396 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX512F3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw -mattr=+avx512vl | FileCheck %s --check-prefix=AVX512BWVL4 5define <16 x i8> @avg_v16i8_mask(<16 x i8> %a, <16 x i8> %b, <16 x i8> %src, i16 %mask) nounwind {6; AVX512F-LABEL: avg_v16i8_mask:7; AVX512F: # %bb.0:8; AVX512F-NEXT: vpavgb %xmm1, %xmm0, %xmm09; AVX512F-NEXT: kmovw %edi, %k110; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 {%k1} {z} = -111; AVX512F-NEXT: vpmovdb %zmm1, %xmm112; AVX512F-NEXT: vpblendvb %xmm1, %xmm0, %xmm2, %xmm013; AVX512F-NEXT: vzeroupper14; AVX512F-NEXT: retq15;16; AVX512BWVL-LABEL: avg_v16i8_mask:17; AVX512BWVL: # %bb.0:18; AVX512BWVL-NEXT: kmovd %edi, %k119; AVX512BWVL-NEXT: vpavgb %xmm1, %xmm0, %xmm2 {%k1}20; AVX512BWVL-NEXT: vmovdqa %xmm2, %xmm021; AVX512BWVL-NEXT: retq22 %za = zext <16 x i8> %a to <16 x i16>23 %zb = zext <16 x i8> %b to <16 x i16>24 %add = add nuw nsw <16 x i16> %za, %zb25 %add1 = add nuw nsw <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>26 %lshr = lshr <16 x i16> %add1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>27 %trunc = trunc <16 x i16> %lshr to <16 x i8>28 %mask1 = bitcast i16 %mask to <16 x i1>29 %res = select <16 x i1> %mask1, <16 x i8> %trunc, <16 x i8> %src30 ret <16 x i8> %res31}32 33define <16 x i8> @avg_v16i8_maskz(<16 x i8> %a, <16 x i8> %b, i16 %mask) nounwind {34; AVX512F-LABEL: avg_v16i8_maskz:35; AVX512F: # %bb.0:36; AVX512F-NEXT: vpavgb %xmm1, %xmm0, %xmm037; AVX512F-NEXT: kmovw %edi, %k138; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 {%k1} {z} = -139; AVX512F-NEXT: vpmovdb %zmm1, %xmm140; AVX512F-NEXT: vpand %xmm0, %xmm1, %xmm041; AVX512F-NEXT: vzeroupper42; AVX512F-NEXT: retq43;44; AVX512BWVL-LABEL: avg_v16i8_maskz:45; AVX512BWVL: # %bb.0:46; AVX512BWVL-NEXT: kmovd %edi, %k147; AVX512BWVL-NEXT: vpavgb %xmm1, %xmm0, %xmm0 {%k1} {z}48; AVX512BWVL-NEXT: retq49 %za = zext <16 x i8> %a to <16 x i16>50 %zb = zext <16 x i8> %b to <16 x i16>51 %add = add nuw nsw <16 x i16> %za, %zb52 %add1 = add nuw nsw <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>53 %lshr = lshr <16 x i16> %add1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>54 %trunc = trunc <16 x i16> %lshr to <16 x i8>55 %mask1 = bitcast i16 %mask to <16 x i1>56 %res = select <16 x i1> %mask1, <16 x i8> %trunc, <16 x i8> zeroinitializer57 ret <16 x i8> %res58}59 60define <32 x i8> @avg_v32i8_mask(<32 x i8> %a, <32 x i8> %b, <32 x i8> %src, i32 %mask) nounwind {61; AVX512F-LABEL: avg_v32i8_mask:62; AVX512F: # %bb.0:63; AVX512F-NEXT: kmovw %edi, %k164; AVX512F-NEXT: shrl $16, %edi65; AVX512F-NEXT: vpavgb %ymm1, %ymm0, %ymm066; AVX512F-NEXT: kmovw %edi, %k267; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 {%k1} {z} = -168; AVX512F-NEXT: vpmovdb %zmm1, %xmm169; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k2} {z} = -170; AVX512F-NEXT: vpmovdb %zmm3, %xmm371; AVX512F-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm172; AVX512F-NEXT: vpblendvb %ymm1, %ymm0, %ymm2, %ymm073; AVX512F-NEXT: retq74;75; AVX512BWVL-LABEL: avg_v32i8_mask:76; AVX512BWVL: # %bb.0:77; AVX512BWVL-NEXT: kmovd %edi, %k178; AVX512BWVL-NEXT: vpavgb %ymm1, %ymm0, %ymm2 {%k1}79; AVX512BWVL-NEXT: vmovdqa %ymm2, %ymm080; AVX512BWVL-NEXT: retq81 %za = zext <32 x i8> %a to <32 x i16>82 %zb = zext <32 x i8> %b to <32 x i16>83 %add = add nuw nsw <32 x i16> %za, %zb84 %add1 = add nuw nsw <32 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>85 %lshr = lshr <32 x i16> %add1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>86 %trunc = trunc <32 x i16> %lshr to <32 x i8>87 %mask1 = bitcast i32 %mask to <32 x i1>88 %res = select <32 x i1> %mask1, <32 x i8> %trunc, <32 x i8> %src89 ret <32 x i8> %res90}91 92define <32 x i8> @avg_v32i8_maskz(<32 x i8> %a, <32 x i8> %b, i32 %mask) nounwind {93; AVX512F-LABEL: avg_v32i8_maskz:94; AVX512F: # %bb.0:95; AVX512F-NEXT: kmovw %edi, %k196; AVX512F-NEXT: shrl $16, %edi97; AVX512F-NEXT: vpavgb %ymm1, %ymm0, %ymm098; AVX512F-NEXT: kmovw %edi, %k299; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 {%k1} {z} = -1100; AVX512F-NEXT: vpmovdb %zmm1, %xmm1101; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 {%k2} {z} = -1102; AVX512F-NEXT: vpmovdb %zmm2, %xmm2103; AVX512F-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1104; AVX512F-NEXT: vpand %ymm0, %ymm1, %ymm0105; AVX512F-NEXT: retq106;107; AVX512BWVL-LABEL: avg_v32i8_maskz:108; AVX512BWVL: # %bb.0:109; AVX512BWVL-NEXT: kmovd %edi, %k1110; AVX512BWVL-NEXT: vpavgb %ymm1, %ymm0, %ymm0 {%k1} {z}111; AVX512BWVL-NEXT: retq112 %za = zext <32 x i8> %a to <32 x i16>113 %zb = zext <32 x i8> %b to <32 x i16>114 %add = add nuw nsw <32 x i16> %za, %zb115 %add1 = add nuw nsw <32 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>116 %lshr = lshr <32 x i16> %add1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>117 %trunc = trunc <32 x i16> %lshr to <32 x i8>118 %mask1 = bitcast i32 %mask to <32 x i1>119 %res = select <32 x i1> %mask1, <32 x i8> %trunc, <32 x i8> zeroinitializer120 ret <32 x i8> %res121}122 123define <64 x i8> @avg_v64i8_mask(<64 x i8> %a, <64 x i8> %b, <64 x i8> %src, i64 %mask) nounwind {124; AVX512F-LABEL: avg_v64i8_mask:125; AVX512F: # %bb.0:126; AVX512F-NEXT: movq %rdi, %rax127; AVX512F-NEXT: movl %edi, %ecx128; AVX512F-NEXT: kmovw %edi, %k1129; AVX512F-NEXT: shrq $32, %rdi130; AVX512F-NEXT: shrq $48, %rax131; AVX512F-NEXT: shrl $16, %ecx132; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm3133; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm4134; AVX512F-NEXT: vpavgb %ymm3, %ymm4, %ymm3135; AVX512F-NEXT: vpavgb %ymm1, %ymm0, %ymm0136; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm1137; AVX512F-NEXT: kmovw %ecx, %k2138; AVX512F-NEXT: kmovw %eax, %k3139; AVX512F-NEXT: kmovw %edi, %k4140; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 {%k4} {z} = -1141; AVX512F-NEXT: vpmovdb %zmm0, %xmm0142; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k3} {z} = -1143; AVX512F-NEXT: vpmovdb %zmm3, %xmm3144; AVX512F-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0145; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k1} {z} = -1146; AVX512F-NEXT: vpmovdb %zmm3, %xmm3147; AVX512F-NEXT: vpternlogd {{.*#+}} zmm4 {%k2} {z} = -1148; AVX512F-NEXT: vpmovdb %zmm4, %xmm4149; AVX512F-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3150; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm0151; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm2 ^ (zmm0 & (zmm1 ^ zmm2))152; AVX512F-NEXT: retq153;154; AVX512BWVL-LABEL: avg_v64i8_mask:155; AVX512BWVL: # %bb.0:156; AVX512BWVL-NEXT: kmovq %rdi, %k1157; AVX512BWVL-NEXT: vpavgb %zmm1, %zmm0, %zmm2 {%k1}158; AVX512BWVL-NEXT: vmovdqa64 %zmm2, %zmm0159; AVX512BWVL-NEXT: retq160 %za = zext <64 x i8> %a to <64 x i16>161 %zb = zext <64 x i8> %b to <64 x i16>162 %add = add nuw nsw <64 x i16> %za, %zb163 %add1 = add nuw nsw <64 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>164 %lshr = lshr <64 x i16> %add1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>165 %trunc = trunc <64 x i16> %lshr to <64 x i8>166 %mask1 = bitcast i64 %mask to <64 x i1>167 %res = select <64 x i1> %mask1, <64 x i8> %trunc, <64 x i8> %src168 ret <64 x i8> %res169}170 171define <64 x i8> @avg_v64i8_maskz(<64 x i8> %a, <64 x i8> %b, i64 %mask) nounwind {172; AVX512F-LABEL: avg_v64i8_maskz:173; AVX512F: # %bb.0:174; AVX512F-NEXT: movq %rdi, %rax175; AVX512F-NEXT: movl %edi, %ecx176; AVX512F-NEXT: kmovw %edi, %k1177; AVX512F-NEXT: shrq $32, %rdi178; AVX512F-NEXT: shrq $48, %rax179; AVX512F-NEXT: shrl $16, %ecx180; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm2181; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm3182; AVX512F-NEXT: vpavgb %ymm2, %ymm3, %ymm2183; AVX512F-NEXT: vpavgb %ymm1, %ymm0, %ymm0184; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0185; AVX512F-NEXT: kmovw %ecx, %k2186; AVX512F-NEXT: kmovw %eax, %k3187; AVX512F-NEXT: kmovw %edi, %k4188; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 {%k4} {z} = -1189; AVX512F-NEXT: vpmovdb %zmm1, %xmm1190; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 {%k3} {z} = -1191; AVX512F-NEXT: vpmovdb %zmm2, %xmm2192; AVX512F-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1193; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 {%k1} {z} = -1194; AVX512F-NEXT: vpmovdb %zmm2, %xmm2195; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k2} {z} = -1196; AVX512F-NEXT: vpmovdb %zmm3, %xmm3197; AVX512F-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2198; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1199; AVX512F-NEXT: vpandq %zmm0, %zmm1, %zmm0200; AVX512F-NEXT: retq201;202; AVX512BWVL-LABEL: avg_v64i8_maskz:203; AVX512BWVL: # %bb.0:204; AVX512BWVL-NEXT: kmovq %rdi, %k1205; AVX512BWVL-NEXT: vpavgb %zmm1, %zmm0, %zmm0 {%k1} {z}206; AVX512BWVL-NEXT: retq207 %za = zext <64 x i8> %a to <64 x i16>208 %zb = zext <64 x i8> %b to <64 x i16>209 %add = add nuw nsw <64 x i16> %za, %zb210 %add1 = add nuw nsw <64 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>211 %lshr = lshr <64 x i16> %add1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>212 %trunc = trunc <64 x i16> %lshr to <64 x i8>213 %mask1 = bitcast i64 %mask to <64 x i1>214 %res = select <64 x i1> %mask1, <64 x i8> %trunc, <64 x i8> zeroinitializer215 ret <64 x i8> %res216}217 218define <8 x i16> @avg_v8i16_mask(<8 x i16> %a, <8 x i16> %b, <8 x i16> %src, i8 %mask) nounwind {219; AVX512F-LABEL: avg_v8i16_mask:220; AVX512F: # %bb.0:221; AVX512F-NEXT: vpavgw %xmm1, %xmm0, %xmm0222; AVX512F-NEXT: kmovw %edi, %k1223; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 {%k1} {z} = -1224; AVX512F-NEXT: vpmovdw %zmm1, %ymm1225; AVX512F-NEXT: vpblendvb %xmm1, %xmm0, %xmm2, %xmm0226; AVX512F-NEXT: vzeroupper227; AVX512F-NEXT: retq228;229; AVX512BWVL-LABEL: avg_v8i16_mask:230; AVX512BWVL: # %bb.0:231; AVX512BWVL-NEXT: kmovd %edi, %k1232; AVX512BWVL-NEXT: vpavgw %xmm1, %xmm0, %xmm2 {%k1}233; AVX512BWVL-NEXT: vmovdqa %xmm2, %xmm0234; AVX512BWVL-NEXT: retq235 %za = zext <8 x i16> %a to <8 x i32>236 %zb = zext <8 x i16> %b to <8 x i32>237 %add = add nuw nsw <8 x i32> %za, %zb238 %add1 = add nuw nsw <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>239 %lshr = lshr <8 x i32> %add1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>240 %trunc = trunc <8 x i32> %lshr to <8 x i16>241 %mask1 = bitcast i8 %mask to <8 x i1>242 %res = select <8 x i1> %mask1, <8 x i16> %trunc, <8 x i16> %src243 ret <8 x i16> %res244}245 246define <8 x i16> @avg_v8i16_maskz(<8 x i16> %a, <8 x i16> %b, i8 %mask) nounwind {247; AVX512F-LABEL: avg_v8i16_maskz:248; AVX512F: # %bb.0:249; AVX512F-NEXT: vpavgw %xmm1, %xmm0, %xmm0250; AVX512F-NEXT: kmovw %edi, %k1251; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 {%k1} {z} = -1252; AVX512F-NEXT: vpmovdw %zmm1, %ymm1253; AVX512F-NEXT: vpand %xmm0, %xmm1, %xmm0254; AVX512F-NEXT: vzeroupper255; AVX512F-NEXT: retq256;257; AVX512BWVL-LABEL: avg_v8i16_maskz:258; AVX512BWVL: # %bb.0:259; AVX512BWVL-NEXT: kmovd %edi, %k1260; AVX512BWVL-NEXT: vpavgw %xmm1, %xmm0, %xmm0 {%k1} {z}261; AVX512BWVL-NEXT: retq262 %za = zext <8 x i16> %a to <8 x i32>263 %zb = zext <8 x i16> %b to <8 x i32>264 %add = add nuw nsw <8 x i32> %za, %zb265 %add1 = add nuw nsw <8 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>266 %lshr = lshr <8 x i32> %add1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>267 %trunc = trunc <8 x i32> %lshr to <8 x i16>268 %mask1 = bitcast i8 %mask to <8 x i1>269 %res = select <8 x i1> %mask1, <8 x i16> %trunc, <8 x i16> zeroinitializer270 ret <8 x i16> %res271}272 273define <16 x i16> @avg_v16i16_mask(<16 x i16> %a, <16 x i16> %b, <16 x i16> %src, i16 %mask) nounwind {274; AVX512F-LABEL: avg_v16i16_mask:275; AVX512F: # %bb.0:276; AVX512F-NEXT: vpavgw %ymm1, %ymm0, %ymm0277; AVX512F-NEXT: kmovw %edi, %k1278; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 {%k1} {z} = -1279; AVX512F-NEXT: vpmovdw %zmm1, %ymm1280; AVX512F-NEXT: vpblendvb %ymm1, %ymm0, %ymm2, %ymm0281; AVX512F-NEXT: retq282;283; AVX512BWVL-LABEL: avg_v16i16_mask:284; AVX512BWVL: # %bb.0:285; AVX512BWVL-NEXT: kmovd %edi, %k1286; AVX512BWVL-NEXT: vpavgw %ymm1, %ymm0, %ymm2 {%k1}287; AVX512BWVL-NEXT: vmovdqa %ymm2, %ymm0288; AVX512BWVL-NEXT: retq289 %za = zext <16 x i16> %a to <16 x i32>290 %zb = zext <16 x i16> %b to <16 x i32>291 %add = add nuw nsw <16 x i32> %za, %zb292 %add1 = add nuw nsw <16 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>293 %lshr = lshr <16 x i32> %add1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>294 %trunc = trunc <16 x i32> %lshr to <16 x i16>295 %mask1 = bitcast i16 %mask to <16 x i1>296 %res = select <16 x i1> %mask1, <16 x i16> %trunc, <16 x i16> %src297 ret <16 x i16> %res298}299 300define <16 x i16> @avg_v16i16_maskz(<16 x i16> %a, <16 x i16> %b, i16 %mask) nounwind {301; AVX512F-LABEL: avg_v16i16_maskz:302; AVX512F: # %bb.0:303; AVX512F-NEXT: vpavgw %ymm1, %ymm0, %ymm0304; AVX512F-NEXT: kmovw %edi, %k1305; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 {%k1} {z} = -1306; AVX512F-NEXT: vpmovdw %zmm1, %ymm1307; AVX512F-NEXT: vpand %ymm0, %ymm1, %ymm0308; AVX512F-NEXT: retq309;310; AVX512BWVL-LABEL: avg_v16i16_maskz:311; AVX512BWVL: # %bb.0:312; AVX512BWVL-NEXT: kmovd %edi, %k1313; AVX512BWVL-NEXT: vpavgw %ymm1, %ymm0, %ymm0 {%k1} {z}314; AVX512BWVL-NEXT: retq315 %za = zext <16 x i16> %a to <16 x i32>316 %zb = zext <16 x i16> %b to <16 x i32>317 %add = add nuw nsw <16 x i32> %za, %zb318 %add1 = add nuw nsw <16 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>319 %lshr = lshr <16 x i32> %add1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>320 %trunc = trunc <16 x i32> %lshr to <16 x i16>321 %mask1 = bitcast i16 %mask to <16 x i1>322 %res = select <16 x i1> %mask1, <16 x i16> %trunc, <16 x i16> zeroinitializer323 ret <16 x i16> %res324}325 326define <32 x i16> @avg_v32i16_mask(<32 x i16> %a, <32 x i16> %b, <32 x i16> %src, i32 %mask) nounwind {327; AVX512F-LABEL: avg_v32i16_mask:328; AVX512F: # %bb.0:329; AVX512F-NEXT: kmovw %edi, %k1330; AVX512F-NEXT: shrl $16, %edi331; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm3332; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm4333; AVX512F-NEXT: vpavgw %ymm3, %ymm4, %ymm3334; AVX512F-NEXT: vpavgw %ymm1, %ymm0, %ymm0335; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm1336; AVX512F-NEXT: kmovw %edi, %k2337; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1338; AVX512F-NEXT: vpmovdw %zmm0, %ymm0339; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k2} {z} = -1340; AVX512F-NEXT: vpmovdw %zmm3, %ymm3341; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0342; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 = zmm2 ^ (zmm0 & (zmm1 ^ zmm2))343; AVX512F-NEXT: retq344;345; AVX512BWVL-LABEL: avg_v32i16_mask:346; AVX512BWVL: # %bb.0:347; AVX512BWVL-NEXT: kmovd %edi, %k1348; AVX512BWVL-NEXT: vpavgw %zmm1, %zmm0, %zmm2 {%k1}349; AVX512BWVL-NEXT: vmovdqa64 %zmm2, %zmm0350; AVX512BWVL-NEXT: retq351 %za = zext <32 x i16> %a to <32 x i32>352 %zb = zext <32 x i16> %b to <32 x i32>353 %add = add nuw nsw <32 x i32> %za, %zb354 %add1 = add nuw nsw <32 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>355 %lshr = lshr <32 x i32> %add1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>356 %trunc = trunc <32 x i32> %lshr to <32 x i16>357 %mask1 = bitcast i32 %mask to <32 x i1>358 %res = select <32 x i1> %mask1, <32 x i16> %trunc, <32 x i16> %src359 ret <32 x i16> %res360}361 362define <32 x i16> @avg_v32i16_maskz(<32 x i16> %a, <32 x i16> %b, i32 %mask) nounwind {363; AVX512F-LABEL: avg_v32i16_maskz:364; AVX512F: # %bb.0:365; AVX512F-NEXT: kmovw %edi, %k1366; AVX512F-NEXT: shrl $16, %edi367; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm2368; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm3369; AVX512F-NEXT: vpavgw %ymm2, %ymm3, %ymm2370; AVX512F-NEXT: vpavgw %ymm1, %ymm0, %ymm0371; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0372; AVX512F-NEXT: kmovw %edi, %k2373; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 {%k1} {z} = -1374; AVX512F-NEXT: vpmovdw %zmm1, %ymm1375; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 {%k2} {z} = -1376; AVX512F-NEXT: vpmovdw %zmm2, %ymm2377; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1378; AVX512F-NEXT: vpandq %zmm0, %zmm1, %zmm0379; AVX512F-NEXT: retq380;381; AVX512BWVL-LABEL: avg_v32i16_maskz:382; AVX512BWVL: # %bb.0:383; AVX512BWVL-NEXT: kmovd %edi, %k1384; AVX512BWVL-NEXT: vpavgw %zmm1, %zmm0, %zmm0 {%k1} {z}385; AVX512BWVL-NEXT: retq386 %za = zext <32 x i16> %a to <32 x i32>387 %zb = zext <32 x i16> %b to <32 x i32>388 %add = add nuw nsw <32 x i32> %za, %zb389 %add1 = add nuw nsw <32 x i32> %add, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>390 %lshr = lshr <32 x i32> %add1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>391 %trunc = trunc <32 x i32> %lshr to <32 x i16>392 %mask1 = bitcast i32 %mask to <32 x i1>393 %res = select <32 x i1> %mask1, <32 x i16> %trunc, <32 x i16> zeroinitializer394 ret <32 x i16> %res395}396