565 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avxvnni | FileCheck %s --check-prefixes=CHECK,AVXVNNI,AVXVNNI-AVX3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avxvnni,+avx512vl | FileCheck %s --check-prefixes=CHECK,AVXVNNI,AVXVNNI-AVX5124; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512VNNI5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+avx512vl | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512VLVNNI6 7define i32 @no_dpbusd(ptr%a, ptr%b, i32 %c, i32 %n) {8; CHECK-LABEL: no_dpbusd:9; CHECK: # %bb.0: # %entry10; CHECK-NEXT: vpmovzxbw {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero11; CHECK-NEXT: vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero12; CHECK-NEXT: vpmaddwd %ymm0, %ymm1, %ymm013; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm114; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm015; CHECK-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]16; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm017; CHECK-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]18; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm019; CHECK-NEXT: vmovd %xmm0, %eax20; CHECK-NEXT: addl %edx, %eax21; CHECK-NEXT: vzeroupper22; CHECK-NEXT: retq23entry:24 %0 = load <16 x i8>, ptr %a, align 1625 %1 = zext <16 x i8> %0 to <16 x i32>26 %2 = load <16 x i8>, ptr %b, align 1627 %3 = zext <16 x i8> %2 to <16 x i32>28 %4 = mul nsw <16 x i32> %3, %129 %5 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %4)30 %op.extra = add nsw i32 %5, %c31 ret i32 %op.extra32}33 34define i32 @vpdpbusd_mutate(ptr%a, ptr%b, i32 %c, i32 %n) {35; AVXVNNI-LABEL: vpdpbusd_mutate:36; AVXVNNI: # %bb.0: # %entry37; AVXVNNI-NEXT: vmovdqa (%rsi), %xmm038; AVXVNNI-NEXT: vpxor %xmm1, %xmm1, %xmm139; AVXVNNI-NEXT: {vex} vpdpbusd (%rdi), %xmm0, %xmm140; AVXVNNI-NEXT: vpshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]41; AVXVNNI-NEXT: vpaddd %xmm0, %xmm1, %xmm042; AVXVNNI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]43; AVXVNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm044; AVXVNNI-NEXT: vmovd %xmm0, %eax45; AVXVNNI-NEXT: addl %edx, %eax46; AVXVNNI-NEXT: retq47;48; AVX512VNNI-LABEL: vpdpbusd_mutate:49; AVX512VNNI: # %bb.0: # %entry50; AVX512VNNI-NEXT: vmovdqa (%rdi), %xmm051; AVX512VNNI-NEXT: vmovdqa (%rsi), %xmm152; AVX512VNNI-NEXT: vpxor %xmm2, %xmm2, %xmm253; AVX512VNNI-NEXT: vpdpbusd %zmm0, %zmm1, %zmm254; AVX512VNNI-NEXT: vpshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]55; AVX512VNNI-NEXT: vpaddd %xmm0, %xmm2, %xmm056; AVX512VNNI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]57; AVX512VNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm058; AVX512VNNI-NEXT: vmovd %xmm0, %eax59; AVX512VNNI-NEXT: addl %edx, %eax60; AVX512VNNI-NEXT: vzeroupper61; AVX512VNNI-NEXT: retq62;63; AVX512VLVNNI-LABEL: vpdpbusd_mutate:64; AVX512VLVNNI: # %bb.0: # %entry65; AVX512VLVNNI-NEXT: vmovdqa (%rsi), %xmm066; AVX512VLVNNI-NEXT: vpxor %xmm1, %xmm1, %xmm167; AVX512VLVNNI-NEXT: vpdpbusd (%rdi), %xmm0, %xmm168; AVX512VLVNNI-NEXT: vpshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]69; AVX512VLVNNI-NEXT: vpaddd %xmm0, %xmm1, %xmm070; AVX512VLVNNI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]71; AVX512VLVNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm072; AVX512VLVNNI-NEXT: vmovd %xmm0, %eax73; AVX512VLVNNI-NEXT: addl %edx, %eax74; AVX512VLVNNI-NEXT: retq75entry:76 %0 = load <16 x i8>, ptr %a, align 1677 %1 = sext <16 x i8> %0 to <16 x i32>78 %2 = load <16 x i8>, ptr %b, align 1679 %3 = zext <16 x i8> %2 to <16 x i32>80 %4 = mul nsw <16 x i32> %3, %181 %5 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %4)82 %op.extra = add nsw i32 %5, %c83 ret i32 %op.extra84}85 86define i32 @mul_zext(ptr%a, ptr%b, i32 %c, i32 %n) {87; AVXVNNI-AVX-LABEL: mul_zext:88; AVXVNNI-AVX: # %bb.0: # %entry89; AVXVNNI-AVX-NEXT: vpmovzxbw {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero90; AVXVNNI-AVX-NEXT: vpmovsxbw (%rsi), %ymm191; AVXVNNI-AVX-NEXT: vpmullw %ymm0, %ymm1, %ymm092; AVXVNNI-AVX-NEXT: vextracti128 $1, %ymm0, %xmm193; AVXVNNI-AVX-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero94; AVXVNNI-AVX-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero95; AVXVNNI-AVX-NEXT: vpaddd %ymm1, %ymm0, %ymm096; AVXVNNI-AVX-NEXT: vextracti128 $1, %ymm0, %xmm197; AVXVNNI-AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm098; AVXVNNI-AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]99; AVXVNNI-AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0100; AVXVNNI-AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]101; AVXVNNI-AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0102; AVXVNNI-AVX-NEXT: vmovd %xmm0, %eax103; AVXVNNI-AVX-NEXT: addl %edx, %eax104; AVXVNNI-AVX-NEXT: vzeroupper105; AVXVNNI-AVX-NEXT: retq106;107; AVXVNNI-AVX512-LABEL: mul_zext:108; AVXVNNI-AVX512: # %bb.0: # %entry109; AVXVNNI-AVX512-NEXT: vpmovzxbw {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero110; AVXVNNI-AVX512-NEXT: vpmovsxbw (%rsi), %ymm1111; AVXVNNI-AVX512-NEXT: vpmullw %ymm0, %ymm1, %ymm0112; AVXVNNI-AVX512-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero113; AVXVNNI-AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1114; AVXVNNI-AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0115; AVXVNNI-AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1116; AVXVNNI-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0117; AVXVNNI-AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]118; AVXVNNI-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0119; AVXVNNI-AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]120; AVXVNNI-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0121; AVXVNNI-AVX512-NEXT: vmovd %xmm0, %eax122; AVXVNNI-AVX512-NEXT: addl %edx, %eax123; AVXVNNI-AVX512-NEXT: vzeroupper124; AVXVNNI-AVX512-NEXT: retq125;126; AVX512-LABEL: mul_zext:127; AVX512: # %bb.0: # %entry128; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero129; AVX512-NEXT: vpmovsxbw (%rsi), %ymm1130; AVX512-NEXT: vpmullw %ymm0, %ymm1, %ymm0131; AVX512-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero132; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1133; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0134; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1135; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0136; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]137; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0138; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]139; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0140; AVX512-NEXT: vmovd %xmm0, %eax141; AVX512-NEXT: addl %edx, %eax142; AVX512-NEXT: vzeroupper143; AVX512-NEXT: retq144entry:145 %0 = load <16 x i8>, ptr %a, align 16146 %1 = zext <16 x i8> %0 to <16 x i16>147 %2 = load <16 x i8>, ptr %b, align 16148 %3 = sext <16 x i8> %2 to <16 x i16>149 %4 = mul nsw <16 x i16> %3, %1150 ; We can't combine to vpdpbusd for zext, because each of the 4 multiplies151 ; done by vpdpbusd compute a signed 16-bit product that will be sign extended152 ; before adding into the accumulator.153 %5 = zext <16 x i16> %4 to <16 x i32>154 %6 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %5)155 %op.extra = add nsw i32 %6, %c156 ret i32 %op.extra157}158 159define i32 @mul_sext(ptr%a, ptr%b, i32 %c, i32 %n) {160; AVXVNNI-AVX-LABEL: mul_sext:161; AVXVNNI-AVX: # %bb.0: # %entry162; AVXVNNI-AVX-NEXT: vpmovzxbw {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero163; AVXVNNI-AVX-NEXT: vpmovsxbw (%rsi), %ymm1164; AVXVNNI-AVX-NEXT: vpmullw %ymm0, %ymm1, %ymm0165; AVXVNNI-AVX-NEXT: vextracti128 $1, %ymm0, %xmm1166; AVXVNNI-AVX-NEXT: vpmovsxwd %xmm1, %ymm1167; AVXVNNI-AVX-NEXT: vpmovsxwd %xmm0, %ymm0168; AVXVNNI-AVX-NEXT: vpaddd %ymm1, %ymm0, %ymm0169; AVXVNNI-AVX-NEXT: vextracti128 $1, %ymm0, %xmm1170; AVXVNNI-AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0171; AVXVNNI-AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]172; AVXVNNI-AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0173; AVXVNNI-AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]174; AVXVNNI-AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0175; AVXVNNI-AVX-NEXT: vmovd %xmm0, %eax176; AVXVNNI-AVX-NEXT: addl %edx, %eax177; AVXVNNI-AVX-NEXT: vzeroupper178; AVXVNNI-AVX-NEXT: retq179;180; AVXVNNI-AVX512-LABEL: mul_sext:181; AVXVNNI-AVX512: # %bb.0: # %entry182; AVXVNNI-AVX512-NEXT: vpmovzxbw {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero183; AVXVNNI-AVX512-NEXT: vpmovsxbw (%rsi), %ymm1184; AVXVNNI-AVX512-NEXT: vpmullw %ymm0, %ymm1, %ymm0185; AVXVNNI-AVX512-NEXT: vpmovsxwd %ymm0, %zmm0186; AVXVNNI-AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1187; AVXVNNI-AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0188; AVXVNNI-AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1189; AVXVNNI-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0190; AVXVNNI-AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]191; AVXVNNI-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0192; AVXVNNI-AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]193; AVXVNNI-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0194; AVXVNNI-AVX512-NEXT: vmovd %xmm0, %eax195; AVXVNNI-AVX512-NEXT: addl %edx, %eax196; AVXVNNI-AVX512-NEXT: vzeroupper197; AVXVNNI-AVX512-NEXT: retq198;199; AVX512-LABEL: mul_sext:200; AVX512: # %bb.0: # %entry201; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero202; AVX512-NEXT: vpmovsxbw (%rsi), %ymm1203; AVX512-NEXT: vpmullw %ymm0, %ymm1, %ymm0204; AVX512-NEXT: vpmovsxwd %ymm0, %zmm0205; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1206; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0207; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1208; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0209; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]210; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0211; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]212; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0213; AVX512-NEXT: vmovd %xmm0, %eax214; AVX512-NEXT: addl %edx, %eax215; AVX512-NEXT: vzeroupper216; AVX512-NEXT: retq217entry:218 %0 = load <16 x i8>, ptr %a, align 16219 %1 = zext <16 x i8> %0 to <16 x i16>220 %2 = load <16 x i8>, ptr %b, align 16221 %3 = sext <16 x i8> %2 to <16 x i16>222 %4 = mul nsw <16 x i16> %3, %1223 ; TODO:224 ; We also need to verify that the multiply has at least 2x the number of bits225 ; of the input. We shouldn't match226 ; (sign_extend (mul (vXi9 (zext (vXi8 X))), (vXi9 (zext (vXi8 Y)))).227 %5 = sext <16 x i16> %4 to <16 x i32>228 %6 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %5)229 %op.extra = add nsw i32 %6, %c230 ret i32 %op.extra231}232 233define i32 @vpdpbusd_512(ptr%a, ptr%b, i32 %c, i32 %n) {234; AVXVNNI-LABEL: vpdpbusd_512:235; AVXVNNI: # %bb.0: # %entry236; AVXVNNI-NEXT: vmovdqa (%rdi), %xmm0237; AVXVNNI-NEXT: vpxor %xmm1, %xmm1, %xmm1238; AVXVNNI-NEXT: {vex} vpdpbusd (%rsi), %xmm0, %xmm1239; AVXVNNI-NEXT: vpshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]240; AVXVNNI-NEXT: vpaddd %xmm0, %xmm1, %xmm0241; AVXVNNI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]242; AVXVNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0243; AVXVNNI-NEXT: vmovd %xmm0, %eax244; AVXVNNI-NEXT: addl %edx, %eax245; AVXVNNI-NEXT: retq246;247; AVX512VNNI-LABEL: vpdpbusd_512:248; AVX512VNNI: # %bb.0: # %entry249; AVX512VNNI-NEXT: vmovdqa (%rdi), %xmm0250; AVX512VNNI-NEXT: vmovdqa (%rsi), %xmm1251; AVX512VNNI-NEXT: vpxor %xmm2, %xmm2, %xmm2252; AVX512VNNI-NEXT: vpdpbusd %zmm1, %zmm0, %zmm2253; AVX512VNNI-NEXT: vpshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]254; AVX512VNNI-NEXT: vpaddd %xmm0, %xmm2, %xmm0255; AVX512VNNI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]256; AVX512VNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0257; AVX512VNNI-NEXT: vmovd %xmm0, %eax258; AVX512VNNI-NEXT: addl %edx, %eax259; AVX512VNNI-NEXT: vzeroupper260; AVX512VNNI-NEXT: retq261;262; AVX512VLVNNI-LABEL: vpdpbusd_512:263; AVX512VLVNNI: # %bb.0: # %entry264; AVX512VLVNNI-NEXT: vmovdqa (%rdi), %xmm0265; AVX512VLVNNI-NEXT: vpxor %xmm1, %xmm1, %xmm1266; AVX512VLVNNI-NEXT: vpdpbusd (%rsi), %xmm0, %xmm1267; AVX512VLVNNI-NEXT: vpshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]268; AVX512VLVNNI-NEXT: vpaddd %xmm0, %xmm1, %xmm0269; AVX512VLVNNI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]270; AVX512VLVNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0271; AVX512VLVNNI-NEXT: vmovd %xmm0, %eax272; AVX512VLVNNI-NEXT: addl %edx, %eax273; AVX512VLVNNI-NEXT: retq274entry:275 %0 = load <16 x i8>, ptr %a, align 16276 %1 = zext <16 x i8> %0 to <16 x i32>277 %2 = load <16 x i8>, ptr %b, align 16278 %3 = sext <16 x i8> %2 to <16 x i32>279 %4 = mul nsw <16 x i32> %3, %1280 %5 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %4)281 %op.extra = add nsw i32 %5, %c282 ret i32 %op.extra283}284 285declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)286 287define i32 @vpdpbusd_256(ptr%a, ptr%b, i32 %c, i32 %n) {288; AVXVNNI-LABEL: vpdpbusd_256:289; AVXVNNI: # %bb.0: # %entry290; AVXVNNI-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero291; AVXVNNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero292; AVXVNNI-NEXT: vpxor %xmm2, %xmm2, %xmm2293; AVXVNNI-NEXT: {vex} vpdpbusd %xmm0, %xmm1, %xmm2294; AVXVNNI-NEXT: vpshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]295; AVXVNNI-NEXT: vpaddd %xmm0, %xmm2, %xmm0296; AVXVNNI-NEXT: vmovd %xmm0, %eax297; AVXVNNI-NEXT: addl %edx, %eax298; AVXVNNI-NEXT: retq299;300; AVX512VNNI-LABEL: vpdpbusd_256:301; AVX512VNNI: # %bb.0: # %entry302; AVX512VNNI-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero303; AVX512VNNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero304; AVX512VNNI-NEXT: vpxor %xmm2, %xmm2, %xmm2305; AVX512VNNI-NEXT: vpdpbusd %zmm0, %zmm1, %zmm2306; AVX512VNNI-NEXT: vpshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]307; AVX512VNNI-NEXT: vpaddd %xmm0, %xmm2, %xmm0308; AVX512VNNI-NEXT: vmovd %xmm0, %eax309; AVX512VNNI-NEXT: addl %edx, %eax310; AVX512VNNI-NEXT: vzeroupper311; AVX512VNNI-NEXT: retq312;313; AVX512VLVNNI-LABEL: vpdpbusd_256:314; AVX512VLVNNI: # %bb.0: # %entry315; AVX512VLVNNI-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero316; AVX512VLVNNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero317; AVX512VLVNNI-NEXT: vpxor %xmm2, %xmm2, %xmm2318; AVX512VLVNNI-NEXT: vpdpbusd %xmm0, %xmm1, %xmm2319; AVX512VLVNNI-NEXT: vpshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]320; AVX512VLVNNI-NEXT: vpaddd %xmm0, %xmm2, %xmm0321; AVX512VLVNNI-NEXT: vmovd %xmm0, %eax322; AVX512VLVNNI-NEXT: addl %edx, %eax323; AVX512VLVNNI-NEXT: retq324entry:325 %0 = load <8 x i8>, ptr %a, align 8326 %1 = zext <8 x i8> %0 to <8 x i32>327 %2 = load <8 x i8>, ptr %b, align 8328 %3 = sext <8 x i8> %2 to <8 x i32>329 %4 = mul nsw <8 x i32> %3, %1330 %5 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %4)331 %op.extra = add nsw i32 %5, %c332 ret i32 %op.extra333}334 335declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)336 337define i32 @vpdpbusd_128(ptr%a, ptr%b, i32 %c, i32 %n) {338; AVXVNNI-AVX-LABEL: vpdpbusd_128:339; AVXVNNI-AVX: # %bb.0: # %entry340; AVXVNNI-AVX-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero341; AVXVNNI-AVX-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero342; AVXVNNI-AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2343; AVXVNNI-AVX-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3]344; AVXVNNI-AVX-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]345; AVXVNNI-AVX-NEXT: {vex} vpdpbusd %xmm1, %xmm0, %xmm2346; AVXVNNI-AVX-NEXT: vmovd %xmm2, %eax347; AVXVNNI-AVX-NEXT: addl %edx, %eax348; AVXVNNI-AVX-NEXT: retq349;350; AVXVNNI-AVX512-LABEL: vpdpbusd_128:351; AVXVNNI-AVX512: # %bb.0: # %entry352; AVXVNNI-AVX512-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero353; AVXVNNI-AVX512-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero354; AVXVNNI-AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2355; AVXVNNI-AVX512-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3]356; AVXVNNI-AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]357; AVXVNNI-AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2358; AVXVNNI-AVX512-NEXT: {vex} vpdpbusd %xmm1, %xmm0, %xmm2359; AVXVNNI-AVX512-NEXT: vmovd %xmm2, %eax360; AVXVNNI-AVX512-NEXT: addl %edx, %eax361; AVXVNNI-AVX512-NEXT: retq362;363; AVX512VNNI-LABEL: vpdpbusd_128:364; AVX512VNNI: # %bb.0: # %entry365; AVX512VNNI-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero366; AVX512VNNI-NEXT: vpxor %xmm1, %xmm1, %xmm1367; AVX512VNNI-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]368; AVX512VNNI-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero369; AVX512VNNI-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3]370; AVX512VNNI-NEXT: vpxor %xmm2, %xmm2, %xmm2371; AVX512VNNI-NEXT: vpdpbusd %zmm0, %zmm1, %zmm2372; AVX512VNNI-NEXT: vmovd %xmm2, %eax373; AVX512VNNI-NEXT: addl %edx, %eax374; AVX512VNNI-NEXT: vzeroupper375; AVX512VNNI-NEXT: retq376;377; AVX512VLVNNI-LABEL: vpdpbusd_128:378; AVX512VLVNNI: # %bb.0: # %entry379; AVX512VLVNNI-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero380; AVX512VLVNNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero381; AVX512VLVNNI-NEXT: vpxor %xmm2, %xmm2, %xmm2382; AVX512VLVNNI-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3]383; AVX512VLVNNI-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]384; AVX512VLVNNI-NEXT: vpxor %xmm2, %xmm2, %xmm2385; AVX512VLVNNI-NEXT: vpdpbusd %xmm1, %xmm0, %xmm2386; AVX512VLVNNI-NEXT: vmovd %xmm2, %eax387; AVX512VLVNNI-NEXT: addl %edx, %eax388; AVX512VLVNNI-NEXT: retq389entry:390 %0 = load <4 x i8>, ptr %a, align 8391 %1 = zext <4 x i8> %0 to <4 x i32>392 %2 = load <4 x i8>, ptr %b, align 8393 %3 = sext <4 x i8> %2 to <4 x i32>394 %4 = mul nsw <4 x i32> %3, %1395 %5 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %4)396 %op.extra = add nsw i32 %5, %c397 ret i32 %op.extra398}399 400declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)401 402define i32 @vpdpbusd_2xi32(ptr%a, ptr%b, i32 %c, i32 %n) {403; AVXVNNI-LABEL: vpdpbusd_2xi32:404; AVXVNNI: # %bb.0: # %entry405; AVXVNNI-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero406; AVXVNNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero407; AVXVNNI-NEXT: vpxor %xmm2, %xmm2, %xmm2408; AVXVNNI-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7]409; AVXVNNI-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7]410; AVXVNNI-NEXT: {vex} vpdpbusd %xmm1, %xmm0, %xmm2411; AVXVNNI-NEXT: vmovd %xmm2, %eax412; AVXVNNI-NEXT: addl %edx, %eax413; AVXVNNI-NEXT: retq414;415; AVX512VNNI-LABEL: vpdpbusd_2xi32:416; AVX512VNNI: # %bb.0: # %entry417; AVX512VNNI-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero418; AVX512VNNI-NEXT: vmovd {{.*#+}} xmm1 = [65535,0,0,0]419; AVX512VNNI-NEXT: vpandq %zmm1, %zmm0, %zmm0420; AVX512VNNI-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero421; AVX512VNNI-NEXT: vpandq %zmm1, %zmm2, %zmm1422; AVX512VNNI-NEXT: vpxor %xmm2, %xmm2, %xmm2423; AVX512VNNI-NEXT: vpdpbusd %zmm0, %zmm1, %zmm2424; AVX512VNNI-NEXT: vmovd %xmm2, %eax425; AVX512VNNI-NEXT: addl %edx, %eax426; AVX512VNNI-NEXT: vzeroupper427; AVX512VNNI-NEXT: retq428;429; AVX512VLVNNI-LABEL: vpdpbusd_2xi32:430; AVX512VLVNNI: # %bb.0: # %entry431; AVX512VLVNNI-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero432; AVX512VLVNNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero433; AVX512VLVNNI-NEXT: vpxor %xmm2, %xmm2, %xmm2434; AVX512VLVNNI-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7]435; AVX512VLVNNI-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7]436; AVX512VLVNNI-NEXT: vpdpbusd %xmm1, %xmm0, %xmm2437; AVX512VLVNNI-NEXT: vmovd %xmm2, %eax438; AVX512VLVNNI-NEXT: addl %edx, %eax439; AVX512VLVNNI-NEXT: retq440entry:441 %0 = load <2 x i8>, ptr %a, align 8442 %1 = zext <2 x i8> %0 to <2 x i32>443 %2 = load <2 x i8>, ptr %b, align 8444 %3 = sext <2 x i8> %2 to <2 x i32>445 %4 = mul nsw <2 x i32> %3, %1446 %5 = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %4)447 %op.extra = add nsw i32 %5, %c448 ret i32 %op.extra449}450 451declare i32 @llvm.vector.reduce.add.v2i32(<2 x i32>)452 453define i32 @vpdpbusd_32xi32(ptr%a, ptr%b, i32 %c, i32 %n) {454; AVXVNNI-LABEL: vpdpbusd_32xi32:455; AVXVNNI: # %bb.0: # %entry456; AVXVNNI-NEXT: vmovdqu (%rdi), %ymm0457; AVXVNNI-NEXT: vpxor %xmm1, %xmm1, %xmm1458; AVXVNNI-NEXT: {vex} vpdpbusd (%rsi), %ymm0, %ymm1459; AVXVNNI-NEXT: vextracti128 $1, %ymm1, %xmm0460; AVXVNNI-NEXT: vpaddd %xmm0, %xmm1, %xmm0461; AVXVNNI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]462; AVXVNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0463; AVXVNNI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]464; AVXVNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0465; AVXVNNI-NEXT: vmovd %xmm0, %eax466; AVXVNNI-NEXT: addl %edx, %eax467; AVXVNNI-NEXT: vzeroupper468; AVXVNNI-NEXT: retq469;470; AVX512VNNI-LABEL: vpdpbusd_32xi32:471; AVX512VNNI: # %bb.0: # %entry472; AVX512VNNI-NEXT: vmovdqu (%rdi), %ymm0473; AVX512VNNI-NEXT: vmovdqu (%rsi), %ymm1474; AVX512VNNI-NEXT: vpxor %xmm2, %xmm2, %xmm2475; AVX512VNNI-NEXT: vpdpbusd %zmm1, %zmm0, %zmm2476; AVX512VNNI-NEXT: vextracti128 $1, %ymm2, %xmm0477; AVX512VNNI-NEXT: vpaddd %xmm0, %xmm2, %xmm0478; AVX512VNNI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]479; AVX512VNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0480; AVX512VNNI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]481; AVX512VNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0482; AVX512VNNI-NEXT: vmovd %xmm0, %eax483; AVX512VNNI-NEXT: addl %edx, %eax484; AVX512VNNI-NEXT: vzeroupper485; AVX512VNNI-NEXT: retq486;487; AVX512VLVNNI-LABEL: vpdpbusd_32xi32:488; AVX512VLVNNI: # %bb.0: # %entry489; AVX512VLVNNI-NEXT: vmovdqu (%rdi), %ymm0490; AVX512VLVNNI-NEXT: vpxor %xmm1, %xmm1, %xmm1491; AVX512VLVNNI-NEXT: vpdpbusd (%rsi), %ymm0, %ymm1492; AVX512VLVNNI-NEXT: vextracti128 $1, %ymm1, %xmm0493; AVX512VLVNNI-NEXT: vpaddd %xmm0, %xmm1, %xmm0494; AVX512VLVNNI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]495; AVX512VLVNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0496; AVX512VLVNNI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]497; AVX512VLVNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0498; AVX512VLVNNI-NEXT: vmovd %xmm0, %eax499; AVX512VLVNNI-NEXT: addl %edx, %eax500; AVX512VLVNNI-NEXT: vzeroupper501; AVX512VLVNNI-NEXT: retq502entry:503 %0 = load <32 x i8>, ptr %a, align 16504 %1 = zext <32 x i8> %0 to <32 x i32>505 %2 = load <32 x i8>, ptr %b, align 16506 %3 = sext <32 x i8> %2 to <32 x i32>507 %4 = mul nsw <32 x i32> %3, %1508 %5 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %4)509 %op.extra = add nsw i32 %5, %c510 ret i32 %op.extra511}512 513declare i32 @llvm.vector.reduce.add.v32i32(<32 x i32>)514 515define i32 @vpdpbusd_64xi32(ptr%a, ptr%b, i32 %c, i32 %n) {516; AVXVNNI-LABEL: vpdpbusd_64xi32:517; AVXVNNI: # %bb.0: # %entry518; AVXVNNI-NEXT: vmovdqu (%rdi), %ymm0519; AVXVNNI-NEXT: vmovdqu 32(%rdi), %ymm1520; AVXVNNI-NEXT: vpxor %xmm2, %xmm2, %xmm2521; AVXVNNI-NEXT: vpxor %xmm3, %xmm3, %xmm3522; AVXVNNI-NEXT: {vex} vpdpbusd 32(%rsi), %ymm1, %ymm3523; AVXVNNI-NEXT: {vex} vpdpbusd (%rsi), %ymm0, %ymm2524; AVXVNNI-NEXT: vpaddd %ymm3, %ymm2, %ymm0525; AVXVNNI-NEXT: vextracti128 $1, %ymm0, %xmm1526; AVXVNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0527; AVXVNNI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]528; AVXVNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0529; AVXVNNI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]530; AVXVNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0531; AVXVNNI-NEXT: vmovd %xmm0, %eax532; AVXVNNI-NEXT: addl %edx, %eax533; AVXVNNI-NEXT: vzeroupper534; AVXVNNI-NEXT: retq535;536; AVX512-LABEL: vpdpbusd_64xi32:537; AVX512: # %bb.0: # %entry538; AVX512-NEXT: vmovdqu64 (%rdi), %zmm0539; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1540; AVX512-NEXT: vpdpbusd (%rsi), %zmm0, %zmm1541; AVX512-NEXT: vextracti64x4 $1, %zmm1, %ymm0542; AVX512-NEXT: vpaddd %zmm0, %zmm1, %zmm0543; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1544; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0545; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]546; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0547; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]548; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0549; AVX512-NEXT: vmovd %xmm0, %eax550; AVX512-NEXT: addl %edx, %eax551; AVX512-NEXT: vzeroupper552; AVX512-NEXT: retq553entry:554 %0 = load <64 x i8>, ptr %a, align 16555 %1 = zext <64 x i8> %0 to <64 x i32>556 %2 = load <64 x i8>, ptr %b, align 16557 %3 = sext <64 x i8> %2 to <64 x i32>558 %4 = mul nsw <64 x i32> %3, %1559 %5 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %4)560 %op.extra = add nsw i32 %5, %c561 ret i32 %op.extra562}563 564declare i32 @llvm.vector.reduce.add.v64i32(<64 x i32>)565