367 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avxvnni | FileCheck %s --check-prefixes=CHECK,AVXVNNI,AVXVNNI-AVX3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avxvnni,+avx512vl | FileCheck %s --check-prefixes=CHECK,AVXVNNI,AVXVNNI-AVX5124; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512VNNI5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+avx512vl | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512VLVNNI6 7define i32 @mul_4xi8_zc_exceed(<4 x i8> %a, i32 %c) {8; CHECK-LABEL: mul_4xi8_zc_exceed:9; CHECK: # %bb.0: # %entry10; CHECK-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero11; CHECK-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [0,0,1,0,2,0,128,0]12; CHECK-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]13; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm014; CHECK-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]15; CHECK-NEXT: vpaddd %xmm0, %xmm1, %xmm016; CHECK-NEXT: vmovd %xmm0, %eax17; CHECK-NEXT: addl %edi, %eax18; CHECK-NEXT: retq19entry:20 %0 = zext <4 x i8> %a to <4 x i32>21 %1 = mul nsw <4 x i32> %0, <i32 0, i32 1, i32 2, i32 128>22 %2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %1)23 %op.extra = add nsw i32 %2, %c24 ret i32 %op.extra25}26 27define i32 @mul_4xi8_zc(<4 x i8> %a, i32 %c) {28; AVXVNNI-AVX-LABEL: mul_4xi8_zc:29; AVXVNNI-AVX: # %bb.0: # %entry30; AVXVNNI-AVX-NEXT: vpxor %xmm1, %xmm1, %xmm131; AVXVNNI-AVX-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]32; AVXVNNI-AVX-NEXT: {vex} vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm133; AVXVNNI-AVX-NEXT: vmovd %xmm1, %eax34; AVXVNNI-AVX-NEXT: addl %edi, %eax35; AVXVNNI-AVX-NEXT: retq36;37; AVXVNNI-AVX512-LABEL: mul_4xi8_zc:38; AVXVNNI-AVX512: # %bb.0: # %entry39; AVXVNNI-AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm140; AVXVNNI-AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]41; AVXVNNI-AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm142; AVXVNNI-AVX512-NEXT: {vex} vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm143; AVXVNNI-AVX512-NEXT: vmovd %xmm1, %eax44; AVXVNNI-AVX512-NEXT: addl %edi, %eax45; AVXVNNI-AVX512-NEXT: retq46;47; AVX512VNNI-LABEL: mul_4xi8_zc:48; AVX512VNNI: # %bb.0: # %entry49; AVX512VNNI-NEXT: vpxor %xmm1, %xmm1, %xmm150; AVX512VNNI-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]51; AVX512VNNI-NEXT: vpxor %xmm1, %xmm1, %xmm152; AVX512VNNI-NEXT: vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm153; AVX512VNNI-NEXT: vmovd %xmm1, %eax54; AVX512VNNI-NEXT: addl %edi, %eax55; AVX512VNNI-NEXT: vzeroupper56; AVX512VNNI-NEXT: retq57;58; AVX512VLVNNI-LABEL: mul_4xi8_zc:59; AVX512VLVNNI: # %bb.0: # %entry60; AVX512VLVNNI-NEXT: vpxor %xmm1, %xmm1, %xmm161; AVX512VLVNNI-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]62; AVX512VLVNNI-NEXT: vpxor %xmm1, %xmm1, %xmm163; AVX512VLVNNI-NEXT: vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm164; AVX512VLVNNI-NEXT: vmovd %xmm1, %eax65; AVX512VLVNNI-NEXT: addl %edi, %eax66; AVX512VLVNNI-NEXT: retq67entry:68 %0 = zext <4 x i8> %a to <4 x i32>69 %1 = mul nsw <4 x i32> %0, <i32 16, i32 1, i32 2, i32 127>70 %2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %1)71 %op.extra = add nsw i32 %2, %c72 ret i32 %op.extra73}74 75define i32 @mul_4xi4_cz(<4 x i4> %a, i32 %c) {76; AVXVNNI-AVX-LABEL: mul_4xi4_cz:77; AVXVNNI-AVX: # %bb.0: # %entry78; AVXVNNI-AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]79; AVXVNNI-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm080; AVXVNNI-AVX-NEXT: vpxor %xmm1, %xmm1, %xmm181; AVXVNNI-AVX-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]82; AVXVNNI-AVX-NEXT: {vex} vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm183; AVXVNNI-AVX-NEXT: vmovd %xmm1, %eax84; AVXVNNI-AVX-NEXT: addl %edi, %eax85; AVXVNNI-AVX-NEXT: retq86;87; AVXVNNI-AVX512-LABEL: mul_4xi4_cz:88; AVXVNNI-AVX512: # %bb.0: # %entry89; AVXVNNI-AVX512-NEXT: vpmovdb %xmm0, %xmm090; AVXVNNI-AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm091; AVXVNNI-AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm192; AVXVNNI-AVX512-NEXT: {vex} vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm193; AVXVNNI-AVX512-NEXT: vmovd %xmm1, %eax94; AVXVNNI-AVX512-NEXT: addl %edi, %eax95; AVXVNNI-AVX512-NEXT: retq96;97; AVX512VNNI-LABEL: mul_4xi4_cz:98; AVX512VNNI: # %bb.0: # %entry99; AVX512VNNI-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]100; AVX512VNNI-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0101; AVX512VNNI-NEXT: vpxor %xmm1, %xmm1, %xmm1102; AVX512VNNI-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]103; AVX512VNNI-NEXT: vpxor %xmm1, %xmm1, %xmm1104; AVX512VNNI-NEXT: vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1105; AVX512VNNI-NEXT: vmovd %xmm1, %eax106; AVX512VNNI-NEXT: addl %edi, %eax107; AVX512VNNI-NEXT: vzeroupper108; AVX512VNNI-NEXT: retq109;110; AVX512VLVNNI-LABEL: mul_4xi4_cz:111; AVX512VLVNNI: # %bb.0: # %entry112; AVX512VLVNNI-NEXT: vpmovdb %xmm0, %xmm0113; AVX512VLVNNI-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0114; AVX512VLVNNI-NEXT: vpxor %xmm1, %xmm1, %xmm1115; AVX512VLVNNI-NEXT: vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1116; AVX512VLVNNI-NEXT: vmovd %xmm1, %eax117; AVX512VLVNNI-NEXT: addl %edi, %eax118; AVX512VLVNNI-NEXT: retq119entry:120 %0 = zext <4 x i4> %a to <4 x i32>121 %1 = mul nsw <4 x i32> <i32 16, i32 1, i32 2, i32 127>, %0122 %2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %1)123 %op.extra = add nsw i32 %2, %c124 ret i32 %op.extra125}126 127define i32 @mul_4xi8_cs(<4 x i8> %a, i32 %c) {128; AVXVNNI-AVX-LABEL: mul_4xi8_cs:129; AVXVNNI-AVX: # %bb.0: # %entry130; AVXVNNI-AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1131; AVXVNNI-AVX-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]132; AVXVNNI-AVX-NEXT: vmovd {{.*#+}} xmm2 = [16,1,2,255,0,0,0,0,0,0,0,0,0,0,0,0]133; AVXVNNI-AVX-NEXT: {vex} vpdpbusd %xmm0, %xmm2, %xmm1134; AVXVNNI-AVX-NEXT: vmovd %xmm1, %eax135; AVXVNNI-AVX-NEXT: addl %edi, %eax136; AVXVNNI-AVX-NEXT: retq137;138; AVXVNNI-AVX512-LABEL: mul_4xi8_cs:139; AVXVNNI-AVX512: # %bb.0: # %entry140; AVXVNNI-AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1141; AVXVNNI-AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]142; AVXVNNI-AVX512-NEXT: vmovd {{.*#+}} xmm1 = [16,1,2,255,0,0,0,0,0,0,0,0,0,0,0,0]143; AVXVNNI-AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2144; AVXVNNI-AVX512-NEXT: {vex} vpdpbusd %xmm0, %xmm1, %xmm2145; AVXVNNI-AVX512-NEXT: vmovd %xmm2, %eax146; AVXVNNI-AVX512-NEXT: addl %edi, %eax147; AVXVNNI-AVX512-NEXT: retq148;149; AVX512VNNI-LABEL: mul_4xi8_cs:150; AVX512VNNI: # %bb.0: # %entry151; AVX512VNNI-NEXT: vpxor %xmm1, %xmm1, %xmm1152; AVX512VNNI-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]153; AVX512VNNI-NEXT: vmovd {{.*#+}} xmm1 = [16,1,2,255,0,0,0,0,0,0,0,0,0,0,0,0]154; AVX512VNNI-NEXT: vpxor %xmm2, %xmm2, %xmm2155; AVX512VNNI-NEXT: vpdpbusd %zmm0, %zmm1, %zmm2156; AVX512VNNI-NEXT: vmovd %xmm2, %eax157; AVX512VNNI-NEXT: addl %edi, %eax158; AVX512VNNI-NEXT: vzeroupper159; AVX512VNNI-NEXT: retq160;161; AVX512VLVNNI-LABEL: mul_4xi8_cs:162; AVX512VLVNNI: # %bb.0: # %entry163; AVX512VLVNNI-NEXT: vpxor %xmm1, %xmm1, %xmm1164; AVX512VLVNNI-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]165; AVX512VLVNNI-NEXT: vmovd {{.*#+}} xmm1 = [16,1,2,255,0,0,0,0,0,0,0,0,0,0,0,0]166; AVX512VLVNNI-NEXT: vpxor %xmm2, %xmm2, %xmm2167; AVX512VLVNNI-NEXT: vpdpbusd %xmm0, %xmm1, %xmm2168; AVX512VLVNNI-NEXT: vmovd %xmm2, %eax169; AVX512VLVNNI-NEXT: addl %edi, %eax170; AVX512VLVNNI-NEXT: retq171entry:172 %0 = sext <4 x i8> %a to <4 x i32>173 %1 = mul nsw <4 x i32> <i32 16, i32 1, i32 2, i32 255>, %0174 %2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %1)175 %op.extra = add nsw i32 %2, %c176 ret i32 %op.extra177}178 179define i32 @mul_4xi8_cs_exceed(<4 x i8> %a, i32 %c) {180; CHECK-LABEL: mul_4xi8_cs_exceed:181; CHECK: # %bb.0: # %entry182; CHECK-NEXT: vpmovsxbd %xmm0, %xmm0183; CHECK-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [0,0,1,0,2,0,256,0]184; CHECK-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]185; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm0186; CHECK-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]187; CHECK-NEXT: vpaddd %xmm0, %xmm1, %xmm0188; CHECK-NEXT: vmovd %xmm0, %eax189; CHECK-NEXT: addl %edi, %eax190; CHECK-NEXT: retq191entry:192 %0 = sext <4 x i8> %a to <4 x i32>193 %1 = mul nsw <4 x i32> <i32 0, i32 1, i32 2, i32 256>, %0194 %2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %1)195 %op.extra = add nsw i32 %2, %c196 ret i32 %op.extra197}198 199define i32 @mul_16xi8_zc(<16 x i8> %a, i32 %c) {200; AVXVNNI-LABEL: mul_16xi8_zc:201; AVXVNNI: # %bb.0: # %entry202; AVXVNNI-NEXT: vpxor %xmm1, %xmm1, %xmm1203; AVXVNNI-NEXT: {vex} vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1204; AVXVNNI-NEXT: vpshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]205; AVXVNNI-NEXT: vpaddd %xmm0, %xmm1, %xmm0206; AVXVNNI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]207; AVXVNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0208; AVXVNNI-NEXT: vmovd %xmm0, %eax209; AVXVNNI-NEXT: addl %edi, %eax210; AVXVNNI-NEXT: retq211;212; AVX512VNNI-LABEL: mul_16xi8_zc:213; AVX512VNNI: # %bb.0: # %entry214; AVX512VNNI-NEXT: vmovdqa %xmm0, %xmm0215; AVX512VNNI-NEXT: vpxor %xmm1, %xmm1, %xmm1216; AVX512VNNI-NEXT: vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1217; AVX512VNNI-NEXT: vpshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]218; AVX512VNNI-NEXT: vpaddd %xmm0, %xmm1, %xmm0219; AVX512VNNI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]220; AVX512VNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0221; AVX512VNNI-NEXT: vmovd %xmm0, %eax222; AVX512VNNI-NEXT: addl %edi, %eax223; AVX512VNNI-NEXT: vzeroupper224; AVX512VNNI-NEXT: retq225;226; AVX512VLVNNI-LABEL: mul_16xi8_zc:227; AVX512VLVNNI: # %bb.0: # %entry228; AVX512VLVNNI-NEXT: vpxor %xmm1, %xmm1, %xmm1229; AVX512VLVNNI-NEXT: vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm1230; AVX512VLVNNI-NEXT: vpshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]231; AVX512VLVNNI-NEXT: vpaddd %xmm0, %xmm1, %xmm0232; AVX512VLVNNI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]233; AVX512VLVNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0234; AVX512VLVNNI-NEXT: vmovd %xmm0, %eax235; AVX512VLVNNI-NEXT: addl %edi, %eax236; AVX512VLVNNI-NEXT: retq237entry:238 %0 = zext <16 x i8> %a to <16 x i32>239 %1 = mul nsw <16 x i32> %0, <i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64>240 %2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %1)241 %op.extra = add nsw i32 %2, %c242 ret i32 %op.extra243}244 245define i32 @mul_32xi8_zc(<32 x i8> %a, i32 %c) {246; AVXVNNI-LABEL: mul_32xi8_zc:247; AVXVNNI: # %bb.0: # %entry248; AVXVNNI-NEXT: vpxor %xmm1, %xmm1, %xmm1249; AVXVNNI-NEXT: {vex} vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm1250; AVXVNNI-NEXT: vextracti128 $1, %ymm1, %xmm0251; AVXVNNI-NEXT: vpaddd %xmm0, %xmm1, %xmm0252; AVXVNNI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]253; AVXVNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0254; AVXVNNI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]255; AVXVNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0256; AVXVNNI-NEXT: vmovd %xmm0, %eax257; AVXVNNI-NEXT: addl %edi, %eax258; AVXVNNI-NEXT: vzeroupper259; AVXVNNI-NEXT: retq260;261; AVX512VNNI-LABEL: mul_32xi8_zc:262; AVX512VNNI: # %bb.0: # %entry263; AVX512VNNI-NEXT: vmovdqa %ymm0, %ymm0264; AVX512VNNI-NEXT: vpxor %xmm1, %xmm1, %xmm1265; AVX512VNNI-NEXT: vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1266; AVX512VNNI-NEXT: vextracti128 $1, %ymm1, %xmm0267; AVX512VNNI-NEXT: vpaddd %xmm0, %xmm1, %xmm0268; AVX512VNNI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]269; AVX512VNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0270; AVX512VNNI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]271; AVX512VNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0272; AVX512VNNI-NEXT: vmovd %xmm0, %eax273; AVX512VNNI-NEXT: addl %edi, %eax274; AVX512VNNI-NEXT: vzeroupper275; AVX512VNNI-NEXT: retq276;277; AVX512VLVNNI-LABEL: mul_32xi8_zc:278; AVX512VLVNNI: # %bb.0: # %entry279; AVX512VLVNNI-NEXT: vpxor %xmm1, %xmm1, %xmm1280; AVX512VLVNNI-NEXT: vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm1281; AVX512VLVNNI-NEXT: vextracti128 $1, %ymm1, %xmm0282; AVX512VLVNNI-NEXT: vpaddd %xmm0, %xmm1, %xmm0283; AVX512VLVNNI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]284; AVX512VLVNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0285; AVX512VLVNNI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]286; AVX512VLVNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0287; AVX512VLVNNI-NEXT: vmovd %xmm0, %eax288; AVX512VLVNNI-NEXT: addl %edi, %eax289; AVX512VLVNNI-NEXT: vzeroupper290; AVX512VLVNNI-NEXT: retq291entry:292 %0 = zext <32 x i8> %a to <32 x i32>293 %1 = mul nsw <32 x i32> %0, <i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64>294 %2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %1)295 %op.extra = add nsw i32 %2, %c296 ret i32 %op.extra297}298 299define i32 @mul_64xi8_zc(<64 x i8> %a, i32 %c) {300; AVXVNNI-AVX-LABEL: mul_64xi8_zc:301; AVXVNNI-AVX: # %bb.0: # %entry302; AVXVNNI-AVX-NEXT: vpbroadcastd {{.*#+}} ymm2 = [0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64]303; AVXVNNI-AVX-NEXT: vpxor %xmm3, %xmm3, %xmm3304; AVXVNNI-AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4305; AVXVNNI-AVX-NEXT: {vex} vpdpbusd %ymm2, %ymm1, %ymm4306; AVXVNNI-AVX-NEXT: {vex} vpdpbusd %ymm2, %ymm0, %ymm3307; AVXVNNI-AVX-NEXT: vpaddd %ymm4, %ymm3, %ymm0308; AVXVNNI-AVX-NEXT: vextracti128 $1, %ymm0, %xmm1309; AVXVNNI-AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0310; AVXVNNI-AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]311; AVXVNNI-AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0312; AVXVNNI-AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]313; AVXVNNI-AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0314; AVXVNNI-AVX-NEXT: vmovd %xmm0, %eax315; AVXVNNI-AVX-NEXT: addl %edi, %eax316; AVXVNNI-AVX-NEXT: vzeroupper317; AVXVNNI-AVX-NEXT: retq318;319; AVXVNNI-AVX512-LABEL: mul_64xi8_zc:320; AVXVNNI-AVX512: # %bb.0: # %entry321; AVXVNNI-AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1322; AVXVNNI-AVX512-NEXT: vpbroadcastd {{.*#+}} ymm2 = [0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64]323; AVXVNNI-AVX512-NEXT: vpxor %xmm3, %xmm3, %xmm3324; AVXVNNI-AVX512-NEXT: vpxor %xmm4, %xmm4, %xmm4325; AVXVNNI-AVX512-NEXT: {vex} vpdpbusd %ymm2, %ymm1, %ymm4326; AVXVNNI-AVX512-NEXT: {vex} vpdpbusd %ymm2, %ymm0, %ymm3327; AVXVNNI-AVX512-NEXT: vpaddd %ymm4, %ymm3, %ymm0328; AVXVNNI-AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1329; AVXVNNI-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0330; AVXVNNI-AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]331; AVXVNNI-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0332; AVXVNNI-AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]333; AVXVNNI-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0334; AVXVNNI-AVX512-NEXT: vmovd %xmm0, %eax335; AVXVNNI-AVX512-NEXT: addl %edi, %eax336; AVXVNNI-AVX512-NEXT: vzeroupper337; AVXVNNI-AVX512-NEXT: retq338;339; AVX512-LABEL: mul_64xi8_zc:340; AVX512: # %bb.0: # %entry341; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1342; AVX512-NEXT: vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm1343; AVX512-NEXT: vextracti64x4 $1, %zmm1, %ymm0344; AVX512-NEXT: vpaddd %zmm0, %zmm1, %zmm0345; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1346; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0347; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]348; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0349; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]350; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0351; AVX512-NEXT: vmovd %xmm0, %eax352; AVX512-NEXT: addl %edi, %eax353; AVX512-NEXT: vzeroupper354; AVX512-NEXT: retq355entry:356 %0 = zext <64 x i8> %a to <64 x i32>357 %1 = mul nsw <64 x i32> %0, <i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64, i32 0, i32 1, i32 2, i32 64>358 %2 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %1)359 %op.extra = add nsw i32 %2, %c360 ret i32 %op.extra361}362 363declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)364declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)365declare i32 @llvm.vector.reduce.add.v32i32(<32 x i32>)366declare i32 @llvm.vector.reduce.add.v64i32(<64 x i32>)367