535 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s --check-prefixes=SSE23; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=SSE44; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=AVX,AVX25; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=AVX,AVX5126 7define i32 @dot_ext_v8i8_v8i32(ptr %a, i64 %a_stride, ptr %b) nounwind {8; SSE2-LABEL: dot_ext_v8i8_v8i32:9; SSE2: # %bb.0: # %entry10; SSE2-NEXT: pushq %rbx11; SSE2-NEXT: movzbl (%rdi), %eax12; SSE2-NEXT: movzbl (%rdi,%rsi), %ecx13; SSE2-NEXT: movzbl (%rdi,%rsi,2), %r8d14; SSE2-NEXT: leaq (%rsi,%rsi,2), %r915; SSE2-NEXT: movzbl (%rdi,%r9), %r10d16; SSE2-NEXT: movzbl (%rdi,%rsi,4), %r11d17; SSE2-NEXT: leaq (%rsi,%rsi,4), %rbx18; SSE2-NEXT: movzbl (%rdi,%rbx), %ebx19; SSE2-NEXT: movzbl (%rdi,%r9,2), %r9d20; SSE2-NEXT: leaq (%rdi,%rsi,8), %rdi21; SSE2-NEXT: subq %rsi, %rdi22; SSE2-NEXT: movzbl (%rdi), %esi23; SSE2-NEXT: shll $16, %ecx24; SSE2-NEXT: orl %eax, %ecx25; SSE2-NEXT: movd %ecx, %xmm026; SSE2-NEXT: pinsrw $2, %r8d, %xmm027; SSE2-NEXT: pinsrw $3, %r10d, %xmm028; SSE2-NEXT: pinsrw $4, %r11d, %xmm029; SSE2-NEXT: pinsrw $5, %ebx, %xmm030; SSE2-NEXT: pinsrw $6, %r9d, %xmm031; SSE2-NEXT: pinsrw $7, %esi, %xmm032; SSE2-NEXT: movdqu (%rdx), %xmm133; SSE2-NEXT: pmaddwd %xmm0, %xmm134; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]35; SSE2-NEXT: paddd %xmm1, %xmm036; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]37; SSE2-NEXT: paddd %xmm0, %xmm138; SSE2-NEXT: movd %xmm1, %eax39; SSE2-NEXT: popq %rbx40; SSE2-NEXT: retq41;42; SSE4-LABEL: dot_ext_v8i8_v8i32:43; SSE4: # %bb.0: # %entry44; SSE4-NEXT: movzbl (%rdi), %eax45; SSE4-NEXT: leaq (%rsi,%rsi,4), %rcx46; SSE4-NEXT: leaq (%rsi,%rsi,2), %r847; SSE4-NEXT: leaq (%rdi,%rsi,8), %r948; SSE4-NEXT: subq %rsi, %r949; SSE4-NEXT: movd %eax, %xmm050; SSE4-NEXT: pinsrb $2, (%rdi,%rsi), %xmm051; SSE4-NEXT: pinsrb $4, (%rdi,%rsi,2), %xmm052; SSE4-NEXT: pinsrb $6, (%rdi,%r8), %xmm053; SSE4-NEXT: pinsrb $8, (%rdi,%rsi,4), %xmm054; SSE4-NEXT: pinsrb $10, (%rdi,%rcx), %xmm055; SSE4-NEXT: pinsrb $12, (%rdi,%r8,2), %xmm056; SSE4-NEXT: pinsrb $14, (%r9), %xmm057; SSE4-NEXT: movdqu (%rdx), %xmm158; SSE4-NEXT: pmaddwd %xmm0, %xmm159; SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]60; SSE4-NEXT: paddd %xmm1, %xmm061; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]62; SSE4-NEXT: paddd %xmm0, %xmm163; SSE4-NEXT: movd %xmm1, %eax64; SSE4-NEXT: retq65;66; AVX-LABEL: dot_ext_v8i8_v8i32:67; AVX: # %bb.0: # %entry68; AVX-NEXT: movzbl (%rdi), %eax69; AVX-NEXT: leaq (%rsi,%rsi,2), %rcx70; AVX-NEXT: leaq (%rsi,%rsi,4), %r871; AVX-NEXT: leaq (%rdi,%rsi,8), %r972; AVX-NEXT: subq %rsi, %r973; AVX-NEXT: vmovd %eax, %xmm074; AVX-NEXT: vpinsrb $2, (%rdi,%rsi), %xmm0, %xmm075; AVX-NEXT: vpinsrb $4, (%rdi,%rsi,2), %xmm0, %xmm076; AVX-NEXT: vpinsrb $6, (%rdi,%rcx), %xmm0, %xmm077; AVX-NEXT: vpinsrb $8, (%rdi,%rsi,4), %xmm0, %xmm078; AVX-NEXT: vpinsrb $10, (%rdi,%r8), %xmm0, %xmm079; AVX-NEXT: vpinsrb $12, (%rdi,%rcx,2), %xmm0, %xmm080; AVX-NEXT: vpinsrb $14, (%r9), %xmm0, %xmm081; AVX-NEXT: vpmaddwd (%rdx), %xmm0, %xmm082; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]83; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm084; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]85; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm086; AVX-NEXT: vmovd %xmm0, %eax87; AVX-NEXT: retq88entry:89 %var0 = load i8, ptr %a, align 190 %arrayidx.1 = getelementptr inbounds i8, ptr %a, i64 %a_stride91 %var1 = load i8, ptr %arrayidx.1, align 192 %mul.2 = shl nsw i64 %a_stride, 193 %arrayidx.2 = getelementptr inbounds i8, ptr %a, i64 %mul.294 %var2 = load i8, ptr %arrayidx.2, align 195 %mul.3 = mul nsw i64 %a_stride, 396 %arrayidx.3 = getelementptr inbounds i8, ptr %a, i64 %mul.397 %var3 = load i8, ptr %arrayidx.3, align 198 %mul.4 = shl nsw i64 %a_stride, 299 %arrayidx.4 = getelementptr inbounds i8, ptr %a, i64 %mul.4100 %var4 = load i8, ptr %arrayidx.4, align 1101 %mul.5 = mul nsw i64 %a_stride, 5102 %arrayidx.5 = getelementptr inbounds i8, ptr %a, i64 %mul.5103 %var5 = load i8, ptr %arrayidx.5, align 1104 %mul.6 = mul nsw i64 %a_stride, 6105 %arrayidx.6 = getelementptr inbounds i8, ptr %a, i64 %mul.6106 %var6 = load i8, ptr %arrayidx.6, align 1107 %mul.7 = mul nsw i64 %a_stride, 7108 %arrayidx.7 = getelementptr inbounds i8, ptr %a, i64 %mul.7109 %var7 = load i8, ptr %arrayidx.7, align 1110 %var8 = insertelement <8 x i8> poison, i8 %var0, i64 0111 %var9 = insertelement <8 x i8> %var8, i8 %var1, i64 1112 %var10 = insertelement <8 x i8> %var9, i8 %var2, i64 2113 %var11 = insertelement <8 x i8> %var10, i8 %var3, i64 3114 %var12 = insertelement <8 x i8> %var11, i8 %var4, i64 4115 %var13 = insertelement <8 x i8> %var12, i8 %var5, i64 5116 %var14 = insertelement <8 x i8> %var13, i8 %var6, i64 6117 %var15 = insertelement <8 x i8> %var14, i8 %var7, i64 7118 %var16 = zext <8 x i8> %var15 to <8 x i32>119 %var17 = load <8 x i16>, ptr %b, align 2120 %var18 = sext <8 x i16> %var17 to <8 x i32>121 %var19 = mul nsw <8 x i32> %var18, %var16122 %var20 = tail call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %var19)123 ret i32 %var20124}125 126define i32 @dot_ext_v4i8_v4i32(ptr %a, i64 %a_stride, ptr %b) nounwind {127; SSE2-LABEL: dot_ext_v4i8_v4i32:128; SSE2: # %bb.0: # %entry129; SSE2-NEXT: movzbl (%rdi), %eax130; SSE2-NEXT: movzbl (%rdi,%rsi), %ecx131; SSE2-NEXT: movzbl (%rdi,%rsi,2), %r8d132; SSE2-NEXT: leaq (%rsi,%rsi,2), %rsi133; SSE2-NEXT: movd %eax, %xmm0134; SSE2-NEXT: pinsrw $2, %ecx, %xmm0135; SSE2-NEXT: pinsrw $4, %r8d, %xmm0136; SSE2-NEXT: movzbl (%rdi,%rsi), %eax137; SSE2-NEXT: pinsrw $6, %eax, %xmm0138; SSE2-NEXT: movq {{.*#+}} xmm1 = mem[0],zero139; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]140; SSE2-NEXT: pmaddwd %xmm0, %xmm1141; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]142; SSE2-NEXT: paddd %xmm1, %xmm0143; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]144; SSE2-NEXT: paddd %xmm0, %xmm1145; SSE2-NEXT: movd %xmm1, %eax146; SSE2-NEXT: retq147;148; SSE4-LABEL: dot_ext_v4i8_v4i32:149; SSE4: # %bb.0: # %entry150; SSE4-NEXT: movzbl (%rdi), %eax151; SSE4-NEXT: leaq (%rsi,%rsi,2), %rcx152; SSE4-NEXT: movd %eax, %xmm0153; SSE4-NEXT: pinsrb $4, (%rdi,%rsi), %xmm0154; SSE4-NEXT: pinsrb $8, (%rdi,%rsi,2), %xmm0155; SSE4-NEXT: pinsrb $12, (%rdi,%rcx), %xmm0156; SSE4-NEXT: pmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero157; SSE4-NEXT: pmaddwd %xmm0, %xmm1158; SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]159; SSE4-NEXT: paddd %xmm1, %xmm0160; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]161; SSE4-NEXT: paddd %xmm0, %xmm1162; SSE4-NEXT: movd %xmm1, %eax163; SSE4-NEXT: retq164;165; AVX-LABEL: dot_ext_v4i8_v4i32:166; AVX: # %bb.0: # %entry167; AVX-NEXT: movzbl (%rdi), %eax168; AVX-NEXT: leaq (%rsi,%rsi,2), %rcx169; AVX-NEXT: vmovd %eax, %xmm0170; AVX-NEXT: vpinsrb $4, (%rdi,%rsi), %xmm0, %xmm0171; AVX-NEXT: vpinsrb $8, (%rdi,%rsi,2), %xmm0, %xmm0172; AVX-NEXT: vpinsrb $12, (%rdi,%rcx), %xmm0, %xmm0173; AVX-NEXT: vpmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero174; AVX-NEXT: vpmaddwd %xmm0, %xmm1, %xmm0175; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]176; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0177; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]178; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0179; AVX-NEXT: vmovd %xmm0, %eax180; AVX-NEXT: retq181entry:182 %var0 = load i8, ptr %a, align 1183 %arrayidx.1 = getelementptr inbounds i8, ptr %a, i64 %a_stride184 %var1 = load i8, ptr %arrayidx.1, align 1185 %mul.2 = shl nsw i64 %a_stride, 1186 %arrayidx.2 = getelementptr inbounds i8, ptr %a, i64 %mul.2187 %var2 = load i8, ptr %arrayidx.2, align 1188 %mul.3 = mul nsw i64 %a_stride, 3189 %arrayidx.3 = getelementptr inbounds i8, ptr %a, i64 %mul.3190 %var3 = load i8, ptr %arrayidx.3, align 1191 %var8 = insertelement <4 x i8> poison, i8 %var0, i64 0192 %var9 = insertelement <4 x i8> %var8, i8 %var1, i64 1193 %var10 = insertelement <4 x i8> %var9, i8 %var2, i64 2194 %var11 = insertelement <4 x i8> %var10, i8 %var3, i64 3195 %var16 = zext <4 x i8> %var11 to <4 x i32>196 %var17 = load <4 x i16>, ptr %b, align 2197 %var18 = sext <4 x i16> %var17 to <4 x i32>198 %var19 = mul nsw <4 x i32> %var18, %var16199 %var20 = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %var19)200 ret i32 %var20201}202 203define i32 @dot_ext_v2i8_v2i32(ptr %a, i64 %a_stride, ptr %b) nounwind {204; SSE2-LABEL: dot_ext_v2i8_v2i32:205; SSE2: # %bb.0:206; SSE2-NEXT: movzbl (%rdi), %eax207; SSE2-NEXT: movzbl (%rdi,%rsi), %ecx208; SSE2-NEXT: movd %eax, %xmm0209; SSE2-NEXT: pinsrw $2, %ecx, %xmm0210; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero211; SSE2-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7]212; SSE2-NEXT: pmaddwd %xmm0, %xmm1213; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]214; SSE2-NEXT: paddd %xmm1, %xmm0215; SSE2-NEXT: movd %xmm0, %eax216; SSE2-NEXT: retq217;218; SSE4-LABEL: dot_ext_v2i8_v2i32:219; SSE4: # %bb.0:220; SSE4-NEXT: movzbl (%rdi), %eax221; SSE4-NEXT: movd %eax, %xmm0222; SSE4-NEXT: pinsrb $4, (%rdi,%rsi), %xmm0223; SSE4-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero224; SSE4-NEXT: pmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero225; SSE4-NEXT: pmaddwd %xmm0, %xmm1226; SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]227; SSE4-NEXT: paddd %xmm1, %xmm0228; SSE4-NEXT: movd %xmm0, %eax229; SSE4-NEXT: retq230;231; AVX-LABEL: dot_ext_v2i8_v2i32:232; AVX: # %bb.0:233; AVX-NEXT: movzbl (%rdi), %eax234; AVX-NEXT: vmovd %eax, %xmm0235; AVX-NEXT: vpinsrb $4, (%rdi,%rsi), %xmm0, %xmm0236; AVX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero237; AVX-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero238; AVX-NEXT: vpmaddwd %xmm0, %xmm1, %xmm0239; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]240; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0241; AVX-NEXT: vmovd %xmm0, %eax242; AVX-NEXT: retq243 %var0 = load i8, ptr %a, align 1244 %arrayidx.1 = getelementptr inbounds i8, ptr %a, i64 %a_stride245 %var1 = load i8, ptr %arrayidx.1, align 1246 %var8 = insertelement <2 x i8> poison, i8 %var0, i64 0247 %var9 = insertelement <2 x i8> %var8, i8 %var1, i64 1248 %var16 = zext <2 x i8> %var9 to <2 x i32>249 %var17 = load <2 x i16>, ptr %b, align 2250 %var18 = sext <2 x i16> %var17 to <2 x i32>251 %var19 = mul nsw <2 x i32> %var18, %var16252 %var20 = tail call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %var19)253 ret i32 %var20254}255 256define i64 @dot_ext_v2i8_v2i64(ptr %a, i64 %a_stride, ptr %b) nounwind {257; SSE2-LABEL: dot_ext_v2i8_v2i64:258; SSE2: # %bb.0:259; SSE2-NEXT: movzbl (%rdi), %eax260; SSE2-NEXT: movzbl (%rdi,%rsi), %ecx261; SSE2-NEXT: movd %eax, %xmm0262; SSE2-NEXT: pinsrw $4, %ecx, %xmm0263; SSE2-NEXT: movzwl (%rdx), %eax264; SSE2-NEXT: movd %eax, %xmm1265; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]266; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]267; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,1,3]268; SSE2-NEXT: pxor %xmm2, %xmm2269; SSE2-NEXT: pcmpgtd %xmm1, %xmm2270; SSE2-NEXT: psrad $24, %xmm1271; SSE2-NEXT: pmuludq %xmm0, %xmm1272; SSE2-NEXT: pmuludq %xmm0, %xmm2273; SSE2-NEXT: psllq $32, %xmm2274; SSE2-NEXT: paddq %xmm1, %xmm2275; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]276; SSE2-NEXT: paddq %xmm2, %xmm0277; SSE2-NEXT: movq %xmm0, %rax278; SSE2-NEXT: retq279;280; SSE4-LABEL: dot_ext_v2i8_v2i64:281; SSE4: # %bb.0:282; SSE4-NEXT: movzbl (%rdi), %eax283; SSE4-NEXT: movd %eax, %xmm0284; SSE4-NEXT: pinsrb $8, (%rdi,%rsi), %xmm0285; SSE4-NEXT: pmovsxbq (%rdx), %xmm1286; SSE4-NEXT: pmuldq %xmm0, %xmm1287; SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]288; SSE4-NEXT: paddq %xmm1, %xmm0289; SSE4-NEXT: movq %xmm0, %rax290; SSE4-NEXT: retq291;292; AVX-LABEL: dot_ext_v2i8_v2i64:293; AVX: # %bb.0:294; AVX-NEXT: movzbl (%rdi), %eax295; AVX-NEXT: vmovd %eax, %xmm0296; AVX-NEXT: vpinsrb $8, (%rdi,%rsi), %xmm0, %xmm0297; AVX-NEXT: vpmovsxbq (%rdx), %xmm1298; AVX-NEXT: vpmuldq %xmm0, %xmm1, %xmm0299; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]300; AVX-NEXT: vpaddq %xmm1, %xmm0, %xmm0301; AVX-NEXT: vmovq %xmm0, %rax302; AVX-NEXT: retq303 %var0 = load i8, ptr %a, align 1304 %arrayidx.1 = getelementptr inbounds i8, ptr %a, i64 %a_stride305 %var1 = load i8, ptr %arrayidx.1, align 1306 %var8 = insertelement <2 x i8> poison, i8 %var0, i64 0307 %var9 = insertelement <2 x i8> %var8, i8 %var1, i64 1308 %var16 = zext <2 x i8> %var9 to <2 x i64>309 %var17 = load <2 x i8>, ptr %b, align 2310 %var18 = sext <2 x i8> %var17 to <2 x i64>311 %var19 = mul nsw <2 x i64> %var18, %var16312 %var20 = tail call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %var19)313 ret i64 %var20314}315 316define i32 @dot_ext_v4i16_v4i32(ptr %a, i64 %a_stride, ptr %b) nounwind {317; SSE2-LABEL: dot_ext_v4i16_v4i32:318; SSE2: # %bb.0: # %entry319; SSE2-NEXT: movzwl (%rdi), %eax320; SSE2-NEXT: movd %eax, %xmm0321; SSE2-NEXT: pinsrw $2, (%rdi,%rsi), %xmm0322; SSE2-NEXT: pinsrw $4, (%rdi,%rsi,2), %xmm0323; SSE2-NEXT: leaq (%rsi,%rsi,2), %rax324; SSE2-NEXT: movdqa %xmm0, %xmm1325; SSE2-NEXT: pinsrw $6, (%rdi,%rax), %xmm1326; SSE2-NEXT: movq {{.*#+}} xmm2 = mem[0],zero327; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]328; SSE2-NEXT: psrad $16, %xmm2329; SSE2-NEXT: pmuludq %xmm2, %xmm0330; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]331; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]332; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]333; SSE2-NEXT: pmuludq %xmm1, %xmm2334; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]335; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]336; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]337; SSE2-NEXT: paddd %xmm0, %xmm1338; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]339; SSE2-NEXT: paddd %xmm1, %xmm0340; SSE2-NEXT: movd %xmm0, %eax341; SSE2-NEXT: retq342;343; SSE4-LABEL: dot_ext_v4i16_v4i32:344; SSE4: # %bb.0: # %entry345; SSE4-NEXT: movzwl (%rdi), %eax346; SSE4-NEXT: leaq (%rsi,%rsi,2), %rcx347; SSE4-NEXT: movd %eax, %xmm0348; SSE4-NEXT: pinsrw $2, (%rdi,%rsi), %xmm0349; SSE4-NEXT: pinsrw $4, (%rdi,%rsi,2), %xmm0350; SSE4-NEXT: pinsrw $6, (%rdi,%rcx), %xmm0351; SSE4-NEXT: pmovsxwd (%rdx), %xmm1352; SSE4-NEXT: pmulld %xmm0, %xmm1353; SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]354; SSE4-NEXT: paddd %xmm1, %xmm0355; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]356; SSE4-NEXT: paddd %xmm0, %xmm1357; SSE4-NEXT: movd %xmm1, %eax358; SSE4-NEXT: retq359;360; AVX-LABEL: dot_ext_v4i16_v4i32:361; AVX: # %bb.0: # %entry362; AVX-NEXT: movzwl (%rdi), %eax363; AVX-NEXT: leaq (%rsi,%rsi,2), %rcx364; AVX-NEXT: vmovd %eax, %xmm0365; AVX-NEXT: vpinsrw $2, (%rdi,%rsi), %xmm0, %xmm0366; AVX-NEXT: vpinsrw $4, (%rdi,%rsi,2), %xmm0, %xmm0367; AVX-NEXT: vpinsrw $6, (%rdi,%rcx), %xmm0, %xmm0368; AVX-NEXT: vpmovsxwd (%rdx), %xmm1369; AVX-NEXT: vpmulld %xmm0, %xmm1, %xmm0370; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]371; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0372; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]373; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0374; AVX-NEXT: vmovd %xmm0, %eax375; AVX-NEXT: retq376entry:377 %var0 = load i16, ptr %a, align 1378 %arrayidx.1 = getelementptr inbounds i8, ptr %a, i64 %a_stride379 %var1 = load i16, ptr %arrayidx.1, align 1380 %mul.2 = shl nsw i64 %a_stride, 1381 %arrayidx.2 = getelementptr inbounds i8, ptr %a, i64 %mul.2382 %var2 = load i16, ptr %arrayidx.2, align 1383 %mul.3 = mul nsw i64 %a_stride, 3384 %arrayidx.3 = getelementptr inbounds i8, ptr %a, i64 %mul.3385 %var3 = load i16, ptr %arrayidx.3, align 1386 %var8 = insertelement <4 x i16> poison, i16 %var0, i64 0387 %var9 = insertelement <4 x i16> %var8, i16 %var1, i64 1388 %var10 = insertelement <4 x i16> %var9, i16 %var2, i64 2389 %var11 = insertelement <4 x i16> %var10, i16 %var3, i64 3390 %var16 = zext <4 x i16> %var11 to <4 x i32>391 %var17 = load <4 x i16>, ptr %b, align 2392 %var18 = sext <4 x i16> %var17 to <4 x i32>393 %var19 = mul nsw <4 x i32> %var18, %var16394 %var20 = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %var19)395 ret i32 %var20396}397 398define i32 @dot_ext_v2i16_v2i32(ptr %a, i64 %a_stride, ptr %b) nounwind {399; SSE2-LABEL: dot_ext_v2i16_v2i32:400; SSE2: # %bb.0:401; SSE2-NEXT: movzwl (%rdi), %eax402; SSE2-NEXT: movd %eax, %xmm0403; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero404; SSE2-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,0,2,1,4,5,6,7]405; SSE2-NEXT: psrad $16, %xmm1406; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]407; SSE2-NEXT: pmuludq %xmm0, %xmm1408; SSE2-NEXT: pinsrw $2, (%rdi,%rsi), %xmm0409; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]410; SSE2-NEXT: pmuludq %xmm0, %xmm2411; SSE2-NEXT: paddd %xmm2, %xmm1412; SSE2-NEXT: movd %xmm1, %eax413; SSE2-NEXT: retq414;415; SSE4-LABEL: dot_ext_v2i16_v2i32:416; SSE4: # %bb.0:417; SSE4-NEXT: movzwl (%rdi), %eax418; SSE4-NEXT: movd %eax, %xmm0419; SSE4-NEXT: pinsrw $1, (%rdi,%rsi), %xmm0420; SSE4-NEXT: pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero421; SSE4-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero422; SSE4-NEXT: pmovsxwd %xmm1, %xmm1423; SSE4-NEXT: pmulld %xmm0, %xmm1424; SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]425; SSE4-NEXT: paddd %xmm1, %xmm0426; SSE4-NEXT: movd %xmm0, %eax427; SSE4-NEXT: retq428;429; AVX-LABEL: dot_ext_v2i16_v2i32:430; AVX: # %bb.0:431; AVX-NEXT: movzwl (%rdi), %eax432; AVX-NEXT: vmovd %eax, %xmm0433; AVX-NEXT: vpinsrw $1, (%rdi,%rsi), %xmm0, %xmm0434; AVX-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero435; AVX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero436; AVX-NEXT: vpmovsxwd %xmm1, %xmm1437; AVX-NEXT: vpmulld %xmm0, %xmm1, %xmm0438; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]439; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0440; AVX-NEXT: vmovd %xmm0, %eax441; AVX-NEXT: retq442 %var0 = load i16, ptr %a, align 1443 %arrayidx.1 = getelementptr inbounds i8, ptr %a, i64 %a_stride444 %var1 = load i16, ptr %arrayidx.1, align 1445 %var8 = insertelement <2 x i16> poison, i16 %var0, i64 0446 %var9 = insertelement <2 x i16> %var8, i16 %var1, i64 1447 %var16 = zext <2 x i16> %var9 to <2 x i32>448 %var17 = load <2 x i16>, ptr %b, align 2449 %var18 = sext <2 x i16> %var17 to <2 x i32>450 %var19 = mul nsw <2 x i32> %var18, %var16451 %var20 = tail call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %var19)452 ret i32 %var20453}454 455define i64 @dot_ext_v2i32_v2i64(ptr %a, i64 %a_stride, ptr %b) nounwind {456; SSE2-LABEL: dot_ext_v2i32_v2i64:457; SSE2: # %bb.0:458; SSE2-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero459; SSE2-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero460; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]461; SSE2-NEXT: movq {{.*#+}} xmm0 = mem[0],zero462; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,1,3]463; SSE2-NEXT: pxor %xmm2, %xmm2464; SSE2-NEXT: pcmpgtd %xmm0, %xmm2465; SSE2-NEXT: pmuludq %xmm1, %xmm2466; SSE2-NEXT: psllq $32, %xmm2467; SSE2-NEXT: pmuludq %xmm0, %xmm1468; SSE2-NEXT: paddq %xmm2, %xmm1469; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]470; SSE2-NEXT: paddq %xmm1, %xmm0471; SSE2-NEXT: movq %xmm0, %rax472; SSE2-NEXT: retq473;474; SSE4-LABEL: dot_ext_v2i32_v2i64:475; SSE4: # %bb.0:476; SSE4-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero477; SSE4-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero478; SSE4-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]479; SSE4-NEXT: pmovsxdq (%rdx), %xmm0480; SSE4-NEXT: movdqa %xmm0, %xmm2481; SSE4-NEXT: pmuludq %xmm1, %xmm2482; SSE4-NEXT: psrlq $32, %xmm0483; SSE4-NEXT: pmuludq %xmm1, %xmm0484; SSE4-NEXT: psllq $32, %xmm0485; SSE4-NEXT: paddq %xmm2, %xmm0486; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]487; SSE4-NEXT: paddq %xmm0, %xmm1488; SSE4-NEXT: movq %xmm1, %rax489; SSE4-NEXT: retq490;491; AVX2-LABEL: dot_ext_v2i32_v2i64:492; AVX2: # %bb.0:493; AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero494; AVX2-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero495; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]496; AVX2-NEXT: vpmovsxdq (%rdx), %xmm1497; AVX2-NEXT: vpmuludq %xmm0, %xmm1, %xmm2498; AVX2-NEXT: vpsrlq $32, %xmm1, %xmm1499; AVX2-NEXT: vpmuludq %xmm0, %xmm1, %xmm0500; AVX2-NEXT: vpsllq $32, %xmm0, %xmm0501; AVX2-NEXT: vpaddq %xmm0, %xmm2, %xmm0502; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]503; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0504; AVX2-NEXT: vmovq %xmm0, %rax505; AVX2-NEXT: retq506;507; AVX512-LABEL: dot_ext_v2i32_v2i64:508; AVX512: # %bb.0:509; AVX512-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero510; AVX512-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero511; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]512; AVX512-NEXT: vpmovsxdq (%rdx), %xmm1513; AVX512-NEXT: vpmullq %xmm0, %xmm1, %xmm0514; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]515; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0516; AVX512-NEXT: vmovq %xmm0, %rax517; AVX512-NEXT: retq518 %var0 = load i32, ptr %a, align 1519 %arrayidx.1 = getelementptr inbounds i8, ptr %a, i64 %a_stride520 %var1 = load i32, ptr %arrayidx.1, align 1521 %var8 = insertelement <2 x i32> poison, i32 %var0, i64 0522 %var9 = insertelement <2 x i32> %var8, i32 %var1, i64 1523 %var16 = zext <2 x i32> %var9 to <2 x i64>524 %var17 = load <2 x i32>, ptr %b, align 2525 %var18 = sext <2 x i32> %var17 to <2 x i64>526 %var19 = mul nsw <2 x i64> %var18, %var16527 %var20 = tail call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %var19)528 ret i64 %var20529}530 531declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)532declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)533declare i32 @llvm.vector.reduce.add.v2i32(<2 x i32>)534declare i64 @llvm.vector.reduce.add.v2i64(<2 x i64>)535