682 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=AVX14; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX25; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+xop | FileCheck %s --check-prefix=XOP6 7; PR37428 - https://bugs.llvm.org/show_bug.cgi?id=374288; This is a larger-than-usual regression test to verify that several backend9; transforms are working together. We want to hoist the expansion of non-uniform10; vector shifts out of a loop if we do not have real vector shift instructions.11; See test/Transforms/CodeGenPrepare/X86/vec-shift.ll for the 1st step in that12; sequence.13 14define void @vector_variable_shift_left_loop(ptr nocapture %arr, ptr nocapture readonly %control, i32 %count, i32 %amt0, i32 %amt1) nounwind {15; SSE-LABEL: vector_variable_shift_left_loop:16; SSE: # %bb.0: # %entry17; SSE-NEXT: testl %edx, %edx18; SSE-NEXT: jle .LBB0_919; SSE-NEXT: # %bb.1: # %for.body.preheader20; SSE-NEXT: movl %ecx, %eax21; SSE-NEXT: movl %edx, %r9d22; SSE-NEXT: cmpl $31, %edx23; SSE-NEXT: ja .LBB0_324; SSE-NEXT: # %bb.2:25; SSE-NEXT: xorl %edx, %edx26; SSE-NEXT: jmp .LBB0_627; SSE-NEXT: .LBB0_3: # %vector.ph28; SSE-NEXT: movl %r9d, %edx29; SSE-NEXT: andl $-32, %edx30; SSE-NEXT: movd %eax, %xmm031; SSE-NEXT: movd %r8d, %xmm132; SSE-NEXT: xorl %ecx, %ecx33; SSE-NEXT: pxor %xmm8, %xmm834; SSE-NEXT: pmovzxdq {{.*#+}} xmm9 = xmm0[0],zero,xmm0[1],zero35; SSE-NEXT: pmovzxdq {{.*#+}} xmm10 = xmm1[0],zero,xmm1[1],zero36; SSE-NEXT: .p2align 437; SSE-NEXT: .LBB0_4: # %vector.body38; SSE-NEXT: # =>This Inner Loop Header: Depth=139; SSE-NEXT: movq {{.*#+}} xmm0 = mem[0],zero40; SSE-NEXT: movq {{.*#+}} xmm1 = mem[0],zero41; SSE-NEXT: movq {{.*#+}} xmm2 = mem[0],zero42; SSE-NEXT: movq {{.*#+}} xmm11 = mem[0],zero43; SSE-NEXT: pcmpeqb %xmm8, %xmm044; SSE-NEXT: pmovsxbd %xmm0, %xmm745; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]46; SSE-NEXT: pmovsxbd %xmm0, %xmm047; SSE-NEXT: pcmpeqb %xmm8, %xmm148; SSE-NEXT: pmovsxbd %xmm1, %xmm549; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]50; SSE-NEXT: pmovsxbd %xmm1, %xmm651; SSE-NEXT: pcmpeqb %xmm8, %xmm252; SSE-NEXT: pmovsxbd %xmm2, %xmm353; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]54; SSE-NEXT: pmovsxbd %xmm1, %xmm455; SSE-NEXT: pcmpeqb %xmm8, %xmm1156; SSE-NEXT: pmovsxbd %xmm11, %xmm157; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm11[1,1,1,1]58; SSE-NEXT: pmovsxbd %xmm2, %xmm259; SSE-NEXT: movdqu 16(%rdi,%rcx,4), %xmm1160; SSE-NEXT: movdqa %xmm11, %xmm1261; SSE-NEXT: pslld %xmm9, %xmm1262; SSE-NEXT: pslld %xmm10, %xmm1163; SSE-NEXT: blendvps %xmm0, %xmm12, %xmm1164; SSE-NEXT: movdqu (%rdi,%rcx,4), %xmm1265; SSE-NEXT: movdqa %xmm12, %xmm1366; SSE-NEXT: pslld %xmm9, %xmm1367; SSE-NEXT: pslld %xmm10, %xmm1268; SSE-NEXT: movdqa %xmm7, %xmm069; SSE-NEXT: blendvps %xmm0, %xmm13, %xmm1270; SSE-NEXT: movdqu 48(%rdi,%rcx,4), %xmm771; SSE-NEXT: movdqa %xmm7, %xmm1372; SSE-NEXT: pslld %xmm9, %xmm1373; SSE-NEXT: pslld %xmm10, %xmm774; SSE-NEXT: movdqa %xmm6, %xmm075; SSE-NEXT: blendvps %xmm0, %xmm13, %xmm776; SSE-NEXT: movdqu 32(%rdi,%rcx,4), %xmm677; SSE-NEXT: movdqa %xmm6, %xmm1378; SSE-NEXT: pslld %xmm9, %xmm1379; SSE-NEXT: pslld %xmm10, %xmm680; SSE-NEXT: movdqa %xmm5, %xmm081; SSE-NEXT: blendvps %xmm0, %xmm13, %xmm682; SSE-NEXT: movdqu 80(%rdi,%rcx,4), %xmm583; SSE-NEXT: movdqa %xmm5, %xmm1384; SSE-NEXT: pslld %xmm9, %xmm1385; SSE-NEXT: pslld %xmm10, %xmm586; SSE-NEXT: movdqa %xmm4, %xmm087; SSE-NEXT: blendvps %xmm0, %xmm13, %xmm588; SSE-NEXT: movdqu 64(%rdi,%rcx,4), %xmm489; SSE-NEXT: movdqa %xmm4, %xmm1390; SSE-NEXT: pslld %xmm9, %xmm1391; SSE-NEXT: pslld %xmm10, %xmm492; SSE-NEXT: movdqa %xmm3, %xmm093; SSE-NEXT: blendvps %xmm0, %xmm13, %xmm494; SSE-NEXT: movdqu 112(%rdi,%rcx,4), %xmm395; SSE-NEXT: movdqa %xmm3, %xmm1396; SSE-NEXT: pslld %xmm9, %xmm1397; SSE-NEXT: pslld %xmm10, %xmm398; SSE-NEXT: movdqa %xmm2, %xmm099; SSE-NEXT: blendvps %xmm0, %xmm13, %xmm3100; SSE-NEXT: movdqu 96(%rdi,%rcx,4), %xmm2101; SSE-NEXT: movdqa %xmm2, %xmm13102; SSE-NEXT: pslld %xmm9, %xmm13103; SSE-NEXT: pslld %xmm10, %xmm2104; SSE-NEXT: movdqa %xmm1, %xmm0105; SSE-NEXT: blendvps %xmm0, %xmm13, %xmm2106; SSE-NEXT: movups %xmm12, (%rdi,%rcx,4)107; SSE-NEXT: movups %xmm11, 16(%rdi,%rcx,4)108; SSE-NEXT: movups %xmm6, 32(%rdi,%rcx,4)109; SSE-NEXT: movups %xmm7, 48(%rdi,%rcx,4)110; SSE-NEXT: movups %xmm4, 64(%rdi,%rcx,4)111; SSE-NEXT: movups %xmm5, 80(%rdi,%rcx,4)112; SSE-NEXT: movups %xmm2, 96(%rdi,%rcx,4)113; SSE-NEXT: movups %xmm3, 112(%rdi,%rcx,4)114; SSE-NEXT: addq $32, %rcx115; SSE-NEXT: cmpq %rcx, %rdx116; SSE-NEXT: jne .LBB0_4117; SSE-NEXT: # %bb.5: # %middle.block118; SSE-NEXT: cmpl %r9d, %edx119; SSE-NEXT: jne .LBB0_6120; SSE-NEXT: .LBB0_9: # %for.cond.cleanup121; SSE-NEXT: retq122; SSE-NEXT: .p2align 4123; SSE-NEXT: .LBB0_8: # %for.body124; SSE-NEXT: # in Loop: Header=BB0_6 Depth=1125; SSE-NEXT: # kill: def $cl killed $cl killed $ecx126; SSE-NEXT: shll %cl, (%rdi,%rdx,4)127; SSE-NEXT: incq %rdx128; SSE-NEXT: cmpq %rdx, %r9129; SSE-NEXT: je .LBB0_9130; SSE-NEXT: .LBB0_6: # %for.body131; SSE-NEXT: # =>This Inner Loop Header: Depth=1132; SSE-NEXT: cmpb $0, (%rsi,%rdx)133; SSE-NEXT: movl %eax, %ecx134; SSE-NEXT: je .LBB0_8135; SSE-NEXT: # %bb.7: # %for.body136; SSE-NEXT: # in Loop: Header=BB0_6 Depth=1137; SSE-NEXT: movl %r8d, %ecx138; SSE-NEXT: jmp .LBB0_8139;140; AVX1-LABEL: vector_variable_shift_left_loop:141; AVX1: # %bb.0: # %entry142; AVX1-NEXT: testl %edx, %edx143; AVX1-NEXT: jle .LBB0_9144; AVX1-NEXT: # %bb.1: # %for.body.preheader145; AVX1-NEXT: movl %ecx, %eax146; AVX1-NEXT: movl %edx, %r9d147; AVX1-NEXT: cmpl $31, %edx148; AVX1-NEXT: ja .LBB0_3149; AVX1-NEXT: # %bb.2:150; AVX1-NEXT: xorl %edx, %edx151; AVX1-NEXT: jmp .LBB0_6152; AVX1-NEXT: .LBB0_3: # %vector.ph153; AVX1-NEXT: movl %r9d, %edx154; AVX1-NEXT: andl $-32, %edx155; AVX1-NEXT: vmovd %eax, %xmm7156; AVX1-NEXT: vmovd %r8d, %xmm8157; AVX1-NEXT: xorl %ecx, %ecx158; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm7[0],zero,xmm7[1],zero159; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill160; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm8[0],zero,xmm8[1],zero161; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill162; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm7[0],zero,xmm7[1],zero163; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill164; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm8[0],zero,xmm8[1],zero165; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill166; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm7[0],zero,xmm7[1],zero167; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm6 = xmm8[0],zero,xmm8[1],zero168; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm7 = xmm7[0],zero,xmm7[1],zero169; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm8 = xmm8[0],zero,xmm8[1],zero170; AVX1-NEXT: .p2align 4171; AVX1-NEXT: .LBB0_4: # %vector.body172; AVX1-NEXT: # =>This Inner Loop Header: Depth=1173; AVX1-NEXT: vmovq {{.*#+}} xmm9 = mem[0],zero174; AVX1-NEXT: vmovq {{.*#+}} xmm10 = mem[0],zero175; AVX1-NEXT: vmovq {{.*#+}} xmm11 = mem[0],zero176; AVX1-NEXT: vmovq {{.*#+}} xmm12 = mem[0],zero177; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2178; AVX1-NEXT: vpcmpeqb %xmm2, %xmm9, %xmm9179; AVX1-NEXT: vpmovsxbd %xmm9, %xmm14180; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[1,1,1,1]181; AVX1-NEXT: vpmovsxbd %xmm9, %xmm15182; AVX1-NEXT: vpcmpeqb %xmm2, %xmm10, %xmm9183; AVX1-NEXT: vpmovsxbd %xmm9, %xmm0184; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[1,1,1,1]185; AVX1-NEXT: vpmovsxbd %xmm9, %xmm1186; AVX1-NEXT: vpcmpeqb %xmm2, %xmm11, %xmm9187; AVX1-NEXT: vpmovsxbd %xmm9, %xmm13188; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[1,1,1,1]189; AVX1-NEXT: vpmovsxbd %xmm9, %xmm11190; AVX1-NEXT: vpcmpeqb %xmm2, %xmm12, %xmm9191; AVX1-NEXT: vpmovsxbd %xmm9, %xmm10192; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[1,1,1,1]193; AVX1-NEXT: vpmovsxbd %xmm9, %xmm9194; AVX1-NEXT: vmovdqu (%rdi,%rcx,4), %xmm12195; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload196; AVX1-NEXT: vpslld %xmm3, %xmm12, %xmm2197; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload198; AVX1-NEXT: vpslld %xmm4, %xmm12, %xmm12199; AVX1-NEXT: vblendvps %xmm14, %xmm2, %xmm12, %xmm12200; AVX1-NEXT: vmovdqu 16(%rdi,%rcx,4), %xmm2201; AVX1-NEXT: vpslld %xmm3, %xmm2, %xmm14202; AVX1-NEXT: vpslld %xmm4, %xmm2, %xmm2203; AVX1-NEXT: vblendvps %xmm15, %xmm14, %xmm2, %xmm2204; AVX1-NEXT: vmovdqu 32(%rdi,%rcx,4), %xmm14205; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload206; AVX1-NEXT: vpslld %xmm3, %xmm14, %xmm15207; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload208; AVX1-NEXT: vpslld %xmm4, %xmm14, %xmm14209; AVX1-NEXT: vblendvps %xmm0, %xmm15, %xmm14, %xmm0210; AVX1-NEXT: vmovdqu 48(%rdi,%rcx,4), %xmm14211; AVX1-NEXT: vpslld %xmm3, %xmm14, %xmm15212; AVX1-NEXT: vpslld %xmm4, %xmm14, %xmm14213; AVX1-NEXT: vblendvps %xmm1, %xmm15, %xmm14, %xmm1214; AVX1-NEXT: vmovdqu 64(%rdi,%rcx,4), %xmm14215; AVX1-NEXT: vpslld %xmm5, %xmm14, %xmm15216; AVX1-NEXT: vpslld %xmm6, %xmm14, %xmm14217; AVX1-NEXT: vblendvps %xmm13, %xmm15, %xmm14, %xmm13218; AVX1-NEXT: vmovdqu 80(%rdi,%rcx,4), %xmm14219; AVX1-NEXT: vpslld %xmm5, %xmm14, %xmm15220; AVX1-NEXT: vpslld %xmm6, %xmm14, %xmm14221; AVX1-NEXT: vblendvps %xmm11, %xmm15, %xmm14, %xmm11222; AVX1-NEXT: vmovdqu 96(%rdi,%rcx,4), %xmm14223; AVX1-NEXT: vpslld %xmm7, %xmm14, %xmm15224; AVX1-NEXT: vpslld %xmm8, %xmm14, %xmm14225; AVX1-NEXT: vblendvps %xmm10, %xmm15, %xmm14, %xmm10226; AVX1-NEXT: vmovdqu 112(%rdi,%rcx,4), %xmm14227; AVX1-NEXT: vpslld %xmm7, %xmm14, %xmm15228; AVX1-NEXT: vpslld %xmm8, %xmm14, %xmm14229; AVX1-NEXT: vblendvps %xmm9, %xmm15, %xmm14, %xmm9230; AVX1-NEXT: vmovups %xmm12, (%rdi,%rcx,4)231; AVX1-NEXT: vmovups %xmm2, 16(%rdi,%rcx,4)232; AVX1-NEXT: vmovups %xmm0, 32(%rdi,%rcx,4)233; AVX1-NEXT: vmovups %xmm1, 48(%rdi,%rcx,4)234; AVX1-NEXT: vmovups %xmm13, 64(%rdi,%rcx,4)235; AVX1-NEXT: vmovups %xmm11, 80(%rdi,%rcx,4)236; AVX1-NEXT: vmovups %xmm10, 96(%rdi,%rcx,4)237; AVX1-NEXT: vmovups %xmm9, 112(%rdi,%rcx,4)238; AVX1-NEXT: addq $32, %rcx239; AVX1-NEXT: cmpq %rcx, %rdx240; AVX1-NEXT: jne .LBB0_4241; AVX1-NEXT: # %bb.5: # %middle.block242; AVX1-NEXT: cmpl %r9d, %edx243; AVX1-NEXT: jne .LBB0_6244; AVX1-NEXT: .LBB0_9: # %for.cond.cleanup245; AVX1-NEXT: vzeroupper246; AVX1-NEXT: retq247; AVX1-NEXT: .p2align 4248; AVX1-NEXT: .LBB0_8: # %for.body249; AVX1-NEXT: # in Loop: Header=BB0_6 Depth=1250; AVX1-NEXT: # kill: def $cl killed $cl killed $ecx251; AVX1-NEXT: shll %cl, (%rdi,%rdx,4)252; AVX1-NEXT: incq %rdx253; AVX1-NEXT: cmpq %rdx, %r9254; AVX1-NEXT: je .LBB0_9255; AVX1-NEXT: .LBB0_6: # %for.body256; AVX1-NEXT: # =>This Inner Loop Header: Depth=1257; AVX1-NEXT: cmpb $0, (%rsi,%rdx)258; AVX1-NEXT: movl %eax, %ecx259; AVX1-NEXT: je .LBB0_8260; AVX1-NEXT: # %bb.7: # %for.body261; AVX1-NEXT: # in Loop: Header=BB0_6 Depth=1262; AVX1-NEXT: movl %r8d, %ecx263; AVX1-NEXT: jmp .LBB0_8264;265; AVX2-LABEL: vector_variable_shift_left_loop:266; AVX2: # %bb.0: # %entry267; AVX2-NEXT: testl %edx, %edx268; AVX2-NEXT: jle .LBB0_9269; AVX2-NEXT: # %bb.1: # %for.body.preheader270; AVX2-NEXT: movl %ecx, %eax271; AVX2-NEXT: movl %edx, %r9d272; AVX2-NEXT: cmpl $31, %edx273; AVX2-NEXT: ja .LBB0_3274; AVX2-NEXT: # %bb.2:275; AVX2-NEXT: xorl %edx, %edx276; AVX2-NEXT: jmp .LBB0_6277; AVX2-NEXT: .LBB0_3: # %vector.ph278; AVX2-NEXT: movl %r9d, %edx279; AVX2-NEXT: andl $-32, %edx280; AVX2-NEXT: vmovd %eax, %xmm0281; AVX2-NEXT: vpbroadcastd %xmm0, %ymm0282; AVX2-NEXT: vmovd %r8d, %xmm1283; AVX2-NEXT: vpbroadcastd %xmm1, %ymm1284; AVX2-NEXT: xorl %ecx, %ecx285; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2286; AVX2-NEXT: .p2align 4287; AVX2-NEXT: .LBB0_4: # %vector.body288; AVX2-NEXT: # =>This Inner Loop Header: Depth=1289; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm3 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero290; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero291; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm5 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero292; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm6 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero293; AVX2-NEXT: vpcmpeqd %ymm2, %ymm3, %ymm3294; AVX2-NEXT: vblendvps %ymm3, %ymm0, %ymm1, %ymm3295; AVX2-NEXT: vpcmpeqd %ymm2, %ymm4, %ymm4296; AVX2-NEXT: vblendvps %ymm4, %ymm0, %ymm1, %ymm4297; AVX2-NEXT: vpcmpeqd %ymm2, %ymm5, %ymm5298; AVX2-NEXT: vblendvps %ymm5, %ymm0, %ymm1, %ymm5299; AVX2-NEXT: vpcmpeqd %ymm2, %ymm6, %ymm6300; AVX2-NEXT: vblendvps %ymm6, %ymm0, %ymm1, %ymm6301; AVX2-NEXT: vmovdqu (%rdi,%rcx,4), %ymm7302; AVX2-NEXT: vpsllvd %ymm3, %ymm7, %ymm3303; AVX2-NEXT: vmovdqu 32(%rdi,%rcx,4), %ymm7304; AVX2-NEXT: vpsllvd %ymm4, %ymm7, %ymm4305; AVX2-NEXT: vmovdqu 64(%rdi,%rcx,4), %ymm7306; AVX2-NEXT: vpsllvd %ymm5, %ymm7, %ymm5307; AVX2-NEXT: vmovdqu 96(%rdi,%rcx,4), %ymm7308; AVX2-NEXT: vpsllvd %ymm6, %ymm7, %ymm6309; AVX2-NEXT: vmovdqu %ymm3, (%rdi,%rcx,4)310; AVX2-NEXT: vmovdqu %ymm4, 32(%rdi,%rcx,4)311; AVX2-NEXT: vmovdqu %ymm5, 64(%rdi,%rcx,4)312; AVX2-NEXT: vmovdqu %ymm6, 96(%rdi,%rcx,4)313; AVX2-NEXT: addq $32, %rcx314; AVX2-NEXT: cmpq %rcx, %rdx315; AVX2-NEXT: jne .LBB0_4316; AVX2-NEXT: # %bb.5: # %middle.block317; AVX2-NEXT: cmpl %r9d, %edx318; AVX2-NEXT: jne .LBB0_6319; AVX2-NEXT: .LBB0_9: # %for.cond.cleanup320; AVX2-NEXT: vzeroupper321; AVX2-NEXT: retq322; AVX2-NEXT: .p2align 4323; AVX2-NEXT: .LBB0_8: # %for.body324; AVX2-NEXT: # in Loop: Header=BB0_6 Depth=1325; AVX2-NEXT: # kill: def $cl killed $cl killed $ecx326; AVX2-NEXT: shll %cl, (%rdi,%rdx,4)327; AVX2-NEXT: incq %rdx328; AVX2-NEXT: cmpq %rdx, %r9329; AVX2-NEXT: je .LBB0_9330; AVX2-NEXT: .LBB0_6: # %for.body331; AVX2-NEXT: # =>This Inner Loop Header: Depth=1332; AVX2-NEXT: cmpb $0, (%rsi,%rdx)333; AVX2-NEXT: movl %eax, %ecx334; AVX2-NEXT: je .LBB0_8335; AVX2-NEXT: # %bb.7: # %for.body336; AVX2-NEXT: # in Loop: Header=BB0_6 Depth=1337; AVX2-NEXT: movl %r8d, %ecx338; AVX2-NEXT: jmp .LBB0_8339;340; XOP-LABEL: vector_variable_shift_left_loop:341; XOP: # %bb.0: # %entry342; XOP-NEXT: testl %edx, %edx343; XOP-NEXT: jle .LBB0_9344; XOP-NEXT: # %bb.1: # %for.body.preheader345; XOP-NEXT: movl %ecx, %eax346; XOP-NEXT: movl %edx, %r9d347; XOP-NEXT: cmpl $31, %edx348; XOP-NEXT: ja .LBB0_3349; XOP-NEXT: # %bb.2:350; XOP-NEXT: xorl %edx, %edx351; XOP-NEXT: jmp .LBB0_6352; XOP-NEXT: .LBB0_3: # %vector.ph353; XOP-NEXT: movl %r9d, %edx354; XOP-NEXT: andl $-32, %edx355; XOP-NEXT: vmovd %eax, %xmm0356; XOP-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]357; XOP-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0358; XOP-NEXT: vmovd %r8d, %xmm1359; XOP-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]360; XOP-NEXT: vinsertf128 $1, %xmm1, %ymm1, %ymm1361; XOP-NEXT: xorl %ecx, %ecx362; XOP-NEXT: vpxor %xmm2, %xmm2, %xmm2363; XOP-NEXT: vextractf128 $1, %ymm0, %xmm3364; XOP-NEXT: vextractf128 $1, %ymm1, %xmm4365; XOP-NEXT: .p2align 4366; XOP-NEXT: .LBB0_4: # %vector.body367; XOP-NEXT: # =>This Inner Loop Header: Depth=1368; XOP-NEXT: vmovq {{.*#+}} xmm5 = mem[0],zero369; XOP-NEXT: vmovq {{.*#+}} xmm6 = mem[0],zero370; XOP-NEXT: vmovq {{.*#+}} xmm7 = mem[0],zero371; XOP-NEXT: vmovq {{.*#+}} xmm8 = mem[0],zero372; XOP-NEXT: vpcomeqb %xmm2, %xmm5, %xmm5373; XOP-NEXT: vpmovsxbd %xmm5, %xmm9374; XOP-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[1,1,1,1]375; XOP-NEXT: vpmovsxbd %xmm5, %xmm5376; XOP-NEXT: vpcomeqb %xmm2, %xmm6, %xmm6377; XOP-NEXT: vpmovsxbd %xmm6, %xmm10378; XOP-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[1,1,1,1]379; XOP-NEXT: vpmovsxbd %xmm6, %xmm6380; XOP-NEXT: vpcomeqb %xmm2, %xmm7, %xmm7381; XOP-NEXT: vpmovsxbd %xmm7, %xmm11382; XOP-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[1,1,1,1]383; XOP-NEXT: vpmovsxbd %xmm7, %xmm7384; XOP-NEXT: vpcomeqb %xmm2, %xmm8, %xmm8385; XOP-NEXT: vpmovsxbd %xmm8, %xmm12386; XOP-NEXT: vpshufd {{.*#+}} xmm8 = xmm8[1,1,1,1]387; XOP-NEXT: vpmovsxbd %xmm8, %xmm8388; XOP-NEXT: vblendvps %xmm5, %xmm3, %xmm4, %xmm5389; XOP-NEXT: vpshld %xmm5, 16(%rdi,%rcx,4), %xmm5390; XOP-NEXT: vblendvps %xmm9, %xmm0, %xmm1, %xmm9391; XOP-NEXT: vpshld %xmm9, (%rdi,%rcx,4), %xmm9392; XOP-NEXT: vblendvps %xmm6, %xmm3, %xmm4, %xmm6393; XOP-NEXT: vpshld %xmm6, 48(%rdi,%rcx,4), %xmm6394; XOP-NEXT: vblendvps %xmm10, %xmm0, %xmm1, %xmm10395; XOP-NEXT: vpshld %xmm10, 32(%rdi,%rcx,4), %xmm10396; XOP-NEXT: vblendvps %xmm7, %xmm3, %xmm4, %xmm7397; XOP-NEXT: vpshld %xmm7, 80(%rdi,%rcx,4), %xmm7398; XOP-NEXT: vblendvps %xmm11, %xmm0, %xmm1, %xmm11399; XOP-NEXT: vpshld %xmm11, 64(%rdi,%rcx,4), %xmm11400; XOP-NEXT: vblendvps %xmm8, %xmm3, %xmm4, %xmm8401; XOP-NEXT: vpshld %xmm8, 112(%rdi,%rcx,4), %xmm8402; XOP-NEXT: vblendvps %xmm12, %xmm0, %xmm1, %xmm12403; XOP-NEXT: vpshld %xmm12, 96(%rdi,%rcx,4), %xmm12404; XOP-NEXT: vmovdqu %xmm9, (%rdi,%rcx,4)405; XOP-NEXT: vmovdqu %xmm5, 16(%rdi,%rcx,4)406; XOP-NEXT: vmovdqu %xmm10, 32(%rdi,%rcx,4)407; XOP-NEXT: vmovdqu %xmm6, 48(%rdi,%rcx,4)408; XOP-NEXT: vmovdqu %xmm11, 64(%rdi,%rcx,4)409; XOP-NEXT: vmovdqu %xmm7, 80(%rdi,%rcx,4)410; XOP-NEXT: vmovdqu %xmm12, 96(%rdi,%rcx,4)411; XOP-NEXT: vmovdqu %xmm8, 112(%rdi,%rcx,4)412; XOP-NEXT: addq $32, %rcx413; XOP-NEXT: cmpq %rcx, %rdx414; XOP-NEXT: jne .LBB0_4415; XOP-NEXT: # %bb.5: # %middle.block416; XOP-NEXT: cmpl %r9d, %edx417; XOP-NEXT: jne .LBB0_6418; XOP-NEXT: .LBB0_9: # %for.cond.cleanup419; XOP-NEXT: vzeroupper420; XOP-NEXT: retq421; XOP-NEXT: .p2align 4422; XOP-NEXT: .LBB0_8: # %for.body423; XOP-NEXT: # in Loop: Header=BB0_6 Depth=1424; XOP-NEXT: # kill: def $cl killed $cl killed $ecx425; XOP-NEXT: shll %cl, (%rdi,%rdx,4)426; XOP-NEXT: incq %rdx427; XOP-NEXT: cmpq %rdx, %r9428; XOP-NEXT: je .LBB0_9429; XOP-NEXT: .LBB0_6: # %for.body430; XOP-NEXT: # =>This Inner Loop Header: Depth=1431; XOP-NEXT: cmpb $0, (%rsi,%rdx)432; XOP-NEXT: movl %eax, %ecx433; XOP-NEXT: je .LBB0_8434; XOP-NEXT: # %bb.7: # %for.body435; XOP-NEXT: # in Loop: Header=BB0_6 Depth=1436; XOP-NEXT: movl %r8d, %ecx437; XOP-NEXT: jmp .LBB0_8438entry:439 %cmp12 = icmp sgt i32 %count, 0440 br i1 %cmp12, label %for.body.preheader, label %for.cond.cleanup441 442for.body.preheader:443 %wide.trip.count = zext i32 %count to i64444 %min.iters.check = icmp ult i32 %count, 32445 br i1 %min.iters.check, label %for.body.preheader40, label %vector.ph446 447for.body.preheader40:448 %indvars.iv.ph = phi i64 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]449 br label %for.body450 451vector.ph:452 %n.vec = and i64 %wide.trip.count, 4294967264453 %broadcast.splatinsert20 = insertelement <8 x i32> poison, i32 %amt0, i32 0454 %broadcast.splat21 = shufflevector <8 x i32> %broadcast.splatinsert20, <8 x i32> poison, <8 x i32> zeroinitializer455 %broadcast.splatinsert22 = insertelement <8 x i32> poison, i32 %amt1, i32 0456 %broadcast.splat23 = shufflevector <8 x i32> %broadcast.splatinsert22, <8 x i32> poison, <8 x i32> zeroinitializer457 %broadcast.splatinsert24 = insertelement <8 x i32> poison, i32 %amt0, i32 0458 %broadcast.splat25 = shufflevector <8 x i32> %broadcast.splatinsert24, <8 x i32> poison, <8 x i32> zeroinitializer459 %broadcast.splatinsert26 = insertelement <8 x i32> poison, i32 %amt1, i32 0460 %broadcast.splat27 = shufflevector <8 x i32> %broadcast.splatinsert26, <8 x i32> poison, <8 x i32> zeroinitializer461 %broadcast.splatinsert28 = insertelement <8 x i32> poison, i32 %amt0, i32 0462 %broadcast.splat29 = shufflevector <8 x i32> %broadcast.splatinsert28, <8 x i32> poison, <8 x i32> zeroinitializer463 %broadcast.splatinsert30 = insertelement <8 x i32> poison, i32 %amt1, i32 0464 %broadcast.splat31 = shufflevector <8 x i32> %broadcast.splatinsert30, <8 x i32> poison, <8 x i32> zeroinitializer465 %broadcast.splatinsert32 = insertelement <8 x i32> poison, i32 %amt0, i32 0466 %broadcast.splat33 = shufflevector <8 x i32> %broadcast.splatinsert32, <8 x i32> poison, <8 x i32> zeroinitializer467 %broadcast.splatinsert34 = insertelement <8 x i32> poison, i32 %amt1, i32 0468 %broadcast.splat35 = shufflevector <8 x i32> %broadcast.splatinsert34, <8 x i32> poison, <8 x i32> zeroinitializer469 br label %vector.body470 471vector.body:472 %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]473 %0 = getelementptr inbounds i8, ptr %control, i64 %index474 %wide.load = load <8 x i8>, ptr %0, align 1475 %1 = getelementptr inbounds i8, ptr %0, i64 8476 %wide.load17 = load <8 x i8>, ptr %1, align 1477 %2 = getelementptr inbounds i8, ptr %0, i64 16478 %wide.load18 = load <8 x i8>, ptr %2, align 1479 %3 = getelementptr inbounds i8, ptr %0, i64 24480 %wide.load19 = load <8 x i8>, ptr %3, align 1481 %4 = icmp eq <8 x i8> %wide.load, zeroinitializer482 %5 = icmp eq <8 x i8> %wide.load17, zeroinitializer483 %6 = icmp eq <8 x i8> %wide.load18, zeroinitializer484 %7 = icmp eq <8 x i8> %wide.load19, zeroinitializer485 %8 = select <8 x i1> %4, <8 x i32> %broadcast.splat21, <8 x i32> %broadcast.splat23486 %9 = select <8 x i1> %5, <8 x i32> %broadcast.splat25, <8 x i32> %broadcast.splat27487 %10 = select <8 x i1> %6, <8 x i32> %broadcast.splat29, <8 x i32> %broadcast.splat31488 %11 = select <8 x i1> %7, <8 x i32> %broadcast.splat33, <8 x i32> %broadcast.splat35489 %12 = getelementptr inbounds i32, ptr %arr, i64 %index490 %wide.load36 = load <8 x i32>, ptr %12, align 4491 %13 = getelementptr inbounds i32, ptr %12, i64 8492 %wide.load37 = load <8 x i32>, ptr %13, align 4493 %14 = getelementptr inbounds i32, ptr %12, i64 16494 %wide.load38 = load <8 x i32>, ptr %14, align 4495 %15 = getelementptr inbounds i32, ptr %12, i64 24496 %wide.load39 = load <8 x i32>, ptr %15, align 4497 %16 = shl <8 x i32> %wide.load36, %8498 %17 = shl <8 x i32> %wide.load37, %9499 %18 = shl <8 x i32> %wide.load38, %10500 %19 = shl <8 x i32> %wide.load39, %11501 store <8 x i32> %16, ptr %12, align 4502 store <8 x i32> %17, ptr %13, align 4503 store <8 x i32> %18, ptr %14, align 4504 store <8 x i32> %19, ptr %15, align 4505 %index.next = add i64 %index, 32506 %20 = icmp eq i64 %index.next, %n.vec507 br i1 %20, label %middle.block, label %vector.body508 509middle.block:510 %cmp.n = icmp eq i64 %n.vec, %wide.trip.count511 br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader40512 513for.cond.cleanup:514 ret void515 516for.body:517 %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ %indvars.iv.ph, %for.body.preheader40 ]518 %arrayidx = getelementptr inbounds i8, ptr %control, i64 %indvars.iv519 %21 = load i8, ptr %arrayidx, align 1520 %tobool = icmp eq i8 %21, 0521 %cond = select i1 %tobool, i32 %amt0, i32 %amt1522 %arrayidx2 = getelementptr inbounds i32, ptr %arr, i64 %indvars.iv523 %22 = load i32, ptr %arrayidx2, align 4524 %shl = shl i32 %22, %cond525 store i32 %shl, ptr %arrayidx2, align 4526 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1527 %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count528 br i1 %exitcond, label %for.cond.cleanup, label %for.body529}530 531define void @vector_variable_shift_left_loop_simpler(ptr nocapture %arr, ptr nocapture readonly %control, i32 %count, i32 %amt0, i32 %amt1, i32 %x) nounwind {532; SSE-LABEL: vector_variable_shift_left_loop_simpler:533; SSE: # %bb.0: # %entry534; SSE-NEXT: testl %edx, %edx535; SSE-NEXT: jle .LBB1_3536; SSE-NEXT: # %bb.1: # %vector.ph537; SSE-NEXT: movl %edx, %eax538; SSE-NEXT: andl $-4, %eax539; SSE-NEXT: movd %ecx, %xmm0540; SSE-NEXT: movd %r8d, %xmm3541; SSE-NEXT: movd %r9d, %xmm1542; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]543; SSE-NEXT: xorl %ecx, %ecx544; SSE-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero545; SSE-NEXT: movdqa %xmm1, %xmm2546; SSE-NEXT: pslld %xmm0, %xmm2547; SSE-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm3[0],zero,xmm3[1],zero548; SSE-NEXT: pslld %xmm0, %xmm1549; SSE-NEXT: pxor %xmm3, %xmm3550; SSE-NEXT: .p2align 4551; SSE-NEXT: .LBB1_2: # %vector.body552; SSE-NEXT: # =>This Inner Loop Header: Depth=1553; SSE-NEXT: pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero554; SSE-NEXT: pcmpeqd %xmm3, %xmm0555; SSE-NEXT: movdqa %xmm1, %xmm4556; SSE-NEXT: blendvps %xmm0, %xmm2, %xmm4557; SSE-NEXT: movups %xmm4, (%rdi,%rcx,4)558; SSE-NEXT: addq $4, %rcx559; SSE-NEXT: cmpq %rcx, %rax560; SSE-NEXT: jne .LBB1_2561; SSE-NEXT: .LBB1_3: # %exit562; SSE-NEXT: retq563;564; AVX1-LABEL: vector_variable_shift_left_loop_simpler:565; AVX1: # %bb.0: # %entry566; AVX1-NEXT: testl %edx, %edx567; AVX1-NEXT: jle .LBB1_3568; AVX1-NEXT: # %bb.1: # %vector.ph569; AVX1-NEXT: movl %edx, %eax570; AVX1-NEXT: andl $-4, %eax571; AVX1-NEXT: vmovd %ecx, %xmm0572; AVX1-NEXT: vmovd %r8d, %xmm1573; AVX1-NEXT: vmovd %r9d, %xmm2574; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]575; AVX1-NEXT: xorl %ecx, %ecx576; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero577; AVX1-NEXT: vpslld %xmm0, %xmm2, %xmm0578; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero579; AVX1-NEXT: vpslld %xmm1, %xmm2, %xmm1580; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2581; AVX1-NEXT: .p2align 4582; AVX1-NEXT: .LBB1_2: # %vector.body583; AVX1-NEXT: # =>This Inner Loop Header: Depth=1584; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm3 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero585; AVX1-NEXT: vpcmpeqd %xmm2, %xmm3, %xmm3586; AVX1-NEXT: vblendvps %xmm3, %xmm0, %xmm1, %xmm3587; AVX1-NEXT: vmovups %xmm3, (%rdi,%rcx,4)588; AVX1-NEXT: addq $4, %rcx589; AVX1-NEXT: cmpq %rcx, %rax590; AVX1-NEXT: jne .LBB1_2591; AVX1-NEXT: .LBB1_3: # %exit592; AVX1-NEXT: retq593;594; AVX2-LABEL: vector_variable_shift_left_loop_simpler:595; AVX2: # %bb.0: # %entry596; AVX2-NEXT: testl %edx, %edx597; AVX2-NEXT: jle .LBB1_3598; AVX2-NEXT: # %bb.1: # %vector.ph599; AVX2-NEXT: movl %edx, %eax600; AVX2-NEXT: andl $-4, %eax601; AVX2-NEXT: vmovd %ecx, %xmm0602; AVX2-NEXT: vpbroadcastd %xmm0, %xmm0603; AVX2-NEXT: vmovd %r8d, %xmm1604; AVX2-NEXT: vpbroadcastd %xmm1, %xmm1605; AVX2-NEXT: vmovd %r9d, %xmm2606; AVX2-NEXT: vpbroadcastd %xmm2, %xmm2607; AVX2-NEXT: xorl %ecx, %ecx608; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3609; AVX2-NEXT: .p2align 4610; AVX2-NEXT: .LBB1_2: # %vector.body611; AVX2-NEXT: # =>This Inner Loop Header: Depth=1612; AVX2-NEXT: vpmovzxbd {{.*#+}} xmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero613; AVX2-NEXT: vpcmpeqd %xmm3, %xmm4, %xmm4614; AVX2-NEXT: vblendvps %xmm4, %xmm0, %xmm1, %xmm4615; AVX2-NEXT: vpsllvd %xmm4, %xmm2, %xmm4616; AVX2-NEXT: vmovdqu %xmm4, (%rdi,%rcx,4)617; AVX2-NEXT: addq $4, %rcx618; AVX2-NEXT: cmpq %rcx, %rax619; AVX2-NEXT: jne .LBB1_2620; AVX2-NEXT: .LBB1_3: # %exit621; AVX2-NEXT: retq622;623; XOP-LABEL: vector_variable_shift_left_loop_simpler:624; XOP: # %bb.0: # %entry625; XOP-NEXT: testl %edx, %edx626; XOP-NEXT: jle .LBB1_3627; XOP-NEXT: # %bb.1: # %vector.ph628; XOP-NEXT: movl %edx, %eax629; XOP-NEXT: andl $-4, %eax630; XOP-NEXT: vmovd %ecx, %xmm0631; XOP-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]632; XOP-NEXT: vmovd %r8d, %xmm1633; XOP-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]634; XOP-NEXT: vmovd %r9d, %xmm2635; XOP-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]636; XOP-NEXT: xorl %ecx, %ecx637; XOP-NEXT: vpxor %xmm3, %xmm3, %xmm3638; XOP-NEXT: .p2align 4639; XOP-NEXT: .LBB1_2: # %vector.body640; XOP-NEXT: # =>This Inner Loop Header: Depth=1641; XOP-NEXT: vpmovzxbd {{.*#+}} xmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero642; XOP-NEXT: vpcomeqd %xmm3, %xmm4, %xmm4643; XOP-NEXT: vblendvps %xmm4, %xmm0, %xmm1, %xmm4644; XOP-NEXT: vpshld %xmm4, %xmm2, %xmm4645; XOP-NEXT: vmovdqu %xmm4, (%rdi,%rcx,4)646; XOP-NEXT: addq $4, %rcx647; XOP-NEXT: cmpq %rcx, %rax648; XOP-NEXT: jne .LBB1_2649; XOP-NEXT: .LBB1_3: # %exit650; XOP-NEXT: retq651entry:652 %cmp16 = icmp sgt i32 %count, 0653 %wide.trip.count = zext i32 %count to i64654 br i1 %cmp16, label %vector.ph, label %exit655 656vector.ph:657 %n.vec = and i64 %wide.trip.count, 4294967292658 %splatinsert18 = insertelement <4 x i32> poison, i32 %amt0, i32 0659 %splat1 = shufflevector <4 x i32> %splatinsert18, <4 x i32> poison, <4 x i32> zeroinitializer660 %splatinsert20 = insertelement <4 x i32> poison, i32 %amt1, i32 0661 %splat2 = shufflevector <4 x i32> %splatinsert20, <4 x i32> poison, <4 x i32> zeroinitializer662 %splatinsert22 = insertelement <4 x i32> poison, i32 %x, i32 0663 %splat3 = shufflevector <4 x i32> %splatinsert22, <4 x i32> poison, <4 x i32> zeroinitializer664 br label %vector.body665 666vector.body:667 %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]668 %0 = getelementptr inbounds i8, ptr %control, i64 %index669 %wide.load = load <4 x i8>, ptr %0, align 1670 %1 = icmp eq <4 x i8> %wide.load, zeroinitializer671 %2 = select <4 x i1> %1, <4 x i32> %splat1, <4 x i32> %splat2672 %3 = shl <4 x i32> %splat3, %2673 %4 = getelementptr inbounds i32, ptr %arr, i64 %index674 store <4 x i32> %3, ptr %4, align 4675 %index.next = add i64 %index, 4676 %5 = icmp eq i64 %index.next, %n.vec677 br i1 %5, label %exit, label %vector.body678 679exit:680 ret void681}682