762 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512vl -mattr=+avx512dq < %s | FileCheck %s --check-prefix=WIDEN_SKX3; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f < %s | FileCheck %s --check-prefix=WIDEN_KNL4; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake < %s | FileCheck %s --check-prefix=WIDEN_AVX25 6define <2 x double> @test_gather_v2i32_index(ptr %base, <2 x i32> %ind, <2 x i1> %mask, <2 x double> %src0) {7; WIDEN_SKX-LABEL: test_gather_v2i32_index:8; WIDEN_SKX: # %bb.0:9; WIDEN_SKX-NEXT: vpsllq $63, %xmm1, %xmm110; WIDEN_SKX-NEXT: vpmovq2m %xmm1, %k011; WIDEN_SKX-NEXT: vpbroadcastq %rdi, %xmm112; WIDEN_SKX-NEXT: vpmovsxdq %xmm0, %xmm013; WIDEN_SKX-NEXT: vpsllq $3, %xmm0, %xmm014; WIDEN_SKX-NEXT: vpaddq %xmm0, %xmm1, %xmm015; WIDEN_SKX-NEXT: kmovw %k0, %eax16; WIDEN_SKX-NEXT: testb $1, %al17; WIDEN_SKX-NEXT: jne .LBB0_118; WIDEN_SKX-NEXT: # %bb.2: # %else19; WIDEN_SKX-NEXT: testb $2, %al20; WIDEN_SKX-NEXT: jne .LBB0_321; WIDEN_SKX-NEXT: .LBB0_4: # %else222; WIDEN_SKX-NEXT: vmovaps %xmm2, %xmm023; WIDEN_SKX-NEXT: retq24; WIDEN_SKX-NEXT: .LBB0_1: # %cond.load25; WIDEN_SKX-NEXT: vmovq %xmm0, %rcx26; WIDEN_SKX-NEXT: vmovlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]27; WIDEN_SKX-NEXT: testb $2, %al28; WIDEN_SKX-NEXT: je .LBB0_429; WIDEN_SKX-NEXT: .LBB0_3: # %cond.load130; WIDEN_SKX-NEXT: vpextrq $1, %xmm0, %rax31; WIDEN_SKX-NEXT: vmovhps {{.*#+}} xmm2 = xmm2[0,1],mem[0,1]32; WIDEN_SKX-NEXT: vmovaps %xmm2, %xmm033; WIDEN_SKX-NEXT: retq34;35; WIDEN_KNL-LABEL: test_gather_v2i32_index:36; WIDEN_KNL: # %bb.0:37; WIDEN_KNL-NEXT: vpsllq $63, %xmm1, %xmm138; WIDEN_KNL-NEXT: vptestmq %zmm1, %zmm1, %k039; WIDEN_KNL-NEXT: vpmovsxdq %xmm0, %xmm040; WIDEN_KNL-NEXT: vpsllq $3, %xmm0, %xmm041; WIDEN_KNL-NEXT: vmovq %rdi, %xmm142; WIDEN_KNL-NEXT: vpbroadcastq %xmm1, %xmm143; WIDEN_KNL-NEXT: vpaddq %xmm0, %xmm1, %xmm044; WIDEN_KNL-NEXT: kmovw %k0, %eax45; WIDEN_KNL-NEXT: testb $1, %al46; WIDEN_KNL-NEXT: jne .LBB0_147; WIDEN_KNL-NEXT: # %bb.2: # %else48; WIDEN_KNL-NEXT: testb $2, %al49; WIDEN_KNL-NEXT: jne .LBB0_350; WIDEN_KNL-NEXT: .LBB0_4: # %else251; WIDEN_KNL-NEXT: vmovaps %xmm2, %xmm052; WIDEN_KNL-NEXT: vzeroupper53; WIDEN_KNL-NEXT: retq54; WIDEN_KNL-NEXT: .LBB0_1: # %cond.load55; WIDEN_KNL-NEXT: vmovq %xmm0, %rcx56; WIDEN_KNL-NEXT: vmovlps {{.*#+}} xmm2 = mem[0,1],xmm2[2,3]57; WIDEN_KNL-NEXT: testb $2, %al58; WIDEN_KNL-NEXT: je .LBB0_459; WIDEN_KNL-NEXT: .LBB0_3: # %cond.load160; WIDEN_KNL-NEXT: vpextrq $1, %xmm0, %rax61; WIDEN_KNL-NEXT: vmovhps {{.*#+}} xmm2 = xmm2[0,1],mem[0,1]62; WIDEN_KNL-NEXT: vmovaps %xmm2, %xmm063; WIDEN_KNL-NEXT: vzeroupper64; WIDEN_KNL-NEXT: retq65;66; WIDEN_AVX2-LABEL: test_gather_v2i32_index:67; WIDEN_AVX2: # %bb.0:68; WIDEN_AVX2-NEXT: vpsllq $63, %xmm1, %xmm169; WIDEN_AVX2-NEXT: vgatherdpd %xmm1, (%rdi,%xmm0,8), %xmm270; WIDEN_AVX2-NEXT: vmovapd %xmm2, %xmm071; WIDEN_AVX2-NEXT: retq72 %gep.random = getelementptr double, ptr %base, <2 x i32> %ind73 %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %gep.random, i32 4, <2 x i1> %mask, <2 x double> %src0)74 ret <2 x double> %res75}76 77define void @test_scatter_v2i32_index(<2 x double> %a1, ptr %base, <2 x i32> %ind, <2 x i1> %mask) {78; WIDEN_SKX-LABEL: test_scatter_v2i32_index:79; WIDEN_SKX: # %bb.0:80; WIDEN_SKX-NEXT: vpsllq $63, %xmm2, %xmm281; WIDEN_SKX-NEXT: vpmovq2m %xmm2, %k082; WIDEN_SKX-NEXT: vpbroadcastq %rdi, %xmm283; WIDEN_SKX-NEXT: vpmovsxdq %xmm1, %xmm184; WIDEN_SKX-NEXT: vpsllq $3, %xmm1, %xmm185; WIDEN_SKX-NEXT: vpaddq %xmm1, %xmm2, %xmm186; WIDEN_SKX-NEXT: kmovw %k0, %eax87; WIDEN_SKX-NEXT: testb $1, %al88; WIDEN_SKX-NEXT: jne .LBB1_189; WIDEN_SKX-NEXT: # %bb.2: # %else90; WIDEN_SKX-NEXT: testb $2, %al91; WIDEN_SKX-NEXT: jne .LBB1_392; WIDEN_SKX-NEXT: .LBB1_4: # %else293; WIDEN_SKX-NEXT: retq94; WIDEN_SKX-NEXT: .LBB1_1: # %cond.store95; WIDEN_SKX-NEXT: vmovq %xmm1, %rcx96; WIDEN_SKX-NEXT: vmovlps %xmm0, (%rcx)97; WIDEN_SKX-NEXT: testb $2, %al98; WIDEN_SKX-NEXT: je .LBB1_499; WIDEN_SKX-NEXT: .LBB1_3: # %cond.store1100; WIDEN_SKX-NEXT: vpextrq $1, %xmm1, %rax101; WIDEN_SKX-NEXT: vmovhps %xmm0, (%rax)102; WIDEN_SKX-NEXT: retq103;104; WIDEN_KNL-LABEL: test_scatter_v2i32_index:105; WIDEN_KNL: # %bb.0:106; WIDEN_KNL-NEXT: vpsllq $63, %xmm2, %xmm2107; WIDEN_KNL-NEXT: vptestmq %zmm2, %zmm2, %k0108; WIDEN_KNL-NEXT: vpmovsxdq %xmm1, %xmm1109; WIDEN_KNL-NEXT: vpsllq $3, %xmm1, %xmm1110; WIDEN_KNL-NEXT: vmovq %rdi, %xmm2111; WIDEN_KNL-NEXT: vpbroadcastq %xmm2, %xmm2112; WIDEN_KNL-NEXT: vpaddq %xmm1, %xmm2, %xmm1113; WIDEN_KNL-NEXT: kmovw %k0, %eax114; WIDEN_KNL-NEXT: testb $1, %al115; WIDEN_KNL-NEXT: jne .LBB1_1116; WIDEN_KNL-NEXT: # %bb.2: # %else117; WIDEN_KNL-NEXT: testb $2, %al118; WIDEN_KNL-NEXT: jne .LBB1_3119; WIDEN_KNL-NEXT: .LBB1_4: # %else2120; WIDEN_KNL-NEXT: vzeroupper121; WIDEN_KNL-NEXT: retq122; WIDEN_KNL-NEXT: .LBB1_1: # %cond.store123; WIDEN_KNL-NEXT: vmovq %xmm1, %rcx124; WIDEN_KNL-NEXT: vmovlps %xmm0, (%rcx)125; WIDEN_KNL-NEXT: testb $2, %al126; WIDEN_KNL-NEXT: je .LBB1_4127; WIDEN_KNL-NEXT: .LBB1_3: # %cond.store1128; WIDEN_KNL-NEXT: vpextrq $1, %xmm1, %rax129; WIDEN_KNL-NEXT: vmovhps %xmm0, (%rax)130; WIDEN_KNL-NEXT: vzeroupper131; WIDEN_KNL-NEXT: retq132;133; WIDEN_AVX2-LABEL: test_scatter_v2i32_index:134; WIDEN_AVX2: # %bb.0:135; WIDEN_AVX2-NEXT: vpmovsxdq %xmm1, %xmm1136; WIDEN_AVX2-NEXT: vpsllq $3, %xmm1, %xmm1137; WIDEN_AVX2-NEXT: vmovq %rdi, %xmm3138; WIDEN_AVX2-NEXT: vpbroadcastq %xmm3, %xmm3139; WIDEN_AVX2-NEXT: vpaddq %xmm1, %xmm3, %xmm1140; WIDEN_AVX2-NEXT: vpsllq $63, %xmm2, %xmm2141; WIDEN_AVX2-NEXT: vmovmskpd %xmm2, %eax142; WIDEN_AVX2-NEXT: testb $1, %al143; WIDEN_AVX2-NEXT: jne .LBB1_1144; WIDEN_AVX2-NEXT: # %bb.2: # %else145; WIDEN_AVX2-NEXT: testb $2, %al146; WIDEN_AVX2-NEXT: jne .LBB1_3147; WIDEN_AVX2-NEXT: .LBB1_4: # %else2148; WIDEN_AVX2-NEXT: retq149; WIDEN_AVX2-NEXT: .LBB1_1: # %cond.store150; WIDEN_AVX2-NEXT: vmovq %xmm1, %rcx151; WIDEN_AVX2-NEXT: vmovlps %xmm0, (%rcx)152; WIDEN_AVX2-NEXT: testb $2, %al153; WIDEN_AVX2-NEXT: je .LBB1_4154; WIDEN_AVX2-NEXT: .LBB1_3: # %cond.store1155; WIDEN_AVX2-NEXT: vpextrq $1, %xmm1, %rax156; WIDEN_AVX2-NEXT: vmovhps %xmm0, (%rax)157; WIDEN_AVX2-NEXT: retq158 %gep = getelementptr double, ptr%base, <2 x i32> %ind159 call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> %a1, <2 x ptr> %gep, i32 4, <2 x i1> %mask)160 ret void161}162 163define <2 x i32> @test_gather_v2i32_data(<2 x ptr> %ptr, <2 x i1> %mask, <2 x i32> %src0) {164; WIDEN_SKX-LABEL: test_gather_v2i32_data:165; WIDEN_SKX: # %bb.0:166; WIDEN_SKX-NEXT: vpsllq $63, %xmm1, %xmm1167; WIDEN_SKX-NEXT: vpmovq2m %xmm1, %k0168; WIDEN_SKX-NEXT: kmovw %k0, %eax169; WIDEN_SKX-NEXT: testb $1, %al170; WIDEN_SKX-NEXT: jne .LBB2_1171; WIDEN_SKX-NEXT: # %bb.2: # %else172; WIDEN_SKX-NEXT: testb $2, %al173; WIDEN_SKX-NEXT: jne .LBB2_3174; WIDEN_SKX-NEXT: .LBB2_4: # %else2175; WIDEN_SKX-NEXT: vmovdqa %xmm2, %xmm0176; WIDEN_SKX-NEXT: retq177; WIDEN_SKX-NEXT: .LBB2_1: # %cond.load178; WIDEN_SKX-NEXT: vmovq %xmm0, %rcx179; WIDEN_SKX-NEXT: vpinsrd $0, (%rcx), %xmm2, %xmm2180; WIDEN_SKX-NEXT: testb $2, %al181; WIDEN_SKX-NEXT: je .LBB2_4182; WIDEN_SKX-NEXT: .LBB2_3: # %cond.load1183; WIDEN_SKX-NEXT: vpextrq $1, %xmm0, %rax184; WIDEN_SKX-NEXT: vpinsrd $1, (%rax), %xmm2, %xmm2185; WIDEN_SKX-NEXT: vmovdqa %xmm2, %xmm0186; WIDEN_SKX-NEXT: retq187;188; WIDEN_KNL-LABEL: test_gather_v2i32_data:189; WIDEN_KNL: # %bb.0:190; WIDEN_KNL-NEXT: vpsllq $63, %xmm1, %xmm1191; WIDEN_KNL-NEXT: vptestmq %zmm1, %zmm1, %k0192; WIDEN_KNL-NEXT: kmovw %k0, %eax193; WIDEN_KNL-NEXT: testb $1, %al194; WIDEN_KNL-NEXT: jne .LBB2_1195; WIDEN_KNL-NEXT: # %bb.2: # %else196; WIDEN_KNL-NEXT: testb $2, %al197; WIDEN_KNL-NEXT: jne .LBB2_3198; WIDEN_KNL-NEXT: .LBB2_4: # %else2199; WIDEN_KNL-NEXT: vmovdqa %xmm2, %xmm0200; WIDEN_KNL-NEXT: vzeroupper201; WIDEN_KNL-NEXT: retq202; WIDEN_KNL-NEXT: .LBB2_1: # %cond.load203; WIDEN_KNL-NEXT: vmovq %xmm0, %rcx204; WIDEN_KNL-NEXT: vpinsrd $0, (%rcx), %xmm2, %xmm2205; WIDEN_KNL-NEXT: testb $2, %al206; WIDEN_KNL-NEXT: je .LBB2_4207; WIDEN_KNL-NEXT: .LBB2_3: # %cond.load1208; WIDEN_KNL-NEXT: vpextrq $1, %xmm0, %rax209; WIDEN_KNL-NEXT: vpinsrd $1, (%rax), %xmm2, %xmm2210; WIDEN_KNL-NEXT: vmovdqa %xmm2, %xmm0211; WIDEN_KNL-NEXT: vzeroupper212; WIDEN_KNL-NEXT: retq213;214; WIDEN_AVX2-LABEL: test_gather_v2i32_data:215; WIDEN_AVX2: # %bb.0:216; WIDEN_AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]217; WIDEN_AVX2-NEXT: vpslld $31, %xmm1, %xmm1218; WIDEN_AVX2-NEXT: vpgatherqd %xmm1, (,%xmm0), %xmm2219; WIDEN_AVX2-NEXT: vmovdqa %xmm2, %xmm0220; WIDEN_AVX2-NEXT: retq221 %res = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> %ptr, i32 4, <2 x i1> %mask, <2 x i32> %src0)222 ret <2 x i32>%res223}224 225define void @test_scatter_v2i32_data(<2 x i32>%a1, <2 x ptr> %ptr, <2 x i1>%mask) {226; WIDEN_SKX-LABEL: test_scatter_v2i32_data:227; WIDEN_SKX: # %bb.0:228; WIDEN_SKX-NEXT: vpsllq $63, %xmm2, %xmm2229; WIDEN_SKX-NEXT: vpmovq2m %xmm2, %k0230; WIDEN_SKX-NEXT: kmovw %k0, %eax231; WIDEN_SKX-NEXT: testb $1, %al232; WIDEN_SKX-NEXT: jne .LBB3_1233; WIDEN_SKX-NEXT: # %bb.2: # %else234; WIDEN_SKX-NEXT: testb $2, %al235; WIDEN_SKX-NEXT: jne .LBB3_3236; WIDEN_SKX-NEXT: .LBB3_4: # %else2237; WIDEN_SKX-NEXT: retq238; WIDEN_SKX-NEXT: .LBB3_1: # %cond.store239; WIDEN_SKX-NEXT: vmovq %xmm1, %rcx240; WIDEN_SKX-NEXT: vmovss %xmm0, (%rcx)241; WIDEN_SKX-NEXT: testb $2, %al242; WIDEN_SKX-NEXT: je .LBB3_4243; WIDEN_SKX-NEXT: .LBB3_3: # %cond.store1244; WIDEN_SKX-NEXT: vpextrq $1, %xmm1, %rax245; WIDEN_SKX-NEXT: vextractps $1, %xmm0, (%rax)246; WIDEN_SKX-NEXT: retq247;248; WIDEN_KNL-LABEL: test_scatter_v2i32_data:249; WIDEN_KNL: # %bb.0:250; WIDEN_KNL-NEXT: vpsllq $63, %xmm2, %xmm2251; WIDEN_KNL-NEXT: vptestmq %zmm2, %zmm2, %k0252; WIDEN_KNL-NEXT: kmovw %k0, %eax253; WIDEN_KNL-NEXT: testb $1, %al254; WIDEN_KNL-NEXT: jne .LBB3_1255; WIDEN_KNL-NEXT: # %bb.2: # %else256; WIDEN_KNL-NEXT: testb $2, %al257; WIDEN_KNL-NEXT: jne .LBB3_3258; WIDEN_KNL-NEXT: .LBB3_4: # %else2259; WIDEN_KNL-NEXT: vzeroupper260; WIDEN_KNL-NEXT: retq261; WIDEN_KNL-NEXT: .LBB3_1: # %cond.store262; WIDEN_KNL-NEXT: vmovq %xmm1, %rcx263; WIDEN_KNL-NEXT: vmovss %xmm0, (%rcx)264; WIDEN_KNL-NEXT: testb $2, %al265; WIDEN_KNL-NEXT: je .LBB3_4266; WIDEN_KNL-NEXT: .LBB3_3: # %cond.store1267; WIDEN_KNL-NEXT: vpextrq $1, %xmm1, %rax268; WIDEN_KNL-NEXT: vextractps $1, %xmm0, (%rax)269; WIDEN_KNL-NEXT: vzeroupper270; WIDEN_KNL-NEXT: retq271;272; WIDEN_AVX2-LABEL: test_scatter_v2i32_data:273; WIDEN_AVX2: # %bb.0:274; WIDEN_AVX2-NEXT: vpsllq $63, %xmm2, %xmm2275; WIDEN_AVX2-NEXT: vmovmskpd %xmm2, %eax276; WIDEN_AVX2-NEXT: testb $1, %al277; WIDEN_AVX2-NEXT: jne .LBB3_1278; WIDEN_AVX2-NEXT: # %bb.2: # %else279; WIDEN_AVX2-NEXT: testb $2, %al280; WIDEN_AVX2-NEXT: jne .LBB3_3281; WIDEN_AVX2-NEXT: .LBB3_4: # %else2282; WIDEN_AVX2-NEXT: retq283; WIDEN_AVX2-NEXT: .LBB3_1: # %cond.store284; WIDEN_AVX2-NEXT: vmovq %xmm1, %rcx285; WIDEN_AVX2-NEXT: vmovss %xmm0, (%rcx)286; WIDEN_AVX2-NEXT: testb $2, %al287; WIDEN_AVX2-NEXT: je .LBB3_4288; WIDEN_AVX2-NEXT: .LBB3_3: # %cond.store1289; WIDEN_AVX2-NEXT: vpextrq $1, %xmm1, %rax290; WIDEN_AVX2-NEXT: vextractps $1, %xmm0, (%rax)291; WIDEN_AVX2-NEXT: retq292 call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> %a1, <2 x ptr> %ptr, i32 4, <2 x i1> %mask)293 ret void294}295 296define <2 x i32> @test_gather_v2i32_data_index(ptr %base, <2 x i32> %ind, <2 x i1> %mask, <2 x i32> %src0) {297; WIDEN_SKX-LABEL: test_gather_v2i32_data_index:298; WIDEN_SKX: # %bb.0:299; WIDEN_SKX-NEXT: vpsllq $63, %xmm1, %xmm1300; WIDEN_SKX-NEXT: vpmovq2m %xmm1, %k0301; WIDEN_SKX-NEXT: vpbroadcastq %rdi, %xmm1302; WIDEN_SKX-NEXT: vpmovsxdq %xmm0, %xmm0303; WIDEN_SKX-NEXT: vpsllq $2, %xmm0, %xmm0304; WIDEN_SKX-NEXT: vpaddq %xmm0, %xmm1, %xmm0305; WIDEN_SKX-NEXT: kmovw %k0, %eax306; WIDEN_SKX-NEXT: testb $1, %al307; WIDEN_SKX-NEXT: jne .LBB4_1308; WIDEN_SKX-NEXT: # %bb.2: # %else309; WIDEN_SKX-NEXT: testb $2, %al310; WIDEN_SKX-NEXT: jne .LBB4_3311; WIDEN_SKX-NEXT: .LBB4_4: # %else2312; WIDEN_SKX-NEXT: vmovdqa %xmm2, %xmm0313; WIDEN_SKX-NEXT: retq314; WIDEN_SKX-NEXT: .LBB4_1: # %cond.load315; WIDEN_SKX-NEXT: vmovq %xmm0, %rcx316; WIDEN_SKX-NEXT: vpinsrd $0, (%rcx), %xmm2, %xmm2317; WIDEN_SKX-NEXT: testb $2, %al318; WIDEN_SKX-NEXT: je .LBB4_4319; WIDEN_SKX-NEXT: .LBB4_3: # %cond.load1320; WIDEN_SKX-NEXT: vpextrq $1, %xmm0, %rax321; WIDEN_SKX-NEXT: vpinsrd $1, (%rax), %xmm2, %xmm2322; WIDEN_SKX-NEXT: vmovdqa %xmm2, %xmm0323; WIDEN_SKX-NEXT: retq324;325; WIDEN_KNL-LABEL: test_gather_v2i32_data_index:326; WIDEN_KNL: # %bb.0:327; WIDEN_KNL-NEXT: vpsllq $63, %xmm1, %xmm1328; WIDEN_KNL-NEXT: vptestmq %zmm1, %zmm1, %k0329; WIDEN_KNL-NEXT: vpmovsxdq %xmm0, %xmm0330; WIDEN_KNL-NEXT: vpsllq $2, %xmm0, %xmm0331; WIDEN_KNL-NEXT: vmovq %rdi, %xmm1332; WIDEN_KNL-NEXT: vpbroadcastq %xmm1, %xmm1333; WIDEN_KNL-NEXT: vpaddq %xmm0, %xmm1, %xmm0334; WIDEN_KNL-NEXT: kmovw %k0, %eax335; WIDEN_KNL-NEXT: testb $1, %al336; WIDEN_KNL-NEXT: jne .LBB4_1337; WIDEN_KNL-NEXT: # %bb.2: # %else338; WIDEN_KNL-NEXT: testb $2, %al339; WIDEN_KNL-NEXT: jne .LBB4_3340; WIDEN_KNL-NEXT: .LBB4_4: # %else2341; WIDEN_KNL-NEXT: vmovdqa %xmm2, %xmm0342; WIDEN_KNL-NEXT: vzeroupper343; WIDEN_KNL-NEXT: retq344; WIDEN_KNL-NEXT: .LBB4_1: # %cond.load345; WIDEN_KNL-NEXT: vmovq %xmm0, %rcx346; WIDEN_KNL-NEXT: vpinsrd $0, (%rcx), %xmm2, %xmm2347; WIDEN_KNL-NEXT: testb $2, %al348; WIDEN_KNL-NEXT: je .LBB4_4349; WIDEN_KNL-NEXT: .LBB4_3: # %cond.load1350; WIDEN_KNL-NEXT: vpextrq $1, %xmm0, %rax351; WIDEN_KNL-NEXT: vpinsrd $1, (%rax), %xmm2, %xmm2352; WIDEN_KNL-NEXT: vmovdqa %xmm2, %xmm0353; WIDEN_KNL-NEXT: vzeroupper354; WIDEN_KNL-NEXT: retq355;356; WIDEN_AVX2-LABEL: test_gather_v2i32_data_index:357; WIDEN_AVX2: # %bb.0:358; WIDEN_AVX2-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,2],zero,zero359; WIDEN_AVX2-NEXT: vpslld $31, %xmm1, %xmm1360; WIDEN_AVX2-NEXT: vpgatherdd %xmm1, (%rdi,%xmm0,4), %xmm2361; WIDEN_AVX2-NEXT: vmovdqa %xmm2, %xmm0362; WIDEN_AVX2-NEXT: retq363 %gep.random = getelementptr i32, ptr %base, <2 x i32> %ind364 %res = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> %gep.random, i32 4, <2 x i1> %mask, <2 x i32> %src0)365 ret <2 x i32> %res366}367 368define void @test_scatter_v2i32_data_index(<2 x i32> %a1, ptr %base, <2 x i32> %ind, <2 x i1> %mask) {369; WIDEN_SKX-LABEL: test_scatter_v2i32_data_index:370; WIDEN_SKX: # %bb.0:371; WIDEN_SKX-NEXT: vpsllq $63, %xmm2, %xmm2372; WIDEN_SKX-NEXT: vpmovq2m %xmm2, %k0373; WIDEN_SKX-NEXT: vpbroadcastq %rdi, %xmm2374; WIDEN_SKX-NEXT: vpmovsxdq %xmm1, %xmm1375; WIDEN_SKX-NEXT: vpsllq $2, %xmm1, %xmm1376; WIDEN_SKX-NEXT: vpaddq %xmm1, %xmm2, %xmm1377; WIDEN_SKX-NEXT: kmovw %k0, %eax378; WIDEN_SKX-NEXT: testb $1, %al379; WIDEN_SKX-NEXT: jne .LBB5_1380; WIDEN_SKX-NEXT: # %bb.2: # %else381; WIDEN_SKX-NEXT: testb $2, %al382; WIDEN_SKX-NEXT: jne .LBB5_3383; WIDEN_SKX-NEXT: .LBB5_4: # %else2384; WIDEN_SKX-NEXT: retq385; WIDEN_SKX-NEXT: .LBB5_1: # %cond.store386; WIDEN_SKX-NEXT: vmovq %xmm1, %rcx387; WIDEN_SKX-NEXT: vmovss %xmm0, (%rcx)388; WIDEN_SKX-NEXT: testb $2, %al389; WIDEN_SKX-NEXT: je .LBB5_4390; WIDEN_SKX-NEXT: .LBB5_3: # %cond.store1391; WIDEN_SKX-NEXT: vpextrq $1, %xmm1, %rax392; WIDEN_SKX-NEXT: vextractps $1, %xmm0, (%rax)393; WIDEN_SKX-NEXT: retq394;395; WIDEN_KNL-LABEL: test_scatter_v2i32_data_index:396; WIDEN_KNL: # %bb.0:397; WIDEN_KNL-NEXT: vpsllq $63, %xmm2, %xmm2398; WIDEN_KNL-NEXT: vptestmq %zmm2, %zmm2, %k0399; WIDEN_KNL-NEXT: vpmovsxdq %xmm1, %xmm1400; WIDEN_KNL-NEXT: vpsllq $2, %xmm1, %xmm1401; WIDEN_KNL-NEXT: vmovq %rdi, %xmm2402; WIDEN_KNL-NEXT: vpbroadcastq %xmm2, %xmm2403; WIDEN_KNL-NEXT: vpaddq %xmm1, %xmm2, %xmm1404; WIDEN_KNL-NEXT: kmovw %k0, %eax405; WIDEN_KNL-NEXT: testb $1, %al406; WIDEN_KNL-NEXT: jne .LBB5_1407; WIDEN_KNL-NEXT: # %bb.2: # %else408; WIDEN_KNL-NEXT: testb $2, %al409; WIDEN_KNL-NEXT: jne .LBB5_3410; WIDEN_KNL-NEXT: .LBB5_4: # %else2411; WIDEN_KNL-NEXT: vzeroupper412; WIDEN_KNL-NEXT: retq413; WIDEN_KNL-NEXT: .LBB5_1: # %cond.store414; WIDEN_KNL-NEXT: vmovq %xmm1, %rcx415; WIDEN_KNL-NEXT: vmovss %xmm0, (%rcx)416; WIDEN_KNL-NEXT: testb $2, %al417; WIDEN_KNL-NEXT: je .LBB5_4418; WIDEN_KNL-NEXT: .LBB5_3: # %cond.store1419; WIDEN_KNL-NEXT: vpextrq $1, %xmm1, %rax420; WIDEN_KNL-NEXT: vextractps $1, %xmm0, (%rax)421; WIDEN_KNL-NEXT: vzeroupper422; WIDEN_KNL-NEXT: retq423;424; WIDEN_AVX2-LABEL: test_scatter_v2i32_data_index:425; WIDEN_AVX2: # %bb.0:426; WIDEN_AVX2-NEXT: vpmovsxdq %xmm1, %xmm1427; WIDEN_AVX2-NEXT: vpsllq $2, %xmm1, %xmm1428; WIDEN_AVX2-NEXT: vmovq %rdi, %xmm3429; WIDEN_AVX2-NEXT: vpbroadcastq %xmm3, %xmm3430; WIDEN_AVX2-NEXT: vpaddq %xmm1, %xmm3, %xmm1431; WIDEN_AVX2-NEXT: vpsllq $63, %xmm2, %xmm2432; WIDEN_AVX2-NEXT: vmovmskpd %xmm2, %eax433; WIDEN_AVX2-NEXT: testb $1, %al434; WIDEN_AVX2-NEXT: jne .LBB5_1435; WIDEN_AVX2-NEXT: # %bb.2: # %else436; WIDEN_AVX2-NEXT: testb $2, %al437; WIDEN_AVX2-NEXT: jne .LBB5_3438; WIDEN_AVX2-NEXT: .LBB5_4: # %else2439; WIDEN_AVX2-NEXT: retq440; WIDEN_AVX2-NEXT: .LBB5_1: # %cond.store441; WIDEN_AVX2-NEXT: vmovq %xmm1, %rcx442; WIDEN_AVX2-NEXT: vmovss %xmm0, (%rcx)443; WIDEN_AVX2-NEXT: testb $2, %al444; WIDEN_AVX2-NEXT: je .LBB5_4445; WIDEN_AVX2-NEXT: .LBB5_3: # %cond.store1446; WIDEN_AVX2-NEXT: vpextrq $1, %xmm1, %rax447; WIDEN_AVX2-NEXT: vextractps $1, %xmm0, (%rax)448; WIDEN_AVX2-NEXT: retq449 %gep = getelementptr i32, ptr%base, <2 x i32> %ind450 call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> %a1, <2 x ptr> %gep, i32 4, <2 x i1> %mask)451 ret void452}453 454define void @test_mscatter_v17f32(ptr %base, <17 x i32> %index, <17 x float> %val)455; WIDEN_SKX-LABEL: test_mscatter_v17f32:456; WIDEN_SKX: # %bb.0:457; WIDEN_SKX-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[2,3]458; WIDEN_SKX-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0,1],xmm6[0],xmm4[3]459; WIDEN_SKX-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0,1,2],xmm7[0]460; WIDEN_SKX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]461; WIDEN_SKX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]462; WIDEN_SKX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[0]463; WIDEN_SKX-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0464; WIDEN_SKX-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero465; WIDEN_SKX-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[2,3]466; WIDEN_SKX-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1],mem[0],xmm1[3]467; WIDEN_SKX-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],mem[0]468; WIDEN_SKX-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero469; WIDEN_SKX-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]470; WIDEN_SKX-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]471; WIDEN_SKX-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]472; WIDEN_SKX-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1473; WIDEN_SKX-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm0474; WIDEN_SKX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero475; WIDEN_SKX-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1476; WIDEN_SKX-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1477; WIDEN_SKX-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1478; WIDEN_SKX-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero479; WIDEN_SKX-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2480; WIDEN_SKX-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2481; WIDEN_SKX-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2482; WIDEN_SKX-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1483; WIDEN_SKX-NEXT: vmovd %esi, %xmm2484; WIDEN_SKX-NEXT: vpinsrd $1, %edx, %xmm2, %xmm2485; WIDEN_SKX-NEXT: vpinsrd $2, %ecx, %xmm2, %xmm2486; WIDEN_SKX-NEXT: vpinsrd $3, %r8d, %xmm2, %xmm2487; WIDEN_SKX-NEXT: vmovd %r9d, %xmm3488; WIDEN_SKX-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm3, %xmm3489; WIDEN_SKX-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm3, %xmm3490; WIDEN_SKX-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm3, %xmm3491; WIDEN_SKX-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2492; WIDEN_SKX-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1493; WIDEN_SKX-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero494; WIDEN_SKX-NEXT: vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero495; WIDEN_SKX-NEXT: kxnorw %k0, %k0, %k1496; WIDEN_SKX-NEXT: vscatterdps %zmm0, (%rdi,%zmm1,4) {%k1}497; WIDEN_SKX-NEXT: movw $1, %ax498; WIDEN_SKX-NEXT: kmovw %eax, %k1499; WIDEN_SKX-NEXT: vscatterdps %zmm2, (%rdi,%zmm3,4) {%k1}500; WIDEN_SKX-NEXT: vzeroupper501; WIDEN_SKX-NEXT: retq502;503; WIDEN_KNL-LABEL: test_mscatter_v17f32:504; WIDEN_KNL: # %bb.0:505; WIDEN_KNL-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[2,3]506; WIDEN_KNL-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0,1],xmm6[0],xmm4[3]507; WIDEN_KNL-NEXT: vinsertps {{.*#+}} xmm4 = xmm4[0,1,2],xmm7[0]508; WIDEN_KNL-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]509; WIDEN_KNL-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]510; WIDEN_KNL-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[0]511; WIDEN_KNL-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm0512; WIDEN_KNL-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero513; WIDEN_KNL-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[2,3]514; WIDEN_KNL-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1],mem[0],xmm1[3]515; WIDEN_KNL-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],mem[0]516; WIDEN_KNL-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero517; WIDEN_KNL-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0],mem[0],xmm2[2,3]518; WIDEN_KNL-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1],mem[0],xmm2[3]519; WIDEN_KNL-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],mem[0]520; WIDEN_KNL-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1521; WIDEN_KNL-NEXT: vinsertf64x4 $1, %ymm1, %zmm0, %zmm0522; WIDEN_KNL-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero523; WIDEN_KNL-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1524; WIDEN_KNL-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1525; WIDEN_KNL-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1526; WIDEN_KNL-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero527; WIDEN_KNL-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2528; WIDEN_KNL-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2529; WIDEN_KNL-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2530; WIDEN_KNL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1531; WIDEN_KNL-NEXT: vmovd %esi, %xmm2532; WIDEN_KNL-NEXT: vpinsrd $1, %edx, %xmm2, %xmm2533; WIDEN_KNL-NEXT: vpinsrd $2, %ecx, %xmm2, %xmm2534; WIDEN_KNL-NEXT: vpinsrd $3, %r8d, %xmm2, %xmm2535; WIDEN_KNL-NEXT: vmovd %r9d, %xmm3536; WIDEN_KNL-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm3, %xmm3537; WIDEN_KNL-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm3, %xmm3538; WIDEN_KNL-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm3, %xmm3539; WIDEN_KNL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2540; WIDEN_KNL-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1541; WIDEN_KNL-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero542; WIDEN_KNL-NEXT: vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero543; WIDEN_KNL-NEXT: kxnorw %k0, %k0, %k1544; WIDEN_KNL-NEXT: vscatterdps %zmm0, (%rdi,%zmm1,4) {%k1}545; WIDEN_KNL-NEXT: movw $1, %ax546; WIDEN_KNL-NEXT: kmovw %eax, %k1547; WIDEN_KNL-NEXT: vscatterdps %zmm2, (%rdi,%zmm3,4) {%k1}548; WIDEN_KNL-NEXT: vzeroupper549; WIDEN_KNL-NEXT: retq550;551; WIDEN_AVX2-LABEL: test_mscatter_v17f32:552; WIDEN_AVX2: # %bb.0:553; WIDEN_AVX2-NEXT: vmovq %rdi, %xmm8554; WIDEN_AVX2-NEXT: vpbroadcastq %xmm8, %ymm8555; WIDEN_AVX2-NEXT: vmovd %esi, %xmm9556; WIDEN_AVX2-NEXT: vpinsrd $1, %edx, %xmm9, %xmm9557; WIDEN_AVX2-NEXT: vpinsrd $2, %ecx, %xmm9, %xmm9558; WIDEN_AVX2-NEXT: vpinsrd $3, %r8d, %xmm9, %xmm9559; WIDEN_AVX2-NEXT: vpmovsxdq %xmm9, %ymm9560; WIDEN_AVX2-NEXT: vpsllq $2, %ymm9, %ymm9561; WIDEN_AVX2-NEXT: vpaddq %ymm9, %ymm8, %ymm9562; WIDEN_AVX2-NEXT: vmovq %xmm9, %rax563; WIDEN_AVX2-NEXT: vmovss %xmm0, (%rax)564; WIDEN_AVX2-NEXT: vpextrq $1, %xmm9, %rax565; WIDEN_AVX2-NEXT: vmovss %xmm1, (%rax)566; WIDEN_AVX2-NEXT: vextracti128 $1, %ymm9, %xmm0567; WIDEN_AVX2-NEXT: vmovq %xmm0, %rax568; WIDEN_AVX2-NEXT: vmovss %xmm2, (%rax)569; WIDEN_AVX2-NEXT: vpextrq $1, %xmm0, %rax570; WIDEN_AVX2-NEXT: vmovss %xmm3, (%rax)571; WIDEN_AVX2-NEXT: vmovd %r9d, %xmm0572; WIDEN_AVX2-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0573; WIDEN_AVX2-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0574; WIDEN_AVX2-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0575; WIDEN_AVX2-NEXT: vpmovsxdq %xmm0, %ymm0576; WIDEN_AVX2-NEXT: vpsllq $2, %ymm0, %ymm0577; WIDEN_AVX2-NEXT: vpaddq %ymm0, %ymm8, %ymm0578; WIDEN_AVX2-NEXT: vmovq %xmm0, %rax579; WIDEN_AVX2-NEXT: vmovss %xmm4, (%rax)580; WIDEN_AVX2-NEXT: vpextrq $1, %xmm0, %rax581; WIDEN_AVX2-NEXT: vmovss %xmm5, (%rax)582; WIDEN_AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0583; WIDEN_AVX2-NEXT: vmovq %xmm0, %rax584; WIDEN_AVX2-NEXT: vmovss %xmm6, (%rax)585; WIDEN_AVX2-NEXT: vpextrq $1, %xmm0, %rax586; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero587; WIDEN_AVX2-NEXT: vmovss %xmm7, (%rax)588; WIDEN_AVX2-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero589; WIDEN_AVX2-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1590; WIDEN_AVX2-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1591; WIDEN_AVX2-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1592; WIDEN_AVX2-NEXT: vpmovsxdq %xmm1, %ymm1593; WIDEN_AVX2-NEXT: vpsllq $2, %ymm1, %ymm1594; WIDEN_AVX2-NEXT: vpaddq %ymm1, %ymm8, %ymm1595; WIDEN_AVX2-NEXT: vmovq %xmm1, %rax596; WIDEN_AVX2-NEXT: vmovss %xmm0, (%rax)597; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero598; WIDEN_AVX2-NEXT: vpextrq $1, %xmm1, %rax599; WIDEN_AVX2-NEXT: vmovss %xmm0, (%rax)600; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero601; WIDEN_AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1602; WIDEN_AVX2-NEXT: vmovq %xmm1, %rax603; WIDEN_AVX2-NEXT: vmovss %xmm0, (%rax)604; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero605; WIDEN_AVX2-NEXT: vpextrq $1, %xmm1, %rax606; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero607; WIDEN_AVX2-NEXT: vmovss %xmm1, (%rax)608; WIDEN_AVX2-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero609; WIDEN_AVX2-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1610; WIDEN_AVX2-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1611; WIDEN_AVX2-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1612; WIDEN_AVX2-NEXT: vpmovsxdq %xmm1, %ymm1613; WIDEN_AVX2-NEXT: vpsllq $2, %ymm1, %ymm1614; WIDEN_AVX2-NEXT: vpaddq %ymm1, %ymm8, %ymm1615; WIDEN_AVX2-NEXT: vmovq %xmm1, %rax616; WIDEN_AVX2-NEXT: vmovss %xmm0, (%rax)617; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero618; WIDEN_AVX2-NEXT: vpextrq $1, %xmm1, %rax619; WIDEN_AVX2-NEXT: vmovss %xmm0, (%rax)620; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero621; WIDEN_AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1622; WIDEN_AVX2-NEXT: vmovq %xmm1, %rax623; WIDEN_AVX2-NEXT: vmovss %xmm0, (%rax)624; WIDEN_AVX2-NEXT: vpextrq $1, %xmm1, %rax625; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero626; WIDEN_AVX2-NEXT: vmovss %xmm0, (%rax)627; WIDEN_AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero628; WIDEN_AVX2-NEXT: vpmovsxdq %xmm0, %xmm0629; WIDEN_AVX2-NEXT: vpsllq $2, %xmm0, %xmm0630; WIDEN_AVX2-NEXT: vpaddq %xmm0, %xmm8, %xmm0631; WIDEN_AVX2-NEXT: vmovq %xmm0, %rax632; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero633; WIDEN_AVX2-NEXT: vmovss %xmm0, (%rax)634; WIDEN_AVX2-NEXT: vzeroupper635; WIDEN_AVX2-NEXT: retq636{637 %gep = getelementptr float, ptr %base, <17 x i32> %index638 call void @llvm.masked.scatter.v17f32.v17p0(<17 x float> %val, <17 x ptr> %gep, i32 4, <17 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)639 ret void640}641 642define <17 x float> @test_mgather_v17f32(ptr %base, <17 x i32> %index)643; WIDEN_SKX-LABEL: test_mgather_v17f32:644; WIDEN_SKX: # %bb.0:645; WIDEN_SKX-NEXT: movq %rdi, %rax646; WIDEN_SKX-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero647; WIDEN_SKX-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0648; WIDEN_SKX-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0649; WIDEN_SKX-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0650; WIDEN_SKX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero651; WIDEN_SKX-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1652; WIDEN_SKX-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1653; WIDEN_SKX-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1654; WIDEN_SKX-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0655; WIDEN_SKX-NEXT: vmovd %edx, %xmm1656; WIDEN_SKX-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm1657; WIDEN_SKX-NEXT: vpinsrd $2, %r8d, %xmm1, %xmm1658; WIDEN_SKX-NEXT: vpinsrd $3, %r9d, %xmm1, %xmm1659; WIDEN_SKX-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero660; WIDEN_SKX-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2661; WIDEN_SKX-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2662; WIDEN_SKX-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2663; WIDEN_SKX-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1664; WIDEN_SKX-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0665; WIDEN_SKX-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero666; WIDEN_SKX-NEXT: kxnorw %k0, %k0, %k1667; WIDEN_SKX-NEXT: vpxor %xmm2, %xmm2, %xmm2668; WIDEN_SKX-NEXT: vxorps %xmm3, %xmm3, %xmm3669; WIDEN_SKX-NEXT: vgatherdps (%rsi,%zmm0,4), %zmm3 {%k1}670; WIDEN_SKX-NEXT: movw $1, %cx671; WIDEN_SKX-NEXT: kmovw %ecx, %k1672; WIDEN_SKX-NEXT: vgatherdps (%rsi,%zmm1,4), %zmm2 {%k1}673; WIDEN_SKX-NEXT: vmovss %xmm2, 64(%rdi)674; WIDEN_SKX-NEXT: vmovaps %zmm3, (%rdi)675; WIDEN_SKX-NEXT: vzeroupper676; WIDEN_SKX-NEXT: retq677;678; WIDEN_KNL-LABEL: test_mgather_v17f32:679; WIDEN_KNL: # %bb.0:680; WIDEN_KNL-NEXT: movq %rdi, %rax681; WIDEN_KNL-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero682; WIDEN_KNL-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0683; WIDEN_KNL-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0684; WIDEN_KNL-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0685; WIDEN_KNL-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero686; WIDEN_KNL-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1687; WIDEN_KNL-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1688; WIDEN_KNL-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1689; WIDEN_KNL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0690; WIDEN_KNL-NEXT: vmovd %edx, %xmm1691; WIDEN_KNL-NEXT: vpinsrd $1, %ecx, %xmm1, %xmm1692; WIDEN_KNL-NEXT: vpinsrd $2, %r8d, %xmm1, %xmm1693; WIDEN_KNL-NEXT: vpinsrd $3, %r9d, %xmm1, %xmm1694; WIDEN_KNL-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero695; WIDEN_KNL-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm2, %xmm2696; WIDEN_KNL-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm2, %xmm2697; WIDEN_KNL-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm2, %xmm2698; WIDEN_KNL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1699; WIDEN_KNL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0700; WIDEN_KNL-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero701; WIDEN_KNL-NEXT: kxnorw %k0, %k0, %k1702; WIDEN_KNL-NEXT: vpxor %xmm2, %xmm2, %xmm2703; WIDEN_KNL-NEXT: vxorps %xmm3, %xmm3, %xmm3704; WIDEN_KNL-NEXT: vgatherdps (%rsi,%zmm0,4), %zmm3 {%k1}705; WIDEN_KNL-NEXT: movw $1, %cx706; WIDEN_KNL-NEXT: kmovw %ecx, %k1707; WIDEN_KNL-NEXT: vgatherdps (%rsi,%zmm1,4), %zmm2 {%k1}708; WIDEN_KNL-NEXT: vmovss %xmm2, 64(%rdi)709; WIDEN_KNL-NEXT: vmovaps %zmm3, (%rdi)710; WIDEN_KNL-NEXT: vzeroupper711; WIDEN_KNL-NEXT: retq712;713; WIDEN_AVX2-LABEL: test_mgather_v17f32:714; WIDEN_AVX2: # %bb.0:715; WIDEN_AVX2-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero716; WIDEN_AVX2-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm0, %xmm0717; WIDEN_AVX2-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm0, %xmm0718; WIDEN_AVX2-NEXT: movq %rdi, %rax719; WIDEN_AVX2-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm0, %xmm0720; WIDEN_AVX2-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero721; WIDEN_AVX2-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1722; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero723; WIDEN_AVX2-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1724; WIDEN_AVX2-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1725; WIDEN_AVX2-NEXT: vmovd %edx, %xmm3726; WIDEN_AVX2-NEXT: vpinsrd $1, %ecx, %xmm3, %xmm3727; WIDEN_AVX2-NEXT: vpinsrd $2, %r8d, %xmm3, %xmm3728; WIDEN_AVX2-NEXT: vpinsrd $3, %r9d, %xmm3, %xmm3729; WIDEN_AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0730; WIDEN_AVX2-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero731; WIDEN_AVX2-NEXT: vpinsrd $1, {{[0-9]+}}(%rsp), %xmm1, %xmm1732; WIDEN_AVX2-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1733; WIDEN_AVX2-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1734; WIDEN_AVX2-NEXT: vinserti128 $1, %xmm1, %ymm3, %ymm1735; WIDEN_AVX2-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3736; WIDEN_AVX2-NEXT: vxorps %xmm4, %xmm4, %xmm4737; WIDEN_AVX2-NEXT: vpcmpeqd %ymm5, %ymm5, %ymm5738; WIDEN_AVX2-NEXT: vxorps %xmm6, %xmm6, %xmm6739; WIDEN_AVX2-NEXT: vgatherdps %ymm5, (%rsi,%ymm1,4), %ymm6740; WIDEN_AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1741; WIDEN_AVX2-NEXT: vgatherdps %ymm3, (%rsi,%ymm0,4), %ymm1742; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm0 = [4294967295,0,0,0]743; WIDEN_AVX2-NEXT: vgatherdps %ymm0, (%rsi,%ymm2,4), %ymm4744; WIDEN_AVX2-NEXT: vmovss %xmm4, 64(%rdi)745; WIDEN_AVX2-NEXT: vmovaps %ymm1, 32(%rdi)746; WIDEN_AVX2-NEXT: vmovaps %ymm6, (%rdi)747; WIDEN_AVX2-NEXT: vzeroupper748; WIDEN_AVX2-NEXT: retq749{750 %gep = getelementptr float, ptr %base, <17 x i32> %index751 %res = call <17 x float> @llvm.masked.gather.v17f32.v17p0(<17 x ptr> %gep, i32 4, <17 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <17 x float> undef)752 ret <17 x float> %res753}754 755declare <17 x float> @llvm.masked.gather.v17f32.v17p0(<17 x ptr>, i32 immarg, <17 x i1>, <17 x float>)756declare void @llvm.masked.scatter.v17f32.v17p0(<17 x float> , <17 x ptr> , i32 , <17 x i1>)757 758declare <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr>, i32, <2 x i1>, <2 x double>)759declare void @llvm.masked.scatter.v2f64.v2p0(<2 x double>, <2 x ptr>, i32, <2 x i1>)760declare <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr>, i32, <2 x i1>, <2 x i32>)761declare void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> , <2 x ptr> , i32 , <2 x i1>)762