740 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2,FALLBACK03; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse4.2 | FileCheck %s --check-prefixes=SSE,SSE42,FALLBACK14; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1-ONLY,FALLBACK25; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2,AVX2-SLOW,FALLBACK36; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2,AVX2-FAST-PERLANE,FALLBACK47; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2,AVX2-FAST,FALLBACK58; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512F,AVX512F-SLOW,FALLBACK69; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512F,AVX512F-FAST,FALLBACK710; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW,FALLBACK811; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-FAST,FALLBACK912 13define void @concat_a_to_shuf_of_a(ptr %a.ptr, ptr %dst) {14; SSE-LABEL: concat_a_to_shuf_of_a:15; SSE: # %bb.0:16; SSE-NEXT: movdqa (%rdi), %xmm017; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]18; SSE-NEXT: movdqa %xmm0, 16(%rsi)19; SSE-NEXT: movdqa %xmm1, (%rsi)20; SSE-NEXT: retq21;22; AVX-LABEL: concat_a_to_shuf_of_a:23; AVX: # %bb.0:24; AVX-NEXT: vmovaps (%rdi), %xmm025; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1]26; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm027; AVX-NEXT: vmovaps %ymm0, (%rsi)28; AVX-NEXT: vzeroupper29; AVX-NEXT: retq30;31; AVX2-LABEL: concat_a_to_shuf_of_a:32; AVX2: # %bb.0:33; AVX2-NEXT: vmovaps (%rdi), %xmm034; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,0,0,1]35; AVX2-NEXT: vmovaps %ymm0, (%rsi)36; AVX2-NEXT: vzeroupper37; AVX2-NEXT: retq38;39; AVX512F-LABEL: concat_a_to_shuf_of_a:40; AVX512F: # %bb.0:41; AVX512F-NEXT: vmovaps (%rdi), %xmm042; AVX512F-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,0,0,1]43; AVX512F-NEXT: vmovaps %ymm0, (%rsi)44; AVX512F-NEXT: vzeroupper45; AVX512F-NEXT: retq46;47; AVX512BW-LABEL: concat_a_to_shuf_of_a:48; AVX512BW: # %bb.0:49; AVX512BW-NEXT: vmovaps (%rdi), %xmm050; AVX512BW-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,0,0,1]51; AVX512BW-NEXT: vmovaps %ymm0, (%rsi)52; AVX512BW-NEXT: vzeroupper53; AVX512BW-NEXT: retq54 %a = load <2 x i64>, ptr %a.ptr, align 6455 %shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>56 %concat = shufflevector <2 x i64> %shuffle, <2 x i64> %a, <4 x i32> <i32 0, i32 1, i32 2, i32 3>57 store <4 x i64> %concat, ptr %dst, align 6458 ret void59}60define void @concat_shuf_of_a_to_a(ptr %a.ptr, ptr %b.ptr, ptr %dst) {61; SSE-LABEL: concat_shuf_of_a_to_a:62; SSE: # %bb.0:63; SSE-NEXT: movdqa (%rdi), %xmm064; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]65; SSE-NEXT: movdqa %xmm0, (%rdx)66; SSE-NEXT: movdqa %xmm1, 16(%rdx)67; SSE-NEXT: retq68;69; AVX-LABEL: concat_shuf_of_a_to_a:70; AVX: # %bb.0:71; AVX-NEXT: vmovaps (%rdi), %xmm072; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1]73; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm074; AVX-NEXT: vmovaps %ymm0, (%rdx)75; AVX-NEXT: vzeroupper76; AVX-NEXT: retq77;78; AVX2-LABEL: concat_shuf_of_a_to_a:79; AVX2: # %bb.0:80; AVX2-NEXT: vmovaps (%rdi), %xmm081; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,1,1,0]82; AVX2-NEXT: vmovaps %ymm0, (%rdx)83; AVX2-NEXT: vzeroupper84; AVX2-NEXT: retq85;86; AVX512F-LABEL: concat_shuf_of_a_to_a:87; AVX512F: # %bb.0:88; AVX512F-NEXT: vmovaps (%rdi), %xmm089; AVX512F-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,1,1,0]90; AVX512F-NEXT: vmovaps %ymm0, (%rdx)91; AVX512F-NEXT: vzeroupper92; AVX512F-NEXT: retq93;94; AVX512BW-LABEL: concat_shuf_of_a_to_a:95; AVX512BW: # %bb.0:96; AVX512BW-NEXT: vmovaps (%rdi), %xmm097; AVX512BW-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,1,1,0]98; AVX512BW-NEXT: vmovaps %ymm0, (%rdx)99; AVX512BW-NEXT: vzeroupper100; AVX512BW-NEXT: retq101 %a = load <2 x i64>, ptr %a.ptr, align 64102 %b = load <2 x i64>, ptr %b.ptr, align 64103 %shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>104 %concat = shufflevector <2 x i64> %a, <2 x i64> %shuffle, <4 x i32> <i32 0, i32 1, i32 2, i32 3>105 store <4 x i64> %concat, ptr %dst, align 64106 ret void107}108 109define void @concat_a_to_shuf_of_a_extrause_of_shuf(ptr %a.ptr, ptr %dst, ptr %shuf.escape.ptr) {110; SSE-LABEL: concat_a_to_shuf_of_a_extrause_of_shuf:111; SSE: # %bb.0:112; SSE-NEXT: movdqa (%rdi), %xmm0113; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]114; SSE-NEXT: movdqa %xmm1, (%rdx)115; SSE-NEXT: movdqa %xmm0, 16(%rsi)116; SSE-NEXT: movdqa %xmm1, (%rsi)117; SSE-NEXT: retq118;119; AVX-LABEL: concat_a_to_shuf_of_a_extrause_of_shuf:120; AVX: # %bb.0:121; AVX-NEXT: vmovaps (%rdi), %xmm0122; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1]123; AVX-NEXT: vmovaps %xmm1, (%rdx)124; AVX-NEXT: vmovaps %xmm0, 16(%rsi)125; AVX-NEXT: vmovaps %xmm1, (%rsi)126; AVX-NEXT: retq127;128; AVX2-LABEL: concat_a_to_shuf_of_a_extrause_of_shuf:129; AVX2: # %bb.0:130; AVX2-NEXT: vmovaps (%rdi), %xmm0131; AVX2-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1]132; AVX2-NEXT: vmovaps %xmm1, (%rdx)133; AVX2-NEXT: vmovaps %xmm0, 16(%rsi)134; AVX2-NEXT: vmovaps %xmm1, (%rsi)135; AVX2-NEXT: retq136;137; AVX512F-LABEL: concat_a_to_shuf_of_a_extrause_of_shuf:138; AVX512F: # %bb.0:139; AVX512F-NEXT: vmovaps (%rdi), %xmm0140; AVX512F-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1]141; AVX512F-NEXT: vmovaps %xmm1, (%rdx)142; AVX512F-NEXT: vmovaps %xmm0, 16(%rsi)143; AVX512F-NEXT: vmovaps %xmm1, (%rsi)144; AVX512F-NEXT: retq145;146; AVX512BW-LABEL: concat_a_to_shuf_of_a_extrause_of_shuf:147; AVX512BW: # %bb.0:148; AVX512BW-NEXT: vmovaps (%rdi), %xmm0149; AVX512BW-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1]150; AVX512BW-NEXT: vmovaps %xmm1, (%rdx)151; AVX512BW-NEXT: vmovaps %xmm0, 16(%rsi)152; AVX512BW-NEXT: vmovaps %xmm1, (%rsi)153; AVX512BW-NEXT: retq154 %a = load <2 x i64>, ptr %a.ptr, align 64155 %shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>156 store <2 x i64> %shuffle, ptr %shuf.escape.ptr, align 64157 %concat = shufflevector <2 x i64> %shuffle, <2 x i64> %a, <4 x i32> <i32 0, i32 1, i32 2, i32 3>158 store <4 x i64> %concat, ptr %dst, align 64159 ret void160}161 162define void @concat_a_to_shuf_of_ab(ptr %a.ptr, ptr %b.ptr, ptr %dst) {163; SSE2-LABEL: concat_a_to_shuf_of_ab:164; SSE2: # %bb.0:165; SSE2-NEXT: movapd (%rdi), %xmm0166; SSE2-NEXT: movapd (%rsi), %xmm1167; SSE2-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]168; SSE2-NEXT: movapd %xmm0, 16(%rdx)169; SSE2-NEXT: movapd %xmm1, (%rdx)170; SSE2-NEXT: retq171;172; SSE42-LABEL: concat_a_to_shuf_of_ab:173; SSE42: # %bb.0:174; SSE42-NEXT: movaps (%rdi), %xmm0175; SSE42-NEXT: movaps (%rsi), %xmm1176; SSE42-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]177; SSE42-NEXT: movaps %xmm0, 16(%rdx)178; SSE42-NEXT: movaps %xmm1, (%rdx)179; SSE42-NEXT: retq180;181; AVX-LABEL: concat_a_to_shuf_of_ab:182; AVX: # %bb.0:183; AVX-NEXT: vmovaps (%rdi), %xmm0184; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0,1],mem[2,3]185; AVX-NEXT: vmovaps %xmm0, 16(%rdx)186; AVX-NEXT: vmovaps %xmm1, (%rdx)187; AVX-NEXT: retq188;189; AVX2-LABEL: concat_a_to_shuf_of_ab:190; AVX2: # %bb.0:191; AVX2-NEXT: vmovaps (%rdi), %xmm0192; AVX2-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0,1],mem[2,3]193; AVX2-NEXT: vmovaps %xmm0, 16(%rdx)194; AVX2-NEXT: vmovaps %xmm1, (%rdx)195; AVX2-NEXT: retq196;197; AVX512F-LABEL: concat_a_to_shuf_of_ab:198; AVX512F: # %bb.0:199; AVX512F-NEXT: vmovaps (%rdi), %xmm0200; AVX512F-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0,1],mem[2,3]201; AVX512F-NEXT: vmovaps %xmm0, 16(%rdx)202; AVX512F-NEXT: vmovaps %xmm1, (%rdx)203; AVX512F-NEXT: retq204;205; AVX512BW-LABEL: concat_a_to_shuf_of_ab:206; AVX512BW: # %bb.0:207; AVX512BW-NEXT: vmovaps (%rdi), %xmm0208; AVX512BW-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0,1],mem[2,3]209; AVX512BW-NEXT: vmovaps %xmm0, 16(%rdx)210; AVX512BW-NEXT: vmovaps %xmm1, (%rdx)211; AVX512BW-NEXT: retq212 %a = load <2 x i64>, ptr %a.ptr, align 64213 %b = load <2 x i64>, ptr %b.ptr, align 64214 %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> <i32 0, i32 3>215 %concat = shufflevector <2 x i64> %shuffle, <2 x i64> %a, <4 x i32> <i32 0, i32 1, i32 2, i32 3>216 store <4 x i64> %concat, ptr %dst, align 64217 ret void218}219define void @concat_b_to_shuf_of_ab(ptr %a.ptr, ptr %b.ptr, ptr %dst) {220; SSE2-LABEL: concat_b_to_shuf_of_ab:221; SSE2: # %bb.0:222; SSE2-NEXT: movaps (%rsi), %xmm0223; SSE2-NEXT: movaps %xmm0, %xmm1224; SSE2-NEXT: movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]225; SSE2-NEXT: movaps %xmm0, 16(%rdx)226; SSE2-NEXT: movaps %xmm1, (%rdx)227; SSE2-NEXT: retq228;229; SSE42-LABEL: concat_b_to_shuf_of_ab:230; SSE42: # %bb.0:231; SSE42-NEXT: movaps (%rsi), %xmm0232; SSE42-NEXT: movaps (%rdi), %xmm1233; SSE42-NEXT: blendps {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,3]234; SSE42-NEXT: movaps %xmm0, 16(%rdx)235; SSE42-NEXT: movaps %xmm1, (%rdx)236; SSE42-NEXT: retq237;238; AVX-LABEL: concat_b_to_shuf_of_ab:239; AVX: # %bb.0:240; AVX-NEXT: vmovaps (%rsi), %xmm0241; AVX-NEXT: vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3]242; AVX-NEXT: vmovaps %xmm0, 16(%rdx)243; AVX-NEXT: vmovaps %xmm1, (%rdx)244; AVX-NEXT: retq245;246; AVX2-LABEL: concat_b_to_shuf_of_ab:247; AVX2: # %bb.0:248; AVX2-NEXT: vmovaps (%rsi), %xmm0249; AVX2-NEXT: vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3]250; AVX2-NEXT: vmovaps %xmm0, 16(%rdx)251; AVX2-NEXT: vmovaps %xmm1, (%rdx)252; AVX2-NEXT: retq253;254; AVX512F-LABEL: concat_b_to_shuf_of_ab:255; AVX512F: # %bb.0:256; AVX512F-NEXT: vmovaps (%rsi), %xmm0257; AVX512F-NEXT: vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3]258; AVX512F-NEXT: vmovaps %xmm0, 16(%rdx)259; AVX512F-NEXT: vmovaps %xmm1, (%rdx)260; AVX512F-NEXT: retq261;262; AVX512BW-LABEL: concat_b_to_shuf_of_ab:263; AVX512BW: # %bb.0:264; AVX512BW-NEXT: vmovaps (%rsi), %xmm0265; AVX512BW-NEXT: vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3]266; AVX512BW-NEXT: vmovaps %xmm0, 16(%rdx)267; AVX512BW-NEXT: vmovaps %xmm1, (%rdx)268; AVX512BW-NEXT: retq269 %a = load <2 x i64>, ptr %a.ptr, align 64270 %b = load <2 x i64>, ptr %b.ptr, align 64271 %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> <i32 0, i32 3>272 %concat = shufflevector <2 x i64> %shuffle, <2 x i64> %b, <4 x i32> <i32 0, i32 1, i32 2, i32 3>273 store <4 x i64> %concat, ptr %dst, align 64274 ret void275}276 277define void @concat_shuf_of_ab_to_a(ptr %a.ptr, ptr %b.ptr, ptr %dst) {278; SSE2-LABEL: concat_shuf_of_ab_to_a:279; SSE2: # %bb.0:280; SSE2-NEXT: movapd (%rdi), %xmm0281; SSE2-NEXT: movapd (%rsi), %xmm1282; SSE2-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]283; SSE2-NEXT: movapd %xmm0, (%rdx)284; SSE2-NEXT: movapd %xmm1, 16(%rdx)285; SSE2-NEXT: retq286;287; SSE42-LABEL: concat_shuf_of_ab_to_a:288; SSE42: # %bb.0:289; SSE42-NEXT: movaps (%rdi), %xmm0290; SSE42-NEXT: movaps (%rsi), %xmm1291; SSE42-NEXT: movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]292; SSE42-NEXT: movaps %xmm1, 16(%rdx)293; SSE42-NEXT: movaps %xmm0, (%rdx)294; SSE42-NEXT: retq295;296; AVX-LABEL: concat_shuf_of_ab_to_a:297; AVX: # %bb.0:298; AVX-NEXT: vmovaps (%rdi), %xmm0299; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0,1],mem[2,3]300; AVX-NEXT: vmovaps %xmm1, 16(%rdx)301; AVX-NEXT: vmovaps %xmm0, (%rdx)302; AVX-NEXT: retq303;304; AVX2-LABEL: concat_shuf_of_ab_to_a:305; AVX2: # %bb.0:306; AVX2-NEXT: vmovaps (%rdi), %xmm0307; AVX2-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0,1],mem[2,3]308; AVX2-NEXT: vmovaps %xmm1, 16(%rdx)309; AVX2-NEXT: vmovaps %xmm0, (%rdx)310; AVX2-NEXT: retq311;312; AVX512F-LABEL: concat_shuf_of_ab_to_a:313; AVX512F: # %bb.0:314; AVX512F-NEXT: vmovaps (%rdi), %xmm0315; AVX512F-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0,1],mem[2,3]316; AVX512F-NEXT: vmovaps %xmm1, 16(%rdx)317; AVX512F-NEXT: vmovaps %xmm0, (%rdx)318; AVX512F-NEXT: retq319;320; AVX512BW-LABEL: concat_shuf_of_ab_to_a:321; AVX512BW: # %bb.0:322; AVX512BW-NEXT: vmovaps (%rdi), %xmm0323; AVX512BW-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0,1],mem[2,3]324; AVX512BW-NEXT: vmovaps %xmm1, 16(%rdx)325; AVX512BW-NEXT: vmovaps %xmm0, (%rdx)326; AVX512BW-NEXT: retq327 %a = load <2 x i64>, ptr %a.ptr, align 64328 %b = load <2 x i64>, ptr %b.ptr, align 64329 %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> <i32 0, i32 3>330 %concat = shufflevector <2 x i64> %a, <2 x i64> %shuffle, <4 x i32> <i32 0, i32 1, i32 2, i32 3>331 store <4 x i64> %concat, ptr %dst, align 64332 ret void333}334define void @concat_shuf_of_ab_to_b(ptr %a.ptr, ptr %b.ptr, ptr %dst) {335; SSE2-LABEL: concat_shuf_of_ab_to_b:336; SSE2: # %bb.0:337; SSE2-NEXT: movaps (%rsi), %xmm0338; SSE2-NEXT: movaps %xmm0, %xmm1339; SSE2-NEXT: movlps {{.*#+}} xmm1 = mem[0,1],xmm1[2,3]340; SSE2-NEXT: movaps %xmm1, 16(%rdx)341; SSE2-NEXT: movaps %xmm0, (%rdx)342; SSE2-NEXT: retq343;344; SSE42-LABEL: concat_shuf_of_ab_to_b:345; SSE42: # %bb.0:346; SSE42-NEXT: movaps (%rsi), %xmm0347; SSE42-NEXT: movaps (%rdi), %xmm1348; SSE42-NEXT: blendps {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,3]349; SSE42-NEXT: movaps %xmm1, 16(%rdx)350; SSE42-NEXT: movaps %xmm0, (%rdx)351; SSE42-NEXT: retq352;353; AVX-LABEL: concat_shuf_of_ab_to_b:354; AVX: # %bb.0:355; AVX-NEXT: vmovaps (%rsi), %xmm0356; AVX-NEXT: vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3]357; AVX-NEXT: vmovaps %xmm1, 16(%rdx)358; AVX-NEXT: vmovaps %xmm0, (%rdx)359; AVX-NEXT: retq360;361; AVX2-LABEL: concat_shuf_of_ab_to_b:362; AVX2: # %bb.0:363; AVX2-NEXT: vmovaps (%rsi), %xmm0364; AVX2-NEXT: vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3]365; AVX2-NEXT: vmovaps %xmm1, 16(%rdx)366; AVX2-NEXT: vmovaps %xmm0, (%rdx)367; AVX2-NEXT: retq368;369; AVX512F-LABEL: concat_shuf_of_ab_to_b:370; AVX512F: # %bb.0:371; AVX512F-NEXT: vmovaps (%rsi), %xmm0372; AVX512F-NEXT: vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3]373; AVX512F-NEXT: vmovaps %xmm1, 16(%rdx)374; AVX512F-NEXT: vmovaps %xmm0, (%rdx)375; AVX512F-NEXT: retq376;377; AVX512BW-LABEL: concat_shuf_of_ab_to_b:378; AVX512BW: # %bb.0:379; AVX512BW-NEXT: vmovaps (%rsi), %xmm0380; AVX512BW-NEXT: vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3]381; AVX512BW-NEXT: vmovaps %xmm1, 16(%rdx)382; AVX512BW-NEXT: vmovaps %xmm0, (%rdx)383; AVX512BW-NEXT: retq384 %a = load <2 x i64>, ptr %a.ptr, align 64385 %b = load <2 x i64>, ptr %b.ptr, align 64386 %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> <i32 0, i32 3>387 %concat = shufflevector <2 x i64> %b, <2 x i64> %shuffle, <4 x i32> <i32 0, i32 1, i32 2, i32 3>388 store <4 x i64> %concat, ptr %dst, align 64389 ret void390}391 392define void @concat_b_to_shuf_of_a(ptr %a.ptr, ptr %b.ptr, ptr %dst) {393; SSE-LABEL: concat_b_to_shuf_of_a:394; SSE: # %bb.0:395; SSE-NEXT: movaps (%rsi), %xmm0396; SSE-NEXT: pshufd {{.*#+}} xmm1 = mem[2,3,0,1]397; SSE-NEXT: movaps %xmm0, 16(%rdx)398; SSE-NEXT: movdqa %xmm1, (%rdx)399; SSE-NEXT: retq400;401; AVX-LABEL: concat_b_to_shuf_of_a:402; AVX: # %bb.0:403; AVX-NEXT: vmovaps (%rsi), %xmm0404; AVX-NEXT: vpermilps {{.*#+}} xmm1 = mem[2,3,0,1]405; AVX-NEXT: vmovaps %xmm0, 16(%rdx)406; AVX-NEXT: vmovaps %xmm1, (%rdx)407; AVX-NEXT: retq408;409; AVX2-LABEL: concat_b_to_shuf_of_a:410; AVX2: # %bb.0:411; AVX2-NEXT: vmovaps (%rsi), %xmm0412; AVX2-NEXT: vpermilps {{.*#+}} xmm1 = mem[2,3,0,1]413; AVX2-NEXT: vmovaps %xmm0, 16(%rdx)414; AVX2-NEXT: vmovaps %xmm1, (%rdx)415; AVX2-NEXT: retq416;417; AVX512F-LABEL: concat_b_to_shuf_of_a:418; AVX512F: # %bb.0:419; AVX512F-NEXT: vmovaps (%rsi), %xmm0420; AVX512F-NEXT: vpermilps {{.*#+}} xmm1 = mem[2,3,0,1]421; AVX512F-NEXT: vmovaps %xmm0, 16(%rdx)422; AVX512F-NEXT: vmovaps %xmm1, (%rdx)423; AVX512F-NEXT: retq424;425; AVX512BW-LABEL: concat_b_to_shuf_of_a:426; AVX512BW: # %bb.0:427; AVX512BW-NEXT: vmovaps (%rsi), %xmm0428; AVX512BW-NEXT: vpermilps {{.*#+}} xmm1 = mem[2,3,0,1]429; AVX512BW-NEXT: vmovaps %xmm0, 16(%rdx)430; AVX512BW-NEXT: vmovaps %xmm1, (%rdx)431; AVX512BW-NEXT: retq432 %a = load <2 x i64>, ptr %a.ptr, align 64433 %b = load <2 x i64>, ptr %b.ptr, align 64434 %shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>435 %concat = shufflevector <2 x i64> %shuffle, <2 x i64> %b, <4 x i32> <i32 0, i32 1, i32 2, i32 3>436 store <4 x i64> %concat, ptr %dst, align 64437 ret void438}439define void @concat_shuf_of_a_to_b(ptr %a.ptr, ptr %b.ptr, ptr %dst) {440; SSE-LABEL: concat_shuf_of_a_to_b:441; SSE: # %bb.0:442; SSE-NEXT: movaps (%rsi), %xmm0443; SSE-NEXT: pshufd {{.*#+}} xmm1 = mem[2,3,0,1]444; SSE-NEXT: movdqa %xmm1, 16(%rdx)445; SSE-NEXT: movaps %xmm0, (%rdx)446; SSE-NEXT: retq447;448; AVX-LABEL: concat_shuf_of_a_to_b:449; AVX: # %bb.0:450; AVX-NEXT: vmovaps (%rsi), %xmm0451; AVX-NEXT: vpermilps {{.*#+}} xmm1 = mem[2,3,0,1]452; AVX-NEXT: vmovaps %xmm1, 16(%rdx)453; AVX-NEXT: vmovaps %xmm0, (%rdx)454; AVX-NEXT: retq455;456; AVX2-LABEL: concat_shuf_of_a_to_b:457; AVX2: # %bb.0:458; AVX2-NEXT: vmovaps (%rsi), %xmm0459; AVX2-NEXT: vpermilps {{.*#+}} xmm1 = mem[2,3,0,1]460; AVX2-NEXT: vmovaps %xmm1, 16(%rdx)461; AVX2-NEXT: vmovaps %xmm0, (%rdx)462; AVX2-NEXT: retq463;464; AVX512F-LABEL: concat_shuf_of_a_to_b:465; AVX512F: # %bb.0:466; AVX512F-NEXT: vmovaps (%rsi), %xmm0467; AVX512F-NEXT: vpermilps {{.*#+}} xmm1 = mem[2,3,0,1]468; AVX512F-NEXT: vmovaps %xmm1, 16(%rdx)469; AVX512F-NEXT: vmovaps %xmm0, (%rdx)470; AVX512F-NEXT: retq471;472; AVX512BW-LABEL: concat_shuf_of_a_to_b:473; AVX512BW: # %bb.0:474; AVX512BW-NEXT: vmovaps (%rsi), %xmm0475; AVX512BW-NEXT: vpermilps {{.*#+}} xmm1 = mem[2,3,0,1]476; AVX512BW-NEXT: vmovaps %xmm1, 16(%rdx)477; AVX512BW-NEXT: vmovaps %xmm0, (%rdx)478; AVX512BW-NEXT: retq479 %a = load <2 x i64>, ptr %a.ptr, align 64480 %b = load <2 x i64>, ptr %b.ptr, align 64481 %shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>482 %concat = shufflevector <2 x i64> %b, <2 x i64> %shuffle, <4 x i32> <i32 0, i32 1, i32 2, i32 3>483 store <4 x i64> %concat, ptr %dst, align 64484 ret void485}486 487define void @concat_poison_to_shuf_of_a(ptr %a.ptr, ptr %dst) {488; SSE-LABEL: concat_poison_to_shuf_of_a:489; SSE: # %bb.0:490; SSE-NEXT: pshufd {{.*#+}} xmm0 = mem[2,3,0,1]491; SSE-NEXT: movdqa %xmm0, (%rsi)492; SSE-NEXT: retq493;494; AVX-LABEL: concat_poison_to_shuf_of_a:495; AVX: # %bb.0:496; AVX-NEXT: vpermilps {{.*#+}} xmm0 = mem[2,3,0,1]497; AVX-NEXT: vmovaps %xmm0, (%rsi)498; AVX-NEXT: retq499;500; AVX2-LABEL: concat_poison_to_shuf_of_a:501; AVX2: # %bb.0:502; AVX2-NEXT: vpermilps {{.*#+}} xmm0 = mem[2,3,0,1]503; AVX2-NEXT: vmovaps %xmm0, (%rsi)504; AVX2-NEXT: retq505;506; AVX512F-LABEL: concat_poison_to_shuf_of_a:507; AVX512F: # %bb.0:508; AVX512F-NEXT: vpermilps {{.*#+}} xmm0 = mem[2,3,0,1]509; AVX512F-NEXT: vmovaps %xmm0, (%rsi)510; AVX512F-NEXT: retq511;512; AVX512BW-LABEL: concat_poison_to_shuf_of_a:513; AVX512BW: # %bb.0:514; AVX512BW-NEXT: vpermilps {{.*#+}} xmm0 = mem[2,3,0,1]515; AVX512BW-NEXT: vmovaps %xmm0, (%rsi)516; AVX512BW-NEXT: retq517 %a = load <2 x i64>, ptr %a.ptr, align 64518 %shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>519 %concat = shufflevector <2 x i64> %shuffle, <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>520 store <4 x i64> %concat, ptr %dst, align 64521 ret void522}523define void @concat_shuf_of_a_to_poison(ptr %a.ptr, ptr %b.ptr, ptr %dst) {524; SSE-LABEL: concat_shuf_of_a_to_poison:525; SSE: # %bb.0:526; SSE-NEXT: pshufd {{.*#+}} xmm0 = mem[2,3,0,1]527; SSE-NEXT: movdqa %xmm0, 16(%rdx)528; SSE-NEXT: retq529;530; AVX-LABEL: concat_shuf_of_a_to_poison:531; AVX: # %bb.0:532; AVX-NEXT: vpermilps {{.*#+}} xmm0 = mem[2,3,0,1]533; AVX-NEXT: vmovaps %xmm0, 16(%rdx)534; AVX-NEXT: retq535;536; AVX2-LABEL: concat_shuf_of_a_to_poison:537; AVX2: # %bb.0:538; AVX2-NEXT: vpermilps {{.*#+}} xmm0 = mem[2,3,0,1]539; AVX2-NEXT: vmovaps %xmm0, 16(%rdx)540; AVX2-NEXT: retq541;542; AVX512F-LABEL: concat_shuf_of_a_to_poison:543; AVX512F: # %bb.0:544; AVX512F-NEXT: vpermilps {{.*#+}} xmm0 = mem[2,3,0,1]545; AVX512F-NEXT: vmovaps %xmm0, 16(%rdx)546; AVX512F-NEXT: retq547;548; AVX512BW-LABEL: concat_shuf_of_a_to_poison:549; AVX512BW: # %bb.0:550; AVX512BW-NEXT: vpermilps {{.*#+}} xmm0 = mem[2,3,0,1]551; AVX512BW-NEXT: vmovaps %xmm0, 16(%rdx)552; AVX512BW-NEXT: retq553 %a = load <2 x i64>, ptr %a.ptr, align 64554 %b = load <2 x i64>, ptr %b.ptr, align 64555 %shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>556 %concat = shufflevector <2 x i64> poison, <2 x i64> %shuffle, <4 x i32> <i32 0, i32 1, i32 2, i32 3>557 store <4 x i64> %concat, ptr %dst, align 64558 ret void559}560 561define void @concat_shuf_of_a_to_itself(ptr %a.ptr, ptr %dst) {562; SSE-LABEL: concat_shuf_of_a_to_itself:563; SSE: # %bb.0:564; SSE-NEXT: pshufd {{.*#+}} xmm0 = mem[2,3,0,1]565; SSE-NEXT: movdqa %xmm0, 16(%rsi)566; SSE-NEXT: movdqa %xmm0, (%rsi)567; SSE-NEXT: retq568;569; AVX-LABEL: concat_shuf_of_a_to_itself:570; AVX: # %bb.0:571; AVX-NEXT: vpermilpd {{.*#+}} xmm0 = mem[1,0]572; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0573; AVX-NEXT: vmovaps %ymm0, (%rsi)574; AVX-NEXT: vzeroupper575; AVX-NEXT: retq576;577; AVX2-LABEL: concat_shuf_of_a_to_itself:578; AVX2: # %bb.0:579; AVX2-NEXT: vmovaps (%rdi), %xmm0580; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,0,1,0]581; AVX2-NEXT: vmovaps %ymm0, (%rsi)582; AVX2-NEXT: vzeroupper583; AVX2-NEXT: retq584;585; AVX512F-LABEL: concat_shuf_of_a_to_itself:586; AVX512F: # %bb.0:587; AVX512F-NEXT: vmovaps (%rdi), %xmm0588; AVX512F-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,0,1,0]589; AVX512F-NEXT: vmovaps %ymm0, (%rsi)590; AVX512F-NEXT: vzeroupper591; AVX512F-NEXT: retq592;593; AVX512BW-LABEL: concat_shuf_of_a_to_itself:594; AVX512BW: # %bb.0:595; AVX512BW-NEXT: vmovaps (%rdi), %xmm0596; AVX512BW-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,0,1,0]597; AVX512BW-NEXT: vmovaps %ymm0, (%rsi)598; AVX512BW-NEXT: vzeroupper599; AVX512BW-NEXT: retq600 %a = load <2 x i64>, ptr %a.ptr, align 64601 %shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>602 %concat = shufflevector <2 x i64> %shuffle, <2 x i64> %shuffle, <4 x i32> <i32 0, i32 1, i32 2, i32 3>603 store <4 x i64> %concat, ptr %dst, align 64604 ret void605}606 607define void @concat_aaa_to_shuf_of_a(ptr %a.ptr, ptr %dst) {608; SSE-LABEL: concat_aaa_to_shuf_of_a:609; SSE: # %bb.0:610; SSE-NEXT: movdqa (%rdi), %xmm0611; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]612; SSE-NEXT: movdqa %xmm0, 32(%rsi)613; SSE-NEXT: movdqa %xmm0, 48(%rsi)614; SSE-NEXT: movdqa %xmm0, 16(%rsi)615; SSE-NEXT: movdqa %xmm1, (%rsi)616; SSE-NEXT: retq617;618; AVX-LABEL: concat_aaa_to_shuf_of_a:619; AVX: # %bb.0:620; AVX-NEXT: vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]621; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1]622; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm1623; AVX-NEXT: vmovaps %ymm0, 32(%rsi)624; AVX-NEXT: vmovaps %ymm1, (%rsi)625; AVX-NEXT: vzeroupper626; AVX-NEXT: retq627;628; AVX2-LABEL: concat_aaa_to_shuf_of_a:629; AVX2: # %bb.0:630; AVX2-NEXT: vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]631; AVX2-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[1,0,0,1]632; AVX2-NEXT: vmovaps %ymm0, 32(%rsi)633; AVX2-NEXT: vmovaps %ymm1, (%rsi)634; AVX2-NEXT: vzeroupper635; AVX2-NEXT: retq636;637; AVX512F-LABEL: concat_aaa_to_shuf_of_a:638; AVX512F: # %bb.0:639; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]640; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]641; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm1642; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0643; AVX512F-NEXT: vmovdqa64 %zmm0, (%rsi)644; AVX512F-NEXT: vzeroupper645; AVX512F-NEXT: retq646;647; AVX512BW-LABEL: concat_aaa_to_shuf_of_a:648; AVX512BW: # %bb.0:649; AVX512BW-NEXT: vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]650; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]651; AVX512BW-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm1652; AVX512BW-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0653; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rsi)654; AVX512BW-NEXT: vzeroupper655; AVX512BW-NEXT: retq656 %a = load <2 x i64>, ptr %a.ptr, align 64657 %shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>658 %concat01 = shufflevector <2 x i64> %shuffle, <2 x i64> %a, <4 x i32> <i32 0, i32 1, i32 2, i32 3>659 %concat23 = shufflevector <2 x i64> %a, <2 x i64> %a, <4 x i32> <i32 0, i32 1, i32 2, i32 3>660 %concat = shufflevector <4 x i64> %concat01, <4 x i64> %concat23, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>661 store <8 x i64> %concat, ptr %dst, align 64662 ret void663}664define void @concat_shuf_of_a_to_aaa(ptr %a.ptr, ptr %dst) {665; SSE-LABEL: concat_shuf_of_a_to_aaa:666; SSE: # %bb.0:667; SSE-NEXT: movdqa (%rdi), %xmm0668; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]669; SSE-NEXT: movdqa %xmm0, 32(%rsi)670; SSE-NEXT: movdqa %xmm0, 16(%rsi)671; SSE-NEXT: movdqa %xmm0, (%rsi)672; SSE-NEXT: movdqa %xmm1, 48(%rsi)673; SSE-NEXT: retq674;675; AVX-LABEL: concat_shuf_of_a_to_aaa:676; AVX: # %bb.0:677; AVX-NEXT: vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]678; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1]679; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1680; AVX-NEXT: vmovaps %ymm0, (%rsi)681; AVX-NEXT: vmovaps %ymm1, 32(%rsi)682; AVX-NEXT: vzeroupper683; AVX-NEXT: retq684;685; AVX2-LABEL: concat_shuf_of_a_to_aaa:686; AVX2: # %bb.0:687; AVX2-NEXT: vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]688; AVX2-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[0,1,1,0]689; AVX2-NEXT: vmovaps %ymm0, (%rsi)690; AVX2-NEXT: vmovaps %ymm1, 32(%rsi)691; AVX2-NEXT: vzeroupper692; AVX2-NEXT: retq693;694; AVX512F-LABEL: concat_shuf_of_a_to_aaa:695; AVX512F: # %bb.0:696; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]697; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]698; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1699; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0700; AVX512F-NEXT: vmovdqa64 %zmm0, (%rsi)701; AVX512F-NEXT: vzeroupper702; AVX512F-NEXT: retq703;704; AVX512BW-LABEL: concat_shuf_of_a_to_aaa:705; AVX512BW: # %bb.0:706; AVX512BW-NEXT: vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1]707; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]708; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1709; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0710; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rsi)711; AVX512BW-NEXT: vzeroupper712; AVX512BW-NEXT: retq713 %a = load <2 x i64>, ptr %a.ptr, align 64714 %shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>715 %concat01 = shufflevector <2 x i64> %a, <2 x i64> %a, <4 x i32> <i32 0, i32 1, i32 2, i32 3>716 %concat23 = shufflevector <2 x i64> %a, <2 x i64> %shuffle, <4 x i32> <i32 0, i32 1, i32 2, i32 3>717 %concat = shufflevector <4 x i64> %concat01, <4 x i64> %concat23, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>718 store <8 x i64> %concat, ptr %dst, align 64719 ret void720}721;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:722; AVX1-ONLY: {{.*}}723; AVX2-FAST: {{.*}}724; AVX2-FAST-PERLANE: {{.*}}725; AVX2-SLOW: {{.*}}726; AVX512BW-FAST: {{.*}}727; AVX512BW-SLOW: {{.*}}728; AVX512F-FAST: {{.*}}729; AVX512F-SLOW: {{.*}}730; FALLBACK0: {{.*}}731; FALLBACK1: {{.*}}732; FALLBACK2: {{.*}}733; FALLBACK3: {{.*}}734; FALLBACK4: {{.*}}735; FALLBACK5: {{.*}}736; FALLBACK6: {{.*}}737; FALLBACK7: {{.*}}738; FALLBACK8: {{.*}}739; FALLBACK9: {{.*}}740