995 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -fast-isel -mtriple=i386-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=CHECK,X863; RUN: llc < %s -fast-isel -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=CHECK,X644 5; NOTE: This should use IR equivalent to what is generated by clang/test/CodeGen/avx512vbmi2-builtins.c6 7define <8 x i64> @test_mm512_mask_compress_epi16(<8 x i64> %__S, i32 %__U, <8 x i64> %__D) {8; X86-LABEL: test_mm512_mask_compress_epi16:9; X86: # %bb.0: # %entry10; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k111; X86-NEXT: vpcompressw %zmm1, %zmm0 {%k1}12; X86-NEXT: retl13;14; X64-LABEL: test_mm512_mask_compress_epi16:15; X64: # %bb.0: # %entry16; X64-NEXT: kmovd %edi, %k117; X64-NEXT: vpcompressw %zmm1, %zmm0 {%k1}18; X64-NEXT: retq19entry:20 %0 = bitcast <8 x i64> %__D to <32 x i16>21 %1 = bitcast <8 x i64> %__S to <32 x i16>22 %2 = tail call <32 x i16> @llvm.x86.avx512.mask.compress.w.512(<32 x i16> %0, <32 x i16> %1, i32 %__U)23 %3 = bitcast <32 x i16> %2 to <8 x i64>24 ret <8 x i64> %325}26 27define <8 x i64> @test_mm512_maskz_compress_epi16(i32 %__U, <8 x i64> %__D) {28; X86-LABEL: test_mm512_maskz_compress_epi16:29; X86: # %bb.0: # %entry30; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k131; X86-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}32; X86-NEXT: retl33;34; X64-LABEL: test_mm512_maskz_compress_epi16:35; X64: # %bb.0: # %entry36; X64-NEXT: kmovd %edi, %k137; X64-NEXT: vpcompressw %zmm0, %zmm0 {%k1} {z}38; X64-NEXT: retq39entry:40 %0 = bitcast <8 x i64> %__D to <32 x i16>41 %1 = tail call <32 x i16> @llvm.x86.avx512.mask.compress.w.512(<32 x i16> %0, <32 x i16> zeroinitializer, i32 %__U)42 %2 = bitcast <32 x i16> %1 to <8 x i64>43 ret <8 x i64> %244}45 46define <8 x i64> @test_mm512_mask_compress_epi8(<8 x i64> %__S, i64 %__U, <8 x i64> %__D) {47; X86-LABEL: test_mm512_mask_compress_epi8:48; X86: # %bb.0: # %entry49; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k050; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k151; X86-NEXT: kunpckdq %k1, %k0, %k152; X86-NEXT: vpcompressb %zmm1, %zmm0 {%k1}53; X86-NEXT: retl54;55; X64-LABEL: test_mm512_mask_compress_epi8:56; X64: # %bb.0: # %entry57; X64-NEXT: kmovq %rdi, %k158; X64-NEXT: vpcompressb %zmm1, %zmm0 {%k1}59; X64-NEXT: retq60entry:61 %0 = bitcast <8 x i64> %__D to <64 x i8>62 %1 = bitcast <8 x i64> %__S to <64 x i8>63 %2 = tail call <64 x i8> @llvm.x86.avx512.mask.compress.b.512(<64 x i8> %0, <64 x i8> %1, i64 %__U)64 %3 = bitcast <64 x i8> %2 to <8 x i64>65 ret <8 x i64> %366}67 68define <8 x i64> @test_mm512_maskz_compress_epi8(i64 %__U, <8 x i64> %__D) {69; X86-LABEL: test_mm512_maskz_compress_epi8:70; X86: # %bb.0: # %entry71; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k072; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k173; X86-NEXT: kunpckdq %k1, %k0, %k174; X86-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}75; X86-NEXT: retl76;77; X64-LABEL: test_mm512_maskz_compress_epi8:78; X64: # %bb.0: # %entry79; X64-NEXT: kmovq %rdi, %k180; X64-NEXT: vpcompressb %zmm0, %zmm0 {%k1} {z}81; X64-NEXT: retq82entry:83 %0 = bitcast <8 x i64> %__D to <64 x i8>84 %1 = tail call <64 x i8> @llvm.x86.avx512.mask.compress.b.512(<64 x i8> %0, <64 x i8> zeroinitializer, i64 %__U)85 %2 = bitcast <64 x i8> %1 to <8 x i64>86 ret <8 x i64> %287}88 89define void @test_mm512_mask_compressstoreu_epi16(ptr %__P, i32 %__U, <8 x i64> %__D) {90; X86-LABEL: test_mm512_mask_compressstoreu_epi16:91; X86: # %bb.0: # %entry92; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k193; X86-NEXT: movl {{[0-9]+}}(%esp), %eax94; X86-NEXT: vpcompressw %zmm0, (%eax) {%k1}95; X86-NEXT: vzeroupper96; X86-NEXT: retl97;98; X64-LABEL: test_mm512_mask_compressstoreu_epi16:99; X64: # %bb.0: # %entry100; X64-NEXT: kmovd %esi, %k1101; X64-NEXT: vpcompressw %zmm0, (%rdi) {%k1}102; X64-NEXT: vzeroupper103; X64-NEXT: retq104entry:105 %0 = bitcast <8 x i64> %__D to <32 x i16>106 %1 = bitcast i32 %__U to <32 x i1>107 tail call void @llvm.masked.compressstore.v32i16(<32 x i16> %0, ptr %__P, <32 x i1> %1)108 ret void109}110 111define void @test_mm512_mask_compressstoreu_epi8(ptr %__P, i64 %__U, <8 x i64> %__D) {112; X86-LABEL: test_mm512_mask_compressstoreu_epi8:113; X86: # %bb.0: # %entry114; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k0115; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1116; X86-NEXT: movl {{[0-9]+}}(%esp), %eax117; X86-NEXT: kunpckdq %k1, %k0, %k1118; X86-NEXT: vpcompressb %zmm0, (%eax) {%k1}119; X86-NEXT: vzeroupper120; X86-NEXT: retl121;122; X64-LABEL: test_mm512_mask_compressstoreu_epi8:123; X64: # %bb.0: # %entry124; X64-NEXT: kmovq %rsi, %k1125; X64-NEXT: vpcompressb %zmm0, (%rdi) {%k1}126; X64-NEXT: vzeroupper127; X64-NEXT: retq128entry:129 %0 = bitcast <8 x i64> %__D to <64 x i8>130 %1 = bitcast i64 %__U to <64 x i1>131 tail call void @llvm.masked.compressstore.v64i8(<64 x i8> %0, ptr %__P, <64 x i1> %1)132 ret void133}134 135define <8 x i64> @test_mm512_mask_expand_epi16(<8 x i64> %__S, i32 %__U, <8 x i64> %__D) {136; X86-LABEL: test_mm512_mask_expand_epi16:137; X86: # %bb.0: # %entry138; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1139; X86-NEXT: vpexpandw %zmm1, %zmm0 {%k1}140; X86-NEXT: retl141;142; X64-LABEL: test_mm512_mask_expand_epi16:143; X64: # %bb.0: # %entry144; X64-NEXT: kmovd %edi, %k1145; X64-NEXT: vpexpandw %zmm1, %zmm0 {%k1}146; X64-NEXT: retq147entry:148 %0 = bitcast <8 x i64> %__D to <32 x i16>149 %1 = bitcast <8 x i64> %__S to <32 x i16>150 %2 = tail call <32 x i16> @llvm.x86.avx512.mask.expand.w.512(<32 x i16> %0, <32 x i16> %1, i32 %__U)151 %3 = bitcast <32 x i16> %2 to <8 x i64>152 ret <8 x i64> %3153}154 155define <8 x i64> @test_mm512_maskz_expand_epi16(i32 %__U, <8 x i64> %__D) {156; X86-LABEL: test_mm512_maskz_expand_epi16:157; X86: # %bb.0: # %entry158; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1159; X86-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}160; X86-NEXT: retl161;162; X64-LABEL: test_mm512_maskz_expand_epi16:163; X64: # %bb.0: # %entry164; X64-NEXT: kmovd %edi, %k1165; X64-NEXT: vpexpandw %zmm0, %zmm0 {%k1} {z}166; X64-NEXT: retq167entry:168 %0 = bitcast <8 x i64> %__D to <32 x i16>169 %1 = tail call <32 x i16> @llvm.x86.avx512.mask.expand.w.512(<32 x i16> %0, <32 x i16> zeroinitializer, i32 %__U)170 %2 = bitcast <32 x i16> %1 to <8 x i64>171 ret <8 x i64> %2172}173 174define <8 x i64> @test_mm512_mask_expand_epi8(<8 x i64> %__S, i64 %__U, <8 x i64> %__D) {175; X86-LABEL: test_mm512_mask_expand_epi8:176; X86: # %bb.0: # %entry177; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k0178; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1179; X86-NEXT: kunpckdq %k1, %k0, %k1180; X86-NEXT: vpexpandb %zmm1, %zmm0 {%k1}181; X86-NEXT: retl182;183; X64-LABEL: test_mm512_mask_expand_epi8:184; X64: # %bb.0: # %entry185; X64-NEXT: kmovq %rdi, %k1186; X64-NEXT: vpexpandb %zmm1, %zmm0 {%k1}187; X64-NEXT: retq188entry:189 %0 = bitcast <8 x i64> %__D to <64 x i8>190 %1 = bitcast <8 x i64> %__S to <64 x i8>191 %2 = tail call <64 x i8> @llvm.x86.avx512.mask.expand.b.512(<64 x i8> %0, <64 x i8> %1, i64 %__U)192 %3 = bitcast <64 x i8> %2 to <8 x i64>193 ret <8 x i64> %3194}195 196define <8 x i64> @test_mm512_maskz_expand_epi8(i64 %__U, <8 x i64> %__D) {197; X86-LABEL: test_mm512_maskz_expand_epi8:198; X86: # %bb.0: # %entry199; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k0200; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1201; X86-NEXT: kunpckdq %k1, %k0, %k1202; X86-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}203; X86-NEXT: retl204;205; X64-LABEL: test_mm512_maskz_expand_epi8:206; X64: # %bb.0: # %entry207; X64-NEXT: kmovq %rdi, %k1208; X64-NEXT: vpexpandb %zmm0, %zmm0 {%k1} {z}209; X64-NEXT: retq210entry:211 %0 = bitcast <8 x i64> %__D to <64 x i8>212 %1 = tail call <64 x i8> @llvm.x86.avx512.mask.expand.b.512(<64 x i8> %0, <64 x i8> zeroinitializer, i64 %__U)213 %2 = bitcast <64 x i8> %1 to <8 x i64>214 ret <8 x i64> %2215}216 217define <8 x i64> @test_mm512_mask_expandloadu_epi16(<8 x i64> %__S, i32 %__U, ptr readonly %__P) {218; X86-LABEL: test_mm512_mask_expandloadu_epi16:219; X86: # %bb.0: # %entry220; X86-NEXT: movl {{[0-9]+}}(%esp), %eax221; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1222; X86-NEXT: vpexpandw (%eax), %zmm0 {%k1}223; X86-NEXT: retl224;225; X64-LABEL: test_mm512_mask_expandloadu_epi16:226; X64: # %bb.0: # %entry227; X64-NEXT: kmovd %edi, %k1228; X64-NEXT: vpexpandw (%rsi), %zmm0 {%k1}229; X64-NEXT: retq230entry:231 %0 = bitcast <8 x i64> %__S to <32 x i16>232 %1 = bitcast i32 %__U to <32 x i1>233 %2 = tail call <32 x i16> @llvm.masked.expandload.v32i16(ptr %__P, <32 x i1> %1, <32 x i16> %0)234 %3 = bitcast <32 x i16> %2 to <8 x i64>235 ret <8 x i64> %3236}237 238define <8 x i64> @test_mm512_maskz_expandloadu_epi16(i32 %__U, ptr readonly %__P) {239; X86-LABEL: test_mm512_maskz_expandloadu_epi16:240; X86: # %bb.0: # %entry241; X86-NEXT: movl {{[0-9]+}}(%esp), %eax242; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1243; X86-NEXT: vpexpandw (%eax), %zmm0 {%k1} {z}244; X86-NEXT: retl245;246; X64-LABEL: test_mm512_maskz_expandloadu_epi16:247; X64: # %bb.0: # %entry248; X64-NEXT: kmovd %edi, %k1249; X64-NEXT: vpexpandw (%rsi), %zmm0 {%k1} {z}250; X64-NEXT: retq251entry:252 %0 = bitcast i32 %__U to <32 x i1>253 %1 = tail call <32 x i16> @llvm.masked.expandload.v32i16(ptr %__P, <32 x i1> %0, <32 x i16> zeroinitializer)254 %2 = bitcast <32 x i16> %1 to <8 x i64>255 ret <8 x i64> %2256}257 258define <8 x i64> @test_mm512_mask_expandloadu_epi8(<8 x i64> %__S, i64 %__U, ptr readonly %__P) {259; X86-LABEL: test_mm512_mask_expandloadu_epi8:260; X86: # %bb.0: # %entry261; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k0262; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1263; X86-NEXT: movl {{[0-9]+}}(%esp), %eax264; X86-NEXT: kunpckdq %k1, %k0, %k1265; X86-NEXT: vpexpandb (%eax), %zmm0 {%k1}266; X86-NEXT: retl267;268; X64-LABEL: test_mm512_mask_expandloadu_epi8:269; X64: # %bb.0: # %entry270; X64-NEXT: kmovq %rdi, %k1271; X64-NEXT: vpexpandb (%rsi), %zmm0 {%k1}272; X64-NEXT: retq273entry:274 %0 = bitcast <8 x i64> %__S to <64 x i8>275 %1 = bitcast i64 %__U to <64 x i1>276 %2 = tail call <64 x i8> @llvm.masked.expandload.v64i8(ptr %__P, <64 x i1> %1, <64 x i8> %0)277 %3 = bitcast <64 x i8> %2 to <8 x i64>278 ret <8 x i64> %3279}280 281define <8 x i64> @test_mm512_maskz_expandloadu_epi8(i64 %__U, ptr readonly %__P) {282; X86-LABEL: test_mm512_maskz_expandloadu_epi8:283; X86: # %bb.0: # %entry284; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k0285; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1286; X86-NEXT: movl {{[0-9]+}}(%esp), %eax287; X86-NEXT: kunpckdq %k1, %k0, %k1288; X86-NEXT: vpexpandb (%eax), %zmm0 {%k1} {z}289; X86-NEXT: retl290;291; X64-LABEL: test_mm512_maskz_expandloadu_epi8:292; X64: # %bb.0: # %entry293; X64-NEXT: kmovq %rdi, %k1294; X64-NEXT: vpexpandb (%rsi), %zmm0 {%k1} {z}295; X64-NEXT: retq296entry:297 %0 = bitcast i64 %__U to <64 x i1>298 %1 = tail call <64 x i8> @llvm.masked.expandload.v64i8(ptr %__P, <64 x i1> %0, <64 x i8> zeroinitializer)299 %2 = bitcast <64 x i8> %1 to <8 x i64>300 ret <8 x i64> %2301}302 303define <8 x i64> @test_mm512_mask_shldi_epi64(<8 x i64> %__S, i8 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {304; X86-LABEL: test_mm512_mask_shldi_epi64:305; X86: # %bb.0: # %entry306; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax307; X86-NEXT: kmovd %eax, %k1308; X86-NEXT: vpshldq $47, %zmm2, %zmm1, %zmm0 {%k1}309; X86-NEXT: retl310;311; X64-LABEL: test_mm512_mask_shldi_epi64:312; X64: # %bb.0: # %entry313; X64-NEXT: kmovd %edi, %k1314; X64-NEXT: vpshldq $47, %zmm2, %zmm1, %zmm0 {%k1}315; X64-NEXT: retq316entry:317 %0 = tail call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %__A, <8 x i64> %__B, <8 x i64> <i64 47, i64 47, i64 47, i64 47, i64 47, i64 47, i64 47, i64 47>)318 %1 = bitcast i8 %__U to <8 x i1>319 %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> %__S320 ret <8 x i64> %2321}322 323declare <8 x i64> @llvm.fshl.v8i64(<8 x i64>, <8 x i64>, <8 x i64>)324 325define <8 x i64> @test_mm512_maskz_shldi_epi64(i8 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {326; X86-LABEL: test_mm512_maskz_shldi_epi64:327; X86: # %bb.0: # %entry328; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax329; X86-NEXT: kmovd %eax, %k1330; X86-NEXT: vpshldq $63, %zmm1, %zmm0, %zmm0 {%k1} {z}331; X86-NEXT: retl332;333; X64-LABEL: test_mm512_maskz_shldi_epi64:334; X64: # %bb.0: # %entry335; X64-NEXT: kmovd %edi, %k1336; X64-NEXT: vpshldq $63, %zmm1, %zmm0, %zmm0 {%k1} {z}337; X64-NEXT: retq338entry:339 %0 = tail call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %__A, <8 x i64> %__B, <8 x i64> <i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63>)340 %1 = bitcast i8 %__U to <8 x i1>341 %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> zeroinitializer342 ret <8 x i64> %2343}344 345define <8 x i64> @test_mm512_shldi_epi64(<8 x i64> %__A, <8 x i64> %__B) {346; CHECK-LABEL: test_mm512_shldi_epi64:347; CHECK: # %bb.0: # %entry348; CHECK-NEXT: vpshldq $31, %zmm1, %zmm0, %zmm0349; CHECK-NEXT: ret{{[l|q]}}350entry:351 %0 = tail call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %__A, <8 x i64> %__B, <8 x i64> <i64 31, i64 31, i64 31, i64 31, i64 31, i64 31, i64 31, i64 31>)352 ret <8 x i64> %0353}354 355define <8 x i64> @test_mm512_mask_shldi_epi32(<8 x i64> %__S, i16 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {356; X86-LABEL: test_mm512_mask_shldi_epi32:357; X86: # %bb.0: # %entry358; X86-NEXT: kmovw {{[0-9]+}}(%esp), %k1359; X86-NEXT: vpshldd $7, %zmm2, %zmm1, %zmm0 {%k1}360; X86-NEXT: retl361;362; X64-LABEL: test_mm512_mask_shldi_epi32:363; X64: # %bb.0: # %entry364; X64-NEXT: kmovd %edi, %k1365; X64-NEXT: vpshldd $7, %zmm2, %zmm1, %zmm0 {%k1}366; X64-NEXT: retq367entry:368 %0 = bitcast <8 x i64> %__A to <16 x i32>369 %1 = bitcast <8 x i64> %__B to <16 x i32>370 %2 = tail call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %0, <16 x i32> %1, <16 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>)371 %3 = bitcast <8 x i64> %__S to <16 x i32>372 %4 = bitcast i16 %__U to <16 x i1>373 %5 = select <16 x i1> %4, <16 x i32> %2, <16 x i32> %3374 %6 = bitcast <16 x i32> %5 to <8 x i64>375 ret <8 x i64> %6376}377 378declare <16 x i32> @llvm.fshl.v16i32(<16 x i32>, <16 x i32>, <16 x i32>)379 380define <8 x i64> @test_mm512_maskz_shldi_epi32(i16 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {381; X86-LABEL: test_mm512_maskz_shldi_epi32:382; X86: # %bb.0: # %entry383; X86-NEXT: kmovw {{[0-9]+}}(%esp), %k1384; X86-NEXT: vpshldd $15, %zmm1, %zmm0, %zmm0 {%k1} {z}385; X86-NEXT: retl386;387; X64-LABEL: test_mm512_maskz_shldi_epi32:388; X64: # %bb.0: # %entry389; X64-NEXT: kmovd %edi, %k1390; X64-NEXT: vpshldd $15, %zmm1, %zmm0, %zmm0 {%k1} {z}391; X64-NEXT: retq392entry:393 %0 = bitcast <8 x i64> %__A to <16 x i32>394 %1 = bitcast <8 x i64> %__B to <16 x i32>395 %2 = tail call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %0, <16 x i32> %1, <16 x i32> <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>)396 %3 = bitcast i16 %__U to <16 x i1>397 %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer398 %5 = bitcast <16 x i32> %4 to <8 x i64>399 ret <8 x i64> %5400}401 402define <8 x i64> @test_mm512_shldi_epi32(<8 x i64> %__A, <8 x i64> %__B) {403; CHECK-LABEL: test_mm512_shldi_epi32:404; CHECK: # %bb.0: # %entry405; CHECK-NEXT: vpshldd $31, %zmm1, %zmm0, %zmm0406; CHECK-NEXT: ret{{[l|q]}}407entry:408 %0 = bitcast <8 x i64> %__A to <16 x i32>409 %1 = bitcast <8 x i64> %__B to <16 x i32>410 %2 = tail call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %0, <16 x i32> %1, <16 x i32> <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>)411 %3 = bitcast <16 x i32> %2 to <8 x i64>412 ret <8 x i64> %3413}414 415define <8 x i64> @test_mm512_mask_shldi_epi16(<8 x i64> %__S, i32 %__U, <8 x i64> %__A, <8 x i64> %__B) {416; X86-LABEL: test_mm512_mask_shldi_epi16:417; X86: # %bb.0: # %entry418; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1419; X86-NEXT: vpshldw $3, %zmm2, %zmm1, %zmm0 {%k1}420; X86-NEXT: retl421;422; X64-LABEL: test_mm512_mask_shldi_epi16:423; X64: # %bb.0: # %entry424; X64-NEXT: kmovd %edi, %k1425; X64-NEXT: vpshldw $3, %zmm2, %zmm1, %zmm0 {%k1}426; X64-NEXT: retq427entry:428 %0 = bitcast <8 x i64> %__A to <32 x i16>429 %1 = bitcast <8 x i64> %__B to <32 x i16>430 %2 = tail call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %0, <32 x i16> %1, <32 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)431 %3 = bitcast <8 x i64> %__S to <32 x i16>432 %4 = bitcast i32 %__U to <32 x i1>433 %5 = select <32 x i1> %4, <32 x i16> %2, <32 x i16> %3434 %6 = bitcast <32 x i16> %5 to <8 x i64>435 ret <8 x i64> %6436}437 438declare <32 x i16> @llvm.fshl.v32i16(<32 x i16>, <32 x i16>, <32 x i16>)439 440define <8 x i64> @test_mm512_maskz_shldi_epi16(i32 %__U, <8 x i64> %__A, <8 x i64> %__B) {441; X86-LABEL: test_mm512_maskz_shldi_epi16:442; X86: # %bb.0: # %entry443; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1444; X86-NEXT: vpshldw $7, %zmm1, %zmm0, %zmm0 {%k1} {z}445; X86-NEXT: retl446;447; X64-LABEL: test_mm512_maskz_shldi_epi16:448; X64: # %bb.0: # %entry449; X64-NEXT: kmovd %edi, %k1450; X64-NEXT: vpshldw $7, %zmm1, %zmm0, %zmm0 {%k1} {z}451; X64-NEXT: retq452entry:453 %0 = bitcast <8 x i64> %__A to <32 x i16>454 %1 = bitcast <8 x i64> %__B to <32 x i16>455 %2 = tail call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %0, <32 x i16> %1, <32 x i16> <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>)456 %3 = bitcast i32 %__U to <32 x i1>457 %4 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> zeroinitializer458 %5 = bitcast <32 x i16> %4 to <8 x i64>459 ret <8 x i64> %5460}461 462define <8 x i64> @test_mm512_shldi_epi16(<8 x i64> %__A, <8 x i64> %__B) {463; CHECK-LABEL: test_mm512_shldi_epi16:464; CHECK: # %bb.0: # %entry465; CHECK-NEXT: vpshldw $15, %zmm1, %zmm0, %zmm0466; CHECK-NEXT: ret{{[l|q]}}467entry:468 %0 = bitcast <8 x i64> %__A to <32 x i16>469 %1 = bitcast <8 x i64> %__B to <32 x i16>470 %2 = tail call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %0, <32 x i16> %1, <32 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)471 %3 = bitcast <32 x i16> %2 to <8 x i64>472 ret <8 x i64> %3473}474 475define <8 x i64> @test_mm512_mask_shrdi_epi64(<8 x i64> %__S, i8 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {476; X86-LABEL: test_mm512_mask_shrdi_epi64:477; X86: # %bb.0: # %entry478; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax479; X86-NEXT: kmovd %eax, %k1480; X86-NEXT: vpshrdq $47, %zmm2, %zmm1, %zmm0 {%k1}481; X86-NEXT: retl482;483; X64-LABEL: test_mm512_mask_shrdi_epi64:484; X64: # %bb.0: # %entry485; X64-NEXT: kmovd %edi, %k1486; X64-NEXT: vpshrdq $47, %zmm2, %zmm1, %zmm0 {%k1}487; X64-NEXT: retq488entry:489 %0 = tail call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %__B, <8 x i64> %__A, <8 x i64> <i64 47, i64 47, i64 47, i64 47, i64 47, i64 47, i64 47, i64 47>)490 %1 = bitcast i8 %__U to <8 x i1>491 %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> %__S492 ret <8 x i64> %2493}494 495declare <8 x i64> @llvm.fshr.v8i64(<8 x i64>, <8 x i64>, <8 x i64>)496 497define <8 x i64> @test_mm512_maskz_shrdi_epi64(i8 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {498; X86-LABEL: test_mm512_maskz_shrdi_epi64:499; X86: # %bb.0: # %entry500; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax501; X86-NEXT: kmovd %eax, %k1502; X86-NEXT: vpshrdq $63, %zmm1, %zmm0, %zmm0 {%k1} {z}503; X86-NEXT: retl504;505; X64-LABEL: test_mm512_maskz_shrdi_epi64:506; X64: # %bb.0: # %entry507; X64-NEXT: kmovd %edi, %k1508; X64-NEXT: vpshrdq $63, %zmm1, %zmm0, %zmm0 {%k1} {z}509; X64-NEXT: retq510entry:511 %0 = tail call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %__B, <8 x i64> %__A, <8 x i64> <i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63>)512 %1 = bitcast i8 %__U to <8 x i1>513 %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> zeroinitializer514 ret <8 x i64> %2515}516 517define <8 x i64> @test_mm512_shrdi_epi64(<8 x i64> %__A, <8 x i64> %__B) {518; CHECK-LABEL: test_mm512_shrdi_epi64:519; CHECK: # %bb.0: # %entry520; CHECK-NEXT: vpshrdq $31, %zmm1, %zmm0, %zmm0521; CHECK-NEXT: ret{{[l|q]}}522entry:523 %0 = tail call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %__B, <8 x i64> %__A, <8 x i64> <i64 31, i64 31, i64 31, i64 31, i64 31, i64 31, i64 31, i64 31>)524 ret <8 x i64> %0525}526 527define <8 x i64> @test_mm512_mask_shrdi_epi32(<8 x i64> %__S, i16 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {528; X86-LABEL: test_mm512_mask_shrdi_epi32:529; X86: # %bb.0: # %entry530; X86-NEXT: kmovw {{[0-9]+}}(%esp), %k1531; X86-NEXT: vpshrdd $7, %zmm2, %zmm1, %zmm0 {%k1}532; X86-NEXT: retl533;534; X64-LABEL: test_mm512_mask_shrdi_epi32:535; X64: # %bb.0: # %entry536; X64-NEXT: kmovd %edi, %k1537; X64-NEXT: vpshrdd $7, %zmm2, %zmm1, %zmm0 {%k1}538; X64-NEXT: retq539entry:540 %0 = bitcast <8 x i64> %__A to <16 x i32>541 %1 = bitcast <8 x i64> %__B to <16 x i32>542 %2 = tail call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %1, <16 x i32> %0, <16 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>)543 %3 = bitcast <8 x i64> %__S to <16 x i32>544 %4 = bitcast i16 %__U to <16 x i1>545 %5 = select <16 x i1> %4, <16 x i32> %2, <16 x i32> %3546 %6 = bitcast <16 x i32> %5 to <8 x i64>547 ret <8 x i64> %6548}549 550declare <16 x i32> @llvm.fshr.v16i32(<16 x i32>, <16 x i32>, <16 x i32>)551 552define <8 x i64> @test_mm512_maskz_shrdi_epi32(i16 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {553; X86-LABEL: test_mm512_maskz_shrdi_epi32:554; X86: # %bb.0: # %entry555; X86-NEXT: kmovw {{[0-9]+}}(%esp), %k1556; X86-NEXT: vpshrdd $15, %zmm1, %zmm0, %zmm0 {%k1} {z}557; X86-NEXT: retl558;559; X64-LABEL: test_mm512_maskz_shrdi_epi32:560; X64: # %bb.0: # %entry561; X64-NEXT: kmovd %edi, %k1562; X64-NEXT: vpshrdd $15, %zmm1, %zmm0, %zmm0 {%k1} {z}563; X64-NEXT: retq564entry:565 %0 = bitcast <8 x i64> %__A to <16 x i32>566 %1 = bitcast <8 x i64> %__B to <16 x i32>567 %2 = tail call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %1, <16 x i32> %0, <16 x i32> <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>)568 %3 = bitcast i16 %__U to <16 x i1>569 %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer570 %5 = bitcast <16 x i32> %4 to <8 x i64>571 ret <8 x i64> %5572}573 574define <8 x i64> @test_mm512_shrdi_epi32(<8 x i64> %__A, <8 x i64> %__B) {575; CHECK-LABEL: test_mm512_shrdi_epi32:576; CHECK: # %bb.0: # %entry577; CHECK-NEXT: vpshrdd $31, %zmm1, %zmm0, %zmm0578; CHECK-NEXT: ret{{[l|q]}}579entry:580 %0 = bitcast <8 x i64> %__A to <16 x i32>581 %1 = bitcast <8 x i64> %__B to <16 x i32>582 %2 = tail call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %1, <16 x i32> %0, <16 x i32> <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>)583 %3 = bitcast <16 x i32> %2 to <8 x i64>584 ret <8 x i64> %3585}586 587define <8 x i64> @test_mm512_mask_shrdi_epi16(<8 x i64> %__S, i32 %__U, <8 x i64> %__A, <8 x i64> %__B) {588; X86-LABEL: test_mm512_mask_shrdi_epi16:589; X86: # %bb.0: # %entry590; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1591; X86-NEXT: vpshrdw $3, %zmm2, %zmm1, %zmm0 {%k1}592; X86-NEXT: retl593;594; X64-LABEL: test_mm512_mask_shrdi_epi16:595; X64: # %bb.0: # %entry596; X64-NEXT: kmovd %edi, %k1597; X64-NEXT: vpshrdw $3, %zmm2, %zmm1, %zmm0 {%k1}598; X64-NEXT: retq599entry:600 %0 = bitcast <8 x i64> %__A to <32 x i16>601 %1 = bitcast <8 x i64> %__B to <32 x i16>602 %2 = tail call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %1, <32 x i16> %0, <32 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)603 %3 = bitcast <8 x i64> %__S to <32 x i16>604 %4 = bitcast i32 %__U to <32 x i1>605 %5 = select <32 x i1> %4, <32 x i16> %2, <32 x i16> %3606 %6 = bitcast <32 x i16> %5 to <8 x i64>607 ret <8 x i64> %6608}609 610declare <32 x i16> @llvm.fshr.v32i16(<32 x i16>, <32 x i16>, <32 x i16>)611 612define <8 x i64> @test_mm512_maskz_shrdi_epi16(i32 %__U, <8 x i64> %__A, <8 x i64> %__B) {613; X86-LABEL: test_mm512_maskz_shrdi_epi16:614; X86: # %bb.0: # %entry615; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1616; X86-NEXT: vpshrdw $15, %zmm1, %zmm0, %zmm0 {%k1} {z}617; X86-NEXT: retl618;619; X64-LABEL: test_mm512_maskz_shrdi_epi16:620; X64: # %bb.0: # %entry621; X64-NEXT: kmovd %edi, %k1622; X64-NEXT: vpshrdw $15, %zmm1, %zmm0, %zmm0 {%k1} {z}623; X64-NEXT: retq624entry:625 %0 = bitcast <8 x i64> %__A to <32 x i16>626 %1 = bitcast <8 x i64> %__B to <32 x i16>627 %2 = tail call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %1, <32 x i16> %0, <32 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)628 %3 = bitcast i32 %__U to <32 x i1>629 %4 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> zeroinitializer630 %5 = bitcast <32 x i16> %4 to <8 x i64>631 ret <8 x i64> %5632}633 634define <8 x i64> @test_mm512_shrdi_epi16(<8 x i64> %__A, <8 x i64> %__B) {635; CHECK-LABEL: test_mm512_shrdi_epi16:636; CHECK: # %bb.0: # %entry637; CHECK-NEXT: vpshrdw $15, %zmm1, %zmm0, %zmm0638; CHECK-NEXT: ret{{[l|q]}}639entry:640 %0 = bitcast <8 x i64> %__A to <32 x i16>641 %1 = bitcast <8 x i64> %__B to <32 x i16>642 %2 = tail call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %1, <32 x i16> %0, <32 x i16> <i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31>)643 %3 = bitcast <32 x i16> %2 to <8 x i64>644 ret <8 x i64> %3645}646 647define <8 x i64> @test_mm512_mask_shldv_epi64(<8 x i64> %__S, i8 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {648; X86-LABEL: test_mm512_mask_shldv_epi64:649; X86: # %bb.0: # %entry650; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax651; X86-NEXT: kmovd %eax, %k1652; X86-NEXT: vpshldvq %zmm2, %zmm1, %zmm0 {%k1}653; X86-NEXT: retl654;655; X64-LABEL: test_mm512_mask_shldv_epi64:656; X64: # %bb.0: # %entry657; X64-NEXT: kmovd %edi, %k1658; X64-NEXT: vpshldvq %zmm2, %zmm1, %zmm0 {%k1}659; X64-NEXT: retq660entry:661 %0 = tail call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B)662 %1 = bitcast i8 %__U to <8 x i1>663 %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> %__S664 ret <8 x i64> %2665}666 667define <8 x i64> @test_mm512_maskz_shldv_epi64(i8 zeroext %__U, <8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B) {668; X86-LABEL: test_mm512_maskz_shldv_epi64:669; X86: # %bb.0: # %entry670; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax671; X86-NEXT: kmovd %eax, %k1672; X86-NEXT: vpshldvq %zmm2, %zmm1, %zmm0 {%k1} {z}673; X86-NEXT: retl674;675; X64-LABEL: test_mm512_maskz_shldv_epi64:676; X64: # %bb.0: # %entry677; X64-NEXT: kmovd %edi, %k1678; X64-NEXT: vpshldvq %zmm2, %zmm1, %zmm0 {%k1} {z}679; X64-NEXT: retq680entry:681 %0 = tail call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B)682 %1 = bitcast i8 %__U to <8 x i1>683 %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> zeroinitializer684 ret <8 x i64> %2685}686 687define <8 x i64> @test_mm512_shldv_epi64(<8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B) {688; CHECK-LABEL: test_mm512_shldv_epi64:689; CHECK: # %bb.0: # %entry690; CHECK-NEXT: vpshldvq %zmm2, %zmm1, %zmm0691; CHECK-NEXT: ret{{[l|q]}}692entry:693 %0 = tail call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B)694 ret <8 x i64> %0695}696 697define <8 x i64> @test_mm512_mask_shldv_epi32(<8 x i64> %__S, i16 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {698; X86-LABEL: test_mm512_mask_shldv_epi32:699; X86: # %bb.0: # %entry700; X86-NEXT: kmovw {{[0-9]+}}(%esp), %k1701; X86-NEXT: vpshldvd %zmm2, %zmm1, %zmm0 {%k1}702; X86-NEXT: retl703;704; X64-LABEL: test_mm512_mask_shldv_epi32:705; X64: # %bb.0: # %entry706; X64-NEXT: kmovd %edi, %k1707; X64-NEXT: vpshldvd %zmm2, %zmm1, %zmm0 {%k1}708; X64-NEXT: retq709entry:710 %0 = bitcast <8 x i64> %__S to <16 x i32>711 %1 = bitcast <8 x i64> %__A to <16 x i32>712 %2 = bitcast <8 x i64> %__B to <16 x i32>713 %3 = tail call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %0, <16 x i32> %1, <16 x i32> %2)714 %4 = bitcast i16 %__U to <16 x i1>715 %5 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> %0716 %6 = bitcast <16 x i32> %5 to <8 x i64>717 ret <8 x i64> %6718}719 720define <8 x i64> @test_mm512_maskz_shldv_epi32(i16 zeroext %__U, <8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B) {721; X86-LABEL: test_mm512_maskz_shldv_epi32:722; X86: # %bb.0: # %entry723; X86-NEXT: kmovw {{[0-9]+}}(%esp), %k1724; X86-NEXT: vpshldvd %zmm2, %zmm1, %zmm0 {%k1} {z}725; X86-NEXT: retl726;727; X64-LABEL: test_mm512_maskz_shldv_epi32:728; X64: # %bb.0: # %entry729; X64-NEXT: kmovd %edi, %k1730; X64-NEXT: vpshldvd %zmm2, %zmm1, %zmm0 {%k1} {z}731; X64-NEXT: retq732entry:733 %0 = bitcast <8 x i64> %__S to <16 x i32>734 %1 = bitcast <8 x i64> %__A to <16 x i32>735 %2 = bitcast <8 x i64> %__B to <16 x i32>736 %3 = tail call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %0, <16 x i32> %1, <16 x i32> %2)737 %4 = bitcast i16 %__U to <16 x i1>738 %5 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> zeroinitializer739 %6 = bitcast <16 x i32> %5 to <8 x i64>740 ret <8 x i64> %6741}742 743define <8 x i64> @test_mm512_shldv_epi32(<8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B) {744; CHECK-LABEL: test_mm512_shldv_epi32:745; CHECK: # %bb.0: # %entry746; CHECK-NEXT: vpshldvd %zmm2, %zmm1, %zmm0747; CHECK-NEXT: ret{{[l|q]}}748entry:749 %0 = bitcast <8 x i64> %__S to <16 x i32>750 %1 = bitcast <8 x i64> %__A to <16 x i32>751 %2 = bitcast <8 x i64> %__B to <16 x i32>752 %3 = tail call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %0, <16 x i32> %1, <16 x i32> %2)753 %4 = bitcast <16 x i32> %3 to <8 x i64>754 ret <8 x i64> %4755}756 757define <8 x i64> @test_mm512_mask_shldv_epi16(<8 x i64> %__S, i32 %__U, <8 x i64> %__A, <8 x i64> %__B) {758; X86-LABEL: test_mm512_mask_shldv_epi16:759; X86: # %bb.0: # %entry760; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1761; X86-NEXT: vpshldvw %zmm2, %zmm1, %zmm0 {%k1}762; X86-NEXT: retl763;764; X64-LABEL: test_mm512_mask_shldv_epi16:765; X64: # %bb.0: # %entry766; X64-NEXT: kmovd %edi, %k1767; X64-NEXT: vpshldvw %zmm2, %zmm1, %zmm0 {%k1}768; X64-NEXT: retq769entry:770 %0 = bitcast <8 x i64> %__S to <32 x i16>771 %1 = bitcast <8 x i64> %__A to <32 x i16>772 %2 = bitcast <8 x i64> %__B to <32 x i16>773 %3 = tail call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %0, <32 x i16> %1, <32 x i16> %2)774 %4 = bitcast i32 %__U to <32 x i1>775 %5 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> %0776 %6 = bitcast <32 x i16> %5 to <8 x i64>777 ret <8 x i64> %6778}779 780define <8 x i64> @test_mm512_maskz_shldv_epi16(i32 %__U, <8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B) {781; X86-LABEL: test_mm512_maskz_shldv_epi16:782; X86: # %bb.0: # %entry783; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1784; X86-NEXT: vpshldvw %zmm2, %zmm1, %zmm0 {%k1} {z}785; X86-NEXT: retl786;787; X64-LABEL: test_mm512_maskz_shldv_epi16:788; X64: # %bb.0: # %entry789; X64-NEXT: kmovd %edi, %k1790; X64-NEXT: vpshldvw %zmm2, %zmm1, %zmm0 {%k1} {z}791; X64-NEXT: retq792entry:793 %0 = bitcast <8 x i64> %__S to <32 x i16>794 %1 = bitcast <8 x i64> %__A to <32 x i16>795 %2 = bitcast <8 x i64> %__B to <32 x i16>796 %3 = tail call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %0, <32 x i16> %1, <32 x i16> %2)797 %4 = bitcast i32 %__U to <32 x i1>798 %5 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> zeroinitializer799 %6 = bitcast <32 x i16> %5 to <8 x i64>800 ret <8 x i64> %6801}802 803define <8 x i64> @test_mm512_shldv_epi16(<8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B) {804; CHECK-LABEL: test_mm512_shldv_epi16:805; CHECK: # %bb.0: # %entry806; CHECK-NEXT: vpshldvw %zmm2, %zmm1, %zmm0807; CHECK-NEXT: ret{{[l|q]}}808entry:809 %0 = bitcast <8 x i64> %__S to <32 x i16>810 %1 = bitcast <8 x i64> %__A to <32 x i16>811 %2 = bitcast <8 x i64> %__B to <32 x i16>812 %3 = tail call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %0, <32 x i16> %1, <32 x i16> %2)813 %4 = bitcast <32 x i16> %3 to <8 x i64>814 ret <8 x i64> %4815}816 817define <8 x i64> @test_mm512_mask_shrdv_epi64(<8 x i64> %__S, i8 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {818; X86-LABEL: test_mm512_mask_shrdv_epi64:819; X86: # %bb.0: # %entry820; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax821; X86-NEXT: kmovd %eax, %k1822; X86-NEXT: vpshrdvq %zmm2, %zmm1, %zmm0 {%k1}823; X86-NEXT: retl824;825; X64-LABEL: test_mm512_mask_shrdv_epi64:826; X64: # %bb.0: # %entry827; X64-NEXT: kmovd %edi, %k1828; X64-NEXT: vpshrdvq %zmm2, %zmm1, %zmm0 {%k1}829; X64-NEXT: retq830entry:831 %0 = tail call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %__A, <8 x i64> %__S, <8 x i64> %__B)832 %1 = bitcast i8 %__U to <8 x i1>833 %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> %__S834 ret <8 x i64> %2835}836 837define <8 x i64> @test_mm512_maskz_shrdv_epi64(i8 zeroext %__U, <8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B) {838; X86-LABEL: test_mm512_maskz_shrdv_epi64:839; X86: # %bb.0: # %entry840; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax841; X86-NEXT: kmovd %eax, %k1842; X86-NEXT: vpshrdvq %zmm2, %zmm1, %zmm0 {%k1} {z}843; X86-NEXT: retl844;845; X64-LABEL: test_mm512_maskz_shrdv_epi64:846; X64: # %bb.0: # %entry847; X64-NEXT: kmovd %edi, %k1848; X64-NEXT: vpshrdvq %zmm2, %zmm1, %zmm0 {%k1} {z}849; X64-NEXT: retq850entry:851 %0 = tail call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %__A, <8 x i64> %__S, <8 x i64> %__B)852 %1 = bitcast i8 %__U to <8 x i1>853 %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> zeroinitializer854 ret <8 x i64> %2855}856 857define <8 x i64> @test_mm512_shrdv_epi64(<8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B) {858; CHECK-LABEL: test_mm512_shrdv_epi64:859; CHECK: # %bb.0: # %entry860; CHECK-NEXT: vpshrdvq %zmm2, %zmm1, %zmm0861; CHECK-NEXT: ret{{[l|q]}}862entry:863 %0 = tail call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %__A, <8 x i64> %__S, <8 x i64> %__B)864 ret <8 x i64> %0865}866 867define <8 x i64> @test_mm512_mask_shrdv_epi32(<8 x i64> %__S, i16 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {868; X86-LABEL: test_mm512_mask_shrdv_epi32:869; X86: # %bb.0: # %entry870; X86-NEXT: kmovw {{[0-9]+}}(%esp), %k1871; X86-NEXT: vpshrdvd %zmm2, %zmm1, %zmm0 {%k1}872; X86-NEXT: retl873;874; X64-LABEL: test_mm512_mask_shrdv_epi32:875; X64: # %bb.0: # %entry876; X64-NEXT: kmovd %edi, %k1877; X64-NEXT: vpshrdvd %zmm2, %zmm1, %zmm0 {%k1}878; X64-NEXT: retq879entry:880 %0 = bitcast <8 x i64> %__S to <16 x i32>881 %1 = bitcast <8 x i64> %__A to <16 x i32>882 %2 = bitcast <8 x i64> %__B to <16 x i32>883 %3 = tail call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %1, <16 x i32> %0, <16 x i32> %2)884 %4 = bitcast i16 %__U to <16 x i1>885 %5 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> %0886 %6 = bitcast <16 x i32> %5 to <8 x i64>887 ret <8 x i64> %6888}889 890define <8 x i64> @test_mm512_maskz_shrdv_epi32(i16 zeroext %__U, <8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B) {891; X86-LABEL: test_mm512_maskz_shrdv_epi32:892; X86: # %bb.0: # %entry893; X86-NEXT: kmovw {{[0-9]+}}(%esp), %k1894; X86-NEXT: vpshrdvd %zmm2, %zmm1, %zmm0 {%k1} {z}895; X86-NEXT: retl896;897; X64-LABEL: test_mm512_maskz_shrdv_epi32:898; X64: # %bb.0: # %entry899; X64-NEXT: kmovd %edi, %k1900; X64-NEXT: vpshrdvd %zmm2, %zmm1, %zmm0 {%k1} {z}901; X64-NEXT: retq902entry:903 %0 = bitcast <8 x i64> %__S to <16 x i32>904 %1 = bitcast <8 x i64> %__A to <16 x i32>905 %2 = bitcast <8 x i64> %__B to <16 x i32>906 %3 = tail call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %1, <16 x i32> %0, <16 x i32> %2)907 %4 = bitcast i16 %__U to <16 x i1>908 %5 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> zeroinitializer909 %6 = bitcast <16 x i32> %5 to <8 x i64>910 ret <8 x i64> %6911}912 913define <8 x i64> @test_mm512_shrdv_epi32(<8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B) {914; CHECK-LABEL: test_mm512_shrdv_epi32:915; CHECK: # %bb.0: # %entry916; CHECK-NEXT: vpshrdvd %zmm2, %zmm1, %zmm0917; CHECK-NEXT: ret{{[l|q]}}918entry:919 %0 = bitcast <8 x i64> %__S to <16 x i32>920 %1 = bitcast <8 x i64> %__A to <16 x i32>921 %2 = bitcast <8 x i64> %__B to <16 x i32>922 %3 = tail call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %1, <16 x i32> %0, <16 x i32> %2)923 %4 = bitcast <16 x i32> %3 to <8 x i64>924 ret <8 x i64> %4925}926 927define <8 x i64> @test_mm512_mask_shrdv_epi16(<8 x i64> %__S, i32 %__U, <8 x i64> %__A, <8 x i64> %__B) {928; X86-LABEL: test_mm512_mask_shrdv_epi16:929; X86: # %bb.0: # %entry930; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1931; X86-NEXT: vpshrdvw %zmm2, %zmm1, %zmm0 {%k1}932; X86-NEXT: retl933;934; X64-LABEL: test_mm512_mask_shrdv_epi16:935; X64: # %bb.0: # %entry936; X64-NEXT: kmovd %edi, %k1937; X64-NEXT: vpshrdvw %zmm2, %zmm1, %zmm0 {%k1}938; X64-NEXT: retq939entry:940 %0 = bitcast <8 x i64> %__S to <32 x i16>941 %1 = bitcast <8 x i64> %__A to <32 x i16>942 %2 = bitcast <8 x i64> %__B to <32 x i16>943 %3 = tail call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %1, <32 x i16> %0, <32 x i16> %2)944 %4 = bitcast i32 %__U to <32 x i1>945 %5 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> %0946 %6 = bitcast <32 x i16> %5 to <8 x i64>947 ret <8 x i64> %6948}949 950define <8 x i64> @test_mm512_maskz_shrdv_epi16(i32 %__U, <8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B) {951; X86-LABEL: test_mm512_maskz_shrdv_epi16:952; X86: # %bb.0: # %entry953; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1954; X86-NEXT: vpshrdvw %zmm2, %zmm1, %zmm0 {%k1} {z}955; X86-NEXT: retl956;957; X64-LABEL: test_mm512_maskz_shrdv_epi16:958; X64: # %bb.0: # %entry959; X64-NEXT: kmovd %edi, %k1960; X64-NEXT: vpshrdvw %zmm2, %zmm1, %zmm0 {%k1} {z}961; X64-NEXT: retq962entry:963 %0 = bitcast <8 x i64> %__S to <32 x i16>964 %1 = bitcast <8 x i64> %__A to <32 x i16>965 %2 = bitcast <8 x i64> %__B to <32 x i16>966 %3 = tail call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %1, <32 x i16> %0, <32 x i16> %2)967 %4 = bitcast i32 %__U to <32 x i1>968 %5 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> zeroinitializer969 %6 = bitcast <32 x i16> %5 to <8 x i64>970 ret <8 x i64> %6971}972 973define <8 x i64> @test_mm512_shrdv_epi16(<8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B) {974; CHECK-LABEL: test_mm512_shrdv_epi16:975; CHECK: # %bb.0: # %entry976; CHECK-NEXT: vpshrdvw %zmm2, %zmm1, %zmm0977; CHECK-NEXT: ret{{[l|q]}}978entry:979 %0 = bitcast <8 x i64> %__S to <32 x i16>980 %1 = bitcast <8 x i64> %__A to <32 x i16>981 %2 = bitcast <8 x i64> %__B to <32 x i16>982 %3 = tail call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %1, <32 x i16> %0, <32 x i16> %2)983 %4 = bitcast <32 x i16> %3 to <8 x i64>984 ret <8 x i64> %4985}986 987declare <32 x i16> @llvm.x86.avx512.mask.compress.w.512(<32 x i16>, <32 x i16>, i32)988declare <64 x i8> @llvm.x86.avx512.mask.compress.b.512(<64 x i8>, <64 x i8>, i64)989declare void @llvm.masked.compressstore.v32i16(<32 x i16>, ptr, <32 x i1>)990declare void @llvm.masked.compressstore.v64i8(<64 x i8>, ptr, <64 x i1>)991declare <32 x i16> @llvm.x86.avx512.mask.expand.w.512(<32 x i16>, <32 x i16>, i32)992declare <64 x i8> @llvm.x86.avx512.mask.expand.b.512(<64 x i8>, <64 x i8>, i64)993declare <32 x i16> @llvm.masked.expandload.v32i16(ptr, <32 x i1>, <32 x i16>)994declare <64 x i8> @llvm.masked.expandload.v64i8(ptr, <64 x i1>, <64 x i8>)995