2101 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 -mattr=prefer-256-bit | FileCheck %s --check-prefixes=CHECK,CHECK-SKX,CHECK-SKX-NOVBMI3; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 -mattr=prefer-256-bit,avx512vbmi | FileCheck %s --check-prefixes=CHECK,CHECK-SKX,CHECK-SKX-VBMI4; Make sure CPUs default to prefer-256-bit. avx512vnni isn't interesting as it just adds an isel peephole for vpmaddwd+vpaddd5; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 | FileCheck %s --check-prefixes=CHECK,CHECK-AVX5126; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=cascadelake | FileCheck %s --check-prefixes=CHECK,CHECK-AVX5127; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=cooperlake | FileCheck %s --check-prefixes=CHECK,CHECK-AVX5128; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=cannonlake | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI,CHECK-VBMI19; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=icelake-client | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI,CHECK-GFNI10; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=icelake-server | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI,CHECK-GFNI11; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=-avx512vnni -mcpu=tigerlake | FileCheck %s --check-prefixes=CHECK,CHECK-VBMI,CHECK-GFNI12 13; This file primarily contains tests for specific places in X86ISelLowering.cpp that needed be made aware of the legalizer not allowing 512-bit vectors due to prefer-256-bit even though AVX512 is enabled.14 15define dso_local void @add256(ptr %a, ptr %b, ptr %c) "min-legal-vector-width"="256" {16; CHECK-LABEL: add256:17; CHECK: # %bb.0:18; CHECK-NEXT: vmovdqa (%rdi), %ymm019; CHECK-NEXT: vmovdqa 32(%rdi), %ymm120; CHECK-NEXT: vpaddd 32(%rsi), %ymm1, %ymm121; CHECK-NEXT: vpaddd (%rsi), %ymm0, %ymm022; CHECK-NEXT: vmovdqa %ymm0, (%rdx)23; CHECK-NEXT: vmovdqa %ymm1, 32(%rdx)24; CHECK-NEXT: vzeroupper25; CHECK-NEXT: retq26 %d = load <16 x i32>, ptr %a27 %e = load <16 x i32>, ptr %b28 %f = add <16 x i32> %d, %e29 store <16 x i32> %f, ptr %c30 ret void31}32 33define dso_local void @add512(ptr %a, ptr %b, ptr %c) "min-legal-vector-width"="512" {34; CHECK-LABEL: add512:35; CHECK: # %bb.0:36; CHECK-NEXT: vmovdqa64 (%rdi), %zmm037; CHECK-NEXT: vpaddd (%rsi), %zmm0, %zmm038; CHECK-NEXT: vmovdqa64 %zmm0, (%rdx)39; CHECK-NEXT: vzeroupper40; CHECK-NEXT: retq41 %d = load <16 x i32>, ptr %a42 %e = load <16 x i32>, ptr %b43 %f = add <16 x i32> %d, %e44 store <16 x i32> %f, ptr %c45 ret void46}47 48define dso_local void @avg_v64i8_256(ptr %a, ptr %b) "min-legal-vector-width"="256" {49; CHECK-LABEL: avg_v64i8_256:50; CHECK: # %bb.0:51; CHECK-NEXT: vmovdqa (%rdi), %ymm052; CHECK-NEXT: vmovdqa 32(%rdi), %ymm153; CHECK-NEXT: vpavgb (%rsi), %ymm0, %ymm054; CHECK-NEXT: vpavgb 32(%rsi), %ymm1, %ymm155; CHECK-NEXT: vmovdqu %ymm1, (%rax)56; CHECK-NEXT: vmovdqu %ymm0, (%rax)57; CHECK-NEXT: vzeroupper58; CHECK-NEXT: retq59 %1 = load <64 x i8>, ptr %a60 %2 = load <64 x i8>, ptr %b61 %3 = zext <64 x i8> %1 to <64 x i32>62 %4 = zext <64 x i8> %2 to <64 x i32>63 %5 = add nuw nsw <64 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>64 %6 = add nuw nsw <64 x i32> %5, %465 %7 = lshr <64 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>66 %8 = trunc <64 x i32> %7 to <64 x i8>67 store <64 x i8> %8, ptr undef, align 468 ret void69}70 71 72define dso_local void @avg_v64i8_512(ptr %a, ptr %b) "min-legal-vector-width"="512" {73; CHECK-LABEL: avg_v64i8_512:74; CHECK: # %bb.0:75; CHECK-NEXT: vmovdqa64 (%rdi), %zmm076; CHECK-NEXT: vpavgb (%rsi), %zmm0, %zmm077; CHECK-NEXT: vmovdqu64 %zmm0, (%rax)78; CHECK-NEXT: vzeroupper79; CHECK-NEXT: retq80 %1 = load <64 x i8>, ptr %a81 %2 = load <64 x i8>, ptr %b82 %3 = zext <64 x i8> %1 to <64 x i32>83 %4 = zext <64 x i8> %2 to <64 x i32>84 %5 = add nuw nsw <64 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>85 %6 = add nuw nsw <64 x i32> %5, %486 %7 = lshr <64 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>87 %8 = trunc <64 x i32> %7 to <64 x i8>88 store <64 x i8> %8, ptr undef, align 489 ret void90}91 92define dso_local void @pmaddwd_32_256(ptr %APtr, ptr %BPtr, ptr %CPtr) "min-legal-vector-width"="256" {93; CHECK-LABEL: pmaddwd_32_256:94; CHECK: # %bb.0:95; CHECK-NEXT: vmovdqa (%rdi), %ymm096; CHECK-NEXT: vmovdqa 32(%rdi), %ymm197; CHECK-NEXT: vpmaddwd 32(%rsi), %ymm1, %ymm198; CHECK-NEXT: vpmaddwd (%rsi), %ymm0, %ymm099; CHECK-NEXT: vmovdqa %ymm0, (%rdx)100; CHECK-NEXT: vmovdqa %ymm1, 32(%rdx)101; CHECK-NEXT: vzeroupper102; CHECK-NEXT: retq103 %A = load <32 x i16>, ptr %APtr104 %B = load <32 x i16>, ptr %BPtr105 %a = sext <32 x i16> %A to <32 x i32>106 %b = sext <32 x i16> %B to <32 x i32>107 %m = mul nsw <32 x i32> %a, %b108 %odd = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>109 %even = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>110 %ret = add <16 x i32> %odd, %even111 store <16 x i32> %ret, ptr %CPtr112 ret void113}114 115define dso_local void @pmaddwd_32_512(ptr %APtr, ptr %BPtr, ptr %CPtr) "min-legal-vector-width"="512" {116; CHECK-LABEL: pmaddwd_32_512:117; CHECK: # %bb.0:118; CHECK-NEXT: vmovdqa64 (%rdi), %zmm0119; CHECK-NEXT: vpmaddwd (%rsi), %zmm0, %zmm0120; CHECK-NEXT: vmovdqa64 %zmm0, (%rdx)121; CHECK-NEXT: vzeroupper122; CHECK-NEXT: retq123 %A = load <32 x i16>, ptr %APtr124 %B = load <32 x i16>, ptr %BPtr125 %a = sext <32 x i16> %A to <32 x i32>126 %b = sext <32 x i16> %B to <32 x i32>127 %m = mul nsw <32 x i32> %a, %b128 %odd = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>129 %even = shufflevector <32 x i32> %m, <32 x i32> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>130 %ret = add <16 x i32> %odd, %even131 store <16 x i32> %ret, ptr %CPtr132 ret void133}134 135define dso_local void @psubus_64i8_max_256(ptr %xptr, ptr %yptr, ptr %zptr) "min-legal-vector-width"="256" {136; CHECK-LABEL: psubus_64i8_max_256:137; CHECK: # %bb.0:138; CHECK-NEXT: vmovdqa (%rdi), %ymm0139; CHECK-NEXT: vmovdqa 32(%rdi), %ymm1140; CHECK-NEXT: vpsubusb 32(%rsi), %ymm1, %ymm1141; CHECK-NEXT: vpsubusb (%rsi), %ymm0, %ymm0142; CHECK-NEXT: vmovdqa %ymm0, (%rdx)143; CHECK-NEXT: vmovdqa %ymm1, 32(%rdx)144; CHECK-NEXT: vzeroupper145; CHECK-NEXT: retq146 %x = load <64 x i8>, ptr %xptr147 %y = load <64 x i8>, ptr %yptr148 %cmp = icmp ult <64 x i8> %x, %y149 %max = select <64 x i1> %cmp, <64 x i8> %y, <64 x i8> %x150 %res = sub <64 x i8> %max, %y151 store <64 x i8> %res, ptr %zptr152 ret void153}154 155define dso_local void @psubus_64i8_max_512(ptr %xptr, ptr %yptr, ptr %zptr) "min-legal-vector-width"="512" {156; CHECK-LABEL: psubus_64i8_max_512:157; CHECK: # %bb.0:158; CHECK-NEXT: vmovdqa64 (%rdi), %zmm0159; CHECK-NEXT: vpsubusb (%rsi), %zmm0, %zmm0160; CHECK-NEXT: vmovdqa64 %zmm0, (%rdx)161; CHECK-NEXT: vzeroupper162; CHECK-NEXT: retq163 %x = load <64 x i8>, ptr %xptr164 %y = load <64 x i8>, ptr %yptr165 %cmp = icmp ult <64 x i8> %x, %y166 %max = select <64 x i1> %cmp, <64 x i8> %y, <64 x i8> %x167 %res = sub <64 x i8> %max, %y168 store <64 x i8> %res, ptr %zptr169 ret void170}171 172define dso_local i32 @_Z9test_charPcS_i_256(ptr nocapture readonly, ptr nocapture readonly, i32) "min-legal-vector-width"="256" {173; CHECK-SKX-LABEL: _Z9test_charPcS_i_256:174; CHECK-SKX: # %bb.0: # %entry175; CHECK-SKX-NEXT: movl %edx, %eax176; CHECK-SKX-NEXT: vpxor %xmm0, %xmm0, %xmm0177; CHECK-SKX-NEXT: xorl %ecx, %ecx178; CHECK-SKX-NEXT: vpxor %xmm1, %xmm1, %xmm1179; CHECK-SKX-NEXT: vpxor %xmm2, %xmm2, %xmm2180; CHECK-SKX-NEXT: .p2align 4181; CHECK-SKX-NEXT: .LBB8_1: # %vector.body182; CHECK-SKX-NEXT: # =>This Inner Loop Header: Depth=1183; CHECK-SKX-NEXT: vpmovsxbw 16(%rdi,%rcx), %ymm3184; CHECK-SKX-NEXT: vpmovsxbw (%rdi,%rcx), %ymm4185; CHECK-SKX-NEXT: vpmovsxbw 16(%rsi,%rcx), %ymm5186; CHECK-SKX-NEXT: vpmaddwd %ymm3, %ymm5, %ymm3187; CHECK-SKX-NEXT: vpaddd %ymm2, %ymm3, %ymm2188; CHECK-SKX-NEXT: vpmovsxbw (%rsi,%rcx), %ymm3189; CHECK-SKX-NEXT: vpmaddwd %ymm4, %ymm3, %ymm3190; CHECK-SKX-NEXT: vpaddd %ymm1, %ymm3, %ymm1191; CHECK-SKX-NEXT: addq $32, %rcx192; CHECK-SKX-NEXT: cmpq %rcx, %rax193; CHECK-SKX-NEXT: jne .LBB8_1194; CHECK-SKX-NEXT: # %bb.2: # %middle.block195; CHECK-SKX-NEXT: vpaddd %ymm0, %ymm1, %ymm1196; CHECK-SKX-NEXT: vpaddd %ymm0, %ymm2, %ymm0197; CHECK-SKX-NEXT: vpaddd %ymm0, %ymm1, %ymm0198; CHECK-SKX-NEXT: vextracti128 $1, %ymm0, %xmm1199; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0200; CHECK-SKX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]201; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0202; CHECK-SKX-NEXT: vpsrlq $32, %xmm0, %xmm1203; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0204; CHECK-SKX-NEXT: vmovd %xmm0, %eax205; CHECK-SKX-NEXT: vzeroupper206; CHECK-SKX-NEXT: retq207;208; CHECK-AVX512-LABEL: _Z9test_charPcS_i_256:209; CHECK-AVX512: # %bb.0: # %entry210; CHECK-AVX512-NEXT: movl %edx, %eax211; CHECK-AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0212; CHECK-AVX512-NEXT: xorl %ecx, %ecx213; CHECK-AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1214; CHECK-AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2215; CHECK-AVX512-NEXT: .p2align 4216; CHECK-AVX512-NEXT: .LBB8_1: # %vector.body217; CHECK-AVX512-NEXT: # =>This Inner Loop Header: Depth=1218; CHECK-AVX512-NEXT: vpmovsxbw 16(%rdi,%rcx), %ymm3219; CHECK-AVX512-NEXT: vpmovsxbw (%rdi,%rcx), %ymm4220; CHECK-AVX512-NEXT: vpmovsxbw 16(%rsi,%rcx), %ymm5221; CHECK-AVX512-NEXT: vpmaddwd %ymm3, %ymm5, %ymm3222; CHECK-AVX512-NEXT: vpaddd %ymm2, %ymm3, %ymm2223; CHECK-AVX512-NEXT: vpmovsxbw (%rsi,%rcx), %ymm3224; CHECK-AVX512-NEXT: vpmaddwd %ymm4, %ymm3, %ymm3225; CHECK-AVX512-NEXT: vpaddd %ymm1, %ymm3, %ymm1226; CHECK-AVX512-NEXT: addq $32, %rcx227; CHECK-AVX512-NEXT: cmpq %rcx, %rax228; CHECK-AVX512-NEXT: jne .LBB8_1229; CHECK-AVX512-NEXT: # %bb.2: # %middle.block230; CHECK-AVX512-NEXT: vpaddd %ymm0, %ymm1, %ymm1231; CHECK-AVX512-NEXT: vpaddd %ymm0, %ymm2, %ymm0232; CHECK-AVX512-NEXT: vpaddd %ymm0, %ymm1, %ymm0233; CHECK-AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1234; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0235; CHECK-AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]236; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0237; CHECK-AVX512-NEXT: vpsrlq $32, %xmm0, %xmm1238; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0239; CHECK-AVX512-NEXT: vmovd %xmm0, %eax240; CHECK-AVX512-NEXT: vzeroupper241; CHECK-AVX512-NEXT: retq242;243; CHECK-VBMI-LABEL: _Z9test_charPcS_i_256:244; CHECK-VBMI: # %bb.0: # %entry245; CHECK-VBMI-NEXT: movl %edx, %eax246; CHECK-VBMI-NEXT: vpxor %xmm0, %xmm0, %xmm0247; CHECK-VBMI-NEXT: xorl %ecx, %ecx248; CHECK-VBMI-NEXT: vpxor %xmm1, %xmm1, %xmm1249; CHECK-VBMI-NEXT: vpxor %xmm2, %xmm2, %xmm2250; CHECK-VBMI-NEXT: .p2align 4251; CHECK-VBMI-NEXT: .LBB8_1: # %vector.body252; CHECK-VBMI-NEXT: # =>This Inner Loop Header: Depth=1253; CHECK-VBMI-NEXT: vpmovsxbw 16(%rdi,%rcx), %ymm3254; CHECK-VBMI-NEXT: vpmovsxbw (%rdi,%rcx), %ymm4255; CHECK-VBMI-NEXT: vpmovsxbw 16(%rsi,%rcx), %ymm5256; CHECK-VBMI-NEXT: vpmaddwd %ymm3, %ymm5, %ymm3257; CHECK-VBMI-NEXT: vpaddd %ymm2, %ymm3, %ymm2258; CHECK-VBMI-NEXT: vpmovsxbw (%rsi,%rcx), %ymm3259; CHECK-VBMI-NEXT: vpmaddwd %ymm4, %ymm3, %ymm3260; CHECK-VBMI-NEXT: vpaddd %ymm1, %ymm3, %ymm1261; CHECK-VBMI-NEXT: addq $32, %rcx262; CHECK-VBMI-NEXT: cmpq %rcx, %rax263; CHECK-VBMI-NEXT: jne .LBB8_1264; CHECK-VBMI-NEXT: # %bb.2: # %middle.block265; CHECK-VBMI-NEXT: vpaddd %ymm0, %ymm1, %ymm1266; CHECK-VBMI-NEXT: vpaddd %ymm0, %ymm2, %ymm0267; CHECK-VBMI-NEXT: vpaddd %ymm0, %ymm1, %ymm0268; CHECK-VBMI-NEXT: vextracti128 $1, %ymm0, %xmm1269; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0270; CHECK-VBMI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]271; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0272; CHECK-VBMI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]273; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0274; CHECK-VBMI-NEXT: vmovd %xmm0, %eax275; CHECK-VBMI-NEXT: vzeroupper276; CHECK-VBMI-NEXT: retq277entry:278 %3 = zext i32 %2 to i64279 br label %vector.body280 281vector.body:282 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ]283 %vec.phi = phi <32 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ]284 %4 = getelementptr inbounds i8, ptr %0, i64 %index285 %5 = bitcast ptr %4 to ptr286 %wide.load = load <32 x i8>, ptr %5, align 1287 %6 = sext <32 x i8> %wide.load to <32 x i32>288 %7 = getelementptr inbounds i8, ptr %1, i64 %index289 %8 = bitcast ptr %7 to ptr290 %wide.load14 = load <32 x i8>, ptr %8, align 1291 %9 = sext <32 x i8> %wide.load14 to <32 x i32>292 %10 = mul nsw <32 x i32> %9, %6293 %11 = add nsw <32 x i32> %10, %vec.phi294 %index.next = add i64 %index, 32295 %12 = icmp eq i64 %index.next, %3296 br i1 %12, label %middle.block, label %vector.body297 298middle.block:299 %rdx.shuf1 = shufflevector <32 x i32> %11, <32 x i32> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>300 %bin.rdx1 = add <32 x i32> %11, %rdx.shuf1301 %rdx.shuf = shufflevector <32 x i32> %bin.rdx1, <32 x i32> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>302 %bin.rdx = add <32 x i32> %bin.rdx1, %rdx.shuf303 %rdx.shuf15 = shufflevector <32 x i32> %bin.rdx, <32 x i32> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>304 %bin.rdx32 = add <32 x i32> %bin.rdx, %rdx.shuf15305 %rdx.shuf17 = shufflevector <32 x i32> %bin.rdx32, <32 x i32> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>306 %bin.rdx18 = add <32 x i32> %bin.rdx32, %rdx.shuf17307 %rdx.shuf19 = shufflevector <32 x i32> %bin.rdx18, <32 x i32> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>308 %bin.rdx20 = add <32 x i32> %bin.rdx18, %rdx.shuf19309 %13 = extractelement <32 x i32> %bin.rdx20, i32 0310 ret i32 %13311}312 313define dso_local i32 @_Z9test_charPcS_i_512(ptr nocapture readonly, ptr nocapture readonly, i32) "min-legal-vector-width"="512" {314; CHECK-SKX-LABEL: _Z9test_charPcS_i_512:315; CHECK-SKX: # %bb.0: # %entry316; CHECK-SKX-NEXT: movl %edx, %eax317; CHECK-SKX-NEXT: vpxor %xmm0, %xmm0, %xmm0318; CHECK-SKX-NEXT: xorl %ecx, %ecx319; CHECK-SKX-NEXT: vpxor %xmm1, %xmm1, %xmm1320; CHECK-SKX-NEXT: .p2align 4321; CHECK-SKX-NEXT: .LBB9_1: # %vector.body322; CHECK-SKX-NEXT: # =>This Inner Loop Header: Depth=1323; CHECK-SKX-NEXT: vpmovsxbw (%rdi,%rcx), %zmm2324; CHECK-SKX-NEXT: vpmovsxbw (%rsi,%rcx), %zmm3325; CHECK-SKX-NEXT: vpmaddwd %zmm2, %zmm3, %zmm2326; CHECK-SKX-NEXT: vpaddd %zmm1, %zmm2, %zmm1327; CHECK-SKX-NEXT: addq $32, %rcx328; CHECK-SKX-NEXT: cmpq %rcx, %rax329; CHECK-SKX-NEXT: jne .LBB9_1330; CHECK-SKX-NEXT: # %bb.2: # %middle.block331; CHECK-SKX-NEXT: vpaddd %zmm0, %zmm1, %zmm0332; CHECK-SKX-NEXT: vextracti64x4 $1, %zmm0, %ymm1333; CHECK-SKX-NEXT: vpaddd %zmm1, %zmm0, %zmm0334; CHECK-SKX-NEXT: vextracti128 $1, %ymm0, %xmm1335; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0336; CHECK-SKX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]337; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0338; CHECK-SKX-NEXT: vpsrlq $32, %xmm0, %xmm1339; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0340; CHECK-SKX-NEXT: vmovd %xmm0, %eax341; CHECK-SKX-NEXT: vzeroupper342; CHECK-SKX-NEXT: retq343;344; CHECK-AVX512-LABEL: _Z9test_charPcS_i_512:345; CHECK-AVX512: # %bb.0: # %entry346; CHECK-AVX512-NEXT: movl %edx, %eax347; CHECK-AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0348; CHECK-AVX512-NEXT: xorl %ecx, %ecx349; CHECK-AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1350; CHECK-AVX512-NEXT: .p2align 4351; CHECK-AVX512-NEXT: .LBB9_1: # %vector.body352; CHECK-AVX512-NEXT: # =>This Inner Loop Header: Depth=1353; CHECK-AVX512-NEXT: vpmovsxbw (%rdi,%rcx), %zmm2354; CHECK-AVX512-NEXT: vpmovsxbw (%rsi,%rcx), %zmm3355; CHECK-AVX512-NEXT: vpmaddwd %zmm2, %zmm3, %zmm2356; CHECK-AVX512-NEXT: vpaddd %zmm1, %zmm2, %zmm1357; CHECK-AVX512-NEXT: addq $32, %rcx358; CHECK-AVX512-NEXT: cmpq %rcx, %rax359; CHECK-AVX512-NEXT: jne .LBB9_1360; CHECK-AVX512-NEXT: # %bb.2: # %middle.block361; CHECK-AVX512-NEXT: vpaddd %zmm0, %zmm1, %zmm0362; CHECK-AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1363; CHECK-AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0364; CHECK-AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1365; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0366; CHECK-AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]367; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0368; CHECK-AVX512-NEXT: vpsrlq $32, %xmm0, %xmm1369; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0370; CHECK-AVX512-NEXT: vmovd %xmm0, %eax371; CHECK-AVX512-NEXT: vzeroupper372; CHECK-AVX512-NEXT: retq373;374; CHECK-VBMI-LABEL: _Z9test_charPcS_i_512:375; CHECK-VBMI: # %bb.0: # %entry376; CHECK-VBMI-NEXT: movl %edx, %eax377; CHECK-VBMI-NEXT: vpxor %xmm0, %xmm0, %xmm0378; CHECK-VBMI-NEXT: xorl %ecx, %ecx379; CHECK-VBMI-NEXT: vpxor %xmm1, %xmm1, %xmm1380; CHECK-VBMI-NEXT: .p2align 4381; CHECK-VBMI-NEXT: .LBB9_1: # %vector.body382; CHECK-VBMI-NEXT: # =>This Inner Loop Header: Depth=1383; CHECK-VBMI-NEXT: vpmovsxbw (%rdi,%rcx), %zmm2384; CHECK-VBMI-NEXT: vpmovsxbw (%rsi,%rcx), %zmm3385; CHECK-VBMI-NEXT: vpmaddwd %zmm2, %zmm3, %zmm2386; CHECK-VBMI-NEXT: vpaddd %zmm1, %zmm2, %zmm1387; CHECK-VBMI-NEXT: addq $32, %rcx388; CHECK-VBMI-NEXT: cmpq %rcx, %rax389; CHECK-VBMI-NEXT: jne .LBB9_1390; CHECK-VBMI-NEXT: # %bb.2: # %middle.block391; CHECK-VBMI-NEXT: vpaddd %zmm0, %zmm1, %zmm0392; CHECK-VBMI-NEXT: vextracti64x4 $1, %zmm0, %ymm1393; CHECK-VBMI-NEXT: vpaddd %zmm1, %zmm0, %zmm0394; CHECK-VBMI-NEXT: vextracti128 $1, %ymm0, %xmm1395; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0396; CHECK-VBMI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]397; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0398; CHECK-VBMI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]399; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0400; CHECK-VBMI-NEXT: vmovd %xmm0, %eax401; CHECK-VBMI-NEXT: vzeroupper402; CHECK-VBMI-NEXT: retq403entry:404 %3 = zext i32 %2 to i64405 br label %vector.body406 407vector.body:408 %index = phi i64 [ %index.next, %vector.body ], [ 0, %entry ]409 %vec.phi = phi <32 x i32> [ %11, %vector.body ], [ zeroinitializer, %entry ]410 %4 = getelementptr inbounds i8, ptr %0, i64 %index411 %5 = bitcast ptr %4 to ptr412 %wide.load = load <32 x i8>, ptr %5, align 1413 %6 = sext <32 x i8> %wide.load to <32 x i32>414 %7 = getelementptr inbounds i8, ptr %1, i64 %index415 %8 = bitcast ptr %7 to ptr416 %wide.load14 = load <32 x i8>, ptr %8, align 1417 %9 = sext <32 x i8> %wide.load14 to <32 x i32>418 %10 = mul nsw <32 x i32> %9, %6419 %11 = add nsw <32 x i32> %10, %vec.phi420 %index.next = add i64 %index, 32421 %12 = icmp eq i64 %index.next, %3422 br i1 %12, label %middle.block, label %vector.body423 424middle.block:425 %rdx.shuf1 = shufflevector <32 x i32> %11, <32 x i32> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>426 %bin.rdx1 = add <32 x i32> %11, %rdx.shuf1427 %rdx.shuf = shufflevector <32 x i32> %bin.rdx1, <32 x i32> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>428 %bin.rdx = add <32 x i32> %bin.rdx1, %rdx.shuf429 %rdx.shuf15 = shufflevector <32 x i32> %bin.rdx, <32 x i32> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>430 %bin.rdx32 = add <32 x i32> %bin.rdx, %rdx.shuf15431 %rdx.shuf17 = shufflevector <32 x i32> %bin.rdx32, <32 x i32> undef, <32 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>432 %bin.rdx18 = add <32 x i32> %bin.rdx32, %rdx.shuf17433 %rdx.shuf19 = shufflevector <32 x i32> %bin.rdx18, <32 x i32> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>434 %bin.rdx20 = add <32 x i32> %bin.rdx18, %rdx.shuf19435 %13 = extractelement <32 x i32> %bin.rdx20, i32 0436 ret i32 %13437}438 439@a = dso_local global [1024 x i8] zeroinitializer, align 16440@b = dso_local global [1024 x i8] zeroinitializer, align 16441 442define dso_local i32 @sad_16i8_256() "min-legal-vector-width"="256" {443; CHECK-SKX-LABEL: sad_16i8_256:444; CHECK-SKX: # %bb.0: # %entry445; CHECK-SKX-NEXT: vpxor %xmm0, %xmm0, %xmm0446; CHECK-SKX-NEXT: movq $-1024, %rax # imm = 0xFC00447; CHECK-SKX-NEXT: vpxor %xmm1, %xmm1, %xmm1448; CHECK-SKX-NEXT: .p2align 4449; CHECK-SKX-NEXT: .LBB10_1: # %vector.body450; CHECK-SKX-NEXT: # =>This Inner Loop Header: Depth=1451; CHECK-SKX-NEXT: vmovdqu a+1024(%rax), %xmm2452; CHECK-SKX-NEXT: vpsadbw b+1024(%rax), %xmm2, %xmm2453; CHECK-SKX-NEXT: vpaddd %ymm1, %ymm2, %ymm1454; CHECK-SKX-NEXT: addq $4, %rax455; CHECK-SKX-NEXT: jne .LBB10_1456; CHECK-SKX-NEXT: # %bb.2: # %middle.block457; CHECK-SKX-NEXT: vpaddd %ymm0, %ymm1, %ymm0458; CHECK-SKX-NEXT: vextracti128 $1, %ymm0, %xmm1459; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0460; CHECK-SKX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]461; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0462; CHECK-SKX-NEXT: vpsrlq $32, %xmm0, %xmm1463; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0464; CHECK-SKX-NEXT: vmovd %xmm0, %eax465; CHECK-SKX-NEXT: vzeroupper466; CHECK-SKX-NEXT: retq467;468; CHECK-AVX512-LABEL: sad_16i8_256:469; CHECK-AVX512: # %bb.0: # %entry470; CHECK-AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0471; CHECK-AVX512-NEXT: movq $-1024, %rax # imm = 0xFC00472; CHECK-AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1473; CHECK-AVX512-NEXT: .p2align 4474; CHECK-AVX512-NEXT: .LBB10_1: # %vector.body475; CHECK-AVX512-NEXT: # =>This Inner Loop Header: Depth=1476; CHECK-AVX512-NEXT: vmovdqu a+1024(%rax), %xmm2477; CHECK-AVX512-NEXT: vpsadbw b+1024(%rax), %xmm2, %xmm2478; CHECK-AVX512-NEXT: vpaddd %ymm1, %ymm2, %ymm1479; CHECK-AVX512-NEXT: addq $4, %rax480; CHECK-AVX512-NEXT: jne .LBB10_1481; CHECK-AVX512-NEXT: # %bb.2: # %middle.block482; CHECK-AVX512-NEXT: vpaddd %ymm0, %ymm1, %ymm0483; CHECK-AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1484; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0485; CHECK-AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]486; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0487; CHECK-AVX512-NEXT: vpsrlq $32, %xmm0, %xmm1488; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0489; CHECK-AVX512-NEXT: vmovd %xmm0, %eax490; CHECK-AVX512-NEXT: vzeroupper491; CHECK-AVX512-NEXT: retq492;493; CHECK-VBMI-LABEL: sad_16i8_256:494; CHECK-VBMI: # %bb.0: # %entry495; CHECK-VBMI-NEXT: vpxor %xmm0, %xmm0, %xmm0496; CHECK-VBMI-NEXT: movq $-1024, %rax # imm = 0xFC00497; CHECK-VBMI-NEXT: vpxor %xmm1, %xmm1, %xmm1498; CHECK-VBMI-NEXT: .p2align 4499; CHECK-VBMI-NEXT: .LBB10_1: # %vector.body500; CHECK-VBMI-NEXT: # =>This Inner Loop Header: Depth=1501; CHECK-VBMI-NEXT: vmovdqu a+1024(%rax), %xmm2502; CHECK-VBMI-NEXT: vpsadbw b+1024(%rax), %xmm2, %xmm2503; CHECK-VBMI-NEXT: vpaddd %ymm1, %ymm2, %ymm1504; CHECK-VBMI-NEXT: addq $4, %rax505; CHECK-VBMI-NEXT: jne .LBB10_1506; CHECK-VBMI-NEXT: # %bb.2: # %middle.block507; CHECK-VBMI-NEXT: vpaddd %ymm0, %ymm1, %ymm0508; CHECK-VBMI-NEXT: vextracti128 $1, %ymm0, %xmm1509; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0510; CHECK-VBMI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]511; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0512; CHECK-VBMI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]513; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0514; CHECK-VBMI-NEXT: vmovd %xmm0, %eax515; CHECK-VBMI-NEXT: vzeroupper516; CHECK-VBMI-NEXT: retq517entry:518 br label %vector.body519 520vector.body:521 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]522 %vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %10, %vector.body ]523 %0 = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index524 %1 = bitcast ptr %0 to ptr525 %wide.load = load <16 x i8>, ptr %1, align 4526 %2 = zext <16 x i8> %wide.load to <16 x i32>527 %3 = getelementptr inbounds [1024 x i8], ptr @b, i64 0, i64 %index528 %4 = bitcast ptr %3 to ptr529 %wide.load1 = load <16 x i8>, ptr %4, align 4530 %5 = zext <16 x i8> %wide.load1 to <16 x i32>531 %6 = sub nsw <16 x i32> %2, %5532 %7 = icmp sgt <16 x i32> %6, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>533 %8 = sub nsw <16 x i32> zeroinitializer, %6534 %9 = select <16 x i1> %7, <16 x i32> %6, <16 x i32> %8535 %10 = add nsw <16 x i32> %9, %vec.phi536 %index.next = add i64 %index, 4537 %11 = icmp eq i64 %index.next, 1024538 br i1 %11, label %middle.block, label %vector.body539 540middle.block:541 %rdx.shuf = shufflevector <16 x i32> %10, <16 x i32> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>542 %bin.rdx = add <16 x i32> %10, %rdx.shuf543 %rdx.shuf2 = shufflevector <16 x i32> %bin.rdx, <16 x i32> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>544 %bin.rdx2 = add <16 x i32> %bin.rdx, %rdx.shuf2545 %rdx.shuf3 = shufflevector <16 x i32> %bin.rdx2, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>546 %bin.rdx3 = add <16 x i32> %bin.rdx2, %rdx.shuf3547 %rdx.shuf4 = shufflevector <16 x i32> %bin.rdx3, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>548 %bin.rdx4 = add <16 x i32> %bin.rdx3, %rdx.shuf4549 %12 = extractelement <16 x i32> %bin.rdx4, i32 0550 ret i32 %12551}552 553define dso_local i32 @sad_16i8_512() "min-legal-vector-width"="512" {554; CHECK-SKX-LABEL: sad_16i8_512:555; CHECK-SKX: # %bb.0: # %entry556; CHECK-SKX-NEXT: vpxor %xmm0, %xmm0, %xmm0557; CHECK-SKX-NEXT: movq $-1024, %rax # imm = 0xFC00558; CHECK-SKX-NEXT: .p2align 4559; CHECK-SKX-NEXT: .LBB11_1: # %vector.body560; CHECK-SKX-NEXT: # =>This Inner Loop Header: Depth=1561; CHECK-SKX-NEXT: vmovdqu a+1024(%rax), %xmm1562; CHECK-SKX-NEXT: vpsadbw b+1024(%rax), %xmm1, %xmm1563; CHECK-SKX-NEXT: vpaddd %zmm0, %zmm1, %zmm0564; CHECK-SKX-NEXT: addq $4, %rax565; CHECK-SKX-NEXT: jne .LBB11_1566; CHECK-SKX-NEXT: # %bb.2: # %middle.block567; CHECK-SKX-NEXT: vextracti64x4 $1, %zmm0, %ymm1568; CHECK-SKX-NEXT: vpaddd %zmm1, %zmm0, %zmm0569; CHECK-SKX-NEXT: vextracti128 $1, %ymm0, %xmm1570; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0571; CHECK-SKX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]572; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0573; CHECK-SKX-NEXT: vpsrlq $32, %xmm0, %xmm1574; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0575; CHECK-SKX-NEXT: vmovd %xmm0, %eax576; CHECK-SKX-NEXT: vzeroupper577; CHECK-SKX-NEXT: retq578;579; CHECK-AVX512-LABEL: sad_16i8_512:580; CHECK-AVX512: # %bb.0: # %entry581; CHECK-AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0582; CHECK-AVX512-NEXT: movq $-1024, %rax # imm = 0xFC00583; CHECK-AVX512-NEXT: .p2align 4584; CHECK-AVX512-NEXT: .LBB11_1: # %vector.body585; CHECK-AVX512-NEXT: # =>This Inner Loop Header: Depth=1586; CHECK-AVX512-NEXT: vmovdqu a+1024(%rax), %xmm1587; CHECK-AVX512-NEXT: vpsadbw b+1024(%rax), %xmm1, %xmm1588; CHECK-AVX512-NEXT: vpaddd %zmm0, %zmm1, %zmm0589; CHECK-AVX512-NEXT: addq $4, %rax590; CHECK-AVX512-NEXT: jne .LBB11_1591; CHECK-AVX512-NEXT: # %bb.2: # %middle.block592; CHECK-AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1593; CHECK-AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0594; CHECK-AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1595; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0596; CHECK-AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]597; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0598; CHECK-AVX512-NEXT: vpsrlq $32, %xmm0, %xmm1599; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0600; CHECK-AVX512-NEXT: vmovd %xmm0, %eax601; CHECK-AVX512-NEXT: vzeroupper602; CHECK-AVX512-NEXT: retq603;604; CHECK-VBMI-LABEL: sad_16i8_512:605; CHECK-VBMI: # %bb.0: # %entry606; CHECK-VBMI-NEXT: vpxor %xmm0, %xmm0, %xmm0607; CHECK-VBMI-NEXT: movq $-1024, %rax # imm = 0xFC00608; CHECK-VBMI-NEXT: .p2align 4609; CHECK-VBMI-NEXT: .LBB11_1: # %vector.body610; CHECK-VBMI-NEXT: # =>This Inner Loop Header: Depth=1611; CHECK-VBMI-NEXT: vmovdqu a+1024(%rax), %xmm1612; CHECK-VBMI-NEXT: vpsadbw b+1024(%rax), %xmm1, %xmm1613; CHECK-VBMI-NEXT: vpaddd %zmm0, %zmm1, %zmm0614; CHECK-VBMI-NEXT: addq $4, %rax615; CHECK-VBMI-NEXT: jne .LBB11_1616; CHECK-VBMI-NEXT: # %bb.2: # %middle.block617; CHECK-VBMI-NEXT: vextracti64x4 $1, %zmm0, %ymm1618; CHECK-VBMI-NEXT: vpaddd %zmm1, %zmm0, %zmm0619; CHECK-VBMI-NEXT: vextracti128 $1, %ymm0, %xmm1620; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0621; CHECK-VBMI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]622; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0623; CHECK-VBMI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]624; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0625; CHECK-VBMI-NEXT: vmovd %xmm0, %eax626; CHECK-VBMI-NEXT: vzeroupper627; CHECK-VBMI-NEXT: retq628entry:629 br label %vector.body630 631vector.body:632 %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]633 %vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %10, %vector.body ]634 %0 = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index635 %1 = bitcast ptr %0 to ptr636 %wide.load = load <16 x i8>, ptr %1, align 4637 %2 = zext <16 x i8> %wide.load to <16 x i32>638 %3 = getelementptr inbounds [1024 x i8], ptr @b, i64 0, i64 %index639 %4 = bitcast ptr %3 to ptr640 %wide.load1 = load <16 x i8>, ptr %4, align 4641 %5 = zext <16 x i8> %wide.load1 to <16 x i32>642 %6 = sub nsw <16 x i32> %2, %5643 %7 = icmp sgt <16 x i32> %6, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>644 %8 = sub nsw <16 x i32> zeroinitializer, %6645 %9 = select <16 x i1> %7, <16 x i32> %6, <16 x i32> %8646 %10 = add nsw <16 x i32> %9, %vec.phi647 %index.next = add i64 %index, 4648 %11 = icmp eq i64 %index.next, 1024649 br i1 %11, label %middle.block, label %vector.body650 651middle.block:652 %rdx.shuf = shufflevector <16 x i32> %10, <16 x i32> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>653 %bin.rdx = add <16 x i32> %10, %rdx.shuf654 %rdx.shuf2 = shufflevector <16 x i32> %bin.rdx, <16 x i32> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>655 %bin.rdx2 = add <16 x i32> %bin.rdx, %rdx.shuf2656 %rdx.shuf3 = shufflevector <16 x i32> %bin.rdx2, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>657 %bin.rdx3 = add <16 x i32> %bin.rdx2, %rdx.shuf3658 %rdx.shuf4 = shufflevector <16 x i32> %bin.rdx3, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>659 %bin.rdx4 = add <16 x i32> %bin.rdx3, %rdx.shuf4660 %12 = extractelement <16 x i32> %bin.rdx4, i32 0661 ret i32 %12662}663 664define dso_local void @sbto16f32_256(<16 x i16> %a, ptr %res) "min-legal-vector-width"="256" {665; CHECK-LABEL: sbto16f32_256:666; CHECK: # %bb.0:667; CHECK-NEXT: vpmovw2m %ymm0, %k0668; CHECK-NEXT: kshiftrw $8, %k0, %k1669; CHECK-NEXT: vpmovm2d %k1, %ymm0670; CHECK-NEXT: vcvtdq2ps %ymm0, %ymm0671; CHECK-NEXT: vpmovm2d %k0, %ymm1672; CHECK-NEXT: vcvtdq2ps %ymm1, %ymm1673; CHECK-NEXT: vmovaps %ymm1, (%rdi)674; CHECK-NEXT: vmovaps %ymm0, 32(%rdi)675; CHECK-NEXT: vzeroupper676; CHECK-NEXT: retq677 %mask = icmp slt <16 x i16> %a, zeroinitializer678 %1 = sitofp <16 x i1> %mask to <16 x float>679 store <16 x float> %1, ptr %res680 ret void681}682 683define dso_local void @sbto16f32_512(<16 x i16> %a, ptr %res) "min-legal-vector-width"="512" {684; CHECK-LABEL: sbto16f32_512:685; CHECK: # %bb.0:686; CHECK-NEXT: vpmovw2m %ymm0, %k0687; CHECK-NEXT: vpmovm2d %k0, %zmm0688; CHECK-NEXT: vcvtdq2ps %zmm0, %zmm0689; CHECK-NEXT: vmovaps %zmm0, (%rdi)690; CHECK-NEXT: vzeroupper691; CHECK-NEXT: retq692 %mask = icmp slt <16 x i16> %a, zeroinitializer693 %1 = sitofp <16 x i1> %mask to <16 x float>694 store <16 x float> %1, ptr %res695 ret void696}697 698define dso_local void @sbto16f64_256(<16 x i16> %a, ptr %res) "min-legal-vector-width"="256" {699; CHECK-LABEL: sbto16f64_256:700; CHECK: # %bb.0:701; CHECK-NEXT: vpmovw2m %ymm0, %k0702; CHECK-NEXT: kshiftrw $8, %k0, %k1703; CHECK-NEXT: vpmovm2d %k1, %ymm0704; CHECK-NEXT: vcvtdq2pd %xmm0, %ymm1705; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm0706; CHECK-NEXT: vcvtdq2pd %xmm0, %ymm0707; CHECK-NEXT: vpmovm2d %k0, %ymm2708; CHECK-NEXT: vcvtdq2pd %xmm2, %ymm3709; CHECK-NEXT: vextracti128 $1, %ymm2, %xmm2710; CHECK-NEXT: vcvtdq2pd %xmm2, %ymm2711; CHECK-NEXT: vmovaps %ymm2, 32(%rdi)712; CHECK-NEXT: vmovaps %ymm3, (%rdi)713; CHECK-NEXT: vmovaps %ymm0, 96(%rdi)714; CHECK-NEXT: vmovaps %ymm1, 64(%rdi)715; CHECK-NEXT: vzeroupper716; CHECK-NEXT: retq717 %mask = icmp slt <16 x i16> %a, zeroinitializer718 %1 = sitofp <16 x i1> %mask to <16 x double>719 store <16 x double> %1, ptr %res720 ret void721}722 723define dso_local void @sbto16f64_512(<16 x i16> %a, ptr %res) "min-legal-vector-width"="512" {724; CHECK-LABEL: sbto16f64_512:725; CHECK: # %bb.0:726; CHECK-NEXT: vpmovw2m %ymm0, %k0727; CHECK-NEXT: vpmovm2d %k0, %zmm0728; CHECK-NEXT: vcvtdq2pd %ymm0, %zmm1729; CHECK-NEXT: vextracti64x4 $1, %zmm0, %ymm0730; CHECK-NEXT: vcvtdq2pd %ymm0, %zmm0731; CHECK-NEXT: vmovaps %zmm0, 64(%rdi)732; CHECK-NEXT: vmovaps %zmm1, (%rdi)733; CHECK-NEXT: vzeroupper734; CHECK-NEXT: retq735 %mask = icmp slt <16 x i16> %a, zeroinitializer736 %1 = sitofp <16 x i1> %mask to <16 x double>737 store <16 x double> %1, ptr %res738 ret void739}740 741define dso_local void @ubto16f32_256(<16 x i16> %a, ptr %res) "min-legal-vector-width"="256" {742; CHECK-LABEL: ubto16f32_256:743; CHECK: # %bb.0:744; CHECK-NEXT: vpmovw2m %ymm0, %k0745; CHECK-NEXT: kshiftrw $8, %k0, %k1746; CHECK-NEXT: vpmovm2d %k1, %ymm0747; CHECK-NEXT: vpsrld $31, %ymm0, %ymm0748; CHECK-NEXT: vcvtdq2ps %ymm0, %ymm0749; CHECK-NEXT: vpmovm2d %k0, %ymm1750; CHECK-NEXT: vpsrld $31, %ymm1, %ymm1751; CHECK-NEXT: vcvtdq2ps %ymm1, %ymm1752; CHECK-NEXT: vmovaps %ymm1, (%rdi)753; CHECK-NEXT: vmovaps %ymm0, 32(%rdi)754; CHECK-NEXT: vzeroupper755; CHECK-NEXT: retq756 %mask = icmp slt <16 x i16> %a, zeroinitializer757 %1 = uitofp <16 x i1> %mask to <16 x float>758 store <16 x float> %1, ptr %res759 ret void760}761 762define dso_local void @ubto16f32_512(<16 x i16> %a, ptr %res) "min-legal-vector-width"="512" {763; CHECK-LABEL: ubto16f32_512:764; CHECK: # %bb.0:765; CHECK-NEXT: vpmovw2m %ymm0, %k0766; CHECK-NEXT: vpmovm2d %k0, %zmm0767; CHECK-NEXT: vpsrld $31, %zmm0, %zmm0768; CHECK-NEXT: vcvtdq2ps %zmm0, %zmm0769; CHECK-NEXT: vmovaps %zmm0, (%rdi)770; CHECK-NEXT: vzeroupper771; CHECK-NEXT: retq772 %mask = icmp slt <16 x i16> %a, zeroinitializer773 %1 = uitofp <16 x i1> %mask to <16 x float>774 store <16 x float> %1, ptr %res775 ret void776}777 778define dso_local void @ubto16f64_256(<16 x i16> %a, ptr %res) "min-legal-vector-width"="256" {779; CHECK-LABEL: ubto16f64_256:780; CHECK: # %bb.0:781; CHECK-NEXT: vpmovw2m %ymm0, %k0782; CHECK-NEXT: kshiftrw $8, %k0, %k1783; CHECK-NEXT: vpmovm2d %k1, %ymm0784; CHECK-NEXT: vpsrld $31, %ymm0, %ymm0785; CHECK-NEXT: vcvtdq2pd %xmm0, %ymm1786; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm0787; CHECK-NEXT: vcvtdq2pd %xmm0, %ymm0788; CHECK-NEXT: vpmovm2d %k0, %ymm2789; CHECK-NEXT: vpsrld $31, %ymm2, %ymm2790; CHECK-NEXT: vcvtdq2pd %xmm2, %ymm3791; CHECK-NEXT: vextracti128 $1, %ymm2, %xmm2792; CHECK-NEXT: vcvtdq2pd %xmm2, %ymm2793; CHECK-NEXT: vmovaps %ymm2, 32(%rdi)794; CHECK-NEXT: vmovaps %ymm3, (%rdi)795; CHECK-NEXT: vmovaps %ymm0, 96(%rdi)796; CHECK-NEXT: vmovaps %ymm1, 64(%rdi)797; CHECK-NEXT: vzeroupper798; CHECK-NEXT: retq799 %mask = icmp slt <16 x i16> %a, zeroinitializer800 %1 = uitofp <16 x i1> %mask to <16 x double>801 store <16 x double> %1, ptr %res802 ret void803}804 805define dso_local void @ubto16f64_512(<16 x i16> %a, ptr %res) "min-legal-vector-width"="512" {806; CHECK-LABEL: ubto16f64_512:807; CHECK: # %bb.0:808; CHECK-NEXT: vpmovw2m %ymm0, %k0809; CHECK-NEXT: vpmovm2d %k0, %zmm0810; CHECK-NEXT: vpsrld $31, %zmm0, %zmm0811; CHECK-NEXT: vcvtdq2pd %ymm0, %zmm1812; CHECK-NEXT: vextracti64x4 $1, %zmm0, %ymm0813; CHECK-NEXT: vcvtdq2pd %ymm0, %zmm0814; CHECK-NEXT: vmovaps %zmm0, 64(%rdi)815; CHECK-NEXT: vmovaps %zmm1, (%rdi)816; CHECK-NEXT: vzeroupper817; CHECK-NEXT: retq818 %mask = icmp slt <16 x i16> %a, zeroinitializer819 %1 = uitofp <16 x i1> %mask to <16 x double>820 store <16 x double> %1, ptr %res821 ret void822}823 824define <16 x i16> @test_16f32toub_256(ptr %ptr, <16 x i16> %passthru) "min-legal-vector-width"="256" {825; CHECK-LABEL: test_16f32toub_256:826; CHECK: # %bb.0:827; CHECK-NEXT: vcvttps2dq (%rdi), %ymm1828; CHECK-NEXT: vpslld $31, %ymm1, %ymm1829; CHECK-NEXT: vpmovd2m %ymm1, %k0830; CHECK-NEXT: vcvttps2dq 32(%rdi), %ymm1831; CHECK-NEXT: vpslld $31, %ymm1, %ymm1832; CHECK-NEXT: vpmovd2m %ymm1, %k1833; CHECK-NEXT: kunpckbw %k0, %k1, %k1834; CHECK-NEXT: vmovdqu16 %ymm0, %ymm0 {%k1} {z}835; CHECK-NEXT: retq836 %a = load <16 x float>, ptr %ptr837 %mask = fptoui <16 x float> %a to <16 x i1>838 %select = select <16 x i1> %mask, <16 x i16> %passthru, <16 x i16> zeroinitializer839 ret <16 x i16> %select840}841 842define <16 x i16> @test_16f32toub_512(ptr %ptr, <16 x i16> %passthru) "min-legal-vector-width"="512" {843; CHECK-LABEL: test_16f32toub_512:844; CHECK: # %bb.0:845; CHECK-NEXT: vcvttps2dq (%rdi), %zmm1846; CHECK-NEXT: vpslld $31, %zmm1, %zmm1847; CHECK-NEXT: vpmovd2m %zmm1, %k1848; CHECK-NEXT: vmovdqu16 %ymm0, %ymm0 {%k1} {z}849; CHECK-NEXT: retq850 %a = load <16 x float>, ptr %ptr851 %mask = fptoui <16 x float> %a to <16 x i1>852 %select = select <16 x i1> %mask, <16 x i16> %passthru, <16 x i16> zeroinitializer853 ret <16 x i16> %select854}855 856define <16 x i16> @test_16f32tosb_256(ptr %ptr, <16 x i16> %passthru) "min-legal-vector-width"="256" {857; CHECK-LABEL: test_16f32tosb_256:858; CHECK: # %bb.0:859; CHECK-NEXT: vcvttps2dq (%rdi), %ymm1860; CHECK-NEXT: vpmovd2m %ymm1, %k0861; CHECK-NEXT: vcvttps2dq 32(%rdi), %ymm1862; CHECK-NEXT: vpmovd2m %ymm1, %k1863; CHECK-NEXT: kunpckbw %k0, %k1, %k1864; CHECK-NEXT: vmovdqu16 %ymm0, %ymm0 {%k1} {z}865; CHECK-NEXT: retq866 %a = load <16 x float>, ptr %ptr867 %mask = fptosi <16 x float> %a to <16 x i1>868 %select = select <16 x i1> %mask, <16 x i16> %passthru, <16 x i16> zeroinitializer869 ret <16 x i16> %select870}871 872define <16 x i16> @test_16f32tosb_512(ptr %ptr, <16 x i16> %passthru) "min-legal-vector-width"="512" {873; CHECK-LABEL: test_16f32tosb_512:874; CHECK: # %bb.0:875; CHECK-NEXT: vcvttps2dq (%rdi), %zmm1876; CHECK-NEXT: vpmovd2m %zmm1, %k1877; CHECK-NEXT: vmovdqu16 %ymm0, %ymm0 {%k1} {z}878; CHECK-NEXT: retq879 %a = load <16 x float>, ptr %ptr880 %mask = fptosi <16 x float> %a to <16 x i1>881 %select = select <16 x i1> %mask, <16 x i16> %passthru, <16 x i16> zeroinitializer882 ret <16 x i16> %select883}884 885define dso_local void @mul256(ptr %a, ptr %b, ptr %c) "min-legal-vector-width"="256" {886; CHECK-SKX-NOVBMI-LABEL: mul256:887; CHECK-SKX-NOVBMI: # %bb.0:888; CHECK-SKX-NOVBMI-NEXT: vmovdqa (%rdi), %ymm0889; CHECK-SKX-NOVBMI-NEXT: vmovdqa 32(%rdi), %ymm1890; CHECK-SKX-NOVBMI-NEXT: vmovdqa (%rsi), %ymm2891; CHECK-SKX-NOVBMI-NEXT: vmovdqa 32(%rsi), %ymm3892; CHECK-SKX-NOVBMI-NEXT: vpmullw %ymm3, %ymm1, %ymm4893; CHECK-SKX-NOVBMI-NEXT: vpbroadcastd {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]894; CHECK-SKX-NOVBMI-NEXT: vpandn %ymm3, %ymm5, %ymm3895; CHECK-SKX-NOVBMI-NEXT: vpmaddubsw %ymm3, %ymm1, %ymm1896; CHECK-SKX-NOVBMI-NEXT: vpsllw $8, %ymm1, %ymm1897; CHECK-SKX-NOVBMI-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm5)898; CHECK-SKX-NOVBMI-NEXT: vpmullw %ymm2, %ymm0, %ymm3899; CHECK-SKX-NOVBMI-NEXT: vpandn %ymm2, %ymm5, %ymm2900; CHECK-SKX-NOVBMI-NEXT: vpmaddubsw %ymm2, %ymm0, %ymm0901; CHECK-SKX-NOVBMI-NEXT: vpsllw $8, %ymm0, %ymm0902; CHECK-SKX-NOVBMI-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm3 & ymm5)903; CHECK-SKX-NOVBMI-NEXT: vmovdqa %ymm0, (%rdx)904; CHECK-SKX-NOVBMI-NEXT: vmovdqa %ymm1, 32(%rdx)905; CHECK-SKX-NOVBMI-NEXT: vzeroupper906; CHECK-SKX-NOVBMI-NEXT: retq907;908; CHECK-SKX-VBMI-LABEL: mul256:909; CHECK-SKX-VBMI: # %bb.0:910; CHECK-SKX-VBMI-NEXT: vmovdqa (%rdi), %ymm0911; CHECK-SKX-VBMI-NEXT: vmovdqa 32(%rdi), %ymm1912; CHECK-SKX-VBMI-NEXT: vmovdqa (%rsi), %ymm2913; CHECK-SKX-VBMI-NEXT: vmovdqa 32(%rsi), %ymm3914; CHECK-SKX-VBMI-NEXT: vpmullw %ymm3, %ymm1, %ymm4915; CHECK-SKX-VBMI-NEXT: vpbroadcastd {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]916; CHECK-SKX-VBMI-NEXT: vpandn %ymm3, %ymm5, %ymm3917; CHECK-SKX-VBMI-NEXT: vpmaddubsw %ymm3, %ymm1, %ymm1918; CHECK-SKX-VBMI-NEXT: vmovdqa {{.*#+}} ymm3 = [0,32,2,34,4,36,6,38,8,40,10,42,12,44,14,46,16,48,18,50,20,52,22,54,24,56,26,58,28,60,30,62]919; CHECK-SKX-VBMI-NEXT: vpermt2b %ymm1, %ymm3, %ymm4920; CHECK-SKX-VBMI-NEXT: vpmullw %ymm2, %ymm0, %ymm1921; CHECK-SKX-VBMI-NEXT: vpandn %ymm2, %ymm5, %ymm2922; CHECK-SKX-VBMI-NEXT: vpmaddubsw %ymm2, %ymm0, %ymm0923; CHECK-SKX-VBMI-NEXT: vpermt2b %ymm0, %ymm3, %ymm1924; CHECK-SKX-VBMI-NEXT: vmovdqa %ymm1, (%rdx)925; CHECK-SKX-VBMI-NEXT: vmovdqa %ymm4, 32(%rdx)926; CHECK-SKX-VBMI-NEXT: vzeroupper927; CHECK-SKX-VBMI-NEXT: retq928;929; CHECK-AVX512-LABEL: mul256:930; CHECK-AVX512: # %bb.0:931; CHECK-AVX512-NEXT: vmovdqa (%rdi), %ymm0932; CHECK-AVX512-NEXT: vmovdqa 32(%rdi), %ymm1933; CHECK-AVX512-NEXT: vmovdqa (%rsi), %ymm2934; CHECK-AVX512-NEXT: vmovdqa 32(%rsi), %ymm3935; CHECK-AVX512-NEXT: vpmullw %ymm3, %ymm1, %ymm4936; CHECK-AVX512-NEXT: vpbroadcastd {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]937; CHECK-AVX512-NEXT: vpandn %ymm3, %ymm5, %ymm3938; CHECK-AVX512-NEXT: vpmaddubsw %ymm3, %ymm1, %ymm1939; CHECK-AVX512-NEXT: vpsllw $8, %ymm1, %ymm1940; CHECK-AVX512-NEXT: vpternlogq {{.*#+}} ymm1 = ymm1 | (ymm4 & ymm5)941; CHECK-AVX512-NEXT: vpmullw %ymm2, %ymm0, %ymm3942; CHECK-AVX512-NEXT: vpandn %ymm2, %ymm5, %ymm2943; CHECK-AVX512-NEXT: vpmaddubsw %ymm2, %ymm0, %ymm0944; CHECK-AVX512-NEXT: vpsllw $8, %ymm0, %ymm0945; CHECK-AVX512-NEXT: vpternlogq {{.*#+}} ymm0 = ymm0 | (ymm3 & ymm5)946; CHECK-AVX512-NEXT: vmovdqa %ymm0, (%rdx)947; CHECK-AVX512-NEXT: vmovdqa %ymm1, 32(%rdx)948; CHECK-AVX512-NEXT: vzeroupper949; CHECK-AVX512-NEXT: retq950;951; CHECK-VBMI-LABEL: mul256:952; CHECK-VBMI: # %bb.0:953; CHECK-VBMI-NEXT: vmovdqa (%rdi), %ymm0954; CHECK-VBMI-NEXT: vmovdqa 32(%rdi), %ymm1955; CHECK-VBMI-NEXT: vmovdqa (%rsi), %ymm2956; CHECK-VBMI-NEXT: vmovdqa 32(%rsi), %ymm3957; CHECK-VBMI-NEXT: vpmullw %ymm3, %ymm1, %ymm4958; CHECK-VBMI-NEXT: vpbroadcastd {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]959; CHECK-VBMI-NEXT: vpandn %ymm3, %ymm5, %ymm3960; CHECK-VBMI-NEXT: vpmaddubsw %ymm3, %ymm1, %ymm1961; CHECK-VBMI-NEXT: vmovdqa {{.*#+}} ymm3 = [0,32,2,34,4,36,6,38,8,40,10,42,12,44,14,46,16,48,18,50,20,52,22,54,24,56,26,58,28,60,30,62]962; CHECK-VBMI-NEXT: vpermt2b %ymm1, %ymm3, %ymm4963; CHECK-VBMI-NEXT: vpmullw %ymm2, %ymm0, %ymm1964; CHECK-VBMI-NEXT: vpandn %ymm2, %ymm5, %ymm2965; CHECK-VBMI-NEXT: vpmaddubsw %ymm2, %ymm0, %ymm0966; CHECK-VBMI-NEXT: vpermt2b %ymm0, %ymm3, %ymm1967; CHECK-VBMI-NEXT: vmovdqa %ymm1, (%rdx)968; CHECK-VBMI-NEXT: vmovdqa %ymm4, 32(%rdx)969; CHECK-VBMI-NEXT: vzeroupper970; CHECK-VBMI-NEXT: retq971 %d = load <64 x i8>, ptr %a972 %e = load <64 x i8>, ptr %b973 %f = mul <64 x i8> %d, %e974 store <64 x i8> %f, ptr %c975 ret void976}977 978define dso_local void @mul512(ptr %a, ptr %b, ptr %c) "min-legal-vector-width"="512" {979; CHECK-SKX-NOVBMI-LABEL: mul512:980; CHECK-SKX-NOVBMI: # %bb.0:981; CHECK-SKX-NOVBMI-NEXT: vmovdqa64 (%rdi), %zmm0982; CHECK-SKX-NOVBMI-NEXT: vmovdqa64 (%rsi), %zmm1983; CHECK-SKX-NOVBMI-NEXT: vpmullw %zmm1, %zmm0, %zmm2984; CHECK-SKX-NOVBMI-NEXT: vpbroadcastd {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]985; CHECK-SKX-NOVBMI-NEXT: vpandnq %zmm1, %zmm3, %zmm1986; CHECK-SKX-NOVBMI-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0987; CHECK-SKX-NOVBMI-NEXT: vpsllw $8, %zmm0, %zmm0988; CHECK-SKX-NOVBMI-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm2 & zmm3)989; CHECK-SKX-NOVBMI-NEXT: vmovdqa64 %zmm0, (%rdx)990; CHECK-SKX-NOVBMI-NEXT: vzeroupper991; CHECK-SKX-NOVBMI-NEXT: retq992;993; CHECK-SKX-VBMI-LABEL: mul512:994; CHECK-SKX-VBMI: # %bb.0:995; CHECK-SKX-VBMI-NEXT: vmovdqa64 (%rdi), %zmm0996; CHECK-SKX-VBMI-NEXT: vmovdqa64 (%rsi), %zmm1997; CHECK-SKX-VBMI-NEXT: vpmullw %zmm1, %zmm0, %zmm2998; CHECK-SKX-VBMI-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1999; CHECK-SKX-VBMI-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm01000; CHECK-SKX-VBMI-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,64,2,66,4,68,6,70,8,72,10,74,12,76,14,78,16,80,18,82,20,84,22,86,24,88,26,90,28,92,30,94,32,96,34,98,36,100,38,102,40,104,42,106,44,108,46,110,48,112,50,114,52,116,54,118,56,120,58,122,60,124,62,126]1001; CHECK-SKX-VBMI-NEXT: vpermi2b %zmm0, %zmm2, %zmm11002; CHECK-SKX-VBMI-NEXT: vmovdqa64 %zmm1, (%rdx)1003; CHECK-SKX-VBMI-NEXT: vzeroupper1004; CHECK-SKX-VBMI-NEXT: retq1005;1006; CHECK-AVX512-LABEL: mul512:1007; CHECK-AVX512: # %bb.0:1008; CHECK-AVX512-NEXT: vmovdqa64 (%rdi), %zmm01009; CHECK-AVX512-NEXT: vmovdqa64 (%rsi), %zmm11010; CHECK-AVX512-NEXT: vpmullw %zmm1, %zmm0, %zmm21011; CHECK-AVX512-NEXT: vpbroadcastd {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]1012; CHECK-AVX512-NEXT: vpandnq %zmm1, %zmm3, %zmm11013; CHECK-AVX512-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm01014; CHECK-AVX512-NEXT: vpsllw $8, %zmm0, %zmm01015; CHECK-AVX512-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 | (zmm2 & zmm3)1016; CHECK-AVX512-NEXT: vmovdqa64 %zmm0, (%rdx)1017; CHECK-AVX512-NEXT: vzeroupper1018; CHECK-AVX512-NEXT: retq1019;1020; CHECK-VBMI-LABEL: mul512:1021; CHECK-VBMI: # %bb.0:1022; CHECK-VBMI-NEXT: vmovdqa64 (%rdi), %zmm01023; CHECK-VBMI-NEXT: vmovdqa64 (%rsi), %zmm11024; CHECK-VBMI-NEXT: vpmullw %zmm1, %zmm0, %zmm21025; CHECK-VBMI-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm11026; CHECK-VBMI-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm01027; CHECK-VBMI-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,64,2,66,4,68,6,70,8,72,10,74,12,76,14,78,16,80,18,82,20,84,22,86,24,88,26,90,28,92,30,94,32,96,34,98,36,100,38,102,40,104,42,106,44,108,46,110,48,112,50,114,52,116,54,118,56,120,58,122,60,124,62,126]1028; CHECK-VBMI-NEXT: vpermi2b %zmm0, %zmm2, %zmm11029; CHECK-VBMI-NEXT: vmovdqa64 %zmm1, (%rdx)1030; CHECK-VBMI-NEXT: vzeroupper1031; CHECK-VBMI-NEXT: retq1032 %d = load <64 x i8>, ptr %a1033 %e = load <64 x i8>, ptr %b1034 %f = mul <64 x i8> %d, %e1035 store <64 x i8> %f, ptr %c1036 ret void1037}1038 1039; This threw an assertion at one point.1040define <4 x i32> @mload_v4i32(<4 x i32> %trigger, ptr %addr, <4 x i32> %dst) "min-legal-vector-width"="256" {1041; CHECK-LABEL: mload_v4i32:1042; CHECK: # %bb.0:1043; CHECK-NEXT: vptestnmd %xmm0, %xmm0, %k11044; CHECK-NEXT: vpblendmd (%rdi), %xmm1, %xmm0 {%k1}1045; CHECK-NEXT: retq1046 %mask = icmp eq <4 x i32> %trigger, zeroinitializer1047 %res = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %addr, i32 4, <4 x i1> %mask, <4 x i32> %dst)1048 ret <4 x i32> %res1049}1050declare <4 x i32> @llvm.masked.load.v4i32.p0(ptr, i32, <4 x i1>, <4 x i32>)1051 1052define <16 x i32> @trunc_v16i64_v16i32(ptr %x) nounwind "min-legal-vector-width"="256" {1053; CHECK-LABEL: trunc_v16i64_v16i32:1054; CHECK: # %bb.0:1055; CHECK-NEXT: vmovdqa (%rdi), %ymm01056; CHECK-NEXT: vmovdqa 32(%rdi), %ymm11057; CHECK-NEXT: vmovdqa 64(%rdi), %ymm21058; CHECK-NEXT: vmovdqa 96(%rdi), %ymm31059; CHECK-NEXT: vpmovqd %ymm0, %xmm01060; CHECK-NEXT: vpmovqd %ymm1, %xmm11061; CHECK-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm01062; CHECK-NEXT: vpmovqd %ymm2, %xmm11063; CHECK-NEXT: vpmovqd %ymm3, %xmm21064; CHECK-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm11065; CHECK-NEXT: retq1066 %a = load <16 x i64>, ptr %x1067 %b = trunc <16 x i64> %a to <16 x i32>1068 ret <16 x i32> %b1069}1070 1071define <16 x i8> @trunc_v16i64_v16i8(ptr %x) nounwind "min-legal-vector-width"="256" {1072; CHECK-LABEL: trunc_v16i64_v16i8:1073; CHECK: # %bb.0:1074; CHECK-NEXT: vmovdqa (%rdi), %ymm01075; CHECK-NEXT: vmovdqa 32(%rdi), %ymm11076; CHECK-NEXT: vmovdqa 64(%rdi), %ymm21077; CHECK-NEXT: vmovdqa 96(%rdi), %ymm31078; CHECK-NEXT: vpmovqb %ymm3, %xmm31079; CHECK-NEXT: vpmovqb %ymm2, %xmm21080; CHECK-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]1081; CHECK-NEXT: vpmovqb %ymm1, %xmm11082; CHECK-NEXT: vpmovqb %ymm0, %xmm01083; CHECK-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]1084; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]1085; CHECK-NEXT: vzeroupper1086; CHECK-NEXT: retq1087 %a = load <16 x i64>, ptr %x1088 %b = trunc <16 x i64> %a to <16 x i8>1089 ret <16 x i8> %b1090}1091 1092define <16 x i8> @trunc_v16i32_v16i8(ptr %x) nounwind "min-legal-vector-width"="256" {1093; CHECK-LABEL: trunc_v16i32_v16i8:1094; CHECK: # %bb.0:1095; CHECK-NEXT: vmovdqa (%rdi), %ymm01096; CHECK-NEXT: vmovdqa 32(%rdi), %ymm11097; CHECK-NEXT: vpmovdb %ymm1, %xmm11098; CHECK-NEXT: vpmovdb %ymm0, %xmm01099; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]1100; CHECK-NEXT: vzeroupper1101; CHECK-NEXT: retq1102 %a = load <16 x i32>, ptr %x1103 %b = trunc <16 x i32> %a to <16 x i8>1104 ret <16 x i8> %b1105}1106 1107define <8 x i8> @trunc_v8i64_v8i8(ptr %x) nounwind "min-legal-vector-width"="256" {1108; CHECK-LABEL: trunc_v8i64_v8i8:1109; CHECK: # %bb.0:1110; CHECK-NEXT: vmovdqa (%rdi), %ymm01111; CHECK-NEXT: vmovdqa 32(%rdi), %ymm11112; CHECK-NEXT: vpmovqb %ymm1, %xmm11113; CHECK-NEXT: vpmovqb %ymm0, %xmm01114; CHECK-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]1115; CHECK-NEXT: vzeroupper1116; CHECK-NEXT: retq1117 %a = load <8 x i64>, ptr %x1118 %b = trunc <8 x i64> %a to <8 x i8>1119 ret <8 x i8> %b1120}1121 1122define <8 x i16> @trunc_v8i64_v8i16(ptr %x) nounwind "min-legal-vector-width"="256" {1123; CHECK-LABEL: trunc_v8i64_v8i16:1124; CHECK: # %bb.0:1125; CHECK-NEXT: vmovdqa (%rdi), %ymm01126; CHECK-NEXT: vmovdqa 32(%rdi), %ymm11127; CHECK-NEXT: vpmovqw %ymm1, %xmm11128; CHECK-NEXT: vpmovqw %ymm0, %xmm01129; CHECK-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]1130; CHECK-NEXT: vzeroupper1131; CHECK-NEXT: retq1132 %a = load <8 x i64>, ptr %x1133 %b = trunc <8 x i64> %a to <8 x i16>1134 ret <8 x i16> %b1135}1136 1137define <8 x i32> @trunc_v8i64_v8i32_zeroes(ptr %x) nounwind "min-legal-vector-width"="256" {1138; CHECK-LABEL: trunc_v8i64_v8i32_zeroes:1139; CHECK: # %bb.0:1140; CHECK-NEXT: vpsrlq $48, 32(%rdi), %ymm01141; CHECK-NEXT: vpsrlq $48, (%rdi), %ymm11142; CHECK-NEXT: vpackusdw %ymm0, %ymm1, %ymm01143; CHECK-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]1144; CHECK-NEXT: retq1145 %a = load <8 x i64>, ptr %x1146 %b = lshr <8 x i64> %a, <i64 48, i64 48, i64 48, i64 48, i64 48, i64 48, i64 48, i64 48>1147 %c = trunc <8 x i64> %b to <8 x i32>1148 ret <8 x i32> %c1149}1150 1151define <16 x i16> @trunc_v16i32_v16i16_zeroes(ptr %x) nounwind "min-legal-vector-width"="256" {1152; CHECK-LABEL: trunc_v16i32_v16i16_zeroes:1153; CHECK: # %bb.0:1154; CHECK-NEXT: vmovdqa (%rdi), %ymm11155; CHECK-NEXT: vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]1156; CHECK-NEXT: vpermi2w 32(%rdi), %ymm1, %ymm01157; CHECK-NEXT: retq1158 %a = load <16 x i32>, ptr %x1159 %b = lshr <16 x i32> %a, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>1160 %c = trunc <16 x i32> %b to <16 x i16>1161 ret <16 x i16> %c1162}1163 1164define <32 x i8> @trunc_v32i16_v32i8_zeroes(ptr %x) nounwind "min-legal-vector-width"="256" {1165; CHECK-SKX-NOVBMI-LABEL: trunc_v32i16_v32i8_zeroes:1166; CHECK-SKX-NOVBMI: # %bb.0:1167; CHECK-SKX-NOVBMI-NEXT: vpsrlw $8, 32(%rdi), %ymm01168; CHECK-SKX-NOVBMI-NEXT: vpsrlw $8, (%rdi), %ymm11169; CHECK-SKX-NOVBMI-NEXT: vpackuswb %ymm0, %ymm1, %ymm01170; CHECK-SKX-NOVBMI-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]1171; CHECK-SKX-NOVBMI-NEXT: retq1172;1173; CHECK-SKX-VBMI-LABEL: trunc_v32i16_v32i8_zeroes:1174; CHECK-SKX-VBMI: # %bb.0:1175; CHECK-SKX-VBMI-NEXT: vmovdqa (%rdi), %ymm11176; CHECK-SKX-VBMI-NEXT: vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]1177; CHECK-SKX-VBMI-NEXT: vpermi2b 32(%rdi), %ymm1, %ymm01178; CHECK-SKX-VBMI-NEXT: retq1179;1180; CHECK-AVX512-LABEL: trunc_v32i16_v32i8_zeroes:1181; CHECK-AVX512: # %bb.0:1182; CHECK-AVX512-NEXT: vpsrlw $8, 32(%rdi), %ymm01183; CHECK-AVX512-NEXT: vpsrlw $8, (%rdi), %ymm11184; CHECK-AVX512-NEXT: vpackuswb %ymm0, %ymm1, %ymm01185; CHECK-AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]1186; CHECK-AVX512-NEXT: retq1187;1188; CHECK-VBMI-LABEL: trunc_v32i16_v32i8_zeroes:1189; CHECK-VBMI: # %bb.0:1190; CHECK-VBMI-NEXT: vmovdqa (%rdi), %ymm11191; CHECK-VBMI-NEXT: vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]1192; CHECK-VBMI-NEXT: vpermi2b 32(%rdi), %ymm1, %ymm01193; CHECK-VBMI-NEXT: retq1194 %a = load <32 x i16>, ptr %x1195 %b = lshr <32 x i16> %a, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>1196 %c = trunc <32 x i16> %b to <32 x i8>1197 ret <32 x i8> %c1198}1199 1200define <8 x i32> @trunc_v8i64_v8i32_sign(ptr %x) nounwind "min-legal-vector-width"="256" {1201; CHECK-LABEL: trunc_v8i64_v8i32_sign:1202; CHECK: # %bb.0:1203; CHECK-NEXT: vpsraq $48, 32(%rdi), %ymm01204; CHECK-NEXT: vpsraq $48, (%rdi), %ymm11205; CHECK-NEXT: vpackssdw %ymm0, %ymm1, %ymm01206; CHECK-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]1207; CHECK-NEXT: retq1208 %a = load <8 x i64>, ptr %x1209 %b = ashr <8 x i64> %a, <i64 48, i64 48, i64 48, i64 48, i64 48, i64 48, i64 48, i64 48>1210 %c = trunc <8 x i64> %b to <8 x i32>1211 ret <8 x i32> %c1212}1213 1214define <16 x i16> @trunc_v16i32_v16i16_sign(ptr %x) nounwind "min-legal-vector-width"="256" {1215; CHECK-LABEL: trunc_v16i32_v16i16_sign:1216; CHECK: # %bb.0:1217; CHECK-NEXT: vmovdqa (%rdi), %ymm11218; CHECK-NEXT: vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31]1219; CHECK-NEXT: vpermi2w 32(%rdi), %ymm1, %ymm01220; CHECK-NEXT: retq1221 %a = load <16 x i32>, ptr %x1222 %b = ashr <16 x i32> %a, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>1223 %c = trunc <16 x i32> %b to <16 x i16>1224 ret <16 x i16> %c1225}1226 1227define <32 x i8> @trunc_v32i16_v32i8_sign(ptr %x) nounwind "min-legal-vector-width"="256" {1228; CHECK-SKX-NOVBMI-LABEL: trunc_v32i16_v32i8_sign:1229; CHECK-SKX-NOVBMI: # %bb.0:1230; CHECK-SKX-NOVBMI-NEXT: vpsrlw $8, 32(%rdi), %ymm01231; CHECK-SKX-NOVBMI-NEXT: vpsrlw $8, (%rdi), %ymm11232; CHECK-SKX-NOVBMI-NEXT: vpackuswb %ymm0, %ymm1, %ymm01233; CHECK-SKX-NOVBMI-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]1234; CHECK-SKX-NOVBMI-NEXT: retq1235;1236; CHECK-SKX-VBMI-LABEL: trunc_v32i16_v32i8_sign:1237; CHECK-SKX-VBMI: # %bb.0:1238; CHECK-SKX-VBMI-NEXT: vmovdqa (%rdi), %ymm11239; CHECK-SKX-VBMI-NEXT: vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]1240; CHECK-SKX-VBMI-NEXT: vpermi2b 32(%rdi), %ymm1, %ymm01241; CHECK-SKX-VBMI-NEXT: retq1242;1243; CHECK-AVX512-LABEL: trunc_v32i16_v32i8_sign:1244; CHECK-AVX512: # %bb.0:1245; CHECK-AVX512-NEXT: vpsrlw $8, 32(%rdi), %ymm01246; CHECK-AVX512-NEXT: vpsrlw $8, (%rdi), %ymm11247; CHECK-AVX512-NEXT: vpackuswb %ymm0, %ymm1, %ymm01248; CHECK-AVX512-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]1249; CHECK-AVX512-NEXT: retq1250;1251; CHECK-VBMI-LABEL: trunc_v32i16_v32i8_sign:1252; CHECK-VBMI: # %bb.0:1253; CHECK-VBMI-NEXT: vmovdqa (%rdi), %ymm11254; CHECK-VBMI-NEXT: vmovdqa {{.*#+}} ymm0 = [1,3,5,7,9,11,13,15,17,19,21,23,25,27,29,31,33,35,37,39,41,43,45,47,49,51,53,55,57,59,61,63]1255; CHECK-VBMI-NEXT: vpermi2b 32(%rdi), %ymm1, %ymm01256; CHECK-VBMI-NEXT: retq1257 %a = load <32 x i16>, ptr %x1258 %b = ashr <32 x i16> %a, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>1259 %c = trunc <32 x i16> %b to <32 x i8>1260 ret <32 x i8> %c1261}1262 1263define dso_local void @zext_v16i8_v16i64(<16 x i8> %x, ptr %y) nounwind "min-legal-vector-width"="256" {1264; CHECK-LABEL: zext_v16i8_v16i64:1265; CHECK: # %bb.0:1266; CHECK-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero1267; CHECK-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]1268; CHECK-NEXT: vpmovzxwq {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero1269; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm11270; CHECK-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]1271; CHECK-NEXT: vpmovzxwq {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero1272; CHECK-NEXT: vpmovzxwq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero1273; CHECK-NEXT: vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero1274; CHECK-NEXT: vmovdqa %ymm0, (%rdi)1275; CHECK-NEXT: vmovdqa %ymm1, 64(%rdi)1276; CHECK-NEXT: vmovdqa %ymm3, 96(%rdi)1277; CHECK-NEXT: vmovdqa %ymm2, 32(%rdi)1278; CHECK-NEXT: vzeroupper1279; CHECK-NEXT: retq1280 %a = zext <16 x i8> %x to <16 x i64>1281 store <16 x i64> %a, ptr %y1282 ret void1283}1284 1285define dso_local void @sext_v16i8_v16i64(<16 x i8> %x, ptr %y) nounwind "min-legal-vector-width"="256" {1286; CHECK-LABEL: sext_v16i8_v16i64:1287; CHECK: # %bb.0:1288; CHECK-NEXT: vpmovsxbw %xmm0, %ymm11289; CHECK-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]1290; CHECK-NEXT: vpmovsxwq %xmm2, %ymm21291; CHECK-NEXT: vextracti128 $1, %ymm1, %xmm11292; CHECK-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]1293; CHECK-NEXT: vpmovsxwq %xmm3, %ymm31294; CHECK-NEXT: vpmovsxwq %xmm1, %ymm11295; CHECK-NEXT: vpmovsxbq %xmm0, %ymm01296; CHECK-NEXT: vmovdqa %ymm0, (%rdi)1297; CHECK-NEXT: vmovdqa %ymm1, 64(%rdi)1298; CHECK-NEXT: vmovdqa %ymm3, 96(%rdi)1299; CHECK-NEXT: vmovdqa %ymm2, 32(%rdi)1300; CHECK-NEXT: vzeroupper1301; CHECK-NEXT: retq1302 %a = sext <16 x i8> %x to <16 x i64>1303 store <16 x i64> %a, ptr %y1304 ret void1305}1306 1307define dso_local void @vselect_split_v8i16_setcc(<8 x i16> %s, <8 x i16> %t, ptr %p, ptr %q, ptr %r) "min-legal-vector-width"="256" {1308; CHECK-LABEL: vselect_split_v8i16_setcc:1309; CHECK: # %bb.0:1310; CHECK-NEXT: vmovdqa (%rsi), %ymm21311; CHECK-NEXT: vmovdqa 32(%rsi), %ymm31312; CHECK-NEXT: vpcmpeqw %xmm1, %xmm0, %k11313; CHECK-NEXT: kshiftrb $4, %k1, %k21314; CHECK-NEXT: vmovdqa64 32(%rdi), %ymm3 {%k2}1315; CHECK-NEXT: vmovdqa64 (%rdi), %ymm2 {%k1}1316; CHECK-NEXT: vmovdqa %ymm2, (%rdx)1317; CHECK-NEXT: vmovdqa %ymm3, 32(%rdx)1318; CHECK-NEXT: vzeroupper1319; CHECK-NEXT: retq1320 %x = load <8 x i64>, ptr %p1321 %y = load <8 x i64>, ptr %q1322 %a = icmp eq <8 x i16> %s, %t1323 %b = select <8 x i1> %a, <8 x i64> %x, <8 x i64> %y1324 store <8 x i64> %b, ptr %r1325 ret void1326}1327 1328define dso_local void @vselect_split_v8i32_setcc(<8 x i32> %s, <8 x i32> %t, ptr %p, ptr %q, ptr %r) "min-legal-vector-width"="256" {1329; CHECK-LABEL: vselect_split_v8i32_setcc:1330; CHECK: # %bb.0:1331; CHECK-NEXT: vmovdqa (%rsi), %ymm21332; CHECK-NEXT: vmovdqa 32(%rsi), %ymm31333; CHECK-NEXT: vpcmpeqd %ymm1, %ymm0, %k11334; CHECK-NEXT: kshiftrb $4, %k1, %k21335; CHECK-NEXT: vmovdqa64 32(%rdi), %ymm3 {%k2}1336; CHECK-NEXT: vmovdqa64 (%rdi), %ymm2 {%k1}1337; CHECK-NEXT: vmovdqa %ymm2, (%rdx)1338; CHECK-NEXT: vmovdqa %ymm3, 32(%rdx)1339; CHECK-NEXT: vzeroupper1340; CHECK-NEXT: retq1341 %x = load <8 x i64>, ptr %p1342 %y = load <8 x i64>, ptr %q1343 %a = icmp eq <8 x i32> %s, %t1344 %b = select <8 x i1> %a, <8 x i64> %x, <8 x i64> %y1345 store <8 x i64> %b, ptr %r1346 ret void1347}1348 1349define dso_local void @vselect_split_v16i8_setcc(<16 x i8> %s, <16 x i8> %t, ptr %p, ptr %q, ptr %r) "min-legal-vector-width"="256" {1350; CHECK-LABEL: vselect_split_v16i8_setcc:1351; CHECK: # %bb.0:1352; CHECK-NEXT: vmovdqa (%rsi), %ymm21353; CHECK-NEXT: vmovdqa 32(%rsi), %ymm31354; CHECK-NEXT: vpcmpeqb %xmm1, %xmm0, %k11355; CHECK-NEXT: kshiftrw $8, %k1, %k21356; CHECK-NEXT: vmovdqa32 32(%rdi), %ymm3 {%k2}1357; CHECK-NEXT: vmovdqa32 (%rdi), %ymm2 {%k1}1358; CHECK-NEXT: vmovdqa %ymm2, (%rdx)1359; CHECK-NEXT: vmovdqa %ymm3, 32(%rdx)1360; CHECK-NEXT: vzeroupper1361; CHECK-NEXT: retq1362 %x = load <16 x i32>, ptr %p1363 %y = load <16 x i32>, ptr %q1364 %a = icmp eq <16 x i8> %s, %t1365 %b = select <16 x i1> %a, <16 x i32> %x, <16 x i32> %y1366 store <16 x i32> %b, ptr %r1367 ret void1368}1369 1370define dso_local void @vselect_split_v16i16_setcc(<16 x i16> %s, <16 x i16> %t, ptr %p, ptr %q, ptr %r) "min-legal-vector-width"="256" {1371; CHECK-LABEL: vselect_split_v16i16_setcc:1372; CHECK: # %bb.0:1373; CHECK-NEXT: vmovdqa (%rsi), %ymm21374; CHECK-NEXT: vmovdqa 32(%rsi), %ymm31375; CHECK-NEXT: vpcmpeqw %ymm1, %ymm0, %k11376; CHECK-NEXT: kshiftrw $8, %k1, %k21377; CHECK-NEXT: vmovdqa32 32(%rdi), %ymm3 {%k2}1378; CHECK-NEXT: vmovdqa32 (%rdi), %ymm2 {%k1}1379; CHECK-NEXT: vmovdqa %ymm2, (%rdx)1380; CHECK-NEXT: vmovdqa %ymm3, 32(%rdx)1381; CHECK-NEXT: vzeroupper1382; CHECK-NEXT: retq1383 %x = load <16 x i32>, ptr %p1384 %y = load <16 x i32>, ptr %q1385 %a = icmp eq <16 x i16> %s, %t1386 %b = select <16 x i1> %a, <16 x i32> %x, <16 x i32> %y1387 store <16 x i32> %b, ptr %r1388 ret void1389}1390 1391define <16 x i8> @trunc_packus_v16i32_v16i8(ptr %p) "min-legal-vector-width"="256" {1392; CHECK-LABEL: trunc_packus_v16i32_v16i8:1393; CHECK: # %bb.0:1394; CHECK-NEXT: vmovdqa (%rdi), %ymm01395; CHECK-NEXT: vpackusdw 32(%rdi), %ymm0, %ymm01396; CHECK-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]1397; CHECK-NEXT: vpmovuswb %ymm0, %xmm01398; CHECK-NEXT: vzeroupper1399; CHECK-NEXT: retq1400 %a = load <16 x i32>, ptr %p1401 %b = icmp slt <16 x i32> %a, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>1402 %c = select <16 x i1> %b, <16 x i32> %a, <16 x i32> <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>1403 %d = icmp sgt <16 x i32> %c, zeroinitializer1404 %e = select <16 x i1> %d, <16 x i32> %c, <16 x i32> zeroinitializer1405 %f = trunc <16 x i32> %e to <16 x i8>1406 ret <16 x i8> %f1407}1408 1409define dso_local void @trunc_packus_v16i32_v16i8_store(ptr %p, ptr %q) "min-legal-vector-width"="256" {1410; CHECK-LABEL: trunc_packus_v16i32_v16i8_store:1411; CHECK: # %bb.0:1412; CHECK-NEXT: vmovdqa (%rdi), %ymm01413; CHECK-NEXT: vpackusdw 32(%rdi), %ymm0, %ymm01414; CHECK-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]1415; CHECK-NEXT: vpmovuswb %ymm0, (%rsi)1416; CHECK-NEXT: vzeroupper1417; CHECK-NEXT: retq1418 %a = load <16 x i32>, ptr %p1419 %b = icmp slt <16 x i32> %a, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>1420 %c = select <16 x i1> %b, <16 x i32> %a, <16 x i32> <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>1421 %d = icmp sgt <16 x i32> %c, zeroinitializer1422 %e = select <16 x i1> %d, <16 x i32> %c, <16 x i32> zeroinitializer1423 %f = trunc <16 x i32> %e to <16 x i8>1424 store <16 x i8> %f, ptr %q1425 ret void1426}1427 1428define <64 x i1> @v64i1_argument_return(<64 x i1> %x) "min-legal-vector-width"="256" {1429; CHECK-LABEL: v64i1_argument_return:1430; CHECK: # %bb.0:1431; CHECK-NEXT: retq1432 ret <64 x i1> %x1433}1434 1435define dso_local void @v64i1_shuffle(ptr %x, ptr %y) "min-legal-vector-width"="256" {1436; CHECK-LABEL: v64i1_shuffle:1437; CHECK: # %bb.0: # %entry1438; CHECK-NEXT: vmovdqa (%rdi), %ymm11439; CHECK-NEXT: vmovdqa 32(%rdi), %ymm01440; CHECK-NEXT: vptestnmb %ymm1, %ymm1, %k01441; CHECK-NEXT: kshiftrd $1, %k0, %k11442; CHECK-NEXT: kshiftlq $63, %k0, %k21443; CHECK-NEXT: kshiftrq $62, %k2, %k21444; CHECK-NEXT: kshiftlq $63, %k1, %k11445; CHECK-NEXT: kshiftrq $63, %k1, %k11446; CHECK-NEXT: korq %k2, %k1, %k11447; CHECK-NEXT: movq $-5, %rax1448; CHECK-NEXT: kmovq %rax, %k21449; CHECK-NEXT: kandq %k2, %k1, %k11450; CHECK-NEXT: kshiftrd $3, %k0, %k21451; CHECK-NEXT: kshiftlq $63, %k2, %k21452; CHECK-NEXT: kshiftrq $61, %k2, %k21453; CHECK-NEXT: korq %k2, %k1, %k11454; CHECK-NEXT: movq $-9, %rax1455; CHECK-NEXT: kmovq %rax, %k21456; CHECK-NEXT: kandq %k2, %k1, %k11457; CHECK-NEXT: kshiftrd $2, %k0, %k21458; CHECK-NEXT: kshiftlq $63, %k2, %k21459; CHECK-NEXT: kshiftrq $60, %k2, %k21460; CHECK-NEXT: korq %k2, %k1, %k11461; CHECK-NEXT: movq $-17, %rax1462; CHECK-NEXT: kmovq %rax, %k21463; CHECK-NEXT: kandq %k2, %k1, %k11464; CHECK-NEXT: kshiftrd $5, %k0, %k21465; CHECK-NEXT: kshiftlq $63, %k2, %k21466; CHECK-NEXT: kshiftrq $59, %k2, %k21467; CHECK-NEXT: korq %k2, %k1, %k11468; CHECK-NEXT: movq $-33, %rax1469; CHECK-NEXT: kmovq %rax, %k21470; CHECK-NEXT: kandq %k2, %k1, %k11471; CHECK-NEXT: kshiftrd $4, %k0, %k21472; CHECK-NEXT: kshiftlq $63, %k2, %k21473; CHECK-NEXT: kshiftrq $58, %k2, %k21474; CHECK-NEXT: korq %k2, %k1, %k11475; CHECK-NEXT: movq $-65, %rax1476; CHECK-NEXT: kmovq %rax, %k21477; CHECK-NEXT: kandq %k2, %k1, %k11478; CHECK-NEXT: kshiftrd $7, %k0, %k21479; CHECK-NEXT: kshiftlq $63, %k2, %k21480; CHECK-NEXT: kshiftrq $57, %k2, %k21481; CHECK-NEXT: korq %k2, %k1, %k11482; CHECK-NEXT: movq $-129, %rax1483; CHECK-NEXT: kmovq %rax, %k21484; CHECK-NEXT: kandq %k2, %k1, %k11485; CHECK-NEXT: kshiftrd $6, %k0, %k21486; CHECK-NEXT: kshiftlq $63, %k2, %k21487; CHECK-NEXT: kshiftrq $56, %k2, %k21488; CHECK-NEXT: korq %k2, %k1, %k11489; CHECK-NEXT: movq $-257, %rax # imm = 0xFEFF1490; CHECK-NEXT: kmovq %rax, %k21491; CHECK-NEXT: kandq %k2, %k1, %k11492; CHECK-NEXT: kshiftrd $9, %k0, %k21493; CHECK-NEXT: kshiftlq $63, %k2, %k21494; CHECK-NEXT: kshiftrq $55, %k2, %k21495; CHECK-NEXT: korq %k2, %k1, %k11496; CHECK-NEXT: movq $-513, %rax # imm = 0xFDFF1497; CHECK-NEXT: kmovq %rax, %k21498; CHECK-NEXT: kandq %k2, %k1, %k11499; CHECK-NEXT: kshiftrd $8, %k0, %k21500; CHECK-NEXT: kshiftlq $63, %k2, %k21501; CHECK-NEXT: kshiftrq $54, %k2, %k21502; CHECK-NEXT: korq %k2, %k1, %k11503; CHECK-NEXT: movq $-1025, %rax # imm = 0xFBFF1504; CHECK-NEXT: kmovq %rax, %k21505; CHECK-NEXT: kandq %k2, %k1, %k11506; CHECK-NEXT: kshiftrd $11, %k0, %k21507; CHECK-NEXT: kshiftlq $63, %k2, %k21508; CHECK-NEXT: kshiftrq $53, %k2, %k21509; CHECK-NEXT: korq %k2, %k1, %k11510; CHECK-NEXT: movq $-2049, %rax # imm = 0xF7FF1511; CHECK-NEXT: kmovq %rax, %k21512; CHECK-NEXT: kandq %k2, %k1, %k11513; CHECK-NEXT: kshiftrd $10, %k0, %k21514; CHECK-NEXT: kshiftlq $63, %k2, %k21515; CHECK-NEXT: kshiftrq $52, %k2, %k21516; CHECK-NEXT: korq %k2, %k1, %k11517; CHECK-NEXT: movq $-4097, %rax # imm = 0xEFFF1518; CHECK-NEXT: kmovq %rax, %k21519; CHECK-NEXT: kandq %k2, %k1, %k11520; CHECK-NEXT: kshiftrd $13, %k0, %k21521; CHECK-NEXT: kshiftlq $63, %k2, %k21522; CHECK-NEXT: kshiftrq $51, %k2, %k21523; CHECK-NEXT: korq %k2, %k1, %k11524; CHECK-NEXT: movq $-8193, %rax # imm = 0xDFFF1525; CHECK-NEXT: kmovq %rax, %k21526; CHECK-NEXT: kandq %k2, %k1, %k11527; CHECK-NEXT: kshiftrd $12, %k0, %k21528; CHECK-NEXT: kshiftlq $63, %k2, %k21529; CHECK-NEXT: kshiftrq $50, %k2, %k21530; CHECK-NEXT: korq %k2, %k1, %k11531; CHECK-NEXT: movq $-16385, %rax # imm = 0xBFFF1532; CHECK-NEXT: kmovq %rax, %k21533; CHECK-NEXT: kandq %k2, %k1, %k11534; CHECK-NEXT: kshiftrd $15, %k0, %k21535; CHECK-NEXT: kshiftlq $63, %k2, %k21536; CHECK-NEXT: kshiftrq $49, %k2, %k21537; CHECK-NEXT: korq %k2, %k1, %k11538; CHECK-NEXT: movq $-32769, %rax # imm = 0xFFFF7FFF1539; CHECK-NEXT: kmovq %rax, %k21540; CHECK-NEXT: kandq %k2, %k1, %k11541; CHECK-NEXT: kshiftrd $14, %k0, %k21542; CHECK-NEXT: kshiftlq $63, %k2, %k21543; CHECK-NEXT: kshiftrq $48, %k2, %k21544; CHECK-NEXT: korq %k2, %k1, %k11545; CHECK-NEXT: movq $-65537, %rax # imm = 0xFFFEFFFF1546; CHECK-NEXT: kmovq %rax, %k21547; CHECK-NEXT: kandq %k2, %k1, %k11548; CHECK-NEXT: kshiftrd $17, %k0, %k21549; CHECK-NEXT: kshiftlq $63, %k2, %k21550; CHECK-NEXT: kshiftrq $47, %k2, %k21551; CHECK-NEXT: korq %k2, %k1, %k11552; CHECK-NEXT: movq $-131073, %rax # imm = 0xFFFDFFFF1553; CHECK-NEXT: kmovq %rax, %k21554; CHECK-NEXT: kandq %k2, %k1, %k11555; CHECK-NEXT: kshiftrd $16, %k0, %k21556; CHECK-NEXT: kshiftlq $63, %k2, %k21557; CHECK-NEXT: kshiftrq $46, %k2, %k21558; CHECK-NEXT: korq %k2, %k1, %k11559; CHECK-NEXT: movq $-262145, %rax # imm = 0xFFFBFFFF1560; CHECK-NEXT: kmovq %rax, %k21561; CHECK-NEXT: kandq %k2, %k1, %k11562; CHECK-NEXT: kshiftrd $19, %k0, %k21563; CHECK-NEXT: kshiftlq $63, %k2, %k21564; CHECK-NEXT: kshiftrq $45, %k2, %k21565; CHECK-NEXT: korq %k2, %k1, %k11566; CHECK-NEXT: movq $-524289, %rax # imm = 0xFFF7FFFF1567; CHECK-NEXT: kmovq %rax, %k21568; CHECK-NEXT: kandq %k2, %k1, %k11569; CHECK-NEXT: kshiftrd $18, %k0, %k21570; CHECK-NEXT: kshiftlq $63, %k2, %k21571; CHECK-NEXT: kshiftrq $44, %k2, %k21572; CHECK-NEXT: korq %k2, %k1, %k11573; CHECK-NEXT: movq $-1048577, %rax # imm = 0xFFEFFFFF1574; CHECK-NEXT: kmovq %rax, %k21575; CHECK-NEXT: kandq %k2, %k1, %k11576; CHECK-NEXT: kshiftrd $21, %k0, %k21577; CHECK-NEXT: kshiftlq $63, %k2, %k21578; CHECK-NEXT: kshiftrq $43, %k2, %k21579; CHECK-NEXT: korq %k2, %k1, %k11580; CHECK-NEXT: movq $-2097153, %rax # imm = 0xFFDFFFFF1581; CHECK-NEXT: kmovq %rax, %k21582; CHECK-NEXT: kandq %k2, %k1, %k11583; CHECK-NEXT: kshiftrd $20, %k0, %k21584; CHECK-NEXT: kshiftlq $63, %k2, %k21585; CHECK-NEXT: kshiftrq $42, %k2, %k21586; CHECK-NEXT: korq %k2, %k1, %k11587; CHECK-NEXT: movq $-4194305, %rax # imm = 0xFFBFFFFF1588; CHECK-NEXT: kmovq %rax, %k21589; CHECK-NEXT: kandq %k2, %k1, %k11590; CHECK-NEXT: kshiftrd $23, %k0, %k21591; CHECK-NEXT: kshiftlq $63, %k2, %k21592; CHECK-NEXT: kshiftrq $41, %k2, %k21593; CHECK-NEXT: korq %k2, %k1, %k11594; CHECK-NEXT: movq $-8388609, %rax # imm = 0xFF7FFFFF1595; CHECK-NEXT: kmovq %rax, %k21596; CHECK-NEXT: kandq %k2, %k1, %k11597; CHECK-NEXT: kshiftrd $22, %k0, %k21598; CHECK-NEXT: kshiftlq $63, %k2, %k21599; CHECK-NEXT: kshiftrq $40, %k2, %k21600; CHECK-NEXT: korq %k2, %k1, %k11601; CHECK-NEXT: movq $-16777217, %rax # imm = 0xFEFFFFFF1602; CHECK-NEXT: kmovq %rax, %k21603; CHECK-NEXT: kandq %k2, %k1, %k11604; CHECK-NEXT: kshiftrd $25, %k0, %k21605; CHECK-NEXT: kshiftlq $63, %k2, %k21606; CHECK-NEXT: kshiftrq $39, %k2, %k21607; CHECK-NEXT: korq %k2, %k1, %k11608; CHECK-NEXT: movq $-33554433, %rax # imm = 0xFDFFFFFF1609; CHECK-NEXT: kmovq %rax, %k21610; CHECK-NEXT: kandq %k2, %k1, %k11611; CHECK-NEXT: kshiftrd $24, %k0, %k21612; CHECK-NEXT: kshiftlq $63, %k2, %k21613; CHECK-NEXT: kshiftrq $38, %k2, %k21614; CHECK-NEXT: korq %k2, %k1, %k11615; CHECK-NEXT: movq $-67108865, %rax # imm = 0xFBFFFFFF1616; CHECK-NEXT: kmovq %rax, %k21617; CHECK-NEXT: kandq %k2, %k1, %k11618; CHECK-NEXT: kshiftrd $27, %k0, %k21619; CHECK-NEXT: kshiftlq $63, %k2, %k21620; CHECK-NEXT: kshiftrq $37, %k2, %k21621; CHECK-NEXT: korq %k2, %k1, %k11622; CHECK-NEXT: movq $-134217729, %rax # imm = 0xF7FFFFFF1623; CHECK-NEXT: kmovq %rax, %k21624; CHECK-NEXT: kandq %k2, %k1, %k11625; CHECK-NEXT: kshiftrd $26, %k0, %k21626; CHECK-NEXT: kshiftlq $63, %k2, %k21627; CHECK-NEXT: kshiftrq $36, %k2, %k21628; CHECK-NEXT: korq %k2, %k1, %k11629; CHECK-NEXT: movq $-268435457, %rax # imm = 0xEFFFFFFF1630; CHECK-NEXT: kmovq %rax, %k21631; CHECK-NEXT: kandq %k2, %k1, %k11632; CHECK-NEXT: kshiftrd $29, %k0, %k21633; CHECK-NEXT: kshiftlq $63, %k2, %k21634; CHECK-NEXT: kshiftrq $35, %k2, %k21635; CHECK-NEXT: korq %k2, %k1, %k11636; CHECK-NEXT: movq $-536870913, %rax # imm = 0xDFFFFFFF1637; CHECK-NEXT: kmovq %rax, %k21638; CHECK-NEXT: kandq %k2, %k1, %k11639; CHECK-NEXT: kshiftrd $28, %k0, %k21640; CHECK-NEXT: kshiftlq $63, %k2, %k21641; CHECK-NEXT: kshiftrq $34, %k2, %k21642; CHECK-NEXT: korq %k2, %k1, %k11643; CHECK-NEXT: movq $-1073741825, %rax # imm = 0xBFFFFFFF1644; CHECK-NEXT: kmovq %rax, %k21645; CHECK-NEXT: kandq %k2, %k1, %k11646; CHECK-NEXT: kshiftrd $31, %k0, %k21647; CHECK-NEXT: kshiftlq $63, %k2, %k21648; CHECK-NEXT: kshiftrq $33, %k2, %k21649; CHECK-NEXT: korq %k2, %k1, %k11650; CHECK-NEXT: movabsq $-2147483649, %rax # imm = 0xFFFFFFFF7FFFFFFF1651; CHECK-NEXT: kmovq %rax, %k21652; CHECK-NEXT: kandq %k2, %k1, %k21653; CHECK-NEXT: vptestnmb %ymm0, %ymm0, %k11654; CHECK-NEXT: kshiftrd $30, %k0, %k01655; CHECK-NEXT: kshiftlq $63, %k0, %k01656; CHECK-NEXT: kshiftrq $32, %k0, %k01657; CHECK-NEXT: korq %k0, %k2, %k01658; CHECK-NEXT: movabsq $-4294967297, %rax # imm = 0xFFFFFFFEFFFFFFFF1659; CHECK-NEXT: kmovq %rax, %k21660; CHECK-NEXT: kandq %k2, %k0, %k01661; CHECK-NEXT: kshiftrd $1, %k1, %k21662; CHECK-NEXT: kshiftlq $63, %k2, %k21663; CHECK-NEXT: kshiftrq $31, %k2, %k21664; CHECK-NEXT: korq %k2, %k0, %k01665; CHECK-NEXT: movabsq $-8589934593, %rax # imm = 0xFFFFFFFDFFFFFFFF1666; CHECK-NEXT: kmovq %rax, %k21667; CHECK-NEXT: kandq %k2, %k0, %k01668; CHECK-NEXT: kshiftlq $63, %k1, %k21669; CHECK-NEXT: kshiftrq $30, %k2, %k21670; CHECK-NEXT: korq %k2, %k0, %k01671; CHECK-NEXT: movabsq $-17179869185, %rax # imm = 0xFFFFFFFBFFFFFFFF1672; CHECK-NEXT: kmovq %rax, %k21673; CHECK-NEXT: kandq %k2, %k0, %k01674; CHECK-NEXT: kshiftrd $3, %k1, %k21675; CHECK-NEXT: kshiftlq $63, %k2, %k21676; CHECK-NEXT: kshiftrq $29, %k2, %k21677; CHECK-NEXT: korq %k2, %k0, %k01678; CHECK-NEXT: movabsq $-34359738369, %rax # imm = 0xFFFFFFF7FFFFFFFF1679; CHECK-NEXT: kmovq %rax, %k21680; CHECK-NEXT: kandq %k2, %k0, %k01681; CHECK-NEXT: kshiftrd $2, %k1, %k21682; CHECK-NEXT: kshiftlq $63, %k2, %k21683; CHECK-NEXT: kshiftrq $28, %k2, %k21684; CHECK-NEXT: korq %k2, %k0, %k01685; CHECK-NEXT: movabsq $-68719476737, %rax # imm = 0xFFFFFFEFFFFFFFFF1686; CHECK-NEXT: kmovq %rax, %k21687; CHECK-NEXT: kandq %k2, %k0, %k01688; CHECK-NEXT: kshiftrd $5, %k1, %k21689; CHECK-NEXT: kshiftlq $63, %k2, %k21690; CHECK-NEXT: kshiftrq $27, %k2, %k21691; CHECK-NEXT: korq %k2, %k0, %k01692; CHECK-NEXT: movabsq $-137438953473, %rax # imm = 0xFFFFFFDFFFFFFFFF1693; CHECK-NEXT: kmovq %rax, %k21694; CHECK-NEXT: kandq %k2, %k0, %k01695; CHECK-NEXT: kshiftrd $4, %k1, %k21696; CHECK-NEXT: kshiftlq $63, %k2, %k21697; CHECK-NEXT: kshiftrq $26, %k2, %k21698; CHECK-NEXT: korq %k2, %k0, %k01699; CHECK-NEXT: movabsq $-274877906945, %rax # imm = 0xFFFFFFBFFFFFFFFF1700; CHECK-NEXT: kmovq %rax, %k21701; CHECK-NEXT: kandq %k2, %k0, %k01702; CHECK-NEXT: kshiftrd $7, %k1, %k21703; CHECK-NEXT: kshiftlq $63, %k2, %k21704; CHECK-NEXT: kshiftrq $25, %k2, %k21705; CHECK-NEXT: korq %k2, %k0, %k01706; CHECK-NEXT: movabsq $-549755813889, %rax # imm = 0xFFFFFF7FFFFFFFFF1707; CHECK-NEXT: kmovq %rax, %k21708; CHECK-NEXT: kandq %k2, %k0, %k01709; CHECK-NEXT: kshiftrd $6, %k1, %k21710; CHECK-NEXT: kshiftlq $63, %k2, %k21711; CHECK-NEXT: kshiftrq $24, %k2, %k21712; CHECK-NEXT: korq %k2, %k0, %k01713; CHECK-NEXT: movabsq $-1099511627777, %rax # imm = 0xFFFFFEFFFFFFFFFF1714; CHECK-NEXT: kmovq %rax, %k21715; CHECK-NEXT: kandq %k2, %k0, %k01716; CHECK-NEXT: kshiftrd $9, %k1, %k21717; CHECK-NEXT: kshiftlq $63, %k2, %k21718; CHECK-NEXT: kshiftrq $23, %k2, %k21719; CHECK-NEXT: korq %k2, %k0, %k01720; CHECK-NEXT: movabsq $-2199023255553, %rax # imm = 0xFFFFFDFFFFFFFFFF1721; CHECK-NEXT: kmovq %rax, %k21722; CHECK-NEXT: kandq %k2, %k0, %k01723; CHECK-NEXT: kshiftrd $8, %k1, %k21724; CHECK-NEXT: kshiftlq $63, %k2, %k21725; CHECK-NEXT: kshiftrq $22, %k2, %k21726; CHECK-NEXT: korq %k2, %k0, %k01727; CHECK-NEXT: movabsq $-4398046511105, %rax # imm = 0xFFFFFBFFFFFFFFFF1728; CHECK-NEXT: kmovq %rax, %k21729; CHECK-NEXT: kandq %k2, %k0, %k01730; CHECK-NEXT: kshiftrd $11, %k1, %k21731; CHECK-NEXT: kshiftlq $63, %k2, %k21732; CHECK-NEXT: kshiftrq $21, %k2, %k21733; CHECK-NEXT: korq %k2, %k0, %k01734; CHECK-NEXT: movabsq $-8796093022209, %rax # imm = 0xFFFFF7FFFFFFFFFF1735; CHECK-NEXT: kmovq %rax, %k21736; CHECK-NEXT: kandq %k2, %k0, %k01737; CHECK-NEXT: kshiftrd $10, %k1, %k21738; CHECK-NEXT: kshiftlq $63, %k2, %k21739; CHECK-NEXT: kshiftrq $20, %k2, %k21740; CHECK-NEXT: korq %k2, %k0, %k01741; CHECK-NEXT: movabsq $-17592186044417, %rax # imm = 0xFFFFEFFFFFFFFFFF1742; CHECK-NEXT: kmovq %rax, %k21743; CHECK-NEXT: kandq %k2, %k0, %k01744; CHECK-NEXT: kshiftrd $13, %k1, %k21745; CHECK-NEXT: kshiftlq $63, %k2, %k21746; CHECK-NEXT: kshiftrq $19, %k2, %k21747; CHECK-NEXT: korq %k2, %k0, %k01748; CHECK-NEXT: movabsq $-35184372088833, %rax # imm = 0xFFFFDFFFFFFFFFFF1749; CHECK-NEXT: kmovq %rax, %k21750; CHECK-NEXT: kandq %k2, %k0, %k01751; CHECK-NEXT: kshiftrd $12, %k1, %k21752; CHECK-NEXT: kshiftlq $63, %k2, %k21753; CHECK-NEXT: kshiftrq $18, %k2, %k21754; CHECK-NEXT: korq %k2, %k0, %k01755; CHECK-NEXT: movabsq $-70368744177665, %rax # imm = 0xFFFFBFFFFFFFFFFF1756; CHECK-NEXT: kmovq %rax, %k21757; CHECK-NEXT: kandq %k2, %k0, %k01758; CHECK-NEXT: kshiftrd $15, %k1, %k21759; CHECK-NEXT: kshiftlq $63, %k2, %k21760; CHECK-NEXT: kshiftrq $17, %k2, %k21761; CHECK-NEXT: korq %k2, %k0, %k01762; CHECK-NEXT: movabsq $-140737488355329, %rax # imm = 0xFFFF7FFFFFFFFFFF1763; CHECK-NEXT: kmovq %rax, %k21764; CHECK-NEXT: kandq %k2, %k0, %k01765; CHECK-NEXT: kshiftrd $14, %k1, %k21766; CHECK-NEXT: kshiftlq $63, %k2, %k21767; CHECK-NEXT: kshiftrq $16, %k2, %k21768; CHECK-NEXT: korq %k2, %k0, %k01769; CHECK-NEXT: movabsq $-281474976710657, %rax # imm = 0xFFFEFFFFFFFFFFFF1770; CHECK-NEXT: kmovq %rax, %k21771; CHECK-NEXT: kandq %k2, %k0, %k01772; CHECK-NEXT: kshiftrd $17, %k1, %k21773; CHECK-NEXT: kshiftlq $63, %k2, %k21774; CHECK-NEXT: kshiftrq $15, %k2, %k21775; CHECK-NEXT: korq %k2, %k0, %k01776; CHECK-NEXT: movabsq $-562949953421313, %rax # imm = 0xFFFDFFFFFFFFFFFF1777; CHECK-NEXT: kmovq %rax, %k21778; CHECK-NEXT: kandq %k2, %k0, %k01779; CHECK-NEXT: kshiftrd $16, %k1, %k21780; CHECK-NEXT: kshiftlq $63, %k2, %k21781; CHECK-NEXT: kshiftrq $14, %k2, %k21782; CHECK-NEXT: korq %k2, %k0, %k01783; CHECK-NEXT: movabsq $-1125899906842625, %rax # imm = 0xFFFBFFFFFFFFFFFF1784; CHECK-NEXT: kmovq %rax, %k21785; CHECK-NEXT: kandq %k2, %k0, %k01786; CHECK-NEXT: kshiftrd $19, %k1, %k21787; CHECK-NEXT: kshiftlq $63, %k2, %k21788; CHECK-NEXT: kshiftrq $13, %k2, %k21789; CHECK-NEXT: korq %k2, %k0, %k01790; CHECK-NEXT: movabsq $-2251799813685249, %rax # imm = 0xFFF7FFFFFFFFFFFF1791; CHECK-NEXT: kmovq %rax, %k21792; CHECK-NEXT: kandq %k2, %k0, %k01793; CHECK-NEXT: kshiftrd $18, %k1, %k21794; CHECK-NEXT: kshiftlq $63, %k2, %k21795; CHECK-NEXT: kshiftrq $12, %k2, %k21796; CHECK-NEXT: korq %k2, %k0, %k01797; CHECK-NEXT: movabsq $-4503599627370497, %rax # imm = 0xFFEFFFFFFFFFFFFF1798; CHECK-NEXT: kmovq %rax, %k21799; CHECK-NEXT: kandq %k2, %k0, %k01800; CHECK-NEXT: kshiftrd $21, %k1, %k21801; CHECK-NEXT: kshiftlq $63, %k2, %k21802; CHECK-NEXT: kshiftrq $11, %k2, %k21803; CHECK-NEXT: korq %k2, %k0, %k01804; CHECK-NEXT: movabsq $-9007199254740993, %rax # imm = 0xFFDFFFFFFFFFFFFF1805; CHECK-NEXT: kmovq %rax, %k21806; CHECK-NEXT: kandq %k2, %k0, %k01807; CHECK-NEXT: kshiftrd $20, %k1, %k21808; CHECK-NEXT: kshiftlq $63, %k2, %k21809; CHECK-NEXT: kshiftrq $10, %k2, %k21810; CHECK-NEXT: korq %k2, %k0, %k01811; CHECK-NEXT: movabsq $-18014398509481985, %rax # imm = 0xFFBFFFFFFFFFFFFF1812; CHECK-NEXT: kmovq %rax, %k21813; CHECK-NEXT: kandq %k2, %k0, %k01814; CHECK-NEXT: kshiftrd $23, %k1, %k21815; CHECK-NEXT: kshiftlq $63, %k2, %k21816; CHECK-NEXT: kshiftrq $9, %k2, %k21817; CHECK-NEXT: korq %k2, %k0, %k01818; CHECK-NEXT: movabsq $-36028797018963969, %rax # imm = 0xFF7FFFFFFFFFFFFF1819; CHECK-NEXT: kmovq %rax, %k21820; CHECK-NEXT: kandq %k2, %k0, %k01821; CHECK-NEXT: kshiftrd $22, %k1, %k21822; CHECK-NEXT: kshiftlq $63, %k2, %k21823; CHECK-NEXT: kshiftrq $8, %k2, %k21824; CHECK-NEXT: korq %k2, %k0, %k01825; CHECK-NEXT: movabsq $-72057594037927937, %rax # imm = 0xFEFFFFFFFFFFFFFF1826; CHECK-NEXT: kmovq %rax, %k21827; CHECK-NEXT: kandq %k2, %k0, %k01828; CHECK-NEXT: kshiftrd $25, %k1, %k21829; CHECK-NEXT: kshiftlq $63, %k2, %k21830; CHECK-NEXT: kshiftrq $7, %k2, %k21831; CHECK-NEXT: korq %k2, %k0, %k01832; CHECK-NEXT: movabsq $-144115188075855873, %rax # imm = 0xFDFFFFFFFFFFFFFF1833; CHECK-NEXT: kmovq %rax, %k21834; CHECK-NEXT: kandq %k2, %k0, %k01835; CHECK-NEXT: kshiftrd $24, %k1, %k21836; CHECK-NEXT: kshiftlq $63, %k2, %k21837; CHECK-NEXT: kshiftrq $6, %k2, %k21838; CHECK-NEXT: korq %k2, %k0, %k01839; CHECK-NEXT: movabsq $-288230376151711745, %rax # imm = 0xFBFFFFFFFFFFFFFF1840; CHECK-NEXT: kmovq %rax, %k21841; CHECK-NEXT: kandq %k2, %k0, %k01842; CHECK-NEXT: kshiftrd $27, %k1, %k21843; CHECK-NEXT: kshiftlq $63, %k2, %k21844; CHECK-NEXT: kshiftrq $5, %k2, %k21845; CHECK-NEXT: korq %k2, %k0, %k01846; CHECK-NEXT: movabsq $-576460752303423489, %rax # imm = 0xF7FFFFFFFFFFFFFF1847; CHECK-NEXT: kmovq %rax, %k21848; CHECK-NEXT: kandq %k2, %k0, %k01849; CHECK-NEXT: kshiftrd $26, %k1, %k21850; CHECK-NEXT: kshiftlq $63, %k2, %k21851; CHECK-NEXT: kshiftrq $4, %k2, %k21852; CHECK-NEXT: korq %k2, %k0, %k01853; CHECK-NEXT: movabsq $-1152921504606846977, %rax # imm = 0xEFFFFFFFFFFFFFFF1854; CHECK-NEXT: kmovq %rax, %k21855; CHECK-NEXT: kandq %k2, %k0, %k01856; CHECK-NEXT: kshiftrd $29, %k1, %k21857; CHECK-NEXT: kshiftlq $63, %k2, %k21858; CHECK-NEXT: kshiftrq $3, %k2, %k21859; CHECK-NEXT: korq %k2, %k0, %k01860; CHECK-NEXT: movabsq $-2305843009213693953, %rax # imm = 0xDFFFFFFFFFFFFFFF1861; CHECK-NEXT: kmovq %rax, %k21862; CHECK-NEXT: kandq %k2, %k0, %k01863; CHECK-NEXT: kshiftrd $28, %k1, %k21864; CHECK-NEXT: kshiftlq $63, %k2, %k21865; CHECK-NEXT: kshiftrq $2, %k2, %k21866; CHECK-NEXT: korq %k2, %k0, %k01867; CHECK-NEXT: movabsq $-4611686018427387905, %rax # imm = 0xBFFFFFFFFFFFFFFF1868; CHECK-NEXT: kmovq %rax, %k21869; CHECK-NEXT: kandq %k2, %k0, %k01870; CHECK-NEXT: kshiftrd $31, %k1, %k21871; CHECK-NEXT: kshiftlq $62, %k2, %k21872; CHECK-NEXT: korq %k2, %k0, %k01873; CHECK-NEXT: kshiftrd $30, %k1, %k11874; CHECK-NEXT: kshiftlq $1, %k0, %k01875; CHECK-NEXT: kshiftrq $1, %k0, %k01876; CHECK-NEXT: kshiftlq $63, %k1, %k11877; CHECK-NEXT: korq %k1, %k0, %k11878; CHECK-NEXT: vmovdqu8 %ymm1, (%rsi) {%k1}1879; CHECK-NEXT: kshiftrq $32, %k1, %k11880; CHECK-NEXT: vmovdqu8 %ymm0, 32(%rsi) {%k1}1881; CHECK-NEXT: vzeroupper1882; CHECK-NEXT: retq1883entry:1884 %a = load <64 x i8>, ptr %x1885 %b = icmp eq <64 x i8> %a, zeroinitializer1886 %shuf = shufflevector <64 x i1> %b, <64 x i1> undef, <64 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14, i32 17, i32 16, i32 19, i32 18, i32 21, i32 20, i32 23, i32 22, i32 25, i32 24, i32 27, i32 26, i32 29, i32 28, i32 31, i32 30, i32 33, i32 32, i32 35, i32 34, i32 37, i32 36, i32 39, i32 38, i32 41, i32 40, i32 43, i32 42, i32 45, i32 44, i32 47, i32 46, i32 49, i32 48, i32 51, i32 50, i32 53, i32 52, i32 55, i32 54, i32 57, i32 56, i32 59, i32 58, i32 61, i32 60, i32 63, i32 62>1887 call void @llvm.masked.store.v64i8.p0(<64 x i8> %a, ptr %y, i32 1, <64 x i1> %shuf)1888 ret void1889}1890declare void @llvm.masked.store.v64i8.p0(<64 x i8>, ptr, i32, <64 x i1>)1891 1892@mem64_dst = dso_local global i64 0, align 81893@mem64_src = dso_local global i64 0, align 81894define dso_local i32 @v64i1_inline_asm() "min-legal-vector-width"="256" {1895; CHECK-LABEL: v64i1_inline_asm:1896; CHECK: # %bb.0:1897; CHECK-NEXT: kmovq mem64_src(%rip), %k01898; CHECK-NEXT: #APP1899; CHECK-NEXT: #NO_APP1900; CHECK-NEXT: kmovq %k0, mem64_dst(%rip)1901; CHECK-NEXT: movl -{{[0-9]+}}(%rsp), %eax1902; CHECK-NEXT: retq1903 %1 = alloca i32, align 41904 %2 = load i64, ptr @mem64_src, align 81905 %3 = call i64 asm "", "=k,k,~{dirflag},~{fpsr},~{flags}"(i64 %2)1906 store i64 %3, ptr @mem64_dst, align 81907 %4 = load i32, ptr %1, align 41908 ret i32 %41909}1910 1911define dso_local void @cmp_v8i64_sext(ptr %xptr, ptr %yptr, ptr %zptr) "min-legal-vector-width"="256" {1912; CHECK-LABEL: cmp_v8i64_sext:1913; CHECK: # %bb.0:1914; CHECK-NEXT: vmovdqa (%rsi), %ymm01915; CHECK-NEXT: vmovdqa 32(%rsi), %ymm11916; CHECK-NEXT: vpcmpgtq 32(%rdi), %ymm1, %ymm11917; CHECK-NEXT: vpcmpgtq (%rdi), %ymm0, %ymm01918; CHECK-NEXT: vmovdqa %ymm0, (%rdx)1919; CHECK-NEXT: vmovdqa %ymm1, 32(%rdx)1920; CHECK-NEXT: vzeroupper1921; CHECK-NEXT: retq1922 %x = load <8 x i64>, ptr %xptr1923 %y = load <8 x i64>, ptr %yptr1924 %cmp = icmp slt <8 x i64> %x, %y1925 %ext = sext <8 x i1> %cmp to <8 x i64>1926 store <8 x i64> %ext, ptr %zptr1927 ret void1928}1929 1930define dso_local void @cmp_v8i64_zext(ptr %xptr, ptr %yptr, ptr %zptr) "min-legal-vector-width"="256" {1931; CHECK-LABEL: cmp_v8i64_zext:1932; CHECK: # %bb.0:1933; CHECK-NEXT: vmovdqa (%rsi), %ymm01934; CHECK-NEXT: vmovdqa 32(%rsi), %ymm11935; CHECK-NEXT: vpcmpgtq 32(%rdi), %ymm1, %ymm11936; CHECK-NEXT: vpcmpgtq (%rdi), %ymm0, %ymm01937; CHECK-NEXT: vpsrlq $63, %ymm1, %ymm11938; CHECK-NEXT: vpsrlq $63, %ymm0, %ymm01939; CHECK-NEXT: vmovdqa %ymm0, (%rdx)1940; CHECK-NEXT: vmovdqa %ymm1, 32(%rdx)1941; CHECK-NEXT: vzeroupper1942; CHECK-NEXT: retq1943 %x = load <8 x i64>, ptr %xptr1944 %y = load <8 x i64>, ptr %yptr1945 %cmp = icmp slt <8 x i64> %x, %y1946 %ext = zext <8 x i1> %cmp to <8 x i64>1947 store <8 x i64> %ext, ptr %zptr1948 ret void1949}1950 1951define <16 x i8> @var_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind "min-legal-vector-width"="256" {1952; CHECK-LABEL: var_rotate_v16i8:1953; CHECK: # %bb.0:1954; CHECK-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm11955; CHECK-NEXT: vpxor %xmm2, %xmm2, %xmm21956; CHECK-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]1957; CHECK-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]1958; CHECK-NEXT: vpsllvw %xmm2, %xmm3, %xmm21959; CHECK-NEXT: vpsrlw $8, %xmm2, %xmm21960; CHECK-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]1961; CHECK-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero1962; CHECK-NEXT: vpsllvw %xmm1, %xmm0, %xmm01963; CHECK-NEXT: vpsrlw $8, %xmm0, %xmm01964; CHECK-NEXT: vpackuswb %xmm2, %xmm0, %xmm01965; CHECK-NEXT: retq1966 %b8 = sub <16 x i8> <i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8>, %b1967 %shl = shl <16 x i8> %a, %b1968 %lshr = lshr <16 x i8> %a, %b81969 %or = or <16 x i8> %shl, %lshr1970 ret <16 x i8> %or1971}1972 1973define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind "min-legal-vector-width"="256" {1974; CHECK-LABEL: var_rotate_v32i8:1975; CHECK: # %bb.0:1976; CHECK-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm11977; CHECK-NEXT: vpxor %xmm2, %xmm2, %xmm21978; CHECK-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]1979; CHECK-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]1980; CHECK-NEXT: vpsllvw %ymm3, %ymm4, %ymm31981; CHECK-NEXT: vpsrlw $8, %ymm3, %ymm31982; CHECK-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]1983; CHECK-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]1984; CHECK-NEXT: vpsllvw %ymm1, %ymm0, %ymm01985; CHECK-NEXT: vpsrlw $8, %ymm0, %ymm01986; CHECK-NEXT: vpackuswb %ymm3, %ymm0, %ymm01987; CHECK-NEXT: retq1988 %b8 = sub <32 x i8> <i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8>, %b1989 %shl = shl <32 x i8> %a, %b1990 %lshr = lshr <32 x i8> %a, %b81991 %or = or <32 x i8> %shl, %lshr1992 ret <32 x i8> %or1993}1994 1995define <32 x i8> @splatvar_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind "min-legal-vector-width"="256" {1996; CHECK-LABEL: splatvar_rotate_v32i8:1997; CHECK: # %bb.0:1998; CHECK-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]1999; CHECK-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm12000; CHECK-NEXT: vpsllw %xmm1, %ymm2, %ymm22001; CHECK-NEXT: vpsrlw $8, %ymm2, %ymm22002; CHECK-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]2003; CHECK-NEXT: vpsllw %xmm1, %ymm0, %ymm02004; CHECK-NEXT: vpsrlw $8, %ymm0, %ymm02005; CHECK-NEXT: vpackuswb %ymm2, %ymm0, %ymm02006; CHECK-NEXT: retq2007 %splat = shufflevector <32 x i8> %b, <32 x i8> undef, <32 x i32> zeroinitializer2008 %splat8 = sub <32 x i8> <i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8>, %splat2009 %shl = shl <32 x i8> %a, %splat2010 %lshr = lshr <32 x i8> %a, %splat82011 %or = or <32 x i8> %shl, %lshr2012 ret <32 x i8> %or2013}2014 2015define <32 x i8> @constant_rotate_v32i8(<32 x i8> %a) nounwind "min-legal-vector-width"="256" {2016; CHECK-LABEL: constant_rotate_v32i8:2017; CHECK: # %bb.0:2018; CHECK-NEXT: vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]2019; CHECK-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm12020; CHECK-NEXT: vpsrlw $8, %ymm1, %ymm12021; CHECK-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]2022; CHECK-NEXT: vpsllvw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm02023; CHECK-NEXT: vpsrlw $8, %ymm0, %ymm02024; CHECK-NEXT: vpackuswb %ymm1, %ymm0, %ymm02025; CHECK-NEXT: retq2026 %shl = shl <32 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1>2027 %lshr = lshr <32 x i8> %a, <i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>2028 %or = or <32 x i8> %shl, %lshr2029 ret <32 x i8> %or2030}2031 2032define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind "min-legal-vector-width"="256" {2033; CHECK-SKX-LABEL: splatconstant_rotate_v32i8:2034; CHECK-SKX: # %bb.0:2035; CHECK-SKX-NEXT: vpsllw $4, %ymm0, %ymm12036; CHECK-SKX-NEXT: vpsrlw $4, %ymm0, %ymm02037; CHECK-SKX-NEXT: vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))2038; CHECK-SKX-NEXT: retq2039;2040; CHECK-AVX512-LABEL: splatconstant_rotate_v32i8:2041; CHECK-AVX512: # %bb.0:2042; CHECK-AVX512-NEXT: vpsllw $4, %ymm0, %ymm12043; CHECK-AVX512-NEXT: vpsrlw $4, %ymm0, %ymm02044; CHECK-AVX512-NEXT: vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))2045; CHECK-AVX512-NEXT: retq2046;2047; CHECK-VBMI1-LABEL: splatconstant_rotate_v32i8:2048; CHECK-VBMI1: # %bb.0:2049; CHECK-VBMI1-NEXT: vpsllw $4, %ymm0, %ymm12050; CHECK-VBMI1-NEXT: vpsrlw $4, %ymm0, %ymm02051; CHECK-VBMI1-NEXT: vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))2052; CHECK-VBMI1-NEXT: retq2053;2054; CHECK-GFNI-LABEL: splatconstant_rotate_v32i8:2055; CHECK-GFNI: # %bb.0:2056; CHECK-GFNI-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm02057; CHECK-GFNI-NEXT: retq2058 %shl = shl <32 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>2059 %lshr = lshr <32 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>2060 %or = or <32 x i8> %shl, %lshr2061 ret <32 x i8> %or2062}2063 2064define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind "min-legal-vector-width"="256" {2065; CHECK-SKX-LABEL: splatconstant_rotate_mask_v32i8:2066; CHECK-SKX: # %bb.0:2067; CHECK-SKX-NEXT: vpsllw $4, %ymm0, %ymm12068; CHECK-SKX-NEXT: vpsrlw $4, %ymm0, %ymm02069; CHECK-SKX-NEXT: vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))2070; CHECK-SKX-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm02071; CHECK-SKX-NEXT: retq2072;2073; CHECK-AVX512-LABEL: splatconstant_rotate_mask_v32i8:2074; CHECK-AVX512: # %bb.0:2075; CHECK-AVX512-NEXT: vpsllw $4, %ymm0, %ymm12076; CHECK-AVX512-NEXT: vpsrlw $4, %ymm0, %ymm02077; CHECK-AVX512-NEXT: vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))2078; CHECK-AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm02079; CHECK-AVX512-NEXT: retq2080;2081; CHECK-VBMI1-LABEL: splatconstant_rotate_mask_v32i8:2082; CHECK-VBMI1: # %bb.0:2083; CHECK-VBMI1-NEXT: vpsllw $4, %ymm0, %ymm12084; CHECK-VBMI1-NEXT: vpsrlw $4, %ymm0, %ymm02085; CHECK-VBMI1-NEXT: vpternlogd {{.*#+}} ymm0 = ymm0 ^ (m32bcst & (ymm0 ^ ymm1))2086; CHECK-VBMI1-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm02087; CHECK-VBMI1-NEXT: retq2088;2089; CHECK-GFNI-LABEL: splatconstant_rotate_mask_v32i8:2090; CHECK-GFNI: # %bb.0:2091; CHECK-GFNI-NEXT: vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm02092; CHECK-GFNI-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm02093; CHECK-GFNI-NEXT: retq2094 %shl = shl <32 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>2095 %lshr = lshr <32 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>2096 %rmask = and <32 x i8> %lshr, <i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55>2097 %lmask = and <32 x i8> %shl, <i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33, i8 33>2098 %or = or <32 x i8> %lmask, %rmask2099 ret <32 x i8> %or2100}2101