brintos / llvm-project-archived public Read only

1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -O3 -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512dq,+avx512vbmi,+avx512cd,+avx512vpopcntdq,+avx512vnni < %s | FileCheck %s3 4target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"5target triple = "x86_64-unknown-unknown"6 7; Stack reload folding tests.8;9; By including a nop call with sideeffects we can force a partial register spill of the10; relevant registers and check that the reload is correctly folded into the instruction.11 12define <16 x i32> @stack_fold_valignd(<16 x i32> %a, <16 x i32> %b) {13; CHECK-LABEL: stack_fold_valignd:14; CHECK:       # %bb.0:15; CHECK-NEXT:    pushq %rax16; CHECK-NEXT:    .cfi_def_cfa_offset 1617; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill18; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill19; CHECK-NEXT:    #APP20; CHECK-NEXT:    nop21; CHECK-NEXT:    #NO_APP22; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload23; CHECK-NEXT:    valignd $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload24; CHECK-NEXT:    # zmm0 = mem[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zmm0[0]25; CHECK-NEXT:    popq %rax26; CHECK-NEXT:    .cfi_def_cfa_offset 827; CHECK-NEXT:    retq28  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()29  %2 = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32><i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>30  ret <16 x i32> %231}32 33define <16 x i32> @stack_fold_valignd_mask(<16 x i32> %a, <16 x i32> %b, ptr %passthru, i16 %mask) {34; CHECK-LABEL: stack_fold_valignd_mask:35; CHECK:       # %bb.0:36; CHECK-NEXT:    pushq %rax37; CHECK-NEXT:    .cfi_def_cfa_offset 1638; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill39; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill40; CHECK-NEXT:    #APP41; CHECK-NEXT:    nop42; CHECK-NEXT:    #NO_APP43; CHECK-NEXT:    kmovd %esi, %k144; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm145; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload46; CHECK-NEXT:    valignd $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 {%k1} # 64-byte Folded Reload47; CHECK-NEXT:    # zmm1 {%k1} = mem[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zmm0[0]48; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm049; CHECK-NEXT:    popq %rax50; CHECK-NEXT:    .cfi_def_cfa_offset 851; CHECK-NEXT:    retq52  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()53  %2 = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>54  %3 = bitcast i16 %mask to <16 x i1>55  %4 = load <16 x i32>, ptr %passthru56  %5 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> %457  ret <16 x i32> %558}59 60define <16 x i32> @stack_fold_valignd_maskz(<16 x i32> %a, <16 x i32> %b, i16 %mask) {61; CHECK-LABEL: stack_fold_valignd_maskz:62; CHECK:       # %bb.0:63; CHECK-NEXT:    pushq %rax64; CHECK-NEXT:    .cfi_def_cfa_offset 1665; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill66; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill67; CHECK-NEXT:    #APP68; CHECK-NEXT:    nop69; CHECK-NEXT:    #NO_APP70; CHECK-NEXT:    kmovd %edi, %k171; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload72; CHECK-NEXT:    valignd $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload73; CHECK-NEXT:    # zmm0 {%k1} {z} = mem[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zmm0[0]74; CHECK-NEXT:    popq %rax75; CHECK-NEXT:    .cfi_def_cfa_offset 876; CHECK-NEXT:    retq77  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()78  %2 = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>79  %3 = bitcast i16 %mask to <16 x i1>80  %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer81  ret <16 x i32> %482}83 84define <8 x i64> @stack_fold_valignq(<8 x i64> %a, <8 x i64> %b) {85; CHECK-LABEL: stack_fold_valignq:86; CHECK:       # %bb.0:87; CHECK-NEXT:    pushq %rax88; CHECK-NEXT:    .cfi_def_cfa_offset 1689; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill90; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill91; CHECK-NEXT:    #APP92; CHECK-NEXT:    nop93; CHECK-NEXT:    #NO_APP94; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload95; CHECK-NEXT:    valignq $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload96; CHECK-NEXT:    # zmm0 = mem[1,2,3,4,5,6,7],zmm0[0]97; CHECK-NEXT:    popq %rax98; CHECK-NEXT:    .cfi_def_cfa_offset 899; CHECK-NEXT:    retq100  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()101  %2 = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>102  ret <8 x i64> %2103}104 105define <8 x i64> @stack_fold_valignq_mask(<8 x i64> %a, <8 x i64> %b, ptr %passthru, i8 %mask) {106; CHECK-LABEL: stack_fold_valignq_mask:107; CHECK:       # %bb.0:108; CHECK-NEXT:    pushq %rax109; CHECK-NEXT:    .cfi_def_cfa_offset 16110; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill111; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill112; CHECK-NEXT:    #APP113; CHECK-NEXT:    nop114; CHECK-NEXT:    #NO_APP115; CHECK-NEXT:    kmovd %esi, %k1116; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm1117; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload118; CHECK-NEXT:    valignq $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 {%k1} # 64-byte Folded Reload119; CHECK-NEXT:    # zmm1 {%k1} = mem[1,2,3,4,5,6,7],zmm0[0]120; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0121; CHECK-NEXT:    popq %rax122; CHECK-NEXT:    .cfi_def_cfa_offset 8123; CHECK-NEXT:    retq124  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()125  %2 = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>126  %3 = bitcast i8 %mask to <8 x i1>127  %4 = load <8 x i64>, ptr %passthru128  %5 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> %4129  ret <8 x i64> %5130}131 132define <8 x i64> @stack_fold_valignq_maskz(<8 x i64> %a, <8 x i64> %b, i8 %mask) {133; CHECK-LABEL: stack_fold_valignq_maskz:134; CHECK:       # %bb.0:135; CHECK-NEXT:    pushq %rax136; CHECK-NEXT:    .cfi_def_cfa_offset 16137; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill138; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill139; CHECK-NEXT:    #APP140; CHECK-NEXT:    nop141; CHECK-NEXT:    #NO_APP142; CHECK-NEXT:    kmovd %edi, %k1143; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload144; CHECK-NEXT:    valignq $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload145; CHECK-NEXT:    # zmm0 {%k1} {z} = mem[1,2,3,4,5,6,7],zmm0[0]146; CHECK-NEXT:    popq %rax147; CHECK-NEXT:    .cfi_def_cfa_offset 8148; CHECK-NEXT:    retq149  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()150  %2 = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>151  %3 = bitcast i8 %mask to <8 x i1>152  %4 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> zeroinitializer153  ret <8 x i64> %4154}155 156define <64 x i8> @stack_fold_pavgb(<64 x i8> %a0, <64 x i8> %a1) {157; CHECK-LABEL: stack_fold_pavgb:158; CHECK:       # %bb.0:159; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill160; CHECK-NEXT:    #APP161; CHECK-NEXT:    nop162; CHECK-NEXT:    #NO_APP163; CHECK-NEXT:    vpavgb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload164; CHECK-NEXT:    retq165  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()166  %2 = call <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8> %a0, <64 x i8> %a1)167  ret <64 x i8> %2168}169declare <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8>, <64 x i8>)170 171define <64 x i8> @stack_fold_pavgb_commuted(<64 x i8> %a0, <64 x i8> %a1) {172; CHECK-LABEL: stack_fold_pavgb_commuted:173; CHECK:       # %bb.0:174; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill175; CHECK-NEXT:    #APP176; CHECK-NEXT:    nop177; CHECK-NEXT:    #NO_APP178; CHECK-NEXT:    vpavgb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload179; CHECK-NEXT:    retq180  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()181  %2 = call <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8> %a1, <64 x i8> %a0)182  ret <64 x i8> %2183}184 185define <64 x i8> @stack_fold_pavgb_mask(<64 x i8> %a0, <64 x i8> %a1, ptr %a2, i64 %mask) {186; CHECK-LABEL: stack_fold_pavgb_mask:187; CHECK:       # %bb.0:188; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill189; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1190; CHECK-NEXT:    #APP191; CHECK-NEXT:    nop192; CHECK-NEXT:    #NO_APP193; CHECK-NEXT:    kmovq %rsi, %k1194; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0195; CHECK-NEXT:    vpavgb {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload196; CHECK-NEXT:    retq197  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()198  %2 = call <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8> %a0, <64 x i8> %a1)199  %3 = bitcast i64 %mask to <64 x i1>200  ; load needed to keep the operation from being scheduled about the asm block201  %4 = load <64 x i8>, ptr %a2202  %5 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> %4203  ret <64 x i8> %5204}205 206define <64 x i8> @stack_fold_pavgb_mask_commuted(<64 x i8> %a0, <64 x i8> %a1, ptr %a2, i64 %mask) {207; CHECK-LABEL: stack_fold_pavgb_mask_commuted:208; CHECK:       # %bb.0:209; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill210; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1211; CHECK-NEXT:    #APP212; CHECK-NEXT:    nop213; CHECK-NEXT:    #NO_APP214; CHECK-NEXT:    kmovq %rsi, %k1215; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0216; CHECK-NEXT:    vpavgb {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload217; CHECK-NEXT:    retq218  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()219  %2 = call <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8> %a1, <64 x i8> %a0)220  %3 = bitcast i64 %mask to <64 x i1>221  ; load needed to keep the operation from being scheduled about the asm block222  %4 = load <64 x i8>, ptr %a2223  %5 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> %4224  ret <64 x i8> %5225}226 227define <64 x i8> @stack_fold_pavgb_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {228; CHECK-LABEL: stack_fold_pavgb_maskz:229; CHECK:       # %bb.0:230; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill231; CHECK-NEXT:    #APP232; CHECK-NEXT:    nop233; CHECK-NEXT:    #NO_APP234; CHECK-NEXT:    kmovq %rdi, %k1235; CHECK-NEXT:    vpavgb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload236; CHECK-NEXT:    retq237  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()238  %2 = call <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8> %a0, <64 x i8> %a1)239  %3 = bitcast i64 %mask to <64 x i1>240  %4 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> zeroinitializer241  ret <64 x i8> %4242}243 244define <64 x i8> @stack_fold_pavgb_maskz_commuted(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {245; CHECK-LABEL: stack_fold_pavgb_maskz_commuted:246; CHECK:       # %bb.0:247; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill248; CHECK-NEXT:    #APP249; CHECK-NEXT:    nop250; CHECK-NEXT:    #NO_APP251; CHECK-NEXT:    kmovq %rdi, %k1252; CHECK-NEXT:    vpavgb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload253; CHECK-NEXT:    retq254  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()255  %2 = call <64 x i8> @llvm.x86.avx512.pavg.b.512(<64 x i8> %a1, <64 x i8> %a0)256  %3 = bitcast i64 %mask to <64 x i1>257  %4 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> zeroinitializer258  ret <64 x i8> %4259}260 261define <32 x i16> @stack_fold_pavgw(<32 x i16> %a0, <32 x i16> %a1) {262; CHECK-LABEL: stack_fold_pavgw:263; CHECK:       # %bb.0:264; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill265; CHECK-NEXT:    #APP266; CHECK-NEXT:    nop267; CHECK-NEXT:    #NO_APP268; CHECK-NEXT:    vpavgw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload269; CHECK-NEXT:    retq270  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()271  %2 = call <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16> %a0, <32 x i16> %a1)272  ret <32 x i16> %2273}274declare <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16>, <32 x i16>)275 276define <32 x i16> @stack_fold_pavgw_commuted(<32 x i16> %a0, <32 x i16> %a1) {277; CHECK-LABEL: stack_fold_pavgw_commuted:278; CHECK:       # %bb.0:279; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill280; CHECK-NEXT:    #APP281; CHECK-NEXT:    nop282; CHECK-NEXT:    #NO_APP283; CHECK-NEXT:    vpavgw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload284; CHECK-NEXT:    retq285  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()286  %2 = call <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16> %a1, <32 x i16> %a0)287  ret <32 x i16> %2288}289 290define <32 x i16> @stack_fold_pavgw_mask(<32 x i16> %a0, <32 x i16> %a1, ptr %a2, i32 %mask) {291; CHECK-LABEL: stack_fold_pavgw_mask:292; CHECK:       # %bb.0:293; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill294; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1295; CHECK-NEXT:    #APP296; CHECK-NEXT:    nop297; CHECK-NEXT:    #NO_APP298; CHECK-NEXT:    kmovd %esi, %k1299; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0300; CHECK-NEXT:    vpavgw {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload301; CHECK-NEXT:    retq302  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()303  %2 = call <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16> %a0, <32 x i16> %a1)304  %3 = bitcast i32 %mask to <32 x i1>305  ; load needed to keep the operation from being scheduled about the asm block306  %4 = load <32 x i16>, ptr %a2307  %5 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> %4308  ret <32 x i16> %5309}310 311define <32 x i16> @stack_fold_pavgw_mask_commuted(<32 x i16> %a0, <32 x i16> %a1, ptr %a2, i32 %mask) {312; CHECK-LABEL: stack_fold_pavgw_mask_commuted:313; CHECK:       # %bb.0:314; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill315; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1316; CHECK-NEXT:    #APP317; CHECK-NEXT:    nop318; CHECK-NEXT:    #NO_APP319; CHECK-NEXT:    kmovd %esi, %k1320; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0321; CHECK-NEXT:    vpavgw {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload322; CHECK-NEXT:    retq323  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()324  %2 = call <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16> %a1, <32 x i16> %a0)325  %3 = bitcast i32 %mask to <32 x i1>326  ; load needed to keep the operation from being scheduled about the asm block327  %4 = load <32 x i16>, ptr %a2328  %5 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> %4329  ret <32 x i16> %5330}331 332define <32 x i16> @stack_fold_pavgw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {333; CHECK-LABEL: stack_fold_pavgw_maskz:334; CHECK:       # %bb.0:335; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill336; CHECK-NEXT:    #APP337; CHECK-NEXT:    nop338; CHECK-NEXT:    #NO_APP339; CHECK-NEXT:    kmovd %edi, %k1340; CHECK-NEXT:    vpavgw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload341; CHECK-NEXT:    retq342  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()343  %2 = call <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16> %a0, <32 x i16> %a1)344  %3 = bitcast i32 %mask to <32 x i1>345  %4 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> zeroinitializer346  ret <32 x i16> %4347}348 349define <32 x i16> @stack_fold_pavgw_maskz_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {350; CHECK-LABEL: stack_fold_pavgw_maskz_commuted:351; CHECK:       # %bb.0:352; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill353; CHECK-NEXT:    #APP354; CHECK-NEXT:    nop355; CHECK-NEXT:    #NO_APP356; CHECK-NEXT:    kmovd %edi, %k1357; CHECK-NEXT:    vpavgw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload358; CHECK-NEXT:    retq359  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()360  %2 = call <32 x i16> @llvm.x86.avx512.pavg.w.512(<32 x i16> %a1, <32 x i16> %a0)361  %3 = bitcast i32 %mask to <32 x i1>362  %4 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> zeroinitializer363  ret <32 x i16> %4364}365 366define <4 x i32> @stack_fold_extracti32x4(<16 x i16> %a0, <16 x i32> %a1) {367; CHECK-LABEL: stack_fold_extracti32x4:368; CHECK:       # %bb.0:369; CHECK-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero370; CHECK-NEXT:    vextracti32x4 $3, %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill371; CHECK-NEXT:    #APP372; CHECK-NEXT:    nop373; CHECK-NEXT:    #NO_APP374; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload375; CHECK-NEXT:    vzeroupper376; CHECK-NEXT:    retq377  ; zext forces execution domain378  %1 = zext <16 x i16> %a0 to <16 x i32>379  %2 = shufflevector <16 x i32> %1, <16 x i32> %a1, <4 x i32> <i32 12, i32 13, i32 14, i32 15>380  %3 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()381  ret <4 x i32> %2382}383 384define <2 x i64> @stack_fold_extracti64x2(<8 x i32> %a0, <8 x i64> %a1) {385; CHECK-LABEL: stack_fold_extracti64x2:386; CHECK:       # %bb.0:387; CHECK-NEXT:    vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero388; CHECK-NEXT:    vextracti32x4 $3, %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill389; CHECK-NEXT:    #APP390; CHECK-NEXT:    nop391; CHECK-NEXT:    #NO_APP392; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload393; CHECK-NEXT:    vzeroupper394; CHECK-NEXT:    retq395  ; zext forces execution domain396  %1 = zext <8 x i32> %a0 to <8 x i64>397  %2 = shufflevector <8 x i64> %1, <8 x i64> %a1, <2 x i32> <i32 6, i32 7>398  %3 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()399  ret <2 x i64> %2400}401 402define <8 x i32> @stack_fold_extracti32x8(<16 x i16> %a0, <16 x i32> %a1) {403; CHECK-LABEL: stack_fold_extracti32x8:404; CHECK:       # %bb.0:405; CHECK-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero406; CHECK-NEXT:    vextracti64x4 $1, %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill407; CHECK-NEXT:    #APP408; CHECK-NEXT:    nop409; CHECK-NEXT:    #NO_APP410; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload411; CHECK-NEXT:    retq412  ; zext forces execution domain413  %1 = zext <16 x i16> %a0 to <16 x i32>414  %2 = shufflevector <16 x i32> %1, <16 x i32> %a1, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>415  %3 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()416  ret <8 x i32> %2417}418 419define <4 x i64> @stack_fold_extracti64x4(<8 x i32> %a0, <8 x i64> %a1) {420; CHECK-LABEL: stack_fold_extracti64x4:421; CHECK:       # %bb.0:422; CHECK-NEXT:    vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero423; CHECK-NEXT:    vextracti64x4 $1, %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill424; CHECK-NEXT:    #APP425; CHECK-NEXT:    nop426; CHECK-NEXT:    #NO_APP427; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload428; CHECK-NEXT:    retq429  ; zext forces execution domain430  %1 = zext <8 x i32> %a0 to <8 x i64>431  %2 = shufflevector <8 x i64> %1, <8 x i64> %a1, <4 x i32> <i32 4, i32 5, i32 6, i32 7>432  %3 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()433  ret <4 x i64> %2434}435 436define <16 x i32> @stack_fold_inserti32x8(<8 x i32> %a0, <8 x i32> %a1) {437; CHECK-LABEL: stack_fold_inserti32x8:438; CHECK:       # %bb.0:439; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill440; CHECK-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0441; CHECK-NEXT:    #APP442; CHECK-NEXT:    nop443; CHECK-NEXT:    #NO_APP444; CHECK-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload445; CHECK-NEXT:    vpternlogd {{.*#+}} zmm1 = -1446; CHECK-NEXT:    vpsubd %zmm1, %zmm0, %zmm0447; CHECK-NEXT:    retq448  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()449  %2 = shufflevector <8 x i32> %a0, <8 x i32> %a1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>450  ; add forces execution domain451  %3 = add <16 x i32> %2, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>452  ret <16 x i32> %3453}454 455define <8 x i64> @stack_fold_inserti64x4(<4 x i64> %a0, <4 x i64> %a1) {456; CHECK-LABEL: stack_fold_inserti64x4:457; CHECK:       # %bb.0:458; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill459; CHECK-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0460; CHECK-NEXT:    #APP461; CHECK-NEXT:    nop462; CHECK-NEXT:    #NO_APP463; CHECK-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload464; CHECK-NEXT:    vpternlogd {{.*#+}} zmm1 = -1465; CHECK-NEXT:    vpsubq %zmm1, %zmm0, %zmm0466; CHECK-NEXT:    retq467  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()468  %2 = shufflevector <4 x i64> %a0, <4 x i64> %a1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>469  ; add forces execution domain470  %3 = add <8 x i64> %2, <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>471  ret <8 x i64> %3472}473 474define <64 x i8> @stack_fold_pabsb(<64 x i8> %a0) {475; CHECK-LABEL: stack_fold_pabsb:476; CHECK:       # %bb.0:477; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill478; CHECK-NEXT:    #APP479; CHECK-NEXT:    nop480; CHECK-NEXT:    #NO_APP481; CHECK-NEXT:    vpabsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload482; CHECK-NEXT:    retq483  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()484  %2 = icmp sgt <64 x i8> %a0, zeroinitializer485  %3 = sub <64 x i8> zeroinitializer, %a0486  %4 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %3487  ret <64 x i8> %4488}489 490define <64 x i8> @stack_fold_pabsb_mask(<64 x i8> %passthru, <64 x i8> %a0, i64 %mask) {491; CHECK-LABEL: stack_fold_pabsb_mask:492; CHECK:       # %bb.0:493; CHECK-NEXT:    pushq %rax494; CHECK-NEXT:    .cfi_def_cfa_offset 16495; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill496; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill497; CHECK-NEXT:    #APP498; CHECK-NEXT:    nop499; CHECK-NEXT:    #NO_APP500; CHECK-NEXT:    kmovq %rdi, %k1501; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload502; CHECK-NEXT:    vpabsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} # 64-byte Folded Reload503; CHECK-NEXT:    popq %rax504; CHECK-NEXT:    .cfi_def_cfa_offset 8505; CHECK-NEXT:    retq506  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()507  %2 = icmp sgt <64 x i8> %a0, zeroinitializer508  %3 = sub <64 x i8> zeroinitializer, %a0509  %4 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %3510  %5 = bitcast i64 %mask to <64 x i1>511  %6 = select <64 x i1> %5, <64 x i8> %4, <64 x i8> %passthru512  ret <64 x i8> %6513}514 515define <64 x i8> @stack_fold_pabsb_maskz(<64 x i8> %a0, i64 %mask) {516; CHECK-LABEL: stack_fold_pabsb_maskz:517; CHECK:       # %bb.0:518; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill519; CHECK-NEXT:    #APP520; CHECK-NEXT:    nop521; CHECK-NEXT:    #NO_APP522; CHECK-NEXT:    kmovq %rdi, %k1523; CHECK-NEXT:    vpabsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 64-byte Folded Reload524; CHECK-NEXT:    retq525  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()526  %2 = icmp sgt <64 x i8> %a0, zeroinitializer527  %3 = sub <64 x i8> zeroinitializer, %a0528  %4 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %3529  %5 = bitcast i64 %mask to <64 x i1>530  %6 = select <64 x i1> %5, <64 x i8> %4, <64 x i8> zeroinitializer531  ret <64 x i8> %6532}533 534define <16 x i32> @stack_fold_pabsd(<16 x i32> %a0) {535; CHECK-LABEL: stack_fold_pabsd:536; CHECK:       # %bb.0:537; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill538; CHECK-NEXT:    #APP539; CHECK-NEXT:    nop540; CHECK-NEXT:    #NO_APP541; CHECK-NEXT:    vpabsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload542; CHECK-NEXT:    retq543  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()544  %2 = icmp sgt <16 x i32> %a0, zeroinitializer545  %3 = sub <16 x i32> zeroinitializer, %a0546  %4 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %3547  ret <16 x i32> %4548}549 550define <16 x i32> @stack_fold_pabsd_mask(<16 x i32> %passthru, <16 x i32> %a0, i16 %mask) {551; CHECK-LABEL: stack_fold_pabsd_mask:552; CHECK:       # %bb.0:553; CHECK-NEXT:    pushq %rax554; CHECK-NEXT:    .cfi_def_cfa_offset 16555; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill556; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill557; CHECK-NEXT:    #APP558; CHECK-NEXT:    nop559; CHECK-NEXT:    #NO_APP560; CHECK-NEXT:    kmovd %edi, %k1561; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload562; CHECK-NEXT:    vpabsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} # 64-byte Folded Reload563; CHECK-NEXT:    popq %rax564; CHECK-NEXT:    .cfi_def_cfa_offset 8565; CHECK-NEXT:    retq566  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()567  %2 = icmp sgt <16 x i32> %a0, zeroinitializer568  %3 = sub <16 x i32> zeroinitializer, %a0569  %4 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %3570  %5 = bitcast i16 %mask to <16 x i1>571  %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> %passthru572  ret <16 x i32> %6573}574 575define <16 x i32> @stack_fold_pabsd_maskz(<16 x i32> %a0, i16 %mask) {576; CHECK-LABEL: stack_fold_pabsd_maskz:577; CHECK:       # %bb.0:578; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill579; CHECK-NEXT:    #APP580; CHECK-NEXT:    nop581; CHECK-NEXT:    #NO_APP582; CHECK-NEXT:    kmovd %edi, %k1583; CHECK-NEXT:    vpabsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 64-byte Folded Reload584; CHECK-NEXT:    retq585  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()586  %2 = icmp sgt <16 x i32> %a0, zeroinitializer587  %3 = sub <16 x i32> zeroinitializer, %a0588  %4 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %3589  %5 = bitcast i16 %mask to <16 x i1>590  %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer591  ret <16 x i32> %6592}593 594define <8 x i64> @stack_fold_pabsq(<8 x i64> %a0) {595; CHECK-LABEL: stack_fold_pabsq:596; CHECK:       # %bb.0:597; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill598; CHECK-NEXT:    #APP599; CHECK-NEXT:    nop600; CHECK-NEXT:    #NO_APP601; CHECK-NEXT:    vpabsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload602; CHECK-NEXT:    retq603  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()604  %2 = icmp sgt <8 x i64> %a0, zeroinitializer605  %3 = sub <8 x i64> zeroinitializer, %a0606  %4 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %3607  ret <8 x i64> %4608}609 610define <8 x i64> @stack_fold_pabsq_mask(<8 x i64> %passthru, <8 x i64> %a0, i8 %mask) {611; CHECK-LABEL: stack_fold_pabsq_mask:612; CHECK:       # %bb.0:613; CHECK-NEXT:    pushq %rax614; CHECK-NEXT:    .cfi_def_cfa_offset 16615; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill616; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill617; CHECK-NEXT:    #APP618; CHECK-NEXT:    nop619; CHECK-NEXT:    #NO_APP620; CHECK-NEXT:    kmovd %edi, %k1621; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload622; CHECK-NEXT:    vpabsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} # 64-byte Folded Reload623; CHECK-NEXT:    popq %rax624; CHECK-NEXT:    .cfi_def_cfa_offset 8625; CHECK-NEXT:    retq626  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()627  %2 = icmp sgt <8 x i64> %a0, zeroinitializer628  %3 = sub <8 x i64> zeroinitializer, %a0629  %4 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %3630  %5 = bitcast i8 %mask to <8 x i1>631  %6 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> %passthru632  ret <8 x i64> %6633}634 635define <8 x i64> @stack_fold_pabsq_maskz(<8 x i64> %a0, i8 %mask) {636; CHECK-LABEL: stack_fold_pabsq_maskz:637; CHECK:       # %bb.0:638; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill639; CHECK-NEXT:    #APP640; CHECK-NEXT:    nop641; CHECK-NEXT:    #NO_APP642; CHECK-NEXT:    kmovd %edi, %k1643; CHECK-NEXT:    vpabsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 64-byte Folded Reload644; CHECK-NEXT:    retq645  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()646  %2 = icmp sgt <8 x i64> %a0, zeroinitializer647  %3 = sub <8 x i64> zeroinitializer, %a0648  %4 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %3649  %5 = bitcast i8 %mask to <8 x i1>650  %6 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> zeroinitializer651  ret <8 x i64> %6652}653 654define <32 x i16> @stack_fold_pabsw(<32 x i16> %a0) {655; CHECK-LABEL: stack_fold_pabsw:656; CHECK:       # %bb.0:657; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill658; CHECK-NEXT:    #APP659; CHECK-NEXT:    nop660; CHECK-NEXT:    #NO_APP661; CHECK-NEXT:    vpabsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload662; CHECK-NEXT:    retq663  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()664  %2 = icmp sgt <32 x i16> %a0, zeroinitializer665  %3 = sub <32 x i16> zeroinitializer, %a0666  %4 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %3667  ret <32 x i16> %4668}669 670define <32 x i16> @stack_fold_pabsw_mask(<32 x i16> %passthru, <32 x i16> %a0, i32 %mask) {671; CHECK-LABEL: stack_fold_pabsw_mask:672; CHECK:       # %bb.0:673; CHECK-NEXT:    pushq %rax674; CHECK-NEXT:    .cfi_def_cfa_offset 16675; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill676; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill677; CHECK-NEXT:    #APP678; CHECK-NEXT:    nop679; CHECK-NEXT:    #NO_APP680; CHECK-NEXT:    kmovd %edi, %k1681; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload682; CHECK-NEXT:    vpabsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} # 64-byte Folded Reload683; CHECK-NEXT:    popq %rax684; CHECK-NEXT:    .cfi_def_cfa_offset 8685; CHECK-NEXT:    retq686  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()687  %2 = icmp sgt <32 x i16> %a0, zeroinitializer688  %3 = sub <32 x i16> zeroinitializer, %a0689  %4 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %3690  %5 = bitcast i32 %mask to <32 x i1>691  %6 = select <32 x i1> %5, <32 x i16> %4, <32 x i16> %passthru692  ret <32 x i16> %6693}694 695define <32 x i16> @stack_fold_pabsw_maskz(<32 x i16> %a0, i32 %mask) {696; CHECK-LABEL: stack_fold_pabsw_maskz:697; CHECK:       # %bb.0:698; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill699; CHECK-NEXT:    #APP700; CHECK-NEXT:    nop701; CHECK-NEXT:    #NO_APP702; CHECK-NEXT:    kmovd %edi, %k1703; CHECK-NEXT:    vpabsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 64-byte Folded Reload704; CHECK-NEXT:    retq705  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()706  %2 = icmp sgt <32 x i16> %a0, zeroinitializer707  %3 = sub <32 x i16> zeroinitializer, %a0708  %4 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %3709  %5 = bitcast i32 %mask to <32 x i1>710  %6 = select <32 x i1> %5, <32 x i16> %4, <32 x i16> zeroinitializer711  ret <32 x i16> %6712}713 714define <32 x i16> @stack_fold_packssdw(<16 x i32> %a0, <16 x i32> %a1) {715; CHECK-LABEL: stack_fold_packssdw:716; CHECK:       # %bb.0:717; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill718; CHECK-NEXT:    #APP719; CHECK-NEXT:    nop720; CHECK-NEXT:    #NO_APP721; CHECK-NEXT:    vpackssdw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload722; CHECK-NEXT:    retq723  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()724  %2 = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> %a0, <16 x i32> %a1)725  ret <32 x i16> %2726}727declare <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32>, <16 x i32>) nounwind readnone728 729define <64 x i8> @stack_fold_packsswb(<32 x i16> %a0, <32 x i16> %a1) {730; CHECK-LABEL: stack_fold_packsswb:731; CHECK:       # %bb.0:732; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill733; CHECK-NEXT:    #APP734; CHECK-NEXT:    nop735; CHECK-NEXT:    #NO_APP736; CHECK-NEXT:    vpacksswb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload737; CHECK-NEXT:    retq738  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()739  %2 = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> %a0, <32 x i16> %a1)740  ret <64 x i8> %2741}742declare <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16>, <32 x i16>) nounwind readnone743 744define <32 x i16> @stack_fold_packusdw(<16 x i32> %a0, <16 x i32> %a1) {745; CHECK-LABEL: stack_fold_packusdw:746; CHECK:       # %bb.0:747; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill748; CHECK-NEXT:    #APP749; CHECK-NEXT:    nop750; CHECK-NEXT:    #NO_APP751; CHECK-NEXT:    vpackusdw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload752; CHECK-NEXT:    retq753  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()754  %2 = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> %a0, <16 x i32> %a1)755  ret <32 x i16> %2756}757declare <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32>, <16 x i32>) nounwind readnone758 759define <32 x i16> @stack_fold_packusdw_mask(ptr %passthru, <16 x i32> %a0, <16 x i32> %a1, i32 %mask) {760; CHECK-LABEL: stack_fold_packusdw_mask:761; CHECK:       # %bb.0:762; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill763; CHECK-NEXT:    #APP764; CHECK-NEXT:    nop765; CHECK-NEXT:    #NO_APP766; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm2767; CHECK-NEXT:    kmovd %esi, %k1768; CHECK-NEXT:    vpackusdw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload769; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0770; CHECK-NEXT:    retq771  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()772  %2 = load <32 x i16>, ptr %passthru773  %3 = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> %a0, <16 x i32> %a1)774  %4 = bitcast i32 %mask to <32 x i1>775  %5 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> %2776  ret <32 x i16> %5777}778 779define <32 x i16> @stack_fold_packusdw_maskz(<16 x i32> %a0, <16 x i32> %a1, i32 %mask) {780; CHECK-LABEL: stack_fold_packusdw_maskz:781; CHECK:       # %bb.0:782; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill783; CHECK-NEXT:    #APP784; CHECK-NEXT:    nop785; CHECK-NEXT:    #NO_APP786; CHECK-NEXT:    kmovd %edi, %k1787; CHECK-NEXT:    vpackusdw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload788; CHECK-NEXT:    retq789  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()790  %2 = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> %a0, <16 x i32> %a1)791  %3 = bitcast i32 %mask to <32 x i1>792  %4 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> zeroinitializer793  ret <32 x i16> %4794}795 796define <64 x i8> @stack_fold_packuswb(<32 x i16> %a0, <32 x i16> %a1) {797; CHECK-LABEL: stack_fold_packuswb:798; CHECK:       # %bb.0:799; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill800; CHECK-NEXT:    #APP801; CHECK-NEXT:    nop802; CHECK-NEXT:    #NO_APP803; CHECK-NEXT:    vpackuswb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload804; CHECK-NEXT:    retq805  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()806  %2 = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> %a0, <32 x i16> %a1)807  ret <64 x i8> %2808}809declare <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16>, <32 x i16>) nounwind readnone810 811define <64 x i8> @stack_fold_paddb(<64 x i8> %a0, <64 x i8> %a1) {812; CHECK-LABEL: stack_fold_paddb:813; CHECK:       # %bb.0:814; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill815; CHECK-NEXT:    #APP816; CHECK-NEXT:    nop817; CHECK-NEXT:    #NO_APP818; CHECK-NEXT:    vpaddb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload819; CHECK-NEXT:    retq820  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()821  %2 = add <64 x i8> %a0, %a1822  ret <64 x i8> %2823}824 825define <64 x i8> @stack_fold_paddb_commuted(<64 x i8> %a0, <64 x i8> %a1) {826; CHECK-LABEL: stack_fold_paddb_commuted:827; CHECK:       # %bb.0:828; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill829; CHECK-NEXT:    #APP830; CHECK-NEXT:    nop831; CHECK-NEXT:    #NO_APP832; CHECK-NEXT:    vpaddb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload833; CHECK-NEXT:    retq834  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()835  %2 = add <64 x i8> %a1, %a0836  ret <64 x i8> %2837}838 839define <64 x i8> @stack_fold_paddb_mask(<64 x i8> %a0, <64 x i8> %a1, ptr %a2, i64 %mask) {840; CHECK-LABEL: stack_fold_paddb_mask:841; CHECK:       # %bb.0:842; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill843; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1844; CHECK-NEXT:    #APP845; CHECK-NEXT:    nop846; CHECK-NEXT:    #NO_APP847; CHECK-NEXT:    kmovq %rsi, %k1848; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0849; CHECK-NEXT:    vpaddb {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload850; CHECK-NEXT:    retq851  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()852  %2 = add <64 x i8> %a0, %a1853  %3 = bitcast i64 %mask to <64 x i1>854  ; load needed to keep the operation from being scheduled about the asm block855  %4 = load <64 x i8>, ptr %a2856  %5 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> %4857  ret <64 x i8> %5858}859 860define <64 x i8> @stack_fold_paddb_mask_commuted(<64 x i8> %a0, <64 x i8> %a1, ptr %a2, i64 %mask) {861; CHECK-LABEL: stack_fold_paddb_mask_commuted:862; CHECK:       # %bb.0:863; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill864; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1865; CHECK-NEXT:    #APP866; CHECK-NEXT:    nop867; CHECK-NEXT:    #NO_APP868; CHECK-NEXT:    kmovq %rsi, %k1869; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0870; CHECK-NEXT:    vpaddb {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload871; CHECK-NEXT:    retq872  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()873  %2 = add <64 x i8> %a1, %a0874  %3 = bitcast i64 %mask to <64 x i1>875  ; load needed to keep the operation from being scheduled about the asm block876  %4 = load <64 x i8>, ptr %a2877  %5 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> %4878  ret <64 x i8> %5879}880 881define <64 x i8> @stack_fold_paddb_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {882; CHECK-LABEL: stack_fold_paddb_maskz:883; CHECK:       # %bb.0:884; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill885; CHECK-NEXT:    #APP886; CHECK-NEXT:    nop887; CHECK-NEXT:    #NO_APP888; CHECK-NEXT:    kmovq %rdi, %k1889; CHECK-NEXT:    vpaddb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload890; CHECK-NEXT:    retq891  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()892  %2 = add <64 x i8> %a0, %a1893  %3 = bitcast i64 %mask to <64 x i1>894  %4 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> zeroinitializer895  ret <64 x i8> %4896}897 898define <64 x i8> @stack_fold_paddb_maskz_commuted(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {899; CHECK-LABEL: stack_fold_paddb_maskz_commuted:900; CHECK:       # %bb.0:901; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill902; CHECK-NEXT:    #APP903; CHECK-NEXT:    nop904; CHECK-NEXT:    #NO_APP905; CHECK-NEXT:    kmovq %rdi, %k1906; CHECK-NEXT:    vpaddb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload907; CHECK-NEXT:    retq908  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()909  %2 = add <64 x i8> %a1, %a0910  %3 = bitcast i64 %mask to <64 x i1>911  %4 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> zeroinitializer912  ret <64 x i8> %4913}914 915define <16 x i32> @stack_fold_paddd(<16 x i32> %a0, <16 x i32> %a1) {916; CHECK-LABEL: stack_fold_paddd:917; CHECK:       # %bb.0:918; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill919; CHECK-NEXT:    #APP920; CHECK-NEXT:    nop921; CHECK-NEXT:    #NO_APP922; CHECK-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload923; CHECK-NEXT:    retq924  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()925  %2 = add <16 x i32> %a0, %a1926  ret <16 x i32> %2927}928 929define <16 x i32> @stack_fold_paddd_commuted(<16 x i32> %a0, <16 x i32> %a1) {930; CHECK-LABEL: stack_fold_paddd_commuted:931; CHECK:       # %bb.0:932; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill933; CHECK-NEXT:    #APP934; CHECK-NEXT:    nop935; CHECK-NEXT:    #NO_APP936; CHECK-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload937; CHECK-NEXT:    retq938  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()939  %2 = add <16 x i32> %a1, %a0940  ret <16 x i32> %2941}942 943define <16 x i32> @stack_fold_paddd_mask(<16 x i32> %a0, <16 x i32> %a1, ptr %a2, i16 %mask) {944; CHECK-LABEL: stack_fold_paddd_mask:945; CHECK:       # %bb.0:946; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill947; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1948; CHECK-NEXT:    #APP949; CHECK-NEXT:    nop950; CHECK-NEXT:    #NO_APP951; CHECK-NEXT:    kmovd %esi, %k1952; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0953; CHECK-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload954; CHECK-NEXT:    retq955  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()956  %2 = add <16 x i32> %a0, %a1957  %3 = bitcast i16 %mask to <16 x i1>958  ; load needed to keep the operation from being scheduled about the asm block959  %4 = load <16 x i32>, ptr %a2960  %5 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> %4961  ret <16 x i32> %5962}963 964define <16 x i32> @stack_fold_paddd_mask_commuted(<16 x i32> %a0, <16 x i32> %a1, ptr %a2, i16 %mask) {965; CHECK-LABEL: stack_fold_paddd_mask_commuted:966; CHECK:       # %bb.0:967; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill968; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1969; CHECK-NEXT:    #APP970; CHECK-NEXT:    nop971; CHECK-NEXT:    #NO_APP972; CHECK-NEXT:    kmovd %esi, %k1973; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm0974; CHECK-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload975; CHECK-NEXT:    retq976  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()977  %2 = add <16 x i32> %a1, %a0978  %3 = bitcast i16 %mask to <16 x i1>979  ; load needed to keep the operation from being scheduled about the asm block980  %4 = load <16 x i32>, ptr %a2981  %5 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> %4982  ret <16 x i32> %5983}984 985define <16 x i32> @stack_fold_paddd_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {986; CHECK-LABEL: stack_fold_paddd_maskz:987; CHECK:       # %bb.0:988; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill989; CHECK-NEXT:    #APP990; CHECK-NEXT:    nop991; CHECK-NEXT:    #NO_APP992; CHECK-NEXT:    kmovd %edi, %k1993; CHECK-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload994; CHECK-NEXT:    retq995  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()996  %2 = add <16 x i32> %a0, %a1997  %3 = bitcast i16 %mask to <16 x i1>998  %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer999  ret <16 x i32> %41000}1001 1002define <16 x i32> @stack_fold_paddd_maskz_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {1003; CHECK-LABEL: stack_fold_paddd_maskz_commuted:1004; CHECK:       # %bb.0:1005; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1006; CHECK-NEXT:    #APP1007; CHECK-NEXT:    nop1008; CHECK-NEXT:    #NO_APP1009; CHECK-NEXT:    kmovd %edi, %k11010; CHECK-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload1011; CHECK-NEXT:    retq1012  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1013  %2 = add <16 x i32> %a1, %a01014  %3 = bitcast i16 %mask to <16 x i1>1015  %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer1016  ret <16 x i32> %41017}1018 1019define <8 x i64> @stack_fold_paddq(<8 x i64> %a0, <8 x i64> %a1) {1020; CHECK-LABEL: stack_fold_paddq:1021; CHECK:       # %bb.0:1022; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1023; CHECK-NEXT:    #APP1024; CHECK-NEXT:    nop1025; CHECK-NEXT:    #NO_APP1026; CHECK-NEXT:    vpaddq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload1027; CHECK-NEXT:    retq1028  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1029  %2 = add <8 x i64> %a0, %a11030  ret <8 x i64> %21031}1032 1033define <8 x i64> @stack_fold_paddq_commuted(<8 x i64> %a0, <8 x i64> %a1) {1034; CHECK-LABEL: stack_fold_paddq_commuted:1035; CHECK:       # %bb.0:1036; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1037; CHECK-NEXT:    #APP1038; CHECK-NEXT:    nop1039; CHECK-NEXT:    #NO_APP1040; CHECK-NEXT:    vpaddq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload1041; CHECK-NEXT:    retq1042  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1043  %2 = add <8 x i64> %a1, %a01044  ret <8 x i64> %21045}1046 1047define <8 x i64> @stack_fold_paddq_mask(<8 x i64> %a0, <8 x i64> %a1, ptr %a2, i8 %mask) {1048; CHECK-LABEL: stack_fold_paddq_mask:1049; CHECK:       # %bb.0:1050; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1051; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm11052; CHECK-NEXT:    #APP1053; CHECK-NEXT:    nop1054; CHECK-NEXT:    #NO_APP1055; CHECK-NEXT:    kmovd %esi, %k11056; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm01057; CHECK-NEXT:    vpaddq {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload1058; CHECK-NEXT:    retq1059  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1060  %2 = add <8 x i64> %a0, %a11061  %3 = bitcast i8 %mask to <8 x i1>1062  ; load needed to keep the operation from being scheduled about the asm block1063  %4 = load <8 x i64>, ptr %a21064  %5 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> %41065  ret <8 x i64> %51066}1067 1068define <8 x i64> @stack_fold_paddq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, ptr %a2, i8 %mask) {1069; CHECK-LABEL: stack_fold_paddq_mask_commuted:1070; CHECK:       # %bb.0:1071; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1072; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm11073; CHECK-NEXT:    #APP1074; CHECK-NEXT:    nop1075; CHECK-NEXT:    #NO_APP1076; CHECK-NEXT:    kmovd %esi, %k11077; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm01078; CHECK-NEXT:    vpaddq {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload1079; CHECK-NEXT:    retq1080  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1081  %2 = add <8 x i64> %a1, %a01082  %3 = bitcast i8 %mask to <8 x i1>1083  ; load needed to keep the operation from being scheduled about the asm block1084  %4 = load <8 x i64>, ptr %a21085  %5 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> %41086  ret <8 x i64> %51087}1088 1089define <8 x i64> @stack_fold_paddq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {1090; CHECK-LABEL: stack_fold_paddq_maskz:1091; CHECK:       # %bb.0:1092; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1093; CHECK-NEXT:    #APP1094; CHECK-NEXT:    nop1095; CHECK-NEXT:    #NO_APP1096; CHECK-NEXT:    kmovd %edi, %k11097; CHECK-NEXT:    vpaddq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload1098; CHECK-NEXT:    retq1099  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1100  %2 = add <8 x i64> %a0, %a11101  %3 = bitcast i8 %mask to <8 x i1>1102  %4 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> zeroinitializer1103  ret <8 x i64> %41104}1105 1106define <8 x i64> @stack_fold_paddq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {1107; CHECK-LABEL: stack_fold_paddq_maskz_commuted:1108; CHECK:       # %bb.0:1109; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1110; CHECK-NEXT:    #APP1111; CHECK-NEXT:    nop1112; CHECK-NEXT:    #NO_APP1113; CHECK-NEXT:    kmovd %edi, %k11114; CHECK-NEXT:    vpaddq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload1115; CHECK-NEXT:    retq1116  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1117  %2 = add <8 x i64> %a1, %a01118  %3 = bitcast i8 %mask to <8 x i1>1119  %4 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> zeroinitializer1120  ret <8 x i64> %41121}1122 1123define <64 x i8> @stack_fold_paddsb(<64 x i8> %a0, <64 x i8> %a1) {1124; CHECK-LABEL: stack_fold_paddsb:1125; CHECK:       # %bb.0:1126; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1127; CHECK-NEXT:    #APP1128; CHECK-NEXT:    nop1129; CHECK-NEXT:    #NO_APP1130; CHECK-NEXT:    vpaddsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload1131; CHECK-NEXT:    retq1132  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1133  %2 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> %a0, <64 x i8> %a1)1134  ret <64 x i8> %21135}1136 1137define <64 x i8> @stack_fold_paddsb_commuted(<64 x i8> %a0, <64 x i8> %a1) {1138; CHECK-LABEL: stack_fold_paddsb_commuted:1139; CHECK:       # %bb.0:1140; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1141; CHECK-NEXT:    #APP1142; CHECK-NEXT:    nop1143; CHECK-NEXT:    #NO_APP1144; CHECK-NEXT:    vpaddsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload1145; CHECK-NEXT:    retq1146  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1147  %2 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> %a1, <64 x i8> %a0)1148  ret <64 x i8> %21149}1150 1151define <64 x i8> @stack_fold_paddsb_mask(<64 x i8> %a0, <64 x i8> %a1, ptr %a2, i64 %mask) {1152; CHECK-LABEL: stack_fold_paddsb_mask:1153; CHECK:       # %bb.0:1154; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1155; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm11156; CHECK-NEXT:    #APP1157; CHECK-NEXT:    nop1158; CHECK-NEXT:    #NO_APP1159; CHECK-NEXT:    kmovq %rsi, %k11160; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm01161; CHECK-NEXT:    vpaddsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload1162; CHECK-NEXT:    retq1163  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1164  %2 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> %a0, <64 x i8> %a1)1165  %3 = bitcast i64 %mask to <64 x i1>1166  ; load needed to keep the operation from being scheduled about the asm block1167  %4 = load <64 x i8>, ptr %a21168  %5 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> %41169  ret <64 x i8> %51170}1171 1172define <64 x i8> @stack_fold_paddsb_mask_commuted(<64 x i8> %a0, <64 x i8> %a1, ptr %a2, i64 %mask) {1173; CHECK-LABEL: stack_fold_paddsb_mask_commuted:1174; CHECK:       # %bb.0:1175; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1176; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm11177; CHECK-NEXT:    #APP1178; CHECK-NEXT:    nop1179; CHECK-NEXT:    #NO_APP1180; CHECK-NEXT:    kmovq %rsi, %k11181; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm01182; CHECK-NEXT:    vpaddsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload1183; CHECK-NEXT:    retq1184  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1185  %2 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> %a1, <64 x i8> %a0)1186  %3 = bitcast i64 %mask to <64 x i1>1187  ; load needed to keep the operation from being scheduled about the asm block1188  %4 = load <64 x i8>, ptr %a21189  %5 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> %41190  ret <64 x i8> %51191}1192 1193define <64 x i8> @stack_fold_paddsb_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {1194; CHECK-LABEL: stack_fold_paddsb_maskz:1195; CHECK:       # %bb.0:1196; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1197; CHECK-NEXT:    #APP1198; CHECK-NEXT:    nop1199; CHECK-NEXT:    #NO_APP1200; CHECK-NEXT:    kmovq %rdi, %k11201; CHECK-NEXT:    vpaddsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload1202; CHECK-NEXT:    retq1203  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1204  %2 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> %a0, <64 x i8> %a1)1205  %3 = bitcast i64 %mask to <64 x i1>1206  %4 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> zeroinitializer1207  ret <64 x i8> %41208}1209 1210define <64 x i8> @stack_fold_paddsb_maskz_commuted(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {1211; CHECK-LABEL: stack_fold_paddsb_maskz_commuted:1212; CHECK:       # %bb.0:1213; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1214; CHECK-NEXT:    #APP1215; CHECK-NEXT:    nop1216; CHECK-NEXT:    #NO_APP1217; CHECK-NEXT:    kmovq %rdi, %k11218; CHECK-NEXT:    vpaddsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload1219; CHECK-NEXT:    retq1220  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1221  %2 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> %a1, <64 x i8> %a0)1222  %3 = bitcast i64 %mask to <64 x i1>1223  %4 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> zeroinitializer1224  ret <64 x i8> %41225}1226 1227define <32 x i16> @stack_fold_paddsw(<32 x i16> %a0, <32 x i16> %a1) {1228; CHECK-LABEL: stack_fold_paddsw:1229; CHECK:       # %bb.0:1230; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1231; CHECK-NEXT:    #APP1232; CHECK-NEXT:    nop1233; CHECK-NEXT:    #NO_APP1234; CHECK-NEXT:    vpaddsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload1235; CHECK-NEXT:    retq1236  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1237  %2 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> %a0, <32 x i16> %a1)1238  ret <32 x i16> %21239}1240 1241define <32 x i16> @stack_fold_paddsw_commuted(<32 x i16> %a0, <32 x i16> %a1) {1242; CHECK-LABEL: stack_fold_paddsw_commuted:1243; CHECK:       # %bb.0:1244; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1245; CHECK-NEXT:    #APP1246; CHECK-NEXT:    nop1247; CHECK-NEXT:    #NO_APP1248; CHECK-NEXT:    vpaddsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload1249; CHECK-NEXT:    retq1250  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1251  %2 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> %a1, <32 x i16> %a0)1252  ret <32 x i16> %21253}1254 1255define <32 x i16> @stack_fold_paddsw_mask(<32 x i16> %a0, <32 x i16> %a1, ptr %a2, i32 %mask) {1256; CHECK-LABEL: stack_fold_paddsw_mask:1257; CHECK:       # %bb.0:1258; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1259; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm11260; CHECK-NEXT:    #APP1261; CHECK-NEXT:    nop1262; CHECK-NEXT:    #NO_APP1263; CHECK-NEXT:    kmovd %esi, %k11264; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm01265; CHECK-NEXT:    vpaddsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload1266; CHECK-NEXT:    retq1267  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1268  %2 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> %a0, <32 x i16> %a1)1269  %3 = bitcast i32 %mask to <32 x i1>1270  ; load needed to keep the operation from being scheduled about the asm block1271  %4 = load <32 x i16>, ptr %a21272  %5 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> %41273  ret <32 x i16> %51274}1275 1276define <32 x i16> @stack_fold_paddsw_mask_commuted(<32 x i16> %a0, <32 x i16> %a1, ptr %a2, i32 %mask) {1277; CHECK-LABEL: stack_fold_paddsw_mask_commuted:1278; CHECK:       # %bb.0:1279; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1280; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm11281; CHECK-NEXT:    #APP1282; CHECK-NEXT:    nop1283; CHECK-NEXT:    #NO_APP1284; CHECK-NEXT:    kmovd %esi, %k11285; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm01286; CHECK-NEXT:    vpaddsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload1287; CHECK-NEXT:    retq1288  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1289  %2 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> %a1, <32 x i16> %a0)1290  %3 = bitcast i32 %mask to <32 x i1>1291  ; load needed to keep the operation from being scheduled about the asm block1292  %4 = load <32 x i16>, ptr %a21293  %5 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> %41294  ret <32 x i16> %51295}1296 1297define <32 x i16> @stack_fold_paddsw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {1298; CHECK-LABEL: stack_fold_paddsw_maskz:1299; CHECK:       # %bb.0:1300; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1301; CHECK-NEXT:    #APP1302; CHECK-NEXT:    nop1303; CHECK-NEXT:    #NO_APP1304; CHECK-NEXT:    kmovd %edi, %k11305; CHECK-NEXT:    vpaddsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload1306; CHECK-NEXT:    retq1307  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1308  %2 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> %a0, <32 x i16> %a1)1309  %3 = bitcast i32 %mask to <32 x i1>1310  %4 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> zeroinitializer1311  ret <32 x i16> %41312}1313 1314define <32 x i16> @stack_fold_paddsw_maskz_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {1315; CHECK-LABEL: stack_fold_paddsw_maskz_commuted:1316; CHECK:       # %bb.0:1317; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1318; CHECK-NEXT:    #APP1319; CHECK-NEXT:    nop1320; CHECK-NEXT:    #NO_APP1321; CHECK-NEXT:    kmovd %edi, %k11322; CHECK-NEXT:    vpaddsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload1323; CHECK-NEXT:    retq1324  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1325  %2 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> %a1, <32 x i16> %a0)1326  %3 = bitcast i32 %mask to <32 x i1>1327  %4 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> zeroinitializer1328  ret <32 x i16> %41329}1330 1331define <64 x i8> @stack_fold_paddusb(<64 x i8> %a0, <64 x i8> %a1) {1332; CHECK-LABEL: stack_fold_paddusb:1333; CHECK:       # %bb.0:1334; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1335; CHECK-NEXT:    #APP1336; CHECK-NEXT:    nop1337; CHECK-NEXT:    #NO_APP1338; CHECK-NEXT:    vpaddusb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload1339; CHECK-NEXT:    retq1340  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1341  %2 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> %a0, <64 x i8> %a1)1342  ret <64 x i8> %21343}1344 1345define <64 x i8> @stack_fold_paddusb_commuted(<64 x i8> %a0, <64 x i8> %a1) {1346; CHECK-LABEL: stack_fold_paddusb_commuted:1347; CHECK:       # %bb.0:1348; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1349; CHECK-NEXT:    #APP1350; CHECK-NEXT:    nop1351; CHECK-NEXT:    #NO_APP1352; CHECK-NEXT:    vpaddusb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload1353; CHECK-NEXT:    retq1354  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1355  %2 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> %a1, <64 x i8> %a0)1356  ret <64 x i8> %21357}1358 1359define <64 x i8> @stack_fold_paddusb_mask(<64 x i8> %a0, <64 x i8> %a1, ptr %a2, i64 %mask) {1360; CHECK-LABEL: stack_fold_paddusb_mask:1361; CHECK:       # %bb.0:1362; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1363; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm11364; CHECK-NEXT:    #APP1365; CHECK-NEXT:    nop1366; CHECK-NEXT:    #NO_APP1367; CHECK-NEXT:    kmovq %rsi, %k11368; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm01369; CHECK-NEXT:    vpaddusb {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload1370; CHECK-NEXT:    retq1371  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1372  %2 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> %a0, <64 x i8> %a1)1373  %3 = bitcast i64 %mask to <64 x i1>1374  ; load needed to keep the operation from being scheduled about the asm block1375  %4 = load <64 x i8>, ptr %a21376  %5 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> %41377  ret <64 x i8> %51378}1379 1380define <64 x i8> @stack_fold_paddusb_mask_commuted(<64 x i8> %a0, <64 x i8> %a1, ptr %a2, i64 %mask) {1381; CHECK-LABEL: stack_fold_paddusb_mask_commuted:1382; CHECK:       # %bb.0:1383; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1384; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm11385; CHECK-NEXT:    #APP1386; CHECK-NEXT:    nop1387; CHECK-NEXT:    #NO_APP1388; CHECK-NEXT:    kmovq %rsi, %k11389; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm01390; CHECK-NEXT:    vpaddusb {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload1391; CHECK-NEXT:    retq1392  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1393  %2 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> %a1, <64 x i8> %a0)1394  %3 = bitcast i64 %mask to <64 x i1>1395  ; load needed to keep the operation from being scheduled about the asm block1396  %4 = load <64 x i8>, ptr %a21397  %5 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> %41398  ret <64 x i8> %51399}1400 1401define <64 x i8> @stack_fold_paddusb_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {1402; CHECK-LABEL: stack_fold_paddusb_maskz:1403; CHECK:       # %bb.0:1404; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1405; CHECK-NEXT:    #APP1406; CHECK-NEXT:    nop1407; CHECK-NEXT:    #NO_APP1408; CHECK-NEXT:    kmovq %rdi, %k11409; CHECK-NEXT:    vpaddusb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload1410; CHECK-NEXT:    retq1411  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1412  %2 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> %a0, <64 x i8> %a1)1413  %3 = bitcast i64 %mask to <64 x i1>1414  %4 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> zeroinitializer1415  ret <64 x i8> %41416}1417 1418define <64 x i8> @stack_fold_paddusb_maskz_commuted(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {1419; CHECK-LABEL: stack_fold_paddusb_maskz_commuted:1420; CHECK:       # %bb.0:1421; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1422; CHECK-NEXT:    #APP1423; CHECK-NEXT:    nop1424; CHECK-NEXT:    #NO_APP1425; CHECK-NEXT:    kmovq %rdi, %k11426; CHECK-NEXT:    vpaddusb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload1427; CHECK-NEXT:    retq1428  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1429  %2 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> %a1, <64 x i8> %a0)1430  %3 = bitcast i64 %mask to <64 x i1>1431  %4 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> zeroinitializer1432  ret <64 x i8> %41433}1434 1435define <32 x i16> @stack_fold_paddusw(<32 x i16> %a0, <32 x i16> %a1) {1436; CHECK-LABEL: stack_fold_paddusw:1437; CHECK:       # %bb.0:1438; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1439; CHECK-NEXT:    #APP1440; CHECK-NEXT:    nop1441; CHECK-NEXT:    #NO_APP1442; CHECK-NEXT:    vpaddusw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload1443; CHECK-NEXT:    retq1444  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1445  %2 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %a0, <32 x i16> %a1)1446  ret <32 x i16> %21447}1448 1449define <32 x i16> @stack_fold_paddusw_commuted(<32 x i16> %a0, <32 x i16> %a1) {1450; CHECK-LABEL: stack_fold_paddusw_commuted:1451; CHECK:       # %bb.0:1452; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1453; CHECK-NEXT:    #APP1454; CHECK-NEXT:    nop1455; CHECK-NEXT:    #NO_APP1456; CHECK-NEXT:    vpaddusw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload1457; CHECK-NEXT:    retq1458  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1459  %2 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %a1, <32 x i16> %a0)1460  ret <32 x i16> %21461}1462 1463define <32 x i16> @stack_fold_paddusw_mask(<32 x i16> %a0, <32 x i16> %a1, ptr %a2, i32 %mask) {1464; CHECK-LABEL: stack_fold_paddusw_mask:1465; CHECK:       # %bb.0:1466; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1467; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm11468; CHECK-NEXT:    #APP1469; CHECK-NEXT:    nop1470; CHECK-NEXT:    #NO_APP1471; CHECK-NEXT:    kmovd %esi, %k11472; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm01473; CHECK-NEXT:    vpaddusw {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload1474; CHECK-NEXT:    retq1475  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1476  %2 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %a0, <32 x i16> %a1)1477  %3 = bitcast i32 %mask to <32 x i1>1478  ; load needed to keep the operation from being scheduled about the asm block1479  %4 = load <32 x i16>, ptr %a21480  %5 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> %41481  ret <32 x i16> %51482}1483 1484define <32 x i16> @stack_fold_paddusw_mask_commuted(<32 x i16> %a0, <32 x i16> %a1, ptr %a2, i32 %mask) {1485; CHECK-LABEL: stack_fold_paddusw_mask_commuted:1486; CHECK:       # %bb.0:1487; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1488; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm11489; CHECK-NEXT:    #APP1490; CHECK-NEXT:    nop1491; CHECK-NEXT:    #NO_APP1492; CHECK-NEXT:    kmovd %esi, %k11493; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm01494; CHECK-NEXT:    vpaddusw {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload1495; CHECK-NEXT:    retq1496  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1497  %2 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %a1, <32 x i16> %a0)1498  %3 = bitcast i32 %mask to <32 x i1>1499  ; load needed to keep the operation from being scheduled about the asm block1500  %4 = load <32 x i16>, ptr %a21501  %5 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> %41502  ret <32 x i16> %51503}1504 1505define <32 x i16> @stack_fold_paddusw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {1506; CHECK-LABEL: stack_fold_paddusw_maskz:1507; CHECK:       # %bb.0:1508; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1509; CHECK-NEXT:    #APP1510; CHECK-NEXT:    nop1511; CHECK-NEXT:    #NO_APP1512; CHECK-NEXT:    kmovd %edi, %k11513; CHECK-NEXT:    vpaddusw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload1514; CHECK-NEXT:    retq1515  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1516  %2 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %a0, <32 x i16> %a1)1517  %3 = bitcast i32 %mask to <32 x i1>1518  %4 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> zeroinitializer1519  ret <32 x i16> %41520}1521 1522define <32 x i16> @stack_fold_paddusw_maskz_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {1523; CHECK-LABEL: stack_fold_paddusw_maskz_commuted:1524; CHECK:       # %bb.0:1525; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1526; CHECK-NEXT:    #APP1527; CHECK-NEXT:    nop1528; CHECK-NEXT:    #NO_APP1529; CHECK-NEXT:    kmovd %edi, %k11530; CHECK-NEXT:    vpaddusw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload1531; CHECK-NEXT:    retq1532  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1533  %2 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %a1, <32 x i16> %a0)1534  %3 = bitcast i32 %mask to <32 x i1>1535  %4 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> zeroinitializer1536  ret <32 x i16> %41537}1538 1539define <32 x i16> @stack_fold_paddw(<32 x i16> %a0, <32 x i16> %a1) {1540; CHECK-LABEL: stack_fold_paddw:1541; CHECK:       # %bb.0:1542; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1543; CHECK-NEXT:    #APP1544; CHECK-NEXT:    nop1545; CHECK-NEXT:    #NO_APP1546; CHECK-NEXT:    vpaddw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload1547; CHECK-NEXT:    retq1548  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1549  %2 = add <32 x i16> %a0, %a11550  ret <32 x i16> %21551}1552 1553define <32 x i16> @stack_fold_paddw_commuted(<32 x i16> %a0, <32 x i16> %a1) {1554; CHECK-LABEL: stack_fold_paddw_commuted:1555; CHECK:       # %bb.0:1556; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1557; CHECK-NEXT:    #APP1558; CHECK-NEXT:    nop1559; CHECK-NEXT:    #NO_APP1560; CHECK-NEXT:    vpaddw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload1561; CHECK-NEXT:    retq1562  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1563  %2 = add <32 x i16> %a1, %a01564  ret <32 x i16> %21565}1566 1567define <32 x i16> @stack_fold_paddw_mask(<32 x i16> %a0, <32 x i16> %a1, ptr %a2, i32 %mask) {1568; CHECK-LABEL: stack_fold_paddw_mask:1569; CHECK:       # %bb.0:1570; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1571; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm11572; CHECK-NEXT:    #APP1573; CHECK-NEXT:    nop1574; CHECK-NEXT:    #NO_APP1575; CHECK-NEXT:    kmovd %esi, %k11576; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm01577; CHECK-NEXT:    vpaddw {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload1578; CHECK-NEXT:    retq1579  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1580  %2 = add <32 x i16> %a0, %a11581  %3 = bitcast i32 %mask to <32 x i1>1582  ; load needed to keep the operation from being scheduled about the asm block1583  %4 = load <32 x i16>, ptr %a21584  %5 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> %41585  ret <32 x i16> %51586}1587 1588define <32 x i16> @stack_fold_paddw_mask_commuted(<32 x i16> %a0, <32 x i16> %a1, ptr %a2, i32 %mask) {1589; CHECK-LABEL: stack_fold_paddw_mask_commuted:1590; CHECK:       # %bb.0:1591; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1592; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm11593; CHECK-NEXT:    #APP1594; CHECK-NEXT:    nop1595; CHECK-NEXT:    #NO_APP1596; CHECK-NEXT:    kmovd %esi, %k11597; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm01598; CHECK-NEXT:    vpaddw {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload1599; CHECK-NEXT:    retq1600  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1601  %2 = add <32 x i16> %a1, %a01602  %3 = bitcast i32 %mask to <32 x i1>1603  ; load needed to keep the operation from being scheduled about the asm block1604  %4 = load <32 x i16>, ptr %a21605  %5 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> %41606  ret <32 x i16> %51607}1608 1609define <32 x i16> @stack_fold_paddw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {1610; CHECK-LABEL: stack_fold_paddw_maskz:1611; CHECK:       # %bb.0:1612; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1613; CHECK-NEXT:    #APP1614; CHECK-NEXT:    nop1615; CHECK-NEXT:    #NO_APP1616; CHECK-NEXT:    kmovd %edi, %k11617; CHECK-NEXT:    vpaddw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload1618; CHECK-NEXT:    retq1619  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1620  %2 = add <32 x i16> %a0, %a11621  %3 = bitcast i32 %mask to <32 x i1>1622  %4 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> zeroinitializer1623  ret <32 x i16> %41624}1625 1626define <32 x i16> @stack_fold_paddw_maskz_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {1627; CHECK-LABEL: stack_fold_paddw_maskz_commuted:1628; CHECK:       # %bb.0:1629; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1630; CHECK-NEXT:    #APP1631; CHECK-NEXT:    nop1632; CHECK-NEXT:    #NO_APP1633; CHECK-NEXT:    kmovd %edi, %k11634; CHECK-NEXT:    vpaddw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload1635; CHECK-NEXT:    retq1636  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1637  %2 = add <32 x i16> %a1, %a01638  %3 = bitcast i32 %mask to <32 x i1>1639  %4 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> zeroinitializer1640  ret <32 x i16> %41641}1642 1643define <64 x i8> @stack_fold_palignr(<64 x i8> %a0, <64 x i8> %a1) {1644; CHECK-LABEL: stack_fold_palignr:1645; CHECK:       # %bb.0:1646; CHECK-NEXT:    pushq %rax1647; CHECK-NEXT:    .cfi_def_cfa_offset 161648; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1649; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1650; CHECK-NEXT:    #APP1651; CHECK-NEXT:    nop1652; CHECK-NEXT:    #NO_APP1653; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload1654; CHECK-NEXT:    vpalignr $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload1655; CHECK-NEXT:    # zmm0 = mem[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zmm0[0],mem[17,18,19,20,21,22,23,24,25,26,27,28,29,30,31],zmm0[16],mem[33,34,35,36,37,38,39,40,41,42,43,44,45,46,47],zmm0[32],mem[49,50,51,52,53,54,55,56,57,58,59,60,61,62,63],zmm0[48]1656; CHECK-NEXT:    popq %rax1657; CHECK-NEXT:    .cfi_def_cfa_offset 81658; CHECK-NEXT:    retq1659  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1660  %2 = shufflevector <64 x i8> %a1, <64 x i8> %a0, <64 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112>1661  ret <64 x i8> %21662}1663 1664define <64 x i8> @stack_fold_palignr_mask(<64 x i8> %a0, <64 x i8> %a1, ptr %passthru, i64 %mask) {1665; CHECK-LABEL: stack_fold_palignr_mask:1666; CHECK:       # %bb.0:1667; CHECK-NEXT:    pushq %rax1668; CHECK-NEXT:    .cfi_def_cfa_offset 161669; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1670; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1671; CHECK-NEXT:    #APP1672; CHECK-NEXT:    nop1673; CHECK-NEXT:    #NO_APP1674; CHECK-NEXT:    kmovq %rsi, %k11675; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm11676; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload1677; CHECK-NEXT:    vpalignr $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 {%k1} # 64-byte Folded Reload1678; CHECK-NEXT:    # zmm1 {%k1} = mem[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zmm0[0],mem[17,18,19,20,21,22,23,24,25,26,27,28,29,30,31],zmm0[16],mem[33,34,35,36,37,38,39,40,41,42,43,44,45,46,47],zmm0[32],mem[49,50,51,52,53,54,55,56,57,58,59,60,61,62,63],zmm0[48]1679; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm01680; CHECK-NEXT:    popq %rax1681; CHECK-NEXT:    .cfi_def_cfa_offset 81682; CHECK-NEXT:    retq1683  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1684  %2 = shufflevector <64 x i8> %a1, <64 x i8> %a0, <64 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112>1685  %3 = bitcast i64 %mask to <64 x i1>1686  %4 = load <64 x i8>, ptr %passthru1687  %5 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> %41688  ret <64 x i8> %51689}1690 1691define <64 x i8> @stack_fold_palignr_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {1692; CHECK-LABEL: stack_fold_palignr_maskz:1693; CHECK:       # %bb.0:1694; CHECK-NEXT:    pushq %rax1695; CHECK-NEXT:    .cfi_def_cfa_offset 161696; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1697; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1698; CHECK-NEXT:    #APP1699; CHECK-NEXT:    nop1700; CHECK-NEXT:    #NO_APP1701; CHECK-NEXT:    kmovq %rdi, %k11702; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload1703; CHECK-NEXT:    vpalignr $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload1704; CHECK-NEXT:    # zmm0 {%k1} {z} = mem[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zmm0[0],mem[17,18,19,20,21,22,23,24,25,26,27,28,29,30,31],zmm0[16],mem[33,34,35,36,37,38,39,40,41,42,43,44,45,46,47],zmm0[32],mem[49,50,51,52,53,54,55,56,57,58,59,60,61,62,63],zmm0[48]1705; CHECK-NEXT:    popq %rax1706; CHECK-NEXT:    .cfi_def_cfa_offset 81707; CHECK-NEXT:    retq1708  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1709  %2 = shufflevector <64 x i8> %a1, <64 x i8> %a0, <64 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112>1710  %3 = bitcast i64 %mask to <64 x i1>1711  %4 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> zeroinitializer1712  ret <64 x i8> %41713}1714 1715define <16 x i32> @stack_fold_pandd(<16 x i32> %a0, <16 x i32> %a1) {1716; CHECK-LABEL: stack_fold_pandd:1717; CHECK:       # %bb.0:1718; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1719; CHECK-NEXT:    #APP1720; CHECK-NEXT:    nop1721; CHECK-NEXT:    #NO_APP1722; CHECK-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload1723; CHECK-NEXT:    retq1724  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1725  %2 = and <16 x i32> %a0, %a11726  ret <16 x i32> %21727}1728 1729define <16 x i32> @stack_fold_pandd_commuted(<16 x i32> %a0, <16 x i32> %a1) {1730; CHECK-LABEL: stack_fold_pandd_commuted:1731; CHECK:       # %bb.0:1732; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1733; CHECK-NEXT:    #APP1734; CHECK-NEXT:    nop1735; CHECK-NEXT:    #NO_APP1736; CHECK-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload1737; CHECK-NEXT:    retq1738  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1739  %2 = and <16 x i32> %a1, %a01740  ret <16 x i32> %21741}1742 1743define <16 x i32> @stack_fold_pandd_mask(<16 x i32> %a0, <16 x i32> %a1, ptr %a2, i16 %mask) {1744; CHECK-LABEL: stack_fold_pandd_mask:1745; CHECK:       # %bb.0:1746; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1747; CHECK-NEXT:    vmovaps %zmm0, %zmm11748; CHECK-NEXT:    #APP1749; CHECK-NEXT:    nop1750; CHECK-NEXT:    #NO_APP1751; CHECK-NEXT:    kmovd %esi, %k11752; CHECK-NEXT:    vmovaps (%rdi), %zmm01753; CHECK-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload1754; CHECK-NEXT:    retq1755  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1756  %2 = and <16 x i32> %a0, %a11757  %3 = bitcast i16 %mask to <16 x i1>1758  ; load needed to keep the operation from being scheduled about the asm block1759  %4 = load <16 x i32>, ptr %a21760  %5 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> %41761  ret <16 x i32> %51762}1763 1764define <16 x i32> @stack_fold_pandd_mask_commuted(<16 x i32> %a0, <16 x i32> %a1, ptr %a2, i16 %mask) {1765; CHECK-LABEL: stack_fold_pandd_mask_commuted:1766; CHECK:       # %bb.0:1767; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1768; CHECK-NEXT:    vmovaps %zmm0, %zmm11769; CHECK-NEXT:    #APP1770; CHECK-NEXT:    nop1771; CHECK-NEXT:    #NO_APP1772; CHECK-NEXT:    kmovd %esi, %k11773; CHECK-NEXT:    vmovaps (%rdi), %zmm01774; CHECK-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload1775; CHECK-NEXT:    retq1776  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1777  %2 = and <16 x i32> %a1, %a01778  %3 = bitcast i16 %mask to <16 x i1>1779  ; load needed to keep the operation from being scheduled about the asm block1780  %4 = load <16 x i32>, ptr %a21781  %5 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> %41782  ret <16 x i32> %51783}1784 1785define <16 x i32> @stack_fold_pandd_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {1786; CHECK-LABEL: stack_fold_pandd_maskz:1787; CHECK:       # %bb.0:1788; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1789; CHECK-NEXT:    #APP1790; CHECK-NEXT:    nop1791; CHECK-NEXT:    #NO_APP1792; CHECK-NEXT:    kmovd %edi, %k11793; CHECK-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload1794; CHECK-NEXT:    retq1795  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1796  %2 = and <16 x i32> %a0, %a11797  %3 = bitcast i16 %mask to <16 x i1>1798  %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer1799  ret <16 x i32> %41800}1801 1802define <16 x i32> @stack_fold_pandd_maskz_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {1803; CHECK-LABEL: stack_fold_pandd_maskz_commuted:1804; CHECK:       # %bb.0:1805; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1806; CHECK-NEXT:    #APP1807; CHECK-NEXT:    nop1808; CHECK-NEXT:    #NO_APP1809; CHECK-NEXT:    kmovd %edi, %k11810; CHECK-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload1811; CHECK-NEXT:    retq1812  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1813  %2 = and <16 x i32> %a1, %a01814  %3 = bitcast i16 %mask to <16 x i1>1815  %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer1816  ret <16 x i32> %41817}1818 1819define <8 x i64> @stack_fold_pandq(<8 x i64> %a0, <8 x i64> %a1) {1820; CHECK-LABEL: stack_fold_pandq:1821; CHECK:       # %bb.0:1822; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1823; CHECK-NEXT:    #APP1824; CHECK-NEXT:    nop1825; CHECK-NEXT:    #NO_APP1826; CHECK-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload1827; CHECK-NEXT:    retq1828  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1829  %2 = and <8 x i64> %a0, %a11830  ret <8 x i64> %21831}1832 1833define <8 x i64> @stack_fold_pandq_commuted(<8 x i64> %a0, <8 x i64> %a1) {1834; CHECK-LABEL: stack_fold_pandq_commuted:1835; CHECK:       # %bb.0:1836; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1837; CHECK-NEXT:    #APP1838; CHECK-NEXT:    nop1839; CHECK-NEXT:    #NO_APP1840; CHECK-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload1841; CHECK-NEXT:    retq1842  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1843  %2 = and <8 x i64> %a1, %a01844  ret <8 x i64> %21845}1846 1847define <8 x i64> @stack_fold_pandq_mask(<8 x i64> %a0, <8 x i64> %a1, ptr %a2, i8 %mask) {1848; CHECK-LABEL: stack_fold_pandq_mask:1849; CHECK:       # %bb.0:1850; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1851; CHECK-NEXT:    vmovapd %zmm0, %zmm11852; CHECK-NEXT:    #APP1853; CHECK-NEXT:    nop1854; CHECK-NEXT:    #NO_APP1855; CHECK-NEXT:    kmovd %esi, %k11856; CHECK-NEXT:    vmovapd (%rdi), %zmm01857; CHECK-NEXT:    vandpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload1858; CHECK-NEXT:    retq1859  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1860  %2 = and <8 x i64> %a0, %a11861  %3 = bitcast i8 %mask to <8 x i1>1862  ; load needed to keep the operation from being scheduled about the asm block1863  %4 = load <8 x i64>, ptr %a21864  %5 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> %41865  ret <8 x i64> %51866}1867 1868define <8 x i64> @stack_fold_pandq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, ptr %a2, i8 %mask) {1869; CHECK-LABEL: stack_fold_pandq_mask_commuted:1870; CHECK:       # %bb.0:1871; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1872; CHECK-NEXT:    vmovapd %zmm0, %zmm11873; CHECK-NEXT:    #APP1874; CHECK-NEXT:    nop1875; CHECK-NEXT:    #NO_APP1876; CHECK-NEXT:    kmovd %esi, %k11877; CHECK-NEXT:    vmovapd (%rdi), %zmm01878; CHECK-NEXT:    vandpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload1879; CHECK-NEXT:    retq1880  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1881  %2 = and <8 x i64> %a1, %a01882  %3 = bitcast i8 %mask to <8 x i1>1883  ; load needed to keep the operation from being scheduled about the asm block1884  %4 = load <8 x i64>, ptr %a21885  %5 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> %41886  ret <8 x i64> %51887}1888 1889define <8 x i64> @stack_fold_pandq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {1890; CHECK-LABEL: stack_fold_pandq_maskz:1891; CHECK:       # %bb.0:1892; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1893; CHECK-NEXT:    #APP1894; CHECK-NEXT:    nop1895; CHECK-NEXT:    #NO_APP1896; CHECK-NEXT:    kmovd %edi, %k11897; CHECK-NEXT:    vandpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload1898; CHECK-NEXT:    retq1899  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1900  %2 = and <8 x i64> %a0, %a11901  %3 = bitcast i8 %mask to <8 x i1>1902  %4 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> zeroinitializer1903  ret <8 x i64> %41904}1905 1906define <8 x i64> @stack_fold_pandq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {1907; CHECK-LABEL: stack_fold_pandq_maskz_commuted:1908; CHECK:       # %bb.0:1909; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1910; CHECK-NEXT:    #APP1911; CHECK-NEXT:    nop1912; CHECK-NEXT:    #NO_APP1913; CHECK-NEXT:    kmovd %edi, %k11914; CHECK-NEXT:    vandpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload1915; CHECK-NEXT:    retq1916  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1917  %2 = and <8 x i64> %a1, %a01918  %3 = bitcast i8 %mask to <8 x i1>1919  %4 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> zeroinitializer1920  ret <8 x i64> %41921}1922 1923define <16 x i32> @stack_fold_vpconflictd(<16 x i32> %a0) {1924; CHECK-LABEL: stack_fold_vpconflictd:1925; CHECK:       # %bb.0:1926; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1927; CHECK-NEXT:    #APP1928; CHECK-NEXT:    nop1929; CHECK-NEXT:    #NO_APP1930; CHECK-NEXT:    vpconflictd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload1931; CHECK-NEXT:    retq1932  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1933  %2 = call <16 x i32> @llvm.x86.avx512.conflict.d.512(<16 x i32> %a0)1934  ret <16 x i32> %21935}1936declare <16 x i32> @llvm.x86.avx512.mask.conflict.d.512(<16 x i32>, <16 x i32>, i16) nounwind readonly1937 1938define <8 x i64> @stack_fold_vpconflictq(<8 x i64> %a0) {1939; CHECK-LABEL: stack_fold_vpconflictq:1940; CHECK:       # %bb.0:1941; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1942; CHECK-NEXT:    #APP1943; CHECK-NEXT:    nop1944; CHECK-NEXT:    #NO_APP1945; CHECK-NEXT:    vpconflictq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload1946; CHECK-NEXT:    retq1947  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1948  %2 = call <8 x i64> @llvm.x86.avx512.conflict.q.512(<8 x i64> %a0)1949  ret <8 x i64> %21950}1951declare <8 x i64> @llvm.x86.avx512.mask.conflict.q.512(<8 x i64>, <8 x i64>, i8) nounwind readnone1952 1953define i64 @stack_fold_pcmpeqb(<64 x i8> %a0, <64 x i8> %a1) {1954; CHECK-LABEL: stack_fold_pcmpeqb:1955; CHECK:       # %bb.0:1956; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1957; CHECK-NEXT:    #APP1958; CHECK-NEXT:    nop1959; CHECK-NEXT:    #NO_APP1960; CHECK-NEXT:    vpcmpeqb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %k0 # 64-byte Folded Reload1961; CHECK-NEXT:    kmovq %k0, %rax1962; CHECK-NEXT:    vzeroupper1963; CHECK-NEXT:    retq1964  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1965  %2 = icmp eq <64 x i8> %a0, %a11966  %3 = bitcast <64 x i1> %2 to i641967  ret i64 %31968}1969 1970define i16 @stack_fold_pcmpeqd(<16 x i32> %a0, <16 x i32> %a1) {1971; CHECK-LABEL: stack_fold_pcmpeqd:1972; CHECK:       # %bb.0:1973; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1974; CHECK-NEXT:    #APP1975; CHECK-NEXT:    nop1976; CHECK-NEXT:    #NO_APP1977; CHECK-NEXT:    vpcmpeqd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %k0 # 64-byte Folded Reload1978; CHECK-NEXT:    kmovd %k0, %eax1979; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax1980; CHECK-NEXT:    vzeroupper1981; CHECK-NEXT:    retq1982  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1983  %2 = icmp eq <16 x i32> %a0, %a11984  %3 = bitcast <16 x i1> %2 to i161985  ret i16 %31986}1987 1988define i8 @stack_fold_pcmpeqq(<8 x i64> %a0, <8 x i64> %a1) {1989; CHECK-LABEL: stack_fold_pcmpeqq:1990; CHECK:       # %bb.0:1991; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1992; CHECK-NEXT:    #APP1993; CHECK-NEXT:    nop1994; CHECK-NEXT:    #NO_APP1995; CHECK-NEXT:    vpcmpeqq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %k0 # 64-byte Folded Reload1996; CHECK-NEXT:    kmovd %k0, %eax1997; CHECK-NEXT:    # kill: def $al killed $al killed $eax1998; CHECK-NEXT:    vzeroupper1999; CHECK-NEXT:    retq2000  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2001  %2 = icmp eq <8 x i64> %a0, %a12002  %3 = bitcast <8 x i1> %2 to i82003  ret i8 %32004}2005 2006define i32 @stack_fold_pcmpeqw(<32 x i16> %a0, <32 x i16> %a1) {2007; CHECK-LABEL: stack_fold_pcmpeqw:2008; CHECK:       # %bb.0:2009; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2010; CHECK-NEXT:    #APP2011; CHECK-NEXT:    nop2012; CHECK-NEXT:    #NO_APP2013; CHECK-NEXT:    vpcmpeqw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %k0 # 64-byte Folded Reload2014; CHECK-NEXT:    kmovd %k0, %eax2015; CHECK-NEXT:    vzeroupper2016; CHECK-NEXT:    retq2017  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2018  %2 = icmp eq <32 x i16> %a0, %a12019  %3 = bitcast <32 x i1> %2 to i322020  ret i32 %32021}2022 2023define <16 x i32> @stack_fold_pcmpeqd_mask(<16 x i32> %a0, <16 x i32> %a1, ptr %a2, i16 %mask, <16 x i32> %b0, <16 x i32> %b1) {2024; CHECK-LABEL: stack_fold_pcmpeqd_mask:2025; CHECK:       # %bb.0:2026; CHECK-NEXT:    subq $136, %rsp2027; CHECK-NEXT:    .cfi_def_cfa_offset 1442028; CHECK-NEXT:    vmovups %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2029; CHECK-NEXT:    vmovups %zmm2, (%rsp) # 64-byte Spill2030; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2031; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2032; CHECK-NEXT:    #APP2033; CHECK-NEXT:    nop2034; CHECK-NEXT:    #NO_APP2035; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload2036; CHECK-NEXT:    vpaddd (%rdi), %zmm0, %zmm02037; CHECK-NEXT:    kmovd %esi, %k12038; CHECK-NEXT:    vpcmpeqd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %k1 {%k1} # 64-byte Folded Reload2039; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload2040; CHECK-NEXT:    vmovdqu64 (%rsp), %zmm1 # 64-byte Reload2041; CHECK-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}2042; CHECK-NEXT:    addq $136, %rsp2043; CHECK-NEXT:    .cfi_def_cfa_offset 82044; CHECK-NEXT:    retq2045  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2046  ; load and add are here to keep the operations below the side effecting block and to avoid folding the wrong load2047  %2 = load <16 x i32>, ptr %a22048  %3 = add <16 x i32> %a1, %22049  %4 = bitcast i16 %mask to <16 x i1>2050  %5 = icmp eq <16 x i32> %3, %a02051  %6 = and <16 x i1> %4, %52052  %7 = select <16 x i1> %6, <16 x i32> %b0, <16 x i32> %b12053  ret <16 x i32> %72054}2055 2056define <16 x i32> @stack_fold_pcmpeqd_mask_commuted(<16 x i32> %a0, <16 x i32> %a1, ptr %a2, i16 %mask, <16 x i32> %b0, <16 x i32> %b1) {2057; CHECK-LABEL: stack_fold_pcmpeqd_mask_commuted:2058; CHECK:       # %bb.0:2059; CHECK-NEXT:    subq $136, %rsp2060; CHECK-NEXT:    .cfi_def_cfa_offset 1442061; CHECK-NEXT:    vmovups %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2062; CHECK-NEXT:    vmovups %zmm2, (%rsp) # 64-byte Spill2063; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2064; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2065; CHECK-NEXT:    #APP2066; CHECK-NEXT:    nop2067; CHECK-NEXT:    #NO_APP2068; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload2069; CHECK-NEXT:    vpaddd (%rdi), %zmm0, %zmm02070; CHECK-NEXT:    kmovd %esi, %k12071; CHECK-NEXT:    vpcmpeqd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %k1 {%k1} # 64-byte Folded Reload2072; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload2073; CHECK-NEXT:    vmovdqu64 (%rsp), %zmm1 # 64-byte Reload2074; CHECK-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}2075; CHECK-NEXT:    addq $136, %rsp2076; CHECK-NEXT:    .cfi_def_cfa_offset 82077; CHECK-NEXT:    retq2078  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2079  ; load and add are here to keep the operations below the side effecting block and to avoid folding the wrong load2080  %2 = load <16 x i32>, ptr %a22081  %3 = add <16 x i32> %a1, %22082  %4 = bitcast i16 %mask to <16 x i1>2083  %5 = icmp eq <16 x i32> %a0, %32084  %6 = and <16 x i1> %4, %52085  %7 = select <16 x i1> %6, <16 x i32> %b0, <16 x i32> %b12086  ret <16 x i32> %72087}2088 2089define <16 x i32> @stack_fold_pcmpled_mask(<16 x i32> %a0, <16 x i32> %a1, ptr %a2, i16 %mask, <16 x i32> %b0, <16 x i32> %b1) {2090; CHECK-LABEL: stack_fold_pcmpled_mask:2091; CHECK:       # %bb.0:2092; CHECK-NEXT:    subq $136, %rsp2093; CHECK-NEXT:    .cfi_def_cfa_offset 1442094; CHECK-NEXT:    vmovups %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2095; CHECK-NEXT:    vmovups %zmm2, (%rsp) # 64-byte Spill2096; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2097; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2098; CHECK-NEXT:    #APP2099; CHECK-NEXT:    nop2100; CHECK-NEXT:    #NO_APP2101; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload2102; CHECK-NEXT:    vpaddd (%rdi), %zmm0, %zmm02103; CHECK-NEXT:    kmovd %esi, %k12104; CHECK-NEXT:    vpcmpled {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %k1 {%k1} # 64-byte Folded Reload2105; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload2106; CHECK-NEXT:    vmovdqu64 (%rsp), %zmm1 # 64-byte Reload2107; CHECK-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}2108; CHECK-NEXT:    addq $136, %rsp2109; CHECK-NEXT:    .cfi_def_cfa_offset 82110; CHECK-NEXT:    retq2111  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2112  ; load and add are here to keep the operations below the side effecting block and to avoid folding the wrong load2113  %2 = load <16 x i32>, ptr %a22114  %3 = add <16 x i32> %a1, %22115  %4 = bitcast i16 %mask to <16 x i1>2116  %5 = icmp sge <16 x i32> %a0, %32117  %6 = and <16 x i1> %4, %52118  %7 = select <16 x i1> %6, <16 x i32> %b0, <16 x i32> %b12119  ret <16 x i32> %72120}2121 2122define i16 @stack_fold_pcmpleud(<16 x i32> %a0, <16 x i32> %a1, ptr %a2, i16 %mask) {2123; CHECK-LABEL: stack_fold_pcmpleud:2124; CHECK:       # %bb.0:2125; CHECK-NEXT:    pushq %rax2126; CHECK-NEXT:    .cfi_def_cfa_offset 162127; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2128; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2129; CHECK-NEXT:    #APP2130; CHECK-NEXT:    nop2131; CHECK-NEXT:    #NO_APP2132; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload2133; CHECK-NEXT:    vpaddd (%rdi), %zmm0, %zmm02134; CHECK-NEXT:    vpcmpleud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %k0 # 64-byte Folded Reload2135; CHECK-NEXT:    kmovd %k0, %eax2136; CHECK-NEXT:    andl %esi, %eax2137; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax2138; CHECK-NEXT:    popq %rcx2139; CHECK-NEXT:    .cfi_def_cfa_offset 82140; CHECK-NEXT:    vzeroupper2141; CHECK-NEXT:    retq2142  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2143  %2 = load <16 x i32>, ptr %a22144  %3 = add <16 x i32> %a1, %22145  %4 = bitcast i16 %mask to <16 x i1>2146  %5 = icmp uge <16 x i32> %a0, %32147  %6 = and <16 x i1> %5, %42148  %7 = bitcast <16 x i1> %6 to i162149  ret i16 %72150}2151 2152define <64 x i8> @stack_fold_permbvar(<64 x i8> %a0, <64 x i8> %a1) {2153; CHECK-LABEL: stack_fold_permbvar:2154; CHECK:       # %bb.0:2155; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2156; CHECK-NEXT:    #APP2157; CHECK-NEXT:    nop2158; CHECK-NEXT:    #NO_APP2159; CHECK-NEXT:    vpermb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload2160; CHECK-NEXT:    retq2161  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2162  %2 = call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %a1, <64 x i8> %a0)2163  ret <64 x i8> %22164}2165declare <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8>, <64 x i8>) nounwind readonly2166 2167define <64 x i8> @stack_fold_permbvar_mask(ptr %passthru, <64 x i8> %a0, <64 x i8> %a1, i64 %mask) {2168; CHECK-LABEL: stack_fold_permbvar_mask:2169; CHECK:       # %bb.0:2170; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2171; CHECK-NEXT:    #APP2172; CHECK-NEXT:    nop2173; CHECK-NEXT:    #NO_APP2174; CHECK-NEXT:    kmovq %rsi, %k12175; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm22176; CHECK-NEXT:    vpermb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload2177; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm02178; CHECK-NEXT:    retq2179  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2180  %2 = call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %a1, <64 x i8> %a0)2181  %3 = bitcast i64 %mask to <64 x i1>2182  ; load needed to keep the operation from being scheduled above the asm block2183  %4 = load <64 x i8>, ptr %passthru2184  %5 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> %42185  ret <64 x i8> %52186}2187 2188define <64 x i8> @stack_fold_permbvar_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {2189; CHECK-LABEL: stack_fold_permbvar_maskz:2190; CHECK:       # %bb.0:2191; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2192; CHECK-NEXT:    #APP2193; CHECK-NEXT:    nop2194; CHECK-NEXT:    #NO_APP2195; CHECK-NEXT:    kmovq %rdi, %k12196; CHECK-NEXT:    vpermb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload2197; CHECK-NEXT:    retq2198  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2199  %2 = call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %a1, <64 x i8> %a0)2200  %3 = bitcast i64 %mask to <64 x i1>2201  %4 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> zeroinitializer2202  ret <64 x i8> %42203}2204 2205define <16 x i32> @stack_fold_permd(<16 x i32> %a0, <16 x i32> %a1) {2206; CHECK-LABEL: stack_fold_permd:2207; CHECK:       # %bb.0:2208; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2209; CHECK-NEXT:    #APP2210; CHECK-NEXT:    nop2211; CHECK-NEXT:    #NO_APP2212; CHECK-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload2213; CHECK-NEXT:    vpternlogd {{.*#+}} zmm1 = -12214; CHECK-NEXT:    vpsubd %zmm1, %zmm0, %zmm02215; CHECK-NEXT:    retq2216  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2217  %2 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %a1, <16 x i32> %a0)2218  ; add forces execution domain2219  %3 = add <16 x i32> %2, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>2220  ret <16 x i32> %32221}2222declare <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32>, <16 x i32>) nounwind readonly2223 2224define <64 x i8> @stack_fold_vpermi2b(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2) {2225; CHECK-LABEL: stack_fold_vpermi2b:2226; CHECK:       # %bb.0:2227; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2228; CHECK-NEXT:    #APP2229; CHECK-NEXT:    nop2230; CHECK-NEXT:    #NO_APP2231; CHECK-NEXT:    vpermi2b {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload2232; CHECK-NEXT:    retq2233  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2234  %2 = call <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8> %x1, <64 x i8> %x0, <64 x i8> %x2)2235  ret <64 x i8> %22236}2237 2238define <16 x i32> @stack_fold_vpermi2d(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) {2239; CHECK-LABEL: stack_fold_vpermi2d:2240; CHECK:       # %bb.0:2241; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2242; CHECK-NEXT:    #APP2243; CHECK-NEXT:    nop2244; CHECK-NEXT:    #NO_APP2245; CHECK-NEXT:    vpermi2d {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload2246; CHECK-NEXT:    retq2247  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2248  %2 = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> %x1, <16 x i32> %x0, <16 x i32> %x2)2249  ret <16 x i32> %22250}2251 2252define <8 x i64> @stack_fold_vpermi2q(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2) {2253; CHECK-LABEL: stack_fold_vpermi2q:2254; CHECK:       # %bb.0:2255; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2256; CHECK-NEXT:    #APP2257; CHECK-NEXT:    nop2258; CHECK-NEXT:    #NO_APP2259; CHECK-NEXT:    vpermi2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload2260; CHECK-NEXT:    retq2261  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2262  %2 = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> %x1, <8 x i64> %x0, <8 x i64> %x2)2263  ret <8 x i64> %22264}2265 2266define <32 x i16> @stack_fold_vpermi2w(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) {2267; CHECK-LABEL: stack_fold_vpermi2w:2268; CHECK:       # %bb.0:2269; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2270; CHECK-NEXT:    #APP2271; CHECK-NEXT:    nop2272; CHECK-NEXT:    #NO_APP2273; CHECK-NEXT:    vpermi2w {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload2274; CHECK-NEXT:    retq2275  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2276  %2 = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> %x1, <32 x i16> %x0, <32 x i16> %x2)2277  ret <32 x i16> %22278}2279 2280define <8 x i64> @stack_fold_permq(<8 x i64> %a0) {2281; CHECK-LABEL: stack_fold_permq:2282; CHECK:       # %bb.0:2283; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2284; CHECK-NEXT:    #APP2285; CHECK-NEXT:    nop2286; CHECK-NEXT:    #NO_APP2287; CHECK-NEXT:    vpermq $235, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload2288; CHECK-NEXT:    # zmm0 = mem[3,2,2,3,7,6,6,7]2289; CHECK-NEXT:    vpternlogd {{.*#+}} zmm1 = -12290; CHECK-NEXT:    vpsubq %zmm1, %zmm0, %zmm02291; CHECK-NEXT:    retq2292  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2293  %2 = shufflevector <8 x i64> %a0, <8 x i64> undef, <8 x i32> <i32 3, i32 2, i32 2, i32 3, i32 7, i32 6, i32 6, i32 7>2294  ; add forces execution domain2295  %3 = add <8 x i64> %2, <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>2296  ret <8 x i64> %32297}2298 2299define <8 x i64> @stack_fold_permq_mask(ptr %passthru, <8 x i64> %a0, i8 %mask) {2300; CHECK-LABEL: stack_fold_permq_mask:2301; CHECK:       # %bb.0:2302; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2303; CHECK-NEXT:    #APP2304; CHECK-NEXT:    nop2305; CHECK-NEXT:    #NO_APP2306; CHECK-NEXT:    kmovd %esi, %k12307; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm02308; CHECK-NEXT:    vpermq $235, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} # 64-byte Folded Reload2309; CHECK-NEXT:    # zmm0 {%k1} = mem[3,2,2,3,7,6,6,7]2310; CHECK-NEXT:    vpternlogd {{.*#+}} zmm1 = -12311; CHECK-NEXT:    vpsubq %zmm1, %zmm0, %zmm02312; CHECK-NEXT:    retq2313  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2314  %2 = shufflevector <8 x i64> %a0, <8 x i64> undef, <8 x i32> <i32 3, i32 2, i32 2, i32 3, i32 7, i32 6, i32 6, i32 7>2315  %3 = bitcast i8 %mask to <8 x i1>2316  ; load needed to keep the operation from being scheduled above the asm block2317  %4 = load <8 x i64>, ptr %passthru2318  %5 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> %42319  ; add forces execution domain2320  %6 = add <8 x i64> %5, <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>2321  ret <8 x i64> %62322}2323 2324define <8 x i64> @stack_fold_permq_maskz(ptr %passthru, <8 x i64> %a0, i8 %mask) {2325; CHECK-LABEL: stack_fold_permq_maskz:2326; CHECK:       # %bb.0:2327; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2328; CHECK-NEXT:    #APP2329; CHECK-NEXT:    nop2330; CHECK-NEXT:    #NO_APP2331; CHECK-NEXT:    kmovd %esi, %k12332; CHECK-NEXT:    vpermq $235, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 64-byte Folded Reload2333; CHECK-NEXT:    # zmm0 {%k1} {z} = mem[3,2,2,3,7,6,6,7]2334; CHECK-NEXT:    retq2335  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2336  %2 = shufflevector <8 x i64> %a0, <8 x i64> undef, <8 x i32> <i32 3, i32 2, i32 2, i32 3, i32 7, i32 6, i32 6, i32 7>2337  %3 = bitcast i8 %mask to <8 x i1>2338  %4 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> zeroinitializer2339  ret <8 x i64> %42340}2341 2342define <8 x i64> @stack_fold_permqvar(<8 x i64> %a0, <8 x i64> %a1) {2343; CHECK-LABEL: stack_fold_permqvar:2344; CHECK:       # %bb.0:2345; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2346; CHECK-NEXT:    #APP2347; CHECK-NEXT:    nop2348; CHECK-NEXT:    #NO_APP2349; CHECK-NEXT:    vpermq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload2350; CHECK-NEXT:    vpternlogd {{.*#+}} zmm1 = -12351; CHECK-NEXT:    vpsubq %zmm1, %zmm0, %zmm02352; CHECK-NEXT:    retq2353  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2354  %2 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a1, <8 x i64> %a0)2355  ; add forces execution domain2356  %3 = add <8 x i64> %2, <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>2357  ret <8 x i64> %32358}2359declare <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64>, <8 x i64>) nounwind readonly2360 2361define <8 x i64> @stack_fold_permqvar_mask(ptr %passthru, <8 x i64> %a0, <8 x i64> %a1, i8 %mask) {2362; CHECK-LABEL: stack_fold_permqvar_mask:2363; CHECK:       # %bb.0:2364; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2365; CHECK-NEXT:    #APP2366; CHECK-NEXT:    nop2367; CHECK-NEXT:    #NO_APP2368; CHECK-NEXT:    kmovd %esi, %k12369; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm12370; CHECK-NEXT:    vpermq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 {%k1} # 64-byte Folded Reload2371; CHECK-NEXT:    vpternlogd {{.*#+}} zmm0 = -12372; CHECK-NEXT:    vpsubq %zmm0, %zmm1, %zmm02373; CHECK-NEXT:    retq2374  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2375  %2 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a1, <8 x i64> %a0)2376  %3 = bitcast i8 %mask to <8 x i1>2377  ; load needed to keep the operation from being scheduled above the asm block2378  %4 = load <8 x i64>, ptr %passthru2379  %5 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> %42380  ; add forces execution domain2381  %6 = add <8 x i64> %5, <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>2382  ret <8 x i64> %62383}2384 2385define <64 x i8> @stack_fold_vpermt2b(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2) {2386; CHECK-LABEL: stack_fold_vpermt2b:2387; CHECK:       # %bb.0:2388; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2389; CHECK-NEXT:    #APP2390; CHECK-NEXT:    nop2391; CHECK-NEXT:    #NO_APP2392; CHECK-NEXT:    vpermt2b {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload2393; CHECK-NEXT:    retq2394  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2395  %2 = call <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2)2396  ret <64 x i8> %22397}2398declare <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8>, <64 x i8>, <64 x i8>)2399 2400define <16 x i32> @stack_fold_vpermt2d(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) {2401; CHECK-LABEL: stack_fold_vpermt2d:2402; CHECK:       # %bb.0:2403; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2404; CHECK-NEXT:    #APP2405; CHECK-NEXT:    nop2406; CHECK-NEXT:    #NO_APP2407; CHECK-NEXT:    vpermt2d {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload2408; CHECK-NEXT:    retq2409  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2410  %2 = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)2411  ret <16 x i32> %22412}2413declare <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32>, <16 x i32>, <16 x i32>)2414 2415define <8 x i64> @stack_fold_vpermt2q(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2) {2416; CHECK-LABEL: stack_fold_vpermt2q:2417; CHECK:       # %bb.0:2418; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2419; CHECK-NEXT:    #APP2420; CHECK-NEXT:    nop2421; CHECK-NEXT:    #NO_APP2422; CHECK-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload2423; CHECK-NEXT:    retq2424  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2425  %2 = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)2426  ret <8 x i64> %22427}2428declare <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64>, <8 x i64>, <8 x i64>)2429 2430define <32 x i16> @stack_fold_vpermt2w(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2) {2431; CHECK-LABEL: stack_fold_vpermt2w:2432; CHECK:       # %bb.0:2433; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2434; CHECK-NEXT:    #APP2435; CHECK-NEXT:    nop2436; CHECK-NEXT:    #NO_APP2437; CHECK-NEXT:    vpermt2w {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload2438; CHECK-NEXT:    retq2439  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2440  %2 = call <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2)2441  ret <32 x i16> %22442}2443declare <32 x i16> @llvm.x86.avx512.vpermi2var.hi.512(<32 x i16>, <32 x i16>, <32 x i16>)2444 2445define <32 x i16> @stack_fold_permwvar(<32 x i16> %a0, <32 x i16> %a1) {2446; CHECK-LABEL: stack_fold_permwvar:2447; CHECK:       # %bb.0:2448; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2449; CHECK-NEXT:    #APP2450; CHECK-NEXT:    nop2451; CHECK-NEXT:    #NO_APP2452; CHECK-NEXT:    vpermw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload2453; CHECK-NEXT:    retq2454  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2455  %2 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a1, <32 x i16> %a0)2456  ret <32 x i16> %22457}2458declare <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16>, <32 x i16>) nounwind readonly2459 2460define <32 x i16> @stack_fold_permwvar_mask(ptr %passthru, <32 x i16> %a0, <32 x i16> %a1, i32 %mask) {2461; CHECK-LABEL: stack_fold_permwvar_mask:2462; CHECK:       # %bb.0:2463; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2464; CHECK-NEXT:    #APP2465; CHECK-NEXT:    nop2466; CHECK-NEXT:    #NO_APP2467; CHECK-NEXT:    kmovd %esi, %k12468; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm22469; CHECK-NEXT:    vpermw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload2470; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm02471; CHECK-NEXT:    retq2472  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2473  %2 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a1, <32 x i16> %a0)2474  %3 = bitcast i32 %mask to <32 x i1>2475  ; load needed to keep the operation from being scheduled above the asm block2476  %4 = load <32 x i16>, ptr %passthru2477  %5 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> %42478  ret <32 x i16> %52479}2480 2481define <32 x i16> @stack_fold_permwvar_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {2482; CHECK-LABEL: stack_fold_permwvar_maskz:2483; CHECK:       # %bb.0:2484; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2485; CHECK-NEXT:    #APP2486; CHECK-NEXT:    nop2487; CHECK-NEXT:    #NO_APP2488; CHECK-NEXT:    kmovd %edi, %k12489; CHECK-NEXT:    vpermw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload2490; CHECK-NEXT:    retq2491  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2492  %2 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a1, <32 x i16> %a0)2493  %3 = bitcast i32 %mask to <32 x i1>2494  %4 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> zeroinitializer2495  ret <32 x i16> %42496}2497 2498define i32 @stack_fold_pextrd(<4 x i32> %a0, <4 x i32> %a1) {2499; CHECK-LABEL: stack_fold_pextrd:2500; CHECK:       # %bb.0:2501; CHECK-NEXT:    pushq %rbp2502; CHECK-NEXT:    .cfi_def_cfa_offset 162503; CHECK-NEXT:    pushq %r152504; CHECK-NEXT:    .cfi_def_cfa_offset 242505; CHECK-NEXT:    pushq %r142506; CHECK-NEXT:    .cfi_def_cfa_offset 322507; CHECK-NEXT:    pushq %r132508; CHECK-NEXT:    .cfi_def_cfa_offset 402509; CHECK-NEXT:    pushq %r122510; CHECK-NEXT:    .cfi_def_cfa_offset 482511; CHECK-NEXT:    pushq %rbx2512; CHECK-NEXT:    .cfi_def_cfa_offset 562513; CHECK-NEXT:    .cfi_offset %rbx, -562514; CHECK-NEXT:    .cfi_offset %r12, -482515; CHECK-NEXT:    .cfi_offset %r13, -402516; CHECK-NEXT:    .cfi_offset %r14, -322517; CHECK-NEXT:    .cfi_offset %r15, -242518; CHECK-NEXT:    .cfi_offset %rbp, -162519; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm02520; CHECK-NEXT:    vpextrd $1, %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill2521; CHECK-NEXT:    #APP2522; CHECK-NEXT:    nop2523; CHECK-NEXT:    #NO_APP2524; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload2525; CHECK-NEXT:    popq %rbx2526; CHECK-NEXT:    .cfi_def_cfa_offset 482527; CHECK-NEXT:    popq %r122528; CHECK-NEXT:    .cfi_def_cfa_offset 402529; CHECK-NEXT:    popq %r132530; CHECK-NEXT:    .cfi_def_cfa_offset 322531; CHECK-NEXT:    popq %r142532; CHECK-NEXT:    .cfi_def_cfa_offset 242533; CHECK-NEXT:    popq %r152534; CHECK-NEXT:    .cfi_def_cfa_offset 162535; CHECK-NEXT:    popq %rbp2536; CHECK-NEXT:    .cfi_def_cfa_offset 82537; CHECK-NEXT:    retq2538  ; add forces execution domain2539  %1 = add <4 x i32> %a0, %a12540  %2 = extractelement <4 x i32> %1, i32 12541  %3 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()2542  ret i32 %22543}2544 2545define i64 @stack_fold_pextrq(<2 x i64> %a0) {2546; CHECK-LABEL: stack_fold_pextrq:2547; CHECK:       # %bb.0:2548; CHECK-NEXT:    pushq %rbp2549; CHECK-NEXT:    .cfi_def_cfa_offset 162550; CHECK-NEXT:    pushq %r152551; CHECK-NEXT:    .cfi_def_cfa_offset 242552; CHECK-NEXT:    pushq %r142553; CHECK-NEXT:    .cfi_def_cfa_offset 322554; CHECK-NEXT:    pushq %r132555; CHECK-NEXT:    .cfi_def_cfa_offset 402556; CHECK-NEXT:    pushq %r122557; CHECK-NEXT:    .cfi_def_cfa_offset 482558; CHECK-NEXT:    pushq %rbx2559; CHECK-NEXT:    .cfi_def_cfa_offset 562560; CHECK-NEXT:    .cfi_offset %rbx, -562561; CHECK-NEXT:    .cfi_offset %r12, -482562; CHECK-NEXT:    .cfi_offset %r13, -402563; CHECK-NEXT:    .cfi_offset %r14, -322564; CHECK-NEXT:    .cfi_offset %r15, -242565; CHECK-NEXT:    .cfi_offset %rbp, -162566; CHECK-NEXT:    vpextrq $1, %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill2567; CHECK-NEXT:    #APP2568; CHECK-NEXT:    nop2569; CHECK-NEXT:    #NO_APP2570; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload2571; CHECK-NEXT:    popq %rbx2572; CHECK-NEXT:    .cfi_def_cfa_offset 482573; CHECK-NEXT:    popq %r122574; CHECK-NEXT:    .cfi_def_cfa_offset 402575; CHECK-NEXT:    popq %r132576; CHECK-NEXT:    .cfi_def_cfa_offset 322577; CHECK-NEXT:    popq %r142578; CHECK-NEXT:    .cfi_def_cfa_offset 242579; CHECK-NEXT:    popq %r152580; CHECK-NEXT:    .cfi_def_cfa_offset 162581; CHECK-NEXT:    popq %rbp2582; CHECK-NEXT:    .cfi_def_cfa_offset 82583; CHECK-NEXT:    retq2584  %1 = extractelement <2 x i64> %a0, i32 12585  %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()2586  ret i64 %12587}2588 2589define <16 x i8> @stack_fold_pinsrb(<16 x i8> %a0, i8 %a1) {2590; CHECK-LABEL: stack_fold_pinsrb:2591; CHECK:       # %bb.0:2592; CHECK-NEXT:    pushq %rbp2593; CHECK-NEXT:    .cfi_def_cfa_offset 162594; CHECK-NEXT:    pushq %r152595; CHECK-NEXT:    .cfi_def_cfa_offset 242596; CHECK-NEXT:    pushq %r142597; CHECK-NEXT:    .cfi_def_cfa_offset 322598; CHECK-NEXT:    pushq %r132599; CHECK-NEXT:    .cfi_def_cfa_offset 402600; CHECK-NEXT:    pushq %r122601; CHECK-NEXT:    .cfi_def_cfa_offset 482602; CHECK-NEXT:    pushq %rbx2603; CHECK-NEXT:    .cfi_def_cfa_offset 562604; CHECK-NEXT:    .cfi_offset %rbx, -562605; CHECK-NEXT:    .cfi_offset %r12, -482606; CHECK-NEXT:    .cfi_offset %r13, -402607; CHECK-NEXT:    .cfi_offset %r14, -322608; CHECK-NEXT:    .cfi_offset %r15, -242609; CHECK-NEXT:    .cfi_offset %rbp, -162610; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill2611; CHECK-NEXT:    #APP2612; CHECK-NEXT:    nop2613; CHECK-NEXT:    #NO_APP2614; CHECK-NEXT:    vpinsrb $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload2615; CHECK-NEXT:    popq %rbx2616; CHECK-NEXT:    .cfi_def_cfa_offset 482617; CHECK-NEXT:    popq %r122618; CHECK-NEXT:    .cfi_def_cfa_offset 402619; CHECK-NEXT:    popq %r132620; CHECK-NEXT:    .cfi_def_cfa_offset 322621; CHECK-NEXT:    popq %r142622; CHECK-NEXT:    .cfi_def_cfa_offset 242623; CHECK-NEXT:    popq %r152624; CHECK-NEXT:    .cfi_def_cfa_offset 162625; CHECK-NEXT:    popq %rbp2626; CHECK-NEXT:    .cfi_def_cfa_offset 82627; CHECK-NEXT:    retq2628  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()2629  %2 = insertelement <16 x i8> %a0, i8 %a1, i32 12630  ret <16 x i8> %22631}2632 2633define <4 x i32> @stack_fold_pinsrd(<4 x i32> %a0, i32 %a1) {2634; CHECK-LABEL: stack_fold_pinsrd:2635; CHECK:       # %bb.0:2636; CHECK-NEXT:    pushq %rbp2637; CHECK-NEXT:    .cfi_def_cfa_offset 162638; CHECK-NEXT:    pushq %r152639; CHECK-NEXT:    .cfi_def_cfa_offset 242640; CHECK-NEXT:    pushq %r142641; CHECK-NEXT:    .cfi_def_cfa_offset 322642; CHECK-NEXT:    pushq %r132643; CHECK-NEXT:    .cfi_def_cfa_offset 402644; CHECK-NEXT:    pushq %r122645; CHECK-NEXT:    .cfi_def_cfa_offset 482646; CHECK-NEXT:    pushq %rbx2647; CHECK-NEXT:    .cfi_def_cfa_offset 562648; CHECK-NEXT:    .cfi_offset %rbx, -562649; CHECK-NEXT:    .cfi_offset %r12, -482650; CHECK-NEXT:    .cfi_offset %r13, -402651; CHECK-NEXT:    .cfi_offset %r14, -322652; CHECK-NEXT:    .cfi_offset %r15, -242653; CHECK-NEXT:    .cfi_offset %rbp, -162654; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill2655; CHECK-NEXT:    #APP2656; CHECK-NEXT:    nop2657; CHECK-NEXT:    #NO_APP2658; CHECK-NEXT:    vpinsrd $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload2659; CHECK-NEXT:    popq %rbx2660; CHECK-NEXT:    .cfi_def_cfa_offset 482661; CHECK-NEXT:    popq %r122662; CHECK-NEXT:    .cfi_def_cfa_offset 402663; CHECK-NEXT:    popq %r132664; CHECK-NEXT:    .cfi_def_cfa_offset 322665; CHECK-NEXT:    popq %r142666; CHECK-NEXT:    .cfi_def_cfa_offset 242667; CHECK-NEXT:    popq %r152668; CHECK-NEXT:    .cfi_def_cfa_offset 162669; CHECK-NEXT:    popq %rbp2670; CHECK-NEXT:    .cfi_def_cfa_offset 82671; CHECK-NEXT:    retq2672  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()2673  %2 = insertelement <4 x i32> %a0, i32 %a1, i32 12674  ret <4 x i32> %22675}2676 2677define <2 x i64> @stack_fold_pinsrq(<2 x i64> %a0, i64 %a1) {2678; CHECK-LABEL: stack_fold_pinsrq:2679; CHECK:       # %bb.0:2680; CHECK-NEXT:    pushq %rbp2681; CHECK-NEXT:    .cfi_def_cfa_offset 162682; CHECK-NEXT:    pushq %r152683; CHECK-NEXT:    .cfi_def_cfa_offset 242684; CHECK-NEXT:    pushq %r142685; CHECK-NEXT:    .cfi_def_cfa_offset 322686; CHECK-NEXT:    pushq %r132687; CHECK-NEXT:    .cfi_def_cfa_offset 402688; CHECK-NEXT:    pushq %r122689; CHECK-NEXT:    .cfi_def_cfa_offset 482690; CHECK-NEXT:    pushq %rbx2691; CHECK-NEXT:    .cfi_def_cfa_offset 562692; CHECK-NEXT:    .cfi_offset %rbx, -562693; CHECK-NEXT:    .cfi_offset %r12, -482694; CHECK-NEXT:    .cfi_offset %r13, -402695; CHECK-NEXT:    .cfi_offset %r14, -322696; CHECK-NEXT:    .cfi_offset %r15, -242697; CHECK-NEXT:    .cfi_offset %rbp, -162698; CHECK-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill2699; CHECK-NEXT:    #APP2700; CHECK-NEXT:    nop2701; CHECK-NEXT:    #NO_APP2702; CHECK-NEXT:    vpinsrq $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 8-byte Folded Reload2703; CHECK-NEXT:    popq %rbx2704; CHECK-NEXT:    .cfi_def_cfa_offset 482705; CHECK-NEXT:    popq %r122706; CHECK-NEXT:    .cfi_def_cfa_offset 402707; CHECK-NEXT:    popq %r132708; CHECK-NEXT:    .cfi_def_cfa_offset 322709; CHECK-NEXT:    popq %r142710; CHECK-NEXT:    .cfi_def_cfa_offset 242711; CHECK-NEXT:    popq %r152712; CHECK-NEXT:    .cfi_def_cfa_offset 162713; CHECK-NEXT:    popq %rbp2714; CHECK-NEXT:    .cfi_def_cfa_offset 82715; CHECK-NEXT:    retq2716  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()2717  %2 = insertelement <2 x i64> %a0, i64 %a1, i32 12718  ret <2 x i64> %22719}2720 2721define <8 x i16> @stack_fold_pinsrw(<8 x i16> %a0, i16 %a1) {2722; CHECK-LABEL: stack_fold_pinsrw:2723; CHECK:       # %bb.0:2724; CHECK-NEXT:    pushq %rbp2725; CHECK-NEXT:    .cfi_def_cfa_offset 162726; CHECK-NEXT:    pushq %r152727; CHECK-NEXT:    .cfi_def_cfa_offset 242728; CHECK-NEXT:    pushq %r142729; CHECK-NEXT:    .cfi_def_cfa_offset 322730; CHECK-NEXT:    pushq %r132731; CHECK-NEXT:    .cfi_def_cfa_offset 402732; CHECK-NEXT:    pushq %r122733; CHECK-NEXT:    .cfi_def_cfa_offset 482734; CHECK-NEXT:    pushq %rbx2735; CHECK-NEXT:    .cfi_def_cfa_offset 562736; CHECK-NEXT:    .cfi_offset %rbx, -562737; CHECK-NEXT:    .cfi_offset %r12, -482738; CHECK-NEXT:    .cfi_offset %r13, -402739; CHECK-NEXT:    .cfi_offset %r14, -322740; CHECK-NEXT:    .cfi_offset %r15, -242741; CHECK-NEXT:    .cfi_offset %rbp, -162742; CHECK-NEXT:    movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill2743; CHECK-NEXT:    #APP2744; CHECK-NEXT:    nop2745; CHECK-NEXT:    #NO_APP2746; CHECK-NEXT:    vpinsrw $1, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload2747; CHECK-NEXT:    popq %rbx2748; CHECK-NEXT:    .cfi_def_cfa_offset 482749; CHECK-NEXT:    popq %r122750; CHECK-NEXT:    .cfi_def_cfa_offset 402751; CHECK-NEXT:    popq %r132752; CHECK-NEXT:    .cfi_def_cfa_offset 322753; CHECK-NEXT:    popq %r142754; CHECK-NEXT:    .cfi_def_cfa_offset 242755; CHECK-NEXT:    popq %r152756; CHECK-NEXT:    .cfi_def_cfa_offset 162757; CHECK-NEXT:    popq %rbp2758; CHECK-NEXT:    .cfi_def_cfa_offset 82759; CHECK-NEXT:    retq2760  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()2761  %2 = insertelement <8 x i16> %a0, i16 %a1, i32 12762  ret <8 x i16> %22763}2764 2765define <16 x i32> @stack_fold_vplzcntd(<16 x i32> %a0) {2766; CHECK-LABEL: stack_fold_vplzcntd:2767; CHECK:       # %bb.0:2768; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2769; CHECK-NEXT:    #APP2770; CHECK-NEXT:    nop2771; CHECK-NEXT:    #NO_APP2772; CHECK-NEXT:    vplzcntd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload2773; CHECK-NEXT:    retq2774  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2775  %2 = call <16 x i32> @llvm.ctlz.v16i32(<16 x i32> %a0, i1 false)2776  ret <16 x i32> %22777}2778 2779define <8 x i64> @stack_fold_vplzcntq(<8 x i64> %a0) {2780; CHECK-LABEL: stack_fold_vplzcntq:2781; CHECK:       # %bb.0:2782; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2783; CHECK-NEXT:    #APP2784; CHECK-NEXT:    nop2785; CHECK-NEXT:    #NO_APP2786; CHECK-NEXT:    vplzcntq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload2787; CHECK-NEXT:    retq2788  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2789  %2 = call <8 x i64> @llvm.ctlz.v8i64(<8 x i64> %a0, i1 false)2790  ret <8 x i64> %22791}2792 2793define <32 x i16> @stack_fold_pmaddubsw_zmm(<64 x i8> %a0, <64 x i8> %a1) {2794; CHECK-LABEL: stack_fold_pmaddubsw_zmm:2795; CHECK:       # %bb.0:2796; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2797; CHECK-NEXT:    #APP2798; CHECK-NEXT:    nop2799; CHECK-NEXT:    #NO_APP2800; CHECK-NEXT:    vpmaddubsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload2801; CHECK-NEXT:    retq2802  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2803  %2 = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %a0, <64 x i8> %a1)2804  ret <32 x i16> %22805}2806declare <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8>, <64 x i8>) nounwind readnone2807 2808define <32 x i16> @stack_fold_pmaddubsw_zmm_mask(ptr %passthru, <64 x i8> %a0, <64 x i8> %a1, i32 %mask) {2809; CHECK-LABEL: stack_fold_pmaddubsw_zmm_mask:2810; CHECK:       # %bb.0:2811; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2812; CHECK-NEXT:    #APP2813; CHECK-NEXT:    nop2814; CHECK-NEXT:    #NO_APP2815; CHECK-NEXT:    kmovd %esi, %k12816; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm22817; CHECK-NEXT:    vpmaddubsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload2818; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm02819; CHECK-NEXT:    retq2820  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2821  %2 = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %a0, <64 x i8> %a1)2822  %3 = bitcast i32 %mask to <32 x i1>2823  ; load needed to keep the operation from being scheduled about the asm block2824  %4 = load <32 x i16>, ptr %passthru2825  %5 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> %42826  ret <32 x i16> %52827}2828 2829define <32 x i16> @stack_fold_pmaddubsw_zmm_maskz(<64 x i8> %a0, <64 x i8> %a1, i32 %mask) {2830; CHECK-LABEL: stack_fold_pmaddubsw_zmm_maskz:2831; CHECK:       # %bb.0:2832; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2833; CHECK-NEXT:    #APP2834; CHECK-NEXT:    nop2835; CHECK-NEXT:    #NO_APP2836; CHECK-NEXT:    kmovd %edi, %k12837; CHECK-NEXT:    vpmaddubsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload2838; CHECK-NEXT:    retq2839  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2840  %2 = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %a0, <64 x i8> %a1)2841  %3 = bitcast i32 %mask to <32 x i1>2842  %4 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> zeroinitializer2843  ret <32 x i16> %42844}2845 2846define <16 x i32> @stack_fold_pmaddwd_zmm(<32 x i16> %a0, <32 x i16> %a1) {2847; CHECK-LABEL: stack_fold_pmaddwd_zmm:2848; CHECK:       # %bb.0:2849; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2850; CHECK-NEXT:    #APP2851; CHECK-NEXT:    nop2852; CHECK-NEXT:    #NO_APP2853; CHECK-NEXT:    vpmaddwd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload2854; CHECK-NEXT:    retq2855  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2856  %2 = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> %a0, <32 x i16> %a1)2857  ret <16 x i32> %22858}2859declare <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16>, <32 x i16>) nounwind readnone2860 2861define <16 x i32> @stack_fold_pmaddwd_zmm_commuted(<32 x i16> %a0, <32 x i16> %a1) {2862; CHECK-LABEL: stack_fold_pmaddwd_zmm_commuted:2863; CHECK:       # %bb.0:2864; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2865; CHECK-NEXT:    #APP2866; CHECK-NEXT:    nop2867; CHECK-NEXT:    #NO_APP2868; CHECK-NEXT:    vpmaddwd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload2869; CHECK-NEXT:    retq2870  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2871  %2 = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> %a1, <32 x i16> %a0)2872  ret <16 x i32> %22873}2874 2875define <16 x i32> @stack_fold_pmaddwd_zmm_mask(ptr %passthru, <32 x i16> %a0, <32 x i16> %a1, i16 %mask) {2876; CHECK-LABEL: stack_fold_pmaddwd_zmm_mask:2877; CHECK:       # %bb.0:2878; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2879; CHECK-NEXT:    #APP2880; CHECK-NEXT:    nop2881; CHECK-NEXT:    #NO_APP2882; CHECK-NEXT:    kmovd %esi, %k12883; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm22884; CHECK-NEXT:    vpmaddwd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload2885; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm02886; CHECK-NEXT:    retq2887  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2888  %2 = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> %a0, <32 x i16> %a1)2889  %3 = bitcast i16 %mask to <16 x i1>2890  ; load needed to keep the operation from being scheduled about the asm block2891  %4 = load <16 x i32>, ptr %passthru2892  %5 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> %42893  ret <16 x i32> %52894}2895 2896define <16 x i32> @stack_fold_pmaddwd_zmm_mask_commuted(ptr %passthru, <32 x i16> %a0, <32 x i16> %a1, i16 %mask) {2897; CHECK-LABEL: stack_fold_pmaddwd_zmm_mask_commuted:2898; CHECK:       # %bb.0:2899; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2900; CHECK-NEXT:    #APP2901; CHECK-NEXT:    nop2902; CHECK-NEXT:    #NO_APP2903; CHECK-NEXT:    kmovd %esi, %k12904; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm22905; CHECK-NEXT:    vpmaddwd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload2906; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm02907; CHECK-NEXT:    retq2908  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2909  %2 = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> %a1, <32 x i16> %a0)2910  %3 = bitcast i16 %mask to <16 x i1>2911  ; load needed to keep the operation from being scheduled about the asm block2912  %4 = load <16 x i32>, ptr %passthru2913  %5 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> %42914  ret <16 x i32> %52915}2916 2917define <16 x i32> @stack_fold_pmaddwd_zmm_maskz(ptr %passthru, <32 x i16> %a0, <32 x i16> %a1, i16 %mask) {2918; CHECK-LABEL: stack_fold_pmaddwd_zmm_maskz:2919; CHECK:       # %bb.0:2920; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2921; CHECK-NEXT:    #APP2922; CHECK-NEXT:    nop2923; CHECK-NEXT:    #NO_APP2924; CHECK-NEXT:    kmovd %esi, %k12925; CHECK-NEXT:    vpmaddwd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload2926; CHECK-NEXT:    retq2927  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2928  %2 = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> %a0, <32 x i16> %a1)2929  %3 = bitcast i16 %mask to <16 x i1>2930  %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer2931  ret <16 x i32> %42932}2933 2934define <16 x i32> @stack_fold_pmaddwd_zmm_maskz_commuted(ptr %passthru, <32 x i16> %a0, <32 x i16> %a1, i16 %mask) {2935; CHECK-LABEL: stack_fold_pmaddwd_zmm_maskz_commuted:2936; CHECK:       # %bb.0:2937; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2938; CHECK-NEXT:    #APP2939; CHECK-NEXT:    nop2940; CHECK-NEXT:    #NO_APP2941; CHECK-NEXT:    kmovd %esi, %k12942; CHECK-NEXT:    vpmaddwd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload2943; CHECK-NEXT:    retq2944  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2945  %2 = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> %a1, <32 x i16> %a0)2946  %3 = bitcast i16 %mask to <16 x i1>2947  %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer2948  ret <16 x i32> %42949}2950 2951define <64 x i8> @stack_fold_pmaxsb(<64 x i8> %a0, <64 x i8> %a1) {2952; CHECK-LABEL: stack_fold_pmaxsb:2953; CHECK:       # %bb.0:2954; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2955; CHECK-NEXT:    #APP2956; CHECK-NEXT:    nop2957; CHECK-NEXT:    #NO_APP2958; CHECK-NEXT:    vpmaxsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload2959; CHECK-NEXT:    retq2960  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2961  %2 = icmp sgt <64 x i8> %a0, %a12962  %3 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %a12963  ret <64 x i8> %32964}2965 2966define <64 x i8> @stack_fold_pmaxsb_commuted(<64 x i8> %a0, <64 x i8> %a1) {2967; CHECK-LABEL: stack_fold_pmaxsb_commuted:2968; CHECK:       # %bb.0:2969; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2970; CHECK-NEXT:    #APP2971; CHECK-NEXT:    nop2972; CHECK-NEXT:    #NO_APP2973; CHECK-NEXT:    vpmaxsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload2974; CHECK-NEXT:    retq2975  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2976  %2 = icmp sgt <64 x i8> %a1, %a02977  %3 = select <64 x i1> %2, <64 x i8> %a1, <64 x i8> %a02978  ret <64 x i8> %32979}2980 2981define <64 x i8> @stack_fold_pmaxsb_mask(<64 x i8> %a0, <64 x i8> %a1, i64 %mask, ptr %passthru) {2982; CHECK-LABEL: stack_fold_pmaxsb_mask:2983; CHECK:       # %bb.0:2984; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill2985; CHECK-NEXT:    #APP2986; CHECK-NEXT:    nop2987; CHECK-NEXT:    #NO_APP2988; CHECK-NEXT:    kmovq %rdi, %k12989; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm22990; CHECK-NEXT:    vpmaxsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload2991; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm02992; CHECK-NEXT:    retq2993  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()2994  %2 = icmp sgt <64 x i8> %a0, %a12995  %3 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %a12996  %4 = bitcast i64 %mask to <64 x i1>2997  ; load needed to keep the operation from being scheduled about the asm block2998  %5 = load <64 x i8>, ptr %passthru2999  %6 = select <64 x i1> %4, <64 x i8> %3, <64 x i8> %53000  ret <64 x i8> %63001}3002 3003define <64 x i8> @stack_fold_pmaxsb_mask_commuted(<64 x i8> %a0, <64 x i8> %a1, i64 %mask, ptr %passthru) {3004; CHECK-LABEL: stack_fold_pmaxsb_mask_commuted:3005; CHECK:       # %bb.0:3006; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3007; CHECK-NEXT:    #APP3008; CHECK-NEXT:    nop3009; CHECK-NEXT:    #NO_APP3010; CHECK-NEXT:    kmovq %rdi, %k13011; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm23012; CHECK-NEXT:    vpmaxsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload3013; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm03014; CHECK-NEXT:    retq3015  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3016  %2 = icmp sgt <64 x i8> %a1, %a03017  %3 = select <64 x i1> %2, <64 x i8> %a1, <64 x i8> %a03018  %4 = bitcast i64 %mask to <64 x i1>3019  ; load needed to keep the operation from being scheduled about the asm block3020  %5 = load <64 x i8>, ptr %passthru3021  %6 = select <64 x i1> %4, <64 x i8> %3, <64 x i8> %53022  ret <64 x i8> %63023}3024 3025define <64 x i8> @stack_fold_pmaxsb_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {3026; CHECK-LABEL: stack_fold_pmaxsb_maskz:3027; CHECK:       # %bb.0:3028; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3029; CHECK-NEXT:    #APP3030; CHECK-NEXT:    nop3031; CHECK-NEXT:    #NO_APP3032; CHECK-NEXT:    kmovq %rdi, %k13033; CHECK-NEXT:    vpmaxsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload3034; CHECK-NEXT:    retq3035  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3036  %2 = icmp sgt <64 x i8> %a0, %a13037  %3 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %a13038  %4 = bitcast i64 %mask to <64 x i1>3039  %5 = select <64 x i1> %4, <64 x i8> %3, <64 x i8> zeroinitializer3040  ret <64 x i8> %53041}3042 3043define <64 x i8> @stack_fold_pmaxsb_maskz_commuted(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {3044; CHECK-LABEL: stack_fold_pmaxsb_maskz_commuted:3045; CHECK:       # %bb.0:3046; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3047; CHECK-NEXT:    #APP3048; CHECK-NEXT:    nop3049; CHECK-NEXT:    #NO_APP3050; CHECK-NEXT:    kmovq %rdi, %k13051; CHECK-NEXT:    vpmaxsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload3052; CHECK-NEXT:    retq3053  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3054  %2 = icmp sgt <64 x i8> %a1, %a03055  %3 = select <64 x i1> %2, <64 x i8> %a1, <64 x i8> %a03056  %4 = bitcast i64 %mask to <64 x i1>3057  %5 = select <64 x i1> %4, <64 x i8> %3, <64 x i8> zeroinitializer3058  ret <64 x i8> %53059}3060 3061define <16 x i32> @stack_fold_pmaxsd(<16 x i32> %a0, <16 x i32> %a1) {3062; CHECK-LABEL: stack_fold_pmaxsd:3063; CHECK:       # %bb.0:3064; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3065; CHECK-NEXT:    #APP3066; CHECK-NEXT:    nop3067; CHECK-NEXT:    #NO_APP3068; CHECK-NEXT:    vpmaxsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload3069; CHECK-NEXT:    retq3070  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3071  %2 = icmp sgt <16 x i32> %a0, %a13072  %3 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %a13073  ret <16 x i32> %33074}3075 3076define <16 x i32> @stack_fold_pmaxsd_commuted(<16 x i32> %a0, <16 x i32> %a1) {3077; CHECK-LABEL: stack_fold_pmaxsd_commuted:3078; CHECK:       # %bb.0:3079; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3080; CHECK-NEXT:    #APP3081; CHECK-NEXT:    nop3082; CHECK-NEXT:    #NO_APP3083; CHECK-NEXT:    vpmaxsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload3084; CHECK-NEXT:    retq3085  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3086  %2 = icmp sgt <16 x i32> %a1, %a03087  %3 = select <16 x i1> %2, <16 x i32> %a1, <16 x i32> %a03088  ret <16 x i32> %33089}3090 3091define <16 x i32> @stack_fold_pmaxsd_mask(<16 x i32> %a0, <16 x i32> %a1, i16 %mask, ptr %passthru) {3092; CHECK-LABEL: stack_fold_pmaxsd_mask:3093; CHECK:       # %bb.0:3094; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3095; CHECK-NEXT:    #APP3096; CHECK-NEXT:    nop3097; CHECK-NEXT:    #NO_APP3098; CHECK-NEXT:    kmovd %edi, %k13099; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm23100; CHECK-NEXT:    vpmaxsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload3101; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm03102; CHECK-NEXT:    retq3103  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3104  %2 = icmp sgt <16 x i32> %a0, %a13105  %3 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %a13106  %4 = bitcast i16 %mask to <16 x i1>3107  ; load needed to keep the operation from being scheduled about the asm block3108  %5 = load <16 x i32>, ptr %passthru3109  %6 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> %53110  ret <16 x i32> %63111}3112 3113define <16 x i32> @stack_fold_pmaxsd_mask_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask, ptr %passthru) {3114; CHECK-LABEL: stack_fold_pmaxsd_mask_commuted:3115; CHECK:       # %bb.0:3116; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3117; CHECK-NEXT:    #APP3118; CHECK-NEXT:    nop3119; CHECK-NEXT:    #NO_APP3120; CHECK-NEXT:    kmovd %edi, %k13121; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm23122; CHECK-NEXT:    vpmaxsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload3123; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm03124; CHECK-NEXT:    retq3125  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3126  %2 = icmp sgt <16 x i32> %a1, %a03127  %3 = select <16 x i1> %2, <16 x i32> %a1, <16 x i32> %a03128  %4 = bitcast i16 %mask to <16 x i1>3129  ; load needed to keep the operation from being scheduled about the asm block3130  %5 = load <16 x i32>, ptr %passthru3131  %6 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> %53132  ret <16 x i32> %63133}3134 3135define <16 x i32> @stack_fold_pmaxsd_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {3136; CHECK-LABEL: stack_fold_pmaxsd_maskz:3137; CHECK:       # %bb.0:3138; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3139; CHECK-NEXT:    #APP3140; CHECK-NEXT:    nop3141; CHECK-NEXT:    #NO_APP3142; CHECK-NEXT:    kmovd %edi, %k13143; CHECK-NEXT:    vpmaxsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload3144; CHECK-NEXT:    retq3145  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3146  %2 = icmp sgt <16 x i32> %a0, %a13147  %3 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %a13148  %4 = bitcast i16 %mask to <16 x i1>3149  %5 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> zeroinitializer3150  ret <16 x i32> %53151}3152 3153define <16 x i32> @stack_fold_pmaxsd_maskz_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {3154; CHECK-LABEL: stack_fold_pmaxsd_maskz_commuted:3155; CHECK:       # %bb.0:3156; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3157; CHECK-NEXT:    #APP3158; CHECK-NEXT:    nop3159; CHECK-NEXT:    #NO_APP3160; CHECK-NEXT:    kmovd %edi, %k13161; CHECK-NEXT:    vpmaxsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload3162; CHECK-NEXT:    retq3163  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3164  %2 = icmp sgt <16 x i32> %a1, %a03165  %3 = select <16 x i1> %2, <16 x i32> %a1, <16 x i32> %a03166  %4 = bitcast i16 %mask to <16 x i1>3167  %5 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> zeroinitializer3168  ret <16 x i32> %53169}3170 3171define <8 x i64> @stack_fold_pmaxsq(<8 x i64> %a0, <8 x i64> %a1) {3172; CHECK-LABEL: stack_fold_pmaxsq:3173; CHECK:       # %bb.0:3174; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3175; CHECK-NEXT:    #APP3176; CHECK-NEXT:    nop3177; CHECK-NEXT:    #NO_APP3178; CHECK-NEXT:    vpmaxsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload3179; CHECK-NEXT:    retq3180  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3181  %2 = icmp sgt <8 x i64> %a0, %a13182  %3 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %a13183  ret <8 x i64> %33184}3185 3186define <8 x i64> @stack_fold_pmaxsq_commuted(<8 x i64> %a0, <8 x i64> %a1) {3187; CHECK-LABEL: stack_fold_pmaxsq_commuted:3188; CHECK:       # %bb.0:3189; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3190; CHECK-NEXT:    #APP3191; CHECK-NEXT:    nop3192; CHECK-NEXT:    #NO_APP3193; CHECK-NEXT:    vpmaxsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload3194; CHECK-NEXT:    retq3195  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3196  %2 = icmp sgt <8 x i64> %a1, %a03197  %3 = select <8 x i1> %2, <8 x i64> %a1, <8 x i64> %a03198  ret <8 x i64> %33199}3200 3201define <8 x i64> @stack_fold_pmaxsq_mask(<8 x i64> %a0, <8 x i64> %a1, i8 %mask, ptr %passthru) {3202; CHECK-LABEL: stack_fold_pmaxsq_mask:3203; CHECK:       # %bb.0:3204; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3205; CHECK-NEXT:    #APP3206; CHECK-NEXT:    nop3207; CHECK-NEXT:    #NO_APP3208; CHECK-NEXT:    kmovd %edi, %k13209; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm23210; CHECK-NEXT:    vpmaxsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload3211; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm03212; CHECK-NEXT:    retq3213  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3214  %2 = icmp sgt <8 x i64> %a0, %a13215  %3 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %a13216  %4 = bitcast i8 %mask to <8 x i1>3217  ; load needed to keep the operation from being scheduled about the asm block3218  %5 = load <8 x i64>, ptr %passthru3219  %6 = select <8 x i1> %4, <8 x i64> %3, <8 x i64> %53220  ret <8 x i64> %63221}3222 3223define <8 x i64> @stack_fold_pmaxsq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask, ptr %passthru) {3224; CHECK-LABEL: stack_fold_pmaxsq_mask_commuted:3225; CHECK:       # %bb.0:3226; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3227; CHECK-NEXT:    #APP3228; CHECK-NEXT:    nop3229; CHECK-NEXT:    #NO_APP3230; CHECK-NEXT:    kmovd %edi, %k13231; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm23232; CHECK-NEXT:    vpmaxsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload3233; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm03234; CHECK-NEXT:    retq3235  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3236  %2 = icmp sgt <8 x i64> %a1, %a03237  %3 = select <8 x i1> %2, <8 x i64> %a1, <8 x i64> %a03238  %4 = bitcast i8 %mask to <8 x i1>3239  ; load needed to keep the operation from being scheduled about the asm block3240  %5 = load <8 x i64>, ptr %passthru3241  %6 = select <8 x i1> %4, <8 x i64> %3, <8 x i64> %53242  ret <8 x i64> %63243}3244 3245define <8 x i64> @stack_fold_pmaxsq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {3246; CHECK-LABEL: stack_fold_pmaxsq_maskz:3247; CHECK:       # %bb.0:3248; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3249; CHECK-NEXT:    #APP3250; CHECK-NEXT:    nop3251; CHECK-NEXT:    #NO_APP3252; CHECK-NEXT:    kmovd %edi, %k13253; CHECK-NEXT:    vpmaxsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload3254; CHECK-NEXT:    retq3255  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3256  %2 = icmp sgt <8 x i64> %a0, %a13257  %3 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %a13258  %4 = bitcast i8 %mask to <8 x i1>3259  %5 = select <8 x i1> %4, <8 x i64> %3, <8 x i64> zeroinitializer3260  ret <8 x i64> %53261}3262 3263define <8 x i64> @stack_fold_pmaxsq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {3264; CHECK-LABEL: stack_fold_pmaxsq_maskz_commuted:3265; CHECK:       # %bb.0:3266; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3267; CHECK-NEXT:    #APP3268; CHECK-NEXT:    nop3269; CHECK-NEXT:    #NO_APP3270; CHECK-NEXT:    kmovd %edi, %k13271; CHECK-NEXT:    vpmaxsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload3272; CHECK-NEXT:    retq3273  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3274  %2 = icmp sgt <8 x i64> %a1, %a03275  %3 = select <8 x i1> %2, <8 x i64> %a1, <8 x i64> %a03276  %4 = bitcast i8 %mask to <8 x i1>3277  %5 = select <8 x i1> %4, <8 x i64> %3, <8 x i64> zeroinitializer3278  ret <8 x i64> %53279}3280 3281define <32 x i16> @stack_fold_pmaxsw(<32 x i16> %a0, <32 x i16> %a1) {3282; CHECK-LABEL: stack_fold_pmaxsw:3283; CHECK:       # %bb.0:3284; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3285; CHECK-NEXT:    #APP3286; CHECK-NEXT:    nop3287; CHECK-NEXT:    #NO_APP3288; CHECK-NEXT:    vpmaxsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload3289; CHECK-NEXT:    retq3290  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3291  %2 = icmp sgt <32 x i16> %a0, %a13292  %3 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %a13293  ret <32 x i16> %33294}3295 3296define <32 x i16> @stack_fold_pmaxsw_commuted(<32 x i16> %a0, <32 x i16> %a1) {3297; CHECK-LABEL: stack_fold_pmaxsw_commuted:3298; CHECK:       # %bb.0:3299; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3300; CHECK-NEXT:    #APP3301; CHECK-NEXT:    nop3302; CHECK-NEXT:    #NO_APP3303; CHECK-NEXT:    vpmaxsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload3304; CHECK-NEXT:    retq3305  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3306  %2 = icmp sgt <32 x i16> %a1, %a03307  %3 = select <32 x i1> %2, <32 x i16> %a1, <32 x i16> %a03308  ret <32 x i16> %33309}3310 3311define <32 x i16> @stack_fold_pmaxsw_mask(<32 x i16> %a0, <32 x i16> %a1, i32 %mask, ptr %passthru) {3312; CHECK-LABEL: stack_fold_pmaxsw_mask:3313; CHECK:       # %bb.0:3314; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3315; CHECK-NEXT:    #APP3316; CHECK-NEXT:    nop3317; CHECK-NEXT:    #NO_APP3318; CHECK-NEXT:    kmovd %edi, %k13319; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm23320; CHECK-NEXT:    vpmaxsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload3321; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm03322; CHECK-NEXT:    retq3323  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3324  %2 = icmp sgt <32 x i16> %a0, %a13325  %3 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %a13326  %4 = bitcast i32 %mask to <32 x i1>3327  ; load needed to keep the operation from being scheduled about the asm block3328  %5 = load <32 x i16>, ptr %passthru3329  %6 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> %53330  ret <32 x i16> %63331}3332 3333define <32 x i16> @stack_fold_pmaxsw_mask_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask, ptr %passthru) {3334; CHECK-LABEL: stack_fold_pmaxsw_mask_commuted:3335; CHECK:       # %bb.0:3336; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3337; CHECK-NEXT:    #APP3338; CHECK-NEXT:    nop3339; CHECK-NEXT:    #NO_APP3340; CHECK-NEXT:    kmovd %edi, %k13341; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm23342; CHECK-NEXT:    vpmaxsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload3343; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm03344; CHECK-NEXT:    retq3345  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3346  %2 = icmp sgt <32 x i16> %a1, %a03347  %3 = select <32 x i1> %2, <32 x i16> %a1, <32 x i16> %a03348  %4 = bitcast i32 %mask to <32 x i1>3349  ; load needed to keep the operation from being scheduled about the asm block3350  %5 = load <32 x i16>, ptr %passthru3351  %6 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> %53352  ret <32 x i16> %63353}3354 3355define <32 x i16> @stack_fold_pmaxsw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {3356; CHECK-LABEL: stack_fold_pmaxsw_maskz:3357; CHECK:       # %bb.0:3358; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3359; CHECK-NEXT:    #APP3360; CHECK-NEXT:    nop3361; CHECK-NEXT:    #NO_APP3362; CHECK-NEXT:    kmovd %edi, %k13363; CHECK-NEXT:    vpmaxsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload3364; CHECK-NEXT:    retq3365  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3366  %2 = icmp sgt <32 x i16> %a0, %a13367  %3 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %a13368  %4 = bitcast i32 %mask to <32 x i1>3369  %5 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> zeroinitializer3370  ret <32 x i16> %53371}3372 3373define <32 x i16> @stack_fold_pmaxsw_maskz_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {3374; CHECK-LABEL: stack_fold_pmaxsw_maskz_commuted:3375; CHECK:       # %bb.0:3376; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3377; CHECK-NEXT:    #APP3378; CHECK-NEXT:    nop3379; CHECK-NEXT:    #NO_APP3380; CHECK-NEXT:    kmovd %edi, %k13381; CHECK-NEXT:    vpmaxsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload3382; CHECK-NEXT:    retq3383  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3384  %2 = icmp sgt <32 x i16> %a1, %a03385  %3 = select <32 x i1> %2, <32 x i16> %a1, <32 x i16> %a03386  %4 = bitcast i32 %mask to <32 x i1>3387  %5 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> zeroinitializer3388  ret <32 x i16> %53389}3390 3391define <64 x i8> @stack_fold_pmaxub(<64 x i8> %a0, <64 x i8> %a1) {3392; CHECK-LABEL: stack_fold_pmaxub:3393; CHECK:       # %bb.0:3394; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3395; CHECK-NEXT:    #APP3396; CHECK-NEXT:    nop3397; CHECK-NEXT:    #NO_APP3398; CHECK-NEXT:    vpmaxub {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload3399; CHECK-NEXT:    retq3400  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3401  %2 = icmp ugt <64 x i8> %a0, %a13402  %3 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %a13403  ret <64 x i8> %33404}3405 3406define <64 x i8> @stack_fold_pmaxub_commuted(<64 x i8> %a0, <64 x i8> %a1) {3407; CHECK-LABEL: stack_fold_pmaxub_commuted:3408; CHECK:       # %bb.0:3409; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3410; CHECK-NEXT:    #APP3411; CHECK-NEXT:    nop3412; CHECK-NEXT:    #NO_APP3413; CHECK-NEXT:    vpmaxub {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload3414; CHECK-NEXT:    retq3415  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3416  %2 = icmp ugt <64 x i8> %a1, %a03417  %3 = select <64 x i1> %2, <64 x i8> %a1, <64 x i8> %a03418  ret <64 x i8> %33419}3420 3421define <64 x i8> @stack_fold_pmaxub_mask(<64 x i8> %a0, <64 x i8> %a1, i64 %mask, ptr %passthru) {3422; CHECK-LABEL: stack_fold_pmaxub_mask:3423; CHECK:       # %bb.0:3424; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3425; CHECK-NEXT:    #APP3426; CHECK-NEXT:    nop3427; CHECK-NEXT:    #NO_APP3428; CHECK-NEXT:    kmovq %rdi, %k13429; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm23430; CHECK-NEXT:    vpmaxub {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload3431; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm03432; CHECK-NEXT:    retq3433  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3434  %2 = icmp ugt <64 x i8> %a0, %a13435  %3 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %a13436  %4 = bitcast i64 %mask to <64 x i1>3437  ; load needed to keep the operation from being scheduled about the asm block3438  %5 = load <64 x i8>, ptr %passthru3439  %6 = select <64 x i1> %4, <64 x i8> %3, <64 x i8> %53440  ret <64 x i8> %63441}3442 3443define <64 x i8> @stack_fold_pmaxub_mask_commuted(<64 x i8> %a0, <64 x i8> %a1, i64 %mask, ptr %passthru) {3444; CHECK-LABEL: stack_fold_pmaxub_mask_commuted:3445; CHECK:       # %bb.0:3446; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3447; CHECK-NEXT:    #APP3448; CHECK-NEXT:    nop3449; CHECK-NEXT:    #NO_APP3450; CHECK-NEXT:    kmovq %rdi, %k13451; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm23452; CHECK-NEXT:    vpmaxub {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload3453; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm03454; CHECK-NEXT:    retq3455  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3456  %2 = icmp ugt <64 x i8> %a1, %a03457  %3 = select <64 x i1> %2, <64 x i8> %a1, <64 x i8> %a03458  %4 = bitcast i64 %mask to <64 x i1>3459  ; load needed to keep the operation from being scheduled about the asm block3460  %5 = load <64 x i8>, ptr %passthru3461  %6 = select <64 x i1> %4, <64 x i8> %3, <64 x i8> %53462  ret <64 x i8> %63463}3464 3465define <64 x i8> @stack_fold_pmaxub_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {3466; CHECK-LABEL: stack_fold_pmaxub_maskz:3467; CHECK:       # %bb.0:3468; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3469; CHECK-NEXT:    #APP3470; CHECK-NEXT:    nop3471; CHECK-NEXT:    #NO_APP3472; CHECK-NEXT:    kmovq %rdi, %k13473; CHECK-NEXT:    vpmaxub {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload3474; CHECK-NEXT:    retq3475  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3476  %2 = icmp ugt <64 x i8> %a0, %a13477  %3 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %a13478  %4 = bitcast i64 %mask to <64 x i1>3479  %5 = select <64 x i1> %4, <64 x i8> %3, <64 x i8> zeroinitializer3480  ret <64 x i8> %53481}3482 3483define <64 x i8> @stack_fold_pmaxub_maskz_commuted(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {3484; CHECK-LABEL: stack_fold_pmaxub_maskz_commuted:3485; CHECK:       # %bb.0:3486; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3487; CHECK-NEXT:    #APP3488; CHECK-NEXT:    nop3489; CHECK-NEXT:    #NO_APP3490; CHECK-NEXT:    kmovq %rdi, %k13491; CHECK-NEXT:    vpmaxub {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload3492; CHECK-NEXT:    retq3493  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3494  %2 = icmp ugt <64 x i8> %a1, %a03495  %3 = select <64 x i1> %2, <64 x i8> %a1, <64 x i8> %a03496  %4 = bitcast i64 %mask to <64 x i1>3497  %5 = select <64 x i1> %4, <64 x i8> %3, <64 x i8> zeroinitializer3498  ret <64 x i8> %53499}3500 3501define <16 x i32> @stack_fold_pmaxud(<16 x i32> %a0, <16 x i32> %a1) {3502; CHECK-LABEL: stack_fold_pmaxud:3503; CHECK:       # %bb.0:3504; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3505; CHECK-NEXT:    #APP3506; CHECK-NEXT:    nop3507; CHECK-NEXT:    #NO_APP3508; CHECK-NEXT:    vpmaxud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload3509; CHECK-NEXT:    retq3510  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3511  %2 = icmp ugt <16 x i32> %a0, %a13512  %3 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %a13513  ret <16 x i32> %33514}3515 3516define <16 x i32> @stack_fold_pmaxud_commuted(<16 x i32> %a0, <16 x i32> %a1) {3517; CHECK-LABEL: stack_fold_pmaxud_commuted:3518; CHECK:       # %bb.0:3519; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3520; CHECK-NEXT:    #APP3521; CHECK-NEXT:    nop3522; CHECK-NEXT:    #NO_APP3523; CHECK-NEXT:    vpmaxud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload3524; CHECK-NEXT:    retq3525  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3526  %2 = icmp ugt <16 x i32> %a1, %a03527  %3 = select <16 x i1> %2, <16 x i32> %a1, <16 x i32> %a03528  ret <16 x i32> %33529}3530 3531define <16 x i32> @stack_fold_pmaxud_mask(<16 x i32> %a0, <16 x i32> %a1, i16 %mask, ptr %passthru) {3532; CHECK-LABEL: stack_fold_pmaxud_mask:3533; CHECK:       # %bb.0:3534; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3535; CHECK-NEXT:    #APP3536; CHECK-NEXT:    nop3537; CHECK-NEXT:    #NO_APP3538; CHECK-NEXT:    kmovd %edi, %k13539; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm23540; CHECK-NEXT:    vpmaxud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload3541; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm03542; CHECK-NEXT:    retq3543  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3544  %2 = icmp ugt <16 x i32> %a0, %a13545  %3 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %a13546  %4 = bitcast i16 %mask to <16 x i1>3547  ; load needed to keep the operation from being scheduled about the asm block3548  %5 = load <16 x i32>, ptr %passthru3549  %6 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> %53550  ret <16 x i32> %63551}3552 3553define <16 x i32> @stack_fold_pmaxud_mask_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask, ptr %passthru) {3554; CHECK-LABEL: stack_fold_pmaxud_mask_commuted:3555; CHECK:       # %bb.0:3556; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3557; CHECK-NEXT:    #APP3558; CHECK-NEXT:    nop3559; CHECK-NEXT:    #NO_APP3560; CHECK-NEXT:    kmovd %edi, %k13561; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm23562; CHECK-NEXT:    vpmaxud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload3563; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm03564; CHECK-NEXT:    retq3565  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3566  %2 = icmp ugt <16 x i32> %a1, %a03567  %3 = select <16 x i1> %2, <16 x i32> %a1, <16 x i32> %a03568  %4 = bitcast i16 %mask to <16 x i1>3569  ; load needed to keep the operation from being scheduled about the asm block3570  %5 = load <16 x i32>, ptr %passthru3571  %6 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> %53572  ret <16 x i32> %63573}3574 3575define <16 x i32> @stack_fold_pmaxud_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {3576; CHECK-LABEL: stack_fold_pmaxud_maskz:3577; CHECK:       # %bb.0:3578; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3579; CHECK-NEXT:    #APP3580; CHECK-NEXT:    nop3581; CHECK-NEXT:    #NO_APP3582; CHECK-NEXT:    kmovd %edi, %k13583; CHECK-NEXT:    vpmaxud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload3584; CHECK-NEXT:    retq3585  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3586  %2 = icmp ugt <16 x i32> %a0, %a13587  %3 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %a13588  %4 = bitcast i16 %mask to <16 x i1>3589  %5 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> zeroinitializer3590  ret <16 x i32> %53591}3592 3593define <16 x i32> @stack_fold_pmaxud_maskz_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {3594; CHECK-LABEL: stack_fold_pmaxud_maskz_commuted:3595; CHECK:       # %bb.0:3596; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3597; CHECK-NEXT:    #APP3598; CHECK-NEXT:    nop3599; CHECK-NEXT:    #NO_APP3600; CHECK-NEXT:    kmovd %edi, %k13601; CHECK-NEXT:    vpmaxud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload3602; CHECK-NEXT:    retq3603  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3604  %2 = icmp ugt <16 x i32> %a1, %a03605  %3 = select <16 x i1> %2, <16 x i32> %a1, <16 x i32> %a03606  %4 = bitcast i16 %mask to <16 x i1>3607  %5 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> zeroinitializer3608  ret <16 x i32> %53609}3610 3611define <8 x i64> @stack_fold_pmaxuq(<8 x i64> %a0, <8 x i64> %a1) {3612; CHECK-LABEL: stack_fold_pmaxuq:3613; CHECK:       # %bb.0:3614; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3615; CHECK-NEXT:    #APP3616; CHECK-NEXT:    nop3617; CHECK-NEXT:    #NO_APP3618; CHECK-NEXT:    vpmaxuq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload3619; CHECK-NEXT:    retq3620  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3621  %2 = icmp ugt <8 x i64> %a0, %a13622  %3 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %a13623  ret <8 x i64> %33624}3625 3626define <8 x i64> @stack_fold_pmaxuq_commuted(<8 x i64> %a0, <8 x i64> %a1) {3627; CHECK-LABEL: stack_fold_pmaxuq_commuted:3628; CHECK:       # %bb.0:3629; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3630; CHECK-NEXT:    #APP3631; CHECK-NEXT:    nop3632; CHECK-NEXT:    #NO_APP3633; CHECK-NEXT:    vpmaxuq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload3634; CHECK-NEXT:    retq3635  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3636  %2 = icmp ugt <8 x i64> %a1, %a03637  %3 = select <8 x i1> %2, <8 x i64> %a1, <8 x i64> %a03638  ret <8 x i64> %33639}3640 3641define <8 x i64> @stack_fold_pmaxuq_mask(<8 x i64> %a0, <8 x i64> %a1, i8 %mask, ptr %passthru) {3642; CHECK-LABEL: stack_fold_pmaxuq_mask:3643; CHECK:       # %bb.0:3644; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3645; CHECK-NEXT:    #APP3646; CHECK-NEXT:    nop3647; CHECK-NEXT:    #NO_APP3648; CHECK-NEXT:    kmovd %edi, %k13649; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm23650; CHECK-NEXT:    vpmaxuq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload3651; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm03652; CHECK-NEXT:    retq3653  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3654  %2 = icmp ugt <8 x i64> %a0, %a13655  %3 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %a13656  %4 = bitcast i8 %mask to <8 x i1>3657  ; load needed to keep the operation from being scheduled about the asm block3658  %5 = load <8 x i64>, ptr %passthru3659  %6 = select <8 x i1> %4, <8 x i64> %3, <8 x i64> %53660  ret <8 x i64> %63661}3662 3663define <8 x i64> @stack_fold_pmaxuq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask, ptr %passthru) {3664; CHECK-LABEL: stack_fold_pmaxuq_mask_commuted:3665; CHECK:       # %bb.0:3666; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3667; CHECK-NEXT:    #APP3668; CHECK-NEXT:    nop3669; CHECK-NEXT:    #NO_APP3670; CHECK-NEXT:    kmovd %edi, %k13671; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm23672; CHECK-NEXT:    vpmaxuq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload3673; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm03674; CHECK-NEXT:    retq3675  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3676  %2 = icmp ugt <8 x i64> %a1, %a03677  %3 = select <8 x i1> %2, <8 x i64> %a1, <8 x i64> %a03678  %4 = bitcast i8 %mask to <8 x i1>3679  ; load needed to keep the operation from being scheduled about the asm block3680  %5 = load <8 x i64>, ptr %passthru3681  %6 = select <8 x i1> %4, <8 x i64> %3, <8 x i64> %53682  ret <8 x i64> %63683}3684 3685define <8 x i64> @stack_fold_pmaxuq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {3686; CHECK-LABEL: stack_fold_pmaxuq_maskz:3687; CHECK:       # %bb.0:3688; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3689; CHECK-NEXT:    #APP3690; CHECK-NEXT:    nop3691; CHECK-NEXT:    #NO_APP3692; CHECK-NEXT:    kmovd %edi, %k13693; CHECK-NEXT:    vpmaxuq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload3694; CHECK-NEXT:    retq3695  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3696  %2 = icmp ugt <8 x i64> %a0, %a13697  %3 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %a13698  %4 = bitcast i8 %mask to <8 x i1>3699  %5 = select <8 x i1> %4, <8 x i64> %3, <8 x i64> zeroinitializer3700  ret <8 x i64> %53701}3702 3703define <8 x i64> @stack_fold_pmaxuq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {3704; CHECK-LABEL: stack_fold_pmaxuq_maskz_commuted:3705; CHECK:       # %bb.0:3706; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3707; CHECK-NEXT:    #APP3708; CHECK-NEXT:    nop3709; CHECK-NEXT:    #NO_APP3710; CHECK-NEXT:    kmovd %edi, %k13711; CHECK-NEXT:    vpmaxuq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload3712; CHECK-NEXT:    retq3713  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3714  %2 = icmp ugt <8 x i64> %a1, %a03715  %3 = select <8 x i1> %2, <8 x i64> %a1, <8 x i64> %a03716  %4 = bitcast i8 %mask to <8 x i1>3717  %5 = select <8 x i1> %4, <8 x i64> %3, <8 x i64> zeroinitializer3718  ret <8 x i64> %53719}3720 3721define <32 x i16> @stack_fold_pmaxuw(<32 x i16> %a0, <32 x i16> %a1) {3722; CHECK-LABEL: stack_fold_pmaxuw:3723; CHECK:       # %bb.0:3724; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3725; CHECK-NEXT:    #APP3726; CHECK-NEXT:    nop3727; CHECK-NEXT:    #NO_APP3728; CHECK-NEXT:    vpmaxuw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload3729; CHECK-NEXT:    retq3730  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3731  %2 = icmp ugt <32 x i16> %a0, %a13732  %3 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %a13733  ret <32 x i16> %33734}3735 3736define <32 x i16> @stack_fold_pmaxuw_commuted(<32 x i16> %a0, <32 x i16> %a1) {3737; CHECK-LABEL: stack_fold_pmaxuw_commuted:3738; CHECK:       # %bb.0:3739; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3740; CHECK-NEXT:    #APP3741; CHECK-NEXT:    nop3742; CHECK-NEXT:    #NO_APP3743; CHECK-NEXT:    vpmaxuw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload3744; CHECK-NEXT:    retq3745  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3746  %2 = icmp ugt <32 x i16> %a1, %a03747  %3 = select <32 x i1> %2, <32 x i16> %a1, <32 x i16> %a03748  ret <32 x i16> %33749}3750 3751define <32 x i16> @stack_fold_pmaxuw_mask(<32 x i16> %a0, <32 x i16> %a1, i32 %mask, ptr %passthru) {3752; CHECK-LABEL: stack_fold_pmaxuw_mask:3753; CHECK:       # %bb.0:3754; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3755; CHECK-NEXT:    #APP3756; CHECK-NEXT:    nop3757; CHECK-NEXT:    #NO_APP3758; CHECK-NEXT:    kmovd %edi, %k13759; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm23760; CHECK-NEXT:    vpmaxuw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload3761; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm03762; CHECK-NEXT:    retq3763  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3764  %2 = icmp ugt <32 x i16> %a0, %a13765  %3 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %a13766  %4 = bitcast i32 %mask to <32 x i1>3767  ; load needed to keep the operation from being scheduled about the asm block3768  %5 = load <32 x i16>, ptr %passthru3769  %6 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> %53770  ret <32 x i16> %63771}3772 3773define <32 x i16> @stack_fold_pmaxuw_mask_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask, ptr %passthru) {3774; CHECK-LABEL: stack_fold_pmaxuw_mask_commuted:3775; CHECK:       # %bb.0:3776; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3777; CHECK-NEXT:    #APP3778; CHECK-NEXT:    nop3779; CHECK-NEXT:    #NO_APP3780; CHECK-NEXT:    kmovd %edi, %k13781; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm23782; CHECK-NEXT:    vpmaxuw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload3783; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm03784; CHECK-NEXT:    retq3785  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3786  %2 = icmp ugt <32 x i16> %a1, %a03787  %3 = select <32 x i1> %2, <32 x i16> %a1, <32 x i16> %a03788  %4 = bitcast i32 %mask to <32 x i1>3789  ; load needed to keep the operation from being scheduled about the asm block3790  %5 = load <32 x i16>, ptr %passthru3791  %6 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> %53792  ret <32 x i16> %63793}3794 3795define <32 x i16> @stack_fold_pmaxuw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {3796; CHECK-LABEL: stack_fold_pmaxuw_maskz:3797; CHECK:       # %bb.0:3798; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3799; CHECK-NEXT:    #APP3800; CHECK-NEXT:    nop3801; CHECK-NEXT:    #NO_APP3802; CHECK-NEXT:    kmovd %edi, %k13803; CHECK-NEXT:    vpmaxuw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload3804; CHECK-NEXT:    retq3805  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3806  %2 = icmp ugt <32 x i16> %a0, %a13807  %3 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %a13808  %4 = bitcast i32 %mask to <32 x i1>3809  %5 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> zeroinitializer3810  ret <32 x i16> %53811}3812 3813define <32 x i16> @stack_fold_pmaxuw_maskz_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {3814; CHECK-LABEL: stack_fold_pmaxuw_maskz_commuted:3815; CHECK:       # %bb.0:3816; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3817; CHECK-NEXT:    #APP3818; CHECK-NEXT:    nop3819; CHECK-NEXT:    #NO_APP3820; CHECK-NEXT:    kmovd %edi, %k13821; CHECK-NEXT:    vpmaxuw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload3822; CHECK-NEXT:    retq3823  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3824  %2 = icmp ugt <32 x i16> %a1, %a03825  %3 = select <32 x i1> %2, <32 x i16> %a1, <32 x i16> %a03826  %4 = bitcast i32 %mask to <32 x i1>3827  %5 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> zeroinitializer3828  ret <32 x i16> %53829}3830 3831define <64 x i8> @stack_fold_pminsb(<64 x i8> %a0, <64 x i8> %a1) {3832; CHECK-LABEL: stack_fold_pminsb:3833; CHECK:       # %bb.0:3834; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3835; CHECK-NEXT:    #APP3836; CHECK-NEXT:    nop3837; CHECK-NEXT:    #NO_APP3838; CHECK-NEXT:    vpminsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload3839; CHECK-NEXT:    retq3840  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3841  %2 = icmp slt <64 x i8> %a0, %a13842  %3 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %a13843  ret <64 x i8> %33844}3845 3846define <64 x i8> @stack_fold_pminsb_commuted(<64 x i8> %a0, <64 x i8> %a1) {3847; CHECK-LABEL: stack_fold_pminsb_commuted:3848; CHECK:       # %bb.0:3849; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3850; CHECK-NEXT:    #APP3851; CHECK-NEXT:    nop3852; CHECK-NEXT:    #NO_APP3853; CHECK-NEXT:    vpminsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload3854; CHECK-NEXT:    retq3855  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3856  %2 = icmp slt <64 x i8> %a1, %a03857  %3 = select <64 x i1> %2, <64 x i8> %a1, <64 x i8> %a03858  ret <64 x i8> %33859}3860 3861define <64 x i8> @stack_fold_pminsb_mask(<64 x i8> %a0, <64 x i8> %a1, i64 %mask, ptr %passthru) {3862; CHECK-LABEL: stack_fold_pminsb_mask:3863; CHECK:       # %bb.0:3864; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3865; CHECK-NEXT:    #APP3866; CHECK-NEXT:    nop3867; CHECK-NEXT:    #NO_APP3868; CHECK-NEXT:    kmovq %rdi, %k13869; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm23870; CHECK-NEXT:    vpminsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload3871; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm03872; CHECK-NEXT:    retq3873  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3874  %2 = icmp slt <64 x i8> %a0, %a13875  %3 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %a13876  %4 = bitcast i64 %mask to <64 x i1>3877  ; load needed to keep the operation from being scheduled about the asm block3878  %5 = load <64 x i8>, ptr %passthru3879  %6 = select <64 x i1> %4, <64 x i8> %3, <64 x i8> %53880  ret <64 x i8> %63881}3882 3883define <64 x i8> @stack_fold_pminsb_mask_commuted(<64 x i8> %a0, <64 x i8> %a1, i64 %mask, ptr %passthru) {3884; CHECK-LABEL: stack_fold_pminsb_mask_commuted:3885; CHECK:       # %bb.0:3886; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3887; CHECK-NEXT:    #APP3888; CHECK-NEXT:    nop3889; CHECK-NEXT:    #NO_APP3890; CHECK-NEXT:    kmovq %rdi, %k13891; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm23892; CHECK-NEXT:    vpminsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload3893; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm03894; CHECK-NEXT:    retq3895  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3896  %2 = icmp slt <64 x i8> %a1, %a03897  %3 = select <64 x i1> %2, <64 x i8> %a1, <64 x i8> %a03898  %4 = bitcast i64 %mask to <64 x i1>3899  ; load needed to keep the operation from being scheduled about the asm block3900  %5 = load <64 x i8>, ptr %passthru3901  %6 = select <64 x i1> %4, <64 x i8> %3, <64 x i8> %53902  ret <64 x i8> %63903}3904 3905define <64 x i8> @stack_fold_pminsb_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {3906; CHECK-LABEL: stack_fold_pminsb_maskz:3907; CHECK:       # %bb.0:3908; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3909; CHECK-NEXT:    #APP3910; CHECK-NEXT:    nop3911; CHECK-NEXT:    #NO_APP3912; CHECK-NEXT:    kmovq %rdi, %k13913; CHECK-NEXT:    vpminsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload3914; CHECK-NEXT:    retq3915  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3916  %2 = icmp slt <64 x i8> %a0, %a13917  %3 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %a13918  %4 = bitcast i64 %mask to <64 x i1>3919  %5 = select <64 x i1> %4, <64 x i8> %3, <64 x i8> zeroinitializer3920  ret <64 x i8> %53921}3922 3923define <64 x i8> @stack_fold_pminsb_maskz_commuted(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {3924; CHECK-LABEL: stack_fold_pminsb_maskz_commuted:3925; CHECK:       # %bb.0:3926; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3927; CHECK-NEXT:    #APP3928; CHECK-NEXT:    nop3929; CHECK-NEXT:    #NO_APP3930; CHECK-NEXT:    kmovq %rdi, %k13931; CHECK-NEXT:    vpminsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload3932; CHECK-NEXT:    retq3933  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3934  %2 = icmp slt <64 x i8> %a1, %a03935  %3 = select <64 x i1> %2, <64 x i8> %a1, <64 x i8> %a03936  %4 = bitcast i64 %mask to <64 x i1>3937  %5 = select <64 x i1> %4, <64 x i8> %3, <64 x i8> zeroinitializer3938  ret <64 x i8> %53939}3940 3941define <16 x i32> @stack_fold_pminsd(<16 x i32> %a0, <16 x i32> %a1) {3942; CHECK-LABEL: stack_fold_pminsd:3943; CHECK:       # %bb.0:3944; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3945; CHECK-NEXT:    #APP3946; CHECK-NEXT:    nop3947; CHECK-NEXT:    #NO_APP3948; CHECK-NEXT:    vpminsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload3949; CHECK-NEXT:    retq3950  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3951  %2 = icmp slt <16 x i32> %a0, %a13952  %3 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %a13953  ret <16 x i32> %33954}3955 3956define <16 x i32> @stack_fold_pminsd_commuted(<16 x i32> %a0, <16 x i32> %a1) {3957; CHECK-LABEL: stack_fold_pminsd_commuted:3958; CHECK:       # %bb.0:3959; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3960; CHECK-NEXT:    #APP3961; CHECK-NEXT:    nop3962; CHECK-NEXT:    #NO_APP3963; CHECK-NEXT:    vpminsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload3964; CHECK-NEXT:    retq3965  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3966  %2 = icmp slt <16 x i32> %a1, %a03967  %3 = select <16 x i1> %2, <16 x i32> %a1, <16 x i32> %a03968  ret <16 x i32> %33969}3970 3971define <16 x i32> @stack_fold_pminsd_mask(<16 x i32> %a0, <16 x i32> %a1, i16 %mask, ptr %passthru) {3972; CHECK-LABEL: stack_fold_pminsd_mask:3973; CHECK:       # %bb.0:3974; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3975; CHECK-NEXT:    #APP3976; CHECK-NEXT:    nop3977; CHECK-NEXT:    #NO_APP3978; CHECK-NEXT:    kmovd %edi, %k13979; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm23980; CHECK-NEXT:    vpminsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload3981; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm03982; CHECK-NEXT:    retq3983  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()3984  %2 = icmp slt <16 x i32> %a0, %a13985  %3 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %a13986  %4 = bitcast i16 %mask to <16 x i1>3987  ; load needed to keep the operation from being scheduled about the asm block3988  %5 = load <16 x i32>, ptr %passthru3989  %6 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> %53990  ret <16 x i32> %63991}3992 3993define <16 x i32> @stack_fold_pminsd_mask_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask, ptr %passthru) {3994; CHECK-LABEL: stack_fold_pminsd_mask_commuted:3995; CHECK:       # %bb.0:3996; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill3997; CHECK-NEXT:    #APP3998; CHECK-NEXT:    nop3999; CHECK-NEXT:    #NO_APP4000; CHECK-NEXT:    kmovd %edi, %k14001; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm24002; CHECK-NEXT:    vpminsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload4003; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm04004; CHECK-NEXT:    retq4005  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4006  %2 = icmp slt <16 x i32> %a1, %a04007  %3 = select <16 x i1> %2, <16 x i32> %a1, <16 x i32> %a04008  %4 = bitcast i16 %mask to <16 x i1>4009  ; load needed to keep the operation from being scheduled about the asm block4010  %5 = load <16 x i32>, ptr %passthru4011  %6 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> %54012  ret <16 x i32> %64013}4014 4015define <16 x i32> @stack_fold_pminsd_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {4016; CHECK-LABEL: stack_fold_pminsd_maskz:4017; CHECK:       # %bb.0:4018; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4019; CHECK-NEXT:    #APP4020; CHECK-NEXT:    nop4021; CHECK-NEXT:    #NO_APP4022; CHECK-NEXT:    kmovd %edi, %k14023; CHECK-NEXT:    vpminsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload4024; CHECK-NEXT:    retq4025  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4026  %2 = icmp slt <16 x i32> %a0, %a14027  %3 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %a14028  %4 = bitcast i16 %mask to <16 x i1>4029  %5 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> zeroinitializer4030  ret <16 x i32> %54031}4032 4033define <16 x i32> @stack_fold_pminsd_maskz_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {4034; CHECK-LABEL: stack_fold_pminsd_maskz_commuted:4035; CHECK:       # %bb.0:4036; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4037; CHECK-NEXT:    #APP4038; CHECK-NEXT:    nop4039; CHECK-NEXT:    #NO_APP4040; CHECK-NEXT:    kmovd %edi, %k14041; CHECK-NEXT:    vpminsd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload4042; CHECK-NEXT:    retq4043  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4044  %2 = icmp slt <16 x i32> %a1, %a04045  %3 = select <16 x i1> %2, <16 x i32> %a1, <16 x i32> %a04046  %4 = bitcast i16 %mask to <16 x i1>4047  %5 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> zeroinitializer4048  ret <16 x i32> %54049}4050 4051define <8 x i64> @stack_fold_pminsq(<8 x i64> %a0, <8 x i64> %a1) {4052; CHECK-LABEL: stack_fold_pminsq:4053; CHECK:       # %bb.0:4054; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4055; CHECK-NEXT:    #APP4056; CHECK-NEXT:    nop4057; CHECK-NEXT:    #NO_APP4058; CHECK-NEXT:    vpminsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload4059; CHECK-NEXT:    retq4060  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4061  %2 = icmp slt <8 x i64> %a0, %a14062  %3 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %a14063  ret <8 x i64> %34064}4065 4066define <8 x i64> @stack_fold_pminsq_commuted(<8 x i64> %a0, <8 x i64> %a1) {4067; CHECK-LABEL: stack_fold_pminsq_commuted:4068; CHECK:       # %bb.0:4069; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4070; CHECK-NEXT:    #APP4071; CHECK-NEXT:    nop4072; CHECK-NEXT:    #NO_APP4073; CHECK-NEXT:    vpminsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload4074; CHECK-NEXT:    retq4075  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4076  %2 = icmp slt <8 x i64> %a1, %a04077  %3 = select <8 x i1> %2, <8 x i64> %a1, <8 x i64> %a04078  ret <8 x i64> %34079}4080 4081define <8 x i64> @stack_fold_pminsq_mask(<8 x i64> %a0, <8 x i64> %a1, i8 %mask, ptr %passthru) {4082; CHECK-LABEL: stack_fold_pminsq_mask:4083; CHECK:       # %bb.0:4084; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4085; CHECK-NEXT:    #APP4086; CHECK-NEXT:    nop4087; CHECK-NEXT:    #NO_APP4088; CHECK-NEXT:    kmovd %edi, %k14089; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm24090; CHECK-NEXT:    vpminsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload4091; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm04092; CHECK-NEXT:    retq4093  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4094  %2 = icmp slt <8 x i64> %a0, %a14095  %3 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %a14096  %4 = bitcast i8 %mask to <8 x i1>4097  ; load needed to keep the operation from being scheduled about the asm block4098  %5 = load <8 x i64>, ptr %passthru4099  %6 = select <8 x i1> %4, <8 x i64> %3, <8 x i64> %54100  ret <8 x i64> %64101}4102 4103define <8 x i64> @stack_fold_pminsq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask, ptr %passthru) {4104; CHECK-LABEL: stack_fold_pminsq_mask_commuted:4105; CHECK:       # %bb.0:4106; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4107; CHECK-NEXT:    #APP4108; CHECK-NEXT:    nop4109; CHECK-NEXT:    #NO_APP4110; CHECK-NEXT:    kmovd %edi, %k14111; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm24112; CHECK-NEXT:    vpminsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload4113; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm04114; CHECK-NEXT:    retq4115  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4116  %2 = icmp slt <8 x i64> %a1, %a04117  %3 = select <8 x i1> %2, <8 x i64> %a1, <8 x i64> %a04118  %4 = bitcast i8 %mask to <8 x i1>4119  ; load needed to keep the operation from being scheduled about the asm block4120  %5 = load <8 x i64>, ptr %passthru4121  %6 = select <8 x i1> %4, <8 x i64> %3, <8 x i64> %54122  ret <8 x i64> %64123}4124 4125define <8 x i64> @stack_fold_pminsq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {4126; CHECK-LABEL: stack_fold_pminsq_maskz:4127; CHECK:       # %bb.0:4128; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4129; CHECK-NEXT:    #APP4130; CHECK-NEXT:    nop4131; CHECK-NEXT:    #NO_APP4132; CHECK-NEXT:    kmovd %edi, %k14133; CHECK-NEXT:    vpminsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload4134; CHECK-NEXT:    retq4135  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4136  %2 = icmp slt <8 x i64> %a0, %a14137  %3 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %a14138  %4 = bitcast i8 %mask to <8 x i1>4139  %5 = select <8 x i1> %4, <8 x i64> %3, <8 x i64> zeroinitializer4140  ret <8 x i64> %54141}4142 4143define <8 x i64> @stack_fold_pminsq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {4144; CHECK-LABEL: stack_fold_pminsq_maskz_commuted:4145; CHECK:       # %bb.0:4146; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4147; CHECK-NEXT:    #APP4148; CHECK-NEXT:    nop4149; CHECK-NEXT:    #NO_APP4150; CHECK-NEXT:    kmovd %edi, %k14151; CHECK-NEXT:    vpminsq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload4152; CHECK-NEXT:    retq4153  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4154  %2 = icmp slt <8 x i64> %a1, %a04155  %3 = select <8 x i1> %2, <8 x i64> %a1, <8 x i64> %a04156  %4 = bitcast i8 %mask to <8 x i1>4157  %5 = select <8 x i1> %4, <8 x i64> %3, <8 x i64> zeroinitializer4158  ret <8 x i64> %54159}4160 4161define <32 x i16> @stack_fold_pminsw(<32 x i16> %a0, <32 x i16> %a1) {4162; CHECK-LABEL: stack_fold_pminsw:4163; CHECK:       # %bb.0:4164; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4165; CHECK-NEXT:    #APP4166; CHECK-NEXT:    nop4167; CHECK-NEXT:    #NO_APP4168; CHECK-NEXT:    vpminsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload4169; CHECK-NEXT:    retq4170  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4171  %2 = icmp slt <32 x i16> %a0, %a14172  %3 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %a14173  ret <32 x i16> %34174}4175 4176define <32 x i16> @stack_fold_pminsw_commuted(<32 x i16> %a0, <32 x i16> %a1) {4177; CHECK-LABEL: stack_fold_pminsw_commuted:4178; CHECK:       # %bb.0:4179; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4180; CHECK-NEXT:    #APP4181; CHECK-NEXT:    nop4182; CHECK-NEXT:    #NO_APP4183; CHECK-NEXT:    vpminsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload4184; CHECK-NEXT:    retq4185  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4186  %2 = icmp slt <32 x i16> %a1, %a04187  %3 = select <32 x i1> %2, <32 x i16> %a1, <32 x i16> %a04188  ret <32 x i16> %34189}4190 4191define <32 x i16> @stack_fold_pminsw_mask(<32 x i16> %a0, <32 x i16> %a1, i32 %mask, ptr %passthru) {4192; CHECK-LABEL: stack_fold_pminsw_mask:4193; CHECK:       # %bb.0:4194; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4195; CHECK-NEXT:    #APP4196; CHECK-NEXT:    nop4197; CHECK-NEXT:    #NO_APP4198; CHECK-NEXT:    kmovd %edi, %k14199; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm24200; CHECK-NEXT:    vpminsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload4201; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm04202; CHECK-NEXT:    retq4203  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4204  %2 = icmp slt <32 x i16> %a0, %a14205  %3 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %a14206  %4 = bitcast i32 %mask to <32 x i1>4207  ; load needed to keep the operation from being scheduled about the asm block4208  %5 = load <32 x i16>, ptr %passthru4209  %6 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> %54210  ret <32 x i16> %64211}4212 4213define <32 x i16> @stack_fold_pminsw_mask_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask, ptr %passthru) {4214; CHECK-LABEL: stack_fold_pminsw_mask_commuted:4215; CHECK:       # %bb.0:4216; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4217; CHECK-NEXT:    #APP4218; CHECK-NEXT:    nop4219; CHECK-NEXT:    #NO_APP4220; CHECK-NEXT:    kmovd %edi, %k14221; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm24222; CHECK-NEXT:    vpminsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload4223; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm04224; CHECK-NEXT:    retq4225  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4226  %2 = icmp slt <32 x i16> %a1, %a04227  %3 = select <32 x i1> %2, <32 x i16> %a1, <32 x i16> %a04228  %4 = bitcast i32 %mask to <32 x i1>4229  ; load needed to keep the operation from being scheduled about the asm block4230  %5 = load <32 x i16>, ptr %passthru4231  %6 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> %54232  ret <32 x i16> %64233}4234 4235define <32 x i16> @stack_fold_pminsw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {4236; CHECK-LABEL: stack_fold_pminsw_maskz:4237; CHECK:       # %bb.0:4238; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4239; CHECK-NEXT:    #APP4240; CHECK-NEXT:    nop4241; CHECK-NEXT:    #NO_APP4242; CHECK-NEXT:    kmovd %edi, %k14243; CHECK-NEXT:    vpminsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload4244; CHECK-NEXT:    retq4245  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4246  %2 = icmp slt <32 x i16> %a0, %a14247  %3 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %a14248  %4 = bitcast i32 %mask to <32 x i1>4249  %5 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> zeroinitializer4250  ret <32 x i16> %54251}4252 4253define <32 x i16> @stack_fold_pminsw_maskz_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {4254; CHECK-LABEL: stack_fold_pminsw_maskz_commuted:4255; CHECK:       # %bb.0:4256; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4257; CHECK-NEXT:    #APP4258; CHECK-NEXT:    nop4259; CHECK-NEXT:    #NO_APP4260; CHECK-NEXT:    kmovd %edi, %k14261; CHECK-NEXT:    vpminsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload4262; CHECK-NEXT:    retq4263  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4264  %2 = icmp slt <32 x i16> %a1, %a04265  %3 = select <32 x i1> %2, <32 x i16> %a1, <32 x i16> %a04266  %4 = bitcast i32 %mask to <32 x i1>4267  %5 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> zeroinitializer4268  ret <32 x i16> %54269}4270 4271define <64 x i8> @stack_fold_pminub(<64 x i8> %a0, <64 x i8> %a1) {4272; CHECK-LABEL: stack_fold_pminub:4273; CHECK:       # %bb.0:4274; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4275; CHECK-NEXT:    #APP4276; CHECK-NEXT:    nop4277; CHECK-NEXT:    #NO_APP4278; CHECK-NEXT:    vpminub {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload4279; CHECK-NEXT:    retq4280  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4281  %2 = icmp ult <64 x i8> %a0, %a14282  %3 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %a14283  ret <64 x i8> %34284}4285 4286define <64 x i8> @stack_fold_pminub_commuted(<64 x i8> %a0, <64 x i8> %a1) {4287; CHECK-LABEL: stack_fold_pminub_commuted:4288; CHECK:       # %bb.0:4289; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4290; CHECK-NEXT:    #APP4291; CHECK-NEXT:    nop4292; CHECK-NEXT:    #NO_APP4293; CHECK-NEXT:    vpminub {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload4294; CHECK-NEXT:    retq4295  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4296  %2 = icmp ult <64 x i8> %a1, %a04297  %3 = select <64 x i1> %2, <64 x i8> %a1, <64 x i8> %a04298  ret <64 x i8> %34299}4300 4301define <64 x i8> @stack_fold_pminub_mask(<64 x i8> %a0, <64 x i8> %a1, i64 %mask, ptr %passthru) {4302; CHECK-LABEL: stack_fold_pminub_mask:4303; CHECK:       # %bb.0:4304; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4305; CHECK-NEXT:    #APP4306; CHECK-NEXT:    nop4307; CHECK-NEXT:    #NO_APP4308; CHECK-NEXT:    kmovq %rdi, %k14309; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm24310; CHECK-NEXT:    vpminub {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload4311; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm04312; CHECK-NEXT:    retq4313  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4314  %2 = icmp ult <64 x i8> %a0, %a14315  %3 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %a14316  %4 = bitcast i64 %mask to <64 x i1>4317  ; load needed to keep the operation from being scheduled about the asm block4318  %5 = load <64 x i8>, ptr %passthru4319  %6 = select <64 x i1> %4, <64 x i8> %3, <64 x i8> %54320  ret <64 x i8> %64321}4322 4323define <64 x i8> @stack_fold_pminub_mask_commuted(<64 x i8> %a0, <64 x i8> %a1, i64 %mask, ptr %passthru) {4324; CHECK-LABEL: stack_fold_pminub_mask_commuted:4325; CHECK:       # %bb.0:4326; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4327; CHECK-NEXT:    #APP4328; CHECK-NEXT:    nop4329; CHECK-NEXT:    #NO_APP4330; CHECK-NEXT:    kmovq %rdi, %k14331; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm24332; CHECK-NEXT:    vpminub {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload4333; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm04334; CHECK-NEXT:    retq4335  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4336  %2 = icmp ult <64 x i8> %a1, %a04337  %3 = select <64 x i1> %2, <64 x i8> %a1, <64 x i8> %a04338  %4 = bitcast i64 %mask to <64 x i1>4339  ; load needed to keep the operation from being scheduled about the asm block4340  %5 = load <64 x i8>, ptr %passthru4341  %6 = select <64 x i1> %4, <64 x i8> %3, <64 x i8> %54342  ret <64 x i8> %64343}4344 4345define <64 x i8> @stack_fold_pminub_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {4346; CHECK-LABEL: stack_fold_pminub_maskz:4347; CHECK:       # %bb.0:4348; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4349; CHECK-NEXT:    #APP4350; CHECK-NEXT:    nop4351; CHECK-NEXT:    #NO_APP4352; CHECK-NEXT:    kmovq %rdi, %k14353; CHECK-NEXT:    vpminub {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload4354; CHECK-NEXT:    retq4355  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4356  %2 = icmp ult <64 x i8> %a0, %a14357  %3 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %a14358  %4 = bitcast i64 %mask to <64 x i1>4359  %5 = select <64 x i1> %4, <64 x i8> %3, <64 x i8> zeroinitializer4360  ret <64 x i8> %54361}4362 4363define <64 x i8> @stack_fold_pminub_maskz_commuted(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {4364; CHECK-LABEL: stack_fold_pminub_maskz_commuted:4365; CHECK:       # %bb.0:4366; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4367; CHECK-NEXT:    #APP4368; CHECK-NEXT:    nop4369; CHECK-NEXT:    #NO_APP4370; CHECK-NEXT:    kmovq %rdi, %k14371; CHECK-NEXT:    vpminub {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload4372; CHECK-NEXT:    retq4373  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4374  %2 = icmp ult <64 x i8> %a1, %a04375  %3 = select <64 x i1> %2, <64 x i8> %a1, <64 x i8> %a04376  %4 = bitcast i64 %mask to <64 x i1>4377  %5 = select <64 x i1> %4, <64 x i8> %3, <64 x i8> zeroinitializer4378  ret <64 x i8> %54379}4380 4381define <16 x i32> @stack_fold_pminud(<16 x i32> %a0, <16 x i32> %a1) {4382; CHECK-LABEL: stack_fold_pminud:4383; CHECK:       # %bb.0:4384; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4385; CHECK-NEXT:    #APP4386; CHECK-NEXT:    nop4387; CHECK-NEXT:    #NO_APP4388; CHECK-NEXT:    vpminud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload4389; CHECK-NEXT:    retq4390  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4391  %2 = icmp ult <16 x i32> %a0, %a14392  %3 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %a14393  ret <16 x i32> %34394}4395 4396define <16 x i32> @stack_fold_pminud_commuted(<16 x i32> %a0, <16 x i32> %a1) {4397; CHECK-LABEL: stack_fold_pminud_commuted:4398; CHECK:       # %bb.0:4399; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4400; CHECK-NEXT:    #APP4401; CHECK-NEXT:    nop4402; CHECK-NEXT:    #NO_APP4403; CHECK-NEXT:    vpminud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload4404; CHECK-NEXT:    retq4405  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4406  %2 = icmp ult <16 x i32> %a1, %a04407  %3 = select <16 x i1> %2, <16 x i32> %a1, <16 x i32> %a04408  ret <16 x i32> %34409}4410 4411define <16 x i32> @stack_fold_pminud_mask(<16 x i32> %a0, <16 x i32> %a1, i16 %mask, ptr %passthru) {4412; CHECK-LABEL: stack_fold_pminud_mask:4413; CHECK:       # %bb.0:4414; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4415; CHECK-NEXT:    #APP4416; CHECK-NEXT:    nop4417; CHECK-NEXT:    #NO_APP4418; CHECK-NEXT:    kmovd %edi, %k14419; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm24420; CHECK-NEXT:    vpminud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload4421; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm04422; CHECK-NEXT:    retq4423  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4424  %2 = icmp ult <16 x i32> %a0, %a14425  %3 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %a14426  %4 = bitcast i16 %mask to <16 x i1>4427  ; load needed to keep the operation from being scheduled about the asm block4428  %5 = load <16 x i32>, ptr %passthru4429  %6 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> %54430  ret <16 x i32> %64431}4432 4433define <16 x i32> @stack_fold_pminud_mask_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask, ptr %passthru) {4434; CHECK-LABEL: stack_fold_pminud_mask_commuted:4435; CHECK:       # %bb.0:4436; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4437; CHECK-NEXT:    #APP4438; CHECK-NEXT:    nop4439; CHECK-NEXT:    #NO_APP4440; CHECK-NEXT:    kmovd %edi, %k14441; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm24442; CHECK-NEXT:    vpminud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload4443; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm04444; CHECK-NEXT:    retq4445  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4446  %2 = icmp ult <16 x i32> %a1, %a04447  %3 = select <16 x i1> %2, <16 x i32> %a1, <16 x i32> %a04448  %4 = bitcast i16 %mask to <16 x i1>4449  ; load needed to keep the operation from being scheduled about the asm block4450  %5 = load <16 x i32>, ptr %passthru4451  %6 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> %54452  ret <16 x i32> %64453}4454 4455define <16 x i32> @stack_fold_pminud_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {4456; CHECK-LABEL: stack_fold_pminud_maskz:4457; CHECK:       # %bb.0:4458; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4459; CHECK-NEXT:    #APP4460; CHECK-NEXT:    nop4461; CHECK-NEXT:    #NO_APP4462; CHECK-NEXT:    kmovd %edi, %k14463; CHECK-NEXT:    vpminud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload4464; CHECK-NEXT:    retq4465  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4466  %2 = icmp ult <16 x i32> %a0, %a14467  %3 = select <16 x i1> %2, <16 x i32> %a0, <16 x i32> %a14468  %4 = bitcast i16 %mask to <16 x i1>4469  %5 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> zeroinitializer4470  ret <16 x i32> %54471}4472 4473define <16 x i32> @stack_fold_pminud_maskz_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {4474; CHECK-LABEL: stack_fold_pminud_maskz_commuted:4475; CHECK:       # %bb.0:4476; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4477; CHECK-NEXT:    #APP4478; CHECK-NEXT:    nop4479; CHECK-NEXT:    #NO_APP4480; CHECK-NEXT:    kmovd %edi, %k14481; CHECK-NEXT:    vpminud {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload4482; CHECK-NEXT:    retq4483  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4484  %2 = icmp ult <16 x i32> %a1, %a04485  %3 = select <16 x i1> %2, <16 x i32> %a1, <16 x i32> %a04486  %4 = bitcast i16 %mask to <16 x i1>4487  %5 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> zeroinitializer4488  ret <16 x i32> %54489}4490 4491define <8 x i64> @stack_fold_pminuq(<8 x i64> %a0, <8 x i64> %a1) {4492; CHECK-LABEL: stack_fold_pminuq:4493; CHECK:       # %bb.0:4494; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4495; CHECK-NEXT:    #APP4496; CHECK-NEXT:    nop4497; CHECK-NEXT:    #NO_APP4498; CHECK-NEXT:    vpminuq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload4499; CHECK-NEXT:    retq4500  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4501  %2 = icmp ult <8 x i64> %a0, %a14502  %3 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %a14503  ret <8 x i64> %34504}4505 4506define <8 x i64> @stack_fold_pminuq_commuted(<8 x i64> %a0, <8 x i64> %a1) {4507; CHECK-LABEL: stack_fold_pminuq_commuted:4508; CHECK:       # %bb.0:4509; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4510; CHECK-NEXT:    #APP4511; CHECK-NEXT:    nop4512; CHECK-NEXT:    #NO_APP4513; CHECK-NEXT:    vpminuq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload4514; CHECK-NEXT:    retq4515  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4516  %2 = icmp ult <8 x i64> %a1, %a04517  %3 = select <8 x i1> %2, <8 x i64> %a1, <8 x i64> %a04518  ret <8 x i64> %34519}4520 4521define <8 x i64> @stack_fold_pminuq_mask(<8 x i64> %a0, <8 x i64> %a1, i8 %mask, ptr %passthru) {4522; CHECK-LABEL: stack_fold_pminuq_mask:4523; CHECK:       # %bb.0:4524; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4525; CHECK-NEXT:    #APP4526; CHECK-NEXT:    nop4527; CHECK-NEXT:    #NO_APP4528; CHECK-NEXT:    kmovd %edi, %k14529; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm24530; CHECK-NEXT:    vpminuq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload4531; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm04532; CHECK-NEXT:    retq4533  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4534  %2 = icmp ult <8 x i64> %a0, %a14535  %3 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %a14536  %4 = bitcast i8 %mask to <8 x i1>4537  ; load needed to keep the operation from being scheduled about the asm block4538  %5 = load <8 x i64>, ptr %passthru4539  %6 = select <8 x i1> %4, <8 x i64> %3, <8 x i64> %54540  ret <8 x i64> %64541}4542 4543define <8 x i64> @stack_fold_pminuq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask, ptr %passthru) {4544; CHECK-LABEL: stack_fold_pminuq_mask_commuted:4545; CHECK:       # %bb.0:4546; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4547; CHECK-NEXT:    #APP4548; CHECK-NEXT:    nop4549; CHECK-NEXT:    #NO_APP4550; CHECK-NEXT:    kmovd %edi, %k14551; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm24552; CHECK-NEXT:    vpminuq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload4553; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm04554; CHECK-NEXT:    retq4555  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4556  %2 = icmp ult <8 x i64> %a1, %a04557  %3 = select <8 x i1> %2, <8 x i64> %a1, <8 x i64> %a04558  %4 = bitcast i8 %mask to <8 x i1>4559  ; load needed to keep the operation from being scheduled about the asm block4560  %5 = load <8 x i64>, ptr %passthru4561  %6 = select <8 x i1> %4, <8 x i64> %3, <8 x i64> %54562  ret <8 x i64> %64563}4564 4565define <8 x i64> @stack_fold_pminuq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {4566; CHECK-LABEL: stack_fold_pminuq_maskz:4567; CHECK:       # %bb.0:4568; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4569; CHECK-NEXT:    #APP4570; CHECK-NEXT:    nop4571; CHECK-NEXT:    #NO_APP4572; CHECK-NEXT:    kmovd %edi, %k14573; CHECK-NEXT:    vpminuq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload4574; CHECK-NEXT:    retq4575  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4576  %2 = icmp ult <8 x i64> %a0, %a14577  %3 = select <8 x i1> %2, <8 x i64> %a0, <8 x i64> %a14578  %4 = bitcast i8 %mask to <8 x i1>4579  %5 = select <8 x i1> %4, <8 x i64> %3, <8 x i64> zeroinitializer4580  ret <8 x i64> %54581}4582 4583define <8 x i64> @stack_fold_pminuq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {4584; CHECK-LABEL: stack_fold_pminuq_maskz_commuted:4585; CHECK:       # %bb.0:4586; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4587; CHECK-NEXT:    #APP4588; CHECK-NEXT:    nop4589; CHECK-NEXT:    #NO_APP4590; CHECK-NEXT:    kmovd %edi, %k14591; CHECK-NEXT:    vpminuq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload4592; CHECK-NEXT:    retq4593  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4594  %2 = icmp ult <8 x i64> %a1, %a04595  %3 = select <8 x i1> %2, <8 x i64> %a1, <8 x i64> %a04596  %4 = bitcast i8 %mask to <8 x i1>4597  %5 = select <8 x i1> %4, <8 x i64> %3, <8 x i64> zeroinitializer4598  ret <8 x i64> %54599}4600 4601define <32 x i16> @stack_fold_pminuw(<32 x i16> %a0, <32 x i16> %a1) {4602; CHECK-LABEL: stack_fold_pminuw:4603; CHECK:       # %bb.0:4604; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4605; CHECK-NEXT:    #APP4606; CHECK-NEXT:    nop4607; CHECK-NEXT:    #NO_APP4608; CHECK-NEXT:    vpminuw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload4609; CHECK-NEXT:    retq4610  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4611  %2 = icmp ult <32 x i16> %a0, %a14612  %3 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %a14613  ret <32 x i16> %34614}4615 4616define <32 x i16> @stack_fold_pminuw_commuted(<32 x i16> %a0, <32 x i16> %a1) {4617; CHECK-LABEL: stack_fold_pminuw_commuted:4618; CHECK:       # %bb.0:4619; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4620; CHECK-NEXT:    #APP4621; CHECK-NEXT:    nop4622; CHECK-NEXT:    #NO_APP4623; CHECK-NEXT:    vpminuw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload4624; CHECK-NEXT:    retq4625  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4626  %2 = icmp ult <32 x i16> %a1, %a04627  %3 = select <32 x i1> %2, <32 x i16> %a1, <32 x i16> %a04628  ret <32 x i16> %34629}4630 4631define <32 x i16> @stack_fold_pminuw_mask(<32 x i16> %a0, <32 x i16> %a1, i32 %mask, ptr %passthru) {4632; CHECK-LABEL: stack_fold_pminuw_mask:4633; CHECK:       # %bb.0:4634; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4635; CHECK-NEXT:    #APP4636; CHECK-NEXT:    nop4637; CHECK-NEXT:    #NO_APP4638; CHECK-NEXT:    kmovd %edi, %k14639; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm24640; CHECK-NEXT:    vpminuw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload4641; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm04642; CHECK-NEXT:    retq4643  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4644  %2 = icmp ult <32 x i16> %a0, %a14645  %3 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %a14646  %4 = bitcast i32 %mask to <32 x i1>4647  ; load needed to keep the operation from being scheduled about the asm block4648  %5 = load <32 x i16>, ptr %passthru4649  %6 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> %54650  ret <32 x i16> %64651}4652 4653define <32 x i16> @stack_fold_pminuw_mask_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask, ptr %passthru) {4654; CHECK-LABEL: stack_fold_pminuw_mask_commuted:4655; CHECK:       # %bb.0:4656; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4657; CHECK-NEXT:    #APP4658; CHECK-NEXT:    nop4659; CHECK-NEXT:    #NO_APP4660; CHECK-NEXT:    kmovd %edi, %k14661; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm24662; CHECK-NEXT:    vpminuw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload4663; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm04664; CHECK-NEXT:    retq4665  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4666  %2 = icmp ult <32 x i16> %a1, %a04667  %3 = select <32 x i1> %2, <32 x i16> %a1, <32 x i16> %a04668  %4 = bitcast i32 %mask to <32 x i1>4669  ; load needed to keep the operation from being scheduled about the asm block4670  %5 = load <32 x i16>, ptr %passthru4671  %6 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> %54672  ret <32 x i16> %64673}4674 4675define <32 x i16> @stack_fold_pminuw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {4676; CHECK-LABEL: stack_fold_pminuw_maskz:4677; CHECK:       # %bb.0:4678; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4679; CHECK-NEXT:    #APP4680; CHECK-NEXT:    nop4681; CHECK-NEXT:    #NO_APP4682; CHECK-NEXT:    kmovd %edi, %k14683; CHECK-NEXT:    vpminuw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload4684; CHECK-NEXT:    retq4685  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4686  %2 = icmp ult <32 x i16> %a0, %a14687  %3 = select <32 x i1> %2, <32 x i16> %a0, <32 x i16> %a14688  %4 = bitcast i32 %mask to <32 x i1>4689  %5 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> zeroinitializer4690  ret <32 x i16> %54691}4692 4693define <32 x i16> @stack_fold_pminuw_maskz_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {4694; CHECK-LABEL: stack_fold_pminuw_maskz_commuted:4695; CHECK:       # %bb.0:4696; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill4697; CHECK-NEXT:    #APP4698; CHECK-NEXT:    nop4699; CHECK-NEXT:    #NO_APP4700; CHECK-NEXT:    kmovd %edi, %k14701; CHECK-NEXT:    vpminuw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload4702; CHECK-NEXT:    retq4703  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4704  %2 = icmp ult <32 x i16> %a1, %a04705  %3 = select <32 x i1> %2, <32 x i16> %a1, <32 x i16> %a04706  %4 = bitcast i32 %mask to <32 x i1>4707  %5 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> zeroinitializer4708  ret <32 x i16> %54709}4710 4711define <16 x i8> @stack_fold_vpmovdb(<16 x i32> %a0) {4712; CHECK-LABEL: stack_fold_vpmovdb:4713; CHECK:       # %bb.0:4714; CHECK-NEXT:    vpmovdb %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill4715; CHECK-NEXT:    #APP4716; CHECK-NEXT:    nop4717; CHECK-NEXT:    #NO_APP4718; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload4719; CHECK-NEXT:    vzeroupper4720; CHECK-NEXT:    retq4721  %1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.512(<16 x i32> %a0, <16 x i8> undef, i16 -1)4722  %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4723  ret <16 x i8> %14724}4725declare <16 x i8> @llvm.x86.avx512.mask.pmov.db.512(<16 x i32>, <16 x i8>, i16)4726 4727define <16 x i16> @stack_fold_vpmovdw(<16 x i32> %a0) {4728; CHECK-LABEL: stack_fold_vpmovdw:4729; CHECK:       # %bb.0:4730; CHECK-NEXT:    vpmovdw %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill4731; CHECK-NEXT:    #APP4732; CHECK-NEXT:    nop4733; CHECK-NEXT:    #NO_APP4734; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload4735; CHECK-NEXT:    retq4736  %1 = call <16 x i16> @llvm.x86.avx512.mask.pmov.dw.512(<16 x i32> %a0, <16 x i16> undef, i16 -1)4737  %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4738  ret <16 x i16> %14739}4740declare <16 x i16> @llvm.x86.avx512.mask.pmov.dw.512(<16 x i32>, <16 x i16>, i16)4741 4742define <2 x i64> @stack_fold_movq_load(<2 x i64> %a0) {4743; CHECK-LABEL: stack_fold_movq_load:4744; CHECK:       # %bb.0:4745; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4746; CHECK-NEXT:    #APP4747; CHECK-NEXT:    nop4748; CHECK-NEXT:    #NO_APP4749; CHECK-NEXT:    vmovq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload4750; CHECK-NEXT:    # xmm0 = mem[0],zero4751; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm14752; CHECK-NEXT:    vpsubq %xmm1, %xmm0, %xmm04753; CHECK-NEXT:    retq4754  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4755  %2 = shufflevector <2 x i64> %a0, <2 x i64> zeroinitializer, <2 x i32> <i32 0, i32 2>4756  ; add forces execution domain4757  %3 = add <2 x i64> %2, <i64 1, i64 1>4758  ret <2 x i64> %34759}4760 4761define <8 x i32> @stack_fold_vpmovqd(<8 x i64> %a0) {4762; CHECK-LABEL: stack_fold_vpmovqd:4763; CHECK:       # %bb.0:4764; CHECK-NEXT:    vpmovqd %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill4765; CHECK-NEXT:    #APP4766; CHECK-NEXT:    nop4767; CHECK-NEXT:    #NO_APP4768; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload4769; CHECK-NEXT:    retq4770  %1 = trunc <8 x i64> %a0 to <8 x i32>4771  %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4772  ret <8 x i32> %14773}4774declare <8 x i32> @llvm.x86.avx512.mask.pmov.qd.512(<8 x i64>, <8 x i32>, i8)4775 4776define <8 x i16> @stack_fold_vpmovqw(<8 x i64> %a0) {4777; CHECK-LABEL: stack_fold_vpmovqw:4778; CHECK:       # %bb.0:4779; CHECK-NEXT:    vpmovqw %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill4780; CHECK-NEXT:    #APP4781; CHECK-NEXT:    nop4782; CHECK-NEXT:    #NO_APP4783; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload4784; CHECK-NEXT:    vzeroupper4785; CHECK-NEXT:    retq4786  %1 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64> %a0, <8 x i16> undef, i8 -1)4787  %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4788  ret <8 x i16> %14789}4790declare <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64>, <8 x i16>, i8)4791 4792define <32 x i8> @stack_fold_vpmovwb(<32 x i16> %a0) {4793; CHECK-LABEL: stack_fold_vpmovwb:4794; CHECK:       # %bb.0:4795; CHECK-NEXT:    vpmovwb %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill4796; CHECK-NEXT:    #APP4797; CHECK-NEXT:    nop4798; CHECK-NEXT:    #NO_APP4799; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload4800; CHECK-NEXT:    retq4801  %1 = trunc <32 x i16> %a0 to <32 x i8>4802  %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4803  ret <32 x i8> %14804}4805declare <32 x i8> @llvm.x86.avx512.mask.pmov.wb.512(<32 x i16>, <32 x i8>, i32)4806 4807define <16 x i8> @stack_fold_vpmovsdb(<16 x i32> %a0) {4808; CHECK-LABEL: stack_fold_vpmovsdb:4809; CHECK:       # %bb.0:4810; CHECK-NEXT:    vpmovsdb %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill4811; CHECK-NEXT:    #APP4812; CHECK-NEXT:    nop4813; CHECK-NEXT:    #NO_APP4814; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload4815; CHECK-NEXT:    vzeroupper4816; CHECK-NEXT:    retq4817  %1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512(<16 x i32> %a0, <16 x i8> undef, i16 -1)4818  %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4819  ret <16 x i8> %14820}4821declare <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512(<16 x i32>, <16 x i8>, i16)4822 4823define <16 x i16> @stack_fold_vpmovsdw(<16 x i32> %a0) {4824; CHECK-LABEL: stack_fold_vpmovsdw:4825; CHECK:       # %bb.0:4826; CHECK-NEXT:    vpmovsdw %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill4827; CHECK-NEXT:    #APP4828; CHECK-NEXT:    nop4829; CHECK-NEXT:    #NO_APP4830; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload4831; CHECK-NEXT:    retq4832  %1 = call <16 x i16> @llvm.x86.avx512.mask.pmovs.dw.512(<16 x i32> %a0, <16 x i16> undef, i16 -1)4833  %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4834  ret <16 x i16> %14835}4836declare <16 x i16> @llvm.x86.avx512.mask.pmovs.dw.512(<16 x i32>, <16 x i16>, i16)4837 4838define <8 x i32> @stack_fold_vpmovsqd(<8 x i64> %a0) {4839; CHECK-LABEL: stack_fold_vpmovsqd:4840; CHECK:       # %bb.0:4841; CHECK-NEXT:    vpmovsqd %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill4842; CHECK-NEXT:    #APP4843; CHECK-NEXT:    nop4844; CHECK-NEXT:    #NO_APP4845; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload4846; CHECK-NEXT:    retq4847  %1 = call <8 x i32> @llvm.x86.avx512.mask.pmovs.qd.512(<8 x i64> %a0, <8 x i32> undef, i8 -1)4848  %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4849  ret <8 x i32> %14850}4851declare <8 x i32> @llvm.x86.avx512.mask.pmovs.qd.512(<8 x i64>, <8 x i32>, i8)4852 4853define <8 x i16> @stack_fold_vpmovsqw(<8 x i64> %a0) {4854; CHECK-LABEL: stack_fold_vpmovsqw:4855; CHECK:       # %bb.0:4856; CHECK-NEXT:    vpmovsqw %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill4857; CHECK-NEXT:    #APP4858; CHECK-NEXT:    nop4859; CHECK-NEXT:    #NO_APP4860; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload4861; CHECK-NEXT:    vzeroupper4862; CHECK-NEXT:    retq4863  %1 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.512(<8 x i64> %a0, <8 x i16> undef, i8 -1)4864  %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4865  ret <8 x i16> %14866}4867declare <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.512(<8 x i64>, <8 x i16>, i8)4868 4869define <32 x i8> @stack_fold_vpmovswb(<32 x i16> %a0) {4870; CHECK-LABEL: stack_fold_vpmovswb:4871; CHECK:       # %bb.0:4872; CHECK-NEXT:    vpmovswb %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill4873; CHECK-NEXT:    #APP4874; CHECK-NEXT:    nop4875; CHECK-NEXT:    #NO_APP4876; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload4877; CHECK-NEXT:    retq4878  %1 = call <32 x i8> @llvm.x86.avx512.mask.pmovs.wb.512(<32 x i16> %a0, <32 x i8> undef, i32 -1)4879  %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4880  ret <32 x i8> %14881}4882declare <32 x i8> @llvm.x86.avx512.mask.pmovs.wb.512(<32 x i16>, <32 x i8>, i32)4883 4884define <16 x i32> @stack_fold_pmovsxbd_zmm(<16 x i8> %a0) {4885; CHECK-LABEL: stack_fold_pmovsxbd_zmm:4886; CHECK:       # %bb.0:4887; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4888; CHECK-NEXT:    #APP4889; CHECK-NEXT:    nop4890; CHECK-NEXT:    #NO_APP4891; CHECK-NEXT:    vpmovsxbd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 16-byte Folded Reload4892; CHECK-NEXT:    retq4893  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4894  %2 = sext <16 x i8> %a0 to <16 x i32>4895  ret <16 x i32> %24896}4897 4898define <8 x i64> @stack_fold_pmovsxbq_zmm(<16 x i8> %a0) {4899; CHECK-LABEL: stack_fold_pmovsxbq_zmm:4900; CHECK:       # %bb.0:4901; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4902; CHECK-NEXT:    #APP4903; CHECK-NEXT:    nop4904; CHECK-NEXT:    #NO_APP4905; CHECK-NEXT:    vpmovsxbq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 16-byte Folded Reload4906; CHECK-NEXT:    retq4907  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4908  %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>4909  %3 = sext <8 x i8> %2 to <8 x i64>4910  ret <8 x i64> %34911}4912 4913define <32 x i16> @stack_fold_pmovsxbw_zmm(<32 x i8> %a0) {4914; CHECK-LABEL: stack_fold_pmovsxbw_zmm:4915; CHECK:       # %bb.0:4916; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4917; CHECK-NEXT:    #APP4918; CHECK-NEXT:    nop4919; CHECK-NEXT:    #NO_APP4920; CHECK-NEXT:    vpmovsxbw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 32-byte Folded Reload4921; CHECK-NEXT:    retq4922  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4923  %2 = sext <32 x i8> %a0 to <32 x i16>4924  ret <32 x i16> %24925}4926 4927define <8 x i64> @stack_fold_pmovsxdq_zmm(<8 x i32> %a0) {4928; CHECK-LABEL: stack_fold_pmovsxdq_zmm:4929; CHECK:       # %bb.0:4930; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4931; CHECK-NEXT:    #APP4932; CHECK-NEXT:    nop4933; CHECK-NEXT:    #NO_APP4934; CHECK-NEXT:    vpmovsxdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 32-byte Folded Reload4935; CHECK-NEXT:    retq4936  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4937  %2 = sext <8 x i32> %a0 to <8 x i64>4938  ret <8 x i64> %24939}4940 4941define <16 x i32> @stack_fold_pmovsxwd_zmm(<16 x i16> %a0) {4942; CHECK-LABEL: stack_fold_pmovsxwd_zmm:4943; CHECK:       # %bb.0:4944; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill4945; CHECK-NEXT:    #APP4946; CHECK-NEXT:    nop4947; CHECK-NEXT:    #NO_APP4948; CHECK-NEXT:    vpmovsxwd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 32-byte Folded Reload4949; CHECK-NEXT:    retq4950  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4951  %2 = sext <16 x i16> %a0 to <16 x i32>4952  ret <16 x i32> %24953}4954 4955define <8 x i64> @stack_fold_pmovsxwq_zmm(<8 x i16> %a0) {4956; CHECK-LABEL: stack_fold_pmovsxwq_zmm:4957; CHECK:       # %bb.0:4958; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4959; CHECK-NEXT:    #APP4960; CHECK-NEXT:    nop4961; CHECK-NEXT:    #NO_APP4962; CHECK-NEXT:    vpmovsxwq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 16-byte Folded Reload4963; CHECK-NEXT:    retq4964  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4965  %2 = sext <8 x i16> %a0 to <8 x i64>4966  ret <8 x i64> %24967}4968 4969define <8 x i64> @stack_fold_pmovsxwq_mask_zmm(<8 x i64> %passthru, <8 x i16> %a0, i8 %mask) {4970; CHECK-LABEL: stack_fold_pmovsxwq_mask_zmm:4971; CHECK:       # %bb.0:4972; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4973; CHECK-NEXT:    #APP4974; CHECK-NEXT:    nop4975; CHECK-NEXT:    #NO_APP4976; CHECK-NEXT:    kmovd %edi, %k14977; CHECK-NEXT:    vpmovsxwq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} # 16-byte Folded Reload4978; CHECK-NEXT:    retq4979  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4980  %2 = sext <8 x i16> %a0 to <8 x i64>4981  %3 = bitcast i8 %mask to <8 x i1>4982  %4 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> %passthru4983  ret <8 x i64> %44984}4985 4986define <8 x i64> @stack_fold_pmovsxwq_maskz_zmm(<8 x i16> %a0, i8 %mask) {4987; CHECK-LABEL: stack_fold_pmovsxwq_maskz_zmm:4988; CHECK:       # %bb.0:4989; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill4990; CHECK-NEXT:    #APP4991; CHECK-NEXT:    nop4992; CHECK-NEXT:    #NO_APP4993; CHECK-NEXT:    kmovd %edi, %k14994; CHECK-NEXT:    vpmovsxwq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 16-byte Folded Reload4995; CHECK-NEXT:    retq4996  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()4997  %2 = sext <8 x i16> %a0 to <8 x i64>4998  %3 = bitcast i8 %mask to <8 x i1>4999  %4 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> zeroinitializer5000  ret <8 x i64> %45001}5002 5003define <16 x i8> @stack_fold_vpmovusdb(<16 x i32> %a0) {5004; CHECK-LABEL: stack_fold_vpmovusdb:5005; CHECK:       # %bb.0:5006; CHECK-NEXT:    vpmovusdb %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill5007; CHECK-NEXT:    #APP5008; CHECK-NEXT:    nop5009; CHECK-NEXT:    #NO_APP5010; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5011; CHECK-NEXT:    vzeroupper5012; CHECK-NEXT:    retq5013  %1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.512(<16 x i32> %a0, <16 x i8> undef, i16 -1)5014  %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5015  ret <16 x i8> %15016}5017declare <16 x i8> @llvm.x86.avx512.mask.pmovus.db.512(<16 x i32>, <16 x i8>, i16)5018 5019define <16 x i16> @stack_fold_vpmovusdw(<16 x i32> %a0) {5020; CHECK-LABEL: stack_fold_vpmovusdw:5021; CHECK:       # %bb.0:5022; CHECK-NEXT:    vpmovusdw %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill5023; CHECK-NEXT:    #APP5024; CHECK-NEXT:    nop5025; CHECK-NEXT:    #NO_APP5026; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload5027; CHECK-NEXT:    retq5028  %1 = call <16 x i16> @llvm.x86.avx512.mask.pmovus.dw.512(<16 x i32> %a0, <16 x i16> undef, i16 -1)5029  %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5030  ret <16 x i16> %15031}5032declare <16 x i16> @llvm.x86.avx512.mask.pmovus.dw.512(<16 x i32>, <16 x i16>, i16)5033 5034define <8 x i32> @stack_fold_vpmovusqd(<8 x i64> %a0) {5035; CHECK-LABEL: stack_fold_vpmovusqd:5036; CHECK:       # %bb.0:5037; CHECK-NEXT:    vpmovusqd %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill5038; CHECK-NEXT:    #APP5039; CHECK-NEXT:    nop5040; CHECK-NEXT:    #NO_APP5041; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload5042; CHECK-NEXT:    retq5043  %1 = call <8 x i32> @llvm.x86.avx512.mask.pmovus.qd.512(<8 x i64> %a0, <8 x i32> undef, i8 -1)5044  %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5045  ret <8 x i32> %15046}5047declare <8 x i32> @llvm.x86.avx512.mask.pmovus.qd.512(<8 x i64>, <8 x i32>, i8)5048 5049define <8 x i16> @stack_fold_vpmovusqw(<8 x i64> %a0) {5050; CHECK-LABEL: stack_fold_vpmovusqw:5051; CHECK:       # %bb.0:5052; CHECK-NEXT:    vpmovusqw %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Folded Spill5053; CHECK-NEXT:    #APP5054; CHECK-NEXT:    nop5055; CHECK-NEXT:    #NO_APP5056; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload5057; CHECK-NEXT:    vzeroupper5058; CHECK-NEXT:    retq5059  %1 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.512(<8 x i64> %a0, <8 x i16> undef, i8 -1)5060  %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5061  ret <8 x i16> %15062}5063declare <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.512(<8 x i64>, <8 x i16>, i8)5064 5065define <32 x i8> @stack_fold_vpmovuswb(<32 x i16> %a0) {5066; CHECK-LABEL: stack_fold_vpmovuswb:5067; CHECK:       # %bb.0:5068; CHECK-NEXT:    vpmovuswb %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Folded Spill5069; CHECK-NEXT:    #APP5070; CHECK-NEXT:    nop5071; CHECK-NEXT:    #NO_APP5072; CHECK-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload5073; CHECK-NEXT:    retq5074  %1 = call <32 x i8> @llvm.x86.avx512.mask.pmovus.wb.512(<32 x i16> %a0, <32 x i8> undef, i32 -1)5075  %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5076  ret <32 x i8> %15077}5078declare <32 x i8> @llvm.x86.avx512.mask.pmovus.wb.512(<32 x i16>, <32 x i8>, i32)5079 5080define <16 x i32> @stack_fold_pmovzxbd_zmm(<16 x i8> %a0) {5081; CHECK-LABEL: stack_fold_pmovzxbd_zmm:5082; CHECK:       # %bb.0:5083; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5084; CHECK-NEXT:    #APP5085; CHECK-NEXT:    nop5086; CHECK-NEXT:    #NO_APP5087; CHECK-NEXT:    vpmovzxbd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 16-byte Folded Reload5088; CHECK-NEXT:    # zmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero5089; CHECK-NEXT:    retq5090  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5091  %2 = zext <16 x i8> %a0 to <16 x i32>5092  ret <16 x i32> %25093}5094 5095define <8 x i64> @stack_fold_pmovzxbq_zmm(<16 x i8> %a0) {5096; CHECK-LABEL: stack_fold_pmovzxbq_zmm:5097; CHECK:       # %bb.0:5098; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5099; CHECK-NEXT:    #APP5100; CHECK-NEXT:    nop5101; CHECK-NEXT:    #NO_APP5102; CHECK-NEXT:    vpmovzxbq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 16-byte Folded Reload5103; CHECK-NEXT:    # zmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero,mem[4],zero,zero,zero,zero,zero,zero,zero,mem[5],zero,zero,zero,zero,zero,zero,zero,mem[6],zero,zero,zero,zero,zero,zero,zero,mem[7],zero,zero,zero,zero,zero,zero,zero5104; CHECK-NEXT:    retq5105  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5106  %2 = shufflevector <16 x i8> %a0, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>5107  %3 = zext <8 x i8> %2 to <8 x i64>5108  ret <8 x i64> %35109}5110 5111define <32 x i16> @stack_fold_pmovzxbw_zmm(<32 x i8> %a0) {5112; CHECK-LABEL: stack_fold_pmovzxbw_zmm:5113; CHECK:       # %bb.0:5114; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5115; CHECK-NEXT:    #APP5116; CHECK-NEXT:    nop5117; CHECK-NEXT:    #NO_APP5118; CHECK-NEXT:    vpmovzxbw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 32-byte Folded Reload5119; CHECK-NEXT:    # zmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero,mem[16],zero,mem[17],zero,mem[18],zero,mem[19],zero,mem[20],zero,mem[21],zero,mem[22],zero,mem[23],zero,mem[24],zero,mem[25],zero,mem[26],zero,mem[27],zero,mem[28],zero,mem[29],zero,mem[30],zero,mem[31],zero5120; CHECK-NEXT:    retq5121  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5122  %2 = zext <32 x i8> %a0 to <32 x i16>5123  ret <32 x i16> %25124}5125 5126define <8 x i64> @stack_fold_pmovzxdq_zmm(<8 x i32> %a0) {5127; CHECK-LABEL: stack_fold_pmovzxdq_zmm:5128; CHECK:       # %bb.0:5129; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5130; CHECK-NEXT:    #APP5131; CHECK-NEXT:    nop5132; CHECK-NEXT:    #NO_APP5133; CHECK-NEXT:    vpmovzxdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 32-byte Folded Reload5134; CHECK-NEXT:    # zmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero5135; CHECK-NEXT:    retq5136  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5137  %2 = zext <8 x i32> %a0 to <8 x i64>5138  ret <8 x i64> %25139}5140 5141define <16 x i32> @stack_fold_pmovzxwd_zmm(<16 x i16> %a0) {5142; CHECK-LABEL: stack_fold_pmovzxwd_zmm:5143; CHECK:       # %bb.0:5144; CHECK-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill5145; CHECK-NEXT:    #APP5146; CHECK-NEXT:    nop5147; CHECK-NEXT:    #NO_APP5148; CHECK-NEXT:    vpmovzxwd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 32-byte Folded Reload5149; CHECK-NEXT:    # zmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero5150; CHECK-NEXT:    retq5151  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5152  %2 = zext <16 x i16> %a0 to <16 x i32>5153  ret <16 x i32> %25154}5155 5156define <8 x i64> @stack_fold_pmovzxwq_zmm(<8 x i16> %a0) {5157; CHECK-LABEL: stack_fold_pmovzxwq_zmm:5158; CHECK:       # %bb.0:5159; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5160; CHECK-NEXT:    #APP5161; CHECK-NEXT:    nop5162; CHECK-NEXT:    #NO_APP5163; CHECK-NEXT:    vpmovzxwq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 16-byte Folded Reload5164; CHECK-NEXT:    # zmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero5165; CHECK-NEXT:    retq5166  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5167  %2 = zext <8 x i16> %a0 to <8 x i64>5168  ret <8 x i64> %25169}5170 5171define <8 x i64> @stack_fold_pmovzxwq_mask_zmm(<8 x i64> %passthru, <8 x i16> %a0, i8 %mask) {5172; CHECK-LABEL: stack_fold_pmovzxwq_mask_zmm:5173; CHECK:       # %bb.0:5174; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5175; CHECK-NEXT:    #APP5176; CHECK-NEXT:    nop5177; CHECK-NEXT:    #NO_APP5178; CHECK-NEXT:    kmovd %edi, %k15179; CHECK-NEXT:    vpmovzxwq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} # 16-byte Folded Reload5180; CHECK-NEXT:    # zmm0 {%k1} = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero5181; CHECK-NEXT:    retq5182  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5183  %2 = zext <8 x i16> %a0 to <8 x i64>5184  %3 = bitcast i8 %mask to <8 x i1>5185  %4 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> %passthru5186  ret <8 x i64> %45187}5188 5189define <8 x i64> @stack_fold_pmovzxwq_maskz_zmm(<8 x i16> %a0, i8 %mask) {5190; CHECK-LABEL: stack_fold_pmovzxwq_maskz_zmm:5191; CHECK:       # %bb.0:5192; CHECK-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill5193; CHECK-NEXT:    #APP5194; CHECK-NEXT:    nop5195; CHECK-NEXT:    #NO_APP5196; CHECK-NEXT:    kmovd %edi, %k15197; CHECK-NEXT:    vpmovzxwq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 16-byte Folded Reload5198; CHECK-NEXT:    # zmm0 {%k1} {z} = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero5199; CHECK-NEXT:    retq5200  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5201  %2 = zext <8 x i16> %a0 to <8 x i64>5202  %3 = bitcast i8 %mask to <8 x i1>5203  %4 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> zeroinitializer5204  ret <8 x i64> %45205}5206 5207define <16 x i32> @stack_fold_pmulld(<16 x i32> %a0, <16 x i32> %a1) {5208; CHECK-LABEL: stack_fold_pmulld:5209; CHECK:       # %bb.0:5210; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5211; CHECK-NEXT:    #APP5212; CHECK-NEXT:    nop5213; CHECK-NEXT:    #NO_APP5214; CHECK-NEXT:    vpmulld {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload5215; CHECK-NEXT:    retq5216  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5217  %2 = mul <16 x i32> %a0, %a15218  ret <16 x i32> %25219}5220 5221define <16 x i32> @stack_fold_pmulld_commuted(<16 x i32> %a0, <16 x i32> %a1) {5222; CHECK-LABEL: stack_fold_pmulld_commuted:5223; CHECK:       # %bb.0:5224; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5225; CHECK-NEXT:    #APP5226; CHECK-NEXT:    nop5227; CHECK-NEXT:    #NO_APP5228; CHECK-NEXT:    vpmulld {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload5229; CHECK-NEXT:    retq5230  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5231  %2 = mul <16 x i32> %a1, %a05232  ret <16 x i32> %25233}5234 5235define <16 x i32> @stack_fold_pmulld_mask(<16 x i32> %a0, <16 x i32> %a1, ptr %a2, i16 %mask) {5236; CHECK-LABEL: stack_fold_pmulld_mask:5237; CHECK:       # %bb.0:5238; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5239; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm15240; CHECK-NEXT:    #APP5241; CHECK-NEXT:    nop5242; CHECK-NEXT:    #NO_APP5243; CHECK-NEXT:    kmovd %esi, %k15244; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm05245; CHECK-NEXT:    vpmulld {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload5246; CHECK-NEXT:    retq5247  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5248  %2 = mul <16 x i32> %a0, %a15249  %3 = bitcast i16 %mask to <16 x i1>5250  ; load needed to keep the operation from being scheduled about the asm block5251  %4 = load <16 x i32>, ptr %a25252  %5 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> %45253  ret <16 x i32> %55254}5255 5256define <16 x i32> @stack_fold_pmulld_mask_commuted(<16 x i32> %a0, <16 x i32> %a1, ptr %a2, i16 %mask) {5257; CHECK-LABEL: stack_fold_pmulld_mask_commuted:5258; CHECK:       # %bb.0:5259; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5260; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm15261; CHECK-NEXT:    #APP5262; CHECK-NEXT:    nop5263; CHECK-NEXT:    #NO_APP5264; CHECK-NEXT:    kmovd %esi, %k15265; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm05266; CHECK-NEXT:    vpmulld {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload5267; CHECK-NEXT:    retq5268  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5269  %2 = mul <16 x i32> %a1, %a05270  %3 = bitcast i16 %mask to <16 x i1>5271  ; load needed to keep the operation from being scheduled about the asm block5272  %4 = load <16 x i32>, ptr %a25273  %5 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> %45274  ret <16 x i32> %55275}5276 5277define <16 x i32> @stack_fold_pmulld_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {5278; CHECK-LABEL: stack_fold_pmulld_maskz:5279; CHECK:       # %bb.0:5280; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5281; CHECK-NEXT:    #APP5282; CHECK-NEXT:    nop5283; CHECK-NEXT:    #NO_APP5284; CHECK-NEXT:    kmovd %edi, %k15285; CHECK-NEXT:    vpmulld {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload5286; CHECK-NEXT:    retq5287  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5288  %2 = mul <16 x i32> %a0, %a15289  %3 = bitcast i16 %mask to <16 x i1>5290  %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer5291  ret <16 x i32> %45292}5293 5294define <16 x i32> @stack_fold_pmulld_maskz_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {5295; CHECK-LABEL: stack_fold_pmulld_maskz_commuted:5296; CHECK:       # %bb.0:5297; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5298; CHECK-NEXT:    #APP5299; CHECK-NEXT:    nop5300; CHECK-NEXT:    #NO_APP5301; CHECK-NEXT:    kmovd %edi, %k15302; CHECK-NEXT:    vpmulld {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload5303; CHECK-NEXT:    retq5304  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5305  %2 = mul <16 x i32> %a1, %a05306  %3 = bitcast i16 %mask to <16 x i1>5307  %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer5308  ret <16 x i32> %45309}5310 5311define <8 x i64> @stack_fold_pmullq(<8 x i64> %a0, <8 x i64> %a1) {5312; CHECK-LABEL: stack_fold_pmullq:5313; CHECK:       # %bb.0:5314; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5315; CHECK-NEXT:    #APP5316; CHECK-NEXT:    nop5317; CHECK-NEXT:    #NO_APP5318; CHECK-NEXT:    vpmullq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload5319; CHECK-NEXT:    retq5320  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5321  %2 = mul <8 x i64> %a0, %a15322  ret <8 x i64> %25323}5324 5325define <8 x i64> @stack_fold_pmullq_commuted(<8 x i64> %a0, <8 x i64> %a1) {5326; CHECK-LABEL: stack_fold_pmullq_commuted:5327; CHECK:       # %bb.0:5328; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5329; CHECK-NEXT:    #APP5330; CHECK-NEXT:    nop5331; CHECK-NEXT:    #NO_APP5332; CHECK-NEXT:    vpmullq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload5333; CHECK-NEXT:    retq5334  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5335  %2 = mul <8 x i64> %a1, %a05336  ret <8 x i64> %25337}5338 5339define <8 x i64> @stack_fold_pmullq_mask(<8 x i64> %a0, <8 x i64> %a1, ptr %a2, i8 %mask) {5340; CHECK-LABEL: stack_fold_pmullq_mask:5341; CHECK:       # %bb.0:5342; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5343; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm15344; CHECK-NEXT:    #APP5345; CHECK-NEXT:    nop5346; CHECK-NEXT:    #NO_APP5347; CHECK-NEXT:    kmovd %esi, %k15348; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm05349; CHECK-NEXT:    vpmullq {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload5350; CHECK-NEXT:    retq5351  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5352  %2 = mul <8 x i64> %a0, %a15353  %3 = bitcast i8 %mask to <8 x i1>5354  ; load needed to keep the operation from being scheduled about the asm block5355  %4 = load <8 x i64>, ptr %a25356  %5 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> %45357  ret <8 x i64> %55358}5359 5360define <8 x i64> @stack_fold_pmullq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, ptr %a2, i8 %mask) {5361; CHECK-LABEL: stack_fold_pmullq_mask_commuted:5362; CHECK:       # %bb.0:5363; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5364; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm15365; CHECK-NEXT:    #APP5366; CHECK-NEXT:    nop5367; CHECK-NEXT:    #NO_APP5368; CHECK-NEXT:    kmovd %esi, %k15369; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm05370; CHECK-NEXT:    vpmullq {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload5371; CHECK-NEXT:    retq5372  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5373  %2 = mul <8 x i64> %a1, %a05374  %3 = bitcast i8 %mask to <8 x i1>5375  ; load needed to keep the operation from being scheduled about the asm block5376  %4 = load <8 x i64>, ptr %a25377  %5 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> %45378  ret <8 x i64> %55379}5380 5381define <8 x i64> @stack_fold_pmullq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {5382; CHECK-LABEL: stack_fold_pmullq_maskz:5383; CHECK:       # %bb.0:5384; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5385; CHECK-NEXT:    #APP5386; CHECK-NEXT:    nop5387; CHECK-NEXT:    #NO_APP5388; CHECK-NEXT:    kmovd %edi, %k15389; CHECK-NEXT:    vpmullq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload5390; CHECK-NEXT:    retq5391  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5392  %2 = mul <8 x i64> %a0, %a15393  %3 = bitcast i8 %mask to <8 x i1>5394  %4 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> zeroinitializer5395  ret <8 x i64> %45396}5397 5398define <8 x i64> @stack_fold_pmullq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {5399; CHECK-LABEL: stack_fold_pmullq_maskz_commuted:5400; CHECK:       # %bb.0:5401; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5402; CHECK-NEXT:    #APP5403; CHECK-NEXT:    nop5404; CHECK-NEXT:    #NO_APP5405; CHECK-NEXT:    kmovd %edi, %k15406; CHECK-NEXT:    vpmullq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload5407; CHECK-NEXT:    retq5408  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5409  %2 = mul <8 x i64> %a1, %a05410  %3 = bitcast i8 %mask to <8 x i1>5411  %4 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> zeroinitializer5412  ret <8 x i64> %45413}5414 5415define <32 x i16> @stack_fold_pmullw(<32 x i16> %a0, <32 x i16> %a1) {5416; CHECK-LABEL: stack_fold_pmullw:5417; CHECK:       # %bb.0:5418; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5419; CHECK-NEXT:    #APP5420; CHECK-NEXT:    nop5421; CHECK-NEXT:    #NO_APP5422; CHECK-NEXT:    vpmullw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload5423; CHECK-NEXT:    retq5424  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5425  %2 = mul <32 x i16> %a0, %a15426  ret <32 x i16> %25427}5428 5429define <32 x i16> @stack_fold_pmullw_commuted(<32 x i16> %a0, <32 x i16> %a1) {5430; CHECK-LABEL: stack_fold_pmullw_commuted:5431; CHECK:       # %bb.0:5432; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5433; CHECK-NEXT:    #APP5434; CHECK-NEXT:    nop5435; CHECK-NEXT:    #NO_APP5436; CHECK-NEXT:    vpmullw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload5437; CHECK-NEXT:    retq5438  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5439  %2 = mul <32 x i16> %a1, %a05440  ret <32 x i16> %25441}5442 5443define <32 x i16> @stack_fold_pmullw_mask(<32 x i16> %a0, <32 x i16> %a1, ptr %a2, i32 %mask) {5444; CHECK-LABEL: stack_fold_pmullw_mask:5445; CHECK:       # %bb.0:5446; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5447; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm15448; CHECK-NEXT:    #APP5449; CHECK-NEXT:    nop5450; CHECK-NEXT:    #NO_APP5451; CHECK-NEXT:    kmovd %esi, %k15452; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm05453; CHECK-NEXT:    vpmullw {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload5454; CHECK-NEXT:    retq5455  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5456  %2 = mul <32 x i16> %a0, %a15457  %3 = bitcast i32 %mask to <32 x i1>5458  ; load needed to keep the operation from being scheduled about the asm block5459  %4 = load <32 x i16>, ptr %a25460  %5 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> %45461  ret <32 x i16> %55462}5463 5464define <32 x i16> @stack_fold_pmullw_mask_commuted(<32 x i16> %a0, <32 x i16> %a1, ptr %a2, i32 %mask) {5465; CHECK-LABEL: stack_fold_pmullw_mask_commuted:5466; CHECK:       # %bb.0:5467; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5468; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm15469; CHECK-NEXT:    #APP5470; CHECK-NEXT:    nop5471; CHECK-NEXT:    #NO_APP5472; CHECK-NEXT:    kmovd %esi, %k15473; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm05474; CHECK-NEXT:    vpmullw {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload5475; CHECK-NEXT:    retq5476  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5477  %2 = mul <32 x i16> %a1, %a05478  %3 = bitcast i32 %mask to <32 x i1>5479  ; load needed to keep the operation from being scheduled about the asm block5480  %4 = load <32 x i16>, ptr %a25481  %5 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> %45482  ret <32 x i16> %55483}5484 5485define <32 x i16> @stack_fold_pmullw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {5486; CHECK-LABEL: stack_fold_pmullw_maskz:5487; CHECK:       # %bb.0:5488; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5489; CHECK-NEXT:    #APP5490; CHECK-NEXT:    nop5491; CHECK-NEXT:    #NO_APP5492; CHECK-NEXT:    kmovd %edi, %k15493; CHECK-NEXT:    vpmullw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload5494; CHECK-NEXT:    retq5495  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5496  %2 = mul <32 x i16> %a0, %a15497  %3 = bitcast i32 %mask to <32 x i1>5498  %4 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> zeroinitializer5499  ret <32 x i16> %45500}5501 5502define <32 x i16> @stack_fold_pmullw_maskz_commuted(<32 x i16> %a0, <32 x i16> %a1, i32 %mask) {5503; CHECK-LABEL: stack_fold_pmullw_maskz_commuted:5504; CHECK:       # %bb.0:5505; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5506; CHECK-NEXT:    #APP5507; CHECK-NEXT:    nop5508; CHECK-NEXT:    #NO_APP5509; CHECK-NEXT:    kmovd %edi, %k15510; CHECK-NEXT:    vpmullw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload5511; CHECK-NEXT:    retq5512  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5513  %2 = mul <32 x i16> %a1, %a05514  %3 = bitcast i32 %mask to <32 x i1>5515  %4 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> zeroinitializer5516  ret <32 x i16> %45517}5518 5519define <8 x i64> @stack_fold_pmuldq(<8 x i64> %a0, <8 x i64> %a1) {5520; CHECK-LABEL: stack_fold_pmuldq:5521; CHECK:       # %bb.0:5522; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5523; CHECK-NEXT:    #APP5524; CHECK-NEXT:    nop5525; CHECK-NEXT:    #NO_APP5526; CHECK-NEXT:    vpmuldq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload5527; CHECK-NEXT:    retq5528  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5529  %2 = shl <8 x i64> %a0, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>5530  %3 = ashr <8 x i64> %2, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>5531  %4 = shl <8 x i64> %a1, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>5532  %5 = ashr <8 x i64> %4, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>5533  %6 = mul <8 x i64> %3, %55534  ret <8 x i64> %65535}5536 5537define <8 x i64> @stack_fold_pmuldq_commuted(<8 x i64> %a0, <8 x i64> %a1) {5538; CHECK-LABEL: stack_fold_pmuldq_commuted:5539; CHECK:       # %bb.0:5540; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5541; CHECK-NEXT:    #APP5542; CHECK-NEXT:    nop5543; CHECK-NEXT:    #NO_APP5544; CHECK-NEXT:    vpmuldq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload5545; CHECK-NEXT:    retq5546  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5547  %2 = shl <8 x i64> %a0, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>5548  %3 = ashr <8 x i64> %2, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>5549  %4 = shl <8 x i64> %a1, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>5550  %5 = ashr <8 x i64> %4, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>5551  %6 = mul <8 x i64> %5, %35552  ret <8 x i64> %65553}5554 5555define <8 x i64> @stack_fold_pmuldq_mask(<8 x i64> %a0, <8 x i64> %a1, ptr %a2, i8 %mask) {5556; CHECK-LABEL: stack_fold_pmuldq_mask:5557; CHECK:       # %bb.0:5558; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5559; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm15560; CHECK-NEXT:    #APP5561; CHECK-NEXT:    nop5562; CHECK-NEXT:    #NO_APP5563; CHECK-NEXT:    kmovd %esi, %k15564; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm05565; CHECK-NEXT:    vpmuldq {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload5566; CHECK-NEXT:    retq5567  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5568  %2 = shl <8 x i64> %a0, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>5569  %3 = ashr <8 x i64> %2, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>5570  %4 = shl <8 x i64> %a1, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>5571  %5 = ashr <8 x i64> %4, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>5572  %6 = mul <8 x i64> %3, %55573  %7 = bitcast i8 %mask to <8 x i1>5574  ; load needed to keep the operation from being scheduled about the asm block5575  %8 = load <8 x i64>, ptr %a25576  %9 = select <8 x i1> %7, <8 x i64> %6, <8 x i64> %85577  ret <8 x i64> %95578}5579 5580define <8 x i64> @stack_fold_pmuldq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, ptr %a2, i8 %mask) {5581; CHECK-LABEL: stack_fold_pmuldq_mask_commuted:5582; CHECK:       # %bb.0:5583; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5584; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm15585; CHECK-NEXT:    #APP5586; CHECK-NEXT:    nop5587; CHECK-NEXT:    #NO_APP5588; CHECK-NEXT:    kmovd %esi, %k15589; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm05590; CHECK-NEXT:    vpmuldq {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload5591; CHECK-NEXT:    retq5592  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5593  %2 = shl <8 x i64> %a0, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>5594  %3 = ashr <8 x i64> %2, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>5595  %4 = shl <8 x i64> %a1, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>5596  %5 = ashr <8 x i64> %4, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>5597  %6 = mul <8 x i64> %5, %35598  %7 = bitcast i8 %mask to <8 x i1>5599  ; load needed to keep the operation from being scheduled about the asm block5600  %8 = load <8 x i64>, ptr %a25601  %9 = select <8 x i1> %7, <8 x i64> %6, <8 x i64> %85602  ret <8 x i64> %95603}5604 5605define <8 x i64> @stack_fold_pmuldq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {5606; CHECK-LABEL: stack_fold_pmuldq_maskz:5607; CHECK:       # %bb.0:5608; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5609; CHECK-NEXT:    #APP5610; CHECK-NEXT:    nop5611; CHECK-NEXT:    #NO_APP5612; CHECK-NEXT:    kmovd %edi, %k15613; CHECK-NEXT:    vpmuldq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload5614; CHECK-NEXT:    retq5615  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5616  %2 = shl <8 x i64> %a0, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>5617  %3 = ashr <8 x i64> %2, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>5618  %4 = shl <8 x i64> %a1, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>5619  %5 = ashr <8 x i64> %4, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>5620  %6 = mul <8 x i64> %3, %55621  %7 = bitcast i8 %mask to <8 x i1>5622  %8 = select <8 x i1> %7, <8 x i64> %6, <8 x i64> zeroinitializer5623  ret <8 x i64> %85624}5625 5626define <8 x i64> @stack_fold_pmuldq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {5627; CHECK-LABEL: stack_fold_pmuldq_maskz_commuted:5628; CHECK:       # %bb.0:5629; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5630; CHECK-NEXT:    #APP5631; CHECK-NEXT:    nop5632; CHECK-NEXT:    #NO_APP5633; CHECK-NEXT:    kmovd %edi, %k15634; CHECK-NEXT:    vpmuldq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload5635; CHECK-NEXT:    retq5636  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5637  %2 = shl <8 x i64> %a0, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>5638  %3 = ashr <8 x i64> %2, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>5639  %4 = shl <8 x i64> %a1, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>5640  %5 = ashr <8 x i64> %4, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>5641  %6 = mul <8 x i64> %5, %35642  %7 = bitcast i8 %mask to <8 x i1>5643  %8 = select <8 x i1> %7, <8 x i64> %6, <8 x i64> zeroinitializer5644  ret <8 x i64> %85645}5646 5647 5648 5649 5650define <8 x i64> @stack_fold_pmuludq(<8 x i64> %a0, <8 x i64> %a1) {5651; CHECK-LABEL: stack_fold_pmuludq:5652; CHECK:       # %bb.0:5653; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5654; CHECK-NEXT:    #APP5655; CHECK-NEXT:    nop5656; CHECK-NEXT:    #NO_APP5657; CHECK-NEXT:    vpmuludq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload5658; CHECK-NEXT:    retq5659  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5660  %2 = and <8 x i64> %a0, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>5661  %3 = and <8 x i64> %a1, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>5662  %4 = mul <8 x i64> %2, %35663  ret <8 x i64> %45664}5665 5666define <8 x i64> @stack_fold_pmuludq_commuted(<8 x i64> %a0, <8 x i64> %a1) {5667; CHECK-LABEL: stack_fold_pmuludq_commuted:5668; CHECK:       # %bb.0:5669; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5670; CHECK-NEXT:    #APP5671; CHECK-NEXT:    nop5672; CHECK-NEXT:    #NO_APP5673; CHECK-NEXT:    vpmuludq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload5674; CHECK-NEXT:    retq5675  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5676  %2 = and <8 x i64> %a0, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>5677  %3 = and <8 x i64> %a1, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>5678  %4 = mul <8 x i64> %3, %25679  ret <8 x i64> %45680}5681 5682define <8 x i64> @stack_fold_pmuludq_mask(<8 x i64> %a0, <8 x i64> %a1, ptr %a2, i8 %mask) {5683; CHECK-LABEL: stack_fold_pmuludq_mask:5684; CHECK:       # %bb.0:5685; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5686; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm15687; CHECK-NEXT:    #APP5688; CHECK-NEXT:    nop5689; CHECK-NEXT:    #NO_APP5690; CHECK-NEXT:    kmovd %esi, %k15691; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm05692; CHECK-NEXT:    vpmuludq {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload5693; CHECK-NEXT:    retq5694  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5695  %2 = and <8 x i64> %a0, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>5696  %3 = and <8 x i64> %a1, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>5697  %4 = mul <8 x i64> %2, %35698  %5 = bitcast i8 %mask to <8 x i1>5699  ; load needed to keep the operation from being scheduled about the asm block5700  %6 = load <8 x i64>, ptr %a25701  %7 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> %65702  ret <8 x i64> %75703}5704 5705define <8 x i64> @stack_fold_pmuludq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, ptr %a2, i8 %mask) {5706; CHECK-LABEL: stack_fold_pmuludq_mask_commuted:5707; CHECK:       # %bb.0:5708; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5709; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm15710; CHECK-NEXT:    #APP5711; CHECK-NEXT:    nop5712; CHECK-NEXT:    #NO_APP5713; CHECK-NEXT:    kmovd %esi, %k15714; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm05715; CHECK-NEXT:    vpmuludq {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload5716; CHECK-NEXT:    retq5717  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5718  %2 = and <8 x i64> %a0, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>5719  %3 = and <8 x i64> %a1, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>5720  %4 = mul <8 x i64> %3, %25721  %5 = bitcast i8 %mask to <8 x i1>5722  ; load needed to keep the operation from being scheduled about the asm block5723  %6 = load <8 x i64>, ptr %a25724  %7 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> %65725  ret <8 x i64> %75726}5727 5728define <8 x i64> @stack_fold_pmuludq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {5729; CHECK-LABEL: stack_fold_pmuludq_maskz:5730; CHECK:       # %bb.0:5731; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5732; CHECK-NEXT:    #APP5733; CHECK-NEXT:    nop5734; CHECK-NEXT:    #NO_APP5735; CHECK-NEXT:    kmovd %edi, %k15736; CHECK-NEXT:    vpmuludq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload5737; CHECK-NEXT:    retq5738  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5739  %2 = and <8 x i64> %a0, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>5740  %3 = and <8 x i64> %a1, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>5741  %4 = mul <8 x i64> %2, %35742  %5 = bitcast i8 %mask to <8 x i1>5743  %6 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> zeroinitializer5744  ret <8 x i64> %65745}5746 5747define <8 x i64> @stack_fold_pmuludq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {5748; CHECK-LABEL: stack_fold_pmuludq_maskz_commuted:5749; CHECK:       # %bb.0:5750; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5751; CHECK-NEXT:    #APP5752; CHECK-NEXT:    nop5753; CHECK-NEXT:    #NO_APP5754; CHECK-NEXT:    kmovd %edi, %k15755; CHECK-NEXT:    vpmuludq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload5756; CHECK-NEXT:    retq5757  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5758  %2 = and <8 x i64> %a0, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>5759  %3 = and <8 x i64> %a1, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>5760  %4 = mul <8 x i64> %3, %25761  %5 = bitcast i8 %mask to <8 x i1>5762  %6 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> zeroinitializer5763  ret <8 x i64> %65764}5765 5766define <16 x i32> @stack_fold_vpopcntd(<16 x i32> %a0) {5767; CHECK-LABEL: stack_fold_vpopcntd:5768; CHECK:       # %bb.0:5769; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5770; CHECK-NEXT:    #APP5771; CHECK-NEXT:    nop5772; CHECK-NEXT:    #NO_APP5773; CHECK-NEXT:    vpopcntd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload5774; CHECK-NEXT:    retq5775  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5776  %2 = call <16 x i32> @llvm.ctpop.v16i32(<16 x i32> %a0)5777  ret <16 x i32> %25778}5779declare <16 x i32> @llvm.ctpop.v16i32(<16 x i32>) nounwind readonly5780 5781define <8 x i64> @stack_fold_vpopcntq(<8 x i64> %a0) {5782; CHECK-LABEL: stack_fold_vpopcntq:5783; CHECK:       # %bb.0:5784; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5785; CHECK-NEXT:    #APP5786; CHECK-NEXT:    nop5787; CHECK-NEXT:    #NO_APP5788; CHECK-NEXT:    vpopcntq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload5789; CHECK-NEXT:    retq5790  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5791  %2 = call <8 x i64> @llvm.ctpop.v8i64(<8 x i64> %a0)5792  ret <8 x i64> %25793}5794declare <8 x i64> @llvm.ctpop.v8i64(<8 x i64>) nounwind readnone5795 5796define <16 x i32> @stack_fold_pord(<16 x i32> %a0, <16 x i32> %a1) {5797; CHECK-LABEL: stack_fold_pord:5798; CHECK:       # %bb.0:5799; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5800; CHECK-NEXT:    #APP5801; CHECK-NEXT:    nop5802; CHECK-NEXT:    #NO_APP5803; CHECK-NEXT:    vorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload5804; CHECK-NEXT:    retq5805  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5806  %2 = or <16 x i32> %a0, %a15807  ret <16 x i32> %25808}5809 5810define <16 x i32> @stack_fold_pord_commuted(<16 x i32> %a0, <16 x i32> %a1) {5811; CHECK-LABEL: stack_fold_pord_commuted:5812; CHECK:       # %bb.0:5813; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5814; CHECK-NEXT:    #APP5815; CHECK-NEXT:    nop5816; CHECK-NEXT:    #NO_APP5817; CHECK-NEXT:    vorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload5818; CHECK-NEXT:    retq5819  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5820  %2 = or <16 x i32> %a1, %a05821  ret <16 x i32> %25822}5823 5824define <16 x i32> @stack_fold_pord_mask(<16 x i32> %a0, <16 x i32> %a1, ptr %a2, i16 %mask) {5825; CHECK-LABEL: stack_fold_pord_mask:5826; CHECK:       # %bb.0:5827; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5828; CHECK-NEXT:    vmovaps %zmm0, %zmm15829; CHECK-NEXT:    #APP5830; CHECK-NEXT:    nop5831; CHECK-NEXT:    #NO_APP5832; CHECK-NEXT:    kmovd %esi, %k15833; CHECK-NEXT:    vmovaps (%rdi), %zmm05834; CHECK-NEXT:    vorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload5835; CHECK-NEXT:    retq5836  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5837  %2 = or <16 x i32> %a0, %a15838  %3 = bitcast i16 %mask to <16 x i1>5839  ; load needed to keep the operation from being scheduled about the asm block5840  %4 = load <16 x i32>, ptr %a25841  %5 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> %45842  ret <16 x i32> %55843}5844 5845define <16 x i32> @stack_fold_pord_mask_commuted(<16 x i32> %a0, <16 x i32> %a1, ptr %a2, i16 %mask) {5846; CHECK-LABEL: stack_fold_pord_mask_commuted:5847; CHECK:       # %bb.0:5848; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5849; CHECK-NEXT:    vmovaps %zmm0, %zmm15850; CHECK-NEXT:    #APP5851; CHECK-NEXT:    nop5852; CHECK-NEXT:    #NO_APP5853; CHECK-NEXT:    kmovd %esi, %k15854; CHECK-NEXT:    vmovaps (%rdi), %zmm05855; CHECK-NEXT:    vorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload5856; CHECK-NEXT:    retq5857  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5858  %2 = or <16 x i32> %a1, %a05859  %3 = bitcast i16 %mask to <16 x i1>5860  ; load needed to keep the operation from being scheduled about the asm block5861  %4 = load <16 x i32>, ptr %a25862  %5 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> %45863  ret <16 x i32> %55864}5865 5866define <16 x i32> @stack_fold_pord_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {5867; CHECK-LABEL: stack_fold_pord_maskz:5868; CHECK:       # %bb.0:5869; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5870; CHECK-NEXT:    #APP5871; CHECK-NEXT:    nop5872; CHECK-NEXT:    #NO_APP5873; CHECK-NEXT:    kmovd %edi, %k15874; CHECK-NEXT:    vorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload5875; CHECK-NEXT:    retq5876  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5877  %2 = or <16 x i32> %a0, %a15878  %3 = bitcast i16 %mask to <16 x i1>5879  %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer5880  ret <16 x i32> %45881}5882 5883define <16 x i32> @stack_fold_pord_maskz_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {5884; CHECK-LABEL: stack_fold_pord_maskz_commuted:5885; CHECK:       # %bb.0:5886; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5887; CHECK-NEXT:    #APP5888; CHECK-NEXT:    nop5889; CHECK-NEXT:    #NO_APP5890; CHECK-NEXT:    kmovd %edi, %k15891; CHECK-NEXT:    vorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload5892; CHECK-NEXT:    retq5893  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5894  %2 = or <16 x i32> %a1, %a05895  %3 = bitcast i16 %mask to <16 x i1>5896  %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer5897  ret <16 x i32> %45898}5899 5900define <8 x i64> @stack_fold_porq(<8 x i64> %a0, <8 x i64> %a1) {5901; CHECK-LABEL: stack_fold_porq:5902; CHECK:       # %bb.0:5903; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5904; CHECK-NEXT:    #APP5905; CHECK-NEXT:    nop5906; CHECK-NEXT:    #NO_APP5907; CHECK-NEXT:    vorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload5908; CHECK-NEXT:    retq5909  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5910  %2 = or <8 x i64> %a0, %a15911  ret <8 x i64> %25912}5913 5914define <8 x i64> @stack_fold_porq_commuted(<8 x i64> %a0, <8 x i64> %a1) {5915; CHECK-LABEL: stack_fold_porq_commuted:5916; CHECK:       # %bb.0:5917; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5918; CHECK-NEXT:    #APP5919; CHECK-NEXT:    nop5920; CHECK-NEXT:    #NO_APP5921; CHECK-NEXT:    vorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload5922; CHECK-NEXT:    retq5923  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5924  %2 = or <8 x i64> %a1, %a05925  ret <8 x i64> %25926}5927 5928define <8 x i64> @stack_fold_porq_mask(<8 x i64> %a0, <8 x i64> %a1, ptr %a2, i8 %mask) {5929; CHECK-LABEL: stack_fold_porq_mask:5930; CHECK:       # %bb.0:5931; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5932; CHECK-NEXT:    vmovapd %zmm0, %zmm15933; CHECK-NEXT:    #APP5934; CHECK-NEXT:    nop5935; CHECK-NEXT:    #NO_APP5936; CHECK-NEXT:    kmovd %esi, %k15937; CHECK-NEXT:    vmovapd (%rdi), %zmm05938; CHECK-NEXT:    vorpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload5939; CHECK-NEXT:    retq5940  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5941  %2 = or <8 x i64> %a0, %a15942  %3 = bitcast i8 %mask to <8 x i1>5943  ; load needed to keep the operation from being scheduled about the asm block5944  %4 = load <8 x i64>, ptr %a25945  %5 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> %45946  ret <8 x i64> %55947}5948 5949define <8 x i64> @stack_fold_porq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, ptr %a2, i8 %mask) {5950; CHECK-LABEL: stack_fold_porq_mask_commuted:5951; CHECK:       # %bb.0:5952; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5953; CHECK-NEXT:    vmovapd %zmm0, %zmm15954; CHECK-NEXT:    #APP5955; CHECK-NEXT:    nop5956; CHECK-NEXT:    #NO_APP5957; CHECK-NEXT:    kmovd %esi, %k15958; CHECK-NEXT:    vmovapd (%rdi), %zmm05959; CHECK-NEXT:    vorpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload5960; CHECK-NEXT:    retq5961  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5962  %2 = or <8 x i64> %a1, %a05963  %3 = bitcast i8 %mask to <8 x i1>5964  ; load needed to keep the operation from being scheduled about the asm block5965  %4 = load <8 x i64>, ptr %a25966  %5 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> %45967  ret <8 x i64> %55968}5969 5970define <8 x i64> @stack_fold_porq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {5971; CHECK-LABEL: stack_fold_porq_maskz:5972; CHECK:       # %bb.0:5973; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5974; CHECK-NEXT:    #APP5975; CHECK-NEXT:    nop5976; CHECK-NEXT:    #NO_APP5977; CHECK-NEXT:    kmovd %edi, %k15978; CHECK-NEXT:    vorpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload5979; CHECK-NEXT:    retq5980  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5981  %2 = or <8 x i64> %a0, %a15982  %3 = bitcast i8 %mask to <8 x i1>5983  %4 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> zeroinitializer5984  ret <8 x i64> %45985}5986 5987define <8 x i64> @stack_fold_porq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {5988; CHECK-LABEL: stack_fold_porq_maskz_commuted:5989; CHECK:       # %bb.0:5990; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill5991; CHECK-NEXT:    #APP5992; CHECK-NEXT:    nop5993; CHECK-NEXT:    #NO_APP5994; CHECK-NEXT:    kmovd %edi, %k15995; CHECK-NEXT:    vorpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload5996; CHECK-NEXT:    retq5997  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()5998  %2 = or <8 x i64> %a1, %a05999  %3 = bitcast i8 %mask to <8 x i1>6000  %4 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> zeroinitializer6001  ret <8 x i64> %46002}6003 6004define <8 x i64> @stack_fold_psadbw(<64 x i8> %a0, <64 x i8> %a1) {6005; CHECK-LABEL: stack_fold_psadbw:6006; CHECK:       # %bb.0:6007; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6008; CHECK-NEXT:    #APP6009; CHECK-NEXT:    nop6010; CHECK-NEXT:    #NO_APP6011; CHECK-NEXT:    vpsadbw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload6012; CHECK-NEXT:    retq6013  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6014  %2 = call <8 x i64> @llvm.x86.avx512.psad.bw.512(<64 x i8> %a0, <64 x i8> %a1)6015  ret <8 x i64> %26016}6017declare <8 x i64> @llvm.x86.avx512.psad.bw.512(<64 x i8>, <64 x i8>) nounwind readnone6018 6019define <8 x i64> @stack_fold_psadbw_commute(<64 x i8> %a0, <64 x i8> %a1) {6020; CHECK-LABEL: stack_fold_psadbw_commute:6021; CHECK:       # %bb.0:6022; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6023; CHECK-NEXT:    #APP6024; CHECK-NEXT:    nop6025; CHECK-NEXT:    #NO_APP6026; CHECK-NEXT:    vpsadbw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload6027; CHECK-NEXT:    retq6028  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6029  %2 = call <8 x i64> @llvm.x86.avx512.psad.bw.512(<64 x i8> %a1, <64 x i8> %a0)6030  ret <8 x i64> %26031}6032 6033define <64 x i8> @stack_fold_pshufb_zmm(<64 x i8> %a0, <64 x i8> %a1) {6034; CHECK-LABEL: stack_fold_pshufb_zmm:6035; CHECK:       # %bb.0:6036; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6037; CHECK-NEXT:    #APP6038; CHECK-NEXT:    nop6039; CHECK-NEXT:    #NO_APP6040; CHECK-NEXT:    vpshufb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload6041; CHECK-NEXT:    retq6042  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6043  %2 = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %a0, <64 x i8> %a1)6044  ret <64 x i8> %26045}6046declare <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8>, <64 x i8>)6047 6048define <64 x i8> @stack_fold_pshufb_zmm_mask(ptr %passthru, <64 x i8> %a0, <64 x i8> %a1, i64 %mask) {6049; CHECK-LABEL: stack_fold_pshufb_zmm_mask:6050; CHECK:       # %bb.0:6051; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6052; CHECK-NEXT:    #APP6053; CHECK-NEXT:    nop6054; CHECK-NEXT:    #NO_APP6055; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm26056; CHECK-NEXT:    kmovq %rsi, %k16057; CHECK-NEXT:    vpshufb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload6058; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm06059; CHECK-NEXT:    retq6060  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6061  %2 = load <64 x i8>, ptr %passthru6062  %3 = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %a0, <64 x i8> %a1)6063  %4 = bitcast i64 %mask to <64 x i1>6064  %5 = select <64 x i1> %4, <64 x i8> %3, <64 x i8> %26065  ret <64 x i8> %56066}6067 6068define <64 x i8> @stack_fold_pshufb_zmm_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {6069; CHECK-LABEL: stack_fold_pshufb_zmm_maskz:6070; CHECK:       # %bb.0:6071; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6072; CHECK-NEXT:    #APP6073; CHECK-NEXT:    nop6074; CHECK-NEXT:    #NO_APP6075; CHECK-NEXT:    kmovq %rdi, %k16076; CHECK-NEXT:    vpshufb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload6077; CHECK-NEXT:    retq6078  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6079  %2 = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> %a0, <64 x i8> %a1)6080  %3 = bitcast i64 %mask to <64 x i1>6081  %4 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> zeroinitializer6082  ret <64 x i8> %46083}6084 6085define <16 x i32> @stack_fold_pshufd_zmm(<16 x i32> %a0) {6086; CHECK-LABEL: stack_fold_pshufd_zmm:6087; CHECK:       # %bb.0:6088; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6089; CHECK-NEXT:    #APP6090; CHECK-NEXT:    nop6091; CHECK-NEXT:    #NO_APP6092; CHECK-NEXT:    vpshufd $27, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload6093; CHECK-NEXT:    # zmm0 = mem[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]6094; CHECK-NEXT:    vpternlogd {{.*#+}} zmm1 = -16095; CHECK-NEXT:    vpsubd %zmm1, %zmm0, %zmm06096; CHECK-NEXT:    retq6097  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6098  %2 = shufflevector <16 x i32> %a0, <16 x i32> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>6099  %3 = add <16 x i32> %2, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>6100  ret <16 x i32> %36101}6102 6103define <16 x i32> @stack_fold_pshufd_zmm_mask(<16 x i32> %passthru, <16 x i32> %a0, i16 %mask) {6104; CHECK-LABEL: stack_fold_pshufd_zmm_mask:6105; CHECK:       # %bb.0:6106; CHECK-NEXT:    pushq %rax6107; CHECK-NEXT:    .cfi_def_cfa_offset 166108; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6109; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6110; CHECK-NEXT:    #APP6111; CHECK-NEXT:    nop6112; CHECK-NEXT:    #NO_APP6113; CHECK-NEXT:    kmovd %edi, %k16114; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload6115; CHECK-NEXT:    vpshufd $27, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} # 64-byte Folded Reload6116; CHECK-NEXT:    # zmm0 {%k1} = mem[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]6117; CHECK-NEXT:    popq %rax6118; CHECK-NEXT:    .cfi_def_cfa_offset 86119; CHECK-NEXT:    retq6120  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6121  %2 = shufflevector <16 x i32> %a0, <16 x i32> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>6122  %3 = bitcast i16 %mask to <16 x i1>6123  %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> %passthru6124  ret <16 x i32> %46125}6126 6127define <16 x i32> @stack_fold_pshufd_zmm_maskz(<16 x i32> %a0, i16 %mask) {6128; CHECK-LABEL: stack_fold_pshufd_zmm_maskz:6129; CHECK:       # %bb.0:6130; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6131; CHECK-NEXT:    #APP6132; CHECK-NEXT:    nop6133; CHECK-NEXT:    #NO_APP6134; CHECK-NEXT:    kmovd %edi, %k16135; CHECK-NEXT:    vpshufd $27, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 64-byte Folded Reload6136; CHECK-NEXT:    # zmm0 {%k1} {z} = mem[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]6137; CHECK-NEXT:    retq6138  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6139  %2 = shufflevector <16 x i32> %a0, <16 x i32> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>6140  %3 = bitcast i16 %mask to <16 x i1>6141  %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer6142  ret <16 x i32> %46143}6144 6145define <32 x i16> @stack_fold_pshufhw_zmm(<32 x i16> %a0) {6146; CHECK-LABEL: stack_fold_pshufhw_zmm:6147; CHECK:       # %bb.0:6148; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6149; CHECK-NEXT:    #APP6150; CHECK-NEXT:    nop6151; CHECK-NEXT:    #NO_APP6152; CHECK-NEXT:    vpshufhw $27, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload6153; CHECK-NEXT:    # zmm0 = mem[0,1,2,3,7,6,5,4,8,9,10,11,15,14,13,12,16,17,18,19,23,22,21,20,24,25,26,27,31,30,29,28]6154; CHECK-NEXT:    retq6155  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6156  %2 = shufflevector <32 x i16> %a0, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 5, i32 4, i32 8, i32 9, i32 10, i32 11, i32 15, i32 14, i32 13, i32 12, i32 16, i32 17, i32 18, i32 19, i32 23, i32 22, i32 21, i32 20, i32 24, i32 25, i32 26, i32 27, i32 31, i32 30, i32 29, i32 28>6157  ret <32 x i16> %26158}6159 6160define <32 x i16> @stack_fold_pshufhw_zmm_mask(<32 x i16> %passthru, <32 x i16> %a0, i32 %mask) {6161; CHECK-LABEL: stack_fold_pshufhw_zmm_mask:6162; CHECK:       # %bb.0:6163; CHECK-NEXT:    pushq %rax6164; CHECK-NEXT:    .cfi_def_cfa_offset 166165; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6166; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6167; CHECK-NEXT:    #APP6168; CHECK-NEXT:    nop6169; CHECK-NEXT:    #NO_APP6170; CHECK-NEXT:    kmovd %edi, %k16171; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload6172; CHECK-NEXT:    vpshufhw $27, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} # 64-byte Folded Reload6173; CHECK-NEXT:    # zmm0 {%k1} = mem[0,1,2,3,7,6,5,4,8,9,10,11,15,14,13,12,16,17,18,19,23,22,21,20,24,25,26,27,31,30,29,28]6174; CHECK-NEXT:    popq %rax6175; CHECK-NEXT:    .cfi_def_cfa_offset 86176; CHECK-NEXT:    retq6177  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6178  %2 = shufflevector <32 x i16> %a0, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 5, i32 4, i32 8, i32 9, i32 10, i32 11, i32 15, i32 14, i32 13, i32 12, i32 16, i32 17, i32 18, i32 19, i32 23, i32 22, i32 21, i32 20, i32 24, i32 25, i32 26, i32 27, i32 31, i32 30, i32 29, i32 28>6179  %3 = bitcast i32 %mask to <32 x i1>6180  %4 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> %passthru6181  ret <32 x i16> %46182}6183 6184define <32 x i16> @stack_fold_pshufhw_zmm_maskz(<32 x i16> %a0, i32 %mask) {6185; CHECK-LABEL: stack_fold_pshufhw_zmm_maskz:6186; CHECK:       # %bb.0:6187; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6188; CHECK-NEXT:    #APP6189; CHECK-NEXT:    nop6190; CHECK-NEXT:    #NO_APP6191; CHECK-NEXT:    kmovd %edi, %k16192; CHECK-NEXT:    vpshufhw $27, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 64-byte Folded Reload6193; CHECK-NEXT:    # zmm0 {%k1} {z} = mem[0,1,2,3,7,6,5,4,8,9,10,11,15,14,13,12,16,17,18,19,23,22,21,20,24,25,26,27,31,30,29,28]6194; CHECK-NEXT:    retq6195  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6196  %2 = shufflevector <32 x i16> %a0, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 5, i32 4, i32 8, i32 9, i32 10, i32 11, i32 15, i32 14, i32 13, i32 12, i32 16, i32 17, i32 18, i32 19, i32 23, i32 22, i32 21, i32 20, i32 24, i32 25, i32 26, i32 27, i32 31, i32 30, i32 29, i32 28>6197  %3 = bitcast i32 %mask to <32 x i1>6198  %4 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> zeroinitializer6199  ret <32 x i16> %46200}6201 6202define <32 x i16> @stack_fold_pshuflw_zmm(<32 x i16> %a0) {6203; CHECK-LABEL: stack_fold_pshuflw_zmm:6204; CHECK:       # %bb.0:6205; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6206; CHECK-NEXT:    #APP6207; CHECK-NEXT:    nop6208; CHECK-NEXT:    #NO_APP6209; CHECK-NEXT:    vpshuflw $27, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload6210; CHECK-NEXT:    # zmm0 = mem[3,2,1,0,4,5,6,7,11,10,9,8,12,13,14,15,19,18,17,16,20,21,22,23,27,26,25,24,28,29,30,31]6211; CHECK-NEXT:    retq6212  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6213  %2 = shufflevector <32 x i16> %a0, <32 x i16> undef, <32 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 10, i32 9, i32 8, i32 12, i32 13, i32 14, i32 15, i32 19, i32 18, i32 17, i32 16, i32 20, i32 21, i32 22, i32 23, i32 27, i32 26, i32 25, i32 24, i32 28, i32 29, i32 30, i32 31>6214  ret <32 x i16> %26215}6216 6217define <32 x i16> @stack_fold_pshuflw_zmm_mask(<32 x i16> %passthru, <32 x i16> %a0, i32 %mask) {6218; CHECK-LABEL: stack_fold_pshuflw_zmm_mask:6219; CHECK:       # %bb.0:6220; CHECK-NEXT:    pushq %rax6221; CHECK-NEXT:    .cfi_def_cfa_offset 166222; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6223; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6224; CHECK-NEXT:    #APP6225; CHECK-NEXT:    nop6226; CHECK-NEXT:    #NO_APP6227; CHECK-NEXT:    kmovd %edi, %k16228; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload6229; CHECK-NEXT:    vpshuflw $27, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} # 64-byte Folded Reload6230; CHECK-NEXT:    # zmm0 {%k1} = mem[3,2,1,0,4,5,6,7,11,10,9,8,12,13,14,15,19,18,17,16,20,21,22,23,27,26,25,24,28,29,30,31]6231; CHECK-NEXT:    popq %rax6232; CHECK-NEXT:    .cfi_def_cfa_offset 86233; CHECK-NEXT:    retq6234  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6235  %2 = shufflevector <32 x i16> %a0, <32 x i16> undef, <32 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 10, i32 9, i32 8, i32 12, i32 13, i32 14, i32 15, i32 19, i32 18, i32 17, i32 16, i32 20, i32 21, i32 22, i32 23, i32 27, i32 26, i32 25, i32 24, i32 28, i32 29, i32 30, i32 31>6236  %3 = bitcast i32 %mask to <32 x i1>6237  %4 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> %passthru6238  ret <32 x i16> %46239}6240 6241define <32 x i16> @stack_fold_pshuflw_zmm_maskz(<32 x i16> %a0, i32 %mask) {6242; CHECK-LABEL: stack_fold_pshuflw_zmm_maskz:6243; CHECK:       # %bb.0:6244; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6245; CHECK-NEXT:    #APP6246; CHECK-NEXT:    nop6247; CHECK-NEXT:    #NO_APP6248; CHECK-NEXT:    kmovd %edi, %k16249; CHECK-NEXT:    vpshuflw $27, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 64-byte Folded Reload6250; CHECK-NEXT:    # zmm0 {%k1} {z} = mem[3,2,1,0,4,5,6,7,11,10,9,8,12,13,14,15,19,18,17,16,20,21,22,23,27,26,25,24,28,29,30,31]6251; CHECK-NEXT:    retq6252  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6253  %2 = shufflevector <32 x i16> %a0, <32 x i16> undef, <32 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 10, i32 9, i32 8, i32 12, i32 13, i32 14, i32 15, i32 19, i32 18, i32 17, i32 16, i32 20, i32 21, i32 22, i32 23, i32 27, i32 26, i32 25, i32 24, i32 28, i32 29, i32 30, i32 31>6254  %3 = bitcast i32 %mask to <32 x i1>6255  %4 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> zeroinitializer6256  ret <32 x i16> %46257}6258 6259define <16 x i32> @stack_fold_pslld(<16 x i32> %a0, <4 x i32> %a1) {6260; CHECK-LABEL: stack_fold_pslld:6261; CHECK:       # %bb.0:6262; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6263; CHECK-NEXT:    #APP6264; CHECK-NEXT:    nop6265; CHECK-NEXT:    #NO_APP6266; CHECK-NEXT:    vpslld {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 16-byte Folded Reload6267; CHECK-NEXT:    retq6268  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6269  %2 = call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %a0, <4 x i32> %a1)6270  ret <16 x i32> %26271}6272declare <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32>, <4 x i32>) nounwind readnone6273 6274define <16 x i32> @stack_fold_pslld_mask(ptr %passthru, <16 x i32> %a0, <4 x i32> %a1, i16 %mask) {6275; CHECK-LABEL: stack_fold_pslld_mask:6276; CHECK:       # %bb.0:6277; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6278; CHECK-NEXT:    #APP6279; CHECK-NEXT:    nop6280; CHECK-NEXT:    #NO_APP6281; CHECK-NEXT:    kmovd %esi, %k16282; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm26283; CHECK-NEXT:    vpslld {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 16-byte Folded Reload6284; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm06285; CHECK-NEXT:    retq6286  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6287  %2 = call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %a0, <4 x i32> %a1)6288  %3 = bitcast i16 %mask to <16 x i1>6289  %4 = load <16 x i32>, ptr %passthru6290  %5 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> %46291  ret <16 x i32> %56292}6293 6294define <16 x i32> @stack_fold_pslld_maskz(<16 x i32> %a0, <4 x i32> %a1, i16 %mask) {6295; CHECK-LABEL: stack_fold_pslld_maskz:6296; CHECK:       # %bb.0:6297; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6298; CHECK-NEXT:    #APP6299; CHECK-NEXT:    nop6300; CHECK-NEXT:    #NO_APP6301; CHECK-NEXT:    kmovd %edi, %k16302; CHECK-NEXT:    vpslld {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 16-byte Folded Reload6303; CHECK-NEXT:    retq6304  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6305  %2 = call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %a0, <4 x i32> %a1)6306  %3 = bitcast i16 %mask to <16 x i1>6307  %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer6308  ret <16 x i32> %46309}6310 6311define <16 x i32> @stack_fold_pslldi(<16 x i32> %a0) {6312; CHECK-LABEL: stack_fold_pslldi:6313; CHECK:       # %bb.0:6314; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6315; CHECK-NEXT:    #APP6316; CHECK-NEXT:    nop6317; CHECK-NEXT:    #NO_APP6318; CHECK-NEXT:    vpslld $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload6319; CHECK-NEXT:    retq6320  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6321  %2 = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> %a0, i32 1)6322  ret <16 x i32> %26323}6324declare <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32>, i32) nounwind readnone6325 6326define <16 x i32> @stack_fold_pslldi_mask(ptr %passthru, <16 x i32> %a0, i16 %mask) {6327; CHECK-LABEL: stack_fold_pslldi_mask:6328; CHECK:       # %bb.0:6329; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6330; CHECK-NEXT:    #APP6331; CHECK-NEXT:    nop6332; CHECK-NEXT:    #NO_APP6333; CHECK-NEXT:    kmovd %esi, %k16334; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm16335; CHECK-NEXT:    vpslld $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 {%k1} # 64-byte Folded Reload6336; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm06337; CHECK-NEXT:    retq6338  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6339  %2 = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> %a0, i32 1)6340  %3 = bitcast i16 %mask to <16 x i1>6341  %4 = load <16 x i32>, ptr %passthru6342  %5 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> %46343  ret <16 x i32> %56344}6345 6346define <16 x i32> @stack_fold_pslldi_maskz(<16 x i32> %a0, i16 %mask) {6347; CHECK-LABEL: stack_fold_pslldi_maskz:6348; CHECK:       # %bb.0:6349; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6350; CHECK-NEXT:    #APP6351; CHECK-NEXT:    nop6352; CHECK-NEXT:    #NO_APP6353; CHECK-NEXT:    kmovd %edi, %k16354; CHECK-NEXT:    vpslld $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 {%k1} {z} # 64-byte Folded Reload6355; CHECK-NEXT:    retq6356  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6357  %2 = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> %a0, i32 1)6358  %3 = bitcast i16 %mask to <16 x i1>6359  %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer6360  ret <16 x i32> %46361}6362 6363define <64 x i8> @stack_fold_pslldq(<64 x i8> %a, <64 x i8> %b) {6364; CHECK-LABEL: stack_fold_pslldq:6365; CHECK:       # %bb.0:6366; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6367; CHECK-NEXT:    #APP6368; CHECK-NEXT:    nop6369; CHECK-NEXT:    #NO_APP6370; CHECK-NEXT:    vpslldq $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload6371; CHECK-NEXT:    # zmm0 = zero,mem[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14],zero,mem[16,17,18,19,20,21,22,23,24,25,26,27,28,29,30],zero,mem[32,33,34,35,36,37,38,39,40,41,42,43,44,45,46],zero,mem[48,49,50,51,52,53,54,55,56,57,58,59,60,61,62]6372; CHECK-NEXT:    retq6373  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6374  %2 = shufflevector <64 x i8> %a, <64 x i8> zeroinitializer, <64 x i32> <i32 79, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 95, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 111, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 127, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62>6375  ret <64 x i8> %26376}6377 6378define <8 x i64> @stack_fold_psllq(<8 x i64> %a0, <2 x i64> %a1) {6379; CHECK-LABEL: stack_fold_psllq:6380; CHECK:       # %bb.0:6381; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6382; CHECK-NEXT:    #APP6383; CHECK-NEXT:    nop6384; CHECK-NEXT:    #NO_APP6385; CHECK-NEXT:    vpsllq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 16-byte Folded Reload6386; CHECK-NEXT:    retq6387  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6388  %2 = call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> %a0, <2 x i64> %a1)6389  ret <8 x i64> %26390}6391declare <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64>, <2 x i64>) nounwind readnone6392 6393define <8 x i64> @stack_fold_psllqi(<8 x i64> %a0) {6394; CHECK-LABEL: stack_fold_psllqi:6395; CHECK:       # %bb.0:6396; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6397; CHECK-NEXT:    #APP6398; CHECK-NEXT:    nop6399; CHECK-NEXT:    #NO_APP6400; CHECK-NEXT:    vpsllq $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload6401; CHECK-NEXT:    retq6402  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6403  %2 = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> %a0, i32 1)6404  ret <8 x i64> %26405}6406declare <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64>, i32) nounwind readnone6407 6408define <16 x i32> @stack_fold_psllvd(<16 x i32> %a0, <16 x i32> %a1) {6409; CHECK-LABEL: stack_fold_psllvd:6410; CHECK:       # %bb.0:6411; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6412; CHECK-NEXT:    #APP6413; CHECK-NEXT:    nop6414; CHECK-NEXT:    #NO_APP6415; CHECK-NEXT:    vpsllvd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload6416; CHECK-NEXT:    retq6417  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6418  %2 = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %a0, <16 x i32> %a1)6419  ret <16 x i32> %26420}6421declare <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32>, <16 x i32>) nounwind readnone6422 6423define <16 x i32> @stack_fold_psllvd_mask(ptr %passthru, <16 x i32> %a0, <16 x i32> %a1, i16 %mask) {6424; CHECK-LABEL: stack_fold_psllvd_mask:6425; CHECK:       # %bb.0:6426; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6427; CHECK-NEXT:    #APP6428; CHECK-NEXT:    nop6429; CHECK-NEXT:    #NO_APP6430; CHECK-NEXT:    kmovd %esi, %k16431; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm26432; CHECK-NEXT:    vpsllvd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload6433; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm06434; CHECK-NEXT:    retq6435  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6436  %2 = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %a0, <16 x i32> %a1)6437  %3 = bitcast i16 %mask to <16 x i1>6438  %4 = load <16 x i32>, ptr %passthru6439  %5 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> %46440  ret <16 x i32> %56441}6442 6443define <16 x i32> @stack_fold_psllvd_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {6444; CHECK-LABEL: stack_fold_psllvd_maskz:6445; CHECK:       # %bb.0:6446; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6447; CHECK-NEXT:    #APP6448; CHECK-NEXT:    nop6449; CHECK-NEXT:    #NO_APP6450; CHECK-NEXT:    kmovd %edi, %k16451; CHECK-NEXT:    vpsllvd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload6452; CHECK-NEXT:    retq6453  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6454  %2 = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %a0, <16 x i32> %a1)6455  %3 = bitcast i16 %mask to <16 x i1>6456  %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer6457  ret <16 x i32> %46458}6459 6460define <8 x i64> @stack_fold_psllvq(<8 x i64> %a0, <8 x i64> %a1) {6461; CHECK-LABEL: stack_fold_psllvq:6462; CHECK:       # %bb.0:6463; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6464; CHECK-NEXT:    #APP6465; CHECK-NEXT:    nop6466; CHECK-NEXT:    #NO_APP6467; CHECK-NEXT:    vpsllvq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload6468; CHECK-NEXT:    retq6469  %1 = tail call <8 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6470  %2 = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> %a0, <8 x i64> %a1)6471  ret <8 x i64> %26472}6473declare <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64>, <8 x i64>) nounwind readnone6474 6475define <32 x i16> @stack_fold_psllvw(<32 x i16> %a0, <32 x i16> %a1) {6476; CHECK-LABEL: stack_fold_psllvw:6477; CHECK:       # %bb.0:6478; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6479; CHECK-NEXT:    #APP6480; CHECK-NEXT:    nop6481; CHECK-NEXT:    #NO_APP6482; CHECK-NEXT:    vpsllvw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload6483; CHECK-NEXT:    retq6484  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6485  %2 = call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %a0, <32 x i16> %a1)6486  ret <32 x i16> %26487}6488declare <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16>, <32 x i16>) nounwind readnone6489 6490define <32 x i16> @stack_fold_psllw(<32 x i16> %a0, <8 x i16> %a1) {6491; CHECK-LABEL: stack_fold_psllw:6492; CHECK:       # %bb.0:6493; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6494; CHECK-NEXT:    #APP6495; CHECK-NEXT:    nop6496; CHECK-NEXT:    #NO_APP6497; CHECK-NEXT:    vpsllw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 16-byte Folded Reload6498; CHECK-NEXT:    retq6499  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6500  %2 = call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> %a0, <8 x i16> %a1)6501  ret <32 x i16> %26502}6503declare <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16>, <8 x i16>) nounwind readnone6504 6505define <32 x i16> @stack_fold_psllwi(<32 x i16> %a0) {6506; CHECK-LABEL: stack_fold_psllwi:6507; CHECK:       # %bb.0:6508; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6509; CHECK-NEXT:    #APP6510; CHECK-NEXT:    nop6511; CHECK-NEXT:    #NO_APP6512; CHECK-NEXT:    vpsllw $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload6513; CHECK-NEXT:    retq6514  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6515  %2 = call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> %a0, i32 1)6516  ret <32 x i16> %26517}6518declare <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16>, i32) nounwind readnone6519 6520define <16 x i32> @stack_fold_psrad(<16 x i32> %a0, <4 x i32> %a1) {6521; CHECK-LABEL: stack_fold_psrad:6522; CHECK:       # %bb.0:6523; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6524; CHECK-NEXT:    #APP6525; CHECK-NEXT:    nop6526; CHECK-NEXT:    #NO_APP6527; CHECK-NEXT:    vpsrad {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 16-byte Folded Reload6528; CHECK-NEXT:    retq6529  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6530  %2 = call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %a0, <4 x i32> %a1)6531  ret <16 x i32> %26532}6533declare <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32>, <4 x i32>) nounwind readnone6534 6535define <16 x i32> @stack_fold_psradi(<16 x i32> %a0) {6536; CHECK-LABEL: stack_fold_psradi:6537; CHECK:       # %bb.0:6538; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6539; CHECK-NEXT:    #APP6540; CHECK-NEXT:    nop6541; CHECK-NEXT:    #NO_APP6542; CHECK-NEXT:    vpsrad $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload6543; CHECK-NEXT:    retq6544  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6545  %2 = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %a0, i32 1)6546  ret <16 x i32> %26547}6548declare <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32>, i32) nounwind readnone6549 6550define <8 x i64> @stack_fold_psraq(<8 x i64> %a0, <2 x i64> %a1) {6551; CHECK-LABEL: stack_fold_psraq:6552; CHECK:       # %bb.0:6553; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6554; CHECK-NEXT:    #APP6555; CHECK-NEXT:    nop6556; CHECK-NEXT:    #NO_APP6557; CHECK-NEXT:    vpsraq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 16-byte Folded Reload6558; CHECK-NEXT:    retq6559  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6560  %2 = call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> %a0, <2 x i64> %a1)6561  ret <8 x i64> %26562}6563declare <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64>, <2 x i64>) nounwind readnone6564 6565define <8 x i64> @stack_fold_psraqi(<8 x i64> %a0) {6566; CHECK-LABEL: stack_fold_psraqi:6567; CHECK:       # %bb.0:6568; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6569; CHECK-NEXT:    #APP6570; CHECK-NEXT:    nop6571; CHECK-NEXT:    #NO_APP6572; CHECK-NEXT:    vpsraq $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload6573; CHECK-NEXT:    retq6574  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6575  %2 = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> %a0, i32 1)6576  ret <8 x i64> %26577}6578declare <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64>, i32) nounwind readnone6579 6580define <16 x i32> @stack_fold_psravd(<16 x i32> %a0, <16 x i32> %a1) {6581; CHECK-LABEL: stack_fold_psravd:6582; CHECK:       # %bb.0:6583; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6584; CHECK-NEXT:    #APP6585; CHECK-NEXT:    nop6586; CHECK-NEXT:    #NO_APP6587; CHECK-NEXT:    vpsravd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload6588; CHECK-NEXT:    retq6589  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6590  %2 = call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> %a0, <16 x i32> %a1)6591  ret <16 x i32> %26592}6593declare <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32>, <16 x i32>) nounwind readnone6594 6595define <8 x i64> @stack_fold_psravq(<8 x i64> %a0, <8 x i64> %a1) {6596; CHECK-LABEL: stack_fold_psravq:6597; CHECK:       # %bb.0:6598; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6599; CHECK-NEXT:    #APP6600; CHECK-NEXT:    nop6601; CHECK-NEXT:    #NO_APP6602; CHECK-NEXT:    vpsravq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload6603; CHECK-NEXT:    retq6604  %1 = tail call <8 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6605  %2 = call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> %a0, <8 x i64> %a1)6606  ret <8 x i64> %26607}6608declare <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64>, <8 x i64>) nounwind readnone6609 6610define <32 x i16> @stack_fold_psravw(<32 x i16> %a0, <32 x i16> %a1) {6611; CHECK-LABEL: stack_fold_psravw:6612; CHECK:       # %bb.0:6613; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6614; CHECK-NEXT:    #APP6615; CHECK-NEXT:    nop6616; CHECK-NEXT:    #NO_APP6617; CHECK-NEXT:    vpsravw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload6618; CHECK-NEXT:    retq6619  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6620  %2 = call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %a0, <32 x i16> %a1)6621  ret <32 x i16> %26622}6623declare <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16>, <32 x i16>) nounwind readnone6624 6625define <32 x i16> @stack_fold_psraw(<32 x i16> %a0, <8 x i16> %a1) {6626; CHECK-LABEL: stack_fold_psraw:6627; CHECK:       # %bb.0:6628; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6629; CHECK-NEXT:    #APP6630; CHECK-NEXT:    nop6631; CHECK-NEXT:    #NO_APP6632; CHECK-NEXT:    vpsraw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 16-byte Folded Reload6633; CHECK-NEXT:    retq6634  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6635  %2 = call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %a0, <8 x i16> %a1)6636  ret <32 x i16> %26637}6638declare <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16>, <8 x i16>) nounwind readnone6639 6640define <32 x i16> @stack_fold_psrawi(<32 x i16> %a0) {6641; CHECK-LABEL: stack_fold_psrawi:6642; CHECK:       # %bb.0:6643; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6644; CHECK-NEXT:    #APP6645; CHECK-NEXT:    nop6646; CHECK-NEXT:    #NO_APP6647; CHECK-NEXT:    vpsraw $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload6648; CHECK-NEXT:    retq6649  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6650  %2 = call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> %a0, i32 1)6651  ret <32 x i16> %26652}6653declare <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16>, i32) nounwind readnone6654 6655define <16 x i32> @stack_fold_psrld(<16 x i32> %a0, <4 x i32> %a1) {6656; CHECK-LABEL: stack_fold_psrld:6657; CHECK:       # %bb.0:6658; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6659; CHECK-NEXT:    #APP6660; CHECK-NEXT:    nop6661; CHECK-NEXT:    #NO_APP6662; CHECK-NEXT:    vpsrld {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 16-byte Folded Reload6663; CHECK-NEXT:    retq6664  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6665  %2 = call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %a0, <4 x i32> %a1)6666  ret <16 x i32> %26667}6668declare <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32>, <4 x i32>) nounwind readnone6669 6670define <16 x i32> @stack_fold_psrldi(<16 x i32> %a0) {6671; CHECK-LABEL: stack_fold_psrldi:6672; CHECK:       # %bb.0:6673; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6674; CHECK-NEXT:    #APP6675; CHECK-NEXT:    nop6676; CHECK-NEXT:    #NO_APP6677; CHECK-NEXT:    vpsrld $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload6678; CHECK-NEXT:    retq6679  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6680  %2 = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> %a0, i32 1)6681  ret <16 x i32> %26682}6683declare <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32>, i32) nounwind readnone6684 6685define <64 x i8> @stack_fold_psrldq(<64 x i8> %a, <64 x i8> %b) {6686; CHECK-LABEL: stack_fold_psrldq:6687; CHECK:       # %bb.0:6688; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6689; CHECK-NEXT:    #APP6690; CHECK-NEXT:    nop6691; CHECK-NEXT:    #NO_APP6692; CHECK-NEXT:    vpsrldq $2, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload6693; CHECK-NEXT:    # zmm0 = mem[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,mem[18,19,20,21,22,23,24,25,26,27,28,29,30,31],zero,zero,mem[34,35,36,37,38,39,40,41,42,43,44,45,46,47],zero,zero,mem[50,51,52,53,54,55,56,57,58,59,60,61,62,63],zero,zero6694; CHECK-NEXT:    retq6695  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6696  %2 = shufflevector <64 x i8> %a, <64 x i8> zeroinitializer, <64 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 64, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 64, i32 64, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 64, i32 64, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 64>6697  ret <64 x i8> %26698}6699 6700define <8 x i64> @stack_fold_psrlq(<8 x i64> %a0, <2 x i64> %a1) {6701; CHECK-LABEL: stack_fold_psrlq:6702; CHECK:       # %bb.0:6703; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6704; CHECK-NEXT:    #APP6705; CHECK-NEXT:    nop6706; CHECK-NEXT:    #NO_APP6707; CHECK-NEXT:    vpsrlq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 16-byte Folded Reload6708; CHECK-NEXT:    retq6709  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6710  %2 = call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> %a0, <2 x i64> %a1)6711  ret <8 x i64> %26712}6713declare <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64>, <2 x i64>) nounwind readnone6714 6715define <8 x i64> @stack_fold_psrlqi(<8 x i64> %a0) {6716; CHECK-LABEL: stack_fold_psrlqi:6717; CHECK:       # %bb.0:6718; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6719; CHECK-NEXT:    #APP6720; CHECK-NEXT:    nop6721; CHECK-NEXT:    #NO_APP6722; CHECK-NEXT:    vpsrlq $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload6723; CHECK-NEXT:    retq6724  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6725  %2 = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> %a0, i32 1)6726  ret <8 x i64> %26727}6728declare <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64>, i32) nounwind readnone6729 6730define <16 x i32> @stack_fold_psrlvd(<16 x i32> %a0, <16 x i32> %a1) {6731; CHECK-LABEL: stack_fold_psrlvd:6732; CHECK:       # %bb.0:6733; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6734; CHECK-NEXT:    #APP6735; CHECK-NEXT:    nop6736; CHECK-NEXT:    #NO_APP6737; CHECK-NEXT:    vpsrlvd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload6738; CHECK-NEXT:    retq6739  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6740  %2 = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> %a0, <16 x i32> %a1)6741  ret <16 x i32> %26742}6743declare <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32>, <16 x i32>) nounwind readnone6744 6745define <8 x i64> @stack_fold_psrlvq(<8 x i64> %a0, <8 x i64> %a1) {6746; CHECK-LABEL: stack_fold_psrlvq:6747; CHECK:       # %bb.0:6748; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6749; CHECK-NEXT:    #APP6750; CHECK-NEXT:    nop6751; CHECK-NEXT:    #NO_APP6752; CHECK-NEXT:    vpsrlvq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload6753; CHECK-NEXT:    retq6754  %1 = tail call <8 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6755  %2 = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> %a0, <8 x i64> %a1)6756  ret <8 x i64> %26757}6758declare <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64>, <8 x i64>) nounwind readnone6759 6760define <32 x i16> @stack_fold_psrlvw(<32 x i16> %a0, <32 x i16> %a1) {6761; CHECK-LABEL: stack_fold_psrlvw:6762; CHECK:       # %bb.0:6763; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6764; CHECK-NEXT:    #APP6765; CHECK-NEXT:    nop6766; CHECK-NEXT:    #NO_APP6767; CHECK-NEXT:    vpsrlvw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload6768; CHECK-NEXT:    retq6769  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6770  %2 = call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %a0, <32 x i16> %a1)6771  ret <32 x i16> %26772}6773declare <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16>, <32 x i16>) nounwind readnone6774 6775define <32 x i16> @stack_fold_psrlw(<32 x i16> %a0, <8 x i16> %a1) {6776; CHECK-LABEL: stack_fold_psrlw:6777; CHECK:       # %bb.0:6778; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill6779; CHECK-NEXT:    #APP6780; CHECK-NEXT:    nop6781; CHECK-NEXT:    #NO_APP6782; CHECK-NEXT:    vpsrlw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 16-byte Folded Reload6783; CHECK-NEXT:    retq6784  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6785  %2 = call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %a0, <8 x i16> %a1)6786  ret <32 x i16> %26787}6788declare <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16>, <8 x i16>) nounwind readnone6789 6790define <32 x i16> @stack_fold_psrlwi(<32 x i16> %a0) {6791; CHECK-LABEL: stack_fold_psrlwi:6792; CHECK:       # %bb.0:6793; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6794; CHECK-NEXT:    #APP6795; CHECK-NEXT:    nop6796; CHECK-NEXT:    #NO_APP6797; CHECK-NEXT:    vpsrlw $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Folded Reload6798; CHECK-NEXT:    retq6799  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6800  %2 = call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> %a0, i32 1)6801  ret <32 x i16> %26802}6803declare <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16>, i32) nounwind readnone6804 6805define <64 x i8> @stack_fold_psubb(<64 x i8> %a0, <64 x i8> %a1) {6806; CHECK-LABEL: stack_fold_psubb:6807; CHECK:       # %bb.0:6808; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6809; CHECK-NEXT:    #APP6810; CHECK-NEXT:    nop6811; CHECK-NEXT:    #NO_APP6812; CHECK-NEXT:    vpsubb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload6813; CHECK-NEXT:    retq6814  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6815  %2 = sub <64 x i8> %a0, %a16816  ret <64 x i8> %26817}6818 6819define <16 x i32> @stack_fold_psubd(<16 x i32> %a0, <16 x i32> %a1) {6820; CHECK-LABEL: stack_fold_psubd:6821; CHECK:       # %bb.0:6822; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6823; CHECK-NEXT:    #APP6824; CHECK-NEXT:    nop6825; CHECK-NEXT:    #NO_APP6826; CHECK-NEXT:    vpsubd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload6827; CHECK-NEXT:    retq6828  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6829  %2 = sub <16 x i32> %a0, %a16830  ret <16 x i32> %26831}6832 6833define <8 x i64> @stack_fold_psubq(<8 x i64> %a0, <8 x i64> %a1) {6834; CHECK-LABEL: stack_fold_psubq:6835; CHECK:       # %bb.0:6836; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6837; CHECK-NEXT:    #APP6838; CHECK-NEXT:    nop6839; CHECK-NEXT:    #NO_APP6840; CHECK-NEXT:    vpsubq {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload6841; CHECK-NEXT:    retq6842  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6843  %2 = sub <8 x i64> %a0, %a16844  ret <8 x i64> %26845}6846 6847define <64 x i8> @stack_fold_psubsb(<64 x i8> %a0, <64 x i8> %a1) {6848; CHECK-LABEL: stack_fold_psubsb:6849; CHECK:       # %bb.0:6850; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6851; CHECK-NEXT:    #APP6852; CHECK-NEXT:    nop6853; CHECK-NEXT:    #NO_APP6854; CHECK-NEXT:    vpsubsb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload6855; CHECK-NEXT:    retq6856  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6857  %2 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> %a0, <64 x i8> %a1)6858  ret <64 x i8> %26859}6860 6861define <32 x i16> @stack_fold_psubsw(<32 x i16> %a0, <32 x i16> %a1) {6862; CHECK-LABEL: stack_fold_psubsw:6863; CHECK:       # %bb.0:6864; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6865; CHECK-NEXT:    #APP6866; CHECK-NEXT:    nop6867; CHECK-NEXT:    #NO_APP6868; CHECK-NEXT:    vpsubsw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload6869; CHECK-NEXT:    retq6870  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6871  %2 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> %a0, <32 x i16> %a1)6872  ret <32 x i16> %26873}6874 6875define <64 x i8> @stack_fold_psubusb(<64 x i8> %a0, <64 x i8> %a1) {6876; CHECK-LABEL: stack_fold_psubusb:6877; CHECK:       # %bb.0:6878; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6879; CHECK-NEXT:    #APP6880; CHECK-NEXT:    nop6881; CHECK-NEXT:    #NO_APP6882; CHECK-NEXT:    vpsubusb {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload6883; CHECK-NEXT:    retq6884  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6885  %2 = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> %a0, <64 x i8> %a1)6886  ret <64 x i8> %26887}6888 6889define <32 x i16> @stack_fold_psubusw(<32 x i16> %a0, <32 x i16> %a1) {6890; CHECK-LABEL: stack_fold_psubusw:6891; CHECK:       # %bb.0:6892; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6893; CHECK-NEXT:    #APP6894; CHECK-NEXT:    nop6895; CHECK-NEXT:    #NO_APP6896; CHECK-NEXT:    vpsubusw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload6897; CHECK-NEXT:    retq6898  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6899  %2 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> %a0, <32 x i16> %a1)6900  ret <32 x i16> %26901}6902 6903define <32 x i16> @stack_fold_psubw(<32 x i16> %a0, <32 x i16> %a1) {6904; CHECK-LABEL: stack_fold_psubw:6905; CHECK:       # %bb.0:6906; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6907; CHECK-NEXT:    #APP6908; CHECK-NEXT:    nop6909; CHECK-NEXT:    #NO_APP6910; CHECK-NEXT:    vpsubw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload6911; CHECK-NEXT:    retq6912  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6913  %2 = sub <32 x i16> %a0, %a16914  ret <32 x i16> %26915}6916 6917define <8 x i64> @stack_fold_shufi64x2(<8 x i64> %a, <8 x i64> %b) {6918; CHECK-LABEL: stack_fold_shufi64x2:6919; CHECK:       # %bb.0:6920; CHECK-NEXT:    pushq %rax6921; CHECK-NEXT:    .cfi_def_cfa_offset 166922; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6923; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6924; CHECK-NEXT:    #APP6925; CHECK-NEXT:    nop6926; CHECK-NEXT:    #NO_APP6927; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload6928; CHECK-NEXT:    vshufi64x2 $24, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload6929; CHECK-NEXT:    # zmm0 = zmm0[0,1,4,5],mem[2,3,0,1]6930; CHECK-NEXT:    popq %rax6931; CHECK-NEXT:    .cfi_def_cfa_offset 86932; CHECK-NEXT:    retq6933  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6934  %2 = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 1, i32 4, i32 5, i32 10, i32 11, i32 8, i32 9>6935  ret <8 x i64> %26936}6937 6938define <8 x i64> @stack_fold_shufi64x2_mask(<8 x i64> %a, <8 x i64> %b, i8 %mask, ptr %passthru) {6939; CHECK-LABEL: stack_fold_shufi64x2_mask:6940; CHECK:       # %bb.0:6941; CHECK-NEXT:    pushq %rax6942; CHECK-NEXT:    .cfi_def_cfa_offset 166943; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6944; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6945; CHECK-NEXT:    #APP6946; CHECK-NEXT:    nop6947; CHECK-NEXT:    #NO_APP6948; CHECK-NEXT:    kmovd %edi, %k16949; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm16950; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload6951; CHECK-NEXT:    vshufi64x2 $24, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 {%k1} # 64-byte Folded Reload6952; CHECK-NEXT:    # zmm1 {%k1} = zmm0[0,1,4,5],mem[2,3,0,1]6953; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm06954; CHECK-NEXT:    popq %rax6955; CHECK-NEXT:    .cfi_def_cfa_offset 86956; CHECK-NEXT:    retq6957  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6958  %2 = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 1, i32 4, i32 5, i32 10, i32 11, i32 8, i32 9>6959  %3 = bitcast i8 %mask to <8 x i1>6960  ; load needed to keep the operation from being scheduled above the asm block6961  %4 = load <8 x i64>, ptr %passthru6962  %5 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> %46963  ret <8 x i64> %56964}6965 6966define <8 x i64> @stack_fold_shufi64x2_maskz(<8 x i64> %a, <8 x i64> %b, i8 %mask, ptr %passthru) {6967; CHECK-LABEL: stack_fold_shufi64x2_maskz:6968; CHECK:       # %bb.0:6969; CHECK-NEXT:    pushq %rax6970; CHECK-NEXT:    .cfi_def_cfa_offset 166971; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6972; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6973; CHECK-NEXT:    #APP6974; CHECK-NEXT:    nop6975; CHECK-NEXT:    #NO_APP6976; CHECK-NEXT:    kmovd %edi, %k16977; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload6978; CHECK-NEXT:    vshufi64x2 $24, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload6979; CHECK-NEXT:    # zmm0 {%k1} {z} = zmm0[0,1,4,5],mem[2,3,0,1]6980; CHECK-NEXT:    popq %rax6981; CHECK-NEXT:    .cfi_def_cfa_offset 86982; CHECK-NEXT:    retq6983  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()6984  %2 = shufflevector <8 x i64> %a, <8 x i64> %b, <8 x i32> <i32 0, i32 1, i32 4, i32 5, i32 10, i32 11, i32 8, i32 9>6985  %3 = bitcast i8 %mask to <8 x i1>6986  %4 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> zeroinitializer6987  ret <8 x i64> %46988}6989 6990define <16 x i32> @stack_fold_shufi32x4_mask(<16 x i32> %a, <16 x i32> %b, i16 %mask, ptr %passthru) {6991; CHECK-LABEL: stack_fold_shufi32x4_mask:6992; CHECK:       # %bb.0:6993; CHECK-NEXT:    pushq %rax6994; CHECK-NEXT:    .cfi_def_cfa_offset 166995; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6996; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill6997; CHECK-NEXT:    #APP6998; CHECK-NEXT:    nop6999; CHECK-NEXT:    #NO_APP7000; CHECK-NEXT:    kmovd %edi, %k17001; CHECK-NEXT:    vmovdqa64 (%rsi), %zmm17002; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload7003; CHECK-NEXT:    vshufi32x4 $20, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 {%k1} # 64-byte Folded Reload7004; CHECK-NEXT:    # zmm1 {%k1} = zmm0[0,1,2,3,4,5,6,7],mem[4,5,6,7,0,1,2,3]7005; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm07006; CHECK-NEXT:    popq %rax7007; CHECK-NEXT:    .cfi_def_cfa_offset 87008; CHECK-NEXT:    retq7009  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()7010  %2 = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 18, i32 19>7011  %3 = bitcast i16 %mask to <16 x i1>7012  ; load needed to keep the operation from being scheduled above the asm block7013  %4 = load <16 x i32>, ptr %passthru7014  %5 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> %47015  ret <16 x i32> %57016}7017 7018define <16 x i32> @stack_fold_shufi32x4_maskz(<16 x i32> %a, <16 x i32> %b, i16 %mask) {7019; CHECK-LABEL: stack_fold_shufi32x4_maskz:7020; CHECK:       # %bb.0:7021; CHECK-NEXT:    pushq %rax7022; CHECK-NEXT:    .cfi_def_cfa_offset 167023; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7024; CHECK-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7025; CHECK-NEXT:    #APP7026; CHECK-NEXT:    nop7027; CHECK-NEXT:    #NO_APP7028; CHECK-NEXT:    kmovd %edi, %k17029; CHECK-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload7030; CHECK-NEXT:    vshufi32x4 $20, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload7031; CHECK-NEXT:    # zmm0 {%k1} {z} = zmm0[0,1,2,3,4,5,6,7],mem[4,5,6,7,0,1,2,3]7032; CHECK-NEXT:    popq %rax7033; CHECK-NEXT:    .cfi_def_cfa_offset 87034; CHECK-NEXT:    retq7035  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()7036  %2 = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 16, i32 17, i32 18, i32 19>7037  %3 = bitcast i16 %mask to <16 x i1>7038  %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer7039  ret <16 x i32> %47040}7041 7042define <16 x i32> @stack_fold_ternlogd(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) {7043; CHECK-LABEL: stack_fold_ternlogd:7044; CHECK:       # %bb.0:7045; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7046; CHECK-NEXT:    #APP7047; CHECK-NEXT:    nop7048; CHECK-NEXT:    #NO_APP7049; CHECK-NEXT:    vpternlogd $33, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload7050; CHECK-NEXT:    # zmm0 = ~(zmm1 | (zmm0 ^ mem))7051; CHECK-NEXT:    retq7052  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()7053  %2 = call <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i32 33)7054  ret <16 x i32> %27055}7056declare <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i32)7057 7058define <8 x i64> @stack_fold_ternlogq(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2) {7059; CHECK-LABEL: stack_fold_ternlogq:7060; CHECK:       # %bb.0:7061; CHECK-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7062; CHECK-NEXT:    #APP7063; CHECK-NEXT:    nop7064; CHECK-NEXT:    #NO_APP7065; CHECK-NEXT:    vpternlogq $33, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload7066; CHECK-NEXT:    # zmm0 = ~(zmm1 | (zmm0 ^ mem))7067; CHECK-NEXT:    retq7068  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()7069  %2 = call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33)7070  ret <8 x i64> %27071}7072 7073declare <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i32)7074 7075define <64 x i8> @stack_fold_punpckhbw_zmm(<64 x i8> %a0, <64 x i8> %a1) {7076; CHECK-LABEL: stack_fold_punpckhbw_zmm:7077; CHECK:       # %bb.0:7078; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7079; CHECK-NEXT:    #APP7080; CHECK-NEXT:    nop7081; CHECK-NEXT:    #NO_APP7082; CHECK-NEXT:    vpunpckhbw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload7083; CHECK-NEXT:    # zmm0 = zmm0[8],mem[8],zmm0[9],mem[9],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[12],mem[12],zmm0[13],mem[13],zmm0[14],mem[14],zmm0[15],mem[15],zmm0[24],mem[24],zmm0[25],mem[25],zmm0[26],mem[26],zmm0[27],mem[27],zmm0[28],mem[28],zmm0[29],mem[29],zmm0[30],mem[30],zmm0[31],mem[31],zmm0[40],mem[40],zmm0[41],mem[41],zmm0[42],mem[42],zmm0[43],mem[43],zmm0[44],mem[44],zmm0[45],mem[45],zmm0[46],mem[46],zmm0[47],mem[47],zmm0[56],mem[56],zmm0[57],mem[57],zmm0[58],mem[58],zmm0[59],mem[59],zmm0[60],mem[60],zmm0[61],mem[61],zmm0[62],mem[62],zmm0[63],mem[63]7084; CHECK-NEXT:    retq7085  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()7086  %2 = shufflevector <64 x i8> %a0, <64 x i8> %a1, <64 x i32> <i32 8, i32 72, i32 9, i32 73, i32 10, i32 74, i32 11, i32 75, i32 12, i32 76, i32 13, i32 77, i32 14, i32 78, i32 15, i32 79, i32 24, i32 88, i32 25, i32 89, i32 26, i32 90, i32 27, i32 91, i32 28, i32 92, i32 29, i32 93, i32 30, i32 94, i32 31, i32 95, i32 40, i32 104, i32 41, i32 105, i32 42, i32 106, i32 43, i32 107, i32 44, i32 108, i32 45, i32 109, i32 46, i32 110, i32 47, i32 111, i32 56, i32 120, i32 57, i32 121, i32 58, i32 122, i32 59, i32 123, i32 60, i32 124, i32 61, i32 125, i32 62, i32 126, i32 63, i32 127>7087  ret <64 x i8> %27088}7089 7090define <64 x i8> @stack_fold_punpckhbw_mask_zmm(ptr %passthru, <64 x i8> %a0, <64 x i8> %a1, i64 %mask) {7091; CHECK-LABEL: stack_fold_punpckhbw_mask_zmm:7092; CHECK:       # %bb.0:7093; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7094; CHECK-NEXT:    #APP7095; CHECK-NEXT:    nop7096; CHECK-NEXT:    #NO_APP7097; CHECK-NEXT:    kmovq %rsi, %k17098; CHECK-NEXT:    vmovdqa64 (%rdi), %zmm27099; CHECK-NEXT:    vpunpckhbw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 {%k1} # 64-byte Folded Reload7100; CHECK-NEXT:    # zmm2 {%k1} = zmm0[8],mem[8],zmm0[9],mem[9],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[12],mem[12],zmm0[13],mem[13],zmm0[14],mem[14],zmm0[15],mem[15],zmm0[24],mem[24],zmm0[25],mem[25],zmm0[26],mem[26],zmm0[27],mem[27],zmm0[28],mem[28],zmm0[29],mem[29],zmm0[30],mem[30],zmm0[31],mem[31],zmm0[40],mem[40],zmm0[41],mem[41],zmm0[42],mem[42],zmm0[43],mem[43],zmm0[44],mem[44],zmm0[45],mem[45],zmm0[46],mem[46],zmm0[47],mem[47],zmm0[56],mem[56],zmm0[57],mem[57],zmm0[58],mem[58],zmm0[59],mem[59],zmm0[60],mem[60],zmm0[61],mem[61],zmm0[62],mem[62],zmm0[63],mem[63]7101; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm07102; CHECK-NEXT:    retq7103  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()7104  %2 = shufflevector <64 x i8> %a0, <64 x i8> %a1, <64 x i32> <i32 8, i32 72, i32 9, i32 73, i32 10, i32 74, i32 11, i32 75, i32 12, i32 76, i32 13, i32 77, i32 14, i32 78, i32 15, i32 79, i32 24, i32 88, i32 25, i32 89, i32 26, i32 90, i32 27, i32 91, i32 28, i32 92, i32 29, i32 93, i32 30, i32 94, i32 31, i32 95, i32 40, i32 104, i32 41, i32 105, i32 42, i32 106, i32 43, i32 107, i32 44, i32 108, i32 45, i32 109, i32 46, i32 110, i32 47, i32 111, i32 56, i32 120, i32 57, i32 121, i32 58, i32 122, i32 59, i32 123, i32 60, i32 124, i32 61, i32 125, i32 62, i32 126, i32 63, i32 127>7105  %3 = bitcast i64 %mask to <64 x i1>7106  ; load needed to keep the operation from being scheduled about the asm block7107  %4 = load <64 x i8>, ptr %passthru7108  %5 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> %47109  ret <64 x i8> %57110}7111 7112define <64 x i8> @stack_fold_punpckhbw_maskz_zmm(<64 x i8> %a0, <64 x i8> %a1, i64 %mask) {7113; CHECK-LABEL: stack_fold_punpckhbw_maskz_zmm:7114; CHECK:       # %bb.0:7115; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7116; CHECK-NEXT:    #APP7117; CHECK-NEXT:    nop7118; CHECK-NEXT:    #NO_APP7119; CHECK-NEXT:    kmovq %rdi, %k17120; CHECK-NEXT:    vpunpckhbw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload7121; CHECK-NEXT:    # zmm0 {%k1} {z} = zmm0[8],mem[8],zmm0[9],mem[9],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[12],mem[12],zmm0[13],mem[13],zmm0[14],mem[14],zmm0[15],mem[15],zmm0[24],mem[24],zmm0[25],mem[25],zmm0[26],mem[26],zmm0[27],mem[27],zmm0[28],mem[28],zmm0[29],mem[29],zmm0[30],mem[30],zmm0[31],mem[31],zmm0[40],mem[40],zmm0[41],mem[41],zmm0[42],mem[42],zmm0[43],mem[43],zmm0[44],mem[44],zmm0[45],mem[45],zmm0[46],mem[46],zmm0[47],mem[47],zmm0[56],mem[56],zmm0[57],mem[57],zmm0[58],mem[58],zmm0[59],mem[59],zmm0[60],mem[60],zmm0[61],mem[61],zmm0[62],mem[62],zmm0[63],mem[63]7122; CHECK-NEXT:    retq7123  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()7124  %2 = shufflevector <64 x i8> %a0, <64 x i8> %a1, <64 x i32> <i32 8, i32 72, i32 9, i32 73, i32 10, i32 74, i32 11, i32 75, i32 12, i32 76, i32 13, i32 77, i32 14, i32 78, i32 15, i32 79, i32 24, i32 88, i32 25, i32 89, i32 26, i32 90, i32 27, i32 91, i32 28, i32 92, i32 29, i32 93, i32 30, i32 94, i32 31, i32 95, i32 40, i32 104, i32 41, i32 105, i32 42, i32 106, i32 43, i32 107, i32 44, i32 108, i32 45, i32 109, i32 46, i32 110, i32 47, i32 111, i32 56, i32 120, i32 57, i32 121, i32 58, i32 122, i32 59, i32 123, i32 60, i32 124, i32 61, i32 125, i32 62, i32 126, i32 63, i32 127>7125  %3 = bitcast i64 %mask to <64 x i1>7126  %4 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> zeroinitializer7127  ret <64 x i8> %47128}7129 7130define <16 x i32> @stack_fold_pxord(<16 x i32> %a0, <16 x i32> %a1) {7131; CHECK-LABEL: stack_fold_pxord:7132; CHECK:       # %bb.0:7133; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7134; CHECK-NEXT:    #APP7135; CHECK-NEXT:    nop7136; CHECK-NEXT:    #NO_APP7137; CHECK-NEXT:    vxorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload7138; CHECK-NEXT:    retq7139  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()7140  %2 = xor <16 x i32> %a0, %a17141  ret <16 x i32> %27142}7143 7144define <16 x i32> @stack_fold_pxord_commuted(<16 x i32> %a0, <16 x i32> %a1) {7145; CHECK-LABEL: stack_fold_pxord_commuted:7146; CHECK:       # %bb.0:7147; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7148; CHECK-NEXT:    #APP7149; CHECK-NEXT:    nop7150; CHECK-NEXT:    #NO_APP7151; CHECK-NEXT:    vxorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload7152; CHECK-NEXT:    retq7153  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()7154  %2 = xor <16 x i32> %a1, %a07155  ret <16 x i32> %27156}7157 7158define <16 x i32> @stack_fold_pxord_mask(<16 x i32> %a0, <16 x i32> %a1, ptr %a2, i16 %mask) {7159; CHECK-LABEL: stack_fold_pxord_mask:7160; CHECK:       # %bb.0:7161; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7162; CHECK-NEXT:    vmovaps %zmm0, %zmm17163; CHECK-NEXT:    #APP7164; CHECK-NEXT:    nop7165; CHECK-NEXT:    #NO_APP7166; CHECK-NEXT:    kmovd %esi, %k17167; CHECK-NEXT:    vmovaps (%rdi), %zmm07168; CHECK-NEXT:    vxorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload7169; CHECK-NEXT:    retq7170  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()7171  %2 = xor <16 x i32> %a0, %a17172  %3 = bitcast i16 %mask to <16 x i1>7173  ; load needed to keep the operation from being scheduled about the asm block7174  %4 = load <16 x i32>, ptr %a27175  %5 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> %47176  ret <16 x i32> %57177}7178 7179define <16 x i32> @stack_fold_pxord_mask_commuted(<16 x i32> %a0, <16 x i32> %a1, ptr %a2, i16 %mask) {7180; CHECK-LABEL: stack_fold_pxord_mask_commuted:7181; CHECK:       # %bb.0:7182; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7183; CHECK-NEXT:    vmovaps %zmm0, %zmm17184; CHECK-NEXT:    #APP7185; CHECK-NEXT:    nop7186; CHECK-NEXT:    #NO_APP7187; CHECK-NEXT:    kmovd %esi, %k17188; CHECK-NEXT:    vmovaps (%rdi), %zmm07189; CHECK-NEXT:    vxorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload7190; CHECK-NEXT:    retq7191  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()7192  %2 = xor <16 x i32> %a1, %a07193  %3 = bitcast i16 %mask to <16 x i1>7194  ; load needed to keep the operation from being scheduled about the asm block7195  %4 = load <16 x i32>, ptr %a27196  %5 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> %47197  ret <16 x i32> %57198}7199 7200define <16 x i32> @stack_fold_pxord_maskz(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {7201; CHECK-LABEL: stack_fold_pxord_maskz:7202; CHECK:       # %bb.0:7203; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7204; CHECK-NEXT:    #APP7205; CHECK-NEXT:    nop7206; CHECK-NEXT:    #NO_APP7207; CHECK-NEXT:    kmovd %edi, %k17208; CHECK-NEXT:    vxorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload7209; CHECK-NEXT:    retq7210  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()7211  %2 = xor <16 x i32> %a0, %a17212  %3 = bitcast i16 %mask to <16 x i1>7213  %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer7214  ret <16 x i32> %47215}7216 7217define <16 x i32> @stack_fold_pxord_maskz_commuted(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {7218; CHECK-LABEL: stack_fold_pxord_maskz_commuted:7219; CHECK:       # %bb.0:7220; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7221; CHECK-NEXT:    #APP7222; CHECK-NEXT:    nop7223; CHECK-NEXT:    #NO_APP7224; CHECK-NEXT:    kmovd %edi, %k17225; CHECK-NEXT:    vxorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload7226; CHECK-NEXT:    retq7227  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()7228  %2 = xor <16 x i32> %a1, %a07229  %3 = bitcast i16 %mask to <16 x i1>7230  %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer7231  ret <16 x i32> %47232}7233 7234define <8 x i64> @stack_fold_pxorq(<8 x i64> %a0, <8 x i64> %a1) {7235; CHECK-LABEL: stack_fold_pxorq:7236; CHECK:       # %bb.0:7237; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7238; CHECK-NEXT:    #APP7239; CHECK-NEXT:    nop7240; CHECK-NEXT:    #NO_APP7241; CHECK-NEXT:    vxorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload7242; CHECK-NEXT:    retq7243  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()7244  %2 = xor <8 x i64> %a0, %a17245  ret <8 x i64> %27246}7247 7248define <8 x i64> @stack_fold_pxorq_commuted(<8 x i64> %a0, <8 x i64> %a1) {7249; CHECK-LABEL: stack_fold_pxorq_commuted:7250; CHECK:       # %bb.0:7251; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7252; CHECK-NEXT:    #APP7253; CHECK-NEXT:    nop7254; CHECK-NEXT:    #NO_APP7255; CHECK-NEXT:    vxorps {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload7256; CHECK-NEXT:    retq7257  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()7258  %2 = xor <8 x i64> %a1, %a07259  ret <8 x i64> %27260}7261 7262define <8 x i64> @stack_fold_pxorq_mask(<8 x i64> %a0, <8 x i64> %a1, ptr %a2, i8 %mask) {7263; CHECK-LABEL: stack_fold_pxorq_mask:7264; CHECK:       # %bb.0:7265; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7266; CHECK-NEXT:    vmovapd %zmm0, %zmm17267; CHECK-NEXT:    #APP7268; CHECK-NEXT:    nop7269; CHECK-NEXT:    #NO_APP7270; CHECK-NEXT:    kmovd %esi, %k17271; CHECK-NEXT:    vmovapd (%rdi), %zmm07272; CHECK-NEXT:    vxorpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload7273; CHECK-NEXT:    retq7274  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()7275  %2 = xor <8 x i64> %a0, %a17276  %3 = bitcast i8 %mask to <8 x i1>7277  ; load needed to keep the operation from being scheduled about the asm block7278  %4 = load <8 x i64>, ptr %a27279  %5 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> %47280  ret <8 x i64> %57281}7282 7283define <8 x i64> @stack_fold_pxorq_mask_commuted(<8 x i64> %a0, <8 x i64> %a1, ptr %a2, i8 %mask) {7284; CHECK-LABEL: stack_fold_pxorq_mask_commuted:7285; CHECK:       # %bb.0:7286; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7287; CHECK-NEXT:    vmovapd %zmm0, %zmm17288; CHECK-NEXT:    #APP7289; CHECK-NEXT:    nop7290; CHECK-NEXT:    #NO_APP7291; CHECK-NEXT:    kmovd %esi, %k17292; CHECK-NEXT:    vmovapd (%rdi), %zmm07293; CHECK-NEXT:    vxorpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 {%k1} # 64-byte Folded Reload7294; CHECK-NEXT:    retq7295  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm0},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()7296  %2 = xor <8 x i64> %a1, %a07297  %3 = bitcast i8 %mask to <8 x i1>7298  ; load needed to keep the operation from being scheduled about the asm block7299  %4 = load <8 x i64>, ptr %a27300  %5 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> %47301  ret <8 x i64> %57302}7303 7304define <8 x i64> @stack_fold_pxorq_maskz(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {7305; CHECK-LABEL: stack_fold_pxorq_maskz:7306; CHECK:       # %bb.0:7307; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7308; CHECK-NEXT:    #APP7309; CHECK-NEXT:    nop7310; CHECK-NEXT:    #NO_APP7311; CHECK-NEXT:    kmovd %edi, %k17312; CHECK-NEXT:    vxorpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload7313; CHECK-NEXT:    retq7314  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()7315  %2 = xor <8 x i64> %a0, %a17316  %3 = bitcast i8 %mask to <8 x i1>7317  %4 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> zeroinitializer7318  ret <8 x i64> %47319}7320 7321define <8 x i64> @stack_fold_pxorq_maskz_commuted(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {7322; CHECK-LABEL: stack_fold_pxorq_maskz_commuted:7323; CHECK:       # %bb.0:7324; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill7325; CHECK-NEXT:    #APP7326; CHECK-NEXT:    nop7327; CHECK-NEXT:    #NO_APP7328; CHECK-NEXT:    kmovd %edi, %k17329; CHECK-NEXT:    vxorpd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 {%k1} {z} # 64-byte Folded Reload7330; CHECK-NEXT:    retq7331  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()7332  %2 = xor <8 x i64> %a1, %a07333  %3 = bitcast i8 %mask to <8 x i1>7334  %4 = select <8 x i1> %3, <8 x i64> %2, <8 x i64> zeroinitializer7335  ret <8 x i64> %47336}7337 7338declare <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8>, <64 x i8>)7339declare <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16>, <32 x i16>)7340declare <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8>, <64 x i8>)7341declare <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16>, <32 x i16>)7342declare <16 x i32> @llvm.x86.avx512.conflict.d.512(<16 x i32>)7343declare <8 x i64> @llvm.x86.avx512.conflict.q.512(<8 x i64>)7344declare <16 x i32> @llvm.ctlz.v16i32(<16 x i32>, i1)7345declare <8 x i64> @llvm.ctlz.v8i64(<8 x i64>, i1)7346declare <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8>, <64 x i8>)7347declare <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16>, <32 x i16>)7348declare <64 x i8> @llvm.usub.sat.v64i8(<64 x i8>, <64 x i8>)7349declare <32 x i16> @llvm.usub.sat.v32i16(<32 x i16>, <32 x i16>)7350