1162 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -verify-machineinstrs -mcpu=sapphirerapids -mattr=+false-deps-perm -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=ENABLE3; RUN: llc -verify-machineinstrs -mcpu=sapphirerapids -mattr=-false-deps-perm -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=DISABLE4 5define <4 x i64> @permq_ri_256(<4 x i64> %a0) {6; ENABLE-LABEL: permq_ri_256:7; ENABLE: # %bb.0:8; ENABLE-NEXT: #APP9; ENABLE-NEXT: nop10; ENABLE-NEXT: #NO_APP11; ENABLE-NEXT: vxorps %xmm1, %xmm1, %xmm112; ENABLE-NEXT: vpermq {{.*#+}} ymm1 = ymm0[1,2,1,0]13; ENABLE-NEXT: vpaddq %ymm0, %ymm1, %ymm014; ENABLE-NEXT: retq15;16; DISABLE-LABEL: permq_ri_256:17; DISABLE: # %bb.0:18; DISABLE-NEXT: #APP19; DISABLE-NEXT: nop20; DISABLE-NEXT: #NO_APP21; DISABLE-NEXT: vpermq {{.*#+}} ymm1 = ymm0[1,2,1,0]22; DISABLE-NEXT: vpaddq %ymm0, %ymm1, %ymm023; DISABLE-NEXT: retq24 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()25 %2 = shufflevector <4 x i64> %a0, <4 x i64> undef, <4 x i32> <i32 1, i32 2, i32 1, i32 0>26 %res = add <4 x i64> %2, %a027 ret <4 x i64> %res28}29 30define <4 x i64> @permq_rr_256(<4 x i64> %a0, <4 x i64> %idx) {31; ENABLE-LABEL: permq_rr_256:32; ENABLE: # %bb.0:33; ENABLE-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill34; ENABLE-NEXT: #APP35; ENABLE-NEXT: nop36; ENABLE-NEXT: #NO_APP37; ENABLE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload38; ENABLE-NEXT: vxorps %xmm1, %xmm1, %xmm139; ENABLE-NEXT: vpermq %ymm0, %ymm2, %ymm140; ENABLE-NEXT: vpaddq %ymm2, %ymm0, %ymm041; ENABLE-NEXT: vpaddq %ymm1, %ymm0, %ymm042; ENABLE-NEXT: retq43;44; DISABLE-LABEL: permq_rr_256:45; DISABLE: # %bb.0:46; DISABLE-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill47; DISABLE-NEXT: #APP48; DISABLE-NEXT: nop49; DISABLE-NEXT: #NO_APP50; DISABLE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload51; DISABLE-NEXT: vpermq %ymm0, %ymm2, %ymm152; DISABLE-NEXT: vpaddq %ymm2, %ymm0, %ymm053; DISABLE-NEXT: vpaddq %ymm1, %ymm0, %ymm054; DISABLE-NEXT: retq55 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()56 %2 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %a0, <4 x i64> %idx)57 %t = add <4 x i64> %a0, %idx58 %res = add <4 x i64> %t, %259 ret <4 x i64> %res60}61 62define <4 x i64> @permq_rm_256(ptr %p0, <4 x i64> %idx) {63; ENABLE-LABEL: permq_rm_256:64; ENABLE: # %bb.0:65; ENABLE-NEXT: #APP66; ENABLE-NEXT: nop67; ENABLE-NEXT: #NO_APP68; ENABLE-NEXT: vxorps %xmm1, %xmm1, %xmm169; ENABLE-NEXT: vpermq (%rdi), %ymm0, %ymm170; ENABLE-NEXT: vpaddq %ymm1, %ymm0, %ymm071; ENABLE-NEXT: retq72;73; DISABLE-LABEL: permq_rm_256:74; DISABLE: # %bb.0:75; DISABLE-NEXT: #APP76; DISABLE-NEXT: nop77; DISABLE-NEXT: #NO_APP78; DISABLE-NEXT: vpermq (%rdi), %ymm0, %ymm179; DISABLE-NEXT: vpaddq %ymm1, %ymm0, %ymm080; DISABLE-NEXT: retq81 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()82 %a0 = load <4 x i64>, ptr %p0, align 6483 %2 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %a0, <4 x i64> %idx)84 %res = add <4 x i64> %idx, %285 ret <4 x i64> %res86}87 88define <4 x i64> @permq_mi_256(ptr %p0) {89; ENABLE-LABEL: permq_mi_256:90; ENABLE: # %bb.0:91; ENABLE-NEXT: #APP92; ENABLE-NEXT: nop93; ENABLE-NEXT: #NO_APP94; ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm095; ENABLE-NEXT: vpermpd {{.*#+}} ymm0 = mem[3,2,2,0]96; ENABLE-NEXT: retq97;98; DISABLE-LABEL: permq_mi_256:99; DISABLE: # %bb.0:100; DISABLE-NEXT: #APP101; DISABLE-NEXT: nop102; DISABLE-NEXT: #NO_APP103; DISABLE-NEXT: vpermpd {{.*#+}} ymm0 = mem[3,2,2,0]104; DISABLE-NEXT: retq105 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()106 %a0 = load <4 x i64>, ptr %p0, align 64107 %2 = shufflevector <4 x i64> %a0, <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 2, i32 0>108 ret <4 x i64> %2109}110 111define <4 x i64> @permq_broadcast_256(ptr %p0, <4 x i64> %idx) {112; ENABLE-LABEL: permq_broadcast_256:113; ENABLE: # %bb.0:114; ENABLE-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill115; ENABLE-NEXT: #APP116; ENABLE-NEXT: nop117; ENABLE-NEXT: #NO_APP118; ENABLE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload119; ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0120; ENABLE-NEXT: vpermq (%rdi){1to4}, %ymm1, %ymm0121; ENABLE-NEXT: vpaddq %ymm1, %ymm0, %ymm0122; ENABLE-NEXT: retq123;124; DISABLE-LABEL: permq_broadcast_256:125; DISABLE: # %bb.0:126; DISABLE-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill127; DISABLE-NEXT: #APP128; DISABLE-NEXT: nop129; DISABLE-NEXT: #NO_APP130; DISABLE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload131; DISABLE-NEXT: vpermq (%rdi){1to4}, %ymm1, %ymm0132; DISABLE-NEXT: vpaddq %ymm1, %ymm0, %ymm0133; DISABLE-NEXT: retq134 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()135 %v0 = load i64, ptr %p0, align 4136 %t0 = insertelement <4 x i64> undef, i64 %v0, i64 0137 %a0 = shufflevector <4 x i64> %t0, <4 x i64> undef, <4 x i32> zeroinitializer138 %2 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %a0, <4 x i64> %idx)139 %res = add <4 x i64> %2, %idx140 ret <4 x i64> %res141}142 143define <4 x i64> @permq_maskz_256(<4 x i64> %a0, <4 x i64> %idx, ptr %mask) {144; ENABLE-LABEL: permq_maskz_256:145; ENABLE: # %bb.0:146; ENABLE-NEXT: #APP147; ENABLE-NEXT: nop148; ENABLE-NEXT: #NO_APP149; ENABLE-NEXT: vxorps %xmm2, %xmm2, %xmm2150; ENABLE-NEXT: vpermq %ymm0, %ymm1, %ymm2151; ENABLE-NEXT: kmovb (%rdi), %k1152; ENABLE-NEXT: vpaddq %ymm1, %ymm0, %ymm0153; ENABLE-NEXT: vpaddq %ymm2, %ymm0, %ymm0 {%k1}154; ENABLE-NEXT: retq155;156; DISABLE-LABEL: permq_maskz_256:157; DISABLE: # %bb.0:158; DISABLE-NEXT: #APP159; DISABLE-NEXT: nop160; DISABLE-NEXT: #NO_APP161; DISABLE-NEXT: vpermq %ymm0, %ymm1, %ymm2162; DISABLE-NEXT: kmovb (%rdi), %k1163; DISABLE-NEXT: vpaddq %ymm1, %ymm0, %ymm0164; DISABLE-NEXT: vpaddq %ymm2, %ymm0, %ymm0 {%k1}165; DISABLE-NEXT: retq166 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()167 %2 = load i8, ptr %mask168 %3 = call <4 x i64> @llvm.x86.avx512.mask.permvar.di.256(<4 x i64> %a0, <4 x i64> %idx, <4 x i64> zeroinitializer, i8 %2)169 %t = add <4 x i64> %a0, %idx170 %res = add <4 x i64> %3, %t171 ret <4 x i64> %res172}173 174declare <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64>, <4 x i64>)175declare <4 x i64> @llvm.x86.avx512.mask.permvar.di.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)176 177define <8 x i64> @permq_rr_512(<8 x i64> %a0, <8 x i64> %idx) {178; ENABLE-LABEL: permq_rr_512:179; ENABLE: # %bb.0:180; ENABLE-NEXT: vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill181; ENABLE-NEXT: #APP182; ENABLE-NEXT: nop183; ENABLE-NEXT: #NO_APP184; ENABLE-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload185; ENABLE-NEXT: vpxor %xmm1, %xmm1, %xmm1186; ENABLE-NEXT: vpermq %zmm0, %zmm2, %zmm1187; ENABLE-NEXT: vpaddq %zmm2, %zmm0, %zmm0188; ENABLE-NEXT: vpaddq %zmm1, %zmm0, %zmm0189; ENABLE-NEXT: retq190;191; DISABLE-LABEL: permq_rr_512:192; DISABLE: # %bb.0:193; DISABLE-NEXT: vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill194; DISABLE-NEXT: #APP195; DISABLE-NEXT: nop196; DISABLE-NEXT: #NO_APP197; DISABLE-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload198; DISABLE-NEXT: vpermq %zmm0, %zmm2, %zmm1199; DISABLE-NEXT: vpaddq %zmm2, %zmm0, %zmm0200; DISABLE-NEXT: vpaddq %zmm1, %zmm0, %zmm0201; DISABLE-NEXT: retq202 %1 = tail call <8 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()203 %2 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a0, <8 x i64> %idx)204 %t = add <8 x i64> %a0, %idx205 %res = add <8 x i64> %t, %2206 ret <8 x i64> %res207}208 209define <8 x i64> @permq_rm_512(ptr %p0, <8 x i64> %idx) {210; ENABLE-LABEL: permq_rm_512:211; ENABLE: # %bb.0:212; ENABLE-NEXT: #APP213; ENABLE-NEXT: nop214; ENABLE-NEXT: #NO_APP215; ENABLE-NEXT: vpxor %xmm1, %xmm1, %xmm1216; ENABLE-NEXT: vpermq (%rdi), %zmm0, %zmm1217; ENABLE-NEXT: vpaddq %zmm1, %zmm0, %zmm0218; ENABLE-NEXT: retq219;220; DISABLE-LABEL: permq_rm_512:221; DISABLE: # %bb.0:222; DISABLE-NEXT: #APP223; DISABLE-NEXT: nop224; DISABLE-NEXT: #NO_APP225; DISABLE-NEXT: vpermq (%rdi), %zmm0, %zmm1226; DISABLE-NEXT: vpaddq %zmm1, %zmm0, %zmm0227; DISABLE-NEXT: retq228 %1 = tail call <8 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()229 %a0 = load <8 x i64>, ptr %p0, align 64230 %2 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a0, <8 x i64> %idx)231 %res = add <8 x i64> %idx, %2232 ret <8 x i64> %res233}234 235define <8 x i64> @permq_broadcast_512(ptr %p0, <8 x i64> %idx) {236; ENABLE-LABEL: permq_broadcast_512:237; ENABLE: # %bb.0:238; ENABLE-NEXT: vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill239; ENABLE-NEXT: #APP240; ENABLE-NEXT: nop241; ENABLE-NEXT: #NO_APP242; ENABLE-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload243; ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0244; ENABLE-NEXT: vpermq (%rdi){1to8}, %zmm1, %zmm0245; ENABLE-NEXT: vpaddq %zmm1, %zmm0, %zmm0246; ENABLE-NEXT: retq247;248; DISABLE-LABEL: permq_broadcast_512:249; DISABLE: # %bb.0:250; DISABLE-NEXT: vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill251; DISABLE-NEXT: #APP252; DISABLE-NEXT: nop253; DISABLE-NEXT: #NO_APP254; DISABLE-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload255; DISABLE-NEXT: vpermq (%rdi){1to8}, %zmm1, %zmm0256; DISABLE-NEXT: vpaddq %zmm1, %zmm0, %zmm0257; DISABLE-NEXT: retq258 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()259 %v0 = load i64, ptr %p0, align 4260 %t0 = insertelement <8 x i64> undef, i64 %v0, i64 0261 %a0 = shufflevector <8 x i64> %t0, <8 x i64> undef, <8 x i32> zeroinitializer262 %2 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a0, <8 x i64> %idx)263 %res = add <8 x i64> %2, %idx264 ret <8 x i64> %res265}266 267define <8 x i64> @permq_maskz_512(<8 x i64> %a0, <8 x i64> %idx, ptr %mask) {268; ENABLE-LABEL: permq_maskz_512:269; ENABLE: # %bb.0:270; ENABLE-NEXT: #APP271; ENABLE-NEXT: nop272; ENABLE-NEXT: #NO_APP273; ENABLE-NEXT: vpxor %xmm2, %xmm2, %xmm2274; ENABLE-NEXT: vpermq %zmm0, %zmm1, %zmm2275; ENABLE-NEXT: kmovb (%rdi), %k1276; ENABLE-NEXT: vpaddq %zmm1, %zmm0, %zmm0277; ENABLE-NEXT: vpaddq %zmm2, %zmm0, %zmm0 {%k1}278; ENABLE-NEXT: retq279;280; DISABLE-LABEL: permq_maskz_512:281; DISABLE: # %bb.0:282; DISABLE-NEXT: #APP283; DISABLE-NEXT: nop284; DISABLE-NEXT: #NO_APP285; DISABLE-NEXT: vpermq %zmm0, %zmm1, %zmm2286; DISABLE-NEXT: kmovb (%rdi), %k1287; DISABLE-NEXT: vpaddq %zmm1, %zmm0, %zmm0288; DISABLE-NEXT: vpaddq %zmm2, %zmm0, %zmm0 {%k1}289; DISABLE-NEXT: retq290 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()291 %2 = load i8, ptr %mask292 %3 = call <8 x i64> @llvm.x86.avx512.mask.permvar.di.512(<8 x i64> %a0, <8 x i64> %idx, <8 x i64> zeroinitializer, i8 %2)293 %t = add <8 x i64> %a0, %idx294 %res = add <8 x i64> %3, %t295 ret <8 x i64> %res296}297 298declare <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64>, <8 x i64>)299declare <8 x i64> @llvm.x86.avx512.mask.permvar.di.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)300 301define <8 x i32> @permd_rr_256(<8 x i32> %a0, <8 x i32> %idx) {302; ENABLE-LABEL: permd_rr_256:303; ENABLE: # %bb.0:304; ENABLE-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill305; ENABLE-NEXT: #APP306; ENABLE-NEXT: nop307; ENABLE-NEXT: #NO_APP308; ENABLE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload309; ENABLE-NEXT: vxorps %xmm1, %xmm1, %xmm1310; ENABLE-NEXT: vpermd %ymm0, %ymm2, %ymm1311; ENABLE-NEXT: vpaddd %ymm2, %ymm0, %ymm0312; ENABLE-NEXT: vpaddd %ymm1, %ymm0, %ymm0313; ENABLE-NEXT: retq314;315; DISABLE-LABEL: permd_rr_256:316; DISABLE: # %bb.0:317; DISABLE-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill318; DISABLE-NEXT: #APP319; DISABLE-NEXT: nop320; DISABLE-NEXT: #NO_APP321; DISABLE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload322; DISABLE-NEXT: vpermd %ymm0, %ymm2, %ymm1323; DISABLE-NEXT: vpaddd %ymm2, %ymm0, %ymm0324; DISABLE-NEXT: vpaddd %ymm1, %ymm0, %ymm0325; DISABLE-NEXT: retq326 %1 = tail call <8 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()327 %2 = call <8 x i32> @llvm.x86.avx512.mask.permvar.si.256(<8 x i32> %a0, <8 x i32> %idx, <8 x i32> undef, i8 -1)328 %t = add <8 x i32> %a0, %idx329 %res = add <8 x i32> %t, %2330 ret <8 x i32> %res331}332 333define <8 x i32> @permd_rm_256(ptr %p0, <8 x i32> %idx) {334; ENABLE-LABEL: permd_rm_256:335; ENABLE: # %bb.0:336; ENABLE-NEXT: #APP337; ENABLE-NEXT: nop338; ENABLE-NEXT: #NO_APP339; ENABLE-NEXT: vxorps %xmm1, %xmm1, %xmm1340; ENABLE-NEXT: vpermd (%rdi), %ymm0, %ymm1341; ENABLE-NEXT: vpaddd %ymm1, %ymm0, %ymm0342; ENABLE-NEXT: retq343;344; DISABLE-LABEL: permd_rm_256:345; DISABLE: # %bb.0:346; DISABLE-NEXT: #APP347; DISABLE-NEXT: nop348; DISABLE-NEXT: #NO_APP349; DISABLE-NEXT: vpermd (%rdi), %ymm0, %ymm1350; DISABLE-NEXT: vpaddd %ymm1, %ymm0, %ymm0351; DISABLE-NEXT: retq352 %1 = tail call <8 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()353 %a0 = load <8 x i32>, ptr %p0, align 64354 %2 = call <8 x i32> @llvm.x86.avx512.mask.permvar.si.256(<8 x i32> %a0, <8 x i32> %idx, <8 x i32> undef, i8 -1)355 %res = add <8 x i32> %idx, %2356 ret <8 x i32> %res357}358 359define <8 x i32> @permd_broadcast_256(ptr %p0, <8 x i32> %idx) {360; ENABLE-LABEL: permd_broadcast_256:361; ENABLE: # %bb.0:362; ENABLE-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill363; ENABLE-NEXT: #APP364; ENABLE-NEXT: nop365; ENABLE-NEXT: #NO_APP366; ENABLE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload367; ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0368; ENABLE-NEXT: vpermd (%rdi){1to8}, %ymm1, %ymm0369; ENABLE-NEXT: vpaddd %ymm1, %ymm0, %ymm0370; ENABLE-NEXT: retq371;372; DISABLE-LABEL: permd_broadcast_256:373; DISABLE: # %bb.0:374; DISABLE-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill375; DISABLE-NEXT: #APP376; DISABLE-NEXT: nop377; DISABLE-NEXT: #NO_APP378; DISABLE-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload379; DISABLE-NEXT: vpermd (%rdi){1to8}, %ymm1, %ymm0380; DISABLE-NEXT: vpaddd %ymm1, %ymm0, %ymm0381; DISABLE-NEXT: retq382 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()383 %v0 = load i32, ptr %p0, align 4384 %t0 = insertelement <8 x i32> undef, i32 %v0, i32 0385 %a0 = shufflevector <8 x i32> %t0, <8 x i32> undef, <8 x i32> zeroinitializer386 %2 = call <8 x i32> @llvm.x86.avx512.mask.permvar.si.256(<8 x i32> %a0, <8 x i32> %idx, <8 x i32> zeroinitializer, i8 -1)387 %res = add <8 x i32> %2, %idx388 ret <8 x i32> %res389}390 391define <8 x i32> @permd_maskz_256(<8 x i32> %a0, <8 x i32> %idx, ptr %mask) {392; ENABLE-LABEL: permd_maskz_256:393; ENABLE: # %bb.0:394; ENABLE-NEXT: #APP395; ENABLE-NEXT: nop396; ENABLE-NEXT: #NO_APP397; ENABLE-NEXT: vxorps %xmm2, %xmm2, %xmm2398; ENABLE-NEXT: vpermd %ymm0, %ymm1, %ymm2399; ENABLE-NEXT: kmovb (%rdi), %k1400; ENABLE-NEXT: vpaddd %ymm1, %ymm0, %ymm0401; ENABLE-NEXT: vpaddd %ymm2, %ymm0, %ymm0 {%k1}402; ENABLE-NEXT: retq403;404; DISABLE-LABEL: permd_maskz_256:405; DISABLE: # %bb.0:406; DISABLE-NEXT: #APP407; DISABLE-NEXT: nop408; DISABLE-NEXT: #NO_APP409; DISABLE-NEXT: vpermd %ymm0, %ymm1, %ymm2410; DISABLE-NEXT: kmovb (%rdi), %k1411; DISABLE-NEXT: vpaddd %ymm1, %ymm0, %ymm0412; DISABLE-NEXT: vpaddd %ymm2, %ymm0, %ymm0 {%k1}413; DISABLE-NEXT: retq414 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()415 %2 = load i8, ptr %mask416 %3 = call <8 x i32> @llvm.x86.avx512.mask.permvar.si.256(<8 x i32> %a0, <8 x i32> %idx, <8 x i32> zeroinitializer, i8 %2)417 %t = add <8 x i32> %a0, %idx418 %res = add <8 x i32> %3, %t419 ret <8 x i32> %res420}421 422declare <8 x i32> @llvm.x86.avx512.mask.permvar.si.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)423 424define <16 x i32> @permd_rr_512(<16 x i32> %a0, <16 x i32> %idx) {425; ENABLE-LABEL: permd_rr_512:426; ENABLE: # %bb.0:427; ENABLE-NEXT: vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill428; ENABLE-NEXT: #APP429; ENABLE-NEXT: nop430; ENABLE-NEXT: #NO_APP431; ENABLE-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload432; ENABLE-NEXT: vpxor %xmm1, %xmm1, %xmm1433; ENABLE-NEXT: vpermd %zmm0, %zmm2, %zmm1434; ENABLE-NEXT: vpaddd %zmm2, %zmm0, %zmm0435; ENABLE-NEXT: vpaddd %zmm1, %zmm0, %zmm0436; ENABLE-NEXT: retq437;438; DISABLE-LABEL: permd_rr_512:439; DISABLE: # %bb.0:440; DISABLE-NEXT: vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill441; DISABLE-NEXT: #APP442; DISABLE-NEXT: nop443; DISABLE-NEXT: #NO_APP444; DISABLE-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload445; DISABLE-NEXT: vpermd %zmm0, %zmm2, %zmm1446; DISABLE-NEXT: vpaddd %zmm2, %zmm0, %zmm0447; DISABLE-NEXT: vpaddd %zmm1, %zmm0, %zmm0448; DISABLE-NEXT: retq449 %1 = tail call <8 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()450 %2 = call <16 x i32> @llvm.x86.avx512.mask.permvar.si.512(<16 x i32> %a0, <16 x i32> %idx, <16 x i32> undef, i16 -1)451 %t = add <16 x i32> %a0, %idx452 %res = add <16 x i32> %t, %2453 ret <16 x i32> %res454}455 456define <16 x i32> @permd_rm_512(ptr %p0, <16 x i32> %idx) {457; ENABLE-LABEL: permd_rm_512:458; ENABLE: # %bb.0:459; ENABLE-NEXT: #APP460; ENABLE-NEXT: nop461; ENABLE-NEXT: #NO_APP462; ENABLE-NEXT: vpxor %xmm1, %xmm1, %xmm1463; ENABLE-NEXT: vpermd (%rdi), %zmm0, %zmm1464; ENABLE-NEXT: vpaddd %zmm1, %zmm0, %zmm0465; ENABLE-NEXT: retq466;467; DISABLE-LABEL: permd_rm_512:468; DISABLE: # %bb.0:469; DISABLE-NEXT: #APP470; DISABLE-NEXT: nop471; DISABLE-NEXT: #NO_APP472; DISABLE-NEXT: vpermd (%rdi), %zmm0, %zmm1473; DISABLE-NEXT: vpaddd %zmm1, %zmm0, %zmm0474; DISABLE-NEXT: retq475 %1 = tail call <8 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()476 %a0 = load <16 x i32>, ptr %p0, align 64477 %2 = call <16 x i32> @llvm.x86.avx512.mask.permvar.si.512(<16 x i32> %a0, <16 x i32> %idx, <16 x i32> undef, i16 -1)478 %res = add <16 x i32> %idx, %2479 ret <16 x i32> %res480}481 482define <16 x i32> @permd_broadcast_512(ptr %p0, <16 x i32> %idx) {483; ENABLE-LABEL: permd_broadcast_512:484; ENABLE: # %bb.0:485; ENABLE-NEXT: vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill486; ENABLE-NEXT: #APP487; ENABLE-NEXT: nop488; ENABLE-NEXT: #NO_APP489; ENABLE-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload490; ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0491; ENABLE-NEXT: vpermd (%rdi){1to16}, %zmm1, %zmm0492; ENABLE-NEXT: vpaddd %zmm1, %zmm0, %zmm0493; ENABLE-NEXT: retq494;495; DISABLE-LABEL: permd_broadcast_512:496; DISABLE: # %bb.0:497; DISABLE-NEXT: vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill498; DISABLE-NEXT: #APP499; DISABLE-NEXT: nop500; DISABLE-NEXT: #NO_APP501; DISABLE-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload502; DISABLE-NEXT: vpermd (%rdi){1to16}, %zmm1, %zmm0503; DISABLE-NEXT: vpaddd %zmm1, %zmm0, %zmm0504; DISABLE-NEXT: retq505 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()506 %v0 = load i32, ptr %p0, align 4507 %t0 = insertelement <16 x i32> undef, i32 %v0, i32 0508 %a0 = shufflevector <16 x i32> %t0, <16 x i32> undef, <16 x i32> zeroinitializer509 %2 = call <16 x i32> @llvm.x86.avx512.mask.permvar.si.512(<16 x i32> %a0, <16 x i32> %idx, <16 x i32> undef, i16 -1)510 %res = add <16 x i32> %2, %idx511 ret <16 x i32> %res512}513 514define <16 x i32> @permd_maskz_512(<16 x i32> %a0, <16 x i32> %idx, ptr %mask) {515; ENABLE-LABEL: permd_maskz_512:516; ENABLE: # %bb.0:517; ENABLE-NEXT: #APP518; ENABLE-NEXT: nop519; ENABLE-NEXT: #NO_APP520; ENABLE-NEXT: vpxor %xmm2, %xmm2, %xmm2521; ENABLE-NEXT: vpermd %zmm0, %zmm1, %zmm2522; ENABLE-NEXT: kmovw (%rdi), %k1523; ENABLE-NEXT: vpaddd %zmm1, %zmm0, %zmm0524; ENABLE-NEXT: vpaddd %zmm2, %zmm0, %zmm0 {%k1}525; ENABLE-NEXT: retq526;527; DISABLE-LABEL: permd_maskz_512:528; DISABLE: # %bb.0:529; DISABLE-NEXT: #APP530; DISABLE-NEXT: nop531; DISABLE-NEXT: #NO_APP532; DISABLE-NEXT: vpermd %zmm0, %zmm1, %zmm2533; DISABLE-NEXT: kmovw (%rdi), %k1534; DISABLE-NEXT: vpaddd %zmm1, %zmm0, %zmm0535; DISABLE-NEXT: vpaddd %zmm2, %zmm0, %zmm0 {%k1}536; DISABLE-NEXT: retq537 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()538 %2 = load i16, ptr %mask539 %3 = call <16 x i32> @llvm.x86.avx512.mask.permvar.si.512(<16 x i32> %a0, <16 x i32> %idx, <16 x i32> zeroinitializer, i16 %2)540 %t = add <16 x i32> %a0, %idx541 %res = add <16 x i32> %3, %t542 ret <16 x i32> %res543}544 545declare <16 x i32> @llvm.x86.avx512.mask.permvar.si.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)546 547define <4 x double> @permpd_ri_256(<4 x double> %a0) {548; ENABLE-LABEL: permpd_ri_256:549; ENABLE: # %bb.0:550; ENABLE-NEXT: #APP551; ENABLE-NEXT: nop552; ENABLE-NEXT: #NO_APP553; ENABLE-NEXT: vxorps %xmm1, %xmm1, %xmm1554; ENABLE-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[1,2,1,0]555; ENABLE-NEXT: vaddpd %ymm0, %ymm1, %ymm0556; ENABLE-NEXT: retq557;558; DISABLE-LABEL: permpd_ri_256:559; DISABLE: # %bb.0:560; DISABLE-NEXT: #APP561; DISABLE-NEXT: nop562; DISABLE-NEXT: #NO_APP563; DISABLE-NEXT: vpermpd {{.*#+}} ymm1 = ymm0[1,2,1,0]564; DISABLE-NEXT: vaddpd %ymm0, %ymm1, %ymm0565; DISABLE-NEXT: retq566 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()567 %2 = shufflevector <4 x double> %a0, <4 x double> undef, <4 x i32> <i32 1, i32 2, i32 1, i32 0>568 %res = fadd <4 x double> %2, %a0569 ret <4 x double> %res570}571 572define <4 x double> @permpd_rr_256(<4 x double> %a0, <4 x i64> %idx) {573; ENABLE-LABEL: permpd_rr_256:574; ENABLE: # %bb.0:575; ENABLE-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill576; ENABLE-NEXT: #APP577; ENABLE-NEXT: nop578; ENABLE-NEXT: #NO_APP579; ENABLE-NEXT: vmovapd %ymm0, %ymm2580; ENABLE-NEXT: vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload581; ENABLE-NEXT: vxorps %xmm1, %xmm1, %xmm1582; ENABLE-NEXT: vpermpd %ymm2, %ymm0, %ymm1583; ENABLE-NEXT: vcvtqq2pd %ymm0, %ymm0584; ENABLE-NEXT: vaddpd %ymm0, %ymm2, %ymm0585; ENABLE-NEXT: vaddpd %ymm0, %ymm1, %ymm0586; ENABLE-NEXT: retq587;588; DISABLE-LABEL: permpd_rr_256:589; DISABLE: # %bb.0:590; DISABLE-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill591; DISABLE-NEXT: #APP592; DISABLE-NEXT: nop593; DISABLE-NEXT: #NO_APP594; DISABLE-NEXT: vmovapd %ymm0, %ymm2595; DISABLE-NEXT: vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload596; DISABLE-NEXT: vpermpd %ymm2, %ymm0, %ymm1597; DISABLE-NEXT: vcvtqq2pd %ymm0, %ymm0598; DISABLE-NEXT: vaddpd %ymm0, %ymm2, %ymm0599; DISABLE-NEXT: vaddpd %ymm0, %ymm1, %ymm0600; DISABLE-NEXT: retq601 %1 = tail call <4 x double> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()602 %2 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %1, <4 x i64> %idx)603 %a1 = sitofp <4 x i64> %idx to <4 x double>604 %t = fadd <4 x double> %1, %a1605 %res = fadd <4 x double> %2, %t606 ret <4 x double> %res607}608 609define <4 x double> @permpd_rm_256(ptr %p0, <4 x i64> %idx) {610; ENABLE-LABEL: permpd_rm_256:611; ENABLE: # %bb.0:612; ENABLE-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill613; ENABLE-NEXT: #APP614; ENABLE-NEXT: nop615; ENABLE-NEXT: #NO_APP616; ENABLE-NEXT: vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload617; ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0618; ENABLE-NEXT: vpermpd (%rdi), %ymm1, %ymm0619; ENABLE-NEXT: vcvtqq2pd %ymm1, %ymm1620; ENABLE-NEXT: vaddpd %ymm1, %ymm0, %ymm0621; ENABLE-NEXT: retq622;623; DISABLE-LABEL: permpd_rm_256:624; DISABLE: # %bb.0:625; DISABLE-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill626; DISABLE-NEXT: #APP627; DISABLE-NEXT: nop628; DISABLE-NEXT: #NO_APP629; DISABLE-NEXT: vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload630; DISABLE-NEXT: vpermpd (%rdi), %ymm1, %ymm0631; DISABLE-NEXT: vcvtqq2pd %ymm1, %ymm1632; DISABLE-NEXT: vaddpd %ymm1, %ymm0, %ymm0633; DISABLE-NEXT: retq634 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()635 %a0 = load <4 x double>, ptr %p0, align 64636 %2 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %a0, <4 x i64> %idx)637 %a1 = sitofp <4 x i64> %idx to <4 x double>638 %res = fadd <4 x double> %2, %a1639 ret <4 x double> %res640}641 642define <4 x double> @permpd_mi_256(ptr %p0) {643; ENABLE-LABEL: permpd_mi_256:644; ENABLE: # %bb.0:645; ENABLE-NEXT: #APP646; ENABLE-NEXT: nop647; ENABLE-NEXT: #NO_APP648; ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0649; ENABLE-NEXT: vpermpd {{.*#+}} ymm0 = mem[3,2,2,0]650; ENABLE-NEXT: retq651;652; DISABLE-LABEL: permpd_mi_256:653; DISABLE: # %bb.0:654; DISABLE-NEXT: #APP655; DISABLE-NEXT: nop656; DISABLE-NEXT: #NO_APP657; DISABLE-NEXT: vpermpd {{.*#+}} ymm0 = mem[3,2,2,0]658; DISABLE-NEXT: retq659 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()660 %a0 = load <4 x double>, ptr %p0, align 64661 %2 = shufflevector <4 x double> %a0, <4 x double> undef, <4 x i32> <i32 3, i32 2, i32 2, i32 0>662 ret <4 x double> %2663}664 665define <4 x double> @permpd_broadcast_256(ptr %p0, <4 x i64> %idx) {666; ENABLE-LABEL: permpd_broadcast_256:667; ENABLE: # %bb.0:668; ENABLE-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill669; ENABLE-NEXT: #APP670; ENABLE-NEXT: nop671; ENABLE-NEXT: #NO_APP672; ENABLE-NEXT: vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload673; ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0674; ENABLE-NEXT: vpermpd (%rdi){1to4}, %ymm1, %ymm0675; ENABLE-NEXT: vcvtqq2pd %ymm1, %ymm1676; ENABLE-NEXT: vaddpd %ymm1, %ymm0, %ymm0677; ENABLE-NEXT: retq678;679; DISABLE-LABEL: permpd_broadcast_256:680; DISABLE: # %bb.0:681; DISABLE-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill682; DISABLE-NEXT: #APP683; DISABLE-NEXT: nop684; DISABLE-NEXT: #NO_APP685; DISABLE-NEXT: vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload686; DISABLE-NEXT: vpermpd (%rdi){1to4}, %ymm1, %ymm0687; DISABLE-NEXT: vcvtqq2pd %ymm1, %ymm1688; DISABLE-NEXT: vaddpd %ymm1, %ymm0, %ymm0689; DISABLE-NEXT: retq690 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()691 %v0 = load double, ptr %p0, align 4692 %t0 = insertelement <4 x double> undef, double %v0, i64 0693 %a0 = shufflevector <4 x double> %t0, <4 x double> undef, <4 x i32> zeroinitializer694 %2 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %a0, <4 x i64> %idx)695 %a1 = sitofp <4 x i64> %idx to <4 x double>696 %res = fadd <4 x double> %2, %a1697 ret <4 x double> %res698}699 700define <4 x double> @permpd_maskz_256(<4 x double> %a0, <4 x i64> %idx, ptr %mask) {701; ENABLE-LABEL: permpd_maskz_256:702; ENABLE: # %bb.0:703; ENABLE-NEXT: #APP704; ENABLE-NEXT: nop705; ENABLE-NEXT: #NO_APP706; ENABLE-NEXT: kmovb (%rdi), %k1707; ENABLE-NEXT: vxorps %xmm2, %xmm2, %xmm2708; ENABLE-NEXT: vpermpd %ymm0, %ymm1, %ymm2 {%k1} {z}709; ENABLE-NEXT: vcvtqq2pd %ymm1, %ymm1710; ENABLE-NEXT: vaddpd %ymm1, %ymm0, %ymm0711; ENABLE-NEXT: vaddpd %ymm0, %ymm2, %ymm0712; ENABLE-NEXT: retq713;714; DISABLE-LABEL: permpd_maskz_256:715; DISABLE: # %bb.0:716; DISABLE-NEXT: #APP717; DISABLE-NEXT: nop718; DISABLE-NEXT: #NO_APP719; DISABLE-NEXT: kmovb (%rdi), %k1720; DISABLE-NEXT: vpermpd %ymm0, %ymm1, %ymm2 {%k1} {z}721; DISABLE-NEXT: vcvtqq2pd %ymm1, %ymm1722; DISABLE-NEXT: vaddpd %ymm1, %ymm0, %ymm0723; DISABLE-NEXT: vaddpd %ymm0, %ymm2, %ymm0724; DISABLE-NEXT: retq725 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()726 %2 = load i8, ptr %mask727 %3 = call <4 x double> @llvm.x86.avx512.mask.permvar.df.256(<4 x double> %a0, <4 x i64> %idx, <4 x double> zeroinitializer, i8 %2)728 %a1 = sitofp <4 x i64> %idx to <4 x double>729 %t = fadd <4 x double> %a0, %a1730 %res = fadd <4 x double> %3, %t731 ret <4 x double> %res732}733 734declare <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double>, <4 x i64>)735declare <4 x double> @llvm.x86.avx512.mask.permvar.df.256(<4 x double>, <4 x i64>, <4 x double>, i8)736 737define <8 x double> @permpd_rr_512(<8 x double> %a0, <8 x i64> %idx) {738; ENABLE-LABEL: permpd_rr_512:739; ENABLE: # %bb.0:740; ENABLE-NEXT: vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill741; ENABLE-NEXT: #APP742; ENABLE-NEXT: nop743; ENABLE-NEXT: #NO_APP744; ENABLE-NEXT: vmovapd %zmm0, %zmm2745; ENABLE-NEXT: vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload746; ENABLE-NEXT: vpxor %xmm1, %xmm1, %xmm1747; ENABLE-NEXT: vpermpd %zmm2, %zmm0, %zmm1748; ENABLE-NEXT: vcvtqq2pd %zmm0, %zmm0749; ENABLE-NEXT: vaddpd %zmm0, %zmm2, %zmm0750; ENABLE-NEXT: vaddpd %zmm0, %zmm1, %zmm0751; ENABLE-NEXT: retq752;753; DISABLE-LABEL: permpd_rr_512:754; DISABLE: # %bb.0:755; DISABLE-NEXT: vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill756; DISABLE-NEXT: #APP757; DISABLE-NEXT: nop758; DISABLE-NEXT: #NO_APP759; DISABLE-NEXT: vmovapd %zmm0, %zmm2760; DISABLE-NEXT: vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload761; DISABLE-NEXT: vpermpd %zmm2, %zmm0, %zmm1762; DISABLE-NEXT: vcvtqq2pd %zmm0, %zmm0763; DISABLE-NEXT: vaddpd %zmm0, %zmm2, %zmm0764; DISABLE-NEXT: vaddpd %zmm0, %zmm1, %zmm0765; DISABLE-NEXT: retq766 %1 = tail call <8 x double> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()767 %2 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %1, <8 x i64> %idx)768 %a1 = sitofp <8 x i64> %idx to <8 x double>769 %t = fadd <8 x double> %1, %a1770 %res = fadd <8 x double> %2, %t771 ret <8 x double> %res772}773 774define <8 x double> @permpd_rm_512(ptr %p0, <8 x i64> %idx) {775; ENABLE-LABEL: permpd_rm_512:776; ENABLE: # %bb.0:777; ENABLE-NEXT: vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill778; ENABLE-NEXT: #APP779; ENABLE-NEXT: nop780; ENABLE-NEXT: #NO_APP781; ENABLE-NEXT: vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload782; ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0783; ENABLE-NEXT: vpermpd (%rdi), %zmm1, %zmm0784; ENABLE-NEXT: vcvtqq2pd %zmm1, %zmm1785; ENABLE-NEXT: vaddpd %zmm1, %zmm0, %zmm0786; ENABLE-NEXT: retq787;788; DISABLE-LABEL: permpd_rm_512:789; DISABLE: # %bb.0:790; DISABLE-NEXT: vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill791; DISABLE-NEXT: #APP792; DISABLE-NEXT: nop793; DISABLE-NEXT: #NO_APP794; DISABLE-NEXT: vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload795; DISABLE-NEXT: vpermpd (%rdi), %zmm1, %zmm0796; DISABLE-NEXT: vcvtqq2pd %zmm1, %zmm1797; DISABLE-NEXT: vaddpd %zmm1, %zmm0, %zmm0798; DISABLE-NEXT: retq799 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()800 %a0 = load <8 x double>, ptr %p0, align 64801 %2 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %a0, <8 x i64> %idx)802 %a1 = sitofp <8 x i64> %idx to <8 x double>803 %res = fadd <8 x double> %2, %a1804 ret <8 x double> %res805}806 807define <8 x double> @permpd_broadcast_512(ptr %p0, <8 x i64> %idx) {808; ENABLE-LABEL: permpd_broadcast_512:809; ENABLE: # %bb.0:810; ENABLE-NEXT: vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill811; ENABLE-NEXT: #APP812; ENABLE-NEXT: nop813; ENABLE-NEXT: #NO_APP814; ENABLE-NEXT: vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload815; ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm0816; ENABLE-NEXT: vpermpd (%rdi){1to8}, %zmm1, %zmm0817; ENABLE-NEXT: vcvtqq2pd %zmm1, %zmm1818; ENABLE-NEXT: vaddpd %zmm1, %zmm0, %zmm0819; ENABLE-NEXT: retq820;821; DISABLE-LABEL: permpd_broadcast_512:822; DISABLE: # %bb.0:823; DISABLE-NEXT: vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill824; DISABLE-NEXT: #APP825; DISABLE-NEXT: nop826; DISABLE-NEXT: #NO_APP827; DISABLE-NEXT: vmovupd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload828; DISABLE-NEXT: vpermpd (%rdi){1to8}, %zmm1, %zmm0829; DISABLE-NEXT: vcvtqq2pd %zmm1, %zmm1830; DISABLE-NEXT: vaddpd %zmm1, %zmm0, %zmm0831; DISABLE-NEXT: retq832 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()833 %v0 = load double, ptr %p0, align 4834 %t0 = insertelement <8 x double> undef, double %v0, i64 0835 %a0 = shufflevector <8 x double> %t0, <8 x double> undef, <8 x i32> zeroinitializer836 %2 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %a0, <8 x i64> %idx)837 %a1 = sitofp <8 x i64> %idx to <8 x double>838 %res = fadd <8 x double> %2, %a1839 ret <8 x double> %res840}841 842define <8 x double> @permpd_maskz_512(<8 x double> %a0, <8 x i64> %idx, ptr %mask) {843; ENABLE-LABEL: permpd_maskz_512:844; ENABLE: # %bb.0:845; ENABLE-NEXT: #APP846; ENABLE-NEXT: nop847; ENABLE-NEXT: #NO_APP848; ENABLE-NEXT: kmovb (%rdi), %k1849; ENABLE-NEXT: vpxor %xmm2, %xmm2, %xmm2850; ENABLE-NEXT: vpermpd %zmm0, %zmm1, %zmm2 {%k1} {z}851; ENABLE-NEXT: vcvtqq2pd %zmm1, %zmm1852; ENABLE-NEXT: vaddpd %zmm1, %zmm0, %zmm0853; ENABLE-NEXT: vaddpd %zmm0, %zmm2, %zmm0854; ENABLE-NEXT: retq855;856; DISABLE-LABEL: permpd_maskz_512:857; DISABLE: # %bb.0:858; DISABLE-NEXT: #APP859; DISABLE-NEXT: nop860; DISABLE-NEXT: #NO_APP861; DISABLE-NEXT: kmovb (%rdi), %k1862; DISABLE-NEXT: vpermpd %zmm0, %zmm1, %zmm2 {%k1} {z}863; DISABLE-NEXT: vcvtqq2pd %zmm1, %zmm1864; DISABLE-NEXT: vaddpd %zmm1, %zmm0, %zmm0865; DISABLE-NEXT: vaddpd %zmm0, %zmm2, %zmm0866; DISABLE-NEXT: retq867 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()868 %2 = load i8, ptr %mask869 %3 = call <8 x double> @llvm.x86.avx512.mask.permvar.df.512(<8 x double> %a0, <8 x i64> %idx, <8 x double> zeroinitializer, i8 %2)870 %a1 = sitofp <8 x i64> %idx to <8 x double>871 %t = fadd <8 x double> %a0, %a1872 %res = fadd <8 x double> %3, %t873 ret <8 x double> %res874}875 876declare <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double>, <8 x i64>)877declare <8 x double> @llvm.x86.avx512.mask.permvar.df.512(<8 x double>, <8 x i64>, <8 x double>, i8)878 879 880define <8 x float> @permps_rr_256(<8 x float> %a0, <8 x i32> %idx) {881; ENABLE-LABEL: permps_rr_256:882; ENABLE: # %bb.0:883; ENABLE-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill884; ENABLE-NEXT: #APP885; ENABLE-NEXT: nop886; ENABLE-NEXT: #NO_APP887; ENABLE-NEXT: vmovaps %ymm0, %ymm2888; ENABLE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload889; ENABLE-NEXT: vxorps %xmm1, %xmm1, %xmm1890; ENABLE-NEXT: vpermps %ymm2, %ymm0, %ymm1891; ENABLE-NEXT: vcvtdq2ps %ymm0, %ymm0892; ENABLE-NEXT: vaddps %ymm0, %ymm2, %ymm0893; ENABLE-NEXT: vaddps %ymm0, %ymm1, %ymm0894; ENABLE-NEXT: retq895;896; DISABLE-LABEL: permps_rr_256:897; DISABLE: # %bb.0:898; DISABLE-NEXT: vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill899; DISABLE-NEXT: #APP900; DISABLE-NEXT: nop901; DISABLE-NEXT: #NO_APP902; DISABLE-NEXT: vmovaps %ymm0, %ymm2903; DISABLE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload904; DISABLE-NEXT: vpermps %ymm2, %ymm0, %ymm1905; DISABLE-NEXT: vcvtdq2ps %ymm0, %ymm0906; DISABLE-NEXT: vaddps %ymm0, %ymm2, %ymm0907; DISABLE-NEXT: vaddps %ymm0, %ymm1, %ymm0908; DISABLE-NEXT: retq909 %1 = tail call <8 x float> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()910 %2 = call <8 x float> @llvm.x86.avx512.mask.permvar.sf.256(<8 x float> %1, <8 x i32> %idx, <8 x float> zeroinitializer, i8 -1)911 %a1 = sitofp <8 x i32> %idx to <8 x float>912 %t = fadd <8 x float> %1, %a1913 %res = fadd <8 x float> %2, %t914 ret <8 x float> %res915}916 917define <8 x float> @permps_rm_256(ptr %p0, <8 x i32> %idx) {918; ENABLE-LABEL: permps_rm_256:919; ENABLE: # %bb.0:920; ENABLE-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill921; ENABLE-NEXT: #APP922; ENABLE-NEXT: nop923; ENABLE-NEXT: #NO_APP924; ENABLE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload925; ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0926; ENABLE-NEXT: vpermps (%rdi), %ymm1, %ymm0927; ENABLE-NEXT: vcvtdq2ps %ymm1, %ymm1928; ENABLE-NEXT: vaddps %ymm1, %ymm0, %ymm0929; ENABLE-NEXT: retq930;931; DISABLE-LABEL: permps_rm_256:932; DISABLE: # %bb.0:933; DISABLE-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill934; DISABLE-NEXT: #APP935; DISABLE-NEXT: nop936; DISABLE-NEXT: #NO_APP937; DISABLE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload938; DISABLE-NEXT: vpermps (%rdi), %ymm1, %ymm0939; DISABLE-NEXT: vcvtdq2ps %ymm1, %ymm1940; DISABLE-NEXT: vaddps %ymm1, %ymm0, %ymm0941; DISABLE-NEXT: retq942 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()943 %a0 = load <8 x float>, ptr %p0, align 64944 %2 = call <8 x float> @llvm.x86.avx512.mask.permvar.sf.256(<8 x float> %a0, <8 x i32> %idx, <8 x float> zeroinitializer, i8 -1)945 %a1 = sitofp <8 x i32> %idx to <8 x float>946 %res = fadd <8 x float> %2, %a1947 ret <8 x float> %res948}949 950define <8 x float> @permps_broadcast_256(ptr %p0, <8 x i32> %idx) {951; ENABLE-LABEL: permps_broadcast_256:952; ENABLE: # %bb.0:953; ENABLE-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill954; ENABLE-NEXT: #APP955; ENABLE-NEXT: nop956; ENABLE-NEXT: #NO_APP957; ENABLE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload958; ENABLE-NEXT: vxorps %xmm0, %xmm0, %xmm0959; ENABLE-NEXT: vpermps (%rdi){1to8}, %ymm1, %ymm0960; ENABLE-NEXT: vcvtdq2ps %ymm1, %ymm1961; ENABLE-NEXT: vaddps %ymm1, %ymm0, %ymm0962; ENABLE-NEXT: retq963;964; DISABLE-LABEL: permps_broadcast_256:965; DISABLE: # %bb.0:966; DISABLE-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill967; DISABLE-NEXT: #APP968; DISABLE-NEXT: nop969; DISABLE-NEXT: #NO_APP970; DISABLE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload971; DISABLE-NEXT: vpermps (%rdi){1to8}, %ymm1, %ymm0972; DISABLE-NEXT: vcvtdq2ps %ymm1, %ymm1973; DISABLE-NEXT: vaddps %ymm1, %ymm0, %ymm0974; DISABLE-NEXT: retq975 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()976 %v0 = load float, ptr %p0, align 4977 %t0 = insertelement <8 x float> undef, float %v0, i32 0978 %a0 = shufflevector <8 x float> %t0, <8 x float> undef, <8 x i32> zeroinitializer979 %2 = call <8 x float> @llvm.x86.avx512.mask.permvar.sf.256(<8 x float> %a0, <8 x i32> %idx, <8 x float> zeroinitializer, i8 -1)980 %a1 = sitofp <8 x i32> %idx to <8 x float>981 %res = fadd <8 x float> %2, %a1982 ret <8 x float> %res983}984 985define <8 x float> @permps_maskz_256(<8 x float> %a0, <8 x i32> %idx, ptr %mask) {986; ENABLE-LABEL: permps_maskz_256:987; ENABLE: # %bb.0:988; ENABLE-NEXT: #APP989; ENABLE-NEXT: nop990; ENABLE-NEXT: #NO_APP991; ENABLE-NEXT: kmovb (%rdi), %k1992; ENABLE-NEXT: vxorps %xmm2, %xmm2, %xmm2993; ENABLE-NEXT: vpermps %ymm0, %ymm1, %ymm2 {%k1} {z}994; ENABLE-NEXT: vcvtdq2ps %ymm1, %ymm1995; ENABLE-NEXT: vaddps %ymm1, %ymm0, %ymm0996; ENABLE-NEXT: vaddps %ymm0, %ymm2, %ymm0997; ENABLE-NEXT: retq998;999; DISABLE-LABEL: permps_maskz_256:1000; DISABLE: # %bb.0:1001; DISABLE-NEXT: #APP1002; DISABLE-NEXT: nop1003; DISABLE-NEXT: #NO_APP1004; DISABLE-NEXT: kmovb (%rdi), %k11005; DISABLE-NEXT: vpermps %ymm0, %ymm1, %ymm2 {%k1} {z}1006; DISABLE-NEXT: vcvtdq2ps %ymm1, %ymm11007; DISABLE-NEXT: vaddps %ymm1, %ymm0, %ymm01008; DISABLE-NEXT: vaddps %ymm0, %ymm2, %ymm01009; DISABLE-NEXT: retq1010 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1011 %2 = load i8, ptr %mask1012 %3 = call <8 x float> @llvm.x86.avx512.mask.permvar.sf.256(<8 x float> %a0, <8 x i32> %idx, <8 x float> zeroinitializer, i8 %2)1013 %a1 = sitofp <8 x i32> %idx to <8 x float>1014 %t = fadd <8 x float> %a0, %a11015 %res = fadd <8 x float> %3, %t1016 ret <8 x float> %res1017}1018 1019declare <8 x float> @llvm.x86.avx512.mask.permvar.sf.256(<8 x float>, <8 x i32>, <8 x float>, i8)1020 1021define <16 x float> @permps_rr_512(<16 x float> %a0, <16 x i32> %idx) {1022; ENABLE-LABEL: permps_rr_512:1023; ENABLE: # %bb.0:1024; ENABLE-NEXT: vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1025; ENABLE-NEXT: #APP1026; ENABLE-NEXT: nop1027; ENABLE-NEXT: #NO_APP1028; ENABLE-NEXT: vmovaps %zmm0, %zmm21029; ENABLE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload1030; ENABLE-NEXT: vpxor %xmm1, %xmm1, %xmm11031; ENABLE-NEXT: vpermps %zmm2, %zmm0, %zmm11032; ENABLE-NEXT: vcvtdq2ps %zmm0, %zmm01033; ENABLE-NEXT: vaddps %zmm0, %zmm2, %zmm01034; ENABLE-NEXT: vaddps %zmm0, %zmm1, %zmm01035; ENABLE-NEXT: retq1036;1037; DISABLE-LABEL: permps_rr_512:1038; DISABLE: # %bb.0:1039; DISABLE-NEXT: vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1040; DISABLE-NEXT: #APP1041; DISABLE-NEXT: nop1042; DISABLE-NEXT: #NO_APP1043; DISABLE-NEXT: vmovaps %zmm0, %zmm21044; DISABLE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload1045; DISABLE-NEXT: vpermps %zmm2, %zmm0, %zmm11046; DISABLE-NEXT: vcvtdq2ps %zmm0, %zmm01047; DISABLE-NEXT: vaddps %zmm0, %zmm2, %zmm01048; DISABLE-NEXT: vaddps %zmm0, %zmm1, %zmm01049; DISABLE-NEXT: retq1050 %1 = tail call <16 x float> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1051 %2 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %1, <16 x i32> %idx)1052 %a1 = sitofp <16 x i32> %idx to <16 x float>1053 %t = fadd <16 x float> %1, %a11054 %res = fadd <16 x float> %2, %t1055 ret <16 x float> %res1056}1057 1058define <16 x float> @permps_rm_512(ptr %p0, <16 x i32> %idx) {1059; ENABLE-LABEL: permps_rm_512:1060; ENABLE: # %bb.0:1061; ENABLE-NEXT: vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1062; ENABLE-NEXT: #APP1063; ENABLE-NEXT: nop1064; ENABLE-NEXT: #NO_APP1065; ENABLE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload1066; ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm01067; ENABLE-NEXT: vpermps (%rdi), %zmm1, %zmm01068; ENABLE-NEXT: vcvtdq2ps %zmm1, %zmm11069; ENABLE-NEXT: vaddps %zmm1, %zmm0, %zmm01070; ENABLE-NEXT: retq1071;1072; DISABLE-LABEL: permps_rm_512:1073; DISABLE: # %bb.0:1074; DISABLE-NEXT: vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1075; DISABLE-NEXT: #APP1076; DISABLE-NEXT: nop1077; DISABLE-NEXT: #NO_APP1078; DISABLE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload1079; DISABLE-NEXT: vpermps (%rdi), %zmm1, %zmm01080; DISABLE-NEXT: vcvtdq2ps %zmm1, %zmm11081; DISABLE-NEXT: vaddps %zmm1, %zmm0, %zmm01082; DISABLE-NEXT: retq1083 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1084 %a0 = load <16 x float>, ptr %p0, align 641085 %2 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %a0, <16 x i32> %idx)1086 %a1 = sitofp <16 x i32> %idx to <16 x float>1087 %res = fadd <16 x float> %2, %a11088 ret <16 x float> %res1089}1090 1091define <16 x float> @permps_broadcast_512(ptr %p0, <16 x i32> %idx) {1092; ENABLE-LABEL: permps_broadcast_512:1093; ENABLE: # %bb.0:1094; ENABLE-NEXT: vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1095; ENABLE-NEXT: #APP1096; ENABLE-NEXT: nop1097; ENABLE-NEXT: #NO_APP1098; ENABLE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload1099; ENABLE-NEXT: vpxor %xmm0, %xmm0, %xmm01100; ENABLE-NEXT: vpermps (%rdi){1to16}, %zmm1, %zmm01101; ENABLE-NEXT: vcvtdq2ps %zmm1, %zmm11102; ENABLE-NEXT: vaddps %zmm1, %zmm0, %zmm01103; ENABLE-NEXT: retq1104;1105; DISABLE-LABEL: permps_broadcast_512:1106; DISABLE: # %bb.0:1107; DISABLE-NEXT: vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill1108; DISABLE-NEXT: #APP1109; DISABLE-NEXT: nop1110; DISABLE-NEXT: #NO_APP1111; DISABLE-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload1112; DISABLE-NEXT: vpermps (%rdi){1to16}, %zmm1, %zmm01113; DISABLE-NEXT: vcvtdq2ps %zmm1, %zmm11114; DISABLE-NEXT: vaddps %zmm1, %zmm0, %zmm01115; DISABLE-NEXT: retq1116 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1117 %v0 = load float, ptr %p0, align 41118 %t0 = insertelement <16 x float> undef, float %v0, i32 01119 %a0 = shufflevector <16 x float> %t0, <16 x float> undef, <16 x i32> zeroinitializer1120 %2 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %a0, <16 x i32> %idx)1121 %a1 = sitofp <16 x i32> %idx to <16 x float>1122 %res = fadd <16 x float> %2, %a11123 ret <16 x float> %res1124}1125 1126define <16 x float> @permps_maskz_512(<16 x float> %a0, <16 x i32> %idx, ptr %mask) {1127; ENABLE-LABEL: permps_maskz_512:1128; ENABLE: # %bb.0:1129; ENABLE-NEXT: #APP1130; ENABLE-NEXT: nop1131; ENABLE-NEXT: #NO_APP1132; ENABLE-NEXT: kmovw (%rdi), %k11133; ENABLE-NEXT: vpxor %xmm2, %xmm2, %xmm21134; ENABLE-NEXT: vpermps %zmm0, %zmm1, %zmm2 {%k1} {z}1135; ENABLE-NEXT: vcvtdq2ps %zmm1, %zmm11136; ENABLE-NEXT: vaddps %zmm1, %zmm0, %zmm01137; ENABLE-NEXT: vaddps %zmm0, %zmm2, %zmm01138; ENABLE-NEXT: retq1139;1140; DISABLE-LABEL: permps_maskz_512:1141; DISABLE: # %bb.0:1142; DISABLE-NEXT: #APP1143; DISABLE-NEXT: nop1144; DISABLE-NEXT: #NO_APP1145; DISABLE-NEXT: kmovw (%rdi), %k11146; DISABLE-NEXT: vpermps %zmm0, %zmm1, %zmm2 {%k1} {z}1147; DISABLE-NEXT: vcvtdq2ps %zmm1, %zmm11148; DISABLE-NEXT: vaddps %zmm1, %zmm0, %zmm01149; DISABLE-NEXT: vaddps %zmm0, %zmm2, %zmm01150; DISABLE-NEXT: retq1151 %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()1152 %2 = load i16, ptr %mask1153 %3 = call <16 x float> @llvm.x86.avx512.mask.permvar.sf.512(<16 x float> %a0, <16 x i32> %idx, <16 x float> zeroinitializer, i16 %2)1154 %a1 = sitofp <16 x i32> %idx to <16 x float>1155 %t = fadd <16 x float> %a0, %a11156 %res = fadd <16 x float> %3, %t1157 ret <16 x float> %res1158}1159 1160declare <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float>, <16 x i32>)1161declare <16 x float> @llvm.x86.avx512.mask.permvar.sf.512(<16 x float>, <16 x i32>, <16 x float>, i16)1162