852 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512f,+avx512bw,+avx512vl,+avx512dq | FileCheck %s3 4declare <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float>, ptr, <16 x i32>, i16, i32)5declare void @llvm.x86.avx512.scatter.dps.512 (ptr, i16, <16 x i32>, <16 x float>, i32)6declare <8 x double> @llvm.x86.avx512.gather.dpd.512 (<8 x double>, ptr, <8 x i32>, i8, i32)7declare void @llvm.x86.avx512.scatter.dpd.512 (ptr, i8, <8 x i32>, <8 x double>, i32)8 9declare <8 x float> @llvm.x86.avx512.gather.qps.512 (<8 x float>, ptr, <8 x i64>, i8, i32)10declare void @llvm.x86.avx512.scatter.qps.512 (ptr, i8, <8 x i64>, <8 x float>, i32)11declare <8 x double> @llvm.x86.avx512.gather.qpd.512 (<8 x double>, ptr, <8 x i64>, i8, i32)12declare void @llvm.x86.avx512.scatter.qpd.512 (ptr, i8, <8 x i64>, <8 x double>, i32)13 14define void @gather_mask_dps(<16 x i32> %ind, <16 x float> %src, i16 %mask, ptr %base, ptr %stbuf) {15; CHECK-LABEL: gather_mask_dps:16; CHECK: ## %bb.0:17; CHECK-NEXT: kmovd %edi, %k118; CHECK-NEXT: kmovq %k1, %k219; CHECK-NEXT: vgatherdps (%rsi,%zmm0,4), %zmm1 {%k2}20; CHECK-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm021; CHECK-NEXT: vscatterdps %zmm1, (%rdx,%zmm0,4) {%k1}22; CHECK-NEXT: vzeroupper23; CHECK-NEXT: retq24 %x = call <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float> %src, ptr %base, <16 x i32>%ind, i16 %mask, i32 4)25 %ind2 = add <16 x i32> %ind, <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>26 call void @llvm.x86.avx512.scatter.dps.512 (ptr %stbuf, i16 %mask, <16 x i32>%ind2, <16 x float> %x, i32 4)27 ret void28}29 30define void @gather_mask_dpd(<8 x i32> %ind, <8 x double> %src, i8 %mask, ptr %base, ptr %stbuf) {31; CHECK-LABEL: gather_mask_dpd:32; CHECK: ## %bb.0:33; CHECK-NEXT: kmovd %edi, %k134; CHECK-NEXT: kmovq %k1, %k235; CHECK-NEXT: vgatherdpd (%rsi,%ymm0,4), %zmm1 {%k2}36; CHECK-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm037; CHECK-NEXT: vscatterdpd %zmm1, (%rdx,%ymm0,4) {%k1}38; CHECK-NEXT: vzeroupper39; CHECK-NEXT: retq40 %x = call <8 x double> @llvm.x86.avx512.gather.dpd.512 (<8 x double> %src, ptr %base, <8 x i32>%ind, i8 %mask, i32 4)41 %ind2 = add <8 x i32> %ind, <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>42 call void @llvm.x86.avx512.scatter.dpd.512 (ptr %stbuf, i8 %mask, <8 x i32>%ind2, <8 x double> %x, i32 4)43 ret void44}45 46define void @gather_mask_qps(<8 x i64> %ind, <8 x float> %src, i8 %mask, ptr %base, ptr %stbuf) {47; CHECK-LABEL: gather_mask_qps:48; CHECK: ## %bb.0:49; CHECK-NEXT: kmovd %edi, %k150; CHECK-NEXT: kmovq %k1, %k251; CHECK-NEXT: vgatherqps (%rsi,%zmm0,4), %ymm1 {%k2}52; CHECK-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm053; CHECK-NEXT: vscatterqps %ymm1, (%rdx,%zmm0,4) {%k1}54; CHECK-NEXT: vzeroupper55; CHECK-NEXT: retq56 %x = call <8 x float> @llvm.x86.avx512.gather.qps.512 (<8 x float> %src, ptr %base, <8 x i64>%ind, i8 %mask, i32 4)57 %ind2 = add <8 x i64> %ind, <i64 0, i64 1, i64 2, i64 3, i64 0, i64 1, i64 2, i64 3>58 call void @llvm.x86.avx512.scatter.qps.512 (ptr %stbuf, i8 %mask, <8 x i64>%ind2, <8 x float> %x, i32 4)59 ret void60}61 62define void @gather_mask_qpd(<8 x i64> %ind, <8 x double> %src, i8 %mask, ptr %base, ptr %stbuf) {63; CHECK-LABEL: gather_mask_qpd:64; CHECK: ## %bb.0:65; CHECK-NEXT: kmovd %edi, %k166; CHECK-NEXT: kmovq %k1, %k267; CHECK-NEXT: vgatherqpd (%rsi,%zmm0,4), %zmm1 {%k2}68; CHECK-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm069; CHECK-NEXT: vscatterqpd %zmm1, (%rdx,%zmm0,4) {%k1}70; CHECK-NEXT: vzeroupper71; CHECK-NEXT: retq72 %x = call <8 x double> @llvm.x86.avx512.gather.qpd.512 (<8 x double> %src, ptr %base, <8 x i64>%ind, i8 %mask, i32 4)73 %ind2 = add <8 x i64> %ind, <i64 0, i64 1, i64 2, i64 3, i64 0, i64 1, i64 2, i64 3>74 call void @llvm.x86.avx512.scatter.qpd.512 (ptr %stbuf, i8 %mask, <8 x i64>%ind2, <8 x double> %x, i32 4)75 ret void76}77;;78;; Integer Gather/Scatter79;;80declare <16 x i32> @llvm.x86.avx512.gather.dpi.512 (<16 x i32>, ptr, <16 x i32>, i16, i32)81declare void @llvm.x86.avx512.scatter.dpi.512 (ptr, i16, <16 x i32>, <16 x i32>, i32)82declare <8 x i64> @llvm.x86.avx512.gather.dpq.512 (<8 x i64>, ptr, <8 x i32>, i8, i32)83declare void @llvm.x86.avx512.scatter.dpq.512 (ptr, i8, <8 x i32>, <8 x i64>, i32)84 85declare <8 x i32> @llvm.x86.avx512.gather.qpi.512 (<8 x i32>, ptr, <8 x i64>, i8, i32)86declare void @llvm.x86.avx512.scatter.qpi.512 (ptr, i8, <8 x i64>, <8 x i32>, i32)87declare <8 x i64> @llvm.x86.avx512.gather.qpq.512 (<8 x i64>, ptr, <8 x i64>, i8, i32)88declare void @llvm.x86.avx512.scatter.qpq.512 (ptr, i8, <8 x i64>, <8 x i64>, i32)89 90define void @gather_mask_dd(<16 x i32> %ind, <16 x i32> %src, i16 %mask, ptr %base, ptr %stbuf) {91; CHECK-LABEL: gather_mask_dd:92; CHECK: ## %bb.0:93; CHECK-NEXT: kmovd %edi, %k194; CHECK-NEXT: kmovq %k1, %k295; CHECK-NEXT: vpgatherdd (%rsi,%zmm0,4), %zmm1 {%k2}96; CHECK-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm097; CHECK-NEXT: vpscatterdd %zmm1, (%rdx,%zmm0,4) {%k1}98; CHECK-NEXT: vzeroupper99; CHECK-NEXT: retq100 %x = call <16 x i32> @llvm.x86.avx512.gather.dpi.512 (<16 x i32> %src, ptr %base, <16 x i32>%ind, i16 %mask, i32 4)101 %ind2 = add <16 x i32> %ind, <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>102 call void @llvm.x86.avx512.scatter.dpi.512 (ptr %stbuf, i16 %mask, <16 x i32>%ind2, <16 x i32> %x, i32 4)103 ret void104}105 106define void @gather_mask_qd(<8 x i64> %ind, <8 x i32> %src, i8 %mask, ptr %base, ptr %stbuf) {107; CHECK-LABEL: gather_mask_qd:108; CHECK: ## %bb.0:109; CHECK-NEXT: kmovd %edi, %k1110; CHECK-NEXT: kmovq %k1, %k2111; CHECK-NEXT: vpgatherqd (%rsi,%zmm0,4), %ymm1 {%k2}112; CHECK-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0113; CHECK-NEXT: vpscatterqd %ymm1, (%rdx,%zmm0,4) {%k1}114; CHECK-NEXT: vzeroupper115; CHECK-NEXT: retq116 %x = call <8 x i32> @llvm.x86.avx512.gather.qpi.512 (<8 x i32> %src, ptr %base, <8 x i64>%ind, i8 %mask, i32 4)117 %ind2 = add <8 x i64> %ind, <i64 0, i64 1, i64 2, i64 3, i64 0, i64 1, i64 2, i64 3>118 call void @llvm.x86.avx512.scatter.qpi.512 (ptr %stbuf, i8 %mask, <8 x i64>%ind2, <8 x i32> %x, i32 4)119 ret void120}121 122define void @gather_mask_qq(<8 x i64> %ind, <8 x i64> %src, i8 %mask, ptr %base, ptr %stbuf) {123; CHECK-LABEL: gather_mask_qq:124; CHECK: ## %bb.0:125; CHECK-NEXT: kmovd %edi, %k1126; CHECK-NEXT: kmovq %k1, %k2127; CHECK-NEXT: vpgatherqq (%rsi,%zmm0,4), %zmm1 {%k2}128; CHECK-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0129; CHECK-NEXT: vpscatterqq %zmm1, (%rdx,%zmm0,4) {%k1}130; CHECK-NEXT: vzeroupper131; CHECK-NEXT: retq132 %x = call <8 x i64> @llvm.x86.avx512.gather.qpq.512 (<8 x i64> %src, ptr %base, <8 x i64>%ind, i8 %mask, i32 4)133 %ind2 = add <8 x i64> %ind, <i64 0, i64 1, i64 2, i64 3, i64 0, i64 1, i64 2, i64 3>134 call void @llvm.x86.avx512.scatter.qpq.512 (ptr %stbuf, i8 %mask, <8 x i64>%ind2, <8 x i64> %x, i32 4)135 ret void136}137 138define void @gather_mask_dq(<8 x i32> %ind, <8 x i64> %src, i8 %mask, ptr %base, ptr %stbuf) {139; CHECK-LABEL: gather_mask_dq:140; CHECK: ## %bb.0:141; CHECK-NEXT: kmovd %edi, %k1142; CHECK-NEXT: kmovq %k1, %k2143; CHECK-NEXT: vpgatherdq (%rsi,%ymm0,4), %zmm1 {%k2}144; CHECK-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0145; CHECK-NEXT: vpscatterdq %zmm1, (%rdx,%ymm0,4) {%k1}146; CHECK-NEXT: vzeroupper147; CHECK-NEXT: retq148 %x = call <8 x i64> @llvm.x86.avx512.gather.dpq.512 (<8 x i64> %src, ptr %base, <8 x i32>%ind, i8 %mask, i32 4)149 %ind2 = add <8 x i32> %ind, <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>150 call void @llvm.x86.avx512.scatter.dpq.512 (ptr %stbuf, i8 %mask, <8 x i32>%ind2, <8 x i64> %x, i32 4)151 ret void152}153 154define void @gather_mask_dpd_execdomain(<8 x i32> %ind, <8 x double> %src, i8 %mask, ptr %base, ptr %stbuf) {155; CHECK-LABEL: gather_mask_dpd_execdomain:156; CHECK: ## %bb.0:157; CHECK-NEXT: kmovd %edi, %k1158; CHECK-NEXT: vgatherdpd (%rsi,%ymm0,4), %zmm1 {%k1}159; CHECK-NEXT: vmovapd %zmm1, (%rdx)160; CHECK-NEXT: vzeroupper161; CHECK-NEXT: retq162 %x = call <8 x double> @llvm.x86.avx512.gather.dpd.512 (<8 x double> %src, ptr %base, <8 x i32>%ind, i8 %mask, i32 4)163 store <8 x double> %x, ptr %stbuf164 ret void165}166 167define void @gather_mask_qpd_execdomain(<8 x i64> %ind, <8 x double> %src, i8 %mask, ptr %base, ptr %stbuf) {168; CHECK-LABEL: gather_mask_qpd_execdomain:169; CHECK: ## %bb.0:170; CHECK-NEXT: kmovd %edi, %k1171; CHECK-NEXT: vgatherqpd (%rsi,%zmm0,4), %zmm1 {%k1}172; CHECK-NEXT: vmovapd %zmm1, (%rdx)173; CHECK-NEXT: vzeroupper174; CHECK-NEXT: retq175 %x = call <8 x double> @llvm.x86.avx512.gather.qpd.512 (<8 x double> %src, ptr %base, <8 x i64>%ind, i8 %mask, i32 4)176 store <8 x double> %x, ptr %stbuf177 ret void178}179 180define <16 x float> @gather_mask_dps_execdomain(<16 x i32> %ind, <16 x float> %src, i16 %mask, ptr %base) {181; CHECK-LABEL: gather_mask_dps_execdomain:182; CHECK: ## %bb.0:183; CHECK-NEXT: kmovd %edi, %k1184; CHECK-NEXT: vgatherdps (%rsi,%zmm0,4), %zmm1 {%k1}185; CHECK-NEXT: vmovaps %zmm1, %zmm0186; CHECK-NEXT: retq187 %res = call <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float> %src, ptr %base, <16 x i32>%ind, i16 %mask, i32 4)188 ret <16 x float> %res;189}190 191define <8 x float> @gather_mask_qps_execdomain(<8 x i64> %ind, <8 x float> %src, i8 %mask, ptr %base) {192; CHECK-LABEL: gather_mask_qps_execdomain:193; CHECK: ## %bb.0:194; CHECK-NEXT: kmovd %edi, %k1195; CHECK-NEXT: vgatherqps (%rsi,%zmm0,4), %ymm1 {%k1}196; CHECK-NEXT: vmovaps %ymm1, %ymm0197; CHECK-NEXT: retq198 %res = call <8 x float> @llvm.x86.avx512.gather.qps.512 (<8 x float> %src, ptr %base, <8 x i64>%ind, i8 %mask, i32 4)199 ret <8 x float> %res;200}201 202define void @scatter_mask_dpd_execdomain(<8 x i32> %ind, ptr %src, i8 %mask, ptr %base, ptr %stbuf) {203; CHECK-LABEL: scatter_mask_dpd_execdomain:204; CHECK: ## %bb.0:205; CHECK-NEXT: kmovd %esi, %k1206; CHECK-NEXT: vmovapd (%rdi), %zmm1207; CHECK-NEXT: vscatterdpd %zmm1, (%rcx,%ymm0,4) {%k1}208; CHECK-NEXT: vzeroupper209; CHECK-NEXT: retq210 %x = load <8 x double>, ptr %src, align 64211 call void @llvm.x86.avx512.scatter.dpd.512 (ptr %stbuf, i8 %mask, <8 x i32>%ind, <8 x double> %x, i32 4)212 ret void213}214 215define void @scatter_mask_qpd_execdomain(<8 x i64> %ind, ptr %src, i8 %mask, ptr %base, ptr %stbuf) {216; CHECK-LABEL: scatter_mask_qpd_execdomain:217; CHECK: ## %bb.0:218; CHECK-NEXT: kmovd %esi, %k1219; CHECK-NEXT: vmovapd (%rdi), %zmm1220; CHECK-NEXT: vscatterqpd %zmm1, (%rcx,%zmm0,4) {%k1}221; CHECK-NEXT: vzeroupper222; CHECK-NEXT: retq223 %x = load <8 x double>, ptr %src, align 64224 call void @llvm.x86.avx512.scatter.qpd.512 (ptr %stbuf, i8 %mask, <8 x i64>%ind, <8 x double> %x, i32 4)225 ret void226}227 228define void @scatter_mask_dps_execdomain(<16 x i32> %ind, ptr %src, i16 %mask, ptr %base, ptr %stbuf) {229; CHECK-LABEL: scatter_mask_dps_execdomain:230; CHECK: ## %bb.0:231; CHECK-NEXT: kmovd %esi, %k1232; CHECK-NEXT: vmovaps (%rdi), %zmm1233; CHECK-NEXT: vscatterdps %zmm1, (%rcx,%zmm0,4) {%k1}234; CHECK-NEXT: vzeroupper235; CHECK-NEXT: retq236 %x = load <16 x float>, ptr %src, align 64237 call void @llvm.x86.avx512.scatter.dps.512 (ptr %stbuf, i16 %mask, <16 x i32>%ind, <16 x float> %x, i32 4)238 ret void239}240 241define void @scatter_mask_qps_execdomain(<8 x i64> %ind, ptr %src, i8 %mask, ptr %base, ptr %stbuf) {242; CHECK-LABEL: scatter_mask_qps_execdomain:243; CHECK: ## %bb.0:244; CHECK-NEXT: kmovd %esi, %k1245; CHECK-NEXT: vmovaps (%rdi), %ymm1246; CHECK-NEXT: vscatterqps %ymm1, (%rcx,%zmm0,4) {%k1}247; CHECK-NEXT: vzeroupper248; CHECK-NEXT: retq249 %x = load <8 x float>, ptr %src, align 32250 call void @llvm.x86.avx512.scatter.qps.512 (ptr %stbuf, i8 %mask, <8 x i64>%ind, <8 x float> %x, i32 4)251 ret void252}253 254define void @gather_qps(<8 x i64> %ind, <8 x float> %src, ptr %base, ptr %stbuf) {255; CHECK-LABEL: gather_qps:256; CHECK: ## %bb.0:257; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1258; CHECK-NEXT: kxnorb %k0, %k0, %k1259; CHECK-NEXT: kxnorb %k0, %k0, %k2260; CHECK-NEXT: vgatherqps (%rdi,%zmm0,4), %ymm1 {%k2}261; CHECK-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0262; CHECK-NEXT: vscatterqps %ymm1, (%rsi,%zmm0,4) {%k1}263; CHECK-NEXT: vzeroupper264; CHECK-NEXT: retq265 %x = call <8 x float> @llvm.x86.avx512.gather.qps.512 (<8 x float> %src, ptr %base, <8 x i64>%ind, i8 -1, i32 4)266 %ind2 = add <8 x i64> %ind, <i64 0, i64 1, i64 2, i64 3, i64 0, i64 1, i64 2, i64 3>267 call void @llvm.x86.avx512.scatter.qps.512 (ptr %stbuf, i8 -1, <8 x i64>%ind2, <8 x float> %x, i32 4)268 ret void269}270 271declare <2 x double> @llvm.x86.avx512.gather3div2.df(<2 x double>, ptr, <2 x i64>, i8, i32)272 273define <2 x double>@test_int_x86_avx512_gather3div2_df(<2 x double> %x0, ptr %x1, <2 x i64> %x2, i8 %x3) {274; CHECK-LABEL: test_int_x86_avx512_gather3div2_df:275; CHECK: ## %bb.0:276; CHECK-NEXT: kmovd %esi, %k1277; CHECK-NEXT: vgatherqpd (%rdi,%xmm1,4), %xmm0 {%k1}278; CHECK-NEXT: kxnorw %k0, %k0, %k1279; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2280; CHECK-NEXT: vgatherqpd (%rdi,%xmm1,2), %xmm2 {%k1}281; CHECK-NEXT: vaddpd %xmm2, %xmm0, %xmm0282; CHECK-NEXT: retq283 %res = call <2 x double> @llvm.x86.avx512.gather3div2.df(<2 x double> %x0, ptr %x1, <2 x i64> %x2, i8 %x3, i32 4)284 %res1 = call <2 x double> @llvm.x86.avx512.gather3div2.df(<2 x double> %x0, ptr %x1, <2 x i64> %x2, i8 -1, i32 2)285 %res2 = fadd <2 x double> %res, %res1286 ret <2 x double> %res2287}288 289declare <2 x i64> @llvm.x86.avx512.gather3div2.di(<2 x i64>, ptr, <2 x i64>, i8, i32)290 291define <2 x i64>@test_int_x86_avx512_gather3div2_di(<2 x i64> %x0, ptr %x1, <2 x i64> %x2, i8 %x3) {292; CHECK-LABEL: test_int_x86_avx512_gather3div2_di:293; CHECK: ## %bb.0:294; CHECK-NEXT: kmovd %esi, %k1295; CHECK-NEXT: vpgatherqq (%rdi,%xmm1,8), %xmm0 {%k1}296; CHECK-NEXT: vpaddq %xmm0, %xmm0, %xmm0297; CHECK-NEXT: retq298 %res = call <2 x i64> @llvm.x86.avx512.gather3div2.di(<2 x i64> %x0, ptr %x1, <2 x i64> %x2, i8 %x3, i32 8)299 %res1 = call <2 x i64> @llvm.x86.avx512.gather3div2.di(<2 x i64> %x0, ptr %x1, <2 x i64> %x2, i8 %x3, i32 8)300 %res2 = add <2 x i64> %res, %res1301 ret <2 x i64> %res2302}303 304declare <4 x double> @llvm.x86.avx512.gather3div4.df(<4 x double>, ptr, <4 x i64>, i8, i32)305 306define <4 x double>@test_int_x86_avx512_gather3div4_df(<4 x double> %x0, ptr %x1, <4 x i64> %x2, i8 %x3) {307; CHECK-LABEL: test_int_x86_avx512_gather3div4_df:308; CHECK: ## %bb.0:309; CHECK-NEXT: kmovd %esi, %k1310; CHECK-NEXT: vgatherqpd (%rdi,%ymm1,4), %ymm0 {%k1}311; CHECK-NEXT: kxnorw %k0, %k0, %k1312; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2313; CHECK-NEXT: vgatherqpd (%rdi,%ymm1,2), %ymm2 {%k1}314; CHECK-NEXT: vaddpd %ymm2, %ymm0, %ymm0315; CHECK-NEXT: retq316 %res = call <4 x double> @llvm.x86.avx512.gather3div4.df(<4 x double> %x0, ptr %x1, <4 x i64> %x2, i8 %x3, i32 4)317 %res1 = call <4 x double> @llvm.x86.avx512.gather3div4.df(<4 x double> %x0, ptr %x1, <4 x i64> %x2, i8 -1, i32 2)318 %res2 = fadd <4 x double> %res, %res1319 ret <4 x double> %res2320}321 322declare <4 x i64> @llvm.x86.avx512.gather3div4.di(<4 x i64>, ptr, <4 x i64>, i8, i32)323 324define <4 x i64>@test_int_x86_avx512_gather3div4_di(<4 x i64> %x0, ptr %x1, <4 x i64> %x2, i8 %x3) {325; CHECK-LABEL: test_int_x86_avx512_gather3div4_di:326; CHECK: ## %bb.0:327; CHECK-NEXT: kmovd %esi, %k1328; CHECK-NEXT: vpgatherqq (%rdi,%ymm1,8), %ymm0 {%k1}329; CHECK-NEXT: kxnorw %k0, %k0, %k1330; CHECK-NEXT: vpxor %xmm2, %xmm2, %xmm2331; CHECK-NEXT: vpgatherqq (%rdi,%ymm1,8), %ymm2 {%k1}332; CHECK-NEXT: vpaddq %ymm2, %ymm0, %ymm0333; CHECK-NEXT: retq334 %res = call <4 x i64> @llvm.x86.avx512.gather3div4.di(<4 x i64> %x0, ptr %x1, <4 x i64> %x2, i8 %x3, i32 8)335 %res1 = call <4 x i64> @llvm.x86.avx512.gather3div4.di(<4 x i64> %x0, ptr %x1, <4 x i64> %x2, i8 -1, i32 8)336 %res2 = add <4 x i64> %res, %res1337 ret <4 x i64> %res2338}339 340declare <4 x float> @llvm.x86.avx512.gather3div4.sf(<4 x float>, ptr, <2 x i64>, i8, i32)341 342define <4 x float>@test_int_x86_avx512_gather3div4_sf(<4 x float> %x0, ptr %x1, <2 x i64> %x2, i8 %x3) {343; CHECK-LABEL: test_int_x86_avx512_gather3div4_sf:344; CHECK: ## %bb.0:345; CHECK-NEXT: kmovd %esi, %k1346; CHECK-NEXT: vgatherqps (%rdi,%xmm1,4), %xmm0 {%k1}347; CHECK-NEXT: kxnorw %k0, %k0, %k1348; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2349; CHECK-NEXT: vgatherqps (%rdi,%xmm1,2), %xmm2 {%k1}350; CHECK-NEXT: vaddps %xmm2, %xmm0, %xmm0351; CHECK-NEXT: retq352 %res = call <4 x float> @llvm.x86.avx512.gather3div4.sf(<4 x float> %x0, ptr %x1, <2 x i64> %x2, i8 %x3, i32 4)353 %res1 = call <4 x float> @llvm.x86.avx512.gather3div4.sf(<4 x float> %x0, ptr %x1, <2 x i64> %x2, i8 -1, i32 2)354 %res2 = fadd <4 x float> %res, %res1355 ret <4 x float> %res2356}357 358declare <4 x i32> @llvm.x86.avx512.gather3div4.si(<4 x i32>, ptr, <2 x i64>, i8, i32)359 360define <4 x i32>@test_int_x86_avx512_gather3div4_si(<4 x i32> %x0, ptr %x1, <2 x i64> %x2, i8 %x3) {361; CHECK-LABEL: test_int_x86_avx512_gather3div4_si:362; CHECK: ## %bb.0:363; CHECK-NEXT: kmovd %esi, %k1364; CHECK-NEXT: kxnorw %k0, %k0, %k2365; CHECK-NEXT: vpxor %xmm2, %xmm2, %xmm2366; CHECK-NEXT: vpgatherqd (%rdi,%xmm1,4), %xmm2 {%k2}367; CHECK-NEXT: vpgatherqd (%rdi,%xmm1,4), %xmm0 {%k1}368; CHECK-NEXT: vpaddd %xmm0, %xmm2, %xmm0369; CHECK-NEXT: retq370 %res = call <4 x i32> @llvm.x86.avx512.gather3div4.si(<4 x i32> %x0, ptr %x1, <2 x i64> %x2, i8 -1, i32 4)371 %res1 = call <4 x i32> @llvm.x86.avx512.gather3div4.si(<4 x i32> %x0, ptr %x1, <2 x i64> %x2, i8 %x3, i32 4)372 %res2 = add <4 x i32> %res, %res1373 ret <4 x i32> %res2374}375 376declare <4 x float> @llvm.x86.avx512.gather3div8.sf(<4 x float>, ptr, <4 x i64>, i8, i32)377 378define <4 x float>@test_int_x86_avx512_gather3div8_sf(<4 x float> %x0, ptr %x1, <4 x i64> %x2, i8 %x3) {379; CHECK-LABEL: test_int_x86_avx512_gather3div8_sf:380; CHECK: ## %bb.0:381; CHECK-NEXT: kmovd %esi, %k1382; CHECK-NEXT: vgatherqps (%rdi,%ymm1,4), %xmm0 {%k1}383; CHECK-NEXT: kxnorw %k0, %k0, %k1384; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2385; CHECK-NEXT: vgatherqps (%rdi,%ymm1,2), %xmm2 {%k1}386; CHECK-NEXT: vaddps %xmm2, %xmm0, %xmm0387; CHECK-NEXT: vzeroupper388; CHECK-NEXT: retq389 %res = call <4 x float> @llvm.x86.avx512.gather3div8.sf(<4 x float> %x0, ptr %x1, <4 x i64> %x2, i8 %x3, i32 4)390 %res1 = call <4 x float> @llvm.x86.avx512.gather3div8.sf(<4 x float> %x0, ptr %x1, <4 x i64> %x2, i8 -1, i32 2)391 %res2 = fadd <4 x float> %res, %res1392 ret <4 x float> %res2393}394 395declare <4 x i32> @llvm.x86.avx512.gather3div8.si(<4 x i32>, ptr, <4 x i64>, i8, i32)396 397define <4 x i32>@test_int_x86_avx512_gather3div8_si(<4 x i32> %x0, ptr %x1, <4 x i64> %x2, i8 %x3) {398; CHECK-LABEL: test_int_x86_avx512_gather3div8_si:399; CHECK: ## %bb.0:400; CHECK-NEXT: kmovd %esi, %k1401; CHECK-NEXT: vmovdqa %xmm0, %xmm2402; CHECK-NEXT: kmovq %k1, %k2403; CHECK-NEXT: vpgatherqd (%rdi,%ymm1,4), %xmm2 {%k2}404; CHECK-NEXT: vpgatherqd (%rdi,%ymm1,2), %xmm0 {%k1}405; CHECK-NEXT: vpaddd %xmm0, %xmm2, %xmm0406; CHECK-NEXT: vzeroupper407; CHECK-NEXT: retq408 %res = call <4 x i32> @llvm.x86.avx512.gather3div8.si(<4 x i32> %x0, ptr %x1, <4 x i64> %x2, i8 %x3, i32 4)409 %res1 = call <4 x i32> @llvm.x86.avx512.gather3div8.si(<4 x i32> %x0, ptr %x1, <4 x i64> %x2, i8 %x3, i32 2)410 %res2 = add <4 x i32> %res, %res1411 ret <4 x i32> %res2412}413 414declare <2 x double> @llvm.x86.avx512.gather3siv2.df(<2 x double>, ptr, <4 x i32>, i8, i32)415 416define <2 x double>@test_int_x86_avx512_gather3siv2_df(<2 x double> %x0, ptr %x1, <4 x i32> %x2, i8 %x3) {417; CHECK-LABEL: test_int_x86_avx512_gather3siv2_df:418; CHECK: ## %bb.0:419; CHECK-NEXT: kmovd %esi, %k1420; CHECK-NEXT: vgatherdpd (%rdi,%xmm1,4), %xmm0 {%k1}421; CHECK-NEXT: kxnorw %k0, %k0, %k1422; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2423; CHECK-NEXT: vgatherdpd (%rdi,%xmm1,2), %xmm2 {%k1}424; CHECK-NEXT: vaddpd %xmm2, %xmm0, %xmm0425; CHECK-NEXT: retq426 %res = call <2 x double> @llvm.x86.avx512.gather3siv2.df(<2 x double> %x0, ptr %x1, <4 x i32> %x2, i8 %x3, i32 4)427 %res1 = call <2 x double> @llvm.x86.avx512.gather3siv2.df(<2 x double> %x0, ptr %x1, <4 x i32> %x2, i8 -1, i32 2)428 %res2 = fadd <2 x double> %res, %res1429 ret <2 x double> %res2430}431 432declare <2 x i64> @llvm.x86.avx512.gather3siv2.di(<2 x i64>, ptr, <4 x i32>, i8, i32)433 434define <2 x i64>@test_int_x86_avx512_gather3siv2_di(<2 x i64> %x0, ptr %x1, <4 x i32> %x2, i8 %x3) {435; CHECK-LABEL: test_int_x86_avx512_gather3siv2_di:436; CHECK: ## %bb.0:437; CHECK-NEXT: kmovd %esi, %k1438; CHECK-NEXT: vpgatherdq (%rdi,%xmm1,8), %xmm0 {%k1}439; CHECK-NEXT: vpaddq %xmm0, %xmm0, %xmm0440; CHECK-NEXT: retq441 %res = call <2 x i64> @llvm.x86.avx512.gather3siv2.di(<2 x i64> %x0, ptr %x1, <4 x i32> %x2, i8 %x3, i32 8)442 %res1 = call <2 x i64> @llvm.x86.avx512.gather3siv2.di(<2 x i64> %x0, ptr %x1, <4 x i32> %x2, i8 %x3, i32 8)443 %res2 = add <2 x i64> %res, %res1444 ret <2 x i64> %res2445}446 447declare <4 x double> @llvm.x86.avx512.gather3siv4.df(<4 x double>, ptr, <4 x i32>, i8, i32)448 449define <4 x double>@test_int_x86_avx512_gather3siv4_df(<4 x double> %x0, ptr %x1, <4 x i32> %x2, i8 %x3) {450; CHECK-LABEL: test_int_x86_avx512_gather3siv4_df:451; CHECK: ## %bb.0:452; CHECK-NEXT: kmovd %esi, %k1453; CHECK-NEXT: vgatherdpd (%rdi,%xmm1,4), %ymm0 {%k1}454; CHECK-NEXT: kxnorw %k0, %k0, %k1455; CHECK-NEXT: vxorpd %xmm2, %xmm2, %xmm2456; CHECK-NEXT: vgatherdpd (%rdi,%xmm1,2), %ymm2 {%k1}457; CHECK-NEXT: vaddpd %ymm2, %ymm0, %ymm0458; CHECK-NEXT: retq459 %res = call <4 x double> @llvm.x86.avx512.gather3siv4.df(<4 x double> %x0, ptr %x1, <4 x i32> %x2, i8 %x3, i32 4)460 %res1 = call <4 x double> @llvm.x86.avx512.gather3siv4.df(<4 x double> %x0, ptr %x1, <4 x i32> %x2, i8 -1, i32 2)461 %res2 = fadd <4 x double> %res, %res1462 ret <4 x double> %res2463}464 465declare <4 x i64> @llvm.x86.avx512.gather3siv4.di(<4 x i64>, ptr, <4 x i32>, i8, i32)466 467define <4 x i64>@test_int_x86_avx512_gather3siv4_di(<4 x i64> %x0, ptr %x1, <4 x i32> %x2, i8 %x3) {468; CHECK-LABEL: test_int_x86_avx512_gather3siv4_di:469; CHECK: ## %bb.0:470; CHECK-NEXT: kmovd %esi, %k1471; CHECK-NEXT: vpgatherdq (%rdi,%xmm1,8), %ymm0 {%k1}472; CHECK-NEXT: vpaddq %ymm0, %ymm0, %ymm0473; CHECK-NEXT: retq474 %res = call <4 x i64> @llvm.x86.avx512.gather3siv4.di(<4 x i64> %x0, ptr %x1, <4 x i32> %x2, i8 %x3, i32 8)475 %res1 = call <4 x i64> @llvm.x86.avx512.gather3siv4.di(<4 x i64> %x0, ptr %x1, <4 x i32> %x2, i8 %x3, i32 8)476 %res2 = add <4 x i64> %res, %res1477 ret <4 x i64> %res2478}479 480declare <4 x float> @llvm.x86.avx512.gather3siv4.sf(<4 x float>, ptr, <4 x i32>, i8, i32)481 482define <4 x float>@test_int_x86_avx512_gather3siv4_sf(<4 x float> %x0, ptr %x1, <4 x i32> %x2, i8 %x3) {483; CHECK-LABEL: test_int_x86_avx512_gather3siv4_sf:484; CHECK: ## %bb.0:485; CHECK-NEXT: kmovd %esi, %k1486; CHECK-NEXT: vgatherdps (%rdi,%xmm1,4), %xmm0 {%k1}487; CHECK-NEXT: kxnorw %k0, %k0, %k1488; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2489; CHECK-NEXT: vgatherdps (%rdi,%xmm1,2), %xmm2 {%k1}490; CHECK-NEXT: vaddps %xmm2, %xmm0, %xmm0491; CHECK-NEXT: retq492 %res = call <4 x float> @llvm.x86.avx512.gather3siv4.sf(<4 x float> %x0, ptr %x1, <4 x i32> %x2, i8 %x3, i32 4)493 %res1 = call <4 x float> @llvm.x86.avx512.gather3siv4.sf(<4 x float> %x0, ptr %x1, <4 x i32> %x2, i8 -1, i32 2)494 %res2 = fadd <4 x float> %res, %res1495 ret <4 x float> %res2496}497 498declare <4 x i32> @llvm.x86.avx512.gather3siv4.si(<4 x i32>, ptr, <4 x i32>, i8, i32)499 500define <4 x i32>@test_int_x86_avx512_gather3siv4_si(<4 x i32> %x0, ptr %x1, <4 x i32> %x2, i8 %x3) {501; CHECK-LABEL: test_int_x86_avx512_gather3siv4_si:502; CHECK: ## %bb.0:503; CHECK-NEXT: kmovd %esi, %k1504; CHECK-NEXT: kxnorw %k0, %k0, %k2505; CHECK-NEXT: vpxor %xmm2, %xmm2, %xmm2506; CHECK-NEXT: vpgatherdd (%rdi,%xmm1,4), %xmm2 {%k2}507; CHECK-NEXT: vpgatherdd (%rdi,%xmm1,2), %xmm0 {%k1}508; CHECK-NEXT: vpaddd %xmm0, %xmm2, %xmm0509; CHECK-NEXT: retq510 %res = call <4 x i32> @llvm.x86.avx512.gather3siv4.si(<4 x i32> %x0, ptr %x1, <4 x i32> %x2, i8 -1, i32 4)511 %res1 = call <4 x i32> @llvm.x86.avx512.gather3siv4.si(<4 x i32> %x0, ptr %x1, <4 x i32> %x2, i8 %x3, i32 2)512 %res2 = add <4 x i32> %res, %res1513 ret <4 x i32> %res2514}515 516declare <8 x float> @llvm.x86.avx512.gather3siv8.sf(<8 x float>, ptr, <8 x i32>, i8, i32)517 518define <8 x float>@test_int_x86_avx512_gather3siv8_sf(<8 x float> %x0, ptr %x1, <8 x i32> %x2, i8 %x3) {519; CHECK-LABEL: test_int_x86_avx512_gather3siv8_sf:520; CHECK: ## %bb.0:521; CHECK-NEXT: kmovd %esi, %k1522; CHECK-NEXT: vgatherdps (%rdi,%ymm1,4), %ymm0 {%k1}523; CHECK-NEXT: kxnorb %k0, %k0, %k1524; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2525; CHECK-NEXT: vgatherdps (%rdi,%ymm1,2), %ymm2 {%k1}526; CHECK-NEXT: vaddps %ymm2, %ymm0, %ymm0527; CHECK-NEXT: retq528 %res = call <8 x float> @llvm.x86.avx512.gather3siv8.sf(<8 x float> %x0, ptr %x1, <8 x i32> %x2, i8 %x3, i32 4)529 %res1 = call <8 x float> @llvm.x86.avx512.gather3siv8.sf(<8 x float> %x0, ptr %x1, <8 x i32> %x2, i8 -1, i32 2)530 %res2 = fadd <8 x float> %res, %res1531 ret <8 x float> %res2532}533 534declare <8 x i32> @llvm.x86.avx512.gather3siv8.si(<8 x i32>, ptr, <8 x i32>, i8, i32)535 536define <8 x i32>@test_int_x86_avx512_gather3siv8_si(<8 x i32> %x0, ptr %x1, <8 x i32> %x2, i8 %x3) {537; CHECK-LABEL: test_int_x86_avx512_gather3siv8_si:538; CHECK: ## %bb.0:539; CHECK-NEXT: kmovd %esi, %k1540; CHECK-NEXT: vmovdqa %ymm0, %ymm2541; CHECK-NEXT: kmovq %k1, %k2542; CHECK-NEXT: vpgatherdd (%rdi,%ymm1,4), %ymm2 {%k2}543; CHECK-NEXT: vpgatherdd (%rdi,%ymm1,2), %ymm0 {%k1}544; CHECK-NEXT: vpaddd %ymm0, %ymm2, %ymm0545; CHECK-NEXT: retq546 %res = call <8 x i32> @llvm.x86.avx512.gather3siv8.si(<8 x i32> %x0, ptr %x1, <8 x i32> %x2, i8 %x3, i32 4)547 %res1 = call <8 x i32> @llvm.x86.avx512.gather3siv8.si(<8 x i32> %x0, ptr %x1, <8 x i32> %x2, i8 %x3, i32 2)548 %res2 = add <8 x i32> %res, %res1549 ret <8 x i32> %res2550}551 552declare void @llvm.x86.avx512.scatterdiv2.df(ptr, i8, <2 x i64>, <2 x double>, i32)553 554define void@test_int_x86_avx512_scatterdiv2_df(ptr %x0, i8 %x1, <2 x i64> %x2, <2 x double> %x3) {555; CHECK-LABEL: test_int_x86_avx512_scatterdiv2_df:556; CHECK: ## %bb.0:557; CHECK-NEXT: kmovd %esi, %k1558; CHECK-NEXT: kxnorw %k0, %k0, %k2559; CHECK-NEXT: vscatterqpd %xmm1, (%rdi,%xmm0,2) {%k2}560; CHECK-NEXT: vscatterqpd %xmm1, (%rdi,%xmm0,4) {%k1}561; CHECK-NEXT: retq562 call void @llvm.x86.avx512.scatterdiv2.df(ptr %x0, i8 -1, <2 x i64> %x2, <2 x double> %x3, i32 2)563 call void @llvm.x86.avx512.scatterdiv2.df(ptr %x0, i8 %x1, <2 x i64> %x2, <2 x double> %x3, i32 4)564 ret void565}566 567declare void @llvm.x86.avx512.scatterdiv2.di(ptr, i8, <2 x i64>, <2 x i64>, i32)568 569define void@test_int_x86_avx512_scatterdiv2_di(ptr %x0, i8 %x1, <2 x i64> %x2, <2 x i64> %x3) {570; CHECK-LABEL: test_int_x86_avx512_scatterdiv2_di:571; CHECK: ## %bb.0:572; CHECK-NEXT: kmovd %esi, %k1573; CHECK-NEXT: vpscatterqq %xmm1, (%rdi,%xmm0,2) {%k1}574; CHECK-NEXT: kxnorw %k0, %k0, %k1575; CHECK-NEXT: vpscatterqq %xmm1, (%rdi,%xmm0,4) {%k1}576; CHECK-NEXT: retq577 call void @llvm.x86.avx512.scatterdiv2.di(ptr %x0, i8 %x1, <2 x i64> %x2, <2 x i64> %x3, i32 2)578 call void @llvm.x86.avx512.scatterdiv2.di(ptr %x0, i8 -1, <2 x i64> %x2, <2 x i64> %x3, i32 4)579 ret void580}581 582declare void @llvm.x86.avx512.scatterdiv4.df(ptr, i8, <4 x i64>, <4 x double>, i32)583 584define void@test_int_x86_avx512_scatterdiv4_df(ptr %x0, i8 %x1, <4 x i64> %x2, <4 x double> %x3) {585; CHECK-LABEL: test_int_x86_avx512_scatterdiv4_df:586; CHECK: ## %bb.0:587; CHECK-NEXT: kmovd %esi, %k1588; CHECK-NEXT: vscatterqpd %ymm1, (%rdi,%ymm0,2) {%k1}589; CHECK-NEXT: kxnorw %k0, %k0, %k1590; CHECK-NEXT: vscatterqpd %ymm1, (%rdi,%ymm0,4) {%k1}591; CHECK-NEXT: vzeroupper592; CHECK-NEXT: retq593 call void @llvm.x86.avx512.scatterdiv4.df(ptr %x0, i8 %x1, <4 x i64> %x2, <4 x double> %x3, i32 2)594 call void @llvm.x86.avx512.scatterdiv4.df(ptr %x0, i8 -1, <4 x i64> %x2, <4 x double> %x3, i32 4)595 ret void596}597 598declare void @llvm.x86.avx512.scatterdiv4.di(ptr, i8, <4 x i64>, <4 x i64>, i32)599 600define void@test_int_x86_avx512_scatterdiv4_di(ptr %x0, i8 %x1, <4 x i64> %x2, <4 x i64> %x3) {601; CHECK-LABEL: test_int_x86_avx512_scatterdiv4_di:602; CHECK: ## %bb.0:603; CHECK-NEXT: kmovd %esi, %k1604; CHECK-NEXT: vpscatterqq %ymm1, (%rdi,%ymm0,2) {%k1}605; CHECK-NEXT: kxnorw %k0, %k0, %k1606; CHECK-NEXT: vpscatterqq %ymm1, (%rdi,%ymm0,4) {%k1}607; CHECK-NEXT: vzeroupper608; CHECK-NEXT: retq609 call void @llvm.x86.avx512.scatterdiv4.di(ptr %x0, i8 %x1, <4 x i64> %x2, <4 x i64> %x3, i32 2)610 call void @llvm.x86.avx512.scatterdiv4.di(ptr %x0, i8 -1, <4 x i64> %x2, <4 x i64> %x3, i32 4)611 ret void612}613 614declare void @llvm.x86.avx512.scatterdiv4.sf(ptr, i8, <2 x i64>, <4 x float>, i32)615 616define void@test_int_x86_avx512_scatterdiv4_sf(ptr %x0, i8 %x1, <2 x i64> %x2, <4 x float> %x3) {617; CHECK-LABEL: test_int_x86_avx512_scatterdiv4_sf:618; CHECK: ## %bb.0:619; CHECK-NEXT: kmovd %esi, %k1620; CHECK-NEXT: vscatterqps %xmm1, (%rdi,%xmm0,2) {%k1}621; CHECK-NEXT: kxnorw %k0, %k0, %k1622; CHECK-NEXT: vscatterqps %xmm1, (%rdi,%xmm0,4) {%k1}623; CHECK-NEXT: retq624 call void @llvm.x86.avx512.scatterdiv4.sf(ptr %x0, i8 %x1, <2 x i64> %x2, <4 x float> %x3, i32 2)625 call void @llvm.x86.avx512.scatterdiv4.sf(ptr %x0, i8 -1, <2 x i64> %x2, <4 x float> %x3, i32 4)626 ret void627}628 629declare void @llvm.x86.avx512.scatterdiv4.si(ptr, i8, <2 x i64>, <4 x i32>, i32)630 631define void@test_int_x86_avx512_scatterdiv4_si(ptr %x0, i8 %x1, <2 x i64> %x2, <4 x i32> %x3) {632; CHECK-LABEL: test_int_x86_avx512_scatterdiv4_si:633; CHECK: ## %bb.0:634; CHECK-NEXT: kmovd %esi, %k1635; CHECK-NEXT: kxnorw %k0, %k0, %k2636; CHECK-NEXT: vpscatterqd %xmm1, (%rdi,%xmm0,2) {%k2}637; CHECK-NEXT: vpscatterqd %xmm1, (%rdi,%xmm0,4) {%k1}638; CHECK-NEXT: retq639 call void @llvm.x86.avx512.scatterdiv4.si(ptr %x0, i8 -1, <2 x i64> %x2, <4 x i32> %x3, i32 2)640 call void @llvm.x86.avx512.scatterdiv4.si(ptr %x0, i8 %x1, <2 x i64> %x2, <4 x i32> %x3, i32 4)641 ret void642}643 644declare void @llvm.x86.avx512.scatterdiv8.sf(ptr, i8, <4 x i64>, <4 x float>, i32)645 646define void@test_int_x86_avx512_scatterdiv8_sf(ptr %x0, i8 %x1, <4 x i64> %x2, <4 x float> %x3) {647; CHECK-LABEL: test_int_x86_avx512_scatterdiv8_sf:648; CHECK: ## %bb.0:649; CHECK-NEXT: kmovd %esi, %k1650; CHECK-NEXT: vscatterqps %xmm1, (%rdi,%ymm0,2) {%k1}651; CHECK-NEXT: kxnorw %k0, %k0, %k1652; CHECK-NEXT: vscatterqps %xmm1, (%rdi,%ymm0,4) {%k1}653; CHECK-NEXT: vzeroupper654; CHECK-NEXT: retq655 call void @llvm.x86.avx512.scatterdiv8.sf(ptr %x0, i8 %x1, <4 x i64> %x2, <4 x float> %x3, i32 2)656 call void @llvm.x86.avx512.scatterdiv8.sf(ptr %x0, i8 -1, <4 x i64> %x2, <4 x float> %x3, i32 4)657 ret void658}659 660declare void @llvm.x86.avx512.scatterdiv8.si(ptr, i8, <4 x i64>, <4 x i32>, i32)661 662define void@test_int_x86_avx512_scatterdiv8_si(ptr %x0, i8 %x1, <4 x i64> %x2, <4 x i32> %x3) {663; CHECK-LABEL: test_int_x86_avx512_scatterdiv8_si:664; CHECK: ## %bb.0:665; CHECK-NEXT: kmovd %esi, %k1666; CHECK-NEXT: vpscatterqd %xmm1, (%rdi,%ymm0,2) {%k1}667; CHECK-NEXT: kxnorw %k0, %k0, %k1668; CHECK-NEXT: vpscatterqd %xmm1, (%rdi,%ymm0,4) {%k1}669; CHECK-NEXT: vzeroupper670; CHECK-NEXT: retq671 call void @llvm.x86.avx512.scatterdiv8.si(ptr %x0, i8 %x1, <4 x i64> %x2, <4 x i32> %x3, i32 2)672 call void @llvm.x86.avx512.scatterdiv8.si(ptr %x0, i8 -1, <4 x i64> %x2, <4 x i32> %x3, i32 4)673 ret void674}675 676declare void @llvm.x86.avx512.scattersiv2.df(ptr, i8, <4 x i32>, <2 x double>, i32)677 678define void@test_int_x86_avx512_scattersiv2_df(ptr %x0, i8 %x1, <4 x i32> %x2, <2 x double> %x3) {679; CHECK-LABEL: test_int_x86_avx512_scattersiv2_df:680; CHECK: ## %bb.0:681; CHECK-NEXT: kmovd %esi, %k1682; CHECK-NEXT: kxnorw %k0, %k0, %k2683; CHECK-NEXT: vscatterdpd %xmm1, (%rdi,%xmm0,2) {%k2}684; CHECK-NEXT: vscatterdpd %xmm1, (%rdi,%xmm0,4) {%k1}685; CHECK-NEXT: retq686 call void @llvm.x86.avx512.scattersiv2.df(ptr %x0, i8 -1, <4 x i32> %x2, <2 x double> %x3, i32 2)687 call void @llvm.x86.avx512.scattersiv2.df(ptr %x0, i8 %x1, <4 x i32> %x2, <2 x double> %x3, i32 4)688 ret void689}690 691declare void @llvm.x86.avx512.scattersiv2.di(ptr, i8, <4 x i32>, <2 x i64>, i32)692 693define void@test_int_x86_avx512_scattersiv2_di(ptr %x0, i8 %x1, <4 x i32> %x2, <2 x i64> %x3) {694; CHECK-LABEL: test_int_x86_avx512_scattersiv2_di:695; CHECK: ## %bb.0:696; CHECK-NEXT: kmovd %esi, %k1697; CHECK-NEXT: kxnorw %k0, %k0, %k2698; CHECK-NEXT: vpscatterdq %xmm1, (%rdi,%xmm0,2) {%k2}699; CHECK-NEXT: vpscatterdq %xmm1, (%rdi,%xmm0,4) {%k1}700; CHECK-NEXT: retq701 call void @llvm.x86.avx512.scattersiv2.di(ptr %x0, i8 -1, <4 x i32> %x2, <2 x i64> %x3, i32 2)702 call void @llvm.x86.avx512.scattersiv2.di(ptr %x0, i8 %x1, <4 x i32> %x2, <2 x i64> %x3, i32 4)703 ret void704}705 706declare void @llvm.x86.avx512.scattersiv4.df(ptr, i8, <4 x i32>, <4 x double>, i32)707 708define void@test_int_x86_avx512_scattersiv4_df(ptr %x0, i8 %x1, <4 x i32> %x2, <4 x double> %x3) {709; CHECK-LABEL: test_int_x86_avx512_scattersiv4_df:710; CHECK: ## %bb.0:711; CHECK-NEXT: kmovd %esi, %k1712; CHECK-NEXT: vscatterdpd %ymm1, (%rdi,%xmm0,2) {%k1}713; CHECK-NEXT: kxnorw %k0, %k0, %k1714; CHECK-NEXT: vscatterdpd %ymm1, (%rdi,%xmm0,4) {%k1}715; CHECK-NEXT: vzeroupper716; CHECK-NEXT: retq717 call void @llvm.x86.avx512.scattersiv4.df(ptr %x0, i8 %x1, <4 x i32> %x2, <4 x double> %x3, i32 2)718 call void @llvm.x86.avx512.scattersiv4.df(ptr %x0, i8 -1, <4 x i32> %x2, <4 x double> %x3, i32 4)719 ret void720}721 722declare void @llvm.x86.avx512.scattersiv4.di(ptr, i8, <4 x i32>, <4 x i64>, i32)723 724define void@test_int_x86_avx512_scattersiv4_di(ptr %x0, i8 %x1, <4 x i32> %x2, <4 x i64> %x3) {725; CHECK-LABEL: test_int_x86_avx512_scattersiv4_di:726; CHECK: ## %bb.0:727; CHECK-NEXT: kmovd %esi, %k1728; CHECK-NEXT: kxnorw %k0, %k0, %k2729; CHECK-NEXT: vpscatterdq %ymm1, (%rdi,%xmm0,2) {%k2}730; CHECK-NEXT: vpscatterdq %ymm1, (%rdi,%xmm0,4) {%k1}731; CHECK-NEXT: vzeroupper732; CHECK-NEXT: retq733 call void @llvm.x86.avx512.scattersiv4.di(ptr %x0, i8 -1, <4 x i32> %x2, <4 x i64> %x3, i32 2)734 call void @llvm.x86.avx512.scattersiv4.di(ptr %x0, i8 %x1, <4 x i32> %x2, <4 x i64> %x3, i32 4)735 ret void736}737 738declare void @llvm.x86.avx512.scattersiv4.sf(ptr, i8, <4 x i32>, <4 x float>, i32)739 740define void@test_int_x86_avx512_scattersiv4_sf(ptr %x0, i8 %x1, <4 x i32> %x2, <4 x float> %x3) {741; CHECK-LABEL: test_int_x86_avx512_scattersiv4_sf:742; CHECK: ## %bb.0:743; CHECK-NEXT: kmovd %esi, %k1744; CHECK-NEXT: vscatterdps %xmm1, (%rdi,%xmm0,2) {%k1}745; CHECK-NEXT: kxnorw %k0, %k0, %k1746; CHECK-NEXT: vscatterdps %xmm1, (%rdi,%xmm0,4) {%k1}747; CHECK-NEXT: retq748 call void @llvm.x86.avx512.scattersiv4.sf(ptr %x0, i8 %x1, <4 x i32> %x2, <4 x float> %x3, i32 2)749 call void @llvm.x86.avx512.scattersiv4.sf(ptr %x0, i8 -1, <4 x i32> %x2, <4 x float> %x3, i32 4)750 ret void751}752 753declare void @llvm.x86.avx512.scattersiv4.si(ptr, i8, <4 x i32>, <4 x i32>, i32)754 755define void@test_int_x86_avx512_scattersiv4_si(ptr %x0, i8 %x1, <4 x i32> %x2, <4 x i32> %x3) {756; CHECK-LABEL: test_int_x86_avx512_scattersiv4_si:757; CHECK: ## %bb.0:758; CHECK-NEXT: kmovd %esi, %k1759; CHECK-NEXT: vpscatterdd %xmm1, (%rdi,%xmm0,2) {%k1}760; CHECK-NEXT: kxnorw %k0, %k0, %k1761; CHECK-NEXT: vpscatterdd %xmm1, (%rdi,%xmm0,4) {%k1}762; CHECK-NEXT: retq763 call void @llvm.x86.avx512.scattersiv4.si(ptr %x0, i8 %x1, <4 x i32> %x2, <4 x i32> %x3, i32 2)764 call void @llvm.x86.avx512.scattersiv4.si(ptr %x0, i8 -1, <4 x i32> %x2, <4 x i32> %x3, i32 4)765 ret void766}767 768declare void @llvm.x86.avx512.scattersiv8.sf(ptr, i8, <8 x i32>, <8 x float>, i32)769 770define void@test_int_x86_avx512_scattersiv8_sf(ptr %x0, i8 %x1, <8 x i32> %x2, <8 x float> %x3) {771; CHECK-LABEL: test_int_x86_avx512_scattersiv8_sf:772; CHECK: ## %bb.0:773; CHECK-NEXT: kmovd %esi, %k1774; CHECK-NEXT: vscatterdps %ymm1, (%rdi,%ymm0,2) {%k1}775; CHECK-NEXT: kxnorb %k0, %k0, %k1776; CHECK-NEXT: vscatterdps %ymm1, (%rdi,%ymm0,4) {%k1}777; CHECK-NEXT: vzeroupper778; CHECK-NEXT: retq779 call void @llvm.x86.avx512.scattersiv8.sf(ptr %x0, i8 %x1, <8 x i32> %x2, <8 x float> %x3, i32 2)780 call void @llvm.x86.avx512.scattersiv8.sf(ptr %x0, i8 -1, <8 x i32> %x2, <8 x float> %x3, i32 4)781 ret void782}783 784declare void @llvm.x86.avx512.scattersiv8.si(ptr, i8, <8 x i32>, <8 x i32>, i32)785 786define void@test_int_x86_avx512_scattersiv8_si(ptr %x0, i8 %x1, <8 x i32> %x2, <8 x i32> %x3) {787; CHECK-LABEL: test_int_x86_avx512_scattersiv8_si:788; CHECK: ## %bb.0:789; CHECK-NEXT: kmovd %esi, %k1790; CHECK-NEXT: vpscatterdd %ymm1, (%rdi,%ymm0,2) {%k1}791; CHECK-NEXT: kxnorb %k0, %k0, %k1792; CHECK-NEXT: vpscatterdd %ymm1, (%rdi,%ymm0,4) {%k1}793; CHECK-NEXT: vzeroupper794; CHECK-NEXT: retq795 call void @llvm.x86.avx512.scattersiv8.si(ptr %x0, i8 %x1, <8 x i32> %x2, <8 x i32> %x3, i32 2)796 call void @llvm.x86.avx512.scattersiv8.si(ptr %x0, i8 -1, <8 x i32> %x2, <8 x i32> %x3, i32 4)797 ret void798}799 800define void @scatter_mask_test(ptr %x0, <8 x i32> %x2, <8 x i32> %x3) {801; CHECK-LABEL: scatter_mask_test:802; CHECK: ## %bb.0:803; CHECK-NEXT: kxnorb %k0, %k0, %k1804; CHECK-NEXT: vpscatterdd %ymm1, (%rdi,%ymm0,2) {%k1}805; CHECK-NEXT: kxorb %k0, %k0, %k1806; CHECK-NEXT: vpscatterdd %ymm1, (%rdi,%ymm0,4) {%k1}807; CHECK-NEXT: movb $1, %al808; CHECK-NEXT: kmovd %eax, %k1809; CHECK-NEXT: vpscatterdd %ymm1, (%rdi,%ymm0,2) {%k1}810; CHECK-NEXT: movb $96, %al811; CHECK-NEXT: kmovd %eax, %k1812; CHECK-NEXT: vpscatterdd %ymm1, (%rdi,%ymm0,4) {%k1}813; CHECK-NEXT: vzeroupper814; CHECK-NEXT: retq815 call void @llvm.x86.avx512.scattersiv8.si(ptr %x0, i8 -1, <8 x i32> %x2, <8 x i32> %x3, i32 2)816 call void @llvm.x86.avx512.scattersiv8.si(ptr %x0, i8 0, <8 x i32> %x2, <8 x i32> %x3, i32 4)817 call void @llvm.x86.avx512.scattersiv8.si(ptr %x0, i8 1, <8 x i32> %x2, <8 x i32> %x3, i32 2)818 call void @llvm.x86.avx512.scattersiv8.si(ptr %x0, i8 96, <8 x i32> %x2, <8 x i32> %x3, i32 4)819 ret void820}821 822define <16 x float> @gather_mask_test(<16 x i32> %ind, <16 x float> %src, ptr %base) {823; CHECK-LABEL: gather_mask_test:824; CHECK: ## %bb.0:825; CHECK-NEXT: kxnorw %k0, %k0, %k1826; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2827; CHECK-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm2 {%k1}828; CHECK-NEXT: kxorw %k0, %k0, %k1829; CHECK-NEXT: vmovaps %zmm1, %zmm3830; CHECK-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm3 {%k1}831; CHECK-NEXT: vaddps %zmm3, %zmm2, %zmm2832; CHECK-NEXT: movw $1, %ax833; CHECK-NEXT: kmovd %eax, %k1834; CHECK-NEXT: vmovaps %zmm1, %zmm3835; CHECK-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm3 {%k1}836; CHECK-NEXT: movw $220, %ax837; CHECK-NEXT: kmovd %eax, %k1838; CHECK-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1}839; CHECK-NEXT: vaddps %zmm3, %zmm1, %zmm0840; CHECK-NEXT: vaddps %zmm2, %zmm0, %zmm0841; CHECK-NEXT: retq842 %res = call <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float> %src, ptr %base, <16 x i32>%ind, i16 -1, i32 4)843 %res1 = call <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float> %src, ptr %base, <16 x i32>%ind, i16 0, i32 4)844 %res2 = call <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float> %src, ptr %base, <16 x i32>%ind, i16 1, i32 4)845 %res3 = call <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float> %src, ptr %base, <16 x i32>%ind, i16 220, i32 4)846 847 %res4 = fadd <16 x float> %res, %res1848 %res5 = fadd <16 x float> %res3, %res2849 %res6 = fadd <16 x float> %res5, %res4850 ret <16 x float> %res6851}852