276 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=x86_64-apple-darwin -mattr=avx512f < %s | FileCheck %s --check-prefix=AVX512 --check-prefix=AVX512F3; RUN: llc -mtriple=x86_64-apple-darwin -mattr=avx512f,avx512bw,avx512vl < %s | FileCheck %s --check-prefix=AVX512 --check-prefix=SKX4 5define <16 x i32> @test1(<16 x i32> %trigger, ptr %addr) {6; AVX512-LABEL: test1:7; AVX512: ## %bb.0:8; AVX512-NEXT: vptestnmd %zmm0, %zmm0, %k19; AVX512-NEXT: vmovdqu32 (%rdi), %zmm0 {%k1} {z}10; AVX512-NEXT: retq11 %mask = icmp eq <16 x i32> %trigger, zeroinitializer12 %res = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr %addr, i32 4, <16 x i1>%mask, <16 x i32>undef)13 ret <16 x i32> %res14}15 16define <16 x i32> @test2(<16 x i32> %trigger, ptr %addr) {17; AVX512-LABEL: test2:18; AVX512: ## %bb.0:19; AVX512-NEXT: vptestnmd %zmm0, %zmm0, %k120; AVX512-NEXT: vmovdqu32 (%rdi), %zmm0 {%k1} {z}21; AVX512-NEXT: retq22 %mask = icmp eq <16 x i32> %trigger, zeroinitializer23 %res = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr %addr, i32 4, <16 x i1>%mask, <16 x i32>zeroinitializer)24 ret <16 x i32> %res25}26 27define void @test3(<16 x i32> %trigger, ptr %addr, <16 x i32> %val) {28; AVX512-LABEL: test3:29; AVX512: ## %bb.0:30; AVX512-NEXT: vptestnmd %zmm0, %zmm0, %k131; AVX512-NEXT: vmovdqu32 %zmm1, (%rdi) {%k1}32; AVX512-NEXT: vzeroupper33; AVX512-NEXT: retq34 %mask = icmp eq <16 x i32> %trigger, zeroinitializer35 call void @llvm.masked.store.v16i32.p0(<16 x i32>%val, ptr %addr, i32 4, <16 x i1>%mask)36 ret void37}38 39define <16 x float> @test4(<16 x i32> %trigger, ptr %addr, <16 x float> %dst) {40; AVX512-LABEL: test4:41; AVX512: ## %bb.0:42; AVX512-NEXT: vptestnmd %zmm0, %zmm0, %k143; AVX512-NEXT: vblendmps (%rdi), %zmm1, %zmm0 {%k1}44; AVX512-NEXT: retq45 %mask = icmp eq <16 x i32> %trigger, zeroinitializer46 %res = call <16 x float> @llvm.masked.load.v16f32.p0(ptr %addr, i32 4, <16 x i1>%mask, <16 x float> %dst)47 ret <16 x float> %res48}49 50define void @test13(<16 x i32> %trigger, ptr %addr, <16 x float> %val) {51; AVX512-LABEL: test13:52; AVX512: ## %bb.0:53; AVX512-NEXT: vptestnmd %zmm0, %zmm0, %k154; AVX512-NEXT: vmovups %zmm1, (%rdi) {%k1}55; AVX512-NEXT: vzeroupper56; AVX512-NEXT: retq57 %mask = icmp eq <16 x i32> %trigger, zeroinitializer58 call void @llvm.masked.store.v16f32.p0(<16 x float>%val, ptr %addr, i32 4, <16 x i1>%mask)59 ret void60}61 62define void @one_mask_bit_set5(ptr %addr, <8 x double> %val) {63; AVX512-LABEL: one_mask_bit_set5:64; AVX512: ## %bb.0:65; AVX512-NEXT: vextractf32x4 $3, %zmm0, %xmm066; AVX512-NEXT: vmovlps %xmm0, 48(%rdi)67; AVX512-NEXT: vzeroupper68; AVX512-NEXT: retq69 call void @llvm.masked.store.v8f64.p0(<8 x double> %val, ptr %addr, i32 4, <8 x i1><i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 false>)70 ret void71}72 73define <8 x double> @load_one_mask_bit_set5(ptr %addr, <8 x double> %val) {74;75; AVX512F-LABEL: load_one_mask_bit_set5:76; AVX512F: ## %bb.0:77; AVX512F-NEXT: movb $-128, %al78; AVX512F-NEXT: kmovw %eax, %k179; AVX512F-NEXT: vbroadcastsd 56(%rdi), %zmm0 {%k1}80; AVX512F-NEXT: retq81;82; SKX-LABEL: load_one_mask_bit_set5:83; SKX: ## %bb.0:84; SKX-NEXT: movb $-128, %al85; SKX-NEXT: kmovd %eax, %k186; SKX-NEXT: vbroadcastsd 56(%rdi), %zmm0 {%k1}87; SKX-NEXT: retq88 %res = call <8 x double> @llvm.masked.load.v8f64.p0(ptr %addr, i32 4, <8 x i1><i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true>, <8 x double> %val)89 ret <8 x double> %res90}91 92declare <16 x i32> @llvm.masked.load.v16i32.p0(ptr, i32, <16 x i1>, <16 x i32>)93declare void @llvm.masked.store.v16i32.p0(<16 x i32>, ptr, i32, <16 x i1>)94declare void @llvm.masked.store.v16f32.p0(<16 x float>, ptr, i32, <16 x i1>)95declare <16 x float> @llvm.masked.load.v16f32.p0(ptr, i32, <16 x i1>, <16 x float>)96declare <8 x double> @llvm.masked.load.v8f64.p0(ptr, i32, <8 x i1>, <8 x double>)97declare void @llvm.masked.store.v8f64.p0(<8 x double>, ptr, i32, <8 x i1>)98 99declare <16 x ptr> @llvm.masked.load.v16p0.p0(ptr, i32, <16 x i1>, <16 x ptr>)100 101define <16 x ptr> @test23(<16 x ptr> %trigger, ptr %addr) {102; AVX512-LABEL: test23:103; AVX512: ## %bb.0:104; AVX512-NEXT: vptestnmq %zmm1, %zmm1, %k1105; AVX512-NEXT: vptestnmq %zmm0, %zmm0, %k2106; AVX512-NEXT: vmovdqu64 (%rdi), %zmm0 {%k2} {z}107; AVX512-NEXT: vmovdqu64 64(%rdi), %zmm1 {%k1} {z}108; AVX512-NEXT: retq109 %mask = icmp eq <16 x ptr> %trigger, zeroinitializer110 %res = call <16 x ptr> @llvm.masked.load.v16p0.p0(ptr %addr, i32 4, <16 x i1>%mask, <16 x ptr>zeroinitializer)111 ret <16 x ptr> %res112}113 114%mystruct = type { i16, i16, [1 x ptr] }115 116 117define <16 x ptr> @test24(<16 x i1> %mask, ptr %addr) {118; AVX512F-LABEL: test24:119; AVX512F: ## %bb.0:120; AVX512F-NEXT: vpmovsxbd %xmm0, %zmm0121; AVX512F-NEXT: vpslld $31, %zmm0, %zmm0122; AVX512F-NEXT: vptestmd %zmm0, %zmm0, %k1123; AVX512F-NEXT: vmovdqu64 (%rdi), %zmm0 {%k1} {z}124; AVX512F-NEXT: kshiftrw $8, %k1, %k1125; AVX512F-NEXT: vmovdqu64 64(%rdi), %zmm1 {%k1} {z}126; AVX512F-NEXT: retq127;128; SKX-LABEL: test24:129; SKX: ## %bb.0:130; SKX-NEXT: vpsllw $7, %xmm0, %xmm0131; SKX-NEXT: vpmovb2m %xmm0, %k1132; SKX-NEXT: vmovdqu64 (%rdi), %zmm0 {%k1} {z}133; SKX-NEXT: kshiftrw $8, %k1, %k1134; SKX-NEXT: vmovdqu64 64(%rdi), %zmm1 {%k1} {z}135; SKX-NEXT: retq136 %res = call <16 x ptr> @llvm.masked.load.v16p0.p0(ptr %addr, i32 4, <16 x i1>%mask, <16 x ptr>zeroinitializer)137 ret <16 x ptr> %res138}139 140define void @test_store_16i64(ptr %ptrs, <16 x i1> %mask, <16 x i64> %src0) {141; AVX512F-LABEL: test_store_16i64:142; AVX512F: ## %bb.0:143; AVX512F-NEXT: vpmovsxbd %xmm0, %zmm0144; AVX512F-NEXT: vpslld $31, %zmm0, %zmm0145; AVX512F-NEXT: vptestmd %zmm0, %zmm0, %k1146; AVX512F-NEXT: vmovdqu64 %zmm1, (%rdi) {%k1}147; AVX512F-NEXT: kshiftrw $8, %k1, %k1148; AVX512F-NEXT: vmovdqu64 %zmm2, 64(%rdi) {%k1}149; AVX512F-NEXT: vzeroupper150; AVX512F-NEXT: retq151;152; SKX-LABEL: test_store_16i64:153; SKX: ## %bb.0:154; SKX-NEXT: vpsllw $7, %xmm0, %xmm0155; SKX-NEXT: vpmovb2m %xmm0, %k1156; SKX-NEXT: vmovdqu64 %zmm1, (%rdi) {%k1}157; SKX-NEXT: kshiftrw $8, %k1, %k1158; SKX-NEXT: vmovdqu64 %zmm2, 64(%rdi) {%k1}159; SKX-NEXT: vzeroupper160; SKX-NEXT: retq161 call void @llvm.masked.store.v16i64.p0(<16 x i64> %src0, ptr %ptrs, i32 4, <16 x i1> %mask)162 ret void163}164declare void @llvm.masked.store.v16i64.p0(<16 x i64> %src0, ptr %ptrs, i32, <16 x i1> %mask)165 166define void @test_store_16f64(ptr %ptrs, <16 x i1> %mask, <16 x double> %src0) {167; AVX512F-LABEL: test_store_16f64:168; AVX512F: ## %bb.0:169; AVX512F-NEXT: vpmovsxbd %xmm0, %zmm0170; AVX512F-NEXT: vpslld $31, %zmm0, %zmm0171; AVX512F-NEXT: vptestmd %zmm0, %zmm0, %k1172; AVX512F-NEXT: vmovupd %zmm1, (%rdi) {%k1}173; AVX512F-NEXT: kshiftrw $8, %k1, %k1174; AVX512F-NEXT: vmovupd %zmm2, 64(%rdi) {%k1}175; AVX512F-NEXT: vzeroupper176; AVX512F-NEXT: retq177;178; SKX-LABEL: test_store_16f64:179; SKX: ## %bb.0:180; SKX-NEXT: vpsllw $7, %xmm0, %xmm0181; SKX-NEXT: vpmovb2m %xmm0, %k1182; SKX-NEXT: vmovupd %zmm1, (%rdi) {%k1}183; SKX-NEXT: kshiftrw $8, %k1, %k1184; SKX-NEXT: vmovupd %zmm2, 64(%rdi) {%k1}185; SKX-NEXT: vzeroupper186; SKX-NEXT: retq187 call void @llvm.masked.store.v16f64.p0(<16 x double> %src0, ptr %ptrs, i32 4, <16 x i1> %mask)188 ret void189}190declare void @llvm.masked.store.v16f64.p0(<16 x double> %src0, ptr %ptrs, i32, <16 x i1> %mask)191 192define <16 x i64> @test_load_16i64(ptr %ptrs, <16 x i1> %mask, <16 x i64> %src0) {193; AVX512F-LABEL: test_load_16i64:194; AVX512F: ## %bb.0:195; AVX512F-NEXT: vpmovsxbd %xmm0, %zmm0196; AVX512F-NEXT: vpslld $31, %zmm0, %zmm0197; AVX512F-NEXT: vptestmd %zmm0, %zmm0, %k1198; AVX512F-NEXT: vpblendmq (%rdi), %zmm1, %zmm0 {%k1}199; AVX512F-NEXT: kshiftrw $8, %k1, %k1200; AVX512F-NEXT: vpblendmq 64(%rdi), %zmm2, %zmm1 {%k1}201; AVX512F-NEXT: retq202;203; SKX-LABEL: test_load_16i64:204; SKX: ## %bb.0:205; SKX-NEXT: vpsllw $7, %xmm0, %xmm0206; SKX-NEXT: vpmovb2m %xmm0, %k1207; SKX-NEXT: vpblendmq (%rdi), %zmm1, %zmm0 {%k1}208; SKX-NEXT: kshiftrw $8, %k1, %k1209; SKX-NEXT: vpblendmq 64(%rdi), %zmm2, %zmm1 {%k1}210; SKX-NEXT: retq211 %res = call <16 x i64> @llvm.masked.load.v16i64.p0(ptr %ptrs, i32 4, <16 x i1> %mask, <16 x i64> %src0)212 ret <16 x i64> %res213}214declare <16 x i64> @llvm.masked.load.v16i64.p0(ptr %ptrs, i32, <16 x i1> %mask, <16 x i64> %src0)215 216define <16 x double> @test_load_16f64(ptr %ptrs, <16 x i1> %mask, <16 x double> %src0) {217; AVX512F-LABEL: test_load_16f64:218; AVX512F: ## %bb.0:219; AVX512F-NEXT: vpmovsxbd %xmm0, %zmm0220; AVX512F-NEXT: vpslld $31, %zmm0, %zmm0221; AVX512F-NEXT: vptestmd %zmm0, %zmm0, %k1222; AVX512F-NEXT: vblendmpd (%rdi), %zmm1, %zmm0 {%k1}223; AVX512F-NEXT: kshiftrw $8, %k1, %k1224; AVX512F-NEXT: vblendmpd 64(%rdi), %zmm2, %zmm1 {%k1}225; AVX512F-NEXT: retq226;227; SKX-LABEL: test_load_16f64:228; SKX: ## %bb.0:229; SKX-NEXT: vpsllw $7, %xmm0, %xmm0230; SKX-NEXT: vpmovb2m %xmm0, %k1231; SKX-NEXT: vblendmpd (%rdi), %zmm1, %zmm0 {%k1}232; SKX-NEXT: kshiftrw $8, %k1, %k1233; SKX-NEXT: vblendmpd 64(%rdi), %zmm2, %zmm1 {%k1}234; SKX-NEXT: retq235 %res = call <16 x double> @llvm.masked.load.v16f64.p0(ptr %ptrs, i32 4, <16 x i1> %mask, <16 x double> %src0)236 ret <16 x double> %res237}238declare <16 x double> @llvm.masked.load.v16f64.p0(ptr %ptrs, i32, <16 x i1> %mask, <16 x double> %src0)239 240define <32 x double> @test_load_32f64(ptr %ptrs, <32 x i1> %mask, <32 x double> %src0) {241; AVX512F-LABEL: test_load_32f64:242; AVX512F: ## %bb.0:243; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm5244; AVX512F-NEXT: vpmovsxbd %xmm5, %zmm5245; AVX512F-NEXT: vpslld $31, %zmm5, %zmm5246; AVX512F-NEXT: vptestmd %zmm5, %zmm5, %k1247; AVX512F-NEXT: vpmovsxbd %xmm0, %zmm0248; AVX512F-NEXT: vpslld $31, %zmm0, %zmm0249; AVX512F-NEXT: vptestmd %zmm0, %zmm0, %k2250; AVX512F-NEXT: vblendmpd (%rdi), %zmm1, %zmm0 {%k2}251; AVX512F-NEXT: vblendmpd 128(%rdi), %zmm3, %zmm5 {%k1}252; AVX512F-NEXT: kshiftrw $8, %k2, %k2253; AVX512F-NEXT: vblendmpd 64(%rdi), %zmm2, %zmm1 {%k2}254; AVX512F-NEXT: kshiftrw $8, %k1, %k1255; AVX512F-NEXT: vblendmpd 192(%rdi), %zmm4, %zmm3 {%k1}256; AVX512F-NEXT: vmovapd %zmm5, %zmm2257; AVX512F-NEXT: retq258;259; SKX-LABEL: test_load_32f64:260; SKX: ## %bb.0:261; SKX-NEXT: vpsllw $7, %ymm0, %ymm0262; SKX-NEXT: vpmovb2m %ymm0, %k1263; SKX-NEXT: vblendmpd (%rdi), %zmm1, %zmm0 {%k1}264; SKX-NEXT: kshiftrd $8, %k1, %k2265; SKX-NEXT: vblendmpd 64(%rdi), %zmm2, %zmm1 {%k2}266; SKX-NEXT: kshiftrd $16, %k1, %k2267; SKX-NEXT: vblendmpd 128(%rdi), %zmm3, %zmm2 {%k2}268; SKX-NEXT: kshiftrd $24, %k1, %k1269; SKX-NEXT: vblendmpd 192(%rdi), %zmm4, %zmm3 {%k1}270; SKX-NEXT: retq271 %res = call <32 x double> @llvm.masked.load.v32f64.p0(ptr %ptrs, i32 4, <32 x i1> %mask, <32 x double> %src0)272 ret <32 x double> %res273}274 275declare <32 x double> @llvm.masked.load.v32f64.p0(ptr %ptrs, i32, <32 x i1> %mask, <32 x double> %src0)276