370 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt < %s -passes=instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s3 4;; MASKED LOADS5 6; If the mask isn't constant, do nothing.7 8define <4 x float> @mload(ptr %f, <4 x i32> %mask) {9; CHECK-LABEL: @mload(10; CHECK-NEXT: [[LD:%.*]] = tail call <4 x float> @llvm.x86.avx.maskload.ps(ptr [[F:%.*]], <4 x i32> [[MASK:%.*]])11; CHECK-NEXT: ret <4 x float> [[LD]]12;13 %ld = tail call <4 x float> @llvm.x86.avx.maskload.ps(ptr %f, <4 x i32> %mask)14 ret <4 x float> %ld15}16 17; If the mask comes from a comparison, convert to an LLVM intrinsic. The backend should optimize further.18 19define <4 x float> @mload_v4f32_cmp(ptr %f, <4 x i32> %src) {20; CHECK-LABEL: @mload_v4f32_cmp(21; CHECK-NEXT: [[ICMP:%.*]] = icmp ne <4 x i32> [[SRC:%.*]], zeroinitializer22; CHECK-NEXT: [[LD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr align 1 [[F:%.*]], <4 x i1> [[ICMP]], <4 x float> zeroinitializer)23; CHECK-NEXT: ret <4 x float> [[LD]]24;25 %icmp = icmp ne <4 x i32> %src, zeroinitializer26 %mask = sext <4 x i1> %icmp to <4 x i32>27 %ld = tail call <4 x float> @llvm.x86.avx.maskload.ps(ptr %f, <4 x i32> %mask)28 ret <4 x float> %ld29}30 31; Zero mask returns a zero vector.32 33define <4 x float> @mload_zeros(ptr %f) {34; CHECK-LABEL: @mload_zeros(35; CHECK-NEXT: ret <4 x float> zeroinitializer36;37 %ld = tail call <4 x float> @llvm.x86.avx.maskload.ps(ptr %f, <4 x i32> zeroinitializer)38 ret <4 x float> %ld39}40 41; Only the sign bit matters.42 43define <4 x float> @mload_fake_ones(ptr %f) {44; CHECK-LABEL: @mload_fake_ones(45; CHECK-NEXT: ret <4 x float> zeroinitializer46;47 %ld = tail call <4 x float> @llvm.x86.avx.maskload.ps(ptr %f, <4 x i32> <i32 1, i32 2, i32 3, i32 2147483647>)48 ret <4 x float> %ld49}50 51; All mask bits are set, so this is just a vector load.52 53define <4 x float> @mload_real_ones(ptr %f) {54; CHECK-LABEL: @mload_real_ones(55; CHECK-NEXT: [[UNMASKEDLOAD:%.*]] = load <4 x float>, ptr [[F:%.*]], align 156; CHECK-NEXT: ret <4 x float> [[UNMASKEDLOAD]]57;58 %ld = tail call <4 x float> @llvm.x86.avx.maskload.ps(ptr %f, <4 x i32> <i32 -1, i32 -2, i32 -3, i32 2147483648>)59 ret <4 x float> %ld60}61 62; It's a constant mask, so convert to an LLVM intrinsic. The backend should optimize further.63 64define <4 x float> @mload_one_one(ptr %f) {65; CHECK-LABEL: @mload_one_one(66; CHECK-NEXT: [[LD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr align 1 [[F:%.*]], <4 x i1> <i1 false, i1 false, i1 false, i1 true>, <4 x float> <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float poison>)67; CHECK-NEXT: ret <4 x float> [[LD]]68;69 %ld = tail call <4 x float> @llvm.x86.avx.maskload.ps(ptr %f, <4 x i32> <i32 0, i32 0, i32 0, i32 -1>)70 ret <4 x float> %ld71}72 73; Try doubles.74 75define <2 x double> @mload_one_one_double(ptr %f) {76; CHECK-LABEL: @mload_one_one_double(77; CHECK-NEXT: [[LD:%.*]] = call <2 x double> @llvm.masked.load.v2f64.p0(ptr align 1 [[F:%.*]], <2 x i1> <i1 true, i1 false>, <2 x double> <double poison, double 0.000000e+00>)78; CHECK-NEXT: ret <2 x double> [[LD]]79;80 %ld = tail call <2 x double> @llvm.x86.avx.maskload.pd(ptr %f, <2 x i64> <i64 -1, i64 0>)81 ret <2 x double> %ld82}83 84; Try 256-bit FP ops.85 86define <8 x float> @mload_v8f32(ptr %f) {87; CHECK-LABEL: @mload_v8f32(88; CHECK-NEXT: [[LD:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0(ptr align 1 [[F:%.*]], <8 x i1> <i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false>, <8 x float> <float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float poison, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>)89; CHECK-NEXT: ret <8 x float> [[LD]]90;91 %ld = tail call <8 x float> @llvm.x86.avx.maskload.ps.256(ptr %f, <8 x i32> <i32 0, i32 0, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 0>)92 ret <8 x float> %ld93}94 95define <8 x float> @mload_v8f32_cmp(ptr %f, <8 x float> %src0, <8 x float> %src1) {96; CHECK-LABEL: @mload_v8f32_cmp(97; CHECK-NEXT: [[ICMP0:%.*]] = fcmp one <8 x float> [[SRC0:%.*]], zeroinitializer98; CHECK-NEXT: [[ICMP1:%.*]] = fcmp one <8 x float> [[SRC1:%.*]], zeroinitializer99; CHECK-NEXT: [[MASK1:%.*]] = and <8 x i1> [[ICMP0]], [[ICMP1]]100; CHECK-NEXT: [[LD:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0(ptr align 1 [[F:%.*]], <8 x i1> [[MASK1]], <8 x float> zeroinitializer)101; CHECK-NEXT: ret <8 x float> [[LD]]102;103 %icmp0 = fcmp one <8 x float> %src0, zeroinitializer104 %icmp1 = fcmp one <8 x float> %src1, zeroinitializer105 %ext0 = sext <8 x i1> %icmp0 to <8 x i32>106 %ext1 = sext <8 x i1> %icmp1 to <8 x i32>107 %mask = and <8 x i32> %ext0, %ext1108 %ld = tail call <8 x float> @llvm.x86.avx.maskload.ps.256(ptr %f, <8 x i32> %mask)109 ret <8 x float> %ld110}111 112define <4 x double> @mload_v4f64(ptr %f) {113; CHECK-LABEL: @mload_v4f64(114; CHECK-NEXT: [[LD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 1 [[F:%.*]], <4 x i1> <i1 true, i1 false, i1 false, i1 false>, <4 x double> <double poison, double 0.000000e+00, double 0.000000e+00, double 0.000000e+00>)115; CHECK-NEXT: ret <4 x double> [[LD]]116;117 %ld = tail call <4 x double> @llvm.x86.avx.maskload.pd.256(ptr %f, <4 x i64> <i64 -1, i64 0, i64 0, i64 0>)118 ret <4 x double> %ld119}120 121; Try the AVX2 variants.122 123define <4 x i32> @mload_v4i32(ptr %f) {124; CHECK-LABEL: @mload_v4i32(125; CHECK-NEXT: [[LD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 1 [[F:%.*]], <4 x i1> <i1 false, i1 false, i1 false, i1 true>, <4 x i32> <i32 0, i32 0, i32 0, i32 poison>)126; CHECK-NEXT: ret <4 x i32> [[LD]]127;128 %ld = tail call <4 x i32> @llvm.x86.avx2.maskload.d(ptr %f, <4 x i32> <i32 0, i32 0, i32 0, i32 -1>)129 ret <4 x i32> %ld130}131 132define <2 x i64> @mload_v2i64(ptr %f) {133; CHECK-LABEL: @mload_v2i64(134; CHECK-NEXT: [[LD:%.*]] = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr align 1 [[F:%.*]], <2 x i1> <i1 true, i1 false>, <2 x i64> <i64 poison, i64 0>)135; CHECK-NEXT: ret <2 x i64> [[LD]]136;137 %ld = tail call <2 x i64> @llvm.x86.avx2.maskload.q(ptr %f, <2 x i64> <i64 -1, i64 0>)138 ret <2 x i64> %ld139}140 141define <8 x i32> @mload_v8i32(ptr %f) {142; CHECK-LABEL: @mload_v8i32(143; CHECK-NEXT: [[LD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 1 [[F:%.*]], <8 x i1> <i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false>, <8 x i32> <i32 0, i32 0, i32 0, i32 poison, i32 0, i32 0, i32 0, i32 0>)144; CHECK-NEXT: ret <8 x i32> [[LD]]145;146 %ld = tail call <8 x i32> @llvm.x86.avx2.maskload.d.256(ptr %f, <8 x i32> <i32 0, i32 0, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 0>)147 ret <8 x i32> %ld148}149 150define <4 x i64> @mload_v4i64(ptr %f) {151; CHECK-LABEL: @mload_v4i64(152; CHECK-NEXT: [[LD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 1 [[F:%.*]], <4 x i1> <i1 true, i1 false, i1 false, i1 false>, <4 x i64> <i64 poison, i64 0, i64 0, i64 0>)153; CHECK-NEXT: ret <4 x i64> [[LD]]154;155 %ld = tail call <4 x i64> @llvm.x86.avx2.maskload.q.256(ptr %f, <4 x i64> <i64 -1, i64 0, i64 0, i64 0>)156 ret <4 x i64> %ld157}158 159define <4 x i64> @mload_v4i64_cmp(ptr %f, <4 x i64> %src) {160; CHECK-LABEL: @mload_v4i64_cmp(161; CHECK-NEXT: [[ICMP:%.*]] = icmp sgt <4 x i64> [[SRC:%.*]], splat (i64 -1)162; CHECK-NEXT: [[LD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 1 [[F:%.*]], <4 x i1> [[ICMP]], <4 x i64> zeroinitializer)163; CHECK-NEXT: ret <4 x i64> [[LD]]164;165 %icmp = icmp sge <4 x i64> %src, zeroinitializer166 %mask = sext <4 x i1> %icmp to <4 x i64>167 %ld = tail call <4 x i64> @llvm.x86.avx2.maskload.q.256(ptr %f, <4 x i64> %mask)168 ret <4 x i64> %ld169}170 171;; MASKED STORES172 173; If the mask isn't constant, do nothing.174 175define void @mstore(ptr %f, <4 x i32> %mask, <4 x float> %v) {176; CHECK-LABEL: @mstore(177; CHECK-NEXT: tail call void @llvm.x86.avx.maskstore.ps(ptr [[F:%.*]], <4 x i32> [[MASK:%.*]], <4 x float> [[V:%.*]])178; CHECK-NEXT: ret void179;180 tail call void @llvm.x86.avx.maskstore.ps(ptr %f, <4 x i32> %mask, <4 x float> %v)181 ret void182}183 184; If the mask comes from a comparison, convert to an LLVM intrinsic. The backend should optimize further.185 186define void @mstore_v4f32_cmp(ptr %f, <4 x i32> %src, <4 x float> %v) {187; CHECK-LABEL: @mstore_v4f32_cmp(188; CHECK-NEXT: [[ICMP:%.*]] = icmp eq <4 x i32> [[SRC:%.*]], zeroinitializer189; CHECK-NEXT: call void @llvm.masked.store.v4f32.p0(<4 x float> [[V:%.*]], ptr align 1 [[F:%.*]], <4 x i1> [[ICMP]])190; CHECK-NEXT: ret void191;192 %icmp = icmp eq <4 x i32> %src, zeroinitializer193 %mask = sext <4 x i1> %icmp to <4 x i32>194 tail call void @llvm.x86.avx.maskstore.ps(ptr %f, <4 x i32> %mask, <4 x float> %v)195 ret void196}197 198; Zero mask is a nop.199 200define void @mstore_zeros(ptr %f, <4 x float> %v) {201; CHECK-LABEL: @mstore_zeros(202; CHECK-NEXT: ret void203;204 tail call void @llvm.x86.avx.maskstore.ps(ptr %f, <4 x i32> zeroinitializer, <4 x float> %v)205 ret void206}207 208; Only the sign bit matters.209 210define void @mstore_fake_ones(ptr %f, <4 x float> %v) {211; CHECK-LABEL: @mstore_fake_ones(212; CHECK-NEXT: ret void213;214 tail call void @llvm.x86.avx.maskstore.ps(ptr %f, <4 x i32> <i32 1, i32 2, i32 3, i32 2147483647>, <4 x float> %v)215 ret void216}217 218; All mask bits are set, so this is just a vector store.219 220define void @mstore_real_ones(ptr %f, <4 x float> %v) {221; CHECK-LABEL: @mstore_real_ones(222; CHECK-NEXT: store <4 x float> [[V:%.*]], ptr [[F:%.*]], align 1223; CHECK-NEXT: ret void224;225 tail call void @llvm.x86.avx.maskstore.ps(ptr %f, <4 x i32> <i32 -1, i32 -2, i32 -3, i32 -2147483648>, <4 x float> %v)226 ret void227}228 229; It's a constant mask, so convert to an LLVM intrinsic. The backend should optimize further.230 231define void @mstore_one_one(ptr %f, <4 x float> %v) {232; CHECK-LABEL: @mstore_one_one(233; CHECK-NEXT: call void @llvm.masked.store.v4f32.p0(<4 x float> [[V:%.*]], ptr align 1 [[F:%.*]], <4 x i1> <i1 false, i1 false, i1 false, i1 true>)234; CHECK-NEXT: ret void235;236 tail call void @llvm.x86.avx.maskstore.ps(ptr %f, <4 x i32> <i32 0, i32 0, i32 0, i32 -1>, <4 x float> %v)237 ret void238}239 240; Try doubles.241 242define void @mstore_one_one_double(ptr %f, <2 x double> %v) {243; CHECK-LABEL: @mstore_one_one_double(244; CHECK-NEXT: call void @llvm.masked.store.v2f64.p0(<2 x double> [[V:%.*]], ptr align 1 [[F:%.*]], <2 x i1> <i1 true, i1 false>)245; CHECK-NEXT: ret void246;247 tail call void @llvm.x86.avx.maskstore.pd(ptr %f, <2 x i64> <i64 -1, i64 0>, <2 x double> %v)248 ret void249}250 251; Try 256-bit FP ops.252 253define void @mstore_v8f32(ptr %f, <8 x float> %v) {254; CHECK-LABEL: @mstore_v8f32(255; CHECK-NEXT: call void @llvm.masked.store.v8f32.p0(<8 x float> [[V:%.*]], ptr align 1 [[F:%.*]], <8 x i1> <i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>)256; CHECK-NEXT: ret void257;258 tail call void @llvm.x86.avx.maskstore.ps.256(ptr %f, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 -1, i32 -2, i32 -3, i32 -4>, <8 x float> %v)259 ret void260}261 262define void @mstore_v4f64(ptr %f, <4 x double> %v) {263; CHECK-LABEL: @mstore_v4f64(264; CHECK-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[V:%.*]], ptr align 1 [[F:%.*]], <4 x i1> <i1 true, i1 false, i1 false, i1 false>)265; CHECK-NEXT: ret void266;267 tail call void @llvm.x86.avx.maskstore.pd.256(ptr %f, <4 x i64> <i64 -1, i64 0, i64 1, i64 2>, <4 x double> %v)268 ret void269}270 271define void @mstore_v4f64_cmp(ptr %f, <4 x i32> %src, <4 x double> %v) {272; CHECK-LABEL: @mstore_v4f64_cmp(273; CHECK-NEXT: [[ICMP:%.*]] = icmp sgt <4 x i32> [[SRC:%.*]], splat (i32 -1)274; CHECK-NEXT: call void @llvm.masked.store.v4f64.p0(<4 x double> [[V:%.*]], ptr align 1 [[F:%.*]], <4 x i1> [[ICMP]])275; CHECK-NEXT: ret void276;277 %icmp = icmp sge <4 x i32> %src, zeroinitializer278 %mask = sext <4 x i1> %icmp to <4 x i64>279 tail call void @llvm.x86.avx.maskstore.pd.256(ptr %f, <4 x i64> %mask, <4 x double> %v)280 ret void281}282 283; Try the AVX2 variants.284 285define void @mstore_v4i32(ptr %f, <4 x i32> %v) {286; CHECK-LABEL: @mstore_v4i32(287; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[V:%.*]], ptr align 1 [[F:%.*]], <4 x i1> <i1 false, i1 false, i1 true, i1 true>)288; CHECK-NEXT: ret void289;290 tail call void @llvm.x86.avx2.maskstore.d(ptr %f, <4 x i32> <i32 0, i32 1, i32 -1, i32 -2>, <4 x i32> %v)291 ret void292}293 294define void @mstore_v2i64(ptr %f, <2 x i64> %v) {295; CHECK-LABEL: @mstore_v2i64(296; CHECK-NEXT: call void @llvm.masked.store.v2i64.p0(<2 x i64> [[V:%.*]], ptr align 1 [[F:%.*]], <2 x i1> <i1 true, i1 false>)297; CHECK-NEXT: ret void298;299 tail call void @llvm.x86.avx2.maskstore.q(ptr %f, <2 x i64> <i64 -1, i64 0>, <2 x i64> %v)300 ret void301 302}303 304define void @mstore_v8i32(ptr %f, <8 x i32> %v) {305; CHECK-LABEL: @mstore_v8i32(306; CHECK-NEXT: call void @llvm.masked.store.v8i32.p0(<8 x i32> [[V:%.*]], ptr align 1 [[F:%.*]], <8 x i1> <i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>)307; CHECK-NEXT: ret void308;309 tail call void @llvm.x86.avx2.maskstore.d.256(ptr %f, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 -1, i32 -2, i32 -3, i32 -4>, <8 x i32> %v)310 ret void311}312 313define void @mstore_v4i64(ptr %f, <4 x i64> %v) {314; CHECK-LABEL: @mstore_v4i64(315; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[V:%.*]], ptr align 1 [[F:%.*]], <4 x i1> <i1 true, i1 false, i1 false, i1 false>)316; CHECK-NEXT: ret void317;318 tail call void @llvm.x86.avx2.maskstore.q.256(ptr %f, <4 x i64> <i64 -1, i64 0, i64 1, i64 2>, <4 x i64> %v)319 ret void320}321 322define void @mstore_v4i64_cmp(ptr %f, <4 x i64> %src0, <4 x i64> %src1, <4 x i64> %v) {323; CHECK-LABEL: @mstore_v4i64_cmp(324; CHECK-NEXT: [[ICMP0:%.*]] = icmp eq <4 x i64> [[SRC0:%.*]], zeroinitializer325; CHECK-NEXT: [[ICMP1:%.*]] = icmp ne <4 x i64> [[SRC1:%.*]], zeroinitializer326; CHECK-NEXT: [[MASK1:%.*]] = and <4 x i1> [[ICMP0]], [[ICMP1]]327; CHECK-NEXT: call void @llvm.masked.store.v4i64.p0(<4 x i64> [[V:%.*]], ptr align 1 [[F:%.*]], <4 x i1> [[MASK1]])328; CHECK-NEXT: ret void329;330 %icmp0 = icmp eq <4 x i64> %src0, zeroinitializer331 %icmp1 = icmp ne <4 x i64> %src1, zeroinitializer332 %ext0 = sext <4 x i1> %icmp0 to <4 x i64>333 %ext1 = sext <4 x i1> %icmp1 to <4 x i64>334 %mask = and <4 x i64> %ext0, %ext1335 tail call void @llvm.x86.avx2.maskstore.q.256(ptr %f, <4 x i64> %mask, <4 x i64> %v)336 ret void337}338 339; The original SSE2 masked store variant.340 341define void @mstore_v16i8_sse2_zeros(<16 x i8> %d, ptr %p) {342; CHECK-LABEL: @mstore_v16i8_sse2_zeros(343; CHECK-NEXT: ret void344;345 tail call void @llvm.x86.sse2.maskmov.dqu(<16 x i8> %d, <16 x i8> zeroinitializer, ptr %p)346 ret void347}348 349declare <4 x float> @llvm.x86.avx.maskload.ps(ptr, <4 x i32>)350declare <2 x double> @llvm.x86.avx.maskload.pd(ptr, <2 x i64>)351declare <8 x float> @llvm.x86.avx.maskload.ps.256(ptr, <8 x i32>)352declare <4 x double> @llvm.x86.avx.maskload.pd.256(ptr, <4 x i64>)353 354declare <4 x i32> @llvm.x86.avx2.maskload.d(ptr, <4 x i32>)355declare <2 x i64> @llvm.x86.avx2.maskload.q(ptr, <2 x i64>)356declare <8 x i32> @llvm.x86.avx2.maskload.d.256(ptr, <8 x i32>)357declare <4 x i64> @llvm.x86.avx2.maskload.q.256(ptr, <4 x i64>)358 359declare void @llvm.x86.avx.maskstore.ps(ptr, <4 x i32>, <4 x float>)360declare void @llvm.x86.avx.maskstore.pd(ptr, <2 x i64>, <2 x double>)361declare void @llvm.x86.avx.maskstore.ps.256(ptr, <8 x i32>, <8 x float>)362declare void @llvm.x86.avx.maskstore.pd.256(ptr, <4 x i64>, <4 x double>)363 364declare void @llvm.x86.avx2.maskstore.d(ptr, <4 x i32>, <4 x i32>)365declare void @llvm.x86.avx2.maskstore.q(ptr, <2 x i64>, <2 x i64>)366declare void @llvm.x86.avx2.maskstore.d.256(ptr, <8 x i32>, <8 x i32>)367declare void @llvm.x86.avx2.maskstore.q.256(ptr, <4 x i64>, <4 x i64>)368 369declare void @llvm.x86.sse2.maskmov.dqu(<16 x i8>, <16 x i8>, ptr)370