155 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt < %s -passes=instcombine -S | FileCheck %s3 4; Fold zeroing of inactive lanes into the load's passthrough parameter.5define <4 x float> @masked_load_and_zero_inactive_1(ptr %ptr, <4 x i1> %mask) {6; CHECK-LABEL: @masked_load_and_zero_inactive_1(7; CHECK-NEXT: [[LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr align 4 [[PTR:%.*]], <4 x i1> [[MASK:%.*]], <4 x float> zeroinitializer)8; CHECK-NEXT: ret <4 x float> [[LOAD]]9;10 %load = call <4 x float> @llvm.masked.load.v4f32.p0(ptr %ptr, i32 4, <4 x i1> %mask, <4 x float> undef)11 %masked = select <4 x i1> %mask, <4 x float> %load, <4 x float> zeroinitializer12 ret <4 x float> %masked13}14 15; As above but reuse the load's existing passthrough.16define <4 x i32> @masked_load_and_zero_inactive_2(ptr %ptr, <4 x i1> %mask) {17; CHECK-LABEL: @masked_load_and_zero_inactive_2(18; CHECK-NEXT: [[LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[PTR:%.*]], <4 x i1> [[MASK:%.*]], <4 x i32> zeroinitializer)19; CHECK-NEXT: ret <4 x i32> [[LOAD]]20;21 %load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %ptr, i32 4, <4 x i1> %mask, <4 x i32> zeroinitializer)22 %masked = select <4 x i1> %mask, <4 x i32> %load, <4 x i32> zeroinitializer23 ret <4 x i32> %masked24}25 26; No transform when the load's passthrough cannot be reused or altered.27define <4 x i32> @masked_load_and_zero_inactive_3(ptr %ptr, <4 x i1> %mask, <4 x i32> %passthrough) {28; CHECK-LABEL: @masked_load_and_zero_inactive_3(29; CHECK-NEXT: [[MASKED:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[PTR:%.*]], <4 x i1> [[MASK:%.*]], <4 x i32> zeroinitializer)30; CHECK-NEXT: ret <4 x i32> [[MASKED]]31;32 %load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %ptr, i32 4, <4 x i1> %mask, <4 x i32> %passthrough)33 %masked = select <4 x i1> %mask, <4 x i32> %load, <4 x i32> zeroinitializer34 ret <4 x i32> %masked35}36 37; Remove redundant select when its mask doesn't overlap with the load mask.38define <4 x i32> @masked_load_and_zero_inactive_4(ptr %ptr, <4 x i1> %inv_mask) {39; CHECK-LABEL: @masked_load_and_zero_inactive_4(40; CHECK-NEXT: [[MASK:%.*]] = xor <4 x i1> [[INV_MASK:%.*]], splat (i1 true)41; CHECK-NEXT: [[LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[PTR:%.*]], <4 x i1> [[MASK]], <4 x i32> zeroinitializer)42; CHECK-NEXT: ret <4 x i32> [[LOAD]]43;44 %mask = xor <4 x i1> %inv_mask, <i1 true, i1 true, i1 true, i1 true>45 %load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %ptr, i32 4, <4 x i1> %mask, <4 x i32> undef)46 %masked = select <4 x i1> %inv_mask, <4 x i32> zeroinitializer, <4 x i32> %load47 ret <4 x i32> %masked48}49 50; As above but reuse the load's existing passthrough.51define <4 x i32> @masked_load_and_zero_inactive_5(ptr %ptr, <4 x i1> %inv_mask) {52; CHECK-LABEL: @masked_load_and_zero_inactive_5(53; CHECK-NEXT: [[MASK:%.*]] = xor <4 x i1> [[INV_MASK:%.*]], splat (i1 true)54; CHECK-NEXT: [[LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[PTR:%.*]], <4 x i1> [[MASK]], <4 x i32> zeroinitializer)55; CHECK-NEXT: ret <4 x i32> [[LOAD]]56;57 %mask = xor <4 x i1> %inv_mask, <i1 true, i1 true, i1 true, i1 true>58 %load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %ptr, i32 4, <4 x i1> %mask, <4 x i32> zeroinitializer)59 %masked = select <4 x i1> %inv_mask, <4 x i32> zeroinitializer, <4 x i32> %load60 ret <4 x i32> %masked61}62 63; No transform when the load's passthrough cannot be reused or altered.64define <4 x i32> @masked_load_and_zero_inactive_6(ptr %ptr, <4 x i1> %inv_mask, <4 x i32> %passthrough) {65; CHECK-LABEL: @masked_load_and_zero_inactive_6(66; CHECK-NEXT: [[MASK:%.*]] = xor <4 x i1> [[INV_MASK:%.*]], splat (i1 true)67; CHECK-NEXT: [[LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[PTR:%.*]], <4 x i1> [[MASK]], <4 x i32> [[PASSTHROUGH:%.*]])68; CHECK-NEXT: [[MASKED:%.*]] = select <4 x i1> [[INV_MASK]], <4 x i32> zeroinitializer, <4 x i32> [[LOAD]]69; CHECK-NEXT: ret <4 x i32> [[MASKED]]70;71 %mask = xor <4 x i1> %inv_mask, <i1 true, i1 true, i1 true, i1 true>72 %load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %ptr, i32 4, <4 x i1> %mask, <4 x i32> %passthrough)73 %masked = select <4 x i1> %inv_mask, <4 x i32> zeroinitializer, <4 x i32> %load74 ret <4 x i32> %masked75}76 77; No transform when select and load masks have no relation.78define <4 x i32> @masked_load_and_zero_inactive_7(ptr %ptr, <4 x i1> %mask1, <4 x i1> %mask2) {79; CHECK-LABEL: @masked_load_and_zero_inactive_7(80; CHECK-NEXT: [[LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[PTR:%.*]], <4 x i1> [[MASK1:%.*]], <4 x i32> zeroinitializer)81; CHECK-NEXT: [[MASKED:%.*]] = select <4 x i1> [[MASK2:%.*]], <4 x i32> zeroinitializer, <4 x i32> [[LOAD]]82; CHECK-NEXT: ret <4 x i32> [[MASKED]]83;84 %load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %ptr, i32 4, <4 x i1> %mask1, <4 x i32> zeroinitializer)85 %masked = select <4 x i1> %mask2, <4 x i32> zeroinitializer, <4 x i32> %load86 ret <4 x i32> %masked87}88 89; A more complex case where we can prove the select mask is a subset of the90; load's inactive lanes and thus the load's passthrough takes effect.91define <4 x float> @masked_load_and_zero_inactive_8(ptr %ptr, <4 x i1> %inv_mask, <4 x i1> %cond) {92; CHECK-LABEL: @masked_load_and_zero_inactive_8(93; CHECK-NEXT: [[MASK:%.*]] = xor <4 x i1> [[INV_MASK:%.*]], splat (i1 true)94; CHECK-NEXT: [[PG:%.*]] = and <4 x i1> [[COND:%.*]], [[MASK]]95; CHECK-NEXT: [[LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr align 4 [[PTR:%.*]], <4 x i1> [[PG]], <4 x float> zeroinitializer)96; CHECK-NEXT: ret <4 x float> [[LOAD]]97;98 %mask = xor <4 x i1> %inv_mask, <i1 true, i1 true, i1 true, i1 true>99 %pg = and <4 x i1> %mask, %cond100 %load = call <4 x float> @llvm.masked.load.v4f32.p0(ptr %ptr, i32 4, <4 x i1> %pg, <4 x float> undef)101 %masked = select <4 x i1> %inv_mask, <4 x float> zeroinitializer, <4 x float> %load102 ret <4 x float> %masked103}104 105define <8 x float> @masked_load_and_scalar_select_cond(ptr %ptr, <8 x i1> %mask, i1 %cond) {106; CHECK-LABEL: @masked_load_and_scalar_select_cond(107; CHECK-NEXT: entry:108; CHECK-NEXT: [[TMP0:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0(ptr align 32 [[PTR:%.*]], <8 x i1> [[MASK:%.*]], <8 x float> undef)109; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[COND:%.*]], <8 x float> zeroinitializer, <8 x float> [[TMP0]]110; CHECK-NEXT: ret <8 x float> [[TMP1]]111;112entry:113 %0 = call <8 x float> @llvm.masked.load.v8f32.p0(ptr %ptr, i32 32, <8 x i1> %mask, <8 x float> undef)114 %1 = select i1 %cond, <8 x float> zeroinitializer, <8 x float> %0115 ret <8 x float> %1116}117 118define <vscale x 4 x float> @fold_sel_into_masked_load_scalable(ptr %loc, <vscale x 4 x i1> %mask, <vscale x 4 x float> %passthrough) {119; CHECK-LABEL: @fold_sel_into_masked_load_scalable(120; CHECK-NEXT: [[SEL:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 1 [[LOC:%.*]], <vscale x 4 x i1> [[MASK:%.*]], <vscale x 4 x float> [[PASSTHROUGH:%.*]])121; CHECK-NEXT: ret <vscale x 4 x float> [[SEL]]122;123 %load = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr %loc, i32 1, <vscale x 4 x i1> %mask, <vscale x 4 x float> zeroinitializer)124 %sel = select <vscale x 4 x i1> %mask, <vscale x 4 x float> %load, <vscale x 4 x float> %passthrough125 ret <vscale x 4 x float> %sel126}127 128define <vscale x 4 x float> @neg_fold_sel_into_masked_load_mask_mismatch(ptr %loc, <vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask2, <vscale x 4 x float> %passthrough) {129; CHECK-LABEL: @neg_fold_sel_into_masked_load_mask_mismatch(130; CHECK-NEXT: [[LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 1 [[LOC:%.*]], <vscale x 4 x i1> [[MASK:%.*]], <vscale x 4 x float> [[PASSTHROUGH:%.*]])131; CHECK-NEXT: [[SEL:%.*]] = select <vscale x 4 x i1> [[MASK2:%.*]], <vscale x 4 x float> [[LOAD]], <vscale x 4 x float> [[PASSTHROUGH]]132; CHECK-NEXT: ret <vscale x 4 x float> [[SEL]]133;134 %load = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr %loc, i32 1, <vscale x 4 x i1> %mask, <vscale x 4 x float> %passthrough)135 %sel = select <vscale x 4 x i1> %mask2, <vscale x 4 x float> %load, <vscale x 4 x float> %passthrough136 ret <vscale x 4 x float> %sel137}138 139define <vscale x 4 x float> @fold_sel_into_masked_load_scalable_one_use_check(ptr %loc1, <vscale x 4 x i1> %mask, <vscale x 4 x float> %passthrough, ptr %loc2) {140; CHECK-LABEL: @fold_sel_into_masked_load_scalable_one_use_check(141; CHECK-NEXT: [[LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr align 1 [[LOC1:%.*]], <vscale x 4 x i1> [[MASK:%.*]], <vscale x 4 x float> zeroinitializer)142; CHECK-NEXT: [[SEL:%.*]] = select <vscale x 4 x i1> [[MASK]], <vscale x 4 x float> [[LOAD]], <vscale x 4 x float> [[PASSTHROUGH:%.*]]143; CHECK-NEXT: call void @llvm.masked.store.nxv4f32.p0(<vscale x 4 x float> [[LOAD]], ptr align 1 [[LOC2:%.*]], <vscale x 4 x i1> [[MASK]])144; CHECK-NEXT: ret <vscale x 4 x float> [[SEL]]145;146 %load = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr %loc1, i32 1, <vscale x 4 x i1> %mask, <vscale x 4 x float> zeroinitializer)147 %sel = select <vscale x 4 x i1> %mask, <vscale x 4 x float> %load, <vscale x 4 x float> %passthrough148 call void @llvm.masked.store.nxv4f32.p0(<vscale x 4 x float> %load, ptr %loc2, i32 1, <vscale x 4 x i1> %mask)149 ret <vscale x 4 x float> %sel150}151 152declare <8 x float> @llvm.masked.load.v8f32.p0(ptr, i32 immarg, <8 x i1>, <8 x float>)153declare <4 x i32> @llvm.masked.load.v4i32.p0(ptr, i32 immarg, <4 x i1>, <4 x i32>)154declare <4 x float> @llvm.masked.load.v4f32.p0(ptr, i32 immarg, <4 x i1>, <4 x float>)155