839 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 62; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -tail-predication=enabled -mattr=+mve,+lob %s -S -o - | FileCheck %s3 4define dso_local arm_aapcs_vfpcc void @mul_v16i8(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) {5; CHECK-LABEL: define dso_local arm_aapcs_vfpcc void @mul_v16i8(6; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {7; CHECK-NEXT: [[ENTRY:.*:]]8; CHECK-NEXT: [[CMP8:%.*]] = icmp eq i32 [[N]], 09; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[N]], 1510; CHECK-NEXT: [[TMP9:%.*]] = lshr i32 [[TMP8]], 411; CHECK-NEXT: [[TMP10:%.*]] = shl nuw i32 [[TMP9]], 412; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[TMP10]], -1613; CHECK-NEXT: [[TMP12:%.*]] = lshr i32 [[TMP11]], 414; CHECK-NEXT: [[TMP13:%.*]] = add nuw nsw i32 [[TMP12]], 115; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_PH:.*]]16; CHECK: [[VECTOR_PH]]:17; CHECK-NEXT: [[START:%.*]] = call i32 @llvm.start.loop.iterations.i32(i32 [[TMP13]])18; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]19; CHECK: [[VECTOR_BODY]]:20; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]21; CHECK-NEXT: [[TMP14:%.*]] = phi i32 [ [[START]], %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]22; CHECK-NEXT: [[TMP0:%.*]] = phi i32 [ [[N]], %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]23; CHECK-NEXT: [[TMP:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[INDEX]]24; CHECK-NEXT: [[TMP1:%.*]] = call <16 x i1> @llvm.arm.mve.vctp8(i32 [[TMP0]])25; CHECK-NEXT: [[TMP2]] = sub i32 [[TMP0]], 1626; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = tail call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 4 [[TMP]], <16 x i1> [[TMP1]], <16 x i8> undef)27; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[INDEX]]28; CHECK-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = tail call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 4 [[TMP3]], <16 x i1> [[TMP1]], <16 x i8> undef)29; CHECK-NEXT: [[MUL:%.*]] = mul nsw <16 x i8> [[WIDE_MASKED_LOAD2]], [[WIDE_MASKED_LOAD]]30; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[C]], i32 [[INDEX]]31; CHECK-NEXT: tail call void @llvm.masked.store.v16i8.p0(<16 x i8> [[MUL]], ptr align 4 [[TMP6]], <16 x i1> [[TMP1]])32; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 1633; CHECK-NEXT: [[TMP15]] = call i32 @llvm.loop.decrement.reg.i32(i32 [[TMP14]], i32 1)34; CHECK-NEXT: [[TMP16:%.*]] = icmp ne i32 [[TMP15]], 035; CHECK-NEXT: br i1 [[TMP16]], label %[[VECTOR_BODY]], label %[[FOR_COND_CLEANUP]]36; CHECK: [[FOR_COND_CLEANUP]]:37; CHECK-NEXT: ret void38;39entry:40 %cmp8 = icmp eq i32 %N, 041 %tmp8 = add i32 %N, 1542 %tmp9 = lshr i32 %tmp8, 443 %tmp10 = shl nuw i32 %tmp9, 444 %tmp11 = add i32 %tmp10, -1645 %tmp12 = lshr i32 %tmp11, 446 %tmp13 = add nuw nsw i32 %tmp12, 147 br i1 %cmp8, label %for.cond.cleanup, label %vector.ph48 49vector.ph: ; preds = %entry50 %start = call i32 @llvm.start.loop.iterations.i32(i32 %tmp13)51 br label %vector.body52 53vector.body: ; preds = %vector.body, %vector.ph54 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]55 %tmp14 = phi i32 [ %start, %vector.ph ], [ %tmp15, %vector.body ]56 %tmp = getelementptr inbounds i8, ptr %a, i32 %index57 %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %N)58 %wide.masked.load = tail call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %tmp, i32 4, <16 x i1> %active.lane.mask, <16 x i8> undef)59 %tmp3 = getelementptr inbounds i8, ptr %b, i32 %index60 %wide.masked.load2 = tail call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %tmp3, i32 4, <16 x i1> %active.lane.mask, <16 x i8> undef)61 %mul = mul nsw <16 x i8> %wide.masked.load2, %wide.masked.load62 %tmp6 = getelementptr inbounds i8, ptr %c, i32 %index63 tail call void @llvm.masked.store.v16i8.p0(<16 x i8> %mul, ptr %tmp6, i32 4, <16 x i1> %active.lane.mask)64 %index.next = add i32 %index, 1665 %tmp15 = call i32 @llvm.loop.decrement.reg.i32(i32 %tmp14, i32 1)66 %tmp16 = icmp ne i32 %tmp15, 067 br i1 %tmp16, label %vector.body, label %for.cond.cleanup68 69for.cond.cleanup: ; preds = %vector.body, %entry70 ret void71}72 73define dso_local arm_aapcs_vfpcc void @mul_v8i16(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) {74; CHECK-LABEL: define dso_local arm_aapcs_vfpcc void @mul_v8i16(75; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {76; CHECK-NEXT: [[ENTRY:.*:]]77; CHECK-NEXT: [[CMP8:%.*]] = icmp eq i32 [[N]], 078; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[N]], 779; CHECK-NEXT: [[TMP9:%.*]] = lshr i32 [[TMP8]], 380; CHECK-NEXT: [[TMP10:%.*]] = shl nuw i32 [[TMP9]], 381; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[TMP10]], -882; CHECK-NEXT: [[TMP12:%.*]] = lshr i32 [[TMP11]], 383; CHECK-NEXT: [[TMP13:%.*]] = add nuw nsw i32 [[TMP12]], 184; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_PH:.*]]85; CHECK: [[VECTOR_PH]]:86; CHECK-NEXT: [[START:%.*]] = call i32 @llvm.start.loop.iterations.i32(i32 [[TMP13]])87; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]88; CHECK: [[VECTOR_BODY]]:89; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]90; CHECK-NEXT: [[TMP14:%.*]] = phi i32 [ [[START]], %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]91; CHECK-NEXT: [[TMP0:%.*]] = phi i32 [ [[N]], %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]92; CHECK-NEXT: [[TMP:%.*]] = getelementptr inbounds i16, ptr [[A]], i32 [[INDEX]]93; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.vctp16(i32 [[TMP0]])94; CHECK-NEXT: [[TMP2]] = sub i32 [[TMP0]], 895; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = tail call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 4 [[TMP]], <8 x i1> [[TMP1]], <8 x i16> undef)96; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[INDEX]]97; CHECK-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = tail call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 4 [[TMP3]], <8 x i1> [[TMP1]], <8 x i16> undef)98; CHECK-NEXT: [[MUL:%.*]] = mul nsw <8 x i16> [[WIDE_MASKED_LOAD2]], [[WIDE_MASKED_LOAD]]99; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[C]], i32 [[INDEX]]100; CHECK-NEXT: tail call void @llvm.masked.store.v8i16.p0(<8 x i16> [[MUL]], ptr align 4 [[TMP6]], <8 x i1> [[TMP1]])101; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8102; CHECK-NEXT: [[TMP15]] = call i32 @llvm.loop.decrement.reg.i32(i32 [[TMP14]], i32 1)103; CHECK-NEXT: [[TMP16:%.*]] = icmp ne i32 [[TMP15]], 0104; CHECK-NEXT: br i1 [[TMP16]], label %[[VECTOR_BODY]], label %[[FOR_COND_CLEANUP]]105; CHECK: [[FOR_COND_CLEANUP]]:106; CHECK-NEXT: ret void107;108entry:109 %cmp8 = icmp eq i32 %N, 0110 %tmp8 = add i32 %N, 7111 %tmp9 = lshr i32 %tmp8, 3112 %tmp10 = shl nuw i32 %tmp9, 3113 %tmp11 = add i32 %tmp10, -8114 %tmp12 = lshr i32 %tmp11, 3115 %tmp13 = add nuw nsw i32 %tmp12, 1116 br i1 %cmp8, label %for.cond.cleanup, label %vector.ph117 118vector.ph: ; preds = %entry119 %start = call i32 @llvm.start.loop.iterations.i32(i32 %tmp13)120 br label %vector.body121 122vector.body: ; preds = %vector.body, %vector.ph123 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]124 %tmp14 = phi i32 [ %start, %vector.ph ], [ %tmp15, %vector.body ]125 %tmp = getelementptr inbounds i16, ptr %a, i32 %index126 %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %N)127 %wide.masked.load = tail call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %tmp, i32 4, <8 x i1> %active.lane.mask, <8 x i16> undef)128 %tmp3 = getelementptr inbounds i16, ptr %b, i32 %index129 %wide.masked.load2 = tail call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %tmp3, i32 4, <8 x i1> %active.lane.mask, <8 x i16> undef)130 %mul = mul nsw <8 x i16> %wide.masked.load2, %wide.masked.load131 %tmp6 = getelementptr inbounds i16, ptr %c, i32 %index132 tail call void @llvm.masked.store.v8i16.p0(<8 x i16> %mul, ptr %tmp6, i32 4, <8 x i1> %active.lane.mask)133 %index.next = add i32 %index, 8134 %tmp15 = call i32 @llvm.loop.decrement.reg.i32(i32 %tmp14, i32 1)135 %tmp16 = icmp ne i32 %tmp15, 0136 br i1 %tmp16, label %vector.body, label %for.cond.cleanup137 138for.cond.cleanup: ; preds = %vector.body, %entry139 ret void140}141 142define dso_local arm_aapcs_vfpcc void @mul_v4i32(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) {143; CHECK-LABEL: define dso_local arm_aapcs_vfpcc void @mul_v4i32(144; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {145; CHECK-NEXT: [[ENTRY:.*:]]146; CHECK-NEXT: [[CMP8:%.*]] = icmp eq i32 [[N]], 0147; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[N]], 3148; CHECK-NEXT: [[TMP9:%.*]] = lshr i32 [[TMP8]], 2149; CHECK-NEXT: [[TMP10:%.*]] = shl nuw i32 [[TMP9]], 2150; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[TMP10]], -4151; CHECK-NEXT: [[TMP12:%.*]] = lshr i32 [[TMP11]], 2152; CHECK-NEXT: [[TMP13:%.*]] = add nuw nsw i32 [[TMP12]], 1153; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_PH:.*]]154; CHECK: [[VECTOR_PH]]:155; CHECK-NEXT: [[START:%.*]] = call i32 @llvm.start.loop.iterations.i32(i32 [[TMP13]])156; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]157; CHECK: [[VECTOR_BODY]]:158; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]159; CHECK-NEXT: [[TMP14:%.*]] = phi i32 [ [[START]], %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]160; CHECK-NEXT: [[TMP0:%.*]] = phi i32 [ [[N]], %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]161; CHECK-NEXT: [[TMP:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[INDEX]]162; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.vctp32(i32 [[TMP0]])163; CHECK-NEXT: [[TMP2]] = sub i32 [[TMP0]], 4164; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP]], <4 x i1> [[TMP1]], <4 x i32> undef)165; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[INDEX]]166; CHECK-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP3]], <4 x i1> [[TMP1]], <4 x i32> undef)167; CHECK-NEXT: [[MUL:%.*]] = mul nsw <4 x i32> [[WIDE_MASKED_LOAD2]], [[WIDE_MASKED_LOAD]]168; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[C]], i32 [[INDEX]]169; CHECK-NEXT: tail call void @llvm.masked.store.v4i32.p0(<4 x i32> [[MUL]], ptr align 4 [[TMP6]], <4 x i1> [[TMP1]])170; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4171; CHECK-NEXT: [[TMP15]] = call i32 @llvm.loop.decrement.reg.i32(i32 [[TMP14]], i32 1)172; CHECK-NEXT: [[TMP16:%.*]] = icmp ne i32 [[TMP15]], 0173; CHECK-NEXT: br i1 [[TMP16]], label %[[VECTOR_BODY]], label %[[FOR_COND_CLEANUP]]174; CHECK: [[FOR_COND_CLEANUP]]:175; CHECK-NEXT: ret void176;177entry:178 %cmp8 = icmp eq i32 %N, 0179 %tmp8 = add i32 %N, 3180 %tmp9 = lshr i32 %tmp8, 2181 %tmp10 = shl nuw i32 %tmp9, 2182 %tmp11 = add i32 %tmp10, -4183 %tmp12 = lshr i32 %tmp11, 2184 %tmp13 = add nuw nsw i32 %tmp12, 1185 br i1 %cmp8, label %for.cond.cleanup, label %vector.ph186 187vector.ph: ; preds = %entry188 %start = call i32 @llvm.start.loop.iterations.i32(i32 %tmp13)189 br label %vector.body190 191vector.body: ; preds = %vector.body, %vector.ph192 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]193 %tmp14 = phi i32 [ %start, %vector.ph ], [ %tmp15, %vector.body ]194 %tmp = getelementptr inbounds i32, ptr %a, i32 %index195 %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %N)196 %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)197 %tmp3 = getelementptr inbounds i32, ptr %b, i32 %index198 %wide.masked.load2 = tail call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp3, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)199 %mul = mul nsw <4 x i32> %wide.masked.load2, %wide.masked.load200 %tmp6 = getelementptr inbounds i32, ptr %c, i32 %index201 tail call void @llvm.masked.store.v4i32.p0(<4 x i32> %mul, ptr %tmp6, i32 4, <4 x i1> %active.lane.mask)202 %index.next = add i32 %index, 4203 %tmp15 = call i32 @llvm.loop.decrement.reg.i32(i32 %tmp14, i32 1)204 %tmp16 = icmp ne i32 %tmp15, 0205 br i1 %tmp16, label %vector.body, label %for.cond.cleanup206 207for.cond.cleanup: ; preds = %vector.body, %entry208 ret void209}210 211define dso_local arm_aapcs_vfpcc void @split_vector(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) {212; CHECK-LABEL: define dso_local arm_aapcs_vfpcc void @split_vector(213; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {214; CHECK-NEXT: [[ENTRY:.*:]]215; CHECK-NEXT: [[CMP8:%.*]] = icmp eq i32 [[N]], 0216; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[N]], 3217; CHECK-NEXT: [[TMP9:%.*]] = lshr i32 [[TMP8]], 2218; CHECK-NEXT: [[TMP10:%.*]] = shl nuw i32 [[TMP9]], 2219; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[TMP10]], -4220; CHECK-NEXT: [[TMP12:%.*]] = lshr i32 [[TMP11]], 2221; CHECK-NEXT: [[TMP13:%.*]] = add nuw nsw i32 [[TMP12]], 1222; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_PH:.*]]223; CHECK: [[VECTOR_PH]]:224; CHECK-NEXT: [[START:%.*]] = call i32 @llvm.start.loop.iterations.i32(i32 [[TMP13]])225; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]226; CHECK: [[VECTOR_BODY]]:227; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]228; CHECK-NEXT: [[TMP14:%.*]] = phi i32 [ [[START]], %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]229; CHECK-NEXT: [[TMP0:%.*]] = phi i32 [ [[N]], %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]230; CHECK-NEXT: [[TMP:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[INDEX]]231; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.vctp32(i32 [[TMP0]])232; CHECK-NEXT: [[TMP2]] = sub i32 [[TMP0]], 4233; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP]], <4 x i1> [[TMP1]], <4 x i32> undef)234; CHECK-NEXT: [[EXTRACT_1_LOW:%.*]] = shufflevector <4 x i32> [[WIDE_MASKED_LOAD]], <4 x i32> undef, <2 x i32> <i32 0, i32 2>235; CHECK-NEXT: [[EXTRACT_1_HIGH:%.*]] = shufflevector <4 x i32> [[WIDE_MASKED_LOAD]], <4 x i32> undef, <2 x i32> <i32 1, i32 3>236; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[INDEX]]237; CHECK-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP3]], <4 x i1> [[TMP1]], <4 x i32> undef)238; CHECK-NEXT: [[EXTRACT_2_LOW:%.*]] = shufflevector <4 x i32> [[WIDE_MASKED_LOAD2]], <4 x i32> undef, <2 x i32> <i32 0, i32 2>239; CHECK-NEXT: [[EXTRACT_2_HIGH:%.*]] = shufflevector <4 x i32> [[WIDE_MASKED_LOAD2]], <4 x i32> undef, <2 x i32> <i32 1, i32 3>240; CHECK-NEXT: [[MUL:%.*]] = mul nsw <2 x i32> [[EXTRACT_1_LOW]], [[EXTRACT_2_LOW]]241; CHECK-NEXT: [[SUB:%.*]] = sub nsw <2 x i32> [[EXTRACT_1_HIGH]], [[EXTRACT_2_HIGH]]242; CHECK-NEXT: [[COMBINE:%.*]] = shufflevector <2 x i32> [[MUL]], <2 x i32> [[SUB]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>243; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[C]], i32 [[INDEX]]244; CHECK-NEXT: tail call void @llvm.masked.store.v4i32.p0(<4 x i32> [[COMBINE]], ptr align 4 [[TMP6]], <4 x i1> [[TMP1]])245; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4246; CHECK-NEXT: [[TMP15]] = call i32 @llvm.loop.decrement.reg.i32(i32 [[TMP14]], i32 1)247; CHECK-NEXT: [[TMP16:%.*]] = icmp ne i32 [[TMP15]], 0248; CHECK-NEXT: br i1 [[TMP16]], label %[[VECTOR_BODY]], label %[[FOR_COND_CLEANUP]]249; CHECK: [[FOR_COND_CLEANUP]]:250; CHECK-NEXT: ret void251;252entry:253 %cmp8 = icmp eq i32 %N, 0254 %tmp8 = add i32 %N, 3255 %tmp9 = lshr i32 %tmp8, 2256 %tmp10 = shl nuw i32 %tmp9, 2257 %tmp11 = add i32 %tmp10, -4258 %tmp12 = lshr i32 %tmp11, 2259 %tmp13 = add nuw nsw i32 %tmp12, 1260 br i1 %cmp8, label %for.cond.cleanup, label %vector.ph261 262vector.ph: ; preds = %entry263 %start = call i32 @llvm.start.loop.iterations.i32(i32 %tmp13)264 br label %vector.body265 266vector.body: ; preds = %vector.body, %vector.ph267 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]268 %tmp14 = phi i32 [ %start, %vector.ph ], [ %tmp15, %vector.body ]269 %tmp = getelementptr inbounds i32, ptr %a, i32 %index270 %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %N)271 %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)272 %extract.1.low = shufflevector <4 x i32> %wide.masked.load, <4 x i32> undef, < 2 x i32> < i32 0, i32 2>273 %extract.1.high = shufflevector <4 x i32> %wide.masked.load, <4 x i32> undef, < 2 x i32> < i32 1, i32 3>274 %tmp3 = getelementptr inbounds i32, ptr %b, i32 %index275 %wide.masked.load2 = tail call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp3, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)276 %extract.2.low = shufflevector <4 x i32> %wide.masked.load2, <4 x i32> undef, < 2 x i32> < i32 0, i32 2>277 %extract.2.high = shufflevector <4 x i32> %wide.masked.load2, <4 x i32> undef, < 2 x i32> < i32 1, i32 3>278 %mul = mul nsw <2 x i32> %extract.1.low, %extract.2.low279 %sub = sub nsw <2 x i32> %extract.1.high, %extract.2.high280 %combine = shufflevector <2 x i32> %mul, <2 x i32> %sub, <4 x i32> <i32 0, i32 1, i32 2, i32 3>281 %tmp6 = getelementptr inbounds i32, ptr %c, i32 %index282 tail call void @llvm.masked.store.v4i32.p0(<4 x i32> %combine, ptr %tmp6, i32 4, <4 x i1> %active.lane.mask)283 %index.next = add i32 %index, 4284 %tmp15 = call i32 @llvm.loop.decrement.reg.i32(i32 %tmp14, i32 1)285 %tmp16 = icmp ne i32 %tmp15, 0286 br i1 %tmp16, label %vector.body, label %for.cond.cleanup287 288for.cond.cleanup: ; preds = %vector.body, %entry289 ret void290}291 292; One of the loads now uses ult predicate.293define dso_local arm_aapcs_vfpcc void @mismatch_load_pred(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) {294; CHECK-LABEL: define dso_local arm_aapcs_vfpcc void @mismatch_load_pred(295; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {296; CHECK-NEXT: [[ENTRY:.*:]]297; CHECK-NEXT: [[CMP8:%.*]] = icmp eq i32 [[N]], 0298; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[N]], 3299; CHECK-NEXT: [[TMP9:%.*]] = lshr i32 [[TMP8]], 2300; CHECK-NEXT: [[TMP10:%.*]] = shl nuw i32 [[TMP9]], 2301; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[TMP10]], -4302; CHECK-NEXT: [[TMP12:%.*]] = lshr i32 [[TMP11]], 2303; CHECK-NEXT: [[TMP13:%.*]] = add nuw nsw i32 [[TMP12]], 1304; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_PH:.*]]305; CHECK: [[VECTOR_PH]]:306; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add i32 [[N]], -1307; CHECK-NEXT: [[BROADCAST_SPLATINSERT10:%.*]] = insertelement <4 x i32> undef, i32 [[TRIP_COUNT_MINUS_1]], i32 0308; CHECK-NEXT: [[BROADCAST_SPLAT11:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT10]], <4 x i32> undef, <4 x i32> zeroinitializer309; CHECK-NEXT: [[START:%.*]] = call i32 @llvm.start.loop.iterations.i32(i32 [[TMP13]])310; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]311; CHECK: [[VECTOR_BODY]]:312; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]313; CHECK-NEXT: [[TMP14:%.*]] = phi i32 [ [[START]], %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]314; CHECK-NEXT: [[TMP0:%.*]] = phi i32 [ [[N]], %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]315; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> undef, i32 [[INDEX]], i32 0316; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> undef, <4 x i32> zeroinitializer317; CHECK-NEXT: [[INDUCTION:%.*]] = add <4 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3>318; CHECK-NEXT: [[TMP:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[INDEX]]319; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.vctp32(i32 [[TMP0]])320; CHECK-NEXT: [[TMP2]] = sub i32 [[TMP0]], 4321; CHECK-NEXT: [[WRONG:%.*]] = icmp ult <4 x i32> [[INDUCTION]], [[BROADCAST_SPLAT11]]322; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP]], <4 x i1> [[TMP1]], <4 x i32> undef)323; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[INDEX]]324; CHECK-NEXT: [[WIDE_MASKED_LOAD12:%.*]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP3]], <4 x i1> [[WRONG]], <4 x i32> undef)325; CHECK-NEXT: [[TMP5:%.*]] = mul nsw <4 x i32> [[WIDE_MASKED_LOAD12]], [[WIDE_MASKED_LOAD]]326; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[C]], i32 [[INDEX]]327; CHECK-NEXT: tail call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP5]], ptr align 4 [[TMP6]], <4 x i1> [[TMP1]])328; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4329; CHECK-NEXT: [[TMP15]] = call i32 @llvm.loop.decrement.reg.i32(i32 [[TMP14]], i32 1)330; CHECK-NEXT: [[TMP16:%.*]] = icmp ne i32 [[TMP15]], 0331; CHECK-NEXT: br i1 [[TMP16]], label %[[VECTOR_BODY]], label %[[FOR_COND_CLEANUP]]332; CHECK: [[FOR_COND_CLEANUP]]:333; CHECK-NEXT: ret void334;335entry:336 %cmp8 = icmp eq i32 %N, 0337 %tmp8 = add i32 %N, 3338 %tmp9 = lshr i32 %tmp8, 2339 %tmp10 = shl nuw i32 %tmp9, 2340 %tmp11 = add i32 %tmp10, -4341 %tmp12 = lshr i32 %tmp11, 2342 %tmp13 = add nuw nsw i32 %tmp12, 1343 br i1 %cmp8, label %for.cond.cleanup, label %vector.ph344 345vector.ph: ; preds = %entry346 %trip.count.minus.1 = add i32 %N, -1347 %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0348 %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer349 %start = call i32 @llvm.start.loop.iterations.i32(i32 %tmp13)350 br label %vector.body351 352vector.body: ; preds = %vector.body, %vector.ph353 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]354 %tmp14 = phi i32 [ %start, %vector.ph ], [ %tmp15, %vector.body ]355 %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0356 %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer357 %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>358 %tmp = getelementptr inbounds i32, ptr %a, i32 %index359 %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %N)360 %wrong = icmp ult <4 x i32> %induction, %broadcast.splat11361 %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)362 %tmp3 = getelementptr inbounds i32, ptr %b, i32 %index363 %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp3, i32 4, <4 x i1> %wrong, <4 x i32> undef)364 %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load365 %tmp6 = getelementptr inbounds i32, ptr %c, i32 %index366 tail call void @llvm.masked.store.v4i32.p0(<4 x i32> %tmp5, ptr %tmp6, i32 4, <4 x i1> %active.lane.mask)367 %index.next = add i32 %index, 4368 %tmp15 = call i32 @llvm.loop.decrement.reg.i32(i32 %tmp14, i32 1)369 %tmp16 = icmp ne i32 %tmp15, 0370 br i1 %tmp16, label %vector.body, label %for.cond.cleanup371 372for.cond.cleanup: ; preds = %vector.body, %entry373 ret void374}375 376; The store now uses ult predicate.377define dso_local arm_aapcs_vfpcc void @mismatch_store_pred(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture %c, i32 %N) {378; CHECK-LABEL: define dso_local arm_aapcs_vfpcc void @mismatch_store_pred(379; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias captures(none) [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {380; CHECK-NEXT: [[ENTRY:.*:]]381; CHECK-NEXT: [[CMP8:%.*]] = icmp eq i32 [[N]], 0382; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[N]], 3383; CHECK-NEXT: [[TMP9:%.*]] = lshr i32 [[TMP8]], 2384; CHECK-NEXT: [[TMP10:%.*]] = shl nuw i32 [[TMP9]], 2385; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[TMP10]], -4386; CHECK-NEXT: [[TMP12:%.*]] = lshr i32 [[TMP11]], 2387; CHECK-NEXT: [[TMP13:%.*]] = add nuw nsw i32 [[TMP12]], 1388; CHECK-NEXT: br i1 [[CMP8]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_PH:.*]]389; CHECK: [[VECTOR_PH]]:390; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = add i32 [[N]], -1391; CHECK-NEXT: [[BROADCAST_SPLATINSERT10:%.*]] = insertelement <4 x i32> undef, i32 [[TRIP_COUNT_MINUS_1]], i32 0392; CHECK-NEXT: [[BROADCAST_SPLAT11:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT10]], <4 x i32> undef, <4 x i32> zeroinitializer393; CHECK-NEXT: [[START:%.*]] = call i32 @llvm.start.loop.iterations.i32(i32 [[TMP13]])394; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]395; CHECK: [[VECTOR_BODY]]:396; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]397; CHECK-NEXT: [[TMP14:%.*]] = phi i32 [ [[START]], %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]398; CHECK-NEXT: [[TMP0:%.*]] = phi i32 [ [[N]], %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]399; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> undef, i32 [[INDEX]], i32 0400; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> undef, <4 x i32> zeroinitializer401; CHECK-NEXT: [[INDUCTION:%.*]] = add <4 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3>402; CHECK-NEXT: [[TMP:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[INDEX]]403; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.vctp32(i32 [[TMP0]])404; CHECK-NEXT: [[TMP2]] = sub i32 [[TMP0]], 4405; CHECK-NEXT: [[WRONG:%.*]] = icmp ult <4 x i32> [[INDUCTION]], [[BROADCAST_SPLAT11]]406; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP]], <4 x i1> [[TMP1]], <4 x i32> undef)407; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[INDEX]]408; CHECK-NEXT: [[WIDE_MASKED_LOAD12:%.*]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP3]], <4 x i1> [[TMP1]], <4 x i32> undef)409; CHECK-NEXT: [[TMP5:%.*]] = mul nsw <4 x i32> [[WIDE_MASKED_LOAD12]], [[WIDE_MASKED_LOAD]]410; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[C]], i32 [[INDEX]]411; CHECK-NEXT: tail call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP5]], ptr align 4 [[TMP6]], <4 x i1> [[WRONG]])412; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4413; CHECK-NEXT: [[TMP15]] = call i32 @llvm.loop.decrement.reg.i32(i32 [[TMP14]], i32 1)414; CHECK-NEXT: [[TMP16:%.*]] = icmp ne i32 [[TMP15]], 0415; CHECK-NEXT: br i1 [[TMP16]], label %[[VECTOR_BODY]], label %[[FOR_COND_CLEANUP]]416; CHECK: [[FOR_COND_CLEANUP]]:417; CHECK-NEXT: ret void418;419entry:420 %cmp8 = icmp eq i32 %N, 0421 %tmp8 = add i32 %N, 3422 %tmp9 = lshr i32 %tmp8, 2423 %tmp10 = shl nuw i32 %tmp9, 2424 %tmp11 = add i32 %tmp10, -4425 %tmp12 = lshr i32 %tmp11, 2426 %tmp13 = add nuw nsw i32 %tmp12, 1427 br i1 %cmp8, label %for.cond.cleanup, label %vector.ph428 429vector.ph: ; preds = %entry430 %trip.count.minus.1 = add i32 %N, -1431 %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0432 %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer433 %start = call i32 @llvm.start.loop.iterations.i32(i32 %tmp13)434 br label %vector.body435 436vector.body: ; preds = %vector.body, %vector.ph437 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]438 %tmp14 = phi i32 [ %start, %vector.ph ], [ %tmp15, %vector.body ]439 %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0440 %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer441 %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>442 %tmp = getelementptr inbounds i32, ptr %a, i32 %index443 %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %N)444 %wrong = icmp ult <4 x i32> %induction, %broadcast.splat11445 %wide.masked.load = tail call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)446 %tmp3 = getelementptr inbounds i32, ptr %b, i32 %index447 %wide.masked.load12 = tail call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp3, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)448 %tmp5 = mul nsw <4 x i32> %wide.masked.load12, %wide.masked.load449 %tmp6 = getelementptr inbounds i32, ptr %c, i32 %index450 tail call void @llvm.masked.store.v4i32.p0(<4 x i32> %tmp5, ptr %tmp6, i32 4, <4 x i1> %wrong)451 %index.next = add i32 %index, 4452 %tmp15 = call i32 @llvm.loop.decrement.reg.i32(i32 %tmp14, i32 1)453 %tmp16 = icmp ne i32 %tmp15, 0454 br i1 %tmp16, label %vector.body, label %for.cond.cleanup455 456for.cond.cleanup: ; preds = %vector.body, %entry457 ret void458}459 460; TODO: Multiple intrinsics not yet supported.461; This is currently rejected, because if the vector body is unrolled, the step462; is not what we expect:463;464; Step value 16 doesn't match vector width 4465;466define dso_local void @interleave4(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, ptr noalias nocapture readonly %C, i32 %N) local_unnamed_addr #0 {467; CHECK-LABEL: define dso_local void @interleave4(468; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias readonly captures(none) [[C:%.*]], i32 [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {469; CHECK-NEXT: [[ENTRY:.*:]]470; CHECK-NEXT: [[CMP8:%.*]] = icmp sgt i32 [[N]], 0471; CHECK-NEXT: [[V0:%.*]] = add i32 [[N]], 15472; CHECK-NEXT: [[V1:%.*]] = lshr i32 [[V0]], 4473; CHECK-NEXT: [[V2:%.*]] = shl nuw i32 [[V1]], 4474; CHECK-NEXT: [[V3:%.*]] = add i32 [[V2]], -16475; CHECK-NEXT: [[V4:%.*]] = lshr i32 [[V3]], 4476; CHECK-NEXT: [[V5:%.*]] = add nuw nsw i32 [[V4]], 1477; CHECK-NEXT: br i1 [[CMP8]], label %[[VECTOR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]478; CHECK: [[VECTOR_PH]]:479; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, ptr [[A]], i32 8480; CHECK-NEXT: [[SCEVGEP30:%.*]] = getelementptr i32, ptr [[C]], i32 8481; CHECK-NEXT: [[SCEVGEP37:%.*]] = getelementptr i32, ptr [[B]], i32 8482; CHECK-NEXT: [[START:%.*]] = call i32 @llvm.start.loop.iterations.i32(i32 [[V5]])483; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]484; CHECK: [[VECTOR_BODY]]:485; CHECK-NEXT: [[LSR_IV38:%.*]] = phi ptr [ [[SCEVGEP39:%.*]], %[[VECTOR_BODY]] ], [ [[SCEVGEP37]], %[[VECTOR_PH]] ]486; CHECK-NEXT: [[LSR_IV31:%.*]] = phi ptr [ [[SCEVGEP32:%.*]], %[[VECTOR_BODY]] ], [ [[SCEVGEP30]], %[[VECTOR_PH]] ]487; CHECK-NEXT: [[LSR_IV:%.*]] = phi ptr [ [[SCEVGEP25:%.*]], %[[VECTOR_BODY]] ], [ [[SCEVGEP]], %[[VECTOR_PH]] ]488; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[V14:%.*]], %[[VECTOR_BODY]] ]489; CHECK-NEXT: [[V6:%.*]] = phi i32 [ [[START]], %[[VECTOR_PH]] ], [ [[V15:%.*]], %[[VECTOR_BODY]] ]490; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 [[N]])491; CHECK-NEXT: [[V7:%.*]] = add i32 [[INDEX]], 4492; CHECK-NEXT: [[ACTIVE_LANE_MASK15:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[V7]], i32 [[N]])493; CHECK-NEXT: [[V8:%.*]] = add i32 [[V7]], 4494; CHECK-NEXT: [[ACTIVE_LANE_MASK16:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[V8]], i32 [[N]])495; CHECK-NEXT: [[V9:%.*]] = add i32 [[V8]], 4496; CHECK-NEXT: [[ACTIVE_LANE_MASK17:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[V9]], i32 [[N]])497; CHECK-NEXT: [[SCEVGEP42:%.*]] = getelementptr <4 x i32>, ptr [[LSR_IV38]], i32 -2498; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[SCEVGEP42]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> undef)499; CHECK-NEXT: [[SCEVGEP43:%.*]] = getelementptr <4 x i32>, ptr [[LSR_IV38]], i32 -1500; CHECK-NEXT: [[WIDE_MASKED_LOAD18:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[SCEVGEP43]], <4 x i1> [[ACTIVE_LANE_MASK15]], <4 x i32> undef)501; CHECK-NEXT: [[WIDE_MASKED_LOAD19:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[LSR_IV38]], <4 x i1> [[ACTIVE_LANE_MASK16]], <4 x i32> undef)502; CHECK-NEXT: [[SCEVGEP41:%.*]] = getelementptr <4 x i32>, ptr [[LSR_IV38]], i32 1503; CHECK-NEXT: [[WIDE_MASKED_LOAD20:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[SCEVGEP41]], <4 x i1> [[ACTIVE_LANE_MASK17]], <4 x i32> undef)504; CHECK-NEXT: [[SCEVGEP34:%.*]] = getelementptr <4 x i32>, ptr [[LSR_IV31]], i32 -2505; CHECK-NEXT: [[WIDE_MASKED_LOAD21:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[SCEVGEP34]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> undef)506; CHECK-NEXT: [[SCEVGEP35:%.*]] = getelementptr <4 x i32>, ptr [[LSR_IV31]], i32 -1507; CHECK-NEXT: [[WIDE_MASKED_LOAD22:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[SCEVGEP35]], <4 x i1> [[ACTIVE_LANE_MASK15]], <4 x i32> undef)508; CHECK-NEXT: [[WIDE_MASKED_LOAD23:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[LSR_IV31]], <4 x i1> [[ACTIVE_LANE_MASK16]], <4 x i32> undef)509; CHECK-NEXT: [[SCEVGEP36:%.*]] = getelementptr <4 x i32>, ptr [[LSR_IV31]], i32 1510; CHECK-NEXT: [[WIDE_MASKED_LOAD24:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[SCEVGEP36]], <4 x i1> [[ACTIVE_LANE_MASK17]], <4 x i32> undef)511; CHECK-NEXT: [[V10:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD21]], [[WIDE_MASKED_LOAD]]512; CHECK-NEXT: [[V11:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD22]], [[WIDE_MASKED_LOAD18]]513; CHECK-NEXT: [[V12:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD23]], [[WIDE_MASKED_LOAD19]]514; CHECK-NEXT: [[V13:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD24]], [[WIDE_MASKED_LOAD20]]515; CHECK-NEXT: [[SCEVGEP27:%.*]] = getelementptr <4 x i32>, ptr [[LSR_IV]], i32 -2516; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[V10]], ptr align 4 [[SCEVGEP27]], <4 x i1> [[ACTIVE_LANE_MASK]])517; CHECK-NEXT: [[SCEVGEP28:%.*]] = getelementptr <4 x i32>, ptr [[LSR_IV]], i32 -1518; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[V11]], ptr align 4 [[SCEVGEP28]], <4 x i1> [[ACTIVE_LANE_MASK15]])519; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[V12]], ptr align 4 [[LSR_IV]], <4 x i1> [[ACTIVE_LANE_MASK16]])520; CHECK-NEXT: [[SCEVGEP29:%.*]] = getelementptr <4 x i32>, ptr [[LSR_IV]], i32 1521; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[V13]], ptr align 4 [[SCEVGEP29]], <4 x i1> [[ACTIVE_LANE_MASK17]])522; CHECK-NEXT: [[SCEVGEP25]] = getelementptr i32, ptr [[LSR_IV]], i32 16523; CHECK-NEXT: [[SCEVGEP32]] = getelementptr i32, ptr [[LSR_IV31]], i32 16524; CHECK-NEXT: [[SCEVGEP39]] = getelementptr i32, ptr [[LSR_IV38]], i32 16525; CHECK-NEXT: [[V14]] = add i32 [[V9]], 4526; CHECK-NEXT: [[V15]] = call i32 @llvm.loop.decrement.reg.i32(i32 [[V6]], i32 1)527; CHECK-NEXT: [[V16:%.*]] = icmp ne i32 [[V15]], 0528; CHECK-NEXT: br i1 [[V16]], label %[[VECTOR_BODY]], label %[[FOR_COND_CLEANUP]]529; CHECK: [[FOR_COND_CLEANUP]]:530; CHECK-NEXT: ret void531;532entry:533 %cmp8 = icmp sgt i32 %N, 0534 %v0 = add i32 %N, 15535 %v1 = lshr i32 %v0, 4536 %v2 = shl nuw i32 %v1, 4537 %v3 = add i32 %v2, -16538 %v4 = lshr i32 %v3, 4539 %v5 = add nuw nsw i32 %v4, 1540 br i1 %cmp8, label %vector.ph, label %for.cond.cleanup541 542 543vector.ph:544 %scevgep = getelementptr i32, ptr %A, i32 8545 %scevgep30 = getelementptr i32, ptr %C, i32 8546 %scevgep37 = getelementptr i32, ptr %B, i32 8547 %start = call i32 @llvm.start.loop.iterations.i32(i32 %v5)548 br label %vector.body549 550vector.body:551 %lsr.iv38 = phi ptr [ %scevgep39, %vector.body ], [ %scevgep37, %vector.ph ]552 %lsr.iv31 = phi ptr [ %scevgep32, %vector.body ], [ %scevgep30, %vector.ph ]553 %lsr.iv = phi ptr [ %scevgep25, %vector.body ], [ %scevgep, %vector.ph ]554 %index = phi i32 [ 0, %vector.ph ], [ %v14, %vector.body ]555 %v6 = phi i32 [ %start, %vector.ph ], [ %v15, %vector.body ]556 %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %N)557 %v7 = add i32 %index, 4558 %active.lane.mask15 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %v7, i32 %N)559 %v8 = add i32 %v7, 4560 %active.lane.mask16 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %v8, i32 %N)561 %v9 = add i32 %v8, 4562 %active.lane.mask17 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %v9, i32 %N)563 %scevgep42 = getelementptr <4 x i32>, ptr %lsr.iv38, i32 -2564 %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %scevgep42, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)565 %scevgep43 = getelementptr <4 x i32>, ptr %lsr.iv38, i32 -1566 %wide.masked.load18 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr nonnull %scevgep43, i32 4, <4 x i1> %active.lane.mask15, <4 x i32> undef)567 %wide.masked.load19 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr nonnull %lsr.iv38, i32 4, <4 x i1> %active.lane.mask16, <4 x i32> undef)568 %scevgep41 = getelementptr <4 x i32>, ptr %lsr.iv38, i32 1569 %wide.masked.load20 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr nonnull %scevgep41, i32 4, <4 x i1> %active.lane.mask17, <4 x i32> undef)570 %scevgep34 = getelementptr <4 x i32>, ptr %lsr.iv31, i32 -2571 %wide.masked.load21 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %scevgep34, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)572 %scevgep35 = getelementptr <4 x i32>, ptr %lsr.iv31, i32 -1573 %wide.masked.load22 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr nonnull %scevgep35, i32 4, <4 x i1> %active.lane.mask15, <4 x i32> undef)574 %wide.masked.load23 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr nonnull %lsr.iv31, i32 4, <4 x i1> %active.lane.mask16, <4 x i32> undef)575 %scevgep36 = getelementptr <4 x i32>, ptr %lsr.iv31, i32 1576 %wide.masked.load24 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr nonnull %scevgep36, i32 4, <4 x i1> %active.lane.mask17, <4 x i32> undef)577 %v10 = add nsw <4 x i32> %wide.masked.load21, %wide.masked.load578 %v11 = add nsw <4 x i32> %wide.masked.load22, %wide.masked.load18579 %v12 = add nsw <4 x i32> %wide.masked.load23, %wide.masked.load19580 %v13 = add nsw <4 x i32> %wide.masked.load24, %wide.masked.load20581 %scevgep27 = getelementptr <4 x i32>, ptr %lsr.iv, i32 -2582 call void @llvm.masked.store.v4i32.p0(<4 x i32> %v10, ptr %scevgep27, i32 4, <4 x i1> %active.lane.mask)583 %scevgep28 = getelementptr <4 x i32>, ptr %lsr.iv, i32 -1584 call void @llvm.masked.store.v4i32.p0(<4 x i32> %v11, ptr %scevgep28, i32 4, <4 x i1> %active.lane.mask15)585 call void @llvm.masked.store.v4i32.p0(<4 x i32> %v12, ptr %lsr.iv, i32 4, <4 x i1> %active.lane.mask16)586 %scevgep29 = getelementptr <4 x i32>, ptr %lsr.iv, i32 1587 call void @llvm.masked.store.v4i32.p0(<4 x i32> %v13, ptr %scevgep29, i32 4, <4 x i1> %active.lane.mask17)588 %scevgep25 = getelementptr i32, ptr %lsr.iv, i32 16589 %scevgep32 = getelementptr i32, ptr %lsr.iv31, i32 16590 %scevgep39 = getelementptr i32, ptr %lsr.iv38, i32 16591 %v14 = add i32 %v9, 4592 %v15 = call i32 @llvm.loop.decrement.reg.i32(i32 %v6, i32 1)593 %v16 = icmp ne i32 %v15, 0594 br i1 %v16, label %vector.body, label %for.cond.cleanup595 596for.cond.cleanup:597 ret void598}599 600define dso_local void @const_expected_in_set_loop(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, ptr noalias nocapture readonly %C, i32 %N) local_unnamed_addr #0 {601; CHECK-LABEL: define dso_local void @const_expected_in_set_loop(602; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias readonly captures(none) [[C:%.*]], i32 [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {603; CHECK-NEXT: [[ENTRY:.*:]]604; CHECK-NEXT: [[CMP8:%.*]] = icmp sgt i32 [[N]], 0605; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], 3606; CHECK-NEXT: [[TMP1:%.*]] = lshr i32 [[TMP0]], 2607; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i32 [[TMP1]], 2608; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[TMP2]], -4609; CHECK-NEXT: [[TMP4:%.*]] = lshr i32 [[TMP3]], 2610; CHECK-NEXT: [[TMP5:%.*]] = add nuw nsw i32 [[TMP4]], 1611; CHECK-NEXT: br i1 [[CMP8]], label %[[VECTOR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]612; CHECK: [[VECTOR_PH]]:613; CHECK-NEXT: [[START:%.*]] = call i32 @llvm.start.loop.iterations.i32(i32 [[TMP5]])614; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]615; CHECK: [[VECTOR_BODY]]:616; CHECK-NEXT: [[LSR_IV17:%.*]] = phi ptr [ [[SCEVGEP18:%.*]], %[[VECTOR_BODY]] ], [ [[A]], %[[VECTOR_PH]] ]617; CHECK-NEXT: [[LSR_IV14:%.*]] = phi ptr [ [[SCEVGEP15:%.*]], %[[VECTOR_BODY]] ], [ [[C]], %[[VECTOR_PH]] ]618; CHECK-NEXT: [[LSR_IV:%.*]] = phi ptr [ [[SCEVGEP:%.*]], %[[VECTOR_BODY]] ], [ [[B]], %[[VECTOR_PH]] ]619; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]620; CHECK-NEXT: [[TMP6:%.*]] = phi i32 [ [[START]], %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]621; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 42)622; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[LSR_IV]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> undef)623; CHECK-NEXT: [[WIDE_MASKED_LOAD12:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[LSR_IV14]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> undef)624; CHECK-NEXT: [[TMP7:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD12]], [[WIDE_MASKED_LOAD]]625; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP7]], ptr align 4 [[LSR_IV17]], <4 x i1> [[ACTIVE_LANE_MASK]])626; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4627; CHECK-NEXT: [[SCEVGEP]] = getelementptr i32, ptr [[LSR_IV]], i32 4628; CHECK-NEXT: [[SCEVGEP15]] = getelementptr i32, ptr [[LSR_IV14]], i32 4629; CHECK-NEXT: [[SCEVGEP18]] = getelementptr i32, ptr [[LSR_IV17]], i32 4630; CHECK-NEXT: [[TMP8]] = call i32 @llvm.loop.decrement.reg.i32(i32 [[TMP6]], i32 1)631; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i32 [[TMP8]], 0632; CHECK-NEXT: br i1 [[TMP9]], label %[[VECTOR_BODY]], label %[[FOR_COND_CLEANUP]]633; CHECK: [[FOR_COND_CLEANUP]]:634; CHECK-NEXT: ret void635;636entry:637 %cmp8 = icmp sgt i32 %N, 0638 %0 = add i32 %N, 3639 %1 = lshr i32 %0, 2640 %2 = shl nuw i32 %1, 2641 %3 = add i32 %2, -4642 %4 = lshr i32 %3, 2643 %5 = add nuw nsw i32 %4, 1644 br i1 %cmp8, label %vector.ph, label %for.cond.cleanup645 646vector.ph:647 %start = call i32 @llvm.start.loop.iterations.i32(i32 %5)648 br label %vector.body649 650vector.body: ; preds = %vector.body, %vector.ph651 %lsr.iv17 = phi ptr [ %scevgep18, %vector.body ], [ %A, %vector.ph ]652 %lsr.iv14 = phi ptr [ %scevgep15, %vector.body ], [ %C, %vector.ph ]653 %lsr.iv = phi ptr [ %scevgep, %vector.body ], [ %B, %vector.ph ]654 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]655 %6 = phi i32 [ %start, %vector.ph ], [ %8, %vector.body ]656 %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 42)657 %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %lsr.iv, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)658 %wide.masked.load12 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %lsr.iv14, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)659 %7 = add nsw <4 x i32> %wide.masked.load12, %wide.masked.load660 call void @llvm.masked.store.v4i32.p0(<4 x i32> %7, ptr %lsr.iv17, i32 4, <4 x i1> %active.lane.mask)661 %index.next = add i32 %index, 4662 %scevgep = getelementptr i32, ptr %lsr.iv, i32 4663 %scevgep15 = getelementptr i32, ptr %lsr.iv14, i32 4664 %scevgep18 = getelementptr i32, ptr %lsr.iv17, i32 4665 %8 = call i32 @llvm.loop.decrement.reg.i32(i32 %6, i32 1)666 %9 = icmp ne i32 %8, 0667 br i1 %9, label %vector.body, label %for.cond.cleanup668 669for.cond.cleanup: ; preds = %vector.body, %entry670 ret void671}672 673define dso_local void @tripcount_arg_not_invariant(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, ptr noalias nocapture readonly %C, i32 %N) local_unnamed_addr #0 {674; CHECK-LABEL: define dso_local void @tripcount_arg_not_invariant(675; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias readonly captures(none) [[C:%.*]], i32 [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {676; CHECK-NEXT: [[ENTRY:.*:]]677; CHECK-NEXT: [[CMP8:%.*]] = icmp sgt i32 [[N]], 0678; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], 3679; CHECK-NEXT: [[TMP1:%.*]] = lshr i32 [[TMP0]], 2680; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i32 [[TMP1]], 2681; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[TMP2]], -4682; CHECK-NEXT: [[TMP4:%.*]] = lshr i32 [[TMP3]], 2683; CHECK-NEXT: [[TMP5:%.*]] = add nuw nsw i32 [[TMP4]], 1684; CHECK-NEXT: br i1 [[CMP8]], label %[[VECTOR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]685; CHECK: [[VECTOR_PH]]:686; CHECK-NEXT: [[START:%.*]] = call i32 @llvm.start.loop.iterations.i32(i32 [[TMP5]])687; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]688; CHECK: [[VECTOR_BODY]]:689; CHECK-NEXT: [[LSR_IV17:%.*]] = phi ptr [ [[SCEVGEP18:%.*]], %[[VECTOR_BODY]] ], [ [[A]], %[[VECTOR_PH]] ]690; CHECK-NEXT: [[LSR_IV14:%.*]] = phi ptr [ [[SCEVGEP15:%.*]], %[[VECTOR_BODY]] ], [ [[C]], %[[VECTOR_PH]] ]691; CHECK-NEXT: [[LSR_IV:%.*]] = phi ptr [ [[SCEVGEP:%.*]], %[[VECTOR_BODY]] ], [ [[B]], %[[VECTOR_PH]] ]692; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]693; CHECK-NEXT: [[TMP6:%.*]] = phi i32 [ [[START]], %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]694; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 [[INDEX]])695; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[LSR_IV]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> undef)696; CHECK-NEXT: [[WIDE_MASKED_LOAD12:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[LSR_IV14]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> undef)697; CHECK-NEXT: [[TMP7:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD12]], [[WIDE_MASKED_LOAD]]698; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP7]], ptr align 4 [[LSR_IV17]], <4 x i1> [[ACTIVE_LANE_MASK]])699; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4700; CHECK-NEXT: [[SCEVGEP]] = getelementptr i32, ptr [[LSR_IV]], i32 4701; CHECK-NEXT: [[SCEVGEP15]] = getelementptr i32, ptr [[LSR_IV14]], i32 4702; CHECK-NEXT: [[SCEVGEP18]] = getelementptr i32, ptr [[LSR_IV17]], i32 4703; CHECK-NEXT: [[TMP8]] = call i32 @llvm.loop.decrement.reg.i32(i32 [[TMP6]], i32 1)704; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i32 [[TMP8]], 0705; CHECK-NEXT: br i1 [[TMP9]], label %[[VECTOR_BODY]], label %[[VECTOR_PH]]706; CHECK: [[FOR_COND_CLEANUP]]:707; CHECK-NEXT: ret void708;709entry:710 %cmp8 = icmp sgt i32 %N, 0711 %0 = add i32 %N, 3712 %1 = lshr i32 %0, 2713 %2 = shl nuw i32 %1, 2714 %3 = add i32 %2, -4715 %4 = lshr i32 %3, 2716 %5 = add nuw nsw i32 %4, 1717 br i1 %cmp8, label %vector.ph, label %for.cond.cleanup718 719vector.ph: ; preds = %entry720 %start = call i32 @llvm.start.loop.iterations.i32(i32 %5)721 br label %vector.body722 723vector.body: ; preds = %vector.body, %vector.ph724 %lsr.iv17 = phi ptr [ %scevgep18, %vector.body ], [ %A, %vector.ph ]725 %lsr.iv14 = phi ptr [ %scevgep15, %vector.body ], [ %C, %vector.ph ]726 %lsr.iv = phi ptr [ %scevgep, %vector.body ], [ %B, %vector.ph ]727 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]728 %6 = phi i32 [ %start, %vector.ph ], [ %8, %vector.body ]729 730 %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %index)731 %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %lsr.iv, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)732 %wide.masked.load12 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %lsr.iv14, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)733 %7 = add nsw <4 x i32> %wide.masked.load12, %wide.masked.load734 call void @llvm.masked.store.v4i32.p0(<4 x i32> %7, ptr %lsr.iv17, i32 4, <4 x i1> %active.lane.mask)735 %index.next = add i32 %index, 4736 %scevgep = getelementptr i32, ptr %lsr.iv, i32 4737 %scevgep15 = getelementptr i32, ptr %lsr.iv14, i32 4738 %scevgep18 = getelementptr i32, ptr %lsr.iv17, i32 4739 %8 = call i32 @llvm.loop.decrement.reg.i32(i32 %6, i32 1)740 %9 = icmp ne i32 %8, 0741 ;br i1 %9, label %vector.body, label %for.cond.cleanup742 br i1 %9, label %vector.body, label %vector.ph743 744for.cond.cleanup: ; preds = %vector.body, %entry745 ret void746}747 748define dso_local void @addrec_base_not_zero(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, ptr noalias nocapture readonly %C, i32 %N) local_unnamed_addr #0 {749; CHECK-LABEL: define dso_local void @addrec_base_not_zero(750; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], ptr noalias readonly captures(none) [[C:%.*]], i32 [[N:%.*]]) local_unnamed_addr #[[ATTR0]] {751; CHECK-NEXT: [[ENTRY:.*:]]752; CHECK-NEXT: [[CMP8:%.*]] = icmp sgt i32 [[N]], 0753; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], 3754; CHECK-NEXT: [[TMP1:%.*]] = lshr i32 [[TMP0]], 2755; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i32 [[TMP1]], 2756; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[TMP2]], -4757; CHECK-NEXT: [[TMP4:%.*]] = lshr i32 [[TMP3]], 2758; CHECK-NEXT: [[TMP5:%.*]] = add nuw nsw i32 [[TMP4]], 1759; CHECK-NEXT: br i1 [[CMP8]], label %[[VECTOR_PH:.*]], label %[[FOR_COND_CLEANUP:.*]]760; CHECK: [[VECTOR_PH]]:761; CHECK-NEXT: [[START:%.*]] = call i32 @llvm.start.loop.iterations.i32(i32 [[TMP5]])762; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]763; CHECK: [[VECTOR_BODY]]:764; CHECK-NEXT: [[LSR_IV17:%.*]] = phi ptr [ [[SCEVGEP18:%.*]], %[[VECTOR_BODY]] ], [ [[A]], %[[VECTOR_PH]] ]765; CHECK-NEXT: [[LSR_IV14:%.*]] = phi ptr [ [[SCEVGEP15:%.*]], %[[VECTOR_BODY]] ], [ [[C]], %[[VECTOR_PH]] ]766; CHECK-NEXT: [[LSR_IV:%.*]] = phi ptr [ [[SCEVGEP:%.*]], %[[VECTOR_BODY]] ], [ [[B]], %[[VECTOR_PH]] ]767; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 1, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]768; CHECK-NEXT: [[TMP6:%.*]] = phi i32 [ [[START]], %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]769; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 [[N]])770; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[LSR_IV]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> undef)771; CHECK-NEXT: [[WIDE_MASKED_LOAD12:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[LSR_IV14]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> undef)772; CHECK-NEXT: [[TMP7:%.*]] = add nsw <4 x i32> [[WIDE_MASKED_LOAD12]], [[WIDE_MASKED_LOAD]]773; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP7]], ptr align 4 [[LSR_IV17]], <4 x i1> [[ACTIVE_LANE_MASK]])774; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4775; CHECK-NEXT: [[SCEVGEP]] = getelementptr i32, ptr [[LSR_IV]], i32 4776; CHECK-NEXT: [[SCEVGEP15]] = getelementptr i32, ptr [[LSR_IV14]], i32 4777; CHECK-NEXT: [[SCEVGEP18]] = getelementptr i32, ptr [[LSR_IV17]], i32 4778; CHECK-NEXT: [[TMP8]] = call i32 @llvm.loop.decrement.reg.i32(i32 [[TMP6]], i32 1)779; CHECK-NEXT: [[TMP9:%.*]] = icmp ne i32 [[TMP8]], 0780; CHECK-NEXT: br i1 [[TMP9]], label %[[VECTOR_BODY]], label %[[VECTOR_PH]]781; CHECK: [[FOR_COND_CLEANUP]]:782; CHECK-NEXT: ret void783;784entry:785 %cmp8 = icmp sgt i32 %N, 0786 %0 = add i32 %N, 3787 %1 = lshr i32 %0, 2788 %2 = shl nuw i32 %1, 2789 %3 = add i32 %2, -4790 %4 = lshr i32 %3, 2791 %5 = add nuw nsw i32 %4, 1792 br i1 %cmp8, label %vector.ph, label %for.cond.cleanup793 794vector.ph: ; preds = %entry795 %start = call i32 @llvm.start.loop.iterations.i32(i32 %5)796 br label %vector.body797 798vector.body: ; preds = %vector.body, %vector.ph799 %lsr.iv17 = phi ptr [ %scevgep18, %vector.body ], [ %A, %vector.ph ]800 %lsr.iv14 = phi ptr [ %scevgep15, %vector.body ], [ %C, %vector.ph ]801 %lsr.iv = phi ptr [ %scevgep, %vector.body ], [ %B, %vector.ph ]802 803; AddRec base is not 0:804 %index = phi i32 [ 1, %vector.ph ], [ %index.next, %vector.body ]805 806 %6 = phi i32 [ %start, %vector.ph ], [ %8, %vector.body ]807 %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %N)808 %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %lsr.iv, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)809 %wide.masked.load12 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %lsr.iv14, i32 4, <4 x i1> %active.lane.mask, <4 x i32> undef)810 %7 = add nsw <4 x i32> %wide.masked.load12, %wide.masked.load811 call void @llvm.masked.store.v4i32.p0(<4 x i32> %7, ptr %lsr.iv17, i32 4, <4 x i1> %active.lane.mask)812 %index.next = add i32 %index, 4813 %scevgep = getelementptr i32, ptr %lsr.iv, i32 4814 %scevgep15 = getelementptr i32, ptr %lsr.iv14, i32 4815 %scevgep18 = getelementptr i32, ptr %lsr.iv17, i32 4816 %8 = call i32 @llvm.loop.decrement.reg.i32(i32 %6, i32 1)817 %9 = icmp ne i32 %8, 0818 ;br i1 %9, label %vector.body, label %for.cond.cleanup819 br i1 %9, label %vector.body, label %vector.ph820 821for.cond.cleanup: ; preds = %vector.body, %entry822 ret void823}824 825 826declare <16 x i8> @llvm.masked.load.v16i8.p0(ptr, i32 immarg, <16 x i1>, <16 x i8>)827declare void @llvm.masked.store.v16i8.p0(<16 x i8>, ptr, i32 immarg, <16 x i1>)828declare <8 x i16> @llvm.masked.load.v8i16.p0(ptr, i32 immarg, <8 x i1>, <8 x i16>)829declare void @llvm.masked.store.v8i16.p0(<8 x i16>, ptr, i32 immarg, <8 x i1>)830declare <4 x i32> @llvm.masked.load.v4i32.p0(ptr, i32 immarg, <4 x i1>, <4 x i32>)831declare void @llvm.masked.store.v2i64.p0(<2 x i64>, ptr, i32 immarg, <2 x i1>)832declare <2 x i64> @llvm.masked.load.v2i64.p0(ptr, i32 immarg, <2 x i1>, <2 x i64>)833declare void @llvm.masked.store.v4i32.p0(<4 x i32>, ptr, i32 immarg, <4 x i1>)834declare i32 @llvm.start.loop.iterations.i32(i32)835declare i32 @llvm.loop.decrement.reg.i32(i32, i32)836declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32)837declare <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32, i32)838declare <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32, i32)839