442 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt --arm-mve-gather-scatter-lowering -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp %s -S -o - | FileCheck %s3 4target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"5 6define arm_aapcs_vfpcc void @push_out_add_sub_block(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n.vec) {7; CHECK-LABEL: @push_out_add_sub_block(8; CHECK-NEXT: vector.ph:9; CHECK-NEXT: [[PUSHEDOUTADD:%.*]] = add <4 x i32> <i32 0, i32 2, i32 4, i32 6>, splat (i32 6)10; CHECK-NEXT: [[SCALEDINDEX:%.*]] = shl <4 x i32> [[PUSHEDOUTADD]], splat (i32 2)11; CHECK-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[DATA:%.*]] to i3212; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP0]], i64 013; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer14; CHECK-NEXT: [[STARTINDEX:%.*]] = add <4 x i32> [[SCALEDINDEX]], [[DOTSPLAT]]15; CHECK-NEXT: [[PREINCREMENTSTARTINDEX:%.*]] = sub <4 x i32> [[STARTINDEX]], splat (i32 32)16; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]17; CHECK: vector.body:18; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY_END:%.*]] ]19; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[PREINCREMENTSTARTINDEX]], [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY_END]] ]20; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX]], 4821; CHECK-NEXT: br i1 [[TMP1]], label [[LOWER_BLOCK:%.*]], label [[END:%.*]]22; CHECK: lower.block:23; CHECK-NEXT: [[TMP2:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4i32.v4i32(<4 x i32> [[VEC_IND]], i32 32)24; CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[TMP2]], 025; CHECK-NEXT: [[TMP4]] = extractvalue { <4 x i32>, <4 x i32> } [[TMP2]], 126; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 [[INDEX]]27; CHECK-NEXT: store <4 x i32> [[TMP3]], ptr [[TMP5]], align 428; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 429; CHECK-NEXT: br label [[VECTOR_BODY_END]]30; CHECK: vector.body.end:31; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC:%.*]]32; CHECK-NEXT: br i1 [[TMP6]], label [[END]], label [[VECTOR_BODY]]33; CHECK: end:34; CHECK-NEXT: ret void35;36 37vector.ph:38 br label %vector.body39 40vector.body: ; preds = %vector.body, %vector.ph41 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body.end ]42 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body.end ]43 %0 = icmp eq i32 %index, 4844 br i1 %0, label %lower.block, label %end45 46lower.block: ; preds = %vector.body47 %1 = add <4 x i32> %vec.ind, <i32 6, i32 6, i32 6, i32 6>48 %2 = getelementptr inbounds i32, ptr %data, <4 x i32> %149 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %2, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)50 %3 = getelementptr inbounds i32, ptr %dst, i32 %index51 %4 = bitcast ptr %3 to ptr52 store <4 x i32> %wide.masked.gather, ptr %4, align 453 %index.next = add i32 %index, 454 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8>55 br label %vector.body.end56 57vector.body.end: ; preds = %lower.block58 %5 = icmp eq i32 %index.next, %n.vec59 br i1 %5, label %end, label %vector.body60 61end:62 ret void;63}64 65define arm_aapcs_vfpcc void @push_out_add_sub_block_commutedphi(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n.vec) {66; CHECK-LABEL: @push_out_add_sub_block_commutedphi(67; CHECK-NEXT: vector.ph:68; CHECK-NEXT: [[PUSHEDOUTADD:%.*]] = add <4 x i32> <i32 0, i32 2, i32 4, i32 6>, splat (i32 6)69; CHECK-NEXT: [[SCALEDINDEX:%.*]] = shl <4 x i32> [[PUSHEDOUTADD]], splat (i32 2)70; CHECK-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[DATA:%.*]] to i3271; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP0]], i64 072; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer73; CHECK-NEXT: [[STARTINDEX:%.*]] = add <4 x i32> [[SCALEDINDEX]], [[DOTSPLAT]]74; CHECK-NEXT: [[PREINCREMENTSTARTINDEX:%.*]] = sub <4 x i32> [[STARTINDEX]], splat (i32 32)75; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]76; CHECK: vector.body:77; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY_END:%.*]] ]78; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[PREINCREMENTSTARTINDEX]], [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY_END]] ]79; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX]], 4880; CHECK-NEXT: br i1 [[TMP1]], label [[LOWER_BLOCK:%.*]], label [[END:%.*]]81; CHECK: lower.block:82; CHECK-NEXT: [[TMP2:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4i32.v4i32(<4 x i32> [[VEC_IND]], i32 32)83; CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[TMP2]], 084; CHECK-NEXT: [[TMP4]] = extractvalue { <4 x i32>, <4 x i32> } [[TMP2]], 185; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 [[INDEX]]86; CHECK-NEXT: store <4 x i32> [[TMP3]], ptr [[TMP5]], align 487; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 488; CHECK-NEXT: br label [[VECTOR_BODY_END]]89; CHECK: vector.body.end:90; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC:%.*]]91; CHECK-NEXT: br i1 [[TMP6]], label [[END]], label [[VECTOR_BODY]]92; CHECK: end:93; CHECK-NEXT: ret void94;95 96vector.ph:97 br label %vector.body98 99vector.body: ; preds = %vector.body, %vector.ph100 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body.end ]101 %vec.ind = phi <4 x i32> [ %vec.ind.next, %vector.body.end ], [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ]102 %0 = icmp eq i32 %index, 48103 br i1 %0, label %lower.block, label %end104 105lower.block: ; preds = %vector.body106 %1 = add <4 x i32> %vec.ind, <i32 6, i32 6, i32 6, i32 6>107 %2 = getelementptr inbounds i32, ptr %data, <4 x i32> %1108 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %2, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)109 %3 = getelementptr inbounds i32, ptr %dst, i32 %index110 %4 = bitcast ptr %3 to ptr111 store <4 x i32> %wide.masked.gather, ptr %4, align 4112 %index.next = add i32 %index, 4113 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8>114 br label %vector.body.end115 116vector.body.end: ; preds = %lower.block117 %5 = icmp eq i32 %index.next, %n.vec118 br i1 %5, label %end, label %vector.body119 120end:121 ret void;122}123 124define arm_aapcs_vfpcc void @push_out_mul_sub_block(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n.vec) {125; CHECK-LABEL: @push_out_mul_sub_block(126; CHECK-NEXT: vector.ph:127; CHECK-NEXT: [[PUSHEDOUTMUL:%.*]] = mul <4 x i32> <i32 0, i32 2, i32 4, i32 6>, splat (i32 3)128; CHECK-NEXT: [[PRODUCT:%.*]] = mul <4 x i32> splat (i32 8), splat (i32 3)129; CHECK-NEXT: [[PUSHEDOUTADD:%.*]] = add <4 x i32> [[PUSHEDOUTMUL]], splat (i32 6)130; CHECK-NEXT: [[SCALEDINDEX:%.*]] = shl <4 x i32> [[PUSHEDOUTADD]], splat (i32 2)131; CHECK-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[DATA:%.*]] to i32132; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP0]], i64 0133; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer134; CHECK-NEXT: [[STARTINDEX:%.*]] = add <4 x i32> [[SCALEDINDEX]], [[DOTSPLAT]]135; CHECK-NEXT: [[PREINCREMENTSTARTINDEX:%.*]] = sub <4 x i32> [[STARTINDEX]], splat (i32 96)136; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]137; CHECK: vector.body:138; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY_END:%.*]] ]139; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[PREINCREMENTSTARTINDEX]], [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY_END]] ]140; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i32 [[INDEX]], 48141; CHECK-NEXT: br i1 [[TMP1]], label [[LOWER_BLOCK:%.*]], label [[END:%.*]]142; CHECK: lower.block:143; CHECK-NEXT: [[TMP2:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4i32.v4i32(<4 x i32> [[VEC_IND]], i32 96)144; CHECK-NEXT: [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[TMP2]], 0145; CHECK-NEXT: [[TMP4]] = extractvalue { <4 x i32>, <4 x i32> } [[TMP2]], 1146; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 [[INDEX]]147; CHECK-NEXT: store <4 x i32> [[TMP3]], ptr [[TMP5]], align 4148; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4149; CHECK-NEXT: br label [[VECTOR_BODY_END]]150; CHECK: vector.body.end:151; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC:%.*]]152; CHECK-NEXT: br i1 [[TMP6]], label [[END]], label [[VECTOR_BODY]]153; CHECK: end:154; CHECK-NEXT: ret void155;156 157vector.ph:158 br label %vector.body159 160vector.body: ; preds = %vector.body, %vector.ph161 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body.end ]162 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body.end ]163 %0 = icmp eq i32 %index, 48164 br i1 %0, label %lower.block, label %end165 166lower.block: ; preds = %vector.body167 %1 = mul <4 x i32> %vec.ind, <i32 3, i32 3, i32 3, i32 3>168 %2 = add <4 x i32> %1, <i32 6, i32 6, i32 6, i32 6>169 %3 = getelementptr inbounds i32, ptr %data, <4 x i32> %2170 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %3, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)171 %4 = getelementptr inbounds i32, ptr %dst, i32 %index172 %5 = bitcast ptr %4 to ptr173 store <4 x i32> %wide.masked.gather, ptr %5, align 4174 %index.next = add i32 %index, 4175 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8>176 br label %vector.body.end177 178vector.body.end: ; preds = %lower.block179 %6 = icmp eq i32 %index.next, %n.vec180 br i1 %6, label %end, label %vector.body181 182end:183 ret void;184}185 186 187define arm_aapcs_vfpcc void @push_out_mul_sub_loop(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n.vec) {188; CHECK-LABEL: @push_out_mul_sub_loop(189; CHECK-NEXT: vector.ph:190; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]191; CHECK: vector.body:192; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY_END:%.*]] ]193; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY_END]] ]194; CHECK-NEXT: br label [[VECTOR_2_PH:%.*]]195; CHECK: vector.2.ph:196; CHECK-NEXT: br label [[VECTOR_2_BODY:%.*]]197; CHECK: vector.2.body:198; CHECK-NEXT: [[TMP0:%.*]] = mul <4 x i32> [[VEC_IND]], splat (i32 3)199; CHECK-NEXT: [[SCALEDINDEX:%.*]] = shl <4 x i32> [[TMP0]], splat (i32 2)200; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[DATA:%.*]] to i32201; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1]], i64 0202; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer203; CHECK-NEXT: [[STARTINDEX:%.*]] = add <4 x i32> [[SCALEDINDEX]], [[DOTSPLAT]]204; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.base.v4i32.v4i32(<4 x i32> [[STARTINDEX]], i32 24)205; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 [[INDEX]]206; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4207; CHECK-NEXT: br label [[VECTOR_2_BODY_END:%.*]]208; CHECK: vector.2.body.end:209; CHECK-NEXT: [[INDEX_2_NEXT:%.*]] = add i32 [[INDEX]], 4210; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_2_NEXT]], 16211; CHECK-NEXT: br i1 [[TMP4]], label [[VECTOR_BODY_END]], label [[VECTOR_2_BODY]]212; CHECK: vector.body.end:213; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4214; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 8)215; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC:%.*]]216; CHECK-NEXT: br i1 [[TMP5]], label [[END:%.*]], label [[VECTOR_BODY]]217; CHECK: end:218; CHECK-NEXT: ret void219;220 221vector.ph:222 br label %vector.body223 224vector.body: ; preds = %vector.body, %vector.ph225 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body.end ]226 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body.end ]227 br label %vector.2.ph228 229vector.2.ph:230 br label %vector.2.body231 232vector.2.body: ; preds = %vector.body233 %0 = mul <4 x i32> %vec.ind, <i32 3, i32 3, i32 3, i32 3>234 %1 = add <4 x i32> %0, <i32 6, i32 6, i32 6, i32 6>235 %2 = getelementptr inbounds i32, ptr %data, <4 x i32> %1236 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %2, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)237 %3 = getelementptr inbounds i32, ptr %dst, i32 %index238 %4 = bitcast ptr %3 to ptr239 store <4 x i32> %wide.masked.gather, ptr %4, align 4240 br label %vector.2.body.end241 242vector.2.body.end: ; preds = %lower.block243 %index.2.next = add i32 %index, 4244 %5 = icmp eq i32 %index.2.next, 16245 br i1 %5, label %vector.body.end, label %vector.2.body246 247vector.body.end: ; preds = %lower.block248 %index.next = add i32 %index, 4249 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8>250 %6 = icmp eq i32 %index.next, %n.vec251 br i1 %6, label %end, label %vector.body252 253end:254 ret void;255}256 257define arm_aapcs_vfpcc void @invariant_add(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n.vec) {258; CHECK-LABEL: @invariant_add(259; CHECK-NEXT: vector.ph:260; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]261; CHECK: vector.body:262; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]263; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]264; CHECK-NEXT: [[L0:%.*]] = mul <4 x i32> [[VEC_IND]], splat (i32 3)265; CHECK-NEXT: [[L1:%.*]] = add <4 x i32> [[L0]], [[VEC_IND]]266; CHECK-NEXT: [[TMP0:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0.v4i32(ptr [[DATA:%.*]], <4 x i32> [[L1]], i32 32, i32 2, i32 1)267; CHECK-NEXT: [[L3:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 [[INDEX]]268; CHECK-NEXT: store <4 x i32> [[TMP0]], ptr [[L3]], align 4269; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4270; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 8)271; CHECK-NEXT: [[L5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC:%.*]]272; CHECK-NEXT: br i1 [[L5]], label [[END:%.*]], label [[VECTOR_BODY]]273; CHECK: end:274; CHECK-NEXT: ret void275;276 277vector.ph:278 br label %vector.body279 280vector.body: ; preds = %vector.body, %vector.ph281 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]282 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body ]283 %l0 = mul <4 x i32> %vec.ind, <i32 3, i32 3, i32 3, i32 3>284 %l1 = add <4 x i32> %l0, %vec.ind285 %l2 = getelementptr inbounds i32, ptr %data, <4 x i32> %l1286 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %l2, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)287 %l3 = getelementptr inbounds i32, ptr %dst, i32 %index288 %l4 = bitcast ptr %l3 to ptr289 store <4 x i32> %wide.masked.gather, ptr %l4, align 4290 %index.next = add i32 %index, 4291 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8>292 %l5 = icmp eq i32 %index.next, %n.vec293 br i1 %l5, label %end, label %vector.body294 295end:296 ret void;297}298 299define void @gatherload(i32 %n, i32 %m, ptr nocapture %a, ptr nocapture readonly %b, i32 %call.us.us) {300; CHECK-LABEL: @gatherload(301; CHECK-NEXT: entry:302; CHECK-NEXT: [[CMP38:%.*]] = icmp sgt i32 [[N:%.*]], 0303; CHECK-NEXT: br i1 [[CMP38]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END16:%.*]]304; CHECK: for.body.lr.ph:305; CHECK-NEXT: [[CMP636:%.*]] = icmp sgt i32 [[M:%.*]], 0306; CHECK-NEXT: br i1 [[CMP636]], label [[FOR_BODY_US_US_PREHEADER:%.*]], label [[FOR_BODY:%.*]]307; CHECK: for.body.us.us.preheader:308; CHECK-NEXT: [[TMP0:%.*]] = shl nuw i32 [[M]], 2309; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i32, ptr [[A:%.*]], i32 [[M]]310; CHECK-NEXT: [[SCEVGEP64:%.*]] = getelementptr i32, ptr [[B:%.*]], i32 [[M]]311; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[M]], 4312; CHECK-NEXT: [[BOUND0:%.*]] = icmp ugt ptr [[SCEVGEP64]], [[A]]313; CHECK-NEXT: [[BOUND1:%.*]] = icmp ugt ptr [[SCEVGEP]], [[B]]314; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]315; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[M]], -4316; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N_VEC]], [[M]]317; CHECK-NEXT: br label [[FOR_BODY_US_US:%.*]]318; CHECK: for.body.us.us:319; CHECK-NEXT: [[I_039_US_US:%.*]] = phi i32 [ [[INC15_US_US:%.*]], [[FOR_COND5_FOR_END13_CRIT_EDGE_US_US:%.*]] ], [ 0, [[FOR_BODY_US_US_PREHEADER]] ]320; CHECK-NEXT: [[VLA_US_US:%.*]] = alloca i32, i32 [[CALL_US_US:%.*]], align 4321; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 4 [[VLA_US_US]], ptr align 4 [[A]], i32 [[TMP0]], i1 false)322; CHECK-NEXT: [[BRMERGE:%.*]] = select i1 [[MIN_ITERS_CHECK]], i1 true, i1 [[FOUND_CONFLICT]]323; CHECK-NEXT: br i1 [[BRMERGE]], label [[FOR_BODY7_US_US_PREHEADER:%.*]], label [[VECTOR_BODY:%.*]]324; CHECK: vector.body:325; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ], [ 0, [[FOR_BODY_US_US]] ]326; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[INDEX]]327; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4328; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0.v4i32(ptr [[VLA_US_US]], <4 x i32> [[WIDE_LOAD]], i32 32, i32 2, i32 1)329; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[INDEX]]330; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[TMP3]], align 4331; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4332; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]333; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]]334; CHECK: middle.block:335; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND5_FOR_END13_CRIT_EDGE_US_US]], label [[FOR_BODY7_US_US_PREHEADER]]336; CHECK: for.body7.us.us.preheader:337; CHECK-NEXT: [[J_137_US_US_PH:%.*]] = phi i32 [ 0, [[FOR_BODY_US_US]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ]338; CHECK-NEXT: br label [[FOR_BODY7_US_US:%.*]]339; CHECK: for.body7.us.us:340; CHECK-NEXT: [[J_137_US_US:%.*]] = phi i32 [ [[INC12_US_US:%.*]], [[FOR_BODY7_US_US]] ], [ [[J_137_US_US_PH]], [[FOR_BODY7_US_US_PREHEADER]] ]341; CHECK-NEXT: [[ARRAYIDX8_US_US:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[J_137_US_US]]342; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX8_US_US]], align 4343; CHECK-NEXT: [[ARRAYIDX9_US_US:%.*]] = getelementptr inbounds i32, ptr [[VLA_US_US]], i32 [[TMP5]]344; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[ARRAYIDX9_US_US]], align 4345; CHECK-NEXT: [[ARRAYIDX10_US_US:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[J_137_US_US]]346; CHECK-NEXT: store i32 [[TMP6]], ptr [[ARRAYIDX10_US_US]], align 4347; CHECK-NEXT: [[INC12_US_US]] = add nuw nsw i32 [[J_137_US_US]], 1348; CHECK-NEXT: [[EXITCOND58_NOT:%.*]] = icmp eq i32 [[INC12_US_US]], [[M]]349; CHECK-NEXT: br i1 [[EXITCOND58_NOT]], label [[FOR_COND5_FOR_END13_CRIT_EDGE_US_US]], label [[FOR_BODY7_US_US]]350; CHECK: for.cond5.for.end13_crit_edge.us.us:351; CHECK-NEXT: [[INC15_US_US]] = add nuw nsw i32 [[I_039_US_US]], 1352; CHECK-NEXT: [[EXITCOND59_NOT:%.*]] = icmp eq i32 [[INC15_US_US]], [[N]]353; CHECK-NEXT: br i1 [[EXITCOND59_NOT]], label [[FOR_END16]], label [[FOR_BODY_US_US]]354; CHECK: for.body:355; CHECK-NEXT: [[I_039:%.*]] = phi i32 [ [[INC15:%.*]], [[FOR_BODY]] ], [ 0, [[FOR_BODY_LR_PH]] ]356; CHECK-NEXT: [[INC15]] = add nuw nsw i32 [[I_039]], 1357; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC15]], [[N]]358; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END16]], label [[FOR_BODY]]359; CHECK: for.end16:360; CHECK-NEXT: ret void361;362entry:363 %a57 = bitcast ptr %a to ptr364 %cmp38 = icmp sgt i32 %n, 0365 br i1 %cmp38, label %for.body.lr.ph, label %for.end16366 367for.body.lr.ph: ; preds = %entry368 %cmp636 = icmp sgt i32 %m, 0369 br i1 %cmp636, label %for.body.us.us.preheader, label %for.body370 371for.body.us.us.preheader: ; preds = %for.body.lr.ph372 %0 = shl nuw i32 %m, 2373 %scevgep = getelementptr i32, ptr %a, i32 %m374 %scevgep64 = getelementptr i32, ptr %b, i32 %m375 %min.iters.check = icmp ult i32 %m, 4376 %bound0 = icmp ugt ptr %scevgep64, %a377 %bound1 = icmp ugt ptr %scevgep, %b378 %found.conflict = and i1 %bound0, %bound1379 %n.vec = and i32 %m, -4380 %cmp.n = icmp eq i32 %n.vec, %m381 br label %for.body.us.us382 383for.body.us.us: ; preds = %for.body.us.us.preheader, %for.cond5.for.end13_crit_edge.us.us384 %i.039.us.us = phi i32 [ %inc15.us.us, %for.cond5.for.end13_crit_edge.us.us ], [ 0, %for.body.us.us.preheader ]385 %1 = add i32 0, 0386 %vla.us.us = alloca i32, i32 %call.us.us, align 4387 %vla.us.us56 = bitcast ptr %vla.us.us to ptr388 call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 4 %vla.us.us56, ptr align 4 %a57, i32 %0, i1 false)389 %brmerge = select i1 %min.iters.check, i1 true, i1 %found.conflict390 br i1 %brmerge, label %for.body7.us.us.preheader, label %vector.body391 392vector.body: ; preds = %for.body.us.us, %vector.body393 %index = phi i32 [ %index.next, %vector.body ], [ 0, %for.body.us.us ]394 %2 = getelementptr inbounds i32, ptr %b, i32 %index395 %3 = bitcast ptr %2 to ptr396 %wide.load = load <4 x i32>, ptr %3, align 4397 %4 = getelementptr inbounds i32, ptr %vla.us.us, <4 x i32> %wide.load398 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %4, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)399 %5 = getelementptr inbounds i32, ptr %a, i32 %index400 %6 = bitcast ptr %5 to ptr401 store <4 x i32> %wide.masked.gather, ptr %6, align 4402 %index.next = add nuw i32 %index, 4403 %7 = icmp eq i32 %index.next, %n.vec404 br i1 %7, label %middle.block, label %vector.body405 406middle.block: ; preds = %vector.body407 br i1 %cmp.n, label %for.cond5.for.end13_crit_edge.us.us, label %for.body7.us.us.preheader408 409for.body7.us.us.preheader: ; preds = %for.body.us.us, %middle.block410 %j.137.us.us.ph = phi i32 [ 0, %for.body.us.us ], [ %n.vec, %middle.block ]411 br label %for.body7.us.us412 413for.body7.us.us: ; preds = %for.body7.us.us.preheader, %for.body7.us.us414 %j.137.us.us = phi i32 [ %inc12.us.us, %for.body7.us.us ], [ %j.137.us.us.ph, %for.body7.us.us.preheader ]415 %arrayidx8.us.us = getelementptr inbounds i32, ptr %b, i32 %j.137.us.us416 %8 = load i32, ptr %arrayidx8.us.us, align 4417 %arrayidx9.us.us = getelementptr inbounds i32, ptr %vla.us.us, i32 %8418 %9 = load i32, ptr %arrayidx9.us.us, align 4419 %arrayidx10.us.us = getelementptr inbounds i32, ptr %a, i32 %j.137.us.us420 store i32 %9, ptr %arrayidx10.us.us, align 4421 %inc12.us.us = add nuw nsw i32 %j.137.us.us, 1422 %exitcond58.not = icmp eq i32 %inc12.us.us, %m423 br i1 %exitcond58.not, label %for.cond5.for.end13_crit_edge.us.us, label %for.body7.us.us424 425for.cond5.for.end13_crit_edge.us.us: ; preds = %for.body7.us.us, %middle.block426 %inc15.us.us = add nuw nsw i32 %i.039.us.us, 1427 %exitcond59.not = icmp eq i32 %inc15.us.us, %n428 br i1 %exitcond59.not, label %for.end16, label %for.body.us.us429 430for.body: ; preds = %for.body.lr.ph, %for.body431 %i.039 = phi i32 [ %inc15, %for.body ], [ 0, %for.body.lr.ph ]432 %inc15 = add nuw nsw i32 %i.039, 1433 %exitcond.not = icmp eq i32 %inc15, %n434 br i1 %exitcond.not, label %for.end16, label %for.body435 436for.end16: ; preds = %for.body, %for.cond5.for.end13_crit_edge.us.us, %entry437 ret void438}439 440declare <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr>, i32, <4 x i1>, <4 x i32>)441declare void @llvm.memcpy.p0.p0.i32(ptr, ptr, i32, i1)442