405 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 52; RUN: opt -p loop-vectorize -S %s | FileCheck %s3 4target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"5target triple = "aarch64-unknown-linux"6 7; Test case from https://github.com/llvm/llvm-project/issues/148431.8define void @test_predicated_load_cast_hint(ptr %dst.1, ptr %dst.2, ptr %src, i8 %n, i64 %off) #0 {9; CHECK-LABEL: define void @test_predicated_load_cast_hint(10; CHECK-SAME: ptr [[DST_1:%.*]], ptr [[DST_2:%.*]], ptr [[SRC:%.*]], i8 [[N:%.*]], i64 [[OFF:%.*]]) #[[ATTR0:[0-9]+]] {11; CHECK-NEXT: [[ENTRY:.*:]]12; CHECK-NEXT: [[N_EXT:%.*]] = sext i8 [[N]] to i3213; CHECK-NEXT: [[N_SUB:%.*]] = add i32 [[N_EXT]], -1514; CHECK-NEXT: [[SMAX16:%.*]] = call i32 @llvm.smax.i32(i32 [[N_SUB]], i32 4)15; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[SMAX16]], -116; CHECK-NEXT: [[TMP1:%.*]] = lshr i32 [[TMP0]], 217; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i32 [[TMP1]], 118; CHECK-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]19; CHECK: [[VECTOR_SCEVCHECK]]:20; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N_SUB]], i32 4)21; CHECK-NEXT: [[TMP3:%.*]] = add nsw i32 [[SMAX]], -122; CHECK-NEXT: [[TMP4:%.*]] = lshr i32 [[TMP3]], 223; CHECK-NEXT: [[TMP5:%.*]] = trunc i32 [[TMP4]] to i824; CHECK-NEXT: [[MUL:%.*]] = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 4, i8 [[TMP5]])25; CHECK-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i8, i1 } [[MUL]], 026; CHECK-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i8, i1 } [[MUL]], 127; CHECK-NEXT: [[TMP6:%.*]] = add i8 4, [[MUL_RESULT]]28; CHECK-NEXT: [[TMP7:%.*]] = icmp ult i8 [[TMP6]], 429; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]30; CHECK-NEXT: [[TMP9:%.*]] = icmp ugt i32 [[TMP4]], 25531; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]32; CHECK-NEXT: [[TMP11:%.*]] = shl i64 [[OFF]], 333; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST_1]], i64 [[TMP11]]34; CHECK-NEXT: [[TMP12:%.*]] = zext i32 [[TMP4]] to i6435; CHECK-NEXT: [[MUL1:%.*]] = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 512, i64 [[TMP12]])36; CHECK-NEXT: [[MUL_RESULT2:%.*]] = extractvalue { i64, i1 } [[MUL1]], 037; CHECK-NEXT: [[MUL_OVERFLOW3:%.*]] = extractvalue { i64, i1 } [[MUL1]], 138; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[SCEVGEP]], i64 [[MUL_RESULT2]]39; CHECK-NEXT: [[TMP15:%.*]] = icmp ult ptr [[TMP14]], [[SCEVGEP]]40; CHECK-NEXT: [[TMP16:%.*]] = or i1 [[TMP15]], [[MUL_OVERFLOW3]]41; CHECK-NEXT: [[TMP17:%.*]] = or i1 [[TMP10]], [[TMP16]]42; CHECK-NEXT: br i1 [[TMP17]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]43; CHECK: [[VECTOR_MEMCHECK]]:44; CHECK-NEXT: [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[DST_2]], i64 145; CHECK-NEXT: [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 146; CHECK-NEXT: [[TMP18:%.*]] = shl i64 [[OFF]], 347; CHECK-NEXT: [[SCEVGEP6:%.*]] = getelementptr i8, ptr [[DST_1]], i64 [[TMP18]]48; CHECK-NEXT: [[SMAX7:%.*]] = call i32 @llvm.smax.i32(i32 [[N_SUB]], i32 4)49; CHECK-NEXT: [[TMP19:%.*]] = add nsw i32 [[SMAX7]], -150; CHECK-NEXT: [[TMP20:%.*]] = zext nneg i32 [[TMP19]] to i6451; CHECK-NEXT: [[TMP21:%.*]] = lshr i64 [[TMP20]], 252; CHECK-NEXT: [[TMP22:%.*]] = shl nuw nsw i64 [[TMP21]], 953; CHECK-NEXT: [[TMP23:%.*]] = add i64 [[TMP22]], [[TMP18]]54; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[TMP23]], 855; CHECK-NEXT: [[SCEVGEP8:%.*]] = getelementptr i8, ptr [[DST_1]], i64 [[TMP24]]56; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST_2]], [[SCEVGEP5]]57; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP4]]58; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]59; CHECK-NEXT: [[BOUND09:%.*]] = icmp ult ptr [[DST_2]], [[SCEVGEP8]]60; CHECK-NEXT: [[BOUND110:%.*]] = icmp ult ptr [[SCEVGEP6]], [[SCEVGEP4]]61; CHECK-NEXT: [[FOUND_CONFLICT11:%.*]] = and i1 [[BOUND09]], [[BOUND110]]62; CHECK-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT11]]63; CHECK-NEXT: [[BOUND012:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP8]]64; CHECK-NEXT: [[BOUND113:%.*]] = icmp ult ptr [[SCEVGEP6]], [[SCEVGEP5]]65; CHECK-NEXT: [[FOUND_CONFLICT14:%.*]] = and i1 [[BOUND012]], [[BOUND113]]66; CHECK-NEXT: [[CONFLICT_RDX15:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT14]]67; CHECK-NEXT: br i1 [[CONFLICT_RDX15]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]68; CHECK: [[VECTOR_PH]]:69; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 0, i32 [[TMP2]])70; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]71; CHECK: [[VECTOR_BODY]]:72; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE22:.*]] ]73; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE22]] ]74; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 4, i8 8, i8 12>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE22]] ]75; CHECK-NEXT: [[TMP28:%.*]] = load i8, ptr [[SRC]], align 1, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]76; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[TMP28]], i64 077; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer78; CHECK-NEXT: [[TMP25:%.*]] = zext <4 x i8> [[BROADCAST_SPLAT]] to <4 x i64>79; CHECK-NEXT: [[TMP26:%.*]] = zext <4 x i8> [[VEC_IND]] to <4 x i64>80; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK]], i32 081; CHECK-NEXT: br i1 [[TMP27]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]82; CHECK: [[PRED_STORE_IF]]:83; CHECK-NEXT: [[TMP102:%.*]] = extractelement <4 x i64> [[TMP26]], i32 084; CHECK-NEXT: [[TMP103:%.*]] = getelementptr [16 x i64], ptr [[DST_1]], i64 [[TMP102]], i64 [[OFF]]85; CHECK-NEXT: [[TMP104:%.*]] = extractelement <4 x i64> [[TMP25]], i32 086; CHECK-NEXT: [[TMP105:%.*]] = or i64 [[TMP104]], 187; CHECK-NEXT: store i64 [[TMP105]], ptr [[TMP103]], align 8, !alias.scope [[META3]]88; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]89; CHECK: [[PRED_STORE_CONTINUE]]:90; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK]], i32 191; CHECK-NEXT: br i1 [[TMP32]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18:.*]]92; CHECK: [[PRED_STORE_IF17]]:93; CHECK-NEXT: [[TMP108:%.*]] = extractelement <4 x i64> [[TMP26]], i32 194; CHECK-NEXT: [[TMP109:%.*]] = getelementptr [16 x i64], ptr [[DST_1]], i64 [[TMP108]], i64 [[OFF]]95; CHECK-NEXT: [[TMP110:%.*]] = extractelement <4 x i64> [[TMP25]], i32 196; CHECK-NEXT: [[TMP111:%.*]] = or i64 [[TMP110]], 197; CHECK-NEXT: store i64 [[TMP111]], ptr [[TMP109]], align 8, !alias.scope [[META3]]98; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE18]]99; CHECK: [[PRED_STORE_CONTINUE18]]:100; CHECK-NEXT: [[TMP37:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK]], i32 2101; CHECK-NEXT: br i1 [[TMP37]], label %[[PRED_STORE_IF19:.*]], label %[[PRED_STORE_CONTINUE20:.*]]102; CHECK: [[PRED_STORE_IF19]]:103; CHECK-NEXT: [[TMP114:%.*]] = extractelement <4 x i64> [[TMP26]], i32 2104; CHECK-NEXT: [[TMP115:%.*]] = getelementptr [16 x i64], ptr [[DST_1]], i64 [[TMP114]], i64 [[OFF]]105; CHECK-NEXT: [[TMP116:%.*]] = extractelement <4 x i64> [[TMP25]], i32 2106; CHECK-NEXT: [[TMP117:%.*]] = or i64 [[TMP116]], 1107; CHECK-NEXT: store i64 [[TMP117]], ptr [[TMP115]], align 8, !alias.scope [[META3]]108; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE20]]109; CHECK: [[PRED_STORE_CONTINUE20]]:110; CHECK-NEXT: [[TMP42:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK]], i32 3111; CHECK-NEXT: br i1 [[TMP42]], label %[[PRED_STORE_IF21:.*]], label %[[PRED_STORE_CONTINUE22]]112; CHECK: [[PRED_STORE_IF21]]:113; CHECK-NEXT: [[TMP120:%.*]] = extractelement <4 x i64> [[TMP26]], i32 3114; CHECK-NEXT: [[TMP121:%.*]] = getelementptr [16 x i64], ptr [[DST_1]], i64 [[TMP120]], i64 [[OFF]]115; CHECK-NEXT: [[TMP122:%.*]] = extractelement <4 x i64> [[TMP25]], i32 3116; CHECK-NEXT: [[TMP123:%.*]] = or i64 [[TMP122]], 1117; CHECK-NEXT: store i64 [[TMP123]], ptr [[TMP121]], align 8, !alias.scope [[META3]]118; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE22]]119; CHECK: [[PRED_STORE_CONTINUE22]]:120; CHECK-NEXT: store i8 0, ptr [[DST_2]], align 1, !alias.scope [[META5:![0-9]+]], !noalias [[META7:![0-9]+]]121; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4122; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX_NEXT]], i32 [[TMP2]])123; CHECK-NEXT: [[TMP47:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0124; CHECK-NEXT: [[TMP48:%.*]] = xor i1 [[TMP47]], true125; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 16)126; CHECK-NEXT: br i1 [[TMP48]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]127; CHECK: [[MIDDLE_BLOCK]]:128; CHECK-NEXT: br label %[[EXIT:.*]]129; CHECK: [[SCALAR_PH]]:130; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i8 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]131; CHECK-NEXT: br label %[[LOOP:.*]]132; CHECK: [[LOOP]]:133; CHECK-NEXT: [[IV:%.*]] = phi i8 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]134; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[SRC]], align 1135; CHECK-NEXT: [[L_EXT:%.*]] = zext i8 [[L]] to i64136; CHECK-NEXT: [[ADD:%.*]] = or i64 [[L_EXT]], 1137; CHECK-NEXT: [[IV_EXT:%.*]] = zext i8 [[IV]] to i64138; CHECK-NEXT: [[GEP_DST_1:%.*]] = getelementptr [16 x i64], ptr [[DST_1]], i64 [[IV_EXT]], i64 [[OFF]]139; CHECK-NEXT: store i64 [[ADD]], ptr [[GEP_DST_1]], align 8140; CHECK-NEXT: store i8 0, ptr [[DST_2]], align 1141; CHECK-NEXT: [[IV_NEXT]] = add i8 [[IV]], 4142; CHECK-NEXT: [[IV_NEXT_EXT:%.*]] = zext i8 [[IV_NEXT]] to i32143; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[N_SUB]], [[IV_NEXT_EXT]]144; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP12:![0-9]+]]145; CHECK: [[EXIT]]:146; CHECK-NEXT: ret void147;148entry:149 %n.ext = sext i8 %n to i32150 %n.sub = add i32 %n.ext, -15151 br label %loop152 153loop:154 %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop ]155 %l = load i8, ptr %src, align 1156 %l.ext = zext i8 %l to i64157 %add = or i64 %l.ext, 1158 %iv.ext = zext i8 %iv to i64159 %gep.dst.1 = getelementptr [16 x i64], ptr %dst.1, i64 %iv.ext, i64 %off160 store i64 %add, ptr %gep.dst.1, align 8161 store i8 0, ptr %dst.2, align 1162 %iv.next = add i8 %iv, 4163 %iv.next.ext = zext i8 %iv.next to i32164 %cmp = icmp sgt i32 %n.sub, %iv.next.ext165 br i1 %cmp, label %loop, label %exit, !llvm.loop !0166 167exit:168 ret void169}170 171; Check computing costs for sdiv/udiv with invariant divisor and tail folding.172; From https://github.com/llvm/llvm-project/issues/160354.173define void @srem_sdiv_with_tail_folding(i32 %d.0, i32 %d.1, ptr %dst, i32 %end) #0 {174; CHECK-LABEL: define void @srem_sdiv_with_tail_folding(175; CHECK-SAME: i32 [[D_0:%.*]], i32 [[D_1:%.*]], ptr [[DST:%.*]], i32 [[END:%.*]]) #[[ATTR0]] {176; CHECK-NEXT: [[ENTRY:.*]]:177; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]178; CHECK: [[LOOP_HEADER]]:179; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]180; CHECK-NEXT: [[IV_SUB:%.*]] = add nsw i32 [[IV]], -1181; CHECK-NEXT: [[REM:%.*]] = srem i32 [[IV_SUB]], [[D_0]]182; CHECK-NEXT: [[REM_1:%.*]] = add nsw i32 [[REM]], 1183; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[REM_1]], [[D_0]]184; CHECK-NEXT: br i1 [[C]], label %[[THEN:.*]], label %[[LOOP_LATCH]]185; CHECK: [[THEN]]:186; CHECK-NEXT: [[DIV:%.*]] = sdiv i32 [[IV_SUB]], [[D_1]]187; CHECK-NEXT: [[ADD_1:%.*]] = add i32 [[DIV]], 1188; CHECK-NEXT: [[ADD_1_EXT:%.*]] = sext i32 [[ADD_1]] to i64189; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr i32, ptr [[DST]], i64 [[ADD_1_EXT]]190; CHECK-NEXT: store i32 [[IV]], ptr [[GEP_DST]], align 4191; CHECK-NEXT: br label %[[LOOP_LATCH]]192; CHECK: [[LOOP_LATCH]]:193; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1194; CHECK-NEXT: [[EC:%.*]] = icmp ne i32 [[IV_NEXT]], [[END]]195; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT:.*]]196; CHECK: [[EXIT]]:197; CHECK-NEXT: ret void198;199entry:200 br label %loop.header201 202loop.header:203 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]204 %iv.sub = add nsw i32 %iv, -1205 %rem = srem i32 %iv.sub, %d.0206 %rem.1 = add nsw i32 %rem, 1207 %c = icmp eq i32 %rem.1, %d.0208 br i1 %c, label %then, label %loop.latch209 210then:211 %div = sdiv i32 %iv.sub, %d.1212 %add.1 = add i32 %div, 1213 %add.1.ext = sext i32 %add.1 to i64214 %gep.dst = getelementptr i32, ptr %dst, i64 %add.1.ext215 store i32 %iv, ptr %gep.dst, align 4216 br label %loop.latch217 218loop.latch:219 %iv.next = add nuw nsw i32 %iv, 1220 %ec = icmp ne i32 %iv.next, %end221 br i1 %ec, label %loop.header, label %exit222 223exit:224 ret void225}226 227; Check computing costs for predicated sdiv/udiv with invariant divisor without tail folding.228; From https://github.com/llvm/llvm-project/issues/160356.229define void @srem_sdiv_without_tail_folding(i32 %d.0, i32 %d.1, ptr %dst, i32 %end) #1 {230; CHECK-LABEL: define void @srem_sdiv_without_tail_folding(231; CHECK-SAME: i32 [[D_0:%.*]], i32 [[D_1:%.*]], ptr [[DST:%.*]], i32 [[END:%.*]]) #[[ATTR1:[0-9]+]] {232; CHECK-NEXT: [[ENTRY:.*]]:233; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[END]], 4234; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]235; CHECK: [[VECTOR_PH]]:236; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[END]], 4237; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[END]], [[N_MOD_VF]]238; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[D_0]], i64 0239; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer240; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]241; CHECK: [[VECTOR_BODY]]:242; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE12:.*]] ]243; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE12]] ]244; CHECK-NEXT: [[TMP0:%.*]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 -1)245; CHECK-NEXT: [[TMP1:%.*]] = srem <4 x i32> [[TMP0]], [[BROADCAST_SPLAT]]246; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], splat (i32 1)247; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i32> [[TMP2]], [[BROADCAST_SPLAT]]248; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP3]], i32 0249; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]250; CHECK: [[PRED_SDIV_IF]]:251; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[TMP0]], i32 0252; CHECK-NEXT: [[TMP6:%.*]] = sdiv i32 [[TMP5]], [[D_1]]253; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[TMP6]], i32 0254; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE]]255; CHECK: [[PRED_SDIV_CONTINUE]]:256; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP7]], %[[PRED_SDIV_IF]] ]257; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP3]], i32 1258; CHECK-NEXT: br i1 [[TMP9]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]259; CHECK: [[PRED_SDIV_IF1]]:260; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i32> [[TMP0]], i32 1261; CHECK-NEXT: [[TMP11:%.*]] = sdiv i32 [[TMP10]], [[D_1]]262; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP11]], i32 1263; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE2]]264; CHECK: [[PRED_SDIV_CONTINUE2]]:265; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x i32> [ [[TMP8]], %[[PRED_SDIV_CONTINUE]] ], [ [[TMP12]], %[[PRED_SDIV_IF1]] ]266; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP3]], i32 2267; CHECK-NEXT: br i1 [[TMP14]], label %[[PRED_SDIV_IF3:.*]], label %[[PRED_SDIV_CONTINUE4:.*]]268; CHECK: [[PRED_SDIV_IF3]]:269; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i32> [[TMP0]], i32 2270; CHECK-NEXT: [[TMP16:%.*]] = sdiv i32 [[TMP15]], [[D_1]]271; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP16]], i32 2272; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE4]]273; CHECK: [[PRED_SDIV_CONTINUE4]]:274; CHECK-NEXT: [[TMP18:%.*]] = phi <4 x i32> [ [[TMP13]], %[[PRED_SDIV_CONTINUE2]] ], [ [[TMP17]], %[[PRED_SDIV_IF3]] ]275; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP3]], i32 3276; CHECK-NEXT: br i1 [[TMP19]], label %[[PRED_SDIV_IF5:.*]], label %[[PRED_SDIV_CONTINUE6:.*]]277; CHECK: [[PRED_SDIV_IF5]]:278; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i32> [[TMP0]], i32 3279; CHECK-NEXT: [[TMP21:%.*]] = sdiv i32 [[TMP20]], [[D_1]]280; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP21]], i32 3281; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE6]]282; CHECK: [[PRED_SDIV_CONTINUE6]]:283; CHECK-NEXT: [[TMP23:%.*]] = phi <4 x i32> [ [[TMP18]], %[[PRED_SDIV_CONTINUE4]] ], [ [[TMP22]], %[[PRED_SDIV_IF5]] ]284; CHECK-NEXT: [[TMP24:%.*]] = add <4 x i32> [[TMP23]], splat (i32 1)285; CHECK-NEXT: [[TMP25:%.*]] = sext <4 x i32> [[TMP24]] to <4 x i64>286; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x i1> [[TMP3]], i32 0287; CHECK-NEXT: br i1 [[TMP26]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]288; CHECK: [[PRED_STORE_IF]]:289; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i64> [[TMP25]], i32 0290; CHECK-NEXT: [[TMP28:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP27]]291; CHECK-NEXT: [[TMP29:%.*]] = add i32 [[INDEX]], 0292; CHECK-NEXT: store i32 [[TMP29]], ptr [[TMP28]], align 4293; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]294; CHECK: [[PRED_STORE_CONTINUE]]:295; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i1> [[TMP3]], i32 1296; CHECK-NEXT: br i1 [[TMP30]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]297; CHECK: [[PRED_STORE_IF7]]:298; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i64> [[TMP25]], i32 1299; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP31]]300; CHECK-NEXT: [[TMP33:%.*]] = add i32 [[INDEX]], 1301; CHECK-NEXT: store i32 [[TMP33]], ptr [[TMP32]], align 4302; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE8]]303; CHECK: [[PRED_STORE_CONTINUE8]]:304; CHECK-NEXT: [[TMP34:%.*]] = extractelement <4 x i1> [[TMP3]], i32 2305; CHECK-NEXT: br i1 [[TMP34]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]306; CHECK: [[PRED_STORE_IF9]]:307; CHECK-NEXT: [[TMP35:%.*]] = extractelement <4 x i64> [[TMP25]], i32 2308; CHECK-NEXT: [[TMP36:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP35]]309; CHECK-NEXT: [[TMP37:%.*]] = add i32 [[INDEX]], 2310; CHECK-NEXT: store i32 [[TMP37]], ptr [[TMP36]], align 4311; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE10]]312; CHECK: [[PRED_STORE_CONTINUE10]]:313; CHECK-NEXT: [[TMP38:%.*]] = extractelement <4 x i1> [[TMP3]], i32 3314; CHECK-NEXT: br i1 [[TMP38]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12]]315; CHECK: [[PRED_STORE_IF11]]:316; CHECK-NEXT: [[TMP39:%.*]] = extractelement <4 x i64> [[TMP25]], i32 3317; CHECK-NEXT: [[TMP40:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP39]]318; CHECK-NEXT: [[TMP41:%.*]] = add i32 [[INDEX]], 3319; CHECK-NEXT: store i32 [[TMP41]], ptr [[TMP40]], align 4320; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE12]]321; CHECK: [[PRED_STORE_CONTINUE12]]:322; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4323; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 4)324; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]325; CHECK-NEXT: br i1 [[TMP42]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]326; CHECK: [[MIDDLE_BLOCK]]:327; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[END]], [[N_VEC]]328; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]329; CHECK: [[SCALAR_PH]]:330; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]331; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]332; CHECK: [[LOOP_HEADER]]:333; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]334; CHECK-NEXT: [[IV_SUB:%.*]] = add nsw i32 [[IV]], -1335; CHECK-NEXT: [[REM:%.*]] = srem i32 [[IV_SUB]], [[D_0]]336; CHECK-NEXT: [[REM_1:%.*]] = add nsw i32 [[REM]], 1337; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[REM_1]], [[D_0]]338; CHECK-NEXT: br i1 [[C]], label %[[THEN:.*]], label %[[LOOP_LATCH]]339; CHECK: [[THEN]]:340; CHECK-NEXT: [[DIV:%.*]] = sdiv i32 [[IV_SUB]], [[D_1]]341; CHECK-NEXT: [[ADD_1:%.*]] = add i32 [[DIV]], 1342; CHECK-NEXT: [[ADD_1_EXT:%.*]] = sext i32 [[ADD_1]] to i64343; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr i32, ptr [[DST]], i64 [[ADD_1_EXT]]344; CHECK-NEXT: store i32 [[IV]], ptr [[GEP_DST]], align 4345; CHECK-NEXT: br label %[[LOOP_LATCH]]346; CHECK: [[LOOP_LATCH]]:347; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1348; CHECK-NEXT: [[EC:%.*]] = icmp ne i32 [[IV_NEXT]], [[END]]349; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT]], !llvm.loop [[LOOP14:![0-9]+]]350; CHECK: [[EXIT]]:351; CHECK-NEXT: ret void352;353entry:354 br label %loop.header355 356loop.header:357 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]358 %iv.sub = add nsw i32 %iv, -1359 %rem = srem i32 %iv.sub, %d.0360 %rem.1 = add nsw i32 %rem, 1361 %c = icmp eq i32 %rem.1, %d.0362 br i1 %c, label %then, label %loop.latch363 364then:365 %div = sdiv i32 %iv.sub, %d.1366 %add.1 = add i32 %div, 1367 %add.1.ext = sext i32 %add.1 to i64368 %gep.dst = getelementptr i32, ptr %dst, i64 %add.1.ext369 store i32 %iv, ptr %gep.dst, align 4370 br label %loop.latch371 372loop.latch:373 %iv.next = add nuw nsw i32 %iv, 1374 %ec = icmp ne i32 %iv.next, %end375 br i1 %ec, label %loop.header, label %exit376 377exit:378 ret void379}380 381attributes #0 = { "target-cpu"="neoverse-v1" }382attributes #1 = { "target-cpu"="neoverse-v2" }383 384!0 = distinct !{!0, !1, !2, !3}385!1 = !{!"llvm.loop.mustprogress"}386!2 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}387!3 = !{!"llvm.loop.vectorize.enable", i1 true}388;.389; CHECK: [[META0]] = !{[[META1:![0-9]+]]}390; CHECK: [[META1]] = distinct !{[[META1]], [[META2:![0-9]+]]}391; CHECK: [[META2]] = distinct !{[[META2]], !"LVerDomain"}392; CHECK: [[META3]] = !{[[META4:![0-9]+]]}393; CHECK: [[META4]] = distinct !{[[META4]], [[META2]]}394; CHECK: [[META5]] = !{[[META6:![0-9]+]]}395; CHECK: [[META6]] = distinct !{[[META6]], [[META2]]}396; CHECK: [[META7]] = !{[[META1]], [[META4]]}397; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META9:![0-9]+]], [[META10:![0-9]+]], [[META11:![0-9]+]]}398; CHECK: [[META9]] = !{!"llvm.loop.mustprogress"}399; CHECK: [[META10]] = !{!"llvm.loop.isvectorized", i32 1}400; CHECK: [[META11]] = !{!"llvm.loop.unroll.runtime.disable"}401; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META9]], [[META10]]}402; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META10]], [[META11]]}403; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META11]], [[META10]]}404;.405