543 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 42; RUN: opt -S < %s -p loop-vectorize | FileCheck %s --check-prefixes=CHECK3 4target triple = "aarch64-unknown-linux-gnu"5 6declare void @init_mem(ptr, i64);7 8define i64 @same_exit_block_pre_inc_use1() #1 {9; CHECK-LABEL: define i64 @same_exit_block_pre_inc_use1(10; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {11; CHECK-NEXT: entry:12; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 413; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 414; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)15; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)16; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()17; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 418; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 64, [[TMP1]]19; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]20; CHECK: vector.ph:21; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()22; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 1623; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 64, [[TMP3]]24; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]25; CHECK-NEXT: [[TMP6:%.*]] = add i64 3, [[N_VEC]]26; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]27; CHECK: vector.body:28; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]29; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]30; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[OFFSET_IDX]]31; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP11]], align 132; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]33; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP13]], align 134; CHECK-NEXT: [[TMP16:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]35; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], [[TMP3]]36; CHECK-NEXT: [[TMP8:%.*]] = freeze <vscale x 16 x i1> [[TMP16]]37; CHECK-NEXT: [[TMP17:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[TMP8]])38; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT3]], [[N_VEC]]39; CHECK-NEXT: [[TMP19:%.*]] = or i1 [[TMP17]], [[TMP18]]40; CHECK-NEXT: br i1 [[TMP19]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]41; CHECK: middle.split:42; CHECK-NEXT: br i1 [[TMP17]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]43; CHECK: middle.block:44; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 64, [[N_VEC]]45; CHECK-NEXT: br i1 [[CMP_N]], label [[LOOP_END:%.*]], label [[SCALAR_PH]]46; CHECK: vector.early.exit:47; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1(<vscale x 16 x i1> [[TMP16]], i1 false)48; CHECK-NEXT: [[TMP20:%.*]] = add i64 [[INDEX1]], [[FIRST_ACTIVE_LANE]]49; CHECK-NEXT: [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP20]]50; CHECK-NEXT: br label [[LOOP_END]]51; CHECK: scalar.ph:52; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]53; CHECK-NEXT: br label [[LOOP:%.*]]54; CHECK: loop:55; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]56; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]57; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 158; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]59; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 160; CHECK-NEXT: [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]61; CHECK-NEXT: br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END]]62; CHECK: loop.inc:63; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 164; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 6765; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP3:![0-9]+]]66; CHECK: loop.end:67; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]68; CHECK-NEXT: ret i64 [[RETVAL]]69;70entry:71 %p1 = alloca [1024 x i8]72 %p2 = alloca [1024 x i8]73 call void @init_mem(ptr %p1, i64 1024)74 call void @init_mem(ptr %p2, i64 1024)75 br label %loop76 77loop:78 %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]79 %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index80 %ld1 = load i8, ptr %arrayidx, align 181 %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index82 %ld2 = load i8, ptr %arrayidx1, align 183 %cmp3 = icmp eq i8 %ld1, %ld284 br i1 %cmp3, label %loop.inc, label %loop.end85 86loop.inc:87 %index.next = add i64 %index, 188 %exitcond = icmp ne i64 %index.next, 6789 br i1 %exitcond, label %loop, label %loop.end90 91loop.end:92 %retval = phi i64 [ %index, %loop ], [ 67, %loop.inc ]93 ret i64 %retval94}95 96 97; In this example the early exit block appears in the list of ExitNotTaken98; SCEVs, but is not computable.99define i64 @same_exit_block_pre_inc_use4() {100; CHECK-LABEL: define i64 @same_exit_block_pre_inc_use4() {101; CHECK-NEXT: entry:102; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i64], align 8103; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i64], align 8104; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)105; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)106; CHECK-NEXT: br label [[VECTOR_PH:%.*]]107; CHECK: vector.ph:108; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]109; CHECK: vector.body:110; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VECTOR_BODY]] ]111; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 3, i64 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]112; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]113; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[P1]], i64 [[OFFSET_IDX]]114; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP1]], align 1115; CHECK-NEXT: [[TMP4:%.*]] = icmp uge <2 x i64> [[VEC_IND]], [[WIDE_LOAD]]116; CHECK-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 2117; CHECK-NEXT: [[TMP2:%.*]] = freeze <2 x i1> [[TMP4]]118; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> [[TMP2]])119; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 64120; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]121; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)122; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]123; CHECK: middle.split:124; CHECK-NEXT: br i1 [[TMP5]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]125; CHECK: middle.block:126; CHECK-NEXT: br label [[LOOP_END:%.*]]127; CHECK: vector.early.exit:128; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> [[TMP4]], i1 false)129; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX1]], [[FIRST_ACTIVE_LANE]]130; CHECK-NEXT: [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP8]]131; CHECK-NEXT: br label [[LOOP_END]]132; CHECK: loop.end:133; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]134; CHECK-NEXT: ret i64 [[RETVAL]]135;136entry:137 %p1 = alloca [1024 x i64]138 %p2 = alloca [1024 x i64]139 call void @init_mem(ptr %p1, i64 1024)140 call void @init_mem(ptr %p2, i64 1024)141 br label %loop142 143loop:144 %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]145 %arrayidx = getelementptr inbounds i64, ptr %p1, i64 %index146 %ld1 = load i64, ptr %arrayidx, align 1147 %cmp3 = icmp ult i64 %index, %ld1148 br i1 %cmp3, label %loop.inc, label %loop.end149 150loop.inc:151 %index.next = add i64 %index, 1152 %exitcond = icmp ne i64 %index.next, 67153 br i1 %exitcond, label %loop, label %loop.end154 155loop.end:156 %retval = phi i64 [ %index, %loop ], [ 67, %loop.inc ]157 ret i64 %retval158}159 160 161define i64 @loop_contains_safe_call() #1 {162; CHECK-LABEL: define i64 @loop_contains_safe_call(163; CHECK-SAME: ) #[[ATTR0]] {164; CHECK-NEXT: entry:165; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 4166; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 4167; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)168; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)169; CHECK-NEXT: br label [[VECTOR_PH:%.*]]170; CHECK: vector.ph:171; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]172; CHECK: vector.body:173; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VECTOR_BODY]] ]174; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]175; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[P1]], i64 [[OFFSET_IDX]]176; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 1177; CHECK-NEXT: [[TMP3:%.*]] = call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> [[WIDE_LOAD]])178; CHECK-NEXT: [[TMP5:%.*]] = fcmp fast oge <4 x float> [[TMP3]], splat (float 3.000000e+00)179; CHECK-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 4180; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP5]]181; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])182; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 64183; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]184; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]185; CHECK: middle.split:186; CHECK-NEXT: br i1 [[TMP6]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]187; CHECK: middle.block:188; CHECK-NEXT: br label [[LOOP_END:%.*]]189; CHECK: vector.early.exit:190; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)191; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX1]], [[FIRST_ACTIVE_LANE]]192; CHECK-NEXT: [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP9]]193; CHECK-NEXT: br label [[LOOP_END]]194; CHECK: loop.end:195; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]196; CHECK-NEXT: ret i64 [[RETVAL]]197;198entry:199 %p1 = alloca [1024 x i8]200 %p2 = alloca [1024 x i8]201 call void @init_mem(ptr %p1, i64 1024)202 call void @init_mem(ptr %p2, i64 1024)203 br label %loop204 205loop:206 %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]207 %arrayidx = getelementptr inbounds float, ptr %p1, i64 %index208 %ld1 = load float, ptr %arrayidx, align 1209 %sqrt = tail call fast float @llvm.sqrt.f32(float %ld1)210 %cmp = fcmp fast ult float %sqrt, 3.0e+00211 br i1 %cmp, label %loop.inc, label %loop.end212 213loop.inc:214 %index.next = add i64 %index, 1215 %exitcond = icmp ne i64 %index.next, 67216 br i1 %exitcond, label %loop, label %loop.end217 218loop.end:219 %retval = phi i64 [ %index, %loop ], [ 67, %loop.inc ]220 ret i64 %retval221}222 223 224define i64 @loop_contains_safe_div() #1 {225; CHECK-LABEL: define i64 @loop_contains_safe_div(226; CHECK-SAME: ) #[[ATTR0]] {227; CHECK-NEXT: entry:228; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 4229; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 4230; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)231; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)232; CHECK-NEXT: br label [[VECTOR_PH:%.*]]233; CHECK: vector.ph:234; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64()235; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP10]], 4236; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 64, [[TMP3]]237; CHECK-NEXT: [[INDEX1:%.*]] = sub i64 64, [[N_MOD_VF]]238; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]239; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]240; CHECK: vector.body:241; CHECK-NEXT: [[INDEX2:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VECTOR_BODY]] ]242; CHECK-NEXT: [[OFFSET_IDX1:%.*]] = add i64 3, [[INDEX2]]243; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[OFFSET_IDX1]]244; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP1]], align 1245; CHECK-NEXT: [[TMP13:%.*]] = udiv <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 20000)246; CHECK-NEXT: [[TMP15:%.*]] = icmp ne <vscale x 4 x i32> [[TMP13]], splat (i32 1)247; CHECK-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX2]], [[TMP3]]248; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 4 x i1> [[TMP15]]249; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP9]])250; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[INDEX1]]251; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]252; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]253; CHECK: middle.split:254; CHECK-NEXT: br i1 [[TMP6]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]255; CHECK: middle.block:256; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 64, [[INDEX1]]257; CHECK-NEXT: br i1 [[CMP_N]], label [[LOOP_END:%.*]], label [[SCALAR_PH:%.*]]258; CHECK: vector.early.exit:259; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv4i1(<vscale x 4 x i1> [[TMP15]], i1 false)260; CHECK-NEXT: [[TMP16:%.*]] = add i64 [[INDEX2]], [[FIRST_ACTIVE_LANE]]261; CHECK-NEXT: [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP16]]262; CHECK-NEXT: br label [[LOOP_END]]263; CHECK: scalar.ph:264; CHECK-NEXT: br label [[LOOP:%.*]]265; CHECK: loop:266; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[OFFSET_IDX]], [[SCALAR_PH]] ]267; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[INDEX]]268; CHECK-NEXT: [[LD1:%.*]] = load i32, ptr [[ARRAYIDX]], align 1269; CHECK-NEXT: [[DIV:%.*]] = udiv i32 [[LD1]], 20000270; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[DIV]], 1271; CHECK-NEXT: br i1 [[CMP]], label [[LOOP_INC]], label [[LOOP_END]]272; CHECK: loop.inc:273; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1274; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67275; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP7:![0-9]+]]276; CHECK: loop.end:277; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]278; CHECK-NEXT: ret i64 [[RETVAL]]279;280entry:281 %p1 = alloca [1024 x i8]282 %p2 = alloca [1024 x i8]283 call void @init_mem(ptr %p1, i64 1024)284 call void @init_mem(ptr %p2, i64 1024)285 br label %loop286 287loop:288 %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]289 %arrayidx = getelementptr inbounds i32, ptr %p1, i64 %index290 %ld1 = load i32, ptr %arrayidx, align 1291 %div = udiv i32 %ld1, 20000292 %cmp = icmp eq i32 %div, 1293 br i1 %cmp, label %loop.inc, label %loop.end294 295loop.inc:296 %index.next = add i64 %index, 1297 %exitcond = icmp ne i64 %index.next, 67298 br i1 %exitcond, label %loop, label %loop.end299 300loop.end:301 %retval = phi i64 [ %index, %loop ], [ 67, %loop.inc ]302 ret i64 %retval303}304 305 306define i64 @loop_contains_load_after_early_exit(ptr dereferenceable(1024) align(8) %p2) {307; DEBUG-LABEL: LV: Checking a loop in 'loop_contains_load_after_early_exit'308; DEBUG: LV: Found an early exit loop with symbolic max backedge taken count: 63309; DEBUG-NEXT: LV: We can vectorize this loop!310; DEBUG-NEXT: LV: Not vectorizing: Auto-vectorization of loops with uncountable early exit is not yet supported.311; CHECK-LABEL: define i64 @loop_contains_load_after_early_exit(312; CHECK-SAME: ptr align 8 dereferenceable(1024) [[P2:%.*]]) {313; CHECK-NEXT: entry:314; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 4315; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)316; CHECK-NEXT: br label [[VECTOR_PH:%.*]]317; CHECK: vector.ph:318; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]319; CHECK: vector.body:320; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ]321; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]322; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[OFFSET_IDX]]323; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 1324; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], splat (i32 1)325; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P2]], i64 [[OFFSET_IDX]]326; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP4]], align 8327; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4328; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP6]]329; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]])330; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64331; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]332; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]333; CHECK: middle.split:334; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[WIDE_LOAD2]], i32 3335; CHECK-NEXT: br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]336; CHECK: middle.block:337; CHECK-NEXT: br label [[LOOP_END:%.*]]338; CHECK: vector.early.exit:339; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)340; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX1]], [[FIRST_ACTIVE_LANE]]341; CHECK-NEXT: [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP11]]342; CHECK-NEXT: br label [[LOOP_END]]343; CHECK: loop.end:344; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]345; CHECK-NEXT: ret i64 [[RETVAL]]346;347entry:348 %p1 = alloca [1024 x i8]349 call void @init_mem(ptr %p1, i64 1024)350 br label %loop351 352loop:353 %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]354 %arrayidx = getelementptr inbounds i32, ptr %p1, i64 %index355 %ld1 = load i32, ptr %arrayidx, align 1356 %cmp = icmp eq i32 %ld1, 1357 br i1 %cmp, label %loop.inc, label %loop.end358 359loop.inc:360 %arrayidx2 = getelementptr inbounds i64, ptr %p2, i64 %index361 %ld2 = load i64, ptr %arrayidx2, align 8362 %index.next = add i64 %index, 1363 %exitcond = icmp ne i64 %index.next, 67364 br i1 %exitcond, label %loop, label %loop.end365 366loop.end:367 %retval = phi i64 [ %index, %loop ], [ %ld2, %loop.inc ]368 ret i64 %retval369}370 371 372; The form of the induction variables requires SCEV predicates.373define i32 @diff_exit_block_needs_scev_check(i32 %end) {374; DEBUG-LABEL: LV: Checking a loop in 'diff_exit_block_needs_scev_check'375; DEBUG: Found an early exit loop with symbolic max backedge taken count: (-1 + (1 umax (zext i10 (trunc i32 %end to i10) to i32)))<nsw>376; DEBUG-NEXT: LV: We can vectorize this loop!377; DEBUG-NEXT: LV: Not vectorizing: Auto-vectorization of loops with uncountable early exit is not yet supported.378; CHECK-LABEL: define i32 @diff_exit_block_needs_scev_check(379; CHECK-SAME: i32 [[END:%.*]]) {380; CHECK-NEXT: entry:381; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i32], align 4382; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i32], align 4383; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)384; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)385; CHECK-NEXT: [[END_CLAMPED:%.*]] = and i32 [[END]], 1023386; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[END]] to i10387; CHECK-NEXT: [[TMP1:%.*]] = zext i10 [[TMP0]] to i64388; CHECK-NEXT: [[UMAX1:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 1)389; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX1]], 12390; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]391; CHECK: vector.scevcheck:392; CHECK-NEXT: [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[END_CLAMPED]], i32 1)393; CHECK-NEXT: [[TMP2:%.*]] = add nsw i32 [[UMAX]], -1394; CHECK-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i8395; CHECK-NEXT: [[TMP4:%.*]] = add i8 1, [[TMP3]]396; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i8 [[TMP4]], 1397; CHECK-NEXT: [[TMP6:%.*]] = icmp ugt i32 [[TMP2]], 255398; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]]399; CHECK-NEXT: br i1 [[TMP7]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]400; CHECK: vector.ph:401; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX1]], 4402; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX1]], [[N_MOD_VF]]403; CHECK-NEXT: [[TMP8:%.*]] = trunc i64 [[N_VEC]] to i8404; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]405; CHECK: vector.body:406; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]407; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[INDEX]]408; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP10]], align 4409; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[P2]], i64 [[INDEX]]410; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP12]], align 4411; CHECK-NEXT: [[TMP14:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD2]]412; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4413; CHECK-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[TMP14]]414; CHECK-NEXT: [[TMP15:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP13]])415; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]416; CHECK-NEXT: [[TMP17:%.*]] = or i1 [[TMP15]], [[TMP16]]417; CHECK-NEXT: br i1 [[TMP17]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]418; CHECK: middle.split:419; CHECK-NEXT: br i1 [[TMP15]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]420; CHECK: middle.block:421; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[UMAX1]], [[N_VEC]]422; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]423; CHECK: vector.early.exit:424; CHECK-NEXT: br label [[FOUND:%.*]]425; CHECK: scalar.ph:426; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i8 [ [[TMP8]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]427; CHECK-NEXT: [[BC_RESUME_VAL3:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]428; CHECK-NEXT: br label [[FOR_BODY:%.*]]429; CHECK: for.body:430; CHECK-NEXT: [[IND:%.*]] = phi i8 [ [[IND_NEXT:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]431; CHECK-NEXT: [[GEP_IND:%.*]] = phi i64 [ [[GEP_IND_NEXT:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ]432; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[GEP_IND]]433; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4434; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[P2]], i64 [[GEP_IND]]435; CHECK-NEXT: [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4436; CHECK-NEXT: [[CMP_EARLY:%.*]] = icmp eq i32 [[TMP18]], [[TMP19]]437; CHECK-NEXT: br i1 [[CMP_EARLY]], label [[FOUND]], label [[FOR_INC]]438; CHECK: for.inc:439; CHECK-NEXT: [[IND_NEXT]] = add i8 [[IND]], 1440; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[IND_NEXT]] to i32441; CHECK-NEXT: [[GEP_IND_NEXT]] = add i64 [[GEP_IND]], 1442; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[CONV]], [[END_CLAMPED]]443; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY]], label [[EXIT]], !llvm.loop [[LOOP10:![0-9]+]]444; CHECK: found:445; CHECK-NEXT: ret i32 1446; CHECK: exit:447; CHECK-NEXT: ret i32 0448;449entry:450 %p1 = alloca [1024 x i32]451 %p2 = alloca [1024 x i32]452 call void @init_mem(ptr %p1, i64 1024)453 call void @init_mem(ptr %p2, i64 1024)454 %end.clamped = and i32 %end, 1023455 br label %for.body456 457for.body:458 %ind = phi i8 [ %ind.next, %for.inc ], [ 0, %entry ]459 %gep.ind = phi i64 [ %gep.ind.next, %for.inc ], [ 0, %entry ]460 %arrayidx1 = getelementptr inbounds i32, ptr %p1, i64 %gep.ind461 %0 = load i32, ptr %arrayidx1, align 4462 %arrayidx2 = getelementptr inbounds i32, ptr %p2, i64 %gep.ind463 %1 = load i32, ptr %arrayidx2, align 4464 %cmp.early = icmp eq i32 %0, %1465 br i1 %cmp.early, label %found, label %for.inc466 467for.inc:468 %ind.next = add i8 %ind, 1469 %conv = zext i8 %ind.next to i32470 %gep.ind.next = add i64 %gep.ind, 1471 %cmp = icmp ult i32 %conv, %end.clamped472 br i1 %cmp, label %for.body, label %exit473 474found:475 ret i32 1476 477exit:478 ret i32 0479}480 481%my.struct = type { i8, i8 }482 483define i64 @same_exit_block_requires_interleaving() {484; CHECK-LABEL: define i64 @same_exit_block_requires_interleaving() {485; CHECK-NEXT: entry:486; CHECK-NEXT: [[P1:%.*]] = alloca [128 x [[MY_STRUCT:%.*]]], align 8487; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 256)488; CHECK-NEXT: br label [[LOOP:%.*]]489; CHECK: loop:490; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ 3, [[ENTRY:%.*]] ]491; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [128 x [[MY_STRUCT]]], ptr [[P1]], i64 0, i64 [[INDEX]]492; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1493; CHECK-NEXT: [[CMP3:%.*]] = icmp eq i8 [[LD1]], 3494; CHECK-NEXT: br i1 [[CMP3]], label [[LOOP_LATCH]], label [[LOOP_END:%.*]]495; CHECK: loop.latch:496; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1497; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 69498; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]]499; CHECK: loop.end:500; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ 0, [[LOOP_LATCH]] ], [ 1, [[LOOP]] ]501; CHECK-NEXT: ret i64 [[RETVAL]]502;503entry:504 %p1 = alloca [128 x %my.struct]505 call void @init_mem(ptr %p1, i64 256)506 br label %loop507 508loop:509 %index = phi i64 [ %index.next, %loop.latch ], [ 3, %entry ]510 %arrayidx = getelementptr inbounds [128 x %my.struct], ptr %p1, i64 0, i64 %index511 %ld1 = load i8, ptr %arrayidx, align 1512 %cmp3 = icmp eq i8 %ld1, 3513 br i1 %cmp3, label %loop.latch, label %loop.end514 515loop.latch:516 %index.next = add i64 %index, 1517 %exitcond = icmp ne i64 %index.next, 69518 br i1 %exitcond, label %loop, label %loop.end519 520loop.end:521 %retval = phi i64 [ 0, %loop.latch ], [ 1, %loop ]522 ret i64 %retval523}524 525declare i32 @foo(i32) readonly526declare <vscale x 4 x i32> @foo_vec(<vscale x 4 x i32>)527 528attributes #0 = { "vector-function-abi-variant"="_ZGVsNxv_foo(foo_vec)" }529attributes #1 = { "target-features"="+sve" vscale_range(1,16) }530;.531; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}532; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}533; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}534; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}535; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}536; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}537; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}538; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}539; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}540; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]}541; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]]}542;.543