181 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -S -loop-predication-skip-profitability-checks=false -passes='require<scalar-evolution>,loop-mssa(loop-predication)' -verify-memoryssa < %s 2>&1 | FileCheck %s3 4; latch block exits to a speculation block. We account for this since deopt is5; very rarely taken. So we do not predicate this loop using that coarse latch6; check.7; LatchExitProbability: 0x04000000 / 0x80000000 = 3.12%8; ExitingBlockProbability: 0x7ffa572a / 0x80000000 = 99.98%9define i64 @donot_predicate(ptr nocapture readonly %arg, i32 %length, ptr nocapture readonly %arg2, ptr nocapture readonly %n_addr, i64 %i) !prof !21 {10; CHECK-LABEL: @donot_predicate(11; CHECK-NEXT: entry:12; CHECK-NEXT: [[LENGTH_EXT:%.*]] = zext i32 [[LENGTH:%.*]] to i6413; CHECK-NEXT: [[N_PRE:%.*]] = load i64, ptr [[N_ADDR:%.*]], align 414; CHECK-NEXT: br label [[HEADER:%.*]]15; CHECK: Header:16; CHECK-NEXT: [[RESULT_IN3:%.*]] = phi ptr [ [[ARG2:%.*]], [[ENTRY:%.*]] ], [ [[ARG:%.*]], [[LATCH:%.*]] ]17; CHECK-NEXT: [[J2:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[J_NEXT:%.*]], [[LATCH]] ]18; CHECK-NEXT: [[WITHIN_BOUNDS:%.*]] = icmp ult i64 [[J2]], [[LENGTH_EXT]]19; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 [[WITHIN_BOUNDS]], i32 9) [ "deopt"() ]20; CHECK-NEXT: [[INNERCMP:%.*]] = icmp eq i64 [[J2]], [[N_PRE]]21; CHECK-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J2]], 122; CHECK-NEXT: br i1 [[INNERCMP]], label [[LATCH]], label [[EXIT:%.*]], !prof [[PROF1:![0-9]+]]23; CHECK: Latch:24; CHECK-NEXT: [[SPECULATE_TRIP_COUNT:%.*]] = icmp ult i64 [[J_NEXT]], 104857625; CHECK-NEXT: br i1 [[SPECULATE_TRIP_COUNT]], label [[HEADER]], label [[DEOPT:%.*]]26; CHECK: deopt:27; CHECK-NEXT: [[COUNTED_SPECULATION_FAILED:%.*]] = call i64 (...) @llvm.experimental.deoptimize.i64(i64 30) [ "deopt"(i32 0) ]28; CHECK-NEXT: ret i64 [[COUNTED_SPECULATION_FAILED]]29; CHECK: exit:30; CHECK-NEXT: [[RESULT_IN3_LCSSA:%.*]] = phi ptr [ [[RESULT_IN3]], [[HEADER]] ]31; CHECK-NEXT: [[RESULT_LE:%.*]] = load i64, ptr [[RESULT_IN3_LCSSA]], align 832; CHECK-NEXT: ret i64 [[RESULT_LE]]33;34entry:35 %length.ext = zext i32 %length to i6436 %n.pre = load i64, ptr %n_addr, align 437 br label %Header38 39Header: ; preds = %entry, %Latch40 %result.in3 = phi ptr [ %arg2, %entry ], [ %arg, %Latch ]41 %j2 = phi i64 [ 0, %entry ], [ %j.next, %Latch ]42 %within.bounds = icmp ult i64 %j2, %length.ext43 call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]44 %innercmp = icmp eq i64 %j2, %n.pre45 %j.next = add nuw nsw i64 %j2, 146 br i1 %innercmp, label %Latch, label %exit, !prof !047 48Latch: ; preds = %Header49 %speculate_trip_count = icmp ult i64 %j.next, 104857650 br i1 %speculate_trip_count, label %Header, label %deopt51 52deopt: ; preds = %Latch53 %counted_speculation_failed = call i64 (...) @llvm.experimental.deoptimize.i64(i64 30) [ "deopt"(i32 0) ]54 ret i64 %counted_speculation_failed55 56exit: ; preds = %Header57 %result.in3.lcssa = phi ptr [ %result.in3, %Header ]58 %result.le = load i64, ptr %result.in3.lcssa, align 859 ret i64 %result.le60}61!0 = !{!"branch_weights", i32 18, i32 104200}62 63; predicate loop since there's no profile information and BPI concluded all64; exiting blocks have same probability of exiting from loop.65define i64 @predicate(ptr nocapture readonly %arg, i32 %length, ptr nocapture readonly %arg2, ptr nocapture readonly %n_addr, i64 %i) !prof !21 {66; CHECK-LABEL: @predicate(67; CHECK-NEXT: entry:68; CHECK-NEXT: [[LENGTH_EXT:%.*]] = zext i32 [[LENGTH:%.*]] to i6469; CHECK-NEXT: [[N_PRE:%.*]] = load i64, ptr [[N_ADDR:%.*]], align 470; CHECK-NEXT: [[TMP0:%.*]] = icmp ule i64 1048576, [[LENGTH_EXT]]71; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i64 0, [[LENGTH_EXT]]72; CHECK-NEXT: [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]73; CHECK-NEXT: [[TMP3:%.*]] = freeze i1 [[TMP2]]74; CHECK-NEXT: br label [[HEADER:%.*]]75; CHECK: Header:76; CHECK-NEXT: [[RESULT_IN3:%.*]] = phi ptr [ [[ARG2:%.*]], [[ENTRY:%.*]] ], [ [[ARG:%.*]], [[LATCH:%.*]] ]77; CHECK-NEXT: [[J2:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[J_NEXT:%.*]], [[LATCH]] ]78; CHECK-NEXT: [[WITHIN_BOUNDS:%.*]] = icmp ult i64 [[J2]], [[LENGTH_EXT]]79; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 [[TMP3]], i32 9) [ "deopt"() ]80; CHECK-NEXT: call void @llvm.assume(i1 [[WITHIN_BOUNDS]])81; CHECK-NEXT: [[INNERCMP:%.*]] = icmp eq i64 [[J2]], [[N_PRE]]82; CHECK-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J2]], 183; CHECK-NEXT: br i1 [[INNERCMP]], label [[LATCH]], label [[EXIT:%.*]]84; CHECK: Latch:85; CHECK-NEXT: [[SPECULATE_TRIP_COUNT:%.*]] = icmp ult i64 [[J_NEXT]], 104857686; CHECK-NEXT: br i1 [[SPECULATE_TRIP_COUNT]], label [[HEADER]], label [[EXITLATCH:%.*]]87; CHECK: exitLatch:88; CHECK-NEXT: ret i64 189; CHECK: exit:90; CHECK-NEXT: [[RESULT_IN3_LCSSA:%.*]] = phi ptr [ [[RESULT_IN3]], [[HEADER]] ]91; CHECK-NEXT: [[RESULT_LE:%.*]] = load i64, ptr [[RESULT_IN3_LCSSA]], align 892; CHECK-NEXT: ret i64 [[RESULT_LE]]93;94entry:95 %length.ext = zext i32 %length to i6496 %n.pre = load i64, ptr %n_addr, align 497 br label %Header98 99Header: ; preds = %entry, %Latch100 %result.in3 = phi ptr [ %arg2, %entry ], [ %arg, %Latch ]101 %j2 = phi i64 [ 0, %entry ], [ %j.next, %Latch ]102 %within.bounds = icmp ult i64 %j2, %length.ext103 call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]104 %innercmp = icmp eq i64 %j2, %n.pre105 %j.next = add nuw nsw i64 %j2, 1106 br i1 %innercmp, label %Latch, label %exit107 108Latch: ; preds = %Header109 %speculate_trip_count = icmp ult i64 %j.next, 1048576110 br i1 %speculate_trip_count, label %Header, label %exitLatch111 112exitLatch: ; preds = %Latch113 ret i64 1114 115exit: ; preds = %Header116 %result.in3.lcssa = phi ptr [ %result.in3, %Header ]117 %result.le = load i64, ptr %result.in3.lcssa, align 8118 ret i64 %result.le119}120 121; Same as test above but with profiling data that the most probable exit from122; the loop is the header exiting block (not the latch block). So do not predicate.123; LatchExitProbability: 0x000020e1 / 0x80000000 = 0.00%124; ExitingBlockProbability: 0x7ffcbb86 / 0x80000000 = 99.99%125define i64 @donot_predicate_prof(ptr nocapture readonly %arg, i32 %length, ptr nocapture readonly %arg2, ptr nocapture readonly %n_addr, i64 %i) !prof !21 {126; CHECK-LABEL: @donot_predicate_prof(127; CHECK-NEXT: entry:128; CHECK-NEXT: [[LENGTH_EXT:%.*]] = zext i32 [[LENGTH:%.*]] to i64129; CHECK-NEXT: [[N_PRE:%.*]] = load i64, ptr [[N_ADDR:%.*]], align 4130; CHECK-NEXT: br label [[HEADER:%.*]]131; CHECK: Header:132; CHECK-NEXT: [[RESULT_IN3:%.*]] = phi ptr [ [[ARG2:%.*]], [[ENTRY:%.*]] ], [ [[ARG:%.*]], [[LATCH:%.*]] ]133; CHECK-NEXT: [[J2:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[J_NEXT:%.*]], [[LATCH]] ]134; CHECK-NEXT: [[WITHIN_BOUNDS:%.*]] = icmp ult i64 [[J2]], [[LENGTH_EXT]]135; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 [[WITHIN_BOUNDS]], i32 9) [ "deopt"() ]136; CHECK-NEXT: [[INNERCMP:%.*]] = icmp eq i64 [[J2]], [[N_PRE]]137; CHECK-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J2]], 1138; CHECK-NEXT: br i1 [[INNERCMP]], label [[LATCH]], label [[EXIT:%.*]], !prof [[PROF2:![0-9]+]]139; CHECK: Latch:140; CHECK-NEXT: [[SPECULATE_TRIP_COUNT:%.*]] = icmp ult i64 [[J_NEXT]], 1048576141; CHECK-NEXT: br i1 [[SPECULATE_TRIP_COUNT]], label [[HEADER]], label [[EXITLATCH:%.*]], !prof [[PROF3:![0-9]+]]142; CHECK: exitLatch:143; CHECK-NEXT: ret i64 1144; CHECK: exit:145; CHECK-NEXT: [[RESULT_IN3_LCSSA:%.*]] = phi ptr [ [[RESULT_IN3]], [[HEADER]] ]146; CHECK-NEXT: [[RESULT_LE:%.*]] = load i64, ptr [[RESULT_IN3_LCSSA]], align 8147; CHECK-NEXT: ret i64 [[RESULT_LE]]148;149entry:150 %length.ext = zext i32 %length to i64151 %n.pre = load i64, ptr %n_addr, align 4152 br label %Header153 154Header: ; preds = %entry, %Latch155 %result.in3 = phi ptr [ %arg2, %entry ], [ %arg, %Latch ]156 %j2 = phi i64 [ 0, %entry ], [ %j.next, %Latch ]157 %within.bounds = icmp ult i64 %j2, %length.ext158 call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]159 %innercmp = icmp eq i64 %j2, %n.pre160 %j.next = add nuw nsw i64 %j2, 1161 br i1 %innercmp, label %Latch, label %exit, !prof !1162 163Latch: ; preds = %Header164 %speculate_trip_count = icmp ult i64 %j.next, 1048576165 br i1 %speculate_trip_count, label %Header, label %exitLatch, !prof !2166 167exitLatch: ; preds = %Latch168 ret i64 1169 170exit: ; preds = %Header171 %result.in3.lcssa = phi ptr [ %result.in3, %Header ]172 %result.le = load i64, ptr %result.in3.lcssa, align 8173 ret i64 %result.le174}175declare i64 @llvm.experimental.deoptimize.i64(...)176declare void @llvm.experimental.guard(i1, ...)177 178!1 = !{!"branch_weights", i32 104, i32 1042861}179!2 = !{!"branch_weights", i32 255129, i32 1}180!21 = !{!"function_entry_count", i64 20000}181