260 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 32; RUN: opt -S -loop-predication-skip-profitability-checks=false -passes='require<scalar-evolution>,loop-mssa(loop-predication)' -verify-memoryssa -loop-predication-latch-probability-scale=2 %s 2>&1 | FileCheck %s --check-prefixes=CHECK-PROF3; RUN: opt -S -loop-predication-skip-profitability-checks=false -passes='require<scalar-evolution>,loop-mssa(loop-predication)' -verify-memoryssa -loop-predication-latch-probability-scale=1.9 %s 2>&1 | FileCheck %s --check-prefixes=CHECK-NOTPROF4 5; LatchExitProbability: 0x20000000 / 0x80000000 = 25.00%6; ExitingBlockProbability: 0x40000000 / 0x80000000 = 50.00%7; Predicate is profitable when the scale factor is 2 and not profitable if it's less than 2.8define i64 @predicate_eq_ones(ptr nocapture readonly %arg, i32 %length, ptr nocapture readonly %arg2, ptr nocapture readonly %n_addr, i64 %i) !prof !21 {9; CHECK-PROF-LABEL: define i64 @predicate_eq_ones(10; CHECK-PROF-SAME: ptr readonly captures(none) [[ARG:%.*]], i32 [[LENGTH:%.*]], ptr readonly captures(none) [[ARG2:%.*]], ptr readonly captures(none) [[N_ADDR:%.*]], i64 [[I:%.*]]) !prof [[PROF0:![0-9]+]] {11; CHECK-PROF-NEXT: entry:12; CHECK-PROF-NEXT: [[LENGTH_EXT:%.*]] = zext i32 [[LENGTH]] to i6413; CHECK-PROF-NEXT: [[N_PRE:%.*]] = load i64, ptr [[N_ADDR]], align 414; CHECK-PROF-NEXT: [[TMP0:%.*]] = icmp ule i64 1048576, [[LENGTH_EXT]]15; CHECK-PROF-NEXT: [[TMP1:%.*]] = icmp ult i64 0, [[LENGTH_EXT]]16; CHECK-PROF-NEXT: [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]17; CHECK-PROF-NEXT: [[TMP3:%.*]] = freeze i1 [[TMP2]]18; CHECK-PROF-NEXT: br label [[HEADER:%.*]]19; CHECK-PROF: Header:20; CHECK-PROF-NEXT: [[RESULT_IN3:%.*]] = phi ptr [ [[ARG2]], [[ENTRY:%.*]] ], [ [[ARG]], [[LATCH:%.*]] ]21; CHECK-PROF-NEXT: [[J2:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[J_NEXT:%.*]], [[LATCH]] ]22; CHECK-PROF-NEXT: [[WITHIN_BOUNDS:%.*]] = icmp ult i64 [[J2]], [[LENGTH_EXT]]23; CHECK-PROF-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 [[TMP3]], i32 9) [ "deopt"() ]24; CHECK-PROF-NEXT: call void @llvm.assume(i1 [[WITHIN_BOUNDS]])25; CHECK-PROF-NEXT: [[INNERCMP:%.*]] = icmp eq i64 [[J2]], [[N_PRE]]26; CHECK-PROF-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J2]], 127; CHECK-PROF-NEXT: br i1 [[INNERCMP]], label [[LATCH]], label [[EXIT:%.*]], !prof [[PROF1:![0-9]+]]28; CHECK-PROF: Latch:29; CHECK-PROF-NEXT: [[SPECULATE_TRIP_COUNT:%.*]] = icmp ult i64 [[J_NEXT]], 104857630; CHECK-PROF-NEXT: br i1 [[SPECULATE_TRIP_COUNT]], label [[HEADER]], label [[EXITLATCH:%.*]], !prof [[PROF2:![0-9]+]]31; CHECK-PROF: exitLatch:32; CHECK-PROF-NEXT: ret i64 133; CHECK-PROF: exit:34; CHECK-PROF-NEXT: [[RESULT_IN3_LCSSA:%.*]] = phi ptr [ [[RESULT_IN3]], [[HEADER]] ]35; CHECK-PROF-NEXT: [[RESULT_LE:%.*]] = load i64, ptr [[RESULT_IN3_LCSSA]], align 836; CHECK-PROF-NEXT: ret i64 [[RESULT_LE]]37;38; CHECK-NOTPROF-LABEL: define i64 @predicate_eq_ones(39; CHECK-NOTPROF-SAME: ptr readonly captures(none) [[ARG:%.*]], i32 [[LENGTH:%.*]], ptr readonly captures(none) [[ARG2:%.*]], ptr readonly captures(none) [[N_ADDR:%.*]], i64 [[I:%.*]]) !prof [[PROF0:![0-9]+]] {40; CHECK-NOTPROF-NEXT: entry:41; CHECK-NOTPROF-NEXT: [[LENGTH_EXT:%.*]] = zext i32 [[LENGTH]] to i6442; CHECK-NOTPROF-NEXT: [[N_PRE:%.*]] = load i64, ptr [[N_ADDR]], align 443; CHECK-NOTPROF-NEXT: br label [[HEADER:%.*]]44; CHECK-NOTPROF: Header:45; CHECK-NOTPROF-NEXT: [[RESULT_IN3:%.*]] = phi ptr [ [[ARG2]], [[ENTRY:%.*]] ], [ [[ARG]], [[LATCH:%.*]] ]46; CHECK-NOTPROF-NEXT: [[J2:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[J_NEXT:%.*]], [[LATCH]] ]47; CHECK-NOTPROF-NEXT: [[WITHIN_BOUNDS:%.*]] = icmp ult i64 [[J2]], [[LENGTH_EXT]]48; CHECK-NOTPROF-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 [[WITHIN_BOUNDS]], i32 9) [ "deopt"() ]49; CHECK-NOTPROF-NEXT: [[INNERCMP:%.*]] = icmp eq i64 [[J2]], [[N_PRE]]50; CHECK-NOTPROF-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J2]], 151; CHECK-NOTPROF-NEXT: br i1 [[INNERCMP]], label [[LATCH]], label [[EXIT:%.*]], !prof [[PROF1:![0-9]+]]52; CHECK-NOTPROF: Latch:53; CHECK-NOTPROF-NEXT: [[SPECULATE_TRIP_COUNT:%.*]] = icmp ult i64 [[J_NEXT]], 104857654; CHECK-NOTPROF-NEXT: br i1 [[SPECULATE_TRIP_COUNT]], label [[HEADER]], label [[EXITLATCH:%.*]], !prof [[PROF2:![0-9]+]]55; CHECK-NOTPROF: exitLatch:56; CHECK-NOTPROF-NEXT: ret i64 157; CHECK-NOTPROF: exit:58; CHECK-NOTPROF-NEXT: [[RESULT_IN3_LCSSA:%.*]] = phi ptr [ [[RESULT_IN3]], [[HEADER]] ]59; CHECK-NOTPROF-NEXT: [[RESULT_LE:%.*]] = load i64, ptr [[RESULT_IN3_LCSSA]], align 860; CHECK-NOTPROF-NEXT: ret i64 [[RESULT_LE]]61;62entry:63 %length.ext = zext i32 %length to i6464 %n.pre = load i64, ptr %n_addr, align 465 br label %Header66 67Header: ; preds = %entry, %Latch68 %result.in3 = phi ptr [ %arg2, %entry ], [ %arg, %Latch ]69 %j2 = phi i64 [ 0, %entry ], [ %j.next, %Latch ]70 %within.bounds = icmp ult i64 %j2, %length.ext71 call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]72 %innercmp = icmp eq i64 %j2, %n.pre73 %j.next = add nuw nsw i64 %j2, 174 br i1 %innercmp, label %Latch, label %exit, !prof !075 76Latch: ; preds = %Header77 %speculate_trip_count = icmp ult i64 %j.next, 104857678 br i1 %speculate_trip_count, label %Header, label %exitLatch, !prof !279 80exitLatch: ; preds = %Latch81 ret i64 182 83exit: ; preds = %Header84 %result.in3.lcssa = phi ptr [ %result.in3, %Header ]85 %result.le = load i64, ptr %result.in3.lcssa, align 886 ret i64 %result.le87}88!0 = !{!"branch_weights", i32 1, i32 1}89 90; Same as the previous one, but with zero weights (should be treated as if no profile - equal probability)91define i64 @predicate_eq_zeroes(ptr nocapture readonly %arg, i32 %length, ptr nocapture readonly %arg2, ptr nocapture readonly %n_addr, i64 %i) !prof !21 {92; CHECK-PROF-LABEL: define i64 @predicate_eq_zeroes(93; CHECK-PROF-SAME: ptr readonly captures(none) [[ARG:%.*]], i32 [[LENGTH:%.*]], ptr readonly captures(none) [[ARG2:%.*]], ptr readonly captures(none) [[N_ADDR:%.*]], i64 [[I:%.*]]) !prof [[PROF0]] {94; CHECK-PROF-NEXT: entry:95; CHECK-PROF-NEXT: [[LENGTH_EXT:%.*]] = zext i32 [[LENGTH]] to i6496; CHECK-PROF-NEXT: [[N_PRE:%.*]] = load i64, ptr [[N_ADDR]], align 497; CHECK-PROF-NEXT: [[TMP0:%.*]] = icmp ule i64 1048576, [[LENGTH_EXT]]98; CHECK-PROF-NEXT: [[TMP1:%.*]] = icmp ult i64 0, [[LENGTH_EXT]]99; CHECK-PROF-NEXT: [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]100; CHECK-PROF-NEXT: [[TMP3:%.*]] = freeze i1 [[TMP2]]101; CHECK-PROF-NEXT: br label [[HEADER:%.*]]102; CHECK-PROF: Header:103; CHECK-PROF-NEXT: [[RESULT_IN3:%.*]] = phi ptr [ [[ARG2]], [[ENTRY:%.*]] ], [ [[ARG]], [[LATCH:%.*]] ]104; CHECK-PROF-NEXT: [[J2:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[J_NEXT:%.*]], [[LATCH]] ]105; CHECK-PROF-NEXT: [[WITHIN_BOUNDS:%.*]] = icmp ult i64 [[J2]], [[LENGTH_EXT]]106; CHECK-PROF-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 [[TMP3]], i32 9) [ "deopt"() ]107; CHECK-PROF-NEXT: call void @llvm.assume(i1 [[WITHIN_BOUNDS]])108; CHECK-PROF-NEXT: [[INNERCMP:%.*]] = icmp eq i64 [[J2]], [[N_PRE]]109; CHECK-PROF-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J2]], 1110; CHECK-PROF-NEXT: br i1 [[INNERCMP]], label [[LATCH]], label [[EXIT:%.*]], !prof [[PROF3:![0-9]+]]111; CHECK-PROF: Latch:112; CHECK-PROF-NEXT: [[SPECULATE_TRIP_COUNT:%.*]] = icmp ult i64 [[J_NEXT]], 1048576113; CHECK-PROF-NEXT: br i1 [[SPECULATE_TRIP_COUNT]], label [[HEADER]], label [[EXITLATCH:%.*]], !prof [[PROF2]]114; CHECK-PROF: exitLatch:115; CHECK-PROF-NEXT: ret i64 1116; CHECK-PROF: exit:117; CHECK-PROF-NEXT: [[RESULT_IN3_LCSSA:%.*]] = phi ptr [ [[RESULT_IN3]], [[HEADER]] ]118; CHECK-PROF-NEXT: [[RESULT_LE:%.*]] = load i64, ptr [[RESULT_IN3_LCSSA]], align 8119; CHECK-PROF-NEXT: ret i64 [[RESULT_LE]]120;121; CHECK-NOTPROF-LABEL: define i64 @predicate_eq_zeroes(122; CHECK-NOTPROF-SAME: ptr readonly captures(none) [[ARG:%.*]], i32 [[LENGTH:%.*]], ptr readonly captures(none) [[ARG2:%.*]], ptr readonly captures(none) [[N_ADDR:%.*]], i64 [[I:%.*]]) !prof [[PROF0]] {123; CHECK-NOTPROF-NEXT: entry:124; CHECK-NOTPROF-NEXT: [[LENGTH_EXT:%.*]] = zext i32 [[LENGTH]] to i64125; CHECK-NOTPROF-NEXT: [[N_PRE:%.*]] = load i64, ptr [[N_ADDR]], align 4126; CHECK-NOTPROF-NEXT: br label [[HEADER:%.*]]127; CHECK-NOTPROF: Header:128; CHECK-NOTPROF-NEXT: [[RESULT_IN3:%.*]] = phi ptr [ [[ARG2]], [[ENTRY:%.*]] ], [ [[ARG]], [[LATCH:%.*]] ]129; CHECK-NOTPROF-NEXT: [[J2:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[J_NEXT:%.*]], [[LATCH]] ]130; CHECK-NOTPROF-NEXT: [[WITHIN_BOUNDS:%.*]] = icmp ult i64 [[J2]], [[LENGTH_EXT]]131; CHECK-NOTPROF-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 [[WITHIN_BOUNDS]], i32 9) [ "deopt"() ]132; CHECK-NOTPROF-NEXT: [[INNERCMP:%.*]] = icmp eq i64 [[J2]], [[N_PRE]]133; CHECK-NOTPROF-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J2]], 1134; CHECK-NOTPROF-NEXT: br i1 [[INNERCMP]], label [[LATCH]], label [[EXIT:%.*]], !prof [[PROF3:![0-9]+]]135; CHECK-NOTPROF: Latch:136; CHECK-NOTPROF-NEXT: [[SPECULATE_TRIP_COUNT:%.*]] = icmp ult i64 [[J_NEXT]], 1048576137; CHECK-NOTPROF-NEXT: br i1 [[SPECULATE_TRIP_COUNT]], label [[HEADER]], label [[EXITLATCH:%.*]], !prof [[PROF2]]138; CHECK-NOTPROF: exitLatch:139; CHECK-NOTPROF-NEXT: ret i64 1140; CHECK-NOTPROF: exit:141; CHECK-NOTPROF-NEXT: [[RESULT_IN3_LCSSA:%.*]] = phi ptr [ [[RESULT_IN3]], [[HEADER]] ]142; CHECK-NOTPROF-NEXT: [[RESULT_LE:%.*]] = load i64, ptr [[RESULT_IN3_LCSSA]], align 8143; CHECK-NOTPROF-NEXT: ret i64 [[RESULT_LE]]144;145entry:146 %length.ext = zext i32 %length to i64147 %n.pre = load i64, ptr %n_addr, align 4148 br label %Header149 150Header: ; preds = %entry, %Latch151 %result.in3 = phi ptr [ %arg2, %entry ], [ %arg, %Latch ]152 %j2 = phi i64 [ 0, %entry ], [ %j.next, %Latch ]153 %within.bounds = icmp ult i64 %j2, %length.ext154 call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]155 %innercmp = icmp eq i64 %j2, %n.pre156 %j.next = add nuw nsw i64 %j2, 1157 br i1 %innercmp, label %Latch, label %exit, !prof !1158 159Latch: ; preds = %Header160 %speculate_trip_count = icmp ult i64 %j.next, 1048576161 br i1 %speculate_trip_count, label %Header, label %exitLatch, !prof !2162 163exitLatch: ; preds = %Latch164 ret i64 1165 166exit: ; preds = %Header167 %result.in3.lcssa = phi ptr [ %result.in3, %Header ]168 %result.le = load i64, ptr %result.in3.lcssa, align 8169 ret i64 %result.le170}171!1 = !{!"branch_weights", i32 0, i32 0}172 173; No profile on br in Header174define i64 @predicate_eq_none(ptr nocapture readonly %arg, i32 %length, ptr nocapture readonly %arg2, ptr nocapture readonly %n_addr, i64 %i) !prof !21 {175; CHECK-PROF-LABEL: define i64 @predicate_eq_none(176; CHECK-PROF-SAME: ptr readonly captures(none) [[ARG:%.*]], i32 [[LENGTH:%.*]], ptr readonly captures(none) [[ARG2:%.*]], ptr readonly captures(none) [[N_ADDR:%.*]], i64 [[I:%.*]]) !prof [[PROF0]] {177; CHECK-PROF-NEXT: entry:178; CHECK-PROF-NEXT: [[LENGTH_EXT:%.*]] = zext i32 [[LENGTH]] to i64179; CHECK-PROF-NEXT: [[N_PRE:%.*]] = load i64, ptr [[N_ADDR]], align 4180; CHECK-PROF-NEXT: [[TMP0:%.*]] = icmp ule i64 1048576, [[LENGTH_EXT]]181; CHECK-PROF-NEXT: [[TMP1:%.*]] = icmp ult i64 0, [[LENGTH_EXT]]182; CHECK-PROF-NEXT: [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]183; CHECK-PROF-NEXT: [[TMP3:%.*]] = freeze i1 [[TMP2]]184; CHECK-PROF-NEXT: br label [[HEADER:%.*]]185; CHECK-PROF: Header:186; CHECK-PROF-NEXT: [[RESULT_IN3:%.*]] = phi ptr [ [[ARG2]], [[ENTRY:%.*]] ], [ [[ARG]], [[LATCH:%.*]] ]187; CHECK-PROF-NEXT: [[J2:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[J_NEXT:%.*]], [[LATCH]] ]188; CHECK-PROF-NEXT: [[WITHIN_BOUNDS:%.*]] = icmp ult i64 [[J2]], [[LENGTH_EXT]]189; CHECK-PROF-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 [[TMP3]], i32 9) [ "deopt"() ]190; CHECK-PROF-NEXT: call void @llvm.assume(i1 [[WITHIN_BOUNDS]])191; CHECK-PROF-NEXT: [[INNERCMP:%.*]] = icmp eq i64 [[J2]], [[N_PRE]]192; CHECK-PROF-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J2]], 1193; CHECK-PROF-NEXT: br i1 [[INNERCMP]], label [[LATCH]], label [[EXIT:%.*]]194; CHECK-PROF: Latch:195; CHECK-PROF-NEXT: [[SPECULATE_TRIP_COUNT:%.*]] = icmp ult i64 [[J_NEXT]], 1048576196; CHECK-PROF-NEXT: br i1 [[SPECULATE_TRIP_COUNT]], label [[HEADER]], label [[EXITLATCH:%.*]], !prof [[PROF2]]197; CHECK-PROF: exitLatch:198; CHECK-PROF-NEXT: ret i64 1199; CHECK-PROF: exit:200; CHECK-PROF-NEXT: [[RESULT_IN3_LCSSA:%.*]] = phi ptr [ [[RESULT_IN3]], [[HEADER]] ]201; CHECK-PROF-NEXT: [[RESULT_LE:%.*]] = load i64, ptr [[RESULT_IN3_LCSSA]], align 8202; CHECK-PROF-NEXT: ret i64 [[RESULT_LE]]203;204; CHECK-NOTPROF-LABEL: define i64 @predicate_eq_none(205; CHECK-NOTPROF-SAME: ptr readonly captures(none) [[ARG:%.*]], i32 [[LENGTH:%.*]], ptr readonly captures(none) [[ARG2:%.*]], ptr readonly captures(none) [[N_ADDR:%.*]], i64 [[I:%.*]]) !prof [[PROF0]] {206; CHECK-NOTPROF-NEXT: entry:207; CHECK-NOTPROF-NEXT: [[LENGTH_EXT:%.*]] = zext i32 [[LENGTH]] to i64208; CHECK-NOTPROF-NEXT: [[N_PRE:%.*]] = load i64, ptr [[N_ADDR]], align 4209; CHECK-NOTPROF-NEXT: br label [[HEADER:%.*]]210; CHECK-NOTPROF: Header:211; CHECK-NOTPROF-NEXT: [[RESULT_IN3:%.*]] = phi ptr [ [[ARG2]], [[ENTRY:%.*]] ], [ [[ARG]], [[LATCH:%.*]] ]212; CHECK-NOTPROF-NEXT: [[J2:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[J_NEXT:%.*]], [[LATCH]] ]213; CHECK-NOTPROF-NEXT: [[WITHIN_BOUNDS:%.*]] = icmp ult i64 [[J2]], [[LENGTH_EXT]]214; CHECK-NOTPROF-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 [[WITHIN_BOUNDS]], i32 9) [ "deopt"() ]215; CHECK-NOTPROF-NEXT: [[INNERCMP:%.*]] = icmp eq i64 [[J2]], [[N_PRE]]216; CHECK-NOTPROF-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J2]], 1217; CHECK-NOTPROF-NEXT: br i1 [[INNERCMP]], label [[LATCH]], label [[EXIT:%.*]]218; CHECK-NOTPROF: Latch:219; CHECK-NOTPROF-NEXT: [[SPECULATE_TRIP_COUNT:%.*]] = icmp ult i64 [[J_NEXT]], 1048576220; CHECK-NOTPROF-NEXT: br i1 [[SPECULATE_TRIP_COUNT]], label [[HEADER]], label [[EXITLATCH:%.*]], !prof [[PROF2]]221; CHECK-NOTPROF: exitLatch:222; CHECK-NOTPROF-NEXT: ret i64 1223; CHECK-NOTPROF: exit:224; CHECK-NOTPROF-NEXT: [[RESULT_IN3_LCSSA:%.*]] = phi ptr [ [[RESULT_IN3]], [[HEADER]] ]225; CHECK-NOTPROF-NEXT: [[RESULT_LE:%.*]] = load i64, ptr [[RESULT_IN3_LCSSA]], align 8226; CHECK-NOTPROF-NEXT: ret i64 [[RESULT_LE]]227;228entry:229 %length.ext = zext i32 %length to i64230 %n.pre = load i64, ptr %n_addr, align 4231 br label %Header232 233Header: ; preds = %entry, %Latch234 %result.in3 = phi ptr [ %arg2, %entry ], [ %arg, %Latch ]235 %j2 = phi i64 [ 0, %entry ], [ %j.next, %Latch ]236 %within.bounds = icmp ult i64 %j2, %length.ext237 call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]238 %innercmp = icmp eq i64 %j2, %n.pre239 %j.next = add nuw nsw i64 %j2, 1240 br i1 %innercmp, label %Latch, label %exit241 242Latch: ; preds = %Header243 %speculate_trip_count = icmp ult i64 %j.next, 1048576244 br i1 %speculate_trip_count, label %Header, label %exitLatch, !prof !2245 246exitLatch: ; preds = %Latch247 ret i64 1248 249exit: ; preds = %Header250 %result.in3.lcssa = phi ptr [ %result.in3, %Header ]251 %result.le = load i64, ptr %result.in3.lcssa, align 8252 ret i64 %result.le253}254 255!2 = !{!"branch_weights", i32 3, i32 1}256!21 = !{!"function_entry_count", i64 20000}257 258declare i64 @llvm.experimental.deoptimize.i64(...)259declare void @llvm.experimental.guard(i1, ...)260