388 lines · plain
1; RUN: opt < %s -passes=pseudo-probe,sample-profile -sample-profile-use-profi -sample-profile-file=%S/Inputs/profile-inference-rebalance-large.prof | opt -passes='print<branch-prob>' -disable-output 2>&1 | FileCheck %s2; RUN: opt < %s -passes=pseudo-probe,sample-profile -sample-profile-use-profi -sample-profile-file=%S/Inputs/profile-inference-rebalance-large.prof | opt -passes='print<block-freq>' -disable-output 2>&1 | FileCheck %s --check-prefix=CHECK23 4; The test verifies that counts can rebalanced in switch statements that contain5; both 'known' and 'unknown' basic blocks.6;7; +---------+8; +----------------- | b15 [?] |9; | +---------+10; | ^11; | |12; | |13; | +---------+ +--------------+ +---------+14; | | b13 [?] | <-- | b11 [3300] | --> | b14 [?] |15; | +---------+ +--------------+ +---------+16; | | | | |17; | | | | |18; | | v | |19; | | +---------+ | |20; | | | b12 [0] | | |21; | | +---------+ | |22; | | | | |23; | | | | |24; | | v v |25; | | +--------------+ |26; | +-----------> | | <-----+27; | | b16 [3300] |28; +----------------> | |29; +--------------+30 31@yydebug = dso_local global i32 0, align 432 33; Function Attrs: nounwind uwtable34define dso_local i32 @foo1(i32 %0, i32 %1) #0 {35b11:36 call void @llvm.pseudoprobe(i64 7682762345278052905, i64 1, i32 0, i64 -1)37 %cmp = icmp ne i32 %0, 038 switch i32 %1, label %b12 [39 i32 1, label %b1340 i32 2, label %b1441 i32 3, label %b1542 i32 4, label %b1643 ]44; CHECK: edge %b11 -> %b12 probability is 0x00000000 / 0x80000000 = 0.00%45; CHECK: edge %b11 -> %b13 probability is 0x20000000 / 0x80000000 = 25.00%46; CHECK: edge %b11 -> %b14 probability is 0x20000000 / 0x80000000 = 25.00%47; CHECK: edge %b11 -> %b15 probability is 0x20000000 / 0x80000000 = 25.00%48; CHECK: edge %b11 -> %b16 probability is 0x20000000 / 0x80000000 = 25.00%49; CHECK2: - b11: float = {{.*}}, int = {{.*}}, count = 330050 51b12:52 call void @llvm.pseudoprobe(i64 7682762345278052905, i64 2, i32 0, i64 -1)53 br label %b1654; CHECK2: - b12: float = {{.*}}, int = {{.*}}, count = 055 56b13:57 call void @llvm.pseudoprobe(i64 7682762345278052905, i64 3, i32 0, i64 -1)58 br label %b1659; CHECK2: - b13: float = {{.*}}, int = {{.*}}, count = 82560 61b14:62 call void @llvm.pseudoprobe(i64 7682762345278052905, i64 4, i32 0, i64 -1)63 br label %b1664; CHECK2: - b14: float = {{.*}}, int = {{.*}}, count = 82565 66b15:67 call void @llvm.pseudoprobe(i64 7682762345278052905, i64 5, i32 0, i64 -1)68 br label %b1669; CHECK2: - b15: float = {{.*}}, int = {{.*}}, count = 82570 71b16:72 call void @llvm.pseudoprobe(i64 7682762345278052905, i64 6, i32 0, i64 -1)73 ret i32 %174; CHECK2: - b16: float = {{.*}}, int = {{.*}}, count = 330075}76 77 78; The test verifies that counts can rebalanced even when control-flow ends at79; a basic block with an unknown count.80;81; +-----------+82; | b21 [128] | -+83; +-----------+ |84; | |85; v |86; +-----------+ |87; | b22 [128] | |88; +-----------+ |89; | |90; v |91; +-----------+ |92; +------------ | b23 [128] | <+93; | +-----------+94; | |95; v v96; +---------+ +-----------+97; | b26 [?] | <-- | b24 [128] |98; +---------+ +-----------+99; | |100; | v101; | +-----------+102; | | b25 [?] |103; | +-----------+104; | |105; | v106; | +-----------+107; +-----------> | b27 [?] | -+108; +-----------+ |109; | |110; v |111; +-----------+ |112; | b28 [?] | |113; +-----------+ |114; | |115; v |116; +-----------+ |117; | b29 [?] | <+118; +-----------+119 120define dso_local i32 @foo2(i32 %0, i32 %1) #0 {121b21:122 call void @llvm.pseudoprobe(i64 2494702099028631698, i64 1, i32 0, i64 -1)123 %cmp = icmp ne i32 %0, 0124 br i1 %cmp, label %b22, label %b23125; CHECK: edge %b21 -> %b22 probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]126; CHECK: edge %b21 -> %b23 probability is 0x00000000 / 0x80000000 = 0.00%127; CHECK2: - b21: float = {{.*}}, int = {{.*}}, count = 128128 129b22:130 call void @llvm.pseudoprobe(i64 2494702099028631698, i64 2, i32 0, i64 -1)131 br label %b23132 133b23:134 call void @llvm.pseudoprobe(i64 2494702099028631698, i64 3, i32 0, i64 -1)135 br i1 %cmp, label %b24, label %b26136; CHECK: edge %b23 -> %b24 probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge]137; CHECK: edge %b23 -> %b26 probability is 0x00000000 / 0x80000000 = 0.00%138; CHECK2: - b23: float = {{.*}}, int = {{.*}}, count = 128139 140b24:141 call void @llvm.pseudoprobe(i64 2494702099028631698, i64 4, i32 0, i64 -1)142 br i1 %cmp, label %b25, label %b26143; CHECK: edge %b24 -> %b25 probability is 0x40000000 / 0x80000000 = 50.00%144; CHECK: edge %b24 -> %b26 probability is 0x40000000 / 0x80000000 = 50.00%145 146b25:147 call void @llvm.pseudoprobe(i64 2494702099028631698, i64 5, i32 0, i64 -1)148 br label %b27149; CHECK2: - b25: float = {{.*}}, int = {{.*}}, count = 64150 151b26:152 call void @llvm.pseudoprobe(i64 2494702099028631698, i64 6, i32 0, i64 -1)153 br label %b27154; CHECK2: - b26: float = {{.*}}, int = {{.*}}, count = 64155 156b27:157 call void @llvm.pseudoprobe(i64 2494702099028631698, i64 7, i32 0, i64 -1)158 br i1 %cmp, label %b28, label %b29159; CHECK: edge %b27 -> %b28 probability is 0x40000000 / 0x80000000 = 50.00%160; CHECK: edge %b27 -> %b29 probability is 0x40000000 / 0x80000000 = 50.00%161; CHECK2: - b27: float = {{.*}}, int = {{.*}}, count = 128162 163b28:164 call void @llvm.pseudoprobe(i64 2494702099028631698, i64 8, i32 0, i64 -1)165 br label %b29166; CHECK2: - b28: float = {{.*}}, int = {{.*}}, count = 64167 168b29:169 call void @llvm.pseudoprobe(i64 2494702099028631698, i64 9, i32 0, i64 -1)170 ret i32 %1171; CHECK2: - b29: float = {{.*}}, int = {{.*}}, count = 128172}173 174 175; The test verifies a flexible mode of rebalancing in which some jumps to known176; basic blocks are ignored.177;178; +------------+179; | b31 [500] |180; +------------+181; |182; v183; +---------+ +------------+184; | b33 [?] | <-- | b32 [1500] | <-----+185; +---------+ +------------+ |186; | | |187; | v |188; | +------------+ +-----------+189; | | b34 [1200] | --> | b36 [900] |190; | +------------+ +-----------+191; | |192; | v193; | +------------+194; | | b35 [?] |195; | +------------+196; | |197; | v198; | +------------+199; +-----------> | b37 [?] | -+200; +------------+ |201; | |202; v |203; +------------+ |204; | b38 [?] | |205; +------------+ |206; | |207; v |208; +------------+ |209; | b39 [500] | <+210; +------------+211;212 213define dso_local i32 @foo3(i32 %0, i32 %1) #0 {214b31:215 call void @llvm.pseudoprobe(i64 -7908226060800700466, i64 1, i32 0, i64 -1)216 %cmp = icmp ne i32 %0, 0217 br label %b32218; CHECK2: - b31: float = {{.*}}, int = {{.*}}, count = 500219 220b32:221 call void @llvm.pseudoprobe(i64 -7908226060800700466, i64 2, i32 0, i64 -1)222 br i1 %cmp, label %b33, label %b34223; CHECK: edge %b32 -> %b33 probability is 0x1999999a / 0x80000000 = 20.00%224; CHECK: edge %b32 -> %b34 probability is 0x66666666 / 0x80000000 = 80.00%225; CHECK2: - b32: float = {{.*}}, int = {{.*}}, count = 1500226 227b33:228 call void @llvm.pseudoprobe(i64 -7908226060800700466, i64 3, i32 0, i64 -1)229 br label %b37230; CHECK2: - b33: float = {{.*}}, int = {{.*}}, count = 300231 232b34:233 call void @llvm.pseudoprobe(i64 -7908226060800700466, i64 4, i32 0, i64 -1)234 br i1 %cmp, label %b35, label %b36235; CHECK: edge %b34 -> %b35 probability is 0x15555555 / 0x80000000 = 16.67%236; CHECK: edge %b34 -> %b36 probability is 0x6aaaaaab / 0x80000000 = 83.33% [HOT edge]237; CHECK2: - b34: float = {{.*}}, int = {{.*}}, count = 1200238 239b35:240 call void @llvm.pseudoprobe(i64 -7908226060800700466, i64 5, i32 0, i64 -1)241 br label %b37242; CHECK2: - b35: float = {{.*}}, int = {{.*}}, count = 200243 244b36:245 call void @llvm.pseudoprobe(i64 -7908226060800700466, i64 6, i32 0, i64 -1)246 br label %b32247; CHECK2: - b36: float = {{.*}}, int = {{.*}}, count = 1000248 249b37:250 call void @llvm.pseudoprobe(i64 -7908226060800700466, i64 7, i32 0, i64 -1)251 br i1 %cmp, label %b38, label %b39252; CHECK: edge %b37 -> %b38 probability is 0x40000000 / 0x80000000 = 50.00%253; CHECK: edge %b37 -> %b39 probability is 0x40000000 / 0x80000000 = 50.00%254; CHECK2: - b37: float = {{.*}}, int = {{.*}}, count = 500255 256b38:257 call void @llvm.pseudoprobe(i64 -7908226060800700466, i64 8, i32 0, i64 -1)258 br label %b39259; CHECK2: - b38: float = {{.*}}, int = {{.*}}, count = 250260 261b39:262 call void @llvm.pseudoprobe(i64 -7908226060800700466, i64 9, i32 0, i64 -1)263 ret i32 %1264; CHECK2: - b39: float = {{.*}}, int = {{.*}}, count = 500265}266 267 268; The test verifies that flow rebalancer can ignore 'unlikely' jumps.269;270; +-----------+271; | b41 [400] | -+272; +-----------+ |273; | |274; | |275; v |276; +-----------+ |277; | b42 [?] | |278; +-----------+ |279; | |280; | |281; v v282; +---------++---------+ +---------------------------+ +---------++---------+283; | b48 [?] || b46 [?] | <-- | | --> | b47 [?] || b49 [?] |284; +---------++---------+ | | +---------++---------+285; | ^ | | | | ^286; | | | | b43 [400] | | |287; | +-------+-------------| | | |288; | | | | | |289; | | | | ------+----------+290; | | +---------------------------+ |291; | | | | |292; | | | | |293; | | v v |294; | | +-----------+ +---------+ |295; | | | b44 [?] | | b45 [?] | |296; | | +-----------+ +---------+ |297; | | | | |298; | | | | |299; | | v v |300; | | +---------------------------+ |301; | +-----------> | | <-----+302; | | b410 [400] |303; | | |304; +----------------------> | |305; +---------------------------+306 307 308define dso_local void @foo4(i32 %0, i32 %1) #0 {309b41:310 call void @llvm.pseudoprobe(i64 -6882312132165544686, i64 1, i32 0, i64 -1)311 %cmp = icmp ne i32 %0, 0312 br i1 %cmp, label %b42, label %b43313; CHECK: edge %b41 -> %b42 probability is 0x40000000 / 0x80000000 = 50.00%314; CHECK: edge %b41 -> %b43 probability is 0x40000000 / 0x80000000 = 50.00%315; CHECK2: - b41: float = {{.*}}, int = {{.*}}, count = 400316 317b42:318 call void @llvm.pseudoprobe(i64 -6882312132165544686, i64 2, i32 0, i64 -1)319 br label %b43320; CHECK2: - b42: float = {{.*}}, int = {{.*}}, count = 200321 322b43:323 call void @llvm.pseudoprobe(i64 -6882312132165544686, i64 3, i32 0, i64 -1)324 switch i32 %1, label %b49 [325 i32 1, label %b44326 i32 2, label %b45327 i32 3, label %b46328 i32 4, label %b47329 i32 5, label %b48330 ]331; CHECK: edge %b43 -> %b49 probability is 0x00000000 / 0x80000000 = 0.00%332; CHECK: edge %b43 -> %b44 probability is 0x1999999a / 0x80000000 = 20.00%333; CHECK: edge %b43 -> %b45 probability is 0x1999999a / 0x80000000 = 20.00%334; CHECK: edge %b43 -> %b46 probability is 0x1999999a / 0x80000000 = 20.00%335; CHECK: edge %b43 -> %b47 probability is 0x1999999a / 0x80000000 = 20.00%336; CHECK: edge %b43 -> %b48 probability is 0x1999999a / 0x80000000 = 20.00%337; CHECK2: - b43: float = {{.*}}, int = {{.*}}, count = 400338 339b44:340 call void @llvm.pseudoprobe(i64 -6882312132165544686, i64 4, i32 0, i64 -1)341 br label %b410342; CHECK2: - b44: float = {{.*}}, int = {{.*}}, count = 80343 344b45:345 call void @llvm.pseudoprobe(i64 -6882312132165544686, i64 5, i32 0, i64 -1)346 br label %b410347; CHECK2: - b45: float = {{.*}}, int = {{.*}}, count = 80348 349b46:350 call void @llvm.pseudoprobe(i64 -6882312132165544686, i64 6, i32 0, i64 -1)351 br label %b410352; CHECK2: - b46: float = {{.*}}, int = {{.*}}, count = 80353 354b47:355 call void @llvm.pseudoprobe(i64 -6882312132165544686, i64 7, i32 0, i64 -1)356 br label %b410357; CHECK2: - b47: float = {{.*}}, int = {{.*}}, count = 80358 359b48:360 call void @llvm.pseudoprobe(i64 -6882312132165544686, i64 8, i32 0, i64 -1)361 br label %b410362; CHECK2: - b48: float = {{.*}}, int = {{.*}}, count = 80363 364b49:365 call void @llvm.pseudoprobe(i64 -6882312132165544686, i64 9, i32 0, i64 -1)366 unreachable367; CHECK2: - b49: float = {{.*}}, int = {{.*}}, count = 0368 369b410:370 call void @llvm.pseudoprobe(i64 -6882312132165544686, i64 10, i32 0, i64 -1)371 ret void372; CHECK2: - b410: float = {{.*}}, int = {{.*}}, count = 400373}374 375 376; Function Attrs: inaccessiblememonly nounwind willreturn377declare void @llvm.pseudoprobe(i64, i64, i32, i64) #4378 379attributes #0 = { noinline nounwind uwtable "use-sample-profile" }380attributes #4 = { inaccessiblememonly nounwind willreturn }381 382!llvm.pseudo_probe_desc = !{!7, !8, !9, !10}383 384!7 = !{i64 7682762345278052905, i64 157181141624, !"foo1", null}385!8 = !{i64 2494702099028631698, i64 208782362068, !"foo2", null}386!9 = !{i64 -7908226060800700466, i64 189901498683, !"foo3", null}387!10 = !{i64 -6882312132165544686, i64 241030178952, !"foo4", null}388