472 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 62; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -amdgpu-enable-uniform-intrinsic-combine=0 -O3 -S < %s | FileCheck %s -check-prefix=CURRENT-CHECK3; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -passes=amdgpu-uniform-intrinsic-combine -S < %s | FileCheck %s -check-prefix=PASS-CHECK4; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -amdgpu-uniform-intrinsic-combine -S < %s | FileCheck %s -check-prefix=PASS-CHECK5; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -O3 -S < %s | FileCheck %s -check-prefix=O3-CHECK6 7define protected amdgpu_kernel void @trivial_waterfall_eq_zero(ptr addrspace(1) %out) {8; CURRENT-CHECK-LABEL: define protected amdgpu_kernel void @trivial_waterfall_eq_zero(9; CURRENT-CHECK-SAME: ptr addrspace(1) writeonly captures(none) [[OUT:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {10; CURRENT-CHECK-NEXT: [[ENTRY:.*:]]11; CURRENT-CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.ballot.i32(i1 true)12; CURRENT-CHECK-NEXT: [[IS_DONE_PEEL:%.*]] = icmp eq i32 [[TMP0]], 013; CURRENT-CHECK-NEXT: br i1 [[IS_DONE_PEEL]], label %[[EXIT:.*]], label %[[IF_PEEL:.*]]14; CURRENT-CHECK: [[IF_PEEL]]:15; CURRENT-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 416; CURRENT-CHECK-NEXT: br label %[[EXIT]]17; CURRENT-CHECK: [[EXIT]]:18; CURRENT-CHECK-NEXT: ret void19;20; PASS-CHECK-LABEL: define protected amdgpu_kernel void @trivial_waterfall_eq_zero(21; PASS-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {22; PASS-CHECK-NEXT: [[ENTRY:.*]]:23; PASS-CHECK-NEXT: br label %[[WHILE:.*]]24; PASS-CHECK: [[WHILE]]:25; PASS-CHECK-NEXT: [[DONE:%.*]] = phi i1 [ false, %[[ENTRY]] ], [ true, %[[IF:.*]] ]26; PASS-CHECK-NEXT: [[NOT_DONE:%.*]] = xor i1 [[DONE]], true27; PASS-CHECK-NEXT: [[BALLOT:%.*]] = tail call i64 @llvm.amdgcn.ballot.i64(i1 [[NOT_DONE]])28; PASS-CHECK-NEXT: [[TMP0:%.*]] = xor i1 [[NOT_DONE]], true29; PASS-CHECK-NEXT: [[IS_DONE:%.*]] = icmp eq i64 [[BALLOT]], 030; PASS-CHECK-NEXT: br i1 [[TMP0]], label %[[EXIT:.*]], label %[[IF]]31; PASS-CHECK: [[IF]]:32; PASS-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 433; PASS-CHECK-NEXT: br label %[[WHILE]]34; PASS-CHECK: [[EXIT]]:35; PASS-CHECK-NEXT: ret void36;37; O3-CHECK-LABEL: define protected amdgpu_kernel void @trivial_waterfall_eq_zero(38; O3-CHECK-SAME: ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {39; O3-CHECK-NEXT: [[ENTRY:.*:]]40; O3-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 441; O3-CHECK-NEXT: ret void42;43entry:44 br label %while45 46while:47 %done = phi i1 [ 0, %entry ], [ 1, %if ]48 %not_done = xor i1 %done, true49 %ballot = tail call i64 @llvm.amdgcn.ballot.i64(i1 %not_done)50 %is_done = icmp eq i64 %ballot, 0 ; in this case is_done = !not_done51 br i1 %is_done, label %exit, label %if52 53if:54 store i32 5, ptr addrspace(1) %out55 br label %while56 57exit:58 ret void59}60 61define protected amdgpu_kernel void @trivial_waterfall_eq_zero_swap_op(ptr addrspace(1) %out) {62; CURRENT-CHECK-LABEL: define protected amdgpu_kernel void @trivial_waterfall_eq_zero_swap_op(63; CURRENT-CHECK-SAME: ptr addrspace(1) writeonly captures(none) [[OUT:%.*]]) local_unnamed_addr #[[ATTR0]] {64; CURRENT-CHECK-NEXT: [[ENTRY:.*:]]65; CURRENT-CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.ballot.i32(i1 true)66; CURRENT-CHECK-NEXT: [[IS_DONE_PEEL:%.*]] = icmp eq i32 [[TMP0]], 067; CURRENT-CHECK-NEXT: br i1 [[IS_DONE_PEEL]], label %[[EXIT:.*]], label %[[IF_PEEL:.*]]68; CURRENT-CHECK: [[IF_PEEL]]:69; CURRENT-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 470; CURRENT-CHECK-NEXT: br label %[[EXIT]]71; CURRENT-CHECK: [[EXIT]]:72; CURRENT-CHECK-NEXT: ret void73;74; PASS-CHECK-LABEL: define protected amdgpu_kernel void @trivial_waterfall_eq_zero_swap_op(75; PASS-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {76; PASS-CHECK-NEXT: [[ENTRY:.*]]:77; PASS-CHECK-NEXT: br label %[[WHILE:.*]]78; PASS-CHECK: [[WHILE]]:79; PASS-CHECK-NEXT: [[DONE:%.*]] = phi i1 [ false, %[[ENTRY]] ], [ true, %[[IF:.*]] ]80; PASS-CHECK-NEXT: [[NOT_DONE:%.*]] = xor i1 [[DONE]], true81; PASS-CHECK-NEXT: [[BALLOT:%.*]] = tail call i64 @llvm.amdgcn.ballot.i64(i1 [[NOT_DONE]])82; PASS-CHECK-NEXT: [[TMP0:%.*]] = xor i1 [[NOT_DONE]], true83; PASS-CHECK-NEXT: [[IS_DONE:%.*]] = icmp eq i64 0, [[BALLOT]]84; PASS-CHECK-NEXT: br i1 [[TMP0]], label %[[EXIT:.*]], label %[[IF]]85; PASS-CHECK: [[IF]]:86; PASS-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 487; PASS-CHECK-NEXT: br label %[[WHILE]]88; PASS-CHECK: [[EXIT]]:89; PASS-CHECK-NEXT: ret void90;91; O3-CHECK-LABEL: define protected amdgpu_kernel void @trivial_waterfall_eq_zero_swap_op(92; O3-CHECK-SAME: ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR0]] {93; O3-CHECK-NEXT: [[ENTRY:.*:]]94; O3-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 495; O3-CHECK-NEXT: ret void96;97entry:98 br label %while99 100while:101 %done = phi i1 [ 0, %entry ], [ 1, %if ]102 %not_done = xor i1 %done, true103 %ballot = tail call i64 @llvm.amdgcn.ballot.i64(i1 %not_done)104 %is_done = icmp eq i64 0, %ballot ; in this case is_done = !not_done105 br i1 %is_done, label %exit, label %if106 107if:108 store i32 5, ptr addrspace(1) %out109 br label %while110 111exit:112 ret void113}114 115define protected amdgpu_kernel void @trivial_waterfall_ne_zero(ptr addrspace(1) %out) {116; CURRENT-CHECK-LABEL: define protected amdgpu_kernel void @trivial_waterfall_ne_zero(117; CURRENT-CHECK-SAME: ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {118; CURRENT-CHECK-NEXT: [[ENTRY:.*:]]119; CURRENT-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 4120; CURRENT-CHECK-NEXT: br label %[[WHILE:.*]]121; CURRENT-CHECK: [[WHILE]]:122; CURRENT-CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.ballot.i32(i1 true)123; CURRENT-CHECK-NEXT: [[IS_DONE_NOT:%.*]] = icmp eq i32 [[TMP0]], 0124; CURRENT-CHECK-NEXT: br i1 [[IS_DONE_NOT]], label %[[WHILE]], label %[[EXIT:.*]], !llvm.loop [[LOOP0:![0-9]+]]125; CURRENT-CHECK: [[EXIT]]:126; CURRENT-CHECK-NEXT: ret void127;128; PASS-CHECK-LABEL: define protected amdgpu_kernel void @trivial_waterfall_ne_zero(129; PASS-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {130; PASS-CHECK-NEXT: [[ENTRY:.*]]:131; PASS-CHECK-NEXT: br label %[[WHILE:.*]]132; PASS-CHECK: [[WHILE]]:133; PASS-CHECK-NEXT: [[DONE:%.*]] = phi i1 [ false, %[[ENTRY]] ], [ true, %[[IF:.*]] ]134; PASS-CHECK-NEXT: [[BALLOT:%.*]] = tail call i64 @llvm.amdgcn.ballot.i64(i1 [[DONE]])135; PASS-CHECK-NEXT: [[IS_DONE:%.*]] = icmp ne i64 0, [[BALLOT]]136; PASS-CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[IF]]137; PASS-CHECK: [[IF]]:138; PASS-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 4139; PASS-CHECK-NEXT: br label %[[WHILE]]140; PASS-CHECK: [[EXIT]]:141; PASS-CHECK-NEXT: ret void142;143; O3-CHECK-LABEL: define protected amdgpu_kernel void @trivial_waterfall_ne_zero(144; O3-CHECK-SAME: ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR0]] {145; O3-CHECK-NEXT: [[ENTRY:.*:]]146; O3-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 4147; O3-CHECK-NEXT: ret void148;149entry:150 br label %while151 152while:153 %done = phi i1 [ 0, %entry ], [ 1, %if ]154 %ballot = tail call i64 @llvm.amdgcn.ballot.i64(i1 %done)155 %is_done = icmp ne i64 0, %ballot ; in this case is_done = done156 br i1 %is_done, label %exit, label %if157 158if:159 store i32 5, ptr addrspace(1) %out160 br label %while161 162exit:163 ret void164}165 166define protected amdgpu_kernel void @trivial_waterfall_ne_zero_swap(ptr addrspace(1) %out) {167; CURRENT-CHECK-LABEL: define protected amdgpu_kernel void @trivial_waterfall_ne_zero_swap(168; CURRENT-CHECK-SAME: ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR1]] {169; CURRENT-CHECK-NEXT: [[ENTRY:.*:]]170; CURRENT-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 4171; CURRENT-CHECK-NEXT: br label %[[WHILE:.*]]172; CURRENT-CHECK: [[WHILE]]:173; CURRENT-CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.ballot.i32(i1 true)174; CURRENT-CHECK-NEXT: [[IS_DONE_NOT:%.*]] = icmp eq i32 [[TMP0]], 0175; CURRENT-CHECK-NEXT: br i1 [[IS_DONE_NOT]], label %[[WHILE]], label %[[EXIT:.*]], !llvm.loop [[LOOP2:![0-9]+]]176; CURRENT-CHECK: [[EXIT]]:177; CURRENT-CHECK-NEXT: ret void178;179; PASS-CHECK-LABEL: define protected amdgpu_kernel void @trivial_waterfall_ne_zero_swap(180; PASS-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {181; PASS-CHECK-NEXT: [[ENTRY:.*]]:182; PASS-CHECK-NEXT: br label %[[WHILE:.*]]183; PASS-CHECK: [[WHILE]]:184; PASS-CHECK-NEXT: [[DONE:%.*]] = phi i1 [ false, %[[ENTRY]] ], [ true, %[[IF:.*]] ]185; PASS-CHECK-NEXT: [[BALLOT:%.*]] = tail call i64 @llvm.amdgcn.ballot.i64(i1 [[DONE]])186; PASS-CHECK-NEXT: [[IS_DONE:%.*]] = icmp ne i64 [[BALLOT]], 0187; PASS-CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[IF]]188; PASS-CHECK: [[IF]]:189; PASS-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 4190; PASS-CHECK-NEXT: br label %[[WHILE]]191; PASS-CHECK: [[EXIT]]:192; PASS-CHECK-NEXT: ret void193;194; O3-CHECK-LABEL: define protected amdgpu_kernel void @trivial_waterfall_ne_zero_swap(195; O3-CHECK-SAME: ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR0]] {196; O3-CHECK-NEXT: [[ENTRY:.*:]]197; O3-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 4198; O3-CHECK-NEXT: ret void199;200entry:201 br label %while202 203while:204 %done = phi i1 [ 0, %entry ], [ 1, %if ]205 %ballot = tail call i64 @llvm.amdgcn.ballot.i64(i1 %done)206 %is_done = icmp ne i64 %ballot, 0 ; in this case is_done = done207 br i1 %is_done, label %exit, label %if208 209if:210 store i32 5, ptr addrspace(1) %out211 br label %while212 213exit:214 ret void215}216 217define protected amdgpu_kernel void @trivial_uniform_waterfall(ptr addrspace(1) %out) {218; CURRENT-CHECK-LABEL: define protected amdgpu_kernel void @trivial_uniform_waterfall(219; CURRENT-CHECK-SAME: ptr addrspace(1) writeonly captures(none) [[OUT:%.*]]) local_unnamed_addr #[[ATTR0]] {220; CURRENT-CHECK-NEXT: [[ENTRY:.*:]]221; CURRENT-CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.ballot.i32(i1 true)222; CURRENT-CHECK-NEXT: [[IS_DONE_PEEL:%.*]] = icmp eq i32 [[TMP0]], 0223; CURRENT-CHECK-NEXT: br i1 [[IS_DONE_PEEL]], label %[[EXIT:.*]], label %[[WORK_PEEL:.*]]224; CURRENT-CHECK: [[WORK_PEEL]]:225; CURRENT-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 4226; CURRENT-CHECK-NEXT: br label %[[EXIT]]227; CURRENT-CHECK: [[EXIT]]:228; CURRENT-CHECK-NEXT: ret void229;230; PASS-CHECK-LABEL: define protected amdgpu_kernel void @trivial_uniform_waterfall(231; PASS-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {232; PASS-CHECK-NEXT: [[ENTRY:.*]]:233; PASS-CHECK-NEXT: br label %[[WHILE:.*]]234; PASS-CHECK: [[WHILE]]:235; PASS-CHECK-NEXT: [[DONE:%.*]] = phi i1 [ false, %[[ENTRY]] ], [ [[NEW_DONE:%.*]], %[[TAIL:.*]] ]236; PASS-CHECK-NEXT: [[NOT_DONE:%.*]] = xor i1 [[DONE]], true237; PASS-CHECK-NEXT: [[BALLOT:%.*]] = tail call i64 @llvm.amdgcn.ballot.i64(i1 [[NOT_DONE]])238; PASS-CHECK-NEXT: [[TMP0:%.*]] = xor i1 [[NOT_DONE]], true239; PASS-CHECK-NEXT: [[IS_DONE:%.*]] = icmp eq i64 [[BALLOT]], 0240; PASS-CHECK-NEXT: br i1 [[TMP0]], label %[[EXIT:.*]], label %[[IF:.*]]241; PASS-CHECK: [[IF]]:242; PASS-CHECK-NEXT: [[FIRST_ACTIVE_ID:%.*]] = tail call noundef i32 @llvm.amdgcn.readfirstlane.i32(i32 0)243; PASS-CHECK-NEXT: [[IS_FIRST_ACTIVE_ID:%.*]] = icmp eq i32 0, [[FIRST_ACTIVE_ID]]244; PASS-CHECK-NEXT: br i1 [[IS_FIRST_ACTIVE_ID]], label %[[WORK:.*]], label %[[TAIL]]245; PASS-CHECK: [[WORK]]:246; PASS-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 4247; PASS-CHECK-NEXT: br label %[[TAIL]]248; PASS-CHECK: [[TAIL]]:249; PASS-CHECK-NEXT: [[NEW_DONE]] = phi i1 [ true, %[[WORK]] ], [ false, %[[IF]] ]250; PASS-CHECK-NEXT: br label %[[WHILE]]251; PASS-CHECK: [[EXIT]]:252; PASS-CHECK-NEXT: ret void253;254; O3-CHECK-LABEL: define protected amdgpu_kernel void @trivial_uniform_waterfall(255; O3-CHECK-SAME: ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR0]] {256; O3-CHECK-NEXT: [[ENTRY:.*:]]257; O3-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 4258; O3-CHECK-NEXT: ret void259;260entry:261 br label %while262 263while:264 %done = phi i1 [ false, %entry ], [ %new_done, %tail ]265 %not_done = xor i1 %done, true266 %ballot = tail call i64 @llvm.amdgcn.ballot.i64(i1 %not_done)267 %is_done = icmp eq i64 %ballot, 0268 br i1 %is_done, label %exit, label %if269 270if:271 %first_active_id = tail call noundef i32 @llvm.amdgcn.readfirstlane.i32(i32 0)272 %is_first_active_id = icmp eq i32 0, %first_active_id273 br i1 %is_first_active_id, label %work, label %tail274 275work:276 store i32 5, ptr addrspace(1) %out277 br label %tail278 279tail:280 %new_done = phi i1 [ true, %work ], [ false, %if ]281 br label %while282 283exit:284 ret void285}286 287define protected amdgpu_kernel void @uniform_waterfall(ptr addrspace(1) %out, i32 %mymask) {288; CURRENT-CHECK-LABEL: define protected amdgpu_kernel void @uniform_waterfall(289; CURRENT-CHECK-SAME: ptr addrspace(1) writeonly captures(none) [[OUT:%.*]], i32 [[MYMASK:%.*]]) local_unnamed_addr #[[ATTR0]] {290; CURRENT-CHECK-NEXT: [[ENTRY:.*:]]291; CURRENT-CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.amdgcn.ballot.i32(i1 true)292; CURRENT-CHECK-NEXT: [[IS_DONE_PEEL:%.*]] = icmp eq i32 [[TMP0]], 0293; CURRENT-CHECK-NEXT: br i1 [[IS_DONE_PEEL]], label %[[EXIT:.*]], label %[[WORK_PEEL:.*]]294; CURRENT-CHECK: [[WORK_PEEL]]:295; CURRENT-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 4296; CURRENT-CHECK-NEXT: br label %[[EXIT]]297; CURRENT-CHECK: [[EXIT]]:298; CURRENT-CHECK-NEXT: ret void299;300; PASS-CHECK-LABEL: define protected amdgpu_kernel void @uniform_waterfall(301; PASS-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], i32 [[MYMASK:%.*]]) #[[ATTR0]] {302; PASS-CHECK-NEXT: [[ENTRY:.*]]:303; PASS-CHECK-NEXT: br label %[[WHILE:.*]]304; PASS-CHECK: [[WHILE]]:305; PASS-CHECK-NEXT: [[DONE:%.*]] = phi i1 [ false, %[[ENTRY]] ], [ [[NEW_DONE:%.*]], %[[TAIL:.*]] ]306; PASS-CHECK-NEXT: [[NOT_DONE:%.*]] = xor i1 [[DONE]], true307; PASS-CHECK-NEXT: [[BALLOT:%.*]] = tail call i64 @llvm.amdgcn.ballot.i64(i1 [[NOT_DONE]])308; PASS-CHECK-NEXT: [[TMP0:%.*]] = xor i1 [[NOT_DONE]], true309; PASS-CHECK-NEXT: [[IS_DONE:%.*]] = icmp eq i64 [[BALLOT]], 0310; PASS-CHECK-NEXT: br i1 [[TMP0]], label %[[EXIT:.*]], label %[[IF:.*]]311; PASS-CHECK: [[IF]]:312; PASS-CHECK-NEXT: [[FIRST_ACTIVE_ID:%.*]] = tail call noundef i32 @llvm.amdgcn.readfirstlane.i32(i32 [[MYMASK]])313; PASS-CHECK-NEXT: [[IS_FIRST_ACTIVE_ID:%.*]] = icmp eq i32 [[MYMASK]], [[FIRST_ACTIVE_ID]]314; PASS-CHECK-NEXT: br i1 [[IS_FIRST_ACTIVE_ID]], label %[[WORK:.*]], label %[[TAIL]]315; PASS-CHECK: [[WORK]]:316; PASS-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 4317; PASS-CHECK-NEXT: br label %[[TAIL]]318; PASS-CHECK: [[TAIL]]:319; PASS-CHECK-NEXT: [[NEW_DONE]] = phi i1 [ true, %[[WORK]] ], [ false, %[[IF]] ]320; PASS-CHECK-NEXT: br label %[[WHILE]]321; PASS-CHECK: [[EXIT]]:322; PASS-CHECK-NEXT: ret void323;324; O3-CHECK-LABEL: define protected amdgpu_kernel void @uniform_waterfall(325; O3-CHECK-SAME: ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[OUT:%.*]], i32 [[MYMASK:%.*]]) local_unnamed_addr #[[ATTR0]] {326; O3-CHECK-NEXT: [[ENTRY:.*:]]327; O3-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 4328; O3-CHECK-NEXT: ret void329;330entry:331 br label %while332 333while:334 %done = phi i1 [ false, %entry ], [ %new_done, %tail ]335 %not_done = xor i1 %done, true336 %ballot = tail call i64 @llvm.amdgcn.ballot.i64(i1 %not_done)337 %is_done = icmp eq i64 %ballot, 0338 br i1 %is_done, label %exit, label %if339 340if:341 %first_active_id = tail call noundef i32 @llvm.amdgcn.readfirstlane.i32(i32 %mymask)342 %is_first_active_id = icmp eq i32 %mymask, %first_active_id343 br i1 %is_first_active_id, label %work, label %tail344 345work:346 store i32 5, ptr addrspace(1) %out347 br label %tail348 349tail:350 %new_done = phi i1 [ true, %work ], [ false, %if ]351 br label %while352 353exit:354 ret void355}356 357define protected amdgpu_kernel void @trivial_waterfall_eq_zero_i32(ptr addrspace(1) %out) {358; CURRENT-CHECK-LABEL: define protected amdgpu_kernel void @trivial_waterfall_eq_zero_i32(359; CURRENT-CHECK-SAME: ptr addrspace(1) writeonly captures(none) [[OUT:%.*]]) local_unnamed_addr #[[ATTR0]] {360; CURRENT-CHECK-NEXT: [[ENTRY:.*:]]361; CURRENT-CHECK-NEXT: [[BALLOT_PEEL:%.*]] = tail call i32 @llvm.amdgcn.ballot.i32(i1 true)362; CURRENT-CHECK-NEXT: [[IS_DONE_PEEL:%.*]] = icmp eq i32 [[BALLOT_PEEL]], 0363; CURRENT-CHECK-NEXT: br i1 [[IS_DONE_PEEL]], label %[[EXIT:.*]], label %[[IF_PEEL:.*]]364; CURRENT-CHECK: [[IF_PEEL]]:365; CURRENT-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 4366; CURRENT-CHECK-NEXT: br label %[[EXIT]]367; CURRENT-CHECK: [[EXIT]]:368; CURRENT-CHECK-NEXT: ret void369;370; PASS-CHECK-LABEL: define protected amdgpu_kernel void @trivial_waterfall_eq_zero_i32(371; PASS-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {372; PASS-CHECK-NEXT: [[ENTRY:.*]]:373; PASS-CHECK-NEXT: br label %[[WHILE:.*]]374; PASS-CHECK: [[WHILE]]:375; PASS-CHECK-NEXT: [[DONE:%.*]] = phi i1 [ false, %[[ENTRY]] ], [ true, %[[IF:.*]] ]376; PASS-CHECK-NEXT: [[NOT_DONE:%.*]] = xor i1 [[DONE]], true377; PASS-CHECK-NEXT: [[BALLOT:%.*]] = tail call i32 @llvm.amdgcn.ballot.i32(i1 [[NOT_DONE]])378; PASS-CHECK-NEXT: [[TMP0:%.*]] = xor i1 [[NOT_DONE]], true379; PASS-CHECK-NEXT: [[IS_DONE:%.*]] = icmp eq i32 [[BALLOT]], 0380; PASS-CHECK-NEXT: br i1 [[TMP0]], label %[[EXIT:.*]], label %[[IF]]381; PASS-CHECK: [[IF]]:382; PASS-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 4383; PASS-CHECK-NEXT: br label %[[WHILE]]384; PASS-CHECK: [[EXIT]]:385; PASS-CHECK-NEXT: ret void386;387; O3-CHECK-LABEL: define protected amdgpu_kernel void @trivial_waterfall_eq_zero_i32(388; O3-CHECK-SAME: ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR0]] {389; O3-CHECK-NEXT: [[ENTRY:.*:]]390; O3-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 4391; O3-CHECK-NEXT: ret void392;393entry:394 br label %while395 396while:397 %done = phi i1 [ 0, %entry ], [ 1, %if ]398 %not_done = xor i1 %done, true399 %ballot = tail call i32 @llvm.amdgcn.ballot.i32(i1 %not_done)400 %is_done = icmp eq i32 %ballot, 0 ; in this case is_done = !not_done401 br i1 %is_done, label %exit, label %if402 403if:404 store i32 5, ptr addrspace(1) %out405 br label %while406 407exit:408 ret void409}410 411define protected amdgpu_kernel void @trivial_waterfall_ne_zero_i32(ptr addrspace(1) %out) {412; CURRENT-CHECK-LABEL: define protected amdgpu_kernel void @trivial_waterfall_ne_zero_i32(413; CURRENT-CHECK-SAME: ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR1]] {414; CURRENT-CHECK-NEXT: [[ENTRY:.*:]]415; CURRENT-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 4416; CURRENT-CHECK-NEXT: br label %[[WHILE:.*]]417; CURRENT-CHECK: [[WHILE]]:418; CURRENT-CHECK-NEXT: [[BALLOT:%.*]] = tail call i32 @llvm.amdgcn.ballot.i32(i1 true)419; CURRENT-CHECK-NEXT: [[IS_DONE_NOT:%.*]] = icmp eq i32 [[BALLOT]], 0420; CURRENT-CHECK-NEXT: br i1 [[IS_DONE_NOT]], label %[[WHILE]], label %[[EXIT:.*]], !llvm.loop [[LOOP3:![0-9]+]]421; CURRENT-CHECK: [[EXIT]]:422; CURRENT-CHECK-NEXT: ret void423;424; PASS-CHECK-LABEL: define protected amdgpu_kernel void @trivial_waterfall_ne_zero_i32(425; PASS-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {426; PASS-CHECK-NEXT: [[ENTRY:.*]]:427; PASS-CHECK-NEXT: br label %[[WHILE:.*]]428; PASS-CHECK: [[WHILE]]:429; PASS-CHECK-NEXT: [[DONE:%.*]] = phi i1 [ false, %[[ENTRY]] ], [ true, %[[IF:.*]] ]430; PASS-CHECK-NEXT: [[BALLOT:%.*]] = tail call i32 @llvm.amdgcn.ballot.i32(i1 [[DONE]])431; PASS-CHECK-NEXT: [[IS_DONE:%.*]] = icmp ne i32 0, [[BALLOT]]432; PASS-CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[IF]]433; PASS-CHECK: [[IF]]:434; PASS-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 4435; PASS-CHECK-NEXT: br label %[[WHILE]]436; PASS-CHECK: [[EXIT]]:437; PASS-CHECK-NEXT: ret void438;439; O3-CHECK-LABEL: define protected amdgpu_kernel void @trivial_waterfall_ne_zero_i32(440; O3-CHECK-SAME: ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR0]] {441; O3-CHECK-NEXT: [[ENTRY:.*:]]442; O3-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 4443; O3-CHECK-NEXT: ret void444;445entry:446 br label %while447 448while:449 %done = phi i1 [ 0, %entry ], [ 1, %if ]450 %ballot = tail call i32 @llvm.amdgcn.ballot.i32(i1 %done)451 %is_done = icmp ne i32 0, %ballot ; in this case is_done = done452 br i1 %is_done, label %exit, label %if453 454if:455 store i32 5, ptr addrspace(1) %out456 br label %while457 458exit:459 ret void460}461 462declare i64 @llvm.amdgcn.ballot.i64(i1) #1463!6 = !{i64 690}464!7 = distinct !{!7, !8}465!8 = !{!"llvm.loop.mustprogress"}466;.467; CURRENT-CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}468; CURRENT-CHECK: [[META1]] = !{!"llvm.loop.peeled.count", i32 1}469; CURRENT-CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[META1]]}470; CURRENT-CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}471;.472