343 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 32; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 -new-reg-bank-select < %s | FileCheck -check-prefix=GFX10 %s3 4; Divergent phis that don't require lowering using lane mask merging5 6; - divergent phi that has divergent incoming value (this makes it divergent)7; but is reachable through only one path - branch instruction that chooses8; path is uniform9 10; - divergent phi that is used only inside the loop and has incoming from11; previous iteration. After phi-elimination (rewrite lane mask in phi def with12; lane mask value from previous iteration), phi will hold lane mask valid for13; current iteration which is fine since it is not used outside of the loop.14 15; And one more that is tricky (is branch divergent or not ?)16; "amdgpu-flat-work-group-size"="1,1" aka single lane execution does not stop17; shader from activating multiple lanes by using some intrinsic (entering wwm18; and using dpp instructions)19; - there are cases with single lane execution where branch instructions are not20; lowered to si_if (or other intrinsic branches) - with intention to use21; uniform branch after instruction selection?22; PhiIncomingAnalysis does not recognize G_BRCOND as divergent branch and does23; not perform lane mask merging24 25 26 27define amdgpu_ps void @divergent_i1_phi_uniform_branch(ptr addrspace(1) %out, i32 %tid, i32 inreg %cond, ptr addrspace(1) %dummyaddr) {28; GFX10-LABEL: divergent_i1_phi_uniform_branch:29; GFX10: ; %bb.0: ; %A30; GFX10-NEXT: s_cmp_lg_u32 s0, 031; GFX10-NEXT: s_cbranch_scc0 .LBB0_232; GFX10-NEXT: ; %bb.1:33; GFX10-NEXT: v_cmp_le_u32_e64 s0, 6, v234; GFX10-NEXT: s_branch .LBB0_335; GFX10-NEXT: .LBB0_2: ; %dummy36; GFX10-NEXT: v_mov_b32_e32 v5, 0x7b37; GFX10-NEXT: v_cmp_gt_u32_e64 s0, 1, v238; GFX10-NEXT: global_store_dword v[3:4], v5, off39; GFX10-NEXT: .LBB0_3: ; %exit40; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, -1, s041; GFX10-NEXT: v_add_nc_u32_e32 v2, 2, v242; GFX10-NEXT: global_store_dword v[0:1], v2, off43; GFX10-NEXT: s_endpgm44A:45 %val_A = icmp uge i32 %tid, 646 %cmp = icmp eq i32 %cond, 047 br i1 %cmp, label %dummy, label %exit48 49dummy:50 store i32 123, ptr addrspace(1) %dummyaddr51 br label %B52 53B:54 %val_B = icmp ult i32 %tid, 155 br label %exit56 57exit:58 %phi = phi i1 [ %val_A, %A ], [ %val_B, %B ]59 %sel = select i1 %phi, i32 1, i32 260 store i32 %sel, ptr addrspace(1) %out61 ret void62}63 64; Fix me - there is no need to merge lane masks here65define amdgpu_ps void @divergent_i1_phi_uniform_branch_simple(ptr addrspace(1) %out, i32 %tid, i32 inreg %cond) {66; GFX10-LABEL: divergent_i1_phi_uniform_branch_simple:67; GFX10: ; %bb.0: ; %A68; GFX10-NEXT: v_cmp_le_u32_e64 s1, 6, v269; GFX10-NEXT: s_cmp_lg_u32 s0, 070; GFX10-NEXT: s_cbranch_scc1 .LBB1_271; GFX10-NEXT: ; %bb.1: ; %B72; GFX10-NEXT: v_cmp_gt_u32_e32 vcc_lo, 1, v273; GFX10-NEXT: s_andn2_b32 s0, s1, exec_lo74; GFX10-NEXT: s_and_b32 s1, exec_lo, vcc_lo75; GFX10-NEXT: s_or_b32 s1, s0, s176; GFX10-NEXT: .LBB1_2: ; %exit77; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, -1, s178; GFX10-NEXT: v_add_nc_u32_e32 v2, 2, v279; GFX10-NEXT: global_store_dword v[0:1], v2, off80; GFX10-NEXT: s_endpgm81A:82 %val_A = icmp uge i32 %tid, 683 %cmp = icmp eq i32 %cond, 084 br i1 %cmp, label %B, label %exit85 86B:87 %val_B = icmp ult i32 %tid, 188 br label %exit89 90exit:91 %phi = phi i1 [ %val_A, %A ], [ %val_B, %B ]92 %sel = select i1 %phi, i32 1, i32 293 store i32 %sel, ptr addrspace(1) %out94 ret void95}96 97 98; Divergent i1 phi that uses value from previous iteration.99; Used only inside the loop (variable name is bool_counter)100define void @divergent_i1_phi_used_inside_loop(float %val, ptr %addr) {101; GFX10-LABEL: divergent_i1_phi_used_inside_loop:102; GFX10: ; %bb.0: ; %entry103; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)104; GFX10-NEXT: s_mov_b32 s4, 0105; GFX10-NEXT: s_mov_b32 s5, 1106; GFX10-NEXT: s_mov_b32 s6, 0107; GFX10-NEXT: ; implicit-def: $sgpr7108; GFX10-NEXT: .LBB2_1: ; %loop109; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1110; GFX10-NEXT: v_cvt_f32_u32_e32 v3, s6111; GFX10-NEXT: s_xor_b32 s5, s5, 1112; GFX10-NEXT: s_and_b32 s8, s5, 1113; GFX10-NEXT: s_cmp_lg_u32 s8, 0114; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v0115; GFX10-NEXT: s_cselect_b32 s8, exec_lo, 0116; GFX10-NEXT: s_add_i32 s6, s6, 1117; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4118; GFX10-NEXT: s_andn2_b32 s7, s7, exec_lo119; GFX10-NEXT: s_and_b32 s8, exec_lo, s8120; GFX10-NEXT: s_or_b32 s7, s7, s8121; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4122; GFX10-NEXT: s_cbranch_execnz .LBB2_1123; GFX10-NEXT: ; %bb.2: ; %exit124; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4125; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s7126; GFX10-NEXT: flat_store_dword v[1:2], v0127; GFX10-NEXT: s_waitcnt lgkmcnt(0)128; GFX10-NEXT: s_setpc_b64 s[30:31]129entry:130 br label %loop131 132loop:133 %counter = phi i32 [ 0, %entry ], [ %counterPlus1, %loop ]134 %bool_counter = phi i1 [ true, %entry ], [ %neg_bool_counter, %loop ]135 %neg_bool_counter = xor i1 %bool_counter, true136 %fcounter = uitofp i32 %counter to float137 %cond = fcmp ogt float %fcounter, %val138 %counterPlus1 = add i32 %counter, 1139 br i1 %cond, label %exit, label %loop140 141exit:142 %select = select i1 %neg_bool_counter, float 1.000000e+00, float 0.000000e+00143 store float %select, ptr %addr144 ret void145}146 147define void @divergent_i1_phi_used_inside_loop_bigger_loop_body(float %val, float %pre_cond_val, ptr %addr, ptr %addr_if, ptr %addr_else) {148; GFX10-LABEL: divergent_i1_phi_used_inside_loop_bigger_loop_body:149; GFX10: ; %bb.0: ; %entry150; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)151; GFX10-NEXT: v_cmp_lt_f32_e64 s4, 1.0, v1152; GFX10-NEXT: v_mov_b32_e32 v1, 0x3e8153; GFX10-NEXT: s_mov_b32 s5, 0154; GFX10-NEXT: s_mov_b32 s6, 0155; GFX10-NEXT: ; implicit-def: $sgpr7156; GFX10-NEXT: s_branch .LBB3_2157; GFX10-NEXT: .LBB3_1: ; %loop_body158; GFX10-NEXT: ; in Loop: Header=BB3_2 Depth=1159; GFX10-NEXT: v_cvt_f32_u32_e32 v8, s6160; GFX10-NEXT: s_mov_b32 s8, exec_lo161; GFX10-NEXT: s_add_i32 s6, s6, 1162; GFX10-NEXT: s_xor_b32 s4, s4, s8163; GFX10-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v0164; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5165; GFX10-NEXT: s_andn2_b32 s7, s7, exec_lo166; GFX10-NEXT: s_and_b32 s8, exec_lo, s4167; GFX10-NEXT: s_or_b32 s7, s7, s8168; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5169; GFX10-NEXT: s_cbranch_execz .LBB3_6170; GFX10-NEXT: .LBB3_2: ; %loop_start171; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1172; GFX10-NEXT: s_cmpk_le_i32 s6, 0x3e8173; GFX10-NEXT: s_mov_b32 s8, 1174; GFX10-NEXT: s_cbranch_scc0 .LBB3_4175; GFX10-NEXT: ; %bb.3: ; %else176; GFX10-NEXT: ; in Loop: Header=BB3_2 Depth=1177; GFX10-NEXT: s_mov_b32 s8, 0178; GFX10-NEXT: flat_store_dword v[6:7], v1179; GFX10-NEXT: .LBB3_4: ; %Flow180; GFX10-NEXT: ; in Loop: Header=BB3_2 Depth=1181; GFX10-NEXT: s_xor_b32 s8, s8, 1182; GFX10-NEXT: s_cmp_lg_u32 s8, 0183; GFX10-NEXT: s_cbranch_scc1 .LBB3_1184; GFX10-NEXT: ; %bb.5: ; %if185; GFX10-NEXT: ; in Loop: Header=BB3_2 Depth=1186; GFX10-NEXT: flat_store_dword v[4:5], v1187; GFX10-NEXT: s_branch .LBB3_1188; GFX10-NEXT: .LBB3_6: ; %exit189; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5190; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s7191; GFX10-NEXT: flat_store_dword v[2:3], v0192; GFX10-NEXT: s_waitcnt lgkmcnt(0)193; GFX10-NEXT: s_setpc_b64 s[30:31]194entry:195 %pre_cond = fcmp ogt float %pre_cond_val, 1.0196 br label %loop_start197 198loop_start:199 %counter = phi i32 [ 0, %entry ], [ %counterPlus1, %loop_body ]200 %bool_counter = phi i1 [ %pre_cond, %entry ], [ %neg_bool_counter, %loop_body ]201 %cond_break = icmp sgt i32 %counter, 1000202 br i1 %cond_break, label %if, label %else203 204if:205 store i32 1000, ptr %addr_if206 br label %loop_body207 208else:209 store i32 1000, ptr %addr_else210 br label %loop_body211 212loop_body:213 %neg_bool_counter = xor i1 %bool_counter, true214 %fcounter = uitofp i32 %counter to float215 %cond = fcmp ogt float %fcounter, %val216 %counterPlus1 = add i32 %counter, 1217 br i1 %cond, label %exit, label %loop_start218 219exit:220 %select = select i1 %neg_bool_counter, float 1.000000e+00, float 0.000000e+00221 store float %select, ptr %addr222 ret void223}224 225; There is a divergent, according to machine uniformity info, g_brcond branch226; here, not lowered to si_if because of "amdgpu-flat-work-group-size"="1,1".227define amdgpu_cs void @single_lane_execution_attribute(i32 inreg %.userdata0, <3 x i32> inreg %.WorkgroupId, <3 x i32> %.LocalInvocationId) #0 {228; GFX10-LABEL: single_lane_execution_attribute:229; GFX10: ; %bb.0: ; %.entry230; GFX10-NEXT: s_getpc_b64 s[4:5]231; GFX10-NEXT: s_mov_b32 s2, s0232; GFX10-NEXT: s_mov_b32 s3, s5233; GFX10-NEXT: v_mbcnt_lo_u32_b32 v1, -1, 0234; GFX10-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x0235; GFX10-NEXT: v_mbcnt_hi_u32_b32 v1, -1, v1236; GFX10-NEXT: v_lshlrev_b32_e32 v2, 2, v1237; GFX10-NEXT: v_and_b32_e32 v3, 1, v1238; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3239; GFX10-NEXT: s_xor_b32 s2, vcc_lo, exec_lo240; GFX10-NEXT: s_waitcnt lgkmcnt(0)241; GFX10-NEXT: buffer_load_dword v2, v2, s[4:7], 0 offen242; GFX10-NEXT: s_and_b32 vcc_lo, exec_lo, s2243; GFX10-NEXT: s_mov_b32 s2, 0244; GFX10-NEXT: s_waitcnt vmcnt(0)245; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v2246; GFX10-NEXT: s_cbranch_vccnz .LBB4_4247; GFX10-NEXT: ; %bb.1: ; %.preheader.preheader248; GFX10-NEXT: s_mov_b32 s3, 0249; GFX10-NEXT: .LBB4_2: ; %.preheader250; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1251; GFX10-NEXT: v_mov_b32_e32 v3, s2252; GFX10-NEXT: v_add_nc_u32_e32 v1, -1, v1253; GFX10-NEXT: s_add_i32 s2, s2, 4254; GFX10-NEXT: buffer_load_dword v3, v3, s[4:7], 0 offen255; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1256; GFX10-NEXT: s_waitcnt vmcnt(0)257; GFX10-NEXT: v_readfirstlane_b32 s12, v3258; GFX10-NEXT: s_add_i32 s3, s12, s3259; GFX10-NEXT: s_cbranch_vccnz .LBB4_2260; GFX10-NEXT: ; %bb.3: ; %.preheader._crit_edge261; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, s3, v2262; GFX10-NEXT: s_or_b32 s2, s0, vcc_lo263; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2264; GFX10-NEXT: s_branch .LBB4_6265; GFX10-NEXT: .LBB4_4:266; GFX10-NEXT: s_mov_b32 s2, exec_lo267; GFX10-NEXT: ; implicit-def: $vgpr1268; GFX10-NEXT: s_and_b32 vcc_lo, exec_lo, s2269; GFX10-NEXT: s_cbranch_vccz .LBB4_6270; GFX10-NEXT: ; %bb.5: ; %.19271; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0272; GFX10-NEXT: v_or_b32_e32 v1, 2, v1273; GFX10-NEXT: .LBB4_6: ; %.22274; GFX10-NEXT: v_add_lshl_u32 v0, v0, s1, 2275; GFX10-NEXT: buffer_store_dword v1, v0, s[8:11], 0 offen276; GFX10-NEXT: s_endpgm277.entry:278 %.0 = call i64 @llvm.amdgcn.s.getpc()279 %.1 = and i64 %.0, -4294967296280 %.2 = zext i32 %.userdata0 to i64281 %.3 = or i64 %.1, %.2282 %.4 = inttoptr i64 %.3 to ptr addrspace(4)283 %.5 = getelementptr i8, ptr addrspace(4) %.4, i64 16284 %.6 = load <4 x i32>, ptr addrspace(4) %.5, align 16285 %.7 = load <4 x i32>, ptr addrspace(4) %.4, align 16286 %.8 = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)287 %.9 = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %.8)288 %.fr11 = freeze i32 %.9289 %.idx = shl i32 %.fr11, 2290 %.10 = call i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32> %.7, i32 %.idx, i32 0, i32 0)291 %.11 = icmp eq i32 %.10, 0292 %.12 = and i32 %.fr11, 1293 %.not = icmp eq i32 %.12, 0294 br i1 %.not, label %.19, label %.preheader295 296.preheader: ; preds = %.entry, %.preheader297 %._96.02 = phi i32 [ %.15, %.preheader ], [ 0, %.entry ]298 %._50.01 = phi i32 [ %.14, %.preheader ], [ 0, %.entry ]299 %.idx5 = shl i32 %._96.02, 2300 %.13 = call i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32> %.7, i32 %.idx5, i32 0, i32 0)301 %.14 = add i32 %.13, %._50.01302 %.15 = add nuw i32 %._96.02, 1303 %.exitcond.not = icmp eq i32 %.15, %.fr11304 br i1 %.exitcond.not, label %.preheader._crit_edge, label %.preheader305 306.preheader._crit_edge: ; preds = %.preheader307 %.16 = icmp eq i32 %.14, %.10308 %.17 = or i1 %.11, %.16309 %.18 = zext i1 %.17 to i32310 br label %.22311 312.19: ; preds = %.entry313 %.20 = zext i1 %.11 to i32314 %.21 = or i32 %.20, 2315 br label %.22316 317.22: ; preds = %.19, %.preheader._crit_edge318 %._51.0 = phi i32 [ %.18, %.preheader._crit_edge ], [ %.21, %.19 ]319 %.WorkgroupId.i0 = extractelement <3 x i32> %.WorkgroupId, i64 0320 %.LocalInvocationId.i0 = extractelement <3 x i32> %.LocalInvocationId, i64 0321 %.i0 = add i32 %.LocalInvocationId.i0, %.WorkgroupId.i0322 %.idx6 = shl i32 %.i0, 2323 call void @llvm.amdgcn.raw.buffer.store.i32(i32 %._51.0, <4 x i32> %.6, i32 %.idx6, i32 0, i32 0)324 ret void325}326 327; Function Attrs: nocallback nofree nosync nounwind willreturn memory(none)328declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32)329 330; Function Attrs: nocallback nofree nosync nounwind willreturn memory(none)331declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32)332 333; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)334declare i64 @llvm.amdgcn.s.getpc()335 336; Function Attrs: nocallback nofree nosync nounwind willreturn memory(read)337declare i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32>, i32, i32, i32 immarg)338 339; Function Attrs: nocallback nofree nosync nounwind willreturn memory(write)340declare void @llvm.amdgcn.raw.buffer.store.i32(i32, <4 x i32>, i32, i32, i32 immarg)341 342attributes #0 = { nounwind memory(readwrite) "amdgpu-flat-work-group-size"="1,1" }343