454 lines · plain
1; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=verde -amdgpu-early-ifcvt=1 -amdgpu-codegenprepare-break-large-phis=0 < %s | FileCheck -check-prefix=GCN %s2; XUN: llc -mtriple=amdgcn -mcpu=tonga -amdgpu-early-ifcvt=1 < %s | FileCheck -check-prefix=GCN %s3 4; Note: breaking up large PHIs is disabled to prevent some testcases from becoming5; branchless.6 7; FIXME: This leaves behind a now unnecessary and with exec8 9; GCN-LABEL: {{^}}test_vccnz_ifcvt_triangle:10; GCN: buffer_load_dword [[VAL:v[0-9]+]]11; GCN: v_cmp_neq_f32_e32 vcc, 1.0, [[VAL]]12; GCN: v_add_f32_e32 [[ADD:v[0-9]+]], [[VAL]], [[VAL]]13; GCN: v_cndmask_b32_e32 [[RESULT:v[0-9]+]], [[ADD]], [[VAL]], vcc14; GCN: buffer_store_dword [[RESULT]]15define amdgpu_kernel void @test_vccnz_ifcvt_triangle(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {16entry:17 %v = load float, ptr addrspace(1) %in18 %cc = fcmp oeq float %v, 1.000000e+0019 br i1 %cc, label %if, label %endif20 21if:22 %u = fadd float %v, %v23 br label %endif24 25endif:26 %r = phi float [ %v, %entry ], [ %u, %if ]27 store float %r, ptr addrspace(1) %out28 ret void29}30 31; GCN-LABEL: {{^}}test_vccnz_ifcvt_diamond:32; GCN: buffer_load_dword [[VAL:v[0-9]+]]33; GCN: v_cmp_neq_f32_e32 vcc, 1.0, [[VAL]]34; GCN-DAG: v_add_f32_e32 [[ADD:v[0-9]+]], [[VAL]], [[VAL]]35; GCN-DAG: v_mul_f32_e32 [[MUL:v[0-9]+]], [[VAL]], [[VAL]]36; GCN: buffer_store_dword [[MUL]]37define amdgpu_kernel void @test_vccnz_ifcvt_diamond(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {38entry:39 %v = load float, ptr addrspace(1) %in40 %cc = fcmp oeq float %v, 1.000000e+0041 br i1 %cc, label %if, label %else42 43if:44 %u0 = fadd float %v, %v45 br label %endif46 47else:48 %u1 = fmul float %v, %v49 br label %endif50 51endif:52 %r = phi float [ %u0, %if ], [ %u1, %else ]53 store float %r, ptr addrspace(1) %out54 ret void55}56 57; GCN-LABEL: {{^}}test_vccnz_ifcvt_triangle_vcc_clobber:58; GCN: s_cbranch_vccnz59; GCN: ; clobber vcc60define amdgpu_kernel void @test_vccnz_ifcvt_triangle_vcc_clobber(ptr addrspace(1) %out, ptr addrspace(1) %in, float %k) #0 {61entry:62 %v = load i32, ptr addrspace(1) %in63 %cc = fcmp oeq float %k, 1.000000e+0064 br i1 %cc, label %if, label %endif65 66if:67 call void asm "; clobber $0", "~{vcc}"() #068 %u = add i32 %v, %v69 br label %endif70 71endif:72 %r = phi i32 [ %v, %entry ], [ %u, %if ]73 store i32 %r, ptr addrspace(1) %out74 ret void75}76 77; Longest chain of cheap instructions to convert78; GCN-LABEL: {{^}}test_vccnz_ifcvt_triangle_max_cheap:79; GCN: v_mul_f3280; GCN: v_mul_f3281; GCN: v_mul_f3282; GCN: v_mul_f3283; GCN: v_mul_f3284; GCN: v_mul_f3285; GCN: v_mul_f3286; GCN: v_mul_f3287; GCN: v_mul_f3288; GCN: v_cndmask_b32_e3289define amdgpu_kernel void @test_vccnz_ifcvt_triangle_max_cheap(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {90entry:91 %v = load float, ptr addrspace(1) %in92 %cc = fcmp oeq float %v, 1.000000e+0093 br i1 %cc, label %if, label %endif94 95if:96 %u.0 = fmul float %v, %v97 %u.1 = fmul float %v, %u.098 %u.2 = fmul float %v, %u.199 %u.3 = fmul float %v, %u.2100 %u.4 = fmul float %v, %u.3101 %u.5 = fmul float %v, %u.4102 %u.6 = fmul float %v, %u.5103 %u.7 = fmul float %v, %u.6104 %u.8 = fmul float %v, %u.7105 br label %endif106 107endif:108 %r = phi float [ %v, %entry ], [ %u.8, %if ]109 store float %r, ptr addrspace(1) %out110 ret void111}112 113; Short chain of cheap instructions to not convert114; GCN-LABEL: {{^}}test_vccnz_ifcvt_triangle_min_expensive:115; GCN: s_cbranch_vccnz [[ENDIF:.LBB[0-9]+_[0-9]+]]116 117; GCN: v_mul_f32118; GCN: v_mul_f32119; GCN: v_mul_f32120; GCN: v_mul_f32121; GCN: v_mul_f32122; GCN: v_mul_f32123; GCN: v_mul_f32124; GCN: v_mul_f32125; GCN: v_mul_f32126; GCN: v_mul_f32127 128; GCN: [[ENDIF]]:129; GCN: buffer_store_dword130define amdgpu_kernel void @test_vccnz_ifcvt_triangle_min_expensive(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {131entry:132 %v = load float, ptr addrspace(1) %in133 %cc = fcmp oeq float %v, 1.000000e+00134 br i1 %cc, label %if, label %endif135 136if:137 %u.0 = fmul float %v, %v138 %u.1 = fmul float %v, %u.0139 %u.2 = fmul float %v, %u.1140 %u.3 = fmul float %v, %u.2141 %u.4 = fmul float %v, %u.3142 %u.5 = fmul float %v, %u.4143 %u.6 = fmul float %v, %u.5144 %u.7 = fmul float %v, %u.6145 %u.8 = fmul float %v, %u.7146 %u.9 = fmul float %v, %u.8147 br label %endif148 149endif:150 %r = phi float [ %v, %entry ], [ %u.9, %if ]151 store float %r, ptr addrspace(1) %out152 ret void153}154 155; Should still branch over fdiv expansion156; GCN-LABEL: {{^}}test_vccnz_ifcvt_triangle_expensive:157; GCN: v_cmp_neq_f32_e32158; GCN: s_cbranch_vccnz [[ENDIF:.LBB[0-9]+_[0-9]+]]159 160; GCN: v_div_scale_f32161 162; GCN: [[ENDIF]]:163; GCN: buffer_store_dword164define amdgpu_kernel void @test_vccnz_ifcvt_triangle_expensive(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {165entry:166 %v = load float, ptr addrspace(1) %in167 %cc = fcmp oeq float %v, 1.000000e+00168 br i1 %cc, label %if, label %endif169 170if:171 %u = fdiv float %v, %v172 br label %endif173 174endif:175 %r = phi float [ %v, %entry ], [ %u, %if ]176 store float %r, ptr addrspace(1) %out177 ret void178}179 180; vcc branch with SGPR inputs181; GCN-LABEL: {{^}}test_vccnz_sgpr_ifcvt_triangle:182; GCN: v_cmp_neq_f32_e64183; GCN: s_cbranch_vccnz [[ENDIF:.LBB[0-9]+_[0-9]+]]184 185; GCN: s_add_i32186 187; GCN: [[ENDIF]]:188; GCN: buffer_store_dword189define amdgpu_kernel void @test_vccnz_sgpr_ifcvt_triangle(ptr addrspace(1) %out, ptr addrspace(4) %in, float %cnd) #0 {190entry:191 %v = load i32, ptr addrspace(4) %in192 %cc = fcmp oeq float %cnd, 1.000000e+00193 br i1 %cc, label %if, label %endif194 195if:196 %u = add i32 %v, %v197 br label %endif198 199endif:200 %r = phi i32 [ %v, %entry ], [ %u, %if ]201 store i32 %r, ptr addrspace(1) %out202 ret void203 204}205 206; GCN-LABEL: {{^}}test_vccnz_ifcvt_triangle_constant_load:207; GCN: v_cndmask_b32208define amdgpu_kernel void @test_vccnz_ifcvt_triangle_constant_load(ptr addrspace(1) %out, ptr addrspace(4) %in) #0 {209entry:210 %v = load float, ptr addrspace(4) %in211 %cc = fcmp oeq float %v, 1.000000e+00212 br i1 %cc, label %if, label %endif213 214if:215 %u = fadd float %v, %v216 br label %endif217 218endif:219 %r = phi float [ %v, %entry ], [ %u, %if ]220 store float %r, ptr addrspace(1) %out221 ret void222}223 224; Due to broken cost heuristic, this is not if converted like225; test_vccnz_ifcvt_triangle_constant_load even though it should be.226 227; GCN-LABEL: {{^}}test_vccnz_ifcvt_triangle_argload:228; GCN: v_cndmask_b32229define amdgpu_kernel void @test_vccnz_ifcvt_triangle_argload(ptr addrspace(1) %out, float %v) #0 {230entry:231 %cc = fcmp oeq float %v, 1.000000e+00232 br i1 %cc, label %if, label %endif233 234if:235 %u = fadd float %v, %v236 br label %endif237 238endif:239 %r = phi float [ %v, %entry ], [ %u, %if ]240 store float %r, ptr addrspace(1) %out241 ret void242}243 244; Scalar branch and scalar inputs245; GCN-LABEL: {{^}}test_scc1_sgpr_ifcvt_triangle:246; GCN: s_load_dword [[VAL:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x0247; GCN: s_add_i32 [[ADD:s[0-9]+]], [[VAL]], [[VAL]]248; GCN: s_cmp_lg_u32 s{{[0-9]+}}, 1249; GCN-NEXT: s_cselect_b32 [[SELECT:s[0-9]+]], [[VAL]], [[ADD]]250define amdgpu_kernel void @test_scc1_sgpr_ifcvt_triangle(ptr addrspace(4) %in, i32 %cond) #0 {251entry:252 %v = load i32, ptr addrspace(4) %in253 %cc = icmp eq i32 %cond, 1254 br i1 %cc, label %if, label %endif255 256if:257 %u = add i32 %v, %v258 br label %endif259 260endif:261 %r = phi i32 [ %v, %entry ], [ %u, %if ]262 call void asm sideeffect "; reg use $0", "s"(i32 %r) #0263 ret void264}265 266; FIXME: Should be able to use VALU compare and select267; Scalar branch but VGPR select operands268; GCN-LABEL: {{^}}test_scc1_vgpr_ifcvt_triangle:269; GCN: s_cmp_lg_u32270; GCN: s_cbranch_scc1 [[ENDIF:.LBB[0-9]+_[0-9]+]]271 272; GCN: v_add_f32_e32273 274; GCN: [[ENDIF]]:275; GCN: buffer_store_dword276define amdgpu_kernel void @test_scc1_vgpr_ifcvt_triangle(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %cond) #0 {277entry:278 %v = load float, ptr addrspace(1) %in279 %cc = icmp eq i32 %cond, 1280 br i1 %cc, label %if, label %endif281 282if:283 %u = fadd float %v, %v284 br label %endif285 286endif:287 %r = phi float [ %v, %entry ], [ %u, %if ]288 store float %r, ptr addrspace(1) %out289 ret void290}291 292; GCN-LABEL: {{^}}test_scc1_sgpr_ifcvt_triangle64:293; GCN: s_add_u32294; GCN: s_addc_u32295; GCN: s_cmp_lg_u32 s{{[0-9]+}}, 1296; GCN-NEXT: s_cselect_b64 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}297define amdgpu_kernel void @test_scc1_sgpr_ifcvt_triangle64(ptr addrspace(4) %in, i32 %cond) #0 {298entry:299 %v = load i64, ptr addrspace(4) %in300 %cc = icmp eq i32 %cond, 1301 br i1 %cc, label %if, label %endif302 303if:304 %u = add i64 %v, %v305 br label %endif306 307endif:308 %r = phi i64 [ %v, %entry ], [ %u, %if ]309 call void asm sideeffect "; reg use $0", "s"(i64 %r) #0310 ret void311}312 313; TODO: Can do s_cselect_b64; s_cselect_b32314; GCN-LABEL: {{^}}test_scc1_sgpr_ifcvt_triangle96:315; GCN: s_add_i32316; GCN: s_add_i32317; GCN: s_add_i32318; GCN: s_cmp_lg_u32 s{{[0-9]+}}, 1319; GCN-NEXT: s_cselect_b32 s320; GCN-NEXT: s_cselect_b32 s321; GCN-NEXT: s_cselect_b32 s322define amdgpu_kernel void @test_scc1_sgpr_ifcvt_triangle96(ptr addrspace(4) %in, i32 %cond) #0 {323entry:324 %v = load <3 x i32>, ptr addrspace(4) %in325 %cc = icmp eq i32 %cond, 1326 br i1 %cc, label %if, label %endif327 328if:329 %u = add <3 x i32> %v, %v330 br label %endif331 332endif:333 %r = phi <3 x i32> [ %v, %entry ], [ %u, %if ]334 %r.ext = shufflevector <3 x i32> %r, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>335 call void asm sideeffect "; reg use $0", "s"(<4 x i32> %r.ext) #0336 ret void337}338 339; GCN-LABEL: {{^}}test_scc1_sgpr_ifcvt_triangle128:340; GCN: s_add_i32341; GCN: s_add_i32342; GCN: s_add_i32343; GCN: s_add_i32344; GCN: s_cmp_lg_u32 s{{[0-9]+}}, 1345; GCN-NEXT: s_cselect_b64 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}346; GCN-NEXT: s_cselect_b64 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}347define amdgpu_kernel void @test_scc1_sgpr_ifcvt_triangle128(ptr addrspace(4) %in, i32 %cond) #0 {348entry:349 %v = load <4 x i32>, ptr addrspace(4) %in350 %cc = icmp eq i32 %cond, 1351 br i1 %cc, label %if, label %endif352 353if:354 %u = add <4 x i32> %v, %v355 br label %endif356 357endif:358 %r = phi <4 x i32> [ %v, %entry ], [ %u, %if ]359 call void asm sideeffect "; reg use $0", "s"(<4 x i32> %r) #0360 ret void361}362 363; GCN-LABEL: {{^}}uniform_if_swap_br_targets_scc_constant_select:364; GCN: s_cmp_lg_u32 s{{[0-9]+}}, 0365; GCN: s_cselect_b32 s{{[0-9]+}}, 0, 1{{$}}366define amdgpu_kernel void @uniform_if_swap_br_targets_scc_constant_select(i32 %cond, ptr addrspace(1) %out) {367entry:368 %cmp0 = icmp eq i32 %cond, 0369 br i1 %cmp0, label %else, label %if370 371if:372 br label %done373 374else:375 br label %done376 377done:378 %value = phi i32 [0, %if], [1, %else]379 store i32 %value, ptr addrspace(1) %out380 ret void381}382 383; GCN-LABEL: {{^}}ifcvt_undef_scc:384; GCN: {{^}}; %bb.0:385; GCN-NEXT: s_load_dwordx2386; GCN-NEXT: s_cselect_b32 s{{[0-9]+}}, 0, 1{{$}}387define amdgpu_kernel void @ifcvt_undef_scc(i32 %cond, ptr addrspace(1) %out) {388entry:389 br i1 poison, label %else, label %if390 391if:392 br label %done393 394else:395 br label %done396 397done:398 %value = phi i32 [0, %if], [1, %else]399 store i32 %value, ptr addrspace(1) %out400 ret void401}402 403; GCN-LABEL: {{^}}test_vccnz_ifcvt_triangle256:404; GCN: v_cmp_neq_f32405; GCN: s_cbranch_vccnz [[ENDIF:.LBB[0-9]+_[0-9]+]]406 407; GCN: v_add_i32408; GCN: v_add_i32409 410; GCN: [[ENDIF]]:411; GCN: buffer_store_dword412define amdgpu_kernel void @test_vccnz_ifcvt_triangle256(ptr addrspace(1) %out, ptr addrspace(1) %in, float %cnd) #0 {413entry:414 %v = load <8 x i32>, ptr addrspace(1) %in415 %cc = fcmp oeq float %cnd, 1.000000e+00416 br i1 %cc, label %if, label %endif417 418if:419 %u = add <8 x i32> %v, %v420 br label %endif421 422endif:423 %r = phi <8 x i32> [ %v, %entry ], [ %u, %if ]424 store <8 x i32> %r, ptr addrspace(1) %out425 ret void426}427 428; GCN-LABEL: {{^}}test_vccnz_ifcvt_triangle512:429; GCN: v_cmp_neq_f32430; GCN: s_cbranch_vccnz [[ENDIF:.LBB[0-9]+_[0-9]+]]431 432; GCN: v_add_i32433; GCN: v_add_i32434 435; GCN: [[ENDIF]]:436; GCN: buffer_store_dword437define amdgpu_kernel void @test_vccnz_ifcvt_triangle512(ptr addrspace(1) %out, ptr addrspace(1) %in, float %cnd) #0 {438entry:439 %v = load <16 x i32>, ptr addrspace(1) %in440 %cc = fcmp oeq float %cnd, 1.000000e+00441 br i1 %cc, label %if, label %endif442 443if:444 %u = add <16 x i32> %v, %v445 br label %endif446 447endif:448 %r = phi <16 x i32> [ %v, %entry ], [ %u, %if ]449 store <16 x i32> %r, ptr addrspace(1) %out450 ret void451}452 453attributes #0 = { nounwind }454