464 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -global-isel < %s | FileCheck %s3 4declare i64 @llvm.amdgcn.ballot.i64(i1)5declare i64 @llvm.ctpop.i64(i64)6 7; Test ballot(0)8 9define amdgpu_cs i64 @constant_false() {10; CHECK-LABEL: constant_false:11; CHECK: ; %bb.0:12; CHECK-NEXT: s_mov_b32 s0, 013; CHECK-NEXT: s_mov_b32 s1, 014; CHECK-NEXT: ; return to shader part epilog15 %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 0)16 ret i64 %ballot17}18 19; Test ballot(1)20 21define amdgpu_cs i64 @constant_true() {22; CHECK-LABEL: constant_true:23; CHECK: ; %bb.0:24; CHECK-NEXT: s_mov_b32 s0, exec_lo25; CHECK-NEXT: s_mov_b32 s1, exec_hi26; CHECK-NEXT: ; return to shader part epilog27 %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 1)28 ret i64 %ballot29}30 31; Test ballot of a non-comparison operation32 33define amdgpu_cs i64 @non_compare(i32 %x) {34; CHECK-LABEL: non_compare:35; CHECK: ; %bb.0:36; CHECK-NEXT: v_and_b32_e32 v0, 1, v037; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v038; CHECK-NEXT: s_and_b64 s[0:1], vcc, exec39; CHECK-NEXT: ; return to shader part epilog40 %trunc = trunc i32 %x to i141 %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %trunc)42 ret i64 %ballot43}44 45; Test ballot of comparisons46 47define amdgpu_cs i64 @compare_ints(i32 %x, i32 %y) {48; CHECK-LABEL: compare_ints:49; CHECK: ; %bb.0:50; CHECK-NEXT: v_cmp_eq_u32_e64 s[0:1], v0, v151; CHECK-NEXT: ; return to shader part epilog52 %cmp = icmp eq i32 %x, %y53 %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %cmp)54 ret i64 %ballot55}56 57define amdgpu_cs i64 @compare_int_with_constant(i32 %x) {58; CHECK-LABEL: compare_int_with_constant:59; CHECK: ; %bb.0:60; CHECK-NEXT: v_mov_b32_e32 v1, 0x6361; CHECK-NEXT: v_cmp_ge_i32_e64 s[0:1], v0, v162; CHECK-NEXT: ; return to shader part epilog63 %cmp = icmp sge i32 %x, 9964 %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %cmp)65 ret i64 %ballot66}67 68define amdgpu_cs i64 @compare_floats(float %x, float %y) {69; CHECK-LABEL: compare_floats:70; CHECK: ; %bb.0:71; CHECK-NEXT: v_cmp_gt_f32_e64 s[0:1], v0, v172; CHECK-NEXT: ; return to shader part epilog73 %cmp = fcmp ogt float %x, %y74 %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %cmp)75 ret i64 %ballot76}77 78define amdgpu_cs i64 @ctpop_of_ballot(float %x, float %y) {79; CHECK-LABEL: ctpop_of_ballot:80; CHECK: ; %bb.0:81; CHECK-NEXT: v_cmp_gt_f32_e32 vcc, v0, v182; CHECK-NEXT: s_bcnt1_i32_b64 s0, vcc83; CHECK-NEXT: s_mov_b32 s1, 084; CHECK-NEXT: ; return to shader part epilog85 %cmp = fcmp ogt float %x, %y86 %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %cmp)87 %bcnt = call i64 @llvm.ctpop.i64(i64 %ballot)88 ret i64 %bcnt89}90 91define amdgpu_cs i32 @branch_divergent_ballot_ne_zero_non_compare(i32 %v) {92; CHECK-LABEL: branch_divergent_ballot_ne_zero_non_compare:93; CHECK: ; %bb.0:94; CHECK-NEXT: v_and_b32_e32 v0, 1, v095; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v096; CHECK-NEXT: s_and_b64 s[0:1], vcc, exec97; CHECK-NEXT: s_cmp_eq_u64 s[0:1], 098; CHECK-NEXT: s_cbranch_scc1 .LBB7_299; CHECK-NEXT: ; %bb.1: ; %true100; CHECK-NEXT: s_mov_b32 s0, 42101; CHECK-NEXT: s_branch .LBB7_3102; CHECK-NEXT: .LBB7_2: ; %false103; CHECK-NEXT: s_mov_b32 s0, 33104; CHECK-NEXT: s_branch .LBB7_3105; CHECK-NEXT: .LBB7_3:106 %c = trunc i32 %v to i1107 %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %c)108 %ballot_ne_zero = icmp ne i64 %ballot, 0109 br i1 %ballot_ne_zero, label %true, label %false110true:111 ret i32 42112false:113 ret i32 33114}115 116define amdgpu_cs i32 @branch_uniform_ballot_ne_zero_non_compare(i32 inreg %v) {117; CHECK-LABEL: branch_uniform_ballot_ne_zero_non_compare:118; CHECK: ; %bb.0:119; CHECK-NEXT: s_xor_b32 s0, s0, 1120; CHECK-NEXT: s_and_b32 s0, s0, 1121; CHECK-NEXT: s_cmp_lg_u32 s0, 0122; CHECK-NEXT: s_cbranch_scc1 .LBB8_2123; CHECK-NEXT: ; %bb.1: ; %true124; CHECK-NEXT: s_mov_b32 s0, 42125; CHECK-NEXT: s_branch .LBB8_3126; CHECK-NEXT: .LBB8_2: ; %false127; CHECK-NEXT: s_mov_b32 s0, 33128; CHECK-NEXT: s_branch .LBB8_3129; CHECK-NEXT: .LBB8_3:130 %c = trunc i32 %v to i1131 %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %c)132 %ballot_ne_zero = icmp ne i64 %ballot, 0133 br i1 %ballot_ne_zero, label %true, label %false134true:135 ret i32 42136false:137 ret i32 33138}139 140define amdgpu_cs i32 @branch_divergent_ballot_eq_zero_non_compare(i32 %v) {141; CHECK-LABEL: branch_divergent_ballot_eq_zero_non_compare:142; CHECK: ; %bb.0:143; CHECK-NEXT: v_and_b32_e32 v0, 1, v0144; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0145; CHECK-NEXT: s_and_b64 s[0:1], vcc, exec146; CHECK-NEXT: s_cbranch_scc0 .LBB9_2147; CHECK-NEXT: ; %bb.1: ; %false148; CHECK-NEXT: s_mov_b32 s0, 33149; CHECK-NEXT: s_branch .LBB9_3150; CHECK-NEXT: .LBB9_2: ; %true151; CHECK-NEXT: s_mov_b32 s0, 42152; CHECK-NEXT: s_branch .LBB9_3153; CHECK-NEXT: .LBB9_3:154 %c = trunc i32 %v to i1155 %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %c)156 %ballot_eq_zero = icmp eq i64 %ballot, 0157 br i1 %ballot_eq_zero, label %true, label %false158true:159 ret i32 42160false:161 ret i32 33162}163 164define amdgpu_cs i32 @branch_uniform_ballot_eq_zero_non_compare(i32 inreg %v) {165; CHECK-LABEL: branch_uniform_ballot_eq_zero_non_compare:166; CHECK: ; %bb.0:167; CHECK-NEXT: s_xor_b32 s0, s0, 1168; CHECK-NEXT: s_xor_b32 s0, s0, 1169; CHECK-NEXT: s_and_b32 s0, s0, 1170; CHECK-NEXT: s_cmp_lg_u32 s0, 0171; CHECK-NEXT: s_cbranch_scc1 .LBB10_2172; CHECK-NEXT: ; %bb.1: ; %true173; CHECK-NEXT: s_mov_b32 s0, 42174; CHECK-NEXT: s_branch .LBB10_3175; CHECK-NEXT: .LBB10_2: ; %false176; CHECK-NEXT: s_mov_b32 s0, 33177; CHECK-NEXT: s_branch .LBB10_3178; CHECK-NEXT: .LBB10_3:179 %c = trunc i32 %v to i1180 %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %c)181 %ballot_eq_zero = icmp eq i64 %ballot, 0182 br i1 %ballot_eq_zero, label %true, label %false183true:184 ret i32 42185false:186 ret i32 33187}188 189define amdgpu_cs i32 @branch_divergent_ballot_ne_zero_compare(i32 %v) {190; CHECK-LABEL: branch_divergent_ballot_ne_zero_compare:191; CHECK: ; %bb.0:192; CHECK-NEXT: v_cmp_gt_u32_e32 vcc, 12, v0193; CHECK-NEXT: s_cmp_eq_u64 vcc, 0194; CHECK-NEXT: s_cbranch_scc1 .LBB11_2195; CHECK-NEXT: ; %bb.1: ; %true196; CHECK-NEXT: s_mov_b32 s0, 42197; CHECK-NEXT: s_branch .LBB11_3198; CHECK-NEXT: .LBB11_2: ; %false199; CHECK-NEXT: s_mov_b32 s0, 33200; CHECK-NEXT: s_branch .LBB11_3201; CHECK-NEXT: .LBB11_3:202 %c = icmp ult i32 %v, 12203 %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %c)204 %ballot_ne_zero = icmp ne i64 %ballot, 0205 br i1 %ballot_ne_zero, label %true, label %false206true:207 ret i32 42208false:209 ret i32 33210}211 212define amdgpu_cs i32 @branch_uniform_ballot_ne_zero_compare(i32 inreg %v) {213; CHECK-LABEL: branch_uniform_ballot_ne_zero_compare:214; CHECK: ; %bb.0:215; CHECK-NEXT: s_cmp_ge_u32 s0, 12216; CHECK-NEXT: s_cbranch_scc1 .LBB12_2217; CHECK-NEXT: ; %bb.1: ; %true218; CHECK-NEXT: s_mov_b32 s0, 42219; CHECK-NEXT: s_branch .LBB12_3220; CHECK-NEXT: .LBB12_2: ; %false221; CHECK-NEXT: s_mov_b32 s0, 33222; CHECK-NEXT: s_branch .LBB12_3223; CHECK-NEXT: .LBB12_3:224 %c = icmp ult i32 %v, 12225 %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %c)226 %ballot_ne_zero = icmp ne i64 %ballot, 0227 br i1 %ballot_ne_zero, label %true, label %false228true:229 ret i32 42230false:231 ret i32 33232}233 234define amdgpu_cs i32 @branch_divergent_ballot_eq_zero_compare(i32 %v) {235; CHECK-LABEL: branch_divergent_ballot_eq_zero_compare:236; CHECK: ; %bb.0:237; CHECK-NEXT: v_cmp_gt_u32_e32 vcc, 12, v0238; CHECK-NEXT: s_cmp_lg_u64 vcc, 0239; CHECK-NEXT: s_cbranch_scc0 .LBB13_2240; CHECK-NEXT: ; %bb.1: ; %false241; CHECK-NEXT: s_mov_b32 s0, 33242; CHECK-NEXT: s_branch .LBB13_3243; CHECK-NEXT: .LBB13_2: ; %true244; CHECK-NEXT: s_mov_b32 s0, 42245; CHECK-NEXT: s_branch .LBB13_3246; CHECK-NEXT: .LBB13_3:247 %c = icmp ult i32 %v, 12248 %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %c)249 %ballot_eq_zero = icmp eq i64 %ballot, 0250 br i1 %ballot_eq_zero, label %true, label %false251true:252 ret i32 42253false:254 ret i32 33255}256 257define amdgpu_cs i32 @branch_uniform_ballot_eq_zero_compare(i32 inreg %v) {258; CHECK-LABEL: branch_uniform_ballot_eq_zero_compare:259; CHECK: ; %bb.0:260; CHECK-NEXT: s_cmp_lt_u32 s0, 12261; CHECK-NEXT: s_cbranch_scc1 .LBB14_2262; CHECK-NEXT: ; %bb.1: ; %true263; CHECK-NEXT: s_mov_b32 s0, 42264; CHECK-NEXT: s_branch .LBB14_3265; CHECK-NEXT: .LBB14_2: ; %false266; CHECK-NEXT: s_mov_b32 s0, 33267; CHECK-NEXT: s_branch .LBB14_3268; CHECK-NEXT: .LBB14_3:269 %c = icmp ult i32 %v, 12270 %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %c)271 %ballot_eq_zero = icmp eq i64 %ballot, 0272 br i1 %ballot_eq_zero, label %true, label %false273true:274 ret i32 42275false:276 ret i32 33277}278 279define amdgpu_cs i32 @branch_divergent_ballot_ne_zero_and(i32 %v1, i32 %v2) {280; CHECK-LABEL: branch_divergent_ballot_ne_zero_and:281; CHECK: ; %bb.0:282; CHECK-NEXT: v_cmp_gt_u32_e32 vcc, 12, v0283; CHECK-NEXT: v_cmp_lt_u32_e64 s[0:1], 34, v1284; CHECK-NEXT: s_and_b64 s[0:1], vcc, s[0:1]285; CHECK-NEXT: s_cmp_eq_u64 s[0:1], 0286; CHECK-NEXT: s_cbranch_scc1 .LBB15_2287; CHECK-NEXT: ; %bb.1: ; %true288; CHECK-NEXT: s_mov_b32 s0, 42289; CHECK-NEXT: s_branch .LBB15_3290; CHECK-NEXT: .LBB15_2: ; %false291; CHECK-NEXT: s_mov_b32 s0, 33292; CHECK-NEXT: s_branch .LBB15_3293; CHECK-NEXT: .LBB15_3:294 %v1c = icmp ult i32 %v1, 12295 %v2c = icmp ugt i32 %v2, 34296 %c = and i1 %v1c, %v2c297 %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %c)298 %ballot_ne_zero = icmp ne i64 %ballot, 0299 br i1 %ballot_ne_zero, label %true, label %false300true:301 ret i32 42302false:303 ret i32 33304}305 306define amdgpu_cs i32 @branch_uniform_ballot_ne_zero_and(i32 inreg %v1, i32 inreg %v2) {307; CHECK-LABEL: branch_uniform_ballot_ne_zero_and:308; CHECK: ; %bb.0:309; CHECK-NEXT: s_cmp_ge_u32 s0, 12310; CHECK-NEXT: s_cselect_b32 s0, 1, 0311; CHECK-NEXT: s_cmp_le_u32 s1, 34312; CHECK-NEXT: s_cselect_b32 s1, 1, 0313; CHECK-NEXT: s_or_b32 s0, s0, s1314; CHECK-NEXT: s_cmp_lg_u32 s0, 0315; CHECK-NEXT: s_cbranch_scc1 .LBB16_2316; CHECK-NEXT: ; %bb.1: ; %true317; CHECK-NEXT: s_mov_b32 s0, 42318; CHECK-NEXT: s_branch .LBB16_3319; CHECK-NEXT: .LBB16_2: ; %false320; CHECK-NEXT: s_mov_b32 s0, 33321; CHECK-NEXT: s_branch .LBB16_3322; CHECK-NEXT: .LBB16_3:323 %v1c = icmp ult i32 %v1, 12324 %v2c = icmp ugt i32 %v2, 34325 %c = and i1 %v1c, %v2c326 %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %c)327 %ballot_ne_zero = icmp ne i64 %ballot, 0328 br i1 %ballot_ne_zero, label %true, label %false329true:330 ret i32 42331false:332 ret i32 33333}334 335define amdgpu_cs i32 @branch_divergent_ballot_eq_zero_and(i32 %v1, i32 %v2) {336; CHECK-LABEL: branch_divergent_ballot_eq_zero_and:337; CHECK: ; %bb.0:338; CHECK-NEXT: v_cmp_gt_u32_e32 vcc, 12, v0339; CHECK-NEXT: v_cmp_lt_u32_e64 s[0:1], 34, v1340; CHECK-NEXT: s_and_b64 s[0:1], vcc, s[0:1]341; CHECK-NEXT: s_cbranch_scc0 .LBB17_2342; CHECK-NEXT: ; %bb.1: ; %false343; CHECK-NEXT: s_mov_b32 s0, 33344; CHECK-NEXT: s_branch .LBB17_3345; CHECK-NEXT: .LBB17_2: ; %true346; CHECK-NEXT: s_mov_b32 s0, 42347; CHECK-NEXT: s_branch .LBB17_3348; CHECK-NEXT: .LBB17_3:349 %v1c = icmp ult i32 %v1, 12350 %v2c = icmp ugt i32 %v2, 34351 %c = and i1 %v1c, %v2c352 %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %c)353 %ballot_eq_zero = icmp eq i64 %ballot, 0354 br i1 %ballot_eq_zero, label %true, label %false355true:356 ret i32 42357false:358 ret i32 33359}360 361define amdgpu_cs i32 @branch_uniform_ballot_eq_zero_and(i32 inreg %v1, i32 inreg %v2) {362; CHECK-LABEL: branch_uniform_ballot_eq_zero_and:363; CHECK: ; %bb.0:364; CHECK-NEXT: s_cmp_lt_u32 s0, 12365; CHECK-NEXT: s_cselect_b32 s0, 1, 0366; CHECK-NEXT: s_cmp_gt_u32 s1, 34367; CHECK-NEXT: s_cselect_b32 s1, 1, 0368; CHECK-NEXT: s_and_b32 s0, s0, s1369; CHECK-NEXT: s_cmp_lg_u32 s0, 0370; CHECK-NEXT: s_cbranch_scc1 .LBB18_2371; CHECK-NEXT: ; %bb.1: ; %true372; CHECK-NEXT: s_mov_b32 s0, 42373; CHECK-NEXT: s_branch .LBB18_3374; CHECK-NEXT: .LBB18_2: ; %false375; CHECK-NEXT: s_mov_b32 s0, 33376; CHECK-NEXT: s_branch .LBB18_3377; CHECK-NEXT: .LBB18_3:378 %v1c = icmp ult i32 %v1, 12379 %v2c = icmp ugt i32 %v2, 34380 %c = and i1 %v1c, %v2c381 %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %c)382 %ballot_eq_zero = icmp eq i64 %ballot, 0383 br i1 %ballot_eq_zero, label %true, label %false384true:385 ret i32 42386false:387 ret i32 33388}389 390define amdgpu_cs i32 @branch_uniform_ballot_sgt_N_compare(i32 inreg %v) {391; CHECK-LABEL: branch_uniform_ballot_sgt_N_compare:392; CHECK: ; %bb.0:393; CHECK-NEXT: s_cmp_lt_u32 s0, 12394; CHECK-NEXT: s_cselect_b32 s0, 1, 0395; CHECK-NEXT: s_and_b32 s0, 1, s0396; CHECK-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, s0397; CHECK-NEXT: v_cmp_le_i64_e64 vcc, s[0:1], 22398; CHECK-NEXT: s_cbranch_vccnz .LBB19_2399; CHECK-NEXT: ; %bb.1: ; %true400; CHECK-NEXT: s_mov_b32 s0, 42401; CHECK-NEXT: s_branch .LBB19_3402; CHECK-NEXT: .LBB19_2: ; %false403; CHECK-NEXT: s_mov_b32 s0, 33404; CHECK-NEXT: s_branch .LBB19_3405; CHECK-NEXT: .LBB19_3:406 %c = icmp ult i32 %v, 12407 %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %c)408 %bc = icmp sgt i64 %ballot, 22409 br i1 %bc, label %true, label %false410true:411 ret i32 42412false:413 ret i32 33414}415 416; Input that is not constant or direct result of a compare.417; Tests setting 0 to inactive lanes.418define amdgpu_ps void @non_cst_non_compare_input(ptr addrspace(1) %out, i32 %tid, i32 %cond) {419; CHECK-LABEL: non_cst_non_compare_input:420; CHECK: ; %bb.0: ; %entry421; CHECK-NEXT: s_and_b32 s0, 1, s0422; CHECK-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3423; CHECK-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, s0424; CHECK-NEXT: s_and_saveexec_b64 s[2:3], vcc425; CHECK-NEXT: s_xor_b64 s[2:3], exec, s[2:3]426; CHECK-NEXT: ; %bb.1: ; %B427; CHECK-NEXT: v_cmp_gt_u32_e32 vcc, 2, v2428; CHECK-NEXT: s_andn2_b64 s[0:1], s[0:1], exec429; CHECK-NEXT: s_and_b64 s[4:5], exec, vcc430; CHECK-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]431; CHECK-NEXT: ; implicit-def: $vgpr2432; CHECK-NEXT: ; %bb.2: ; %Flow433; CHECK-NEXT: s_andn2_saveexec_b64 s[2:3], s[2:3]434; CHECK-NEXT: ; %bb.3: ; %A435; CHECK-NEXT: v_cmp_le_u32_e32 vcc, 1, v2436; CHECK-NEXT: s_andn2_b64 s[0:1], s[0:1], exec437; CHECK-NEXT: s_and_b64 s[4:5], exec, vcc438; CHECK-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]439; CHECK-NEXT: ; %bb.4: ; %exit440; CHECK-NEXT: s_or_b64 exec, exec, s[2:3]441; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec442; CHECK-NEXT: v_mov_b32_e32 v3, s1443; CHECK-NEXT: v_mov_b32_e32 v2, s0444; CHECK-NEXT: global_store_dwordx2 v[0:1], v[2:3], off445; CHECK-NEXT: s_endpgm446entry:447 %cmp = icmp eq i32 %cond, 0448 br i1 %cmp, label %A, label %B449 450A:451 %val_A = icmp uge i32 %tid, 1452 br label %exit453 454B:455 %val_B = icmp ult i32 %tid, 2456 br label %exit457 458exit:459 %phi = phi i1 [ %val_A, %A ], [ %val_B, %B ]460 %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %phi)461 store i64 %ballot, ptr addrspace(1) %out462 ret void463}464