364 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 22; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -mattr=+wavefrontsize64 -global-isel=1 < %s | FileCheck -check-prefix=GISEL_W64 %s3; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -mattr=+wavefrontsize64 -global-isel=0 < %s | FileCheck -check-prefix=SDAG_W64 %s4; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -mattr=+wavefrontsize32 -global-isel=1 < %s | FileCheck -check-prefix=GISEL_W32 %s5; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -mattr=+wavefrontsize32 -global-isel=0 < %s | FileCheck -check-prefix=SDAG_W32 %s6 7declare i1 @llvm.amdgcn.inverse.ballot.i64(i64)8 9; Test ballot(0)10define amdgpu_cs void @constant_false_inverse_ballot(ptr addrspace(1) %out) {11; GISEL_W64-LABEL: constant_false_inverse_ballot:12; GISEL_W64: ; %bb.0: ; %entry13; GISEL_W64-NEXT: s_mov_b64 s[0:1], 014; GISEL_W64-NEXT: v_mov_b32_e32 v3, 015; GISEL_W64-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]16; GISEL_W64-NEXT: global_store_b64 v[0:1], v[2:3], off17; GISEL_W64-NEXT: s_endpgm18;19; SDAG_W64-LABEL: constant_false_inverse_ballot:20; SDAG_W64: ; %bb.0: ; %entry21; SDAG_W64-NEXT: s_mov_b32 s2, 022; SDAG_W64-NEXT: s_mov_b64 s[0:1], 023; SDAG_W64-NEXT: v_mov_b32_e32 v3, s224; SDAG_W64-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]25; SDAG_W64-NEXT: global_store_b64 v[0:1], v[2:3], off26; SDAG_W64-NEXT: s_endpgm27;28; GISEL_W32-LABEL: constant_false_inverse_ballot:29; GISEL_W32: ; %bb.0: ; %entry30; GISEL_W32-NEXT: s_mov_b32 s0, 031; GISEL_W32-NEXT: v_mov_b32_e32 v3, 032; GISEL_W32-NEXT: v_cndmask_b32_e64 v2, 0, 1, s033; GISEL_W32-NEXT: global_store_b64 v[0:1], v[2:3], off34; GISEL_W32-NEXT: s_endpgm35;36; SDAG_W32-LABEL: constant_false_inverse_ballot:37; SDAG_W32: ; %bb.0: ; %entry38; SDAG_W32-NEXT: s_mov_b32 s0, 039; SDAG_W32-NEXT: v_cndmask_b32_e64 v2, 0, 1, s040; SDAG_W32-NEXT: v_mov_b32_e32 v3, s041; SDAG_W32-NEXT: global_store_b64 v[0:1], v[2:3], off42; SDAG_W32-NEXT: s_endpgm43entry:44 %ballot = call i1 @llvm.amdgcn.inverse.ballot.i64(i64 0)45 %sel = select i1 %ballot, i64 1, i64 046 store i64 %sel, ptr addrspace(1) %out47 ret void48}49 50; Test ballot(1)51 52define amdgpu_cs void @constant_true_inverse_ballot(ptr addrspace(1) %out) {53; GISEL_W64-LABEL: constant_true_inverse_ballot:54; GISEL_W64: ; %bb.0: ; %entry55; GISEL_W64-NEXT: s_mov_b64 s[0:1], -156; GISEL_W64-NEXT: v_mov_b32_e32 v3, 057; GISEL_W64-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]58; GISEL_W64-NEXT: global_store_b64 v[0:1], v[2:3], off59; GISEL_W64-NEXT: s_endpgm60;61; SDAG_W64-LABEL: constant_true_inverse_ballot:62; SDAG_W64: ; %bb.0: ; %entry63; SDAG_W64-NEXT: s_mov_b32 s2, 064; SDAG_W64-NEXT: s_mov_b64 s[0:1], -165; SDAG_W64-NEXT: v_mov_b32_e32 v3, s266; SDAG_W64-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]67; SDAG_W64-NEXT: global_store_b64 v[0:1], v[2:3], off68; SDAG_W64-NEXT: s_endpgm69;70; GISEL_W32-LABEL: constant_true_inverse_ballot:71; GISEL_W32: ; %bb.0: ; %entry72; GISEL_W32-NEXT: s_mov_b32 s0, -173; GISEL_W32-NEXT: v_mov_b32_e32 v3, 074; GISEL_W32-NEXT: v_cndmask_b32_e64 v2, 0, 1, s075; GISEL_W32-NEXT: global_store_b64 v[0:1], v[2:3], off76; GISEL_W32-NEXT: s_endpgm77;78; SDAG_W32-LABEL: constant_true_inverse_ballot:79; SDAG_W32: ; %bb.0: ; %entry80; SDAG_W32-NEXT: s_mov_b32 s0, 081; SDAG_W32-NEXT: s_mov_b32 s1, -182; SDAG_W32-NEXT: v_mov_b32_e32 v3, s083; SDAG_W32-NEXT: v_cndmask_b32_e64 v2, 0, 1, s184; SDAG_W32-NEXT: global_store_b64 v[0:1], v[2:3], off85; SDAG_W32-NEXT: s_endpgm86entry:87 %ballot = call i1 @llvm.amdgcn.inverse.ballot.i64(i64 u0xFFFFFFFFFFFFFFFF)88 %sel = select i1 %ballot, i64 1, i64 089 store i64 %sel, ptr addrspace(1) %out90 ret void91}92 93; Test ballot(u0x0040F8010000)94 95define amdgpu_cs void @constant_mask_inverse_ballot(ptr addrspace(1) %out) {96; GISEL_W64-LABEL: constant_mask_inverse_ballot:97; GISEL_W64: ; %bb.0: ; %entry98; GISEL_W64-NEXT: s_mov_b32 s0, 0xf801000099; GISEL_W64-NEXT: s_mov_b32 s1, 64100; GISEL_W64-NEXT: v_mov_b32_e32 v3, 0101; GISEL_W64-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]102; GISEL_W64-NEXT: global_store_b64 v[0:1], v[2:3], off103; GISEL_W64-NEXT: s_endpgm104;105; SDAG_W64-LABEL: constant_mask_inverse_ballot:106; SDAG_W64: ; %bb.0: ; %entry107; SDAG_W64-NEXT: s_mov_b32 s0, 0xf8010000108; SDAG_W64-NEXT: s_mov_b32 s1, 64109; SDAG_W64-NEXT: s_mov_b32 s2, 0110; SDAG_W64-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]111; SDAG_W64-NEXT: v_mov_b32_e32 v3, s2112; SDAG_W64-NEXT: global_store_b64 v[0:1], v[2:3], off113; SDAG_W64-NEXT: s_endpgm114;115; GISEL_W32-LABEL: constant_mask_inverse_ballot:116; GISEL_W32: ; %bb.0: ; %entry117; GISEL_W32-NEXT: s_mov_b32 s0, 0xf8010000118; GISEL_W32-NEXT: v_mov_b32_e32 v3, 0119; GISEL_W32-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0120; GISEL_W32-NEXT: global_store_b64 v[0:1], v[2:3], off121; GISEL_W32-NEXT: s_endpgm122;123; SDAG_W32-LABEL: constant_mask_inverse_ballot:124; SDAG_W32: ; %bb.0: ; %entry125; SDAG_W32-NEXT: s_mov_b32 s0, 0126; SDAG_W32-NEXT: s_mov_b32 s1, 0xf8010000127; SDAG_W32-NEXT: v_mov_b32_e32 v3, s0128; SDAG_W32-NEXT: v_cndmask_b32_e64 v2, 0, 1, s1129; SDAG_W32-NEXT: global_store_b64 v[0:1], v[2:3], off130; SDAG_W32-NEXT: s_endpgm131entry:132 %ballot = call i1 @llvm.amdgcn.inverse.ballot.i64(i64 u0x0040F8010000)133 %sel = select i1 %ballot, i64 1, i64 0134 store i64 %sel, ptr addrspace(1) %out135 ret void136}137 138; Test inverse ballot using a vgpr as input139 140define amdgpu_cs void @vgpr_inverse_ballot(i64 %input, ptr addrspace(1) %out) {141; GISEL_W64-LABEL: vgpr_inverse_ballot:142; GISEL_W64: ; %bb.0: ; %entry143; GISEL_W64-NEXT: v_readfirstlane_b32 s0, v0144; GISEL_W64-NEXT: v_readfirstlane_b32 s1, v1145; GISEL_W64-NEXT: v_mov_b32_e32 v5, 0146; GISEL_W64-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[0:1]147; GISEL_W64-NEXT: global_store_b64 v[2:3], v[4:5], off148; GISEL_W64-NEXT: s_endpgm149;150; SDAG_W64-LABEL: vgpr_inverse_ballot:151; SDAG_W64: ; %bb.0: ; %entry152; SDAG_W64-NEXT: v_readfirstlane_b32 s0, v0153; SDAG_W64-NEXT: v_readfirstlane_b32 s1, v1154; SDAG_W64-NEXT: s_mov_b32 s2, 0155; SDAG_W64-NEXT: v_mov_b32_e32 v1, s2156; SDAG_W64-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]157; SDAG_W64-NEXT: global_store_b64 v[2:3], v[0:1], off158; SDAG_W64-NEXT: s_endpgm159;160; GISEL_W32-LABEL: vgpr_inverse_ballot:161; GISEL_W32: ; %bb.0: ; %entry162; GISEL_W32-NEXT: v_readfirstlane_b32 s0, v0163; GISEL_W32-NEXT: v_mov_b32_e32 v1, 0164; GISEL_W32-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0165; GISEL_W32-NEXT: global_store_b64 v[2:3], v[0:1], off166; GISEL_W32-NEXT: s_endpgm167;168; SDAG_W32-LABEL: vgpr_inverse_ballot:169; SDAG_W32: ; %bb.0: ; %entry170; SDAG_W32-NEXT: v_readfirstlane_b32 s1, v0171; SDAG_W32-NEXT: s_mov_b32 s0, 0172; SDAG_W32-NEXT: v_mov_b32_e32 v1, s0173; SDAG_W32-NEXT: v_cndmask_b32_e64 v0, 0, 1, s1174; SDAG_W32-NEXT: global_store_b64 v[2:3], v[0:1], off175; SDAG_W32-NEXT: s_endpgm176entry:177 %ballot = call i1 @llvm.amdgcn.inverse.ballot.i64(i64 %input)178 %sel = select i1 %ballot, i64 1, i64 0179 store i64 %sel, ptr addrspace(1) %out180 ret void181}182 183define amdgpu_cs void @sgpr_inverse_ballot(i64 inreg %input, ptr addrspace(1) %out) {184; GISEL_W64-LABEL: sgpr_inverse_ballot:185; GISEL_W64: ; %bb.0: ; %entry186; GISEL_W64-NEXT: v_mov_b32_e32 v3, 0187; GISEL_W64-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]188; GISEL_W64-NEXT: global_store_b64 v[0:1], v[2:3], off189; GISEL_W64-NEXT: s_endpgm190;191; SDAG_W64-LABEL: sgpr_inverse_ballot:192; SDAG_W64: ; %bb.0: ; %entry193; SDAG_W64-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]194; SDAG_W64-NEXT: s_mov_b32 s0, 0195; SDAG_W64-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)196; SDAG_W64-NEXT: v_mov_b32_e32 v3, s0197; SDAG_W64-NEXT: global_store_b64 v[0:1], v[2:3], off198; SDAG_W64-NEXT: s_endpgm199;200; GISEL_W32-LABEL: sgpr_inverse_ballot:201; GISEL_W32: ; %bb.0: ; %entry202; GISEL_W32-NEXT: v_mov_b32_e32 v3, 0203; GISEL_W32-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0204; GISEL_W32-NEXT: global_store_b64 v[0:1], v[2:3], off205; GISEL_W32-NEXT: s_endpgm206;207; SDAG_W32-LABEL: sgpr_inverse_ballot:208; SDAG_W32: ; %bb.0: ; %entry209; SDAG_W32-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0210; SDAG_W32-NEXT: s_mov_b32 s0, 0211; SDAG_W32-NEXT: v_mov_b32_e32 v3, s0212; SDAG_W32-NEXT: global_store_b64 v[0:1], v[2:3], off213; SDAG_W32-NEXT: s_endpgm214entry:215 %ballot = call i1 @llvm.amdgcn.inverse.ballot.i64(i64 %input)216 %sel = select i1 %ballot, i64 1, i64 0217 store i64 %sel, ptr addrspace(1) %out218 ret void219}220 221; Test ballot after phi222define amdgpu_cs void @phi_uniform(i64 inreg %s0_1, i64 inreg %s2, ptr addrspace(1) %out) {223; GISEL_W64-LABEL: phi_uniform:224; GISEL_W64: ; %bb.0: ; %entry225; GISEL_W64-NEXT: s_cmp_lg_u64 s[2:3], 0226; GISEL_W64-NEXT: s_cbranch_scc1 .LBB5_2227; GISEL_W64-NEXT: ; %bb.1: ; %if228; GISEL_W64-NEXT: s_add_u32 s0, s0, 1229; GISEL_W64-NEXT: s_addc_u32 s1, s1, 0230; GISEL_W64-NEXT: .LBB5_2: ; %endif231; GISEL_W64-NEXT: v_mov_b32_e32 v3, 0232; GISEL_W64-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]233; GISEL_W64-NEXT: global_store_b64 v[0:1], v[2:3], off234; GISEL_W64-NEXT: s_endpgm235;236; SDAG_W64-LABEL: phi_uniform:237; SDAG_W64: ; %bb.0: ; %entry238; SDAG_W64-NEXT: s_cmp_lg_u64 s[2:3], 0239; SDAG_W64-NEXT: s_cbranch_scc1 .LBB5_2240; SDAG_W64-NEXT: ; %bb.1: ; %if241; SDAG_W64-NEXT: s_add_u32 s0, s0, 1242; SDAG_W64-NEXT: s_addc_u32 s1, s1, 0243; SDAG_W64-NEXT: .LBB5_2: ; %endif244; SDAG_W64-NEXT: s_mov_b32 s2, 0245; SDAG_W64-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]246; SDAG_W64-NEXT: v_mov_b32_e32 v3, s2247; SDAG_W64-NEXT: global_store_b64 v[0:1], v[2:3], off248; SDAG_W64-NEXT: s_endpgm249;250; GISEL_W32-LABEL: phi_uniform:251; GISEL_W32: ; %bb.0: ; %entry252; GISEL_W32-NEXT: s_cmp_lg_u64 s[2:3], 0253; GISEL_W32-NEXT: s_cbranch_scc1 .LBB5_2254; GISEL_W32-NEXT: ; %bb.1: ; %if255; GISEL_W32-NEXT: s_add_u32 s0, s0, 1256; GISEL_W32-NEXT: .LBB5_2: ; %endif257; GISEL_W32-NEXT: v_mov_b32_e32 v3, 0258; GISEL_W32-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0259; GISEL_W32-NEXT: global_store_b64 v[0:1], v[2:3], off260; GISEL_W32-NEXT: s_endpgm261;262; SDAG_W32-LABEL: phi_uniform:263; SDAG_W32: ; %bb.0: ; %entry264; SDAG_W32-NEXT: s_cmp_lg_u64 s[2:3], 0265; SDAG_W32-NEXT: s_cbranch_scc1 .LBB5_2266; SDAG_W32-NEXT: ; %bb.1: ; %if267; SDAG_W32-NEXT: s_add_u32 s0, s0, 1268; SDAG_W32-NEXT: .LBB5_2: ; %endif269; SDAG_W32-NEXT: s_mov_b32 s1, 0270; SDAG_W32-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0271; SDAG_W32-NEXT: v_mov_b32_e32 v3, s1272; SDAG_W32-NEXT: global_store_b64 v[0:1], v[2:3], off273; SDAG_W32-NEXT: s_endpgm274entry:275 %cc = icmp ne i64 %s2, 0276 br i1 %cc, label %endif, label %if277 278if:279 %tmp = add i64 %s0_1, 1280 br label %endif281 282endif:283 %input = phi i64 [ %s0_1, %entry ], [ %tmp, %if ]284 285 %ballot = call i1 @llvm.amdgcn.inverse.ballot.i64(i64 %input)286 %sel = select i1 %ballot, i64 1, i64 0287 store i64 %sel, ptr addrspace(1) %out288 ret void289}290 291; Test for branching292; The change in the branch affects all lanes, not just the branching ones.293; This test will be fixed once GISel correctly takes uniformity analysis into account.294define amdgpu_cs void @inverse_ballot_branch(i64 inreg %s0_1, i64 inreg %s2, ptr addrspace(1) %out) {295; GISEL_W64-LABEL: inverse_ballot_branch:296; GISEL_W64: ; %bb.0: ; %entry297; GISEL_W64-NEXT: s_xor_b64 s[4:5], s[2:3], -1298; GISEL_W64-NEXT: s_and_saveexec_b64 s[2:3], s[4:5]299; GISEL_W64-NEXT: ; %bb.1: ; %if300; GISEL_W64-NEXT: s_add_u32 s0, s0, 1301; GISEL_W64-NEXT: s_addc_u32 s1, s1, 0302; GISEL_W64-NEXT: ; %bb.2: ; %endif303; GISEL_W64-NEXT: s_or_b64 exec, exec, s[2:3]304; GISEL_W64-NEXT: v_mov_b32_e32 v3, s1305; GISEL_W64-NEXT: v_mov_b32_e32 v2, s0306; GISEL_W64-NEXT: global_store_b64 v[0:1], v[2:3], off307; GISEL_W64-NEXT: s_endpgm308;309; SDAG_W64-LABEL: inverse_ballot_branch:310; SDAG_W64: ; %bb.0: ; %entry311; SDAG_W64-NEXT: v_mov_b32_e32 v3, s1312; SDAG_W64-NEXT: v_mov_b32_e32 v2, s0313; SDAG_W64-NEXT: s_xor_b64 s[4:5], s[2:3], -1314; SDAG_W64-NEXT: s_and_saveexec_b64 s[2:3], s[4:5]315; SDAG_W64-NEXT: ; %bb.1: ; %if316; SDAG_W64-NEXT: s_add_u32 s0, s0, 1317; SDAG_W64-NEXT: s_addc_u32 s1, s1, 0318; SDAG_W64-NEXT: v_mov_b32_e32 v3, s1319; SDAG_W64-NEXT: v_mov_b32_e32 v2, s0320; SDAG_W64-NEXT: ; %bb.2: ; %endif321; SDAG_W64-NEXT: s_or_b64 exec, exec, s[2:3]322; SDAG_W64-NEXT: global_store_b64 v[0:1], v[2:3], off323; SDAG_W64-NEXT: s_endpgm324;325; GISEL_W32-LABEL: inverse_ballot_branch:326; GISEL_W32: ; %bb.0: ; %entry327; GISEL_W32-NEXT: s_xor_b32 s3, s2, -1328; GISEL_W32-NEXT: s_and_saveexec_b32 s2, s3329; GISEL_W32-NEXT: ; %bb.1: ; %if330; GISEL_W32-NEXT: s_add_u32 s0, s0, 1331; GISEL_W32-NEXT: s_addc_u32 s1, s1, 0332; GISEL_W32-NEXT: ; %bb.2: ; %endif333; GISEL_W32-NEXT: s_or_b32 exec_lo, exec_lo, s2334; GISEL_W32-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0335; GISEL_W32-NEXT: global_store_b64 v[0:1], v[2:3], off336; GISEL_W32-NEXT: s_endpgm337;338; SDAG_W32-LABEL: inverse_ballot_branch:339; SDAG_W32: ; %bb.0: ; %entry340; SDAG_W32-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0341; SDAG_W32-NEXT: s_xor_b32 s3, s2, -1342; SDAG_W32-NEXT: s_and_saveexec_b32 s2, s3343; SDAG_W32-NEXT: ; %bb.1: ; %if344; SDAG_W32-NEXT: s_add_u32 s0, s0, 1345; SDAG_W32-NEXT: s_addc_u32 s1, s1, 0346; SDAG_W32-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0347; SDAG_W32-NEXT: ; %bb.2: ; %endif348; SDAG_W32-NEXT: s_or_b32 exec_lo, exec_lo, s2349; SDAG_W32-NEXT: global_store_b64 v[0:1], v[2:3], off350; SDAG_W32-NEXT: s_endpgm351entry:352 %ballot = call i1 @llvm.amdgcn.inverse.ballot.i64(i64 %s2)353 br i1 %ballot, label %endif, label %if354 355if:356 %tmp = add i64 %s0_1, 1357 br label %endif358 359endif:360 %sel = phi i64 [ %s0_1, %entry ], [ %tmp, %if ]361 store i64 %sel, ptr addrspace(1) %out362 ret void363}364