535 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 42; RUN: llc -mtriple=amdgcn -mcpu=tahiti < %s | FileCheck -check-prefix=SI %s3; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=VI %s4 5declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone6 7declare i64 @llvm.ctpop.i64(i64) nounwind readnone8declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>) nounwind readnone9declare <4 x i64> @llvm.ctpop.v4i64(<4 x i64>) nounwind readnone10declare <8 x i64> @llvm.ctpop.v8i64(<8 x i64>) nounwind readnone11declare <16 x i64> @llvm.ctpop.v16i64(<16 x i64>) nounwind readnone12 13declare i65 @llvm.ctpop.i65(i65) nounwind readnone14declare i128 @llvm.ctpop.i128(i128) nounwind readnone15 16define amdgpu_kernel void @s_ctpop_i64(ptr addrspace(1) noalias %out, [8 x i32], i64 %val) nounwind {17; SI-LABEL: s_ctpop_i64:18; SI: ; %bb.0:19; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x1320; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x921; SI-NEXT: s_mov_b32 s3, 0xf00022; SI-NEXT: s_mov_b32 s2, -123; SI-NEXT: s_waitcnt lgkmcnt(0)24; SI-NEXT: s_bcnt1_i32_b64 s4, s[6:7]25; SI-NEXT: v_mov_b32_e32 v0, s426; SI-NEXT: buffer_store_dword v0, off, s[0:3], 027; SI-NEXT: s_endpgm28;29; VI-LABEL: s_ctpop_i64:30; VI: ; %bb.0:31; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x4c32; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2433; VI-NEXT: s_mov_b32 s3, 0xf00034; VI-NEXT: s_mov_b32 s2, -135; VI-NEXT: s_waitcnt lgkmcnt(0)36; VI-NEXT: s_bcnt1_i32_b64 s4, s[6:7]37; VI-NEXT: v_mov_b32_e32 v0, s438; VI-NEXT: buffer_store_dword v0, off, s[0:3], 039; VI-NEXT: s_endpgm40 %ctpop = call i64 @llvm.ctpop.i64(i64 %val) nounwind readnone41 %truncctpop = trunc i64 %ctpop to i3242 store i32 %truncctpop, ptr addrspace(1) %out, align 443 ret void44}45 46define amdgpu_kernel void @v_ctpop_i64(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) nounwind {47; SI-LABEL: v_ctpop_i64:48; SI: ; %bb.0:49; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x950; SI-NEXT: s_mov_b32 s7, 0xf00051; SI-NEXT: s_mov_b32 s10, 052; SI-NEXT: s_mov_b32 s11, s753; SI-NEXT: v_lshlrev_b32_e32 v0, 3, v054; SI-NEXT: s_waitcnt lgkmcnt(0)55; SI-NEXT: s_mov_b64 s[8:9], s[2:3]56; SI-NEXT: v_mov_b32_e32 v1, 057; SI-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[8:11], 0 addr6458; SI-NEXT: s_mov_b32 s6, -159; SI-NEXT: s_mov_b32 s4, s060; SI-NEXT: s_mov_b32 s5, s161; SI-NEXT: s_waitcnt vmcnt(0)62; SI-NEXT: v_bcnt_u32_b32_e64 v0, v0, 063; SI-NEXT: v_bcnt_u32_b32_e32 v0, v1, v064; SI-NEXT: buffer_store_dword v0, off, s[4:7], 065; SI-NEXT: s_endpgm66;67; VI-LABEL: v_ctpop_i64:68; VI: ; %bb.0:69; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2470; VI-NEXT: v_lshlrev_b32_e32 v0, 3, v071; VI-NEXT: s_waitcnt lgkmcnt(0)72; VI-NEXT: v_mov_b32_e32 v1, s373; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v074; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc75; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]76; VI-NEXT: s_mov_b32 s3, 0xf00077; VI-NEXT: s_mov_b32 s2, -178; VI-NEXT: s_waitcnt vmcnt(0)79; VI-NEXT: v_bcnt_u32_b32 v0, v0, 080; VI-NEXT: v_bcnt_u32_b32 v0, v1, v081; VI-NEXT: buffer_store_dword v0, off, s[0:3], 082; VI-NEXT: s_endpgm83 %tid = call i32 @llvm.amdgcn.workitem.id.x()84 %in.gep = getelementptr i64, ptr addrspace(1) %in, i32 %tid85 %val = load i64, ptr addrspace(1) %in.gep, align 886 %ctpop = call i64 @llvm.ctpop.i64(i64 %val) nounwind readnone87 %truncctpop = trunc i64 %ctpop to i3288 store i32 %truncctpop, ptr addrspace(1) %out, align 489 ret void90}91 92define amdgpu_kernel void @v_ctpop_i64_user(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in, i64 %s.val) nounwind {93; SI-LABEL: v_ctpop_i64_user:94; SI: ; %bb.0:95; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x996; SI-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0xd97; SI-NEXT: s_mov_b32 s7, 0xf00098; SI-NEXT: s_mov_b32 s10, 099; SI-NEXT: s_mov_b32 s11, s7100; SI-NEXT: s_waitcnt lgkmcnt(0)101; SI-NEXT: s_mov_b64 s[8:9], s[2:3]102; SI-NEXT: v_lshlrev_b32_e32 v0, 3, v0103; SI-NEXT: v_mov_b32_e32 v1, 0104; SI-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[8:11], 0 addr64105; SI-NEXT: s_mov_b32 s6, -1106; SI-NEXT: s_mov_b32 s4, s0107; SI-NEXT: s_mov_b32 s5, s1108; SI-NEXT: s_waitcnt vmcnt(0)109; SI-NEXT: v_bcnt_u32_b32_e64 v0, v0, 0110; SI-NEXT: v_bcnt_u32_b32_e32 v0, v1, v0111; SI-NEXT: v_mov_b32_e32 v1, s13112; SI-NEXT: v_or_b32_e32 v0, s12, v0113; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0114; SI-NEXT: s_endpgm115;116; VI-LABEL: v_ctpop_i64_user:117; VI: ; %bb.0:118; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24119; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34120; VI-NEXT: v_lshlrev_b32_e32 v0, 3, v0121; VI-NEXT: s_waitcnt lgkmcnt(0)122; VI-NEXT: v_mov_b32_e32 v1, s3123; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v0124; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc125; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]126; VI-NEXT: s_mov_b32 s3, 0xf000127; VI-NEXT: s_mov_b32 s2, -1128; VI-NEXT: s_waitcnt vmcnt(0)129; VI-NEXT: v_bcnt_u32_b32 v0, v0, 0130; VI-NEXT: v_bcnt_u32_b32 v0, v1, v0131; VI-NEXT: v_mov_b32_e32 v1, s5132; VI-NEXT: v_or_b32_e32 v0, s4, v0133; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0134; VI-NEXT: s_endpgm135 %tid = call i32 @llvm.amdgcn.workitem.id.x()136 %in.gep = getelementptr i64, ptr addrspace(1) %in, i32 %tid137 %val = load i64, ptr addrspace(1) %in.gep, align 8138 %ctpop = call i64 @llvm.ctpop.i64(i64 %val) nounwind readnone139 %or = or i64 %ctpop, %s.val140 store i64 %or, ptr addrspace(1) %out141 ret void142}143 144define amdgpu_kernel void @s_ctpop_v2i64(ptr addrspace(1) noalias %out, <2 x i64> %val) nounwind {145; SI-LABEL: s_ctpop_v2i64:146; SI: ; %bb.0:147; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xd148; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9149; SI-NEXT: s_mov_b32 s7, 0xf000150; SI-NEXT: s_mov_b32 s6, -1151; SI-NEXT: s_waitcnt lgkmcnt(0)152; SI-NEXT: s_bcnt1_i32_b64 s0, s[0:1]153; SI-NEXT: s_bcnt1_i32_b64 s1, s[2:3]154; SI-NEXT: v_mov_b32_e32 v0, s0155; SI-NEXT: v_mov_b32_e32 v1, s1156; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0157; SI-NEXT: s_endpgm158;159; VI-LABEL: s_ctpop_v2i64:160; VI: ; %bb.0:161; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34162; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24163; VI-NEXT: s_mov_b32 s7, 0xf000164; VI-NEXT: s_mov_b32 s6, -1165; VI-NEXT: s_waitcnt lgkmcnt(0)166; VI-NEXT: s_bcnt1_i32_b64 s0, s[0:1]167; VI-NEXT: s_bcnt1_i32_b64 s1, s[2:3]168; VI-NEXT: v_mov_b32_e32 v0, s0169; VI-NEXT: v_mov_b32_e32 v1, s1170; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0171; VI-NEXT: s_endpgm172 %ctpop = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %val) nounwind readnone173 %truncctpop = trunc <2 x i64> %ctpop to <2 x i32>174 store <2 x i32> %truncctpop, ptr addrspace(1) %out, align 8175 ret void176}177 178define amdgpu_kernel void @s_ctpop_v4i64(ptr addrspace(1) noalias %out, <4 x i64> %val) nounwind {179; SI-LABEL: s_ctpop_v4i64:180; SI: ; %bb.0:181; SI-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x11182; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9183; SI-NEXT: s_mov_b32 s3, 0xf000184; SI-NEXT: s_mov_b32 s2, -1185; SI-NEXT: s_waitcnt lgkmcnt(0)186; SI-NEXT: s_bcnt1_i32_b64 s4, s[8:9]187; SI-NEXT: s_bcnt1_i32_b64 s5, s[10:11]188; SI-NEXT: s_bcnt1_i32_b64 s6, s[12:13]189; SI-NEXT: s_bcnt1_i32_b64 s7, s[14:15]190; SI-NEXT: v_mov_b32_e32 v0, s4191; SI-NEXT: v_mov_b32_e32 v1, s5192; SI-NEXT: v_mov_b32_e32 v2, s6193; SI-NEXT: v_mov_b32_e32 v3, s7194; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0195; SI-NEXT: s_endpgm196;197; VI-LABEL: s_ctpop_v4i64:198; VI: ; %bb.0:199; VI-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x44200; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24201; VI-NEXT: s_mov_b32 s3, 0xf000202; VI-NEXT: s_mov_b32 s2, -1203; VI-NEXT: s_waitcnt lgkmcnt(0)204; VI-NEXT: s_bcnt1_i32_b64 s4, s[8:9]205; VI-NEXT: s_bcnt1_i32_b64 s5, s[10:11]206; VI-NEXT: s_bcnt1_i32_b64 s6, s[12:13]207; VI-NEXT: s_bcnt1_i32_b64 s7, s[14:15]208; VI-NEXT: v_mov_b32_e32 v0, s4209; VI-NEXT: v_mov_b32_e32 v1, s5210; VI-NEXT: v_mov_b32_e32 v2, s6211; VI-NEXT: v_mov_b32_e32 v3, s7212; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0213; VI-NEXT: s_endpgm214 %ctpop = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %val) nounwind readnone215 %truncctpop = trunc <4 x i64> %ctpop to <4 x i32>216 store <4 x i32> %truncctpop, ptr addrspace(1) %out, align 16217 ret void218}219 220define amdgpu_kernel void @v_ctpop_v2i64(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) nounwind {221; SI-LABEL: v_ctpop_v2i64:222; SI: ; %bb.0:223; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9224; SI-NEXT: s_mov_b32 s7, 0xf000225; SI-NEXT: s_mov_b32 s10, 0226; SI-NEXT: s_mov_b32 s11, s7227; SI-NEXT: v_lshlrev_b32_e32 v0, 4, v0228; SI-NEXT: s_waitcnt lgkmcnt(0)229; SI-NEXT: s_mov_b64 s[8:9], s[2:3]230; SI-NEXT: v_mov_b32_e32 v1, 0231; SI-NEXT: buffer_load_dwordx4 v[0:3], v[0:1], s[8:11], 0 addr64232; SI-NEXT: s_mov_b32 s6, -1233; SI-NEXT: s_mov_b32 s4, s0234; SI-NEXT: s_mov_b32 s5, s1235; SI-NEXT: s_waitcnt vmcnt(0)236; SI-NEXT: v_bcnt_u32_b32_e64 v0, v0, 0237; SI-NEXT: v_bcnt_u32_b32_e64 v2, v2, 0238; SI-NEXT: v_bcnt_u32_b32_e32 v0, v1, v0239; SI-NEXT: v_bcnt_u32_b32_e32 v1, v3, v2240; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0241; SI-NEXT: s_endpgm242;243; VI-LABEL: v_ctpop_v2i64:244; VI: ; %bb.0:245; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24246; VI-NEXT: v_lshlrev_b32_e32 v0, 4, v0247; VI-NEXT: s_waitcnt lgkmcnt(0)248; VI-NEXT: v_mov_b32_e32 v1, s3249; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v0250; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc251; VI-NEXT: flat_load_dwordx4 v[0:3], v[0:1]252; VI-NEXT: s_mov_b32 s3, 0xf000253; VI-NEXT: s_mov_b32 s2, -1254; VI-NEXT: s_waitcnt vmcnt(0)255; VI-NEXT: v_bcnt_u32_b32 v0, v0, 0256; VI-NEXT: v_bcnt_u32_b32 v2, v2, 0257; VI-NEXT: v_bcnt_u32_b32 v0, v1, v0258; VI-NEXT: v_bcnt_u32_b32 v1, v3, v2259; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0260; VI-NEXT: s_endpgm261 %tid = call i32 @llvm.amdgcn.workitem.id.x()262 %in.gep = getelementptr <2 x i64>, ptr addrspace(1) %in, i32 %tid263 %val = load <2 x i64>, ptr addrspace(1) %in.gep, align 16264 %ctpop = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %val) nounwind readnone265 %truncctpop = trunc <2 x i64> %ctpop to <2 x i32>266 store <2 x i32> %truncctpop, ptr addrspace(1) %out, align 8267 ret void268}269 270define amdgpu_kernel void @v_ctpop_v4i64(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) nounwind {271; SI-LABEL: v_ctpop_v4i64:272; SI: ; %bb.0:273; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9274; SI-NEXT: s_mov_b32 s7, 0xf000275; SI-NEXT: s_mov_b32 s10, 0276; SI-NEXT: s_mov_b32 s11, s7277; SI-NEXT: v_lshlrev_b32_e32 v4, 5, v0278; SI-NEXT: s_waitcnt lgkmcnt(0)279; SI-NEXT: s_mov_b64 s[8:9], s[2:3]280; SI-NEXT: v_mov_b32_e32 v5, 0281; SI-NEXT: buffer_load_dwordx4 v[0:3], v[4:5], s[8:11], 0 addr64282; SI-NEXT: buffer_load_dwordx4 v[4:7], v[4:5], s[8:11], 0 addr64 offset:16283; SI-NEXT: s_mov_b32 s6, -1284; SI-NEXT: s_mov_b32 s4, s0285; SI-NEXT: s_mov_b32 s5, s1286; SI-NEXT: s_waitcnt vmcnt(1)287; SI-NEXT: v_bcnt_u32_b32_e64 v0, v0, 0288; SI-NEXT: v_bcnt_u32_b32_e64 v2, v2, 0289; SI-NEXT: s_waitcnt vmcnt(0)290; SI-NEXT: v_bcnt_u32_b32_e64 v4, v4, 0291; SI-NEXT: v_bcnt_u32_b32_e64 v6, v6, 0292; SI-NEXT: v_bcnt_u32_b32_e32 v0, v1, v0293; SI-NEXT: v_bcnt_u32_b32_e32 v1, v3, v2294; SI-NEXT: v_bcnt_u32_b32_e32 v2, v5, v4295; SI-NEXT: v_bcnt_u32_b32_e32 v3, v7, v6296; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0297; SI-NEXT: s_endpgm298;299; VI-LABEL: v_ctpop_v4i64:300; VI: ; %bb.0:301; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24302; VI-NEXT: v_lshlrev_b32_e32 v0, 5, v0303; VI-NEXT: s_waitcnt lgkmcnt(0)304; VI-NEXT: v_mov_b32_e32 v1, s3305; VI-NEXT: v_add_u32_e32 v4, vcc, s2, v0306; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc307; VI-NEXT: flat_load_dwordx4 v[0:3], v[4:5]308; VI-NEXT: v_add_u32_e32 v4, vcc, 16, v4309; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc310; VI-NEXT: flat_load_dwordx4 v[4:7], v[4:5]311; VI-NEXT: s_mov_b32 s3, 0xf000312; VI-NEXT: s_mov_b32 s2, -1313; VI-NEXT: s_waitcnt vmcnt(1)314; VI-NEXT: v_bcnt_u32_b32 v0, v0, 0315; VI-NEXT: v_bcnt_u32_b32 v8, v2, 0316; VI-NEXT: v_bcnt_u32_b32 v2, v1, v0317; VI-NEXT: v_bcnt_u32_b32 v3, v3, v8318; VI-NEXT: s_waitcnt vmcnt(0)319; VI-NEXT: v_bcnt_u32_b32 v4, v4, 0320; VI-NEXT: v_bcnt_u32_b32 v6, v6, 0321; VI-NEXT: v_bcnt_u32_b32 v4, v5, v4322; VI-NEXT: v_bcnt_u32_b32 v5, v7, v6323; VI-NEXT: buffer_store_dwordx4 v[2:5], off, s[0:3], 0324; VI-NEXT: s_endpgm325 %tid = call i32 @llvm.amdgcn.workitem.id.x()326 %in.gep = getelementptr <4 x i64>, ptr addrspace(1) %in, i32 %tid327 %val = load <4 x i64>, ptr addrspace(1) %in.gep, align 32328 %ctpop = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %val) nounwind readnone329 %truncctpop = trunc <4 x i64> %ctpop to <4 x i32>330 store <4 x i32> %truncctpop, ptr addrspace(1) %out, align 16331 ret void332}333 334define amdgpu_kernel void @ctpop_i64_in_br(ptr addrspace(1) %out, ptr addrspace(1) %in, i64 %ctpop_arg, i32 %cond) {335; SI-LABEL: ctpop_i64_in_br:336; SI: ; %bb.0: ; %entry337; SI-NEXT: s_load_dword s8, s[4:5], 0xf338; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9339; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd340; SI-NEXT: s_waitcnt lgkmcnt(0)341; SI-NEXT: s_cmp_lg_u32 s8, 0342; SI-NEXT: s_cbranch_scc0 .LBB7_4343; SI-NEXT: ; %bb.1: ; %else344; SI-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x2345; SI-NEXT: s_mov_b64 s[2:3], 0346; SI-NEXT: s_andn2_b64 vcc, exec, s[2:3]347; SI-NEXT: s_waitcnt lgkmcnt(0)348; SI-NEXT: s_mov_b64 vcc, vcc349; SI-NEXT: s_cbranch_vccnz .LBB7_3350; SI-NEXT: .LBB7_2: ; %if351; SI-NEXT: s_bcnt1_i32_b64 s4, s[6:7]352; SI-NEXT: s_mov_b32 s5, 0353; SI-NEXT: .LBB7_3: ; %endif354; SI-NEXT: v_mov_b32_e32 v0, s4355; SI-NEXT: s_mov_b32 s3, 0xf000356; SI-NEXT: s_mov_b32 s2, -1357; SI-NEXT: v_mov_b32_e32 v1, s5358; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0359; SI-NEXT: s_endpgm360; SI-NEXT: .LBB7_4:361; SI-NEXT: ; implicit-def: $sgpr4_sgpr5362; SI-NEXT: s_branch .LBB7_2363;364; VI-LABEL: ctpop_i64_in_br:365; VI: ; %bb.0: ; %entry366; VI-NEXT: s_load_dword s8, s[4:5], 0x3c367; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24368; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34369; VI-NEXT: s_waitcnt lgkmcnt(0)370; VI-NEXT: s_cmp_lg_u32 s8, 0371; VI-NEXT: s_cbranch_scc0 .LBB7_4372; VI-NEXT: ; %bb.1: ; %else373; VI-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x8374; VI-NEXT: s_cbranch_execnz .LBB7_3375; VI-NEXT: .LBB7_2: ; %if376; VI-NEXT: s_waitcnt lgkmcnt(0)377; VI-NEXT: s_bcnt1_i32_b64 s4, s[6:7]378; VI-NEXT: s_mov_b32 s5, 0379; VI-NEXT: .LBB7_3: ; %endif380; VI-NEXT: s_waitcnt lgkmcnt(0)381; VI-NEXT: v_mov_b32_e32 v0, s4382; VI-NEXT: s_mov_b32 s3, 0xf000383; VI-NEXT: s_mov_b32 s2, -1384; VI-NEXT: v_mov_b32_e32 v1, s5385; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0386; VI-NEXT: s_endpgm387; VI-NEXT: .LBB7_4:388; VI-NEXT: ; implicit-def: $sgpr4_sgpr5389; VI-NEXT: s_branch .LBB7_2390entry:391 %tmp0 = icmp eq i32 %cond, 0392 br i1 %tmp0, label %if, label %else393 394if:395 %tmp2 = call i64 @llvm.ctpop.i64(i64 %ctpop_arg)396 br label %endif397 398else:399 %tmp3 = getelementptr i64, ptr addrspace(1) %in, i32 1400 %tmp4 = load i64, ptr addrspace(1) %tmp3401 br label %endif402 403endif:404 %tmp5 = phi i64 [%tmp2, %if], [%tmp4, %else]405 store i64 %tmp5, ptr addrspace(1) %out406 ret void407}408 409define amdgpu_kernel void @s_ctpop_i128(ptr addrspace(1) noalias %out, i128 %val) nounwind {410; SI-LABEL: s_ctpop_i128:411; SI: ; %bb.0:412; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb413; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9414; SI-NEXT: s_mov_b32 s7, 0xf000415; SI-NEXT: s_mov_b32 s6, -1416; SI-NEXT: s_waitcnt lgkmcnt(0)417; SI-NEXT: s_bcnt1_i32_b64 s2, s[2:3]418; SI-NEXT: s_bcnt1_i32_b64 s0, s[0:1]419; SI-NEXT: s_add_i32 s0, s0, s2420; SI-NEXT: v_mov_b32_e32 v0, s0421; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0422; SI-NEXT: s_endpgm423;424; VI-LABEL: s_ctpop_i128:425; VI: ; %bb.0:426; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c427; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24428; VI-NEXT: s_mov_b32 s7, 0xf000429; VI-NEXT: s_mov_b32 s6, -1430; VI-NEXT: s_waitcnt lgkmcnt(0)431; VI-NEXT: s_bcnt1_i32_b64 s2, s[2:3]432; VI-NEXT: s_bcnt1_i32_b64 s0, s[0:1]433; VI-NEXT: s_add_i32 s0, s0, s2434; VI-NEXT: v_mov_b32_e32 v0, s0435; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0436; VI-NEXT: s_endpgm437 %ctpop = call i128 @llvm.ctpop.i128(i128 %val) nounwind readnone438 %truncctpop = trunc i128 %ctpop to i32439 store i32 %truncctpop, ptr addrspace(1) %out, align 4440 ret void441}442 443define amdgpu_kernel void @s_ctpop_i65(ptr addrspace(1) noalias %out, i65 %val) nounwind {444; SI-LABEL: s_ctpop_i65:445; SI: ; %bb.0:446; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9447; SI-NEXT: s_load_dword s8, s[4:5], 0xd448; SI-NEXT: s_mov_b32 s7, 0xf000449; SI-NEXT: s_mov_b32 s6, -1450; SI-NEXT: s_waitcnt lgkmcnt(0)451; SI-NEXT: s_mov_b32 s4, s0452; SI-NEXT: s_and_b32 s0, s8, 0xff453; SI-NEXT: s_mov_b32 s5, s1454; SI-NEXT: s_bcnt1_i32_b32 s0, s0455; SI-NEXT: s_bcnt1_i32_b64 s1, s[2:3]456; SI-NEXT: s_add_i32 s0, s1, s0457; SI-NEXT: v_mov_b32_e32 v0, s0458; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0459; SI-NEXT: s_endpgm460;461; VI-LABEL: s_ctpop_i65:462; VI: ; %bb.0:463; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24464; VI-NEXT: s_load_dword s8, s[4:5], 0x34465; VI-NEXT: s_mov_b32 s7, 0xf000466; VI-NEXT: s_mov_b32 s6, -1467; VI-NEXT: s_waitcnt lgkmcnt(0)468; VI-NEXT: s_mov_b32 s4, s0469; VI-NEXT: s_and_b32 s0, s8, 0xff470; VI-NEXT: s_mov_b32 s5, s1471; VI-NEXT: s_bcnt1_i32_b32 s0, s0472; VI-NEXT: s_bcnt1_i32_b64 s1, s[2:3]473; VI-NEXT: s_add_i32 s0, s1, s0474; VI-NEXT: v_mov_b32_e32 v0, s0475; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0476; VI-NEXT: s_endpgm477 %ctpop = call i65 @llvm.ctpop.i65(i65 %val) nounwind readnone478 %truncctpop = trunc i65 %ctpop to i32479 store i32 %truncctpop, ptr addrspace(1) %out, align 4480 ret void481}482 483; FIXME: Should not have extra add484define amdgpu_kernel void @v_ctpop_i128(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) nounwind {485; SI-LABEL: v_ctpop_i128:486; SI: ; %bb.0:487; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9488; SI-NEXT: s_mov_b32 s7, 0xf000489; SI-NEXT: s_mov_b32 s10, 0490; SI-NEXT: s_mov_b32 s11, s7491; SI-NEXT: v_lshlrev_b32_e32 v0, 4, v0492; SI-NEXT: s_waitcnt lgkmcnt(0)493; SI-NEXT: s_mov_b64 s[8:9], s[2:3]494; SI-NEXT: v_mov_b32_e32 v1, 0495; SI-NEXT: buffer_load_dwordx4 v[0:3], v[0:1], s[8:11], 0 addr64496; SI-NEXT: s_mov_b32 s6, -1497; SI-NEXT: s_mov_b32 s4, s0498; SI-NEXT: s_mov_b32 s5, s1499; SI-NEXT: s_waitcnt vmcnt(0)500; SI-NEXT: v_bcnt_u32_b32_e64 v2, v2, 0501; SI-NEXT: v_bcnt_u32_b32_e64 v0, v0, 0502; SI-NEXT: v_bcnt_u32_b32_e32 v2, v3, v2503; SI-NEXT: v_bcnt_u32_b32_e32 v0, v1, v0504; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v2505; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0506; SI-NEXT: s_endpgm507;508; VI-LABEL: v_ctpop_i128:509; VI: ; %bb.0:510; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24511; VI-NEXT: v_lshlrev_b32_e32 v0, 4, v0512; VI-NEXT: s_waitcnt lgkmcnt(0)513; VI-NEXT: v_mov_b32_e32 v1, s3514; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v0515; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc516; VI-NEXT: flat_load_dwordx4 v[0:3], v[0:1]517; VI-NEXT: s_mov_b32 s3, 0xf000518; VI-NEXT: s_mov_b32 s2, -1519; VI-NEXT: s_waitcnt vmcnt(0)520; VI-NEXT: v_bcnt_u32_b32 v2, v2, 0521; VI-NEXT: v_bcnt_u32_b32 v0, v0, 0522; VI-NEXT: v_bcnt_u32_b32 v2, v3, v2523; VI-NEXT: v_bcnt_u32_b32 v0, v1, v0524; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2525; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0526; VI-NEXT: s_endpgm527 %tid = call i32 @llvm.amdgcn.workitem.id.x()528 %in.gep = getelementptr i128, ptr addrspace(1) %in, i32 %tid529 %val = load i128, ptr addrspace(1) %in.gep, align 8530 %ctpop = call i128 @llvm.ctpop.i128(i128 %val) nounwind readnone531 %truncctpop = trunc i128 %ctpop to i32532 store i32 %truncctpop, ptr addrspace(1) %out, align 4533 ret void534}535