331 lines · plain
1; RUN: llc -mtriple=amdgcn -mcpu=tahiti < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=FUNC -check-prefix=SI %s2; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=FUNC -check-prefix=VI %s3; RUN: llc -mtriple=r600 -mcpu=cypress < %s | FileCheck -enable-var-scope -check-prefix=EG -check-prefix=FUNC %s4 5declare i32 @llvm.ctpop.i32(i32) nounwind readnone6declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>) nounwind readnone7declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>) nounwind readnone8declare <8 x i32> @llvm.ctpop.v8i32(<8 x i32>) nounwind readnone9declare <16 x i32> @llvm.ctpop.v16i32(<16 x i32>) nounwind readnone10 11declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone12 13; FUNC-LABEL: {{^}}s_ctpop_i32:14; GCN: s_load_dword [[SVAL:s[0-9]+]],15; GCN: s_bcnt1_i32_b32 [[SRESULT:s[0-9]+]], [[SVAL]]16; GCN: v_mov_b32_e32 [[VRESULT:v[0-9]+]], [[SRESULT]]17; GCN: buffer_store_dword [[VRESULT]],18; GCN: s_endpgm19 20; EG: BCNT_INT21define amdgpu_kernel void @s_ctpop_i32(ptr addrspace(1) noalias %out, i32 %val) nounwind {22 %ctpop = call i32 @llvm.ctpop.i32(i32 %val) nounwind readnone23 store i32 %ctpop, ptr addrspace(1) %out, align 424 ret void25}26 27; XXX - Why 0 in register?28; FUNC-LABEL: {{^}}v_ctpop_i32:29; GCN: {{buffer|flat}}_load_dword [[VAL:v[0-9]+]],30; GCN: v_bcnt_u32_b32{{(_e64)*}} [[RESULT:v[0-9]+]], [[VAL]], 031; GCN: buffer_store_dword [[RESULT]],32; GCN: s_endpgm33 34; EG: BCNT_INT35define amdgpu_kernel void @v_ctpop_i32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) nounwind {36 %tid = call i32 @llvm.amdgcn.workitem.id.x()37 %in.gep = getelementptr i32, ptr addrspace(1) %in, i32 %tid38 %val = load i32, ptr addrspace(1) %in.gep, align 439 %ctpop = call i32 @llvm.ctpop.i32(i32 %val) nounwind readnone40 store i32 %ctpop, ptr addrspace(1) %out, align 441 ret void42}43 44; FUNC-LABEL: {{^}}v_ctpop_add_chain_i32:45; SI: buffer_load_dword [[VAL0:v[0-9]+]],46; SI: buffer_load_dword [[VAL1:v[0-9]+]],47; VI: flat_load_dword [[VAL0:v[0-9]+]],48; VI: flat_load_dword [[VAL1:v[0-9]+]],49; GCN: v_bcnt_u32_b32{{(_e64)*}} [[MIDRESULT:v[0-9]+]], [[VAL1]], 050; SI: v_bcnt_u32_b32_e32 [[RESULT:v[0-9]+]], [[VAL0]], [[MIDRESULT]]51; VI: v_bcnt_u32_b32 [[RESULT:v[0-9]+]], [[VAL0]], [[MIDRESULT]]52; GCN: buffer_store_dword [[RESULT]],53; GCN: s_endpgm54 55; EG: BCNT_INT56; EG: BCNT_INT57define amdgpu_kernel void @v_ctpop_add_chain_i32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in0, ptr addrspace(1) noalias %in1) nounwind {58 %tid = call i32 @llvm.amdgcn.workitem.id.x()59 %in0.gep = getelementptr i32, ptr addrspace(1) %in0, i32 %tid60 %in1.gep = getelementptr i32, ptr addrspace(1) %in1, i32 %tid61 %val0 = load volatile i32, ptr addrspace(1) %in0.gep, align 462 %val1 = load volatile i32, ptr addrspace(1) %in1.gep, align 463 %ctpop0 = call i32 @llvm.ctpop.i32(i32 %val0) nounwind readnone64 %ctpop1 = call i32 @llvm.ctpop.i32(i32 %val1) nounwind readnone65 %add = add i32 %ctpop0, %ctpop166 store i32 %add, ptr addrspace(1) %out, align 467 ret void68}69 70; FUNC-LABEL: {{^}}v_ctpop_add_sgpr_i32:71; GCN: {{buffer|flat}}_load_dword [[VAL0:v[0-9]+]],72; GCN: s_waitcnt73; GCN-NEXT: v_bcnt_u32_b32{{(_e64)*}} [[RESULT:v[0-9]+]], [[VAL0]], s{{[0-9]+}}74; GCN: buffer_store_dword [[RESULT]],75; GCN: s_endpgm76define amdgpu_kernel void @v_ctpop_add_sgpr_i32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in, i32 %sval) nounwind {77 %tid = call i32 @llvm.amdgcn.workitem.id.x()78 %in.gep = getelementptr i32, ptr addrspace(1) %in, i32 %tid79 %val = load i32, ptr addrspace(1) %in.gep, align 480 %ctpop = call i32 @llvm.ctpop.i32(i32 %val) nounwind readnone81 %add = add i32 %ctpop, %sval82 store i32 %add, ptr addrspace(1) %out, align 483 ret void84}85 86; FUNC-LABEL: {{^}}v_ctpop_v2i32:87; GCN: v_bcnt_u32_b32{{(_e64)*}}88; GCN: v_bcnt_u32_b32{{(_e64)*}}89; GCN: s_endpgm90 91; EG: BCNT_INT92; EG: BCNT_INT93define amdgpu_kernel void @v_ctpop_v2i32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) nounwind {94 %tid = call i32 @llvm.amdgcn.workitem.id.x()95 %in.gep = getelementptr <2 x i32>, ptr addrspace(1) %in, i32 %tid96 %val = load <2 x i32>, ptr addrspace(1) %in.gep, align 897 %ctpop = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %val) nounwind readnone98 store <2 x i32> %ctpop, ptr addrspace(1) %out, align 899 ret void100}101 102; FUNC-LABEL: {{^}}v_ctpop_v4i32:103; GCN: v_bcnt_u32_b32{{(_e64)*}}104; GCN: v_bcnt_u32_b32{{(_e64)*}}105; GCN: v_bcnt_u32_b32{{(_e64)*}}106; GCN: v_bcnt_u32_b32{{(_e64)*}}107; GCN: s_endpgm108 109; EG: BCNT_INT110; EG: BCNT_INT111; EG: BCNT_INT112; EG: BCNT_INT113define amdgpu_kernel void @v_ctpop_v4i32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) nounwind {114 %tid = call i32 @llvm.amdgcn.workitem.id.x()115 %in.gep = getelementptr <4 x i32>, ptr addrspace(1) %in, i32 %tid116 %val = load <4 x i32>, ptr addrspace(1) %in.gep, align 16117 %ctpop = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %val) nounwind readnone118 store <4 x i32> %ctpop, ptr addrspace(1) %out, align 16119 ret void120}121 122; FUNC-LABEL: {{^}}v_ctpop_v8i32:123; GCN: v_bcnt_u32_b32{{(_e64)*}}124; GCN: v_bcnt_u32_b32{{(_e64)*}}125; GCN: v_bcnt_u32_b32{{(_e64)*}}126; GCN: v_bcnt_u32_b32{{(_e64)*}}127; GCN: v_bcnt_u32_b32{{(_e64)*}}128; GCN: v_bcnt_u32_b32{{(_e64)*}}129; GCN: v_bcnt_u32_b32{{(_e64)*}}130; GCN: v_bcnt_u32_b32{{(_e64)*}}131; GCN: s_endpgm132 133; EG: BCNT_INT134; EG: BCNT_INT135; EG: BCNT_INT136; EG: BCNT_INT137; EG: BCNT_INT138; EG: BCNT_INT139; EG: BCNT_INT140; EG: BCNT_INT141define amdgpu_kernel void @v_ctpop_v8i32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) nounwind {142 %tid = call i32 @llvm.amdgcn.workitem.id.x()143 %in.gep = getelementptr <8 x i32>, ptr addrspace(1) %in, i32 %tid144 %val = load <8 x i32>, ptr addrspace(1) %in.gep, align 32145 %ctpop = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> %val) nounwind readnone146 store <8 x i32> %ctpop, ptr addrspace(1) %out, align 32147 ret void148}149 150; FUNC-LABEL: {{^}}v_ctpop_v16i32:151; GCN: v_bcnt_u32_b32{{(_e64)*}}152; GCN: v_bcnt_u32_b32{{(_e64)*}}153; GCN: v_bcnt_u32_b32{{(_e64)*}}154; GCN: v_bcnt_u32_b32{{(_e64)*}}155; GCN: v_bcnt_u32_b32{{(_e64)*}}156; GCN: v_bcnt_u32_b32{{(_e64)*}}157; GCN: v_bcnt_u32_b32{{(_e64)*}}158; GCN: v_bcnt_u32_b32{{(_e64)*}}159; GCN: v_bcnt_u32_b32{{(_e64)*}}160; GCN: v_bcnt_u32_b32{{(_e64)*}}161; GCN: v_bcnt_u32_b32{{(_e64)*}}162; GCN: v_bcnt_u32_b32{{(_e64)*}}163; GCN: v_bcnt_u32_b32{{(_e64)*}}164; GCN: v_bcnt_u32_b32{{(_e64)*}}165; GCN: v_bcnt_u32_b32{{(_e64)*}}166; GCN: v_bcnt_u32_b32{{(_e64)*}}167; GCN: s_endpgm168 169; EG: BCNT_INT170; EG: BCNT_INT171; EG: BCNT_INT172; EG: BCNT_INT173; EG: BCNT_INT174; EG: BCNT_INT175; EG: BCNT_INT176; EG: BCNT_INT177; EG: BCNT_INT178; EG: BCNT_INT179; EG: BCNT_INT180; EG: BCNT_INT181; EG: BCNT_INT182; EG: BCNT_INT183; EG: BCNT_INT184; EG: BCNT_INT185define amdgpu_kernel void @v_ctpop_v16i32(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) nounwind {186 %tid = call i32 @llvm.amdgcn.workitem.id.x()187 %in.gep = getelementptr <16 x i32>, ptr addrspace(1) %in, i32 %tid188 %val = load <16 x i32>, ptr addrspace(1) %in.gep, align 32189 %ctpop = call <16 x i32> @llvm.ctpop.v16i32(<16 x i32> %val) nounwind readnone190 store <16 x i32> %ctpop, ptr addrspace(1) %out, align 32191 ret void192}193 194; FUNC-LABEL: {{^}}v_ctpop_i32_add_inline_constant:195; GCN: {{buffer|flat}}_load_dword [[VAL:v[0-9]+]],196; GCN: v_bcnt_u32_b32{{(_e64)*}} [[RESULT:v[0-9]+]], [[VAL]], 4197; GCN: buffer_store_dword [[RESULT]],198; GCN: s_endpgm199 200; EG: BCNT_INT201define amdgpu_kernel void @v_ctpop_i32_add_inline_constant(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) nounwind {202 %tid = call i32 @llvm.amdgcn.workitem.id.x()203 %in.gep = getelementptr i32, ptr addrspace(1) %in, i32 %tid204 %val = load i32, ptr addrspace(1) %in.gep, align 4205 %ctpop = call i32 @llvm.ctpop.i32(i32 %val) nounwind readnone206 %add = add i32 %ctpop, 4207 store i32 %add, ptr addrspace(1) %out, align 4208 ret void209}210 211; FUNC-LABEL: {{^}}v_ctpop_i32_add_inline_constant_inv:212; GCN: {{buffer|flat}}_load_dword [[VAL:v[0-9]+]],213; GCN: v_bcnt_u32_b32{{(_e64)*}} [[RESULT:v[0-9]+]], [[VAL]], 4214; GCN: buffer_store_dword [[RESULT]],215; GCN: s_endpgm216 217; EG: BCNT_INT218define amdgpu_kernel void @v_ctpop_i32_add_inline_constant_inv(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) nounwind {219 %tid = call i32 @llvm.amdgcn.workitem.id.x()220 %in.gep = getelementptr i32, ptr addrspace(1) %in, i32 %tid221 %val = load i32, ptr addrspace(1) %in.gep, align 4222 %ctpop = call i32 @llvm.ctpop.i32(i32 %val) nounwind readnone223 %add = add i32 4, %ctpop224 store i32 %add, ptr addrspace(1) %out, align 4225 ret void226}227 228; FUNC-LABEL: {{^}}v_ctpop_i32_add_literal:229; GCN-DAG: {{buffer|flat}}_load_dword [[VAL:v[0-9]+]],230; GCN-DAG: s_mov_b32 [[LIT:s[0-9]+]], 0x1869f231; SI: v_bcnt_u32_b32_e64 [[RESULT:v[0-9]+]], [[VAL]], [[LIT]]232; VI: v_bcnt_u32_b32 [[RESULT:v[0-9]+]], [[VAL]], [[LIT]]233; GCN: buffer_store_dword [[RESULT]],234; GCN: s_endpgm235define amdgpu_kernel void @v_ctpop_i32_add_literal(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) nounwind {236 %tid = call i32 @llvm.amdgcn.workitem.id.x()237 %in.gep = getelementptr i32, ptr addrspace(1) %in, i32 %tid238 %val = load i32, ptr addrspace(1) %in.gep, align 4239 %ctpop = call i32 @llvm.ctpop.i32(i32 %val) nounwind readnone240 %add = add i32 %ctpop, 99999241 store i32 %add, ptr addrspace(1) %out, align 4242 ret void243}244 245; FUNC-LABEL: {{^}}v_ctpop_i32_add_var:246; GCN-DAG: {{buffer|flat}}_load_dword [[VAL:v[0-9]+]],247; GCN-DAG: s_load_dword [[VAR:s[0-9]+]],248; GCN: v_bcnt_u32_b32{{(_e64)*}} [[RESULT:v[0-9]+]], [[VAL]], [[VAR]]249; GCN: buffer_store_dword [[RESULT]],250; GCN: s_endpgm251 252; EG: BCNT_INT253define amdgpu_kernel void @v_ctpop_i32_add_var(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in, i32 %const) nounwind {254 %tid = call i32 @llvm.amdgcn.workitem.id.x()255 %in.gep = getelementptr i32, ptr addrspace(1) %in, i32 %tid256 %val = load i32, ptr addrspace(1) %in.gep, align 4257 %ctpop = call i32 @llvm.ctpop.i32(i32 %val) nounwind readnone258 %add = add i32 %ctpop, %const259 store i32 %add, ptr addrspace(1) %out, align 4260 ret void261}262 263; FUNC-LABEL: {{^}}v_ctpop_i32_add_var_inv:264; GCN-DAG: {{buffer|flat}}_load_dword [[VAL:v[0-9]+]],265; GCN-DAG: s_load_dword [[VAR:s[0-9]+]],266; GCN: v_bcnt_u32_b32{{(_e64)*}} [[RESULT:v[0-9]+]], [[VAL]], [[VAR]]267; GCN: buffer_store_dword [[RESULT]],268; GCN: s_endpgm269 270; EG: BCNT_INT271define amdgpu_kernel void @v_ctpop_i32_add_var_inv(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in, i32 %const) nounwind {272 %tid = call i32 @llvm.amdgcn.workitem.id.x()273 %in.gep = getelementptr i32, ptr addrspace(1) %in, i32 %tid274 %val = load i32, ptr addrspace(1) %in.gep, align 4275 %ctpop = call i32 @llvm.ctpop.i32(i32 %val) nounwind readnone276 %add = add i32 %const, %ctpop277 store i32 %add, ptr addrspace(1) %out, align 4278 ret void279}280 281; FUNC-LABEL: {{^}}v_ctpop_i32_add_vvar_inv:282; SI: buffer_load_dword [[VAR:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0 addr64283; SI: buffer_load_dword [[VAL:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], 0 addr64284; SI: v_bcnt_u32_b32_e32 [[RESULT:v[0-9]+]], [[VAR]], [[VAL]]285; VI: flat_load_dword [[VAR:v[0-9]+]], v[{{[0-9]+:[0-9]+}}]286; VI: flat_load_dword [[VAL:v[0-9]+]], v[{{[0-9]+:[0-9]+}}]287; VI: v_bcnt_u32_b32 [[RESULT:v[0-9]+]], [[VAR]], [[VAL]]288; GCN: buffer_store_dword [[RESULT]],289; GCN: s_endpgm290 291; EG: BCNT_INT292define amdgpu_kernel void @v_ctpop_i32_add_vvar_inv(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in, ptr addrspace(1) noalias %constptr) nounwind {293 %tid = call i32 @llvm.amdgcn.workitem.id.x()294 %in.gep = getelementptr i32, ptr addrspace(1) %in, i32 %tid295 %val = load i32, ptr addrspace(1) %in.gep, align 4296 %ctpop = call i32 @llvm.ctpop.i32(i32 %val) nounwind readnone297 %gep = getelementptr i32, ptr addrspace(1) %constptr, i32 %tid298 %const = load i32, ptr addrspace(1) %gep, align 4299 %add = add i32 %const, %ctpop300 store i32 %add, ptr addrspace(1) %out, align 4301 ret void302}303 304; FUNC-LABEL: {{^}}ctpop_i32_in_br:305; SI: s_load_dword [[VAL:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0xd306; VI: s_load_dword [[VAL:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0x34307; GCN: s_bcnt1_i32_b32 [[SRESULT:s[0-9]+]], [[VAL]]308; GCN: v_mov_b32_e32 [[RESULT:v[0-9]+]], [[SRESULT]]309; GCN: buffer_store_dword [[RESULT]],310; GCN: s_endpgm311; EG: BCNT_INT312define amdgpu_kernel void @ctpop_i32_in_br(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %ctpop_arg, [8 x i32], i32 %cond) {313entry:314 %tmp0 = icmp eq i32 %cond, 0315 br i1 %tmp0, label %if, label %else316 317if:318 %tmp2 = call i32 @llvm.ctpop.i32(i32 %ctpop_arg)319 br label %endif320 321else:322 %tmp3 = getelementptr i32, ptr addrspace(1) %in, i32 1323 %tmp4 = load i32, ptr addrspace(1) %tmp3324 br label %endif325 326endif:327 %tmp5 = phi i32 [%tmp2, %if], [%tmp4, %else]328 store i32 %tmp5, ptr addrspace(1) %out329 ret void330}331