122 lines · plain
1; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck %s -check-prefixes=GCN,DPP64,GFX90A,DPP64-GFX9 -DCTL=row_newbcast2; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck %s -check-prefixes=GCN,DPP64,DPPMOV64,DPP64-GFX9,GFX942 -DCTL=row_newbcast3; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck %s -check-prefixes=GCN,DPP32,GFX10PLUS,GFX10 -DCTL=row_share4; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck %s -check-prefixes=GCN,DPP32,GFX10PLUS,GFX11 -DCTL=row_share5; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck %s -check-prefixes=GCN,DPP32,GFX1250 -DCTL=row_share6; RUN: llc -mtriple=amdgcn -mcpu=gfx1251 < %s | FileCheck %s -check-prefixes=GCN,DPP64,DPPMOV64,DPP64-GFX1251 -DCTL=row_share7 8; GCN-LABEL: {{^}}dpp64_ceil:9; GCN: global_load_{{dwordx2|b64}} [[V:v\[[0-9:]+\]]],10; DPP64: v_ceil_f64_dpp [[V]], [[V]] [[CTL]]:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}11; DPP32-COUNT-2: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} [[CTL]]:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}12define amdgpu_kernel void @dpp64_ceil(ptr addrspace(1) %arg, i64 %in1) {13 %id = tail call i32 @llvm.amdgcn.workitem.id.x()14 %gep = getelementptr inbounds i64, ptr addrspace(1) %arg, i32 %id15 %load = load i64, ptr addrspace(1) %gep16 %tmp0 = call i64 @llvm.amdgcn.update.dpp.i64(i64 %in1, i64 %load, i32 337, i32 15, i32 15, i1 1) #017 %tmp1 = bitcast i64 %tmp0 to double18 %round = tail call double @llvm.ceil.f64(double %tmp1)19 %tmp2 = bitcast double %round to i6420 store i64 %tmp2, ptr addrspace(1) %gep21 ret void22}23 24; GCN-LABEL: {{^}}dpp64_rcp:25; GCN: global_load_{{dwordx2|b64}} [[V:v\[[0-9:]+\]]],26; DPP64-GFX9: v_rcp_f64_dpp [[V]], [[V]] [[CTL]]:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}27; DPP64-GFX1251: v_mov_b64_dpp v[{{[0-9:]+}}], [[V]] [[CTL]]:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}28; DPP64-GFX1251: v_rcp_f64_e3229; DPP32-COUNT-2: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} [[CTL]]:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}30define amdgpu_kernel void @dpp64_rcp(ptr addrspace(1) %arg, i64 %in1) {31 %id = tail call i32 @llvm.amdgcn.workitem.id.x()32 %gep = getelementptr inbounds i64, ptr addrspace(1) %arg, i32 %id33 %load = load i64, ptr addrspace(1) %gep34 %tmp0 = call i64 @llvm.amdgcn.update.dpp.i64(i64 %in1, i64 %load, i32 337, i32 15, i32 15, i1 1) #035 %tmp1 = bitcast i64 %tmp0 to double36 %rcp = call double @llvm.amdgcn.rcp.f64(double %tmp1)37 %tmp2 = bitcast double %rcp to i6438 store i64 %tmp2, ptr addrspace(1) %gep39 ret void40}41 42; GCN-LABEL: {{^}}dpp64_rcp_unsupported_ctl:43; GCN-COUNT-2: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}44; GCN: v_rcp_f64_e3245define amdgpu_kernel void @dpp64_rcp_unsupported_ctl(ptr addrspace(1) %arg, i64 %in1) {46 %id = tail call i32 @llvm.amdgcn.workitem.id.x()47 %gep = getelementptr inbounds i64, ptr addrspace(1) %arg, i32 %id48 %load = load i64, ptr addrspace(1) %gep49 %tmp0 = call i64 @llvm.amdgcn.update.dpp.i64(i64 %in1, i64 %load, i32 1, i32 15, i32 15, i1 1) #050 %tmp1 = bitcast i64 %tmp0 to double51 %rcp = fdiv fast double 1.0, %tmp152 %tmp2 = bitcast double %rcp to i6453 store i64 %tmp2, ptr addrspace(1) %gep54 ret void55}56 57; GCN-LABEL: {{^}}dpp64_div:58; GCN: global_load_{{dwordx2|b64}} [[V:v\[[0-9:]+\]]],59; DPPMOV64: v_mov_b64_dpp v[{{[0-9:]+}}], [[V]] [[CTL]]:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}60; GFX90A-COUNT-2: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} [[CTL]]:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}61; DPP32-COUNT-2: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} [[CTL]]:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}62; GCN: v_div_scale_f6463; GCN: v_rcp_f64_e3264define amdgpu_kernel void @dpp64_div(ptr addrspace(1) %arg, i64 %in1) {65 %id = tail call i32 @llvm.amdgcn.workitem.id.x()66 %gep = getelementptr inbounds i64, ptr addrspace(1) %arg, i32 %id67 %load = load i64, ptr addrspace(1) %gep68 %tmp0 = call i64 @llvm.amdgcn.update.dpp.i64(i64 %in1, i64 %load, i32 337, i32 15, i32 15, i1 1) #069 %tmp1 = bitcast i64 %tmp0 to double70 %rcp = fdiv double 15.0, %tmp171 %tmp2 = bitcast double %rcp to i6472 store i64 %tmp2, ptr addrspace(1) %gep73 ret void74}75 76; On GFX9 it fails to combine because v_mul_lo_u32 has no e32 or dpp form.77; GCN-LABEL: {{^}}dpp_mul_row_share:78; GCN: global_load_{{dword|b32}} [[V:v[0-9]+]],79; DPP64-GFX9: v_mov_b32_e32 [[V2:v[0-9]+]], [[V]]80; DPP64-GFX9: v_mov_b32_dpp [[V2]], [[V2]] {{row_share|row_newbcast}}:0 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}81; DPP64-GFX9: v_mul_lo_u32 [[V]], [[V2]], [[V]]{{$}}82; GFX1250: v_mov_b32_e32 [[V2:v[0-9]+]], [[V]]83; GFX1250: v_mov_b32_dpp [[V2]], [[V2]] {{row_share|row_newbcast}}:0 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}84; GFX1250: v_mul_lo_u32 [[V]], [[V2]], [[V]]{{$}}85; DPP64-GFX1251: v_mul_lo_u32_e64_dpp [[V]], [[V]], [[V]] [[CTL]]:0 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}86define amdgpu_kernel void @dpp_mul_row_share(ptr addrspace(1) %arg) {87 %id = tail call i32 @llvm.amdgcn.workitem.id.x()88 %gep = getelementptr inbounds i32, ptr addrspace(1) %arg, i32 %id89 %load = load i32, ptr addrspace(1) %gep90 %tmp0 = call i32 @llvm.amdgcn.update.dpp.i32(i32 %load, i32 %load, i32 336, i32 15, i32 15, i1 1)91 %mul = mul i32 %tmp0, %load92 store i32 %mul, ptr addrspace(1) %gep93 ret void94}95 96; GCN-LABEL: {{^}}dpp64_loop:97; GCN: v_mov_b32_dpp98; DPP64: v_mov_b32_dpp99; GFX90A: v_add_co_u32_e32100; GFX90A: v_addc_co_u32_e32101; GFX942: v_lshl_add_u64102; GFX10PLUS: v_mov_b32_dpp103; GFX10PLUS: v_add_co_u32104; GFX10: v_add_co_ci_u32_e32105; GFX11: v_add_co_ci_u32_e64106define amdgpu_cs void @dpp64_loop(i64 %arg, i64 %val) {107bb:108 br label %bb1109bb1:110 %i = call i64 @llvm.amdgcn.update.dpp.i64(i64 poison, i64 %val, i32 0, i32 0, i32 0, i1 false)111 %i2 = add i64 %i, %arg112 %i3 = atomicrmw add ptr addrspace(1) null, i64 %i2 monotonic, align 8113 br label %bb1114}115 116declare i32 @llvm.amdgcn.workitem.id.x()117declare i64 @llvm.amdgcn.update.dpp.i64(i64, i64, i32, i32, i32, i1) #0118declare double @llvm.ceil.f64(double)119declare double @llvm.amdgcn.rcp.f64(double)120 121attributes #0 = { nounwind readnone convergent }122