315 lines · plain
1; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mattr=+mad-mac-f32-insts -denormal-fp-math-f32=preserve-sign < %s | FileCheck --check-prefixes=SI,GCN %s2; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tonga -denormal-fp-math=preserve-sign -denormal-fp-math-f32=preserve-sign -mattr=-flat-for-global < %s | FileCheck --check-prefixes=VI-FLUSH,GCN %s3; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tonga -denormal-fp-math=ieee -denormal-fp-math-f32=preserve-sign -mattr=-flat-for-global < %s | FileCheck -check-prefix=GCN %s4 5; GCN-LABEL: {{^}}mac_vvv:6; GCN: buffer_load_dword [[A:v[0-9]+]], off, s[{{[0-9]+:[0-9]+}}], 0 glc{{$}}7; GCN: buffer_load_dword [[B:v[0-9]+]], off, s[{{[0-9]+:[0-9]+}}], 0 offset:48; GCN: buffer_load_dword [[C:v[0-9]+]], off, s[{{[0-9]+:[0-9]+}}], 0 offset:89; GCN: v_mac_f32_e32 [[C]], [[A]], [[B]]10; GCN: buffer_store_dword [[C]]11define amdgpu_kernel void @mac_vvv(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {12entry:13 %b_ptr = getelementptr float, ptr addrspace(1) %in, i32 114 %c_ptr = getelementptr float, ptr addrspace(1) %in, i32 215 16 %a = load volatile float, ptr addrspace(1) %in17 %b = load volatile float, ptr addrspace(1) %b_ptr18 %c = load volatile float, ptr addrspace(1) %c_ptr19 20 %tmp0 = fmul float %a, %b21 %tmp1 = fadd float %tmp0, %c22 store float %tmp1, ptr addrspace(1) %out23 ret void24}25 26; GCN-LABEL: {{^}}mad_inline_sgpr_inline:27; GCN-NOT: v_mac_f3228; GCN: v_mad_f32 v{{[0-9]}}, s{{[0-9]+}}, 0.5, 0.529define amdgpu_kernel void @mad_inline_sgpr_inline(ptr addrspace(1) %out, float %in) #0 {30entry:31 %tmp0 = fmul float 0.5, %in32 %tmp1 = fadd float %tmp0, 0.533 store float %tmp1, ptr addrspace(1) %out34 ret void35}36 37; GCN-LABEL: {{^}}mad_vvs:38; GCN-NOT: v_mac_f3239; GCN: v_mad_f32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, s{{[0-9]+}}40define amdgpu_kernel void @mad_vvs(ptr addrspace(1) %out, ptr addrspace(1) %in, float %c) #0 {41entry:42 %b_ptr = getelementptr float, ptr addrspace(1) %in, i32 143 44 %a = load float, ptr addrspace(1) %in45 %b = load float, ptr addrspace(1) %b_ptr46 47 %tmp0 = fmul float %a, %b48 %tmp1 = fadd float %tmp0, %c49 store float %tmp1, ptr addrspace(1) %out50 ret void51}52 53; GCN-LABEL: {{^}}mac_ssv:54; GCN: v_mac_f32_e64 v{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}55define amdgpu_kernel void @mac_ssv(ptr addrspace(1) %out, ptr addrspace(1) %in, float %a) #0 {56entry:57 %c = load float, ptr addrspace(1) %in58 59 %tmp0 = fmul float %a, %a60 %tmp1 = fadd float %tmp0, %c61 store float %tmp1, ptr addrspace(1) %out62 ret void63}64 65; GCN-LABEL: {{^}}mac_mad_same_add:66; GCN: v_mad_f32 v{{[0-9]}}, v{{[0-9]+}}, v{{[0-9]+}}, [[ADD:v[0-9]+]]67; GCN: v_mac_f32_e32 [[ADD]], v{{[0-9]+}}, v{{[0-9]+}}68define amdgpu_kernel void @mac_mad_same_add(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {69entry:70 %b_ptr = getelementptr float, ptr addrspace(1) %in, i32 171 %c_ptr = getelementptr float, ptr addrspace(1) %in, i32 272 %d_ptr = getelementptr float, ptr addrspace(1) %in, i32 373 %e_ptr = getelementptr float, ptr addrspace(1) %in, i32 474 75 %a = load volatile float, ptr addrspace(1) %in76 %b = load volatile float, ptr addrspace(1) %b_ptr77 %c = load volatile float, ptr addrspace(1) %c_ptr78 %d = load volatile float, ptr addrspace(1) %d_ptr79 %e = load volatile float, ptr addrspace(1) %e_ptr80 81 %tmp0 = fmul float %a, %b82 %tmp1 = fadd float %tmp0, %c83 84 %tmp2 = fmul float %d, %e85 %tmp3 = fadd float %tmp2, %c86 87 %out1 = getelementptr float, ptr addrspace(1) %out, i32 188 store float %tmp1, ptr addrspace(1) %out89 store float %tmp3, ptr addrspace(1) %out190 ret void91}92 93; There is no advantage to using v_mac when one of the operands is negated94; and v_mad accepts more operand types.95 96; GCN-LABEL: {{^}}mad_neg_src0:97; GCN-NOT: v_mac_f3298; GCN: v_mad_f32 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[-0-9]}}99define amdgpu_kernel void @mad_neg_src0(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {100entry:101 %b_ptr = getelementptr float, ptr addrspace(1) %in, i32 1102 %c_ptr = getelementptr float, ptr addrspace(1) %in, i32 2103 104 %a = load float, ptr addrspace(1) %in105 %b = load float, ptr addrspace(1) %b_ptr106 %c = load float, ptr addrspace(1) %c_ptr107 108 %neg_a = fneg float %a109 %tmp0 = fmul float %neg_a, %b110 %tmp1 = fadd float %tmp0, %c111 112 store float %tmp1, ptr addrspace(1) %out113 ret void114}115 116; GCN-LABEL: {{^}}nsz_mad_sub0_src0:117; GCN-NOT: v_mac_f32118; GCN: v_mad_f32 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[-0-9]}}119define amdgpu_kernel void @nsz_mad_sub0_src0(ptr addrspace(1) %out, ptr addrspace(1) %in) {120entry:121 %b_ptr = getelementptr float, ptr addrspace(1) %in, i32 1122 %c_ptr = getelementptr float, ptr addrspace(1) %in, i32 2123 124 %a = load float, ptr addrspace(1) %in125 %b = load float, ptr addrspace(1) %b_ptr126 %c = load float, ptr addrspace(1) %c_ptr127 128 %neg_a = fsub nsz float 0.0, %a129 %tmp0 = fmul float %neg_a, %b130 %tmp1 = fadd float %tmp0, %c131 132 store float %tmp1, ptr addrspace(1) %out133 ret void134}135 136; GCN-LABEL: {{^}}safe_mad_sub0_src0:137; GCN: v_sub_f32_e32 [[SUB0:v[0-9]+]], 0,138; GCN: v_ma{{[cd]}}_f32{{[_e32]*}} v{{[0-9]+}}, [[SUB0]], v{{[0-9]+}}139define amdgpu_kernel void @safe_mad_sub0_src0(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {140entry:141 %b_ptr = getelementptr float, ptr addrspace(1) %in, i32 1142 %c_ptr = getelementptr float, ptr addrspace(1) %in, i32 2143 144 %a = load float, ptr addrspace(1) %in145 %b = load float, ptr addrspace(1) %b_ptr146 %c = load float, ptr addrspace(1) %c_ptr147 148 %neg_a = fsub float 0.0, %a149 %tmp0 = fmul float %neg_a, %b150 %tmp1 = fadd float %tmp0, %c151 152 store float %tmp1, ptr addrspace(1) %out153 ret void154}155 156; GCN-LABEL: {{^}}mad_neg_src1:157; GCN-NOT: v_mac_f32158; GCN: v_mad_f32 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[-0-9]}}159define amdgpu_kernel void @mad_neg_src1(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {160entry:161 %b_ptr = getelementptr float, ptr addrspace(1) %in, i32 1162 %c_ptr = getelementptr float, ptr addrspace(1) %in, i32 2163 164 %a = load float, ptr addrspace(1) %in165 %b = load float, ptr addrspace(1) %b_ptr166 %c = load float, ptr addrspace(1) %c_ptr167 168 %neg_b = fneg float %b169 %tmp0 = fmul float %a, %neg_b170 %tmp1 = fadd float %tmp0, %c171 172 store float %tmp1, ptr addrspace(1) %out173 ret void174}175 176; GCN-LABEL: {{^}}nsz_mad_sub0_src1:177; GCN-NOT: v_mac_f32178; GCN: v_mad_f32 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, v{{[-0-9]}}179define amdgpu_kernel void @nsz_mad_sub0_src1(ptr addrspace(1) %out, ptr addrspace(1) %in) {180entry:181 %b_ptr = getelementptr float, ptr addrspace(1) %in, i32 1182 %c_ptr = getelementptr float, ptr addrspace(1) %in, i32 2183 184 %a = load float, ptr addrspace(1) %in185 %b = load float, ptr addrspace(1) %b_ptr186 %c = load float, ptr addrspace(1) %c_ptr187 188 %neg_b = fsub nsz float 0.0, %b189 %tmp0 = fmul float %a, %neg_b190 %tmp1 = fadd float %tmp0, %c191 192 store float %tmp1, ptr addrspace(1) %out193 ret void194}195 196; GCN-LABEL: {{^}}mad_neg_src2:197; GCN-NOT: v_mac198; GCN: v_mad_f32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, -v{{[-0-9]}}199define amdgpu_kernel void @mad_neg_src2(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {200entry:201 %b_ptr = getelementptr float, ptr addrspace(1) %in, i32 1202 %c_ptr = getelementptr float, ptr addrspace(1) %in, i32 2203 204 %a = load float, ptr addrspace(1) %in205 %b = load float, ptr addrspace(1) %b_ptr206 %c = load float, ptr addrspace(1) %c_ptr207 208 %neg_c = fneg float %c209 %tmp0 = fmul float %a, %b210 %tmp1 = fadd float %tmp0, %neg_c211 212 store float %tmp1, ptr addrspace(1) %out213 ret void214}215 216; Without special casing the inline constant check for v_mac_f32's217; src2, this fails to fold the 1.0 into a mad.218 219; GCN-LABEL: {{^}}fold_inline_imm_into_mac_src2_f32:220; GCN: {{buffer|flat}}_load_dword [[A:v[0-9]+]]221; GCN: {{buffer|flat}}_load_dword [[B:v[0-9]+]]222 223; GCN: v_add_f32_e32 [[TMP2:v[0-9]+]], [[A]], [[A]]224; GCN: v_mad_f32 v{{[0-9]+}}, [[TMP2]], -4.0, 1.0225define amdgpu_kernel void @fold_inline_imm_into_mac_src2_f32(ptr addrspace(1) %out, ptr addrspace(1) %a, ptr addrspace(1) %b) #3 {226bb:227 %tid = call i32 @llvm.amdgcn.workitem.id.x()228 %tid.ext = sext i32 %tid to i64229 %gep.a = getelementptr inbounds float, ptr addrspace(1) %a, i64 %tid.ext230 %gep.b = getelementptr inbounds float, ptr addrspace(1) %b, i64 %tid.ext231 %gep.out = getelementptr inbounds float, ptr addrspace(1) %out, i64 %tid.ext232 %tmp = load volatile float, ptr addrspace(1) %gep.a233 %tmp1 = load volatile float, ptr addrspace(1) %gep.b234 %tmp2 = fadd float %tmp, %tmp235 %tmp3 = fmul float %tmp2, 4.0236 %tmp4 = fsub float 1.0, %tmp3237 %tmp5 = fadd float %tmp4, %tmp1238 %tmp6 = fadd float %tmp1, %tmp1239 %tmp7 = fmul float %tmp6, %tmp240 %tmp8 = fsub float 1.0, %tmp7241 %tmp9 = fmul float %tmp8, 8.0242 %tmp10 = fadd float %tmp5, %tmp9243 store float %tmp10, ptr addrspace(1) %gep.out244 ret void245}246 247; GCN-LABEL: {{^}}fold_inline_imm_into_mac_src2_f16:248; GCN: {{buffer|flat}}_load_ushort [[A:v[0-9]+]]249; GCN: {{buffer|flat}}_load_ushort [[B:v[0-9]+]]250 251; SI-DAG: v_cvt_f32_f16_e32 [[CVT_A:v[0-9]+]], [[A]]252; SI-DAG: v_cvt_f32_f16_e32 [[CVT_B:v[0-9]+]], [[B]]253 254; SI: v_add_f32_e32 [[TMP2:v[0-9]+]], [[CVT_A]], [[CVT_A]]255; SI: v_mad_f32 v{{[0-9]+}}, [[TMP2]], -4.0, 1.0256; SI: v_madmk_f32 v{{[0-9]+}}, v{{[0-9]+}}, 0x41000000, v{{[0-9]+}}257 258; VI-FLUSH: v_add_f16_e32 [[TMP2:v[0-9]+]], [[A]], [[A]]259; VI-FLUSH: v_mad_f16 v{{[0-9]+}}, [[TMP2]], -4.0, 1.0260define amdgpu_kernel void @fold_inline_imm_into_mac_src2_f16(ptr addrspace(1) %out, ptr addrspace(1) %a, ptr addrspace(1) %b) #3 {261bb:262 %tid = call i32 @llvm.amdgcn.workitem.id.x()263 %tid.ext = sext i32 %tid to i64264 %gep.a = getelementptr inbounds half, ptr addrspace(1) %a, i64 %tid.ext265 %gep.b = getelementptr inbounds half, ptr addrspace(1) %b, i64 %tid.ext266 %gep.out = getelementptr inbounds half, ptr addrspace(1) %out, i64 %tid.ext267 %tmp = load volatile half, ptr addrspace(1) %gep.a268 %tmp1 = load volatile half, ptr addrspace(1) %gep.b269 %tmp2 = fadd half %tmp, %tmp270 %tmp3 = fmul half %tmp2, 4.0271 %tmp4 = fsub half 1.0, %tmp3272 %tmp5 = fadd half %tmp4, %tmp1273 %tmp6 = fadd half %tmp1, %tmp1274 %tmp7 = fmul half %tmp6, %tmp275 %tmp8 = fsub half 1.0, %tmp7276 %tmp9 = fmul half %tmp8, 8.0277 %tmp10 = fadd half %tmp5, %tmp9278 store half %tmp10, ptr addrspace(1) %gep.out279 ret void280}281 282; Need to assume denormal handling is needed for dynamic denormal mode283; GCN-LABEL: {{^}}v_mac_f32_dynamic:284; GCN: v_mul_f32285; GCN: v_add_f32286define float @v_mac_f32_dynamic(float %a, float %b, float %c) "denormal-fp-math-f32"="dynamic,dynamic" {287 %mul = fmul float %a, %b288 %mad = fadd float %mul, %c289 ret float %mad290}291 292; GCN-LABEL: {{^}}v_mac_f32_dynamic_daz:293; GCN: v_mul_f32294; GCN: v_add_f32295define float @v_mac_f32_dynamic_daz(float %a, float %b, float %c) "denormal-fp-math-f32"="preserve-sign,dynamic" {296 %mul = fmul float %a, %b297 %mad = fadd float %mul, %c298 ret float %mad299}300 301; GCN-LABEL: {{^}}v_mac_f32_dynamic_ftz:302; GCN: v_mul_f32303; GCN: v_add_f32304define float @v_mac_f32_dynamic_ftz(float %a, float %b, float %c) "denormal-fp-math-f32"="dynamic,preserve-sign" {305 %mul = fmul float %a, %b306 %mad = fadd float %mul, %c307 ret float %mad308}309 310declare i32 @llvm.amdgcn.workitem.id.x() #2311 312attributes #0 = { nounwind "no-signed-zeros-fp-math"="false" }313attributes #2 = { nounwind readnone }314attributes #3 = { nounwind }315