414 lines · plain
1; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -denormal-fp-math-f32=preserve-sign < %s | FileCheck %s -check-prefixes=GCN,GFX9002; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -denormal-fp-math-f32=preserve-sign < %s | FileCheck %s -check-prefixes=GCN,GCN-DL-UNSAFE,GFX906-DL-UNSAFE3; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -denormal-fp-math-f32=preserve-sign < %s | FileCheck %s -check-prefixes=GCN,GCN-DL-UNSAFE,GFX10-DL-UNSAFE,GFX10-CONTRACT4; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -denormal-fp-math-f32=preserve-sign < %s | FileCheck %s -check-prefixes=GCN,GCN-DL-UNSAFE,GFX10-DL-UNSAFE,GFX10-CONTRACT5; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -denormal-fp-math-f32=preserve-sign < %s | FileCheck %s -check-prefixes=GCN,GFX9066; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -denormal-fp-math=preserve-sign -fp-contract=fast < %s | FileCheck %s -check-prefixes=GCN,GFX906-CONTRACT7; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -denormal-fp-math=ieee -fp-contract=fast < %s | FileCheck %s -check-prefixes=GCN,GFX906-DENORM-CONTRACT8; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -denormal-fp-math-f32=preserve-sign -mattr="+dot7-insts,-dot10-insts" < %s | FileCheck %s -check-prefixes=GCN,GFX906-DOT10-DISABLED9; (fadd (fmul S1.x, S2.x), (fadd (fmul (S1.y, S2.y), z))) -> (fdot2 S1, S2, z)10 11; Tests to make sure fdot2 is not generated when vector elements of dot-product expressions12; are not converted from f16 to f32.13; GCN-LABEL: {{^}}dotproduct_f16_contract14; GFX900: v_fma_f1615; GFX900: v_fma_f1616 17; GFX906-DL-UNSAFE: v_fma_f1618; GFX10-CONTRACT: v_fmac_f1619 20; GFX906-CONTRACT: v_mac_f16_e3221; GFX906-DENORM-CONTRACT: v_fma_f1622; GFX906-DOT10-DISABLED: v_fma_f1623 24define amdgpu_kernel void @dotproduct_f16_contract(ptr addrspace(1) %src1,25 ptr addrspace(1) %src2,26 ptr addrspace(1) nocapture %dst) {27entry:28 %src1.vec = load <2 x half>, ptr addrspace(1) %src129 %src2.vec = load <2 x half>, ptr addrspace(1) %src230 31 %src1.el1 = extractelement <2 x half> %src1.vec, i64 032 %src2.el1 = extractelement <2 x half> %src2.vec, i64 033 34 %src1.el2 = extractelement <2 x half> %src1.vec, i64 135 %src2.el2 = extractelement <2 x half> %src2.vec, i64 136 37 %mul2 = fmul contract half %src1.el2, %src2.el238 %mul1 = fmul contract half %src1.el1, %src2.el139 %acc = load half, ptr addrspace(1) %dst, align 240 %acc1 = fadd contract half %mul2, %acc41 %acc2 = fadd contract half %mul1, %acc142 store half %acc2, ptr addrspace(1) %dst, align 243 ret void44}45 46; GCN-LABEL: {{^}}dotproduct_f1647 48; GFX906: v_mul_f16_e3249; GFX906: v_mul_f16_e3250 51define amdgpu_kernel void @dotproduct_f16(ptr addrspace(1) %src1,52 ptr addrspace(1) %src2,53 ptr addrspace(1) nocapture %dst) {54entry:55 %src1.vec = load <2 x half>, ptr addrspace(1) %src156 %src2.vec = load <2 x half>, ptr addrspace(1) %src257 58 %src1.el1 = extractelement <2 x half> %src1.vec, i64 059 %src2.el1 = extractelement <2 x half> %src2.vec, i64 060 61 %src1.el2 = extractelement <2 x half> %src1.vec, i64 162 %src2.el2 = extractelement <2 x half> %src2.vec, i64 163 64 %mul2 = fmul half %src1.el2, %src2.el265 %mul1 = fmul half %src1.el1, %src2.el166 %acc = load half, ptr addrspace(1) %dst, align 267 %acc1 = fadd half %mul2, %acc68 %acc2 = fadd half %mul1, %acc169 store half %acc2, ptr addrspace(1) %dst, align 270 ret void71}72 73; We only want to generate fdot2 if:74; - vector element of dot product is converted from f16 to f32, and75; - the vectors are of type <2 x half>, and76; - "dot10-insts" is enabled77 78; GCN-LABEL: {{^}}dotproduct_f16_f32_contract79 80; GFX906-DL-UNSAFE: v_dot2_f32_f1681; GFX10-DL-UNSAFE: v_dot2c_f32_f1682 83; GFX906-CONTRACT: v_dot2_f32_f1684 85; GFX906-DENORM-CONTRACT: v_dot2_f32_f1686; GFX906-DOT10-DISABLED: v_fma_mix_f3287define amdgpu_kernel void @dotproduct_f16_f32_contract(ptr addrspace(1) %src1,88 ptr addrspace(1) %src2,89 ptr addrspace(1) nocapture %dst) {90entry:91 %src1.vec = load <2 x half>, ptr addrspace(1) %src192 %src2.vec = load <2 x half>, ptr addrspace(1) %src293 94 %src1.el1 = extractelement <2 x half> %src1.vec, i64 095 %csrc1.el1 = fpext half %src1.el1 to float96 %src2.el1 = extractelement <2 x half> %src2.vec, i64 097 %csrc2.el1 = fpext half %src2.el1 to float98 99 %src1.el2 = extractelement <2 x half> %src1.vec, i64 1100 %csrc1.el2 = fpext half %src1.el2 to float101 %src2.el2 = extractelement <2 x half> %src2.vec, i64 1102 %csrc2.el2 = fpext half %src2.el2 to float103 104 %mul2 = fmul contract float %csrc1.el2, %csrc2.el2105 %mul1 = fmul contract float %csrc1.el1, %csrc2.el1106 %acc = load float, ptr addrspace(1) %dst, align 4107 %acc1 = fadd contract float %mul2, %acc108 %acc2 = fadd contract float %mul1, %acc1109 store float %acc2, ptr addrspace(1) %dst, align 4110 ret void111}112 113; GCN-LABEL: {{^}}dotproduct_f16_f32114; GFX900: v_mad_mix_f32115; GFX900: v_mad_mix_f32116 117; GFX906: v_mad_f32118; GFX906: v_mac_f32_e32119 120define amdgpu_kernel void @dotproduct_f16_f32(ptr addrspace(1) %src1,121 ptr addrspace(1) %src2,122 ptr addrspace(1) nocapture %dst) {123entry:124 %src1.vec = load <2 x half>, ptr addrspace(1) %src1125 %src2.vec = load <2 x half>, ptr addrspace(1) %src2126 127 %src1.el1 = extractelement <2 x half> %src1.vec, i64 0128 %csrc1.el1 = fpext half %src1.el1 to float129 %src2.el1 = extractelement <2 x half> %src2.vec, i64 0130 %csrc2.el1 = fpext half %src2.el1 to float131 132 %src1.el2 = extractelement <2 x half> %src1.vec, i64 1133 %csrc1.el2 = fpext half %src1.el2 to float134 %src2.el2 = extractelement <2 x half> %src2.vec, i64 1135 %csrc2.el2 = fpext half %src2.el2 to float136 137 %mul2 = fmul float %csrc1.el2, %csrc2.el2138 %mul1 = fmul float %csrc1.el1, %csrc2.el1139 %acc = load float, ptr addrspace(1) %dst, align 4140 %acc1 = fadd float %mul2, %acc141 %acc2 = fadd float %mul1, %acc1142 store float %acc2, ptr addrspace(1) %dst, align 4143 ret void144}145 146; We only want to generate fdot2 if:147; - vector element of dot product is converted from f16 to f32, and148; - the vectors are of type <2 x half>, and149; - "dot10-insts" is enabled150 151; GCN-LABEL: {{^}}dotproduct_diffvecorder_contract152; GFX906-DL-UNSAFE: v_dot2_f32_f16153; GFX10-DL-UNSAFE: v_dot2c_f32_f16154 155; GFX906-CONTRACT: v_dot2_f32_f16156; GFX906-DENORM-CONTRACT: v_dot2_f32_f16157; GFX906-DOT10-DISABLED: v_fma_mix_f32158define amdgpu_kernel void @dotproduct_diffvecorder_contract(ptr addrspace(1) %src1,159 ptr addrspace(1) %src2,160 ptr addrspace(1) nocapture %dst) {161entry:162 %src1.vec = load <2 x half>, ptr addrspace(1) %src1163 %src2.vec = load <2 x half>, ptr addrspace(1) %src2164 165 %src1.el1 = extractelement <2 x half> %src1.vec, i64 0166 %csrc1.el1 = fpext half %src1.el1 to float167 %src2.el1 = extractelement <2 x half> %src2.vec, i64 0168 %csrc2.el1 = fpext half %src2.el1 to float169 170 %src1.el2 = extractelement <2 x half> %src1.vec, i64 1171 %csrc1.el2 = fpext half %src1.el2 to float172 %src2.el2 = extractelement <2 x half> %src2.vec, i64 1173 %csrc2.el2 = fpext half %src2.el2 to float174 175 %mul2 = fmul contract float %csrc2.el2, %csrc1.el2176 %mul1 = fmul contract float %csrc1.el1, %csrc2.el1177 %acc = load float, ptr addrspace(1) %dst, align 4178 %acc1 = fadd contract float %mul2, %acc179 %acc2 = fadd contract float %mul1, %acc1180 store float %acc2, ptr addrspace(1) %dst, align 4181 ret void182}183 184; GCN-LABEL: {{^}}dotproduct_diffvecorder185; GFX900: v_mad_mix_f32186; GFX900: v_mad_mix_f32187 188; GFX906: v_mad_f32189; GFX906: v_mac_f32_e32190 191define amdgpu_kernel void @dotproduct_diffvecorder(ptr addrspace(1) %src1,192 ptr addrspace(1) %src2,193 ptr addrspace(1) nocapture %dst) {194entry:195 %src1.vec = load <2 x half>, ptr addrspace(1) %src1196 %src2.vec = load <2 x half>, ptr addrspace(1) %src2197 198 %src1.el1 = extractelement <2 x half> %src1.vec, i64 0199 %csrc1.el1 = fpext half %src1.el1 to float200 %src2.el1 = extractelement <2 x half> %src2.vec, i64 0201 %csrc2.el1 = fpext half %src2.el1 to float202 203 %src1.el2 = extractelement <2 x half> %src1.vec, i64 1204 %csrc1.el2 = fpext half %src1.el2 to float205 %src2.el2 = extractelement <2 x half> %src2.vec, i64 1206 %csrc2.el2 = fpext half %src2.el2 to float207 208 %mul2 = fmul float %csrc2.el2, %csrc1.el2209 %mul1 = fmul float %csrc1.el1, %csrc2.el1210 %acc = load float, ptr addrspace(1) %dst, align 4211 %acc1 = fadd float %mul2, %acc212 %acc2 = fadd float %mul1, %acc1213 store float %acc2, ptr addrspace(1) %dst, align 4214 ret void215}216 217; Tests to make sure dot product is not generated when the vectors are not of <2 x half>.218; GCN-LABEL: {{^}}dotproduct_v4f16_contract219 220; GCN-DL-UNSAFE: v_fma_mix_f32221 222; GFX906-CONTRACT: v_fma_mix_f32223; GFX906-DENORM-CONTRACT: v_fma_mix_f32224; GFX906-DOT10-DISABLED: v_fma_mix_f32225define amdgpu_kernel void @dotproduct_v4f16_contract(ptr addrspace(1) %src1,226 ptr addrspace(1) %src2,227 ptr addrspace(1) nocapture %dst) {228entry:229 %src1.vec = load <4 x half>, ptr addrspace(1) %src1230 %src2.vec = load <4 x half>, ptr addrspace(1) %src2231 232 %src1.el1 = extractelement <4 x half> %src1.vec, i64 0233 %csrc1.el1 = fpext half %src1.el1 to float234 %src2.el1 = extractelement <4 x half> %src2.vec, i64 0235 %csrc2.el1 = fpext half %src2.el1 to float236 237 %src1.el2 = extractelement <4 x half> %src1.vec, i64 1238 %csrc1.el2 = fpext half %src1.el2 to float239 %src2.el2 = extractelement <4 x half> %src2.vec, i64 1240 %csrc2.el2 = fpext half %src2.el2 to float241 242 %mul2 = fmul contract float %csrc1.el2, %csrc2.el2243 %mul1 = fmul float %csrc1.el1, %csrc2.el1244 %acc = load float, ptr addrspace(1) %dst, align 4245 %acc1 = fadd contract float %mul2, %acc246 %acc2 = fadd contract float %mul1, %acc1247 store float %acc2, ptr addrspace(1) %dst, align 4248 ret void249}250 251; GCN-LABEL: {{^}}dotproduct_v4f16252; GFX900: v_mad_mix_f32253 254; GFX906: v_mad_f32255; GFX906: v_mac_f32_e32256 257define amdgpu_kernel void @dotproduct_v4f16(ptr addrspace(1) %src1,258 ptr addrspace(1) %src2,259 ptr addrspace(1) nocapture %dst) {260entry:261 %src1.vec = load <4 x half>, ptr addrspace(1) %src1262 %src2.vec = load <4 x half>, ptr addrspace(1) %src2263 264 %src1.el1 = extractelement <4 x half> %src1.vec, i64 0265 %csrc1.el1 = fpext half %src1.el1 to float266 %src2.el1 = extractelement <4 x half> %src2.vec, i64 0267 %csrc2.el1 = fpext half %src2.el1 to float268 269 %src1.el2 = extractelement <4 x half> %src1.vec, i64 1270 %csrc1.el2 = fpext half %src1.el2 to float271 %src2.el2 = extractelement <4 x half> %src2.vec, i64 1272 %csrc2.el2 = fpext half %src2.el2 to float273 274 %mul2 = fmul float %csrc1.el2, %csrc2.el2275 %mul1 = fmul float %csrc1.el1, %csrc2.el1276 %acc = load float, ptr addrspace(1) %dst, align 4277 %acc1 = fadd float %mul2, %acc278 %acc2 = fadd float %mul1, %acc1279 store float %acc2, ptr addrspace(1) %dst, align 4280 ret void281}282 283; GCN-LABEL: {{^}}NotAdotproductContract284 285; GCN-DL-UNSAFE: v_fma_mix_f32286 287; GFX906-CONTRACT: v_fma_mix_f32288; GFX906-DENORM-CONTRACT: v_fma_mix_f32289; GFX906-DOT10-DISABLED: v_fma_mix_f32290define amdgpu_kernel void @NotAdotproductContract(ptr addrspace(1) %src1,291 ptr addrspace(1) %src2,292 ptr addrspace(1) nocapture %dst) {293entry:294 %src1.vec = load <2 x half>, ptr addrspace(1) %src1295 %src2.vec = load <2 x half>, ptr addrspace(1) %src2296 297 %src1.el1 = extractelement <2 x half> %src1.vec, i64 0298 %csrc1.el1 = fpext half %src1.el1 to float299 %src2.el1 = extractelement <2 x half> %src2.vec, i64 0300 %csrc2.el1 = fpext half %src2.el1 to float301 302 %src1.el2 = extractelement <2 x half> %src1.vec, i64 1303 %csrc1.el2 = fpext half %src1.el2 to float304 %src2.el2 = extractelement <2 x half> %src2.vec, i64 1305 %csrc2.el2 = fpext half %src2.el2 to float306 307 %mul2 = fmul contract float %csrc1.el2, %csrc1.el1308 %mul1 = fmul contract float %csrc2.el1, %csrc2.el2309 %acc = load float, ptr addrspace(1) %dst, align 4310 %acc1 = fadd contract float %mul2, %acc311 %acc2 = fadd contract float %mul1, %acc1312 store float %acc2, ptr addrspace(1) %dst, align 4313 ret void314}315 316; GCN-LABEL: {{^}}NotAdotproduct317; GFX900: v_mad_mix_f32318; GFX900: v_mad_mix_f32319 320; GFX906: v_mad_f32321; GFX906: v_mac_f32_e32322 323define amdgpu_kernel void @NotAdotproduct(ptr addrspace(1) %src1,324 ptr addrspace(1) %src2,325 ptr addrspace(1) nocapture %dst) {326entry:327 %src1.vec = load <2 x half>, ptr addrspace(1) %src1328 %src2.vec = load <2 x half>, ptr addrspace(1) %src2329 330 %src1.el1 = extractelement <2 x half> %src1.vec, i64 0331 %csrc1.el1 = fpext half %src1.el1 to float332 %src2.el1 = extractelement <2 x half> %src2.vec, i64 0333 %csrc2.el1 = fpext half %src2.el1 to float334 335 %src1.el2 = extractelement <2 x half> %src1.vec, i64 1336 %csrc1.el2 = fpext half %src1.el2 to float337 %src2.el2 = extractelement <2 x half> %src2.vec, i64 1338 %csrc2.el2 = fpext half %src2.el2 to float339 340 %mul2 = fmul float %csrc1.el2, %csrc1.el1341 %mul1 = fmul float %csrc2.el1, %csrc2.el2342 %acc = load float, ptr addrspace(1) %dst, align 4343 %acc1 = fadd float %mul2, %acc344 %acc2 = fadd float %mul1, %acc1345 store float %acc2, ptr addrspace(1) %dst, align 4346 ret void347}348 349; GCN-LABEL: {{^}}Diff_Idx_NotAdotproductContract350 351; GCN-DL-UNSAFE: v_fma_mix_f32352 353; GFX906-CONTRACT: v_fma_mix_f32354; GFX906-DENORM-CONTRACT: v_fma_mix_f32355; GFX906-DOT10-DISABLED: v_fma_mix_f32356define amdgpu_kernel void @Diff_Idx_NotAdotproductContract(ptr addrspace(1) %src1,357 ptr addrspace(1) %src2,358 ptr addrspace(1) nocapture %dst) {359entry:360 %src1.vec = load <2 x half>, ptr addrspace(1) %src1361 %src2.vec = load <2 x half>, ptr addrspace(1) %src2362 363 %src1.el1 = extractelement <2 x half> %src1.vec, i64 0364 %csrc1.el1 = fpext half %src1.el1 to float365 %src2.el1 = extractelement <2 x half> %src2.vec, i64 0366 %csrc2.el1 = fpext half %src2.el1 to float367 368 %src1.el2 = extractelement <2 x half> %src1.vec, i64 1369 %csrc1.el2 = fpext half %src1.el2 to float370 %src2.el2 = extractelement <2 x half> %src2.vec, i64 1371 %csrc2.el2 = fpext half %src2.el2 to float372 373 %mul2 = fmul contract float %csrc1.el2, %csrc2.el1374 %mul1 = fmul contract float %csrc1.el1, %csrc2.el2375 %acc = load float, ptr addrspace(1) %dst, align 4376 %acc1 = fadd contract float %mul2, %acc377 %acc2 = fadd contract float %mul1, %acc1378 store float %acc2, ptr addrspace(1) %dst, align 4379 ret void380}381 382; GCN-LABEL: {{^}}Diff_Idx_NotAdotproduct383; GFX900: v_mad_mix_f32384; GFX900: v_mad_mix_f32385 386; GFX906: v_mad_f32387; GFX906: v_mac_f32_e32388 389define amdgpu_kernel void @Diff_Idx_NotAdotproduct(ptr addrspace(1) %src1,390 ptr addrspace(1) %src2,391 ptr addrspace(1) nocapture %dst) {392entry:393 %src1.vec = load <2 x half>, ptr addrspace(1) %src1394 %src2.vec = load <2 x half>, ptr addrspace(1) %src2395 396 %src1.el1 = extractelement <2 x half> %src1.vec, i64 0397 %csrc1.el1 = fpext half %src1.el1 to float398 %src2.el1 = extractelement <2 x half> %src2.vec, i64 0399 %csrc2.el1 = fpext half %src2.el1 to float400 401 %src1.el2 = extractelement <2 x half> %src1.vec, i64 1402 %csrc1.el2 = fpext half %src1.el2 to float403 %src2.el2 = extractelement <2 x half> %src2.vec, i64 1404 %csrc2.el2 = fpext half %src2.el2 to float405 406 %mul2 = fmul float %csrc1.el2, %csrc2.el1407 %mul1 = fmul float %csrc1.el1, %csrc2.el2408 %acc = load float, ptr addrspace(1) %dst, align 4409 %acc1 = fadd float %mul2, %acc410 %acc2 = fadd float %mul1, %acc1411 store float %acc2, ptr addrspace(1) %dst, align 4412 ret void413}414