618 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 42; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck --check-prefix=GCN %s3 4; TODO: Add global-isel when it can support bf165define amdgpu_ps void @llvm_sqrt_bf16_v(ptr addrspace(1) %out, bfloat %src) {6; GCN-LABEL: llvm_sqrt_bf16_v:7; GCN: ; %bb.0:8; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 19; GCN-NEXT: v_sqrt_bf16_e32 v2, v210; GCN-NEXT: global_store_b16 v[0:1], v2, off11; GCN-NEXT: s_endpgm12 %sqrt = call bfloat @llvm.sqrt.bf16(bfloat %src)13 store bfloat %sqrt, ptr addrspace(1) %out, align 214 ret void15}16 17define amdgpu_ps void @llvm_sqrt_bf16_s(ptr addrspace(1) %out, bfloat inreg %src) {18; GCN-LABEL: llvm_sqrt_bf16_s:19; GCN: ; %bb.0:20; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 121; GCN-NEXT: v_sqrt_bf16_e32 v2, s022; GCN-NEXT: global_store_b16 v[0:1], v2, off23; GCN-NEXT: s_endpgm24 %sqrt = call bfloat @llvm.sqrt.bf16(bfloat %src)25 store bfloat %sqrt, ptr addrspace(1) %out, align 226 ret void27}28 29define amdgpu_ps void @v_test_add_v2bf16_vv(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> %b) {30; GCN-LABEL: v_test_add_v2bf16_vv:31; GCN: ; %bb.0:32; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 133; GCN-NEXT: v_pk_add_bf16 v2, v2, v334; GCN-NEXT: global_store_b32 v[0:1], v2, off35; GCN-NEXT: s_endpgm36 %add = fadd <2 x bfloat> %a, %b37 store <2 x bfloat> %add, ptr addrspace(1) %out38 ret void39}40 41define amdgpu_ps void @v_test_add_v2bf16_vs(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b) {42; GCN-LABEL: v_test_add_v2bf16_vs:43; GCN: ; %bb.0:44; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 145; GCN-NEXT: v_pk_add_bf16 v2, v2, s046; GCN-NEXT: global_store_b32 v[0:1], v2, off47; GCN-NEXT: s_endpgm48 %add = fadd <2 x bfloat> %a, %b49 store <2 x bfloat> %add, ptr addrspace(1) %out50 ret void51}52 53define amdgpu_ps void @v_test_add_v2bf16_ss(ptr addrspace(1) %out, <2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {54; GCN-LABEL: v_test_add_v2bf16_ss:55; GCN: ; %bb.0:56; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 157; GCN-NEXT: v_pk_add_bf16 v2, s0, s158; GCN-NEXT: global_store_b32 v[0:1], v2, off59; GCN-NEXT: s_endpgm60 %add = fadd <2 x bfloat> %a, %b61 store <2 x bfloat> %add, ptr addrspace(1) %out62 ret void63}64 65define amdgpu_ps void @v_test_add_v2bf16_vc(ptr addrspace(1) %out, <2 x bfloat> %a) {66; GCN-LABEL: v_test_add_v2bf16_vc:67; GCN: ; %bb.0:68; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 169; GCN-NEXT: v_pk_add_bf16 v2, v2, 2.0 op_sel_hi:[1,0]70; GCN-NEXT: global_store_b32 v[0:1], v2, off71; GCN-NEXT: s_endpgm72 %add = fadd <2 x bfloat> %a, <bfloat 2.0, bfloat 2.0>73 store <2 x bfloat> %add, ptr addrspace(1) %out74 ret void75}76 77define amdgpu_ps void @v_test_add_v2bf16_vl(ptr addrspace(1) %out, <2 x bfloat> %a) {78; GCN-LABEL: v_test_add_v2bf16_vl:79; GCN: ; %bb.0:80; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 181; GCN-NEXT: v_pk_add_bf16 v2, 0x42c83f80, v282; GCN-NEXT: global_store_b32 v[0:1], v2, off83; GCN-NEXT: s_endpgm84 %add = fadd <2 x bfloat> %a, <bfloat 1.0, bfloat 100.0>85 store <2 x bfloat> %add, ptr addrspace(1) %out86 ret void87}88 89define amdgpu_ps void @v_test_sub_v2bf16_vv(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> %b) {90; GCN-LABEL: v_test_sub_v2bf16_vv:91; GCN: ; %bb.0:92; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 193; GCN-NEXT: v_pk_add_bf16 v2, v2, v3 neg_lo:[0,1] neg_hi:[0,1]94; GCN-NEXT: global_store_b32 v[0:1], v2, off95; GCN-NEXT: s_endpgm96 %add = fsub <2 x bfloat> %a, %b97 store <2 x bfloat> %add, ptr addrspace(1) %out98 ret void99}100 101define amdgpu_ps void @v_test_sub_v2bf16_vs(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b) {102; GCN-LABEL: v_test_sub_v2bf16_vs:103; GCN: ; %bb.0:104; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1105; GCN-NEXT: v_pk_add_bf16 v2, v2, s0 neg_lo:[0,1] neg_hi:[0,1]106; GCN-NEXT: global_store_b32 v[0:1], v2, off107; GCN-NEXT: s_endpgm108 %add = fsub <2 x bfloat> %a, %b109 store <2 x bfloat> %add, ptr addrspace(1) %out110 ret void111}112 113define amdgpu_ps void @v_test_sub_v2bf16_ss(ptr addrspace(1) %out, <2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {114; GCN-LABEL: v_test_sub_v2bf16_ss:115; GCN: ; %bb.0:116; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1117; GCN-NEXT: v_pk_add_bf16 v2, s0, s1 neg_lo:[0,1] neg_hi:[0,1]118; GCN-NEXT: global_store_b32 v[0:1], v2, off119; GCN-NEXT: s_endpgm120 %add = fsub <2 x bfloat> %a, %b121 store <2 x bfloat> %add, ptr addrspace(1) %out122 ret void123}124 125define amdgpu_ps void @v_test_sub_v2bf16_vc(ptr addrspace(1) %out, <2 x bfloat> %a) {126; GCN-LABEL: v_test_sub_v2bf16_vc:127; GCN: ; %bb.0:128; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1129; GCN-NEXT: v_pk_add_bf16 v2, v2, -2.0 op_sel_hi:[1,0]130; GCN-NEXT: global_store_b32 v[0:1], v2, off131; GCN-NEXT: s_endpgm132 %add = fsub <2 x bfloat> %a, <bfloat 2.0, bfloat 2.0>133 store <2 x bfloat> %add, ptr addrspace(1) %out134 ret void135}136 137define amdgpu_ps void @v_test_sub_v2bf16_vl(ptr addrspace(1) %out, <2 x bfloat> %a) {138; GCN-LABEL: v_test_sub_v2bf16_vl:139; GCN: ; %bb.0:140; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1141; GCN-NEXT: v_pk_add_bf16 v2, 0xc2c8bf80, v2142; GCN-NEXT: global_store_b32 v[0:1], v2, off143; GCN-NEXT: s_endpgm144 %add = fsub <2 x bfloat> %a, <bfloat 1.0, bfloat 100.0>145 store <2 x bfloat> %add, ptr addrspace(1) %out146 ret void147}148 149define amdgpu_ps void @v_test_sub_v2bf16_lv(ptr addrspace(1) %out, <2 x bfloat> %a) {150; GCN-LABEL: v_test_sub_v2bf16_lv:151; GCN: ; %bb.0:152; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1153; GCN-NEXT: v_pk_add_bf16 v2, 0x42c83f80, v2 neg_lo:[0,1] neg_hi:[0,1]154; GCN-NEXT: global_store_b32 v[0:1], v2, off155; GCN-NEXT: s_endpgm156 %add = fsub <2 x bfloat> <bfloat 1.0, bfloat 100.0>, %a157 store <2 x bfloat> %add, ptr addrspace(1) %out158 ret void159}160 161define amdgpu_ps void @v_test_sub_v2bf16_iv(ptr addrspace(1) %out, <2 x bfloat> %a) {162; GCN-LABEL: v_test_sub_v2bf16_iv:163; GCN: ; %bb.0:164; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1165; GCN-NEXT: v_pk_add_bf16 v2, v2, 1.0 op_sel_hi:[1,0] neg_lo:[1,0] neg_hi:[1,0]166; GCN-NEXT: global_store_b32 v[0:1], v2, off167; GCN-NEXT: s_endpgm168 %add = fsub <2 x bfloat> <bfloat 1.0, bfloat 1.0>, %a169 store <2 x bfloat> %add, ptr addrspace(1) %out170 ret void171}172 173define amdgpu_ps void @v_test_mul_v2bf16_vv(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> %b) {174; GCN-LABEL: v_test_mul_v2bf16_vv:175; GCN: ; %bb.0:176; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1177; GCN-NEXT: v_pk_mul_bf16 v2, v2, v3178; GCN-NEXT: global_store_b32 v[0:1], v2, off179; GCN-NEXT: s_endpgm180 %mul = fmul <2 x bfloat> %a, %b181 store <2 x bfloat> %mul, ptr addrspace(1) %out182 ret void183}184 185define amdgpu_ps void @v_test_mul_v2bf16_vs(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b) {186; GCN-LABEL: v_test_mul_v2bf16_vs:187; GCN: ; %bb.0:188; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1189; GCN-NEXT: v_pk_mul_bf16 v2, v2, s0190; GCN-NEXT: global_store_b32 v[0:1], v2, off191; GCN-NEXT: s_endpgm192 %mul = fmul <2 x bfloat> %a, %b193 store <2 x bfloat> %mul, ptr addrspace(1) %out194 ret void195}196 197define amdgpu_ps void @v_test_mul_v2bf16_ss(ptr addrspace(1) %out, <2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {198; GCN-LABEL: v_test_mul_v2bf16_ss:199; GCN: ; %bb.0:200; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1201; GCN-NEXT: v_pk_mul_bf16 v2, s0, s1202; GCN-NEXT: global_store_b32 v[0:1], v2, off203; GCN-NEXT: s_endpgm204 %mul = fmul <2 x bfloat> %a, %b205 store <2 x bfloat> %mul, ptr addrspace(1) %out206 ret void207}208 209; FIXME: We can do better folding inline constant instead of a literal.210 211define amdgpu_ps void @v_test_mul_v2bf16_vc(ptr addrspace(1) %out, <2 x bfloat> %a) {212; GCN-LABEL: v_test_mul_v2bf16_vc:213; GCN: ; %bb.0:214; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1215; GCN-NEXT: v_pk_mul_bf16 v2, v2, 0.5 op_sel_hi:[1,0]216; GCN-NEXT: global_store_b32 v[0:1], v2, off217; GCN-NEXT: s_endpgm218 %mul = fmul <2 x bfloat> %a, <bfloat 0.5, bfloat 0.5>219 store <2 x bfloat> %mul, ptr addrspace(1) %out220 ret void221}222 223define amdgpu_ps void @v_test_mul_v2bf16_vl(ptr addrspace(1) %out, <2 x bfloat> %a) {224; GCN-LABEL: v_test_mul_v2bf16_vl:225; GCN: ; %bb.0:226; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1227; GCN-NEXT: v_pk_mul_bf16 v2, 0x42c83f80, v2228; GCN-NEXT: global_store_b32 v[0:1], v2, off229; GCN-NEXT: s_endpgm230 %mul = fmul <2 x bfloat> %a, <bfloat 1.0, bfloat 100.0>231 store <2 x bfloat> %mul, ptr addrspace(1) %out232 ret void233}234 235define amdgpu_ps void @v_test_min_v2bf16_vv(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> %b) {236; GCN-LABEL: v_test_min_v2bf16_vv:237; GCN: ; %bb.0:238; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1239; GCN-NEXT: v_pk_min_num_bf16 v2, v2, v3240; GCN-NEXT: global_store_b32 v[0:1], v2, off241; GCN-NEXT: s_endpgm242 %min = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)243 store <2 x bfloat> %min, ptr addrspace(1) %out244 ret void245}246 247define amdgpu_ps void @v_test_min_v2bf16_vs(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b) {248; GCN-LABEL: v_test_min_v2bf16_vs:249; GCN: ; %bb.0:250; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1251; GCN-NEXT: v_pk_min_num_bf16 v2, v2, s0252; GCN-NEXT: global_store_b32 v[0:1], v2, off253; GCN-NEXT: s_endpgm254 %min = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)255 store <2 x bfloat> %min, ptr addrspace(1) %out256 ret void257}258 259define amdgpu_ps void @v_test_min_v2bf16_ss(ptr addrspace(1) %out, <2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {260; GCN-LABEL: v_test_min_v2bf16_ss:261; GCN: ; %bb.0:262; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1263; GCN-NEXT: v_pk_min_num_bf16 v2, s0, s1264; GCN-NEXT: global_store_b32 v[0:1], v2, off265; GCN-NEXT: s_endpgm266 %min = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)267 store <2 x bfloat> %min, ptr addrspace(1) %out268 ret void269}270 271define amdgpu_ps void @v_test_min_v2bf16_vc(ptr addrspace(1) %out, <2 x bfloat> %a) {272; GCN-LABEL: v_test_min_v2bf16_vc:273; GCN: ; %bb.0:274; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1275; GCN-NEXT: v_pk_min_num_bf16 v2, v2, 0.5 op_sel_hi:[1,0]276; GCN-NEXT: global_store_b32 v[0:1], v2, off277; GCN-NEXT: s_endpgm278 %min = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 0.5, bfloat 0.5>)279 store <2 x bfloat> %min, ptr addrspace(1) %out280 ret void281}282 283define amdgpu_ps void @v_test_min_v2bf16_vl(ptr addrspace(1) %out, <2 x bfloat> %a) {284; GCN-LABEL: v_test_min_v2bf16_vl:285; GCN: ; %bb.0:286; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1287; GCN-NEXT: v_pk_min_num_bf16 v2, 0x42c83f80, v2288; GCN-NEXT: global_store_b32 v[0:1], v2, off289; GCN-NEXT: s_endpgm290 %min = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 1.0, bfloat 100.0>)291 store <2 x bfloat> %min, ptr addrspace(1) %out292 ret void293}294 295define amdgpu_ps void @v_test_max_v2bf16_vv(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> %b) {296; GCN-LABEL: v_test_max_v2bf16_vv:297; GCN: ; %bb.0:298; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1299; GCN-NEXT: v_pk_max_num_bf16 v2, v2, v3300; GCN-NEXT: global_store_b32 v[0:1], v2, off301; GCN-NEXT: s_endpgm302 %max = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)303 store <2 x bfloat> %max, ptr addrspace(1) %out304 ret void305}306 307define amdgpu_ps void @v_test_max_v2bf16_vs(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b) {308; GCN-LABEL: v_test_max_v2bf16_vs:309; GCN: ; %bb.0:310; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1311; GCN-NEXT: v_pk_max_num_bf16 v2, v2, s0312; GCN-NEXT: global_store_b32 v[0:1], v2, off313; GCN-NEXT: s_endpgm314 %max = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)315 store <2 x bfloat> %max, ptr addrspace(1) %out316 ret void317}318 319define amdgpu_ps void @v_test_max_v2bf16_ss(ptr addrspace(1) %out, <2 x bfloat> inreg %a, <2 x bfloat> inreg %b) {320; GCN-LABEL: v_test_max_v2bf16_ss:321; GCN: ; %bb.0:322; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1323; GCN-NEXT: v_pk_max_num_bf16 v2, s0, s1324; GCN-NEXT: global_store_b32 v[0:1], v2, off325; GCN-NEXT: s_endpgm326 %max = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)327 store <2 x bfloat> %max, ptr addrspace(1) %out328 ret void329}330 331define amdgpu_ps void @v_test_max_v2bf16_vc(ptr addrspace(1) %out, <2 x bfloat> %a) {332; GCN-LABEL: v_test_max_v2bf16_vc:333; GCN: ; %bb.0:334; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1335; GCN-NEXT: v_pk_max_num_bf16 v2, v2, 0.5 op_sel_hi:[1,0]336; GCN-NEXT: global_store_b32 v[0:1], v2, off337; GCN-NEXT: s_endpgm338 %max = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 0.5, bfloat 0.5>)339 store <2 x bfloat> %max, ptr addrspace(1) %out340 ret void341}342 343define amdgpu_ps void @v_test_max_v2bf16_vl(ptr addrspace(1) %out, <2 x bfloat> %a) {344; GCN-LABEL: v_test_max_v2bf16_vl:345; GCN: ; %bb.0:346; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1347; GCN-NEXT: v_pk_max_num_bf16 v2, 0x42c83f80, v2348; GCN-NEXT: global_store_b32 v[0:1], v2, off349; GCN-NEXT: s_endpgm350 %max = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 1.0, bfloat 100.0>)351 store <2 x bfloat> %max, ptr addrspace(1) %out352 ret void353}354 355define amdgpu_ps bfloat @test_clamp_bf16(bfloat %src) {356; GCN-LABEL: test_clamp_bf16:357; GCN: ; %bb.0:358; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1359; GCN-NEXT: v_pk_max_num_bf16 v0, v0, v0 clamp360; GCN-NEXT: ; return to shader part epilog361 %max = call bfloat @llvm.maxnum.bf16(bfloat %src, bfloat 0.0)362 %clamp = call bfloat @llvm.minnum.bf16(bfloat %max, bfloat 1.0)363 ret bfloat %clamp364}365 366define amdgpu_ps bfloat @test_clamp_bf16_s(bfloat inreg %src) {367; GCN-LABEL: test_clamp_bf16_s:368; GCN: ; %bb.0:369; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1370; GCN-NEXT: v_pk_max_num_bf16 v0, s0, s0 clamp371; GCN-NEXT: ; return to shader part epilog372 %max = call bfloat @llvm.maxnum.bf16(bfloat %src, bfloat 0.0)373 %clamp = call bfloat @llvm.minnum.bf16(bfloat %max, bfloat 1.0)374 ret bfloat %clamp375}376 377define amdgpu_ps float @test_clamp_v2bf16(<2 x bfloat> %src) {378; GCN-LABEL: test_clamp_v2bf16:379; GCN: ; %bb.0:380; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1381; GCN-NEXT: v_pk_max_num_bf16 v0, v0, v0 clamp382; GCN-NEXT: ; return to shader part epilog383 %max = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %src, <2 x bfloat> <bfloat 0.0, bfloat 0.0>)384 %clamp = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %max, <2 x bfloat> <bfloat 1.0, bfloat 1.0>)385 %ret = bitcast <2 x bfloat> %clamp to float386 ret float %ret387}388 389define amdgpu_ps float @test_clamp_v2bf16_s(<2 x bfloat> inreg %src) {390; GCN-LABEL: test_clamp_v2bf16_s:391; GCN: ; %bb.0:392; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1393; GCN-NEXT: v_pk_max_num_bf16 v0, s0, s0 clamp394; GCN-NEXT: ; return to shader part epilog395 %max = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %src, <2 x bfloat> <bfloat 0.0, bfloat 0.0>)396 %clamp = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %max, <2 x bfloat> <bfloat 1.0, bfloat 1.0>)397 %ret = bitcast <2 x bfloat> %clamp to float398 ret float %ret399}400 401define amdgpu_ps bfloat @test_clamp_bf16_folding(bfloat %src) {402; GCN-LABEL: test_clamp_bf16_folding:403; GCN: ; %bb.0:404; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1405; GCN-NEXT: v_exp_bf16_e32 v0, v0406; GCN-NEXT: v_nop407; GCN-NEXT: s_delay_alu instid0(TRANS32_DEP_1)408; GCN-NEXT: v_pk_max_num_bf16 v0, v0, v0 clamp409; GCN-NEXT: ; return to shader part epilog410 %exp = call bfloat @llvm.exp2.bf16(bfloat %src)411 %max = call bfloat @llvm.maxnum.bf16(bfloat %exp, bfloat 0.0)412 %clamp = call bfloat @llvm.minnum.bf16(bfloat %max, bfloat 1.0)413 ret bfloat %clamp414}415 416define amdgpu_ps float @test_clamp_v2bf16_folding(<2 x bfloat> %src0, <2 x bfloat> %src1) {417; GCN-LABEL: test_clamp_v2bf16_folding:418; GCN: ; %bb.0:419; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1420; GCN-NEXT: v_pk_mul_bf16 v0, v0, v1 clamp421; GCN-NEXT: ; return to shader part epilog422 %mul = fmul <2 x bfloat> %src0, %src1423 %max = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %mul, <2 x bfloat> <bfloat 0.0, bfloat 0.0>)424 %clamp = call <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %max, <2 x bfloat> <bfloat 1.0, bfloat 1.0>)425 %ret = bitcast <2 x bfloat> %clamp to float426 ret float %ret427}428 429define amdgpu_ps void @v_test_mul_add_v2bf16_vvv(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) {430; GCN-LABEL: v_test_mul_add_v2bf16_vvv:431; GCN: ; %bb.0:432; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1433; GCN-NEXT: v_pk_fma_bf16 v2, v2, v3, v4434; GCN-NEXT: global_store_b32 v[0:1], v2, off435; GCN-NEXT: s_endpgm436 %mul = fmul contract <2 x bfloat> %a, %b437 %add = fadd contract <2 x bfloat> %mul, %c438 store <2 x bfloat> %add, ptr addrspace(1) %out439 ret void440}441 442define amdgpu_ps void @v_test_mul_add_v2bf16_vss(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b, <2 x bfloat> inreg %c) {443; GCN-LABEL: v_test_mul_add_v2bf16_vss:444; GCN: ; %bb.0:445; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1446; GCN-NEXT: v_pk_fma_bf16 v2, v2, s0, s1447; GCN-NEXT: global_store_b32 v[0:1], v2, off448; GCN-NEXT: s_endpgm449 %mul = fmul contract <2 x bfloat> %a, %b450 %add = fadd contract <2 x bfloat> %mul, %c451 store <2 x bfloat> %add, ptr addrspace(1) %out452 ret void453}454 455define amdgpu_ps void @v_test_mul_add_v2bf16_sss(ptr addrspace(1) %out, <2 x bfloat> inreg %a, <2 x bfloat> inreg %b, <2 x bfloat> inreg %c) {456; GCN-LABEL: v_test_mul_add_v2bf16_sss:457; GCN: ; %bb.0:458; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1459; GCN-NEXT: v_mov_b32_e32 v2, s2460; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)461; GCN-NEXT: v_pk_fma_bf16 v2, s0, s1, v2462; GCN-NEXT: global_store_b32 v[0:1], v2, off463; GCN-NEXT: s_endpgm464 %mul = fmul contract <2 x bfloat> %a, %b465 %add = fadd contract <2 x bfloat> %mul, %c466 store <2 x bfloat> %add, ptr addrspace(1) %out467 ret void468}469 470define amdgpu_ps void @v_test_mul_add_v2bf16_vsc(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b) {471; GCN-LABEL: v_test_mul_add_v2bf16_vsc:472; GCN: ; %bb.0:473; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1474; GCN-NEXT: v_pk_fma_bf16 v2, v2, s0, 0.5 op_sel_hi:[1,1,0]475; GCN-NEXT: global_store_b32 v[0:1], v2, off476; GCN-NEXT: s_endpgm477 %mul = fmul contract <2 x bfloat> %a, %b478 %add = fadd contract <2 x bfloat> %mul, <bfloat 0.5, bfloat 0.5>479 store <2 x bfloat> %add, ptr addrspace(1) %out480 ret void481}482 483define amdgpu_ps void @v_test_mul_add_v2bf16_vll(ptr addrspace(1) %out, <2 x bfloat> %a) {484; GCN-LABEL: v_test_mul_add_v2bf16_vll:485; GCN: ; %bb.0:486; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1487; GCN-NEXT: s_mov_b32 s0, 0x43484000488; GCN-NEXT: s_delay_alu instid0(SALU_CYCLE_1)489; GCN-NEXT: v_pk_fma_bf16 v2, 0x42c83f80, v2, s0490; GCN-NEXT: global_store_b32 v[0:1], v2, off491; GCN-NEXT: s_endpgm492 %mul = fmul contract <2 x bfloat> %a, <bfloat 1.0, bfloat 100.0>493 %add = fadd contract <2 x bfloat> %mul, <bfloat 2.0, bfloat 200.0>494 store <2 x bfloat> %add, ptr addrspace(1) %out495 ret void496}497 498define amdgpu_ps void @v_test_fma_v2bf16_vvv(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) {499; GCN-LABEL: v_test_fma_v2bf16_vvv:500; GCN: ; %bb.0:501; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1502; GCN-NEXT: v_pk_fma_bf16 v2, v2, v3, v4503; GCN-NEXT: global_store_b32 v[0:1], v2, off504; GCN-NEXT: s_endpgm505 %fma = call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)506 store <2 x bfloat> %fma, ptr addrspace(1) %out507 ret void508}509 510define amdgpu_ps void @v_test_fma_v2bf16_vss(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b, <2 x bfloat> inreg %c) {511; GCN-LABEL: v_test_fma_v2bf16_vss:512; GCN: ; %bb.0:513; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1514; GCN-NEXT: v_pk_fma_bf16 v2, v2, s0, s1515; GCN-NEXT: global_store_b32 v[0:1], v2, off516; GCN-NEXT: s_endpgm517 %fma = call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)518 store <2 x bfloat> %fma, ptr addrspace(1) %out519 ret void520}521 522define amdgpu_ps void @v_test_fma_v2bf16_sss(ptr addrspace(1) %out, <2 x bfloat> inreg %a, <2 x bfloat> inreg %b, <2 x bfloat> inreg %c) {523; GCN-LABEL: v_test_fma_v2bf16_sss:524; GCN: ; %bb.0:525; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1526; GCN-NEXT: v_mov_b32_e32 v2, s2527; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1)528; GCN-NEXT: v_pk_fma_bf16 v2, s0, s1, v2529; GCN-NEXT: global_store_b32 v[0:1], v2, off530; GCN-NEXT: s_endpgm531 %fma = call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)532 store <2 x bfloat> %fma, ptr addrspace(1) %out533 ret void534}535 536define amdgpu_ps void @v_test_fma_v2bf16_vsc(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b) {537; GCN-LABEL: v_test_fma_v2bf16_vsc:538; GCN: ; %bb.0:539; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1540; GCN-NEXT: v_pk_fma_bf16 v2, v2, s0, 0.5 op_sel_hi:[1,1,0]541; GCN-NEXT: global_store_b32 v[0:1], v2, off542; GCN-NEXT: s_endpgm543 %fma = call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> <bfloat 0.5, bfloat 0.5>)544 store <2 x bfloat> %fma, ptr addrspace(1) %out545 ret void546}547 548define amdgpu_ps void @v_test_fma_v2bf16_vll(ptr addrspace(1) %out, <2 x bfloat> %a) {549; GCN-LABEL: v_test_fma_v2bf16_vll:550; GCN: ; %bb.0:551; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1552; GCN-NEXT: s_mov_b32 s0, 0x42c83f80553; GCN-NEXT: s_delay_alu instid0(SALU_CYCLE_1)554; GCN-NEXT: v_pk_fma_bf16 v2, v2, s0, 0x43484000555; GCN-NEXT: global_store_b32 v[0:1], v2, off556; GCN-NEXT: s_endpgm557 %fma = call <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 1.0, bfloat 100.0>, <2 x bfloat> <bfloat 2.0, bfloat 200.0>)558 store <2 x bfloat> %fma, ptr addrspace(1) %out559 ret void560}561 562define amdgpu_ps void @llvm_log2_bf16_v(ptr addrspace(1) %out, bfloat %src) {563; GCN-LABEL: llvm_log2_bf16_v:564; GCN: ; %bb.0:565; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1566; GCN-NEXT: v_log_bf16_e32 v2, v2567; GCN-NEXT: global_store_b16 v[0:1], v2, off568; GCN-NEXT: s_endpgm569 %log = call bfloat @llvm.log2.bf16(bfloat %src)570 store bfloat %log, ptr addrspace(1) %out, align 2571 ret void572}573 574define amdgpu_ps void @llvm_log2_bf16_s(ptr addrspace(1) %out, bfloat inreg %src) {575; GCN-LABEL: llvm_log2_bf16_s:576; GCN: ; %bb.0:577; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1578; GCN-NEXT: v_log_bf16_e32 v2, s0579; GCN-NEXT: global_store_b16 v[0:1], v2, off580; GCN-NEXT: s_endpgm581 %log = call bfloat @llvm.log2.bf16(bfloat %src)582 store bfloat %log, ptr addrspace(1) %out, align 2583 ret void584}585 586define amdgpu_ps void @llvm_exp2_bf16_v(ptr addrspace(1) %out, bfloat %src) {587; GCN-LABEL: llvm_exp2_bf16_v:588; GCN: ; %bb.0:589; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1590; GCN-NEXT: v_exp_bf16_e32 v2, v2591; GCN-NEXT: global_store_b16 v[0:1], v2, off592; GCN-NEXT: s_endpgm593 %exp = call bfloat @llvm.exp2.bf16(bfloat %src)594 store bfloat %exp, ptr addrspace(1) %out, align 2595 ret void596}597 598define amdgpu_ps void @llvm_exp2_bf16_s(ptr addrspace(1) %out, bfloat inreg %src) {599; GCN-LABEL: llvm_exp2_bf16_s:600; GCN: ; %bb.0:601; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1602; GCN-NEXT: v_exp_bf16_e32 v2, s0603; GCN-NEXT: global_store_b16 v[0:1], v2, off604; GCN-NEXT: s_endpgm605 %exp = call bfloat @llvm.exp2.bf16(bfloat %src)606 store bfloat %exp, ptr addrspace(1) %out, align 2607 ret void608}609 610declare bfloat @llvm.minnum.bf16(bfloat, bfloat)611declare bfloat @llvm.maxnum.bf16(bfloat, bfloat)612declare <2 x bfloat> @llvm.minnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)613declare <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)614declare <2 x bfloat> @llvm.fma.v2bf16(<2 x bfloat>, <2 x bfloat>, <2 x bfloat>)615declare bfloat @llvm.sqrt.bf16(bfloat)616declare bfloat @llvm.log2.bf16(bfloat)617declare bfloat @llvm.exp2.bf16(bfloat)618