635 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 42; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GCN,GFX-942 %s3; RUN: llc -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck --check-prefixes=GCN,GFX-950 %s4; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck --check-prefix=GFX1250 %s5 6; TODO: Add global-isel when it can support bf167 8define amdgpu_ps float @v_test_cvt_bf16_f32_v(bfloat %v) {9; GCN-LABEL: v_test_cvt_bf16_f32_v:10; GCN: ; %bb.0:11; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v012; GCN-NEXT: ; return to shader part epilog13;14; GFX1250-LABEL: v_test_cvt_bf16_f32_v:15; GFX1250: ; %bb.0:16; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 117; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v018; GFX1250-NEXT: ; return to shader part epilog19 %cvt = fpext bfloat %v to float20 ret float %cvt21}22 23define amdgpu_ps float @v_test_cvt_bf16_f32_s(bfloat inreg %v) {24; GCN-LABEL: v_test_cvt_bf16_f32_s:25; GCN: ; %bb.0:26; GCN-NEXT: s_lshl_b32 s0, s0, 1627; GCN-NEXT: v_mov_b32_e32 v0, s028; GCN-NEXT: ; return to shader part epilog29;30; GFX1250-LABEL: v_test_cvt_bf16_f32_s:31; GFX1250: ; %bb.0:32; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 133; GFX1250-NEXT: s_lshl_b32 s0, s0, 1634; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)35; GFX1250-NEXT: v_mov_b32_e32 v0, s036; GFX1250-NEXT: ; return to shader part epilog37 %cvt = fpext bfloat %v to float38 ret float %cvt39}40 41define amdgpu_ps float @v_test_cvt_v2f32_v2bf16_v(<2 x float> %src) {42; GFX-942-LABEL: v_test_cvt_v2f32_v2bf16_v:43; GFX-942: ; %bb.0:44; GFX-942-NEXT: v_bfe_u32 v2, v0, 16, 145; GFX-942-NEXT: s_movk_i32 s0, 0x7fff46; GFX-942-NEXT: v_add3_u32 v2, v2, v0, s047; GFX-942-NEXT: v_or_b32_e32 v3, 0x400000, v048; GFX-942-NEXT: v_cmp_u_f32_e32 vcc, v0, v049; GFX-942-NEXT: s_nop 150; GFX-942-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc51; GFX-942-NEXT: v_bfe_u32 v2, v1, 16, 152; GFX-942-NEXT: v_add3_u32 v2, v2, v1, s053; GFX-942-NEXT: v_or_b32_e32 v3, 0x400000, v154; GFX-942-NEXT: v_cmp_u_f32_e32 vcc, v1, v155; GFX-942-NEXT: s_mov_b32 s0, 0x706030256; GFX-942-NEXT: s_nop 057; GFX-942-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc58; GFX-942-NEXT: v_perm_b32 v0, v1, v0, s059; GFX-942-NEXT: ; return to shader part epilog60;61; GFX-950-LABEL: v_test_cvt_v2f32_v2bf16_v:62; GFX-950: ; %bb.0:63; GFX-950-NEXT: v_cvt_pk_bf16_f32 v0, v0, v164; GFX-950-NEXT: ; return to shader part epilog65;66; GFX1250-LABEL: v_test_cvt_v2f32_v2bf16_v:67; GFX1250: ; %bb.0:68; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 169; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v170; GFX1250-NEXT: ; return to shader part epilog71 %res = fptrunc <2 x float> %src to <2 x bfloat>72 %cast = bitcast <2 x bfloat> %res to float73 ret float %cast74}75 76define amdgpu_ps float @v_test_cvt_v2f32_v2bf16_s(<2 x float> inreg %src) {77; GFX-942-LABEL: v_test_cvt_v2f32_v2bf16_s:78; GFX-942: ; %bb.0:79; GFX-942-NEXT: s_bfe_u32 s2, s1, 0x1001080; GFX-942-NEXT: s_add_i32 s2, s2, s181; GFX-942-NEXT: s_or_b32 s4, s1, 0x40000082; GFX-942-NEXT: s_add_i32 s5, s2, 0x7fff83; GFX-942-NEXT: v_cmp_u_f32_e64 s[2:3], s1, s184; GFX-942-NEXT: s_and_b64 s[2:3], s[2:3], exec85; GFX-942-NEXT: s_cselect_b32 s1, s4, s586; GFX-942-NEXT: s_lshr_b32 s2, s1, 1687; GFX-942-NEXT: s_bfe_u32 s1, s0, 0x1001088; GFX-942-NEXT: s_add_i32 s1, s1, s089; GFX-942-NEXT: s_or_b32 s3, s0, 0x40000090; GFX-942-NEXT: s_add_i32 s4, s1, 0x7fff91; GFX-942-NEXT: v_cmp_u_f32_e64 s[0:1], s0, s092; GFX-942-NEXT: s_and_b64 s[0:1], s[0:1], exec93; GFX-942-NEXT: s_cselect_b32 s0, s3, s494; GFX-942-NEXT: s_lshr_b32 s0, s0, 1695; GFX-942-NEXT: s_pack_ll_b32_b16 s0, s0, s296; GFX-942-NEXT: v_mov_b32_e32 v0, s097; GFX-942-NEXT: ; return to shader part epilog98;99; GFX-950-LABEL: v_test_cvt_v2f32_v2bf16_s:100; GFX-950: ; %bb.0:101; GFX-950-NEXT: v_mov_b32_e32 v0, s1102; GFX-950-NEXT: v_cvt_pk_bf16_f32 v0, s0, v0103; GFX-950-NEXT: ; return to shader part epilog104;105; GFX1250-LABEL: v_test_cvt_v2f32_v2bf16_s:106; GFX1250: ; %bb.0:107; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1108; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, s0, s1109; GFX1250-NEXT: ; return to shader part epilog110 %res = fptrunc <2 x float> %src to <2 x bfloat>111 %cast = bitcast <2 x bfloat> %res to float112 ret float %cast113}114 115define amdgpu_ps float @v_test_cvt_f32_bf16_v(float %src) {116; GFX-942-LABEL: v_test_cvt_f32_bf16_v:117; GFX-942: ; %bb.0:118; GFX-942-NEXT: v_bfe_u32 v1, v0, 16, 1119; GFX-942-NEXT: s_movk_i32 s0, 0x7fff120; GFX-942-NEXT: v_add3_u32 v1, v1, v0, s0121; GFX-942-NEXT: v_or_b32_e32 v2, 0x400000, v0122; GFX-942-NEXT: v_cmp_u_f32_e32 vcc, v0, v0123; GFX-942-NEXT: s_nop 1124; GFX-942-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc125; GFX-942-NEXT: v_and_b32_e32 v0, 0xffff0000, v0126; GFX-942-NEXT: ; return to shader part epilog127;128; GFX-950-LABEL: v_test_cvt_f32_bf16_v:129; GFX-950: ; %bb.0:130; GFX-950-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0131; GFX-950-NEXT: v_lshlrev_b32_e32 v0, 16, v0132; GFX-950-NEXT: ; return to shader part epilog133;134; GFX1250-LABEL: v_test_cvt_f32_bf16_v:135; GFX1250: ; %bb.0:136; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1137; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0138; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)139; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0140; GFX1250-NEXT: ; return to shader part epilog141 %trunc = fptrunc float %src to bfloat142 %ext = fpext bfloat %trunc to float143 ret float %ext144}145 146define amdgpu_ps float @v_test_cvt_v2f64_v2bf16_v(<2 x double> %src) {147; GFX-942-LABEL: v_test_cvt_v2f64_v2bf16_v:148; GFX-942: ; %bb.0:149; GFX-942-NEXT: v_cvt_f32_f64_e32 v6, v[0:1]150; GFX-942-NEXT: v_cvt_f64_f32_e32 v[4:5], v6151; GFX-942-NEXT: v_and_b32_e32 v7, 1, v6152; GFX-942-NEXT: v_cmp_gt_f64_e64 s[2:3], |v[0:1]|, |v[4:5]|153; GFX-942-NEXT: v_cmp_nlg_f64_e32 vcc, v[0:1], v[4:5]154; GFX-942-NEXT: v_cmp_eq_u32_e64 s[0:1], 1, v7155; GFX-942-NEXT: v_cndmask_b32_e64 v4, -1, 1, s[2:3]156; GFX-942-NEXT: v_add_u32_e32 v4, v6, v4157; GFX-942-NEXT: s_or_b64 vcc, vcc, s[0:1]158; GFX-942-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc159; GFX-942-NEXT: v_bfe_u32 v5, v4, 16, 1160; GFX-942-NEXT: s_movk_i32 s4, 0x7fff161; GFX-942-NEXT: v_add3_u32 v5, v5, v4, s4162; GFX-942-NEXT: v_or_b32_e32 v4, 0x400000, v4163; GFX-942-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[0:1]164; GFX-942-NEXT: s_nop 1165; GFX-942-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc166; GFX-942-NEXT: v_cvt_f32_f64_e32 v5, v[2:3]167; GFX-942-NEXT: v_cvt_f64_f32_e32 v[0:1], v5168; GFX-942-NEXT: v_and_b32_e32 v6, 1, v5169; GFX-942-NEXT: v_cmp_gt_f64_e64 s[2:3], |v[2:3]|, |v[0:1]|170; GFX-942-NEXT: v_cmp_nlg_f64_e32 vcc, v[2:3], v[0:1]171; GFX-942-NEXT: v_cmp_eq_u32_e64 s[0:1], 1, v6172; GFX-942-NEXT: v_cndmask_b32_e64 v0, -1, 1, s[2:3]173; GFX-942-NEXT: v_add_u32_e32 v0, v5, v0174; GFX-942-NEXT: s_or_b64 vcc, vcc, s[0:1]175; GFX-942-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc176; GFX-942-NEXT: v_bfe_u32 v1, v0, 16, 1177; GFX-942-NEXT: v_add3_u32 v1, v1, v0, s4178; GFX-942-NEXT: v_or_b32_e32 v0, 0x400000, v0179; GFX-942-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[2:3]180; GFX-942-NEXT: s_mov_b32 s0, 0x7060302181; GFX-942-NEXT: s_nop 0182; GFX-942-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc183; GFX-942-NEXT: v_perm_b32 v0, v0, v4, s0184; GFX-942-NEXT: ; return to shader part epilog185;186; GFX-950-LABEL: v_test_cvt_v2f64_v2bf16_v:187; GFX-950: ; %bb.0:188; GFX-950-NEXT: v_cvt_f32_f64_e32 v6, v[2:3]189; GFX-950-NEXT: v_and_b32_e32 v4, 1, v6190; GFX-950-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4191; GFX-950-NEXT: v_cvt_f64_f32_e32 v[4:5], v6192; GFX-950-NEXT: v_cmp_gt_f64_e64 s[2:3], |v[2:3]|, |v[4:5]|193; GFX-950-NEXT: v_cmp_nlg_f64_e64 s[0:1], v[2:3], v[4:5]194; GFX-950-NEXT: v_cvt_f32_f64_e32 v7, v[0:1]195; GFX-950-NEXT: v_cndmask_b32_e64 v2, -1, 1, s[2:3]196; GFX-950-NEXT: v_add_u32_e32 v2, v6, v2197; GFX-950-NEXT: s_or_b64 vcc, vcc, s[0:1]198; GFX-950-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc199; GFX-950-NEXT: v_cvt_f64_f32_e32 v[2:3], v7200; GFX-950-NEXT: v_and_b32_e32 v8, 1, v7201; GFX-950-NEXT: v_cmp_gt_f64_e64 s[2:3], |v[0:1]|, |v[2:3]|202; GFX-950-NEXT: v_cmp_ne_u32_e32 vcc, 0, v8203; GFX-950-NEXT: v_cmp_nlg_f64_e64 s[0:1], v[0:1], v[2:3]204; GFX-950-NEXT: v_cndmask_b32_e64 v0, -1, 1, s[2:3]205; GFX-950-NEXT: v_add_u32_e32 v0, v7, v0206; GFX-950-NEXT: s_or_b64 vcc, vcc, s[0:1]207; GFX-950-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc208; GFX-950-NEXT: v_cvt_pk_bf16_f32 v0, v0, v4209; GFX-950-NEXT: ; return to shader part epilog210;211; GFX1250-LABEL: v_test_cvt_v2f64_v2bf16_v:212; GFX1250: ; %bb.0:213; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1214; GFX1250-NEXT: v_cvt_f32_f64_e32 v8, v[2:3]215; GFX1250-NEXT: v_cvt_f32_f64_e32 v9, v[0:1]216; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)217; GFX1250-NEXT: v_cvt_f64_f32_e32 v[4:5], v8218; GFX1250-NEXT: v_cvt_f64_f32_e32 v[6:7], v9219; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)220; GFX1250-NEXT: v_cmp_gt_f64_e64 s1, |v[2:3]|, |v[4:5]|221; GFX1250-NEXT: v_cmp_nlg_f64_e32 vcc_lo, v[2:3], v[4:5]222; GFX1250-NEXT: v_cmp_nlg_f64_e64 s0, v[0:1], v[6:7]223; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)224; GFX1250-NEXT: v_cndmask_b32_e64 v2, -1, 1, s1225; GFX1250-NEXT: v_cmp_gt_f64_e64 s1, |v[0:1]|, |v[6:7]|226; GFX1250-NEXT: v_dual_add_nc_u32 v1, v8, v2 :: v_dual_bitop2_b32 v10, 1, v8 bitop3:0x40227; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)228; GFX1250-NEXT: v_cndmask_b32_e64 v0, -1, 1, s1229; GFX1250-NEXT: v_and_b32_e32 v11, 1, v9230; GFX1250-NEXT: v_cmp_ne_u32_e64 s1, 0, v10231; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)232; GFX1250-NEXT: v_add_nc_u32_e32 v0, v9, v0233; GFX1250-NEXT: v_cmp_ne_u32_e64 s2, 0, v11234; GFX1250-NEXT: s_or_b32 vcc_lo, s1, vcc_lo235; GFX1250-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc_lo236; GFX1250-NEXT: s_or_b32 vcc_lo, s2, s0237; GFX1250-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc_lo238; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)239; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1240; GFX1250-NEXT: ; return to shader part epilog241 %res = fptrunc <2 x double> %src to <2 x bfloat>242 %cast = bitcast <2 x bfloat> %res to float243 ret float %cast244}245 246define amdgpu_ps float @fptrunc_f32_f32_to_v2bf16(float %a, float %b) {247; GFX-942-LABEL: fptrunc_f32_f32_to_v2bf16:248; GFX-942: ; %bb.0: ; %entry249; GFX-942-NEXT: v_bfe_u32 v2, v0, 16, 1250; GFX-942-NEXT: s_movk_i32 s0, 0x7fff251; GFX-942-NEXT: v_add3_u32 v2, v2, v0, s0252; GFX-942-NEXT: v_or_b32_e32 v3, 0x400000, v0253; GFX-942-NEXT: v_cmp_u_f32_e32 vcc, v0, v0254; GFX-942-NEXT: s_nop 1255; GFX-942-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc256; GFX-942-NEXT: v_bfe_u32 v2, v1, 16, 1257; GFX-942-NEXT: v_add3_u32 v2, v2, v1, s0258; GFX-942-NEXT: v_or_b32_e32 v3, 0x400000, v1259; GFX-942-NEXT: v_cmp_u_f32_e32 vcc, v1, v1260; GFX-942-NEXT: s_mov_b32 s0, 0x7060302261; GFX-942-NEXT: s_nop 0262; GFX-942-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc263; GFX-942-NEXT: v_perm_b32 v0, v1, v0, s0264; GFX-942-NEXT: ; return to shader part epilog265;266; GFX-950-LABEL: fptrunc_f32_f32_to_v2bf16:267; GFX-950: ; %bb.0: ; %entry268; GFX-950-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1269; GFX-950-NEXT: ; return to shader part epilog270;271; GFX1250-LABEL: fptrunc_f32_f32_to_v2bf16:272; GFX1250: ; %bb.0: ; %entry273; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1274; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1275; GFX1250-NEXT: ; return to shader part epilog276entry:277 %a.cvt = fptrunc float %a to bfloat278 %b.cvt = fptrunc float %b to bfloat279 %v2.1 = insertelement <2 x bfloat> poison, bfloat %a.cvt, i32 0280 %v2.2 = insertelement <2 x bfloat> %v2.1, bfloat %b.cvt, i32 1281 %ret = bitcast <2 x bfloat> %v2.2 to float282 ret float %ret283}284 285define amdgpu_ps float @fptrunc_f32_f32_to_v2bf16_mods(float %a, float %b) {286; GFX-942-LABEL: fptrunc_f32_f32_to_v2bf16_mods:287; GFX-942: ; %bb.0: ; %entry288; GFX-942-NEXT: v_xor_b32_e32 v2, 0x80000000, v0289; GFX-942-NEXT: v_bfe_u32 v3, v2, 16, 1290; GFX-942-NEXT: s_movk_i32 s0, 0x7fff291; GFX-942-NEXT: v_add3_u32 v3, v3, v2, s0292; GFX-942-NEXT: v_or_b32_e32 v2, 0x400000, v2293; GFX-942-NEXT: v_cmp_u_f32_e64 vcc, -v0, -v0294; GFX-942-NEXT: s_nop 1295; GFX-942-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc296; GFX-942-NEXT: v_and_b32_e32 v2, 0x7fffffff, v1297; GFX-942-NEXT: v_bfe_u32 v3, v2, 16, 1298; GFX-942-NEXT: v_add3_u32 v3, v3, v2, s0299; GFX-942-NEXT: v_or_b32_e32 v2, 0x400000, v2300; GFX-942-NEXT: v_cmp_u_f32_e64 vcc, |v1|, |v1|301; GFX-942-NEXT: s_mov_b32 s0, 0x7060302302; GFX-942-NEXT: s_nop 0303; GFX-942-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc304; GFX-942-NEXT: v_perm_b32 v0, v1, v0, s0305; GFX-942-NEXT: ; return to shader part epilog306;307; GFX-950-LABEL: fptrunc_f32_f32_to_v2bf16_mods:308; GFX-950: ; %bb.0: ; %entry309; GFX-950-NEXT: v_cvt_pk_bf16_f32 v0, -v0, |v1|310; GFX-950-NEXT: ; return to shader part epilog311;312; GFX1250-LABEL: fptrunc_f32_f32_to_v2bf16_mods:313; GFX1250: ; %bb.0: ; %entry314; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1315; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, -v0, |v1|316; GFX1250-NEXT: ; return to shader part epilog317entry:318 %a.neg = fneg float %a319 %a.cvt = fptrunc float %a.neg to bfloat320 %b.abs = call float @llvm.fabs.f32(float %b)321 %b.cvt = fptrunc float %b.abs to bfloat322 %v2.1 = insertelement <2 x bfloat> poison, bfloat %a.cvt, i32 0323 %v2.2 = insertelement <2 x bfloat> %v2.1, bfloat %b.cvt, i32 1324 %ret = bitcast <2 x bfloat> %v2.2 to float325 ret float %ret326}327 328define amdgpu_ps void @fptrunc_f32_to_bf16(float %a, ptr %out) {329; GFX-942-LABEL: fptrunc_f32_to_bf16:330; GFX-942: ; %bb.0: ; %entry331; GFX-942-NEXT: v_mov_b32_e32 v3, v2332; GFX-942-NEXT: v_mov_b32_e32 v2, v1333; GFX-942-NEXT: v_bfe_u32 v1, v0, 16, 1334; GFX-942-NEXT: s_movk_i32 s0, 0x7fff335; GFX-942-NEXT: v_add3_u32 v1, v1, v0, s0336; GFX-942-NEXT: v_or_b32_e32 v4, 0x400000, v0337; GFX-942-NEXT: v_cmp_u_f32_e32 vcc, v0, v0338; GFX-942-NEXT: s_nop 1339; GFX-942-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc340; GFX-942-NEXT: flat_store_short_d16_hi v[2:3], v0341; GFX-942-NEXT: s_endpgm342;343; GFX-950-LABEL: fptrunc_f32_to_bf16:344; GFX-950: ; %bb.0: ; %entry345; GFX-950-NEXT: v_mov_b32_e32 v3, v2346; GFX-950-NEXT: v_mov_b32_e32 v2, v1347; GFX-950-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0348; GFX-950-NEXT: flat_store_short v[2:3], v0349; GFX-950-NEXT: s_endpgm350;351; GFX1250-LABEL: fptrunc_f32_to_bf16:352; GFX1250: ; %bb.0: ; %entry353; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1354; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1355; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0356; GFX1250-NEXT: flat_store_b16 v[2:3], v0357; GFX1250-NEXT: s_endpgm358entry:359 %a.cvt = fptrunc float %a to bfloat360 store bfloat %a.cvt, ptr %out361 ret void362}363 364define amdgpu_ps void @fptrunc_f32_to_bf16_abs(float %a, ptr %out) {365; GFX-942-LABEL: fptrunc_f32_to_bf16_abs:366; GFX-942: ; %bb.0: ; %entry367; GFX-942-NEXT: v_mov_b32_e32 v3, v2368; GFX-942-NEXT: v_mov_b32_e32 v2, v1369; GFX-942-NEXT: v_and_b32_e32 v1, 0x7fffffff, v0370; GFX-942-NEXT: v_bfe_u32 v4, v1, 16, 1371; GFX-942-NEXT: s_movk_i32 s0, 0x7fff372; GFX-942-NEXT: v_add3_u32 v4, v4, v1, s0373; GFX-942-NEXT: v_or_b32_e32 v1, 0x400000, v1374; GFX-942-NEXT: v_cmp_u_f32_e64 vcc, |v0|, |v0|375; GFX-942-NEXT: s_nop 1376; GFX-942-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc377; GFX-942-NEXT: flat_store_short_d16_hi v[2:3], v0378; GFX-942-NEXT: s_endpgm379;380; GFX-950-LABEL: fptrunc_f32_to_bf16_abs:381; GFX-950: ; %bb.0: ; %entry382; GFX-950-NEXT: v_mov_b32_e32 v3, v2383; GFX-950-NEXT: v_mov_b32_e32 v2, v1384; GFX-950-NEXT: v_cvt_pk_bf16_f32 v0, |v0|, s0385; GFX-950-NEXT: flat_store_short v[2:3], v0386; GFX-950-NEXT: s_endpgm387;388; GFX1250-LABEL: fptrunc_f32_to_bf16_abs:389; GFX1250: ; %bb.0: ; %entry390; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1391; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1392; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, |v0|, s0393; GFX1250-NEXT: flat_store_b16 v[2:3], v0394; GFX1250-NEXT: s_endpgm395entry:396 %a.abs = call float @llvm.fabs.f32(float %a)397 %a.cvt = fptrunc float %a.abs to bfloat398 store bfloat %a.cvt, ptr %out399 ret void400}401 402define amdgpu_ps void @fptrunc_f32_to_bf16_neg(float %a, ptr %out) {403; GFX-942-LABEL: fptrunc_f32_to_bf16_neg:404; GFX-942: ; %bb.0: ; %entry405; GFX-942-NEXT: v_mov_b32_e32 v3, v2406; GFX-942-NEXT: v_mov_b32_e32 v2, v1407; GFX-942-NEXT: v_xor_b32_e32 v1, 0x80000000, v0408; GFX-942-NEXT: v_bfe_u32 v4, v1, 16, 1409; GFX-942-NEXT: s_movk_i32 s0, 0x7fff410; GFX-942-NEXT: v_add3_u32 v4, v4, v1, s0411; GFX-942-NEXT: v_or_b32_e32 v1, 0x400000, v1412; GFX-942-NEXT: v_cmp_u_f32_e64 vcc, -v0, -v0413; GFX-942-NEXT: s_nop 1414; GFX-942-NEXT: v_cndmask_b32_e32 v0, v4, v1, vcc415; GFX-942-NEXT: flat_store_short_d16_hi v[2:3], v0416; GFX-942-NEXT: s_endpgm417;418; GFX-950-LABEL: fptrunc_f32_to_bf16_neg:419; GFX-950: ; %bb.0: ; %entry420; GFX-950-NEXT: v_mov_b32_e32 v3, v2421; GFX-950-NEXT: v_mov_b32_e32 v2, v1422; GFX-950-NEXT: v_cvt_pk_bf16_f32 v0, -v0, s0423; GFX-950-NEXT: flat_store_short v[2:3], v0424; GFX-950-NEXT: s_endpgm425;426; GFX1250-LABEL: fptrunc_f32_to_bf16_neg:427; GFX1250: ; %bb.0: ; %entry428; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1429; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1430; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, -v0, s0431; GFX1250-NEXT: flat_store_b16 v[2:3], v0432; GFX1250-NEXT: s_endpgm433entry:434 %a.neg = fneg float %a435 %a.cvt = fptrunc float %a.neg to bfloat436 store bfloat %a.cvt, ptr %out437 ret void438}439 440define amdgpu_ps void @fptrunc_f64_to_bf16(double %a, ptr %out) {441; GFX-942-LABEL: fptrunc_f64_to_bf16:442; GFX-942: ; %bb.0: ; %entry443; GFX-942-NEXT: v_cvt_f32_f64_e32 v6, v[0:1]444; GFX-942-NEXT: v_cvt_f64_f32_e32 v[4:5], v6445; GFX-942-NEXT: v_and_b32_e32 v7, 1, v6446; GFX-942-NEXT: v_cmp_gt_f64_e64 s[2:3], |v[0:1]|, |v[4:5]|447; GFX-942-NEXT: v_cmp_nlg_f64_e32 vcc, v[0:1], v[4:5]448; GFX-942-NEXT: v_cmp_eq_u32_e64 s[0:1], 1, v7449; GFX-942-NEXT: v_cndmask_b32_e64 v4, -1, 1, s[2:3]450; GFX-942-NEXT: v_add_u32_e32 v4, v6, v4451; GFX-942-NEXT: s_or_b64 vcc, vcc, s[0:1]452; GFX-942-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc453; GFX-942-NEXT: v_bfe_u32 v5, v4, 16, 1454; GFX-942-NEXT: s_movk_i32 s0, 0x7fff455; GFX-942-NEXT: v_add3_u32 v5, v5, v4, s0456; GFX-942-NEXT: v_or_b32_e32 v4, 0x400000, v4457; GFX-942-NEXT: v_cmp_u_f64_e32 vcc, v[0:1], v[0:1]458; GFX-942-NEXT: s_nop 1459; GFX-942-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc460; GFX-942-NEXT: flat_store_short_d16_hi v[2:3], v0461; GFX-942-NEXT: s_endpgm462;463; GFX-950-LABEL: fptrunc_f64_to_bf16:464; GFX-950: ; %bb.0: ; %entry465; GFX-950-NEXT: v_cvt_f32_f64_e32 v6, v[0:1]466; GFX-950-NEXT: v_cvt_f64_f32_e32 v[4:5], v6467; GFX-950-NEXT: v_and_b32_e32 v7, 1, v6468; GFX-950-NEXT: v_cmp_gt_f64_e64 s[2:3], |v[0:1]|, |v[4:5]|469; GFX-950-NEXT: v_cmp_nlg_f64_e32 vcc, v[0:1], v[4:5]470; GFX-950-NEXT: v_cmp_eq_u32_e64 s[0:1], 1, v7471; GFX-950-NEXT: v_cndmask_b32_e64 v0, -1, 1, s[2:3]472; GFX-950-NEXT: v_add_u32_e32 v0, v6, v0473; GFX-950-NEXT: s_or_b64 vcc, vcc, s[0:1]474; GFX-950-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc475; GFX-950-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0476; GFX-950-NEXT: flat_store_short v[2:3], v0477; GFX-950-NEXT: s_endpgm478;479; GFX1250-LABEL: fptrunc_f64_to_bf16:480; GFX1250: ; %bb.0: ; %entry481; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1482; GFX1250-NEXT: v_cvt_f32_f64_e32 v6, v[0:1]483; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)484; GFX1250-NEXT: v_cvt_f64_f32_e32 v[4:5], v6485; GFX1250-NEXT: v_cmp_gt_f64_e64 s0, |v[0:1]|, |v[4:5]|486; GFX1250-NEXT: v_cmp_nlg_f64_e32 vcc_lo, v[0:1], v[4:5]487; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)488; GFX1250-NEXT: v_cndmask_b32_e64 v0, -1, 1, s0489; GFX1250-NEXT: v_dual_add_nc_u32 v0, v6, v0 :: v_dual_bitop2_b32 v7, 1, v6 bitop3:0x40490; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)491; GFX1250-NEXT: v_cmp_eq_u32_e64 s0, 1, v7492; GFX1250-NEXT: s_or_b32 vcc_lo, vcc_lo, s0493; GFX1250-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc_lo494; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0495; GFX1250-NEXT: flat_store_b16 v[2:3], v0496; GFX1250-NEXT: s_endpgm497entry:498 %a.cvt = fptrunc double %a to bfloat499 store bfloat %a.cvt, ptr %out500 ret void501}502 503define amdgpu_ps void @fptrunc_f64_to_bf16_neg(double %a, ptr %out) {504; GFX-942-LABEL: fptrunc_f64_to_bf16_neg:505; GFX-942: ; %bb.0: ; %entry506; GFX-942-NEXT: v_cvt_f32_f64_e64 v6, -v[0:1]507; GFX-942-NEXT: v_cvt_f64_f32_e32 v[4:5], v6508; GFX-942-NEXT: v_and_b32_e32 v7, 1, v6509; GFX-942-NEXT: v_cmp_gt_f64_e64 s[2:3], |v[0:1]|, |v[4:5]|510; GFX-942-NEXT: v_cmp_nlg_f64_e64 s[0:1], -v[0:1], v[4:5]511; GFX-942-NEXT: v_cmp_eq_u32_e32 vcc, 1, v7512; GFX-942-NEXT: v_cndmask_b32_e64 v4, -1, 1, s[2:3]513; GFX-942-NEXT: v_add_u32_e32 v4, v6, v4514; GFX-942-NEXT: s_or_b64 vcc, s[0:1], vcc515; GFX-942-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc516; GFX-942-NEXT: v_bfe_u32 v5, v4, 16, 1517; GFX-942-NEXT: s_movk_i32 s0, 0x7fff518; GFX-942-NEXT: v_add3_u32 v5, v5, v4, s0519; GFX-942-NEXT: v_or_b32_e32 v4, 0x400000, v4520; GFX-942-NEXT: v_cmp_u_f64_e64 vcc, -v[0:1], -v[0:1]521; GFX-942-NEXT: s_nop 1522; GFX-942-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc523; GFX-942-NEXT: flat_store_short_d16_hi v[2:3], v0524; GFX-942-NEXT: s_endpgm525;526; GFX-950-LABEL: fptrunc_f64_to_bf16_neg:527; GFX-950: ; %bb.0: ; %entry528; GFX-950-NEXT: v_cvt_f32_f64_e64 v6, -v[0:1]529; GFX-950-NEXT: v_cvt_f64_f32_e32 v[4:5], v6530; GFX-950-NEXT: v_and_b32_e32 v7, 1, v6531; GFX-950-NEXT: v_cmp_gt_f64_e64 s[2:3], |v[0:1]|, |v[4:5]|532; GFX-950-NEXT: v_cmp_nlg_f64_e64 s[0:1], -v[0:1], v[4:5]533; GFX-950-NEXT: v_cmp_eq_u32_e32 vcc, 1, v7534; GFX-950-NEXT: v_cndmask_b32_e64 v0, -1, 1, s[2:3]535; GFX-950-NEXT: v_add_u32_e32 v0, v6, v0536; GFX-950-NEXT: s_or_b64 vcc, s[0:1], vcc537; GFX-950-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc538; GFX-950-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0539; GFX-950-NEXT: flat_store_short v[2:3], v0540; GFX-950-NEXT: s_endpgm541;542; GFX1250-LABEL: fptrunc_f64_to_bf16_neg:543; GFX1250: ; %bb.0: ; %entry544; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1545; GFX1250-NEXT: v_cvt_f32_f64_e64 v6, -v[0:1]546; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)547; GFX1250-NEXT: v_cvt_f64_f32_e32 v[4:5], v6548; GFX1250-NEXT: v_cmp_gt_f64_e64 s1, |v[0:1]|, |v[4:5]|549; GFX1250-NEXT: v_cmp_nlg_f64_e64 s0, -v[0:1], v[4:5]550; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)551; GFX1250-NEXT: v_cndmask_b32_e64 v0, -1, 1, s1552; GFX1250-NEXT: v_dual_add_nc_u32 v0, v6, v0 :: v_dual_bitop2_b32 v7, 1, v6 bitop3:0x40553; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)554; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v7555; GFX1250-NEXT: s_or_b32 vcc_lo, s0, vcc_lo556; GFX1250-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc_lo557; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)558; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0559; GFX1250-NEXT: flat_store_b16 v[2:3], v0560; GFX1250-NEXT: s_endpgm561entry:562 %a.neg = fneg double %a563 %a.cvt = fptrunc double %a.neg to bfloat564 store bfloat %a.cvt, ptr %out565 ret void566}567 568define amdgpu_ps void @fptrunc_f64_to_bf16_abs(double %a, ptr %out) {569; GFX-942-LABEL: fptrunc_f64_to_bf16_abs:570; GFX-942: ; %bb.0: ; %entry571; GFX-942-NEXT: v_cvt_f32_f64_e64 v6, |v[0:1]|572; GFX-942-NEXT: v_cvt_f64_f32_e32 v[4:5], v6573; GFX-942-NEXT: v_and_b32_e32 v7, 1, v6574; GFX-942-NEXT: v_cmp_gt_f64_e64 s[2:3], |v[0:1]|, |v[4:5]|575; GFX-942-NEXT: v_cmp_nlg_f64_e64 s[0:1], |v[0:1]|, v[4:5]576; GFX-942-NEXT: v_cmp_eq_u32_e32 vcc, 1, v7577; GFX-942-NEXT: v_cndmask_b32_e64 v4, -1, 1, s[2:3]578; GFX-942-NEXT: v_add_u32_e32 v4, v6, v4579; GFX-942-NEXT: s_or_b64 vcc, s[0:1], vcc580; GFX-942-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc581; GFX-942-NEXT: v_bfe_u32 v5, v4, 16, 1582; GFX-942-NEXT: s_movk_i32 s0, 0x7fff583; GFX-942-NEXT: v_add3_u32 v5, v5, v4, s0584; GFX-942-NEXT: v_or_b32_e32 v4, 0x400000, v4585; GFX-942-NEXT: v_cmp_u_f64_e64 vcc, |v[0:1]|, |v[0:1]|586; GFX-942-NEXT: s_nop 1587; GFX-942-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc588; GFX-942-NEXT: flat_store_short_d16_hi v[2:3], v0589; GFX-942-NEXT: s_endpgm590;591; GFX-950-LABEL: fptrunc_f64_to_bf16_abs:592; GFX-950: ; %bb.0: ; %entry593; GFX-950-NEXT: v_cvt_f32_f64_e64 v6, |v[0:1]|594; GFX-950-NEXT: v_cvt_f64_f32_e32 v[4:5], v6595; GFX-950-NEXT: v_and_b32_e32 v7, 1, v6596; GFX-950-NEXT: v_cmp_gt_f64_e64 s[2:3], |v[0:1]|, |v[4:5]|597; GFX-950-NEXT: v_cmp_nlg_f64_e64 s[0:1], |v[0:1]|, v[4:5]598; GFX-950-NEXT: v_cmp_eq_u32_e32 vcc, 1, v7599; GFX-950-NEXT: v_cndmask_b32_e64 v0, -1, 1, s[2:3]600; GFX-950-NEXT: v_add_u32_e32 v0, v6, v0601; GFX-950-NEXT: s_or_b64 vcc, s[0:1], vcc602; GFX-950-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc603; GFX-950-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0604; GFX-950-NEXT: flat_store_short v[2:3], v0605; GFX-950-NEXT: s_endpgm606;607; GFX1250-LABEL: fptrunc_f64_to_bf16_abs:608; GFX1250: ; %bb.0: ; %entry609; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1610; GFX1250-NEXT: v_cvt_f32_f64_e64 v6, |v[0:1]|611; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)612; GFX1250-NEXT: v_cvt_f64_f32_e32 v[4:5], v6613; GFX1250-NEXT: v_cmp_gt_f64_e64 s1, |v[0:1]|, |v[4:5]|614; GFX1250-NEXT: v_cmp_nlg_f64_e64 s0, |v[0:1]|, v[4:5]615; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)616; GFX1250-NEXT: v_cndmask_b32_e64 v0, -1, 1, s1617; GFX1250-NEXT: v_dual_add_nc_u32 v0, v6, v0 :: v_dual_bitop2_b32 v7, 1, v6 bitop3:0x40618; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)619; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v7620; GFX1250-NEXT: s_or_b32 vcc_lo, s0, vcc_lo621; GFX1250-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc_lo622; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)623; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0624; GFX1250-NEXT: flat_store_b16 v[2:3], v0625; GFX1250-NEXT: s_endpgm626entry:627 %a.abs = call double @llvm.fabs.f64(double %a)628 %a.cvt = fptrunc double %a.abs to bfloat629 store bfloat %a.cvt, ptr %out630 ret void631}632 633declare float @llvm.fabs.f32(float)634declare double @llvm.fabs.f64(double)635