1145 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 22; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri < %s | FileCheck --check-prefixes=CIVI,CI %s3; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=tonga < %s | FileCheck --check-prefixes=CIVI,VI %s4; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefixes=GFX9 %s5; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck --check-prefixes=GFX11,GFX11-TRUE16 %s6; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck --check-prefixes=GFX11,GFX11-FAKE16 %s7 8define amdgpu_kernel void @fneg_fabs_fadd_bf16(ptr addrspace(1) %out, bfloat %x, bfloat %y) {9; CI-LABEL: fneg_fabs_fadd_bf16:10; CI: ; %bb.0:11; CI-NEXT: s_load_dword s2, s[8:9], 0x212; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x013; CI-NEXT: s_add_i32 s12, s12, s1714; CI-NEXT: s_mov_b32 flat_scratch_lo, s1315; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 816; CI-NEXT: s_waitcnt lgkmcnt(0)17; CI-NEXT: s_and_b32 s3, s2, 0x7fff18; CI-NEXT: s_lshl_b32 s3, s3, 1619; CI-NEXT: s_and_b32 s2, s2, 0xffff000020; CI-NEXT: v_mov_b32_e32 v0, s321; CI-NEXT: v_sub_f32_e32 v0, s2, v022; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v023; CI-NEXT: v_mov_b32_e32 v0, s024; CI-NEXT: v_mov_b32_e32 v1, s125; CI-NEXT: flat_store_short v[0:1], v226; CI-NEXT: s_endpgm27;28; VI-LABEL: fneg_fabs_fadd_bf16:29; VI: ; %bb.0:30; VI-NEXT: s_load_dword s2, s[8:9], 0x831; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x032; VI-NEXT: s_add_i32 s12, s12, s1733; VI-NEXT: s_mov_b32 flat_scratch_lo, s1334; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 835; VI-NEXT: s_waitcnt lgkmcnt(0)36; VI-NEXT: s_and_b32 s3, s2, 0x7fff37; VI-NEXT: s_lshl_b32 s3, s3, 1638; VI-NEXT: s_and_b32 s2, s2, 0xffff000039; VI-NEXT: v_mov_b32_e32 v0, s340; VI-NEXT: v_sub_f32_e32 v0, s2, v041; VI-NEXT: v_bfe_u32 v1, v0, 16, 142; VI-NEXT: v_add_u32_e32 v1, vcc, v1, v043; VI-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v144; VI-NEXT: v_or_b32_e32 v2, 0x400000, v045; VI-NEXT: v_cmp_u_f32_e32 vcc, v0, v046; VI-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc47; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v048; VI-NEXT: v_mov_b32_e32 v0, s049; VI-NEXT: v_mov_b32_e32 v1, s150; VI-NEXT: flat_store_short v[0:1], v251; VI-NEXT: s_endpgm52;53; GFX9-LABEL: fneg_fabs_fadd_bf16:54; GFX9: ; %bb.0:55; GFX9-NEXT: s_load_dword s2, s[8:9], 0x856; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x057; GFX9-NEXT: v_mov_b32_e32 v0, 058; GFX9-NEXT: s_waitcnt lgkmcnt(0)59; GFX9-NEXT: s_and_b32 s3, s2, 0x7fff60; GFX9-NEXT: s_lshl_b32 s3, s3, 1661; GFX9-NEXT: s_and_b32 s2, s2, 0xffff000062; GFX9-NEXT: v_mov_b32_e32 v1, s363; GFX9-NEXT: v_sub_f32_e32 v1, s2, v164; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 165; GFX9-NEXT: v_add_u32_e32 v2, v2, v166; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v167; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v268; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v169; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc70; GFX9-NEXT: global_store_short_d16_hi v0, v1, s[0:1]71; GFX9-NEXT: s_endpgm72;73; GFX11-LABEL: fneg_fabs_fadd_bf16:74; GFX11: ; %bb.0:75; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x876; GFX11-NEXT: s_waitcnt lgkmcnt(0)77; GFX11-NEXT: s_and_b32 s1, s0, 0x7fff78; GFX11-NEXT: s_and_b32 s0, s0, 0xffff000079; GFX11-NEXT: s_lshl_b32 s1, s1, 1680; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)81; GFX11-NEXT: v_sub_f32_e64 v0, s0, s182; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x083; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 184; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v085; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v086; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)87; GFX11-NEXT: v_add_nc_u32_e32 v1, v1, v088; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v189; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)90; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_cndmask_b32 v0, v1, v291; GFX11-NEXT: s_waitcnt lgkmcnt(0)92; GFX11-NEXT: global_store_d16_hi_b16 v3, v0, s[0:1]93; GFX11-NEXT: s_endpgm94 %fabs = call bfloat @llvm.fabs.bf16(bfloat %x)95 %fsub = fsub bfloat -0.0, %fabs96 %fadd = fadd bfloat %y, %fsub97 store bfloat %fadd, ptr addrspace(1) %out, align 298 ret void99}100 101define amdgpu_kernel void @fneg_fabs_fmul_bf16(ptr addrspace(1) %out, bfloat %x, bfloat %y) {102; CI-LABEL: fneg_fabs_fmul_bf16:103; CI: ; %bb.0:104; CI-NEXT: s_load_dword s2, s[8:9], 0x2105; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0106; CI-NEXT: s_add_i32 s12, s12, s17107; CI-NEXT: s_mov_b32 flat_scratch_lo, s13108; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8109; CI-NEXT: s_waitcnt lgkmcnt(0)110; CI-NEXT: s_lshl_b32 s3, s2, 16111; CI-NEXT: s_and_b32 s2, s2, 0xffff0000112; CI-NEXT: v_mov_b32_e32 v0, s3113; CI-NEXT: v_mul_f32_e64 v0, s2, -|v0|114; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v0115; CI-NEXT: v_mov_b32_e32 v0, s0116; CI-NEXT: v_mov_b32_e32 v1, s1117; CI-NEXT: flat_store_short v[0:1], v2118; CI-NEXT: s_endpgm119;120; VI-LABEL: fneg_fabs_fmul_bf16:121; VI: ; %bb.0:122; VI-NEXT: s_load_dword s2, s[8:9], 0x8123; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0124; VI-NEXT: s_add_i32 s12, s12, s17125; VI-NEXT: s_mov_b32 flat_scratch_lo, s13126; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8127; VI-NEXT: s_waitcnt lgkmcnt(0)128; VI-NEXT: s_or_b32 s3, s2, 0x8000129; VI-NEXT: s_lshl_b32 s3, s3, 16130; VI-NEXT: s_and_b32 s2, s2, 0xffff0000131; VI-NEXT: v_mov_b32_e32 v0, s3132; VI-NEXT: v_mul_f32_e32 v0, s2, v0133; VI-NEXT: v_bfe_u32 v1, v0, 16, 1134; VI-NEXT: v_add_u32_e32 v1, vcc, v1, v0135; VI-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1136; VI-NEXT: v_or_b32_e32 v2, 0x400000, v0137; VI-NEXT: v_cmp_u_f32_e32 vcc, v0, v0138; VI-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc139; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v0140; VI-NEXT: v_mov_b32_e32 v0, s0141; VI-NEXT: v_mov_b32_e32 v1, s1142; VI-NEXT: flat_store_short v[0:1], v2143; VI-NEXT: s_endpgm144;145; GFX9-LABEL: fneg_fabs_fmul_bf16:146; GFX9: ; %bb.0:147; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8148; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0149; GFX9-NEXT: v_mov_b32_e32 v0, 0150; GFX9-NEXT: s_waitcnt lgkmcnt(0)151; GFX9-NEXT: s_or_b32 s3, s2, 0x8000152; GFX9-NEXT: s_lshl_b32 s3, s3, 16153; GFX9-NEXT: s_and_b32 s2, s2, 0xffff0000154; GFX9-NEXT: v_mov_b32_e32 v1, s3155; GFX9-NEXT: v_mul_f32_e32 v1, s2, v1156; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1157; GFX9-NEXT: v_add_u32_e32 v2, v2, v1158; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1159; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2160; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1161; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc162; GFX9-NEXT: global_store_short_d16_hi v0, v1, s[0:1]163; GFX9-NEXT: s_endpgm164;165; GFX11-LABEL: fneg_fabs_fmul_bf16:166; GFX11: ; %bb.0:167; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8168; GFX11-NEXT: s_waitcnt lgkmcnt(0)169; GFX11-NEXT: s_or_b32 s1, s0, 0x8000170; GFX11-NEXT: s_and_b32 s0, s0, 0xffff0000171; GFX11-NEXT: s_lshl_b32 s1, s1, 16172; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)173; GFX11-NEXT: v_mul_f32_e64 v0, s0, s1174; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0175; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1176; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0177; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0178; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)179; GFX11-NEXT: v_add_nc_u32_e32 v1, v1, v0180; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1181; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)182; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_cndmask_b32 v0, v1, v2183; GFX11-NEXT: s_waitcnt lgkmcnt(0)184; GFX11-NEXT: global_store_d16_hi_b16 v3, v0, s[0:1]185; GFX11-NEXT: s_endpgm186 %fabs = call bfloat @llvm.fabs.bf16(bfloat %x)187 %fsub = fsub bfloat -0.0, %fabs188 %fmul = fmul bfloat %y, %fsub189 store bfloat %fmul, ptr addrspace(1) %out, align 2190 ret void191}192 193; DAGCombiner will transform:194; (fabs (bf16 bitcast (i16 a))) => (bf16 bitcast (and (i16 a), 0x7FFFFFFF))195; unless isFabsFree returns true196define amdgpu_kernel void @fneg_fabs_free_bf16(ptr addrspace(1) %out, i16 %in) {197; CI-LABEL: fneg_fabs_free_bf16:198; CI: ; %bb.0:199; CI-NEXT: s_load_dword s2, s[8:9], 0x2200; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0201; CI-NEXT: s_add_i32 s12, s12, s17202; CI-NEXT: s_mov_b32 flat_scratch_lo, s13203; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8204; CI-NEXT: s_waitcnt lgkmcnt(0)205; CI-NEXT: s_bitset1_b32 s2, 15206; CI-NEXT: v_mov_b32_e32 v0, s0207; CI-NEXT: v_mov_b32_e32 v1, s1208; CI-NEXT: v_mov_b32_e32 v2, s2209; CI-NEXT: flat_store_short v[0:1], v2210; CI-NEXT: s_endpgm211;212; VI-LABEL: fneg_fabs_free_bf16:213; VI: ; %bb.0:214; VI-NEXT: s_load_dword s2, s[8:9], 0x8215; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0216; VI-NEXT: s_add_i32 s12, s12, s17217; VI-NEXT: s_mov_b32 flat_scratch_lo, s13218; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8219; VI-NEXT: s_waitcnt lgkmcnt(0)220; VI-NEXT: s_bitset1_b32 s2, 15221; VI-NEXT: v_mov_b32_e32 v0, s0222; VI-NEXT: v_mov_b32_e32 v1, s1223; VI-NEXT: v_mov_b32_e32 v2, s2224; VI-NEXT: flat_store_short v[0:1], v2225; VI-NEXT: s_endpgm226;227; GFX9-LABEL: fneg_fabs_free_bf16:228; GFX9: ; %bb.0:229; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8230; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0231; GFX9-NEXT: v_mov_b32_e32 v0, 0232; GFX9-NEXT: s_waitcnt lgkmcnt(0)233; GFX9-NEXT: s_bitset1_b32 s2, 15234; GFX9-NEXT: v_mov_b32_e32 v1, s2235; GFX9-NEXT: global_store_short v0, v1, s[0:1]236; GFX9-NEXT: s_endpgm237;238; GFX11-TRUE16-LABEL: fneg_fabs_free_bf16:239; GFX11-TRUE16: ; %bb.0:240; GFX11-TRUE16-NEXT: s_clause 0x1241; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8242; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0243; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0244; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)245; GFX11-TRUE16-NEXT: s_bitset1_b32 s2, 15246; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)247; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2248; GFX11-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]249; GFX11-TRUE16-NEXT: s_endpgm250;251; GFX11-FAKE16-LABEL: fneg_fabs_free_bf16:252; GFX11-FAKE16: ; %bb.0:253; GFX11-FAKE16-NEXT: s_clause 0x1254; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8255; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0256; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)257; GFX11-FAKE16-NEXT: s_bitset1_b32 s2, 15258; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)259; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2260; GFX11-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]261; GFX11-FAKE16-NEXT: s_endpgm262 %bc = bitcast i16 %in to bfloat263 %fabs = call bfloat @llvm.fabs.bf16(bfloat %bc)264 %fsub = fsub bfloat -0.0, %fabs265 store bfloat %fsub, ptr addrspace(1) %out266 ret void267}268 269define amdgpu_kernel void @fneg_fabs_bf16(ptr addrspace(1) %out, bfloat %in) {270; CI-LABEL: fneg_fabs_bf16:271; CI: ; %bb.0:272; CI-NEXT: s_load_dword s2, s[8:9], 0x2273; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0274; CI-NEXT: s_add_i32 s12, s12, s17275; CI-NEXT: s_mov_b32 flat_scratch_lo, s13276; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8277; CI-NEXT: s_waitcnt lgkmcnt(0)278; CI-NEXT: s_bitset1_b32 s2, 15279; CI-NEXT: v_mov_b32_e32 v0, s0280; CI-NEXT: v_mov_b32_e32 v1, s1281; CI-NEXT: v_mov_b32_e32 v2, s2282; CI-NEXT: flat_store_short v[0:1], v2283; CI-NEXT: s_endpgm284;285; VI-LABEL: fneg_fabs_bf16:286; VI: ; %bb.0:287; VI-NEXT: s_load_dword s2, s[8:9], 0x8288; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0289; VI-NEXT: s_add_i32 s12, s12, s17290; VI-NEXT: s_mov_b32 flat_scratch_lo, s13291; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8292; VI-NEXT: s_waitcnt lgkmcnt(0)293; VI-NEXT: s_bitset1_b32 s2, 15294; VI-NEXT: v_mov_b32_e32 v0, s0295; VI-NEXT: v_mov_b32_e32 v1, s1296; VI-NEXT: v_mov_b32_e32 v2, s2297; VI-NEXT: flat_store_short v[0:1], v2298; VI-NEXT: s_endpgm299;300; GFX9-LABEL: fneg_fabs_bf16:301; GFX9: ; %bb.0:302; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8303; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0304; GFX9-NEXT: v_mov_b32_e32 v0, 0305; GFX9-NEXT: s_waitcnt lgkmcnt(0)306; GFX9-NEXT: s_bitset1_b32 s2, 15307; GFX9-NEXT: v_mov_b32_e32 v1, s2308; GFX9-NEXT: global_store_short v0, v1, s[0:1]309; GFX9-NEXT: s_endpgm310;311; GFX11-TRUE16-LABEL: fneg_fabs_bf16:312; GFX11-TRUE16: ; %bb.0:313; GFX11-TRUE16-NEXT: s_clause 0x1314; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8315; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0316; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0317; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)318; GFX11-TRUE16-NEXT: s_bitset1_b32 s2, 15319; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)320; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2321; GFX11-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]322; GFX11-TRUE16-NEXT: s_endpgm323;324; GFX11-FAKE16-LABEL: fneg_fabs_bf16:325; GFX11-FAKE16: ; %bb.0:326; GFX11-FAKE16-NEXT: s_clause 0x1327; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8328; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0329; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)330; GFX11-FAKE16-NEXT: s_bitset1_b32 s2, 15331; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)332; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2333; GFX11-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]334; GFX11-FAKE16-NEXT: s_endpgm335 %fabs = call bfloat @llvm.fabs.bf16(bfloat %in)336 %fsub = fsub bfloat -0.0, %fabs337 store bfloat %fsub, ptr addrspace(1) %out, align 2338 ret void339}340 341define amdgpu_kernel void @v_fneg_fabs_bf16(ptr addrspace(1) %out, ptr addrspace(1) %in) {342; CIVI-LABEL: v_fneg_fabs_bf16:343; CIVI: ; %bb.0:344; CIVI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0345; CIVI-NEXT: s_add_i32 s12, s12, s17346; CIVI-NEXT: s_mov_b32 flat_scratch_lo, s13347; CIVI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8348; CIVI-NEXT: s_waitcnt lgkmcnt(0)349; CIVI-NEXT: v_mov_b32_e32 v0, s2350; CIVI-NEXT: v_mov_b32_e32 v1, s3351; CIVI-NEXT: flat_load_ushort v2, v[0:1]352; CIVI-NEXT: v_mov_b32_e32 v0, s0353; CIVI-NEXT: v_mov_b32_e32 v1, s1354; CIVI-NEXT: s_waitcnt vmcnt(0)355; CIVI-NEXT: v_or_b32_e32 v2, 0x8000, v2356; CIVI-NEXT: flat_store_short v[0:1], v2357; CIVI-NEXT: s_endpgm358;359; GFX9-LABEL: v_fneg_fabs_bf16:360; GFX9: ; %bb.0:361; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0362; GFX9-NEXT: v_mov_b32_e32 v0, 0363; GFX9-NEXT: s_waitcnt lgkmcnt(0)364; GFX9-NEXT: global_load_ushort v1, v0, s[2:3]365; GFX9-NEXT: s_waitcnt vmcnt(0)366; GFX9-NEXT: v_or_b32_e32 v1, 0x8000, v1367; GFX9-NEXT: global_store_short v0, v1, s[0:1]368; GFX9-NEXT: s_endpgm369;370; GFX11-TRUE16-LABEL: v_fneg_fabs_bf16:371; GFX11-TRUE16: ; %bb.0:372; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0373; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0374; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)375; GFX11-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3]376; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)377; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, 0x8000, v0378; GFX11-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]379; GFX11-TRUE16-NEXT: s_endpgm380;381; GFX11-FAKE16-LABEL: v_fneg_fabs_bf16:382; GFX11-FAKE16: ; %bb.0:383; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0384; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0385; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)386; GFX11-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]387; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)388; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, 0x8000, v1389; GFX11-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]390; GFX11-FAKE16-NEXT: s_endpgm391 %val = load bfloat, ptr addrspace(1) %in, align 2392 %fabs = call bfloat @llvm.fabs.bf16(bfloat %val)393 %fsub = fsub bfloat -0.0, %fabs394 store bfloat %fsub, ptr addrspace(1) %out, align 2395 ret void396}397 398define amdgpu_kernel void @s_fneg_fabs_v2bf16_non_bc_src(ptr addrspace(1) %out, <2 x bfloat> %in) {399; CI-LABEL: s_fneg_fabs_v2bf16_non_bc_src:400; CI: ; %bb.0:401; CI-NEXT: s_load_dword s2, s[8:9], 0x2402; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0403; CI-NEXT: s_add_i32 s12, s12, s17404; CI-NEXT: s_mov_b32 flat_scratch_lo, s13405; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8406; CI-NEXT: s_waitcnt lgkmcnt(0)407; CI-NEXT: s_lshl_b32 s3, s2, 16408; CI-NEXT: s_and_b32 s2, s2, 0xffff0000409; CI-NEXT: v_add_f32_e64 v1, s2, 2.0410; CI-NEXT: v_add_f32_e64 v0, s3, 1.0411; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1412; CI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16413; CI-NEXT: v_or_b32_e32 v2, 0x80008000, v0414; CI-NEXT: v_mov_b32_e32 v0, s0415; CI-NEXT: v_mov_b32_e32 v1, s1416; CI-NEXT: flat_store_dword v[0:1], v2417; CI-NEXT: s_endpgm418;419; VI-LABEL: s_fneg_fabs_v2bf16_non_bc_src:420; VI: ; %bb.0:421; VI-NEXT: s_load_dword s2, s[8:9], 0x8422; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0423; VI-NEXT: s_add_i32 s12, s12, s17424; VI-NEXT: s_mov_b32 flat_scratch_lo, s13425; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8426; VI-NEXT: s_waitcnt lgkmcnt(0)427; VI-NEXT: s_lshl_b32 s3, s2, 16428; VI-NEXT: v_add_f32_e64 v0, s3, 1.0429; VI-NEXT: v_bfe_u32 v1, v0, 16, 1430; VI-NEXT: v_add_u32_e32 v1, vcc, v1, v0431; VI-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1432; VI-NEXT: v_or_b32_e32 v2, 0x400000, v0433; VI-NEXT: v_cmp_u_f32_e32 vcc, v0, v0434; VI-NEXT: s_and_b32 s2, s2, 0xffff0000435; VI-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc436; VI-NEXT: v_add_f32_e64 v1, s2, 2.0437; VI-NEXT: v_bfe_u32 v2, v1, 16, 1438; VI-NEXT: v_add_u32_e32 v2, vcc, v2, v1439; VI-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2440; VI-NEXT: v_or_b32_e32 v3, 0x400000, v1441; VI-NEXT: v_cmp_u_f32_e32 vcc, v1, v1442; VI-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc443; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v1444; VI-NEXT: v_lshrrev_b64 v[0:1], 16, v[0:1]445; VI-NEXT: v_or_b32_e32 v2, 0x80008000, v0446; VI-NEXT: v_mov_b32_e32 v0, s0447; VI-NEXT: v_mov_b32_e32 v1, s1448; VI-NEXT: flat_store_dword v[0:1], v2449; VI-NEXT: s_endpgm450;451; GFX9-LABEL: s_fneg_fabs_v2bf16_non_bc_src:452; GFX9: ; %bb.0:453; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8454; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0455; GFX9-NEXT: v_mov_b32_e32 v0, 0456; GFX9-NEXT: s_waitcnt lgkmcnt(0)457; GFX9-NEXT: s_and_b32 s3, s2, 0xffff0000458; GFX9-NEXT: v_add_f32_e64 v1, s3, 2.0459; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1460; GFX9-NEXT: v_add_u32_e32 v2, v2, v1461; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1462; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2463; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1464; GFX9-NEXT: s_lshl_b32 s2, s2, 16465; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc466; GFX9-NEXT: v_add_f32_e64 v2, s2, 1.0467; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1468; GFX9-NEXT: v_add_u32_e32 v3, v3, v2469; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3470; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2471; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2472; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc473; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff474; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1475; GFX9-NEXT: v_and_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1476; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2477; GFX9-NEXT: v_or_b32_e32 v1, 0x80008000, v1478; GFX9-NEXT: global_store_dword v0, v1, s[0:1]479; GFX9-NEXT: s_endpgm480;481; GFX11-TRUE16-LABEL: s_fneg_fabs_v2bf16_non_bc_src:482; GFX11-TRUE16: ; %bb.0:483; GFX11-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x8484; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)485; GFX11-TRUE16-NEXT: s_and_b32 s1, s0, 0xffff0000486; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16487; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, s1, 2.0488; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, s0, 1.0489; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0490; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)491; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1492; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1493; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0494; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0495; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1496; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)497; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v1498; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3499; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v0500; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)501; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2502; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo503; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1504; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)505; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0506; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo507; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1508; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)509; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l510; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0511; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x80008000, v1512; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)513; GFX11-TRUE16-NEXT: global_store_b32 v0, v1, s[0:1]514; GFX11-TRUE16-NEXT: s_endpgm515;516; GFX11-FAKE16-LABEL: s_fneg_fabs_v2bf16_non_bc_src:517; GFX11-FAKE16: ; %bb.0:518; GFX11-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x8519; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)520; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s0, 16521; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xffff0000522; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, s1, 1.0523; GFX11-FAKE16-NEXT: v_add_f32_e64 v1, s0, 2.0524; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0525; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)526; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1527; GFX11-FAKE16-NEXT: v_bfe_u32 v3, v1, 16, 1528; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v0529; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0530; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v1531; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)532; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, v3, v1533; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3534; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, v2, v0535; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)536; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2537; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo538; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1539; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)540; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0541; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v1, v3, v5 :: v_dual_and_b32 v0, 0xffff, v0542; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)543; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1544; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0545; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, 0546; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)547; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, 0x80008000, v0548; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)549; GFX11-FAKE16-NEXT: global_store_b32 v1, v0, s[0:1]550; GFX11-FAKE16-NEXT: s_endpgm551 %add = fadd <2 x bfloat> %in, <bfloat 1.0, bfloat 2.0>552 %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %add)553 %fneg.fabs = fsub <2 x bfloat> <bfloat -0.0, bfloat -0.0>, %fabs554 store <2 x bfloat> %fneg.fabs, ptr addrspace(1) %out555 ret void556}557 558; FIXME: single bit op559; Combine turns this into integer op when bitcast source (from load)560define amdgpu_kernel void @s_fneg_fabs_v2bf16_bc_src(ptr addrspace(1) %out, <2 x bfloat> %in) {561; CI-LABEL: s_fneg_fabs_v2bf16_bc_src:562; CI: ; %bb.0:563; CI-NEXT: s_load_dword s2, s[8:9], 0x2564; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0565; CI-NEXT: s_add_i32 s12, s12, s17566; CI-NEXT: s_mov_b32 flat_scratch_lo, s13567; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8568; CI-NEXT: s_waitcnt lgkmcnt(0)569; CI-NEXT: s_or_b32 s2, s2, 0x80008000570; CI-NEXT: v_mov_b32_e32 v0, s0571; CI-NEXT: v_mov_b32_e32 v1, s1572; CI-NEXT: v_mov_b32_e32 v2, s2573; CI-NEXT: flat_store_dword v[0:1], v2574; CI-NEXT: s_endpgm575;576; VI-LABEL: s_fneg_fabs_v2bf16_bc_src:577; VI: ; %bb.0:578; VI-NEXT: s_load_dword s2, s[8:9], 0x8579; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0580; VI-NEXT: s_add_i32 s12, s12, s17581; VI-NEXT: s_mov_b32 flat_scratch_lo, s13582; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8583; VI-NEXT: s_waitcnt lgkmcnt(0)584; VI-NEXT: s_or_b32 s2, s2, 0x80008000585; VI-NEXT: v_mov_b32_e32 v0, s0586; VI-NEXT: v_mov_b32_e32 v1, s1587; VI-NEXT: v_mov_b32_e32 v2, s2588; VI-NEXT: flat_store_dword v[0:1], v2589; VI-NEXT: s_endpgm590;591; GFX9-LABEL: s_fneg_fabs_v2bf16_bc_src:592; GFX9: ; %bb.0:593; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8594; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0595; GFX9-NEXT: v_mov_b32_e32 v0, 0596; GFX9-NEXT: s_waitcnt lgkmcnt(0)597; GFX9-NEXT: s_or_b32 s2, s2, 0x80008000598; GFX9-NEXT: v_mov_b32_e32 v1, s2599; GFX9-NEXT: global_store_dword v0, v1, s[0:1]600; GFX9-NEXT: s_endpgm601;602; GFX11-LABEL: s_fneg_fabs_v2bf16_bc_src:603; GFX11: ; %bb.0:604; GFX11-NEXT: s_clause 0x1605; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8606; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0607; GFX11-NEXT: s_waitcnt lgkmcnt(0)608; GFX11-NEXT: s_or_b32 s2, s2, 0x80008000609; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)610; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2611; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]612; GFX11-NEXT: s_endpgm613 %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %in)614 %fneg.fabs = fsub <2 x bfloat> <bfloat -0.0, bfloat -0.0>, %fabs615 store <2 x bfloat> %fneg.fabs, ptr addrspace(1) %out616 ret void617}618 619define amdgpu_kernel void @fneg_fabs_v4bf16(ptr addrspace(1) %out, <4 x bfloat> %in) {620; CI-LABEL: fneg_fabs_v4bf16:621; CI: ; %bb.0:622; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0623; CI-NEXT: s_add_i32 s12, s12, s17624; CI-NEXT: s_mov_b32 flat_scratch_lo, s13625; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8626; CI-NEXT: s_waitcnt lgkmcnt(0)627; CI-NEXT: s_or_b32 s3, s3, 0x80008000628; CI-NEXT: s_or_b32 s2, s2, 0x80008000629; CI-NEXT: v_mov_b32_e32 v3, s1630; CI-NEXT: v_mov_b32_e32 v0, s2631; CI-NEXT: v_mov_b32_e32 v1, s3632; CI-NEXT: v_mov_b32_e32 v2, s0633; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]634; CI-NEXT: s_endpgm635;636; VI-LABEL: fneg_fabs_v4bf16:637; VI: ; %bb.0:638; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0639; VI-NEXT: s_add_i32 s12, s12, s17640; VI-NEXT: s_mov_b32 flat_scratch_lo, s13641; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8642; VI-NEXT: s_waitcnt lgkmcnt(0)643; VI-NEXT: s_lshr_b32 s4, s2, 16644; VI-NEXT: s_lshr_b32 s5, s3, 16645; VI-NEXT: s_bitset1_b32 s3, 15646; VI-NEXT: s_bitset1_b32 s2, 15647; VI-NEXT: s_bitset1_b32 s5, 15648; VI-NEXT: s_bitset1_b32 s4, 15649; VI-NEXT: s_and_b32 s3, 0xffff, s3650; VI-NEXT: s_lshl_b32 s5, s5, 16651; VI-NEXT: s_and_b32 s2, 0xffff, s2652; VI-NEXT: s_lshl_b32 s4, s4, 16653; VI-NEXT: s_or_b32 s3, s3, s5654; VI-NEXT: s_or_b32 s2, s2, s4655; VI-NEXT: v_mov_b32_e32 v3, s1656; VI-NEXT: v_mov_b32_e32 v0, s2657; VI-NEXT: v_mov_b32_e32 v1, s3658; VI-NEXT: v_mov_b32_e32 v2, s0659; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]660; VI-NEXT: s_endpgm661;662; GFX9-LABEL: fneg_fabs_v4bf16:663; GFX9: ; %bb.0:664; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0665; GFX9-NEXT: v_mov_b32_e32 v2, 0666; GFX9-NEXT: s_waitcnt lgkmcnt(0)667; GFX9-NEXT: s_lshr_b32 s4, s2, 16668; GFX9-NEXT: s_lshr_b32 s5, s3, 16669; GFX9-NEXT: s_bitset1_b32 s3, 15670; GFX9-NEXT: s_bitset1_b32 s2, 15671; GFX9-NEXT: s_bitset1_b32 s5, 15672; GFX9-NEXT: s_bitset1_b32 s4, 15673; GFX9-NEXT: s_pack_ll_b32_b16 s3, s3, s5674; GFX9-NEXT: s_pack_ll_b32_b16 s2, s2, s4675; GFX9-NEXT: v_mov_b32_e32 v0, s2676; GFX9-NEXT: v_mov_b32_e32 v1, s3677; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]678; GFX9-NEXT: s_endpgm679;680; GFX11-LABEL: fneg_fabs_v4bf16:681; GFX11: ; %bb.0:682; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0683; GFX11-NEXT: s_waitcnt lgkmcnt(0)684; GFX11-NEXT: s_lshr_b32 s4, s2, 16685; GFX11-NEXT: s_lshr_b32 s5, s3, 16686; GFX11-NEXT: s_bitset1_b32 s3, 15687; GFX11-NEXT: s_bitset1_b32 s2, 15688; GFX11-NEXT: s_bitset1_b32 s4, 15689; GFX11-NEXT: s_bitset1_b32 s5, 15690; GFX11-NEXT: s_pack_ll_b32_b16 s2, s2, s4691; GFX11-NEXT: s_pack_ll_b32_b16 s3, s3, s5692; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)693; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3694; GFX11-NEXT: v_mov_b32_e32 v0, s2695; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]696; GFX11-NEXT: s_endpgm697 %fabs = call <4 x bfloat> @llvm.fabs.v4bf16(<4 x bfloat> %in)698 %fsub = fsub <4 x bfloat> <bfloat -0.0, bfloat -0.0, bfloat -0.0, bfloat -0.0>, %fabs699 store <4 x bfloat> %fsub, ptr addrspace(1) %out700 ret void701}702 703define amdgpu_kernel void @fold_user_fneg_fabs_v2bf16(ptr addrspace(1) %out, <2 x bfloat> %in) #0 {704; CI-LABEL: fold_user_fneg_fabs_v2bf16:705; CI: ; %bb.0:706; CI-NEXT: s_load_dword s2, s[8:9], 0x2707; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0708; CI-NEXT: s_add_i32 s12, s12, s17709; CI-NEXT: s_mov_b32 flat_scratch_lo, s13710; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8711; CI-NEXT: s_waitcnt lgkmcnt(0)712; CI-NEXT: s_and_b32 s3, s2, 0x7fff713; CI-NEXT: s_and_b32 s2, s2, 0x7fff0000714; CI-NEXT: v_mul_f32_e64 v0, s2, -4.0715; CI-NEXT: s_lshl_b32 s2, s3, 16716; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v0717; CI-NEXT: v_mul_f32_e64 v0, s2, -4.0718; CI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16719; CI-NEXT: v_mov_b32_e32 v2, s1720; CI-NEXT: v_mov_b32_e32 v1, s0721; CI-NEXT: flat_store_dword v[1:2], v0722; CI-NEXT: s_endpgm723;724; VI-LABEL: fold_user_fneg_fabs_v2bf16:725; VI: ; %bb.0:726; VI-NEXT: s_load_dword s2, s[8:9], 0x8727; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0728; VI-NEXT: s_add_i32 s12, s12, s17729; VI-NEXT: s_mov_b32 flat_scratch_lo, s13730; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8731; VI-NEXT: s_waitcnt lgkmcnt(0)732; VI-NEXT: s_and_b32 s3, s2, 0x7fff733; VI-NEXT: s_lshl_b32 s3, s3, 16734; VI-NEXT: v_mul_f32_e64 v0, s3, -4.0735; VI-NEXT: v_bfe_u32 v1, v0, 16, 1736; VI-NEXT: v_add_u32_e32 v1, vcc, v1, v0737; VI-NEXT: s_lshr_b32 s2, s2, 16738; VI-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1739; VI-NEXT: s_and_b32 s2, s2, 0x7fff740; VI-NEXT: v_or_b32_e32 v2, 0x400000, v0741; VI-NEXT: v_cmp_u_f32_e32 vcc, v0, v0742; VI-NEXT: s_lshl_b32 s2, s2, 16743; VI-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc744; VI-NEXT: v_mul_f32_e64 v1, s2, -4.0745; VI-NEXT: v_bfe_u32 v2, v1, 16, 1746; VI-NEXT: v_add_u32_e32 v2, vcc, v2, v1747; VI-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2748; VI-NEXT: v_or_b32_e32 v3, 0x400000, v1749; VI-NEXT: v_cmp_u_f32_e32 vcc, v1, v1750; VI-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc751; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v1752; VI-NEXT: v_lshrrev_b64 v[0:1], 16, v[0:1]753; VI-NEXT: v_mov_b32_e32 v2, s1754; VI-NEXT: v_mov_b32_e32 v1, s0755; VI-NEXT: flat_store_dword v[1:2], v0756; VI-NEXT: s_endpgm757;758; GFX9-LABEL: fold_user_fneg_fabs_v2bf16:759; GFX9: ; %bb.0:760; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8761; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0762; GFX9-NEXT: v_mov_b32_e32 v0, 0763; GFX9-NEXT: s_waitcnt lgkmcnt(0)764; GFX9-NEXT: s_and_b32 s3, s2, 0x7fff765; GFX9-NEXT: s_lshl_b32 s3, s3, 16766; GFX9-NEXT: v_mul_f32_e64 v1, s3, -4.0767; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1768; GFX9-NEXT: s_lshr_b32 s2, s2, 16769; GFX9-NEXT: v_add_u32_e32 v2, v2, v1770; GFX9-NEXT: s_and_b32 s2, s2, 0x7fff771; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1772; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2773; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1774; GFX9-NEXT: s_lshl_b32 s2, s2, 16775; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc776; GFX9-NEXT: v_mul_f32_e64 v2, s2, -4.0777; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1778; GFX9-NEXT: v_add_u32_e32 v3, v3, v2779; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v3780; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2781; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2782; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc783; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff784; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2785; GFX9-NEXT: v_and_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1786; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1787; GFX9-NEXT: global_store_dword v0, v1, s[0:1]788; GFX9-NEXT: s_endpgm789;790; GFX11-TRUE16-LABEL: fold_user_fneg_fabs_v2bf16:791; GFX11-TRUE16: ; %bb.0:792; GFX11-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x8793; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)794; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16795; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0x7fff796; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0x7fff797; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16798; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16799; GFX11-TRUE16-NEXT: v_mul_f32_e64 v1, s0, -4.0800; GFX11-TRUE16-NEXT: v_mul_f32_e64 v0, s1, -4.0801; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0802; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)803; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1804; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1805; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0806; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0807; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1808; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v1809; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)810; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3811; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v0812; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2813; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)814; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo815; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1816; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0817; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0818; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo819; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)820; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1821; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l822; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)823; GFX11-TRUE16-NEXT: global_store_b32 v2, v1, s[0:1]824; GFX11-TRUE16-NEXT: s_endpgm825;826; GFX11-FAKE16-LABEL: fold_user_fneg_fabs_v2bf16:827; GFX11-FAKE16: ; %bb.0:828; GFX11-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x8829; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)830; GFX11-FAKE16-NEXT: s_and_b32 s1, s0, 0x7fff831; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 16832; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 16833; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0x7fff834; GFX11-FAKE16-NEXT: v_mul_f32_e64 v0, s1, -4.0835; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 16836; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)837; GFX11-FAKE16-NEXT: v_mul_f32_e64 v1, s0, -4.0838; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0839; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1840; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v0841; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0842; GFX11-FAKE16-NEXT: v_bfe_u32 v3, v1, 16, 1843; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v1844; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)845; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, v3, v1846; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3847; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, v2, v0848; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)849; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2850; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo851; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1852; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, 0853; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)854; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0855; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v1, v3, v5 :: v_dual_and_b32 v0, 0xffff, v0856; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)857; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1858; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0859; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)860; GFX11-FAKE16-NEXT: global_store_b32 v2, v0, s[0:1]861; GFX11-FAKE16-NEXT: s_endpgm862 %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %in)863 %fneg.fabs = fsub <2 x bfloat> <bfloat -0.0, bfloat -0.0>, %fabs864 %mul = fmul <2 x bfloat> %fneg.fabs, <bfloat 4.0, bfloat 4.0>865 store <2 x bfloat> %mul, ptr addrspace(1) %out866 ret void867}868 869define amdgpu_kernel void @s_fneg_multi_use_fabs_v2bf16(ptr addrspace(1) %out0, ptr addrspace(1) %out1, <2 x bfloat> %in) {870; CI-LABEL: s_fneg_multi_use_fabs_v2bf16:871; CI: ; %bb.0:872; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0873; CI-NEXT: s_load_dword s4, s[8:9], 0x4874; CI-NEXT: s_add_i32 s12, s12, s17875; CI-NEXT: s_mov_b32 flat_scratch_lo, s13876; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8877; CI-NEXT: s_waitcnt lgkmcnt(0)878; CI-NEXT: v_mov_b32_e32 v0, s0879; CI-NEXT: s_and_b32 s0, s4, 0x7fff7fff880; CI-NEXT: v_mov_b32_e32 v2, s2881; CI-NEXT: s_or_b32 s2, s0, 0x8000882; CI-NEXT: v_mov_b32_e32 v1, s1883; CI-NEXT: s_and_b32 s1, s4, 0x7fff0000884; CI-NEXT: s_and_b32 s2, s2, 0xffff885; CI-NEXT: s_or_b32 s1, s1, s2886; CI-NEXT: s_bitset1_b32 s1, 31887; CI-NEXT: v_mov_b32_e32 v4, s0888; CI-NEXT: v_mov_b32_e32 v3, s3889; CI-NEXT: flat_store_dword v[0:1], v4890; CI-NEXT: v_mov_b32_e32 v0, s1891; CI-NEXT: flat_store_dword v[2:3], v0892; CI-NEXT: s_endpgm893;894; VI-LABEL: s_fneg_multi_use_fabs_v2bf16:895; VI: ; %bb.0:896; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0897; VI-NEXT: s_load_dword s4, s[8:9], 0x10898; VI-NEXT: s_add_i32 s12, s12, s17899; VI-NEXT: s_mov_b32 flat_scratch_lo, s13900; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8901; VI-NEXT: s_waitcnt lgkmcnt(0)902; VI-NEXT: v_mov_b32_e32 v0, s0903; VI-NEXT: s_and_b32 s0, s4, 0x7fff7fff904; VI-NEXT: v_mov_b32_e32 v1, s1905; VI-NEXT: s_or_b32 s1, s4, 0x80008000906; VI-NEXT: v_mov_b32_e32 v4, s0907; VI-NEXT: v_mov_b32_e32 v2, s2908; VI-NEXT: v_mov_b32_e32 v3, s3909; VI-NEXT: flat_store_dword v[0:1], v4910; VI-NEXT: v_mov_b32_e32 v0, s1911; VI-NEXT: flat_store_dword v[2:3], v0912; VI-NEXT: s_endpgm913;914; GFX9-LABEL: s_fneg_multi_use_fabs_v2bf16:915; GFX9: ; %bb.0:916; GFX9-NEXT: s_load_dword s4, s[8:9], 0x10917; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0918; GFX9-NEXT: v_mov_b32_e32 v0, 0919; GFX9-NEXT: s_waitcnt lgkmcnt(0)920; GFX9-NEXT: s_and_b32 s5, s4, 0x7fff7fff921; GFX9-NEXT: s_or_b32 s4, s4, 0x80008000922; GFX9-NEXT: v_mov_b32_e32 v1, s5923; GFX9-NEXT: v_mov_b32_e32 v2, s4924; GFX9-NEXT: global_store_dword v0, v1, s[0:1]925; GFX9-NEXT: global_store_dword v0, v2, s[2:3]926; GFX9-NEXT: s_endpgm927;928; GFX11-LABEL: s_fneg_multi_use_fabs_v2bf16:929; GFX11: ; %bb.0:930; GFX11-NEXT: s_clause 0x1931; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x10932; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0933; GFX11-NEXT: s_waitcnt lgkmcnt(0)934; GFX11-NEXT: s_and_b32 s4, s6, 0x7fff7fff935; GFX11-NEXT: s_or_b32 s5, s6, 0x80008000936; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4937; GFX11-NEXT: v_mov_b32_e32 v2, s5938; GFX11-NEXT: s_clause 0x1939; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]940; GFX11-NEXT: global_store_b32 v0, v2, s[2:3]941; GFX11-NEXT: s_endpgm942 %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %in)943 %fneg = fsub <2 x bfloat> <bfloat -0.0, bfloat -0.0>, %fabs944 store <2 x bfloat> %fabs, ptr addrspace(1) %out0945 store <2 x bfloat> %fneg, ptr addrspace(1) %out1946 ret void947}948 949define amdgpu_kernel void @s_fneg_multi_use_fabs_foldable_neg_v2bf16(ptr addrspace(1) %out0, ptr addrspace(1) %out1, <2 x bfloat> %in) {950; CI-LABEL: s_fneg_multi_use_fabs_foldable_neg_v2bf16:951; CI: ; %bb.0:952; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0953; CI-NEXT: s_load_dword s4, s[8:9], 0x4954; CI-NEXT: s_add_i32 s12, s12, s17955; CI-NEXT: s_mov_b32 flat_scratch_lo, s13956; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8957; CI-NEXT: s_waitcnt lgkmcnt(0)958; CI-NEXT: v_mov_b32_e32 v1, s1959; CI-NEXT: v_mov_b32_e32 v2, s2960; CI-NEXT: s_and_b32 s1, s4, 0x7fff961; CI-NEXT: s_and_b32 s2, s4, 0x7fff0000962; CI-NEXT: v_mul_f32_e64 v4, s2, -4.0963; CI-NEXT: s_lshl_b32 s1, s1, 16964; CI-NEXT: v_lshrrev_b32_e32 v5, 16, v4965; CI-NEXT: v_mul_f32_e64 v4, s1, -4.0966; CI-NEXT: v_mov_b32_e32 v0, s0967; CI-NEXT: s_and_b32 s0, s4, 0x7fff7fff968; CI-NEXT: v_lshr_b64 v[4:5], v[4:5], 16969; CI-NEXT: v_mov_b32_e32 v5, s0970; CI-NEXT: v_mov_b32_e32 v3, s3971; CI-NEXT: flat_store_dword v[0:1], v5972; CI-NEXT: flat_store_dword v[2:3], v4973; CI-NEXT: s_endpgm974;975; VI-LABEL: s_fneg_multi_use_fabs_foldable_neg_v2bf16:976; VI: ; %bb.0:977; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0978; VI-NEXT: s_load_dword s4, s[8:9], 0x10979; VI-NEXT: s_add_i32 s12, s12, s17980; VI-NEXT: s_mov_b32 flat_scratch_lo, s13981; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8982; VI-NEXT: s_waitcnt lgkmcnt(0)983; VI-NEXT: v_mov_b32_e32 v1, s1984; VI-NEXT: s_and_b32 s1, s4, 0x7fff985; VI-NEXT: s_lshl_b32 s1, s1, 16986; VI-NEXT: v_mul_f32_e64 v4, s1, -4.0987; VI-NEXT: v_bfe_u32 v5, v4, 16, 1988; VI-NEXT: v_add_u32_e32 v5, vcc, v5, v4989; VI-NEXT: s_lshr_b32 s1, s4, 16990; VI-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5991; VI-NEXT: s_and_b32 s1, s1, 0x7fff992; VI-NEXT: v_or_b32_e32 v6, 0x400000, v4993; VI-NEXT: v_cmp_u_f32_e32 vcc, v4, v4994; VI-NEXT: s_lshl_b32 s1, s1, 16995; VI-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc996; VI-NEXT: v_mul_f32_e64 v5, s1, -4.0997; VI-NEXT: v_bfe_u32 v6, v5, 16, 1998; VI-NEXT: v_add_u32_e32 v6, vcc, v6, v5999; VI-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v61000; VI-NEXT: v_or_b32_e32 v7, 0x400000, v51001; VI-NEXT: v_cmp_u_f32_e32 vcc, v5, v51002; VI-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc1003; VI-NEXT: v_lshrrev_b32_e32 v5, 16, v51004; VI-NEXT: v_mov_b32_e32 v0, s01005; VI-NEXT: s_and_b32 s0, s4, 0x7fff7fff1006; VI-NEXT: v_lshrrev_b64 v[4:5], 16, v[4:5]1007; VI-NEXT: v_mov_b32_e32 v5, s01008; VI-NEXT: v_mov_b32_e32 v2, s21009; VI-NEXT: v_mov_b32_e32 v3, s31010; VI-NEXT: flat_store_dword v[0:1], v51011; VI-NEXT: flat_store_dword v[2:3], v41012; VI-NEXT: s_endpgm1013;1014; GFX9-LABEL: s_fneg_multi_use_fabs_foldable_neg_v2bf16:1015; GFX9: ; %bb.0:1016; GFX9-NEXT: s_load_dword s4, s[8:9], 0x101017; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x01018; GFX9-NEXT: v_mov_b32_e32 v0, 01019; GFX9-NEXT: s_waitcnt lgkmcnt(0)1020; GFX9-NEXT: s_and_b32 s6, s4, 0x7fff1021; GFX9-NEXT: s_lshl_b32 s6, s6, 161022; GFX9-NEXT: v_mul_f32_e64 v1, s6, -4.01023; GFX9-NEXT: s_and_b32 s5, s4, 0x7fff7fff1024; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 11025; GFX9-NEXT: s_lshr_b32 s4, s4, 161026; GFX9-NEXT: v_add_u32_e32 v2, v2, v11027; GFX9-NEXT: s_and_b32 s4, s4, 0x7fff1028; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v11029; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v21030; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v11031; GFX9-NEXT: s_lshl_b32 s4, s4, 161032; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc1033; GFX9-NEXT: v_mul_f32_e64 v2, s4, -4.01034; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 11035; GFX9-NEXT: v_add_u32_e32 v3, v3, v21036; GFX9-NEXT: v_add_u32_e32 v3, 0x7fff, v31037; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v21038; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v21039; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc1040; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff1041; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v21042; GFX9-NEXT: v_and_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_11043; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v11044; GFX9-NEXT: v_mov_b32_e32 v2, s51045; GFX9-NEXT: global_store_dword v0, v2, s[0:1]1046; GFX9-NEXT: global_store_dword v0, v1, s[2:3]1047; GFX9-NEXT: s_endpgm1048;1049; GFX11-TRUE16-LABEL: s_fneg_multi_use_fabs_foldable_neg_v2bf16:1050; GFX11-TRUE16: ; %bb.0:1051; GFX11-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x101052; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)1053; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s6, 161054; GFX11-TRUE16-NEXT: s_and_b32 s1, s6, 0x7fff1055; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0x7fff1056; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)1057; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 161058; GFX11-TRUE16-NEXT: v_mul_f32_e64 v0, s0, -4.01059; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s1, 161060; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)1061; GFX11-TRUE16-NEXT: v_mul_f32_e64 v1, s0, -4.01062; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x01063; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 11064; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v01065; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v01066; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 11067; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v11068; GFX11-TRUE16-NEXT: s_and_b32 s4, s6, 0x7fff7fff1069; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)1070; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v11071; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v31072; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v01073; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1074; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v21075; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo1076; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v11077; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 01078; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)1079; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v01080; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo1081; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s41082; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v11083; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)1084; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l1085; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)1086; GFX11-TRUE16-NEXT: s_clause 0x11087; GFX11-TRUE16-NEXT: global_store_b32 v2, v3, s[0:1]1088; GFX11-TRUE16-NEXT: global_store_b32 v2, v1, s[2:3]1089; GFX11-TRUE16-NEXT: s_endpgm1090;1091; GFX11-FAKE16-LABEL: s_fneg_multi_use_fabs_foldable_neg_v2bf16:1092; GFX11-FAKE16: ; %bb.0:1093; GFX11-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x101094; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)1095; GFX11-FAKE16-NEXT: s_and_b32 s0, s6, 0x7fff1096; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s6, 161097; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 161098; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0x7fff1099; GFX11-FAKE16-NEXT: v_mul_f32_e64 v0, s0, -4.01100; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s1, 161101; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)1102; GFX11-FAKE16-NEXT: v_mul_f32_e64 v1, s0, -4.01103; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x01104; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 11105; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v01106; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v01107; GFX11-FAKE16-NEXT: v_bfe_u32 v3, v1, 16, 11108; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v11109; GFX11-FAKE16-NEXT: s_and_b32 s4, s6, 0x7fff7fff1110; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)1111; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, v3, v11112; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v31113; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, v2, v01114; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1115; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v21116; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo1117; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v11118; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, 01119; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)1120; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v01121; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo1122; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_and_b32 v0, 0xffff, v01123; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)1124; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v11125; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v01126; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)1127; GFX11-FAKE16-NEXT: s_clause 0x11128; GFX11-FAKE16-NEXT: global_store_b32 v2, v3, s[0:1]1129; GFX11-FAKE16-NEXT: global_store_b32 v2, v0, s[2:3]1130; GFX11-FAKE16-NEXT: s_endpgm1131 %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %in)1132 %fneg = fsub <2 x bfloat> <bfloat -0.0, bfloat -0.0>, %fabs1133 %mul = fmul <2 x bfloat> %fneg, <bfloat 4.0, bfloat 4.0>1134 store <2 x bfloat> %fabs, ptr addrspace(1) %out01135 store <2 x bfloat> %mul, ptr addrspace(1) %out11136 ret void1137}1138 1139declare bfloat @llvm.fabs.bf16(bfloat) #11140declare <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat>) #11141declare <4 x bfloat> @llvm.fabs.v4bf16(<4 x bfloat>) #11142 1143attributes #0 = { nounwind }1144attributes #1 = { nounwind readnone }1145