1018 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 22; RUN: llc -amdgpu-scalarize-global-loads=false -mcpu=kaveri -mtriple=amdgcn--amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=CIVI,CI %s3; RUN: llc -amdgpu-scalarize-global-loads=false -mcpu=tonga -mtriple=amdgcn--amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=CIVI,GFX8 %s4; RUN: llc -amdgpu-scalarize-global-loads=false -mcpu=gfx900 -mtriple=amdgcn--amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s5; RUN: llc -amdgpu-scalarize-global-loads=false -mcpu=gfx1100 -mattr=+real-true16 -mtriple=amdgcn--amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-TRUE16 %s6; RUN: llc -amdgpu-scalarize-global-loads=false -mcpu=gfx1100 -mattr=-real-true16 -mtriple=amdgcn--amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-FAKE16 %s7 8define amdgpu_kernel void @s_fneg_bf16(ptr addrspace(1) %out, bfloat %in) #0 {9; CI-LABEL: s_fneg_bf16:10; CI: ; %bb.0:11; CI-NEXT: s_load_dword s2, s[8:9], 0x212; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x013; CI-NEXT: s_add_i32 s12, s12, s1714; CI-NEXT: s_mov_b32 flat_scratch_lo, s1315; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 816; CI-NEXT: s_waitcnt lgkmcnt(0)17; CI-NEXT: s_xor_b32 s2, s2, 0x800018; CI-NEXT: v_mov_b32_e32 v0, s019; CI-NEXT: v_mov_b32_e32 v1, s120; CI-NEXT: v_mov_b32_e32 v2, s221; CI-NEXT: flat_store_short v[0:1], v222; CI-NEXT: s_endpgm23;24; GFX8-LABEL: s_fneg_bf16:25; GFX8: ; %bb.0:26; GFX8-NEXT: s_load_dword s2, s[8:9], 0x827; GFX8-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x028; GFX8-NEXT: s_add_i32 s12, s12, s1729; GFX8-NEXT: s_mov_b32 flat_scratch_lo, s1330; GFX8-NEXT: s_lshr_b32 flat_scratch_hi, s12, 831; GFX8-NEXT: s_waitcnt lgkmcnt(0)32; GFX8-NEXT: s_xor_b32 s2, s2, 0x800033; GFX8-NEXT: v_mov_b32_e32 v0, s034; GFX8-NEXT: v_mov_b32_e32 v1, s135; GFX8-NEXT: v_mov_b32_e32 v2, s236; GFX8-NEXT: flat_store_short v[0:1], v237; GFX8-NEXT: s_endpgm38;39; GFX9-LABEL: s_fneg_bf16:40; GFX9: ; %bb.0:41; GFX9-NEXT: s_load_dword s2, s[8:9], 0x842; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x043; GFX9-NEXT: v_mov_b32_e32 v0, 044; GFX9-NEXT: s_waitcnt lgkmcnt(0)45; GFX9-NEXT: s_xor_b32 s2, s2, 0x800046; GFX9-NEXT: v_mov_b32_e32 v1, s247; GFX9-NEXT: global_store_short v0, v1, s[0:1]48; GFX9-NEXT: s_endpgm49;50; GFX11-TRUE16-LABEL: s_fneg_bf16:51; GFX11-TRUE16: ; %bb.0:52; GFX11-TRUE16-NEXT: s_clause 0x153; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x854; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x055; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 056; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)57; GFX11-TRUE16-NEXT: s_xor_b32 s2, s2, 0x800058; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)59; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s260; GFX11-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]61; GFX11-TRUE16-NEXT: s_endpgm62;63; GFX11-FAKE16-LABEL: s_fneg_bf16:64; GFX11-FAKE16: ; %bb.0:65; GFX11-FAKE16-NEXT: s_clause 0x166; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x867; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x068; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)69; GFX11-FAKE16-NEXT: s_xor_b32 s2, s2, 0x800070; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)71; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s272; GFX11-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]73; GFX11-FAKE16-NEXT: s_endpgm74 %fneg = fsub bfloat -0.0, %in75 store bfloat %fneg, ptr addrspace(1) %out76 ret void77}78 79; FIXME: Should be able to use bit operations when illegal type as80; well.81define amdgpu_kernel void @v_fneg_bf16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {82; CI-LABEL: v_fneg_bf16:83; CI: ; %bb.0:84; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x285; CI-NEXT: v_lshlrev_b32_e32 v0, 1, v086; CI-NEXT: s_add_i32 s12, s12, s1787; CI-NEXT: s_mov_b32 flat_scratch_lo, s1388; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 889; CI-NEXT: s_waitcnt lgkmcnt(0)90; CI-NEXT: v_mov_b32_e32 v1, s191; CI-NEXT: v_add_i32_e32 v0, vcc, s0, v092; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc93; CI-NEXT: flat_load_ushort v2, v[0:1]94; CI-NEXT: s_waitcnt vmcnt(0)95; CI-NEXT: v_xor_b32_e32 v2, 0x8000, v296; CI-NEXT: flat_store_short v[0:1], v297; CI-NEXT: s_endpgm98;99; GFX8-LABEL: v_fneg_bf16:100; GFX8: ; %bb.0:101; GFX8-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x8102; GFX8-NEXT: v_lshlrev_b32_e32 v0, 1, v0103; GFX8-NEXT: s_add_i32 s12, s12, s17104; GFX8-NEXT: s_mov_b32 flat_scratch_lo, s13105; GFX8-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8106; GFX8-NEXT: s_waitcnt lgkmcnt(0)107; GFX8-NEXT: v_mov_b32_e32 v1, s1108; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v0109; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc110; GFX8-NEXT: flat_load_ushort v2, v[0:1]111; GFX8-NEXT: s_waitcnt vmcnt(0)112; GFX8-NEXT: v_xor_b32_e32 v2, 0x8000, v2113; GFX8-NEXT: flat_store_short v[0:1], v2114; GFX8-NEXT: s_endpgm115;116; GFX9-LABEL: v_fneg_bf16:117; GFX9: ; %bb.0:118; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x8119; GFX9-NEXT: v_lshlrev_b32_e32 v0, 1, v0120; GFX9-NEXT: s_waitcnt lgkmcnt(0)121; GFX9-NEXT: global_load_ushort v1, v0, s[0:1]122; GFX9-NEXT: s_waitcnt vmcnt(0)123; GFX9-NEXT: v_xor_b32_e32 v1, 0x8000, v1124; GFX9-NEXT: global_store_short v0, v1, s[0:1]125; GFX9-NEXT: s_endpgm126;127; GFX11-TRUE16-LABEL: v_fneg_bf16:128; GFX11-TRUE16: ; %bb.0:129; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x8130; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0131; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)132; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 1, v0133; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)134; GFX11-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[0:1]135; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)136; GFX11-TRUE16-NEXT: v_xor_b16 v0.l, 0x8000, v0.l137; GFX11-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]138; GFX11-TRUE16-NEXT: s_endpgm139;140; GFX11-FAKE16-LABEL: v_fneg_bf16:141; GFX11-FAKE16: ; %bb.0:142; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x8143; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0144; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)145; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 1, v0146; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)147; GFX11-FAKE16-NEXT: global_load_u16 v1, v0, s[0:1]148; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)149; GFX11-FAKE16-NEXT: v_xor_b32_e32 v1, 0x8000, v1150; GFX11-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]151; GFX11-FAKE16-NEXT: s_endpgm152 %tid = call i32 @llvm.amdgcn.workitem.id.x()153 %gep.in = getelementptr inbounds bfloat, ptr addrspace(1) %in, i32 %tid154 %gep.out = getelementptr inbounds bfloat, ptr addrspace(1) %in, i32 %tid155 %val = load bfloat, ptr addrspace(1) %gep.in, align 2156 %fneg = fsub bfloat -0.0, %val157 store bfloat %fneg, ptr addrspace(1) %gep.out158 ret void159}160 161define amdgpu_kernel void @s_fneg_free_bf16(ptr addrspace(1) %out, i16 %in) #0 {162; CI-LABEL: s_fneg_free_bf16:163; CI: ; %bb.0:164; CI-NEXT: s_load_dword s2, s[8:9], 0x2165; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0166; CI-NEXT: s_add_i32 s12, s12, s17167; CI-NEXT: s_mov_b32 flat_scratch_lo, s13168; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8169; CI-NEXT: s_waitcnt lgkmcnt(0)170; CI-NEXT: s_xor_b32 s2, s2, 0x8000171; CI-NEXT: v_mov_b32_e32 v0, s0172; CI-NEXT: v_mov_b32_e32 v1, s1173; CI-NEXT: v_mov_b32_e32 v2, s2174; CI-NEXT: flat_store_short v[0:1], v2175; CI-NEXT: s_endpgm176;177; GFX8-LABEL: s_fneg_free_bf16:178; GFX8: ; %bb.0:179; GFX8-NEXT: s_load_dword s2, s[8:9], 0x8180; GFX8-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0181; GFX8-NEXT: s_add_i32 s12, s12, s17182; GFX8-NEXT: s_mov_b32 flat_scratch_lo, s13183; GFX8-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8184; GFX8-NEXT: s_waitcnt lgkmcnt(0)185; GFX8-NEXT: s_xor_b32 s2, s2, 0x8000186; GFX8-NEXT: v_mov_b32_e32 v0, s0187; GFX8-NEXT: v_mov_b32_e32 v1, s1188; GFX8-NEXT: v_mov_b32_e32 v2, s2189; GFX8-NEXT: flat_store_short v[0:1], v2190; GFX8-NEXT: s_endpgm191;192; GFX9-LABEL: s_fneg_free_bf16:193; GFX9: ; %bb.0:194; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8195; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0196; GFX9-NEXT: v_mov_b32_e32 v0, 0197; GFX9-NEXT: s_waitcnt lgkmcnt(0)198; GFX9-NEXT: s_xor_b32 s2, s2, 0x8000199; GFX9-NEXT: v_mov_b32_e32 v1, s2200; GFX9-NEXT: global_store_short v0, v1, s[0:1]201; GFX9-NEXT: s_endpgm202;203; GFX11-TRUE16-LABEL: s_fneg_free_bf16:204; GFX11-TRUE16: ; %bb.0:205; GFX11-TRUE16-NEXT: s_clause 0x1206; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8207; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0208; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0209; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)210; GFX11-TRUE16-NEXT: s_xor_b32 s2, s2, 0x8000211; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)212; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2213; GFX11-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]214; GFX11-TRUE16-NEXT: s_endpgm215;216; GFX11-FAKE16-LABEL: s_fneg_free_bf16:217; GFX11-FAKE16: ; %bb.0:218; GFX11-FAKE16-NEXT: s_clause 0x1219; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8220; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0221; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)222; GFX11-FAKE16-NEXT: s_xor_b32 s2, s2, 0x8000223; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)224; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2225; GFX11-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]226; GFX11-FAKE16-NEXT: s_endpgm227 %bc = bitcast i16 %in to bfloat228 %fsub = fsub bfloat -0.0, %bc229 store bfloat %fsub, ptr addrspace(1) %out230 ret void231}232 233define amdgpu_kernel void @v_fneg_fold_bf16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {234; CI-LABEL: v_fneg_fold_bf16:235; CI: ; %bb.0:236; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0237; CI-NEXT: s_add_i32 s12, s12, s17238; CI-NEXT: s_mov_b32 flat_scratch_lo, s13239; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8240; CI-NEXT: s_waitcnt lgkmcnt(0)241; CI-NEXT: v_mov_b32_e32 v0, s2242; CI-NEXT: v_mov_b32_e32 v1, s3243; CI-NEXT: flat_load_ushort v2, v[0:1]244; CI-NEXT: v_mov_b32_e32 v0, s0245; CI-NEXT: v_mov_b32_e32 v1, s1246; CI-NEXT: s_waitcnt vmcnt(0)247; CI-NEXT: v_xor_b32_e32 v3, 0x8000, v2248; CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2249; CI-NEXT: v_lshlrev_b32_e32 v3, 16, v3250; CI-NEXT: v_mul_f32_e32 v2, v3, v2251; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v2252; CI-NEXT: flat_store_short v[0:1], v2253; CI-NEXT: s_endpgm254;255; GFX8-LABEL: v_fneg_fold_bf16:256; GFX8: ; %bb.0:257; GFX8-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0258; GFX8-NEXT: s_add_i32 s12, s12, s17259; GFX8-NEXT: s_mov_b32 flat_scratch_lo, s13260; GFX8-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8261; GFX8-NEXT: v_mov_b32_e32 v3, 0x8000262; GFX8-NEXT: s_waitcnt lgkmcnt(0)263; GFX8-NEXT: v_mov_b32_e32 v0, s2264; GFX8-NEXT: v_mov_b32_e32 v1, s3265; GFX8-NEXT: flat_load_ushort v2, v[0:1]266; GFX8-NEXT: v_mov_b32_e32 v0, s0267; GFX8-NEXT: v_mov_b32_e32 v1, s1268; GFX8-NEXT: s_waitcnt vmcnt(0)269; GFX8-NEXT: v_xor_b32_sdwa v3, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD270; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2271; GFX8-NEXT: v_mul_f32_e32 v2, v3, v2272; GFX8-NEXT: v_bfe_u32 v3, v2, 16, 1273; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v2274; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3275; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v2276; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2277; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc278; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2279; GFX8-NEXT: flat_store_short v[0:1], v2280; GFX8-NEXT: s_endpgm281;282; GFX9-LABEL: v_fneg_fold_bf16:283; GFX9: ; %bb.0:284; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0285; GFX9-NEXT: v_mov_b32_e32 v0, 0286; GFX9-NEXT: s_waitcnt lgkmcnt(0)287; GFX9-NEXT: global_load_ushort v1, v0, s[2:3]288; GFX9-NEXT: s_mov_b32 s2, 0x8000289; GFX9-NEXT: s_waitcnt vmcnt(0)290; GFX9-NEXT: v_xor_b32_sdwa v2, v1, s2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD291; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1292; GFX9-NEXT: v_mul_f32_e32 v1, v2, v1293; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1294; GFX9-NEXT: v_add_u32_e32 v2, v2, v1295; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1296; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2297; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1298; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc299; GFX9-NEXT: global_store_short_d16_hi v0, v1, s[0:1]300; GFX9-NEXT: s_endpgm301;302; GFX11-TRUE16-LABEL: v_fneg_fold_bf16:303; GFX11-TRUE16: ; %bb.0:304; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0305; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0306; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)307; GFX11-TRUE16-NEXT: global_load_d16_b16 v1, v0, s[2:3]308; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)309; GFX11-TRUE16-NEXT: v_xor_b32_e32 v2, 0x8000, v1310; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1311; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)312; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2313; GFX11-TRUE16-NEXT: v_mul_f32_e32 v1, v2, v1314; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)315; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1316; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v1317; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)318; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2319; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1320; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1321; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc_lo322; GFX11-TRUE16-NEXT: global_store_d16_hi_b16 v0, v1, s[0:1]323; GFX11-TRUE16-NEXT: s_endpgm324;325; GFX11-FAKE16-LABEL: v_fneg_fold_bf16:326; GFX11-FAKE16: ; %bb.0:327; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0328; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0329; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)330; GFX11-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]331; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)332; GFX11-FAKE16-NEXT: v_xor_b32_e32 v2, 0x8000, v1333; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1334; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)335; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2336; GFX11-FAKE16-NEXT: v_mul_f32_e32 v1, v2, v1337; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)338; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v1, 16, 1339; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, v2, v1340; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)341; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2342; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v1343; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1344; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc_lo345; GFX11-FAKE16-NEXT: global_store_d16_hi_b16 v0, v1, s[0:1]346; GFX11-FAKE16-NEXT: s_endpgm347 %val = load bfloat, ptr addrspace(1) %in348 %fsub = fsub bfloat -0.0, %val349 %fmul = fmul bfloat %fsub, %val350 store bfloat %fmul, ptr addrspace(1) %out351 ret void352}353 354define amdgpu_kernel void @s_fneg_v2bf16(ptr addrspace(1) %out, <2 x bfloat> %in) #0 {355; CI-LABEL: s_fneg_v2bf16:356; CI: ; %bb.0:357; CI-NEXT: s_load_dword s2, s[8:9], 0x2358; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0359; CI-NEXT: s_add_i32 s12, s12, s17360; CI-NEXT: s_mov_b32 flat_scratch_lo, s13361; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8362; CI-NEXT: s_waitcnt lgkmcnt(0)363; CI-NEXT: s_and_b32 s3, s2, 0xffff0000364; CI-NEXT: s_xor_b32 s2, s2, 0x8000365; CI-NEXT: s_and_b32 s2, s2, 0xffff366; CI-NEXT: s_or_b32 s2, s2, s3367; CI-NEXT: s_add_i32 s2, s2, 0x80000000368; CI-NEXT: v_mov_b32_e32 v0, s0369; CI-NEXT: v_mov_b32_e32 v1, s1370; CI-NEXT: v_mov_b32_e32 v2, s2371; CI-NEXT: flat_store_dword v[0:1], v2372; CI-NEXT: s_endpgm373;374; GFX8-LABEL: s_fneg_v2bf16:375; GFX8: ; %bb.0:376; GFX8-NEXT: s_load_dword s2, s[8:9], 0x8377; GFX8-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0378; GFX8-NEXT: s_add_i32 s12, s12, s17379; GFX8-NEXT: s_mov_b32 flat_scratch_lo, s13380; GFX8-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8381; GFX8-NEXT: s_waitcnt lgkmcnt(0)382; GFX8-NEXT: s_xor_b32 s2, s2, 0x80008000383; GFX8-NEXT: v_mov_b32_e32 v0, s0384; GFX8-NEXT: v_mov_b32_e32 v1, s1385; GFX8-NEXT: v_mov_b32_e32 v2, s2386; GFX8-NEXT: flat_store_dword v[0:1], v2387; GFX8-NEXT: s_endpgm388;389; GFX9-LABEL: s_fneg_v2bf16:390; GFX9: ; %bb.0:391; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8392; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0393; GFX9-NEXT: v_mov_b32_e32 v0, 0394; GFX9-NEXT: s_waitcnt lgkmcnt(0)395; GFX9-NEXT: s_xor_b32 s2, s2, 0x80008000396; GFX9-NEXT: v_mov_b32_e32 v1, s2397; GFX9-NEXT: global_store_dword v0, v1, s[0:1]398; GFX9-NEXT: s_endpgm399;400; GFX11-LABEL: s_fneg_v2bf16:401; GFX11: ; %bb.0:402; GFX11-NEXT: s_clause 0x1403; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8404; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0405; GFX11-NEXT: s_waitcnt lgkmcnt(0)406; GFX11-NEXT: s_xor_b32 s2, s2, 0x80008000407; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)408; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2409; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]410; GFX11-NEXT: s_endpgm411 %fneg = fsub <2 x bfloat> <bfloat -0.0, bfloat -0.0>, %in412 store <2 x bfloat> %fneg, ptr addrspace(1) %out413 ret void414}415 416define amdgpu_kernel void @s_fneg_v2bf16_nonload(ptr addrspace(1) %out) #0 {417; CI-LABEL: s_fneg_v2bf16_nonload:418; CI: ; %bb.0:419; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0420; CI-NEXT: s_add_i32 s12, s12, s17421; CI-NEXT: ;;#ASMSTART422; CI-NEXT: ; def s2423; CI-NEXT: ;;#ASMEND424; CI-NEXT: s_and_b32 s3, s2, 0xffff0000425; CI-NEXT: s_xor_b32 s2, s2, 0x8000426; CI-NEXT: s_and_b32 s2, s2, 0xffff427; CI-NEXT: s_or_b32 s2, s2, s3428; CI-NEXT: s_add_i32 s2, s2, 0x80000000429; CI-NEXT: s_waitcnt lgkmcnt(0)430; CI-NEXT: v_mov_b32_e32 v0, s0431; CI-NEXT: s_mov_b32 flat_scratch_lo, s13432; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8433; CI-NEXT: v_mov_b32_e32 v1, s1434; CI-NEXT: v_mov_b32_e32 v2, s2435; CI-NEXT: flat_store_dword v[0:1], v2436; CI-NEXT: s_endpgm437;438; GFX8-LABEL: s_fneg_v2bf16_nonload:439; GFX8: ; %bb.0:440; GFX8-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0441; GFX8-NEXT: s_add_i32 s12, s12, s17442; GFX8-NEXT: ;;#ASMSTART443; GFX8-NEXT: ; def s2444; GFX8-NEXT: ;;#ASMEND445; GFX8-NEXT: s_xor_b32 s2, s2, 0x80008000446; GFX8-NEXT: s_mov_b32 flat_scratch_lo, s13447; GFX8-NEXT: s_waitcnt lgkmcnt(0)448; GFX8-NEXT: v_mov_b32_e32 v0, s0449; GFX8-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8450; GFX8-NEXT: v_mov_b32_e32 v1, s1451; GFX8-NEXT: v_mov_b32_e32 v2, s2452; GFX8-NEXT: flat_store_dword v[0:1], v2453; GFX8-NEXT: s_endpgm454;455; GFX9-LABEL: s_fneg_v2bf16_nonload:456; GFX9: ; %bb.0:457; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0458; GFX9-NEXT: ;;#ASMSTART459; GFX9-NEXT: ; def s2460; GFX9-NEXT: ;;#ASMEND461; GFX9-NEXT: s_xor_b32 s2, s2, 0x80008000462; GFX9-NEXT: v_mov_b32_e32 v0, 0463; GFX9-NEXT: v_mov_b32_e32 v1, s2464; GFX9-NEXT: s_waitcnt lgkmcnt(0)465; GFX9-NEXT: global_store_dword v0, v1, s[0:1]466; GFX9-NEXT: s_endpgm467;468; GFX11-LABEL: s_fneg_v2bf16_nonload:469; GFX11: ; %bb.0:470; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0471; GFX11-NEXT: ;;#ASMSTART472; GFX11-NEXT: ; def s2473; GFX11-NEXT: ;;#ASMEND474; GFX11-NEXT: s_xor_b32 s2, s2, 0x80008000475; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)476; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2477; GFX11-NEXT: s_waitcnt lgkmcnt(0)478; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]479; GFX11-NEXT: s_endpgm480 %in = call i32 asm sideeffect "; def $0", "=s"()481 %in.bc = bitcast i32 %in to <2 x bfloat>482 %fneg = fsub <2 x bfloat> <bfloat -0.0, bfloat -0.0>, %in.bc483 store <2 x bfloat> %fneg, ptr addrspace(1) %out484 ret void485}486 487define amdgpu_kernel void @v_fneg_v2bf16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {488; CI-LABEL: v_fneg_v2bf16:489; CI: ; %bb.0:490; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x2491; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0492; CI-NEXT: s_add_i32 s12, s12, s17493; CI-NEXT: s_mov_b32 flat_scratch_lo, s13494; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8495; CI-NEXT: s_waitcnt lgkmcnt(0)496; CI-NEXT: v_mov_b32_e32 v1, s1497; CI-NEXT: v_add_i32_e32 v0, vcc, s0, v0498; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc499; CI-NEXT: flat_load_dword v2, v[0:1]500; CI-NEXT: s_mov_b32 s0, 0xffff501; CI-NEXT: s_waitcnt vmcnt(0)502; CI-NEXT: v_xor_b32_e32 v3, 0x8000, v2503; CI-NEXT: v_bfi_b32 v2, s0, v3, v2504; CI-NEXT: v_add_i32_e32 v2, vcc, 0x80000000, v2505; CI-NEXT: flat_store_dword v[0:1], v2506; CI-NEXT: s_endpgm507;508; GFX8-LABEL: v_fneg_v2bf16:509; GFX8: ; %bb.0:510; GFX8-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x8511; GFX8-NEXT: v_lshlrev_b32_e32 v0, 2, v0512; GFX8-NEXT: s_add_i32 s12, s12, s17513; GFX8-NEXT: s_mov_b32 flat_scratch_lo, s13514; GFX8-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8515; GFX8-NEXT: s_waitcnt lgkmcnt(0)516; GFX8-NEXT: v_mov_b32_e32 v1, s1517; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v0518; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc519; GFX8-NEXT: flat_load_dword v2, v[0:1]520; GFX8-NEXT: s_waitcnt vmcnt(0)521; GFX8-NEXT: v_xor_b32_e32 v2, 0x80008000, v2522; GFX8-NEXT: flat_store_dword v[0:1], v2523; GFX8-NEXT: s_endpgm524;525; GFX9-LABEL: v_fneg_v2bf16:526; GFX9: ; %bb.0:527; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x8528; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0529; GFX9-NEXT: s_waitcnt lgkmcnt(0)530; GFX9-NEXT: global_load_dword v1, v0, s[0:1]531; GFX9-NEXT: s_waitcnt vmcnt(0)532; GFX9-NEXT: v_xor_b32_e32 v1, 0x80008000, v1533; GFX9-NEXT: global_store_dword v0, v1, s[0:1]534; GFX9-NEXT: s_endpgm535;536; GFX11-LABEL: v_fneg_v2bf16:537; GFX11: ; %bb.0:538; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x8539; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0540; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)541; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0542; GFX11-NEXT: s_waitcnt lgkmcnt(0)543; GFX11-NEXT: global_load_b32 v1, v0, s[0:1]544; GFX11-NEXT: s_waitcnt vmcnt(0)545; GFX11-NEXT: v_xor_b32_e32 v1, 0x80008000, v1546; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]547; GFX11-NEXT: s_endpgm548 %tid = call i32 @llvm.amdgcn.workitem.id.x()549 %gep.in = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %in, i32 %tid550 %gep.out = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %in, i32 %tid551 %val = load <2 x bfloat>, ptr addrspace(1) %gep.in, align 2552 %fneg = fsub <2 x bfloat> <bfloat -0.0, bfloat -0.0>, %val553 store <2 x bfloat> %fneg, ptr addrspace(1) %gep.out554 ret void555}556 557define amdgpu_kernel void @fneg_free_v2bf16(ptr addrspace(1) %out, i32 %in) #0 {558; CI-LABEL: fneg_free_v2bf16:559; CI: ; %bb.0:560; CI-NEXT: s_load_dword s2, s[8:9], 0x2561; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0562; CI-NEXT: s_add_i32 s12, s12, s17563; CI-NEXT: s_mov_b32 flat_scratch_lo, s13564; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8565; CI-NEXT: s_waitcnt lgkmcnt(0)566; CI-NEXT: s_and_b32 s3, s2, 0xffff0000567; CI-NEXT: s_xor_b32 s2, s2, 0x8000568; CI-NEXT: s_and_b32 s2, s2, 0xffff569; CI-NEXT: s_or_b32 s2, s2, s3570; CI-NEXT: s_add_i32 s2, s2, 0x80000000571; CI-NEXT: v_mov_b32_e32 v0, s0572; CI-NEXT: v_mov_b32_e32 v1, s1573; CI-NEXT: v_mov_b32_e32 v2, s2574; CI-NEXT: flat_store_dword v[0:1], v2575; CI-NEXT: s_endpgm576;577; GFX8-LABEL: fneg_free_v2bf16:578; GFX8: ; %bb.0:579; GFX8-NEXT: s_load_dword s2, s[8:9], 0x8580; GFX8-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0581; GFX8-NEXT: s_add_i32 s12, s12, s17582; GFX8-NEXT: s_mov_b32 flat_scratch_lo, s13583; GFX8-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8584; GFX8-NEXT: s_waitcnt lgkmcnt(0)585; GFX8-NEXT: s_xor_b32 s2, s2, 0x80008000586; GFX8-NEXT: v_mov_b32_e32 v0, s0587; GFX8-NEXT: v_mov_b32_e32 v1, s1588; GFX8-NEXT: v_mov_b32_e32 v2, s2589; GFX8-NEXT: flat_store_dword v[0:1], v2590; GFX8-NEXT: s_endpgm591;592; GFX9-LABEL: fneg_free_v2bf16:593; GFX9: ; %bb.0:594; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8595; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0596; GFX9-NEXT: v_mov_b32_e32 v0, 0597; GFX9-NEXT: s_waitcnt lgkmcnt(0)598; GFX9-NEXT: s_xor_b32 s2, s2, 0x80008000599; GFX9-NEXT: v_mov_b32_e32 v1, s2600; GFX9-NEXT: global_store_dword v0, v1, s[0:1]601; GFX9-NEXT: s_endpgm602;603; GFX11-LABEL: fneg_free_v2bf16:604; GFX11: ; %bb.0:605; GFX11-NEXT: s_clause 0x1606; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8607; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0608; GFX11-NEXT: s_waitcnt lgkmcnt(0)609; GFX11-NEXT: s_xor_b32 s2, s2, 0x80008000610; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)611; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2612; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]613; GFX11-NEXT: s_endpgm614 %bc = bitcast i32 %in to <2 x bfloat>615 %fsub = fsub <2 x bfloat> <bfloat -0.0, bfloat -0.0>, %bc616 store <2 x bfloat> %fsub, ptr addrspace(1) %out617 ret void618}619 620define amdgpu_kernel void @v_fneg_fold_v2bf16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {621; CI-LABEL: v_fneg_fold_v2bf16:622; CI: ; %bb.0:623; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0624; CI-NEXT: s_add_i32 s12, s12, s17625; CI-NEXT: s_mov_b32 flat_scratch_lo, s13626; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8627; CI-NEXT: s_waitcnt lgkmcnt(0)628; CI-NEXT: v_mov_b32_e32 v0, s2629; CI-NEXT: v_mov_b32_e32 v1, s3630; CI-NEXT: flat_load_dword v1, v[0:1]631; CI-NEXT: v_mov_b32_e32 v0, s0632; CI-NEXT: s_waitcnt vmcnt(0)633; CI-NEXT: v_xor_b32_e32 v2, 0x8000, v1634; CI-NEXT: v_lshlrev_b32_e32 v3, 16, v1635; CI-NEXT: v_and_b32_e32 v1, 0xffff0000, v1636; CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2637; CI-NEXT: v_mul_f32_e64 v4, -v1, v1638; CI-NEXT: v_mul_f32_e32 v1, v2, v3639; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v4640; CI-NEXT: v_lshr_b64 v[2:3], v[1:2], 16641; CI-NEXT: v_mov_b32_e32 v1, s1642; CI-NEXT: flat_store_dword v[0:1], v2643; CI-NEXT: s_endpgm644;645; GFX8-LABEL: v_fneg_fold_v2bf16:646; GFX8: ; %bb.0:647; GFX8-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0648; GFX8-NEXT: s_add_i32 s12, s12, s17649; GFX8-NEXT: s_mov_b32 flat_scratch_lo, s13650; GFX8-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8651; GFX8-NEXT: v_mov_b32_e32 v2, 0x8000652; GFX8-NEXT: s_waitcnt lgkmcnt(0)653; GFX8-NEXT: v_mov_b32_e32 v0, s2654; GFX8-NEXT: v_mov_b32_e32 v1, s3655; GFX8-NEXT: flat_load_dword v1, v[0:1]656; GFX8-NEXT: v_mov_b32_e32 v0, s0657; GFX8-NEXT: s_waitcnt vmcnt(0)658; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1659; GFX8-NEXT: v_xor_b32_sdwa v4, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD660; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v1661; GFX8-NEXT: v_xor_b32_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD662; GFX8-NEXT: v_mul_f32_e32 v2, v4, v3663; GFX8-NEXT: v_mul_f32_e32 v3, v1, v5664; GFX8-NEXT: v_bfe_u32 v1, v2, 16, 1665; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1666; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v2667; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3668; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1669; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5670; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v2671; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2672; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v3673; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc674; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3675; GFX8-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc676; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v2677; GFX8-NEXT: v_lshrrev_b64 v[2:3], 16, v[1:2]678; GFX8-NEXT: v_mov_b32_e32 v1, s1679; GFX8-NEXT: flat_store_dword v[0:1], v2680; GFX8-NEXT: s_endpgm681;682; GFX9-LABEL: v_fneg_fold_v2bf16:683; GFX9: ; %bb.0:684; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0685; GFX9-NEXT: v_mov_b32_e32 v0, 0686; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff687; GFX9-NEXT: s_waitcnt lgkmcnt(0)688; GFX9-NEXT: global_load_dword v1, v0, s[2:3]689; GFX9-NEXT: s_mov_b32 s2, 0x8000690; GFX9-NEXT: s_waitcnt vmcnt(0)691; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v1692; GFX9-NEXT: v_xor_b32_sdwa v4, v1, s2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD693; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v1694; GFX9-NEXT: v_xor_b32_sdwa v1, v1, s2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD695; GFX9-NEXT: v_mul_f32_e32 v3, v4, v3696; GFX9-NEXT: v_mul_f32_e32 v1, v1, v5697; GFX9-NEXT: v_bfe_u32 v4, v3, 16, 1698; GFX9-NEXT: v_bfe_u32 v6, v1, 16, 1699; GFX9-NEXT: v_add_u32_e32 v4, v4, v3700; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v3701; GFX9-NEXT: v_add_u32_e32 v6, v6, v1702; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4703; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3704; GFX9-NEXT: v_or_b32_e32 v7, 0x400000, v1705; GFX9-NEXT: v_add_u32_e32 v6, 0x7fff, v6706; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc707; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1708; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc709; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1710; GFX9-NEXT: v_and_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1711; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2712; GFX9-NEXT: global_store_dword v0, v1, s[0:1]713; GFX9-NEXT: s_endpgm714;715; GFX11-TRUE16-LABEL: v_fneg_fold_v2bf16:716; GFX11-TRUE16: ; %bb.0:717; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0718; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0719; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)720; GFX11-TRUE16-NEXT: global_load_b32 v1, v0, s[2:3]721; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)722; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1723; GFX11-TRUE16-NEXT: v_xor_b32_e32 v3, 0x8000, v1724; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1725; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1726; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)727; GFX11-TRUE16-NEXT: v_xor_b32_e32 v2, 0x8000, v2728; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3729; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)730; GFX11-TRUE16-NEXT: v_dual_mul_f32 v1, v3, v1 :: v_dual_lshlrev_b32 v2, 16, v2731; GFX11-TRUE16-NEXT: v_mul_f32_e32 v2, v2, v4732; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)733; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1734; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1735; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1736; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)737; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, v4, v1738; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2739; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2740; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v2741; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)742; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v4743; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3744; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)745; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v5, vcc_lo746; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1747; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo748; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)749; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2750; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1751; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)752; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l753; GFX11-TRUE16-NEXT: global_store_b32 v0, v1, s[0:1]754; GFX11-TRUE16-NEXT: s_endpgm755;756; GFX11-FAKE16-LABEL: v_fneg_fold_v2bf16:757; GFX11-FAKE16: ; %bb.0:758; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0759; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0760; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)761; GFX11-FAKE16-NEXT: global_load_b32 v1, v0, s[2:3]762; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)763; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1764; GFX11-FAKE16-NEXT: v_xor_b32_e32 v3, 0x8000, v1765; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1766; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1767; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)768; GFX11-FAKE16-NEXT: v_xor_b32_e32 v2, 0x8000, v2769; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3770; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)771; GFX11-FAKE16-NEXT: v_dual_mul_f32 v3, v3, v4 :: v_dual_lshlrev_b32 v2, 16, v2772; GFX11-FAKE16-NEXT: v_mul_f32_e32 v1, v2, v1773; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)774; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v3, 16, 1775; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v3776; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3777; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, v2, v3778; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)779; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2780; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v1, 16, 1781; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v1782; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo783; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)784; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, v4, v1785; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1786; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2787; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)788; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v4789; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2790; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)791; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo792; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1793; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)794; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v1, 16, v2795; GFX11-FAKE16-NEXT: global_store_b32 v0, v1, s[0:1]796; GFX11-FAKE16-NEXT: s_endpgm797 %val = load <2 x bfloat>, ptr addrspace(1) %in798 %fsub = fsub <2 x bfloat> <bfloat -0.0, bfloat -0.0>, %val799 %fmul = fmul <2 x bfloat> %fsub, %val800 store <2 x bfloat> %fmul, ptr addrspace(1) %out801 ret void802}803 804define amdgpu_kernel void @v_extract_fneg_fold_v2bf16(ptr addrspace(1) %in) #0 {805; CI-LABEL: v_extract_fneg_fold_v2bf16:806; CI: ; %bb.0:807; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0808; CI-NEXT: s_add_i32 s12, s12, s17809; CI-NEXT: s_mov_b32 flat_scratch_lo, s13810; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8811; CI-NEXT: s_waitcnt lgkmcnt(0)812; CI-NEXT: v_mov_b32_e32 v0, s0813; CI-NEXT: v_mov_b32_e32 v1, s1814; CI-NEXT: flat_load_dword v0, v[0:1]815; CI-NEXT: s_waitcnt vmcnt(0)816; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v0817; CI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0818; CI-NEXT: v_mul_f32_e32 v1, -4.0, v1819; CI-NEXT: v_sub_f32_e32 v0, 2.0, v0820; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1821; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0822; CI-NEXT: flat_store_short v[0:1], v1823; CI-NEXT: s_waitcnt vmcnt(0)824; CI-NEXT: flat_store_short v[0:1], v0825; CI-NEXT: s_waitcnt vmcnt(0)826; CI-NEXT: s_endpgm827;828; GFX8-LABEL: v_extract_fneg_fold_v2bf16:829; GFX8: ; %bb.0:830; GFX8-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0831; GFX8-NEXT: s_add_i32 s12, s12, s17832; GFX8-NEXT: s_mov_b32 flat_scratch_lo, s13833; GFX8-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8834; GFX8-NEXT: s_waitcnt lgkmcnt(0)835; GFX8-NEXT: v_mov_b32_e32 v0, s0836; GFX8-NEXT: v_mov_b32_e32 v1, s1837; GFX8-NEXT: flat_load_dword v0, v[0:1]838; GFX8-NEXT: s_waitcnt vmcnt(0)839; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v0840; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v0841; GFX8-NEXT: v_mul_f32_e32 v1, -4.0, v1842; GFX8-NEXT: v_sub_f32_e32 v0, 2.0, v0843; GFX8-NEXT: v_bfe_u32 v2, v1, 16, 1844; GFX8-NEXT: v_bfe_u32 v4, v0, 16, 1845; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v1846; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v0847; GFX8-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2848; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4849; GFX8-NEXT: v_or_b32_e32 v3, 0x400000, v1850; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1851; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v0852; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc853; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0854; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc855; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1856; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0857; GFX8-NEXT: flat_store_short v[0:1], v1858; GFX8-NEXT: s_waitcnt vmcnt(0)859; GFX8-NEXT: flat_store_short v[0:1], v0860; GFX8-NEXT: s_waitcnt vmcnt(0)861; GFX8-NEXT: s_endpgm862;863; GFX9-LABEL: v_extract_fneg_fold_v2bf16:864; GFX9: ; %bb.0:865; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0866; GFX9-NEXT: v_mov_b32_e32 v0, 0867; GFX9-NEXT: s_waitcnt lgkmcnt(0)868; GFX9-NEXT: global_load_dword v0, v0, s[0:1]869; GFX9-NEXT: s_waitcnt vmcnt(0)870; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v0871; GFX9-NEXT: v_and_b32_e32 v0, 0xffff0000, v0872; GFX9-NEXT: v_mul_f32_e32 v1, -4.0, v1873; GFX9-NEXT: v_sub_f32_e32 v0, 2.0, v0874; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1875; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1876; GFX9-NEXT: v_add_u32_e32 v2, v2, v1877; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1878; GFX9-NEXT: v_add_u32_e32 v4, v4, v0879; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2880; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1881; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v0882; GFX9-NEXT: v_add_u32_e32 v4, 0x7fff, v4883; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc884; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0885; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc886; GFX9-NEXT: global_store_short_d16_hi v[0:1], v1, off887; GFX9-NEXT: s_waitcnt vmcnt(0)888; GFX9-NEXT: global_store_short_d16_hi v[0:1], v0, off889; GFX9-NEXT: s_waitcnt vmcnt(0)890; GFX9-NEXT: s_endpgm891;892; GFX11-LABEL: v_extract_fneg_fold_v2bf16:893; GFX11: ; %bb.0:894; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0895; GFX11-NEXT: v_mov_b32_e32 v0, 0896; GFX11-NEXT: s_waitcnt lgkmcnt(0)897; GFX11-NEXT: global_load_b32 v0, v0, s[0:1]898; GFX11-NEXT: s_waitcnt vmcnt(0)899; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v0900; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)901; GFX11-NEXT: v_dual_mul_f32 v1, -4.0, v1 :: v_dual_and_b32 v0, 0xffff0000, v0902; GFX11-NEXT: v_sub_f32_e32 v0, 2.0, v0903; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)904; GFX11-NEXT: v_bfe_u32 v2, v1, 16, 1905; GFX11-NEXT: v_bfe_u32 v3, v0, 16, 1906; GFX11-NEXT: v_or_b32_e32 v4, 0x400000, v1907; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1908; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)909; GFX11-NEXT: v_add_nc_u32_e32 v2, v2, v1910; GFX11-NEXT: v_add_nc_u32_e32 v3, v3, v0911; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)912; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2913; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3914; GFX11-NEXT: v_or_b32_e32 v5, 0x400000, v0915; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)916; GFX11-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo917; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0918; GFX11-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo919; GFX11-NEXT: global_store_d16_hi_b16 v[0:1], v1, off dlc920; GFX11-NEXT: s_waitcnt_vscnt null, 0x0921; GFX11-NEXT: global_store_d16_hi_b16 v[0:1], v0, off dlc922; GFX11-NEXT: s_waitcnt_vscnt null, 0x0923; GFX11-NEXT: s_endpgm924 %val = load <2 x bfloat>, ptr addrspace(1) %in925 %fneg = fsub <2 x bfloat> <bfloat -0.0, bfloat -0.0>, %val926 %elt0 = extractelement <2 x bfloat> %fneg, i32 0927 %elt1 = extractelement <2 x bfloat> %fneg, i32 1928 929 %fmul0 = fmul bfloat %elt0, 4.0930 %fadd1 = fadd bfloat %elt1, 2.0931 store volatile bfloat %fmul0, ptr addrspace(1) poison932 store volatile bfloat %fadd1, ptr addrspace(1) poison933 ret void934}935 936define amdgpu_kernel void @v_extract_fneg_no_fold_v2bf16(ptr addrspace(1) %in) #0 {937; CI-LABEL: v_extract_fneg_no_fold_v2bf16:938; CI: ; %bb.0:939; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0940; CI-NEXT: s_add_i32 s12, s12, s17941; CI-NEXT: s_mov_b32 flat_scratch_lo, s13942; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8943; CI-NEXT: s_waitcnt lgkmcnt(0)944; CI-NEXT: v_mov_b32_e32 v0, s0945; CI-NEXT: v_mov_b32_e32 v1, s1946; CI-NEXT: flat_load_dword v0, v[0:1]947; CI-NEXT: s_waitcnt vmcnt(0)948; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v0949; CI-NEXT: v_xor_b32_e32 v0, 0x8000, v0950; CI-NEXT: v_xor_b32_e32 v1, 0x8000, v1951; CI-NEXT: flat_store_short v[0:1], v0952; CI-NEXT: s_waitcnt vmcnt(0)953; CI-NEXT: flat_store_short v[0:1], v1954; CI-NEXT: s_waitcnt vmcnt(0)955; CI-NEXT: s_endpgm956;957; GFX8-LABEL: v_extract_fneg_no_fold_v2bf16:958; GFX8: ; %bb.0:959; GFX8-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0960; GFX8-NEXT: s_add_i32 s12, s12, s17961; GFX8-NEXT: s_mov_b32 flat_scratch_lo, s13962; GFX8-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8963; GFX8-NEXT: s_waitcnt lgkmcnt(0)964; GFX8-NEXT: v_mov_b32_e32 v0, s0965; GFX8-NEXT: v_mov_b32_e32 v1, s1966; GFX8-NEXT: flat_load_dword v0, v[0:1]967; GFX8-NEXT: s_waitcnt vmcnt(0)968; GFX8-NEXT: v_xor_b32_e32 v0, 0x80008000, v0969; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v0970; GFX8-NEXT: flat_store_short v[0:1], v0971; GFX8-NEXT: s_waitcnt vmcnt(0)972; GFX8-NEXT: flat_store_short v[0:1], v1973; GFX8-NEXT: s_waitcnt vmcnt(0)974; GFX8-NEXT: s_endpgm975;976; GFX9-LABEL: v_extract_fneg_no_fold_v2bf16:977; GFX9: ; %bb.0:978; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0979; GFX9-NEXT: v_mov_b32_e32 v0, 0980; GFX9-NEXT: s_waitcnt lgkmcnt(0)981; GFX9-NEXT: global_load_dword v0, v0, s[0:1]982; GFX9-NEXT: s_waitcnt vmcnt(0)983; GFX9-NEXT: v_xor_b32_e32 v0, 0x80008000, v0984; GFX9-NEXT: global_store_short v[0:1], v0, off985; GFX9-NEXT: s_waitcnt vmcnt(0)986; GFX9-NEXT: global_store_short_d16_hi v[0:1], v0, off987; GFX9-NEXT: s_waitcnt vmcnt(0)988; GFX9-NEXT: s_endpgm989;990; GFX11-LABEL: v_extract_fneg_no_fold_v2bf16:991; GFX11: ; %bb.0:992; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0993; GFX11-NEXT: v_mov_b32_e32 v0, 0994; GFX11-NEXT: s_waitcnt lgkmcnt(0)995; GFX11-NEXT: global_load_b32 v0, v0, s[0:1]996; GFX11-NEXT: s_waitcnt vmcnt(0)997; GFX11-NEXT: v_xor_b32_e32 v0, 0x80008000, v0998; GFX11-NEXT: global_store_b16 v[0:1], v0, off dlc999; GFX11-NEXT: s_waitcnt_vscnt null, 0x01000; GFX11-NEXT: global_store_d16_hi_b16 v[0:1], v0, off dlc1001; GFX11-NEXT: s_waitcnt_vscnt null, 0x01002; GFX11-NEXT: s_endpgm1003 %val = load <2 x bfloat>, ptr addrspace(1) %in1004 %fneg = fsub <2 x bfloat> <bfloat -0.0, bfloat -0.0>, %val1005 %elt0 = extractelement <2 x bfloat> %fneg, i32 01006 %elt1 = extractelement <2 x bfloat> %fneg, i32 11007 store volatile bfloat %elt0, ptr addrspace(1) poison1008 store volatile bfloat %elt1, ptr addrspace(1) poison1009 ret void1010}1011 1012declare i32 @llvm.amdgcn.workitem.id.x() #11013 1014attributes #0 = { nounwind }1015attributes #1 = { nounwind readnone }1016;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:1017; CIVI: {{.*}}1018