1147 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=kaveri < %s | FileCheck -enable-var-scope -check-prefixes=CI %s3; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=tonga < %s | FileCheck -enable-var-scope -check-prefixes=VI %s4; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx900 < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s5; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-TRUE16 %s6; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-FAKE16 %s7 8; DAGCombiner will transform:9; (fabs (bf16 bitcast (i16 a))) => (bf16 bitcast (and (i16 a), 0x7FFFFFFF))10; unless isFabsFree returns true11 12define amdgpu_kernel void @s_fabs_free_bf16(ptr addrspace(1) %out, i16 %in) {13; CI-LABEL: s_fabs_free_bf16:14; CI: ; %bb.0:15; CI-NEXT: s_load_dword s2, s[8:9], 0x216; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x017; CI-NEXT: s_add_i32 s12, s12, s1718; CI-NEXT: s_mov_b32 flat_scratch_lo, s1319; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 820; CI-NEXT: s_waitcnt lgkmcnt(0)21; CI-NEXT: s_and_b32 s2, s2, 0x7fff22; CI-NEXT: v_mov_b32_e32 v0, s023; CI-NEXT: v_mov_b32_e32 v1, s124; CI-NEXT: v_mov_b32_e32 v2, s225; CI-NEXT: flat_store_short v[0:1], v226; CI-NEXT: s_endpgm27;28; VI-LABEL: s_fabs_free_bf16:29; VI: ; %bb.0:30; VI-NEXT: s_load_dword s2, s[8:9], 0x831; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x032; VI-NEXT: s_add_i32 s12, s12, s1733; VI-NEXT: s_mov_b32 flat_scratch_lo, s1334; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 835; VI-NEXT: s_waitcnt lgkmcnt(0)36; VI-NEXT: s_and_b32 s2, s2, 0x7fff37; VI-NEXT: v_mov_b32_e32 v0, s038; VI-NEXT: v_mov_b32_e32 v1, s139; VI-NEXT: v_mov_b32_e32 v2, s240; VI-NEXT: flat_store_short v[0:1], v241; VI-NEXT: s_endpgm42;43; GFX9-LABEL: s_fabs_free_bf16:44; GFX9: ; %bb.0:45; GFX9-NEXT: s_load_dword s2, s[8:9], 0x846; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x047; GFX9-NEXT: v_mov_b32_e32 v0, 048; GFX9-NEXT: s_waitcnt lgkmcnt(0)49; GFX9-NEXT: s_and_b32 s2, s2, 0x7fff50; GFX9-NEXT: v_mov_b32_e32 v1, s251; GFX9-NEXT: global_store_short v0, v1, s[0:1]52; GFX9-NEXT: s_endpgm53;54; GFX11-TRUE16-LABEL: s_fabs_free_bf16:55; GFX11-TRUE16: ; %bb.0:56; GFX11-TRUE16-NEXT: s_clause 0x157; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x858; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x059; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 060; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)61; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, 0x7fff62; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)63; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s264; GFX11-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]65; GFX11-TRUE16-NEXT: s_endpgm66;67; GFX11-FAKE16-LABEL: s_fabs_free_bf16:68; GFX11-FAKE16: ; %bb.0:69; GFX11-FAKE16-NEXT: s_clause 0x170; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x871; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x072; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)73; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, 0x7fff74; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)75; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s276; GFX11-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]77; GFX11-FAKE16-NEXT: s_endpgm78 %bc= bitcast i16 %in to bfloat79 %fabs = call bfloat @llvm.fabs.bf16(bfloat %bc)80 store bfloat %fabs, ptr addrspace(1) %out81 ret void82}83 84define amdgpu_kernel void @s_fabs_bf16(ptr addrspace(1) %out, bfloat %in) {85; CI-LABEL: s_fabs_bf16:86; CI: ; %bb.0:87; CI-NEXT: s_load_dword s2, s[8:9], 0x288; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x089; CI-NEXT: s_add_i32 s12, s12, s1790; CI-NEXT: s_mov_b32 flat_scratch_lo, s1391; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 892; CI-NEXT: s_waitcnt lgkmcnt(0)93; CI-NEXT: s_and_b32 s2, s2, 0x7fff94; CI-NEXT: v_mov_b32_e32 v0, s095; CI-NEXT: v_mov_b32_e32 v1, s196; CI-NEXT: v_mov_b32_e32 v2, s297; CI-NEXT: flat_store_short v[0:1], v298; CI-NEXT: s_endpgm99;100; VI-LABEL: s_fabs_bf16:101; VI: ; %bb.0:102; VI-NEXT: s_load_dword s2, s[8:9], 0x8103; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0104; VI-NEXT: s_add_i32 s12, s12, s17105; VI-NEXT: s_mov_b32 flat_scratch_lo, s13106; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8107; VI-NEXT: s_waitcnt lgkmcnt(0)108; VI-NEXT: s_and_b32 s2, s2, 0x7fff109; VI-NEXT: v_mov_b32_e32 v0, s0110; VI-NEXT: v_mov_b32_e32 v1, s1111; VI-NEXT: v_mov_b32_e32 v2, s2112; VI-NEXT: flat_store_short v[0:1], v2113; VI-NEXT: s_endpgm114;115; GFX9-LABEL: s_fabs_bf16:116; GFX9: ; %bb.0:117; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8118; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0119; GFX9-NEXT: v_mov_b32_e32 v0, 0120; GFX9-NEXT: s_waitcnt lgkmcnt(0)121; GFX9-NEXT: s_and_b32 s2, s2, 0x7fff122; GFX9-NEXT: v_mov_b32_e32 v1, s2123; GFX9-NEXT: global_store_short v0, v1, s[0:1]124; GFX9-NEXT: s_endpgm125;126; GFX11-TRUE16-LABEL: s_fabs_bf16:127; GFX11-TRUE16: ; %bb.0:128; GFX11-TRUE16-NEXT: s_clause 0x1129; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8130; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0131; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0132; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)133; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, 0x7fff134; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)135; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2136; GFX11-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]137; GFX11-TRUE16-NEXT: s_endpgm138;139; GFX11-FAKE16-LABEL: s_fabs_bf16:140; GFX11-FAKE16: ; %bb.0:141; GFX11-FAKE16-NEXT: s_clause 0x1142; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8143; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0144; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)145; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, 0x7fff146; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)147; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2148; GFX11-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]149; GFX11-FAKE16-NEXT: s_endpgm150 %fabs = call bfloat @llvm.fabs.bf16(bfloat %in)151 store bfloat %fabs, ptr addrspace(1) %out152 ret void153}154 155define amdgpu_kernel void @s_fabs_v2bf16(ptr addrspace(1) %out, <2 x bfloat> %in) {156; CI-LABEL: s_fabs_v2bf16:157; CI: ; %bb.0:158; CI-NEXT: s_load_dword s2, s[8:9], 0x2159; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0160; CI-NEXT: s_add_i32 s12, s12, s17161; CI-NEXT: s_mov_b32 flat_scratch_lo, s13162; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8163; CI-NEXT: s_waitcnt lgkmcnt(0)164; CI-NEXT: s_and_b32 s2, s2, 0x7fff7fff165; CI-NEXT: v_mov_b32_e32 v0, s0166; CI-NEXT: v_mov_b32_e32 v1, s1167; CI-NEXT: v_mov_b32_e32 v2, s2168; CI-NEXT: flat_store_dword v[0:1], v2169; CI-NEXT: s_endpgm170;171; VI-LABEL: s_fabs_v2bf16:172; VI: ; %bb.0:173; VI-NEXT: s_load_dword s2, s[8:9], 0x8174; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0175; VI-NEXT: s_add_i32 s12, s12, s17176; VI-NEXT: s_mov_b32 flat_scratch_lo, s13177; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8178; VI-NEXT: s_waitcnt lgkmcnt(0)179; VI-NEXT: s_and_b32 s2, s2, 0x7fff7fff180; VI-NEXT: v_mov_b32_e32 v0, s0181; VI-NEXT: v_mov_b32_e32 v1, s1182; VI-NEXT: v_mov_b32_e32 v2, s2183; VI-NEXT: flat_store_dword v[0:1], v2184; VI-NEXT: s_endpgm185;186; GFX9-LABEL: s_fabs_v2bf16:187; GFX9: ; %bb.0:188; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8189; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0190; GFX9-NEXT: v_mov_b32_e32 v0, 0191; GFX9-NEXT: s_waitcnt lgkmcnt(0)192; GFX9-NEXT: s_and_b32 s2, s2, 0x7fff7fff193; GFX9-NEXT: v_mov_b32_e32 v1, s2194; GFX9-NEXT: global_store_dword v0, v1, s[0:1]195; GFX9-NEXT: s_endpgm196;197; GFX11-LABEL: s_fabs_v2bf16:198; GFX11: ; %bb.0:199; GFX11-NEXT: s_clause 0x1200; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8201; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0202; GFX11-NEXT: s_waitcnt lgkmcnt(0)203; GFX11-NEXT: s_and_b32 s2, s2, 0x7fff7fff204; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)205; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2206; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]207; GFX11-NEXT: s_endpgm208 %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %in)209 store <2 x bfloat> %fabs, ptr addrspace(1) %out210 ret void211}212 213define amdgpu_kernel void @s_fabs_v4bf16(ptr addrspace(1) %out, <4 x bfloat> %in) {214; CI-LABEL: s_fabs_v4bf16:215; CI: ; %bb.0:216; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0217; CI-NEXT: s_add_i32 s12, s12, s17218; CI-NEXT: s_mov_b32 flat_scratch_lo, s13219; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8220; CI-NEXT: s_waitcnt lgkmcnt(0)221; CI-NEXT: s_and_b32 s3, s3, 0x7fff7fff222; CI-NEXT: s_and_b32 s2, s2, 0x7fff7fff223; CI-NEXT: v_mov_b32_e32 v3, s1224; CI-NEXT: v_mov_b32_e32 v0, s2225; CI-NEXT: v_mov_b32_e32 v1, s3226; CI-NEXT: v_mov_b32_e32 v2, s0227; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]228; CI-NEXT: s_endpgm229;230; VI-LABEL: s_fabs_v4bf16:231; VI: ; %bb.0:232; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0233; VI-NEXT: s_add_i32 s12, s12, s17234; VI-NEXT: s_mov_b32 flat_scratch_lo, s13235; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8236; VI-NEXT: s_waitcnt lgkmcnt(0)237; VI-NEXT: s_and_b32 s4, s3, 0x7fff238; VI-NEXT: s_lshr_b32 s3, s3, 16239; VI-NEXT: s_and_b32 s5, s2, 0x7fff240; VI-NEXT: s_lshr_b32 s2, s2, 16241; VI-NEXT: s_and_b32 s3, s3, 0x7fff242; VI-NEXT: s_and_b32 s2, s2, 0x7fff243; VI-NEXT: s_and_b32 s4, 0xffff, s4244; VI-NEXT: s_and_b32 s5, 0xffff, s5245; VI-NEXT: s_lshl_b32 s3, s3, 16246; VI-NEXT: s_lshl_b32 s2, s2, 16247; VI-NEXT: s_or_b32 s3, s4, s3248; VI-NEXT: s_or_b32 s2, s5, s2249; VI-NEXT: v_mov_b32_e32 v3, s1250; VI-NEXT: v_mov_b32_e32 v0, s2251; VI-NEXT: v_mov_b32_e32 v1, s3252; VI-NEXT: v_mov_b32_e32 v2, s0253; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]254; VI-NEXT: s_endpgm255;256; GFX9-LABEL: s_fabs_v4bf16:257; GFX9: ; %bb.0:258; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0259; GFX9-NEXT: v_mov_b32_e32 v2, 0260; GFX9-NEXT: s_waitcnt lgkmcnt(0)261; GFX9-NEXT: s_and_b32 s4, s3, 0x7fff262; GFX9-NEXT: s_lshr_b32 s3, s3, 16263; GFX9-NEXT: s_and_b32 s5, s2, 0x7fff264; GFX9-NEXT: s_lshr_b32 s2, s2, 16265; GFX9-NEXT: s_and_b32 s3, s3, 0x7fff266; GFX9-NEXT: s_and_b32 s2, s2, 0x7fff267; GFX9-NEXT: s_pack_ll_b32_b16 s3, s4, s3268; GFX9-NEXT: s_pack_ll_b32_b16 s2, s5, s2269; GFX9-NEXT: v_mov_b32_e32 v0, s2270; GFX9-NEXT: v_mov_b32_e32 v1, s3271; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]272; GFX9-NEXT: s_endpgm273;274; GFX11-LABEL: s_fabs_v4bf16:275; GFX11: ; %bb.0:276; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0277; GFX11-NEXT: s_waitcnt lgkmcnt(0)278; GFX11-NEXT: s_and_b32 s4, s3, 0x7fff279; GFX11-NEXT: s_lshr_b32 s3, s3, 16280; GFX11-NEXT: s_lshr_b32 s5, s2, 16281; GFX11-NEXT: s_and_b32 s2, s2, 0x7fff282; GFX11-NEXT: s_and_b32 s5, s5, 0x7fff283; GFX11-NEXT: s_and_b32 s3, s3, 0x7fff284; GFX11-NEXT: s_pack_ll_b32_b16 s2, s2, s5285; GFX11-NEXT: s_pack_ll_b32_b16 s3, s4, s3286; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)287; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3288; GFX11-NEXT: v_mov_b32_e32 v0, s2289; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]290; GFX11-NEXT: s_endpgm291 %fabs = call <4 x bfloat> @llvm.fabs.v4bf16(<4 x bfloat> %in)292 store <4 x bfloat> %fabs, ptr addrspace(1) %out293 ret void294}295 296define amdgpu_kernel void @fabs_fold_bf16(ptr addrspace(1) %out, bfloat %in0, bfloat %in1) {297; CI-LABEL: fabs_fold_bf16:298; CI: ; %bb.0:299; CI-NEXT: s_load_dword s2, s[8:9], 0x2300; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0301; CI-NEXT: s_add_i32 s12, s12, s17302; CI-NEXT: s_mov_b32 flat_scratch_lo, s13303; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8304; CI-NEXT: s_waitcnt lgkmcnt(0)305; CI-NEXT: s_and_b32 s3, s2, 0x7fff306; CI-NEXT: s_and_b32 s2, s2, 0xffff0000307; CI-NEXT: s_lshl_b32 s3, s3, 16308; CI-NEXT: v_mov_b32_e32 v0, s2309; CI-NEXT: v_mul_f32_e32 v0, s3, v0310; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v0311; CI-NEXT: v_mov_b32_e32 v0, s0312; CI-NEXT: v_mov_b32_e32 v1, s1313; CI-NEXT: flat_store_short v[0:1], v2314; CI-NEXT: s_endpgm315;316; VI-LABEL: fabs_fold_bf16:317; VI: ; %bb.0:318; VI-NEXT: s_load_dword s2, s[8:9], 0x8319; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0320; VI-NEXT: s_add_i32 s12, s12, s17321; VI-NEXT: s_mov_b32 flat_scratch_lo, s13322; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8323; VI-NEXT: s_waitcnt lgkmcnt(0)324; VI-NEXT: s_and_b32 s3, s2, 0x7fff325; VI-NEXT: s_and_b32 s2, s2, 0xffff0000326; VI-NEXT: s_lshl_b32 s3, s3, 16327; VI-NEXT: v_mov_b32_e32 v0, s2328; VI-NEXT: v_mul_f32_e32 v0, s3, v0329; VI-NEXT: v_bfe_u32 v1, v0, 16, 1330; VI-NEXT: v_add_u32_e32 v1, vcc, v1, v0331; VI-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1332; VI-NEXT: v_or_b32_e32 v2, 0x400000, v0333; VI-NEXT: v_cmp_u_f32_e32 vcc, v0, v0334; VI-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc335; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v0336; VI-NEXT: v_mov_b32_e32 v0, s0337; VI-NEXT: v_mov_b32_e32 v1, s1338; VI-NEXT: flat_store_short v[0:1], v2339; VI-NEXT: s_endpgm340;341; GFX9-LABEL: fabs_fold_bf16:342; GFX9: ; %bb.0:343; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8344; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0345; GFX9-NEXT: v_mov_b32_e32 v0, 0346; GFX9-NEXT: s_waitcnt lgkmcnt(0)347; GFX9-NEXT: s_and_b32 s3, s2, 0x7fff348; GFX9-NEXT: s_and_b32 s2, s2, 0xffff0000349; GFX9-NEXT: s_lshl_b32 s3, s3, 16350; GFX9-NEXT: v_mov_b32_e32 v1, s2351; GFX9-NEXT: v_mul_f32_e32 v1, s3, v1352; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1353; GFX9-NEXT: v_add_u32_e32 v2, v2, v1354; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2355; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1356; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1357; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc358; GFX9-NEXT: global_store_short_d16_hi v0, v1, s[0:1]359; GFX9-NEXT: s_endpgm360;361; GFX11-LABEL: fabs_fold_bf16:362; GFX11: ; %bb.0:363; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8364; GFX11-NEXT: s_waitcnt lgkmcnt(0)365; GFX11-NEXT: s_and_b32 s1, s0, 0x7fff366; GFX11-NEXT: s_and_b32 s0, s0, 0xffff0000367; GFX11-NEXT: s_lshl_b32 s1, s1, 16368; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)369; GFX11-NEXT: v_mul_f32_e64 v0, s1, s0370; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0371; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1372; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0373; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0374; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)375; GFX11-NEXT: v_add_nc_u32_e32 v1, v1, v0376; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1377; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)378; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_cndmask_b32 v0, v1, v2379; GFX11-NEXT: s_waitcnt lgkmcnt(0)380; GFX11-NEXT: global_store_d16_hi_b16 v3, v0, s[0:1]381; GFX11-NEXT: s_endpgm382 %fabs = call bfloat @llvm.fabs.bf16(bfloat %in0)383 %fmul = fmul bfloat %fabs, %in1384 store bfloat %fmul, ptr addrspace(1) %out385 ret void386}387 388define amdgpu_kernel void @v_fabs_v2bf16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {389; CI-LABEL: v_fabs_v2bf16:390; CI: ; %bb.0:391; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x2392; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0393; CI-NEXT: s_add_i32 s12, s12, s17394; CI-NEXT: s_mov_b32 flat_scratch_lo, s13395; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8396; CI-NEXT: s_waitcnt lgkmcnt(0)397; CI-NEXT: v_mov_b32_e32 v1, s1398; CI-NEXT: v_add_i32_e32 v0, vcc, s0, v0399; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc400; CI-NEXT: flat_load_dword v2, v[0:1]401; CI-NEXT: s_waitcnt vmcnt(0)402; CI-NEXT: v_and_b32_e32 v2, 0x7fff7fff, v2403; CI-NEXT: flat_store_dword v[0:1], v2404; CI-NEXT: s_endpgm405;406; VI-LABEL: v_fabs_v2bf16:407; VI: ; %bb.0:408; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x8409; VI-NEXT: v_lshlrev_b32_e32 v0, 2, v0410; VI-NEXT: s_add_i32 s12, s12, s17411; VI-NEXT: s_mov_b32 flat_scratch_lo, s13412; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8413; VI-NEXT: s_waitcnt lgkmcnt(0)414; VI-NEXT: v_mov_b32_e32 v1, s1415; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v0416; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc417; VI-NEXT: flat_load_dword v2, v[0:1]418; VI-NEXT: s_waitcnt vmcnt(0)419; VI-NEXT: v_and_b32_e32 v2, 0x7fff7fff, v2420; VI-NEXT: flat_store_dword v[0:1], v2421; VI-NEXT: s_endpgm422;423; GFX9-LABEL: v_fabs_v2bf16:424; GFX9: ; %bb.0:425; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x8426; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0427; GFX9-NEXT: s_waitcnt lgkmcnt(0)428; GFX9-NEXT: global_load_dword v1, v0, s[0:1]429; GFX9-NEXT: s_waitcnt vmcnt(0)430; GFX9-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v1431; GFX9-NEXT: global_store_dword v0, v1, s[0:1]432; GFX9-NEXT: s_endpgm433;434; GFX11-LABEL: v_fabs_v2bf16:435; GFX11: ; %bb.0:436; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x8437; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0438; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)439; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0440; GFX11-NEXT: s_waitcnt lgkmcnt(0)441; GFX11-NEXT: global_load_b32 v1, v0, s[0:1]442; GFX11-NEXT: s_waitcnt vmcnt(0)443; GFX11-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v1444; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]445; GFX11-NEXT: s_endpgm446 %tid = call i32 @llvm.amdgcn.workitem.id.x()447 %gep.in = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %in, i32 %tid448 %gep.out = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %in, i32 %tid449 %val = load <2 x bfloat>, ptr addrspace(1) %gep.in, align 2450 %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %val)451 store <2 x bfloat> %fabs, ptr addrspace(1) %gep.out452 ret void453}454 455define amdgpu_kernel void @fabs_free_v2bf16(ptr addrspace(1) %out, i32 %in) #0 {456; CI-LABEL: fabs_free_v2bf16:457; CI: ; %bb.0:458; CI-NEXT: s_load_dword s2, s[8:9], 0x2459; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0460; CI-NEXT: s_add_i32 s12, s12, s17461; CI-NEXT: s_mov_b32 flat_scratch_lo, s13462; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8463; CI-NEXT: s_waitcnt lgkmcnt(0)464; CI-NEXT: s_and_b32 s2, s2, 0x7fff7fff465; CI-NEXT: v_mov_b32_e32 v0, s0466; CI-NEXT: v_mov_b32_e32 v1, s1467; CI-NEXT: v_mov_b32_e32 v2, s2468; CI-NEXT: flat_store_dword v[0:1], v2469; CI-NEXT: s_endpgm470;471; VI-LABEL: fabs_free_v2bf16:472; VI: ; %bb.0:473; VI-NEXT: s_load_dword s2, s[8:9], 0x8474; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0475; VI-NEXT: s_add_i32 s12, s12, s17476; VI-NEXT: s_mov_b32 flat_scratch_lo, s13477; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8478; VI-NEXT: s_waitcnt lgkmcnt(0)479; VI-NEXT: s_and_b32 s2, s2, 0x7fff7fff480; VI-NEXT: v_mov_b32_e32 v0, s0481; VI-NEXT: v_mov_b32_e32 v1, s1482; VI-NEXT: v_mov_b32_e32 v2, s2483; VI-NEXT: flat_store_dword v[0:1], v2484; VI-NEXT: s_endpgm485;486; GFX9-LABEL: fabs_free_v2bf16:487; GFX9: ; %bb.0:488; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8489; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0490; GFX9-NEXT: v_mov_b32_e32 v0, 0491; GFX9-NEXT: s_waitcnt lgkmcnt(0)492; GFX9-NEXT: s_and_b32 s2, s2, 0x7fff7fff493; GFX9-NEXT: v_mov_b32_e32 v1, s2494; GFX9-NEXT: global_store_dword v0, v1, s[0:1]495; GFX9-NEXT: s_endpgm496;497; GFX11-LABEL: fabs_free_v2bf16:498; GFX11: ; %bb.0:499; GFX11-NEXT: s_clause 0x1500; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8501; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0502; GFX11-NEXT: s_waitcnt lgkmcnt(0)503; GFX11-NEXT: s_and_b32 s2, s2, 0x7fff7fff504; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)505; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2506; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]507; GFX11-NEXT: s_endpgm508 %bc = bitcast i32 %in to <2 x bfloat>509 %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %bc)510 store <2 x bfloat> %fabs, ptr addrspace(1) %out511 ret void512}513 514; FIXME: Should do fabs after conversion to avoid converting multiple515; times in this particular case.516define amdgpu_kernel void @v_fabs_fold_self_v2bf16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {517; CI-LABEL: v_fabs_fold_self_v2bf16:518; CI: ; %bb.0:519; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0520; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0521; CI-NEXT: s_add_i32 s12, s12, s17522; CI-NEXT: s_mov_b32 flat_scratch_lo, s13523; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8524; CI-NEXT: s_waitcnt lgkmcnt(0)525; CI-NEXT: v_mov_b32_e32 v1, s3526; CI-NEXT: v_add_i32_e32 v0, vcc, s2, v0527; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc528; CI-NEXT: flat_load_dword v2, v[0:1]529; CI-NEXT: v_mov_b32_e32 v0, s0530; CI-NEXT: v_mov_b32_e32 v1, s1531; CI-NEXT: s_waitcnt vmcnt(0)532; CI-NEXT: v_and_b32_e32 v3, 0x7fff, v2533; CI-NEXT: v_lshlrev_b32_e32 v4, 16, v2534; CI-NEXT: v_and_b32_e32 v5, 0xffff0000, v2535; CI-NEXT: v_and_b32_e32 v2, 0x7fff0000, v2536; CI-NEXT: v_lshlrev_b32_e32 v3, 16, v3537; CI-NEXT: v_mul_f32_e32 v2, v2, v5538; CI-NEXT: v_mul_f32_e32 v3, v3, v4539; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v2540; CI-NEXT: v_alignbit_b32 v2, v2, v3, 16541; CI-NEXT: flat_store_dword v[0:1], v2542; CI-NEXT: s_endpgm543;544; VI-LABEL: v_fabs_fold_self_v2bf16:545; VI: ; %bb.0:546; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0547; VI-NEXT: v_lshlrev_b32_e32 v0, 2, v0548; VI-NEXT: s_add_i32 s12, s12, s17549; VI-NEXT: s_mov_b32 flat_scratch_lo, s13550; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8551; VI-NEXT: s_waitcnt lgkmcnt(0)552; VI-NEXT: v_mov_b32_e32 v1, s3553; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v0554; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc555; VI-NEXT: flat_load_dword v2, v[0:1]556; VI-NEXT: v_mov_b32_e32 v3, 0x7fff557; VI-NEXT: s_movk_i32 s2, 0x7fff558; VI-NEXT: v_mov_b32_e32 v0, s0559; VI-NEXT: v_mov_b32_e32 v1, s1560; VI-NEXT: s_waitcnt vmcnt(0)561; VI-NEXT: v_lshlrev_b32_e32 v4, 16, v2562; VI-NEXT: v_and_b32_sdwa v5, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD563; VI-NEXT: v_and_b32_e32 v6, 0xffff0000, v2564; VI-NEXT: v_and_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1565; VI-NEXT: v_mul_f32_e32 v3, v5, v4566; VI-NEXT: v_mul_f32_e32 v2, v2, v6567; VI-NEXT: v_bfe_u32 v4, v3, 16, 1568; VI-NEXT: v_bfe_u32 v6, v2, 16, 1569; VI-NEXT: v_add_u32_e32 v4, vcc, v4, v3570; VI-NEXT: v_add_u32_e32 v6, vcc, v6, v2571; VI-NEXT: v_add_u32_e32 v4, vcc, s2, v4572; VI-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6573; VI-NEXT: v_or_b32_e32 v5, 0x400000, v3574; VI-NEXT: v_cmp_u_f32_e32 vcc, v3, v3575; VI-NEXT: v_or_b32_e32 v7, 0x400000, v2576; VI-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc577; VI-NEXT: v_cmp_u_f32_e32 vcc, v2, v2578; VI-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc579; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2580; VI-NEXT: v_alignbit_b32 v2, v2, v3, 16581; VI-NEXT: flat_store_dword v[0:1], v2582; VI-NEXT: s_endpgm583;584; GFX9-LABEL: v_fabs_fold_self_v2bf16:585; GFX9: ; %bb.0:586; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0587; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0588; GFX9-NEXT: v_mov_b32_e32 v1, 0589; GFX9-NEXT: s_waitcnt lgkmcnt(0)590; GFX9-NEXT: global_load_dword v0, v0, s[2:3]591; GFX9-NEXT: s_movk_i32 s2, 0x7fff592; GFX9-NEXT: s_waitcnt vmcnt(0)593; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v0594; GFX9-NEXT: v_and_b32_sdwa v3, s2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD595; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v0596; GFX9-NEXT: v_and_b32_sdwa v0, s2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1597; GFX9-NEXT: v_mul_f32_e32 v2, v3, v2598; GFX9-NEXT: v_mul_f32_e32 v0, v0, v4599; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1600; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2601; GFX9-NEXT: v_bfe_u32 v5, v0, 16, 1602; GFX9-NEXT: v_add3_u32 v3, v3, v2, s2603; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2604; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v0605; GFX9-NEXT: v_add3_u32 v5, v5, v0, s2606; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc607; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0608; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc609; GFX9-NEXT: s_mov_b32 s2, 0x7060302610; GFX9-NEXT: v_perm_b32 v0, v0, v2, s2611; GFX9-NEXT: global_store_dword v1, v0, s[0:1]612; GFX9-NEXT: s_endpgm613;614; GFX11-TRUE16-LABEL: v_fabs_fold_self_v2bf16:615; GFX11-TRUE16: ; %bb.0:616; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0617; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0618; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0619; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)620; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0621; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)622; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[2:3]623; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)624; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0625; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.l626; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)627; GFX11-TRUE16-NEXT: v_dual_mul_f32 v2, v1, v2 :: v_dual_and_b32 v3, 0xffff0000, v0628; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.h629; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2630; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)631; GFX11-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v3632; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1633; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2634; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1635; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)636; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff637; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0638; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0639; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff640; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)641; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo642; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0643; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo644; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)645; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h646; GFX11-TRUE16-NEXT: global_store_b32 v2, v0, s[0:1]647; GFX11-TRUE16-NEXT: s_endpgm648;649; GFX11-FAKE16-LABEL: v_fabs_fold_self_v2bf16:650; GFX11-FAKE16: ; %bb.0:651; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0652; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0653; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)654; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0655; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)656; GFX11-FAKE16-NEXT: global_load_b32 v0, v0, s[2:3]657; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)658; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0659; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0x7fff, v0660; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0661; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0662; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)663; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2664; GFX11-FAKE16-NEXT: v_dual_mul_f32 v2, v2, v3 :: v_dual_and_b32 v1, 0x7fff, v1665; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)666; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1667; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v2668; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)669; GFX11-FAKE16-NEXT: v_mul_f32_e32 v0, v1, v0670; GFX11-FAKE16-NEXT: v_bfe_u32 v1, v2, 16, 1671; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2672; GFX11-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1673; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)674; GFX11-FAKE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff675; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0676; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, 0677; GFX11-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff678; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)679; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo680; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0681; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo682; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)683; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x7060302684; GFX11-FAKE16-NEXT: global_store_b32 v2, v0, s[0:1]685; GFX11-FAKE16-NEXT: s_endpgm686 %tid = call i32 @llvm.amdgcn.workitem.id.x()687 %gep = getelementptr <2 x bfloat>, ptr addrspace(1) %in, i32 %tid688 %val = load <2 x bfloat>, ptr addrspace(1) %gep689 %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %val)690 %fmul = fmul <2 x bfloat> %fabs, %val691 store <2 x bfloat> %fmul, ptr addrspace(1) %out692 ret void693}694 695define amdgpu_kernel void @v_fabs_fold_v2bf16(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %other.val) #0 {696; CI-LABEL: v_fabs_fold_v2bf16:697; CI: ; %bb.0:698; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0699; CI-NEXT: s_load_dword s4, s[8:9], 0x4700; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0701; CI-NEXT: s_add_i32 s12, s12, s17702; CI-NEXT: s_mov_b32 flat_scratch_lo, s13703; CI-NEXT: s_waitcnt lgkmcnt(0)704; CI-NEXT: v_mov_b32_e32 v1, s3705; CI-NEXT: v_add_i32_e32 v0, vcc, s2, v0706; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8707; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc708; CI-NEXT: flat_load_dword v2, v[0:1]709; CI-NEXT: v_mov_b32_e32 v1, s1710; CI-NEXT: s_and_b32 s1, s4, 0xffff0000711; CI-NEXT: v_mov_b32_e32 v0, s0712; CI-NEXT: s_lshl_b32 s0, s4, 16713; CI-NEXT: s_waitcnt vmcnt(0)714; CI-NEXT: v_and_b32_e32 v3, 0x7fff0000, v2715; CI-NEXT: v_and_b32_e32 v2, 0x7fff, v2716; CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2717; CI-NEXT: v_mul_f32_e32 v3, s1, v3718; CI-NEXT: v_mul_f32_e32 v2, s0, v2719; CI-NEXT: v_lshrrev_b32_e32 v3, 16, v3720; CI-NEXT: v_alignbit_b32 v2, v3, v2, 16721; CI-NEXT: flat_store_dword v[0:1], v2722; CI-NEXT: s_endpgm723;724; VI-LABEL: v_fabs_fold_v2bf16:725; VI: ; %bb.0:726; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0727; VI-NEXT: s_load_dword s4, s[8:9], 0x10728; VI-NEXT: v_lshlrev_b32_e32 v0, 2, v0729; VI-NEXT: s_add_i32 s12, s12, s17730; VI-NEXT: s_mov_b32 flat_scratch_lo, s13731; VI-NEXT: s_waitcnt lgkmcnt(0)732; VI-NEXT: v_mov_b32_e32 v1, s3733; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v0734; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8735; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc736; VI-NEXT: flat_load_dword v2, v[0:1]737; VI-NEXT: v_mov_b32_e32 v3, 0x7fff738; VI-NEXT: v_mov_b32_e32 v0, s0739; VI-NEXT: v_mov_b32_e32 v1, s1740; VI-NEXT: s_lshl_b32 s0, s4, 16741; VI-NEXT: s_and_b32 s1, s4, 0xffff0000742; VI-NEXT: s_movk_i32 s2, 0x7fff743; VI-NEXT: s_waitcnt vmcnt(0)744; VI-NEXT: v_and_b32_sdwa v4, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD745; VI-NEXT: v_and_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1746; VI-NEXT: v_mul_f32_e32 v3, s0, v4747; VI-NEXT: v_mul_f32_e32 v2, s1, v2748; VI-NEXT: v_bfe_u32 v4, v3, 16, 1749; VI-NEXT: v_bfe_u32 v6, v2, 16, 1750; VI-NEXT: v_add_u32_e32 v4, vcc, v4, v3751; VI-NEXT: v_add_u32_e32 v6, vcc, v6, v2752; VI-NEXT: v_add_u32_e32 v4, vcc, s2, v4753; VI-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6754; VI-NEXT: v_or_b32_e32 v5, 0x400000, v3755; VI-NEXT: v_cmp_u_f32_e32 vcc, v3, v3756; VI-NEXT: v_or_b32_e32 v7, 0x400000, v2757; VI-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc758; VI-NEXT: v_cmp_u_f32_e32 vcc, v2, v2759; VI-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc760; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2761; VI-NEXT: v_alignbit_b32 v2, v2, v3, 16762; VI-NEXT: flat_store_dword v[0:1], v2763; VI-NEXT: s_endpgm764;765; GFX9-LABEL: v_fabs_fold_v2bf16:766; GFX9: ; %bb.0:767; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0768; GFX9-NEXT: s_load_dword s4, s[8:9], 0x10769; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0770; GFX9-NEXT: v_mov_b32_e32 v1, 0771; GFX9-NEXT: s_waitcnt lgkmcnt(0)772; GFX9-NEXT: global_load_dword v0, v0, s[2:3]773; GFX9-NEXT: s_movk_i32 s2, 0x7fff774; GFX9-NEXT: s_lshl_b32 s3, s4, 16775; GFX9-NEXT: s_and_b32 s4, s4, 0xffff0000776; GFX9-NEXT: s_waitcnt vmcnt(0)777; GFX9-NEXT: v_and_b32_sdwa v2, s2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD778; GFX9-NEXT: v_and_b32_sdwa v0, s2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1779; GFX9-NEXT: v_mul_f32_e32 v2, s3, v2780; GFX9-NEXT: v_mul_f32_e32 v0, s4, v0781; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1782; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v2783; GFX9-NEXT: v_bfe_u32 v5, v0, 16, 1784; GFX9-NEXT: v_add3_u32 v3, v3, v2, s2785; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2786; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v0787; GFX9-NEXT: v_add3_u32 v5, v5, v0, s2788; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc789; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0790; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc791; GFX9-NEXT: s_mov_b32 s2, 0x7060302792; GFX9-NEXT: v_perm_b32 v0, v0, v2, s2793; GFX9-NEXT: global_store_dword v1, v0, s[0:1]794; GFX9-NEXT: s_endpgm795;796; GFX11-TRUE16-LABEL: v_fabs_fold_v2bf16:797; GFX11-TRUE16: ; %bb.0:798; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0799; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0800; GFX11-TRUE16-NEXT: s_load_b32 s4, s[4:5], 0x10801; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0802; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)803; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0804; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)805; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[2:3]806; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s4, 16807; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)808; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.l809; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)810; GFX11-TRUE16-NEXT: v_mul_f32_e32 v2, s2, v1811; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.h812; GFX11-TRUE16-NEXT: s_and_b32 s2, s4, 0xffff0000813; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2814; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)815; GFX11-TRUE16-NEXT: v_mul_f32_e32 v0, s2, v1816; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1817; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2818; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1819; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)820; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff821; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0822; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0823; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff824; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)825; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo826; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0827; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo828; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)829; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h830; GFX11-TRUE16-NEXT: global_store_b32 v2, v0, s[0:1]831; GFX11-TRUE16-NEXT: s_endpgm832;833; GFX11-FAKE16-LABEL: v_fabs_fold_v2bf16:834; GFX11-FAKE16: ; %bb.0:835; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0836; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0837; GFX11-FAKE16-NEXT: s_load_b32 s4, s[4:5], 0x10838; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)839; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0840; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)841; GFX11-FAKE16-NEXT: global_load_b32 v0, v0, s[2:3]842; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s4, 16843; GFX11-FAKE16-NEXT: s_and_b32 s3, s4, 0xffff0000844; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)845; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0846; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x7fff, v0847; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)848; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0849; GFX11-FAKE16-NEXT: v_dual_mul_f32 v0, s2, v0 :: v_dual_and_b32 v1, 0x7fff, v1850; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)851; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1852; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v0853; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0854; GFX11-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff855; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)856; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v0, v2, v4 :: v_dual_lshlrev_b32 v1, 16, v1857; GFX11-FAKE16-NEXT: v_dual_mul_f32 v1, s3, v1 :: v_dual_mov_b32 v2, 0858; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)859; GFX11-FAKE16-NEXT: v_bfe_u32 v3, v1, 16, 1860; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v1861; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1862; GFX11-FAKE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff863; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)864; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo865; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302866; GFX11-FAKE16-NEXT: global_store_b32 v2, v0, s[0:1]867; GFX11-FAKE16-NEXT: s_endpgm868 %tid = call i32 @llvm.amdgcn.workitem.id.x()869 %gep = getelementptr <2 x bfloat>, ptr addrspace(1) %in, i32 %tid870 %val = load <2 x bfloat>, ptr addrspace(1) %gep871 %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %val)872 %other.val.cvt = bitcast i32 %other.val to <2 x bfloat>873 %fmul = fmul <2 x bfloat> %fabs, %other.val.cvt874 store <2 x bfloat> %fmul, ptr addrspace(1) %out875 ret void876}877 878define amdgpu_kernel void @v_extract_fabs_fold_v2bf16(ptr addrspace(1) %in) #0 {879; CI-LABEL: v_extract_fabs_fold_v2bf16:880; CI: ; %bb.0:881; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0882; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0883; CI-NEXT: s_add_i32 s12, s12, s17884; CI-NEXT: s_mov_b32 flat_scratch_lo, s13885; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8886; CI-NEXT: s_waitcnt lgkmcnt(0)887; CI-NEXT: v_mov_b32_e32 v1, s1888; CI-NEXT: v_add_i32_e32 v0, vcc, s0, v0889; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc890; CI-NEXT: flat_load_dword v0, v[0:1]891; CI-NEXT: s_waitcnt vmcnt(0)892; CI-NEXT: v_and_b32_e32 v1, 0x7fff, v0893; CI-NEXT: v_and_b32_e32 v0, 0x7fff0000, v0894; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1895; CI-NEXT: v_add_f32_e32 v0, 2.0, v0896; CI-NEXT: v_mul_f32_e32 v1, 4.0, v1897; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0898; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1899; CI-NEXT: flat_store_short v[0:1], v1900; CI-NEXT: s_waitcnt vmcnt(0)901; CI-NEXT: flat_store_short v[0:1], v0902; CI-NEXT: s_waitcnt vmcnt(0)903; CI-NEXT: s_endpgm904;905; VI-LABEL: v_extract_fabs_fold_v2bf16:906; VI: ; %bb.0:907; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0908; VI-NEXT: v_lshlrev_b32_e32 v0, 2, v0909; VI-NEXT: s_add_i32 s12, s12, s17910; VI-NEXT: s_mov_b32 flat_scratch_lo, s13911; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8912; VI-NEXT: s_waitcnt lgkmcnt(0)913; VI-NEXT: v_mov_b32_e32 v1, s1914; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v0915; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc916; VI-NEXT: flat_load_dword v0, v[0:1]917; VI-NEXT: v_mov_b32_e32 v1, 0x7fff918; VI-NEXT: s_movk_i32 s0, 0x7fff919; VI-NEXT: s_waitcnt vmcnt(0)920; VI-NEXT: v_and_b32_sdwa v2, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD921; VI-NEXT: v_and_b32_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1922; VI-NEXT: v_mul_f32_e32 v1, 4.0, v2923; VI-NEXT: v_add_f32_e32 v0, 2.0, v0924; VI-NEXT: v_bfe_u32 v2, v1, 16, 1925; VI-NEXT: v_bfe_u32 v4, v0, 16, 1926; VI-NEXT: v_add_u32_e32 v2, vcc, v2, v1927; VI-NEXT: v_add_u32_e32 v4, vcc, v4, v0928; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v2929; VI-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4930; VI-NEXT: v_or_b32_e32 v3, 0x400000, v1931; VI-NEXT: v_cmp_u_f32_e32 vcc, v1, v1932; VI-NEXT: v_or_b32_e32 v5, 0x400000, v0933; VI-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc934; VI-NEXT: v_cmp_u_f32_e32 vcc, v0, v0935; VI-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc936; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v1937; VI-NEXT: v_lshrrev_b32_e32 v0, 16, v0938; VI-NEXT: flat_store_short v[0:1], v1939; VI-NEXT: s_waitcnt vmcnt(0)940; VI-NEXT: flat_store_short v[0:1], v0941; VI-NEXT: s_waitcnt vmcnt(0)942; VI-NEXT: s_endpgm943;944; GFX9-LABEL: v_extract_fabs_fold_v2bf16:945; GFX9: ; %bb.0:946; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0947; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0948; GFX9-NEXT: s_waitcnt lgkmcnt(0)949; GFX9-NEXT: global_load_dword v0, v0, s[0:1]950; GFX9-NEXT: s_movk_i32 s0, 0x7fff951; GFX9-NEXT: s_waitcnt vmcnt(0)952; GFX9-NEXT: v_and_b32_sdwa v1, s0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD953; GFX9-NEXT: v_and_b32_sdwa v0, s0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1954; GFX9-NEXT: v_mul_f32_e32 v1, 4.0, v1955; GFX9-NEXT: v_add_f32_e32 v0, 2.0, v0956; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1957; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1958; GFX9-NEXT: v_bfe_u32 v4, v0, 16, 1959; GFX9-NEXT: v_add3_u32 v2, v2, v1, s0960; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1961; GFX9-NEXT: v_or_b32_e32 v5, 0x400000, v0962; GFX9-NEXT: v_add3_u32 v4, v4, v0, s0963; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc964; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v0965; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc966; GFX9-NEXT: global_store_short_d16_hi v[0:1], v1, off967; GFX9-NEXT: s_waitcnt vmcnt(0)968; GFX9-NEXT: global_store_short_d16_hi v[0:1], v0, off969; GFX9-NEXT: s_waitcnt vmcnt(0)970; GFX9-NEXT: s_endpgm971;972; GFX11-TRUE16-LABEL: v_extract_fabs_fold_v2bf16:973; GFX11-TRUE16: ; %bb.0:974; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0975; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0976; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0977; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)978; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0979; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)980; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[0:1]981; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)982; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.l983; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0x7fff, v0.h984; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)985; GFX11-TRUE16-NEXT: v_mul_f32_e32 v2, 4.0, v1986; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l987; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)988; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2989; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 2.0, v1990; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1991; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2992; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)993; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1994; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff995; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0996; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)997; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff998; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo999; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v01000; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)1001; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo1002; GFX11-TRUE16-NEXT: global_store_d16_hi_b16 v[0:1], v1, off dlc1003; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x01004; GFX11-TRUE16-NEXT: global_store_d16_hi_b16 v[0:1], v0, off dlc1005; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x01006; GFX11-TRUE16-NEXT: s_endpgm1007;1008; GFX11-FAKE16-LABEL: v_extract_fabs_fold_v2bf16:1009; GFX11-FAKE16: ; %bb.0:1010; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x01011; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v01012; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)1013; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 2, v01014; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)1015; GFX11-FAKE16-NEXT: global_load_b32 v0, v0, s[0:1]1016; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)1017; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v01018; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x7fff, v01019; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1020; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v01021; GFX11-FAKE16-NEXT: v_dual_mul_f32 v0, 4.0, v0 :: v_dual_and_b32 v1, 0x7fff, v11022; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)1023; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 11024; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v01025; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v01026; GFX11-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff1027; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1028; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v0, v2, v4 :: v_dual_lshlrev_b32 v1, 16, v11029; GFX11-FAKE16-NEXT: v_add_f32_e32 v1, 2.0, v11030; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)1031; GFX11-FAKE16-NEXT: v_bfe_u32 v3, v1, 16, 11032; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v11033; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v11034; GFX11-FAKE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff1035; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)1036; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo1037; GFX11-FAKE16-NEXT: global_store_d16_hi_b16 v[0:1], v0, off dlc1038; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x01039; GFX11-FAKE16-NEXT: global_store_d16_hi_b16 v[0:1], v1, off dlc1040; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x01041; GFX11-FAKE16-NEXT: s_endpgm1042 %tid = call i32 @llvm.amdgcn.workitem.id.x()1043 %gep.in = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %in, i32 %tid1044 %val = load <2 x bfloat>, ptr addrspace(1) %gep.in1045 %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %val)1046 %elt0 = extractelement <2 x bfloat> %fabs, i32 01047 %elt1 = extractelement <2 x bfloat> %fabs, i32 11048 1049 %fmul0 = fmul bfloat %elt0, 4.01050 %fadd1 = fadd bfloat %elt1, 2.01051 store volatile bfloat %fmul0, ptr addrspace(1) poison1052 store volatile bfloat %fadd1, ptr addrspace(1) poison1053 ret void1054}1055 1056define amdgpu_kernel void @v_extract_fabs_no_fold_v2bf16(ptr addrspace(1) %in) #0 {1057; CI-LABEL: v_extract_fabs_no_fold_v2bf16:1058; CI: ; %bb.0:1059; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01060; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v01061; CI-NEXT: s_add_i32 s12, s12, s171062; CI-NEXT: s_mov_b32 flat_scratch_lo, s131063; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81064; CI-NEXT: s_waitcnt lgkmcnt(0)1065; CI-NEXT: v_mov_b32_e32 v1, s11066; CI-NEXT: v_add_i32_e32 v0, vcc, s0, v01067; CI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1068; CI-NEXT: flat_load_dword v0, v[0:1]1069; CI-NEXT: s_waitcnt vmcnt(0)1070; CI-NEXT: v_bfe_u32 v1, v0, 16, 151071; CI-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v01072; CI-NEXT: flat_store_short v[0:1], v01073; CI-NEXT: s_waitcnt vmcnt(0)1074; CI-NEXT: flat_store_short v[0:1], v11075; CI-NEXT: s_waitcnt vmcnt(0)1076; CI-NEXT: s_endpgm1077;1078; VI-LABEL: v_extract_fabs_no_fold_v2bf16:1079; VI: ; %bb.0:1080; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01081; VI-NEXT: v_lshlrev_b32_e32 v0, 2, v01082; VI-NEXT: s_add_i32 s12, s12, s171083; VI-NEXT: s_mov_b32 flat_scratch_lo, s131084; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81085; VI-NEXT: s_waitcnt lgkmcnt(0)1086; VI-NEXT: v_mov_b32_e32 v1, s11087; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v01088; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1089; VI-NEXT: flat_load_dword v0, v[0:1]1090; VI-NEXT: s_waitcnt vmcnt(0)1091; VI-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v01092; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v01093; VI-NEXT: flat_store_short v[0:1], v01094; VI-NEXT: s_waitcnt vmcnt(0)1095; VI-NEXT: flat_store_short v[0:1], v11096; VI-NEXT: s_waitcnt vmcnt(0)1097; VI-NEXT: s_endpgm1098;1099; GFX9-LABEL: v_extract_fabs_no_fold_v2bf16:1100; GFX9: ; %bb.0:1101; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x01102; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v01103; GFX9-NEXT: s_waitcnt lgkmcnt(0)1104; GFX9-NEXT: global_load_dword v0, v0, s[0:1]1105; GFX9-NEXT: s_waitcnt vmcnt(0)1106; GFX9-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v01107; GFX9-NEXT: global_store_short v[0:1], v0, off1108; GFX9-NEXT: s_waitcnt vmcnt(0)1109; GFX9-NEXT: global_store_short_d16_hi v[0:1], v0, off1110; GFX9-NEXT: s_waitcnt vmcnt(0)1111; GFX9-NEXT: s_endpgm1112;1113; GFX11-LABEL: v_extract_fabs_no_fold_v2bf16:1114; GFX11: ; %bb.0:1115; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x01116; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v01117; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)1118; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v01119; GFX11-NEXT: s_waitcnt lgkmcnt(0)1120; GFX11-NEXT: global_load_b32 v0, v0, s[0:1]1121; GFX11-NEXT: s_waitcnt vmcnt(0)1122; GFX11-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v01123; GFX11-NEXT: global_store_b16 v[0:1], v0, off dlc1124; GFX11-NEXT: s_waitcnt_vscnt null, 0x01125; GFX11-NEXT: global_store_d16_hi_b16 v[0:1], v0, off dlc1126; GFX11-NEXT: s_waitcnt_vscnt null, 0x01127; GFX11-NEXT: s_endpgm1128 %tid = call i32 @llvm.amdgcn.workitem.id.x()1129 %gep.in = getelementptr inbounds <2 x bfloat>, ptr addrspace(1) %in, i32 %tid1130 %val = load <2 x bfloat>, ptr addrspace(1) %gep.in1131 %fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %val)1132 %elt0 = extractelement <2 x bfloat> %fabs, i32 01133 %elt1 = extractelement <2 x bfloat> %fabs, i32 11134 store volatile bfloat %elt0, ptr addrspace(1) poison1135 store volatile bfloat %elt1, ptr addrspace(1) poison1136 ret void1137}1138 1139declare bfloat @llvm.fabs.bf16(bfloat) #11140declare <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat>) #11141declare <4 x bfloat> @llvm.fabs.v4bf16(<4 x bfloat>) #11142declare i32 @llvm.amdgcn.workitem.id.x() #11143 1144attributes #0 = { nounwind }1145attributes #1 = { nounwind readnone }1146 1147