918 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI,SIVI %s3; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=fiji -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,SIVI,VIGFX9 %s4; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx900 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,VIGFX9 %s5 6declare half @llvm.fma.f16(half %a, half %b, half %c)7declare <2 x half> @llvm.fma.v2f16(<2 x half> %a, <2 x half> %b, <2 x half> %c)8declare <4 x half> @llvm.fma.v4f16(<4 x half> %a, <4 x half> %b, <4 x half> %c)9 10define amdgpu_kernel void @fma_f16(11; SI-LABEL: fma_f16:12; SI: ; %bb.0:13; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x914; SI-NEXT: s_mov_b32 s11, 0xf00015; SI-NEXT: s_mov_b32 s10, -116; SI-NEXT: s_mov_b32 s14, s1017; SI-NEXT: s_mov_b32 s15, s1118; SI-NEXT: s_waitcnt lgkmcnt(0)19; SI-NEXT: s_mov_b32 s12, s220; SI-NEXT: s_mov_b32 s13, s321; SI-NEXT: s_mov_b32 s16, s422; SI-NEXT: s_mov_b32 s17, s523; SI-NEXT: s_mov_b32 s18, s1024; SI-NEXT: s_mov_b32 s19, s1125; SI-NEXT: s_mov_b32 s4, s626; SI-NEXT: s_mov_b32 s5, s727; SI-NEXT: s_mov_b32 s6, s1028; SI-NEXT: s_mov_b32 s7, s1129; SI-NEXT: buffer_load_ushort v0, off, s[12:15], 030; SI-NEXT: buffer_load_ushort v1, off, s[16:19], 031; SI-NEXT: buffer_load_ushort v2, off, s[4:7], 032; SI-NEXT: s_mov_b32 s8, s033; SI-NEXT: s_mov_b32 s9, s134; SI-NEXT: s_waitcnt vmcnt(2)35; SI-NEXT: v_cvt_f32_f16_e32 v0, v036; SI-NEXT: s_waitcnt vmcnt(1)37; SI-NEXT: v_cvt_f32_f16_e32 v1, v138; SI-NEXT: s_waitcnt vmcnt(0)39; SI-NEXT: v_cvt_f32_f16_e32 v2, v240; SI-NEXT: v_fma_f32 v0, v0, v1, v241; SI-NEXT: v_cvt_f16_f32_e32 v0, v042; SI-NEXT: buffer_store_short v0, off, s[8:11], 043; SI-NEXT: s_endpgm44;45; VI-LABEL: fma_f16:46; VI: ; %bb.0:47; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x2448; VI-NEXT: s_mov_b32 s11, 0xf00049; VI-NEXT: s_mov_b32 s10, -150; VI-NEXT: s_mov_b32 s14, s1051; VI-NEXT: s_mov_b32 s15, s1152; VI-NEXT: s_waitcnt lgkmcnt(0)53; VI-NEXT: s_mov_b32 s12, s254; VI-NEXT: s_mov_b32 s13, s355; VI-NEXT: s_mov_b32 s16, s456; VI-NEXT: s_mov_b32 s17, s557; VI-NEXT: s_mov_b32 s18, s1058; VI-NEXT: s_mov_b32 s19, s1159; VI-NEXT: s_mov_b32 s4, s660; VI-NEXT: s_mov_b32 s5, s761; VI-NEXT: s_mov_b32 s6, s1062; VI-NEXT: s_mov_b32 s7, s1163; VI-NEXT: buffer_load_ushort v0, off, s[12:15], 064; VI-NEXT: buffer_load_ushort v1, off, s[16:19], 065; VI-NEXT: buffer_load_ushort v2, off, s[4:7], 066; VI-NEXT: s_mov_b32 s8, s067; VI-NEXT: s_mov_b32 s9, s168; VI-NEXT: s_waitcnt vmcnt(0)69; VI-NEXT: v_fma_f16 v0, v0, v1, v270; VI-NEXT: buffer_store_short v0, off, s[8:11], 071; VI-NEXT: s_endpgm72;73; GFX9-LABEL: fma_f16:74; GFX9: ; %bb.0:75; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2476; GFX9-NEXT: s_mov_b32 s3, 0xf00077; GFX9-NEXT: s_mov_b32 s2, -178; GFX9-NEXT: s_mov_b32 s6, s279; GFX9-NEXT: s_mov_b32 s7, s380; GFX9-NEXT: s_waitcnt lgkmcnt(0)81; GFX9-NEXT: s_mov_b32 s4, s1082; GFX9-NEXT: s_mov_b32 s5, s1183; GFX9-NEXT: s_mov_b32 s16, s1284; GFX9-NEXT: s_mov_b32 s17, s1385; GFX9-NEXT: s_mov_b32 s18, s286; GFX9-NEXT: s_mov_b32 s19, s387; GFX9-NEXT: s_mov_b32 s12, s1488; GFX9-NEXT: s_mov_b32 s13, s1589; GFX9-NEXT: s_mov_b32 s14, s290; GFX9-NEXT: s_mov_b32 s15, s391; GFX9-NEXT: buffer_load_ushort v0, off, s[4:7], 092; GFX9-NEXT: buffer_load_ushort v1, off, s[16:19], 093; GFX9-NEXT: buffer_load_ushort v2, off, s[12:15], 094; GFX9-NEXT: s_mov_b32 s0, s895; GFX9-NEXT: s_mov_b32 s1, s996; GFX9-NEXT: s_waitcnt vmcnt(0)97; GFX9-NEXT: v_fma_f16 v0, v0, v1, v298; GFX9-NEXT: buffer_store_short v0, off, s[0:3], 099; GFX9-NEXT: s_endpgm100 ptr addrspace(1) %r,101 ptr addrspace(1) %a,102 ptr addrspace(1) %b,103 ptr addrspace(1) %c) {104 %a.val = load half, ptr addrspace(1) %a105 %b.val = load half, ptr addrspace(1) %b106 %c.val = load half, ptr addrspace(1) %c107 %r.val = call half @llvm.fma.f16(half %a.val, half %b.val, half %c.val)108 store half %r.val, ptr addrspace(1) %r109 ret void110}111 112define amdgpu_kernel void @fma_f16_imm_a(113; SI-LABEL: fma_f16_imm_a:114; SI: ; %bb.0:115; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9116; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd117; SI-NEXT: s_mov_b32 s7, 0xf000118; SI-NEXT: s_mov_b32 s6, -1119; SI-NEXT: s_mov_b32 s14, s6120; SI-NEXT: s_waitcnt lgkmcnt(0)121; SI-NEXT: s_mov_b32 s12, s2122; SI-NEXT: s_mov_b32 s13, s3123; SI-NEXT: s_mov_b32 s15, s7124; SI-NEXT: s_mov_b32 s10, s6125; SI-NEXT: s_mov_b32 s11, s7126; SI-NEXT: buffer_load_ushort v0, off, s[12:15], 0127; SI-NEXT: buffer_load_ushort v1, off, s[8:11], 0128; SI-NEXT: s_mov_b32 s2, 0x40400000129; SI-NEXT: s_mov_b32 s4, s0130; SI-NEXT: s_mov_b32 s5, s1131; SI-NEXT: s_waitcnt vmcnt(1)132; SI-NEXT: v_cvt_f32_f16_e32 v0, v0133; SI-NEXT: s_waitcnt vmcnt(0)134; SI-NEXT: v_cvt_f32_f16_e32 v1, v1135; SI-NEXT: v_fma_f32 v0, v0, s2, v1136; SI-NEXT: v_cvt_f16_f32_e32 v0, v0137; SI-NEXT: buffer_store_short v0, off, s[4:7], 0138; SI-NEXT: s_endpgm139;140; VI-LABEL: fma_f16_imm_a:141; VI: ; %bb.0:142; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24143; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34144; VI-NEXT: s_mov_b32 s7, 0xf000145; VI-NEXT: s_mov_b32 s6, -1146; VI-NEXT: s_mov_b32 s14, s6147; VI-NEXT: s_waitcnt lgkmcnt(0)148; VI-NEXT: s_mov_b32 s12, s2149; VI-NEXT: s_mov_b32 s13, s3150; VI-NEXT: s_mov_b32 s15, s7151; VI-NEXT: s_mov_b32 s10, s6152; VI-NEXT: s_mov_b32 s11, s7153; VI-NEXT: buffer_load_ushort v0, off, s[12:15], 0154; VI-NEXT: buffer_load_ushort v1, off, s[8:11], 0155; VI-NEXT: s_mov_b32 s4, s0156; VI-NEXT: s_movk_i32 s0, 0x4200157; VI-NEXT: s_mov_b32 s5, s1158; VI-NEXT: s_waitcnt vmcnt(0)159; VI-NEXT: v_fma_f16 v0, v0, s0, v1160; VI-NEXT: buffer_store_short v0, off, s[4:7], 0161; VI-NEXT: s_endpgm162;163; GFX9-LABEL: fma_f16_imm_a:164; GFX9: ; %bb.0:165; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24166; GFX9-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34167; GFX9-NEXT: s_mov_b32 s7, 0xf000168; GFX9-NEXT: s_mov_b32 s6, -1169; GFX9-NEXT: s_mov_b32 s14, s6170; GFX9-NEXT: s_waitcnt lgkmcnt(0)171; GFX9-NEXT: s_mov_b32 s12, s2172; GFX9-NEXT: s_mov_b32 s13, s3173; GFX9-NEXT: s_mov_b32 s15, s7174; GFX9-NEXT: s_mov_b32 s10, s6175; GFX9-NEXT: s_mov_b32 s11, s7176; GFX9-NEXT: buffer_load_ushort v0, off, s[12:15], 0177; GFX9-NEXT: buffer_load_ushort v1, off, s[8:11], 0178; GFX9-NEXT: s_mov_b32 s4, s0179; GFX9-NEXT: s_movk_i32 s0, 0x4200180; GFX9-NEXT: s_mov_b32 s5, s1181; GFX9-NEXT: s_waitcnt vmcnt(0)182; GFX9-NEXT: v_fma_f16 v0, v0, s0, v1183; GFX9-NEXT: buffer_store_short v0, off, s[4:7], 0184; GFX9-NEXT: s_endpgm185 ptr addrspace(1) %r,186 ptr addrspace(1) %b,187 ptr addrspace(1) %c) {188 %b.val = load half, ptr addrspace(1) %b189 %c.val = load half, ptr addrspace(1) %c190 %r.val = call half @llvm.fma.f16(half 3.0, half %b.val, half %c.val)191 store half %r.val, ptr addrspace(1) %r192 ret void193}194 195define amdgpu_kernel void @fma_f16_imm_b(196; SI-LABEL: fma_f16_imm_b:197; SI: ; %bb.0:198; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9199; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd200; SI-NEXT: s_mov_b32 s7, 0xf000201; SI-NEXT: s_mov_b32 s6, -1202; SI-NEXT: s_mov_b32 s14, s6203; SI-NEXT: s_waitcnt lgkmcnt(0)204; SI-NEXT: s_mov_b32 s12, s2205; SI-NEXT: s_mov_b32 s13, s3206; SI-NEXT: s_mov_b32 s15, s7207; SI-NEXT: s_mov_b32 s10, s6208; SI-NEXT: s_mov_b32 s11, s7209; SI-NEXT: buffer_load_ushort v0, off, s[12:15], 0210; SI-NEXT: buffer_load_ushort v1, off, s[8:11], 0211; SI-NEXT: s_mov_b32 s2, 0x40400000212; SI-NEXT: s_mov_b32 s4, s0213; SI-NEXT: s_mov_b32 s5, s1214; SI-NEXT: s_waitcnt vmcnt(1)215; SI-NEXT: v_cvt_f32_f16_e32 v0, v0216; SI-NEXT: s_waitcnt vmcnt(0)217; SI-NEXT: v_cvt_f32_f16_e32 v1, v1218; SI-NEXT: v_fma_f32 v0, v0, s2, v1219; SI-NEXT: v_cvt_f16_f32_e32 v0, v0220; SI-NEXT: buffer_store_short v0, off, s[4:7], 0221; SI-NEXT: s_endpgm222;223; VI-LABEL: fma_f16_imm_b:224; VI: ; %bb.0:225; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24226; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34227; VI-NEXT: s_mov_b32 s7, 0xf000228; VI-NEXT: s_mov_b32 s6, -1229; VI-NEXT: s_mov_b32 s14, s6230; VI-NEXT: s_waitcnt lgkmcnt(0)231; VI-NEXT: s_mov_b32 s12, s2232; VI-NEXT: s_mov_b32 s13, s3233; VI-NEXT: s_mov_b32 s15, s7234; VI-NEXT: s_mov_b32 s10, s6235; VI-NEXT: s_mov_b32 s11, s7236; VI-NEXT: buffer_load_ushort v0, off, s[12:15], 0237; VI-NEXT: buffer_load_ushort v1, off, s[8:11], 0238; VI-NEXT: s_mov_b32 s4, s0239; VI-NEXT: s_movk_i32 s0, 0x4200240; VI-NEXT: s_mov_b32 s5, s1241; VI-NEXT: s_waitcnt vmcnt(0)242; VI-NEXT: v_fma_f16 v0, v0, s0, v1243; VI-NEXT: buffer_store_short v0, off, s[4:7], 0244; VI-NEXT: s_endpgm245;246; GFX9-LABEL: fma_f16_imm_b:247; GFX9: ; %bb.0:248; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24249; GFX9-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34250; GFX9-NEXT: s_mov_b32 s7, 0xf000251; GFX9-NEXT: s_mov_b32 s6, -1252; GFX9-NEXT: s_mov_b32 s14, s6253; GFX9-NEXT: s_waitcnt lgkmcnt(0)254; GFX9-NEXT: s_mov_b32 s12, s2255; GFX9-NEXT: s_mov_b32 s13, s3256; GFX9-NEXT: s_mov_b32 s15, s7257; GFX9-NEXT: s_mov_b32 s10, s6258; GFX9-NEXT: s_mov_b32 s11, s7259; GFX9-NEXT: buffer_load_ushort v0, off, s[12:15], 0260; GFX9-NEXT: buffer_load_ushort v1, off, s[8:11], 0261; GFX9-NEXT: s_mov_b32 s4, s0262; GFX9-NEXT: s_movk_i32 s0, 0x4200263; GFX9-NEXT: s_mov_b32 s5, s1264; GFX9-NEXT: s_waitcnt vmcnt(0)265; GFX9-NEXT: v_fma_f16 v0, v0, s0, v1266; GFX9-NEXT: buffer_store_short v0, off, s[4:7], 0267; GFX9-NEXT: s_endpgm268 ptr addrspace(1) %r,269 ptr addrspace(1) %a,270 ptr addrspace(1) %c) {271 %a.val = load half, ptr addrspace(1) %a272 %c.val = load half, ptr addrspace(1) %c273 %r.val = call half @llvm.fma.f16(half %a.val, half 3.0, half %c.val)274 store half %r.val, ptr addrspace(1) %r275 ret void276}277 278define amdgpu_kernel void @fma_f16_imm_c(279; SI-LABEL: fma_f16_imm_c:280; SI: ; %bb.0:281; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9282; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd283; SI-NEXT: s_mov_b32 s7, 0xf000284; SI-NEXT: s_mov_b32 s6, -1285; SI-NEXT: s_mov_b32 s14, s6286; SI-NEXT: s_waitcnt lgkmcnt(0)287; SI-NEXT: s_mov_b32 s12, s2288; SI-NEXT: s_mov_b32 s13, s3289; SI-NEXT: s_mov_b32 s15, s7290; SI-NEXT: s_mov_b32 s10, s6291; SI-NEXT: s_mov_b32 s11, s7292; SI-NEXT: buffer_load_ushort v0, off, s[12:15], 0293; SI-NEXT: buffer_load_ushort v1, off, s[8:11], 0294; SI-NEXT: s_mov_b32 s2, 0x40400000295; SI-NEXT: s_mov_b32 s4, s0296; SI-NEXT: s_mov_b32 s5, s1297; SI-NEXT: s_waitcnt vmcnt(1)298; SI-NEXT: v_cvt_f32_f16_e32 v0, v0299; SI-NEXT: s_waitcnt vmcnt(0)300; SI-NEXT: v_cvt_f32_f16_e32 v1, v1301; SI-NEXT: v_fma_f32 v0, v0, v1, s2302; SI-NEXT: v_cvt_f16_f32_e32 v0, v0303; SI-NEXT: buffer_store_short v0, off, s[4:7], 0304; SI-NEXT: s_endpgm305;306; VI-LABEL: fma_f16_imm_c:307; VI: ; %bb.0:308; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24309; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34310; VI-NEXT: s_mov_b32 s7, 0xf000311; VI-NEXT: s_mov_b32 s6, -1312; VI-NEXT: s_mov_b32 s14, s6313; VI-NEXT: s_waitcnt lgkmcnt(0)314; VI-NEXT: s_mov_b32 s12, s2315; VI-NEXT: s_mov_b32 s13, s3316; VI-NEXT: s_mov_b32 s15, s7317; VI-NEXT: s_mov_b32 s10, s6318; VI-NEXT: s_mov_b32 s11, s7319; VI-NEXT: buffer_load_ushort v0, off, s[12:15], 0320; VI-NEXT: buffer_load_ushort v1, off, s[8:11], 0321; VI-NEXT: s_mov_b32 s4, s0322; VI-NEXT: s_movk_i32 s0, 0x4200323; VI-NEXT: s_mov_b32 s5, s1324; VI-NEXT: s_waitcnt vmcnt(0)325; VI-NEXT: v_fma_f16 v0, v0, v1, s0326; VI-NEXT: buffer_store_short v0, off, s[4:7], 0327; VI-NEXT: s_endpgm328;329; GFX9-LABEL: fma_f16_imm_c:330; GFX9: ; %bb.0:331; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24332; GFX9-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34333; GFX9-NEXT: s_mov_b32 s7, 0xf000334; GFX9-NEXT: s_mov_b32 s6, -1335; GFX9-NEXT: s_mov_b32 s14, s6336; GFX9-NEXT: s_waitcnt lgkmcnt(0)337; GFX9-NEXT: s_mov_b32 s12, s2338; GFX9-NEXT: s_mov_b32 s13, s3339; GFX9-NEXT: s_mov_b32 s15, s7340; GFX9-NEXT: s_mov_b32 s10, s6341; GFX9-NEXT: s_mov_b32 s11, s7342; GFX9-NEXT: buffer_load_ushort v0, off, s[12:15], 0343; GFX9-NEXT: buffer_load_ushort v1, off, s[8:11], 0344; GFX9-NEXT: s_mov_b32 s4, s0345; GFX9-NEXT: s_movk_i32 s0, 0x4200346; GFX9-NEXT: s_mov_b32 s5, s1347; GFX9-NEXT: s_waitcnt vmcnt(0)348; GFX9-NEXT: v_fma_f16 v0, v0, v1, s0349; GFX9-NEXT: buffer_store_short v0, off, s[4:7], 0350; GFX9-NEXT: s_endpgm351 ptr addrspace(1) %r,352 ptr addrspace(1) %a,353 ptr addrspace(1) %b) {354 %a.val = load half, ptr addrspace(1) %a355 %b.val = load half, ptr addrspace(1) %b356 %r.val = call half @llvm.fma.f16(half %a.val, half %b.val, half 3.0)357 store half %r.val, ptr addrspace(1) %r358 ret void359}360 361define amdgpu_kernel void @fma_v2f16(362; SI-LABEL: fma_v2f16:363; SI: ; %bb.0:364; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9365; SI-NEXT: s_mov_b32 s11, 0xf000366; SI-NEXT: s_mov_b32 s10, -1367; SI-NEXT: s_mov_b32 s14, s10368; SI-NEXT: s_mov_b32 s15, s11369; SI-NEXT: s_waitcnt lgkmcnt(0)370; SI-NEXT: s_mov_b32 s12, s2371; SI-NEXT: s_mov_b32 s13, s3372; SI-NEXT: s_mov_b32 s16, s4373; SI-NEXT: s_mov_b32 s17, s5374; SI-NEXT: s_mov_b32 s18, s10375; SI-NEXT: s_mov_b32 s19, s11376; SI-NEXT: s_mov_b32 s4, s6377; SI-NEXT: s_mov_b32 s5, s7378; SI-NEXT: s_mov_b32 s6, s10379; SI-NEXT: buffer_load_dword v0, off, s[12:15], 0380; SI-NEXT: s_mov_b32 s7, s11381; SI-NEXT: buffer_load_dword v1, off, s[16:19], 0382; SI-NEXT: buffer_load_dword v2, off, s[4:7], 0383; SI-NEXT: s_mov_b32 s8, s0384; SI-NEXT: s_mov_b32 s9, s1385; SI-NEXT: s_waitcnt vmcnt(2)386; SI-NEXT: v_cvt_f32_f16_e32 v3, v0387; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0388; SI-NEXT: s_waitcnt vmcnt(1)389; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v1390; SI-NEXT: s_waitcnt vmcnt(0)391; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v2392; SI-NEXT: v_cvt_f32_f16_e32 v0, v0393; SI-NEXT: v_cvt_f32_f16_e32 v4, v4394; SI-NEXT: v_cvt_f32_f16_e32 v5, v5395; SI-NEXT: v_cvt_f32_f16_e32 v1, v1396; SI-NEXT: v_cvt_f32_f16_e32 v2, v2397; SI-NEXT: v_fma_f32 v0, v0, v4, v5398; SI-NEXT: v_cvt_f16_f32_e32 v0, v0399; SI-NEXT: v_fma_f32 v1, v3, v1, v2400; SI-NEXT: v_cvt_f16_f32_e32 v1, v1401; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0402; SI-NEXT: v_or_b32_e32 v0, v1, v0403; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0404; SI-NEXT: s_endpgm405;406; VI-LABEL: fma_v2f16:407; VI: ; %bb.0:408; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24409; VI-NEXT: s_mov_b32 s11, 0xf000410; VI-NEXT: s_mov_b32 s10, -1411; VI-NEXT: s_mov_b32 s14, s10412; VI-NEXT: s_mov_b32 s15, s11413; VI-NEXT: s_waitcnt lgkmcnt(0)414; VI-NEXT: s_mov_b32 s16, s4415; VI-NEXT: s_mov_b32 s17, s5416; VI-NEXT: s_mov_b32 s4, s6417; VI-NEXT: s_mov_b32 s5, s7418; VI-NEXT: s_mov_b32 s6, s10419; VI-NEXT: s_mov_b32 s7, s11420; VI-NEXT: s_mov_b32 s12, s2421; VI-NEXT: s_mov_b32 s13, s3422; VI-NEXT: s_mov_b32 s18, s10423; VI-NEXT: s_mov_b32 s19, s11424; VI-NEXT: buffer_load_dword v0, off, s[4:7], 0425; VI-NEXT: buffer_load_dword v1, off, s[16:19], 0426; VI-NEXT: buffer_load_dword v2, off, s[12:15], 0427; VI-NEXT: s_mov_b32 s8, s0428; VI-NEXT: s_mov_b32 s9, s1429; VI-NEXT: s_waitcnt vmcnt(2)430; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v0431; VI-NEXT: s_waitcnt vmcnt(1)432; VI-NEXT: v_lshrrev_b32_e32 v4, 16, v1433; VI-NEXT: s_waitcnt vmcnt(0)434; VI-NEXT: v_lshrrev_b32_e32 v5, 16, v2435; VI-NEXT: v_fma_f16 v3, v5, v4, v3436; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3437; VI-NEXT: v_fma_f16 v0, v2, v1, v0438; VI-NEXT: v_or_b32_e32 v0, v0, v3439; VI-NEXT: buffer_store_dword v0, off, s[8:11], 0440; VI-NEXT: s_endpgm441;442; GFX9-LABEL: fma_v2f16:443; GFX9: ; %bb.0:444; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24445; GFX9-NEXT: s_mov_b32 s3, 0xf000446; GFX9-NEXT: s_mov_b32 s2, -1447; GFX9-NEXT: s_mov_b32 s6, s2448; GFX9-NEXT: s_mov_b32 s7, s3449; GFX9-NEXT: s_waitcnt lgkmcnt(0)450; GFX9-NEXT: s_mov_b32 s4, s10451; GFX9-NEXT: s_mov_b32 s5, s11452; GFX9-NEXT: s_mov_b32 s16, s12453; GFX9-NEXT: s_mov_b32 s17, s13454; GFX9-NEXT: s_mov_b32 s18, s2455; GFX9-NEXT: s_mov_b32 s19, s3456; GFX9-NEXT: s_mov_b32 s12, s14457; GFX9-NEXT: s_mov_b32 s13, s15458; GFX9-NEXT: s_mov_b32 s14, s2459; GFX9-NEXT: s_mov_b32 s15, s3460; GFX9-NEXT: buffer_load_dword v0, off, s[4:7], 0461; GFX9-NEXT: buffer_load_dword v1, off, s[16:19], 0462; GFX9-NEXT: buffer_load_dword v2, off, s[12:15], 0463; GFX9-NEXT: s_mov_b32 s0, s8464; GFX9-NEXT: s_mov_b32 s1, s9465; GFX9-NEXT: s_waitcnt vmcnt(0)466; GFX9-NEXT: v_pk_fma_f16 v0, v0, v1, v2467; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0468; GFX9-NEXT: s_endpgm469 ptr addrspace(1) %r,470 ptr addrspace(1) %a,471 ptr addrspace(1) %b,472 ptr addrspace(1) %c) {473 %a.val = load <2 x half>, ptr addrspace(1) %a474 %b.val = load <2 x half>, ptr addrspace(1) %b475 %c.val = load <2 x half>, ptr addrspace(1) %c476 %r.val = call <2 x half> @llvm.fma.v2f16(<2 x half> %a.val, <2 x half> %b.val, <2 x half> %c.val)477 store <2 x half> %r.val, ptr addrspace(1) %r478 ret void479}480 481define amdgpu_kernel void @fma_v2f16_imm_a(482; SI-LABEL: fma_v2f16_imm_a:483; SI: ; %bb.0:484; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9485; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd486; SI-NEXT: s_mov_b32 s7, 0xf000487; SI-NEXT: s_mov_b32 s6, -1488; SI-NEXT: s_mov_b32 s10, s6489; SI-NEXT: s_mov_b32 s11, s7490; SI-NEXT: s_waitcnt lgkmcnt(0)491; SI-NEXT: s_mov_b32 s12, s2492; SI-NEXT: s_mov_b32 s13, s3493; SI-NEXT: s_mov_b32 s14, s6494; SI-NEXT: s_mov_b32 s15, s7495; SI-NEXT: buffer_load_dword v0, off, s[8:11], 0496; SI-NEXT: buffer_load_dword v1, off, s[12:15], 0497; SI-NEXT: s_mov_b32 s2, 0x40400000498; SI-NEXT: s_mov_b32 s4, s0499; SI-NEXT: s_mov_b32 s5, s1500; SI-NEXT: s_waitcnt vmcnt(1)501; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0502; SI-NEXT: s_waitcnt vmcnt(0)503; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1504; SI-NEXT: v_cvt_f32_f16_e32 v2, v2505; SI-NEXT: v_cvt_f32_f16_e32 v3, v3506; SI-NEXT: v_cvt_f32_f16_e32 v0, v0507; SI-NEXT: v_cvt_f32_f16_e32 v1, v1508; SI-NEXT: v_fma_f32 v2, v3, s2, v2509; SI-NEXT: v_cvt_f16_f32_e32 v2, v2510; SI-NEXT: v_fma_f32 v0, v1, s2, v0511; SI-NEXT: v_cvt_f16_f32_e32 v0, v0512; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2513; SI-NEXT: v_or_b32_e32 v0, v0, v1514; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0515; SI-NEXT: s_endpgm516;517; VI-LABEL: fma_v2f16_imm_a:518; VI: ; %bb.0:519; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24520; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34521; VI-NEXT: s_mov_b32 s7, 0xf000522; VI-NEXT: s_mov_b32 s6, -1523; VI-NEXT: s_mov_b32 s10, s6524; VI-NEXT: s_mov_b32 s11, s7525; VI-NEXT: s_waitcnt lgkmcnt(0)526; VI-NEXT: s_mov_b32 s12, s2527; VI-NEXT: s_mov_b32 s13, s3528; VI-NEXT: s_mov_b32 s14, s6529; VI-NEXT: s_mov_b32 s15, s7530; VI-NEXT: buffer_load_dword v0, off, s[8:11], 0531; VI-NEXT: buffer_load_dword v1, off, s[12:15], 0532; VI-NEXT: s_movk_i32 s2, 0x4200533; VI-NEXT: s_mov_b32 s4, s0534; VI-NEXT: s_mov_b32 s5, s1535; VI-NEXT: s_waitcnt vmcnt(1)536; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v0537; VI-NEXT: s_waitcnt vmcnt(0)538; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v1539; VI-NEXT: v_fma_f16 v2, v3, s2, v2540; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2541; VI-NEXT: v_fma_f16 v0, v1, s2, v0542; VI-NEXT: v_or_b32_e32 v0, v0, v2543; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0544; VI-NEXT: s_endpgm545;546; GFX9-LABEL: fma_v2f16_imm_a:547; GFX9: ; %bb.0:548; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24549; GFX9-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34550; GFX9-NEXT: s_mov_b32 s7, 0xf000551; GFX9-NEXT: s_mov_b32 s6, -1552; GFX9-NEXT: s_mov_b32 s14, s6553; GFX9-NEXT: s_waitcnt lgkmcnt(0)554; GFX9-NEXT: s_mov_b32 s12, s2555; GFX9-NEXT: s_mov_b32 s13, s3556; GFX9-NEXT: s_mov_b32 s15, s7557; GFX9-NEXT: s_mov_b32 s10, s6558; GFX9-NEXT: s_mov_b32 s11, s7559; GFX9-NEXT: buffer_load_dword v0, off, s[12:15], 0560; GFX9-NEXT: buffer_load_dword v1, off, s[8:11], 0561; GFX9-NEXT: s_mov_b32 s4, s0562; GFX9-NEXT: s_movk_i32 s0, 0x4200563; GFX9-NEXT: s_mov_b32 s5, s1564; GFX9-NEXT: s_waitcnt vmcnt(0)565; GFX9-NEXT: v_pk_fma_f16 v0, v0, s0, v1 op_sel_hi:[1,0,1]566; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0567; GFX9-NEXT: s_endpgm568 ptr addrspace(1) %r,569 ptr addrspace(1) %b,570 ptr addrspace(1) %c) {571 %b.val = load <2 x half>, ptr addrspace(1) %b572 %c.val = load <2 x half>, ptr addrspace(1) %c573 %r.val = call <2 x half> @llvm.fma.v2f16(<2 x half> <half 3.0, half 3.0>, <2 x half> %b.val, <2 x half> %c.val)574 store <2 x half> %r.val, ptr addrspace(1) %r575 ret void576}577 578define amdgpu_kernel void @fma_v2f16_imm_b(579; SI-LABEL: fma_v2f16_imm_b:580; SI: ; %bb.0:581; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9582; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd583; SI-NEXT: s_mov_b32 s7, 0xf000584; SI-NEXT: s_mov_b32 s6, -1585; SI-NEXT: s_mov_b32 s10, s6586; SI-NEXT: s_mov_b32 s11, s7587; SI-NEXT: s_waitcnt lgkmcnt(0)588; SI-NEXT: s_mov_b32 s12, s2589; SI-NEXT: s_mov_b32 s13, s3590; SI-NEXT: s_mov_b32 s14, s6591; SI-NEXT: s_mov_b32 s15, s7592; SI-NEXT: buffer_load_dword v0, off, s[8:11], 0593; SI-NEXT: buffer_load_dword v1, off, s[12:15], 0594; SI-NEXT: s_mov_b32 s2, 0x40400000595; SI-NEXT: s_mov_b32 s4, s0596; SI-NEXT: s_mov_b32 s5, s1597; SI-NEXT: s_waitcnt vmcnt(1)598; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0599; SI-NEXT: s_waitcnt vmcnt(0)600; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1601; SI-NEXT: v_cvt_f32_f16_e32 v2, v2602; SI-NEXT: v_cvt_f32_f16_e32 v3, v3603; SI-NEXT: v_cvt_f32_f16_e32 v0, v0604; SI-NEXT: v_cvt_f32_f16_e32 v1, v1605; SI-NEXT: v_fma_f32 v2, v3, s2, v2606; SI-NEXT: v_cvt_f16_f32_e32 v2, v2607; SI-NEXT: v_fma_f32 v0, v1, s2, v0608; SI-NEXT: v_cvt_f16_f32_e32 v0, v0609; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2610; SI-NEXT: v_or_b32_e32 v0, v0, v1611; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0612; SI-NEXT: s_endpgm613;614; VI-LABEL: fma_v2f16_imm_b:615; VI: ; %bb.0:616; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24617; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34618; VI-NEXT: s_mov_b32 s7, 0xf000619; VI-NEXT: s_mov_b32 s6, -1620; VI-NEXT: s_mov_b32 s10, s6621; VI-NEXT: s_mov_b32 s11, s7622; VI-NEXT: s_waitcnt lgkmcnt(0)623; VI-NEXT: s_mov_b32 s12, s2624; VI-NEXT: s_mov_b32 s13, s3625; VI-NEXT: s_mov_b32 s14, s6626; VI-NEXT: s_mov_b32 s15, s7627; VI-NEXT: buffer_load_dword v0, off, s[8:11], 0628; VI-NEXT: buffer_load_dword v1, off, s[12:15], 0629; VI-NEXT: s_movk_i32 s2, 0x4200630; VI-NEXT: s_mov_b32 s4, s0631; VI-NEXT: s_mov_b32 s5, s1632; VI-NEXT: s_waitcnt vmcnt(1)633; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v0634; VI-NEXT: s_waitcnt vmcnt(0)635; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v1636; VI-NEXT: v_fma_f16 v2, v3, s2, v2637; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2638; VI-NEXT: v_fma_f16 v0, v1, s2, v0639; VI-NEXT: v_or_b32_e32 v0, v0, v2640; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0641; VI-NEXT: s_endpgm642;643; GFX9-LABEL: fma_v2f16_imm_b:644; GFX9: ; %bb.0:645; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24646; GFX9-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34647; GFX9-NEXT: s_mov_b32 s7, 0xf000648; GFX9-NEXT: s_mov_b32 s6, -1649; GFX9-NEXT: s_mov_b32 s14, s6650; GFX9-NEXT: s_waitcnt lgkmcnt(0)651; GFX9-NEXT: s_mov_b32 s12, s2652; GFX9-NEXT: s_mov_b32 s13, s3653; GFX9-NEXT: s_mov_b32 s15, s7654; GFX9-NEXT: s_mov_b32 s10, s6655; GFX9-NEXT: s_mov_b32 s11, s7656; GFX9-NEXT: buffer_load_dword v0, off, s[12:15], 0657; GFX9-NEXT: buffer_load_dword v1, off, s[8:11], 0658; GFX9-NEXT: s_mov_b32 s4, s0659; GFX9-NEXT: s_movk_i32 s0, 0x4200660; GFX9-NEXT: s_mov_b32 s5, s1661; GFX9-NEXT: s_waitcnt vmcnt(0)662; GFX9-NEXT: v_pk_fma_f16 v0, v0, s0, v1 op_sel_hi:[1,0,1]663; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0664; GFX9-NEXT: s_endpgm665 ptr addrspace(1) %r,666 ptr addrspace(1) %a,667 ptr addrspace(1) %c) {668 %a.val = load <2 x half>, ptr addrspace(1) %a669 %c.val = load <2 x half>, ptr addrspace(1) %c670 %r.val = call <2 x half> @llvm.fma.v2f16(<2 x half> %a.val, <2 x half> <half 3.0, half 3.0>, <2 x half> %c.val)671 store <2 x half> %r.val, ptr addrspace(1) %r672 ret void673}674 675define amdgpu_kernel void @fma_v2f16_imm_c(676; SI-LABEL: fma_v2f16_imm_c:677; SI: ; %bb.0:678; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9679; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd680; SI-NEXT: s_mov_b32 s7, 0xf000681; SI-NEXT: s_mov_b32 s6, -1682; SI-NEXT: s_mov_b32 s10, s6683; SI-NEXT: s_mov_b32 s11, s7684; SI-NEXT: s_waitcnt lgkmcnt(0)685; SI-NEXT: s_mov_b32 s12, s2686; SI-NEXT: s_mov_b32 s13, s3687; SI-NEXT: s_mov_b32 s14, s6688; SI-NEXT: s_mov_b32 s15, s7689; SI-NEXT: buffer_load_dword v0, off, s[8:11], 0690; SI-NEXT: buffer_load_dword v1, off, s[12:15], 0691; SI-NEXT: s_mov_b32 s2, 0x40400000692; SI-NEXT: s_mov_b32 s4, s0693; SI-NEXT: s_mov_b32 s5, s1694; SI-NEXT: s_waitcnt vmcnt(1)695; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0696; SI-NEXT: s_waitcnt vmcnt(0)697; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1698; SI-NEXT: v_cvt_f32_f16_e32 v2, v2699; SI-NEXT: v_cvt_f32_f16_e32 v3, v3700; SI-NEXT: v_cvt_f32_f16_e32 v0, v0701; SI-NEXT: v_cvt_f32_f16_e32 v1, v1702; SI-NEXT: v_fma_f32 v2, v3, v2, s2703; SI-NEXT: v_cvt_f16_f32_e32 v2, v2704; SI-NEXT: v_fma_f32 v0, v1, v0, s2705; SI-NEXT: v_cvt_f16_f32_e32 v0, v0706; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v2707; SI-NEXT: v_or_b32_e32 v0, v0, v1708; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0709; SI-NEXT: s_endpgm710;711; VI-LABEL: fma_v2f16_imm_c:712; VI: ; %bb.0:713; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24714; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34715; VI-NEXT: s_mov_b32 s7, 0xf000716; VI-NEXT: s_mov_b32 s6, -1717; VI-NEXT: s_mov_b32 s10, s6718; VI-NEXT: s_mov_b32 s11, s7719; VI-NEXT: s_waitcnt lgkmcnt(0)720; VI-NEXT: s_mov_b32 s12, s2721; VI-NEXT: s_mov_b32 s13, s3722; VI-NEXT: s_mov_b32 s14, s6723; VI-NEXT: s_mov_b32 s15, s7724; VI-NEXT: buffer_load_dword v0, off, s[8:11], 0725; VI-NEXT: buffer_load_dword v1, off, s[12:15], 0726; VI-NEXT: s_movk_i32 s2, 0x4200727; VI-NEXT: s_mov_b32 s4, s0728; VI-NEXT: s_mov_b32 s5, s1729; VI-NEXT: s_waitcnt vmcnt(1)730; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v0731; VI-NEXT: s_waitcnt vmcnt(0)732; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v1733; VI-NEXT: v_fma_f16 v2, v3, v2, s2734; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2735; VI-NEXT: v_fma_f16 v0, v1, v0, s2736; VI-NEXT: v_or_b32_e32 v0, v0, v2737; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0738; VI-NEXT: s_endpgm739;740; GFX9-LABEL: fma_v2f16_imm_c:741; GFX9: ; %bb.0:742; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24743; GFX9-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34744; GFX9-NEXT: s_mov_b32 s7, 0xf000745; GFX9-NEXT: s_mov_b32 s6, -1746; GFX9-NEXT: s_mov_b32 s14, s6747; GFX9-NEXT: s_waitcnt lgkmcnt(0)748; GFX9-NEXT: s_mov_b32 s12, s2749; GFX9-NEXT: s_mov_b32 s13, s3750; GFX9-NEXT: s_mov_b32 s15, s7751; GFX9-NEXT: s_mov_b32 s10, s6752; GFX9-NEXT: s_mov_b32 s11, s7753; GFX9-NEXT: buffer_load_dword v0, off, s[12:15], 0754; GFX9-NEXT: buffer_load_dword v1, off, s[8:11], 0755; GFX9-NEXT: s_mov_b32 s4, s0756; GFX9-NEXT: s_movk_i32 s0, 0x4200757; GFX9-NEXT: s_mov_b32 s5, s1758; GFX9-NEXT: s_waitcnt vmcnt(0)759; GFX9-NEXT: v_pk_fma_f16 v0, v0, v1, s0 op_sel_hi:[1,1,0]760; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0761; GFX9-NEXT: s_endpgm762 ptr addrspace(1) %r,763 ptr addrspace(1) %a,764 ptr addrspace(1) %b) {765 %a.val = load <2 x half>, ptr addrspace(1) %a766 %b.val = load <2 x half>, ptr addrspace(1) %b767 %r.val = call <2 x half> @llvm.fma.v2f16(<2 x half> %a.val, <2 x half> %b.val, <2 x half> <half 3.0, half 3.0>)768 store <2 x half> %r.val, ptr addrspace(1) %r769 ret void770}771 772define amdgpu_kernel void @fma_v4f16(773; SI-LABEL: fma_v4f16:774; SI: ; %bb.0:775; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9776; SI-NEXT: s_mov_b32 s11, 0xf000777; SI-NEXT: s_mov_b32 s10, -1778; SI-NEXT: s_mov_b32 s14, s10779; SI-NEXT: s_mov_b32 s15, s11780; SI-NEXT: s_waitcnt lgkmcnt(0)781; SI-NEXT: s_mov_b32 s12, s2782; SI-NEXT: s_mov_b32 s13, s3783; SI-NEXT: s_mov_b32 s16, s4784; SI-NEXT: s_mov_b32 s17, s5785; SI-NEXT: s_mov_b32 s18, s10786; SI-NEXT: s_mov_b32 s19, s11787; SI-NEXT: s_mov_b32 s4, s6788; SI-NEXT: s_mov_b32 s5, s7789; SI-NEXT: s_mov_b32 s6, s10790; SI-NEXT: s_mov_b32 s7, s11791; SI-NEXT: buffer_load_dwordx2 v[0:1], off, s[12:15], 0792; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[16:19], 0793; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0794; SI-NEXT: s_mov_b32 s8, s0795; SI-NEXT: s_mov_b32 s9, s1796; SI-NEXT: s_waitcnt vmcnt(2)797; SI-NEXT: v_cvt_f32_f16_e32 v6, v0798; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0799; SI-NEXT: v_cvt_f32_f16_e32 v7, v1800; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1801; SI-NEXT: s_waitcnt vmcnt(1)802; SI-NEXT: v_cvt_f32_f16_e32 v8, v2803; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2804; SI-NEXT: v_cvt_f32_f16_e32 v9, v3805; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v3806; SI-NEXT: s_waitcnt vmcnt(0)807; SI-NEXT: v_cvt_f32_f16_e32 v10, v4808; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v4809; SI-NEXT: v_cvt_f32_f16_e32 v11, v5810; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v5811; SI-NEXT: v_cvt_f32_f16_e32 v0, v0812; SI-NEXT: v_cvt_f32_f16_e32 v1, v1813; SI-NEXT: v_cvt_f32_f16_e32 v2, v2814; SI-NEXT: v_cvt_f32_f16_e32 v3, v3815; SI-NEXT: v_cvt_f32_f16_e32 v5, v5816; SI-NEXT: v_cvt_f32_f16_e32 v4, v4817; SI-NEXT: v_fma_f32 v7, v7, v9, v11818; SI-NEXT: v_fma_f32 v6, v6, v8, v10819; SI-NEXT: v_fma_f32 v1, v1, v3, v5820; SI-NEXT: v_fma_f32 v0, v0, v2, v4821; SI-NEXT: v_cvt_f16_f32_e32 v1, v1822; SI-NEXT: v_cvt_f16_f32_e32 v0, v0823; SI-NEXT: v_cvt_f16_f32_e32 v2, v7824; SI-NEXT: v_cvt_f16_f32_e32 v3, v6825; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1826; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0827; SI-NEXT: v_or_b32_e32 v1, v2, v1828; SI-NEXT: v_or_b32_e32 v0, v3, v0829; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0830; SI-NEXT: s_endpgm831;832; VI-LABEL: fma_v4f16:833; VI: ; %bb.0:834; VI-NEXT: s_load_dwordx8 s[4:11], s[4:5], 0x24835; VI-NEXT: s_mov_b32 s3, 0xf000836; VI-NEXT: s_mov_b32 s2, -1837; VI-NEXT: s_mov_b32 s14, s2838; VI-NEXT: s_mov_b32 s15, s3839; VI-NEXT: s_waitcnt lgkmcnt(0)840; VI-NEXT: s_mov_b32 s16, s8841; VI-NEXT: s_mov_b32 s17, s9842; VI-NEXT: s_mov_b32 s8, s10843; VI-NEXT: s_mov_b32 s9, s11844; VI-NEXT: s_mov_b32 s10, s2845; VI-NEXT: s_mov_b32 s11, s3846; VI-NEXT: s_mov_b32 s12, s6847; VI-NEXT: s_mov_b32 s13, s7848; VI-NEXT: s_mov_b32 s18, s2849; VI-NEXT: s_mov_b32 s19, s3850; VI-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0851; VI-NEXT: buffer_load_dwordx2 v[2:3], off, s[16:19], 0852; VI-NEXT: buffer_load_dwordx2 v[4:5], off, s[12:15], 0853; VI-NEXT: s_mov_b32 s0, s4854; VI-NEXT: s_mov_b32 s1, s5855; VI-NEXT: s_waitcnt vmcnt(2)856; VI-NEXT: v_lshrrev_b32_e32 v6, 16, v1857; VI-NEXT: s_waitcnt vmcnt(1)858; VI-NEXT: v_lshrrev_b32_e32 v7, 16, v3859; VI-NEXT: s_waitcnt vmcnt(0)860; VI-NEXT: v_lshrrev_b32_e32 v8, 16, v5861; VI-NEXT: v_fma_f16 v1, v5, v3, v1862; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v0863; VI-NEXT: v_lshrrev_b32_e32 v5, 16, v2864; VI-NEXT: v_lshrrev_b32_e32 v9, 16, v4865; VI-NEXT: v_fma_f16 v0, v4, v2, v0866; VI-NEXT: v_fma_f16 v2, v8, v7, v6867; VI-NEXT: v_fma_f16 v3, v9, v5, v3868; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2869; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3870; VI-NEXT: v_or_b32_e32 v1, v1, v2871; VI-NEXT: v_or_b32_e32 v0, v0, v3872; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0873; VI-NEXT: s_endpgm874;875; GFX9-LABEL: fma_v4f16:876; GFX9: ; %bb.0:877; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24878; GFX9-NEXT: s_mov_b32 s3, 0xf000879; GFX9-NEXT: s_mov_b32 s2, -1880; GFX9-NEXT: s_mov_b32 s6, s2881; GFX9-NEXT: s_mov_b32 s7, s3882; GFX9-NEXT: s_waitcnt lgkmcnt(0)883; GFX9-NEXT: s_mov_b32 s4, s10884; GFX9-NEXT: s_mov_b32 s5, s11885; GFX9-NEXT: s_mov_b32 s16, s12886; GFX9-NEXT: s_mov_b32 s17, s13887; GFX9-NEXT: s_mov_b32 s18, s2888; GFX9-NEXT: s_mov_b32 s19, s3889; GFX9-NEXT: s_mov_b32 s12, s14890; GFX9-NEXT: s_mov_b32 s13, s15891; GFX9-NEXT: s_mov_b32 s14, s2892; GFX9-NEXT: s_mov_b32 s15, s3893; GFX9-NEXT: buffer_load_dwordx2 v[0:1], off, s[4:7], 0894; GFX9-NEXT: buffer_load_dwordx2 v[2:3], off, s[16:19], 0895; GFX9-NEXT: buffer_load_dwordx2 v[4:5], off, s[12:15], 0896; GFX9-NEXT: s_mov_b32 s0, s8897; GFX9-NEXT: s_mov_b32 s1, s9898; GFX9-NEXT: s_waitcnt vmcnt(0)899; GFX9-NEXT: v_pk_fma_f16 v1, v1, v3, v5900; GFX9-NEXT: v_pk_fma_f16 v0, v0, v2, v4901; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0902; GFX9-NEXT: s_endpgm903 ptr addrspace(1) %r,904 ptr addrspace(1) %a,905 ptr addrspace(1) %b,906 ptr addrspace(1) %c) {907 %a.val = load <4 x half>, ptr addrspace(1) %a908 %b.val = load <4 x half>, ptr addrspace(1) %b909 %c.val = load <4 x half>, ptr addrspace(1) %c910 %r.val = call <4 x half> @llvm.fma.v4f16(<4 x half> %a.val, <4 x half> %b.val, <4 x half> %c.val)911 store <4 x half> %r.val, ptr addrspace(1) %r912 ret void913}914;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:915; GCN: {{.*}}916; SIVI: {{.*}}917; VIGFX9: {{.*}}918