928 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -mattr=+fast-fmaf,+mad-mac-f32-insts -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-FLUSH %s3; RUN: llc -mtriple=amdgcn -mattr=-fast-fmaf,+mad-mac-f32-insts -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-FLUSH %s4 5; RUN: llc -mtriple=amdgcn -mattr=+fast-fmaf,+mad-mac-f32-insts -denormal-fp-math-f32=ieee < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-FASTFMA %s6; RUN: llc -mtriple=amdgcn -mattr=-fast-fmaf,+mad-mac-f32-insts -denormal-fp-math-f32=ieee < %s | FileCheck -check-prefix=GCN -check-prefix=GCN-SLOWFMA %s7 8; FIXME: This should also fold when fma is actually fast if an FMA9; exists in the original program.10 11; (fadd (fma x, y, (fmul u, v), z) -> (fma x, y (fma u, v, z))12define amdgpu_kernel void @fast_add_fmuladd_fmul() #0 {13; GCN-FLUSH-LABEL: fast_add_fmuladd_fmul:14; GCN-FLUSH: ; %bb.0:15; GCN-FLUSH-NEXT: s_mov_b32 s3, 0xf00016; GCN-FLUSH-NEXT: s_mov_b32 s2, -117; GCN-FLUSH-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc18; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)19; GCN-FLUSH-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc20; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)21; GCN-FLUSH-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc22; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)23; GCN-FLUSH-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc24; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)25; GCN-FLUSH-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc26; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)27; GCN-FLUSH-NEXT: v_mac_f32_e32 v2, v3, v428; GCN-FLUSH-NEXT: v_mac_f32_e32 v2, v0, v129; GCN-FLUSH-NEXT: buffer_store_dword v2, off, s[0:3], 030; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)31; GCN-FLUSH-NEXT: s_endpgm32;33; GCN-FASTFMA-LABEL: fast_add_fmuladd_fmul:34; GCN-FASTFMA: ; %bb.0:35; GCN-FASTFMA-NEXT: s_mov_b32 s3, 0xf00036; GCN-FASTFMA-NEXT: s_mov_b32 s2, -137; GCN-FASTFMA-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc38; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)39; GCN-FASTFMA-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc40; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)41; GCN-FASTFMA-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc42; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)43; GCN-FASTFMA-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc44; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)45; GCN-FASTFMA-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc46; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)47; GCN-FASTFMA-NEXT: v_fma_f32 v2, v3, v4, v248; GCN-FASTFMA-NEXT: v_fma_f32 v0, v0, v1, v249; GCN-FASTFMA-NEXT: buffer_store_dword v0, off, s[0:3], 050; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)51; GCN-FASTFMA-NEXT: s_endpgm52;53; GCN-SLOWFMA-LABEL: fast_add_fmuladd_fmul:54; GCN-SLOWFMA: ; %bb.0:55; GCN-SLOWFMA-NEXT: s_mov_b32 s3, 0xf00056; GCN-SLOWFMA-NEXT: s_mov_b32 s2, -157; GCN-SLOWFMA-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc58; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)59; GCN-SLOWFMA-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc60; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)61; GCN-SLOWFMA-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc62; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)63; GCN-SLOWFMA-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc64; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)65; GCN-SLOWFMA-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc66; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)67; GCN-SLOWFMA-NEXT: v_mul_f32_e32 v3, v3, v468; GCN-SLOWFMA-NEXT: v_mul_f32_e32 v0, v0, v169; GCN-SLOWFMA-NEXT: v_add_f32_e32 v0, v0, v370; GCN-SLOWFMA-NEXT: v_add_f32_e32 v0, v0, v271; GCN-SLOWFMA-NEXT: buffer_store_dword v0, off, s[0:3], 072; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)73; GCN-SLOWFMA-NEXT: s_endpgm74 %x = load volatile float, ptr addrspace(1) poison75 %y = load volatile float, ptr addrspace(1) poison76 %z = load volatile float, ptr addrspace(1) poison77 %u = load volatile float, ptr addrspace(1) poison78 %v = load volatile float, ptr addrspace(1) poison79 %mul.u.v = fmul fast float %u, %v80 %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v)81 %add = fadd fast float %fma, %z82 store volatile float %add, ptr addrspace(1) poison83 ret void84}85 86define amdgpu_kernel void @fast_sub_fmuladd_fmul() #0 {87; GCN-FLUSH-LABEL: fast_sub_fmuladd_fmul:88; GCN-FLUSH: ; %bb.0:89; GCN-FLUSH-NEXT: s_mov_b32 s3, 0xf00090; GCN-FLUSH-NEXT: s_mov_b32 s2, -191; GCN-FLUSH-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc92; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)93; GCN-FLUSH-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc94; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)95; GCN-FLUSH-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc96; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)97; GCN-FLUSH-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc98; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)99; GCN-FLUSH-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc100; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)101; GCN-FLUSH-NEXT: v_mad_f32 v2, v3, v4, -v2102; GCN-FLUSH-NEXT: v_mac_f32_e32 v2, v0, v1103; GCN-FLUSH-NEXT: buffer_store_dword v2, off, s[0:3], 0104; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)105; GCN-FLUSH-NEXT: s_endpgm106;107; GCN-FASTFMA-LABEL: fast_sub_fmuladd_fmul:108; GCN-FASTFMA: ; %bb.0:109; GCN-FASTFMA-NEXT: s_mov_b32 s3, 0xf000110; GCN-FASTFMA-NEXT: s_mov_b32 s2, -1111; GCN-FASTFMA-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc112; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)113; GCN-FASTFMA-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc114; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)115; GCN-FASTFMA-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc116; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)117; GCN-FASTFMA-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc118; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)119; GCN-FASTFMA-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc120; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)121; GCN-FASTFMA-NEXT: v_fma_f32 v2, v3, v4, -v2122; GCN-FASTFMA-NEXT: v_fma_f32 v0, v0, v1, v2123; GCN-FASTFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0124; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)125; GCN-FASTFMA-NEXT: s_endpgm126;127; GCN-SLOWFMA-LABEL: fast_sub_fmuladd_fmul:128; GCN-SLOWFMA: ; %bb.0:129; GCN-SLOWFMA-NEXT: s_mov_b32 s3, 0xf000130; GCN-SLOWFMA-NEXT: s_mov_b32 s2, -1131; GCN-SLOWFMA-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc132; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)133; GCN-SLOWFMA-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc134; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)135; GCN-SLOWFMA-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc136; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)137; GCN-SLOWFMA-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc138; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)139; GCN-SLOWFMA-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc140; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)141; GCN-SLOWFMA-NEXT: v_mul_f32_e32 v3, v3, v4142; GCN-SLOWFMA-NEXT: v_mul_f32_e32 v0, v0, v1143; GCN-SLOWFMA-NEXT: v_add_f32_e32 v0, v0, v3144; GCN-SLOWFMA-NEXT: v_sub_f32_e32 v0, v0, v2145; GCN-SLOWFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0146; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)147; GCN-SLOWFMA-NEXT: s_endpgm148 %x = load volatile float, ptr addrspace(1) poison149 %y = load volatile float, ptr addrspace(1) poison150 %z = load volatile float, ptr addrspace(1) poison151 %u = load volatile float, ptr addrspace(1) poison152 %v = load volatile float, ptr addrspace(1) poison153 %mul.u.v = fmul fast float %u, %v154 %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v)155 %add = fsub fast float %fma, %z156 store volatile float %add, ptr addrspace(1) poison157 ret void158}159 160define amdgpu_kernel void @fast_add_fmuladd_fmul_multi_use_mul() #0 {161; GCN-FLUSH-LABEL: fast_add_fmuladd_fmul_multi_use_mul:162; GCN-FLUSH: ; %bb.0:163; GCN-FLUSH-NEXT: s_mov_b32 s3, 0xf000164; GCN-FLUSH-NEXT: s_mov_b32 s2, -1165; GCN-FLUSH-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc166; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)167; GCN-FLUSH-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc168; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)169; GCN-FLUSH-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc170; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)171; GCN-FLUSH-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc172; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)173; GCN-FLUSH-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc174; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)175; GCN-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v4176; GCN-FLUSH-NEXT: buffer_store_dword v3, off, s[0:3], 0177; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0)178; GCN-FLUSH-NEXT: v_mac_f32_e32 v3, v0, v1179; GCN-FLUSH-NEXT: v_add_f32_e32 v0, v3, v2180; GCN-FLUSH-NEXT: buffer_store_dword v0, off, s[0:3], 0181; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)182; GCN-FLUSH-NEXT: s_endpgm183;184; GCN-FASTFMA-LABEL: fast_add_fmuladd_fmul_multi_use_mul:185; GCN-FASTFMA: ; %bb.0:186; GCN-FASTFMA-NEXT: s_mov_b32 s3, 0xf000187; GCN-FASTFMA-NEXT: s_mov_b32 s2, -1188; GCN-FASTFMA-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc189; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)190; GCN-FASTFMA-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc191; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)192; GCN-FASTFMA-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc193; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)194; GCN-FASTFMA-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc195; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)196; GCN-FASTFMA-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc197; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)198; GCN-FASTFMA-NEXT: v_mul_f32_e32 v3, v3, v4199; GCN-FASTFMA-NEXT: buffer_store_dword v3, off, s[0:3], 0200; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)201; GCN-FASTFMA-NEXT: v_fma_f32 v0, v0, v1, v3202; GCN-FASTFMA-NEXT: v_add_f32_e32 v0, v0, v2203; GCN-FASTFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0204; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)205; GCN-FASTFMA-NEXT: s_endpgm206;207; GCN-SLOWFMA-LABEL: fast_add_fmuladd_fmul_multi_use_mul:208; GCN-SLOWFMA: ; %bb.0:209; GCN-SLOWFMA-NEXT: s_mov_b32 s3, 0xf000210; GCN-SLOWFMA-NEXT: s_mov_b32 s2, -1211; GCN-SLOWFMA-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc212; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)213; GCN-SLOWFMA-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc214; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)215; GCN-SLOWFMA-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc216; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)217; GCN-SLOWFMA-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc218; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)219; GCN-SLOWFMA-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc220; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)221; GCN-SLOWFMA-NEXT: v_mul_f32_e32 v3, v3, v4222; GCN-SLOWFMA-NEXT: v_mul_f32_e32 v0, v0, v1223; GCN-SLOWFMA-NEXT: buffer_store_dword v3, off, s[0:3], 0224; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)225; GCN-SLOWFMA-NEXT: v_add_f32_e32 v0, v0, v3226; GCN-SLOWFMA-NEXT: v_add_f32_e32 v0, v0, v2227; GCN-SLOWFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0228; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)229; GCN-SLOWFMA-NEXT: s_endpgm230 %x = load volatile float, ptr addrspace(1) poison231 %y = load volatile float, ptr addrspace(1) poison232 %z = load volatile float, ptr addrspace(1) poison233 %u = load volatile float, ptr addrspace(1) poison234 %v = load volatile float, ptr addrspace(1) poison235 %mul.u.v = fmul fast float %u, %v236 store volatile float %mul.u.v, ptr addrspace(1) poison237 %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v)238 %add = fadd fast float %fma, %z239 store volatile float %add, ptr addrspace(1) poison240 ret void241}242 243define amdgpu_kernel void @fast_add_fmuladd_fmul_multi_use_mul_commute() #0 {244; GCN-FLUSH-LABEL: fast_add_fmuladd_fmul_multi_use_mul_commute:245; GCN-FLUSH: ; %bb.0:246; GCN-FLUSH-NEXT: s_mov_b32 s3, 0xf000247; GCN-FLUSH-NEXT: s_mov_b32 s2, -1248; GCN-FLUSH-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc249; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)250; GCN-FLUSH-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc251; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)252; GCN-FLUSH-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc253; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)254; GCN-FLUSH-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc255; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)256; GCN-FLUSH-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc257; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)258; GCN-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v4259; GCN-FLUSH-NEXT: buffer_store_dword v3, off, s[0:3], 0260; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0)261; GCN-FLUSH-NEXT: v_mac_f32_e32 v3, v0, v1262; GCN-FLUSH-NEXT: v_add_f32_e32 v0, v2, v3263; GCN-FLUSH-NEXT: buffer_store_dword v0, off, s[0:3], 0264; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)265; GCN-FLUSH-NEXT: s_endpgm266;267; GCN-FASTFMA-LABEL: fast_add_fmuladd_fmul_multi_use_mul_commute:268; GCN-FASTFMA: ; %bb.0:269; GCN-FASTFMA-NEXT: s_mov_b32 s3, 0xf000270; GCN-FASTFMA-NEXT: s_mov_b32 s2, -1271; GCN-FASTFMA-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc272; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)273; GCN-FASTFMA-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc274; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)275; GCN-FASTFMA-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc276; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)277; GCN-FASTFMA-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc278; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)279; GCN-FASTFMA-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc280; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)281; GCN-FASTFMA-NEXT: v_mul_f32_e32 v3, v3, v4282; GCN-FASTFMA-NEXT: buffer_store_dword v3, off, s[0:3], 0283; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)284; GCN-FASTFMA-NEXT: v_fma_f32 v0, v0, v1, v3285; GCN-FASTFMA-NEXT: v_add_f32_e32 v0, v2, v0286; GCN-FASTFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0287; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)288; GCN-FASTFMA-NEXT: s_endpgm289;290; GCN-SLOWFMA-LABEL: fast_add_fmuladd_fmul_multi_use_mul_commute:291; GCN-SLOWFMA: ; %bb.0:292; GCN-SLOWFMA-NEXT: s_mov_b32 s3, 0xf000293; GCN-SLOWFMA-NEXT: s_mov_b32 s2, -1294; GCN-SLOWFMA-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc295; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)296; GCN-SLOWFMA-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc297; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)298; GCN-SLOWFMA-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc299; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)300; GCN-SLOWFMA-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc301; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)302; GCN-SLOWFMA-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc303; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)304; GCN-SLOWFMA-NEXT: v_mul_f32_e32 v3, v3, v4305; GCN-SLOWFMA-NEXT: v_mul_f32_e32 v0, v0, v1306; GCN-SLOWFMA-NEXT: buffer_store_dword v3, off, s[0:3], 0307; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)308; GCN-SLOWFMA-NEXT: v_add_f32_e32 v0, v0, v3309; GCN-SLOWFMA-NEXT: v_add_f32_e32 v0, v2, v0310; GCN-SLOWFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0311; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)312; GCN-SLOWFMA-NEXT: s_endpgm313 %x = load volatile float, ptr addrspace(1) poison314 %y = load volatile float, ptr addrspace(1) poison315 %z = load volatile float, ptr addrspace(1) poison316 %u = load volatile float, ptr addrspace(1) poison317 %v = load volatile float, ptr addrspace(1) poison318 %mul.u.v = fmul fast float %u, %v319 store volatile float %mul.u.v, ptr addrspace(1) poison320 %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v)321 %add = fadd fast float %z, %fma322 store volatile float %add, ptr addrspace(1) poison323 ret void324}325 326define amdgpu_kernel void @fast_add_fmuladd_fmul_multi_use_fmuladd() #0 {327; GCN-FLUSH-LABEL: fast_add_fmuladd_fmul_multi_use_fmuladd:328; GCN-FLUSH: ; %bb.0:329; GCN-FLUSH-NEXT: s_mov_b32 s3, 0xf000330; GCN-FLUSH-NEXT: s_mov_b32 s2, -1331; GCN-FLUSH-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc332; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)333; GCN-FLUSH-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc334; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)335; GCN-FLUSH-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc336; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)337; GCN-FLUSH-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc338; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)339; GCN-FLUSH-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc340; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)341; GCN-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v4342; GCN-FLUSH-NEXT: v_mac_f32_e32 v3, v0, v1343; GCN-FLUSH-NEXT: buffer_store_dword v3, off, s[0:3], 0344; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)345; GCN-FLUSH-NEXT: v_add_f32_e32 v0, v3, v2346; GCN-FLUSH-NEXT: buffer_store_dword v0, off, s[0:3], 0347; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)348; GCN-FLUSH-NEXT: s_endpgm349;350; GCN-FASTFMA-LABEL: fast_add_fmuladd_fmul_multi_use_fmuladd:351; GCN-FASTFMA: ; %bb.0:352; GCN-FASTFMA-NEXT: s_mov_b32 s3, 0xf000353; GCN-FASTFMA-NEXT: s_mov_b32 s2, -1354; GCN-FASTFMA-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc355; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)356; GCN-FASTFMA-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc357; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)358; GCN-FASTFMA-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc359; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)360; GCN-FASTFMA-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc361; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)362; GCN-FASTFMA-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc363; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)364; GCN-FASTFMA-NEXT: v_mul_f32_e32 v3, v3, v4365; GCN-FASTFMA-NEXT: v_fma_f32 v0, v0, v1, v3366; GCN-FASTFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0367; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0) expcnt(0)368; GCN-FASTFMA-NEXT: v_add_f32_e32 v0, v0, v2369; GCN-FASTFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0370; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)371; GCN-FASTFMA-NEXT: s_endpgm372;373; GCN-SLOWFMA-LABEL: fast_add_fmuladd_fmul_multi_use_fmuladd:374; GCN-SLOWFMA: ; %bb.0:375; GCN-SLOWFMA-NEXT: s_mov_b32 s3, 0xf000376; GCN-SLOWFMA-NEXT: s_mov_b32 s2, -1377; GCN-SLOWFMA-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc378; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)379; GCN-SLOWFMA-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc380; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)381; GCN-SLOWFMA-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc382; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)383; GCN-SLOWFMA-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc384; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)385; GCN-SLOWFMA-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc386; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)387; GCN-SLOWFMA-NEXT: v_mul_f32_e32 v3, v3, v4388; GCN-SLOWFMA-NEXT: v_mul_f32_e32 v0, v0, v1389; GCN-SLOWFMA-NEXT: v_add_f32_e32 v0, v0, v3390; GCN-SLOWFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0391; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0) expcnt(0)392; GCN-SLOWFMA-NEXT: v_add_f32_e32 v0, v0, v2393; GCN-SLOWFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0394; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)395; GCN-SLOWFMA-NEXT: s_endpgm396 %x = load volatile float, ptr addrspace(1) poison397 %y = load volatile float, ptr addrspace(1) poison398 %z = load volatile float, ptr addrspace(1) poison399 %u = load volatile float, ptr addrspace(1) poison400 %v = load volatile float, ptr addrspace(1) poison401 %mul.u.v = fmul fast float %u, %v402 %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v)403 store volatile float %fma, ptr addrspace(1) poison404 %add = fadd fast float %fma, %z405 store volatile float %add, ptr addrspace(1) poison406 ret void407}408 409define amdgpu_kernel void @fast_add_fmuladd_fmul_multi_use_fmuladd_commute() #0 {410; GCN-FLUSH-LABEL: fast_add_fmuladd_fmul_multi_use_fmuladd_commute:411; GCN-FLUSH: ; %bb.0:412; GCN-FLUSH-NEXT: s_mov_b32 s3, 0xf000413; GCN-FLUSH-NEXT: s_mov_b32 s2, -1414; GCN-FLUSH-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc415; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)416; GCN-FLUSH-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc417; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)418; GCN-FLUSH-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc419; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)420; GCN-FLUSH-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc421; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)422; GCN-FLUSH-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc423; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)424; GCN-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v4425; GCN-FLUSH-NEXT: v_mac_f32_e32 v3, v0, v1426; GCN-FLUSH-NEXT: buffer_store_dword v3, off, s[0:3], 0427; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)428; GCN-FLUSH-NEXT: v_add_f32_e32 v0, v2, v3429; GCN-FLUSH-NEXT: buffer_store_dword v0, off, s[0:3], 0430; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)431; GCN-FLUSH-NEXT: s_endpgm432;433; GCN-FASTFMA-LABEL: fast_add_fmuladd_fmul_multi_use_fmuladd_commute:434; GCN-FASTFMA: ; %bb.0:435; GCN-FASTFMA-NEXT: s_mov_b32 s3, 0xf000436; GCN-FASTFMA-NEXT: s_mov_b32 s2, -1437; GCN-FASTFMA-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc438; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)439; GCN-FASTFMA-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc440; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)441; GCN-FASTFMA-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc442; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)443; GCN-FASTFMA-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc444; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)445; GCN-FASTFMA-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc446; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)447; GCN-FASTFMA-NEXT: v_mul_f32_e32 v3, v3, v4448; GCN-FASTFMA-NEXT: v_fma_f32 v0, v0, v1, v3449; GCN-FASTFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0450; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0) expcnt(0)451; GCN-FASTFMA-NEXT: v_add_f32_e32 v0, v2, v0452; GCN-FASTFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0453; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)454; GCN-FASTFMA-NEXT: s_endpgm455;456; GCN-SLOWFMA-LABEL: fast_add_fmuladd_fmul_multi_use_fmuladd_commute:457; GCN-SLOWFMA: ; %bb.0:458; GCN-SLOWFMA-NEXT: s_mov_b32 s3, 0xf000459; GCN-SLOWFMA-NEXT: s_mov_b32 s2, -1460; GCN-SLOWFMA-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc461; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)462; GCN-SLOWFMA-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc463; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)464; GCN-SLOWFMA-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc465; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)466; GCN-SLOWFMA-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc467; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)468; GCN-SLOWFMA-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc469; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)470; GCN-SLOWFMA-NEXT: v_mul_f32_e32 v3, v3, v4471; GCN-SLOWFMA-NEXT: v_mul_f32_e32 v0, v0, v1472; GCN-SLOWFMA-NEXT: v_add_f32_e32 v0, v0, v3473; GCN-SLOWFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0474; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0) expcnt(0)475; GCN-SLOWFMA-NEXT: v_add_f32_e32 v0, v2, v0476; GCN-SLOWFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0477; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)478; GCN-SLOWFMA-NEXT: s_endpgm479 %x = load volatile float, ptr addrspace(1) poison480 %y = load volatile float, ptr addrspace(1) poison481 %z = load volatile float, ptr addrspace(1) poison482 %u = load volatile float, ptr addrspace(1) poison483 %v = load volatile float, ptr addrspace(1) poison484 %mul.u.v = fmul fast float %u, %v485 %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v)486 store volatile float %fma, ptr addrspace(1) poison487 %add = fadd fast float %z, %fma488 store volatile float %add, ptr addrspace(1) poison489 ret void490}491 492define amdgpu_kernel void @fast_sub_fmuladd_fmul_multi_use_mul() #0 {493; GCN-FLUSH-LABEL: fast_sub_fmuladd_fmul_multi_use_mul:494; GCN-FLUSH: ; %bb.0:495; GCN-FLUSH-NEXT: s_mov_b32 s3, 0xf000496; GCN-FLUSH-NEXT: s_mov_b32 s2, -1497; GCN-FLUSH-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc498; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)499; GCN-FLUSH-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc500; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)501; GCN-FLUSH-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc502; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)503; GCN-FLUSH-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc504; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)505; GCN-FLUSH-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc506; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)507; GCN-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v4508; GCN-FLUSH-NEXT: v_mad_f32 v0, v0, v1, v3509; GCN-FLUSH-NEXT: v_sub_f32_e32 v0, v0, v2510; GCN-FLUSH-NEXT: buffer_store_dword v3, off, s[0:3], 0511; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)512; GCN-FLUSH-NEXT: buffer_store_dword v0, off, s[0:3], 0513; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)514; GCN-FLUSH-NEXT: s_endpgm515;516; GCN-FASTFMA-LABEL: fast_sub_fmuladd_fmul_multi_use_mul:517; GCN-FASTFMA: ; %bb.0:518; GCN-FASTFMA-NEXT: s_mov_b32 s3, 0xf000519; GCN-FASTFMA-NEXT: s_mov_b32 s2, -1520; GCN-FASTFMA-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc521; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)522; GCN-FASTFMA-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc523; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)524; GCN-FASTFMA-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc525; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)526; GCN-FASTFMA-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc527; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)528; GCN-FASTFMA-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc529; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)530; GCN-FASTFMA-NEXT: v_mul_f32_e32 v3, v3, v4531; GCN-FASTFMA-NEXT: v_fma_f32 v0, v0, v1, v3532; GCN-FASTFMA-NEXT: v_sub_f32_e32 v0, v0, v2533; GCN-FASTFMA-NEXT: buffer_store_dword v3, off, s[0:3], 0534; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)535; GCN-FASTFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0536; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)537; GCN-FASTFMA-NEXT: s_endpgm538;539; GCN-SLOWFMA-LABEL: fast_sub_fmuladd_fmul_multi_use_mul:540; GCN-SLOWFMA: ; %bb.0:541; GCN-SLOWFMA-NEXT: s_mov_b32 s3, 0xf000542; GCN-SLOWFMA-NEXT: s_mov_b32 s2, -1543; GCN-SLOWFMA-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc544; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)545; GCN-SLOWFMA-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc546; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)547; GCN-SLOWFMA-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc548; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)549; GCN-SLOWFMA-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc550; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)551; GCN-SLOWFMA-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc552; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)553; GCN-SLOWFMA-NEXT: v_mul_f32_e32 v3, v3, v4554; GCN-SLOWFMA-NEXT: v_mul_f32_e32 v0, v0, v1555; GCN-SLOWFMA-NEXT: v_add_f32_e32 v0, v0, v3556; GCN-SLOWFMA-NEXT: v_sub_f32_e32 v0, v0, v2557; GCN-SLOWFMA-NEXT: buffer_store_dword v3, off, s[0:3], 0558; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)559; GCN-SLOWFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0560; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)561; GCN-SLOWFMA-NEXT: s_endpgm562 %x = load volatile float, ptr addrspace(1) poison563 %y = load volatile float, ptr addrspace(1) poison564 %z = load volatile float, ptr addrspace(1) poison565 %u = load volatile float, ptr addrspace(1) poison566 %v = load volatile float, ptr addrspace(1) poison567 %mul.u.v = fmul fast float %u, %v568 %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v)569 %sub = fsub fast float %fma, %z570 store volatile float %mul.u.v, ptr addrspace(1) poison571 store volatile float %sub, ptr addrspace(1) poison572 ret void573}574 575define amdgpu_kernel void @fast_sub_fmuladd_fmul_multi_use_fmuladd_lhs() #0 {576; GCN-FLUSH-LABEL: fast_sub_fmuladd_fmul_multi_use_fmuladd_lhs:577; GCN-FLUSH: ; %bb.0:578; GCN-FLUSH-NEXT: s_mov_b32 s3, 0xf000579; GCN-FLUSH-NEXT: s_mov_b32 s2, -1580; GCN-FLUSH-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc581; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)582; GCN-FLUSH-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc583; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)584; GCN-FLUSH-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc585; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)586; GCN-FLUSH-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc587; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)588; GCN-FLUSH-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc589; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)590; GCN-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v4591; GCN-FLUSH-NEXT: v_mac_f32_e32 v3, v0, v1592; GCN-FLUSH-NEXT: v_sub_f32_e32 v0, v3, v2593; GCN-FLUSH-NEXT: buffer_store_dword v3, off, s[0:3], 0594; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)595; GCN-FLUSH-NEXT: buffer_store_dword v0, off, s[0:3], 0596; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)597; GCN-FLUSH-NEXT: s_endpgm598;599; GCN-FASTFMA-LABEL: fast_sub_fmuladd_fmul_multi_use_fmuladd_lhs:600; GCN-FASTFMA: ; %bb.0:601; GCN-FASTFMA-NEXT: s_mov_b32 s3, 0xf000602; GCN-FASTFMA-NEXT: s_mov_b32 s2, -1603; GCN-FASTFMA-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc604; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)605; GCN-FASTFMA-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc606; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)607; GCN-FASTFMA-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc608; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)609; GCN-FASTFMA-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc610; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)611; GCN-FASTFMA-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc612; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)613; GCN-FASTFMA-NEXT: v_mul_f32_e32 v3, v3, v4614; GCN-FASTFMA-NEXT: v_fma_f32 v0, v0, v1, v3615; GCN-FASTFMA-NEXT: v_sub_f32_e32 v1, v0, v2616; GCN-FASTFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0617; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)618; GCN-FASTFMA-NEXT: buffer_store_dword v1, off, s[0:3], 0619; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)620; GCN-FASTFMA-NEXT: s_endpgm621;622; GCN-SLOWFMA-LABEL: fast_sub_fmuladd_fmul_multi_use_fmuladd_lhs:623; GCN-SLOWFMA: ; %bb.0:624; GCN-SLOWFMA-NEXT: s_mov_b32 s3, 0xf000625; GCN-SLOWFMA-NEXT: s_mov_b32 s2, -1626; GCN-SLOWFMA-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc627; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)628; GCN-SLOWFMA-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc629; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)630; GCN-SLOWFMA-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc631; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)632; GCN-SLOWFMA-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc633; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)634; GCN-SLOWFMA-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc635; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)636; GCN-SLOWFMA-NEXT: v_mul_f32_e32 v3, v3, v4637; GCN-SLOWFMA-NEXT: v_mul_f32_e32 v0, v0, v1638; GCN-SLOWFMA-NEXT: v_add_f32_e32 v0, v0, v3639; GCN-SLOWFMA-NEXT: v_sub_f32_e32 v1, v0, v2640; GCN-SLOWFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0641; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)642; GCN-SLOWFMA-NEXT: buffer_store_dword v1, off, s[0:3], 0643; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)644; GCN-SLOWFMA-NEXT: s_endpgm645 %x = load volatile float, ptr addrspace(1) poison646 %y = load volatile float, ptr addrspace(1) poison647 %z = load volatile float, ptr addrspace(1) poison648 %u = load volatile float, ptr addrspace(1) poison649 %v = load volatile float, ptr addrspace(1) poison650 %mul.u.v = fmul fast float %u, %v651 %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v)652 %add = fsub fast float %fma, %z653 store volatile float %fma, ptr addrspace(1) poison654 store volatile float %add, ptr addrspace(1) poison655 ret void656}657 658define amdgpu_kernel void @fast_sub_fmuladd_fmul_multi_use_fmuladd_rhs() #0 {659; GCN-FLUSH-LABEL: fast_sub_fmuladd_fmul_multi_use_fmuladd_rhs:660; GCN-FLUSH: ; %bb.0:661; GCN-FLUSH-NEXT: s_mov_b32 s3, 0xf000662; GCN-FLUSH-NEXT: s_mov_b32 s2, -1663; GCN-FLUSH-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc664; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)665; GCN-FLUSH-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc666; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)667; GCN-FLUSH-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc668; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)669; GCN-FLUSH-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc670; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)671; GCN-FLUSH-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc672; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)673; GCN-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v4674; GCN-FLUSH-NEXT: v_mac_f32_e32 v3, v0, v1675; GCN-FLUSH-NEXT: v_sub_f32_e32 v0, v2, v3676; GCN-FLUSH-NEXT: buffer_store_dword v3, off, s[0:3], 0677; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)678; GCN-FLUSH-NEXT: buffer_store_dword v0, off, s[0:3], 0679; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)680; GCN-FLUSH-NEXT: s_endpgm681;682; GCN-FASTFMA-LABEL: fast_sub_fmuladd_fmul_multi_use_fmuladd_rhs:683; GCN-FASTFMA: ; %bb.0:684; GCN-FASTFMA-NEXT: s_mov_b32 s3, 0xf000685; GCN-FASTFMA-NEXT: s_mov_b32 s2, -1686; GCN-FASTFMA-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc687; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)688; GCN-FASTFMA-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc689; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)690; GCN-FASTFMA-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc691; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)692; GCN-FASTFMA-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc693; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)694; GCN-FASTFMA-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc695; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)696; GCN-FASTFMA-NEXT: v_mul_f32_e32 v3, v3, v4697; GCN-FASTFMA-NEXT: v_fma_f32 v0, v0, v1, v3698; GCN-FASTFMA-NEXT: v_sub_f32_e32 v1, v2, v0699; GCN-FASTFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0700; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)701; GCN-FASTFMA-NEXT: buffer_store_dword v1, off, s[0:3], 0702; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)703; GCN-FASTFMA-NEXT: s_endpgm704;705; GCN-SLOWFMA-LABEL: fast_sub_fmuladd_fmul_multi_use_fmuladd_rhs:706; GCN-SLOWFMA: ; %bb.0:707; GCN-SLOWFMA-NEXT: s_mov_b32 s3, 0xf000708; GCN-SLOWFMA-NEXT: s_mov_b32 s2, -1709; GCN-SLOWFMA-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc710; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)711; GCN-SLOWFMA-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc712; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)713; GCN-SLOWFMA-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc714; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)715; GCN-SLOWFMA-NEXT: buffer_load_dword v3, off, s[0:3], 0 glc716; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)717; GCN-SLOWFMA-NEXT: buffer_load_dword v4, off, s[0:3], 0 glc718; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)719; GCN-SLOWFMA-NEXT: v_mul_f32_e32 v3, v3, v4720; GCN-SLOWFMA-NEXT: v_mul_f32_e32 v0, v0, v1721; GCN-SLOWFMA-NEXT: v_add_f32_e32 v0, v0, v3722; GCN-SLOWFMA-NEXT: v_sub_f32_e32 v1, v2, v0723; GCN-SLOWFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0724; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)725; GCN-SLOWFMA-NEXT: buffer_store_dword v1, off, s[0:3], 0726; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)727; GCN-SLOWFMA-NEXT: s_endpgm728 %x = load volatile float, ptr addrspace(1) poison729 %y = load volatile float, ptr addrspace(1) poison730 %z = load volatile float, ptr addrspace(1) poison731 %u = load volatile float, ptr addrspace(1) poison732 %v = load volatile float, ptr addrspace(1) poison733 %mul.u.v = fmul fast float %u, %v734 %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v)735 %add = fsub fast float %z, %fma736 store volatile float %fma, ptr addrspace(1) poison737 store volatile float %add, ptr addrspace(1) poison738 ret void739}740 741define amdgpu_kernel void @fast_sub_fmuladd_fpext_fmul_multi_use_fmuladd_lhs() #0 {742; GCN-FLUSH-LABEL: fast_sub_fmuladd_fpext_fmul_multi_use_fmuladd_lhs:743; GCN-FLUSH: ; %bb.0:744; GCN-FLUSH-NEXT: s_mov_b32 s3, 0xf000745; GCN-FLUSH-NEXT: s_mov_b32 s2, -1746; GCN-FLUSH-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc747; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)748; GCN-FLUSH-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc749; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)750; GCN-FLUSH-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc751; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)752; GCN-FLUSH-NEXT: buffer_load_ushort v3, off, s[0:3], 0 glc753; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)754; GCN-FLUSH-NEXT: buffer_load_ushort v4, off, s[0:3], 0 glc755; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)756; GCN-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v3757; GCN-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, v4758; GCN-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v4759; GCN-FLUSH-NEXT: v_mac_f32_e32 v3, v0, v1760; GCN-FLUSH-NEXT: v_sub_f32_e32 v0, v3, v2761; GCN-FLUSH-NEXT: buffer_store_dword v3, off, s[0:3], 0762; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)763; GCN-FLUSH-NEXT: buffer_store_dword v0, off, s[0:3], 0764; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)765; GCN-FLUSH-NEXT: s_endpgm766;767; GCN-FASTFMA-LABEL: fast_sub_fmuladd_fpext_fmul_multi_use_fmuladd_lhs:768; GCN-FASTFMA: ; %bb.0:769; GCN-FASTFMA-NEXT: s_mov_b32 s3, 0xf000770; GCN-FASTFMA-NEXT: s_mov_b32 s2, -1771; GCN-FASTFMA-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc772; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)773; GCN-FASTFMA-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc774; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)775; GCN-FASTFMA-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc776; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)777; GCN-FASTFMA-NEXT: buffer_load_ushort v3, off, s[0:3], 0 glc778; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)779; GCN-FASTFMA-NEXT: buffer_load_ushort v4, off, s[0:3], 0 glc780; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)781; GCN-FASTFMA-NEXT: v_cvt_f32_f16_e32 v3, v3782; GCN-FASTFMA-NEXT: v_cvt_f32_f16_e32 v4, v4783; GCN-FASTFMA-NEXT: v_mul_f32_e32 v3, v3, v4784; GCN-FASTFMA-NEXT: v_fma_f32 v0, v0, v1, v3785; GCN-FASTFMA-NEXT: v_sub_f32_e32 v1, v0, v2786; GCN-FASTFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0787; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)788; GCN-FASTFMA-NEXT: buffer_store_dword v1, off, s[0:3], 0789; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)790; GCN-FASTFMA-NEXT: s_endpgm791;792; GCN-SLOWFMA-LABEL: fast_sub_fmuladd_fpext_fmul_multi_use_fmuladd_lhs:793; GCN-SLOWFMA: ; %bb.0:794; GCN-SLOWFMA-NEXT: s_mov_b32 s3, 0xf000795; GCN-SLOWFMA-NEXT: s_mov_b32 s2, -1796; GCN-SLOWFMA-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc797; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)798; GCN-SLOWFMA-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc799; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)800; GCN-SLOWFMA-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc801; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)802; GCN-SLOWFMA-NEXT: buffer_load_ushort v3, off, s[0:3], 0 glc803; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)804; GCN-SLOWFMA-NEXT: buffer_load_ushort v4, off, s[0:3], 0 glc805; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)806; GCN-SLOWFMA-NEXT: v_cvt_f32_f16_e32 v3, v3807; GCN-SLOWFMA-NEXT: v_cvt_f32_f16_e32 v4, v4808; GCN-SLOWFMA-NEXT: v_mul_f32_e32 v0, v0, v1809; GCN-SLOWFMA-NEXT: v_mul_f32_e32 v1, v3, v4810; GCN-SLOWFMA-NEXT: v_add_f32_e32 v0, v0, v1811; GCN-SLOWFMA-NEXT: v_sub_f32_e32 v1, v0, v2812; GCN-SLOWFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0813; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)814; GCN-SLOWFMA-NEXT: buffer_store_dword v1, off, s[0:3], 0815; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)816; GCN-SLOWFMA-NEXT: s_endpgm817 %x = load volatile float, ptr addrspace(1) poison818 %y = load volatile float, ptr addrspace(1) poison819 %z = load volatile float, ptr addrspace(1) poison820 %u = load volatile half, ptr addrspace(1) poison821 %v = load volatile half, ptr addrspace(1) poison822 %mul.u.v.half = fmul fast half %u, %v823 %mul.u.v = fpext half %mul.u.v.half to float824 %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v)825 %add = fsub fast float %fma, %z826 store volatile float %fma, ptr addrspace(1) poison827 store volatile float %add, ptr addrspace(1) poison828 ret void829}830 831define amdgpu_kernel void @fast_sub_fmuladd_fpext_fmul_multi_use_fmuladd_rhs() #0 {832; GCN-FLUSH-LABEL: fast_sub_fmuladd_fpext_fmul_multi_use_fmuladd_rhs:833; GCN-FLUSH: ; %bb.0:834; GCN-FLUSH-NEXT: s_mov_b32 s3, 0xf000835; GCN-FLUSH-NEXT: s_mov_b32 s2, -1836; GCN-FLUSH-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc837; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)838; GCN-FLUSH-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc839; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)840; GCN-FLUSH-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc841; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)842; GCN-FLUSH-NEXT: buffer_load_ushort v3, off, s[0:3], 0 glc843; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)844; GCN-FLUSH-NEXT: buffer_load_ushort v4, off, s[0:3], 0 glc845; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)846; GCN-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v3847; GCN-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, v4848; GCN-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v4849; GCN-FLUSH-NEXT: v_mac_f32_e32 v3, v0, v1850; GCN-FLUSH-NEXT: v_sub_f32_e32 v0, v2, v3851; GCN-FLUSH-NEXT: buffer_store_dword v3, off, s[0:3], 0852; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)853; GCN-FLUSH-NEXT: buffer_store_dword v0, off, s[0:3], 0854; GCN-FLUSH-NEXT: s_waitcnt vmcnt(0)855; GCN-FLUSH-NEXT: s_endpgm856;857; GCN-FASTFMA-LABEL: fast_sub_fmuladd_fpext_fmul_multi_use_fmuladd_rhs:858; GCN-FASTFMA: ; %bb.0:859; GCN-FASTFMA-NEXT: s_mov_b32 s3, 0xf000860; GCN-FASTFMA-NEXT: s_mov_b32 s2, -1861; GCN-FASTFMA-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc862; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)863; GCN-FASTFMA-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc864; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)865; GCN-FASTFMA-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc866; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)867; GCN-FASTFMA-NEXT: buffer_load_ushort v3, off, s[0:3], 0 glc868; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)869; GCN-FASTFMA-NEXT: buffer_load_ushort v4, off, s[0:3], 0 glc870; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)871; GCN-FASTFMA-NEXT: v_cvt_f32_f16_e32 v3, v3872; GCN-FASTFMA-NEXT: v_cvt_f32_f16_e32 v4, v4873; GCN-FASTFMA-NEXT: v_mul_f32_e32 v3, v3, v4874; GCN-FASTFMA-NEXT: v_fma_f32 v0, v0, v1, v3875; GCN-FASTFMA-NEXT: v_sub_f32_e32 v1, v2, v0876; GCN-FASTFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0877; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)878; GCN-FASTFMA-NEXT: buffer_store_dword v1, off, s[0:3], 0879; GCN-FASTFMA-NEXT: s_waitcnt vmcnt(0)880; GCN-FASTFMA-NEXT: s_endpgm881;882; GCN-SLOWFMA-LABEL: fast_sub_fmuladd_fpext_fmul_multi_use_fmuladd_rhs:883; GCN-SLOWFMA: ; %bb.0:884; GCN-SLOWFMA-NEXT: s_mov_b32 s3, 0xf000885; GCN-SLOWFMA-NEXT: s_mov_b32 s2, -1886; GCN-SLOWFMA-NEXT: buffer_load_dword v0, off, s[0:3], 0 glc887; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)888; GCN-SLOWFMA-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc889; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)890; GCN-SLOWFMA-NEXT: buffer_load_dword v2, off, s[0:3], 0 glc891; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)892; GCN-SLOWFMA-NEXT: buffer_load_ushort v3, off, s[0:3], 0 glc893; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)894; GCN-SLOWFMA-NEXT: buffer_load_ushort v4, off, s[0:3], 0 glc895; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)896; GCN-SLOWFMA-NEXT: v_cvt_f32_f16_e32 v3, v3897; GCN-SLOWFMA-NEXT: v_cvt_f32_f16_e32 v4, v4898; GCN-SLOWFMA-NEXT: v_mul_f32_e32 v0, v0, v1899; GCN-SLOWFMA-NEXT: v_mul_f32_e32 v1, v3, v4900; GCN-SLOWFMA-NEXT: v_add_f32_e32 v0, v0, v1901; GCN-SLOWFMA-NEXT: v_sub_f32_e32 v1, v2, v0902; GCN-SLOWFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0903; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)904; GCN-SLOWFMA-NEXT: buffer_store_dword v1, off, s[0:3], 0905; GCN-SLOWFMA-NEXT: s_waitcnt vmcnt(0)906; GCN-SLOWFMA-NEXT: s_endpgm907 %x = load volatile float, ptr addrspace(1) poison908 %y = load volatile float, ptr addrspace(1) poison909 %z = load volatile float, ptr addrspace(1) poison910 %u = load volatile half, ptr addrspace(1) poison911 %v = load volatile half, ptr addrspace(1) poison912 %mul.u.v.half = fmul fast half %u, %v913 %mul.u.v = fpext half %mul.u.v.half to float914 %fma = call fast float @llvm.fmuladd.f32(float %x, float %y, float %mul.u.v)915 %add = fsub fast float %z, %fma916 store volatile float %fma, ptr addrspace(1) poison917 store volatile float %add, ptr addrspace(1) poison918 ret void919}920 921declare float @llvm.fma.f32(float, float, float) #1922declare float @llvm.fmuladd.f32(float, float, float) #1923 924attributes #0 = { nounwind }925attributes #1 = { nounwind readnone }926;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:927; GCN: {{.*}}928