544 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx908 < %s | FileCheck -check-prefix=GFX908 %s3; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx90a < %s | FileCheck -check-prefix=GFX90A %s4; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx942 < %s | FileCheck -check-prefix=GFX942 %s5; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX1200 %s6; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefix=GFX1250 %s7 8define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset(float %val, ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) #0 {9; GFX908-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:10; GFX908: ; %bb.0:11; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12; GFX908-NEXT: buffer_atomic_add_f32 v0, v[1:2], s[16:19], s20 idxen offen13; GFX908-NEXT: s_waitcnt vmcnt(0)14; GFX908-NEXT: s_setpc_b64 s[30:31]15;16; GFX90A-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:17; GFX90A: ; %bb.0:18; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)19; GFX90A-NEXT: v_mov_b32_e32 v3, v220; GFX90A-NEXT: v_mov_b32_e32 v2, v121; GFX90A-NEXT: buffer_atomic_add_f32 v0, v[2:3], s[16:19], s20 idxen offen22; GFX90A-NEXT: s_waitcnt vmcnt(0)23; GFX90A-NEXT: s_setpc_b64 s[30:31]24;25; GFX942-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:26; GFX942: ; %bb.0:27; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)28; GFX942-NEXT: v_mov_b32_e32 v3, v229; GFX942-NEXT: v_mov_b32_e32 v2, v130; GFX942-NEXT: buffer_atomic_add_f32 v0, v[2:3], s[0:3], s16 idxen offen31; GFX942-NEXT: s_waitcnt vmcnt(0)32; GFX942-NEXT: s_setpc_b64 s[30:31]33;34; GFX1200-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:35; GFX1200: ; %bb.0:36; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x037; GFX1200-NEXT: s_wait_expcnt 0x038; GFX1200-NEXT: s_wait_samplecnt 0x039; GFX1200-NEXT: s_wait_bvhcnt 0x040; GFX1200-NEXT: s_wait_kmcnt 0x041; GFX1200-NEXT: buffer_atomic_add_f32 v0, v[1:2], s[0:3], s16 idxen offen42; GFX1200-NEXT: s_setpc_b64 s[30:31]43;44; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:45; GFX1250: ; %bb.0:46; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x047; GFX1250-NEXT: s_wait_kmcnt 0x048; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v149; GFX1250-NEXT: buffer_atomic_add_f32 v0, v[2:3], s[0:3], s16 idxen offen50; GFX1250-NEXT: s_set_pc_i64 s[30:31]51 %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)52 ret void53}54 55; Natural mapping, no voffset56define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset(float %val, ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 inreg %soffset) #0 {57; GFX908-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:58; GFX908: ; %bb.0:59; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)60; GFX908-NEXT: buffer_atomic_add_f32 v0, v1, s[16:19], s20 idxen61; GFX908-NEXT: s_waitcnt vmcnt(0)62; GFX908-NEXT: s_setpc_b64 s[30:31]63;64; GFX90A-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:65; GFX90A: ; %bb.0:66; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)67; GFX90A-NEXT: buffer_atomic_add_f32 v0, v1, s[16:19], s20 idxen68; GFX90A-NEXT: s_waitcnt vmcnt(0)69; GFX90A-NEXT: s_setpc_b64 s[30:31]70;71; GFX942-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:72; GFX942: ; %bb.0:73; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)74; GFX942-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], s16 idxen75; GFX942-NEXT: s_waitcnt vmcnt(0)76; GFX942-NEXT: s_setpc_b64 s[30:31]77;78; GFX1200-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:79; GFX1200: ; %bb.0:80; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x081; GFX1200-NEXT: s_wait_expcnt 0x082; GFX1200-NEXT: s_wait_samplecnt 0x083; GFX1200-NEXT: s_wait_bvhcnt 0x084; GFX1200-NEXT: s_wait_kmcnt 0x085; GFX1200-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], s16 idxen86; GFX1200-NEXT: s_setpc_b64 s[30:31]87;88; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:89; GFX1250: ; %bb.0:90; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x091; GFX1250-NEXT: s_wait_kmcnt 0x092; GFX1250-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], s16 idxen93; GFX1250-NEXT: s_set_pc_i64 s[30:31]94 %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 0)95 ret void96}97 98define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc(float %val, ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) #0 {99; GFX908-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:100; GFX908: ; %bb.0:101; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)102; GFX908-NEXT: buffer_atomic_add_f32 v0, v[1:2], s[16:19], s20 idxen offen slc103; GFX908-NEXT: s_waitcnt vmcnt(0)104; GFX908-NEXT: s_setpc_b64 s[30:31]105;106; GFX90A-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:107; GFX90A: ; %bb.0:108; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)109; GFX90A-NEXT: v_mov_b32_e32 v3, v2110; GFX90A-NEXT: v_mov_b32_e32 v2, v1111; GFX90A-NEXT: buffer_atomic_add_f32 v0, v[2:3], s[16:19], s20 idxen offen slc112; GFX90A-NEXT: s_waitcnt vmcnt(0)113; GFX90A-NEXT: s_setpc_b64 s[30:31]114;115; GFX942-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:116; GFX942: ; %bb.0:117; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)118; GFX942-NEXT: v_mov_b32_e32 v3, v2119; GFX942-NEXT: v_mov_b32_e32 v2, v1120; GFX942-NEXT: buffer_atomic_add_f32 v0, v[2:3], s[0:3], s16 idxen offen nt121; GFX942-NEXT: s_waitcnt vmcnt(0)122; GFX942-NEXT: s_setpc_b64 s[30:31]123;124; GFX1200-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:125; GFX1200: ; %bb.0:126; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0127; GFX1200-NEXT: s_wait_expcnt 0x0128; GFX1200-NEXT: s_wait_samplecnt 0x0129; GFX1200-NEXT: s_wait_bvhcnt 0x0130; GFX1200-NEXT: s_wait_kmcnt 0x0131; GFX1200-NEXT: buffer_atomic_add_f32 v0, v[1:2], s[0:3], s16 idxen offen th:TH_ATOMIC_NT132; GFX1200-NEXT: s_setpc_b64 s[30:31]133;134; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:135; GFX1250: ; %bb.0:136; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0137; GFX1250-NEXT: s_wait_kmcnt 0x0138; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1139; GFX1250-NEXT: buffer_atomic_add_f32 v0, v[2:3], s[0:3], s16 idxen offen th:TH_ATOMIC_NT140; GFX1250-NEXT: s_set_pc_i64 s[30:31]141 %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 2)142 ret void143}144 145define void @struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<2 x half> %val, ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) #0 {146; GFX908-LABEL: struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:147; GFX908: ; %bb.0:148; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)149; GFX908-NEXT: buffer_atomic_pk_add_f16 v0, v[1:2], s[16:19], s20 idxen offen150; GFX908-NEXT: s_waitcnt vmcnt(0)151; GFX908-NEXT: s_setpc_b64 s[30:31]152;153; GFX90A-LABEL: struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:154; GFX90A: ; %bb.0:155; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)156; GFX90A-NEXT: v_mov_b32_e32 v3, v2157; GFX90A-NEXT: v_mov_b32_e32 v2, v1158; GFX90A-NEXT: buffer_atomic_pk_add_f16 v0, v[2:3], s[16:19], s20 idxen offen159; GFX90A-NEXT: s_waitcnt vmcnt(0)160; GFX90A-NEXT: s_setpc_b64 s[30:31]161;162; GFX942-LABEL: struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:163; GFX942: ; %bb.0:164; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)165; GFX942-NEXT: v_mov_b32_e32 v3, v2166; GFX942-NEXT: v_mov_b32_e32 v2, v1167; GFX942-NEXT: buffer_atomic_pk_add_f16 v0, v[2:3], s[0:3], s16 idxen offen168; GFX942-NEXT: s_waitcnt vmcnt(0)169; GFX942-NEXT: s_setpc_b64 s[30:31]170;171; GFX1200-LABEL: struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:172; GFX1200: ; %bb.0:173; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0174; GFX1200-NEXT: s_wait_expcnt 0x0175; GFX1200-NEXT: s_wait_samplecnt 0x0176; GFX1200-NEXT: s_wait_bvhcnt 0x0177; GFX1200-NEXT: s_wait_kmcnt 0x0178; GFX1200-NEXT: buffer_atomic_pk_add_f16 v0, v[1:2], s[0:3], s16 idxen offen179; GFX1200-NEXT: s_setpc_b64 s[30:31]180;181; GFX1250-LABEL: struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:182; GFX1250: ; %bb.0:183; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0184; GFX1250-NEXT: s_wait_kmcnt 0x0185; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1186; GFX1250-NEXT: buffer_atomic_pk_add_f16 v0, v[2:3], s[0:3], s16 idxen offen187; GFX1250-NEXT: s_set_pc_i64 s[30:31]188 %ret = call <2 x half> @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2f16(<2 x half> %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)189 ret void190}191 192; Test waterfall loop193define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset) #0 {194; GFX908-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:195; GFX908: ; %bb.0:196; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)197; GFX908-NEXT: s_mov_b64 s[6:7], exec198; GFX908-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1199; GFX908-NEXT: v_readfirstlane_b32 s8, v1200; GFX908-NEXT: v_readfirstlane_b32 s9, v2201; GFX908-NEXT: v_readfirstlane_b32 s10, v3202; GFX908-NEXT: v_readfirstlane_b32 s11, v4203; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[1:2]204; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[3:4]205; GFX908-NEXT: v_readfirstlane_b32 s12, v7206; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]207; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, s12, v7208; GFX908-NEXT: s_and_b64 s[4:5], s[4:5], vcc209; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]210; GFX908-NEXT: s_nop 0211; GFX908-NEXT: buffer_atomic_add_f32 v0, v[5:6], s[8:11], s12 idxen offen212; GFX908-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4213; GFX908-NEXT: ; implicit-def: $vgpr7214; GFX908-NEXT: ; implicit-def: $vgpr0215; GFX908-NEXT: ; implicit-def: $vgpr5_vgpr6216; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]217; GFX908-NEXT: s_cbranch_execnz .LBB4_1218; GFX908-NEXT: ; %bb.2:219; GFX908-NEXT: s_mov_b64 exec, s[6:7]220; GFX908-NEXT: s_waitcnt vmcnt(0)221; GFX908-NEXT: s_setpc_b64 s[30:31]222;223; GFX90A-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:224; GFX90A: ; %bb.0:225; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)226; GFX90A-NEXT: v_mov_b32_e32 v9, v6227; GFX90A-NEXT: v_mov_b32_e32 v8, v5228; GFX90A-NEXT: v_mov_b32_e32 v5, v4229; GFX90A-NEXT: v_mov_b32_e32 v4, v3230; GFX90A-NEXT: v_mov_b32_e32 v3, v2231; GFX90A-NEXT: v_mov_b32_e32 v2, v1232; GFX90A-NEXT: s_mov_b64 s[6:7], exec233; GFX90A-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1234; GFX90A-NEXT: v_readfirstlane_b32 s8, v2235; GFX90A-NEXT: v_readfirstlane_b32 s9, v3236; GFX90A-NEXT: v_readfirstlane_b32 s10, v4237; GFX90A-NEXT: v_readfirstlane_b32 s11, v5238; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[2:3]239; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[4:5]240; GFX90A-NEXT: v_readfirstlane_b32 s12, v7241; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]242; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, s12, v7243; GFX90A-NEXT: s_and_b64 s[4:5], s[4:5], vcc244; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]245; GFX90A-NEXT: s_nop 0246; GFX90A-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[8:11], s12 idxen offen247; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5248; GFX90A-NEXT: ; implicit-def: $vgpr7249; GFX90A-NEXT: ; implicit-def: $vgpr0250; GFX90A-NEXT: ; implicit-def: $vgpr8_vgpr9251; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]252; GFX90A-NEXT: s_cbranch_execnz .LBB4_1253; GFX90A-NEXT: ; %bb.2:254; GFX90A-NEXT: s_mov_b64 exec, s[6:7]255; GFX90A-NEXT: s_waitcnt vmcnt(0)256; GFX90A-NEXT: s_setpc_b64 s[30:31]257;258; GFX942-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:259; GFX942: ; %bb.0:260; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)261; GFX942-NEXT: v_mov_b32_e32 v9, v6262; GFX942-NEXT: v_mov_b32_e32 v8, v5263; GFX942-NEXT: v_mov_b32_e32 v5, v4264; GFX942-NEXT: v_mov_b32_e32 v4, v3265; GFX942-NEXT: v_mov_b32_e32 v3, v2266; GFX942-NEXT: v_mov_b32_e32 v2, v1267; GFX942-NEXT: s_mov_b64 s[2:3], exec268; GFX942-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1269; GFX942-NEXT: v_readfirstlane_b32 s4, v2270; GFX942-NEXT: v_readfirstlane_b32 s5, v3271; GFX942-NEXT: v_readfirstlane_b32 s6, v4272; GFX942-NEXT: v_readfirstlane_b32 s7, v5273; GFX942-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[2:3]274; GFX942-NEXT: v_readfirstlane_b32 s8, v7275; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[4:5]276; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]277; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, s8, v7278; GFX942-NEXT: s_and_b64 s[0:1], s[0:1], vcc279; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]280; GFX942-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[4:7], s8 idxen offen281; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5282; GFX942-NEXT: ; implicit-def: $vgpr7283; GFX942-NEXT: ; implicit-def: $vgpr0284; GFX942-NEXT: ; implicit-def: $vgpr8_vgpr9285; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]286; GFX942-NEXT: s_cbranch_execnz .LBB4_1287; GFX942-NEXT: ; %bb.2:288; GFX942-NEXT: s_mov_b64 exec, s[2:3]289; GFX942-NEXT: s_waitcnt vmcnt(0)290; GFX942-NEXT: s_setpc_b64 s[30:31]291;292; GFX1200-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:293; GFX1200: ; %bb.0:294; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0295; GFX1200-NEXT: s_wait_expcnt 0x0296; GFX1200-NEXT: s_wait_samplecnt 0x0297; GFX1200-NEXT: s_wait_bvhcnt 0x0298; GFX1200-NEXT: s_wait_kmcnt 0x0299; GFX1200-NEXT: s_mov_b32 s2, exec_lo300; GFX1200-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1301; GFX1200-NEXT: v_readfirstlane_b32 s4, v1302; GFX1200-NEXT: v_readfirstlane_b32 s5, v2303; GFX1200-NEXT: v_readfirstlane_b32 s6, v3304; GFX1200-NEXT: v_readfirstlane_b32 s7, v4305; GFX1200-NEXT: v_readfirstlane_b32 s3, v7306; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)307; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]308; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)309; GFX1200-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]310; GFX1200-NEXT: v_cmp_eq_u32_e64 s1, s3, v7311; GFX1200-NEXT: s_and_b32 s0, vcc_lo, s0312; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)313; GFX1200-NEXT: s_and_b32 s0, s0, s1314; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)315; GFX1200-NEXT: s_and_saveexec_b32 s0, s0316; GFX1200-NEXT: buffer_atomic_add_f32 v0, v[5:6], s[4:7], s3 idxen offen317; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4318; GFX1200-NEXT: ; implicit-def: $vgpr7319; GFX1200-NEXT: ; implicit-def: $vgpr0320; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6321; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s0322; GFX1200-NEXT: s_cbranch_execnz .LBB4_1323; GFX1200-NEXT: ; %bb.2:324; GFX1200-NEXT: s_mov_b32 exec_lo, s2325; GFX1200-NEXT: s_setpc_b64 s[30:31]326;327; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:328; GFX1250: ; %bb.0:329; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0330; GFX1250-NEXT: s_wait_kmcnt 0x0331; GFX1250-NEXT: v_dual_mov_b32 v9, v6 :: v_dual_mov_b32 v8, v5332; GFX1250-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3333; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1334; GFX1250-NEXT: s_mov_b32 s2, exec_lo335; GFX1250-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1336; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)337; GFX1250-NEXT: v_readfirstlane_b32 s4, v2338; GFX1250-NEXT: v_readfirstlane_b32 s5, v3339; GFX1250-NEXT: v_readfirstlane_b32 s6, v4340; GFX1250-NEXT: v_readfirstlane_b32 s7, v5341; GFX1250-NEXT: v_readfirstlane_b32 s3, v7342; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)343; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]344; GFX1250-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[4:5]345; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)346; GFX1250-NEXT: v_cmp_eq_u32_e64 s1, s3, v7347; GFX1250-NEXT: s_and_b32 s0, vcc_lo, s0348; GFX1250-NEXT: s_and_b32 s0, s0, s1349; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)350; GFX1250-NEXT: s_and_saveexec_b32 s0, s0351; GFX1250-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[4:7], s3 idxen offen352; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5353; GFX1250-NEXT: ; implicit-def: $vgpr7354; GFX1250-NEXT: ; implicit-def: $vgpr0355; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9356; GFX1250-NEXT: s_wait_xcnt 0x0357; GFX1250-NEXT: s_xor_b32 exec_lo, exec_lo, s0358; GFX1250-NEXT: s_cbranch_execnz .LBB4_1359; GFX1250-NEXT: ; %bb.2:360; GFX1250-NEXT: s_mov_b32 exec_lo, s2361; GFX1250-NEXT: s_set_pc_i64 s[30:31]362 %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)363 ret void364}365 366; Test waterfall loop367define void @struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset(<2 x half> %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset) #0 {368; GFX908-LABEL: struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:369; GFX908: ; %bb.0:370; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)371; GFX908-NEXT: s_mov_b64 s[6:7], exec372; GFX908-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1373; GFX908-NEXT: v_readfirstlane_b32 s8, v1374; GFX908-NEXT: v_readfirstlane_b32 s9, v2375; GFX908-NEXT: v_readfirstlane_b32 s10, v3376; GFX908-NEXT: v_readfirstlane_b32 s11, v4377; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[1:2]378; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[3:4]379; GFX908-NEXT: v_readfirstlane_b32 s12, v7380; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]381; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, s12, v7382; GFX908-NEXT: s_and_b64 s[4:5], s[4:5], vcc383; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]384; GFX908-NEXT: s_nop 0385; GFX908-NEXT: buffer_atomic_pk_add_f16 v0, v[5:6], s[8:11], s12 idxen offen386; GFX908-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4387; GFX908-NEXT: ; implicit-def: $vgpr7388; GFX908-NEXT: ; implicit-def: $vgpr0389; GFX908-NEXT: ; implicit-def: $vgpr5_vgpr6390; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]391; GFX908-NEXT: s_cbranch_execnz .LBB5_1392; GFX908-NEXT: ; %bb.2:393; GFX908-NEXT: s_mov_b64 exec, s[6:7]394; GFX908-NEXT: s_waitcnt vmcnt(0)395; GFX908-NEXT: s_setpc_b64 s[30:31]396;397; GFX90A-LABEL: struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:398; GFX90A: ; %bb.0:399; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)400; GFX90A-NEXT: v_mov_b32_e32 v9, v6401; GFX90A-NEXT: v_mov_b32_e32 v8, v5402; GFX90A-NEXT: v_mov_b32_e32 v5, v4403; GFX90A-NEXT: v_mov_b32_e32 v4, v3404; GFX90A-NEXT: v_mov_b32_e32 v3, v2405; GFX90A-NEXT: v_mov_b32_e32 v2, v1406; GFX90A-NEXT: s_mov_b64 s[6:7], exec407; GFX90A-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1408; GFX90A-NEXT: v_readfirstlane_b32 s8, v2409; GFX90A-NEXT: v_readfirstlane_b32 s9, v3410; GFX90A-NEXT: v_readfirstlane_b32 s10, v4411; GFX90A-NEXT: v_readfirstlane_b32 s11, v5412; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[2:3]413; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[4:5]414; GFX90A-NEXT: v_readfirstlane_b32 s12, v7415; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]416; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, s12, v7417; GFX90A-NEXT: s_and_b64 s[4:5], s[4:5], vcc418; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]419; GFX90A-NEXT: s_nop 0420; GFX90A-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[8:11], s12 idxen offen421; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5422; GFX90A-NEXT: ; implicit-def: $vgpr7423; GFX90A-NEXT: ; implicit-def: $vgpr0424; GFX90A-NEXT: ; implicit-def: $vgpr8_vgpr9425; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]426; GFX90A-NEXT: s_cbranch_execnz .LBB5_1427; GFX90A-NEXT: ; %bb.2:428; GFX90A-NEXT: s_mov_b64 exec, s[6:7]429; GFX90A-NEXT: s_waitcnt vmcnt(0)430; GFX90A-NEXT: s_setpc_b64 s[30:31]431;432; GFX942-LABEL: struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:433; GFX942: ; %bb.0:434; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)435; GFX942-NEXT: v_mov_b32_e32 v9, v6436; GFX942-NEXT: v_mov_b32_e32 v8, v5437; GFX942-NEXT: v_mov_b32_e32 v5, v4438; GFX942-NEXT: v_mov_b32_e32 v4, v3439; GFX942-NEXT: v_mov_b32_e32 v3, v2440; GFX942-NEXT: v_mov_b32_e32 v2, v1441; GFX942-NEXT: s_mov_b64 s[2:3], exec442; GFX942-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1443; GFX942-NEXT: v_readfirstlane_b32 s4, v2444; GFX942-NEXT: v_readfirstlane_b32 s5, v3445; GFX942-NEXT: v_readfirstlane_b32 s6, v4446; GFX942-NEXT: v_readfirstlane_b32 s7, v5447; GFX942-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[2:3]448; GFX942-NEXT: v_readfirstlane_b32 s8, v7449; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[4:5]450; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]451; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, s8, v7452; GFX942-NEXT: s_and_b64 s[0:1], s[0:1], vcc453; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]454; GFX942-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[4:7], s8 idxen offen455; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5456; GFX942-NEXT: ; implicit-def: $vgpr7457; GFX942-NEXT: ; implicit-def: $vgpr0458; GFX942-NEXT: ; implicit-def: $vgpr8_vgpr9459; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]460; GFX942-NEXT: s_cbranch_execnz .LBB5_1461; GFX942-NEXT: ; %bb.2:462; GFX942-NEXT: s_mov_b64 exec, s[2:3]463; GFX942-NEXT: s_waitcnt vmcnt(0)464; GFX942-NEXT: s_setpc_b64 s[30:31]465;466; GFX1200-LABEL: struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:467; GFX1200: ; %bb.0:468; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0469; GFX1200-NEXT: s_wait_expcnt 0x0470; GFX1200-NEXT: s_wait_samplecnt 0x0471; GFX1200-NEXT: s_wait_bvhcnt 0x0472; GFX1200-NEXT: s_wait_kmcnt 0x0473; GFX1200-NEXT: s_mov_b32 s2, exec_lo474; GFX1200-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1475; GFX1200-NEXT: v_readfirstlane_b32 s4, v1476; GFX1200-NEXT: v_readfirstlane_b32 s5, v2477; GFX1200-NEXT: v_readfirstlane_b32 s6, v3478; GFX1200-NEXT: v_readfirstlane_b32 s7, v4479; GFX1200-NEXT: v_readfirstlane_b32 s3, v7480; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)481; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]482; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)483; GFX1200-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]484; GFX1200-NEXT: v_cmp_eq_u32_e64 s1, s3, v7485; GFX1200-NEXT: s_and_b32 s0, vcc_lo, s0486; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)487; GFX1200-NEXT: s_and_b32 s0, s0, s1488; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)489; GFX1200-NEXT: s_and_saveexec_b32 s0, s0490; GFX1200-NEXT: buffer_atomic_pk_add_f16 v0, v[5:6], s[4:7], s3 idxen offen491; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4492; GFX1200-NEXT: ; implicit-def: $vgpr7493; GFX1200-NEXT: ; implicit-def: $vgpr0494; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6495; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s0496; GFX1200-NEXT: s_cbranch_execnz .LBB5_1497; GFX1200-NEXT: ; %bb.2:498; GFX1200-NEXT: s_mov_b32 exec_lo, s2499; GFX1200-NEXT: s_setpc_b64 s[30:31]500;501; GFX1250-LABEL: struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:502; GFX1250: ; %bb.0:503; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0504; GFX1250-NEXT: s_wait_kmcnt 0x0505; GFX1250-NEXT: v_dual_mov_b32 v9, v6 :: v_dual_mov_b32 v8, v5506; GFX1250-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3507; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1508; GFX1250-NEXT: s_mov_b32 s2, exec_lo509; GFX1250-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1510; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)511; GFX1250-NEXT: v_readfirstlane_b32 s4, v2512; GFX1250-NEXT: v_readfirstlane_b32 s5, v3513; GFX1250-NEXT: v_readfirstlane_b32 s6, v4514; GFX1250-NEXT: v_readfirstlane_b32 s7, v5515; GFX1250-NEXT: v_readfirstlane_b32 s3, v7516; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)517; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]518; GFX1250-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[4:5]519; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)520; GFX1250-NEXT: v_cmp_eq_u32_e64 s1, s3, v7521; GFX1250-NEXT: s_and_b32 s0, vcc_lo, s0522; GFX1250-NEXT: s_and_b32 s0, s0, s1523; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)524; GFX1250-NEXT: s_and_saveexec_b32 s0, s0525; GFX1250-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[4:7], s3 idxen offen526; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5527; GFX1250-NEXT: ; implicit-def: $vgpr7528; GFX1250-NEXT: ; implicit-def: $vgpr0529; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9530; GFX1250-NEXT: s_wait_xcnt 0x0531; GFX1250-NEXT: s_xor_b32 exec_lo, exec_lo, s0532; GFX1250-NEXT: s_cbranch_execnz .LBB5_1533; GFX1250-NEXT: ; %bb.2:534; GFX1250-NEXT: s_mov_b32 exec_lo, s2535; GFX1250-NEXT: s_set_pc_i64 s[30:31]536 %ret = call <2 x half> @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2f16(<2 x half> %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)537 ret void538}539 540declare float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float, ptr addrspace(8), i32, i32, i32, i32 immarg)541declare <2 x half> @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2f16(<2 x half>, ptr addrspace(8), i32, i32, i32, i32 immarg)542 543attributes #0 = { nounwind }544