2423 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx90a -amdgpu-atomic-optimizer-strategy=None | FileCheck %s -check-prefix=GFX90A3; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx942 -amdgpu-atomic-optimizer-strategy=None | FileCheck %s -check-prefix=GFX9424; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1250 -amdgpu-atomic-optimizer-strategy=None | FileCheck %s -check-prefix=GFX12505 6declare double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double, <4 x i32>, i32, i32, i32, i32 immarg)7declare double @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f64(double, ptr addrspace(8), i32, i32, i32, i32 immarg)8declare double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double, <4 x i32>, i32, i32, i32 immarg)9declare double @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f64(double, ptr addrspace(8), i32, i32, i32 immarg)10declare double @llvm.amdgcn.struct.buffer.atomic.fmin.f64(double, <4 x i32>, i32, i32, i32, i32 immarg)11declare double @llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f64(double, ptr addrspace(8), i32, i32, i32, i32 immarg)12declare double @llvm.amdgcn.raw.buffer.atomic.fmin.f64(double, <4 x i32>, i32, i32, i32 immarg)13declare double @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f64(double, ptr addrspace(8), i32, i32, i32 immarg)14declare double @llvm.amdgcn.struct.buffer.atomic.fmax.f64(double, <4 x i32>, i32, i32, i32, i32 immarg)15declare double @llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f64(double, ptr addrspace(8), i32, i32, i32, i32 immarg)16declare double @llvm.amdgcn.raw.buffer.atomic.fmax.f64(double, <4 x i32>, i32, i32, i32 immarg)17declare double @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f64(double, ptr addrspace(8), i32, i32, i32 immarg)18declare double @llvm.amdgcn.ds.fadd.f64(ptr addrspace(3) nocapture, double, i32, i32, i1)19 20define amdgpu_kernel void @raw_buffer_atomic_add_noret_f64(<4 x i32> %rsrc, double %data, i32 %vindex) {21; GFX90A-LABEL: raw_buffer_atomic_add_noret_f64:22; GFX90A: ; %bb.0: ; %main_body23; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3424; GFX90A-NEXT: s_load_dword s8, s[4:5], 0x3c25; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2426; GFX90A-NEXT: s_waitcnt lgkmcnt(0)27; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]28; GFX90A-NEXT: v_mov_b32_e32 v0, s829; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 offen30; GFX90A-NEXT: s_endpgm31;32; GFX942-LABEL: raw_buffer_atomic_add_noret_f64:33; GFX942: ; %bb.0: ; %main_body34; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3435; GFX942-NEXT: s_load_dword s8, s[4:5], 0x3c36; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2437; GFX942-NEXT: s_waitcnt lgkmcnt(0)38; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]39; GFX942-NEXT: v_mov_b32_e32 v0, s840; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 offen41; GFX942-NEXT: s_endpgm42;43; GFX1250-LABEL: raw_buffer_atomic_add_noret_f64:44; GFX1250: ; %bb.0: ; %main_body45; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 146; GFX1250-NEXT: s_clause 0x147; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x3448; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x2449; GFX1250-NEXT: s_wait_kmcnt 0x050; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]51; GFX1250-NEXT: v_mov_b32_e32 v2, s1052; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null offen53; GFX1250-NEXT: s_endpgm54main_body:55 %ret = call double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0)56 ret void57}58 59define amdgpu_ps void @raw_buffer_atomic_add_rtn_f64(<4 x i32> inreg %rsrc, double %data, i32 %vindex) {60; GFX90A-LABEL: raw_buffer_atomic_add_rtn_f64:61; GFX90A: ; %bb.0: ; %main_body62; GFX90A-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 offen glc63; GFX90A-NEXT: s_waitcnt vmcnt(0)64; GFX90A-NEXT: flat_store_dwordx2 v[0:1], v[0:1]65; GFX90A-NEXT: s_endpgm66;67; GFX942-LABEL: raw_buffer_atomic_add_rtn_f64:68; GFX942: ; %bb.0: ; %main_body69; GFX942-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 offen sc070; GFX942-NEXT: s_waitcnt vmcnt(0)71; GFX942-NEXT: flat_store_dwordx2 v[0:1], v[0:1]72; GFX942-NEXT: s_endpgm73;74; GFX1250-LABEL: raw_buffer_atomic_add_rtn_f64:75; GFX1250: ; %bb.0: ; %main_body76; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 177; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN78; GFX1250-NEXT: s_wait_loadcnt 0x079; GFX1250-NEXT: flat_store_b64 v[0:1], v[0:1]80; GFX1250-NEXT: s_endpgm81main_body:82 %ret = call double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0)83 store double %ret, ptr poison84 ret void85}86 87define amdgpu_kernel void @raw_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {88; GFX90A-LABEL: raw_buffer_atomic_add_rtn_f64_off4_slc:89; GFX90A: ; %bb.0: ; %main_body90; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3491; GFX90A-NEXT: s_load_dword s10, s[4:5], 0x3c92; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2493; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x4494; GFX90A-NEXT: s_waitcnt lgkmcnt(0)95; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]96; GFX90A-NEXT: v_mov_b32_e32 v0, s1097; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 4 offen glc slc98; GFX90A-NEXT: v_mov_b32_e32 v0, 099; GFX90A-NEXT: s_waitcnt vmcnt(0)100; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]101; GFX90A-NEXT: s_endpgm102;103; GFX942-LABEL: raw_buffer_atomic_add_rtn_f64_off4_slc:104; GFX942: ; %bb.0: ; %main_body105; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34106; GFX942-NEXT: s_load_dword s10, s[4:5], 0x3c107; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24108; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44109; GFX942-NEXT: s_waitcnt lgkmcnt(0)110; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]111; GFX942-NEXT: v_mov_b32_e32 v0, s10112; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt113; GFX942-NEXT: v_mov_b32_e32 v0, 0114; GFX942-NEXT: s_waitcnt vmcnt(0)115; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]116; GFX942-NEXT: s_endpgm117;118; GFX1250-LABEL: raw_buffer_atomic_add_rtn_f64_off4_slc:119; GFX1250: ; %bb.0: ; %main_body120; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1121; GFX1250-NEXT: s_clause 0x1122; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34123; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24124; GFX1250-NEXT: s_mov_b32 s6, 4125; GFX1250-NEXT: s_wait_kmcnt 0x0126; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]127; GFX1250-NEXT: v_mov_b32_e32 v2, s10128; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN129; GFX1250-NEXT: s_wait_xcnt 0x0130; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44131; GFX1250-NEXT: v_mov_b32_e32 v2, 0132; GFX1250-NEXT: s_wait_loadcnt 0x0133; GFX1250-NEXT: s_wait_kmcnt 0x0134; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]135; GFX1250-NEXT: s_endpgm136main_body:137 %ret = call double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 2)138 store double %ret, ptr addrspace(1) %out, align 8139 ret void140}141 142define amdgpu_kernel void @raw_ptr_buffer_atomic_add_noret_f64(ptr addrspace(8) %rsrc, double %data, i32 %vindex) {143; GFX90A-LABEL: raw_ptr_buffer_atomic_add_noret_f64:144; GFX90A: ; %bb.0: ; %main_body145; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34146; GFX90A-NEXT: s_load_dword s8, s[4:5], 0x3c147; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24148; GFX90A-NEXT: s_waitcnt lgkmcnt(0)149; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]150; GFX90A-NEXT: v_mov_b32_e32 v0, s8151; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 offen152; GFX90A-NEXT: s_endpgm153;154; GFX942-LABEL: raw_ptr_buffer_atomic_add_noret_f64:155; GFX942: ; %bb.0: ; %main_body156; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34157; GFX942-NEXT: s_load_dword s8, s[4:5], 0x3c158; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24159; GFX942-NEXT: s_waitcnt lgkmcnt(0)160; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]161; GFX942-NEXT: v_mov_b32_e32 v0, s8162; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 offen163; GFX942-NEXT: s_endpgm164;165; GFX1250-LABEL: raw_ptr_buffer_atomic_add_noret_f64:166; GFX1250: ; %bb.0: ; %main_body167; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1168; GFX1250-NEXT: s_clause 0x1169; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34170; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24171; GFX1250-NEXT: s_wait_kmcnt 0x0172; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]173; GFX1250-NEXT: v_mov_b32_e32 v2, s10174; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null offen175; GFX1250-NEXT: s_endpgm176main_body:177 %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0)178 ret void179}180 181define amdgpu_ps void @raw_ptr_buffer_atomic_add_rtn_f64(ptr addrspace(8) inreg %rsrc, double %data, i32 %vindex) {182; GFX90A-LABEL: raw_ptr_buffer_atomic_add_rtn_f64:183; GFX90A: ; %bb.0: ; %main_body184; GFX90A-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 offen glc185; GFX90A-NEXT: s_waitcnt vmcnt(0)186; GFX90A-NEXT: flat_store_dwordx2 v[0:1], v[0:1]187; GFX90A-NEXT: s_endpgm188;189; GFX942-LABEL: raw_ptr_buffer_atomic_add_rtn_f64:190; GFX942: ; %bb.0: ; %main_body191; GFX942-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 offen sc0192; GFX942-NEXT: s_waitcnt vmcnt(0)193; GFX942-NEXT: flat_store_dwordx2 v[0:1], v[0:1]194; GFX942-NEXT: s_endpgm195;196; GFX1250-LABEL: raw_ptr_buffer_atomic_add_rtn_f64:197; GFX1250: ; %bb.0: ; %main_body198; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1199; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN200; GFX1250-NEXT: s_wait_loadcnt 0x0201; GFX1250-NEXT: flat_store_b64 v[0:1], v[0:1]202; GFX1250-NEXT: s_endpgm203main_body:204 %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0)205 store double %ret, ptr poison206 ret void207}208 209define amdgpu_kernel void @raw_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr addrspace(8) %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {210; GFX90A-LABEL: raw_ptr_buffer_atomic_add_rtn_f64_off4_slc:211; GFX90A: ; %bb.0: ; %main_body212; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34213; GFX90A-NEXT: s_load_dword s10, s[4:5], 0x3c214; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24215; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44216; GFX90A-NEXT: s_waitcnt lgkmcnt(0)217; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]218; GFX90A-NEXT: v_mov_b32_e32 v0, s10219; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 4 offen glc slc220; GFX90A-NEXT: v_mov_b32_e32 v0, 0221; GFX90A-NEXT: s_waitcnt vmcnt(0)222; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]223; GFX90A-NEXT: s_endpgm224;225; GFX942-LABEL: raw_ptr_buffer_atomic_add_rtn_f64_off4_slc:226; GFX942: ; %bb.0: ; %main_body227; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34228; GFX942-NEXT: s_load_dword s10, s[4:5], 0x3c229; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24230; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44231; GFX942-NEXT: s_waitcnt lgkmcnt(0)232; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]233; GFX942-NEXT: v_mov_b32_e32 v0, s10234; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt235; GFX942-NEXT: v_mov_b32_e32 v0, 0236; GFX942-NEXT: s_waitcnt vmcnt(0)237; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]238; GFX942-NEXT: s_endpgm239;240; GFX1250-LABEL: raw_ptr_buffer_atomic_add_rtn_f64_off4_slc:241; GFX1250: ; %bb.0: ; %main_body242; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1243; GFX1250-NEXT: s_clause 0x1244; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34245; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24246; GFX1250-NEXT: s_mov_b32 s6, 4247; GFX1250-NEXT: s_wait_kmcnt 0x0248; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]249; GFX1250-NEXT: v_mov_b32_e32 v2, s10250; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN251; GFX1250-NEXT: s_wait_xcnt 0x0252; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44253; GFX1250-NEXT: v_mov_b32_e32 v2, 0254; GFX1250-NEXT: s_wait_loadcnt 0x0255; GFX1250-NEXT: s_wait_kmcnt 0x0256; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]257; GFX1250-NEXT: s_endpgm258main_body:259 %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 2)260 store double %ret, ptr addrspace(1) %out, align 8261 ret void262}263 264define amdgpu_kernel void @struct_buffer_atomic_add_noret_f64(<4 x i32> %rsrc, double %data, i32 %vindex) {265; GFX90A-LABEL: struct_buffer_atomic_add_noret_f64:266; GFX90A: ; %bb.0: ; %main_body267; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34268; GFX90A-NEXT: s_load_dword s8, s[4:5], 0x3c269; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24270; GFX90A-NEXT: s_waitcnt lgkmcnt(0)271; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]272; GFX90A-NEXT: v_mov_b32_e32 v0, s8273; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen274; GFX90A-NEXT: s_endpgm275;276; GFX942-LABEL: struct_buffer_atomic_add_noret_f64:277; GFX942: ; %bb.0: ; %main_body278; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34279; GFX942-NEXT: s_load_dword s8, s[4:5], 0x3c280; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24281; GFX942-NEXT: s_waitcnt lgkmcnt(0)282; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]283; GFX942-NEXT: v_mov_b32_e32 v0, s8284; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen285; GFX942-NEXT: s_endpgm286;287; GFX1250-LABEL: struct_buffer_atomic_add_noret_f64:288; GFX1250: ; %bb.0: ; %main_body289; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1290; GFX1250-NEXT: s_clause 0x1291; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34292; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24293; GFX1250-NEXT: s_wait_kmcnt 0x0294; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]295; GFX1250-NEXT: v_mov_b32_e32 v2, s10296; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen297; GFX1250-NEXT: s_endpgm298main_body:299 %ret = call double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0)300 ret void301}302 303define amdgpu_ps void @struct_buffer_atomic_add_rtn_f64(<4 x i32> inreg %rsrc, double %data, i32 %vindex) {304; GFX90A-LABEL: struct_buffer_atomic_add_rtn_f64:305; GFX90A: ; %bb.0: ; %main_body306; GFX90A-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 idxen glc307; GFX90A-NEXT: s_waitcnt vmcnt(0)308; GFX90A-NEXT: flat_store_dwordx2 v[0:1], v[0:1]309; GFX90A-NEXT: s_endpgm310;311; GFX942-LABEL: struct_buffer_atomic_add_rtn_f64:312; GFX942: ; %bb.0: ; %main_body313; GFX942-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 idxen sc0314; GFX942-NEXT: s_waitcnt vmcnt(0)315; GFX942-NEXT: flat_store_dwordx2 v[0:1], v[0:1]316; GFX942-NEXT: s_endpgm317;318; GFX1250-LABEL: struct_buffer_atomic_add_rtn_f64:319; GFX1250: ; %bb.0: ; %main_body320; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1321; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN322; GFX1250-NEXT: s_wait_loadcnt 0x0323; GFX1250-NEXT: flat_store_b64 v[0:1], v[0:1]324; GFX1250-NEXT: s_endpgm325main_body:326 %ret = call double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0)327 store double %ret, ptr poison328 ret void329}330 331define amdgpu_kernel void @struct_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {332; GFX90A-LABEL: struct_buffer_atomic_add_rtn_f64_off4_slc:333; GFX90A: ; %bb.0: ; %main_body334; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34335; GFX90A-NEXT: s_load_dword s10, s[4:5], 0x3c336; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24337; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44338; GFX90A-NEXT: s_waitcnt lgkmcnt(0)339; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]340; GFX90A-NEXT: v_mov_b32_e32 v0, s10341; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc342; GFX90A-NEXT: v_mov_b32_e32 v0, 0343; GFX90A-NEXT: s_waitcnt vmcnt(0)344; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]345; GFX90A-NEXT: s_endpgm346;347; GFX942-LABEL: struct_buffer_atomic_add_rtn_f64_off4_slc:348; GFX942: ; %bb.0: ; %main_body349; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34350; GFX942-NEXT: s_load_dword s10, s[4:5], 0x3c351; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24352; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44353; GFX942-NEXT: s_waitcnt lgkmcnt(0)354; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]355; GFX942-NEXT: v_mov_b32_e32 v0, s10356; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt357; GFX942-NEXT: v_mov_b32_e32 v0, 0358; GFX942-NEXT: s_waitcnt vmcnt(0)359; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]360; GFX942-NEXT: s_endpgm361;362; GFX1250-LABEL: struct_buffer_atomic_add_rtn_f64_off4_slc:363; GFX1250: ; %bb.0: ; %main_body364; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1365; GFX1250-NEXT: s_clause 0x1366; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34367; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24368; GFX1250-NEXT: s_wait_kmcnt 0x0369; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]370; GFX1250-NEXT: v_mov_b32_e32 v2, s10371; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN372; GFX1250-NEXT: s_wait_xcnt 0x0373; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44374; GFX1250-NEXT: v_mov_b32_e32 v2, 0375; GFX1250-NEXT: s_wait_loadcnt 0x0376; GFX1250-NEXT: s_wait_kmcnt 0x0377; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]378; GFX1250-NEXT: s_endpgm379main_body:380 %ret = call double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 0, i32 2)381 store double %ret, ptr addrspace(1) %out, align 8382 ret void383}384 385define amdgpu_kernel void @struct_ptr_buffer_atomic_add_noret_f64(ptr addrspace(8) %rsrc, double %data, i32 %vindex) {386; GFX90A-LABEL: struct_ptr_buffer_atomic_add_noret_f64:387; GFX90A: ; %bb.0: ; %main_body388; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34389; GFX90A-NEXT: s_load_dword s8, s[4:5], 0x3c390; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24391; GFX90A-NEXT: s_waitcnt lgkmcnt(0)392; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]393; GFX90A-NEXT: v_mov_b32_e32 v0, s8394; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen395; GFX90A-NEXT: s_endpgm396;397; GFX942-LABEL: struct_ptr_buffer_atomic_add_noret_f64:398; GFX942: ; %bb.0: ; %main_body399; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34400; GFX942-NEXT: s_load_dword s8, s[4:5], 0x3c401; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24402; GFX942-NEXT: s_waitcnt lgkmcnt(0)403; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]404; GFX942-NEXT: v_mov_b32_e32 v0, s8405; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen406; GFX942-NEXT: s_endpgm407;408; GFX1250-LABEL: struct_ptr_buffer_atomic_add_noret_f64:409; GFX1250: ; %bb.0: ; %main_body410; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1411; GFX1250-NEXT: s_clause 0x1412; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34413; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24414; GFX1250-NEXT: s_wait_kmcnt 0x0415; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]416; GFX1250-NEXT: v_mov_b32_e32 v2, s10417; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen418; GFX1250-NEXT: s_endpgm419main_body:420 %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0, i32 0)421 ret void422}423 424define amdgpu_ps void @struct_ptr_buffer_atomic_add_rtn_f64(ptr addrspace(8) inreg %rsrc, double %data, i32 %vindex) {425; GFX90A-LABEL: struct_ptr_buffer_atomic_add_rtn_f64:426; GFX90A: ; %bb.0: ; %main_body427; GFX90A-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 idxen glc428; GFX90A-NEXT: s_waitcnt vmcnt(0)429; GFX90A-NEXT: flat_store_dwordx2 v[0:1], v[0:1]430; GFX90A-NEXT: s_endpgm431;432; GFX942-LABEL: struct_ptr_buffer_atomic_add_rtn_f64:433; GFX942: ; %bb.0: ; %main_body434; GFX942-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 idxen sc0435; GFX942-NEXT: s_waitcnt vmcnt(0)436; GFX942-NEXT: flat_store_dwordx2 v[0:1], v[0:1]437; GFX942-NEXT: s_endpgm438;439; GFX1250-LABEL: struct_ptr_buffer_atomic_add_rtn_f64:440; GFX1250: ; %bb.0: ; %main_body441; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1442; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN443; GFX1250-NEXT: s_wait_loadcnt 0x0444; GFX1250-NEXT: flat_store_b64 v[0:1], v[0:1]445; GFX1250-NEXT: s_endpgm446main_body:447 %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0, i32 0)448 store double %ret, ptr poison449 ret void450}451 452define amdgpu_kernel void @struct_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr addrspace(8) %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {453; GFX90A-LABEL: struct_ptr_buffer_atomic_add_rtn_f64_off4_slc:454; GFX90A: ; %bb.0: ; %main_body455; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34456; GFX90A-NEXT: s_load_dword s10, s[4:5], 0x3c457; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24458; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44459; GFX90A-NEXT: s_waitcnt lgkmcnt(0)460; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]461; GFX90A-NEXT: v_mov_b32_e32 v0, s10462; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc463; GFX90A-NEXT: v_mov_b32_e32 v0, 0464; GFX90A-NEXT: s_waitcnt vmcnt(0)465; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]466; GFX90A-NEXT: s_endpgm467;468; GFX942-LABEL: struct_ptr_buffer_atomic_add_rtn_f64_off4_slc:469; GFX942: ; %bb.0: ; %main_body470; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34471; GFX942-NEXT: s_load_dword s10, s[4:5], 0x3c472; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24473; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44474; GFX942-NEXT: s_waitcnt lgkmcnt(0)475; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]476; GFX942-NEXT: v_mov_b32_e32 v0, s10477; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt478; GFX942-NEXT: v_mov_b32_e32 v0, 0479; GFX942-NEXT: s_waitcnt vmcnt(0)480; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]481; GFX942-NEXT: s_endpgm482;483; GFX1250-LABEL: struct_ptr_buffer_atomic_add_rtn_f64_off4_slc:484; GFX1250: ; %bb.0: ; %main_body485; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1486; GFX1250-NEXT: s_clause 0x1487; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34488; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24489; GFX1250-NEXT: s_wait_kmcnt 0x0490; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]491; GFX1250-NEXT: v_mov_b32_e32 v2, s10492; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN493; GFX1250-NEXT: s_wait_xcnt 0x0494; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44495; GFX1250-NEXT: v_mov_b32_e32 v2, 0496; GFX1250-NEXT: s_wait_loadcnt 0x0497; GFX1250-NEXT: s_wait_kmcnt 0x0498; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]499; GFX1250-NEXT: s_endpgm500main_body:501 %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 0, i32 2)502 store double %ret, ptr addrspace(1) %out, align 8503 ret void504}505 506define amdgpu_kernel void @raw_buffer_atomic_min_noret_f64(<4 x i32> %rsrc, double %data, i32 %vindex) {507; GFX90A-LABEL: raw_buffer_atomic_min_noret_f64:508; GFX90A: ; %bb.0: ; %main_body509; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34510; GFX90A-NEXT: s_load_dword s8, s[4:5], 0x3c511; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24512; GFX90A-NEXT: s_waitcnt lgkmcnt(0)513; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]514; GFX90A-NEXT: v_mov_b32_e32 v0, s8515; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 offen516; GFX90A-NEXT: s_endpgm517;518; GFX942-LABEL: raw_buffer_atomic_min_noret_f64:519; GFX942: ; %bb.0: ; %main_body520; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34521; GFX942-NEXT: s_load_dword s8, s[4:5], 0x3c522; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24523; GFX942-NEXT: s_waitcnt lgkmcnt(0)524; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]525; GFX942-NEXT: v_mov_b32_e32 v0, s8526; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 offen527; GFX942-NEXT: s_endpgm528;529; GFX1250-LABEL: raw_buffer_atomic_min_noret_f64:530; GFX1250: ; %bb.0: ; %main_body531; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1532; GFX1250-NEXT: s_clause 0x1533; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34534; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24535; GFX1250-NEXT: s_wait_kmcnt 0x0536; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]537; GFX1250-NEXT: v_mov_b32_e32 v2, s10538; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null offen539; GFX1250-NEXT: s_endpgm540main_body:541 %ret = call double @llvm.amdgcn.raw.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0)542 ret void543}544 545define amdgpu_ps void @raw_buffer_atomic_min_rtn_f64(<4 x i32> inreg %rsrc, double %data, i32 %vindex) {546; GFX90A-LABEL: raw_buffer_atomic_min_rtn_f64:547; GFX90A: ; %bb.0: ; %main_body548; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen glc549; GFX90A-NEXT: s_waitcnt vmcnt(0)550; GFX90A-NEXT: flat_store_dwordx2 v[0:1], v[0:1]551; GFX90A-NEXT: s_endpgm552;553; GFX942-LABEL: raw_buffer_atomic_min_rtn_f64:554; GFX942: ; %bb.0: ; %main_body555; GFX942-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen sc0556; GFX942-NEXT: s_waitcnt vmcnt(0)557; GFX942-NEXT: flat_store_dwordx2 v[0:1], v[0:1]558; GFX942-NEXT: s_endpgm559;560; GFX1250-LABEL: raw_buffer_atomic_min_rtn_f64:561; GFX1250: ; %bb.0: ; %main_body562; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1563; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN564; GFX1250-NEXT: s_wait_loadcnt 0x0565; GFX1250-NEXT: flat_store_b64 v[0:1], v[0:1]566; GFX1250-NEXT: s_endpgm567main_body:568 %ret = call double @llvm.amdgcn.raw.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0)569 store double %ret, ptr poison570 ret void571}572 573define amdgpu_kernel void @raw_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {574; GFX90A-LABEL: raw_buffer_atomic_min_rtn_f64_off4_slc:575; GFX90A: ; %bb.0: ; %main_body576; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34577; GFX90A-NEXT: s_load_dword s10, s[4:5], 0x3c578; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24579; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44580; GFX90A-NEXT: s_waitcnt lgkmcnt(0)581; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]582; GFX90A-NEXT: v_mov_b32_e32 v0, s10583; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 4 offen glc slc584; GFX90A-NEXT: v_mov_b32_e32 v0, 0585; GFX90A-NEXT: s_waitcnt vmcnt(0)586; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]587; GFX90A-NEXT: s_endpgm588;589; GFX942-LABEL: raw_buffer_atomic_min_rtn_f64_off4_slc:590; GFX942: ; %bb.0: ; %main_body591; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34592; GFX942-NEXT: s_load_dword s10, s[4:5], 0x3c593; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24594; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44595; GFX942-NEXT: s_waitcnt lgkmcnt(0)596; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]597; GFX942-NEXT: v_mov_b32_e32 v0, s10598; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt599; GFX942-NEXT: v_mov_b32_e32 v0, 0600; GFX942-NEXT: s_waitcnt vmcnt(0)601; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]602; GFX942-NEXT: s_endpgm603;604; GFX1250-LABEL: raw_buffer_atomic_min_rtn_f64_off4_slc:605; GFX1250: ; %bb.0: ; %main_body606; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1607; GFX1250-NEXT: s_clause 0x1608; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34609; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24610; GFX1250-NEXT: s_mov_b32 s6, 4611; GFX1250-NEXT: s_wait_kmcnt 0x0612; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]613; GFX1250-NEXT: v_mov_b32_e32 v2, s10614; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN615; GFX1250-NEXT: s_wait_xcnt 0x0616; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44617; GFX1250-NEXT: v_mov_b32_e32 v2, 0618; GFX1250-NEXT: s_wait_loadcnt 0x0619; GFX1250-NEXT: s_wait_kmcnt 0x0620; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]621; GFX1250-NEXT: s_endpgm622main_body:623 %ret = call double @llvm.amdgcn.raw.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 2)624 store double %ret, ptr addrspace(1) %out, align 8625 ret void626}627 628define amdgpu_kernel void @raw_ptr_buffer_atomic_min_noret_f64(ptr addrspace(8) %rsrc, double %data, i32 %vindex) {629; GFX90A-LABEL: raw_ptr_buffer_atomic_min_noret_f64:630; GFX90A: ; %bb.0: ; %main_body631; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34632; GFX90A-NEXT: s_load_dword s8, s[4:5], 0x3c633; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24634; GFX90A-NEXT: s_waitcnt lgkmcnt(0)635; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]636; GFX90A-NEXT: v_mov_b32_e32 v0, s8637; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 offen638; GFX90A-NEXT: s_endpgm639;640; GFX942-LABEL: raw_ptr_buffer_atomic_min_noret_f64:641; GFX942: ; %bb.0: ; %main_body642; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34643; GFX942-NEXT: s_load_dword s8, s[4:5], 0x3c644; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24645; GFX942-NEXT: s_waitcnt lgkmcnt(0)646; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]647; GFX942-NEXT: v_mov_b32_e32 v0, s8648; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 offen649; GFX942-NEXT: s_endpgm650;651; GFX1250-LABEL: raw_ptr_buffer_atomic_min_noret_f64:652; GFX1250: ; %bb.0: ; %main_body653; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1654; GFX1250-NEXT: s_clause 0x1655; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34656; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24657; GFX1250-NEXT: s_wait_kmcnt 0x0658; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]659; GFX1250-NEXT: v_mov_b32_e32 v2, s10660; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null offen661; GFX1250-NEXT: s_endpgm662main_body:663 %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0)664 ret void665}666 667define amdgpu_ps void @raw_ptr_buffer_atomic_min_rtn_f64(ptr addrspace(8) inreg %rsrc, double %data, i32 %vindex) {668; GFX90A-LABEL: raw_ptr_buffer_atomic_min_rtn_f64:669; GFX90A: ; %bb.0: ; %main_body670; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen glc671; GFX90A-NEXT: s_waitcnt vmcnt(0)672; GFX90A-NEXT: flat_store_dwordx2 v[0:1], v[0:1]673; GFX90A-NEXT: s_endpgm674;675; GFX942-LABEL: raw_ptr_buffer_atomic_min_rtn_f64:676; GFX942: ; %bb.0: ; %main_body677; GFX942-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen sc0678; GFX942-NEXT: s_waitcnt vmcnt(0)679; GFX942-NEXT: flat_store_dwordx2 v[0:1], v[0:1]680; GFX942-NEXT: s_endpgm681;682; GFX1250-LABEL: raw_ptr_buffer_atomic_min_rtn_f64:683; GFX1250: ; %bb.0: ; %main_body684; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1685; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN686; GFX1250-NEXT: s_wait_loadcnt 0x0687; GFX1250-NEXT: flat_store_b64 v[0:1], v[0:1]688; GFX1250-NEXT: s_endpgm689main_body:690 %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0)691 store double %ret, ptr poison692 ret void693}694 695define amdgpu_kernel void @raw_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr addrspace(8) %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {696; GFX90A-LABEL: raw_ptr_buffer_atomic_min_rtn_f64_off4_slc:697; GFX90A: ; %bb.0: ; %main_body698; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34699; GFX90A-NEXT: s_load_dword s10, s[4:5], 0x3c700; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24701; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44702; GFX90A-NEXT: s_waitcnt lgkmcnt(0)703; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]704; GFX90A-NEXT: v_mov_b32_e32 v0, s10705; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 4 offen glc slc706; GFX90A-NEXT: v_mov_b32_e32 v0, 0707; GFX90A-NEXT: s_waitcnt vmcnt(0)708; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]709; GFX90A-NEXT: s_endpgm710;711; GFX942-LABEL: raw_ptr_buffer_atomic_min_rtn_f64_off4_slc:712; GFX942: ; %bb.0: ; %main_body713; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34714; GFX942-NEXT: s_load_dword s10, s[4:5], 0x3c715; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24716; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44717; GFX942-NEXT: s_waitcnt lgkmcnt(0)718; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]719; GFX942-NEXT: v_mov_b32_e32 v0, s10720; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt721; GFX942-NEXT: v_mov_b32_e32 v0, 0722; GFX942-NEXT: s_waitcnt vmcnt(0)723; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]724; GFX942-NEXT: s_endpgm725;726; GFX1250-LABEL: raw_ptr_buffer_atomic_min_rtn_f64_off4_slc:727; GFX1250: ; %bb.0: ; %main_body728; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1729; GFX1250-NEXT: s_clause 0x1730; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34731; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24732; GFX1250-NEXT: s_mov_b32 s6, 4733; GFX1250-NEXT: s_wait_kmcnt 0x0734; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]735; GFX1250-NEXT: v_mov_b32_e32 v2, s10736; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN737; GFX1250-NEXT: s_wait_xcnt 0x0738; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44739; GFX1250-NEXT: v_mov_b32_e32 v2, 0740; GFX1250-NEXT: s_wait_loadcnt 0x0741; GFX1250-NEXT: s_wait_kmcnt 0x0742; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]743; GFX1250-NEXT: s_endpgm744main_body:745 %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 2)746 store double %ret, ptr addrspace(1) %out, align 8747 ret void748}749 750define amdgpu_kernel void @struct_buffer_atomic_min_noret_f64(<4 x i32> %rsrc, double %data, i32 %vindex) {751; GFX90A-LABEL: struct_buffer_atomic_min_noret_f64:752; GFX90A: ; %bb.0: ; %main_body753; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34754; GFX90A-NEXT: s_load_dword s8, s[4:5], 0x3c755; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24756; GFX90A-NEXT: s_waitcnt lgkmcnt(0)757; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]758; GFX90A-NEXT: v_mov_b32_e32 v0, s8759; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen760; GFX90A-NEXT: s_endpgm761;762; GFX942-LABEL: struct_buffer_atomic_min_noret_f64:763; GFX942: ; %bb.0: ; %main_body764; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34765; GFX942-NEXT: s_load_dword s8, s[4:5], 0x3c766; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24767; GFX942-NEXT: s_waitcnt lgkmcnt(0)768; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]769; GFX942-NEXT: v_mov_b32_e32 v0, s8770; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen771; GFX942-NEXT: s_endpgm772;773; GFX1250-LABEL: struct_buffer_atomic_min_noret_f64:774; GFX1250: ; %bb.0: ; %main_body775; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1776; GFX1250-NEXT: s_clause 0x1777; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34778; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24779; GFX1250-NEXT: s_wait_kmcnt 0x0780; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]781; GFX1250-NEXT: v_mov_b32_e32 v2, s10782; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen783; GFX1250-NEXT: s_endpgm784main_body:785 %ret = call double @llvm.amdgcn.struct.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0)786 ret void787}788 789define amdgpu_ps void @struct_buffer_atomic_min_rtn_f64(<4 x i32> inreg %rsrc, double %data, i32 %vindex) {790; GFX90A-LABEL: struct_buffer_atomic_min_rtn_f64:791; GFX90A: ; %bb.0: ; %main_body792; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 idxen glc793; GFX90A-NEXT: s_waitcnt vmcnt(0)794; GFX90A-NEXT: flat_store_dwordx2 v[0:1], v[0:1]795; GFX90A-NEXT: s_endpgm796;797; GFX942-LABEL: struct_buffer_atomic_min_rtn_f64:798; GFX942: ; %bb.0: ; %main_body799; GFX942-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 idxen sc0800; GFX942-NEXT: s_waitcnt vmcnt(0)801; GFX942-NEXT: flat_store_dwordx2 v[0:1], v[0:1]802; GFX942-NEXT: s_endpgm803;804; GFX1250-LABEL: struct_buffer_atomic_min_rtn_f64:805; GFX1250: ; %bb.0: ; %main_body806; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1807; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN808; GFX1250-NEXT: s_wait_loadcnt 0x0809; GFX1250-NEXT: flat_store_b64 v[0:1], v[0:1]810; GFX1250-NEXT: s_endpgm811main_body:812 %ret = call double @llvm.amdgcn.struct.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0)813 store double %ret, ptr poison814 ret void815}816 817define amdgpu_kernel void @struct_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {818; GFX90A-LABEL: struct_buffer_atomic_min_rtn_f64_off4_slc:819; GFX90A: ; %bb.0: ; %main_body820; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34821; GFX90A-NEXT: s_load_dword s10, s[4:5], 0x3c822; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24823; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44824; GFX90A-NEXT: s_waitcnt lgkmcnt(0)825; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]826; GFX90A-NEXT: v_mov_b32_e32 v0, s10827; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc828; GFX90A-NEXT: v_mov_b32_e32 v0, 0829; GFX90A-NEXT: s_waitcnt vmcnt(0)830; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]831; GFX90A-NEXT: s_endpgm832;833; GFX942-LABEL: struct_buffer_atomic_min_rtn_f64_off4_slc:834; GFX942: ; %bb.0: ; %main_body835; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34836; GFX942-NEXT: s_load_dword s10, s[4:5], 0x3c837; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24838; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44839; GFX942-NEXT: s_waitcnt lgkmcnt(0)840; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]841; GFX942-NEXT: v_mov_b32_e32 v0, s10842; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt843; GFX942-NEXT: v_mov_b32_e32 v0, 0844; GFX942-NEXT: s_waitcnt vmcnt(0)845; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]846; GFX942-NEXT: s_endpgm847;848; GFX1250-LABEL: struct_buffer_atomic_min_rtn_f64_off4_slc:849; GFX1250: ; %bb.0: ; %main_body850; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1851; GFX1250-NEXT: s_clause 0x1852; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34853; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24854; GFX1250-NEXT: s_wait_kmcnt 0x0855; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]856; GFX1250-NEXT: v_mov_b32_e32 v2, s10857; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN858; GFX1250-NEXT: s_wait_xcnt 0x0859; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44860; GFX1250-NEXT: v_mov_b32_e32 v2, 0861; GFX1250-NEXT: s_wait_loadcnt 0x0862; GFX1250-NEXT: s_wait_kmcnt 0x0863; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]864; GFX1250-NEXT: s_endpgm865main_body:866 %ret = call double @llvm.amdgcn.struct.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 0, i32 2)867 store double %ret, ptr addrspace(1) %out, align 8868 ret void869}870 871define amdgpu_kernel void @struct_ptr_buffer_atomic_min_noret_f64(ptr addrspace(8) %rsrc, double %data, i32 %vindex) {872; GFX90A-LABEL: struct_ptr_buffer_atomic_min_noret_f64:873; GFX90A: ; %bb.0: ; %main_body874; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34875; GFX90A-NEXT: s_load_dword s8, s[4:5], 0x3c876; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24877; GFX90A-NEXT: s_waitcnt lgkmcnt(0)878; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]879; GFX90A-NEXT: v_mov_b32_e32 v0, s8880; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen881; GFX90A-NEXT: s_endpgm882;883; GFX942-LABEL: struct_ptr_buffer_atomic_min_noret_f64:884; GFX942: ; %bb.0: ; %main_body885; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34886; GFX942-NEXT: s_load_dword s8, s[4:5], 0x3c887; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24888; GFX942-NEXT: s_waitcnt lgkmcnt(0)889; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]890; GFX942-NEXT: v_mov_b32_e32 v0, s8891; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen892; GFX942-NEXT: s_endpgm893;894; GFX1250-LABEL: struct_ptr_buffer_atomic_min_noret_f64:895; GFX1250: ; %bb.0: ; %main_body896; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1897; GFX1250-NEXT: s_clause 0x1898; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34899; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24900; GFX1250-NEXT: s_wait_kmcnt 0x0901; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]902; GFX1250-NEXT: v_mov_b32_e32 v2, s10903; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen904; GFX1250-NEXT: s_endpgm905main_body:906 %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0, i32 0)907 ret void908}909 910define amdgpu_ps void @struct_ptr_buffer_atomic_min_rtn_f64(ptr addrspace(8) inreg %rsrc, double %data, i32 %vindex) {911; GFX90A-LABEL: struct_ptr_buffer_atomic_min_rtn_f64:912; GFX90A: ; %bb.0: ; %main_body913; GFX90A-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 idxen glc914; GFX90A-NEXT: s_waitcnt vmcnt(0)915; GFX90A-NEXT: flat_store_dwordx2 v[0:1], v[0:1]916; GFX90A-NEXT: s_endpgm917;918; GFX942-LABEL: struct_ptr_buffer_atomic_min_rtn_f64:919; GFX942: ; %bb.0: ; %main_body920; GFX942-NEXT: buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 idxen sc0921; GFX942-NEXT: s_waitcnt vmcnt(0)922; GFX942-NEXT: flat_store_dwordx2 v[0:1], v[0:1]923; GFX942-NEXT: s_endpgm924;925; GFX1250-LABEL: struct_ptr_buffer_atomic_min_rtn_f64:926; GFX1250: ; %bb.0: ; %main_body927; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1928; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN929; GFX1250-NEXT: s_wait_loadcnt 0x0930; GFX1250-NEXT: flat_store_b64 v[0:1], v[0:1]931; GFX1250-NEXT: s_endpgm932main_body:933 %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0, i32 0)934 store double %ret, ptr poison935 ret void936}937 938define amdgpu_kernel void @struct_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr addrspace(8) %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {939; GFX90A-LABEL: struct_ptr_buffer_atomic_min_rtn_f64_off4_slc:940; GFX90A: ; %bb.0: ; %main_body941; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34942; GFX90A-NEXT: s_load_dword s10, s[4:5], 0x3c943; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24944; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44945; GFX90A-NEXT: s_waitcnt lgkmcnt(0)946; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]947; GFX90A-NEXT: v_mov_b32_e32 v0, s10948; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc949; GFX90A-NEXT: v_mov_b32_e32 v0, 0950; GFX90A-NEXT: s_waitcnt vmcnt(0)951; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]952; GFX90A-NEXT: s_endpgm953;954; GFX942-LABEL: struct_ptr_buffer_atomic_min_rtn_f64_off4_slc:955; GFX942: ; %bb.0: ; %main_body956; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34957; GFX942-NEXT: s_load_dword s10, s[4:5], 0x3c958; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24959; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44960; GFX942-NEXT: s_waitcnt lgkmcnt(0)961; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]962; GFX942-NEXT: v_mov_b32_e32 v0, s10963; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt964; GFX942-NEXT: v_mov_b32_e32 v0, 0965; GFX942-NEXT: s_waitcnt vmcnt(0)966; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]967; GFX942-NEXT: s_endpgm968;969; GFX1250-LABEL: struct_ptr_buffer_atomic_min_rtn_f64_off4_slc:970; GFX1250: ; %bb.0: ; %main_body971; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1972; GFX1250-NEXT: s_clause 0x1973; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34974; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24975; GFX1250-NEXT: s_wait_kmcnt 0x0976; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]977; GFX1250-NEXT: v_mov_b32_e32 v2, s10978; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN979; GFX1250-NEXT: s_wait_xcnt 0x0980; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x44981; GFX1250-NEXT: v_mov_b32_e32 v2, 0982; GFX1250-NEXT: s_wait_loadcnt 0x0983; GFX1250-NEXT: s_wait_kmcnt 0x0984; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]985; GFX1250-NEXT: s_endpgm986main_body:987 %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 0, i32 2)988 store double %ret, ptr addrspace(1) %out, align 8989 ret void990}991 992define amdgpu_kernel void @raw_buffer_atomic_max_noret_f64(<4 x i32> %rsrc, double %data, i32 %vindex) {993; GFX90A-LABEL: raw_buffer_atomic_max_noret_f64:994; GFX90A: ; %bb.0: ; %main_body995; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34996; GFX90A-NEXT: s_load_dword s8, s[4:5], 0x3c997; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24998; GFX90A-NEXT: s_waitcnt lgkmcnt(0)999; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]1000; GFX90A-NEXT: v_mov_b32_e32 v0, s81001; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 offen1002; GFX90A-NEXT: s_endpgm1003;1004; GFX942-LABEL: raw_buffer_atomic_max_noret_f64:1005; GFX942: ; %bb.0: ; %main_body1006; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341007; GFX942-NEXT: s_load_dword s8, s[4:5], 0x3c1008; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241009; GFX942-NEXT: s_waitcnt lgkmcnt(0)1010; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]1011; GFX942-NEXT: v_mov_b32_e32 v0, s81012; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 offen1013; GFX942-NEXT: s_endpgm1014;1015; GFX1250-LABEL: raw_buffer_atomic_max_noret_f64:1016; GFX1250: ; %bb.0: ; %main_body1017; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11018; GFX1250-NEXT: s_clause 0x11019; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x341020; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x241021; GFX1250-NEXT: s_wait_kmcnt 0x01022; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]1023; GFX1250-NEXT: v_mov_b32_e32 v2, s101024; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null offen1025; GFX1250-NEXT: s_endpgm1026main_body:1027 %ret = call double @llvm.amdgcn.raw.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0)1028 ret void1029}1030 1031define amdgpu_ps void @raw_buffer_atomic_max_rtn_f64(<4 x i32> inreg %rsrc, double %data, i32 %vindex) {1032; GFX90A-LABEL: raw_buffer_atomic_max_rtn_f64:1033; GFX90A: ; %bb.0: ; %main_body1034; GFX90A-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen glc1035; GFX90A-NEXT: s_waitcnt vmcnt(0)1036; GFX90A-NEXT: flat_store_dwordx2 v[0:1], v[0:1]1037; GFX90A-NEXT: s_endpgm1038;1039; GFX942-LABEL: raw_buffer_atomic_max_rtn_f64:1040; GFX942: ; %bb.0: ; %main_body1041; GFX942-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen sc01042; GFX942-NEXT: s_waitcnt vmcnt(0)1043; GFX942-NEXT: flat_store_dwordx2 v[0:1], v[0:1]1044; GFX942-NEXT: s_endpgm1045;1046; GFX1250-LABEL: raw_buffer_atomic_max_rtn_f64:1047; GFX1250: ; %bb.0: ; %main_body1048; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11049; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN1050; GFX1250-NEXT: s_wait_loadcnt 0x01051; GFX1250-NEXT: flat_store_b64 v[0:1], v[0:1]1052; GFX1250-NEXT: s_endpgm1053main_body:1054 %ret = call double @llvm.amdgcn.raw.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0)1055 store double %ret, ptr poison1056 ret void1057}1058 1059define amdgpu_kernel void @raw_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {1060; GFX90A-LABEL: raw_buffer_atomic_max_rtn_f64_off4_slc:1061; GFX90A: ; %bb.0: ; %main_body1062; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341063; GFX90A-NEXT: s_load_dword s10, s[4:5], 0x3c1064; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241065; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x441066; GFX90A-NEXT: s_waitcnt lgkmcnt(0)1067; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]1068; GFX90A-NEXT: v_mov_b32_e32 v0, s101069; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 4 offen glc slc1070; GFX90A-NEXT: v_mov_b32_e32 v0, 01071; GFX90A-NEXT: s_waitcnt vmcnt(0)1072; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]1073; GFX90A-NEXT: s_endpgm1074;1075; GFX942-LABEL: raw_buffer_atomic_max_rtn_f64_off4_slc:1076; GFX942: ; %bb.0: ; %main_body1077; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341078; GFX942-NEXT: s_load_dword s10, s[4:5], 0x3c1079; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241080; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x441081; GFX942-NEXT: s_waitcnt lgkmcnt(0)1082; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]1083; GFX942-NEXT: v_mov_b32_e32 v0, s101084; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt1085; GFX942-NEXT: v_mov_b32_e32 v0, 01086; GFX942-NEXT: s_waitcnt vmcnt(0)1087; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]1088; GFX942-NEXT: s_endpgm1089;1090; GFX1250-LABEL: raw_buffer_atomic_max_rtn_f64_off4_slc:1091; GFX1250: ; %bb.0: ; %main_body1092; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11093; GFX1250-NEXT: s_clause 0x11094; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x341095; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x241096; GFX1250-NEXT: s_mov_b32 s6, 41097; GFX1250-NEXT: s_wait_kmcnt 0x01098; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]1099; GFX1250-NEXT: v_mov_b32_e32 v2, s101100; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN1101; GFX1250-NEXT: s_wait_xcnt 0x01102; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x441103; GFX1250-NEXT: v_mov_b32_e32 v2, 01104; GFX1250-NEXT: s_wait_loadcnt 0x01105; GFX1250-NEXT: s_wait_kmcnt 0x01106; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]1107; GFX1250-NEXT: s_endpgm1108main_body:1109 %ret = call double @llvm.amdgcn.raw.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 2)1110 store double %ret, ptr addrspace(1) %out, align 81111 ret void1112}1113 1114define amdgpu_kernel void @raw_ptr_buffer_atomic_max_noret_f64(ptr addrspace(8) %rsrc, double %data, i32 %vindex) {1115; GFX90A-LABEL: raw_ptr_buffer_atomic_max_noret_f64:1116; GFX90A: ; %bb.0: ; %main_body1117; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341118; GFX90A-NEXT: s_load_dword s8, s[4:5], 0x3c1119; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241120; GFX90A-NEXT: s_waitcnt lgkmcnt(0)1121; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]1122; GFX90A-NEXT: v_mov_b32_e32 v0, s81123; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 offen1124; GFX90A-NEXT: s_endpgm1125;1126; GFX942-LABEL: raw_ptr_buffer_atomic_max_noret_f64:1127; GFX942: ; %bb.0: ; %main_body1128; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341129; GFX942-NEXT: s_load_dword s8, s[4:5], 0x3c1130; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241131; GFX942-NEXT: s_waitcnt lgkmcnt(0)1132; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]1133; GFX942-NEXT: v_mov_b32_e32 v0, s81134; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 offen1135; GFX942-NEXT: s_endpgm1136;1137; GFX1250-LABEL: raw_ptr_buffer_atomic_max_noret_f64:1138; GFX1250: ; %bb.0: ; %main_body1139; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11140; GFX1250-NEXT: s_clause 0x11141; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x341142; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x241143; GFX1250-NEXT: s_wait_kmcnt 0x01144; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]1145; GFX1250-NEXT: v_mov_b32_e32 v2, s101146; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null offen1147; GFX1250-NEXT: s_endpgm1148main_body:1149 %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0)1150 ret void1151}1152 1153define amdgpu_ps void @raw_ptr_buffer_atomic_max_rtn_f64(ptr addrspace(8) inreg %rsrc, double %data, i32 %vindex) {1154; GFX90A-LABEL: raw_ptr_buffer_atomic_max_rtn_f64:1155; GFX90A: ; %bb.0: ; %main_body1156; GFX90A-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen glc1157; GFX90A-NEXT: s_waitcnt vmcnt(0)1158; GFX90A-NEXT: flat_store_dwordx2 v[0:1], v[0:1]1159; GFX90A-NEXT: s_endpgm1160;1161; GFX942-LABEL: raw_ptr_buffer_atomic_max_rtn_f64:1162; GFX942: ; %bb.0: ; %main_body1163; GFX942-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen sc01164; GFX942-NEXT: s_waitcnt vmcnt(0)1165; GFX942-NEXT: flat_store_dwordx2 v[0:1], v[0:1]1166; GFX942-NEXT: s_endpgm1167;1168; GFX1250-LABEL: raw_ptr_buffer_atomic_max_rtn_f64:1169; GFX1250: ; %bb.0: ; %main_body1170; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11171; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN1172; GFX1250-NEXT: s_wait_loadcnt 0x01173; GFX1250-NEXT: flat_store_b64 v[0:1], v[0:1]1174; GFX1250-NEXT: s_endpgm1175main_body:1176 %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0)1177 store double %ret, ptr poison1178 ret void1179}1180 1181define amdgpu_kernel void @raw_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr addrspace(8) %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {1182; GFX90A-LABEL: raw_ptr_buffer_atomic_max_rtn_f64_off4_slc:1183; GFX90A: ; %bb.0: ; %main_body1184; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341185; GFX90A-NEXT: s_load_dword s10, s[4:5], 0x3c1186; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241187; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x441188; GFX90A-NEXT: s_waitcnt lgkmcnt(0)1189; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]1190; GFX90A-NEXT: v_mov_b32_e32 v0, s101191; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 4 offen glc slc1192; GFX90A-NEXT: v_mov_b32_e32 v0, 01193; GFX90A-NEXT: s_waitcnt vmcnt(0)1194; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]1195; GFX90A-NEXT: s_endpgm1196;1197; GFX942-LABEL: raw_ptr_buffer_atomic_max_rtn_f64_off4_slc:1198; GFX942: ; %bb.0: ; %main_body1199; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341200; GFX942-NEXT: s_load_dword s10, s[4:5], 0x3c1201; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241202; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x441203; GFX942-NEXT: s_waitcnt lgkmcnt(0)1204; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]1205; GFX942-NEXT: v_mov_b32_e32 v0, s101206; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt1207; GFX942-NEXT: v_mov_b32_e32 v0, 01208; GFX942-NEXT: s_waitcnt vmcnt(0)1209; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]1210; GFX942-NEXT: s_endpgm1211;1212; GFX1250-LABEL: raw_ptr_buffer_atomic_max_rtn_f64_off4_slc:1213; GFX1250: ; %bb.0: ; %main_body1214; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11215; GFX1250-NEXT: s_clause 0x11216; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x341217; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x241218; GFX1250-NEXT: s_mov_b32 s6, 41219; GFX1250-NEXT: s_wait_kmcnt 0x01220; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]1221; GFX1250-NEXT: v_mov_b32_e32 v2, s101222; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN1223; GFX1250-NEXT: s_wait_xcnt 0x01224; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x441225; GFX1250-NEXT: v_mov_b32_e32 v2, 01226; GFX1250-NEXT: s_wait_loadcnt 0x01227; GFX1250-NEXT: s_wait_kmcnt 0x01228; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]1229; GFX1250-NEXT: s_endpgm1230main_body:1231 %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 2)1232 store double %ret, ptr addrspace(1) %out, align 81233 ret void1234}1235 1236define amdgpu_kernel void @struct_buffer_atomic_max_noret_f64(<4 x i32> %rsrc, double %data, i32 %vindex) {1237; GFX90A-LABEL: struct_buffer_atomic_max_noret_f64:1238; GFX90A: ; %bb.0: ; %main_body1239; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341240; GFX90A-NEXT: s_load_dword s8, s[4:5], 0x3c1241; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241242; GFX90A-NEXT: s_waitcnt lgkmcnt(0)1243; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]1244; GFX90A-NEXT: v_mov_b32_e32 v0, s81245; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen1246; GFX90A-NEXT: s_endpgm1247;1248; GFX942-LABEL: struct_buffer_atomic_max_noret_f64:1249; GFX942: ; %bb.0: ; %main_body1250; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341251; GFX942-NEXT: s_load_dword s8, s[4:5], 0x3c1252; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241253; GFX942-NEXT: s_waitcnt lgkmcnt(0)1254; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]1255; GFX942-NEXT: v_mov_b32_e32 v0, s81256; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen1257; GFX942-NEXT: s_endpgm1258;1259; GFX1250-LABEL: struct_buffer_atomic_max_noret_f64:1260; GFX1250: ; %bb.0: ; %main_body1261; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11262; GFX1250-NEXT: s_clause 0x11263; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x341264; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x241265; GFX1250-NEXT: s_wait_kmcnt 0x01266; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]1267; GFX1250-NEXT: v_mov_b32_e32 v2, s101268; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen1269; GFX1250-NEXT: s_endpgm1270main_body:1271 %ret = call double @llvm.amdgcn.struct.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0)1272 ret void1273}1274 1275define amdgpu_ps void @struct_buffer_atomic_max_rtn_f64(<4 x i32> inreg %rsrc, double %data, i32 %vindex) {1276; GFX90A-LABEL: struct_buffer_atomic_max_rtn_f64:1277; GFX90A: ; %bb.0: ; %main_body1278; GFX90A-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 idxen glc1279; GFX90A-NEXT: s_waitcnt vmcnt(0)1280; GFX90A-NEXT: flat_store_dwordx2 v[0:1], v[0:1]1281; GFX90A-NEXT: s_endpgm1282;1283; GFX942-LABEL: struct_buffer_atomic_max_rtn_f64:1284; GFX942: ; %bb.0: ; %main_body1285; GFX942-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 idxen sc01286; GFX942-NEXT: s_waitcnt vmcnt(0)1287; GFX942-NEXT: flat_store_dwordx2 v[0:1], v[0:1]1288; GFX942-NEXT: s_endpgm1289;1290; GFX1250-LABEL: struct_buffer_atomic_max_rtn_f64:1291; GFX1250: ; %bb.0: ; %main_body1292; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11293; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN1294; GFX1250-NEXT: s_wait_loadcnt 0x01295; GFX1250-NEXT: flat_store_b64 v[0:1], v[0:1]1296; GFX1250-NEXT: s_endpgm1297main_body:1298 %ret = call double @llvm.amdgcn.struct.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0)1299 store double %ret, ptr poison1300 ret void1301}1302 1303define amdgpu_kernel void @struct_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {1304; GFX90A-LABEL: struct_buffer_atomic_max_rtn_f64_off4_slc:1305; GFX90A: ; %bb.0: ; %main_body1306; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341307; GFX90A-NEXT: s_load_dword s10, s[4:5], 0x3c1308; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241309; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x441310; GFX90A-NEXT: s_waitcnt lgkmcnt(0)1311; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]1312; GFX90A-NEXT: v_mov_b32_e32 v0, s101313; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc1314; GFX90A-NEXT: v_mov_b32_e32 v0, 01315; GFX90A-NEXT: s_waitcnt vmcnt(0)1316; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]1317; GFX90A-NEXT: s_endpgm1318;1319; GFX942-LABEL: struct_buffer_atomic_max_rtn_f64_off4_slc:1320; GFX942: ; %bb.0: ; %main_body1321; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341322; GFX942-NEXT: s_load_dword s10, s[4:5], 0x3c1323; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241324; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x441325; GFX942-NEXT: s_waitcnt lgkmcnt(0)1326; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]1327; GFX942-NEXT: v_mov_b32_e32 v0, s101328; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt1329; GFX942-NEXT: v_mov_b32_e32 v0, 01330; GFX942-NEXT: s_waitcnt vmcnt(0)1331; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]1332; GFX942-NEXT: s_endpgm1333;1334; GFX1250-LABEL: struct_buffer_atomic_max_rtn_f64_off4_slc:1335; GFX1250: ; %bb.0: ; %main_body1336; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11337; GFX1250-NEXT: s_clause 0x11338; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x341339; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x241340; GFX1250-NEXT: s_wait_kmcnt 0x01341; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]1342; GFX1250-NEXT: v_mov_b32_e32 v2, s101343; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN1344; GFX1250-NEXT: s_wait_xcnt 0x01345; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x441346; GFX1250-NEXT: v_mov_b32_e32 v2, 01347; GFX1250-NEXT: s_wait_loadcnt 0x01348; GFX1250-NEXT: s_wait_kmcnt 0x01349; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]1350; GFX1250-NEXT: s_endpgm1351main_body:1352 %ret = call double @llvm.amdgcn.struct.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 0, i32 2)1353 store double %ret, ptr addrspace(1) %out, align 81354 ret void1355}1356 1357define amdgpu_kernel void @struct_ptr_buffer_atomic_max_noret_f64(ptr addrspace(8) %rsrc, double %data, i32 %vindex) {1358; GFX90A-LABEL: struct_ptr_buffer_atomic_max_noret_f64:1359; GFX90A: ; %bb.0: ; %main_body1360; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341361; GFX90A-NEXT: s_load_dword s8, s[4:5], 0x3c1362; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241363; GFX90A-NEXT: s_waitcnt lgkmcnt(0)1364; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]1365; GFX90A-NEXT: v_mov_b32_e32 v0, s81366; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen1367; GFX90A-NEXT: s_endpgm1368;1369; GFX942-LABEL: struct_ptr_buffer_atomic_max_noret_f64:1370; GFX942: ; %bb.0: ; %main_body1371; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341372; GFX942-NEXT: s_load_dword s8, s[4:5], 0x3c1373; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241374; GFX942-NEXT: s_waitcnt lgkmcnt(0)1375; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]1376; GFX942-NEXT: v_mov_b32_e32 v0, s81377; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen1378; GFX942-NEXT: s_endpgm1379;1380; GFX1250-LABEL: struct_ptr_buffer_atomic_max_noret_f64:1381; GFX1250: ; %bb.0: ; %main_body1382; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11383; GFX1250-NEXT: s_clause 0x11384; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x341385; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x241386; GFX1250-NEXT: s_wait_kmcnt 0x01387; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]1388; GFX1250-NEXT: v_mov_b32_e32 v2, s101389; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen1390; GFX1250-NEXT: s_endpgm1391main_body:1392 %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0, i32 0)1393 ret void1394}1395 1396define amdgpu_ps void @struct_ptr_buffer_atomic_max_rtn_f64(ptr addrspace(8) inreg %rsrc, double %data, i32 %vindex) {1397; GFX90A-LABEL: struct_ptr_buffer_atomic_max_rtn_f64:1398; GFX90A: ; %bb.0: ; %main_body1399; GFX90A-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 idxen glc1400; GFX90A-NEXT: s_waitcnt vmcnt(0)1401; GFX90A-NEXT: flat_store_dwordx2 v[0:1], v[0:1]1402; GFX90A-NEXT: s_endpgm1403;1404; GFX942-LABEL: struct_ptr_buffer_atomic_max_rtn_f64:1405; GFX942: ; %bb.0: ; %main_body1406; GFX942-NEXT: buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 idxen sc01407; GFX942-NEXT: s_waitcnt vmcnt(0)1408; GFX942-NEXT: flat_store_dwordx2 v[0:1], v[0:1]1409; GFX942-NEXT: s_endpgm1410;1411; GFX1250-LABEL: struct_ptr_buffer_atomic_max_rtn_f64:1412; GFX1250: ; %bb.0: ; %main_body1413; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11414; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN1415; GFX1250-NEXT: s_wait_loadcnt 0x01416; GFX1250-NEXT: flat_store_b64 v[0:1], v[0:1]1417; GFX1250-NEXT: s_endpgm1418main_body:1419 %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0, i32 0)1420 store double %ret, ptr poison1421 ret void1422}1423 1424define amdgpu_kernel void @struct_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr addrspace(8) %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {1425; GFX90A-LABEL: struct_ptr_buffer_atomic_max_rtn_f64_off4_slc:1426; GFX90A: ; %bb.0: ; %main_body1427; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341428; GFX90A-NEXT: s_load_dword s10, s[4:5], 0x3c1429; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241430; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x441431; GFX90A-NEXT: s_waitcnt lgkmcnt(0)1432; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]1433; GFX90A-NEXT: v_mov_b32_e32 v0, s101434; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc1435; GFX90A-NEXT: v_mov_b32_e32 v0, 01436; GFX90A-NEXT: s_waitcnt vmcnt(0)1437; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]1438; GFX90A-NEXT: s_endpgm1439;1440; GFX942-LABEL: struct_ptr_buffer_atomic_max_rtn_f64_off4_slc:1441; GFX942: ; %bb.0: ; %main_body1442; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341443; GFX942-NEXT: s_load_dword s10, s[4:5], 0x3c1444; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241445; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x441446; GFX942-NEXT: s_waitcnt lgkmcnt(0)1447; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]1448; GFX942-NEXT: v_mov_b32_e32 v0, s101449; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt1450; GFX942-NEXT: v_mov_b32_e32 v0, 01451; GFX942-NEXT: s_waitcnt vmcnt(0)1452; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]1453; GFX942-NEXT: s_endpgm1454;1455; GFX1250-LABEL: struct_ptr_buffer_atomic_max_rtn_f64_off4_slc:1456; GFX1250: ; %bb.0: ; %main_body1457; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11458; GFX1250-NEXT: s_clause 0x11459; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x341460; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x241461; GFX1250-NEXT: s_wait_kmcnt 0x01462; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]1463; GFX1250-NEXT: v_mov_b32_e32 v2, s101464; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN1465; GFX1250-NEXT: s_wait_xcnt 0x01466; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x441467; GFX1250-NEXT: v_mov_b32_e32 v2, 01468; GFX1250-NEXT: s_wait_loadcnt 0x01469; GFX1250-NEXT: s_wait_kmcnt 0x01470; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]1471; GFX1250-NEXT: s_endpgm1472main_body:1473 %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 0, i32 2)1474 store double %ret, ptr addrspace(1) %out, align 81475 ret void1476}1477 1478define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %ptr) #1 {1479; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat:1480; GFX90A: ; %bb.0: ; %main_body1481; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241482; GFX90A-NEXT: v_mov_b32_e32 v0, 01483; GFX90A-NEXT: v_mov_b32_e32 v2, 01484; GFX90A-NEXT: v_mov_b32_e32 v1, 0x401000001485; GFX90A-NEXT: buffer_wbl21486; GFX90A-NEXT: s_waitcnt lgkmcnt(0)1487; GFX90A-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1]1488; GFX90A-NEXT: s_waitcnt vmcnt(0)1489; GFX90A-NEXT: buffer_invl21490; GFX90A-NEXT: buffer_wbinvl1_vol1491; GFX90A-NEXT: s_endpgm1492;1493; GFX942-LABEL: global_atomic_fadd_f64_noret_pat:1494; GFX942: ; %bb.0: ; %main_body1495; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241496; GFX942-NEXT: v_mov_b32_e32 v2, 01497; GFX942-NEXT: v_mov_b64_e32 v[0:1], 4.01498; GFX942-NEXT: buffer_wbl2 sc0 sc11499; GFX942-NEXT: s_waitcnt lgkmcnt(0)1500; GFX942-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] sc11501; GFX942-NEXT: s_waitcnt vmcnt(0)1502; GFX942-NEXT: buffer_inv sc0 sc11503; GFX942-NEXT: s_endpgm1504;1505; GFX1250-LABEL: global_atomic_fadd_f64_noret_pat:1506; GFX1250: ; %bb.0: ; %main_body1507; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11508; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x241509; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.01510; GFX1250-NEXT: v_mov_b32_e32 v2, 01511; GFX1250-NEXT: global_wb scope:SCOPE_SYS1512; GFX1250-NEXT: s_wait_storecnt 0x01513; GFX1250-NEXT: s_wait_kmcnt 0x01514; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_SYS1515; GFX1250-NEXT: s_wait_storecnt 0x01516; GFX1250-NEXT: global_inv scope:SCOPE_SYS1517; GFX1250-NEXT: s_endpgm1518main_body:1519 %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 seq_cst, !amdgpu.no.fine.grained.memory !01520 ret void1521}1522 1523define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(1) %ptr) #1 {1524; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_agent:1525; GFX90A: ; %bb.0: ; %main_body1526; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241527; GFX90A-NEXT: v_mov_b32_e32 v0, 01528; GFX90A-NEXT: v_mov_b32_e32 v2, 01529; GFX90A-NEXT: v_mov_b32_e32 v1, 0x401000001530; GFX90A-NEXT: s_waitcnt lgkmcnt(0)1531; GFX90A-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1]1532; GFX90A-NEXT: s_waitcnt vmcnt(0)1533; GFX90A-NEXT: buffer_wbinvl1_vol1534; GFX90A-NEXT: s_endpgm1535;1536; GFX942-LABEL: global_atomic_fadd_f64_noret_pat_agent:1537; GFX942: ; %bb.0: ; %main_body1538; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241539; GFX942-NEXT: v_mov_b32_e32 v2, 01540; GFX942-NEXT: v_mov_b64_e32 v[0:1], 4.01541; GFX942-NEXT: buffer_wbl2 sc11542; GFX942-NEXT: s_waitcnt lgkmcnt(0)1543; GFX942-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1]1544; GFX942-NEXT: s_waitcnt vmcnt(0)1545; GFX942-NEXT: buffer_inv sc11546; GFX942-NEXT: s_endpgm1547;1548; GFX1250-LABEL: global_atomic_fadd_f64_noret_pat_agent:1549; GFX1250: ; %bb.0: ; %main_body1550; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11551; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x241552; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.01553; GFX1250-NEXT: v_mov_b32_e32 v2, 01554; GFX1250-NEXT: global_wb scope:SCOPE_DEV1555; GFX1250-NEXT: s_wait_storecnt 0x01556; GFX1250-NEXT: s_wait_kmcnt 0x01557; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_DEV1558; GFX1250-NEXT: s_wait_storecnt 0x01559; GFX1250-NEXT: global_inv scope:SCOPE_DEV1560; GFX1250-NEXT: s_endpgm1561main_body:1562 %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01563 ret void1564}1565 1566define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace(1) %ptr) #1 {1567; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_system:1568; GFX90A: ; %bb.0: ; %main_body1569; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241570; GFX90A-NEXT: v_mov_b32_e32 v0, 01571; GFX90A-NEXT: v_mov_b32_e32 v2, 01572; GFX90A-NEXT: v_mov_b32_e32 v1, 0x401000001573; GFX90A-NEXT: buffer_wbl21574; GFX90A-NEXT: s_waitcnt lgkmcnt(0)1575; GFX90A-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1]1576; GFX90A-NEXT: s_waitcnt vmcnt(0)1577; GFX90A-NEXT: buffer_invl21578; GFX90A-NEXT: buffer_wbinvl1_vol1579; GFX90A-NEXT: s_endpgm1580;1581; GFX942-LABEL: global_atomic_fadd_f64_noret_pat_system:1582; GFX942: ; %bb.0: ; %main_body1583; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241584; GFX942-NEXT: v_mov_b32_e32 v2, 01585; GFX942-NEXT: v_mov_b64_e32 v[0:1], 4.01586; GFX942-NEXT: buffer_wbl2 sc0 sc11587; GFX942-NEXT: s_waitcnt lgkmcnt(0)1588; GFX942-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] sc11589; GFX942-NEXT: s_waitcnt vmcnt(0)1590; GFX942-NEXT: buffer_inv sc0 sc11591; GFX942-NEXT: s_endpgm1592;1593; GFX1250-LABEL: global_atomic_fadd_f64_noret_pat_system:1594; GFX1250: ; %bb.0: ; %main_body1595; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11596; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x241597; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.01598; GFX1250-NEXT: v_mov_b32_e32 v2, 01599; GFX1250-NEXT: global_wb scope:SCOPE_SYS1600; GFX1250-NEXT: s_wait_storecnt 0x01601; GFX1250-NEXT: s_wait_kmcnt 0x01602; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_SYS1603; GFX1250-NEXT: s_wait_storecnt 0x01604; GFX1250-NEXT: global_inv scope:SCOPE_SYS1605; GFX1250-NEXT: s_endpgm1606main_body:1607 %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") seq_cst, !amdgpu.no.fine.grained.memory !01608 ret void1609}1610 1611define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_flush(ptr addrspace(1) %ptr) #0 {1612; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_flush:1613; GFX90A: ; %bb.0: ; %main_body1614; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241615; GFX90A-NEXT: v_mov_b32_e32 v0, 01616; GFX90A-NEXT: v_mov_b32_e32 v2, 01617; GFX90A-NEXT: v_mov_b32_e32 v1, 0x401000001618; GFX90A-NEXT: s_waitcnt lgkmcnt(0)1619; GFX90A-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1]1620; GFX90A-NEXT: s_waitcnt vmcnt(0)1621; GFX90A-NEXT: buffer_wbinvl1_vol1622; GFX90A-NEXT: s_endpgm1623;1624; GFX942-LABEL: global_atomic_fadd_f64_noret_pat_flush:1625; GFX942: ; %bb.0: ; %main_body1626; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241627; GFX942-NEXT: v_mov_b32_e32 v2, 01628; GFX942-NEXT: v_mov_b64_e32 v[0:1], 4.01629; GFX942-NEXT: buffer_wbl2 sc11630; GFX942-NEXT: s_waitcnt lgkmcnt(0)1631; GFX942-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1]1632; GFX942-NEXT: s_waitcnt vmcnt(0)1633; GFX942-NEXT: buffer_inv sc11634; GFX942-NEXT: s_endpgm1635;1636; GFX1250-LABEL: global_atomic_fadd_f64_noret_pat_flush:1637; GFX1250: ; %bb.0: ; %main_body1638; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11639; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x241640; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.01641; GFX1250-NEXT: v_mov_b32_e32 v2, 01642; GFX1250-NEXT: global_wb scope:SCOPE_DEV1643; GFX1250-NEXT: s_wait_storecnt 0x01644; GFX1250-NEXT: s_wait_kmcnt 0x01645; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_DEV1646; GFX1250-NEXT: s_wait_storecnt 0x01647; GFX1250-NEXT: global_inv scope:SCOPE_DEV1648; GFX1250-NEXT: s_endpgm1649main_body:1650 %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01651 ret void1652}1653 1654define double @global_atomic_fadd_f64_rtn_pat(ptr addrspace(1) %ptr, double %data) #1 {1655; GFX90A-LABEL: global_atomic_fadd_f64_rtn_pat:1656; GFX90A: ; %bb.0: ; %main_body1657; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1658; GFX90A-NEXT: v_mov_b32_e32 v2, 01659; GFX90A-NEXT: v_mov_b32_e32 v3, 0x401000001660; GFX90A-NEXT: buffer_wbl21661; GFX90A-NEXT: global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off glc1662; GFX90A-NEXT: s_waitcnt vmcnt(0)1663; GFX90A-NEXT: buffer_invl21664; GFX90A-NEXT: buffer_wbinvl1_vol1665; GFX90A-NEXT: s_setpc_b64 s[30:31]1666;1667; GFX942-LABEL: global_atomic_fadd_f64_rtn_pat:1668; GFX942: ; %bb.0: ; %main_body1669; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1670; GFX942-NEXT: v_mov_b64_e32 v[2:3], 4.01671; GFX942-NEXT: buffer_wbl2 sc0 sc11672; GFX942-NEXT: global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off sc0 sc11673; GFX942-NEXT: s_waitcnt vmcnt(0)1674; GFX942-NEXT: buffer_inv sc0 sc11675; GFX942-NEXT: s_setpc_b64 s[30:31]1676;1677; GFX1250-LABEL: global_atomic_fadd_f64_rtn_pat:1678; GFX1250: ; %bb.0: ; %main_body1679; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x01680; GFX1250-NEXT: s_wait_kmcnt 0x01681; GFX1250-NEXT: v_mov_b64_e32 v[2:3], 4.01682; GFX1250-NEXT: global_wb scope:SCOPE_SYS1683; GFX1250-NEXT: s_wait_storecnt 0x01684; GFX1250-NEXT: global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS1685; GFX1250-NEXT: s_wait_loadcnt 0x01686; GFX1250-NEXT: global_inv scope:SCOPE_SYS1687; GFX1250-NEXT: s_wait_loadcnt 0x01688; GFX1250-NEXT: s_set_pc_i64 s[30:31]1689main_body:1690 %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 seq_cst, !amdgpu.no.fine.grained.memory !01691 ret double %ret1692}1693 1694define double @global_atomic_fadd_f64_rtn_pat_agent(ptr addrspace(1) %ptr, double %data) #1 {1695; GFX90A-LABEL: global_atomic_fadd_f64_rtn_pat_agent:1696; GFX90A: ; %bb.0: ; %main_body1697; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1698; GFX90A-NEXT: v_mov_b32_e32 v2, 01699; GFX90A-NEXT: v_mov_b32_e32 v3, 0x401000001700; GFX90A-NEXT: global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off glc1701; GFX90A-NEXT: s_waitcnt vmcnt(0)1702; GFX90A-NEXT: buffer_wbinvl1_vol1703; GFX90A-NEXT: s_setpc_b64 s[30:31]1704;1705; GFX942-LABEL: global_atomic_fadd_f64_rtn_pat_agent:1706; GFX942: ; %bb.0: ; %main_body1707; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1708; GFX942-NEXT: v_mov_b64_e32 v[2:3], 4.01709; GFX942-NEXT: buffer_wbl2 sc11710; GFX942-NEXT: global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off sc01711; GFX942-NEXT: s_waitcnt vmcnt(0)1712; GFX942-NEXT: buffer_inv sc11713; GFX942-NEXT: s_setpc_b64 s[30:31]1714;1715; GFX1250-LABEL: global_atomic_fadd_f64_rtn_pat_agent:1716; GFX1250: ; %bb.0: ; %main_body1717; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x01718; GFX1250-NEXT: s_wait_kmcnt 0x01719; GFX1250-NEXT: v_mov_b64_e32 v[2:3], 4.01720; GFX1250-NEXT: global_wb scope:SCOPE_DEV1721; GFX1250-NEXT: s_wait_storecnt 0x01722; GFX1250-NEXT: global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV1723; GFX1250-NEXT: s_wait_loadcnt 0x01724; GFX1250-NEXT: global_inv scope:SCOPE_DEV1725; GFX1250-NEXT: s_wait_loadcnt 0x01726; GFX1250-NEXT: s_set_pc_i64 s[30:31]1727main_body:1728 %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01729 ret double %ret1730}1731 1732define double @global_atomic_fadd_f64_rtn_pat_system(ptr addrspace(1) %ptr, double %data) #1 {1733; GFX90A-LABEL: global_atomic_fadd_f64_rtn_pat_system:1734; GFX90A: ; %bb.0: ; %main_body1735; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1736; GFX90A-NEXT: v_mov_b32_e32 v2, 01737; GFX90A-NEXT: v_mov_b32_e32 v3, 0x401000001738; GFX90A-NEXT: buffer_wbl21739; GFX90A-NEXT: global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off glc1740; GFX90A-NEXT: s_waitcnt vmcnt(0)1741; GFX90A-NEXT: buffer_invl21742; GFX90A-NEXT: buffer_wbinvl1_vol1743; GFX90A-NEXT: s_setpc_b64 s[30:31]1744;1745; GFX942-LABEL: global_atomic_fadd_f64_rtn_pat_system:1746; GFX942: ; %bb.0: ; %main_body1747; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1748; GFX942-NEXT: v_mov_b64_e32 v[2:3], 4.01749; GFX942-NEXT: buffer_wbl2 sc0 sc11750; GFX942-NEXT: global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off sc0 sc11751; GFX942-NEXT: s_waitcnt vmcnt(0)1752; GFX942-NEXT: buffer_inv sc0 sc11753; GFX942-NEXT: s_setpc_b64 s[30:31]1754;1755; GFX1250-LABEL: global_atomic_fadd_f64_rtn_pat_system:1756; GFX1250: ; %bb.0: ; %main_body1757; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x01758; GFX1250-NEXT: s_wait_kmcnt 0x01759; GFX1250-NEXT: v_mov_b64_e32 v[2:3], 4.01760; GFX1250-NEXT: global_wb scope:SCOPE_SYS1761; GFX1250-NEXT: s_wait_storecnt 0x01762; GFX1250-NEXT: global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS1763; GFX1250-NEXT: s_wait_loadcnt 0x01764; GFX1250-NEXT: global_inv scope:SCOPE_SYS1765; GFX1250-NEXT: s_wait_loadcnt 0x01766; GFX1250-NEXT: s_set_pc_i64 s[30:31]1767main_body:1768 %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") seq_cst, !amdgpu.no.fine.grained.memory !01769 ret double %ret1770}1771 1772define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrspace(1) %ptr) {1773; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe:1774; GFX90A: ; %bb.0: ; %main_body1775; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241776; GFX90A-NEXT: s_mov_b64 s[2:3], 01777; GFX90A-NEXT: v_mov_b32_e32 v4, 01778; GFX90A-NEXT: s_waitcnt lgkmcnt(0)1779; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x01780; GFX90A-NEXT: s_waitcnt lgkmcnt(0)1781; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]1782; GFX90A-NEXT: .LBB43_1: ; %atomicrmw.start1783; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=11784; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.01785; GFX90A-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc1786; GFX90A-NEXT: s_waitcnt vmcnt(0)1787; GFX90A-NEXT: buffer_wbinvl1_vol1788; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]1789; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3]1790; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]1791; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3]1792; GFX90A-NEXT: s_cbranch_execnz .LBB43_11793; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end1794; GFX90A-NEXT: s_endpgm1795;1796; GFX942-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe:1797; GFX942: ; %bb.0: ; %main_body1798; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241799; GFX942-NEXT: v_mov_b32_e32 v2, 01800; GFX942-NEXT: v_mov_b64_e32 v[0:1], 4.01801; GFX942-NEXT: buffer_wbl2 sc11802; GFX942-NEXT: s_waitcnt lgkmcnt(0)1803; GFX942-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1]1804; GFX942-NEXT: s_waitcnt vmcnt(0)1805; GFX942-NEXT: buffer_inv sc11806; GFX942-NEXT: s_endpgm1807;1808; GFX1250-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe:1809; GFX1250: ; %bb.0: ; %main_body1810; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11811; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x241812; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.01813; GFX1250-NEXT: v_mov_b32_e32 v2, 01814; GFX1250-NEXT: global_wb scope:SCOPE_DEV1815; GFX1250-NEXT: s_wait_storecnt 0x01816; GFX1250-NEXT: s_wait_kmcnt 0x01817; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_DEV1818; GFX1250-NEXT: s_wait_storecnt 0x01819; GFX1250-NEXT: global_inv scope:SCOPE_DEV1820; GFX1250-NEXT: s_endpgm1821main_body:1822 %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst1823 ret void1824}1825 1826define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat(ptr %ptr) #1 {1827; GFX90A-LABEL: flat_atomic_fadd_f64_noret_pat:1828; GFX90A: ; %bb.0: ; %main_body1829; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241830; GFX90A-NEXT: v_mov_b32_e32 v0, 01831; GFX90A-NEXT: v_mov_b32_e32 v1, 0x401000001832; GFX90A-NEXT: s_waitcnt lgkmcnt(0)1833; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1]1834; GFX90A-NEXT: buffer_wbl21835; GFX90A-NEXT: flat_atomic_add_f64 v[2:3], v[0:1]1836; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1837; GFX90A-NEXT: buffer_invl21838; GFX90A-NEXT: buffer_wbinvl1_vol1839; GFX90A-NEXT: s_endpgm1840;1841; GFX942-LABEL: flat_atomic_fadd_f64_noret_pat:1842; GFX942: ; %bb.0: ; %main_body1843; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241844; GFX942-NEXT: v_mov_b64_e32 v[0:1], 4.01845; GFX942-NEXT: s_waitcnt lgkmcnt(0)1846; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[0:1]1847; GFX942-NEXT: buffer_wbl2 sc0 sc11848; GFX942-NEXT: flat_atomic_add_f64 v[2:3], v[0:1] sc11849; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1850; GFX942-NEXT: buffer_inv sc0 sc11851; GFX942-NEXT: s_endpgm1852;1853; GFX1250-LABEL: flat_atomic_fadd_f64_noret_pat:1854; GFX1250: ; %bb.0: ; %main_body1855; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11856; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x241857; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.01858; GFX1250-NEXT: v_mov_b32_e32 v2, 01859; GFX1250-NEXT: global_wb scope:SCOPE_SYS1860; GFX1250-NEXT: s_wait_storecnt 0x01861; GFX1250-NEXT: s_wait_kmcnt 0x01862; GFX1250-NEXT: flat_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_SYS1863; GFX1250-NEXT: s_wait_storecnt_dscnt 0x01864; GFX1250-NEXT: global_inv scope:SCOPE_SYS1865; GFX1250-NEXT: s_endpgm1866main_body:1867 %ret = atomicrmw fadd ptr %ptr, double 4.0 seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !01868 ret void1869}1870 1871define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_agent(ptr %ptr) #1 {1872; GFX90A-LABEL: flat_atomic_fadd_f64_noret_pat_agent:1873; GFX90A: ; %bb.0: ; %main_body1874; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241875; GFX90A-NEXT: v_mov_b32_e32 v0, 01876; GFX90A-NEXT: v_mov_b32_e32 v1, 0x401000001877; GFX90A-NEXT: s_waitcnt lgkmcnt(0)1878; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1]1879; GFX90A-NEXT: flat_atomic_add_f64 v[2:3], v[0:1]1880; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1881; GFX90A-NEXT: buffer_wbinvl1_vol1882; GFX90A-NEXT: s_endpgm1883;1884; GFX942-LABEL: flat_atomic_fadd_f64_noret_pat_agent:1885; GFX942: ; %bb.0: ; %main_body1886; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241887; GFX942-NEXT: v_mov_b64_e32 v[0:1], 4.01888; GFX942-NEXT: s_waitcnt lgkmcnt(0)1889; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[0:1]1890; GFX942-NEXT: buffer_wbl2 sc11891; GFX942-NEXT: flat_atomic_add_f64 v[2:3], v[0:1]1892; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1893; GFX942-NEXT: buffer_inv sc11894; GFX942-NEXT: s_endpgm1895;1896; GFX1250-LABEL: flat_atomic_fadd_f64_noret_pat_agent:1897; GFX1250: ; %bb.0: ; %main_body1898; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11899; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x241900; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.01901; GFX1250-NEXT: v_mov_b32_e32 v2, 01902; GFX1250-NEXT: global_wb scope:SCOPE_DEV1903; GFX1250-NEXT: s_wait_storecnt 0x01904; GFX1250-NEXT: s_wait_kmcnt 0x01905; GFX1250-NEXT: flat_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_DEV1906; GFX1250-NEXT: s_wait_storecnt_dscnt 0x01907; GFX1250-NEXT: global_inv scope:SCOPE_DEV1908; GFX1250-NEXT: s_endpgm1909main_body:1910 %ret = atomicrmw fadd ptr %ptr, double 4.0 syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !01911 ret void1912}1913 1914define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_system(ptr %ptr) #1 {1915; GFX90A-LABEL: flat_atomic_fadd_f64_noret_pat_system:1916; GFX90A: ; %bb.0: ; %main_body1917; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241918; GFX90A-NEXT: v_mov_b32_e32 v0, 01919; GFX90A-NEXT: v_mov_b32_e32 v1, 0x401000001920; GFX90A-NEXT: s_waitcnt lgkmcnt(0)1921; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1]1922; GFX90A-NEXT: buffer_wbl21923; GFX90A-NEXT: flat_atomic_add_f64 v[2:3], v[0:1]1924; GFX90A-NEXT: s_waitcnt vmcnt(0)1925; GFX90A-NEXT: buffer_invl21926; GFX90A-NEXT: buffer_wbinvl1_vol1927; GFX90A-NEXT: s_endpgm1928;1929; GFX942-LABEL: flat_atomic_fadd_f64_noret_pat_system:1930; GFX942: ; %bb.0: ; %main_body1931; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241932; GFX942-NEXT: v_mov_b64_e32 v[0:1], 4.01933; GFX942-NEXT: s_waitcnt lgkmcnt(0)1934; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[0:1]1935; GFX942-NEXT: buffer_wbl2 sc0 sc11936; GFX942-NEXT: flat_atomic_add_f64 v[2:3], v[0:1] sc11937; GFX942-NEXT: s_waitcnt vmcnt(0)1938; GFX942-NEXT: buffer_inv sc0 sc11939; GFX942-NEXT: s_endpgm1940;1941; GFX1250-LABEL: flat_atomic_fadd_f64_noret_pat_system:1942; GFX1250: ; %bb.0: ; %main_body1943; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11944; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x241945; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.01946; GFX1250-NEXT: v_mov_b32_e32 v2, 01947; GFX1250-NEXT: global_wb scope:SCOPE_SYS1948; GFX1250-NEXT: s_wait_storecnt 0x01949; GFX1250-NEXT: s_wait_kmcnt 0x01950; GFX1250-NEXT: flat_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_SYS1951; GFX1250-NEXT: s_wait_storecnt 0x01952; GFX1250-NEXT: global_inv scope:SCOPE_SYS1953; GFX1250-NEXT: s_endpgm1954main_body:1955 %ret = atomicrmw fadd ptr %ptr, double 4.0 syncscope("one-as") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !01956 ret void1957}1958 1959define double @flat_atomic_fadd_f64_rtn_pat(ptr %ptr) #1 {1960; GFX90A-LABEL: flat_atomic_fadd_f64_rtn_pat:1961; GFX90A: ; %bb.0: ; %main_body1962; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1963; GFX90A-NEXT: v_mov_b32_e32 v2, 01964; GFX90A-NEXT: v_mov_b32_e32 v3, 0x401000001965; GFX90A-NEXT: buffer_wbl21966; GFX90A-NEXT: flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] glc1967; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1968; GFX90A-NEXT: buffer_invl21969; GFX90A-NEXT: buffer_wbinvl1_vol1970; GFX90A-NEXT: s_setpc_b64 s[30:31]1971;1972; GFX942-LABEL: flat_atomic_fadd_f64_rtn_pat:1973; GFX942: ; %bb.0: ; %main_body1974; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1975; GFX942-NEXT: v_mov_b64_e32 v[2:3], 4.01976; GFX942-NEXT: buffer_wbl2 sc0 sc11977; GFX942-NEXT: flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] sc0 sc11978; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1979; GFX942-NEXT: buffer_inv sc0 sc11980; GFX942-NEXT: s_setpc_b64 s[30:31]1981;1982; GFX1250-LABEL: flat_atomic_fadd_f64_rtn_pat:1983; GFX1250: ; %bb.0: ; %main_body1984; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x01985; GFX1250-NEXT: s_wait_kmcnt 0x01986; GFX1250-NEXT: v_mov_b64_e32 v[2:3], 4.01987; GFX1250-NEXT: global_wb scope:SCOPE_SYS1988; GFX1250-NEXT: s_wait_storecnt 0x01989; GFX1250-NEXT: flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS1990; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x01991; GFX1250-NEXT: global_inv scope:SCOPE_SYS1992; GFX1250-NEXT: s_wait_loadcnt 0x01993; GFX1250-NEXT: s_set_pc_i64 s[30:31]1994main_body:1995 %ret = atomicrmw fadd ptr %ptr, double 4.0 seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !01996 ret double %ret1997}1998 1999define double @flat_atomic_fadd_f64_rtn_pat_agent(ptr %ptr) #1 {2000; GFX90A-LABEL: flat_atomic_fadd_f64_rtn_pat_agent:2001; GFX90A: ; %bb.0: ; %main_body2002; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2003; GFX90A-NEXT: v_mov_b32_e32 v2, 02004; GFX90A-NEXT: v_mov_b32_e32 v3, 0x401000002005; GFX90A-NEXT: flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] glc2006; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2007; GFX90A-NEXT: buffer_wbinvl1_vol2008; GFX90A-NEXT: s_setpc_b64 s[30:31]2009;2010; GFX942-LABEL: flat_atomic_fadd_f64_rtn_pat_agent:2011; GFX942: ; %bb.0: ; %main_body2012; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2013; GFX942-NEXT: v_mov_b64_e32 v[2:3], 4.02014; GFX942-NEXT: buffer_wbl2 sc12015; GFX942-NEXT: flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] sc02016; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2017; GFX942-NEXT: buffer_inv sc12018; GFX942-NEXT: s_setpc_b64 s[30:31]2019;2020; GFX1250-LABEL: flat_atomic_fadd_f64_rtn_pat_agent:2021; GFX1250: ; %bb.0: ; %main_body2022; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x02023; GFX1250-NEXT: s_wait_kmcnt 0x02024; GFX1250-NEXT: v_mov_b64_e32 v[2:3], 4.02025; GFX1250-NEXT: global_wb scope:SCOPE_DEV2026; GFX1250-NEXT: s_wait_storecnt 0x02027; GFX1250-NEXT: flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV2028; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x02029; GFX1250-NEXT: global_inv scope:SCOPE_DEV2030; GFX1250-NEXT: s_wait_loadcnt 0x02031; GFX1250-NEXT: s_set_pc_i64 s[30:31]2032main_body:2033 %ret = atomicrmw fadd ptr %ptr, double 4.0 syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !02034 ret double %ret2035}2036 2037define double @flat_atomic_fadd_f64_rtn_pat_system(ptr %ptr) #1 {2038; GFX90A-LABEL: flat_atomic_fadd_f64_rtn_pat_system:2039; GFX90A: ; %bb.0: ; %main_body2040; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2041; GFX90A-NEXT: v_mov_b32_e32 v2, 02042; GFX90A-NEXT: v_mov_b32_e32 v3, 0x401000002043; GFX90A-NEXT: buffer_wbl22044; GFX90A-NEXT: flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] glc2045; GFX90A-NEXT: s_waitcnt vmcnt(0)2046; GFX90A-NEXT: buffer_invl22047; GFX90A-NEXT: buffer_wbinvl1_vol2048; GFX90A-NEXT: s_waitcnt lgkmcnt(0)2049; GFX90A-NEXT: s_setpc_b64 s[30:31]2050;2051; GFX942-LABEL: flat_atomic_fadd_f64_rtn_pat_system:2052; GFX942: ; %bb.0: ; %main_body2053; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2054; GFX942-NEXT: v_mov_b64_e32 v[2:3], 4.02055; GFX942-NEXT: buffer_wbl2 sc0 sc12056; GFX942-NEXT: flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] sc0 sc12057; GFX942-NEXT: s_waitcnt vmcnt(0)2058; GFX942-NEXT: buffer_inv sc0 sc12059; GFX942-NEXT: s_waitcnt lgkmcnt(0)2060; GFX942-NEXT: s_setpc_b64 s[30:31]2061;2062; GFX1250-LABEL: flat_atomic_fadd_f64_rtn_pat_system:2063; GFX1250: ; %bb.0: ; %main_body2064; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x02065; GFX1250-NEXT: s_wait_kmcnt 0x02066; GFX1250-NEXT: v_mov_b64_e32 v[2:3], 4.02067; GFX1250-NEXT: global_wb scope:SCOPE_SYS2068; GFX1250-NEXT: s_wait_storecnt 0x02069; GFX1250-NEXT: flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS2070; GFX1250-NEXT: s_wait_loadcnt 0x02071; GFX1250-NEXT: global_inv scope:SCOPE_SYS2072; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x02073; GFX1250-NEXT: s_set_pc_i64 s[30:31]2074main_body:2075 %ret = atomicrmw fadd ptr %ptr, double 4.0 syncscope("one-as") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !02076 ret double %ret2077}2078 2079define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_agent_safe(ptr %ptr) {2080; GFX90A-LABEL: flat_atomic_fadd_f64_noret_pat_agent_safe:2081; GFX90A: ; %bb.0: ; %main_body2082; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x242083; GFX90A-NEXT: s_mov_b64 s[0:1], 02084; GFX90A-NEXT: s_waitcnt lgkmcnt(0)2085; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]2086; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]2087; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[2:3], s[2:3] op_sel:[0,1]2088; GFX90A-NEXT: .LBB50_1: ; %atomicrmw.start2089; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=12090; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2091; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.02092; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc2093; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2094; GFX90A-NEXT: buffer_wbinvl1_vol2095; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]2096; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1]2097; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]2098; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1]2099; GFX90A-NEXT: s_cbranch_execnz .LBB50_12100; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end2101; GFX90A-NEXT: s_endpgm2102;2103; GFX942-LABEL: flat_atomic_fadd_f64_noret_pat_agent_safe:2104; GFX942: ; %bb.0: ; %main_body2105; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x242106; GFX942-NEXT: v_mov_b64_e32 v[0:1], 4.02107; GFX942-NEXT: s_waitcnt lgkmcnt(0)2108; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[0:1]2109; GFX942-NEXT: buffer_wbl2 sc12110; GFX942-NEXT: flat_atomic_add_f64 v[2:3], v[0:1]2111; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2112; GFX942-NEXT: buffer_inv sc12113; GFX942-NEXT: s_endpgm2114;2115; GFX1250-LABEL: flat_atomic_fadd_f64_noret_pat_agent_safe:2116; GFX1250: ; %bb.0: ; %main_body2117; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12118; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x242119; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.02120; GFX1250-NEXT: v_mov_b32_e32 v2, 02121; GFX1250-NEXT: global_wb scope:SCOPE_DEV2122; GFX1250-NEXT: s_wait_storecnt 0x02123; GFX1250-NEXT: s_wait_kmcnt 0x02124; GFX1250-NEXT: flat_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_DEV2125; GFX1250-NEXT: s_wait_storecnt_dscnt 0x02126; GFX1250-NEXT: global_inv scope:SCOPE_DEV2127; GFX1250-NEXT: s_endpgm2128main_body:2129 %ret = atomicrmw fadd ptr %ptr, double 4.0 syncscope("agent") seq_cst, !noalias.addrspace !12130 ret void2131}2132 2133define amdgpu_kernel void @local_atomic_fadd_f64_noret(ptr addrspace(3) %ptr, double %data) {2134; GFX90A-LABEL: local_atomic_fadd_f64_noret:2135; GFX90A: ; %bb.0: ; %main_body2136; GFX90A-NEXT: s_load_dword s2, s[4:5], 0x242137; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c2138; GFX90A-NEXT: s_waitcnt lgkmcnt(0)2139; GFX90A-NEXT: v_mov_b32_e32 v0, s22140; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1]2141; GFX90A-NEXT: ds_add_f64 v0, v[2:3]2142; GFX90A-NEXT: s_waitcnt lgkmcnt(0)2143; GFX90A-NEXT: s_endpgm2144;2145; GFX942-LABEL: local_atomic_fadd_f64_noret:2146; GFX942: ; %bb.0: ; %main_body2147; GFX942-NEXT: s_load_dword s2, s[4:5], 0x242148; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c2149; GFX942-NEXT: s_waitcnt lgkmcnt(0)2150; GFX942-NEXT: v_mov_b32_e32 v0, s22151; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[0:1]2152; GFX942-NEXT: ds_add_f64 v0, v[2:3]2153; GFX942-NEXT: s_waitcnt lgkmcnt(0)2154; GFX942-NEXT: s_endpgm2155;2156; GFX1250-LABEL: local_atomic_fadd_f64_noret:2157; GFX1250: ; %bb.0: ; %main_body2158; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12159; GFX1250-NEXT: s_clause 0x12160; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x242161; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c2162; GFX1250-NEXT: s_wait_kmcnt 0x02163; GFX1250-NEXT: v_mov_b32_e32 v2, s22164; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]2165; GFX1250-NEXT: ds_add_f64 v2, v[0:1]2166; GFX1250-NEXT: s_wait_dscnt 0x02167; GFX1250-NEXT: s_endpgm2168main_body:2169 %ret = call double @llvm.amdgcn.ds.fadd.f64(ptr addrspace(3) %ptr, double %data, i32 0, i32 0, i1 0)2170 ret void2171}2172 2173define double @local_atomic_fadd_f64_rtn(ptr addrspace(3) %ptr, double %data) {2174; GFX90A-LABEL: local_atomic_fadd_f64_rtn:2175; GFX90A: ; %bb.0: ; %main_body2176; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2177; GFX90A-NEXT: v_mov_b32_e32 v3, v22178; GFX90A-NEXT: v_mov_b32_e32 v2, v12179; GFX90A-NEXT: ds_add_rtn_f64 v[0:1], v0, v[2:3]2180; GFX90A-NEXT: s_waitcnt lgkmcnt(0)2181; GFX90A-NEXT: s_setpc_b64 s[30:31]2182;2183; GFX942-LABEL: local_atomic_fadd_f64_rtn:2184; GFX942: ; %bb.0: ; %main_body2185; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2186; GFX942-NEXT: v_mov_b32_e32 v3, v22187; GFX942-NEXT: v_mov_b32_e32 v2, v12188; GFX942-NEXT: ds_add_rtn_f64 v[0:1], v0, v[2:3]2189; GFX942-NEXT: s_waitcnt lgkmcnt(0)2190; GFX942-NEXT: s_setpc_b64 s[30:31]2191;2192; GFX1250-LABEL: local_atomic_fadd_f64_rtn:2193; GFX1250: ; %bb.0: ; %main_body2194; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x02195; GFX1250-NEXT: s_wait_kmcnt 0x02196; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v12197; GFX1250-NEXT: s_wait_storecnt 0x02198; GFX1250-NEXT: ds_add_rtn_f64 v[0:1], v0, v[2:3]2199; GFX1250-NEXT: s_wait_dscnt 0x02200; GFX1250-NEXT: s_set_pc_i64 s[30:31]2201main_body:2202 %ret = call double @llvm.amdgcn.ds.fadd.f64(ptr addrspace(3) %ptr, double %data, i32 0, i32 0, i1 0)2203 ret double %ret2204}2205 2206define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr) #1 {2207; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat:2208; GFX90A: ; %bb.0: ; %main_body2209; GFX90A-NEXT: s_load_dword s0, s[4:5], 0x242210; GFX90A-NEXT: v_mov_b32_e32 v0, 02211; GFX90A-NEXT: v_mov_b32_e32 v1, 0x401000002212; GFX90A-NEXT: s_waitcnt lgkmcnt(0)2213; GFX90A-NEXT: v_mov_b32_e32 v2, s02214; GFX90A-NEXT: ds_add_f64 v2, v[0:1]2215; GFX90A-NEXT: s_waitcnt lgkmcnt(0)2216; GFX90A-NEXT: s_endpgm2217;2218; GFX942-LABEL: local_atomic_fadd_f64_noret_pat:2219; GFX942: ; %bb.0: ; %main_body2220; GFX942-NEXT: s_load_dword s0, s[4:5], 0x242221; GFX942-NEXT: v_mov_b64_e32 v[0:1], 4.02222; GFX942-NEXT: s_waitcnt lgkmcnt(0)2223; GFX942-NEXT: v_mov_b32_e32 v2, s02224; GFX942-NEXT: ds_add_f64 v2, v[0:1]2225; GFX942-NEXT: s_waitcnt lgkmcnt(0)2226; GFX942-NEXT: s_endpgm2227;2228; GFX1250-LABEL: local_atomic_fadd_f64_noret_pat:2229; GFX1250: ; %bb.0: ; %main_body2230; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12231; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x242232; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.02233; GFX1250-NEXT: s_wait_kmcnt 0x02234; GFX1250-NEXT: v_mov_b32_e32 v2, s02235; GFX1250-NEXT: ds_add_f64 v2, v[0:1]2236; GFX1250-NEXT: s_wait_dscnt 0x02237; GFX1250-NEXT: s_endpgm2238main_body:2239 %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst, !amdgpu.no.fine.grained.memory !02240 ret void2241}2242 2243define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3) %ptr) #0 {2244; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat_flush:2245; GFX90A: ; %bb.0: ; %main_body2246; GFX90A-NEXT: s_load_dword s0, s[4:5], 0x242247; GFX90A-NEXT: v_mov_b32_e32 v0, 02248; GFX90A-NEXT: v_mov_b32_e32 v1, 0x401000002249; GFX90A-NEXT: s_waitcnt lgkmcnt(0)2250; GFX90A-NEXT: v_mov_b32_e32 v2, s02251; GFX90A-NEXT: ds_add_f64 v2, v[0:1]2252; GFX90A-NEXT: s_waitcnt lgkmcnt(0)2253; GFX90A-NEXT: s_endpgm2254;2255; GFX942-LABEL: local_atomic_fadd_f64_noret_pat_flush:2256; GFX942: ; %bb.0: ; %main_body2257; GFX942-NEXT: s_load_dword s0, s[4:5], 0x242258; GFX942-NEXT: v_mov_b64_e32 v[0:1], 4.02259; GFX942-NEXT: s_waitcnt lgkmcnt(0)2260; GFX942-NEXT: v_mov_b32_e32 v2, s02261; GFX942-NEXT: ds_add_f64 v2, v[0:1]2262; GFX942-NEXT: s_waitcnt lgkmcnt(0)2263; GFX942-NEXT: s_endpgm2264;2265; GFX1250-LABEL: local_atomic_fadd_f64_noret_pat_flush:2266; GFX1250: ; %bb.0: ; %main_body2267; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12268; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x242269; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.02270; GFX1250-NEXT: s_wait_kmcnt 0x02271; GFX1250-NEXT: v_mov_b32_e32 v2, s02272; GFX1250-NEXT: ds_add_f64 v2, v[0:1]2273; GFX1250-NEXT: s_wait_dscnt 0x02274; GFX1250-NEXT: s_endpgm2275main_body:2276 %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst, !amdgpu.no.fine.grained.memory !02277 ret void2278}2279 2280define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrspace(3) %ptr) #4 {2281; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe:2282; GFX90A: ; %bb.0: ; %main_body2283; GFX90A-NEXT: s_load_dword s0, s[4:5], 0x242284; GFX90A-NEXT: v_mov_b32_e32 v0, 02285; GFX90A-NEXT: v_mov_b32_e32 v1, 0x401000002286; GFX90A-NEXT: s_waitcnt lgkmcnt(0)2287; GFX90A-NEXT: v_mov_b32_e32 v2, s02288; GFX90A-NEXT: ds_add_f64 v2, v[0:1]2289; GFX90A-NEXT: s_waitcnt lgkmcnt(0)2290; GFX90A-NEXT: s_endpgm2291;2292; GFX942-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe:2293; GFX942: ; %bb.0: ; %main_body2294; GFX942-NEXT: s_load_dword s0, s[4:5], 0x242295; GFX942-NEXT: v_mov_b64_e32 v[0:1], 4.02296; GFX942-NEXT: s_waitcnt lgkmcnt(0)2297; GFX942-NEXT: v_mov_b32_e32 v2, s02298; GFX942-NEXT: ds_add_f64 v2, v[0:1]2299; GFX942-NEXT: s_waitcnt lgkmcnt(0)2300; GFX942-NEXT: s_endpgm2301;2302; GFX1250-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe:2303; GFX1250: ; %bb.0: ; %main_body2304; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12305; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x242306; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 4.02307; GFX1250-NEXT: s_wait_kmcnt 0x02308; GFX1250-NEXT: v_mov_b32_e32 v2, s02309; GFX1250-NEXT: ds_add_f64 v2, v[0:1]2310; GFX1250-NEXT: s_wait_dscnt 0x02311; GFX1250-NEXT: s_endpgm2312main_body:2313 %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst2314 ret void2315}2316 2317define double @local_atomic_fadd_f64_rtn_pat(ptr addrspace(3) %ptr, double %data) #1 {2318; GFX90A-LABEL: local_atomic_fadd_f64_rtn_pat:2319; GFX90A: ; %bb.0: ; %main_body2320; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2321; GFX90A-NEXT: v_mov_b32_e32 v2, 02322; GFX90A-NEXT: v_mov_b32_e32 v3, 0x401000002323; GFX90A-NEXT: ds_add_rtn_f64 v[0:1], v0, v[2:3]2324; GFX90A-NEXT: s_waitcnt lgkmcnt(0)2325; GFX90A-NEXT: s_setpc_b64 s[30:31]2326;2327; GFX942-LABEL: local_atomic_fadd_f64_rtn_pat:2328; GFX942: ; %bb.0: ; %main_body2329; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2330; GFX942-NEXT: v_mov_b64_e32 v[2:3], 4.02331; GFX942-NEXT: ds_add_rtn_f64 v[0:1], v0, v[2:3]2332; GFX942-NEXT: s_waitcnt lgkmcnt(0)2333; GFX942-NEXT: s_setpc_b64 s[30:31]2334;2335; GFX1250-LABEL: local_atomic_fadd_f64_rtn_pat:2336; GFX1250: ; %bb.0: ; %main_body2337; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x02338; GFX1250-NEXT: s_wait_kmcnt 0x02339; GFX1250-NEXT: v_mov_b64_e32 v[2:3], 4.02340; GFX1250-NEXT: s_wait_storecnt 0x02341; GFX1250-NEXT: ds_add_rtn_f64 v[0:1], v0, v[2:3]2342; GFX1250-NEXT: s_wait_dscnt 0x02343; GFX1250-NEXT: s_set_pc_i64 s[30:31]2344main_body:2345 %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst, !amdgpu.no.fine.grained.memory !02346 ret double %ret2347}2348 2349define double @local_atomic_fadd_f64_rtn_ieee_unsafe(ptr addrspace(3) %ptr, double %data) #2 {2350; GFX90A-LABEL: local_atomic_fadd_f64_rtn_ieee_unsafe:2351; GFX90A: ; %bb.0: ; %main_body2352; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2353; GFX90A-NEXT: v_mov_b32_e32 v3, v22354; GFX90A-NEXT: v_mov_b32_e32 v2, v12355; GFX90A-NEXT: ds_add_rtn_f64 v[0:1], v0, v[2:3]2356; GFX90A-NEXT: s_waitcnt lgkmcnt(0)2357; GFX90A-NEXT: s_setpc_b64 s[30:31]2358;2359; GFX942-LABEL: local_atomic_fadd_f64_rtn_ieee_unsafe:2360; GFX942: ; %bb.0: ; %main_body2361; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2362; GFX942-NEXT: v_mov_b32_e32 v3, v22363; GFX942-NEXT: v_mov_b32_e32 v2, v12364; GFX942-NEXT: ds_add_rtn_f64 v[0:1], v0, v[2:3]2365; GFX942-NEXT: s_waitcnt lgkmcnt(0)2366; GFX942-NEXT: s_setpc_b64 s[30:31]2367;2368; GFX1250-LABEL: local_atomic_fadd_f64_rtn_ieee_unsafe:2369; GFX1250: ; %bb.0: ; %main_body2370; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x02371; GFX1250-NEXT: s_wait_kmcnt 0x02372; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v12373; GFX1250-NEXT: s_wait_storecnt 0x02374; GFX1250-NEXT: ds_add_rtn_f64 v[0:1], v0, v[2:3]2375; GFX1250-NEXT: s_wait_dscnt 0x02376; GFX1250-NEXT: s_set_pc_i64 s[30:31]2377main_body:2378 %ret = call double @llvm.amdgcn.ds.fadd.f64(ptr addrspace(3) %ptr, double %data, i32 0, i32 0, i1 0)2379 ret double %ret2380}2381 2382define double @local_atomic_fadd_f64_rtn_ieee_safe(ptr addrspace(3) %ptr, double %data) #3 {2383; GFX90A-LABEL: local_atomic_fadd_f64_rtn_ieee_safe:2384; GFX90A: ; %bb.0: ; %main_body2385; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2386; GFX90A-NEXT: v_mov_b32_e32 v3, v22387; GFX90A-NEXT: v_mov_b32_e32 v2, v12388; GFX90A-NEXT: ds_add_rtn_f64 v[0:1], v0, v[2:3]2389; GFX90A-NEXT: s_waitcnt lgkmcnt(0)2390; GFX90A-NEXT: s_setpc_b64 s[30:31]2391;2392; GFX942-LABEL: local_atomic_fadd_f64_rtn_ieee_safe:2393; GFX942: ; %bb.0: ; %main_body2394; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2395; GFX942-NEXT: v_mov_b32_e32 v3, v22396; GFX942-NEXT: v_mov_b32_e32 v2, v12397; GFX942-NEXT: ds_add_rtn_f64 v[0:1], v0, v[2:3]2398; GFX942-NEXT: s_waitcnt lgkmcnt(0)2399; GFX942-NEXT: s_setpc_b64 s[30:31]2400;2401; GFX1250-LABEL: local_atomic_fadd_f64_rtn_ieee_safe:2402; GFX1250: ; %bb.0: ; %main_body2403; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x02404; GFX1250-NEXT: s_wait_kmcnt 0x02405; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v12406; GFX1250-NEXT: s_wait_storecnt 0x02407; GFX1250-NEXT: ds_add_rtn_f64 v[0:1], v0, v[2:3]2408; GFX1250-NEXT: s_wait_dscnt 0x02409; GFX1250-NEXT: s_set_pc_i64 s[30:31]2410main_body:2411 %ret = call double @llvm.amdgcn.ds.fadd.f64(ptr addrspace(3) %ptr, double %data, i32 0, i32 0, i1 0)2412 ret double %ret2413}2414 2415attributes #0 = { "denormal-fp-math"="preserve-sign,preserve-sign" }2416attributes #1 = { nounwind }2417attributes #2 = { "denormal-fp-math"="ieee,ieee" }2418attributes #3 = { "denormal-fp-math"="ieee,ieee" }2419attributes #4 = { "denormal-fp-math"="preserve-sign,preserve-sign" }2420 2421!0 = !{}2422!1 = !{i32 5, i32 6}2423