brintos / llvm-project-archived public Read only

1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck %s -check-prefix=GFX90A3; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck %s -check-prefix=GFX9424; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck %s -check-prefix=GFX12505 6declare double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double, <4 x i32>, i32, i32, i32, i32 immarg)7declare double @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f64(double, ptr addrspace(8), i32, i32, i32, i32 immarg)8declare double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double, <4 x i32>, i32, i32, i32 immarg)9declare double @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f64(double, ptr addrspace(8), i32, i32, i32 immarg)10declare double @llvm.amdgcn.struct.buffer.atomic.fmin.f64(double, <4 x i32>, i32, i32, i32, i32 immarg)11declare double @llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f64(double, ptr addrspace(8), i32, i32, i32, i32 immarg)12declare double @llvm.amdgcn.raw.buffer.atomic.fmin.f64(double, <4 x i32>, i32, i32, i32 immarg)13declare double @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f64(double, ptr addrspace(8), i32, i32, i32 immarg)14declare double @llvm.amdgcn.struct.buffer.atomic.fmax.f64(double, <4 x i32>, i32, i32, i32, i32 immarg)15declare double @llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f64(double, ptr addrspace(8), i32, i32, i32, i32 immarg)16declare double @llvm.amdgcn.raw.buffer.atomic.fmax.f64(double, <4 x i32>, i32, i32, i32 immarg)17declare double @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f64(double, ptr addrspace(8), i32, i32, i32 immarg)18 19define amdgpu_kernel void @raw_buffer_atomic_add_noret_f64(<4 x i32> %rsrc, double %data, i32 %vindex) {20; GFX90A-LABEL: raw_buffer_atomic_add_noret_f64:21; GFX90A:       ; %bb.0: ; %main_body22; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3423; GFX90A-NEXT:    s_load_dword s8, s[4:5], 0x3c24; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2425; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)26; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]27; GFX90A-NEXT:    v_mov_b32_e32 v2, s828; GFX90A-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 offen29; GFX90A-NEXT:    s_endpgm30;31; GFX942-LABEL: raw_buffer_atomic_add_noret_f64:32; GFX942:       ; %bb.0: ; %main_body33; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3434; GFX942-NEXT:    s_load_dword s8, s[4:5], 0x3c35; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2436; GFX942-NEXT:    s_waitcnt lgkmcnt(0)37; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[6:7]38; GFX942-NEXT:    v_mov_b32_e32 v2, s839; GFX942-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 offen40; GFX942-NEXT:    s_endpgm41;42; GFX1250-LABEL: raw_buffer_atomic_add_noret_f64:43; GFX1250:       ; %bb.0: ; %main_body44; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 145; GFX1250-NEXT:    s_clause 0x146; GFX1250-NEXT:    s_load_b96 s[8:10], s[4:5], 0x3447; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2448; GFX1250-NEXT:    s_wait_kmcnt 0x049; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]50; GFX1250-NEXT:    v_mov_b32_e32 v2, s1051; GFX1250-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], null offen52; GFX1250-NEXT:    s_endpgm53main_body:54  %ret = call double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0)55  ret void56}57 58define amdgpu_ps void @raw_buffer_atomic_add_rtn_f64(<4 x i32> inreg %rsrc, double %data, i32 %vindex) {59; GFX90A-LABEL: raw_buffer_atomic_add_rtn_f64:60; GFX90A:       ; %bb.0: ; %main_body61; GFX90A-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 offen glc62; GFX90A-NEXT:    s_waitcnt vmcnt(0)63; GFX90A-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]64; GFX90A-NEXT:    s_endpgm65;66; GFX942-LABEL: raw_buffer_atomic_add_rtn_f64:67; GFX942:       ; %bb.0: ; %main_body68; GFX942-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 offen sc069; GFX942-NEXT:    s_waitcnt vmcnt(0)70; GFX942-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]71; GFX942-NEXT:    s_endpgm72;73; GFX1250-LABEL: raw_buffer_atomic_add_rtn_f64:74; GFX1250:       ; %bb.0: ; %main_body75; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 176; GFX1250-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN77; GFX1250-NEXT:    s_wait_loadcnt 0x078; GFX1250-NEXT:    flat_store_b64 v[0:1], v[0:1]79; GFX1250-NEXT:    s_endpgm80main_body:81  %ret = call double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0)82  store double %ret, ptr poison83  ret void84}85 86define amdgpu_kernel void @raw_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {87; GFX90A-LABEL: raw_buffer_atomic_add_rtn_f64_off4_slc:88; GFX90A:       ; %bb.0: ; %main_body89; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x3490; GFX90A-NEXT:    s_load_dword s10, s[4:5], 0x3c91; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2492; GFX90A-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x4493; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)94; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]95; GFX90A-NEXT:    v_mov_b32_e32 v0, s1096; GFX90A-NEXT:    buffer_atomic_add_f64 v[2:3], v0, s[0:3], 4 offen glc slc97; GFX90A-NEXT:    v_mov_b32_e32 v0, 098; GFX90A-NEXT:    s_waitcnt vmcnt(0)99; GFX90A-NEXT:    global_store_dwordx2 v0, v[2:3], s[8:9]100; GFX90A-NEXT:    s_endpgm101;102; GFX942-LABEL: raw_buffer_atomic_add_rtn_f64_off4_slc:103; GFX942:       ; %bb.0: ; %main_body104; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34105; GFX942-NEXT:    s_load_dword s10, s[4:5], 0x3c106; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24107; GFX942-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x44108; GFX942-NEXT:    s_waitcnt lgkmcnt(0)109; GFX942-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]110; GFX942-NEXT:    v_mov_b32_e32 v0, s10111; GFX942-NEXT:    buffer_atomic_add_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt112; GFX942-NEXT:    v_mov_b32_e32 v0, 0113; GFX942-NEXT:    s_waitcnt vmcnt(0)114; GFX942-NEXT:    global_store_dwordx2 v0, v[2:3], s[8:9]115; GFX942-NEXT:    s_endpgm116;117; GFX1250-LABEL: raw_buffer_atomic_add_rtn_f64_off4_slc:118; GFX1250:       ; %bb.0: ; %main_body119; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1120; GFX1250-NEXT:    s_clause 0x1121; GFX1250-NEXT:    s_load_b96 s[8:10], s[4:5], 0x34122; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24123; GFX1250-NEXT:    s_mov_b32 s6, 4124; GFX1250-NEXT:    s_wait_kmcnt 0x0125; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]126; GFX1250-NEXT:    v_mov_b32_e32 v2, s10127; GFX1250-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN128; GFX1250-NEXT:    s_wait_xcnt 0x0129; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x44130; GFX1250-NEXT:    v_mov_b32_e32 v2, 0131; GFX1250-NEXT:    s_wait_loadcnt 0x0132; GFX1250-NEXT:    s_wait_kmcnt 0x0133; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[0:1]134; GFX1250-NEXT:    s_endpgm135main_body:136  %ret = call double @llvm.amdgcn.raw.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 2)137  store double %ret, ptr addrspace(1) %out, align 8138  ret void139}140 141define amdgpu_kernel void @raw_ptr_buffer_atomic_add_noret_f64(ptr addrspace(8) %rsrc, double %data, i32 %vindex) {142; GFX90A-LABEL: raw_ptr_buffer_atomic_add_noret_f64:143; GFX90A:       ; %bb.0: ; %main_body144; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34145; GFX90A-NEXT:    s_load_dword s8, s[4:5], 0x3c146; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24147; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)148; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]149; GFX90A-NEXT:    v_mov_b32_e32 v2, s8150; GFX90A-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 offen151; GFX90A-NEXT:    s_endpgm152;153; GFX942-LABEL: raw_ptr_buffer_atomic_add_noret_f64:154; GFX942:       ; %bb.0: ; %main_body155; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34156; GFX942-NEXT:    s_load_dword s8, s[4:5], 0x3c157; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24158; GFX942-NEXT:    s_waitcnt lgkmcnt(0)159; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[6:7]160; GFX942-NEXT:    v_mov_b32_e32 v2, s8161; GFX942-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 offen162; GFX942-NEXT:    s_endpgm163;164; GFX1250-LABEL: raw_ptr_buffer_atomic_add_noret_f64:165; GFX1250:       ; %bb.0: ; %main_body166; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1167; GFX1250-NEXT:    s_clause 0x1168; GFX1250-NEXT:    s_load_b96 s[8:10], s[4:5], 0x34169; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24170; GFX1250-NEXT:    s_wait_kmcnt 0x0171; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]172; GFX1250-NEXT:    v_mov_b32_e32 v2, s10173; GFX1250-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], null offen174; GFX1250-NEXT:    s_endpgm175main_body:176  %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0)177  ret void178}179 180define amdgpu_ps void @raw_ptr_buffer_atomic_add_rtn_f64(ptr addrspace(8) inreg %rsrc, double %data, i32 %vindex) {181; GFX90A-LABEL: raw_ptr_buffer_atomic_add_rtn_f64:182; GFX90A:       ; %bb.0: ; %main_body183; GFX90A-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 offen glc184; GFX90A-NEXT:    s_waitcnt vmcnt(0)185; GFX90A-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]186; GFX90A-NEXT:    s_endpgm187;188; GFX942-LABEL: raw_ptr_buffer_atomic_add_rtn_f64:189; GFX942:       ; %bb.0: ; %main_body190; GFX942-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 offen sc0191; GFX942-NEXT:    s_waitcnt vmcnt(0)192; GFX942-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]193; GFX942-NEXT:    s_endpgm194;195; GFX1250-LABEL: raw_ptr_buffer_atomic_add_rtn_f64:196; GFX1250:       ; %bb.0: ; %main_body197; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1198; GFX1250-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN199; GFX1250-NEXT:    s_wait_loadcnt 0x0200; GFX1250-NEXT:    flat_store_b64 v[0:1], v[0:1]201; GFX1250-NEXT:    s_endpgm202main_body:203  %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0)204  store double %ret, ptr poison205  ret void206}207 208define amdgpu_kernel void @raw_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr addrspace(8) %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {209; GFX90A-LABEL: raw_ptr_buffer_atomic_add_rtn_f64_off4_slc:210; GFX90A:       ; %bb.0: ; %main_body211; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34212; GFX90A-NEXT:    s_load_dword s10, s[4:5], 0x3c213; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24214; GFX90A-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x44215; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)216; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]217; GFX90A-NEXT:    v_mov_b32_e32 v0, s10218; GFX90A-NEXT:    buffer_atomic_add_f64 v[2:3], v0, s[0:3], 4 offen glc slc219; GFX90A-NEXT:    v_mov_b32_e32 v0, 0220; GFX90A-NEXT:    s_waitcnt vmcnt(0)221; GFX90A-NEXT:    global_store_dwordx2 v0, v[2:3], s[8:9]222; GFX90A-NEXT:    s_endpgm223;224; GFX942-LABEL: raw_ptr_buffer_atomic_add_rtn_f64_off4_slc:225; GFX942:       ; %bb.0: ; %main_body226; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34227; GFX942-NEXT:    s_load_dword s10, s[4:5], 0x3c228; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24229; GFX942-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x44230; GFX942-NEXT:    s_waitcnt lgkmcnt(0)231; GFX942-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]232; GFX942-NEXT:    v_mov_b32_e32 v0, s10233; GFX942-NEXT:    buffer_atomic_add_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt234; GFX942-NEXT:    v_mov_b32_e32 v0, 0235; GFX942-NEXT:    s_waitcnt vmcnt(0)236; GFX942-NEXT:    global_store_dwordx2 v0, v[2:3], s[8:9]237; GFX942-NEXT:    s_endpgm238;239; GFX1250-LABEL: raw_ptr_buffer_atomic_add_rtn_f64_off4_slc:240; GFX1250:       ; %bb.0: ; %main_body241; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1242; GFX1250-NEXT:    s_clause 0x1243; GFX1250-NEXT:    s_load_b96 s[8:10], s[4:5], 0x34244; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24245; GFX1250-NEXT:    s_mov_b32 s6, 4246; GFX1250-NEXT:    s_wait_kmcnt 0x0247; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]248; GFX1250-NEXT:    v_mov_b32_e32 v2, s10249; GFX1250-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN250; GFX1250-NEXT:    s_wait_xcnt 0x0251; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x44252; GFX1250-NEXT:    v_mov_b32_e32 v2, 0253; GFX1250-NEXT:    s_wait_loadcnt 0x0254; GFX1250-NEXT:    s_wait_kmcnt 0x0255; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[0:1]256; GFX1250-NEXT:    s_endpgm257main_body:258  %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 2)259  store double %ret, ptr addrspace(1) %out, align 8260  ret void261}262 263define amdgpu_kernel void @struct_buffer_atomic_add_noret_f64(<4 x i32> %rsrc, double %data, i32 %vindex) {264; GFX90A-LABEL: struct_buffer_atomic_add_noret_f64:265; GFX90A:       ; %bb.0: ; %main_body266; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34267; GFX90A-NEXT:    s_load_dword s8, s[4:5], 0x3c268; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24269; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)270; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]271; GFX90A-NEXT:    v_mov_b32_e32 v2, s8272; GFX90A-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 idxen273; GFX90A-NEXT:    s_endpgm274;275; GFX942-LABEL: struct_buffer_atomic_add_noret_f64:276; GFX942:       ; %bb.0: ; %main_body277; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34278; GFX942-NEXT:    s_load_dword s8, s[4:5], 0x3c279; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24280; GFX942-NEXT:    s_waitcnt lgkmcnt(0)281; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[6:7]282; GFX942-NEXT:    v_mov_b32_e32 v2, s8283; GFX942-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 idxen284; GFX942-NEXT:    s_endpgm285;286; GFX1250-LABEL: struct_buffer_atomic_add_noret_f64:287; GFX1250:       ; %bb.0: ; %main_body288; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1289; GFX1250-NEXT:    s_clause 0x1290; GFX1250-NEXT:    s_load_b96 s[8:10], s[4:5], 0x34291; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24292; GFX1250-NEXT:    s_wait_kmcnt 0x0293; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]294; GFX1250-NEXT:    v_mov_b32_e32 v2, s10295; GFX1250-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen296; GFX1250-NEXT:    s_endpgm297main_body:298  %ret = call double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0)299  ret void300}301 302define amdgpu_ps void @struct_buffer_atomic_add_rtn_f64(<4 x i32> inreg %rsrc, double %data, i32 %vindex) {303; GFX90A-LABEL: struct_buffer_atomic_add_rtn_f64:304; GFX90A:       ; %bb.0: ; %main_body305; GFX90A-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 idxen glc306; GFX90A-NEXT:    s_waitcnt vmcnt(0)307; GFX90A-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]308; GFX90A-NEXT:    s_endpgm309;310; GFX942-LABEL: struct_buffer_atomic_add_rtn_f64:311; GFX942:       ; %bb.0: ; %main_body312; GFX942-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 idxen sc0313; GFX942-NEXT:    s_waitcnt vmcnt(0)314; GFX942-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]315; GFX942-NEXT:    s_endpgm316;317; GFX1250-LABEL: struct_buffer_atomic_add_rtn_f64:318; GFX1250:       ; %bb.0: ; %main_body319; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1320; GFX1250-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN321; GFX1250-NEXT:    s_wait_loadcnt 0x0322; GFX1250-NEXT:    flat_store_b64 v[0:1], v[0:1]323; GFX1250-NEXT:    s_endpgm324main_body:325  %ret = call double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0)326  store double %ret, ptr poison327  ret void328}329 330define amdgpu_kernel void @struct_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {331; GFX90A-LABEL: struct_buffer_atomic_add_rtn_f64_off4_slc:332; GFX90A:       ; %bb.0: ; %main_body333; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34334; GFX90A-NEXT:    s_load_dword s10, s[4:5], 0x3c335; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24336; GFX90A-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x44337; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)338; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]339; GFX90A-NEXT:    v_mov_b32_e32 v0, s10340; GFX90A-NEXT:    buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc341; GFX90A-NEXT:    v_mov_b32_e32 v0, 0342; GFX90A-NEXT:    s_waitcnt vmcnt(0)343; GFX90A-NEXT:    global_store_dwordx2 v0, v[2:3], s[8:9]344; GFX90A-NEXT:    s_endpgm345;346; GFX942-LABEL: struct_buffer_atomic_add_rtn_f64_off4_slc:347; GFX942:       ; %bb.0: ; %main_body348; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34349; GFX942-NEXT:    s_load_dword s10, s[4:5], 0x3c350; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24351; GFX942-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x44352; GFX942-NEXT:    s_waitcnt lgkmcnt(0)353; GFX942-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]354; GFX942-NEXT:    v_mov_b32_e32 v0, s10355; GFX942-NEXT:    buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt356; GFX942-NEXT:    v_mov_b32_e32 v0, 0357; GFX942-NEXT:    s_waitcnt vmcnt(0)358; GFX942-NEXT:    global_store_dwordx2 v0, v[2:3], s[8:9]359; GFX942-NEXT:    s_endpgm360;361; GFX1250-LABEL: struct_buffer_atomic_add_rtn_f64_off4_slc:362; GFX1250:       ; %bb.0: ; %main_body363; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1364; GFX1250-NEXT:    s_clause 0x1365; GFX1250-NEXT:    s_load_b96 s[8:10], s[4:5], 0x34366; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24367; GFX1250-NEXT:    s_wait_kmcnt 0x0368; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]369; GFX1250-NEXT:    v_mov_b32_e32 v2, s10370; GFX1250-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN371; GFX1250-NEXT:    s_wait_xcnt 0x0372; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x44373; GFX1250-NEXT:    v_mov_b32_e32 v2, 0374; GFX1250-NEXT:    s_wait_loadcnt 0x0375; GFX1250-NEXT:    s_wait_kmcnt 0x0376; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[0:1]377; GFX1250-NEXT:    s_endpgm378main_body:379  %ret = call double @llvm.amdgcn.struct.buffer.atomic.fadd.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 0, i32 2)380  store double %ret, ptr addrspace(1) %out, align 8381  ret void382}383 384define amdgpu_kernel void @struct_ptr_buffer_atomic_add_noret_f64(ptr addrspace(8) %rsrc, double %data, i32 %vindex) {385; GFX90A-LABEL: struct_ptr_buffer_atomic_add_noret_f64:386; GFX90A:       ; %bb.0: ; %main_body387; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34388; GFX90A-NEXT:    s_load_dword s8, s[4:5], 0x3c389; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24390; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)391; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]392; GFX90A-NEXT:    v_mov_b32_e32 v2, s8393; GFX90A-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 idxen394; GFX90A-NEXT:    s_endpgm395;396; GFX942-LABEL: struct_ptr_buffer_atomic_add_noret_f64:397; GFX942:       ; %bb.0: ; %main_body398; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34399; GFX942-NEXT:    s_load_dword s8, s[4:5], 0x3c400; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24401; GFX942-NEXT:    s_waitcnt lgkmcnt(0)402; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[6:7]403; GFX942-NEXT:    v_mov_b32_e32 v2, s8404; GFX942-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 idxen405; GFX942-NEXT:    s_endpgm406;407; GFX1250-LABEL: struct_ptr_buffer_atomic_add_noret_f64:408; GFX1250:       ; %bb.0: ; %main_body409; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1410; GFX1250-NEXT:    s_clause 0x1411; GFX1250-NEXT:    s_load_b96 s[8:10], s[4:5], 0x34412; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24413; GFX1250-NEXT:    s_wait_kmcnt 0x0414; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]415; GFX1250-NEXT:    v_mov_b32_e32 v2, s10416; GFX1250-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen417; GFX1250-NEXT:    s_endpgm418main_body:419  %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0, i32 0)420  ret void421}422 423define amdgpu_ps void @struct_ptr_buffer_atomic_add_rtn_f64(ptr addrspace(8) inreg %rsrc, double %data, i32 %vindex) {424; GFX90A-LABEL: struct_ptr_buffer_atomic_add_rtn_f64:425; GFX90A:       ; %bb.0: ; %main_body426; GFX90A-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 idxen glc427; GFX90A-NEXT:    s_waitcnt vmcnt(0)428; GFX90A-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]429; GFX90A-NEXT:    s_endpgm430;431; GFX942-LABEL: struct_ptr_buffer_atomic_add_rtn_f64:432; GFX942:       ; %bb.0: ; %main_body433; GFX942-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 idxen sc0434; GFX942-NEXT:    s_waitcnt vmcnt(0)435; GFX942-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]436; GFX942-NEXT:    s_endpgm437;438; GFX1250-LABEL: struct_ptr_buffer_atomic_add_rtn_f64:439; GFX1250:       ; %bb.0: ; %main_body440; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1441; GFX1250-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN442; GFX1250-NEXT:    s_wait_loadcnt 0x0443; GFX1250-NEXT:    flat_store_b64 v[0:1], v[0:1]444; GFX1250-NEXT:    s_endpgm445main_body:446  %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0, i32 0)447  store double %ret, ptr poison448  ret void449}450 451define amdgpu_kernel void @struct_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr addrspace(8) %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {452; GFX90A-LABEL: struct_ptr_buffer_atomic_add_rtn_f64_off4_slc:453; GFX90A:       ; %bb.0: ; %main_body454; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34455; GFX90A-NEXT:    s_load_dword s10, s[4:5], 0x3c456; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24457; GFX90A-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x44458; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)459; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]460; GFX90A-NEXT:    v_mov_b32_e32 v0, s10461; GFX90A-NEXT:    buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc462; GFX90A-NEXT:    v_mov_b32_e32 v0, 0463; GFX90A-NEXT:    s_waitcnt vmcnt(0)464; GFX90A-NEXT:    global_store_dwordx2 v0, v[2:3], s[8:9]465; GFX90A-NEXT:    s_endpgm466;467; GFX942-LABEL: struct_ptr_buffer_atomic_add_rtn_f64_off4_slc:468; GFX942:       ; %bb.0: ; %main_body469; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34470; GFX942-NEXT:    s_load_dword s10, s[4:5], 0x3c471; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24472; GFX942-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x44473; GFX942-NEXT:    s_waitcnt lgkmcnt(0)474; GFX942-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]475; GFX942-NEXT:    v_mov_b32_e32 v0, s10476; GFX942-NEXT:    buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt477; GFX942-NEXT:    v_mov_b32_e32 v0, 0478; GFX942-NEXT:    s_waitcnt vmcnt(0)479; GFX942-NEXT:    global_store_dwordx2 v0, v[2:3], s[8:9]480; GFX942-NEXT:    s_endpgm481;482; GFX1250-LABEL: struct_ptr_buffer_atomic_add_rtn_f64_off4_slc:483; GFX1250:       ; %bb.0: ; %main_body484; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1485; GFX1250-NEXT:    s_clause 0x1486; GFX1250-NEXT:    s_load_b96 s[8:10], s[4:5], 0x34487; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24488; GFX1250-NEXT:    s_wait_kmcnt 0x0489; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]490; GFX1250-NEXT:    v_mov_b32_e32 v2, s10491; GFX1250-NEXT:    buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN492; GFX1250-NEXT:    s_wait_xcnt 0x0493; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x44494; GFX1250-NEXT:    v_mov_b32_e32 v2, 0495; GFX1250-NEXT:    s_wait_loadcnt 0x0496; GFX1250-NEXT:    s_wait_kmcnt 0x0497; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[0:1]498; GFX1250-NEXT:    s_endpgm499main_body:500  %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 0, i32 2)501  store double %ret, ptr addrspace(1) %out, align 8502  ret void503}504 505define amdgpu_kernel void @raw_buffer_atomic_min_noret_f64(<4 x i32> %rsrc, double %data, i32 %vindex) {506; GFX90A-LABEL: raw_buffer_atomic_min_noret_f64:507; GFX90A:       ; %bb.0: ; %main_body508; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34509; GFX90A-NEXT:    s_load_dword s8, s[4:5], 0x3c510; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24511; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)512; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]513; GFX90A-NEXT:    v_mov_b32_e32 v2, s8514; GFX90A-NEXT:    buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen515; GFX90A-NEXT:    s_endpgm516;517; GFX942-LABEL: raw_buffer_atomic_min_noret_f64:518; GFX942:       ; %bb.0: ; %main_body519; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34520; GFX942-NEXT:    s_load_dword s8, s[4:5], 0x3c521; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24522; GFX942-NEXT:    s_waitcnt lgkmcnt(0)523; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[6:7]524; GFX942-NEXT:    v_mov_b32_e32 v2, s8525; GFX942-NEXT:    buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen526; GFX942-NEXT:    s_endpgm527;528; GFX1250-LABEL: raw_buffer_atomic_min_noret_f64:529; GFX1250:       ; %bb.0: ; %main_body530; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1531; GFX1250-NEXT:    s_clause 0x1532; GFX1250-NEXT:    s_load_b96 s[8:10], s[4:5], 0x34533; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24534; GFX1250-NEXT:    s_wait_kmcnt 0x0535; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]536; GFX1250-NEXT:    v_mov_b32_e32 v2, s10537; GFX1250-NEXT:    buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null offen538; GFX1250-NEXT:    s_endpgm539main_body:540  %ret = call double @llvm.amdgcn.raw.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0)541  ret void542}543 544define amdgpu_ps void @raw_buffer_atomic_min_rtn_f64(<4 x i32> inreg %rsrc, double %data, i32 %vindex) {545; GFX90A-LABEL: raw_buffer_atomic_min_rtn_f64:546; GFX90A:       ; %bb.0: ; %main_body547; GFX90A-NEXT:    buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen glc548; GFX90A-NEXT:    s_waitcnt vmcnt(0)549; GFX90A-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]550; GFX90A-NEXT:    s_endpgm551;552; GFX942-LABEL: raw_buffer_atomic_min_rtn_f64:553; GFX942:       ; %bb.0: ; %main_body554; GFX942-NEXT:    buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen sc0555; GFX942-NEXT:    s_waitcnt vmcnt(0)556; GFX942-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]557; GFX942-NEXT:    s_endpgm558;559; GFX1250-LABEL: raw_buffer_atomic_min_rtn_f64:560; GFX1250:       ; %bb.0: ; %main_body561; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1562; GFX1250-NEXT:    buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN563; GFX1250-NEXT:    s_wait_loadcnt 0x0564; GFX1250-NEXT:    flat_store_b64 v[0:1], v[0:1]565; GFX1250-NEXT:    s_endpgm566main_body:567  %ret = call double @llvm.amdgcn.raw.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0)568  store double %ret, ptr poison569  ret void570}571 572define amdgpu_kernel void @raw_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {573; GFX90A-LABEL: raw_buffer_atomic_min_rtn_f64_off4_slc:574; GFX90A:       ; %bb.0: ; %main_body575; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34576; GFX90A-NEXT:    s_load_dword s10, s[4:5], 0x3c577; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24578; GFX90A-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x44579; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)580; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]581; GFX90A-NEXT:    v_mov_b32_e32 v0, s10582; GFX90A-NEXT:    buffer_atomic_min_f64 v[2:3], v0, s[0:3], 4 offen glc slc583; GFX90A-NEXT:    v_mov_b32_e32 v0, 0584; GFX90A-NEXT:    s_waitcnt vmcnt(0)585; GFX90A-NEXT:    global_store_dwordx2 v0, v[2:3], s[8:9]586; GFX90A-NEXT:    s_endpgm587;588; GFX942-LABEL: raw_buffer_atomic_min_rtn_f64_off4_slc:589; GFX942:       ; %bb.0: ; %main_body590; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34591; GFX942-NEXT:    s_load_dword s10, s[4:5], 0x3c592; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24593; GFX942-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x44594; GFX942-NEXT:    s_waitcnt lgkmcnt(0)595; GFX942-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]596; GFX942-NEXT:    v_mov_b32_e32 v0, s10597; GFX942-NEXT:    buffer_atomic_min_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt598; GFX942-NEXT:    v_mov_b32_e32 v0, 0599; GFX942-NEXT:    s_waitcnt vmcnt(0)600; GFX942-NEXT:    global_store_dwordx2 v0, v[2:3], s[8:9]601; GFX942-NEXT:    s_endpgm602;603; GFX1250-LABEL: raw_buffer_atomic_min_rtn_f64_off4_slc:604; GFX1250:       ; %bb.0: ; %main_body605; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1606; GFX1250-NEXT:    s_clause 0x1607; GFX1250-NEXT:    s_load_b96 s[8:10], s[4:5], 0x34608; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24609; GFX1250-NEXT:    s_mov_b32 s6, 4610; GFX1250-NEXT:    s_wait_kmcnt 0x0611; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]612; GFX1250-NEXT:    v_mov_b32_e32 v2, s10613; GFX1250-NEXT:    buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN614; GFX1250-NEXT:    s_wait_xcnt 0x0615; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x44616; GFX1250-NEXT:    v_mov_b32_e32 v2, 0617; GFX1250-NEXT:    s_wait_loadcnt 0x0618; GFX1250-NEXT:    s_wait_kmcnt 0x0619; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[0:1]620; GFX1250-NEXT:    s_endpgm621main_body:622  %ret = call double @llvm.amdgcn.raw.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 2)623  store double %ret, ptr addrspace(1) %out, align 8624  ret void625}626 627define amdgpu_kernel void @raw_ptr_buffer_atomic_min_noret_f64(ptr addrspace(8) %rsrc, double %data, i32 %vindex) {628; GFX90A-LABEL: raw_ptr_buffer_atomic_min_noret_f64:629; GFX90A:       ; %bb.0: ; %main_body630; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34631; GFX90A-NEXT:    s_load_dword s8, s[4:5], 0x3c632; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24633; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)634; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]635; GFX90A-NEXT:    v_mov_b32_e32 v2, s8636; GFX90A-NEXT:    buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen637; GFX90A-NEXT:    s_endpgm638;639; GFX942-LABEL: raw_ptr_buffer_atomic_min_noret_f64:640; GFX942:       ; %bb.0: ; %main_body641; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34642; GFX942-NEXT:    s_load_dword s8, s[4:5], 0x3c643; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24644; GFX942-NEXT:    s_waitcnt lgkmcnt(0)645; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[6:7]646; GFX942-NEXT:    v_mov_b32_e32 v2, s8647; GFX942-NEXT:    buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen648; GFX942-NEXT:    s_endpgm649;650; GFX1250-LABEL: raw_ptr_buffer_atomic_min_noret_f64:651; GFX1250:       ; %bb.0: ; %main_body652; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1653; GFX1250-NEXT:    s_clause 0x1654; GFX1250-NEXT:    s_load_b96 s[8:10], s[4:5], 0x34655; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24656; GFX1250-NEXT:    s_wait_kmcnt 0x0657; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]658; GFX1250-NEXT:    v_mov_b32_e32 v2, s10659; GFX1250-NEXT:    buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null offen660; GFX1250-NEXT:    s_endpgm661main_body:662  %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0)663  ret void664}665 666define amdgpu_ps void @raw_ptr_buffer_atomic_min_rtn_f64(ptr addrspace(8) inreg %rsrc, double %data, i32 %vindex) {667; GFX90A-LABEL: raw_ptr_buffer_atomic_min_rtn_f64:668; GFX90A:       ; %bb.0: ; %main_body669; GFX90A-NEXT:    buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen glc670; GFX90A-NEXT:    s_waitcnt vmcnt(0)671; GFX90A-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]672; GFX90A-NEXT:    s_endpgm673;674; GFX942-LABEL: raw_ptr_buffer_atomic_min_rtn_f64:675; GFX942:       ; %bb.0: ; %main_body676; GFX942-NEXT:    buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 offen sc0677; GFX942-NEXT:    s_waitcnt vmcnt(0)678; GFX942-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]679; GFX942-NEXT:    s_endpgm680;681; GFX1250-LABEL: raw_ptr_buffer_atomic_min_rtn_f64:682; GFX1250:       ; %bb.0: ; %main_body683; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1684; GFX1250-NEXT:    buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN685; GFX1250-NEXT:    s_wait_loadcnt 0x0686; GFX1250-NEXT:    flat_store_b64 v[0:1], v[0:1]687; GFX1250-NEXT:    s_endpgm688main_body:689  %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0)690  store double %ret, ptr poison691  ret void692}693 694define amdgpu_kernel void @raw_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr addrspace(8) %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {695; GFX90A-LABEL: raw_ptr_buffer_atomic_min_rtn_f64_off4_slc:696; GFX90A:       ; %bb.0: ; %main_body697; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34698; GFX90A-NEXT:    s_load_dword s10, s[4:5], 0x3c699; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24700; GFX90A-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x44701; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)702; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]703; GFX90A-NEXT:    v_mov_b32_e32 v0, s10704; GFX90A-NEXT:    buffer_atomic_min_f64 v[2:3], v0, s[0:3], 4 offen glc slc705; GFX90A-NEXT:    v_mov_b32_e32 v0, 0706; GFX90A-NEXT:    s_waitcnt vmcnt(0)707; GFX90A-NEXT:    global_store_dwordx2 v0, v[2:3], s[8:9]708; GFX90A-NEXT:    s_endpgm709;710; GFX942-LABEL: raw_ptr_buffer_atomic_min_rtn_f64_off4_slc:711; GFX942:       ; %bb.0: ; %main_body712; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34713; GFX942-NEXT:    s_load_dword s10, s[4:5], 0x3c714; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24715; GFX942-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x44716; GFX942-NEXT:    s_waitcnt lgkmcnt(0)717; GFX942-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]718; GFX942-NEXT:    v_mov_b32_e32 v0, s10719; GFX942-NEXT:    buffer_atomic_min_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt720; GFX942-NEXT:    v_mov_b32_e32 v0, 0721; GFX942-NEXT:    s_waitcnt vmcnt(0)722; GFX942-NEXT:    global_store_dwordx2 v0, v[2:3], s[8:9]723; GFX942-NEXT:    s_endpgm724;725; GFX1250-LABEL: raw_ptr_buffer_atomic_min_rtn_f64_off4_slc:726; GFX1250:       ; %bb.0: ; %main_body727; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1728; GFX1250-NEXT:    s_clause 0x1729; GFX1250-NEXT:    s_load_b96 s[8:10], s[4:5], 0x34730; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24731; GFX1250-NEXT:    s_mov_b32 s6, 4732; GFX1250-NEXT:    s_wait_kmcnt 0x0733; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]734; GFX1250-NEXT:    v_mov_b32_e32 v2, s10735; GFX1250-NEXT:    buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN736; GFX1250-NEXT:    s_wait_xcnt 0x0737; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x44738; GFX1250-NEXT:    v_mov_b32_e32 v2, 0739; GFX1250-NEXT:    s_wait_loadcnt 0x0740; GFX1250-NEXT:    s_wait_kmcnt 0x0741; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[0:1]742; GFX1250-NEXT:    s_endpgm743main_body:744  %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmin.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 2)745  store double %ret, ptr addrspace(1) %out, align 8746  ret void747}748 749define amdgpu_kernel void @struct_buffer_atomic_min_noret_f64(<4 x i32> %rsrc, double %data, i32 %vindex) {750; GFX90A-LABEL: struct_buffer_atomic_min_noret_f64:751; GFX90A:       ; %bb.0: ; %main_body752; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34753; GFX90A-NEXT:    s_load_dword s8, s[4:5], 0x3c754; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24755; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)756; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]757; GFX90A-NEXT:    v_mov_b32_e32 v2, s8758; GFX90A-NEXT:    buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 idxen759; GFX90A-NEXT:    s_endpgm760;761; GFX942-LABEL: struct_buffer_atomic_min_noret_f64:762; GFX942:       ; %bb.0: ; %main_body763; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34764; GFX942-NEXT:    s_load_dword s8, s[4:5], 0x3c765; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24766; GFX942-NEXT:    s_waitcnt lgkmcnt(0)767; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[6:7]768; GFX942-NEXT:    v_mov_b32_e32 v2, s8769; GFX942-NEXT:    buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 idxen770; GFX942-NEXT:    s_endpgm771;772; GFX1250-LABEL: struct_buffer_atomic_min_noret_f64:773; GFX1250:       ; %bb.0: ; %main_body774; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1775; GFX1250-NEXT:    s_clause 0x1776; GFX1250-NEXT:    s_load_b96 s[8:10], s[4:5], 0x34777; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24778; GFX1250-NEXT:    s_wait_kmcnt 0x0779; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]780; GFX1250-NEXT:    v_mov_b32_e32 v2, s10781; GFX1250-NEXT:    buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen782; GFX1250-NEXT:    s_endpgm783main_body:784  %ret = call double @llvm.amdgcn.struct.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0)785  ret void786}787 788define amdgpu_ps void @struct_buffer_atomic_min_rtn_f64(<4 x i32> inreg %rsrc, double %data, i32 %vindex) {789; GFX90A-LABEL: struct_buffer_atomic_min_rtn_f64:790; GFX90A:       ; %bb.0: ; %main_body791; GFX90A-NEXT:    buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 idxen glc792; GFX90A-NEXT:    s_waitcnt vmcnt(0)793; GFX90A-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]794; GFX90A-NEXT:    s_endpgm795;796; GFX942-LABEL: struct_buffer_atomic_min_rtn_f64:797; GFX942:       ; %bb.0: ; %main_body798; GFX942-NEXT:    buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 idxen sc0799; GFX942-NEXT:    s_waitcnt vmcnt(0)800; GFX942-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]801; GFX942-NEXT:    s_endpgm802;803; GFX1250-LABEL: struct_buffer_atomic_min_rtn_f64:804; GFX1250:       ; %bb.0: ; %main_body805; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1806; GFX1250-NEXT:    buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN807; GFX1250-NEXT:    s_wait_loadcnt 0x0808; GFX1250-NEXT:    flat_store_b64 v[0:1], v[0:1]809; GFX1250-NEXT:    s_endpgm810main_body:811  %ret = call double @llvm.amdgcn.struct.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0)812  store double %ret, ptr poison813  ret void814}815 816define amdgpu_kernel void @struct_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {817; GFX90A-LABEL: struct_buffer_atomic_min_rtn_f64_off4_slc:818; GFX90A:       ; %bb.0: ; %main_body819; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34820; GFX90A-NEXT:    s_load_dword s10, s[4:5], 0x3c821; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24822; GFX90A-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x44823; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)824; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]825; GFX90A-NEXT:    v_mov_b32_e32 v0, s10826; GFX90A-NEXT:    buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc827; GFX90A-NEXT:    v_mov_b32_e32 v0, 0828; GFX90A-NEXT:    s_waitcnt vmcnt(0)829; GFX90A-NEXT:    global_store_dwordx2 v0, v[2:3], s[8:9]830; GFX90A-NEXT:    s_endpgm831;832; GFX942-LABEL: struct_buffer_atomic_min_rtn_f64_off4_slc:833; GFX942:       ; %bb.0: ; %main_body834; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34835; GFX942-NEXT:    s_load_dword s10, s[4:5], 0x3c836; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24837; GFX942-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x44838; GFX942-NEXT:    s_waitcnt lgkmcnt(0)839; GFX942-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]840; GFX942-NEXT:    v_mov_b32_e32 v0, s10841; GFX942-NEXT:    buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt842; GFX942-NEXT:    v_mov_b32_e32 v0, 0843; GFX942-NEXT:    s_waitcnt vmcnt(0)844; GFX942-NEXT:    global_store_dwordx2 v0, v[2:3], s[8:9]845; GFX942-NEXT:    s_endpgm846;847; GFX1250-LABEL: struct_buffer_atomic_min_rtn_f64_off4_slc:848; GFX1250:       ; %bb.0: ; %main_body849; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1850; GFX1250-NEXT:    s_clause 0x1851; GFX1250-NEXT:    s_load_b96 s[8:10], s[4:5], 0x34852; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24853; GFX1250-NEXT:    s_wait_kmcnt 0x0854; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]855; GFX1250-NEXT:    v_mov_b32_e32 v2, s10856; GFX1250-NEXT:    buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN857; GFX1250-NEXT:    s_wait_xcnt 0x0858; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x44859; GFX1250-NEXT:    v_mov_b32_e32 v2, 0860; GFX1250-NEXT:    s_wait_loadcnt 0x0861; GFX1250-NEXT:    s_wait_kmcnt 0x0862; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[0:1]863; GFX1250-NEXT:    s_endpgm864main_body:865  %ret = call double @llvm.amdgcn.struct.buffer.atomic.fmin.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 0, i32 2)866  store double %ret, ptr addrspace(1) %out, align 8867  ret void868}869 870define amdgpu_kernel void @struct_ptr_buffer_atomic_min_noret_f64(ptr addrspace(8) %rsrc, double %data, i32 %vindex) {871; GFX90A-LABEL: struct_ptr_buffer_atomic_min_noret_f64:872; GFX90A:       ; %bb.0: ; %main_body873; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34874; GFX90A-NEXT:    s_load_dword s8, s[4:5], 0x3c875; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24876; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)877; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]878; GFX90A-NEXT:    v_mov_b32_e32 v2, s8879; GFX90A-NEXT:    buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 idxen880; GFX90A-NEXT:    s_endpgm881;882; GFX942-LABEL: struct_ptr_buffer_atomic_min_noret_f64:883; GFX942:       ; %bb.0: ; %main_body884; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34885; GFX942-NEXT:    s_load_dword s8, s[4:5], 0x3c886; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24887; GFX942-NEXT:    s_waitcnt lgkmcnt(0)888; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[6:7]889; GFX942-NEXT:    v_mov_b32_e32 v2, s8890; GFX942-NEXT:    buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 idxen891; GFX942-NEXT:    s_endpgm892;893; GFX1250-LABEL: struct_ptr_buffer_atomic_min_noret_f64:894; GFX1250:       ; %bb.0: ; %main_body895; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1896; GFX1250-NEXT:    s_clause 0x1897; GFX1250-NEXT:    s_load_b96 s[8:10], s[4:5], 0x34898; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24899; GFX1250-NEXT:    s_wait_kmcnt 0x0900; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]901; GFX1250-NEXT:    v_mov_b32_e32 v2, s10902; GFX1250-NEXT:    buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen903; GFX1250-NEXT:    s_endpgm904main_body:905  %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0, i32 0)906  ret void907}908 909define amdgpu_ps void @struct_ptr_buffer_atomic_min_rtn_f64(ptr addrspace(8) inreg %rsrc, double %data, i32 %vindex) {910; GFX90A-LABEL: struct_ptr_buffer_atomic_min_rtn_f64:911; GFX90A:       ; %bb.0: ; %main_body912; GFX90A-NEXT:    buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 idxen glc913; GFX90A-NEXT:    s_waitcnt vmcnt(0)914; GFX90A-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]915; GFX90A-NEXT:    s_endpgm916;917; GFX942-LABEL: struct_ptr_buffer_atomic_min_rtn_f64:918; GFX942:       ; %bb.0: ; %main_body919; GFX942-NEXT:    buffer_atomic_min_f64 v[0:1], v2, s[0:3], 0 idxen sc0920; GFX942-NEXT:    s_waitcnt vmcnt(0)921; GFX942-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]922; GFX942-NEXT:    s_endpgm923;924; GFX1250-LABEL: struct_ptr_buffer_atomic_min_rtn_f64:925; GFX1250:       ; %bb.0: ; %main_body926; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1927; GFX1250-NEXT:    buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN928; GFX1250-NEXT:    s_wait_loadcnt 0x0929; GFX1250-NEXT:    flat_store_b64 v[0:1], v[0:1]930; GFX1250-NEXT:    s_endpgm931main_body:932  %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0, i32 0)933  store double %ret, ptr poison934  ret void935}936 937define amdgpu_kernel void @struct_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr addrspace(8) %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {938; GFX90A-LABEL: struct_ptr_buffer_atomic_min_rtn_f64_off4_slc:939; GFX90A:       ; %bb.0: ; %main_body940; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34941; GFX90A-NEXT:    s_load_dword s10, s[4:5], 0x3c942; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24943; GFX90A-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x44944; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)945; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]946; GFX90A-NEXT:    v_mov_b32_e32 v0, s10947; GFX90A-NEXT:    buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc948; GFX90A-NEXT:    v_mov_b32_e32 v0, 0949; GFX90A-NEXT:    s_waitcnt vmcnt(0)950; GFX90A-NEXT:    global_store_dwordx2 v0, v[2:3], s[8:9]951; GFX90A-NEXT:    s_endpgm952;953; GFX942-LABEL: struct_ptr_buffer_atomic_min_rtn_f64_off4_slc:954; GFX942:       ; %bb.0: ; %main_body955; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34956; GFX942-NEXT:    s_load_dword s10, s[4:5], 0x3c957; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24958; GFX942-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x44959; GFX942-NEXT:    s_waitcnt lgkmcnt(0)960; GFX942-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]961; GFX942-NEXT:    v_mov_b32_e32 v0, s10962; GFX942-NEXT:    buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt963; GFX942-NEXT:    v_mov_b32_e32 v0, 0964; GFX942-NEXT:    s_waitcnt vmcnt(0)965; GFX942-NEXT:    global_store_dwordx2 v0, v[2:3], s[8:9]966; GFX942-NEXT:    s_endpgm967;968; GFX1250-LABEL: struct_ptr_buffer_atomic_min_rtn_f64_off4_slc:969; GFX1250:       ; %bb.0: ; %main_body970; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1971; GFX1250-NEXT:    s_clause 0x1972; GFX1250-NEXT:    s_load_b96 s[8:10], s[4:5], 0x34973; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24974; GFX1250-NEXT:    s_wait_kmcnt 0x0975; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]976; GFX1250-NEXT:    v_mov_b32_e32 v2, s10977; GFX1250-NEXT:    buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN978; GFX1250-NEXT:    s_wait_xcnt 0x0979; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x44980; GFX1250-NEXT:    v_mov_b32_e32 v2, 0981; GFX1250-NEXT:    s_wait_loadcnt 0x0982; GFX1250-NEXT:    s_wait_kmcnt 0x0983; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[0:1]984; GFX1250-NEXT:    s_endpgm985main_body:986  %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 0, i32 2)987  store double %ret, ptr addrspace(1) %out, align 8988  ret void989}990 991define amdgpu_kernel void @raw_buffer_atomic_max_noret_f64(<4 x i32> %rsrc, double %data, i32 %vindex) {992; GFX90A-LABEL: raw_buffer_atomic_max_noret_f64:993; GFX90A:       ; %bb.0: ; %main_body994; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34995; GFX90A-NEXT:    s_load_dword s8, s[4:5], 0x3c996; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24997; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)998; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]999; GFX90A-NEXT:    v_mov_b32_e32 v2, s81000; GFX90A-NEXT:    buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen1001; GFX90A-NEXT:    s_endpgm1002;1003; GFX942-LABEL: raw_buffer_atomic_max_noret_f64:1004; GFX942:       ; %bb.0: ; %main_body1005; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x341006; GFX942-NEXT:    s_load_dword s8, s[4:5], 0x3c1007; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241008; GFX942-NEXT:    s_waitcnt lgkmcnt(0)1009; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[6:7]1010; GFX942-NEXT:    v_mov_b32_e32 v2, s81011; GFX942-NEXT:    buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen1012; GFX942-NEXT:    s_endpgm1013;1014; GFX1250-LABEL: raw_buffer_atomic_max_noret_f64:1015; GFX1250:       ; %bb.0: ; %main_body1016; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11017; GFX1250-NEXT:    s_clause 0x11018; GFX1250-NEXT:    s_load_b96 s[8:10], s[4:5], 0x341019; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x241020; GFX1250-NEXT:    s_wait_kmcnt 0x01021; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]1022; GFX1250-NEXT:    v_mov_b32_e32 v2, s101023; GFX1250-NEXT:    buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null offen1024; GFX1250-NEXT:    s_endpgm1025main_body:1026  %ret = call double @llvm.amdgcn.raw.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0)1027  ret void1028}1029 1030define amdgpu_ps void @raw_buffer_atomic_max_rtn_f64(<4 x i32> inreg %rsrc, double %data, i32 %vindex) {1031; GFX90A-LABEL: raw_buffer_atomic_max_rtn_f64:1032; GFX90A:       ; %bb.0: ; %main_body1033; GFX90A-NEXT:    buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen glc1034; GFX90A-NEXT:    s_waitcnt vmcnt(0)1035; GFX90A-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]1036; GFX90A-NEXT:    s_endpgm1037;1038; GFX942-LABEL: raw_buffer_atomic_max_rtn_f64:1039; GFX942:       ; %bb.0: ; %main_body1040; GFX942-NEXT:    buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen sc01041; GFX942-NEXT:    s_waitcnt vmcnt(0)1042; GFX942-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]1043; GFX942-NEXT:    s_endpgm1044;1045; GFX1250-LABEL: raw_buffer_atomic_max_rtn_f64:1046; GFX1250:       ; %bb.0: ; %main_body1047; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11048; GFX1250-NEXT:    buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN1049; GFX1250-NEXT:    s_wait_loadcnt 0x01050; GFX1250-NEXT:    flat_store_b64 v[0:1], v[0:1]1051; GFX1250-NEXT:    s_endpgm1052main_body:1053  %ret = call double @llvm.amdgcn.raw.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0)1054  store double %ret, ptr poison1055  ret void1056}1057 1058define amdgpu_kernel void @raw_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {1059; GFX90A-LABEL: raw_buffer_atomic_max_rtn_f64_off4_slc:1060; GFX90A:       ; %bb.0: ; %main_body1061; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x341062; GFX90A-NEXT:    s_load_dword s10, s[4:5], 0x3c1063; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241064; GFX90A-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x441065; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)1066; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]1067; GFX90A-NEXT:    v_mov_b32_e32 v0, s101068; GFX90A-NEXT:    buffer_atomic_max_f64 v[2:3], v0, s[0:3], 4 offen glc slc1069; GFX90A-NEXT:    v_mov_b32_e32 v0, 01070; GFX90A-NEXT:    s_waitcnt vmcnt(0)1071; GFX90A-NEXT:    global_store_dwordx2 v0, v[2:3], s[8:9]1072; GFX90A-NEXT:    s_endpgm1073;1074; GFX942-LABEL: raw_buffer_atomic_max_rtn_f64_off4_slc:1075; GFX942:       ; %bb.0: ; %main_body1076; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x341077; GFX942-NEXT:    s_load_dword s10, s[4:5], 0x3c1078; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241079; GFX942-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x441080; GFX942-NEXT:    s_waitcnt lgkmcnt(0)1081; GFX942-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]1082; GFX942-NEXT:    v_mov_b32_e32 v0, s101083; GFX942-NEXT:    buffer_atomic_max_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt1084; GFX942-NEXT:    v_mov_b32_e32 v0, 01085; GFX942-NEXT:    s_waitcnt vmcnt(0)1086; GFX942-NEXT:    global_store_dwordx2 v0, v[2:3], s[8:9]1087; GFX942-NEXT:    s_endpgm1088;1089; GFX1250-LABEL: raw_buffer_atomic_max_rtn_f64_off4_slc:1090; GFX1250:       ; %bb.0: ; %main_body1091; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11092; GFX1250-NEXT:    s_clause 0x11093; GFX1250-NEXT:    s_load_b96 s[8:10], s[4:5], 0x341094; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x241095; GFX1250-NEXT:    s_mov_b32 s6, 41096; GFX1250-NEXT:    s_wait_kmcnt 0x01097; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]1098; GFX1250-NEXT:    v_mov_b32_e32 v2, s101099; GFX1250-NEXT:    buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN1100; GFX1250-NEXT:    s_wait_xcnt 0x01101; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x441102; GFX1250-NEXT:    v_mov_b32_e32 v2, 01103; GFX1250-NEXT:    s_wait_loadcnt 0x01104; GFX1250-NEXT:    s_wait_kmcnt 0x01105; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[0:1]1106; GFX1250-NEXT:    s_endpgm1107main_body:1108  %ret = call double @llvm.amdgcn.raw.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 2)1109  store double %ret, ptr addrspace(1) %out, align 81110  ret void1111}1112 1113define amdgpu_kernel void @raw_ptr_buffer_atomic_max_noret_f64(ptr addrspace(8) %rsrc, double %data, i32 %vindex) {1114; GFX90A-LABEL: raw_ptr_buffer_atomic_max_noret_f64:1115; GFX90A:       ; %bb.0: ; %main_body1116; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x341117; GFX90A-NEXT:    s_load_dword s8, s[4:5], 0x3c1118; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241119; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)1120; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]1121; GFX90A-NEXT:    v_mov_b32_e32 v2, s81122; GFX90A-NEXT:    buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen1123; GFX90A-NEXT:    s_endpgm1124;1125; GFX942-LABEL: raw_ptr_buffer_atomic_max_noret_f64:1126; GFX942:       ; %bb.0: ; %main_body1127; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x341128; GFX942-NEXT:    s_load_dword s8, s[4:5], 0x3c1129; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241130; GFX942-NEXT:    s_waitcnt lgkmcnt(0)1131; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[6:7]1132; GFX942-NEXT:    v_mov_b32_e32 v2, s81133; GFX942-NEXT:    buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen1134; GFX942-NEXT:    s_endpgm1135;1136; GFX1250-LABEL: raw_ptr_buffer_atomic_max_noret_f64:1137; GFX1250:       ; %bb.0: ; %main_body1138; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11139; GFX1250-NEXT:    s_clause 0x11140; GFX1250-NEXT:    s_load_b96 s[8:10], s[4:5], 0x341141; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x241142; GFX1250-NEXT:    s_wait_kmcnt 0x01143; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]1144; GFX1250-NEXT:    v_mov_b32_e32 v2, s101145; GFX1250-NEXT:    buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null offen1146; GFX1250-NEXT:    s_endpgm1147main_body:1148  %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0)1149  ret void1150}1151 1152define amdgpu_ps void @raw_ptr_buffer_atomic_max_rtn_f64(ptr addrspace(8) inreg %rsrc, double %data, i32 %vindex) {1153; GFX90A-LABEL: raw_ptr_buffer_atomic_max_rtn_f64:1154; GFX90A:       ; %bb.0: ; %main_body1155; GFX90A-NEXT:    buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen glc1156; GFX90A-NEXT:    s_waitcnt vmcnt(0)1157; GFX90A-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]1158; GFX90A-NEXT:    s_endpgm1159;1160; GFX942-LABEL: raw_ptr_buffer_atomic_max_rtn_f64:1161; GFX942:       ; %bb.0: ; %main_body1162; GFX942-NEXT:    buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 offen sc01163; GFX942-NEXT:    s_waitcnt vmcnt(0)1164; GFX942-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]1165; GFX942-NEXT:    s_endpgm1166;1167; GFX1250-LABEL: raw_ptr_buffer_atomic_max_rtn_f64:1168; GFX1250:       ; %bb.0: ; %main_body1169; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11170; GFX1250-NEXT:    buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null offen th:TH_ATOMIC_RETURN1171; GFX1250-NEXT:    s_wait_loadcnt 0x01172; GFX1250-NEXT:    flat_store_b64 v[0:1], v[0:1]1173; GFX1250-NEXT:    s_endpgm1174main_body:1175  %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0)1176  store double %ret, ptr poison1177  ret void1178}1179 1180define amdgpu_kernel void @raw_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr addrspace(8) %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {1181; GFX90A-LABEL: raw_ptr_buffer_atomic_max_rtn_f64_off4_slc:1182; GFX90A:       ; %bb.0: ; %main_body1183; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x341184; GFX90A-NEXT:    s_load_dword s10, s[4:5], 0x3c1185; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241186; GFX90A-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x441187; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)1188; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]1189; GFX90A-NEXT:    v_mov_b32_e32 v0, s101190; GFX90A-NEXT:    buffer_atomic_max_f64 v[2:3], v0, s[0:3], 4 offen glc slc1191; GFX90A-NEXT:    v_mov_b32_e32 v0, 01192; GFX90A-NEXT:    s_waitcnt vmcnt(0)1193; GFX90A-NEXT:    global_store_dwordx2 v0, v[2:3], s[8:9]1194; GFX90A-NEXT:    s_endpgm1195;1196; GFX942-LABEL: raw_ptr_buffer_atomic_max_rtn_f64_off4_slc:1197; GFX942:       ; %bb.0: ; %main_body1198; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x341199; GFX942-NEXT:    s_load_dword s10, s[4:5], 0x3c1200; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241201; GFX942-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x441202; GFX942-NEXT:    s_waitcnt lgkmcnt(0)1203; GFX942-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]1204; GFX942-NEXT:    v_mov_b32_e32 v0, s101205; GFX942-NEXT:    buffer_atomic_max_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt1206; GFX942-NEXT:    v_mov_b32_e32 v0, 01207; GFX942-NEXT:    s_waitcnt vmcnt(0)1208; GFX942-NEXT:    global_store_dwordx2 v0, v[2:3], s[8:9]1209; GFX942-NEXT:    s_endpgm1210;1211; GFX1250-LABEL: raw_ptr_buffer_atomic_max_rtn_f64_off4_slc:1212; GFX1250:       ; %bb.0: ; %main_body1213; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11214; GFX1250-NEXT:    s_clause 0x11215; GFX1250-NEXT:    s_load_b96 s[8:10], s[4:5], 0x341216; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x241217; GFX1250-NEXT:    s_mov_b32 s6, 41218; GFX1250-NEXT:    s_wait_kmcnt 0x01219; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]1220; GFX1250-NEXT:    v_mov_b32_e32 v2, s101221; GFX1250-NEXT:    buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN1222; GFX1250-NEXT:    s_wait_xcnt 0x01223; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x441224; GFX1250-NEXT:    v_mov_b32_e32 v2, 01225; GFX1250-NEXT:    s_wait_loadcnt 0x01226; GFX1250-NEXT:    s_wait_kmcnt 0x01227; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[0:1]1228; GFX1250-NEXT:    s_endpgm1229main_body:1230  %ret = call double @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 2)1231  store double %ret, ptr addrspace(1) %out, align 81232  ret void1233}1234 1235define amdgpu_kernel void @struct_buffer_atomic_max_noret_f64(<4 x i32> %rsrc, double %data, i32 %vindex) {1236; GFX90A-LABEL: struct_buffer_atomic_max_noret_f64:1237; GFX90A:       ; %bb.0: ; %main_body1238; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x341239; GFX90A-NEXT:    s_load_dword s8, s[4:5], 0x3c1240; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241241; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)1242; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]1243; GFX90A-NEXT:    v_mov_b32_e32 v2, s81244; GFX90A-NEXT:    buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 idxen1245; GFX90A-NEXT:    s_endpgm1246;1247; GFX942-LABEL: struct_buffer_atomic_max_noret_f64:1248; GFX942:       ; %bb.0: ; %main_body1249; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x341250; GFX942-NEXT:    s_load_dword s8, s[4:5], 0x3c1251; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241252; GFX942-NEXT:    s_waitcnt lgkmcnt(0)1253; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[6:7]1254; GFX942-NEXT:    v_mov_b32_e32 v2, s81255; GFX942-NEXT:    buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 idxen1256; GFX942-NEXT:    s_endpgm1257;1258; GFX1250-LABEL: struct_buffer_atomic_max_noret_f64:1259; GFX1250:       ; %bb.0: ; %main_body1260; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11261; GFX1250-NEXT:    s_clause 0x11262; GFX1250-NEXT:    s_load_b96 s[8:10], s[4:5], 0x341263; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x241264; GFX1250-NEXT:    s_wait_kmcnt 0x01265; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]1266; GFX1250-NEXT:    v_mov_b32_e32 v2, s101267; GFX1250-NEXT:    buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen1268; GFX1250-NEXT:    s_endpgm1269main_body:1270  %ret = call double @llvm.amdgcn.struct.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0)1271  ret void1272}1273 1274define amdgpu_ps void @struct_buffer_atomic_max_rtn_f64(<4 x i32> inreg %rsrc, double %data, i32 %vindex) {1275; GFX90A-LABEL: struct_buffer_atomic_max_rtn_f64:1276; GFX90A:       ; %bb.0: ; %main_body1277; GFX90A-NEXT:    buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 idxen glc1278; GFX90A-NEXT:    s_waitcnt vmcnt(0)1279; GFX90A-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]1280; GFX90A-NEXT:    s_endpgm1281;1282; GFX942-LABEL: struct_buffer_atomic_max_rtn_f64:1283; GFX942:       ; %bb.0: ; %main_body1284; GFX942-NEXT:    buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 idxen sc01285; GFX942-NEXT:    s_waitcnt vmcnt(0)1286; GFX942-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]1287; GFX942-NEXT:    s_endpgm1288;1289; GFX1250-LABEL: struct_buffer_atomic_max_rtn_f64:1290; GFX1250:       ; %bb.0: ; %main_body1291; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11292; GFX1250-NEXT:    buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN1293; GFX1250-NEXT:    s_wait_loadcnt 0x01294; GFX1250-NEXT:    flat_store_b64 v[0:1], v[0:1]1295; GFX1250-NEXT:    s_endpgm1296main_body:1297  %ret = call double @llvm.amdgcn.struct.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 0, i32 0, i32 0)1298  store double %ret, ptr poison1299  ret void1300}1301 1302define amdgpu_kernel void @struct_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {1303; GFX90A-LABEL: struct_buffer_atomic_max_rtn_f64_off4_slc:1304; GFX90A:       ; %bb.0: ; %main_body1305; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x341306; GFX90A-NEXT:    s_load_dword s10, s[4:5], 0x3c1307; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241308; GFX90A-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x441309; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)1310; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]1311; GFX90A-NEXT:    v_mov_b32_e32 v0, s101312; GFX90A-NEXT:    buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc1313; GFX90A-NEXT:    v_mov_b32_e32 v0, 01314; GFX90A-NEXT:    s_waitcnt vmcnt(0)1315; GFX90A-NEXT:    global_store_dwordx2 v0, v[2:3], s[8:9]1316; GFX90A-NEXT:    s_endpgm1317;1318; GFX942-LABEL: struct_buffer_atomic_max_rtn_f64_off4_slc:1319; GFX942:       ; %bb.0: ; %main_body1320; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x341321; GFX942-NEXT:    s_load_dword s10, s[4:5], 0x3c1322; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241323; GFX942-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x441324; GFX942-NEXT:    s_waitcnt lgkmcnt(0)1325; GFX942-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]1326; GFX942-NEXT:    v_mov_b32_e32 v0, s101327; GFX942-NEXT:    buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt1328; GFX942-NEXT:    v_mov_b32_e32 v0, 01329; GFX942-NEXT:    s_waitcnt vmcnt(0)1330; GFX942-NEXT:    global_store_dwordx2 v0, v[2:3], s[8:9]1331; GFX942-NEXT:    s_endpgm1332;1333; GFX1250-LABEL: struct_buffer_atomic_max_rtn_f64_off4_slc:1334; GFX1250:       ; %bb.0: ; %main_body1335; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11336; GFX1250-NEXT:    s_clause 0x11337; GFX1250-NEXT:    s_load_b96 s[8:10], s[4:5], 0x341338; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x241339; GFX1250-NEXT:    s_wait_kmcnt 0x01340; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]1341; GFX1250-NEXT:    v_mov_b32_e32 v2, s101342; GFX1250-NEXT:    buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN1343; GFX1250-NEXT:    s_wait_xcnt 0x01344; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x441345; GFX1250-NEXT:    v_mov_b32_e32 v2, 01346; GFX1250-NEXT:    s_wait_loadcnt 0x01347; GFX1250-NEXT:    s_wait_kmcnt 0x01348; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[0:1]1349; GFX1250-NEXT:    s_endpgm1350main_body:1351  %ret = call double @llvm.amdgcn.struct.buffer.atomic.fmax.f64(double %data, <4 x i32> %rsrc, i32 %vindex, i32 4, i32 0, i32 2)1352  store double %ret, ptr addrspace(1) %out, align 81353  ret void1354}1355 1356define amdgpu_kernel void @struct_ptr_buffer_atomic_max_noret_f64(ptr addrspace(8) %rsrc, double %data, i32 %vindex) {1357; GFX90A-LABEL: struct_ptr_buffer_atomic_max_noret_f64:1358; GFX90A:       ; %bb.0: ; %main_body1359; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x341360; GFX90A-NEXT:    s_load_dword s8, s[4:5], 0x3c1361; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241362; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)1363; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[6:7], s[6:7] op_sel:[0,1]1364; GFX90A-NEXT:    v_mov_b32_e32 v2, s81365; GFX90A-NEXT:    buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 idxen1366; GFX90A-NEXT:    s_endpgm1367;1368; GFX942-LABEL: struct_ptr_buffer_atomic_max_noret_f64:1369; GFX942:       ; %bb.0: ; %main_body1370; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x341371; GFX942-NEXT:    s_load_dword s8, s[4:5], 0x3c1372; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241373; GFX942-NEXT:    s_waitcnt lgkmcnt(0)1374; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[6:7]1375; GFX942-NEXT:    v_mov_b32_e32 v2, s81376; GFX942-NEXT:    buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 idxen1377; GFX942-NEXT:    s_endpgm1378;1379; GFX1250-LABEL: struct_ptr_buffer_atomic_max_noret_f64:1380; GFX1250:       ; %bb.0: ; %main_body1381; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11382; GFX1250-NEXT:    s_clause 0x11383; GFX1250-NEXT:    s_load_b96 s[8:10], s[4:5], 0x341384; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x241385; GFX1250-NEXT:    s_wait_kmcnt 0x01386; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]1387; GFX1250-NEXT:    v_mov_b32_e32 v2, s101388; GFX1250-NEXT:    buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen1389; GFX1250-NEXT:    s_endpgm1390main_body:1391  %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0, i32 0)1392  ret void1393}1394 1395define amdgpu_ps void @struct_ptr_buffer_atomic_max_rtn_f64(ptr addrspace(8) inreg %rsrc, double %data, i32 %vindex) {1396; GFX90A-LABEL: struct_ptr_buffer_atomic_max_rtn_f64:1397; GFX90A:       ; %bb.0: ; %main_body1398; GFX90A-NEXT:    buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 idxen glc1399; GFX90A-NEXT:    s_waitcnt vmcnt(0)1400; GFX90A-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]1401; GFX90A-NEXT:    s_endpgm1402;1403; GFX942-LABEL: struct_ptr_buffer_atomic_max_rtn_f64:1404; GFX942:       ; %bb.0: ; %main_body1405; GFX942-NEXT:    buffer_atomic_max_f64 v[0:1], v2, s[0:3], 0 idxen sc01406; GFX942-NEXT:    s_waitcnt vmcnt(0)1407; GFX942-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]1408; GFX942-NEXT:    s_endpgm1409;1410; GFX1250-LABEL: struct_ptr_buffer_atomic_max_rtn_f64:1411; GFX1250:       ; %bb.0: ; %main_body1412; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11413; GFX1250-NEXT:    buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen th:TH_ATOMIC_RETURN1414; GFX1250-NEXT:    s_wait_loadcnt 0x01415; GFX1250-NEXT:    flat_store_b64 v[0:1], v[0:1]1416; GFX1250-NEXT:    s_endpgm1417main_body:1418  %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 0, i32 0)1419  store double %ret, ptr poison1420  ret void1421}1422 1423define amdgpu_kernel void @struct_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr addrspace(8) %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {1424; GFX90A-LABEL: struct_ptr_buffer_atomic_max_rtn_f64_off4_slc:1425; GFX90A:       ; %bb.0: ; %main_body1426; GFX90A-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x341427; GFX90A-NEXT:    s_load_dword s10, s[4:5], 0x3c1428; GFX90A-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241429; GFX90A-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x441430; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)1431; GFX90A-NEXT:    v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]1432; GFX90A-NEXT:    v_mov_b32_e32 v0, s101433; GFX90A-NEXT:    buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc1434; GFX90A-NEXT:    v_mov_b32_e32 v0, 01435; GFX90A-NEXT:    s_waitcnt vmcnt(0)1436; GFX90A-NEXT:    global_store_dwordx2 v0, v[2:3], s[8:9]1437; GFX90A-NEXT:    s_endpgm1438;1439; GFX942-LABEL: struct_ptr_buffer_atomic_max_rtn_f64_off4_slc:1440; GFX942:       ; %bb.0: ; %main_body1441; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x341442; GFX942-NEXT:    s_load_dword s10, s[4:5], 0x3c1443; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x241444; GFX942-NEXT:    s_load_dwordx2 s[8:9], s[4:5], 0x441445; GFX942-NEXT:    s_waitcnt lgkmcnt(0)1446; GFX942-NEXT:    v_mov_b64_e32 v[2:3], s[6:7]1447; GFX942-NEXT:    v_mov_b32_e32 v0, s101448; GFX942-NEXT:    buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt1449; GFX942-NEXT:    v_mov_b32_e32 v0, 01450; GFX942-NEXT:    s_waitcnt vmcnt(0)1451; GFX942-NEXT:    global_store_dwordx2 v0, v[2:3], s[8:9]1452; GFX942-NEXT:    s_endpgm1453;1454; GFX1250-LABEL: struct_ptr_buffer_atomic_max_rtn_f64_off4_slc:1455; GFX1250:       ; %bb.0: ; %main_body1456; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11457; GFX1250-NEXT:    s_clause 0x11458; GFX1250-NEXT:    s_load_b96 s[8:10], s[4:5], 0x341459; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x241460; GFX1250-NEXT:    s_wait_kmcnt 0x01461; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]1462; GFX1250-NEXT:    v_mov_b32_e32 v2, s101463; GFX1250-NEXT:    buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN1464; GFX1250-NEXT:    s_wait_xcnt 0x01465; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x441466; GFX1250-NEXT:    v_mov_b32_e32 v2, 01467; GFX1250-NEXT:    s_wait_loadcnt 0x01468; GFX1250-NEXT:    s_wait_kmcnt 0x01469; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[0:1]1470; GFX1250-NEXT:    s_endpgm1471main_body:1472  %ret = call double @llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f64(double %data, ptr addrspace(8) %rsrc, i32 %vindex, i32 4, i32 0, i32 2)1473  store double %ret, ptr addrspace(1) %out, align 81474  ret void1475}1476 1477define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %ptr) #1 {1478; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat:1479; GFX90A:       ; %bb.0: ; %main_body1480; GFX90A-NEXT:    s_mov_b64 s[0:1], exec1481; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 01482; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v01483; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01484; GFX90A-NEXT:    s_and_saveexec_b64 s[2:3], vcc1485; GFX90A-NEXT:    s_cbranch_execz .LBB36_21486; GFX90A-NEXT:  ; %bb.1:1487; GFX90A-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x241488; GFX90A-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]1489; GFX90A-NEXT:    v_cvt_f64_u32_e32 v[0:1], s01490; GFX90A-NEXT:    v_mul_f64 v[0:1], v[0:1], 4.01491; GFX90A-NEXT:    v_mov_b32_e32 v2, 01492; GFX90A-NEXT:    buffer_wbl21493; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)1494; GFX90A-NEXT:    global_atomic_add_f64 v2, v[0:1], s[2:3]1495; GFX90A-NEXT:    s_waitcnt vmcnt(0)1496; GFX90A-NEXT:    buffer_invl21497; GFX90A-NEXT:    buffer_wbinvl1_vol1498; GFX90A-NEXT:  .LBB36_2:1499; GFX90A-NEXT:    s_endpgm1500;1501; GFX942-LABEL: global_atomic_fadd_f64_noret_pat:1502; GFX942:       ; %bb.0: ; %main_body1503; GFX942-NEXT:    s_mov_b64 s[0:1], exec1504; GFX942-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 01505; GFX942-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v01506; GFX942-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01507; GFX942-NEXT:    s_and_saveexec_b64 s[2:3], vcc1508; GFX942-NEXT:    s_cbranch_execz .LBB36_21509; GFX942-NEXT:  ; %bb.1:1510; GFX942-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x241511; GFX942-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]1512; GFX942-NEXT:    v_cvt_f64_u32_e32 v[0:1], s01513; GFX942-NEXT:    v_mul_f64 v[0:1], v[0:1], 4.01514; GFX942-NEXT:    v_mov_b32_e32 v2, 01515; GFX942-NEXT:    buffer_wbl2 sc0 sc11516; GFX942-NEXT:    s_waitcnt lgkmcnt(0)1517; GFX942-NEXT:    global_atomic_add_f64 v2, v[0:1], s[2:3] sc11518; GFX942-NEXT:    s_waitcnt vmcnt(0)1519; GFX942-NEXT:    buffer_inv sc0 sc11520; GFX942-NEXT:  .LBB36_2:1521; GFX942-NEXT:    s_endpgm1522;1523; GFX1250-LABEL: global_atomic_fadd_f64_noret_pat:1524; GFX1250:       ; %bb.0: ; %main_body1525; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11526; GFX1250-NEXT:    s_mov_b32 s0, exec_lo1527; GFX1250-NEXT:    s_mov_b32 s1, exec_lo1528; GFX1250-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 01529; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)1530; GFX1250-NEXT:    v_cmpx_eq_u32_e32 0, v01531; GFX1250-NEXT:    s_cbranch_execz .LBB36_21532; GFX1250-NEXT:  ; %bb.1:1533; GFX1250-NEXT:    s_bcnt1_i32_b32 s0, s01534; GFX1250-NEXT:    v_mov_b32_e32 v2, 01535; GFX1250-NEXT:    v_cvt_f64_u32_e32 v[0:1], s01536; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241537; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)1538; GFX1250-NEXT:    v_mul_f64_e32 v[0:1], 4.0, v[0:1]1539; GFX1250-NEXT:    global_wb scope:SCOPE_SYS1540; GFX1250-NEXT:    s_wait_storecnt 0x01541; GFX1250-NEXT:    s_wait_kmcnt 0x01542; GFX1250-NEXT:    global_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_SYS1543; GFX1250-NEXT:    s_wait_storecnt 0x01544; GFX1250-NEXT:    global_inv scope:SCOPE_SYS1545; GFX1250-NEXT:  .LBB36_2:1546; GFX1250-NEXT:    s_endpgm1547main_body:1548  %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 seq_cst, !amdgpu.no.fine.grained.memory !01549  ret void1550}1551 1552define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(1) %ptr) #1 {1553; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_agent:1554; GFX90A:       ; %bb.0: ; %main_body1555; GFX90A-NEXT:    s_mov_b64 s[0:1], exec1556; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 01557; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v01558; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01559; GFX90A-NEXT:    s_and_saveexec_b64 s[2:3], vcc1560; GFX90A-NEXT:    s_cbranch_execz .LBB37_21561; GFX90A-NEXT:  ; %bb.1:1562; GFX90A-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x241563; GFX90A-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]1564; GFX90A-NEXT:    v_cvt_f64_u32_e32 v[0:1], s01565; GFX90A-NEXT:    v_mul_f64 v[0:1], v[0:1], 4.01566; GFX90A-NEXT:    v_mov_b32_e32 v2, 01567; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)1568; GFX90A-NEXT:    global_atomic_add_f64 v2, v[0:1], s[2:3]1569; GFX90A-NEXT:    s_waitcnt vmcnt(0)1570; GFX90A-NEXT:    buffer_wbinvl1_vol1571; GFX90A-NEXT:  .LBB37_2:1572; GFX90A-NEXT:    s_endpgm1573;1574; GFX942-LABEL: global_atomic_fadd_f64_noret_pat_agent:1575; GFX942:       ; %bb.0: ; %main_body1576; GFX942-NEXT:    s_mov_b64 s[0:1], exec1577; GFX942-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 01578; GFX942-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v01579; GFX942-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01580; GFX942-NEXT:    s_and_saveexec_b64 s[2:3], vcc1581; GFX942-NEXT:    s_cbranch_execz .LBB37_21582; GFX942-NEXT:  ; %bb.1:1583; GFX942-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x241584; GFX942-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]1585; GFX942-NEXT:    v_cvt_f64_u32_e32 v[0:1], s01586; GFX942-NEXT:    v_mul_f64 v[0:1], v[0:1], 4.01587; GFX942-NEXT:    v_mov_b32_e32 v2, 01588; GFX942-NEXT:    buffer_wbl2 sc11589; GFX942-NEXT:    s_waitcnt lgkmcnt(0)1590; GFX942-NEXT:    global_atomic_add_f64 v2, v[0:1], s[2:3]1591; GFX942-NEXT:    s_waitcnt vmcnt(0)1592; GFX942-NEXT:    buffer_inv sc11593; GFX942-NEXT:  .LBB37_2:1594; GFX942-NEXT:    s_endpgm1595;1596; GFX1250-LABEL: global_atomic_fadd_f64_noret_pat_agent:1597; GFX1250:       ; %bb.0: ; %main_body1598; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11599; GFX1250-NEXT:    s_mov_b32 s0, exec_lo1600; GFX1250-NEXT:    s_mov_b32 s1, exec_lo1601; GFX1250-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 01602; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)1603; GFX1250-NEXT:    v_cmpx_eq_u32_e32 0, v01604; GFX1250-NEXT:    s_cbranch_execz .LBB37_21605; GFX1250-NEXT:  ; %bb.1:1606; GFX1250-NEXT:    s_bcnt1_i32_b32 s0, s01607; GFX1250-NEXT:    v_mov_b32_e32 v2, 01608; GFX1250-NEXT:    v_cvt_f64_u32_e32 v[0:1], s01609; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241610; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)1611; GFX1250-NEXT:    v_mul_f64_e32 v[0:1], 4.0, v[0:1]1612; GFX1250-NEXT:    global_wb scope:SCOPE_DEV1613; GFX1250-NEXT:    s_wait_storecnt 0x01614; GFX1250-NEXT:    s_wait_kmcnt 0x01615; GFX1250-NEXT:    global_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_DEV1616; GFX1250-NEXT:    s_wait_storecnt 0x01617; GFX1250-NEXT:    global_inv scope:SCOPE_DEV1618; GFX1250-NEXT:  .LBB37_2:1619; GFX1250-NEXT:    s_endpgm1620main_body:1621  %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01622  ret void1623}1624 1625define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace(1) %ptr) #1 {1626; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_system:1627; GFX90A:       ; %bb.0: ; %main_body1628; GFX90A-NEXT:    s_mov_b64 s[0:1], exec1629; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 01630; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v01631; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01632; GFX90A-NEXT:    s_and_saveexec_b64 s[2:3], vcc1633; GFX90A-NEXT:    s_cbranch_execz .LBB38_21634; GFX90A-NEXT:  ; %bb.1:1635; GFX90A-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x241636; GFX90A-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]1637; GFX90A-NEXT:    v_cvt_f64_u32_e32 v[0:1], s01638; GFX90A-NEXT:    v_mul_f64 v[0:1], v[0:1], 4.01639; GFX90A-NEXT:    v_mov_b32_e32 v2, 01640; GFX90A-NEXT:    buffer_wbl21641; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)1642; GFX90A-NEXT:    global_atomic_add_f64 v2, v[0:1], s[2:3]1643; GFX90A-NEXT:    s_waitcnt vmcnt(0)1644; GFX90A-NEXT:    buffer_invl21645; GFX90A-NEXT:    buffer_wbinvl1_vol1646; GFX90A-NEXT:  .LBB38_2:1647; GFX90A-NEXT:    s_endpgm1648;1649; GFX942-LABEL: global_atomic_fadd_f64_noret_pat_system:1650; GFX942:       ; %bb.0: ; %main_body1651; GFX942-NEXT:    s_mov_b64 s[0:1], exec1652; GFX942-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 01653; GFX942-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v01654; GFX942-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01655; GFX942-NEXT:    s_and_saveexec_b64 s[2:3], vcc1656; GFX942-NEXT:    s_cbranch_execz .LBB38_21657; GFX942-NEXT:  ; %bb.1:1658; GFX942-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x241659; GFX942-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]1660; GFX942-NEXT:    v_cvt_f64_u32_e32 v[0:1], s01661; GFX942-NEXT:    v_mul_f64 v[0:1], v[0:1], 4.01662; GFX942-NEXT:    v_mov_b32_e32 v2, 01663; GFX942-NEXT:    buffer_wbl2 sc0 sc11664; GFX942-NEXT:    s_waitcnt lgkmcnt(0)1665; GFX942-NEXT:    global_atomic_add_f64 v2, v[0:1], s[2:3] sc11666; GFX942-NEXT:    s_waitcnt vmcnt(0)1667; GFX942-NEXT:    buffer_inv sc0 sc11668; GFX942-NEXT:  .LBB38_2:1669; GFX942-NEXT:    s_endpgm1670;1671; GFX1250-LABEL: global_atomic_fadd_f64_noret_pat_system:1672; GFX1250:       ; %bb.0: ; %main_body1673; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11674; GFX1250-NEXT:    s_mov_b32 s0, exec_lo1675; GFX1250-NEXT:    s_mov_b32 s1, exec_lo1676; GFX1250-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 01677; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)1678; GFX1250-NEXT:    v_cmpx_eq_u32_e32 0, v01679; GFX1250-NEXT:    s_cbranch_execz .LBB38_21680; GFX1250-NEXT:  ; %bb.1:1681; GFX1250-NEXT:    s_bcnt1_i32_b32 s0, s01682; GFX1250-NEXT:    v_mov_b32_e32 v2, 01683; GFX1250-NEXT:    v_cvt_f64_u32_e32 v[0:1], s01684; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241685; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)1686; GFX1250-NEXT:    v_mul_f64_e32 v[0:1], 4.0, v[0:1]1687; GFX1250-NEXT:    global_wb scope:SCOPE_SYS1688; GFX1250-NEXT:    s_wait_storecnt 0x01689; GFX1250-NEXT:    s_wait_kmcnt 0x01690; GFX1250-NEXT:    global_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_SYS1691; GFX1250-NEXT:    s_wait_storecnt 0x01692; GFX1250-NEXT:    global_inv scope:SCOPE_SYS1693; GFX1250-NEXT:  .LBB38_2:1694; GFX1250-NEXT:    s_endpgm1695main_body:1696  %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") seq_cst, !amdgpu.no.fine.grained.memory !01697  ret void1698}1699 1700define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_flush(ptr addrspace(1) %ptr) #0 {1701; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_flush:1702; GFX90A:       ; %bb.0: ; %main_body1703; GFX90A-NEXT:    s_mov_b64 s[0:1], exec1704; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 01705; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v01706; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01707; GFX90A-NEXT:    s_and_saveexec_b64 s[2:3], vcc1708; GFX90A-NEXT:    s_cbranch_execz .LBB39_21709; GFX90A-NEXT:  ; %bb.1:1710; GFX90A-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x241711; GFX90A-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]1712; GFX90A-NEXT:    v_cvt_f64_u32_e32 v[0:1], s01713; GFX90A-NEXT:    v_mul_f64 v[0:1], v[0:1], 4.01714; GFX90A-NEXT:    v_mov_b32_e32 v2, 01715; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)1716; GFX90A-NEXT:    global_atomic_add_f64 v2, v[0:1], s[2:3]1717; GFX90A-NEXT:    s_waitcnt vmcnt(0)1718; GFX90A-NEXT:    buffer_wbinvl1_vol1719; GFX90A-NEXT:  .LBB39_2:1720; GFX90A-NEXT:    s_endpgm1721;1722; GFX942-LABEL: global_atomic_fadd_f64_noret_pat_flush:1723; GFX942:       ; %bb.0: ; %main_body1724; GFX942-NEXT:    s_mov_b64 s[0:1], exec1725; GFX942-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 01726; GFX942-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v01727; GFX942-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01728; GFX942-NEXT:    s_and_saveexec_b64 s[2:3], vcc1729; GFX942-NEXT:    s_cbranch_execz .LBB39_21730; GFX942-NEXT:  ; %bb.1:1731; GFX942-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x241732; GFX942-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]1733; GFX942-NEXT:    v_cvt_f64_u32_e32 v[0:1], s01734; GFX942-NEXT:    v_mul_f64 v[0:1], v[0:1], 4.01735; GFX942-NEXT:    v_mov_b32_e32 v2, 01736; GFX942-NEXT:    buffer_wbl2 sc11737; GFX942-NEXT:    s_waitcnt lgkmcnt(0)1738; GFX942-NEXT:    global_atomic_add_f64 v2, v[0:1], s[2:3]1739; GFX942-NEXT:    s_waitcnt vmcnt(0)1740; GFX942-NEXT:    buffer_inv sc11741; GFX942-NEXT:  .LBB39_2:1742; GFX942-NEXT:    s_endpgm1743;1744; GFX1250-LABEL: global_atomic_fadd_f64_noret_pat_flush:1745; GFX1250:       ; %bb.0: ; %main_body1746; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11747; GFX1250-NEXT:    s_mov_b32 s0, exec_lo1748; GFX1250-NEXT:    s_mov_b32 s1, exec_lo1749; GFX1250-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 01750; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)1751; GFX1250-NEXT:    v_cmpx_eq_u32_e32 0, v01752; GFX1250-NEXT:    s_cbranch_execz .LBB39_21753; GFX1250-NEXT:  ; %bb.1:1754; GFX1250-NEXT:    s_bcnt1_i32_b32 s0, s01755; GFX1250-NEXT:    v_mov_b32_e32 v2, 01756; GFX1250-NEXT:    v_cvt_f64_u32_e32 v[0:1], s01757; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241758; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)1759; GFX1250-NEXT:    v_mul_f64_e32 v[0:1], 4.0, v[0:1]1760; GFX1250-NEXT:    global_wb scope:SCOPE_DEV1761; GFX1250-NEXT:    s_wait_storecnt 0x01762; GFX1250-NEXT:    s_wait_kmcnt 0x01763; GFX1250-NEXT:    global_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_DEV1764; GFX1250-NEXT:    s_wait_storecnt 0x01765; GFX1250-NEXT:    global_inv scope:SCOPE_DEV1766; GFX1250-NEXT:  .LBB39_2:1767; GFX1250-NEXT:    s_endpgm1768main_body:1769  %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01770  ret void1771}1772 1773define double @global_atomic_fadd_f64_rtn_pat(ptr addrspace(1) %ptr, double %data) #1 {1774; GFX90A-LABEL: global_atomic_fadd_f64_rtn_pat:1775; GFX90A:       ; %bb.0: ; %main_body1776; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1777; GFX90A-NEXT:    v_mov_b32_e32 v2, 01778; GFX90A-NEXT:    v_mov_b32_e32 v3, 0x401000001779; GFX90A-NEXT:    buffer_wbl21780; GFX90A-NEXT:    global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off glc1781; GFX90A-NEXT:    s_waitcnt vmcnt(0)1782; GFX90A-NEXT:    buffer_invl21783; GFX90A-NEXT:    buffer_wbinvl1_vol1784; GFX90A-NEXT:    s_setpc_b64 s[30:31]1785;1786; GFX942-LABEL: global_atomic_fadd_f64_rtn_pat:1787; GFX942:       ; %bb.0: ; %main_body1788; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1789; GFX942-NEXT:    v_mov_b64_e32 v[2:3], 4.01790; GFX942-NEXT:    buffer_wbl2 sc0 sc11791; GFX942-NEXT:    global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off sc0 sc11792; GFX942-NEXT:    s_waitcnt vmcnt(0)1793; GFX942-NEXT:    buffer_inv sc0 sc11794; GFX942-NEXT:    s_setpc_b64 s[30:31]1795;1796; GFX1250-LABEL: global_atomic_fadd_f64_rtn_pat:1797; GFX1250:       ; %bb.0: ; %main_body1798; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x01799; GFX1250-NEXT:    s_wait_kmcnt 0x01800; GFX1250-NEXT:    v_mov_b64_e32 v[2:3], 4.01801; GFX1250-NEXT:    global_wb scope:SCOPE_SYS1802; GFX1250-NEXT:    s_wait_storecnt 0x01803; GFX1250-NEXT:    global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS1804; GFX1250-NEXT:    s_wait_loadcnt 0x01805; GFX1250-NEXT:    global_inv scope:SCOPE_SYS1806; GFX1250-NEXT:    s_wait_loadcnt 0x01807; GFX1250-NEXT:    s_set_pc_i64 s[30:31]1808main_body:1809  %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 seq_cst, !amdgpu.no.fine.grained.memory !01810  ret double %ret1811}1812 1813define double @global_atomic_fadd_f64_rtn_pat_agent(ptr addrspace(1) %ptr, double %data) #1 {1814; GFX90A-LABEL: global_atomic_fadd_f64_rtn_pat_agent:1815; GFX90A:       ; %bb.0: ; %main_body1816; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1817; GFX90A-NEXT:    v_mov_b32_e32 v2, 01818; GFX90A-NEXT:    v_mov_b32_e32 v3, 0x401000001819; GFX90A-NEXT:    global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off glc1820; GFX90A-NEXT:    s_waitcnt vmcnt(0)1821; GFX90A-NEXT:    buffer_wbinvl1_vol1822; GFX90A-NEXT:    s_setpc_b64 s[30:31]1823;1824; GFX942-LABEL: global_atomic_fadd_f64_rtn_pat_agent:1825; GFX942:       ; %bb.0: ; %main_body1826; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1827; GFX942-NEXT:    v_mov_b64_e32 v[2:3], 4.01828; GFX942-NEXT:    buffer_wbl2 sc11829; GFX942-NEXT:    global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off sc01830; GFX942-NEXT:    s_waitcnt vmcnt(0)1831; GFX942-NEXT:    buffer_inv sc11832; GFX942-NEXT:    s_setpc_b64 s[30:31]1833;1834; GFX1250-LABEL: global_atomic_fadd_f64_rtn_pat_agent:1835; GFX1250:       ; %bb.0: ; %main_body1836; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x01837; GFX1250-NEXT:    s_wait_kmcnt 0x01838; GFX1250-NEXT:    v_mov_b64_e32 v[2:3], 4.01839; GFX1250-NEXT:    global_wb scope:SCOPE_DEV1840; GFX1250-NEXT:    s_wait_storecnt 0x01841; GFX1250-NEXT:    global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV1842; GFX1250-NEXT:    s_wait_loadcnt 0x01843; GFX1250-NEXT:    global_inv scope:SCOPE_DEV1844; GFX1250-NEXT:    s_wait_loadcnt 0x01845; GFX1250-NEXT:    s_set_pc_i64 s[30:31]1846main_body:1847  %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01848  ret double %ret1849}1850 1851define double @global_atomic_fadd_f64_rtn_pat_system(ptr addrspace(1) %ptr, double %data) #1 {1852; GFX90A-LABEL: global_atomic_fadd_f64_rtn_pat_system:1853; GFX90A:       ; %bb.0: ; %main_body1854; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1855; GFX90A-NEXT:    v_mov_b32_e32 v2, 01856; GFX90A-NEXT:    v_mov_b32_e32 v3, 0x401000001857; GFX90A-NEXT:    buffer_wbl21858; GFX90A-NEXT:    global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off glc1859; GFX90A-NEXT:    s_waitcnt vmcnt(0)1860; GFX90A-NEXT:    buffer_invl21861; GFX90A-NEXT:    buffer_wbinvl1_vol1862; GFX90A-NEXT:    s_setpc_b64 s[30:31]1863;1864; GFX942-LABEL: global_atomic_fadd_f64_rtn_pat_system:1865; GFX942:       ; %bb.0: ; %main_body1866; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1867; GFX942-NEXT:    v_mov_b64_e32 v[2:3], 4.01868; GFX942-NEXT:    buffer_wbl2 sc0 sc11869; GFX942-NEXT:    global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off sc0 sc11870; GFX942-NEXT:    s_waitcnt vmcnt(0)1871; GFX942-NEXT:    buffer_inv sc0 sc11872; GFX942-NEXT:    s_setpc_b64 s[30:31]1873;1874; GFX1250-LABEL: global_atomic_fadd_f64_rtn_pat_system:1875; GFX1250:       ; %bb.0: ; %main_body1876; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x01877; GFX1250-NEXT:    s_wait_kmcnt 0x01878; GFX1250-NEXT:    v_mov_b64_e32 v[2:3], 4.01879; GFX1250-NEXT:    global_wb scope:SCOPE_SYS1880; GFX1250-NEXT:    s_wait_storecnt 0x01881; GFX1250-NEXT:    global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS1882; GFX1250-NEXT:    s_wait_loadcnt 0x01883; GFX1250-NEXT:    global_inv scope:SCOPE_SYS1884; GFX1250-NEXT:    s_wait_loadcnt 0x01885; GFX1250-NEXT:    s_set_pc_i64 s[30:31]1886main_body:1887  %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("one-as") seq_cst, !amdgpu.no.fine.grained.memory !01888  ret double %ret1889}1890 1891define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrspace(1) %ptr) {1892; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe:1893; GFX90A:       ; %bb.0: ; %main_body1894; GFX90A-NEXT:    s_mov_b64 s[0:1], exec1895; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 01896; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v01897; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01898; GFX90A-NEXT:    s_and_saveexec_b64 s[2:3], vcc1899; GFX90A-NEXT:    s_cbranch_execz .LBB43_21900; GFX90A-NEXT:  ; %bb.1:1901; GFX90A-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x241902; GFX90A-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]1903; GFX90A-NEXT:    v_cvt_f64_u32_e32 v[0:1], s01904; GFX90A-NEXT:    v_mul_f64 v[0:1], v[0:1], 4.01905; GFX90A-NEXT:    v_mov_b32_e32 v2, 01906; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)1907; GFX90A-NEXT:    global_atomic_add_f64 v2, v[0:1], s[2:3]1908; GFX90A-NEXT:    s_waitcnt vmcnt(0)1909; GFX90A-NEXT:    buffer_wbinvl1_vol1910; GFX90A-NEXT:  .LBB43_2:1911; GFX90A-NEXT:    s_endpgm1912;1913; GFX942-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe:1914; GFX942:       ; %bb.0: ; %main_body1915; GFX942-NEXT:    s_mov_b64 s[0:1], exec1916; GFX942-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 01917; GFX942-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v01918; GFX942-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v01919; GFX942-NEXT:    s_and_saveexec_b64 s[2:3], vcc1920; GFX942-NEXT:    s_cbranch_execz .LBB43_21921; GFX942-NEXT:  ; %bb.1:1922; GFX942-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x241923; GFX942-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]1924; GFX942-NEXT:    v_cvt_f64_u32_e32 v[0:1], s01925; GFX942-NEXT:    v_mul_f64 v[0:1], v[0:1], 4.01926; GFX942-NEXT:    v_mov_b32_e32 v2, 01927; GFX942-NEXT:    buffer_wbl2 sc11928; GFX942-NEXT:    s_waitcnt lgkmcnt(0)1929; GFX942-NEXT:    global_atomic_add_f64 v2, v[0:1], s[2:3]1930; GFX942-NEXT:    s_waitcnt vmcnt(0)1931; GFX942-NEXT:    buffer_inv sc11932; GFX942-NEXT:  .LBB43_2:1933; GFX942-NEXT:    s_endpgm1934;1935; GFX1250-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe:1936; GFX1250:       ; %bb.0: ; %main_body1937; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11938; GFX1250-NEXT:    s_mov_b32 s0, exec_lo1939; GFX1250-NEXT:    s_mov_b32 s1, exec_lo1940; GFX1250-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 01941; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)1942; GFX1250-NEXT:    v_cmpx_eq_u32_e32 0, v01943; GFX1250-NEXT:    s_cbranch_execz .LBB43_21944; GFX1250-NEXT:  ; %bb.1:1945; GFX1250-NEXT:    s_bcnt1_i32_b32 s0, s01946; GFX1250-NEXT:    v_mov_b32_e32 v2, 01947; GFX1250-NEXT:    v_cvt_f64_u32_e32 v[0:1], s01948; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241949; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)1950; GFX1250-NEXT:    v_mul_f64_e32 v[0:1], 4.0, v[0:1]1951; GFX1250-NEXT:    global_wb scope:SCOPE_DEV1952; GFX1250-NEXT:    s_wait_storecnt 0x01953; GFX1250-NEXT:    s_wait_kmcnt 0x01954; GFX1250-NEXT:    global_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_DEV1955; GFX1250-NEXT:    s_wait_storecnt 0x01956; GFX1250-NEXT:    global_inv scope:SCOPE_DEV1957; GFX1250-NEXT:  .LBB43_2:1958; GFX1250-NEXT:    s_endpgm1959main_body:1960  %ret = atomicrmw fadd ptr addrspace(1) %ptr, double 4.0 syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01961  ret void1962}1963 1964define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat(ptr %ptr) #1 {1965; GFX90A-LABEL: flat_atomic_fadd_f64_noret_pat:1966; GFX90A:       ; %bb.0: ; %main_body1967; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241968; GFX90A-NEXT:    v_mov_b32_e32 v2, 01969; GFX90A-NEXT:    v_mov_b32_e32 v3, 0x401000001970; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)1971; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]1972; GFX90A-NEXT:    buffer_wbl21973; GFX90A-NEXT:    flat_atomic_add_f64 v[0:1], v[2:3]1974; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1975; GFX90A-NEXT:    buffer_invl21976; GFX90A-NEXT:    buffer_wbinvl1_vol1977; GFX90A-NEXT:    s_endpgm1978;1979; GFX942-LABEL: flat_atomic_fadd_f64_noret_pat:1980; GFX942:       ; %bb.0: ; %main_body1981; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x241982; GFX942-NEXT:    v_mov_b64_e32 v[2:3], 4.01983; GFX942-NEXT:    s_waitcnt lgkmcnt(0)1984; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]1985; GFX942-NEXT:    buffer_wbl2 sc0 sc11986; GFX942-NEXT:    flat_atomic_add_f64 v[0:1], v[2:3] sc11987; GFX942-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)1988; GFX942-NEXT:    buffer_inv sc0 sc11989; GFX942-NEXT:    s_endpgm1990;1991; GFX1250-LABEL: flat_atomic_fadd_f64_noret_pat:1992; GFX1250:       ; %bb.0: ; %main_body1993; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11994; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x241995; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], 4.01996; GFX1250-NEXT:    v_mov_b32_e32 v2, 01997; GFX1250-NEXT:    global_wb scope:SCOPE_SYS1998; GFX1250-NEXT:    s_wait_storecnt 0x01999; GFX1250-NEXT:    s_wait_kmcnt 0x02000; GFX1250-NEXT:    flat_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_SYS2001; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x02002; GFX1250-NEXT:    global_inv scope:SCOPE_SYS2003; GFX1250-NEXT:    s_endpgm2004main_body:2005  %ret = atomicrmw fadd ptr %ptr, double 4.0 seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !02006  ret void2007}2008 2009define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_agent(ptr %ptr) #1 {2010; GFX90A-LABEL: flat_atomic_fadd_f64_noret_pat_agent:2011; GFX90A:       ; %bb.0: ; %main_body2012; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x242013; GFX90A-NEXT:    v_mov_b32_e32 v2, 02014; GFX90A-NEXT:    v_mov_b32_e32 v3, 0x401000002015; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)2016; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]2017; GFX90A-NEXT:    flat_atomic_add_f64 v[0:1], v[2:3]2018; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2019; GFX90A-NEXT:    buffer_wbinvl1_vol2020; GFX90A-NEXT:    s_endpgm2021;2022; GFX942-LABEL: flat_atomic_fadd_f64_noret_pat_agent:2023; GFX942:       ; %bb.0: ; %main_body2024; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x242025; GFX942-NEXT:    v_mov_b64_e32 v[2:3], 4.02026; GFX942-NEXT:    s_waitcnt lgkmcnt(0)2027; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]2028; GFX942-NEXT:    buffer_wbl2 sc12029; GFX942-NEXT:    flat_atomic_add_f64 v[0:1], v[2:3]2030; GFX942-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2031; GFX942-NEXT:    buffer_inv sc12032; GFX942-NEXT:    s_endpgm2033;2034; GFX1250-LABEL: flat_atomic_fadd_f64_noret_pat_agent:2035; GFX1250:       ; %bb.0: ; %main_body2036; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12037; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x242038; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], 4.02039; GFX1250-NEXT:    v_mov_b32_e32 v2, 02040; GFX1250-NEXT:    global_wb scope:SCOPE_DEV2041; GFX1250-NEXT:    s_wait_storecnt 0x02042; GFX1250-NEXT:    s_wait_kmcnt 0x02043; GFX1250-NEXT:    flat_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_DEV2044; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x02045; GFX1250-NEXT:    global_inv scope:SCOPE_DEV2046; GFX1250-NEXT:    s_endpgm2047main_body:2048  %ret = atomicrmw fadd ptr %ptr, double 4.0 syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !02049  ret void2050}2051 2052define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_system(ptr %ptr) #1 {2053; GFX90A-LABEL: flat_atomic_fadd_f64_noret_pat_system:2054; GFX90A:       ; %bb.0: ; %main_body2055; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x242056; GFX90A-NEXT:    v_mov_b32_e32 v2, 02057; GFX90A-NEXT:    v_mov_b32_e32 v3, 0x401000002058; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)2059; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]2060; GFX90A-NEXT:    buffer_wbl22061; GFX90A-NEXT:    flat_atomic_add_f64 v[0:1], v[2:3]2062; GFX90A-NEXT:    s_waitcnt vmcnt(0)2063; GFX90A-NEXT:    buffer_invl22064; GFX90A-NEXT:    buffer_wbinvl1_vol2065; GFX90A-NEXT:    s_endpgm2066;2067; GFX942-LABEL: flat_atomic_fadd_f64_noret_pat_system:2068; GFX942:       ; %bb.0: ; %main_body2069; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x242070; GFX942-NEXT:    v_mov_b64_e32 v[2:3], 4.02071; GFX942-NEXT:    s_waitcnt lgkmcnt(0)2072; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]2073; GFX942-NEXT:    buffer_wbl2 sc0 sc12074; GFX942-NEXT:    flat_atomic_add_f64 v[0:1], v[2:3] sc12075; GFX942-NEXT:    s_waitcnt vmcnt(0)2076; GFX942-NEXT:    buffer_inv sc0 sc12077; GFX942-NEXT:    s_endpgm2078;2079; GFX1250-LABEL: flat_atomic_fadd_f64_noret_pat_system:2080; GFX1250:       ; %bb.0: ; %main_body2081; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12082; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x242083; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], 4.02084; GFX1250-NEXT:    v_mov_b32_e32 v2, 02085; GFX1250-NEXT:    global_wb scope:SCOPE_SYS2086; GFX1250-NEXT:    s_wait_storecnt 0x02087; GFX1250-NEXT:    s_wait_kmcnt 0x02088; GFX1250-NEXT:    flat_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_SYS2089; GFX1250-NEXT:    s_wait_storecnt 0x02090; GFX1250-NEXT:    global_inv scope:SCOPE_SYS2091; GFX1250-NEXT:    s_endpgm2092main_body:2093  %ret = atomicrmw fadd ptr %ptr, double 4.0 syncscope("one-as") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !02094  ret void2095}2096 2097define double @flat_atomic_fadd_f64_rtn_pat(ptr %ptr) #1 {2098; GFX90A-LABEL: flat_atomic_fadd_f64_rtn_pat:2099; GFX90A:       ; %bb.0: ; %main_body2100; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2101; GFX90A-NEXT:    v_mov_b32_e32 v2, 02102; GFX90A-NEXT:    v_mov_b32_e32 v3, 0x401000002103; GFX90A-NEXT:    buffer_wbl22104; GFX90A-NEXT:    flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] glc2105; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2106; GFX90A-NEXT:    buffer_invl22107; GFX90A-NEXT:    buffer_wbinvl1_vol2108; GFX90A-NEXT:    s_setpc_b64 s[30:31]2109;2110; GFX942-LABEL: flat_atomic_fadd_f64_rtn_pat:2111; GFX942:       ; %bb.0: ; %main_body2112; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2113; GFX942-NEXT:    v_mov_b64_e32 v[2:3], 4.02114; GFX942-NEXT:    buffer_wbl2 sc0 sc12115; GFX942-NEXT:    flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] sc0 sc12116; GFX942-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2117; GFX942-NEXT:    buffer_inv sc0 sc12118; GFX942-NEXT:    s_setpc_b64 s[30:31]2119;2120; GFX1250-LABEL: flat_atomic_fadd_f64_rtn_pat:2121; GFX1250:       ; %bb.0: ; %main_body2122; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x02123; GFX1250-NEXT:    s_wait_kmcnt 0x02124; GFX1250-NEXT:    v_mov_b64_e32 v[2:3], 4.02125; GFX1250-NEXT:    global_wb scope:SCOPE_SYS2126; GFX1250-NEXT:    s_wait_storecnt 0x02127; GFX1250-NEXT:    flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS2128; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x02129; GFX1250-NEXT:    global_inv scope:SCOPE_SYS2130; GFX1250-NEXT:    s_wait_loadcnt 0x02131; GFX1250-NEXT:    s_set_pc_i64 s[30:31]2132main_body:2133  %ret = atomicrmw fadd ptr %ptr, double 4.0 seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !02134  ret double %ret2135}2136 2137define double @flat_atomic_fadd_f64_rtn_pat_agent(ptr %ptr) #1 {2138; GFX90A-LABEL: flat_atomic_fadd_f64_rtn_pat_agent:2139; GFX90A:       ; %bb.0: ; %main_body2140; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2141; GFX90A-NEXT:    v_mov_b32_e32 v2, 02142; GFX90A-NEXT:    v_mov_b32_e32 v3, 0x401000002143; GFX90A-NEXT:    flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] glc2144; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2145; GFX90A-NEXT:    buffer_wbinvl1_vol2146; GFX90A-NEXT:    s_setpc_b64 s[30:31]2147;2148; GFX942-LABEL: flat_atomic_fadd_f64_rtn_pat_agent:2149; GFX942:       ; %bb.0: ; %main_body2150; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2151; GFX942-NEXT:    v_mov_b64_e32 v[2:3], 4.02152; GFX942-NEXT:    buffer_wbl2 sc12153; GFX942-NEXT:    flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] sc02154; GFX942-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2155; GFX942-NEXT:    buffer_inv sc12156; GFX942-NEXT:    s_setpc_b64 s[30:31]2157;2158; GFX1250-LABEL: flat_atomic_fadd_f64_rtn_pat_agent:2159; GFX1250:       ; %bb.0: ; %main_body2160; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x02161; GFX1250-NEXT:    s_wait_kmcnt 0x02162; GFX1250-NEXT:    v_mov_b64_e32 v[2:3], 4.02163; GFX1250-NEXT:    global_wb scope:SCOPE_DEV2164; GFX1250-NEXT:    s_wait_storecnt 0x02165; GFX1250-NEXT:    flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV2166; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x02167; GFX1250-NEXT:    global_inv scope:SCOPE_DEV2168; GFX1250-NEXT:    s_wait_loadcnt 0x02169; GFX1250-NEXT:    s_set_pc_i64 s[30:31]2170main_body:2171  %ret = atomicrmw fadd ptr %ptr, double 4.0 syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !02172  ret double %ret2173}2174 2175define double @flat_atomic_fadd_f64_rtn_pat_system(ptr %ptr) #1 {2176; GFX90A-LABEL: flat_atomic_fadd_f64_rtn_pat_system:2177; GFX90A:       ; %bb.0: ; %main_body2178; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2179; GFX90A-NEXT:    v_mov_b32_e32 v2, 02180; GFX90A-NEXT:    v_mov_b32_e32 v3, 0x401000002181; GFX90A-NEXT:    buffer_wbl22182; GFX90A-NEXT:    flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] glc2183; GFX90A-NEXT:    s_waitcnt vmcnt(0)2184; GFX90A-NEXT:    buffer_invl22185; GFX90A-NEXT:    buffer_wbinvl1_vol2186; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)2187; GFX90A-NEXT:    s_setpc_b64 s[30:31]2188;2189; GFX942-LABEL: flat_atomic_fadd_f64_rtn_pat_system:2190; GFX942:       ; %bb.0: ; %main_body2191; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2192; GFX942-NEXT:    v_mov_b64_e32 v[2:3], 4.02193; GFX942-NEXT:    buffer_wbl2 sc0 sc12194; GFX942-NEXT:    flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] sc0 sc12195; GFX942-NEXT:    s_waitcnt vmcnt(0)2196; GFX942-NEXT:    buffer_inv sc0 sc12197; GFX942-NEXT:    s_waitcnt lgkmcnt(0)2198; GFX942-NEXT:    s_setpc_b64 s[30:31]2199;2200; GFX1250-LABEL: flat_atomic_fadd_f64_rtn_pat_system:2201; GFX1250:       ; %bb.0: ; %main_body2202; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x02203; GFX1250-NEXT:    s_wait_kmcnt 0x02204; GFX1250-NEXT:    v_mov_b64_e32 v[2:3], 4.02205; GFX1250-NEXT:    global_wb scope:SCOPE_SYS2206; GFX1250-NEXT:    s_wait_storecnt 0x02207; GFX1250-NEXT:    flat_atomic_add_f64 v[0:1], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS2208; GFX1250-NEXT:    s_wait_loadcnt 0x02209; GFX1250-NEXT:    global_inv scope:SCOPE_SYS2210; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x02211; GFX1250-NEXT:    s_set_pc_i64 s[30:31]2212main_body:2213  %ret = atomicrmw fadd ptr %ptr, double 4.0 syncscope("one-as") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !02214  ret double %ret2215}2216 2217define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_agent_safe(ptr %ptr) {2218; GFX90A-LABEL: flat_atomic_fadd_f64_noret_pat_agent_safe:2219; GFX90A:       ; %bb.0: ; %main_body2220; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x242221; GFX90A-NEXT:    v_mov_b32_e32 v2, 02222; GFX90A-NEXT:    v_mov_b32_e32 v3, 0x401000002223; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)2224; GFX90A-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]2225; GFX90A-NEXT:    flat_atomic_add_f64 v[0:1], v[2:3]2226; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2227; GFX90A-NEXT:    buffer_wbinvl1_vol2228; GFX90A-NEXT:    s_endpgm2229;2230; GFX942-LABEL: flat_atomic_fadd_f64_noret_pat_agent_safe:2231; GFX942:       ; %bb.0: ; %main_body2232; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x242233; GFX942-NEXT:    v_mov_b64_e32 v[2:3], 4.02234; GFX942-NEXT:    s_waitcnt lgkmcnt(0)2235; GFX942-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]2236; GFX942-NEXT:    buffer_wbl2 sc12237; GFX942-NEXT:    flat_atomic_add_f64 v[0:1], v[2:3]2238; GFX942-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)2239; GFX942-NEXT:    buffer_inv sc12240; GFX942-NEXT:    s_endpgm2241;2242; GFX1250-LABEL: flat_atomic_fadd_f64_noret_pat_agent_safe:2243; GFX1250:       ; %bb.0: ; %main_body2244; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12245; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x242246; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], 4.02247; GFX1250-NEXT:    v_mov_b32_e32 v2, 02248; GFX1250-NEXT:    global_wb scope:SCOPE_DEV2249; GFX1250-NEXT:    s_wait_storecnt 0x02250; GFX1250-NEXT:    s_wait_kmcnt 0x02251; GFX1250-NEXT:    flat_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_DEV2252; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x02253; GFX1250-NEXT:    global_inv scope:SCOPE_DEV2254; GFX1250-NEXT:    s_endpgm2255main_body:2256  %ret = atomicrmw fadd ptr %ptr, double 4.0 syncscope("agent") seq_cst, !noalias.addrspace !1, !amdgpu.no.fine.grained.memory !02257  ret void2258}2259 2260define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr) #1 {2261; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat:2262; GFX90A:       ; %bb.0: ; %main_body2263; GFX90A-NEXT:    s_mov_b64 s[0:1], exec2264; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 02265; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v02266; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v02267; GFX90A-NEXT:    s_and_saveexec_b64 s[2:3], vcc2268; GFX90A-NEXT:    s_cbranch_execz .LBB51_22269; GFX90A-NEXT:  ; %bb.1:2270; GFX90A-NEXT:    s_load_dword s2, s[4:5], 0x242271; GFX90A-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]2272; GFX90A-NEXT:    v_cvt_f64_u32_e32 v[0:1], s02273; GFX90A-NEXT:    v_mul_f64 v[0:1], v[0:1], 4.02274; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)2275; GFX90A-NEXT:    v_mov_b32_e32 v2, s22276; GFX90A-NEXT:    ds_add_f64 v2, v[0:1]2277; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)2278; GFX90A-NEXT:  .LBB51_2:2279; GFX90A-NEXT:    s_endpgm2280;2281; GFX942-LABEL: local_atomic_fadd_f64_noret_pat:2282; GFX942:       ; %bb.0: ; %main_body2283; GFX942-NEXT:    s_mov_b64 s[0:1], exec2284; GFX942-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 02285; GFX942-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v02286; GFX942-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v02287; GFX942-NEXT:    s_and_saveexec_b64 s[2:3], vcc2288; GFX942-NEXT:    s_cbranch_execz .LBB51_22289; GFX942-NEXT:  ; %bb.1:2290; GFX942-NEXT:    s_load_dword s2, s[4:5], 0x242291; GFX942-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]2292; GFX942-NEXT:    v_cvt_f64_u32_e32 v[0:1], s02293; GFX942-NEXT:    v_mul_f64 v[0:1], v[0:1], 4.02294; GFX942-NEXT:    s_waitcnt lgkmcnt(0)2295; GFX942-NEXT:    v_mov_b32_e32 v2, s22296; GFX942-NEXT:    ds_add_f64 v2, v[0:1]2297; GFX942-NEXT:    s_waitcnt lgkmcnt(0)2298; GFX942-NEXT:  .LBB51_2:2299; GFX942-NEXT:    s_endpgm2300;2301; GFX1250-LABEL: local_atomic_fadd_f64_noret_pat:2302; GFX1250:       ; %bb.0: ; %main_body2303; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12304; GFX1250-NEXT:    s_mov_b32 s0, exec_lo2305; GFX1250-NEXT:    s_mov_b32 s1, exec_lo2306; GFX1250-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 02307; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)2308; GFX1250-NEXT:    v_cmpx_eq_u32_e32 0, v02309; GFX1250-NEXT:    s_cbranch_execz .LBB51_22310; GFX1250-NEXT:  ; %bb.1:2311; GFX1250-NEXT:    s_bcnt1_i32_b32 s0, s02312; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)2313; GFX1250-NEXT:    v_cvt_f64_u32_e32 v[0:1], s02314; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x242315; GFX1250-NEXT:    s_wait_kmcnt 0x02316; GFX1250-NEXT:    v_dual_mul_f64 v[0:1], 4.0, v[0:1] :: v_dual_mov_b32 v2, s02317; GFX1250-NEXT:    ds_add_f64 v2, v[0:1]2318; GFX1250-NEXT:    s_wait_dscnt 0x02319; GFX1250-NEXT:  .LBB51_2:2320; GFX1250-NEXT:    s_endpgm2321main_body:2322  %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst, !amdgpu.no.fine.grained.memory !02323  ret void2324}2325 2326define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3) %ptr) #0 {2327; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat_flush:2328; GFX90A:       ; %bb.0: ; %main_body2329; GFX90A-NEXT:    s_mov_b64 s[0:1], exec2330; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 02331; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v02332; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v02333; GFX90A-NEXT:    s_and_saveexec_b64 s[2:3], vcc2334; GFX90A-NEXT:    s_cbranch_execz .LBB52_22335; GFX90A-NEXT:  ; %bb.1:2336; GFX90A-NEXT:    s_load_dword s2, s[4:5], 0x242337; GFX90A-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]2338; GFX90A-NEXT:    v_cvt_f64_u32_e32 v[0:1], s02339; GFX90A-NEXT:    v_mul_f64 v[0:1], v[0:1], 4.02340; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)2341; GFX90A-NEXT:    v_mov_b32_e32 v2, s22342; GFX90A-NEXT:    ds_add_f64 v2, v[0:1]2343; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)2344; GFX90A-NEXT:  .LBB52_2:2345; GFX90A-NEXT:    s_endpgm2346;2347; GFX942-LABEL: local_atomic_fadd_f64_noret_pat_flush:2348; GFX942:       ; %bb.0: ; %main_body2349; GFX942-NEXT:    s_mov_b64 s[0:1], exec2350; GFX942-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 02351; GFX942-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v02352; GFX942-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v02353; GFX942-NEXT:    s_and_saveexec_b64 s[2:3], vcc2354; GFX942-NEXT:    s_cbranch_execz .LBB52_22355; GFX942-NEXT:  ; %bb.1:2356; GFX942-NEXT:    s_load_dword s2, s[4:5], 0x242357; GFX942-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]2358; GFX942-NEXT:    v_cvt_f64_u32_e32 v[0:1], s02359; GFX942-NEXT:    v_mul_f64 v[0:1], v[0:1], 4.02360; GFX942-NEXT:    s_waitcnt lgkmcnt(0)2361; GFX942-NEXT:    v_mov_b32_e32 v2, s22362; GFX942-NEXT:    ds_add_f64 v2, v[0:1]2363; GFX942-NEXT:    s_waitcnt lgkmcnt(0)2364; GFX942-NEXT:  .LBB52_2:2365; GFX942-NEXT:    s_endpgm2366;2367; GFX1250-LABEL: local_atomic_fadd_f64_noret_pat_flush:2368; GFX1250:       ; %bb.0: ; %main_body2369; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12370; GFX1250-NEXT:    s_mov_b32 s0, exec_lo2371; GFX1250-NEXT:    s_mov_b32 s1, exec_lo2372; GFX1250-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 02373; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)2374; GFX1250-NEXT:    v_cmpx_eq_u32_e32 0, v02375; GFX1250-NEXT:    s_cbranch_execz .LBB52_22376; GFX1250-NEXT:  ; %bb.1:2377; GFX1250-NEXT:    s_bcnt1_i32_b32 s0, s02378; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)2379; GFX1250-NEXT:    v_cvt_f64_u32_e32 v[0:1], s02380; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x242381; GFX1250-NEXT:    s_wait_kmcnt 0x02382; GFX1250-NEXT:    v_dual_mul_f64 v[0:1], 4.0, v[0:1] :: v_dual_mov_b32 v2, s02383; GFX1250-NEXT:    ds_add_f64 v2, v[0:1]2384; GFX1250-NEXT:    s_wait_dscnt 0x02385; GFX1250-NEXT:  .LBB52_2:2386; GFX1250-NEXT:    s_endpgm2387main_body:2388  %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst, !amdgpu.no.fine.grained.memory !02389  ret void2390}2391 2392define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrspace(3) %ptr) #2 {2393; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe:2394; GFX90A:       ; %bb.0: ; %main_body2395; GFX90A-NEXT:    s_mov_b64 s[0:1], exec2396; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 02397; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v02398; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v02399; GFX90A-NEXT:    s_and_saveexec_b64 s[2:3], vcc2400; GFX90A-NEXT:    s_cbranch_execz .LBB53_22401; GFX90A-NEXT:  ; %bb.1:2402; GFX90A-NEXT:    s_load_dword s2, s[4:5], 0x242403; GFX90A-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]2404; GFX90A-NEXT:    v_cvt_f64_u32_e32 v[0:1], s02405; GFX90A-NEXT:    v_mul_f64 v[0:1], v[0:1], 4.02406; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)2407; GFX90A-NEXT:    v_mov_b32_e32 v2, s22408; GFX90A-NEXT:    ds_add_f64 v2, v[0:1]2409; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)2410; GFX90A-NEXT:  .LBB53_2:2411; GFX90A-NEXT:    s_endpgm2412;2413; GFX942-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe:2414; GFX942:       ; %bb.0: ; %main_body2415; GFX942-NEXT:    s_mov_b64 s[0:1], exec2416; GFX942-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 02417; GFX942-NEXT:    v_mbcnt_hi_u32_b32 v0, s1, v02418; GFX942-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v02419; GFX942-NEXT:    s_and_saveexec_b64 s[2:3], vcc2420; GFX942-NEXT:    s_cbranch_execz .LBB53_22421; GFX942-NEXT:  ; %bb.1:2422; GFX942-NEXT:    s_load_dword s2, s[4:5], 0x242423; GFX942-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]2424; GFX942-NEXT:    v_cvt_f64_u32_e32 v[0:1], s02425; GFX942-NEXT:    v_mul_f64 v[0:1], v[0:1], 4.02426; GFX942-NEXT:    s_waitcnt lgkmcnt(0)2427; GFX942-NEXT:    v_mov_b32_e32 v2, s22428; GFX942-NEXT:    ds_add_f64 v2, v[0:1]2429; GFX942-NEXT:    s_waitcnt lgkmcnt(0)2430; GFX942-NEXT:  .LBB53_2:2431; GFX942-NEXT:    s_endpgm2432;2433; GFX1250-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe:2434; GFX1250:       ; %bb.0: ; %main_body2435; GFX1250-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12436; GFX1250-NEXT:    s_mov_b32 s0, exec_lo2437; GFX1250-NEXT:    s_mov_b32 s1, exec_lo2438; GFX1250-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 02439; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)2440; GFX1250-NEXT:    v_cmpx_eq_u32_e32 0, v02441; GFX1250-NEXT:    s_cbranch_execz .LBB53_22442; GFX1250-NEXT:  ; %bb.1:2443; GFX1250-NEXT:    s_bcnt1_i32_b32 s0, s02444; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)2445; GFX1250-NEXT:    v_cvt_f64_u32_e32 v[0:1], s02446; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x242447; GFX1250-NEXT:    s_wait_kmcnt 0x02448; GFX1250-NEXT:    v_dual_mul_f64 v[0:1], 4.0, v[0:1] :: v_dual_mov_b32 v2, s02449; GFX1250-NEXT:    ds_add_f64 v2, v[0:1]2450; GFX1250-NEXT:    s_wait_dscnt 0x02451; GFX1250-NEXT:  .LBB53_2:2452; GFX1250-NEXT:    s_endpgm2453main_body:2454  %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst, !amdgpu.no.fine.grained.memory !02455  ret void2456}2457 2458define double @local_atomic_fadd_f64_rtn_pat(ptr addrspace(3) %ptr, double %data) #1 {2459; GFX90A-LABEL: local_atomic_fadd_f64_rtn_pat:2460; GFX90A:       ; %bb.0: ; %main_body2461; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2462; GFX90A-NEXT:    v_mov_b32_e32 v2, 02463; GFX90A-NEXT:    v_mov_b32_e32 v3, 0x401000002464; GFX90A-NEXT:    ds_add_rtn_f64 v[0:1], v0, v[2:3]2465; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)2466; GFX90A-NEXT:    s_setpc_b64 s[30:31]2467;2468; GFX942-LABEL: local_atomic_fadd_f64_rtn_pat:2469; GFX942:       ; %bb.0: ; %main_body2470; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2471; GFX942-NEXT:    v_mov_b64_e32 v[2:3], 4.02472; GFX942-NEXT:    ds_add_rtn_f64 v[0:1], v0, v[2:3]2473; GFX942-NEXT:    s_waitcnt lgkmcnt(0)2474; GFX942-NEXT:    s_setpc_b64 s[30:31]2475;2476; GFX1250-LABEL: local_atomic_fadd_f64_rtn_pat:2477; GFX1250:       ; %bb.0: ; %main_body2478; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x02479; GFX1250-NEXT:    s_wait_kmcnt 0x02480; GFX1250-NEXT:    v_mov_b64_e32 v[2:3], 4.02481; GFX1250-NEXT:    s_wait_storecnt 0x02482; GFX1250-NEXT:    ds_add_rtn_f64 v[0:1], v0, v[2:3]2483; GFX1250-NEXT:    s_wait_dscnt 0x02484; GFX1250-NEXT:    s_set_pc_i64 s[30:31]2485main_body:2486  %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst2487  ret double %ret2488}2489 2490attributes #0 = { nounwind "denormal-fp-math"="preserve-sign,preserve-sign" }2491attributes #1 = { nounwind }2492attributes #2 = { nounwind "denormal-fp-math"="preserve-sign,preserve-sign" }2493 2494!0 = !{}2495!1 = !{i32 5, i32 6}2496