7505 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -mcpu=bonaire -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -enable-var-scope -check-prefixes=CI %s3; RUN: llc -mtriple=amdgcn -mcpu=tonga -amdgpu-atomic-optimizer-strategy=None -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=VI %s4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s5; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck -enable-var-scope -check-prefixes=GFX12 %s6 7define amdgpu_kernel void @atomic_add_i64_offset(ptr addrspace(1) %out, i64 %in) {8; CI-LABEL: atomic_add_i64_offset:9; CI: ; %bb.0: ; %entry10; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x911; CI-NEXT: s_waitcnt lgkmcnt(0)12; CI-NEXT: v_mov_b32_e32 v0, s213; CI-NEXT: v_mov_b32_e32 v1, s314; CI-NEXT: s_mov_b32 s3, 0xf00015; CI-NEXT: s_mov_b32 s2, -116; CI-NEXT: buffer_atomic_add_x2 v[0:1], off, s[0:3], 0 offset:3217; CI-NEXT: s_waitcnt vmcnt(0)18; CI-NEXT: buffer_wbinvl1_vol19; CI-NEXT: s_endpgm20;21; VI-LABEL: atomic_add_i64_offset:22; VI: ; %bb.0: ; %entry23; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2424; VI-NEXT: s_waitcnt lgkmcnt(0)25; VI-NEXT: v_mov_b32_e32 v0, s226; VI-NEXT: v_mov_b32_e32 v1, s327; VI-NEXT: s_mov_b32 s3, 0xf00028; VI-NEXT: s_mov_b32 s2, -129; VI-NEXT: buffer_atomic_add_x2 v[0:1], off, s[0:3], 0 offset:3230; VI-NEXT: s_waitcnt vmcnt(0)31; VI-NEXT: buffer_wbinvl1_vol32; VI-NEXT: s_endpgm33;34; GFX9-LABEL: atomic_add_i64_offset:35; GFX9: ; %bb.0: ; %entry36; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2437; GFX9-NEXT: v_mov_b32_e32 v2, 038; GFX9-NEXT: s_waitcnt lgkmcnt(0)39; GFX9-NEXT: v_mov_b32_e32 v0, s240; GFX9-NEXT: v_mov_b32_e32 v1, s341; GFX9-NEXT: global_atomic_add_x2 v2, v[0:1], s[0:1] offset:3242; GFX9-NEXT: s_waitcnt vmcnt(0)43; GFX9-NEXT: buffer_wbinvl1_vol44; GFX9-NEXT: s_endpgm45;46; GFX12-LABEL: atomic_add_i64_offset:47; GFX12: ; %bb.0: ; %entry48; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2449; GFX12-NEXT: s_wait_kmcnt 0x050; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s351; GFX12-NEXT: v_mov_b32_e32 v0, s252; GFX12-NEXT: global_atomic_add_u64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_DEV53; GFX12-NEXT: s_wait_storecnt 0x054; GFX12-NEXT: global_inv scope:SCOPE_DEV55; GFX12-NEXT: s_endpgm56entry:57 %gep = getelementptr i64, ptr addrspace(1) %out, i64 458 %tmp0 = atomicrmw volatile add ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst59 ret void60}61 62define amdgpu_kernel void @atomic_add_i64_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in) {63; CI-LABEL: atomic_add_i64_ret_offset:64; CI: ; %bb.0: ; %entry65; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x966; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd67; CI-NEXT: s_mov_b32 s7, 0xf00068; CI-NEXT: s_mov_b32 s6, -169; CI-NEXT: s_waitcnt lgkmcnt(0)70; CI-NEXT: s_mov_b32 s4, s271; CI-NEXT: s_mov_b32 s5, s372; CI-NEXT: v_mov_b32_e32 v0, s873; CI-NEXT: v_mov_b32_e32 v1, s974; CI-NEXT: s_mov_b32 s2, s675; CI-NEXT: s_mov_b32 s3, s776; CI-NEXT: buffer_atomic_add_x2 v[0:1], off, s[0:3], 0 offset:32 glc77; CI-NEXT: s_waitcnt vmcnt(0)78; CI-NEXT: buffer_wbinvl1_vol79; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 080; CI-NEXT: s_endpgm81;82; VI-LABEL: atomic_add_i64_ret_offset:83; VI: ; %bb.0: ; %entry84; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2485; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x3486; VI-NEXT: s_mov_b32 s7, 0xf00087; VI-NEXT: s_mov_b32 s6, -188; VI-NEXT: s_waitcnt lgkmcnt(0)89; VI-NEXT: s_mov_b32 s4, s290; VI-NEXT: s_mov_b32 s5, s391; VI-NEXT: v_mov_b32_e32 v0, s892; VI-NEXT: v_mov_b32_e32 v1, s993; VI-NEXT: s_mov_b32 s2, s694; VI-NEXT: s_mov_b32 s3, s795; VI-NEXT: buffer_atomic_add_x2 v[0:1], off, s[0:3], 0 offset:32 glc96; VI-NEXT: s_waitcnt vmcnt(0)97; VI-NEXT: buffer_wbinvl1_vol98; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 099; VI-NEXT: s_endpgm100;101; GFX9-LABEL: atomic_add_i64_ret_offset:102; GFX9: ; %bb.0: ; %entry103; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34104; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24105; GFX9-NEXT: v_mov_b32_e32 v2, 0106; GFX9-NEXT: s_waitcnt lgkmcnt(0)107; GFX9-NEXT: v_mov_b32_e32 v0, s6108; GFX9-NEXT: v_mov_b32_e32 v1, s7109; GFX9-NEXT: global_atomic_add_x2 v[0:1], v2, v[0:1], s[0:1] offset:32 glc110; GFX9-NEXT: s_waitcnt vmcnt(0)111; GFX9-NEXT: buffer_wbinvl1_vol112; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]113; GFX9-NEXT: s_endpgm114;115; GFX12-LABEL: atomic_add_i64_ret_offset:116; GFX12: ; %bb.0: ; %entry117; GFX12-NEXT: s_clause 0x1118; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x34119; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24120; GFX12-NEXT: s_wait_kmcnt 0x0121; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s7122; GFX12-NEXT: v_mov_b32_e32 v0, s6123; GFX12-NEXT: global_atomic_add_u64 v[0:1], v2, v[0:1], s[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV124; GFX12-NEXT: s_wait_loadcnt 0x0125; GFX12-NEXT: global_inv scope:SCOPE_DEV126; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]127; GFX12-NEXT: s_endpgm128entry:129 %gep = getelementptr i64, ptr addrspace(1) %out, i64 4130 %tmp0 = atomicrmw volatile add ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst131 store i64 %tmp0, ptr addrspace(1) %out2132 ret void133}134 135define amdgpu_kernel void @atomic_add_i64_addr64_offset(ptr addrspace(1) %out, i64 %in, i64 %index) {136; CI-LABEL: atomic_add_i64_addr64_offset:137; CI: ; %bb.0: ; %entry138; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9139; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd140; CI-NEXT: s_waitcnt lgkmcnt(0)141; CI-NEXT: v_mov_b32_e32 v0, s2142; CI-NEXT: s_lshl_b64 s[4:5], s[4:5], 3143; CI-NEXT: v_mov_b32_e32 v2, s4144; CI-NEXT: v_mov_b32_e32 v1, s3145; CI-NEXT: s_mov_b32 s3, 0xf000146; CI-NEXT: s_mov_b32 s2, 0147; CI-NEXT: v_mov_b32_e32 v3, s5148; CI-NEXT: buffer_atomic_add_x2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:32149; CI-NEXT: s_waitcnt vmcnt(0)150; CI-NEXT: buffer_wbinvl1_vol151; CI-NEXT: s_endpgm152;153; VI-LABEL: atomic_add_i64_addr64_offset:154; VI: ; %bb.0: ; %entry155; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24156; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34157; VI-NEXT: s_waitcnt lgkmcnt(0)158; VI-NEXT: v_mov_b32_e32 v0, s2159; VI-NEXT: v_mov_b32_e32 v1, s3160; VI-NEXT: s_lshl_b64 s[2:3], s[4:5], 3161; VI-NEXT: s_add_u32 s0, s0, s2162; VI-NEXT: s_addc_u32 s1, s1, s3163; VI-NEXT: s_add_u32 s0, s0, 32164; VI-NEXT: s_addc_u32 s1, s1, 0165; VI-NEXT: v_mov_b32_e32 v3, s1166; VI-NEXT: v_mov_b32_e32 v2, s0167; VI-NEXT: flat_atomic_add_x2 v[2:3], v[0:1]168; VI-NEXT: s_waitcnt vmcnt(0)169; VI-NEXT: buffer_wbinvl1_vol170; VI-NEXT: s_endpgm171;172; GFX9-LABEL: atomic_add_i64_addr64_offset:173; GFX9: ; %bb.0: ; %entry174; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24175; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34176; GFX9-NEXT: v_mov_b32_e32 v2, 0177; GFX9-NEXT: s_waitcnt lgkmcnt(0)178; GFX9-NEXT: v_mov_b32_e32 v0, s2179; GFX9-NEXT: v_mov_b32_e32 v1, s3180; GFX9-NEXT: s_lshl_b64 s[2:3], s[6:7], 3181; GFX9-NEXT: s_add_u32 s0, s0, s2182; GFX9-NEXT: s_addc_u32 s1, s1, s3183; GFX9-NEXT: global_atomic_add_x2 v2, v[0:1], s[0:1] offset:32184; GFX9-NEXT: s_waitcnt vmcnt(0)185; GFX9-NEXT: buffer_wbinvl1_vol186; GFX9-NEXT: s_endpgm187;188; GFX12-LABEL: atomic_add_i64_addr64_offset:189; GFX12: ; %bb.0: ; %entry190; GFX12-NEXT: s_clause 0x1191; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24192; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x34193; GFX12-NEXT: v_mov_b32_e32 v2, 0194; GFX12-NEXT: s_wait_kmcnt 0x0195; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3196; GFX12-NEXT: s_lshl_b64 s[2:3], s[4:5], 3197; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)198; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]199; GFX12-NEXT: global_atomic_add_u64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_DEV200; GFX12-NEXT: s_wait_storecnt 0x0201; GFX12-NEXT: global_inv scope:SCOPE_DEV202; GFX12-NEXT: s_endpgm203entry:204 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index205 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4206 %tmp0 = atomicrmw volatile add ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst207 ret void208}209 210define amdgpu_kernel void @atomic_add_i64_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {211; CI-LABEL: atomic_add_i64_ret_addr64_offset:212; CI: ; %bb.0: ; %entry213; CI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9214; CI-NEXT: s_mov_b32 s11, 0xf000215; CI-NEXT: s_mov_b32 s10, -1216; CI-NEXT: s_waitcnt lgkmcnt(0)217; CI-NEXT: v_mov_b32_e32 v0, s4218; CI-NEXT: v_mov_b32_e32 v1, s5219; CI-NEXT: s_lshl_b64 s[4:5], s[6:7], 3220; CI-NEXT: v_mov_b32_e32 v2, s4221; CI-NEXT: s_mov_b32 s8, s2222; CI-NEXT: s_mov_b32 s9, s3223; CI-NEXT: s_mov_b32 s2, 0224; CI-NEXT: s_mov_b32 s3, s11225; CI-NEXT: v_mov_b32_e32 v3, s5226; CI-NEXT: buffer_atomic_add_x2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:32 glc227; CI-NEXT: s_waitcnt vmcnt(0)228; CI-NEXT: buffer_wbinvl1_vol229; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0230; CI-NEXT: s_endpgm231;232; VI-LABEL: atomic_add_i64_ret_addr64_offset:233; VI: ; %bb.0: ; %entry234; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24235; VI-NEXT: s_waitcnt lgkmcnt(0)236; VI-NEXT: v_mov_b32_e32 v0, s4237; VI-NEXT: v_mov_b32_e32 v1, s5238; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 3239; VI-NEXT: s_add_u32 s0, s0, s4240; VI-NEXT: s_addc_u32 s1, s1, s5241; VI-NEXT: s_add_u32 s0, s0, 32242; VI-NEXT: s_addc_u32 s1, s1, 0243; VI-NEXT: v_mov_b32_e32 v3, s1244; VI-NEXT: v_mov_b32_e32 v2, s0245; VI-NEXT: flat_atomic_add_x2 v[0:1], v[2:3], v[0:1] glc246; VI-NEXT: s_waitcnt vmcnt(0)247; VI-NEXT: buffer_wbinvl1_vol248; VI-NEXT: s_mov_b32 s7, 0xf000249; VI-NEXT: s_mov_b32 s6, -1250; VI-NEXT: s_mov_b32 s4, s2251; VI-NEXT: s_mov_b32 s5, s3252; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0253; VI-NEXT: s_endpgm254;255; GFX9-LABEL: atomic_add_i64_ret_addr64_offset:256; GFX9: ; %bb.0: ; %entry257; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24258; GFX9-NEXT: v_mov_b32_e32 v2, 0259; GFX9-NEXT: s_waitcnt lgkmcnt(0)260; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 3261; GFX9-NEXT: s_add_u32 s0, s8, s0262; GFX9-NEXT: v_mov_b32_e32 v0, s12263; GFX9-NEXT: v_mov_b32_e32 v1, s13264; GFX9-NEXT: s_addc_u32 s1, s9, s1265; GFX9-NEXT: global_atomic_add_x2 v[0:1], v2, v[0:1], s[0:1] offset:32 glc266; GFX9-NEXT: s_waitcnt vmcnt(0)267; GFX9-NEXT: buffer_wbinvl1_vol268; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]269; GFX9-NEXT: s_endpgm270;271; GFX12-LABEL: atomic_add_i64_ret_addr64_offset:272; GFX12: ; %bb.0: ; %entry273; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x24274; GFX12-NEXT: s_wait_kmcnt 0x0275; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s5276; GFX12-NEXT: v_mov_b32_e32 v0, s4277; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 3278; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)279; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]280; GFX12-NEXT: global_atomic_add_u64 v[0:1], v2, v[0:1], s[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV281; GFX12-NEXT: s_wait_loadcnt 0x0282; GFX12-NEXT: global_inv scope:SCOPE_DEV283; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]284; GFX12-NEXT: s_endpgm285entry:286 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index287 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4288 %tmp0 = atomicrmw volatile add ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst289 store i64 %tmp0, ptr addrspace(1) %out2290 ret void291}292 293define amdgpu_kernel void @atomic_add_i64(ptr addrspace(1) %out, i64 %in) {294; CI-LABEL: atomic_add_i64:295; CI: ; %bb.0: ; %entry296; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9297; CI-NEXT: s_mov_b32 s7, 0xf000298; CI-NEXT: s_mov_b32 s6, -1299; CI-NEXT: s_waitcnt lgkmcnt(0)300; CI-NEXT: s_mov_b32 s4, s0301; CI-NEXT: s_mov_b32 s5, s1302; CI-NEXT: v_mov_b32_e32 v0, s2303; CI-NEXT: v_mov_b32_e32 v1, s3304; CI-NEXT: buffer_atomic_add_x2 v[0:1], off, s[4:7], 0305; CI-NEXT: s_waitcnt vmcnt(0)306; CI-NEXT: buffer_wbinvl1_vol307; CI-NEXT: s_endpgm308;309; VI-LABEL: atomic_add_i64:310; VI: ; %bb.0: ; %entry311; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24312; VI-NEXT: s_mov_b32 s7, 0xf000313; VI-NEXT: s_mov_b32 s6, -1314; VI-NEXT: s_waitcnt lgkmcnt(0)315; VI-NEXT: s_mov_b32 s4, s0316; VI-NEXT: s_mov_b32 s5, s1317; VI-NEXT: v_mov_b32_e32 v0, s2318; VI-NEXT: v_mov_b32_e32 v1, s3319; VI-NEXT: buffer_atomic_add_x2 v[0:1], off, s[4:7], 0320; VI-NEXT: s_waitcnt vmcnt(0)321; VI-NEXT: buffer_wbinvl1_vol322; VI-NEXT: s_endpgm323;324; GFX9-LABEL: atomic_add_i64:325; GFX9: ; %bb.0: ; %entry326; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24327; GFX9-NEXT: v_mov_b32_e32 v2, 0328; GFX9-NEXT: s_waitcnt lgkmcnt(0)329; GFX9-NEXT: v_mov_b32_e32 v0, s2330; GFX9-NEXT: v_mov_b32_e32 v1, s3331; GFX9-NEXT: global_atomic_add_x2 v2, v[0:1], s[0:1]332; GFX9-NEXT: s_waitcnt vmcnt(0)333; GFX9-NEXT: buffer_wbinvl1_vol334; GFX9-NEXT: s_endpgm335;336; GFX12-LABEL: atomic_add_i64:337; GFX12: ; %bb.0: ; %entry338; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24339; GFX12-NEXT: s_wait_kmcnt 0x0340; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3341; GFX12-NEXT: v_mov_b32_e32 v0, s2342; GFX12-NEXT: global_atomic_add_u64 v2, v[0:1], s[0:1] scope:SCOPE_DEV343; GFX12-NEXT: s_wait_storecnt 0x0344; GFX12-NEXT: global_inv scope:SCOPE_DEV345; GFX12-NEXT: s_endpgm346entry:347 %tmp0 = atomicrmw volatile add ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst348 ret void349}350 351define amdgpu_kernel void @atomic_add_i64_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in) {352; CI-LABEL: atomic_add_i64_ret:353; CI: ; %bb.0: ; %entry354; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9355; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd356; CI-NEXT: s_mov_b32 s7, 0xf000357; CI-NEXT: s_mov_b32 s6, -1358; CI-NEXT: s_waitcnt lgkmcnt(0)359; CI-NEXT: s_mov_b32 s4, s0360; CI-NEXT: s_mov_b32 s5, s1361; CI-NEXT: v_mov_b32_e32 v0, s8362; CI-NEXT: v_mov_b32_e32 v1, s9363; CI-NEXT: buffer_atomic_add_x2 v[0:1], off, s[4:7], 0 glc364; CI-NEXT: s_waitcnt vmcnt(0)365; CI-NEXT: buffer_wbinvl1_vol366; CI-NEXT: s_mov_b32 s4, s2367; CI-NEXT: s_mov_b32 s5, s3368; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0369; CI-NEXT: s_endpgm370;371; VI-LABEL: atomic_add_i64_ret:372; VI: ; %bb.0: ; %entry373; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24374; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34375; VI-NEXT: s_mov_b32 s7, 0xf000376; VI-NEXT: s_mov_b32 s6, -1377; VI-NEXT: s_waitcnt lgkmcnt(0)378; VI-NEXT: s_mov_b32 s4, s0379; VI-NEXT: s_mov_b32 s5, s1380; VI-NEXT: v_mov_b32_e32 v0, s8381; VI-NEXT: v_mov_b32_e32 v1, s9382; VI-NEXT: buffer_atomic_add_x2 v[0:1], off, s[4:7], 0 glc383; VI-NEXT: s_waitcnt vmcnt(0)384; VI-NEXT: buffer_wbinvl1_vol385; VI-NEXT: s_mov_b32 s4, s2386; VI-NEXT: s_mov_b32 s5, s3387; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0388; VI-NEXT: s_endpgm389;390; GFX9-LABEL: atomic_add_i64_ret:391; GFX9: ; %bb.0: ; %entry392; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34393; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24394; GFX9-NEXT: v_mov_b32_e32 v2, 0395; GFX9-NEXT: s_waitcnt lgkmcnt(0)396; GFX9-NEXT: v_mov_b32_e32 v0, s6397; GFX9-NEXT: v_mov_b32_e32 v1, s7398; GFX9-NEXT: global_atomic_add_x2 v[0:1], v2, v[0:1], s[0:1] glc399; GFX9-NEXT: s_waitcnt vmcnt(0)400; GFX9-NEXT: buffer_wbinvl1_vol401; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]402; GFX9-NEXT: s_endpgm403;404; GFX12-LABEL: atomic_add_i64_ret:405; GFX12: ; %bb.0: ; %entry406; GFX12-NEXT: s_clause 0x1407; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x34408; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24409; GFX12-NEXT: s_wait_kmcnt 0x0410; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s7411; GFX12-NEXT: v_mov_b32_e32 v0, s6412; GFX12-NEXT: global_atomic_add_u64 v[0:1], v2, v[0:1], s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV413; GFX12-NEXT: s_wait_loadcnt 0x0414; GFX12-NEXT: global_inv scope:SCOPE_DEV415; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]416; GFX12-NEXT: s_endpgm417entry:418 %tmp0 = atomicrmw volatile add ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst419 store i64 %tmp0, ptr addrspace(1) %out2420 ret void421}422 423define amdgpu_kernel void @atomic_add_i64_addr64(ptr addrspace(1) %out, i64 %in, i64 %index) {424; CI-LABEL: atomic_add_i64_addr64:425; CI: ; %bb.0: ; %entry426; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9427; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd428; CI-NEXT: s_waitcnt lgkmcnt(0)429; CI-NEXT: v_mov_b32_e32 v0, s2430; CI-NEXT: s_lshl_b64 s[4:5], s[4:5], 3431; CI-NEXT: v_mov_b32_e32 v2, s4432; CI-NEXT: v_mov_b32_e32 v1, s3433; CI-NEXT: s_mov_b32 s3, 0xf000434; CI-NEXT: s_mov_b32 s2, 0435; CI-NEXT: v_mov_b32_e32 v3, s5436; CI-NEXT: buffer_atomic_add_x2 v[0:1], v[2:3], s[0:3], 0 addr64437; CI-NEXT: s_waitcnt vmcnt(0)438; CI-NEXT: buffer_wbinvl1_vol439; CI-NEXT: s_endpgm440;441; VI-LABEL: atomic_add_i64_addr64:442; VI: ; %bb.0: ; %entry443; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24444; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34445; VI-NEXT: s_waitcnt lgkmcnt(0)446; VI-NEXT: v_mov_b32_e32 v0, s2447; VI-NEXT: v_mov_b32_e32 v1, s3448; VI-NEXT: s_lshl_b64 s[2:3], s[4:5], 3449; VI-NEXT: s_add_u32 s0, s0, s2450; VI-NEXT: s_addc_u32 s1, s1, s3451; VI-NEXT: v_mov_b32_e32 v3, s1452; VI-NEXT: v_mov_b32_e32 v2, s0453; VI-NEXT: flat_atomic_add_x2 v[2:3], v[0:1]454; VI-NEXT: s_waitcnt vmcnt(0)455; VI-NEXT: buffer_wbinvl1_vol456; VI-NEXT: s_endpgm457;458; GFX9-LABEL: atomic_add_i64_addr64:459; GFX9: ; %bb.0: ; %entry460; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24461; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34462; GFX9-NEXT: v_mov_b32_e32 v2, 0463; GFX9-NEXT: s_waitcnt lgkmcnt(0)464; GFX9-NEXT: v_mov_b32_e32 v0, s2465; GFX9-NEXT: v_mov_b32_e32 v1, s3466; GFX9-NEXT: s_lshl_b64 s[2:3], s[6:7], 3467; GFX9-NEXT: s_add_u32 s0, s0, s2468; GFX9-NEXT: s_addc_u32 s1, s1, s3469; GFX9-NEXT: global_atomic_add_x2 v2, v[0:1], s[0:1]470; GFX9-NEXT: s_waitcnt vmcnt(0)471; GFX9-NEXT: buffer_wbinvl1_vol472; GFX9-NEXT: s_endpgm473;474; GFX12-LABEL: atomic_add_i64_addr64:475; GFX12: ; %bb.0: ; %entry476; GFX12-NEXT: s_clause 0x1477; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24478; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x34479; GFX12-NEXT: v_mov_b32_e32 v2, 0480; GFX12-NEXT: s_wait_kmcnt 0x0481; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3482; GFX12-NEXT: s_lshl_b64 s[2:3], s[4:5], 3483; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)484; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]485; GFX12-NEXT: global_atomic_add_u64 v2, v[0:1], s[0:1] scope:SCOPE_DEV486; GFX12-NEXT: s_wait_storecnt 0x0487; GFX12-NEXT: global_inv scope:SCOPE_DEV488; GFX12-NEXT: s_endpgm489entry:490 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index491 %tmp0 = atomicrmw volatile add ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst492 ret void493}494 495define amdgpu_kernel void @atomic_add_i64_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {496; CI-LABEL: atomic_add_i64_ret_addr64:497; CI: ; %bb.0: ; %entry498; CI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9499; CI-NEXT: s_mov_b32 s11, 0xf000500; CI-NEXT: s_mov_b32 s10, -1501; CI-NEXT: s_waitcnt lgkmcnt(0)502; CI-NEXT: v_mov_b32_e32 v0, s4503; CI-NEXT: v_mov_b32_e32 v1, s5504; CI-NEXT: s_lshl_b64 s[4:5], s[6:7], 3505; CI-NEXT: v_mov_b32_e32 v2, s4506; CI-NEXT: s_mov_b32 s8, s2507; CI-NEXT: s_mov_b32 s9, s3508; CI-NEXT: s_mov_b32 s2, 0509; CI-NEXT: s_mov_b32 s3, s11510; CI-NEXT: v_mov_b32_e32 v3, s5511; CI-NEXT: buffer_atomic_add_x2 v[0:1], v[2:3], s[0:3], 0 addr64 glc512; CI-NEXT: s_waitcnt vmcnt(0)513; CI-NEXT: buffer_wbinvl1_vol514; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0515; CI-NEXT: s_endpgm516;517; VI-LABEL: atomic_add_i64_ret_addr64:518; VI: ; %bb.0: ; %entry519; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24520; VI-NEXT: s_waitcnt lgkmcnt(0)521; VI-NEXT: v_mov_b32_e32 v0, s4522; VI-NEXT: v_mov_b32_e32 v1, s5523; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 3524; VI-NEXT: s_add_u32 s0, s0, s4525; VI-NEXT: s_addc_u32 s1, s1, s5526; VI-NEXT: v_mov_b32_e32 v3, s1527; VI-NEXT: v_mov_b32_e32 v2, s0528; VI-NEXT: flat_atomic_add_x2 v[0:1], v[2:3], v[0:1] glc529; VI-NEXT: s_waitcnt vmcnt(0)530; VI-NEXT: buffer_wbinvl1_vol531; VI-NEXT: s_mov_b32 s7, 0xf000532; VI-NEXT: s_mov_b32 s6, -1533; VI-NEXT: s_mov_b32 s4, s2534; VI-NEXT: s_mov_b32 s5, s3535; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0536; VI-NEXT: s_endpgm537;538; GFX9-LABEL: atomic_add_i64_ret_addr64:539; GFX9: ; %bb.0: ; %entry540; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24541; GFX9-NEXT: v_mov_b32_e32 v2, 0542; GFX9-NEXT: s_waitcnt lgkmcnt(0)543; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 3544; GFX9-NEXT: s_add_u32 s0, s8, s0545; GFX9-NEXT: v_mov_b32_e32 v0, s12546; GFX9-NEXT: v_mov_b32_e32 v1, s13547; GFX9-NEXT: s_addc_u32 s1, s9, s1548; GFX9-NEXT: global_atomic_add_x2 v[0:1], v2, v[0:1], s[0:1] glc549; GFX9-NEXT: s_waitcnt vmcnt(0)550; GFX9-NEXT: buffer_wbinvl1_vol551; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]552; GFX9-NEXT: s_endpgm553;554; GFX12-LABEL: atomic_add_i64_ret_addr64:555; GFX12: ; %bb.0: ; %entry556; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x24557; GFX12-NEXT: s_wait_kmcnt 0x0558; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s5559; GFX12-NEXT: v_mov_b32_e32 v0, s4560; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 3561; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)562; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]563; GFX12-NEXT: global_atomic_add_u64 v[0:1], v2, v[0:1], s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV564; GFX12-NEXT: s_wait_loadcnt 0x0565; GFX12-NEXT: global_inv scope:SCOPE_DEV566; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]567; GFX12-NEXT: s_endpgm568entry:569 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index570 %tmp0 = atomicrmw volatile add ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst571 store i64 %tmp0, ptr addrspace(1) %out2572 ret void573}574 575define amdgpu_kernel void @atomic_and_i64_offset(ptr addrspace(1) %out, i64 %in) {576; CI-LABEL: atomic_and_i64_offset:577; CI: ; %bb.0: ; %entry578; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9579; CI-NEXT: s_waitcnt lgkmcnt(0)580; CI-NEXT: v_mov_b32_e32 v0, s2581; CI-NEXT: v_mov_b32_e32 v1, s3582; CI-NEXT: s_mov_b32 s3, 0xf000583; CI-NEXT: s_mov_b32 s2, -1584; CI-NEXT: buffer_atomic_and_x2 v[0:1], off, s[0:3], 0 offset:32585; CI-NEXT: s_waitcnt vmcnt(0)586; CI-NEXT: buffer_wbinvl1_vol587; CI-NEXT: s_endpgm588;589; VI-LABEL: atomic_and_i64_offset:590; VI: ; %bb.0: ; %entry591; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24592; VI-NEXT: s_waitcnt lgkmcnt(0)593; VI-NEXT: v_mov_b32_e32 v0, s2594; VI-NEXT: v_mov_b32_e32 v1, s3595; VI-NEXT: s_mov_b32 s3, 0xf000596; VI-NEXT: s_mov_b32 s2, -1597; VI-NEXT: buffer_atomic_and_x2 v[0:1], off, s[0:3], 0 offset:32598; VI-NEXT: s_waitcnt vmcnt(0)599; VI-NEXT: buffer_wbinvl1_vol600; VI-NEXT: s_endpgm601;602; GFX9-LABEL: atomic_and_i64_offset:603; GFX9: ; %bb.0: ; %entry604; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24605; GFX9-NEXT: v_mov_b32_e32 v2, 0606; GFX9-NEXT: s_waitcnt lgkmcnt(0)607; GFX9-NEXT: v_mov_b32_e32 v0, s2608; GFX9-NEXT: v_mov_b32_e32 v1, s3609; GFX9-NEXT: global_atomic_and_x2 v2, v[0:1], s[0:1] offset:32610; GFX9-NEXT: s_waitcnt vmcnt(0)611; GFX9-NEXT: buffer_wbinvl1_vol612; GFX9-NEXT: s_endpgm613;614; GFX12-LABEL: atomic_and_i64_offset:615; GFX12: ; %bb.0: ; %entry616; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24617; GFX12-NEXT: s_wait_kmcnt 0x0618; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3619; GFX12-NEXT: v_mov_b32_e32 v0, s2620; GFX12-NEXT: global_atomic_and_b64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_DEV621; GFX12-NEXT: s_wait_storecnt 0x0622; GFX12-NEXT: global_inv scope:SCOPE_DEV623; GFX12-NEXT: s_endpgm624entry:625 %gep = getelementptr i64, ptr addrspace(1) %out, i64 4626 %tmp0 = atomicrmw volatile and ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0627 ret void628}629 630define amdgpu_kernel void @atomic_and_i64_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in) {631; CI-LABEL: atomic_and_i64_ret_offset:632; CI: ; %bb.0: ; %entry633; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9634; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd635; CI-NEXT: s_mov_b32 s7, 0xf000636; CI-NEXT: s_mov_b32 s6, -1637; CI-NEXT: s_waitcnt lgkmcnt(0)638; CI-NEXT: s_mov_b32 s4, s2639; CI-NEXT: s_mov_b32 s5, s3640; CI-NEXT: v_mov_b32_e32 v0, s8641; CI-NEXT: v_mov_b32_e32 v1, s9642; CI-NEXT: s_mov_b32 s2, s6643; CI-NEXT: s_mov_b32 s3, s7644; CI-NEXT: buffer_atomic_and_x2 v[0:1], off, s[0:3], 0 offset:32 glc645; CI-NEXT: s_waitcnt vmcnt(0)646; CI-NEXT: buffer_wbinvl1_vol647; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0648; CI-NEXT: s_endpgm649;650; VI-LABEL: atomic_and_i64_ret_offset:651; VI: ; %bb.0: ; %entry652; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24653; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34654; VI-NEXT: s_mov_b32 s7, 0xf000655; VI-NEXT: s_mov_b32 s6, -1656; VI-NEXT: s_waitcnt lgkmcnt(0)657; VI-NEXT: s_mov_b32 s4, s2658; VI-NEXT: s_mov_b32 s5, s3659; VI-NEXT: v_mov_b32_e32 v0, s8660; VI-NEXT: v_mov_b32_e32 v1, s9661; VI-NEXT: s_mov_b32 s2, s6662; VI-NEXT: s_mov_b32 s3, s7663; VI-NEXT: buffer_atomic_and_x2 v[0:1], off, s[0:3], 0 offset:32 glc664; VI-NEXT: s_waitcnt vmcnt(0)665; VI-NEXT: buffer_wbinvl1_vol666; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0667; VI-NEXT: s_endpgm668;669; GFX9-LABEL: atomic_and_i64_ret_offset:670; GFX9: ; %bb.0: ; %entry671; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34672; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24673; GFX9-NEXT: v_mov_b32_e32 v2, 0674; GFX9-NEXT: s_waitcnt lgkmcnt(0)675; GFX9-NEXT: v_mov_b32_e32 v0, s6676; GFX9-NEXT: v_mov_b32_e32 v1, s7677; GFX9-NEXT: global_atomic_and_x2 v[0:1], v2, v[0:1], s[0:1] offset:32 glc678; GFX9-NEXT: s_waitcnt vmcnt(0)679; GFX9-NEXT: buffer_wbinvl1_vol680; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]681; GFX9-NEXT: s_endpgm682;683; GFX12-LABEL: atomic_and_i64_ret_offset:684; GFX12: ; %bb.0: ; %entry685; GFX12-NEXT: s_clause 0x1686; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x34687; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24688; GFX12-NEXT: s_wait_kmcnt 0x0689; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s7690; GFX12-NEXT: v_mov_b32_e32 v0, s6691; GFX12-NEXT: global_atomic_and_b64 v[0:1], v2, v[0:1], s[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV692; GFX12-NEXT: s_wait_loadcnt 0x0693; GFX12-NEXT: global_inv scope:SCOPE_DEV694; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]695; GFX12-NEXT: s_endpgm696entry:697 %gep = getelementptr i64, ptr addrspace(1) %out, i64 4698 %tmp0 = atomicrmw volatile and ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0699 store i64 %tmp0, ptr addrspace(1) %out2700 ret void701}702 703define amdgpu_kernel void @atomic_and_i64_addr64_offset(ptr addrspace(1) %out, i64 %in, i64 %index) {704; CI-LABEL: atomic_and_i64_addr64_offset:705; CI: ; %bb.0: ; %entry706; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9707; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd708; CI-NEXT: s_waitcnt lgkmcnt(0)709; CI-NEXT: v_mov_b32_e32 v0, s2710; CI-NEXT: s_lshl_b64 s[4:5], s[4:5], 3711; CI-NEXT: v_mov_b32_e32 v2, s4712; CI-NEXT: v_mov_b32_e32 v1, s3713; CI-NEXT: s_mov_b32 s3, 0xf000714; CI-NEXT: s_mov_b32 s2, 0715; CI-NEXT: v_mov_b32_e32 v3, s5716; CI-NEXT: buffer_atomic_and_x2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:32717; CI-NEXT: s_waitcnt vmcnt(0)718; CI-NEXT: buffer_wbinvl1_vol719; CI-NEXT: s_endpgm720;721; VI-LABEL: atomic_and_i64_addr64_offset:722; VI: ; %bb.0: ; %entry723; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24724; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34725; VI-NEXT: s_waitcnt lgkmcnt(0)726; VI-NEXT: v_mov_b32_e32 v0, s2727; VI-NEXT: v_mov_b32_e32 v1, s3728; VI-NEXT: s_lshl_b64 s[2:3], s[4:5], 3729; VI-NEXT: s_add_u32 s0, s0, s2730; VI-NEXT: s_addc_u32 s1, s1, s3731; VI-NEXT: s_add_u32 s0, s0, 32732; VI-NEXT: s_addc_u32 s1, s1, 0733; VI-NEXT: v_mov_b32_e32 v3, s1734; VI-NEXT: v_mov_b32_e32 v2, s0735; VI-NEXT: flat_atomic_and_x2 v[2:3], v[0:1]736; VI-NEXT: s_waitcnt vmcnt(0)737; VI-NEXT: buffer_wbinvl1_vol738; VI-NEXT: s_endpgm739;740; GFX9-LABEL: atomic_and_i64_addr64_offset:741; GFX9: ; %bb.0: ; %entry742; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24743; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34744; GFX9-NEXT: v_mov_b32_e32 v2, 0745; GFX9-NEXT: s_waitcnt lgkmcnt(0)746; GFX9-NEXT: v_mov_b32_e32 v0, s2747; GFX9-NEXT: v_mov_b32_e32 v1, s3748; GFX9-NEXT: s_lshl_b64 s[2:3], s[6:7], 3749; GFX9-NEXT: s_add_u32 s0, s0, s2750; GFX9-NEXT: s_addc_u32 s1, s1, s3751; GFX9-NEXT: global_atomic_and_x2 v2, v[0:1], s[0:1] offset:32752; GFX9-NEXT: s_waitcnt vmcnt(0)753; GFX9-NEXT: buffer_wbinvl1_vol754; GFX9-NEXT: s_endpgm755;756; GFX12-LABEL: atomic_and_i64_addr64_offset:757; GFX12: ; %bb.0: ; %entry758; GFX12-NEXT: s_clause 0x1759; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24760; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x34761; GFX12-NEXT: v_mov_b32_e32 v2, 0762; GFX12-NEXT: s_wait_kmcnt 0x0763; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3764; GFX12-NEXT: s_lshl_b64 s[2:3], s[4:5], 3765; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)766; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]767; GFX12-NEXT: global_atomic_and_b64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_DEV768; GFX12-NEXT: s_wait_storecnt 0x0769; GFX12-NEXT: global_inv scope:SCOPE_DEV770; GFX12-NEXT: s_endpgm771entry:772 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index773 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4774 %tmp0 = atomicrmw volatile and ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0775 ret void776}777 778define amdgpu_kernel void @atomic_and_i64_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {779; CI-LABEL: atomic_and_i64_ret_addr64_offset:780; CI: ; %bb.0: ; %entry781; CI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9782; CI-NEXT: s_mov_b32 s11, 0xf000783; CI-NEXT: s_mov_b32 s10, -1784; CI-NEXT: s_waitcnt lgkmcnt(0)785; CI-NEXT: v_mov_b32_e32 v0, s4786; CI-NEXT: v_mov_b32_e32 v1, s5787; CI-NEXT: s_lshl_b64 s[4:5], s[6:7], 3788; CI-NEXT: v_mov_b32_e32 v2, s4789; CI-NEXT: s_mov_b32 s8, s2790; CI-NEXT: s_mov_b32 s9, s3791; CI-NEXT: s_mov_b32 s2, 0792; CI-NEXT: s_mov_b32 s3, s11793; CI-NEXT: v_mov_b32_e32 v3, s5794; CI-NEXT: buffer_atomic_and_x2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:32 glc795; CI-NEXT: s_waitcnt vmcnt(0)796; CI-NEXT: buffer_wbinvl1_vol797; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0798; CI-NEXT: s_endpgm799;800; VI-LABEL: atomic_and_i64_ret_addr64_offset:801; VI: ; %bb.0: ; %entry802; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24803; VI-NEXT: s_waitcnt lgkmcnt(0)804; VI-NEXT: v_mov_b32_e32 v0, s4805; VI-NEXT: v_mov_b32_e32 v1, s5806; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 3807; VI-NEXT: s_add_u32 s0, s0, s4808; VI-NEXT: s_addc_u32 s1, s1, s5809; VI-NEXT: s_add_u32 s0, s0, 32810; VI-NEXT: s_addc_u32 s1, s1, 0811; VI-NEXT: v_mov_b32_e32 v3, s1812; VI-NEXT: v_mov_b32_e32 v2, s0813; VI-NEXT: flat_atomic_and_x2 v[0:1], v[2:3], v[0:1] glc814; VI-NEXT: s_waitcnt vmcnt(0)815; VI-NEXT: buffer_wbinvl1_vol816; VI-NEXT: s_mov_b32 s7, 0xf000817; VI-NEXT: s_mov_b32 s6, -1818; VI-NEXT: s_mov_b32 s4, s2819; VI-NEXT: s_mov_b32 s5, s3820; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0821; VI-NEXT: s_endpgm822;823; GFX9-LABEL: atomic_and_i64_ret_addr64_offset:824; GFX9: ; %bb.0: ; %entry825; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24826; GFX9-NEXT: v_mov_b32_e32 v2, 0827; GFX9-NEXT: s_waitcnt lgkmcnt(0)828; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 3829; GFX9-NEXT: s_add_u32 s0, s8, s0830; GFX9-NEXT: v_mov_b32_e32 v0, s12831; GFX9-NEXT: v_mov_b32_e32 v1, s13832; GFX9-NEXT: s_addc_u32 s1, s9, s1833; GFX9-NEXT: global_atomic_and_x2 v[0:1], v2, v[0:1], s[0:1] offset:32 glc834; GFX9-NEXT: s_waitcnt vmcnt(0)835; GFX9-NEXT: buffer_wbinvl1_vol836; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]837; GFX9-NEXT: s_endpgm838;839; GFX12-LABEL: atomic_and_i64_ret_addr64_offset:840; GFX12: ; %bb.0: ; %entry841; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x24842; GFX12-NEXT: s_wait_kmcnt 0x0843; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s5844; GFX12-NEXT: v_mov_b32_e32 v0, s4845; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 3846; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)847; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]848; GFX12-NEXT: global_atomic_and_b64 v[0:1], v2, v[0:1], s[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV849; GFX12-NEXT: s_wait_loadcnt 0x0850; GFX12-NEXT: global_inv scope:SCOPE_DEV851; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]852; GFX12-NEXT: s_endpgm853entry:854 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index855 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4856 %tmp0 = atomicrmw volatile and ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0857 store i64 %tmp0, ptr addrspace(1) %out2858 ret void859}860 861define amdgpu_kernel void @atomic_and_i64(ptr addrspace(1) %out, i64 %in) {862; CI-LABEL: atomic_and_i64:863; CI: ; %bb.0: ; %entry864; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9865; CI-NEXT: s_mov_b32 s7, 0xf000866; CI-NEXT: s_mov_b32 s6, -1867; CI-NEXT: s_waitcnt lgkmcnt(0)868; CI-NEXT: s_mov_b32 s4, s0869; CI-NEXT: s_mov_b32 s5, s1870; CI-NEXT: v_mov_b32_e32 v0, s2871; CI-NEXT: v_mov_b32_e32 v1, s3872; CI-NEXT: buffer_atomic_and_x2 v[0:1], off, s[4:7], 0873; CI-NEXT: s_waitcnt vmcnt(0)874; CI-NEXT: buffer_wbinvl1_vol875; CI-NEXT: s_endpgm876;877; VI-LABEL: atomic_and_i64:878; VI: ; %bb.0: ; %entry879; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24880; VI-NEXT: s_mov_b32 s7, 0xf000881; VI-NEXT: s_mov_b32 s6, -1882; VI-NEXT: s_waitcnt lgkmcnt(0)883; VI-NEXT: s_mov_b32 s4, s0884; VI-NEXT: s_mov_b32 s5, s1885; VI-NEXT: v_mov_b32_e32 v0, s2886; VI-NEXT: v_mov_b32_e32 v1, s3887; VI-NEXT: buffer_atomic_and_x2 v[0:1], off, s[4:7], 0888; VI-NEXT: s_waitcnt vmcnt(0)889; VI-NEXT: buffer_wbinvl1_vol890; VI-NEXT: s_endpgm891;892; GFX9-LABEL: atomic_and_i64:893; GFX9: ; %bb.0: ; %entry894; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24895; GFX9-NEXT: v_mov_b32_e32 v2, 0896; GFX9-NEXT: s_waitcnt lgkmcnt(0)897; GFX9-NEXT: v_mov_b32_e32 v0, s2898; GFX9-NEXT: v_mov_b32_e32 v1, s3899; GFX9-NEXT: global_atomic_and_x2 v2, v[0:1], s[0:1]900; GFX9-NEXT: s_waitcnt vmcnt(0)901; GFX9-NEXT: buffer_wbinvl1_vol902; GFX9-NEXT: s_endpgm903;904; GFX12-LABEL: atomic_and_i64:905; GFX12: ; %bb.0: ; %entry906; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24907; GFX12-NEXT: s_wait_kmcnt 0x0908; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3909; GFX12-NEXT: v_mov_b32_e32 v0, s2910; GFX12-NEXT: global_atomic_and_b64 v2, v[0:1], s[0:1] scope:SCOPE_DEV911; GFX12-NEXT: s_wait_storecnt 0x0912; GFX12-NEXT: global_inv scope:SCOPE_DEV913; GFX12-NEXT: s_endpgm914entry:915 %tmp0 = atomicrmw volatile and ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0916 ret void917}918 919define amdgpu_kernel void @atomic_and_i64_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in) {920; CI-LABEL: atomic_and_i64_ret:921; CI: ; %bb.0: ; %entry922; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9923; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd924; CI-NEXT: s_mov_b32 s7, 0xf000925; CI-NEXT: s_mov_b32 s6, -1926; CI-NEXT: s_waitcnt lgkmcnt(0)927; CI-NEXT: s_mov_b32 s4, s0928; CI-NEXT: s_mov_b32 s5, s1929; CI-NEXT: v_mov_b32_e32 v0, s8930; CI-NEXT: v_mov_b32_e32 v1, s9931; CI-NEXT: buffer_atomic_and_x2 v[0:1], off, s[4:7], 0 glc932; CI-NEXT: s_waitcnt vmcnt(0)933; CI-NEXT: buffer_wbinvl1_vol934; CI-NEXT: s_mov_b32 s4, s2935; CI-NEXT: s_mov_b32 s5, s3936; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0937; CI-NEXT: s_endpgm938;939; VI-LABEL: atomic_and_i64_ret:940; VI: ; %bb.0: ; %entry941; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24942; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34943; VI-NEXT: s_mov_b32 s7, 0xf000944; VI-NEXT: s_mov_b32 s6, -1945; VI-NEXT: s_waitcnt lgkmcnt(0)946; VI-NEXT: s_mov_b32 s4, s0947; VI-NEXT: s_mov_b32 s5, s1948; VI-NEXT: v_mov_b32_e32 v0, s8949; VI-NEXT: v_mov_b32_e32 v1, s9950; VI-NEXT: buffer_atomic_and_x2 v[0:1], off, s[4:7], 0 glc951; VI-NEXT: s_waitcnt vmcnt(0)952; VI-NEXT: buffer_wbinvl1_vol953; VI-NEXT: s_mov_b32 s4, s2954; VI-NEXT: s_mov_b32 s5, s3955; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0956; VI-NEXT: s_endpgm957;958; GFX9-LABEL: atomic_and_i64_ret:959; GFX9: ; %bb.0: ; %entry960; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34961; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24962; GFX9-NEXT: v_mov_b32_e32 v2, 0963; GFX9-NEXT: s_waitcnt lgkmcnt(0)964; GFX9-NEXT: v_mov_b32_e32 v0, s6965; GFX9-NEXT: v_mov_b32_e32 v1, s7966; GFX9-NEXT: global_atomic_and_x2 v[0:1], v2, v[0:1], s[0:1] glc967; GFX9-NEXT: s_waitcnt vmcnt(0)968; GFX9-NEXT: buffer_wbinvl1_vol969; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]970; GFX9-NEXT: s_endpgm971;972; GFX12-LABEL: atomic_and_i64_ret:973; GFX12: ; %bb.0: ; %entry974; GFX12-NEXT: s_clause 0x1975; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x34976; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24977; GFX12-NEXT: s_wait_kmcnt 0x0978; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s7979; GFX12-NEXT: v_mov_b32_e32 v0, s6980; GFX12-NEXT: global_atomic_and_b64 v[0:1], v2, v[0:1], s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV981; GFX12-NEXT: s_wait_loadcnt 0x0982; GFX12-NEXT: global_inv scope:SCOPE_DEV983; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]984; GFX12-NEXT: s_endpgm985entry:986 %tmp0 = atomicrmw volatile and ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0987 store i64 %tmp0, ptr addrspace(1) %out2988 ret void989}990 991define amdgpu_kernel void @atomic_and_i64_addr64(ptr addrspace(1) %out, i64 %in, i64 %index) {992; CI-LABEL: atomic_and_i64_addr64:993; CI: ; %bb.0: ; %entry994; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9995; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd996; CI-NEXT: s_waitcnt lgkmcnt(0)997; CI-NEXT: v_mov_b32_e32 v0, s2998; CI-NEXT: s_lshl_b64 s[4:5], s[4:5], 3999; CI-NEXT: v_mov_b32_e32 v2, s41000; CI-NEXT: v_mov_b32_e32 v1, s31001; CI-NEXT: s_mov_b32 s3, 0xf0001002; CI-NEXT: s_mov_b32 s2, 01003; CI-NEXT: v_mov_b32_e32 v3, s51004; CI-NEXT: buffer_atomic_and_x2 v[0:1], v[2:3], s[0:3], 0 addr641005; CI-NEXT: s_waitcnt vmcnt(0)1006; CI-NEXT: buffer_wbinvl1_vol1007; CI-NEXT: s_endpgm1008;1009; VI-LABEL: atomic_and_i64_addr64:1010; VI: ; %bb.0: ; %entry1011; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241012; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341013; VI-NEXT: s_waitcnt lgkmcnt(0)1014; VI-NEXT: v_mov_b32_e32 v0, s21015; VI-NEXT: v_mov_b32_e32 v1, s31016; VI-NEXT: s_lshl_b64 s[2:3], s[4:5], 31017; VI-NEXT: s_add_u32 s0, s0, s21018; VI-NEXT: s_addc_u32 s1, s1, s31019; VI-NEXT: v_mov_b32_e32 v3, s11020; VI-NEXT: v_mov_b32_e32 v2, s01021; VI-NEXT: flat_atomic_and_x2 v[2:3], v[0:1]1022; VI-NEXT: s_waitcnt vmcnt(0)1023; VI-NEXT: buffer_wbinvl1_vol1024; VI-NEXT: s_endpgm1025;1026; GFX9-LABEL: atomic_and_i64_addr64:1027; GFX9: ; %bb.0: ; %entry1028; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241029; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341030; GFX9-NEXT: v_mov_b32_e32 v2, 01031; GFX9-NEXT: s_waitcnt lgkmcnt(0)1032; GFX9-NEXT: v_mov_b32_e32 v0, s21033; GFX9-NEXT: v_mov_b32_e32 v1, s31034; GFX9-NEXT: s_lshl_b64 s[2:3], s[6:7], 31035; GFX9-NEXT: s_add_u32 s0, s0, s21036; GFX9-NEXT: s_addc_u32 s1, s1, s31037; GFX9-NEXT: global_atomic_and_x2 v2, v[0:1], s[0:1]1038; GFX9-NEXT: s_waitcnt vmcnt(0)1039; GFX9-NEXT: buffer_wbinvl1_vol1040; GFX9-NEXT: s_endpgm1041;1042; GFX12-LABEL: atomic_and_i64_addr64:1043; GFX12: ; %bb.0: ; %entry1044; GFX12-NEXT: s_clause 0x11045; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x241046; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x341047; GFX12-NEXT: v_mov_b32_e32 v2, 01048; GFX12-NEXT: s_wait_kmcnt 0x01049; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s31050; GFX12-NEXT: s_lshl_b64 s[2:3], s[4:5], 31051; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)1052; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]1053; GFX12-NEXT: global_atomic_and_b64 v2, v[0:1], s[0:1] scope:SCOPE_DEV1054; GFX12-NEXT: s_wait_storecnt 0x01055; GFX12-NEXT: global_inv scope:SCOPE_DEV1056; GFX12-NEXT: s_endpgm1057entry:1058 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index1059 %tmp0 = atomicrmw volatile and ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01060 ret void1061}1062 1063define amdgpu_kernel void @atomic_and_i64_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {1064; CI-LABEL: atomic_and_i64_ret_addr64:1065; CI: ; %bb.0: ; %entry1066; CI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x91067; CI-NEXT: s_mov_b32 s11, 0xf0001068; CI-NEXT: s_mov_b32 s10, -11069; CI-NEXT: s_waitcnt lgkmcnt(0)1070; CI-NEXT: v_mov_b32_e32 v0, s41071; CI-NEXT: v_mov_b32_e32 v1, s51072; CI-NEXT: s_lshl_b64 s[4:5], s[6:7], 31073; CI-NEXT: v_mov_b32_e32 v2, s41074; CI-NEXT: s_mov_b32 s8, s21075; CI-NEXT: s_mov_b32 s9, s31076; CI-NEXT: s_mov_b32 s2, 01077; CI-NEXT: s_mov_b32 s3, s111078; CI-NEXT: v_mov_b32_e32 v3, s51079; CI-NEXT: buffer_atomic_and_x2 v[0:1], v[2:3], s[0:3], 0 addr64 glc1080; CI-NEXT: s_waitcnt vmcnt(0)1081; CI-NEXT: buffer_wbinvl1_vol1082; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 01083; CI-NEXT: s_endpgm1084;1085; VI-LABEL: atomic_and_i64_ret_addr64:1086; VI: ; %bb.0: ; %entry1087; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x241088; VI-NEXT: s_waitcnt lgkmcnt(0)1089; VI-NEXT: v_mov_b32_e32 v0, s41090; VI-NEXT: v_mov_b32_e32 v1, s51091; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 31092; VI-NEXT: s_add_u32 s0, s0, s41093; VI-NEXT: s_addc_u32 s1, s1, s51094; VI-NEXT: v_mov_b32_e32 v3, s11095; VI-NEXT: v_mov_b32_e32 v2, s01096; VI-NEXT: flat_atomic_and_x2 v[0:1], v[2:3], v[0:1] glc1097; VI-NEXT: s_waitcnt vmcnt(0)1098; VI-NEXT: buffer_wbinvl1_vol1099; VI-NEXT: s_mov_b32 s7, 0xf0001100; VI-NEXT: s_mov_b32 s6, -11101; VI-NEXT: s_mov_b32 s4, s21102; VI-NEXT: s_mov_b32 s5, s31103; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 01104; VI-NEXT: s_endpgm1105;1106; GFX9-LABEL: atomic_and_i64_ret_addr64:1107; GFX9: ; %bb.0: ; %entry1108; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x241109; GFX9-NEXT: v_mov_b32_e32 v2, 01110; GFX9-NEXT: s_waitcnt lgkmcnt(0)1111; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 31112; GFX9-NEXT: s_add_u32 s0, s8, s01113; GFX9-NEXT: v_mov_b32_e32 v0, s121114; GFX9-NEXT: v_mov_b32_e32 v1, s131115; GFX9-NEXT: s_addc_u32 s1, s9, s11116; GFX9-NEXT: global_atomic_and_x2 v[0:1], v2, v[0:1], s[0:1] glc1117; GFX9-NEXT: s_waitcnt vmcnt(0)1118; GFX9-NEXT: buffer_wbinvl1_vol1119; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]1120; GFX9-NEXT: s_endpgm1121;1122; GFX12-LABEL: atomic_and_i64_ret_addr64:1123; GFX12: ; %bb.0: ; %entry1124; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x241125; GFX12-NEXT: s_wait_kmcnt 0x01126; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s51127; GFX12-NEXT: v_mov_b32_e32 v0, s41128; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 31129; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)1130; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]1131; GFX12-NEXT: global_atomic_and_b64 v[0:1], v2, v[0:1], s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV1132; GFX12-NEXT: s_wait_loadcnt 0x01133; GFX12-NEXT: global_inv scope:SCOPE_DEV1134; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]1135; GFX12-NEXT: s_endpgm1136entry:1137 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index1138 %tmp0 = atomicrmw volatile and ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01139 store i64 %tmp0, ptr addrspace(1) %out21140 ret void1141}1142 1143define amdgpu_kernel void @atomic_sub_i64_offset(ptr addrspace(1) %out, i64 %in) {1144; CI-LABEL: atomic_sub_i64_offset:1145; CI: ; %bb.0: ; %entry1146; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91147; CI-NEXT: s_waitcnt lgkmcnt(0)1148; CI-NEXT: v_mov_b32_e32 v0, s21149; CI-NEXT: v_mov_b32_e32 v1, s31150; CI-NEXT: s_mov_b32 s3, 0xf0001151; CI-NEXT: s_mov_b32 s2, -11152; CI-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[0:3], 0 offset:321153; CI-NEXT: s_waitcnt vmcnt(0)1154; CI-NEXT: buffer_wbinvl1_vol1155; CI-NEXT: s_endpgm1156;1157; VI-LABEL: atomic_sub_i64_offset:1158; VI: ; %bb.0: ; %entry1159; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241160; VI-NEXT: s_waitcnt lgkmcnt(0)1161; VI-NEXT: v_mov_b32_e32 v0, s21162; VI-NEXT: v_mov_b32_e32 v1, s31163; VI-NEXT: s_mov_b32 s3, 0xf0001164; VI-NEXT: s_mov_b32 s2, -11165; VI-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[0:3], 0 offset:321166; VI-NEXT: s_waitcnt vmcnt(0)1167; VI-NEXT: buffer_wbinvl1_vol1168; VI-NEXT: s_endpgm1169;1170; GFX9-LABEL: atomic_sub_i64_offset:1171; GFX9: ; %bb.0: ; %entry1172; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241173; GFX9-NEXT: v_mov_b32_e32 v2, 01174; GFX9-NEXT: s_waitcnt lgkmcnt(0)1175; GFX9-NEXT: v_mov_b32_e32 v0, s21176; GFX9-NEXT: v_mov_b32_e32 v1, s31177; GFX9-NEXT: global_atomic_sub_x2 v2, v[0:1], s[0:1] offset:321178; GFX9-NEXT: s_waitcnt vmcnt(0)1179; GFX9-NEXT: buffer_wbinvl1_vol1180; GFX9-NEXT: s_endpgm1181;1182; GFX12-LABEL: atomic_sub_i64_offset:1183; GFX12: ; %bb.0: ; %entry1184; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x241185; GFX12-NEXT: s_wait_kmcnt 0x01186; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s31187; GFX12-NEXT: v_mov_b32_e32 v0, s21188; GFX12-NEXT: global_atomic_sub_u64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_DEV1189; GFX12-NEXT: s_wait_storecnt 0x01190; GFX12-NEXT: global_inv scope:SCOPE_DEV1191; GFX12-NEXT: s_endpgm1192entry:1193 %gep = getelementptr i64, ptr addrspace(1) %out, i64 41194 %tmp0 = atomicrmw volatile sub ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01195 ret void1196}1197 1198define amdgpu_kernel void @atomic_sub_i64_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in) {1199; CI-LABEL: atomic_sub_i64_ret_offset:1200; CI: ; %bb.0: ; %entry1201; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91202; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd1203; CI-NEXT: s_mov_b32 s7, 0xf0001204; CI-NEXT: s_mov_b32 s6, -11205; CI-NEXT: s_waitcnt lgkmcnt(0)1206; CI-NEXT: s_mov_b32 s4, s21207; CI-NEXT: s_mov_b32 s5, s31208; CI-NEXT: v_mov_b32_e32 v0, s81209; CI-NEXT: v_mov_b32_e32 v1, s91210; CI-NEXT: s_mov_b32 s2, s61211; CI-NEXT: s_mov_b32 s3, s71212; CI-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[0:3], 0 offset:32 glc1213; CI-NEXT: s_waitcnt vmcnt(0)1214; CI-NEXT: buffer_wbinvl1_vol1215; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 01216; CI-NEXT: s_endpgm1217;1218; VI-LABEL: atomic_sub_i64_ret_offset:1219; VI: ; %bb.0: ; %entry1220; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241221; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x341222; VI-NEXT: s_mov_b32 s7, 0xf0001223; VI-NEXT: s_mov_b32 s6, -11224; VI-NEXT: s_waitcnt lgkmcnt(0)1225; VI-NEXT: s_mov_b32 s4, s21226; VI-NEXT: s_mov_b32 s5, s31227; VI-NEXT: v_mov_b32_e32 v0, s81228; VI-NEXT: v_mov_b32_e32 v1, s91229; VI-NEXT: s_mov_b32 s2, s61230; VI-NEXT: s_mov_b32 s3, s71231; VI-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[0:3], 0 offset:32 glc1232; VI-NEXT: s_waitcnt vmcnt(0)1233; VI-NEXT: buffer_wbinvl1_vol1234; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 01235; VI-NEXT: s_endpgm1236;1237; GFX9-LABEL: atomic_sub_i64_ret_offset:1238; GFX9: ; %bb.0: ; %entry1239; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341240; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241241; GFX9-NEXT: v_mov_b32_e32 v2, 01242; GFX9-NEXT: s_waitcnt lgkmcnt(0)1243; GFX9-NEXT: v_mov_b32_e32 v0, s61244; GFX9-NEXT: v_mov_b32_e32 v1, s71245; GFX9-NEXT: global_atomic_sub_x2 v[0:1], v2, v[0:1], s[0:1] offset:32 glc1246; GFX9-NEXT: s_waitcnt vmcnt(0)1247; GFX9-NEXT: buffer_wbinvl1_vol1248; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]1249; GFX9-NEXT: s_endpgm1250;1251; GFX12-LABEL: atomic_sub_i64_ret_offset:1252; GFX12: ; %bb.0: ; %entry1253; GFX12-NEXT: s_clause 0x11254; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x341255; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x241256; GFX12-NEXT: s_wait_kmcnt 0x01257; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s71258; GFX12-NEXT: v_mov_b32_e32 v0, s61259; GFX12-NEXT: global_atomic_sub_u64 v[0:1], v2, v[0:1], s[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV1260; GFX12-NEXT: s_wait_loadcnt 0x01261; GFX12-NEXT: global_inv scope:SCOPE_DEV1262; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]1263; GFX12-NEXT: s_endpgm1264entry:1265 %gep = getelementptr i64, ptr addrspace(1) %out, i64 41266 %tmp0 = atomicrmw volatile sub ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01267 store i64 %tmp0, ptr addrspace(1) %out21268 ret void1269}1270 1271define amdgpu_kernel void @atomic_sub_i64_addr64_offset(ptr addrspace(1) %out, i64 %in, i64 %index) {1272; CI-LABEL: atomic_sub_i64_addr64_offset:1273; CI: ; %bb.0: ; %entry1274; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91275; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd1276; CI-NEXT: s_waitcnt lgkmcnt(0)1277; CI-NEXT: v_mov_b32_e32 v0, s21278; CI-NEXT: s_lshl_b64 s[4:5], s[4:5], 31279; CI-NEXT: v_mov_b32_e32 v2, s41280; CI-NEXT: v_mov_b32_e32 v1, s31281; CI-NEXT: s_mov_b32 s3, 0xf0001282; CI-NEXT: s_mov_b32 s2, 01283; CI-NEXT: v_mov_b32_e32 v3, s51284; CI-NEXT: buffer_atomic_sub_x2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:321285; CI-NEXT: s_waitcnt vmcnt(0)1286; CI-NEXT: buffer_wbinvl1_vol1287; CI-NEXT: s_endpgm1288;1289; VI-LABEL: atomic_sub_i64_addr64_offset:1290; VI: ; %bb.0: ; %entry1291; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241292; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341293; VI-NEXT: s_waitcnt lgkmcnt(0)1294; VI-NEXT: v_mov_b32_e32 v0, s21295; VI-NEXT: v_mov_b32_e32 v1, s31296; VI-NEXT: s_lshl_b64 s[2:3], s[4:5], 31297; VI-NEXT: s_add_u32 s0, s0, s21298; VI-NEXT: s_addc_u32 s1, s1, s31299; VI-NEXT: s_add_u32 s0, s0, 321300; VI-NEXT: s_addc_u32 s1, s1, 01301; VI-NEXT: v_mov_b32_e32 v3, s11302; VI-NEXT: v_mov_b32_e32 v2, s01303; VI-NEXT: flat_atomic_sub_x2 v[2:3], v[0:1]1304; VI-NEXT: s_waitcnt vmcnt(0)1305; VI-NEXT: buffer_wbinvl1_vol1306; VI-NEXT: s_endpgm1307;1308; GFX9-LABEL: atomic_sub_i64_addr64_offset:1309; GFX9: ; %bb.0: ; %entry1310; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241311; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341312; GFX9-NEXT: v_mov_b32_e32 v2, 01313; GFX9-NEXT: s_waitcnt lgkmcnt(0)1314; GFX9-NEXT: v_mov_b32_e32 v0, s21315; GFX9-NEXT: v_mov_b32_e32 v1, s31316; GFX9-NEXT: s_lshl_b64 s[2:3], s[6:7], 31317; GFX9-NEXT: s_add_u32 s0, s0, s21318; GFX9-NEXT: s_addc_u32 s1, s1, s31319; GFX9-NEXT: global_atomic_sub_x2 v2, v[0:1], s[0:1] offset:321320; GFX9-NEXT: s_waitcnt vmcnt(0)1321; GFX9-NEXT: buffer_wbinvl1_vol1322; GFX9-NEXT: s_endpgm1323;1324; GFX12-LABEL: atomic_sub_i64_addr64_offset:1325; GFX12: ; %bb.0: ; %entry1326; GFX12-NEXT: s_clause 0x11327; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x241328; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x341329; GFX12-NEXT: v_mov_b32_e32 v2, 01330; GFX12-NEXT: s_wait_kmcnt 0x01331; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s31332; GFX12-NEXT: s_lshl_b64 s[2:3], s[4:5], 31333; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)1334; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]1335; GFX12-NEXT: global_atomic_sub_u64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_DEV1336; GFX12-NEXT: s_wait_storecnt 0x01337; GFX12-NEXT: global_inv scope:SCOPE_DEV1338; GFX12-NEXT: s_endpgm1339entry:1340 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index1341 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 41342 %tmp0 = atomicrmw volatile sub ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01343 ret void1344}1345 1346define amdgpu_kernel void @atomic_sub_i64_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {1347; CI-LABEL: atomic_sub_i64_ret_addr64_offset:1348; CI: ; %bb.0: ; %entry1349; CI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x91350; CI-NEXT: s_mov_b32 s11, 0xf0001351; CI-NEXT: s_mov_b32 s10, -11352; CI-NEXT: s_waitcnt lgkmcnt(0)1353; CI-NEXT: v_mov_b32_e32 v0, s41354; CI-NEXT: v_mov_b32_e32 v1, s51355; CI-NEXT: s_lshl_b64 s[4:5], s[6:7], 31356; CI-NEXT: v_mov_b32_e32 v2, s41357; CI-NEXT: s_mov_b32 s8, s21358; CI-NEXT: s_mov_b32 s9, s31359; CI-NEXT: s_mov_b32 s2, 01360; CI-NEXT: s_mov_b32 s3, s111361; CI-NEXT: v_mov_b32_e32 v3, s51362; CI-NEXT: buffer_atomic_sub_x2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:32 glc1363; CI-NEXT: s_waitcnt vmcnt(0)1364; CI-NEXT: buffer_wbinvl1_vol1365; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 01366; CI-NEXT: s_endpgm1367;1368; VI-LABEL: atomic_sub_i64_ret_addr64_offset:1369; VI: ; %bb.0: ; %entry1370; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x241371; VI-NEXT: s_waitcnt lgkmcnt(0)1372; VI-NEXT: v_mov_b32_e32 v0, s41373; VI-NEXT: v_mov_b32_e32 v1, s51374; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 31375; VI-NEXT: s_add_u32 s0, s0, s41376; VI-NEXT: s_addc_u32 s1, s1, s51377; VI-NEXT: s_add_u32 s0, s0, 321378; VI-NEXT: s_addc_u32 s1, s1, 01379; VI-NEXT: v_mov_b32_e32 v3, s11380; VI-NEXT: v_mov_b32_e32 v2, s01381; VI-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3], v[0:1] glc1382; VI-NEXT: s_waitcnt vmcnt(0)1383; VI-NEXT: buffer_wbinvl1_vol1384; VI-NEXT: s_mov_b32 s7, 0xf0001385; VI-NEXT: s_mov_b32 s6, -11386; VI-NEXT: s_mov_b32 s4, s21387; VI-NEXT: s_mov_b32 s5, s31388; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 01389; VI-NEXT: s_endpgm1390;1391; GFX9-LABEL: atomic_sub_i64_ret_addr64_offset:1392; GFX9: ; %bb.0: ; %entry1393; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x241394; GFX9-NEXT: v_mov_b32_e32 v2, 01395; GFX9-NEXT: s_waitcnt lgkmcnt(0)1396; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 31397; GFX9-NEXT: s_add_u32 s0, s8, s01398; GFX9-NEXT: v_mov_b32_e32 v0, s121399; GFX9-NEXT: v_mov_b32_e32 v1, s131400; GFX9-NEXT: s_addc_u32 s1, s9, s11401; GFX9-NEXT: global_atomic_sub_x2 v[0:1], v2, v[0:1], s[0:1] offset:32 glc1402; GFX9-NEXT: s_waitcnt vmcnt(0)1403; GFX9-NEXT: buffer_wbinvl1_vol1404; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]1405; GFX9-NEXT: s_endpgm1406;1407; GFX12-LABEL: atomic_sub_i64_ret_addr64_offset:1408; GFX12: ; %bb.0: ; %entry1409; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x241410; GFX12-NEXT: s_wait_kmcnt 0x01411; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s51412; GFX12-NEXT: v_mov_b32_e32 v0, s41413; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 31414; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)1415; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]1416; GFX12-NEXT: global_atomic_sub_u64 v[0:1], v2, v[0:1], s[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV1417; GFX12-NEXT: s_wait_loadcnt 0x01418; GFX12-NEXT: global_inv scope:SCOPE_DEV1419; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]1420; GFX12-NEXT: s_endpgm1421entry:1422 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index1423 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 41424 %tmp0 = atomicrmw volatile sub ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01425 store i64 %tmp0, ptr addrspace(1) %out21426 ret void1427}1428 1429define amdgpu_kernel void @atomic_sub_i64(ptr addrspace(1) %out, i64 %in) {1430; CI-LABEL: atomic_sub_i64:1431; CI: ; %bb.0: ; %entry1432; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91433; CI-NEXT: s_mov_b32 s7, 0xf0001434; CI-NEXT: s_mov_b32 s6, -11435; CI-NEXT: s_waitcnt lgkmcnt(0)1436; CI-NEXT: s_mov_b32 s4, s01437; CI-NEXT: s_mov_b32 s5, s11438; CI-NEXT: v_mov_b32_e32 v0, s21439; CI-NEXT: v_mov_b32_e32 v1, s31440; CI-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[4:7], 01441; CI-NEXT: s_waitcnt vmcnt(0)1442; CI-NEXT: buffer_wbinvl1_vol1443; CI-NEXT: s_endpgm1444;1445; VI-LABEL: atomic_sub_i64:1446; VI: ; %bb.0: ; %entry1447; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241448; VI-NEXT: s_mov_b32 s7, 0xf0001449; VI-NEXT: s_mov_b32 s6, -11450; VI-NEXT: s_waitcnt lgkmcnt(0)1451; VI-NEXT: s_mov_b32 s4, s01452; VI-NEXT: s_mov_b32 s5, s11453; VI-NEXT: v_mov_b32_e32 v0, s21454; VI-NEXT: v_mov_b32_e32 v1, s31455; VI-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[4:7], 01456; VI-NEXT: s_waitcnt vmcnt(0)1457; VI-NEXT: buffer_wbinvl1_vol1458; VI-NEXT: s_endpgm1459;1460; GFX9-LABEL: atomic_sub_i64:1461; GFX9: ; %bb.0: ; %entry1462; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241463; GFX9-NEXT: v_mov_b32_e32 v2, 01464; GFX9-NEXT: s_waitcnt lgkmcnt(0)1465; GFX9-NEXT: v_mov_b32_e32 v0, s21466; GFX9-NEXT: v_mov_b32_e32 v1, s31467; GFX9-NEXT: global_atomic_sub_x2 v2, v[0:1], s[0:1]1468; GFX9-NEXT: s_waitcnt vmcnt(0)1469; GFX9-NEXT: buffer_wbinvl1_vol1470; GFX9-NEXT: s_endpgm1471;1472; GFX12-LABEL: atomic_sub_i64:1473; GFX12: ; %bb.0: ; %entry1474; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x241475; GFX12-NEXT: s_wait_kmcnt 0x01476; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s31477; GFX12-NEXT: v_mov_b32_e32 v0, s21478; GFX12-NEXT: global_atomic_sub_u64 v2, v[0:1], s[0:1] scope:SCOPE_DEV1479; GFX12-NEXT: s_wait_storecnt 0x01480; GFX12-NEXT: global_inv scope:SCOPE_DEV1481; GFX12-NEXT: s_endpgm1482entry:1483 %tmp0 = atomicrmw volatile sub ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01484 ret void1485}1486 1487define amdgpu_kernel void @atomic_sub_i64_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in) {1488; CI-LABEL: atomic_sub_i64_ret:1489; CI: ; %bb.0: ; %entry1490; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91491; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd1492; CI-NEXT: s_mov_b32 s7, 0xf0001493; CI-NEXT: s_mov_b32 s6, -11494; CI-NEXT: s_waitcnt lgkmcnt(0)1495; CI-NEXT: s_mov_b32 s4, s01496; CI-NEXT: s_mov_b32 s5, s11497; CI-NEXT: v_mov_b32_e32 v0, s81498; CI-NEXT: v_mov_b32_e32 v1, s91499; CI-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[4:7], 0 glc1500; CI-NEXT: s_waitcnt vmcnt(0)1501; CI-NEXT: buffer_wbinvl1_vol1502; CI-NEXT: s_mov_b32 s4, s21503; CI-NEXT: s_mov_b32 s5, s31504; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 01505; CI-NEXT: s_endpgm1506;1507; VI-LABEL: atomic_sub_i64_ret:1508; VI: ; %bb.0: ; %entry1509; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241510; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x341511; VI-NEXT: s_mov_b32 s7, 0xf0001512; VI-NEXT: s_mov_b32 s6, -11513; VI-NEXT: s_waitcnt lgkmcnt(0)1514; VI-NEXT: s_mov_b32 s4, s01515; VI-NEXT: s_mov_b32 s5, s11516; VI-NEXT: v_mov_b32_e32 v0, s81517; VI-NEXT: v_mov_b32_e32 v1, s91518; VI-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[4:7], 0 glc1519; VI-NEXT: s_waitcnt vmcnt(0)1520; VI-NEXT: buffer_wbinvl1_vol1521; VI-NEXT: s_mov_b32 s4, s21522; VI-NEXT: s_mov_b32 s5, s31523; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 01524; VI-NEXT: s_endpgm1525;1526; GFX9-LABEL: atomic_sub_i64_ret:1527; GFX9: ; %bb.0: ; %entry1528; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341529; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241530; GFX9-NEXT: v_mov_b32_e32 v2, 01531; GFX9-NEXT: s_waitcnt lgkmcnt(0)1532; GFX9-NEXT: v_mov_b32_e32 v0, s61533; GFX9-NEXT: v_mov_b32_e32 v1, s71534; GFX9-NEXT: global_atomic_sub_x2 v[0:1], v2, v[0:1], s[0:1] glc1535; GFX9-NEXT: s_waitcnt vmcnt(0)1536; GFX9-NEXT: buffer_wbinvl1_vol1537; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]1538; GFX9-NEXT: s_endpgm1539;1540; GFX12-LABEL: atomic_sub_i64_ret:1541; GFX12: ; %bb.0: ; %entry1542; GFX12-NEXT: s_clause 0x11543; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x341544; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x241545; GFX12-NEXT: s_wait_kmcnt 0x01546; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s71547; GFX12-NEXT: v_mov_b32_e32 v0, s61548; GFX12-NEXT: global_atomic_sub_u64 v[0:1], v2, v[0:1], s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV1549; GFX12-NEXT: s_wait_loadcnt 0x01550; GFX12-NEXT: global_inv scope:SCOPE_DEV1551; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]1552; GFX12-NEXT: s_endpgm1553entry:1554 %tmp0 = atomicrmw volatile sub ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01555 store i64 %tmp0, ptr addrspace(1) %out21556 ret void1557}1558 1559define amdgpu_kernel void @atomic_sub_i64_addr64(ptr addrspace(1) %out, i64 %in, i64 %index) {1560; CI-LABEL: atomic_sub_i64_addr64:1561; CI: ; %bb.0: ; %entry1562; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91563; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd1564; CI-NEXT: s_waitcnt lgkmcnt(0)1565; CI-NEXT: v_mov_b32_e32 v0, s21566; CI-NEXT: s_lshl_b64 s[4:5], s[4:5], 31567; CI-NEXT: v_mov_b32_e32 v2, s41568; CI-NEXT: v_mov_b32_e32 v1, s31569; CI-NEXT: s_mov_b32 s3, 0xf0001570; CI-NEXT: s_mov_b32 s2, 01571; CI-NEXT: v_mov_b32_e32 v3, s51572; CI-NEXT: buffer_atomic_sub_x2 v[0:1], v[2:3], s[0:3], 0 addr641573; CI-NEXT: s_waitcnt vmcnt(0)1574; CI-NEXT: buffer_wbinvl1_vol1575; CI-NEXT: s_endpgm1576;1577; VI-LABEL: atomic_sub_i64_addr64:1578; VI: ; %bb.0: ; %entry1579; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241580; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341581; VI-NEXT: s_waitcnt lgkmcnt(0)1582; VI-NEXT: v_mov_b32_e32 v0, s21583; VI-NEXT: v_mov_b32_e32 v1, s31584; VI-NEXT: s_lshl_b64 s[2:3], s[4:5], 31585; VI-NEXT: s_add_u32 s0, s0, s21586; VI-NEXT: s_addc_u32 s1, s1, s31587; VI-NEXT: v_mov_b32_e32 v3, s11588; VI-NEXT: v_mov_b32_e32 v2, s01589; VI-NEXT: flat_atomic_sub_x2 v[2:3], v[0:1]1590; VI-NEXT: s_waitcnt vmcnt(0)1591; VI-NEXT: buffer_wbinvl1_vol1592; VI-NEXT: s_endpgm1593;1594; GFX9-LABEL: atomic_sub_i64_addr64:1595; GFX9: ; %bb.0: ; %entry1596; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241597; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341598; GFX9-NEXT: v_mov_b32_e32 v2, 01599; GFX9-NEXT: s_waitcnt lgkmcnt(0)1600; GFX9-NEXT: v_mov_b32_e32 v0, s21601; GFX9-NEXT: v_mov_b32_e32 v1, s31602; GFX9-NEXT: s_lshl_b64 s[2:3], s[6:7], 31603; GFX9-NEXT: s_add_u32 s0, s0, s21604; GFX9-NEXT: s_addc_u32 s1, s1, s31605; GFX9-NEXT: global_atomic_sub_x2 v2, v[0:1], s[0:1]1606; GFX9-NEXT: s_waitcnt vmcnt(0)1607; GFX9-NEXT: buffer_wbinvl1_vol1608; GFX9-NEXT: s_endpgm1609;1610; GFX12-LABEL: atomic_sub_i64_addr64:1611; GFX12: ; %bb.0: ; %entry1612; GFX12-NEXT: s_clause 0x11613; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x241614; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x341615; GFX12-NEXT: v_mov_b32_e32 v2, 01616; GFX12-NEXT: s_wait_kmcnt 0x01617; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s31618; GFX12-NEXT: s_lshl_b64 s[2:3], s[4:5], 31619; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)1620; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]1621; GFX12-NEXT: global_atomic_sub_u64 v2, v[0:1], s[0:1] scope:SCOPE_DEV1622; GFX12-NEXT: s_wait_storecnt 0x01623; GFX12-NEXT: global_inv scope:SCOPE_DEV1624; GFX12-NEXT: s_endpgm1625entry:1626 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index1627 %tmp0 = atomicrmw volatile sub ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01628 ret void1629}1630 1631define amdgpu_kernel void @atomic_sub_i64_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {1632; CI-LABEL: atomic_sub_i64_ret_addr64:1633; CI: ; %bb.0: ; %entry1634; CI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x91635; CI-NEXT: s_mov_b32 s11, 0xf0001636; CI-NEXT: s_mov_b32 s10, -11637; CI-NEXT: s_waitcnt lgkmcnt(0)1638; CI-NEXT: v_mov_b32_e32 v0, s41639; CI-NEXT: v_mov_b32_e32 v1, s51640; CI-NEXT: s_lshl_b64 s[4:5], s[6:7], 31641; CI-NEXT: v_mov_b32_e32 v2, s41642; CI-NEXT: s_mov_b32 s8, s21643; CI-NEXT: s_mov_b32 s9, s31644; CI-NEXT: s_mov_b32 s2, 01645; CI-NEXT: s_mov_b32 s3, s111646; CI-NEXT: v_mov_b32_e32 v3, s51647; CI-NEXT: buffer_atomic_sub_x2 v[0:1], v[2:3], s[0:3], 0 addr64 glc1648; CI-NEXT: s_waitcnt vmcnt(0)1649; CI-NEXT: buffer_wbinvl1_vol1650; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 01651; CI-NEXT: s_endpgm1652;1653; VI-LABEL: atomic_sub_i64_ret_addr64:1654; VI: ; %bb.0: ; %entry1655; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x241656; VI-NEXT: s_waitcnt lgkmcnt(0)1657; VI-NEXT: v_mov_b32_e32 v0, s41658; VI-NEXT: v_mov_b32_e32 v1, s51659; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 31660; VI-NEXT: s_add_u32 s0, s0, s41661; VI-NEXT: s_addc_u32 s1, s1, s51662; VI-NEXT: v_mov_b32_e32 v3, s11663; VI-NEXT: v_mov_b32_e32 v2, s01664; VI-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3], v[0:1] glc1665; VI-NEXT: s_waitcnt vmcnt(0)1666; VI-NEXT: buffer_wbinvl1_vol1667; VI-NEXT: s_mov_b32 s7, 0xf0001668; VI-NEXT: s_mov_b32 s6, -11669; VI-NEXT: s_mov_b32 s4, s21670; VI-NEXT: s_mov_b32 s5, s31671; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 01672; VI-NEXT: s_endpgm1673;1674; GFX9-LABEL: atomic_sub_i64_ret_addr64:1675; GFX9: ; %bb.0: ; %entry1676; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x241677; GFX9-NEXT: v_mov_b32_e32 v2, 01678; GFX9-NEXT: s_waitcnt lgkmcnt(0)1679; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 31680; GFX9-NEXT: s_add_u32 s0, s8, s01681; GFX9-NEXT: v_mov_b32_e32 v0, s121682; GFX9-NEXT: v_mov_b32_e32 v1, s131683; GFX9-NEXT: s_addc_u32 s1, s9, s11684; GFX9-NEXT: global_atomic_sub_x2 v[0:1], v2, v[0:1], s[0:1] glc1685; GFX9-NEXT: s_waitcnt vmcnt(0)1686; GFX9-NEXT: buffer_wbinvl1_vol1687; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]1688; GFX9-NEXT: s_endpgm1689;1690; GFX12-LABEL: atomic_sub_i64_ret_addr64:1691; GFX12: ; %bb.0: ; %entry1692; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x241693; GFX12-NEXT: s_wait_kmcnt 0x01694; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s51695; GFX12-NEXT: v_mov_b32_e32 v0, s41696; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 31697; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)1698; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]1699; GFX12-NEXT: global_atomic_sub_u64 v[0:1], v2, v[0:1], s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV1700; GFX12-NEXT: s_wait_loadcnt 0x01701; GFX12-NEXT: global_inv scope:SCOPE_DEV1702; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]1703; GFX12-NEXT: s_endpgm1704entry:1705 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index1706 %tmp0 = atomicrmw volatile sub ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01707 store i64 %tmp0, ptr addrspace(1) %out21708 ret void1709}1710 1711define amdgpu_kernel void @atomic_max_i64_offset(ptr addrspace(1) %out, i64 %in) {1712; CI-LABEL: atomic_max_i64_offset:1713; CI: ; %bb.0: ; %entry1714; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91715; CI-NEXT: s_waitcnt lgkmcnt(0)1716; CI-NEXT: v_mov_b32_e32 v0, s21717; CI-NEXT: v_mov_b32_e32 v1, s31718; CI-NEXT: s_mov_b32 s3, 0xf0001719; CI-NEXT: s_mov_b32 s2, -11720; CI-NEXT: buffer_atomic_smax_x2 v[0:1], off, s[0:3], 0 offset:321721; CI-NEXT: s_endpgm1722;1723; VI-LABEL: atomic_max_i64_offset:1724; VI: ; %bb.0: ; %entry1725; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241726; VI-NEXT: s_waitcnt lgkmcnt(0)1727; VI-NEXT: v_mov_b32_e32 v0, s21728; VI-NEXT: v_mov_b32_e32 v1, s31729; VI-NEXT: s_mov_b32 s3, 0xf0001730; VI-NEXT: s_mov_b32 s2, -11731; VI-NEXT: buffer_atomic_smax_x2 v[0:1], off, s[0:3], 0 offset:321732; VI-NEXT: s_endpgm1733;1734; GFX9-LABEL: atomic_max_i64_offset:1735; GFX9: ; %bb.0: ; %entry1736; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241737; GFX9-NEXT: v_mov_b32_e32 v2, 01738; GFX9-NEXT: s_waitcnt lgkmcnt(0)1739; GFX9-NEXT: v_mov_b32_e32 v0, s21740; GFX9-NEXT: v_mov_b32_e32 v1, s31741; GFX9-NEXT: global_atomic_smax_x2 v2, v[0:1], s[0:1] offset:321742; GFX9-NEXT: s_endpgm1743;1744; GFX12-LABEL: atomic_max_i64_offset:1745; GFX12: ; %bb.0: ; %entry1746; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x241747; GFX12-NEXT: s_wait_kmcnt 0x01748; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s31749; GFX12-NEXT: v_mov_b32_e32 v0, s21750; GFX12-NEXT: global_atomic_max_i64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_SE1751; GFX12-NEXT: s_wait_storecnt 0x01752; GFX12-NEXT: global_inv scope:SCOPE_SE1753; GFX12-NEXT: s_endpgm1754entry:1755 %gep = getelementptr i64, ptr addrspace(1) %out, i64 41756 %tmp0 = atomicrmw volatile max ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !01757 ret void1758}1759 1760define amdgpu_kernel void @atomic_max_i64_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in) {1761; CI-LABEL: atomic_max_i64_ret_offset:1762; CI: ; %bb.0: ; %entry1763; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91764; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd1765; CI-NEXT: s_mov_b32 s7, 0xf0001766; CI-NEXT: s_mov_b32 s6, -11767; CI-NEXT: s_waitcnt lgkmcnt(0)1768; CI-NEXT: s_mov_b32 s4, s21769; CI-NEXT: s_mov_b32 s5, s31770; CI-NEXT: v_mov_b32_e32 v0, s81771; CI-NEXT: v_mov_b32_e32 v1, s91772; CI-NEXT: s_mov_b32 s2, s61773; CI-NEXT: s_mov_b32 s3, s71774; CI-NEXT: buffer_atomic_smax_x2 v[0:1], off, s[0:3], 0 offset:32 glc1775; CI-NEXT: s_waitcnt vmcnt(0)1776; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 01777; CI-NEXT: s_endpgm1778;1779; VI-LABEL: atomic_max_i64_ret_offset:1780; VI: ; %bb.0: ; %entry1781; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241782; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x341783; VI-NEXT: s_mov_b32 s7, 0xf0001784; VI-NEXT: s_mov_b32 s6, -11785; VI-NEXT: s_waitcnt lgkmcnt(0)1786; VI-NEXT: s_mov_b32 s4, s21787; VI-NEXT: s_mov_b32 s5, s31788; VI-NEXT: v_mov_b32_e32 v0, s81789; VI-NEXT: v_mov_b32_e32 v1, s91790; VI-NEXT: s_mov_b32 s2, s61791; VI-NEXT: s_mov_b32 s3, s71792; VI-NEXT: buffer_atomic_smax_x2 v[0:1], off, s[0:3], 0 offset:32 glc1793; VI-NEXT: s_waitcnt vmcnt(0)1794; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 01795; VI-NEXT: s_endpgm1796;1797; GFX9-LABEL: atomic_max_i64_ret_offset:1798; GFX9: ; %bb.0: ; %entry1799; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341800; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241801; GFX9-NEXT: v_mov_b32_e32 v2, 01802; GFX9-NEXT: s_waitcnt lgkmcnt(0)1803; GFX9-NEXT: v_mov_b32_e32 v0, s61804; GFX9-NEXT: v_mov_b32_e32 v1, s71805; GFX9-NEXT: global_atomic_smax_x2 v[0:1], v2, v[0:1], s[0:1] offset:32 glc1806; GFX9-NEXT: s_waitcnt vmcnt(0)1807; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]1808; GFX9-NEXT: s_endpgm1809;1810; GFX12-LABEL: atomic_max_i64_ret_offset:1811; GFX12: ; %bb.0: ; %entry1812; GFX12-NEXT: s_clause 0x11813; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x341814; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x241815; GFX12-NEXT: s_wait_kmcnt 0x01816; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s71817; GFX12-NEXT: v_mov_b32_e32 v0, s61818; GFX12-NEXT: global_atomic_max_i64 v[0:1], v2, v[0:1], s[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_SE1819; GFX12-NEXT: s_wait_loadcnt 0x01820; GFX12-NEXT: global_inv scope:SCOPE_SE1821; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]1822; GFX12-NEXT: s_endpgm1823entry:1824 %gep = getelementptr i64, ptr addrspace(1) %out, i64 41825 %tmp0 = atomicrmw volatile max ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !01826 store i64 %tmp0, ptr addrspace(1) %out21827 ret void1828}1829 1830define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr addrspace(1) %out, i64 %in, i64 %index) {1831; CI-LABEL: atomic_max_i64_addr64_offset:1832; CI: ; %bb.0: ; %entry1833; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91834; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd1835; CI-NEXT: s_waitcnt lgkmcnt(0)1836; CI-NEXT: v_mov_b32_e32 v0, s21837; CI-NEXT: s_lshl_b64 s[4:5], s[4:5], 31838; CI-NEXT: v_mov_b32_e32 v2, s41839; CI-NEXT: v_mov_b32_e32 v1, s31840; CI-NEXT: s_mov_b32 s3, 0xf0001841; CI-NEXT: s_mov_b32 s2, 01842; CI-NEXT: v_mov_b32_e32 v3, s51843; CI-NEXT: buffer_atomic_smax_x2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:321844; CI-NEXT: s_endpgm1845;1846; VI-LABEL: atomic_max_i64_addr64_offset:1847; VI: ; %bb.0: ; %entry1848; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241849; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341850; VI-NEXT: s_waitcnt lgkmcnt(0)1851; VI-NEXT: v_mov_b32_e32 v0, s21852; VI-NEXT: v_mov_b32_e32 v1, s31853; VI-NEXT: s_lshl_b64 s[2:3], s[4:5], 31854; VI-NEXT: s_add_u32 s0, s0, s21855; VI-NEXT: s_addc_u32 s1, s1, s31856; VI-NEXT: s_add_u32 s0, s0, 321857; VI-NEXT: s_addc_u32 s1, s1, 01858; VI-NEXT: v_mov_b32_e32 v3, s11859; VI-NEXT: v_mov_b32_e32 v2, s01860; VI-NEXT: flat_atomic_smax_x2 v[2:3], v[0:1]1861; VI-NEXT: s_endpgm1862;1863; GFX9-LABEL: atomic_max_i64_addr64_offset:1864; GFX9: ; %bb.0: ; %entry1865; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241866; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341867; GFX9-NEXT: v_mov_b32_e32 v2, 01868; GFX9-NEXT: s_waitcnt lgkmcnt(0)1869; GFX9-NEXT: v_mov_b32_e32 v0, s21870; GFX9-NEXT: v_mov_b32_e32 v1, s31871; GFX9-NEXT: s_lshl_b64 s[2:3], s[6:7], 31872; GFX9-NEXT: s_add_u32 s0, s0, s21873; GFX9-NEXT: s_addc_u32 s1, s1, s31874; GFX9-NEXT: global_atomic_smax_x2 v2, v[0:1], s[0:1] offset:321875; GFX9-NEXT: s_endpgm1876;1877; GFX12-LABEL: atomic_max_i64_addr64_offset:1878; GFX12: ; %bb.0: ; %entry1879; GFX12-NEXT: s_clause 0x11880; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x241881; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x341882; GFX12-NEXT: v_mov_b32_e32 v2, 01883; GFX12-NEXT: s_wait_kmcnt 0x01884; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s31885; GFX12-NEXT: s_lshl_b64 s[2:3], s[4:5], 31886; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)1887; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]1888; GFX12-NEXT: global_atomic_max_i64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_SE1889; GFX12-NEXT: s_wait_storecnt 0x01890; GFX12-NEXT: global_inv scope:SCOPE_SE1891; GFX12-NEXT: s_endpgm1892entry:1893 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index1894 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 41895 %tmp0 = atomicrmw volatile max ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !01896 ret void1897}1898 1899define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {1900; CI-LABEL: atomic_max_i64_ret_addr64_offset:1901; CI: ; %bb.0: ; %entry1902; CI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x91903; CI-NEXT: s_mov_b32 s11, 0xf0001904; CI-NEXT: s_mov_b32 s10, -11905; CI-NEXT: s_waitcnt lgkmcnt(0)1906; CI-NEXT: v_mov_b32_e32 v0, s41907; CI-NEXT: v_mov_b32_e32 v1, s51908; CI-NEXT: s_lshl_b64 s[4:5], s[6:7], 31909; CI-NEXT: v_mov_b32_e32 v2, s41910; CI-NEXT: s_mov_b32 s8, s21911; CI-NEXT: s_mov_b32 s9, s31912; CI-NEXT: s_mov_b32 s2, 01913; CI-NEXT: s_mov_b32 s3, s111914; CI-NEXT: v_mov_b32_e32 v3, s51915; CI-NEXT: buffer_atomic_smax_x2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:32 glc1916; CI-NEXT: s_waitcnt vmcnt(0)1917; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 01918; CI-NEXT: s_endpgm1919;1920; VI-LABEL: atomic_max_i64_ret_addr64_offset:1921; VI: ; %bb.0: ; %entry1922; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x241923; VI-NEXT: s_waitcnt lgkmcnt(0)1924; VI-NEXT: v_mov_b32_e32 v0, s41925; VI-NEXT: v_mov_b32_e32 v1, s51926; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 31927; VI-NEXT: s_add_u32 s0, s0, s41928; VI-NEXT: s_addc_u32 s1, s1, s51929; VI-NEXT: s_add_u32 s0, s0, 321930; VI-NEXT: s_addc_u32 s1, s1, 01931; VI-NEXT: v_mov_b32_e32 v3, s11932; VI-NEXT: v_mov_b32_e32 v2, s01933; VI-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3], v[0:1] glc1934; VI-NEXT: s_mov_b32 s7, 0xf0001935; VI-NEXT: s_mov_b32 s6, -11936; VI-NEXT: s_mov_b32 s4, s21937; VI-NEXT: s_mov_b32 s5, s31938; VI-NEXT: s_waitcnt vmcnt(0)1939; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 01940; VI-NEXT: s_endpgm1941;1942; GFX9-LABEL: atomic_max_i64_ret_addr64_offset:1943; GFX9: ; %bb.0: ; %entry1944; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x241945; GFX9-NEXT: v_mov_b32_e32 v2, 01946; GFX9-NEXT: s_waitcnt lgkmcnt(0)1947; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 31948; GFX9-NEXT: s_add_u32 s0, s8, s01949; GFX9-NEXT: v_mov_b32_e32 v0, s121950; GFX9-NEXT: v_mov_b32_e32 v1, s131951; GFX9-NEXT: s_addc_u32 s1, s9, s11952; GFX9-NEXT: global_atomic_smax_x2 v[0:1], v2, v[0:1], s[0:1] offset:32 glc1953; GFX9-NEXT: s_waitcnt vmcnt(0)1954; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]1955; GFX9-NEXT: s_endpgm1956;1957; GFX12-LABEL: atomic_max_i64_ret_addr64_offset:1958; GFX12: ; %bb.0: ; %entry1959; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x241960; GFX12-NEXT: s_wait_kmcnt 0x01961; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s51962; GFX12-NEXT: v_mov_b32_e32 v0, s41963; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 31964; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)1965; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]1966; GFX12-NEXT: global_atomic_max_i64 v[0:1], v2, v[0:1], s[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_SE1967; GFX12-NEXT: s_wait_loadcnt 0x01968; GFX12-NEXT: global_inv scope:SCOPE_SE1969; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]1970; GFX12-NEXT: s_endpgm1971entry:1972 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index1973 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 41974 %tmp0 = atomicrmw volatile max ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !01975 store i64 %tmp0, ptr addrspace(1) %out21976 ret void1977}1978 1979define amdgpu_kernel void @atomic_max_i64(ptr addrspace(1) %out, i64 %in) {1980; CI-LABEL: atomic_max_i64:1981; CI: ; %bb.0: ; %entry1982; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91983; CI-NEXT: s_mov_b32 s7, 0xf0001984; CI-NEXT: s_mov_b32 s6, -11985; CI-NEXT: s_waitcnt lgkmcnt(0)1986; CI-NEXT: s_mov_b32 s4, s01987; CI-NEXT: s_mov_b32 s5, s11988; CI-NEXT: v_mov_b32_e32 v0, s21989; CI-NEXT: v_mov_b32_e32 v1, s31990; CI-NEXT: buffer_atomic_smax_x2 v[0:1], off, s[4:7], 01991; CI-NEXT: s_endpgm1992;1993; VI-LABEL: atomic_max_i64:1994; VI: ; %bb.0: ; %entry1995; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241996; VI-NEXT: s_mov_b32 s7, 0xf0001997; VI-NEXT: s_mov_b32 s6, -11998; VI-NEXT: s_waitcnt lgkmcnt(0)1999; VI-NEXT: s_mov_b32 s4, s02000; VI-NEXT: s_mov_b32 s5, s12001; VI-NEXT: v_mov_b32_e32 v0, s22002; VI-NEXT: v_mov_b32_e32 v1, s32003; VI-NEXT: buffer_atomic_smax_x2 v[0:1], off, s[4:7], 02004; VI-NEXT: s_endpgm2005;2006; GFX9-LABEL: atomic_max_i64:2007; GFX9: ; %bb.0: ; %entry2008; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242009; GFX9-NEXT: v_mov_b32_e32 v2, 02010; GFX9-NEXT: s_waitcnt lgkmcnt(0)2011; GFX9-NEXT: v_mov_b32_e32 v0, s22012; GFX9-NEXT: v_mov_b32_e32 v1, s32013; GFX9-NEXT: global_atomic_smax_x2 v2, v[0:1], s[0:1]2014; GFX9-NEXT: s_endpgm2015;2016; GFX12-LABEL: atomic_max_i64:2017; GFX12: ; %bb.0: ; %entry2018; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x242019; GFX12-NEXT: s_wait_kmcnt 0x02020; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s32021; GFX12-NEXT: v_mov_b32_e32 v0, s22022; GFX12-NEXT: global_atomic_max_i64 v2, v[0:1], s[0:1] scope:SCOPE_SE2023; GFX12-NEXT: s_wait_storecnt 0x02024; GFX12-NEXT: global_inv scope:SCOPE_SE2025; GFX12-NEXT: s_endpgm2026entry:2027 %tmp0 = atomicrmw volatile max ptr addrspace(1) %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02028 ret void2029}2030 2031define amdgpu_kernel void @atomic_max_i64_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in) {2032; CI-LABEL: atomic_max_i64_ret:2033; CI: ; %bb.0: ; %entry2034; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92035; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd2036; CI-NEXT: s_mov_b32 s7, 0xf0002037; CI-NEXT: s_mov_b32 s6, -12038; CI-NEXT: s_waitcnt lgkmcnt(0)2039; CI-NEXT: s_mov_b32 s4, s02040; CI-NEXT: s_mov_b32 s5, s12041; CI-NEXT: v_mov_b32_e32 v0, s82042; CI-NEXT: v_mov_b32_e32 v1, s92043; CI-NEXT: buffer_atomic_smax_x2 v[0:1], off, s[4:7], 0 glc2044; CI-NEXT: s_mov_b32 s4, s22045; CI-NEXT: s_mov_b32 s5, s32046; CI-NEXT: s_waitcnt vmcnt(0)2047; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 02048; CI-NEXT: s_endpgm2049;2050; VI-LABEL: atomic_max_i64_ret:2051; VI: ; %bb.0: ; %entry2052; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242053; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x342054; VI-NEXT: s_mov_b32 s7, 0xf0002055; VI-NEXT: s_mov_b32 s6, -12056; VI-NEXT: s_waitcnt lgkmcnt(0)2057; VI-NEXT: s_mov_b32 s4, s02058; VI-NEXT: s_mov_b32 s5, s12059; VI-NEXT: v_mov_b32_e32 v0, s82060; VI-NEXT: v_mov_b32_e32 v1, s92061; VI-NEXT: buffer_atomic_smax_x2 v[0:1], off, s[4:7], 0 glc2062; VI-NEXT: s_mov_b32 s4, s22063; VI-NEXT: s_mov_b32 s5, s32064; VI-NEXT: s_waitcnt vmcnt(0)2065; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 02066; VI-NEXT: s_endpgm2067;2068; GFX9-LABEL: atomic_max_i64_ret:2069; GFX9: ; %bb.0: ; %entry2070; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342071; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242072; GFX9-NEXT: v_mov_b32_e32 v2, 02073; GFX9-NEXT: s_waitcnt lgkmcnt(0)2074; GFX9-NEXT: v_mov_b32_e32 v0, s62075; GFX9-NEXT: v_mov_b32_e32 v1, s72076; GFX9-NEXT: global_atomic_smax_x2 v[0:1], v2, v[0:1], s[0:1] glc2077; GFX9-NEXT: s_waitcnt vmcnt(0)2078; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]2079; GFX9-NEXT: s_endpgm2080;2081; GFX12-LABEL: atomic_max_i64_ret:2082; GFX12: ; %bb.0: ; %entry2083; GFX12-NEXT: s_clause 0x12084; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x342085; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x242086; GFX12-NEXT: s_wait_kmcnt 0x02087; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s72088; GFX12-NEXT: v_mov_b32_e32 v0, s62089; GFX12-NEXT: global_atomic_max_i64 v[0:1], v2, v[0:1], s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SE2090; GFX12-NEXT: s_wait_loadcnt 0x02091; GFX12-NEXT: global_inv scope:SCOPE_SE2092; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]2093; GFX12-NEXT: s_endpgm2094entry:2095 %tmp0 = atomicrmw volatile max ptr addrspace(1) %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02096 store i64 %tmp0, ptr addrspace(1) %out22097 ret void2098}2099 2100define amdgpu_kernel void @atomic_max_i64_addr64(ptr addrspace(1) %out, i64 %in, i64 %index) {2101; CI-LABEL: atomic_max_i64_addr64:2102; CI: ; %bb.0: ; %entry2103; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92104; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd2105; CI-NEXT: s_waitcnt lgkmcnt(0)2106; CI-NEXT: v_mov_b32_e32 v0, s22107; CI-NEXT: s_lshl_b64 s[4:5], s[4:5], 32108; CI-NEXT: v_mov_b32_e32 v2, s42109; CI-NEXT: v_mov_b32_e32 v1, s32110; CI-NEXT: s_mov_b32 s3, 0xf0002111; CI-NEXT: s_mov_b32 s2, 02112; CI-NEXT: v_mov_b32_e32 v3, s52113; CI-NEXT: buffer_atomic_smax_x2 v[0:1], v[2:3], s[0:3], 0 addr642114; CI-NEXT: s_endpgm2115;2116; VI-LABEL: atomic_max_i64_addr64:2117; VI: ; %bb.0: ; %entry2118; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242119; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x342120; VI-NEXT: s_waitcnt lgkmcnt(0)2121; VI-NEXT: v_mov_b32_e32 v0, s22122; VI-NEXT: v_mov_b32_e32 v1, s32123; VI-NEXT: s_lshl_b64 s[2:3], s[4:5], 32124; VI-NEXT: s_add_u32 s0, s0, s22125; VI-NEXT: s_addc_u32 s1, s1, s32126; VI-NEXT: v_mov_b32_e32 v3, s12127; VI-NEXT: v_mov_b32_e32 v2, s02128; VI-NEXT: flat_atomic_smax_x2 v[2:3], v[0:1]2129; VI-NEXT: s_endpgm2130;2131; GFX9-LABEL: atomic_max_i64_addr64:2132; GFX9: ; %bb.0: ; %entry2133; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242134; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342135; GFX9-NEXT: v_mov_b32_e32 v2, 02136; GFX9-NEXT: s_waitcnt lgkmcnt(0)2137; GFX9-NEXT: v_mov_b32_e32 v0, s22138; GFX9-NEXT: v_mov_b32_e32 v1, s32139; GFX9-NEXT: s_lshl_b64 s[2:3], s[6:7], 32140; GFX9-NEXT: s_add_u32 s0, s0, s22141; GFX9-NEXT: s_addc_u32 s1, s1, s32142; GFX9-NEXT: global_atomic_smax_x2 v2, v[0:1], s[0:1]2143; GFX9-NEXT: s_endpgm2144;2145; GFX12-LABEL: atomic_max_i64_addr64:2146; GFX12: ; %bb.0: ; %entry2147; GFX12-NEXT: s_clause 0x12148; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x242149; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x342150; GFX12-NEXT: v_mov_b32_e32 v2, 02151; GFX12-NEXT: s_wait_kmcnt 0x02152; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s32153; GFX12-NEXT: s_lshl_b64 s[2:3], s[4:5], 32154; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)2155; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]2156; GFX12-NEXT: global_atomic_max_i64 v2, v[0:1], s[0:1] scope:SCOPE_SE2157; GFX12-NEXT: s_wait_storecnt 0x02158; GFX12-NEXT: global_inv scope:SCOPE_SE2159; GFX12-NEXT: s_endpgm2160entry:2161 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index2162 %tmp0 = atomicrmw volatile max ptr addrspace(1) %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02163 ret void2164}2165 2166define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {2167; CI-LABEL: atomic_max_i64_ret_addr64:2168; CI: ; %bb.0: ; %entry2169; CI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x92170; CI-NEXT: s_mov_b32 s11, 0xf0002171; CI-NEXT: s_mov_b32 s10, -12172; CI-NEXT: s_waitcnt lgkmcnt(0)2173; CI-NEXT: v_mov_b32_e32 v0, s42174; CI-NEXT: v_mov_b32_e32 v1, s52175; CI-NEXT: s_lshl_b64 s[4:5], s[6:7], 32176; CI-NEXT: v_mov_b32_e32 v2, s42177; CI-NEXT: s_mov_b32 s8, s22178; CI-NEXT: s_mov_b32 s9, s32179; CI-NEXT: s_mov_b32 s2, 02180; CI-NEXT: s_mov_b32 s3, s112181; CI-NEXT: v_mov_b32_e32 v3, s52182; CI-NEXT: buffer_atomic_smax_x2 v[0:1], v[2:3], s[0:3], 0 addr64 glc2183; CI-NEXT: s_waitcnt vmcnt(0)2184; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 02185; CI-NEXT: s_endpgm2186;2187; VI-LABEL: atomic_max_i64_ret_addr64:2188; VI: ; %bb.0: ; %entry2189; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x242190; VI-NEXT: s_waitcnt lgkmcnt(0)2191; VI-NEXT: v_mov_b32_e32 v0, s42192; VI-NEXT: v_mov_b32_e32 v1, s52193; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 32194; VI-NEXT: s_add_u32 s0, s0, s42195; VI-NEXT: s_addc_u32 s1, s1, s52196; VI-NEXT: v_mov_b32_e32 v3, s12197; VI-NEXT: v_mov_b32_e32 v2, s02198; VI-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3], v[0:1] glc2199; VI-NEXT: s_mov_b32 s7, 0xf0002200; VI-NEXT: s_mov_b32 s6, -12201; VI-NEXT: s_mov_b32 s4, s22202; VI-NEXT: s_mov_b32 s5, s32203; VI-NEXT: s_waitcnt vmcnt(0)2204; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 02205; VI-NEXT: s_endpgm2206;2207; GFX9-LABEL: atomic_max_i64_ret_addr64:2208; GFX9: ; %bb.0: ; %entry2209; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x242210; GFX9-NEXT: v_mov_b32_e32 v2, 02211; GFX9-NEXT: s_waitcnt lgkmcnt(0)2212; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 32213; GFX9-NEXT: s_add_u32 s0, s8, s02214; GFX9-NEXT: v_mov_b32_e32 v0, s122215; GFX9-NEXT: v_mov_b32_e32 v1, s132216; GFX9-NEXT: s_addc_u32 s1, s9, s12217; GFX9-NEXT: global_atomic_smax_x2 v[0:1], v2, v[0:1], s[0:1] glc2218; GFX9-NEXT: s_waitcnt vmcnt(0)2219; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]2220; GFX9-NEXT: s_endpgm2221;2222; GFX12-LABEL: atomic_max_i64_ret_addr64:2223; GFX12: ; %bb.0: ; %entry2224; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x242225; GFX12-NEXT: s_wait_kmcnt 0x02226; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s52227; GFX12-NEXT: v_mov_b32_e32 v0, s42228; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 32229; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)2230; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]2231; GFX12-NEXT: global_atomic_max_i64 v[0:1], v2, v[0:1], s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SE2232; GFX12-NEXT: s_wait_loadcnt 0x02233; GFX12-NEXT: global_inv scope:SCOPE_SE2234; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]2235; GFX12-NEXT: s_endpgm2236entry:2237 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index2238 %tmp0 = atomicrmw volatile max ptr addrspace(1) %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02239 store i64 %tmp0, ptr addrspace(1) %out22240 ret void2241}2242 2243define amdgpu_kernel void @atomic_umax_i64_offset(ptr addrspace(1) %out, i64 %in) {2244; CI-LABEL: atomic_umax_i64_offset:2245; CI: ; %bb.0: ; %entry2246; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92247; CI-NEXT: s_waitcnt lgkmcnt(0)2248; CI-NEXT: v_mov_b32_e32 v0, s22249; CI-NEXT: v_mov_b32_e32 v1, s32250; CI-NEXT: s_mov_b32 s3, 0xf0002251; CI-NEXT: s_mov_b32 s2, -12252; CI-NEXT: buffer_atomic_umax_x2 v[0:1], off, s[0:3], 0 offset:322253; CI-NEXT: s_endpgm2254;2255; VI-LABEL: atomic_umax_i64_offset:2256; VI: ; %bb.0: ; %entry2257; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242258; VI-NEXT: s_waitcnt lgkmcnt(0)2259; VI-NEXT: v_mov_b32_e32 v0, s22260; VI-NEXT: v_mov_b32_e32 v1, s32261; VI-NEXT: s_mov_b32 s3, 0xf0002262; VI-NEXT: s_mov_b32 s2, -12263; VI-NEXT: buffer_atomic_umax_x2 v[0:1], off, s[0:3], 0 offset:322264; VI-NEXT: s_endpgm2265;2266; GFX9-LABEL: atomic_umax_i64_offset:2267; GFX9: ; %bb.0: ; %entry2268; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242269; GFX9-NEXT: v_mov_b32_e32 v2, 02270; GFX9-NEXT: s_waitcnt lgkmcnt(0)2271; GFX9-NEXT: v_mov_b32_e32 v0, s22272; GFX9-NEXT: v_mov_b32_e32 v1, s32273; GFX9-NEXT: global_atomic_umax_x2 v2, v[0:1], s[0:1] offset:322274; GFX9-NEXT: s_endpgm2275;2276; GFX12-LABEL: atomic_umax_i64_offset:2277; GFX12: ; %bb.0: ; %entry2278; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x242279; GFX12-NEXT: s_wait_kmcnt 0x02280; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s32281; GFX12-NEXT: v_mov_b32_e32 v0, s22282; GFX12-NEXT: global_atomic_max_u64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_SE2283; GFX12-NEXT: s_wait_storecnt 0x02284; GFX12-NEXT: global_inv scope:SCOPE_SE2285; GFX12-NEXT: s_endpgm2286entry:2287 %gep = getelementptr i64, ptr addrspace(1) %out, i64 42288 %tmp0 = atomicrmw volatile umax ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02289 ret void2290}2291 2292define amdgpu_kernel void @atomic_umax_i64_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in) {2293; CI-LABEL: atomic_umax_i64_ret_offset:2294; CI: ; %bb.0: ; %entry2295; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92296; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd2297; CI-NEXT: s_mov_b32 s7, 0xf0002298; CI-NEXT: s_mov_b32 s6, -12299; CI-NEXT: s_waitcnt lgkmcnt(0)2300; CI-NEXT: s_mov_b32 s4, s22301; CI-NEXT: s_mov_b32 s5, s32302; CI-NEXT: v_mov_b32_e32 v0, s82303; CI-NEXT: v_mov_b32_e32 v1, s92304; CI-NEXT: s_mov_b32 s2, s62305; CI-NEXT: s_mov_b32 s3, s72306; CI-NEXT: buffer_atomic_umax_x2 v[0:1], off, s[0:3], 0 offset:32 glc2307; CI-NEXT: s_waitcnt vmcnt(0)2308; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 02309; CI-NEXT: s_endpgm2310;2311; VI-LABEL: atomic_umax_i64_ret_offset:2312; VI: ; %bb.0: ; %entry2313; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242314; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x342315; VI-NEXT: s_mov_b32 s7, 0xf0002316; VI-NEXT: s_mov_b32 s6, -12317; VI-NEXT: s_waitcnt lgkmcnt(0)2318; VI-NEXT: s_mov_b32 s4, s22319; VI-NEXT: s_mov_b32 s5, s32320; VI-NEXT: v_mov_b32_e32 v0, s82321; VI-NEXT: v_mov_b32_e32 v1, s92322; VI-NEXT: s_mov_b32 s2, s62323; VI-NEXT: s_mov_b32 s3, s72324; VI-NEXT: buffer_atomic_umax_x2 v[0:1], off, s[0:3], 0 offset:32 glc2325; VI-NEXT: s_waitcnt vmcnt(0)2326; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 02327; VI-NEXT: s_endpgm2328;2329; GFX9-LABEL: atomic_umax_i64_ret_offset:2330; GFX9: ; %bb.0: ; %entry2331; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342332; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242333; GFX9-NEXT: v_mov_b32_e32 v2, 02334; GFX9-NEXT: s_waitcnt lgkmcnt(0)2335; GFX9-NEXT: v_mov_b32_e32 v0, s62336; GFX9-NEXT: v_mov_b32_e32 v1, s72337; GFX9-NEXT: global_atomic_umax_x2 v[0:1], v2, v[0:1], s[0:1] offset:32 glc2338; GFX9-NEXT: s_waitcnt vmcnt(0)2339; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]2340; GFX9-NEXT: s_endpgm2341;2342; GFX12-LABEL: atomic_umax_i64_ret_offset:2343; GFX12: ; %bb.0: ; %entry2344; GFX12-NEXT: s_clause 0x12345; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x342346; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x242347; GFX12-NEXT: s_wait_kmcnt 0x02348; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s72349; GFX12-NEXT: v_mov_b32_e32 v0, s62350; GFX12-NEXT: global_atomic_max_u64 v[0:1], v2, v[0:1], s[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_SE2351; GFX12-NEXT: s_wait_loadcnt 0x02352; GFX12-NEXT: global_inv scope:SCOPE_SE2353; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]2354; GFX12-NEXT: s_endpgm2355entry:2356 %gep = getelementptr i64, ptr addrspace(1) %out, i64 42357 %tmp0 = atomicrmw volatile umax ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02358 store i64 %tmp0, ptr addrspace(1) %out22359 ret void2360}2361 2362define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr addrspace(1) %out, i64 %in, i64 %index) {2363; CI-LABEL: atomic_umax_i64_addr64_offset:2364; CI: ; %bb.0: ; %entry2365; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92366; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd2367; CI-NEXT: s_waitcnt lgkmcnt(0)2368; CI-NEXT: v_mov_b32_e32 v0, s22369; CI-NEXT: s_lshl_b64 s[4:5], s[4:5], 32370; CI-NEXT: v_mov_b32_e32 v2, s42371; CI-NEXT: v_mov_b32_e32 v1, s32372; CI-NEXT: s_mov_b32 s3, 0xf0002373; CI-NEXT: s_mov_b32 s2, 02374; CI-NEXT: v_mov_b32_e32 v3, s52375; CI-NEXT: buffer_atomic_umax_x2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:322376; CI-NEXT: s_endpgm2377;2378; VI-LABEL: atomic_umax_i64_addr64_offset:2379; VI: ; %bb.0: ; %entry2380; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242381; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x342382; VI-NEXT: s_waitcnt lgkmcnt(0)2383; VI-NEXT: v_mov_b32_e32 v0, s22384; VI-NEXT: v_mov_b32_e32 v1, s32385; VI-NEXT: s_lshl_b64 s[2:3], s[4:5], 32386; VI-NEXT: s_add_u32 s0, s0, s22387; VI-NEXT: s_addc_u32 s1, s1, s32388; VI-NEXT: s_add_u32 s0, s0, 322389; VI-NEXT: s_addc_u32 s1, s1, 02390; VI-NEXT: v_mov_b32_e32 v3, s12391; VI-NEXT: v_mov_b32_e32 v2, s02392; VI-NEXT: flat_atomic_umax_x2 v[2:3], v[0:1]2393; VI-NEXT: s_endpgm2394;2395; GFX9-LABEL: atomic_umax_i64_addr64_offset:2396; GFX9: ; %bb.0: ; %entry2397; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242398; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342399; GFX9-NEXT: v_mov_b32_e32 v2, 02400; GFX9-NEXT: s_waitcnt lgkmcnt(0)2401; GFX9-NEXT: v_mov_b32_e32 v0, s22402; GFX9-NEXT: v_mov_b32_e32 v1, s32403; GFX9-NEXT: s_lshl_b64 s[2:3], s[6:7], 32404; GFX9-NEXT: s_add_u32 s0, s0, s22405; GFX9-NEXT: s_addc_u32 s1, s1, s32406; GFX9-NEXT: global_atomic_umax_x2 v2, v[0:1], s[0:1] offset:322407; GFX9-NEXT: s_endpgm2408;2409; GFX12-LABEL: atomic_umax_i64_addr64_offset:2410; GFX12: ; %bb.0: ; %entry2411; GFX12-NEXT: s_clause 0x12412; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x242413; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x342414; GFX12-NEXT: v_mov_b32_e32 v2, 02415; GFX12-NEXT: s_wait_kmcnt 0x02416; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s32417; GFX12-NEXT: s_lshl_b64 s[2:3], s[4:5], 32418; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)2419; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]2420; GFX12-NEXT: global_atomic_max_u64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_SE2421; GFX12-NEXT: s_wait_storecnt 0x02422; GFX12-NEXT: global_inv scope:SCOPE_SE2423; GFX12-NEXT: s_endpgm2424entry:2425 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index2426 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 42427 %tmp0 = atomicrmw volatile umax ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02428 ret void2429}2430 2431define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {2432; CI-LABEL: atomic_umax_i64_ret_addr64_offset:2433; CI: ; %bb.0: ; %entry2434; CI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x92435; CI-NEXT: s_mov_b32 s11, 0xf0002436; CI-NEXT: s_mov_b32 s10, -12437; CI-NEXT: s_waitcnt lgkmcnt(0)2438; CI-NEXT: v_mov_b32_e32 v0, s42439; CI-NEXT: v_mov_b32_e32 v1, s52440; CI-NEXT: s_lshl_b64 s[4:5], s[6:7], 32441; CI-NEXT: v_mov_b32_e32 v2, s42442; CI-NEXT: s_mov_b32 s8, s22443; CI-NEXT: s_mov_b32 s9, s32444; CI-NEXT: s_mov_b32 s2, 02445; CI-NEXT: s_mov_b32 s3, s112446; CI-NEXT: v_mov_b32_e32 v3, s52447; CI-NEXT: buffer_atomic_umax_x2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:32 glc2448; CI-NEXT: s_waitcnt vmcnt(0)2449; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 02450; CI-NEXT: s_endpgm2451;2452; VI-LABEL: atomic_umax_i64_ret_addr64_offset:2453; VI: ; %bb.0: ; %entry2454; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x242455; VI-NEXT: s_waitcnt lgkmcnt(0)2456; VI-NEXT: v_mov_b32_e32 v0, s42457; VI-NEXT: v_mov_b32_e32 v1, s52458; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 32459; VI-NEXT: s_add_u32 s0, s0, s42460; VI-NEXT: s_addc_u32 s1, s1, s52461; VI-NEXT: s_add_u32 s0, s0, 322462; VI-NEXT: s_addc_u32 s1, s1, 02463; VI-NEXT: v_mov_b32_e32 v3, s12464; VI-NEXT: v_mov_b32_e32 v2, s02465; VI-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3], v[0:1] glc2466; VI-NEXT: s_mov_b32 s7, 0xf0002467; VI-NEXT: s_mov_b32 s6, -12468; VI-NEXT: s_mov_b32 s4, s22469; VI-NEXT: s_mov_b32 s5, s32470; VI-NEXT: s_waitcnt vmcnt(0)2471; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 02472; VI-NEXT: s_endpgm2473;2474; GFX9-LABEL: atomic_umax_i64_ret_addr64_offset:2475; GFX9: ; %bb.0: ; %entry2476; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x242477; GFX9-NEXT: v_mov_b32_e32 v2, 02478; GFX9-NEXT: s_waitcnt lgkmcnt(0)2479; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 32480; GFX9-NEXT: s_add_u32 s0, s8, s02481; GFX9-NEXT: v_mov_b32_e32 v0, s122482; GFX9-NEXT: v_mov_b32_e32 v1, s132483; GFX9-NEXT: s_addc_u32 s1, s9, s12484; GFX9-NEXT: global_atomic_umax_x2 v[0:1], v2, v[0:1], s[0:1] offset:32 glc2485; GFX9-NEXT: s_waitcnt vmcnt(0)2486; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]2487; GFX9-NEXT: s_endpgm2488;2489; GFX12-LABEL: atomic_umax_i64_ret_addr64_offset:2490; GFX12: ; %bb.0: ; %entry2491; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x242492; GFX12-NEXT: s_wait_kmcnt 0x02493; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s52494; GFX12-NEXT: v_mov_b32_e32 v0, s42495; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 32496; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)2497; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]2498; GFX12-NEXT: global_atomic_max_u64 v[0:1], v2, v[0:1], s[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_SE2499; GFX12-NEXT: s_wait_loadcnt 0x02500; GFX12-NEXT: global_inv scope:SCOPE_SE2501; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]2502; GFX12-NEXT: s_endpgm2503entry:2504 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index2505 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 42506 %tmp0 = atomicrmw volatile umax ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02507 store i64 %tmp0, ptr addrspace(1) %out22508 ret void2509}2510 2511define amdgpu_kernel void @atomic_umax_i64(ptr addrspace(1) %out, i64 %in) {2512; CI-LABEL: atomic_umax_i64:2513; CI: ; %bb.0: ; %entry2514; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92515; CI-NEXT: s_mov_b32 s7, 0xf0002516; CI-NEXT: s_mov_b32 s6, -12517; CI-NEXT: s_waitcnt lgkmcnt(0)2518; CI-NEXT: s_mov_b32 s4, s02519; CI-NEXT: s_mov_b32 s5, s12520; CI-NEXT: v_mov_b32_e32 v0, s22521; CI-NEXT: v_mov_b32_e32 v1, s32522; CI-NEXT: buffer_atomic_umax_x2 v[0:1], off, s[4:7], 02523; CI-NEXT: s_endpgm2524;2525; VI-LABEL: atomic_umax_i64:2526; VI: ; %bb.0: ; %entry2527; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242528; VI-NEXT: s_mov_b32 s7, 0xf0002529; VI-NEXT: s_mov_b32 s6, -12530; VI-NEXT: s_waitcnt lgkmcnt(0)2531; VI-NEXT: s_mov_b32 s4, s02532; VI-NEXT: s_mov_b32 s5, s12533; VI-NEXT: v_mov_b32_e32 v0, s22534; VI-NEXT: v_mov_b32_e32 v1, s32535; VI-NEXT: buffer_atomic_umax_x2 v[0:1], off, s[4:7], 02536; VI-NEXT: s_endpgm2537;2538; GFX9-LABEL: atomic_umax_i64:2539; GFX9: ; %bb.0: ; %entry2540; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242541; GFX9-NEXT: v_mov_b32_e32 v2, 02542; GFX9-NEXT: s_waitcnt lgkmcnt(0)2543; GFX9-NEXT: v_mov_b32_e32 v0, s22544; GFX9-NEXT: v_mov_b32_e32 v1, s32545; GFX9-NEXT: global_atomic_umax_x2 v2, v[0:1], s[0:1]2546; GFX9-NEXT: s_endpgm2547;2548; GFX12-LABEL: atomic_umax_i64:2549; GFX12: ; %bb.0: ; %entry2550; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x242551; GFX12-NEXT: s_wait_kmcnt 0x02552; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s32553; GFX12-NEXT: v_mov_b32_e32 v0, s22554; GFX12-NEXT: global_atomic_max_u64 v2, v[0:1], s[0:1] scope:SCOPE_SE2555; GFX12-NEXT: s_wait_storecnt 0x02556; GFX12-NEXT: global_inv scope:SCOPE_SE2557; GFX12-NEXT: s_endpgm2558entry:2559 %tmp0 = atomicrmw volatile umax ptr addrspace(1) %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02560 ret void2561}2562 2563define amdgpu_kernel void @atomic_umax_i64_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in) {2564; CI-LABEL: atomic_umax_i64_ret:2565; CI: ; %bb.0: ; %entry2566; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92567; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd2568; CI-NEXT: s_mov_b32 s7, 0xf0002569; CI-NEXT: s_mov_b32 s6, -12570; CI-NEXT: s_waitcnt lgkmcnt(0)2571; CI-NEXT: s_mov_b32 s4, s02572; CI-NEXT: s_mov_b32 s5, s12573; CI-NEXT: v_mov_b32_e32 v0, s82574; CI-NEXT: v_mov_b32_e32 v1, s92575; CI-NEXT: buffer_atomic_umax_x2 v[0:1], off, s[4:7], 0 glc2576; CI-NEXT: s_mov_b32 s4, s22577; CI-NEXT: s_mov_b32 s5, s32578; CI-NEXT: s_waitcnt vmcnt(0)2579; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 02580; CI-NEXT: s_endpgm2581;2582; VI-LABEL: atomic_umax_i64_ret:2583; VI: ; %bb.0: ; %entry2584; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242585; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x342586; VI-NEXT: s_mov_b32 s7, 0xf0002587; VI-NEXT: s_mov_b32 s6, -12588; VI-NEXT: s_waitcnt lgkmcnt(0)2589; VI-NEXT: s_mov_b32 s4, s02590; VI-NEXT: s_mov_b32 s5, s12591; VI-NEXT: v_mov_b32_e32 v0, s82592; VI-NEXT: v_mov_b32_e32 v1, s92593; VI-NEXT: buffer_atomic_umax_x2 v[0:1], off, s[4:7], 0 glc2594; VI-NEXT: s_mov_b32 s4, s22595; VI-NEXT: s_mov_b32 s5, s32596; VI-NEXT: s_waitcnt vmcnt(0)2597; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 02598; VI-NEXT: s_endpgm2599;2600; GFX9-LABEL: atomic_umax_i64_ret:2601; GFX9: ; %bb.0: ; %entry2602; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342603; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242604; GFX9-NEXT: v_mov_b32_e32 v2, 02605; GFX9-NEXT: s_waitcnt lgkmcnt(0)2606; GFX9-NEXT: v_mov_b32_e32 v0, s62607; GFX9-NEXT: v_mov_b32_e32 v1, s72608; GFX9-NEXT: global_atomic_umax_x2 v[0:1], v2, v[0:1], s[0:1] glc2609; GFX9-NEXT: s_waitcnt vmcnt(0)2610; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]2611; GFX9-NEXT: s_endpgm2612;2613; GFX12-LABEL: atomic_umax_i64_ret:2614; GFX12: ; %bb.0: ; %entry2615; GFX12-NEXT: s_clause 0x12616; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x342617; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x242618; GFX12-NEXT: s_wait_kmcnt 0x02619; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s72620; GFX12-NEXT: v_mov_b32_e32 v0, s62621; GFX12-NEXT: global_atomic_max_u64 v[0:1], v2, v[0:1], s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SE2622; GFX12-NEXT: s_wait_loadcnt 0x02623; GFX12-NEXT: global_inv scope:SCOPE_SE2624; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]2625; GFX12-NEXT: s_endpgm2626entry:2627 %tmp0 = atomicrmw volatile umax ptr addrspace(1) %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02628 store i64 %tmp0, ptr addrspace(1) %out22629 ret void2630}2631 2632define amdgpu_kernel void @atomic_umax_i64_addr64(ptr addrspace(1) %out, i64 %in, i64 %index) {2633; CI-LABEL: atomic_umax_i64_addr64:2634; CI: ; %bb.0: ; %entry2635; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92636; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd2637; CI-NEXT: s_waitcnt lgkmcnt(0)2638; CI-NEXT: v_mov_b32_e32 v0, s22639; CI-NEXT: s_lshl_b64 s[4:5], s[4:5], 32640; CI-NEXT: v_mov_b32_e32 v2, s42641; CI-NEXT: v_mov_b32_e32 v1, s32642; CI-NEXT: s_mov_b32 s3, 0xf0002643; CI-NEXT: s_mov_b32 s2, 02644; CI-NEXT: v_mov_b32_e32 v3, s52645; CI-NEXT: buffer_atomic_umax_x2 v[0:1], v[2:3], s[0:3], 0 addr642646; CI-NEXT: s_endpgm2647;2648; VI-LABEL: atomic_umax_i64_addr64:2649; VI: ; %bb.0: ; %entry2650; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242651; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x342652; VI-NEXT: s_waitcnt lgkmcnt(0)2653; VI-NEXT: v_mov_b32_e32 v0, s22654; VI-NEXT: v_mov_b32_e32 v1, s32655; VI-NEXT: s_lshl_b64 s[2:3], s[4:5], 32656; VI-NEXT: s_add_u32 s0, s0, s22657; VI-NEXT: s_addc_u32 s1, s1, s32658; VI-NEXT: v_mov_b32_e32 v3, s12659; VI-NEXT: v_mov_b32_e32 v2, s02660; VI-NEXT: flat_atomic_umax_x2 v[2:3], v[0:1]2661; VI-NEXT: s_endpgm2662;2663; GFX9-LABEL: atomic_umax_i64_addr64:2664; GFX9: ; %bb.0: ; %entry2665; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242666; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342667; GFX9-NEXT: v_mov_b32_e32 v2, 02668; GFX9-NEXT: s_waitcnt lgkmcnt(0)2669; GFX9-NEXT: v_mov_b32_e32 v0, s22670; GFX9-NEXT: v_mov_b32_e32 v1, s32671; GFX9-NEXT: s_lshl_b64 s[2:3], s[6:7], 32672; GFX9-NEXT: s_add_u32 s0, s0, s22673; GFX9-NEXT: s_addc_u32 s1, s1, s32674; GFX9-NEXT: global_atomic_umax_x2 v2, v[0:1], s[0:1]2675; GFX9-NEXT: s_endpgm2676;2677; GFX12-LABEL: atomic_umax_i64_addr64:2678; GFX12: ; %bb.0: ; %entry2679; GFX12-NEXT: s_clause 0x12680; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x242681; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x342682; GFX12-NEXT: v_mov_b32_e32 v2, 02683; GFX12-NEXT: s_wait_kmcnt 0x02684; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s32685; GFX12-NEXT: s_lshl_b64 s[2:3], s[4:5], 32686; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)2687; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]2688; GFX12-NEXT: global_atomic_max_u64 v2, v[0:1], s[0:1] scope:SCOPE_SE2689; GFX12-NEXT: s_wait_storecnt 0x02690; GFX12-NEXT: global_inv scope:SCOPE_SE2691; GFX12-NEXT: s_endpgm2692entry:2693 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index2694 %tmp0 = atomicrmw volatile umax ptr addrspace(1) %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02695 ret void2696}2697 2698define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {2699; CI-LABEL: atomic_umax_i64_ret_addr64:2700; CI: ; %bb.0: ; %entry2701; CI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x92702; CI-NEXT: s_mov_b32 s11, 0xf0002703; CI-NEXT: s_mov_b32 s10, -12704; CI-NEXT: s_waitcnt lgkmcnt(0)2705; CI-NEXT: v_mov_b32_e32 v0, s42706; CI-NEXT: v_mov_b32_e32 v1, s52707; CI-NEXT: s_lshl_b64 s[4:5], s[6:7], 32708; CI-NEXT: v_mov_b32_e32 v2, s42709; CI-NEXT: s_mov_b32 s8, s22710; CI-NEXT: s_mov_b32 s9, s32711; CI-NEXT: s_mov_b32 s2, 02712; CI-NEXT: s_mov_b32 s3, s112713; CI-NEXT: v_mov_b32_e32 v3, s52714; CI-NEXT: buffer_atomic_umax_x2 v[0:1], v[2:3], s[0:3], 0 addr64 glc2715; CI-NEXT: s_waitcnt vmcnt(0)2716; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 02717; CI-NEXT: s_endpgm2718;2719; VI-LABEL: atomic_umax_i64_ret_addr64:2720; VI: ; %bb.0: ; %entry2721; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x242722; VI-NEXT: s_waitcnt lgkmcnt(0)2723; VI-NEXT: v_mov_b32_e32 v0, s42724; VI-NEXT: v_mov_b32_e32 v1, s52725; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 32726; VI-NEXT: s_add_u32 s0, s0, s42727; VI-NEXT: s_addc_u32 s1, s1, s52728; VI-NEXT: v_mov_b32_e32 v3, s12729; VI-NEXT: v_mov_b32_e32 v2, s02730; VI-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3], v[0:1] glc2731; VI-NEXT: s_mov_b32 s7, 0xf0002732; VI-NEXT: s_mov_b32 s6, -12733; VI-NEXT: s_mov_b32 s4, s22734; VI-NEXT: s_mov_b32 s5, s32735; VI-NEXT: s_waitcnt vmcnt(0)2736; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 02737; VI-NEXT: s_endpgm2738;2739; GFX9-LABEL: atomic_umax_i64_ret_addr64:2740; GFX9: ; %bb.0: ; %entry2741; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x242742; GFX9-NEXT: v_mov_b32_e32 v2, 02743; GFX9-NEXT: s_waitcnt lgkmcnt(0)2744; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 32745; GFX9-NEXT: s_add_u32 s0, s8, s02746; GFX9-NEXT: v_mov_b32_e32 v0, s122747; GFX9-NEXT: v_mov_b32_e32 v1, s132748; GFX9-NEXT: s_addc_u32 s1, s9, s12749; GFX9-NEXT: global_atomic_umax_x2 v[0:1], v2, v[0:1], s[0:1] glc2750; GFX9-NEXT: s_waitcnt vmcnt(0)2751; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]2752; GFX9-NEXT: s_endpgm2753;2754; GFX12-LABEL: atomic_umax_i64_ret_addr64:2755; GFX12: ; %bb.0: ; %entry2756; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x242757; GFX12-NEXT: s_wait_kmcnt 0x02758; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s52759; GFX12-NEXT: v_mov_b32_e32 v0, s42760; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 32761; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)2762; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]2763; GFX12-NEXT: global_atomic_max_u64 v[0:1], v2, v[0:1], s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SE2764; GFX12-NEXT: s_wait_loadcnt 0x02765; GFX12-NEXT: global_inv scope:SCOPE_SE2766; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]2767; GFX12-NEXT: s_endpgm2768entry:2769 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index2770 %tmp0 = atomicrmw volatile umax ptr addrspace(1) %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02771 store i64 %tmp0, ptr addrspace(1) %out22772 ret void2773}2774 2775define amdgpu_kernel void @atomic_min_i64_offset(ptr addrspace(1) %out, i64 %in) {2776; CI-LABEL: atomic_min_i64_offset:2777; CI: ; %bb.0: ; %entry2778; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92779; CI-NEXT: s_waitcnt lgkmcnt(0)2780; CI-NEXT: v_mov_b32_e32 v0, s22781; CI-NEXT: v_mov_b32_e32 v1, s32782; CI-NEXT: s_mov_b32 s3, 0xf0002783; CI-NEXT: s_mov_b32 s2, -12784; CI-NEXT: buffer_atomic_smin_x2 v[0:1], off, s[0:3], 0 offset:322785; CI-NEXT: s_endpgm2786;2787; VI-LABEL: atomic_min_i64_offset:2788; VI: ; %bb.0: ; %entry2789; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242790; VI-NEXT: s_waitcnt lgkmcnt(0)2791; VI-NEXT: v_mov_b32_e32 v0, s22792; VI-NEXT: v_mov_b32_e32 v1, s32793; VI-NEXT: s_mov_b32 s3, 0xf0002794; VI-NEXT: s_mov_b32 s2, -12795; VI-NEXT: buffer_atomic_smin_x2 v[0:1], off, s[0:3], 0 offset:322796; VI-NEXT: s_endpgm2797;2798; GFX9-LABEL: atomic_min_i64_offset:2799; GFX9: ; %bb.0: ; %entry2800; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242801; GFX9-NEXT: v_mov_b32_e32 v2, 02802; GFX9-NEXT: s_waitcnt lgkmcnt(0)2803; GFX9-NEXT: v_mov_b32_e32 v0, s22804; GFX9-NEXT: v_mov_b32_e32 v1, s32805; GFX9-NEXT: global_atomic_smin_x2 v2, v[0:1], s[0:1] offset:322806; GFX9-NEXT: s_endpgm2807;2808; GFX12-LABEL: atomic_min_i64_offset:2809; GFX12: ; %bb.0: ; %entry2810; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x242811; GFX12-NEXT: s_wait_kmcnt 0x02812; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s32813; GFX12-NEXT: v_mov_b32_e32 v0, s22814; GFX12-NEXT: global_atomic_min_i64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_SE2815; GFX12-NEXT: s_wait_storecnt 0x02816; GFX12-NEXT: global_inv scope:SCOPE_SE2817; GFX12-NEXT: s_endpgm2818entry:2819 %gep = getelementptr i64, ptr addrspace(1) %out, i64 42820 %tmp0 = atomicrmw volatile min ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02821 ret void2822}2823 2824define amdgpu_kernel void @atomic_min_i64_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in) {2825; CI-LABEL: atomic_min_i64_ret_offset:2826; CI: ; %bb.0: ; %entry2827; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92828; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd2829; CI-NEXT: s_mov_b32 s7, 0xf0002830; CI-NEXT: s_mov_b32 s6, -12831; CI-NEXT: s_waitcnt lgkmcnt(0)2832; CI-NEXT: s_mov_b32 s4, s22833; CI-NEXT: s_mov_b32 s5, s32834; CI-NEXT: v_mov_b32_e32 v0, s82835; CI-NEXT: v_mov_b32_e32 v1, s92836; CI-NEXT: s_mov_b32 s2, s62837; CI-NEXT: s_mov_b32 s3, s72838; CI-NEXT: buffer_atomic_smin_x2 v[0:1], off, s[0:3], 0 offset:32 glc2839; CI-NEXT: s_waitcnt vmcnt(0)2840; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 02841; CI-NEXT: s_endpgm2842;2843; VI-LABEL: atomic_min_i64_ret_offset:2844; VI: ; %bb.0: ; %entry2845; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242846; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x342847; VI-NEXT: s_mov_b32 s7, 0xf0002848; VI-NEXT: s_mov_b32 s6, -12849; VI-NEXT: s_waitcnt lgkmcnt(0)2850; VI-NEXT: s_mov_b32 s4, s22851; VI-NEXT: s_mov_b32 s5, s32852; VI-NEXT: v_mov_b32_e32 v0, s82853; VI-NEXT: v_mov_b32_e32 v1, s92854; VI-NEXT: s_mov_b32 s2, s62855; VI-NEXT: s_mov_b32 s3, s72856; VI-NEXT: buffer_atomic_smin_x2 v[0:1], off, s[0:3], 0 offset:32 glc2857; VI-NEXT: s_waitcnt vmcnt(0)2858; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 02859; VI-NEXT: s_endpgm2860;2861; GFX9-LABEL: atomic_min_i64_ret_offset:2862; GFX9: ; %bb.0: ; %entry2863; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342864; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242865; GFX9-NEXT: v_mov_b32_e32 v2, 02866; GFX9-NEXT: s_waitcnt lgkmcnt(0)2867; GFX9-NEXT: v_mov_b32_e32 v0, s62868; GFX9-NEXT: v_mov_b32_e32 v1, s72869; GFX9-NEXT: global_atomic_smin_x2 v[0:1], v2, v[0:1], s[0:1] offset:32 glc2870; GFX9-NEXT: s_waitcnt vmcnt(0)2871; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]2872; GFX9-NEXT: s_endpgm2873;2874; GFX12-LABEL: atomic_min_i64_ret_offset:2875; GFX12: ; %bb.0: ; %entry2876; GFX12-NEXT: s_clause 0x12877; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x342878; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x242879; GFX12-NEXT: s_wait_kmcnt 0x02880; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s72881; GFX12-NEXT: v_mov_b32_e32 v0, s62882; GFX12-NEXT: global_atomic_min_i64 v[0:1], v2, v[0:1], s[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_SE2883; GFX12-NEXT: s_wait_loadcnt 0x02884; GFX12-NEXT: global_inv scope:SCOPE_SE2885; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]2886; GFX12-NEXT: s_endpgm2887entry:2888 %gep = getelementptr i64, ptr addrspace(1) %out, i64 42889 %tmp0 = atomicrmw volatile min ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02890 store i64 %tmp0, ptr addrspace(1) %out22891 ret void2892}2893 2894define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr addrspace(1) %out, i64 %in, i64 %index) {2895; CI-LABEL: atomic_min_i64_addr64_offset:2896; CI: ; %bb.0: ; %entry2897; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92898; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd2899; CI-NEXT: s_waitcnt lgkmcnt(0)2900; CI-NEXT: v_mov_b32_e32 v0, s22901; CI-NEXT: s_lshl_b64 s[4:5], s[4:5], 32902; CI-NEXT: v_mov_b32_e32 v2, s42903; CI-NEXT: v_mov_b32_e32 v1, s32904; CI-NEXT: s_mov_b32 s3, 0xf0002905; CI-NEXT: s_mov_b32 s2, 02906; CI-NEXT: v_mov_b32_e32 v3, s52907; CI-NEXT: buffer_atomic_smin_x2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:322908; CI-NEXT: s_endpgm2909;2910; VI-LABEL: atomic_min_i64_addr64_offset:2911; VI: ; %bb.0: ; %entry2912; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242913; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x342914; VI-NEXT: s_waitcnt lgkmcnt(0)2915; VI-NEXT: v_mov_b32_e32 v0, s22916; VI-NEXT: v_mov_b32_e32 v1, s32917; VI-NEXT: s_lshl_b64 s[2:3], s[4:5], 32918; VI-NEXT: s_add_u32 s0, s0, s22919; VI-NEXT: s_addc_u32 s1, s1, s32920; VI-NEXT: s_add_u32 s0, s0, 322921; VI-NEXT: s_addc_u32 s1, s1, 02922; VI-NEXT: v_mov_b32_e32 v3, s12923; VI-NEXT: v_mov_b32_e32 v2, s02924; VI-NEXT: flat_atomic_smin_x2 v[2:3], v[0:1]2925; VI-NEXT: s_endpgm2926;2927; GFX9-LABEL: atomic_min_i64_addr64_offset:2928; GFX9: ; %bb.0: ; %entry2929; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242930; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342931; GFX9-NEXT: v_mov_b32_e32 v2, 02932; GFX9-NEXT: s_waitcnt lgkmcnt(0)2933; GFX9-NEXT: v_mov_b32_e32 v0, s22934; GFX9-NEXT: v_mov_b32_e32 v1, s32935; GFX9-NEXT: s_lshl_b64 s[2:3], s[6:7], 32936; GFX9-NEXT: s_add_u32 s0, s0, s22937; GFX9-NEXT: s_addc_u32 s1, s1, s32938; GFX9-NEXT: global_atomic_smin_x2 v2, v[0:1], s[0:1] offset:322939; GFX9-NEXT: s_endpgm2940;2941; GFX12-LABEL: atomic_min_i64_addr64_offset:2942; GFX12: ; %bb.0: ; %entry2943; GFX12-NEXT: s_clause 0x12944; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x242945; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x342946; GFX12-NEXT: v_mov_b32_e32 v2, 02947; GFX12-NEXT: s_wait_kmcnt 0x02948; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s32949; GFX12-NEXT: s_lshl_b64 s[2:3], s[4:5], 32950; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)2951; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]2952; GFX12-NEXT: global_atomic_min_i64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_SE2953; GFX12-NEXT: s_wait_storecnt 0x02954; GFX12-NEXT: global_inv scope:SCOPE_SE2955; GFX12-NEXT: s_endpgm2956entry:2957 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index2958 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 42959 %tmp0 = atomicrmw volatile min ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !02960 ret void2961}2962 2963define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {2964; CI-LABEL: atomic_min_i64_ret_addr64_offset:2965; CI: ; %bb.0: ; %entry2966; CI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x92967; CI-NEXT: s_mov_b32 s11, 0xf0002968; CI-NEXT: s_mov_b32 s10, -12969; CI-NEXT: s_waitcnt lgkmcnt(0)2970; CI-NEXT: v_mov_b32_e32 v0, s42971; CI-NEXT: v_mov_b32_e32 v1, s52972; CI-NEXT: s_lshl_b64 s[4:5], s[6:7], 32973; CI-NEXT: v_mov_b32_e32 v2, s42974; CI-NEXT: s_mov_b32 s8, s22975; CI-NEXT: s_mov_b32 s9, s32976; CI-NEXT: s_mov_b32 s2, 02977; CI-NEXT: s_mov_b32 s3, s112978; CI-NEXT: v_mov_b32_e32 v3, s52979; CI-NEXT: buffer_atomic_smin_x2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:32 glc2980; CI-NEXT: s_waitcnt vmcnt(0)2981; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 02982; CI-NEXT: s_endpgm2983;2984; VI-LABEL: atomic_min_i64_ret_addr64_offset:2985; VI: ; %bb.0: ; %entry2986; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x242987; VI-NEXT: s_waitcnt lgkmcnt(0)2988; VI-NEXT: v_mov_b32_e32 v0, s42989; VI-NEXT: v_mov_b32_e32 v1, s52990; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 32991; VI-NEXT: s_add_u32 s0, s0, s42992; VI-NEXT: s_addc_u32 s1, s1, s52993; VI-NEXT: s_add_u32 s0, s0, 322994; VI-NEXT: s_addc_u32 s1, s1, 02995; VI-NEXT: v_mov_b32_e32 v3, s12996; VI-NEXT: v_mov_b32_e32 v2, s02997; VI-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3], v[0:1] glc2998; VI-NEXT: s_mov_b32 s7, 0xf0002999; VI-NEXT: s_mov_b32 s6, -13000; VI-NEXT: s_mov_b32 s4, s23001; VI-NEXT: s_mov_b32 s5, s33002; VI-NEXT: s_waitcnt vmcnt(0)3003; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 03004; VI-NEXT: s_endpgm3005;3006; GFX9-LABEL: atomic_min_i64_ret_addr64_offset:3007; GFX9: ; %bb.0: ; %entry3008; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x243009; GFX9-NEXT: v_mov_b32_e32 v2, 03010; GFX9-NEXT: s_waitcnt lgkmcnt(0)3011; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 33012; GFX9-NEXT: s_add_u32 s0, s8, s03013; GFX9-NEXT: v_mov_b32_e32 v0, s123014; GFX9-NEXT: v_mov_b32_e32 v1, s133015; GFX9-NEXT: s_addc_u32 s1, s9, s13016; GFX9-NEXT: global_atomic_smin_x2 v[0:1], v2, v[0:1], s[0:1] offset:32 glc3017; GFX9-NEXT: s_waitcnt vmcnt(0)3018; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]3019; GFX9-NEXT: s_endpgm3020;3021; GFX12-LABEL: atomic_min_i64_ret_addr64_offset:3022; GFX12: ; %bb.0: ; %entry3023; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x243024; GFX12-NEXT: s_wait_kmcnt 0x03025; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s53026; GFX12-NEXT: v_mov_b32_e32 v0, s43027; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 33028; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)3029; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]3030; GFX12-NEXT: global_atomic_min_i64 v[0:1], v2, v[0:1], s[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_SE3031; GFX12-NEXT: s_wait_loadcnt 0x03032; GFX12-NEXT: global_inv scope:SCOPE_SE3033; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]3034; GFX12-NEXT: s_endpgm3035entry:3036 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index3037 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 43038 %tmp0 = atomicrmw volatile min ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03039 store i64 %tmp0, ptr addrspace(1) %out23040 ret void3041}3042 3043define amdgpu_kernel void @atomic_min_i64(ptr addrspace(1) %out, i64 %in) {3044; CI-LABEL: atomic_min_i64:3045; CI: ; %bb.0: ; %entry3046; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93047; CI-NEXT: s_mov_b32 s7, 0xf0003048; CI-NEXT: s_mov_b32 s6, -13049; CI-NEXT: s_waitcnt lgkmcnt(0)3050; CI-NEXT: s_mov_b32 s4, s03051; CI-NEXT: s_mov_b32 s5, s13052; CI-NEXT: v_mov_b32_e32 v0, s23053; CI-NEXT: v_mov_b32_e32 v1, s33054; CI-NEXT: buffer_atomic_smin_x2 v[0:1], off, s[4:7], 03055; CI-NEXT: s_endpgm3056;3057; VI-LABEL: atomic_min_i64:3058; VI: ; %bb.0: ; %entry3059; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243060; VI-NEXT: s_mov_b32 s7, 0xf0003061; VI-NEXT: s_mov_b32 s6, -13062; VI-NEXT: s_waitcnt lgkmcnt(0)3063; VI-NEXT: s_mov_b32 s4, s03064; VI-NEXT: s_mov_b32 s5, s13065; VI-NEXT: v_mov_b32_e32 v0, s23066; VI-NEXT: v_mov_b32_e32 v1, s33067; VI-NEXT: buffer_atomic_smin_x2 v[0:1], off, s[4:7], 03068; VI-NEXT: s_endpgm3069;3070; GFX9-LABEL: atomic_min_i64:3071; GFX9: ; %bb.0: ; %entry3072; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243073; GFX9-NEXT: v_mov_b32_e32 v2, 03074; GFX9-NEXT: s_waitcnt lgkmcnt(0)3075; GFX9-NEXT: v_mov_b32_e32 v0, s23076; GFX9-NEXT: v_mov_b32_e32 v1, s33077; GFX9-NEXT: global_atomic_smin_x2 v2, v[0:1], s[0:1]3078; GFX9-NEXT: s_endpgm3079;3080; GFX12-LABEL: atomic_min_i64:3081; GFX12: ; %bb.0: ; %entry3082; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x243083; GFX12-NEXT: s_wait_kmcnt 0x03084; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s33085; GFX12-NEXT: v_mov_b32_e32 v0, s23086; GFX12-NEXT: global_atomic_min_i64 v2, v[0:1], s[0:1] scope:SCOPE_SE3087; GFX12-NEXT: s_wait_storecnt 0x03088; GFX12-NEXT: global_inv scope:SCOPE_SE3089; GFX12-NEXT: s_endpgm3090entry:3091 %tmp0 = atomicrmw volatile min ptr addrspace(1) %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03092 ret void3093}3094 3095define amdgpu_kernel void @atomic_min_i64_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in) {3096; CI-LABEL: atomic_min_i64_ret:3097; CI: ; %bb.0: ; %entry3098; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93099; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd3100; CI-NEXT: s_mov_b32 s7, 0xf0003101; CI-NEXT: s_mov_b32 s6, -13102; CI-NEXT: s_waitcnt lgkmcnt(0)3103; CI-NEXT: s_mov_b32 s4, s03104; CI-NEXT: s_mov_b32 s5, s13105; CI-NEXT: v_mov_b32_e32 v0, s83106; CI-NEXT: v_mov_b32_e32 v1, s93107; CI-NEXT: buffer_atomic_smin_x2 v[0:1], off, s[4:7], 0 glc3108; CI-NEXT: s_mov_b32 s4, s23109; CI-NEXT: s_mov_b32 s5, s33110; CI-NEXT: s_waitcnt vmcnt(0)3111; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 03112; CI-NEXT: s_endpgm3113;3114; VI-LABEL: atomic_min_i64_ret:3115; VI: ; %bb.0: ; %entry3116; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243117; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x343118; VI-NEXT: s_mov_b32 s7, 0xf0003119; VI-NEXT: s_mov_b32 s6, -13120; VI-NEXT: s_waitcnt lgkmcnt(0)3121; VI-NEXT: s_mov_b32 s4, s03122; VI-NEXT: s_mov_b32 s5, s13123; VI-NEXT: v_mov_b32_e32 v0, s83124; VI-NEXT: v_mov_b32_e32 v1, s93125; VI-NEXT: buffer_atomic_smin_x2 v[0:1], off, s[4:7], 0 glc3126; VI-NEXT: s_mov_b32 s4, s23127; VI-NEXT: s_mov_b32 s5, s33128; VI-NEXT: s_waitcnt vmcnt(0)3129; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 03130; VI-NEXT: s_endpgm3131;3132; GFX9-LABEL: atomic_min_i64_ret:3133; GFX9: ; %bb.0: ; %entry3134; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343135; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243136; GFX9-NEXT: v_mov_b32_e32 v2, 03137; GFX9-NEXT: s_waitcnt lgkmcnt(0)3138; GFX9-NEXT: v_mov_b32_e32 v0, s63139; GFX9-NEXT: v_mov_b32_e32 v1, s73140; GFX9-NEXT: global_atomic_smin_x2 v[0:1], v2, v[0:1], s[0:1] glc3141; GFX9-NEXT: s_waitcnt vmcnt(0)3142; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]3143; GFX9-NEXT: s_endpgm3144;3145; GFX12-LABEL: atomic_min_i64_ret:3146; GFX12: ; %bb.0: ; %entry3147; GFX12-NEXT: s_clause 0x13148; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x343149; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x243150; GFX12-NEXT: s_wait_kmcnt 0x03151; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s73152; GFX12-NEXT: v_mov_b32_e32 v0, s63153; GFX12-NEXT: global_atomic_min_i64 v[0:1], v2, v[0:1], s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SE3154; GFX12-NEXT: s_wait_loadcnt 0x03155; GFX12-NEXT: global_inv scope:SCOPE_SE3156; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]3157; GFX12-NEXT: s_endpgm3158entry:3159 %tmp0 = atomicrmw volatile min ptr addrspace(1) %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03160 store i64 %tmp0, ptr addrspace(1) %out23161 ret void3162}3163 3164define amdgpu_kernel void @atomic_min_i64_addr64(ptr addrspace(1) %out, i64 %in, i64 %index) {3165; CI-LABEL: atomic_min_i64_addr64:3166; CI: ; %bb.0: ; %entry3167; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93168; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd3169; CI-NEXT: s_waitcnt lgkmcnt(0)3170; CI-NEXT: v_mov_b32_e32 v0, s23171; CI-NEXT: s_lshl_b64 s[4:5], s[4:5], 33172; CI-NEXT: v_mov_b32_e32 v2, s43173; CI-NEXT: v_mov_b32_e32 v1, s33174; CI-NEXT: s_mov_b32 s3, 0xf0003175; CI-NEXT: s_mov_b32 s2, 03176; CI-NEXT: v_mov_b32_e32 v3, s53177; CI-NEXT: buffer_atomic_smin_x2 v[0:1], v[2:3], s[0:3], 0 addr643178; CI-NEXT: s_endpgm3179;3180; VI-LABEL: atomic_min_i64_addr64:3181; VI: ; %bb.0: ; %entry3182; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243183; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x343184; VI-NEXT: s_waitcnt lgkmcnt(0)3185; VI-NEXT: v_mov_b32_e32 v0, s23186; VI-NEXT: v_mov_b32_e32 v1, s33187; VI-NEXT: s_lshl_b64 s[2:3], s[4:5], 33188; VI-NEXT: s_add_u32 s0, s0, s23189; VI-NEXT: s_addc_u32 s1, s1, s33190; VI-NEXT: v_mov_b32_e32 v3, s13191; VI-NEXT: v_mov_b32_e32 v2, s03192; VI-NEXT: flat_atomic_smin_x2 v[2:3], v[0:1]3193; VI-NEXT: s_endpgm3194;3195; GFX9-LABEL: atomic_min_i64_addr64:3196; GFX9: ; %bb.0: ; %entry3197; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243198; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343199; GFX9-NEXT: v_mov_b32_e32 v2, 03200; GFX9-NEXT: s_waitcnt lgkmcnt(0)3201; GFX9-NEXT: v_mov_b32_e32 v0, s23202; GFX9-NEXT: v_mov_b32_e32 v1, s33203; GFX9-NEXT: s_lshl_b64 s[2:3], s[6:7], 33204; GFX9-NEXT: s_add_u32 s0, s0, s23205; GFX9-NEXT: s_addc_u32 s1, s1, s33206; GFX9-NEXT: global_atomic_smin_x2 v2, v[0:1], s[0:1]3207; GFX9-NEXT: s_endpgm3208;3209; GFX12-LABEL: atomic_min_i64_addr64:3210; GFX12: ; %bb.0: ; %entry3211; GFX12-NEXT: s_clause 0x13212; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x243213; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x343214; GFX12-NEXT: v_mov_b32_e32 v2, 03215; GFX12-NEXT: s_wait_kmcnt 0x03216; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s33217; GFX12-NEXT: s_lshl_b64 s[2:3], s[4:5], 33218; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)3219; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]3220; GFX12-NEXT: global_atomic_min_i64 v2, v[0:1], s[0:1] scope:SCOPE_SE3221; GFX12-NEXT: s_wait_storecnt 0x03222; GFX12-NEXT: global_inv scope:SCOPE_SE3223; GFX12-NEXT: s_endpgm3224entry:3225 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index3226 %tmp0 = atomicrmw volatile min ptr addrspace(1) %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03227 ret void3228}3229 3230define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {3231; CI-LABEL: atomic_min_i64_ret_addr64:3232; CI: ; %bb.0: ; %entry3233; CI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x93234; CI-NEXT: s_mov_b32 s11, 0xf0003235; CI-NEXT: s_mov_b32 s10, -13236; CI-NEXT: s_waitcnt lgkmcnt(0)3237; CI-NEXT: v_mov_b32_e32 v0, s43238; CI-NEXT: v_mov_b32_e32 v1, s53239; CI-NEXT: s_lshl_b64 s[4:5], s[6:7], 33240; CI-NEXT: v_mov_b32_e32 v2, s43241; CI-NEXT: s_mov_b32 s8, s23242; CI-NEXT: s_mov_b32 s9, s33243; CI-NEXT: s_mov_b32 s2, 03244; CI-NEXT: s_mov_b32 s3, s113245; CI-NEXT: v_mov_b32_e32 v3, s53246; CI-NEXT: buffer_atomic_smin_x2 v[0:1], v[2:3], s[0:3], 0 addr64 glc3247; CI-NEXT: s_waitcnt vmcnt(0)3248; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 03249; CI-NEXT: s_endpgm3250;3251; VI-LABEL: atomic_min_i64_ret_addr64:3252; VI: ; %bb.0: ; %entry3253; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x243254; VI-NEXT: s_waitcnt lgkmcnt(0)3255; VI-NEXT: v_mov_b32_e32 v0, s43256; VI-NEXT: v_mov_b32_e32 v1, s53257; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 33258; VI-NEXT: s_add_u32 s0, s0, s43259; VI-NEXT: s_addc_u32 s1, s1, s53260; VI-NEXT: v_mov_b32_e32 v3, s13261; VI-NEXT: v_mov_b32_e32 v2, s03262; VI-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3], v[0:1] glc3263; VI-NEXT: s_mov_b32 s7, 0xf0003264; VI-NEXT: s_mov_b32 s6, -13265; VI-NEXT: s_mov_b32 s4, s23266; VI-NEXT: s_mov_b32 s5, s33267; VI-NEXT: s_waitcnt vmcnt(0)3268; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 03269; VI-NEXT: s_endpgm3270;3271; GFX9-LABEL: atomic_min_i64_ret_addr64:3272; GFX9: ; %bb.0: ; %entry3273; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x243274; GFX9-NEXT: v_mov_b32_e32 v2, 03275; GFX9-NEXT: s_waitcnt lgkmcnt(0)3276; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 33277; GFX9-NEXT: s_add_u32 s0, s8, s03278; GFX9-NEXT: v_mov_b32_e32 v0, s123279; GFX9-NEXT: v_mov_b32_e32 v1, s133280; GFX9-NEXT: s_addc_u32 s1, s9, s13281; GFX9-NEXT: global_atomic_smin_x2 v[0:1], v2, v[0:1], s[0:1] glc3282; GFX9-NEXT: s_waitcnt vmcnt(0)3283; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]3284; GFX9-NEXT: s_endpgm3285;3286; GFX12-LABEL: atomic_min_i64_ret_addr64:3287; GFX12: ; %bb.0: ; %entry3288; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x243289; GFX12-NEXT: s_wait_kmcnt 0x03290; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s53291; GFX12-NEXT: v_mov_b32_e32 v0, s43292; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 33293; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)3294; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]3295; GFX12-NEXT: global_atomic_min_i64 v[0:1], v2, v[0:1], s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SE3296; GFX12-NEXT: s_wait_loadcnt 0x03297; GFX12-NEXT: global_inv scope:SCOPE_SE3298; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]3299; GFX12-NEXT: s_endpgm3300entry:3301 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index3302 %tmp0 = atomicrmw volatile min ptr addrspace(1) %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03303 store i64 %tmp0, ptr addrspace(1) %out23304 ret void3305}3306 3307define amdgpu_kernel void @atomic_umin_i64_offset(ptr addrspace(1) %out, i64 %in) {3308; CI-LABEL: atomic_umin_i64_offset:3309; CI: ; %bb.0: ; %entry3310; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93311; CI-NEXT: s_waitcnt lgkmcnt(0)3312; CI-NEXT: v_mov_b32_e32 v0, s23313; CI-NEXT: v_mov_b32_e32 v1, s33314; CI-NEXT: s_mov_b32 s3, 0xf0003315; CI-NEXT: s_mov_b32 s2, -13316; CI-NEXT: buffer_atomic_umin_x2 v[0:1], off, s[0:3], 0 offset:323317; CI-NEXT: s_endpgm3318;3319; VI-LABEL: atomic_umin_i64_offset:3320; VI: ; %bb.0: ; %entry3321; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243322; VI-NEXT: s_waitcnt lgkmcnt(0)3323; VI-NEXT: v_mov_b32_e32 v0, s23324; VI-NEXT: v_mov_b32_e32 v1, s33325; VI-NEXT: s_mov_b32 s3, 0xf0003326; VI-NEXT: s_mov_b32 s2, -13327; VI-NEXT: buffer_atomic_umin_x2 v[0:1], off, s[0:3], 0 offset:323328; VI-NEXT: s_endpgm3329;3330; GFX9-LABEL: atomic_umin_i64_offset:3331; GFX9: ; %bb.0: ; %entry3332; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243333; GFX9-NEXT: v_mov_b32_e32 v2, 03334; GFX9-NEXT: s_waitcnt lgkmcnt(0)3335; GFX9-NEXT: v_mov_b32_e32 v0, s23336; GFX9-NEXT: v_mov_b32_e32 v1, s33337; GFX9-NEXT: global_atomic_umin_x2 v2, v[0:1], s[0:1] offset:323338; GFX9-NEXT: s_endpgm3339;3340; GFX12-LABEL: atomic_umin_i64_offset:3341; GFX12: ; %bb.0: ; %entry3342; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x243343; GFX12-NEXT: s_wait_kmcnt 0x03344; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s33345; GFX12-NEXT: v_mov_b32_e32 v0, s23346; GFX12-NEXT: global_atomic_min_u64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_SE3347; GFX12-NEXT: s_wait_storecnt 0x03348; GFX12-NEXT: global_inv scope:SCOPE_SE3349; GFX12-NEXT: s_endpgm3350entry:3351 %gep = getelementptr i64, ptr addrspace(1) %out, i64 43352 %tmp0 = atomicrmw volatile umin ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03353 ret void3354}3355 3356define amdgpu_kernel void @atomic_umin_i64_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in) {3357; CI-LABEL: atomic_umin_i64_ret_offset:3358; CI: ; %bb.0: ; %entry3359; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93360; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd3361; CI-NEXT: s_mov_b32 s7, 0xf0003362; CI-NEXT: s_mov_b32 s6, -13363; CI-NEXT: s_waitcnt lgkmcnt(0)3364; CI-NEXT: s_mov_b32 s4, s23365; CI-NEXT: s_mov_b32 s5, s33366; CI-NEXT: v_mov_b32_e32 v0, s83367; CI-NEXT: v_mov_b32_e32 v1, s93368; CI-NEXT: s_mov_b32 s2, s63369; CI-NEXT: s_mov_b32 s3, s73370; CI-NEXT: buffer_atomic_umin_x2 v[0:1], off, s[0:3], 0 offset:32 glc3371; CI-NEXT: s_waitcnt vmcnt(0)3372; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 03373; CI-NEXT: s_endpgm3374;3375; VI-LABEL: atomic_umin_i64_ret_offset:3376; VI: ; %bb.0: ; %entry3377; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243378; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x343379; VI-NEXT: s_mov_b32 s7, 0xf0003380; VI-NEXT: s_mov_b32 s6, -13381; VI-NEXT: s_waitcnt lgkmcnt(0)3382; VI-NEXT: s_mov_b32 s4, s23383; VI-NEXT: s_mov_b32 s5, s33384; VI-NEXT: v_mov_b32_e32 v0, s83385; VI-NEXT: v_mov_b32_e32 v1, s93386; VI-NEXT: s_mov_b32 s2, s63387; VI-NEXT: s_mov_b32 s3, s73388; VI-NEXT: buffer_atomic_umin_x2 v[0:1], off, s[0:3], 0 offset:32 glc3389; VI-NEXT: s_waitcnt vmcnt(0)3390; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 03391; VI-NEXT: s_endpgm3392;3393; GFX9-LABEL: atomic_umin_i64_ret_offset:3394; GFX9: ; %bb.0: ; %entry3395; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343396; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243397; GFX9-NEXT: v_mov_b32_e32 v2, 03398; GFX9-NEXT: s_waitcnt lgkmcnt(0)3399; GFX9-NEXT: v_mov_b32_e32 v0, s63400; GFX9-NEXT: v_mov_b32_e32 v1, s73401; GFX9-NEXT: global_atomic_umin_x2 v[0:1], v2, v[0:1], s[0:1] offset:32 glc3402; GFX9-NEXT: s_waitcnt vmcnt(0)3403; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]3404; GFX9-NEXT: s_endpgm3405;3406; GFX12-LABEL: atomic_umin_i64_ret_offset:3407; GFX12: ; %bb.0: ; %entry3408; GFX12-NEXT: s_clause 0x13409; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x343410; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x243411; GFX12-NEXT: s_wait_kmcnt 0x03412; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s73413; GFX12-NEXT: v_mov_b32_e32 v0, s63414; GFX12-NEXT: global_atomic_min_u64 v[0:1], v2, v[0:1], s[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_SE3415; GFX12-NEXT: s_wait_loadcnt 0x03416; GFX12-NEXT: global_inv scope:SCOPE_SE3417; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]3418; GFX12-NEXT: s_endpgm3419entry:3420 %gep = getelementptr i64, ptr addrspace(1) %out, i64 43421 %tmp0 = atomicrmw volatile umin ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03422 store i64 %tmp0, ptr addrspace(1) %out23423 ret void3424}3425 3426define amdgpu_kernel void @atomic_umin_i64_addr64_offset(ptr addrspace(1) %out, i64 %in, i64 %index) {3427; CI-LABEL: atomic_umin_i64_addr64_offset:3428; CI: ; %bb.0: ; %entry3429; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93430; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd3431; CI-NEXT: s_waitcnt lgkmcnt(0)3432; CI-NEXT: v_mov_b32_e32 v0, s23433; CI-NEXT: s_lshl_b64 s[4:5], s[4:5], 33434; CI-NEXT: v_mov_b32_e32 v2, s43435; CI-NEXT: v_mov_b32_e32 v1, s33436; CI-NEXT: s_mov_b32 s3, 0xf0003437; CI-NEXT: s_mov_b32 s2, 03438; CI-NEXT: v_mov_b32_e32 v3, s53439; CI-NEXT: buffer_atomic_umin_x2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:323440; CI-NEXT: s_endpgm3441;3442; VI-LABEL: atomic_umin_i64_addr64_offset:3443; VI: ; %bb.0: ; %entry3444; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243445; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x343446; VI-NEXT: s_waitcnt lgkmcnt(0)3447; VI-NEXT: v_mov_b32_e32 v0, s23448; VI-NEXT: v_mov_b32_e32 v1, s33449; VI-NEXT: s_lshl_b64 s[2:3], s[4:5], 33450; VI-NEXT: s_add_u32 s0, s0, s23451; VI-NEXT: s_addc_u32 s1, s1, s33452; VI-NEXT: s_add_u32 s0, s0, 323453; VI-NEXT: s_addc_u32 s1, s1, 03454; VI-NEXT: v_mov_b32_e32 v3, s13455; VI-NEXT: v_mov_b32_e32 v2, s03456; VI-NEXT: flat_atomic_umin_x2 v[2:3], v[0:1]3457; VI-NEXT: s_endpgm3458;3459; GFX9-LABEL: atomic_umin_i64_addr64_offset:3460; GFX9: ; %bb.0: ; %entry3461; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243462; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343463; GFX9-NEXT: v_mov_b32_e32 v2, 03464; GFX9-NEXT: s_waitcnt lgkmcnt(0)3465; GFX9-NEXT: v_mov_b32_e32 v0, s23466; GFX9-NEXT: v_mov_b32_e32 v1, s33467; GFX9-NEXT: s_lshl_b64 s[2:3], s[6:7], 33468; GFX9-NEXT: s_add_u32 s0, s0, s23469; GFX9-NEXT: s_addc_u32 s1, s1, s33470; GFX9-NEXT: global_atomic_umin_x2 v2, v[0:1], s[0:1] offset:323471; GFX9-NEXT: s_endpgm3472;3473; GFX12-LABEL: atomic_umin_i64_addr64_offset:3474; GFX12: ; %bb.0: ; %entry3475; GFX12-NEXT: s_clause 0x13476; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x243477; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x343478; GFX12-NEXT: v_mov_b32_e32 v2, 03479; GFX12-NEXT: s_wait_kmcnt 0x03480; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s33481; GFX12-NEXT: s_lshl_b64 s[2:3], s[4:5], 33482; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)3483; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]3484; GFX12-NEXT: global_atomic_min_u64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_SE3485; GFX12-NEXT: s_wait_storecnt 0x03486; GFX12-NEXT: global_inv scope:SCOPE_SE3487; GFX12-NEXT: s_endpgm3488entry:3489 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index3490 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 43491 %tmp0 = atomicrmw volatile umin ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03492 ret void3493}3494 3495define amdgpu_kernel void @atomic_umin_i64_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {3496; CI-LABEL: atomic_umin_i64_ret_addr64_offset:3497; CI: ; %bb.0: ; %entry3498; CI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x93499; CI-NEXT: s_mov_b32 s11, 0xf0003500; CI-NEXT: s_mov_b32 s10, -13501; CI-NEXT: s_waitcnt lgkmcnt(0)3502; CI-NEXT: v_mov_b32_e32 v0, s43503; CI-NEXT: v_mov_b32_e32 v1, s53504; CI-NEXT: s_lshl_b64 s[4:5], s[6:7], 33505; CI-NEXT: v_mov_b32_e32 v2, s43506; CI-NEXT: s_mov_b32 s8, s23507; CI-NEXT: s_mov_b32 s9, s33508; CI-NEXT: s_mov_b32 s2, 03509; CI-NEXT: s_mov_b32 s3, s113510; CI-NEXT: v_mov_b32_e32 v3, s53511; CI-NEXT: buffer_atomic_umin_x2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:32 glc3512; CI-NEXT: s_waitcnt vmcnt(0)3513; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 03514; CI-NEXT: s_endpgm3515;3516; VI-LABEL: atomic_umin_i64_ret_addr64_offset:3517; VI: ; %bb.0: ; %entry3518; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x243519; VI-NEXT: s_waitcnt lgkmcnt(0)3520; VI-NEXT: v_mov_b32_e32 v0, s43521; VI-NEXT: v_mov_b32_e32 v1, s53522; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 33523; VI-NEXT: s_add_u32 s0, s0, s43524; VI-NEXT: s_addc_u32 s1, s1, s53525; VI-NEXT: s_add_u32 s0, s0, 323526; VI-NEXT: s_addc_u32 s1, s1, 03527; VI-NEXT: v_mov_b32_e32 v3, s13528; VI-NEXT: v_mov_b32_e32 v2, s03529; VI-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3], v[0:1] glc3530; VI-NEXT: s_mov_b32 s7, 0xf0003531; VI-NEXT: s_mov_b32 s6, -13532; VI-NEXT: s_mov_b32 s4, s23533; VI-NEXT: s_mov_b32 s5, s33534; VI-NEXT: s_waitcnt vmcnt(0)3535; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 03536; VI-NEXT: s_endpgm3537;3538; GFX9-LABEL: atomic_umin_i64_ret_addr64_offset:3539; GFX9: ; %bb.0: ; %entry3540; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x243541; GFX9-NEXT: v_mov_b32_e32 v2, 03542; GFX9-NEXT: s_waitcnt lgkmcnt(0)3543; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 33544; GFX9-NEXT: s_add_u32 s0, s8, s03545; GFX9-NEXT: v_mov_b32_e32 v0, s123546; GFX9-NEXT: v_mov_b32_e32 v1, s133547; GFX9-NEXT: s_addc_u32 s1, s9, s13548; GFX9-NEXT: global_atomic_umin_x2 v[0:1], v2, v[0:1], s[0:1] offset:32 glc3549; GFX9-NEXT: s_waitcnt vmcnt(0)3550; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]3551; GFX9-NEXT: s_endpgm3552;3553; GFX12-LABEL: atomic_umin_i64_ret_addr64_offset:3554; GFX12: ; %bb.0: ; %entry3555; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x243556; GFX12-NEXT: s_wait_kmcnt 0x03557; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s53558; GFX12-NEXT: v_mov_b32_e32 v0, s43559; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 33560; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)3561; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]3562; GFX12-NEXT: global_atomic_min_u64 v[0:1], v2, v[0:1], s[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_SE3563; GFX12-NEXT: s_wait_loadcnt 0x03564; GFX12-NEXT: global_inv scope:SCOPE_SE3565; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]3566; GFX12-NEXT: s_endpgm3567entry:3568 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index3569 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 43570 %tmp0 = atomicrmw volatile umin ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03571 store i64 %tmp0, ptr addrspace(1) %out23572 ret void3573}3574 3575define amdgpu_kernel void @atomic_umin_i64(ptr addrspace(1) %out, i64 %in) {3576; CI-LABEL: atomic_umin_i64:3577; CI: ; %bb.0: ; %entry3578; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93579; CI-NEXT: s_mov_b32 s7, 0xf0003580; CI-NEXT: s_mov_b32 s6, -13581; CI-NEXT: s_waitcnt lgkmcnt(0)3582; CI-NEXT: s_mov_b32 s4, s03583; CI-NEXT: s_mov_b32 s5, s13584; CI-NEXT: v_mov_b32_e32 v0, s23585; CI-NEXT: v_mov_b32_e32 v1, s33586; CI-NEXT: buffer_atomic_umin_x2 v[0:1], off, s[4:7], 03587; CI-NEXT: s_endpgm3588;3589; VI-LABEL: atomic_umin_i64:3590; VI: ; %bb.0: ; %entry3591; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243592; VI-NEXT: s_mov_b32 s7, 0xf0003593; VI-NEXT: s_mov_b32 s6, -13594; VI-NEXT: s_waitcnt lgkmcnt(0)3595; VI-NEXT: s_mov_b32 s4, s03596; VI-NEXT: s_mov_b32 s5, s13597; VI-NEXT: v_mov_b32_e32 v0, s23598; VI-NEXT: v_mov_b32_e32 v1, s33599; VI-NEXT: buffer_atomic_umin_x2 v[0:1], off, s[4:7], 03600; VI-NEXT: s_endpgm3601;3602; GFX9-LABEL: atomic_umin_i64:3603; GFX9: ; %bb.0: ; %entry3604; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243605; GFX9-NEXT: v_mov_b32_e32 v2, 03606; GFX9-NEXT: s_waitcnt lgkmcnt(0)3607; GFX9-NEXT: v_mov_b32_e32 v0, s23608; GFX9-NEXT: v_mov_b32_e32 v1, s33609; GFX9-NEXT: global_atomic_umin_x2 v2, v[0:1], s[0:1]3610; GFX9-NEXT: s_endpgm3611;3612; GFX12-LABEL: atomic_umin_i64:3613; GFX12: ; %bb.0: ; %entry3614; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x243615; GFX12-NEXT: s_wait_kmcnt 0x03616; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s33617; GFX12-NEXT: v_mov_b32_e32 v0, s23618; GFX12-NEXT: global_atomic_min_u64 v2, v[0:1], s[0:1] scope:SCOPE_SE3619; GFX12-NEXT: s_wait_storecnt 0x03620; GFX12-NEXT: global_inv scope:SCOPE_SE3621; GFX12-NEXT: s_endpgm3622entry:3623 %tmp0 = atomicrmw volatile umin ptr addrspace(1) %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03624 ret void3625}3626 3627define amdgpu_kernel void @atomic_umin_i64_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in) {3628; CI-LABEL: atomic_umin_i64_ret:3629; CI: ; %bb.0: ; %entry3630; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93631; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd3632; CI-NEXT: s_mov_b32 s7, 0xf0003633; CI-NEXT: s_mov_b32 s6, -13634; CI-NEXT: s_waitcnt lgkmcnt(0)3635; CI-NEXT: s_mov_b32 s4, s03636; CI-NEXT: s_mov_b32 s5, s13637; CI-NEXT: v_mov_b32_e32 v0, s83638; CI-NEXT: v_mov_b32_e32 v1, s93639; CI-NEXT: buffer_atomic_umin_x2 v[0:1], off, s[4:7], 0 glc3640; CI-NEXT: s_mov_b32 s4, s23641; CI-NEXT: s_mov_b32 s5, s33642; CI-NEXT: s_waitcnt vmcnt(0)3643; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 03644; CI-NEXT: s_endpgm3645;3646; VI-LABEL: atomic_umin_i64_ret:3647; VI: ; %bb.0: ; %entry3648; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243649; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x343650; VI-NEXT: s_mov_b32 s7, 0xf0003651; VI-NEXT: s_mov_b32 s6, -13652; VI-NEXT: s_waitcnt lgkmcnt(0)3653; VI-NEXT: s_mov_b32 s4, s03654; VI-NEXT: s_mov_b32 s5, s13655; VI-NEXT: v_mov_b32_e32 v0, s83656; VI-NEXT: v_mov_b32_e32 v1, s93657; VI-NEXT: buffer_atomic_umin_x2 v[0:1], off, s[4:7], 0 glc3658; VI-NEXT: s_mov_b32 s4, s23659; VI-NEXT: s_mov_b32 s5, s33660; VI-NEXT: s_waitcnt vmcnt(0)3661; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 03662; VI-NEXT: s_endpgm3663;3664; GFX9-LABEL: atomic_umin_i64_ret:3665; GFX9: ; %bb.0: ; %entry3666; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343667; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243668; GFX9-NEXT: v_mov_b32_e32 v2, 03669; GFX9-NEXT: s_waitcnt lgkmcnt(0)3670; GFX9-NEXT: v_mov_b32_e32 v0, s63671; GFX9-NEXT: v_mov_b32_e32 v1, s73672; GFX9-NEXT: global_atomic_umin_x2 v[0:1], v2, v[0:1], s[0:1] glc3673; GFX9-NEXT: s_waitcnt vmcnt(0)3674; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]3675; GFX9-NEXT: s_endpgm3676;3677; GFX12-LABEL: atomic_umin_i64_ret:3678; GFX12: ; %bb.0: ; %entry3679; GFX12-NEXT: s_clause 0x13680; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x343681; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x243682; GFX12-NEXT: s_wait_kmcnt 0x03683; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s73684; GFX12-NEXT: v_mov_b32_e32 v0, s63685; GFX12-NEXT: global_atomic_min_u64 v[0:1], v2, v[0:1], s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SE3686; GFX12-NEXT: s_wait_loadcnt 0x03687; GFX12-NEXT: global_inv scope:SCOPE_SE3688; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]3689; GFX12-NEXT: s_endpgm3690entry:3691 %tmp0 = atomicrmw volatile umin ptr addrspace(1) %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03692 store i64 %tmp0, ptr addrspace(1) %out23693 ret void3694}3695 3696define amdgpu_kernel void @atomic_umin_i64_addr64(ptr addrspace(1) %out, i64 %in, i64 %index) {3697; CI-LABEL: atomic_umin_i64_addr64:3698; CI: ; %bb.0: ; %entry3699; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93700; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd3701; CI-NEXT: s_waitcnt lgkmcnt(0)3702; CI-NEXT: v_mov_b32_e32 v0, s23703; CI-NEXT: s_lshl_b64 s[4:5], s[4:5], 33704; CI-NEXT: v_mov_b32_e32 v2, s43705; CI-NEXT: v_mov_b32_e32 v1, s33706; CI-NEXT: s_mov_b32 s3, 0xf0003707; CI-NEXT: s_mov_b32 s2, 03708; CI-NEXT: v_mov_b32_e32 v3, s53709; CI-NEXT: buffer_atomic_umin_x2 v[0:1], v[2:3], s[0:3], 0 addr643710; CI-NEXT: s_endpgm3711;3712; VI-LABEL: atomic_umin_i64_addr64:3713; VI: ; %bb.0: ; %entry3714; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243715; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x343716; VI-NEXT: s_waitcnt lgkmcnt(0)3717; VI-NEXT: v_mov_b32_e32 v0, s23718; VI-NEXT: v_mov_b32_e32 v1, s33719; VI-NEXT: s_lshl_b64 s[2:3], s[4:5], 33720; VI-NEXT: s_add_u32 s0, s0, s23721; VI-NEXT: s_addc_u32 s1, s1, s33722; VI-NEXT: v_mov_b32_e32 v3, s13723; VI-NEXT: v_mov_b32_e32 v2, s03724; VI-NEXT: flat_atomic_umin_x2 v[2:3], v[0:1]3725; VI-NEXT: s_endpgm3726;3727; GFX9-LABEL: atomic_umin_i64_addr64:3728; GFX9: ; %bb.0: ; %entry3729; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243730; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343731; GFX9-NEXT: v_mov_b32_e32 v2, 03732; GFX9-NEXT: s_waitcnt lgkmcnt(0)3733; GFX9-NEXT: v_mov_b32_e32 v0, s23734; GFX9-NEXT: v_mov_b32_e32 v1, s33735; GFX9-NEXT: s_lshl_b64 s[2:3], s[6:7], 33736; GFX9-NEXT: s_add_u32 s0, s0, s23737; GFX9-NEXT: s_addc_u32 s1, s1, s33738; GFX9-NEXT: global_atomic_umin_x2 v2, v[0:1], s[0:1]3739; GFX9-NEXT: s_endpgm3740;3741; GFX12-LABEL: atomic_umin_i64_addr64:3742; GFX12: ; %bb.0: ; %entry3743; GFX12-NEXT: s_clause 0x13744; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x243745; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x343746; GFX12-NEXT: v_mov_b32_e32 v2, 03747; GFX12-NEXT: s_wait_kmcnt 0x03748; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s33749; GFX12-NEXT: s_lshl_b64 s[2:3], s[4:5], 33750; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)3751; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]3752; GFX12-NEXT: global_atomic_min_u64 v2, v[0:1], s[0:1] scope:SCOPE_SE3753; GFX12-NEXT: s_wait_storecnt 0x03754; GFX12-NEXT: global_inv scope:SCOPE_SE3755; GFX12-NEXT: s_endpgm3756entry:3757 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index3758 %tmp0 = atomicrmw volatile umin ptr addrspace(1) %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03759 ret void3760}3761 3762define amdgpu_kernel void @atomic_umin_i64_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {3763; CI-LABEL: atomic_umin_i64_ret_addr64:3764; CI: ; %bb.0: ; %entry3765; CI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x93766; CI-NEXT: s_mov_b32 s11, 0xf0003767; CI-NEXT: s_mov_b32 s10, -13768; CI-NEXT: s_waitcnt lgkmcnt(0)3769; CI-NEXT: v_mov_b32_e32 v0, s43770; CI-NEXT: v_mov_b32_e32 v1, s53771; CI-NEXT: s_lshl_b64 s[4:5], s[6:7], 33772; CI-NEXT: v_mov_b32_e32 v2, s43773; CI-NEXT: s_mov_b32 s8, s23774; CI-NEXT: s_mov_b32 s9, s33775; CI-NEXT: s_mov_b32 s2, 03776; CI-NEXT: s_mov_b32 s3, s113777; CI-NEXT: v_mov_b32_e32 v3, s53778; CI-NEXT: buffer_atomic_umin_x2 v[0:1], v[2:3], s[0:3], 0 addr64 glc3779; CI-NEXT: s_waitcnt vmcnt(0)3780; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 03781; CI-NEXT: s_endpgm3782;3783; VI-LABEL: atomic_umin_i64_ret_addr64:3784; VI: ; %bb.0: ; %entry3785; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x243786; VI-NEXT: s_waitcnt lgkmcnt(0)3787; VI-NEXT: v_mov_b32_e32 v0, s43788; VI-NEXT: v_mov_b32_e32 v1, s53789; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 33790; VI-NEXT: s_add_u32 s0, s0, s43791; VI-NEXT: s_addc_u32 s1, s1, s53792; VI-NEXT: v_mov_b32_e32 v3, s13793; VI-NEXT: v_mov_b32_e32 v2, s03794; VI-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3], v[0:1] glc3795; VI-NEXT: s_mov_b32 s7, 0xf0003796; VI-NEXT: s_mov_b32 s6, -13797; VI-NEXT: s_mov_b32 s4, s23798; VI-NEXT: s_mov_b32 s5, s33799; VI-NEXT: s_waitcnt vmcnt(0)3800; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 03801; VI-NEXT: s_endpgm3802;3803; GFX9-LABEL: atomic_umin_i64_ret_addr64:3804; GFX9: ; %bb.0: ; %entry3805; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x243806; GFX9-NEXT: v_mov_b32_e32 v2, 03807; GFX9-NEXT: s_waitcnt lgkmcnt(0)3808; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 33809; GFX9-NEXT: s_add_u32 s0, s8, s03810; GFX9-NEXT: v_mov_b32_e32 v0, s123811; GFX9-NEXT: v_mov_b32_e32 v1, s133812; GFX9-NEXT: s_addc_u32 s1, s9, s13813; GFX9-NEXT: global_atomic_umin_x2 v[0:1], v2, v[0:1], s[0:1] glc3814; GFX9-NEXT: s_waitcnt vmcnt(0)3815; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]3816; GFX9-NEXT: s_endpgm3817;3818; GFX12-LABEL: atomic_umin_i64_ret_addr64:3819; GFX12: ; %bb.0: ; %entry3820; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x243821; GFX12-NEXT: s_wait_kmcnt 0x03822; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s53823; GFX12-NEXT: v_mov_b32_e32 v0, s43824; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 33825; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)3826; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]3827; GFX12-NEXT: global_atomic_min_u64 v[0:1], v2, v[0:1], s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SE3828; GFX12-NEXT: s_wait_loadcnt 0x03829; GFX12-NEXT: global_inv scope:SCOPE_SE3830; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]3831; GFX12-NEXT: s_endpgm3832entry:3833 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index3834 %tmp0 = atomicrmw volatile umin ptr addrspace(1) %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !03835 store i64 %tmp0, ptr addrspace(1) %out23836 ret void3837}3838 3839define amdgpu_kernel void @atomic_or_i64_offset(ptr addrspace(1) %out, i64 %in) {3840; CI-LABEL: atomic_or_i64_offset:3841; CI: ; %bb.0: ; %entry3842; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93843; CI-NEXT: s_waitcnt lgkmcnt(0)3844; CI-NEXT: v_mov_b32_e32 v0, s23845; CI-NEXT: v_mov_b32_e32 v1, s33846; CI-NEXT: s_mov_b32 s3, 0xf0003847; CI-NEXT: s_mov_b32 s2, -13848; CI-NEXT: buffer_atomic_or_x2 v[0:1], off, s[0:3], 0 offset:323849; CI-NEXT: s_waitcnt vmcnt(0)3850; CI-NEXT: buffer_wbinvl1_vol3851; CI-NEXT: s_endpgm3852;3853; VI-LABEL: atomic_or_i64_offset:3854; VI: ; %bb.0: ; %entry3855; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243856; VI-NEXT: s_waitcnt lgkmcnt(0)3857; VI-NEXT: v_mov_b32_e32 v0, s23858; VI-NEXT: v_mov_b32_e32 v1, s33859; VI-NEXT: s_mov_b32 s3, 0xf0003860; VI-NEXT: s_mov_b32 s2, -13861; VI-NEXT: buffer_atomic_or_x2 v[0:1], off, s[0:3], 0 offset:323862; VI-NEXT: s_waitcnt vmcnt(0)3863; VI-NEXT: buffer_wbinvl1_vol3864; VI-NEXT: s_endpgm3865;3866; GFX9-LABEL: atomic_or_i64_offset:3867; GFX9: ; %bb.0: ; %entry3868; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243869; GFX9-NEXT: v_mov_b32_e32 v2, 03870; GFX9-NEXT: s_waitcnt lgkmcnt(0)3871; GFX9-NEXT: v_mov_b32_e32 v0, s23872; GFX9-NEXT: v_mov_b32_e32 v1, s33873; GFX9-NEXT: global_atomic_or_x2 v2, v[0:1], s[0:1] offset:323874; GFX9-NEXT: s_waitcnt vmcnt(0)3875; GFX9-NEXT: buffer_wbinvl1_vol3876; GFX9-NEXT: s_endpgm3877;3878; GFX12-LABEL: atomic_or_i64_offset:3879; GFX12: ; %bb.0: ; %entry3880; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x243881; GFX12-NEXT: s_wait_kmcnt 0x03882; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s33883; GFX12-NEXT: v_mov_b32_e32 v0, s23884; GFX12-NEXT: global_atomic_or_b64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_DEV3885; GFX12-NEXT: s_wait_storecnt 0x03886; GFX12-NEXT: global_inv scope:SCOPE_DEV3887; GFX12-NEXT: s_endpgm3888entry:3889 %gep = getelementptr i64, ptr addrspace(1) %out, i64 43890 %tmp0 = atomicrmw volatile or ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !03891 ret void3892}3893 3894define amdgpu_kernel void @atomic_or_i64_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in) {3895; CI-LABEL: atomic_or_i64_ret_offset:3896; CI: ; %bb.0: ; %entry3897; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93898; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd3899; CI-NEXT: s_mov_b32 s7, 0xf0003900; CI-NEXT: s_mov_b32 s6, -13901; CI-NEXT: s_waitcnt lgkmcnt(0)3902; CI-NEXT: s_mov_b32 s4, s23903; CI-NEXT: s_mov_b32 s5, s33904; CI-NEXT: v_mov_b32_e32 v0, s83905; CI-NEXT: v_mov_b32_e32 v1, s93906; CI-NEXT: s_mov_b32 s2, s63907; CI-NEXT: s_mov_b32 s3, s73908; CI-NEXT: buffer_atomic_or_x2 v[0:1], off, s[0:3], 0 offset:32 glc3909; CI-NEXT: s_waitcnt vmcnt(0)3910; CI-NEXT: buffer_wbinvl1_vol3911; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 03912; CI-NEXT: s_endpgm3913;3914; VI-LABEL: atomic_or_i64_ret_offset:3915; VI: ; %bb.0: ; %entry3916; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243917; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x343918; VI-NEXT: s_mov_b32 s7, 0xf0003919; VI-NEXT: s_mov_b32 s6, -13920; VI-NEXT: s_waitcnt lgkmcnt(0)3921; VI-NEXT: s_mov_b32 s4, s23922; VI-NEXT: s_mov_b32 s5, s33923; VI-NEXT: v_mov_b32_e32 v0, s83924; VI-NEXT: v_mov_b32_e32 v1, s93925; VI-NEXT: s_mov_b32 s2, s63926; VI-NEXT: s_mov_b32 s3, s73927; VI-NEXT: buffer_atomic_or_x2 v[0:1], off, s[0:3], 0 offset:32 glc3928; VI-NEXT: s_waitcnt vmcnt(0)3929; VI-NEXT: buffer_wbinvl1_vol3930; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 03931; VI-NEXT: s_endpgm3932;3933; GFX9-LABEL: atomic_or_i64_ret_offset:3934; GFX9: ; %bb.0: ; %entry3935; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x343936; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243937; GFX9-NEXT: v_mov_b32_e32 v2, 03938; GFX9-NEXT: s_waitcnt lgkmcnt(0)3939; GFX9-NEXT: v_mov_b32_e32 v0, s63940; GFX9-NEXT: v_mov_b32_e32 v1, s73941; GFX9-NEXT: global_atomic_or_x2 v[0:1], v2, v[0:1], s[0:1] offset:32 glc3942; GFX9-NEXT: s_waitcnt vmcnt(0)3943; GFX9-NEXT: buffer_wbinvl1_vol3944; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]3945; GFX9-NEXT: s_endpgm3946;3947; GFX12-LABEL: atomic_or_i64_ret_offset:3948; GFX12: ; %bb.0: ; %entry3949; GFX12-NEXT: s_clause 0x13950; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x343951; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x243952; GFX12-NEXT: s_wait_kmcnt 0x03953; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s73954; GFX12-NEXT: v_mov_b32_e32 v0, s63955; GFX12-NEXT: global_atomic_or_b64 v[0:1], v2, v[0:1], s[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV3956; GFX12-NEXT: s_wait_loadcnt 0x03957; GFX12-NEXT: global_inv scope:SCOPE_DEV3958; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]3959; GFX12-NEXT: s_endpgm3960entry:3961 %gep = getelementptr i64, ptr addrspace(1) %out, i64 43962 %tmp0 = atomicrmw volatile or ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !03963 store i64 %tmp0, ptr addrspace(1) %out23964 ret void3965}3966 3967define amdgpu_kernel void @atomic_or_i64_addr64_offset(ptr addrspace(1) %out, i64 %in, i64 %index) {3968; CI-LABEL: atomic_or_i64_addr64_offset:3969; CI: ; %bb.0: ; %entry3970; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93971; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd3972; CI-NEXT: s_waitcnt lgkmcnt(0)3973; CI-NEXT: v_mov_b32_e32 v0, s23974; CI-NEXT: s_lshl_b64 s[4:5], s[4:5], 33975; CI-NEXT: v_mov_b32_e32 v2, s43976; CI-NEXT: v_mov_b32_e32 v1, s33977; CI-NEXT: s_mov_b32 s3, 0xf0003978; CI-NEXT: s_mov_b32 s2, 03979; CI-NEXT: v_mov_b32_e32 v3, s53980; CI-NEXT: buffer_atomic_or_x2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:323981; CI-NEXT: s_waitcnt vmcnt(0)3982; CI-NEXT: buffer_wbinvl1_vol3983; CI-NEXT: s_endpgm3984;3985; VI-LABEL: atomic_or_i64_addr64_offset:3986; VI: ; %bb.0: ; %entry3987; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243988; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x343989; VI-NEXT: s_waitcnt lgkmcnt(0)3990; VI-NEXT: v_mov_b32_e32 v0, s23991; VI-NEXT: v_mov_b32_e32 v1, s33992; VI-NEXT: s_lshl_b64 s[2:3], s[4:5], 33993; VI-NEXT: s_add_u32 s0, s0, s23994; VI-NEXT: s_addc_u32 s1, s1, s33995; VI-NEXT: s_add_u32 s0, s0, 323996; VI-NEXT: s_addc_u32 s1, s1, 03997; VI-NEXT: v_mov_b32_e32 v3, s13998; VI-NEXT: v_mov_b32_e32 v2, s03999; VI-NEXT: flat_atomic_or_x2 v[2:3], v[0:1]4000; VI-NEXT: s_waitcnt vmcnt(0)4001; VI-NEXT: buffer_wbinvl1_vol4002; VI-NEXT: s_endpgm4003;4004; GFX9-LABEL: atomic_or_i64_addr64_offset:4005; GFX9: ; %bb.0: ; %entry4006; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244007; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x344008; GFX9-NEXT: v_mov_b32_e32 v2, 04009; GFX9-NEXT: s_waitcnt lgkmcnt(0)4010; GFX9-NEXT: v_mov_b32_e32 v0, s24011; GFX9-NEXT: v_mov_b32_e32 v1, s34012; GFX9-NEXT: s_lshl_b64 s[2:3], s[6:7], 34013; GFX9-NEXT: s_add_u32 s0, s0, s24014; GFX9-NEXT: s_addc_u32 s1, s1, s34015; GFX9-NEXT: global_atomic_or_x2 v2, v[0:1], s[0:1] offset:324016; GFX9-NEXT: s_waitcnt vmcnt(0)4017; GFX9-NEXT: buffer_wbinvl1_vol4018; GFX9-NEXT: s_endpgm4019;4020; GFX12-LABEL: atomic_or_i64_addr64_offset:4021; GFX12: ; %bb.0: ; %entry4022; GFX12-NEXT: s_clause 0x14023; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x244024; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x344025; GFX12-NEXT: v_mov_b32_e32 v2, 04026; GFX12-NEXT: s_wait_kmcnt 0x04027; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s34028; GFX12-NEXT: s_lshl_b64 s[2:3], s[4:5], 34029; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)4030; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]4031; GFX12-NEXT: global_atomic_or_b64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_DEV4032; GFX12-NEXT: s_wait_storecnt 0x04033; GFX12-NEXT: global_inv scope:SCOPE_DEV4034; GFX12-NEXT: s_endpgm4035entry:4036 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index4037 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 44038 %tmp0 = atomicrmw volatile or ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !04039 ret void4040}4041 4042define amdgpu_kernel void @atomic_or_i64_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {4043; CI-LABEL: atomic_or_i64_ret_addr64_offset:4044; CI: ; %bb.0: ; %entry4045; CI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x94046; CI-NEXT: s_mov_b32 s11, 0xf0004047; CI-NEXT: s_mov_b32 s10, -14048; CI-NEXT: s_waitcnt lgkmcnt(0)4049; CI-NEXT: v_mov_b32_e32 v0, s44050; CI-NEXT: v_mov_b32_e32 v1, s54051; CI-NEXT: s_lshl_b64 s[4:5], s[6:7], 34052; CI-NEXT: v_mov_b32_e32 v2, s44053; CI-NEXT: s_mov_b32 s8, s24054; CI-NEXT: s_mov_b32 s9, s34055; CI-NEXT: s_mov_b32 s2, 04056; CI-NEXT: s_mov_b32 s3, s114057; CI-NEXT: v_mov_b32_e32 v3, s54058; CI-NEXT: buffer_atomic_or_x2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:32 glc4059; CI-NEXT: s_waitcnt vmcnt(0)4060; CI-NEXT: buffer_wbinvl1_vol4061; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 04062; CI-NEXT: s_endpgm4063;4064; VI-LABEL: atomic_or_i64_ret_addr64_offset:4065; VI: ; %bb.0: ; %entry4066; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x244067; VI-NEXT: s_waitcnt lgkmcnt(0)4068; VI-NEXT: v_mov_b32_e32 v0, s44069; VI-NEXT: v_mov_b32_e32 v1, s54070; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 34071; VI-NEXT: s_add_u32 s0, s0, s44072; VI-NEXT: s_addc_u32 s1, s1, s54073; VI-NEXT: s_add_u32 s0, s0, 324074; VI-NEXT: s_addc_u32 s1, s1, 04075; VI-NEXT: v_mov_b32_e32 v3, s14076; VI-NEXT: v_mov_b32_e32 v2, s04077; VI-NEXT: flat_atomic_or_x2 v[0:1], v[2:3], v[0:1] glc4078; VI-NEXT: s_waitcnt vmcnt(0)4079; VI-NEXT: buffer_wbinvl1_vol4080; VI-NEXT: s_mov_b32 s7, 0xf0004081; VI-NEXT: s_mov_b32 s6, -14082; VI-NEXT: s_mov_b32 s4, s24083; VI-NEXT: s_mov_b32 s5, s34084; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 04085; VI-NEXT: s_endpgm4086;4087; GFX9-LABEL: atomic_or_i64_ret_addr64_offset:4088; GFX9: ; %bb.0: ; %entry4089; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x244090; GFX9-NEXT: v_mov_b32_e32 v2, 04091; GFX9-NEXT: s_waitcnt lgkmcnt(0)4092; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 34093; GFX9-NEXT: s_add_u32 s0, s8, s04094; GFX9-NEXT: v_mov_b32_e32 v0, s124095; GFX9-NEXT: v_mov_b32_e32 v1, s134096; GFX9-NEXT: s_addc_u32 s1, s9, s14097; GFX9-NEXT: global_atomic_or_x2 v[0:1], v2, v[0:1], s[0:1] offset:32 glc4098; GFX9-NEXT: s_waitcnt vmcnt(0)4099; GFX9-NEXT: buffer_wbinvl1_vol4100; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]4101; GFX9-NEXT: s_endpgm4102;4103; GFX12-LABEL: atomic_or_i64_ret_addr64_offset:4104; GFX12: ; %bb.0: ; %entry4105; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x244106; GFX12-NEXT: s_wait_kmcnt 0x04107; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s54108; GFX12-NEXT: v_mov_b32_e32 v0, s44109; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 34110; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)4111; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]4112; GFX12-NEXT: global_atomic_or_b64 v[0:1], v2, v[0:1], s[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV4113; GFX12-NEXT: s_wait_loadcnt 0x04114; GFX12-NEXT: global_inv scope:SCOPE_DEV4115; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]4116; GFX12-NEXT: s_endpgm4117entry:4118 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index4119 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 44120 %tmp0 = atomicrmw volatile or ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !04121 store i64 %tmp0, ptr addrspace(1) %out24122 ret void4123}4124 4125define amdgpu_kernel void @atomic_or_i64(ptr addrspace(1) %out, i64 %in) {4126; CI-LABEL: atomic_or_i64:4127; CI: ; %bb.0: ; %entry4128; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94129; CI-NEXT: s_mov_b32 s7, 0xf0004130; CI-NEXT: s_mov_b32 s6, -14131; CI-NEXT: s_waitcnt lgkmcnt(0)4132; CI-NEXT: s_mov_b32 s4, s04133; CI-NEXT: s_mov_b32 s5, s14134; CI-NEXT: v_mov_b32_e32 v0, s24135; CI-NEXT: v_mov_b32_e32 v1, s34136; CI-NEXT: buffer_atomic_or_x2 v[0:1], off, s[4:7], 04137; CI-NEXT: s_waitcnt vmcnt(0)4138; CI-NEXT: buffer_wbinvl1_vol4139; CI-NEXT: s_endpgm4140;4141; VI-LABEL: atomic_or_i64:4142; VI: ; %bb.0: ; %entry4143; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244144; VI-NEXT: s_mov_b32 s7, 0xf0004145; VI-NEXT: s_mov_b32 s6, -14146; VI-NEXT: s_waitcnt lgkmcnt(0)4147; VI-NEXT: s_mov_b32 s4, s04148; VI-NEXT: s_mov_b32 s5, s14149; VI-NEXT: v_mov_b32_e32 v0, s24150; VI-NEXT: v_mov_b32_e32 v1, s34151; VI-NEXT: buffer_atomic_or_x2 v[0:1], off, s[4:7], 04152; VI-NEXT: s_waitcnt vmcnt(0)4153; VI-NEXT: buffer_wbinvl1_vol4154; VI-NEXT: s_endpgm4155;4156; GFX9-LABEL: atomic_or_i64:4157; GFX9: ; %bb.0: ; %entry4158; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244159; GFX9-NEXT: v_mov_b32_e32 v2, 04160; GFX9-NEXT: s_waitcnt lgkmcnt(0)4161; GFX9-NEXT: v_mov_b32_e32 v0, s24162; GFX9-NEXT: v_mov_b32_e32 v1, s34163; GFX9-NEXT: global_atomic_or_x2 v2, v[0:1], s[0:1]4164; GFX9-NEXT: s_waitcnt vmcnt(0)4165; GFX9-NEXT: buffer_wbinvl1_vol4166; GFX9-NEXT: s_endpgm4167;4168; GFX12-LABEL: atomic_or_i64:4169; GFX12: ; %bb.0: ; %entry4170; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x244171; GFX12-NEXT: s_wait_kmcnt 0x04172; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s34173; GFX12-NEXT: v_mov_b32_e32 v0, s24174; GFX12-NEXT: global_atomic_or_b64 v2, v[0:1], s[0:1] scope:SCOPE_DEV4175; GFX12-NEXT: s_wait_storecnt 0x04176; GFX12-NEXT: global_inv scope:SCOPE_DEV4177; GFX12-NEXT: s_endpgm4178entry:4179 %tmp0 = atomicrmw volatile or ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !04180 ret void4181}4182 4183define amdgpu_kernel void @atomic_or_i64_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in) {4184; CI-LABEL: atomic_or_i64_ret:4185; CI: ; %bb.0: ; %entry4186; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94187; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd4188; CI-NEXT: s_mov_b32 s7, 0xf0004189; CI-NEXT: s_mov_b32 s6, -14190; CI-NEXT: s_waitcnt lgkmcnt(0)4191; CI-NEXT: s_mov_b32 s4, s04192; CI-NEXT: s_mov_b32 s5, s14193; CI-NEXT: v_mov_b32_e32 v0, s84194; CI-NEXT: v_mov_b32_e32 v1, s94195; CI-NEXT: buffer_atomic_or_x2 v[0:1], off, s[4:7], 0 glc4196; CI-NEXT: s_waitcnt vmcnt(0)4197; CI-NEXT: buffer_wbinvl1_vol4198; CI-NEXT: s_mov_b32 s4, s24199; CI-NEXT: s_mov_b32 s5, s34200; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 04201; CI-NEXT: s_endpgm4202;4203; VI-LABEL: atomic_or_i64_ret:4204; VI: ; %bb.0: ; %entry4205; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244206; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x344207; VI-NEXT: s_mov_b32 s7, 0xf0004208; VI-NEXT: s_mov_b32 s6, -14209; VI-NEXT: s_waitcnt lgkmcnt(0)4210; VI-NEXT: s_mov_b32 s4, s04211; VI-NEXT: s_mov_b32 s5, s14212; VI-NEXT: v_mov_b32_e32 v0, s84213; VI-NEXT: v_mov_b32_e32 v1, s94214; VI-NEXT: buffer_atomic_or_x2 v[0:1], off, s[4:7], 0 glc4215; VI-NEXT: s_waitcnt vmcnt(0)4216; VI-NEXT: buffer_wbinvl1_vol4217; VI-NEXT: s_mov_b32 s4, s24218; VI-NEXT: s_mov_b32 s5, s34219; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 04220; VI-NEXT: s_endpgm4221;4222; GFX9-LABEL: atomic_or_i64_ret:4223; GFX9: ; %bb.0: ; %entry4224; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x344225; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244226; GFX9-NEXT: v_mov_b32_e32 v2, 04227; GFX9-NEXT: s_waitcnt lgkmcnt(0)4228; GFX9-NEXT: v_mov_b32_e32 v0, s64229; GFX9-NEXT: v_mov_b32_e32 v1, s74230; GFX9-NEXT: global_atomic_or_x2 v[0:1], v2, v[0:1], s[0:1] glc4231; GFX9-NEXT: s_waitcnt vmcnt(0)4232; GFX9-NEXT: buffer_wbinvl1_vol4233; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]4234; GFX9-NEXT: s_endpgm4235;4236; GFX12-LABEL: atomic_or_i64_ret:4237; GFX12: ; %bb.0: ; %entry4238; GFX12-NEXT: s_clause 0x14239; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x344240; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x244241; GFX12-NEXT: s_wait_kmcnt 0x04242; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s74243; GFX12-NEXT: v_mov_b32_e32 v0, s64244; GFX12-NEXT: global_atomic_or_b64 v[0:1], v2, v[0:1], s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV4245; GFX12-NEXT: s_wait_loadcnt 0x04246; GFX12-NEXT: global_inv scope:SCOPE_DEV4247; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]4248; GFX12-NEXT: s_endpgm4249entry:4250 %tmp0 = atomicrmw volatile or ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !04251 store i64 %tmp0, ptr addrspace(1) %out24252 ret void4253}4254 4255define amdgpu_kernel void @atomic_or_i64_addr64(ptr addrspace(1) %out, i64 %in, i64 %index) {4256; CI-LABEL: atomic_or_i64_addr64:4257; CI: ; %bb.0: ; %entry4258; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94259; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd4260; CI-NEXT: s_waitcnt lgkmcnt(0)4261; CI-NEXT: v_mov_b32_e32 v0, s24262; CI-NEXT: s_lshl_b64 s[4:5], s[4:5], 34263; CI-NEXT: v_mov_b32_e32 v2, s44264; CI-NEXT: v_mov_b32_e32 v1, s34265; CI-NEXT: s_mov_b32 s3, 0xf0004266; CI-NEXT: s_mov_b32 s2, 04267; CI-NEXT: v_mov_b32_e32 v3, s54268; CI-NEXT: buffer_atomic_or_x2 v[0:1], v[2:3], s[0:3], 0 addr644269; CI-NEXT: s_waitcnt vmcnt(0)4270; CI-NEXT: buffer_wbinvl1_vol4271; CI-NEXT: s_endpgm4272;4273; VI-LABEL: atomic_or_i64_addr64:4274; VI: ; %bb.0: ; %entry4275; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244276; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x344277; VI-NEXT: s_waitcnt lgkmcnt(0)4278; VI-NEXT: v_mov_b32_e32 v0, s24279; VI-NEXT: v_mov_b32_e32 v1, s34280; VI-NEXT: s_lshl_b64 s[2:3], s[4:5], 34281; VI-NEXT: s_add_u32 s0, s0, s24282; VI-NEXT: s_addc_u32 s1, s1, s34283; VI-NEXT: v_mov_b32_e32 v3, s14284; VI-NEXT: v_mov_b32_e32 v2, s04285; VI-NEXT: flat_atomic_or_x2 v[2:3], v[0:1]4286; VI-NEXT: s_waitcnt vmcnt(0)4287; VI-NEXT: buffer_wbinvl1_vol4288; VI-NEXT: s_endpgm4289;4290; GFX9-LABEL: atomic_or_i64_addr64:4291; GFX9: ; %bb.0: ; %entry4292; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244293; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x344294; GFX9-NEXT: v_mov_b32_e32 v2, 04295; GFX9-NEXT: s_waitcnt lgkmcnt(0)4296; GFX9-NEXT: v_mov_b32_e32 v0, s24297; GFX9-NEXT: v_mov_b32_e32 v1, s34298; GFX9-NEXT: s_lshl_b64 s[2:3], s[6:7], 34299; GFX9-NEXT: s_add_u32 s0, s0, s24300; GFX9-NEXT: s_addc_u32 s1, s1, s34301; GFX9-NEXT: global_atomic_or_x2 v2, v[0:1], s[0:1]4302; GFX9-NEXT: s_waitcnt vmcnt(0)4303; GFX9-NEXT: buffer_wbinvl1_vol4304; GFX9-NEXT: s_endpgm4305;4306; GFX12-LABEL: atomic_or_i64_addr64:4307; GFX12: ; %bb.0: ; %entry4308; GFX12-NEXT: s_clause 0x14309; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x244310; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x344311; GFX12-NEXT: v_mov_b32_e32 v2, 04312; GFX12-NEXT: s_wait_kmcnt 0x04313; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s34314; GFX12-NEXT: s_lshl_b64 s[2:3], s[4:5], 34315; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)4316; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]4317; GFX12-NEXT: global_atomic_or_b64 v2, v[0:1], s[0:1] scope:SCOPE_DEV4318; GFX12-NEXT: s_wait_storecnt 0x04319; GFX12-NEXT: global_inv scope:SCOPE_DEV4320; GFX12-NEXT: s_endpgm4321entry:4322 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index4323 %tmp0 = atomicrmw volatile or ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !04324 ret void4325}4326 4327define amdgpu_kernel void @atomic_or_i64_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {4328; CI-LABEL: atomic_or_i64_ret_addr64:4329; CI: ; %bb.0: ; %entry4330; CI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x94331; CI-NEXT: s_mov_b32 s11, 0xf0004332; CI-NEXT: s_mov_b32 s10, -14333; CI-NEXT: s_waitcnt lgkmcnt(0)4334; CI-NEXT: v_mov_b32_e32 v0, s44335; CI-NEXT: v_mov_b32_e32 v1, s54336; CI-NEXT: s_lshl_b64 s[4:5], s[6:7], 34337; CI-NEXT: v_mov_b32_e32 v2, s44338; CI-NEXT: s_mov_b32 s8, s24339; CI-NEXT: s_mov_b32 s9, s34340; CI-NEXT: s_mov_b32 s2, 04341; CI-NEXT: s_mov_b32 s3, s114342; CI-NEXT: v_mov_b32_e32 v3, s54343; CI-NEXT: buffer_atomic_or_x2 v[0:1], v[2:3], s[0:3], 0 addr64 glc4344; CI-NEXT: s_waitcnt vmcnt(0)4345; CI-NEXT: buffer_wbinvl1_vol4346; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 04347; CI-NEXT: s_endpgm4348;4349; VI-LABEL: atomic_or_i64_ret_addr64:4350; VI: ; %bb.0: ; %entry4351; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x244352; VI-NEXT: s_waitcnt lgkmcnt(0)4353; VI-NEXT: v_mov_b32_e32 v0, s44354; VI-NEXT: v_mov_b32_e32 v1, s54355; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 34356; VI-NEXT: s_add_u32 s0, s0, s44357; VI-NEXT: s_addc_u32 s1, s1, s54358; VI-NEXT: v_mov_b32_e32 v3, s14359; VI-NEXT: v_mov_b32_e32 v2, s04360; VI-NEXT: flat_atomic_or_x2 v[0:1], v[2:3], v[0:1] glc4361; VI-NEXT: s_waitcnt vmcnt(0)4362; VI-NEXT: buffer_wbinvl1_vol4363; VI-NEXT: s_mov_b32 s7, 0xf0004364; VI-NEXT: s_mov_b32 s6, -14365; VI-NEXT: s_mov_b32 s4, s24366; VI-NEXT: s_mov_b32 s5, s34367; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 04368; VI-NEXT: s_endpgm4369;4370; GFX9-LABEL: atomic_or_i64_ret_addr64:4371; GFX9: ; %bb.0: ; %entry4372; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x244373; GFX9-NEXT: v_mov_b32_e32 v2, 04374; GFX9-NEXT: s_waitcnt lgkmcnt(0)4375; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 34376; GFX9-NEXT: s_add_u32 s0, s8, s04377; GFX9-NEXT: v_mov_b32_e32 v0, s124378; GFX9-NEXT: v_mov_b32_e32 v1, s134379; GFX9-NEXT: s_addc_u32 s1, s9, s14380; GFX9-NEXT: global_atomic_or_x2 v[0:1], v2, v[0:1], s[0:1] glc4381; GFX9-NEXT: s_waitcnt vmcnt(0)4382; GFX9-NEXT: buffer_wbinvl1_vol4383; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]4384; GFX9-NEXT: s_endpgm4385;4386; GFX12-LABEL: atomic_or_i64_ret_addr64:4387; GFX12: ; %bb.0: ; %entry4388; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x244389; GFX12-NEXT: s_wait_kmcnt 0x04390; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s54391; GFX12-NEXT: v_mov_b32_e32 v0, s44392; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 34393; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)4394; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]4395; GFX12-NEXT: global_atomic_or_b64 v[0:1], v2, v[0:1], s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV4396; GFX12-NEXT: s_wait_loadcnt 0x04397; GFX12-NEXT: global_inv scope:SCOPE_DEV4398; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]4399; GFX12-NEXT: s_endpgm4400entry:4401 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index4402 %tmp0 = atomicrmw volatile or ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !04403 store i64 %tmp0, ptr addrspace(1) %out24404 ret void4405}4406 4407define amdgpu_kernel void @atomic_xchg_i64_offset(ptr addrspace(1) %out, i64 %in) {4408; CI-LABEL: atomic_xchg_i64_offset:4409; CI: ; %bb.0: ; %entry4410; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94411; CI-NEXT: s_waitcnt lgkmcnt(0)4412; CI-NEXT: v_mov_b32_e32 v0, s24413; CI-NEXT: v_mov_b32_e32 v1, s34414; CI-NEXT: s_mov_b32 s3, 0xf0004415; CI-NEXT: s_mov_b32 s2, -14416; CI-NEXT: buffer_atomic_swap_x2 v[0:1], off, s[0:3], 0 offset:324417; CI-NEXT: s_waitcnt vmcnt(0)4418; CI-NEXT: buffer_wbinvl1_vol4419; CI-NEXT: s_endpgm4420;4421; VI-LABEL: atomic_xchg_i64_offset:4422; VI: ; %bb.0: ; %entry4423; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244424; VI-NEXT: s_waitcnt lgkmcnt(0)4425; VI-NEXT: v_mov_b32_e32 v0, s24426; VI-NEXT: v_mov_b32_e32 v1, s34427; VI-NEXT: s_mov_b32 s3, 0xf0004428; VI-NEXT: s_mov_b32 s2, -14429; VI-NEXT: buffer_atomic_swap_x2 v[0:1], off, s[0:3], 0 offset:324430; VI-NEXT: s_waitcnt vmcnt(0)4431; VI-NEXT: buffer_wbinvl1_vol4432; VI-NEXT: s_endpgm4433;4434; GFX9-LABEL: atomic_xchg_i64_offset:4435; GFX9: ; %bb.0: ; %entry4436; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244437; GFX9-NEXT: v_mov_b32_e32 v2, 04438; GFX9-NEXT: s_waitcnt lgkmcnt(0)4439; GFX9-NEXT: v_mov_b32_e32 v0, s24440; GFX9-NEXT: v_mov_b32_e32 v1, s34441; GFX9-NEXT: global_atomic_swap_x2 v2, v[0:1], s[0:1] offset:324442; GFX9-NEXT: s_waitcnt vmcnt(0)4443; GFX9-NEXT: buffer_wbinvl1_vol4444; GFX9-NEXT: s_endpgm4445;4446; GFX12-LABEL: atomic_xchg_i64_offset:4447; GFX12: ; %bb.0: ; %entry4448; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x244449; GFX12-NEXT: s_wait_kmcnt 0x04450; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s34451; GFX12-NEXT: v_mov_b32_e32 v0, s24452; GFX12-NEXT: global_atomic_swap_b64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_DEV4453; GFX12-NEXT: s_wait_storecnt 0x04454; GFX12-NEXT: global_inv scope:SCOPE_DEV4455; GFX12-NEXT: s_endpgm4456entry:4457 %gep = getelementptr i64, ptr addrspace(1) %out, i64 44458 %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !04459 ret void4460}4461 4462define amdgpu_kernel void @atomic_xchg_f64_offset(ptr addrspace(1) %out, double %in) {4463; CI-LABEL: atomic_xchg_f64_offset:4464; CI: ; %bb.0: ; %entry4465; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94466; CI-NEXT: s_waitcnt lgkmcnt(0)4467; CI-NEXT: v_mov_b32_e32 v0, s24468; CI-NEXT: v_mov_b32_e32 v1, s34469; CI-NEXT: s_mov_b32 s3, 0xf0004470; CI-NEXT: s_mov_b32 s2, -14471; CI-NEXT: buffer_atomic_swap_x2 v[0:1], off, s[0:3], 0 offset:324472; CI-NEXT: s_waitcnt vmcnt(0)4473; CI-NEXT: buffer_wbinvl1_vol4474; CI-NEXT: s_endpgm4475;4476; VI-LABEL: atomic_xchg_f64_offset:4477; VI: ; %bb.0: ; %entry4478; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244479; VI-NEXT: s_waitcnt lgkmcnt(0)4480; VI-NEXT: v_mov_b32_e32 v0, s24481; VI-NEXT: v_mov_b32_e32 v1, s34482; VI-NEXT: s_mov_b32 s3, 0xf0004483; VI-NEXT: s_mov_b32 s2, -14484; VI-NEXT: buffer_atomic_swap_x2 v[0:1], off, s[0:3], 0 offset:324485; VI-NEXT: s_waitcnt vmcnt(0)4486; VI-NEXT: buffer_wbinvl1_vol4487; VI-NEXT: s_endpgm4488;4489; GFX9-LABEL: atomic_xchg_f64_offset:4490; GFX9: ; %bb.0: ; %entry4491; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244492; GFX9-NEXT: v_mov_b32_e32 v2, 04493; GFX9-NEXT: s_waitcnt lgkmcnt(0)4494; GFX9-NEXT: v_mov_b32_e32 v0, s24495; GFX9-NEXT: v_mov_b32_e32 v1, s34496; GFX9-NEXT: global_atomic_swap_x2 v2, v[0:1], s[0:1] offset:324497; GFX9-NEXT: s_waitcnt vmcnt(0)4498; GFX9-NEXT: buffer_wbinvl1_vol4499; GFX9-NEXT: s_endpgm4500;4501; GFX12-LABEL: atomic_xchg_f64_offset:4502; GFX12: ; %bb.0: ; %entry4503; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x244504; GFX12-NEXT: s_wait_kmcnt 0x04505; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s34506; GFX12-NEXT: v_mov_b32_e32 v0, s24507; GFX12-NEXT: global_atomic_swap_b64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_DEV4508; GFX12-NEXT: s_wait_storecnt 0x04509; GFX12-NEXT: global_inv scope:SCOPE_DEV4510; GFX12-NEXT: s_endpgm4511entry:4512 %gep = getelementptr double, ptr addrspace(1) %out, i64 44513 %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %gep, double %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !04514 ret void4515}4516 4517define amdgpu_kernel void @atomic_xchg_pointer_offset(ptr addrspace(1) %out, ptr %in) {4518; CI-LABEL: atomic_xchg_pointer_offset:4519; CI: ; %bb.0: ; %entry4520; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94521; CI-NEXT: s_waitcnt lgkmcnt(0)4522; CI-NEXT: v_mov_b32_e32 v0, s24523; CI-NEXT: v_mov_b32_e32 v1, s34524; CI-NEXT: s_mov_b32 s3, 0xf0004525; CI-NEXT: s_mov_b32 s2, -14526; CI-NEXT: buffer_atomic_swap_x2 v[0:1], off, s[0:3], 0 offset:324527; CI-NEXT: s_waitcnt vmcnt(0)4528; CI-NEXT: buffer_wbinvl1_vol4529; CI-NEXT: s_endpgm4530;4531; VI-LABEL: atomic_xchg_pointer_offset:4532; VI: ; %bb.0: ; %entry4533; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244534; VI-NEXT: s_waitcnt lgkmcnt(0)4535; VI-NEXT: v_mov_b32_e32 v0, s24536; VI-NEXT: v_mov_b32_e32 v1, s34537; VI-NEXT: s_mov_b32 s3, 0xf0004538; VI-NEXT: s_mov_b32 s2, -14539; VI-NEXT: buffer_atomic_swap_x2 v[0:1], off, s[0:3], 0 offset:324540; VI-NEXT: s_waitcnt vmcnt(0)4541; VI-NEXT: buffer_wbinvl1_vol4542; VI-NEXT: s_endpgm4543;4544; GFX9-LABEL: atomic_xchg_pointer_offset:4545; GFX9: ; %bb.0: ; %entry4546; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244547; GFX9-NEXT: v_mov_b32_e32 v2, 04548; GFX9-NEXT: s_waitcnt lgkmcnt(0)4549; GFX9-NEXT: v_mov_b32_e32 v0, s24550; GFX9-NEXT: v_mov_b32_e32 v1, s34551; GFX9-NEXT: global_atomic_swap_x2 v2, v[0:1], s[0:1] offset:324552; GFX9-NEXT: s_waitcnt vmcnt(0)4553; GFX9-NEXT: buffer_wbinvl1_vol4554; GFX9-NEXT: s_endpgm4555;4556; GFX12-LABEL: atomic_xchg_pointer_offset:4557; GFX12: ; %bb.0: ; %entry4558; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x244559; GFX12-NEXT: s_wait_kmcnt 0x04560; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s34561; GFX12-NEXT: v_mov_b32_e32 v0, s24562; GFX12-NEXT: global_atomic_swap_b64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_DEV4563; GFX12-NEXT: s_wait_storecnt 0x04564; GFX12-NEXT: global_inv scope:SCOPE_DEV4565; GFX12-NEXT: s_endpgm4566entry:4567 %gep = getelementptr ptr, ptr addrspace(1) %out, i64 44568 %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %gep, ptr %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !04569 ret void4570}4571 4572define amdgpu_kernel void @atomic_xchg_i64_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in) {4573; CI-LABEL: atomic_xchg_i64_ret_offset:4574; CI: ; %bb.0: ; %entry4575; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94576; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd4577; CI-NEXT: s_mov_b32 s7, 0xf0004578; CI-NEXT: s_mov_b32 s6, -14579; CI-NEXT: s_waitcnt lgkmcnt(0)4580; CI-NEXT: s_mov_b32 s4, s24581; CI-NEXT: s_mov_b32 s5, s34582; CI-NEXT: v_mov_b32_e32 v0, s84583; CI-NEXT: v_mov_b32_e32 v1, s94584; CI-NEXT: s_mov_b32 s2, s64585; CI-NEXT: s_mov_b32 s3, s74586; CI-NEXT: buffer_atomic_swap_x2 v[0:1], off, s[0:3], 0 offset:32 glc4587; CI-NEXT: s_waitcnt vmcnt(0)4588; CI-NEXT: buffer_wbinvl1_vol4589; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 04590; CI-NEXT: s_endpgm4591;4592; VI-LABEL: atomic_xchg_i64_ret_offset:4593; VI: ; %bb.0: ; %entry4594; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244595; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x344596; VI-NEXT: s_mov_b32 s7, 0xf0004597; VI-NEXT: s_mov_b32 s6, -14598; VI-NEXT: s_waitcnt lgkmcnt(0)4599; VI-NEXT: s_mov_b32 s4, s24600; VI-NEXT: s_mov_b32 s5, s34601; VI-NEXT: v_mov_b32_e32 v0, s84602; VI-NEXT: v_mov_b32_e32 v1, s94603; VI-NEXT: s_mov_b32 s2, s64604; VI-NEXT: s_mov_b32 s3, s74605; VI-NEXT: buffer_atomic_swap_x2 v[0:1], off, s[0:3], 0 offset:32 glc4606; VI-NEXT: s_waitcnt vmcnt(0)4607; VI-NEXT: buffer_wbinvl1_vol4608; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 04609; VI-NEXT: s_endpgm4610;4611; GFX9-LABEL: atomic_xchg_i64_ret_offset:4612; GFX9: ; %bb.0: ; %entry4613; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x344614; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244615; GFX9-NEXT: v_mov_b32_e32 v2, 04616; GFX9-NEXT: s_waitcnt lgkmcnt(0)4617; GFX9-NEXT: v_mov_b32_e32 v0, s64618; GFX9-NEXT: v_mov_b32_e32 v1, s74619; GFX9-NEXT: global_atomic_swap_x2 v[0:1], v2, v[0:1], s[0:1] offset:32 glc4620; GFX9-NEXT: s_waitcnt vmcnt(0)4621; GFX9-NEXT: buffer_wbinvl1_vol4622; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]4623; GFX9-NEXT: s_endpgm4624;4625; GFX12-LABEL: atomic_xchg_i64_ret_offset:4626; GFX12: ; %bb.0: ; %entry4627; GFX12-NEXT: s_clause 0x14628; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x344629; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x244630; GFX12-NEXT: s_wait_kmcnt 0x04631; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s74632; GFX12-NEXT: v_mov_b32_e32 v0, s64633; GFX12-NEXT: global_atomic_swap_b64 v[0:1], v2, v[0:1], s[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV4634; GFX12-NEXT: s_wait_loadcnt 0x04635; GFX12-NEXT: global_inv scope:SCOPE_DEV4636; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]4637; GFX12-NEXT: s_endpgm4638entry:4639 %gep = getelementptr i64, ptr addrspace(1) %out, i64 44640 %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !04641 store i64 %tmp0, ptr addrspace(1) %out24642 ret void4643}4644 4645define amdgpu_kernel void @atomic_xchg_i64_addr64_offset(ptr addrspace(1) %out, i64 %in, i64 %index) {4646; CI-LABEL: atomic_xchg_i64_addr64_offset:4647; CI: ; %bb.0: ; %entry4648; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94649; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd4650; CI-NEXT: s_waitcnt lgkmcnt(0)4651; CI-NEXT: v_mov_b32_e32 v0, s24652; CI-NEXT: s_lshl_b64 s[4:5], s[4:5], 34653; CI-NEXT: v_mov_b32_e32 v2, s44654; CI-NEXT: v_mov_b32_e32 v1, s34655; CI-NEXT: s_mov_b32 s3, 0xf0004656; CI-NEXT: s_mov_b32 s2, 04657; CI-NEXT: v_mov_b32_e32 v3, s54658; CI-NEXT: buffer_atomic_swap_x2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:324659; CI-NEXT: s_waitcnt vmcnt(0)4660; CI-NEXT: buffer_wbinvl1_vol4661; CI-NEXT: s_endpgm4662;4663; VI-LABEL: atomic_xchg_i64_addr64_offset:4664; VI: ; %bb.0: ; %entry4665; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244666; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x344667; VI-NEXT: s_waitcnt lgkmcnt(0)4668; VI-NEXT: v_mov_b32_e32 v0, s24669; VI-NEXT: v_mov_b32_e32 v1, s34670; VI-NEXT: s_lshl_b64 s[2:3], s[4:5], 34671; VI-NEXT: s_add_u32 s0, s0, s24672; VI-NEXT: s_addc_u32 s1, s1, s34673; VI-NEXT: s_add_u32 s0, s0, 324674; VI-NEXT: s_addc_u32 s1, s1, 04675; VI-NEXT: v_mov_b32_e32 v3, s14676; VI-NEXT: v_mov_b32_e32 v2, s04677; VI-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]4678; VI-NEXT: s_waitcnt vmcnt(0)4679; VI-NEXT: buffer_wbinvl1_vol4680; VI-NEXT: s_endpgm4681;4682; GFX9-LABEL: atomic_xchg_i64_addr64_offset:4683; GFX9: ; %bb.0: ; %entry4684; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244685; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x344686; GFX9-NEXT: v_mov_b32_e32 v2, 04687; GFX9-NEXT: s_waitcnt lgkmcnt(0)4688; GFX9-NEXT: v_mov_b32_e32 v0, s24689; GFX9-NEXT: v_mov_b32_e32 v1, s34690; GFX9-NEXT: s_lshl_b64 s[2:3], s[6:7], 34691; GFX9-NEXT: s_add_u32 s0, s0, s24692; GFX9-NEXT: s_addc_u32 s1, s1, s34693; GFX9-NEXT: global_atomic_swap_x2 v2, v[0:1], s[0:1] offset:324694; GFX9-NEXT: s_waitcnt vmcnt(0)4695; GFX9-NEXT: buffer_wbinvl1_vol4696; GFX9-NEXT: s_endpgm4697;4698; GFX12-LABEL: atomic_xchg_i64_addr64_offset:4699; GFX12: ; %bb.0: ; %entry4700; GFX12-NEXT: s_clause 0x14701; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x244702; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x344703; GFX12-NEXT: v_mov_b32_e32 v2, 04704; GFX12-NEXT: s_wait_kmcnt 0x04705; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s34706; GFX12-NEXT: s_lshl_b64 s[2:3], s[4:5], 34707; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)4708; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]4709; GFX12-NEXT: global_atomic_swap_b64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_DEV4710; GFX12-NEXT: s_wait_storecnt 0x04711; GFX12-NEXT: global_inv scope:SCOPE_DEV4712; GFX12-NEXT: s_endpgm4713entry:4714 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index4715 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 44716 %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !04717 ret void4718}4719 4720define amdgpu_kernel void @atomic_xchg_i64_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {4721; CI-LABEL: atomic_xchg_i64_ret_addr64_offset:4722; CI: ; %bb.0: ; %entry4723; CI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x94724; CI-NEXT: s_mov_b32 s11, 0xf0004725; CI-NEXT: s_mov_b32 s10, -14726; CI-NEXT: s_waitcnt lgkmcnt(0)4727; CI-NEXT: v_mov_b32_e32 v0, s44728; CI-NEXT: v_mov_b32_e32 v1, s54729; CI-NEXT: s_lshl_b64 s[4:5], s[6:7], 34730; CI-NEXT: v_mov_b32_e32 v2, s44731; CI-NEXT: s_mov_b32 s8, s24732; CI-NEXT: s_mov_b32 s9, s34733; CI-NEXT: s_mov_b32 s2, 04734; CI-NEXT: s_mov_b32 s3, s114735; CI-NEXT: v_mov_b32_e32 v3, s54736; CI-NEXT: buffer_atomic_swap_x2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:32 glc4737; CI-NEXT: s_waitcnt vmcnt(0)4738; CI-NEXT: buffer_wbinvl1_vol4739; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 04740; CI-NEXT: s_endpgm4741;4742; VI-LABEL: atomic_xchg_i64_ret_addr64_offset:4743; VI: ; %bb.0: ; %entry4744; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x244745; VI-NEXT: s_waitcnt lgkmcnt(0)4746; VI-NEXT: v_mov_b32_e32 v0, s44747; VI-NEXT: v_mov_b32_e32 v1, s54748; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 34749; VI-NEXT: s_add_u32 s0, s0, s44750; VI-NEXT: s_addc_u32 s1, s1, s54751; VI-NEXT: s_add_u32 s0, s0, 324752; VI-NEXT: s_addc_u32 s1, s1, 04753; VI-NEXT: v_mov_b32_e32 v3, s14754; VI-NEXT: v_mov_b32_e32 v2, s04755; VI-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc4756; VI-NEXT: s_waitcnt vmcnt(0)4757; VI-NEXT: buffer_wbinvl1_vol4758; VI-NEXT: s_mov_b32 s7, 0xf0004759; VI-NEXT: s_mov_b32 s6, -14760; VI-NEXT: s_mov_b32 s4, s24761; VI-NEXT: s_mov_b32 s5, s34762; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 04763; VI-NEXT: s_endpgm4764;4765; GFX9-LABEL: atomic_xchg_i64_ret_addr64_offset:4766; GFX9: ; %bb.0: ; %entry4767; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x244768; GFX9-NEXT: v_mov_b32_e32 v2, 04769; GFX9-NEXT: s_waitcnt lgkmcnt(0)4770; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 34771; GFX9-NEXT: s_add_u32 s0, s8, s04772; GFX9-NEXT: v_mov_b32_e32 v0, s124773; GFX9-NEXT: v_mov_b32_e32 v1, s134774; GFX9-NEXT: s_addc_u32 s1, s9, s14775; GFX9-NEXT: global_atomic_swap_x2 v[0:1], v2, v[0:1], s[0:1] offset:32 glc4776; GFX9-NEXT: s_waitcnt vmcnt(0)4777; GFX9-NEXT: buffer_wbinvl1_vol4778; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]4779; GFX9-NEXT: s_endpgm4780;4781; GFX12-LABEL: atomic_xchg_i64_ret_addr64_offset:4782; GFX12: ; %bb.0: ; %entry4783; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x244784; GFX12-NEXT: s_wait_kmcnt 0x04785; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s54786; GFX12-NEXT: v_mov_b32_e32 v0, s44787; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 34788; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)4789; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]4790; GFX12-NEXT: global_atomic_swap_b64 v[0:1], v2, v[0:1], s[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV4791; GFX12-NEXT: s_wait_loadcnt 0x04792; GFX12-NEXT: global_inv scope:SCOPE_DEV4793; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]4794; GFX12-NEXT: s_endpgm4795entry:4796 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index4797 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 44798 %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !04799 store i64 %tmp0, ptr addrspace(1) %out24800 ret void4801}4802 4803define amdgpu_kernel void @atomic_xchg_i64(ptr addrspace(1) %out, i64 %in) {4804; CI-LABEL: atomic_xchg_i64:4805; CI: ; %bb.0: ; %entry4806; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94807; CI-NEXT: s_mov_b32 s7, 0xf0004808; CI-NEXT: s_mov_b32 s6, -14809; CI-NEXT: s_waitcnt lgkmcnt(0)4810; CI-NEXT: s_mov_b32 s4, s04811; CI-NEXT: s_mov_b32 s5, s14812; CI-NEXT: v_mov_b32_e32 v0, s24813; CI-NEXT: v_mov_b32_e32 v1, s34814; CI-NEXT: buffer_atomic_swap_x2 v[0:1], off, s[4:7], 04815; CI-NEXT: s_waitcnt vmcnt(0)4816; CI-NEXT: buffer_wbinvl1_vol4817; CI-NEXT: s_endpgm4818;4819; VI-LABEL: atomic_xchg_i64:4820; VI: ; %bb.0: ; %entry4821; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244822; VI-NEXT: s_mov_b32 s7, 0xf0004823; VI-NEXT: s_mov_b32 s6, -14824; VI-NEXT: s_waitcnt lgkmcnt(0)4825; VI-NEXT: s_mov_b32 s4, s04826; VI-NEXT: s_mov_b32 s5, s14827; VI-NEXT: v_mov_b32_e32 v0, s24828; VI-NEXT: v_mov_b32_e32 v1, s34829; VI-NEXT: buffer_atomic_swap_x2 v[0:1], off, s[4:7], 04830; VI-NEXT: s_waitcnt vmcnt(0)4831; VI-NEXT: buffer_wbinvl1_vol4832; VI-NEXT: s_endpgm4833;4834; GFX9-LABEL: atomic_xchg_i64:4835; GFX9: ; %bb.0: ; %entry4836; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244837; GFX9-NEXT: v_mov_b32_e32 v2, 04838; GFX9-NEXT: s_waitcnt lgkmcnt(0)4839; GFX9-NEXT: v_mov_b32_e32 v0, s24840; GFX9-NEXT: v_mov_b32_e32 v1, s34841; GFX9-NEXT: global_atomic_swap_x2 v2, v[0:1], s[0:1]4842; GFX9-NEXT: s_waitcnt vmcnt(0)4843; GFX9-NEXT: buffer_wbinvl1_vol4844; GFX9-NEXT: s_endpgm4845;4846; GFX12-LABEL: atomic_xchg_i64:4847; GFX12: ; %bb.0: ; %entry4848; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x244849; GFX12-NEXT: s_wait_kmcnt 0x04850; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s34851; GFX12-NEXT: v_mov_b32_e32 v0, s24852; GFX12-NEXT: global_atomic_swap_b64 v2, v[0:1], s[0:1] scope:SCOPE_DEV4853; GFX12-NEXT: s_wait_storecnt 0x04854; GFX12-NEXT: global_inv scope:SCOPE_DEV4855; GFX12-NEXT: s_endpgm4856entry:4857 %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !04858 ret void4859}4860 4861define amdgpu_kernel void @atomic_xchg_i64_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in) {4862; CI-LABEL: atomic_xchg_i64_ret:4863; CI: ; %bb.0: ; %entry4864; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94865; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd4866; CI-NEXT: s_mov_b32 s7, 0xf0004867; CI-NEXT: s_mov_b32 s6, -14868; CI-NEXT: s_waitcnt lgkmcnt(0)4869; CI-NEXT: s_mov_b32 s4, s04870; CI-NEXT: s_mov_b32 s5, s14871; CI-NEXT: v_mov_b32_e32 v0, s84872; CI-NEXT: v_mov_b32_e32 v1, s94873; CI-NEXT: buffer_atomic_swap_x2 v[0:1], off, s[4:7], 0 glc4874; CI-NEXT: s_waitcnt vmcnt(0)4875; CI-NEXT: buffer_wbinvl1_vol4876; CI-NEXT: s_mov_b32 s4, s24877; CI-NEXT: s_mov_b32 s5, s34878; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 04879; CI-NEXT: s_endpgm4880;4881; VI-LABEL: atomic_xchg_i64_ret:4882; VI: ; %bb.0: ; %entry4883; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244884; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x344885; VI-NEXT: s_mov_b32 s7, 0xf0004886; VI-NEXT: s_mov_b32 s6, -14887; VI-NEXT: s_waitcnt lgkmcnt(0)4888; VI-NEXT: s_mov_b32 s4, s04889; VI-NEXT: s_mov_b32 s5, s14890; VI-NEXT: v_mov_b32_e32 v0, s84891; VI-NEXT: v_mov_b32_e32 v1, s94892; VI-NEXT: buffer_atomic_swap_x2 v[0:1], off, s[4:7], 0 glc4893; VI-NEXT: s_waitcnt vmcnt(0)4894; VI-NEXT: buffer_wbinvl1_vol4895; VI-NEXT: s_mov_b32 s4, s24896; VI-NEXT: s_mov_b32 s5, s34897; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 04898; VI-NEXT: s_endpgm4899;4900; GFX9-LABEL: atomic_xchg_i64_ret:4901; GFX9: ; %bb.0: ; %entry4902; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x344903; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244904; GFX9-NEXT: v_mov_b32_e32 v2, 04905; GFX9-NEXT: s_waitcnt lgkmcnt(0)4906; GFX9-NEXT: v_mov_b32_e32 v0, s64907; GFX9-NEXT: v_mov_b32_e32 v1, s74908; GFX9-NEXT: global_atomic_swap_x2 v[0:1], v2, v[0:1], s[0:1] glc4909; GFX9-NEXT: s_waitcnt vmcnt(0)4910; GFX9-NEXT: buffer_wbinvl1_vol4911; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]4912; GFX9-NEXT: s_endpgm4913;4914; GFX12-LABEL: atomic_xchg_i64_ret:4915; GFX12: ; %bb.0: ; %entry4916; GFX12-NEXT: s_clause 0x14917; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x344918; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x244919; GFX12-NEXT: s_wait_kmcnt 0x04920; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s74921; GFX12-NEXT: v_mov_b32_e32 v0, s64922; GFX12-NEXT: global_atomic_swap_b64 v[0:1], v2, v[0:1], s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV4923; GFX12-NEXT: s_wait_loadcnt 0x04924; GFX12-NEXT: global_inv scope:SCOPE_DEV4925; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]4926; GFX12-NEXT: s_endpgm4927entry:4928 %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !04929 store i64 %tmp0, ptr addrspace(1) %out24930 ret void4931}4932 4933define amdgpu_kernel void @atomic_xchg_i64_addr64(ptr addrspace(1) %out, i64 %in, i64 %index) {4934; CI-LABEL: atomic_xchg_i64_addr64:4935; CI: ; %bb.0: ; %entry4936; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94937; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd4938; CI-NEXT: s_waitcnt lgkmcnt(0)4939; CI-NEXT: v_mov_b32_e32 v0, s24940; CI-NEXT: s_lshl_b64 s[4:5], s[4:5], 34941; CI-NEXT: v_mov_b32_e32 v2, s44942; CI-NEXT: v_mov_b32_e32 v1, s34943; CI-NEXT: s_mov_b32 s3, 0xf0004944; CI-NEXT: s_mov_b32 s2, 04945; CI-NEXT: v_mov_b32_e32 v3, s54946; CI-NEXT: buffer_atomic_swap_x2 v[0:1], v[2:3], s[0:3], 0 addr644947; CI-NEXT: s_waitcnt vmcnt(0)4948; CI-NEXT: buffer_wbinvl1_vol4949; CI-NEXT: s_endpgm4950;4951; VI-LABEL: atomic_xchg_i64_addr64:4952; VI: ; %bb.0: ; %entry4953; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244954; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x344955; VI-NEXT: s_waitcnt lgkmcnt(0)4956; VI-NEXT: v_mov_b32_e32 v0, s24957; VI-NEXT: v_mov_b32_e32 v1, s34958; VI-NEXT: s_lshl_b64 s[2:3], s[4:5], 34959; VI-NEXT: s_add_u32 s0, s0, s24960; VI-NEXT: s_addc_u32 s1, s1, s34961; VI-NEXT: v_mov_b32_e32 v3, s14962; VI-NEXT: v_mov_b32_e32 v2, s04963; VI-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]4964; VI-NEXT: s_waitcnt vmcnt(0)4965; VI-NEXT: buffer_wbinvl1_vol4966; VI-NEXT: s_endpgm4967;4968; GFX9-LABEL: atomic_xchg_i64_addr64:4969; GFX9: ; %bb.0: ; %entry4970; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244971; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x344972; GFX9-NEXT: v_mov_b32_e32 v2, 04973; GFX9-NEXT: s_waitcnt lgkmcnt(0)4974; GFX9-NEXT: v_mov_b32_e32 v0, s24975; GFX9-NEXT: v_mov_b32_e32 v1, s34976; GFX9-NEXT: s_lshl_b64 s[2:3], s[6:7], 34977; GFX9-NEXT: s_add_u32 s0, s0, s24978; GFX9-NEXT: s_addc_u32 s1, s1, s34979; GFX9-NEXT: global_atomic_swap_x2 v2, v[0:1], s[0:1]4980; GFX9-NEXT: s_waitcnt vmcnt(0)4981; GFX9-NEXT: buffer_wbinvl1_vol4982; GFX9-NEXT: s_endpgm4983;4984; GFX12-LABEL: atomic_xchg_i64_addr64:4985; GFX12: ; %bb.0: ; %entry4986; GFX12-NEXT: s_clause 0x14987; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x244988; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x344989; GFX12-NEXT: v_mov_b32_e32 v2, 04990; GFX12-NEXT: s_wait_kmcnt 0x04991; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s34992; GFX12-NEXT: s_lshl_b64 s[2:3], s[4:5], 34993; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)4994; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]4995; GFX12-NEXT: global_atomic_swap_b64 v2, v[0:1], s[0:1] scope:SCOPE_DEV4996; GFX12-NEXT: s_wait_storecnt 0x04997; GFX12-NEXT: global_inv scope:SCOPE_DEV4998; GFX12-NEXT: s_endpgm4999entry:5000 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index5001 %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !05002 ret void5003}5004 5005define amdgpu_kernel void @atomic_xchg_i64_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {5006; CI-LABEL: atomic_xchg_i64_ret_addr64:5007; CI: ; %bb.0: ; %entry5008; CI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x95009; CI-NEXT: s_mov_b32 s11, 0xf0005010; CI-NEXT: s_mov_b32 s10, -15011; CI-NEXT: s_waitcnt lgkmcnt(0)5012; CI-NEXT: v_mov_b32_e32 v0, s45013; CI-NEXT: v_mov_b32_e32 v1, s55014; CI-NEXT: s_lshl_b64 s[4:5], s[6:7], 35015; CI-NEXT: v_mov_b32_e32 v2, s45016; CI-NEXT: s_mov_b32 s8, s25017; CI-NEXT: s_mov_b32 s9, s35018; CI-NEXT: s_mov_b32 s2, 05019; CI-NEXT: s_mov_b32 s3, s115020; CI-NEXT: v_mov_b32_e32 v3, s55021; CI-NEXT: buffer_atomic_swap_x2 v[0:1], v[2:3], s[0:3], 0 addr64 glc5022; CI-NEXT: s_waitcnt vmcnt(0)5023; CI-NEXT: buffer_wbinvl1_vol5024; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 05025; CI-NEXT: s_endpgm5026;5027; VI-LABEL: atomic_xchg_i64_ret_addr64:5028; VI: ; %bb.0: ; %entry5029; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x245030; VI-NEXT: s_waitcnt lgkmcnt(0)5031; VI-NEXT: v_mov_b32_e32 v0, s45032; VI-NEXT: v_mov_b32_e32 v1, s55033; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 35034; VI-NEXT: s_add_u32 s0, s0, s45035; VI-NEXT: s_addc_u32 s1, s1, s55036; VI-NEXT: v_mov_b32_e32 v3, s15037; VI-NEXT: v_mov_b32_e32 v2, s05038; VI-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc5039; VI-NEXT: s_waitcnt vmcnt(0)5040; VI-NEXT: buffer_wbinvl1_vol5041; VI-NEXT: s_mov_b32 s7, 0xf0005042; VI-NEXT: s_mov_b32 s6, -15043; VI-NEXT: s_mov_b32 s4, s25044; VI-NEXT: s_mov_b32 s5, s35045; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 05046; VI-NEXT: s_endpgm5047;5048; GFX9-LABEL: atomic_xchg_i64_ret_addr64:5049; GFX9: ; %bb.0: ; %entry5050; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x245051; GFX9-NEXT: v_mov_b32_e32 v2, 05052; GFX9-NEXT: s_waitcnt lgkmcnt(0)5053; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 35054; GFX9-NEXT: s_add_u32 s0, s8, s05055; GFX9-NEXT: v_mov_b32_e32 v0, s125056; GFX9-NEXT: v_mov_b32_e32 v1, s135057; GFX9-NEXT: s_addc_u32 s1, s9, s15058; GFX9-NEXT: global_atomic_swap_x2 v[0:1], v2, v[0:1], s[0:1] glc5059; GFX9-NEXT: s_waitcnt vmcnt(0)5060; GFX9-NEXT: buffer_wbinvl1_vol5061; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]5062; GFX9-NEXT: s_endpgm5063;5064; GFX12-LABEL: atomic_xchg_i64_ret_addr64:5065; GFX12: ; %bb.0: ; %entry5066; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x245067; GFX12-NEXT: s_wait_kmcnt 0x05068; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s55069; GFX12-NEXT: v_mov_b32_e32 v0, s45070; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 35071; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)5072; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]5073; GFX12-NEXT: global_atomic_swap_b64 v[0:1], v2, v[0:1], s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV5074; GFX12-NEXT: s_wait_loadcnt 0x05075; GFX12-NEXT: global_inv scope:SCOPE_DEV5076; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]5077; GFX12-NEXT: s_endpgm5078entry:5079 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index5080 %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !05081 store i64 %tmp0, ptr addrspace(1) %out25082 ret void5083}5084 5085define amdgpu_kernel void @atomic_xor_i64_offset(ptr addrspace(1) %out, i64 %in) {5086; CI-LABEL: atomic_xor_i64_offset:5087; CI: ; %bb.0: ; %entry5088; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95089; CI-NEXT: s_waitcnt lgkmcnt(0)5090; CI-NEXT: v_mov_b32_e32 v0, s25091; CI-NEXT: v_mov_b32_e32 v1, s35092; CI-NEXT: s_mov_b32 s3, 0xf0005093; CI-NEXT: s_mov_b32 s2, -15094; CI-NEXT: buffer_atomic_xor_x2 v[0:1], off, s[0:3], 0 offset:325095; CI-NEXT: s_waitcnt vmcnt(0)5096; CI-NEXT: buffer_wbinvl1_vol5097; CI-NEXT: s_endpgm5098;5099; VI-LABEL: atomic_xor_i64_offset:5100; VI: ; %bb.0: ; %entry5101; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245102; VI-NEXT: s_waitcnt lgkmcnt(0)5103; VI-NEXT: v_mov_b32_e32 v0, s25104; VI-NEXT: v_mov_b32_e32 v1, s35105; VI-NEXT: s_mov_b32 s3, 0xf0005106; VI-NEXT: s_mov_b32 s2, -15107; VI-NEXT: buffer_atomic_xor_x2 v[0:1], off, s[0:3], 0 offset:325108; VI-NEXT: s_waitcnt vmcnt(0)5109; VI-NEXT: buffer_wbinvl1_vol5110; VI-NEXT: s_endpgm5111;5112; GFX9-LABEL: atomic_xor_i64_offset:5113; GFX9: ; %bb.0: ; %entry5114; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245115; GFX9-NEXT: v_mov_b32_e32 v2, 05116; GFX9-NEXT: s_waitcnt lgkmcnt(0)5117; GFX9-NEXT: v_mov_b32_e32 v0, s25118; GFX9-NEXT: v_mov_b32_e32 v1, s35119; GFX9-NEXT: global_atomic_xor_x2 v2, v[0:1], s[0:1] offset:325120; GFX9-NEXT: s_waitcnt vmcnt(0)5121; GFX9-NEXT: buffer_wbinvl1_vol5122; GFX9-NEXT: s_endpgm5123;5124; GFX12-LABEL: atomic_xor_i64_offset:5125; GFX12: ; %bb.0: ; %entry5126; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x245127; GFX12-NEXT: s_wait_kmcnt 0x05128; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s35129; GFX12-NEXT: v_mov_b32_e32 v0, s25130; GFX12-NEXT: global_atomic_xor_b64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_DEV5131; GFX12-NEXT: s_wait_storecnt 0x05132; GFX12-NEXT: global_inv scope:SCOPE_DEV5133; GFX12-NEXT: s_endpgm5134entry:5135 %gep = getelementptr i64, ptr addrspace(1) %out, i64 45136 %tmp0 = atomicrmw volatile xor ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !05137 ret void5138}5139 5140define amdgpu_kernel void @atomic_xor_i64_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in) {5141; CI-LABEL: atomic_xor_i64_ret_offset:5142; CI: ; %bb.0: ; %entry5143; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95144; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd5145; CI-NEXT: s_mov_b32 s7, 0xf0005146; CI-NEXT: s_mov_b32 s6, -15147; CI-NEXT: s_waitcnt lgkmcnt(0)5148; CI-NEXT: s_mov_b32 s4, s25149; CI-NEXT: s_mov_b32 s5, s35150; CI-NEXT: v_mov_b32_e32 v0, s85151; CI-NEXT: v_mov_b32_e32 v1, s95152; CI-NEXT: s_mov_b32 s2, s65153; CI-NEXT: s_mov_b32 s3, s75154; CI-NEXT: buffer_atomic_xor_x2 v[0:1], off, s[0:3], 0 offset:32 glc5155; CI-NEXT: s_waitcnt vmcnt(0)5156; CI-NEXT: buffer_wbinvl1_vol5157; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 05158; CI-NEXT: s_endpgm5159;5160; VI-LABEL: atomic_xor_i64_ret_offset:5161; VI: ; %bb.0: ; %entry5162; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245163; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x345164; VI-NEXT: s_mov_b32 s7, 0xf0005165; VI-NEXT: s_mov_b32 s6, -15166; VI-NEXT: s_waitcnt lgkmcnt(0)5167; VI-NEXT: s_mov_b32 s4, s25168; VI-NEXT: s_mov_b32 s5, s35169; VI-NEXT: v_mov_b32_e32 v0, s85170; VI-NEXT: v_mov_b32_e32 v1, s95171; VI-NEXT: s_mov_b32 s2, s65172; VI-NEXT: s_mov_b32 s3, s75173; VI-NEXT: buffer_atomic_xor_x2 v[0:1], off, s[0:3], 0 offset:32 glc5174; VI-NEXT: s_waitcnt vmcnt(0)5175; VI-NEXT: buffer_wbinvl1_vol5176; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 05177; VI-NEXT: s_endpgm5178;5179; GFX9-LABEL: atomic_xor_i64_ret_offset:5180; GFX9: ; %bb.0: ; %entry5181; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x345182; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245183; GFX9-NEXT: v_mov_b32_e32 v2, 05184; GFX9-NEXT: s_waitcnt lgkmcnt(0)5185; GFX9-NEXT: v_mov_b32_e32 v0, s65186; GFX9-NEXT: v_mov_b32_e32 v1, s75187; GFX9-NEXT: global_atomic_xor_x2 v[0:1], v2, v[0:1], s[0:1] offset:32 glc5188; GFX9-NEXT: s_waitcnt vmcnt(0)5189; GFX9-NEXT: buffer_wbinvl1_vol5190; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]5191; GFX9-NEXT: s_endpgm5192;5193; GFX12-LABEL: atomic_xor_i64_ret_offset:5194; GFX12: ; %bb.0: ; %entry5195; GFX12-NEXT: s_clause 0x15196; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x345197; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x245198; GFX12-NEXT: s_wait_kmcnt 0x05199; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s75200; GFX12-NEXT: v_mov_b32_e32 v0, s65201; GFX12-NEXT: global_atomic_xor_b64 v[0:1], v2, v[0:1], s[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV5202; GFX12-NEXT: s_wait_loadcnt 0x05203; GFX12-NEXT: global_inv scope:SCOPE_DEV5204; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]5205; GFX12-NEXT: s_endpgm5206entry:5207 %gep = getelementptr i64, ptr addrspace(1) %out, i64 45208 %tmp0 = atomicrmw volatile xor ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !05209 store i64 %tmp0, ptr addrspace(1) %out25210 ret void5211}5212 5213define amdgpu_kernel void @atomic_xor_i64_addr64_offset(ptr addrspace(1) %out, i64 %in, i64 %index) {5214; CI-LABEL: atomic_xor_i64_addr64_offset:5215; CI: ; %bb.0: ; %entry5216; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95217; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd5218; CI-NEXT: s_waitcnt lgkmcnt(0)5219; CI-NEXT: v_mov_b32_e32 v0, s25220; CI-NEXT: s_lshl_b64 s[4:5], s[4:5], 35221; CI-NEXT: v_mov_b32_e32 v2, s45222; CI-NEXT: v_mov_b32_e32 v1, s35223; CI-NEXT: s_mov_b32 s3, 0xf0005224; CI-NEXT: s_mov_b32 s2, 05225; CI-NEXT: v_mov_b32_e32 v3, s55226; CI-NEXT: buffer_atomic_xor_x2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:325227; CI-NEXT: s_waitcnt vmcnt(0)5228; CI-NEXT: buffer_wbinvl1_vol5229; CI-NEXT: s_endpgm5230;5231; VI-LABEL: atomic_xor_i64_addr64_offset:5232; VI: ; %bb.0: ; %entry5233; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245234; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x345235; VI-NEXT: s_waitcnt lgkmcnt(0)5236; VI-NEXT: v_mov_b32_e32 v0, s25237; VI-NEXT: v_mov_b32_e32 v1, s35238; VI-NEXT: s_lshl_b64 s[2:3], s[4:5], 35239; VI-NEXT: s_add_u32 s0, s0, s25240; VI-NEXT: s_addc_u32 s1, s1, s35241; VI-NEXT: s_add_u32 s0, s0, 325242; VI-NEXT: s_addc_u32 s1, s1, 05243; VI-NEXT: v_mov_b32_e32 v3, s15244; VI-NEXT: v_mov_b32_e32 v2, s05245; VI-NEXT: flat_atomic_xor_x2 v[2:3], v[0:1]5246; VI-NEXT: s_waitcnt vmcnt(0)5247; VI-NEXT: buffer_wbinvl1_vol5248; VI-NEXT: s_endpgm5249;5250; GFX9-LABEL: atomic_xor_i64_addr64_offset:5251; GFX9: ; %bb.0: ; %entry5252; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245253; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x345254; GFX9-NEXT: v_mov_b32_e32 v2, 05255; GFX9-NEXT: s_waitcnt lgkmcnt(0)5256; GFX9-NEXT: v_mov_b32_e32 v0, s25257; GFX9-NEXT: v_mov_b32_e32 v1, s35258; GFX9-NEXT: s_lshl_b64 s[2:3], s[6:7], 35259; GFX9-NEXT: s_add_u32 s0, s0, s25260; GFX9-NEXT: s_addc_u32 s1, s1, s35261; GFX9-NEXT: global_atomic_xor_x2 v2, v[0:1], s[0:1] offset:325262; GFX9-NEXT: s_waitcnt vmcnt(0)5263; GFX9-NEXT: buffer_wbinvl1_vol5264; GFX9-NEXT: s_endpgm5265;5266; GFX12-LABEL: atomic_xor_i64_addr64_offset:5267; GFX12: ; %bb.0: ; %entry5268; GFX12-NEXT: s_clause 0x15269; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x245270; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x345271; GFX12-NEXT: v_mov_b32_e32 v2, 05272; GFX12-NEXT: s_wait_kmcnt 0x05273; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s35274; GFX12-NEXT: s_lshl_b64 s[2:3], s[4:5], 35275; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)5276; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]5277; GFX12-NEXT: global_atomic_xor_b64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_DEV5278; GFX12-NEXT: s_wait_storecnt 0x05279; GFX12-NEXT: global_inv scope:SCOPE_DEV5280; GFX12-NEXT: s_endpgm5281entry:5282 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index5283 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 45284 %tmp0 = atomicrmw volatile xor ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !05285 ret void5286}5287 5288define amdgpu_kernel void @atomic_xor_i64_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {5289; CI-LABEL: atomic_xor_i64_ret_addr64_offset:5290; CI: ; %bb.0: ; %entry5291; CI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x95292; CI-NEXT: s_mov_b32 s11, 0xf0005293; CI-NEXT: s_mov_b32 s10, -15294; CI-NEXT: s_waitcnt lgkmcnt(0)5295; CI-NEXT: v_mov_b32_e32 v0, s45296; CI-NEXT: v_mov_b32_e32 v1, s55297; CI-NEXT: s_lshl_b64 s[4:5], s[6:7], 35298; CI-NEXT: v_mov_b32_e32 v2, s45299; CI-NEXT: s_mov_b32 s8, s25300; CI-NEXT: s_mov_b32 s9, s35301; CI-NEXT: s_mov_b32 s2, 05302; CI-NEXT: s_mov_b32 s3, s115303; CI-NEXT: v_mov_b32_e32 v3, s55304; CI-NEXT: buffer_atomic_xor_x2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:32 glc5305; CI-NEXT: s_waitcnt vmcnt(0)5306; CI-NEXT: buffer_wbinvl1_vol5307; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 05308; CI-NEXT: s_endpgm5309;5310; VI-LABEL: atomic_xor_i64_ret_addr64_offset:5311; VI: ; %bb.0: ; %entry5312; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x245313; VI-NEXT: s_waitcnt lgkmcnt(0)5314; VI-NEXT: v_mov_b32_e32 v0, s45315; VI-NEXT: v_mov_b32_e32 v1, s55316; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 35317; VI-NEXT: s_add_u32 s0, s0, s45318; VI-NEXT: s_addc_u32 s1, s1, s55319; VI-NEXT: s_add_u32 s0, s0, 325320; VI-NEXT: s_addc_u32 s1, s1, 05321; VI-NEXT: v_mov_b32_e32 v3, s15322; VI-NEXT: v_mov_b32_e32 v2, s05323; VI-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3], v[0:1] glc5324; VI-NEXT: s_waitcnt vmcnt(0)5325; VI-NEXT: buffer_wbinvl1_vol5326; VI-NEXT: s_mov_b32 s7, 0xf0005327; VI-NEXT: s_mov_b32 s6, -15328; VI-NEXT: s_mov_b32 s4, s25329; VI-NEXT: s_mov_b32 s5, s35330; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 05331; VI-NEXT: s_endpgm5332;5333; GFX9-LABEL: atomic_xor_i64_ret_addr64_offset:5334; GFX9: ; %bb.0: ; %entry5335; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x245336; GFX9-NEXT: v_mov_b32_e32 v2, 05337; GFX9-NEXT: s_waitcnt lgkmcnt(0)5338; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 35339; GFX9-NEXT: s_add_u32 s0, s8, s05340; GFX9-NEXT: v_mov_b32_e32 v0, s125341; GFX9-NEXT: v_mov_b32_e32 v1, s135342; GFX9-NEXT: s_addc_u32 s1, s9, s15343; GFX9-NEXT: global_atomic_xor_x2 v[0:1], v2, v[0:1], s[0:1] offset:32 glc5344; GFX9-NEXT: s_waitcnt vmcnt(0)5345; GFX9-NEXT: buffer_wbinvl1_vol5346; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]5347; GFX9-NEXT: s_endpgm5348;5349; GFX12-LABEL: atomic_xor_i64_ret_addr64_offset:5350; GFX12: ; %bb.0: ; %entry5351; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x245352; GFX12-NEXT: s_wait_kmcnt 0x05353; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s55354; GFX12-NEXT: v_mov_b32_e32 v0, s45355; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 35356; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)5357; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]5358; GFX12-NEXT: global_atomic_xor_b64 v[0:1], v2, v[0:1], s[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV5359; GFX12-NEXT: s_wait_loadcnt 0x05360; GFX12-NEXT: global_inv scope:SCOPE_DEV5361; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]5362; GFX12-NEXT: s_endpgm5363entry:5364 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index5365 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 45366 %tmp0 = atomicrmw volatile xor ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !05367 store i64 %tmp0, ptr addrspace(1) %out25368 ret void5369}5370 5371define amdgpu_kernel void @atomic_xor_i64(ptr addrspace(1) %out, i64 %in) {5372; CI-LABEL: atomic_xor_i64:5373; CI: ; %bb.0: ; %entry5374; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95375; CI-NEXT: s_mov_b32 s7, 0xf0005376; CI-NEXT: s_mov_b32 s6, -15377; CI-NEXT: s_waitcnt lgkmcnt(0)5378; CI-NEXT: s_mov_b32 s4, s05379; CI-NEXT: s_mov_b32 s5, s15380; CI-NEXT: v_mov_b32_e32 v0, s25381; CI-NEXT: v_mov_b32_e32 v1, s35382; CI-NEXT: buffer_atomic_xor_x2 v[0:1], off, s[4:7], 05383; CI-NEXT: s_waitcnt vmcnt(0)5384; CI-NEXT: buffer_wbinvl1_vol5385; CI-NEXT: s_endpgm5386;5387; VI-LABEL: atomic_xor_i64:5388; VI: ; %bb.0: ; %entry5389; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245390; VI-NEXT: s_mov_b32 s7, 0xf0005391; VI-NEXT: s_mov_b32 s6, -15392; VI-NEXT: s_waitcnt lgkmcnt(0)5393; VI-NEXT: s_mov_b32 s4, s05394; VI-NEXT: s_mov_b32 s5, s15395; VI-NEXT: v_mov_b32_e32 v0, s25396; VI-NEXT: v_mov_b32_e32 v1, s35397; VI-NEXT: buffer_atomic_xor_x2 v[0:1], off, s[4:7], 05398; VI-NEXT: s_waitcnt vmcnt(0)5399; VI-NEXT: buffer_wbinvl1_vol5400; VI-NEXT: s_endpgm5401;5402; GFX9-LABEL: atomic_xor_i64:5403; GFX9: ; %bb.0: ; %entry5404; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245405; GFX9-NEXT: v_mov_b32_e32 v2, 05406; GFX9-NEXT: s_waitcnt lgkmcnt(0)5407; GFX9-NEXT: v_mov_b32_e32 v0, s25408; GFX9-NEXT: v_mov_b32_e32 v1, s35409; GFX9-NEXT: global_atomic_xor_x2 v2, v[0:1], s[0:1]5410; GFX9-NEXT: s_waitcnt vmcnt(0)5411; GFX9-NEXT: buffer_wbinvl1_vol5412; GFX9-NEXT: s_endpgm5413;5414; GFX12-LABEL: atomic_xor_i64:5415; GFX12: ; %bb.0: ; %entry5416; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x245417; GFX12-NEXT: s_wait_kmcnt 0x05418; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s35419; GFX12-NEXT: v_mov_b32_e32 v0, s25420; GFX12-NEXT: global_atomic_xor_b64 v2, v[0:1], s[0:1] scope:SCOPE_DEV5421; GFX12-NEXT: s_wait_storecnt 0x05422; GFX12-NEXT: global_inv scope:SCOPE_DEV5423; GFX12-NEXT: s_endpgm5424entry:5425 %tmp0 = atomicrmw volatile xor ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !05426 ret void5427}5428 5429define amdgpu_kernel void @atomic_xor_i64_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in) {5430; CI-LABEL: atomic_xor_i64_ret:5431; CI: ; %bb.0: ; %entry5432; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95433; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd5434; CI-NEXT: s_mov_b32 s7, 0xf0005435; CI-NEXT: s_mov_b32 s6, -15436; CI-NEXT: s_waitcnt lgkmcnt(0)5437; CI-NEXT: s_mov_b32 s4, s05438; CI-NEXT: s_mov_b32 s5, s15439; CI-NEXT: v_mov_b32_e32 v0, s85440; CI-NEXT: v_mov_b32_e32 v1, s95441; CI-NEXT: buffer_atomic_xor_x2 v[0:1], off, s[4:7], 0 glc5442; CI-NEXT: s_waitcnt vmcnt(0)5443; CI-NEXT: buffer_wbinvl1_vol5444; CI-NEXT: s_mov_b32 s4, s25445; CI-NEXT: s_mov_b32 s5, s35446; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 05447; CI-NEXT: s_endpgm5448;5449; VI-LABEL: atomic_xor_i64_ret:5450; VI: ; %bb.0: ; %entry5451; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245452; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x345453; VI-NEXT: s_mov_b32 s7, 0xf0005454; VI-NEXT: s_mov_b32 s6, -15455; VI-NEXT: s_waitcnt lgkmcnt(0)5456; VI-NEXT: s_mov_b32 s4, s05457; VI-NEXT: s_mov_b32 s5, s15458; VI-NEXT: v_mov_b32_e32 v0, s85459; VI-NEXT: v_mov_b32_e32 v1, s95460; VI-NEXT: buffer_atomic_xor_x2 v[0:1], off, s[4:7], 0 glc5461; VI-NEXT: s_waitcnt vmcnt(0)5462; VI-NEXT: buffer_wbinvl1_vol5463; VI-NEXT: s_mov_b32 s4, s25464; VI-NEXT: s_mov_b32 s5, s35465; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 05466; VI-NEXT: s_endpgm5467;5468; GFX9-LABEL: atomic_xor_i64_ret:5469; GFX9: ; %bb.0: ; %entry5470; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x345471; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245472; GFX9-NEXT: v_mov_b32_e32 v2, 05473; GFX9-NEXT: s_waitcnt lgkmcnt(0)5474; GFX9-NEXT: v_mov_b32_e32 v0, s65475; GFX9-NEXT: v_mov_b32_e32 v1, s75476; GFX9-NEXT: global_atomic_xor_x2 v[0:1], v2, v[0:1], s[0:1] glc5477; GFX9-NEXT: s_waitcnt vmcnt(0)5478; GFX9-NEXT: buffer_wbinvl1_vol5479; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]5480; GFX9-NEXT: s_endpgm5481;5482; GFX12-LABEL: atomic_xor_i64_ret:5483; GFX12: ; %bb.0: ; %entry5484; GFX12-NEXT: s_clause 0x15485; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x345486; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x245487; GFX12-NEXT: s_wait_kmcnt 0x05488; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s75489; GFX12-NEXT: v_mov_b32_e32 v0, s65490; GFX12-NEXT: global_atomic_xor_b64 v[0:1], v2, v[0:1], s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV5491; GFX12-NEXT: s_wait_loadcnt 0x05492; GFX12-NEXT: global_inv scope:SCOPE_DEV5493; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]5494; GFX12-NEXT: s_endpgm5495entry:5496 %tmp0 = atomicrmw volatile xor ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !05497 store i64 %tmp0, ptr addrspace(1) %out25498 ret void5499}5500 5501define amdgpu_kernel void @atomic_xor_i64_addr64(ptr addrspace(1) %out, i64 %in, i64 %index) {5502; CI-LABEL: atomic_xor_i64_addr64:5503; CI: ; %bb.0: ; %entry5504; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95505; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd5506; CI-NEXT: s_waitcnt lgkmcnt(0)5507; CI-NEXT: v_mov_b32_e32 v0, s25508; CI-NEXT: s_lshl_b64 s[4:5], s[4:5], 35509; CI-NEXT: v_mov_b32_e32 v2, s45510; CI-NEXT: v_mov_b32_e32 v1, s35511; CI-NEXT: s_mov_b32 s3, 0xf0005512; CI-NEXT: s_mov_b32 s2, 05513; CI-NEXT: v_mov_b32_e32 v3, s55514; CI-NEXT: buffer_atomic_xor_x2 v[0:1], v[2:3], s[0:3], 0 addr645515; CI-NEXT: s_waitcnt vmcnt(0)5516; CI-NEXT: buffer_wbinvl1_vol5517; CI-NEXT: s_endpgm5518;5519; VI-LABEL: atomic_xor_i64_addr64:5520; VI: ; %bb.0: ; %entry5521; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245522; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x345523; VI-NEXT: s_waitcnt lgkmcnt(0)5524; VI-NEXT: v_mov_b32_e32 v0, s25525; VI-NEXT: v_mov_b32_e32 v1, s35526; VI-NEXT: s_lshl_b64 s[2:3], s[4:5], 35527; VI-NEXT: s_add_u32 s0, s0, s25528; VI-NEXT: s_addc_u32 s1, s1, s35529; VI-NEXT: v_mov_b32_e32 v3, s15530; VI-NEXT: v_mov_b32_e32 v2, s05531; VI-NEXT: flat_atomic_xor_x2 v[2:3], v[0:1]5532; VI-NEXT: s_waitcnt vmcnt(0)5533; VI-NEXT: buffer_wbinvl1_vol5534; VI-NEXT: s_endpgm5535;5536; GFX9-LABEL: atomic_xor_i64_addr64:5537; GFX9: ; %bb.0: ; %entry5538; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245539; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x345540; GFX9-NEXT: v_mov_b32_e32 v2, 05541; GFX9-NEXT: s_waitcnt lgkmcnt(0)5542; GFX9-NEXT: v_mov_b32_e32 v0, s25543; GFX9-NEXT: v_mov_b32_e32 v1, s35544; GFX9-NEXT: s_lshl_b64 s[2:3], s[6:7], 35545; GFX9-NEXT: s_add_u32 s0, s0, s25546; GFX9-NEXT: s_addc_u32 s1, s1, s35547; GFX9-NEXT: global_atomic_xor_x2 v2, v[0:1], s[0:1]5548; GFX9-NEXT: s_waitcnt vmcnt(0)5549; GFX9-NEXT: buffer_wbinvl1_vol5550; GFX9-NEXT: s_endpgm5551;5552; GFX12-LABEL: atomic_xor_i64_addr64:5553; GFX12: ; %bb.0: ; %entry5554; GFX12-NEXT: s_clause 0x15555; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x245556; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x345557; GFX12-NEXT: v_mov_b32_e32 v2, 05558; GFX12-NEXT: s_wait_kmcnt 0x05559; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s35560; GFX12-NEXT: s_lshl_b64 s[2:3], s[4:5], 35561; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)5562; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]5563; GFX12-NEXT: global_atomic_xor_b64 v2, v[0:1], s[0:1] scope:SCOPE_DEV5564; GFX12-NEXT: s_wait_storecnt 0x05565; GFX12-NEXT: global_inv scope:SCOPE_DEV5566; GFX12-NEXT: s_endpgm5567entry:5568 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index5569 %tmp0 = atomicrmw volatile xor ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !05570 ret void5571}5572 5573define amdgpu_kernel void @atomic_xor_i64_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index) {5574; CI-LABEL: atomic_xor_i64_ret_addr64:5575; CI: ; %bb.0: ; %entry5576; CI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x95577; CI-NEXT: s_mov_b32 s11, 0xf0005578; CI-NEXT: s_mov_b32 s10, -15579; CI-NEXT: s_waitcnt lgkmcnt(0)5580; CI-NEXT: v_mov_b32_e32 v0, s45581; CI-NEXT: v_mov_b32_e32 v1, s55582; CI-NEXT: s_lshl_b64 s[4:5], s[6:7], 35583; CI-NEXT: v_mov_b32_e32 v2, s45584; CI-NEXT: s_mov_b32 s8, s25585; CI-NEXT: s_mov_b32 s9, s35586; CI-NEXT: s_mov_b32 s2, 05587; CI-NEXT: s_mov_b32 s3, s115588; CI-NEXT: v_mov_b32_e32 v3, s55589; CI-NEXT: buffer_atomic_xor_x2 v[0:1], v[2:3], s[0:3], 0 addr64 glc5590; CI-NEXT: s_waitcnt vmcnt(0)5591; CI-NEXT: buffer_wbinvl1_vol5592; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 05593; CI-NEXT: s_endpgm5594;5595; VI-LABEL: atomic_xor_i64_ret_addr64:5596; VI: ; %bb.0: ; %entry5597; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x245598; VI-NEXT: s_waitcnt lgkmcnt(0)5599; VI-NEXT: v_mov_b32_e32 v0, s45600; VI-NEXT: v_mov_b32_e32 v1, s55601; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 35602; VI-NEXT: s_add_u32 s0, s0, s45603; VI-NEXT: s_addc_u32 s1, s1, s55604; VI-NEXT: v_mov_b32_e32 v3, s15605; VI-NEXT: v_mov_b32_e32 v2, s05606; VI-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3], v[0:1] glc5607; VI-NEXT: s_waitcnt vmcnt(0)5608; VI-NEXT: buffer_wbinvl1_vol5609; VI-NEXT: s_mov_b32 s7, 0xf0005610; VI-NEXT: s_mov_b32 s6, -15611; VI-NEXT: s_mov_b32 s4, s25612; VI-NEXT: s_mov_b32 s5, s35613; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 05614; VI-NEXT: s_endpgm5615;5616; GFX9-LABEL: atomic_xor_i64_ret_addr64:5617; GFX9: ; %bb.0: ; %entry5618; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x245619; GFX9-NEXT: v_mov_b32_e32 v2, 05620; GFX9-NEXT: s_waitcnt lgkmcnt(0)5621; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 35622; GFX9-NEXT: s_add_u32 s0, s8, s05623; GFX9-NEXT: v_mov_b32_e32 v0, s125624; GFX9-NEXT: v_mov_b32_e32 v1, s135625; GFX9-NEXT: s_addc_u32 s1, s9, s15626; GFX9-NEXT: global_atomic_xor_x2 v[0:1], v2, v[0:1], s[0:1] glc5627; GFX9-NEXT: s_waitcnt vmcnt(0)5628; GFX9-NEXT: buffer_wbinvl1_vol5629; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]5630; GFX9-NEXT: s_endpgm5631;5632; GFX12-LABEL: atomic_xor_i64_ret_addr64:5633; GFX12: ; %bb.0: ; %entry5634; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x245635; GFX12-NEXT: s_wait_kmcnt 0x05636; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s55637; GFX12-NEXT: v_mov_b32_e32 v0, s45638; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 35639; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)5640; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]5641; GFX12-NEXT: global_atomic_xor_b64 v[0:1], v2, v[0:1], s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV5642; GFX12-NEXT: s_wait_loadcnt 0x05643; GFX12-NEXT: global_inv scope:SCOPE_DEV5644; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]5645; GFX12-NEXT: s_endpgm5646entry:5647 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index5648 %tmp0 = atomicrmw volatile xor ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !05649 store i64 %tmp0, ptr addrspace(1) %out25650 ret void5651}5652 5653define amdgpu_kernel void @atomic_cmpxchg_i64_offset(ptr addrspace(1) %out, i64 %in, i64 %old) {5654; CI-LABEL: atomic_cmpxchg_i64_offset:5655; CI: ; %bb.0: ; %entry5656; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95657; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd5658; CI-NEXT: s_mov_b32 s7, 0xf0005659; CI-NEXT: s_mov_b32 s6, -15660; CI-NEXT: s_waitcnt lgkmcnt(0)5661; CI-NEXT: s_mov_b32 s4, s05662; CI-NEXT: s_mov_b32 s5, s15663; CI-NEXT: v_mov_b32_e32 v0, s25664; CI-NEXT: v_mov_b32_e32 v1, s35665; CI-NEXT: v_mov_b32_e32 v2, s85666; CI-NEXT: v_mov_b32_e32 v3, s95667; CI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 offset:325668; CI-NEXT: s_waitcnt vmcnt(0)5669; CI-NEXT: buffer_wbinvl1_vol5670; CI-NEXT: s_endpgm5671;5672; VI-LABEL: atomic_cmpxchg_i64_offset:5673; VI: ; %bb.0: ; %entry5674; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245675; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x345676; VI-NEXT: s_mov_b32 s7, 0xf0005677; VI-NEXT: s_mov_b32 s6, -15678; VI-NEXT: s_waitcnt lgkmcnt(0)5679; VI-NEXT: s_mov_b32 s4, s05680; VI-NEXT: s_mov_b32 s5, s15681; VI-NEXT: v_mov_b32_e32 v0, s25682; VI-NEXT: v_mov_b32_e32 v1, s35683; VI-NEXT: v_mov_b32_e32 v2, s85684; VI-NEXT: v_mov_b32_e32 v3, s95685; VI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 offset:325686; VI-NEXT: s_waitcnt vmcnt(0)5687; VI-NEXT: buffer_wbinvl1_vol5688; VI-NEXT: s_endpgm5689;5690; GFX9-LABEL: atomic_cmpxchg_i64_offset:5691; GFX9: ; %bb.0: ; %entry5692; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245693; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x345694; GFX9-NEXT: v_mov_b32_e32 v4, 05695; GFX9-NEXT: s_waitcnt lgkmcnt(0)5696; GFX9-NEXT: v_mov_b32_e32 v0, s25697; GFX9-NEXT: v_mov_b32_e32 v1, s35698; GFX9-NEXT: v_mov_b32_e32 v2, s65699; GFX9-NEXT: v_mov_b32_e32 v3, s75700; GFX9-NEXT: global_atomic_cmpswap_x2 v4, v[0:3], s[0:1] offset:325701; GFX9-NEXT: s_waitcnt vmcnt(0)5702; GFX9-NEXT: buffer_wbinvl1_vol5703; GFX9-NEXT: s_endpgm5704;5705; GFX12-LABEL: atomic_cmpxchg_i64_offset:5706; GFX12: ; %bb.0: ; %entry5707; GFX12-NEXT: s_clause 0x15708; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x245709; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x345710; GFX12-NEXT: s_wait_kmcnt 0x05711; GFX12-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s35712; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v3, s55713; GFX12-NEXT: v_mov_b32_e32 v2, s45714; GFX12-NEXT: global_atomic_cmpswap_b64 v4, v[0:3], s[0:1] offset:32 scope:SCOPE_DEV5715; GFX12-NEXT: s_wait_storecnt 0x05716; GFX12-NEXT: global_inv scope:SCOPE_DEV5717; GFX12-NEXT: s_endpgm5718entry:5719 %gep = getelementptr i64, ptr addrspace(1) %out, i64 45720 %val = cmpxchg volatile ptr addrspace(1) %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst5721 ret void5722}5723 5724define amdgpu_kernel void @atomic_cmpxchg_i64_soffset(ptr addrspace(1) %out, i64 %in, i64 %old) {5725; CI-LABEL: atomic_cmpxchg_i64_soffset:5726; CI: ; %bb.0: ; %entry5727; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95728; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd5729; CI-NEXT: s_mov_b32 s7, 0xf0005730; CI-NEXT: s_mov_b32 s6, -15731; CI-NEXT: s_waitcnt lgkmcnt(0)5732; CI-NEXT: s_mov_b32 s4, s05733; CI-NEXT: s_mov_b32 s5, s15734; CI-NEXT: s_mov_b32 s0, 0x119405735; CI-NEXT: v_mov_b32_e32 v0, s25736; CI-NEXT: v_mov_b32_e32 v1, s35737; CI-NEXT: v_mov_b32_e32 v2, s85738; CI-NEXT: v_mov_b32_e32 v3, s95739; CI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], s05740; CI-NEXT: s_waitcnt vmcnt(0)5741; CI-NEXT: buffer_wbinvl1_vol5742; CI-NEXT: s_endpgm5743;5744; VI-LABEL: atomic_cmpxchg_i64_soffset:5745; VI: ; %bb.0: ; %entry5746; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245747; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x345748; VI-NEXT: s_mov_b32 s7, 0xf0005749; VI-NEXT: s_mov_b32 s6, -15750; VI-NEXT: s_waitcnt lgkmcnt(0)5751; VI-NEXT: s_mov_b32 s4, s05752; VI-NEXT: s_mov_b32 s5, s15753; VI-NEXT: s_mov_b32 s0, 0x119405754; VI-NEXT: v_mov_b32_e32 v0, s25755; VI-NEXT: v_mov_b32_e32 v1, s35756; VI-NEXT: v_mov_b32_e32 v2, s85757; VI-NEXT: v_mov_b32_e32 v3, s95758; VI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], s05759; VI-NEXT: s_waitcnt vmcnt(0)5760; VI-NEXT: buffer_wbinvl1_vol5761; VI-NEXT: s_endpgm5762;5763; GFX9-LABEL: atomic_cmpxchg_i64_soffset:5764; GFX9: ; %bb.0: ; %entry5765; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245766; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x345767; GFX9-NEXT: v_mov_b32_e32 v4, 0x110005768; GFX9-NEXT: s_waitcnt lgkmcnt(0)5769; GFX9-NEXT: v_mov_b32_e32 v0, s25770; GFX9-NEXT: v_mov_b32_e32 v1, s35771; GFX9-NEXT: v_mov_b32_e32 v2, s65772; GFX9-NEXT: v_mov_b32_e32 v3, s75773; GFX9-NEXT: global_atomic_cmpswap_x2 v4, v[0:3], s[0:1] offset:23685774; GFX9-NEXT: s_waitcnt vmcnt(0)5775; GFX9-NEXT: buffer_wbinvl1_vol5776; GFX9-NEXT: s_endpgm5777;5778; GFX12-LABEL: atomic_cmpxchg_i64_soffset:5779; GFX12: ; %bb.0: ; %entry5780; GFX12-NEXT: s_clause 0x15781; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x245782; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x345783; GFX12-NEXT: s_wait_kmcnt 0x05784; GFX12-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s35785; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v3, s55786; GFX12-NEXT: v_mov_b32_e32 v2, s45787; GFX12-NEXT: global_atomic_cmpswap_b64 v4, v[0:3], s[0:1] offset:72000 scope:SCOPE_DEV5788; GFX12-NEXT: s_wait_storecnt 0x05789; GFX12-NEXT: global_inv scope:SCOPE_DEV5790; GFX12-NEXT: s_endpgm5791entry:5792 %gep = getelementptr i64, ptr addrspace(1) %out, i64 90005793 %val = cmpxchg volatile ptr addrspace(1) %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst5794 ret void5795}5796 5797define amdgpu_kernel void @atomic_cmpxchg_i64_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %old) {5798; CI-LABEL: atomic_cmpxchg_i64_ret_offset:5799; CI: ; %bb.0: ; %entry5800; CI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x95801; CI-NEXT: s_mov_b32 s11, 0xf0005802; CI-NEXT: s_mov_b32 s10, -15803; CI-NEXT: s_waitcnt lgkmcnt(0)5804; CI-NEXT: s_mov_b32 s8, s25805; CI-NEXT: s_mov_b32 s9, s35806; CI-NEXT: s_mov_b32 s2, s105807; CI-NEXT: s_mov_b32 s3, s115808; CI-NEXT: v_mov_b32_e32 v0, s45809; CI-NEXT: v_mov_b32_e32 v1, s55810; CI-NEXT: v_mov_b32_e32 v2, s65811; CI-NEXT: v_mov_b32_e32 v3, s75812; CI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[0:3], 0 offset:32 glc5813; CI-NEXT: s_waitcnt vmcnt(0)5814; CI-NEXT: buffer_wbinvl1_vol5815; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 05816; CI-NEXT: s_endpgm5817;5818; VI-LABEL: atomic_cmpxchg_i64_ret_offset:5819; VI: ; %bb.0: ; %entry5820; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x245821; VI-NEXT: s_mov_b32 s11, 0xf0005822; VI-NEXT: s_mov_b32 s10, -15823; VI-NEXT: s_waitcnt lgkmcnt(0)5824; VI-NEXT: s_mov_b32 s8, s25825; VI-NEXT: s_mov_b32 s9, s35826; VI-NEXT: s_mov_b32 s2, s105827; VI-NEXT: s_mov_b32 s3, s115828; VI-NEXT: v_mov_b32_e32 v0, s45829; VI-NEXT: v_mov_b32_e32 v1, s55830; VI-NEXT: v_mov_b32_e32 v2, s65831; VI-NEXT: v_mov_b32_e32 v3, s75832; VI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[0:3], 0 offset:32 glc5833; VI-NEXT: s_waitcnt vmcnt(0)5834; VI-NEXT: buffer_wbinvl1_vol5835; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 05836; VI-NEXT: s_endpgm5837;5838; GFX9-LABEL: atomic_cmpxchg_i64_ret_offset:5839; GFX9: ; %bb.0: ; %entry5840; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x245841; GFX9-NEXT: v_mov_b32_e32 v4, 05842; GFX9-NEXT: s_waitcnt lgkmcnt(0)5843; GFX9-NEXT: v_mov_b32_e32 v0, s125844; GFX9-NEXT: v_mov_b32_e32 v1, s135845; GFX9-NEXT: v_mov_b32_e32 v2, s145846; GFX9-NEXT: v_mov_b32_e32 v3, s155847; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[8:9] offset:32 glc5848; GFX9-NEXT: s_waitcnt vmcnt(0)5849; GFX9-NEXT: buffer_wbinvl1_vol5850; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[10:11]5851; GFX9-NEXT: s_endpgm5852;5853; GFX12-LABEL: atomic_cmpxchg_i64_ret_offset:5854; GFX12: ; %bb.0: ; %entry5855; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x245856; GFX12-NEXT: s_wait_kmcnt 0x05857; GFX12-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s55858; GFX12-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v3, s75859; GFX12-NEXT: v_mov_b32_e32 v2, s65860; GFX12-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV5861; GFX12-NEXT: s_wait_loadcnt 0x05862; GFX12-NEXT: global_inv scope:SCOPE_DEV5863; GFX12-NEXT: global_store_b64 v4, v[0:1], s[2:3]5864; GFX12-NEXT: s_endpgm5865entry:5866 %gep = getelementptr i64, ptr addrspace(1) %out, i64 45867 %val = cmpxchg volatile ptr addrspace(1) %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst5868 %extract0 = extractvalue { i64, i1 } %val, 05869 store i64 %extract0, ptr addrspace(1) %out25870 ret void5871}5872 5873define amdgpu_kernel void @atomic_cmpxchg_i64_addr64_offset(ptr addrspace(1) %out, i64 %in, i64 %index, i64 %old) {5874; CI-LABEL: atomic_cmpxchg_i64_addr64_offset:5875; CI: ; %bb.0: ; %entry5876; CI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x95877; CI-NEXT: s_mov_b32 s11, 0xf0005878; CI-NEXT: s_mov_b32 s10, 05879; CI-NEXT: s_waitcnt lgkmcnt(0)5880; CI-NEXT: s_lshl_b64 s[4:5], s[4:5], 35881; CI-NEXT: v_mov_b32_e32 v4, s45882; CI-NEXT: s_mov_b64 s[8:9], s[0:1]5883; CI-NEXT: v_mov_b32_e32 v0, s25884; CI-NEXT: v_mov_b32_e32 v1, s35885; CI-NEXT: v_mov_b32_e32 v2, s65886; CI-NEXT: v_mov_b32_e32 v3, s75887; CI-NEXT: v_mov_b32_e32 v5, s55888; CI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[8:11], 0 addr64 offset:325889; CI-NEXT: s_waitcnt vmcnt(0)5890; CI-NEXT: buffer_wbinvl1_vol5891; CI-NEXT: s_endpgm5892;5893; VI-LABEL: atomic_cmpxchg_i64_addr64_offset:5894; VI: ; %bb.0: ; %entry5895; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x245896; VI-NEXT: s_waitcnt lgkmcnt(0)5897; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], 35898; VI-NEXT: s_add_u32 s0, s0, s45899; VI-NEXT: s_addc_u32 s1, s1, s55900; VI-NEXT: s_add_u32 s0, s0, 325901; VI-NEXT: s_addc_u32 s1, s1, 05902; VI-NEXT: v_mov_b32_e32 v5, s15903; VI-NEXT: v_mov_b32_e32 v0, s25904; VI-NEXT: v_mov_b32_e32 v1, s35905; VI-NEXT: v_mov_b32_e32 v2, s65906; VI-NEXT: v_mov_b32_e32 v3, s75907; VI-NEXT: v_mov_b32_e32 v4, s05908; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:3]5909; VI-NEXT: s_waitcnt vmcnt(0)5910; VI-NEXT: buffer_wbinvl1_vol5911; VI-NEXT: s_endpgm5912;5913; GFX9-LABEL: atomic_cmpxchg_i64_addr64_offset:5914; GFX9: ; %bb.0: ; %entry5915; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x245916; GFX9-NEXT: v_mov_b32_e32 v4, 05917; GFX9-NEXT: s_waitcnt lgkmcnt(0)5918; GFX9-NEXT: s_lshl_b64 s[0:1], s[12:13], 35919; GFX9-NEXT: s_add_u32 s0, s8, s05920; GFX9-NEXT: v_mov_b32_e32 v0, s105921; GFX9-NEXT: v_mov_b32_e32 v1, s115922; GFX9-NEXT: v_mov_b32_e32 v2, s145923; GFX9-NEXT: s_addc_u32 s1, s9, s15924; GFX9-NEXT: v_mov_b32_e32 v3, s155925; GFX9-NEXT: global_atomic_cmpswap_x2 v4, v[0:3], s[0:1] offset:325926; GFX9-NEXT: s_waitcnt vmcnt(0)5927; GFX9-NEXT: buffer_wbinvl1_vol5928; GFX9-NEXT: s_endpgm5929;5930; GFX12-LABEL: atomic_cmpxchg_i64_addr64_offset:5931; GFX12: ; %bb.0: ; %entry5932; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x245933; GFX12-NEXT: s_wait_kmcnt 0x05934; GFX12-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s35935; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v3, s75936; GFX12-NEXT: v_mov_b32_e32 v2, s65937; GFX12-NEXT: s_lshl_b64 s[2:3], s[4:5], 35938; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)5939; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]5940; GFX12-NEXT: global_atomic_cmpswap_b64 v4, v[0:3], s[0:1] offset:32 scope:SCOPE_DEV5941; GFX12-NEXT: s_wait_storecnt 0x05942; GFX12-NEXT: global_inv scope:SCOPE_DEV5943; GFX12-NEXT: s_endpgm5944entry:5945 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index5946 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 45947 %val = cmpxchg volatile ptr addrspace(1) %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst5948 ret void5949}5950 5951define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index, i64 %old) {5952; CI-LABEL: atomic_cmpxchg_i64_ret_addr64_offset:5953; CI: ; %bb.0: ; %entry5954; CI-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x95955; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x115956; CI-NEXT: s_mov_b32 s3, 0xf0005957; CI-NEXT: s_mov_b32 s2, -15958; CI-NEXT: s_waitcnt lgkmcnt(0)5959; CI-NEXT: s_lshl_b64 s[6:7], s[14:15], 35960; CI-NEXT: v_mov_b32_e32 v4, s65961; CI-NEXT: s_mov_b32 s0, s105962; CI-NEXT: s_mov_b32 s1, s115963; CI-NEXT: s_mov_b32 s10, 05964; CI-NEXT: s_mov_b32 s11, s35965; CI-NEXT: v_mov_b32_e32 v0, s125966; CI-NEXT: v_mov_b32_e32 v1, s135967; CI-NEXT: v_mov_b32_e32 v2, s45968; CI-NEXT: v_mov_b32_e32 v3, s55969; CI-NEXT: v_mov_b32_e32 v5, s75970; CI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[8:11], 0 addr64 offset:32 glc5971; CI-NEXT: s_waitcnt vmcnt(0)5972; CI-NEXT: buffer_wbinvl1_vol5973; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 05974; CI-NEXT: s_endpgm5975;5976; VI-LABEL: atomic_cmpxchg_i64_ret_addr64_offset:5977; VI: ; %bb.0: ; %entry5978; VI-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x245979; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x445980; VI-NEXT: s_waitcnt lgkmcnt(0)5981; VI-NEXT: s_lshl_b64 s[2:3], s[14:15], 35982; VI-NEXT: v_mov_b32_e32 v2, s05983; VI-NEXT: s_add_u32 s0, s8, s25984; VI-NEXT: s_addc_u32 s3, s9, s35985; VI-NEXT: s_add_u32 s2, s0, 325986; VI-NEXT: s_addc_u32 s3, s3, 05987; VI-NEXT: v_mov_b32_e32 v5, s35988; VI-NEXT: v_mov_b32_e32 v0, s125989; VI-NEXT: v_mov_b32_e32 v1, s135990; VI-NEXT: v_mov_b32_e32 v3, s15991; VI-NEXT: v_mov_b32_e32 v4, s25992; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc5993; VI-NEXT: s_waitcnt vmcnt(0)5994; VI-NEXT: buffer_wbinvl1_vol5995; VI-NEXT: s_mov_b32 s3, 0xf0005996; VI-NEXT: s_mov_b32 s2, -15997; VI-NEXT: s_mov_b32 s0, s105998; VI-NEXT: s_mov_b32 s1, s115999; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 06000; VI-NEXT: s_endpgm6001;6002; GFX9-LABEL: atomic_cmpxchg_i64_ret_addr64_offset:6003; GFX9: ; %bb.0: ; %entry6004; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x246005; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x446006; GFX9-NEXT: v_mov_b32_e32 v4, 06007; GFX9-NEXT: s_waitcnt lgkmcnt(0)6008; GFX9-NEXT: s_lshl_b64 s[2:3], s[14:15], 36009; GFX9-NEXT: s_add_u32 s2, s8, s26010; GFX9-NEXT: v_mov_b32_e32 v0, s126011; GFX9-NEXT: v_mov_b32_e32 v1, s136012; GFX9-NEXT: s_addc_u32 s3, s9, s36013; GFX9-NEXT: v_mov_b32_e32 v2, s06014; GFX9-NEXT: v_mov_b32_e32 v3, s16015; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] offset:32 glc6016; GFX9-NEXT: s_waitcnt vmcnt(0)6017; GFX9-NEXT: buffer_wbinvl1_vol6018; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[10:11]6019; GFX9-NEXT: s_endpgm6020;6021; GFX12-LABEL: atomic_cmpxchg_i64_ret_addr64_offset:6022; GFX12: ; %bb.0: ; %entry6023; GFX12-NEXT: s_clause 0x16024; GFX12-NEXT: s_load_b256 s[8:15], s[4:5], 0x246025; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x446026; GFX12-NEXT: s_wait_kmcnt 0x06027; GFX12-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s136028; GFX12-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v3, s16029; GFX12-NEXT: v_mov_b32_e32 v2, s06030; GFX12-NEXT: s_lshl_b64 s[0:1], s[14:15], 36031; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)6032; GFX12-NEXT: s_add_nc_u64 s[0:1], s[8:9], s[0:1]6033; GFX12-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV6034; GFX12-NEXT: s_wait_loadcnt 0x06035; GFX12-NEXT: global_inv scope:SCOPE_DEV6036; GFX12-NEXT: global_store_b64 v4, v[0:1], s[10:11]6037; GFX12-NEXT: s_endpgm6038entry:6039 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index6040 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 46041 %val = cmpxchg volatile ptr addrspace(1) %gep, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst6042 %extract0 = extractvalue { i64, i1 } %val, 06043 store i64 %extract0, ptr addrspace(1) %out26044 ret void6045}6046 6047define amdgpu_kernel void @atomic_cmpxchg_i64(ptr addrspace(1) %out, i64 %in, i64 %old) {6048; CI-LABEL: atomic_cmpxchg_i64:6049; CI: ; %bb.0: ; %entry6050; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96051; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd6052; CI-NEXT: s_mov_b32 s7, 0xf0006053; CI-NEXT: s_mov_b32 s6, -16054; CI-NEXT: s_waitcnt lgkmcnt(0)6055; CI-NEXT: s_mov_b32 s4, s06056; CI-NEXT: s_mov_b32 s5, s16057; CI-NEXT: v_mov_b32_e32 v0, s26058; CI-NEXT: v_mov_b32_e32 v1, s36059; CI-NEXT: v_mov_b32_e32 v2, s86060; CI-NEXT: v_mov_b32_e32 v3, s96061; CI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 06062; CI-NEXT: s_waitcnt vmcnt(0)6063; CI-NEXT: buffer_wbinvl1_vol6064; CI-NEXT: s_endpgm6065;6066; VI-LABEL: atomic_cmpxchg_i64:6067; VI: ; %bb.0: ; %entry6068; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246069; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x346070; VI-NEXT: s_mov_b32 s7, 0xf0006071; VI-NEXT: s_mov_b32 s6, -16072; VI-NEXT: s_waitcnt lgkmcnt(0)6073; VI-NEXT: s_mov_b32 s4, s06074; VI-NEXT: s_mov_b32 s5, s16075; VI-NEXT: v_mov_b32_e32 v0, s26076; VI-NEXT: v_mov_b32_e32 v1, s36077; VI-NEXT: v_mov_b32_e32 v2, s86078; VI-NEXT: v_mov_b32_e32 v3, s96079; VI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 06080; VI-NEXT: s_waitcnt vmcnt(0)6081; VI-NEXT: buffer_wbinvl1_vol6082; VI-NEXT: s_endpgm6083;6084; GFX9-LABEL: atomic_cmpxchg_i64:6085; GFX9: ; %bb.0: ; %entry6086; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246087; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346088; GFX9-NEXT: v_mov_b32_e32 v4, 06089; GFX9-NEXT: s_waitcnt lgkmcnt(0)6090; GFX9-NEXT: v_mov_b32_e32 v0, s26091; GFX9-NEXT: v_mov_b32_e32 v1, s36092; GFX9-NEXT: v_mov_b32_e32 v2, s66093; GFX9-NEXT: v_mov_b32_e32 v3, s76094; GFX9-NEXT: global_atomic_cmpswap_x2 v4, v[0:3], s[0:1]6095; GFX9-NEXT: s_waitcnt vmcnt(0)6096; GFX9-NEXT: buffer_wbinvl1_vol6097; GFX9-NEXT: s_endpgm6098;6099; GFX12-LABEL: atomic_cmpxchg_i64:6100; GFX12: ; %bb.0: ; %entry6101; GFX12-NEXT: s_clause 0x16102; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x246103; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x346104; GFX12-NEXT: s_wait_kmcnt 0x06105; GFX12-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s36106; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v3, s56107; GFX12-NEXT: v_mov_b32_e32 v2, s46108; GFX12-NEXT: global_atomic_cmpswap_b64 v4, v[0:3], s[0:1] scope:SCOPE_DEV6109; GFX12-NEXT: s_wait_storecnt 0x06110; GFX12-NEXT: global_inv scope:SCOPE_DEV6111; GFX12-NEXT: s_endpgm6112entry:6113 %val = cmpxchg volatile ptr addrspace(1) %out, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst6114 ret void6115}6116 6117define amdgpu_kernel void @atomic_cmpxchg_i64_ret(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %old) {6118; CI-LABEL: atomic_cmpxchg_i64_ret:6119; CI: ; %bb.0: ; %entry6120; CI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x96121; CI-NEXT: s_mov_b32 s11, 0xf0006122; CI-NEXT: s_mov_b32 s10, -16123; CI-NEXT: s_waitcnt lgkmcnt(0)6124; CI-NEXT: s_mov_b32 s8, s06125; CI-NEXT: s_mov_b32 s9, s16126; CI-NEXT: v_mov_b32_e32 v0, s46127; CI-NEXT: v_mov_b32_e32 v1, s56128; CI-NEXT: v_mov_b32_e32 v2, s66129; CI-NEXT: v_mov_b32_e32 v3, s76130; CI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[8:11], 0 glc6131; CI-NEXT: s_waitcnt vmcnt(0)6132; CI-NEXT: buffer_wbinvl1_vol6133; CI-NEXT: s_mov_b32 s8, s26134; CI-NEXT: s_mov_b32 s9, s36135; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 06136; CI-NEXT: s_endpgm6137;6138; VI-LABEL: atomic_cmpxchg_i64_ret:6139; VI: ; %bb.0: ; %entry6140; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x246141; VI-NEXT: s_mov_b32 s11, 0xf0006142; VI-NEXT: s_mov_b32 s10, -16143; VI-NEXT: s_waitcnt lgkmcnt(0)6144; VI-NEXT: s_mov_b32 s8, s06145; VI-NEXT: s_mov_b32 s9, s16146; VI-NEXT: v_mov_b32_e32 v0, s46147; VI-NEXT: v_mov_b32_e32 v1, s56148; VI-NEXT: v_mov_b32_e32 v2, s66149; VI-NEXT: v_mov_b32_e32 v3, s76150; VI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[8:11], 0 glc6151; VI-NEXT: s_waitcnt vmcnt(0)6152; VI-NEXT: buffer_wbinvl1_vol6153; VI-NEXT: s_mov_b32 s8, s26154; VI-NEXT: s_mov_b32 s9, s36155; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 06156; VI-NEXT: s_endpgm6157;6158; GFX9-LABEL: atomic_cmpxchg_i64_ret:6159; GFX9: ; %bb.0: ; %entry6160; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x246161; GFX9-NEXT: v_mov_b32_e32 v4, 06162; GFX9-NEXT: s_waitcnt lgkmcnt(0)6163; GFX9-NEXT: v_mov_b32_e32 v0, s126164; GFX9-NEXT: v_mov_b32_e32 v1, s136165; GFX9-NEXT: v_mov_b32_e32 v2, s146166; GFX9-NEXT: v_mov_b32_e32 v3, s156167; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[8:9] glc6168; GFX9-NEXT: s_waitcnt vmcnt(0)6169; GFX9-NEXT: buffer_wbinvl1_vol6170; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[10:11]6171; GFX9-NEXT: s_endpgm6172;6173; GFX12-LABEL: atomic_cmpxchg_i64_ret:6174; GFX12: ; %bb.0: ; %entry6175; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x246176; GFX12-NEXT: s_wait_kmcnt 0x06177; GFX12-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s56178; GFX12-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v3, s76179; GFX12-NEXT: v_mov_b32_e32 v2, s66180; GFX12-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV6181; GFX12-NEXT: s_wait_loadcnt 0x06182; GFX12-NEXT: global_inv scope:SCOPE_DEV6183; GFX12-NEXT: global_store_b64 v4, v[0:1], s[2:3]6184; GFX12-NEXT: s_endpgm6185entry:6186 %val = cmpxchg volatile ptr addrspace(1) %out, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst6187 %extract0 = extractvalue { i64, i1 } %val, 06188 store i64 %extract0, ptr addrspace(1) %out26189 ret void6190}6191 6192define amdgpu_kernel void @atomic_cmpxchg_i64_addr64(ptr addrspace(1) %out, i64 %in, i64 %index, i64 %old) {6193; CI-LABEL: atomic_cmpxchg_i64_addr64:6194; CI: ; %bb.0: ; %entry6195; CI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x96196; CI-NEXT: s_mov_b32 s11, 0xf0006197; CI-NEXT: s_mov_b32 s10, 06198; CI-NEXT: s_waitcnt lgkmcnt(0)6199; CI-NEXT: s_mov_b64 s[8:9], s[0:1]6200; CI-NEXT: s_lshl_b64 s[0:1], s[4:5], 36201; CI-NEXT: v_mov_b32_e32 v5, s16202; CI-NEXT: v_mov_b32_e32 v0, s26203; CI-NEXT: v_mov_b32_e32 v1, s36204; CI-NEXT: v_mov_b32_e32 v2, s66205; CI-NEXT: v_mov_b32_e32 v3, s76206; CI-NEXT: v_mov_b32_e32 v4, s06207; CI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[8:11], 0 addr646208; CI-NEXT: s_waitcnt vmcnt(0)6209; CI-NEXT: buffer_wbinvl1_vol6210; CI-NEXT: s_endpgm6211;6212; VI-LABEL: atomic_cmpxchg_i64_addr64:6213; VI: ; %bb.0: ; %entry6214; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x246215; VI-NEXT: s_waitcnt lgkmcnt(0)6216; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], 36217; VI-NEXT: s_add_u32 s0, s0, s46218; VI-NEXT: s_addc_u32 s1, s1, s56219; VI-NEXT: v_mov_b32_e32 v5, s16220; VI-NEXT: v_mov_b32_e32 v0, s26221; VI-NEXT: v_mov_b32_e32 v1, s36222; VI-NEXT: v_mov_b32_e32 v2, s66223; VI-NEXT: v_mov_b32_e32 v3, s76224; VI-NEXT: v_mov_b32_e32 v4, s06225; VI-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:3]6226; VI-NEXT: s_waitcnt vmcnt(0)6227; VI-NEXT: buffer_wbinvl1_vol6228; VI-NEXT: s_endpgm6229;6230; GFX9-LABEL: atomic_cmpxchg_i64_addr64:6231; GFX9: ; %bb.0: ; %entry6232; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x246233; GFX9-NEXT: v_mov_b32_e32 v4, 06234; GFX9-NEXT: s_waitcnt lgkmcnt(0)6235; GFX9-NEXT: s_lshl_b64 s[0:1], s[12:13], 36236; GFX9-NEXT: s_add_u32 s0, s8, s06237; GFX9-NEXT: v_mov_b32_e32 v0, s106238; GFX9-NEXT: v_mov_b32_e32 v1, s116239; GFX9-NEXT: v_mov_b32_e32 v2, s146240; GFX9-NEXT: s_addc_u32 s1, s9, s16241; GFX9-NEXT: v_mov_b32_e32 v3, s156242; GFX9-NEXT: global_atomic_cmpswap_x2 v4, v[0:3], s[0:1]6243; GFX9-NEXT: s_waitcnt vmcnt(0)6244; GFX9-NEXT: buffer_wbinvl1_vol6245; GFX9-NEXT: s_endpgm6246;6247; GFX12-LABEL: atomic_cmpxchg_i64_addr64:6248; GFX12: ; %bb.0: ; %entry6249; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x246250; GFX12-NEXT: s_wait_kmcnt 0x06251; GFX12-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s36252; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v3, s76253; GFX12-NEXT: v_mov_b32_e32 v2, s66254; GFX12-NEXT: s_lshl_b64 s[2:3], s[4:5], 36255; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)6256; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]6257; GFX12-NEXT: global_atomic_cmpswap_b64 v4, v[0:3], s[0:1] scope:SCOPE_DEV6258; GFX12-NEXT: s_wait_storecnt 0x06259; GFX12-NEXT: global_inv scope:SCOPE_DEV6260; GFX12-NEXT: s_endpgm6261entry:6262 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index6263 %val = cmpxchg volatile ptr addrspace(1) %ptr, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst6264 ret void6265}6266 6267define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in, i64 %index, i64 %old) {6268; CI-LABEL: atomic_cmpxchg_i64_ret_addr64:6269; CI: ; %bb.0: ; %entry6270; CI-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x96271; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x116272; CI-NEXT: s_mov_b32 s3, 0xf0006273; CI-NEXT: s_mov_b32 s2, -16274; CI-NEXT: s_waitcnt lgkmcnt(0)6275; CI-NEXT: s_lshl_b64 s[6:7], s[14:15], 36276; CI-NEXT: v_mov_b32_e32 v4, s66277; CI-NEXT: s_mov_b32 s0, s106278; CI-NEXT: s_mov_b32 s1, s116279; CI-NEXT: s_mov_b32 s10, 06280; CI-NEXT: s_mov_b32 s11, s36281; CI-NEXT: v_mov_b32_e32 v0, s126282; CI-NEXT: v_mov_b32_e32 v1, s136283; CI-NEXT: v_mov_b32_e32 v2, s46284; CI-NEXT: v_mov_b32_e32 v3, s56285; CI-NEXT: v_mov_b32_e32 v5, s76286; CI-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[8:11], 0 addr64 glc6287; CI-NEXT: s_waitcnt vmcnt(0)6288; CI-NEXT: buffer_wbinvl1_vol6289; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 06290; CI-NEXT: s_endpgm6291;6292; VI-LABEL: atomic_cmpxchg_i64_ret_addr64:6293; VI: ; %bb.0: ; %entry6294; VI-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x246295; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x446296; VI-NEXT: s_waitcnt lgkmcnt(0)6297; VI-NEXT: s_lshl_b64 s[2:3], s[14:15], 36298; VI-NEXT: s_add_u32 s2, s8, s26299; VI-NEXT: s_addc_u32 s3, s9, s36300; VI-NEXT: v_mov_b32_e32 v5, s36301; VI-NEXT: v_mov_b32_e32 v0, s126302; VI-NEXT: v_mov_b32_e32 v1, s136303; VI-NEXT: v_mov_b32_e32 v2, s06304; VI-NEXT: v_mov_b32_e32 v3, s16305; VI-NEXT: v_mov_b32_e32 v4, s26306; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc6307; VI-NEXT: s_waitcnt vmcnt(0)6308; VI-NEXT: buffer_wbinvl1_vol6309; VI-NEXT: s_mov_b32 s3, 0xf0006310; VI-NEXT: s_mov_b32 s2, -16311; VI-NEXT: s_mov_b32 s0, s106312; VI-NEXT: s_mov_b32 s1, s116313; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 06314; VI-NEXT: s_endpgm6315;6316; GFX9-LABEL: atomic_cmpxchg_i64_ret_addr64:6317; GFX9: ; %bb.0: ; %entry6318; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x246319; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x446320; GFX9-NEXT: v_mov_b32_e32 v4, 06321; GFX9-NEXT: s_waitcnt lgkmcnt(0)6322; GFX9-NEXT: s_lshl_b64 s[2:3], s[14:15], 36323; GFX9-NEXT: s_add_u32 s2, s8, s26324; GFX9-NEXT: v_mov_b32_e32 v0, s126325; GFX9-NEXT: v_mov_b32_e32 v1, s136326; GFX9-NEXT: s_addc_u32 s3, s9, s36327; GFX9-NEXT: v_mov_b32_e32 v2, s06328; GFX9-NEXT: v_mov_b32_e32 v3, s16329; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc6330; GFX9-NEXT: s_waitcnt vmcnt(0)6331; GFX9-NEXT: buffer_wbinvl1_vol6332; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[10:11]6333; GFX9-NEXT: s_endpgm6334;6335; GFX12-LABEL: atomic_cmpxchg_i64_ret_addr64:6336; GFX12: ; %bb.0: ; %entry6337; GFX12-NEXT: s_clause 0x16338; GFX12-NEXT: s_load_b256 s[8:15], s[4:5], 0x246339; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x446340; GFX12-NEXT: s_wait_kmcnt 0x06341; GFX12-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s136342; GFX12-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v3, s16343; GFX12-NEXT: v_mov_b32_e32 v2, s06344; GFX12-NEXT: s_lshl_b64 s[0:1], s[14:15], 36345; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)6346; GFX12-NEXT: s_add_nc_u64 s[0:1], s[8:9], s[0:1]6347; GFX12-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV6348; GFX12-NEXT: s_wait_loadcnt 0x06349; GFX12-NEXT: global_inv scope:SCOPE_DEV6350; GFX12-NEXT: global_store_b64 v4, v[0:1], s[10:11]6351; GFX12-NEXT: s_endpgm6352entry:6353 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index6354 %val = cmpxchg volatile ptr addrspace(1) %ptr, i64 %old, i64 %in syncscope("agent") seq_cst seq_cst6355 %extract0 = extractvalue { i64, i1 } %val, 06356 store i64 %extract0, ptr addrspace(1) %out26357 ret void6358}6359 6360define amdgpu_kernel void @atomic_load_i64_offset(ptr addrspace(1) %in, ptr addrspace(1) %out) {6361; CI-LABEL: atomic_load_i64_offset:6362; CI: ; %bb.0: ; %entry6363; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96364; CI-NEXT: s_mov_b32 s7, 0xf0006365; CI-NEXT: s_mov_b32 s6, -16366; CI-NEXT: s_waitcnt lgkmcnt(0)6367; CI-NEXT: s_mov_b32 s4, s26368; CI-NEXT: s_mov_b32 s5, s36369; CI-NEXT: s_mov_b32 s2, s66370; CI-NEXT: s_mov_b32 s3, s76371; CI-NEXT: buffer_load_dwordx2 v[0:1], off, s[0:3], 0 offset:32 glc6372; CI-NEXT: s_waitcnt vmcnt(0)6373; CI-NEXT: buffer_wbinvl1_vol6374; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 06375; CI-NEXT: s_endpgm6376;6377; VI-LABEL: atomic_load_i64_offset:6378; VI: ; %bb.0: ; %entry6379; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246380; VI-NEXT: s_mov_b32 s7, 0xf0006381; VI-NEXT: s_mov_b32 s6, -16382; VI-NEXT: s_waitcnt lgkmcnt(0)6383; VI-NEXT: s_add_u32 s0, s0, 326384; VI-NEXT: s_addc_u32 s1, s1, 06385; VI-NEXT: v_mov_b32_e32 v0, s06386; VI-NEXT: v_mov_b32_e32 v1, s16387; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc6388; VI-NEXT: s_waitcnt vmcnt(0)6389; VI-NEXT: buffer_wbinvl1_vol6390; VI-NEXT: s_mov_b32 s4, s26391; VI-NEXT: s_mov_b32 s5, s36392; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 06393; VI-NEXT: s_endpgm6394;6395; GFX9-LABEL: atomic_load_i64_offset:6396; GFX9: ; %bb.0: ; %entry6397; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246398; GFX9-NEXT: v_mov_b32_e32 v2, 06399; GFX9-NEXT: s_waitcnt lgkmcnt(0)6400; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1] offset:32 glc6401; GFX9-NEXT: s_waitcnt vmcnt(0)6402; GFX9-NEXT: buffer_wbinvl1_vol6403; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]6404; GFX9-NEXT: s_endpgm6405;6406; GFX12-LABEL: atomic_load_i64_offset:6407; GFX12: ; %bb.0: ; %entry6408; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x246409; GFX12-NEXT: v_mov_b32_e32 v2, 06410; GFX12-NEXT: s_wait_kmcnt 0x06411; GFX12-NEXT: global_load_b64 v[0:1], v2, s[0:1] offset:32 scope:SCOPE_SYS6412; GFX12-NEXT: s_wait_loadcnt 0x06413; GFX12-NEXT: global_inv scope:SCOPE_SYS6414; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]6415; GFX12-NEXT: s_endpgm6416entry:6417 %gep = getelementptr i64, ptr addrspace(1) %in, i64 46418 %val = load atomic i64, ptr addrspace(1) %gep seq_cst, align 86419 store i64 %val, ptr addrspace(1) %out6420 ret void6421}6422 6423define amdgpu_kernel void @atomic_load_i64_neg_offset(ptr addrspace(1) %in, ptr addrspace(1) %out) {6424; CI-LABEL: atomic_load_i64_neg_offset:6425; CI: ; %bb.0: ; %entry6426; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96427; CI-NEXT: s_mov_b32 s7, 0xf0006428; CI-NEXT: v_not_b32_e32 v0, 316429; CI-NEXT: v_mov_b32_e32 v1, -16430; CI-NEXT: s_mov_b32 s6, -16431; CI-NEXT: s_waitcnt lgkmcnt(0)6432; CI-NEXT: s_mov_b32 s4, s26433; CI-NEXT: s_mov_b32 s5, s36434; CI-NEXT: s_mov_b32 s2, 06435; CI-NEXT: s_mov_b32 s3, s76436; CI-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc6437; CI-NEXT: s_waitcnt vmcnt(0)6438; CI-NEXT: buffer_wbinvl1_vol6439; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 06440; CI-NEXT: s_endpgm6441;6442; VI-LABEL: atomic_load_i64_neg_offset:6443; VI: ; %bb.0: ; %entry6444; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246445; VI-NEXT: s_mov_b32 s7, 0xf0006446; VI-NEXT: s_mov_b32 s6, -16447; VI-NEXT: s_waitcnt lgkmcnt(0)6448; VI-NEXT: s_add_u32 s0, s0, 0xffffffe06449; VI-NEXT: s_addc_u32 s1, s1, -16450; VI-NEXT: v_mov_b32_e32 v0, s06451; VI-NEXT: v_mov_b32_e32 v1, s16452; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc6453; VI-NEXT: s_waitcnt vmcnt(0)6454; VI-NEXT: buffer_wbinvl1_vol6455; VI-NEXT: s_mov_b32 s4, s26456; VI-NEXT: s_mov_b32 s5, s36457; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 06458; VI-NEXT: s_endpgm6459;6460; GFX9-LABEL: atomic_load_i64_neg_offset:6461; GFX9: ; %bb.0: ; %entry6462; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246463; GFX9-NEXT: v_mov_b32_e32 v2, 06464; GFX9-NEXT: s_waitcnt lgkmcnt(0)6465; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1] offset:-32 glc6466; GFX9-NEXT: s_waitcnt vmcnt(0)6467; GFX9-NEXT: buffer_wbinvl1_vol6468; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]6469; GFX9-NEXT: s_endpgm6470;6471; GFX12-LABEL: atomic_load_i64_neg_offset:6472; GFX12: ; %bb.0: ; %entry6473; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x246474; GFX12-NEXT: v_mov_b32_e32 v2, 06475; GFX12-NEXT: s_wait_kmcnt 0x06476; GFX12-NEXT: global_load_b64 v[0:1], v2, s[0:1] offset:-32 scope:SCOPE_SYS6477; GFX12-NEXT: s_wait_loadcnt 0x06478; GFX12-NEXT: global_inv scope:SCOPE_SYS6479; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]6480; GFX12-NEXT: s_endpgm6481entry:6482 %gep = getelementptr i64, ptr addrspace(1) %in, i64 -46483 %val = load atomic i64, ptr addrspace(1) %gep seq_cst, align 86484 store i64 %val, ptr addrspace(1) %out6485 ret void6486}6487 6488define amdgpu_kernel void @atomic_load_i64(ptr addrspace(1) %in, ptr addrspace(1) %out) {6489; CI-LABEL: atomic_load_i64:6490; CI: ; %bb.0: ; %entry6491; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96492; CI-NEXT: s_mov_b32 s7, 0xf0006493; CI-NEXT: s_mov_b32 s6, -16494; CI-NEXT: s_waitcnt lgkmcnt(0)6495; CI-NEXT: s_mov_b32 s4, s06496; CI-NEXT: s_mov_b32 s5, s16497; CI-NEXT: buffer_load_dwordx2 v[0:1], off, s[4:7], 0 glc6498; CI-NEXT: s_waitcnt vmcnt(0)6499; CI-NEXT: buffer_wbinvl1_vol6500; CI-NEXT: s_mov_b32 s4, s26501; CI-NEXT: s_mov_b32 s5, s36502; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 06503; CI-NEXT: s_endpgm6504;6505; VI-LABEL: atomic_load_i64:6506; VI: ; %bb.0: ; %entry6507; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246508; VI-NEXT: s_mov_b32 s7, 0xf0006509; VI-NEXT: s_mov_b32 s6, -16510; VI-NEXT: s_waitcnt lgkmcnt(0)6511; VI-NEXT: v_mov_b32_e32 v0, s06512; VI-NEXT: v_mov_b32_e32 v1, s16513; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc6514; VI-NEXT: s_waitcnt vmcnt(0)6515; VI-NEXT: buffer_wbinvl1_vol6516; VI-NEXT: s_mov_b32 s4, s26517; VI-NEXT: s_mov_b32 s5, s36518; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 06519; VI-NEXT: s_endpgm6520;6521; GFX9-LABEL: atomic_load_i64:6522; GFX9: ; %bb.0: ; %entry6523; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246524; GFX9-NEXT: v_mov_b32_e32 v2, 06525; GFX9-NEXT: s_waitcnt lgkmcnt(0)6526; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1] glc6527; GFX9-NEXT: s_waitcnt vmcnt(0)6528; GFX9-NEXT: buffer_wbinvl1_vol6529; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]6530; GFX9-NEXT: s_endpgm6531;6532; GFX12-LABEL: atomic_load_i64:6533; GFX12: ; %bb.0: ; %entry6534; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x246535; GFX12-NEXT: v_mov_b32_e32 v2, 06536; GFX12-NEXT: s_wait_kmcnt 0x06537; GFX12-NEXT: global_load_b64 v[0:1], v2, s[0:1] scope:SCOPE_DEV6538; GFX12-NEXT: s_wait_loadcnt 0x06539; GFX12-NEXT: global_inv scope:SCOPE_DEV6540; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]6541; GFX12-NEXT: s_endpgm6542entry:6543 %val = load atomic i64, ptr addrspace(1) %in syncscope("agent") seq_cst, align 86544 store i64 %val, ptr addrspace(1) %out6545 ret void6546}6547 6548define amdgpu_kernel void @atomic_load_i64_addr64_offset(ptr addrspace(1) %in, ptr addrspace(1) %out, i64 %index) {6549; CI-LABEL: atomic_load_i64_addr64_offset:6550; CI: ; %bb.0: ; %entry6551; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96552; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd6553; CI-NEXT: s_mov_b32 s7, 0xf0006554; CI-NEXT: s_mov_b32 s6, -16555; CI-NEXT: s_waitcnt lgkmcnt(0)6556; CI-NEXT: s_mov_b32 s4, s26557; CI-NEXT: s_lshl_b64 s[8:9], s[8:9], 36558; CI-NEXT: v_mov_b32_e32 v0, s86559; CI-NEXT: s_mov_b32 s5, s36560; CI-NEXT: s_mov_b32 s2, 06561; CI-NEXT: s_mov_b32 s3, s76562; CI-NEXT: v_mov_b32_e32 v1, s96563; CI-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 offset:32 glc6564; CI-NEXT: s_waitcnt vmcnt(0)6565; CI-NEXT: buffer_wbinvl1_vol6566; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 06567; CI-NEXT: s_endpgm6568;6569; VI-LABEL: atomic_load_i64_addr64_offset:6570; VI: ; %bb.0: ; %entry6571; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346572; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246573; VI-NEXT: s_waitcnt lgkmcnt(0)6574; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 36575; VI-NEXT: s_add_u32 s0, s0, s46576; VI-NEXT: s_addc_u32 s1, s1, s56577; VI-NEXT: s_add_u32 s0, s0, 326578; VI-NEXT: s_addc_u32 s1, s1, 06579; VI-NEXT: v_mov_b32_e32 v0, s06580; VI-NEXT: v_mov_b32_e32 v1, s16581; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc6582; VI-NEXT: s_waitcnt vmcnt(0)6583; VI-NEXT: buffer_wbinvl1_vol6584; VI-NEXT: s_mov_b32 s7, 0xf0006585; VI-NEXT: s_mov_b32 s6, -16586; VI-NEXT: s_mov_b32 s4, s26587; VI-NEXT: s_mov_b32 s5, s36588; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 06589; VI-NEXT: s_endpgm6590;6591; GFX9-LABEL: atomic_load_i64_addr64_offset:6592; GFX9: ; %bb.0: ; %entry6593; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346594; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246595; GFX9-NEXT: v_mov_b32_e32 v2, 06596; GFX9-NEXT: s_waitcnt lgkmcnt(0)6597; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 36598; GFX9-NEXT: s_add_u32 s0, s0, s46599; GFX9-NEXT: s_addc_u32 s1, s1, s56600; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1] offset:32 glc6601; GFX9-NEXT: s_waitcnt vmcnt(0)6602; GFX9-NEXT: buffer_wbinvl1_vol6603; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]6604; GFX9-NEXT: s_endpgm6605;6606; GFX12-LABEL: atomic_load_i64_addr64_offset:6607; GFX12: ; %bb.0: ; %entry6608; GFX12-NEXT: s_clause 0x16609; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x346610; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x246611; GFX12-NEXT: v_mov_b32_e32 v2, 06612; GFX12-NEXT: s_wait_kmcnt 0x06613; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 36614; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6615; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]6616; GFX12-NEXT: global_load_b64 v[0:1], v2, s[0:1] offset:32 scope:SCOPE_SYS6617; GFX12-NEXT: s_wait_loadcnt 0x06618; GFX12-NEXT: global_inv scope:SCOPE_SYS6619; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]6620; GFX12-NEXT: s_endpgm6621entry:6622 %ptr = getelementptr i64, ptr addrspace(1) %in, i64 %index6623 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 46624 %val = load atomic i64, ptr addrspace(1) %gep seq_cst, align 86625 store i64 %val, ptr addrspace(1) %out6626 ret void6627}6628 6629define amdgpu_kernel void @atomic_load_i64_addr64(ptr addrspace(1) %in, ptr addrspace(1) %out, i64 %index) {6630; CI-LABEL: atomic_load_i64_addr64:6631; CI: ; %bb.0: ; %entry6632; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96633; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd6634; CI-NEXT: s_mov_b32 s7, 0xf0006635; CI-NEXT: s_mov_b32 s6, -16636; CI-NEXT: s_waitcnt lgkmcnt(0)6637; CI-NEXT: s_mov_b32 s4, s26638; CI-NEXT: s_lshl_b64 s[8:9], s[8:9], 36639; CI-NEXT: v_mov_b32_e32 v0, s86640; CI-NEXT: s_mov_b32 s5, s36641; CI-NEXT: s_mov_b32 s2, 06642; CI-NEXT: s_mov_b32 s3, s76643; CI-NEXT: v_mov_b32_e32 v1, s96644; CI-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 glc6645; CI-NEXT: s_waitcnt vmcnt(0)6646; CI-NEXT: buffer_wbinvl1_vol6647; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 06648; CI-NEXT: s_endpgm6649;6650; VI-LABEL: atomic_load_i64_addr64:6651; VI: ; %bb.0: ; %entry6652; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346653; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246654; VI-NEXT: s_waitcnt lgkmcnt(0)6655; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 36656; VI-NEXT: s_add_u32 s0, s0, s46657; VI-NEXT: s_addc_u32 s1, s1, s56658; VI-NEXT: v_mov_b32_e32 v0, s06659; VI-NEXT: v_mov_b32_e32 v1, s16660; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc6661; VI-NEXT: s_waitcnt vmcnt(0)6662; VI-NEXT: buffer_wbinvl1_vol6663; VI-NEXT: s_mov_b32 s7, 0xf0006664; VI-NEXT: s_mov_b32 s6, -16665; VI-NEXT: s_mov_b32 s4, s26666; VI-NEXT: s_mov_b32 s5, s36667; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 06668; VI-NEXT: s_endpgm6669;6670; GFX9-LABEL: atomic_load_i64_addr64:6671; GFX9: ; %bb.0: ; %entry6672; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346673; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246674; GFX9-NEXT: v_mov_b32_e32 v2, 06675; GFX9-NEXT: s_waitcnt lgkmcnt(0)6676; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 36677; GFX9-NEXT: s_add_u32 s0, s0, s46678; GFX9-NEXT: s_addc_u32 s1, s1, s56679; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1] glc6680; GFX9-NEXT: s_waitcnt vmcnt(0)6681; GFX9-NEXT: buffer_wbinvl1_vol6682; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]6683; GFX9-NEXT: s_endpgm6684;6685; GFX12-LABEL: atomic_load_i64_addr64:6686; GFX12: ; %bb.0: ; %entry6687; GFX12-NEXT: s_clause 0x16688; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x346689; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x246690; GFX12-NEXT: v_mov_b32_e32 v2, 06691; GFX12-NEXT: s_wait_kmcnt 0x06692; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 36693; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6694; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]6695; GFX12-NEXT: global_load_b64 v[0:1], v2, s[0:1] scope:SCOPE_SYS6696; GFX12-NEXT: s_wait_loadcnt 0x06697; GFX12-NEXT: global_inv scope:SCOPE_SYS6698; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]6699; GFX12-NEXT: s_endpgm6700entry:6701 %ptr = getelementptr i64, ptr addrspace(1) %in, i64 %index6702 %val = load atomic i64, ptr addrspace(1) %ptr seq_cst, align 86703 store i64 %val, ptr addrspace(1) %out6704 ret void6705}6706 6707define amdgpu_kernel void @atomic_load_f64_addr64_offset(ptr addrspace(1) %in, ptr addrspace(1) %out, i64 %index) {6708; CI-LABEL: atomic_load_f64_addr64_offset:6709; CI: ; %bb.0: ; %entry6710; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96711; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd6712; CI-NEXT: s_mov_b32 s7, 0xf0006713; CI-NEXT: s_mov_b32 s6, -16714; CI-NEXT: s_waitcnt lgkmcnt(0)6715; CI-NEXT: s_mov_b32 s4, s26716; CI-NEXT: s_lshl_b64 s[8:9], s[8:9], 36717; CI-NEXT: v_mov_b32_e32 v0, s86718; CI-NEXT: s_mov_b32 s5, s36719; CI-NEXT: s_mov_b32 s2, 06720; CI-NEXT: s_mov_b32 s3, s76721; CI-NEXT: v_mov_b32_e32 v1, s96722; CI-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64 offset:32 glc6723; CI-NEXT: s_waitcnt vmcnt(0)6724; CI-NEXT: buffer_wbinvl1_vol6725; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 06726; CI-NEXT: s_endpgm6727;6728; VI-LABEL: atomic_load_f64_addr64_offset:6729; VI: ; %bb.0: ; %entry6730; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346731; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246732; VI-NEXT: s_waitcnt lgkmcnt(0)6733; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 36734; VI-NEXT: s_add_u32 s0, s0, s46735; VI-NEXT: s_addc_u32 s1, s1, s56736; VI-NEXT: s_add_u32 s0, s0, 326737; VI-NEXT: s_addc_u32 s1, s1, 06738; VI-NEXT: v_mov_b32_e32 v0, s06739; VI-NEXT: v_mov_b32_e32 v1, s16740; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc6741; VI-NEXT: s_waitcnt vmcnt(0)6742; VI-NEXT: buffer_wbinvl1_vol6743; VI-NEXT: s_mov_b32 s7, 0xf0006744; VI-NEXT: s_mov_b32 s6, -16745; VI-NEXT: s_mov_b32 s4, s26746; VI-NEXT: s_mov_b32 s5, s36747; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 06748; VI-NEXT: s_endpgm6749;6750; GFX9-LABEL: atomic_load_f64_addr64_offset:6751; GFX9: ; %bb.0: ; %entry6752; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346753; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246754; GFX9-NEXT: v_mov_b32_e32 v2, 06755; GFX9-NEXT: s_waitcnt lgkmcnt(0)6756; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 36757; GFX9-NEXT: s_add_u32 s0, s0, s46758; GFX9-NEXT: s_addc_u32 s1, s1, s56759; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1] offset:32 glc6760; GFX9-NEXT: s_waitcnt vmcnt(0)6761; GFX9-NEXT: buffer_wbinvl1_vol6762; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]6763; GFX9-NEXT: s_endpgm6764;6765; GFX12-LABEL: atomic_load_f64_addr64_offset:6766; GFX12: ; %bb.0: ; %entry6767; GFX12-NEXT: s_clause 0x16768; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x346769; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x246770; GFX12-NEXT: v_mov_b32_e32 v2, 06771; GFX12-NEXT: s_wait_kmcnt 0x06772; GFX12-NEXT: s_lshl_b64 s[4:5], s[6:7], 36773; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6774; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]6775; GFX12-NEXT: global_load_b64 v[0:1], v2, s[0:1] offset:32 scope:SCOPE_SYS6776; GFX12-NEXT: s_wait_loadcnt 0x06777; GFX12-NEXT: global_inv scope:SCOPE_SYS6778; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]6779; GFX12-NEXT: s_endpgm6780entry:6781 %ptr = getelementptr double, ptr addrspace(1) %in, i64 %index6782 %gep = getelementptr double, ptr addrspace(1) %ptr, i64 46783 %val = load atomic double, ptr addrspace(1) %gep seq_cst, align 86784 store double %val, ptr addrspace(1) %out6785 ret void6786}6787 6788define amdgpu_kernel void @atomic_store_i64_offset(i64 %in, ptr addrspace(1) %out) {6789; CI-LABEL: atomic_store_i64_offset:6790; CI: ; %bb.0: ; %entry6791; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96792; CI-NEXT: s_mov_b32 s7, 0xf0006793; CI-NEXT: s_mov_b32 s6, -16794; CI-NEXT: s_waitcnt lgkmcnt(0)6795; CI-NEXT: v_mov_b32_e32 v0, s06796; CI-NEXT: v_mov_b32_e32 v1, s16797; CI-NEXT: s_mov_b32 s4, s26798; CI-NEXT: s_mov_b32 s5, s36799; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 offset:326800; CI-NEXT: s_endpgm6801;6802; VI-LABEL: atomic_store_i64_offset:6803; VI: ; %bb.0: ; %entry6804; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246805; VI-NEXT: s_waitcnt lgkmcnt(0)6806; VI-NEXT: v_mov_b32_e32 v0, s06807; VI-NEXT: s_add_u32 s0, s2, 326808; VI-NEXT: v_mov_b32_e32 v1, s16809; VI-NEXT: s_addc_u32 s1, s3, 06810; VI-NEXT: v_mov_b32_e32 v3, s16811; VI-NEXT: v_mov_b32_e32 v2, s06812; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]6813; VI-NEXT: s_endpgm6814;6815; GFX9-LABEL: atomic_store_i64_offset:6816; GFX9: ; %bb.0: ; %entry6817; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246818; GFX9-NEXT: v_mov_b32_e32 v2, 06819; GFX9-NEXT: s_waitcnt lgkmcnt(0)6820; GFX9-NEXT: v_mov_b32_e32 v0, s06821; GFX9-NEXT: v_mov_b32_e32 v1, s16822; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] offset:326823; GFX9-NEXT: s_endpgm6824;6825; GFX12-LABEL: atomic_store_i64_offset:6826; GFX12: ; %bb.0: ; %entry6827; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x246828; GFX12-NEXT: v_mov_b32_e32 v2, 06829; GFX12-NEXT: s_wait_kmcnt 0x06830; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16831; GFX12-NEXT: global_wb scope:SCOPE_SYS6832; GFX12-NEXT: s_wait_storecnt 0x06833; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3] offset:32 scope:SCOPE_SYS6834; GFX12-NEXT: s_endpgm6835entry:6836 %gep = getelementptr i64, ptr addrspace(1) %out, i64 46837 store atomic i64 %in, ptr addrspace(1) %gep seq_cst, align 86838 ret void6839}6840 6841define amdgpu_kernel void @atomic_store_i64(i64 %in, ptr addrspace(1) %out) {6842; CI-LABEL: atomic_store_i64:6843; CI: ; %bb.0: ; %entry6844; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96845; CI-NEXT: s_mov_b32 s7, 0xf0006846; CI-NEXT: s_mov_b32 s6, -16847; CI-NEXT: s_waitcnt lgkmcnt(0)6848; CI-NEXT: v_mov_b32_e32 v0, s06849; CI-NEXT: v_mov_b32_e32 v1, s16850; CI-NEXT: s_mov_b32 s4, s26851; CI-NEXT: s_mov_b32 s5, s36852; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 06853; CI-NEXT: s_endpgm6854;6855; VI-LABEL: atomic_store_i64:6856; VI: ; %bb.0: ; %entry6857; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246858; VI-NEXT: s_waitcnt lgkmcnt(0)6859; VI-NEXT: v_mov_b32_e32 v0, s06860; VI-NEXT: v_mov_b32_e32 v1, s16861; VI-NEXT: v_mov_b32_e32 v2, s26862; VI-NEXT: v_mov_b32_e32 v3, s36863; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]6864; VI-NEXT: s_endpgm6865;6866; GFX9-LABEL: atomic_store_i64:6867; GFX9: ; %bb.0: ; %entry6868; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246869; GFX9-NEXT: v_mov_b32_e32 v2, 06870; GFX9-NEXT: s_waitcnt lgkmcnt(0)6871; GFX9-NEXT: v_mov_b32_e32 v0, s06872; GFX9-NEXT: v_mov_b32_e32 v1, s16873; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]6874; GFX9-NEXT: s_endpgm6875;6876; GFX12-LABEL: atomic_store_i64:6877; GFX12: ; %bb.0: ; %entry6878; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x246879; GFX12-NEXT: v_mov_b32_e32 v2, 06880; GFX12-NEXT: s_wait_kmcnt 0x06881; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16882; GFX12-NEXT: global_wb scope:SCOPE_SYS6883; GFX12-NEXT: s_wait_storecnt 0x06884; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3] scope:SCOPE_SYS6885; GFX12-NEXT: s_endpgm6886entry:6887 store atomic i64 %in, ptr addrspace(1) %out seq_cst, align 86888 ret void6889}6890 6891define amdgpu_kernel void @atomic_store_i64_addr64_offset(i64 %in, ptr addrspace(1) %out, i64 %index) {6892; CI-LABEL: atomic_store_i64_addr64_offset:6893; CI: ; %bb.0: ; %entry6894; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96895; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd6896; CI-NEXT: s_mov_b32 s7, 0xf0006897; CI-NEXT: s_mov_b32 s6, 06898; CI-NEXT: s_waitcnt lgkmcnt(0)6899; CI-NEXT: v_mov_b32_e32 v0, s06900; CI-NEXT: v_mov_b32_e32 v1, s16901; CI-NEXT: s_lshl_b64 s[0:1], s[4:5], 36902; CI-NEXT: v_mov_b32_e32 v3, s16903; CI-NEXT: s_mov_b64 s[4:5], s[2:3]6904; CI-NEXT: v_mov_b32_e32 v2, s06905; CI-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[4:7], 0 addr64 offset:326906; CI-NEXT: s_endpgm6907;6908; VI-LABEL: atomic_store_i64_addr64_offset:6909; VI: ; %bb.0: ; %entry6910; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246911; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x346912; VI-NEXT: s_waitcnt lgkmcnt(0)6913; VI-NEXT: v_mov_b32_e32 v0, s06914; VI-NEXT: v_mov_b32_e32 v1, s16915; VI-NEXT: s_lshl_b64 s[0:1], s[4:5], 36916; VI-NEXT: s_add_u32 s0, s2, s06917; VI-NEXT: s_addc_u32 s1, s3, s16918; VI-NEXT: s_add_u32 s0, s0, 326919; VI-NEXT: s_addc_u32 s1, s1, 06920; VI-NEXT: v_mov_b32_e32 v3, s16921; VI-NEXT: v_mov_b32_e32 v2, s06922; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]6923; VI-NEXT: s_endpgm6924;6925; GFX9-LABEL: atomic_store_i64_addr64_offset:6926; GFX9: ; %bb.0: ; %entry6927; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246928; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346929; GFX9-NEXT: v_mov_b32_e32 v2, 06930; GFX9-NEXT: s_waitcnt lgkmcnt(0)6931; GFX9-NEXT: v_mov_b32_e32 v0, s06932; GFX9-NEXT: v_mov_b32_e32 v1, s16933; GFX9-NEXT: s_lshl_b64 s[0:1], s[6:7], 36934; GFX9-NEXT: s_add_u32 s0, s2, s06935; GFX9-NEXT: s_addc_u32 s1, s3, s16936; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] offset:326937; GFX9-NEXT: s_endpgm6938;6939; GFX12-LABEL: atomic_store_i64_addr64_offset:6940; GFX12: ; %bb.0: ; %entry6941; GFX12-NEXT: s_clause 0x16942; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x246943; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x346944; GFX12-NEXT: v_mov_b32_e32 v2, 06945; GFX12-NEXT: s_wait_kmcnt 0x06946; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16947; GFX12-NEXT: s_lshl_b64 s[0:1], s[4:5], 36948; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)6949; GFX12-NEXT: s_add_nc_u64 s[0:1], s[2:3], s[0:1]6950; GFX12-NEXT: global_wb scope:SCOPE_SYS6951; GFX12-NEXT: s_wait_storecnt 0x06952; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_SYS6953; GFX12-NEXT: s_endpgm6954entry:6955 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index6956 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 46957 store atomic i64 %in, ptr addrspace(1) %gep seq_cst, align 86958 ret void6959}6960 6961define amdgpu_kernel void @atomic_store_i64_addr64(i64 %in, ptr addrspace(1) %out, i64 %index) {6962; CI-LABEL: atomic_store_i64_addr64:6963; CI: ; %bb.0: ; %entry6964; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96965; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd6966; CI-NEXT: s_mov_b32 s7, 0xf0006967; CI-NEXT: s_mov_b32 s6, 06968; CI-NEXT: s_waitcnt lgkmcnt(0)6969; CI-NEXT: v_mov_b32_e32 v0, s06970; CI-NEXT: v_mov_b32_e32 v1, s16971; CI-NEXT: s_lshl_b64 s[0:1], s[8:9], 36972; CI-NEXT: v_mov_b32_e32 v3, s16973; CI-NEXT: s_mov_b64 s[4:5], s[2:3]6974; CI-NEXT: v_mov_b32_e32 v2, s06975; CI-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[4:7], 0 addr646976; CI-NEXT: s_endpgm6977;6978; VI-LABEL: atomic_store_i64_addr64:6979; VI: ; %bb.0: ; %entry6980; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246981; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x346982; VI-NEXT: s_waitcnt lgkmcnt(0)6983; VI-NEXT: v_mov_b32_e32 v0, s06984; VI-NEXT: v_mov_b32_e32 v1, s16985; VI-NEXT: s_lshl_b64 s[0:1], s[4:5], 36986; VI-NEXT: s_add_u32 s0, s2, s06987; VI-NEXT: s_addc_u32 s1, s3, s16988; VI-NEXT: v_mov_b32_e32 v3, s16989; VI-NEXT: v_mov_b32_e32 v2, s06990; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]6991; VI-NEXT: s_endpgm6992;6993; GFX9-LABEL: atomic_store_i64_addr64:6994; GFX9: ; %bb.0: ; %entry6995; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246996; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346997; GFX9-NEXT: v_mov_b32_e32 v2, 06998; GFX9-NEXT: s_waitcnt lgkmcnt(0)6999; GFX9-NEXT: v_mov_b32_e32 v0, s07000; GFX9-NEXT: v_mov_b32_e32 v1, s17001; GFX9-NEXT: s_lshl_b64 s[0:1], s[6:7], 37002; GFX9-NEXT: s_add_u32 s0, s2, s07003; GFX9-NEXT: s_addc_u32 s1, s3, s17004; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]7005; GFX9-NEXT: s_endpgm7006;7007; GFX12-LABEL: atomic_store_i64_addr64:7008; GFX12: ; %bb.0: ; %entry7009; GFX12-NEXT: s_clause 0x17010; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x247011; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x347012; GFX12-NEXT: v_mov_b32_e32 v2, 07013; GFX12-NEXT: s_wait_kmcnt 0x07014; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17015; GFX12-NEXT: s_lshl_b64 s[0:1], s[4:5], 37016; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)7017; GFX12-NEXT: s_add_nc_u64 s[0:1], s[2:3], s[0:1]7018; GFX12-NEXT: global_wb scope:SCOPE_SYS7019; GFX12-NEXT: s_wait_storecnt 0x07020; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1] scope:SCOPE_SYS7021; GFX12-NEXT: s_endpgm7022entry:7023 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index7024 store atomic i64 %in, ptr addrspace(1) %ptr seq_cst, align 87025 ret void7026}7027 7028define amdgpu_kernel void @atomic_store_f64_addr64_offset(double %in, ptr addrspace(1) %out, i64 %index) {7029; CI-LABEL: atomic_store_f64_addr64_offset:7030; CI: ; %bb.0: ; %entry7031; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x97032; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd7033; CI-NEXT: s_mov_b32 s7, 0xf0007034; CI-NEXT: s_mov_b32 s6, 07035; CI-NEXT: s_waitcnt lgkmcnt(0)7036; CI-NEXT: v_mov_b32_e32 v0, s07037; CI-NEXT: v_mov_b32_e32 v1, s17038; CI-NEXT: s_lshl_b64 s[0:1], s[4:5], 37039; CI-NEXT: v_mov_b32_e32 v3, s17040; CI-NEXT: s_mov_b64 s[4:5], s[2:3]7041; CI-NEXT: v_mov_b32_e32 v2, s07042; CI-NEXT: buffer_store_dwordx2 v[0:1], v[2:3], s[4:7], 0 addr64 offset:327043; CI-NEXT: s_endpgm7044;7045; VI-LABEL: atomic_store_f64_addr64_offset:7046; VI: ; %bb.0: ; %entry7047; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247048; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x347049; VI-NEXT: s_waitcnt lgkmcnt(0)7050; VI-NEXT: v_mov_b32_e32 v0, s07051; VI-NEXT: v_mov_b32_e32 v1, s17052; VI-NEXT: s_lshl_b64 s[0:1], s[4:5], 37053; VI-NEXT: s_add_u32 s0, s2, s07054; VI-NEXT: s_addc_u32 s1, s3, s17055; VI-NEXT: s_add_u32 s0, s0, 327056; VI-NEXT: s_addc_u32 s1, s1, 07057; VI-NEXT: v_mov_b32_e32 v3, s17058; VI-NEXT: v_mov_b32_e32 v2, s07059; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]7060; VI-NEXT: s_endpgm7061;7062; GFX9-LABEL: atomic_store_f64_addr64_offset:7063; GFX9: ; %bb.0: ; %entry7064; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247065; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x347066; GFX9-NEXT: v_mov_b32_e32 v2, 07067; GFX9-NEXT: s_waitcnt lgkmcnt(0)7068; GFX9-NEXT: v_mov_b32_e32 v0, s07069; GFX9-NEXT: v_mov_b32_e32 v1, s17070; GFX9-NEXT: s_lshl_b64 s[0:1], s[6:7], 37071; GFX9-NEXT: s_add_u32 s0, s2, s07072; GFX9-NEXT: s_addc_u32 s1, s3, s17073; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] offset:327074; GFX9-NEXT: s_endpgm7075;7076; GFX12-LABEL: atomic_store_f64_addr64_offset:7077; GFX12: ; %bb.0: ; %entry7078; GFX12-NEXT: s_clause 0x17079; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x247080; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x347081; GFX12-NEXT: v_mov_b32_e32 v2, 07082; GFX12-NEXT: s_wait_kmcnt 0x07083; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17084; GFX12-NEXT: s_lshl_b64 s[0:1], s[4:5], 37085; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)7086; GFX12-NEXT: s_add_nc_u64 s[0:1], s[2:3], s[0:1]7087; GFX12-NEXT: global_wb scope:SCOPE_SYS7088; GFX12-NEXT: s_wait_storecnt 0x07089; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_SYS7090; GFX12-NEXT: s_endpgm7091entry:7092 %ptr = getelementptr double, ptr addrspace(1) %out, i64 %index7093 %gep = getelementptr double, ptr addrspace(1) %ptr, i64 47094 store atomic double %in, ptr addrspace(1) %gep seq_cst, align 87095 ret void7096}7097 7098define amdgpu_kernel void @atomic_inc_i64_offset(ptr addrspace(1) %out, i64 %in) {7099; CI-LABEL: atomic_inc_i64_offset:7100; CI: ; %bb.0: ; %entry7101; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x97102; CI-NEXT: s_waitcnt lgkmcnt(0)7103; CI-NEXT: v_mov_b32_e32 v0, s27104; CI-NEXT: v_mov_b32_e32 v1, s37105; CI-NEXT: s_mov_b32 s3, 0xf0007106; CI-NEXT: s_mov_b32 s2, -17107; CI-NEXT: buffer_atomic_inc_x2 v[0:1], off, s[0:3], 0 offset:327108; CI-NEXT: s_waitcnt vmcnt(0)7109; CI-NEXT: buffer_wbinvl1_vol7110; CI-NEXT: s_endpgm7111;7112; VI-LABEL: atomic_inc_i64_offset:7113; VI: ; %bb.0: ; %entry7114; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247115; VI-NEXT: s_waitcnt lgkmcnt(0)7116; VI-NEXT: v_mov_b32_e32 v0, s27117; VI-NEXT: v_mov_b32_e32 v1, s37118; VI-NEXT: s_mov_b32 s3, 0xf0007119; VI-NEXT: s_mov_b32 s2, -17120; VI-NEXT: buffer_atomic_inc_x2 v[0:1], off, s[0:3], 0 offset:327121; VI-NEXT: s_waitcnt vmcnt(0)7122; VI-NEXT: buffer_wbinvl1_vol7123; VI-NEXT: s_endpgm7124;7125; GFX9-LABEL: atomic_inc_i64_offset:7126; GFX9: ; %bb.0: ; %entry7127; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247128; GFX9-NEXT: v_mov_b32_e32 v2, 07129; GFX9-NEXT: s_waitcnt lgkmcnt(0)7130; GFX9-NEXT: v_mov_b32_e32 v0, s27131; GFX9-NEXT: v_mov_b32_e32 v1, s37132; GFX9-NEXT: global_atomic_inc_x2 v2, v[0:1], s[0:1] offset:327133; GFX9-NEXT: s_waitcnt vmcnt(0)7134; GFX9-NEXT: buffer_wbinvl1_vol7135; GFX9-NEXT: s_endpgm7136;7137; GFX12-LABEL: atomic_inc_i64_offset:7138; GFX12: ; %bb.0: ; %entry7139; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x247140; GFX12-NEXT: s_wait_kmcnt 0x07141; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s37142; GFX12-NEXT: v_mov_b32_e32 v0, s27143; GFX12-NEXT: global_atomic_inc_u64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_DEV7144; GFX12-NEXT: s_wait_storecnt 0x07145; GFX12-NEXT: global_inv scope:SCOPE_DEV7146; GFX12-NEXT: s_endpgm7147entry:7148 %gep = getelementptr i64, ptr addrspace(1) %out, i64 47149 %tmp0 = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !07150 ret void7151}7152 7153define amdgpu_kernel void @atomic_inc_i64_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in) {7154; CI-LABEL: atomic_inc_i64_ret_offset:7155; CI: ; %bb.0: ; %entry7156; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x97157; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd7158; CI-NEXT: s_mov_b32 s7, 0xf0007159; CI-NEXT: s_mov_b32 s6, -17160; CI-NEXT: s_waitcnt lgkmcnt(0)7161; CI-NEXT: s_mov_b32 s4, s27162; CI-NEXT: s_mov_b32 s5, s37163; CI-NEXT: v_mov_b32_e32 v0, s87164; CI-NEXT: v_mov_b32_e32 v1, s97165; CI-NEXT: s_mov_b32 s2, s67166; CI-NEXT: s_mov_b32 s3, s77167; CI-NEXT: buffer_atomic_inc_x2 v[0:1], off, s[0:3], 0 offset:32 glc7168; CI-NEXT: s_waitcnt vmcnt(0)7169; CI-NEXT: buffer_wbinvl1_vol7170; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 07171; CI-NEXT: s_endpgm7172;7173; VI-LABEL: atomic_inc_i64_ret_offset:7174; VI: ; %bb.0: ; %entry7175; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247176; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x347177; VI-NEXT: s_mov_b32 s7, 0xf0007178; VI-NEXT: s_mov_b32 s6, -17179; VI-NEXT: s_waitcnt lgkmcnt(0)7180; VI-NEXT: s_mov_b32 s4, s27181; VI-NEXT: s_mov_b32 s5, s37182; VI-NEXT: v_mov_b32_e32 v0, s87183; VI-NEXT: v_mov_b32_e32 v1, s97184; VI-NEXT: s_mov_b32 s2, s67185; VI-NEXT: s_mov_b32 s3, s77186; VI-NEXT: buffer_atomic_inc_x2 v[0:1], off, s[0:3], 0 offset:32 glc7187; VI-NEXT: s_waitcnt vmcnt(0)7188; VI-NEXT: buffer_wbinvl1_vol7189; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 07190; VI-NEXT: s_endpgm7191;7192; GFX9-LABEL: atomic_inc_i64_ret_offset:7193; GFX9: ; %bb.0: ; %entry7194; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x347195; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247196; GFX9-NEXT: v_mov_b32_e32 v2, 07197; GFX9-NEXT: s_waitcnt lgkmcnt(0)7198; GFX9-NEXT: v_mov_b32_e32 v0, s67199; GFX9-NEXT: v_mov_b32_e32 v1, s77200; GFX9-NEXT: global_atomic_inc_x2 v[0:1], v2, v[0:1], s[0:1] offset:32 glc7201; GFX9-NEXT: s_waitcnt vmcnt(0)7202; GFX9-NEXT: buffer_wbinvl1_vol7203; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]7204; GFX9-NEXT: s_endpgm7205;7206; GFX12-LABEL: atomic_inc_i64_ret_offset:7207; GFX12: ; %bb.0: ; %entry7208; GFX12-NEXT: s_clause 0x17209; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x347210; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x247211; GFX12-NEXT: s_wait_kmcnt 0x07212; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s77213; GFX12-NEXT: v_mov_b32_e32 v0, s67214; GFX12-NEXT: global_atomic_inc_u64 v[0:1], v2, v[0:1], s[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV7215; GFX12-NEXT: s_wait_loadcnt 0x07216; GFX12-NEXT: global_inv scope:SCOPE_DEV7217; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]7218; GFX12-NEXT: s_endpgm7219entry:7220 %gep = getelementptr i64, ptr addrspace(1) %out, i64 47221 %tmp0 = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !07222 store i64 %tmp0, ptr addrspace(1) %out27223 ret void7224}7225 7226define amdgpu_kernel void @atomic_inc_i64_incr64_offset(ptr addrspace(1) %out, i64 %in, i64 %index) {7227; CI-LABEL: atomic_inc_i64_incr64_offset:7228; CI: ; %bb.0: ; %entry7229; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x97230; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd7231; CI-NEXT: s_waitcnt lgkmcnt(0)7232; CI-NEXT: v_mov_b32_e32 v0, s27233; CI-NEXT: s_lshl_b64 s[4:5], s[4:5], 37234; CI-NEXT: v_mov_b32_e32 v2, s47235; CI-NEXT: v_mov_b32_e32 v1, s37236; CI-NEXT: s_mov_b32 s3, 0xf0007237; CI-NEXT: s_mov_b32 s2, 07238; CI-NEXT: v_mov_b32_e32 v3, s57239; CI-NEXT: buffer_atomic_inc_x2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:327240; CI-NEXT: s_waitcnt vmcnt(0)7241; CI-NEXT: buffer_wbinvl1_vol7242; CI-NEXT: s_endpgm7243;7244; VI-LABEL: atomic_inc_i64_incr64_offset:7245; VI: ; %bb.0: ; %entry7246; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247247; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x347248; VI-NEXT: s_waitcnt lgkmcnt(0)7249; VI-NEXT: v_mov_b32_e32 v0, s27250; VI-NEXT: v_mov_b32_e32 v1, s37251; VI-NEXT: s_lshl_b64 s[2:3], s[4:5], 37252; VI-NEXT: s_add_u32 s0, s0, s27253; VI-NEXT: s_addc_u32 s1, s1, s37254; VI-NEXT: s_add_u32 s0, s0, 327255; VI-NEXT: s_addc_u32 s1, s1, 07256; VI-NEXT: v_mov_b32_e32 v3, s17257; VI-NEXT: v_mov_b32_e32 v2, s07258; VI-NEXT: flat_atomic_inc_x2 v[2:3], v[0:1]7259; VI-NEXT: s_waitcnt vmcnt(0)7260; VI-NEXT: buffer_wbinvl1_vol7261; VI-NEXT: s_endpgm7262;7263; GFX9-LABEL: atomic_inc_i64_incr64_offset:7264; GFX9: ; %bb.0: ; %entry7265; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247266; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x347267; GFX9-NEXT: v_mov_b32_e32 v2, 07268; GFX9-NEXT: s_waitcnt lgkmcnt(0)7269; GFX9-NEXT: v_mov_b32_e32 v0, s27270; GFX9-NEXT: v_mov_b32_e32 v1, s37271; GFX9-NEXT: s_lshl_b64 s[2:3], s[6:7], 37272; GFX9-NEXT: s_add_u32 s0, s0, s27273; GFX9-NEXT: s_addc_u32 s1, s1, s37274; GFX9-NEXT: global_atomic_inc_x2 v2, v[0:1], s[0:1] offset:327275; GFX9-NEXT: s_waitcnt vmcnt(0)7276; GFX9-NEXT: buffer_wbinvl1_vol7277; GFX9-NEXT: s_endpgm7278;7279; GFX12-LABEL: atomic_inc_i64_incr64_offset:7280; GFX12: ; %bb.0: ; %entry7281; GFX12-NEXT: s_clause 0x17282; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x247283; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x347284; GFX12-NEXT: v_mov_b32_e32 v2, 07285; GFX12-NEXT: s_wait_kmcnt 0x07286; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s37287; GFX12-NEXT: s_lshl_b64 s[2:3], s[4:5], 37288; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)7289; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]7290; GFX12-NEXT: global_atomic_inc_u64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_DEV7291; GFX12-NEXT: s_wait_storecnt 0x07292; GFX12-NEXT: global_inv scope:SCOPE_DEV7293; GFX12-NEXT: s_endpgm7294entry:7295 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index7296 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 47297 %tmp0 = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !07298 ret void7299}7300 7301define amdgpu_kernel void @atomic_dec_i64_offset(ptr addrspace(1) %out, i64 %in) {7302; CI-LABEL: atomic_dec_i64_offset:7303; CI: ; %bb.0: ; %entry7304; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x97305; CI-NEXT: s_waitcnt lgkmcnt(0)7306; CI-NEXT: v_mov_b32_e32 v0, s27307; CI-NEXT: v_mov_b32_e32 v1, s37308; CI-NEXT: s_mov_b32 s3, 0xf0007309; CI-NEXT: s_mov_b32 s2, -17310; CI-NEXT: buffer_atomic_dec_x2 v[0:1], off, s[0:3], 0 offset:327311; CI-NEXT: s_waitcnt vmcnt(0)7312; CI-NEXT: buffer_wbinvl1_vol7313; CI-NEXT: s_endpgm7314;7315; VI-LABEL: atomic_dec_i64_offset:7316; VI: ; %bb.0: ; %entry7317; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247318; VI-NEXT: s_waitcnt lgkmcnt(0)7319; VI-NEXT: v_mov_b32_e32 v0, s27320; VI-NEXT: v_mov_b32_e32 v1, s37321; VI-NEXT: s_mov_b32 s3, 0xf0007322; VI-NEXT: s_mov_b32 s2, -17323; VI-NEXT: buffer_atomic_dec_x2 v[0:1], off, s[0:3], 0 offset:327324; VI-NEXT: s_waitcnt vmcnt(0)7325; VI-NEXT: buffer_wbinvl1_vol7326; VI-NEXT: s_endpgm7327;7328; GFX9-LABEL: atomic_dec_i64_offset:7329; GFX9: ; %bb.0: ; %entry7330; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247331; GFX9-NEXT: v_mov_b32_e32 v2, 07332; GFX9-NEXT: s_waitcnt lgkmcnt(0)7333; GFX9-NEXT: v_mov_b32_e32 v0, s27334; GFX9-NEXT: v_mov_b32_e32 v1, s37335; GFX9-NEXT: global_atomic_dec_x2 v2, v[0:1], s[0:1] offset:327336; GFX9-NEXT: s_waitcnt vmcnt(0)7337; GFX9-NEXT: buffer_wbinvl1_vol7338; GFX9-NEXT: s_endpgm7339;7340; GFX12-LABEL: atomic_dec_i64_offset:7341; GFX12: ; %bb.0: ; %entry7342; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x247343; GFX12-NEXT: s_wait_kmcnt 0x07344; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s37345; GFX12-NEXT: v_mov_b32_e32 v0, s27346; GFX12-NEXT: global_atomic_dec_u64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_DEV7347; GFX12-NEXT: s_wait_storecnt 0x07348; GFX12-NEXT: global_inv scope:SCOPE_DEV7349; GFX12-NEXT: s_endpgm7350entry:7351 %gep = getelementptr i64, ptr addrspace(1) %out, i64 47352 %tmp0 = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !07353 ret void7354}7355 7356define amdgpu_kernel void @atomic_dec_i64_ret_offset(ptr addrspace(1) %out, ptr addrspace(1) %out2, i64 %in) {7357; CI-LABEL: atomic_dec_i64_ret_offset:7358; CI: ; %bb.0: ; %entry7359; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x97360; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd7361; CI-NEXT: s_mov_b32 s7, 0xf0007362; CI-NEXT: s_mov_b32 s6, -17363; CI-NEXT: s_waitcnt lgkmcnt(0)7364; CI-NEXT: s_mov_b32 s4, s27365; CI-NEXT: s_mov_b32 s5, s37366; CI-NEXT: v_mov_b32_e32 v0, s87367; CI-NEXT: v_mov_b32_e32 v1, s97368; CI-NEXT: s_mov_b32 s2, s67369; CI-NEXT: s_mov_b32 s3, s77370; CI-NEXT: buffer_atomic_dec_x2 v[0:1], off, s[0:3], 0 offset:32 glc7371; CI-NEXT: s_waitcnt vmcnt(0)7372; CI-NEXT: buffer_wbinvl1_vol7373; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 07374; CI-NEXT: s_endpgm7375;7376; VI-LABEL: atomic_dec_i64_ret_offset:7377; VI: ; %bb.0: ; %entry7378; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247379; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x347380; VI-NEXT: s_mov_b32 s7, 0xf0007381; VI-NEXT: s_mov_b32 s6, -17382; VI-NEXT: s_waitcnt lgkmcnt(0)7383; VI-NEXT: s_mov_b32 s4, s27384; VI-NEXT: s_mov_b32 s5, s37385; VI-NEXT: v_mov_b32_e32 v0, s87386; VI-NEXT: v_mov_b32_e32 v1, s97387; VI-NEXT: s_mov_b32 s2, s67388; VI-NEXT: s_mov_b32 s3, s77389; VI-NEXT: buffer_atomic_dec_x2 v[0:1], off, s[0:3], 0 offset:32 glc7390; VI-NEXT: s_waitcnt vmcnt(0)7391; VI-NEXT: buffer_wbinvl1_vol7392; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 07393; VI-NEXT: s_endpgm7394;7395; GFX9-LABEL: atomic_dec_i64_ret_offset:7396; GFX9: ; %bb.0: ; %entry7397; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x347398; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247399; GFX9-NEXT: v_mov_b32_e32 v2, 07400; GFX9-NEXT: s_waitcnt lgkmcnt(0)7401; GFX9-NEXT: v_mov_b32_e32 v0, s67402; GFX9-NEXT: v_mov_b32_e32 v1, s77403; GFX9-NEXT: global_atomic_dec_x2 v[0:1], v2, v[0:1], s[0:1] offset:32 glc7404; GFX9-NEXT: s_waitcnt vmcnt(0)7405; GFX9-NEXT: buffer_wbinvl1_vol7406; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]7407; GFX9-NEXT: s_endpgm7408;7409; GFX12-LABEL: atomic_dec_i64_ret_offset:7410; GFX12: ; %bb.0: ; %entry7411; GFX12-NEXT: s_clause 0x17412; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x347413; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x247414; GFX12-NEXT: s_wait_kmcnt 0x07415; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s77416; GFX12-NEXT: v_mov_b32_e32 v0, s67417; GFX12-NEXT: global_atomic_dec_u64 v[0:1], v2, v[0:1], s[0:1] offset:32 th:TH_ATOMIC_RETURN scope:SCOPE_DEV7418; GFX12-NEXT: s_wait_loadcnt 0x07419; GFX12-NEXT: global_inv scope:SCOPE_DEV7420; GFX12-NEXT: global_store_b64 v2, v[0:1], s[2:3]7421; GFX12-NEXT: s_endpgm7422entry:7423 %gep = getelementptr i64, ptr addrspace(1) %out, i64 47424 %tmp0 = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !07425 store i64 %tmp0, ptr addrspace(1) %out27426 ret void7427}7428 7429define amdgpu_kernel void @atomic_dec_i64_decr64_offset(ptr addrspace(1) %out, i64 %in, i64 %index) {7430; CI-LABEL: atomic_dec_i64_decr64_offset:7431; CI: ; %bb.0: ; %entry7432; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x97433; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd7434; CI-NEXT: s_waitcnt lgkmcnt(0)7435; CI-NEXT: v_mov_b32_e32 v0, s27436; CI-NEXT: s_lshl_b64 s[4:5], s[4:5], 37437; CI-NEXT: v_mov_b32_e32 v2, s47438; CI-NEXT: v_mov_b32_e32 v1, s37439; CI-NEXT: s_mov_b32 s3, 0xf0007440; CI-NEXT: s_mov_b32 s2, 07441; CI-NEXT: v_mov_b32_e32 v3, s57442; CI-NEXT: buffer_atomic_dec_x2 v[0:1], v[2:3], s[0:3], 0 addr64 offset:327443; CI-NEXT: s_waitcnt vmcnt(0)7444; CI-NEXT: buffer_wbinvl1_vol7445; CI-NEXT: s_endpgm7446;7447; VI-LABEL: atomic_dec_i64_decr64_offset:7448; VI: ; %bb.0: ; %entry7449; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247450; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x347451; VI-NEXT: s_waitcnt lgkmcnt(0)7452; VI-NEXT: v_mov_b32_e32 v0, s27453; VI-NEXT: v_mov_b32_e32 v1, s37454; VI-NEXT: s_lshl_b64 s[2:3], s[4:5], 37455; VI-NEXT: s_add_u32 s0, s0, s27456; VI-NEXT: s_addc_u32 s1, s1, s37457; VI-NEXT: s_add_u32 s0, s0, 327458; VI-NEXT: s_addc_u32 s1, s1, 07459; VI-NEXT: v_mov_b32_e32 v3, s17460; VI-NEXT: v_mov_b32_e32 v2, s07461; VI-NEXT: flat_atomic_dec_x2 v[2:3], v[0:1]7462; VI-NEXT: s_waitcnt vmcnt(0)7463; VI-NEXT: buffer_wbinvl1_vol7464; VI-NEXT: s_endpgm7465;7466; GFX9-LABEL: atomic_dec_i64_decr64_offset:7467; GFX9: ; %bb.0: ; %entry7468; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247469; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x347470; GFX9-NEXT: v_mov_b32_e32 v2, 07471; GFX9-NEXT: s_waitcnt lgkmcnt(0)7472; GFX9-NEXT: v_mov_b32_e32 v0, s27473; GFX9-NEXT: v_mov_b32_e32 v1, s37474; GFX9-NEXT: s_lshl_b64 s[2:3], s[6:7], 37475; GFX9-NEXT: s_add_u32 s0, s0, s27476; GFX9-NEXT: s_addc_u32 s1, s1, s37477; GFX9-NEXT: global_atomic_dec_x2 v2, v[0:1], s[0:1] offset:327478; GFX9-NEXT: s_waitcnt vmcnt(0)7479; GFX9-NEXT: buffer_wbinvl1_vol7480; GFX9-NEXT: s_endpgm7481;7482; GFX12-LABEL: atomic_dec_i64_decr64_offset:7483; GFX12: ; %bb.0: ; %entry7484; GFX12-NEXT: s_clause 0x17485; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x247486; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x347487; GFX12-NEXT: v_mov_b32_e32 v2, 07488; GFX12-NEXT: s_wait_kmcnt 0x07489; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s37490; GFX12-NEXT: s_lshl_b64 s[2:3], s[4:5], 37491; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)7492; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]7493; GFX12-NEXT: global_atomic_dec_u64 v2, v[0:1], s[0:1] offset:32 scope:SCOPE_DEV7494; GFX12-NEXT: s_wait_storecnt 0x07495; GFX12-NEXT: global_inv scope:SCOPE_DEV7496; GFX12-NEXT: s_endpgm7497entry:7498 %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index7499 %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 47500 %tmp0 = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !07501 ret void7502}7503 7504!0 = !{}7505