18710 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx600 < %s | FileCheck --check-prefixes=GFX6 %s3; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx700 < %s | FileCheck --check-prefixes=GFX7 %s4; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1010 < %s | FileCheck --check-prefixes=GFX10-WGP %s5; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1010 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX10-CU %s6; RUN: llc -mtriple=amdgcn-amd-amdpal -O0 -mcpu=gfx700 -amdgcn-skip-cache-invalidations < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s7; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx90a < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s8; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx90a -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s9; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX942-NOTTGSPLIT %s10; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx942 -mattr=+tgsplit < %s | FileCheck -check-prefixes=GFX942-TGSPLIT %s11; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s12; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s13; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s14; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s15; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s16 17define amdgpu_kernel void @local_workgroup_unordered_load(18; GFX6-LABEL: local_workgroup_unordered_load:19; GFX6: ; %bb.0: ; %entry20; GFX6-NEXT: s_load_dword s5, s[8:9], 0x021; GFX6-NEXT: s_load_dword s4, s[8:9], 0x122; GFX6-NEXT: ; kill: def $sgpr6 killed $sgpr423; GFX6-NEXT: ; kill: def $sgpr6 killed $sgpr524; GFX6-NEXT: s_mov_b32 m0, -125; GFX6-NEXT: s_waitcnt lgkmcnt(0)26; GFX6-NEXT: v_mov_b32_e32 v0, s527; GFX6-NEXT: ds_read_b32 v1, v028; GFX6-NEXT: s_mov_b32 m0, -129; GFX6-NEXT: v_mov_b32_e32 v0, s430; GFX6-NEXT: s_waitcnt lgkmcnt(0)31; GFX6-NEXT: ds_write_b32 v0, v132; GFX6-NEXT: s_endpgm33;34; GFX7-LABEL: local_workgroup_unordered_load:35; GFX7: ; %bb.0: ; %entry36; GFX7-NEXT: s_load_dword s5, s[8:9], 0x037; GFX7-NEXT: s_load_dword s4, s[8:9], 0x138; GFX7-NEXT: s_mov_b32 m0, -139; GFX7-NEXT: s_waitcnt lgkmcnt(0)40; GFX7-NEXT: v_mov_b32_e32 v0, s541; GFX7-NEXT: ds_read_b32 v1, v042; GFX7-NEXT: s_mov_b32 m0, -143; GFX7-NEXT: v_mov_b32_e32 v0, s444; GFX7-NEXT: s_waitcnt lgkmcnt(0)45; GFX7-NEXT: ds_write_b32 v0, v146; GFX7-NEXT: s_endpgm47;48; GFX10-WGP-LABEL: local_workgroup_unordered_load:49; GFX10-WGP: ; %bb.0: ; %entry50; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x051; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x452; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)53; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s554; GFX10-WGP-NEXT: ds_read_b32 v1, v055; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s456; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)57; GFX10-WGP-NEXT: ds_write_b32 v0, v158; GFX10-WGP-NEXT: s_endpgm59;60; GFX10-CU-LABEL: local_workgroup_unordered_load:61; GFX10-CU: ; %bb.0: ; %entry62; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x063; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x464; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)65; GFX10-CU-NEXT: v_mov_b32_e32 v0, s566; GFX10-CU-NEXT: ds_read_b32 v1, v067; GFX10-CU-NEXT: v_mov_b32_e32 v0, s468; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)69; GFX10-CU-NEXT: ds_write_b32 v0, v170; GFX10-CU-NEXT: s_endpgm71;72; SKIP-CACHE-INV-LABEL: local_workgroup_unordered_load:73; SKIP-CACHE-INV: ; %bb.0: ; %entry74; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x075; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x176; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -177; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)78; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s179; SKIP-CACHE-INV-NEXT: ds_read_b32 v1, v080; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -181; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s082; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)83; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v184; SKIP-CACHE-INV-NEXT: s_endpgm85;86; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_unordered_load:87; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry88; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x089; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x490; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)91; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s592; GFX90A-NOTTGSPLIT-NEXT: ds_read_b32 v1, v093; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s494; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)95; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v196; GFX90A-NOTTGSPLIT-NEXT: s_endpgm97;98; GFX90A-TGSPLIT-LABEL: local_workgroup_unordered_load:99; GFX90A-TGSPLIT: ; %bb.0: ; %entry100; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x0101; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x4102; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)103; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s5104; GFX90A-TGSPLIT-NEXT: ds_read_b32 v1, v0105; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s4106; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)107; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v1108; GFX90A-TGSPLIT-NEXT: s_endpgm109;110; GFX942-NOTTGSPLIT-LABEL: local_workgroup_unordered_load:111; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry112; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x0113; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x4114; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)115; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s1116; GFX942-NOTTGSPLIT-NEXT: ds_read_b32 v1, v0117; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0118; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)119; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v1120; GFX942-NOTTGSPLIT-NEXT: s_endpgm121;122; GFX942-TGSPLIT-LABEL: local_workgroup_unordered_load:123; GFX942-TGSPLIT: ; %bb.0: ; %entry124; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x0125; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x4126; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)127; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s1128; GFX942-TGSPLIT-NEXT: ds_read_b32 v1, v0129; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0130; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)131; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v1132; GFX942-TGSPLIT-NEXT: s_endpgm133;134; GFX11-WGP-LABEL: local_workgroup_unordered_load:135; GFX11-WGP: ; %bb.0: ; %entry136; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x0137; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x4138; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)139; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s1140; GFX11-WGP-NEXT: ds_load_b32 v1, v0141; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0142; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)143; GFX11-WGP-NEXT: ds_store_b32 v0, v1144; GFX11-WGP-NEXT: s_endpgm145;146; GFX11-CU-LABEL: local_workgroup_unordered_load:147; GFX11-CU: ; %bb.0: ; %entry148; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x0149; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x4150; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)151; GFX11-CU-NEXT: v_mov_b32_e32 v0, s1152; GFX11-CU-NEXT: ds_load_b32 v1, v0153; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0154; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)155; GFX11-CU-NEXT: ds_store_b32 v0, v1156; GFX11-CU-NEXT: s_endpgm157;158; GFX12-WGP-LABEL: local_workgroup_unordered_load:159; GFX12-WGP: ; %bb.0: ; %entry160; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x0161; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x4162; GFX12-WGP-NEXT: s_wait_kmcnt 0x0163; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s1164; GFX12-WGP-NEXT: ds_load_b32 v1, v0165; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0166; GFX12-WGP-NEXT: s_wait_dscnt 0x0167; GFX12-WGP-NEXT: ds_store_b32 v0, v1168; GFX12-WGP-NEXT: s_endpgm169;170; GFX12-CU-LABEL: local_workgroup_unordered_load:171; GFX12-CU: ; %bb.0: ; %entry172; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x0173; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x4174; GFX12-CU-NEXT: s_wait_kmcnt 0x0175; GFX12-CU-NEXT: v_mov_b32_e32 v0, s1176; GFX12-CU-NEXT: ds_load_b32 v1, v0177; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0178; GFX12-CU-NEXT: s_wait_dscnt 0x0179; GFX12-CU-NEXT: ds_store_b32 v0, v1180; GFX12-CU-NEXT: s_endpgm181;182; GFX1250-LABEL: local_workgroup_unordered_load:183; GFX1250: ; %bb.0: ; %entry184; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1185; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x0186; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x4187; GFX1250-NEXT: s_wait_kmcnt 0x0188; GFX1250-NEXT: v_mov_b32_e32 v0, s1189; GFX1250-NEXT: ds_load_b32 v1, v0190; GFX1250-NEXT: v_mov_b32_e32 v0, s0191; GFX1250-NEXT: s_wait_dscnt 0x0192; GFX1250-NEXT: ds_store_b32 v0, v1193; GFX1250-NEXT: s_endpgm194 ptr addrspace(3) %in, ptr addrspace(3) %out) {195entry:196 %val = load atomic i32, ptr addrspace(3) %in syncscope("workgroup") unordered, align 4197 store i32 %val, ptr addrspace(3) %out198 ret void199}200 201define amdgpu_kernel void @local_workgroup_monotonic_load(202; GFX6-LABEL: local_workgroup_monotonic_load:203; GFX6: ; %bb.0: ; %entry204; GFX6-NEXT: s_load_dword s5, s[8:9], 0x0205; GFX6-NEXT: s_load_dword s4, s[8:9], 0x1206; GFX6-NEXT: ; kill: def $sgpr6 killed $sgpr4207; GFX6-NEXT: ; kill: def $sgpr6 killed $sgpr5208; GFX6-NEXT: s_mov_b32 m0, -1209; GFX6-NEXT: s_waitcnt lgkmcnt(0)210; GFX6-NEXT: v_mov_b32_e32 v0, s5211; GFX6-NEXT: ds_read_b32 v1, v0212; GFX6-NEXT: s_mov_b32 m0, -1213; GFX6-NEXT: v_mov_b32_e32 v0, s4214; GFX6-NEXT: s_waitcnt lgkmcnt(0)215; GFX6-NEXT: ds_write_b32 v0, v1216; GFX6-NEXT: s_endpgm217;218; GFX7-LABEL: local_workgroup_monotonic_load:219; GFX7: ; %bb.0: ; %entry220; GFX7-NEXT: s_load_dword s5, s[8:9], 0x0221; GFX7-NEXT: s_load_dword s4, s[8:9], 0x1222; GFX7-NEXT: s_mov_b32 m0, -1223; GFX7-NEXT: s_waitcnt lgkmcnt(0)224; GFX7-NEXT: v_mov_b32_e32 v0, s5225; GFX7-NEXT: ds_read_b32 v1, v0226; GFX7-NEXT: s_mov_b32 m0, -1227; GFX7-NEXT: v_mov_b32_e32 v0, s4228; GFX7-NEXT: s_waitcnt lgkmcnt(0)229; GFX7-NEXT: ds_write_b32 v0, v1230; GFX7-NEXT: s_endpgm231;232; GFX10-WGP-LABEL: local_workgroup_monotonic_load:233; GFX10-WGP: ; %bb.0: ; %entry234; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x0235; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x4236; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)237; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s5238; GFX10-WGP-NEXT: ds_read_b32 v1, v0239; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4240; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)241; GFX10-WGP-NEXT: ds_write_b32 v0, v1242; GFX10-WGP-NEXT: s_endpgm243;244; GFX10-CU-LABEL: local_workgroup_monotonic_load:245; GFX10-CU: ; %bb.0: ; %entry246; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x0247; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x4248; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)249; GFX10-CU-NEXT: v_mov_b32_e32 v0, s5250; GFX10-CU-NEXT: ds_read_b32 v1, v0251; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4252; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)253; GFX10-CU-NEXT: ds_write_b32 v0, v1254; GFX10-CU-NEXT: s_endpgm255;256; SKIP-CACHE-INV-LABEL: local_workgroup_monotonic_load:257; SKIP-CACHE-INV: ; %bb.0: ; %entry258; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x0259; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x1260; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -1261; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)262; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s1263; SKIP-CACHE-INV-NEXT: ds_read_b32 v1, v0264; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -1265; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0266; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)267; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v1268; SKIP-CACHE-INV-NEXT: s_endpgm269;270; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_monotonic_load:271; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry272; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x0273; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x4274; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)275; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s5276; GFX90A-NOTTGSPLIT-NEXT: ds_read_b32 v1, v0277; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s4278; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)279; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v1280; GFX90A-NOTTGSPLIT-NEXT: s_endpgm281;282; GFX90A-TGSPLIT-LABEL: local_workgroup_monotonic_load:283; GFX90A-TGSPLIT: ; %bb.0: ; %entry284; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x0285; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x4286; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)287; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s5288; GFX90A-TGSPLIT-NEXT: ds_read_b32 v1, v0289; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s4290; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)291; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v1292; GFX90A-TGSPLIT-NEXT: s_endpgm293;294; GFX942-NOTTGSPLIT-LABEL: local_workgroup_monotonic_load:295; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry296; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x0297; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x4298; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)299; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s1300; GFX942-NOTTGSPLIT-NEXT: ds_read_b32 v1, v0301; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0302; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)303; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v1304; GFX942-NOTTGSPLIT-NEXT: s_endpgm305;306; GFX942-TGSPLIT-LABEL: local_workgroup_monotonic_load:307; GFX942-TGSPLIT: ; %bb.0: ; %entry308; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x0309; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x4310; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)311; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s1312; GFX942-TGSPLIT-NEXT: ds_read_b32 v1, v0313; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0314; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)315; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v1316; GFX942-TGSPLIT-NEXT: s_endpgm317;318; GFX11-WGP-LABEL: local_workgroup_monotonic_load:319; GFX11-WGP: ; %bb.0: ; %entry320; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x0321; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x4322; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)323; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s1324; GFX11-WGP-NEXT: ds_load_b32 v1, v0325; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0326; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)327; GFX11-WGP-NEXT: ds_store_b32 v0, v1328; GFX11-WGP-NEXT: s_endpgm329;330; GFX11-CU-LABEL: local_workgroup_monotonic_load:331; GFX11-CU: ; %bb.0: ; %entry332; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x0333; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x4334; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)335; GFX11-CU-NEXT: v_mov_b32_e32 v0, s1336; GFX11-CU-NEXT: ds_load_b32 v1, v0337; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0338; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)339; GFX11-CU-NEXT: ds_store_b32 v0, v1340; GFX11-CU-NEXT: s_endpgm341;342; GFX12-WGP-LABEL: local_workgroup_monotonic_load:343; GFX12-WGP: ; %bb.0: ; %entry344; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x0345; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x4346; GFX12-WGP-NEXT: s_wait_kmcnt 0x0347; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s1348; GFX12-WGP-NEXT: ds_load_b32 v1, v0349; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0350; GFX12-WGP-NEXT: s_wait_dscnt 0x0351; GFX12-WGP-NEXT: ds_store_b32 v0, v1352; GFX12-WGP-NEXT: s_endpgm353;354; GFX12-CU-LABEL: local_workgroup_monotonic_load:355; GFX12-CU: ; %bb.0: ; %entry356; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x0357; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x4358; GFX12-CU-NEXT: s_wait_kmcnt 0x0359; GFX12-CU-NEXT: v_mov_b32_e32 v0, s1360; GFX12-CU-NEXT: ds_load_b32 v1, v0361; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0362; GFX12-CU-NEXT: s_wait_dscnt 0x0363; GFX12-CU-NEXT: ds_store_b32 v0, v1364; GFX12-CU-NEXT: s_endpgm365;366; GFX1250-LABEL: local_workgroup_monotonic_load:367; GFX1250: ; %bb.0: ; %entry368; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1369; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x0370; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x4371; GFX1250-NEXT: s_wait_kmcnt 0x0372; GFX1250-NEXT: v_mov_b32_e32 v0, s1373; GFX1250-NEXT: ds_load_b32 v1, v0374; GFX1250-NEXT: v_mov_b32_e32 v0, s0375; GFX1250-NEXT: s_wait_dscnt 0x0376; GFX1250-NEXT: ds_store_b32 v0, v1377; GFX1250-NEXT: s_endpgm378 ptr addrspace(3) %in, ptr addrspace(3) %out) {379entry:380 %val = load atomic i32, ptr addrspace(3) %in syncscope("workgroup") monotonic, align 4381 store i32 %val, ptr addrspace(3) %out382 ret void383}384 385define amdgpu_kernel void @local_workgroup_acquire_load(386; GFX6-LABEL: local_workgroup_acquire_load:387; GFX6: ; %bb.0: ; %entry388; GFX6-NEXT: s_load_dword s5, s[8:9], 0x0389; GFX6-NEXT: s_load_dword s4, s[8:9], 0x1390; GFX6-NEXT: ; kill: def $sgpr6 killed $sgpr4391; GFX6-NEXT: ; kill: def $sgpr6 killed $sgpr5392; GFX6-NEXT: s_mov_b32 m0, -1393; GFX6-NEXT: s_waitcnt lgkmcnt(0)394; GFX6-NEXT: v_mov_b32_e32 v0, s5395; GFX6-NEXT: ds_read_b32 v1, v0396; GFX6-NEXT: s_waitcnt lgkmcnt(0)397; GFX6-NEXT: s_mov_b32 m0, -1398; GFX6-NEXT: v_mov_b32_e32 v0, s4399; GFX6-NEXT: ds_write_b32 v0, v1400; GFX6-NEXT: s_endpgm401;402; GFX7-LABEL: local_workgroup_acquire_load:403; GFX7: ; %bb.0: ; %entry404; GFX7-NEXT: s_load_dword s5, s[8:9], 0x0405; GFX7-NEXT: s_load_dword s4, s[8:9], 0x1406; GFX7-NEXT: s_mov_b32 m0, -1407; GFX7-NEXT: s_waitcnt lgkmcnt(0)408; GFX7-NEXT: v_mov_b32_e32 v0, s5409; GFX7-NEXT: ds_read_b32 v1, v0410; GFX7-NEXT: s_waitcnt lgkmcnt(0)411; GFX7-NEXT: s_mov_b32 m0, -1412; GFX7-NEXT: v_mov_b32_e32 v0, s4413; GFX7-NEXT: ds_write_b32 v0, v1414; GFX7-NEXT: s_endpgm415;416; GFX10-WGP-LABEL: local_workgroup_acquire_load:417; GFX10-WGP: ; %bb.0: ; %entry418; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x0419; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x4420; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)421; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s5422; GFX10-WGP-NEXT: ds_read_b32 v1, v0423; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)424; GFX10-WGP-NEXT: buffer_gl0_inv425; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4426; GFX10-WGP-NEXT: ds_write_b32 v0, v1427; GFX10-WGP-NEXT: s_endpgm428;429; GFX10-CU-LABEL: local_workgroup_acquire_load:430; GFX10-CU: ; %bb.0: ; %entry431; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x0432; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x4433; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)434; GFX10-CU-NEXT: v_mov_b32_e32 v0, s5435; GFX10-CU-NEXT: ds_read_b32 v1, v0436; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)437; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4438; GFX10-CU-NEXT: ds_write_b32 v0, v1439; GFX10-CU-NEXT: s_endpgm440;441; SKIP-CACHE-INV-LABEL: local_workgroup_acquire_load:442; SKIP-CACHE-INV: ; %bb.0: ; %entry443; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x0444; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x1445; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -1446; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)447; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s1448; SKIP-CACHE-INV-NEXT: ds_read_b32 v1, v0449; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)450; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -1451; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0452; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v1453; SKIP-CACHE-INV-NEXT: s_endpgm454;455; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_acquire_load:456; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry457; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x0458; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x4459; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)460; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s5461; GFX90A-NOTTGSPLIT-NEXT: ds_read_b32 v1, v0462; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)463; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s4464; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v1465; GFX90A-NOTTGSPLIT-NEXT: s_endpgm466;467; GFX90A-TGSPLIT-LABEL: local_workgroup_acquire_load:468; GFX90A-TGSPLIT: ; %bb.0: ; %entry469; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x0470; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x4471; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)472; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s5473; GFX90A-TGSPLIT-NEXT: ds_read_b32 v1, v0474; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol475; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s4476; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)477; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v1478; GFX90A-TGSPLIT-NEXT: s_endpgm479;480; GFX942-NOTTGSPLIT-LABEL: local_workgroup_acquire_load:481; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry482; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x0483; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x4484; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)485; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s1486; GFX942-NOTTGSPLIT-NEXT: ds_read_b32 v1, v0487; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)488; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0489; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v1490; GFX942-NOTTGSPLIT-NEXT: s_endpgm491;492; GFX942-TGSPLIT-LABEL: local_workgroup_acquire_load:493; GFX942-TGSPLIT: ; %bb.0: ; %entry494; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x0495; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x4496; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)497; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s1498; GFX942-TGSPLIT-NEXT: ds_read_b32 v1, v0499; GFX942-TGSPLIT-NEXT: buffer_inv sc0500; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0501; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)502; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v1503; GFX942-TGSPLIT-NEXT: s_endpgm504;505; GFX11-WGP-LABEL: local_workgroup_acquire_load:506; GFX11-WGP: ; %bb.0: ; %entry507; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x0508; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x4509; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)510; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s1511; GFX11-WGP-NEXT: ds_load_b32 v1, v0512; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)513; GFX11-WGP-NEXT: buffer_gl0_inv514; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0515; GFX11-WGP-NEXT: ds_store_b32 v0, v1516; GFX11-WGP-NEXT: s_endpgm517;518; GFX11-CU-LABEL: local_workgroup_acquire_load:519; GFX11-CU: ; %bb.0: ; %entry520; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x0521; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x4522; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)523; GFX11-CU-NEXT: v_mov_b32_e32 v0, s1524; GFX11-CU-NEXT: ds_load_b32 v1, v0525; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)526; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0527; GFX11-CU-NEXT: ds_store_b32 v0, v1528; GFX11-CU-NEXT: s_endpgm529;530; GFX12-WGP-LABEL: local_workgroup_acquire_load:531; GFX12-WGP: ; %bb.0: ; %entry532; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x0533; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x4534; GFX12-WGP-NEXT: s_wait_kmcnt 0x0535; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s1536; GFX12-WGP-NEXT: ds_load_b32 v1, v0537; GFX12-WGP-NEXT: s_wait_dscnt 0x0538; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE539; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0540; GFX12-WGP-NEXT: ds_store_b32 v0, v1541; GFX12-WGP-NEXT: s_endpgm542;543; GFX12-CU-LABEL: local_workgroup_acquire_load:544; GFX12-CU: ; %bb.0: ; %entry545; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x0546; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x4547; GFX12-CU-NEXT: s_wait_kmcnt 0x0548; GFX12-CU-NEXT: v_mov_b32_e32 v0, s1549; GFX12-CU-NEXT: ds_load_b32 v1, v0550; GFX12-CU-NEXT: s_wait_dscnt 0x0551; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0552; GFX12-CU-NEXT: ds_store_b32 v0, v1553; GFX12-CU-NEXT: s_endpgm554;555; GFX1250-LABEL: local_workgroup_acquire_load:556; GFX1250: ; %bb.0: ; %entry557; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1558; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x0559; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x4560; GFX1250-NEXT: s_wait_kmcnt 0x0561; GFX1250-NEXT: v_mov_b32_e32 v0, s1562; GFX1250-NEXT: ds_load_b32 v1, v0563; GFX1250-NEXT: s_wait_dscnt 0x0564; GFX1250-NEXT: v_mov_b32_e32 v0, s0565; GFX1250-NEXT: ds_store_b32 v0, v1566; GFX1250-NEXT: s_endpgm567 ptr addrspace(3) %in, ptr addrspace(3) %out) {568entry:569 %val = load atomic i32, ptr addrspace(3) %in syncscope("workgroup") acquire, align 4570 store i32 %val, ptr addrspace(3) %out571 ret void572}573 574define amdgpu_kernel void @local_workgroup_seq_cst_load(575; GFX6-LABEL: local_workgroup_seq_cst_load:576; GFX6: ; %bb.0: ; %entry577; GFX6-NEXT: s_load_dword s5, s[8:9], 0x0578; GFX6-NEXT: s_load_dword s4, s[8:9], 0x1579; GFX6-NEXT: ; kill: def $sgpr6 killed $sgpr4580; GFX6-NEXT: ; kill: def $sgpr6 killed $sgpr5581; GFX6-NEXT: s_mov_b32 m0, -1582; GFX6-NEXT: s_waitcnt lgkmcnt(0)583; GFX6-NEXT: v_mov_b32_e32 v0, s5584; GFX6-NEXT: s_waitcnt lgkmcnt(0)585; GFX6-NEXT: ds_read_b32 v1, v0586; GFX6-NEXT: s_waitcnt lgkmcnt(0)587; GFX6-NEXT: s_mov_b32 m0, -1588; GFX6-NEXT: v_mov_b32_e32 v0, s4589; GFX6-NEXT: ds_write_b32 v0, v1590; GFX6-NEXT: s_endpgm591;592; GFX7-LABEL: local_workgroup_seq_cst_load:593; GFX7: ; %bb.0: ; %entry594; GFX7-NEXT: s_load_dword s5, s[8:9], 0x0595; GFX7-NEXT: s_load_dword s4, s[8:9], 0x1596; GFX7-NEXT: s_mov_b32 m0, -1597; GFX7-NEXT: s_waitcnt lgkmcnt(0)598; GFX7-NEXT: v_mov_b32_e32 v0, s5599; GFX7-NEXT: s_waitcnt lgkmcnt(0)600; GFX7-NEXT: ds_read_b32 v1, v0601; GFX7-NEXT: s_waitcnt lgkmcnt(0)602; GFX7-NEXT: s_mov_b32 m0, -1603; GFX7-NEXT: v_mov_b32_e32 v0, s4604; GFX7-NEXT: ds_write_b32 v0, v1605; GFX7-NEXT: s_endpgm606;607; GFX10-WGP-LABEL: local_workgroup_seq_cst_load:608; GFX10-WGP: ; %bb.0: ; %entry609; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x0610; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x4611; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)612; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s5613; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)614; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0615; GFX10-WGP-NEXT: ds_read_b32 v1, v0616; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)617; GFX10-WGP-NEXT: buffer_gl0_inv618; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4619; GFX10-WGP-NEXT: ds_write_b32 v0, v1620; GFX10-WGP-NEXT: s_endpgm621;622; GFX10-CU-LABEL: local_workgroup_seq_cst_load:623; GFX10-CU: ; %bb.0: ; %entry624; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x0625; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x4626; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)627; GFX10-CU-NEXT: v_mov_b32_e32 v0, s5628; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)629; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0630; GFX10-CU-NEXT: ds_read_b32 v1, v0631; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)632; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4633; GFX10-CU-NEXT: ds_write_b32 v0, v1634; GFX10-CU-NEXT: s_endpgm635;636; SKIP-CACHE-INV-LABEL: local_workgroup_seq_cst_load:637; SKIP-CACHE-INV: ; %bb.0: ; %entry638; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x0639; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x1640; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -1641; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)642; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s1643; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)644; SKIP-CACHE-INV-NEXT: ds_read_b32 v1, v0645; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)646; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -1647; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0648; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v1649; SKIP-CACHE-INV-NEXT: s_endpgm650;651; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_seq_cst_load:652; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry653; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x0654; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x4655; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)656; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s5657; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)658; GFX90A-NOTTGSPLIT-NEXT: ds_read_b32 v1, v0659; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)660; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s4661; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v1662; GFX90A-NOTTGSPLIT-NEXT: s_endpgm663;664; GFX90A-TGSPLIT-LABEL: local_workgroup_seq_cst_load:665; GFX90A-TGSPLIT: ; %bb.0: ; %entry666; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x0667; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x4668; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)669; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s5670; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)671; GFX90A-TGSPLIT-NEXT: ds_read_b32 v1, v0672; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol673; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s4674; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)675; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v1676; GFX90A-TGSPLIT-NEXT: s_endpgm677;678; GFX942-NOTTGSPLIT-LABEL: local_workgroup_seq_cst_load:679; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry680; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x0681; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x4682; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)683; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s1684; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)685; GFX942-NOTTGSPLIT-NEXT: ds_read_b32 v1, v0686; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)687; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0688; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v1689; GFX942-NOTTGSPLIT-NEXT: s_endpgm690;691; GFX942-TGSPLIT-LABEL: local_workgroup_seq_cst_load:692; GFX942-TGSPLIT: ; %bb.0: ; %entry693; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x0694; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x4695; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)696; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s1697; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)698; GFX942-TGSPLIT-NEXT: ds_read_b32 v1, v0699; GFX942-TGSPLIT-NEXT: buffer_inv sc0700; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0701; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)702; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v1703; GFX942-TGSPLIT-NEXT: s_endpgm704;705; GFX11-WGP-LABEL: local_workgroup_seq_cst_load:706; GFX11-WGP: ; %bb.0: ; %entry707; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x0708; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x4709; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)710; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s1711; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)712; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0713; GFX11-WGP-NEXT: ds_load_b32 v1, v0714; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)715; GFX11-WGP-NEXT: buffer_gl0_inv716; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0717; GFX11-WGP-NEXT: ds_store_b32 v0, v1718; GFX11-WGP-NEXT: s_endpgm719;720; GFX11-CU-LABEL: local_workgroup_seq_cst_load:721; GFX11-CU: ; %bb.0: ; %entry722; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x0723; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x4724; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)725; GFX11-CU-NEXT: v_mov_b32_e32 v0, s1726; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)727; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0728; GFX11-CU-NEXT: ds_load_b32 v1, v0729; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)730; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0731; GFX11-CU-NEXT: ds_store_b32 v0, v1732; GFX11-CU-NEXT: s_endpgm733;734; GFX12-WGP-LABEL: local_workgroup_seq_cst_load:735; GFX12-WGP: ; %bb.0: ; %entry736; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x0737; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x4738; GFX12-WGP-NEXT: s_wait_kmcnt 0x0739; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s1740; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0741; GFX12-WGP-NEXT: s_wait_samplecnt 0x0742; GFX12-WGP-NEXT: s_wait_storecnt 0x0743; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0744; GFX12-WGP-NEXT: ds_load_b32 v1, v0745; GFX12-WGP-NEXT: s_wait_dscnt 0x0746; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE747; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0748; GFX12-WGP-NEXT: ds_store_b32 v0, v1749; GFX12-WGP-NEXT: s_endpgm750;751; GFX12-CU-LABEL: local_workgroup_seq_cst_load:752; GFX12-CU: ; %bb.0: ; %entry753; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x0754; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x4755; GFX12-CU-NEXT: s_wait_kmcnt 0x0756; GFX12-CU-NEXT: v_mov_b32_e32 v0, s1757; GFX12-CU-NEXT: s_wait_bvhcnt 0x0758; GFX12-CU-NEXT: s_wait_samplecnt 0x0759; GFX12-CU-NEXT: s_wait_storecnt 0x0760; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0761; GFX12-CU-NEXT: ds_load_b32 v1, v0762; GFX12-CU-NEXT: s_wait_dscnt 0x0763; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0764; GFX12-CU-NEXT: ds_store_b32 v0, v1765; GFX12-CU-NEXT: s_endpgm766;767; GFX1250-LABEL: local_workgroup_seq_cst_load:768; GFX1250: ; %bb.0: ; %entry769; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1770; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x0771; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x4772; GFX1250-NEXT: s_wait_kmcnt 0x0773; GFX1250-NEXT: v_mov_b32_e32 v0, s1774; GFX1250-NEXT: s_wait_storecnt 0x0775; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0776; GFX1250-NEXT: ds_load_b32 v1, v0777; GFX1250-NEXT: s_wait_dscnt 0x0778; GFX1250-NEXT: v_mov_b32_e32 v0, s0779; GFX1250-NEXT: ds_store_b32 v0, v1780; GFX1250-NEXT: s_endpgm781 ptr addrspace(3) %in, ptr addrspace(3) %out) {782entry:783 %val = load atomic i32, ptr addrspace(3) %in syncscope("workgroup") seq_cst, align 4784 store i32 %val, ptr addrspace(3) %out785 ret void786}787 788define amdgpu_kernel void @local_workgroup_unordered_store(789; GFX6-LABEL: local_workgroup_unordered_store:790; GFX6: ; %bb.0: ; %entry791; GFX6-NEXT: s_load_dword s5, s[8:9], 0x1792; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr5793; GFX6-NEXT: s_load_dword s4, s[8:9], 0x0794; GFX6-NEXT: s_mov_b32 m0, -1795; GFX6-NEXT: s_waitcnt lgkmcnt(0)796; GFX6-NEXT: v_mov_b32_e32 v0, s5797; GFX6-NEXT: v_mov_b32_e32 v1, s4798; GFX6-NEXT: ds_write_b32 v0, v1799; GFX6-NEXT: s_endpgm800;801; GFX7-LABEL: local_workgroup_unordered_store:802; GFX7: ; %bb.0: ; %entry803; GFX7-NEXT: s_load_dword s4, s[8:9], 0x0804; GFX7-NEXT: s_load_dword s5, s[8:9], 0x1805; GFX7-NEXT: s_mov_b32 m0, -1806; GFX7-NEXT: s_waitcnt lgkmcnt(0)807; GFX7-NEXT: v_mov_b32_e32 v0, s5808; GFX7-NEXT: v_mov_b32_e32 v1, s4809; GFX7-NEXT: ds_write_b32 v0, v1810; GFX7-NEXT: s_endpgm811;812; GFX10-WGP-LABEL: local_workgroup_unordered_store:813; GFX10-WGP: ; %bb.0: ; %entry814; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x0815; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x4816; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)817; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s5818; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s4819; GFX10-WGP-NEXT: ds_write_b32 v0, v1820; GFX10-WGP-NEXT: s_endpgm821;822; GFX10-CU-LABEL: local_workgroup_unordered_store:823; GFX10-CU: ; %bb.0: ; %entry824; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x0825; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x4826; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)827; GFX10-CU-NEXT: v_mov_b32_e32 v0, s5828; GFX10-CU-NEXT: v_mov_b32_e32 v1, s4829; GFX10-CU-NEXT: ds_write_b32 v0, v1830; GFX10-CU-NEXT: s_endpgm831;832; SKIP-CACHE-INV-LABEL: local_workgroup_unordered_store:833; SKIP-CACHE-INV: ; %bb.0: ; %entry834; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x0835; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x1836; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -1837; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)838; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s1839; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s0840; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v1841; SKIP-CACHE-INV-NEXT: s_endpgm842;843; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_unordered_store:844; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry845; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x0846; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x4847; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)848; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s5849; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s4850; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v1851; GFX90A-NOTTGSPLIT-NEXT: s_endpgm852;853; GFX90A-TGSPLIT-LABEL: local_workgroup_unordered_store:854; GFX90A-TGSPLIT: ; %bb.0: ; %entry855; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x0856; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x4857; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)858; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s5859; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s4860; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v1861; GFX90A-TGSPLIT-NEXT: s_endpgm862;863; GFX942-NOTTGSPLIT-LABEL: local_workgroup_unordered_store:864; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry865; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x0866; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x4867; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)868; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s1869; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s0870; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v1871; GFX942-NOTTGSPLIT-NEXT: s_endpgm872;873; GFX942-TGSPLIT-LABEL: local_workgroup_unordered_store:874; GFX942-TGSPLIT: ; %bb.0: ; %entry875; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x0876; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x4877; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)878; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s1879; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s0880; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v1881; GFX942-TGSPLIT-NEXT: s_endpgm882;883; GFX11-WGP-LABEL: local_workgroup_unordered_store:884; GFX11-WGP: ; %bb.0: ; %entry885; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0886; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x4887; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)888; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s1889; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s0890; GFX11-WGP-NEXT: ds_store_b32 v0, v1891; GFX11-WGP-NEXT: s_endpgm892;893; GFX11-CU-LABEL: local_workgroup_unordered_store:894; GFX11-CU: ; %bb.0: ; %entry895; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x0896; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x4897; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)898; GFX11-CU-NEXT: v_mov_b32_e32 v0, s1899; GFX11-CU-NEXT: v_mov_b32_e32 v1, s0900; GFX11-CU-NEXT: ds_store_b32 v0, v1901; GFX11-CU-NEXT: s_endpgm902;903; GFX12-WGP-LABEL: local_workgroup_unordered_store:904; GFX12-WGP: ; %bb.0: ; %entry905; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0906; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x4907; GFX12-WGP-NEXT: s_wait_kmcnt 0x0908; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s1909; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s0910; GFX12-WGP-NEXT: ds_store_b32 v0, v1911; GFX12-WGP-NEXT: s_endpgm912;913; GFX12-CU-LABEL: local_workgroup_unordered_store:914; GFX12-CU: ; %bb.0: ; %entry915; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x0916; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x4917; GFX12-CU-NEXT: s_wait_kmcnt 0x0918; GFX12-CU-NEXT: v_mov_b32_e32 v0, s1919; GFX12-CU-NEXT: v_mov_b32_e32 v1, s0920; GFX12-CU-NEXT: ds_store_b32 v0, v1921; GFX12-CU-NEXT: s_endpgm922;923; GFX1250-LABEL: local_workgroup_unordered_store:924; GFX1250: ; %bb.0: ; %entry925; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1926; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0927; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x4928; GFX1250-NEXT: s_wait_kmcnt 0x0929; GFX1250-NEXT: v_mov_b32_e32 v0, s1930; GFX1250-NEXT: v_mov_b32_e32 v1, s0931; GFX1250-NEXT: ds_store_b32 v0, v1932; GFX1250-NEXT: s_endpgm933 i32 %in, ptr addrspace(3) %out) {934entry:935 store atomic i32 %in, ptr addrspace(3) %out syncscope("workgroup") unordered, align 4936 ret void937}938 939define amdgpu_kernel void @local_workgroup_monotonic_store(940; GFX6-LABEL: local_workgroup_monotonic_store:941; GFX6: ; %bb.0: ; %entry942; GFX6-NEXT: s_load_dword s5, s[8:9], 0x1943; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr5944; GFX6-NEXT: s_load_dword s4, s[8:9], 0x0945; GFX6-NEXT: s_mov_b32 m0, -1946; GFX6-NEXT: s_waitcnt lgkmcnt(0)947; GFX6-NEXT: v_mov_b32_e32 v0, s5948; GFX6-NEXT: v_mov_b32_e32 v1, s4949; GFX6-NEXT: ds_write_b32 v0, v1950; GFX6-NEXT: s_endpgm951;952; GFX7-LABEL: local_workgroup_monotonic_store:953; GFX7: ; %bb.0: ; %entry954; GFX7-NEXT: s_load_dword s4, s[8:9], 0x0955; GFX7-NEXT: s_load_dword s5, s[8:9], 0x1956; GFX7-NEXT: s_mov_b32 m0, -1957; GFX7-NEXT: s_waitcnt lgkmcnt(0)958; GFX7-NEXT: v_mov_b32_e32 v0, s5959; GFX7-NEXT: v_mov_b32_e32 v1, s4960; GFX7-NEXT: ds_write_b32 v0, v1961; GFX7-NEXT: s_endpgm962;963; GFX10-WGP-LABEL: local_workgroup_monotonic_store:964; GFX10-WGP: ; %bb.0: ; %entry965; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x0966; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x4967; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)968; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s5969; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s4970; GFX10-WGP-NEXT: ds_write_b32 v0, v1971; GFX10-WGP-NEXT: s_endpgm972;973; GFX10-CU-LABEL: local_workgroup_monotonic_store:974; GFX10-CU: ; %bb.0: ; %entry975; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x0976; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x4977; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)978; GFX10-CU-NEXT: v_mov_b32_e32 v0, s5979; GFX10-CU-NEXT: v_mov_b32_e32 v1, s4980; GFX10-CU-NEXT: ds_write_b32 v0, v1981; GFX10-CU-NEXT: s_endpgm982;983; SKIP-CACHE-INV-LABEL: local_workgroup_monotonic_store:984; SKIP-CACHE-INV: ; %bb.0: ; %entry985; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x0986; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x1987; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -1988; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)989; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s1990; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s0991; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v1992; SKIP-CACHE-INV-NEXT: s_endpgm993;994; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_monotonic_store:995; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry996; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x0997; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x4998; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)999; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s51000; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s41001; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v11002; GFX90A-NOTTGSPLIT-NEXT: s_endpgm1003;1004; GFX90A-TGSPLIT-LABEL: local_workgroup_monotonic_store:1005; GFX90A-TGSPLIT: ; %bb.0: ; %entry1006; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x01007; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x41008; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1009; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s51010; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s41011; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v11012; GFX90A-TGSPLIT-NEXT: s_endpgm1013;1014; GFX942-NOTTGSPLIT-LABEL: local_workgroup_monotonic_store:1015; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry1016; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x01017; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x41018; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1019; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s11020; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s01021; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v11022; GFX942-NOTTGSPLIT-NEXT: s_endpgm1023;1024; GFX942-TGSPLIT-LABEL: local_workgroup_monotonic_store:1025; GFX942-TGSPLIT: ; %bb.0: ; %entry1026; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x01027; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x41028; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1029; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s11030; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s01031; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v11032; GFX942-TGSPLIT-NEXT: s_endpgm1033;1034; GFX11-WGP-LABEL: local_workgroup_monotonic_store:1035; GFX11-WGP: ; %bb.0: ; %entry1036; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x01037; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x41038; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)1039; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s11040; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s01041; GFX11-WGP-NEXT: ds_store_b32 v0, v11042; GFX11-WGP-NEXT: s_endpgm1043;1044; GFX11-CU-LABEL: local_workgroup_monotonic_store:1045; GFX11-CU: ; %bb.0: ; %entry1046; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x01047; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x41048; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)1049; GFX11-CU-NEXT: v_mov_b32_e32 v0, s11050; GFX11-CU-NEXT: v_mov_b32_e32 v1, s01051; GFX11-CU-NEXT: ds_store_b32 v0, v11052; GFX11-CU-NEXT: s_endpgm1053;1054; GFX12-WGP-LABEL: local_workgroup_monotonic_store:1055; GFX12-WGP: ; %bb.0: ; %entry1056; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x01057; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x41058; GFX12-WGP-NEXT: s_wait_kmcnt 0x01059; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s11060; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s01061; GFX12-WGP-NEXT: ds_store_b32 v0, v11062; GFX12-WGP-NEXT: s_endpgm1063;1064; GFX12-CU-LABEL: local_workgroup_monotonic_store:1065; GFX12-CU: ; %bb.0: ; %entry1066; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x01067; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x41068; GFX12-CU-NEXT: s_wait_kmcnt 0x01069; GFX12-CU-NEXT: v_mov_b32_e32 v0, s11070; GFX12-CU-NEXT: v_mov_b32_e32 v1, s01071; GFX12-CU-NEXT: ds_store_b32 v0, v11072; GFX12-CU-NEXT: s_endpgm1073;1074; GFX1250-LABEL: local_workgroup_monotonic_store:1075; GFX1250: ; %bb.0: ; %entry1076; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11077; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x01078; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x41079; GFX1250-NEXT: s_wait_kmcnt 0x01080; GFX1250-NEXT: v_mov_b32_e32 v0, s11081; GFX1250-NEXT: v_mov_b32_e32 v1, s01082; GFX1250-NEXT: ds_store_b32 v0, v11083; GFX1250-NEXT: s_endpgm1084 i32 %in, ptr addrspace(3) %out) {1085entry:1086 store atomic i32 %in, ptr addrspace(3) %out syncscope("workgroup") monotonic, align 41087 ret void1088}1089 1090define amdgpu_kernel void @local_workgroup_release_store(1091; GFX6-LABEL: local_workgroup_release_store:1092; GFX6: ; %bb.0: ; %entry1093; GFX6-NEXT: s_load_dword s5, s[8:9], 0x11094; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr51095; GFX6-NEXT: s_load_dword s4, s[8:9], 0x01096; GFX6-NEXT: s_mov_b32 m0, -11097; GFX6-NEXT: s_waitcnt lgkmcnt(0)1098; GFX6-NEXT: v_mov_b32_e32 v0, s51099; GFX6-NEXT: v_mov_b32_e32 v1, s41100; GFX6-NEXT: s_waitcnt lgkmcnt(0)1101; GFX6-NEXT: ds_write_b32 v0, v11102; GFX6-NEXT: s_endpgm1103;1104; GFX7-LABEL: local_workgroup_release_store:1105; GFX7: ; %bb.0: ; %entry1106; GFX7-NEXT: s_load_dword s4, s[8:9], 0x01107; GFX7-NEXT: s_load_dword s5, s[8:9], 0x11108; GFX7-NEXT: s_mov_b32 m0, -11109; GFX7-NEXT: s_waitcnt lgkmcnt(0)1110; GFX7-NEXT: v_mov_b32_e32 v0, s51111; GFX7-NEXT: v_mov_b32_e32 v1, s41112; GFX7-NEXT: s_waitcnt lgkmcnt(0)1113; GFX7-NEXT: ds_write_b32 v0, v11114; GFX7-NEXT: s_endpgm1115;1116; GFX10-WGP-LABEL: local_workgroup_release_store:1117; GFX10-WGP: ; %bb.0: ; %entry1118; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x01119; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x41120; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)1121; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s51122; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s41123; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1124; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x01125; GFX10-WGP-NEXT: ds_write_b32 v0, v11126; GFX10-WGP-NEXT: s_endpgm1127;1128; GFX10-CU-LABEL: local_workgroup_release_store:1129; GFX10-CU: ; %bb.0: ; %entry1130; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x01131; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x41132; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)1133; GFX10-CU-NEXT: v_mov_b32_e32 v0, s51134; GFX10-CU-NEXT: v_mov_b32_e32 v1, s41135; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1136; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x01137; GFX10-CU-NEXT: ds_write_b32 v0, v11138; GFX10-CU-NEXT: s_endpgm1139;1140; SKIP-CACHE-INV-LABEL: local_workgroup_release_store:1141; SKIP-CACHE-INV: ; %bb.0: ; %entry1142; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x01143; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x11144; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -11145; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)1146; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s11147; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s01148; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)1149; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v11150; SKIP-CACHE-INV-NEXT: s_endpgm1151;1152; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_release_store:1153; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry1154; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x01155; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x41156; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1157; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s51158; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s41159; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1160; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v11161; GFX90A-NOTTGSPLIT-NEXT: s_endpgm1162;1163; GFX90A-TGSPLIT-LABEL: local_workgroup_release_store:1164; GFX90A-TGSPLIT: ; %bb.0: ; %entry1165; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x01166; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x41167; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1168; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s51169; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s41170; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1171; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v11172; GFX90A-TGSPLIT-NEXT: s_endpgm1173;1174; GFX942-NOTTGSPLIT-LABEL: local_workgroup_release_store:1175; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry1176; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x01177; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x41178; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1179; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s11180; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s01181; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1182; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v11183; GFX942-NOTTGSPLIT-NEXT: s_endpgm1184;1185; GFX942-TGSPLIT-LABEL: local_workgroup_release_store:1186; GFX942-TGSPLIT: ; %bb.0: ; %entry1187; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x01188; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x41189; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1190; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s11191; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s01192; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1193; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v11194; GFX942-TGSPLIT-NEXT: s_endpgm1195;1196; GFX11-WGP-LABEL: local_workgroup_release_store:1197; GFX11-WGP: ; %bb.0: ; %entry1198; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x01199; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x41200; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)1201; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s11202; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s01203; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1204; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x01205; GFX11-WGP-NEXT: ds_store_b32 v0, v11206; GFX11-WGP-NEXT: s_endpgm1207;1208; GFX11-CU-LABEL: local_workgroup_release_store:1209; GFX11-CU: ; %bb.0: ; %entry1210; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x01211; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x41212; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)1213; GFX11-CU-NEXT: v_mov_b32_e32 v0, s11214; GFX11-CU-NEXT: v_mov_b32_e32 v1, s01215; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1216; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x01217; GFX11-CU-NEXT: ds_store_b32 v0, v11218; GFX11-CU-NEXT: s_endpgm1219;1220; GFX12-WGP-LABEL: local_workgroup_release_store:1221; GFX12-WGP: ; %bb.0: ; %entry1222; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x01223; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x41224; GFX12-WGP-NEXT: s_wait_kmcnt 0x01225; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s11226; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s01227; GFX12-WGP-NEXT: s_wait_bvhcnt 0x01228; GFX12-WGP-NEXT: s_wait_samplecnt 0x01229; GFX12-WGP-NEXT: s_wait_storecnt 0x01230; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x01231; GFX12-WGP-NEXT: ds_store_b32 v0, v11232; GFX12-WGP-NEXT: s_endpgm1233;1234; GFX12-CU-LABEL: local_workgroup_release_store:1235; GFX12-CU: ; %bb.0: ; %entry1236; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x01237; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x41238; GFX12-CU-NEXT: s_wait_kmcnt 0x01239; GFX12-CU-NEXT: v_mov_b32_e32 v0, s11240; GFX12-CU-NEXT: v_mov_b32_e32 v1, s01241; GFX12-CU-NEXT: s_wait_bvhcnt 0x01242; GFX12-CU-NEXT: s_wait_samplecnt 0x01243; GFX12-CU-NEXT: s_wait_storecnt 0x01244; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x01245; GFX12-CU-NEXT: ds_store_b32 v0, v11246; GFX12-CU-NEXT: s_endpgm1247;1248; GFX1250-LABEL: local_workgroup_release_store:1249; GFX1250: ; %bb.0: ; %entry1250; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11251; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x01252; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x41253; GFX1250-NEXT: s_wait_kmcnt 0x01254; GFX1250-NEXT: v_mov_b32_e32 v0, s11255; GFX1250-NEXT: v_mov_b32_e32 v1, s01256; GFX1250-NEXT: s_wait_storecnt 0x01257; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x01258; GFX1250-NEXT: ds_store_b32 v0, v11259; GFX1250-NEXT: s_endpgm1260 i32 %in, ptr addrspace(3) %out) {1261entry:1262 store atomic i32 %in, ptr addrspace(3) %out syncscope("workgroup") release, align 41263 ret void1264}1265 1266define amdgpu_kernel void @local_workgroup_seq_cst_store(1267; GFX6-LABEL: local_workgroup_seq_cst_store:1268; GFX6: ; %bb.0: ; %entry1269; GFX6-NEXT: s_load_dword s5, s[8:9], 0x11270; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr51271; GFX6-NEXT: s_load_dword s4, s[8:9], 0x01272; GFX6-NEXT: s_mov_b32 m0, -11273; GFX6-NEXT: s_waitcnt lgkmcnt(0)1274; GFX6-NEXT: v_mov_b32_e32 v0, s51275; GFX6-NEXT: v_mov_b32_e32 v1, s41276; GFX6-NEXT: s_waitcnt lgkmcnt(0)1277; GFX6-NEXT: ds_write_b32 v0, v11278; GFX6-NEXT: s_endpgm1279;1280; GFX7-LABEL: local_workgroup_seq_cst_store:1281; GFX7: ; %bb.0: ; %entry1282; GFX7-NEXT: s_load_dword s4, s[8:9], 0x01283; GFX7-NEXT: s_load_dword s5, s[8:9], 0x11284; GFX7-NEXT: s_mov_b32 m0, -11285; GFX7-NEXT: s_waitcnt lgkmcnt(0)1286; GFX7-NEXT: v_mov_b32_e32 v0, s51287; GFX7-NEXT: v_mov_b32_e32 v1, s41288; GFX7-NEXT: s_waitcnt lgkmcnt(0)1289; GFX7-NEXT: ds_write_b32 v0, v11290; GFX7-NEXT: s_endpgm1291;1292; GFX10-WGP-LABEL: local_workgroup_seq_cst_store:1293; GFX10-WGP: ; %bb.0: ; %entry1294; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x01295; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x41296; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)1297; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s51298; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s41299; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1300; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x01301; GFX10-WGP-NEXT: ds_write_b32 v0, v11302; GFX10-WGP-NEXT: s_endpgm1303;1304; GFX10-CU-LABEL: local_workgroup_seq_cst_store:1305; GFX10-CU: ; %bb.0: ; %entry1306; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x01307; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x41308; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)1309; GFX10-CU-NEXT: v_mov_b32_e32 v0, s51310; GFX10-CU-NEXT: v_mov_b32_e32 v1, s41311; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1312; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x01313; GFX10-CU-NEXT: ds_write_b32 v0, v11314; GFX10-CU-NEXT: s_endpgm1315;1316; SKIP-CACHE-INV-LABEL: local_workgroup_seq_cst_store:1317; SKIP-CACHE-INV: ; %bb.0: ; %entry1318; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x01319; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x11320; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -11321; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)1322; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s11323; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s01324; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)1325; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v11326; SKIP-CACHE-INV-NEXT: s_endpgm1327;1328; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_seq_cst_store:1329; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry1330; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x01331; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x41332; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1333; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s51334; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s41335; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1336; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v11337; GFX90A-NOTTGSPLIT-NEXT: s_endpgm1338;1339; GFX90A-TGSPLIT-LABEL: local_workgroup_seq_cst_store:1340; GFX90A-TGSPLIT: ; %bb.0: ; %entry1341; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x01342; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x41343; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1344; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s51345; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s41346; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1347; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v11348; GFX90A-TGSPLIT-NEXT: s_endpgm1349;1350; GFX942-NOTTGSPLIT-LABEL: local_workgroup_seq_cst_store:1351; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry1352; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x01353; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x41354; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1355; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s11356; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s01357; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1358; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v11359; GFX942-NOTTGSPLIT-NEXT: s_endpgm1360;1361; GFX942-TGSPLIT-LABEL: local_workgroup_seq_cst_store:1362; GFX942-TGSPLIT: ; %bb.0: ; %entry1363; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x01364; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x41365; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1366; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s11367; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s01368; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1369; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v11370; GFX942-TGSPLIT-NEXT: s_endpgm1371;1372; GFX11-WGP-LABEL: local_workgroup_seq_cst_store:1373; GFX11-WGP: ; %bb.0: ; %entry1374; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x01375; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x41376; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)1377; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s11378; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s01379; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1380; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x01381; GFX11-WGP-NEXT: ds_store_b32 v0, v11382; GFX11-WGP-NEXT: s_endpgm1383;1384; GFX11-CU-LABEL: local_workgroup_seq_cst_store:1385; GFX11-CU: ; %bb.0: ; %entry1386; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x01387; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x41388; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)1389; GFX11-CU-NEXT: v_mov_b32_e32 v0, s11390; GFX11-CU-NEXT: v_mov_b32_e32 v1, s01391; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1392; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x01393; GFX11-CU-NEXT: ds_store_b32 v0, v11394; GFX11-CU-NEXT: s_endpgm1395;1396; GFX12-WGP-LABEL: local_workgroup_seq_cst_store:1397; GFX12-WGP: ; %bb.0: ; %entry1398; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x01399; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x41400; GFX12-WGP-NEXT: s_wait_kmcnt 0x01401; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s11402; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s01403; GFX12-WGP-NEXT: s_wait_bvhcnt 0x01404; GFX12-WGP-NEXT: s_wait_samplecnt 0x01405; GFX12-WGP-NEXT: s_wait_storecnt 0x01406; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x01407; GFX12-WGP-NEXT: ds_store_b32 v0, v11408; GFX12-WGP-NEXT: s_endpgm1409;1410; GFX12-CU-LABEL: local_workgroup_seq_cst_store:1411; GFX12-CU: ; %bb.0: ; %entry1412; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x01413; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x41414; GFX12-CU-NEXT: s_wait_kmcnt 0x01415; GFX12-CU-NEXT: v_mov_b32_e32 v0, s11416; GFX12-CU-NEXT: v_mov_b32_e32 v1, s01417; GFX12-CU-NEXT: s_wait_bvhcnt 0x01418; GFX12-CU-NEXT: s_wait_samplecnt 0x01419; GFX12-CU-NEXT: s_wait_storecnt 0x01420; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x01421; GFX12-CU-NEXT: ds_store_b32 v0, v11422; GFX12-CU-NEXT: s_endpgm1423;1424; GFX1250-LABEL: local_workgroup_seq_cst_store:1425; GFX1250: ; %bb.0: ; %entry1426; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11427; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x01428; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x41429; GFX1250-NEXT: s_wait_kmcnt 0x01430; GFX1250-NEXT: v_mov_b32_e32 v0, s11431; GFX1250-NEXT: v_mov_b32_e32 v1, s01432; GFX1250-NEXT: s_wait_storecnt 0x01433; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x01434; GFX1250-NEXT: ds_store_b32 v0, v11435; GFX1250-NEXT: s_endpgm1436 i32 %in, ptr addrspace(3) %out) {1437entry:1438 store atomic i32 %in, ptr addrspace(3) %out syncscope("workgroup") seq_cst, align 41439 ret void1440}1441 1442define amdgpu_kernel void @local_workgroup_monotonic_atomicrmw(1443; GFX6-LABEL: local_workgroup_monotonic_atomicrmw:1444; GFX6: ; %bb.0: ; %entry1445; GFX6-NEXT: s_load_dword s5, s[8:9], 0x01446; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr51447; GFX6-NEXT: s_load_dword s4, s[8:9], 0x11448; GFX6-NEXT: s_mov_b32 m0, -11449; GFX6-NEXT: s_waitcnt lgkmcnt(0)1450; GFX6-NEXT: v_mov_b32_e32 v0, s51451; GFX6-NEXT: v_mov_b32_e32 v1, s41452; GFX6-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11453; GFX6-NEXT: s_endpgm1454;1455; GFX7-LABEL: local_workgroup_monotonic_atomicrmw:1456; GFX7: ; %bb.0: ; %entry1457; GFX7-NEXT: s_load_dword s5, s[8:9], 0x01458; GFX7-NEXT: s_load_dword s4, s[8:9], 0x11459; GFX7-NEXT: s_mov_b32 m0, -11460; GFX7-NEXT: s_waitcnt lgkmcnt(0)1461; GFX7-NEXT: v_mov_b32_e32 v0, s51462; GFX7-NEXT: v_mov_b32_e32 v1, s41463; GFX7-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11464; GFX7-NEXT: s_endpgm1465;1466; GFX10-WGP-LABEL: local_workgroup_monotonic_atomicrmw:1467; GFX10-WGP: ; %bb.0: ; %entry1468; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x01469; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x41470; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)1471; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s51472; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s41473; GFX10-WGP-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11474; GFX10-WGP-NEXT: s_endpgm1475;1476; GFX10-CU-LABEL: local_workgroup_monotonic_atomicrmw:1477; GFX10-CU: ; %bb.0: ; %entry1478; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x01479; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x41480; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)1481; GFX10-CU-NEXT: v_mov_b32_e32 v0, s51482; GFX10-CU-NEXT: v_mov_b32_e32 v1, s41483; GFX10-CU-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11484; GFX10-CU-NEXT: s_endpgm1485;1486; SKIP-CACHE-INV-LABEL: local_workgroup_monotonic_atomicrmw:1487; SKIP-CACHE-INV: ; %bb.0: ; %entry1488; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x01489; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x11490; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -11491; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)1492; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s11493; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s01494; SKIP-CACHE-INV-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11495; SKIP-CACHE-INV-NEXT: s_endpgm1496;1497; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_monotonic_atomicrmw:1498; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry1499; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x01500; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x41501; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1502; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s51503; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s41504; GFX90A-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11505; GFX90A-NOTTGSPLIT-NEXT: s_endpgm1506;1507; GFX90A-TGSPLIT-LABEL: local_workgroup_monotonic_atomicrmw:1508; GFX90A-TGSPLIT: ; %bb.0: ; %entry1509; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x01510; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x41511; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1512; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s51513; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s41514; GFX90A-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11515; GFX90A-TGSPLIT-NEXT: s_endpgm1516;1517; GFX942-NOTTGSPLIT-LABEL: local_workgroup_monotonic_atomicrmw:1518; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry1519; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x01520; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x41521; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1522; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s11523; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s01524; GFX942-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11525; GFX942-NOTTGSPLIT-NEXT: s_endpgm1526;1527; GFX942-TGSPLIT-LABEL: local_workgroup_monotonic_atomicrmw:1528; GFX942-TGSPLIT: ; %bb.0: ; %entry1529; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x01530; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x41531; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1532; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s11533; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s01534; GFX942-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11535; GFX942-TGSPLIT-NEXT: s_endpgm1536;1537; GFX11-WGP-LABEL: local_workgroup_monotonic_atomicrmw:1538; GFX11-WGP: ; %bb.0: ; %entry1539; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x01540; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x41541; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)1542; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s11543; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s01544; GFX11-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v11545; GFX11-WGP-NEXT: s_endpgm1546;1547; GFX11-CU-LABEL: local_workgroup_monotonic_atomicrmw:1548; GFX11-CU: ; %bb.0: ; %entry1549; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x01550; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x41551; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)1552; GFX11-CU-NEXT: v_mov_b32_e32 v0, s11553; GFX11-CU-NEXT: v_mov_b32_e32 v1, s01554; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v11555; GFX11-CU-NEXT: s_endpgm1556;1557; GFX12-WGP-LABEL: local_workgroup_monotonic_atomicrmw:1558; GFX12-WGP: ; %bb.0: ; %entry1559; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x01560; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x41561; GFX12-WGP-NEXT: s_wait_kmcnt 0x01562; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s11563; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s01564; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v11565; GFX12-WGP-NEXT: s_endpgm1566;1567; GFX12-CU-LABEL: local_workgroup_monotonic_atomicrmw:1568; GFX12-CU: ; %bb.0: ; %entry1569; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x01570; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x41571; GFX12-CU-NEXT: s_wait_kmcnt 0x01572; GFX12-CU-NEXT: v_mov_b32_e32 v0, s11573; GFX12-CU-NEXT: v_mov_b32_e32 v1, s01574; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v11575; GFX12-CU-NEXT: s_endpgm1576;1577; GFX1250-LABEL: local_workgroup_monotonic_atomicrmw:1578; GFX1250: ; %bb.0: ; %entry1579; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11580; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x01581; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x41582; GFX1250-NEXT: s_wait_kmcnt 0x01583; GFX1250-NEXT: v_mov_b32_e32 v0, s11584; GFX1250-NEXT: v_mov_b32_e32 v1, s01585; GFX1250-NEXT: ds_storexchg_rtn_b32 v0, v0, v11586; GFX1250-NEXT: s_endpgm1587 ptr addrspace(3) %out, i32 %in) {1588entry:1589 %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup") monotonic1590 ret void1591}1592 1593define amdgpu_kernel void @local_workgroup_acquire_atomicrmw(1594; GFX6-LABEL: local_workgroup_acquire_atomicrmw:1595; GFX6: ; %bb.0: ; %entry1596; GFX6-NEXT: s_load_dword s5, s[8:9], 0x01597; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr51598; GFX6-NEXT: s_load_dword s4, s[8:9], 0x11599; GFX6-NEXT: s_mov_b32 m0, -11600; GFX6-NEXT: s_waitcnt lgkmcnt(0)1601; GFX6-NEXT: v_mov_b32_e32 v0, s51602; GFX6-NEXT: v_mov_b32_e32 v1, s41603; GFX6-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11604; GFX6-NEXT: s_waitcnt lgkmcnt(0)1605; GFX6-NEXT: s_endpgm1606;1607; GFX7-LABEL: local_workgroup_acquire_atomicrmw:1608; GFX7: ; %bb.0: ; %entry1609; GFX7-NEXT: s_load_dword s5, s[8:9], 0x01610; GFX7-NEXT: s_load_dword s4, s[8:9], 0x11611; GFX7-NEXT: s_mov_b32 m0, -11612; GFX7-NEXT: s_waitcnt lgkmcnt(0)1613; GFX7-NEXT: v_mov_b32_e32 v0, s51614; GFX7-NEXT: v_mov_b32_e32 v1, s41615; GFX7-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11616; GFX7-NEXT: s_waitcnt lgkmcnt(0)1617; GFX7-NEXT: s_endpgm1618;1619; GFX10-WGP-LABEL: local_workgroup_acquire_atomicrmw:1620; GFX10-WGP: ; %bb.0: ; %entry1621; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x01622; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x41623; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)1624; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s51625; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s41626; GFX10-WGP-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11627; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)1628; GFX10-WGP-NEXT: buffer_gl0_inv1629; GFX10-WGP-NEXT: s_endpgm1630;1631; GFX10-CU-LABEL: local_workgroup_acquire_atomicrmw:1632; GFX10-CU: ; %bb.0: ; %entry1633; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x01634; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x41635; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)1636; GFX10-CU-NEXT: v_mov_b32_e32 v0, s51637; GFX10-CU-NEXT: v_mov_b32_e32 v1, s41638; GFX10-CU-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11639; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)1640; GFX10-CU-NEXT: s_endpgm1641;1642; SKIP-CACHE-INV-LABEL: local_workgroup_acquire_atomicrmw:1643; SKIP-CACHE-INV: ; %bb.0: ; %entry1644; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x01645; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x11646; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -11647; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)1648; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s11649; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s01650; SKIP-CACHE-INV-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11651; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)1652; SKIP-CACHE-INV-NEXT: s_endpgm1653;1654; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_acquire_atomicrmw:1655; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry1656; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x01657; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x41658; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1659; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s51660; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s41661; GFX90A-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11662; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1663; GFX90A-NOTTGSPLIT-NEXT: s_endpgm1664;1665; GFX90A-TGSPLIT-LABEL: local_workgroup_acquire_atomicrmw:1666; GFX90A-TGSPLIT: ; %bb.0: ; %entry1667; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x01668; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x41669; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1670; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s51671; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s41672; GFX90A-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11673; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol1674; GFX90A-TGSPLIT-NEXT: s_endpgm1675;1676; GFX942-NOTTGSPLIT-LABEL: local_workgroup_acquire_atomicrmw:1677; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry1678; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x01679; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x41680; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1681; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s11682; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s01683; GFX942-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11684; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1685; GFX942-NOTTGSPLIT-NEXT: s_endpgm1686;1687; GFX942-TGSPLIT-LABEL: local_workgroup_acquire_atomicrmw:1688; GFX942-TGSPLIT: ; %bb.0: ; %entry1689; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x01690; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x41691; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1692; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s11693; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s01694; GFX942-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11695; GFX942-TGSPLIT-NEXT: buffer_inv sc01696; GFX942-TGSPLIT-NEXT: s_endpgm1697;1698; GFX11-WGP-LABEL: local_workgroup_acquire_atomicrmw:1699; GFX11-WGP: ; %bb.0: ; %entry1700; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x01701; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x41702; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)1703; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s11704; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s01705; GFX11-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v11706; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)1707; GFX11-WGP-NEXT: buffer_gl0_inv1708; GFX11-WGP-NEXT: s_endpgm1709;1710; GFX11-CU-LABEL: local_workgroup_acquire_atomicrmw:1711; GFX11-CU: ; %bb.0: ; %entry1712; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x01713; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x41714; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)1715; GFX11-CU-NEXT: v_mov_b32_e32 v0, s11716; GFX11-CU-NEXT: v_mov_b32_e32 v1, s01717; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v11718; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)1719; GFX11-CU-NEXT: s_endpgm1720;1721; GFX12-WGP-LABEL: local_workgroup_acquire_atomicrmw:1722; GFX12-WGP: ; %bb.0: ; %entry1723; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x01724; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x41725; GFX12-WGP-NEXT: s_wait_kmcnt 0x01726; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s11727; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s01728; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v11729; GFX12-WGP-NEXT: s_wait_dscnt 0x01730; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE1731; GFX12-WGP-NEXT: s_endpgm1732;1733; GFX12-CU-LABEL: local_workgroup_acquire_atomicrmw:1734; GFX12-CU: ; %bb.0: ; %entry1735; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x01736; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x41737; GFX12-CU-NEXT: s_wait_kmcnt 0x01738; GFX12-CU-NEXT: v_mov_b32_e32 v0, s11739; GFX12-CU-NEXT: v_mov_b32_e32 v1, s01740; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v11741; GFX12-CU-NEXT: s_wait_dscnt 0x01742; GFX12-CU-NEXT: s_endpgm1743;1744; GFX1250-LABEL: local_workgroup_acquire_atomicrmw:1745; GFX1250: ; %bb.0: ; %entry1746; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11747; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x01748; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x41749; GFX1250-NEXT: s_wait_kmcnt 0x01750; GFX1250-NEXT: v_mov_b32_e32 v0, s11751; GFX1250-NEXT: v_mov_b32_e32 v1, s01752; GFX1250-NEXT: ds_storexchg_rtn_b32 v0, v0, v11753; GFX1250-NEXT: s_wait_dscnt 0x01754; GFX1250-NEXT: s_endpgm1755 ptr addrspace(3) %out, i32 %in) {1756entry:1757 %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup") acquire1758 ret void1759}1760 1761define amdgpu_kernel void @local_workgroup_release_atomicrmw(1762; GFX6-LABEL: local_workgroup_release_atomicrmw:1763; GFX6: ; %bb.0: ; %entry1764; GFX6-NEXT: s_load_dword s5, s[8:9], 0x01765; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr51766; GFX6-NEXT: s_load_dword s4, s[8:9], 0x11767; GFX6-NEXT: s_mov_b32 m0, -11768; GFX6-NEXT: s_waitcnt lgkmcnt(0)1769; GFX6-NEXT: v_mov_b32_e32 v0, s51770; GFX6-NEXT: v_mov_b32_e32 v1, s41771; GFX6-NEXT: s_waitcnt lgkmcnt(0)1772; GFX6-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11773; GFX6-NEXT: s_endpgm1774;1775; GFX7-LABEL: local_workgroup_release_atomicrmw:1776; GFX7: ; %bb.0: ; %entry1777; GFX7-NEXT: s_load_dword s5, s[8:9], 0x01778; GFX7-NEXT: s_load_dword s4, s[8:9], 0x11779; GFX7-NEXT: s_mov_b32 m0, -11780; GFX7-NEXT: s_waitcnt lgkmcnt(0)1781; GFX7-NEXT: v_mov_b32_e32 v0, s51782; GFX7-NEXT: v_mov_b32_e32 v1, s41783; GFX7-NEXT: s_waitcnt lgkmcnt(0)1784; GFX7-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11785; GFX7-NEXT: s_endpgm1786;1787; GFX10-WGP-LABEL: local_workgroup_release_atomicrmw:1788; GFX10-WGP: ; %bb.0: ; %entry1789; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x01790; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x41791; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)1792; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s51793; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s41794; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1795; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x01796; GFX10-WGP-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11797; GFX10-WGP-NEXT: s_endpgm1798;1799; GFX10-CU-LABEL: local_workgroup_release_atomicrmw:1800; GFX10-CU: ; %bb.0: ; %entry1801; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x01802; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x41803; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)1804; GFX10-CU-NEXT: v_mov_b32_e32 v0, s51805; GFX10-CU-NEXT: v_mov_b32_e32 v1, s41806; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1807; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x01808; GFX10-CU-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11809; GFX10-CU-NEXT: s_endpgm1810;1811; SKIP-CACHE-INV-LABEL: local_workgroup_release_atomicrmw:1812; SKIP-CACHE-INV: ; %bb.0: ; %entry1813; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x01814; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x11815; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -11816; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)1817; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s11818; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s01819; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)1820; SKIP-CACHE-INV-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11821; SKIP-CACHE-INV-NEXT: s_endpgm1822;1823; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_release_atomicrmw:1824; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry1825; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x01826; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x41827; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1828; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s51829; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s41830; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1831; GFX90A-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11832; GFX90A-NOTTGSPLIT-NEXT: s_endpgm1833;1834; GFX90A-TGSPLIT-LABEL: local_workgroup_release_atomicrmw:1835; GFX90A-TGSPLIT: ; %bb.0: ; %entry1836; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x01837; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x41838; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1839; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s51840; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s41841; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1842; GFX90A-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11843; GFX90A-TGSPLIT-NEXT: s_endpgm1844;1845; GFX942-NOTTGSPLIT-LABEL: local_workgroup_release_atomicrmw:1846; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry1847; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x01848; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x41849; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1850; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s11851; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s01852; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1853; GFX942-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11854; GFX942-NOTTGSPLIT-NEXT: s_endpgm1855;1856; GFX942-TGSPLIT-LABEL: local_workgroup_release_atomicrmw:1857; GFX942-TGSPLIT: ; %bb.0: ; %entry1858; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x01859; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x41860; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)1861; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s11862; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s01863; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1864; GFX942-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11865; GFX942-TGSPLIT-NEXT: s_endpgm1866;1867; GFX11-WGP-LABEL: local_workgroup_release_atomicrmw:1868; GFX11-WGP: ; %bb.0: ; %entry1869; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x01870; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x41871; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)1872; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s11873; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s01874; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1875; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x01876; GFX11-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v11877; GFX11-WGP-NEXT: s_endpgm1878;1879; GFX11-CU-LABEL: local_workgroup_release_atomicrmw:1880; GFX11-CU: ; %bb.0: ; %entry1881; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x01882; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x41883; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)1884; GFX11-CU-NEXT: v_mov_b32_e32 v0, s11885; GFX11-CU-NEXT: v_mov_b32_e32 v1, s01886; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1887; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x01888; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v11889; GFX11-CU-NEXT: s_endpgm1890;1891; GFX12-WGP-LABEL: local_workgroup_release_atomicrmw:1892; GFX12-WGP: ; %bb.0: ; %entry1893; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x01894; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x41895; GFX12-WGP-NEXT: s_wait_kmcnt 0x01896; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s11897; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s01898; GFX12-WGP-NEXT: s_wait_bvhcnt 0x01899; GFX12-WGP-NEXT: s_wait_samplecnt 0x01900; GFX12-WGP-NEXT: s_wait_storecnt 0x01901; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x01902; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v11903; GFX12-WGP-NEXT: s_endpgm1904;1905; GFX12-CU-LABEL: local_workgroup_release_atomicrmw:1906; GFX12-CU: ; %bb.0: ; %entry1907; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x01908; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x41909; GFX12-CU-NEXT: s_wait_kmcnt 0x01910; GFX12-CU-NEXT: v_mov_b32_e32 v0, s11911; GFX12-CU-NEXT: v_mov_b32_e32 v1, s01912; GFX12-CU-NEXT: s_wait_bvhcnt 0x01913; GFX12-CU-NEXT: s_wait_samplecnt 0x01914; GFX12-CU-NEXT: s_wait_storecnt 0x01915; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x01916; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v11917; GFX12-CU-NEXT: s_endpgm1918;1919; GFX1250-LABEL: local_workgroup_release_atomicrmw:1920; GFX1250: ; %bb.0: ; %entry1921; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11922; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x01923; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x41924; GFX1250-NEXT: s_wait_kmcnt 0x01925; GFX1250-NEXT: v_mov_b32_e32 v0, s11926; GFX1250-NEXT: v_mov_b32_e32 v1, s01927; GFX1250-NEXT: s_wait_storecnt 0x01928; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x01929; GFX1250-NEXT: ds_storexchg_rtn_b32 v0, v0, v11930; GFX1250-NEXT: s_endpgm1931 ptr addrspace(3) %out, i32 %in) {1932entry:1933 %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup") release1934 ret void1935}1936 1937define amdgpu_kernel void @local_workgroup_acq_rel_atomicrmw(1938; GFX6-LABEL: local_workgroup_acq_rel_atomicrmw:1939; GFX6: ; %bb.0: ; %entry1940; GFX6-NEXT: s_load_dword s5, s[8:9], 0x01941; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr51942; GFX6-NEXT: s_load_dword s4, s[8:9], 0x11943; GFX6-NEXT: s_mov_b32 m0, -11944; GFX6-NEXT: s_waitcnt lgkmcnt(0)1945; GFX6-NEXT: v_mov_b32_e32 v0, s51946; GFX6-NEXT: v_mov_b32_e32 v1, s41947; GFX6-NEXT: s_waitcnt lgkmcnt(0)1948; GFX6-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11949; GFX6-NEXT: s_waitcnt lgkmcnt(0)1950; GFX6-NEXT: s_endpgm1951;1952; GFX7-LABEL: local_workgroup_acq_rel_atomicrmw:1953; GFX7: ; %bb.0: ; %entry1954; GFX7-NEXT: s_load_dword s5, s[8:9], 0x01955; GFX7-NEXT: s_load_dword s4, s[8:9], 0x11956; GFX7-NEXT: s_mov_b32 m0, -11957; GFX7-NEXT: s_waitcnt lgkmcnt(0)1958; GFX7-NEXT: v_mov_b32_e32 v0, s51959; GFX7-NEXT: v_mov_b32_e32 v1, s41960; GFX7-NEXT: s_waitcnt lgkmcnt(0)1961; GFX7-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11962; GFX7-NEXT: s_waitcnt lgkmcnt(0)1963; GFX7-NEXT: s_endpgm1964;1965; GFX10-WGP-LABEL: local_workgroup_acq_rel_atomicrmw:1966; GFX10-WGP: ; %bb.0: ; %entry1967; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x01968; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x41969; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)1970; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s51971; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s41972; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1973; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x01974; GFX10-WGP-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11975; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)1976; GFX10-WGP-NEXT: buffer_gl0_inv1977; GFX10-WGP-NEXT: s_endpgm1978;1979; GFX10-CU-LABEL: local_workgroup_acq_rel_atomicrmw:1980; GFX10-CU: ; %bb.0: ; %entry1981; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x01982; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x41983; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)1984; GFX10-CU-NEXT: v_mov_b32_e32 v0, s51985; GFX10-CU-NEXT: v_mov_b32_e32 v1, s41986; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1987; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x01988; GFX10-CU-NEXT: ds_wrxchg_rtn_b32 v0, v0, v11989; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)1990; GFX10-CU-NEXT: s_endpgm1991;1992; SKIP-CACHE-INV-LABEL: local_workgroup_acq_rel_atomicrmw:1993; SKIP-CACHE-INV: ; %bb.0: ; %entry1994; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x01995; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x11996; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -11997; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)1998; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s11999; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s02000; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)2001; SKIP-CACHE-INV-NEXT: ds_wrxchg_rtn_b32 v0, v0, v12002; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)2003; SKIP-CACHE-INV-NEXT: s_endpgm2004;2005; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_acq_rel_atomicrmw:2006; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry2007; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x02008; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x42009; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2010; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s52011; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s42012; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2013; GFX90A-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v12014; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2015; GFX90A-NOTTGSPLIT-NEXT: s_endpgm2016;2017; GFX90A-TGSPLIT-LABEL: local_workgroup_acq_rel_atomicrmw:2018; GFX90A-TGSPLIT: ; %bb.0: ; %entry2019; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x02020; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x42021; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2022; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s52023; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s42024; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2025; GFX90A-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v12026; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol2027; GFX90A-TGSPLIT-NEXT: s_endpgm2028;2029; GFX942-NOTTGSPLIT-LABEL: local_workgroup_acq_rel_atomicrmw:2030; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry2031; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x02032; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x42033; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2034; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s12035; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s02036; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2037; GFX942-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v12038; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2039; GFX942-NOTTGSPLIT-NEXT: s_endpgm2040;2041; GFX942-TGSPLIT-LABEL: local_workgroup_acq_rel_atomicrmw:2042; GFX942-TGSPLIT: ; %bb.0: ; %entry2043; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x02044; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x42045; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2046; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s12047; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s02048; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2049; GFX942-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v12050; GFX942-TGSPLIT-NEXT: buffer_inv sc02051; GFX942-TGSPLIT-NEXT: s_endpgm2052;2053; GFX11-WGP-LABEL: local_workgroup_acq_rel_atomicrmw:2054; GFX11-WGP: ; %bb.0: ; %entry2055; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x02056; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x42057; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)2058; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s12059; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s02060; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2061; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x02062; GFX11-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v12063; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)2064; GFX11-WGP-NEXT: buffer_gl0_inv2065; GFX11-WGP-NEXT: s_endpgm2066;2067; GFX11-CU-LABEL: local_workgroup_acq_rel_atomicrmw:2068; GFX11-CU: ; %bb.0: ; %entry2069; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x02070; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x42071; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)2072; GFX11-CU-NEXT: v_mov_b32_e32 v0, s12073; GFX11-CU-NEXT: v_mov_b32_e32 v1, s02074; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2075; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x02076; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v12077; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)2078; GFX11-CU-NEXT: s_endpgm2079;2080; GFX12-WGP-LABEL: local_workgroup_acq_rel_atomicrmw:2081; GFX12-WGP: ; %bb.0: ; %entry2082; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x02083; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x42084; GFX12-WGP-NEXT: s_wait_kmcnt 0x02085; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s12086; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s02087; GFX12-WGP-NEXT: s_wait_bvhcnt 0x02088; GFX12-WGP-NEXT: s_wait_samplecnt 0x02089; GFX12-WGP-NEXT: s_wait_storecnt 0x02090; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x02091; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v12092; GFX12-WGP-NEXT: s_wait_dscnt 0x02093; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE2094; GFX12-WGP-NEXT: s_endpgm2095;2096; GFX12-CU-LABEL: local_workgroup_acq_rel_atomicrmw:2097; GFX12-CU: ; %bb.0: ; %entry2098; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x02099; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x42100; GFX12-CU-NEXT: s_wait_kmcnt 0x02101; GFX12-CU-NEXT: v_mov_b32_e32 v0, s12102; GFX12-CU-NEXT: v_mov_b32_e32 v1, s02103; GFX12-CU-NEXT: s_wait_bvhcnt 0x02104; GFX12-CU-NEXT: s_wait_samplecnt 0x02105; GFX12-CU-NEXT: s_wait_storecnt 0x02106; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x02107; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v12108; GFX12-CU-NEXT: s_wait_dscnt 0x02109; GFX12-CU-NEXT: s_endpgm2110;2111; GFX1250-LABEL: local_workgroup_acq_rel_atomicrmw:2112; GFX1250: ; %bb.0: ; %entry2113; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12114; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x02115; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x42116; GFX1250-NEXT: s_wait_kmcnt 0x02117; GFX1250-NEXT: v_mov_b32_e32 v0, s12118; GFX1250-NEXT: v_mov_b32_e32 v1, s02119; GFX1250-NEXT: s_wait_storecnt 0x02120; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x02121; GFX1250-NEXT: ds_storexchg_rtn_b32 v0, v0, v12122; GFX1250-NEXT: s_wait_dscnt 0x02123; GFX1250-NEXT: s_endpgm2124 ptr addrspace(3) %out, i32 %in) {2125entry:2126 %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup") acq_rel2127 ret void2128}2129 2130define amdgpu_kernel void @local_workgroup_seq_cst_atomicrmw(2131; GFX6-LABEL: local_workgroup_seq_cst_atomicrmw:2132; GFX6: ; %bb.0: ; %entry2133; GFX6-NEXT: s_load_dword s5, s[8:9], 0x02134; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr52135; GFX6-NEXT: s_load_dword s4, s[8:9], 0x12136; GFX6-NEXT: s_mov_b32 m0, -12137; GFX6-NEXT: s_waitcnt lgkmcnt(0)2138; GFX6-NEXT: v_mov_b32_e32 v0, s52139; GFX6-NEXT: v_mov_b32_e32 v1, s42140; GFX6-NEXT: s_waitcnt lgkmcnt(0)2141; GFX6-NEXT: ds_wrxchg_rtn_b32 v0, v0, v12142; GFX6-NEXT: s_waitcnt lgkmcnt(0)2143; GFX6-NEXT: s_endpgm2144;2145; GFX7-LABEL: local_workgroup_seq_cst_atomicrmw:2146; GFX7: ; %bb.0: ; %entry2147; GFX7-NEXT: s_load_dword s5, s[8:9], 0x02148; GFX7-NEXT: s_load_dword s4, s[8:9], 0x12149; GFX7-NEXT: s_mov_b32 m0, -12150; GFX7-NEXT: s_waitcnt lgkmcnt(0)2151; GFX7-NEXT: v_mov_b32_e32 v0, s52152; GFX7-NEXT: v_mov_b32_e32 v1, s42153; GFX7-NEXT: s_waitcnt lgkmcnt(0)2154; GFX7-NEXT: ds_wrxchg_rtn_b32 v0, v0, v12155; GFX7-NEXT: s_waitcnt lgkmcnt(0)2156; GFX7-NEXT: s_endpgm2157;2158; GFX10-WGP-LABEL: local_workgroup_seq_cst_atomicrmw:2159; GFX10-WGP: ; %bb.0: ; %entry2160; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x02161; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x42162; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)2163; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s52164; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s42165; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2166; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x02167; GFX10-WGP-NEXT: ds_wrxchg_rtn_b32 v0, v0, v12168; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)2169; GFX10-WGP-NEXT: buffer_gl0_inv2170; GFX10-WGP-NEXT: s_endpgm2171;2172; GFX10-CU-LABEL: local_workgroup_seq_cst_atomicrmw:2173; GFX10-CU: ; %bb.0: ; %entry2174; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x02175; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x42176; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)2177; GFX10-CU-NEXT: v_mov_b32_e32 v0, s52178; GFX10-CU-NEXT: v_mov_b32_e32 v1, s42179; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2180; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x02181; GFX10-CU-NEXT: ds_wrxchg_rtn_b32 v0, v0, v12182; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)2183; GFX10-CU-NEXT: s_endpgm2184;2185; SKIP-CACHE-INV-LABEL: local_workgroup_seq_cst_atomicrmw:2186; SKIP-CACHE-INV: ; %bb.0: ; %entry2187; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x02188; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x12189; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -12190; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)2191; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s12192; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s02193; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)2194; SKIP-CACHE-INV-NEXT: ds_wrxchg_rtn_b32 v0, v0, v12195; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)2196; SKIP-CACHE-INV-NEXT: s_endpgm2197;2198; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_seq_cst_atomicrmw:2199; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry2200; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x02201; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x42202; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2203; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s52204; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s42205; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2206; GFX90A-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v12207; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2208; GFX90A-NOTTGSPLIT-NEXT: s_endpgm2209;2210; GFX90A-TGSPLIT-LABEL: local_workgroup_seq_cst_atomicrmw:2211; GFX90A-TGSPLIT: ; %bb.0: ; %entry2212; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x02213; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x42214; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2215; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s52216; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s42217; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2218; GFX90A-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v12219; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol2220; GFX90A-TGSPLIT-NEXT: s_endpgm2221;2222; GFX942-NOTTGSPLIT-LABEL: local_workgroup_seq_cst_atomicrmw:2223; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry2224; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x02225; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x42226; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2227; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s12228; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s02229; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2230; GFX942-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v12231; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2232; GFX942-NOTTGSPLIT-NEXT: s_endpgm2233;2234; GFX942-TGSPLIT-LABEL: local_workgroup_seq_cst_atomicrmw:2235; GFX942-TGSPLIT: ; %bb.0: ; %entry2236; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x02237; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x42238; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2239; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s12240; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s02241; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2242; GFX942-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v12243; GFX942-TGSPLIT-NEXT: buffer_inv sc02244; GFX942-TGSPLIT-NEXT: s_endpgm2245;2246; GFX11-WGP-LABEL: local_workgroup_seq_cst_atomicrmw:2247; GFX11-WGP: ; %bb.0: ; %entry2248; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x02249; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x42250; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)2251; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s12252; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s02253; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2254; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x02255; GFX11-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v12256; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)2257; GFX11-WGP-NEXT: buffer_gl0_inv2258; GFX11-WGP-NEXT: s_endpgm2259;2260; GFX11-CU-LABEL: local_workgroup_seq_cst_atomicrmw:2261; GFX11-CU: ; %bb.0: ; %entry2262; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x02263; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x42264; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)2265; GFX11-CU-NEXT: v_mov_b32_e32 v0, s12266; GFX11-CU-NEXT: v_mov_b32_e32 v1, s02267; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2268; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x02269; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v12270; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)2271; GFX11-CU-NEXT: s_endpgm2272;2273; GFX12-WGP-LABEL: local_workgroup_seq_cst_atomicrmw:2274; GFX12-WGP: ; %bb.0: ; %entry2275; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x02276; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x42277; GFX12-WGP-NEXT: s_wait_kmcnt 0x02278; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s12279; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s02280; GFX12-WGP-NEXT: s_wait_bvhcnt 0x02281; GFX12-WGP-NEXT: s_wait_samplecnt 0x02282; GFX12-WGP-NEXT: s_wait_storecnt 0x02283; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x02284; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v12285; GFX12-WGP-NEXT: s_wait_dscnt 0x02286; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE2287; GFX12-WGP-NEXT: s_endpgm2288;2289; GFX12-CU-LABEL: local_workgroup_seq_cst_atomicrmw:2290; GFX12-CU: ; %bb.0: ; %entry2291; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x02292; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x42293; GFX12-CU-NEXT: s_wait_kmcnt 0x02294; GFX12-CU-NEXT: v_mov_b32_e32 v0, s12295; GFX12-CU-NEXT: v_mov_b32_e32 v1, s02296; GFX12-CU-NEXT: s_wait_bvhcnt 0x02297; GFX12-CU-NEXT: s_wait_samplecnt 0x02298; GFX12-CU-NEXT: s_wait_storecnt 0x02299; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x02300; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v12301; GFX12-CU-NEXT: s_wait_dscnt 0x02302; GFX12-CU-NEXT: s_endpgm2303;2304; GFX1250-LABEL: local_workgroup_seq_cst_atomicrmw:2305; GFX1250: ; %bb.0: ; %entry2306; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12307; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x02308; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x42309; GFX1250-NEXT: s_wait_kmcnt 0x02310; GFX1250-NEXT: v_mov_b32_e32 v0, s12311; GFX1250-NEXT: v_mov_b32_e32 v1, s02312; GFX1250-NEXT: s_wait_storecnt 0x02313; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x02314; GFX1250-NEXT: ds_storexchg_rtn_b32 v0, v0, v12315; GFX1250-NEXT: s_wait_dscnt 0x02316; GFX1250-NEXT: s_endpgm2317 ptr addrspace(3) %out, i32 %in) {2318entry:2319 %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup") seq_cst2320 ret void2321}2322 2323define amdgpu_kernel void @local_workgroup_acquire_ret_atomicrmw(2324; GFX6-LABEL: local_workgroup_acquire_ret_atomicrmw:2325; GFX6: ; %bb.0: ; %entry2326; GFX6-NEXT: s_load_dword s4, s[8:9], 0x02327; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr42328; GFX6-NEXT: s_load_dword s5, s[8:9], 0x12329; GFX6-NEXT: s_mov_b32 m0, -12330; GFX6-NEXT: s_waitcnt lgkmcnt(0)2331; GFX6-NEXT: v_mov_b32_e32 v0, s42332; GFX6-NEXT: v_mov_b32_e32 v1, s52333; GFX6-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12334; GFX6-NEXT: s_waitcnt lgkmcnt(0)2335; GFX6-NEXT: s_mov_b32 m0, -12336; GFX6-NEXT: v_mov_b32_e32 v0, s42337; GFX6-NEXT: ds_write_b32 v0, v12338; GFX6-NEXT: s_endpgm2339;2340; GFX7-LABEL: local_workgroup_acquire_ret_atomicrmw:2341; GFX7: ; %bb.0: ; %entry2342; GFX7-NEXT: s_load_dword s4, s[8:9], 0x02343; GFX7-NEXT: s_load_dword s5, s[8:9], 0x12344; GFX7-NEXT: s_mov_b32 m0, -12345; GFX7-NEXT: s_waitcnt lgkmcnt(0)2346; GFX7-NEXT: v_mov_b32_e32 v0, s42347; GFX7-NEXT: v_mov_b32_e32 v1, s52348; GFX7-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12349; GFX7-NEXT: s_waitcnt lgkmcnt(0)2350; GFX7-NEXT: s_mov_b32 m0, -12351; GFX7-NEXT: v_mov_b32_e32 v0, s42352; GFX7-NEXT: ds_write_b32 v0, v12353; GFX7-NEXT: s_endpgm2354;2355; GFX10-WGP-LABEL: local_workgroup_acquire_ret_atomicrmw:2356; GFX10-WGP: ; %bb.0: ; %entry2357; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x02358; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x42359; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)2360; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s42361; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s52362; GFX10-WGP-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12363; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)2364; GFX10-WGP-NEXT: buffer_gl0_inv2365; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s42366; GFX10-WGP-NEXT: ds_write_b32 v0, v12367; GFX10-WGP-NEXT: s_endpgm2368;2369; GFX10-CU-LABEL: local_workgroup_acquire_ret_atomicrmw:2370; GFX10-CU: ; %bb.0: ; %entry2371; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x02372; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x42373; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)2374; GFX10-CU-NEXT: v_mov_b32_e32 v0, s42375; GFX10-CU-NEXT: v_mov_b32_e32 v1, s52376; GFX10-CU-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12377; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)2378; GFX10-CU-NEXT: v_mov_b32_e32 v0, s42379; GFX10-CU-NEXT: ds_write_b32 v0, v12380; GFX10-CU-NEXT: s_endpgm2381;2382; SKIP-CACHE-INV-LABEL: local_workgroup_acquire_ret_atomicrmw:2383; SKIP-CACHE-INV: ; %bb.0: ; %entry2384; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x02385; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x12386; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -12387; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)2388; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s02389; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s12390; SKIP-CACHE-INV-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12391; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)2392; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -12393; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s02394; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v12395; SKIP-CACHE-INV-NEXT: s_endpgm2396;2397; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_acquire_ret_atomicrmw:2398; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry2399; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x02400; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x42401; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2402; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s42403; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s52404; GFX90A-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12405; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2406; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s42407; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v12408; GFX90A-NOTTGSPLIT-NEXT: s_endpgm2409;2410; GFX90A-TGSPLIT-LABEL: local_workgroup_acquire_ret_atomicrmw:2411; GFX90A-TGSPLIT: ; %bb.0: ; %entry2412; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x02413; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x42414; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2415; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s42416; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s52417; GFX90A-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12418; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol2419; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s42420; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2421; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v12422; GFX90A-TGSPLIT-NEXT: s_endpgm2423;2424; GFX942-NOTTGSPLIT-LABEL: local_workgroup_acquire_ret_atomicrmw:2425; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry2426; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x02427; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x42428; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2429; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s02430; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s12431; GFX942-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12432; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2433; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s02434; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v12435; GFX942-NOTTGSPLIT-NEXT: s_endpgm2436;2437; GFX942-TGSPLIT-LABEL: local_workgroup_acquire_ret_atomicrmw:2438; GFX942-TGSPLIT: ; %bb.0: ; %entry2439; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x02440; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x42441; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2442; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s02443; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s12444; GFX942-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12445; GFX942-TGSPLIT-NEXT: buffer_inv sc02446; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s02447; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2448; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v12449; GFX942-TGSPLIT-NEXT: s_endpgm2450;2451; GFX11-WGP-LABEL: local_workgroup_acquire_ret_atomicrmw:2452; GFX11-WGP: ; %bb.0: ; %entry2453; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x02454; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x42455; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)2456; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s02457; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s12458; GFX11-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v12459; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)2460; GFX11-WGP-NEXT: buffer_gl0_inv2461; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s02462; GFX11-WGP-NEXT: ds_store_b32 v0, v12463; GFX11-WGP-NEXT: s_endpgm2464;2465; GFX11-CU-LABEL: local_workgroup_acquire_ret_atomicrmw:2466; GFX11-CU: ; %bb.0: ; %entry2467; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x02468; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x42469; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)2470; GFX11-CU-NEXT: v_mov_b32_e32 v0, s02471; GFX11-CU-NEXT: v_mov_b32_e32 v1, s12472; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v12473; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)2474; GFX11-CU-NEXT: v_mov_b32_e32 v0, s02475; GFX11-CU-NEXT: ds_store_b32 v0, v12476; GFX11-CU-NEXT: s_endpgm2477;2478; GFX12-WGP-LABEL: local_workgroup_acquire_ret_atomicrmw:2479; GFX12-WGP: ; %bb.0: ; %entry2480; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x02481; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x42482; GFX12-WGP-NEXT: s_wait_kmcnt 0x02483; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s02484; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s12485; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v12486; GFX12-WGP-NEXT: s_wait_dscnt 0x02487; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE2488; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s02489; GFX12-WGP-NEXT: ds_store_b32 v0, v12490; GFX12-WGP-NEXT: s_endpgm2491;2492; GFX12-CU-LABEL: local_workgroup_acquire_ret_atomicrmw:2493; GFX12-CU: ; %bb.0: ; %entry2494; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x02495; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x42496; GFX12-CU-NEXT: s_wait_kmcnt 0x02497; GFX12-CU-NEXT: v_mov_b32_e32 v0, s02498; GFX12-CU-NEXT: v_mov_b32_e32 v1, s12499; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v12500; GFX12-CU-NEXT: s_wait_dscnt 0x02501; GFX12-CU-NEXT: v_mov_b32_e32 v0, s02502; GFX12-CU-NEXT: ds_store_b32 v0, v12503; GFX12-CU-NEXT: s_endpgm2504;2505; GFX1250-LABEL: local_workgroup_acquire_ret_atomicrmw:2506; GFX1250: ; %bb.0: ; %entry2507; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12508; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x02509; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x42510; GFX1250-NEXT: s_wait_kmcnt 0x02511; GFX1250-NEXT: v_mov_b32_e32 v0, s02512; GFX1250-NEXT: v_mov_b32_e32 v1, s12513; GFX1250-NEXT: ds_storexchg_rtn_b32 v1, v0, v12514; GFX1250-NEXT: s_wait_dscnt 0x02515; GFX1250-NEXT: v_mov_b32_e32 v0, s02516; GFX1250-NEXT: ds_store_b32 v0, v12517; GFX1250-NEXT: s_endpgm2518 ptr addrspace(3) %out, i32 %in) {2519entry:2520 %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup") acquire2521 store i32 %val, ptr addrspace(3) %out, align 42522 ret void2523}2524 2525define amdgpu_kernel void @local_workgroup_acq_rel_ret_atomicrmw(2526; GFX6-LABEL: local_workgroup_acq_rel_ret_atomicrmw:2527; GFX6: ; %bb.0: ; %entry2528; GFX6-NEXT: s_load_dword s4, s[8:9], 0x02529; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr42530; GFX6-NEXT: s_load_dword s5, s[8:9], 0x12531; GFX6-NEXT: s_mov_b32 m0, -12532; GFX6-NEXT: s_waitcnt lgkmcnt(0)2533; GFX6-NEXT: v_mov_b32_e32 v0, s42534; GFX6-NEXT: v_mov_b32_e32 v1, s52535; GFX6-NEXT: s_waitcnt lgkmcnt(0)2536; GFX6-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12537; GFX6-NEXT: s_waitcnt lgkmcnt(0)2538; GFX6-NEXT: s_mov_b32 m0, -12539; GFX6-NEXT: v_mov_b32_e32 v0, s42540; GFX6-NEXT: ds_write_b32 v0, v12541; GFX6-NEXT: s_endpgm2542;2543; GFX7-LABEL: local_workgroup_acq_rel_ret_atomicrmw:2544; GFX7: ; %bb.0: ; %entry2545; GFX7-NEXT: s_load_dword s4, s[8:9], 0x02546; GFX7-NEXT: s_load_dword s5, s[8:9], 0x12547; GFX7-NEXT: s_mov_b32 m0, -12548; GFX7-NEXT: s_waitcnt lgkmcnt(0)2549; GFX7-NEXT: v_mov_b32_e32 v0, s42550; GFX7-NEXT: v_mov_b32_e32 v1, s52551; GFX7-NEXT: s_waitcnt lgkmcnt(0)2552; GFX7-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12553; GFX7-NEXT: s_waitcnt lgkmcnt(0)2554; GFX7-NEXT: s_mov_b32 m0, -12555; GFX7-NEXT: v_mov_b32_e32 v0, s42556; GFX7-NEXT: ds_write_b32 v0, v12557; GFX7-NEXT: s_endpgm2558;2559; GFX10-WGP-LABEL: local_workgroup_acq_rel_ret_atomicrmw:2560; GFX10-WGP: ; %bb.0: ; %entry2561; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x02562; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x42563; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)2564; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s42565; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s52566; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2567; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x02568; GFX10-WGP-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12569; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)2570; GFX10-WGP-NEXT: buffer_gl0_inv2571; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s42572; GFX10-WGP-NEXT: ds_write_b32 v0, v12573; GFX10-WGP-NEXT: s_endpgm2574;2575; GFX10-CU-LABEL: local_workgroup_acq_rel_ret_atomicrmw:2576; GFX10-CU: ; %bb.0: ; %entry2577; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x02578; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x42579; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)2580; GFX10-CU-NEXT: v_mov_b32_e32 v0, s42581; GFX10-CU-NEXT: v_mov_b32_e32 v1, s52582; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2583; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x02584; GFX10-CU-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12585; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)2586; GFX10-CU-NEXT: v_mov_b32_e32 v0, s42587; GFX10-CU-NEXT: ds_write_b32 v0, v12588; GFX10-CU-NEXT: s_endpgm2589;2590; SKIP-CACHE-INV-LABEL: local_workgroup_acq_rel_ret_atomicrmw:2591; SKIP-CACHE-INV: ; %bb.0: ; %entry2592; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x02593; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x12594; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -12595; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)2596; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s02597; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s12598; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)2599; SKIP-CACHE-INV-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12600; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)2601; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -12602; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s02603; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v12604; SKIP-CACHE-INV-NEXT: s_endpgm2605;2606; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_acq_rel_ret_atomicrmw:2607; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry2608; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x02609; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x42610; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2611; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s42612; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s52613; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2614; GFX90A-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12615; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2616; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s42617; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v12618; GFX90A-NOTTGSPLIT-NEXT: s_endpgm2619;2620; GFX90A-TGSPLIT-LABEL: local_workgroup_acq_rel_ret_atomicrmw:2621; GFX90A-TGSPLIT: ; %bb.0: ; %entry2622; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x02623; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x42624; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2625; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s42626; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s52627; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2628; GFX90A-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12629; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol2630; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s42631; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2632; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v12633; GFX90A-TGSPLIT-NEXT: s_endpgm2634;2635; GFX942-NOTTGSPLIT-LABEL: local_workgroup_acq_rel_ret_atomicrmw:2636; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry2637; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x02638; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x42639; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2640; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s02641; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s12642; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2643; GFX942-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12644; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2645; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s02646; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v12647; GFX942-NOTTGSPLIT-NEXT: s_endpgm2648;2649; GFX942-TGSPLIT-LABEL: local_workgroup_acq_rel_ret_atomicrmw:2650; GFX942-TGSPLIT: ; %bb.0: ; %entry2651; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x02652; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x42653; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2654; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s02655; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s12656; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2657; GFX942-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12658; GFX942-TGSPLIT-NEXT: buffer_inv sc02659; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s02660; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2661; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v12662; GFX942-TGSPLIT-NEXT: s_endpgm2663;2664; GFX11-WGP-LABEL: local_workgroup_acq_rel_ret_atomicrmw:2665; GFX11-WGP: ; %bb.0: ; %entry2666; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x02667; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x42668; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)2669; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s02670; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s12671; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2672; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x02673; GFX11-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v12674; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)2675; GFX11-WGP-NEXT: buffer_gl0_inv2676; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s02677; GFX11-WGP-NEXT: ds_store_b32 v0, v12678; GFX11-WGP-NEXT: s_endpgm2679;2680; GFX11-CU-LABEL: local_workgroup_acq_rel_ret_atomicrmw:2681; GFX11-CU: ; %bb.0: ; %entry2682; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x02683; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x42684; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)2685; GFX11-CU-NEXT: v_mov_b32_e32 v0, s02686; GFX11-CU-NEXT: v_mov_b32_e32 v1, s12687; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2688; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x02689; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v12690; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)2691; GFX11-CU-NEXT: v_mov_b32_e32 v0, s02692; GFX11-CU-NEXT: ds_store_b32 v0, v12693; GFX11-CU-NEXT: s_endpgm2694;2695; GFX12-WGP-LABEL: local_workgroup_acq_rel_ret_atomicrmw:2696; GFX12-WGP: ; %bb.0: ; %entry2697; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x02698; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x42699; GFX12-WGP-NEXT: s_wait_kmcnt 0x02700; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s02701; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s12702; GFX12-WGP-NEXT: s_wait_bvhcnt 0x02703; GFX12-WGP-NEXT: s_wait_samplecnt 0x02704; GFX12-WGP-NEXT: s_wait_storecnt 0x02705; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x02706; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v12707; GFX12-WGP-NEXT: s_wait_dscnt 0x02708; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE2709; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s02710; GFX12-WGP-NEXT: ds_store_b32 v0, v12711; GFX12-WGP-NEXT: s_endpgm2712;2713; GFX12-CU-LABEL: local_workgroup_acq_rel_ret_atomicrmw:2714; GFX12-CU: ; %bb.0: ; %entry2715; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x02716; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x42717; GFX12-CU-NEXT: s_wait_kmcnt 0x02718; GFX12-CU-NEXT: v_mov_b32_e32 v0, s02719; GFX12-CU-NEXT: v_mov_b32_e32 v1, s12720; GFX12-CU-NEXT: s_wait_bvhcnt 0x02721; GFX12-CU-NEXT: s_wait_samplecnt 0x02722; GFX12-CU-NEXT: s_wait_storecnt 0x02723; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x02724; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v12725; GFX12-CU-NEXT: s_wait_dscnt 0x02726; GFX12-CU-NEXT: v_mov_b32_e32 v0, s02727; GFX12-CU-NEXT: ds_store_b32 v0, v12728; GFX12-CU-NEXT: s_endpgm2729;2730; GFX1250-LABEL: local_workgroup_acq_rel_ret_atomicrmw:2731; GFX1250: ; %bb.0: ; %entry2732; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12733; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x02734; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x42735; GFX1250-NEXT: s_wait_kmcnt 0x02736; GFX1250-NEXT: v_mov_b32_e32 v0, s02737; GFX1250-NEXT: v_mov_b32_e32 v1, s12738; GFX1250-NEXT: s_wait_storecnt 0x02739; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x02740; GFX1250-NEXT: ds_storexchg_rtn_b32 v1, v0, v12741; GFX1250-NEXT: s_wait_dscnt 0x02742; GFX1250-NEXT: v_mov_b32_e32 v0, s02743; GFX1250-NEXT: ds_store_b32 v0, v12744; GFX1250-NEXT: s_endpgm2745 ptr addrspace(3) %out, i32 %in) {2746entry:2747 %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup") acq_rel2748 store i32 %val, ptr addrspace(3) %out, align 42749 ret void2750}2751 2752define amdgpu_kernel void @local_workgroup_seq_cst_ret_atomicrmw(2753; GFX6-LABEL: local_workgroup_seq_cst_ret_atomicrmw:2754; GFX6: ; %bb.0: ; %entry2755; GFX6-NEXT: s_load_dword s4, s[8:9], 0x02756; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr42757; GFX6-NEXT: s_load_dword s5, s[8:9], 0x12758; GFX6-NEXT: s_mov_b32 m0, -12759; GFX6-NEXT: s_waitcnt lgkmcnt(0)2760; GFX6-NEXT: v_mov_b32_e32 v0, s42761; GFX6-NEXT: v_mov_b32_e32 v1, s52762; GFX6-NEXT: s_waitcnt lgkmcnt(0)2763; GFX6-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12764; GFX6-NEXT: s_waitcnt lgkmcnt(0)2765; GFX6-NEXT: s_mov_b32 m0, -12766; GFX6-NEXT: v_mov_b32_e32 v0, s42767; GFX6-NEXT: ds_write_b32 v0, v12768; GFX6-NEXT: s_endpgm2769;2770; GFX7-LABEL: local_workgroup_seq_cst_ret_atomicrmw:2771; GFX7: ; %bb.0: ; %entry2772; GFX7-NEXT: s_load_dword s4, s[8:9], 0x02773; GFX7-NEXT: s_load_dword s5, s[8:9], 0x12774; GFX7-NEXT: s_mov_b32 m0, -12775; GFX7-NEXT: s_waitcnt lgkmcnt(0)2776; GFX7-NEXT: v_mov_b32_e32 v0, s42777; GFX7-NEXT: v_mov_b32_e32 v1, s52778; GFX7-NEXT: s_waitcnt lgkmcnt(0)2779; GFX7-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12780; GFX7-NEXT: s_waitcnt lgkmcnt(0)2781; GFX7-NEXT: s_mov_b32 m0, -12782; GFX7-NEXT: v_mov_b32_e32 v0, s42783; GFX7-NEXT: ds_write_b32 v0, v12784; GFX7-NEXT: s_endpgm2785;2786; GFX10-WGP-LABEL: local_workgroup_seq_cst_ret_atomicrmw:2787; GFX10-WGP: ; %bb.0: ; %entry2788; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x02789; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x42790; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)2791; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s42792; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s52793; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2794; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x02795; GFX10-WGP-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12796; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)2797; GFX10-WGP-NEXT: buffer_gl0_inv2798; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s42799; GFX10-WGP-NEXT: ds_write_b32 v0, v12800; GFX10-WGP-NEXT: s_endpgm2801;2802; GFX10-CU-LABEL: local_workgroup_seq_cst_ret_atomicrmw:2803; GFX10-CU: ; %bb.0: ; %entry2804; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x02805; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x42806; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)2807; GFX10-CU-NEXT: v_mov_b32_e32 v0, s42808; GFX10-CU-NEXT: v_mov_b32_e32 v1, s52809; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2810; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x02811; GFX10-CU-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12812; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)2813; GFX10-CU-NEXT: v_mov_b32_e32 v0, s42814; GFX10-CU-NEXT: ds_write_b32 v0, v12815; GFX10-CU-NEXT: s_endpgm2816;2817; SKIP-CACHE-INV-LABEL: local_workgroup_seq_cst_ret_atomicrmw:2818; SKIP-CACHE-INV: ; %bb.0: ; %entry2819; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x02820; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x12821; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -12822; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)2823; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s02824; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s12825; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)2826; SKIP-CACHE-INV-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12827; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)2828; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -12829; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s02830; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v12831; SKIP-CACHE-INV-NEXT: s_endpgm2832;2833; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_seq_cst_ret_atomicrmw:2834; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry2835; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x02836; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x42837; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2838; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s42839; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s52840; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2841; GFX90A-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12842; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2843; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s42844; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v12845; GFX90A-NOTTGSPLIT-NEXT: s_endpgm2846;2847; GFX90A-TGSPLIT-LABEL: local_workgroup_seq_cst_ret_atomicrmw:2848; GFX90A-TGSPLIT: ; %bb.0: ; %entry2849; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x02850; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x42851; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2852; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s42853; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s52854; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2855; GFX90A-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12856; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol2857; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s42858; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2859; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v12860; GFX90A-TGSPLIT-NEXT: s_endpgm2861;2862; GFX942-NOTTGSPLIT-LABEL: local_workgroup_seq_cst_ret_atomicrmw:2863; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry2864; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x02865; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x42866; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2867; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s02868; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s12869; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2870; GFX942-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12871; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2872; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s02873; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v12874; GFX942-NOTTGSPLIT-NEXT: s_endpgm2875;2876; GFX942-TGSPLIT-LABEL: local_workgroup_seq_cst_ret_atomicrmw:2877; GFX942-TGSPLIT: ; %bb.0: ; %entry2878; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x02879; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x42880; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2881; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s02882; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s12883; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2884; GFX942-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v1, v0, v12885; GFX942-TGSPLIT-NEXT: buffer_inv sc02886; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s02887; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)2888; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v12889; GFX942-TGSPLIT-NEXT: s_endpgm2890;2891; GFX11-WGP-LABEL: local_workgroup_seq_cst_ret_atomicrmw:2892; GFX11-WGP: ; %bb.0: ; %entry2893; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x02894; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x42895; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)2896; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s02897; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s12898; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2899; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x02900; GFX11-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v12901; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)2902; GFX11-WGP-NEXT: buffer_gl0_inv2903; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s02904; GFX11-WGP-NEXT: ds_store_b32 v0, v12905; GFX11-WGP-NEXT: s_endpgm2906;2907; GFX11-CU-LABEL: local_workgroup_seq_cst_ret_atomicrmw:2908; GFX11-CU: ; %bb.0: ; %entry2909; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x02910; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x42911; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)2912; GFX11-CU-NEXT: v_mov_b32_e32 v0, s02913; GFX11-CU-NEXT: v_mov_b32_e32 v1, s12914; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2915; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x02916; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v12917; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)2918; GFX11-CU-NEXT: v_mov_b32_e32 v0, s02919; GFX11-CU-NEXT: ds_store_b32 v0, v12920; GFX11-CU-NEXT: s_endpgm2921;2922; GFX12-WGP-LABEL: local_workgroup_seq_cst_ret_atomicrmw:2923; GFX12-WGP: ; %bb.0: ; %entry2924; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x02925; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x42926; GFX12-WGP-NEXT: s_wait_kmcnt 0x02927; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s02928; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s12929; GFX12-WGP-NEXT: s_wait_bvhcnt 0x02930; GFX12-WGP-NEXT: s_wait_samplecnt 0x02931; GFX12-WGP-NEXT: s_wait_storecnt 0x02932; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x02933; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v12934; GFX12-WGP-NEXT: s_wait_dscnt 0x02935; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE2936; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s02937; GFX12-WGP-NEXT: ds_store_b32 v0, v12938; GFX12-WGP-NEXT: s_endpgm2939;2940; GFX12-CU-LABEL: local_workgroup_seq_cst_ret_atomicrmw:2941; GFX12-CU: ; %bb.0: ; %entry2942; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x02943; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x42944; GFX12-CU-NEXT: s_wait_kmcnt 0x02945; GFX12-CU-NEXT: v_mov_b32_e32 v0, s02946; GFX12-CU-NEXT: v_mov_b32_e32 v1, s12947; GFX12-CU-NEXT: s_wait_bvhcnt 0x02948; GFX12-CU-NEXT: s_wait_samplecnt 0x02949; GFX12-CU-NEXT: s_wait_storecnt 0x02950; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x02951; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v12952; GFX12-CU-NEXT: s_wait_dscnt 0x02953; GFX12-CU-NEXT: v_mov_b32_e32 v0, s02954; GFX12-CU-NEXT: ds_store_b32 v0, v12955; GFX12-CU-NEXT: s_endpgm2956;2957; GFX1250-LABEL: local_workgroup_seq_cst_ret_atomicrmw:2958; GFX1250: ; %bb.0: ; %entry2959; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12960; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x02961; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x42962; GFX1250-NEXT: s_wait_kmcnt 0x02963; GFX1250-NEXT: v_mov_b32_e32 v0, s02964; GFX1250-NEXT: v_mov_b32_e32 v1, s12965; GFX1250-NEXT: s_wait_storecnt 0x02966; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x02967; GFX1250-NEXT: ds_storexchg_rtn_b32 v1, v0, v12968; GFX1250-NEXT: s_wait_dscnt 0x02969; GFX1250-NEXT: v_mov_b32_e32 v0, s02970; GFX1250-NEXT: ds_store_b32 v0, v12971; GFX1250-NEXT: s_endpgm2972 ptr addrspace(3) %out, i32 %in) {2973entry:2974 %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup") seq_cst2975 store i32 %val, ptr addrspace(3) %out, align 42976 ret void2977}2978 2979define amdgpu_kernel void @local_workgroup_monotonic_monotonic_cmpxchg(2980; GFX6-LABEL: local_workgroup_monotonic_monotonic_cmpxchg:2981; GFX6: ; %bb.0: ; %entry2982; GFX6-NEXT: s_load_dword s6, s[8:9], 0x02983; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr62984; GFX6-NEXT: s_load_dword s4, s[8:9], 0x12985; GFX6-NEXT: s_load_dword s5, s[8:9], 0x22986; GFX6-NEXT: s_mov_b32 m0, -12987; GFX6-NEXT: s_waitcnt lgkmcnt(0)2988; GFX6-NEXT: v_mov_b32_e32 v0, s62989; GFX6-NEXT: v_mov_b32_e32 v1, s52990; GFX6-NEXT: v_mov_b32_e32 v2, s42991; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:162992; GFX6-NEXT: s_endpgm2993;2994; GFX7-LABEL: local_workgroup_monotonic_monotonic_cmpxchg:2995; GFX7: ; %bb.0: ; %entry2996; GFX7-NEXT: s_load_dword s6, s[8:9], 0x02997; GFX7-NEXT: s_load_dword s4, s[8:9], 0x12998; GFX7-NEXT: s_load_dword s5, s[8:9], 0x22999; GFX7-NEXT: s_mov_b32 m0, -13000; GFX7-NEXT: s_waitcnt lgkmcnt(0)3001; GFX7-NEXT: v_mov_b32_e32 v0, s63002; GFX7-NEXT: v_mov_b32_e32 v1, s53003; GFX7-NEXT: v_mov_b32_e32 v2, s43004; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163005; GFX7-NEXT: s_endpgm3006;3007; GFX10-WGP-LABEL: local_workgroup_monotonic_monotonic_cmpxchg:3008; GFX10-WGP: ; %bb.0: ; %entry3009; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x03010; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x43011; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x83012; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)3013; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s63014; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s53015; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s43016; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163017; GFX10-WGP-NEXT: s_endpgm3018;3019; GFX10-CU-LABEL: local_workgroup_monotonic_monotonic_cmpxchg:3020; GFX10-CU: ; %bb.0: ; %entry3021; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x03022; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x43023; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x83024; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)3025; GFX10-CU-NEXT: v_mov_b32_e32 v0, s63026; GFX10-CU-NEXT: v_mov_b32_e32 v1, s53027; GFX10-CU-NEXT: v_mov_b32_e32 v2, s43028; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163029; GFX10-CU-NEXT: s_endpgm3030;3031; SKIP-CACHE-INV-LABEL: local_workgroup_monotonic_monotonic_cmpxchg:3032; SKIP-CACHE-INV: ; %bb.0: ; %entry3033; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x03034; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x13035; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x23036; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -13037; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)3038; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s23039; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s13040; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s03041; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163042; SKIP-CACHE-INV-NEXT: s_endpgm3043;3044; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_monotonic_monotonic_cmpxchg:3045; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry3046; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x03047; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x43048; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x83049; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3050; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s63051; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s53052; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s43053; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163054; GFX90A-NOTTGSPLIT-NEXT: s_endpgm3055;3056; GFX90A-TGSPLIT-LABEL: local_workgroup_monotonic_monotonic_cmpxchg:3057; GFX90A-TGSPLIT: ; %bb.0: ; %entry3058; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x03059; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x43060; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x83061; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3062; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s63063; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s53064; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s43065; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163066; GFX90A-TGSPLIT-NEXT: s_endpgm3067;3068; GFX942-NOTTGSPLIT-LABEL: local_workgroup_monotonic_monotonic_cmpxchg:3069; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry3070; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x03071; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x43072; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x83073; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3074; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s23075; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s13076; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s03077; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163078; GFX942-NOTTGSPLIT-NEXT: s_endpgm3079;3080; GFX942-TGSPLIT-LABEL: local_workgroup_monotonic_monotonic_cmpxchg:3081; GFX942-TGSPLIT: ; %bb.0: ; %entry3082; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x03083; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x43084; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x83085; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3086; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s23087; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s13088; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s03089; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163090; GFX942-TGSPLIT-NEXT: s_endpgm3091;3092; GFX11-WGP-LABEL: local_workgroup_monotonic_monotonic_cmpxchg:3093; GFX11-WGP: ; %bb.0: ; %entry3094; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x03095; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x43096; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x83097; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)3098; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s23099; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s13100; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s03101; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163102; GFX11-WGP-NEXT: s_endpgm3103;3104; GFX11-CU-LABEL: local_workgroup_monotonic_monotonic_cmpxchg:3105; GFX11-CU: ; %bb.0: ; %entry3106; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x03107; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x43108; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x83109; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)3110; GFX11-CU-NEXT: v_mov_b32_e32 v0, s23111; GFX11-CU-NEXT: v_mov_b32_e32 v1, s13112; GFX11-CU-NEXT: v_mov_b32_e32 v2, s03113; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163114; GFX11-CU-NEXT: s_endpgm3115;3116; GFX12-WGP-LABEL: local_workgroup_monotonic_monotonic_cmpxchg:3117; GFX12-WGP: ; %bb.0: ; %entry3118; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x03119; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x43120; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x83121; GFX12-WGP-NEXT: s_wait_kmcnt 0x03122; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s23123; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s13124; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s03125; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163126; GFX12-WGP-NEXT: s_endpgm3127;3128; GFX12-CU-LABEL: local_workgroup_monotonic_monotonic_cmpxchg:3129; GFX12-CU: ; %bb.0: ; %entry3130; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x03131; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x43132; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x83133; GFX12-CU-NEXT: s_wait_kmcnt 0x03134; GFX12-CU-NEXT: v_mov_b32_e32 v0, s23135; GFX12-CU-NEXT: v_mov_b32_e32 v1, s13136; GFX12-CU-NEXT: v_mov_b32_e32 v2, s03137; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163138; GFX12-CU-NEXT: s_endpgm3139;3140; GFX1250-LABEL: local_workgroup_monotonic_monotonic_cmpxchg:3141; GFX1250: ; %bb.0: ; %entry3142; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 13143; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x03144; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x43145; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x83146; GFX1250-NEXT: s_wait_kmcnt 0x03147; GFX1250-NEXT: v_mov_b32_e32 v0, s23148; GFX1250-NEXT: v_mov_b32_e32 v1, s13149; GFX1250-NEXT: v_mov_b32_e32 v2, s03150; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163151; GFX1250-NEXT: s_endpgm3152 ptr addrspace(3) %out, i32 %in, i32 %old) {3153entry:3154 %gep = getelementptr i32, ptr addrspace(3) %out, i32 43155 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") monotonic monotonic3156 ret void3157}3158 3159define amdgpu_kernel void @local_workgroup_acquire_monotonic_cmpxchg(3160; GFX6-LABEL: local_workgroup_acquire_monotonic_cmpxchg:3161; GFX6: ; %bb.0: ; %entry3162; GFX6-NEXT: s_load_dword s6, s[8:9], 0x03163; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr63164; GFX6-NEXT: s_load_dword s4, s[8:9], 0x13165; GFX6-NEXT: s_load_dword s5, s[8:9], 0x23166; GFX6-NEXT: s_mov_b32 m0, -13167; GFX6-NEXT: s_waitcnt lgkmcnt(0)3168; GFX6-NEXT: v_mov_b32_e32 v0, s63169; GFX6-NEXT: v_mov_b32_e32 v1, s53170; GFX6-NEXT: v_mov_b32_e32 v2, s43171; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163172; GFX6-NEXT: s_waitcnt lgkmcnt(0)3173; GFX6-NEXT: s_endpgm3174;3175; GFX7-LABEL: local_workgroup_acquire_monotonic_cmpxchg:3176; GFX7: ; %bb.0: ; %entry3177; GFX7-NEXT: s_load_dword s6, s[8:9], 0x03178; GFX7-NEXT: s_load_dword s4, s[8:9], 0x13179; GFX7-NEXT: s_load_dword s5, s[8:9], 0x23180; GFX7-NEXT: s_mov_b32 m0, -13181; GFX7-NEXT: s_waitcnt lgkmcnt(0)3182; GFX7-NEXT: v_mov_b32_e32 v0, s63183; GFX7-NEXT: v_mov_b32_e32 v1, s53184; GFX7-NEXT: v_mov_b32_e32 v2, s43185; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163186; GFX7-NEXT: s_waitcnt lgkmcnt(0)3187; GFX7-NEXT: s_endpgm3188;3189; GFX10-WGP-LABEL: local_workgroup_acquire_monotonic_cmpxchg:3190; GFX10-WGP: ; %bb.0: ; %entry3191; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x03192; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x43193; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x83194; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)3195; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s63196; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s53197; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s43198; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163199; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)3200; GFX10-WGP-NEXT: buffer_gl0_inv3201; GFX10-WGP-NEXT: s_endpgm3202;3203; GFX10-CU-LABEL: local_workgroup_acquire_monotonic_cmpxchg:3204; GFX10-CU: ; %bb.0: ; %entry3205; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x03206; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x43207; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x83208; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)3209; GFX10-CU-NEXT: v_mov_b32_e32 v0, s63210; GFX10-CU-NEXT: v_mov_b32_e32 v1, s53211; GFX10-CU-NEXT: v_mov_b32_e32 v2, s43212; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163213; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)3214; GFX10-CU-NEXT: s_endpgm3215;3216; SKIP-CACHE-INV-LABEL: local_workgroup_acquire_monotonic_cmpxchg:3217; SKIP-CACHE-INV: ; %bb.0: ; %entry3218; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x03219; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x13220; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x23221; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -13222; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)3223; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s23224; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s13225; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s03226; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163227; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)3228; SKIP-CACHE-INV-NEXT: s_endpgm3229;3230; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_acquire_monotonic_cmpxchg:3231; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry3232; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x03233; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x43234; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x83235; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3236; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s63237; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s53238; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s43239; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163240; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3241; GFX90A-NOTTGSPLIT-NEXT: s_endpgm3242;3243; GFX90A-TGSPLIT-LABEL: local_workgroup_acquire_monotonic_cmpxchg:3244; GFX90A-TGSPLIT: ; %bb.0: ; %entry3245; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x03246; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x43247; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x83248; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3249; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s63250; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s53251; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s43252; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163253; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol3254; GFX90A-TGSPLIT-NEXT: s_endpgm3255;3256; GFX942-NOTTGSPLIT-LABEL: local_workgroup_acquire_monotonic_cmpxchg:3257; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry3258; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x03259; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x43260; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x83261; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3262; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s23263; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s13264; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s03265; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163266; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3267; GFX942-NOTTGSPLIT-NEXT: s_endpgm3268;3269; GFX942-TGSPLIT-LABEL: local_workgroup_acquire_monotonic_cmpxchg:3270; GFX942-TGSPLIT: ; %bb.0: ; %entry3271; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x03272; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x43273; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x83274; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3275; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s23276; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s13277; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s03278; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163279; GFX942-TGSPLIT-NEXT: buffer_inv sc03280; GFX942-TGSPLIT-NEXT: s_endpgm3281;3282; GFX11-WGP-LABEL: local_workgroup_acquire_monotonic_cmpxchg:3283; GFX11-WGP: ; %bb.0: ; %entry3284; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x03285; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x43286; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x83287; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)3288; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s23289; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s13290; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s03291; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163292; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)3293; GFX11-WGP-NEXT: buffer_gl0_inv3294; GFX11-WGP-NEXT: s_endpgm3295;3296; GFX11-CU-LABEL: local_workgroup_acquire_monotonic_cmpxchg:3297; GFX11-CU: ; %bb.0: ; %entry3298; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x03299; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x43300; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x83301; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)3302; GFX11-CU-NEXT: v_mov_b32_e32 v0, s23303; GFX11-CU-NEXT: v_mov_b32_e32 v1, s13304; GFX11-CU-NEXT: v_mov_b32_e32 v2, s03305; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163306; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)3307; GFX11-CU-NEXT: s_endpgm3308;3309; GFX12-WGP-LABEL: local_workgroup_acquire_monotonic_cmpxchg:3310; GFX12-WGP: ; %bb.0: ; %entry3311; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x03312; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x43313; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x83314; GFX12-WGP-NEXT: s_wait_kmcnt 0x03315; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s23316; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s13317; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s03318; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163319; GFX12-WGP-NEXT: s_wait_dscnt 0x03320; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE3321; GFX12-WGP-NEXT: s_endpgm3322;3323; GFX12-CU-LABEL: local_workgroup_acquire_monotonic_cmpxchg:3324; GFX12-CU: ; %bb.0: ; %entry3325; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x03326; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x43327; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x83328; GFX12-CU-NEXT: s_wait_kmcnt 0x03329; GFX12-CU-NEXT: v_mov_b32_e32 v0, s23330; GFX12-CU-NEXT: v_mov_b32_e32 v1, s13331; GFX12-CU-NEXT: v_mov_b32_e32 v2, s03332; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163333; GFX12-CU-NEXT: s_wait_dscnt 0x03334; GFX12-CU-NEXT: s_endpgm3335;3336; GFX1250-LABEL: local_workgroup_acquire_monotonic_cmpxchg:3337; GFX1250: ; %bb.0: ; %entry3338; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 13339; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x03340; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x43341; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x83342; GFX1250-NEXT: s_wait_kmcnt 0x03343; GFX1250-NEXT: v_mov_b32_e32 v0, s23344; GFX1250-NEXT: v_mov_b32_e32 v1, s13345; GFX1250-NEXT: v_mov_b32_e32 v2, s03346; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163347; GFX1250-NEXT: s_wait_dscnt 0x03348; GFX1250-NEXT: s_endpgm3349 ptr addrspace(3) %out, i32 %in, i32 %old) {3350entry:3351 %gep = getelementptr i32, ptr addrspace(3) %out, i32 43352 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic3353 ret void3354}3355 3356define amdgpu_kernel void @local_workgroup_release_monotonic_cmpxchg(3357; GFX6-LABEL: local_workgroup_release_monotonic_cmpxchg:3358; GFX6: ; %bb.0: ; %entry3359; GFX6-NEXT: s_load_dword s6, s[8:9], 0x03360; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr63361; GFX6-NEXT: s_load_dword s4, s[8:9], 0x13362; GFX6-NEXT: s_load_dword s5, s[8:9], 0x23363; GFX6-NEXT: s_mov_b32 m0, -13364; GFX6-NEXT: s_waitcnt lgkmcnt(0)3365; GFX6-NEXT: v_mov_b32_e32 v0, s63366; GFX6-NEXT: v_mov_b32_e32 v1, s53367; GFX6-NEXT: v_mov_b32_e32 v2, s43368; GFX6-NEXT: s_waitcnt lgkmcnt(0)3369; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163370; GFX6-NEXT: s_endpgm3371;3372; GFX7-LABEL: local_workgroup_release_monotonic_cmpxchg:3373; GFX7: ; %bb.0: ; %entry3374; GFX7-NEXT: s_load_dword s6, s[8:9], 0x03375; GFX7-NEXT: s_load_dword s4, s[8:9], 0x13376; GFX7-NEXT: s_load_dword s5, s[8:9], 0x23377; GFX7-NEXT: s_mov_b32 m0, -13378; GFX7-NEXT: s_waitcnt lgkmcnt(0)3379; GFX7-NEXT: v_mov_b32_e32 v0, s63380; GFX7-NEXT: v_mov_b32_e32 v1, s53381; GFX7-NEXT: v_mov_b32_e32 v2, s43382; GFX7-NEXT: s_waitcnt lgkmcnt(0)3383; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163384; GFX7-NEXT: s_endpgm3385;3386; GFX10-WGP-LABEL: local_workgroup_release_monotonic_cmpxchg:3387; GFX10-WGP: ; %bb.0: ; %entry3388; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x03389; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x43390; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x83391; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)3392; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s63393; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s53394; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s43395; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3396; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x03397; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163398; GFX10-WGP-NEXT: s_endpgm3399;3400; GFX10-CU-LABEL: local_workgroup_release_monotonic_cmpxchg:3401; GFX10-CU: ; %bb.0: ; %entry3402; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x03403; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x43404; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x83405; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)3406; GFX10-CU-NEXT: v_mov_b32_e32 v0, s63407; GFX10-CU-NEXT: v_mov_b32_e32 v1, s53408; GFX10-CU-NEXT: v_mov_b32_e32 v2, s43409; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3410; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x03411; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163412; GFX10-CU-NEXT: s_endpgm3413;3414; SKIP-CACHE-INV-LABEL: local_workgroup_release_monotonic_cmpxchg:3415; SKIP-CACHE-INV: ; %bb.0: ; %entry3416; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x03417; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x13418; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x23419; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -13420; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)3421; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s23422; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s13423; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s03424; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)3425; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163426; SKIP-CACHE-INV-NEXT: s_endpgm3427;3428; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_release_monotonic_cmpxchg:3429; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry3430; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x03431; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x43432; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x83433; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3434; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s63435; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s53436; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s43437; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3438; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163439; GFX90A-NOTTGSPLIT-NEXT: s_endpgm3440;3441; GFX90A-TGSPLIT-LABEL: local_workgroup_release_monotonic_cmpxchg:3442; GFX90A-TGSPLIT: ; %bb.0: ; %entry3443; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x03444; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x43445; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x83446; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3447; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s63448; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s53449; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s43450; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3451; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163452; GFX90A-TGSPLIT-NEXT: s_endpgm3453;3454; GFX942-NOTTGSPLIT-LABEL: local_workgroup_release_monotonic_cmpxchg:3455; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry3456; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x03457; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x43458; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x83459; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3460; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s23461; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s13462; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s03463; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3464; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163465; GFX942-NOTTGSPLIT-NEXT: s_endpgm3466;3467; GFX942-TGSPLIT-LABEL: local_workgroup_release_monotonic_cmpxchg:3468; GFX942-TGSPLIT: ; %bb.0: ; %entry3469; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x03470; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x43471; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x83472; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3473; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s23474; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s13475; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s03476; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3477; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163478; GFX942-TGSPLIT-NEXT: s_endpgm3479;3480; GFX11-WGP-LABEL: local_workgroup_release_monotonic_cmpxchg:3481; GFX11-WGP: ; %bb.0: ; %entry3482; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x03483; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x43484; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x83485; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)3486; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s23487; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s13488; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s03489; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3490; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x03491; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163492; GFX11-WGP-NEXT: s_endpgm3493;3494; GFX11-CU-LABEL: local_workgroup_release_monotonic_cmpxchg:3495; GFX11-CU: ; %bb.0: ; %entry3496; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x03497; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x43498; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x83499; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)3500; GFX11-CU-NEXT: v_mov_b32_e32 v0, s23501; GFX11-CU-NEXT: v_mov_b32_e32 v1, s13502; GFX11-CU-NEXT: v_mov_b32_e32 v2, s03503; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3504; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x03505; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163506; GFX11-CU-NEXT: s_endpgm3507;3508; GFX12-WGP-LABEL: local_workgroup_release_monotonic_cmpxchg:3509; GFX12-WGP: ; %bb.0: ; %entry3510; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x03511; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x43512; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x83513; GFX12-WGP-NEXT: s_wait_kmcnt 0x03514; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s23515; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s13516; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s03517; GFX12-WGP-NEXT: s_wait_bvhcnt 0x03518; GFX12-WGP-NEXT: s_wait_samplecnt 0x03519; GFX12-WGP-NEXT: s_wait_storecnt 0x03520; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x03521; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163522; GFX12-WGP-NEXT: s_endpgm3523;3524; GFX12-CU-LABEL: local_workgroup_release_monotonic_cmpxchg:3525; GFX12-CU: ; %bb.0: ; %entry3526; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x03527; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x43528; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x83529; GFX12-CU-NEXT: s_wait_kmcnt 0x03530; GFX12-CU-NEXT: v_mov_b32_e32 v0, s23531; GFX12-CU-NEXT: v_mov_b32_e32 v1, s13532; GFX12-CU-NEXT: v_mov_b32_e32 v2, s03533; GFX12-CU-NEXT: s_wait_bvhcnt 0x03534; GFX12-CU-NEXT: s_wait_samplecnt 0x03535; GFX12-CU-NEXT: s_wait_storecnt 0x03536; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x03537; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163538; GFX12-CU-NEXT: s_endpgm3539;3540; GFX1250-LABEL: local_workgroup_release_monotonic_cmpxchg:3541; GFX1250: ; %bb.0: ; %entry3542; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 13543; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x03544; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x43545; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x83546; GFX1250-NEXT: s_wait_kmcnt 0x03547; GFX1250-NEXT: v_mov_b32_e32 v0, s23548; GFX1250-NEXT: v_mov_b32_e32 v1, s13549; GFX1250-NEXT: v_mov_b32_e32 v2, s03550; GFX1250-NEXT: s_wait_storecnt 0x03551; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x03552; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163553; GFX1250-NEXT: s_endpgm3554 ptr addrspace(3) %out, i32 %in, i32 %old) {3555entry:3556 %gep = getelementptr i32, ptr addrspace(3) %out, i32 43557 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") release monotonic3558 ret void3559}3560 3561define amdgpu_kernel void @local_workgroup_acq_rel_monotonic_cmpxchg(3562; GFX6-LABEL: local_workgroup_acq_rel_monotonic_cmpxchg:3563; GFX6: ; %bb.0: ; %entry3564; GFX6-NEXT: s_load_dword s6, s[8:9], 0x03565; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr63566; GFX6-NEXT: s_load_dword s4, s[8:9], 0x13567; GFX6-NEXT: s_load_dword s5, s[8:9], 0x23568; GFX6-NEXT: s_mov_b32 m0, -13569; GFX6-NEXT: s_waitcnt lgkmcnt(0)3570; GFX6-NEXT: v_mov_b32_e32 v0, s63571; GFX6-NEXT: v_mov_b32_e32 v1, s53572; GFX6-NEXT: v_mov_b32_e32 v2, s43573; GFX6-NEXT: s_waitcnt lgkmcnt(0)3574; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163575; GFX6-NEXT: s_waitcnt lgkmcnt(0)3576; GFX6-NEXT: s_endpgm3577;3578; GFX7-LABEL: local_workgroup_acq_rel_monotonic_cmpxchg:3579; GFX7: ; %bb.0: ; %entry3580; GFX7-NEXT: s_load_dword s6, s[8:9], 0x03581; GFX7-NEXT: s_load_dword s4, s[8:9], 0x13582; GFX7-NEXT: s_load_dword s5, s[8:9], 0x23583; GFX7-NEXT: s_mov_b32 m0, -13584; GFX7-NEXT: s_waitcnt lgkmcnt(0)3585; GFX7-NEXT: v_mov_b32_e32 v0, s63586; GFX7-NEXT: v_mov_b32_e32 v1, s53587; GFX7-NEXT: v_mov_b32_e32 v2, s43588; GFX7-NEXT: s_waitcnt lgkmcnt(0)3589; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163590; GFX7-NEXT: s_waitcnt lgkmcnt(0)3591; GFX7-NEXT: s_endpgm3592;3593; GFX10-WGP-LABEL: local_workgroup_acq_rel_monotonic_cmpxchg:3594; GFX10-WGP: ; %bb.0: ; %entry3595; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x03596; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x43597; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x83598; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)3599; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s63600; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s53601; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s43602; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3603; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x03604; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163605; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)3606; GFX10-WGP-NEXT: buffer_gl0_inv3607; GFX10-WGP-NEXT: s_endpgm3608;3609; GFX10-CU-LABEL: local_workgroup_acq_rel_monotonic_cmpxchg:3610; GFX10-CU: ; %bb.0: ; %entry3611; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x03612; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x43613; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x83614; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)3615; GFX10-CU-NEXT: v_mov_b32_e32 v0, s63616; GFX10-CU-NEXT: v_mov_b32_e32 v1, s53617; GFX10-CU-NEXT: v_mov_b32_e32 v2, s43618; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3619; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x03620; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163621; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)3622; GFX10-CU-NEXT: s_endpgm3623;3624; SKIP-CACHE-INV-LABEL: local_workgroup_acq_rel_monotonic_cmpxchg:3625; SKIP-CACHE-INV: ; %bb.0: ; %entry3626; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x03627; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x13628; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x23629; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -13630; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)3631; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s23632; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s13633; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s03634; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)3635; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163636; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)3637; SKIP-CACHE-INV-NEXT: s_endpgm3638;3639; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_acq_rel_monotonic_cmpxchg:3640; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry3641; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x03642; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x43643; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x83644; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3645; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s63646; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s53647; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s43648; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3649; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163650; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3651; GFX90A-NOTTGSPLIT-NEXT: s_endpgm3652;3653; GFX90A-TGSPLIT-LABEL: local_workgroup_acq_rel_monotonic_cmpxchg:3654; GFX90A-TGSPLIT: ; %bb.0: ; %entry3655; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x03656; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x43657; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x83658; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3659; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s63660; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s53661; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s43662; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3663; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163664; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol3665; GFX90A-TGSPLIT-NEXT: s_endpgm3666;3667; GFX942-NOTTGSPLIT-LABEL: local_workgroup_acq_rel_monotonic_cmpxchg:3668; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry3669; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x03670; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x43671; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x83672; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3673; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s23674; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s13675; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s03676; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3677; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163678; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3679; GFX942-NOTTGSPLIT-NEXT: s_endpgm3680;3681; GFX942-TGSPLIT-LABEL: local_workgroup_acq_rel_monotonic_cmpxchg:3682; GFX942-TGSPLIT: ; %bb.0: ; %entry3683; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x03684; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x43685; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x83686; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3687; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s23688; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s13689; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s03690; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3691; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163692; GFX942-TGSPLIT-NEXT: buffer_inv sc03693; GFX942-TGSPLIT-NEXT: s_endpgm3694;3695; GFX11-WGP-LABEL: local_workgroup_acq_rel_monotonic_cmpxchg:3696; GFX11-WGP: ; %bb.0: ; %entry3697; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x03698; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x43699; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x83700; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)3701; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s23702; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s13703; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s03704; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3705; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x03706; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163707; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)3708; GFX11-WGP-NEXT: buffer_gl0_inv3709; GFX11-WGP-NEXT: s_endpgm3710;3711; GFX11-CU-LABEL: local_workgroup_acq_rel_monotonic_cmpxchg:3712; GFX11-CU: ; %bb.0: ; %entry3713; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x03714; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x43715; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x83716; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)3717; GFX11-CU-NEXT: v_mov_b32_e32 v0, s23718; GFX11-CU-NEXT: v_mov_b32_e32 v1, s13719; GFX11-CU-NEXT: v_mov_b32_e32 v2, s03720; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3721; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x03722; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163723; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)3724; GFX11-CU-NEXT: s_endpgm3725;3726; GFX12-WGP-LABEL: local_workgroup_acq_rel_monotonic_cmpxchg:3727; GFX12-WGP: ; %bb.0: ; %entry3728; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x03729; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x43730; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x83731; GFX12-WGP-NEXT: s_wait_kmcnt 0x03732; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s23733; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s13734; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s03735; GFX12-WGP-NEXT: s_wait_bvhcnt 0x03736; GFX12-WGP-NEXT: s_wait_samplecnt 0x03737; GFX12-WGP-NEXT: s_wait_storecnt 0x03738; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x03739; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163740; GFX12-WGP-NEXT: s_wait_dscnt 0x03741; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE3742; GFX12-WGP-NEXT: s_endpgm3743;3744; GFX12-CU-LABEL: local_workgroup_acq_rel_monotonic_cmpxchg:3745; GFX12-CU: ; %bb.0: ; %entry3746; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x03747; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x43748; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x83749; GFX12-CU-NEXT: s_wait_kmcnt 0x03750; GFX12-CU-NEXT: v_mov_b32_e32 v0, s23751; GFX12-CU-NEXT: v_mov_b32_e32 v1, s13752; GFX12-CU-NEXT: v_mov_b32_e32 v2, s03753; GFX12-CU-NEXT: s_wait_bvhcnt 0x03754; GFX12-CU-NEXT: s_wait_samplecnt 0x03755; GFX12-CU-NEXT: s_wait_storecnt 0x03756; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x03757; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163758; GFX12-CU-NEXT: s_wait_dscnt 0x03759; GFX12-CU-NEXT: s_endpgm3760;3761; GFX1250-LABEL: local_workgroup_acq_rel_monotonic_cmpxchg:3762; GFX1250: ; %bb.0: ; %entry3763; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 13764; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x03765; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x43766; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x83767; GFX1250-NEXT: s_wait_kmcnt 0x03768; GFX1250-NEXT: v_mov_b32_e32 v0, s23769; GFX1250-NEXT: v_mov_b32_e32 v1, s13770; GFX1250-NEXT: v_mov_b32_e32 v2, s03771; GFX1250-NEXT: s_wait_storecnt 0x03772; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x03773; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163774; GFX1250-NEXT: s_wait_dscnt 0x03775; GFX1250-NEXT: s_endpgm3776 ptr addrspace(3) %out, i32 %in, i32 %old) {3777entry:3778 %gep = getelementptr i32, ptr addrspace(3) %out, i32 43779 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic3780 ret void3781}3782 3783define amdgpu_kernel void @local_workgroup_seq_cst_monotonic_cmpxchg(3784; GFX6-LABEL: local_workgroup_seq_cst_monotonic_cmpxchg:3785; GFX6: ; %bb.0: ; %entry3786; GFX6-NEXT: s_load_dword s6, s[8:9], 0x03787; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr63788; GFX6-NEXT: s_load_dword s4, s[8:9], 0x13789; GFX6-NEXT: s_load_dword s5, s[8:9], 0x23790; GFX6-NEXT: s_mov_b32 m0, -13791; GFX6-NEXT: s_waitcnt lgkmcnt(0)3792; GFX6-NEXT: v_mov_b32_e32 v0, s63793; GFX6-NEXT: v_mov_b32_e32 v1, s53794; GFX6-NEXT: v_mov_b32_e32 v2, s43795; GFX6-NEXT: s_waitcnt lgkmcnt(0)3796; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163797; GFX6-NEXT: s_waitcnt lgkmcnt(0)3798; GFX6-NEXT: s_endpgm3799;3800; GFX7-LABEL: local_workgroup_seq_cst_monotonic_cmpxchg:3801; GFX7: ; %bb.0: ; %entry3802; GFX7-NEXT: s_load_dword s6, s[8:9], 0x03803; GFX7-NEXT: s_load_dword s4, s[8:9], 0x13804; GFX7-NEXT: s_load_dword s5, s[8:9], 0x23805; GFX7-NEXT: s_mov_b32 m0, -13806; GFX7-NEXT: s_waitcnt lgkmcnt(0)3807; GFX7-NEXT: v_mov_b32_e32 v0, s63808; GFX7-NEXT: v_mov_b32_e32 v1, s53809; GFX7-NEXT: v_mov_b32_e32 v2, s43810; GFX7-NEXT: s_waitcnt lgkmcnt(0)3811; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163812; GFX7-NEXT: s_waitcnt lgkmcnt(0)3813; GFX7-NEXT: s_endpgm3814;3815; GFX10-WGP-LABEL: local_workgroup_seq_cst_monotonic_cmpxchg:3816; GFX10-WGP: ; %bb.0: ; %entry3817; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x03818; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x43819; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x83820; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)3821; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s63822; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s53823; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s43824; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3825; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x03826; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163827; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)3828; GFX10-WGP-NEXT: buffer_gl0_inv3829; GFX10-WGP-NEXT: s_endpgm3830;3831; GFX10-CU-LABEL: local_workgroup_seq_cst_monotonic_cmpxchg:3832; GFX10-CU: ; %bb.0: ; %entry3833; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x03834; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x43835; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x83836; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)3837; GFX10-CU-NEXT: v_mov_b32_e32 v0, s63838; GFX10-CU-NEXT: v_mov_b32_e32 v1, s53839; GFX10-CU-NEXT: v_mov_b32_e32 v2, s43840; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3841; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x03842; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163843; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)3844; GFX10-CU-NEXT: s_endpgm3845;3846; SKIP-CACHE-INV-LABEL: local_workgroup_seq_cst_monotonic_cmpxchg:3847; SKIP-CACHE-INV: ; %bb.0: ; %entry3848; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x03849; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x13850; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x23851; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -13852; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)3853; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s23854; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s13855; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s03856; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)3857; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163858; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)3859; SKIP-CACHE-INV-NEXT: s_endpgm3860;3861; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_seq_cst_monotonic_cmpxchg:3862; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry3863; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x03864; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x43865; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x83866; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3867; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s63868; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s53869; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s43870; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3871; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163872; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3873; GFX90A-NOTTGSPLIT-NEXT: s_endpgm3874;3875; GFX90A-TGSPLIT-LABEL: local_workgroup_seq_cst_monotonic_cmpxchg:3876; GFX90A-TGSPLIT: ; %bb.0: ; %entry3877; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x03878; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x43879; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x83880; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3881; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s63882; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s53883; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s43884; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3885; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163886; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol3887; GFX90A-TGSPLIT-NEXT: s_endpgm3888;3889; GFX942-NOTTGSPLIT-LABEL: local_workgroup_seq_cst_monotonic_cmpxchg:3890; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry3891; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x03892; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x43893; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x83894; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3895; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s23896; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s13897; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s03898; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3899; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163900; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3901; GFX942-NOTTGSPLIT-NEXT: s_endpgm3902;3903; GFX942-TGSPLIT-LABEL: local_workgroup_seq_cst_monotonic_cmpxchg:3904; GFX942-TGSPLIT: ; %bb.0: ; %entry3905; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x03906; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x43907; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x83908; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)3909; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s23910; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s13911; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s03912; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3913; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:163914; GFX942-TGSPLIT-NEXT: buffer_inv sc03915; GFX942-TGSPLIT-NEXT: s_endpgm3916;3917; GFX11-WGP-LABEL: local_workgroup_seq_cst_monotonic_cmpxchg:3918; GFX11-WGP: ; %bb.0: ; %entry3919; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x03920; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x43921; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x83922; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)3923; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s23924; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s13925; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s03926; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3927; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x03928; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163929; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)3930; GFX11-WGP-NEXT: buffer_gl0_inv3931; GFX11-WGP-NEXT: s_endpgm3932;3933; GFX11-CU-LABEL: local_workgroup_seq_cst_monotonic_cmpxchg:3934; GFX11-CU: ; %bb.0: ; %entry3935; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x03936; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x43937; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x83938; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)3939; GFX11-CU-NEXT: v_mov_b32_e32 v0, s23940; GFX11-CU-NEXT: v_mov_b32_e32 v1, s13941; GFX11-CU-NEXT: v_mov_b32_e32 v2, s03942; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3943; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x03944; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163945; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)3946; GFX11-CU-NEXT: s_endpgm3947;3948; GFX12-WGP-LABEL: local_workgroup_seq_cst_monotonic_cmpxchg:3949; GFX12-WGP: ; %bb.0: ; %entry3950; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x03951; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x43952; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x83953; GFX12-WGP-NEXT: s_wait_kmcnt 0x03954; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s23955; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s13956; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s03957; GFX12-WGP-NEXT: s_wait_bvhcnt 0x03958; GFX12-WGP-NEXT: s_wait_samplecnt 0x03959; GFX12-WGP-NEXT: s_wait_storecnt 0x03960; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x03961; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163962; GFX12-WGP-NEXT: s_wait_dscnt 0x03963; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE3964; GFX12-WGP-NEXT: s_endpgm3965;3966; GFX12-CU-LABEL: local_workgroup_seq_cst_monotonic_cmpxchg:3967; GFX12-CU: ; %bb.0: ; %entry3968; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x03969; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x43970; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x83971; GFX12-CU-NEXT: s_wait_kmcnt 0x03972; GFX12-CU-NEXT: v_mov_b32_e32 v0, s23973; GFX12-CU-NEXT: v_mov_b32_e32 v1, s13974; GFX12-CU-NEXT: v_mov_b32_e32 v2, s03975; GFX12-CU-NEXT: s_wait_bvhcnt 0x03976; GFX12-CU-NEXT: s_wait_samplecnt 0x03977; GFX12-CU-NEXT: s_wait_storecnt 0x03978; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x03979; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163980; GFX12-CU-NEXT: s_wait_dscnt 0x03981; GFX12-CU-NEXT: s_endpgm3982;3983; GFX1250-LABEL: local_workgroup_seq_cst_monotonic_cmpxchg:3984; GFX1250: ; %bb.0: ; %entry3985; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 13986; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x03987; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x43988; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x83989; GFX1250-NEXT: s_wait_kmcnt 0x03990; GFX1250-NEXT: v_mov_b32_e32 v0, s23991; GFX1250-NEXT: v_mov_b32_e32 v1, s13992; GFX1250-NEXT: v_mov_b32_e32 v2, s03993; GFX1250-NEXT: s_wait_storecnt 0x03994; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x03995; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:163996; GFX1250-NEXT: s_wait_dscnt 0x03997; GFX1250-NEXT: s_endpgm3998 ptr addrspace(3) %out, i32 %in, i32 %old) {3999entry:4000 %gep = getelementptr i32, ptr addrspace(3) %out, i32 44001 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic4002 ret void4003}4004 4005define amdgpu_kernel void @local_workgroup_monotonic_acquire_cmpxchg(4006; GFX6-LABEL: local_workgroup_monotonic_acquire_cmpxchg:4007; GFX6: ; %bb.0: ; %entry4008; GFX6-NEXT: s_load_dword s6, s[8:9], 0x04009; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr64010; GFX6-NEXT: s_load_dword s4, s[8:9], 0x14011; GFX6-NEXT: s_load_dword s5, s[8:9], 0x24012; GFX6-NEXT: s_mov_b32 m0, -14013; GFX6-NEXT: s_waitcnt lgkmcnt(0)4014; GFX6-NEXT: v_mov_b32_e32 v0, s64015; GFX6-NEXT: v_mov_b32_e32 v1, s54016; GFX6-NEXT: v_mov_b32_e32 v2, s44017; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164018; GFX6-NEXT: s_waitcnt lgkmcnt(0)4019; GFX6-NEXT: s_endpgm4020;4021; GFX7-LABEL: local_workgroup_monotonic_acquire_cmpxchg:4022; GFX7: ; %bb.0: ; %entry4023; GFX7-NEXT: s_load_dword s6, s[8:9], 0x04024; GFX7-NEXT: s_load_dword s4, s[8:9], 0x14025; GFX7-NEXT: s_load_dword s5, s[8:9], 0x24026; GFX7-NEXT: s_mov_b32 m0, -14027; GFX7-NEXT: s_waitcnt lgkmcnt(0)4028; GFX7-NEXT: v_mov_b32_e32 v0, s64029; GFX7-NEXT: v_mov_b32_e32 v1, s54030; GFX7-NEXT: v_mov_b32_e32 v2, s44031; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164032; GFX7-NEXT: s_waitcnt lgkmcnt(0)4033; GFX7-NEXT: s_endpgm4034;4035; GFX10-WGP-LABEL: local_workgroup_monotonic_acquire_cmpxchg:4036; GFX10-WGP: ; %bb.0: ; %entry4037; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x04038; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x44039; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x84040; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)4041; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s64042; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s54043; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s44044; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164045; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)4046; GFX10-WGP-NEXT: buffer_gl0_inv4047; GFX10-WGP-NEXT: s_endpgm4048;4049; GFX10-CU-LABEL: local_workgroup_monotonic_acquire_cmpxchg:4050; GFX10-CU: ; %bb.0: ; %entry4051; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x04052; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x44053; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x84054; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)4055; GFX10-CU-NEXT: v_mov_b32_e32 v0, s64056; GFX10-CU-NEXT: v_mov_b32_e32 v1, s54057; GFX10-CU-NEXT: v_mov_b32_e32 v2, s44058; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164059; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)4060; GFX10-CU-NEXT: s_endpgm4061;4062; SKIP-CACHE-INV-LABEL: local_workgroup_monotonic_acquire_cmpxchg:4063; SKIP-CACHE-INV: ; %bb.0: ; %entry4064; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x04065; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x14066; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x24067; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -14068; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)4069; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s24070; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s14071; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s04072; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164073; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)4074; SKIP-CACHE-INV-NEXT: s_endpgm4075;4076; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_monotonic_acquire_cmpxchg:4077; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry4078; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x04079; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x44080; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x84081; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4082; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s64083; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s54084; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s44085; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164086; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4087; GFX90A-NOTTGSPLIT-NEXT: s_endpgm4088;4089; GFX90A-TGSPLIT-LABEL: local_workgroup_monotonic_acquire_cmpxchg:4090; GFX90A-TGSPLIT: ; %bb.0: ; %entry4091; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x04092; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x44093; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x84094; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4095; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s64096; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s54097; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s44098; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164099; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol4100; GFX90A-TGSPLIT-NEXT: s_endpgm4101;4102; GFX942-NOTTGSPLIT-LABEL: local_workgroup_monotonic_acquire_cmpxchg:4103; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry4104; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x04105; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x44106; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x84107; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4108; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s24109; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s14110; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s04111; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164112; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4113; GFX942-NOTTGSPLIT-NEXT: s_endpgm4114;4115; GFX942-TGSPLIT-LABEL: local_workgroup_monotonic_acquire_cmpxchg:4116; GFX942-TGSPLIT: ; %bb.0: ; %entry4117; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x04118; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x44119; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x84120; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4121; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s24122; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s14123; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s04124; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164125; GFX942-TGSPLIT-NEXT: buffer_inv sc04126; GFX942-TGSPLIT-NEXT: s_endpgm4127;4128; GFX11-WGP-LABEL: local_workgroup_monotonic_acquire_cmpxchg:4129; GFX11-WGP: ; %bb.0: ; %entry4130; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x04131; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x44132; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x84133; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)4134; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s24135; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s14136; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s04137; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:164138; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)4139; GFX11-WGP-NEXT: buffer_gl0_inv4140; GFX11-WGP-NEXT: s_endpgm4141;4142; GFX11-CU-LABEL: local_workgroup_monotonic_acquire_cmpxchg:4143; GFX11-CU: ; %bb.0: ; %entry4144; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x04145; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x44146; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x84147; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)4148; GFX11-CU-NEXT: v_mov_b32_e32 v0, s24149; GFX11-CU-NEXT: v_mov_b32_e32 v1, s14150; GFX11-CU-NEXT: v_mov_b32_e32 v2, s04151; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:164152; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)4153; GFX11-CU-NEXT: s_endpgm4154;4155; GFX12-WGP-LABEL: local_workgroup_monotonic_acquire_cmpxchg:4156; GFX12-WGP: ; %bb.0: ; %entry4157; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x04158; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x44159; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x84160; GFX12-WGP-NEXT: s_wait_kmcnt 0x04161; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s24162; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s14163; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s04164; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:164165; GFX12-WGP-NEXT: s_wait_dscnt 0x04166; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE4167; GFX12-WGP-NEXT: s_endpgm4168;4169; GFX12-CU-LABEL: local_workgroup_monotonic_acquire_cmpxchg:4170; GFX12-CU: ; %bb.0: ; %entry4171; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x04172; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x44173; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x84174; GFX12-CU-NEXT: s_wait_kmcnt 0x04175; GFX12-CU-NEXT: v_mov_b32_e32 v0, s24176; GFX12-CU-NEXT: v_mov_b32_e32 v1, s14177; GFX12-CU-NEXT: v_mov_b32_e32 v2, s04178; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:164179; GFX12-CU-NEXT: s_wait_dscnt 0x04180; GFX12-CU-NEXT: s_endpgm4181;4182; GFX1250-LABEL: local_workgroup_monotonic_acquire_cmpxchg:4183; GFX1250: ; %bb.0: ; %entry4184; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 14185; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x04186; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x44187; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x84188; GFX1250-NEXT: s_wait_kmcnt 0x04189; GFX1250-NEXT: v_mov_b32_e32 v0, s24190; GFX1250-NEXT: v_mov_b32_e32 v1, s14191; GFX1250-NEXT: v_mov_b32_e32 v2, s04192; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:164193; GFX1250-NEXT: s_wait_dscnt 0x04194; GFX1250-NEXT: s_endpgm4195 ptr addrspace(3) %out, i32 %in, i32 %old) {4196entry:4197 %gep = getelementptr i32, ptr addrspace(3) %out, i32 44198 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") monotonic acquire4199 ret void4200}4201 4202define amdgpu_kernel void @local_workgroup_acquire_acquire_cmpxchg(4203; GFX6-LABEL: local_workgroup_acquire_acquire_cmpxchg:4204; GFX6: ; %bb.0: ; %entry4205; GFX6-NEXT: s_load_dword s6, s[8:9], 0x04206; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr64207; GFX6-NEXT: s_load_dword s4, s[8:9], 0x14208; GFX6-NEXT: s_load_dword s5, s[8:9], 0x24209; GFX6-NEXT: s_mov_b32 m0, -14210; GFX6-NEXT: s_waitcnt lgkmcnt(0)4211; GFX6-NEXT: v_mov_b32_e32 v0, s64212; GFX6-NEXT: v_mov_b32_e32 v1, s54213; GFX6-NEXT: v_mov_b32_e32 v2, s44214; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164215; GFX6-NEXT: s_waitcnt lgkmcnt(0)4216; GFX6-NEXT: s_endpgm4217;4218; GFX7-LABEL: local_workgroup_acquire_acquire_cmpxchg:4219; GFX7: ; %bb.0: ; %entry4220; GFX7-NEXT: s_load_dword s6, s[8:9], 0x04221; GFX7-NEXT: s_load_dword s4, s[8:9], 0x14222; GFX7-NEXT: s_load_dword s5, s[8:9], 0x24223; GFX7-NEXT: s_mov_b32 m0, -14224; GFX7-NEXT: s_waitcnt lgkmcnt(0)4225; GFX7-NEXT: v_mov_b32_e32 v0, s64226; GFX7-NEXT: v_mov_b32_e32 v1, s54227; GFX7-NEXT: v_mov_b32_e32 v2, s44228; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164229; GFX7-NEXT: s_waitcnt lgkmcnt(0)4230; GFX7-NEXT: s_endpgm4231;4232; GFX10-WGP-LABEL: local_workgroup_acquire_acquire_cmpxchg:4233; GFX10-WGP: ; %bb.0: ; %entry4234; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x04235; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x44236; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x84237; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)4238; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s64239; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s54240; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s44241; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164242; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)4243; GFX10-WGP-NEXT: buffer_gl0_inv4244; GFX10-WGP-NEXT: s_endpgm4245;4246; GFX10-CU-LABEL: local_workgroup_acquire_acquire_cmpxchg:4247; GFX10-CU: ; %bb.0: ; %entry4248; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x04249; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x44250; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x84251; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)4252; GFX10-CU-NEXT: v_mov_b32_e32 v0, s64253; GFX10-CU-NEXT: v_mov_b32_e32 v1, s54254; GFX10-CU-NEXT: v_mov_b32_e32 v2, s44255; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164256; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)4257; GFX10-CU-NEXT: s_endpgm4258;4259; SKIP-CACHE-INV-LABEL: local_workgroup_acquire_acquire_cmpxchg:4260; SKIP-CACHE-INV: ; %bb.0: ; %entry4261; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x04262; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x14263; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x24264; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -14265; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)4266; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s24267; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s14268; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s04269; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164270; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)4271; SKIP-CACHE-INV-NEXT: s_endpgm4272;4273; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_acquire_acquire_cmpxchg:4274; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry4275; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x04276; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x44277; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x84278; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4279; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s64280; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s54281; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s44282; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164283; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4284; GFX90A-NOTTGSPLIT-NEXT: s_endpgm4285;4286; GFX90A-TGSPLIT-LABEL: local_workgroup_acquire_acquire_cmpxchg:4287; GFX90A-TGSPLIT: ; %bb.0: ; %entry4288; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x04289; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x44290; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x84291; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4292; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s64293; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s54294; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s44295; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164296; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol4297; GFX90A-TGSPLIT-NEXT: s_endpgm4298;4299; GFX942-NOTTGSPLIT-LABEL: local_workgroup_acquire_acquire_cmpxchg:4300; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry4301; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x04302; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x44303; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x84304; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4305; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s24306; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s14307; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s04308; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164309; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4310; GFX942-NOTTGSPLIT-NEXT: s_endpgm4311;4312; GFX942-TGSPLIT-LABEL: local_workgroup_acquire_acquire_cmpxchg:4313; GFX942-TGSPLIT: ; %bb.0: ; %entry4314; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x04315; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x44316; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x84317; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4318; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s24319; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s14320; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s04321; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164322; GFX942-TGSPLIT-NEXT: buffer_inv sc04323; GFX942-TGSPLIT-NEXT: s_endpgm4324;4325; GFX11-WGP-LABEL: local_workgroup_acquire_acquire_cmpxchg:4326; GFX11-WGP: ; %bb.0: ; %entry4327; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x04328; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x44329; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x84330; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)4331; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s24332; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s14333; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s04334; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:164335; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)4336; GFX11-WGP-NEXT: buffer_gl0_inv4337; GFX11-WGP-NEXT: s_endpgm4338;4339; GFX11-CU-LABEL: local_workgroup_acquire_acquire_cmpxchg:4340; GFX11-CU: ; %bb.0: ; %entry4341; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x04342; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x44343; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x84344; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)4345; GFX11-CU-NEXT: v_mov_b32_e32 v0, s24346; GFX11-CU-NEXT: v_mov_b32_e32 v1, s14347; GFX11-CU-NEXT: v_mov_b32_e32 v2, s04348; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:164349; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)4350; GFX11-CU-NEXT: s_endpgm4351;4352; GFX12-WGP-LABEL: local_workgroup_acquire_acquire_cmpxchg:4353; GFX12-WGP: ; %bb.0: ; %entry4354; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x04355; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x44356; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x84357; GFX12-WGP-NEXT: s_wait_kmcnt 0x04358; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s24359; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s14360; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s04361; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:164362; GFX12-WGP-NEXT: s_wait_dscnt 0x04363; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE4364; GFX12-WGP-NEXT: s_endpgm4365;4366; GFX12-CU-LABEL: local_workgroup_acquire_acquire_cmpxchg:4367; GFX12-CU: ; %bb.0: ; %entry4368; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x04369; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x44370; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x84371; GFX12-CU-NEXT: s_wait_kmcnt 0x04372; GFX12-CU-NEXT: v_mov_b32_e32 v0, s24373; GFX12-CU-NEXT: v_mov_b32_e32 v1, s14374; GFX12-CU-NEXT: v_mov_b32_e32 v2, s04375; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:164376; GFX12-CU-NEXT: s_wait_dscnt 0x04377; GFX12-CU-NEXT: s_endpgm4378;4379; GFX1250-LABEL: local_workgroup_acquire_acquire_cmpxchg:4380; GFX1250: ; %bb.0: ; %entry4381; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 14382; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x04383; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x44384; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x84385; GFX1250-NEXT: s_wait_kmcnt 0x04386; GFX1250-NEXT: v_mov_b32_e32 v0, s24387; GFX1250-NEXT: v_mov_b32_e32 v1, s14388; GFX1250-NEXT: v_mov_b32_e32 v2, s04389; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:164390; GFX1250-NEXT: s_wait_dscnt 0x04391; GFX1250-NEXT: s_endpgm4392 ptr addrspace(3) %out, i32 %in, i32 %old) {4393entry:4394 %gep = getelementptr i32, ptr addrspace(3) %out, i32 44395 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire4396 ret void4397}4398 4399define amdgpu_kernel void @local_workgroup_release_acquire_cmpxchg(4400; GFX6-LABEL: local_workgroup_release_acquire_cmpxchg:4401; GFX6: ; %bb.0: ; %entry4402; GFX6-NEXT: s_load_dword s6, s[8:9], 0x04403; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr64404; GFX6-NEXT: s_load_dword s4, s[8:9], 0x14405; GFX6-NEXT: s_load_dword s5, s[8:9], 0x24406; GFX6-NEXT: s_mov_b32 m0, -14407; GFX6-NEXT: s_waitcnt lgkmcnt(0)4408; GFX6-NEXT: v_mov_b32_e32 v0, s64409; GFX6-NEXT: v_mov_b32_e32 v1, s54410; GFX6-NEXT: v_mov_b32_e32 v2, s44411; GFX6-NEXT: s_waitcnt lgkmcnt(0)4412; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164413; GFX6-NEXT: s_waitcnt lgkmcnt(0)4414; GFX6-NEXT: s_endpgm4415;4416; GFX7-LABEL: local_workgroup_release_acquire_cmpxchg:4417; GFX7: ; %bb.0: ; %entry4418; GFX7-NEXT: s_load_dword s6, s[8:9], 0x04419; GFX7-NEXT: s_load_dword s4, s[8:9], 0x14420; GFX7-NEXT: s_load_dword s5, s[8:9], 0x24421; GFX7-NEXT: s_mov_b32 m0, -14422; GFX7-NEXT: s_waitcnt lgkmcnt(0)4423; GFX7-NEXT: v_mov_b32_e32 v0, s64424; GFX7-NEXT: v_mov_b32_e32 v1, s54425; GFX7-NEXT: v_mov_b32_e32 v2, s44426; GFX7-NEXT: s_waitcnt lgkmcnt(0)4427; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164428; GFX7-NEXT: s_waitcnt lgkmcnt(0)4429; GFX7-NEXT: s_endpgm4430;4431; GFX10-WGP-LABEL: local_workgroup_release_acquire_cmpxchg:4432; GFX10-WGP: ; %bb.0: ; %entry4433; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x04434; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x44435; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x84436; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)4437; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s64438; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s54439; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s44440; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4441; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x04442; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164443; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)4444; GFX10-WGP-NEXT: buffer_gl0_inv4445; GFX10-WGP-NEXT: s_endpgm4446;4447; GFX10-CU-LABEL: local_workgroup_release_acquire_cmpxchg:4448; GFX10-CU: ; %bb.0: ; %entry4449; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x04450; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x44451; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x84452; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)4453; GFX10-CU-NEXT: v_mov_b32_e32 v0, s64454; GFX10-CU-NEXT: v_mov_b32_e32 v1, s54455; GFX10-CU-NEXT: v_mov_b32_e32 v2, s44456; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4457; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x04458; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164459; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)4460; GFX10-CU-NEXT: s_endpgm4461;4462; SKIP-CACHE-INV-LABEL: local_workgroup_release_acquire_cmpxchg:4463; SKIP-CACHE-INV: ; %bb.0: ; %entry4464; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x04465; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x14466; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x24467; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -14468; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)4469; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s24470; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s14471; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s04472; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)4473; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164474; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)4475; SKIP-CACHE-INV-NEXT: s_endpgm4476;4477; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_release_acquire_cmpxchg:4478; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry4479; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x04480; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x44481; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x84482; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4483; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s64484; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s54485; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s44486; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4487; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164488; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4489; GFX90A-NOTTGSPLIT-NEXT: s_endpgm4490;4491; GFX90A-TGSPLIT-LABEL: local_workgroup_release_acquire_cmpxchg:4492; GFX90A-TGSPLIT: ; %bb.0: ; %entry4493; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x04494; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x44495; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x84496; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4497; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s64498; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s54499; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s44500; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4501; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164502; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol4503; GFX90A-TGSPLIT-NEXT: s_endpgm4504;4505; GFX942-NOTTGSPLIT-LABEL: local_workgroup_release_acquire_cmpxchg:4506; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry4507; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x04508; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x44509; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x84510; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4511; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s24512; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s14513; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s04514; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4515; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164516; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4517; GFX942-NOTTGSPLIT-NEXT: s_endpgm4518;4519; GFX942-TGSPLIT-LABEL: local_workgroup_release_acquire_cmpxchg:4520; GFX942-TGSPLIT: ; %bb.0: ; %entry4521; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x04522; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x44523; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x84524; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4525; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s24526; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s14527; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s04528; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4529; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164530; GFX942-TGSPLIT-NEXT: buffer_inv sc04531; GFX942-TGSPLIT-NEXT: s_endpgm4532;4533; GFX11-WGP-LABEL: local_workgroup_release_acquire_cmpxchg:4534; GFX11-WGP: ; %bb.0: ; %entry4535; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x04536; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x44537; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x84538; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)4539; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s24540; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s14541; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s04542; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4543; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x04544; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:164545; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)4546; GFX11-WGP-NEXT: buffer_gl0_inv4547; GFX11-WGP-NEXT: s_endpgm4548;4549; GFX11-CU-LABEL: local_workgroup_release_acquire_cmpxchg:4550; GFX11-CU: ; %bb.0: ; %entry4551; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x04552; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x44553; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x84554; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)4555; GFX11-CU-NEXT: v_mov_b32_e32 v0, s24556; GFX11-CU-NEXT: v_mov_b32_e32 v1, s14557; GFX11-CU-NEXT: v_mov_b32_e32 v2, s04558; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4559; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x04560; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:164561; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)4562; GFX11-CU-NEXT: s_endpgm4563;4564; GFX12-WGP-LABEL: local_workgroup_release_acquire_cmpxchg:4565; GFX12-WGP: ; %bb.0: ; %entry4566; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x04567; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x44568; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x84569; GFX12-WGP-NEXT: s_wait_kmcnt 0x04570; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s24571; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s14572; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s04573; GFX12-WGP-NEXT: s_wait_bvhcnt 0x04574; GFX12-WGP-NEXT: s_wait_samplecnt 0x04575; GFX12-WGP-NEXT: s_wait_storecnt 0x04576; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x04577; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:164578; GFX12-WGP-NEXT: s_wait_dscnt 0x04579; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE4580; GFX12-WGP-NEXT: s_endpgm4581;4582; GFX12-CU-LABEL: local_workgroup_release_acquire_cmpxchg:4583; GFX12-CU: ; %bb.0: ; %entry4584; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x04585; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x44586; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x84587; GFX12-CU-NEXT: s_wait_kmcnt 0x04588; GFX12-CU-NEXT: v_mov_b32_e32 v0, s24589; GFX12-CU-NEXT: v_mov_b32_e32 v1, s14590; GFX12-CU-NEXT: v_mov_b32_e32 v2, s04591; GFX12-CU-NEXT: s_wait_bvhcnt 0x04592; GFX12-CU-NEXT: s_wait_samplecnt 0x04593; GFX12-CU-NEXT: s_wait_storecnt 0x04594; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x04595; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:164596; GFX12-CU-NEXT: s_wait_dscnt 0x04597; GFX12-CU-NEXT: s_endpgm4598;4599; GFX1250-LABEL: local_workgroup_release_acquire_cmpxchg:4600; GFX1250: ; %bb.0: ; %entry4601; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 14602; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x04603; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x44604; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x84605; GFX1250-NEXT: s_wait_kmcnt 0x04606; GFX1250-NEXT: v_mov_b32_e32 v0, s24607; GFX1250-NEXT: v_mov_b32_e32 v1, s14608; GFX1250-NEXT: v_mov_b32_e32 v2, s04609; GFX1250-NEXT: s_wait_storecnt 0x04610; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x04611; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:164612; GFX1250-NEXT: s_wait_dscnt 0x04613; GFX1250-NEXT: s_endpgm4614 ptr addrspace(3) %out, i32 %in, i32 %old) {4615entry:4616 %gep = getelementptr i32, ptr addrspace(3) %out, i32 44617 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") release acquire4618 ret void4619}4620 4621define amdgpu_kernel void @local_workgroup_acq_rel_acquire_cmpxchg(4622; GFX6-LABEL: local_workgroup_acq_rel_acquire_cmpxchg:4623; GFX6: ; %bb.0: ; %entry4624; GFX6-NEXT: s_load_dword s6, s[8:9], 0x04625; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr64626; GFX6-NEXT: s_load_dword s4, s[8:9], 0x14627; GFX6-NEXT: s_load_dword s5, s[8:9], 0x24628; GFX6-NEXT: s_mov_b32 m0, -14629; GFX6-NEXT: s_waitcnt lgkmcnt(0)4630; GFX6-NEXT: v_mov_b32_e32 v0, s64631; GFX6-NEXT: v_mov_b32_e32 v1, s54632; GFX6-NEXT: v_mov_b32_e32 v2, s44633; GFX6-NEXT: s_waitcnt lgkmcnt(0)4634; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164635; GFX6-NEXT: s_waitcnt lgkmcnt(0)4636; GFX6-NEXT: s_endpgm4637;4638; GFX7-LABEL: local_workgroup_acq_rel_acquire_cmpxchg:4639; GFX7: ; %bb.0: ; %entry4640; GFX7-NEXT: s_load_dword s6, s[8:9], 0x04641; GFX7-NEXT: s_load_dword s4, s[8:9], 0x14642; GFX7-NEXT: s_load_dword s5, s[8:9], 0x24643; GFX7-NEXT: s_mov_b32 m0, -14644; GFX7-NEXT: s_waitcnt lgkmcnt(0)4645; GFX7-NEXT: v_mov_b32_e32 v0, s64646; GFX7-NEXT: v_mov_b32_e32 v1, s54647; GFX7-NEXT: v_mov_b32_e32 v2, s44648; GFX7-NEXT: s_waitcnt lgkmcnt(0)4649; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164650; GFX7-NEXT: s_waitcnt lgkmcnt(0)4651; GFX7-NEXT: s_endpgm4652;4653; GFX10-WGP-LABEL: local_workgroup_acq_rel_acquire_cmpxchg:4654; GFX10-WGP: ; %bb.0: ; %entry4655; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x04656; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x44657; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x84658; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)4659; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s64660; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s54661; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s44662; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4663; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x04664; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164665; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)4666; GFX10-WGP-NEXT: buffer_gl0_inv4667; GFX10-WGP-NEXT: s_endpgm4668;4669; GFX10-CU-LABEL: local_workgroup_acq_rel_acquire_cmpxchg:4670; GFX10-CU: ; %bb.0: ; %entry4671; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x04672; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x44673; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x84674; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)4675; GFX10-CU-NEXT: v_mov_b32_e32 v0, s64676; GFX10-CU-NEXT: v_mov_b32_e32 v1, s54677; GFX10-CU-NEXT: v_mov_b32_e32 v2, s44678; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4679; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x04680; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164681; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)4682; GFX10-CU-NEXT: s_endpgm4683;4684; SKIP-CACHE-INV-LABEL: local_workgroup_acq_rel_acquire_cmpxchg:4685; SKIP-CACHE-INV: ; %bb.0: ; %entry4686; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x04687; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x14688; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x24689; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -14690; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)4691; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s24692; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s14693; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s04694; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)4695; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164696; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)4697; SKIP-CACHE-INV-NEXT: s_endpgm4698;4699; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_acq_rel_acquire_cmpxchg:4700; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry4701; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x04702; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x44703; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x84704; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4705; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s64706; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s54707; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s44708; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4709; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164710; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4711; GFX90A-NOTTGSPLIT-NEXT: s_endpgm4712;4713; GFX90A-TGSPLIT-LABEL: local_workgroup_acq_rel_acquire_cmpxchg:4714; GFX90A-TGSPLIT: ; %bb.0: ; %entry4715; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x04716; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x44717; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x84718; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4719; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s64720; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s54721; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s44722; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4723; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164724; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol4725; GFX90A-TGSPLIT-NEXT: s_endpgm4726;4727; GFX942-NOTTGSPLIT-LABEL: local_workgroup_acq_rel_acquire_cmpxchg:4728; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry4729; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x04730; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x44731; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x84732; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4733; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s24734; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s14735; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s04736; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4737; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164738; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4739; GFX942-NOTTGSPLIT-NEXT: s_endpgm4740;4741; GFX942-TGSPLIT-LABEL: local_workgroup_acq_rel_acquire_cmpxchg:4742; GFX942-TGSPLIT: ; %bb.0: ; %entry4743; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x04744; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x44745; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x84746; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4747; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s24748; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s14749; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s04750; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4751; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164752; GFX942-TGSPLIT-NEXT: buffer_inv sc04753; GFX942-TGSPLIT-NEXT: s_endpgm4754;4755; GFX11-WGP-LABEL: local_workgroup_acq_rel_acquire_cmpxchg:4756; GFX11-WGP: ; %bb.0: ; %entry4757; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x04758; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x44759; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x84760; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)4761; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s24762; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s14763; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s04764; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4765; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x04766; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:164767; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)4768; GFX11-WGP-NEXT: buffer_gl0_inv4769; GFX11-WGP-NEXT: s_endpgm4770;4771; GFX11-CU-LABEL: local_workgroup_acq_rel_acquire_cmpxchg:4772; GFX11-CU: ; %bb.0: ; %entry4773; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x04774; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x44775; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x84776; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)4777; GFX11-CU-NEXT: v_mov_b32_e32 v0, s24778; GFX11-CU-NEXT: v_mov_b32_e32 v1, s14779; GFX11-CU-NEXT: v_mov_b32_e32 v2, s04780; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4781; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x04782; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:164783; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)4784; GFX11-CU-NEXT: s_endpgm4785;4786; GFX12-WGP-LABEL: local_workgroup_acq_rel_acquire_cmpxchg:4787; GFX12-WGP: ; %bb.0: ; %entry4788; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x04789; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x44790; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x84791; GFX12-WGP-NEXT: s_wait_kmcnt 0x04792; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s24793; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s14794; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s04795; GFX12-WGP-NEXT: s_wait_bvhcnt 0x04796; GFX12-WGP-NEXT: s_wait_samplecnt 0x04797; GFX12-WGP-NEXT: s_wait_storecnt 0x04798; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x04799; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:164800; GFX12-WGP-NEXT: s_wait_dscnt 0x04801; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE4802; GFX12-WGP-NEXT: s_endpgm4803;4804; GFX12-CU-LABEL: local_workgroup_acq_rel_acquire_cmpxchg:4805; GFX12-CU: ; %bb.0: ; %entry4806; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x04807; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x44808; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x84809; GFX12-CU-NEXT: s_wait_kmcnt 0x04810; GFX12-CU-NEXT: v_mov_b32_e32 v0, s24811; GFX12-CU-NEXT: v_mov_b32_e32 v1, s14812; GFX12-CU-NEXT: v_mov_b32_e32 v2, s04813; GFX12-CU-NEXT: s_wait_bvhcnt 0x04814; GFX12-CU-NEXT: s_wait_samplecnt 0x04815; GFX12-CU-NEXT: s_wait_storecnt 0x04816; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x04817; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:164818; GFX12-CU-NEXT: s_wait_dscnt 0x04819; GFX12-CU-NEXT: s_endpgm4820;4821; GFX1250-LABEL: local_workgroup_acq_rel_acquire_cmpxchg:4822; GFX1250: ; %bb.0: ; %entry4823; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 14824; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x04825; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x44826; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x84827; GFX1250-NEXT: s_wait_kmcnt 0x04828; GFX1250-NEXT: v_mov_b32_e32 v0, s24829; GFX1250-NEXT: v_mov_b32_e32 v1, s14830; GFX1250-NEXT: v_mov_b32_e32 v2, s04831; GFX1250-NEXT: s_wait_storecnt 0x04832; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x04833; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:164834; GFX1250-NEXT: s_wait_dscnt 0x04835; GFX1250-NEXT: s_endpgm4836 ptr addrspace(3) %out, i32 %in, i32 %old) {4837entry:4838 %gep = getelementptr i32, ptr addrspace(3) %out, i32 44839 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire4840 ret void4841}4842 4843define amdgpu_kernel void @local_workgroup_seq_cst_acquire_cmpxchg(4844; GFX6-LABEL: local_workgroup_seq_cst_acquire_cmpxchg:4845; GFX6: ; %bb.0: ; %entry4846; GFX6-NEXT: s_load_dword s6, s[8:9], 0x04847; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr64848; GFX6-NEXT: s_load_dword s4, s[8:9], 0x14849; GFX6-NEXT: s_load_dword s5, s[8:9], 0x24850; GFX6-NEXT: s_mov_b32 m0, -14851; GFX6-NEXT: s_waitcnt lgkmcnt(0)4852; GFX6-NEXT: v_mov_b32_e32 v0, s64853; GFX6-NEXT: v_mov_b32_e32 v1, s54854; GFX6-NEXT: v_mov_b32_e32 v2, s44855; GFX6-NEXT: s_waitcnt lgkmcnt(0)4856; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164857; GFX6-NEXT: s_waitcnt lgkmcnt(0)4858; GFX6-NEXT: s_endpgm4859;4860; GFX7-LABEL: local_workgroup_seq_cst_acquire_cmpxchg:4861; GFX7: ; %bb.0: ; %entry4862; GFX7-NEXT: s_load_dword s6, s[8:9], 0x04863; GFX7-NEXT: s_load_dword s4, s[8:9], 0x14864; GFX7-NEXT: s_load_dword s5, s[8:9], 0x24865; GFX7-NEXT: s_mov_b32 m0, -14866; GFX7-NEXT: s_waitcnt lgkmcnt(0)4867; GFX7-NEXT: v_mov_b32_e32 v0, s64868; GFX7-NEXT: v_mov_b32_e32 v1, s54869; GFX7-NEXT: v_mov_b32_e32 v2, s44870; GFX7-NEXT: s_waitcnt lgkmcnt(0)4871; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164872; GFX7-NEXT: s_waitcnt lgkmcnt(0)4873; GFX7-NEXT: s_endpgm4874;4875; GFX10-WGP-LABEL: local_workgroup_seq_cst_acquire_cmpxchg:4876; GFX10-WGP: ; %bb.0: ; %entry4877; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x04878; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x44879; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x84880; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)4881; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s64882; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s54883; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s44884; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4885; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x04886; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164887; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)4888; GFX10-WGP-NEXT: buffer_gl0_inv4889; GFX10-WGP-NEXT: s_endpgm4890;4891; GFX10-CU-LABEL: local_workgroup_seq_cst_acquire_cmpxchg:4892; GFX10-CU: ; %bb.0: ; %entry4893; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x04894; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x44895; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x84896; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)4897; GFX10-CU-NEXT: v_mov_b32_e32 v0, s64898; GFX10-CU-NEXT: v_mov_b32_e32 v1, s54899; GFX10-CU-NEXT: v_mov_b32_e32 v2, s44900; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4901; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x04902; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164903; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)4904; GFX10-CU-NEXT: s_endpgm4905;4906; SKIP-CACHE-INV-LABEL: local_workgroup_seq_cst_acquire_cmpxchg:4907; SKIP-CACHE-INV: ; %bb.0: ; %entry4908; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x04909; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x14910; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x24911; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -14912; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)4913; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s24914; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s14915; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s04916; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)4917; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164918; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)4919; SKIP-CACHE-INV-NEXT: s_endpgm4920;4921; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_seq_cst_acquire_cmpxchg:4922; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry4923; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x04924; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x44925; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x84926; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4927; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s64928; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s54929; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s44930; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4931; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164932; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4933; GFX90A-NOTTGSPLIT-NEXT: s_endpgm4934;4935; GFX90A-TGSPLIT-LABEL: local_workgroup_seq_cst_acquire_cmpxchg:4936; GFX90A-TGSPLIT: ; %bb.0: ; %entry4937; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x04938; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x44939; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x84940; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4941; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s64942; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s54943; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s44944; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4945; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164946; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol4947; GFX90A-TGSPLIT-NEXT: s_endpgm4948;4949; GFX942-NOTTGSPLIT-LABEL: local_workgroup_seq_cst_acquire_cmpxchg:4950; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry4951; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x04952; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x44953; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x84954; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4955; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s24956; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s14957; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s04958; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4959; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164960; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4961; GFX942-NOTTGSPLIT-NEXT: s_endpgm4962;4963; GFX942-TGSPLIT-LABEL: local_workgroup_seq_cst_acquire_cmpxchg:4964; GFX942-TGSPLIT: ; %bb.0: ; %entry4965; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x04966; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x44967; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x84968; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)4969; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s24970; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s14971; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s04972; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4973; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:164974; GFX942-TGSPLIT-NEXT: buffer_inv sc04975; GFX942-TGSPLIT-NEXT: s_endpgm4976;4977; GFX11-WGP-LABEL: local_workgroup_seq_cst_acquire_cmpxchg:4978; GFX11-WGP: ; %bb.0: ; %entry4979; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x04980; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x44981; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x84982; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)4983; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s24984; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s14985; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s04986; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4987; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x04988; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:164989; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)4990; GFX11-WGP-NEXT: buffer_gl0_inv4991; GFX11-WGP-NEXT: s_endpgm4992;4993; GFX11-CU-LABEL: local_workgroup_seq_cst_acquire_cmpxchg:4994; GFX11-CU: ; %bb.0: ; %entry4995; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x04996; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x44997; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x84998; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)4999; GFX11-CU-NEXT: v_mov_b32_e32 v0, s25000; GFX11-CU-NEXT: v_mov_b32_e32 v1, s15001; GFX11-CU-NEXT: v_mov_b32_e32 v2, s05002; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5003; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x05004; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:165005; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)5006; GFX11-CU-NEXT: s_endpgm5007;5008; GFX12-WGP-LABEL: local_workgroup_seq_cst_acquire_cmpxchg:5009; GFX12-WGP: ; %bb.0: ; %entry5010; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x05011; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x45012; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x85013; GFX12-WGP-NEXT: s_wait_kmcnt 0x05014; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s25015; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s15016; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s05017; GFX12-WGP-NEXT: s_wait_bvhcnt 0x05018; GFX12-WGP-NEXT: s_wait_samplecnt 0x05019; GFX12-WGP-NEXT: s_wait_storecnt 0x05020; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x05021; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:165022; GFX12-WGP-NEXT: s_wait_dscnt 0x05023; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE5024; GFX12-WGP-NEXT: s_endpgm5025;5026; GFX12-CU-LABEL: local_workgroup_seq_cst_acquire_cmpxchg:5027; GFX12-CU: ; %bb.0: ; %entry5028; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x05029; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x45030; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x85031; GFX12-CU-NEXT: s_wait_kmcnt 0x05032; GFX12-CU-NEXT: v_mov_b32_e32 v0, s25033; GFX12-CU-NEXT: v_mov_b32_e32 v1, s15034; GFX12-CU-NEXT: v_mov_b32_e32 v2, s05035; GFX12-CU-NEXT: s_wait_bvhcnt 0x05036; GFX12-CU-NEXT: s_wait_samplecnt 0x05037; GFX12-CU-NEXT: s_wait_storecnt 0x05038; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x05039; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:165040; GFX12-CU-NEXT: s_wait_dscnt 0x05041; GFX12-CU-NEXT: s_endpgm5042;5043; GFX1250-LABEL: local_workgroup_seq_cst_acquire_cmpxchg:5044; GFX1250: ; %bb.0: ; %entry5045; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 15046; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x05047; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x45048; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x85049; GFX1250-NEXT: s_wait_kmcnt 0x05050; GFX1250-NEXT: v_mov_b32_e32 v0, s25051; GFX1250-NEXT: v_mov_b32_e32 v1, s15052; GFX1250-NEXT: v_mov_b32_e32 v2, s05053; GFX1250-NEXT: s_wait_storecnt 0x05054; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x05055; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:165056; GFX1250-NEXT: s_wait_dscnt 0x05057; GFX1250-NEXT: s_endpgm5058 ptr addrspace(3) %out, i32 %in, i32 %old) {5059entry:5060 %gep = getelementptr i32, ptr addrspace(3) %out, i32 45061 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire5062 ret void5063}5064 5065define amdgpu_kernel void @local_workgroup_monotonic_seq_cst_cmpxchg(5066; GFX6-LABEL: local_workgroup_monotonic_seq_cst_cmpxchg:5067; GFX6: ; %bb.0: ; %entry5068; GFX6-NEXT: s_load_dword s6, s[8:9], 0x05069; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr65070; GFX6-NEXT: s_load_dword s4, s[8:9], 0x15071; GFX6-NEXT: s_load_dword s5, s[8:9], 0x25072; GFX6-NEXT: s_mov_b32 m0, -15073; GFX6-NEXT: s_waitcnt lgkmcnt(0)5074; GFX6-NEXT: v_mov_b32_e32 v0, s65075; GFX6-NEXT: v_mov_b32_e32 v1, s55076; GFX6-NEXT: v_mov_b32_e32 v2, s45077; GFX6-NEXT: s_waitcnt lgkmcnt(0)5078; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165079; GFX6-NEXT: s_waitcnt lgkmcnt(0)5080; GFX6-NEXT: s_endpgm5081;5082; GFX7-LABEL: local_workgroup_monotonic_seq_cst_cmpxchg:5083; GFX7: ; %bb.0: ; %entry5084; GFX7-NEXT: s_load_dword s6, s[8:9], 0x05085; GFX7-NEXT: s_load_dword s4, s[8:9], 0x15086; GFX7-NEXT: s_load_dword s5, s[8:9], 0x25087; GFX7-NEXT: s_mov_b32 m0, -15088; GFX7-NEXT: s_waitcnt lgkmcnt(0)5089; GFX7-NEXT: v_mov_b32_e32 v0, s65090; GFX7-NEXT: v_mov_b32_e32 v1, s55091; GFX7-NEXT: v_mov_b32_e32 v2, s45092; GFX7-NEXT: s_waitcnt lgkmcnt(0)5093; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165094; GFX7-NEXT: s_waitcnt lgkmcnt(0)5095; GFX7-NEXT: s_endpgm5096;5097; GFX10-WGP-LABEL: local_workgroup_monotonic_seq_cst_cmpxchg:5098; GFX10-WGP: ; %bb.0: ; %entry5099; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x05100; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x45101; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x85102; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)5103; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s65104; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s55105; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s45106; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5107; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x05108; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165109; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)5110; GFX10-WGP-NEXT: buffer_gl0_inv5111; GFX10-WGP-NEXT: s_endpgm5112;5113; GFX10-CU-LABEL: local_workgroup_monotonic_seq_cst_cmpxchg:5114; GFX10-CU: ; %bb.0: ; %entry5115; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x05116; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x45117; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x85118; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)5119; GFX10-CU-NEXT: v_mov_b32_e32 v0, s65120; GFX10-CU-NEXT: v_mov_b32_e32 v1, s55121; GFX10-CU-NEXT: v_mov_b32_e32 v2, s45122; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5123; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x05124; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165125; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)5126; GFX10-CU-NEXT: s_endpgm5127;5128; SKIP-CACHE-INV-LABEL: local_workgroup_monotonic_seq_cst_cmpxchg:5129; SKIP-CACHE-INV: ; %bb.0: ; %entry5130; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x05131; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x15132; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x25133; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -15134; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)5135; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s25136; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s15137; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s05138; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)5139; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165140; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)5141; SKIP-CACHE-INV-NEXT: s_endpgm5142;5143; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_monotonic_seq_cst_cmpxchg:5144; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry5145; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x05146; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x45147; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x85148; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5149; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s65150; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s55151; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s45152; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5153; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165154; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5155; GFX90A-NOTTGSPLIT-NEXT: s_endpgm5156;5157; GFX90A-TGSPLIT-LABEL: local_workgroup_monotonic_seq_cst_cmpxchg:5158; GFX90A-TGSPLIT: ; %bb.0: ; %entry5159; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x05160; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x45161; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x85162; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5163; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s65164; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s55165; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s45166; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5167; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165168; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol5169; GFX90A-TGSPLIT-NEXT: s_endpgm5170;5171; GFX942-NOTTGSPLIT-LABEL: local_workgroup_monotonic_seq_cst_cmpxchg:5172; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry5173; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x05174; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x45175; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x85176; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5177; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s25178; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s15179; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s05180; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5181; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165182; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5183; GFX942-NOTTGSPLIT-NEXT: s_endpgm5184;5185; GFX942-TGSPLIT-LABEL: local_workgroup_monotonic_seq_cst_cmpxchg:5186; GFX942-TGSPLIT: ; %bb.0: ; %entry5187; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x05188; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x45189; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x85190; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5191; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s25192; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s15193; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s05194; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5195; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165196; GFX942-TGSPLIT-NEXT: buffer_inv sc05197; GFX942-TGSPLIT-NEXT: s_endpgm5198;5199; GFX11-WGP-LABEL: local_workgroup_monotonic_seq_cst_cmpxchg:5200; GFX11-WGP: ; %bb.0: ; %entry5201; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x05202; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x45203; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x85204; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)5205; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s25206; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s15207; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s05208; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5209; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x05210; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:165211; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)5212; GFX11-WGP-NEXT: buffer_gl0_inv5213; GFX11-WGP-NEXT: s_endpgm5214;5215; GFX11-CU-LABEL: local_workgroup_monotonic_seq_cst_cmpxchg:5216; GFX11-CU: ; %bb.0: ; %entry5217; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x05218; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x45219; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x85220; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)5221; GFX11-CU-NEXT: v_mov_b32_e32 v0, s25222; GFX11-CU-NEXT: v_mov_b32_e32 v1, s15223; GFX11-CU-NEXT: v_mov_b32_e32 v2, s05224; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5225; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x05226; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:165227; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)5228; GFX11-CU-NEXT: s_endpgm5229;5230; GFX12-WGP-LABEL: local_workgroup_monotonic_seq_cst_cmpxchg:5231; GFX12-WGP: ; %bb.0: ; %entry5232; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x05233; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x45234; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x85235; GFX12-WGP-NEXT: s_wait_kmcnt 0x05236; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s25237; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s15238; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s05239; GFX12-WGP-NEXT: s_wait_bvhcnt 0x05240; GFX12-WGP-NEXT: s_wait_samplecnt 0x05241; GFX12-WGP-NEXT: s_wait_storecnt 0x05242; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x05243; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:165244; GFX12-WGP-NEXT: s_wait_dscnt 0x05245; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE5246; GFX12-WGP-NEXT: s_endpgm5247;5248; GFX12-CU-LABEL: local_workgroup_monotonic_seq_cst_cmpxchg:5249; GFX12-CU: ; %bb.0: ; %entry5250; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x05251; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x45252; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x85253; GFX12-CU-NEXT: s_wait_kmcnt 0x05254; GFX12-CU-NEXT: v_mov_b32_e32 v0, s25255; GFX12-CU-NEXT: v_mov_b32_e32 v1, s15256; GFX12-CU-NEXT: v_mov_b32_e32 v2, s05257; GFX12-CU-NEXT: s_wait_bvhcnt 0x05258; GFX12-CU-NEXT: s_wait_samplecnt 0x05259; GFX12-CU-NEXT: s_wait_storecnt 0x05260; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x05261; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:165262; GFX12-CU-NEXT: s_wait_dscnt 0x05263; GFX12-CU-NEXT: s_endpgm5264;5265; GFX1250-LABEL: local_workgroup_monotonic_seq_cst_cmpxchg:5266; GFX1250: ; %bb.0: ; %entry5267; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 15268; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x05269; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x45270; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x85271; GFX1250-NEXT: s_wait_kmcnt 0x05272; GFX1250-NEXT: v_mov_b32_e32 v0, s25273; GFX1250-NEXT: v_mov_b32_e32 v1, s15274; GFX1250-NEXT: v_mov_b32_e32 v2, s05275; GFX1250-NEXT: s_wait_storecnt 0x05276; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x05277; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:165278; GFX1250-NEXT: s_wait_dscnt 0x05279; GFX1250-NEXT: s_endpgm5280 ptr addrspace(3) %out, i32 %in, i32 %old) {5281entry:5282 %gep = getelementptr i32, ptr addrspace(3) %out, i32 45283 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") monotonic seq_cst5284 ret void5285}5286 5287define amdgpu_kernel void @local_workgroup_acquire_seq_cst_cmpxchg(5288; GFX6-LABEL: local_workgroup_acquire_seq_cst_cmpxchg:5289; GFX6: ; %bb.0: ; %entry5290; GFX6-NEXT: s_load_dword s6, s[8:9], 0x05291; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr65292; GFX6-NEXT: s_load_dword s4, s[8:9], 0x15293; GFX6-NEXT: s_load_dword s5, s[8:9], 0x25294; GFX6-NEXT: s_mov_b32 m0, -15295; GFX6-NEXT: s_waitcnt lgkmcnt(0)5296; GFX6-NEXT: v_mov_b32_e32 v0, s65297; GFX6-NEXT: v_mov_b32_e32 v1, s55298; GFX6-NEXT: v_mov_b32_e32 v2, s45299; GFX6-NEXT: s_waitcnt lgkmcnt(0)5300; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165301; GFX6-NEXT: s_waitcnt lgkmcnt(0)5302; GFX6-NEXT: s_endpgm5303;5304; GFX7-LABEL: local_workgroup_acquire_seq_cst_cmpxchg:5305; GFX7: ; %bb.0: ; %entry5306; GFX7-NEXT: s_load_dword s6, s[8:9], 0x05307; GFX7-NEXT: s_load_dword s4, s[8:9], 0x15308; GFX7-NEXT: s_load_dword s5, s[8:9], 0x25309; GFX7-NEXT: s_mov_b32 m0, -15310; GFX7-NEXT: s_waitcnt lgkmcnt(0)5311; GFX7-NEXT: v_mov_b32_e32 v0, s65312; GFX7-NEXT: v_mov_b32_e32 v1, s55313; GFX7-NEXT: v_mov_b32_e32 v2, s45314; GFX7-NEXT: s_waitcnt lgkmcnt(0)5315; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165316; GFX7-NEXT: s_waitcnt lgkmcnt(0)5317; GFX7-NEXT: s_endpgm5318;5319; GFX10-WGP-LABEL: local_workgroup_acquire_seq_cst_cmpxchg:5320; GFX10-WGP: ; %bb.0: ; %entry5321; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x05322; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x45323; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x85324; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)5325; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s65326; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s55327; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s45328; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5329; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x05330; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165331; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)5332; GFX10-WGP-NEXT: buffer_gl0_inv5333; GFX10-WGP-NEXT: s_endpgm5334;5335; GFX10-CU-LABEL: local_workgroup_acquire_seq_cst_cmpxchg:5336; GFX10-CU: ; %bb.0: ; %entry5337; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x05338; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x45339; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x85340; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)5341; GFX10-CU-NEXT: v_mov_b32_e32 v0, s65342; GFX10-CU-NEXT: v_mov_b32_e32 v1, s55343; GFX10-CU-NEXT: v_mov_b32_e32 v2, s45344; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5345; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x05346; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165347; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)5348; GFX10-CU-NEXT: s_endpgm5349;5350; SKIP-CACHE-INV-LABEL: local_workgroup_acquire_seq_cst_cmpxchg:5351; SKIP-CACHE-INV: ; %bb.0: ; %entry5352; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x05353; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x15354; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x25355; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -15356; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)5357; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s25358; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s15359; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s05360; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)5361; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165362; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)5363; SKIP-CACHE-INV-NEXT: s_endpgm5364;5365; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_acquire_seq_cst_cmpxchg:5366; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry5367; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x05368; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x45369; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x85370; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5371; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s65372; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s55373; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s45374; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5375; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165376; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5377; GFX90A-NOTTGSPLIT-NEXT: s_endpgm5378;5379; GFX90A-TGSPLIT-LABEL: local_workgroup_acquire_seq_cst_cmpxchg:5380; GFX90A-TGSPLIT: ; %bb.0: ; %entry5381; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x05382; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x45383; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x85384; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5385; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s65386; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s55387; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s45388; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5389; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165390; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol5391; GFX90A-TGSPLIT-NEXT: s_endpgm5392;5393; GFX942-NOTTGSPLIT-LABEL: local_workgroup_acquire_seq_cst_cmpxchg:5394; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry5395; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x05396; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x45397; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x85398; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5399; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s25400; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s15401; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s05402; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5403; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165404; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5405; GFX942-NOTTGSPLIT-NEXT: s_endpgm5406;5407; GFX942-TGSPLIT-LABEL: local_workgroup_acquire_seq_cst_cmpxchg:5408; GFX942-TGSPLIT: ; %bb.0: ; %entry5409; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x05410; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x45411; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x85412; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5413; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s25414; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s15415; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s05416; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5417; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165418; GFX942-TGSPLIT-NEXT: buffer_inv sc05419; GFX942-TGSPLIT-NEXT: s_endpgm5420;5421; GFX11-WGP-LABEL: local_workgroup_acquire_seq_cst_cmpxchg:5422; GFX11-WGP: ; %bb.0: ; %entry5423; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x05424; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x45425; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x85426; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)5427; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s25428; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s15429; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s05430; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5431; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x05432; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:165433; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)5434; GFX11-WGP-NEXT: buffer_gl0_inv5435; GFX11-WGP-NEXT: s_endpgm5436;5437; GFX11-CU-LABEL: local_workgroup_acquire_seq_cst_cmpxchg:5438; GFX11-CU: ; %bb.0: ; %entry5439; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x05440; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x45441; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x85442; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)5443; GFX11-CU-NEXT: v_mov_b32_e32 v0, s25444; GFX11-CU-NEXT: v_mov_b32_e32 v1, s15445; GFX11-CU-NEXT: v_mov_b32_e32 v2, s05446; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5447; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x05448; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:165449; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)5450; GFX11-CU-NEXT: s_endpgm5451;5452; GFX12-WGP-LABEL: local_workgroup_acquire_seq_cst_cmpxchg:5453; GFX12-WGP: ; %bb.0: ; %entry5454; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x05455; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x45456; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x85457; GFX12-WGP-NEXT: s_wait_kmcnt 0x05458; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s25459; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s15460; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s05461; GFX12-WGP-NEXT: s_wait_bvhcnt 0x05462; GFX12-WGP-NEXT: s_wait_samplecnt 0x05463; GFX12-WGP-NEXT: s_wait_storecnt 0x05464; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x05465; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:165466; GFX12-WGP-NEXT: s_wait_dscnt 0x05467; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE5468; GFX12-WGP-NEXT: s_endpgm5469;5470; GFX12-CU-LABEL: local_workgroup_acquire_seq_cst_cmpxchg:5471; GFX12-CU: ; %bb.0: ; %entry5472; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x05473; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x45474; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x85475; GFX12-CU-NEXT: s_wait_kmcnt 0x05476; GFX12-CU-NEXT: v_mov_b32_e32 v0, s25477; GFX12-CU-NEXT: v_mov_b32_e32 v1, s15478; GFX12-CU-NEXT: v_mov_b32_e32 v2, s05479; GFX12-CU-NEXT: s_wait_bvhcnt 0x05480; GFX12-CU-NEXT: s_wait_samplecnt 0x05481; GFX12-CU-NEXT: s_wait_storecnt 0x05482; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x05483; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:165484; GFX12-CU-NEXT: s_wait_dscnt 0x05485; GFX12-CU-NEXT: s_endpgm5486;5487; GFX1250-LABEL: local_workgroup_acquire_seq_cst_cmpxchg:5488; GFX1250: ; %bb.0: ; %entry5489; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 15490; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x05491; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x45492; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x85493; GFX1250-NEXT: s_wait_kmcnt 0x05494; GFX1250-NEXT: v_mov_b32_e32 v0, s25495; GFX1250-NEXT: v_mov_b32_e32 v1, s15496; GFX1250-NEXT: v_mov_b32_e32 v2, s05497; GFX1250-NEXT: s_wait_storecnt 0x05498; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x05499; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:165500; GFX1250-NEXT: s_wait_dscnt 0x05501; GFX1250-NEXT: s_endpgm5502 ptr addrspace(3) %out, i32 %in, i32 %old) {5503entry:5504 %gep = getelementptr i32, ptr addrspace(3) %out, i32 45505 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") acquire seq_cst5506 ret void5507}5508 5509define amdgpu_kernel void @local_workgroup_release_seq_cst_cmpxchg(5510; GFX6-LABEL: local_workgroup_release_seq_cst_cmpxchg:5511; GFX6: ; %bb.0: ; %entry5512; GFX6-NEXT: s_load_dword s6, s[8:9], 0x05513; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr65514; GFX6-NEXT: s_load_dword s4, s[8:9], 0x15515; GFX6-NEXT: s_load_dword s5, s[8:9], 0x25516; GFX6-NEXT: s_mov_b32 m0, -15517; GFX6-NEXT: s_waitcnt lgkmcnt(0)5518; GFX6-NEXT: v_mov_b32_e32 v0, s65519; GFX6-NEXT: v_mov_b32_e32 v1, s55520; GFX6-NEXT: v_mov_b32_e32 v2, s45521; GFX6-NEXT: s_waitcnt lgkmcnt(0)5522; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165523; GFX6-NEXT: s_waitcnt lgkmcnt(0)5524; GFX6-NEXT: s_endpgm5525;5526; GFX7-LABEL: local_workgroup_release_seq_cst_cmpxchg:5527; GFX7: ; %bb.0: ; %entry5528; GFX7-NEXT: s_load_dword s6, s[8:9], 0x05529; GFX7-NEXT: s_load_dword s4, s[8:9], 0x15530; GFX7-NEXT: s_load_dword s5, s[8:9], 0x25531; GFX7-NEXT: s_mov_b32 m0, -15532; GFX7-NEXT: s_waitcnt lgkmcnt(0)5533; GFX7-NEXT: v_mov_b32_e32 v0, s65534; GFX7-NEXT: v_mov_b32_e32 v1, s55535; GFX7-NEXT: v_mov_b32_e32 v2, s45536; GFX7-NEXT: s_waitcnt lgkmcnt(0)5537; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165538; GFX7-NEXT: s_waitcnt lgkmcnt(0)5539; GFX7-NEXT: s_endpgm5540;5541; GFX10-WGP-LABEL: local_workgroup_release_seq_cst_cmpxchg:5542; GFX10-WGP: ; %bb.0: ; %entry5543; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x05544; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x45545; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x85546; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)5547; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s65548; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s55549; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s45550; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5551; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x05552; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165553; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)5554; GFX10-WGP-NEXT: buffer_gl0_inv5555; GFX10-WGP-NEXT: s_endpgm5556;5557; GFX10-CU-LABEL: local_workgroup_release_seq_cst_cmpxchg:5558; GFX10-CU: ; %bb.0: ; %entry5559; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x05560; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x45561; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x85562; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)5563; GFX10-CU-NEXT: v_mov_b32_e32 v0, s65564; GFX10-CU-NEXT: v_mov_b32_e32 v1, s55565; GFX10-CU-NEXT: v_mov_b32_e32 v2, s45566; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5567; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x05568; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165569; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)5570; GFX10-CU-NEXT: s_endpgm5571;5572; SKIP-CACHE-INV-LABEL: local_workgroup_release_seq_cst_cmpxchg:5573; SKIP-CACHE-INV: ; %bb.0: ; %entry5574; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x05575; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x15576; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x25577; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -15578; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)5579; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s25580; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s15581; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s05582; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)5583; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165584; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)5585; SKIP-CACHE-INV-NEXT: s_endpgm5586;5587; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_release_seq_cst_cmpxchg:5588; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry5589; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x05590; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x45591; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x85592; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5593; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s65594; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s55595; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s45596; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5597; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165598; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5599; GFX90A-NOTTGSPLIT-NEXT: s_endpgm5600;5601; GFX90A-TGSPLIT-LABEL: local_workgroup_release_seq_cst_cmpxchg:5602; GFX90A-TGSPLIT: ; %bb.0: ; %entry5603; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x05604; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x45605; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x85606; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5607; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s65608; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s55609; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s45610; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5611; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165612; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol5613; GFX90A-TGSPLIT-NEXT: s_endpgm5614;5615; GFX942-NOTTGSPLIT-LABEL: local_workgroup_release_seq_cst_cmpxchg:5616; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry5617; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x05618; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x45619; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x85620; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5621; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s25622; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s15623; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s05624; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5625; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165626; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5627; GFX942-NOTTGSPLIT-NEXT: s_endpgm5628;5629; GFX942-TGSPLIT-LABEL: local_workgroup_release_seq_cst_cmpxchg:5630; GFX942-TGSPLIT: ; %bb.0: ; %entry5631; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x05632; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x45633; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x85634; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5635; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s25636; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s15637; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s05638; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5639; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165640; GFX942-TGSPLIT-NEXT: buffer_inv sc05641; GFX942-TGSPLIT-NEXT: s_endpgm5642;5643; GFX11-WGP-LABEL: local_workgroup_release_seq_cst_cmpxchg:5644; GFX11-WGP: ; %bb.0: ; %entry5645; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x05646; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x45647; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x85648; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)5649; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s25650; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s15651; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s05652; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5653; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x05654; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:165655; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)5656; GFX11-WGP-NEXT: buffer_gl0_inv5657; GFX11-WGP-NEXT: s_endpgm5658;5659; GFX11-CU-LABEL: local_workgroup_release_seq_cst_cmpxchg:5660; GFX11-CU: ; %bb.0: ; %entry5661; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x05662; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x45663; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x85664; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)5665; GFX11-CU-NEXT: v_mov_b32_e32 v0, s25666; GFX11-CU-NEXT: v_mov_b32_e32 v1, s15667; GFX11-CU-NEXT: v_mov_b32_e32 v2, s05668; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5669; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x05670; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:165671; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)5672; GFX11-CU-NEXT: s_endpgm5673;5674; GFX12-WGP-LABEL: local_workgroup_release_seq_cst_cmpxchg:5675; GFX12-WGP: ; %bb.0: ; %entry5676; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x05677; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x45678; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x85679; GFX12-WGP-NEXT: s_wait_kmcnt 0x05680; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s25681; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s15682; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s05683; GFX12-WGP-NEXT: s_wait_bvhcnt 0x05684; GFX12-WGP-NEXT: s_wait_samplecnt 0x05685; GFX12-WGP-NEXT: s_wait_storecnt 0x05686; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x05687; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:165688; GFX12-WGP-NEXT: s_wait_dscnt 0x05689; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE5690; GFX12-WGP-NEXT: s_endpgm5691;5692; GFX12-CU-LABEL: local_workgroup_release_seq_cst_cmpxchg:5693; GFX12-CU: ; %bb.0: ; %entry5694; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x05695; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x45696; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x85697; GFX12-CU-NEXT: s_wait_kmcnt 0x05698; GFX12-CU-NEXT: v_mov_b32_e32 v0, s25699; GFX12-CU-NEXT: v_mov_b32_e32 v1, s15700; GFX12-CU-NEXT: v_mov_b32_e32 v2, s05701; GFX12-CU-NEXT: s_wait_bvhcnt 0x05702; GFX12-CU-NEXT: s_wait_samplecnt 0x05703; GFX12-CU-NEXT: s_wait_storecnt 0x05704; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x05705; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:165706; GFX12-CU-NEXT: s_wait_dscnt 0x05707; GFX12-CU-NEXT: s_endpgm5708;5709; GFX1250-LABEL: local_workgroup_release_seq_cst_cmpxchg:5710; GFX1250: ; %bb.0: ; %entry5711; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 15712; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x05713; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x45714; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x85715; GFX1250-NEXT: s_wait_kmcnt 0x05716; GFX1250-NEXT: v_mov_b32_e32 v0, s25717; GFX1250-NEXT: v_mov_b32_e32 v1, s15718; GFX1250-NEXT: v_mov_b32_e32 v2, s05719; GFX1250-NEXT: s_wait_storecnt 0x05720; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x05721; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:165722; GFX1250-NEXT: s_wait_dscnt 0x05723; GFX1250-NEXT: s_endpgm5724 ptr addrspace(3) %out, i32 %in, i32 %old) {5725entry:5726 %gep = getelementptr i32, ptr addrspace(3) %out, i32 45727 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") release seq_cst5728 ret void5729}5730 5731define amdgpu_kernel void @local_workgroup_acq_rel_seq_cst_cmpxchg(5732; GFX6-LABEL: local_workgroup_acq_rel_seq_cst_cmpxchg:5733; GFX6: ; %bb.0: ; %entry5734; GFX6-NEXT: s_load_dword s6, s[8:9], 0x05735; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr65736; GFX6-NEXT: s_load_dword s4, s[8:9], 0x15737; GFX6-NEXT: s_load_dword s5, s[8:9], 0x25738; GFX6-NEXT: s_mov_b32 m0, -15739; GFX6-NEXT: s_waitcnt lgkmcnt(0)5740; GFX6-NEXT: v_mov_b32_e32 v0, s65741; GFX6-NEXT: v_mov_b32_e32 v1, s55742; GFX6-NEXT: v_mov_b32_e32 v2, s45743; GFX6-NEXT: s_waitcnt lgkmcnt(0)5744; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165745; GFX6-NEXT: s_waitcnt lgkmcnt(0)5746; GFX6-NEXT: s_endpgm5747;5748; GFX7-LABEL: local_workgroup_acq_rel_seq_cst_cmpxchg:5749; GFX7: ; %bb.0: ; %entry5750; GFX7-NEXT: s_load_dword s6, s[8:9], 0x05751; GFX7-NEXT: s_load_dword s4, s[8:9], 0x15752; GFX7-NEXT: s_load_dword s5, s[8:9], 0x25753; GFX7-NEXT: s_mov_b32 m0, -15754; GFX7-NEXT: s_waitcnt lgkmcnt(0)5755; GFX7-NEXT: v_mov_b32_e32 v0, s65756; GFX7-NEXT: v_mov_b32_e32 v1, s55757; GFX7-NEXT: v_mov_b32_e32 v2, s45758; GFX7-NEXT: s_waitcnt lgkmcnt(0)5759; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165760; GFX7-NEXT: s_waitcnt lgkmcnt(0)5761; GFX7-NEXT: s_endpgm5762;5763; GFX10-WGP-LABEL: local_workgroup_acq_rel_seq_cst_cmpxchg:5764; GFX10-WGP: ; %bb.0: ; %entry5765; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x05766; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x45767; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x85768; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)5769; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s65770; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s55771; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s45772; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5773; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x05774; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165775; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)5776; GFX10-WGP-NEXT: buffer_gl0_inv5777; GFX10-WGP-NEXT: s_endpgm5778;5779; GFX10-CU-LABEL: local_workgroup_acq_rel_seq_cst_cmpxchg:5780; GFX10-CU: ; %bb.0: ; %entry5781; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x05782; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x45783; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x85784; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)5785; GFX10-CU-NEXT: v_mov_b32_e32 v0, s65786; GFX10-CU-NEXT: v_mov_b32_e32 v1, s55787; GFX10-CU-NEXT: v_mov_b32_e32 v2, s45788; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5789; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x05790; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165791; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)5792; GFX10-CU-NEXT: s_endpgm5793;5794; SKIP-CACHE-INV-LABEL: local_workgroup_acq_rel_seq_cst_cmpxchg:5795; SKIP-CACHE-INV: ; %bb.0: ; %entry5796; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x05797; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x15798; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x25799; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -15800; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)5801; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s25802; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s15803; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s05804; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)5805; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165806; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)5807; SKIP-CACHE-INV-NEXT: s_endpgm5808;5809; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_acq_rel_seq_cst_cmpxchg:5810; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry5811; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x05812; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x45813; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x85814; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5815; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s65816; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s55817; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s45818; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5819; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165820; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5821; GFX90A-NOTTGSPLIT-NEXT: s_endpgm5822;5823; GFX90A-TGSPLIT-LABEL: local_workgroup_acq_rel_seq_cst_cmpxchg:5824; GFX90A-TGSPLIT: ; %bb.0: ; %entry5825; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x05826; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x45827; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x85828; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5829; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s65830; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s55831; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s45832; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5833; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165834; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol5835; GFX90A-TGSPLIT-NEXT: s_endpgm5836;5837; GFX942-NOTTGSPLIT-LABEL: local_workgroup_acq_rel_seq_cst_cmpxchg:5838; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry5839; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x05840; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x45841; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x85842; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5843; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s25844; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s15845; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s05846; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5847; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165848; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5849; GFX942-NOTTGSPLIT-NEXT: s_endpgm5850;5851; GFX942-TGSPLIT-LABEL: local_workgroup_acq_rel_seq_cst_cmpxchg:5852; GFX942-TGSPLIT: ; %bb.0: ; %entry5853; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x05854; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x45855; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x85856; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)5857; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s25858; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s15859; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s05860; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5861; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165862; GFX942-TGSPLIT-NEXT: buffer_inv sc05863; GFX942-TGSPLIT-NEXT: s_endpgm5864;5865; GFX11-WGP-LABEL: local_workgroup_acq_rel_seq_cst_cmpxchg:5866; GFX11-WGP: ; %bb.0: ; %entry5867; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x05868; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x45869; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x85870; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)5871; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s25872; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s15873; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s05874; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5875; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x05876; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:165877; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)5878; GFX11-WGP-NEXT: buffer_gl0_inv5879; GFX11-WGP-NEXT: s_endpgm5880;5881; GFX11-CU-LABEL: local_workgroup_acq_rel_seq_cst_cmpxchg:5882; GFX11-CU: ; %bb.0: ; %entry5883; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x05884; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x45885; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x85886; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)5887; GFX11-CU-NEXT: v_mov_b32_e32 v0, s25888; GFX11-CU-NEXT: v_mov_b32_e32 v1, s15889; GFX11-CU-NEXT: v_mov_b32_e32 v2, s05890; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5891; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x05892; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:165893; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)5894; GFX11-CU-NEXT: s_endpgm5895;5896; GFX12-WGP-LABEL: local_workgroup_acq_rel_seq_cst_cmpxchg:5897; GFX12-WGP: ; %bb.0: ; %entry5898; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x05899; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x45900; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x85901; GFX12-WGP-NEXT: s_wait_kmcnt 0x05902; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s25903; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s15904; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s05905; GFX12-WGP-NEXT: s_wait_bvhcnt 0x05906; GFX12-WGP-NEXT: s_wait_samplecnt 0x05907; GFX12-WGP-NEXT: s_wait_storecnt 0x05908; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x05909; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:165910; GFX12-WGP-NEXT: s_wait_dscnt 0x05911; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE5912; GFX12-WGP-NEXT: s_endpgm5913;5914; GFX12-CU-LABEL: local_workgroup_acq_rel_seq_cst_cmpxchg:5915; GFX12-CU: ; %bb.0: ; %entry5916; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x05917; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x45918; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x85919; GFX12-CU-NEXT: s_wait_kmcnt 0x05920; GFX12-CU-NEXT: v_mov_b32_e32 v0, s25921; GFX12-CU-NEXT: v_mov_b32_e32 v1, s15922; GFX12-CU-NEXT: v_mov_b32_e32 v2, s05923; GFX12-CU-NEXT: s_wait_bvhcnt 0x05924; GFX12-CU-NEXT: s_wait_samplecnt 0x05925; GFX12-CU-NEXT: s_wait_storecnt 0x05926; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x05927; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:165928; GFX12-CU-NEXT: s_wait_dscnt 0x05929; GFX12-CU-NEXT: s_endpgm5930;5931; GFX1250-LABEL: local_workgroup_acq_rel_seq_cst_cmpxchg:5932; GFX1250: ; %bb.0: ; %entry5933; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 15934; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x05935; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x45936; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x85937; GFX1250-NEXT: s_wait_kmcnt 0x05938; GFX1250-NEXT: v_mov_b32_e32 v0, s25939; GFX1250-NEXT: v_mov_b32_e32 v1, s15940; GFX1250-NEXT: v_mov_b32_e32 v2, s05941; GFX1250-NEXT: s_wait_storecnt 0x05942; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x05943; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:165944; GFX1250-NEXT: s_wait_dscnt 0x05945; GFX1250-NEXT: s_endpgm5946 ptr addrspace(3) %out, i32 %in, i32 %old) {5947entry:5948 %gep = getelementptr i32, ptr addrspace(3) %out, i32 45949 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel seq_cst5950 ret void5951}5952 5953define amdgpu_kernel void @local_workgroup_seq_cst_seq_cst_cmpxchg(5954; GFX6-LABEL: local_workgroup_seq_cst_seq_cst_cmpxchg:5955; GFX6: ; %bb.0: ; %entry5956; GFX6-NEXT: s_load_dword s6, s[8:9], 0x05957; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr65958; GFX6-NEXT: s_load_dword s4, s[8:9], 0x15959; GFX6-NEXT: s_load_dword s5, s[8:9], 0x25960; GFX6-NEXT: s_mov_b32 m0, -15961; GFX6-NEXT: s_waitcnt lgkmcnt(0)5962; GFX6-NEXT: v_mov_b32_e32 v0, s65963; GFX6-NEXT: v_mov_b32_e32 v1, s55964; GFX6-NEXT: v_mov_b32_e32 v2, s45965; GFX6-NEXT: s_waitcnt lgkmcnt(0)5966; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165967; GFX6-NEXT: s_waitcnt lgkmcnt(0)5968; GFX6-NEXT: s_endpgm5969;5970; GFX7-LABEL: local_workgroup_seq_cst_seq_cst_cmpxchg:5971; GFX7: ; %bb.0: ; %entry5972; GFX7-NEXT: s_load_dword s6, s[8:9], 0x05973; GFX7-NEXT: s_load_dword s4, s[8:9], 0x15974; GFX7-NEXT: s_load_dword s5, s[8:9], 0x25975; GFX7-NEXT: s_mov_b32 m0, -15976; GFX7-NEXT: s_waitcnt lgkmcnt(0)5977; GFX7-NEXT: v_mov_b32_e32 v0, s65978; GFX7-NEXT: v_mov_b32_e32 v1, s55979; GFX7-NEXT: v_mov_b32_e32 v2, s45980; GFX7-NEXT: s_waitcnt lgkmcnt(0)5981; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165982; GFX7-NEXT: s_waitcnt lgkmcnt(0)5983; GFX7-NEXT: s_endpgm5984;5985; GFX10-WGP-LABEL: local_workgroup_seq_cst_seq_cst_cmpxchg:5986; GFX10-WGP: ; %bb.0: ; %entry5987; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x05988; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x45989; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x85990; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)5991; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s65992; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s55993; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s45994; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5995; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x05996; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:165997; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)5998; GFX10-WGP-NEXT: buffer_gl0_inv5999; GFX10-WGP-NEXT: s_endpgm6000;6001; GFX10-CU-LABEL: local_workgroup_seq_cst_seq_cst_cmpxchg:6002; GFX10-CU: ; %bb.0: ; %entry6003; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x06004; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x46005; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x86006; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)6007; GFX10-CU-NEXT: v_mov_b32_e32 v0, s66008; GFX10-CU-NEXT: v_mov_b32_e32 v1, s56009; GFX10-CU-NEXT: v_mov_b32_e32 v2, s46010; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6011; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x06012; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:166013; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)6014; GFX10-CU-NEXT: s_endpgm6015;6016; SKIP-CACHE-INV-LABEL: local_workgroup_seq_cst_seq_cst_cmpxchg:6017; SKIP-CACHE-INV: ; %bb.0: ; %entry6018; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x06019; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x16020; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x26021; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -16022; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)6023; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s26024; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s16025; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s06026; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)6027; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:166028; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)6029; SKIP-CACHE-INV-NEXT: s_endpgm6030;6031; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_seq_cst_seq_cst_cmpxchg:6032; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry6033; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x06034; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x46035; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x86036; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6037; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s66038; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s56039; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s46040; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6041; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:166042; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6043; GFX90A-NOTTGSPLIT-NEXT: s_endpgm6044;6045; GFX90A-TGSPLIT-LABEL: local_workgroup_seq_cst_seq_cst_cmpxchg:6046; GFX90A-TGSPLIT: ; %bb.0: ; %entry6047; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x06048; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x46049; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x86050; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6051; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s66052; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s56053; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s46054; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6055; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:166056; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol6057; GFX90A-TGSPLIT-NEXT: s_endpgm6058;6059; GFX942-NOTTGSPLIT-LABEL: local_workgroup_seq_cst_seq_cst_cmpxchg:6060; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry6061; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x06062; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x46063; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x86064; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6065; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s26066; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s16067; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s06068; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6069; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:166070; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6071; GFX942-NOTTGSPLIT-NEXT: s_endpgm6072;6073; GFX942-TGSPLIT-LABEL: local_workgroup_seq_cst_seq_cst_cmpxchg:6074; GFX942-TGSPLIT: ; %bb.0: ; %entry6075; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x06076; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x46077; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x86078; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6079; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s26080; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s16081; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s06082; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6083; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:166084; GFX942-TGSPLIT-NEXT: buffer_inv sc06085; GFX942-TGSPLIT-NEXT: s_endpgm6086;6087; GFX11-WGP-LABEL: local_workgroup_seq_cst_seq_cst_cmpxchg:6088; GFX11-WGP: ; %bb.0: ; %entry6089; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x06090; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x46091; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x86092; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)6093; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s26094; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s16095; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s06096; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6097; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x06098; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:166099; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)6100; GFX11-WGP-NEXT: buffer_gl0_inv6101; GFX11-WGP-NEXT: s_endpgm6102;6103; GFX11-CU-LABEL: local_workgroup_seq_cst_seq_cst_cmpxchg:6104; GFX11-CU: ; %bb.0: ; %entry6105; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x06106; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x46107; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x86108; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)6109; GFX11-CU-NEXT: v_mov_b32_e32 v0, s26110; GFX11-CU-NEXT: v_mov_b32_e32 v1, s16111; GFX11-CU-NEXT: v_mov_b32_e32 v2, s06112; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6113; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x06114; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:166115; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)6116; GFX11-CU-NEXT: s_endpgm6117;6118; GFX12-WGP-LABEL: local_workgroup_seq_cst_seq_cst_cmpxchg:6119; GFX12-WGP: ; %bb.0: ; %entry6120; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x06121; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x46122; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x86123; GFX12-WGP-NEXT: s_wait_kmcnt 0x06124; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s26125; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s16126; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s06127; GFX12-WGP-NEXT: s_wait_bvhcnt 0x06128; GFX12-WGP-NEXT: s_wait_samplecnt 0x06129; GFX12-WGP-NEXT: s_wait_storecnt 0x06130; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x06131; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:166132; GFX12-WGP-NEXT: s_wait_dscnt 0x06133; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE6134; GFX12-WGP-NEXT: s_endpgm6135;6136; GFX12-CU-LABEL: local_workgroup_seq_cst_seq_cst_cmpxchg:6137; GFX12-CU: ; %bb.0: ; %entry6138; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x06139; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x46140; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x86141; GFX12-CU-NEXT: s_wait_kmcnt 0x06142; GFX12-CU-NEXT: v_mov_b32_e32 v0, s26143; GFX12-CU-NEXT: v_mov_b32_e32 v1, s16144; GFX12-CU-NEXT: v_mov_b32_e32 v2, s06145; GFX12-CU-NEXT: s_wait_bvhcnt 0x06146; GFX12-CU-NEXT: s_wait_samplecnt 0x06147; GFX12-CU-NEXT: s_wait_storecnt 0x06148; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x06149; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:166150; GFX12-CU-NEXT: s_wait_dscnt 0x06151; GFX12-CU-NEXT: s_endpgm6152;6153; GFX1250-LABEL: local_workgroup_seq_cst_seq_cst_cmpxchg:6154; GFX1250: ; %bb.0: ; %entry6155; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 16156; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x06157; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x46158; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x86159; GFX1250-NEXT: s_wait_kmcnt 0x06160; GFX1250-NEXT: v_mov_b32_e32 v0, s26161; GFX1250-NEXT: v_mov_b32_e32 v1, s16162; GFX1250-NEXT: v_mov_b32_e32 v2, s06163; GFX1250-NEXT: s_wait_storecnt 0x06164; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x06165; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:166166; GFX1250-NEXT: s_wait_dscnt 0x06167; GFX1250-NEXT: s_endpgm6168 ptr addrspace(3) %out, i32 %in, i32 %old) {6169entry:6170 %gep = getelementptr i32, ptr addrspace(3) %out, i32 46171 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst6172 ret void6173}6174 6175define amdgpu_kernel void @local_workgroup_monotonic_monotonic_ret_cmpxchg(6176; GFX6-LABEL: local_workgroup_monotonic_monotonic_ret_cmpxchg:6177; GFX6: ; %bb.0: ; %entry6178; GFX6-NEXT: s_load_dword s4, s[8:9], 0x06179; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr46180; GFX6-NEXT: s_load_dword s5, s[8:9], 0x16181; GFX6-NEXT: s_load_dword s6, s[8:9], 0x26182; GFX6-NEXT: s_mov_b32 m0, -16183; GFX6-NEXT: s_waitcnt lgkmcnt(0)6184; GFX6-NEXT: v_mov_b32_e32 v0, s46185; GFX6-NEXT: v_mov_b32_e32 v1, s66186; GFX6-NEXT: v_mov_b32_e32 v2, s56187; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166188; GFX6-NEXT: s_mov_b32 m0, -16189; GFX6-NEXT: v_mov_b32_e32 v0, s46190; GFX6-NEXT: s_waitcnt lgkmcnt(0)6191; GFX6-NEXT: ds_write_b32 v0, v16192; GFX6-NEXT: s_endpgm6193;6194; GFX7-LABEL: local_workgroup_monotonic_monotonic_ret_cmpxchg:6195; GFX7: ; %bb.0: ; %entry6196; GFX7-NEXT: s_load_dword s4, s[8:9], 0x06197; GFX7-NEXT: s_load_dword s5, s[8:9], 0x16198; GFX7-NEXT: s_load_dword s6, s[8:9], 0x26199; GFX7-NEXT: s_mov_b32 m0, -16200; GFX7-NEXT: s_waitcnt lgkmcnt(0)6201; GFX7-NEXT: v_mov_b32_e32 v0, s46202; GFX7-NEXT: v_mov_b32_e32 v1, s66203; GFX7-NEXT: v_mov_b32_e32 v2, s56204; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166205; GFX7-NEXT: s_mov_b32 m0, -16206; GFX7-NEXT: v_mov_b32_e32 v0, s46207; GFX7-NEXT: s_waitcnt lgkmcnt(0)6208; GFX7-NEXT: ds_write_b32 v0, v16209; GFX7-NEXT: s_endpgm6210;6211; GFX10-WGP-LABEL: local_workgroup_monotonic_monotonic_ret_cmpxchg:6212; GFX10-WGP: ; %bb.0: ; %entry6213; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x06214; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x46215; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x86216; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)6217; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s46218; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s66219; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s56220; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166221; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s46222; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)6223; GFX10-WGP-NEXT: ds_write_b32 v0, v16224; GFX10-WGP-NEXT: s_endpgm6225;6226; GFX10-CU-LABEL: local_workgroup_monotonic_monotonic_ret_cmpxchg:6227; GFX10-CU: ; %bb.0: ; %entry6228; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x06229; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x46230; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x86231; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)6232; GFX10-CU-NEXT: v_mov_b32_e32 v0, s46233; GFX10-CU-NEXT: v_mov_b32_e32 v1, s66234; GFX10-CU-NEXT: v_mov_b32_e32 v2, s56235; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166236; GFX10-CU-NEXT: v_mov_b32_e32 v0, s46237; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)6238; GFX10-CU-NEXT: ds_write_b32 v0, v16239; GFX10-CU-NEXT: s_endpgm6240;6241; SKIP-CACHE-INV-LABEL: local_workgroup_monotonic_monotonic_ret_cmpxchg:6242; SKIP-CACHE-INV: ; %bb.0: ; %entry6243; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x06244; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x16245; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x26246; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -16247; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)6248; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s06249; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s26250; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s16251; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166252; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -16253; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s06254; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)6255; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v16256; SKIP-CACHE-INV-NEXT: s_endpgm6257;6258; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_monotonic_monotonic_ret_cmpxchg:6259; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry6260; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x06261; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x46262; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x86263; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6264; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s46265; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s66266; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s56267; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166268; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s46269; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6270; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v16271; GFX90A-NOTTGSPLIT-NEXT: s_endpgm6272;6273; GFX90A-TGSPLIT-LABEL: local_workgroup_monotonic_monotonic_ret_cmpxchg:6274; GFX90A-TGSPLIT: ; %bb.0: ; %entry6275; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x06276; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x46277; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x86278; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6279; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s46280; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s66281; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s56282; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166283; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s46284; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6285; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v16286; GFX90A-TGSPLIT-NEXT: s_endpgm6287;6288; GFX942-NOTTGSPLIT-LABEL: local_workgroup_monotonic_monotonic_ret_cmpxchg:6289; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry6290; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x06291; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x46292; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x86293; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6294; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s06295; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s26296; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s16297; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166298; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s06299; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6300; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v16301; GFX942-NOTTGSPLIT-NEXT: s_endpgm6302;6303; GFX942-TGSPLIT-LABEL: local_workgroup_monotonic_monotonic_ret_cmpxchg:6304; GFX942-TGSPLIT: ; %bb.0: ; %entry6305; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x06306; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x46307; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x86308; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6309; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s06310; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s26311; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s16312; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166313; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s06314; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6315; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v16316; GFX942-TGSPLIT-NEXT: s_endpgm6317;6318; GFX11-WGP-LABEL: local_workgroup_monotonic_monotonic_ret_cmpxchg:6319; GFX11-WGP: ; %bb.0: ; %entry6320; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x06321; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x46322; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x86323; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)6324; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s06325; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s26326; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s16327; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:166328; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s06329; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)6330; GFX11-WGP-NEXT: ds_store_b32 v0, v16331; GFX11-WGP-NEXT: s_endpgm6332;6333; GFX11-CU-LABEL: local_workgroup_monotonic_monotonic_ret_cmpxchg:6334; GFX11-CU: ; %bb.0: ; %entry6335; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x06336; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x46337; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x86338; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)6339; GFX11-CU-NEXT: v_mov_b32_e32 v0, s06340; GFX11-CU-NEXT: v_mov_b32_e32 v1, s26341; GFX11-CU-NEXT: v_mov_b32_e32 v2, s16342; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:166343; GFX11-CU-NEXT: v_mov_b32_e32 v0, s06344; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)6345; GFX11-CU-NEXT: ds_store_b32 v0, v16346; GFX11-CU-NEXT: s_endpgm6347;6348; GFX12-WGP-LABEL: local_workgroup_monotonic_monotonic_ret_cmpxchg:6349; GFX12-WGP: ; %bb.0: ; %entry6350; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x06351; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x46352; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x86353; GFX12-WGP-NEXT: s_wait_kmcnt 0x06354; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s06355; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s26356; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s16357; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:166358; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s06359; GFX12-WGP-NEXT: s_wait_dscnt 0x06360; GFX12-WGP-NEXT: ds_store_b32 v0, v16361; GFX12-WGP-NEXT: s_endpgm6362;6363; GFX12-CU-LABEL: local_workgroup_monotonic_monotonic_ret_cmpxchg:6364; GFX12-CU: ; %bb.0: ; %entry6365; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x06366; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x46367; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x86368; GFX12-CU-NEXT: s_wait_kmcnt 0x06369; GFX12-CU-NEXT: v_mov_b32_e32 v0, s06370; GFX12-CU-NEXT: v_mov_b32_e32 v1, s26371; GFX12-CU-NEXT: v_mov_b32_e32 v2, s16372; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:166373; GFX12-CU-NEXT: v_mov_b32_e32 v0, s06374; GFX12-CU-NEXT: s_wait_dscnt 0x06375; GFX12-CU-NEXT: ds_store_b32 v0, v16376; GFX12-CU-NEXT: s_endpgm6377;6378; GFX1250-LABEL: local_workgroup_monotonic_monotonic_ret_cmpxchg:6379; GFX1250: ; %bb.0: ; %entry6380; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 16381; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x06382; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x46383; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x86384; GFX1250-NEXT: s_wait_kmcnt 0x06385; GFX1250-NEXT: v_mov_b32_e32 v0, s06386; GFX1250-NEXT: v_mov_b32_e32 v1, s26387; GFX1250-NEXT: v_mov_b32_e32 v2, s16388; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:166389; GFX1250-NEXT: v_mov_b32_e32 v0, s06390; GFX1250-NEXT: s_wait_dscnt 0x06391; GFX1250-NEXT: ds_store_b32 v0, v16392; GFX1250-NEXT: s_endpgm6393 ptr addrspace(3) %out, i32 %in, i32 %old) {6394entry:6395 %gep = getelementptr i32, ptr addrspace(3) %out, i32 46396 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") monotonic monotonic6397 %val0 = extractvalue { i32, i1 } %val, 06398 store i32 %val0, ptr addrspace(3) %out, align 46399 ret void6400}6401 6402define amdgpu_kernel void @local_workgroup_acquire_monotonic_ret_cmpxchg(6403; GFX6-LABEL: local_workgroup_acquire_monotonic_ret_cmpxchg:6404; GFX6: ; %bb.0: ; %entry6405; GFX6-NEXT: s_load_dword s4, s[8:9], 0x06406; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr46407; GFX6-NEXT: s_load_dword s5, s[8:9], 0x16408; GFX6-NEXT: s_load_dword s6, s[8:9], 0x26409; GFX6-NEXT: s_mov_b32 m0, -16410; GFX6-NEXT: s_waitcnt lgkmcnt(0)6411; GFX6-NEXT: v_mov_b32_e32 v0, s46412; GFX6-NEXT: v_mov_b32_e32 v1, s66413; GFX6-NEXT: v_mov_b32_e32 v2, s56414; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166415; GFX6-NEXT: s_waitcnt lgkmcnt(0)6416; GFX6-NEXT: s_mov_b32 m0, -16417; GFX6-NEXT: v_mov_b32_e32 v0, s46418; GFX6-NEXT: ds_write_b32 v0, v16419; GFX6-NEXT: s_endpgm6420;6421; GFX7-LABEL: local_workgroup_acquire_monotonic_ret_cmpxchg:6422; GFX7: ; %bb.0: ; %entry6423; GFX7-NEXT: s_load_dword s4, s[8:9], 0x06424; GFX7-NEXT: s_load_dword s5, s[8:9], 0x16425; GFX7-NEXT: s_load_dword s6, s[8:9], 0x26426; GFX7-NEXT: s_mov_b32 m0, -16427; GFX7-NEXT: s_waitcnt lgkmcnt(0)6428; GFX7-NEXT: v_mov_b32_e32 v0, s46429; GFX7-NEXT: v_mov_b32_e32 v1, s66430; GFX7-NEXT: v_mov_b32_e32 v2, s56431; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166432; GFX7-NEXT: s_waitcnt lgkmcnt(0)6433; GFX7-NEXT: s_mov_b32 m0, -16434; GFX7-NEXT: v_mov_b32_e32 v0, s46435; GFX7-NEXT: ds_write_b32 v0, v16436; GFX7-NEXT: s_endpgm6437;6438; GFX10-WGP-LABEL: local_workgroup_acquire_monotonic_ret_cmpxchg:6439; GFX10-WGP: ; %bb.0: ; %entry6440; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x06441; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x46442; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x86443; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)6444; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s46445; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s66446; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s56447; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166448; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)6449; GFX10-WGP-NEXT: buffer_gl0_inv6450; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s46451; GFX10-WGP-NEXT: ds_write_b32 v0, v16452; GFX10-WGP-NEXT: s_endpgm6453;6454; GFX10-CU-LABEL: local_workgroup_acquire_monotonic_ret_cmpxchg:6455; GFX10-CU: ; %bb.0: ; %entry6456; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x06457; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x46458; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x86459; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)6460; GFX10-CU-NEXT: v_mov_b32_e32 v0, s46461; GFX10-CU-NEXT: v_mov_b32_e32 v1, s66462; GFX10-CU-NEXT: v_mov_b32_e32 v2, s56463; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166464; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)6465; GFX10-CU-NEXT: v_mov_b32_e32 v0, s46466; GFX10-CU-NEXT: ds_write_b32 v0, v16467; GFX10-CU-NEXT: s_endpgm6468;6469; SKIP-CACHE-INV-LABEL: local_workgroup_acquire_monotonic_ret_cmpxchg:6470; SKIP-CACHE-INV: ; %bb.0: ; %entry6471; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x06472; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x16473; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x26474; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -16475; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)6476; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s06477; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s26478; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s16479; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166480; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)6481; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -16482; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s06483; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v16484; SKIP-CACHE-INV-NEXT: s_endpgm6485;6486; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_acquire_monotonic_ret_cmpxchg:6487; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry6488; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x06489; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x46490; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x86491; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6492; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s46493; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s66494; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s56495; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166496; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6497; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s46498; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v16499; GFX90A-NOTTGSPLIT-NEXT: s_endpgm6500;6501; GFX90A-TGSPLIT-LABEL: local_workgroup_acquire_monotonic_ret_cmpxchg:6502; GFX90A-TGSPLIT: ; %bb.0: ; %entry6503; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x06504; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x46505; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x86506; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6507; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s46508; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s66509; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s56510; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166511; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol6512; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s46513; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6514; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v16515; GFX90A-TGSPLIT-NEXT: s_endpgm6516;6517; GFX942-NOTTGSPLIT-LABEL: local_workgroup_acquire_monotonic_ret_cmpxchg:6518; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry6519; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x06520; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x46521; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x86522; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6523; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s06524; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s26525; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s16526; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166527; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6528; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s06529; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v16530; GFX942-NOTTGSPLIT-NEXT: s_endpgm6531;6532; GFX942-TGSPLIT-LABEL: local_workgroup_acquire_monotonic_ret_cmpxchg:6533; GFX942-TGSPLIT: ; %bb.0: ; %entry6534; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x06535; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x46536; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x86537; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6538; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s06539; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s26540; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s16541; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166542; GFX942-TGSPLIT-NEXT: buffer_inv sc06543; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s06544; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6545; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v16546; GFX942-TGSPLIT-NEXT: s_endpgm6547;6548; GFX11-WGP-LABEL: local_workgroup_acquire_monotonic_ret_cmpxchg:6549; GFX11-WGP: ; %bb.0: ; %entry6550; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x06551; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x46552; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x86553; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)6554; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s06555; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s26556; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s16557; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:166558; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)6559; GFX11-WGP-NEXT: buffer_gl0_inv6560; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s06561; GFX11-WGP-NEXT: ds_store_b32 v0, v16562; GFX11-WGP-NEXT: s_endpgm6563;6564; GFX11-CU-LABEL: local_workgroup_acquire_monotonic_ret_cmpxchg:6565; GFX11-CU: ; %bb.0: ; %entry6566; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x06567; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x46568; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x86569; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)6570; GFX11-CU-NEXT: v_mov_b32_e32 v0, s06571; GFX11-CU-NEXT: v_mov_b32_e32 v1, s26572; GFX11-CU-NEXT: v_mov_b32_e32 v2, s16573; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:166574; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)6575; GFX11-CU-NEXT: v_mov_b32_e32 v0, s06576; GFX11-CU-NEXT: ds_store_b32 v0, v16577; GFX11-CU-NEXT: s_endpgm6578;6579; GFX12-WGP-LABEL: local_workgroup_acquire_monotonic_ret_cmpxchg:6580; GFX12-WGP: ; %bb.0: ; %entry6581; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x06582; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x46583; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x86584; GFX12-WGP-NEXT: s_wait_kmcnt 0x06585; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s06586; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s26587; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s16588; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:166589; GFX12-WGP-NEXT: s_wait_dscnt 0x06590; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE6591; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s06592; GFX12-WGP-NEXT: ds_store_b32 v0, v16593; GFX12-WGP-NEXT: s_endpgm6594;6595; GFX12-CU-LABEL: local_workgroup_acquire_monotonic_ret_cmpxchg:6596; GFX12-CU: ; %bb.0: ; %entry6597; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x06598; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x46599; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x86600; GFX12-CU-NEXT: s_wait_kmcnt 0x06601; GFX12-CU-NEXT: v_mov_b32_e32 v0, s06602; GFX12-CU-NEXT: v_mov_b32_e32 v1, s26603; GFX12-CU-NEXT: v_mov_b32_e32 v2, s16604; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:166605; GFX12-CU-NEXT: s_wait_dscnt 0x06606; GFX12-CU-NEXT: v_mov_b32_e32 v0, s06607; GFX12-CU-NEXT: ds_store_b32 v0, v16608; GFX12-CU-NEXT: s_endpgm6609;6610; GFX1250-LABEL: local_workgroup_acquire_monotonic_ret_cmpxchg:6611; GFX1250: ; %bb.0: ; %entry6612; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 16613; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x06614; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x46615; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x86616; GFX1250-NEXT: s_wait_kmcnt 0x06617; GFX1250-NEXT: v_mov_b32_e32 v0, s06618; GFX1250-NEXT: v_mov_b32_e32 v1, s26619; GFX1250-NEXT: v_mov_b32_e32 v2, s16620; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:166621; GFX1250-NEXT: s_wait_dscnt 0x06622; GFX1250-NEXT: v_mov_b32_e32 v0, s06623; GFX1250-NEXT: ds_store_b32 v0, v16624; GFX1250-NEXT: s_endpgm6625 ptr addrspace(3) %out, i32 %in, i32 %old) {6626entry:6627 %gep = getelementptr i32, ptr addrspace(3) %out, i32 46628 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic6629 %val0 = extractvalue { i32, i1 } %val, 06630 store i32 %val0, ptr addrspace(3) %out, align 46631 ret void6632}6633 6634define amdgpu_kernel void @local_workgroup_release_monotonic_ret_cmpxchg(6635; GFX6-LABEL: local_workgroup_release_monotonic_ret_cmpxchg:6636; GFX6: ; %bb.0: ; %entry6637; GFX6-NEXT: s_load_dword s4, s[8:9], 0x06638; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr46639; GFX6-NEXT: s_load_dword s5, s[8:9], 0x16640; GFX6-NEXT: s_load_dword s6, s[8:9], 0x26641; GFX6-NEXT: s_mov_b32 m0, -16642; GFX6-NEXT: s_waitcnt lgkmcnt(0)6643; GFX6-NEXT: v_mov_b32_e32 v0, s46644; GFX6-NEXT: v_mov_b32_e32 v1, s66645; GFX6-NEXT: v_mov_b32_e32 v2, s56646; GFX6-NEXT: s_waitcnt lgkmcnt(0)6647; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166648; GFX6-NEXT: s_mov_b32 m0, -16649; GFX6-NEXT: v_mov_b32_e32 v0, s46650; GFX6-NEXT: s_waitcnt lgkmcnt(0)6651; GFX6-NEXT: ds_write_b32 v0, v16652; GFX6-NEXT: s_endpgm6653;6654; GFX7-LABEL: local_workgroup_release_monotonic_ret_cmpxchg:6655; GFX7: ; %bb.0: ; %entry6656; GFX7-NEXT: s_load_dword s4, s[8:9], 0x06657; GFX7-NEXT: s_load_dword s5, s[8:9], 0x16658; GFX7-NEXT: s_load_dword s6, s[8:9], 0x26659; GFX7-NEXT: s_mov_b32 m0, -16660; GFX7-NEXT: s_waitcnt lgkmcnt(0)6661; GFX7-NEXT: v_mov_b32_e32 v0, s46662; GFX7-NEXT: v_mov_b32_e32 v1, s66663; GFX7-NEXT: v_mov_b32_e32 v2, s56664; GFX7-NEXT: s_waitcnt lgkmcnt(0)6665; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166666; GFX7-NEXT: s_mov_b32 m0, -16667; GFX7-NEXT: v_mov_b32_e32 v0, s46668; GFX7-NEXT: s_waitcnt lgkmcnt(0)6669; GFX7-NEXT: ds_write_b32 v0, v16670; GFX7-NEXT: s_endpgm6671;6672; GFX10-WGP-LABEL: local_workgroup_release_monotonic_ret_cmpxchg:6673; GFX10-WGP: ; %bb.0: ; %entry6674; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x06675; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x46676; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x86677; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)6678; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s46679; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s66680; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s56681; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6682; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x06683; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166684; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s46685; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)6686; GFX10-WGP-NEXT: ds_write_b32 v0, v16687; GFX10-WGP-NEXT: s_endpgm6688;6689; GFX10-CU-LABEL: local_workgroup_release_monotonic_ret_cmpxchg:6690; GFX10-CU: ; %bb.0: ; %entry6691; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x06692; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x46693; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x86694; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)6695; GFX10-CU-NEXT: v_mov_b32_e32 v0, s46696; GFX10-CU-NEXT: v_mov_b32_e32 v1, s66697; GFX10-CU-NEXT: v_mov_b32_e32 v2, s56698; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6699; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x06700; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166701; GFX10-CU-NEXT: v_mov_b32_e32 v0, s46702; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)6703; GFX10-CU-NEXT: ds_write_b32 v0, v16704; GFX10-CU-NEXT: s_endpgm6705;6706; SKIP-CACHE-INV-LABEL: local_workgroup_release_monotonic_ret_cmpxchg:6707; SKIP-CACHE-INV: ; %bb.0: ; %entry6708; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x06709; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x16710; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x26711; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -16712; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)6713; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s06714; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s26715; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s16716; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)6717; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166718; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -16719; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s06720; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)6721; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v16722; SKIP-CACHE-INV-NEXT: s_endpgm6723;6724; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_release_monotonic_ret_cmpxchg:6725; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry6726; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x06727; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x46728; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x86729; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6730; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s46731; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s66732; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s56733; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6734; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166735; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s46736; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6737; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v16738; GFX90A-NOTTGSPLIT-NEXT: s_endpgm6739;6740; GFX90A-TGSPLIT-LABEL: local_workgroup_release_monotonic_ret_cmpxchg:6741; GFX90A-TGSPLIT: ; %bb.0: ; %entry6742; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x06743; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x46744; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x86745; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6746; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s46747; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s66748; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s56749; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6750; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166751; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s46752; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6753; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v16754; GFX90A-TGSPLIT-NEXT: s_endpgm6755;6756; GFX942-NOTTGSPLIT-LABEL: local_workgroup_release_monotonic_ret_cmpxchg:6757; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry6758; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x06759; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x46760; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x86761; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6762; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s06763; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s26764; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s16765; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6766; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166767; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s06768; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6769; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v16770; GFX942-NOTTGSPLIT-NEXT: s_endpgm6771;6772; GFX942-TGSPLIT-LABEL: local_workgroup_release_monotonic_ret_cmpxchg:6773; GFX942-TGSPLIT: ; %bb.0: ; %entry6774; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x06775; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x46776; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x86777; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6778; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s06779; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s26780; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s16781; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6782; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166783; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s06784; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6785; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v16786; GFX942-TGSPLIT-NEXT: s_endpgm6787;6788; GFX11-WGP-LABEL: local_workgroup_release_monotonic_ret_cmpxchg:6789; GFX11-WGP: ; %bb.0: ; %entry6790; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x06791; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x46792; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x86793; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)6794; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s06795; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s26796; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s16797; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6798; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x06799; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:166800; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s06801; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)6802; GFX11-WGP-NEXT: ds_store_b32 v0, v16803; GFX11-WGP-NEXT: s_endpgm6804;6805; GFX11-CU-LABEL: local_workgroup_release_monotonic_ret_cmpxchg:6806; GFX11-CU: ; %bb.0: ; %entry6807; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x06808; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x46809; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x86810; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)6811; GFX11-CU-NEXT: v_mov_b32_e32 v0, s06812; GFX11-CU-NEXT: v_mov_b32_e32 v1, s26813; GFX11-CU-NEXT: v_mov_b32_e32 v2, s16814; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6815; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x06816; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:166817; GFX11-CU-NEXT: v_mov_b32_e32 v0, s06818; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)6819; GFX11-CU-NEXT: ds_store_b32 v0, v16820; GFX11-CU-NEXT: s_endpgm6821;6822; GFX12-WGP-LABEL: local_workgroup_release_monotonic_ret_cmpxchg:6823; GFX12-WGP: ; %bb.0: ; %entry6824; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x06825; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x46826; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x86827; GFX12-WGP-NEXT: s_wait_kmcnt 0x06828; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s06829; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s26830; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s16831; GFX12-WGP-NEXT: s_wait_bvhcnt 0x06832; GFX12-WGP-NEXT: s_wait_samplecnt 0x06833; GFX12-WGP-NEXT: s_wait_storecnt 0x06834; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x06835; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:166836; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s06837; GFX12-WGP-NEXT: s_wait_dscnt 0x06838; GFX12-WGP-NEXT: ds_store_b32 v0, v16839; GFX12-WGP-NEXT: s_endpgm6840;6841; GFX12-CU-LABEL: local_workgroup_release_monotonic_ret_cmpxchg:6842; GFX12-CU: ; %bb.0: ; %entry6843; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x06844; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x46845; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x86846; GFX12-CU-NEXT: s_wait_kmcnt 0x06847; GFX12-CU-NEXT: v_mov_b32_e32 v0, s06848; GFX12-CU-NEXT: v_mov_b32_e32 v1, s26849; GFX12-CU-NEXT: v_mov_b32_e32 v2, s16850; GFX12-CU-NEXT: s_wait_bvhcnt 0x06851; GFX12-CU-NEXT: s_wait_samplecnt 0x06852; GFX12-CU-NEXT: s_wait_storecnt 0x06853; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x06854; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:166855; GFX12-CU-NEXT: v_mov_b32_e32 v0, s06856; GFX12-CU-NEXT: s_wait_dscnt 0x06857; GFX12-CU-NEXT: ds_store_b32 v0, v16858; GFX12-CU-NEXT: s_endpgm6859;6860; GFX1250-LABEL: local_workgroup_release_monotonic_ret_cmpxchg:6861; GFX1250: ; %bb.0: ; %entry6862; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 16863; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x06864; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x46865; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x86866; GFX1250-NEXT: s_wait_kmcnt 0x06867; GFX1250-NEXT: v_mov_b32_e32 v0, s06868; GFX1250-NEXT: v_mov_b32_e32 v1, s26869; GFX1250-NEXT: v_mov_b32_e32 v2, s16870; GFX1250-NEXT: s_wait_storecnt 0x06871; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x06872; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:166873; GFX1250-NEXT: v_mov_b32_e32 v0, s06874; GFX1250-NEXT: s_wait_dscnt 0x06875; GFX1250-NEXT: ds_store_b32 v0, v16876; GFX1250-NEXT: s_endpgm6877 ptr addrspace(3) %out, i32 %in, i32 %old) {6878entry:6879 %gep = getelementptr i32, ptr addrspace(3) %out, i32 46880 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") release monotonic6881 %val0 = extractvalue { i32, i1 } %val, 06882 store i32 %val0, ptr addrspace(3) %out, align 46883 ret void6884}6885 6886define amdgpu_kernel void @local_workgroup_acq_rel_monotonic_ret_cmpxchg(6887; GFX6-LABEL: local_workgroup_acq_rel_monotonic_ret_cmpxchg:6888; GFX6: ; %bb.0: ; %entry6889; GFX6-NEXT: s_load_dword s4, s[8:9], 0x06890; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr46891; GFX6-NEXT: s_load_dword s5, s[8:9], 0x16892; GFX6-NEXT: s_load_dword s6, s[8:9], 0x26893; GFX6-NEXT: s_mov_b32 m0, -16894; GFX6-NEXT: s_waitcnt lgkmcnt(0)6895; GFX6-NEXT: v_mov_b32_e32 v0, s46896; GFX6-NEXT: v_mov_b32_e32 v1, s66897; GFX6-NEXT: v_mov_b32_e32 v2, s56898; GFX6-NEXT: s_waitcnt lgkmcnt(0)6899; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166900; GFX6-NEXT: s_waitcnt lgkmcnt(0)6901; GFX6-NEXT: s_mov_b32 m0, -16902; GFX6-NEXT: v_mov_b32_e32 v0, s46903; GFX6-NEXT: ds_write_b32 v0, v16904; GFX6-NEXT: s_endpgm6905;6906; GFX7-LABEL: local_workgroup_acq_rel_monotonic_ret_cmpxchg:6907; GFX7: ; %bb.0: ; %entry6908; GFX7-NEXT: s_load_dword s4, s[8:9], 0x06909; GFX7-NEXT: s_load_dword s5, s[8:9], 0x16910; GFX7-NEXT: s_load_dword s6, s[8:9], 0x26911; GFX7-NEXT: s_mov_b32 m0, -16912; GFX7-NEXT: s_waitcnt lgkmcnt(0)6913; GFX7-NEXT: v_mov_b32_e32 v0, s46914; GFX7-NEXT: v_mov_b32_e32 v1, s66915; GFX7-NEXT: v_mov_b32_e32 v2, s56916; GFX7-NEXT: s_waitcnt lgkmcnt(0)6917; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166918; GFX7-NEXT: s_waitcnt lgkmcnt(0)6919; GFX7-NEXT: s_mov_b32 m0, -16920; GFX7-NEXT: v_mov_b32_e32 v0, s46921; GFX7-NEXT: ds_write_b32 v0, v16922; GFX7-NEXT: s_endpgm6923;6924; GFX10-WGP-LABEL: local_workgroup_acq_rel_monotonic_ret_cmpxchg:6925; GFX10-WGP: ; %bb.0: ; %entry6926; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x06927; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x46928; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x86929; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)6930; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s46931; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s66932; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s56933; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6934; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x06935; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166936; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)6937; GFX10-WGP-NEXT: buffer_gl0_inv6938; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s46939; GFX10-WGP-NEXT: ds_write_b32 v0, v16940; GFX10-WGP-NEXT: s_endpgm6941;6942; GFX10-CU-LABEL: local_workgroup_acq_rel_monotonic_ret_cmpxchg:6943; GFX10-CU: ; %bb.0: ; %entry6944; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x06945; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x46946; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x86947; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)6948; GFX10-CU-NEXT: v_mov_b32_e32 v0, s46949; GFX10-CU-NEXT: v_mov_b32_e32 v1, s66950; GFX10-CU-NEXT: v_mov_b32_e32 v2, s56951; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6952; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x06953; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166954; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)6955; GFX10-CU-NEXT: v_mov_b32_e32 v0, s46956; GFX10-CU-NEXT: ds_write_b32 v0, v16957; GFX10-CU-NEXT: s_endpgm6958;6959; SKIP-CACHE-INV-LABEL: local_workgroup_acq_rel_monotonic_ret_cmpxchg:6960; SKIP-CACHE-INV: ; %bb.0: ; %entry6961; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x06962; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x16963; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x26964; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -16965; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)6966; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s06967; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s26968; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s16969; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)6970; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166971; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)6972; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -16973; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s06974; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v16975; SKIP-CACHE-INV-NEXT: s_endpgm6976;6977; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_acq_rel_monotonic_ret_cmpxchg:6978; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry6979; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x06980; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x46981; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x86982; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6983; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s46984; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s66985; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s56986; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6987; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:166988; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6989; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s46990; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v16991; GFX90A-NOTTGSPLIT-NEXT: s_endpgm6992;6993; GFX90A-TGSPLIT-LABEL: local_workgroup_acq_rel_monotonic_ret_cmpxchg:6994; GFX90A-TGSPLIT: ; %bb.0: ; %entry6995; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x06996; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x46997; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x86998; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)6999; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s47000; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s67001; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s57002; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7003; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167004; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol7005; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s47006; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7007; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v17008; GFX90A-TGSPLIT-NEXT: s_endpgm7009;7010; GFX942-NOTTGSPLIT-LABEL: local_workgroup_acq_rel_monotonic_ret_cmpxchg:7011; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry7012; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x07013; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x47014; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x87015; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7016; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s07017; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s27018; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s17019; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7020; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167021; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7022; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s07023; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v17024; GFX942-NOTTGSPLIT-NEXT: s_endpgm7025;7026; GFX942-TGSPLIT-LABEL: local_workgroup_acq_rel_monotonic_ret_cmpxchg:7027; GFX942-TGSPLIT: ; %bb.0: ; %entry7028; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x07029; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x47030; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x87031; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7032; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s07033; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s27034; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s17035; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7036; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167037; GFX942-TGSPLIT-NEXT: buffer_inv sc07038; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s07039; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7040; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v17041; GFX942-TGSPLIT-NEXT: s_endpgm7042;7043; GFX11-WGP-LABEL: local_workgroup_acq_rel_monotonic_ret_cmpxchg:7044; GFX11-WGP: ; %bb.0: ; %entry7045; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x07046; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x47047; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x87048; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)7049; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s07050; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s27051; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s17052; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7053; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x07054; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:167055; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)7056; GFX11-WGP-NEXT: buffer_gl0_inv7057; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s07058; GFX11-WGP-NEXT: ds_store_b32 v0, v17059; GFX11-WGP-NEXT: s_endpgm7060;7061; GFX11-CU-LABEL: local_workgroup_acq_rel_monotonic_ret_cmpxchg:7062; GFX11-CU: ; %bb.0: ; %entry7063; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x07064; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x47065; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x87066; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)7067; GFX11-CU-NEXT: v_mov_b32_e32 v0, s07068; GFX11-CU-NEXT: v_mov_b32_e32 v1, s27069; GFX11-CU-NEXT: v_mov_b32_e32 v2, s17070; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7071; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x07072; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:167073; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)7074; GFX11-CU-NEXT: v_mov_b32_e32 v0, s07075; GFX11-CU-NEXT: ds_store_b32 v0, v17076; GFX11-CU-NEXT: s_endpgm7077;7078; GFX12-WGP-LABEL: local_workgroup_acq_rel_monotonic_ret_cmpxchg:7079; GFX12-WGP: ; %bb.0: ; %entry7080; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x07081; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x47082; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x87083; GFX12-WGP-NEXT: s_wait_kmcnt 0x07084; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s07085; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s27086; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s17087; GFX12-WGP-NEXT: s_wait_bvhcnt 0x07088; GFX12-WGP-NEXT: s_wait_samplecnt 0x07089; GFX12-WGP-NEXT: s_wait_storecnt 0x07090; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x07091; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:167092; GFX12-WGP-NEXT: s_wait_dscnt 0x07093; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE7094; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s07095; GFX12-WGP-NEXT: ds_store_b32 v0, v17096; GFX12-WGP-NEXT: s_endpgm7097;7098; GFX12-CU-LABEL: local_workgroup_acq_rel_monotonic_ret_cmpxchg:7099; GFX12-CU: ; %bb.0: ; %entry7100; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x07101; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x47102; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x87103; GFX12-CU-NEXT: s_wait_kmcnt 0x07104; GFX12-CU-NEXT: v_mov_b32_e32 v0, s07105; GFX12-CU-NEXT: v_mov_b32_e32 v1, s27106; GFX12-CU-NEXT: v_mov_b32_e32 v2, s17107; GFX12-CU-NEXT: s_wait_bvhcnt 0x07108; GFX12-CU-NEXT: s_wait_samplecnt 0x07109; GFX12-CU-NEXT: s_wait_storecnt 0x07110; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x07111; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:167112; GFX12-CU-NEXT: s_wait_dscnt 0x07113; GFX12-CU-NEXT: v_mov_b32_e32 v0, s07114; GFX12-CU-NEXT: ds_store_b32 v0, v17115; GFX12-CU-NEXT: s_endpgm7116;7117; GFX1250-LABEL: local_workgroup_acq_rel_monotonic_ret_cmpxchg:7118; GFX1250: ; %bb.0: ; %entry7119; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 17120; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x07121; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x47122; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x87123; GFX1250-NEXT: s_wait_kmcnt 0x07124; GFX1250-NEXT: v_mov_b32_e32 v0, s07125; GFX1250-NEXT: v_mov_b32_e32 v1, s27126; GFX1250-NEXT: v_mov_b32_e32 v2, s17127; GFX1250-NEXT: s_wait_storecnt 0x07128; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x07129; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:167130; GFX1250-NEXT: s_wait_dscnt 0x07131; GFX1250-NEXT: v_mov_b32_e32 v0, s07132; GFX1250-NEXT: ds_store_b32 v0, v17133; GFX1250-NEXT: s_endpgm7134 ptr addrspace(3) %out, i32 %in, i32 %old) {7135entry:7136 %gep = getelementptr i32, ptr addrspace(3) %out, i32 47137 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic7138 %val0 = extractvalue { i32, i1 } %val, 07139 store i32 %val0, ptr addrspace(3) %out, align 47140 ret void7141}7142 7143define amdgpu_kernel void @local_workgroup_seq_cst_monotonic_ret_cmpxchg(7144; GFX6-LABEL: local_workgroup_seq_cst_monotonic_ret_cmpxchg:7145; GFX6: ; %bb.0: ; %entry7146; GFX6-NEXT: s_load_dword s4, s[8:9], 0x07147; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr47148; GFX6-NEXT: s_load_dword s5, s[8:9], 0x17149; GFX6-NEXT: s_load_dword s6, s[8:9], 0x27150; GFX6-NEXT: s_mov_b32 m0, -17151; GFX6-NEXT: s_waitcnt lgkmcnt(0)7152; GFX6-NEXT: v_mov_b32_e32 v0, s47153; GFX6-NEXT: v_mov_b32_e32 v1, s67154; GFX6-NEXT: v_mov_b32_e32 v2, s57155; GFX6-NEXT: s_waitcnt lgkmcnt(0)7156; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167157; GFX6-NEXT: s_waitcnt lgkmcnt(0)7158; GFX6-NEXT: s_mov_b32 m0, -17159; GFX6-NEXT: v_mov_b32_e32 v0, s47160; GFX6-NEXT: ds_write_b32 v0, v17161; GFX6-NEXT: s_endpgm7162;7163; GFX7-LABEL: local_workgroup_seq_cst_monotonic_ret_cmpxchg:7164; GFX7: ; %bb.0: ; %entry7165; GFX7-NEXT: s_load_dword s4, s[8:9], 0x07166; GFX7-NEXT: s_load_dword s5, s[8:9], 0x17167; GFX7-NEXT: s_load_dword s6, s[8:9], 0x27168; GFX7-NEXT: s_mov_b32 m0, -17169; GFX7-NEXT: s_waitcnt lgkmcnt(0)7170; GFX7-NEXT: v_mov_b32_e32 v0, s47171; GFX7-NEXT: v_mov_b32_e32 v1, s67172; GFX7-NEXT: v_mov_b32_e32 v2, s57173; GFX7-NEXT: s_waitcnt lgkmcnt(0)7174; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167175; GFX7-NEXT: s_waitcnt lgkmcnt(0)7176; GFX7-NEXT: s_mov_b32 m0, -17177; GFX7-NEXT: v_mov_b32_e32 v0, s47178; GFX7-NEXT: ds_write_b32 v0, v17179; GFX7-NEXT: s_endpgm7180;7181; GFX10-WGP-LABEL: local_workgroup_seq_cst_monotonic_ret_cmpxchg:7182; GFX10-WGP: ; %bb.0: ; %entry7183; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x07184; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x47185; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x87186; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)7187; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s47188; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s67189; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s57190; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7191; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x07192; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167193; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)7194; GFX10-WGP-NEXT: buffer_gl0_inv7195; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s47196; GFX10-WGP-NEXT: ds_write_b32 v0, v17197; GFX10-WGP-NEXT: s_endpgm7198;7199; GFX10-CU-LABEL: local_workgroup_seq_cst_monotonic_ret_cmpxchg:7200; GFX10-CU: ; %bb.0: ; %entry7201; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x07202; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x47203; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x87204; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)7205; GFX10-CU-NEXT: v_mov_b32_e32 v0, s47206; GFX10-CU-NEXT: v_mov_b32_e32 v1, s67207; GFX10-CU-NEXT: v_mov_b32_e32 v2, s57208; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7209; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x07210; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167211; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)7212; GFX10-CU-NEXT: v_mov_b32_e32 v0, s47213; GFX10-CU-NEXT: ds_write_b32 v0, v17214; GFX10-CU-NEXT: s_endpgm7215;7216; SKIP-CACHE-INV-LABEL: local_workgroup_seq_cst_monotonic_ret_cmpxchg:7217; SKIP-CACHE-INV: ; %bb.0: ; %entry7218; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x07219; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x17220; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x27221; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -17222; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)7223; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s07224; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s27225; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s17226; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)7227; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167228; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)7229; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -17230; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s07231; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v17232; SKIP-CACHE-INV-NEXT: s_endpgm7233;7234; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_seq_cst_monotonic_ret_cmpxchg:7235; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry7236; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x07237; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x47238; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x87239; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7240; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s47241; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s67242; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s57243; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7244; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167245; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7246; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s47247; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v17248; GFX90A-NOTTGSPLIT-NEXT: s_endpgm7249;7250; GFX90A-TGSPLIT-LABEL: local_workgroup_seq_cst_monotonic_ret_cmpxchg:7251; GFX90A-TGSPLIT: ; %bb.0: ; %entry7252; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x07253; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x47254; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x87255; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7256; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s47257; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s67258; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s57259; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7260; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167261; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol7262; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s47263; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7264; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v17265; GFX90A-TGSPLIT-NEXT: s_endpgm7266;7267; GFX942-NOTTGSPLIT-LABEL: local_workgroup_seq_cst_monotonic_ret_cmpxchg:7268; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry7269; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x07270; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x47271; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x87272; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7273; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s07274; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s27275; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s17276; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7277; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167278; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7279; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s07280; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v17281; GFX942-NOTTGSPLIT-NEXT: s_endpgm7282;7283; GFX942-TGSPLIT-LABEL: local_workgroup_seq_cst_monotonic_ret_cmpxchg:7284; GFX942-TGSPLIT: ; %bb.0: ; %entry7285; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x07286; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x47287; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x87288; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7289; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s07290; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s27291; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s17292; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7293; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167294; GFX942-TGSPLIT-NEXT: buffer_inv sc07295; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s07296; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7297; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v17298; GFX942-TGSPLIT-NEXT: s_endpgm7299;7300; GFX11-WGP-LABEL: local_workgroup_seq_cst_monotonic_ret_cmpxchg:7301; GFX11-WGP: ; %bb.0: ; %entry7302; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x07303; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x47304; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x87305; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)7306; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s07307; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s27308; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s17309; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7310; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x07311; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:167312; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)7313; GFX11-WGP-NEXT: buffer_gl0_inv7314; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s07315; GFX11-WGP-NEXT: ds_store_b32 v0, v17316; GFX11-WGP-NEXT: s_endpgm7317;7318; GFX11-CU-LABEL: local_workgroup_seq_cst_monotonic_ret_cmpxchg:7319; GFX11-CU: ; %bb.0: ; %entry7320; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x07321; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x47322; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x87323; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)7324; GFX11-CU-NEXT: v_mov_b32_e32 v0, s07325; GFX11-CU-NEXT: v_mov_b32_e32 v1, s27326; GFX11-CU-NEXT: v_mov_b32_e32 v2, s17327; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7328; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x07329; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:167330; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)7331; GFX11-CU-NEXT: v_mov_b32_e32 v0, s07332; GFX11-CU-NEXT: ds_store_b32 v0, v17333; GFX11-CU-NEXT: s_endpgm7334;7335; GFX12-WGP-LABEL: local_workgroup_seq_cst_monotonic_ret_cmpxchg:7336; GFX12-WGP: ; %bb.0: ; %entry7337; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x07338; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x47339; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x87340; GFX12-WGP-NEXT: s_wait_kmcnt 0x07341; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s07342; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s27343; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s17344; GFX12-WGP-NEXT: s_wait_bvhcnt 0x07345; GFX12-WGP-NEXT: s_wait_samplecnt 0x07346; GFX12-WGP-NEXT: s_wait_storecnt 0x07347; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x07348; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:167349; GFX12-WGP-NEXT: s_wait_dscnt 0x07350; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE7351; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s07352; GFX12-WGP-NEXT: ds_store_b32 v0, v17353; GFX12-WGP-NEXT: s_endpgm7354;7355; GFX12-CU-LABEL: local_workgroup_seq_cst_monotonic_ret_cmpxchg:7356; GFX12-CU: ; %bb.0: ; %entry7357; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x07358; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x47359; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x87360; GFX12-CU-NEXT: s_wait_kmcnt 0x07361; GFX12-CU-NEXT: v_mov_b32_e32 v0, s07362; GFX12-CU-NEXT: v_mov_b32_e32 v1, s27363; GFX12-CU-NEXT: v_mov_b32_e32 v2, s17364; GFX12-CU-NEXT: s_wait_bvhcnt 0x07365; GFX12-CU-NEXT: s_wait_samplecnt 0x07366; GFX12-CU-NEXT: s_wait_storecnt 0x07367; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x07368; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:167369; GFX12-CU-NEXT: s_wait_dscnt 0x07370; GFX12-CU-NEXT: v_mov_b32_e32 v0, s07371; GFX12-CU-NEXT: ds_store_b32 v0, v17372; GFX12-CU-NEXT: s_endpgm7373;7374; GFX1250-LABEL: local_workgroup_seq_cst_monotonic_ret_cmpxchg:7375; GFX1250: ; %bb.0: ; %entry7376; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 17377; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x07378; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x47379; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x87380; GFX1250-NEXT: s_wait_kmcnt 0x07381; GFX1250-NEXT: v_mov_b32_e32 v0, s07382; GFX1250-NEXT: v_mov_b32_e32 v1, s27383; GFX1250-NEXT: v_mov_b32_e32 v2, s17384; GFX1250-NEXT: s_wait_storecnt 0x07385; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x07386; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:167387; GFX1250-NEXT: s_wait_dscnt 0x07388; GFX1250-NEXT: v_mov_b32_e32 v0, s07389; GFX1250-NEXT: ds_store_b32 v0, v17390; GFX1250-NEXT: s_endpgm7391 ptr addrspace(3) %out, i32 %in, i32 %old) {7392entry:7393 %gep = getelementptr i32, ptr addrspace(3) %out, i32 47394 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic7395 %val0 = extractvalue { i32, i1 } %val, 07396 store i32 %val0, ptr addrspace(3) %out, align 47397 ret void7398}7399 7400define amdgpu_kernel void @local_workgroup_monotonic_acquire_ret_cmpxchg(7401; GFX6-LABEL: local_workgroup_monotonic_acquire_ret_cmpxchg:7402; GFX6: ; %bb.0: ; %entry7403; GFX6-NEXT: s_load_dword s4, s[8:9], 0x07404; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr47405; GFX6-NEXT: s_load_dword s5, s[8:9], 0x17406; GFX6-NEXT: s_load_dword s6, s[8:9], 0x27407; GFX6-NEXT: s_mov_b32 m0, -17408; GFX6-NEXT: s_waitcnt lgkmcnt(0)7409; GFX6-NEXT: v_mov_b32_e32 v0, s47410; GFX6-NEXT: v_mov_b32_e32 v1, s67411; GFX6-NEXT: v_mov_b32_e32 v2, s57412; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167413; GFX6-NEXT: s_waitcnt lgkmcnt(0)7414; GFX6-NEXT: s_mov_b32 m0, -17415; GFX6-NEXT: v_mov_b32_e32 v0, s47416; GFX6-NEXT: ds_write_b32 v0, v17417; GFX6-NEXT: s_endpgm7418;7419; GFX7-LABEL: local_workgroup_monotonic_acquire_ret_cmpxchg:7420; GFX7: ; %bb.0: ; %entry7421; GFX7-NEXT: s_load_dword s4, s[8:9], 0x07422; GFX7-NEXT: s_load_dword s5, s[8:9], 0x17423; GFX7-NEXT: s_load_dword s6, s[8:9], 0x27424; GFX7-NEXT: s_mov_b32 m0, -17425; GFX7-NEXT: s_waitcnt lgkmcnt(0)7426; GFX7-NEXT: v_mov_b32_e32 v0, s47427; GFX7-NEXT: v_mov_b32_e32 v1, s67428; GFX7-NEXT: v_mov_b32_e32 v2, s57429; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167430; GFX7-NEXT: s_waitcnt lgkmcnt(0)7431; GFX7-NEXT: s_mov_b32 m0, -17432; GFX7-NEXT: v_mov_b32_e32 v0, s47433; GFX7-NEXT: ds_write_b32 v0, v17434; GFX7-NEXT: s_endpgm7435;7436; GFX10-WGP-LABEL: local_workgroup_monotonic_acquire_ret_cmpxchg:7437; GFX10-WGP: ; %bb.0: ; %entry7438; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x07439; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x47440; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x87441; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)7442; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s47443; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s67444; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s57445; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167446; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)7447; GFX10-WGP-NEXT: buffer_gl0_inv7448; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s47449; GFX10-WGP-NEXT: ds_write_b32 v0, v17450; GFX10-WGP-NEXT: s_endpgm7451;7452; GFX10-CU-LABEL: local_workgroup_monotonic_acquire_ret_cmpxchg:7453; GFX10-CU: ; %bb.0: ; %entry7454; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x07455; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x47456; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x87457; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)7458; GFX10-CU-NEXT: v_mov_b32_e32 v0, s47459; GFX10-CU-NEXT: v_mov_b32_e32 v1, s67460; GFX10-CU-NEXT: v_mov_b32_e32 v2, s57461; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167462; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)7463; GFX10-CU-NEXT: v_mov_b32_e32 v0, s47464; GFX10-CU-NEXT: ds_write_b32 v0, v17465; GFX10-CU-NEXT: s_endpgm7466;7467; SKIP-CACHE-INV-LABEL: local_workgroup_monotonic_acquire_ret_cmpxchg:7468; SKIP-CACHE-INV: ; %bb.0: ; %entry7469; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x07470; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x17471; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x27472; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -17473; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)7474; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s07475; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s27476; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s17477; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167478; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)7479; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -17480; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s07481; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v17482; SKIP-CACHE-INV-NEXT: s_endpgm7483;7484; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_monotonic_acquire_ret_cmpxchg:7485; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry7486; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x07487; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x47488; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x87489; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7490; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s47491; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s67492; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s57493; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167494; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7495; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s47496; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v17497; GFX90A-NOTTGSPLIT-NEXT: s_endpgm7498;7499; GFX90A-TGSPLIT-LABEL: local_workgroup_monotonic_acquire_ret_cmpxchg:7500; GFX90A-TGSPLIT: ; %bb.0: ; %entry7501; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x07502; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x47503; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x87504; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7505; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s47506; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s67507; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s57508; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167509; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol7510; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s47511; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7512; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v17513; GFX90A-TGSPLIT-NEXT: s_endpgm7514;7515; GFX942-NOTTGSPLIT-LABEL: local_workgroup_monotonic_acquire_ret_cmpxchg:7516; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry7517; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x07518; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x47519; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x87520; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7521; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s07522; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s27523; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s17524; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167525; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7526; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s07527; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v17528; GFX942-NOTTGSPLIT-NEXT: s_endpgm7529;7530; GFX942-TGSPLIT-LABEL: local_workgroup_monotonic_acquire_ret_cmpxchg:7531; GFX942-TGSPLIT: ; %bb.0: ; %entry7532; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x07533; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x47534; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x87535; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7536; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s07537; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s27538; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s17539; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167540; GFX942-TGSPLIT-NEXT: buffer_inv sc07541; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s07542; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7543; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v17544; GFX942-TGSPLIT-NEXT: s_endpgm7545;7546; GFX11-WGP-LABEL: local_workgroup_monotonic_acquire_ret_cmpxchg:7547; GFX11-WGP: ; %bb.0: ; %entry7548; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x07549; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x47550; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x87551; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)7552; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s07553; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s27554; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s17555; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:167556; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)7557; GFX11-WGP-NEXT: buffer_gl0_inv7558; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s07559; GFX11-WGP-NEXT: ds_store_b32 v0, v17560; GFX11-WGP-NEXT: s_endpgm7561;7562; GFX11-CU-LABEL: local_workgroup_monotonic_acquire_ret_cmpxchg:7563; GFX11-CU: ; %bb.0: ; %entry7564; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x07565; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x47566; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x87567; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)7568; GFX11-CU-NEXT: v_mov_b32_e32 v0, s07569; GFX11-CU-NEXT: v_mov_b32_e32 v1, s27570; GFX11-CU-NEXT: v_mov_b32_e32 v2, s17571; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:167572; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)7573; GFX11-CU-NEXT: v_mov_b32_e32 v0, s07574; GFX11-CU-NEXT: ds_store_b32 v0, v17575; GFX11-CU-NEXT: s_endpgm7576;7577; GFX12-WGP-LABEL: local_workgroup_monotonic_acquire_ret_cmpxchg:7578; GFX12-WGP: ; %bb.0: ; %entry7579; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x07580; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x47581; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x87582; GFX12-WGP-NEXT: s_wait_kmcnt 0x07583; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s07584; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s27585; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s17586; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:167587; GFX12-WGP-NEXT: s_wait_dscnt 0x07588; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE7589; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s07590; GFX12-WGP-NEXT: ds_store_b32 v0, v17591; GFX12-WGP-NEXT: s_endpgm7592;7593; GFX12-CU-LABEL: local_workgroup_monotonic_acquire_ret_cmpxchg:7594; GFX12-CU: ; %bb.0: ; %entry7595; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x07596; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x47597; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x87598; GFX12-CU-NEXT: s_wait_kmcnt 0x07599; GFX12-CU-NEXT: v_mov_b32_e32 v0, s07600; GFX12-CU-NEXT: v_mov_b32_e32 v1, s27601; GFX12-CU-NEXT: v_mov_b32_e32 v2, s17602; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:167603; GFX12-CU-NEXT: s_wait_dscnt 0x07604; GFX12-CU-NEXT: v_mov_b32_e32 v0, s07605; GFX12-CU-NEXT: ds_store_b32 v0, v17606; GFX12-CU-NEXT: s_endpgm7607;7608; GFX1250-LABEL: local_workgroup_monotonic_acquire_ret_cmpxchg:7609; GFX1250: ; %bb.0: ; %entry7610; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 17611; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x07612; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x47613; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x87614; GFX1250-NEXT: s_wait_kmcnt 0x07615; GFX1250-NEXT: v_mov_b32_e32 v0, s07616; GFX1250-NEXT: v_mov_b32_e32 v1, s27617; GFX1250-NEXT: v_mov_b32_e32 v2, s17618; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:167619; GFX1250-NEXT: s_wait_dscnt 0x07620; GFX1250-NEXT: v_mov_b32_e32 v0, s07621; GFX1250-NEXT: ds_store_b32 v0, v17622; GFX1250-NEXT: s_endpgm7623 ptr addrspace(3) %out, i32 %in, i32 %old) {7624entry:7625 %gep = getelementptr i32, ptr addrspace(3) %out, i32 47626 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") monotonic acquire7627 %val0 = extractvalue { i32, i1 } %val, 07628 store i32 %val0, ptr addrspace(3) %out, align 47629 ret void7630}7631 7632define amdgpu_kernel void @local_workgroup_acquire_acquire_ret_cmpxchg(7633; GFX6-LABEL: local_workgroup_acquire_acquire_ret_cmpxchg:7634; GFX6: ; %bb.0: ; %entry7635; GFX6-NEXT: s_load_dword s4, s[8:9], 0x07636; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr47637; GFX6-NEXT: s_load_dword s5, s[8:9], 0x17638; GFX6-NEXT: s_load_dword s6, s[8:9], 0x27639; GFX6-NEXT: s_mov_b32 m0, -17640; GFX6-NEXT: s_waitcnt lgkmcnt(0)7641; GFX6-NEXT: v_mov_b32_e32 v0, s47642; GFX6-NEXT: v_mov_b32_e32 v1, s67643; GFX6-NEXT: v_mov_b32_e32 v2, s57644; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167645; GFX6-NEXT: s_waitcnt lgkmcnt(0)7646; GFX6-NEXT: s_mov_b32 m0, -17647; GFX6-NEXT: v_mov_b32_e32 v0, s47648; GFX6-NEXT: ds_write_b32 v0, v17649; GFX6-NEXT: s_endpgm7650;7651; GFX7-LABEL: local_workgroup_acquire_acquire_ret_cmpxchg:7652; GFX7: ; %bb.0: ; %entry7653; GFX7-NEXT: s_load_dword s4, s[8:9], 0x07654; GFX7-NEXT: s_load_dword s5, s[8:9], 0x17655; GFX7-NEXT: s_load_dword s6, s[8:9], 0x27656; GFX7-NEXT: s_mov_b32 m0, -17657; GFX7-NEXT: s_waitcnt lgkmcnt(0)7658; GFX7-NEXT: v_mov_b32_e32 v0, s47659; GFX7-NEXT: v_mov_b32_e32 v1, s67660; GFX7-NEXT: v_mov_b32_e32 v2, s57661; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167662; GFX7-NEXT: s_waitcnt lgkmcnt(0)7663; GFX7-NEXT: s_mov_b32 m0, -17664; GFX7-NEXT: v_mov_b32_e32 v0, s47665; GFX7-NEXT: ds_write_b32 v0, v17666; GFX7-NEXT: s_endpgm7667;7668; GFX10-WGP-LABEL: local_workgroup_acquire_acquire_ret_cmpxchg:7669; GFX10-WGP: ; %bb.0: ; %entry7670; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x07671; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x47672; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x87673; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)7674; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s47675; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s67676; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s57677; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167678; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)7679; GFX10-WGP-NEXT: buffer_gl0_inv7680; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s47681; GFX10-WGP-NEXT: ds_write_b32 v0, v17682; GFX10-WGP-NEXT: s_endpgm7683;7684; GFX10-CU-LABEL: local_workgroup_acquire_acquire_ret_cmpxchg:7685; GFX10-CU: ; %bb.0: ; %entry7686; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x07687; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x47688; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x87689; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)7690; GFX10-CU-NEXT: v_mov_b32_e32 v0, s47691; GFX10-CU-NEXT: v_mov_b32_e32 v1, s67692; GFX10-CU-NEXT: v_mov_b32_e32 v2, s57693; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167694; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)7695; GFX10-CU-NEXT: v_mov_b32_e32 v0, s47696; GFX10-CU-NEXT: ds_write_b32 v0, v17697; GFX10-CU-NEXT: s_endpgm7698;7699; SKIP-CACHE-INV-LABEL: local_workgroup_acquire_acquire_ret_cmpxchg:7700; SKIP-CACHE-INV: ; %bb.0: ; %entry7701; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x07702; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x17703; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x27704; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -17705; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)7706; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s07707; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s27708; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s17709; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167710; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)7711; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -17712; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s07713; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v17714; SKIP-CACHE-INV-NEXT: s_endpgm7715;7716; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_acquire_acquire_ret_cmpxchg:7717; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry7718; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x07719; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x47720; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x87721; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7722; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s47723; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s67724; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s57725; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167726; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7727; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s47728; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v17729; GFX90A-NOTTGSPLIT-NEXT: s_endpgm7730;7731; GFX90A-TGSPLIT-LABEL: local_workgroup_acquire_acquire_ret_cmpxchg:7732; GFX90A-TGSPLIT: ; %bb.0: ; %entry7733; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x07734; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x47735; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x87736; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7737; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s47738; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s67739; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s57740; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167741; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol7742; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s47743; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7744; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v17745; GFX90A-TGSPLIT-NEXT: s_endpgm7746;7747; GFX942-NOTTGSPLIT-LABEL: local_workgroup_acquire_acquire_ret_cmpxchg:7748; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry7749; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x07750; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x47751; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x87752; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7753; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s07754; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s27755; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s17756; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167757; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7758; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s07759; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v17760; GFX942-NOTTGSPLIT-NEXT: s_endpgm7761;7762; GFX942-TGSPLIT-LABEL: local_workgroup_acquire_acquire_ret_cmpxchg:7763; GFX942-TGSPLIT: ; %bb.0: ; %entry7764; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x07765; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x47766; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x87767; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7768; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s07769; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s27770; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s17771; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167772; GFX942-TGSPLIT-NEXT: buffer_inv sc07773; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s07774; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7775; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v17776; GFX942-TGSPLIT-NEXT: s_endpgm7777;7778; GFX11-WGP-LABEL: local_workgroup_acquire_acquire_ret_cmpxchg:7779; GFX11-WGP: ; %bb.0: ; %entry7780; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x07781; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x47782; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x87783; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)7784; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s07785; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s27786; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s17787; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:167788; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)7789; GFX11-WGP-NEXT: buffer_gl0_inv7790; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s07791; GFX11-WGP-NEXT: ds_store_b32 v0, v17792; GFX11-WGP-NEXT: s_endpgm7793;7794; GFX11-CU-LABEL: local_workgroup_acquire_acquire_ret_cmpxchg:7795; GFX11-CU: ; %bb.0: ; %entry7796; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x07797; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x47798; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x87799; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)7800; GFX11-CU-NEXT: v_mov_b32_e32 v0, s07801; GFX11-CU-NEXT: v_mov_b32_e32 v1, s27802; GFX11-CU-NEXT: v_mov_b32_e32 v2, s17803; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:167804; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)7805; GFX11-CU-NEXT: v_mov_b32_e32 v0, s07806; GFX11-CU-NEXT: ds_store_b32 v0, v17807; GFX11-CU-NEXT: s_endpgm7808;7809; GFX12-WGP-LABEL: local_workgroup_acquire_acquire_ret_cmpxchg:7810; GFX12-WGP: ; %bb.0: ; %entry7811; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x07812; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x47813; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x87814; GFX12-WGP-NEXT: s_wait_kmcnt 0x07815; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s07816; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s27817; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s17818; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:167819; GFX12-WGP-NEXT: s_wait_dscnt 0x07820; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE7821; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s07822; GFX12-WGP-NEXT: ds_store_b32 v0, v17823; GFX12-WGP-NEXT: s_endpgm7824;7825; GFX12-CU-LABEL: local_workgroup_acquire_acquire_ret_cmpxchg:7826; GFX12-CU: ; %bb.0: ; %entry7827; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x07828; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x47829; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x87830; GFX12-CU-NEXT: s_wait_kmcnt 0x07831; GFX12-CU-NEXT: v_mov_b32_e32 v0, s07832; GFX12-CU-NEXT: v_mov_b32_e32 v1, s27833; GFX12-CU-NEXT: v_mov_b32_e32 v2, s17834; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:167835; GFX12-CU-NEXT: s_wait_dscnt 0x07836; GFX12-CU-NEXT: v_mov_b32_e32 v0, s07837; GFX12-CU-NEXT: ds_store_b32 v0, v17838; GFX12-CU-NEXT: s_endpgm7839;7840; GFX1250-LABEL: local_workgroup_acquire_acquire_ret_cmpxchg:7841; GFX1250: ; %bb.0: ; %entry7842; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 17843; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x07844; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x47845; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x87846; GFX1250-NEXT: s_wait_kmcnt 0x07847; GFX1250-NEXT: v_mov_b32_e32 v0, s07848; GFX1250-NEXT: v_mov_b32_e32 v1, s27849; GFX1250-NEXT: v_mov_b32_e32 v2, s17850; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:167851; GFX1250-NEXT: s_wait_dscnt 0x07852; GFX1250-NEXT: v_mov_b32_e32 v0, s07853; GFX1250-NEXT: ds_store_b32 v0, v17854; GFX1250-NEXT: s_endpgm7855 ptr addrspace(3) %out, i32 %in, i32 %old) {7856entry:7857 %gep = getelementptr i32, ptr addrspace(3) %out, i32 47858 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire7859 %val0 = extractvalue { i32, i1 } %val, 07860 store i32 %val0, ptr addrspace(3) %out, align 47861 ret void7862}7863 7864define amdgpu_kernel void @local_workgroup_release_acquire_ret_cmpxchg(7865; GFX6-LABEL: local_workgroup_release_acquire_ret_cmpxchg:7866; GFX6: ; %bb.0: ; %entry7867; GFX6-NEXT: s_load_dword s4, s[8:9], 0x07868; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr47869; GFX6-NEXT: s_load_dword s5, s[8:9], 0x17870; GFX6-NEXT: s_load_dword s6, s[8:9], 0x27871; GFX6-NEXT: s_mov_b32 m0, -17872; GFX6-NEXT: s_waitcnt lgkmcnt(0)7873; GFX6-NEXT: v_mov_b32_e32 v0, s47874; GFX6-NEXT: v_mov_b32_e32 v1, s67875; GFX6-NEXT: v_mov_b32_e32 v2, s57876; GFX6-NEXT: s_waitcnt lgkmcnt(0)7877; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167878; GFX6-NEXT: s_waitcnt lgkmcnt(0)7879; GFX6-NEXT: s_mov_b32 m0, -17880; GFX6-NEXT: v_mov_b32_e32 v0, s47881; GFX6-NEXT: ds_write_b32 v0, v17882; GFX6-NEXT: s_endpgm7883;7884; GFX7-LABEL: local_workgroup_release_acquire_ret_cmpxchg:7885; GFX7: ; %bb.0: ; %entry7886; GFX7-NEXT: s_load_dword s4, s[8:9], 0x07887; GFX7-NEXT: s_load_dword s5, s[8:9], 0x17888; GFX7-NEXT: s_load_dword s6, s[8:9], 0x27889; GFX7-NEXT: s_mov_b32 m0, -17890; GFX7-NEXT: s_waitcnt lgkmcnt(0)7891; GFX7-NEXT: v_mov_b32_e32 v0, s47892; GFX7-NEXT: v_mov_b32_e32 v1, s67893; GFX7-NEXT: v_mov_b32_e32 v2, s57894; GFX7-NEXT: s_waitcnt lgkmcnt(0)7895; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167896; GFX7-NEXT: s_waitcnt lgkmcnt(0)7897; GFX7-NEXT: s_mov_b32 m0, -17898; GFX7-NEXT: v_mov_b32_e32 v0, s47899; GFX7-NEXT: ds_write_b32 v0, v17900; GFX7-NEXT: s_endpgm7901;7902; GFX10-WGP-LABEL: local_workgroup_release_acquire_ret_cmpxchg:7903; GFX10-WGP: ; %bb.0: ; %entry7904; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x07905; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x47906; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x87907; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)7908; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s47909; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s67910; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s57911; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7912; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x07913; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167914; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)7915; GFX10-WGP-NEXT: buffer_gl0_inv7916; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s47917; GFX10-WGP-NEXT: ds_write_b32 v0, v17918; GFX10-WGP-NEXT: s_endpgm7919;7920; GFX10-CU-LABEL: local_workgroup_release_acquire_ret_cmpxchg:7921; GFX10-CU: ; %bb.0: ; %entry7922; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x07923; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x47924; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x87925; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)7926; GFX10-CU-NEXT: v_mov_b32_e32 v0, s47927; GFX10-CU-NEXT: v_mov_b32_e32 v1, s67928; GFX10-CU-NEXT: v_mov_b32_e32 v2, s57929; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7930; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x07931; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167932; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)7933; GFX10-CU-NEXT: v_mov_b32_e32 v0, s47934; GFX10-CU-NEXT: ds_write_b32 v0, v17935; GFX10-CU-NEXT: s_endpgm7936;7937; SKIP-CACHE-INV-LABEL: local_workgroup_release_acquire_ret_cmpxchg:7938; SKIP-CACHE-INV: ; %bb.0: ; %entry7939; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x07940; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x17941; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x27942; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -17943; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)7944; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s07945; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s27946; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s17947; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)7948; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167949; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)7950; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -17951; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s07952; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v17953; SKIP-CACHE-INV-NEXT: s_endpgm7954;7955; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_release_acquire_ret_cmpxchg:7956; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry7957; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x07958; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x47959; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x87960; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7961; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s47962; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s67963; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s57964; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7965; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167966; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7967; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s47968; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v17969; GFX90A-NOTTGSPLIT-NEXT: s_endpgm7970;7971; GFX90A-TGSPLIT-LABEL: local_workgroup_release_acquire_ret_cmpxchg:7972; GFX90A-TGSPLIT: ; %bb.0: ; %entry7973; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x07974; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x47975; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x87976; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7977; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s47978; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s67979; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s57980; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7981; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167982; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol7983; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s47984; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7985; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v17986; GFX90A-TGSPLIT-NEXT: s_endpgm7987;7988; GFX942-NOTTGSPLIT-LABEL: local_workgroup_release_acquire_ret_cmpxchg:7989; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry7990; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x07991; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x47992; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x87993; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7994; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s07995; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s27996; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s17997; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)7998; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:167999; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8000; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s08001; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v18002; GFX942-NOTTGSPLIT-NEXT: s_endpgm8003;8004; GFX942-TGSPLIT-LABEL: local_workgroup_release_acquire_ret_cmpxchg:8005; GFX942-TGSPLIT: ; %bb.0: ; %entry8006; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x08007; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x48008; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x88009; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8010; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s08011; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s28012; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s18013; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8014; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168015; GFX942-TGSPLIT-NEXT: buffer_inv sc08016; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s08017; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8018; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v18019; GFX942-TGSPLIT-NEXT: s_endpgm8020;8021; GFX11-WGP-LABEL: local_workgroup_release_acquire_ret_cmpxchg:8022; GFX11-WGP: ; %bb.0: ; %entry8023; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x08024; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x48025; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x88026; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)8027; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s08028; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s28029; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s18030; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8031; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x08032; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:168033; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)8034; GFX11-WGP-NEXT: buffer_gl0_inv8035; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s08036; GFX11-WGP-NEXT: ds_store_b32 v0, v18037; GFX11-WGP-NEXT: s_endpgm8038;8039; GFX11-CU-LABEL: local_workgroup_release_acquire_ret_cmpxchg:8040; GFX11-CU: ; %bb.0: ; %entry8041; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x08042; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x48043; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x88044; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)8045; GFX11-CU-NEXT: v_mov_b32_e32 v0, s08046; GFX11-CU-NEXT: v_mov_b32_e32 v1, s28047; GFX11-CU-NEXT: v_mov_b32_e32 v2, s18048; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8049; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x08050; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:168051; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)8052; GFX11-CU-NEXT: v_mov_b32_e32 v0, s08053; GFX11-CU-NEXT: ds_store_b32 v0, v18054; GFX11-CU-NEXT: s_endpgm8055;8056; GFX12-WGP-LABEL: local_workgroup_release_acquire_ret_cmpxchg:8057; GFX12-WGP: ; %bb.0: ; %entry8058; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x08059; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x48060; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x88061; GFX12-WGP-NEXT: s_wait_kmcnt 0x08062; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s08063; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s28064; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s18065; GFX12-WGP-NEXT: s_wait_bvhcnt 0x08066; GFX12-WGP-NEXT: s_wait_samplecnt 0x08067; GFX12-WGP-NEXT: s_wait_storecnt 0x08068; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x08069; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:168070; GFX12-WGP-NEXT: s_wait_dscnt 0x08071; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE8072; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s08073; GFX12-WGP-NEXT: ds_store_b32 v0, v18074; GFX12-WGP-NEXT: s_endpgm8075;8076; GFX12-CU-LABEL: local_workgroup_release_acquire_ret_cmpxchg:8077; GFX12-CU: ; %bb.0: ; %entry8078; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x08079; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x48080; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x88081; GFX12-CU-NEXT: s_wait_kmcnt 0x08082; GFX12-CU-NEXT: v_mov_b32_e32 v0, s08083; GFX12-CU-NEXT: v_mov_b32_e32 v1, s28084; GFX12-CU-NEXT: v_mov_b32_e32 v2, s18085; GFX12-CU-NEXT: s_wait_bvhcnt 0x08086; GFX12-CU-NEXT: s_wait_samplecnt 0x08087; GFX12-CU-NEXT: s_wait_storecnt 0x08088; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x08089; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:168090; GFX12-CU-NEXT: s_wait_dscnt 0x08091; GFX12-CU-NEXT: v_mov_b32_e32 v0, s08092; GFX12-CU-NEXT: ds_store_b32 v0, v18093; GFX12-CU-NEXT: s_endpgm8094;8095; GFX1250-LABEL: local_workgroup_release_acquire_ret_cmpxchg:8096; GFX1250: ; %bb.0: ; %entry8097; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 18098; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x08099; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x48100; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x88101; GFX1250-NEXT: s_wait_kmcnt 0x08102; GFX1250-NEXT: v_mov_b32_e32 v0, s08103; GFX1250-NEXT: v_mov_b32_e32 v1, s28104; GFX1250-NEXT: v_mov_b32_e32 v2, s18105; GFX1250-NEXT: s_wait_storecnt 0x08106; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x08107; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:168108; GFX1250-NEXT: s_wait_dscnt 0x08109; GFX1250-NEXT: v_mov_b32_e32 v0, s08110; GFX1250-NEXT: ds_store_b32 v0, v18111; GFX1250-NEXT: s_endpgm8112 ptr addrspace(3) %out, i32 %in, i32 %old) {8113entry:8114 %gep = getelementptr i32, ptr addrspace(3) %out, i32 48115 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") release acquire8116 %val0 = extractvalue { i32, i1 } %val, 08117 store i32 %val0, ptr addrspace(3) %out, align 48118 ret void8119}8120 8121define amdgpu_kernel void @local_workgroup_acq_rel_acquire_ret_cmpxchg(8122; GFX6-LABEL: local_workgroup_acq_rel_acquire_ret_cmpxchg:8123; GFX6: ; %bb.0: ; %entry8124; GFX6-NEXT: s_load_dword s4, s[8:9], 0x08125; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr48126; GFX6-NEXT: s_load_dword s5, s[8:9], 0x18127; GFX6-NEXT: s_load_dword s6, s[8:9], 0x28128; GFX6-NEXT: s_mov_b32 m0, -18129; GFX6-NEXT: s_waitcnt lgkmcnt(0)8130; GFX6-NEXT: v_mov_b32_e32 v0, s48131; GFX6-NEXT: v_mov_b32_e32 v1, s68132; GFX6-NEXT: v_mov_b32_e32 v2, s58133; GFX6-NEXT: s_waitcnt lgkmcnt(0)8134; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168135; GFX6-NEXT: s_waitcnt lgkmcnt(0)8136; GFX6-NEXT: s_mov_b32 m0, -18137; GFX6-NEXT: v_mov_b32_e32 v0, s48138; GFX6-NEXT: ds_write_b32 v0, v18139; GFX6-NEXT: s_endpgm8140;8141; GFX7-LABEL: local_workgroup_acq_rel_acquire_ret_cmpxchg:8142; GFX7: ; %bb.0: ; %entry8143; GFX7-NEXT: s_load_dword s4, s[8:9], 0x08144; GFX7-NEXT: s_load_dword s5, s[8:9], 0x18145; GFX7-NEXT: s_load_dword s6, s[8:9], 0x28146; GFX7-NEXT: s_mov_b32 m0, -18147; GFX7-NEXT: s_waitcnt lgkmcnt(0)8148; GFX7-NEXT: v_mov_b32_e32 v0, s48149; GFX7-NEXT: v_mov_b32_e32 v1, s68150; GFX7-NEXT: v_mov_b32_e32 v2, s58151; GFX7-NEXT: s_waitcnt lgkmcnt(0)8152; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168153; GFX7-NEXT: s_waitcnt lgkmcnt(0)8154; GFX7-NEXT: s_mov_b32 m0, -18155; GFX7-NEXT: v_mov_b32_e32 v0, s48156; GFX7-NEXT: ds_write_b32 v0, v18157; GFX7-NEXT: s_endpgm8158;8159; GFX10-WGP-LABEL: local_workgroup_acq_rel_acquire_ret_cmpxchg:8160; GFX10-WGP: ; %bb.0: ; %entry8161; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x08162; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x48163; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x88164; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)8165; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s48166; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s68167; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s58168; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8169; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x08170; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168171; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)8172; GFX10-WGP-NEXT: buffer_gl0_inv8173; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s48174; GFX10-WGP-NEXT: ds_write_b32 v0, v18175; GFX10-WGP-NEXT: s_endpgm8176;8177; GFX10-CU-LABEL: local_workgroup_acq_rel_acquire_ret_cmpxchg:8178; GFX10-CU: ; %bb.0: ; %entry8179; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x08180; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x48181; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x88182; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)8183; GFX10-CU-NEXT: v_mov_b32_e32 v0, s48184; GFX10-CU-NEXT: v_mov_b32_e32 v1, s68185; GFX10-CU-NEXT: v_mov_b32_e32 v2, s58186; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8187; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x08188; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168189; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)8190; GFX10-CU-NEXT: v_mov_b32_e32 v0, s48191; GFX10-CU-NEXT: ds_write_b32 v0, v18192; GFX10-CU-NEXT: s_endpgm8193;8194; SKIP-CACHE-INV-LABEL: local_workgroup_acq_rel_acquire_ret_cmpxchg:8195; SKIP-CACHE-INV: ; %bb.0: ; %entry8196; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x08197; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x18198; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x28199; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -18200; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)8201; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s08202; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s28203; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s18204; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)8205; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168206; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)8207; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -18208; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s08209; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v18210; SKIP-CACHE-INV-NEXT: s_endpgm8211;8212; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_acq_rel_acquire_ret_cmpxchg:8213; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry8214; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x08215; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x48216; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x88217; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8218; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s48219; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s68220; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s58221; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8222; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168223; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8224; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s48225; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v18226; GFX90A-NOTTGSPLIT-NEXT: s_endpgm8227;8228; GFX90A-TGSPLIT-LABEL: local_workgroup_acq_rel_acquire_ret_cmpxchg:8229; GFX90A-TGSPLIT: ; %bb.0: ; %entry8230; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x08231; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x48232; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x88233; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8234; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s48235; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s68236; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s58237; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8238; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168239; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol8240; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s48241; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8242; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v18243; GFX90A-TGSPLIT-NEXT: s_endpgm8244;8245; GFX942-NOTTGSPLIT-LABEL: local_workgroup_acq_rel_acquire_ret_cmpxchg:8246; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry8247; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x08248; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x48249; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x88250; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8251; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s08252; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s28253; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s18254; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8255; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168256; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8257; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s08258; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v18259; GFX942-NOTTGSPLIT-NEXT: s_endpgm8260;8261; GFX942-TGSPLIT-LABEL: local_workgroup_acq_rel_acquire_ret_cmpxchg:8262; GFX942-TGSPLIT: ; %bb.0: ; %entry8263; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x08264; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x48265; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x88266; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8267; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s08268; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s28269; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s18270; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8271; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168272; GFX942-TGSPLIT-NEXT: buffer_inv sc08273; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s08274; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8275; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v18276; GFX942-TGSPLIT-NEXT: s_endpgm8277;8278; GFX11-WGP-LABEL: local_workgroup_acq_rel_acquire_ret_cmpxchg:8279; GFX11-WGP: ; %bb.0: ; %entry8280; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x08281; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x48282; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x88283; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)8284; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s08285; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s28286; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s18287; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8288; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x08289; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:168290; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)8291; GFX11-WGP-NEXT: buffer_gl0_inv8292; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s08293; GFX11-WGP-NEXT: ds_store_b32 v0, v18294; GFX11-WGP-NEXT: s_endpgm8295;8296; GFX11-CU-LABEL: local_workgroup_acq_rel_acquire_ret_cmpxchg:8297; GFX11-CU: ; %bb.0: ; %entry8298; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x08299; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x48300; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x88301; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)8302; GFX11-CU-NEXT: v_mov_b32_e32 v0, s08303; GFX11-CU-NEXT: v_mov_b32_e32 v1, s28304; GFX11-CU-NEXT: v_mov_b32_e32 v2, s18305; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8306; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x08307; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:168308; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)8309; GFX11-CU-NEXT: v_mov_b32_e32 v0, s08310; GFX11-CU-NEXT: ds_store_b32 v0, v18311; GFX11-CU-NEXT: s_endpgm8312;8313; GFX12-WGP-LABEL: local_workgroup_acq_rel_acquire_ret_cmpxchg:8314; GFX12-WGP: ; %bb.0: ; %entry8315; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x08316; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x48317; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x88318; GFX12-WGP-NEXT: s_wait_kmcnt 0x08319; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s08320; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s28321; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s18322; GFX12-WGP-NEXT: s_wait_bvhcnt 0x08323; GFX12-WGP-NEXT: s_wait_samplecnt 0x08324; GFX12-WGP-NEXT: s_wait_storecnt 0x08325; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x08326; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:168327; GFX12-WGP-NEXT: s_wait_dscnt 0x08328; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE8329; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s08330; GFX12-WGP-NEXT: ds_store_b32 v0, v18331; GFX12-WGP-NEXT: s_endpgm8332;8333; GFX12-CU-LABEL: local_workgroup_acq_rel_acquire_ret_cmpxchg:8334; GFX12-CU: ; %bb.0: ; %entry8335; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x08336; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x48337; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x88338; GFX12-CU-NEXT: s_wait_kmcnt 0x08339; GFX12-CU-NEXT: v_mov_b32_e32 v0, s08340; GFX12-CU-NEXT: v_mov_b32_e32 v1, s28341; GFX12-CU-NEXT: v_mov_b32_e32 v2, s18342; GFX12-CU-NEXT: s_wait_bvhcnt 0x08343; GFX12-CU-NEXT: s_wait_samplecnt 0x08344; GFX12-CU-NEXT: s_wait_storecnt 0x08345; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x08346; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:168347; GFX12-CU-NEXT: s_wait_dscnt 0x08348; GFX12-CU-NEXT: v_mov_b32_e32 v0, s08349; GFX12-CU-NEXT: ds_store_b32 v0, v18350; GFX12-CU-NEXT: s_endpgm8351;8352; GFX1250-LABEL: local_workgroup_acq_rel_acquire_ret_cmpxchg:8353; GFX1250: ; %bb.0: ; %entry8354; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 18355; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x08356; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x48357; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x88358; GFX1250-NEXT: s_wait_kmcnt 0x08359; GFX1250-NEXT: v_mov_b32_e32 v0, s08360; GFX1250-NEXT: v_mov_b32_e32 v1, s28361; GFX1250-NEXT: v_mov_b32_e32 v2, s18362; GFX1250-NEXT: s_wait_storecnt 0x08363; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x08364; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:168365; GFX1250-NEXT: s_wait_dscnt 0x08366; GFX1250-NEXT: v_mov_b32_e32 v0, s08367; GFX1250-NEXT: ds_store_b32 v0, v18368; GFX1250-NEXT: s_endpgm8369 ptr addrspace(3) %out, i32 %in, i32 %old) {8370entry:8371 %gep = getelementptr i32, ptr addrspace(3) %out, i32 48372 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire8373 %val0 = extractvalue { i32, i1 } %val, 08374 store i32 %val0, ptr addrspace(3) %out, align 48375 ret void8376}8377 8378define amdgpu_kernel void @local_workgroup_seq_cst_acquire_ret_cmpxchg(8379; GFX6-LABEL: local_workgroup_seq_cst_acquire_ret_cmpxchg:8380; GFX6: ; %bb.0: ; %entry8381; GFX6-NEXT: s_load_dword s4, s[8:9], 0x08382; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr48383; GFX6-NEXT: s_load_dword s5, s[8:9], 0x18384; GFX6-NEXT: s_load_dword s6, s[8:9], 0x28385; GFX6-NEXT: s_mov_b32 m0, -18386; GFX6-NEXT: s_waitcnt lgkmcnt(0)8387; GFX6-NEXT: v_mov_b32_e32 v0, s48388; GFX6-NEXT: v_mov_b32_e32 v1, s68389; GFX6-NEXT: v_mov_b32_e32 v2, s58390; GFX6-NEXT: s_waitcnt lgkmcnt(0)8391; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168392; GFX6-NEXT: s_waitcnt lgkmcnt(0)8393; GFX6-NEXT: s_mov_b32 m0, -18394; GFX6-NEXT: v_mov_b32_e32 v0, s48395; GFX6-NEXT: ds_write_b32 v0, v18396; GFX6-NEXT: s_endpgm8397;8398; GFX7-LABEL: local_workgroup_seq_cst_acquire_ret_cmpxchg:8399; GFX7: ; %bb.0: ; %entry8400; GFX7-NEXT: s_load_dword s4, s[8:9], 0x08401; GFX7-NEXT: s_load_dword s5, s[8:9], 0x18402; GFX7-NEXT: s_load_dword s6, s[8:9], 0x28403; GFX7-NEXT: s_mov_b32 m0, -18404; GFX7-NEXT: s_waitcnt lgkmcnt(0)8405; GFX7-NEXT: v_mov_b32_e32 v0, s48406; GFX7-NEXT: v_mov_b32_e32 v1, s68407; GFX7-NEXT: v_mov_b32_e32 v2, s58408; GFX7-NEXT: s_waitcnt lgkmcnt(0)8409; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168410; GFX7-NEXT: s_waitcnt lgkmcnt(0)8411; GFX7-NEXT: s_mov_b32 m0, -18412; GFX7-NEXT: v_mov_b32_e32 v0, s48413; GFX7-NEXT: ds_write_b32 v0, v18414; GFX7-NEXT: s_endpgm8415;8416; GFX10-WGP-LABEL: local_workgroup_seq_cst_acquire_ret_cmpxchg:8417; GFX10-WGP: ; %bb.0: ; %entry8418; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x08419; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x48420; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x88421; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)8422; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s48423; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s68424; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s58425; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8426; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x08427; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168428; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)8429; GFX10-WGP-NEXT: buffer_gl0_inv8430; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s48431; GFX10-WGP-NEXT: ds_write_b32 v0, v18432; GFX10-WGP-NEXT: s_endpgm8433;8434; GFX10-CU-LABEL: local_workgroup_seq_cst_acquire_ret_cmpxchg:8435; GFX10-CU: ; %bb.0: ; %entry8436; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x08437; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x48438; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x88439; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)8440; GFX10-CU-NEXT: v_mov_b32_e32 v0, s48441; GFX10-CU-NEXT: v_mov_b32_e32 v1, s68442; GFX10-CU-NEXT: v_mov_b32_e32 v2, s58443; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8444; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x08445; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168446; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)8447; GFX10-CU-NEXT: v_mov_b32_e32 v0, s48448; GFX10-CU-NEXT: ds_write_b32 v0, v18449; GFX10-CU-NEXT: s_endpgm8450;8451; SKIP-CACHE-INV-LABEL: local_workgroup_seq_cst_acquire_ret_cmpxchg:8452; SKIP-CACHE-INV: ; %bb.0: ; %entry8453; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x08454; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x18455; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x28456; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -18457; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)8458; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s08459; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s28460; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s18461; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)8462; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168463; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)8464; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -18465; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s08466; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v18467; SKIP-CACHE-INV-NEXT: s_endpgm8468;8469; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_seq_cst_acquire_ret_cmpxchg:8470; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry8471; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x08472; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x48473; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x88474; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8475; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s48476; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s68477; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s58478; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8479; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168480; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8481; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s48482; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v18483; GFX90A-NOTTGSPLIT-NEXT: s_endpgm8484;8485; GFX90A-TGSPLIT-LABEL: local_workgroup_seq_cst_acquire_ret_cmpxchg:8486; GFX90A-TGSPLIT: ; %bb.0: ; %entry8487; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x08488; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x48489; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x88490; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8491; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s48492; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s68493; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s58494; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8495; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168496; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol8497; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s48498; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8499; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v18500; GFX90A-TGSPLIT-NEXT: s_endpgm8501;8502; GFX942-NOTTGSPLIT-LABEL: local_workgroup_seq_cst_acquire_ret_cmpxchg:8503; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry8504; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x08505; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x48506; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x88507; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8508; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s08509; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s28510; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s18511; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8512; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168513; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8514; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s08515; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v18516; GFX942-NOTTGSPLIT-NEXT: s_endpgm8517;8518; GFX942-TGSPLIT-LABEL: local_workgroup_seq_cst_acquire_ret_cmpxchg:8519; GFX942-TGSPLIT: ; %bb.0: ; %entry8520; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x08521; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x48522; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x88523; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8524; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s08525; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s28526; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s18527; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8528; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168529; GFX942-TGSPLIT-NEXT: buffer_inv sc08530; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s08531; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8532; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v18533; GFX942-TGSPLIT-NEXT: s_endpgm8534;8535; GFX11-WGP-LABEL: local_workgroup_seq_cst_acquire_ret_cmpxchg:8536; GFX11-WGP: ; %bb.0: ; %entry8537; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x08538; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x48539; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x88540; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)8541; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s08542; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s28543; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s18544; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8545; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x08546; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:168547; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)8548; GFX11-WGP-NEXT: buffer_gl0_inv8549; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s08550; GFX11-WGP-NEXT: ds_store_b32 v0, v18551; GFX11-WGP-NEXT: s_endpgm8552;8553; GFX11-CU-LABEL: local_workgroup_seq_cst_acquire_ret_cmpxchg:8554; GFX11-CU: ; %bb.0: ; %entry8555; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x08556; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x48557; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x88558; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)8559; GFX11-CU-NEXT: v_mov_b32_e32 v0, s08560; GFX11-CU-NEXT: v_mov_b32_e32 v1, s28561; GFX11-CU-NEXT: v_mov_b32_e32 v2, s18562; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8563; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x08564; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:168565; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)8566; GFX11-CU-NEXT: v_mov_b32_e32 v0, s08567; GFX11-CU-NEXT: ds_store_b32 v0, v18568; GFX11-CU-NEXT: s_endpgm8569;8570; GFX12-WGP-LABEL: local_workgroup_seq_cst_acquire_ret_cmpxchg:8571; GFX12-WGP: ; %bb.0: ; %entry8572; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x08573; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x48574; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x88575; GFX12-WGP-NEXT: s_wait_kmcnt 0x08576; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s08577; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s28578; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s18579; GFX12-WGP-NEXT: s_wait_bvhcnt 0x08580; GFX12-WGP-NEXT: s_wait_samplecnt 0x08581; GFX12-WGP-NEXT: s_wait_storecnt 0x08582; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x08583; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:168584; GFX12-WGP-NEXT: s_wait_dscnt 0x08585; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE8586; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s08587; GFX12-WGP-NEXT: ds_store_b32 v0, v18588; GFX12-WGP-NEXT: s_endpgm8589;8590; GFX12-CU-LABEL: local_workgroup_seq_cst_acquire_ret_cmpxchg:8591; GFX12-CU: ; %bb.0: ; %entry8592; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x08593; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x48594; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x88595; GFX12-CU-NEXT: s_wait_kmcnt 0x08596; GFX12-CU-NEXT: v_mov_b32_e32 v0, s08597; GFX12-CU-NEXT: v_mov_b32_e32 v1, s28598; GFX12-CU-NEXT: v_mov_b32_e32 v2, s18599; GFX12-CU-NEXT: s_wait_bvhcnt 0x08600; GFX12-CU-NEXT: s_wait_samplecnt 0x08601; GFX12-CU-NEXT: s_wait_storecnt 0x08602; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x08603; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:168604; GFX12-CU-NEXT: s_wait_dscnt 0x08605; GFX12-CU-NEXT: v_mov_b32_e32 v0, s08606; GFX12-CU-NEXT: ds_store_b32 v0, v18607; GFX12-CU-NEXT: s_endpgm8608;8609; GFX1250-LABEL: local_workgroup_seq_cst_acquire_ret_cmpxchg:8610; GFX1250: ; %bb.0: ; %entry8611; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 18612; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x08613; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x48614; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x88615; GFX1250-NEXT: s_wait_kmcnt 0x08616; GFX1250-NEXT: v_mov_b32_e32 v0, s08617; GFX1250-NEXT: v_mov_b32_e32 v1, s28618; GFX1250-NEXT: v_mov_b32_e32 v2, s18619; GFX1250-NEXT: s_wait_storecnt 0x08620; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x08621; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:168622; GFX1250-NEXT: s_wait_dscnt 0x08623; GFX1250-NEXT: v_mov_b32_e32 v0, s08624; GFX1250-NEXT: ds_store_b32 v0, v18625; GFX1250-NEXT: s_endpgm8626 ptr addrspace(3) %out, i32 %in, i32 %old) {8627entry:8628 %gep = getelementptr i32, ptr addrspace(3) %out, i32 48629 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire8630 %val0 = extractvalue { i32, i1 } %val, 08631 store i32 %val0, ptr addrspace(3) %out, align 48632 ret void8633}8634 8635define amdgpu_kernel void @local_workgroup_monotonic_seq_cst_ret_cmpxchg(8636; GFX6-LABEL: local_workgroup_monotonic_seq_cst_ret_cmpxchg:8637; GFX6: ; %bb.0: ; %entry8638; GFX6-NEXT: s_load_dword s4, s[8:9], 0x08639; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr48640; GFX6-NEXT: s_load_dword s5, s[8:9], 0x18641; GFX6-NEXT: s_load_dword s6, s[8:9], 0x28642; GFX6-NEXT: s_mov_b32 m0, -18643; GFX6-NEXT: s_waitcnt lgkmcnt(0)8644; GFX6-NEXT: v_mov_b32_e32 v0, s48645; GFX6-NEXT: v_mov_b32_e32 v1, s68646; GFX6-NEXT: v_mov_b32_e32 v2, s58647; GFX6-NEXT: s_waitcnt lgkmcnt(0)8648; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168649; GFX6-NEXT: s_waitcnt lgkmcnt(0)8650; GFX6-NEXT: s_mov_b32 m0, -18651; GFX6-NEXT: v_mov_b32_e32 v0, s48652; GFX6-NEXT: ds_write_b32 v0, v18653; GFX6-NEXT: s_endpgm8654;8655; GFX7-LABEL: local_workgroup_monotonic_seq_cst_ret_cmpxchg:8656; GFX7: ; %bb.0: ; %entry8657; GFX7-NEXT: s_load_dword s4, s[8:9], 0x08658; GFX7-NEXT: s_load_dword s5, s[8:9], 0x18659; GFX7-NEXT: s_load_dword s6, s[8:9], 0x28660; GFX7-NEXT: s_mov_b32 m0, -18661; GFX7-NEXT: s_waitcnt lgkmcnt(0)8662; GFX7-NEXT: v_mov_b32_e32 v0, s48663; GFX7-NEXT: v_mov_b32_e32 v1, s68664; GFX7-NEXT: v_mov_b32_e32 v2, s58665; GFX7-NEXT: s_waitcnt lgkmcnt(0)8666; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168667; GFX7-NEXT: s_waitcnt lgkmcnt(0)8668; GFX7-NEXT: s_mov_b32 m0, -18669; GFX7-NEXT: v_mov_b32_e32 v0, s48670; GFX7-NEXT: ds_write_b32 v0, v18671; GFX7-NEXT: s_endpgm8672;8673; GFX10-WGP-LABEL: local_workgroup_monotonic_seq_cst_ret_cmpxchg:8674; GFX10-WGP: ; %bb.0: ; %entry8675; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x08676; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x48677; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x88678; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)8679; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s48680; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s68681; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s58682; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8683; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x08684; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168685; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)8686; GFX10-WGP-NEXT: buffer_gl0_inv8687; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s48688; GFX10-WGP-NEXT: ds_write_b32 v0, v18689; GFX10-WGP-NEXT: s_endpgm8690;8691; GFX10-CU-LABEL: local_workgroup_monotonic_seq_cst_ret_cmpxchg:8692; GFX10-CU: ; %bb.0: ; %entry8693; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x08694; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x48695; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x88696; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)8697; GFX10-CU-NEXT: v_mov_b32_e32 v0, s48698; GFX10-CU-NEXT: v_mov_b32_e32 v1, s68699; GFX10-CU-NEXT: v_mov_b32_e32 v2, s58700; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8701; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x08702; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168703; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)8704; GFX10-CU-NEXT: v_mov_b32_e32 v0, s48705; GFX10-CU-NEXT: ds_write_b32 v0, v18706; GFX10-CU-NEXT: s_endpgm8707;8708; SKIP-CACHE-INV-LABEL: local_workgroup_monotonic_seq_cst_ret_cmpxchg:8709; SKIP-CACHE-INV: ; %bb.0: ; %entry8710; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x08711; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x18712; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x28713; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -18714; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)8715; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s08716; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s28717; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s18718; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)8719; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168720; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)8721; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -18722; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s08723; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v18724; SKIP-CACHE-INV-NEXT: s_endpgm8725;8726; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_monotonic_seq_cst_ret_cmpxchg:8727; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry8728; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x08729; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x48730; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x88731; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8732; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s48733; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s68734; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s58735; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8736; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168737; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8738; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s48739; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v18740; GFX90A-NOTTGSPLIT-NEXT: s_endpgm8741;8742; GFX90A-TGSPLIT-LABEL: local_workgroup_monotonic_seq_cst_ret_cmpxchg:8743; GFX90A-TGSPLIT: ; %bb.0: ; %entry8744; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x08745; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x48746; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x88747; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8748; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s48749; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s68750; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s58751; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8752; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168753; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol8754; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s48755; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8756; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v18757; GFX90A-TGSPLIT-NEXT: s_endpgm8758;8759; GFX942-NOTTGSPLIT-LABEL: local_workgroup_monotonic_seq_cst_ret_cmpxchg:8760; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry8761; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x08762; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x48763; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x88764; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8765; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s08766; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s28767; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s18768; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8769; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168770; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8771; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s08772; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v18773; GFX942-NOTTGSPLIT-NEXT: s_endpgm8774;8775; GFX942-TGSPLIT-LABEL: local_workgroup_monotonic_seq_cst_ret_cmpxchg:8776; GFX942-TGSPLIT: ; %bb.0: ; %entry8777; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x08778; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x48779; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x88780; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8781; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s08782; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s28783; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s18784; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8785; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168786; GFX942-TGSPLIT-NEXT: buffer_inv sc08787; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s08788; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8789; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v18790; GFX942-TGSPLIT-NEXT: s_endpgm8791;8792; GFX11-WGP-LABEL: local_workgroup_monotonic_seq_cst_ret_cmpxchg:8793; GFX11-WGP: ; %bb.0: ; %entry8794; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x08795; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x48796; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x88797; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)8798; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s08799; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s28800; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s18801; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8802; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x08803; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:168804; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)8805; GFX11-WGP-NEXT: buffer_gl0_inv8806; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s08807; GFX11-WGP-NEXT: ds_store_b32 v0, v18808; GFX11-WGP-NEXT: s_endpgm8809;8810; GFX11-CU-LABEL: local_workgroup_monotonic_seq_cst_ret_cmpxchg:8811; GFX11-CU: ; %bb.0: ; %entry8812; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x08813; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x48814; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x88815; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)8816; GFX11-CU-NEXT: v_mov_b32_e32 v0, s08817; GFX11-CU-NEXT: v_mov_b32_e32 v1, s28818; GFX11-CU-NEXT: v_mov_b32_e32 v2, s18819; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8820; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x08821; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:168822; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)8823; GFX11-CU-NEXT: v_mov_b32_e32 v0, s08824; GFX11-CU-NEXT: ds_store_b32 v0, v18825; GFX11-CU-NEXT: s_endpgm8826;8827; GFX12-WGP-LABEL: local_workgroup_monotonic_seq_cst_ret_cmpxchg:8828; GFX12-WGP: ; %bb.0: ; %entry8829; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x08830; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x48831; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x88832; GFX12-WGP-NEXT: s_wait_kmcnt 0x08833; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s08834; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s28835; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s18836; GFX12-WGP-NEXT: s_wait_bvhcnt 0x08837; GFX12-WGP-NEXT: s_wait_samplecnt 0x08838; GFX12-WGP-NEXT: s_wait_storecnt 0x08839; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x08840; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:168841; GFX12-WGP-NEXT: s_wait_dscnt 0x08842; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE8843; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s08844; GFX12-WGP-NEXT: ds_store_b32 v0, v18845; GFX12-WGP-NEXT: s_endpgm8846;8847; GFX12-CU-LABEL: local_workgroup_monotonic_seq_cst_ret_cmpxchg:8848; GFX12-CU: ; %bb.0: ; %entry8849; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x08850; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x48851; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x88852; GFX12-CU-NEXT: s_wait_kmcnt 0x08853; GFX12-CU-NEXT: v_mov_b32_e32 v0, s08854; GFX12-CU-NEXT: v_mov_b32_e32 v1, s28855; GFX12-CU-NEXT: v_mov_b32_e32 v2, s18856; GFX12-CU-NEXT: s_wait_bvhcnt 0x08857; GFX12-CU-NEXT: s_wait_samplecnt 0x08858; GFX12-CU-NEXT: s_wait_storecnt 0x08859; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x08860; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:168861; GFX12-CU-NEXT: s_wait_dscnt 0x08862; GFX12-CU-NEXT: v_mov_b32_e32 v0, s08863; GFX12-CU-NEXT: ds_store_b32 v0, v18864; GFX12-CU-NEXT: s_endpgm8865;8866; GFX1250-LABEL: local_workgroup_monotonic_seq_cst_ret_cmpxchg:8867; GFX1250: ; %bb.0: ; %entry8868; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 18869; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x08870; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x48871; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x88872; GFX1250-NEXT: s_wait_kmcnt 0x08873; GFX1250-NEXT: v_mov_b32_e32 v0, s08874; GFX1250-NEXT: v_mov_b32_e32 v1, s28875; GFX1250-NEXT: v_mov_b32_e32 v2, s18876; GFX1250-NEXT: s_wait_storecnt 0x08877; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x08878; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:168879; GFX1250-NEXT: s_wait_dscnt 0x08880; GFX1250-NEXT: v_mov_b32_e32 v0, s08881; GFX1250-NEXT: ds_store_b32 v0, v18882; GFX1250-NEXT: s_endpgm8883 ptr addrspace(3) %out, i32 %in, i32 %old) {8884entry:8885 %gep = getelementptr i32, ptr addrspace(3) %out, i32 48886 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") monotonic seq_cst8887 %val0 = extractvalue { i32, i1 } %val, 08888 store i32 %val0, ptr addrspace(3) %out, align 48889 ret void8890}8891 8892define amdgpu_kernel void @local_workgroup_acquire_seq_cst_ret_cmpxchg(8893; GFX6-LABEL: local_workgroup_acquire_seq_cst_ret_cmpxchg:8894; GFX6: ; %bb.0: ; %entry8895; GFX6-NEXT: s_load_dword s4, s[8:9], 0x08896; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr48897; GFX6-NEXT: s_load_dword s5, s[8:9], 0x18898; GFX6-NEXT: s_load_dword s6, s[8:9], 0x28899; GFX6-NEXT: s_mov_b32 m0, -18900; GFX6-NEXT: s_waitcnt lgkmcnt(0)8901; GFX6-NEXT: v_mov_b32_e32 v0, s48902; GFX6-NEXT: v_mov_b32_e32 v1, s68903; GFX6-NEXT: v_mov_b32_e32 v2, s58904; GFX6-NEXT: s_waitcnt lgkmcnt(0)8905; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168906; GFX6-NEXT: s_waitcnt lgkmcnt(0)8907; GFX6-NEXT: s_mov_b32 m0, -18908; GFX6-NEXT: v_mov_b32_e32 v0, s48909; GFX6-NEXT: ds_write_b32 v0, v18910; GFX6-NEXT: s_endpgm8911;8912; GFX7-LABEL: local_workgroup_acquire_seq_cst_ret_cmpxchg:8913; GFX7: ; %bb.0: ; %entry8914; GFX7-NEXT: s_load_dword s4, s[8:9], 0x08915; GFX7-NEXT: s_load_dword s5, s[8:9], 0x18916; GFX7-NEXT: s_load_dword s6, s[8:9], 0x28917; GFX7-NEXT: s_mov_b32 m0, -18918; GFX7-NEXT: s_waitcnt lgkmcnt(0)8919; GFX7-NEXT: v_mov_b32_e32 v0, s48920; GFX7-NEXT: v_mov_b32_e32 v1, s68921; GFX7-NEXT: v_mov_b32_e32 v2, s58922; GFX7-NEXT: s_waitcnt lgkmcnt(0)8923; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168924; GFX7-NEXT: s_waitcnt lgkmcnt(0)8925; GFX7-NEXT: s_mov_b32 m0, -18926; GFX7-NEXT: v_mov_b32_e32 v0, s48927; GFX7-NEXT: ds_write_b32 v0, v18928; GFX7-NEXT: s_endpgm8929;8930; GFX10-WGP-LABEL: local_workgroup_acquire_seq_cst_ret_cmpxchg:8931; GFX10-WGP: ; %bb.0: ; %entry8932; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x08933; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x48934; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x88935; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)8936; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s48937; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s68938; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s58939; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8940; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x08941; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168942; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)8943; GFX10-WGP-NEXT: buffer_gl0_inv8944; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s48945; GFX10-WGP-NEXT: ds_write_b32 v0, v18946; GFX10-WGP-NEXT: s_endpgm8947;8948; GFX10-CU-LABEL: local_workgroup_acquire_seq_cst_ret_cmpxchg:8949; GFX10-CU: ; %bb.0: ; %entry8950; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x08951; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x48952; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x88953; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)8954; GFX10-CU-NEXT: v_mov_b32_e32 v0, s48955; GFX10-CU-NEXT: v_mov_b32_e32 v1, s68956; GFX10-CU-NEXT: v_mov_b32_e32 v2, s58957; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8958; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x08959; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168960; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)8961; GFX10-CU-NEXT: v_mov_b32_e32 v0, s48962; GFX10-CU-NEXT: ds_write_b32 v0, v18963; GFX10-CU-NEXT: s_endpgm8964;8965; SKIP-CACHE-INV-LABEL: local_workgroup_acquire_seq_cst_ret_cmpxchg:8966; SKIP-CACHE-INV: ; %bb.0: ; %entry8967; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x08968; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x18969; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x28970; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -18971; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)8972; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s08973; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s28974; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s18975; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)8976; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168977; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)8978; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -18979; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s08980; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v18981; SKIP-CACHE-INV-NEXT: s_endpgm8982;8983; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_acquire_seq_cst_ret_cmpxchg:8984; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry8985; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x08986; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x48987; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x88988; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8989; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s48990; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s68991; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s58992; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8993; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:168994; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)8995; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s48996; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v18997; GFX90A-NOTTGSPLIT-NEXT: s_endpgm8998;8999; GFX90A-TGSPLIT-LABEL: local_workgroup_acquire_seq_cst_ret_cmpxchg:9000; GFX90A-TGSPLIT: ; %bb.0: ; %entry9001; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x09002; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x49003; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x89004; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9005; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s49006; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s69007; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s59008; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9009; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169010; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol9011; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s49012; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9013; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v19014; GFX90A-TGSPLIT-NEXT: s_endpgm9015;9016; GFX942-NOTTGSPLIT-LABEL: local_workgroup_acquire_seq_cst_ret_cmpxchg:9017; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry9018; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x09019; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x49020; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x89021; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9022; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s09023; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s29024; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s19025; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9026; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169027; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9028; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s09029; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v19030; GFX942-NOTTGSPLIT-NEXT: s_endpgm9031;9032; GFX942-TGSPLIT-LABEL: local_workgroup_acquire_seq_cst_ret_cmpxchg:9033; GFX942-TGSPLIT: ; %bb.0: ; %entry9034; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x09035; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x49036; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x89037; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9038; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s09039; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s29040; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s19041; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9042; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169043; GFX942-TGSPLIT-NEXT: buffer_inv sc09044; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s09045; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9046; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v19047; GFX942-TGSPLIT-NEXT: s_endpgm9048;9049; GFX11-WGP-LABEL: local_workgroup_acquire_seq_cst_ret_cmpxchg:9050; GFX11-WGP: ; %bb.0: ; %entry9051; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x09052; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x49053; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x89054; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)9055; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s09056; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s29057; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s19058; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9059; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x09060; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:169061; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)9062; GFX11-WGP-NEXT: buffer_gl0_inv9063; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s09064; GFX11-WGP-NEXT: ds_store_b32 v0, v19065; GFX11-WGP-NEXT: s_endpgm9066;9067; GFX11-CU-LABEL: local_workgroup_acquire_seq_cst_ret_cmpxchg:9068; GFX11-CU: ; %bb.0: ; %entry9069; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x09070; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x49071; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x89072; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)9073; GFX11-CU-NEXT: v_mov_b32_e32 v0, s09074; GFX11-CU-NEXT: v_mov_b32_e32 v1, s29075; GFX11-CU-NEXT: v_mov_b32_e32 v2, s19076; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9077; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x09078; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:169079; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)9080; GFX11-CU-NEXT: v_mov_b32_e32 v0, s09081; GFX11-CU-NEXT: ds_store_b32 v0, v19082; GFX11-CU-NEXT: s_endpgm9083;9084; GFX12-WGP-LABEL: local_workgroup_acquire_seq_cst_ret_cmpxchg:9085; GFX12-WGP: ; %bb.0: ; %entry9086; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x09087; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x49088; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x89089; GFX12-WGP-NEXT: s_wait_kmcnt 0x09090; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s09091; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s29092; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s19093; GFX12-WGP-NEXT: s_wait_bvhcnt 0x09094; GFX12-WGP-NEXT: s_wait_samplecnt 0x09095; GFX12-WGP-NEXT: s_wait_storecnt 0x09096; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x09097; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:169098; GFX12-WGP-NEXT: s_wait_dscnt 0x09099; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE9100; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s09101; GFX12-WGP-NEXT: ds_store_b32 v0, v19102; GFX12-WGP-NEXT: s_endpgm9103;9104; GFX12-CU-LABEL: local_workgroup_acquire_seq_cst_ret_cmpxchg:9105; GFX12-CU: ; %bb.0: ; %entry9106; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x09107; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x49108; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x89109; GFX12-CU-NEXT: s_wait_kmcnt 0x09110; GFX12-CU-NEXT: v_mov_b32_e32 v0, s09111; GFX12-CU-NEXT: v_mov_b32_e32 v1, s29112; GFX12-CU-NEXT: v_mov_b32_e32 v2, s19113; GFX12-CU-NEXT: s_wait_bvhcnt 0x09114; GFX12-CU-NEXT: s_wait_samplecnt 0x09115; GFX12-CU-NEXT: s_wait_storecnt 0x09116; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x09117; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:169118; GFX12-CU-NEXT: s_wait_dscnt 0x09119; GFX12-CU-NEXT: v_mov_b32_e32 v0, s09120; GFX12-CU-NEXT: ds_store_b32 v0, v19121; GFX12-CU-NEXT: s_endpgm9122;9123; GFX1250-LABEL: local_workgroup_acquire_seq_cst_ret_cmpxchg:9124; GFX1250: ; %bb.0: ; %entry9125; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 19126; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x09127; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x49128; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x89129; GFX1250-NEXT: s_wait_kmcnt 0x09130; GFX1250-NEXT: v_mov_b32_e32 v0, s09131; GFX1250-NEXT: v_mov_b32_e32 v1, s29132; GFX1250-NEXT: v_mov_b32_e32 v2, s19133; GFX1250-NEXT: s_wait_storecnt 0x09134; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x09135; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:169136; GFX1250-NEXT: s_wait_dscnt 0x09137; GFX1250-NEXT: v_mov_b32_e32 v0, s09138; GFX1250-NEXT: ds_store_b32 v0, v19139; GFX1250-NEXT: s_endpgm9140 ptr addrspace(3) %out, i32 %in, i32 %old) {9141entry:9142 %gep = getelementptr i32, ptr addrspace(3) %out, i32 49143 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") acquire seq_cst9144 %val0 = extractvalue { i32, i1 } %val, 09145 store i32 %val0, ptr addrspace(3) %out, align 49146 ret void9147}9148 9149define amdgpu_kernel void @local_workgroup_release_seq_cst_ret_cmpxchg(9150; GFX6-LABEL: local_workgroup_release_seq_cst_ret_cmpxchg:9151; GFX6: ; %bb.0: ; %entry9152; GFX6-NEXT: s_load_dword s4, s[8:9], 0x09153; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr49154; GFX6-NEXT: s_load_dword s5, s[8:9], 0x19155; GFX6-NEXT: s_load_dword s6, s[8:9], 0x29156; GFX6-NEXT: s_mov_b32 m0, -19157; GFX6-NEXT: s_waitcnt lgkmcnt(0)9158; GFX6-NEXT: v_mov_b32_e32 v0, s49159; GFX6-NEXT: v_mov_b32_e32 v1, s69160; GFX6-NEXT: v_mov_b32_e32 v2, s59161; GFX6-NEXT: s_waitcnt lgkmcnt(0)9162; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169163; GFX6-NEXT: s_waitcnt lgkmcnt(0)9164; GFX6-NEXT: s_mov_b32 m0, -19165; GFX6-NEXT: v_mov_b32_e32 v0, s49166; GFX6-NEXT: ds_write_b32 v0, v19167; GFX6-NEXT: s_endpgm9168;9169; GFX7-LABEL: local_workgroup_release_seq_cst_ret_cmpxchg:9170; GFX7: ; %bb.0: ; %entry9171; GFX7-NEXT: s_load_dword s4, s[8:9], 0x09172; GFX7-NEXT: s_load_dword s5, s[8:9], 0x19173; GFX7-NEXT: s_load_dword s6, s[8:9], 0x29174; GFX7-NEXT: s_mov_b32 m0, -19175; GFX7-NEXT: s_waitcnt lgkmcnt(0)9176; GFX7-NEXT: v_mov_b32_e32 v0, s49177; GFX7-NEXT: v_mov_b32_e32 v1, s69178; GFX7-NEXT: v_mov_b32_e32 v2, s59179; GFX7-NEXT: s_waitcnt lgkmcnt(0)9180; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169181; GFX7-NEXT: s_waitcnt lgkmcnt(0)9182; GFX7-NEXT: s_mov_b32 m0, -19183; GFX7-NEXT: v_mov_b32_e32 v0, s49184; GFX7-NEXT: ds_write_b32 v0, v19185; GFX7-NEXT: s_endpgm9186;9187; GFX10-WGP-LABEL: local_workgroup_release_seq_cst_ret_cmpxchg:9188; GFX10-WGP: ; %bb.0: ; %entry9189; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x09190; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x49191; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x89192; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)9193; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s49194; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s69195; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s59196; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9197; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x09198; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169199; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)9200; GFX10-WGP-NEXT: buffer_gl0_inv9201; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s49202; GFX10-WGP-NEXT: ds_write_b32 v0, v19203; GFX10-WGP-NEXT: s_endpgm9204;9205; GFX10-CU-LABEL: local_workgroup_release_seq_cst_ret_cmpxchg:9206; GFX10-CU: ; %bb.0: ; %entry9207; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x09208; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x49209; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x89210; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)9211; GFX10-CU-NEXT: v_mov_b32_e32 v0, s49212; GFX10-CU-NEXT: v_mov_b32_e32 v1, s69213; GFX10-CU-NEXT: v_mov_b32_e32 v2, s59214; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9215; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x09216; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169217; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)9218; GFX10-CU-NEXT: v_mov_b32_e32 v0, s49219; GFX10-CU-NEXT: ds_write_b32 v0, v19220; GFX10-CU-NEXT: s_endpgm9221;9222; SKIP-CACHE-INV-LABEL: local_workgroup_release_seq_cst_ret_cmpxchg:9223; SKIP-CACHE-INV: ; %bb.0: ; %entry9224; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x09225; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x19226; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x29227; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -19228; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)9229; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s09230; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s29231; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s19232; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)9233; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169234; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)9235; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -19236; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s09237; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v19238; SKIP-CACHE-INV-NEXT: s_endpgm9239;9240; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_release_seq_cst_ret_cmpxchg:9241; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry9242; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x09243; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x49244; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x89245; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9246; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s49247; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s69248; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s59249; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9250; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169251; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9252; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s49253; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v19254; GFX90A-NOTTGSPLIT-NEXT: s_endpgm9255;9256; GFX90A-TGSPLIT-LABEL: local_workgroup_release_seq_cst_ret_cmpxchg:9257; GFX90A-TGSPLIT: ; %bb.0: ; %entry9258; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x09259; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x49260; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x89261; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9262; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s49263; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s69264; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s59265; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9266; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169267; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol9268; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s49269; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9270; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v19271; GFX90A-TGSPLIT-NEXT: s_endpgm9272;9273; GFX942-NOTTGSPLIT-LABEL: local_workgroup_release_seq_cst_ret_cmpxchg:9274; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry9275; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x09276; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x49277; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x89278; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9279; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s09280; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s29281; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s19282; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9283; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169284; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9285; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s09286; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v19287; GFX942-NOTTGSPLIT-NEXT: s_endpgm9288;9289; GFX942-TGSPLIT-LABEL: local_workgroup_release_seq_cst_ret_cmpxchg:9290; GFX942-TGSPLIT: ; %bb.0: ; %entry9291; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x09292; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x49293; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x89294; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9295; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s09296; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s29297; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s19298; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9299; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169300; GFX942-TGSPLIT-NEXT: buffer_inv sc09301; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s09302; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9303; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v19304; GFX942-TGSPLIT-NEXT: s_endpgm9305;9306; GFX11-WGP-LABEL: local_workgroup_release_seq_cst_ret_cmpxchg:9307; GFX11-WGP: ; %bb.0: ; %entry9308; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x09309; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x49310; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x89311; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)9312; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s09313; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s29314; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s19315; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9316; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x09317; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:169318; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)9319; GFX11-WGP-NEXT: buffer_gl0_inv9320; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s09321; GFX11-WGP-NEXT: ds_store_b32 v0, v19322; GFX11-WGP-NEXT: s_endpgm9323;9324; GFX11-CU-LABEL: local_workgroup_release_seq_cst_ret_cmpxchg:9325; GFX11-CU: ; %bb.0: ; %entry9326; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x09327; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x49328; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x89329; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)9330; GFX11-CU-NEXT: v_mov_b32_e32 v0, s09331; GFX11-CU-NEXT: v_mov_b32_e32 v1, s29332; GFX11-CU-NEXT: v_mov_b32_e32 v2, s19333; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9334; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x09335; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:169336; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)9337; GFX11-CU-NEXT: v_mov_b32_e32 v0, s09338; GFX11-CU-NEXT: ds_store_b32 v0, v19339; GFX11-CU-NEXT: s_endpgm9340;9341; GFX12-WGP-LABEL: local_workgroup_release_seq_cst_ret_cmpxchg:9342; GFX12-WGP: ; %bb.0: ; %entry9343; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x09344; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x49345; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x89346; GFX12-WGP-NEXT: s_wait_kmcnt 0x09347; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s09348; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s29349; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s19350; GFX12-WGP-NEXT: s_wait_bvhcnt 0x09351; GFX12-WGP-NEXT: s_wait_samplecnt 0x09352; GFX12-WGP-NEXT: s_wait_storecnt 0x09353; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x09354; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:169355; GFX12-WGP-NEXT: s_wait_dscnt 0x09356; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE9357; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s09358; GFX12-WGP-NEXT: ds_store_b32 v0, v19359; GFX12-WGP-NEXT: s_endpgm9360;9361; GFX12-CU-LABEL: local_workgroup_release_seq_cst_ret_cmpxchg:9362; GFX12-CU: ; %bb.0: ; %entry9363; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x09364; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x49365; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x89366; GFX12-CU-NEXT: s_wait_kmcnt 0x09367; GFX12-CU-NEXT: v_mov_b32_e32 v0, s09368; GFX12-CU-NEXT: v_mov_b32_e32 v1, s29369; GFX12-CU-NEXT: v_mov_b32_e32 v2, s19370; GFX12-CU-NEXT: s_wait_bvhcnt 0x09371; GFX12-CU-NEXT: s_wait_samplecnt 0x09372; GFX12-CU-NEXT: s_wait_storecnt 0x09373; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x09374; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:169375; GFX12-CU-NEXT: s_wait_dscnt 0x09376; GFX12-CU-NEXT: v_mov_b32_e32 v0, s09377; GFX12-CU-NEXT: ds_store_b32 v0, v19378; GFX12-CU-NEXT: s_endpgm9379;9380; GFX1250-LABEL: local_workgroup_release_seq_cst_ret_cmpxchg:9381; GFX1250: ; %bb.0: ; %entry9382; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 19383; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x09384; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x49385; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x89386; GFX1250-NEXT: s_wait_kmcnt 0x09387; GFX1250-NEXT: v_mov_b32_e32 v0, s09388; GFX1250-NEXT: v_mov_b32_e32 v1, s29389; GFX1250-NEXT: v_mov_b32_e32 v2, s19390; GFX1250-NEXT: s_wait_storecnt 0x09391; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x09392; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:169393; GFX1250-NEXT: s_wait_dscnt 0x09394; GFX1250-NEXT: v_mov_b32_e32 v0, s09395; GFX1250-NEXT: ds_store_b32 v0, v19396; GFX1250-NEXT: s_endpgm9397 ptr addrspace(3) %out, i32 %in, i32 %old) {9398entry:9399 %gep = getelementptr i32, ptr addrspace(3) %out, i32 49400 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") release seq_cst9401 %val0 = extractvalue { i32, i1 } %val, 09402 store i32 %val0, ptr addrspace(3) %out, align 49403 ret void9404}9405 9406define amdgpu_kernel void @local_workgroup_acq_rel_seq_cst_ret_cmpxchg(9407; GFX6-LABEL: local_workgroup_acq_rel_seq_cst_ret_cmpxchg:9408; GFX6: ; %bb.0: ; %entry9409; GFX6-NEXT: s_load_dword s4, s[8:9], 0x09410; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr49411; GFX6-NEXT: s_load_dword s5, s[8:9], 0x19412; GFX6-NEXT: s_load_dword s6, s[8:9], 0x29413; GFX6-NEXT: s_mov_b32 m0, -19414; GFX6-NEXT: s_waitcnt lgkmcnt(0)9415; GFX6-NEXT: v_mov_b32_e32 v0, s49416; GFX6-NEXT: v_mov_b32_e32 v1, s69417; GFX6-NEXT: v_mov_b32_e32 v2, s59418; GFX6-NEXT: s_waitcnt lgkmcnt(0)9419; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169420; GFX6-NEXT: s_waitcnt lgkmcnt(0)9421; GFX6-NEXT: s_mov_b32 m0, -19422; GFX6-NEXT: v_mov_b32_e32 v0, s49423; GFX6-NEXT: ds_write_b32 v0, v19424; GFX6-NEXT: s_endpgm9425;9426; GFX7-LABEL: local_workgroup_acq_rel_seq_cst_ret_cmpxchg:9427; GFX7: ; %bb.0: ; %entry9428; GFX7-NEXT: s_load_dword s4, s[8:9], 0x09429; GFX7-NEXT: s_load_dword s5, s[8:9], 0x19430; GFX7-NEXT: s_load_dword s6, s[8:9], 0x29431; GFX7-NEXT: s_mov_b32 m0, -19432; GFX7-NEXT: s_waitcnt lgkmcnt(0)9433; GFX7-NEXT: v_mov_b32_e32 v0, s49434; GFX7-NEXT: v_mov_b32_e32 v1, s69435; GFX7-NEXT: v_mov_b32_e32 v2, s59436; GFX7-NEXT: s_waitcnt lgkmcnt(0)9437; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169438; GFX7-NEXT: s_waitcnt lgkmcnt(0)9439; GFX7-NEXT: s_mov_b32 m0, -19440; GFX7-NEXT: v_mov_b32_e32 v0, s49441; GFX7-NEXT: ds_write_b32 v0, v19442; GFX7-NEXT: s_endpgm9443;9444; GFX10-WGP-LABEL: local_workgroup_acq_rel_seq_cst_ret_cmpxchg:9445; GFX10-WGP: ; %bb.0: ; %entry9446; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x09447; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x49448; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x89449; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)9450; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s49451; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s69452; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s59453; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9454; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x09455; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169456; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)9457; GFX10-WGP-NEXT: buffer_gl0_inv9458; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s49459; GFX10-WGP-NEXT: ds_write_b32 v0, v19460; GFX10-WGP-NEXT: s_endpgm9461;9462; GFX10-CU-LABEL: local_workgroup_acq_rel_seq_cst_ret_cmpxchg:9463; GFX10-CU: ; %bb.0: ; %entry9464; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x09465; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x49466; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x89467; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)9468; GFX10-CU-NEXT: v_mov_b32_e32 v0, s49469; GFX10-CU-NEXT: v_mov_b32_e32 v1, s69470; GFX10-CU-NEXT: v_mov_b32_e32 v2, s59471; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9472; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x09473; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169474; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)9475; GFX10-CU-NEXT: v_mov_b32_e32 v0, s49476; GFX10-CU-NEXT: ds_write_b32 v0, v19477; GFX10-CU-NEXT: s_endpgm9478;9479; SKIP-CACHE-INV-LABEL: local_workgroup_acq_rel_seq_cst_ret_cmpxchg:9480; SKIP-CACHE-INV: ; %bb.0: ; %entry9481; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x09482; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x19483; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x29484; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -19485; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)9486; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s09487; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s29488; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s19489; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)9490; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169491; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)9492; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -19493; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s09494; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v19495; SKIP-CACHE-INV-NEXT: s_endpgm9496;9497; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_acq_rel_seq_cst_ret_cmpxchg:9498; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry9499; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x09500; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x49501; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x89502; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9503; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s49504; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s69505; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s59506; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9507; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169508; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9509; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s49510; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v19511; GFX90A-NOTTGSPLIT-NEXT: s_endpgm9512;9513; GFX90A-TGSPLIT-LABEL: local_workgroup_acq_rel_seq_cst_ret_cmpxchg:9514; GFX90A-TGSPLIT: ; %bb.0: ; %entry9515; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x09516; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x49517; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x89518; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9519; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s49520; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s69521; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s59522; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9523; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169524; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol9525; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s49526; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9527; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v19528; GFX90A-TGSPLIT-NEXT: s_endpgm9529;9530; GFX942-NOTTGSPLIT-LABEL: local_workgroup_acq_rel_seq_cst_ret_cmpxchg:9531; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry9532; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x09533; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x49534; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x89535; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9536; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s09537; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s29538; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s19539; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9540; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169541; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9542; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s09543; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v19544; GFX942-NOTTGSPLIT-NEXT: s_endpgm9545;9546; GFX942-TGSPLIT-LABEL: local_workgroup_acq_rel_seq_cst_ret_cmpxchg:9547; GFX942-TGSPLIT: ; %bb.0: ; %entry9548; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x09549; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x49550; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x89551; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9552; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s09553; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s29554; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s19555; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9556; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169557; GFX942-TGSPLIT-NEXT: buffer_inv sc09558; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s09559; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9560; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v19561; GFX942-TGSPLIT-NEXT: s_endpgm9562;9563; GFX11-WGP-LABEL: local_workgroup_acq_rel_seq_cst_ret_cmpxchg:9564; GFX11-WGP: ; %bb.0: ; %entry9565; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x09566; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x49567; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x89568; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)9569; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s09570; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s29571; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s19572; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9573; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x09574; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:169575; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)9576; GFX11-WGP-NEXT: buffer_gl0_inv9577; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s09578; GFX11-WGP-NEXT: ds_store_b32 v0, v19579; GFX11-WGP-NEXT: s_endpgm9580;9581; GFX11-CU-LABEL: local_workgroup_acq_rel_seq_cst_ret_cmpxchg:9582; GFX11-CU: ; %bb.0: ; %entry9583; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x09584; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x49585; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x89586; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)9587; GFX11-CU-NEXT: v_mov_b32_e32 v0, s09588; GFX11-CU-NEXT: v_mov_b32_e32 v1, s29589; GFX11-CU-NEXT: v_mov_b32_e32 v2, s19590; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9591; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x09592; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:169593; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)9594; GFX11-CU-NEXT: v_mov_b32_e32 v0, s09595; GFX11-CU-NEXT: ds_store_b32 v0, v19596; GFX11-CU-NEXT: s_endpgm9597;9598; GFX12-WGP-LABEL: local_workgroup_acq_rel_seq_cst_ret_cmpxchg:9599; GFX12-WGP: ; %bb.0: ; %entry9600; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x09601; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x49602; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x89603; GFX12-WGP-NEXT: s_wait_kmcnt 0x09604; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s09605; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s29606; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s19607; GFX12-WGP-NEXT: s_wait_bvhcnt 0x09608; GFX12-WGP-NEXT: s_wait_samplecnt 0x09609; GFX12-WGP-NEXT: s_wait_storecnt 0x09610; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x09611; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:169612; GFX12-WGP-NEXT: s_wait_dscnt 0x09613; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE9614; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s09615; GFX12-WGP-NEXT: ds_store_b32 v0, v19616; GFX12-WGP-NEXT: s_endpgm9617;9618; GFX12-CU-LABEL: local_workgroup_acq_rel_seq_cst_ret_cmpxchg:9619; GFX12-CU: ; %bb.0: ; %entry9620; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x09621; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x49622; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x89623; GFX12-CU-NEXT: s_wait_kmcnt 0x09624; GFX12-CU-NEXT: v_mov_b32_e32 v0, s09625; GFX12-CU-NEXT: v_mov_b32_e32 v1, s29626; GFX12-CU-NEXT: v_mov_b32_e32 v2, s19627; GFX12-CU-NEXT: s_wait_bvhcnt 0x09628; GFX12-CU-NEXT: s_wait_samplecnt 0x09629; GFX12-CU-NEXT: s_wait_storecnt 0x09630; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x09631; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:169632; GFX12-CU-NEXT: s_wait_dscnt 0x09633; GFX12-CU-NEXT: v_mov_b32_e32 v0, s09634; GFX12-CU-NEXT: ds_store_b32 v0, v19635; GFX12-CU-NEXT: s_endpgm9636;9637; GFX1250-LABEL: local_workgroup_acq_rel_seq_cst_ret_cmpxchg:9638; GFX1250: ; %bb.0: ; %entry9639; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 19640; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x09641; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x49642; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x89643; GFX1250-NEXT: s_wait_kmcnt 0x09644; GFX1250-NEXT: v_mov_b32_e32 v0, s09645; GFX1250-NEXT: v_mov_b32_e32 v1, s29646; GFX1250-NEXT: v_mov_b32_e32 v2, s19647; GFX1250-NEXT: s_wait_storecnt 0x09648; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x09649; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:169650; GFX1250-NEXT: s_wait_dscnt 0x09651; GFX1250-NEXT: v_mov_b32_e32 v0, s09652; GFX1250-NEXT: ds_store_b32 v0, v19653; GFX1250-NEXT: s_endpgm9654 ptr addrspace(3) %out, i32 %in, i32 %old) {9655entry:9656 %gep = getelementptr i32, ptr addrspace(3) %out, i32 49657 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel seq_cst9658 %val0 = extractvalue { i32, i1 } %val, 09659 store i32 %val0, ptr addrspace(3) %out, align 49660 ret void9661}9662 9663define amdgpu_kernel void @local_workgroup_seq_cst_seq_cst_ret_cmpxchg(9664; GFX6-LABEL: local_workgroup_seq_cst_seq_cst_ret_cmpxchg:9665; GFX6: ; %bb.0: ; %entry9666; GFX6-NEXT: s_load_dword s4, s[8:9], 0x09667; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr49668; GFX6-NEXT: s_load_dword s5, s[8:9], 0x19669; GFX6-NEXT: s_load_dword s6, s[8:9], 0x29670; GFX6-NEXT: s_mov_b32 m0, -19671; GFX6-NEXT: s_waitcnt lgkmcnt(0)9672; GFX6-NEXT: v_mov_b32_e32 v0, s49673; GFX6-NEXT: v_mov_b32_e32 v1, s69674; GFX6-NEXT: v_mov_b32_e32 v2, s59675; GFX6-NEXT: s_waitcnt lgkmcnt(0)9676; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169677; GFX6-NEXT: s_waitcnt lgkmcnt(0)9678; GFX6-NEXT: s_mov_b32 m0, -19679; GFX6-NEXT: v_mov_b32_e32 v0, s49680; GFX6-NEXT: ds_write_b32 v0, v19681; GFX6-NEXT: s_endpgm9682;9683; GFX7-LABEL: local_workgroup_seq_cst_seq_cst_ret_cmpxchg:9684; GFX7: ; %bb.0: ; %entry9685; GFX7-NEXT: s_load_dword s4, s[8:9], 0x09686; GFX7-NEXT: s_load_dword s5, s[8:9], 0x19687; GFX7-NEXT: s_load_dword s6, s[8:9], 0x29688; GFX7-NEXT: s_mov_b32 m0, -19689; GFX7-NEXT: s_waitcnt lgkmcnt(0)9690; GFX7-NEXT: v_mov_b32_e32 v0, s49691; GFX7-NEXT: v_mov_b32_e32 v1, s69692; GFX7-NEXT: v_mov_b32_e32 v2, s59693; GFX7-NEXT: s_waitcnt lgkmcnt(0)9694; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169695; GFX7-NEXT: s_waitcnt lgkmcnt(0)9696; GFX7-NEXT: s_mov_b32 m0, -19697; GFX7-NEXT: v_mov_b32_e32 v0, s49698; GFX7-NEXT: ds_write_b32 v0, v19699; GFX7-NEXT: s_endpgm9700;9701; GFX10-WGP-LABEL: local_workgroup_seq_cst_seq_cst_ret_cmpxchg:9702; GFX10-WGP: ; %bb.0: ; %entry9703; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x09704; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x49705; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x89706; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)9707; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s49708; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s69709; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s59710; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9711; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x09712; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169713; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)9714; GFX10-WGP-NEXT: buffer_gl0_inv9715; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s49716; GFX10-WGP-NEXT: ds_write_b32 v0, v19717; GFX10-WGP-NEXT: s_endpgm9718;9719; GFX10-CU-LABEL: local_workgroup_seq_cst_seq_cst_ret_cmpxchg:9720; GFX10-CU: ; %bb.0: ; %entry9721; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x09722; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x49723; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x89724; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)9725; GFX10-CU-NEXT: v_mov_b32_e32 v0, s49726; GFX10-CU-NEXT: v_mov_b32_e32 v1, s69727; GFX10-CU-NEXT: v_mov_b32_e32 v2, s59728; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9729; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x09730; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169731; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)9732; GFX10-CU-NEXT: v_mov_b32_e32 v0, s49733; GFX10-CU-NEXT: ds_write_b32 v0, v19734; GFX10-CU-NEXT: s_endpgm9735;9736; SKIP-CACHE-INV-LABEL: local_workgroup_seq_cst_seq_cst_ret_cmpxchg:9737; SKIP-CACHE-INV: ; %bb.0: ; %entry9738; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x09739; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x19740; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x29741; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -19742; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)9743; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s09744; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s29745; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s19746; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)9747; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169748; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)9749; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -19750; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s09751; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v19752; SKIP-CACHE-INV-NEXT: s_endpgm9753;9754; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_seq_cst_seq_cst_ret_cmpxchg:9755; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry9756; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x09757; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x49758; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x89759; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9760; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s49761; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s69762; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s59763; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9764; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169765; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9766; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s49767; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v19768; GFX90A-NOTTGSPLIT-NEXT: s_endpgm9769;9770; GFX90A-TGSPLIT-LABEL: local_workgroup_seq_cst_seq_cst_ret_cmpxchg:9771; GFX90A-TGSPLIT: ; %bb.0: ; %entry9772; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x09773; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x49774; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x89775; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9776; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s49777; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s69778; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s59779; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9780; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169781; GFX90A-TGSPLIT-NEXT: buffer_wbinvl1_vol9782; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s49783; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9784; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v19785; GFX90A-TGSPLIT-NEXT: s_endpgm9786;9787; GFX942-NOTTGSPLIT-LABEL: local_workgroup_seq_cst_seq_cst_ret_cmpxchg:9788; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry9789; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x09790; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x49791; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x89792; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9793; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s09794; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s29795; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s19796; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9797; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169798; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9799; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s09800; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v19801; GFX942-NOTTGSPLIT-NEXT: s_endpgm9802;9803; GFX942-TGSPLIT-LABEL: local_workgroup_seq_cst_seq_cst_ret_cmpxchg:9804; GFX942-TGSPLIT: ; %bb.0: ; %entry9805; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x09806; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x49807; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x89808; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9809; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s09810; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s29811; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s19812; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9813; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:169814; GFX942-TGSPLIT-NEXT: buffer_inv sc09815; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s09816; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9817; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v19818; GFX942-TGSPLIT-NEXT: s_endpgm9819;9820; GFX11-WGP-LABEL: local_workgroup_seq_cst_seq_cst_ret_cmpxchg:9821; GFX11-WGP: ; %bb.0: ; %entry9822; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x09823; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x49824; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x89825; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)9826; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s09827; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s29828; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s19829; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9830; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x09831; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:169832; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)9833; GFX11-WGP-NEXT: buffer_gl0_inv9834; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s09835; GFX11-WGP-NEXT: ds_store_b32 v0, v19836; GFX11-WGP-NEXT: s_endpgm9837;9838; GFX11-CU-LABEL: local_workgroup_seq_cst_seq_cst_ret_cmpxchg:9839; GFX11-CU: ; %bb.0: ; %entry9840; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x09841; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x49842; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x89843; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)9844; GFX11-CU-NEXT: v_mov_b32_e32 v0, s09845; GFX11-CU-NEXT: v_mov_b32_e32 v1, s29846; GFX11-CU-NEXT: v_mov_b32_e32 v2, s19847; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9848; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x09849; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:169850; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)9851; GFX11-CU-NEXT: v_mov_b32_e32 v0, s09852; GFX11-CU-NEXT: ds_store_b32 v0, v19853; GFX11-CU-NEXT: s_endpgm9854;9855; GFX12-WGP-LABEL: local_workgroup_seq_cst_seq_cst_ret_cmpxchg:9856; GFX12-WGP: ; %bb.0: ; %entry9857; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x09858; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x49859; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x89860; GFX12-WGP-NEXT: s_wait_kmcnt 0x09861; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s09862; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s29863; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s19864; GFX12-WGP-NEXT: s_wait_bvhcnt 0x09865; GFX12-WGP-NEXT: s_wait_samplecnt 0x09866; GFX12-WGP-NEXT: s_wait_storecnt 0x09867; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x09868; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:169869; GFX12-WGP-NEXT: s_wait_dscnt 0x09870; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE9871; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s09872; GFX12-WGP-NEXT: ds_store_b32 v0, v19873; GFX12-WGP-NEXT: s_endpgm9874;9875; GFX12-CU-LABEL: local_workgroup_seq_cst_seq_cst_ret_cmpxchg:9876; GFX12-CU: ; %bb.0: ; %entry9877; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x09878; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x49879; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x89880; GFX12-CU-NEXT: s_wait_kmcnt 0x09881; GFX12-CU-NEXT: v_mov_b32_e32 v0, s09882; GFX12-CU-NEXT: v_mov_b32_e32 v1, s29883; GFX12-CU-NEXT: v_mov_b32_e32 v2, s19884; GFX12-CU-NEXT: s_wait_bvhcnt 0x09885; GFX12-CU-NEXT: s_wait_samplecnt 0x09886; GFX12-CU-NEXT: s_wait_storecnt 0x09887; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x09888; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:169889; GFX12-CU-NEXT: s_wait_dscnt 0x09890; GFX12-CU-NEXT: v_mov_b32_e32 v0, s09891; GFX12-CU-NEXT: ds_store_b32 v0, v19892; GFX12-CU-NEXT: s_endpgm9893;9894; GFX1250-LABEL: local_workgroup_seq_cst_seq_cst_ret_cmpxchg:9895; GFX1250: ; %bb.0: ; %entry9896; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 19897; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x09898; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x49899; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x89900; GFX1250-NEXT: s_wait_kmcnt 0x09901; GFX1250-NEXT: v_mov_b32_e32 v0, s09902; GFX1250-NEXT: v_mov_b32_e32 v1, s29903; GFX1250-NEXT: v_mov_b32_e32 v2, s19904; GFX1250-NEXT: s_wait_storecnt 0x09905; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x09906; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:169907; GFX1250-NEXT: s_wait_dscnt 0x09908; GFX1250-NEXT: v_mov_b32_e32 v0, s09909; GFX1250-NEXT: ds_store_b32 v0, v19910; GFX1250-NEXT: s_endpgm9911 ptr addrspace(3) %out, i32 %in, i32 %old) {9912entry:9913 %gep = getelementptr i32, ptr addrspace(3) %out, i32 49914 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst9915 %val0 = extractvalue { i32, i1 } %val, 09916 store i32 %val0, ptr addrspace(3) %out, align 49917 ret void9918}9919 9920define amdgpu_kernel void @local_workgroup_one_as_unordered_load(9921; GFX6-LABEL: local_workgroup_one_as_unordered_load:9922; GFX6: ; %bb.0: ; %entry9923; GFX6-NEXT: s_load_dword s5, s[8:9], 0x09924; GFX6-NEXT: s_load_dword s4, s[8:9], 0x19925; GFX6-NEXT: ; kill: def $sgpr6 killed $sgpr49926; GFX6-NEXT: ; kill: def $sgpr6 killed $sgpr59927; GFX6-NEXT: s_mov_b32 m0, -19928; GFX6-NEXT: s_waitcnt lgkmcnt(0)9929; GFX6-NEXT: v_mov_b32_e32 v0, s59930; GFX6-NEXT: ds_read_b32 v1, v09931; GFX6-NEXT: s_mov_b32 m0, -19932; GFX6-NEXT: v_mov_b32_e32 v0, s49933; GFX6-NEXT: s_waitcnt lgkmcnt(0)9934; GFX6-NEXT: ds_write_b32 v0, v19935; GFX6-NEXT: s_endpgm9936;9937; GFX7-LABEL: local_workgroup_one_as_unordered_load:9938; GFX7: ; %bb.0: ; %entry9939; GFX7-NEXT: s_load_dword s5, s[8:9], 0x09940; GFX7-NEXT: s_load_dword s4, s[8:9], 0x19941; GFX7-NEXT: s_mov_b32 m0, -19942; GFX7-NEXT: s_waitcnt lgkmcnt(0)9943; GFX7-NEXT: v_mov_b32_e32 v0, s59944; GFX7-NEXT: ds_read_b32 v1, v09945; GFX7-NEXT: s_mov_b32 m0, -19946; GFX7-NEXT: v_mov_b32_e32 v0, s49947; GFX7-NEXT: s_waitcnt lgkmcnt(0)9948; GFX7-NEXT: ds_write_b32 v0, v19949; GFX7-NEXT: s_endpgm9950;9951; GFX10-WGP-LABEL: local_workgroup_one_as_unordered_load:9952; GFX10-WGP: ; %bb.0: ; %entry9953; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x09954; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x49955; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)9956; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s59957; GFX10-WGP-NEXT: ds_read_b32 v1, v09958; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s49959; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)9960; GFX10-WGP-NEXT: ds_write_b32 v0, v19961; GFX10-WGP-NEXT: s_endpgm9962;9963; GFX10-CU-LABEL: local_workgroup_one_as_unordered_load:9964; GFX10-CU: ; %bb.0: ; %entry9965; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x09966; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x49967; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)9968; GFX10-CU-NEXT: v_mov_b32_e32 v0, s59969; GFX10-CU-NEXT: ds_read_b32 v1, v09970; GFX10-CU-NEXT: v_mov_b32_e32 v0, s49971; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)9972; GFX10-CU-NEXT: ds_write_b32 v0, v19973; GFX10-CU-NEXT: s_endpgm9974;9975; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_unordered_load:9976; SKIP-CACHE-INV: ; %bb.0: ; %entry9977; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x09978; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x19979; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -19980; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)9981; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s19982; SKIP-CACHE-INV-NEXT: ds_read_b32 v1, v09983; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -19984; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s09985; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)9986; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v19987; SKIP-CACHE-INV-NEXT: s_endpgm9988;9989; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_unordered_load:9990; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry9991; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x09992; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x49993; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9994; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s59995; GFX90A-NOTTGSPLIT-NEXT: ds_read_b32 v1, v09996; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s49997; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)9998; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v19999; GFX90A-NOTTGSPLIT-NEXT: s_endpgm10000;10001; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_unordered_load:10002; GFX90A-TGSPLIT: ; %bb.0: ; %entry10003; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x010004; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x410005; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10006; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s510007; GFX90A-TGSPLIT-NEXT: ds_read_b32 v1, v010008; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s410009; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10010; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v110011; GFX90A-TGSPLIT-NEXT: s_endpgm10012;10013; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_unordered_load:10014; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry10015; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x010016; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x410017; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10018; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s110019; GFX942-NOTTGSPLIT-NEXT: ds_read_b32 v1, v010020; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s010021; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10022; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v110023; GFX942-NOTTGSPLIT-NEXT: s_endpgm10024;10025; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_unordered_load:10026; GFX942-TGSPLIT: ; %bb.0: ; %entry10027; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x010028; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x410029; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10030; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s110031; GFX942-TGSPLIT-NEXT: ds_read_b32 v1, v010032; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s010033; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10034; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v110035; GFX942-TGSPLIT-NEXT: s_endpgm10036;10037; GFX11-WGP-LABEL: local_workgroup_one_as_unordered_load:10038; GFX11-WGP: ; %bb.0: ; %entry10039; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x010040; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x410041; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)10042; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s110043; GFX11-WGP-NEXT: ds_load_b32 v1, v010044; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s010045; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)10046; GFX11-WGP-NEXT: ds_store_b32 v0, v110047; GFX11-WGP-NEXT: s_endpgm10048;10049; GFX11-CU-LABEL: local_workgroup_one_as_unordered_load:10050; GFX11-CU: ; %bb.0: ; %entry10051; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x010052; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x410053; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)10054; GFX11-CU-NEXT: v_mov_b32_e32 v0, s110055; GFX11-CU-NEXT: ds_load_b32 v1, v010056; GFX11-CU-NEXT: v_mov_b32_e32 v0, s010057; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)10058; GFX11-CU-NEXT: ds_store_b32 v0, v110059; GFX11-CU-NEXT: s_endpgm10060;10061; GFX12-WGP-LABEL: local_workgroup_one_as_unordered_load:10062; GFX12-WGP: ; %bb.0: ; %entry10063; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x010064; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x410065; GFX12-WGP-NEXT: s_wait_kmcnt 0x010066; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s110067; GFX12-WGP-NEXT: ds_load_b32 v1, v010068; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s010069; GFX12-WGP-NEXT: s_wait_dscnt 0x010070; GFX12-WGP-NEXT: ds_store_b32 v0, v110071; GFX12-WGP-NEXT: s_endpgm10072;10073; GFX12-CU-LABEL: local_workgroup_one_as_unordered_load:10074; GFX12-CU: ; %bb.0: ; %entry10075; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x010076; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x410077; GFX12-CU-NEXT: s_wait_kmcnt 0x010078; GFX12-CU-NEXT: v_mov_b32_e32 v0, s110079; GFX12-CU-NEXT: ds_load_b32 v1, v010080; GFX12-CU-NEXT: v_mov_b32_e32 v0, s010081; GFX12-CU-NEXT: s_wait_dscnt 0x010082; GFX12-CU-NEXT: ds_store_b32 v0, v110083; GFX12-CU-NEXT: s_endpgm10084;10085; GFX1250-LABEL: local_workgroup_one_as_unordered_load:10086; GFX1250: ; %bb.0: ; %entry10087; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 110088; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x010089; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x410090; GFX1250-NEXT: s_wait_kmcnt 0x010091; GFX1250-NEXT: v_mov_b32_e32 v0, s110092; GFX1250-NEXT: ds_load_b32 v1, v010093; GFX1250-NEXT: v_mov_b32_e32 v0, s010094; GFX1250-NEXT: s_wait_dscnt 0x010095; GFX1250-NEXT: ds_store_b32 v0, v110096; GFX1250-NEXT: s_endpgm10097 ptr addrspace(3) %in, ptr addrspace(3) %out) {10098entry:10099 %val = load atomic i32, ptr addrspace(3) %in syncscope("workgroup-one-as") unordered, align 410100 store i32 %val, ptr addrspace(3) %out10101 ret void10102}10103 10104define amdgpu_kernel void @local_workgroup_one_as_monotonic_load(10105; GFX6-LABEL: local_workgroup_one_as_monotonic_load:10106; GFX6: ; %bb.0: ; %entry10107; GFX6-NEXT: s_load_dword s5, s[8:9], 0x010108; GFX6-NEXT: s_load_dword s4, s[8:9], 0x110109; GFX6-NEXT: ; kill: def $sgpr6 killed $sgpr410110; GFX6-NEXT: ; kill: def $sgpr6 killed $sgpr510111; GFX6-NEXT: s_mov_b32 m0, -110112; GFX6-NEXT: s_waitcnt lgkmcnt(0)10113; GFX6-NEXT: v_mov_b32_e32 v0, s510114; GFX6-NEXT: ds_read_b32 v1, v010115; GFX6-NEXT: s_mov_b32 m0, -110116; GFX6-NEXT: v_mov_b32_e32 v0, s410117; GFX6-NEXT: s_waitcnt lgkmcnt(0)10118; GFX6-NEXT: ds_write_b32 v0, v110119; GFX6-NEXT: s_endpgm10120;10121; GFX7-LABEL: local_workgroup_one_as_monotonic_load:10122; GFX7: ; %bb.0: ; %entry10123; GFX7-NEXT: s_load_dword s5, s[8:9], 0x010124; GFX7-NEXT: s_load_dword s4, s[8:9], 0x110125; GFX7-NEXT: s_mov_b32 m0, -110126; GFX7-NEXT: s_waitcnt lgkmcnt(0)10127; GFX7-NEXT: v_mov_b32_e32 v0, s510128; GFX7-NEXT: ds_read_b32 v1, v010129; GFX7-NEXT: s_mov_b32 m0, -110130; GFX7-NEXT: v_mov_b32_e32 v0, s410131; GFX7-NEXT: s_waitcnt lgkmcnt(0)10132; GFX7-NEXT: ds_write_b32 v0, v110133; GFX7-NEXT: s_endpgm10134;10135; GFX10-WGP-LABEL: local_workgroup_one_as_monotonic_load:10136; GFX10-WGP: ; %bb.0: ; %entry10137; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x010138; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x410139; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)10140; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s510141; GFX10-WGP-NEXT: ds_read_b32 v1, v010142; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s410143; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)10144; GFX10-WGP-NEXT: ds_write_b32 v0, v110145; GFX10-WGP-NEXT: s_endpgm10146;10147; GFX10-CU-LABEL: local_workgroup_one_as_monotonic_load:10148; GFX10-CU: ; %bb.0: ; %entry10149; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x010150; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x410151; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)10152; GFX10-CU-NEXT: v_mov_b32_e32 v0, s510153; GFX10-CU-NEXT: ds_read_b32 v1, v010154; GFX10-CU-NEXT: v_mov_b32_e32 v0, s410155; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)10156; GFX10-CU-NEXT: ds_write_b32 v0, v110157; GFX10-CU-NEXT: s_endpgm10158;10159; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_monotonic_load:10160; SKIP-CACHE-INV: ; %bb.0: ; %entry10161; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x010162; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x110163; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -110164; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)10165; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s110166; SKIP-CACHE-INV-NEXT: ds_read_b32 v1, v010167; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -110168; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s010169; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)10170; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v110171; SKIP-CACHE-INV-NEXT: s_endpgm10172;10173; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_monotonic_load:10174; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry10175; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x010176; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x410177; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10178; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s510179; GFX90A-NOTTGSPLIT-NEXT: ds_read_b32 v1, v010180; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s410181; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10182; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v110183; GFX90A-NOTTGSPLIT-NEXT: s_endpgm10184;10185; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_monotonic_load:10186; GFX90A-TGSPLIT: ; %bb.0: ; %entry10187; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x010188; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x410189; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10190; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s510191; GFX90A-TGSPLIT-NEXT: ds_read_b32 v1, v010192; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s410193; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10194; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v110195; GFX90A-TGSPLIT-NEXT: s_endpgm10196;10197; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_monotonic_load:10198; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry10199; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x010200; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x410201; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10202; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s110203; GFX942-NOTTGSPLIT-NEXT: ds_read_b32 v1, v010204; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s010205; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10206; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v110207; GFX942-NOTTGSPLIT-NEXT: s_endpgm10208;10209; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_monotonic_load:10210; GFX942-TGSPLIT: ; %bb.0: ; %entry10211; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x010212; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x410213; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10214; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s110215; GFX942-TGSPLIT-NEXT: ds_read_b32 v1, v010216; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s010217; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10218; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v110219; GFX942-TGSPLIT-NEXT: s_endpgm10220;10221; GFX11-WGP-LABEL: local_workgroup_one_as_monotonic_load:10222; GFX11-WGP: ; %bb.0: ; %entry10223; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x010224; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x410225; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)10226; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s110227; GFX11-WGP-NEXT: ds_load_b32 v1, v010228; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s010229; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)10230; GFX11-WGP-NEXT: ds_store_b32 v0, v110231; GFX11-WGP-NEXT: s_endpgm10232;10233; GFX11-CU-LABEL: local_workgroup_one_as_monotonic_load:10234; GFX11-CU: ; %bb.0: ; %entry10235; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x010236; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x410237; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)10238; GFX11-CU-NEXT: v_mov_b32_e32 v0, s110239; GFX11-CU-NEXT: ds_load_b32 v1, v010240; GFX11-CU-NEXT: v_mov_b32_e32 v0, s010241; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)10242; GFX11-CU-NEXT: ds_store_b32 v0, v110243; GFX11-CU-NEXT: s_endpgm10244;10245; GFX12-WGP-LABEL: local_workgroup_one_as_monotonic_load:10246; GFX12-WGP: ; %bb.0: ; %entry10247; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x010248; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x410249; GFX12-WGP-NEXT: s_wait_kmcnt 0x010250; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s110251; GFX12-WGP-NEXT: ds_load_b32 v1, v010252; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s010253; GFX12-WGP-NEXT: s_wait_dscnt 0x010254; GFX12-WGP-NEXT: ds_store_b32 v0, v110255; GFX12-WGP-NEXT: s_endpgm10256;10257; GFX12-CU-LABEL: local_workgroup_one_as_monotonic_load:10258; GFX12-CU: ; %bb.0: ; %entry10259; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x010260; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x410261; GFX12-CU-NEXT: s_wait_kmcnt 0x010262; GFX12-CU-NEXT: v_mov_b32_e32 v0, s110263; GFX12-CU-NEXT: ds_load_b32 v1, v010264; GFX12-CU-NEXT: v_mov_b32_e32 v0, s010265; GFX12-CU-NEXT: s_wait_dscnt 0x010266; GFX12-CU-NEXT: ds_store_b32 v0, v110267; GFX12-CU-NEXT: s_endpgm10268;10269; GFX1250-LABEL: local_workgroup_one_as_monotonic_load:10270; GFX1250: ; %bb.0: ; %entry10271; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 110272; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x010273; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x410274; GFX1250-NEXT: s_wait_kmcnt 0x010275; GFX1250-NEXT: v_mov_b32_e32 v0, s110276; GFX1250-NEXT: ds_load_b32 v1, v010277; GFX1250-NEXT: v_mov_b32_e32 v0, s010278; GFX1250-NEXT: s_wait_dscnt 0x010279; GFX1250-NEXT: ds_store_b32 v0, v110280; GFX1250-NEXT: s_endpgm10281 ptr addrspace(3) %in, ptr addrspace(3) %out) {10282entry:10283 %val = load atomic i32, ptr addrspace(3) %in syncscope("workgroup-one-as") monotonic, align 410284 store i32 %val, ptr addrspace(3) %out10285 ret void10286}10287 10288define amdgpu_kernel void @local_workgroup_one_as_acquire_load(10289; GFX6-LABEL: local_workgroup_one_as_acquire_load:10290; GFX6: ; %bb.0: ; %entry10291; GFX6-NEXT: s_load_dword s5, s[8:9], 0x010292; GFX6-NEXT: s_load_dword s4, s[8:9], 0x110293; GFX6-NEXT: ; kill: def $sgpr6 killed $sgpr410294; GFX6-NEXT: ; kill: def $sgpr6 killed $sgpr510295; GFX6-NEXT: s_mov_b32 m0, -110296; GFX6-NEXT: s_waitcnt lgkmcnt(0)10297; GFX6-NEXT: v_mov_b32_e32 v0, s510298; GFX6-NEXT: ds_read_b32 v1, v010299; GFX6-NEXT: s_mov_b32 m0, -110300; GFX6-NEXT: v_mov_b32_e32 v0, s410301; GFX6-NEXT: s_waitcnt lgkmcnt(0)10302; GFX6-NEXT: ds_write_b32 v0, v110303; GFX6-NEXT: s_endpgm10304;10305; GFX7-LABEL: local_workgroup_one_as_acquire_load:10306; GFX7: ; %bb.0: ; %entry10307; GFX7-NEXT: s_load_dword s5, s[8:9], 0x010308; GFX7-NEXT: s_load_dword s4, s[8:9], 0x110309; GFX7-NEXT: s_mov_b32 m0, -110310; GFX7-NEXT: s_waitcnt lgkmcnt(0)10311; GFX7-NEXT: v_mov_b32_e32 v0, s510312; GFX7-NEXT: ds_read_b32 v1, v010313; GFX7-NEXT: s_mov_b32 m0, -110314; GFX7-NEXT: v_mov_b32_e32 v0, s410315; GFX7-NEXT: s_waitcnt lgkmcnt(0)10316; GFX7-NEXT: ds_write_b32 v0, v110317; GFX7-NEXT: s_endpgm10318;10319; GFX10-WGP-LABEL: local_workgroup_one_as_acquire_load:10320; GFX10-WGP: ; %bb.0: ; %entry10321; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x010322; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x410323; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)10324; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s510325; GFX10-WGP-NEXT: ds_read_b32 v1, v010326; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s410327; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)10328; GFX10-WGP-NEXT: ds_write_b32 v0, v110329; GFX10-WGP-NEXT: s_endpgm10330;10331; GFX10-CU-LABEL: local_workgroup_one_as_acquire_load:10332; GFX10-CU: ; %bb.0: ; %entry10333; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x010334; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x410335; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)10336; GFX10-CU-NEXT: v_mov_b32_e32 v0, s510337; GFX10-CU-NEXT: ds_read_b32 v1, v010338; GFX10-CU-NEXT: v_mov_b32_e32 v0, s410339; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)10340; GFX10-CU-NEXT: ds_write_b32 v0, v110341; GFX10-CU-NEXT: s_endpgm10342;10343; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acquire_load:10344; SKIP-CACHE-INV: ; %bb.0: ; %entry10345; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x010346; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x110347; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -110348; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)10349; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s110350; SKIP-CACHE-INV-NEXT: ds_read_b32 v1, v010351; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -110352; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s010353; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)10354; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v110355; SKIP-CACHE-INV-NEXT: s_endpgm10356;10357; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_acquire_load:10358; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry10359; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x010360; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x410361; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10362; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s510363; GFX90A-NOTTGSPLIT-NEXT: ds_read_b32 v1, v010364; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s410365; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10366; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v110367; GFX90A-NOTTGSPLIT-NEXT: s_endpgm10368;10369; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_acquire_load:10370; GFX90A-TGSPLIT: ; %bb.0: ; %entry10371; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x010372; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x410373; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10374; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s510375; GFX90A-TGSPLIT-NEXT: ds_read_b32 v1, v010376; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s410377; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10378; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v110379; GFX90A-TGSPLIT-NEXT: s_endpgm10380;10381; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_acquire_load:10382; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry10383; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x010384; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x410385; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10386; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s110387; GFX942-NOTTGSPLIT-NEXT: ds_read_b32 v1, v010388; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s010389; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10390; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v110391; GFX942-NOTTGSPLIT-NEXT: s_endpgm10392;10393; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_acquire_load:10394; GFX942-TGSPLIT: ; %bb.0: ; %entry10395; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x010396; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x410397; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10398; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s110399; GFX942-TGSPLIT-NEXT: ds_read_b32 v1, v010400; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s010401; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10402; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v110403; GFX942-TGSPLIT-NEXT: s_endpgm10404;10405; GFX11-WGP-LABEL: local_workgroup_one_as_acquire_load:10406; GFX11-WGP: ; %bb.0: ; %entry10407; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x010408; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x410409; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)10410; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s110411; GFX11-WGP-NEXT: ds_load_b32 v1, v010412; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s010413; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)10414; GFX11-WGP-NEXT: ds_store_b32 v0, v110415; GFX11-WGP-NEXT: s_endpgm10416;10417; GFX11-CU-LABEL: local_workgroup_one_as_acquire_load:10418; GFX11-CU: ; %bb.0: ; %entry10419; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x010420; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x410421; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)10422; GFX11-CU-NEXT: v_mov_b32_e32 v0, s110423; GFX11-CU-NEXT: ds_load_b32 v1, v010424; GFX11-CU-NEXT: v_mov_b32_e32 v0, s010425; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)10426; GFX11-CU-NEXT: ds_store_b32 v0, v110427; GFX11-CU-NEXT: s_endpgm10428;10429; GFX12-WGP-LABEL: local_workgroup_one_as_acquire_load:10430; GFX12-WGP: ; %bb.0: ; %entry10431; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x010432; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x410433; GFX12-WGP-NEXT: s_wait_kmcnt 0x010434; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s110435; GFX12-WGP-NEXT: ds_load_b32 v1, v010436; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s010437; GFX12-WGP-NEXT: s_wait_dscnt 0x010438; GFX12-WGP-NEXT: ds_store_b32 v0, v110439; GFX12-WGP-NEXT: s_endpgm10440;10441; GFX12-CU-LABEL: local_workgroup_one_as_acquire_load:10442; GFX12-CU: ; %bb.0: ; %entry10443; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x010444; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x410445; GFX12-CU-NEXT: s_wait_kmcnt 0x010446; GFX12-CU-NEXT: v_mov_b32_e32 v0, s110447; GFX12-CU-NEXT: ds_load_b32 v1, v010448; GFX12-CU-NEXT: v_mov_b32_e32 v0, s010449; GFX12-CU-NEXT: s_wait_dscnt 0x010450; GFX12-CU-NEXT: ds_store_b32 v0, v110451; GFX12-CU-NEXT: s_endpgm10452;10453; GFX1250-LABEL: local_workgroup_one_as_acquire_load:10454; GFX1250: ; %bb.0: ; %entry10455; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 110456; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x010457; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x410458; GFX1250-NEXT: s_wait_kmcnt 0x010459; GFX1250-NEXT: v_mov_b32_e32 v0, s110460; GFX1250-NEXT: ds_load_b32 v1, v010461; GFX1250-NEXT: v_mov_b32_e32 v0, s010462; GFX1250-NEXT: s_wait_dscnt 0x010463; GFX1250-NEXT: ds_store_b32 v0, v110464; GFX1250-NEXT: s_endpgm10465 ptr addrspace(3) %in, ptr addrspace(3) %out) {10466entry:10467 %val = load atomic i32, ptr addrspace(3) %in syncscope("workgroup-one-as") acquire, align 410468 store i32 %val, ptr addrspace(3) %out10469 ret void10470}10471 10472define amdgpu_kernel void @local_workgroup_one_as_seq_cst_load(10473; GFX6-LABEL: local_workgroup_one_as_seq_cst_load:10474; GFX6: ; %bb.0: ; %entry10475; GFX6-NEXT: s_load_dword s5, s[8:9], 0x010476; GFX6-NEXT: s_load_dword s4, s[8:9], 0x110477; GFX6-NEXT: ; kill: def $sgpr6 killed $sgpr410478; GFX6-NEXT: ; kill: def $sgpr6 killed $sgpr510479; GFX6-NEXT: s_mov_b32 m0, -110480; GFX6-NEXT: s_waitcnt lgkmcnt(0)10481; GFX6-NEXT: v_mov_b32_e32 v0, s510482; GFX6-NEXT: ds_read_b32 v1, v010483; GFX6-NEXT: s_mov_b32 m0, -110484; GFX6-NEXT: v_mov_b32_e32 v0, s410485; GFX6-NEXT: s_waitcnt lgkmcnt(0)10486; GFX6-NEXT: ds_write_b32 v0, v110487; GFX6-NEXT: s_endpgm10488;10489; GFX7-LABEL: local_workgroup_one_as_seq_cst_load:10490; GFX7: ; %bb.0: ; %entry10491; GFX7-NEXT: s_load_dword s5, s[8:9], 0x010492; GFX7-NEXT: s_load_dword s4, s[8:9], 0x110493; GFX7-NEXT: s_mov_b32 m0, -110494; GFX7-NEXT: s_waitcnt lgkmcnt(0)10495; GFX7-NEXT: v_mov_b32_e32 v0, s510496; GFX7-NEXT: ds_read_b32 v1, v010497; GFX7-NEXT: s_mov_b32 m0, -110498; GFX7-NEXT: v_mov_b32_e32 v0, s410499; GFX7-NEXT: s_waitcnt lgkmcnt(0)10500; GFX7-NEXT: ds_write_b32 v0, v110501; GFX7-NEXT: s_endpgm10502;10503; GFX10-WGP-LABEL: local_workgroup_one_as_seq_cst_load:10504; GFX10-WGP: ; %bb.0: ; %entry10505; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x010506; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x410507; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)10508; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s510509; GFX10-WGP-NEXT: ds_read_b32 v1, v010510; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s410511; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)10512; GFX10-WGP-NEXT: ds_write_b32 v0, v110513; GFX10-WGP-NEXT: s_endpgm10514;10515; GFX10-CU-LABEL: local_workgroup_one_as_seq_cst_load:10516; GFX10-CU: ; %bb.0: ; %entry10517; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x010518; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x410519; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)10520; GFX10-CU-NEXT: v_mov_b32_e32 v0, s510521; GFX10-CU-NEXT: ds_read_b32 v1, v010522; GFX10-CU-NEXT: v_mov_b32_e32 v0, s410523; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)10524; GFX10-CU-NEXT: ds_write_b32 v0, v110525; GFX10-CU-NEXT: s_endpgm10526;10527; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_seq_cst_load:10528; SKIP-CACHE-INV: ; %bb.0: ; %entry10529; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x010530; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x110531; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -110532; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)10533; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s110534; SKIP-CACHE-INV-NEXT: ds_read_b32 v1, v010535; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -110536; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s010537; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)10538; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v110539; SKIP-CACHE-INV-NEXT: s_endpgm10540;10541; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_seq_cst_load:10542; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry10543; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x010544; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x410545; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10546; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s510547; GFX90A-NOTTGSPLIT-NEXT: ds_read_b32 v1, v010548; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s410549; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10550; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v110551; GFX90A-NOTTGSPLIT-NEXT: s_endpgm10552;10553; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_seq_cst_load:10554; GFX90A-TGSPLIT: ; %bb.0: ; %entry10555; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x010556; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x410557; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10558; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s510559; GFX90A-TGSPLIT-NEXT: ds_read_b32 v1, v010560; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s410561; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10562; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v110563; GFX90A-TGSPLIT-NEXT: s_endpgm10564;10565; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_seq_cst_load:10566; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry10567; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x010568; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x410569; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10570; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s110571; GFX942-NOTTGSPLIT-NEXT: ds_read_b32 v1, v010572; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s010573; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10574; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v110575; GFX942-NOTTGSPLIT-NEXT: s_endpgm10576;10577; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_seq_cst_load:10578; GFX942-TGSPLIT: ; %bb.0: ; %entry10579; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x010580; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x410581; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10582; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s110583; GFX942-TGSPLIT-NEXT: ds_read_b32 v1, v010584; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s010585; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10586; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v110587; GFX942-TGSPLIT-NEXT: s_endpgm10588;10589; GFX11-WGP-LABEL: local_workgroup_one_as_seq_cst_load:10590; GFX11-WGP: ; %bb.0: ; %entry10591; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x010592; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x410593; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)10594; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s110595; GFX11-WGP-NEXT: ds_load_b32 v1, v010596; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s010597; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)10598; GFX11-WGP-NEXT: ds_store_b32 v0, v110599; GFX11-WGP-NEXT: s_endpgm10600;10601; GFX11-CU-LABEL: local_workgroup_one_as_seq_cst_load:10602; GFX11-CU: ; %bb.0: ; %entry10603; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x010604; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x410605; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)10606; GFX11-CU-NEXT: v_mov_b32_e32 v0, s110607; GFX11-CU-NEXT: ds_load_b32 v1, v010608; GFX11-CU-NEXT: v_mov_b32_e32 v0, s010609; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)10610; GFX11-CU-NEXT: ds_store_b32 v0, v110611; GFX11-CU-NEXT: s_endpgm10612;10613; GFX12-WGP-LABEL: local_workgroup_one_as_seq_cst_load:10614; GFX12-WGP: ; %bb.0: ; %entry10615; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x010616; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x410617; GFX12-WGP-NEXT: s_wait_kmcnt 0x010618; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s110619; GFX12-WGP-NEXT: ds_load_b32 v1, v010620; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s010621; GFX12-WGP-NEXT: s_wait_dscnt 0x010622; GFX12-WGP-NEXT: ds_store_b32 v0, v110623; GFX12-WGP-NEXT: s_endpgm10624;10625; GFX12-CU-LABEL: local_workgroup_one_as_seq_cst_load:10626; GFX12-CU: ; %bb.0: ; %entry10627; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x010628; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x410629; GFX12-CU-NEXT: s_wait_kmcnt 0x010630; GFX12-CU-NEXT: v_mov_b32_e32 v0, s110631; GFX12-CU-NEXT: ds_load_b32 v1, v010632; GFX12-CU-NEXT: v_mov_b32_e32 v0, s010633; GFX12-CU-NEXT: s_wait_dscnt 0x010634; GFX12-CU-NEXT: ds_store_b32 v0, v110635; GFX12-CU-NEXT: s_endpgm10636;10637; GFX1250-LABEL: local_workgroup_one_as_seq_cst_load:10638; GFX1250: ; %bb.0: ; %entry10639; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 110640; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x010641; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x410642; GFX1250-NEXT: s_wait_kmcnt 0x010643; GFX1250-NEXT: v_mov_b32_e32 v0, s110644; GFX1250-NEXT: ds_load_b32 v1, v010645; GFX1250-NEXT: v_mov_b32_e32 v0, s010646; GFX1250-NEXT: s_wait_dscnt 0x010647; GFX1250-NEXT: ds_store_b32 v0, v110648; GFX1250-NEXT: s_endpgm10649 ptr addrspace(3) %in, ptr addrspace(3) %out) {10650entry:10651 %val = load atomic i32, ptr addrspace(3) %in syncscope("workgroup-one-as") seq_cst, align 410652 store i32 %val, ptr addrspace(3) %out10653 ret void10654}10655 10656define amdgpu_kernel void @local_workgroup_one_as_unordered_store(10657; GFX6-LABEL: local_workgroup_one_as_unordered_store:10658; GFX6: ; %bb.0: ; %entry10659; GFX6-NEXT: s_load_dword s5, s[8:9], 0x110660; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr510661; GFX6-NEXT: s_load_dword s4, s[8:9], 0x010662; GFX6-NEXT: s_mov_b32 m0, -110663; GFX6-NEXT: s_waitcnt lgkmcnt(0)10664; GFX6-NEXT: v_mov_b32_e32 v0, s510665; GFX6-NEXT: v_mov_b32_e32 v1, s410666; GFX6-NEXT: ds_write_b32 v0, v110667; GFX6-NEXT: s_endpgm10668;10669; GFX7-LABEL: local_workgroup_one_as_unordered_store:10670; GFX7: ; %bb.0: ; %entry10671; GFX7-NEXT: s_load_dword s4, s[8:9], 0x010672; GFX7-NEXT: s_load_dword s5, s[8:9], 0x110673; GFX7-NEXT: s_mov_b32 m0, -110674; GFX7-NEXT: s_waitcnt lgkmcnt(0)10675; GFX7-NEXT: v_mov_b32_e32 v0, s510676; GFX7-NEXT: v_mov_b32_e32 v1, s410677; GFX7-NEXT: ds_write_b32 v0, v110678; GFX7-NEXT: s_endpgm10679;10680; GFX10-WGP-LABEL: local_workgroup_one_as_unordered_store:10681; GFX10-WGP: ; %bb.0: ; %entry10682; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x010683; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x410684; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)10685; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s510686; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s410687; GFX10-WGP-NEXT: ds_write_b32 v0, v110688; GFX10-WGP-NEXT: s_endpgm10689;10690; GFX10-CU-LABEL: local_workgroup_one_as_unordered_store:10691; GFX10-CU: ; %bb.0: ; %entry10692; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x010693; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x410694; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)10695; GFX10-CU-NEXT: v_mov_b32_e32 v0, s510696; GFX10-CU-NEXT: v_mov_b32_e32 v1, s410697; GFX10-CU-NEXT: ds_write_b32 v0, v110698; GFX10-CU-NEXT: s_endpgm10699;10700; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_unordered_store:10701; SKIP-CACHE-INV: ; %bb.0: ; %entry10702; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x010703; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x110704; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -110705; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)10706; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s110707; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s010708; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v110709; SKIP-CACHE-INV-NEXT: s_endpgm10710;10711; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_unordered_store:10712; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry10713; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x010714; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x410715; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10716; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s510717; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s410718; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v110719; GFX90A-NOTTGSPLIT-NEXT: s_endpgm10720;10721; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_unordered_store:10722; GFX90A-TGSPLIT: ; %bb.0: ; %entry10723; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x010724; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x410725; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10726; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s510727; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s410728; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v110729; GFX90A-TGSPLIT-NEXT: s_endpgm10730;10731; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_unordered_store:10732; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry10733; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x010734; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x410735; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10736; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s110737; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s010738; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v110739; GFX942-NOTTGSPLIT-NEXT: s_endpgm10740;10741; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_unordered_store:10742; GFX942-TGSPLIT: ; %bb.0: ; %entry10743; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x010744; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x410745; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10746; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s110747; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s010748; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v110749; GFX942-TGSPLIT-NEXT: s_endpgm10750;10751; GFX11-WGP-LABEL: local_workgroup_one_as_unordered_store:10752; GFX11-WGP: ; %bb.0: ; %entry10753; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x010754; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x410755; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)10756; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s110757; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s010758; GFX11-WGP-NEXT: ds_store_b32 v0, v110759; GFX11-WGP-NEXT: s_endpgm10760;10761; GFX11-CU-LABEL: local_workgroup_one_as_unordered_store:10762; GFX11-CU: ; %bb.0: ; %entry10763; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x010764; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x410765; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)10766; GFX11-CU-NEXT: v_mov_b32_e32 v0, s110767; GFX11-CU-NEXT: v_mov_b32_e32 v1, s010768; GFX11-CU-NEXT: ds_store_b32 v0, v110769; GFX11-CU-NEXT: s_endpgm10770;10771; GFX12-WGP-LABEL: local_workgroup_one_as_unordered_store:10772; GFX12-WGP: ; %bb.0: ; %entry10773; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x010774; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x410775; GFX12-WGP-NEXT: s_wait_kmcnt 0x010776; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s110777; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s010778; GFX12-WGP-NEXT: ds_store_b32 v0, v110779; GFX12-WGP-NEXT: s_endpgm10780;10781; GFX12-CU-LABEL: local_workgroup_one_as_unordered_store:10782; GFX12-CU: ; %bb.0: ; %entry10783; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x010784; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x410785; GFX12-CU-NEXT: s_wait_kmcnt 0x010786; GFX12-CU-NEXT: v_mov_b32_e32 v0, s110787; GFX12-CU-NEXT: v_mov_b32_e32 v1, s010788; GFX12-CU-NEXT: ds_store_b32 v0, v110789; GFX12-CU-NEXT: s_endpgm10790;10791; GFX1250-LABEL: local_workgroup_one_as_unordered_store:10792; GFX1250: ; %bb.0: ; %entry10793; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 110794; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x010795; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x410796; GFX1250-NEXT: s_wait_kmcnt 0x010797; GFX1250-NEXT: v_mov_b32_e32 v0, s110798; GFX1250-NEXT: v_mov_b32_e32 v1, s010799; GFX1250-NEXT: ds_store_b32 v0, v110800; GFX1250-NEXT: s_endpgm10801 i32 %in, ptr addrspace(3) %out) {10802entry:10803 store atomic i32 %in, ptr addrspace(3) %out syncscope("workgroup-one-as") unordered, align 410804 ret void10805}10806 10807define amdgpu_kernel void @local_workgroup_one_as_monotonic_store(10808; GFX6-LABEL: local_workgroup_one_as_monotonic_store:10809; GFX6: ; %bb.0: ; %entry10810; GFX6-NEXT: s_load_dword s5, s[8:9], 0x110811; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr510812; GFX6-NEXT: s_load_dword s4, s[8:9], 0x010813; GFX6-NEXT: s_mov_b32 m0, -110814; GFX6-NEXT: s_waitcnt lgkmcnt(0)10815; GFX6-NEXT: v_mov_b32_e32 v0, s510816; GFX6-NEXT: v_mov_b32_e32 v1, s410817; GFX6-NEXT: ds_write_b32 v0, v110818; GFX6-NEXT: s_endpgm10819;10820; GFX7-LABEL: local_workgroup_one_as_monotonic_store:10821; GFX7: ; %bb.0: ; %entry10822; GFX7-NEXT: s_load_dword s4, s[8:9], 0x010823; GFX7-NEXT: s_load_dword s5, s[8:9], 0x110824; GFX7-NEXT: s_mov_b32 m0, -110825; GFX7-NEXT: s_waitcnt lgkmcnt(0)10826; GFX7-NEXT: v_mov_b32_e32 v0, s510827; GFX7-NEXT: v_mov_b32_e32 v1, s410828; GFX7-NEXT: ds_write_b32 v0, v110829; GFX7-NEXT: s_endpgm10830;10831; GFX10-WGP-LABEL: local_workgroup_one_as_monotonic_store:10832; GFX10-WGP: ; %bb.0: ; %entry10833; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x010834; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x410835; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)10836; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s510837; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s410838; GFX10-WGP-NEXT: ds_write_b32 v0, v110839; GFX10-WGP-NEXT: s_endpgm10840;10841; GFX10-CU-LABEL: local_workgroup_one_as_monotonic_store:10842; GFX10-CU: ; %bb.0: ; %entry10843; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x010844; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x410845; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)10846; GFX10-CU-NEXT: v_mov_b32_e32 v0, s510847; GFX10-CU-NEXT: v_mov_b32_e32 v1, s410848; GFX10-CU-NEXT: ds_write_b32 v0, v110849; GFX10-CU-NEXT: s_endpgm10850;10851; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_monotonic_store:10852; SKIP-CACHE-INV: ; %bb.0: ; %entry10853; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x010854; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x110855; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -110856; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)10857; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s110858; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s010859; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v110860; SKIP-CACHE-INV-NEXT: s_endpgm10861;10862; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_monotonic_store:10863; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry10864; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x010865; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x410866; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10867; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s510868; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s410869; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v110870; GFX90A-NOTTGSPLIT-NEXT: s_endpgm10871;10872; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_monotonic_store:10873; GFX90A-TGSPLIT: ; %bb.0: ; %entry10874; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x010875; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x410876; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10877; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s510878; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s410879; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v110880; GFX90A-TGSPLIT-NEXT: s_endpgm10881;10882; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_monotonic_store:10883; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry10884; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x010885; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x410886; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10887; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s110888; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s010889; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v110890; GFX942-NOTTGSPLIT-NEXT: s_endpgm10891;10892; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_monotonic_store:10893; GFX942-TGSPLIT: ; %bb.0: ; %entry10894; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x010895; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x410896; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)10897; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s110898; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s010899; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v110900; GFX942-TGSPLIT-NEXT: s_endpgm10901;10902; GFX11-WGP-LABEL: local_workgroup_one_as_monotonic_store:10903; GFX11-WGP: ; %bb.0: ; %entry10904; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x010905; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x410906; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)10907; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s110908; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s010909; GFX11-WGP-NEXT: ds_store_b32 v0, v110910; GFX11-WGP-NEXT: s_endpgm10911;10912; GFX11-CU-LABEL: local_workgroup_one_as_monotonic_store:10913; GFX11-CU: ; %bb.0: ; %entry10914; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x010915; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x410916; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)10917; GFX11-CU-NEXT: v_mov_b32_e32 v0, s110918; GFX11-CU-NEXT: v_mov_b32_e32 v1, s010919; GFX11-CU-NEXT: ds_store_b32 v0, v110920; GFX11-CU-NEXT: s_endpgm10921;10922; GFX12-WGP-LABEL: local_workgroup_one_as_monotonic_store:10923; GFX12-WGP: ; %bb.0: ; %entry10924; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x010925; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x410926; GFX12-WGP-NEXT: s_wait_kmcnt 0x010927; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s110928; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s010929; GFX12-WGP-NEXT: ds_store_b32 v0, v110930; GFX12-WGP-NEXT: s_endpgm10931;10932; GFX12-CU-LABEL: local_workgroup_one_as_monotonic_store:10933; GFX12-CU: ; %bb.0: ; %entry10934; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x010935; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x410936; GFX12-CU-NEXT: s_wait_kmcnt 0x010937; GFX12-CU-NEXT: v_mov_b32_e32 v0, s110938; GFX12-CU-NEXT: v_mov_b32_e32 v1, s010939; GFX12-CU-NEXT: ds_store_b32 v0, v110940; GFX12-CU-NEXT: s_endpgm10941;10942; GFX1250-LABEL: local_workgroup_one_as_monotonic_store:10943; GFX1250: ; %bb.0: ; %entry10944; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 110945; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x010946; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x410947; GFX1250-NEXT: s_wait_kmcnt 0x010948; GFX1250-NEXT: v_mov_b32_e32 v0, s110949; GFX1250-NEXT: v_mov_b32_e32 v1, s010950; GFX1250-NEXT: ds_store_b32 v0, v110951; GFX1250-NEXT: s_endpgm10952 i32 %in, ptr addrspace(3) %out) {10953entry:10954 store atomic i32 %in, ptr addrspace(3) %out syncscope("workgroup-one-as") monotonic, align 410955 ret void10956}10957 10958define amdgpu_kernel void @local_workgroup_one_as_release_store(10959; GFX6-LABEL: local_workgroup_one_as_release_store:10960; GFX6: ; %bb.0: ; %entry10961; GFX6-NEXT: s_load_dword s5, s[8:9], 0x110962; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr510963; GFX6-NEXT: s_load_dword s4, s[8:9], 0x010964; GFX6-NEXT: s_mov_b32 m0, -110965; GFX6-NEXT: s_waitcnt lgkmcnt(0)10966; GFX6-NEXT: v_mov_b32_e32 v0, s510967; GFX6-NEXT: v_mov_b32_e32 v1, s410968; GFX6-NEXT: ds_write_b32 v0, v110969; GFX6-NEXT: s_endpgm10970;10971; GFX7-LABEL: local_workgroup_one_as_release_store:10972; GFX7: ; %bb.0: ; %entry10973; GFX7-NEXT: s_load_dword s4, s[8:9], 0x010974; GFX7-NEXT: s_load_dword s5, s[8:9], 0x110975; GFX7-NEXT: s_mov_b32 m0, -110976; GFX7-NEXT: s_waitcnt lgkmcnt(0)10977; GFX7-NEXT: v_mov_b32_e32 v0, s510978; GFX7-NEXT: v_mov_b32_e32 v1, s410979; GFX7-NEXT: ds_write_b32 v0, v110980; GFX7-NEXT: s_endpgm10981;10982; GFX10-WGP-LABEL: local_workgroup_one_as_release_store:10983; GFX10-WGP: ; %bb.0: ; %entry10984; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x010985; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x410986; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)10987; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s510988; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s410989; GFX10-WGP-NEXT: ds_write_b32 v0, v110990; GFX10-WGP-NEXT: s_endpgm10991;10992; GFX10-CU-LABEL: local_workgroup_one_as_release_store:10993; GFX10-CU: ; %bb.0: ; %entry10994; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x010995; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x410996; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)10997; GFX10-CU-NEXT: v_mov_b32_e32 v0, s510998; GFX10-CU-NEXT: v_mov_b32_e32 v1, s410999; GFX10-CU-NEXT: ds_write_b32 v0, v111000; GFX10-CU-NEXT: s_endpgm11001;11002; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_release_store:11003; SKIP-CACHE-INV: ; %bb.0: ; %entry11004; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x011005; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x111006; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -111007; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)11008; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s111009; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s011010; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v111011; SKIP-CACHE-INV-NEXT: s_endpgm11012;11013; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_release_store:11014; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry11015; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x011016; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x411017; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11018; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s511019; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s411020; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v111021; GFX90A-NOTTGSPLIT-NEXT: s_endpgm11022;11023; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_release_store:11024; GFX90A-TGSPLIT: ; %bb.0: ; %entry11025; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x011026; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x411027; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11028; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s511029; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s411030; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v111031; GFX90A-TGSPLIT-NEXT: s_endpgm11032;11033; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_release_store:11034; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry11035; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x011036; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x411037; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11038; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s111039; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s011040; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v111041; GFX942-NOTTGSPLIT-NEXT: s_endpgm11042;11043; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_release_store:11044; GFX942-TGSPLIT: ; %bb.0: ; %entry11045; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x011046; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x411047; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11048; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s111049; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s011050; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v111051; GFX942-TGSPLIT-NEXT: s_endpgm11052;11053; GFX11-WGP-LABEL: local_workgroup_one_as_release_store:11054; GFX11-WGP: ; %bb.0: ; %entry11055; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x011056; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x411057; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)11058; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s111059; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s011060; GFX11-WGP-NEXT: ds_store_b32 v0, v111061; GFX11-WGP-NEXT: s_endpgm11062;11063; GFX11-CU-LABEL: local_workgroup_one_as_release_store:11064; GFX11-CU: ; %bb.0: ; %entry11065; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x011066; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x411067; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)11068; GFX11-CU-NEXT: v_mov_b32_e32 v0, s111069; GFX11-CU-NEXT: v_mov_b32_e32 v1, s011070; GFX11-CU-NEXT: ds_store_b32 v0, v111071; GFX11-CU-NEXT: s_endpgm11072;11073; GFX12-WGP-LABEL: local_workgroup_one_as_release_store:11074; GFX12-WGP: ; %bb.0: ; %entry11075; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x011076; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x411077; GFX12-WGP-NEXT: s_wait_kmcnt 0x011078; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s111079; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s011080; GFX12-WGP-NEXT: ds_store_b32 v0, v111081; GFX12-WGP-NEXT: s_endpgm11082;11083; GFX12-CU-LABEL: local_workgroup_one_as_release_store:11084; GFX12-CU: ; %bb.0: ; %entry11085; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x011086; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x411087; GFX12-CU-NEXT: s_wait_kmcnt 0x011088; GFX12-CU-NEXT: v_mov_b32_e32 v0, s111089; GFX12-CU-NEXT: v_mov_b32_e32 v1, s011090; GFX12-CU-NEXT: ds_store_b32 v0, v111091; GFX12-CU-NEXT: s_endpgm11092;11093; GFX1250-LABEL: local_workgroup_one_as_release_store:11094; GFX1250: ; %bb.0: ; %entry11095; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 111096; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x011097; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x411098; GFX1250-NEXT: s_wait_kmcnt 0x011099; GFX1250-NEXT: v_mov_b32_e32 v0, s111100; GFX1250-NEXT: v_mov_b32_e32 v1, s011101; GFX1250-NEXT: ds_store_b32 v0, v111102; GFX1250-NEXT: s_endpgm11103 i32 %in, ptr addrspace(3) %out) {11104entry:11105 store atomic i32 %in, ptr addrspace(3) %out syncscope("workgroup-one-as") release, align 411106 ret void11107}11108 11109define amdgpu_kernel void @local_workgroup_one_as_seq_cst_store(11110; GFX6-LABEL: local_workgroup_one_as_seq_cst_store:11111; GFX6: ; %bb.0: ; %entry11112; GFX6-NEXT: s_load_dword s5, s[8:9], 0x111113; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr511114; GFX6-NEXT: s_load_dword s4, s[8:9], 0x011115; GFX6-NEXT: s_mov_b32 m0, -111116; GFX6-NEXT: s_waitcnt lgkmcnt(0)11117; GFX6-NEXT: v_mov_b32_e32 v0, s511118; GFX6-NEXT: v_mov_b32_e32 v1, s411119; GFX6-NEXT: ds_write_b32 v0, v111120; GFX6-NEXT: s_endpgm11121;11122; GFX7-LABEL: local_workgroup_one_as_seq_cst_store:11123; GFX7: ; %bb.0: ; %entry11124; GFX7-NEXT: s_load_dword s4, s[8:9], 0x011125; GFX7-NEXT: s_load_dword s5, s[8:9], 0x111126; GFX7-NEXT: s_mov_b32 m0, -111127; GFX7-NEXT: s_waitcnt lgkmcnt(0)11128; GFX7-NEXT: v_mov_b32_e32 v0, s511129; GFX7-NEXT: v_mov_b32_e32 v1, s411130; GFX7-NEXT: ds_write_b32 v0, v111131; GFX7-NEXT: s_endpgm11132;11133; GFX10-WGP-LABEL: local_workgroup_one_as_seq_cst_store:11134; GFX10-WGP: ; %bb.0: ; %entry11135; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x011136; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x411137; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)11138; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s511139; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s411140; GFX10-WGP-NEXT: ds_write_b32 v0, v111141; GFX10-WGP-NEXT: s_endpgm11142;11143; GFX10-CU-LABEL: local_workgroup_one_as_seq_cst_store:11144; GFX10-CU: ; %bb.0: ; %entry11145; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x011146; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x411147; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)11148; GFX10-CU-NEXT: v_mov_b32_e32 v0, s511149; GFX10-CU-NEXT: v_mov_b32_e32 v1, s411150; GFX10-CU-NEXT: ds_write_b32 v0, v111151; GFX10-CU-NEXT: s_endpgm11152;11153; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_seq_cst_store:11154; SKIP-CACHE-INV: ; %bb.0: ; %entry11155; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x011156; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x111157; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -111158; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)11159; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s111160; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s011161; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v111162; SKIP-CACHE-INV-NEXT: s_endpgm11163;11164; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_seq_cst_store:11165; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry11166; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x011167; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x411168; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11169; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s511170; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s411171; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v111172; GFX90A-NOTTGSPLIT-NEXT: s_endpgm11173;11174; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_seq_cst_store:11175; GFX90A-TGSPLIT: ; %bb.0: ; %entry11176; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x011177; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x411178; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11179; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s511180; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s411181; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v111182; GFX90A-TGSPLIT-NEXT: s_endpgm11183;11184; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_seq_cst_store:11185; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry11186; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x011187; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x411188; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11189; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s111190; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s011191; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v111192; GFX942-NOTTGSPLIT-NEXT: s_endpgm11193;11194; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_seq_cst_store:11195; GFX942-TGSPLIT: ; %bb.0: ; %entry11196; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x011197; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x411198; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11199; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s111200; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s011201; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v111202; GFX942-TGSPLIT-NEXT: s_endpgm11203;11204; GFX11-WGP-LABEL: local_workgroup_one_as_seq_cst_store:11205; GFX11-WGP: ; %bb.0: ; %entry11206; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x011207; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x411208; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)11209; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s111210; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s011211; GFX11-WGP-NEXT: ds_store_b32 v0, v111212; GFX11-WGP-NEXT: s_endpgm11213;11214; GFX11-CU-LABEL: local_workgroup_one_as_seq_cst_store:11215; GFX11-CU: ; %bb.0: ; %entry11216; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x011217; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x411218; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)11219; GFX11-CU-NEXT: v_mov_b32_e32 v0, s111220; GFX11-CU-NEXT: v_mov_b32_e32 v1, s011221; GFX11-CU-NEXT: ds_store_b32 v0, v111222; GFX11-CU-NEXT: s_endpgm11223;11224; GFX12-WGP-LABEL: local_workgroup_one_as_seq_cst_store:11225; GFX12-WGP: ; %bb.0: ; %entry11226; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x011227; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x411228; GFX12-WGP-NEXT: s_wait_kmcnt 0x011229; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s111230; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s011231; GFX12-WGP-NEXT: ds_store_b32 v0, v111232; GFX12-WGP-NEXT: s_endpgm11233;11234; GFX12-CU-LABEL: local_workgroup_one_as_seq_cst_store:11235; GFX12-CU: ; %bb.0: ; %entry11236; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x011237; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x411238; GFX12-CU-NEXT: s_wait_kmcnt 0x011239; GFX12-CU-NEXT: v_mov_b32_e32 v0, s111240; GFX12-CU-NEXT: v_mov_b32_e32 v1, s011241; GFX12-CU-NEXT: ds_store_b32 v0, v111242; GFX12-CU-NEXT: s_endpgm11243;11244; GFX1250-LABEL: local_workgroup_one_as_seq_cst_store:11245; GFX1250: ; %bb.0: ; %entry11246; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 111247; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x011248; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x411249; GFX1250-NEXT: s_wait_kmcnt 0x011250; GFX1250-NEXT: v_mov_b32_e32 v0, s111251; GFX1250-NEXT: v_mov_b32_e32 v1, s011252; GFX1250-NEXT: ds_store_b32 v0, v111253; GFX1250-NEXT: s_endpgm11254 i32 %in, ptr addrspace(3) %out) {11255entry:11256 store atomic i32 %in, ptr addrspace(3) %out syncscope("workgroup-one-as") seq_cst, align 411257 ret void11258}11259 11260define amdgpu_kernel void @local_workgroup_one_as_monotonic_atomicrmw(11261; GFX6-LABEL: local_workgroup_one_as_monotonic_atomicrmw:11262; GFX6: ; %bb.0: ; %entry11263; GFX6-NEXT: s_load_dword s5, s[8:9], 0x011264; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr511265; GFX6-NEXT: s_load_dword s4, s[8:9], 0x111266; GFX6-NEXT: s_mov_b32 m0, -111267; GFX6-NEXT: s_waitcnt lgkmcnt(0)11268; GFX6-NEXT: v_mov_b32_e32 v0, s511269; GFX6-NEXT: v_mov_b32_e32 v1, s411270; GFX6-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111271; GFX6-NEXT: s_endpgm11272;11273; GFX7-LABEL: local_workgroup_one_as_monotonic_atomicrmw:11274; GFX7: ; %bb.0: ; %entry11275; GFX7-NEXT: s_load_dword s5, s[8:9], 0x011276; GFX7-NEXT: s_load_dword s4, s[8:9], 0x111277; GFX7-NEXT: s_mov_b32 m0, -111278; GFX7-NEXT: s_waitcnt lgkmcnt(0)11279; GFX7-NEXT: v_mov_b32_e32 v0, s511280; GFX7-NEXT: v_mov_b32_e32 v1, s411281; GFX7-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111282; GFX7-NEXT: s_endpgm11283;11284; GFX10-WGP-LABEL: local_workgroup_one_as_monotonic_atomicrmw:11285; GFX10-WGP: ; %bb.0: ; %entry11286; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x011287; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x411288; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)11289; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s511290; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s411291; GFX10-WGP-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111292; GFX10-WGP-NEXT: s_endpgm11293;11294; GFX10-CU-LABEL: local_workgroup_one_as_monotonic_atomicrmw:11295; GFX10-CU: ; %bb.0: ; %entry11296; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x011297; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x411298; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)11299; GFX10-CU-NEXT: v_mov_b32_e32 v0, s511300; GFX10-CU-NEXT: v_mov_b32_e32 v1, s411301; GFX10-CU-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111302; GFX10-CU-NEXT: s_endpgm11303;11304; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_monotonic_atomicrmw:11305; SKIP-CACHE-INV: ; %bb.0: ; %entry11306; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x011307; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x111308; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -111309; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)11310; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s111311; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s011312; SKIP-CACHE-INV-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111313; SKIP-CACHE-INV-NEXT: s_endpgm11314;11315; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_monotonic_atomicrmw:11316; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry11317; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x011318; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x411319; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11320; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s511321; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s411322; GFX90A-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111323; GFX90A-NOTTGSPLIT-NEXT: s_endpgm11324;11325; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_monotonic_atomicrmw:11326; GFX90A-TGSPLIT: ; %bb.0: ; %entry11327; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x011328; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x411329; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11330; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s511331; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s411332; GFX90A-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111333; GFX90A-TGSPLIT-NEXT: s_endpgm11334;11335; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_monotonic_atomicrmw:11336; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry11337; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x011338; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x411339; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11340; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s111341; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s011342; GFX942-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111343; GFX942-NOTTGSPLIT-NEXT: s_endpgm11344;11345; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_monotonic_atomicrmw:11346; GFX942-TGSPLIT: ; %bb.0: ; %entry11347; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x011348; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x411349; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11350; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s111351; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s011352; GFX942-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111353; GFX942-TGSPLIT-NEXT: s_endpgm11354;11355; GFX11-WGP-LABEL: local_workgroup_one_as_monotonic_atomicrmw:11356; GFX11-WGP: ; %bb.0: ; %entry11357; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x011358; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x411359; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)11360; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s111361; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s011362; GFX11-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v111363; GFX11-WGP-NEXT: s_endpgm11364;11365; GFX11-CU-LABEL: local_workgroup_one_as_monotonic_atomicrmw:11366; GFX11-CU: ; %bb.0: ; %entry11367; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x011368; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x411369; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)11370; GFX11-CU-NEXT: v_mov_b32_e32 v0, s111371; GFX11-CU-NEXT: v_mov_b32_e32 v1, s011372; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v111373; GFX11-CU-NEXT: s_endpgm11374;11375; GFX12-WGP-LABEL: local_workgroup_one_as_monotonic_atomicrmw:11376; GFX12-WGP: ; %bb.0: ; %entry11377; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x011378; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x411379; GFX12-WGP-NEXT: s_wait_kmcnt 0x011380; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s111381; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s011382; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v111383; GFX12-WGP-NEXT: s_endpgm11384;11385; GFX12-CU-LABEL: local_workgroup_one_as_monotonic_atomicrmw:11386; GFX12-CU: ; %bb.0: ; %entry11387; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x011388; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x411389; GFX12-CU-NEXT: s_wait_kmcnt 0x011390; GFX12-CU-NEXT: v_mov_b32_e32 v0, s111391; GFX12-CU-NEXT: v_mov_b32_e32 v1, s011392; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v111393; GFX12-CU-NEXT: s_endpgm11394;11395; GFX1250-LABEL: local_workgroup_one_as_monotonic_atomicrmw:11396; GFX1250: ; %bb.0: ; %entry11397; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 111398; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x011399; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x411400; GFX1250-NEXT: s_wait_kmcnt 0x011401; GFX1250-NEXT: v_mov_b32_e32 v0, s111402; GFX1250-NEXT: v_mov_b32_e32 v1, s011403; GFX1250-NEXT: ds_storexchg_rtn_b32 v0, v0, v111404; GFX1250-NEXT: s_endpgm11405 ptr addrspace(3) %out, i32 %in) {11406entry:11407 %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup-one-as") monotonic11408 ret void11409}11410 11411define amdgpu_kernel void @local_workgroup_one_as_acquire_atomicrmw(11412; GFX6-LABEL: local_workgroup_one_as_acquire_atomicrmw:11413; GFX6: ; %bb.0: ; %entry11414; GFX6-NEXT: s_load_dword s5, s[8:9], 0x011415; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr511416; GFX6-NEXT: s_load_dword s4, s[8:9], 0x111417; GFX6-NEXT: s_mov_b32 m0, -111418; GFX6-NEXT: s_waitcnt lgkmcnt(0)11419; GFX6-NEXT: v_mov_b32_e32 v0, s511420; GFX6-NEXT: v_mov_b32_e32 v1, s411421; GFX6-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111422; GFX6-NEXT: s_endpgm11423;11424; GFX7-LABEL: local_workgroup_one_as_acquire_atomicrmw:11425; GFX7: ; %bb.0: ; %entry11426; GFX7-NEXT: s_load_dword s5, s[8:9], 0x011427; GFX7-NEXT: s_load_dword s4, s[8:9], 0x111428; GFX7-NEXT: s_mov_b32 m0, -111429; GFX7-NEXT: s_waitcnt lgkmcnt(0)11430; GFX7-NEXT: v_mov_b32_e32 v0, s511431; GFX7-NEXT: v_mov_b32_e32 v1, s411432; GFX7-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111433; GFX7-NEXT: s_endpgm11434;11435; GFX10-WGP-LABEL: local_workgroup_one_as_acquire_atomicrmw:11436; GFX10-WGP: ; %bb.0: ; %entry11437; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x011438; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x411439; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)11440; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s511441; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s411442; GFX10-WGP-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111443; GFX10-WGP-NEXT: s_endpgm11444;11445; GFX10-CU-LABEL: local_workgroup_one_as_acquire_atomicrmw:11446; GFX10-CU: ; %bb.0: ; %entry11447; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x011448; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x411449; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)11450; GFX10-CU-NEXT: v_mov_b32_e32 v0, s511451; GFX10-CU-NEXT: v_mov_b32_e32 v1, s411452; GFX10-CU-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111453; GFX10-CU-NEXT: s_endpgm11454;11455; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acquire_atomicrmw:11456; SKIP-CACHE-INV: ; %bb.0: ; %entry11457; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x011458; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x111459; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -111460; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)11461; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s111462; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s011463; SKIP-CACHE-INV-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111464; SKIP-CACHE-INV-NEXT: s_endpgm11465;11466; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_acquire_atomicrmw:11467; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry11468; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x011469; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x411470; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11471; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s511472; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s411473; GFX90A-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111474; GFX90A-NOTTGSPLIT-NEXT: s_endpgm11475;11476; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_acquire_atomicrmw:11477; GFX90A-TGSPLIT: ; %bb.0: ; %entry11478; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x011479; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x411480; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11481; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s511482; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s411483; GFX90A-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111484; GFX90A-TGSPLIT-NEXT: s_endpgm11485;11486; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_acquire_atomicrmw:11487; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry11488; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x011489; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x411490; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11491; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s111492; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s011493; GFX942-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111494; GFX942-NOTTGSPLIT-NEXT: s_endpgm11495;11496; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_acquire_atomicrmw:11497; GFX942-TGSPLIT: ; %bb.0: ; %entry11498; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x011499; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x411500; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11501; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s111502; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s011503; GFX942-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111504; GFX942-TGSPLIT-NEXT: s_endpgm11505;11506; GFX11-WGP-LABEL: local_workgroup_one_as_acquire_atomicrmw:11507; GFX11-WGP: ; %bb.0: ; %entry11508; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x011509; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x411510; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)11511; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s111512; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s011513; GFX11-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v111514; GFX11-WGP-NEXT: s_endpgm11515;11516; GFX11-CU-LABEL: local_workgroup_one_as_acquire_atomicrmw:11517; GFX11-CU: ; %bb.0: ; %entry11518; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x011519; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x411520; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)11521; GFX11-CU-NEXT: v_mov_b32_e32 v0, s111522; GFX11-CU-NEXT: v_mov_b32_e32 v1, s011523; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v111524; GFX11-CU-NEXT: s_endpgm11525;11526; GFX12-WGP-LABEL: local_workgroup_one_as_acquire_atomicrmw:11527; GFX12-WGP: ; %bb.0: ; %entry11528; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x011529; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x411530; GFX12-WGP-NEXT: s_wait_kmcnt 0x011531; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s111532; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s011533; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v111534; GFX12-WGP-NEXT: s_endpgm11535;11536; GFX12-CU-LABEL: local_workgroup_one_as_acquire_atomicrmw:11537; GFX12-CU: ; %bb.0: ; %entry11538; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x011539; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x411540; GFX12-CU-NEXT: s_wait_kmcnt 0x011541; GFX12-CU-NEXT: v_mov_b32_e32 v0, s111542; GFX12-CU-NEXT: v_mov_b32_e32 v1, s011543; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v111544; GFX12-CU-NEXT: s_endpgm11545;11546; GFX1250-LABEL: local_workgroup_one_as_acquire_atomicrmw:11547; GFX1250: ; %bb.0: ; %entry11548; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 111549; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x011550; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x411551; GFX1250-NEXT: s_wait_kmcnt 0x011552; GFX1250-NEXT: v_mov_b32_e32 v0, s111553; GFX1250-NEXT: v_mov_b32_e32 v1, s011554; GFX1250-NEXT: ds_storexchg_rtn_b32 v0, v0, v111555; GFX1250-NEXT: s_endpgm11556 ptr addrspace(3) %out, i32 %in) {11557entry:11558 %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup-one-as") acquire11559 ret void11560}11561 11562define amdgpu_kernel void @local_workgroup_one_as_release_atomicrmw(11563; GFX6-LABEL: local_workgroup_one_as_release_atomicrmw:11564; GFX6: ; %bb.0: ; %entry11565; GFX6-NEXT: s_load_dword s5, s[8:9], 0x011566; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr511567; GFX6-NEXT: s_load_dword s4, s[8:9], 0x111568; GFX6-NEXT: s_mov_b32 m0, -111569; GFX6-NEXT: s_waitcnt lgkmcnt(0)11570; GFX6-NEXT: v_mov_b32_e32 v0, s511571; GFX6-NEXT: v_mov_b32_e32 v1, s411572; GFX6-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111573; GFX6-NEXT: s_endpgm11574;11575; GFX7-LABEL: local_workgroup_one_as_release_atomicrmw:11576; GFX7: ; %bb.0: ; %entry11577; GFX7-NEXT: s_load_dword s5, s[8:9], 0x011578; GFX7-NEXT: s_load_dword s4, s[8:9], 0x111579; GFX7-NEXT: s_mov_b32 m0, -111580; GFX7-NEXT: s_waitcnt lgkmcnt(0)11581; GFX7-NEXT: v_mov_b32_e32 v0, s511582; GFX7-NEXT: v_mov_b32_e32 v1, s411583; GFX7-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111584; GFX7-NEXT: s_endpgm11585;11586; GFX10-WGP-LABEL: local_workgroup_one_as_release_atomicrmw:11587; GFX10-WGP: ; %bb.0: ; %entry11588; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x011589; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x411590; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)11591; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s511592; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s411593; GFX10-WGP-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111594; GFX10-WGP-NEXT: s_endpgm11595;11596; GFX10-CU-LABEL: local_workgroup_one_as_release_atomicrmw:11597; GFX10-CU: ; %bb.0: ; %entry11598; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x011599; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x411600; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)11601; GFX10-CU-NEXT: v_mov_b32_e32 v0, s511602; GFX10-CU-NEXT: v_mov_b32_e32 v1, s411603; GFX10-CU-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111604; GFX10-CU-NEXT: s_endpgm11605;11606; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_release_atomicrmw:11607; SKIP-CACHE-INV: ; %bb.0: ; %entry11608; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x011609; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x111610; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -111611; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)11612; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s111613; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s011614; SKIP-CACHE-INV-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111615; SKIP-CACHE-INV-NEXT: s_endpgm11616;11617; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_release_atomicrmw:11618; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry11619; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x011620; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x411621; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11622; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s511623; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s411624; GFX90A-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111625; GFX90A-NOTTGSPLIT-NEXT: s_endpgm11626;11627; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_release_atomicrmw:11628; GFX90A-TGSPLIT: ; %bb.0: ; %entry11629; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x011630; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x411631; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11632; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s511633; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s411634; GFX90A-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111635; GFX90A-TGSPLIT-NEXT: s_endpgm11636;11637; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_release_atomicrmw:11638; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry11639; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x011640; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x411641; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11642; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s111643; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s011644; GFX942-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111645; GFX942-NOTTGSPLIT-NEXT: s_endpgm11646;11647; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_release_atomicrmw:11648; GFX942-TGSPLIT: ; %bb.0: ; %entry11649; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x011650; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x411651; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11652; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s111653; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s011654; GFX942-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111655; GFX942-TGSPLIT-NEXT: s_endpgm11656;11657; GFX11-WGP-LABEL: local_workgroup_one_as_release_atomicrmw:11658; GFX11-WGP: ; %bb.0: ; %entry11659; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x011660; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x411661; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)11662; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s111663; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s011664; GFX11-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v111665; GFX11-WGP-NEXT: s_endpgm11666;11667; GFX11-CU-LABEL: local_workgroup_one_as_release_atomicrmw:11668; GFX11-CU: ; %bb.0: ; %entry11669; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x011670; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x411671; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)11672; GFX11-CU-NEXT: v_mov_b32_e32 v0, s111673; GFX11-CU-NEXT: v_mov_b32_e32 v1, s011674; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v111675; GFX11-CU-NEXT: s_endpgm11676;11677; GFX12-WGP-LABEL: local_workgroup_one_as_release_atomicrmw:11678; GFX12-WGP: ; %bb.0: ; %entry11679; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x011680; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x411681; GFX12-WGP-NEXT: s_wait_kmcnt 0x011682; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s111683; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s011684; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v111685; GFX12-WGP-NEXT: s_endpgm11686;11687; GFX12-CU-LABEL: local_workgroup_one_as_release_atomicrmw:11688; GFX12-CU: ; %bb.0: ; %entry11689; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x011690; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x411691; GFX12-CU-NEXT: s_wait_kmcnt 0x011692; GFX12-CU-NEXT: v_mov_b32_e32 v0, s111693; GFX12-CU-NEXT: v_mov_b32_e32 v1, s011694; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v111695; GFX12-CU-NEXT: s_endpgm11696;11697; GFX1250-LABEL: local_workgroup_one_as_release_atomicrmw:11698; GFX1250: ; %bb.0: ; %entry11699; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 111700; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x011701; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x411702; GFX1250-NEXT: s_wait_kmcnt 0x011703; GFX1250-NEXT: v_mov_b32_e32 v0, s111704; GFX1250-NEXT: v_mov_b32_e32 v1, s011705; GFX1250-NEXT: ds_storexchg_rtn_b32 v0, v0, v111706; GFX1250-NEXT: s_endpgm11707 ptr addrspace(3) %out, i32 %in) {11708entry:11709 %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup-one-as") release11710 ret void11711}11712 11713define amdgpu_kernel void @local_workgroup_one_as_acq_rel_atomicrmw(11714; GFX6-LABEL: local_workgroup_one_as_acq_rel_atomicrmw:11715; GFX6: ; %bb.0: ; %entry11716; GFX6-NEXT: s_load_dword s5, s[8:9], 0x011717; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr511718; GFX6-NEXT: s_load_dword s4, s[8:9], 0x111719; GFX6-NEXT: s_mov_b32 m0, -111720; GFX6-NEXT: s_waitcnt lgkmcnt(0)11721; GFX6-NEXT: v_mov_b32_e32 v0, s511722; GFX6-NEXT: v_mov_b32_e32 v1, s411723; GFX6-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111724; GFX6-NEXT: s_endpgm11725;11726; GFX7-LABEL: local_workgroup_one_as_acq_rel_atomicrmw:11727; GFX7: ; %bb.0: ; %entry11728; GFX7-NEXT: s_load_dword s5, s[8:9], 0x011729; GFX7-NEXT: s_load_dword s4, s[8:9], 0x111730; GFX7-NEXT: s_mov_b32 m0, -111731; GFX7-NEXT: s_waitcnt lgkmcnt(0)11732; GFX7-NEXT: v_mov_b32_e32 v0, s511733; GFX7-NEXT: v_mov_b32_e32 v1, s411734; GFX7-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111735; GFX7-NEXT: s_endpgm11736;11737; GFX10-WGP-LABEL: local_workgroup_one_as_acq_rel_atomicrmw:11738; GFX10-WGP: ; %bb.0: ; %entry11739; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x011740; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x411741; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)11742; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s511743; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s411744; GFX10-WGP-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111745; GFX10-WGP-NEXT: s_endpgm11746;11747; GFX10-CU-LABEL: local_workgroup_one_as_acq_rel_atomicrmw:11748; GFX10-CU: ; %bb.0: ; %entry11749; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x011750; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x411751; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)11752; GFX10-CU-NEXT: v_mov_b32_e32 v0, s511753; GFX10-CU-NEXT: v_mov_b32_e32 v1, s411754; GFX10-CU-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111755; GFX10-CU-NEXT: s_endpgm11756;11757; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acq_rel_atomicrmw:11758; SKIP-CACHE-INV: ; %bb.0: ; %entry11759; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x011760; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x111761; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -111762; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)11763; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s111764; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s011765; SKIP-CACHE-INV-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111766; SKIP-CACHE-INV-NEXT: s_endpgm11767;11768; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_acq_rel_atomicrmw:11769; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry11770; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x011771; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x411772; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11773; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s511774; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s411775; GFX90A-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111776; GFX90A-NOTTGSPLIT-NEXT: s_endpgm11777;11778; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_acq_rel_atomicrmw:11779; GFX90A-TGSPLIT: ; %bb.0: ; %entry11780; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x011781; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x411782; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11783; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s511784; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s411785; GFX90A-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111786; GFX90A-TGSPLIT-NEXT: s_endpgm11787;11788; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_acq_rel_atomicrmw:11789; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry11790; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x011791; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x411792; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11793; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s111794; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s011795; GFX942-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111796; GFX942-NOTTGSPLIT-NEXT: s_endpgm11797;11798; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_acq_rel_atomicrmw:11799; GFX942-TGSPLIT: ; %bb.0: ; %entry11800; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x011801; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x411802; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11803; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s111804; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s011805; GFX942-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111806; GFX942-TGSPLIT-NEXT: s_endpgm11807;11808; GFX11-WGP-LABEL: local_workgroup_one_as_acq_rel_atomicrmw:11809; GFX11-WGP: ; %bb.0: ; %entry11810; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x011811; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x411812; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)11813; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s111814; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s011815; GFX11-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v111816; GFX11-WGP-NEXT: s_endpgm11817;11818; GFX11-CU-LABEL: local_workgroup_one_as_acq_rel_atomicrmw:11819; GFX11-CU: ; %bb.0: ; %entry11820; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x011821; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x411822; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)11823; GFX11-CU-NEXT: v_mov_b32_e32 v0, s111824; GFX11-CU-NEXT: v_mov_b32_e32 v1, s011825; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v111826; GFX11-CU-NEXT: s_endpgm11827;11828; GFX12-WGP-LABEL: local_workgroup_one_as_acq_rel_atomicrmw:11829; GFX12-WGP: ; %bb.0: ; %entry11830; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x011831; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x411832; GFX12-WGP-NEXT: s_wait_kmcnt 0x011833; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s111834; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s011835; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v111836; GFX12-WGP-NEXT: s_endpgm11837;11838; GFX12-CU-LABEL: local_workgroup_one_as_acq_rel_atomicrmw:11839; GFX12-CU: ; %bb.0: ; %entry11840; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x011841; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x411842; GFX12-CU-NEXT: s_wait_kmcnt 0x011843; GFX12-CU-NEXT: v_mov_b32_e32 v0, s111844; GFX12-CU-NEXT: v_mov_b32_e32 v1, s011845; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v111846; GFX12-CU-NEXT: s_endpgm11847;11848; GFX1250-LABEL: local_workgroup_one_as_acq_rel_atomicrmw:11849; GFX1250: ; %bb.0: ; %entry11850; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 111851; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x011852; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x411853; GFX1250-NEXT: s_wait_kmcnt 0x011854; GFX1250-NEXT: v_mov_b32_e32 v0, s111855; GFX1250-NEXT: v_mov_b32_e32 v1, s011856; GFX1250-NEXT: ds_storexchg_rtn_b32 v0, v0, v111857; GFX1250-NEXT: s_endpgm11858 ptr addrspace(3) %out, i32 %in) {11859entry:11860 %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup-one-as") acq_rel11861 ret void11862}11863 11864define amdgpu_kernel void @local_workgroup_one_as_seq_cst_atomicrmw(11865; GFX6-LABEL: local_workgroup_one_as_seq_cst_atomicrmw:11866; GFX6: ; %bb.0: ; %entry11867; GFX6-NEXT: s_load_dword s5, s[8:9], 0x011868; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr511869; GFX6-NEXT: s_load_dword s4, s[8:9], 0x111870; GFX6-NEXT: s_mov_b32 m0, -111871; GFX6-NEXT: s_waitcnt lgkmcnt(0)11872; GFX6-NEXT: v_mov_b32_e32 v0, s511873; GFX6-NEXT: v_mov_b32_e32 v1, s411874; GFX6-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111875; GFX6-NEXT: s_endpgm11876;11877; GFX7-LABEL: local_workgroup_one_as_seq_cst_atomicrmw:11878; GFX7: ; %bb.0: ; %entry11879; GFX7-NEXT: s_load_dword s5, s[8:9], 0x011880; GFX7-NEXT: s_load_dword s4, s[8:9], 0x111881; GFX7-NEXT: s_mov_b32 m0, -111882; GFX7-NEXT: s_waitcnt lgkmcnt(0)11883; GFX7-NEXT: v_mov_b32_e32 v0, s511884; GFX7-NEXT: v_mov_b32_e32 v1, s411885; GFX7-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111886; GFX7-NEXT: s_endpgm11887;11888; GFX10-WGP-LABEL: local_workgroup_one_as_seq_cst_atomicrmw:11889; GFX10-WGP: ; %bb.0: ; %entry11890; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x011891; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x411892; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)11893; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s511894; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s411895; GFX10-WGP-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111896; GFX10-WGP-NEXT: s_endpgm11897;11898; GFX10-CU-LABEL: local_workgroup_one_as_seq_cst_atomicrmw:11899; GFX10-CU: ; %bb.0: ; %entry11900; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x011901; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x411902; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)11903; GFX10-CU-NEXT: v_mov_b32_e32 v0, s511904; GFX10-CU-NEXT: v_mov_b32_e32 v1, s411905; GFX10-CU-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111906; GFX10-CU-NEXT: s_endpgm11907;11908; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_seq_cst_atomicrmw:11909; SKIP-CACHE-INV: ; %bb.0: ; %entry11910; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x011911; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x111912; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -111913; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)11914; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s111915; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s011916; SKIP-CACHE-INV-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111917; SKIP-CACHE-INV-NEXT: s_endpgm11918;11919; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_seq_cst_atomicrmw:11920; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry11921; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x011922; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x411923; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11924; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s511925; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s411926; GFX90A-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111927; GFX90A-NOTTGSPLIT-NEXT: s_endpgm11928;11929; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_seq_cst_atomicrmw:11930; GFX90A-TGSPLIT: ; %bb.0: ; %entry11931; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x011932; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x411933; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11934; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s511935; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s411936; GFX90A-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111937; GFX90A-TGSPLIT-NEXT: s_endpgm11938;11939; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_seq_cst_atomicrmw:11940; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry11941; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x011942; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x411943; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11944; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s111945; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s011946; GFX942-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111947; GFX942-NOTTGSPLIT-NEXT: s_endpgm11948;11949; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_seq_cst_atomicrmw:11950; GFX942-TGSPLIT: ; %bb.0: ; %entry11951; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x011952; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x411953; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)11954; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s111955; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s011956; GFX942-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v0, v0, v111957; GFX942-TGSPLIT-NEXT: s_endpgm11958;11959; GFX11-WGP-LABEL: local_workgroup_one_as_seq_cst_atomicrmw:11960; GFX11-WGP: ; %bb.0: ; %entry11961; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x011962; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x411963; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)11964; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s111965; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s011966; GFX11-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v111967; GFX11-WGP-NEXT: s_endpgm11968;11969; GFX11-CU-LABEL: local_workgroup_one_as_seq_cst_atomicrmw:11970; GFX11-CU: ; %bb.0: ; %entry11971; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x011972; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x411973; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)11974; GFX11-CU-NEXT: v_mov_b32_e32 v0, s111975; GFX11-CU-NEXT: v_mov_b32_e32 v1, s011976; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v111977; GFX11-CU-NEXT: s_endpgm11978;11979; GFX12-WGP-LABEL: local_workgroup_one_as_seq_cst_atomicrmw:11980; GFX12-WGP: ; %bb.0: ; %entry11981; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x011982; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x411983; GFX12-WGP-NEXT: s_wait_kmcnt 0x011984; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s111985; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s011986; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v0, v0, v111987; GFX12-WGP-NEXT: s_endpgm11988;11989; GFX12-CU-LABEL: local_workgroup_one_as_seq_cst_atomicrmw:11990; GFX12-CU: ; %bb.0: ; %entry11991; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x011992; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x411993; GFX12-CU-NEXT: s_wait_kmcnt 0x011994; GFX12-CU-NEXT: v_mov_b32_e32 v0, s111995; GFX12-CU-NEXT: v_mov_b32_e32 v1, s011996; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v0, v0, v111997; GFX12-CU-NEXT: s_endpgm11998;11999; GFX1250-LABEL: local_workgroup_one_as_seq_cst_atomicrmw:12000; GFX1250: ; %bb.0: ; %entry12001; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 112002; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x012003; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x412004; GFX1250-NEXT: s_wait_kmcnt 0x012005; GFX1250-NEXT: v_mov_b32_e32 v0, s112006; GFX1250-NEXT: v_mov_b32_e32 v1, s012007; GFX1250-NEXT: ds_storexchg_rtn_b32 v0, v0, v112008; GFX1250-NEXT: s_endpgm12009 ptr addrspace(3) %out, i32 %in) {12010entry:12011 %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup-one-as") seq_cst12012 ret void12013}12014 12015define amdgpu_kernel void @local_workgroup_one_as_acquire_ret_atomicrmw(12016; GFX6-LABEL: local_workgroup_one_as_acquire_ret_atomicrmw:12017; GFX6: ; %bb.0: ; %entry12018; GFX6-NEXT: s_load_dword s4, s[8:9], 0x012019; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr412020; GFX6-NEXT: s_load_dword s5, s[8:9], 0x112021; GFX6-NEXT: s_mov_b32 m0, -112022; GFX6-NEXT: s_waitcnt lgkmcnt(0)12023; GFX6-NEXT: v_mov_b32_e32 v0, s412024; GFX6-NEXT: v_mov_b32_e32 v1, s512025; GFX6-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112026; GFX6-NEXT: s_mov_b32 m0, -112027; GFX6-NEXT: v_mov_b32_e32 v0, s412028; GFX6-NEXT: s_waitcnt lgkmcnt(0)12029; GFX6-NEXT: ds_write_b32 v0, v112030; GFX6-NEXT: s_endpgm12031;12032; GFX7-LABEL: local_workgroup_one_as_acquire_ret_atomicrmw:12033; GFX7: ; %bb.0: ; %entry12034; GFX7-NEXT: s_load_dword s4, s[8:9], 0x012035; GFX7-NEXT: s_load_dword s5, s[8:9], 0x112036; GFX7-NEXT: s_mov_b32 m0, -112037; GFX7-NEXT: s_waitcnt lgkmcnt(0)12038; GFX7-NEXT: v_mov_b32_e32 v0, s412039; GFX7-NEXT: v_mov_b32_e32 v1, s512040; GFX7-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112041; GFX7-NEXT: s_mov_b32 m0, -112042; GFX7-NEXT: v_mov_b32_e32 v0, s412043; GFX7-NEXT: s_waitcnt lgkmcnt(0)12044; GFX7-NEXT: ds_write_b32 v0, v112045; GFX7-NEXT: s_endpgm12046;12047; GFX10-WGP-LABEL: local_workgroup_one_as_acquire_ret_atomicrmw:12048; GFX10-WGP: ; %bb.0: ; %entry12049; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x012050; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x412051; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)12052; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s412053; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s512054; GFX10-WGP-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112055; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s412056; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)12057; GFX10-WGP-NEXT: ds_write_b32 v0, v112058; GFX10-WGP-NEXT: s_endpgm12059;12060; GFX10-CU-LABEL: local_workgroup_one_as_acquire_ret_atomicrmw:12061; GFX10-CU: ; %bb.0: ; %entry12062; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x012063; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x412064; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)12065; GFX10-CU-NEXT: v_mov_b32_e32 v0, s412066; GFX10-CU-NEXT: v_mov_b32_e32 v1, s512067; GFX10-CU-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112068; GFX10-CU-NEXT: v_mov_b32_e32 v0, s412069; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)12070; GFX10-CU-NEXT: ds_write_b32 v0, v112071; GFX10-CU-NEXT: s_endpgm12072;12073; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acquire_ret_atomicrmw:12074; SKIP-CACHE-INV: ; %bb.0: ; %entry12075; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x012076; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x112077; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -112078; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)12079; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s012080; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s112081; SKIP-CACHE-INV-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112082; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -112083; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s012084; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)12085; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v112086; SKIP-CACHE-INV-NEXT: s_endpgm12087;12088; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_acquire_ret_atomicrmw:12089; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry12090; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x012091; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x412092; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12093; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s412094; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s512095; GFX90A-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112096; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s412097; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12098; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v112099; GFX90A-NOTTGSPLIT-NEXT: s_endpgm12100;12101; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_acquire_ret_atomicrmw:12102; GFX90A-TGSPLIT: ; %bb.0: ; %entry12103; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x012104; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x412105; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12106; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s412107; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s512108; GFX90A-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112109; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s412110; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12111; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v112112; GFX90A-TGSPLIT-NEXT: s_endpgm12113;12114; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_acquire_ret_atomicrmw:12115; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry12116; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x012117; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x412118; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12119; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s012120; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s112121; GFX942-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112122; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s012123; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12124; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v112125; GFX942-NOTTGSPLIT-NEXT: s_endpgm12126;12127; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_acquire_ret_atomicrmw:12128; GFX942-TGSPLIT: ; %bb.0: ; %entry12129; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x012130; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x412131; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12132; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s012133; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s112134; GFX942-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112135; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s012136; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12137; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v112138; GFX942-TGSPLIT-NEXT: s_endpgm12139;12140; GFX11-WGP-LABEL: local_workgroup_one_as_acquire_ret_atomicrmw:12141; GFX11-WGP: ; %bb.0: ; %entry12142; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x012143; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x412144; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)12145; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s012146; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s112147; GFX11-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v112148; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s012149; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)12150; GFX11-WGP-NEXT: ds_store_b32 v0, v112151; GFX11-WGP-NEXT: s_endpgm12152;12153; GFX11-CU-LABEL: local_workgroup_one_as_acquire_ret_atomicrmw:12154; GFX11-CU: ; %bb.0: ; %entry12155; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x012156; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x412157; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)12158; GFX11-CU-NEXT: v_mov_b32_e32 v0, s012159; GFX11-CU-NEXT: v_mov_b32_e32 v1, s112160; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v112161; GFX11-CU-NEXT: v_mov_b32_e32 v0, s012162; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)12163; GFX11-CU-NEXT: ds_store_b32 v0, v112164; GFX11-CU-NEXT: s_endpgm12165;12166; GFX12-WGP-LABEL: local_workgroup_one_as_acquire_ret_atomicrmw:12167; GFX12-WGP: ; %bb.0: ; %entry12168; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x012169; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x412170; GFX12-WGP-NEXT: s_wait_kmcnt 0x012171; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s012172; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s112173; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v112174; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s012175; GFX12-WGP-NEXT: s_wait_dscnt 0x012176; GFX12-WGP-NEXT: ds_store_b32 v0, v112177; GFX12-WGP-NEXT: s_endpgm12178;12179; GFX12-CU-LABEL: local_workgroup_one_as_acquire_ret_atomicrmw:12180; GFX12-CU: ; %bb.0: ; %entry12181; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x012182; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x412183; GFX12-CU-NEXT: s_wait_kmcnt 0x012184; GFX12-CU-NEXT: v_mov_b32_e32 v0, s012185; GFX12-CU-NEXT: v_mov_b32_e32 v1, s112186; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v112187; GFX12-CU-NEXT: v_mov_b32_e32 v0, s012188; GFX12-CU-NEXT: s_wait_dscnt 0x012189; GFX12-CU-NEXT: ds_store_b32 v0, v112190; GFX12-CU-NEXT: s_endpgm12191;12192; GFX1250-LABEL: local_workgroup_one_as_acquire_ret_atomicrmw:12193; GFX1250: ; %bb.0: ; %entry12194; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 112195; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x012196; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x412197; GFX1250-NEXT: s_wait_kmcnt 0x012198; GFX1250-NEXT: v_mov_b32_e32 v0, s012199; GFX1250-NEXT: v_mov_b32_e32 v1, s112200; GFX1250-NEXT: ds_storexchg_rtn_b32 v1, v0, v112201; GFX1250-NEXT: v_mov_b32_e32 v0, s012202; GFX1250-NEXT: s_wait_dscnt 0x012203; GFX1250-NEXT: ds_store_b32 v0, v112204; GFX1250-NEXT: s_endpgm12205 ptr addrspace(3) %out, i32 %in) {12206entry:12207 %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup-one-as") acquire12208 store i32 %val, ptr addrspace(3) %out, align 412209 ret void12210}12211 12212define amdgpu_kernel void @local_workgroup_one_as_acq_rel_ret_atomicrmw(12213; GFX6-LABEL: local_workgroup_one_as_acq_rel_ret_atomicrmw:12214; GFX6: ; %bb.0: ; %entry12215; GFX6-NEXT: s_load_dword s4, s[8:9], 0x012216; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr412217; GFX6-NEXT: s_load_dword s5, s[8:9], 0x112218; GFX6-NEXT: s_mov_b32 m0, -112219; GFX6-NEXT: s_waitcnt lgkmcnt(0)12220; GFX6-NEXT: v_mov_b32_e32 v0, s412221; GFX6-NEXT: v_mov_b32_e32 v1, s512222; GFX6-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112223; GFX6-NEXT: s_mov_b32 m0, -112224; GFX6-NEXT: v_mov_b32_e32 v0, s412225; GFX6-NEXT: s_waitcnt lgkmcnt(0)12226; GFX6-NEXT: ds_write_b32 v0, v112227; GFX6-NEXT: s_endpgm12228;12229; GFX7-LABEL: local_workgroup_one_as_acq_rel_ret_atomicrmw:12230; GFX7: ; %bb.0: ; %entry12231; GFX7-NEXT: s_load_dword s4, s[8:9], 0x012232; GFX7-NEXT: s_load_dword s5, s[8:9], 0x112233; GFX7-NEXT: s_mov_b32 m0, -112234; GFX7-NEXT: s_waitcnt lgkmcnt(0)12235; GFX7-NEXT: v_mov_b32_e32 v0, s412236; GFX7-NEXT: v_mov_b32_e32 v1, s512237; GFX7-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112238; GFX7-NEXT: s_mov_b32 m0, -112239; GFX7-NEXT: v_mov_b32_e32 v0, s412240; GFX7-NEXT: s_waitcnt lgkmcnt(0)12241; GFX7-NEXT: ds_write_b32 v0, v112242; GFX7-NEXT: s_endpgm12243;12244; GFX10-WGP-LABEL: local_workgroup_one_as_acq_rel_ret_atomicrmw:12245; GFX10-WGP: ; %bb.0: ; %entry12246; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x012247; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x412248; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)12249; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s412250; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s512251; GFX10-WGP-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112252; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s412253; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)12254; GFX10-WGP-NEXT: ds_write_b32 v0, v112255; GFX10-WGP-NEXT: s_endpgm12256;12257; GFX10-CU-LABEL: local_workgroup_one_as_acq_rel_ret_atomicrmw:12258; GFX10-CU: ; %bb.0: ; %entry12259; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x012260; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x412261; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)12262; GFX10-CU-NEXT: v_mov_b32_e32 v0, s412263; GFX10-CU-NEXT: v_mov_b32_e32 v1, s512264; GFX10-CU-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112265; GFX10-CU-NEXT: v_mov_b32_e32 v0, s412266; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)12267; GFX10-CU-NEXT: ds_write_b32 v0, v112268; GFX10-CU-NEXT: s_endpgm12269;12270; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acq_rel_ret_atomicrmw:12271; SKIP-CACHE-INV: ; %bb.0: ; %entry12272; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x012273; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x112274; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -112275; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)12276; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s012277; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s112278; SKIP-CACHE-INV-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112279; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -112280; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s012281; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)12282; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v112283; SKIP-CACHE-INV-NEXT: s_endpgm12284;12285; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_acq_rel_ret_atomicrmw:12286; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry12287; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x012288; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x412289; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12290; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s412291; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s512292; GFX90A-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112293; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s412294; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12295; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v112296; GFX90A-NOTTGSPLIT-NEXT: s_endpgm12297;12298; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_acq_rel_ret_atomicrmw:12299; GFX90A-TGSPLIT: ; %bb.0: ; %entry12300; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x012301; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x412302; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12303; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s412304; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s512305; GFX90A-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112306; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s412307; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12308; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v112309; GFX90A-TGSPLIT-NEXT: s_endpgm12310;12311; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_acq_rel_ret_atomicrmw:12312; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry12313; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x012314; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x412315; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12316; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s012317; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s112318; GFX942-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112319; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s012320; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12321; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v112322; GFX942-NOTTGSPLIT-NEXT: s_endpgm12323;12324; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_acq_rel_ret_atomicrmw:12325; GFX942-TGSPLIT: ; %bb.0: ; %entry12326; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x012327; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x412328; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12329; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s012330; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s112331; GFX942-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112332; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s012333; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12334; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v112335; GFX942-TGSPLIT-NEXT: s_endpgm12336;12337; GFX11-WGP-LABEL: local_workgroup_one_as_acq_rel_ret_atomicrmw:12338; GFX11-WGP: ; %bb.0: ; %entry12339; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x012340; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x412341; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)12342; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s012343; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s112344; GFX11-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v112345; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s012346; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)12347; GFX11-WGP-NEXT: ds_store_b32 v0, v112348; GFX11-WGP-NEXT: s_endpgm12349;12350; GFX11-CU-LABEL: local_workgroup_one_as_acq_rel_ret_atomicrmw:12351; GFX11-CU: ; %bb.0: ; %entry12352; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x012353; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x412354; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)12355; GFX11-CU-NEXT: v_mov_b32_e32 v0, s012356; GFX11-CU-NEXT: v_mov_b32_e32 v1, s112357; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v112358; GFX11-CU-NEXT: v_mov_b32_e32 v0, s012359; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)12360; GFX11-CU-NEXT: ds_store_b32 v0, v112361; GFX11-CU-NEXT: s_endpgm12362;12363; GFX12-WGP-LABEL: local_workgroup_one_as_acq_rel_ret_atomicrmw:12364; GFX12-WGP: ; %bb.0: ; %entry12365; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x012366; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x412367; GFX12-WGP-NEXT: s_wait_kmcnt 0x012368; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s012369; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s112370; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v112371; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s012372; GFX12-WGP-NEXT: s_wait_dscnt 0x012373; GFX12-WGP-NEXT: ds_store_b32 v0, v112374; GFX12-WGP-NEXT: s_endpgm12375;12376; GFX12-CU-LABEL: local_workgroup_one_as_acq_rel_ret_atomicrmw:12377; GFX12-CU: ; %bb.0: ; %entry12378; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x012379; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x412380; GFX12-CU-NEXT: s_wait_kmcnt 0x012381; GFX12-CU-NEXT: v_mov_b32_e32 v0, s012382; GFX12-CU-NEXT: v_mov_b32_e32 v1, s112383; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v112384; GFX12-CU-NEXT: v_mov_b32_e32 v0, s012385; GFX12-CU-NEXT: s_wait_dscnt 0x012386; GFX12-CU-NEXT: ds_store_b32 v0, v112387; GFX12-CU-NEXT: s_endpgm12388;12389; GFX1250-LABEL: local_workgroup_one_as_acq_rel_ret_atomicrmw:12390; GFX1250: ; %bb.0: ; %entry12391; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 112392; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x012393; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x412394; GFX1250-NEXT: s_wait_kmcnt 0x012395; GFX1250-NEXT: v_mov_b32_e32 v0, s012396; GFX1250-NEXT: v_mov_b32_e32 v1, s112397; GFX1250-NEXT: ds_storexchg_rtn_b32 v1, v0, v112398; GFX1250-NEXT: v_mov_b32_e32 v0, s012399; GFX1250-NEXT: s_wait_dscnt 0x012400; GFX1250-NEXT: ds_store_b32 v0, v112401; GFX1250-NEXT: s_endpgm12402 ptr addrspace(3) %out, i32 %in) {12403entry:12404 %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup-one-as") acq_rel12405 store i32 %val, ptr addrspace(3) %out, align 412406 ret void12407}12408 12409define amdgpu_kernel void @local_workgroup_one_as_seq_cst_ret_atomicrmw(12410; GFX6-LABEL: local_workgroup_one_as_seq_cst_ret_atomicrmw:12411; GFX6: ; %bb.0: ; %entry12412; GFX6-NEXT: s_load_dword s4, s[8:9], 0x012413; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr412414; GFX6-NEXT: s_load_dword s5, s[8:9], 0x112415; GFX6-NEXT: s_mov_b32 m0, -112416; GFX6-NEXT: s_waitcnt lgkmcnt(0)12417; GFX6-NEXT: v_mov_b32_e32 v0, s412418; GFX6-NEXT: v_mov_b32_e32 v1, s512419; GFX6-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112420; GFX6-NEXT: s_mov_b32 m0, -112421; GFX6-NEXT: v_mov_b32_e32 v0, s412422; GFX6-NEXT: s_waitcnt lgkmcnt(0)12423; GFX6-NEXT: ds_write_b32 v0, v112424; GFX6-NEXT: s_endpgm12425;12426; GFX7-LABEL: local_workgroup_one_as_seq_cst_ret_atomicrmw:12427; GFX7: ; %bb.0: ; %entry12428; GFX7-NEXT: s_load_dword s4, s[8:9], 0x012429; GFX7-NEXT: s_load_dword s5, s[8:9], 0x112430; GFX7-NEXT: s_mov_b32 m0, -112431; GFX7-NEXT: s_waitcnt lgkmcnt(0)12432; GFX7-NEXT: v_mov_b32_e32 v0, s412433; GFX7-NEXT: v_mov_b32_e32 v1, s512434; GFX7-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112435; GFX7-NEXT: s_mov_b32 m0, -112436; GFX7-NEXT: v_mov_b32_e32 v0, s412437; GFX7-NEXT: s_waitcnt lgkmcnt(0)12438; GFX7-NEXT: ds_write_b32 v0, v112439; GFX7-NEXT: s_endpgm12440;12441; GFX10-WGP-LABEL: local_workgroup_one_as_seq_cst_ret_atomicrmw:12442; GFX10-WGP: ; %bb.0: ; %entry12443; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x012444; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x412445; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)12446; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s412447; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s512448; GFX10-WGP-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112449; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s412450; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)12451; GFX10-WGP-NEXT: ds_write_b32 v0, v112452; GFX10-WGP-NEXT: s_endpgm12453;12454; GFX10-CU-LABEL: local_workgroup_one_as_seq_cst_ret_atomicrmw:12455; GFX10-CU: ; %bb.0: ; %entry12456; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x012457; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x412458; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)12459; GFX10-CU-NEXT: v_mov_b32_e32 v0, s412460; GFX10-CU-NEXT: v_mov_b32_e32 v1, s512461; GFX10-CU-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112462; GFX10-CU-NEXT: v_mov_b32_e32 v0, s412463; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)12464; GFX10-CU-NEXT: ds_write_b32 v0, v112465; GFX10-CU-NEXT: s_endpgm12466;12467; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_seq_cst_ret_atomicrmw:12468; SKIP-CACHE-INV: ; %bb.0: ; %entry12469; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x012470; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x112471; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -112472; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)12473; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s012474; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s112475; SKIP-CACHE-INV-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112476; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -112477; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s012478; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)12479; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v112480; SKIP-CACHE-INV-NEXT: s_endpgm12481;12482; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_seq_cst_ret_atomicrmw:12483; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry12484; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x012485; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x412486; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12487; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s412488; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s512489; GFX90A-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112490; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s412491; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12492; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v112493; GFX90A-NOTTGSPLIT-NEXT: s_endpgm12494;12495; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_seq_cst_ret_atomicrmw:12496; GFX90A-TGSPLIT: ; %bb.0: ; %entry12497; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x012498; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x412499; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12500; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s412501; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s512502; GFX90A-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112503; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s412504; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12505; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v112506; GFX90A-TGSPLIT-NEXT: s_endpgm12507;12508; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_seq_cst_ret_atomicrmw:12509; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry12510; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x012511; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x412512; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12513; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s012514; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s112515; GFX942-NOTTGSPLIT-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112516; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s012517; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12518; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v112519; GFX942-NOTTGSPLIT-NEXT: s_endpgm12520;12521; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_seq_cst_ret_atomicrmw:12522; GFX942-TGSPLIT: ; %bb.0: ; %entry12523; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x012524; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x412525; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12526; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s012527; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s112528; GFX942-TGSPLIT-NEXT: ds_wrxchg_rtn_b32 v1, v0, v112529; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s012530; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12531; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v112532; GFX942-TGSPLIT-NEXT: s_endpgm12533;12534; GFX11-WGP-LABEL: local_workgroup_one_as_seq_cst_ret_atomicrmw:12535; GFX11-WGP: ; %bb.0: ; %entry12536; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x012537; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x412538; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)12539; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s012540; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s112541; GFX11-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v112542; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s012543; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)12544; GFX11-WGP-NEXT: ds_store_b32 v0, v112545; GFX11-WGP-NEXT: s_endpgm12546;12547; GFX11-CU-LABEL: local_workgroup_one_as_seq_cst_ret_atomicrmw:12548; GFX11-CU: ; %bb.0: ; %entry12549; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x012550; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x412551; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)12552; GFX11-CU-NEXT: v_mov_b32_e32 v0, s012553; GFX11-CU-NEXT: v_mov_b32_e32 v1, s112554; GFX11-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v112555; GFX11-CU-NEXT: v_mov_b32_e32 v0, s012556; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)12557; GFX11-CU-NEXT: ds_store_b32 v0, v112558; GFX11-CU-NEXT: s_endpgm12559;12560; GFX12-WGP-LABEL: local_workgroup_one_as_seq_cst_ret_atomicrmw:12561; GFX12-WGP: ; %bb.0: ; %entry12562; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x012563; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x412564; GFX12-WGP-NEXT: s_wait_kmcnt 0x012565; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s012566; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s112567; GFX12-WGP-NEXT: ds_storexchg_rtn_b32 v1, v0, v112568; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s012569; GFX12-WGP-NEXT: s_wait_dscnt 0x012570; GFX12-WGP-NEXT: ds_store_b32 v0, v112571; GFX12-WGP-NEXT: s_endpgm12572;12573; GFX12-CU-LABEL: local_workgroup_one_as_seq_cst_ret_atomicrmw:12574; GFX12-CU: ; %bb.0: ; %entry12575; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x012576; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x412577; GFX12-CU-NEXT: s_wait_kmcnt 0x012578; GFX12-CU-NEXT: v_mov_b32_e32 v0, s012579; GFX12-CU-NEXT: v_mov_b32_e32 v1, s112580; GFX12-CU-NEXT: ds_storexchg_rtn_b32 v1, v0, v112581; GFX12-CU-NEXT: v_mov_b32_e32 v0, s012582; GFX12-CU-NEXT: s_wait_dscnt 0x012583; GFX12-CU-NEXT: ds_store_b32 v0, v112584; GFX12-CU-NEXT: s_endpgm12585;12586; GFX1250-LABEL: local_workgroup_one_as_seq_cst_ret_atomicrmw:12587; GFX1250: ; %bb.0: ; %entry12588; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 112589; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x012590; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x412591; GFX1250-NEXT: s_wait_kmcnt 0x012592; GFX1250-NEXT: v_mov_b32_e32 v0, s012593; GFX1250-NEXT: v_mov_b32_e32 v1, s112594; GFX1250-NEXT: ds_storexchg_rtn_b32 v1, v0, v112595; GFX1250-NEXT: v_mov_b32_e32 v0, s012596; GFX1250-NEXT: s_wait_dscnt 0x012597; GFX1250-NEXT: ds_store_b32 v0, v112598; GFX1250-NEXT: s_endpgm12599 ptr addrspace(3) %out, i32 %in) {12600entry:12601 %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup-one-as") seq_cst12602 store i32 %val, ptr addrspace(3) %out, align 412603 ret void12604}12605 12606define amdgpu_kernel void @local_workgroup_one_as_monotonic_monotonic_cmpxchg(12607; GFX6-LABEL: local_workgroup_one_as_monotonic_monotonic_cmpxchg:12608; GFX6: ; %bb.0: ; %entry12609; GFX6-NEXT: s_load_dword s6, s[8:9], 0x012610; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr612611; GFX6-NEXT: s_load_dword s4, s[8:9], 0x112612; GFX6-NEXT: s_load_dword s5, s[8:9], 0x212613; GFX6-NEXT: s_mov_b32 m0, -112614; GFX6-NEXT: s_waitcnt lgkmcnt(0)12615; GFX6-NEXT: v_mov_b32_e32 v0, s612616; GFX6-NEXT: v_mov_b32_e32 v1, s512617; GFX6-NEXT: v_mov_b32_e32 v2, s412618; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1612619; GFX6-NEXT: s_endpgm12620;12621; GFX7-LABEL: local_workgroup_one_as_monotonic_monotonic_cmpxchg:12622; GFX7: ; %bb.0: ; %entry12623; GFX7-NEXT: s_load_dword s6, s[8:9], 0x012624; GFX7-NEXT: s_load_dword s4, s[8:9], 0x112625; GFX7-NEXT: s_load_dword s5, s[8:9], 0x212626; GFX7-NEXT: s_mov_b32 m0, -112627; GFX7-NEXT: s_waitcnt lgkmcnt(0)12628; GFX7-NEXT: v_mov_b32_e32 v0, s612629; GFX7-NEXT: v_mov_b32_e32 v1, s512630; GFX7-NEXT: v_mov_b32_e32 v2, s412631; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1612632; GFX7-NEXT: s_endpgm12633;12634; GFX10-WGP-LABEL: local_workgroup_one_as_monotonic_monotonic_cmpxchg:12635; GFX10-WGP: ; %bb.0: ; %entry12636; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x012637; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x412638; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x812639; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)12640; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s612641; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s512642; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s412643; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1612644; GFX10-WGP-NEXT: s_endpgm12645;12646; GFX10-CU-LABEL: local_workgroup_one_as_monotonic_monotonic_cmpxchg:12647; GFX10-CU: ; %bb.0: ; %entry12648; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x012649; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x412650; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x812651; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)12652; GFX10-CU-NEXT: v_mov_b32_e32 v0, s612653; GFX10-CU-NEXT: v_mov_b32_e32 v1, s512654; GFX10-CU-NEXT: v_mov_b32_e32 v2, s412655; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1612656; GFX10-CU-NEXT: s_endpgm12657;12658; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_monotonic_monotonic_cmpxchg:12659; SKIP-CACHE-INV: ; %bb.0: ; %entry12660; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x012661; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x112662; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x212663; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -112664; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)12665; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s212666; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s112667; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s012668; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1612669; SKIP-CACHE-INV-NEXT: s_endpgm12670;12671; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_monotonic_monotonic_cmpxchg:12672; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry12673; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x012674; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x412675; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x812676; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12677; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s612678; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s512679; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s412680; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1612681; GFX90A-NOTTGSPLIT-NEXT: s_endpgm12682;12683; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_monotonic_monotonic_cmpxchg:12684; GFX90A-TGSPLIT: ; %bb.0: ; %entry12685; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x012686; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x412687; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x812688; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12689; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s612690; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s512691; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s412692; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1612693; GFX90A-TGSPLIT-NEXT: s_endpgm12694;12695; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_monotonic_monotonic_cmpxchg:12696; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry12697; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x012698; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x412699; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x812700; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12701; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s212702; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s112703; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s012704; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1612705; GFX942-NOTTGSPLIT-NEXT: s_endpgm12706;12707; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_monotonic_monotonic_cmpxchg:12708; GFX942-TGSPLIT: ; %bb.0: ; %entry12709; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x012710; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x412711; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x812712; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12713; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s212714; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s112715; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s012716; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1612717; GFX942-TGSPLIT-NEXT: s_endpgm12718;12719; GFX11-WGP-LABEL: local_workgroup_one_as_monotonic_monotonic_cmpxchg:12720; GFX11-WGP: ; %bb.0: ; %entry12721; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x012722; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x412723; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x812724; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)12725; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s212726; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s112727; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s012728; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1612729; GFX11-WGP-NEXT: s_endpgm12730;12731; GFX11-CU-LABEL: local_workgroup_one_as_monotonic_monotonic_cmpxchg:12732; GFX11-CU: ; %bb.0: ; %entry12733; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x012734; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x412735; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x812736; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)12737; GFX11-CU-NEXT: v_mov_b32_e32 v0, s212738; GFX11-CU-NEXT: v_mov_b32_e32 v1, s112739; GFX11-CU-NEXT: v_mov_b32_e32 v2, s012740; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1612741; GFX11-CU-NEXT: s_endpgm12742;12743; GFX12-WGP-LABEL: local_workgroup_one_as_monotonic_monotonic_cmpxchg:12744; GFX12-WGP: ; %bb.0: ; %entry12745; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x012746; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x412747; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x812748; GFX12-WGP-NEXT: s_wait_kmcnt 0x012749; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s212750; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s112751; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s012752; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1612753; GFX12-WGP-NEXT: s_endpgm12754;12755; GFX12-CU-LABEL: local_workgroup_one_as_monotonic_monotonic_cmpxchg:12756; GFX12-CU: ; %bb.0: ; %entry12757; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x012758; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x412759; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x812760; GFX12-CU-NEXT: s_wait_kmcnt 0x012761; GFX12-CU-NEXT: v_mov_b32_e32 v0, s212762; GFX12-CU-NEXT: v_mov_b32_e32 v1, s112763; GFX12-CU-NEXT: v_mov_b32_e32 v2, s012764; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1612765; GFX12-CU-NEXT: s_endpgm12766;12767; GFX1250-LABEL: local_workgroup_one_as_monotonic_monotonic_cmpxchg:12768; GFX1250: ; %bb.0: ; %entry12769; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 112770; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x012771; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x412772; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x812773; GFX1250-NEXT: s_wait_kmcnt 0x012774; GFX1250-NEXT: v_mov_b32_e32 v0, s212775; GFX1250-NEXT: v_mov_b32_e32 v1, s112776; GFX1250-NEXT: v_mov_b32_e32 v2, s012777; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1612778; GFX1250-NEXT: s_endpgm12779 ptr addrspace(3) %out, i32 %in, i32 %old) {12780entry:12781 %gep = getelementptr i32, ptr addrspace(3) %out, i32 412782 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic monotonic12783 ret void12784}12785 12786define amdgpu_kernel void @local_workgroup_one_as_acquire_monotonic_cmpxchg(12787; GFX6-LABEL: local_workgroup_one_as_acquire_monotonic_cmpxchg:12788; GFX6: ; %bb.0: ; %entry12789; GFX6-NEXT: s_load_dword s6, s[8:9], 0x012790; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr612791; GFX6-NEXT: s_load_dword s4, s[8:9], 0x112792; GFX6-NEXT: s_load_dword s5, s[8:9], 0x212793; GFX6-NEXT: s_mov_b32 m0, -112794; GFX6-NEXT: s_waitcnt lgkmcnt(0)12795; GFX6-NEXT: v_mov_b32_e32 v0, s612796; GFX6-NEXT: v_mov_b32_e32 v1, s512797; GFX6-NEXT: v_mov_b32_e32 v2, s412798; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1612799; GFX6-NEXT: s_endpgm12800;12801; GFX7-LABEL: local_workgroup_one_as_acquire_monotonic_cmpxchg:12802; GFX7: ; %bb.0: ; %entry12803; GFX7-NEXT: s_load_dword s6, s[8:9], 0x012804; GFX7-NEXT: s_load_dword s4, s[8:9], 0x112805; GFX7-NEXT: s_load_dword s5, s[8:9], 0x212806; GFX7-NEXT: s_mov_b32 m0, -112807; GFX7-NEXT: s_waitcnt lgkmcnt(0)12808; GFX7-NEXT: v_mov_b32_e32 v0, s612809; GFX7-NEXT: v_mov_b32_e32 v1, s512810; GFX7-NEXT: v_mov_b32_e32 v2, s412811; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1612812; GFX7-NEXT: s_endpgm12813;12814; GFX10-WGP-LABEL: local_workgroup_one_as_acquire_monotonic_cmpxchg:12815; GFX10-WGP: ; %bb.0: ; %entry12816; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x012817; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x412818; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x812819; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)12820; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s612821; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s512822; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s412823; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1612824; GFX10-WGP-NEXT: s_endpgm12825;12826; GFX10-CU-LABEL: local_workgroup_one_as_acquire_monotonic_cmpxchg:12827; GFX10-CU: ; %bb.0: ; %entry12828; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x012829; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x412830; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x812831; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)12832; GFX10-CU-NEXT: v_mov_b32_e32 v0, s612833; GFX10-CU-NEXT: v_mov_b32_e32 v1, s512834; GFX10-CU-NEXT: v_mov_b32_e32 v2, s412835; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1612836; GFX10-CU-NEXT: s_endpgm12837;12838; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acquire_monotonic_cmpxchg:12839; SKIP-CACHE-INV: ; %bb.0: ; %entry12840; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x012841; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x112842; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x212843; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -112844; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)12845; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s212846; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s112847; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s012848; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1612849; SKIP-CACHE-INV-NEXT: s_endpgm12850;12851; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_acquire_monotonic_cmpxchg:12852; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry12853; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x012854; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x412855; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x812856; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12857; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s612858; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s512859; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s412860; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1612861; GFX90A-NOTTGSPLIT-NEXT: s_endpgm12862;12863; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_acquire_monotonic_cmpxchg:12864; GFX90A-TGSPLIT: ; %bb.0: ; %entry12865; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x012866; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x412867; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x812868; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12869; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s612870; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s512871; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s412872; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1612873; GFX90A-TGSPLIT-NEXT: s_endpgm12874;12875; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_acquire_monotonic_cmpxchg:12876; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry12877; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x012878; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x412879; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x812880; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12881; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s212882; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s112883; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s012884; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1612885; GFX942-NOTTGSPLIT-NEXT: s_endpgm12886;12887; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_acquire_monotonic_cmpxchg:12888; GFX942-TGSPLIT: ; %bb.0: ; %entry12889; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x012890; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x412891; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x812892; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)12893; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s212894; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s112895; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s012896; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1612897; GFX942-TGSPLIT-NEXT: s_endpgm12898;12899; GFX11-WGP-LABEL: local_workgroup_one_as_acquire_monotonic_cmpxchg:12900; GFX11-WGP: ; %bb.0: ; %entry12901; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x012902; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x412903; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x812904; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)12905; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s212906; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s112907; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s012908; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1612909; GFX11-WGP-NEXT: s_endpgm12910;12911; GFX11-CU-LABEL: local_workgroup_one_as_acquire_monotonic_cmpxchg:12912; GFX11-CU: ; %bb.0: ; %entry12913; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x012914; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x412915; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x812916; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)12917; GFX11-CU-NEXT: v_mov_b32_e32 v0, s212918; GFX11-CU-NEXT: v_mov_b32_e32 v1, s112919; GFX11-CU-NEXT: v_mov_b32_e32 v2, s012920; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1612921; GFX11-CU-NEXT: s_endpgm12922;12923; GFX12-WGP-LABEL: local_workgroup_one_as_acquire_monotonic_cmpxchg:12924; GFX12-WGP: ; %bb.0: ; %entry12925; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x012926; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x412927; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x812928; GFX12-WGP-NEXT: s_wait_kmcnt 0x012929; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s212930; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s112931; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s012932; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1612933; GFX12-WGP-NEXT: s_endpgm12934;12935; GFX12-CU-LABEL: local_workgroup_one_as_acquire_monotonic_cmpxchg:12936; GFX12-CU: ; %bb.0: ; %entry12937; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x012938; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x412939; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x812940; GFX12-CU-NEXT: s_wait_kmcnt 0x012941; GFX12-CU-NEXT: v_mov_b32_e32 v0, s212942; GFX12-CU-NEXT: v_mov_b32_e32 v1, s112943; GFX12-CU-NEXT: v_mov_b32_e32 v2, s012944; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1612945; GFX12-CU-NEXT: s_endpgm12946;12947; GFX1250-LABEL: local_workgroup_one_as_acquire_monotonic_cmpxchg:12948; GFX1250: ; %bb.0: ; %entry12949; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 112950; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x012951; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x412952; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x812953; GFX1250-NEXT: s_wait_kmcnt 0x012954; GFX1250-NEXT: v_mov_b32_e32 v0, s212955; GFX1250-NEXT: v_mov_b32_e32 v1, s112956; GFX1250-NEXT: v_mov_b32_e32 v2, s012957; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1612958; GFX1250-NEXT: s_endpgm12959 ptr addrspace(3) %out, i32 %in, i32 %old) {12960entry:12961 %gep = getelementptr i32, ptr addrspace(3) %out, i32 412962 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic12963 ret void12964}12965 12966define amdgpu_kernel void @local_workgroup_one_as_release_monotonic_cmpxchg(12967; GFX6-LABEL: local_workgroup_one_as_release_monotonic_cmpxchg:12968; GFX6: ; %bb.0: ; %entry12969; GFX6-NEXT: s_load_dword s6, s[8:9], 0x012970; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr612971; GFX6-NEXT: s_load_dword s4, s[8:9], 0x112972; GFX6-NEXT: s_load_dword s5, s[8:9], 0x212973; GFX6-NEXT: s_mov_b32 m0, -112974; GFX6-NEXT: s_waitcnt lgkmcnt(0)12975; GFX6-NEXT: v_mov_b32_e32 v0, s612976; GFX6-NEXT: v_mov_b32_e32 v1, s512977; GFX6-NEXT: v_mov_b32_e32 v2, s412978; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1612979; GFX6-NEXT: s_endpgm12980;12981; GFX7-LABEL: local_workgroup_one_as_release_monotonic_cmpxchg:12982; GFX7: ; %bb.0: ; %entry12983; GFX7-NEXT: s_load_dword s6, s[8:9], 0x012984; GFX7-NEXT: s_load_dword s4, s[8:9], 0x112985; GFX7-NEXT: s_load_dword s5, s[8:9], 0x212986; GFX7-NEXT: s_mov_b32 m0, -112987; GFX7-NEXT: s_waitcnt lgkmcnt(0)12988; GFX7-NEXT: v_mov_b32_e32 v0, s612989; GFX7-NEXT: v_mov_b32_e32 v1, s512990; GFX7-NEXT: v_mov_b32_e32 v2, s412991; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1612992; GFX7-NEXT: s_endpgm12993;12994; GFX10-WGP-LABEL: local_workgroup_one_as_release_monotonic_cmpxchg:12995; GFX10-WGP: ; %bb.0: ; %entry12996; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x012997; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x412998; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x812999; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)13000; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s613001; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s513002; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s413003; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613004; GFX10-WGP-NEXT: s_endpgm13005;13006; GFX10-CU-LABEL: local_workgroup_one_as_release_monotonic_cmpxchg:13007; GFX10-CU: ; %bb.0: ; %entry13008; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x013009; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x413010; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x813011; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)13012; GFX10-CU-NEXT: v_mov_b32_e32 v0, s613013; GFX10-CU-NEXT: v_mov_b32_e32 v1, s513014; GFX10-CU-NEXT: v_mov_b32_e32 v2, s413015; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613016; GFX10-CU-NEXT: s_endpgm13017;13018; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_release_monotonic_cmpxchg:13019; SKIP-CACHE-INV: ; %bb.0: ; %entry13020; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x013021; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x113022; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x213023; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -113024; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)13025; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s213026; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s113027; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s013028; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613029; SKIP-CACHE-INV-NEXT: s_endpgm13030;13031; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_release_monotonic_cmpxchg:13032; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry13033; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x013034; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x413035; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x813036; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)13037; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s613038; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s513039; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s413040; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613041; GFX90A-NOTTGSPLIT-NEXT: s_endpgm13042;13043; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_release_monotonic_cmpxchg:13044; GFX90A-TGSPLIT: ; %bb.0: ; %entry13045; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x013046; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x413047; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x813048; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)13049; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s613050; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s513051; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s413052; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613053; GFX90A-TGSPLIT-NEXT: s_endpgm13054;13055; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_release_monotonic_cmpxchg:13056; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry13057; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x013058; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x413059; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x813060; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)13061; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s213062; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s113063; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s013064; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613065; GFX942-NOTTGSPLIT-NEXT: s_endpgm13066;13067; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_release_monotonic_cmpxchg:13068; GFX942-TGSPLIT: ; %bb.0: ; %entry13069; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x013070; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x413071; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x813072; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)13073; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s213074; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s113075; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s013076; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613077; GFX942-TGSPLIT-NEXT: s_endpgm13078;13079; GFX11-WGP-LABEL: local_workgroup_one_as_release_monotonic_cmpxchg:13080; GFX11-WGP: ; %bb.0: ; %entry13081; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x013082; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x413083; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x813084; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)13085; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s213086; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s113087; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s013088; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613089; GFX11-WGP-NEXT: s_endpgm13090;13091; GFX11-CU-LABEL: local_workgroup_one_as_release_monotonic_cmpxchg:13092; GFX11-CU: ; %bb.0: ; %entry13093; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x013094; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x413095; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x813096; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)13097; GFX11-CU-NEXT: v_mov_b32_e32 v0, s213098; GFX11-CU-NEXT: v_mov_b32_e32 v1, s113099; GFX11-CU-NEXT: v_mov_b32_e32 v2, s013100; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613101; GFX11-CU-NEXT: s_endpgm13102;13103; GFX12-WGP-LABEL: local_workgroup_one_as_release_monotonic_cmpxchg:13104; GFX12-WGP: ; %bb.0: ; %entry13105; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x013106; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x413107; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x813108; GFX12-WGP-NEXT: s_wait_kmcnt 0x013109; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s213110; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s113111; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s013112; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613113; GFX12-WGP-NEXT: s_endpgm13114;13115; GFX12-CU-LABEL: local_workgroup_one_as_release_monotonic_cmpxchg:13116; GFX12-CU: ; %bb.0: ; %entry13117; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x013118; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x413119; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x813120; GFX12-CU-NEXT: s_wait_kmcnt 0x013121; GFX12-CU-NEXT: v_mov_b32_e32 v0, s213122; GFX12-CU-NEXT: v_mov_b32_e32 v1, s113123; GFX12-CU-NEXT: v_mov_b32_e32 v2, s013124; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613125; GFX12-CU-NEXT: s_endpgm13126;13127; GFX1250-LABEL: local_workgroup_one_as_release_monotonic_cmpxchg:13128; GFX1250: ; %bb.0: ; %entry13129; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 113130; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x013131; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x413132; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x813133; GFX1250-NEXT: s_wait_kmcnt 0x013134; GFX1250-NEXT: v_mov_b32_e32 v0, s213135; GFX1250-NEXT: v_mov_b32_e32 v1, s113136; GFX1250-NEXT: v_mov_b32_e32 v2, s013137; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613138; GFX1250-NEXT: s_endpgm13139 ptr addrspace(3) %out, i32 %in, i32 %old) {13140entry:13141 %gep = getelementptr i32, ptr addrspace(3) %out, i32 413142 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release monotonic13143 ret void13144}13145 13146define amdgpu_kernel void @local_workgroup_one_as_acq_rel_monotonic_cmpxchg(13147; GFX6-LABEL: local_workgroup_one_as_acq_rel_monotonic_cmpxchg:13148; GFX6: ; %bb.0: ; %entry13149; GFX6-NEXT: s_load_dword s6, s[8:9], 0x013150; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr613151; GFX6-NEXT: s_load_dword s4, s[8:9], 0x113152; GFX6-NEXT: s_load_dword s5, s[8:9], 0x213153; GFX6-NEXT: s_mov_b32 m0, -113154; GFX6-NEXT: s_waitcnt lgkmcnt(0)13155; GFX6-NEXT: v_mov_b32_e32 v0, s613156; GFX6-NEXT: v_mov_b32_e32 v1, s513157; GFX6-NEXT: v_mov_b32_e32 v2, s413158; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613159; GFX6-NEXT: s_endpgm13160;13161; GFX7-LABEL: local_workgroup_one_as_acq_rel_monotonic_cmpxchg:13162; GFX7: ; %bb.0: ; %entry13163; GFX7-NEXT: s_load_dword s6, s[8:9], 0x013164; GFX7-NEXT: s_load_dword s4, s[8:9], 0x113165; GFX7-NEXT: s_load_dword s5, s[8:9], 0x213166; GFX7-NEXT: s_mov_b32 m0, -113167; GFX7-NEXT: s_waitcnt lgkmcnt(0)13168; GFX7-NEXT: v_mov_b32_e32 v0, s613169; GFX7-NEXT: v_mov_b32_e32 v1, s513170; GFX7-NEXT: v_mov_b32_e32 v2, s413171; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613172; GFX7-NEXT: s_endpgm13173;13174; GFX10-WGP-LABEL: local_workgroup_one_as_acq_rel_monotonic_cmpxchg:13175; GFX10-WGP: ; %bb.0: ; %entry13176; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x013177; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x413178; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x813179; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)13180; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s613181; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s513182; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s413183; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613184; GFX10-WGP-NEXT: s_endpgm13185;13186; GFX10-CU-LABEL: local_workgroup_one_as_acq_rel_monotonic_cmpxchg:13187; GFX10-CU: ; %bb.0: ; %entry13188; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x013189; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x413190; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x813191; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)13192; GFX10-CU-NEXT: v_mov_b32_e32 v0, s613193; GFX10-CU-NEXT: v_mov_b32_e32 v1, s513194; GFX10-CU-NEXT: v_mov_b32_e32 v2, s413195; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613196; GFX10-CU-NEXT: s_endpgm13197;13198; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acq_rel_monotonic_cmpxchg:13199; SKIP-CACHE-INV: ; %bb.0: ; %entry13200; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x013201; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x113202; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x213203; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -113204; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)13205; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s213206; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s113207; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s013208; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613209; SKIP-CACHE-INV-NEXT: s_endpgm13210;13211; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_acq_rel_monotonic_cmpxchg:13212; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry13213; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x013214; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x413215; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x813216; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)13217; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s613218; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s513219; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s413220; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613221; GFX90A-NOTTGSPLIT-NEXT: s_endpgm13222;13223; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_acq_rel_monotonic_cmpxchg:13224; GFX90A-TGSPLIT: ; %bb.0: ; %entry13225; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x013226; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x413227; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x813228; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)13229; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s613230; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s513231; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s413232; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613233; GFX90A-TGSPLIT-NEXT: s_endpgm13234;13235; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_acq_rel_monotonic_cmpxchg:13236; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry13237; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x013238; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x413239; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x813240; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)13241; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s213242; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s113243; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s013244; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613245; GFX942-NOTTGSPLIT-NEXT: s_endpgm13246;13247; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_acq_rel_monotonic_cmpxchg:13248; GFX942-TGSPLIT: ; %bb.0: ; %entry13249; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x013250; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x413251; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x813252; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)13253; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s213254; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s113255; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s013256; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613257; GFX942-TGSPLIT-NEXT: s_endpgm13258;13259; GFX11-WGP-LABEL: local_workgroup_one_as_acq_rel_monotonic_cmpxchg:13260; GFX11-WGP: ; %bb.0: ; %entry13261; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x013262; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x413263; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x813264; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)13265; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s213266; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s113267; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s013268; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613269; GFX11-WGP-NEXT: s_endpgm13270;13271; GFX11-CU-LABEL: local_workgroup_one_as_acq_rel_monotonic_cmpxchg:13272; GFX11-CU: ; %bb.0: ; %entry13273; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x013274; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x413275; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x813276; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)13277; GFX11-CU-NEXT: v_mov_b32_e32 v0, s213278; GFX11-CU-NEXT: v_mov_b32_e32 v1, s113279; GFX11-CU-NEXT: v_mov_b32_e32 v2, s013280; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613281; GFX11-CU-NEXT: s_endpgm13282;13283; GFX12-WGP-LABEL: local_workgroup_one_as_acq_rel_monotonic_cmpxchg:13284; GFX12-WGP: ; %bb.0: ; %entry13285; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x013286; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x413287; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x813288; GFX12-WGP-NEXT: s_wait_kmcnt 0x013289; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s213290; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s113291; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s013292; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613293; GFX12-WGP-NEXT: s_endpgm13294;13295; GFX12-CU-LABEL: local_workgroup_one_as_acq_rel_monotonic_cmpxchg:13296; GFX12-CU: ; %bb.0: ; %entry13297; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x013298; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x413299; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x813300; GFX12-CU-NEXT: s_wait_kmcnt 0x013301; GFX12-CU-NEXT: v_mov_b32_e32 v0, s213302; GFX12-CU-NEXT: v_mov_b32_e32 v1, s113303; GFX12-CU-NEXT: v_mov_b32_e32 v2, s013304; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613305; GFX12-CU-NEXT: s_endpgm13306;13307; GFX1250-LABEL: local_workgroup_one_as_acq_rel_monotonic_cmpxchg:13308; GFX1250: ; %bb.0: ; %entry13309; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 113310; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x013311; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x413312; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x813313; GFX1250-NEXT: s_wait_kmcnt 0x013314; GFX1250-NEXT: v_mov_b32_e32 v0, s213315; GFX1250-NEXT: v_mov_b32_e32 v1, s113316; GFX1250-NEXT: v_mov_b32_e32 v2, s013317; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613318; GFX1250-NEXT: s_endpgm13319 ptr addrspace(3) %out, i32 %in, i32 %old) {13320entry:13321 %gep = getelementptr i32, ptr addrspace(3) %out, i32 413322 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic13323 ret void13324}13325 13326define amdgpu_kernel void @local_workgroup_one_as_seq_cst_monotonic_cmpxchg(13327; GFX6-LABEL: local_workgroup_one_as_seq_cst_monotonic_cmpxchg:13328; GFX6: ; %bb.0: ; %entry13329; GFX6-NEXT: s_load_dword s6, s[8:9], 0x013330; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr613331; GFX6-NEXT: s_load_dword s4, s[8:9], 0x113332; GFX6-NEXT: s_load_dword s5, s[8:9], 0x213333; GFX6-NEXT: s_mov_b32 m0, -113334; GFX6-NEXT: s_waitcnt lgkmcnt(0)13335; GFX6-NEXT: v_mov_b32_e32 v0, s613336; GFX6-NEXT: v_mov_b32_e32 v1, s513337; GFX6-NEXT: v_mov_b32_e32 v2, s413338; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613339; GFX6-NEXT: s_endpgm13340;13341; GFX7-LABEL: local_workgroup_one_as_seq_cst_monotonic_cmpxchg:13342; GFX7: ; %bb.0: ; %entry13343; GFX7-NEXT: s_load_dword s6, s[8:9], 0x013344; GFX7-NEXT: s_load_dword s4, s[8:9], 0x113345; GFX7-NEXT: s_load_dword s5, s[8:9], 0x213346; GFX7-NEXT: s_mov_b32 m0, -113347; GFX7-NEXT: s_waitcnt lgkmcnt(0)13348; GFX7-NEXT: v_mov_b32_e32 v0, s613349; GFX7-NEXT: v_mov_b32_e32 v1, s513350; GFX7-NEXT: v_mov_b32_e32 v2, s413351; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613352; GFX7-NEXT: s_endpgm13353;13354; GFX10-WGP-LABEL: local_workgroup_one_as_seq_cst_monotonic_cmpxchg:13355; GFX10-WGP: ; %bb.0: ; %entry13356; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x013357; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x413358; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x813359; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)13360; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s613361; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s513362; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s413363; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613364; GFX10-WGP-NEXT: s_endpgm13365;13366; GFX10-CU-LABEL: local_workgroup_one_as_seq_cst_monotonic_cmpxchg:13367; GFX10-CU: ; %bb.0: ; %entry13368; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x013369; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x413370; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x813371; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)13372; GFX10-CU-NEXT: v_mov_b32_e32 v0, s613373; GFX10-CU-NEXT: v_mov_b32_e32 v1, s513374; GFX10-CU-NEXT: v_mov_b32_e32 v2, s413375; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613376; GFX10-CU-NEXT: s_endpgm13377;13378; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_seq_cst_monotonic_cmpxchg:13379; SKIP-CACHE-INV: ; %bb.0: ; %entry13380; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x013381; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x113382; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x213383; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -113384; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)13385; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s213386; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s113387; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s013388; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613389; SKIP-CACHE-INV-NEXT: s_endpgm13390;13391; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_seq_cst_monotonic_cmpxchg:13392; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry13393; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x013394; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x413395; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x813396; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)13397; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s613398; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s513399; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s413400; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613401; GFX90A-NOTTGSPLIT-NEXT: s_endpgm13402;13403; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_seq_cst_monotonic_cmpxchg:13404; GFX90A-TGSPLIT: ; %bb.0: ; %entry13405; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x013406; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x413407; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x813408; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)13409; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s613410; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s513411; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s413412; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613413; GFX90A-TGSPLIT-NEXT: s_endpgm13414;13415; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_seq_cst_monotonic_cmpxchg:13416; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry13417; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x013418; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x413419; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x813420; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)13421; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s213422; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s113423; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s013424; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613425; GFX942-NOTTGSPLIT-NEXT: s_endpgm13426;13427; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_seq_cst_monotonic_cmpxchg:13428; GFX942-TGSPLIT: ; %bb.0: ; %entry13429; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x013430; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x413431; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x813432; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)13433; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s213434; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s113435; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s013436; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613437; GFX942-TGSPLIT-NEXT: s_endpgm13438;13439; GFX11-WGP-LABEL: local_workgroup_one_as_seq_cst_monotonic_cmpxchg:13440; GFX11-WGP: ; %bb.0: ; %entry13441; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x013442; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x413443; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x813444; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)13445; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s213446; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s113447; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s013448; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613449; GFX11-WGP-NEXT: s_endpgm13450;13451; GFX11-CU-LABEL: local_workgroup_one_as_seq_cst_monotonic_cmpxchg:13452; GFX11-CU: ; %bb.0: ; %entry13453; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x013454; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x413455; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x813456; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)13457; GFX11-CU-NEXT: v_mov_b32_e32 v0, s213458; GFX11-CU-NEXT: v_mov_b32_e32 v1, s113459; GFX11-CU-NEXT: v_mov_b32_e32 v2, s013460; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613461; GFX11-CU-NEXT: s_endpgm13462;13463; GFX12-WGP-LABEL: local_workgroup_one_as_seq_cst_monotonic_cmpxchg:13464; GFX12-WGP: ; %bb.0: ; %entry13465; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x013466; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x413467; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x813468; GFX12-WGP-NEXT: s_wait_kmcnt 0x013469; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s213470; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s113471; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s013472; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613473; GFX12-WGP-NEXT: s_endpgm13474;13475; GFX12-CU-LABEL: local_workgroup_one_as_seq_cst_monotonic_cmpxchg:13476; GFX12-CU: ; %bb.0: ; %entry13477; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x013478; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x413479; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x813480; GFX12-CU-NEXT: s_wait_kmcnt 0x013481; GFX12-CU-NEXT: v_mov_b32_e32 v0, s213482; GFX12-CU-NEXT: v_mov_b32_e32 v1, s113483; GFX12-CU-NEXT: v_mov_b32_e32 v2, s013484; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613485; GFX12-CU-NEXT: s_endpgm13486;13487; GFX1250-LABEL: local_workgroup_one_as_seq_cst_monotonic_cmpxchg:13488; GFX1250: ; %bb.0: ; %entry13489; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 113490; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x013491; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x413492; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x813493; GFX1250-NEXT: s_wait_kmcnt 0x013494; GFX1250-NEXT: v_mov_b32_e32 v0, s213495; GFX1250-NEXT: v_mov_b32_e32 v1, s113496; GFX1250-NEXT: v_mov_b32_e32 v2, s013497; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613498; GFX1250-NEXT: s_endpgm13499 ptr addrspace(3) %out, i32 %in, i32 %old) {13500entry:13501 %gep = getelementptr i32, ptr addrspace(3) %out, i32 413502 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic13503 ret void13504}13505 13506define amdgpu_kernel void @local_workgroup_one_as_monotonic_acquire_cmpxchg(13507; GFX6-LABEL: local_workgroup_one_as_monotonic_acquire_cmpxchg:13508; GFX6: ; %bb.0: ; %entry13509; GFX6-NEXT: s_load_dword s6, s[8:9], 0x013510; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr613511; GFX6-NEXT: s_load_dword s4, s[8:9], 0x113512; GFX6-NEXT: s_load_dword s5, s[8:9], 0x213513; GFX6-NEXT: s_mov_b32 m0, -113514; GFX6-NEXT: s_waitcnt lgkmcnt(0)13515; GFX6-NEXT: v_mov_b32_e32 v0, s613516; GFX6-NEXT: v_mov_b32_e32 v1, s513517; GFX6-NEXT: v_mov_b32_e32 v2, s413518; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613519; GFX6-NEXT: s_endpgm13520;13521; GFX7-LABEL: local_workgroup_one_as_monotonic_acquire_cmpxchg:13522; GFX7: ; %bb.0: ; %entry13523; GFX7-NEXT: s_load_dword s6, s[8:9], 0x013524; GFX7-NEXT: s_load_dword s4, s[8:9], 0x113525; GFX7-NEXT: s_load_dword s5, s[8:9], 0x213526; GFX7-NEXT: s_mov_b32 m0, -113527; GFX7-NEXT: s_waitcnt lgkmcnt(0)13528; GFX7-NEXT: v_mov_b32_e32 v0, s613529; GFX7-NEXT: v_mov_b32_e32 v1, s513530; GFX7-NEXT: v_mov_b32_e32 v2, s413531; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613532; GFX7-NEXT: s_endpgm13533;13534; GFX10-WGP-LABEL: local_workgroup_one_as_monotonic_acquire_cmpxchg:13535; GFX10-WGP: ; %bb.0: ; %entry13536; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x013537; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x413538; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x813539; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)13540; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s613541; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s513542; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s413543; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613544; GFX10-WGP-NEXT: s_endpgm13545;13546; GFX10-CU-LABEL: local_workgroup_one_as_monotonic_acquire_cmpxchg:13547; GFX10-CU: ; %bb.0: ; %entry13548; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x013549; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x413550; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x813551; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)13552; GFX10-CU-NEXT: v_mov_b32_e32 v0, s613553; GFX10-CU-NEXT: v_mov_b32_e32 v1, s513554; GFX10-CU-NEXT: v_mov_b32_e32 v2, s413555; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613556; GFX10-CU-NEXT: s_endpgm13557;13558; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_monotonic_acquire_cmpxchg:13559; SKIP-CACHE-INV: ; %bb.0: ; %entry13560; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x013561; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x113562; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x213563; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -113564; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)13565; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s213566; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s113567; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s013568; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613569; SKIP-CACHE-INV-NEXT: s_endpgm13570;13571; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_monotonic_acquire_cmpxchg:13572; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry13573; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x013574; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x413575; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x813576; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)13577; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s613578; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s513579; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s413580; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613581; GFX90A-NOTTGSPLIT-NEXT: s_endpgm13582;13583; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_monotonic_acquire_cmpxchg:13584; GFX90A-TGSPLIT: ; %bb.0: ; %entry13585; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x013586; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x413587; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x813588; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)13589; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s613590; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s513591; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s413592; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613593; GFX90A-TGSPLIT-NEXT: s_endpgm13594;13595; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_monotonic_acquire_cmpxchg:13596; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry13597; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x013598; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x413599; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x813600; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)13601; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s213602; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s113603; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s013604; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613605; GFX942-NOTTGSPLIT-NEXT: s_endpgm13606;13607; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_monotonic_acquire_cmpxchg:13608; GFX942-TGSPLIT: ; %bb.0: ; %entry13609; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x013610; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x413611; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x813612; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)13613; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s213614; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s113615; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s013616; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613617; GFX942-TGSPLIT-NEXT: s_endpgm13618;13619; GFX11-WGP-LABEL: local_workgroup_one_as_monotonic_acquire_cmpxchg:13620; GFX11-WGP: ; %bb.0: ; %entry13621; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x013622; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x413623; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x813624; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)13625; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s213626; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s113627; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s013628; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613629; GFX11-WGP-NEXT: s_endpgm13630;13631; GFX11-CU-LABEL: local_workgroup_one_as_monotonic_acquire_cmpxchg:13632; GFX11-CU: ; %bb.0: ; %entry13633; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x013634; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x413635; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x813636; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)13637; GFX11-CU-NEXT: v_mov_b32_e32 v0, s213638; GFX11-CU-NEXT: v_mov_b32_e32 v1, s113639; GFX11-CU-NEXT: v_mov_b32_e32 v2, s013640; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613641; GFX11-CU-NEXT: s_endpgm13642;13643; GFX12-WGP-LABEL: local_workgroup_one_as_monotonic_acquire_cmpxchg:13644; GFX12-WGP: ; %bb.0: ; %entry13645; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x013646; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x413647; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x813648; GFX12-WGP-NEXT: s_wait_kmcnt 0x013649; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s213650; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s113651; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s013652; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613653; GFX12-WGP-NEXT: s_endpgm13654;13655; GFX12-CU-LABEL: local_workgroup_one_as_monotonic_acquire_cmpxchg:13656; GFX12-CU: ; %bb.0: ; %entry13657; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x013658; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x413659; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x813660; GFX12-CU-NEXT: s_wait_kmcnt 0x013661; GFX12-CU-NEXT: v_mov_b32_e32 v0, s213662; GFX12-CU-NEXT: v_mov_b32_e32 v1, s113663; GFX12-CU-NEXT: v_mov_b32_e32 v2, s013664; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613665; GFX12-CU-NEXT: s_endpgm13666;13667; GFX1250-LABEL: local_workgroup_one_as_monotonic_acquire_cmpxchg:13668; GFX1250: ; %bb.0: ; %entry13669; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 113670; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x013671; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x413672; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x813673; GFX1250-NEXT: s_wait_kmcnt 0x013674; GFX1250-NEXT: v_mov_b32_e32 v0, s213675; GFX1250-NEXT: v_mov_b32_e32 v1, s113676; GFX1250-NEXT: v_mov_b32_e32 v2, s013677; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613678; GFX1250-NEXT: s_endpgm13679 ptr addrspace(3) %out, i32 %in, i32 %old) {13680entry:13681 %gep = getelementptr i32, ptr addrspace(3) %out, i32 413682 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic acquire13683 ret void13684}13685 13686define amdgpu_kernel void @local_workgroup_one_as_acquire_acquire_cmpxchg(13687; GFX6-LABEL: local_workgroup_one_as_acquire_acquire_cmpxchg:13688; GFX6: ; %bb.0: ; %entry13689; GFX6-NEXT: s_load_dword s6, s[8:9], 0x013690; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr613691; GFX6-NEXT: s_load_dword s4, s[8:9], 0x113692; GFX6-NEXT: s_load_dword s5, s[8:9], 0x213693; GFX6-NEXT: s_mov_b32 m0, -113694; GFX6-NEXT: s_waitcnt lgkmcnt(0)13695; GFX6-NEXT: v_mov_b32_e32 v0, s613696; GFX6-NEXT: v_mov_b32_e32 v1, s513697; GFX6-NEXT: v_mov_b32_e32 v2, s413698; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613699; GFX6-NEXT: s_endpgm13700;13701; GFX7-LABEL: local_workgroup_one_as_acquire_acquire_cmpxchg:13702; GFX7: ; %bb.0: ; %entry13703; GFX7-NEXT: s_load_dword s6, s[8:9], 0x013704; GFX7-NEXT: s_load_dword s4, s[8:9], 0x113705; GFX7-NEXT: s_load_dword s5, s[8:9], 0x213706; GFX7-NEXT: s_mov_b32 m0, -113707; GFX7-NEXT: s_waitcnt lgkmcnt(0)13708; GFX7-NEXT: v_mov_b32_e32 v0, s613709; GFX7-NEXT: v_mov_b32_e32 v1, s513710; GFX7-NEXT: v_mov_b32_e32 v2, s413711; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613712; GFX7-NEXT: s_endpgm13713;13714; GFX10-WGP-LABEL: local_workgroup_one_as_acquire_acquire_cmpxchg:13715; GFX10-WGP: ; %bb.0: ; %entry13716; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x013717; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x413718; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x813719; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)13720; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s613721; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s513722; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s413723; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613724; GFX10-WGP-NEXT: s_endpgm13725;13726; GFX10-CU-LABEL: local_workgroup_one_as_acquire_acquire_cmpxchg:13727; GFX10-CU: ; %bb.0: ; %entry13728; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x013729; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x413730; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x813731; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)13732; GFX10-CU-NEXT: v_mov_b32_e32 v0, s613733; GFX10-CU-NEXT: v_mov_b32_e32 v1, s513734; GFX10-CU-NEXT: v_mov_b32_e32 v2, s413735; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613736; GFX10-CU-NEXT: s_endpgm13737;13738; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acquire_acquire_cmpxchg:13739; SKIP-CACHE-INV: ; %bb.0: ; %entry13740; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x013741; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x113742; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x213743; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -113744; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)13745; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s213746; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s113747; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s013748; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613749; SKIP-CACHE-INV-NEXT: s_endpgm13750;13751; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_acquire_acquire_cmpxchg:13752; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry13753; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x013754; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x413755; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x813756; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)13757; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s613758; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s513759; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s413760; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613761; GFX90A-NOTTGSPLIT-NEXT: s_endpgm13762;13763; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_acquire_acquire_cmpxchg:13764; GFX90A-TGSPLIT: ; %bb.0: ; %entry13765; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x013766; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x413767; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x813768; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)13769; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s613770; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s513771; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s413772; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613773; GFX90A-TGSPLIT-NEXT: s_endpgm13774;13775; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_acquire_acquire_cmpxchg:13776; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry13777; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x013778; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x413779; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x813780; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)13781; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s213782; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s113783; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s013784; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613785; GFX942-NOTTGSPLIT-NEXT: s_endpgm13786;13787; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_acquire_acquire_cmpxchg:13788; GFX942-TGSPLIT: ; %bb.0: ; %entry13789; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x013790; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x413791; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x813792; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)13793; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s213794; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s113795; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s013796; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613797; GFX942-TGSPLIT-NEXT: s_endpgm13798;13799; GFX11-WGP-LABEL: local_workgroup_one_as_acquire_acquire_cmpxchg:13800; GFX11-WGP: ; %bb.0: ; %entry13801; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x013802; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x413803; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x813804; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)13805; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s213806; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s113807; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s013808; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613809; GFX11-WGP-NEXT: s_endpgm13810;13811; GFX11-CU-LABEL: local_workgroup_one_as_acquire_acquire_cmpxchg:13812; GFX11-CU: ; %bb.0: ; %entry13813; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x013814; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x413815; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x813816; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)13817; GFX11-CU-NEXT: v_mov_b32_e32 v0, s213818; GFX11-CU-NEXT: v_mov_b32_e32 v1, s113819; GFX11-CU-NEXT: v_mov_b32_e32 v2, s013820; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613821; GFX11-CU-NEXT: s_endpgm13822;13823; GFX12-WGP-LABEL: local_workgroup_one_as_acquire_acquire_cmpxchg:13824; GFX12-WGP: ; %bb.0: ; %entry13825; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x013826; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x413827; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x813828; GFX12-WGP-NEXT: s_wait_kmcnt 0x013829; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s213830; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s113831; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s013832; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613833; GFX12-WGP-NEXT: s_endpgm13834;13835; GFX12-CU-LABEL: local_workgroup_one_as_acquire_acquire_cmpxchg:13836; GFX12-CU: ; %bb.0: ; %entry13837; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x013838; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x413839; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x813840; GFX12-CU-NEXT: s_wait_kmcnt 0x013841; GFX12-CU-NEXT: v_mov_b32_e32 v0, s213842; GFX12-CU-NEXT: v_mov_b32_e32 v1, s113843; GFX12-CU-NEXT: v_mov_b32_e32 v2, s013844; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613845; GFX12-CU-NEXT: s_endpgm13846;13847; GFX1250-LABEL: local_workgroup_one_as_acquire_acquire_cmpxchg:13848; GFX1250: ; %bb.0: ; %entry13849; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 113850; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x013851; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x413852; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x813853; GFX1250-NEXT: s_wait_kmcnt 0x013854; GFX1250-NEXT: v_mov_b32_e32 v0, s213855; GFX1250-NEXT: v_mov_b32_e32 v1, s113856; GFX1250-NEXT: v_mov_b32_e32 v2, s013857; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613858; GFX1250-NEXT: s_endpgm13859 ptr addrspace(3) %out, i32 %in, i32 %old) {13860entry:13861 %gep = getelementptr i32, ptr addrspace(3) %out, i32 413862 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire13863 ret void13864}13865 13866define amdgpu_kernel void @local_workgroup_one_as_release_acquire_cmpxchg(13867; GFX6-LABEL: local_workgroup_one_as_release_acquire_cmpxchg:13868; GFX6: ; %bb.0: ; %entry13869; GFX6-NEXT: s_load_dword s6, s[8:9], 0x013870; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr613871; GFX6-NEXT: s_load_dword s4, s[8:9], 0x113872; GFX6-NEXT: s_load_dword s5, s[8:9], 0x213873; GFX6-NEXT: s_mov_b32 m0, -113874; GFX6-NEXT: s_waitcnt lgkmcnt(0)13875; GFX6-NEXT: v_mov_b32_e32 v0, s613876; GFX6-NEXT: v_mov_b32_e32 v1, s513877; GFX6-NEXT: v_mov_b32_e32 v2, s413878; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613879; GFX6-NEXT: s_endpgm13880;13881; GFX7-LABEL: local_workgroup_one_as_release_acquire_cmpxchg:13882; GFX7: ; %bb.0: ; %entry13883; GFX7-NEXT: s_load_dword s6, s[8:9], 0x013884; GFX7-NEXT: s_load_dword s4, s[8:9], 0x113885; GFX7-NEXT: s_load_dword s5, s[8:9], 0x213886; GFX7-NEXT: s_mov_b32 m0, -113887; GFX7-NEXT: s_waitcnt lgkmcnt(0)13888; GFX7-NEXT: v_mov_b32_e32 v0, s613889; GFX7-NEXT: v_mov_b32_e32 v1, s513890; GFX7-NEXT: v_mov_b32_e32 v2, s413891; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613892; GFX7-NEXT: s_endpgm13893;13894; GFX10-WGP-LABEL: local_workgroup_one_as_release_acquire_cmpxchg:13895; GFX10-WGP: ; %bb.0: ; %entry13896; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x013897; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x413898; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x813899; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)13900; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s613901; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s513902; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s413903; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613904; GFX10-WGP-NEXT: s_endpgm13905;13906; GFX10-CU-LABEL: local_workgroup_one_as_release_acquire_cmpxchg:13907; GFX10-CU: ; %bb.0: ; %entry13908; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x013909; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x413910; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x813911; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)13912; GFX10-CU-NEXT: v_mov_b32_e32 v0, s613913; GFX10-CU-NEXT: v_mov_b32_e32 v1, s513914; GFX10-CU-NEXT: v_mov_b32_e32 v2, s413915; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613916; GFX10-CU-NEXT: s_endpgm13917;13918; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_release_acquire_cmpxchg:13919; SKIP-CACHE-INV: ; %bb.0: ; %entry13920; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x013921; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x113922; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x213923; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -113924; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)13925; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s213926; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s113927; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s013928; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613929; SKIP-CACHE-INV-NEXT: s_endpgm13930;13931; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_release_acquire_cmpxchg:13932; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry13933; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x013934; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x413935; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x813936; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)13937; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s613938; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s513939; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s413940; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613941; GFX90A-NOTTGSPLIT-NEXT: s_endpgm13942;13943; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_release_acquire_cmpxchg:13944; GFX90A-TGSPLIT: ; %bb.0: ; %entry13945; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x013946; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x413947; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x813948; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)13949; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s613950; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s513951; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s413952; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613953; GFX90A-TGSPLIT-NEXT: s_endpgm13954;13955; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_release_acquire_cmpxchg:13956; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry13957; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x013958; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x413959; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x813960; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)13961; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s213962; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s113963; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s013964; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613965; GFX942-NOTTGSPLIT-NEXT: s_endpgm13966;13967; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_release_acquire_cmpxchg:13968; GFX942-TGSPLIT: ; %bb.0: ; %entry13969; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x013970; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x413971; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x813972; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)13973; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s213974; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s113975; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s013976; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1613977; GFX942-TGSPLIT-NEXT: s_endpgm13978;13979; GFX11-WGP-LABEL: local_workgroup_one_as_release_acquire_cmpxchg:13980; GFX11-WGP: ; %bb.0: ; %entry13981; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x013982; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x413983; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x813984; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)13985; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s213986; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s113987; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s013988; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1613989; GFX11-WGP-NEXT: s_endpgm13990;13991; GFX11-CU-LABEL: local_workgroup_one_as_release_acquire_cmpxchg:13992; GFX11-CU: ; %bb.0: ; %entry13993; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x013994; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x413995; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x813996; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)13997; GFX11-CU-NEXT: v_mov_b32_e32 v0, s213998; GFX11-CU-NEXT: v_mov_b32_e32 v1, s113999; GFX11-CU-NEXT: v_mov_b32_e32 v2, s014000; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614001; GFX11-CU-NEXT: s_endpgm14002;14003; GFX12-WGP-LABEL: local_workgroup_one_as_release_acquire_cmpxchg:14004; GFX12-WGP: ; %bb.0: ; %entry14005; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x014006; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x414007; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x814008; GFX12-WGP-NEXT: s_wait_kmcnt 0x014009; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s214010; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s114011; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s014012; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614013; GFX12-WGP-NEXT: s_endpgm14014;14015; GFX12-CU-LABEL: local_workgroup_one_as_release_acquire_cmpxchg:14016; GFX12-CU: ; %bb.0: ; %entry14017; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x014018; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x414019; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x814020; GFX12-CU-NEXT: s_wait_kmcnt 0x014021; GFX12-CU-NEXT: v_mov_b32_e32 v0, s214022; GFX12-CU-NEXT: v_mov_b32_e32 v1, s114023; GFX12-CU-NEXT: v_mov_b32_e32 v2, s014024; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614025; GFX12-CU-NEXT: s_endpgm14026;14027; GFX1250-LABEL: local_workgroup_one_as_release_acquire_cmpxchg:14028; GFX1250: ; %bb.0: ; %entry14029; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 114030; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x014031; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x414032; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x814033; GFX1250-NEXT: s_wait_kmcnt 0x014034; GFX1250-NEXT: v_mov_b32_e32 v0, s214035; GFX1250-NEXT: v_mov_b32_e32 v1, s114036; GFX1250-NEXT: v_mov_b32_e32 v2, s014037; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614038; GFX1250-NEXT: s_endpgm14039 ptr addrspace(3) %out, i32 %in, i32 %old) {14040entry:14041 %gep = getelementptr i32, ptr addrspace(3) %out, i32 414042 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire14043 ret void14044}14045 14046define amdgpu_kernel void @local_workgroup_one_as_acq_rel_acquire_cmpxchg(14047; GFX6-LABEL: local_workgroup_one_as_acq_rel_acquire_cmpxchg:14048; GFX6: ; %bb.0: ; %entry14049; GFX6-NEXT: s_load_dword s6, s[8:9], 0x014050; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr614051; GFX6-NEXT: s_load_dword s4, s[8:9], 0x114052; GFX6-NEXT: s_load_dword s5, s[8:9], 0x214053; GFX6-NEXT: s_mov_b32 m0, -114054; GFX6-NEXT: s_waitcnt lgkmcnt(0)14055; GFX6-NEXT: v_mov_b32_e32 v0, s614056; GFX6-NEXT: v_mov_b32_e32 v1, s514057; GFX6-NEXT: v_mov_b32_e32 v2, s414058; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614059; GFX6-NEXT: s_endpgm14060;14061; GFX7-LABEL: local_workgroup_one_as_acq_rel_acquire_cmpxchg:14062; GFX7: ; %bb.0: ; %entry14063; GFX7-NEXT: s_load_dword s6, s[8:9], 0x014064; GFX7-NEXT: s_load_dword s4, s[8:9], 0x114065; GFX7-NEXT: s_load_dword s5, s[8:9], 0x214066; GFX7-NEXT: s_mov_b32 m0, -114067; GFX7-NEXT: s_waitcnt lgkmcnt(0)14068; GFX7-NEXT: v_mov_b32_e32 v0, s614069; GFX7-NEXT: v_mov_b32_e32 v1, s514070; GFX7-NEXT: v_mov_b32_e32 v2, s414071; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614072; GFX7-NEXT: s_endpgm14073;14074; GFX10-WGP-LABEL: local_workgroup_one_as_acq_rel_acquire_cmpxchg:14075; GFX10-WGP: ; %bb.0: ; %entry14076; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x014077; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x414078; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x814079; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)14080; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s614081; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s514082; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s414083; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614084; GFX10-WGP-NEXT: s_endpgm14085;14086; GFX10-CU-LABEL: local_workgroup_one_as_acq_rel_acquire_cmpxchg:14087; GFX10-CU: ; %bb.0: ; %entry14088; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x014089; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x414090; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x814091; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)14092; GFX10-CU-NEXT: v_mov_b32_e32 v0, s614093; GFX10-CU-NEXT: v_mov_b32_e32 v1, s514094; GFX10-CU-NEXT: v_mov_b32_e32 v2, s414095; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614096; GFX10-CU-NEXT: s_endpgm14097;14098; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acq_rel_acquire_cmpxchg:14099; SKIP-CACHE-INV: ; %bb.0: ; %entry14100; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x014101; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x114102; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x214103; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -114104; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)14105; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s214106; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s114107; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s014108; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614109; SKIP-CACHE-INV-NEXT: s_endpgm14110;14111; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_acq_rel_acquire_cmpxchg:14112; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry14113; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x014114; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x414115; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x814116; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)14117; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s614118; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s514119; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s414120; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614121; GFX90A-NOTTGSPLIT-NEXT: s_endpgm14122;14123; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_acq_rel_acquire_cmpxchg:14124; GFX90A-TGSPLIT: ; %bb.0: ; %entry14125; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x014126; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x414127; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x814128; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)14129; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s614130; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s514131; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s414132; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614133; GFX90A-TGSPLIT-NEXT: s_endpgm14134;14135; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_acq_rel_acquire_cmpxchg:14136; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry14137; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x014138; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x414139; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x814140; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)14141; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s214142; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s114143; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s014144; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614145; GFX942-NOTTGSPLIT-NEXT: s_endpgm14146;14147; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_acq_rel_acquire_cmpxchg:14148; GFX942-TGSPLIT: ; %bb.0: ; %entry14149; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x014150; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x414151; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x814152; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)14153; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s214154; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s114155; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s014156; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614157; GFX942-TGSPLIT-NEXT: s_endpgm14158;14159; GFX11-WGP-LABEL: local_workgroup_one_as_acq_rel_acquire_cmpxchg:14160; GFX11-WGP: ; %bb.0: ; %entry14161; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x014162; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x414163; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x814164; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)14165; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s214166; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s114167; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s014168; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614169; GFX11-WGP-NEXT: s_endpgm14170;14171; GFX11-CU-LABEL: local_workgroup_one_as_acq_rel_acquire_cmpxchg:14172; GFX11-CU: ; %bb.0: ; %entry14173; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x014174; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x414175; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x814176; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)14177; GFX11-CU-NEXT: v_mov_b32_e32 v0, s214178; GFX11-CU-NEXT: v_mov_b32_e32 v1, s114179; GFX11-CU-NEXT: v_mov_b32_e32 v2, s014180; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614181; GFX11-CU-NEXT: s_endpgm14182;14183; GFX12-WGP-LABEL: local_workgroup_one_as_acq_rel_acquire_cmpxchg:14184; GFX12-WGP: ; %bb.0: ; %entry14185; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x014186; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x414187; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x814188; GFX12-WGP-NEXT: s_wait_kmcnt 0x014189; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s214190; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s114191; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s014192; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614193; GFX12-WGP-NEXT: s_endpgm14194;14195; GFX12-CU-LABEL: local_workgroup_one_as_acq_rel_acquire_cmpxchg:14196; GFX12-CU: ; %bb.0: ; %entry14197; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x014198; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x414199; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x814200; GFX12-CU-NEXT: s_wait_kmcnt 0x014201; GFX12-CU-NEXT: v_mov_b32_e32 v0, s214202; GFX12-CU-NEXT: v_mov_b32_e32 v1, s114203; GFX12-CU-NEXT: v_mov_b32_e32 v2, s014204; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614205; GFX12-CU-NEXT: s_endpgm14206;14207; GFX1250-LABEL: local_workgroup_one_as_acq_rel_acquire_cmpxchg:14208; GFX1250: ; %bb.0: ; %entry14209; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 114210; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x014211; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x414212; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x814213; GFX1250-NEXT: s_wait_kmcnt 0x014214; GFX1250-NEXT: v_mov_b32_e32 v0, s214215; GFX1250-NEXT: v_mov_b32_e32 v1, s114216; GFX1250-NEXT: v_mov_b32_e32 v2, s014217; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614218; GFX1250-NEXT: s_endpgm14219 ptr addrspace(3) %out, i32 %in, i32 %old) {14220entry:14221 %gep = getelementptr i32, ptr addrspace(3) %out, i32 414222 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire14223 ret void14224}14225 14226define amdgpu_kernel void @local_workgroup_one_as_seq_cst_acquire_cmpxchg(14227; GFX6-LABEL: local_workgroup_one_as_seq_cst_acquire_cmpxchg:14228; GFX6: ; %bb.0: ; %entry14229; GFX6-NEXT: s_load_dword s6, s[8:9], 0x014230; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr614231; GFX6-NEXT: s_load_dword s4, s[8:9], 0x114232; GFX6-NEXT: s_load_dword s5, s[8:9], 0x214233; GFX6-NEXT: s_mov_b32 m0, -114234; GFX6-NEXT: s_waitcnt lgkmcnt(0)14235; GFX6-NEXT: v_mov_b32_e32 v0, s614236; GFX6-NEXT: v_mov_b32_e32 v1, s514237; GFX6-NEXT: v_mov_b32_e32 v2, s414238; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614239; GFX6-NEXT: s_endpgm14240;14241; GFX7-LABEL: local_workgroup_one_as_seq_cst_acquire_cmpxchg:14242; GFX7: ; %bb.0: ; %entry14243; GFX7-NEXT: s_load_dword s6, s[8:9], 0x014244; GFX7-NEXT: s_load_dword s4, s[8:9], 0x114245; GFX7-NEXT: s_load_dword s5, s[8:9], 0x214246; GFX7-NEXT: s_mov_b32 m0, -114247; GFX7-NEXT: s_waitcnt lgkmcnt(0)14248; GFX7-NEXT: v_mov_b32_e32 v0, s614249; GFX7-NEXT: v_mov_b32_e32 v1, s514250; GFX7-NEXT: v_mov_b32_e32 v2, s414251; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614252; GFX7-NEXT: s_endpgm14253;14254; GFX10-WGP-LABEL: local_workgroup_one_as_seq_cst_acquire_cmpxchg:14255; GFX10-WGP: ; %bb.0: ; %entry14256; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x014257; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x414258; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x814259; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)14260; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s614261; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s514262; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s414263; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614264; GFX10-WGP-NEXT: s_endpgm14265;14266; GFX10-CU-LABEL: local_workgroup_one_as_seq_cst_acquire_cmpxchg:14267; GFX10-CU: ; %bb.0: ; %entry14268; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x014269; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x414270; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x814271; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)14272; GFX10-CU-NEXT: v_mov_b32_e32 v0, s614273; GFX10-CU-NEXT: v_mov_b32_e32 v1, s514274; GFX10-CU-NEXT: v_mov_b32_e32 v2, s414275; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614276; GFX10-CU-NEXT: s_endpgm14277;14278; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_seq_cst_acquire_cmpxchg:14279; SKIP-CACHE-INV: ; %bb.0: ; %entry14280; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x014281; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x114282; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x214283; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -114284; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)14285; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s214286; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s114287; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s014288; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614289; SKIP-CACHE-INV-NEXT: s_endpgm14290;14291; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_seq_cst_acquire_cmpxchg:14292; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry14293; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x014294; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x414295; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x814296; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)14297; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s614298; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s514299; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s414300; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614301; GFX90A-NOTTGSPLIT-NEXT: s_endpgm14302;14303; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_seq_cst_acquire_cmpxchg:14304; GFX90A-TGSPLIT: ; %bb.0: ; %entry14305; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x014306; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x414307; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x814308; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)14309; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s614310; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s514311; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s414312; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614313; GFX90A-TGSPLIT-NEXT: s_endpgm14314;14315; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_seq_cst_acquire_cmpxchg:14316; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry14317; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x014318; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x414319; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x814320; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)14321; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s214322; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s114323; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s014324; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614325; GFX942-NOTTGSPLIT-NEXT: s_endpgm14326;14327; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_seq_cst_acquire_cmpxchg:14328; GFX942-TGSPLIT: ; %bb.0: ; %entry14329; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x014330; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x414331; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x814332; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)14333; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s214334; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s114335; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s014336; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614337; GFX942-TGSPLIT-NEXT: s_endpgm14338;14339; GFX11-WGP-LABEL: local_workgroup_one_as_seq_cst_acquire_cmpxchg:14340; GFX11-WGP: ; %bb.0: ; %entry14341; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x014342; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x414343; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x814344; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)14345; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s214346; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s114347; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s014348; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614349; GFX11-WGP-NEXT: s_endpgm14350;14351; GFX11-CU-LABEL: local_workgroup_one_as_seq_cst_acquire_cmpxchg:14352; GFX11-CU: ; %bb.0: ; %entry14353; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x014354; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x414355; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x814356; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)14357; GFX11-CU-NEXT: v_mov_b32_e32 v0, s214358; GFX11-CU-NEXT: v_mov_b32_e32 v1, s114359; GFX11-CU-NEXT: v_mov_b32_e32 v2, s014360; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614361; GFX11-CU-NEXT: s_endpgm14362;14363; GFX12-WGP-LABEL: local_workgroup_one_as_seq_cst_acquire_cmpxchg:14364; GFX12-WGP: ; %bb.0: ; %entry14365; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x014366; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x414367; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x814368; GFX12-WGP-NEXT: s_wait_kmcnt 0x014369; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s214370; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s114371; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s014372; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614373; GFX12-WGP-NEXT: s_endpgm14374;14375; GFX12-CU-LABEL: local_workgroup_one_as_seq_cst_acquire_cmpxchg:14376; GFX12-CU: ; %bb.0: ; %entry14377; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x014378; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x414379; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x814380; GFX12-CU-NEXT: s_wait_kmcnt 0x014381; GFX12-CU-NEXT: v_mov_b32_e32 v0, s214382; GFX12-CU-NEXT: v_mov_b32_e32 v1, s114383; GFX12-CU-NEXT: v_mov_b32_e32 v2, s014384; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614385; GFX12-CU-NEXT: s_endpgm14386;14387; GFX1250-LABEL: local_workgroup_one_as_seq_cst_acquire_cmpxchg:14388; GFX1250: ; %bb.0: ; %entry14389; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 114390; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x014391; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x414392; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x814393; GFX1250-NEXT: s_wait_kmcnt 0x014394; GFX1250-NEXT: v_mov_b32_e32 v0, s214395; GFX1250-NEXT: v_mov_b32_e32 v1, s114396; GFX1250-NEXT: v_mov_b32_e32 v2, s014397; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614398; GFX1250-NEXT: s_endpgm14399 ptr addrspace(3) %out, i32 %in, i32 %old) {14400entry:14401 %gep = getelementptr i32, ptr addrspace(3) %out, i32 414402 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire14403 ret void14404}14405 14406define amdgpu_kernel void @local_workgroup_one_as_monotonic_seq_cst_cmpxchg(14407; GFX6-LABEL: local_workgroup_one_as_monotonic_seq_cst_cmpxchg:14408; GFX6: ; %bb.0: ; %entry14409; GFX6-NEXT: s_load_dword s6, s[8:9], 0x014410; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr614411; GFX6-NEXT: s_load_dword s4, s[8:9], 0x114412; GFX6-NEXT: s_load_dword s5, s[8:9], 0x214413; GFX6-NEXT: s_mov_b32 m0, -114414; GFX6-NEXT: s_waitcnt lgkmcnt(0)14415; GFX6-NEXT: v_mov_b32_e32 v0, s614416; GFX6-NEXT: v_mov_b32_e32 v1, s514417; GFX6-NEXT: v_mov_b32_e32 v2, s414418; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614419; GFX6-NEXT: s_endpgm14420;14421; GFX7-LABEL: local_workgroup_one_as_monotonic_seq_cst_cmpxchg:14422; GFX7: ; %bb.0: ; %entry14423; GFX7-NEXT: s_load_dword s6, s[8:9], 0x014424; GFX7-NEXT: s_load_dword s4, s[8:9], 0x114425; GFX7-NEXT: s_load_dword s5, s[8:9], 0x214426; GFX7-NEXT: s_mov_b32 m0, -114427; GFX7-NEXT: s_waitcnt lgkmcnt(0)14428; GFX7-NEXT: v_mov_b32_e32 v0, s614429; GFX7-NEXT: v_mov_b32_e32 v1, s514430; GFX7-NEXT: v_mov_b32_e32 v2, s414431; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614432; GFX7-NEXT: s_endpgm14433;14434; GFX10-WGP-LABEL: local_workgroup_one_as_monotonic_seq_cst_cmpxchg:14435; GFX10-WGP: ; %bb.0: ; %entry14436; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x014437; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x414438; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x814439; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)14440; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s614441; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s514442; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s414443; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614444; GFX10-WGP-NEXT: s_endpgm14445;14446; GFX10-CU-LABEL: local_workgroup_one_as_monotonic_seq_cst_cmpxchg:14447; GFX10-CU: ; %bb.0: ; %entry14448; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x014449; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x414450; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x814451; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)14452; GFX10-CU-NEXT: v_mov_b32_e32 v0, s614453; GFX10-CU-NEXT: v_mov_b32_e32 v1, s514454; GFX10-CU-NEXT: v_mov_b32_e32 v2, s414455; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614456; GFX10-CU-NEXT: s_endpgm14457;14458; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_monotonic_seq_cst_cmpxchg:14459; SKIP-CACHE-INV: ; %bb.0: ; %entry14460; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x014461; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x114462; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x214463; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -114464; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)14465; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s214466; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s114467; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s014468; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614469; SKIP-CACHE-INV-NEXT: s_endpgm14470;14471; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_monotonic_seq_cst_cmpxchg:14472; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry14473; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x014474; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x414475; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x814476; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)14477; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s614478; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s514479; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s414480; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614481; GFX90A-NOTTGSPLIT-NEXT: s_endpgm14482;14483; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_monotonic_seq_cst_cmpxchg:14484; GFX90A-TGSPLIT: ; %bb.0: ; %entry14485; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x014486; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x414487; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x814488; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)14489; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s614490; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s514491; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s414492; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614493; GFX90A-TGSPLIT-NEXT: s_endpgm14494;14495; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_monotonic_seq_cst_cmpxchg:14496; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry14497; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x014498; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x414499; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x814500; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)14501; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s214502; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s114503; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s014504; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614505; GFX942-NOTTGSPLIT-NEXT: s_endpgm14506;14507; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_monotonic_seq_cst_cmpxchg:14508; GFX942-TGSPLIT: ; %bb.0: ; %entry14509; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x014510; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x414511; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x814512; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)14513; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s214514; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s114515; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s014516; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614517; GFX942-TGSPLIT-NEXT: s_endpgm14518;14519; GFX11-WGP-LABEL: local_workgroup_one_as_monotonic_seq_cst_cmpxchg:14520; GFX11-WGP: ; %bb.0: ; %entry14521; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x014522; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x414523; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x814524; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)14525; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s214526; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s114527; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s014528; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614529; GFX11-WGP-NEXT: s_endpgm14530;14531; GFX11-CU-LABEL: local_workgroup_one_as_monotonic_seq_cst_cmpxchg:14532; GFX11-CU: ; %bb.0: ; %entry14533; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x014534; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x414535; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x814536; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)14537; GFX11-CU-NEXT: v_mov_b32_e32 v0, s214538; GFX11-CU-NEXT: v_mov_b32_e32 v1, s114539; GFX11-CU-NEXT: v_mov_b32_e32 v2, s014540; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614541; GFX11-CU-NEXT: s_endpgm14542;14543; GFX12-WGP-LABEL: local_workgroup_one_as_monotonic_seq_cst_cmpxchg:14544; GFX12-WGP: ; %bb.0: ; %entry14545; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x014546; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x414547; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x814548; GFX12-WGP-NEXT: s_wait_kmcnt 0x014549; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s214550; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s114551; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s014552; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614553; GFX12-WGP-NEXT: s_endpgm14554;14555; GFX12-CU-LABEL: local_workgroup_one_as_monotonic_seq_cst_cmpxchg:14556; GFX12-CU: ; %bb.0: ; %entry14557; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x014558; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x414559; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x814560; GFX12-CU-NEXT: s_wait_kmcnt 0x014561; GFX12-CU-NEXT: v_mov_b32_e32 v0, s214562; GFX12-CU-NEXT: v_mov_b32_e32 v1, s114563; GFX12-CU-NEXT: v_mov_b32_e32 v2, s014564; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614565; GFX12-CU-NEXT: s_endpgm14566;14567; GFX1250-LABEL: local_workgroup_one_as_monotonic_seq_cst_cmpxchg:14568; GFX1250: ; %bb.0: ; %entry14569; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 114570; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x014571; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x414572; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x814573; GFX1250-NEXT: s_wait_kmcnt 0x014574; GFX1250-NEXT: v_mov_b32_e32 v0, s214575; GFX1250-NEXT: v_mov_b32_e32 v1, s114576; GFX1250-NEXT: v_mov_b32_e32 v2, s014577; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614578; GFX1250-NEXT: s_endpgm14579 ptr addrspace(3) %out, i32 %in, i32 %old) {14580entry:14581 %gep = getelementptr i32, ptr addrspace(3) %out, i32 414582 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic seq_cst14583 ret void14584}14585 14586define amdgpu_kernel void @local_workgroup_one_as_acquire_seq_cst_cmpxchg(14587; GFX6-LABEL: local_workgroup_one_as_acquire_seq_cst_cmpxchg:14588; GFX6: ; %bb.0: ; %entry14589; GFX6-NEXT: s_load_dword s6, s[8:9], 0x014590; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr614591; GFX6-NEXT: s_load_dword s4, s[8:9], 0x114592; GFX6-NEXT: s_load_dword s5, s[8:9], 0x214593; GFX6-NEXT: s_mov_b32 m0, -114594; GFX6-NEXT: s_waitcnt lgkmcnt(0)14595; GFX6-NEXT: v_mov_b32_e32 v0, s614596; GFX6-NEXT: v_mov_b32_e32 v1, s514597; GFX6-NEXT: v_mov_b32_e32 v2, s414598; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614599; GFX6-NEXT: s_endpgm14600;14601; GFX7-LABEL: local_workgroup_one_as_acquire_seq_cst_cmpxchg:14602; GFX7: ; %bb.0: ; %entry14603; GFX7-NEXT: s_load_dword s6, s[8:9], 0x014604; GFX7-NEXT: s_load_dword s4, s[8:9], 0x114605; GFX7-NEXT: s_load_dword s5, s[8:9], 0x214606; GFX7-NEXT: s_mov_b32 m0, -114607; GFX7-NEXT: s_waitcnt lgkmcnt(0)14608; GFX7-NEXT: v_mov_b32_e32 v0, s614609; GFX7-NEXT: v_mov_b32_e32 v1, s514610; GFX7-NEXT: v_mov_b32_e32 v2, s414611; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614612; GFX7-NEXT: s_endpgm14613;14614; GFX10-WGP-LABEL: local_workgroup_one_as_acquire_seq_cst_cmpxchg:14615; GFX10-WGP: ; %bb.0: ; %entry14616; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x014617; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x414618; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x814619; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)14620; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s614621; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s514622; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s414623; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614624; GFX10-WGP-NEXT: s_endpgm14625;14626; GFX10-CU-LABEL: local_workgroup_one_as_acquire_seq_cst_cmpxchg:14627; GFX10-CU: ; %bb.0: ; %entry14628; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x014629; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x414630; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x814631; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)14632; GFX10-CU-NEXT: v_mov_b32_e32 v0, s614633; GFX10-CU-NEXT: v_mov_b32_e32 v1, s514634; GFX10-CU-NEXT: v_mov_b32_e32 v2, s414635; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614636; GFX10-CU-NEXT: s_endpgm14637;14638; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acquire_seq_cst_cmpxchg:14639; SKIP-CACHE-INV: ; %bb.0: ; %entry14640; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x014641; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x114642; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x214643; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -114644; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)14645; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s214646; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s114647; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s014648; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614649; SKIP-CACHE-INV-NEXT: s_endpgm14650;14651; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_acquire_seq_cst_cmpxchg:14652; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry14653; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x014654; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x414655; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x814656; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)14657; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s614658; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s514659; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s414660; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614661; GFX90A-NOTTGSPLIT-NEXT: s_endpgm14662;14663; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_acquire_seq_cst_cmpxchg:14664; GFX90A-TGSPLIT: ; %bb.0: ; %entry14665; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x014666; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x414667; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x814668; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)14669; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s614670; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s514671; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s414672; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614673; GFX90A-TGSPLIT-NEXT: s_endpgm14674;14675; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_acquire_seq_cst_cmpxchg:14676; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry14677; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x014678; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x414679; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x814680; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)14681; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s214682; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s114683; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s014684; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614685; GFX942-NOTTGSPLIT-NEXT: s_endpgm14686;14687; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_acquire_seq_cst_cmpxchg:14688; GFX942-TGSPLIT: ; %bb.0: ; %entry14689; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x014690; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x414691; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x814692; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)14693; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s214694; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s114695; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s014696; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614697; GFX942-TGSPLIT-NEXT: s_endpgm14698;14699; GFX11-WGP-LABEL: local_workgroup_one_as_acquire_seq_cst_cmpxchg:14700; GFX11-WGP: ; %bb.0: ; %entry14701; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x014702; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x414703; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x814704; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)14705; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s214706; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s114707; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s014708; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614709; GFX11-WGP-NEXT: s_endpgm14710;14711; GFX11-CU-LABEL: local_workgroup_one_as_acquire_seq_cst_cmpxchg:14712; GFX11-CU: ; %bb.0: ; %entry14713; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x014714; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x414715; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x814716; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)14717; GFX11-CU-NEXT: v_mov_b32_e32 v0, s214718; GFX11-CU-NEXT: v_mov_b32_e32 v1, s114719; GFX11-CU-NEXT: v_mov_b32_e32 v2, s014720; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614721; GFX11-CU-NEXT: s_endpgm14722;14723; GFX12-WGP-LABEL: local_workgroup_one_as_acquire_seq_cst_cmpxchg:14724; GFX12-WGP: ; %bb.0: ; %entry14725; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x014726; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x414727; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x814728; GFX12-WGP-NEXT: s_wait_kmcnt 0x014729; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s214730; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s114731; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s014732; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614733; GFX12-WGP-NEXT: s_endpgm14734;14735; GFX12-CU-LABEL: local_workgroup_one_as_acquire_seq_cst_cmpxchg:14736; GFX12-CU: ; %bb.0: ; %entry14737; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x014738; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x414739; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x814740; GFX12-CU-NEXT: s_wait_kmcnt 0x014741; GFX12-CU-NEXT: v_mov_b32_e32 v0, s214742; GFX12-CU-NEXT: v_mov_b32_e32 v1, s114743; GFX12-CU-NEXT: v_mov_b32_e32 v2, s014744; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614745; GFX12-CU-NEXT: s_endpgm14746;14747; GFX1250-LABEL: local_workgroup_one_as_acquire_seq_cst_cmpxchg:14748; GFX1250: ; %bb.0: ; %entry14749; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 114750; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x014751; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x414752; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x814753; GFX1250-NEXT: s_wait_kmcnt 0x014754; GFX1250-NEXT: v_mov_b32_e32 v0, s214755; GFX1250-NEXT: v_mov_b32_e32 v1, s114756; GFX1250-NEXT: v_mov_b32_e32 v2, s014757; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614758; GFX1250-NEXT: s_endpgm14759 ptr addrspace(3) %out, i32 %in, i32 %old) {14760entry:14761 %gep = getelementptr i32, ptr addrspace(3) %out, i32 414762 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire seq_cst14763 ret void14764}14765 14766define amdgpu_kernel void @local_workgroup_one_as_release_seq_cst_cmpxchg(14767; GFX6-LABEL: local_workgroup_one_as_release_seq_cst_cmpxchg:14768; GFX6: ; %bb.0: ; %entry14769; GFX6-NEXT: s_load_dword s6, s[8:9], 0x014770; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr614771; GFX6-NEXT: s_load_dword s4, s[8:9], 0x114772; GFX6-NEXT: s_load_dword s5, s[8:9], 0x214773; GFX6-NEXT: s_mov_b32 m0, -114774; GFX6-NEXT: s_waitcnt lgkmcnt(0)14775; GFX6-NEXT: v_mov_b32_e32 v0, s614776; GFX6-NEXT: v_mov_b32_e32 v1, s514777; GFX6-NEXT: v_mov_b32_e32 v2, s414778; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614779; GFX6-NEXT: s_endpgm14780;14781; GFX7-LABEL: local_workgroup_one_as_release_seq_cst_cmpxchg:14782; GFX7: ; %bb.0: ; %entry14783; GFX7-NEXT: s_load_dword s6, s[8:9], 0x014784; GFX7-NEXT: s_load_dword s4, s[8:9], 0x114785; GFX7-NEXT: s_load_dword s5, s[8:9], 0x214786; GFX7-NEXT: s_mov_b32 m0, -114787; GFX7-NEXT: s_waitcnt lgkmcnt(0)14788; GFX7-NEXT: v_mov_b32_e32 v0, s614789; GFX7-NEXT: v_mov_b32_e32 v1, s514790; GFX7-NEXT: v_mov_b32_e32 v2, s414791; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614792; GFX7-NEXT: s_endpgm14793;14794; GFX10-WGP-LABEL: local_workgroup_one_as_release_seq_cst_cmpxchg:14795; GFX10-WGP: ; %bb.0: ; %entry14796; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x014797; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x414798; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x814799; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)14800; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s614801; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s514802; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s414803; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614804; GFX10-WGP-NEXT: s_endpgm14805;14806; GFX10-CU-LABEL: local_workgroup_one_as_release_seq_cst_cmpxchg:14807; GFX10-CU: ; %bb.0: ; %entry14808; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x014809; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x414810; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x814811; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)14812; GFX10-CU-NEXT: v_mov_b32_e32 v0, s614813; GFX10-CU-NEXT: v_mov_b32_e32 v1, s514814; GFX10-CU-NEXT: v_mov_b32_e32 v2, s414815; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614816; GFX10-CU-NEXT: s_endpgm14817;14818; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_release_seq_cst_cmpxchg:14819; SKIP-CACHE-INV: ; %bb.0: ; %entry14820; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x014821; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x114822; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x214823; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -114824; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)14825; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s214826; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s114827; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s014828; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614829; SKIP-CACHE-INV-NEXT: s_endpgm14830;14831; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_release_seq_cst_cmpxchg:14832; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry14833; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x014834; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x414835; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x814836; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)14837; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s614838; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s514839; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s414840; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614841; GFX90A-NOTTGSPLIT-NEXT: s_endpgm14842;14843; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_release_seq_cst_cmpxchg:14844; GFX90A-TGSPLIT: ; %bb.0: ; %entry14845; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x014846; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x414847; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x814848; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)14849; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s614850; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s514851; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s414852; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614853; GFX90A-TGSPLIT-NEXT: s_endpgm14854;14855; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_release_seq_cst_cmpxchg:14856; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry14857; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x014858; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x414859; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x814860; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)14861; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s214862; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s114863; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s014864; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614865; GFX942-NOTTGSPLIT-NEXT: s_endpgm14866;14867; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_release_seq_cst_cmpxchg:14868; GFX942-TGSPLIT: ; %bb.0: ; %entry14869; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x014870; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x414871; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x814872; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)14873; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s214874; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s114875; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s014876; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614877; GFX942-TGSPLIT-NEXT: s_endpgm14878;14879; GFX11-WGP-LABEL: local_workgroup_one_as_release_seq_cst_cmpxchg:14880; GFX11-WGP: ; %bb.0: ; %entry14881; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x014882; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x414883; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x814884; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)14885; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s214886; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s114887; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s014888; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614889; GFX11-WGP-NEXT: s_endpgm14890;14891; GFX11-CU-LABEL: local_workgroup_one_as_release_seq_cst_cmpxchg:14892; GFX11-CU: ; %bb.0: ; %entry14893; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x014894; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x414895; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x814896; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)14897; GFX11-CU-NEXT: v_mov_b32_e32 v0, s214898; GFX11-CU-NEXT: v_mov_b32_e32 v1, s114899; GFX11-CU-NEXT: v_mov_b32_e32 v2, s014900; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614901; GFX11-CU-NEXT: s_endpgm14902;14903; GFX12-WGP-LABEL: local_workgroup_one_as_release_seq_cst_cmpxchg:14904; GFX12-WGP: ; %bb.0: ; %entry14905; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x014906; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x414907; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x814908; GFX12-WGP-NEXT: s_wait_kmcnt 0x014909; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s214910; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s114911; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s014912; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614913; GFX12-WGP-NEXT: s_endpgm14914;14915; GFX12-CU-LABEL: local_workgroup_one_as_release_seq_cst_cmpxchg:14916; GFX12-CU: ; %bb.0: ; %entry14917; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x014918; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x414919; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x814920; GFX12-CU-NEXT: s_wait_kmcnt 0x014921; GFX12-CU-NEXT: v_mov_b32_e32 v0, s214922; GFX12-CU-NEXT: v_mov_b32_e32 v1, s114923; GFX12-CU-NEXT: v_mov_b32_e32 v2, s014924; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614925; GFX12-CU-NEXT: s_endpgm14926;14927; GFX1250-LABEL: local_workgroup_one_as_release_seq_cst_cmpxchg:14928; GFX1250: ; %bb.0: ; %entry14929; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 114930; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x014931; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x414932; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x814933; GFX1250-NEXT: s_wait_kmcnt 0x014934; GFX1250-NEXT: v_mov_b32_e32 v0, s214935; GFX1250-NEXT: v_mov_b32_e32 v1, s114936; GFX1250-NEXT: v_mov_b32_e32 v2, s014937; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1614938; GFX1250-NEXT: s_endpgm14939 ptr addrspace(3) %out, i32 %in, i32 %old) {14940entry:14941 %gep = getelementptr i32, ptr addrspace(3) %out, i32 414942 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release seq_cst14943 ret void14944}14945 14946define amdgpu_kernel void @local_workgroup_one_as_acq_rel_seq_cst_cmpxchg(14947; GFX6-LABEL: local_workgroup_one_as_acq_rel_seq_cst_cmpxchg:14948; GFX6: ; %bb.0: ; %entry14949; GFX6-NEXT: s_load_dword s6, s[8:9], 0x014950; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr614951; GFX6-NEXT: s_load_dword s4, s[8:9], 0x114952; GFX6-NEXT: s_load_dword s5, s[8:9], 0x214953; GFX6-NEXT: s_mov_b32 m0, -114954; GFX6-NEXT: s_waitcnt lgkmcnt(0)14955; GFX6-NEXT: v_mov_b32_e32 v0, s614956; GFX6-NEXT: v_mov_b32_e32 v1, s514957; GFX6-NEXT: v_mov_b32_e32 v2, s414958; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614959; GFX6-NEXT: s_endpgm14960;14961; GFX7-LABEL: local_workgroup_one_as_acq_rel_seq_cst_cmpxchg:14962; GFX7: ; %bb.0: ; %entry14963; GFX7-NEXT: s_load_dword s6, s[8:9], 0x014964; GFX7-NEXT: s_load_dword s4, s[8:9], 0x114965; GFX7-NEXT: s_load_dword s5, s[8:9], 0x214966; GFX7-NEXT: s_mov_b32 m0, -114967; GFX7-NEXT: s_waitcnt lgkmcnt(0)14968; GFX7-NEXT: v_mov_b32_e32 v0, s614969; GFX7-NEXT: v_mov_b32_e32 v1, s514970; GFX7-NEXT: v_mov_b32_e32 v2, s414971; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614972; GFX7-NEXT: s_endpgm14973;14974; GFX10-WGP-LABEL: local_workgroup_one_as_acq_rel_seq_cst_cmpxchg:14975; GFX10-WGP: ; %bb.0: ; %entry14976; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x014977; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x414978; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x814979; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)14980; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s614981; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s514982; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s414983; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614984; GFX10-WGP-NEXT: s_endpgm14985;14986; GFX10-CU-LABEL: local_workgroup_one_as_acq_rel_seq_cst_cmpxchg:14987; GFX10-CU: ; %bb.0: ; %entry14988; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x014989; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x414990; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x814991; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)14992; GFX10-CU-NEXT: v_mov_b32_e32 v0, s614993; GFX10-CU-NEXT: v_mov_b32_e32 v1, s514994; GFX10-CU-NEXT: v_mov_b32_e32 v2, s414995; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1614996; GFX10-CU-NEXT: s_endpgm14997;14998; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acq_rel_seq_cst_cmpxchg:14999; SKIP-CACHE-INV: ; %bb.0: ; %entry15000; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x015001; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x115002; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x215003; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -115004; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)15005; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s215006; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s115007; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s015008; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1615009; SKIP-CACHE-INV-NEXT: s_endpgm15010;15011; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_acq_rel_seq_cst_cmpxchg:15012; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry15013; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x015014; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x415015; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x815016; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15017; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s615018; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s515019; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s415020; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1615021; GFX90A-NOTTGSPLIT-NEXT: s_endpgm15022;15023; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_acq_rel_seq_cst_cmpxchg:15024; GFX90A-TGSPLIT: ; %bb.0: ; %entry15025; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x015026; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x415027; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x815028; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15029; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s615030; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s515031; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s415032; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1615033; GFX90A-TGSPLIT-NEXT: s_endpgm15034;15035; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_acq_rel_seq_cst_cmpxchg:15036; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry15037; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x015038; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x415039; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x815040; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15041; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s215042; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s115043; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s015044; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1615045; GFX942-NOTTGSPLIT-NEXT: s_endpgm15046;15047; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_acq_rel_seq_cst_cmpxchg:15048; GFX942-TGSPLIT: ; %bb.0: ; %entry15049; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x015050; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x415051; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x815052; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15053; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s215054; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s115055; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s015056; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1615057; GFX942-TGSPLIT-NEXT: s_endpgm15058;15059; GFX11-WGP-LABEL: local_workgroup_one_as_acq_rel_seq_cst_cmpxchg:15060; GFX11-WGP: ; %bb.0: ; %entry15061; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x015062; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x415063; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x815064; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)15065; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s215066; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s115067; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s015068; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1615069; GFX11-WGP-NEXT: s_endpgm15070;15071; GFX11-CU-LABEL: local_workgroup_one_as_acq_rel_seq_cst_cmpxchg:15072; GFX11-CU: ; %bb.0: ; %entry15073; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x015074; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x415075; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x815076; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)15077; GFX11-CU-NEXT: v_mov_b32_e32 v0, s215078; GFX11-CU-NEXT: v_mov_b32_e32 v1, s115079; GFX11-CU-NEXT: v_mov_b32_e32 v2, s015080; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1615081; GFX11-CU-NEXT: s_endpgm15082;15083; GFX12-WGP-LABEL: local_workgroup_one_as_acq_rel_seq_cst_cmpxchg:15084; GFX12-WGP: ; %bb.0: ; %entry15085; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x015086; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x415087; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x815088; GFX12-WGP-NEXT: s_wait_kmcnt 0x015089; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s215090; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s115091; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s015092; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1615093; GFX12-WGP-NEXT: s_endpgm15094;15095; GFX12-CU-LABEL: local_workgroup_one_as_acq_rel_seq_cst_cmpxchg:15096; GFX12-CU: ; %bb.0: ; %entry15097; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x015098; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x415099; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x815100; GFX12-CU-NEXT: s_wait_kmcnt 0x015101; GFX12-CU-NEXT: v_mov_b32_e32 v0, s215102; GFX12-CU-NEXT: v_mov_b32_e32 v1, s115103; GFX12-CU-NEXT: v_mov_b32_e32 v2, s015104; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1615105; GFX12-CU-NEXT: s_endpgm15106;15107; GFX1250-LABEL: local_workgroup_one_as_acq_rel_seq_cst_cmpxchg:15108; GFX1250: ; %bb.0: ; %entry15109; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 115110; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x015111; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x415112; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x815113; GFX1250-NEXT: s_wait_kmcnt 0x015114; GFX1250-NEXT: v_mov_b32_e32 v0, s215115; GFX1250-NEXT: v_mov_b32_e32 v1, s115116; GFX1250-NEXT: v_mov_b32_e32 v2, s015117; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1615118; GFX1250-NEXT: s_endpgm15119 ptr addrspace(3) %out, i32 %in, i32 %old) {15120entry:15121 %gep = getelementptr i32, ptr addrspace(3) %out, i32 415122 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel seq_cst15123 ret void15124}15125 15126define amdgpu_kernel void @local_workgroup_one_as_seq_cst_seq_cst_cmpxchg(15127; GFX6-LABEL: local_workgroup_one_as_seq_cst_seq_cst_cmpxchg:15128; GFX6: ; %bb.0: ; %entry15129; GFX6-NEXT: s_load_dword s6, s[8:9], 0x015130; GFX6-NEXT: ; kill: def $sgpr4 killed $sgpr615131; GFX6-NEXT: s_load_dword s4, s[8:9], 0x115132; GFX6-NEXT: s_load_dword s5, s[8:9], 0x215133; GFX6-NEXT: s_mov_b32 m0, -115134; GFX6-NEXT: s_waitcnt lgkmcnt(0)15135; GFX6-NEXT: v_mov_b32_e32 v0, s615136; GFX6-NEXT: v_mov_b32_e32 v1, s515137; GFX6-NEXT: v_mov_b32_e32 v2, s415138; GFX6-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1615139; GFX6-NEXT: s_endpgm15140;15141; GFX7-LABEL: local_workgroup_one_as_seq_cst_seq_cst_cmpxchg:15142; GFX7: ; %bb.0: ; %entry15143; GFX7-NEXT: s_load_dword s6, s[8:9], 0x015144; GFX7-NEXT: s_load_dword s4, s[8:9], 0x115145; GFX7-NEXT: s_load_dword s5, s[8:9], 0x215146; GFX7-NEXT: s_mov_b32 m0, -115147; GFX7-NEXT: s_waitcnt lgkmcnt(0)15148; GFX7-NEXT: v_mov_b32_e32 v0, s615149; GFX7-NEXT: v_mov_b32_e32 v1, s515150; GFX7-NEXT: v_mov_b32_e32 v2, s415151; GFX7-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1615152; GFX7-NEXT: s_endpgm15153;15154; GFX10-WGP-LABEL: local_workgroup_one_as_seq_cst_seq_cst_cmpxchg:15155; GFX10-WGP: ; %bb.0: ; %entry15156; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x015157; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x415158; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x815159; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)15160; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s615161; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s515162; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s415163; GFX10-WGP-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1615164; GFX10-WGP-NEXT: s_endpgm15165;15166; GFX10-CU-LABEL: local_workgroup_one_as_seq_cst_seq_cst_cmpxchg:15167; GFX10-CU: ; %bb.0: ; %entry15168; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x015169; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x415170; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x815171; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)15172; GFX10-CU-NEXT: v_mov_b32_e32 v0, s615173; GFX10-CU-NEXT: v_mov_b32_e32 v1, s515174; GFX10-CU-NEXT: v_mov_b32_e32 v2, s415175; GFX10-CU-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1615176; GFX10-CU-NEXT: s_endpgm15177;15178; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_seq_cst_seq_cst_cmpxchg:15179; SKIP-CACHE-INV: ; %bb.0: ; %entry15180; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x015181; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x115182; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x215183; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -115184; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)15185; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s215186; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s115187; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s015188; SKIP-CACHE-INV-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1615189; SKIP-CACHE-INV-NEXT: s_endpgm15190;15191; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_seq_cst_seq_cst_cmpxchg:15192; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry15193; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x015194; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x415195; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x815196; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15197; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s615198; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s515199; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s415200; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1615201; GFX90A-NOTTGSPLIT-NEXT: s_endpgm15202;15203; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_seq_cst_seq_cst_cmpxchg:15204; GFX90A-TGSPLIT: ; %bb.0: ; %entry15205; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x015206; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x415207; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x815208; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15209; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s615210; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s515211; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s415212; GFX90A-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1615213; GFX90A-TGSPLIT-NEXT: s_endpgm15214;15215; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_seq_cst_seq_cst_cmpxchg:15216; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry15217; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x015218; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x415219; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x815220; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15221; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s215222; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s115223; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s015224; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1615225; GFX942-NOTTGSPLIT-NEXT: s_endpgm15226;15227; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_seq_cst_seq_cst_cmpxchg:15228; GFX942-TGSPLIT: ; %bb.0: ; %entry15229; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x015230; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x415231; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x815232; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15233; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s215234; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s115235; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s015236; GFX942-TGSPLIT-NEXT: ds_cmpst_b32 v0, v1, v2 offset:1615237; GFX942-TGSPLIT-NEXT: s_endpgm15238;15239; GFX11-WGP-LABEL: local_workgroup_one_as_seq_cst_seq_cst_cmpxchg:15240; GFX11-WGP: ; %bb.0: ; %entry15241; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x015242; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x415243; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x815244; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)15245; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s215246; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s115247; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s015248; GFX11-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1615249; GFX11-WGP-NEXT: s_endpgm15250;15251; GFX11-CU-LABEL: local_workgroup_one_as_seq_cst_seq_cst_cmpxchg:15252; GFX11-CU: ; %bb.0: ; %entry15253; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x015254; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x415255; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x815256; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)15257; GFX11-CU-NEXT: v_mov_b32_e32 v0, s215258; GFX11-CU-NEXT: v_mov_b32_e32 v1, s115259; GFX11-CU-NEXT: v_mov_b32_e32 v2, s015260; GFX11-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1615261; GFX11-CU-NEXT: s_endpgm15262;15263; GFX12-WGP-LABEL: local_workgroup_one_as_seq_cst_seq_cst_cmpxchg:15264; GFX12-WGP: ; %bb.0: ; %entry15265; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x015266; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x415267; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x815268; GFX12-WGP-NEXT: s_wait_kmcnt 0x015269; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s215270; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s115271; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s015272; GFX12-WGP-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1615273; GFX12-WGP-NEXT: s_endpgm15274;15275; GFX12-CU-LABEL: local_workgroup_one_as_seq_cst_seq_cst_cmpxchg:15276; GFX12-CU: ; %bb.0: ; %entry15277; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x015278; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x415279; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x815280; GFX12-CU-NEXT: s_wait_kmcnt 0x015281; GFX12-CU-NEXT: v_mov_b32_e32 v0, s215282; GFX12-CU-NEXT: v_mov_b32_e32 v1, s115283; GFX12-CU-NEXT: v_mov_b32_e32 v2, s015284; GFX12-CU-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1615285; GFX12-CU-NEXT: s_endpgm15286;15287; GFX1250-LABEL: local_workgroup_one_as_seq_cst_seq_cst_cmpxchg:15288; GFX1250: ; %bb.0: ; %entry15289; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 115290; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x015291; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x415292; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x815293; GFX1250-NEXT: s_wait_kmcnt 0x015294; GFX1250-NEXT: v_mov_b32_e32 v0, s215295; GFX1250-NEXT: v_mov_b32_e32 v1, s115296; GFX1250-NEXT: v_mov_b32_e32 v2, s015297; GFX1250-NEXT: ds_cmpstore_b32 v0, v1, v2 offset:1615298; GFX1250-NEXT: s_endpgm15299 ptr addrspace(3) %out, i32 %in, i32 %old) {15300entry:15301 %gep = getelementptr i32, ptr addrspace(3) %out, i32 415302 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst15303 ret void15304}15305 15306define amdgpu_kernel void @local_workgroup_one_as_monotonic_monotonic_ret_cmpxchg(15307; GFX6-LABEL: local_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:15308; GFX6: ; %bb.0: ; %entry15309; GFX6-NEXT: s_load_dword s4, s[8:9], 0x015310; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr415311; GFX6-NEXT: s_load_dword s5, s[8:9], 0x115312; GFX6-NEXT: s_load_dword s6, s[8:9], 0x215313; GFX6-NEXT: s_mov_b32 m0, -115314; GFX6-NEXT: s_waitcnt lgkmcnt(0)15315; GFX6-NEXT: v_mov_b32_e32 v0, s415316; GFX6-NEXT: v_mov_b32_e32 v1, s615317; GFX6-NEXT: v_mov_b32_e32 v2, s515318; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615319; GFX6-NEXT: s_mov_b32 m0, -115320; GFX6-NEXT: v_mov_b32_e32 v0, s415321; GFX6-NEXT: s_waitcnt lgkmcnt(0)15322; GFX6-NEXT: ds_write_b32 v0, v115323; GFX6-NEXT: s_endpgm15324;15325; GFX7-LABEL: local_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:15326; GFX7: ; %bb.0: ; %entry15327; GFX7-NEXT: s_load_dword s4, s[8:9], 0x015328; GFX7-NEXT: s_load_dword s5, s[8:9], 0x115329; GFX7-NEXT: s_load_dword s6, s[8:9], 0x215330; GFX7-NEXT: s_mov_b32 m0, -115331; GFX7-NEXT: s_waitcnt lgkmcnt(0)15332; GFX7-NEXT: v_mov_b32_e32 v0, s415333; GFX7-NEXT: v_mov_b32_e32 v1, s615334; GFX7-NEXT: v_mov_b32_e32 v2, s515335; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615336; GFX7-NEXT: s_mov_b32 m0, -115337; GFX7-NEXT: v_mov_b32_e32 v0, s415338; GFX7-NEXT: s_waitcnt lgkmcnt(0)15339; GFX7-NEXT: ds_write_b32 v0, v115340; GFX7-NEXT: s_endpgm15341;15342; GFX10-WGP-LABEL: local_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:15343; GFX10-WGP: ; %bb.0: ; %entry15344; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x015345; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x415346; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x815347; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)15348; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s415349; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s615350; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s515351; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615352; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s415353; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)15354; GFX10-WGP-NEXT: ds_write_b32 v0, v115355; GFX10-WGP-NEXT: s_endpgm15356;15357; GFX10-CU-LABEL: local_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:15358; GFX10-CU: ; %bb.0: ; %entry15359; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x015360; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x415361; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x815362; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)15363; GFX10-CU-NEXT: v_mov_b32_e32 v0, s415364; GFX10-CU-NEXT: v_mov_b32_e32 v1, s615365; GFX10-CU-NEXT: v_mov_b32_e32 v2, s515366; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615367; GFX10-CU-NEXT: v_mov_b32_e32 v0, s415368; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)15369; GFX10-CU-NEXT: ds_write_b32 v0, v115370; GFX10-CU-NEXT: s_endpgm15371;15372; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:15373; SKIP-CACHE-INV: ; %bb.0: ; %entry15374; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x015375; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x115376; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x215377; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -115378; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)15379; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s015380; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s215381; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s115382; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615383; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -115384; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s015385; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)15386; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v115387; SKIP-CACHE-INV-NEXT: s_endpgm15388;15389; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:15390; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry15391; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x015392; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x415393; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x815394; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15395; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s415396; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s615397; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s515398; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615399; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s415400; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15401; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v115402; GFX90A-NOTTGSPLIT-NEXT: s_endpgm15403;15404; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:15405; GFX90A-TGSPLIT: ; %bb.0: ; %entry15406; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x015407; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x415408; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x815409; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15410; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s415411; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s615412; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s515413; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615414; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s415415; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15416; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v115417; GFX90A-TGSPLIT-NEXT: s_endpgm15418;15419; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:15420; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry15421; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x015422; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x415423; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x815424; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15425; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s015426; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s215427; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s115428; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615429; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s015430; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15431; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v115432; GFX942-NOTTGSPLIT-NEXT: s_endpgm15433;15434; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:15435; GFX942-TGSPLIT: ; %bb.0: ; %entry15436; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x015437; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x415438; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x815439; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15440; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s015441; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s215442; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s115443; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615444; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s015445; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15446; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v115447; GFX942-TGSPLIT-NEXT: s_endpgm15448;15449; GFX11-WGP-LABEL: local_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:15450; GFX11-WGP: ; %bb.0: ; %entry15451; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x015452; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x415453; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x815454; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)15455; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s015456; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s215457; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s115458; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1615459; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s015460; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)15461; GFX11-WGP-NEXT: ds_store_b32 v0, v115462; GFX11-WGP-NEXT: s_endpgm15463;15464; GFX11-CU-LABEL: local_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:15465; GFX11-CU: ; %bb.0: ; %entry15466; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x015467; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x415468; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x815469; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)15470; GFX11-CU-NEXT: v_mov_b32_e32 v0, s015471; GFX11-CU-NEXT: v_mov_b32_e32 v1, s215472; GFX11-CU-NEXT: v_mov_b32_e32 v2, s115473; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1615474; GFX11-CU-NEXT: v_mov_b32_e32 v0, s015475; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)15476; GFX11-CU-NEXT: ds_store_b32 v0, v115477; GFX11-CU-NEXT: s_endpgm15478;15479; GFX12-WGP-LABEL: local_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:15480; GFX12-WGP: ; %bb.0: ; %entry15481; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x015482; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x415483; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x815484; GFX12-WGP-NEXT: s_wait_kmcnt 0x015485; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s015486; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s215487; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s115488; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1615489; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s015490; GFX12-WGP-NEXT: s_wait_dscnt 0x015491; GFX12-WGP-NEXT: ds_store_b32 v0, v115492; GFX12-WGP-NEXT: s_endpgm15493;15494; GFX12-CU-LABEL: local_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:15495; GFX12-CU: ; %bb.0: ; %entry15496; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x015497; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x415498; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x815499; GFX12-CU-NEXT: s_wait_kmcnt 0x015500; GFX12-CU-NEXT: v_mov_b32_e32 v0, s015501; GFX12-CU-NEXT: v_mov_b32_e32 v1, s215502; GFX12-CU-NEXT: v_mov_b32_e32 v2, s115503; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1615504; GFX12-CU-NEXT: v_mov_b32_e32 v0, s015505; GFX12-CU-NEXT: s_wait_dscnt 0x015506; GFX12-CU-NEXT: ds_store_b32 v0, v115507; GFX12-CU-NEXT: s_endpgm15508;15509; GFX1250-LABEL: local_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:15510; GFX1250: ; %bb.0: ; %entry15511; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 115512; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x015513; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x415514; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x815515; GFX1250-NEXT: s_wait_kmcnt 0x015516; GFX1250-NEXT: v_mov_b32_e32 v0, s015517; GFX1250-NEXT: v_mov_b32_e32 v1, s215518; GFX1250-NEXT: v_mov_b32_e32 v2, s115519; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1615520; GFX1250-NEXT: v_mov_b32_e32 v0, s015521; GFX1250-NEXT: s_wait_dscnt 0x015522; GFX1250-NEXT: ds_store_b32 v0, v115523; GFX1250-NEXT: s_endpgm15524 ptr addrspace(3) %out, i32 %in, i32 %old) {15525entry:15526 %gep = getelementptr i32, ptr addrspace(3) %out, i32 415527 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic monotonic15528 %val0 = extractvalue { i32, i1 } %val, 015529 store i32 %val0, ptr addrspace(3) %out, align 415530 ret void15531}15532 15533define amdgpu_kernel void @local_workgroup_one_as_acquire_monotonic_ret_cmpxchg(15534; GFX6-LABEL: local_workgroup_one_as_acquire_monotonic_ret_cmpxchg:15535; GFX6: ; %bb.0: ; %entry15536; GFX6-NEXT: s_load_dword s4, s[8:9], 0x015537; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr415538; GFX6-NEXT: s_load_dword s5, s[8:9], 0x115539; GFX6-NEXT: s_load_dword s6, s[8:9], 0x215540; GFX6-NEXT: s_mov_b32 m0, -115541; GFX6-NEXT: s_waitcnt lgkmcnt(0)15542; GFX6-NEXT: v_mov_b32_e32 v0, s415543; GFX6-NEXT: v_mov_b32_e32 v1, s615544; GFX6-NEXT: v_mov_b32_e32 v2, s515545; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615546; GFX6-NEXT: s_mov_b32 m0, -115547; GFX6-NEXT: v_mov_b32_e32 v0, s415548; GFX6-NEXT: s_waitcnt lgkmcnt(0)15549; GFX6-NEXT: ds_write_b32 v0, v115550; GFX6-NEXT: s_endpgm15551;15552; GFX7-LABEL: local_workgroup_one_as_acquire_monotonic_ret_cmpxchg:15553; GFX7: ; %bb.0: ; %entry15554; GFX7-NEXT: s_load_dword s4, s[8:9], 0x015555; GFX7-NEXT: s_load_dword s5, s[8:9], 0x115556; GFX7-NEXT: s_load_dword s6, s[8:9], 0x215557; GFX7-NEXT: s_mov_b32 m0, -115558; GFX7-NEXT: s_waitcnt lgkmcnt(0)15559; GFX7-NEXT: v_mov_b32_e32 v0, s415560; GFX7-NEXT: v_mov_b32_e32 v1, s615561; GFX7-NEXT: v_mov_b32_e32 v2, s515562; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615563; GFX7-NEXT: s_mov_b32 m0, -115564; GFX7-NEXT: v_mov_b32_e32 v0, s415565; GFX7-NEXT: s_waitcnt lgkmcnt(0)15566; GFX7-NEXT: ds_write_b32 v0, v115567; GFX7-NEXT: s_endpgm15568;15569; GFX10-WGP-LABEL: local_workgroup_one_as_acquire_monotonic_ret_cmpxchg:15570; GFX10-WGP: ; %bb.0: ; %entry15571; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x015572; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x415573; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x815574; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)15575; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s415576; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s615577; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s515578; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615579; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s415580; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)15581; GFX10-WGP-NEXT: ds_write_b32 v0, v115582; GFX10-WGP-NEXT: s_endpgm15583;15584; GFX10-CU-LABEL: local_workgroup_one_as_acquire_monotonic_ret_cmpxchg:15585; GFX10-CU: ; %bb.0: ; %entry15586; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x015587; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x415588; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x815589; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)15590; GFX10-CU-NEXT: v_mov_b32_e32 v0, s415591; GFX10-CU-NEXT: v_mov_b32_e32 v1, s615592; GFX10-CU-NEXT: v_mov_b32_e32 v2, s515593; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615594; GFX10-CU-NEXT: v_mov_b32_e32 v0, s415595; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)15596; GFX10-CU-NEXT: ds_write_b32 v0, v115597; GFX10-CU-NEXT: s_endpgm15598;15599; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acquire_monotonic_ret_cmpxchg:15600; SKIP-CACHE-INV: ; %bb.0: ; %entry15601; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x015602; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x115603; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x215604; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -115605; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)15606; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s015607; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s215608; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s115609; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615610; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -115611; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s015612; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)15613; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v115614; SKIP-CACHE-INV-NEXT: s_endpgm15615;15616; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_acquire_monotonic_ret_cmpxchg:15617; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry15618; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x015619; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x415620; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x815621; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15622; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s415623; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s615624; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s515625; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615626; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s415627; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15628; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v115629; GFX90A-NOTTGSPLIT-NEXT: s_endpgm15630;15631; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_acquire_monotonic_ret_cmpxchg:15632; GFX90A-TGSPLIT: ; %bb.0: ; %entry15633; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x015634; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x415635; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x815636; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15637; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s415638; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s615639; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s515640; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615641; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s415642; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15643; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v115644; GFX90A-TGSPLIT-NEXT: s_endpgm15645;15646; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_acquire_monotonic_ret_cmpxchg:15647; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry15648; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x015649; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x415650; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x815651; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15652; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s015653; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s215654; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s115655; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615656; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s015657; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15658; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v115659; GFX942-NOTTGSPLIT-NEXT: s_endpgm15660;15661; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_acquire_monotonic_ret_cmpxchg:15662; GFX942-TGSPLIT: ; %bb.0: ; %entry15663; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x015664; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x415665; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x815666; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15667; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s015668; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s215669; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s115670; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615671; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s015672; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15673; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v115674; GFX942-TGSPLIT-NEXT: s_endpgm15675;15676; GFX11-WGP-LABEL: local_workgroup_one_as_acquire_monotonic_ret_cmpxchg:15677; GFX11-WGP: ; %bb.0: ; %entry15678; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x015679; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x415680; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x815681; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)15682; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s015683; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s215684; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s115685; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1615686; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s015687; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)15688; GFX11-WGP-NEXT: ds_store_b32 v0, v115689; GFX11-WGP-NEXT: s_endpgm15690;15691; GFX11-CU-LABEL: local_workgroup_one_as_acquire_monotonic_ret_cmpxchg:15692; GFX11-CU: ; %bb.0: ; %entry15693; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x015694; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x415695; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x815696; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)15697; GFX11-CU-NEXT: v_mov_b32_e32 v0, s015698; GFX11-CU-NEXT: v_mov_b32_e32 v1, s215699; GFX11-CU-NEXT: v_mov_b32_e32 v2, s115700; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1615701; GFX11-CU-NEXT: v_mov_b32_e32 v0, s015702; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)15703; GFX11-CU-NEXT: ds_store_b32 v0, v115704; GFX11-CU-NEXT: s_endpgm15705;15706; GFX12-WGP-LABEL: local_workgroup_one_as_acquire_monotonic_ret_cmpxchg:15707; GFX12-WGP: ; %bb.0: ; %entry15708; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x015709; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x415710; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x815711; GFX12-WGP-NEXT: s_wait_kmcnt 0x015712; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s015713; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s215714; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s115715; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1615716; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s015717; GFX12-WGP-NEXT: s_wait_dscnt 0x015718; GFX12-WGP-NEXT: ds_store_b32 v0, v115719; GFX12-WGP-NEXT: s_endpgm15720;15721; GFX12-CU-LABEL: local_workgroup_one_as_acquire_monotonic_ret_cmpxchg:15722; GFX12-CU: ; %bb.0: ; %entry15723; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x015724; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x415725; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x815726; GFX12-CU-NEXT: s_wait_kmcnt 0x015727; GFX12-CU-NEXT: v_mov_b32_e32 v0, s015728; GFX12-CU-NEXT: v_mov_b32_e32 v1, s215729; GFX12-CU-NEXT: v_mov_b32_e32 v2, s115730; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1615731; GFX12-CU-NEXT: v_mov_b32_e32 v0, s015732; GFX12-CU-NEXT: s_wait_dscnt 0x015733; GFX12-CU-NEXT: ds_store_b32 v0, v115734; GFX12-CU-NEXT: s_endpgm15735;15736; GFX1250-LABEL: local_workgroup_one_as_acquire_monotonic_ret_cmpxchg:15737; GFX1250: ; %bb.0: ; %entry15738; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 115739; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x015740; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x415741; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x815742; GFX1250-NEXT: s_wait_kmcnt 0x015743; GFX1250-NEXT: v_mov_b32_e32 v0, s015744; GFX1250-NEXT: v_mov_b32_e32 v1, s215745; GFX1250-NEXT: v_mov_b32_e32 v2, s115746; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1615747; GFX1250-NEXT: v_mov_b32_e32 v0, s015748; GFX1250-NEXT: s_wait_dscnt 0x015749; GFX1250-NEXT: ds_store_b32 v0, v115750; GFX1250-NEXT: s_endpgm15751 ptr addrspace(3) %out, i32 %in, i32 %old) {15752entry:15753 %gep = getelementptr i32, ptr addrspace(3) %out, i32 415754 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic15755 %val0 = extractvalue { i32, i1 } %val, 015756 store i32 %val0, ptr addrspace(3) %out, align 415757 ret void15758}15759 15760define amdgpu_kernel void @local_workgroup_one_as_release_monotonic_ret_cmpxchg(15761; GFX6-LABEL: local_workgroup_one_as_release_monotonic_ret_cmpxchg:15762; GFX6: ; %bb.0: ; %entry15763; GFX6-NEXT: s_load_dword s4, s[8:9], 0x015764; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr415765; GFX6-NEXT: s_load_dword s5, s[8:9], 0x115766; GFX6-NEXT: s_load_dword s6, s[8:9], 0x215767; GFX6-NEXT: s_mov_b32 m0, -115768; GFX6-NEXT: s_waitcnt lgkmcnt(0)15769; GFX6-NEXT: v_mov_b32_e32 v0, s415770; GFX6-NEXT: v_mov_b32_e32 v1, s615771; GFX6-NEXT: v_mov_b32_e32 v2, s515772; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615773; GFX6-NEXT: s_mov_b32 m0, -115774; GFX6-NEXT: v_mov_b32_e32 v0, s415775; GFX6-NEXT: s_waitcnt lgkmcnt(0)15776; GFX6-NEXT: ds_write_b32 v0, v115777; GFX6-NEXT: s_endpgm15778;15779; GFX7-LABEL: local_workgroup_one_as_release_monotonic_ret_cmpxchg:15780; GFX7: ; %bb.0: ; %entry15781; GFX7-NEXT: s_load_dword s4, s[8:9], 0x015782; GFX7-NEXT: s_load_dword s5, s[8:9], 0x115783; GFX7-NEXT: s_load_dword s6, s[8:9], 0x215784; GFX7-NEXT: s_mov_b32 m0, -115785; GFX7-NEXT: s_waitcnt lgkmcnt(0)15786; GFX7-NEXT: v_mov_b32_e32 v0, s415787; GFX7-NEXT: v_mov_b32_e32 v1, s615788; GFX7-NEXT: v_mov_b32_e32 v2, s515789; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615790; GFX7-NEXT: s_mov_b32 m0, -115791; GFX7-NEXT: v_mov_b32_e32 v0, s415792; GFX7-NEXT: s_waitcnt lgkmcnt(0)15793; GFX7-NEXT: ds_write_b32 v0, v115794; GFX7-NEXT: s_endpgm15795;15796; GFX10-WGP-LABEL: local_workgroup_one_as_release_monotonic_ret_cmpxchg:15797; GFX10-WGP: ; %bb.0: ; %entry15798; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x015799; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x415800; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x815801; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)15802; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s415803; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s615804; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s515805; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615806; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s415807; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)15808; GFX10-WGP-NEXT: ds_write_b32 v0, v115809; GFX10-WGP-NEXT: s_endpgm15810;15811; GFX10-CU-LABEL: local_workgroup_one_as_release_monotonic_ret_cmpxchg:15812; GFX10-CU: ; %bb.0: ; %entry15813; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x015814; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x415815; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x815816; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)15817; GFX10-CU-NEXT: v_mov_b32_e32 v0, s415818; GFX10-CU-NEXT: v_mov_b32_e32 v1, s615819; GFX10-CU-NEXT: v_mov_b32_e32 v2, s515820; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615821; GFX10-CU-NEXT: v_mov_b32_e32 v0, s415822; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)15823; GFX10-CU-NEXT: ds_write_b32 v0, v115824; GFX10-CU-NEXT: s_endpgm15825;15826; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_release_monotonic_ret_cmpxchg:15827; SKIP-CACHE-INV: ; %bb.0: ; %entry15828; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x015829; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x115830; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x215831; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -115832; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)15833; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s015834; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s215835; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s115836; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615837; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -115838; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s015839; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)15840; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v115841; SKIP-CACHE-INV-NEXT: s_endpgm15842;15843; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_release_monotonic_ret_cmpxchg:15844; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry15845; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x015846; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x415847; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x815848; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15849; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s415850; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s615851; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s515852; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615853; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s415854; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15855; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v115856; GFX90A-NOTTGSPLIT-NEXT: s_endpgm15857;15858; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_release_monotonic_ret_cmpxchg:15859; GFX90A-TGSPLIT: ; %bb.0: ; %entry15860; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x015861; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x415862; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x815863; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15864; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s415865; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s615866; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s515867; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615868; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s415869; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15870; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v115871; GFX90A-TGSPLIT-NEXT: s_endpgm15872;15873; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_release_monotonic_ret_cmpxchg:15874; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry15875; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x015876; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x415877; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x815878; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15879; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s015880; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s215881; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s115882; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615883; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s015884; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15885; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v115886; GFX942-NOTTGSPLIT-NEXT: s_endpgm15887;15888; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_release_monotonic_ret_cmpxchg:15889; GFX942-TGSPLIT: ; %bb.0: ; %entry15890; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x015891; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x415892; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x815893; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15894; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s015895; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s215896; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s115897; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1615898; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s015899; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)15900; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v115901; GFX942-TGSPLIT-NEXT: s_endpgm15902;15903; GFX11-WGP-LABEL: local_workgroup_one_as_release_monotonic_ret_cmpxchg:15904; GFX11-WGP: ; %bb.0: ; %entry15905; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x015906; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x415907; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x815908; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)15909; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s015910; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s215911; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s115912; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1615913; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s015914; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)15915; GFX11-WGP-NEXT: ds_store_b32 v0, v115916; GFX11-WGP-NEXT: s_endpgm15917;15918; GFX11-CU-LABEL: local_workgroup_one_as_release_monotonic_ret_cmpxchg:15919; GFX11-CU: ; %bb.0: ; %entry15920; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x015921; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x415922; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x815923; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)15924; GFX11-CU-NEXT: v_mov_b32_e32 v0, s015925; GFX11-CU-NEXT: v_mov_b32_e32 v1, s215926; GFX11-CU-NEXT: v_mov_b32_e32 v2, s115927; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1615928; GFX11-CU-NEXT: v_mov_b32_e32 v0, s015929; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)15930; GFX11-CU-NEXT: ds_store_b32 v0, v115931; GFX11-CU-NEXT: s_endpgm15932;15933; GFX12-WGP-LABEL: local_workgroup_one_as_release_monotonic_ret_cmpxchg:15934; GFX12-WGP: ; %bb.0: ; %entry15935; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x015936; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x415937; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x815938; GFX12-WGP-NEXT: s_wait_kmcnt 0x015939; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s015940; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s215941; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s115942; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1615943; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s015944; GFX12-WGP-NEXT: s_wait_dscnt 0x015945; GFX12-WGP-NEXT: ds_store_b32 v0, v115946; GFX12-WGP-NEXT: s_endpgm15947;15948; GFX12-CU-LABEL: local_workgroup_one_as_release_monotonic_ret_cmpxchg:15949; GFX12-CU: ; %bb.0: ; %entry15950; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x015951; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x415952; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x815953; GFX12-CU-NEXT: s_wait_kmcnt 0x015954; GFX12-CU-NEXT: v_mov_b32_e32 v0, s015955; GFX12-CU-NEXT: v_mov_b32_e32 v1, s215956; GFX12-CU-NEXT: v_mov_b32_e32 v2, s115957; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1615958; GFX12-CU-NEXT: v_mov_b32_e32 v0, s015959; GFX12-CU-NEXT: s_wait_dscnt 0x015960; GFX12-CU-NEXT: ds_store_b32 v0, v115961; GFX12-CU-NEXT: s_endpgm15962;15963; GFX1250-LABEL: local_workgroup_one_as_release_monotonic_ret_cmpxchg:15964; GFX1250: ; %bb.0: ; %entry15965; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 115966; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x015967; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x415968; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x815969; GFX1250-NEXT: s_wait_kmcnt 0x015970; GFX1250-NEXT: v_mov_b32_e32 v0, s015971; GFX1250-NEXT: v_mov_b32_e32 v1, s215972; GFX1250-NEXT: v_mov_b32_e32 v2, s115973; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1615974; GFX1250-NEXT: v_mov_b32_e32 v0, s015975; GFX1250-NEXT: s_wait_dscnt 0x015976; GFX1250-NEXT: ds_store_b32 v0, v115977; GFX1250-NEXT: s_endpgm15978 ptr addrspace(3) %out, i32 %in, i32 %old) {15979entry:15980 %gep = getelementptr i32, ptr addrspace(3) %out, i32 415981 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release monotonic15982 %val0 = extractvalue { i32, i1 } %val, 015983 store i32 %val0, ptr addrspace(3) %out, align 415984 ret void15985}15986 15987define amdgpu_kernel void @local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg(15988; GFX6-LABEL: local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:15989; GFX6: ; %bb.0: ; %entry15990; GFX6-NEXT: s_load_dword s4, s[8:9], 0x015991; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr415992; GFX6-NEXT: s_load_dword s5, s[8:9], 0x115993; GFX6-NEXT: s_load_dword s6, s[8:9], 0x215994; GFX6-NEXT: s_mov_b32 m0, -115995; GFX6-NEXT: s_waitcnt lgkmcnt(0)15996; GFX6-NEXT: v_mov_b32_e32 v0, s415997; GFX6-NEXT: v_mov_b32_e32 v1, s615998; GFX6-NEXT: v_mov_b32_e32 v2, s515999; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616000; GFX6-NEXT: s_mov_b32 m0, -116001; GFX6-NEXT: v_mov_b32_e32 v0, s416002; GFX6-NEXT: s_waitcnt lgkmcnt(0)16003; GFX6-NEXT: ds_write_b32 v0, v116004; GFX6-NEXT: s_endpgm16005;16006; GFX7-LABEL: local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:16007; GFX7: ; %bb.0: ; %entry16008; GFX7-NEXT: s_load_dword s4, s[8:9], 0x016009; GFX7-NEXT: s_load_dword s5, s[8:9], 0x116010; GFX7-NEXT: s_load_dword s6, s[8:9], 0x216011; GFX7-NEXT: s_mov_b32 m0, -116012; GFX7-NEXT: s_waitcnt lgkmcnt(0)16013; GFX7-NEXT: v_mov_b32_e32 v0, s416014; GFX7-NEXT: v_mov_b32_e32 v1, s616015; GFX7-NEXT: v_mov_b32_e32 v2, s516016; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616017; GFX7-NEXT: s_mov_b32 m0, -116018; GFX7-NEXT: v_mov_b32_e32 v0, s416019; GFX7-NEXT: s_waitcnt lgkmcnt(0)16020; GFX7-NEXT: ds_write_b32 v0, v116021; GFX7-NEXT: s_endpgm16022;16023; GFX10-WGP-LABEL: local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:16024; GFX10-WGP: ; %bb.0: ; %entry16025; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x016026; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x416027; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x816028; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)16029; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s416030; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s616031; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s516032; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616033; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s416034; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)16035; GFX10-WGP-NEXT: ds_write_b32 v0, v116036; GFX10-WGP-NEXT: s_endpgm16037;16038; GFX10-CU-LABEL: local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:16039; GFX10-CU: ; %bb.0: ; %entry16040; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x016041; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x416042; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x816043; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)16044; GFX10-CU-NEXT: v_mov_b32_e32 v0, s416045; GFX10-CU-NEXT: v_mov_b32_e32 v1, s616046; GFX10-CU-NEXT: v_mov_b32_e32 v2, s516047; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616048; GFX10-CU-NEXT: v_mov_b32_e32 v0, s416049; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)16050; GFX10-CU-NEXT: ds_write_b32 v0, v116051; GFX10-CU-NEXT: s_endpgm16052;16053; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:16054; SKIP-CACHE-INV: ; %bb.0: ; %entry16055; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x016056; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x116057; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x216058; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -116059; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)16060; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s016061; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s216062; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s116063; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616064; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -116065; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s016066; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)16067; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v116068; SKIP-CACHE-INV-NEXT: s_endpgm16069;16070; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:16071; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry16072; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x016073; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x416074; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x816075; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16076; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s416077; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s616078; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s516079; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616080; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s416081; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16082; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v116083; GFX90A-NOTTGSPLIT-NEXT: s_endpgm16084;16085; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:16086; GFX90A-TGSPLIT: ; %bb.0: ; %entry16087; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x016088; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x416089; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x816090; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16091; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s416092; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s616093; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s516094; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616095; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s416096; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16097; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v116098; GFX90A-TGSPLIT-NEXT: s_endpgm16099;16100; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:16101; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry16102; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x016103; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x416104; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x816105; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16106; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s016107; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s216108; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s116109; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616110; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s016111; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16112; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v116113; GFX942-NOTTGSPLIT-NEXT: s_endpgm16114;16115; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:16116; GFX942-TGSPLIT: ; %bb.0: ; %entry16117; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x016118; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x416119; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x816120; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16121; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s016122; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s216123; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s116124; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616125; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s016126; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16127; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v116128; GFX942-TGSPLIT-NEXT: s_endpgm16129;16130; GFX11-WGP-LABEL: local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:16131; GFX11-WGP: ; %bb.0: ; %entry16132; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x016133; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x416134; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x816135; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)16136; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s016137; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s216138; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s116139; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1616140; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s016141; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)16142; GFX11-WGP-NEXT: ds_store_b32 v0, v116143; GFX11-WGP-NEXT: s_endpgm16144;16145; GFX11-CU-LABEL: local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:16146; GFX11-CU: ; %bb.0: ; %entry16147; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x016148; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x416149; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x816150; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)16151; GFX11-CU-NEXT: v_mov_b32_e32 v0, s016152; GFX11-CU-NEXT: v_mov_b32_e32 v1, s216153; GFX11-CU-NEXT: v_mov_b32_e32 v2, s116154; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1616155; GFX11-CU-NEXT: v_mov_b32_e32 v0, s016156; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)16157; GFX11-CU-NEXT: ds_store_b32 v0, v116158; GFX11-CU-NEXT: s_endpgm16159;16160; GFX12-WGP-LABEL: local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:16161; GFX12-WGP: ; %bb.0: ; %entry16162; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x016163; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x416164; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x816165; GFX12-WGP-NEXT: s_wait_kmcnt 0x016166; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s016167; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s216168; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s116169; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1616170; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s016171; GFX12-WGP-NEXT: s_wait_dscnt 0x016172; GFX12-WGP-NEXT: ds_store_b32 v0, v116173; GFX12-WGP-NEXT: s_endpgm16174;16175; GFX12-CU-LABEL: local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:16176; GFX12-CU: ; %bb.0: ; %entry16177; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x016178; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x416179; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x816180; GFX12-CU-NEXT: s_wait_kmcnt 0x016181; GFX12-CU-NEXT: v_mov_b32_e32 v0, s016182; GFX12-CU-NEXT: v_mov_b32_e32 v1, s216183; GFX12-CU-NEXT: v_mov_b32_e32 v2, s116184; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1616185; GFX12-CU-NEXT: v_mov_b32_e32 v0, s016186; GFX12-CU-NEXT: s_wait_dscnt 0x016187; GFX12-CU-NEXT: ds_store_b32 v0, v116188; GFX12-CU-NEXT: s_endpgm16189;16190; GFX1250-LABEL: local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:16191; GFX1250: ; %bb.0: ; %entry16192; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 116193; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x016194; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x416195; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x816196; GFX1250-NEXT: s_wait_kmcnt 0x016197; GFX1250-NEXT: v_mov_b32_e32 v0, s016198; GFX1250-NEXT: v_mov_b32_e32 v1, s216199; GFX1250-NEXT: v_mov_b32_e32 v2, s116200; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1616201; GFX1250-NEXT: v_mov_b32_e32 v0, s016202; GFX1250-NEXT: s_wait_dscnt 0x016203; GFX1250-NEXT: ds_store_b32 v0, v116204; GFX1250-NEXT: s_endpgm16205 ptr addrspace(3) %out, i32 %in, i32 %old) {16206entry:16207 %gep = getelementptr i32, ptr addrspace(3) %out, i32 416208 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic16209 %val0 = extractvalue { i32, i1 } %val, 016210 store i32 %val0, ptr addrspace(3) %out, align 416211 ret void16212}16213 16214define amdgpu_kernel void @local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg(16215; GFX6-LABEL: local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:16216; GFX6: ; %bb.0: ; %entry16217; GFX6-NEXT: s_load_dword s4, s[8:9], 0x016218; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr416219; GFX6-NEXT: s_load_dword s5, s[8:9], 0x116220; GFX6-NEXT: s_load_dword s6, s[8:9], 0x216221; GFX6-NEXT: s_mov_b32 m0, -116222; GFX6-NEXT: s_waitcnt lgkmcnt(0)16223; GFX6-NEXT: v_mov_b32_e32 v0, s416224; GFX6-NEXT: v_mov_b32_e32 v1, s616225; GFX6-NEXT: v_mov_b32_e32 v2, s516226; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616227; GFX6-NEXT: s_mov_b32 m0, -116228; GFX6-NEXT: v_mov_b32_e32 v0, s416229; GFX6-NEXT: s_waitcnt lgkmcnt(0)16230; GFX6-NEXT: ds_write_b32 v0, v116231; GFX6-NEXT: s_endpgm16232;16233; GFX7-LABEL: local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:16234; GFX7: ; %bb.0: ; %entry16235; GFX7-NEXT: s_load_dword s4, s[8:9], 0x016236; GFX7-NEXT: s_load_dword s5, s[8:9], 0x116237; GFX7-NEXT: s_load_dword s6, s[8:9], 0x216238; GFX7-NEXT: s_mov_b32 m0, -116239; GFX7-NEXT: s_waitcnt lgkmcnt(0)16240; GFX7-NEXT: v_mov_b32_e32 v0, s416241; GFX7-NEXT: v_mov_b32_e32 v1, s616242; GFX7-NEXT: v_mov_b32_e32 v2, s516243; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616244; GFX7-NEXT: s_mov_b32 m0, -116245; GFX7-NEXT: v_mov_b32_e32 v0, s416246; GFX7-NEXT: s_waitcnt lgkmcnt(0)16247; GFX7-NEXT: ds_write_b32 v0, v116248; GFX7-NEXT: s_endpgm16249;16250; GFX10-WGP-LABEL: local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:16251; GFX10-WGP: ; %bb.0: ; %entry16252; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x016253; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x416254; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x816255; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)16256; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s416257; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s616258; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s516259; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616260; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s416261; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)16262; GFX10-WGP-NEXT: ds_write_b32 v0, v116263; GFX10-WGP-NEXT: s_endpgm16264;16265; GFX10-CU-LABEL: local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:16266; GFX10-CU: ; %bb.0: ; %entry16267; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x016268; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x416269; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x816270; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)16271; GFX10-CU-NEXT: v_mov_b32_e32 v0, s416272; GFX10-CU-NEXT: v_mov_b32_e32 v1, s616273; GFX10-CU-NEXT: v_mov_b32_e32 v2, s516274; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616275; GFX10-CU-NEXT: v_mov_b32_e32 v0, s416276; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)16277; GFX10-CU-NEXT: ds_write_b32 v0, v116278; GFX10-CU-NEXT: s_endpgm16279;16280; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:16281; SKIP-CACHE-INV: ; %bb.0: ; %entry16282; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x016283; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x116284; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x216285; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -116286; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)16287; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s016288; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s216289; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s116290; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616291; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -116292; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s016293; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)16294; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v116295; SKIP-CACHE-INV-NEXT: s_endpgm16296;16297; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:16298; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry16299; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x016300; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x416301; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x816302; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16303; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s416304; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s616305; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s516306; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616307; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s416308; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16309; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v116310; GFX90A-NOTTGSPLIT-NEXT: s_endpgm16311;16312; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:16313; GFX90A-TGSPLIT: ; %bb.0: ; %entry16314; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x016315; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x416316; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x816317; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16318; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s416319; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s616320; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s516321; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616322; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s416323; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16324; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v116325; GFX90A-TGSPLIT-NEXT: s_endpgm16326;16327; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:16328; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry16329; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x016330; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x416331; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x816332; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16333; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s016334; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s216335; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s116336; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616337; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s016338; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16339; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v116340; GFX942-NOTTGSPLIT-NEXT: s_endpgm16341;16342; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:16343; GFX942-TGSPLIT: ; %bb.0: ; %entry16344; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x016345; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x416346; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x816347; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16348; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s016349; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s216350; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s116351; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616352; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s016353; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16354; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v116355; GFX942-TGSPLIT-NEXT: s_endpgm16356;16357; GFX11-WGP-LABEL: local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:16358; GFX11-WGP: ; %bb.0: ; %entry16359; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x016360; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x416361; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x816362; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)16363; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s016364; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s216365; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s116366; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1616367; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s016368; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)16369; GFX11-WGP-NEXT: ds_store_b32 v0, v116370; GFX11-WGP-NEXT: s_endpgm16371;16372; GFX11-CU-LABEL: local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:16373; GFX11-CU: ; %bb.0: ; %entry16374; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x016375; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x416376; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x816377; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)16378; GFX11-CU-NEXT: v_mov_b32_e32 v0, s016379; GFX11-CU-NEXT: v_mov_b32_e32 v1, s216380; GFX11-CU-NEXT: v_mov_b32_e32 v2, s116381; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1616382; GFX11-CU-NEXT: v_mov_b32_e32 v0, s016383; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)16384; GFX11-CU-NEXT: ds_store_b32 v0, v116385; GFX11-CU-NEXT: s_endpgm16386;16387; GFX12-WGP-LABEL: local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:16388; GFX12-WGP: ; %bb.0: ; %entry16389; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x016390; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x416391; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x816392; GFX12-WGP-NEXT: s_wait_kmcnt 0x016393; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s016394; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s216395; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s116396; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1616397; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s016398; GFX12-WGP-NEXT: s_wait_dscnt 0x016399; GFX12-WGP-NEXT: ds_store_b32 v0, v116400; GFX12-WGP-NEXT: s_endpgm16401;16402; GFX12-CU-LABEL: local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:16403; GFX12-CU: ; %bb.0: ; %entry16404; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x016405; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x416406; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x816407; GFX12-CU-NEXT: s_wait_kmcnt 0x016408; GFX12-CU-NEXT: v_mov_b32_e32 v0, s016409; GFX12-CU-NEXT: v_mov_b32_e32 v1, s216410; GFX12-CU-NEXT: v_mov_b32_e32 v2, s116411; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1616412; GFX12-CU-NEXT: v_mov_b32_e32 v0, s016413; GFX12-CU-NEXT: s_wait_dscnt 0x016414; GFX12-CU-NEXT: ds_store_b32 v0, v116415; GFX12-CU-NEXT: s_endpgm16416;16417; GFX1250-LABEL: local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:16418; GFX1250: ; %bb.0: ; %entry16419; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 116420; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x016421; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x416422; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x816423; GFX1250-NEXT: s_wait_kmcnt 0x016424; GFX1250-NEXT: v_mov_b32_e32 v0, s016425; GFX1250-NEXT: v_mov_b32_e32 v1, s216426; GFX1250-NEXT: v_mov_b32_e32 v2, s116427; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1616428; GFX1250-NEXT: v_mov_b32_e32 v0, s016429; GFX1250-NEXT: s_wait_dscnt 0x016430; GFX1250-NEXT: ds_store_b32 v0, v116431; GFX1250-NEXT: s_endpgm16432 ptr addrspace(3) %out, i32 %in, i32 %old) {16433entry:16434 %gep = getelementptr i32, ptr addrspace(3) %out, i32 416435 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic16436 %val0 = extractvalue { i32, i1 } %val, 016437 store i32 %val0, ptr addrspace(3) %out, align 416438 ret void16439}16440 16441define amdgpu_kernel void @local_workgroup_one_as_monotonic_acquire_ret_cmpxchg(16442; GFX6-LABEL: local_workgroup_one_as_monotonic_acquire_ret_cmpxchg:16443; GFX6: ; %bb.0: ; %entry16444; GFX6-NEXT: s_load_dword s4, s[8:9], 0x016445; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr416446; GFX6-NEXT: s_load_dword s5, s[8:9], 0x116447; GFX6-NEXT: s_load_dword s6, s[8:9], 0x216448; GFX6-NEXT: s_mov_b32 m0, -116449; GFX6-NEXT: s_waitcnt lgkmcnt(0)16450; GFX6-NEXT: v_mov_b32_e32 v0, s416451; GFX6-NEXT: v_mov_b32_e32 v1, s616452; GFX6-NEXT: v_mov_b32_e32 v2, s516453; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616454; GFX6-NEXT: s_mov_b32 m0, -116455; GFX6-NEXT: v_mov_b32_e32 v0, s416456; GFX6-NEXT: s_waitcnt lgkmcnt(0)16457; GFX6-NEXT: ds_write_b32 v0, v116458; GFX6-NEXT: s_endpgm16459;16460; GFX7-LABEL: local_workgroup_one_as_monotonic_acquire_ret_cmpxchg:16461; GFX7: ; %bb.0: ; %entry16462; GFX7-NEXT: s_load_dword s4, s[8:9], 0x016463; GFX7-NEXT: s_load_dword s5, s[8:9], 0x116464; GFX7-NEXT: s_load_dword s6, s[8:9], 0x216465; GFX7-NEXT: s_mov_b32 m0, -116466; GFX7-NEXT: s_waitcnt lgkmcnt(0)16467; GFX7-NEXT: v_mov_b32_e32 v0, s416468; GFX7-NEXT: v_mov_b32_e32 v1, s616469; GFX7-NEXT: v_mov_b32_e32 v2, s516470; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616471; GFX7-NEXT: s_mov_b32 m0, -116472; GFX7-NEXT: v_mov_b32_e32 v0, s416473; GFX7-NEXT: s_waitcnt lgkmcnt(0)16474; GFX7-NEXT: ds_write_b32 v0, v116475; GFX7-NEXT: s_endpgm16476;16477; GFX10-WGP-LABEL: local_workgroup_one_as_monotonic_acquire_ret_cmpxchg:16478; GFX10-WGP: ; %bb.0: ; %entry16479; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x016480; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x416481; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x816482; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)16483; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s416484; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s616485; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s516486; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616487; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s416488; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)16489; GFX10-WGP-NEXT: ds_write_b32 v0, v116490; GFX10-WGP-NEXT: s_endpgm16491;16492; GFX10-CU-LABEL: local_workgroup_one_as_monotonic_acquire_ret_cmpxchg:16493; GFX10-CU: ; %bb.0: ; %entry16494; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x016495; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x416496; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x816497; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)16498; GFX10-CU-NEXT: v_mov_b32_e32 v0, s416499; GFX10-CU-NEXT: v_mov_b32_e32 v1, s616500; GFX10-CU-NEXT: v_mov_b32_e32 v2, s516501; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616502; GFX10-CU-NEXT: v_mov_b32_e32 v0, s416503; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)16504; GFX10-CU-NEXT: ds_write_b32 v0, v116505; GFX10-CU-NEXT: s_endpgm16506;16507; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_monotonic_acquire_ret_cmpxchg:16508; SKIP-CACHE-INV: ; %bb.0: ; %entry16509; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x016510; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x116511; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x216512; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -116513; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)16514; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s016515; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s216516; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s116517; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616518; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -116519; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s016520; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)16521; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v116522; SKIP-CACHE-INV-NEXT: s_endpgm16523;16524; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_monotonic_acquire_ret_cmpxchg:16525; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry16526; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x016527; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x416528; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x816529; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16530; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s416531; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s616532; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s516533; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616534; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s416535; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16536; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v116537; GFX90A-NOTTGSPLIT-NEXT: s_endpgm16538;16539; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_monotonic_acquire_ret_cmpxchg:16540; GFX90A-TGSPLIT: ; %bb.0: ; %entry16541; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x016542; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x416543; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x816544; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16545; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s416546; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s616547; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s516548; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616549; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s416550; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16551; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v116552; GFX90A-TGSPLIT-NEXT: s_endpgm16553;16554; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_monotonic_acquire_ret_cmpxchg:16555; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry16556; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x016557; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x416558; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x816559; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16560; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s016561; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s216562; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s116563; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616564; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s016565; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16566; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v116567; GFX942-NOTTGSPLIT-NEXT: s_endpgm16568;16569; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_monotonic_acquire_ret_cmpxchg:16570; GFX942-TGSPLIT: ; %bb.0: ; %entry16571; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x016572; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x416573; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x816574; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16575; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s016576; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s216577; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s116578; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616579; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s016580; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16581; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v116582; GFX942-TGSPLIT-NEXT: s_endpgm16583;16584; GFX11-WGP-LABEL: local_workgroup_one_as_monotonic_acquire_ret_cmpxchg:16585; GFX11-WGP: ; %bb.0: ; %entry16586; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x016587; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x416588; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x816589; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)16590; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s016591; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s216592; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s116593; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1616594; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s016595; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)16596; GFX11-WGP-NEXT: ds_store_b32 v0, v116597; GFX11-WGP-NEXT: s_endpgm16598;16599; GFX11-CU-LABEL: local_workgroup_one_as_monotonic_acquire_ret_cmpxchg:16600; GFX11-CU: ; %bb.0: ; %entry16601; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x016602; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x416603; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x816604; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)16605; GFX11-CU-NEXT: v_mov_b32_e32 v0, s016606; GFX11-CU-NEXT: v_mov_b32_e32 v1, s216607; GFX11-CU-NEXT: v_mov_b32_e32 v2, s116608; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1616609; GFX11-CU-NEXT: v_mov_b32_e32 v0, s016610; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)16611; GFX11-CU-NEXT: ds_store_b32 v0, v116612; GFX11-CU-NEXT: s_endpgm16613;16614; GFX12-WGP-LABEL: local_workgroup_one_as_monotonic_acquire_ret_cmpxchg:16615; GFX12-WGP: ; %bb.0: ; %entry16616; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x016617; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x416618; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x816619; GFX12-WGP-NEXT: s_wait_kmcnt 0x016620; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s016621; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s216622; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s116623; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1616624; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s016625; GFX12-WGP-NEXT: s_wait_dscnt 0x016626; GFX12-WGP-NEXT: ds_store_b32 v0, v116627; GFX12-WGP-NEXT: s_endpgm16628;16629; GFX12-CU-LABEL: local_workgroup_one_as_monotonic_acquire_ret_cmpxchg:16630; GFX12-CU: ; %bb.0: ; %entry16631; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x016632; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x416633; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x816634; GFX12-CU-NEXT: s_wait_kmcnt 0x016635; GFX12-CU-NEXT: v_mov_b32_e32 v0, s016636; GFX12-CU-NEXT: v_mov_b32_e32 v1, s216637; GFX12-CU-NEXT: v_mov_b32_e32 v2, s116638; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1616639; GFX12-CU-NEXT: v_mov_b32_e32 v0, s016640; GFX12-CU-NEXT: s_wait_dscnt 0x016641; GFX12-CU-NEXT: ds_store_b32 v0, v116642; GFX12-CU-NEXT: s_endpgm16643;16644; GFX1250-LABEL: local_workgroup_one_as_monotonic_acquire_ret_cmpxchg:16645; GFX1250: ; %bb.0: ; %entry16646; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 116647; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x016648; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x416649; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x816650; GFX1250-NEXT: s_wait_kmcnt 0x016651; GFX1250-NEXT: v_mov_b32_e32 v0, s016652; GFX1250-NEXT: v_mov_b32_e32 v1, s216653; GFX1250-NEXT: v_mov_b32_e32 v2, s116654; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1616655; GFX1250-NEXT: v_mov_b32_e32 v0, s016656; GFX1250-NEXT: s_wait_dscnt 0x016657; GFX1250-NEXT: ds_store_b32 v0, v116658; GFX1250-NEXT: s_endpgm16659 ptr addrspace(3) %out, i32 %in, i32 %old) {16660entry:16661 %gep = getelementptr i32, ptr addrspace(3) %out, i32 416662 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic acquire16663 %val0 = extractvalue { i32, i1 } %val, 016664 store i32 %val0, ptr addrspace(3) %out, align 416665 ret void16666}16667 16668define amdgpu_kernel void @local_workgroup_one_as_acquire_acquire_ret_cmpxchg(16669; GFX6-LABEL: local_workgroup_one_as_acquire_acquire_ret_cmpxchg:16670; GFX6: ; %bb.0: ; %entry16671; GFX6-NEXT: s_load_dword s4, s[8:9], 0x016672; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr416673; GFX6-NEXT: s_load_dword s5, s[8:9], 0x116674; GFX6-NEXT: s_load_dword s6, s[8:9], 0x216675; GFX6-NEXT: s_mov_b32 m0, -116676; GFX6-NEXT: s_waitcnt lgkmcnt(0)16677; GFX6-NEXT: v_mov_b32_e32 v0, s416678; GFX6-NEXT: v_mov_b32_e32 v1, s616679; GFX6-NEXT: v_mov_b32_e32 v2, s516680; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616681; GFX6-NEXT: s_mov_b32 m0, -116682; GFX6-NEXT: v_mov_b32_e32 v0, s416683; GFX6-NEXT: s_waitcnt lgkmcnt(0)16684; GFX6-NEXT: ds_write_b32 v0, v116685; GFX6-NEXT: s_endpgm16686;16687; GFX7-LABEL: local_workgroup_one_as_acquire_acquire_ret_cmpxchg:16688; GFX7: ; %bb.0: ; %entry16689; GFX7-NEXT: s_load_dword s4, s[8:9], 0x016690; GFX7-NEXT: s_load_dword s5, s[8:9], 0x116691; GFX7-NEXT: s_load_dword s6, s[8:9], 0x216692; GFX7-NEXT: s_mov_b32 m0, -116693; GFX7-NEXT: s_waitcnt lgkmcnt(0)16694; GFX7-NEXT: v_mov_b32_e32 v0, s416695; GFX7-NEXT: v_mov_b32_e32 v1, s616696; GFX7-NEXT: v_mov_b32_e32 v2, s516697; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616698; GFX7-NEXT: s_mov_b32 m0, -116699; GFX7-NEXT: v_mov_b32_e32 v0, s416700; GFX7-NEXT: s_waitcnt lgkmcnt(0)16701; GFX7-NEXT: ds_write_b32 v0, v116702; GFX7-NEXT: s_endpgm16703;16704; GFX10-WGP-LABEL: local_workgroup_one_as_acquire_acquire_ret_cmpxchg:16705; GFX10-WGP: ; %bb.0: ; %entry16706; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x016707; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x416708; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x816709; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)16710; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s416711; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s616712; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s516713; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616714; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s416715; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)16716; GFX10-WGP-NEXT: ds_write_b32 v0, v116717; GFX10-WGP-NEXT: s_endpgm16718;16719; GFX10-CU-LABEL: local_workgroup_one_as_acquire_acquire_ret_cmpxchg:16720; GFX10-CU: ; %bb.0: ; %entry16721; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x016722; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x416723; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x816724; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)16725; GFX10-CU-NEXT: v_mov_b32_e32 v0, s416726; GFX10-CU-NEXT: v_mov_b32_e32 v1, s616727; GFX10-CU-NEXT: v_mov_b32_e32 v2, s516728; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616729; GFX10-CU-NEXT: v_mov_b32_e32 v0, s416730; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)16731; GFX10-CU-NEXT: ds_write_b32 v0, v116732; GFX10-CU-NEXT: s_endpgm16733;16734; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acquire_acquire_ret_cmpxchg:16735; SKIP-CACHE-INV: ; %bb.0: ; %entry16736; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x016737; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x116738; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x216739; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -116740; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)16741; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s016742; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s216743; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s116744; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616745; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -116746; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s016747; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)16748; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v116749; SKIP-CACHE-INV-NEXT: s_endpgm16750;16751; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_acquire_acquire_ret_cmpxchg:16752; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry16753; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x016754; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x416755; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x816756; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16757; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s416758; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s616759; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s516760; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616761; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s416762; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16763; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v116764; GFX90A-NOTTGSPLIT-NEXT: s_endpgm16765;16766; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_acquire_acquire_ret_cmpxchg:16767; GFX90A-TGSPLIT: ; %bb.0: ; %entry16768; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x016769; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x416770; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x816771; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16772; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s416773; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s616774; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s516775; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616776; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s416777; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16778; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v116779; GFX90A-TGSPLIT-NEXT: s_endpgm16780;16781; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_acquire_acquire_ret_cmpxchg:16782; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry16783; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x016784; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x416785; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x816786; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16787; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s016788; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s216789; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s116790; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616791; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s016792; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16793; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v116794; GFX942-NOTTGSPLIT-NEXT: s_endpgm16795;16796; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_acquire_acquire_ret_cmpxchg:16797; GFX942-TGSPLIT: ; %bb.0: ; %entry16798; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x016799; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x416800; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x816801; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16802; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s016803; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s216804; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s116805; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616806; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s016807; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16808; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v116809; GFX942-TGSPLIT-NEXT: s_endpgm16810;16811; GFX11-WGP-LABEL: local_workgroup_one_as_acquire_acquire_ret_cmpxchg:16812; GFX11-WGP: ; %bb.0: ; %entry16813; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x016814; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x416815; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x816816; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)16817; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s016818; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s216819; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s116820; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1616821; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s016822; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)16823; GFX11-WGP-NEXT: ds_store_b32 v0, v116824; GFX11-WGP-NEXT: s_endpgm16825;16826; GFX11-CU-LABEL: local_workgroup_one_as_acquire_acquire_ret_cmpxchg:16827; GFX11-CU: ; %bb.0: ; %entry16828; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x016829; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x416830; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x816831; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)16832; GFX11-CU-NEXT: v_mov_b32_e32 v0, s016833; GFX11-CU-NEXT: v_mov_b32_e32 v1, s216834; GFX11-CU-NEXT: v_mov_b32_e32 v2, s116835; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1616836; GFX11-CU-NEXT: v_mov_b32_e32 v0, s016837; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)16838; GFX11-CU-NEXT: ds_store_b32 v0, v116839; GFX11-CU-NEXT: s_endpgm16840;16841; GFX12-WGP-LABEL: local_workgroup_one_as_acquire_acquire_ret_cmpxchg:16842; GFX12-WGP: ; %bb.0: ; %entry16843; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x016844; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x416845; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x816846; GFX12-WGP-NEXT: s_wait_kmcnt 0x016847; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s016848; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s216849; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s116850; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1616851; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s016852; GFX12-WGP-NEXT: s_wait_dscnt 0x016853; GFX12-WGP-NEXT: ds_store_b32 v0, v116854; GFX12-WGP-NEXT: s_endpgm16855;16856; GFX12-CU-LABEL: local_workgroup_one_as_acquire_acquire_ret_cmpxchg:16857; GFX12-CU: ; %bb.0: ; %entry16858; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x016859; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x416860; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x816861; GFX12-CU-NEXT: s_wait_kmcnt 0x016862; GFX12-CU-NEXT: v_mov_b32_e32 v0, s016863; GFX12-CU-NEXT: v_mov_b32_e32 v1, s216864; GFX12-CU-NEXT: v_mov_b32_e32 v2, s116865; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1616866; GFX12-CU-NEXT: v_mov_b32_e32 v0, s016867; GFX12-CU-NEXT: s_wait_dscnt 0x016868; GFX12-CU-NEXT: ds_store_b32 v0, v116869; GFX12-CU-NEXT: s_endpgm16870;16871; GFX1250-LABEL: local_workgroup_one_as_acquire_acquire_ret_cmpxchg:16872; GFX1250: ; %bb.0: ; %entry16873; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 116874; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x016875; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x416876; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x816877; GFX1250-NEXT: s_wait_kmcnt 0x016878; GFX1250-NEXT: v_mov_b32_e32 v0, s016879; GFX1250-NEXT: v_mov_b32_e32 v1, s216880; GFX1250-NEXT: v_mov_b32_e32 v2, s116881; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1616882; GFX1250-NEXT: v_mov_b32_e32 v0, s016883; GFX1250-NEXT: s_wait_dscnt 0x016884; GFX1250-NEXT: ds_store_b32 v0, v116885; GFX1250-NEXT: s_endpgm16886 ptr addrspace(3) %out, i32 %in, i32 %old) {16887entry:16888 %gep = getelementptr i32, ptr addrspace(3) %out, i32 416889 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire16890 %val0 = extractvalue { i32, i1 } %val, 016891 store i32 %val0, ptr addrspace(3) %out, align 416892 ret void16893}16894 16895define amdgpu_kernel void @local_workgroup_one_as_release_acquire_ret_cmpxchg(16896; GFX6-LABEL: local_workgroup_one_as_release_acquire_ret_cmpxchg:16897; GFX6: ; %bb.0: ; %entry16898; GFX6-NEXT: s_load_dword s4, s[8:9], 0x016899; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr416900; GFX6-NEXT: s_load_dword s5, s[8:9], 0x116901; GFX6-NEXT: s_load_dword s6, s[8:9], 0x216902; GFX6-NEXT: s_mov_b32 m0, -116903; GFX6-NEXT: s_waitcnt lgkmcnt(0)16904; GFX6-NEXT: v_mov_b32_e32 v0, s416905; GFX6-NEXT: v_mov_b32_e32 v1, s616906; GFX6-NEXT: v_mov_b32_e32 v2, s516907; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616908; GFX6-NEXT: s_mov_b32 m0, -116909; GFX6-NEXT: v_mov_b32_e32 v0, s416910; GFX6-NEXT: s_waitcnt lgkmcnt(0)16911; GFX6-NEXT: ds_write_b32 v0, v116912; GFX6-NEXT: s_endpgm16913;16914; GFX7-LABEL: local_workgroup_one_as_release_acquire_ret_cmpxchg:16915; GFX7: ; %bb.0: ; %entry16916; GFX7-NEXT: s_load_dword s4, s[8:9], 0x016917; GFX7-NEXT: s_load_dword s5, s[8:9], 0x116918; GFX7-NEXT: s_load_dword s6, s[8:9], 0x216919; GFX7-NEXT: s_mov_b32 m0, -116920; GFX7-NEXT: s_waitcnt lgkmcnt(0)16921; GFX7-NEXT: v_mov_b32_e32 v0, s416922; GFX7-NEXT: v_mov_b32_e32 v1, s616923; GFX7-NEXT: v_mov_b32_e32 v2, s516924; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616925; GFX7-NEXT: s_mov_b32 m0, -116926; GFX7-NEXT: v_mov_b32_e32 v0, s416927; GFX7-NEXT: s_waitcnt lgkmcnt(0)16928; GFX7-NEXT: ds_write_b32 v0, v116929; GFX7-NEXT: s_endpgm16930;16931; GFX10-WGP-LABEL: local_workgroup_one_as_release_acquire_ret_cmpxchg:16932; GFX10-WGP: ; %bb.0: ; %entry16933; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x016934; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x416935; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x816936; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)16937; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s416938; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s616939; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s516940; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616941; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s416942; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)16943; GFX10-WGP-NEXT: ds_write_b32 v0, v116944; GFX10-WGP-NEXT: s_endpgm16945;16946; GFX10-CU-LABEL: local_workgroup_one_as_release_acquire_ret_cmpxchg:16947; GFX10-CU: ; %bb.0: ; %entry16948; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x016949; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x416950; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x816951; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)16952; GFX10-CU-NEXT: v_mov_b32_e32 v0, s416953; GFX10-CU-NEXT: v_mov_b32_e32 v1, s616954; GFX10-CU-NEXT: v_mov_b32_e32 v2, s516955; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616956; GFX10-CU-NEXT: v_mov_b32_e32 v0, s416957; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)16958; GFX10-CU-NEXT: ds_write_b32 v0, v116959; GFX10-CU-NEXT: s_endpgm16960;16961; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_release_acquire_ret_cmpxchg:16962; SKIP-CACHE-INV: ; %bb.0: ; %entry16963; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x016964; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x116965; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x216966; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -116967; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)16968; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s016969; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s216970; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s116971; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616972; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -116973; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s016974; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)16975; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v116976; SKIP-CACHE-INV-NEXT: s_endpgm16977;16978; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_release_acquire_ret_cmpxchg:16979; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry16980; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x016981; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x416982; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x816983; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16984; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s416985; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s616986; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s516987; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1616988; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s416989; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16990; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v116991; GFX90A-NOTTGSPLIT-NEXT: s_endpgm16992;16993; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_release_acquire_ret_cmpxchg:16994; GFX90A-TGSPLIT: ; %bb.0: ; %entry16995; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x016996; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x416997; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x816998; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)16999; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s417000; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s617001; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s517002; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617003; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s417004; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17005; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v117006; GFX90A-TGSPLIT-NEXT: s_endpgm17007;17008; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_release_acquire_ret_cmpxchg:17009; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry17010; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x017011; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x417012; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x817013; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17014; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s017015; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s217016; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s117017; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617018; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s017019; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17020; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v117021; GFX942-NOTTGSPLIT-NEXT: s_endpgm17022;17023; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_release_acquire_ret_cmpxchg:17024; GFX942-TGSPLIT: ; %bb.0: ; %entry17025; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x017026; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x417027; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x817028; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17029; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s017030; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s217031; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s117032; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617033; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s017034; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17035; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v117036; GFX942-TGSPLIT-NEXT: s_endpgm17037;17038; GFX11-WGP-LABEL: local_workgroup_one_as_release_acquire_ret_cmpxchg:17039; GFX11-WGP: ; %bb.0: ; %entry17040; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x017041; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x417042; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x817043; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)17044; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s017045; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s217046; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s117047; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1617048; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s017049; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)17050; GFX11-WGP-NEXT: ds_store_b32 v0, v117051; GFX11-WGP-NEXT: s_endpgm17052;17053; GFX11-CU-LABEL: local_workgroup_one_as_release_acquire_ret_cmpxchg:17054; GFX11-CU: ; %bb.0: ; %entry17055; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x017056; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x417057; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x817058; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)17059; GFX11-CU-NEXT: v_mov_b32_e32 v0, s017060; GFX11-CU-NEXT: v_mov_b32_e32 v1, s217061; GFX11-CU-NEXT: v_mov_b32_e32 v2, s117062; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1617063; GFX11-CU-NEXT: v_mov_b32_e32 v0, s017064; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)17065; GFX11-CU-NEXT: ds_store_b32 v0, v117066; GFX11-CU-NEXT: s_endpgm17067;17068; GFX12-WGP-LABEL: local_workgroup_one_as_release_acquire_ret_cmpxchg:17069; GFX12-WGP: ; %bb.0: ; %entry17070; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x017071; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x417072; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x817073; GFX12-WGP-NEXT: s_wait_kmcnt 0x017074; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s017075; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s217076; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s117077; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1617078; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s017079; GFX12-WGP-NEXT: s_wait_dscnt 0x017080; GFX12-WGP-NEXT: ds_store_b32 v0, v117081; GFX12-WGP-NEXT: s_endpgm17082;17083; GFX12-CU-LABEL: local_workgroup_one_as_release_acquire_ret_cmpxchg:17084; GFX12-CU: ; %bb.0: ; %entry17085; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x017086; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x417087; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x817088; GFX12-CU-NEXT: s_wait_kmcnt 0x017089; GFX12-CU-NEXT: v_mov_b32_e32 v0, s017090; GFX12-CU-NEXT: v_mov_b32_e32 v1, s217091; GFX12-CU-NEXT: v_mov_b32_e32 v2, s117092; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1617093; GFX12-CU-NEXT: v_mov_b32_e32 v0, s017094; GFX12-CU-NEXT: s_wait_dscnt 0x017095; GFX12-CU-NEXT: ds_store_b32 v0, v117096; GFX12-CU-NEXT: s_endpgm17097;17098; GFX1250-LABEL: local_workgroup_one_as_release_acquire_ret_cmpxchg:17099; GFX1250: ; %bb.0: ; %entry17100; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 117101; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x017102; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x417103; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x817104; GFX1250-NEXT: s_wait_kmcnt 0x017105; GFX1250-NEXT: v_mov_b32_e32 v0, s017106; GFX1250-NEXT: v_mov_b32_e32 v1, s217107; GFX1250-NEXT: v_mov_b32_e32 v2, s117108; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1617109; GFX1250-NEXT: v_mov_b32_e32 v0, s017110; GFX1250-NEXT: s_wait_dscnt 0x017111; GFX1250-NEXT: ds_store_b32 v0, v117112; GFX1250-NEXT: s_endpgm17113 ptr addrspace(3) %out, i32 %in, i32 %old) {17114entry:17115 %gep = getelementptr i32, ptr addrspace(3) %out, i32 417116 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire17117 %val0 = extractvalue { i32, i1 } %val, 017118 store i32 %val0, ptr addrspace(3) %out, align 417119 ret void17120}17121 17122define amdgpu_kernel void @local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(17123; GFX6-LABEL: local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:17124; GFX6: ; %bb.0: ; %entry17125; GFX6-NEXT: s_load_dword s4, s[8:9], 0x017126; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr417127; GFX6-NEXT: s_load_dword s5, s[8:9], 0x117128; GFX6-NEXT: s_load_dword s6, s[8:9], 0x217129; GFX6-NEXT: s_mov_b32 m0, -117130; GFX6-NEXT: s_waitcnt lgkmcnt(0)17131; GFX6-NEXT: v_mov_b32_e32 v0, s417132; GFX6-NEXT: v_mov_b32_e32 v1, s617133; GFX6-NEXT: v_mov_b32_e32 v2, s517134; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617135; GFX6-NEXT: s_mov_b32 m0, -117136; GFX6-NEXT: v_mov_b32_e32 v0, s417137; GFX6-NEXT: s_waitcnt lgkmcnt(0)17138; GFX6-NEXT: ds_write_b32 v0, v117139; GFX6-NEXT: s_endpgm17140;17141; GFX7-LABEL: local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:17142; GFX7: ; %bb.0: ; %entry17143; GFX7-NEXT: s_load_dword s4, s[8:9], 0x017144; GFX7-NEXT: s_load_dword s5, s[8:9], 0x117145; GFX7-NEXT: s_load_dword s6, s[8:9], 0x217146; GFX7-NEXT: s_mov_b32 m0, -117147; GFX7-NEXT: s_waitcnt lgkmcnt(0)17148; GFX7-NEXT: v_mov_b32_e32 v0, s417149; GFX7-NEXT: v_mov_b32_e32 v1, s617150; GFX7-NEXT: v_mov_b32_e32 v2, s517151; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617152; GFX7-NEXT: s_mov_b32 m0, -117153; GFX7-NEXT: v_mov_b32_e32 v0, s417154; GFX7-NEXT: s_waitcnt lgkmcnt(0)17155; GFX7-NEXT: ds_write_b32 v0, v117156; GFX7-NEXT: s_endpgm17157;17158; GFX10-WGP-LABEL: local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:17159; GFX10-WGP: ; %bb.0: ; %entry17160; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x017161; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x417162; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x817163; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)17164; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s417165; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s617166; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s517167; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617168; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s417169; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)17170; GFX10-WGP-NEXT: ds_write_b32 v0, v117171; GFX10-WGP-NEXT: s_endpgm17172;17173; GFX10-CU-LABEL: local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:17174; GFX10-CU: ; %bb.0: ; %entry17175; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x017176; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x417177; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x817178; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)17179; GFX10-CU-NEXT: v_mov_b32_e32 v0, s417180; GFX10-CU-NEXT: v_mov_b32_e32 v1, s617181; GFX10-CU-NEXT: v_mov_b32_e32 v2, s517182; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617183; GFX10-CU-NEXT: v_mov_b32_e32 v0, s417184; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)17185; GFX10-CU-NEXT: ds_write_b32 v0, v117186; GFX10-CU-NEXT: s_endpgm17187;17188; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:17189; SKIP-CACHE-INV: ; %bb.0: ; %entry17190; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x017191; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x117192; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x217193; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -117194; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)17195; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s017196; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s217197; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s117198; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617199; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -117200; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s017201; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)17202; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v117203; SKIP-CACHE-INV-NEXT: s_endpgm17204;17205; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:17206; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry17207; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x017208; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x417209; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x817210; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17211; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s417212; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s617213; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s517214; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617215; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s417216; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17217; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v117218; GFX90A-NOTTGSPLIT-NEXT: s_endpgm17219;17220; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:17221; GFX90A-TGSPLIT: ; %bb.0: ; %entry17222; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x017223; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x417224; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x817225; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17226; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s417227; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s617228; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s517229; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617230; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s417231; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17232; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v117233; GFX90A-TGSPLIT-NEXT: s_endpgm17234;17235; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:17236; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry17237; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x017238; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x417239; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x817240; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17241; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s017242; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s217243; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s117244; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617245; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s017246; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17247; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v117248; GFX942-NOTTGSPLIT-NEXT: s_endpgm17249;17250; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:17251; GFX942-TGSPLIT: ; %bb.0: ; %entry17252; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x017253; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x417254; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x817255; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17256; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s017257; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s217258; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s117259; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617260; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s017261; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17262; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v117263; GFX942-TGSPLIT-NEXT: s_endpgm17264;17265; GFX11-WGP-LABEL: local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:17266; GFX11-WGP: ; %bb.0: ; %entry17267; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x017268; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x417269; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x817270; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)17271; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s017272; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s217273; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s117274; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1617275; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s017276; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)17277; GFX11-WGP-NEXT: ds_store_b32 v0, v117278; GFX11-WGP-NEXT: s_endpgm17279;17280; GFX11-CU-LABEL: local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:17281; GFX11-CU: ; %bb.0: ; %entry17282; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x017283; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x417284; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x817285; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)17286; GFX11-CU-NEXT: v_mov_b32_e32 v0, s017287; GFX11-CU-NEXT: v_mov_b32_e32 v1, s217288; GFX11-CU-NEXT: v_mov_b32_e32 v2, s117289; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1617290; GFX11-CU-NEXT: v_mov_b32_e32 v0, s017291; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)17292; GFX11-CU-NEXT: ds_store_b32 v0, v117293; GFX11-CU-NEXT: s_endpgm17294;17295; GFX12-WGP-LABEL: local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:17296; GFX12-WGP: ; %bb.0: ; %entry17297; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x017298; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x417299; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x817300; GFX12-WGP-NEXT: s_wait_kmcnt 0x017301; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s017302; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s217303; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s117304; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1617305; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s017306; GFX12-WGP-NEXT: s_wait_dscnt 0x017307; GFX12-WGP-NEXT: ds_store_b32 v0, v117308; GFX12-WGP-NEXT: s_endpgm17309;17310; GFX12-CU-LABEL: local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:17311; GFX12-CU: ; %bb.0: ; %entry17312; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x017313; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x417314; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x817315; GFX12-CU-NEXT: s_wait_kmcnt 0x017316; GFX12-CU-NEXT: v_mov_b32_e32 v0, s017317; GFX12-CU-NEXT: v_mov_b32_e32 v1, s217318; GFX12-CU-NEXT: v_mov_b32_e32 v2, s117319; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1617320; GFX12-CU-NEXT: v_mov_b32_e32 v0, s017321; GFX12-CU-NEXT: s_wait_dscnt 0x017322; GFX12-CU-NEXT: ds_store_b32 v0, v117323; GFX12-CU-NEXT: s_endpgm17324;17325; GFX1250-LABEL: local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:17326; GFX1250: ; %bb.0: ; %entry17327; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 117328; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x017329; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x417330; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x817331; GFX1250-NEXT: s_wait_kmcnt 0x017332; GFX1250-NEXT: v_mov_b32_e32 v0, s017333; GFX1250-NEXT: v_mov_b32_e32 v1, s217334; GFX1250-NEXT: v_mov_b32_e32 v2, s117335; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1617336; GFX1250-NEXT: v_mov_b32_e32 v0, s017337; GFX1250-NEXT: s_wait_dscnt 0x017338; GFX1250-NEXT: ds_store_b32 v0, v117339; GFX1250-NEXT: s_endpgm17340 ptr addrspace(3) %out, i32 %in, i32 %old) {17341entry:17342 %gep = getelementptr i32, ptr addrspace(3) %out, i32 417343 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire17344 %val0 = extractvalue { i32, i1 } %val, 017345 store i32 %val0, ptr addrspace(3) %out, align 417346 ret void17347}17348 17349define amdgpu_kernel void @local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(17350; GFX6-LABEL: local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:17351; GFX6: ; %bb.0: ; %entry17352; GFX6-NEXT: s_load_dword s4, s[8:9], 0x017353; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr417354; GFX6-NEXT: s_load_dword s5, s[8:9], 0x117355; GFX6-NEXT: s_load_dword s6, s[8:9], 0x217356; GFX6-NEXT: s_mov_b32 m0, -117357; GFX6-NEXT: s_waitcnt lgkmcnt(0)17358; GFX6-NEXT: v_mov_b32_e32 v0, s417359; GFX6-NEXT: v_mov_b32_e32 v1, s617360; GFX6-NEXT: v_mov_b32_e32 v2, s517361; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617362; GFX6-NEXT: s_mov_b32 m0, -117363; GFX6-NEXT: v_mov_b32_e32 v0, s417364; GFX6-NEXT: s_waitcnt lgkmcnt(0)17365; GFX6-NEXT: ds_write_b32 v0, v117366; GFX6-NEXT: s_endpgm17367;17368; GFX7-LABEL: local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:17369; GFX7: ; %bb.0: ; %entry17370; GFX7-NEXT: s_load_dword s4, s[8:9], 0x017371; GFX7-NEXT: s_load_dword s5, s[8:9], 0x117372; GFX7-NEXT: s_load_dword s6, s[8:9], 0x217373; GFX7-NEXT: s_mov_b32 m0, -117374; GFX7-NEXT: s_waitcnt lgkmcnt(0)17375; GFX7-NEXT: v_mov_b32_e32 v0, s417376; GFX7-NEXT: v_mov_b32_e32 v1, s617377; GFX7-NEXT: v_mov_b32_e32 v2, s517378; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617379; GFX7-NEXT: s_mov_b32 m0, -117380; GFX7-NEXT: v_mov_b32_e32 v0, s417381; GFX7-NEXT: s_waitcnt lgkmcnt(0)17382; GFX7-NEXT: ds_write_b32 v0, v117383; GFX7-NEXT: s_endpgm17384;17385; GFX10-WGP-LABEL: local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:17386; GFX10-WGP: ; %bb.0: ; %entry17387; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x017388; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x417389; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x817390; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)17391; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s417392; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s617393; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s517394; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617395; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s417396; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)17397; GFX10-WGP-NEXT: ds_write_b32 v0, v117398; GFX10-WGP-NEXT: s_endpgm17399;17400; GFX10-CU-LABEL: local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:17401; GFX10-CU: ; %bb.0: ; %entry17402; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x017403; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x417404; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x817405; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)17406; GFX10-CU-NEXT: v_mov_b32_e32 v0, s417407; GFX10-CU-NEXT: v_mov_b32_e32 v1, s617408; GFX10-CU-NEXT: v_mov_b32_e32 v2, s517409; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617410; GFX10-CU-NEXT: v_mov_b32_e32 v0, s417411; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)17412; GFX10-CU-NEXT: ds_write_b32 v0, v117413; GFX10-CU-NEXT: s_endpgm17414;17415; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:17416; SKIP-CACHE-INV: ; %bb.0: ; %entry17417; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x017418; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x117419; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x217420; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -117421; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)17422; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s017423; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s217424; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s117425; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617426; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -117427; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s017428; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)17429; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v117430; SKIP-CACHE-INV-NEXT: s_endpgm17431;17432; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:17433; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry17434; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x017435; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x417436; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x817437; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17438; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s417439; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s617440; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s517441; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617442; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s417443; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17444; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v117445; GFX90A-NOTTGSPLIT-NEXT: s_endpgm17446;17447; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:17448; GFX90A-TGSPLIT: ; %bb.0: ; %entry17449; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x017450; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x417451; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x817452; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17453; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s417454; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s617455; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s517456; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617457; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s417458; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17459; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v117460; GFX90A-TGSPLIT-NEXT: s_endpgm17461;17462; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:17463; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry17464; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x017465; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x417466; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x817467; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17468; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s017469; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s217470; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s117471; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617472; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s017473; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17474; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v117475; GFX942-NOTTGSPLIT-NEXT: s_endpgm17476;17477; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:17478; GFX942-TGSPLIT: ; %bb.0: ; %entry17479; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x017480; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x417481; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x817482; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17483; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s017484; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s217485; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s117486; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617487; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s017488; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17489; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v117490; GFX942-TGSPLIT-NEXT: s_endpgm17491;17492; GFX11-WGP-LABEL: local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:17493; GFX11-WGP: ; %bb.0: ; %entry17494; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x017495; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x417496; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x817497; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)17498; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s017499; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s217500; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s117501; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1617502; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s017503; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)17504; GFX11-WGP-NEXT: ds_store_b32 v0, v117505; GFX11-WGP-NEXT: s_endpgm17506;17507; GFX11-CU-LABEL: local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:17508; GFX11-CU: ; %bb.0: ; %entry17509; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x017510; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x417511; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x817512; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)17513; GFX11-CU-NEXT: v_mov_b32_e32 v0, s017514; GFX11-CU-NEXT: v_mov_b32_e32 v1, s217515; GFX11-CU-NEXT: v_mov_b32_e32 v2, s117516; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1617517; GFX11-CU-NEXT: v_mov_b32_e32 v0, s017518; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)17519; GFX11-CU-NEXT: ds_store_b32 v0, v117520; GFX11-CU-NEXT: s_endpgm17521;17522; GFX12-WGP-LABEL: local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:17523; GFX12-WGP: ; %bb.0: ; %entry17524; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x017525; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x417526; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x817527; GFX12-WGP-NEXT: s_wait_kmcnt 0x017528; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s017529; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s217530; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s117531; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1617532; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s017533; GFX12-WGP-NEXT: s_wait_dscnt 0x017534; GFX12-WGP-NEXT: ds_store_b32 v0, v117535; GFX12-WGP-NEXT: s_endpgm17536;17537; GFX12-CU-LABEL: local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:17538; GFX12-CU: ; %bb.0: ; %entry17539; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x017540; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x417541; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x817542; GFX12-CU-NEXT: s_wait_kmcnt 0x017543; GFX12-CU-NEXT: v_mov_b32_e32 v0, s017544; GFX12-CU-NEXT: v_mov_b32_e32 v1, s217545; GFX12-CU-NEXT: v_mov_b32_e32 v2, s117546; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1617547; GFX12-CU-NEXT: v_mov_b32_e32 v0, s017548; GFX12-CU-NEXT: s_wait_dscnt 0x017549; GFX12-CU-NEXT: ds_store_b32 v0, v117550; GFX12-CU-NEXT: s_endpgm17551;17552; GFX1250-LABEL: local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:17553; GFX1250: ; %bb.0: ; %entry17554; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 117555; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x017556; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x417557; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x817558; GFX1250-NEXT: s_wait_kmcnt 0x017559; GFX1250-NEXT: v_mov_b32_e32 v0, s017560; GFX1250-NEXT: v_mov_b32_e32 v1, s217561; GFX1250-NEXT: v_mov_b32_e32 v2, s117562; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1617563; GFX1250-NEXT: v_mov_b32_e32 v0, s017564; GFX1250-NEXT: s_wait_dscnt 0x017565; GFX1250-NEXT: ds_store_b32 v0, v117566; GFX1250-NEXT: s_endpgm17567 ptr addrspace(3) %out, i32 %in, i32 %old) {17568entry:17569 %gep = getelementptr i32, ptr addrspace(3) %out, i32 417570 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire17571 %val0 = extractvalue { i32, i1 } %val, 017572 store i32 %val0, ptr addrspace(3) %out, align 417573 ret void17574}17575 17576define amdgpu_kernel void @local_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg(17577; GFX6-LABEL: local_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:17578; GFX6: ; %bb.0: ; %entry17579; GFX6-NEXT: s_load_dword s4, s[8:9], 0x017580; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr417581; GFX6-NEXT: s_load_dword s5, s[8:9], 0x117582; GFX6-NEXT: s_load_dword s6, s[8:9], 0x217583; GFX6-NEXT: s_mov_b32 m0, -117584; GFX6-NEXT: s_waitcnt lgkmcnt(0)17585; GFX6-NEXT: v_mov_b32_e32 v0, s417586; GFX6-NEXT: v_mov_b32_e32 v1, s617587; GFX6-NEXT: v_mov_b32_e32 v2, s517588; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617589; GFX6-NEXT: s_mov_b32 m0, -117590; GFX6-NEXT: v_mov_b32_e32 v0, s417591; GFX6-NEXT: s_waitcnt lgkmcnt(0)17592; GFX6-NEXT: ds_write_b32 v0, v117593; GFX6-NEXT: s_endpgm17594;17595; GFX7-LABEL: local_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:17596; GFX7: ; %bb.0: ; %entry17597; GFX7-NEXT: s_load_dword s4, s[8:9], 0x017598; GFX7-NEXT: s_load_dword s5, s[8:9], 0x117599; GFX7-NEXT: s_load_dword s6, s[8:9], 0x217600; GFX7-NEXT: s_mov_b32 m0, -117601; GFX7-NEXT: s_waitcnt lgkmcnt(0)17602; GFX7-NEXT: v_mov_b32_e32 v0, s417603; GFX7-NEXT: v_mov_b32_e32 v1, s617604; GFX7-NEXT: v_mov_b32_e32 v2, s517605; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617606; GFX7-NEXT: s_mov_b32 m0, -117607; GFX7-NEXT: v_mov_b32_e32 v0, s417608; GFX7-NEXT: s_waitcnt lgkmcnt(0)17609; GFX7-NEXT: ds_write_b32 v0, v117610; GFX7-NEXT: s_endpgm17611;17612; GFX10-WGP-LABEL: local_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:17613; GFX10-WGP: ; %bb.0: ; %entry17614; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x017615; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x417616; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x817617; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)17618; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s417619; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s617620; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s517621; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617622; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s417623; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)17624; GFX10-WGP-NEXT: ds_write_b32 v0, v117625; GFX10-WGP-NEXT: s_endpgm17626;17627; GFX10-CU-LABEL: local_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:17628; GFX10-CU: ; %bb.0: ; %entry17629; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x017630; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x417631; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x817632; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)17633; GFX10-CU-NEXT: v_mov_b32_e32 v0, s417634; GFX10-CU-NEXT: v_mov_b32_e32 v1, s617635; GFX10-CU-NEXT: v_mov_b32_e32 v2, s517636; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617637; GFX10-CU-NEXT: v_mov_b32_e32 v0, s417638; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)17639; GFX10-CU-NEXT: ds_write_b32 v0, v117640; GFX10-CU-NEXT: s_endpgm17641;17642; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:17643; SKIP-CACHE-INV: ; %bb.0: ; %entry17644; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x017645; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x117646; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x217647; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -117648; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)17649; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s017650; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s217651; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s117652; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617653; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -117654; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s017655; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)17656; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v117657; SKIP-CACHE-INV-NEXT: s_endpgm17658;17659; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:17660; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry17661; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x017662; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x417663; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x817664; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17665; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s417666; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s617667; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s517668; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617669; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s417670; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17671; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v117672; GFX90A-NOTTGSPLIT-NEXT: s_endpgm17673;17674; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:17675; GFX90A-TGSPLIT: ; %bb.0: ; %entry17676; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x017677; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x417678; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x817679; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17680; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s417681; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s617682; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s517683; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617684; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s417685; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17686; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v117687; GFX90A-TGSPLIT-NEXT: s_endpgm17688;17689; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:17690; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry17691; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x017692; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x417693; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x817694; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17695; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s017696; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s217697; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s117698; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617699; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s017700; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17701; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v117702; GFX942-NOTTGSPLIT-NEXT: s_endpgm17703;17704; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:17705; GFX942-TGSPLIT: ; %bb.0: ; %entry17706; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x017707; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x417708; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x817709; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17710; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s017711; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s217712; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s117713; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617714; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s017715; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17716; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v117717; GFX942-TGSPLIT-NEXT: s_endpgm17718;17719; GFX11-WGP-LABEL: local_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:17720; GFX11-WGP: ; %bb.0: ; %entry17721; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x017722; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x417723; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x817724; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)17725; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s017726; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s217727; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s117728; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1617729; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s017730; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)17731; GFX11-WGP-NEXT: ds_store_b32 v0, v117732; GFX11-WGP-NEXT: s_endpgm17733;17734; GFX11-CU-LABEL: local_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:17735; GFX11-CU: ; %bb.0: ; %entry17736; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x017737; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x417738; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x817739; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)17740; GFX11-CU-NEXT: v_mov_b32_e32 v0, s017741; GFX11-CU-NEXT: v_mov_b32_e32 v1, s217742; GFX11-CU-NEXT: v_mov_b32_e32 v2, s117743; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1617744; GFX11-CU-NEXT: v_mov_b32_e32 v0, s017745; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)17746; GFX11-CU-NEXT: ds_store_b32 v0, v117747; GFX11-CU-NEXT: s_endpgm17748;17749; GFX12-WGP-LABEL: local_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:17750; GFX12-WGP: ; %bb.0: ; %entry17751; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x017752; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x417753; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x817754; GFX12-WGP-NEXT: s_wait_kmcnt 0x017755; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s017756; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s217757; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s117758; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1617759; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s017760; GFX12-WGP-NEXT: s_wait_dscnt 0x017761; GFX12-WGP-NEXT: ds_store_b32 v0, v117762; GFX12-WGP-NEXT: s_endpgm17763;17764; GFX12-CU-LABEL: local_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:17765; GFX12-CU: ; %bb.0: ; %entry17766; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x017767; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x417768; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x817769; GFX12-CU-NEXT: s_wait_kmcnt 0x017770; GFX12-CU-NEXT: v_mov_b32_e32 v0, s017771; GFX12-CU-NEXT: v_mov_b32_e32 v1, s217772; GFX12-CU-NEXT: v_mov_b32_e32 v2, s117773; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1617774; GFX12-CU-NEXT: v_mov_b32_e32 v0, s017775; GFX12-CU-NEXT: s_wait_dscnt 0x017776; GFX12-CU-NEXT: ds_store_b32 v0, v117777; GFX12-CU-NEXT: s_endpgm17778;17779; GFX1250-LABEL: local_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:17780; GFX1250: ; %bb.0: ; %entry17781; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 117782; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x017783; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x417784; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x817785; GFX1250-NEXT: s_wait_kmcnt 0x017786; GFX1250-NEXT: v_mov_b32_e32 v0, s017787; GFX1250-NEXT: v_mov_b32_e32 v1, s217788; GFX1250-NEXT: v_mov_b32_e32 v2, s117789; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1617790; GFX1250-NEXT: v_mov_b32_e32 v0, s017791; GFX1250-NEXT: s_wait_dscnt 0x017792; GFX1250-NEXT: ds_store_b32 v0, v117793; GFX1250-NEXT: s_endpgm17794 ptr addrspace(3) %out, i32 %in, i32 %old) {17795entry:17796 %gep = getelementptr i32, ptr addrspace(3) %out, i32 417797 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic seq_cst17798 %val0 = extractvalue { i32, i1 } %val, 017799 store i32 %val0, ptr addrspace(3) %out, align 417800 ret void17801}17802 17803define amdgpu_kernel void @local_workgroup_one_as_acquire_seq_cst_ret_cmpxchg(17804; GFX6-LABEL: local_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:17805; GFX6: ; %bb.0: ; %entry17806; GFX6-NEXT: s_load_dword s4, s[8:9], 0x017807; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr417808; GFX6-NEXT: s_load_dword s5, s[8:9], 0x117809; GFX6-NEXT: s_load_dword s6, s[8:9], 0x217810; GFX6-NEXT: s_mov_b32 m0, -117811; GFX6-NEXT: s_waitcnt lgkmcnt(0)17812; GFX6-NEXT: v_mov_b32_e32 v0, s417813; GFX6-NEXT: v_mov_b32_e32 v1, s617814; GFX6-NEXT: v_mov_b32_e32 v2, s517815; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617816; GFX6-NEXT: s_mov_b32 m0, -117817; GFX6-NEXT: v_mov_b32_e32 v0, s417818; GFX6-NEXT: s_waitcnt lgkmcnt(0)17819; GFX6-NEXT: ds_write_b32 v0, v117820; GFX6-NEXT: s_endpgm17821;17822; GFX7-LABEL: local_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:17823; GFX7: ; %bb.0: ; %entry17824; GFX7-NEXT: s_load_dword s4, s[8:9], 0x017825; GFX7-NEXT: s_load_dword s5, s[8:9], 0x117826; GFX7-NEXT: s_load_dword s6, s[8:9], 0x217827; GFX7-NEXT: s_mov_b32 m0, -117828; GFX7-NEXT: s_waitcnt lgkmcnt(0)17829; GFX7-NEXT: v_mov_b32_e32 v0, s417830; GFX7-NEXT: v_mov_b32_e32 v1, s617831; GFX7-NEXT: v_mov_b32_e32 v2, s517832; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617833; GFX7-NEXT: s_mov_b32 m0, -117834; GFX7-NEXT: v_mov_b32_e32 v0, s417835; GFX7-NEXT: s_waitcnt lgkmcnt(0)17836; GFX7-NEXT: ds_write_b32 v0, v117837; GFX7-NEXT: s_endpgm17838;17839; GFX10-WGP-LABEL: local_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:17840; GFX10-WGP: ; %bb.0: ; %entry17841; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x017842; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x417843; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x817844; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)17845; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s417846; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s617847; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s517848; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617849; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s417850; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)17851; GFX10-WGP-NEXT: ds_write_b32 v0, v117852; GFX10-WGP-NEXT: s_endpgm17853;17854; GFX10-CU-LABEL: local_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:17855; GFX10-CU: ; %bb.0: ; %entry17856; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x017857; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x417858; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x817859; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)17860; GFX10-CU-NEXT: v_mov_b32_e32 v0, s417861; GFX10-CU-NEXT: v_mov_b32_e32 v1, s617862; GFX10-CU-NEXT: v_mov_b32_e32 v2, s517863; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617864; GFX10-CU-NEXT: v_mov_b32_e32 v0, s417865; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)17866; GFX10-CU-NEXT: ds_write_b32 v0, v117867; GFX10-CU-NEXT: s_endpgm17868;17869; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:17870; SKIP-CACHE-INV: ; %bb.0: ; %entry17871; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x017872; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x117873; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x217874; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -117875; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)17876; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s017877; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s217878; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s117879; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617880; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -117881; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s017882; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)17883; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v117884; SKIP-CACHE-INV-NEXT: s_endpgm17885;17886; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:17887; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry17888; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x017889; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x417890; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x817891; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17892; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s417893; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s617894; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s517895; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617896; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s417897; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17898; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v117899; GFX90A-NOTTGSPLIT-NEXT: s_endpgm17900;17901; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:17902; GFX90A-TGSPLIT: ; %bb.0: ; %entry17903; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x017904; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x417905; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x817906; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17907; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s417908; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s617909; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s517910; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617911; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s417912; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17913; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v117914; GFX90A-TGSPLIT-NEXT: s_endpgm17915;17916; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:17917; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry17918; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x017919; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x417920; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x817921; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17922; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s017923; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s217924; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s117925; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617926; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s017927; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17928; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v117929; GFX942-NOTTGSPLIT-NEXT: s_endpgm17930;17931; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:17932; GFX942-TGSPLIT: ; %bb.0: ; %entry17933; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x017934; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x417935; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x817936; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17937; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s017938; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s217939; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s117940; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1617941; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s017942; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)17943; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v117944; GFX942-TGSPLIT-NEXT: s_endpgm17945;17946; GFX11-WGP-LABEL: local_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:17947; GFX11-WGP: ; %bb.0: ; %entry17948; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x017949; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x417950; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x817951; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)17952; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s017953; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s217954; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s117955; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1617956; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s017957; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)17958; GFX11-WGP-NEXT: ds_store_b32 v0, v117959; GFX11-WGP-NEXT: s_endpgm17960;17961; GFX11-CU-LABEL: local_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:17962; GFX11-CU: ; %bb.0: ; %entry17963; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x017964; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x417965; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x817966; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)17967; GFX11-CU-NEXT: v_mov_b32_e32 v0, s017968; GFX11-CU-NEXT: v_mov_b32_e32 v1, s217969; GFX11-CU-NEXT: v_mov_b32_e32 v2, s117970; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1617971; GFX11-CU-NEXT: v_mov_b32_e32 v0, s017972; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)17973; GFX11-CU-NEXT: ds_store_b32 v0, v117974; GFX11-CU-NEXT: s_endpgm17975;17976; GFX12-WGP-LABEL: local_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:17977; GFX12-WGP: ; %bb.0: ; %entry17978; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x017979; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x417980; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x817981; GFX12-WGP-NEXT: s_wait_kmcnt 0x017982; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s017983; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s217984; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s117985; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1617986; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s017987; GFX12-WGP-NEXT: s_wait_dscnt 0x017988; GFX12-WGP-NEXT: ds_store_b32 v0, v117989; GFX12-WGP-NEXT: s_endpgm17990;17991; GFX12-CU-LABEL: local_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:17992; GFX12-CU: ; %bb.0: ; %entry17993; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x017994; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x417995; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x817996; GFX12-CU-NEXT: s_wait_kmcnt 0x017997; GFX12-CU-NEXT: v_mov_b32_e32 v0, s017998; GFX12-CU-NEXT: v_mov_b32_e32 v1, s217999; GFX12-CU-NEXT: v_mov_b32_e32 v2, s118000; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1618001; GFX12-CU-NEXT: v_mov_b32_e32 v0, s018002; GFX12-CU-NEXT: s_wait_dscnt 0x018003; GFX12-CU-NEXT: ds_store_b32 v0, v118004; GFX12-CU-NEXT: s_endpgm18005;18006; GFX1250-LABEL: local_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:18007; GFX1250: ; %bb.0: ; %entry18008; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 118009; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x018010; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x418011; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x818012; GFX1250-NEXT: s_wait_kmcnt 0x018013; GFX1250-NEXT: v_mov_b32_e32 v0, s018014; GFX1250-NEXT: v_mov_b32_e32 v1, s218015; GFX1250-NEXT: v_mov_b32_e32 v2, s118016; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1618017; GFX1250-NEXT: v_mov_b32_e32 v0, s018018; GFX1250-NEXT: s_wait_dscnt 0x018019; GFX1250-NEXT: ds_store_b32 v0, v118020; GFX1250-NEXT: s_endpgm18021 ptr addrspace(3) %out, i32 %in, i32 %old) {18022entry:18023 %gep = getelementptr i32, ptr addrspace(3) %out, i32 418024 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire seq_cst18025 %val0 = extractvalue { i32, i1 } %val, 018026 store i32 %val0, ptr addrspace(3) %out, align 418027 ret void18028}18029 18030define amdgpu_kernel void @local_workgroup_one_as_release_seq_cst_ret_cmpxchg(18031; GFX6-LABEL: local_workgroup_one_as_release_seq_cst_ret_cmpxchg:18032; GFX6: ; %bb.0: ; %entry18033; GFX6-NEXT: s_load_dword s4, s[8:9], 0x018034; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr418035; GFX6-NEXT: s_load_dword s5, s[8:9], 0x118036; GFX6-NEXT: s_load_dword s6, s[8:9], 0x218037; GFX6-NEXT: s_mov_b32 m0, -118038; GFX6-NEXT: s_waitcnt lgkmcnt(0)18039; GFX6-NEXT: v_mov_b32_e32 v0, s418040; GFX6-NEXT: v_mov_b32_e32 v1, s618041; GFX6-NEXT: v_mov_b32_e32 v2, s518042; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618043; GFX6-NEXT: s_mov_b32 m0, -118044; GFX6-NEXT: v_mov_b32_e32 v0, s418045; GFX6-NEXT: s_waitcnt lgkmcnt(0)18046; GFX6-NEXT: ds_write_b32 v0, v118047; GFX6-NEXT: s_endpgm18048;18049; GFX7-LABEL: local_workgroup_one_as_release_seq_cst_ret_cmpxchg:18050; GFX7: ; %bb.0: ; %entry18051; GFX7-NEXT: s_load_dword s4, s[8:9], 0x018052; GFX7-NEXT: s_load_dword s5, s[8:9], 0x118053; GFX7-NEXT: s_load_dword s6, s[8:9], 0x218054; GFX7-NEXT: s_mov_b32 m0, -118055; GFX7-NEXT: s_waitcnt lgkmcnt(0)18056; GFX7-NEXT: v_mov_b32_e32 v0, s418057; GFX7-NEXT: v_mov_b32_e32 v1, s618058; GFX7-NEXT: v_mov_b32_e32 v2, s518059; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618060; GFX7-NEXT: s_mov_b32 m0, -118061; GFX7-NEXT: v_mov_b32_e32 v0, s418062; GFX7-NEXT: s_waitcnt lgkmcnt(0)18063; GFX7-NEXT: ds_write_b32 v0, v118064; GFX7-NEXT: s_endpgm18065;18066; GFX10-WGP-LABEL: local_workgroup_one_as_release_seq_cst_ret_cmpxchg:18067; GFX10-WGP: ; %bb.0: ; %entry18068; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x018069; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x418070; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x818071; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)18072; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s418073; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s618074; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s518075; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618076; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s418077; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)18078; GFX10-WGP-NEXT: ds_write_b32 v0, v118079; GFX10-WGP-NEXT: s_endpgm18080;18081; GFX10-CU-LABEL: local_workgroup_one_as_release_seq_cst_ret_cmpxchg:18082; GFX10-CU: ; %bb.0: ; %entry18083; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x018084; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x418085; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x818086; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)18087; GFX10-CU-NEXT: v_mov_b32_e32 v0, s418088; GFX10-CU-NEXT: v_mov_b32_e32 v1, s618089; GFX10-CU-NEXT: v_mov_b32_e32 v2, s518090; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618091; GFX10-CU-NEXT: v_mov_b32_e32 v0, s418092; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)18093; GFX10-CU-NEXT: ds_write_b32 v0, v118094; GFX10-CU-NEXT: s_endpgm18095;18096; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_release_seq_cst_ret_cmpxchg:18097; SKIP-CACHE-INV: ; %bb.0: ; %entry18098; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x018099; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x118100; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x218101; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -118102; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)18103; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s018104; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s218105; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s118106; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618107; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -118108; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s018109; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)18110; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v118111; SKIP-CACHE-INV-NEXT: s_endpgm18112;18113; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_release_seq_cst_ret_cmpxchg:18114; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry18115; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x018116; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x418117; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x818118; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)18119; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s418120; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s618121; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s518122; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618123; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s418124; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)18125; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v118126; GFX90A-NOTTGSPLIT-NEXT: s_endpgm18127;18128; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_release_seq_cst_ret_cmpxchg:18129; GFX90A-TGSPLIT: ; %bb.0: ; %entry18130; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x018131; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x418132; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x818133; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)18134; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s418135; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s618136; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s518137; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618138; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s418139; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)18140; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v118141; GFX90A-TGSPLIT-NEXT: s_endpgm18142;18143; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_release_seq_cst_ret_cmpxchg:18144; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry18145; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x018146; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x418147; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x818148; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)18149; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s018150; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s218151; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s118152; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618153; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s018154; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)18155; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v118156; GFX942-NOTTGSPLIT-NEXT: s_endpgm18157;18158; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_release_seq_cst_ret_cmpxchg:18159; GFX942-TGSPLIT: ; %bb.0: ; %entry18160; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x018161; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x418162; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x818163; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)18164; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s018165; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s218166; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s118167; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618168; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s018169; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)18170; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v118171; GFX942-TGSPLIT-NEXT: s_endpgm18172;18173; GFX11-WGP-LABEL: local_workgroup_one_as_release_seq_cst_ret_cmpxchg:18174; GFX11-WGP: ; %bb.0: ; %entry18175; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x018176; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x418177; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x818178; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)18179; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s018180; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s218181; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s118182; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1618183; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s018184; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)18185; GFX11-WGP-NEXT: ds_store_b32 v0, v118186; GFX11-WGP-NEXT: s_endpgm18187;18188; GFX11-CU-LABEL: local_workgroup_one_as_release_seq_cst_ret_cmpxchg:18189; GFX11-CU: ; %bb.0: ; %entry18190; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x018191; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x418192; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x818193; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)18194; GFX11-CU-NEXT: v_mov_b32_e32 v0, s018195; GFX11-CU-NEXT: v_mov_b32_e32 v1, s218196; GFX11-CU-NEXT: v_mov_b32_e32 v2, s118197; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1618198; GFX11-CU-NEXT: v_mov_b32_e32 v0, s018199; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)18200; GFX11-CU-NEXT: ds_store_b32 v0, v118201; GFX11-CU-NEXT: s_endpgm18202;18203; GFX12-WGP-LABEL: local_workgroup_one_as_release_seq_cst_ret_cmpxchg:18204; GFX12-WGP: ; %bb.0: ; %entry18205; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x018206; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x418207; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x818208; GFX12-WGP-NEXT: s_wait_kmcnt 0x018209; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s018210; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s218211; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s118212; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1618213; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s018214; GFX12-WGP-NEXT: s_wait_dscnt 0x018215; GFX12-WGP-NEXT: ds_store_b32 v0, v118216; GFX12-WGP-NEXT: s_endpgm18217;18218; GFX12-CU-LABEL: local_workgroup_one_as_release_seq_cst_ret_cmpxchg:18219; GFX12-CU: ; %bb.0: ; %entry18220; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x018221; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x418222; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x818223; GFX12-CU-NEXT: s_wait_kmcnt 0x018224; GFX12-CU-NEXT: v_mov_b32_e32 v0, s018225; GFX12-CU-NEXT: v_mov_b32_e32 v1, s218226; GFX12-CU-NEXT: v_mov_b32_e32 v2, s118227; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1618228; GFX12-CU-NEXT: v_mov_b32_e32 v0, s018229; GFX12-CU-NEXT: s_wait_dscnt 0x018230; GFX12-CU-NEXT: ds_store_b32 v0, v118231; GFX12-CU-NEXT: s_endpgm18232;18233; GFX1250-LABEL: local_workgroup_one_as_release_seq_cst_ret_cmpxchg:18234; GFX1250: ; %bb.0: ; %entry18235; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 118236; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x018237; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x418238; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x818239; GFX1250-NEXT: s_wait_kmcnt 0x018240; GFX1250-NEXT: v_mov_b32_e32 v0, s018241; GFX1250-NEXT: v_mov_b32_e32 v1, s218242; GFX1250-NEXT: v_mov_b32_e32 v2, s118243; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1618244; GFX1250-NEXT: v_mov_b32_e32 v0, s018245; GFX1250-NEXT: s_wait_dscnt 0x018246; GFX1250-NEXT: ds_store_b32 v0, v118247; GFX1250-NEXT: s_endpgm18248 ptr addrspace(3) %out, i32 %in, i32 %old) {18249entry:18250 %gep = getelementptr i32, ptr addrspace(3) %out, i32 418251 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release seq_cst18252 %val0 = extractvalue { i32, i1 } %val, 018253 store i32 %val0, ptr addrspace(3) %out, align 418254 ret void18255}18256 18257define amdgpu_kernel void @local_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg(18258; GFX6-LABEL: local_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:18259; GFX6: ; %bb.0: ; %entry18260; GFX6-NEXT: s_load_dword s4, s[8:9], 0x018261; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr418262; GFX6-NEXT: s_load_dword s5, s[8:9], 0x118263; GFX6-NEXT: s_load_dword s6, s[8:9], 0x218264; GFX6-NEXT: s_mov_b32 m0, -118265; GFX6-NEXT: s_waitcnt lgkmcnt(0)18266; GFX6-NEXT: v_mov_b32_e32 v0, s418267; GFX6-NEXT: v_mov_b32_e32 v1, s618268; GFX6-NEXT: v_mov_b32_e32 v2, s518269; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618270; GFX6-NEXT: s_mov_b32 m0, -118271; GFX6-NEXT: v_mov_b32_e32 v0, s418272; GFX6-NEXT: s_waitcnt lgkmcnt(0)18273; GFX6-NEXT: ds_write_b32 v0, v118274; GFX6-NEXT: s_endpgm18275;18276; GFX7-LABEL: local_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:18277; GFX7: ; %bb.0: ; %entry18278; GFX7-NEXT: s_load_dword s4, s[8:9], 0x018279; GFX7-NEXT: s_load_dword s5, s[8:9], 0x118280; GFX7-NEXT: s_load_dword s6, s[8:9], 0x218281; GFX7-NEXT: s_mov_b32 m0, -118282; GFX7-NEXT: s_waitcnt lgkmcnt(0)18283; GFX7-NEXT: v_mov_b32_e32 v0, s418284; GFX7-NEXT: v_mov_b32_e32 v1, s618285; GFX7-NEXT: v_mov_b32_e32 v2, s518286; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618287; GFX7-NEXT: s_mov_b32 m0, -118288; GFX7-NEXT: v_mov_b32_e32 v0, s418289; GFX7-NEXT: s_waitcnt lgkmcnt(0)18290; GFX7-NEXT: ds_write_b32 v0, v118291; GFX7-NEXT: s_endpgm18292;18293; GFX10-WGP-LABEL: local_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:18294; GFX10-WGP: ; %bb.0: ; %entry18295; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x018296; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x418297; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x818298; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)18299; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s418300; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s618301; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s518302; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618303; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s418304; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)18305; GFX10-WGP-NEXT: ds_write_b32 v0, v118306; GFX10-WGP-NEXT: s_endpgm18307;18308; GFX10-CU-LABEL: local_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:18309; GFX10-CU: ; %bb.0: ; %entry18310; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x018311; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x418312; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x818313; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)18314; GFX10-CU-NEXT: v_mov_b32_e32 v0, s418315; GFX10-CU-NEXT: v_mov_b32_e32 v1, s618316; GFX10-CU-NEXT: v_mov_b32_e32 v2, s518317; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618318; GFX10-CU-NEXT: v_mov_b32_e32 v0, s418319; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)18320; GFX10-CU-NEXT: ds_write_b32 v0, v118321; GFX10-CU-NEXT: s_endpgm18322;18323; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:18324; SKIP-CACHE-INV: ; %bb.0: ; %entry18325; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x018326; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x118327; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x218328; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -118329; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)18330; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s018331; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s218332; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s118333; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618334; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -118335; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s018336; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)18337; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v118338; SKIP-CACHE-INV-NEXT: s_endpgm18339;18340; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:18341; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry18342; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x018343; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x418344; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x818345; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)18346; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s418347; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s618348; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s518349; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618350; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s418351; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)18352; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v118353; GFX90A-NOTTGSPLIT-NEXT: s_endpgm18354;18355; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:18356; GFX90A-TGSPLIT: ; %bb.0: ; %entry18357; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x018358; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x418359; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x818360; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)18361; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s418362; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s618363; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s518364; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618365; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s418366; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)18367; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v118368; GFX90A-TGSPLIT-NEXT: s_endpgm18369;18370; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:18371; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry18372; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x018373; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x418374; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x818375; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)18376; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s018377; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s218378; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s118379; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618380; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s018381; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)18382; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v118383; GFX942-NOTTGSPLIT-NEXT: s_endpgm18384;18385; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:18386; GFX942-TGSPLIT: ; %bb.0: ; %entry18387; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x018388; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x418389; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x818390; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)18391; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s018392; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s218393; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s118394; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618395; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s018396; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)18397; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v118398; GFX942-TGSPLIT-NEXT: s_endpgm18399;18400; GFX11-WGP-LABEL: local_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:18401; GFX11-WGP: ; %bb.0: ; %entry18402; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x018403; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x418404; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x818405; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)18406; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s018407; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s218408; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s118409; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1618410; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s018411; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)18412; GFX11-WGP-NEXT: ds_store_b32 v0, v118413; GFX11-WGP-NEXT: s_endpgm18414;18415; GFX11-CU-LABEL: local_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:18416; GFX11-CU: ; %bb.0: ; %entry18417; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x018418; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x418419; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x818420; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)18421; GFX11-CU-NEXT: v_mov_b32_e32 v0, s018422; GFX11-CU-NEXT: v_mov_b32_e32 v1, s218423; GFX11-CU-NEXT: v_mov_b32_e32 v2, s118424; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1618425; GFX11-CU-NEXT: v_mov_b32_e32 v0, s018426; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)18427; GFX11-CU-NEXT: ds_store_b32 v0, v118428; GFX11-CU-NEXT: s_endpgm18429;18430; GFX12-WGP-LABEL: local_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:18431; GFX12-WGP: ; %bb.0: ; %entry18432; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x018433; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x418434; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x818435; GFX12-WGP-NEXT: s_wait_kmcnt 0x018436; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s018437; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s218438; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s118439; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1618440; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s018441; GFX12-WGP-NEXT: s_wait_dscnt 0x018442; GFX12-WGP-NEXT: ds_store_b32 v0, v118443; GFX12-WGP-NEXT: s_endpgm18444;18445; GFX12-CU-LABEL: local_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:18446; GFX12-CU: ; %bb.0: ; %entry18447; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x018448; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x418449; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x818450; GFX12-CU-NEXT: s_wait_kmcnt 0x018451; GFX12-CU-NEXT: v_mov_b32_e32 v0, s018452; GFX12-CU-NEXT: v_mov_b32_e32 v1, s218453; GFX12-CU-NEXT: v_mov_b32_e32 v2, s118454; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1618455; GFX12-CU-NEXT: v_mov_b32_e32 v0, s018456; GFX12-CU-NEXT: s_wait_dscnt 0x018457; GFX12-CU-NEXT: ds_store_b32 v0, v118458; GFX12-CU-NEXT: s_endpgm18459;18460; GFX1250-LABEL: local_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:18461; GFX1250: ; %bb.0: ; %entry18462; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 118463; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x018464; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x418465; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x818466; GFX1250-NEXT: s_wait_kmcnt 0x018467; GFX1250-NEXT: v_mov_b32_e32 v0, s018468; GFX1250-NEXT: v_mov_b32_e32 v1, s218469; GFX1250-NEXT: v_mov_b32_e32 v2, s118470; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1618471; GFX1250-NEXT: v_mov_b32_e32 v0, s018472; GFX1250-NEXT: s_wait_dscnt 0x018473; GFX1250-NEXT: ds_store_b32 v0, v118474; GFX1250-NEXT: s_endpgm18475 ptr addrspace(3) %out, i32 %in, i32 %old) {18476entry:18477 %gep = getelementptr i32, ptr addrspace(3) %out, i32 418478 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel seq_cst18479 %val0 = extractvalue { i32, i1 } %val, 018480 store i32 %val0, ptr addrspace(3) %out, align 418481 ret void18482}18483 18484define amdgpu_kernel void @local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(18485; GFX6-LABEL: local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:18486; GFX6: ; %bb.0: ; %entry18487; GFX6-NEXT: s_load_dword s4, s[8:9], 0x018488; GFX6-NEXT: ; kill: def $sgpr5 killed $sgpr418489; GFX6-NEXT: s_load_dword s5, s[8:9], 0x118490; GFX6-NEXT: s_load_dword s6, s[8:9], 0x218491; GFX6-NEXT: s_mov_b32 m0, -118492; GFX6-NEXT: s_waitcnt lgkmcnt(0)18493; GFX6-NEXT: v_mov_b32_e32 v0, s418494; GFX6-NEXT: v_mov_b32_e32 v1, s618495; GFX6-NEXT: v_mov_b32_e32 v2, s518496; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618497; GFX6-NEXT: s_mov_b32 m0, -118498; GFX6-NEXT: v_mov_b32_e32 v0, s418499; GFX6-NEXT: s_waitcnt lgkmcnt(0)18500; GFX6-NEXT: ds_write_b32 v0, v118501; GFX6-NEXT: s_endpgm18502;18503; GFX7-LABEL: local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:18504; GFX7: ; %bb.0: ; %entry18505; GFX7-NEXT: s_load_dword s4, s[8:9], 0x018506; GFX7-NEXT: s_load_dword s5, s[8:9], 0x118507; GFX7-NEXT: s_load_dword s6, s[8:9], 0x218508; GFX7-NEXT: s_mov_b32 m0, -118509; GFX7-NEXT: s_waitcnt lgkmcnt(0)18510; GFX7-NEXT: v_mov_b32_e32 v0, s418511; GFX7-NEXT: v_mov_b32_e32 v1, s618512; GFX7-NEXT: v_mov_b32_e32 v2, s518513; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618514; GFX7-NEXT: s_mov_b32 m0, -118515; GFX7-NEXT: v_mov_b32_e32 v0, s418516; GFX7-NEXT: s_waitcnt lgkmcnt(0)18517; GFX7-NEXT: ds_write_b32 v0, v118518; GFX7-NEXT: s_endpgm18519;18520; GFX10-WGP-LABEL: local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:18521; GFX10-WGP: ; %bb.0: ; %entry18522; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x018523; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x418524; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x818525; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)18526; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s418527; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s618528; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s518529; GFX10-WGP-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618530; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s418531; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)18532; GFX10-WGP-NEXT: ds_write_b32 v0, v118533; GFX10-WGP-NEXT: s_endpgm18534;18535; GFX10-CU-LABEL: local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:18536; GFX10-CU: ; %bb.0: ; %entry18537; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x018538; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x418539; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x818540; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)18541; GFX10-CU-NEXT: v_mov_b32_e32 v0, s418542; GFX10-CU-NEXT: v_mov_b32_e32 v1, s618543; GFX10-CU-NEXT: v_mov_b32_e32 v2, s518544; GFX10-CU-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618545; GFX10-CU-NEXT: v_mov_b32_e32 v0, s418546; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)18547; GFX10-CU-NEXT: ds_write_b32 v0, v118548; GFX10-CU-NEXT: s_endpgm18549;18550; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:18551; SKIP-CACHE-INV: ; %bb.0: ; %entry18552; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x018553; SKIP-CACHE-INV-NEXT: s_load_dword s1, s[4:5], 0x118554; SKIP-CACHE-INV-NEXT: s_load_dword s2, s[4:5], 0x218555; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -118556; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)18557; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s018558; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s218559; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s118560; SKIP-CACHE-INV-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618561; SKIP-CACHE-INV-NEXT: s_mov_b32 m0, -118562; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s018563; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)18564; SKIP-CACHE-INV-NEXT: ds_write_b32 v0, v118565; SKIP-CACHE-INV-NEXT: s_endpgm18566;18567; GFX90A-NOTTGSPLIT-LABEL: local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:18568; GFX90A-NOTTGSPLIT: ; %bb.0: ; %entry18569; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x018570; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x418571; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x818572; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)18573; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s418574; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s618575; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s518576; GFX90A-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618577; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s418578; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)18579; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v118580; GFX90A-NOTTGSPLIT-NEXT: s_endpgm18581;18582; GFX90A-TGSPLIT-LABEL: local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:18583; GFX90A-TGSPLIT: ; %bb.0: ; %entry18584; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x018585; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x418586; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x818587; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)18588; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s418589; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s618590; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s518591; GFX90A-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618592; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s418593; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)18594; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v118595; GFX90A-TGSPLIT-NEXT: s_endpgm18596;18597; GFX942-NOTTGSPLIT-LABEL: local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:18598; GFX942-NOTTGSPLIT: ; %bb.0: ; %entry18599; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x018600; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x418601; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x818602; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)18603; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s018604; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s218605; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s118606; GFX942-NOTTGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618607; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s018608; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)18609; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v118610; GFX942-NOTTGSPLIT-NEXT: s_endpgm18611;18612; GFX942-TGSPLIT-LABEL: local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:18613; GFX942-TGSPLIT: ; %bb.0: ; %entry18614; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x018615; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x418616; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x818617; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)18618; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s018619; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s218620; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s118621; GFX942-TGSPLIT-NEXT: ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:1618622; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s018623; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)18624; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v118625; GFX942-TGSPLIT-NEXT: s_endpgm18626;18627; GFX11-WGP-LABEL: local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:18628; GFX11-WGP: ; %bb.0: ; %entry18629; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x018630; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x418631; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x818632; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)18633; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s018634; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s218635; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s118636; GFX11-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1618637; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s018638; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)18639; GFX11-WGP-NEXT: ds_store_b32 v0, v118640; GFX11-WGP-NEXT: s_endpgm18641;18642; GFX11-CU-LABEL: local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:18643; GFX11-CU: ; %bb.0: ; %entry18644; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x018645; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x418646; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x818647; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)18648; GFX11-CU-NEXT: v_mov_b32_e32 v0, s018649; GFX11-CU-NEXT: v_mov_b32_e32 v1, s218650; GFX11-CU-NEXT: v_mov_b32_e32 v2, s118651; GFX11-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1618652; GFX11-CU-NEXT: v_mov_b32_e32 v0, s018653; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)18654; GFX11-CU-NEXT: ds_store_b32 v0, v118655; GFX11-CU-NEXT: s_endpgm18656;18657; GFX12-WGP-LABEL: local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:18658; GFX12-WGP: ; %bb.0: ; %entry18659; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x018660; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x418661; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x818662; GFX12-WGP-NEXT: s_wait_kmcnt 0x018663; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s018664; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s218665; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s118666; GFX12-WGP-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1618667; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s018668; GFX12-WGP-NEXT: s_wait_dscnt 0x018669; GFX12-WGP-NEXT: ds_store_b32 v0, v118670; GFX12-WGP-NEXT: s_endpgm18671;18672; GFX12-CU-LABEL: local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:18673; GFX12-CU: ; %bb.0: ; %entry18674; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x018675; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x418676; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x818677; GFX12-CU-NEXT: s_wait_kmcnt 0x018678; GFX12-CU-NEXT: v_mov_b32_e32 v0, s018679; GFX12-CU-NEXT: v_mov_b32_e32 v1, s218680; GFX12-CU-NEXT: v_mov_b32_e32 v2, s118681; GFX12-CU-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1618682; GFX12-CU-NEXT: v_mov_b32_e32 v0, s018683; GFX12-CU-NEXT: s_wait_dscnt 0x018684; GFX12-CU-NEXT: ds_store_b32 v0, v118685; GFX12-CU-NEXT: s_endpgm18686;18687; GFX1250-LABEL: local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:18688; GFX1250: ; %bb.0: ; %entry18689; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 118690; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x018691; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x418692; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x818693; GFX1250-NEXT: s_wait_kmcnt 0x018694; GFX1250-NEXT: v_mov_b32_e32 v0, s018695; GFX1250-NEXT: v_mov_b32_e32 v1, s218696; GFX1250-NEXT: v_mov_b32_e32 v2, s118697; GFX1250-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:1618698; GFX1250-NEXT: v_mov_b32_e32 v0, s018699; GFX1250-NEXT: s_wait_dscnt 0x018700; GFX1250-NEXT: ds_store_b32 v0, v118701; GFX1250-NEXT: s_endpgm18702 ptr addrspace(3) %out, i32 %in, i32 %old) {18703entry:18704 %gep = getelementptr i32, ptr addrspace(3) %out, i32 418705 %val = cmpxchg volatile ptr addrspace(3) %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst18706 %val0 = extractvalue { i32, i1 } %val, 018707 store i32 %val0, ptr addrspace(3) %out, align 418708 ret void18709}18710