10856 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -mcpu=bonaire < %s | FileCheck -check-prefixes=GCN1 %s3; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -check-prefixes=GCN2 %s4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN3 %s5; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s6; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s7 8define amdgpu_kernel void @atomic_add_i32_offset(ptr %out, i32 %in) {9; GCN1-LABEL: atomic_add_i32_offset:10; GCN1: ; %bb.0: ; %entry11; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x912; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb13; GCN1-NEXT: s_waitcnt lgkmcnt(0)14; GCN1-NEXT: s_add_u32 s0, s0, 1615; GCN1-NEXT: s_addc_u32 s1, s1, 016; GCN1-NEXT: v_mov_b32_e32 v0, s017; GCN1-NEXT: v_mov_b32_e32 v1, s118; GCN1-NEXT: v_mov_b32_e32 v2, s219; GCN1-NEXT: flat_atomic_add v[0:1], v220; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)21; GCN1-NEXT: buffer_wbinvl1_vol22; GCN1-NEXT: s_endpgm23;24; GCN2-LABEL: atomic_add_i32_offset:25; GCN2: ; %bb.0: ; %entry26; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2427; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c28; GCN2-NEXT: s_waitcnt lgkmcnt(0)29; GCN2-NEXT: s_add_u32 s0, s0, 1630; GCN2-NEXT: s_addc_u32 s1, s1, 031; GCN2-NEXT: v_mov_b32_e32 v0, s032; GCN2-NEXT: v_mov_b32_e32 v1, s133; GCN2-NEXT: v_mov_b32_e32 v2, s234; GCN2-NEXT: flat_atomic_add v[0:1], v235; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)36; GCN2-NEXT: buffer_wbinvl1_vol37; GCN2-NEXT: s_endpgm38;39; GCN3-LABEL: atomic_add_i32_offset:40; GCN3: ; %bb.0: ; %entry41; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2442; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c43; GCN3-NEXT: s_waitcnt lgkmcnt(0)44; GCN3-NEXT: v_mov_b32_e32 v0, s045; GCN3-NEXT: v_mov_b32_e32 v1, s146; GCN3-NEXT: v_mov_b32_e32 v2, s247; GCN3-NEXT: flat_atomic_add v[0:1], v2 offset:1648; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)49; GCN3-NEXT: buffer_wbinvl1_vol50; GCN3-NEXT: s_endpgm51;52; GFX11-LABEL: atomic_add_i32_offset:53; GFX11: ; %bb.0: ; %entry54; GFX11-NEXT: s_clause 0x155; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x2456; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c57; GFX11-NEXT: s_waitcnt lgkmcnt(0)58; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s159; GFX11-NEXT: v_mov_b32_e32 v2, s260; GFX11-NEXT: flat_atomic_add_u32 v[0:1], v2 offset:1661; GFX11-NEXT: s_waitcnt lgkmcnt(0)62; GFX11-NEXT: s_waitcnt_vscnt null, 0x063; GFX11-NEXT: buffer_gl1_inv64; GFX11-NEXT: buffer_gl0_inv65; GFX11-NEXT: s_endpgm66entry:67 %gep = getelementptr inbounds i32, ptr %out, i32 468 %val = atomicrmw add ptr %gep, i32 %in syncscope("agent") seq_cst69 ret void70}71 72define amdgpu_kernel void @atomic_add_i32_max_offset(ptr %out, i32 %in) {73; GCN1-LABEL: atomic_add_i32_max_offset:74; GCN1: ; %bb.0: ; %entry75; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x976; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb77; GCN1-NEXT: s_waitcnt lgkmcnt(0)78; GCN1-NEXT: s_add_u32 s0, s0, 0xffc79; GCN1-NEXT: s_addc_u32 s1, s1, 080; GCN1-NEXT: v_mov_b32_e32 v0, s081; GCN1-NEXT: v_mov_b32_e32 v1, s182; GCN1-NEXT: v_mov_b32_e32 v2, s283; GCN1-NEXT: flat_atomic_add v[0:1], v284; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)85; GCN1-NEXT: buffer_wbinvl1_vol86; GCN1-NEXT: s_endpgm87;88; GCN2-LABEL: atomic_add_i32_max_offset:89; GCN2: ; %bb.0: ; %entry90; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2491; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c92; GCN2-NEXT: s_waitcnt lgkmcnt(0)93; GCN2-NEXT: s_add_u32 s0, s0, 0xffc94; GCN2-NEXT: s_addc_u32 s1, s1, 095; GCN2-NEXT: v_mov_b32_e32 v0, s096; GCN2-NEXT: v_mov_b32_e32 v1, s197; GCN2-NEXT: v_mov_b32_e32 v2, s298; GCN2-NEXT: flat_atomic_add v[0:1], v299; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)100; GCN2-NEXT: buffer_wbinvl1_vol101; GCN2-NEXT: s_endpgm102;103; GCN3-LABEL: atomic_add_i32_max_offset:104; GCN3: ; %bb.0: ; %entry105; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24106; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c107; GCN3-NEXT: s_waitcnt lgkmcnt(0)108; GCN3-NEXT: v_mov_b32_e32 v0, s0109; GCN3-NEXT: v_mov_b32_e32 v1, s1110; GCN3-NEXT: v_mov_b32_e32 v2, s2111; GCN3-NEXT: flat_atomic_add v[0:1], v2 offset:4092112; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)113; GCN3-NEXT: buffer_wbinvl1_vol114; GCN3-NEXT: s_endpgm115;116; GFX11-LABEL: atomic_add_i32_max_offset:117; GFX11: ; %bb.0: ; %entry118; GFX11-NEXT: s_clause 0x1119; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24120; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c121; GFX11-NEXT: s_waitcnt lgkmcnt(0)122; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1123; GFX11-NEXT: v_mov_b32_e32 v2, s2124; GFX11-NEXT: flat_atomic_add_u32 v[0:1], v2 offset:4092125; GFX11-NEXT: s_waitcnt lgkmcnt(0)126; GFX11-NEXT: s_waitcnt_vscnt null, 0x0127; GFX11-NEXT: buffer_gl1_inv128; GFX11-NEXT: buffer_gl0_inv129; GFX11-NEXT: s_endpgm130entry:131 %gep = getelementptr inbounds i32, ptr %out, i32 1023132 %val = atomicrmw volatile add ptr %gep, i32 %in syncscope("agent") seq_cst133 ret void134}135 136define amdgpu_kernel void @atomic_add_i32_max_offset_p1(ptr %out, i32 %in) {137; GCN1-LABEL: atomic_add_i32_max_offset_p1:138; GCN1: ; %bb.0: ; %entry139; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9140; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb141; GCN1-NEXT: s_waitcnt lgkmcnt(0)142; GCN1-NEXT: s_add_u32 s0, s0, 0x1000143; GCN1-NEXT: s_addc_u32 s1, s1, 0144; GCN1-NEXT: v_mov_b32_e32 v0, s0145; GCN1-NEXT: v_mov_b32_e32 v1, s1146; GCN1-NEXT: v_mov_b32_e32 v2, s2147; GCN1-NEXT: flat_atomic_add v[0:1], v2148; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)149; GCN1-NEXT: buffer_wbinvl1_vol150; GCN1-NEXT: s_endpgm151;152; GCN2-LABEL: atomic_add_i32_max_offset_p1:153; GCN2: ; %bb.0: ; %entry154; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24155; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c156; GCN2-NEXT: s_waitcnt lgkmcnt(0)157; GCN2-NEXT: s_add_u32 s0, s0, 0x1000158; GCN2-NEXT: s_addc_u32 s1, s1, 0159; GCN2-NEXT: v_mov_b32_e32 v0, s0160; GCN2-NEXT: v_mov_b32_e32 v1, s1161; GCN2-NEXT: v_mov_b32_e32 v2, s2162; GCN2-NEXT: flat_atomic_add v[0:1], v2163; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)164; GCN2-NEXT: buffer_wbinvl1_vol165; GCN2-NEXT: s_endpgm166;167; GCN3-LABEL: atomic_add_i32_max_offset_p1:168; GCN3: ; %bb.0: ; %entry169; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24170; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c171; GCN3-NEXT: s_waitcnt lgkmcnt(0)172; GCN3-NEXT: v_mov_b32_e32 v0, s0173; GCN3-NEXT: v_mov_b32_e32 v1, s1174; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 0x1000, v0175; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc176; GCN3-NEXT: v_mov_b32_e32 v2, s2177; GCN3-NEXT: flat_atomic_add v[0:1], v2178; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)179; GCN3-NEXT: buffer_wbinvl1_vol180; GCN3-NEXT: s_endpgm181;182; GFX11-LABEL: atomic_add_i32_max_offset_p1:183; GFX11: ; %bb.0: ; %entry184; GFX11-NEXT: s_clause 0x1185; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24186; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c187; GFX11-NEXT: s_waitcnt lgkmcnt(0)188; GFX11-NEXT: v_add_co_u32 v0, s0, 0x1000, s0189; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)190; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, s1, s0191; GFX11-NEXT: v_mov_b32_e32 v2, s2192; GFX11-NEXT: flat_atomic_add_u32 v[0:1], v2193; GFX11-NEXT: s_waitcnt lgkmcnt(0)194; GFX11-NEXT: s_waitcnt_vscnt null, 0x0195; GFX11-NEXT: buffer_gl1_inv196; GFX11-NEXT: buffer_gl0_inv197; GFX11-NEXT: s_endpgm198entry:199 %gep = getelementptr inbounds i32, ptr %out, i32 1024200 %val = atomicrmw volatile add ptr %gep, i32 %in syncscope("agent") seq_cst201 ret void202}203 204define amdgpu_kernel void @atomic_add_i32_ret_offset(ptr %out, ptr %out2, i32 %in) {205; GCN1-LABEL: atomic_add_i32_ret_offset:206; GCN1: ; %bb.0: ; %entry207; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9208; GCN1-NEXT: s_load_dword s4, s[4:5], 0xd209; GCN1-NEXT: s_waitcnt lgkmcnt(0)210; GCN1-NEXT: s_add_u32 s0, s0, 16211; GCN1-NEXT: s_addc_u32 s1, s1, 0212; GCN1-NEXT: v_mov_b32_e32 v0, s0213; GCN1-NEXT: v_mov_b32_e32 v1, s1214; GCN1-NEXT: v_mov_b32_e32 v2, s4215; GCN1-NEXT: flat_atomic_add v2, v[0:1], v2 glc216; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)217; GCN1-NEXT: buffer_wbinvl1_vol218; GCN1-NEXT: v_mov_b32_e32 v0, s2219; GCN1-NEXT: v_mov_b32_e32 v1, s3220; GCN1-NEXT: flat_store_dword v[0:1], v2221; GCN1-NEXT: s_endpgm222;223; GCN2-LABEL: atomic_add_i32_ret_offset:224; GCN2: ; %bb.0: ; %entry225; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24226; GCN2-NEXT: s_load_dword s4, s[4:5], 0x34227; GCN2-NEXT: s_waitcnt lgkmcnt(0)228; GCN2-NEXT: s_add_u32 s0, s0, 16229; GCN2-NEXT: s_addc_u32 s1, s1, 0230; GCN2-NEXT: v_mov_b32_e32 v0, s0231; GCN2-NEXT: v_mov_b32_e32 v1, s1232; GCN2-NEXT: v_mov_b32_e32 v2, s4233; GCN2-NEXT: flat_atomic_add v2, v[0:1], v2 glc234; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)235; GCN2-NEXT: buffer_wbinvl1_vol236; GCN2-NEXT: v_mov_b32_e32 v0, s2237; GCN2-NEXT: v_mov_b32_e32 v1, s3238; GCN2-NEXT: flat_store_dword v[0:1], v2239; GCN2-NEXT: s_endpgm240;241; GCN3-LABEL: atomic_add_i32_ret_offset:242; GCN3: ; %bb.0: ; %entry243; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24244; GCN3-NEXT: s_load_dword s6, s[4:5], 0x34245; GCN3-NEXT: s_waitcnt lgkmcnt(0)246; GCN3-NEXT: v_mov_b32_e32 v0, s0247; GCN3-NEXT: v_mov_b32_e32 v1, s1248; GCN3-NEXT: v_mov_b32_e32 v2, s6249; GCN3-NEXT: flat_atomic_add v2, v[0:1], v2 offset:16 glc250; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)251; GCN3-NEXT: buffer_wbinvl1_vol252; GCN3-NEXT: v_mov_b32_e32 v0, s2253; GCN3-NEXT: v_mov_b32_e32 v1, s3254; GCN3-NEXT: flat_store_dword v[0:1], v2255; GCN3-NEXT: s_endpgm256;257; GFX11-LABEL: atomic_add_i32_ret_offset:258; GFX11: ; %bb.0: ; %entry259; GFX11-NEXT: s_clause 0x1260; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24261; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x34262; GFX11-NEXT: s_waitcnt lgkmcnt(0)263; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1264; GFX11-NEXT: v_mov_b32_e32 v2, s4265; GFX11-NEXT: flat_atomic_add_u32 v2, v[0:1], v2 offset:16 glc266; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)267; GFX11-NEXT: buffer_gl1_inv268; GFX11-NEXT: buffer_gl0_inv269; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3270; GFX11-NEXT: flat_store_b32 v[0:1], v2271; GFX11-NEXT: s_endpgm272entry:273 %gep = getelementptr inbounds i32, ptr %out, i32 4274 %val = atomicrmw volatile add ptr %gep, i32 %in syncscope("agent") seq_cst275 store i32 %val, ptr %out2276 ret void277}278 279define amdgpu_kernel void @atomic_add_i32_addr64_offset(ptr %out, i32 %in, i64 %index) {280; GCN1-LABEL: atomic_add_i32_addr64_offset:281; GCN1: ; %bb.0: ; %entry282; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd283; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9284; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb285; GCN1-NEXT: s_waitcnt lgkmcnt(0)286; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2287; GCN1-NEXT: s_add_u32 s0, s2, s0288; GCN1-NEXT: s_addc_u32 s1, s3, s1289; GCN1-NEXT: s_add_u32 s0, s0, 16290; GCN1-NEXT: s_addc_u32 s1, s1, 0291; GCN1-NEXT: v_mov_b32_e32 v0, s0292; GCN1-NEXT: v_mov_b32_e32 v1, s1293; GCN1-NEXT: v_mov_b32_e32 v2, s4294; GCN1-NEXT: flat_atomic_add v[0:1], v2295; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)296; GCN1-NEXT: buffer_wbinvl1_vol297; GCN1-NEXT: s_endpgm298;299; GCN2-LABEL: atomic_add_i32_addr64_offset:300; GCN2: ; %bb.0: ; %entry301; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34302; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24303; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c304; GCN2-NEXT: s_waitcnt lgkmcnt(0)305; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2306; GCN2-NEXT: s_add_u32 s0, s2, s0307; GCN2-NEXT: s_addc_u32 s1, s3, s1308; GCN2-NEXT: s_add_u32 s0, s0, 16309; GCN2-NEXT: s_addc_u32 s1, s1, 0310; GCN2-NEXT: v_mov_b32_e32 v0, s0311; GCN2-NEXT: v_mov_b32_e32 v1, s1312; GCN2-NEXT: v_mov_b32_e32 v2, s4313; GCN2-NEXT: flat_atomic_add v[0:1], v2314; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)315; GCN2-NEXT: buffer_wbinvl1_vol316; GCN2-NEXT: s_endpgm317;318; GCN3-LABEL: atomic_add_i32_addr64_offset:319; GCN3: ; %bb.0: ; %entry320; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34321; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24322; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c323; GCN3-NEXT: s_waitcnt lgkmcnt(0)324; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2325; GCN3-NEXT: s_add_u32 s0, s2, s0326; GCN3-NEXT: s_addc_u32 s1, s3, s1327; GCN3-NEXT: v_mov_b32_e32 v0, s0328; GCN3-NEXT: v_mov_b32_e32 v1, s1329; GCN3-NEXT: v_mov_b32_e32 v2, s6330; GCN3-NEXT: flat_atomic_add v[0:1], v2 offset:16331; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)332; GCN3-NEXT: buffer_wbinvl1_vol333; GCN3-NEXT: s_endpgm334;335; GFX11-LABEL: atomic_add_i32_addr64_offset:336; GFX11: ; %bb.0: ; %entry337; GFX11-NEXT: s_clause 0x2338; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34339; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24340; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c341; GFX11-NEXT: s_waitcnt lgkmcnt(0)342; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2343; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)344; GFX11-NEXT: s_add_u32 s0, s2, s0345; GFX11-NEXT: s_addc_u32 s1, s3, s1346; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1347; GFX11-NEXT: v_mov_b32_e32 v2, s4348; GFX11-NEXT: flat_atomic_add_u32 v[0:1], v2 offset:16349; GFX11-NEXT: s_waitcnt lgkmcnt(0)350; GFX11-NEXT: s_waitcnt_vscnt null, 0x0351; GFX11-NEXT: buffer_gl1_inv352; GFX11-NEXT: buffer_gl0_inv353; GFX11-NEXT: s_endpgm354entry:355 %ptr = getelementptr inbounds i32, ptr %out, i64 %index356 %gep = getelementptr inbounds i32, ptr %ptr, i32 4357 %val = atomicrmw volatile add ptr %gep, i32 %in syncscope("agent") seq_cst358 ret void359}360 361define amdgpu_kernel void @atomic_add_i32_ret_addr64_offset(ptr %out, ptr %out2, i32 %in, i64 %index) {362; GCN1-LABEL: atomic_add_i32_ret_addr64_offset:363; GCN1: ; %bb.0: ; %entry364; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf365; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9366; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd367; GCN1-NEXT: s_waitcnt lgkmcnt(0)368; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2369; GCN1-NEXT: s_add_u32 s0, s0, s4370; GCN1-NEXT: s_addc_u32 s1, s1, s5371; GCN1-NEXT: s_add_u32 s0, s0, 16372; GCN1-NEXT: s_addc_u32 s1, s1, 0373; GCN1-NEXT: v_mov_b32_e32 v0, s0374; GCN1-NEXT: v_mov_b32_e32 v1, s1375; GCN1-NEXT: v_mov_b32_e32 v2, s8376; GCN1-NEXT: flat_atomic_add v2, v[0:1], v2 glc377; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)378; GCN1-NEXT: buffer_wbinvl1_vol379; GCN1-NEXT: v_mov_b32_e32 v0, s2380; GCN1-NEXT: v_mov_b32_e32 v1, s3381; GCN1-NEXT: flat_store_dword v[0:1], v2382; GCN1-NEXT: s_endpgm383;384; GCN2-LABEL: atomic_add_i32_ret_addr64_offset:385; GCN2: ; %bb.0: ; %entry386; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c387; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24388; GCN2-NEXT: s_load_dword s8, s[4:5], 0x34389; GCN2-NEXT: s_waitcnt lgkmcnt(0)390; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2391; GCN2-NEXT: s_add_u32 s0, s0, s4392; GCN2-NEXT: s_addc_u32 s1, s1, s5393; GCN2-NEXT: s_add_u32 s0, s0, 16394; GCN2-NEXT: s_addc_u32 s1, s1, 0395; GCN2-NEXT: v_mov_b32_e32 v0, s0396; GCN2-NEXT: v_mov_b32_e32 v1, s1397; GCN2-NEXT: v_mov_b32_e32 v2, s8398; GCN2-NEXT: flat_atomic_add v2, v[0:1], v2 glc399; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)400; GCN2-NEXT: buffer_wbinvl1_vol401; GCN2-NEXT: v_mov_b32_e32 v0, s2402; GCN2-NEXT: v_mov_b32_e32 v1, s3403; GCN2-NEXT: flat_store_dword v[0:1], v2404; GCN2-NEXT: s_endpgm405;406; GCN3-LABEL: atomic_add_i32_ret_addr64_offset:407; GCN3: ; %bb.0: ; %entry408; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c409; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24410; GCN3-NEXT: s_load_dword s8, s[4:5], 0x34411; GCN3-NEXT: s_waitcnt lgkmcnt(0)412; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2413; GCN3-NEXT: s_add_u32 s0, s0, s4414; GCN3-NEXT: s_addc_u32 s1, s1, s5415; GCN3-NEXT: v_mov_b32_e32 v0, s0416; GCN3-NEXT: v_mov_b32_e32 v1, s1417; GCN3-NEXT: v_mov_b32_e32 v2, s8418; GCN3-NEXT: flat_atomic_add v2, v[0:1], v2 offset:16 glc419; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)420; GCN3-NEXT: buffer_wbinvl1_vol421; GCN3-NEXT: v_mov_b32_e32 v0, s2422; GCN3-NEXT: v_mov_b32_e32 v1, s3423; GCN3-NEXT: flat_store_dword v[0:1], v2424; GCN3-NEXT: s_endpgm425;426; GFX11-LABEL: atomic_add_i32_ret_addr64_offset:427; GFX11: ; %bb.0: ; %entry428; GFX11-NEXT: s_clause 0x2429; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c430; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24431; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x34432; GFX11-NEXT: s_waitcnt lgkmcnt(0)433; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2434; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)435; GFX11-NEXT: s_add_u32 s0, s0, s4436; GFX11-NEXT: s_addc_u32 s1, s1, s5437; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1438; GFX11-NEXT: v_mov_b32_e32 v2, s8439; GFX11-NEXT: flat_atomic_add_u32 v2, v[0:1], v2 offset:16 glc440; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)441; GFX11-NEXT: buffer_gl1_inv442; GFX11-NEXT: buffer_gl0_inv443; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3444; GFX11-NEXT: flat_store_b32 v[0:1], v2445; GFX11-NEXT: s_endpgm446entry:447 %ptr = getelementptr inbounds i32, ptr %out, i64 %index448 %gep = getelementptr inbounds i32, ptr %ptr, i32 4449 %val = atomicrmw volatile add ptr %gep, i32 %in syncscope("agent") seq_cst450 store i32 %val, ptr %out2451 ret void452}453 454define amdgpu_kernel void @atomic_add_i32(ptr %out, i32 %in) {455; GCN1-LABEL: atomic_add_i32:456; GCN1: ; %bb.0: ; %entry457; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9458; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb459; GCN1-NEXT: s_waitcnt lgkmcnt(0)460; GCN1-NEXT: v_mov_b32_e32 v0, s0461; GCN1-NEXT: v_mov_b32_e32 v1, s1462; GCN1-NEXT: v_mov_b32_e32 v2, s2463; GCN1-NEXT: flat_atomic_add v[0:1], v2464; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)465; GCN1-NEXT: buffer_wbinvl1_vol466; GCN1-NEXT: s_endpgm467;468; GCN2-LABEL: atomic_add_i32:469; GCN2: ; %bb.0: ; %entry470; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24471; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c472; GCN2-NEXT: s_waitcnt lgkmcnt(0)473; GCN2-NEXT: v_mov_b32_e32 v0, s0474; GCN2-NEXT: v_mov_b32_e32 v1, s1475; GCN2-NEXT: v_mov_b32_e32 v2, s2476; GCN2-NEXT: flat_atomic_add v[0:1], v2477; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)478; GCN2-NEXT: buffer_wbinvl1_vol479; GCN2-NEXT: s_endpgm480;481; GCN3-LABEL: atomic_add_i32:482; GCN3: ; %bb.0: ; %entry483; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24484; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c485; GCN3-NEXT: s_waitcnt lgkmcnt(0)486; GCN3-NEXT: v_mov_b32_e32 v0, s0487; GCN3-NEXT: v_mov_b32_e32 v1, s1488; GCN3-NEXT: v_mov_b32_e32 v2, s2489; GCN3-NEXT: flat_atomic_add v[0:1], v2490; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)491; GCN3-NEXT: buffer_wbinvl1_vol492; GCN3-NEXT: s_endpgm493;494; GFX11-LABEL: atomic_add_i32:495; GFX11: ; %bb.0: ; %entry496; GFX11-NEXT: s_clause 0x1497; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24498; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c499; GFX11-NEXT: s_waitcnt lgkmcnt(0)500; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1501; GFX11-NEXT: v_mov_b32_e32 v2, s2502; GFX11-NEXT: flat_atomic_add_u32 v[0:1], v2503; GFX11-NEXT: s_waitcnt lgkmcnt(0)504; GFX11-NEXT: s_waitcnt_vscnt null, 0x0505; GFX11-NEXT: buffer_gl1_inv506; GFX11-NEXT: buffer_gl0_inv507; GFX11-NEXT: s_endpgm508entry:509 %val = atomicrmw volatile add ptr %out, i32 %in syncscope("agent") seq_cst510 ret void511}512 513define amdgpu_kernel void @atomic_add_i32_ret(ptr %out, ptr %out2, i32 %in) {514; GCN1-LABEL: atomic_add_i32_ret:515; GCN1: ; %bb.0: ; %entry516; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9517; GCN1-NEXT: s_load_dword s4, s[4:5], 0xd518; GCN1-NEXT: s_waitcnt lgkmcnt(0)519; GCN1-NEXT: v_mov_b32_e32 v0, s0520; GCN1-NEXT: v_mov_b32_e32 v1, s1521; GCN1-NEXT: v_mov_b32_e32 v2, s4522; GCN1-NEXT: flat_atomic_add v2, v[0:1], v2 glc523; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)524; GCN1-NEXT: buffer_wbinvl1_vol525; GCN1-NEXT: v_mov_b32_e32 v0, s2526; GCN1-NEXT: v_mov_b32_e32 v1, s3527; GCN1-NEXT: flat_store_dword v[0:1], v2528; GCN1-NEXT: s_endpgm529;530; GCN2-LABEL: atomic_add_i32_ret:531; GCN2: ; %bb.0: ; %entry532; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24533; GCN2-NEXT: s_load_dword s4, s[4:5], 0x34534; GCN2-NEXT: s_waitcnt lgkmcnt(0)535; GCN2-NEXT: v_mov_b32_e32 v0, s0536; GCN2-NEXT: v_mov_b32_e32 v1, s1537; GCN2-NEXT: v_mov_b32_e32 v2, s4538; GCN2-NEXT: flat_atomic_add v2, v[0:1], v2 glc539; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)540; GCN2-NEXT: buffer_wbinvl1_vol541; GCN2-NEXT: v_mov_b32_e32 v0, s2542; GCN2-NEXT: v_mov_b32_e32 v1, s3543; GCN2-NEXT: flat_store_dword v[0:1], v2544; GCN2-NEXT: s_endpgm545;546; GCN3-LABEL: atomic_add_i32_ret:547; GCN3: ; %bb.0: ; %entry548; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24549; GCN3-NEXT: s_load_dword s6, s[4:5], 0x34550; GCN3-NEXT: s_waitcnt lgkmcnt(0)551; GCN3-NEXT: v_mov_b32_e32 v0, s0552; GCN3-NEXT: v_mov_b32_e32 v1, s1553; GCN3-NEXT: v_mov_b32_e32 v2, s6554; GCN3-NEXT: flat_atomic_add v2, v[0:1], v2 glc555; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)556; GCN3-NEXT: buffer_wbinvl1_vol557; GCN3-NEXT: v_mov_b32_e32 v0, s2558; GCN3-NEXT: v_mov_b32_e32 v1, s3559; GCN3-NEXT: flat_store_dword v[0:1], v2560; GCN3-NEXT: s_endpgm561;562; GFX11-LABEL: atomic_add_i32_ret:563; GFX11: ; %bb.0: ; %entry564; GFX11-NEXT: s_clause 0x1565; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24566; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x34567; GFX11-NEXT: s_waitcnt lgkmcnt(0)568; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1569; GFX11-NEXT: v_mov_b32_e32 v2, s4570; GFX11-NEXT: flat_atomic_add_u32 v2, v[0:1], v2 glc571; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)572; GFX11-NEXT: buffer_gl1_inv573; GFX11-NEXT: buffer_gl0_inv574; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3575; GFX11-NEXT: flat_store_b32 v[0:1], v2576; GFX11-NEXT: s_endpgm577entry:578 %val = atomicrmw volatile add ptr %out, i32 %in syncscope("agent") seq_cst579 store i32 %val, ptr %out2580 ret void581}582 583define amdgpu_kernel void @atomic_add_i32_addr64(ptr %out, i32 %in, i64 %index) {584; GCN1-LABEL: atomic_add_i32_addr64:585; GCN1: ; %bb.0: ; %entry586; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd587; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9588; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb589; GCN1-NEXT: s_waitcnt lgkmcnt(0)590; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2591; GCN1-NEXT: s_add_u32 s0, s2, s0592; GCN1-NEXT: s_addc_u32 s1, s3, s1593; GCN1-NEXT: v_mov_b32_e32 v0, s0594; GCN1-NEXT: v_mov_b32_e32 v1, s1595; GCN1-NEXT: v_mov_b32_e32 v2, s4596; GCN1-NEXT: flat_atomic_add v[0:1], v2597; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)598; GCN1-NEXT: buffer_wbinvl1_vol599; GCN1-NEXT: s_endpgm600;601; GCN2-LABEL: atomic_add_i32_addr64:602; GCN2: ; %bb.0: ; %entry603; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34604; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24605; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c606; GCN2-NEXT: s_waitcnt lgkmcnt(0)607; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2608; GCN2-NEXT: s_add_u32 s0, s2, s0609; GCN2-NEXT: s_addc_u32 s1, s3, s1610; GCN2-NEXT: v_mov_b32_e32 v0, s0611; GCN2-NEXT: v_mov_b32_e32 v1, s1612; GCN2-NEXT: v_mov_b32_e32 v2, s4613; GCN2-NEXT: flat_atomic_add v[0:1], v2614; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)615; GCN2-NEXT: buffer_wbinvl1_vol616; GCN2-NEXT: s_endpgm617;618; GCN3-LABEL: atomic_add_i32_addr64:619; GCN3: ; %bb.0: ; %entry620; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34621; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24622; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c623; GCN3-NEXT: s_waitcnt lgkmcnt(0)624; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2625; GCN3-NEXT: s_add_u32 s0, s2, s0626; GCN3-NEXT: s_addc_u32 s1, s3, s1627; GCN3-NEXT: v_mov_b32_e32 v0, s0628; GCN3-NEXT: v_mov_b32_e32 v1, s1629; GCN3-NEXT: v_mov_b32_e32 v2, s6630; GCN3-NEXT: flat_atomic_add v[0:1], v2631; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)632; GCN3-NEXT: buffer_wbinvl1_vol633; GCN3-NEXT: s_endpgm634;635; GFX11-LABEL: atomic_add_i32_addr64:636; GFX11: ; %bb.0: ; %entry637; GFX11-NEXT: s_clause 0x2638; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34639; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24640; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c641; GFX11-NEXT: s_waitcnt lgkmcnt(0)642; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2643; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)644; GFX11-NEXT: s_add_u32 s0, s2, s0645; GFX11-NEXT: s_addc_u32 s1, s3, s1646; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1647; GFX11-NEXT: v_mov_b32_e32 v2, s4648; GFX11-NEXT: flat_atomic_add_u32 v[0:1], v2649; GFX11-NEXT: s_waitcnt lgkmcnt(0)650; GFX11-NEXT: s_waitcnt_vscnt null, 0x0651; GFX11-NEXT: buffer_gl1_inv652; GFX11-NEXT: buffer_gl0_inv653; GFX11-NEXT: s_endpgm654entry:655 %ptr = getelementptr inbounds i32, ptr %out, i64 %index656 %val = atomicrmw volatile add ptr %ptr, i32 %in syncscope("agent") seq_cst657 ret void658}659 660define amdgpu_kernel void @atomic_add_i32_ret_addr64(ptr %out, ptr %out2, i32 %in, i64 %index) {661; GCN1-LABEL: atomic_add_i32_ret_addr64:662; GCN1: ; %bb.0: ; %entry663; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf664; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9665; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd666; GCN1-NEXT: s_waitcnt lgkmcnt(0)667; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2668; GCN1-NEXT: s_add_u32 s0, s0, s4669; GCN1-NEXT: s_addc_u32 s1, s1, s5670; GCN1-NEXT: v_mov_b32_e32 v0, s0671; GCN1-NEXT: v_mov_b32_e32 v1, s1672; GCN1-NEXT: v_mov_b32_e32 v2, s8673; GCN1-NEXT: flat_atomic_add v2, v[0:1], v2 glc674; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)675; GCN1-NEXT: buffer_wbinvl1_vol676; GCN1-NEXT: v_mov_b32_e32 v0, s2677; GCN1-NEXT: v_mov_b32_e32 v1, s3678; GCN1-NEXT: flat_store_dword v[0:1], v2679; GCN1-NEXT: s_endpgm680;681; GCN2-LABEL: atomic_add_i32_ret_addr64:682; GCN2: ; %bb.0: ; %entry683; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c684; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24685; GCN2-NEXT: s_load_dword s8, s[4:5], 0x34686; GCN2-NEXT: s_waitcnt lgkmcnt(0)687; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2688; GCN2-NEXT: s_add_u32 s0, s0, s4689; GCN2-NEXT: s_addc_u32 s1, s1, s5690; GCN2-NEXT: v_mov_b32_e32 v0, s0691; GCN2-NEXT: v_mov_b32_e32 v1, s1692; GCN2-NEXT: v_mov_b32_e32 v2, s8693; GCN2-NEXT: flat_atomic_add v2, v[0:1], v2 glc694; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)695; GCN2-NEXT: buffer_wbinvl1_vol696; GCN2-NEXT: v_mov_b32_e32 v0, s2697; GCN2-NEXT: v_mov_b32_e32 v1, s3698; GCN2-NEXT: flat_store_dword v[0:1], v2699; GCN2-NEXT: s_endpgm700;701; GCN3-LABEL: atomic_add_i32_ret_addr64:702; GCN3: ; %bb.0: ; %entry703; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c704; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24705; GCN3-NEXT: s_load_dword s8, s[4:5], 0x34706; GCN3-NEXT: s_waitcnt lgkmcnt(0)707; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2708; GCN3-NEXT: s_add_u32 s0, s0, s4709; GCN3-NEXT: s_addc_u32 s1, s1, s5710; GCN3-NEXT: v_mov_b32_e32 v0, s0711; GCN3-NEXT: v_mov_b32_e32 v1, s1712; GCN3-NEXT: v_mov_b32_e32 v2, s8713; GCN3-NEXT: flat_atomic_add v2, v[0:1], v2 glc714; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)715; GCN3-NEXT: buffer_wbinvl1_vol716; GCN3-NEXT: v_mov_b32_e32 v0, s2717; GCN3-NEXT: v_mov_b32_e32 v1, s3718; GCN3-NEXT: flat_store_dword v[0:1], v2719; GCN3-NEXT: s_endpgm720;721; GFX11-LABEL: atomic_add_i32_ret_addr64:722; GFX11: ; %bb.0: ; %entry723; GFX11-NEXT: s_clause 0x2724; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c725; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24726; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x34727; GFX11-NEXT: s_waitcnt lgkmcnt(0)728; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2729; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)730; GFX11-NEXT: s_add_u32 s0, s0, s4731; GFX11-NEXT: s_addc_u32 s1, s1, s5732; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1733; GFX11-NEXT: v_mov_b32_e32 v2, s8734; GFX11-NEXT: flat_atomic_add_u32 v2, v[0:1], v2 glc735; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)736; GFX11-NEXT: buffer_gl1_inv737; GFX11-NEXT: buffer_gl0_inv738; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3739; GFX11-NEXT: flat_store_b32 v[0:1], v2740; GFX11-NEXT: s_endpgm741entry:742 %ptr = getelementptr inbounds i32, ptr %out, i64 %index743 %val = atomicrmw volatile add ptr %ptr, i32 %in syncscope("agent") seq_cst744 store i32 %val, ptr %out2745 ret void746}747 748define amdgpu_kernel void @atomic_and_i32_offset(ptr %out, i32 %in) {749; GCN1-LABEL: atomic_and_i32_offset:750; GCN1: ; %bb.0: ; %entry751; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9752; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb753; GCN1-NEXT: s_waitcnt lgkmcnt(0)754; GCN1-NEXT: s_add_u32 s0, s0, 16755; GCN1-NEXT: s_addc_u32 s1, s1, 0756; GCN1-NEXT: v_mov_b32_e32 v0, s0757; GCN1-NEXT: v_mov_b32_e32 v1, s1758; GCN1-NEXT: v_mov_b32_e32 v2, s2759; GCN1-NEXT: flat_atomic_and v[0:1], v2760; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)761; GCN1-NEXT: buffer_wbinvl1_vol762; GCN1-NEXT: s_endpgm763;764; GCN2-LABEL: atomic_and_i32_offset:765; GCN2: ; %bb.0: ; %entry766; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24767; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c768; GCN2-NEXT: s_waitcnt lgkmcnt(0)769; GCN2-NEXT: s_add_u32 s0, s0, 16770; GCN2-NEXT: s_addc_u32 s1, s1, 0771; GCN2-NEXT: v_mov_b32_e32 v0, s0772; GCN2-NEXT: v_mov_b32_e32 v1, s1773; GCN2-NEXT: v_mov_b32_e32 v2, s2774; GCN2-NEXT: flat_atomic_and v[0:1], v2775; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)776; GCN2-NEXT: buffer_wbinvl1_vol777; GCN2-NEXT: s_endpgm778;779; GCN3-LABEL: atomic_and_i32_offset:780; GCN3: ; %bb.0: ; %entry781; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24782; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c783; GCN3-NEXT: s_waitcnt lgkmcnt(0)784; GCN3-NEXT: v_mov_b32_e32 v0, s0785; GCN3-NEXT: v_mov_b32_e32 v1, s1786; GCN3-NEXT: v_mov_b32_e32 v2, s2787; GCN3-NEXT: flat_atomic_and v[0:1], v2 offset:16788; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)789; GCN3-NEXT: buffer_wbinvl1_vol790; GCN3-NEXT: s_endpgm791;792; GFX11-LABEL: atomic_and_i32_offset:793; GFX11: ; %bb.0: ; %entry794; GFX11-NEXT: s_clause 0x1795; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24796; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c797; GFX11-NEXT: s_waitcnt lgkmcnt(0)798; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1799; GFX11-NEXT: v_mov_b32_e32 v2, s2800; GFX11-NEXT: flat_atomic_and_b32 v[0:1], v2 offset:16801; GFX11-NEXT: s_waitcnt lgkmcnt(0)802; GFX11-NEXT: s_waitcnt_vscnt null, 0x0803; GFX11-NEXT: buffer_gl1_inv804; GFX11-NEXT: buffer_gl0_inv805; GFX11-NEXT: s_endpgm806entry:807 %gep = getelementptr inbounds i32, ptr %out, i32 4808 %val = atomicrmw volatile and ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0809 ret void810}811 812define amdgpu_kernel void @atomic_and_i32_ret_offset(ptr %out, ptr %out2, i32 %in) {813; GCN1-LABEL: atomic_and_i32_ret_offset:814; GCN1: ; %bb.0: ; %entry815; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9816; GCN1-NEXT: s_load_dword s4, s[4:5], 0xd817; GCN1-NEXT: s_waitcnt lgkmcnt(0)818; GCN1-NEXT: s_add_u32 s0, s0, 16819; GCN1-NEXT: s_addc_u32 s1, s1, 0820; GCN1-NEXT: v_mov_b32_e32 v0, s0821; GCN1-NEXT: v_mov_b32_e32 v1, s1822; GCN1-NEXT: v_mov_b32_e32 v2, s4823; GCN1-NEXT: flat_atomic_and v2, v[0:1], v2 glc824; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)825; GCN1-NEXT: buffer_wbinvl1_vol826; GCN1-NEXT: v_mov_b32_e32 v0, s2827; GCN1-NEXT: v_mov_b32_e32 v1, s3828; GCN1-NEXT: flat_store_dword v[0:1], v2829; GCN1-NEXT: s_endpgm830;831; GCN2-LABEL: atomic_and_i32_ret_offset:832; GCN2: ; %bb.0: ; %entry833; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24834; GCN2-NEXT: s_load_dword s4, s[4:5], 0x34835; GCN2-NEXT: s_waitcnt lgkmcnt(0)836; GCN2-NEXT: s_add_u32 s0, s0, 16837; GCN2-NEXT: s_addc_u32 s1, s1, 0838; GCN2-NEXT: v_mov_b32_e32 v0, s0839; GCN2-NEXT: v_mov_b32_e32 v1, s1840; GCN2-NEXT: v_mov_b32_e32 v2, s4841; GCN2-NEXT: flat_atomic_and v2, v[0:1], v2 glc842; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)843; GCN2-NEXT: buffer_wbinvl1_vol844; GCN2-NEXT: v_mov_b32_e32 v0, s2845; GCN2-NEXT: v_mov_b32_e32 v1, s3846; GCN2-NEXT: flat_store_dword v[0:1], v2847; GCN2-NEXT: s_endpgm848;849; GCN3-LABEL: atomic_and_i32_ret_offset:850; GCN3: ; %bb.0: ; %entry851; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24852; GCN3-NEXT: s_load_dword s6, s[4:5], 0x34853; GCN3-NEXT: s_waitcnt lgkmcnt(0)854; GCN3-NEXT: v_mov_b32_e32 v0, s0855; GCN3-NEXT: v_mov_b32_e32 v1, s1856; GCN3-NEXT: v_mov_b32_e32 v2, s6857; GCN3-NEXT: flat_atomic_and v2, v[0:1], v2 offset:16 glc858; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)859; GCN3-NEXT: buffer_wbinvl1_vol860; GCN3-NEXT: v_mov_b32_e32 v0, s2861; GCN3-NEXT: v_mov_b32_e32 v1, s3862; GCN3-NEXT: flat_store_dword v[0:1], v2863; GCN3-NEXT: s_endpgm864;865; GFX11-LABEL: atomic_and_i32_ret_offset:866; GFX11: ; %bb.0: ; %entry867; GFX11-NEXT: s_clause 0x1868; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24869; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x34870; GFX11-NEXT: s_waitcnt lgkmcnt(0)871; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1872; GFX11-NEXT: v_mov_b32_e32 v2, s4873; GFX11-NEXT: flat_atomic_and_b32 v2, v[0:1], v2 offset:16 glc874; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)875; GFX11-NEXT: buffer_gl1_inv876; GFX11-NEXT: buffer_gl0_inv877; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3878; GFX11-NEXT: flat_store_b32 v[0:1], v2879; GFX11-NEXT: s_endpgm880entry:881 %gep = getelementptr inbounds i32, ptr %out, i32 4882 %val = atomicrmw volatile and ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0883 store i32 %val, ptr %out2884 ret void885}886 887define amdgpu_kernel void @atomic_and_i32_addr64_offset(ptr %out, i32 %in, i64 %index) {888; GCN1-LABEL: atomic_and_i32_addr64_offset:889; GCN1: ; %bb.0: ; %entry890; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd891; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9892; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb893; GCN1-NEXT: s_waitcnt lgkmcnt(0)894; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2895; GCN1-NEXT: s_add_u32 s0, s2, s0896; GCN1-NEXT: s_addc_u32 s1, s3, s1897; GCN1-NEXT: s_add_u32 s0, s0, 16898; GCN1-NEXT: s_addc_u32 s1, s1, 0899; GCN1-NEXT: v_mov_b32_e32 v0, s0900; GCN1-NEXT: v_mov_b32_e32 v1, s1901; GCN1-NEXT: v_mov_b32_e32 v2, s4902; GCN1-NEXT: flat_atomic_and v[0:1], v2903; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)904; GCN1-NEXT: buffer_wbinvl1_vol905; GCN1-NEXT: s_endpgm906;907; GCN2-LABEL: atomic_and_i32_addr64_offset:908; GCN2: ; %bb.0: ; %entry909; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34910; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24911; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c912; GCN2-NEXT: s_waitcnt lgkmcnt(0)913; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2914; GCN2-NEXT: s_add_u32 s0, s2, s0915; GCN2-NEXT: s_addc_u32 s1, s3, s1916; GCN2-NEXT: s_add_u32 s0, s0, 16917; GCN2-NEXT: s_addc_u32 s1, s1, 0918; GCN2-NEXT: v_mov_b32_e32 v0, s0919; GCN2-NEXT: v_mov_b32_e32 v1, s1920; GCN2-NEXT: v_mov_b32_e32 v2, s4921; GCN2-NEXT: flat_atomic_and v[0:1], v2922; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)923; GCN2-NEXT: buffer_wbinvl1_vol924; GCN2-NEXT: s_endpgm925;926; GCN3-LABEL: atomic_and_i32_addr64_offset:927; GCN3: ; %bb.0: ; %entry928; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34929; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24930; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c931; GCN3-NEXT: s_waitcnt lgkmcnt(0)932; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2933; GCN3-NEXT: s_add_u32 s0, s2, s0934; GCN3-NEXT: s_addc_u32 s1, s3, s1935; GCN3-NEXT: v_mov_b32_e32 v0, s0936; GCN3-NEXT: v_mov_b32_e32 v1, s1937; GCN3-NEXT: v_mov_b32_e32 v2, s6938; GCN3-NEXT: flat_atomic_and v[0:1], v2 offset:16939; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)940; GCN3-NEXT: buffer_wbinvl1_vol941; GCN3-NEXT: s_endpgm942;943; GFX11-LABEL: atomic_and_i32_addr64_offset:944; GFX11: ; %bb.0: ; %entry945; GFX11-NEXT: s_clause 0x2946; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34947; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24948; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c949; GFX11-NEXT: s_waitcnt lgkmcnt(0)950; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2951; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)952; GFX11-NEXT: s_add_u32 s0, s2, s0953; GFX11-NEXT: s_addc_u32 s1, s3, s1954; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1955; GFX11-NEXT: v_mov_b32_e32 v2, s4956; GFX11-NEXT: flat_atomic_and_b32 v[0:1], v2 offset:16957; GFX11-NEXT: s_waitcnt lgkmcnt(0)958; GFX11-NEXT: s_waitcnt_vscnt null, 0x0959; GFX11-NEXT: buffer_gl1_inv960; GFX11-NEXT: buffer_gl0_inv961; GFX11-NEXT: s_endpgm962entry:963 %ptr = getelementptr inbounds i32, ptr %out, i64 %index964 %gep = getelementptr inbounds i32, ptr %ptr, i32 4965 %val = atomicrmw volatile and ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0966 ret void967}968 969define amdgpu_kernel void @atomic_and_i32_ret_addr64_offset(ptr %out, ptr %out2, i32 %in, i64 %index) {970; GCN1-LABEL: atomic_and_i32_ret_addr64_offset:971; GCN1: ; %bb.0: ; %entry972; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf973; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9974; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd975; GCN1-NEXT: s_waitcnt lgkmcnt(0)976; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2977; GCN1-NEXT: s_add_u32 s0, s0, s4978; GCN1-NEXT: s_addc_u32 s1, s1, s5979; GCN1-NEXT: s_add_u32 s0, s0, 16980; GCN1-NEXT: s_addc_u32 s1, s1, 0981; GCN1-NEXT: v_mov_b32_e32 v0, s0982; GCN1-NEXT: v_mov_b32_e32 v1, s1983; GCN1-NEXT: v_mov_b32_e32 v2, s8984; GCN1-NEXT: flat_atomic_and v2, v[0:1], v2 glc985; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)986; GCN1-NEXT: buffer_wbinvl1_vol987; GCN1-NEXT: v_mov_b32_e32 v0, s2988; GCN1-NEXT: v_mov_b32_e32 v1, s3989; GCN1-NEXT: flat_store_dword v[0:1], v2990; GCN1-NEXT: s_endpgm991;992; GCN2-LABEL: atomic_and_i32_ret_addr64_offset:993; GCN2: ; %bb.0: ; %entry994; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c995; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24996; GCN2-NEXT: s_load_dword s8, s[4:5], 0x34997; GCN2-NEXT: s_waitcnt lgkmcnt(0)998; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2999; GCN2-NEXT: s_add_u32 s0, s0, s41000; GCN2-NEXT: s_addc_u32 s1, s1, s51001; GCN2-NEXT: s_add_u32 s0, s0, 161002; GCN2-NEXT: s_addc_u32 s1, s1, 01003; GCN2-NEXT: v_mov_b32_e32 v0, s01004; GCN2-NEXT: v_mov_b32_e32 v1, s11005; GCN2-NEXT: v_mov_b32_e32 v2, s81006; GCN2-NEXT: flat_atomic_and v2, v[0:1], v2 glc1007; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1008; GCN2-NEXT: buffer_wbinvl1_vol1009; GCN2-NEXT: v_mov_b32_e32 v0, s21010; GCN2-NEXT: v_mov_b32_e32 v1, s31011; GCN2-NEXT: flat_store_dword v[0:1], v21012; GCN2-NEXT: s_endpgm1013;1014; GCN3-LABEL: atomic_and_i32_ret_addr64_offset:1015; GCN3: ; %bb.0: ; %entry1016; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c1017; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241018; GCN3-NEXT: s_load_dword s8, s[4:5], 0x341019; GCN3-NEXT: s_waitcnt lgkmcnt(0)1020; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 21021; GCN3-NEXT: s_add_u32 s0, s0, s41022; GCN3-NEXT: s_addc_u32 s1, s1, s51023; GCN3-NEXT: v_mov_b32_e32 v0, s01024; GCN3-NEXT: v_mov_b32_e32 v1, s11025; GCN3-NEXT: v_mov_b32_e32 v2, s81026; GCN3-NEXT: flat_atomic_and v2, v[0:1], v2 offset:16 glc1027; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1028; GCN3-NEXT: buffer_wbinvl1_vol1029; GCN3-NEXT: v_mov_b32_e32 v0, s21030; GCN3-NEXT: v_mov_b32_e32 v1, s31031; GCN3-NEXT: flat_store_dword v[0:1], v21032; GCN3-NEXT: s_endpgm1033;1034; GFX11-LABEL: atomic_and_i32_ret_addr64_offset:1035; GFX11: ; %bb.0: ; %entry1036; GFX11-NEXT: s_clause 0x21037; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c1038; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x241039; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x341040; GFX11-NEXT: s_waitcnt lgkmcnt(0)1041; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 21042; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)1043; GFX11-NEXT: s_add_u32 s0, s0, s41044; GFX11-NEXT: s_addc_u32 s1, s1, s51045; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11046; GFX11-NEXT: v_mov_b32_e32 v2, s81047; GFX11-NEXT: flat_atomic_and_b32 v2, v[0:1], v2 offset:16 glc1048; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1049; GFX11-NEXT: buffer_gl1_inv1050; GFX11-NEXT: buffer_gl0_inv1051; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s31052; GFX11-NEXT: flat_store_b32 v[0:1], v21053; GFX11-NEXT: s_endpgm1054entry:1055 %ptr = getelementptr inbounds i32, ptr %out, i64 %index1056 %gep = getelementptr inbounds i32, ptr %ptr, i32 41057 %val = atomicrmw volatile and ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01058 store i32 %val, ptr %out21059 ret void1060}1061 1062define amdgpu_kernel void @atomic_and_i32(ptr %out, i32 %in) {1063; GCN1-LABEL: atomic_and_i32:1064; GCN1: ; %bb.0: ; %entry1065; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x91066; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb1067; GCN1-NEXT: s_waitcnt lgkmcnt(0)1068; GCN1-NEXT: v_mov_b32_e32 v0, s01069; GCN1-NEXT: v_mov_b32_e32 v1, s11070; GCN1-NEXT: v_mov_b32_e32 v2, s21071; GCN1-NEXT: flat_atomic_and v[0:1], v21072; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1073; GCN1-NEXT: buffer_wbinvl1_vol1074; GCN1-NEXT: s_endpgm1075;1076; GCN2-LABEL: atomic_and_i32:1077; GCN2: ; %bb.0: ; %entry1078; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241079; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c1080; GCN2-NEXT: s_waitcnt lgkmcnt(0)1081; GCN2-NEXT: v_mov_b32_e32 v0, s01082; GCN2-NEXT: v_mov_b32_e32 v1, s11083; GCN2-NEXT: v_mov_b32_e32 v2, s21084; GCN2-NEXT: flat_atomic_and v[0:1], v21085; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1086; GCN2-NEXT: buffer_wbinvl1_vol1087; GCN2-NEXT: s_endpgm1088;1089; GCN3-LABEL: atomic_and_i32:1090; GCN3: ; %bb.0: ; %entry1091; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241092; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c1093; GCN3-NEXT: s_waitcnt lgkmcnt(0)1094; GCN3-NEXT: v_mov_b32_e32 v0, s01095; GCN3-NEXT: v_mov_b32_e32 v1, s11096; GCN3-NEXT: v_mov_b32_e32 v2, s21097; GCN3-NEXT: flat_atomic_and v[0:1], v21098; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1099; GCN3-NEXT: buffer_wbinvl1_vol1100; GCN3-NEXT: s_endpgm1101;1102; GFX11-LABEL: atomic_and_i32:1103; GFX11: ; %bb.0: ; %entry1104; GFX11-NEXT: s_clause 0x11105; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x241106; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c1107; GFX11-NEXT: s_waitcnt lgkmcnt(0)1108; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11109; GFX11-NEXT: v_mov_b32_e32 v2, s21110; GFX11-NEXT: flat_atomic_and_b32 v[0:1], v21111; GFX11-NEXT: s_waitcnt lgkmcnt(0)1112; GFX11-NEXT: s_waitcnt_vscnt null, 0x01113; GFX11-NEXT: buffer_gl1_inv1114; GFX11-NEXT: buffer_gl0_inv1115; GFX11-NEXT: s_endpgm1116entry:1117 %val = atomicrmw volatile and ptr %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01118 ret void1119}1120 1121define amdgpu_kernel void @atomic_and_i32_ret(ptr %out, ptr %out2, i32 %in) {1122; GCN1-LABEL: atomic_and_i32_ret:1123; GCN1: ; %bb.0: ; %entry1124; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91125; GCN1-NEXT: s_load_dword s4, s[4:5], 0xd1126; GCN1-NEXT: s_waitcnt lgkmcnt(0)1127; GCN1-NEXT: v_mov_b32_e32 v0, s01128; GCN1-NEXT: v_mov_b32_e32 v1, s11129; GCN1-NEXT: v_mov_b32_e32 v2, s41130; GCN1-NEXT: flat_atomic_and v2, v[0:1], v2 glc1131; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1132; GCN1-NEXT: buffer_wbinvl1_vol1133; GCN1-NEXT: v_mov_b32_e32 v0, s21134; GCN1-NEXT: v_mov_b32_e32 v1, s31135; GCN1-NEXT: flat_store_dword v[0:1], v21136; GCN1-NEXT: s_endpgm1137;1138; GCN2-LABEL: atomic_and_i32_ret:1139; GCN2: ; %bb.0: ; %entry1140; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241141; GCN2-NEXT: s_load_dword s4, s[4:5], 0x341142; GCN2-NEXT: s_waitcnt lgkmcnt(0)1143; GCN2-NEXT: v_mov_b32_e32 v0, s01144; GCN2-NEXT: v_mov_b32_e32 v1, s11145; GCN2-NEXT: v_mov_b32_e32 v2, s41146; GCN2-NEXT: flat_atomic_and v2, v[0:1], v2 glc1147; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1148; GCN2-NEXT: buffer_wbinvl1_vol1149; GCN2-NEXT: v_mov_b32_e32 v0, s21150; GCN2-NEXT: v_mov_b32_e32 v1, s31151; GCN2-NEXT: flat_store_dword v[0:1], v21152; GCN2-NEXT: s_endpgm1153;1154; GCN3-LABEL: atomic_and_i32_ret:1155; GCN3: ; %bb.0: ; %entry1156; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241157; GCN3-NEXT: s_load_dword s6, s[4:5], 0x341158; GCN3-NEXT: s_waitcnt lgkmcnt(0)1159; GCN3-NEXT: v_mov_b32_e32 v0, s01160; GCN3-NEXT: v_mov_b32_e32 v1, s11161; GCN3-NEXT: v_mov_b32_e32 v2, s61162; GCN3-NEXT: flat_atomic_and v2, v[0:1], v2 glc1163; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1164; GCN3-NEXT: buffer_wbinvl1_vol1165; GCN3-NEXT: v_mov_b32_e32 v0, s21166; GCN3-NEXT: v_mov_b32_e32 v1, s31167; GCN3-NEXT: flat_store_dword v[0:1], v21168; GCN3-NEXT: s_endpgm1169;1170; GFX11-LABEL: atomic_and_i32_ret:1171; GFX11: ; %bb.0: ; %entry1172; GFX11-NEXT: s_clause 0x11173; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x241174; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x341175; GFX11-NEXT: s_waitcnt lgkmcnt(0)1176; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11177; GFX11-NEXT: v_mov_b32_e32 v2, s41178; GFX11-NEXT: flat_atomic_and_b32 v2, v[0:1], v2 glc1179; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1180; GFX11-NEXT: buffer_gl1_inv1181; GFX11-NEXT: buffer_gl0_inv1182; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s31183; GFX11-NEXT: flat_store_b32 v[0:1], v21184; GFX11-NEXT: s_endpgm1185entry:1186 %val = atomicrmw volatile and ptr %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01187 store i32 %val, ptr %out21188 ret void1189}1190 1191define amdgpu_kernel void @atomic_and_i32_addr64(ptr %out, i32 %in, i64 %index) {1192; GCN1-LABEL: atomic_and_i32_addr64:1193; GCN1: ; %bb.0: ; %entry1194; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd1195; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x91196; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb1197; GCN1-NEXT: s_waitcnt lgkmcnt(0)1198; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 21199; GCN1-NEXT: s_add_u32 s0, s2, s01200; GCN1-NEXT: s_addc_u32 s1, s3, s11201; GCN1-NEXT: v_mov_b32_e32 v0, s01202; GCN1-NEXT: v_mov_b32_e32 v1, s11203; GCN1-NEXT: v_mov_b32_e32 v2, s41204; GCN1-NEXT: flat_atomic_and v[0:1], v21205; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1206; GCN1-NEXT: buffer_wbinvl1_vol1207; GCN1-NEXT: s_endpgm1208;1209; GCN2-LABEL: atomic_and_i32_addr64:1210; GCN2: ; %bb.0: ; %entry1211; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x341212; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x241213; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c1214; GCN2-NEXT: s_waitcnt lgkmcnt(0)1215; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 21216; GCN2-NEXT: s_add_u32 s0, s2, s01217; GCN2-NEXT: s_addc_u32 s1, s3, s11218; GCN2-NEXT: v_mov_b32_e32 v0, s01219; GCN2-NEXT: v_mov_b32_e32 v1, s11220; GCN2-NEXT: v_mov_b32_e32 v2, s41221; GCN2-NEXT: flat_atomic_and v[0:1], v21222; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1223; GCN2-NEXT: buffer_wbinvl1_vol1224; GCN2-NEXT: s_endpgm1225;1226; GCN3-LABEL: atomic_and_i32_addr64:1227; GCN3: ; %bb.0: ; %entry1228; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x341229; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x241230; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c1231; GCN3-NEXT: s_waitcnt lgkmcnt(0)1232; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 21233; GCN3-NEXT: s_add_u32 s0, s2, s01234; GCN3-NEXT: s_addc_u32 s1, s3, s11235; GCN3-NEXT: v_mov_b32_e32 v0, s01236; GCN3-NEXT: v_mov_b32_e32 v1, s11237; GCN3-NEXT: v_mov_b32_e32 v2, s61238; GCN3-NEXT: flat_atomic_and v[0:1], v21239; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1240; GCN3-NEXT: buffer_wbinvl1_vol1241; GCN3-NEXT: s_endpgm1242;1243; GFX11-LABEL: atomic_and_i32_addr64:1244; GFX11: ; %bb.0: ; %entry1245; GFX11-NEXT: s_clause 0x21246; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x341247; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x241248; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c1249; GFX11-NEXT: s_waitcnt lgkmcnt(0)1250; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 21251; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)1252; GFX11-NEXT: s_add_u32 s0, s2, s01253; GFX11-NEXT: s_addc_u32 s1, s3, s11254; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11255; GFX11-NEXT: v_mov_b32_e32 v2, s41256; GFX11-NEXT: flat_atomic_and_b32 v[0:1], v21257; GFX11-NEXT: s_waitcnt lgkmcnt(0)1258; GFX11-NEXT: s_waitcnt_vscnt null, 0x01259; GFX11-NEXT: buffer_gl1_inv1260; GFX11-NEXT: buffer_gl0_inv1261; GFX11-NEXT: s_endpgm1262entry:1263 %ptr = getelementptr inbounds i32, ptr %out, i64 %index1264 %val = atomicrmw volatile and ptr %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01265 ret void1266}1267 1268define amdgpu_kernel void @atomic_and_i32_ret_addr64(ptr %out, ptr %out2, i32 %in, i64 %index) {1269; GCN1-LABEL: atomic_and_i32_ret_addr64:1270; GCN1: ; %bb.0: ; %entry1271; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf1272; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91273; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd1274; GCN1-NEXT: s_waitcnt lgkmcnt(0)1275; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 21276; GCN1-NEXT: s_add_u32 s0, s0, s41277; GCN1-NEXT: s_addc_u32 s1, s1, s51278; GCN1-NEXT: v_mov_b32_e32 v0, s01279; GCN1-NEXT: v_mov_b32_e32 v1, s11280; GCN1-NEXT: v_mov_b32_e32 v2, s81281; GCN1-NEXT: flat_atomic_and v2, v[0:1], v2 glc1282; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1283; GCN1-NEXT: buffer_wbinvl1_vol1284; GCN1-NEXT: v_mov_b32_e32 v0, s21285; GCN1-NEXT: v_mov_b32_e32 v1, s31286; GCN1-NEXT: flat_store_dword v[0:1], v21287; GCN1-NEXT: s_endpgm1288;1289; GCN2-LABEL: atomic_and_i32_ret_addr64:1290; GCN2: ; %bb.0: ; %entry1291; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c1292; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241293; GCN2-NEXT: s_load_dword s8, s[4:5], 0x341294; GCN2-NEXT: s_waitcnt lgkmcnt(0)1295; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 21296; GCN2-NEXT: s_add_u32 s0, s0, s41297; GCN2-NEXT: s_addc_u32 s1, s1, s51298; GCN2-NEXT: v_mov_b32_e32 v0, s01299; GCN2-NEXT: v_mov_b32_e32 v1, s11300; GCN2-NEXT: v_mov_b32_e32 v2, s81301; GCN2-NEXT: flat_atomic_and v2, v[0:1], v2 glc1302; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1303; GCN2-NEXT: buffer_wbinvl1_vol1304; GCN2-NEXT: v_mov_b32_e32 v0, s21305; GCN2-NEXT: v_mov_b32_e32 v1, s31306; GCN2-NEXT: flat_store_dword v[0:1], v21307; GCN2-NEXT: s_endpgm1308;1309; GCN3-LABEL: atomic_and_i32_ret_addr64:1310; GCN3: ; %bb.0: ; %entry1311; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c1312; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241313; GCN3-NEXT: s_load_dword s8, s[4:5], 0x341314; GCN3-NEXT: s_waitcnt lgkmcnt(0)1315; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 21316; GCN3-NEXT: s_add_u32 s0, s0, s41317; GCN3-NEXT: s_addc_u32 s1, s1, s51318; GCN3-NEXT: v_mov_b32_e32 v0, s01319; GCN3-NEXT: v_mov_b32_e32 v1, s11320; GCN3-NEXT: v_mov_b32_e32 v2, s81321; GCN3-NEXT: flat_atomic_and v2, v[0:1], v2 glc1322; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1323; GCN3-NEXT: buffer_wbinvl1_vol1324; GCN3-NEXT: v_mov_b32_e32 v0, s21325; GCN3-NEXT: v_mov_b32_e32 v1, s31326; GCN3-NEXT: flat_store_dword v[0:1], v21327; GCN3-NEXT: s_endpgm1328;1329; GFX11-LABEL: atomic_and_i32_ret_addr64:1330; GFX11: ; %bb.0: ; %entry1331; GFX11-NEXT: s_clause 0x21332; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c1333; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x241334; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x341335; GFX11-NEXT: s_waitcnt lgkmcnt(0)1336; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 21337; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)1338; GFX11-NEXT: s_add_u32 s0, s0, s41339; GFX11-NEXT: s_addc_u32 s1, s1, s51340; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11341; GFX11-NEXT: v_mov_b32_e32 v2, s81342; GFX11-NEXT: flat_atomic_and_b32 v2, v[0:1], v2 glc1343; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1344; GFX11-NEXT: buffer_gl1_inv1345; GFX11-NEXT: buffer_gl0_inv1346; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s31347; GFX11-NEXT: flat_store_b32 v[0:1], v21348; GFX11-NEXT: s_endpgm1349entry:1350 %ptr = getelementptr inbounds i32, ptr %out, i64 %index1351 %val = atomicrmw volatile and ptr %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01352 store i32 %val, ptr %out21353 ret void1354}1355 1356define amdgpu_kernel void @atomic_sub_i32_offset(ptr %out, i32 %in) {1357; GCN1-LABEL: atomic_sub_i32_offset:1358; GCN1: ; %bb.0: ; %entry1359; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x91360; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb1361; GCN1-NEXT: s_waitcnt lgkmcnt(0)1362; GCN1-NEXT: s_add_u32 s0, s0, 161363; GCN1-NEXT: s_addc_u32 s1, s1, 01364; GCN1-NEXT: v_mov_b32_e32 v0, s01365; GCN1-NEXT: v_mov_b32_e32 v1, s11366; GCN1-NEXT: v_mov_b32_e32 v2, s21367; GCN1-NEXT: flat_atomic_sub v[0:1], v21368; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1369; GCN1-NEXT: buffer_wbinvl1_vol1370; GCN1-NEXT: s_endpgm1371;1372; GCN2-LABEL: atomic_sub_i32_offset:1373; GCN2: ; %bb.0: ; %entry1374; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241375; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c1376; GCN2-NEXT: s_waitcnt lgkmcnt(0)1377; GCN2-NEXT: s_add_u32 s0, s0, 161378; GCN2-NEXT: s_addc_u32 s1, s1, 01379; GCN2-NEXT: v_mov_b32_e32 v0, s01380; GCN2-NEXT: v_mov_b32_e32 v1, s11381; GCN2-NEXT: v_mov_b32_e32 v2, s21382; GCN2-NEXT: flat_atomic_sub v[0:1], v21383; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1384; GCN2-NEXT: buffer_wbinvl1_vol1385; GCN2-NEXT: s_endpgm1386;1387; GCN3-LABEL: atomic_sub_i32_offset:1388; GCN3: ; %bb.0: ; %entry1389; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241390; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c1391; GCN3-NEXT: s_waitcnt lgkmcnt(0)1392; GCN3-NEXT: v_mov_b32_e32 v0, s01393; GCN3-NEXT: v_mov_b32_e32 v1, s11394; GCN3-NEXT: v_mov_b32_e32 v2, s21395; GCN3-NEXT: flat_atomic_sub v[0:1], v2 offset:161396; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1397; GCN3-NEXT: buffer_wbinvl1_vol1398; GCN3-NEXT: s_endpgm1399;1400; GFX11-LABEL: atomic_sub_i32_offset:1401; GFX11: ; %bb.0: ; %entry1402; GFX11-NEXT: s_clause 0x11403; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x241404; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c1405; GFX11-NEXT: s_waitcnt lgkmcnt(0)1406; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11407; GFX11-NEXT: v_mov_b32_e32 v2, s21408; GFX11-NEXT: flat_atomic_sub_u32 v[0:1], v2 offset:161409; GFX11-NEXT: s_waitcnt lgkmcnt(0)1410; GFX11-NEXT: s_waitcnt_vscnt null, 0x01411; GFX11-NEXT: buffer_gl1_inv1412; GFX11-NEXT: buffer_gl0_inv1413; GFX11-NEXT: s_endpgm1414entry:1415 %gep = getelementptr inbounds i32, ptr %out, i32 41416 %val = atomicrmw volatile sub ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01417 ret void1418}1419 1420define amdgpu_kernel void @atomic_sub_i32_ret_offset(ptr %out, ptr %out2, i32 %in) {1421; GCN1-LABEL: atomic_sub_i32_ret_offset:1422; GCN1: ; %bb.0: ; %entry1423; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91424; GCN1-NEXT: s_load_dword s4, s[4:5], 0xd1425; GCN1-NEXT: s_waitcnt lgkmcnt(0)1426; GCN1-NEXT: s_add_u32 s0, s0, 161427; GCN1-NEXT: s_addc_u32 s1, s1, 01428; GCN1-NEXT: v_mov_b32_e32 v0, s01429; GCN1-NEXT: v_mov_b32_e32 v1, s11430; GCN1-NEXT: v_mov_b32_e32 v2, s41431; GCN1-NEXT: flat_atomic_sub v2, v[0:1], v2 glc1432; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1433; GCN1-NEXT: buffer_wbinvl1_vol1434; GCN1-NEXT: v_mov_b32_e32 v0, s21435; GCN1-NEXT: v_mov_b32_e32 v1, s31436; GCN1-NEXT: flat_store_dword v[0:1], v21437; GCN1-NEXT: s_endpgm1438;1439; GCN2-LABEL: atomic_sub_i32_ret_offset:1440; GCN2: ; %bb.0: ; %entry1441; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241442; GCN2-NEXT: s_load_dword s4, s[4:5], 0x341443; GCN2-NEXT: s_waitcnt lgkmcnt(0)1444; GCN2-NEXT: s_add_u32 s0, s0, 161445; GCN2-NEXT: s_addc_u32 s1, s1, 01446; GCN2-NEXT: v_mov_b32_e32 v0, s01447; GCN2-NEXT: v_mov_b32_e32 v1, s11448; GCN2-NEXT: v_mov_b32_e32 v2, s41449; GCN2-NEXT: flat_atomic_sub v2, v[0:1], v2 glc1450; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1451; GCN2-NEXT: buffer_wbinvl1_vol1452; GCN2-NEXT: v_mov_b32_e32 v0, s21453; GCN2-NEXT: v_mov_b32_e32 v1, s31454; GCN2-NEXT: flat_store_dword v[0:1], v21455; GCN2-NEXT: s_endpgm1456;1457; GCN3-LABEL: atomic_sub_i32_ret_offset:1458; GCN3: ; %bb.0: ; %entry1459; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241460; GCN3-NEXT: s_load_dword s6, s[4:5], 0x341461; GCN3-NEXT: s_waitcnt lgkmcnt(0)1462; GCN3-NEXT: v_mov_b32_e32 v0, s01463; GCN3-NEXT: v_mov_b32_e32 v1, s11464; GCN3-NEXT: v_mov_b32_e32 v2, s61465; GCN3-NEXT: flat_atomic_sub v2, v[0:1], v2 offset:16 glc1466; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1467; GCN3-NEXT: buffer_wbinvl1_vol1468; GCN3-NEXT: v_mov_b32_e32 v0, s21469; GCN3-NEXT: v_mov_b32_e32 v1, s31470; GCN3-NEXT: flat_store_dword v[0:1], v21471; GCN3-NEXT: s_endpgm1472;1473; GFX11-LABEL: atomic_sub_i32_ret_offset:1474; GFX11: ; %bb.0: ; %entry1475; GFX11-NEXT: s_clause 0x11476; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x241477; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x341478; GFX11-NEXT: s_waitcnt lgkmcnt(0)1479; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11480; GFX11-NEXT: v_mov_b32_e32 v2, s41481; GFX11-NEXT: flat_atomic_sub_u32 v2, v[0:1], v2 offset:16 glc1482; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1483; GFX11-NEXT: buffer_gl1_inv1484; GFX11-NEXT: buffer_gl0_inv1485; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s31486; GFX11-NEXT: flat_store_b32 v[0:1], v21487; GFX11-NEXT: s_endpgm1488entry:1489 %gep = getelementptr inbounds i32, ptr %out, i32 41490 %val = atomicrmw volatile sub ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01491 store i32 %val, ptr %out21492 ret void1493}1494 1495define amdgpu_kernel void @atomic_sub_i32_addr64_offset(ptr %out, i32 %in, i64 %index) {1496; GCN1-LABEL: atomic_sub_i32_addr64_offset:1497; GCN1: ; %bb.0: ; %entry1498; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd1499; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x91500; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb1501; GCN1-NEXT: s_waitcnt lgkmcnt(0)1502; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 21503; GCN1-NEXT: s_add_u32 s0, s2, s01504; GCN1-NEXT: s_addc_u32 s1, s3, s11505; GCN1-NEXT: s_add_u32 s0, s0, 161506; GCN1-NEXT: s_addc_u32 s1, s1, 01507; GCN1-NEXT: v_mov_b32_e32 v0, s01508; GCN1-NEXT: v_mov_b32_e32 v1, s11509; GCN1-NEXT: v_mov_b32_e32 v2, s41510; GCN1-NEXT: flat_atomic_sub v[0:1], v21511; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1512; GCN1-NEXT: buffer_wbinvl1_vol1513; GCN1-NEXT: s_endpgm1514;1515; GCN2-LABEL: atomic_sub_i32_addr64_offset:1516; GCN2: ; %bb.0: ; %entry1517; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x341518; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x241519; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c1520; GCN2-NEXT: s_waitcnt lgkmcnt(0)1521; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 21522; GCN2-NEXT: s_add_u32 s0, s2, s01523; GCN2-NEXT: s_addc_u32 s1, s3, s11524; GCN2-NEXT: s_add_u32 s0, s0, 161525; GCN2-NEXT: s_addc_u32 s1, s1, 01526; GCN2-NEXT: v_mov_b32_e32 v0, s01527; GCN2-NEXT: v_mov_b32_e32 v1, s11528; GCN2-NEXT: v_mov_b32_e32 v2, s41529; GCN2-NEXT: flat_atomic_sub v[0:1], v21530; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1531; GCN2-NEXT: buffer_wbinvl1_vol1532; GCN2-NEXT: s_endpgm1533;1534; GCN3-LABEL: atomic_sub_i32_addr64_offset:1535; GCN3: ; %bb.0: ; %entry1536; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x341537; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x241538; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c1539; GCN3-NEXT: s_waitcnt lgkmcnt(0)1540; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 21541; GCN3-NEXT: s_add_u32 s0, s2, s01542; GCN3-NEXT: s_addc_u32 s1, s3, s11543; GCN3-NEXT: v_mov_b32_e32 v0, s01544; GCN3-NEXT: v_mov_b32_e32 v1, s11545; GCN3-NEXT: v_mov_b32_e32 v2, s61546; GCN3-NEXT: flat_atomic_sub v[0:1], v2 offset:161547; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1548; GCN3-NEXT: buffer_wbinvl1_vol1549; GCN3-NEXT: s_endpgm1550;1551; GFX11-LABEL: atomic_sub_i32_addr64_offset:1552; GFX11: ; %bb.0: ; %entry1553; GFX11-NEXT: s_clause 0x21554; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x341555; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x241556; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c1557; GFX11-NEXT: s_waitcnt lgkmcnt(0)1558; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 21559; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)1560; GFX11-NEXT: s_add_u32 s0, s2, s01561; GFX11-NEXT: s_addc_u32 s1, s3, s11562; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11563; GFX11-NEXT: v_mov_b32_e32 v2, s41564; GFX11-NEXT: flat_atomic_sub_u32 v[0:1], v2 offset:161565; GFX11-NEXT: s_waitcnt lgkmcnt(0)1566; GFX11-NEXT: s_waitcnt_vscnt null, 0x01567; GFX11-NEXT: buffer_gl1_inv1568; GFX11-NEXT: buffer_gl0_inv1569; GFX11-NEXT: s_endpgm1570entry:1571 %ptr = getelementptr inbounds i32, ptr %out, i64 %index1572 %gep = getelementptr inbounds i32, ptr %ptr, i32 41573 %val = atomicrmw volatile sub ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01574 ret void1575}1576 1577define amdgpu_kernel void @atomic_sub_i32_ret_addr64_offset(ptr %out, ptr %out2, i32 %in, i64 %index) {1578; GCN1-LABEL: atomic_sub_i32_ret_addr64_offset:1579; GCN1: ; %bb.0: ; %entry1580; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf1581; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91582; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd1583; GCN1-NEXT: s_waitcnt lgkmcnt(0)1584; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 21585; GCN1-NEXT: s_add_u32 s0, s0, s41586; GCN1-NEXT: s_addc_u32 s1, s1, s51587; GCN1-NEXT: s_add_u32 s0, s0, 161588; GCN1-NEXT: s_addc_u32 s1, s1, 01589; GCN1-NEXT: v_mov_b32_e32 v0, s01590; GCN1-NEXT: v_mov_b32_e32 v1, s11591; GCN1-NEXT: v_mov_b32_e32 v2, s81592; GCN1-NEXT: flat_atomic_sub v2, v[0:1], v2 glc1593; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1594; GCN1-NEXT: buffer_wbinvl1_vol1595; GCN1-NEXT: v_mov_b32_e32 v0, s21596; GCN1-NEXT: v_mov_b32_e32 v1, s31597; GCN1-NEXT: flat_store_dword v[0:1], v21598; GCN1-NEXT: s_endpgm1599;1600; GCN2-LABEL: atomic_sub_i32_ret_addr64_offset:1601; GCN2: ; %bb.0: ; %entry1602; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c1603; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241604; GCN2-NEXT: s_load_dword s8, s[4:5], 0x341605; GCN2-NEXT: s_waitcnt lgkmcnt(0)1606; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 21607; GCN2-NEXT: s_add_u32 s0, s0, s41608; GCN2-NEXT: s_addc_u32 s1, s1, s51609; GCN2-NEXT: s_add_u32 s0, s0, 161610; GCN2-NEXT: s_addc_u32 s1, s1, 01611; GCN2-NEXT: v_mov_b32_e32 v0, s01612; GCN2-NEXT: v_mov_b32_e32 v1, s11613; GCN2-NEXT: v_mov_b32_e32 v2, s81614; GCN2-NEXT: flat_atomic_sub v2, v[0:1], v2 glc1615; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1616; GCN2-NEXT: buffer_wbinvl1_vol1617; GCN2-NEXT: v_mov_b32_e32 v0, s21618; GCN2-NEXT: v_mov_b32_e32 v1, s31619; GCN2-NEXT: flat_store_dword v[0:1], v21620; GCN2-NEXT: s_endpgm1621;1622; GCN3-LABEL: atomic_sub_i32_ret_addr64_offset:1623; GCN3: ; %bb.0: ; %entry1624; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c1625; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241626; GCN3-NEXT: s_load_dword s8, s[4:5], 0x341627; GCN3-NEXT: s_waitcnt lgkmcnt(0)1628; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 21629; GCN3-NEXT: s_add_u32 s0, s0, s41630; GCN3-NEXT: s_addc_u32 s1, s1, s51631; GCN3-NEXT: v_mov_b32_e32 v0, s01632; GCN3-NEXT: v_mov_b32_e32 v1, s11633; GCN3-NEXT: v_mov_b32_e32 v2, s81634; GCN3-NEXT: flat_atomic_sub v2, v[0:1], v2 offset:16 glc1635; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1636; GCN3-NEXT: buffer_wbinvl1_vol1637; GCN3-NEXT: v_mov_b32_e32 v0, s21638; GCN3-NEXT: v_mov_b32_e32 v1, s31639; GCN3-NEXT: flat_store_dword v[0:1], v21640; GCN3-NEXT: s_endpgm1641;1642; GFX11-LABEL: atomic_sub_i32_ret_addr64_offset:1643; GFX11: ; %bb.0: ; %entry1644; GFX11-NEXT: s_clause 0x21645; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c1646; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x241647; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x341648; GFX11-NEXT: s_waitcnt lgkmcnt(0)1649; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 21650; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)1651; GFX11-NEXT: s_add_u32 s0, s0, s41652; GFX11-NEXT: s_addc_u32 s1, s1, s51653; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11654; GFX11-NEXT: v_mov_b32_e32 v2, s81655; GFX11-NEXT: flat_atomic_sub_u32 v2, v[0:1], v2 offset:16 glc1656; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1657; GFX11-NEXT: buffer_gl1_inv1658; GFX11-NEXT: buffer_gl0_inv1659; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s31660; GFX11-NEXT: flat_store_b32 v[0:1], v21661; GFX11-NEXT: s_endpgm1662entry:1663 %ptr = getelementptr inbounds i32, ptr %out, i64 %index1664 %gep = getelementptr inbounds i32, ptr %ptr, i32 41665 %val = atomicrmw volatile sub ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01666 store i32 %val, ptr %out21667 ret void1668}1669 1670define amdgpu_kernel void @atomic_sub_i32(ptr %out, i32 %in) {1671; GCN1-LABEL: atomic_sub_i32:1672; GCN1: ; %bb.0: ; %entry1673; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x91674; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb1675; GCN1-NEXT: s_waitcnt lgkmcnt(0)1676; GCN1-NEXT: v_mov_b32_e32 v0, s01677; GCN1-NEXT: v_mov_b32_e32 v1, s11678; GCN1-NEXT: v_mov_b32_e32 v2, s21679; GCN1-NEXT: flat_atomic_sub v[0:1], v21680; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1681; GCN1-NEXT: buffer_wbinvl1_vol1682; GCN1-NEXT: s_endpgm1683;1684; GCN2-LABEL: atomic_sub_i32:1685; GCN2: ; %bb.0: ; %entry1686; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241687; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c1688; GCN2-NEXT: s_waitcnt lgkmcnt(0)1689; GCN2-NEXT: v_mov_b32_e32 v0, s01690; GCN2-NEXT: v_mov_b32_e32 v1, s11691; GCN2-NEXT: v_mov_b32_e32 v2, s21692; GCN2-NEXT: flat_atomic_sub v[0:1], v21693; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1694; GCN2-NEXT: buffer_wbinvl1_vol1695; GCN2-NEXT: s_endpgm1696;1697; GCN3-LABEL: atomic_sub_i32:1698; GCN3: ; %bb.0: ; %entry1699; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241700; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c1701; GCN3-NEXT: s_waitcnt lgkmcnt(0)1702; GCN3-NEXT: v_mov_b32_e32 v0, s01703; GCN3-NEXT: v_mov_b32_e32 v1, s11704; GCN3-NEXT: v_mov_b32_e32 v2, s21705; GCN3-NEXT: flat_atomic_sub v[0:1], v21706; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1707; GCN3-NEXT: buffer_wbinvl1_vol1708; GCN3-NEXT: s_endpgm1709;1710; GFX11-LABEL: atomic_sub_i32:1711; GFX11: ; %bb.0: ; %entry1712; GFX11-NEXT: s_clause 0x11713; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x241714; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c1715; GFX11-NEXT: s_waitcnt lgkmcnt(0)1716; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11717; GFX11-NEXT: v_mov_b32_e32 v2, s21718; GFX11-NEXT: flat_atomic_sub_u32 v[0:1], v21719; GFX11-NEXT: s_waitcnt lgkmcnt(0)1720; GFX11-NEXT: s_waitcnt_vscnt null, 0x01721; GFX11-NEXT: buffer_gl1_inv1722; GFX11-NEXT: buffer_gl0_inv1723; GFX11-NEXT: s_endpgm1724entry:1725 %val = atomicrmw volatile sub ptr %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01726 ret void1727}1728 1729define amdgpu_kernel void @atomic_sub_i32_ret(ptr %out, ptr %out2, i32 %in) {1730; GCN1-LABEL: atomic_sub_i32_ret:1731; GCN1: ; %bb.0: ; %entry1732; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91733; GCN1-NEXT: s_load_dword s4, s[4:5], 0xd1734; GCN1-NEXT: s_waitcnt lgkmcnt(0)1735; GCN1-NEXT: v_mov_b32_e32 v0, s01736; GCN1-NEXT: v_mov_b32_e32 v1, s11737; GCN1-NEXT: v_mov_b32_e32 v2, s41738; GCN1-NEXT: flat_atomic_sub v2, v[0:1], v2 glc1739; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1740; GCN1-NEXT: buffer_wbinvl1_vol1741; GCN1-NEXT: v_mov_b32_e32 v0, s21742; GCN1-NEXT: v_mov_b32_e32 v1, s31743; GCN1-NEXT: flat_store_dword v[0:1], v21744; GCN1-NEXT: s_endpgm1745;1746; GCN2-LABEL: atomic_sub_i32_ret:1747; GCN2: ; %bb.0: ; %entry1748; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241749; GCN2-NEXT: s_load_dword s4, s[4:5], 0x341750; GCN2-NEXT: s_waitcnt lgkmcnt(0)1751; GCN2-NEXT: v_mov_b32_e32 v0, s01752; GCN2-NEXT: v_mov_b32_e32 v1, s11753; GCN2-NEXT: v_mov_b32_e32 v2, s41754; GCN2-NEXT: flat_atomic_sub v2, v[0:1], v2 glc1755; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1756; GCN2-NEXT: buffer_wbinvl1_vol1757; GCN2-NEXT: v_mov_b32_e32 v0, s21758; GCN2-NEXT: v_mov_b32_e32 v1, s31759; GCN2-NEXT: flat_store_dword v[0:1], v21760; GCN2-NEXT: s_endpgm1761;1762; GCN3-LABEL: atomic_sub_i32_ret:1763; GCN3: ; %bb.0: ; %entry1764; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241765; GCN3-NEXT: s_load_dword s6, s[4:5], 0x341766; GCN3-NEXT: s_waitcnt lgkmcnt(0)1767; GCN3-NEXT: v_mov_b32_e32 v0, s01768; GCN3-NEXT: v_mov_b32_e32 v1, s11769; GCN3-NEXT: v_mov_b32_e32 v2, s61770; GCN3-NEXT: flat_atomic_sub v2, v[0:1], v2 glc1771; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1772; GCN3-NEXT: buffer_wbinvl1_vol1773; GCN3-NEXT: v_mov_b32_e32 v0, s21774; GCN3-NEXT: v_mov_b32_e32 v1, s31775; GCN3-NEXT: flat_store_dword v[0:1], v21776; GCN3-NEXT: s_endpgm1777;1778; GFX11-LABEL: atomic_sub_i32_ret:1779; GFX11: ; %bb.0: ; %entry1780; GFX11-NEXT: s_clause 0x11781; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x241782; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x341783; GFX11-NEXT: s_waitcnt lgkmcnt(0)1784; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11785; GFX11-NEXT: v_mov_b32_e32 v2, s41786; GFX11-NEXT: flat_atomic_sub_u32 v2, v[0:1], v2 glc1787; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1788; GFX11-NEXT: buffer_gl1_inv1789; GFX11-NEXT: buffer_gl0_inv1790; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s31791; GFX11-NEXT: flat_store_b32 v[0:1], v21792; GFX11-NEXT: s_endpgm1793entry:1794 %val = atomicrmw volatile sub ptr %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01795 store i32 %val, ptr %out21796 ret void1797}1798 1799define amdgpu_kernel void @atomic_sub_i32_addr64(ptr %out, i32 %in, i64 %index) {1800; GCN1-LABEL: atomic_sub_i32_addr64:1801; GCN1: ; %bb.0: ; %entry1802; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd1803; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x91804; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb1805; GCN1-NEXT: s_waitcnt lgkmcnt(0)1806; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 21807; GCN1-NEXT: s_add_u32 s0, s2, s01808; GCN1-NEXT: s_addc_u32 s1, s3, s11809; GCN1-NEXT: v_mov_b32_e32 v0, s01810; GCN1-NEXT: v_mov_b32_e32 v1, s11811; GCN1-NEXT: v_mov_b32_e32 v2, s41812; GCN1-NEXT: flat_atomic_sub v[0:1], v21813; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1814; GCN1-NEXT: buffer_wbinvl1_vol1815; GCN1-NEXT: s_endpgm1816;1817; GCN2-LABEL: atomic_sub_i32_addr64:1818; GCN2: ; %bb.0: ; %entry1819; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x341820; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x241821; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c1822; GCN2-NEXT: s_waitcnt lgkmcnt(0)1823; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 21824; GCN2-NEXT: s_add_u32 s0, s2, s01825; GCN2-NEXT: s_addc_u32 s1, s3, s11826; GCN2-NEXT: v_mov_b32_e32 v0, s01827; GCN2-NEXT: v_mov_b32_e32 v1, s11828; GCN2-NEXT: v_mov_b32_e32 v2, s41829; GCN2-NEXT: flat_atomic_sub v[0:1], v21830; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1831; GCN2-NEXT: buffer_wbinvl1_vol1832; GCN2-NEXT: s_endpgm1833;1834; GCN3-LABEL: atomic_sub_i32_addr64:1835; GCN3: ; %bb.0: ; %entry1836; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x341837; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x241838; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c1839; GCN3-NEXT: s_waitcnt lgkmcnt(0)1840; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 21841; GCN3-NEXT: s_add_u32 s0, s2, s01842; GCN3-NEXT: s_addc_u32 s1, s3, s11843; GCN3-NEXT: v_mov_b32_e32 v0, s01844; GCN3-NEXT: v_mov_b32_e32 v1, s11845; GCN3-NEXT: v_mov_b32_e32 v2, s61846; GCN3-NEXT: flat_atomic_sub v[0:1], v21847; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1848; GCN3-NEXT: buffer_wbinvl1_vol1849; GCN3-NEXT: s_endpgm1850;1851; GFX11-LABEL: atomic_sub_i32_addr64:1852; GFX11: ; %bb.0: ; %entry1853; GFX11-NEXT: s_clause 0x21854; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x341855; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x241856; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c1857; GFX11-NEXT: s_waitcnt lgkmcnt(0)1858; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 21859; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)1860; GFX11-NEXT: s_add_u32 s0, s2, s01861; GFX11-NEXT: s_addc_u32 s1, s3, s11862; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11863; GFX11-NEXT: v_mov_b32_e32 v2, s41864; GFX11-NEXT: flat_atomic_sub_u32 v[0:1], v21865; GFX11-NEXT: s_waitcnt lgkmcnt(0)1866; GFX11-NEXT: s_waitcnt_vscnt null, 0x01867; GFX11-NEXT: buffer_gl1_inv1868; GFX11-NEXT: buffer_gl0_inv1869; GFX11-NEXT: s_endpgm1870entry:1871 %ptr = getelementptr inbounds i32, ptr %out, i64 %index1872 %val = atomicrmw volatile sub ptr %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01873 ret void1874}1875 1876define amdgpu_kernel void @atomic_sub_i32_ret_addr64(ptr %out, ptr %out2, i32 %in, i64 %index) {1877; GCN1-LABEL: atomic_sub_i32_ret_addr64:1878; GCN1: ; %bb.0: ; %entry1879; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf1880; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91881; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd1882; GCN1-NEXT: s_waitcnt lgkmcnt(0)1883; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 21884; GCN1-NEXT: s_add_u32 s0, s0, s41885; GCN1-NEXT: s_addc_u32 s1, s1, s51886; GCN1-NEXT: v_mov_b32_e32 v0, s01887; GCN1-NEXT: v_mov_b32_e32 v1, s11888; GCN1-NEXT: v_mov_b32_e32 v2, s81889; GCN1-NEXT: flat_atomic_sub v2, v[0:1], v2 glc1890; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1891; GCN1-NEXT: buffer_wbinvl1_vol1892; GCN1-NEXT: v_mov_b32_e32 v0, s21893; GCN1-NEXT: v_mov_b32_e32 v1, s31894; GCN1-NEXT: flat_store_dword v[0:1], v21895; GCN1-NEXT: s_endpgm1896;1897; GCN2-LABEL: atomic_sub_i32_ret_addr64:1898; GCN2: ; %bb.0: ; %entry1899; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c1900; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241901; GCN2-NEXT: s_load_dword s8, s[4:5], 0x341902; GCN2-NEXT: s_waitcnt lgkmcnt(0)1903; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 21904; GCN2-NEXT: s_add_u32 s0, s0, s41905; GCN2-NEXT: s_addc_u32 s1, s1, s51906; GCN2-NEXT: v_mov_b32_e32 v0, s01907; GCN2-NEXT: v_mov_b32_e32 v1, s11908; GCN2-NEXT: v_mov_b32_e32 v2, s81909; GCN2-NEXT: flat_atomic_sub v2, v[0:1], v2 glc1910; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1911; GCN2-NEXT: buffer_wbinvl1_vol1912; GCN2-NEXT: v_mov_b32_e32 v0, s21913; GCN2-NEXT: v_mov_b32_e32 v1, s31914; GCN2-NEXT: flat_store_dword v[0:1], v21915; GCN2-NEXT: s_endpgm1916;1917; GCN3-LABEL: atomic_sub_i32_ret_addr64:1918; GCN3: ; %bb.0: ; %entry1919; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c1920; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241921; GCN3-NEXT: s_load_dword s8, s[4:5], 0x341922; GCN3-NEXT: s_waitcnt lgkmcnt(0)1923; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 21924; GCN3-NEXT: s_add_u32 s0, s0, s41925; GCN3-NEXT: s_addc_u32 s1, s1, s51926; GCN3-NEXT: v_mov_b32_e32 v0, s01927; GCN3-NEXT: v_mov_b32_e32 v1, s11928; GCN3-NEXT: v_mov_b32_e32 v2, s81929; GCN3-NEXT: flat_atomic_sub v2, v[0:1], v2 glc1930; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1931; GCN3-NEXT: buffer_wbinvl1_vol1932; GCN3-NEXT: v_mov_b32_e32 v0, s21933; GCN3-NEXT: v_mov_b32_e32 v1, s31934; GCN3-NEXT: flat_store_dword v[0:1], v21935; GCN3-NEXT: s_endpgm1936;1937; GFX11-LABEL: atomic_sub_i32_ret_addr64:1938; GFX11: ; %bb.0: ; %entry1939; GFX11-NEXT: s_clause 0x21940; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c1941; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x241942; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x341943; GFX11-NEXT: s_waitcnt lgkmcnt(0)1944; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 21945; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)1946; GFX11-NEXT: s_add_u32 s0, s0, s41947; GFX11-NEXT: s_addc_u32 s1, s1, s51948; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11949; GFX11-NEXT: v_mov_b32_e32 v2, s81950; GFX11-NEXT: flat_atomic_sub_u32 v2, v[0:1], v2 glc1951; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1952; GFX11-NEXT: buffer_gl1_inv1953; GFX11-NEXT: buffer_gl0_inv1954; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s31955; GFX11-NEXT: flat_store_b32 v[0:1], v21956; GFX11-NEXT: s_endpgm1957entry:1958 %ptr = getelementptr inbounds i32, ptr %out, i64 %index1959 %val = atomicrmw volatile sub ptr %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01960 store i32 %val, ptr %out21961 ret void1962}1963 1964define amdgpu_kernel void @atomic_max_i32_offset(ptr %out, i32 %in) {1965; GCN1-LABEL: atomic_max_i32_offset:1966; GCN1: ; %bb.0: ; %entry1967; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x91968; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb1969; GCN1-NEXT: s_waitcnt lgkmcnt(0)1970; GCN1-NEXT: s_add_u32 s0, s0, 161971; GCN1-NEXT: s_addc_u32 s1, s1, 01972; GCN1-NEXT: v_mov_b32_e32 v0, s01973; GCN1-NEXT: v_mov_b32_e32 v1, s11974; GCN1-NEXT: v_mov_b32_e32 v2, s21975; GCN1-NEXT: flat_atomic_smax v[0:1], v21976; GCN1-NEXT: s_waitcnt lgkmcnt(0)1977; GCN1-NEXT: s_endpgm1978;1979; GCN2-LABEL: atomic_max_i32_offset:1980; GCN2: ; %bb.0: ; %entry1981; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241982; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c1983; GCN2-NEXT: s_waitcnt lgkmcnt(0)1984; GCN2-NEXT: s_add_u32 s0, s0, 161985; GCN2-NEXT: s_addc_u32 s1, s1, 01986; GCN2-NEXT: v_mov_b32_e32 v0, s01987; GCN2-NEXT: v_mov_b32_e32 v1, s11988; GCN2-NEXT: v_mov_b32_e32 v2, s21989; GCN2-NEXT: flat_atomic_smax v[0:1], v21990; GCN2-NEXT: s_waitcnt lgkmcnt(0)1991; GCN2-NEXT: s_endpgm1992;1993; GCN3-LABEL: atomic_max_i32_offset:1994; GCN3: ; %bb.0: ; %entry1995; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x241996; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c1997; GCN3-NEXT: s_waitcnt lgkmcnt(0)1998; GCN3-NEXT: v_mov_b32_e32 v0, s01999; GCN3-NEXT: v_mov_b32_e32 v1, s12000; GCN3-NEXT: v_mov_b32_e32 v2, s22001; GCN3-NEXT: flat_atomic_smax v[0:1], v2 offset:162002; GCN3-NEXT: s_waitcnt lgkmcnt(0)2003; GCN3-NEXT: s_endpgm2004;2005; GFX11-LABEL: atomic_max_i32_offset:2006; GFX11: ; %bb.0: ; %entry2007; GFX11-NEXT: s_clause 0x12008; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x242009; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c2010; GFX11-NEXT: s_waitcnt lgkmcnt(0)2011; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12012; GFX11-NEXT: v_mov_b32_e32 v2, s22013; GFX11-NEXT: flat_atomic_max_i32 v[0:1], v2 offset:162014; GFX11-NEXT: s_waitcnt lgkmcnt(0)2015; GFX11-NEXT: s_waitcnt_vscnt null, 0x02016; GFX11-NEXT: buffer_gl0_inv2017; GFX11-NEXT: s_endpgm2018entry:2019 %gep = getelementptr inbounds i32, ptr %out, i32 42020 %val = atomicrmw volatile max ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !02021 ret void2022}2023 2024define amdgpu_kernel void @atomic_max_i32_ret_offset(ptr %out, ptr %out2, i32 %in) {2025; GCN1-LABEL: atomic_max_i32_ret_offset:2026; GCN1: ; %bb.0: ; %entry2027; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92028; GCN1-NEXT: s_load_dword s4, s[4:5], 0xd2029; GCN1-NEXT: s_waitcnt lgkmcnt(0)2030; GCN1-NEXT: s_add_u32 s0, s0, 162031; GCN1-NEXT: s_addc_u32 s1, s1, 02032; GCN1-NEXT: v_mov_b32_e32 v0, s02033; GCN1-NEXT: v_mov_b32_e32 v1, s12034; GCN1-NEXT: v_mov_b32_e32 v2, s42035; GCN1-NEXT: flat_atomic_smax v2, v[0:1], v2 glc2036; GCN1-NEXT: s_waitcnt lgkmcnt(0)2037; GCN1-NEXT: v_mov_b32_e32 v0, s22038; GCN1-NEXT: v_mov_b32_e32 v1, s32039; GCN1-NEXT: s_waitcnt vmcnt(0)2040; GCN1-NEXT: flat_store_dword v[0:1], v22041; GCN1-NEXT: s_endpgm2042;2043; GCN2-LABEL: atomic_max_i32_ret_offset:2044; GCN2: ; %bb.0: ; %entry2045; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242046; GCN2-NEXT: s_load_dword s4, s[4:5], 0x342047; GCN2-NEXT: s_waitcnt lgkmcnt(0)2048; GCN2-NEXT: s_add_u32 s0, s0, 162049; GCN2-NEXT: s_addc_u32 s1, s1, 02050; GCN2-NEXT: v_mov_b32_e32 v0, s02051; GCN2-NEXT: v_mov_b32_e32 v1, s12052; GCN2-NEXT: v_mov_b32_e32 v2, s42053; GCN2-NEXT: flat_atomic_smax v2, v[0:1], v2 glc2054; GCN2-NEXT: s_waitcnt lgkmcnt(0)2055; GCN2-NEXT: v_mov_b32_e32 v0, s22056; GCN2-NEXT: v_mov_b32_e32 v1, s32057; GCN2-NEXT: s_waitcnt vmcnt(0)2058; GCN2-NEXT: flat_store_dword v[0:1], v22059; GCN2-NEXT: s_endpgm2060;2061; GCN3-LABEL: atomic_max_i32_ret_offset:2062; GCN3: ; %bb.0: ; %entry2063; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242064; GCN3-NEXT: s_load_dword s6, s[4:5], 0x342065; GCN3-NEXT: s_waitcnt lgkmcnt(0)2066; GCN3-NEXT: v_mov_b32_e32 v0, s02067; GCN3-NEXT: v_mov_b32_e32 v1, s12068; GCN3-NEXT: v_mov_b32_e32 v2, s62069; GCN3-NEXT: flat_atomic_smax v2, v[0:1], v2 offset:16 glc2070; GCN3-NEXT: s_waitcnt lgkmcnt(0)2071; GCN3-NEXT: v_mov_b32_e32 v0, s22072; GCN3-NEXT: v_mov_b32_e32 v1, s32073; GCN3-NEXT: s_waitcnt vmcnt(0)2074; GCN3-NEXT: flat_store_dword v[0:1], v22075; GCN3-NEXT: s_endpgm2076;2077; GFX11-LABEL: atomic_max_i32_ret_offset:2078; GFX11: ; %bb.0: ; %entry2079; GFX11-NEXT: s_clause 0x12080; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x242081; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x342082; GFX11-NEXT: s_waitcnt lgkmcnt(0)2083; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12084; GFX11-NEXT: v_mov_b32_e32 v2, s42085; GFX11-NEXT: flat_atomic_max_i32 v2, v[0:1], v2 offset:16 glc2086; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2087; GFX11-NEXT: buffer_gl0_inv2088; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s32089; GFX11-NEXT: flat_store_b32 v[0:1], v22090; GFX11-NEXT: s_endpgm2091entry:2092 %gep = getelementptr inbounds i32, ptr %out, i32 42093 %val = atomicrmw volatile max ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !02094 store i32 %val, ptr %out22095 ret void2096}2097 2098define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr %out, i32 %in, i64 %index) {2099; GCN1-LABEL: atomic_max_i32_addr64_offset:2100; GCN1: ; %bb.0: ; %entry2101; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd2102; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x92103; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb2104; GCN1-NEXT: s_waitcnt lgkmcnt(0)2105; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 22106; GCN1-NEXT: s_add_u32 s0, s2, s02107; GCN1-NEXT: s_addc_u32 s1, s3, s12108; GCN1-NEXT: s_add_u32 s0, s0, 162109; GCN1-NEXT: s_addc_u32 s1, s1, 02110; GCN1-NEXT: v_mov_b32_e32 v0, s02111; GCN1-NEXT: v_mov_b32_e32 v1, s12112; GCN1-NEXT: v_mov_b32_e32 v2, s42113; GCN1-NEXT: flat_atomic_smax v[0:1], v22114; GCN1-NEXT: s_waitcnt lgkmcnt(0)2115; GCN1-NEXT: s_endpgm2116;2117; GCN2-LABEL: atomic_max_i32_addr64_offset:2118; GCN2: ; %bb.0: ; %entry2119; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x342120; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x242121; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c2122; GCN2-NEXT: s_waitcnt lgkmcnt(0)2123; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 22124; GCN2-NEXT: s_add_u32 s0, s2, s02125; GCN2-NEXT: s_addc_u32 s1, s3, s12126; GCN2-NEXT: s_add_u32 s0, s0, 162127; GCN2-NEXT: s_addc_u32 s1, s1, 02128; GCN2-NEXT: v_mov_b32_e32 v0, s02129; GCN2-NEXT: v_mov_b32_e32 v1, s12130; GCN2-NEXT: v_mov_b32_e32 v2, s42131; GCN2-NEXT: flat_atomic_smax v[0:1], v22132; GCN2-NEXT: s_waitcnt lgkmcnt(0)2133; GCN2-NEXT: s_endpgm2134;2135; GCN3-LABEL: atomic_max_i32_addr64_offset:2136; GCN3: ; %bb.0: ; %entry2137; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x342138; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x242139; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c2140; GCN3-NEXT: s_waitcnt lgkmcnt(0)2141; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 22142; GCN3-NEXT: s_add_u32 s0, s2, s02143; GCN3-NEXT: s_addc_u32 s1, s3, s12144; GCN3-NEXT: v_mov_b32_e32 v0, s02145; GCN3-NEXT: v_mov_b32_e32 v1, s12146; GCN3-NEXT: v_mov_b32_e32 v2, s62147; GCN3-NEXT: flat_atomic_smax v[0:1], v2 offset:162148; GCN3-NEXT: s_waitcnt lgkmcnt(0)2149; GCN3-NEXT: s_endpgm2150;2151; GFX11-LABEL: atomic_max_i32_addr64_offset:2152; GFX11: ; %bb.0: ; %entry2153; GFX11-NEXT: s_clause 0x22154; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x342155; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x242156; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c2157; GFX11-NEXT: s_waitcnt lgkmcnt(0)2158; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 22159; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)2160; GFX11-NEXT: s_add_u32 s0, s2, s02161; GFX11-NEXT: s_addc_u32 s1, s3, s12162; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12163; GFX11-NEXT: v_mov_b32_e32 v2, s42164; GFX11-NEXT: flat_atomic_max_i32 v[0:1], v2 offset:162165; GFX11-NEXT: s_waitcnt lgkmcnt(0)2166; GFX11-NEXT: s_waitcnt_vscnt null, 0x02167; GFX11-NEXT: buffer_gl0_inv2168; GFX11-NEXT: s_endpgm2169entry:2170 %ptr = getelementptr inbounds i32, ptr %out, i64 %index2171 %gep = getelementptr inbounds i32, ptr %ptr, i32 42172 %val = atomicrmw volatile max ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !02173 ret void2174}2175 2176define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr %out, ptr %out2, i32 %in, i64 %index) {2177; GCN1-LABEL: atomic_max_i32_ret_addr64_offset:2178; GCN1: ; %bb.0: ; %entry2179; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf2180; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92181; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd2182; GCN1-NEXT: s_waitcnt lgkmcnt(0)2183; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 22184; GCN1-NEXT: s_add_u32 s0, s0, s42185; GCN1-NEXT: s_addc_u32 s1, s1, s52186; GCN1-NEXT: s_add_u32 s0, s0, 162187; GCN1-NEXT: s_addc_u32 s1, s1, 02188; GCN1-NEXT: v_mov_b32_e32 v0, s02189; GCN1-NEXT: v_mov_b32_e32 v1, s12190; GCN1-NEXT: v_mov_b32_e32 v2, s82191; GCN1-NEXT: flat_atomic_smax v2, v[0:1], v2 glc2192; GCN1-NEXT: s_waitcnt lgkmcnt(0)2193; GCN1-NEXT: v_mov_b32_e32 v0, s22194; GCN1-NEXT: v_mov_b32_e32 v1, s32195; GCN1-NEXT: s_waitcnt vmcnt(0)2196; GCN1-NEXT: flat_store_dword v[0:1], v22197; GCN1-NEXT: s_endpgm2198;2199; GCN2-LABEL: atomic_max_i32_ret_addr64_offset:2200; GCN2: ; %bb.0: ; %entry2201; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c2202; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242203; GCN2-NEXT: s_load_dword s8, s[4:5], 0x342204; GCN2-NEXT: s_waitcnt lgkmcnt(0)2205; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 22206; GCN2-NEXT: s_add_u32 s0, s0, s42207; GCN2-NEXT: s_addc_u32 s1, s1, s52208; GCN2-NEXT: s_add_u32 s0, s0, 162209; GCN2-NEXT: s_addc_u32 s1, s1, 02210; GCN2-NEXT: v_mov_b32_e32 v0, s02211; GCN2-NEXT: v_mov_b32_e32 v1, s12212; GCN2-NEXT: v_mov_b32_e32 v2, s82213; GCN2-NEXT: flat_atomic_smax v2, v[0:1], v2 glc2214; GCN2-NEXT: s_waitcnt lgkmcnt(0)2215; GCN2-NEXT: v_mov_b32_e32 v0, s22216; GCN2-NEXT: v_mov_b32_e32 v1, s32217; GCN2-NEXT: s_waitcnt vmcnt(0)2218; GCN2-NEXT: flat_store_dword v[0:1], v22219; GCN2-NEXT: s_endpgm2220;2221; GCN3-LABEL: atomic_max_i32_ret_addr64_offset:2222; GCN3: ; %bb.0: ; %entry2223; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c2224; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242225; GCN3-NEXT: s_load_dword s8, s[4:5], 0x342226; GCN3-NEXT: s_waitcnt lgkmcnt(0)2227; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 22228; GCN3-NEXT: s_add_u32 s0, s0, s42229; GCN3-NEXT: s_addc_u32 s1, s1, s52230; GCN3-NEXT: v_mov_b32_e32 v0, s02231; GCN3-NEXT: v_mov_b32_e32 v1, s12232; GCN3-NEXT: v_mov_b32_e32 v2, s82233; GCN3-NEXT: flat_atomic_smax v2, v[0:1], v2 offset:16 glc2234; GCN3-NEXT: s_waitcnt lgkmcnt(0)2235; GCN3-NEXT: v_mov_b32_e32 v0, s22236; GCN3-NEXT: v_mov_b32_e32 v1, s32237; GCN3-NEXT: s_waitcnt vmcnt(0)2238; GCN3-NEXT: flat_store_dword v[0:1], v22239; GCN3-NEXT: s_endpgm2240;2241; GFX11-LABEL: atomic_max_i32_ret_addr64_offset:2242; GFX11: ; %bb.0: ; %entry2243; GFX11-NEXT: s_clause 0x22244; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c2245; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x242246; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x342247; GFX11-NEXT: s_waitcnt lgkmcnt(0)2248; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 22249; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)2250; GFX11-NEXT: s_add_u32 s0, s0, s42251; GFX11-NEXT: s_addc_u32 s1, s1, s52252; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12253; GFX11-NEXT: v_mov_b32_e32 v2, s82254; GFX11-NEXT: flat_atomic_max_i32 v2, v[0:1], v2 offset:16 glc2255; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2256; GFX11-NEXT: buffer_gl0_inv2257; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s32258; GFX11-NEXT: flat_store_b32 v[0:1], v22259; GFX11-NEXT: s_endpgm2260entry:2261 %ptr = getelementptr inbounds i32, ptr %out, i64 %index2262 %gep = getelementptr inbounds i32, ptr %ptr, i32 42263 %val = atomicrmw volatile max ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !02264 store i32 %val, ptr %out22265 ret void2266}2267 2268define amdgpu_kernel void @atomic_max_i32(ptr %out, i32 %in) {2269; GCN1-LABEL: atomic_max_i32:2270; GCN1: ; %bb.0: ; %entry2271; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x92272; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb2273; GCN1-NEXT: s_waitcnt lgkmcnt(0)2274; GCN1-NEXT: v_mov_b32_e32 v0, s02275; GCN1-NEXT: v_mov_b32_e32 v1, s12276; GCN1-NEXT: v_mov_b32_e32 v2, s22277; GCN1-NEXT: flat_atomic_smax v[0:1], v22278; GCN1-NEXT: s_waitcnt lgkmcnt(0)2279; GCN1-NEXT: s_endpgm2280;2281; GCN2-LABEL: atomic_max_i32:2282; GCN2: ; %bb.0: ; %entry2283; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x242284; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c2285; GCN2-NEXT: s_waitcnt lgkmcnt(0)2286; GCN2-NEXT: v_mov_b32_e32 v0, s02287; GCN2-NEXT: v_mov_b32_e32 v1, s12288; GCN2-NEXT: v_mov_b32_e32 v2, s22289; GCN2-NEXT: flat_atomic_smax v[0:1], v22290; GCN2-NEXT: s_waitcnt lgkmcnt(0)2291; GCN2-NEXT: s_endpgm2292;2293; GCN3-LABEL: atomic_max_i32:2294; GCN3: ; %bb.0: ; %entry2295; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x242296; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c2297; GCN3-NEXT: s_waitcnt lgkmcnt(0)2298; GCN3-NEXT: v_mov_b32_e32 v0, s02299; GCN3-NEXT: v_mov_b32_e32 v1, s12300; GCN3-NEXT: v_mov_b32_e32 v2, s22301; GCN3-NEXT: flat_atomic_smax v[0:1], v22302; GCN3-NEXT: s_waitcnt lgkmcnt(0)2303; GCN3-NEXT: s_endpgm2304;2305; GFX11-LABEL: atomic_max_i32:2306; GFX11: ; %bb.0: ; %entry2307; GFX11-NEXT: s_clause 0x12308; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x242309; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c2310; GFX11-NEXT: s_waitcnt lgkmcnt(0)2311; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12312; GFX11-NEXT: v_mov_b32_e32 v2, s22313; GFX11-NEXT: flat_atomic_max_i32 v[0:1], v22314; GFX11-NEXT: s_waitcnt lgkmcnt(0)2315; GFX11-NEXT: s_waitcnt_vscnt null, 0x02316; GFX11-NEXT: buffer_gl0_inv2317; GFX11-NEXT: s_endpgm2318entry:2319 %val = atomicrmw volatile max ptr %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !02320 ret void2321}2322 2323define amdgpu_kernel void @atomic_max_i32_ret(ptr %out, ptr %out2, i32 %in) {2324; GCN1-LABEL: atomic_max_i32_ret:2325; GCN1: ; %bb.0: ; %entry2326; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92327; GCN1-NEXT: s_load_dword s4, s[4:5], 0xd2328; GCN1-NEXT: s_waitcnt lgkmcnt(0)2329; GCN1-NEXT: v_mov_b32_e32 v0, s02330; GCN1-NEXT: v_mov_b32_e32 v1, s12331; GCN1-NEXT: v_mov_b32_e32 v2, s42332; GCN1-NEXT: flat_atomic_smax v2, v[0:1], v2 glc2333; GCN1-NEXT: s_waitcnt lgkmcnt(0)2334; GCN1-NEXT: v_mov_b32_e32 v0, s22335; GCN1-NEXT: v_mov_b32_e32 v1, s32336; GCN1-NEXT: s_waitcnt vmcnt(0)2337; GCN1-NEXT: flat_store_dword v[0:1], v22338; GCN1-NEXT: s_endpgm2339;2340; GCN2-LABEL: atomic_max_i32_ret:2341; GCN2: ; %bb.0: ; %entry2342; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242343; GCN2-NEXT: s_load_dword s4, s[4:5], 0x342344; GCN2-NEXT: s_waitcnt lgkmcnt(0)2345; GCN2-NEXT: v_mov_b32_e32 v0, s02346; GCN2-NEXT: v_mov_b32_e32 v1, s12347; GCN2-NEXT: v_mov_b32_e32 v2, s42348; GCN2-NEXT: flat_atomic_smax v2, v[0:1], v2 glc2349; GCN2-NEXT: s_waitcnt lgkmcnt(0)2350; GCN2-NEXT: v_mov_b32_e32 v0, s22351; GCN2-NEXT: v_mov_b32_e32 v1, s32352; GCN2-NEXT: s_waitcnt vmcnt(0)2353; GCN2-NEXT: flat_store_dword v[0:1], v22354; GCN2-NEXT: s_endpgm2355;2356; GCN3-LABEL: atomic_max_i32_ret:2357; GCN3: ; %bb.0: ; %entry2358; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242359; GCN3-NEXT: s_load_dword s6, s[4:5], 0x342360; GCN3-NEXT: s_waitcnt lgkmcnt(0)2361; GCN3-NEXT: v_mov_b32_e32 v0, s02362; GCN3-NEXT: v_mov_b32_e32 v1, s12363; GCN3-NEXT: v_mov_b32_e32 v2, s62364; GCN3-NEXT: flat_atomic_smax v2, v[0:1], v2 glc2365; GCN3-NEXT: s_waitcnt lgkmcnt(0)2366; GCN3-NEXT: v_mov_b32_e32 v0, s22367; GCN3-NEXT: v_mov_b32_e32 v1, s32368; GCN3-NEXT: s_waitcnt vmcnt(0)2369; GCN3-NEXT: flat_store_dword v[0:1], v22370; GCN3-NEXT: s_endpgm2371;2372; GFX11-LABEL: atomic_max_i32_ret:2373; GFX11: ; %bb.0: ; %entry2374; GFX11-NEXT: s_clause 0x12375; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x242376; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x342377; GFX11-NEXT: s_waitcnt lgkmcnt(0)2378; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12379; GFX11-NEXT: v_mov_b32_e32 v2, s42380; GFX11-NEXT: flat_atomic_max_i32 v2, v[0:1], v2 glc2381; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2382; GFX11-NEXT: buffer_gl0_inv2383; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s32384; GFX11-NEXT: flat_store_b32 v[0:1], v22385; GFX11-NEXT: s_endpgm2386entry:2387 %val = atomicrmw volatile max ptr %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !02388 store i32 %val, ptr %out22389 ret void2390}2391 2392define amdgpu_kernel void @atomic_max_i32_addr64(ptr %out, i32 %in, i64 %index) {2393; GCN1-LABEL: atomic_max_i32_addr64:2394; GCN1: ; %bb.0: ; %entry2395; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd2396; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x92397; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb2398; GCN1-NEXT: s_waitcnt lgkmcnt(0)2399; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 22400; GCN1-NEXT: s_add_u32 s0, s2, s02401; GCN1-NEXT: s_addc_u32 s1, s3, s12402; GCN1-NEXT: v_mov_b32_e32 v0, s02403; GCN1-NEXT: v_mov_b32_e32 v1, s12404; GCN1-NEXT: v_mov_b32_e32 v2, s42405; GCN1-NEXT: flat_atomic_smax v[0:1], v22406; GCN1-NEXT: s_waitcnt lgkmcnt(0)2407; GCN1-NEXT: s_endpgm2408;2409; GCN2-LABEL: atomic_max_i32_addr64:2410; GCN2: ; %bb.0: ; %entry2411; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x342412; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x242413; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c2414; GCN2-NEXT: s_waitcnt lgkmcnt(0)2415; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 22416; GCN2-NEXT: s_add_u32 s0, s2, s02417; GCN2-NEXT: s_addc_u32 s1, s3, s12418; GCN2-NEXT: v_mov_b32_e32 v0, s02419; GCN2-NEXT: v_mov_b32_e32 v1, s12420; GCN2-NEXT: v_mov_b32_e32 v2, s42421; GCN2-NEXT: flat_atomic_smax v[0:1], v22422; GCN2-NEXT: s_waitcnt lgkmcnt(0)2423; GCN2-NEXT: s_endpgm2424;2425; GCN3-LABEL: atomic_max_i32_addr64:2426; GCN3: ; %bb.0: ; %entry2427; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x342428; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x242429; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c2430; GCN3-NEXT: s_waitcnt lgkmcnt(0)2431; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 22432; GCN3-NEXT: s_add_u32 s0, s2, s02433; GCN3-NEXT: s_addc_u32 s1, s3, s12434; GCN3-NEXT: v_mov_b32_e32 v0, s02435; GCN3-NEXT: v_mov_b32_e32 v1, s12436; GCN3-NEXT: v_mov_b32_e32 v2, s62437; GCN3-NEXT: flat_atomic_smax v[0:1], v22438; GCN3-NEXT: s_waitcnt lgkmcnt(0)2439; GCN3-NEXT: s_endpgm2440;2441; GFX11-LABEL: atomic_max_i32_addr64:2442; GFX11: ; %bb.0: ; %entry2443; GFX11-NEXT: s_clause 0x22444; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x342445; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x242446; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c2447; GFX11-NEXT: s_waitcnt lgkmcnt(0)2448; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 22449; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)2450; GFX11-NEXT: s_add_u32 s0, s2, s02451; GFX11-NEXT: s_addc_u32 s1, s3, s12452; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12453; GFX11-NEXT: v_mov_b32_e32 v2, s42454; GFX11-NEXT: flat_atomic_max_i32 v[0:1], v22455; GFX11-NEXT: s_waitcnt lgkmcnt(0)2456; GFX11-NEXT: s_waitcnt_vscnt null, 0x02457; GFX11-NEXT: buffer_gl0_inv2458; GFX11-NEXT: s_endpgm2459entry:2460 %ptr = getelementptr inbounds i32, ptr %out, i64 %index2461 %val = atomicrmw volatile max ptr %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !02462 ret void2463}2464 2465define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr %out, ptr %out2, i32 %in, i64 %index) {2466; GCN1-LABEL: atomic_max_i32_ret_addr64:2467; GCN1: ; %bb.0: ; %entry2468; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf2469; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92470; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd2471; GCN1-NEXT: s_waitcnt lgkmcnt(0)2472; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 22473; GCN1-NEXT: s_add_u32 s0, s0, s42474; GCN1-NEXT: s_addc_u32 s1, s1, s52475; GCN1-NEXT: v_mov_b32_e32 v0, s02476; GCN1-NEXT: v_mov_b32_e32 v1, s12477; GCN1-NEXT: v_mov_b32_e32 v2, s82478; GCN1-NEXT: flat_atomic_smax v2, v[0:1], v2 glc2479; GCN1-NEXT: s_waitcnt lgkmcnt(0)2480; GCN1-NEXT: v_mov_b32_e32 v0, s22481; GCN1-NEXT: v_mov_b32_e32 v1, s32482; GCN1-NEXT: s_waitcnt vmcnt(0)2483; GCN1-NEXT: flat_store_dword v[0:1], v22484; GCN1-NEXT: s_endpgm2485;2486; GCN2-LABEL: atomic_max_i32_ret_addr64:2487; GCN2: ; %bb.0: ; %entry2488; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c2489; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242490; GCN2-NEXT: s_load_dword s8, s[4:5], 0x342491; GCN2-NEXT: s_waitcnt lgkmcnt(0)2492; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 22493; GCN2-NEXT: s_add_u32 s0, s0, s42494; GCN2-NEXT: s_addc_u32 s1, s1, s52495; GCN2-NEXT: v_mov_b32_e32 v0, s02496; GCN2-NEXT: v_mov_b32_e32 v1, s12497; GCN2-NEXT: v_mov_b32_e32 v2, s82498; GCN2-NEXT: flat_atomic_smax v2, v[0:1], v2 glc2499; GCN2-NEXT: s_waitcnt lgkmcnt(0)2500; GCN2-NEXT: v_mov_b32_e32 v0, s22501; GCN2-NEXT: v_mov_b32_e32 v1, s32502; GCN2-NEXT: s_waitcnt vmcnt(0)2503; GCN2-NEXT: flat_store_dword v[0:1], v22504; GCN2-NEXT: s_endpgm2505;2506; GCN3-LABEL: atomic_max_i32_ret_addr64:2507; GCN3: ; %bb.0: ; %entry2508; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c2509; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242510; GCN3-NEXT: s_load_dword s8, s[4:5], 0x342511; GCN3-NEXT: s_waitcnt lgkmcnt(0)2512; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 22513; GCN3-NEXT: s_add_u32 s0, s0, s42514; GCN3-NEXT: s_addc_u32 s1, s1, s52515; GCN3-NEXT: v_mov_b32_e32 v0, s02516; GCN3-NEXT: v_mov_b32_e32 v1, s12517; GCN3-NEXT: v_mov_b32_e32 v2, s82518; GCN3-NEXT: flat_atomic_smax v2, v[0:1], v2 glc2519; GCN3-NEXT: s_waitcnt lgkmcnt(0)2520; GCN3-NEXT: v_mov_b32_e32 v0, s22521; GCN3-NEXT: v_mov_b32_e32 v1, s32522; GCN3-NEXT: s_waitcnt vmcnt(0)2523; GCN3-NEXT: flat_store_dword v[0:1], v22524; GCN3-NEXT: s_endpgm2525;2526; GFX11-LABEL: atomic_max_i32_ret_addr64:2527; GFX11: ; %bb.0: ; %entry2528; GFX11-NEXT: s_clause 0x22529; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c2530; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x242531; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x342532; GFX11-NEXT: s_waitcnt lgkmcnt(0)2533; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 22534; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)2535; GFX11-NEXT: s_add_u32 s0, s0, s42536; GFX11-NEXT: s_addc_u32 s1, s1, s52537; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12538; GFX11-NEXT: v_mov_b32_e32 v2, s82539; GFX11-NEXT: flat_atomic_max_i32 v2, v[0:1], v2 glc2540; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2541; GFX11-NEXT: buffer_gl0_inv2542; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s32543; GFX11-NEXT: flat_store_b32 v[0:1], v22544; GFX11-NEXT: s_endpgm2545entry:2546 %ptr = getelementptr inbounds i32, ptr %out, i64 %index2547 %val = atomicrmw volatile max ptr %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !02548 store i32 %val, ptr %out22549 ret void2550}2551 2552define amdgpu_kernel void @atomic_umax_i32_offset(ptr %out, i32 %in) {2553; GCN1-LABEL: atomic_umax_i32_offset:2554; GCN1: ; %bb.0: ; %entry2555; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x92556; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb2557; GCN1-NEXT: s_waitcnt lgkmcnt(0)2558; GCN1-NEXT: s_add_u32 s0, s0, 162559; GCN1-NEXT: s_addc_u32 s1, s1, 02560; GCN1-NEXT: v_mov_b32_e32 v0, s02561; GCN1-NEXT: v_mov_b32_e32 v1, s12562; GCN1-NEXT: v_mov_b32_e32 v2, s22563; GCN1-NEXT: flat_atomic_umax v[0:1], v22564; GCN1-NEXT: s_waitcnt lgkmcnt(0)2565; GCN1-NEXT: s_endpgm2566;2567; GCN2-LABEL: atomic_umax_i32_offset:2568; GCN2: ; %bb.0: ; %entry2569; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x242570; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c2571; GCN2-NEXT: s_waitcnt lgkmcnt(0)2572; GCN2-NEXT: s_add_u32 s0, s0, 162573; GCN2-NEXT: s_addc_u32 s1, s1, 02574; GCN2-NEXT: v_mov_b32_e32 v0, s02575; GCN2-NEXT: v_mov_b32_e32 v1, s12576; GCN2-NEXT: v_mov_b32_e32 v2, s22577; GCN2-NEXT: flat_atomic_umax v[0:1], v22578; GCN2-NEXT: s_waitcnt lgkmcnt(0)2579; GCN2-NEXT: s_endpgm2580;2581; GCN3-LABEL: atomic_umax_i32_offset:2582; GCN3: ; %bb.0: ; %entry2583; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x242584; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c2585; GCN3-NEXT: s_waitcnt lgkmcnt(0)2586; GCN3-NEXT: v_mov_b32_e32 v0, s02587; GCN3-NEXT: v_mov_b32_e32 v1, s12588; GCN3-NEXT: v_mov_b32_e32 v2, s22589; GCN3-NEXT: flat_atomic_umax v[0:1], v2 offset:162590; GCN3-NEXT: s_waitcnt lgkmcnt(0)2591; GCN3-NEXT: s_endpgm2592;2593; GFX11-LABEL: atomic_umax_i32_offset:2594; GFX11: ; %bb.0: ; %entry2595; GFX11-NEXT: s_clause 0x12596; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x242597; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c2598; GFX11-NEXT: s_waitcnt lgkmcnt(0)2599; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12600; GFX11-NEXT: v_mov_b32_e32 v2, s22601; GFX11-NEXT: flat_atomic_max_u32 v[0:1], v2 offset:162602; GFX11-NEXT: s_waitcnt lgkmcnt(0)2603; GFX11-NEXT: s_waitcnt_vscnt null, 0x02604; GFX11-NEXT: buffer_gl0_inv2605; GFX11-NEXT: s_endpgm2606entry:2607 %gep = getelementptr inbounds i32, ptr %out, i32 42608 %val = atomicrmw volatile umax ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !02609 ret void2610}2611 2612define amdgpu_kernel void @atomic_umax_i32_ret_offset(ptr %out, ptr %out2, i32 %in) {2613; GCN1-LABEL: atomic_umax_i32_ret_offset:2614; GCN1: ; %bb.0: ; %entry2615; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92616; GCN1-NEXT: s_load_dword s4, s[4:5], 0xd2617; GCN1-NEXT: s_waitcnt lgkmcnt(0)2618; GCN1-NEXT: s_add_u32 s0, s0, 162619; GCN1-NEXT: s_addc_u32 s1, s1, 02620; GCN1-NEXT: v_mov_b32_e32 v0, s02621; GCN1-NEXT: v_mov_b32_e32 v1, s12622; GCN1-NEXT: v_mov_b32_e32 v2, s42623; GCN1-NEXT: flat_atomic_umax v2, v[0:1], v2 glc2624; GCN1-NEXT: s_waitcnt lgkmcnt(0)2625; GCN1-NEXT: v_mov_b32_e32 v0, s22626; GCN1-NEXT: v_mov_b32_e32 v1, s32627; GCN1-NEXT: s_waitcnt vmcnt(0)2628; GCN1-NEXT: flat_store_dword v[0:1], v22629; GCN1-NEXT: s_endpgm2630;2631; GCN2-LABEL: atomic_umax_i32_ret_offset:2632; GCN2: ; %bb.0: ; %entry2633; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242634; GCN2-NEXT: s_load_dword s4, s[4:5], 0x342635; GCN2-NEXT: s_waitcnt lgkmcnt(0)2636; GCN2-NEXT: s_add_u32 s0, s0, 162637; GCN2-NEXT: s_addc_u32 s1, s1, 02638; GCN2-NEXT: v_mov_b32_e32 v0, s02639; GCN2-NEXT: v_mov_b32_e32 v1, s12640; GCN2-NEXT: v_mov_b32_e32 v2, s42641; GCN2-NEXT: flat_atomic_umax v2, v[0:1], v2 glc2642; GCN2-NEXT: s_waitcnt lgkmcnt(0)2643; GCN2-NEXT: v_mov_b32_e32 v0, s22644; GCN2-NEXT: v_mov_b32_e32 v1, s32645; GCN2-NEXT: s_waitcnt vmcnt(0)2646; GCN2-NEXT: flat_store_dword v[0:1], v22647; GCN2-NEXT: s_endpgm2648;2649; GCN3-LABEL: atomic_umax_i32_ret_offset:2650; GCN3: ; %bb.0: ; %entry2651; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242652; GCN3-NEXT: s_load_dword s6, s[4:5], 0x342653; GCN3-NEXT: s_waitcnt lgkmcnt(0)2654; GCN3-NEXT: v_mov_b32_e32 v0, s02655; GCN3-NEXT: v_mov_b32_e32 v1, s12656; GCN3-NEXT: v_mov_b32_e32 v2, s62657; GCN3-NEXT: flat_atomic_umax v2, v[0:1], v2 offset:16 glc2658; GCN3-NEXT: s_waitcnt lgkmcnt(0)2659; GCN3-NEXT: v_mov_b32_e32 v0, s22660; GCN3-NEXT: v_mov_b32_e32 v1, s32661; GCN3-NEXT: s_waitcnt vmcnt(0)2662; GCN3-NEXT: flat_store_dword v[0:1], v22663; GCN3-NEXT: s_endpgm2664;2665; GFX11-LABEL: atomic_umax_i32_ret_offset:2666; GFX11: ; %bb.0: ; %entry2667; GFX11-NEXT: s_clause 0x12668; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x242669; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x342670; GFX11-NEXT: s_waitcnt lgkmcnt(0)2671; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12672; GFX11-NEXT: v_mov_b32_e32 v2, s42673; GFX11-NEXT: flat_atomic_max_u32 v2, v[0:1], v2 offset:16 glc2674; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2675; GFX11-NEXT: buffer_gl0_inv2676; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s32677; GFX11-NEXT: flat_store_b32 v[0:1], v22678; GFX11-NEXT: s_endpgm2679entry:2680 %gep = getelementptr inbounds i32, ptr %out, i32 42681 %val = atomicrmw volatile umax ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !02682 store i32 %val, ptr %out22683 ret void2684}2685 2686define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr %out, i32 %in, i64 %index) {2687; GCN1-LABEL: atomic_umax_i32_addr64_offset:2688; GCN1: ; %bb.0: ; %entry2689; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd2690; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x92691; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb2692; GCN1-NEXT: s_waitcnt lgkmcnt(0)2693; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 22694; GCN1-NEXT: s_add_u32 s0, s2, s02695; GCN1-NEXT: s_addc_u32 s1, s3, s12696; GCN1-NEXT: s_add_u32 s0, s0, 162697; GCN1-NEXT: s_addc_u32 s1, s1, 02698; GCN1-NEXT: v_mov_b32_e32 v0, s02699; GCN1-NEXT: v_mov_b32_e32 v1, s12700; GCN1-NEXT: v_mov_b32_e32 v2, s42701; GCN1-NEXT: flat_atomic_umax v[0:1], v22702; GCN1-NEXT: s_waitcnt lgkmcnt(0)2703; GCN1-NEXT: s_endpgm2704;2705; GCN2-LABEL: atomic_umax_i32_addr64_offset:2706; GCN2: ; %bb.0: ; %entry2707; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x342708; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x242709; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c2710; GCN2-NEXT: s_waitcnt lgkmcnt(0)2711; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 22712; GCN2-NEXT: s_add_u32 s0, s2, s02713; GCN2-NEXT: s_addc_u32 s1, s3, s12714; GCN2-NEXT: s_add_u32 s0, s0, 162715; GCN2-NEXT: s_addc_u32 s1, s1, 02716; GCN2-NEXT: v_mov_b32_e32 v0, s02717; GCN2-NEXT: v_mov_b32_e32 v1, s12718; GCN2-NEXT: v_mov_b32_e32 v2, s42719; GCN2-NEXT: flat_atomic_umax v[0:1], v22720; GCN2-NEXT: s_waitcnt lgkmcnt(0)2721; GCN2-NEXT: s_endpgm2722;2723; GCN3-LABEL: atomic_umax_i32_addr64_offset:2724; GCN3: ; %bb.0: ; %entry2725; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x342726; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x242727; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c2728; GCN3-NEXT: s_waitcnt lgkmcnt(0)2729; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 22730; GCN3-NEXT: s_add_u32 s0, s2, s02731; GCN3-NEXT: s_addc_u32 s1, s3, s12732; GCN3-NEXT: v_mov_b32_e32 v0, s02733; GCN3-NEXT: v_mov_b32_e32 v1, s12734; GCN3-NEXT: v_mov_b32_e32 v2, s62735; GCN3-NEXT: flat_atomic_umax v[0:1], v2 offset:162736; GCN3-NEXT: s_waitcnt lgkmcnt(0)2737; GCN3-NEXT: s_endpgm2738;2739; GFX11-LABEL: atomic_umax_i32_addr64_offset:2740; GFX11: ; %bb.0: ; %entry2741; GFX11-NEXT: s_clause 0x22742; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x342743; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x242744; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c2745; GFX11-NEXT: s_waitcnt lgkmcnt(0)2746; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 22747; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)2748; GFX11-NEXT: s_add_u32 s0, s2, s02749; GFX11-NEXT: s_addc_u32 s1, s3, s12750; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12751; GFX11-NEXT: v_mov_b32_e32 v2, s42752; GFX11-NEXT: flat_atomic_max_u32 v[0:1], v2 offset:162753; GFX11-NEXT: s_waitcnt lgkmcnt(0)2754; GFX11-NEXT: s_waitcnt_vscnt null, 0x02755; GFX11-NEXT: buffer_gl0_inv2756; GFX11-NEXT: s_endpgm2757entry:2758 %ptr = getelementptr inbounds i32, ptr %out, i64 %index2759 %gep = getelementptr inbounds i32, ptr %ptr, i32 42760 %val = atomicrmw volatile umax ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !02761 ret void2762}2763 2764define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr %out, ptr %out2, i32 %in, i64 %index) {2765; GCN1-LABEL: atomic_umax_i32_ret_addr64_offset:2766; GCN1: ; %bb.0: ; %entry2767; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf2768; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92769; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd2770; GCN1-NEXT: s_waitcnt lgkmcnt(0)2771; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 22772; GCN1-NEXT: s_add_u32 s0, s0, s42773; GCN1-NEXT: s_addc_u32 s1, s1, s52774; GCN1-NEXT: s_add_u32 s0, s0, 162775; GCN1-NEXT: s_addc_u32 s1, s1, 02776; GCN1-NEXT: v_mov_b32_e32 v0, s02777; GCN1-NEXT: v_mov_b32_e32 v1, s12778; GCN1-NEXT: v_mov_b32_e32 v2, s82779; GCN1-NEXT: flat_atomic_umax v2, v[0:1], v2 glc2780; GCN1-NEXT: s_waitcnt lgkmcnt(0)2781; GCN1-NEXT: v_mov_b32_e32 v0, s22782; GCN1-NEXT: v_mov_b32_e32 v1, s32783; GCN1-NEXT: s_waitcnt vmcnt(0)2784; GCN1-NEXT: flat_store_dword v[0:1], v22785; GCN1-NEXT: s_endpgm2786;2787; GCN2-LABEL: atomic_umax_i32_ret_addr64_offset:2788; GCN2: ; %bb.0: ; %entry2789; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c2790; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242791; GCN2-NEXT: s_load_dword s8, s[4:5], 0x342792; GCN2-NEXT: s_waitcnt lgkmcnt(0)2793; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 22794; GCN2-NEXT: s_add_u32 s0, s0, s42795; GCN2-NEXT: s_addc_u32 s1, s1, s52796; GCN2-NEXT: s_add_u32 s0, s0, 162797; GCN2-NEXT: s_addc_u32 s1, s1, 02798; GCN2-NEXT: v_mov_b32_e32 v0, s02799; GCN2-NEXT: v_mov_b32_e32 v1, s12800; GCN2-NEXT: v_mov_b32_e32 v2, s82801; GCN2-NEXT: flat_atomic_umax v2, v[0:1], v2 glc2802; GCN2-NEXT: s_waitcnt lgkmcnt(0)2803; GCN2-NEXT: v_mov_b32_e32 v0, s22804; GCN2-NEXT: v_mov_b32_e32 v1, s32805; GCN2-NEXT: s_waitcnt vmcnt(0)2806; GCN2-NEXT: flat_store_dword v[0:1], v22807; GCN2-NEXT: s_endpgm2808;2809; GCN3-LABEL: atomic_umax_i32_ret_addr64_offset:2810; GCN3: ; %bb.0: ; %entry2811; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c2812; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242813; GCN3-NEXT: s_load_dword s8, s[4:5], 0x342814; GCN3-NEXT: s_waitcnt lgkmcnt(0)2815; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 22816; GCN3-NEXT: s_add_u32 s0, s0, s42817; GCN3-NEXT: s_addc_u32 s1, s1, s52818; GCN3-NEXT: v_mov_b32_e32 v0, s02819; GCN3-NEXT: v_mov_b32_e32 v1, s12820; GCN3-NEXT: v_mov_b32_e32 v2, s82821; GCN3-NEXT: flat_atomic_umax v2, v[0:1], v2 offset:16 glc2822; GCN3-NEXT: s_waitcnt lgkmcnt(0)2823; GCN3-NEXT: v_mov_b32_e32 v0, s22824; GCN3-NEXT: v_mov_b32_e32 v1, s32825; GCN3-NEXT: s_waitcnt vmcnt(0)2826; GCN3-NEXT: flat_store_dword v[0:1], v22827; GCN3-NEXT: s_endpgm2828;2829; GFX11-LABEL: atomic_umax_i32_ret_addr64_offset:2830; GFX11: ; %bb.0: ; %entry2831; GFX11-NEXT: s_clause 0x22832; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c2833; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x242834; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x342835; GFX11-NEXT: s_waitcnt lgkmcnt(0)2836; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 22837; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)2838; GFX11-NEXT: s_add_u32 s0, s0, s42839; GFX11-NEXT: s_addc_u32 s1, s1, s52840; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12841; GFX11-NEXT: v_mov_b32_e32 v2, s82842; GFX11-NEXT: flat_atomic_max_u32 v2, v[0:1], v2 offset:16 glc2843; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2844; GFX11-NEXT: buffer_gl0_inv2845; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s32846; GFX11-NEXT: flat_store_b32 v[0:1], v22847; GFX11-NEXT: s_endpgm2848entry:2849 %ptr = getelementptr inbounds i32, ptr %out, i64 %index2850 %gep = getelementptr inbounds i32, ptr %ptr, i32 42851 %val = atomicrmw volatile umax ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !02852 store i32 %val, ptr %out22853 ret void2854}2855 2856define amdgpu_kernel void @atomic_umax_i32(ptr %out, i32 %in) {2857; GCN1-LABEL: atomic_umax_i32:2858; GCN1: ; %bb.0: ; %entry2859; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x92860; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb2861; GCN1-NEXT: s_waitcnt lgkmcnt(0)2862; GCN1-NEXT: v_mov_b32_e32 v0, s02863; GCN1-NEXT: v_mov_b32_e32 v1, s12864; GCN1-NEXT: v_mov_b32_e32 v2, s22865; GCN1-NEXT: flat_atomic_umax v[0:1], v22866; GCN1-NEXT: s_waitcnt lgkmcnt(0)2867; GCN1-NEXT: s_endpgm2868;2869; GCN2-LABEL: atomic_umax_i32:2870; GCN2: ; %bb.0: ; %entry2871; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x242872; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c2873; GCN2-NEXT: s_waitcnt lgkmcnt(0)2874; GCN2-NEXT: v_mov_b32_e32 v0, s02875; GCN2-NEXT: v_mov_b32_e32 v1, s12876; GCN2-NEXT: v_mov_b32_e32 v2, s22877; GCN2-NEXT: flat_atomic_umax v[0:1], v22878; GCN2-NEXT: s_waitcnt lgkmcnt(0)2879; GCN2-NEXT: s_endpgm2880;2881; GCN3-LABEL: atomic_umax_i32:2882; GCN3: ; %bb.0: ; %entry2883; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x242884; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c2885; GCN3-NEXT: s_waitcnt lgkmcnt(0)2886; GCN3-NEXT: v_mov_b32_e32 v0, s02887; GCN3-NEXT: v_mov_b32_e32 v1, s12888; GCN3-NEXT: v_mov_b32_e32 v2, s22889; GCN3-NEXT: flat_atomic_umax v[0:1], v22890; GCN3-NEXT: s_waitcnt lgkmcnt(0)2891; GCN3-NEXT: s_endpgm2892;2893; GFX11-LABEL: atomic_umax_i32:2894; GFX11: ; %bb.0: ; %entry2895; GFX11-NEXT: s_clause 0x12896; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x242897; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c2898; GFX11-NEXT: s_waitcnt lgkmcnt(0)2899; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12900; GFX11-NEXT: v_mov_b32_e32 v2, s22901; GFX11-NEXT: flat_atomic_max_u32 v[0:1], v22902; GFX11-NEXT: s_waitcnt lgkmcnt(0)2903; GFX11-NEXT: s_waitcnt_vscnt null, 0x02904; GFX11-NEXT: buffer_gl0_inv2905; GFX11-NEXT: s_endpgm2906entry:2907 %val = atomicrmw volatile umax ptr %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !02908 ret void2909}2910 2911define amdgpu_kernel void @atomic_umax_i32_ret(ptr %out, ptr %out2, i32 %in) {2912; GCN1-LABEL: atomic_umax_i32_ret:2913; GCN1: ; %bb.0: ; %entry2914; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92915; GCN1-NEXT: s_load_dword s4, s[4:5], 0xd2916; GCN1-NEXT: s_waitcnt lgkmcnt(0)2917; GCN1-NEXT: v_mov_b32_e32 v0, s02918; GCN1-NEXT: v_mov_b32_e32 v1, s12919; GCN1-NEXT: v_mov_b32_e32 v2, s42920; GCN1-NEXT: flat_atomic_umax v2, v[0:1], v2 glc2921; GCN1-NEXT: s_waitcnt lgkmcnt(0)2922; GCN1-NEXT: v_mov_b32_e32 v0, s22923; GCN1-NEXT: v_mov_b32_e32 v1, s32924; GCN1-NEXT: s_waitcnt vmcnt(0)2925; GCN1-NEXT: flat_store_dword v[0:1], v22926; GCN1-NEXT: s_endpgm2927;2928; GCN2-LABEL: atomic_umax_i32_ret:2929; GCN2: ; %bb.0: ; %entry2930; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242931; GCN2-NEXT: s_load_dword s4, s[4:5], 0x342932; GCN2-NEXT: s_waitcnt lgkmcnt(0)2933; GCN2-NEXT: v_mov_b32_e32 v0, s02934; GCN2-NEXT: v_mov_b32_e32 v1, s12935; GCN2-NEXT: v_mov_b32_e32 v2, s42936; GCN2-NEXT: flat_atomic_umax v2, v[0:1], v2 glc2937; GCN2-NEXT: s_waitcnt lgkmcnt(0)2938; GCN2-NEXT: v_mov_b32_e32 v0, s22939; GCN2-NEXT: v_mov_b32_e32 v1, s32940; GCN2-NEXT: s_waitcnt vmcnt(0)2941; GCN2-NEXT: flat_store_dword v[0:1], v22942; GCN2-NEXT: s_endpgm2943;2944; GCN3-LABEL: atomic_umax_i32_ret:2945; GCN3: ; %bb.0: ; %entry2946; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242947; GCN3-NEXT: s_load_dword s6, s[4:5], 0x342948; GCN3-NEXT: s_waitcnt lgkmcnt(0)2949; GCN3-NEXT: v_mov_b32_e32 v0, s02950; GCN3-NEXT: v_mov_b32_e32 v1, s12951; GCN3-NEXT: v_mov_b32_e32 v2, s62952; GCN3-NEXT: flat_atomic_umax v2, v[0:1], v2 glc2953; GCN3-NEXT: s_waitcnt lgkmcnt(0)2954; GCN3-NEXT: v_mov_b32_e32 v0, s22955; GCN3-NEXT: v_mov_b32_e32 v1, s32956; GCN3-NEXT: s_waitcnt vmcnt(0)2957; GCN3-NEXT: flat_store_dword v[0:1], v22958; GCN3-NEXT: s_endpgm2959;2960; GFX11-LABEL: atomic_umax_i32_ret:2961; GFX11: ; %bb.0: ; %entry2962; GFX11-NEXT: s_clause 0x12963; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x242964; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x342965; GFX11-NEXT: s_waitcnt lgkmcnt(0)2966; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s12967; GFX11-NEXT: v_mov_b32_e32 v2, s42968; GFX11-NEXT: flat_atomic_max_u32 v2, v[0:1], v2 glc2969; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2970; GFX11-NEXT: buffer_gl0_inv2971; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s32972; GFX11-NEXT: flat_store_b32 v[0:1], v22973; GFX11-NEXT: s_endpgm2974entry:2975 %val = atomicrmw volatile umax ptr %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !02976 store i32 %val, ptr %out22977 ret void2978}2979 2980define amdgpu_kernel void @atomic_umax_i32_addr64(ptr %out, i32 %in, i64 %index) {2981; GCN1-LABEL: atomic_umax_i32_addr64:2982; GCN1: ; %bb.0: ; %entry2983; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd2984; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x92985; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb2986; GCN1-NEXT: s_waitcnt lgkmcnt(0)2987; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 22988; GCN1-NEXT: s_add_u32 s0, s2, s02989; GCN1-NEXT: s_addc_u32 s1, s3, s12990; GCN1-NEXT: v_mov_b32_e32 v0, s02991; GCN1-NEXT: v_mov_b32_e32 v1, s12992; GCN1-NEXT: v_mov_b32_e32 v2, s42993; GCN1-NEXT: flat_atomic_umax v[0:1], v22994; GCN1-NEXT: s_waitcnt lgkmcnt(0)2995; GCN1-NEXT: s_endpgm2996;2997; GCN2-LABEL: atomic_umax_i32_addr64:2998; GCN2: ; %bb.0: ; %entry2999; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x343000; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x243001; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c3002; GCN2-NEXT: s_waitcnt lgkmcnt(0)3003; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 23004; GCN2-NEXT: s_add_u32 s0, s2, s03005; GCN2-NEXT: s_addc_u32 s1, s3, s13006; GCN2-NEXT: v_mov_b32_e32 v0, s03007; GCN2-NEXT: v_mov_b32_e32 v1, s13008; GCN2-NEXT: v_mov_b32_e32 v2, s43009; GCN2-NEXT: flat_atomic_umax v[0:1], v23010; GCN2-NEXT: s_waitcnt lgkmcnt(0)3011; GCN2-NEXT: s_endpgm3012;3013; GCN3-LABEL: atomic_umax_i32_addr64:3014; GCN3: ; %bb.0: ; %entry3015; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x343016; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x243017; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c3018; GCN3-NEXT: s_waitcnt lgkmcnt(0)3019; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 23020; GCN3-NEXT: s_add_u32 s0, s2, s03021; GCN3-NEXT: s_addc_u32 s1, s3, s13022; GCN3-NEXT: v_mov_b32_e32 v0, s03023; GCN3-NEXT: v_mov_b32_e32 v1, s13024; GCN3-NEXT: v_mov_b32_e32 v2, s63025; GCN3-NEXT: flat_atomic_umax v[0:1], v23026; GCN3-NEXT: s_waitcnt lgkmcnt(0)3027; GCN3-NEXT: s_endpgm3028;3029; GFX11-LABEL: atomic_umax_i32_addr64:3030; GFX11: ; %bb.0: ; %entry3031; GFX11-NEXT: s_clause 0x23032; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x343033; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x243034; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c3035; GFX11-NEXT: s_waitcnt lgkmcnt(0)3036; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 23037; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)3038; GFX11-NEXT: s_add_u32 s0, s2, s03039; GFX11-NEXT: s_addc_u32 s1, s3, s13040; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13041; GFX11-NEXT: v_mov_b32_e32 v2, s43042; GFX11-NEXT: flat_atomic_max_u32 v[0:1], v23043; GFX11-NEXT: s_waitcnt lgkmcnt(0)3044; GFX11-NEXT: s_waitcnt_vscnt null, 0x03045; GFX11-NEXT: buffer_gl0_inv3046; GFX11-NEXT: s_endpgm3047entry:3048 %ptr = getelementptr inbounds i32, ptr %out, i64 %index3049 %val = atomicrmw volatile umax ptr %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !03050 ret void3051}3052 3053define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr %out, ptr %out2, i32 %in, i64 %index) {3054; GCN1-LABEL: atomic_umax_i32_ret_addr64:3055; GCN1: ; %bb.0: ; %entry3056; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf3057; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93058; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd3059; GCN1-NEXT: s_waitcnt lgkmcnt(0)3060; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 23061; GCN1-NEXT: s_add_u32 s0, s0, s43062; GCN1-NEXT: s_addc_u32 s1, s1, s53063; GCN1-NEXT: v_mov_b32_e32 v0, s03064; GCN1-NEXT: v_mov_b32_e32 v1, s13065; GCN1-NEXT: v_mov_b32_e32 v2, s83066; GCN1-NEXT: flat_atomic_umax v2, v[0:1], v2 glc3067; GCN1-NEXT: s_waitcnt lgkmcnt(0)3068; GCN1-NEXT: v_mov_b32_e32 v0, s23069; GCN1-NEXT: v_mov_b32_e32 v1, s33070; GCN1-NEXT: s_waitcnt vmcnt(0)3071; GCN1-NEXT: flat_store_dword v[0:1], v23072; GCN1-NEXT: s_endpgm3073;3074; GCN2-LABEL: atomic_umax_i32_ret_addr64:3075; GCN2: ; %bb.0: ; %entry3076; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c3077; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243078; GCN2-NEXT: s_load_dword s8, s[4:5], 0x343079; GCN2-NEXT: s_waitcnt lgkmcnt(0)3080; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 23081; GCN2-NEXT: s_add_u32 s0, s0, s43082; GCN2-NEXT: s_addc_u32 s1, s1, s53083; GCN2-NEXT: v_mov_b32_e32 v0, s03084; GCN2-NEXT: v_mov_b32_e32 v1, s13085; GCN2-NEXT: v_mov_b32_e32 v2, s83086; GCN2-NEXT: flat_atomic_umax v2, v[0:1], v2 glc3087; GCN2-NEXT: s_waitcnt lgkmcnt(0)3088; GCN2-NEXT: v_mov_b32_e32 v0, s23089; GCN2-NEXT: v_mov_b32_e32 v1, s33090; GCN2-NEXT: s_waitcnt vmcnt(0)3091; GCN2-NEXT: flat_store_dword v[0:1], v23092; GCN2-NEXT: s_endpgm3093;3094; GCN3-LABEL: atomic_umax_i32_ret_addr64:3095; GCN3: ; %bb.0: ; %entry3096; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c3097; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243098; GCN3-NEXT: s_load_dword s8, s[4:5], 0x343099; GCN3-NEXT: s_waitcnt lgkmcnt(0)3100; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 23101; GCN3-NEXT: s_add_u32 s0, s0, s43102; GCN3-NEXT: s_addc_u32 s1, s1, s53103; GCN3-NEXT: v_mov_b32_e32 v0, s03104; GCN3-NEXT: v_mov_b32_e32 v1, s13105; GCN3-NEXT: v_mov_b32_e32 v2, s83106; GCN3-NEXT: flat_atomic_umax v2, v[0:1], v2 glc3107; GCN3-NEXT: s_waitcnt lgkmcnt(0)3108; GCN3-NEXT: v_mov_b32_e32 v0, s23109; GCN3-NEXT: v_mov_b32_e32 v1, s33110; GCN3-NEXT: s_waitcnt vmcnt(0)3111; GCN3-NEXT: flat_store_dword v[0:1], v23112; GCN3-NEXT: s_endpgm3113;3114; GFX11-LABEL: atomic_umax_i32_ret_addr64:3115; GFX11: ; %bb.0: ; %entry3116; GFX11-NEXT: s_clause 0x23117; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c3118; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x243119; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x343120; GFX11-NEXT: s_waitcnt lgkmcnt(0)3121; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 23122; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)3123; GFX11-NEXT: s_add_u32 s0, s0, s43124; GFX11-NEXT: s_addc_u32 s1, s1, s53125; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13126; GFX11-NEXT: v_mov_b32_e32 v2, s83127; GFX11-NEXT: flat_atomic_max_u32 v2, v[0:1], v2 glc3128; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3129; GFX11-NEXT: buffer_gl0_inv3130; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s33131; GFX11-NEXT: flat_store_b32 v[0:1], v23132; GFX11-NEXT: s_endpgm3133entry:3134 %ptr = getelementptr inbounds i32, ptr %out, i64 %index3135 %val = atomicrmw volatile umax ptr %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !03136 store i32 %val, ptr %out23137 ret void3138}3139 3140define amdgpu_kernel void @atomic_min_i32_offset(ptr %out, i32 %in) {3141; GCN1-LABEL: atomic_min_i32_offset:3142; GCN1: ; %bb.0: ; %entry3143; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x93144; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb3145; GCN1-NEXT: s_waitcnt lgkmcnt(0)3146; GCN1-NEXT: s_add_u32 s0, s0, 163147; GCN1-NEXT: s_addc_u32 s1, s1, 03148; GCN1-NEXT: v_mov_b32_e32 v0, s03149; GCN1-NEXT: v_mov_b32_e32 v1, s13150; GCN1-NEXT: v_mov_b32_e32 v2, s23151; GCN1-NEXT: flat_atomic_smin v[0:1], v23152; GCN1-NEXT: s_waitcnt lgkmcnt(0)3153; GCN1-NEXT: s_endpgm3154;3155; GCN2-LABEL: atomic_min_i32_offset:3156; GCN2: ; %bb.0: ; %entry3157; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x243158; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c3159; GCN2-NEXT: s_waitcnt lgkmcnt(0)3160; GCN2-NEXT: s_add_u32 s0, s0, 163161; GCN2-NEXT: s_addc_u32 s1, s1, 03162; GCN2-NEXT: v_mov_b32_e32 v0, s03163; GCN2-NEXT: v_mov_b32_e32 v1, s13164; GCN2-NEXT: v_mov_b32_e32 v2, s23165; GCN2-NEXT: flat_atomic_smin v[0:1], v23166; GCN2-NEXT: s_waitcnt lgkmcnt(0)3167; GCN2-NEXT: s_endpgm3168;3169; GCN3-LABEL: atomic_min_i32_offset:3170; GCN3: ; %bb.0: ; %entry3171; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x243172; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c3173; GCN3-NEXT: s_waitcnt lgkmcnt(0)3174; GCN3-NEXT: v_mov_b32_e32 v0, s03175; GCN3-NEXT: v_mov_b32_e32 v1, s13176; GCN3-NEXT: v_mov_b32_e32 v2, s23177; GCN3-NEXT: flat_atomic_smin v[0:1], v2 offset:163178; GCN3-NEXT: s_waitcnt lgkmcnt(0)3179; GCN3-NEXT: s_endpgm3180;3181; GFX11-LABEL: atomic_min_i32_offset:3182; GFX11: ; %bb.0: ; %entry3183; GFX11-NEXT: s_clause 0x13184; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x243185; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c3186; GFX11-NEXT: s_waitcnt lgkmcnt(0)3187; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13188; GFX11-NEXT: v_mov_b32_e32 v2, s23189; GFX11-NEXT: flat_atomic_min_i32 v[0:1], v2 offset:163190; GFX11-NEXT: s_waitcnt lgkmcnt(0)3191; GFX11-NEXT: s_waitcnt_vscnt null, 0x03192; GFX11-NEXT: buffer_gl0_inv3193; GFX11-NEXT: s_endpgm3194entry:3195 %gep = getelementptr inbounds i32, ptr %out, i32 43196 %val = atomicrmw volatile min ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !03197 ret void3198}3199 3200define amdgpu_kernel void @atomic_min_i32_ret_offset(ptr %out, ptr %out2, i32 %in) {3201; GCN1-LABEL: atomic_min_i32_ret_offset:3202; GCN1: ; %bb.0: ; %entry3203; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93204; GCN1-NEXT: s_load_dword s4, s[4:5], 0xd3205; GCN1-NEXT: s_waitcnt lgkmcnt(0)3206; GCN1-NEXT: s_add_u32 s0, s0, 163207; GCN1-NEXT: s_addc_u32 s1, s1, 03208; GCN1-NEXT: v_mov_b32_e32 v0, s03209; GCN1-NEXT: v_mov_b32_e32 v1, s13210; GCN1-NEXT: v_mov_b32_e32 v2, s43211; GCN1-NEXT: flat_atomic_smin v2, v[0:1], v2 glc3212; GCN1-NEXT: s_waitcnt lgkmcnt(0)3213; GCN1-NEXT: v_mov_b32_e32 v0, s23214; GCN1-NEXT: v_mov_b32_e32 v1, s33215; GCN1-NEXT: s_waitcnt vmcnt(0)3216; GCN1-NEXT: flat_store_dword v[0:1], v23217; GCN1-NEXT: s_endpgm3218;3219; GCN2-LABEL: atomic_min_i32_ret_offset:3220; GCN2: ; %bb.0: ; %entry3221; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243222; GCN2-NEXT: s_load_dword s4, s[4:5], 0x343223; GCN2-NEXT: s_waitcnt lgkmcnt(0)3224; GCN2-NEXT: s_add_u32 s0, s0, 163225; GCN2-NEXT: s_addc_u32 s1, s1, 03226; GCN2-NEXT: v_mov_b32_e32 v0, s03227; GCN2-NEXT: v_mov_b32_e32 v1, s13228; GCN2-NEXT: v_mov_b32_e32 v2, s43229; GCN2-NEXT: flat_atomic_smin v2, v[0:1], v2 glc3230; GCN2-NEXT: s_waitcnt lgkmcnt(0)3231; GCN2-NEXT: v_mov_b32_e32 v0, s23232; GCN2-NEXT: v_mov_b32_e32 v1, s33233; GCN2-NEXT: s_waitcnt vmcnt(0)3234; GCN2-NEXT: flat_store_dword v[0:1], v23235; GCN2-NEXT: s_endpgm3236;3237; GCN3-LABEL: atomic_min_i32_ret_offset:3238; GCN3: ; %bb.0: ; %entry3239; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243240; GCN3-NEXT: s_load_dword s6, s[4:5], 0x343241; GCN3-NEXT: s_waitcnt lgkmcnt(0)3242; GCN3-NEXT: v_mov_b32_e32 v0, s03243; GCN3-NEXT: v_mov_b32_e32 v1, s13244; GCN3-NEXT: v_mov_b32_e32 v2, s63245; GCN3-NEXT: flat_atomic_smin v2, v[0:1], v2 offset:16 glc3246; GCN3-NEXT: s_waitcnt lgkmcnt(0)3247; GCN3-NEXT: v_mov_b32_e32 v0, s23248; GCN3-NEXT: v_mov_b32_e32 v1, s33249; GCN3-NEXT: s_waitcnt vmcnt(0)3250; GCN3-NEXT: flat_store_dword v[0:1], v23251; GCN3-NEXT: s_endpgm3252;3253; GFX11-LABEL: atomic_min_i32_ret_offset:3254; GFX11: ; %bb.0: ; %entry3255; GFX11-NEXT: s_clause 0x13256; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x243257; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x343258; GFX11-NEXT: s_waitcnt lgkmcnt(0)3259; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13260; GFX11-NEXT: v_mov_b32_e32 v2, s43261; GFX11-NEXT: flat_atomic_min_i32 v2, v[0:1], v2 offset:16 glc3262; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3263; GFX11-NEXT: buffer_gl0_inv3264; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s33265; GFX11-NEXT: flat_store_b32 v[0:1], v23266; GFX11-NEXT: s_endpgm3267entry:3268 %gep = getelementptr inbounds i32, ptr %out, i32 43269 %val = atomicrmw volatile min ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !03270 store i32 %val, ptr %out23271 ret void3272}3273 3274define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr %out, i32 %in, i64 %index) {3275; GCN1-LABEL: atomic_min_i32_addr64_offset:3276; GCN1: ; %bb.0: ; %entry3277; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd3278; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x93279; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb3280; GCN1-NEXT: s_waitcnt lgkmcnt(0)3281; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 23282; GCN1-NEXT: s_add_u32 s0, s2, s03283; GCN1-NEXT: s_addc_u32 s1, s3, s13284; GCN1-NEXT: s_add_u32 s0, s0, 163285; GCN1-NEXT: s_addc_u32 s1, s1, 03286; GCN1-NEXT: v_mov_b32_e32 v0, s03287; GCN1-NEXT: v_mov_b32_e32 v1, s13288; GCN1-NEXT: v_mov_b32_e32 v2, s43289; GCN1-NEXT: flat_atomic_smin v[0:1], v23290; GCN1-NEXT: s_waitcnt lgkmcnt(0)3291; GCN1-NEXT: s_endpgm3292;3293; GCN2-LABEL: atomic_min_i32_addr64_offset:3294; GCN2: ; %bb.0: ; %entry3295; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x343296; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x243297; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c3298; GCN2-NEXT: s_waitcnt lgkmcnt(0)3299; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 23300; GCN2-NEXT: s_add_u32 s0, s2, s03301; GCN2-NEXT: s_addc_u32 s1, s3, s13302; GCN2-NEXT: s_add_u32 s0, s0, 163303; GCN2-NEXT: s_addc_u32 s1, s1, 03304; GCN2-NEXT: v_mov_b32_e32 v0, s03305; GCN2-NEXT: v_mov_b32_e32 v1, s13306; GCN2-NEXT: v_mov_b32_e32 v2, s43307; GCN2-NEXT: flat_atomic_smin v[0:1], v23308; GCN2-NEXT: s_waitcnt lgkmcnt(0)3309; GCN2-NEXT: s_endpgm3310;3311; GCN3-LABEL: atomic_min_i32_addr64_offset:3312; GCN3: ; %bb.0: ; %entry3313; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x343314; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x243315; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c3316; GCN3-NEXT: s_waitcnt lgkmcnt(0)3317; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 23318; GCN3-NEXT: s_add_u32 s0, s2, s03319; GCN3-NEXT: s_addc_u32 s1, s3, s13320; GCN3-NEXT: v_mov_b32_e32 v0, s03321; GCN3-NEXT: v_mov_b32_e32 v1, s13322; GCN3-NEXT: v_mov_b32_e32 v2, s63323; GCN3-NEXT: flat_atomic_smin v[0:1], v2 offset:163324; GCN3-NEXT: s_waitcnt lgkmcnt(0)3325; GCN3-NEXT: s_endpgm3326;3327; GFX11-LABEL: atomic_min_i32_addr64_offset:3328; GFX11: ; %bb.0: ; %entry3329; GFX11-NEXT: s_clause 0x23330; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x343331; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x243332; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c3333; GFX11-NEXT: s_waitcnt lgkmcnt(0)3334; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 23335; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)3336; GFX11-NEXT: s_add_u32 s0, s2, s03337; GFX11-NEXT: s_addc_u32 s1, s3, s13338; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13339; GFX11-NEXT: v_mov_b32_e32 v2, s43340; GFX11-NEXT: flat_atomic_min_i32 v[0:1], v2 offset:163341; GFX11-NEXT: s_waitcnt lgkmcnt(0)3342; GFX11-NEXT: s_waitcnt_vscnt null, 0x03343; GFX11-NEXT: buffer_gl0_inv3344; GFX11-NEXT: s_endpgm3345entry:3346 %ptr = getelementptr inbounds i32, ptr %out, i64 %index3347 %gep = getelementptr inbounds i32, ptr %ptr, i32 43348 %val = atomicrmw volatile min ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !03349 ret void3350}3351 3352define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr %out, ptr %out2, i32 %in, i64 %index) {3353; GCN1-LABEL: atomic_min_i32_ret_addr64_offset:3354; GCN1: ; %bb.0: ; %entry3355; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf3356; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93357; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd3358; GCN1-NEXT: s_waitcnt lgkmcnt(0)3359; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 23360; GCN1-NEXT: s_add_u32 s0, s0, s43361; GCN1-NEXT: s_addc_u32 s1, s1, s53362; GCN1-NEXT: s_add_u32 s0, s0, 163363; GCN1-NEXT: s_addc_u32 s1, s1, 03364; GCN1-NEXT: v_mov_b32_e32 v0, s03365; GCN1-NEXT: v_mov_b32_e32 v1, s13366; GCN1-NEXT: v_mov_b32_e32 v2, s83367; GCN1-NEXT: flat_atomic_smin v2, v[0:1], v2 glc3368; GCN1-NEXT: s_waitcnt lgkmcnt(0)3369; GCN1-NEXT: v_mov_b32_e32 v0, s23370; GCN1-NEXT: v_mov_b32_e32 v1, s33371; GCN1-NEXT: s_waitcnt vmcnt(0)3372; GCN1-NEXT: flat_store_dword v[0:1], v23373; GCN1-NEXT: s_endpgm3374;3375; GCN2-LABEL: atomic_min_i32_ret_addr64_offset:3376; GCN2: ; %bb.0: ; %entry3377; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c3378; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243379; GCN2-NEXT: s_load_dword s8, s[4:5], 0x343380; GCN2-NEXT: s_waitcnt lgkmcnt(0)3381; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 23382; GCN2-NEXT: s_add_u32 s0, s0, s43383; GCN2-NEXT: s_addc_u32 s1, s1, s53384; GCN2-NEXT: s_add_u32 s0, s0, 163385; GCN2-NEXT: s_addc_u32 s1, s1, 03386; GCN2-NEXT: v_mov_b32_e32 v0, s03387; GCN2-NEXT: v_mov_b32_e32 v1, s13388; GCN2-NEXT: v_mov_b32_e32 v2, s83389; GCN2-NEXT: flat_atomic_smin v2, v[0:1], v2 glc3390; GCN2-NEXT: s_waitcnt lgkmcnt(0)3391; GCN2-NEXT: v_mov_b32_e32 v0, s23392; GCN2-NEXT: v_mov_b32_e32 v1, s33393; GCN2-NEXT: s_waitcnt vmcnt(0)3394; GCN2-NEXT: flat_store_dword v[0:1], v23395; GCN2-NEXT: s_endpgm3396;3397; GCN3-LABEL: atomic_min_i32_ret_addr64_offset:3398; GCN3: ; %bb.0: ; %entry3399; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c3400; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243401; GCN3-NEXT: s_load_dword s8, s[4:5], 0x343402; GCN3-NEXT: s_waitcnt lgkmcnt(0)3403; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 23404; GCN3-NEXT: s_add_u32 s0, s0, s43405; GCN3-NEXT: s_addc_u32 s1, s1, s53406; GCN3-NEXT: v_mov_b32_e32 v0, s03407; GCN3-NEXT: v_mov_b32_e32 v1, s13408; GCN3-NEXT: v_mov_b32_e32 v2, s83409; GCN3-NEXT: flat_atomic_smin v2, v[0:1], v2 offset:16 glc3410; GCN3-NEXT: s_waitcnt lgkmcnt(0)3411; GCN3-NEXT: v_mov_b32_e32 v0, s23412; GCN3-NEXT: v_mov_b32_e32 v1, s33413; GCN3-NEXT: s_waitcnt vmcnt(0)3414; GCN3-NEXT: flat_store_dword v[0:1], v23415; GCN3-NEXT: s_endpgm3416;3417; GFX11-LABEL: atomic_min_i32_ret_addr64_offset:3418; GFX11: ; %bb.0: ; %entry3419; GFX11-NEXT: s_clause 0x23420; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c3421; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x243422; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x343423; GFX11-NEXT: s_waitcnt lgkmcnt(0)3424; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 23425; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)3426; GFX11-NEXT: s_add_u32 s0, s0, s43427; GFX11-NEXT: s_addc_u32 s1, s1, s53428; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13429; GFX11-NEXT: v_mov_b32_e32 v2, s83430; GFX11-NEXT: flat_atomic_min_i32 v2, v[0:1], v2 offset:16 glc3431; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3432; GFX11-NEXT: buffer_gl0_inv3433; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s33434; GFX11-NEXT: flat_store_b32 v[0:1], v23435; GFX11-NEXT: s_endpgm3436entry:3437 %ptr = getelementptr inbounds i32, ptr %out, i64 %index3438 %gep = getelementptr inbounds i32, ptr %ptr, i32 43439 %val = atomicrmw volatile min ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !03440 store i32 %val, ptr %out23441 ret void3442}3443 3444define amdgpu_kernel void @atomic_min_i32(ptr %out, i32 %in) {3445; GCN1-LABEL: atomic_min_i32:3446; GCN1: ; %bb.0: ; %entry3447; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x93448; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb3449; GCN1-NEXT: s_waitcnt lgkmcnt(0)3450; GCN1-NEXT: v_mov_b32_e32 v0, s03451; GCN1-NEXT: v_mov_b32_e32 v1, s13452; GCN1-NEXT: v_mov_b32_e32 v2, s23453; GCN1-NEXT: flat_atomic_smin v[0:1], v23454; GCN1-NEXT: s_waitcnt lgkmcnt(0)3455; GCN1-NEXT: s_endpgm3456;3457; GCN2-LABEL: atomic_min_i32:3458; GCN2: ; %bb.0: ; %entry3459; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x243460; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c3461; GCN2-NEXT: s_waitcnt lgkmcnt(0)3462; GCN2-NEXT: v_mov_b32_e32 v0, s03463; GCN2-NEXT: v_mov_b32_e32 v1, s13464; GCN2-NEXT: v_mov_b32_e32 v2, s23465; GCN2-NEXT: flat_atomic_smin v[0:1], v23466; GCN2-NEXT: s_waitcnt lgkmcnt(0)3467; GCN2-NEXT: s_endpgm3468;3469; GCN3-LABEL: atomic_min_i32:3470; GCN3: ; %bb.0: ; %entry3471; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x243472; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c3473; GCN3-NEXT: s_waitcnt lgkmcnt(0)3474; GCN3-NEXT: v_mov_b32_e32 v0, s03475; GCN3-NEXT: v_mov_b32_e32 v1, s13476; GCN3-NEXT: v_mov_b32_e32 v2, s23477; GCN3-NEXT: flat_atomic_smin v[0:1], v23478; GCN3-NEXT: s_waitcnt lgkmcnt(0)3479; GCN3-NEXT: s_endpgm3480;3481; GFX11-LABEL: atomic_min_i32:3482; GFX11: ; %bb.0: ; %entry3483; GFX11-NEXT: s_clause 0x13484; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x243485; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c3486; GFX11-NEXT: s_waitcnt lgkmcnt(0)3487; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13488; GFX11-NEXT: v_mov_b32_e32 v2, s23489; GFX11-NEXT: flat_atomic_min_i32 v[0:1], v23490; GFX11-NEXT: s_waitcnt lgkmcnt(0)3491; GFX11-NEXT: s_waitcnt_vscnt null, 0x03492; GFX11-NEXT: buffer_gl0_inv3493; GFX11-NEXT: s_endpgm3494entry:3495 %val = atomicrmw volatile min ptr %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !03496 ret void3497}3498 3499define amdgpu_kernel void @atomic_min_i32_ret(ptr %out, ptr %out2, i32 %in) {3500; GCN1-LABEL: atomic_min_i32_ret:3501; GCN1: ; %bb.0: ; %entry3502; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93503; GCN1-NEXT: s_load_dword s4, s[4:5], 0xd3504; GCN1-NEXT: s_waitcnt lgkmcnt(0)3505; GCN1-NEXT: v_mov_b32_e32 v0, s03506; GCN1-NEXT: v_mov_b32_e32 v1, s13507; GCN1-NEXT: v_mov_b32_e32 v2, s43508; GCN1-NEXT: flat_atomic_smin v2, v[0:1], v2 glc3509; GCN1-NEXT: s_waitcnt lgkmcnt(0)3510; GCN1-NEXT: v_mov_b32_e32 v0, s23511; GCN1-NEXT: v_mov_b32_e32 v1, s33512; GCN1-NEXT: s_waitcnt vmcnt(0)3513; GCN1-NEXT: flat_store_dword v[0:1], v23514; GCN1-NEXT: s_endpgm3515;3516; GCN2-LABEL: atomic_min_i32_ret:3517; GCN2: ; %bb.0: ; %entry3518; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243519; GCN2-NEXT: s_load_dword s4, s[4:5], 0x343520; GCN2-NEXT: s_waitcnt lgkmcnt(0)3521; GCN2-NEXT: v_mov_b32_e32 v0, s03522; GCN2-NEXT: v_mov_b32_e32 v1, s13523; GCN2-NEXT: v_mov_b32_e32 v2, s43524; GCN2-NEXT: flat_atomic_smin v2, v[0:1], v2 glc3525; GCN2-NEXT: s_waitcnt lgkmcnt(0)3526; GCN2-NEXT: v_mov_b32_e32 v0, s23527; GCN2-NEXT: v_mov_b32_e32 v1, s33528; GCN2-NEXT: s_waitcnt vmcnt(0)3529; GCN2-NEXT: flat_store_dword v[0:1], v23530; GCN2-NEXT: s_endpgm3531;3532; GCN3-LABEL: atomic_min_i32_ret:3533; GCN3: ; %bb.0: ; %entry3534; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243535; GCN3-NEXT: s_load_dword s6, s[4:5], 0x343536; GCN3-NEXT: s_waitcnt lgkmcnt(0)3537; GCN3-NEXT: v_mov_b32_e32 v0, s03538; GCN3-NEXT: v_mov_b32_e32 v1, s13539; GCN3-NEXT: v_mov_b32_e32 v2, s63540; GCN3-NEXT: flat_atomic_smin v2, v[0:1], v2 glc3541; GCN3-NEXT: s_waitcnt lgkmcnt(0)3542; GCN3-NEXT: v_mov_b32_e32 v0, s23543; GCN3-NEXT: v_mov_b32_e32 v1, s33544; GCN3-NEXT: s_waitcnt vmcnt(0)3545; GCN3-NEXT: flat_store_dword v[0:1], v23546; GCN3-NEXT: s_endpgm3547;3548; GFX11-LABEL: atomic_min_i32_ret:3549; GFX11: ; %bb.0: ; %entry3550; GFX11-NEXT: s_clause 0x13551; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x243552; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x343553; GFX11-NEXT: s_waitcnt lgkmcnt(0)3554; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13555; GFX11-NEXT: v_mov_b32_e32 v2, s43556; GFX11-NEXT: flat_atomic_min_i32 v2, v[0:1], v2 glc3557; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3558; GFX11-NEXT: buffer_gl0_inv3559; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s33560; GFX11-NEXT: flat_store_b32 v[0:1], v23561; GFX11-NEXT: s_endpgm3562entry:3563 %val = atomicrmw volatile min ptr %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !03564 store i32 %val, ptr %out23565 ret void3566}3567 3568define amdgpu_kernel void @atomic_min_i32_addr64(ptr %out, i32 %in, i64 %index) {3569; GCN1-LABEL: atomic_min_i32_addr64:3570; GCN1: ; %bb.0: ; %entry3571; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd3572; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x93573; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb3574; GCN1-NEXT: s_waitcnt lgkmcnt(0)3575; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 23576; GCN1-NEXT: s_add_u32 s0, s2, s03577; GCN1-NEXT: s_addc_u32 s1, s3, s13578; GCN1-NEXT: v_mov_b32_e32 v0, s03579; GCN1-NEXT: v_mov_b32_e32 v1, s13580; GCN1-NEXT: v_mov_b32_e32 v2, s43581; GCN1-NEXT: flat_atomic_smin v[0:1], v23582; GCN1-NEXT: s_waitcnt lgkmcnt(0)3583; GCN1-NEXT: s_endpgm3584;3585; GCN2-LABEL: atomic_min_i32_addr64:3586; GCN2: ; %bb.0: ; %entry3587; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x343588; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x243589; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c3590; GCN2-NEXT: s_waitcnt lgkmcnt(0)3591; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 23592; GCN2-NEXT: s_add_u32 s0, s2, s03593; GCN2-NEXT: s_addc_u32 s1, s3, s13594; GCN2-NEXT: v_mov_b32_e32 v0, s03595; GCN2-NEXT: v_mov_b32_e32 v1, s13596; GCN2-NEXT: v_mov_b32_e32 v2, s43597; GCN2-NEXT: flat_atomic_smin v[0:1], v23598; GCN2-NEXT: s_waitcnt lgkmcnt(0)3599; GCN2-NEXT: s_endpgm3600;3601; GCN3-LABEL: atomic_min_i32_addr64:3602; GCN3: ; %bb.0: ; %entry3603; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x343604; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x243605; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c3606; GCN3-NEXT: s_waitcnt lgkmcnt(0)3607; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 23608; GCN3-NEXT: s_add_u32 s0, s2, s03609; GCN3-NEXT: s_addc_u32 s1, s3, s13610; GCN3-NEXT: v_mov_b32_e32 v0, s03611; GCN3-NEXT: v_mov_b32_e32 v1, s13612; GCN3-NEXT: v_mov_b32_e32 v2, s63613; GCN3-NEXT: flat_atomic_smin v[0:1], v23614; GCN3-NEXT: s_waitcnt lgkmcnt(0)3615; GCN3-NEXT: s_endpgm3616;3617; GFX11-LABEL: atomic_min_i32_addr64:3618; GFX11: ; %bb.0: ; %entry3619; GFX11-NEXT: s_clause 0x23620; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x343621; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x243622; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c3623; GFX11-NEXT: s_waitcnt lgkmcnt(0)3624; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 23625; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)3626; GFX11-NEXT: s_add_u32 s0, s2, s03627; GFX11-NEXT: s_addc_u32 s1, s3, s13628; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13629; GFX11-NEXT: v_mov_b32_e32 v2, s43630; GFX11-NEXT: flat_atomic_min_i32 v[0:1], v23631; GFX11-NEXT: s_waitcnt lgkmcnt(0)3632; GFX11-NEXT: s_waitcnt_vscnt null, 0x03633; GFX11-NEXT: buffer_gl0_inv3634; GFX11-NEXT: s_endpgm3635entry:3636 %ptr = getelementptr inbounds i32, ptr %out, i64 %index3637 %val = atomicrmw volatile min ptr %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !03638 ret void3639}3640 3641define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr %out, ptr %out2, i32 %in, i64 %index) {3642; GCN1-LABEL: atomic_min_i32_ret_addr64:3643; GCN1: ; %bb.0: ; %entry3644; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf3645; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93646; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd3647; GCN1-NEXT: s_waitcnt lgkmcnt(0)3648; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 23649; GCN1-NEXT: s_add_u32 s0, s0, s43650; GCN1-NEXT: s_addc_u32 s1, s1, s53651; GCN1-NEXT: v_mov_b32_e32 v0, s03652; GCN1-NEXT: v_mov_b32_e32 v1, s13653; GCN1-NEXT: v_mov_b32_e32 v2, s83654; GCN1-NEXT: flat_atomic_smin v2, v[0:1], v2 glc3655; GCN1-NEXT: s_waitcnt lgkmcnt(0)3656; GCN1-NEXT: v_mov_b32_e32 v0, s23657; GCN1-NEXT: v_mov_b32_e32 v1, s33658; GCN1-NEXT: s_waitcnt vmcnt(0)3659; GCN1-NEXT: flat_store_dword v[0:1], v23660; GCN1-NEXT: s_endpgm3661;3662; GCN2-LABEL: atomic_min_i32_ret_addr64:3663; GCN2: ; %bb.0: ; %entry3664; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c3665; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243666; GCN2-NEXT: s_load_dword s8, s[4:5], 0x343667; GCN2-NEXT: s_waitcnt lgkmcnt(0)3668; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 23669; GCN2-NEXT: s_add_u32 s0, s0, s43670; GCN2-NEXT: s_addc_u32 s1, s1, s53671; GCN2-NEXT: v_mov_b32_e32 v0, s03672; GCN2-NEXT: v_mov_b32_e32 v1, s13673; GCN2-NEXT: v_mov_b32_e32 v2, s83674; GCN2-NEXT: flat_atomic_smin v2, v[0:1], v2 glc3675; GCN2-NEXT: s_waitcnt lgkmcnt(0)3676; GCN2-NEXT: v_mov_b32_e32 v0, s23677; GCN2-NEXT: v_mov_b32_e32 v1, s33678; GCN2-NEXT: s_waitcnt vmcnt(0)3679; GCN2-NEXT: flat_store_dword v[0:1], v23680; GCN2-NEXT: s_endpgm3681;3682; GCN3-LABEL: atomic_min_i32_ret_addr64:3683; GCN3: ; %bb.0: ; %entry3684; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c3685; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243686; GCN3-NEXT: s_load_dword s8, s[4:5], 0x343687; GCN3-NEXT: s_waitcnt lgkmcnt(0)3688; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 23689; GCN3-NEXT: s_add_u32 s0, s0, s43690; GCN3-NEXT: s_addc_u32 s1, s1, s53691; GCN3-NEXT: v_mov_b32_e32 v0, s03692; GCN3-NEXT: v_mov_b32_e32 v1, s13693; GCN3-NEXT: v_mov_b32_e32 v2, s83694; GCN3-NEXT: flat_atomic_smin v2, v[0:1], v2 glc3695; GCN3-NEXT: s_waitcnt lgkmcnt(0)3696; GCN3-NEXT: v_mov_b32_e32 v0, s23697; GCN3-NEXT: v_mov_b32_e32 v1, s33698; GCN3-NEXT: s_waitcnt vmcnt(0)3699; GCN3-NEXT: flat_store_dword v[0:1], v23700; GCN3-NEXT: s_endpgm3701;3702; GFX11-LABEL: atomic_min_i32_ret_addr64:3703; GFX11: ; %bb.0: ; %entry3704; GFX11-NEXT: s_clause 0x23705; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c3706; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x243707; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x343708; GFX11-NEXT: s_waitcnt lgkmcnt(0)3709; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 23710; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)3711; GFX11-NEXT: s_add_u32 s0, s0, s43712; GFX11-NEXT: s_addc_u32 s1, s1, s53713; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13714; GFX11-NEXT: v_mov_b32_e32 v2, s83715; GFX11-NEXT: flat_atomic_min_i32 v2, v[0:1], v2 glc3716; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3717; GFX11-NEXT: buffer_gl0_inv3718; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s33719; GFX11-NEXT: flat_store_b32 v[0:1], v23720; GFX11-NEXT: s_endpgm3721entry:3722 %ptr = getelementptr inbounds i32, ptr %out, i64 %index3723 %val = atomicrmw volatile min ptr %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !03724 store i32 %val, ptr %out23725 ret void3726}3727 3728define amdgpu_kernel void @atomic_umin_i32_offset(ptr %out, i32 %in) {3729; GCN1-LABEL: atomic_umin_i32_offset:3730; GCN1: ; %bb.0: ; %entry3731; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x93732; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb3733; GCN1-NEXT: s_waitcnt lgkmcnt(0)3734; GCN1-NEXT: s_add_u32 s0, s0, 163735; GCN1-NEXT: s_addc_u32 s1, s1, 03736; GCN1-NEXT: v_mov_b32_e32 v0, s03737; GCN1-NEXT: v_mov_b32_e32 v1, s13738; GCN1-NEXT: v_mov_b32_e32 v2, s23739; GCN1-NEXT: flat_atomic_umin v[0:1], v23740; GCN1-NEXT: s_waitcnt lgkmcnt(0)3741; GCN1-NEXT: s_endpgm3742;3743; GCN2-LABEL: atomic_umin_i32_offset:3744; GCN2: ; %bb.0: ; %entry3745; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x243746; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c3747; GCN2-NEXT: s_waitcnt lgkmcnt(0)3748; GCN2-NEXT: s_add_u32 s0, s0, 163749; GCN2-NEXT: s_addc_u32 s1, s1, 03750; GCN2-NEXT: v_mov_b32_e32 v0, s03751; GCN2-NEXT: v_mov_b32_e32 v1, s13752; GCN2-NEXT: v_mov_b32_e32 v2, s23753; GCN2-NEXT: flat_atomic_umin v[0:1], v23754; GCN2-NEXT: s_waitcnt lgkmcnt(0)3755; GCN2-NEXT: s_endpgm3756;3757; GCN3-LABEL: atomic_umin_i32_offset:3758; GCN3: ; %bb.0: ; %entry3759; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x243760; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c3761; GCN3-NEXT: s_waitcnt lgkmcnt(0)3762; GCN3-NEXT: v_mov_b32_e32 v0, s03763; GCN3-NEXT: v_mov_b32_e32 v1, s13764; GCN3-NEXT: v_mov_b32_e32 v2, s23765; GCN3-NEXT: flat_atomic_umin v[0:1], v2 offset:163766; GCN3-NEXT: s_waitcnt lgkmcnt(0)3767; GCN3-NEXT: s_endpgm3768;3769; GFX11-LABEL: atomic_umin_i32_offset:3770; GFX11: ; %bb.0: ; %entry3771; GFX11-NEXT: s_clause 0x13772; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x243773; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c3774; GFX11-NEXT: s_waitcnt lgkmcnt(0)3775; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13776; GFX11-NEXT: v_mov_b32_e32 v2, s23777; GFX11-NEXT: flat_atomic_min_u32 v[0:1], v2 offset:163778; GFX11-NEXT: s_waitcnt lgkmcnt(0)3779; GFX11-NEXT: s_waitcnt_vscnt null, 0x03780; GFX11-NEXT: buffer_gl0_inv3781; GFX11-NEXT: s_endpgm3782entry:3783 %gep = getelementptr inbounds i32, ptr %out, i32 43784 %val = atomicrmw volatile umin ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !03785 ret void3786}3787 3788define amdgpu_kernel void @atomic_umin_i32_ret_offset(ptr %out, ptr %out2, i32 %in) {3789; GCN1-LABEL: atomic_umin_i32_ret_offset:3790; GCN1: ; %bb.0: ; %entry3791; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93792; GCN1-NEXT: s_load_dword s4, s[4:5], 0xd3793; GCN1-NEXT: s_waitcnt lgkmcnt(0)3794; GCN1-NEXT: s_add_u32 s0, s0, 163795; GCN1-NEXT: s_addc_u32 s1, s1, 03796; GCN1-NEXT: v_mov_b32_e32 v0, s03797; GCN1-NEXT: v_mov_b32_e32 v1, s13798; GCN1-NEXT: v_mov_b32_e32 v2, s43799; GCN1-NEXT: flat_atomic_umin v2, v[0:1], v2 glc3800; GCN1-NEXT: s_waitcnt lgkmcnt(0)3801; GCN1-NEXT: v_mov_b32_e32 v0, s23802; GCN1-NEXT: v_mov_b32_e32 v1, s33803; GCN1-NEXT: s_waitcnt vmcnt(0)3804; GCN1-NEXT: flat_store_dword v[0:1], v23805; GCN1-NEXT: s_endpgm3806;3807; GCN2-LABEL: atomic_umin_i32_ret_offset:3808; GCN2: ; %bb.0: ; %entry3809; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243810; GCN2-NEXT: s_load_dword s4, s[4:5], 0x343811; GCN2-NEXT: s_waitcnt lgkmcnt(0)3812; GCN2-NEXT: s_add_u32 s0, s0, 163813; GCN2-NEXT: s_addc_u32 s1, s1, 03814; GCN2-NEXT: v_mov_b32_e32 v0, s03815; GCN2-NEXT: v_mov_b32_e32 v1, s13816; GCN2-NEXT: v_mov_b32_e32 v2, s43817; GCN2-NEXT: flat_atomic_umin v2, v[0:1], v2 glc3818; GCN2-NEXT: s_waitcnt lgkmcnt(0)3819; GCN2-NEXT: v_mov_b32_e32 v0, s23820; GCN2-NEXT: v_mov_b32_e32 v1, s33821; GCN2-NEXT: s_waitcnt vmcnt(0)3822; GCN2-NEXT: flat_store_dword v[0:1], v23823; GCN2-NEXT: s_endpgm3824;3825; GCN3-LABEL: atomic_umin_i32_ret_offset:3826; GCN3: ; %bb.0: ; %entry3827; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243828; GCN3-NEXT: s_load_dword s6, s[4:5], 0x343829; GCN3-NEXT: s_waitcnt lgkmcnt(0)3830; GCN3-NEXT: v_mov_b32_e32 v0, s03831; GCN3-NEXT: v_mov_b32_e32 v1, s13832; GCN3-NEXT: v_mov_b32_e32 v2, s63833; GCN3-NEXT: flat_atomic_umin v2, v[0:1], v2 offset:16 glc3834; GCN3-NEXT: s_waitcnt lgkmcnt(0)3835; GCN3-NEXT: v_mov_b32_e32 v0, s23836; GCN3-NEXT: v_mov_b32_e32 v1, s33837; GCN3-NEXT: s_waitcnt vmcnt(0)3838; GCN3-NEXT: flat_store_dword v[0:1], v23839; GCN3-NEXT: s_endpgm3840;3841; GFX11-LABEL: atomic_umin_i32_ret_offset:3842; GFX11: ; %bb.0: ; %entry3843; GFX11-NEXT: s_clause 0x13844; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x243845; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x343846; GFX11-NEXT: s_waitcnt lgkmcnt(0)3847; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13848; GFX11-NEXT: v_mov_b32_e32 v2, s43849; GFX11-NEXT: flat_atomic_min_u32 v2, v[0:1], v2 offset:16 glc3850; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3851; GFX11-NEXT: buffer_gl0_inv3852; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s33853; GFX11-NEXT: flat_store_b32 v[0:1], v23854; GFX11-NEXT: s_endpgm3855entry:3856 %gep = getelementptr inbounds i32, ptr %out, i32 43857 %val = atomicrmw volatile umin ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !03858 store i32 %val, ptr %out23859 ret void3860}3861 3862define amdgpu_kernel void @atomic_umin_i32_addr64_offset(ptr %out, i32 %in, i64 %index) {3863; GCN1-LABEL: atomic_umin_i32_addr64_offset:3864; GCN1: ; %bb.0: ; %entry3865; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd3866; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x93867; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb3868; GCN1-NEXT: s_waitcnt lgkmcnt(0)3869; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 23870; GCN1-NEXT: s_add_u32 s0, s2, s03871; GCN1-NEXT: s_addc_u32 s1, s3, s13872; GCN1-NEXT: s_add_u32 s0, s0, 163873; GCN1-NEXT: s_addc_u32 s1, s1, 03874; GCN1-NEXT: v_mov_b32_e32 v0, s03875; GCN1-NEXT: v_mov_b32_e32 v1, s13876; GCN1-NEXT: v_mov_b32_e32 v2, s43877; GCN1-NEXT: flat_atomic_umin v[0:1], v23878; GCN1-NEXT: s_waitcnt lgkmcnt(0)3879; GCN1-NEXT: s_endpgm3880;3881; GCN2-LABEL: atomic_umin_i32_addr64_offset:3882; GCN2: ; %bb.0: ; %entry3883; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x343884; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x243885; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c3886; GCN2-NEXT: s_waitcnt lgkmcnt(0)3887; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 23888; GCN2-NEXT: s_add_u32 s0, s2, s03889; GCN2-NEXT: s_addc_u32 s1, s3, s13890; GCN2-NEXT: s_add_u32 s0, s0, 163891; GCN2-NEXT: s_addc_u32 s1, s1, 03892; GCN2-NEXT: v_mov_b32_e32 v0, s03893; GCN2-NEXT: v_mov_b32_e32 v1, s13894; GCN2-NEXT: v_mov_b32_e32 v2, s43895; GCN2-NEXT: flat_atomic_umin v[0:1], v23896; GCN2-NEXT: s_waitcnt lgkmcnt(0)3897; GCN2-NEXT: s_endpgm3898;3899; GCN3-LABEL: atomic_umin_i32_addr64_offset:3900; GCN3: ; %bb.0: ; %entry3901; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x343902; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x243903; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c3904; GCN3-NEXT: s_waitcnt lgkmcnt(0)3905; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 23906; GCN3-NEXT: s_add_u32 s0, s2, s03907; GCN3-NEXT: s_addc_u32 s1, s3, s13908; GCN3-NEXT: v_mov_b32_e32 v0, s03909; GCN3-NEXT: v_mov_b32_e32 v1, s13910; GCN3-NEXT: v_mov_b32_e32 v2, s63911; GCN3-NEXT: flat_atomic_umin v[0:1], v2 offset:163912; GCN3-NEXT: s_waitcnt lgkmcnt(0)3913; GCN3-NEXT: s_endpgm3914;3915; GFX11-LABEL: atomic_umin_i32_addr64_offset:3916; GFX11: ; %bb.0: ; %entry3917; GFX11-NEXT: s_clause 0x23918; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x343919; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x243920; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c3921; GFX11-NEXT: s_waitcnt lgkmcnt(0)3922; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 23923; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)3924; GFX11-NEXT: s_add_u32 s0, s2, s03925; GFX11-NEXT: s_addc_u32 s1, s3, s13926; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s13927; GFX11-NEXT: v_mov_b32_e32 v2, s43928; GFX11-NEXT: flat_atomic_min_u32 v[0:1], v2 offset:163929; GFX11-NEXT: s_waitcnt lgkmcnt(0)3930; GFX11-NEXT: s_waitcnt_vscnt null, 0x03931; GFX11-NEXT: buffer_gl0_inv3932; GFX11-NEXT: s_endpgm3933entry:3934 %ptr = getelementptr inbounds i32, ptr %out, i64 %index3935 %gep = getelementptr inbounds i32, ptr %ptr, i32 43936 %val = atomicrmw volatile umin ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !03937 ret void3938}3939 3940define amdgpu_kernel void @atomic_umin_i32_ret_addr64_offset(ptr %out, ptr %out2, i32 %in, i64 %index) {3941; GCN1-LABEL: atomic_umin_i32_ret_addr64_offset:3942; GCN1: ; %bb.0: ; %entry3943; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf3944; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93945; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd3946; GCN1-NEXT: s_waitcnt lgkmcnt(0)3947; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 23948; GCN1-NEXT: s_add_u32 s0, s0, s43949; GCN1-NEXT: s_addc_u32 s1, s1, s53950; GCN1-NEXT: s_add_u32 s0, s0, 163951; GCN1-NEXT: s_addc_u32 s1, s1, 03952; GCN1-NEXT: v_mov_b32_e32 v0, s03953; GCN1-NEXT: v_mov_b32_e32 v1, s13954; GCN1-NEXT: v_mov_b32_e32 v2, s83955; GCN1-NEXT: flat_atomic_umin v2, v[0:1], v2 glc3956; GCN1-NEXT: s_waitcnt lgkmcnt(0)3957; GCN1-NEXT: v_mov_b32_e32 v0, s23958; GCN1-NEXT: v_mov_b32_e32 v1, s33959; GCN1-NEXT: s_waitcnt vmcnt(0)3960; GCN1-NEXT: flat_store_dword v[0:1], v23961; GCN1-NEXT: s_endpgm3962;3963; GCN2-LABEL: atomic_umin_i32_ret_addr64_offset:3964; GCN2: ; %bb.0: ; %entry3965; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c3966; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243967; GCN2-NEXT: s_load_dword s8, s[4:5], 0x343968; GCN2-NEXT: s_waitcnt lgkmcnt(0)3969; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 23970; GCN2-NEXT: s_add_u32 s0, s0, s43971; GCN2-NEXT: s_addc_u32 s1, s1, s53972; GCN2-NEXT: s_add_u32 s0, s0, 163973; GCN2-NEXT: s_addc_u32 s1, s1, 03974; GCN2-NEXT: v_mov_b32_e32 v0, s03975; GCN2-NEXT: v_mov_b32_e32 v1, s13976; GCN2-NEXT: v_mov_b32_e32 v2, s83977; GCN2-NEXT: flat_atomic_umin v2, v[0:1], v2 glc3978; GCN2-NEXT: s_waitcnt lgkmcnt(0)3979; GCN2-NEXT: v_mov_b32_e32 v0, s23980; GCN2-NEXT: v_mov_b32_e32 v1, s33981; GCN2-NEXT: s_waitcnt vmcnt(0)3982; GCN2-NEXT: flat_store_dword v[0:1], v23983; GCN2-NEXT: s_endpgm3984;3985; GCN3-LABEL: atomic_umin_i32_ret_addr64_offset:3986; GCN3: ; %bb.0: ; %entry3987; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c3988; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243989; GCN3-NEXT: s_load_dword s8, s[4:5], 0x343990; GCN3-NEXT: s_waitcnt lgkmcnt(0)3991; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 23992; GCN3-NEXT: s_add_u32 s0, s0, s43993; GCN3-NEXT: s_addc_u32 s1, s1, s53994; GCN3-NEXT: v_mov_b32_e32 v0, s03995; GCN3-NEXT: v_mov_b32_e32 v1, s13996; GCN3-NEXT: v_mov_b32_e32 v2, s83997; GCN3-NEXT: flat_atomic_umin v2, v[0:1], v2 offset:16 glc3998; GCN3-NEXT: s_waitcnt lgkmcnt(0)3999; GCN3-NEXT: v_mov_b32_e32 v0, s24000; GCN3-NEXT: v_mov_b32_e32 v1, s34001; GCN3-NEXT: s_waitcnt vmcnt(0)4002; GCN3-NEXT: flat_store_dword v[0:1], v24003; GCN3-NEXT: s_endpgm4004;4005; GFX11-LABEL: atomic_umin_i32_ret_addr64_offset:4006; GFX11: ; %bb.0: ; %entry4007; GFX11-NEXT: s_clause 0x24008; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c4009; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x244010; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x344011; GFX11-NEXT: s_waitcnt lgkmcnt(0)4012; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 24013; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)4014; GFX11-NEXT: s_add_u32 s0, s0, s44015; GFX11-NEXT: s_addc_u32 s1, s1, s54016; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14017; GFX11-NEXT: v_mov_b32_e32 v2, s84018; GFX11-NEXT: flat_atomic_min_u32 v2, v[0:1], v2 offset:16 glc4019; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4020; GFX11-NEXT: buffer_gl0_inv4021; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s34022; GFX11-NEXT: flat_store_b32 v[0:1], v24023; GFX11-NEXT: s_endpgm4024entry:4025 %ptr = getelementptr inbounds i32, ptr %out, i64 %index4026 %gep = getelementptr inbounds i32, ptr %ptr, i32 44027 %val = atomicrmw volatile umin ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !04028 store i32 %val, ptr %out24029 ret void4030}4031 4032define amdgpu_kernel void @atomic_umin_i32(ptr %out, i32 %in) {4033; GCN1-LABEL: atomic_umin_i32:4034; GCN1: ; %bb.0: ; %entry4035; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x94036; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb4037; GCN1-NEXT: s_waitcnt lgkmcnt(0)4038; GCN1-NEXT: v_mov_b32_e32 v0, s04039; GCN1-NEXT: v_mov_b32_e32 v1, s14040; GCN1-NEXT: v_mov_b32_e32 v2, s24041; GCN1-NEXT: flat_atomic_umin v[0:1], v24042; GCN1-NEXT: s_waitcnt lgkmcnt(0)4043; GCN1-NEXT: s_endpgm4044;4045; GCN2-LABEL: atomic_umin_i32:4046; GCN2: ; %bb.0: ; %entry4047; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x244048; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c4049; GCN2-NEXT: s_waitcnt lgkmcnt(0)4050; GCN2-NEXT: v_mov_b32_e32 v0, s04051; GCN2-NEXT: v_mov_b32_e32 v1, s14052; GCN2-NEXT: v_mov_b32_e32 v2, s24053; GCN2-NEXT: flat_atomic_umin v[0:1], v24054; GCN2-NEXT: s_waitcnt lgkmcnt(0)4055; GCN2-NEXT: s_endpgm4056;4057; GCN3-LABEL: atomic_umin_i32:4058; GCN3: ; %bb.0: ; %entry4059; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x244060; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c4061; GCN3-NEXT: s_waitcnt lgkmcnt(0)4062; GCN3-NEXT: v_mov_b32_e32 v0, s04063; GCN3-NEXT: v_mov_b32_e32 v1, s14064; GCN3-NEXT: v_mov_b32_e32 v2, s24065; GCN3-NEXT: flat_atomic_umin v[0:1], v24066; GCN3-NEXT: s_waitcnt lgkmcnt(0)4067; GCN3-NEXT: s_endpgm4068;4069; GFX11-LABEL: atomic_umin_i32:4070; GFX11: ; %bb.0: ; %entry4071; GFX11-NEXT: s_clause 0x14072; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x244073; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c4074; GFX11-NEXT: s_waitcnt lgkmcnt(0)4075; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14076; GFX11-NEXT: v_mov_b32_e32 v2, s24077; GFX11-NEXT: flat_atomic_min_u32 v[0:1], v24078; GFX11-NEXT: s_waitcnt lgkmcnt(0)4079; GFX11-NEXT: s_waitcnt_vscnt null, 0x04080; GFX11-NEXT: buffer_gl0_inv4081; GFX11-NEXT: s_endpgm4082entry:4083 %val = atomicrmw volatile umin ptr %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !04084 ret void4085}4086 4087define amdgpu_kernel void @atomic_umin_i32_ret(ptr %out, ptr %out2, i32 %in) {4088; GCN1-LABEL: atomic_umin_i32_ret:4089; GCN1: ; %bb.0: ; %entry4090; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94091; GCN1-NEXT: s_load_dword s4, s[4:5], 0xd4092; GCN1-NEXT: s_waitcnt lgkmcnt(0)4093; GCN1-NEXT: v_mov_b32_e32 v0, s04094; GCN1-NEXT: v_mov_b32_e32 v1, s14095; GCN1-NEXT: v_mov_b32_e32 v2, s44096; GCN1-NEXT: flat_atomic_umin v2, v[0:1], v2 glc4097; GCN1-NEXT: s_waitcnt lgkmcnt(0)4098; GCN1-NEXT: v_mov_b32_e32 v0, s24099; GCN1-NEXT: v_mov_b32_e32 v1, s34100; GCN1-NEXT: s_waitcnt vmcnt(0)4101; GCN1-NEXT: flat_store_dword v[0:1], v24102; GCN1-NEXT: s_endpgm4103;4104; GCN2-LABEL: atomic_umin_i32_ret:4105; GCN2: ; %bb.0: ; %entry4106; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244107; GCN2-NEXT: s_load_dword s4, s[4:5], 0x344108; GCN2-NEXT: s_waitcnt lgkmcnt(0)4109; GCN2-NEXT: v_mov_b32_e32 v0, s04110; GCN2-NEXT: v_mov_b32_e32 v1, s14111; GCN2-NEXT: v_mov_b32_e32 v2, s44112; GCN2-NEXT: flat_atomic_umin v2, v[0:1], v2 glc4113; GCN2-NEXT: s_waitcnt lgkmcnt(0)4114; GCN2-NEXT: v_mov_b32_e32 v0, s24115; GCN2-NEXT: v_mov_b32_e32 v1, s34116; GCN2-NEXT: s_waitcnt vmcnt(0)4117; GCN2-NEXT: flat_store_dword v[0:1], v24118; GCN2-NEXT: s_endpgm4119;4120; GCN3-LABEL: atomic_umin_i32_ret:4121; GCN3: ; %bb.0: ; %entry4122; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244123; GCN3-NEXT: s_load_dword s6, s[4:5], 0x344124; GCN3-NEXT: s_waitcnt lgkmcnt(0)4125; GCN3-NEXT: v_mov_b32_e32 v0, s04126; GCN3-NEXT: v_mov_b32_e32 v1, s14127; GCN3-NEXT: v_mov_b32_e32 v2, s64128; GCN3-NEXT: flat_atomic_umin v2, v[0:1], v2 glc4129; GCN3-NEXT: s_waitcnt lgkmcnt(0)4130; GCN3-NEXT: v_mov_b32_e32 v0, s24131; GCN3-NEXT: v_mov_b32_e32 v1, s34132; GCN3-NEXT: s_waitcnt vmcnt(0)4133; GCN3-NEXT: flat_store_dword v[0:1], v24134; GCN3-NEXT: s_endpgm4135;4136; GFX11-LABEL: atomic_umin_i32_ret:4137; GFX11: ; %bb.0: ; %entry4138; GFX11-NEXT: s_clause 0x14139; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x244140; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x344141; GFX11-NEXT: s_waitcnt lgkmcnt(0)4142; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14143; GFX11-NEXT: v_mov_b32_e32 v2, s44144; GFX11-NEXT: flat_atomic_min_u32 v2, v[0:1], v2 glc4145; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4146; GFX11-NEXT: buffer_gl0_inv4147; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s34148; GFX11-NEXT: flat_store_b32 v[0:1], v24149; GFX11-NEXT: s_endpgm4150entry:4151 %val = atomicrmw volatile umin ptr %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !04152 store i32 %val, ptr %out24153 ret void4154}4155 4156define amdgpu_kernel void @atomic_umin_i32_addr64(ptr %out, i32 %in, i64 %index) {4157; GCN1-LABEL: atomic_umin_i32_addr64:4158; GCN1: ; %bb.0: ; %entry4159; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd4160; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x94161; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb4162; GCN1-NEXT: s_waitcnt lgkmcnt(0)4163; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 24164; GCN1-NEXT: s_add_u32 s0, s2, s04165; GCN1-NEXT: s_addc_u32 s1, s3, s14166; GCN1-NEXT: v_mov_b32_e32 v0, s04167; GCN1-NEXT: v_mov_b32_e32 v1, s14168; GCN1-NEXT: v_mov_b32_e32 v2, s44169; GCN1-NEXT: flat_atomic_umin v[0:1], v24170; GCN1-NEXT: s_waitcnt lgkmcnt(0)4171; GCN1-NEXT: s_endpgm4172;4173; GCN2-LABEL: atomic_umin_i32_addr64:4174; GCN2: ; %bb.0: ; %entry4175; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x344176; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x244177; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c4178; GCN2-NEXT: s_waitcnt lgkmcnt(0)4179; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 24180; GCN2-NEXT: s_add_u32 s0, s2, s04181; GCN2-NEXT: s_addc_u32 s1, s3, s14182; GCN2-NEXT: v_mov_b32_e32 v0, s04183; GCN2-NEXT: v_mov_b32_e32 v1, s14184; GCN2-NEXT: v_mov_b32_e32 v2, s44185; GCN2-NEXT: flat_atomic_umin v[0:1], v24186; GCN2-NEXT: s_waitcnt lgkmcnt(0)4187; GCN2-NEXT: s_endpgm4188;4189; GCN3-LABEL: atomic_umin_i32_addr64:4190; GCN3: ; %bb.0: ; %entry4191; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x344192; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x244193; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c4194; GCN3-NEXT: s_waitcnt lgkmcnt(0)4195; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 24196; GCN3-NEXT: s_add_u32 s0, s2, s04197; GCN3-NEXT: s_addc_u32 s1, s3, s14198; GCN3-NEXT: v_mov_b32_e32 v0, s04199; GCN3-NEXT: v_mov_b32_e32 v1, s14200; GCN3-NEXT: v_mov_b32_e32 v2, s64201; GCN3-NEXT: flat_atomic_umin v[0:1], v24202; GCN3-NEXT: s_waitcnt lgkmcnt(0)4203; GCN3-NEXT: s_endpgm4204;4205; GFX11-LABEL: atomic_umin_i32_addr64:4206; GFX11: ; %bb.0: ; %entry4207; GFX11-NEXT: s_clause 0x24208; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x344209; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x244210; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c4211; GFX11-NEXT: s_waitcnt lgkmcnt(0)4212; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 24213; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)4214; GFX11-NEXT: s_add_u32 s0, s2, s04215; GFX11-NEXT: s_addc_u32 s1, s3, s14216; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14217; GFX11-NEXT: v_mov_b32_e32 v2, s44218; GFX11-NEXT: flat_atomic_min_u32 v[0:1], v24219; GFX11-NEXT: s_waitcnt lgkmcnt(0)4220; GFX11-NEXT: s_waitcnt_vscnt null, 0x04221; GFX11-NEXT: buffer_gl0_inv4222; GFX11-NEXT: s_endpgm4223entry:4224 %ptr = getelementptr inbounds i32, ptr %out, i64 %index4225 %val = atomicrmw volatile umin ptr %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !04226 ret void4227}4228 4229define amdgpu_kernel void @atomic_umin_i32_ret_addr64(ptr %out, ptr %out2, i32 %in, i64 %index) {4230; GCN1-LABEL: atomic_umin_i32_ret_addr64:4231; GCN1: ; %bb.0: ; %entry4232; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf4233; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94234; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd4235; GCN1-NEXT: s_waitcnt lgkmcnt(0)4236; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 24237; GCN1-NEXT: s_add_u32 s0, s0, s44238; GCN1-NEXT: s_addc_u32 s1, s1, s54239; GCN1-NEXT: v_mov_b32_e32 v0, s04240; GCN1-NEXT: v_mov_b32_e32 v1, s14241; GCN1-NEXT: v_mov_b32_e32 v2, s84242; GCN1-NEXT: flat_atomic_umin v2, v[0:1], v2 glc4243; GCN1-NEXT: s_waitcnt lgkmcnt(0)4244; GCN1-NEXT: v_mov_b32_e32 v0, s24245; GCN1-NEXT: v_mov_b32_e32 v1, s34246; GCN1-NEXT: s_waitcnt vmcnt(0)4247; GCN1-NEXT: flat_store_dword v[0:1], v24248; GCN1-NEXT: s_endpgm4249;4250; GCN2-LABEL: atomic_umin_i32_ret_addr64:4251; GCN2: ; %bb.0: ; %entry4252; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c4253; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244254; GCN2-NEXT: s_load_dword s8, s[4:5], 0x344255; GCN2-NEXT: s_waitcnt lgkmcnt(0)4256; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 24257; GCN2-NEXT: s_add_u32 s0, s0, s44258; GCN2-NEXT: s_addc_u32 s1, s1, s54259; GCN2-NEXT: v_mov_b32_e32 v0, s04260; GCN2-NEXT: v_mov_b32_e32 v1, s14261; GCN2-NEXT: v_mov_b32_e32 v2, s84262; GCN2-NEXT: flat_atomic_umin v2, v[0:1], v2 glc4263; GCN2-NEXT: s_waitcnt lgkmcnt(0)4264; GCN2-NEXT: v_mov_b32_e32 v0, s24265; GCN2-NEXT: v_mov_b32_e32 v1, s34266; GCN2-NEXT: s_waitcnt vmcnt(0)4267; GCN2-NEXT: flat_store_dword v[0:1], v24268; GCN2-NEXT: s_endpgm4269;4270; GCN3-LABEL: atomic_umin_i32_ret_addr64:4271; GCN3: ; %bb.0: ; %entry4272; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c4273; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244274; GCN3-NEXT: s_load_dword s8, s[4:5], 0x344275; GCN3-NEXT: s_waitcnt lgkmcnt(0)4276; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 24277; GCN3-NEXT: s_add_u32 s0, s0, s44278; GCN3-NEXT: s_addc_u32 s1, s1, s54279; GCN3-NEXT: v_mov_b32_e32 v0, s04280; GCN3-NEXT: v_mov_b32_e32 v1, s14281; GCN3-NEXT: v_mov_b32_e32 v2, s84282; GCN3-NEXT: flat_atomic_umin v2, v[0:1], v2 glc4283; GCN3-NEXT: s_waitcnt lgkmcnt(0)4284; GCN3-NEXT: v_mov_b32_e32 v0, s24285; GCN3-NEXT: v_mov_b32_e32 v1, s34286; GCN3-NEXT: s_waitcnt vmcnt(0)4287; GCN3-NEXT: flat_store_dword v[0:1], v24288; GCN3-NEXT: s_endpgm4289;4290; GFX11-LABEL: atomic_umin_i32_ret_addr64:4291; GFX11: ; %bb.0: ; %entry4292; GFX11-NEXT: s_clause 0x24293; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c4294; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x244295; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x344296; GFX11-NEXT: s_waitcnt lgkmcnt(0)4297; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 24298; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)4299; GFX11-NEXT: s_add_u32 s0, s0, s44300; GFX11-NEXT: s_addc_u32 s1, s1, s54301; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14302; GFX11-NEXT: v_mov_b32_e32 v2, s84303; GFX11-NEXT: flat_atomic_min_u32 v2, v[0:1], v2 glc4304; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4305; GFX11-NEXT: buffer_gl0_inv4306; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s34307; GFX11-NEXT: flat_store_b32 v[0:1], v24308; GFX11-NEXT: s_endpgm4309entry:4310 %ptr = getelementptr inbounds i32, ptr %out, i64 %index4311 %val = atomicrmw volatile umin ptr %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !04312 store i32 %val, ptr %out24313 ret void4314}4315 4316define amdgpu_kernel void @atomic_or_i32_offset(ptr %out, i32 %in) {4317; GCN1-LABEL: atomic_or_i32_offset:4318; GCN1: ; %bb.0: ; %entry4319; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x94320; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb4321; GCN1-NEXT: s_waitcnt lgkmcnt(0)4322; GCN1-NEXT: s_add_u32 s0, s0, 164323; GCN1-NEXT: s_addc_u32 s1, s1, 04324; GCN1-NEXT: v_mov_b32_e32 v0, s04325; GCN1-NEXT: v_mov_b32_e32 v1, s14326; GCN1-NEXT: v_mov_b32_e32 v2, s24327; GCN1-NEXT: flat_atomic_or v[0:1], v24328; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4329; GCN1-NEXT: buffer_wbinvl1_vol4330; GCN1-NEXT: s_endpgm4331;4332; GCN2-LABEL: atomic_or_i32_offset:4333; GCN2: ; %bb.0: ; %entry4334; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x244335; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c4336; GCN2-NEXT: s_waitcnt lgkmcnt(0)4337; GCN2-NEXT: s_add_u32 s0, s0, 164338; GCN2-NEXT: s_addc_u32 s1, s1, 04339; GCN2-NEXT: v_mov_b32_e32 v0, s04340; GCN2-NEXT: v_mov_b32_e32 v1, s14341; GCN2-NEXT: v_mov_b32_e32 v2, s24342; GCN2-NEXT: flat_atomic_or v[0:1], v24343; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4344; GCN2-NEXT: buffer_wbinvl1_vol4345; GCN2-NEXT: s_endpgm4346;4347; GCN3-LABEL: atomic_or_i32_offset:4348; GCN3: ; %bb.0: ; %entry4349; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x244350; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c4351; GCN3-NEXT: s_waitcnt lgkmcnt(0)4352; GCN3-NEXT: v_mov_b32_e32 v0, s04353; GCN3-NEXT: v_mov_b32_e32 v1, s14354; GCN3-NEXT: v_mov_b32_e32 v2, s24355; GCN3-NEXT: flat_atomic_or v[0:1], v2 offset:164356; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4357; GCN3-NEXT: buffer_wbinvl1_vol4358; GCN3-NEXT: s_endpgm4359;4360; GFX11-LABEL: atomic_or_i32_offset:4361; GFX11: ; %bb.0: ; %entry4362; GFX11-NEXT: s_clause 0x14363; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x244364; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c4365; GFX11-NEXT: s_waitcnt lgkmcnt(0)4366; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14367; GFX11-NEXT: v_mov_b32_e32 v2, s24368; GFX11-NEXT: flat_atomic_or_b32 v[0:1], v2 offset:164369; GFX11-NEXT: s_waitcnt lgkmcnt(0)4370; GFX11-NEXT: s_waitcnt_vscnt null, 0x04371; GFX11-NEXT: buffer_gl1_inv4372; GFX11-NEXT: buffer_gl0_inv4373; GFX11-NEXT: s_endpgm4374entry:4375 %gep = getelementptr inbounds i32, ptr %out, i32 44376 %val = atomicrmw volatile or ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !04377 ret void4378}4379 4380define amdgpu_kernel void @atomic_or_i32_ret_offset(ptr %out, ptr %out2, i32 %in) {4381; GCN1-LABEL: atomic_or_i32_ret_offset:4382; GCN1: ; %bb.0: ; %entry4383; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94384; GCN1-NEXT: s_load_dword s4, s[4:5], 0xd4385; GCN1-NEXT: s_waitcnt lgkmcnt(0)4386; GCN1-NEXT: s_add_u32 s0, s0, 164387; GCN1-NEXT: s_addc_u32 s1, s1, 04388; GCN1-NEXT: v_mov_b32_e32 v0, s04389; GCN1-NEXT: v_mov_b32_e32 v1, s14390; GCN1-NEXT: v_mov_b32_e32 v2, s44391; GCN1-NEXT: flat_atomic_or v2, v[0:1], v2 glc4392; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4393; GCN1-NEXT: buffer_wbinvl1_vol4394; GCN1-NEXT: v_mov_b32_e32 v0, s24395; GCN1-NEXT: v_mov_b32_e32 v1, s34396; GCN1-NEXT: flat_store_dword v[0:1], v24397; GCN1-NEXT: s_endpgm4398;4399; GCN2-LABEL: atomic_or_i32_ret_offset:4400; GCN2: ; %bb.0: ; %entry4401; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244402; GCN2-NEXT: s_load_dword s4, s[4:5], 0x344403; GCN2-NEXT: s_waitcnt lgkmcnt(0)4404; GCN2-NEXT: s_add_u32 s0, s0, 164405; GCN2-NEXT: s_addc_u32 s1, s1, 04406; GCN2-NEXT: v_mov_b32_e32 v0, s04407; GCN2-NEXT: v_mov_b32_e32 v1, s14408; GCN2-NEXT: v_mov_b32_e32 v2, s44409; GCN2-NEXT: flat_atomic_or v2, v[0:1], v2 glc4410; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4411; GCN2-NEXT: buffer_wbinvl1_vol4412; GCN2-NEXT: v_mov_b32_e32 v0, s24413; GCN2-NEXT: v_mov_b32_e32 v1, s34414; GCN2-NEXT: flat_store_dword v[0:1], v24415; GCN2-NEXT: s_endpgm4416;4417; GCN3-LABEL: atomic_or_i32_ret_offset:4418; GCN3: ; %bb.0: ; %entry4419; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244420; GCN3-NEXT: s_load_dword s6, s[4:5], 0x344421; GCN3-NEXT: s_waitcnt lgkmcnt(0)4422; GCN3-NEXT: v_mov_b32_e32 v0, s04423; GCN3-NEXT: v_mov_b32_e32 v1, s14424; GCN3-NEXT: v_mov_b32_e32 v2, s64425; GCN3-NEXT: flat_atomic_or v2, v[0:1], v2 offset:16 glc4426; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4427; GCN3-NEXT: buffer_wbinvl1_vol4428; GCN3-NEXT: v_mov_b32_e32 v0, s24429; GCN3-NEXT: v_mov_b32_e32 v1, s34430; GCN3-NEXT: flat_store_dword v[0:1], v24431; GCN3-NEXT: s_endpgm4432;4433; GFX11-LABEL: atomic_or_i32_ret_offset:4434; GFX11: ; %bb.0: ; %entry4435; GFX11-NEXT: s_clause 0x14436; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x244437; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x344438; GFX11-NEXT: s_waitcnt lgkmcnt(0)4439; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14440; GFX11-NEXT: v_mov_b32_e32 v2, s44441; GFX11-NEXT: flat_atomic_or_b32 v2, v[0:1], v2 offset:16 glc4442; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4443; GFX11-NEXT: buffer_gl1_inv4444; GFX11-NEXT: buffer_gl0_inv4445; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s34446; GFX11-NEXT: flat_store_b32 v[0:1], v24447; GFX11-NEXT: s_endpgm4448entry:4449 %gep = getelementptr inbounds i32, ptr %out, i32 44450 %val = atomicrmw volatile or ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !04451 store i32 %val, ptr %out24452 ret void4453}4454 4455define amdgpu_kernel void @atomic_or_i32_addr64_offset(ptr %out, i32 %in, i64 %index) {4456; GCN1-LABEL: atomic_or_i32_addr64_offset:4457; GCN1: ; %bb.0: ; %entry4458; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd4459; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x94460; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb4461; GCN1-NEXT: s_waitcnt lgkmcnt(0)4462; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 24463; GCN1-NEXT: s_add_u32 s0, s2, s04464; GCN1-NEXT: s_addc_u32 s1, s3, s14465; GCN1-NEXT: s_add_u32 s0, s0, 164466; GCN1-NEXT: s_addc_u32 s1, s1, 04467; GCN1-NEXT: v_mov_b32_e32 v0, s04468; GCN1-NEXT: v_mov_b32_e32 v1, s14469; GCN1-NEXT: v_mov_b32_e32 v2, s44470; GCN1-NEXT: flat_atomic_or v[0:1], v24471; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4472; GCN1-NEXT: buffer_wbinvl1_vol4473; GCN1-NEXT: s_endpgm4474;4475; GCN2-LABEL: atomic_or_i32_addr64_offset:4476; GCN2: ; %bb.0: ; %entry4477; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x344478; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x244479; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c4480; GCN2-NEXT: s_waitcnt lgkmcnt(0)4481; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 24482; GCN2-NEXT: s_add_u32 s0, s2, s04483; GCN2-NEXT: s_addc_u32 s1, s3, s14484; GCN2-NEXT: s_add_u32 s0, s0, 164485; GCN2-NEXT: s_addc_u32 s1, s1, 04486; GCN2-NEXT: v_mov_b32_e32 v0, s04487; GCN2-NEXT: v_mov_b32_e32 v1, s14488; GCN2-NEXT: v_mov_b32_e32 v2, s44489; GCN2-NEXT: flat_atomic_or v[0:1], v24490; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4491; GCN2-NEXT: buffer_wbinvl1_vol4492; GCN2-NEXT: s_endpgm4493;4494; GCN3-LABEL: atomic_or_i32_addr64_offset:4495; GCN3: ; %bb.0: ; %entry4496; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x344497; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x244498; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c4499; GCN3-NEXT: s_waitcnt lgkmcnt(0)4500; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 24501; GCN3-NEXT: s_add_u32 s0, s2, s04502; GCN3-NEXT: s_addc_u32 s1, s3, s14503; GCN3-NEXT: v_mov_b32_e32 v0, s04504; GCN3-NEXT: v_mov_b32_e32 v1, s14505; GCN3-NEXT: v_mov_b32_e32 v2, s64506; GCN3-NEXT: flat_atomic_or v[0:1], v2 offset:164507; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4508; GCN3-NEXT: buffer_wbinvl1_vol4509; GCN3-NEXT: s_endpgm4510;4511; GFX11-LABEL: atomic_or_i32_addr64_offset:4512; GFX11: ; %bb.0: ; %entry4513; GFX11-NEXT: s_clause 0x24514; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x344515; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x244516; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c4517; GFX11-NEXT: s_waitcnt lgkmcnt(0)4518; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 24519; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)4520; GFX11-NEXT: s_add_u32 s0, s2, s04521; GFX11-NEXT: s_addc_u32 s1, s3, s14522; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14523; GFX11-NEXT: v_mov_b32_e32 v2, s44524; GFX11-NEXT: flat_atomic_or_b32 v[0:1], v2 offset:164525; GFX11-NEXT: s_waitcnt lgkmcnt(0)4526; GFX11-NEXT: s_waitcnt_vscnt null, 0x04527; GFX11-NEXT: buffer_gl1_inv4528; GFX11-NEXT: buffer_gl0_inv4529; GFX11-NEXT: s_endpgm4530entry:4531 %ptr = getelementptr inbounds i32, ptr %out, i64 %index4532 %gep = getelementptr inbounds i32, ptr %ptr, i32 44533 %val = atomicrmw volatile or ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !04534 ret void4535}4536 4537define amdgpu_kernel void @atomic_or_i32_ret_addr64_offset(ptr %out, ptr %out2, i32 %in, i64 %index) {4538; GCN1-LABEL: atomic_or_i32_ret_addr64_offset:4539; GCN1: ; %bb.0: ; %entry4540; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf4541; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94542; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd4543; GCN1-NEXT: s_waitcnt lgkmcnt(0)4544; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 24545; GCN1-NEXT: s_add_u32 s0, s0, s44546; GCN1-NEXT: s_addc_u32 s1, s1, s54547; GCN1-NEXT: s_add_u32 s0, s0, 164548; GCN1-NEXT: s_addc_u32 s1, s1, 04549; GCN1-NEXT: v_mov_b32_e32 v0, s04550; GCN1-NEXT: v_mov_b32_e32 v1, s14551; GCN1-NEXT: v_mov_b32_e32 v2, s84552; GCN1-NEXT: flat_atomic_or v2, v[0:1], v2 glc4553; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4554; GCN1-NEXT: buffer_wbinvl1_vol4555; GCN1-NEXT: v_mov_b32_e32 v0, s24556; GCN1-NEXT: v_mov_b32_e32 v1, s34557; GCN1-NEXT: flat_store_dword v[0:1], v24558; GCN1-NEXT: s_endpgm4559;4560; GCN2-LABEL: atomic_or_i32_ret_addr64_offset:4561; GCN2: ; %bb.0: ; %entry4562; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c4563; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244564; GCN2-NEXT: s_load_dword s8, s[4:5], 0x344565; GCN2-NEXT: s_waitcnt lgkmcnt(0)4566; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 24567; GCN2-NEXT: s_add_u32 s0, s0, s44568; GCN2-NEXT: s_addc_u32 s1, s1, s54569; GCN2-NEXT: s_add_u32 s0, s0, 164570; GCN2-NEXT: s_addc_u32 s1, s1, 04571; GCN2-NEXT: v_mov_b32_e32 v0, s04572; GCN2-NEXT: v_mov_b32_e32 v1, s14573; GCN2-NEXT: v_mov_b32_e32 v2, s84574; GCN2-NEXT: flat_atomic_or v2, v[0:1], v2 glc4575; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4576; GCN2-NEXT: buffer_wbinvl1_vol4577; GCN2-NEXT: v_mov_b32_e32 v0, s24578; GCN2-NEXT: v_mov_b32_e32 v1, s34579; GCN2-NEXT: flat_store_dword v[0:1], v24580; GCN2-NEXT: s_endpgm4581;4582; GCN3-LABEL: atomic_or_i32_ret_addr64_offset:4583; GCN3: ; %bb.0: ; %entry4584; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c4585; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244586; GCN3-NEXT: s_load_dword s8, s[4:5], 0x344587; GCN3-NEXT: s_waitcnt lgkmcnt(0)4588; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 24589; GCN3-NEXT: s_add_u32 s0, s0, s44590; GCN3-NEXT: s_addc_u32 s1, s1, s54591; GCN3-NEXT: v_mov_b32_e32 v0, s04592; GCN3-NEXT: v_mov_b32_e32 v1, s14593; GCN3-NEXT: v_mov_b32_e32 v2, s84594; GCN3-NEXT: flat_atomic_or v2, v[0:1], v2 offset:16 glc4595; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4596; GCN3-NEXT: buffer_wbinvl1_vol4597; GCN3-NEXT: v_mov_b32_e32 v0, s24598; GCN3-NEXT: v_mov_b32_e32 v1, s34599; GCN3-NEXT: flat_store_dword v[0:1], v24600; GCN3-NEXT: s_endpgm4601;4602; GFX11-LABEL: atomic_or_i32_ret_addr64_offset:4603; GFX11: ; %bb.0: ; %entry4604; GFX11-NEXT: s_clause 0x24605; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c4606; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x244607; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x344608; GFX11-NEXT: s_waitcnt lgkmcnt(0)4609; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 24610; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)4611; GFX11-NEXT: s_add_u32 s0, s0, s44612; GFX11-NEXT: s_addc_u32 s1, s1, s54613; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14614; GFX11-NEXT: v_mov_b32_e32 v2, s84615; GFX11-NEXT: flat_atomic_or_b32 v2, v[0:1], v2 offset:16 glc4616; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4617; GFX11-NEXT: buffer_gl1_inv4618; GFX11-NEXT: buffer_gl0_inv4619; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s34620; GFX11-NEXT: flat_store_b32 v[0:1], v24621; GFX11-NEXT: s_endpgm4622entry:4623 %ptr = getelementptr inbounds i32, ptr %out, i64 %index4624 %gep = getelementptr inbounds i32, ptr %ptr, i32 44625 %val = atomicrmw volatile or ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !04626 store i32 %val, ptr %out24627 ret void4628}4629 4630define amdgpu_kernel void @atomic_or_i32(ptr %out, i32 %in) {4631; GCN1-LABEL: atomic_or_i32:4632; GCN1: ; %bb.0: ; %entry4633; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x94634; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb4635; GCN1-NEXT: s_waitcnt lgkmcnt(0)4636; GCN1-NEXT: v_mov_b32_e32 v0, s04637; GCN1-NEXT: v_mov_b32_e32 v1, s14638; GCN1-NEXT: v_mov_b32_e32 v2, s24639; GCN1-NEXT: flat_atomic_or v[0:1], v24640; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4641; GCN1-NEXT: buffer_wbinvl1_vol4642; GCN1-NEXT: s_endpgm4643;4644; GCN2-LABEL: atomic_or_i32:4645; GCN2: ; %bb.0: ; %entry4646; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x244647; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c4648; GCN2-NEXT: s_waitcnt lgkmcnt(0)4649; GCN2-NEXT: v_mov_b32_e32 v0, s04650; GCN2-NEXT: v_mov_b32_e32 v1, s14651; GCN2-NEXT: v_mov_b32_e32 v2, s24652; GCN2-NEXT: flat_atomic_or v[0:1], v24653; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4654; GCN2-NEXT: buffer_wbinvl1_vol4655; GCN2-NEXT: s_endpgm4656;4657; GCN3-LABEL: atomic_or_i32:4658; GCN3: ; %bb.0: ; %entry4659; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x244660; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c4661; GCN3-NEXT: s_waitcnt lgkmcnt(0)4662; GCN3-NEXT: v_mov_b32_e32 v0, s04663; GCN3-NEXT: v_mov_b32_e32 v1, s14664; GCN3-NEXT: v_mov_b32_e32 v2, s24665; GCN3-NEXT: flat_atomic_or v[0:1], v24666; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4667; GCN3-NEXT: buffer_wbinvl1_vol4668; GCN3-NEXT: s_endpgm4669;4670; GFX11-LABEL: atomic_or_i32:4671; GFX11: ; %bb.0: ; %entry4672; GFX11-NEXT: s_clause 0x14673; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x244674; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c4675; GFX11-NEXT: s_waitcnt lgkmcnt(0)4676; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14677; GFX11-NEXT: v_mov_b32_e32 v2, s24678; GFX11-NEXT: flat_atomic_or_b32 v[0:1], v24679; GFX11-NEXT: s_waitcnt lgkmcnt(0)4680; GFX11-NEXT: s_waitcnt_vscnt null, 0x04681; GFX11-NEXT: buffer_gl1_inv4682; GFX11-NEXT: buffer_gl0_inv4683; GFX11-NEXT: s_endpgm4684entry:4685 %val = atomicrmw volatile or ptr %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !04686 ret void4687}4688 4689define amdgpu_kernel void @atomic_or_i32_ret(ptr %out, ptr %out2, i32 %in) {4690; GCN1-LABEL: atomic_or_i32_ret:4691; GCN1: ; %bb.0: ; %entry4692; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94693; GCN1-NEXT: s_load_dword s4, s[4:5], 0xd4694; GCN1-NEXT: s_waitcnt lgkmcnt(0)4695; GCN1-NEXT: v_mov_b32_e32 v0, s04696; GCN1-NEXT: v_mov_b32_e32 v1, s14697; GCN1-NEXT: v_mov_b32_e32 v2, s44698; GCN1-NEXT: flat_atomic_or v2, v[0:1], v2 glc4699; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4700; GCN1-NEXT: buffer_wbinvl1_vol4701; GCN1-NEXT: v_mov_b32_e32 v0, s24702; GCN1-NEXT: v_mov_b32_e32 v1, s34703; GCN1-NEXT: flat_store_dword v[0:1], v24704; GCN1-NEXT: s_endpgm4705;4706; GCN2-LABEL: atomic_or_i32_ret:4707; GCN2: ; %bb.0: ; %entry4708; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244709; GCN2-NEXT: s_load_dword s4, s[4:5], 0x344710; GCN2-NEXT: s_waitcnt lgkmcnt(0)4711; GCN2-NEXT: v_mov_b32_e32 v0, s04712; GCN2-NEXT: v_mov_b32_e32 v1, s14713; GCN2-NEXT: v_mov_b32_e32 v2, s44714; GCN2-NEXT: flat_atomic_or v2, v[0:1], v2 glc4715; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4716; GCN2-NEXT: buffer_wbinvl1_vol4717; GCN2-NEXT: v_mov_b32_e32 v0, s24718; GCN2-NEXT: v_mov_b32_e32 v1, s34719; GCN2-NEXT: flat_store_dword v[0:1], v24720; GCN2-NEXT: s_endpgm4721;4722; GCN3-LABEL: atomic_or_i32_ret:4723; GCN3: ; %bb.0: ; %entry4724; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244725; GCN3-NEXT: s_load_dword s6, s[4:5], 0x344726; GCN3-NEXT: s_waitcnt lgkmcnt(0)4727; GCN3-NEXT: v_mov_b32_e32 v0, s04728; GCN3-NEXT: v_mov_b32_e32 v1, s14729; GCN3-NEXT: v_mov_b32_e32 v2, s64730; GCN3-NEXT: flat_atomic_or v2, v[0:1], v2 glc4731; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4732; GCN3-NEXT: buffer_wbinvl1_vol4733; GCN3-NEXT: v_mov_b32_e32 v0, s24734; GCN3-NEXT: v_mov_b32_e32 v1, s34735; GCN3-NEXT: flat_store_dword v[0:1], v24736; GCN3-NEXT: s_endpgm4737;4738; GFX11-LABEL: atomic_or_i32_ret:4739; GFX11: ; %bb.0: ; %entry4740; GFX11-NEXT: s_clause 0x14741; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x244742; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x344743; GFX11-NEXT: s_waitcnt lgkmcnt(0)4744; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14745; GFX11-NEXT: v_mov_b32_e32 v2, s44746; GFX11-NEXT: flat_atomic_or_b32 v2, v[0:1], v2 glc4747; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4748; GFX11-NEXT: buffer_gl1_inv4749; GFX11-NEXT: buffer_gl0_inv4750; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s34751; GFX11-NEXT: flat_store_b32 v[0:1], v24752; GFX11-NEXT: s_endpgm4753entry:4754 %val = atomicrmw volatile or ptr %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !04755 store i32 %val, ptr %out24756 ret void4757}4758 4759define amdgpu_kernel void @atomic_or_i32_addr64(ptr %out, i32 %in, i64 %index) {4760; GCN1-LABEL: atomic_or_i32_addr64:4761; GCN1: ; %bb.0: ; %entry4762; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd4763; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x94764; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb4765; GCN1-NEXT: s_waitcnt lgkmcnt(0)4766; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 24767; GCN1-NEXT: s_add_u32 s0, s2, s04768; GCN1-NEXT: s_addc_u32 s1, s3, s14769; GCN1-NEXT: v_mov_b32_e32 v0, s04770; GCN1-NEXT: v_mov_b32_e32 v1, s14771; GCN1-NEXT: v_mov_b32_e32 v2, s44772; GCN1-NEXT: flat_atomic_or v[0:1], v24773; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4774; GCN1-NEXT: buffer_wbinvl1_vol4775; GCN1-NEXT: s_endpgm4776;4777; GCN2-LABEL: atomic_or_i32_addr64:4778; GCN2: ; %bb.0: ; %entry4779; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x344780; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x244781; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c4782; GCN2-NEXT: s_waitcnt lgkmcnt(0)4783; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 24784; GCN2-NEXT: s_add_u32 s0, s2, s04785; GCN2-NEXT: s_addc_u32 s1, s3, s14786; GCN2-NEXT: v_mov_b32_e32 v0, s04787; GCN2-NEXT: v_mov_b32_e32 v1, s14788; GCN2-NEXT: v_mov_b32_e32 v2, s44789; GCN2-NEXT: flat_atomic_or v[0:1], v24790; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4791; GCN2-NEXT: buffer_wbinvl1_vol4792; GCN2-NEXT: s_endpgm4793;4794; GCN3-LABEL: atomic_or_i32_addr64:4795; GCN3: ; %bb.0: ; %entry4796; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x344797; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x244798; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c4799; GCN3-NEXT: s_waitcnt lgkmcnt(0)4800; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 24801; GCN3-NEXT: s_add_u32 s0, s2, s04802; GCN3-NEXT: s_addc_u32 s1, s3, s14803; GCN3-NEXT: v_mov_b32_e32 v0, s04804; GCN3-NEXT: v_mov_b32_e32 v1, s14805; GCN3-NEXT: v_mov_b32_e32 v2, s64806; GCN3-NEXT: flat_atomic_or v[0:1], v24807; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4808; GCN3-NEXT: buffer_wbinvl1_vol4809; GCN3-NEXT: s_endpgm4810;4811; GFX11-LABEL: atomic_or_i32_addr64:4812; GFX11: ; %bb.0: ; %entry4813; GFX11-NEXT: s_clause 0x24814; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x344815; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x244816; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c4817; GFX11-NEXT: s_waitcnt lgkmcnt(0)4818; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 24819; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)4820; GFX11-NEXT: s_add_u32 s0, s2, s04821; GFX11-NEXT: s_addc_u32 s1, s3, s14822; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14823; GFX11-NEXT: v_mov_b32_e32 v2, s44824; GFX11-NEXT: flat_atomic_or_b32 v[0:1], v24825; GFX11-NEXT: s_waitcnt lgkmcnt(0)4826; GFX11-NEXT: s_waitcnt_vscnt null, 0x04827; GFX11-NEXT: buffer_gl1_inv4828; GFX11-NEXT: buffer_gl0_inv4829; GFX11-NEXT: s_endpgm4830entry:4831 %ptr = getelementptr inbounds i32, ptr %out, i64 %index4832 %val = atomicrmw volatile or ptr %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !04833 ret void4834}4835 4836define amdgpu_kernel void @atomic_or_i32_ret_addr64(ptr %out, ptr %out2, i32 %in, i64 %index) {4837; GCN1-LABEL: atomic_or_i32_ret_addr64:4838; GCN1: ; %bb.0: ; %entry4839; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf4840; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94841; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd4842; GCN1-NEXT: s_waitcnt lgkmcnt(0)4843; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 24844; GCN1-NEXT: s_add_u32 s0, s0, s44845; GCN1-NEXT: s_addc_u32 s1, s1, s54846; GCN1-NEXT: v_mov_b32_e32 v0, s04847; GCN1-NEXT: v_mov_b32_e32 v1, s14848; GCN1-NEXT: v_mov_b32_e32 v2, s84849; GCN1-NEXT: flat_atomic_or v2, v[0:1], v2 glc4850; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4851; GCN1-NEXT: buffer_wbinvl1_vol4852; GCN1-NEXT: v_mov_b32_e32 v0, s24853; GCN1-NEXT: v_mov_b32_e32 v1, s34854; GCN1-NEXT: flat_store_dword v[0:1], v24855; GCN1-NEXT: s_endpgm4856;4857; GCN2-LABEL: atomic_or_i32_ret_addr64:4858; GCN2: ; %bb.0: ; %entry4859; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c4860; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244861; GCN2-NEXT: s_load_dword s8, s[4:5], 0x344862; GCN2-NEXT: s_waitcnt lgkmcnt(0)4863; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 24864; GCN2-NEXT: s_add_u32 s0, s0, s44865; GCN2-NEXT: s_addc_u32 s1, s1, s54866; GCN2-NEXT: v_mov_b32_e32 v0, s04867; GCN2-NEXT: v_mov_b32_e32 v1, s14868; GCN2-NEXT: v_mov_b32_e32 v2, s84869; GCN2-NEXT: flat_atomic_or v2, v[0:1], v2 glc4870; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4871; GCN2-NEXT: buffer_wbinvl1_vol4872; GCN2-NEXT: v_mov_b32_e32 v0, s24873; GCN2-NEXT: v_mov_b32_e32 v1, s34874; GCN2-NEXT: flat_store_dword v[0:1], v24875; GCN2-NEXT: s_endpgm4876;4877; GCN3-LABEL: atomic_or_i32_ret_addr64:4878; GCN3: ; %bb.0: ; %entry4879; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c4880; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244881; GCN3-NEXT: s_load_dword s8, s[4:5], 0x344882; GCN3-NEXT: s_waitcnt lgkmcnt(0)4883; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 24884; GCN3-NEXT: s_add_u32 s0, s0, s44885; GCN3-NEXT: s_addc_u32 s1, s1, s54886; GCN3-NEXT: v_mov_b32_e32 v0, s04887; GCN3-NEXT: v_mov_b32_e32 v1, s14888; GCN3-NEXT: v_mov_b32_e32 v2, s84889; GCN3-NEXT: flat_atomic_or v2, v[0:1], v2 glc4890; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4891; GCN3-NEXT: buffer_wbinvl1_vol4892; GCN3-NEXT: v_mov_b32_e32 v0, s24893; GCN3-NEXT: v_mov_b32_e32 v1, s34894; GCN3-NEXT: flat_store_dword v[0:1], v24895; GCN3-NEXT: s_endpgm4896;4897; GFX11-LABEL: atomic_or_i32_ret_addr64:4898; GFX11: ; %bb.0: ; %entry4899; GFX11-NEXT: s_clause 0x24900; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c4901; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x244902; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x344903; GFX11-NEXT: s_waitcnt lgkmcnt(0)4904; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 24905; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)4906; GFX11-NEXT: s_add_u32 s0, s0, s44907; GFX11-NEXT: s_addc_u32 s1, s1, s54908; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14909; GFX11-NEXT: v_mov_b32_e32 v2, s84910; GFX11-NEXT: flat_atomic_or_b32 v2, v[0:1], v2 glc4911; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4912; GFX11-NEXT: buffer_gl1_inv4913; GFX11-NEXT: buffer_gl0_inv4914; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s34915; GFX11-NEXT: flat_store_b32 v[0:1], v24916; GFX11-NEXT: s_endpgm4917entry:4918 %ptr = getelementptr inbounds i32, ptr %out, i64 %index4919 %val = atomicrmw volatile or ptr %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !04920 store i32 %val, ptr %out24921 ret void4922}4923 4924define amdgpu_kernel void @atomic_xchg_i32_offset(ptr %out, i32 %in) {4925; GCN1-LABEL: atomic_xchg_i32_offset:4926; GCN1: ; %bb.0: ; %entry4927; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x94928; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb4929; GCN1-NEXT: s_waitcnt lgkmcnt(0)4930; GCN1-NEXT: s_add_u32 s0, s0, 164931; GCN1-NEXT: s_addc_u32 s1, s1, 04932; GCN1-NEXT: v_mov_b32_e32 v0, s04933; GCN1-NEXT: v_mov_b32_e32 v1, s14934; GCN1-NEXT: v_mov_b32_e32 v2, s24935; GCN1-NEXT: flat_atomic_swap v[0:1], v24936; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4937; GCN1-NEXT: buffer_wbinvl1_vol4938; GCN1-NEXT: s_endpgm4939;4940; GCN2-LABEL: atomic_xchg_i32_offset:4941; GCN2: ; %bb.0: ; %entry4942; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x244943; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c4944; GCN2-NEXT: s_waitcnt lgkmcnt(0)4945; GCN2-NEXT: s_add_u32 s0, s0, 164946; GCN2-NEXT: s_addc_u32 s1, s1, 04947; GCN2-NEXT: v_mov_b32_e32 v0, s04948; GCN2-NEXT: v_mov_b32_e32 v1, s14949; GCN2-NEXT: v_mov_b32_e32 v2, s24950; GCN2-NEXT: flat_atomic_swap v[0:1], v24951; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4952; GCN2-NEXT: buffer_wbinvl1_vol4953; GCN2-NEXT: s_endpgm4954;4955; GCN3-LABEL: atomic_xchg_i32_offset:4956; GCN3: ; %bb.0: ; %entry4957; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x244958; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c4959; GCN3-NEXT: s_waitcnt lgkmcnt(0)4960; GCN3-NEXT: v_mov_b32_e32 v0, s04961; GCN3-NEXT: v_mov_b32_e32 v1, s14962; GCN3-NEXT: v_mov_b32_e32 v2, s24963; GCN3-NEXT: flat_atomic_swap v[0:1], v2 offset:164964; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4965; GCN3-NEXT: buffer_wbinvl1_vol4966; GCN3-NEXT: s_endpgm4967;4968; GFX11-LABEL: atomic_xchg_i32_offset:4969; GFX11: ; %bb.0: ; %entry4970; GFX11-NEXT: s_clause 0x14971; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x244972; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c4973; GFX11-NEXT: s_waitcnt lgkmcnt(0)4974; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s14975; GFX11-NEXT: v_mov_b32_e32 v2, s24976; GFX11-NEXT: flat_atomic_swap_b32 v[0:1], v2 offset:164977; GFX11-NEXT: s_waitcnt lgkmcnt(0)4978; GFX11-NEXT: s_waitcnt_vscnt null, 0x04979; GFX11-NEXT: buffer_gl1_inv4980; GFX11-NEXT: buffer_gl0_inv4981; GFX11-NEXT: s_endpgm4982entry:4983 %gep = getelementptr inbounds i32, ptr %out, i32 44984 %val = atomicrmw volatile xchg ptr %gep, i32 %in syncscope("agent") seq_cst4985 ret void4986}4987 4988define amdgpu_kernel void @atomic_xchg_f32_offset(ptr %out, float %in) {4989; GCN1-LABEL: atomic_xchg_f32_offset:4990; GCN1: ; %bb.0: ; %entry4991; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x94992; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb4993; GCN1-NEXT: s_waitcnt lgkmcnt(0)4994; GCN1-NEXT: s_add_u32 s0, s0, 164995; GCN1-NEXT: s_addc_u32 s1, s1, 04996; GCN1-NEXT: v_mov_b32_e32 v0, s04997; GCN1-NEXT: v_mov_b32_e32 v1, s14998; GCN1-NEXT: v_mov_b32_e32 v2, s24999; GCN1-NEXT: flat_atomic_swap v[0:1], v25000; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5001; GCN1-NEXT: buffer_wbinvl1_vol5002; GCN1-NEXT: s_endpgm5003;5004; GCN2-LABEL: atomic_xchg_f32_offset:5005; GCN2: ; %bb.0: ; %entry5006; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x245007; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c5008; GCN2-NEXT: s_waitcnt lgkmcnt(0)5009; GCN2-NEXT: s_add_u32 s0, s0, 165010; GCN2-NEXT: s_addc_u32 s1, s1, 05011; GCN2-NEXT: v_mov_b32_e32 v0, s05012; GCN2-NEXT: v_mov_b32_e32 v1, s15013; GCN2-NEXT: v_mov_b32_e32 v2, s25014; GCN2-NEXT: flat_atomic_swap v[0:1], v25015; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5016; GCN2-NEXT: buffer_wbinvl1_vol5017; GCN2-NEXT: s_endpgm5018;5019; GCN3-LABEL: atomic_xchg_f32_offset:5020; GCN3: ; %bb.0: ; %entry5021; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x245022; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c5023; GCN3-NEXT: s_waitcnt lgkmcnt(0)5024; GCN3-NEXT: v_mov_b32_e32 v0, s05025; GCN3-NEXT: v_mov_b32_e32 v1, s15026; GCN3-NEXT: v_mov_b32_e32 v2, s25027; GCN3-NEXT: flat_atomic_swap v[0:1], v2 offset:165028; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5029; GCN3-NEXT: buffer_wbinvl1_vol5030; GCN3-NEXT: s_endpgm5031;5032; GFX11-LABEL: atomic_xchg_f32_offset:5033; GFX11: ; %bb.0: ; %entry5034; GFX11-NEXT: s_clause 0x15035; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x245036; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c5037; GFX11-NEXT: s_waitcnt lgkmcnt(0)5038; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15039; GFX11-NEXT: v_mov_b32_e32 v2, s25040; GFX11-NEXT: flat_atomic_swap_b32 v[0:1], v2 offset:165041; GFX11-NEXT: s_waitcnt lgkmcnt(0)5042; GFX11-NEXT: s_waitcnt_vscnt null, 0x05043; GFX11-NEXT: buffer_gl1_inv5044; GFX11-NEXT: buffer_gl0_inv5045; GFX11-NEXT: s_endpgm5046entry:5047 %gep = getelementptr inbounds float, ptr %out, i32 45048 %val = atomicrmw volatile xchg ptr %gep, float %in syncscope("agent") seq_cst5049 ret void5050}5051 5052define amdgpu_kernel void @atomic_xchg_i32_ret_offset(ptr %out, ptr %out2, i32 %in) {5053; GCN1-LABEL: atomic_xchg_i32_ret_offset:5054; GCN1: ; %bb.0: ; %entry5055; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95056; GCN1-NEXT: s_load_dword s4, s[4:5], 0xd5057; GCN1-NEXT: s_waitcnt lgkmcnt(0)5058; GCN1-NEXT: s_add_u32 s0, s0, 165059; GCN1-NEXT: s_addc_u32 s1, s1, 05060; GCN1-NEXT: v_mov_b32_e32 v0, s05061; GCN1-NEXT: v_mov_b32_e32 v1, s15062; GCN1-NEXT: v_mov_b32_e32 v2, s45063; GCN1-NEXT: flat_atomic_swap v2, v[0:1], v2 glc5064; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5065; GCN1-NEXT: buffer_wbinvl1_vol5066; GCN1-NEXT: v_mov_b32_e32 v0, s25067; GCN1-NEXT: v_mov_b32_e32 v1, s35068; GCN1-NEXT: flat_store_dword v[0:1], v25069; GCN1-NEXT: s_endpgm5070;5071; GCN2-LABEL: atomic_xchg_i32_ret_offset:5072; GCN2: ; %bb.0: ; %entry5073; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245074; GCN2-NEXT: s_load_dword s4, s[4:5], 0x345075; GCN2-NEXT: s_waitcnt lgkmcnt(0)5076; GCN2-NEXT: s_add_u32 s0, s0, 165077; GCN2-NEXT: s_addc_u32 s1, s1, 05078; GCN2-NEXT: v_mov_b32_e32 v0, s05079; GCN2-NEXT: v_mov_b32_e32 v1, s15080; GCN2-NEXT: v_mov_b32_e32 v2, s45081; GCN2-NEXT: flat_atomic_swap v2, v[0:1], v2 glc5082; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5083; GCN2-NEXT: buffer_wbinvl1_vol5084; GCN2-NEXT: v_mov_b32_e32 v0, s25085; GCN2-NEXT: v_mov_b32_e32 v1, s35086; GCN2-NEXT: flat_store_dword v[0:1], v25087; GCN2-NEXT: s_endpgm5088;5089; GCN3-LABEL: atomic_xchg_i32_ret_offset:5090; GCN3: ; %bb.0: ; %entry5091; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245092; GCN3-NEXT: s_load_dword s6, s[4:5], 0x345093; GCN3-NEXT: s_waitcnt lgkmcnt(0)5094; GCN3-NEXT: v_mov_b32_e32 v0, s05095; GCN3-NEXT: v_mov_b32_e32 v1, s15096; GCN3-NEXT: v_mov_b32_e32 v2, s65097; GCN3-NEXT: flat_atomic_swap v2, v[0:1], v2 offset:16 glc5098; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5099; GCN3-NEXT: buffer_wbinvl1_vol5100; GCN3-NEXT: v_mov_b32_e32 v0, s25101; GCN3-NEXT: v_mov_b32_e32 v1, s35102; GCN3-NEXT: flat_store_dword v[0:1], v25103; GCN3-NEXT: s_endpgm5104;5105; GFX11-LABEL: atomic_xchg_i32_ret_offset:5106; GFX11: ; %bb.0: ; %entry5107; GFX11-NEXT: s_clause 0x15108; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x245109; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x345110; GFX11-NEXT: s_waitcnt lgkmcnt(0)5111; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15112; GFX11-NEXT: v_mov_b32_e32 v2, s45113; GFX11-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 offset:16 glc5114; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5115; GFX11-NEXT: buffer_gl1_inv5116; GFX11-NEXT: buffer_gl0_inv5117; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s35118; GFX11-NEXT: flat_store_b32 v[0:1], v25119; GFX11-NEXT: s_endpgm5120entry:5121 %gep = getelementptr inbounds i32, ptr %out, i32 45122 %val = atomicrmw volatile xchg ptr %gep, i32 %in syncscope("agent") seq_cst5123 store i32 %val, ptr %out25124 ret void5125}5126 5127define amdgpu_kernel void @atomic_xchg_i32_addr64_offset(ptr %out, i32 %in, i64 %index) {5128; GCN1-LABEL: atomic_xchg_i32_addr64_offset:5129; GCN1: ; %bb.0: ; %entry5130; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd5131; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x95132; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb5133; GCN1-NEXT: s_waitcnt lgkmcnt(0)5134; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 25135; GCN1-NEXT: s_add_u32 s0, s2, s05136; GCN1-NEXT: s_addc_u32 s1, s3, s15137; GCN1-NEXT: s_add_u32 s0, s0, 165138; GCN1-NEXT: s_addc_u32 s1, s1, 05139; GCN1-NEXT: v_mov_b32_e32 v0, s05140; GCN1-NEXT: v_mov_b32_e32 v1, s15141; GCN1-NEXT: v_mov_b32_e32 v2, s45142; GCN1-NEXT: flat_atomic_swap v[0:1], v25143; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5144; GCN1-NEXT: buffer_wbinvl1_vol5145; GCN1-NEXT: s_endpgm5146;5147; GCN2-LABEL: atomic_xchg_i32_addr64_offset:5148; GCN2: ; %bb.0: ; %entry5149; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x345150; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x245151; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c5152; GCN2-NEXT: s_waitcnt lgkmcnt(0)5153; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 25154; GCN2-NEXT: s_add_u32 s0, s2, s05155; GCN2-NEXT: s_addc_u32 s1, s3, s15156; GCN2-NEXT: s_add_u32 s0, s0, 165157; GCN2-NEXT: s_addc_u32 s1, s1, 05158; GCN2-NEXT: v_mov_b32_e32 v0, s05159; GCN2-NEXT: v_mov_b32_e32 v1, s15160; GCN2-NEXT: v_mov_b32_e32 v2, s45161; GCN2-NEXT: flat_atomic_swap v[0:1], v25162; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5163; GCN2-NEXT: buffer_wbinvl1_vol5164; GCN2-NEXT: s_endpgm5165;5166; GCN3-LABEL: atomic_xchg_i32_addr64_offset:5167; GCN3: ; %bb.0: ; %entry5168; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x345169; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x245170; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c5171; GCN3-NEXT: s_waitcnt lgkmcnt(0)5172; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 25173; GCN3-NEXT: s_add_u32 s0, s2, s05174; GCN3-NEXT: s_addc_u32 s1, s3, s15175; GCN3-NEXT: v_mov_b32_e32 v0, s05176; GCN3-NEXT: v_mov_b32_e32 v1, s15177; GCN3-NEXT: v_mov_b32_e32 v2, s65178; GCN3-NEXT: flat_atomic_swap v[0:1], v2 offset:165179; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5180; GCN3-NEXT: buffer_wbinvl1_vol5181; GCN3-NEXT: s_endpgm5182;5183; GFX11-LABEL: atomic_xchg_i32_addr64_offset:5184; GFX11: ; %bb.0: ; %entry5185; GFX11-NEXT: s_clause 0x25186; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x345187; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x245188; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c5189; GFX11-NEXT: s_waitcnt lgkmcnt(0)5190; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 25191; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)5192; GFX11-NEXT: s_add_u32 s0, s2, s05193; GFX11-NEXT: s_addc_u32 s1, s3, s15194; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15195; GFX11-NEXT: v_mov_b32_e32 v2, s45196; GFX11-NEXT: flat_atomic_swap_b32 v[0:1], v2 offset:165197; GFX11-NEXT: s_waitcnt lgkmcnt(0)5198; GFX11-NEXT: s_waitcnt_vscnt null, 0x05199; GFX11-NEXT: buffer_gl1_inv5200; GFX11-NEXT: buffer_gl0_inv5201; GFX11-NEXT: s_endpgm5202entry:5203 %ptr = getelementptr inbounds i32, ptr %out, i64 %index5204 %gep = getelementptr inbounds i32, ptr %ptr, i32 45205 %val = atomicrmw volatile xchg ptr %gep, i32 %in syncscope("agent") seq_cst5206 ret void5207}5208 5209define amdgpu_kernel void @atomic_xchg_i32_ret_addr64_offset(ptr %out, ptr %out2, i32 %in, i64 %index) {5210; GCN1-LABEL: atomic_xchg_i32_ret_addr64_offset:5211; GCN1: ; %bb.0: ; %entry5212; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf5213; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95214; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd5215; GCN1-NEXT: s_waitcnt lgkmcnt(0)5216; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 25217; GCN1-NEXT: s_add_u32 s0, s0, s45218; GCN1-NEXT: s_addc_u32 s1, s1, s55219; GCN1-NEXT: s_add_u32 s0, s0, 165220; GCN1-NEXT: s_addc_u32 s1, s1, 05221; GCN1-NEXT: v_mov_b32_e32 v0, s05222; GCN1-NEXT: v_mov_b32_e32 v1, s15223; GCN1-NEXT: v_mov_b32_e32 v2, s85224; GCN1-NEXT: flat_atomic_swap v2, v[0:1], v2 glc5225; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5226; GCN1-NEXT: buffer_wbinvl1_vol5227; GCN1-NEXT: v_mov_b32_e32 v0, s25228; GCN1-NEXT: v_mov_b32_e32 v1, s35229; GCN1-NEXT: flat_store_dword v[0:1], v25230; GCN1-NEXT: s_endpgm5231;5232; GCN2-LABEL: atomic_xchg_i32_ret_addr64_offset:5233; GCN2: ; %bb.0: ; %entry5234; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c5235; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245236; GCN2-NEXT: s_load_dword s8, s[4:5], 0x345237; GCN2-NEXT: s_waitcnt lgkmcnt(0)5238; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 25239; GCN2-NEXT: s_add_u32 s0, s0, s45240; GCN2-NEXT: s_addc_u32 s1, s1, s55241; GCN2-NEXT: s_add_u32 s0, s0, 165242; GCN2-NEXT: s_addc_u32 s1, s1, 05243; GCN2-NEXT: v_mov_b32_e32 v0, s05244; GCN2-NEXT: v_mov_b32_e32 v1, s15245; GCN2-NEXT: v_mov_b32_e32 v2, s85246; GCN2-NEXT: flat_atomic_swap v2, v[0:1], v2 glc5247; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5248; GCN2-NEXT: buffer_wbinvl1_vol5249; GCN2-NEXT: v_mov_b32_e32 v0, s25250; GCN2-NEXT: v_mov_b32_e32 v1, s35251; GCN2-NEXT: flat_store_dword v[0:1], v25252; GCN2-NEXT: s_endpgm5253;5254; GCN3-LABEL: atomic_xchg_i32_ret_addr64_offset:5255; GCN3: ; %bb.0: ; %entry5256; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c5257; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245258; GCN3-NEXT: s_load_dword s8, s[4:5], 0x345259; GCN3-NEXT: s_waitcnt lgkmcnt(0)5260; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 25261; GCN3-NEXT: s_add_u32 s0, s0, s45262; GCN3-NEXT: s_addc_u32 s1, s1, s55263; GCN3-NEXT: v_mov_b32_e32 v0, s05264; GCN3-NEXT: v_mov_b32_e32 v1, s15265; GCN3-NEXT: v_mov_b32_e32 v2, s85266; GCN3-NEXT: flat_atomic_swap v2, v[0:1], v2 offset:16 glc5267; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5268; GCN3-NEXT: buffer_wbinvl1_vol5269; GCN3-NEXT: v_mov_b32_e32 v0, s25270; GCN3-NEXT: v_mov_b32_e32 v1, s35271; GCN3-NEXT: flat_store_dword v[0:1], v25272; GCN3-NEXT: s_endpgm5273;5274; GFX11-LABEL: atomic_xchg_i32_ret_addr64_offset:5275; GFX11: ; %bb.0: ; %entry5276; GFX11-NEXT: s_clause 0x25277; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c5278; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x245279; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x345280; GFX11-NEXT: s_waitcnt lgkmcnt(0)5281; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 25282; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)5283; GFX11-NEXT: s_add_u32 s0, s0, s45284; GFX11-NEXT: s_addc_u32 s1, s1, s55285; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15286; GFX11-NEXT: v_mov_b32_e32 v2, s85287; GFX11-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 offset:16 glc5288; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5289; GFX11-NEXT: buffer_gl1_inv5290; GFX11-NEXT: buffer_gl0_inv5291; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s35292; GFX11-NEXT: flat_store_b32 v[0:1], v25293; GFX11-NEXT: s_endpgm5294entry:5295 %ptr = getelementptr inbounds i32, ptr %out, i64 %index5296 %gep = getelementptr inbounds i32, ptr %ptr, i32 45297 %val = atomicrmw volatile xchg ptr %gep, i32 %in syncscope("agent") seq_cst5298 store i32 %val, ptr %out25299 ret void5300}5301 5302define amdgpu_kernel void @atomic_xchg_i32(ptr %out, i32 %in) {5303; GCN1-LABEL: atomic_xchg_i32:5304; GCN1: ; %bb.0: ; %entry5305; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x95306; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb5307; GCN1-NEXT: s_waitcnt lgkmcnt(0)5308; GCN1-NEXT: v_mov_b32_e32 v0, s05309; GCN1-NEXT: v_mov_b32_e32 v1, s15310; GCN1-NEXT: v_mov_b32_e32 v2, s25311; GCN1-NEXT: flat_atomic_swap v[0:1], v25312; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5313; GCN1-NEXT: buffer_wbinvl1_vol5314; GCN1-NEXT: s_endpgm5315;5316; GCN2-LABEL: atomic_xchg_i32:5317; GCN2: ; %bb.0: ; %entry5318; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x245319; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c5320; GCN2-NEXT: s_waitcnt lgkmcnt(0)5321; GCN2-NEXT: v_mov_b32_e32 v0, s05322; GCN2-NEXT: v_mov_b32_e32 v1, s15323; GCN2-NEXT: v_mov_b32_e32 v2, s25324; GCN2-NEXT: flat_atomic_swap v[0:1], v25325; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5326; GCN2-NEXT: buffer_wbinvl1_vol5327; GCN2-NEXT: s_endpgm5328;5329; GCN3-LABEL: atomic_xchg_i32:5330; GCN3: ; %bb.0: ; %entry5331; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x245332; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c5333; GCN3-NEXT: s_waitcnt lgkmcnt(0)5334; GCN3-NEXT: v_mov_b32_e32 v0, s05335; GCN3-NEXT: v_mov_b32_e32 v1, s15336; GCN3-NEXT: v_mov_b32_e32 v2, s25337; GCN3-NEXT: flat_atomic_swap v[0:1], v25338; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5339; GCN3-NEXT: buffer_wbinvl1_vol5340; GCN3-NEXT: s_endpgm5341;5342; GFX11-LABEL: atomic_xchg_i32:5343; GFX11: ; %bb.0: ; %entry5344; GFX11-NEXT: s_clause 0x15345; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x245346; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c5347; GFX11-NEXT: s_waitcnt lgkmcnt(0)5348; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15349; GFX11-NEXT: v_mov_b32_e32 v2, s25350; GFX11-NEXT: flat_atomic_swap_b32 v[0:1], v25351; GFX11-NEXT: s_waitcnt lgkmcnt(0)5352; GFX11-NEXT: s_waitcnt_vscnt null, 0x05353; GFX11-NEXT: buffer_gl1_inv5354; GFX11-NEXT: buffer_gl0_inv5355; GFX11-NEXT: s_endpgm5356entry:5357 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") seq_cst5358 ret void5359}5360 5361define amdgpu_kernel void @atomic_xchg_i32_ret(ptr %out, ptr %out2, i32 %in) {5362; GCN1-LABEL: atomic_xchg_i32_ret:5363; GCN1: ; %bb.0: ; %entry5364; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95365; GCN1-NEXT: s_load_dword s4, s[4:5], 0xd5366; GCN1-NEXT: s_waitcnt lgkmcnt(0)5367; GCN1-NEXT: v_mov_b32_e32 v0, s05368; GCN1-NEXT: v_mov_b32_e32 v1, s15369; GCN1-NEXT: v_mov_b32_e32 v2, s45370; GCN1-NEXT: flat_atomic_swap v2, v[0:1], v2 glc5371; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5372; GCN1-NEXT: buffer_wbinvl1_vol5373; GCN1-NEXT: v_mov_b32_e32 v0, s25374; GCN1-NEXT: v_mov_b32_e32 v1, s35375; GCN1-NEXT: flat_store_dword v[0:1], v25376; GCN1-NEXT: s_endpgm5377;5378; GCN2-LABEL: atomic_xchg_i32_ret:5379; GCN2: ; %bb.0: ; %entry5380; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245381; GCN2-NEXT: s_load_dword s4, s[4:5], 0x345382; GCN2-NEXT: s_waitcnt lgkmcnt(0)5383; GCN2-NEXT: v_mov_b32_e32 v0, s05384; GCN2-NEXT: v_mov_b32_e32 v1, s15385; GCN2-NEXT: v_mov_b32_e32 v2, s45386; GCN2-NEXT: flat_atomic_swap v2, v[0:1], v2 glc5387; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5388; GCN2-NEXT: buffer_wbinvl1_vol5389; GCN2-NEXT: v_mov_b32_e32 v0, s25390; GCN2-NEXT: v_mov_b32_e32 v1, s35391; GCN2-NEXT: flat_store_dword v[0:1], v25392; GCN2-NEXT: s_endpgm5393;5394; GCN3-LABEL: atomic_xchg_i32_ret:5395; GCN3: ; %bb.0: ; %entry5396; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245397; GCN3-NEXT: s_load_dword s6, s[4:5], 0x345398; GCN3-NEXT: s_waitcnt lgkmcnt(0)5399; GCN3-NEXT: v_mov_b32_e32 v0, s05400; GCN3-NEXT: v_mov_b32_e32 v1, s15401; GCN3-NEXT: v_mov_b32_e32 v2, s65402; GCN3-NEXT: flat_atomic_swap v2, v[0:1], v2 glc5403; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5404; GCN3-NEXT: buffer_wbinvl1_vol5405; GCN3-NEXT: v_mov_b32_e32 v0, s25406; GCN3-NEXT: v_mov_b32_e32 v1, s35407; GCN3-NEXT: flat_store_dword v[0:1], v25408; GCN3-NEXT: s_endpgm5409;5410; GFX11-LABEL: atomic_xchg_i32_ret:5411; GFX11: ; %bb.0: ; %entry5412; GFX11-NEXT: s_clause 0x15413; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x245414; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x345415; GFX11-NEXT: s_waitcnt lgkmcnt(0)5416; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15417; GFX11-NEXT: v_mov_b32_e32 v2, s45418; GFX11-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc5419; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5420; GFX11-NEXT: buffer_gl1_inv5421; GFX11-NEXT: buffer_gl0_inv5422; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s35423; GFX11-NEXT: flat_store_b32 v[0:1], v25424; GFX11-NEXT: s_endpgm5425entry:5426 %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") seq_cst5427 store i32 %val, ptr %out25428 ret void5429}5430 5431define amdgpu_kernel void @atomic_xchg_i32_addr64(ptr %out, i32 %in, i64 %index) {5432; GCN1-LABEL: atomic_xchg_i32_addr64:5433; GCN1: ; %bb.0: ; %entry5434; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd5435; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x95436; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb5437; GCN1-NEXT: s_waitcnt lgkmcnt(0)5438; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 25439; GCN1-NEXT: s_add_u32 s0, s2, s05440; GCN1-NEXT: s_addc_u32 s1, s3, s15441; GCN1-NEXT: v_mov_b32_e32 v0, s05442; GCN1-NEXT: v_mov_b32_e32 v1, s15443; GCN1-NEXT: v_mov_b32_e32 v2, s45444; GCN1-NEXT: flat_atomic_swap v[0:1], v25445; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5446; GCN1-NEXT: buffer_wbinvl1_vol5447; GCN1-NEXT: s_endpgm5448;5449; GCN2-LABEL: atomic_xchg_i32_addr64:5450; GCN2: ; %bb.0: ; %entry5451; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x345452; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x245453; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c5454; GCN2-NEXT: s_waitcnt lgkmcnt(0)5455; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 25456; GCN2-NEXT: s_add_u32 s0, s2, s05457; GCN2-NEXT: s_addc_u32 s1, s3, s15458; GCN2-NEXT: v_mov_b32_e32 v0, s05459; GCN2-NEXT: v_mov_b32_e32 v1, s15460; GCN2-NEXT: v_mov_b32_e32 v2, s45461; GCN2-NEXT: flat_atomic_swap v[0:1], v25462; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5463; GCN2-NEXT: buffer_wbinvl1_vol5464; GCN2-NEXT: s_endpgm5465;5466; GCN3-LABEL: atomic_xchg_i32_addr64:5467; GCN3: ; %bb.0: ; %entry5468; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x345469; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x245470; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c5471; GCN3-NEXT: s_waitcnt lgkmcnt(0)5472; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 25473; GCN3-NEXT: s_add_u32 s0, s2, s05474; GCN3-NEXT: s_addc_u32 s1, s3, s15475; GCN3-NEXT: v_mov_b32_e32 v0, s05476; GCN3-NEXT: v_mov_b32_e32 v1, s15477; GCN3-NEXT: v_mov_b32_e32 v2, s65478; GCN3-NEXT: flat_atomic_swap v[0:1], v25479; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5480; GCN3-NEXT: buffer_wbinvl1_vol5481; GCN3-NEXT: s_endpgm5482;5483; GFX11-LABEL: atomic_xchg_i32_addr64:5484; GFX11: ; %bb.0: ; %entry5485; GFX11-NEXT: s_clause 0x25486; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x345487; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x245488; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c5489; GFX11-NEXT: s_waitcnt lgkmcnt(0)5490; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 25491; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)5492; GFX11-NEXT: s_add_u32 s0, s2, s05493; GFX11-NEXT: s_addc_u32 s1, s3, s15494; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15495; GFX11-NEXT: v_mov_b32_e32 v2, s45496; GFX11-NEXT: flat_atomic_swap_b32 v[0:1], v25497; GFX11-NEXT: s_waitcnt lgkmcnt(0)5498; GFX11-NEXT: s_waitcnt_vscnt null, 0x05499; GFX11-NEXT: buffer_gl1_inv5500; GFX11-NEXT: buffer_gl0_inv5501; GFX11-NEXT: s_endpgm5502entry:5503 %ptr = getelementptr inbounds i32, ptr %out, i64 %index5504 %val = atomicrmw volatile xchg ptr %ptr, i32 %in syncscope("agent") seq_cst5505 ret void5506}5507 5508define amdgpu_kernel void @atomic_xchg_i32_ret_addr64(ptr %out, ptr %out2, i32 %in, i64 %index) {5509; GCN1-LABEL: atomic_xchg_i32_ret_addr64:5510; GCN1: ; %bb.0: ; %entry5511; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf5512; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95513; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd5514; GCN1-NEXT: s_waitcnt lgkmcnt(0)5515; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 25516; GCN1-NEXT: s_add_u32 s0, s0, s45517; GCN1-NEXT: s_addc_u32 s1, s1, s55518; GCN1-NEXT: v_mov_b32_e32 v0, s05519; GCN1-NEXT: v_mov_b32_e32 v1, s15520; GCN1-NEXT: v_mov_b32_e32 v2, s85521; GCN1-NEXT: flat_atomic_swap v2, v[0:1], v2 glc5522; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5523; GCN1-NEXT: buffer_wbinvl1_vol5524; GCN1-NEXT: v_mov_b32_e32 v0, s25525; GCN1-NEXT: v_mov_b32_e32 v1, s35526; GCN1-NEXT: flat_store_dword v[0:1], v25527; GCN1-NEXT: s_endpgm5528;5529; GCN2-LABEL: atomic_xchg_i32_ret_addr64:5530; GCN2: ; %bb.0: ; %entry5531; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c5532; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245533; GCN2-NEXT: s_load_dword s8, s[4:5], 0x345534; GCN2-NEXT: s_waitcnt lgkmcnt(0)5535; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 25536; GCN2-NEXT: s_add_u32 s0, s0, s45537; GCN2-NEXT: s_addc_u32 s1, s1, s55538; GCN2-NEXT: v_mov_b32_e32 v0, s05539; GCN2-NEXT: v_mov_b32_e32 v1, s15540; GCN2-NEXT: v_mov_b32_e32 v2, s85541; GCN2-NEXT: flat_atomic_swap v2, v[0:1], v2 glc5542; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5543; GCN2-NEXT: buffer_wbinvl1_vol5544; GCN2-NEXT: v_mov_b32_e32 v0, s25545; GCN2-NEXT: v_mov_b32_e32 v1, s35546; GCN2-NEXT: flat_store_dword v[0:1], v25547; GCN2-NEXT: s_endpgm5548;5549; GCN3-LABEL: atomic_xchg_i32_ret_addr64:5550; GCN3: ; %bb.0: ; %entry5551; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c5552; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245553; GCN3-NEXT: s_load_dword s8, s[4:5], 0x345554; GCN3-NEXT: s_waitcnt lgkmcnt(0)5555; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 25556; GCN3-NEXT: s_add_u32 s0, s0, s45557; GCN3-NEXT: s_addc_u32 s1, s1, s55558; GCN3-NEXT: v_mov_b32_e32 v0, s05559; GCN3-NEXT: v_mov_b32_e32 v1, s15560; GCN3-NEXT: v_mov_b32_e32 v2, s85561; GCN3-NEXT: flat_atomic_swap v2, v[0:1], v2 glc5562; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5563; GCN3-NEXT: buffer_wbinvl1_vol5564; GCN3-NEXT: v_mov_b32_e32 v0, s25565; GCN3-NEXT: v_mov_b32_e32 v1, s35566; GCN3-NEXT: flat_store_dword v[0:1], v25567; GCN3-NEXT: s_endpgm5568;5569; GFX11-LABEL: atomic_xchg_i32_ret_addr64:5570; GFX11: ; %bb.0: ; %entry5571; GFX11-NEXT: s_clause 0x25572; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c5573; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x245574; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x345575; GFX11-NEXT: s_waitcnt lgkmcnt(0)5576; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 25577; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)5578; GFX11-NEXT: s_add_u32 s0, s0, s45579; GFX11-NEXT: s_addc_u32 s1, s1, s55580; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15581; GFX11-NEXT: v_mov_b32_e32 v2, s85582; GFX11-NEXT: flat_atomic_swap_b32 v2, v[0:1], v2 glc5583; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5584; GFX11-NEXT: buffer_gl1_inv5585; GFX11-NEXT: buffer_gl0_inv5586; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s35587; GFX11-NEXT: flat_store_b32 v[0:1], v25588; GFX11-NEXT: s_endpgm5589entry:5590 %ptr = getelementptr inbounds i32, ptr %out, i64 %index5591 %val = atomicrmw volatile xchg ptr %ptr, i32 %in syncscope("agent") seq_cst5592 store i32 %val, ptr %out25593 ret void5594}5595 5596; CMP_SWAP5597 5598define amdgpu_kernel void @atomic_cmpxchg_i32_offset(ptr %out, i32 %in, i32 %old) {5599; GCN1-LABEL: atomic_cmpxchg_i32_offset:5600; GCN1: ; %bb.0: ; %entry5601; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95602; GCN1-NEXT: s_waitcnt lgkmcnt(0)5603; GCN1-NEXT: s_add_u32 s0, s0, 165604; GCN1-NEXT: s_addc_u32 s1, s1, 05605; GCN1-NEXT: v_mov_b32_e32 v0, s05606; GCN1-NEXT: v_mov_b32_e32 v2, s25607; GCN1-NEXT: v_mov_b32_e32 v1, s15608; GCN1-NEXT: v_mov_b32_e32 v3, s35609; GCN1-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]5610; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5611; GCN1-NEXT: buffer_wbinvl1_vol5612; GCN1-NEXT: s_endpgm5613;5614; GCN2-LABEL: atomic_cmpxchg_i32_offset:5615; GCN2: ; %bb.0: ; %entry5616; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245617; GCN2-NEXT: s_waitcnt lgkmcnt(0)5618; GCN2-NEXT: s_add_u32 s0, s0, 165619; GCN2-NEXT: s_addc_u32 s1, s1, 05620; GCN2-NEXT: v_mov_b32_e32 v0, s05621; GCN2-NEXT: v_mov_b32_e32 v2, s25622; GCN2-NEXT: v_mov_b32_e32 v1, s15623; GCN2-NEXT: v_mov_b32_e32 v3, s35624; GCN2-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]5625; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5626; GCN2-NEXT: buffer_wbinvl1_vol5627; GCN2-NEXT: s_endpgm5628;5629; GCN3-LABEL: atomic_cmpxchg_i32_offset:5630; GCN3: ; %bb.0: ; %entry5631; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245632; GCN3-NEXT: s_waitcnt lgkmcnt(0)5633; GCN3-NEXT: v_mov_b32_e32 v0, s05634; GCN3-NEXT: v_mov_b32_e32 v2, s25635; GCN3-NEXT: v_mov_b32_e32 v1, s15636; GCN3-NEXT: v_mov_b32_e32 v3, s35637; GCN3-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:165638; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5639; GCN3-NEXT: buffer_wbinvl1_vol5640; GCN3-NEXT: s_endpgm5641;5642; GFX11-LABEL: atomic_cmpxchg_i32_offset:5643; GFX11: ; %bb.0: ; %entry5644; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x245645; GFX11-NEXT: s_waitcnt lgkmcnt(0)5646; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15647; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s35648; GFX11-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:165649; GFX11-NEXT: s_waitcnt lgkmcnt(0)5650; GFX11-NEXT: s_waitcnt_vscnt null, 0x05651; GFX11-NEXT: buffer_gl1_inv5652; GFX11-NEXT: buffer_gl0_inv5653; GFX11-NEXT: s_endpgm5654entry:5655 %gep = getelementptr inbounds i32, ptr %out, i32 45656 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst, !amdgpu.no.fine.grained.memory !05657 ret void5658}5659 5660define amdgpu_kernel void @atomic_cmpxchg_i32_ret_offset(ptr %out, ptr %out2, i32 %in, i32 %old) {5661; GCN1-LABEL: atomic_cmpxchg_i32_ret_offset:5662; GCN1: ; %bb.0: ; %entry5663; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95664; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd5665; GCN1-NEXT: s_waitcnt lgkmcnt(0)5666; GCN1-NEXT: s_add_u32 s0, s0, 165667; GCN1-NEXT: s_addc_u32 s1, s1, 05668; GCN1-NEXT: v_mov_b32_e32 v0, s05669; GCN1-NEXT: v_mov_b32_e32 v2, s45670; GCN1-NEXT: v_mov_b32_e32 v1, s15671; GCN1-NEXT: v_mov_b32_e32 v3, s55672; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc5673; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5674; GCN1-NEXT: buffer_wbinvl1_vol5675; GCN1-NEXT: v_mov_b32_e32 v0, s25676; GCN1-NEXT: v_mov_b32_e32 v1, s35677; GCN1-NEXT: flat_store_dword v[0:1], v25678; GCN1-NEXT: s_endpgm5679;5680; GCN2-LABEL: atomic_cmpxchg_i32_ret_offset:5681; GCN2: ; %bb.0: ; %entry5682; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245683; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x345684; GCN2-NEXT: s_waitcnt lgkmcnt(0)5685; GCN2-NEXT: s_add_u32 s0, s0, 165686; GCN2-NEXT: s_addc_u32 s1, s1, 05687; GCN2-NEXT: v_mov_b32_e32 v0, s05688; GCN2-NEXT: v_mov_b32_e32 v2, s45689; GCN2-NEXT: v_mov_b32_e32 v1, s15690; GCN2-NEXT: v_mov_b32_e32 v3, s55691; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc5692; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5693; GCN2-NEXT: buffer_wbinvl1_vol5694; GCN2-NEXT: v_mov_b32_e32 v0, s25695; GCN2-NEXT: v_mov_b32_e32 v1, s35696; GCN2-NEXT: flat_store_dword v[0:1], v25697; GCN2-NEXT: s_endpgm5698;5699; GCN3-LABEL: atomic_cmpxchg_i32_ret_offset:5700; GCN3: ; %bb.0: ; %entry5701; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245702; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x345703; GCN3-NEXT: s_waitcnt lgkmcnt(0)5704; GCN3-NEXT: v_mov_b32_e32 v0, s05705; GCN3-NEXT: v_mov_b32_e32 v2, s65706; GCN3-NEXT: v_mov_b32_e32 v1, s15707; GCN3-NEXT: v_mov_b32_e32 v3, s75708; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc5709; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5710; GCN3-NEXT: buffer_wbinvl1_vol5711; GCN3-NEXT: v_mov_b32_e32 v0, s25712; GCN3-NEXT: v_mov_b32_e32 v1, s35713; GCN3-NEXT: flat_store_dword v[0:1], v25714; GCN3-NEXT: s_endpgm5715;5716; GFX11-LABEL: atomic_cmpxchg_i32_ret_offset:5717; GFX11: ; %bb.0: ; %entry5718; GFX11-NEXT: s_clause 0x15719; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x245720; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x345721; GFX11-NEXT: s_waitcnt lgkmcnt(0)5722; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15723; GFX11-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s55724; GFX11-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc5725; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5726; GFX11-NEXT: buffer_gl1_inv5727; GFX11-NEXT: buffer_gl0_inv5728; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s35729; GFX11-NEXT: flat_store_b32 v[0:1], v25730; GFX11-NEXT: s_endpgm5731entry:5732 %gep = getelementptr inbounds i32, ptr %out, i32 45733 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst5734 %flag = extractvalue { i32, i1 } %val, 05735 store i32 %flag, ptr %out25736 ret void5737}5738 5739define amdgpu_kernel void @atomic_cmpxchg_i32_addr64_offset(ptr %out, i32 %in, i64 %index, i32 %old) {5740; GCN1-LABEL: atomic_cmpxchg_i32_addr64_offset:5741; GCN1: ; %bb.0: ; %entry5742; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd5743; GCN1-NEXT: s_load_dword s6, s[4:5], 0xb5744; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x95745; GCN1-NEXT: s_load_dword s4, s[4:5], 0xf5746; GCN1-NEXT: s_waitcnt lgkmcnt(0)5747; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 25748; GCN1-NEXT: v_mov_b32_e32 v0, s65749; GCN1-NEXT: s_add_u32 s0, s2, s05750; GCN1-NEXT: s_addc_u32 s1, s3, s15751; GCN1-NEXT: s_add_u32 s0, s0, 165752; GCN1-NEXT: s_addc_u32 s1, s1, 05753; GCN1-NEXT: v_mov_b32_e32 v3, s15754; GCN1-NEXT: v_mov_b32_e32 v1, s45755; GCN1-NEXT: v_mov_b32_e32 v2, s05756; GCN1-NEXT: flat_atomic_cmpswap v[2:3], v[0:1]5757; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5758; GCN1-NEXT: buffer_wbinvl1_vol5759; GCN1-NEXT: s_endpgm5760;5761; GCN2-LABEL: atomic_cmpxchg_i32_addr64_offset:5762; GCN2: ; %bb.0: ; %entry5763; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x345764; GCN2-NEXT: s_load_dword s6, s[4:5], 0x2c5765; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x245766; GCN2-NEXT: s_load_dword s4, s[4:5], 0x3c5767; GCN2-NEXT: s_waitcnt lgkmcnt(0)5768; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 25769; GCN2-NEXT: v_mov_b32_e32 v0, s65770; GCN2-NEXT: s_add_u32 s0, s2, s05771; GCN2-NEXT: s_addc_u32 s1, s3, s15772; GCN2-NEXT: s_add_u32 s0, s0, 165773; GCN2-NEXT: s_addc_u32 s1, s1, 05774; GCN2-NEXT: v_mov_b32_e32 v3, s15775; GCN2-NEXT: v_mov_b32_e32 v1, s45776; GCN2-NEXT: v_mov_b32_e32 v2, s05777; GCN2-NEXT: flat_atomic_cmpswap v[2:3], v[0:1]5778; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5779; GCN2-NEXT: buffer_wbinvl1_vol5780; GCN2-NEXT: s_endpgm5781;5782; GCN3-LABEL: atomic_cmpxchg_i32_addr64_offset:5783; GCN3: ; %bb.0: ; %entry5784; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x345785; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c5786; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x245787; GCN3-NEXT: s_load_dword s7, s[4:5], 0x3c5788; GCN3-NEXT: s_waitcnt lgkmcnt(0)5789; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 25790; GCN3-NEXT: v_mov_b32_e32 v0, s65791; GCN3-NEXT: s_add_u32 s0, s2, s05792; GCN3-NEXT: s_addc_u32 s1, s3, s15793; GCN3-NEXT: v_mov_b32_e32 v3, s15794; GCN3-NEXT: v_mov_b32_e32 v1, s75795; GCN3-NEXT: v_mov_b32_e32 v2, s05796; GCN3-NEXT: flat_atomic_cmpswap v[2:3], v[0:1] offset:165797; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5798; GCN3-NEXT: buffer_wbinvl1_vol5799; GCN3-NEXT: s_endpgm5800;5801; GFX11-LABEL: atomic_cmpxchg_i32_addr64_offset:5802; GFX11: ; %bb.0: ; %entry5803; GFX11-NEXT: s_clause 0x35804; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x345805; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x245806; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c5807; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x3c5808; GFX11-NEXT: s_waitcnt lgkmcnt(0)5809; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 25810; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5811; GFX11-NEXT: s_add_u32 s0, s2, s05812; GFX11-NEXT: s_addc_u32 s1, s3, s15813; GFX11-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s45814; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s05815; GFX11-NEXT: flat_atomic_cmpswap_b32 v[2:3], v[0:1] offset:165816; GFX11-NEXT: s_waitcnt lgkmcnt(0)5817; GFX11-NEXT: s_waitcnt_vscnt null, 0x05818; GFX11-NEXT: buffer_gl1_inv5819; GFX11-NEXT: buffer_gl0_inv5820; GFX11-NEXT: s_endpgm5821entry:5822 %ptr = getelementptr inbounds i32, ptr %out, i64 %index5823 %gep = getelementptr inbounds i32, ptr %ptr, i32 45824 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst5825 ret void5826}5827 5828define amdgpu_kernel void @atomic_cmpxchg_i32_ret_addr64_offset(ptr %out, ptr %out2, i32 %in, i64 %index, i32 %old) {5829; GCN1-LABEL: atomic_cmpxchg_i32_ret_addr64_offset:5830; GCN1: ; %bb.0: ; %entry5831; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf5832; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd5833; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95834; GCN1-NEXT: s_load_dword s9, s[4:5], 0x115835; GCN1-NEXT: s_waitcnt lgkmcnt(0)5836; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 25837; GCN1-NEXT: v_mov_b32_e32 v0, s85838; GCN1-NEXT: s_add_u32 s0, s0, s45839; GCN1-NEXT: s_addc_u32 s1, s1, s55840; GCN1-NEXT: s_add_u32 s0, s0, 165841; GCN1-NEXT: s_addc_u32 s1, s1, 05842; GCN1-NEXT: v_mov_b32_e32 v3, s15843; GCN1-NEXT: v_mov_b32_e32 v1, s95844; GCN1-NEXT: v_mov_b32_e32 v2, s05845; GCN1-NEXT: flat_atomic_cmpswap v2, v[2:3], v[0:1] glc5846; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5847; GCN1-NEXT: buffer_wbinvl1_vol5848; GCN1-NEXT: v_mov_b32_e32 v0, s25849; GCN1-NEXT: v_mov_b32_e32 v1, s35850; GCN1-NEXT: flat_store_dword v[0:1], v25851; GCN1-NEXT: s_endpgm5852;5853; GCN2-LABEL: atomic_cmpxchg_i32_ret_addr64_offset:5854; GCN2: ; %bb.0: ; %entry5855; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c5856; GCN2-NEXT: s_load_dword s8, s[4:5], 0x345857; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245858; GCN2-NEXT: s_load_dword s9, s[4:5], 0x445859; GCN2-NEXT: s_waitcnt lgkmcnt(0)5860; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 25861; GCN2-NEXT: v_mov_b32_e32 v0, s85862; GCN2-NEXT: s_add_u32 s0, s0, s45863; GCN2-NEXT: s_addc_u32 s1, s1, s55864; GCN2-NEXT: s_add_u32 s0, s0, 165865; GCN2-NEXT: s_addc_u32 s1, s1, 05866; GCN2-NEXT: v_mov_b32_e32 v3, s15867; GCN2-NEXT: v_mov_b32_e32 v1, s95868; GCN2-NEXT: v_mov_b32_e32 v2, s05869; GCN2-NEXT: flat_atomic_cmpswap v2, v[2:3], v[0:1] glc5870; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5871; GCN2-NEXT: buffer_wbinvl1_vol5872; GCN2-NEXT: v_mov_b32_e32 v0, s25873; GCN2-NEXT: v_mov_b32_e32 v1, s35874; GCN2-NEXT: flat_store_dword v[0:1], v25875; GCN2-NEXT: s_endpgm5876;5877; GCN3-LABEL: atomic_cmpxchg_i32_ret_addr64_offset:5878; GCN3: ; %bb.0: ; %entry5879; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c5880; GCN3-NEXT: s_load_dword s8, s[4:5], 0x345881; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245882; GCN3-NEXT: s_load_dword s9, s[4:5], 0x445883; GCN3-NEXT: s_waitcnt lgkmcnt(0)5884; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 25885; GCN3-NEXT: v_mov_b32_e32 v0, s85886; GCN3-NEXT: s_add_u32 s0, s0, s45887; GCN3-NEXT: s_addc_u32 s1, s1, s55888; GCN3-NEXT: v_mov_b32_e32 v3, s15889; GCN3-NEXT: v_mov_b32_e32 v1, s95890; GCN3-NEXT: v_mov_b32_e32 v2, s05891; GCN3-NEXT: flat_atomic_cmpswap v2, v[2:3], v[0:1] offset:16 glc5892; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5893; GCN3-NEXT: buffer_wbinvl1_vol5894; GCN3-NEXT: v_mov_b32_e32 v0, s25895; GCN3-NEXT: v_mov_b32_e32 v1, s35896; GCN3-NEXT: flat_store_dword v[0:1], v25897; GCN3-NEXT: s_endpgm5898;5899; GFX11-LABEL: atomic_cmpxchg_i32_ret_addr64_offset:5900; GFX11: ; %bb.0: ; %entry5901; GFX11-NEXT: s_clause 0x35902; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c5903; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x245904; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x345905; GFX11-NEXT: s_load_b32 s9, s[4:5], 0x445906; GFX11-NEXT: s_waitcnt lgkmcnt(0)5907; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 25908; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5909; GFX11-NEXT: s_add_u32 s0, s0, s45910; GFX11-NEXT: s_addc_u32 s1, s1, s55911; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s95912; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s05913; GFX11-NEXT: flat_atomic_cmpswap_b32 v2, v[2:3], v[0:1] offset:16 glc5914; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5915; GFX11-NEXT: buffer_gl1_inv5916; GFX11-NEXT: buffer_gl0_inv5917; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s35918; GFX11-NEXT: flat_store_b32 v[0:1], v25919; GFX11-NEXT: s_endpgm5920entry:5921 %ptr = getelementptr inbounds i32, ptr %out, i64 %index5922 %gep = getelementptr inbounds i32, ptr %ptr, i32 45923 %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst5924 %flag = extractvalue { i32, i1 } %val, 05925 store i32 %flag, ptr %out25926 ret void5927}5928 5929define amdgpu_kernel void @atomic_cmpxchg_i32(ptr %out, i32 %in, i32 %old) {5930; GCN1-LABEL: atomic_cmpxchg_i32:5931; GCN1: ; %bb.0: ; %entry5932; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95933; GCN1-NEXT: s_waitcnt lgkmcnt(0)5934; GCN1-NEXT: v_mov_b32_e32 v0, s05935; GCN1-NEXT: v_mov_b32_e32 v2, s25936; GCN1-NEXT: v_mov_b32_e32 v1, s15937; GCN1-NEXT: v_mov_b32_e32 v3, s35938; GCN1-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]5939; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5940; GCN1-NEXT: buffer_wbinvl1_vol5941; GCN1-NEXT: s_endpgm5942;5943; GCN2-LABEL: atomic_cmpxchg_i32:5944; GCN2: ; %bb.0: ; %entry5945; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245946; GCN2-NEXT: s_waitcnt lgkmcnt(0)5947; GCN2-NEXT: v_mov_b32_e32 v0, s05948; GCN2-NEXT: v_mov_b32_e32 v2, s25949; GCN2-NEXT: v_mov_b32_e32 v1, s15950; GCN2-NEXT: v_mov_b32_e32 v3, s35951; GCN2-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]5952; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5953; GCN2-NEXT: buffer_wbinvl1_vol5954; GCN2-NEXT: s_endpgm5955;5956; GCN3-LABEL: atomic_cmpxchg_i32:5957; GCN3: ; %bb.0: ; %entry5958; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245959; GCN3-NEXT: s_waitcnt lgkmcnt(0)5960; GCN3-NEXT: v_mov_b32_e32 v0, s05961; GCN3-NEXT: v_mov_b32_e32 v2, s25962; GCN3-NEXT: v_mov_b32_e32 v1, s15963; GCN3-NEXT: v_mov_b32_e32 v3, s35964; GCN3-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]5965; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5966; GCN3-NEXT: buffer_wbinvl1_vol5967; GCN3-NEXT: s_endpgm5968;5969; GFX11-LABEL: atomic_cmpxchg_i32:5970; GFX11: ; %bb.0: ; %entry5971; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x245972; GFX11-NEXT: s_waitcnt lgkmcnt(0)5973; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s15974; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s35975; GFX11-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3]5976; GFX11-NEXT: s_waitcnt lgkmcnt(0)5977; GFX11-NEXT: s_waitcnt_vscnt null, 0x05978; GFX11-NEXT: buffer_gl1_inv5979; GFX11-NEXT: buffer_gl0_inv5980; GFX11-NEXT: s_endpgm5981entry:5982 %val = cmpxchg volatile ptr %out, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst5983 ret void5984}5985 5986define amdgpu_kernel void @atomic_cmpxchg_i32_ret(ptr %out, ptr %out2, i32 %in, i32 %old) {5987; GCN1-LABEL: atomic_cmpxchg_i32_ret:5988; GCN1: ; %bb.0: ; %entry5989; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95990; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd5991; GCN1-NEXT: s_waitcnt lgkmcnt(0)5992; GCN1-NEXT: v_mov_b32_e32 v0, s05993; GCN1-NEXT: v_mov_b32_e32 v2, s45994; GCN1-NEXT: v_mov_b32_e32 v1, s15995; GCN1-NEXT: v_mov_b32_e32 v3, s55996; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc5997; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5998; GCN1-NEXT: buffer_wbinvl1_vol5999; GCN1-NEXT: v_mov_b32_e32 v0, s26000; GCN1-NEXT: v_mov_b32_e32 v1, s36001; GCN1-NEXT: flat_store_dword v[0:1], v26002; GCN1-NEXT: s_endpgm6003;6004; GCN2-LABEL: atomic_cmpxchg_i32_ret:6005; GCN2: ; %bb.0: ; %entry6006; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246007; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x346008; GCN2-NEXT: s_waitcnt lgkmcnt(0)6009; GCN2-NEXT: v_mov_b32_e32 v0, s06010; GCN2-NEXT: v_mov_b32_e32 v2, s46011; GCN2-NEXT: v_mov_b32_e32 v1, s16012; GCN2-NEXT: v_mov_b32_e32 v3, s56013; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc6014; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6015; GCN2-NEXT: buffer_wbinvl1_vol6016; GCN2-NEXT: v_mov_b32_e32 v0, s26017; GCN2-NEXT: v_mov_b32_e32 v1, s36018; GCN2-NEXT: flat_store_dword v[0:1], v26019; GCN2-NEXT: s_endpgm6020;6021; GCN3-LABEL: atomic_cmpxchg_i32_ret:6022; GCN3: ; %bb.0: ; %entry6023; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246024; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346025; GCN3-NEXT: s_waitcnt lgkmcnt(0)6026; GCN3-NEXT: v_mov_b32_e32 v0, s06027; GCN3-NEXT: v_mov_b32_e32 v2, s66028; GCN3-NEXT: v_mov_b32_e32 v1, s16029; GCN3-NEXT: v_mov_b32_e32 v3, s76030; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc6031; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6032; GCN3-NEXT: buffer_wbinvl1_vol6033; GCN3-NEXT: v_mov_b32_e32 v0, s26034; GCN3-NEXT: v_mov_b32_e32 v1, s36035; GCN3-NEXT: flat_store_dword v[0:1], v26036; GCN3-NEXT: s_endpgm6037;6038; GFX11-LABEL: atomic_cmpxchg_i32_ret:6039; GFX11: ; %bb.0: ; %entry6040; GFX11-NEXT: s_clause 0x16041; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x246042; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x346043; GFX11-NEXT: s_waitcnt lgkmcnt(0)6044; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16045; GFX11-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s56046; GFX11-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc6047; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6048; GFX11-NEXT: buffer_gl1_inv6049; GFX11-NEXT: buffer_gl0_inv6050; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s36051; GFX11-NEXT: flat_store_b32 v[0:1], v26052; GFX11-NEXT: s_endpgm6053entry:6054 %val = cmpxchg volatile ptr %out, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst6055 %flag = extractvalue { i32, i1 } %val, 06056 store i32 %flag, ptr %out26057 ret void6058}6059 6060define amdgpu_kernel void @atomic_cmpxchg_i32_addr64(ptr %out, i32 %in, i64 %index, i32 %old) {6061; GCN1-LABEL: atomic_cmpxchg_i32_addr64:6062; GCN1: ; %bb.0: ; %entry6063; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd6064; GCN1-NEXT: s_load_dword s6, s[4:5], 0xb6065; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x96066; GCN1-NEXT: s_load_dword s4, s[4:5], 0xf6067; GCN1-NEXT: s_waitcnt lgkmcnt(0)6068; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 26069; GCN1-NEXT: v_mov_b32_e32 v0, s66070; GCN1-NEXT: s_add_u32 s0, s2, s06071; GCN1-NEXT: s_addc_u32 s1, s3, s16072; GCN1-NEXT: v_mov_b32_e32 v3, s16073; GCN1-NEXT: v_mov_b32_e32 v1, s46074; GCN1-NEXT: v_mov_b32_e32 v2, s06075; GCN1-NEXT: flat_atomic_cmpswap v[2:3], v[0:1]6076; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6077; GCN1-NEXT: buffer_wbinvl1_vol6078; GCN1-NEXT: s_endpgm6079;6080; GCN2-LABEL: atomic_cmpxchg_i32_addr64:6081; GCN2: ; %bb.0: ; %entry6082; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x346083; GCN2-NEXT: s_load_dword s6, s[4:5], 0x2c6084; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x246085; GCN2-NEXT: s_load_dword s4, s[4:5], 0x3c6086; GCN2-NEXT: s_waitcnt lgkmcnt(0)6087; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 26088; GCN2-NEXT: v_mov_b32_e32 v0, s66089; GCN2-NEXT: s_add_u32 s0, s2, s06090; GCN2-NEXT: s_addc_u32 s1, s3, s16091; GCN2-NEXT: v_mov_b32_e32 v3, s16092; GCN2-NEXT: v_mov_b32_e32 v1, s46093; GCN2-NEXT: v_mov_b32_e32 v2, s06094; GCN2-NEXT: flat_atomic_cmpswap v[2:3], v[0:1]6095; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6096; GCN2-NEXT: buffer_wbinvl1_vol6097; GCN2-NEXT: s_endpgm6098;6099; GCN3-LABEL: atomic_cmpxchg_i32_addr64:6100; GCN3: ; %bb.0: ; %entry6101; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x346102; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c6103; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x246104; GCN3-NEXT: s_load_dword s7, s[4:5], 0x3c6105; GCN3-NEXT: s_waitcnt lgkmcnt(0)6106; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 26107; GCN3-NEXT: v_mov_b32_e32 v0, s66108; GCN3-NEXT: s_add_u32 s0, s2, s06109; GCN3-NEXT: s_addc_u32 s1, s3, s16110; GCN3-NEXT: v_mov_b32_e32 v3, s16111; GCN3-NEXT: v_mov_b32_e32 v1, s76112; GCN3-NEXT: v_mov_b32_e32 v2, s06113; GCN3-NEXT: flat_atomic_cmpswap v[2:3], v[0:1]6114; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6115; GCN3-NEXT: buffer_wbinvl1_vol6116; GCN3-NEXT: s_endpgm6117;6118; GFX11-LABEL: atomic_cmpxchg_i32_addr64:6119; GFX11: ; %bb.0: ; %entry6120; GFX11-NEXT: s_clause 0x36121; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x346122; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x246123; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c6124; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x3c6125; GFX11-NEXT: s_waitcnt lgkmcnt(0)6126; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 26127; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6128; GFX11-NEXT: s_add_u32 s0, s2, s06129; GFX11-NEXT: s_addc_u32 s1, s3, s16130; GFX11-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s46131; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s06132; GFX11-NEXT: flat_atomic_cmpswap_b32 v[2:3], v[0:1]6133; GFX11-NEXT: s_waitcnt lgkmcnt(0)6134; GFX11-NEXT: s_waitcnt_vscnt null, 0x06135; GFX11-NEXT: buffer_gl1_inv6136; GFX11-NEXT: buffer_gl0_inv6137; GFX11-NEXT: s_endpgm6138entry:6139 %ptr = getelementptr inbounds i32, ptr %out, i64 %index6140 %val = cmpxchg volatile ptr %ptr, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst6141 ret void6142}6143 6144define amdgpu_kernel void @atomic_cmpxchg_i32_ret_addr64(ptr %out, ptr %out2, i32 %in, i64 %index, i32 %old) {6145; GCN1-LABEL: atomic_cmpxchg_i32_ret_addr64:6146; GCN1: ; %bb.0: ; %entry6147; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf6148; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd6149; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96150; GCN1-NEXT: s_load_dword s9, s[4:5], 0x116151; GCN1-NEXT: s_waitcnt lgkmcnt(0)6152; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 26153; GCN1-NEXT: v_mov_b32_e32 v0, s86154; GCN1-NEXT: s_add_u32 s0, s0, s46155; GCN1-NEXT: s_addc_u32 s1, s1, s56156; GCN1-NEXT: v_mov_b32_e32 v3, s16157; GCN1-NEXT: v_mov_b32_e32 v1, s96158; GCN1-NEXT: v_mov_b32_e32 v2, s06159; GCN1-NEXT: flat_atomic_cmpswap v2, v[2:3], v[0:1] glc6160; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6161; GCN1-NEXT: buffer_wbinvl1_vol6162; GCN1-NEXT: v_mov_b32_e32 v0, s26163; GCN1-NEXT: v_mov_b32_e32 v1, s36164; GCN1-NEXT: flat_store_dword v[0:1], v26165; GCN1-NEXT: s_endpgm6166;6167; GCN2-LABEL: atomic_cmpxchg_i32_ret_addr64:6168; GCN2: ; %bb.0: ; %entry6169; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c6170; GCN2-NEXT: s_load_dword s8, s[4:5], 0x346171; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246172; GCN2-NEXT: s_load_dword s9, s[4:5], 0x446173; GCN2-NEXT: s_waitcnt lgkmcnt(0)6174; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 26175; GCN2-NEXT: v_mov_b32_e32 v0, s86176; GCN2-NEXT: s_add_u32 s0, s0, s46177; GCN2-NEXT: s_addc_u32 s1, s1, s56178; GCN2-NEXT: v_mov_b32_e32 v3, s16179; GCN2-NEXT: v_mov_b32_e32 v1, s96180; GCN2-NEXT: v_mov_b32_e32 v2, s06181; GCN2-NEXT: flat_atomic_cmpswap v2, v[2:3], v[0:1] glc6182; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6183; GCN2-NEXT: buffer_wbinvl1_vol6184; GCN2-NEXT: v_mov_b32_e32 v0, s26185; GCN2-NEXT: v_mov_b32_e32 v1, s36186; GCN2-NEXT: flat_store_dword v[0:1], v26187; GCN2-NEXT: s_endpgm6188;6189; GCN3-LABEL: atomic_cmpxchg_i32_ret_addr64:6190; GCN3: ; %bb.0: ; %entry6191; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c6192; GCN3-NEXT: s_load_dword s8, s[4:5], 0x346193; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246194; GCN3-NEXT: s_load_dword s9, s[4:5], 0x446195; GCN3-NEXT: s_waitcnt lgkmcnt(0)6196; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 26197; GCN3-NEXT: v_mov_b32_e32 v0, s86198; GCN3-NEXT: s_add_u32 s0, s0, s46199; GCN3-NEXT: s_addc_u32 s1, s1, s56200; GCN3-NEXT: v_mov_b32_e32 v3, s16201; GCN3-NEXT: v_mov_b32_e32 v1, s96202; GCN3-NEXT: v_mov_b32_e32 v2, s06203; GCN3-NEXT: flat_atomic_cmpswap v2, v[2:3], v[0:1] glc6204; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6205; GCN3-NEXT: buffer_wbinvl1_vol6206; GCN3-NEXT: v_mov_b32_e32 v0, s26207; GCN3-NEXT: v_mov_b32_e32 v1, s36208; GCN3-NEXT: flat_store_dword v[0:1], v26209; GCN3-NEXT: s_endpgm6210;6211; GFX11-LABEL: atomic_cmpxchg_i32_ret_addr64:6212; GFX11: ; %bb.0: ; %entry6213; GFX11-NEXT: s_clause 0x36214; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c6215; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x246216; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x346217; GFX11-NEXT: s_load_b32 s9, s[4:5], 0x446218; GFX11-NEXT: s_waitcnt lgkmcnt(0)6219; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 26220; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6221; GFX11-NEXT: s_add_u32 s0, s0, s46222; GFX11-NEXT: s_addc_u32 s1, s1, s56223; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s96224; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s06225; GFX11-NEXT: flat_atomic_cmpswap_b32 v2, v[2:3], v[0:1] glc6226; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6227; GFX11-NEXT: buffer_gl1_inv6228; GFX11-NEXT: buffer_gl0_inv6229; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s36230; GFX11-NEXT: flat_store_b32 v[0:1], v26231; GFX11-NEXT: s_endpgm6232entry:6233 %ptr = getelementptr inbounds i32, ptr %out, i64 %index6234 %val = cmpxchg volatile ptr %ptr, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst6235 %flag = extractvalue { i32, i1 } %val, 06236 store i32 %flag, ptr %out26237 ret void6238}6239 6240define amdgpu_kernel void @atomic_xor_i32_offset(ptr %out, i32 %in) {6241; GCN1-LABEL: atomic_xor_i32_offset:6242; GCN1: ; %bb.0: ; %entry6243; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x96244; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb6245; GCN1-NEXT: s_waitcnt lgkmcnt(0)6246; GCN1-NEXT: s_add_u32 s0, s0, 166247; GCN1-NEXT: s_addc_u32 s1, s1, 06248; GCN1-NEXT: v_mov_b32_e32 v0, s06249; GCN1-NEXT: v_mov_b32_e32 v1, s16250; GCN1-NEXT: v_mov_b32_e32 v2, s26251; GCN1-NEXT: flat_atomic_xor v[0:1], v26252; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6253; GCN1-NEXT: buffer_wbinvl1_vol6254; GCN1-NEXT: s_endpgm6255;6256; GCN2-LABEL: atomic_xor_i32_offset:6257; GCN2: ; %bb.0: ; %entry6258; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x246259; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c6260; GCN2-NEXT: s_waitcnt lgkmcnt(0)6261; GCN2-NEXT: s_add_u32 s0, s0, 166262; GCN2-NEXT: s_addc_u32 s1, s1, 06263; GCN2-NEXT: v_mov_b32_e32 v0, s06264; GCN2-NEXT: v_mov_b32_e32 v1, s16265; GCN2-NEXT: v_mov_b32_e32 v2, s26266; GCN2-NEXT: flat_atomic_xor v[0:1], v26267; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6268; GCN2-NEXT: buffer_wbinvl1_vol6269; GCN2-NEXT: s_endpgm6270;6271; GCN3-LABEL: atomic_xor_i32_offset:6272; GCN3: ; %bb.0: ; %entry6273; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x246274; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c6275; GCN3-NEXT: s_waitcnt lgkmcnt(0)6276; GCN3-NEXT: v_mov_b32_e32 v0, s06277; GCN3-NEXT: v_mov_b32_e32 v1, s16278; GCN3-NEXT: v_mov_b32_e32 v2, s26279; GCN3-NEXT: flat_atomic_xor v[0:1], v2 offset:166280; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6281; GCN3-NEXT: buffer_wbinvl1_vol6282; GCN3-NEXT: s_endpgm6283;6284; GFX11-LABEL: atomic_xor_i32_offset:6285; GFX11: ; %bb.0: ; %entry6286; GFX11-NEXT: s_clause 0x16287; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x246288; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c6289; GFX11-NEXT: s_waitcnt lgkmcnt(0)6290; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16291; GFX11-NEXT: v_mov_b32_e32 v2, s26292; GFX11-NEXT: flat_atomic_xor_b32 v[0:1], v2 offset:166293; GFX11-NEXT: s_waitcnt lgkmcnt(0)6294; GFX11-NEXT: s_waitcnt_vscnt null, 0x06295; GFX11-NEXT: buffer_gl1_inv6296; GFX11-NEXT: buffer_gl0_inv6297; GFX11-NEXT: s_endpgm6298entry:6299 %gep = getelementptr inbounds i32, ptr %out, i32 46300 %val = atomicrmw volatile xor ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !06301 ret void6302}6303 6304define amdgpu_kernel void @atomic_xor_i32_ret_offset(ptr %out, ptr %out2, i32 %in) {6305; GCN1-LABEL: atomic_xor_i32_ret_offset:6306; GCN1: ; %bb.0: ; %entry6307; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96308; GCN1-NEXT: s_load_dword s4, s[4:5], 0xd6309; GCN1-NEXT: s_waitcnt lgkmcnt(0)6310; GCN1-NEXT: s_add_u32 s0, s0, 166311; GCN1-NEXT: s_addc_u32 s1, s1, 06312; GCN1-NEXT: v_mov_b32_e32 v0, s06313; GCN1-NEXT: v_mov_b32_e32 v1, s16314; GCN1-NEXT: v_mov_b32_e32 v2, s46315; GCN1-NEXT: flat_atomic_xor v2, v[0:1], v2 glc6316; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6317; GCN1-NEXT: buffer_wbinvl1_vol6318; GCN1-NEXT: v_mov_b32_e32 v0, s26319; GCN1-NEXT: v_mov_b32_e32 v1, s36320; GCN1-NEXT: flat_store_dword v[0:1], v26321; GCN1-NEXT: s_endpgm6322;6323; GCN2-LABEL: atomic_xor_i32_ret_offset:6324; GCN2: ; %bb.0: ; %entry6325; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246326; GCN2-NEXT: s_load_dword s4, s[4:5], 0x346327; GCN2-NEXT: s_waitcnt lgkmcnt(0)6328; GCN2-NEXT: s_add_u32 s0, s0, 166329; GCN2-NEXT: s_addc_u32 s1, s1, 06330; GCN2-NEXT: v_mov_b32_e32 v0, s06331; GCN2-NEXT: v_mov_b32_e32 v1, s16332; GCN2-NEXT: v_mov_b32_e32 v2, s46333; GCN2-NEXT: flat_atomic_xor v2, v[0:1], v2 glc6334; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6335; GCN2-NEXT: buffer_wbinvl1_vol6336; GCN2-NEXT: v_mov_b32_e32 v0, s26337; GCN2-NEXT: v_mov_b32_e32 v1, s36338; GCN2-NEXT: flat_store_dword v[0:1], v26339; GCN2-NEXT: s_endpgm6340;6341; GCN3-LABEL: atomic_xor_i32_ret_offset:6342; GCN3: ; %bb.0: ; %entry6343; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246344; GCN3-NEXT: s_load_dword s6, s[4:5], 0x346345; GCN3-NEXT: s_waitcnt lgkmcnt(0)6346; GCN3-NEXT: v_mov_b32_e32 v0, s06347; GCN3-NEXT: v_mov_b32_e32 v1, s16348; GCN3-NEXT: v_mov_b32_e32 v2, s66349; GCN3-NEXT: flat_atomic_xor v2, v[0:1], v2 offset:16 glc6350; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6351; GCN3-NEXT: buffer_wbinvl1_vol6352; GCN3-NEXT: v_mov_b32_e32 v0, s26353; GCN3-NEXT: v_mov_b32_e32 v1, s36354; GCN3-NEXT: flat_store_dword v[0:1], v26355; GCN3-NEXT: s_endpgm6356;6357; GFX11-LABEL: atomic_xor_i32_ret_offset:6358; GFX11: ; %bb.0: ; %entry6359; GFX11-NEXT: s_clause 0x16360; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x246361; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x346362; GFX11-NEXT: s_waitcnt lgkmcnt(0)6363; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16364; GFX11-NEXT: v_mov_b32_e32 v2, s46365; GFX11-NEXT: flat_atomic_xor_b32 v2, v[0:1], v2 offset:16 glc6366; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6367; GFX11-NEXT: buffer_gl1_inv6368; GFX11-NEXT: buffer_gl0_inv6369; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s36370; GFX11-NEXT: flat_store_b32 v[0:1], v26371; GFX11-NEXT: s_endpgm6372entry:6373 %gep = getelementptr inbounds i32, ptr %out, i32 46374 %val = atomicrmw volatile xor ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !06375 store i32 %val, ptr %out26376 ret void6377}6378 6379define amdgpu_kernel void @atomic_xor_i32_addr64_offset(ptr %out, i32 %in, i64 %index) {6380; GCN1-LABEL: atomic_xor_i32_addr64_offset:6381; GCN1: ; %bb.0: ; %entry6382; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd6383; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x96384; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb6385; GCN1-NEXT: s_waitcnt lgkmcnt(0)6386; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 26387; GCN1-NEXT: s_add_u32 s0, s2, s06388; GCN1-NEXT: s_addc_u32 s1, s3, s16389; GCN1-NEXT: s_add_u32 s0, s0, 166390; GCN1-NEXT: s_addc_u32 s1, s1, 06391; GCN1-NEXT: v_mov_b32_e32 v0, s06392; GCN1-NEXT: v_mov_b32_e32 v1, s16393; GCN1-NEXT: v_mov_b32_e32 v2, s46394; GCN1-NEXT: flat_atomic_xor v[0:1], v26395; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6396; GCN1-NEXT: buffer_wbinvl1_vol6397; GCN1-NEXT: s_endpgm6398;6399; GCN2-LABEL: atomic_xor_i32_addr64_offset:6400; GCN2: ; %bb.0: ; %entry6401; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x346402; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x246403; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c6404; GCN2-NEXT: s_waitcnt lgkmcnt(0)6405; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 26406; GCN2-NEXT: s_add_u32 s0, s2, s06407; GCN2-NEXT: s_addc_u32 s1, s3, s16408; GCN2-NEXT: s_add_u32 s0, s0, 166409; GCN2-NEXT: s_addc_u32 s1, s1, 06410; GCN2-NEXT: v_mov_b32_e32 v0, s06411; GCN2-NEXT: v_mov_b32_e32 v1, s16412; GCN2-NEXT: v_mov_b32_e32 v2, s46413; GCN2-NEXT: flat_atomic_xor v[0:1], v26414; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6415; GCN2-NEXT: buffer_wbinvl1_vol6416; GCN2-NEXT: s_endpgm6417;6418; GCN3-LABEL: atomic_xor_i32_addr64_offset:6419; GCN3: ; %bb.0: ; %entry6420; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x346421; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x246422; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c6423; GCN3-NEXT: s_waitcnt lgkmcnt(0)6424; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 26425; GCN3-NEXT: s_add_u32 s0, s2, s06426; GCN3-NEXT: s_addc_u32 s1, s3, s16427; GCN3-NEXT: v_mov_b32_e32 v0, s06428; GCN3-NEXT: v_mov_b32_e32 v1, s16429; GCN3-NEXT: v_mov_b32_e32 v2, s66430; GCN3-NEXT: flat_atomic_xor v[0:1], v2 offset:166431; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6432; GCN3-NEXT: buffer_wbinvl1_vol6433; GCN3-NEXT: s_endpgm6434;6435; GFX11-LABEL: atomic_xor_i32_addr64_offset:6436; GFX11: ; %bb.0: ; %entry6437; GFX11-NEXT: s_clause 0x26438; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x346439; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x246440; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c6441; GFX11-NEXT: s_waitcnt lgkmcnt(0)6442; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 26443; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)6444; GFX11-NEXT: s_add_u32 s0, s2, s06445; GFX11-NEXT: s_addc_u32 s1, s3, s16446; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16447; GFX11-NEXT: v_mov_b32_e32 v2, s46448; GFX11-NEXT: flat_atomic_xor_b32 v[0:1], v2 offset:166449; GFX11-NEXT: s_waitcnt lgkmcnt(0)6450; GFX11-NEXT: s_waitcnt_vscnt null, 0x06451; GFX11-NEXT: buffer_gl1_inv6452; GFX11-NEXT: buffer_gl0_inv6453; GFX11-NEXT: s_endpgm6454entry:6455 %ptr = getelementptr inbounds i32, ptr %out, i64 %index6456 %gep = getelementptr inbounds i32, ptr %ptr, i32 46457 %val = atomicrmw volatile xor ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !06458 ret void6459}6460 6461define amdgpu_kernel void @atomic_xor_i32_ret_addr64_offset(ptr %out, ptr %out2, i32 %in, i64 %index) {6462; GCN1-LABEL: atomic_xor_i32_ret_addr64_offset:6463; GCN1: ; %bb.0: ; %entry6464; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf6465; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96466; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd6467; GCN1-NEXT: s_waitcnt lgkmcnt(0)6468; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 26469; GCN1-NEXT: s_add_u32 s0, s0, s46470; GCN1-NEXT: s_addc_u32 s1, s1, s56471; GCN1-NEXT: s_add_u32 s0, s0, 166472; GCN1-NEXT: s_addc_u32 s1, s1, 06473; GCN1-NEXT: v_mov_b32_e32 v0, s06474; GCN1-NEXT: v_mov_b32_e32 v1, s16475; GCN1-NEXT: v_mov_b32_e32 v2, s86476; GCN1-NEXT: flat_atomic_xor v2, v[0:1], v2 glc6477; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6478; GCN1-NEXT: buffer_wbinvl1_vol6479; GCN1-NEXT: v_mov_b32_e32 v0, s26480; GCN1-NEXT: v_mov_b32_e32 v1, s36481; GCN1-NEXT: flat_store_dword v[0:1], v26482; GCN1-NEXT: s_endpgm6483;6484; GCN2-LABEL: atomic_xor_i32_ret_addr64_offset:6485; GCN2: ; %bb.0: ; %entry6486; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c6487; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246488; GCN2-NEXT: s_load_dword s8, s[4:5], 0x346489; GCN2-NEXT: s_waitcnt lgkmcnt(0)6490; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 26491; GCN2-NEXT: s_add_u32 s0, s0, s46492; GCN2-NEXT: s_addc_u32 s1, s1, s56493; GCN2-NEXT: s_add_u32 s0, s0, 166494; GCN2-NEXT: s_addc_u32 s1, s1, 06495; GCN2-NEXT: v_mov_b32_e32 v0, s06496; GCN2-NEXT: v_mov_b32_e32 v1, s16497; GCN2-NEXT: v_mov_b32_e32 v2, s86498; GCN2-NEXT: flat_atomic_xor v2, v[0:1], v2 glc6499; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6500; GCN2-NEXT: buffer_wbinvl1_vol6501; GCN2-NEXT: v_mov_b32_e32 v0, s26502; GCN2-NEXT: v_mov_b32_e32 v1, s36503; GCN2-NEXT: flat_store_dword v[0:1], v26504; GCN2-NEXT: s_endpgm6505;6506; GCN3-LABEL: atomic_xor_i32_ret_addr64_offset:6507; GCN3: ; %bb.0: ; %entry6508; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c6509; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246510; GCN3-NEXT: s_load_dword s8, s[4:5], 0x346511; GCN3-NEXT: s_waitcnt lgkmcnt(0)6512; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 26513; GCN3-NEXT: s_add_u32 s0, s0, s46514; GCN3-NEXT: s_addc_u32 s1, s1, s56515; GCN3-NEXT: v_mov_b32_e32 v0, s06516; GCN3-NEXT: v_mov_b32_e32 v1, s16517; GCN3-NEXT: v_mov_b32_e32 v2, s86518; GCN3-NEXT: flat_atomic_xor v2, v[0:1], v2 offset:16 glc6519; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6520; GCN3-NEXT: buffer_wbinvl1_vol6521; GCN3-NEXT: v_mov_b32_e32 v0, s26522; GCN3-NEXT: v_mov_b32_e32 v1, s36523; GCN3-NEXT: flat_store_dword v[0:1], v26524; GCN3-NEXT: s_endpgm6525;6526; GFX11-LABEL: atomic_xor_i32_ret_addr64_offset:6527; GFX11: ; %bb.0: ; %entry6528; GFX11-NEXT: s_clause 0x26529; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c6530; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x246531; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x346532; GFX11-NEXT: s_waitcnt lgkmcnt(0)6533; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 26534; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)6535; GFX11-NEXT: s_add_u32 s0, s0, s46536; GFX11-NEXT: s_addc_u32 s1, s1, s56537; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16538; GFX11-NEXT: v_mov_b32_e32 v2, s86539; GFX11-NEXT: flat_atomic_xor_b32 v2, v[0:1], v2 offset:16 glc6540; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6541; GFX11-NEXT: buffer_gl1_inv6542; GFX11-NEXT: buffer_gl0_inv6543; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s36544; GFX11-NEXT: flat_store_b32 v[0:1], v26545; GFX11-NEXT: s_endpgm6546entry:6547 %ptr = getelementptr inbounds i32, ptr %out, i64 %index6548 %gep = getelementptr inbounds i32, ptr %ptr, i32 46549 %val = atomicrmw volatile xor ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !06550 store i32 %val, ptr %out26551 ret void6552}6553 6554define amdgpu_kernel void @atomic_xor_i32(ptr %out, i32 %in) {6555; GCN1-LABEL: atomic_xor_i32:6556; GCN1: ; %bb.0: ; %entry6557; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x96558; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb6559; GCN1-NEXT: s_waitcnt lgkmcnt(0)6560; GCN1-NEXT: v_mov_b32_e32 v0, s06561; GCN1-NEXT: v_mov_b32_e32 v1, s16562; GCN1-NEXT: v_mov_b32_e32 v2, s26563; GCN1-NEXT: flat_atomic_xor v[0:1], v26564; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6565; GCN1-NEXT: buffer_wbinvl1_vol6566; GCN1-NEXT: s_endpgm6567;6568; GCN2-LABEL: atomic_xor_i32:6569; GCN2: ; %bb.0: ; %entry6570; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x246571; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c6572; GCN2-NEXT: s_waitcnt lgkmcnt(0)6573; GCN2-NEXT: v_mov_b32_e32 v0, s06574; GCN2-NEXT: v_mov_b32_e32 v1, s16575; GCN2-NEXT: v_mov_b32_e32 v2, s26576; GCN2-NEXT: flat_atomic_xor v[0:1], v26577; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6578; GCN2-NEXT: buffer_wbinvl1_vol6579; GCN2-NEXT: s_endpgm6580;6581; GCN3-LABEL: atomic_xor_i32:6582; GCN3: ; %bb.0: ; %entry6583; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x246584; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c6585; GCN3-NEXT: s_waitcnt lgkmcnt(0)6586; GCN3-NEXT: v_mov_b32_e32 v0, s06587; GCN3-NEXT: v_mov_b32_e32 v1, s16588; GCN3-NEXT: v_mov_b32_e32 v2, s26589; GCN3-NEXT: flat_atomic_xor v[0:1], v26590; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6591; GCN3-NEXT: buffer_wbinvl1_vol6592; GCN3-NEXT: s_endpgm6593;6594; GFX11-LABEL: atomic_xor_i32:6595; GFX11: ; %bb.0: ; %entry6596; GFX11-NEXT: s_clause 0x16597; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x246598; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c6599; GFX11-NEXT: s_waitcnt lgkmcnt(0)6600; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16601; GFX11-NEXT: v_mov_b32_e32 v2, s26602; GFX11-NEXT: flat_atomic_xor_b32 v[0:1], v26603; GFX11-NEXT: s_waitcnt lgkmcnt(0)6604; GFX11-NEXT: s_waitcnt_vscnt null, 0x06605; GFX11-NEXT: buffer_gl1_inv6606; GFX11-NEXT: buffer_gl0_inv6607; GFX11-NEXT: s_endpgm6608entry:6609 %val = atomicrmw volatile xor ptr %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !06610 ret void6611}6612 6613define amdgpu_kernel void @atomic_xor_i32_ret(ptr %out, ptr %out2, i32 %in) {6614; GCN1-LABEL: atomic_xor_i32_ret:6615; GCN1: ; %bb.0: ; %entry6616; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96617; GCN1-NEXT: s_load_dword s4, s[4:5], 0xd6618; GCN1-NEXT: s_waitcnt lgkmcnt(0)6619; GCN1-NEXT: v_mov_b32_e32 v0, s06620; GCN1-NEXT: v_mov_b32_e32 v1, s16621; GCN1-NEXT: v_mov_b32_e32 v2, s46622; GCN1-NEXT: flat_atomic_xor v2, v[0:1], v2 glc6623; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6624; GCN1-NEXT: buffer_wbinvl1_vol6625; GCN1-NEXT: v_mov_b32_e32 v0, s26626; GCN1-NEXT: v_mov_b32_e32 v1, s36627; GCN1-NEXT: flat_store_dword v[0:1], v26628; GCN1-NEXT: s_endpgm6629;6630; GCN2-LABEL: atomic_xor_i32_ret:6631; GCN2: ; %bb.0: ; %entry6632; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246633; GCN2-NEXT: s_load_dword s4, s[4:5], 0x346634; GCN2-NEXT: s_waitcnt lgkmcnt(0)6635; GCN2-NEXT: v_mov_b32_e32 v0, s06636; GCN2-NEXT: v_mov_b32_e32 v1, s16637; GCN2-NEXT: v_mov_b32_e32 v2, s46638; GCN2-NEXT: flat_atomic_xor v2, v[0:1], v2 glc6639; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6640; GCN2-NEXT: buffer_wbinvl1_vol6641; GCN2-NEXT: v_mov_b32_e32 v0, s26642; GCN2-NEXT: v_mov_b32_e32 v1, s36643; GCN2-NEXT: flat_store_dword v[0:1], v26644; GCN2-NEXT: s_endpgm6645;6646; GCN3-LABEL: atomic_xor_i32_ret:6647; GCN3: ; %bb.0: ; %entry6648; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246649; GCN3-NEXT: s_load_dword s6, s[4:5], 0x346650; GCN3-NEXT: s_waitcnt lgkmcnt(0)6651; GCN3-NEXT: v_mov_b32_e32 v0, s06652; GCN3-NEXT: v_mov_b32_e32 v1, s16653; GCN3-NEXT: v_mov_b32_e32 v2, s66654; GCN3-NEXT: flat_atomic_xor v2, v[0:1], v2 glc6655; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6656; GCN3-NEXT: buffer_wbinvl1_vol6657; GCN3-NEXT: v_mov_b32_e32 v0, s26658; GCN3-NEXT: v_mov_b32_e32 v1, s36659; GCN3-NEXT: flat_store_dword v[0:1], v26660; GCN3-NEXT: s_endpgm6661;6662; GFX11-LABEL: atomic_xor_i32_ret:6663; GFX11: ; %bb.0: ; %entry6664; GFX11-NEXT: s_clause 0x16665; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x246666; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x346667; GFX11-NEXT: s_waitcnt lgkmcnt(0)6668; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16669; GFX11-NEXT: v_mov_b32_e32 v2, s46670; GFX11-NEXT: flat_atomic_xor_b32 v2, v[0:1], v2 glc6671; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6672; GFX11-NEXT: buffer_gl1_inv6673; GFX11-NEXT: buffer_gl0_inv6674; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s36675; GFX11-NEXT: flat_store_b32 v[0:1], v26676; GFX11-NEXT: s_endpgm6677entry:6678 %val = atomicrmw volatile xor ptr %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !06679 store i32 %val, ptr %out26680 ret void6681}6682 6683define amdgpu_kernel void @atomic_xor_i32_addr64(ptr %out, i32 %in, i64 %index) {6684; GCN1-LABEL: atomic_xor_i32_addr64:6685; GCN1: ; %bb.0: ; %entry6686; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd6687; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x96688; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb6689; GCN1-NEXT: s_waitcnt lgkmcnt(0)6690; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 26691; GCN1-NEXT: s_add_u32 s0, s2, s06692; GCN1-NEXT: s_addc_u32 s1, s3, s16693; GCN1-NEXT: v_mov_b32_e32 v0, s06694; GCN1-NEXT: v_mov_b32_e32 v1, s16695; GCN1-NEXT: v_mov_b32_e32 v2, s46696; GCN1-NEXT: flat_atomic_xor v[0:1], v26697; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6698; GCN1-NEXT: buffer_wbinvl1_vol6699; GCN1-NEXT: s_endpgm6700;6701; GCN2-LABEL: atomic_xor_i32_addr64:6702; GCN2: ; %bb.0: ; %entry6703; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x346704; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x246705; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c6706; GCN2-NEXT: s_waitcnt lgkmcnt(0)6707; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 26708; GCN2-NEXT: s_add_u32 s0, s2, s06709; GCN2-NEXT: s_addc_u32 s1, s3, s16710; GCN2-NEXT: v_mov_b32_e32 v0, s06711; GCN2-NEXT: v_mov_b32_e32 v1, s16712; GCN2-NEXT: v_mov_b32_e32 v2, s46713; GCN2-NEXT: flat_atomic_xor v[0:1], v26714; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6715; GCN2-NEXT: buffer_wbinvl1_vol6716; GCN2-NEXT: s_endpgm6717;6718; GCN3-LABEL: atomic_xor_i32_addr64:6719; GCN3: ; %bb.0: ; %entry6720; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x346721; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x246722; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c6723; GCN3-NEXT: s_waitcnt lgkmcnt(0)6724; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 26725; GCN3-NEXT: s_add_u32 s0, s2, s06726; GCN3-NEXT: s_addc_u32 s1, s3, s16727; GCN3-NEXT: v_mov_b32_e32 v0, s06728; GCN3-NEXT: v_mov_b32_e32 v1, s16729; GCN3-NEXT: v_mov_b32_e32 v2, s66730; GCN3-NEXT: flat_atomic_xor v[0:1], v26731; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6732; GCN3-NEXT: buffer_wbinvl1_vol6733; GCN3-NEXT: s_endpgm6734;6735; GFX11-LABEL: atomic_xor_i32_addr64:6736; GFX11: ; %bb.0: ; %entry6737; GFX11-NEXT: s_clause 0x26738; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x346739; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x246740; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c6741; GFX11-NEXT: s_waitcnt lgkmcnt(0)6742; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 26743; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)6744; GFX11-NEXT: s_add_u32 s0, s2, s06745; GFX11-NEXT: s_addc_u32 s1, s3, s16746; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16747; GFX11-NEXT: v_mov_b32_e32 v2, s46748; GFX11-NEXT: flat_atomic_xor_b32 v[0:1], v26749; GFX11-NEXT: s_waitcnt lgkmcnt(0)6750; GFX11-NEXT: s_waitcnt_vscnt null, 0x06751; GFX11-NEXT: buffer_gl1_inv6752; GFX11-NEXT: buffer_gl0_inv6753; GFX11-NEXT: s_endpgm6754entry:6755 %ptr = getelementptr inbounds i32, ptr %out, i64 %index6756 %val = atomicrmw volatile xor ptr %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !06757 ret void6758}6759 6760define amdgpu_kernel void @atomic_xor_i32_ret_addr64(ptr %out, ptr %out2, i32 %in, i64 %index) {6761; GCN1-LABEL: atomic_xor_i32_ret_addr64:6762; GCN1: ; %bb.0: ; %entry6763; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf6764; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96765; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd6766; GCN1-NEXT: s_waitcnt lgkmcnt(0)6767; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 26768; GCN1-NEXT: s_add_u32 s0, s0, s46769; GCN1-NEXT: s_addc_u32 s1, s1, s56770; GCN1-NEXT: v_mov_b32_e32 v0, s06771; GCN1-NEXT: v_mov_b32_e32 v1, s16772; GCN1-NEXT: v_mov_b32_e32 v2, s86773; GCN1-NEXT: flat_atomic_xor v2, v[0:1], v2 glc6774; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6775; GCN1-NEXT: buffer_wbinvl1_vol6776; GCN1-NEXT: v_mov_b32_e32 v0, s26777; GCN1-NEXT: v_mov_b32_e32 v1, s36778; GCN1-NEXT: flat_store_dword v[0:1], v26779; GCN1-NEXT: s_endpgm6780;6781; GCN2-LABEL: atomic_xor_i32_ret_addr64:6782; GCN2: ; %bb.0: ; %entry6783; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c6784; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246785; GCN2-NEXT: s_load_dword s8, s[4:5], 0x346786; GCN2-NEXT: s_waitcnt lgkmcnt(0)6787; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 26788; GCN2-NEXT: s_add_u32 s0, s0, s46789; GCN2-NEXT: s_addc_u32 s1, s1, s56790; GCN2-NEXT: v_mov_b32_e32 v0, s06791; GCN2-NEXT: v_mov_b32_e32 v1, s16792; GCN2-NEXT: v_mov_b32_e32 v2, s86793; GCN2-NEXT: flat_atomic_xor v2, v[0:1], v2 glc6794; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6795; GCN2-NEXT: buffer_wbinvl1_vol6796; GCN2-NEXT: v_mov_b32_e32 v0, s26797; GCN2-NEXT: v_mov_b32_e32 v1, s36798; GCN2-NEXT: flat_store_dword v[0:1], v26799; GCN2-NEXT: s_endpgm6800;6801; GCN3-LABEL: atomic_xor_i32_ret_addr64:6802; GCN3: ; %bb.0: ; %entry6803; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c6804; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246805; GCN3-NEXT: s_load_dword s8, s[4:5], 0x346806; GCN3-NEXT: s_waitcnt lgkmcnt(0)6807; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 26808; GCN3-NEXT: s_add_u32 s0, s0, s46809; GCN3-NEXT: s_addc_u32 s1, s1, s56810; GCN3-NEXT: v_mov_b32_e32 v0, s06811; GCN3-NEXT: v_mov_b32_e32 v1, s16812; GCN3-NEXT: v_mov_b32_e32 v2, s86813; GCN3-NEXT: flat_atomic_xor v2, v[0:1], v2 glc6814; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6815; GCN3-NEXT: buffer_wbinvl1_vol6816; GCN3-NEXT: v_mov_b32_e32 v0, s26817; GCN3-NEXT: v_mov_b32_e32 v1, s36818; GCN3-NEXT: flat_store_dword v[0:1], v26819; GCN3-NEXT: s_endpgm6820;6821; GFX11-LABEL: atomic_xor_i32_ret_addr64:6822; GFX11: ; %bb.0: ; %entry6823; GFX11-NEXT: s_clause 0x26824; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c6825; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x246826; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x346827; GFX11-NEXT: s_waitcnt lgkmcnt(0)6828; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 26829; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)6830; GFX11-NEXT: s_add_u32 s0, s0, s46831; GFX11-NEXT: s_addc_u32 s1, s1, s56832; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16833; GFX11-NEXT: v_mov_b32_e32 v2, s86834; GFX11-NEXT: flat_atomic_xor_b32 v2, v[0:1], v2 glc6835; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6836; GFX11-NEXT: buffer_gl1_inv6837; GFX11-NEXT: buffer_gl0_inv6838; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s36839; GFX11-NEXT: flat_store_b32 v[0:1], v26840; GFX11-NEXT: s_endpgm6841entry:6842 %ptr = getelementptr inbounds i32, ptr %out, i64 %index6843 %val = atomicrmw volatile xor ptr %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !06844 store i32 %val, ptr %out26845 ret void6846}6847 6848define amdgpu_kernel void @atomic_load_i32_offset(ptr %in, ptr %out) {6849; GCN1-LABEL: atomic_load_i32_offset:6850; GCN1: ; %bb.0: ; %entry6851; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96852; GCN1-NEXT: s_waitcnt lgkmcnt(0)6853; GCN1-NEXT: s_add_u32 s0, s0, 166854; GCN1-NEXT: s_addc_u32 s1, s1, 06855; GCN1-NEXT: v_mov_b32_e32 v0, s06856; GCN1-NEXT: v_mov_b32_e32 v1, s16857; GCN1-NEXT: flat_load_dword v2, v[0:1] glc6858; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6859; GCN1-NEXT: buffer_wbinvl1_vol6860; GCN1-NEXT: v_mov_b32_e32 v0, s26861; GCN1-NEXT: v_mov_b32_e32 v1, s36862; GCN1-NEXT: flat_store_dword v[0:1], v26863; GCN1-NEXT: s_endpgm6864;6865; GCN2-LABEL: atomic_load_i32_offset:6866; GCN2: ; %bb.0: ; %entry6867; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246868; GCN2-NEXT: s_waitcnt lgkmcnt(0)6869; GCN2-NEXT: s_add_u32 s0, s0, 166870; GCN2-NEXT: s_addc_u32 s1, s1, 06871; GCN2-NEXT: v_mov_b32_e32 v0, s06872; GCN2-NEXT: v_mov_b32_e32 v1, s16873; GCN2-NEXT: flat_load_dword v2, v[0:1] glc6874; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6875; GCN2-NEXT: buffer_wbinvl1_vol6876; GCN2-NEXT: v_mov_b32_e32 v0, s26877; GCN2-NEXT: v_mov_b32_e32 v1, s36878; GCN2-NEXT: flat_store_dword v[0:1], v26879; GCN2-NEXT: s_endpgm6880;6881; GCN3-LABEL: atomic_load_i32_offset:6882; GCN3: ; %bb.0: ; %entry6883; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246884; GCN3-NEXT: s_waitcnt lgkmcnt(0)6885; GCN3-NEXT: v_mov_b32_e32 v0, s06886; GCN3-NEXT: v_mov_b32_e32 v1, s16887; GCN3-NEXT: flat_load_dword v2, v[0:1] offset:16 glc6888; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6889; GCN3-NEXT: buffer_wbinvl1_vol6890; GCN3-NEXT: v_mov_b32_e32 v0, s26891; GCN3-NEXT: v_mov_b32_e32 v1, s36892; GCN3-NEXT: flat_store_dword v[0:1], v26893; GCN3-NEXT: s_endpgm6894;6895; GFX11-LABEL: atomic_load_i32_offset:6896; GFX11: ; %bb.0: ; %entry6897; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x246898; GFX11-NEXT: s_waitcnt lgkmcnt(0)6899; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16900; GFX11-NEXT: flat_load_b32 v2, v[0:1] offset:16 glc6901; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6902; GFX11-NEXT: buffer_gl1_inv6903; GFX11-NEXT: buffer_gl0_inv6904; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s36905; GFX11-NEXT: flat_store_b32 v[0:1], v26906; GFX11-NEXT: s_endpgm6907entry:6908 %gep = getelementptr inbounds i32, ptr %in, i32 46909 %val = load atomic i32, ptr %gep seq_cst, align 46910 store i32 %val, ptr %out6911 ret void6912}6913 6914define amdgpu_kernel void @atomic_load_i32(ptr %in, ptr %out) {6915; GCN1-LABEL: atomic_load_i32:6916; GCN1: ; %bb.0: ; %entry6917; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96918; GCN1-NEXT: s_waitcnt lgkmcnt(0)6919; GCN1-NEXT: v_mov_b32_e32 v0, s06920; GCN1-NEXT: v_mov_b32_e32 v1, s16921; GCN1-NEXT: flat_load_dword v2, v[0:1] glc6922; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6923; GCN1-NEXT: buffer_wbinvl1_vol6924; GCN1-NEXT: v_mov_b32_e32 v0, s26925; GCN1-NEXT: v_mov_b32_e32 v1, s36926; GCN1-NEXT: flat_store_dword v[0:1], v26927; GCN1-NEXT: s_endpgm6928;6929; GCN2-LABEL: atomic_load_i32:6930; GCN2: ; %bb.0: ; %entry6931; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246932; GCN2-NEXT: s_waitcnt lgkmcnt(0)6933; GCN2-NEXT: v_mov_b32_e32 v0, s06934; GCN2-NEXT: v_mov_b32_e32 v1, s16935; GCN2-NEXT: flat_load_dword v2, v[0:1] glc6936; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6937; GCN2-NEXT: buffer_wbinvl1_vol6938; GCN2-NEXT: v_mov_b32_e32 v0, s26939; GCN2-NEXT: v_mov_b32_e32 v1, s36940; GCN2-NEXT: flat_store_dword v[0:1], v26941; GCN2-NEXT: s_endpgm6942;6943; GCN3-LABEL: atomic_load_i32:6944; GCN3: ; %bb.0: ; %entry6945; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246946; GCN3-NEXT: s_waitcnt lgkmcnt(0)6947; GCN3-NEXT: v_mov_b32_e32 v0, s06948; GCN3-NEXT: v_mov_b32_e32 v1, s16949; GCN3-NEXT: flat_load_dword v2, v[0:1] glc6950; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6951; GCN3-NEXT: buffer_wbinvl1_vol6952; GCN3-NEXT: v_mov_b32_e32 v0, s26953; GCN3-NEXT: v_mov_b32_e32 v1, s36954; GCN3-NEXT: flat_store_dword v[0:1], v26955; GCN3-NEXT: s_endpgm6956;6957; GFX11-LABEL: atomic_load_i32:6958; GFX11: ; %bb.0: ; %entry6959; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x246960; GFX11-NEXT: s_waitcnt lgkmcnt(0)6961; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s16962; GFX11-NEXT: flat_load_b32 v2, v[0:1] glc6963; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6964; GFX11-NEXT: buffer_gl1_inv6965; GFX11-NEXT: buffer_gl0_inv6966; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s36967; GFX11-NEXT: flat_store_b32 v[0:1], v26968; GFX11-NEXT: s_endpgm6969entry:6970 %val = load atomic i32, ptr %in seq_cst, align 46971 store i32 %val, ptr %out6972 ret void6973}6974 6975define amdgpu_kernel void @atomic_load_i32_addr64_offset(ptr %in, ptr %out, i64 %index) {6976; GCN1-LABEL: atomic_load_i32_addr64_offset:6977; GCN1: ; %bb.0: ; %entry6978; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd6979; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96980; GCN1-NEXT: s_waitcnt lgkmcnt(0)6981; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 26982; GCN1-NEXT: s_add_u32 s0, s0, s46983; GCN1-NEXT: s_addc_u32 s1, s1, s56984; GCN1-NEXT: s_add_u32 s0, s0, 166985; GCN1-NEXT: s_addc_u32 s1, s1, 06986; GCN1-NEXT: v_mov_b32_e32 v0, s06987; GCN1-NEXT: v_mov_b32_e32 v1, s16988; GCN1-NEXT: flat_load_dword v2, v[0:1] glc6989; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6990; GCN1-NEXT: buffer_wbinvl1_vol6991; GCN1-NEXT: v_mov_b32_e32 v0, s26992; GCN1-NEXT: v_mov_b32_e32 v1, s36993; GCN1-NEXT: flat_store_dword v[0:1], v26994; GCN1-NEXT: s_endpgm6995;6996; GCN2-LABEL: atomic_load_i32_addr64_offset:6997; GCN2: ; %bb.0: ; %entry6998; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346999; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247000; GCN2-NEXT: s_waitcnt lgkmcnt(0)7001; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 27002; GCN2-NEXT: s_add_u32 s0, s0, s47003; GCN2-NEXT: s_addc_u32 s1, s1, s57004; GCN2-NEXT: s_add_u32 s0, s0, 167005; GCN2-NEXT: s_addc_u32 s1, s1, 07006; GCN2-NEXT: v_mov_b32_e32 v0, s07007; GCN2-NEXT: v_mov_b32_e32 v1, s17008; GCN2-NEXT: flat_load_dword v2, v[0:1] glc7009; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7010; GCN2-NEXT: buffer_wbinvl1_vol7011; GCN2-NEXT: v_mov_b32_e32 v0, s27012; GCN2-NEXT: v_mov_b32_e32 v1, s37013; GCN2-NEXT: flat_store_dword v[0:1], v27014; GCN2-NEXT: s_endpgm7015;7016; GCN3-LABEL: atomic_load_i32_addr64_offset:7017; GCN3: ; %bb.0: ; %entry7018; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x347019; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247020; GCN3-NEXT: s_waitcnt lgkmcnt(0)7021; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 27022; GCN3-NEXT: s_add_u32 s0, s0, s47023; GCN3-NEXT: s_addc_u32 s1, s1, s57024; GCN3-NEXT: v_mov_b32_e32 v0, s07025; GCN3-NEXT: v_mov_b32_e32 v1, s17026; GCN3-NEXT: flat_load_dword v2, v[0:1] offset:16 glc7027; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7028; GCN3-NEXT: buffer_wbinvl1_vol7029; GCN3-NEXT: v_mov_b32_e32 v0, s27030; GCN3-NEXT: v_mov_b32_e32 v1, s37031; GCN3-NEXT: flat_store_dword v[0:1], v27032; GCN3-NEXT: s_endpgm7033;7034; GFX11-LABEL: atomic_load_i32_addr64_offset:7035; GFX11: ; %bb.0: ; %entry7036; GFX11-NEXT: s_clause 0x17037; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x347038; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x247039; GFX11-NEXT: s_waitcnt lgkmcnt(0)7040; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 27041; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)7042; GFX11-NEXT: s_add_u32 s0, s0, s47043; GFX11-NEXT: s_addc_u32 s1, s1, s57044; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17045; GFX11-NEXT: flat_load_b32 v2, v[0:1] offset:16 glc7046; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7047; GFX11-NEXT: buffer_gl1_inv7048; GFX11-NEXT: buffer_gl0_inv7049; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s37050; GFX11-NEXT: flat_store_b32 v[0:1], v27051; GFX11-NEXT: s_endpgm7052entry:7053 %ptr = getelementptr inbounds i32, ptr %in, i64 %index7054 %gep = getelementptr inbounds i32, ptr %ptr, i32 47055 %val = load atomic i32, ptr %gep seq_cst, align 47056 store i32 %val, ptr %out7057 ret void7058}7059 7060define amdgpu_kernel void @atomic_load_i32_addr64(ptr %in, ptr %out, i64 %index) {7061; GCN1-LABEL: atomic_load_i32_addr64:7062; GCN1: ; %bb.0: ; %entry7063; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd7064; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x97065; GCN1-NEXT: s_waitcnt lgkmcnt(0)7066; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 27067; GCN1-NEXT: s_add_u32 s0, s0, s47068; GCN1-NEXT: s_addc_u32 s1, s1, s57069; GCN1-NEXT: v_mov_b32_e32 v0, s07070; GCN1-NEXT: v_mov_b32_e32 v1, s17071; GCN1-NEXT: flat_load_dword v2, v[0:1] glc7072; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7073; GCN1-NEXT: buffer_wbinvl1_vol7074; GCN1-NEXT: v_mov_b32_e32 v0, s27075; GCN1-NEXT: v_mov_b32_e32 v1, s37076; GCN1-NEXT: flat_store_dword v[0:1], v27077; GCN1-NEXT: s_endpgm7078;7079; GCN2-LABEL: atomic_load_i32_addr64:7080; GCN2: ; %bb.0: ; %entry7081; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x347082; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247083; GCN2-NEXT: s_waitcnt lgkmcnt(0)7084; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 27085; GCN2-NEXT: s_add_u32 s0, s0, s47086; GCN2-NEXT: s_addc_u32 s1, s1, s57087; GCN2-NEXT: v_mov_b32_e32 v0, s07088; GCN2-NEXT: v_mov_b32_e32 v1, s17089; GCN2-NEXT: flat_load_dword v2, v[0:1] glc7090; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7091; GCN2-NEXT: buffer_wbinvl1_vol7092; GCN2-NEXT: v_mov_b32_e32 v0, s27093; GCN2-NEXT: v_mov_b32_e32 v1, s37094; GCN2-NEXT: flat_store_dword v[0:1], v27095; GCN2-NEXT: s_endpgm7096;7097; GCN3-LABEL: atomic_load_i32_addr64:7098; GCN3: ; %bb.0: ; %entry7099; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x347100; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247101; GCN3-NEXT: s_waitcnt lgkmcnt(0)7102; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 27103; GCN3-NEXT: s_add_u32 s0, s0, s47104; GCN3-NEXT: s_addc_u32 s1, s1, s57105; GCN3-NEXT: v_mov_b32_e32 v0, s07106; GCN3-NEXT: v_mov_b32_e32 v1, s17107; GCN3-NEXT: flat_load_dword v2, v[0:1] glc7108; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7109; GCN3-NEXT: buffer_wbinvl1_vol7110; GCN3-NEXT: v_mov_b32_e32 v0, s27111; GCN3-NEXT: v_mov_b32_e32 v1, s37112; GCN3-NEXT: flat_store_dword v[0:1], v27113; GCN3-NEXT: s_endpgm7114;7115; GFX11-LABEL: atomic_load_i32_addr64:7116; GFX11: ; %bb.0: ; %entry7117; GFX11-NEXT: s_clause 0x17118; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x347119; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x247120; GFX11-NEXT: s_waitcnt lgkmcnt(0)7121; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 27122; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)7123; GFX11-NEXT: s_add_u32 s0, s0, s47124; GFX11-NEXT: s_addc_u32 s1, s1, s57125; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17126; GFX11-NEXT: flat_load_b32 v2, v[0:1] glc7127; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7128; GFX11-NEXT: buffer_gl1_inv7129; GFX11-NEXT: buffer_gl0_inv7130; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s37131; GFX11-NEXT: flat_store_b32 v[0:1], v27132; GFX11-NEXT: s_endpgm7133entry:7134 %ptr = getelementptr inbounds i32, ptr %in, i64 %index7135 %val = load atomic i32, ptr %ptr seq_cst, align 47136 store i32 %val, ptr %out7137 ret void7138}7139 7140define amdgpu_kernel void @atomic_store_i32_offset(i32 %in, ptr %out) {7141; GCN1-LABEL: atomic_store_i32_offset:7142; GCN1: ; %bb.0: ; %entry7143; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xb7144; GCN1-NEXT: s_load_dword s2, s[4:5], 0x97145; GCN1-NEXT: s_waitcnt lgkmcnt(0)7146; GCN1-NEXT: s_add_u32 s0, s0, 167147; GCN1-NEXT: s_addc_u32 s1, s1, 07148; GCN1-NEXT: v_mov_b32_e32 v0, s07149; GCN1-NEXT: v_mov_b32_e32 v1, s17150; GCN1-NEXT: v_mov_b32_e32 v2, s27151; GCN1-NEXT: flat_store_dword v[0:1], v27152; GCN1-NEXT: s_endpgm7153;7154; GCN2-LABEL: atomic_store_i32_offset:7155; GCN2: ; %bb.0: ; %entry7156; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c7157; GCN2-NEXT: s_load_dword s2, s[4:5], 0x247158; GCN2-NEXT: s_waitcnt lgkmcnt(0)7159; GCN2-NEXT: s_add_u32 s0, s0, 167160; GCN2-NEXT: s_addc_u32 s1, s1, 07161; GCN2-NEXT: v_mov_b32_e32 v0, s07162; GCN2-NEXT: v_mov_b32_e32 v1, s17163; GCN2-NEXT: v_mov_b32_e32 v2, s27164; GCN2-NEXT: flat_store_dword v[0:1], v27165; GCN2-NEXT: s_endpgm7166;7167; GCN3-LABEL: atomic_store_i32_offset:7168; GCN3: ; %bb.0: ; %entry7169; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c7170; GCN3-NEXT: s_load_dword s2, s[4:5], 0x247171; GCN3-NEXT: s_waitcnt lgkmcnt(0)7172; GCN3-NEXT: v_mov_b32_e32 v0, s07173; GCN3-NEXT: v_mov_b32_e32 v1, s17174; GCN3-NEXT: v_mov_b32_e32 v2, s27175; GCN3-NEXT: flat_store_dword v[0:1], v2 offset:167176; GCN3-NEXT: s_endpgm7177;7178; GFX11-LABEL: atomic_store_i32_offset:7179; GFX11: ; %bb.0: ; %entry7180; GFX11-NEXT: s_clause 0x17181; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c7182; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x247183; GFX11-NEXT: s_waitcnt lgkmcnt(0)7184; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17185; GFX11-NEXT: v_mov_b32_e32 v2, s27186; GFX11-NEXT: flat_store_b32 v[0:1], v2 offset:167187; GFX11-NEXT: s_endpgm7188entry:7189 %gep = getelementptr inbounds i32, ptr %out, i32 47190 store atomic i32 %in, ptr %gep seq_cst, align 47191 ret void7192}7193 7194define amdgpu_kernel void @atomic_store_i32(i32 %in, ptr %out) {7195; GCN1-LABEL: atomic_store_i32:7196; GCN1: ; %bb.0: ; %entry7197; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xb7198; GCN1-NEXT: s_load_dword s2, s[4:5], 0x97199; GCN1-NEXT: s_waitcnt lgkmcnt(0)7200; GCN1-NEXT: v_mov_b32_e32 v0, s07201; GCN1-NEXT: v_mov_b32_e32 v1, s17202; GCN1-NEXT: v_mov_b32_e32 v2, s27203; GCN1-NEXT: flat_store_dword v[0:1], v27204; GCN1-NEXT: s_endpgm7205;7206; GCN2-LABEL: atomic_store_i32:7207; GCN2: ; %bb.0: ; %entry7208; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c7209; GCN2-NEXT: s_load_dword s2, s[4:5], 0x247210; GCN2-NEXT: s_waitcnt lgkmcnt(0)7211; GCN2-NEXT: v_mov_b32_e32 v0, s07212; GCN2-NEXT: v_mov_b32_e32 v1, s17213; GCN2-NEXT: v_mov_b32_e32 v2, s27214; GCN2-NEXT: flat_store_dword v[0:1], v27215; GCN2-NEXT: s_endpgm7216;7217; GCN3-LABEL: atomic_store_i32:7218; GCN3: ; %bb.0: ; %entry7219; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c7220; GCN3-NEXT: s_load_dword s2, s[4:5], 0x247221; GCN3-NEXT: s_waitcnt lgkmcnt(0)7222; GCN3-NEXT: v_mov_b32_e32 v0, s07223; GCN3-NEXT: v_mov_b32_e32 v1, s17224; GCN3-NEXT: v_mov_b32_e32 v2, s27225; GCN3-NEXT: flat_store_dword v[0:1], v27226; GCN3-NEXT: s_endpgm7227;7228; GFX11-LABEL: atomic_store_i32:7229; GFX11: ; %bb.0: ; %entry7230; GFX11-NEXT: s_clause 0x17231; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c7232; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x247233; GFX11-NEXT: s_waitcnt lgkmcnt(0)7234; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17235; GFX11-NEXT: v_mov_b32_e32 v2, s27236; GFX11-NEXT: flat_store_b32 v[0:1], v27237; GFX11-NEXT: s_endpgm7238entry:7239 store atomic i32 %in, ptr %out seq_cst, align 47240 ret void7241}7242 7243define amdgpu_kernel void @atomic_store_i32_addr64_offset(i32 %in, ptr %out, i64 %index) {7244; GCN1-LABEL: atomic_store_i32_addr64_offset:7245; GCN1: ; %bb.0: ; %entry7246; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb7247; GCN1-NEXT: s_load_dword s4, s[4:5], 0x97248; GCN1-NEXT: s_waitcnt lgkmcnt(0)7249; GCN1-NEXT: s_lshl_b64 s[2:3], s[2:3], 27250; GCN1-NEXT: s_add_u32 s0, s0, s27251; GCN1-NEXT: s_addc_u32 s1, s1, s37252; GCN1-NEXT: s_add_u32 s0, s0, 167253; GCN1-NEXT: s_addc_u32 s1, s1, 07254; GCN1-NEXT: v_mov_b32_e32 v0, s07255; GCN1-NEXT: v_mov_b32_e32 v1, s17256; GCN1-NEXT: v_mov_b32_e32 v2, s47257; GCN1-NEXT: flat_store_dword v[0:1], v27258; GCN1-NEXT: s_endpgm7259;7260; GCN2-LABEL: atomic_store_i32_addr64_offset:7261; GCN2: ; %bb.0: ; %entry7262; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c7263; GCN2-NEXT: s_load_dword s4, s[4:5], 0x247264; GCN2-NEXT: s_waitcnt lgkmcnt(0)7265; GCN2-NEXT: s_lshl_b64 s[2:3], s[2:3], 27266; GCN2-NEXT: s_add_u32 s0, s0, s27267; GCN2-NEXT: s_addc_u32 s1, s1, s37268; GCN2-NEXT: s_add_u32 s0, s0, 167269; GCN2-NEXT: s_addc_u32 s1, s1, 07270; GCN2-NEXT: v_mov_b32_e32 v0, s07271; GCN2-NEXT: v_mov_b32_e32 v1, s17272; GCN2-NEXT: v_mov_b32_e32 v2, s47273; GCN2-NEXT: flat_store_dword v[0:1], v27274; GCN2-NEXT: s_endpgm7275;7276; GCN3-LABEL: atomic_store_i32_addr64_offset:7277; GCN3: ; %bb.0: ; %entry7278; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c7279; GCN3-NEXT: s_load_dword s6, s[4:5], 0x247280; GCN3-NEXT: s_waitcnt lgkmcnt(0)7281; GCN3-NEXT: s_lshl_b64 s[2:3], s[2:3], 27282; GCN3-NEXT: s_add_u32 s0, s0, s27283; GCN3-NEXT: s_addc_u32 s1, s1, s37284; GCN3-NEXT: v_mov_b32_e32 v0, s07285; GCN3-NEXT: v_mov_b32_e32 v1, s17286; GCN3-NEXT: v_mov_b32_e32 v2, s67287; GCN3-NEXT: flat_store_dword v[0:1], v2 offset:167288; GCN3-NEXT: s_endpgm7289;7290; GFX11-LABEL: atomic_store_i32_addr64_offset:7291; GFX11: ; %bb.0: ; %entry7292; GFX11-NEXT: s_clause 0x17293; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c7294; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x247295; GFX11-NEXT: s_waitcnt lgkmcnt(0)7296; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], 27297; GFX11-NEXT: v_mov_b32_e32 v2, s47298; GFX11-NEXT: s_add_u32 s0, s0, s27299; GFX11-NEXT: s_addc_u32 s1, s1, s37300; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7301; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17302; GFX11-NEXT: flat_store_b32 v[0:1], v2 offset:167303; GFX11-NEXT: s_endpgm7304entry:7305 %ptr = getelementptr inbounds i32, ptr %out, i64 %index7306 %gep = getelementptr inbounds i32, ptr %ptr, i32 47307 store atomic i32 %in, ptr %gep seq_cst, align 47308 ret void7309}7310 7311define amdgpu_kernel void @atomic_store_i32_addr64(i32 %in, ptr %out, i64 %index) {7312; GCN1-LABEL: atomic_store_i32_addr64:7313; GCN1: ; %bb.0: ; %entry7314; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb7315; GCN1-NEXT: s_load_dword s4, s[4:5], 0x97316; GCN1-NEXT: s_waitcnt lgkmcnt(0)7317; GCN1-NEXT: s_lshl_b64 s[2:3], s[2:3], 27318; GCN1-NEXT: s_add_u32 s0, s0, s27319; GCN1-NEXT: s_addc_u32 s1, s1, s37320; GCN1-NEXT: v_mov_b32_e32 v0, s07321; GCN1-NEXT: v_mov_b32_e32 v1, s17322; GCN1-NEXT: v_mov_b32_e32 v2, s47323; GCN1-NEXT: flat_store_dword v[0:1], v27324; GCN1-NEXT: s_endpgm7325;7326; GCN2-LABEL: atomic_store_i32_addr64:7327; GCN2: ; %bb.0: ; %entry7328; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c7329; GCN2-NEXT: s_load_dword s4, s[4:5], 0x247330; GCN2-NEXT: s_waitcnt lgkmcnt(0)7331; GCN2-NEXT: s_lshl_b64 s[2:3], s[2:3], 27332; GCN2-NEXT: s_add_u32 s0, s0, s27333; GCN2-NEXT: s_addc_u32 s1, s1, s37334; GCN2-NEXT: v_mov_b32_e32 v0, s07335; GCN2-NEXT: v_mov_b32_e32 v1, s17336; GCN2-NEXT: v_mov_b32_e32 v2, s47337; GCN2-NEXT: flat_store_dword v[0:1], v27338; GCN2-NEXT: s_endpgm7339;7340; GCN3-LABEL: atomic_store_i32_addr64:7341; GCN3: ; %bb.0: ; %entry7342; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c7343; GCN3-NEXT: s_load_dword s6, s[4:5], 0x247344; GCN3-NEXT: s_waitcnt lgkmcnt(0)7345; GCN3-NEXT: s_lshl_b64 s[2:3], s[2:3], 27346; GCN3-NEXT: s_add_u32 s0, s0, s27347; GCN3-NEXT: s_addc_u32 s1, s1, s37348; GCN3-NEXT: v_mov_b32_e32 v0, s07349; GCN3-NEXT: v_mov_b32_e32 v1, s17350; GCN3-NEXT: v_mov_b32_e32 v2, s67351; GCN3-NEXT: flat_store_dword v[0:1], v27352; GCN3-NEXT: s_endpgm7353;7354; GFX11-LABEL: atomic_store_i32_addr64:7355; GFX11: ; %bb.0: ; %entry7356; GFX11-NEXT: s_clause 0x17357; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c7358; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x247359; GFX11-NEXT: s_waitcnt lgkmcnt(0)7360; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], 27361; GFX11-NEXT: v_mov_b32_e32 v2, s47362; GFX11-NEXT: s_add_u32 s0, s0, s27363; GFX11-NEXT: s_addc_u32 s1, s1, s37364; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7365; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17366; GFX11-NEXT: flat_store_b32 v[0:1], v27367; GFX11-NEXT: s_endpgm7368entry:7369 %ptr = getelementptr inbounds i32, ptr %out, i64 %index7370 store atomic i32 %in, ptr %ptr seq_cst, align 47371 ret void7372}7373 7374define amdgpu_kernel void @atomic_load_f32_offset(ptr %in, ptr %out) {7375; GCN1-LABEL: atomic_load_f32_offset:7376; GCN1: ; %bb.0: ; %entry7377; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x97378; GCN1-NEXT: s_waitcnt lgkmcnt(0)7379; GCN1-NEXT: s_add_u32 s0, s0, 167380; GCN1-NEXT: s_addc_u32 s1, s1, 07381; GCN1-NEXT: v_mov_b32_e32 v0, s07382; GCN1-NEXT: v_mov_b32_e32 v1, s17383; GCN1-NEXT: flat_load_dword v2, v[0:1] glc7384; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7385; GCN1-NEXT: buffer_wbinvl1_vol7386; GCN1-NEXT: v_mov_b32_e32 v0, s27387; GCN1-NEXT: v_mov_b32_e32 v1, s37388; GCN1-NEXT: flat_store_dword v[0:1], v27389; GCN1-NEXT: s_endpgm7390;7391; GCN2-LABEL: atomic_load_f32_offset:7392; GCN2: ; %bb.0: ; %entry7393; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247394; GCN2-NEXT: s_waitcnt lgkmcnt(0)7395; GCN2-NEXT: s_add_u32 s0, s0, 167396; GCN2-NEXT: s_addc_u32 s1, s1, 07397; GCN2-NEXT: v_mov_b32_e32 v0, s07398; GCN2-NEXT: v_mov_b32_e32 v1, s17399; GCN2-NEXT: flat_load_dword v2, v[0:1] glc7400; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7401; GCN2-NEXT: buffer_wbinvl1_vol7402; GCN2-NEXT: v_mov_b32_e32 v0, s27403; GCN2-NEXT: v_mov_b32_e32 v1, s37404; GCN2-NEXT: flat_store_dword v[0:1], v27405; GCN2-NEXT: s_endpgm7406;7407; GCN3-LABEL: atomic_load_f32_offset:7408; GCN3: ; %bb.0: ; %entry7409; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247410; GCN3-NEXT: s_waitcnt lgkmcnt(0)7411; GCN3-NEXT: v_mov_b32_e32 v0, s07412; GCN3-NEXT: v_mov_b32_e32 v1, s17413; GCN3-NEXT: flat_load_dword v2, v[0:1] offset:16 glc7414; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7415; GCN3-NEXT: buffer_wbinvl1_vol7416; GCN3-NEXT: v_mov_b32_e32 v0, s27417; GCN3-NEXT: v_mov_b32_e32 v1, s37418; GCN3-NEXT: flat_store_dword v[0:1], v27419; GCN3-NEXT: s_endpgm7420;7421; GFX11-LABEL: atomic_load_f32_offset:7422; GFX11: ; %bb.0: ; %entry7423; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x247424; GFX11-NEXT: s_waitcnt lgkmcnt(0)7425; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17426; GFX11-NEXT: flat_load_b32 v2, v[0:1] offset:16 glc7427; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7428; GFX11-NEXT: buffer_gl1_inv7429; GFX11-NEXT: buffer_gl0_inv7430; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s37431; GFX11-NEXT: flat_store_b32 v[0:1], v27432; GFX11-NEXT: s_endpgm7433entry:7434 %gep = getelementptr inbounds float, ptr %in, i32 47435 %val = load atomic float, ptr %gep seq_cst, align 47436 store float %val, ptr %out7437 ret void7438}7439 7440define amdgpu_kernel void @atomic_load_f32(ptr %in, ptr %out) {7441; GCN1-LABEL: atomic_load_f32:7442; GCN1: ; %bb.0: ; %entry7443; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x97444; GCN1-NEXT: s_waitcnt lgkmcnt(0)7445; GCN1-NEXT: v_mov_b32_e32 v0, s07446; GCN1-NEXT: v_mov_b32_e32 v1, s17447; GCN1-NEXT: flat_load_dword v2, v[0:1] glc7448; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7449; GCN1-NEXT: buffer_wbinvl1_vol7450; GCN1-NEXT: v_mov_b32_e32 v0, s27451; GCN1-NEXT: v_mov_b32_e32 v1, s37452; GCN1-NEXT: flat_store_dword v[0:1], v27453; GCN1-NEXT: s_endpgm7454;7455; GCN2-LABEL: atomic_load_f32:7456; GCN2: ; %bb.0: ; %entry7457; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247458; GCN2-NEXT: s_waitcnt lgkmcnt(0)7459; GCN2-NEXT: v_mov_b32_e32 v0, s07460; GCN2-NEXT: v_mov_b32_e32 v1, s17461; GCN2-NEXT: flat_load_dword v2, v[0:1] glc7462; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7463; GCN2-NEXT: buffer_wbinvl1_vol7464; GCN2-NEXT: v_mov_b32_e32 v0, s27465; GCN2-NEXT: v_mov_b32_e32 v1, s37466; GCN2-NEXT: flat_store_dword v[0:1], v27467; GCN2-NEXT: s_endpgm7468;7469; GCN3-LABEL: atomic_load_f32:7470; GCN3: ; %bb.0: ; %entry7471; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247472; GCN3-NEXT: s_waitcnt lgkmcnt(0)7473; GCN3-NEXT: v_mov_b32_e32 v0, s07474; GCN3-NEXT: v_mov_b32_e32 v1, s17475; GCN3-NEXT: flat_load_dword v2, v[0:1] glc7476; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7477; GCN3-NEXT: buffer_wbinvl1_vol7478; GCN3-NEXT: v_mov_b32_e32 v0, s27479; GCN3-NEXT: v_mov_b32_e32 v1, s37480; GCN3-NEXT: flat_store_dword v[0:1], v27481; GCN3-NEXT: s_endpgm7482;7483; GFX11-LABEL: atomic_load_f32:7484; GFX11: ; %bb.0: ; %entry7485; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x247486; GFX11-NEXT: s_waitcnt lgkmcnt(0)7487; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17488; GFX11-NEXT: flat_load_b32 v2, v[0:1] glc7489; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7490; GFX11-NEXT: buffer_gl1_inv7491; GFX11-NEXT: buffer_gl0_inv7492; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s37493; GFX11-NEXT: flat_store_b32 v[0:1], v27494; GFX11-NEXT: s_endpgm7495entry:7496 %val = load atomic float, ptr %in seq_cst, align 47497 store float %val, ptr %out7498 ret void7499}7500 7501define amdgpu_kernel void @atomic_load_f32_addr64_offset(ptr %in, ptr %out, i64 %index) {7502; GCN1-LABEL: atomic_load_f32_addr64_offset:7503; GCN1: ; %bb.0: ; %entry7504; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd7505; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x97506; GCN1-NEXT: s_waitcnt lgkmcnt(0)7507; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 27508; GCN1-NEXT: s_add_u32 s0, s0, s47509; GCN1-NEXT: s_addc_u32 s1, s1, s57510; GCN1-NEXT: s_add_u32 s0, s0, 167511; GCN1-NEXT: s_addc_u32 s1, s1, 07512; GCN1-NEXT: v_mov_b32_e32 v0, s07513; GCN1-NEXT: v_mov_b32_e32 v1, s17514; GCN1-NEXT: flat_load_dword v2, v[0:1] glc7515; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7516; GCN1-NEXT: buffer_wbinvl1_vol7517; GCN1-NEXT: v_mov_b32_e32 v0, s27518; GCN1-NEXT: v_mov_b32_e32 v1, s37519; GCN1-NEXT: flat_store_dword v[0:1], v27520; GCN1-NEXT: s_endpgm7521;7522; GCN2-LABEL: atomic_load_f32_addr64_offset:7523; GCN2: ; %bb.0: ; %entry7524; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x347525; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247526; GCN2-NEXT: s_waitcnt lgkmcnt(0)7527; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 27528; GCN2-NEXT: s_add_u32 s0, s0, s47529; GCN2-NEXT: s_addc_u32 s1, s1, s57530; GCN2-NEXT: s_add_u32 s0, s0, 167531; GCN2-NEXT: s_addc_u32 s1, s1, 07532; GCN2-NEXT: v_mov_b32_e32 v0, s07533; GCN2-NEXT: v_mov_b32_e32 v1, s17534; GCN2-NEXT: flat_load_dword v2, v[0:1] glc7535; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7536; GCN2-NEXT: buffer_wbinvl1_vol7537; GCN2-NEXT: v_mov_b32_e32 v0, s27538; GCN2-NEXT: v_mov_b32_e32 v1, s37539; GCN2-NEXT: flat_store_dword v[0:1], v27540; GCN2-NEXT: s_endpgm7541;7542; GCN3-LABEL: atomic_load_f32_addr64_offset:7543; GCN3: ; %bb.0: ; %entry7544; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x347545; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247546; GCN3-NEXT: s_waitcnt lgkmcnt(0)7547; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 27548; GCN3-NEXT: s_add_u32 s0, s0, s47549; GCN3-NEXT: s_addc_u32 s1, s1, s57550; GCN3-NEXT: v_mov_b32_e32 v0, s07551; GCN3-NEXT: v_mov_b32_e32 v1, s17552; GCN3-NEXT: flat_load_dword v2, v[0:1] offset:16 glc7553; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7554; GCN3-NEXT: buffer_wbinvl1_vol7555; GCN3-NEXT: v_mov_b32_e32 v0, s27556; GCN3-NEXT: v_mov_b32_e32 v1, s37557; GCN3-NEXT: flat_store_dword v[0:1], v27558; GCN3-NEXT: s_endpgm7559;7560; GFX11-LABEL: atomic_load_f32_addr64_offset:7561; GFX11: ; %bb.0: ; %entry7562; GFX11-NEXT: s_clause 0x17563; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x347564; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x247565; GFX11-NEXT: s_waitcnt lgkmcnt(0)7566; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 27567; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)7568; GFX11-NEXT: s_add_u32 s0, s0, s47569; GFX11-NEXT: s_addc_u32 s1, s1, s57570; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17571; GFX11-NEXT: flat_load_b32 v2, v[0:1] offset:16 glc7572; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7573; GFX11-NEXT: buffer_gl1_inv7574; GFX11-NEXT: buffer_gl0_inv7575; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s37576; GFX11-NEXT: flat_store_b32 v[0:1], v27577; GFX11-NEXT: s_endpgm7578entry:7579 %ptr = getelementptr inbounds float, ptr %in, i64 %index7580 %gep = getelementptr inbounds float, ptr %ptr, i32 47581 %val = load atomic float, ptr %gep seq_cst, align 47582 store float %val, ptr %out7583 ret void7584}7585 7586define amdgpu_kernel void @atomic_load_f32_addr64(ptr %in, ptr %out, i64 %index) {7587; GCN1-LABEL: atomic_load_f32_addr64:7588; GCN1: ; %bb.0: ; %entry7589; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd7590; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x97591; GCN1-NEXT: s_waitcnt lgkmcnt(0)7592; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 27593; GCN1-NEXT: s_add_u32 s0, s0, s47594; GCN1-NEXT: s_addc_u32 s1, s1, s57595; GCN1-NEXT: v_mov_b32_e32 v0, s07596; GCN1-NEXT: v_mov_b32_e32 v1, s17597; GCN1-NEXT: flat_load_dword v2, v[0:1] glc7598; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7599; GCN1-NEXT: buffer_wbinvl1_vol7600; GCN1-NEXT: v_mov_b32_e32 v0, s27601; GCN1-NEXT: v_mov_b32_e32 v1, s37602; GCN1-NEXT: flat_store_dword v[0:1], v27603; GCN1-NEXT: s_endpgm7604;7605; GCN2-LABEL: atomic_load_f32_addr64:7606; GCN2: ; %bb.0: ; %entry7607; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x347608; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247609; GCN2-NEXT: s_waitcnt lgkmcnt(0)7610; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 27611; GCN2-NEXT: s_add_u32 s0, s0, s47612; GCN2-NEXT: s_addc_u32 s1, s1, s57613; GCN2-NEXT: v_mov_b32_e32 v0, s07614; GCN2-NEXT: v_mov_b32_e32 v1, s17615; GCN2-NEXT: flat_load_dword v2, v[0:1] glc7616; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7617; GCN2-NEXT: buffer_wbinvl1_vol7618; GCN2-NEXT: v_mov_b32_e32 v0, s27619; GCN2-NEXT: v_mov_b32_e32 v1, s37620; GCN2-NEXT: flat_store_dword v[0:1], v27621; GCN2-NEXT: s_endpgm7622;7623; GCN3-LABEL: atomic_load_f32_addr64:7624; GCN3: ; %bb.0: ; %entry7625; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x347626; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247627; GCN3-NEXT: s_waitcnt lgkmcnt(0)7628; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 27629; GCN3-NEXT: s_add_u32 s0, s0, s47630; GCN3-NEXT: s_addc_u32 s1, s1, s57631; GCN3-NEXT: v_mov_b32_e32 v0, s07632; GCN3-NEXT: v_mov_b32_e32 v1, s17633; GCN3-NEXT: flat_load_dword v2, v[0:1] glc7634; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7635; GCN3-NEXT: buffer_wbinvl1_vol7636; GCN3-NEXT: v_mov_b32_e32 v0, s27637; GCN3-NEXT: v_mov_b32_e32 v1, s37638; GCN3-NEXT: flat_store_dword v[0:1], v27639; GCN3-NEXT: s_endpgm7640;7641; GFX11-LABEL: atomic_load_f32_addr64:7642; GFX11: ; %bb.0: ; %entry7643; GFX11-NEXT: s_clause 0x17644; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x347645; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x247646; GFX11-NEXT: s_waitcnt lgkmcnt(0)7647; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 27648; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)7649; GFX11-NEXT: s_add_u32 s0, s0, s47650; GFX11-NEXT: s_addc_u32 s1, s1, s57651; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17652; GFX11-NEXT: flat_load_b32 v2, v[0:1] glc7653; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7654; GFX11-NEXT: buffer_gl1_inv7655; GFX11-NEXT: buffer_gl0_inv7656; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s37657; GFX11-NEXT: flat_store_b32 v[0:1], v27658; GFX11-NEXT: s_endpgm7659entry:7660 %ptr = getelementptr inbounds float, ptr %in, i64 %index7661 %val = load atomic float, ptr %ptr seq_cst, align 47662 store float %val, ptr %out7663 ret void7664}7665 7666define amdgpu_kernel void @atomic_store_f32_offset(float %in, ptr %out) {7667; GCN1-LABEL: atomic_store_f32_offset:7668; GCN1: ; %bb.0: ; %entry7669; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xb7670; GCN1-NEXT: s_load_dword s2, s[4:5], 0x97671; GCN1-NEXT: s_waitcnt lgkmcnt(0)7672; GCN1-NEXT: s_add_u32 s0, s0, 167673; GCN1-NEXT: s_addc_u32 s1, s1, 07674; GCN1-NEXT: v_mov_b32_e32 v0, s07675; GCN1-NEXT: v_mov_b32_e32 v1, s17676; GCN1-NEXT: v_mov_b32_e32 v2, s27677; GCN1-NEXT: flat_store_dword v[0:1], v27678; GCN1-NEXT: s_endpgm7679;7680; GCN2-LABEL: atomic_store_f32_offset:7681; GCN2: ; %bb.0: ; %entry7682; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c7683; GCN2-NEXT: s_load_dword s2, s[4:5], 0x247684; GCN2-NEXT: s_waitcnt lgkmcnt(0)7685; GCN2-NEXT: s_add_u32 s0, s0, 167686; GCN2-NEXT: s_addc_u32 s1, s1, 07687; GCN2-NEXT: v_mov_b32_e32 v0, s07688; GCN2-NEXT: v_mov_b32_e32 v1, s17689; GCN2-NEXT: v_mov_b32_e32 v2, s27690; GCN2-NEXT: flat_store_dword v[0:1], v27691; GCN2-NEXT: s_endpgm7692;7693; GCN3-LABEL: atomic_store_f32_offset:7694; GCN3: ; %bb.0: ; %entry7695; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c7696; GCN3-NEXT: s_load_dword s2, s[4:5], 0x247697; GCN3-NEXT: s_waitcnt lgkmcnt(0)7698; GCN3-NEXT: v_mov_b32_e32 v0, s07699; GCN3-NEXT: v_mov_b32_e32 v1, s17700; GCN3-NEXT: v_mov_b32_e32 v2, s27701; GCN3-NEXT: flat_store_dword v[0:1], v2 offset:167702; GCN3-NEXT: s_endpgm7703;7704; GFX11-LABEL: atomic_store_f32_offset:7705; GFX11: ; %bb.0: ; %entry7706; GFX11-NEXT: s_clause 0x17707; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c7708; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x247709; GFX11-NEXT: s_waitcnt lgkmcnt(0)7710; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17711; GFX11-NEXT: v_mov_b32_e32 v2, s27712; GFX11-NEXT: flat_store_b32 v[0:1], v2 offset:167713; GFX11-NEXT: s_endpgm7714entry:7715 %gep = getelementptr inbounds float, ptr %out, i32 47716 store atomic float %in, ptr %gep seq_cst, align 47717 ret void7718}7719 7720define amdgpu_kernel void @atomic_store_f32(float %in, ptr %out) {7721; GCN1-LABEL: atomic_store_f32:7722; GCN1: ; %bb.0: ; %entry7723; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xb7724; GCN1-NEXT: s_load_dword s2, s[4:5], 0x97725; GCN1-NEXT: s_waitcnt lgkmcnt(0)7726; GCN1-NEXT: v_mov_b32_e32 v0, s07727; GCN1-NEXT: v_mov_b32_e32 v1, s17728; GCN1-NEXT: v_mov_b32_e32 v2, s27729; GCN1-NEXT: flat_store_dword v[0:1], v27730; GCN1-NEXT: s_endpgm7731;7732; GCN2-LABEL: atomic_store_f32:7733; GCN2: ; %bb.0: ; %entry7734; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c7735; GCN2-NEXT: s_load_dword s2, s[4:5], 0x247736; GCN2-NEXT: s_waitcnt lgkmcnt(0)7737; GCN2-NEXT: v_mov_b32_e32 v0, s07738; GCN2-NEXT: v_mov_b32_e32 v1, s17739; GCN2-NEXT: v_mov_b32_e32 v2, s27740; GCN2-NEXT: flat_store_dword v[0:1], v27741; GCN2-NEXT: s_endpgm7742;7743; GCN3-LABEL: atomic_store_f32:7744; GCN3: ; %bb.0: ; %entry7745; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c7746; GCN3-NEXT: s_load_dword s2, s[4:5], 0x247747; GCN3-NEXT: s_waitcnt lgkmcnt(0)7748; GCN3-NEXT: v_mov_b32_e32 v0, s07749; GCN3-NEXT: v_mov_b32_e32 v1, s17750; GCN3-NEXT: v_mov_b32_e32 v2, s27751; GCN3-NEXT: flat_store_dword v[0:1], v27752; GCN3-NEXT: s_endpgm7753;7754; GFX11-LABEL: atomic_store_f32:7755; GFX11: ; %bb.0: ; %entry7756; GFX11-NEXT: s_clause 0x17757; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c7758; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x247759; GFX11-NEXT: s_waitcnt lgkmcnt(0)7760; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17761; GFX11-NEXT: v_mov_b32_e32 v2, s27762; GFX11-NEXT: flat_store_b32 v[0:1], v27763; GFX11-NEXT: s_endpgm7764entry:7765 store atomic float %in, ptr %out seq_cst, align 47766 ret void7767}7768 7769define amdgpu_kernel void @atomic_store_f32_addr64_offset(float %in, ptr %out, i64 %index) {7770; GCN1-LABEL: atomic_store_f32_addr64_offset:7771; GCN1: ; %bb.0: ; %entry7772; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb7773; GCN1-NEXT: s_load_dword s4, s[4:5], 0x97774; GCN1-NEXT: s_waitcnt lgkmcnt(0)7775; GCN1-NEXT: s_lshl_b64 s[2:3], s[2:3], 27776; GCN1-NEXT: s_add_u32 s0, s0, s27777; GCN1-NEXT: s_addc_u32 s1, s1, s37778; GCN1-NEXT: s_add_u32 s0, s0, 167779; GCN1-NEXT: s_addc_u32 s1, s1, 07780; GCN1-NEXT: v_mov_b32_e32 v0, s07781; GCN1-NEXT: v_mov_b32_e32 v1, s17782; GCN1-NEXT: v_mov_b32_e32 v2, s47783; GCN1-NEXT: flat_store_dword v[0:1], v27784; GCN1-NEXT: s_endpgm7785;7786; GCN2-LABEL: atomic_store_f32_addr64_offset:7787; GCN2: ; %bb.0: ; %entry7788; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c7789; GCN2-NEXT: s_load_dword s4, s[4:5], 0x247790; GCN2-NEXT: s_waitcnt lgkmcnt(0)7791; GCN2-NEXT: s_lshl_b64 s[2:3], s[2:3], 27792; GCN2-NEXT: s_add_u32 s0, s0, s27793; GCN2-NEXT: s_addc_u32 s1, s1, s37794; GCN2-NEXT: s_add_u32 s0, s0, 167795; GCN2-NEXT: s_addc_u32 s1, s1, 07796; GCN2-NEXT: v_mov_b32_e32 v0, s07797; GCN2-NEXT: v_mov_b32_e32 v1, s17798; GCN2-NEXT: v_mov_b32_e32 v2, s47799; GCN2-NEXT: flat_store_dword v[0:1], v27800; GCN2-NEXT: s_endpgm7801;7802; GCN3-LABEL: atomic_store_f32_addr64_offset:7803; GCN3: ; %bb.0: ; %entry7804; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c7805; GCN3-NEXT: s_load_dword s6, s[4:5], 0x247806; GCN3-NEXT: s_waitcnt lgkmcnt(0)7807; GCN3-NEXT: s_lshl_b64 s[2:3], s[2:3], 27808; GCN3-NEXT: s_add_u32 s0, s0, s27809; GCN3-NEXT: s_addc_u32 s1, s1, s37810; GCN3-NEXT: v_mov_b32_e32 v0, s07811; GCN3-NEXT: v_mov_b32_e32 v1, s17812; GCN3-NEXT: v_mov_b32_e32 v2, s67813; GCN3-NEXT: flat_store_dword v[0:1], v2 offset:167814; GCN3-NEXT: s_endpgm7815;7816; GFX11-LABEL: atomic_store_f32_addr64_offset:7817; GFX11: ; %bb.0: ; %entry7818; GFX11-NEXT: s_clause 0x17819; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c7820; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x247821; GFX11-NEXT: s_waitcnt lgkmcnt(0)7822; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], 27823; GFX11-NEXT: v_mov_b32_e32 v2, s47824; GFX11-NEXT: s_add_u32 s0, s0, s27825; GFX11-NEXT: s_addc_u32 s1, s1, s37826; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7827; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17828; GFX11-NEXT: flat_store_b32 v[0:1], v2 offset:167829; GFX11-NEXT: s_endpgm7830entry:7831 %ptr = getelementptr inbounds float, ptr %out, i64 %index7832 %gep = getelementptr inbounds float, ptr %ptr, i32 47833 store atomic float %in, ptr %gep seq_cst, align 47834 ret void7835}7836 7837define amdgpu_kernel void @atomic_store_f32_addr64(float %in, ptr %out, i64 %index) {7838; GCN1-LABEL: atomic_store_f32_addr64:7839; GCN1: ; %bb.0: ; %entry7840; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb7841; GCN1-NEXT: s_load_dword s4, s[4:5], 0x97842; GCN1-NEXT: s_waitcnt lgkmcnt(0)7843; GCN1-NEXT: s_lshl_b64 s[2:3], s[2:3], 27844; GCN1-NEXT: s_add_u32 s0, s0, s27845; GCN1-NEXT: s_addc_u32 s1, s1, s37846; GCN1-NEXT: v_mov_b32_e32 v0, s07847; GCN1-NEXT: v_mov_b32_e32 v1, s17848; GCN1-NEXT: v_mov_b32_e32 v2, s47849; GCN1-NEXT: flat_store_dword v[0:1], v27850; GCN1-NEXT: s_endpgm7851;7852; GCN2-LABEL: atomic_store_f32_addr64:7853; GCN2: ; %bb.0: ; %entry7854; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c7855; GCN2-NEXT: s_load_dword s4, s[4:5], 0x247856; GCN2-NEXT: s_waitcnt lgkmcnt(0)7857; GCN2-NEXT: s_lshl_b64 s[2:3], s[2:3], 27858; GCN2-NEXT: s_add_u32 s0, s0, s27859; GCN2-NEXT: s_addc_u32 s1, s1, s37860; GCN2-NEXT: v_mov_b32_e32 v0, s07861; GCN2-NEXT: v_mov_b32_e32 v1, s17862; GCN2-NEXT: v_mov_b32_e32 v2, s47863; GCN2-NEXT: flat_store_dword v[0:1], v27864; GCN2-NEXT: s_endpgm7865;7866; GCN3-LABEL: atomic_store_f32_addr64:7867; GCN3: ; %bb.0: ; %entry7868; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c7869; GCN3-NEXT: s_load_dword s6, s[4:5], 0x247870; GCN3-NEXT: s_waitcnt lgkmcnt(0)7871; GCN3-NEXT: s_lshl_b64 s[2:3], s[2:3], 27872; GCN3-NEXT: s_add_u32 s0, s0, s27873; GCN3-NEXT: s_addc_u32 s1, s1, s37874; GCN3-NEXT: v_mov_b32_e32 v0, s07875; GCN3-NEXT: v_mov_b32_e32 v1, s17876; GCN3-NEXT: v_mov_b32_e32 v2, s67877; GCN3-NEXT: flat_store_dword v[0:1], v27878; GCN3-NEXT: s_endpgm7879;7880; GFX11-LABEL: atomic_store_f32_addr64:7881; GFX11: ; %bb.0: ; %entry7882; GFX11-NEXT: s_clause 0x17883; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c7884; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x247885; GFX11-NEXT: s_waitcnt lgkmcnt(0)7886; GFX11-NEXT: s_lshl_b64 s[2:3], s[2:3], 27887; GFX11-NEXT: v_mov_b32_e32 v2, s47888; GFX11-NEXT: s_add_u32 s0, s0, s27889; GFX11-NEXT: s_addc_u32 s1, s1, s37890; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7891; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17892; GFX11-NEXT: flat_store_b32 v[0:1], v27893; GFX11-NEXT: s_endpgm7894entry:7895 %ptr = getelementptr inbounds float, ptr %out, i64 %index7896 store atomic float %in, ptr %ptr seq_cst, align 47897 ret void7898}7899 7900define amdgpu_kernel void @atomic_load_i8_offset(ptr %in, ptr %out) {7901; GCN1-LABEL: atomic_load_i8_offset:7902; GCN1: ; %bb.0: ; %entry7903; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x97904; GCN1-NEXT: s_waitcnt lgkmcnt(0)7905; GCN1-NEXT: s_add_u32 s0, s0, 167906; GCN1-NEXT: s_addc_u32 s1, s1, 07907; GCN1-NEXT: v_mov_b32_e32 v0, s07908; GCN1-NEXT: v_mov_b32_e32 v1, s17909; GCN1-NEXT: flat_load_ubyte v2, v[0:1] glc7910; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7911; GCN1-NEXT: buffer_wbinvl1_vol7912; GCN1-NEXT: v_mov_b32_e32 v0, s27913; GCN1-NEXT: v_mov_b32_e32 v1, s37914; GCN1-NEXT: flat_store_byte v[0:1], v27915; GCN1-NEXT: s_endpgm7916;7917; GCN2-LABEL: atomic_load_i8_offset:7918; GCN2: ; %bb.0: ; %entry7919; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247920; GCN2-NEXT: s_waitcnt lgkmcnt(0)7921; GCN2-NEXT: s_add_u32 s0, s0, 167922; GCN2-NEXT: s_addc_u32 s1, s1, 07923; GCN2-NEXT: v_mov_b32_e32 v0, s07924; GCN2-NEXT: v_mov_b32_e32 v1, s17925; GCN2-NEXT: flat_load_ubyte v2, v[0:1] glc7926; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7927; GCN2-NEXT: buffer_wbinvl1_vol7928; GCN2-NEXT: v_mov_b32_e32 v0, s27929; GCN2-NEXT: v_mov_b32_e32 v1, s37930; GCN2-NEXT: flat_store_byte v[0:1], v27931; GCN2-NEXT: s_endpgm7932;7933; GCN3-LABEL: atomic_load_i8_offset:7934; GCN3: ; %bb.0: ; %entry7935; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247936; GCN3-NEXT: s_waitcnt lgkmcnt(0)7937; GCN3-NEXT: v_mov_b32_e32 v0, s07938; GCN3-NEXT: v_mov_b32_e32 v1, s17939; GCN3-NEXT: flat_load_ubyte v2, v[0:1] offset:16 glc7940; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7941; GCN3-NEXT: buffer_wbinvl1_vol7942; GCN3-NEXT: v_mov_b32_e32 v0, s27943; GCN3-NEXT: v_mov_b32_e32 v1, s37944; GCN3-NEXT: flat_store_byte v[0:1], v27945; GCN3-NEXT: s_endpgm7946;7947; GFX11-TRUE16-LABEL: atomic_load_i8_offset:7948; GFX11-TRUE16: ; %bb.0: ; %entry7949; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x247950; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)7951; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17952; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s37953; GFX11-TRUE16-NEXT: flat_load_d16_u8 v0, v[0:1] offset:16 glc7954; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7955; GFX11-TRUE16-NEXT: buffer_gl1_inv7956; GFX11-TRUE16-NEXT: buffer_gl0_inv7957; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s27958; GFX11-TRUE16-NEXT: flat_store_b8 v[1:2], v07959; GFX11-TRUE16-NEXT: s_endpgm7960;7961; GFX11-FAKE16-LABEL: atomic_load_i8_offset:7962; GFX11-FAKE16: ; %bb.0: ; %entry7963; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x247964; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)7965; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s17966; GFX11-FAKE16-NEXT: flat_load_u8 v2, v[0:1] offset:16 glc7967; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7968; GFX11-FAKE16-NEXT: buffer_gl1_inv7969; GFX11-FAKE16-NEXT: buffer_gl0_inv7970; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s37971; GFX11-FAKE16-NEXT: flat_store_b8 v[0:1], v27972; GFX11-FAKE16-NEXT: s_endpgm7973entry:7974 %gep = getelementptr inbounds i8, ptr %in, i64 167975 %val = load atomic i8, ptr %gep seq_cst, align 17976 store i8 %val, ptr %out7977 ret void7978}7979 7980define amdgpu_kernel void @atomic_load_i8(ptr %in, ptr %out) {7981; GCN1-LABEL: atomic_load_i8:7982; GCN1: ; %bb.0: ; %entry7983; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x97984; GCN1-NEXT: s_waitcnt lgkmcnt(0)7985; GCN1-NEXT: v_mov_b32_e32 v0, s07986; GCN1-NEXT: v_mov_b32_e32 v1, s17987; GCN1-NEXT: flat_load_ubyte v2, v[0:1] glc7988; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7989; GCN1-NEXT: buffer_wbinvl1_vol7990; GCN1-NEXT: v_mov_b32_e32 v0, s27991; GCN1-NEXT: v_mov_b32_e32 v1, s37992; GCN1-NEXT: flat_store_byte v[0:1], v27993; GCN1-NEXT: s_endpgm7994;7995; GCN2-LABEL: atomic_load_i8:7996; GCN2: ; %bb.0: ; %entry7997; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247998; GCN2-NEXT: s_waitcnt lgkmcnt(0)7999; GCN2-NEXT: v_mov_b32_e32 v0, s08000; GCN2-NEXT: v_mov_b32_e32 v1, s18001; GCN2-NEXT: flat_load_ubyte v2, v[0:1] glc8002; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8003; GCN2-NEXT: buffer_wbinvl1_vol8004; GCN2-NEXT: v_mov_b32_e32 v0, s28005; GCN2-NEXT: v_mov_b32_e32 v1, s38006; GCN2-NEXT: flat_store_byte v[0:1], v28007; GCN2-NEXT: s_endpgm8008;8009; GCN3-LABEL: atomic_load_i8:8010; GCN3: ; %bb.0: ; %entry8011; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x248012; GCN3-NEXT: s_waitcnt lgkmcnt(0)8013; GCN3-NEXT: v_mov_b32_e32 v0, s08014; GCN3-NEXT: v_mov_b32_e32 v1, s18015; GCN3-NEXT: flat_load_ubyte v2, v[0:1] glc8016; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8017; GCN3-NEXT: buffer_wbinvl1_vol8018; GCN3-NEXT: v_mov_b32_e32 v0, s28019; GCN3-NEXT: v_mov_b32_e32 v1, s38020; GCN3-NEXT: flat_store_byte v[0:1], v28021; GCN3-NEXT: s_endpgm8022;8023; GFX11-TRUE16-LABEL: atomic_load_i8:8024; GFX11-TRUE16: ; %bb.0: ; %entry8025; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x248026; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)8027; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18028; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s38029; GFX11-TRUE16-NEXT: flat_load_d16_u8 v0, v[0:1] glc8030; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8031; GFX11-TRUE16-NEXT: buffer_gl1_inv8032; GFX11-TRUE16-NEXT: buffer_gl0_inv8033; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s28034; GFX11-TRUE16-NEXT: flat_store_b8 v[1:2], v08035; GFX11-TRUE16-NEXT: s_endpgm8036;8037; GFX11-FAKE16-LABEL: atomic_load_i8:8038; GFX11-FAKE16: ; %bb.0: ; %entry8039; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x248040; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)8041; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18042; GFX11-FAKE16-NEXT: flat_load_u8 v2, v[0:1] glc8043; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8044; GFX11-FAKE16-NEXT: buffer_gl1_inv8045; GFX11-FAKE16-NEXT: buffer_gl0_inv8046; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s38047; GFX11-FAKE16-NEXT: flat_store_b8 v[0:1], v28048; GFX11-FAKE16-NEXT: s_endpgm8049entry:8050 %val = load atomic i8, ptr %in seq_cst, align 18051 store i8 %val, ptr %out8052 ret void8053}8054 8055define amdgpu_kernel void @atomic_load_i8_addr64_offset(ptr %in, ptr %out, i64 %index) {8056; GCN1-LABEL: atomic_load_i8_addr64_offset:8057; GCN1: ; %bb.0: ; %entry8058; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x98059; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd8060; GCN1-NEXT: s_waitcnt lgkmcnt(0)8061; GCN1-NEXT: s_add_u32 s0, s0, s48062; GCN1-NEXT: s_addc_u32 s1, s1, s58063; GCN1-NEXT: s_add_u32 s0, s0, 168064; GCN1-NEXT: s_addc_u32 s1, s1, 08065; GCN1-NEXT: v_mov_b32_e32 v0, s08066; GCN1-NEXT: v_mov_b32_e32 v1, s18067; GCN1-NEXT: flat_load_ubyte v2, v[0:1] glc8068; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8069; GCN1-NEXT: buffer_wbinvl1_vol8070; GCN1-NEXT: v_mov_b32_e32 v0, s28071; GCN1-NEXT: v_mov_b32_e32 v1, s38072; GCN1-NEXT: flat_store_byte v[0:1], v28073; GCN1-NEXT: s_endpgm8074;8075; GCN2-LABEL: atomic_load_i8_addr64_offset:8076; GCN2: ; %bb.0: ; %entry8077; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x248078; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x348079; GCN2-NEXT: s_waitcnt lgkmcnt(0)8080; GCN2-NEXT: s_add_u32 s0, s0, s48081; GCN2-NEXT: s_addc_u32 s1, s1, s58082; GCN2-NEXT: s_add_u32 s0, s0, 168083; GCN2-NEXT: s_addc_u32 s1, s1, 08084; GCN2-NEXT: v_mov_b32_e32 v0, s08085; GCN2-NEXT: v_mov_b32_e32 v1, s18086; GCN2-NEXT: flat_load_ubyte v2, v[0:1] glc8087; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8088; GCN2-NEXT: buffer_wbinvl1_vol8089; GCN2-NEXT: v_mov_b32_e32 v0, s28090; GCN2-NEXT: v_mov_b32_e32 v1, s38091; GCN2-NEXT: flat_store_byte v[0:1], v28092; GCN2-NEXT: s_endpgm8093;8094; GCN3-LABEL: atomic_load_i8_addr64_offset:8095; GCN3: ; %bb.0: ; %entry8096; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x248097; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x348098; GCN3-NEXT: s_waitcnt lgkmcnt(0)8099; GCN3-NEXT: s_add_u32 s0, s0, s68100; GCN3-NEXT: s_addc_u32 s1, s1, s78101; GCN3-NEXT: v_mov_b32_e32 v0, s08102; GCN3-NEXT: v_mov_b32_e32 v1, s18103; GCN3-NEXT: flat_load_ubyte v2, v[0:1] offset:16 glc8104; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8105; GCN3-NEXT: buffer_wbinvl1_vol8106; GCN3-NEXT: v_mov_b32_e32 v0, s28107; GCN3-NEXT: v_mov_b32_e32 v1, s38108; GCN3-NEXT: flat_store_byte v[0:1], v28109; GCN3-NEXT: s_endpgm8110;8111; GFX11-TRUE16-LABEL: atomic_load_i8_addr64_offset:8112; GFX11-TRUE16: ; %bb.0: ; %entry8113; GFX11-TRUE16-NEXT: s_clause 0x18114; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x248115; GFX11-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x348116; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)8117; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s38118; GFX11-TRUE16-NEXT: s_add_u32 s0, s0, s48119; GFX11-TRUE16-NEXT: s_addc_u32 s1, s1, s58120; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8121; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18122; GFX11-TRUE16-NEXT: flat_load_d16_u8 v0, v[0:1] offset:16 glc8123; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8124; GFX11-TRUE16-NEXT: buffer_gl1_inv8125; GFX11-TRUE16-NEXT: buffer_gl0_inv8126; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s28127; GFX11-TRUE16-NEXT: flat_store_b8 v[1:2], v08128; GFX11-TRUE16-NEXT: s_endpgm8129;8130; GFX11-FAKE16-LABEL: atomic_load_i8_addr64_offset:8131; GFX11-FAKE16: ; %bb.0: ; %entry8132; GFX11-FAKE16-NEXT: s_clause 0x18133; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x248134; GFX11-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x348135; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)8136; GFX11-FAKE16-NEXT: s_add_u32 s0, s0, s48137; GFX11-FAKE16-NEXT: s_addc_u32 s1, s1, s58138; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8139; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18140; GFX11-FAKE16-NEXT: flat_load_u8 v2, v[0:1] offset:16 glc8141; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8142; GFX11-FAKE16-NEXT: buffer_gl1_inv8143; GFX11-FAKE16-NEXT: buffer_gl0_inv8144; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s38145; GFX11-FAKE16-NEXT: flat_store_b8 v[0:1], v28146; GFX11-FAKE16-NEXT: s_endpgm8147entry:8148 %ptr = getelementptr inbounds i8, ptr %in, i64 %index8149 %gep = getelementptr inbounds i8, ptr %ptr, i64 168150 %val = load atomic i8, ptr %gep seq_cst, align 18151 store i8 %val, ptr %out8152 ret void8153}8154 8155define amdgpu_kernel void @atomic_store_i8_offset(i8 %in, ptr %out) {8156; GCN1-LABEL: atomic_store_i8_offset:8157; GCN1: ; %bb.0: ; %entry8158; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xb8159; GCN1-NEXT: s_load_dword s2, s[4:5], 0x98160; GCN1-NEXT: s_waitcnt lgkmcnt(0)8161; GCN1-NEXT: s_add_u32 s0, s0, 168162; GCN1-NEXT: s_addc_u32 s1, s1, 08163; GCN1-NEXT: v_mov_b32_e32 v0, s08164; GCN1-NEXT: v_mov_b32_e32 v1, s18165; GCN1-NEXT: v_mov_b32_e32 v2, s28166; GCN1-NEXT: flat_store_byte v[0:1], v28167; GCN1-NEXT: s_endpgm8168;8169; GCN2-LABEL: atomic_store_i8_offset:8170; GCN2: ; %bb.0: ; %entry8171; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c8172; GCN2-NEXT: s_load_dword s2, s[4:5], 0x248173; GCN2-NEXT: s_waitcnt lgkmcnt(0)8174; GCN2-NEXT: s_add_u32 s0, s0, 168175; GCN2-NEXT: s_addc_u32 s1, s1, 08176; GCN2-NEXT: v_mov_b32_e32 v0, s08177; GCN2-NEXT: v_mov_b32_e32 v1, s18178; GCN2-NEXT: v_mov_b32_e32 v2, s28179; GCN2-NEXT: flat_store_byte v[0:1], v28180; GCN2-NEXT: s_endpgm8181;8182; GCN3-LABEL: atomic_store_i8_offset:8183; GCN3: ; %bb.0: ; %entry8184; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c8185; GCN3-NEXT: s_load_dword s2, s[4:5], 0x248186; GCN3-NEXT: s_waitcnt lgkmcnt(0)8187; GCN3-NEXT: v_mov_b32_e32 v0, s08188; GCN3-NEXT: v_mov_b32_e32 v1, s18189; GCN3-NEXT: v_mov_b32_e32 v2, s28190; GCN3-NEXT: flat_store_byte v[0:1], v2 offset:168191; GCN3-NEXT: s_endpgm8192;8193; GFX11-TRUE16-LABEL: atomic_store_i8_offset:8194; GFX11-TRUE16: ; %bb.0: ; %entry8195; GFX11-TRUE16-NEXT: s_clause 0x18196; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c8197; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x248198; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)8199; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s08200; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s28201; GFX11-TRUE16-NEXT: flat_store_b8 v[1:2], v0 offset:168202; GFX11-TRUE16-NEXT: s_endpgm8203;8204; GFX11-FAKE16-LABEL: atomic_store_i8_offset:8205; GFX11-FAKE16: ; %bb.0: ; %entry8206; GFX11-FAKE16-NEXT: s_clause 0x18207; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c8208; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x248209; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)8210; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18211; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s28212; GFX11-FAKE16-NEXT: flat_store_b8 v[0:1], v2 offset:168213; GFX11-FAKE16-NEXT: s_endpgm8214entry:8215 %gep = getelementptr inbounds i8, ptr %out, i64 168216 store atomic i8 %in, ptr %gep seq_cst, align 18217 ret void8218}8219 8220define amdgpu_kernel void @atomic_store_i8(i8 %in, ptr %out) {8221; GCN1-LABEL: atomic_store_i8:8222; GCN1: ; %bb.0: ; %entry8223; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xb8224; GCN1-NEXT: s_load_dword s2, s[4:5], 0x98225; GCN1-NEXT: s_waitcnt lgkmcnt(0)8226; GCN1-NEXT: v_mov_b32_e32 v0, s08227; GCN1-NEXT: v_mov_b32_e32 v1, s18228; GCN1-NEXT: v_mov_b32_e32 v2, s28229; GCN1-NEXT: flat_store_byte v[0:1], v28230; GCN1-NEXT: s_endpgm8231;8232; GCN2-LABEL: atomic_store_i8:8233; GCN2: ; %bb.0: ; %entry8234; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c8235; GCN2-NEXT: s_load_dword s2, s[4:5], 0x248236; GCN2-NEXT: s_waitcnt lgkmcnt(0)8237; GCN2-NEXT: v_mov_b32_e32 v0, s08238; GCN2-NEXT: v_mov_b32_e32 v1, s18239; GCN2-NEXT: v_mov_b32_e32 v2, s28240; GCN2-NEXT: flat_store_byte v[0:1], v28241; GCN2-NEXT: s_endpgm8242;8243; GCN3-LABEL: atomic_store_i8:8244; GCN3: ; %bb.0: ; %entry8245; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c8246; GCN3-NEXT: s_load_dword s2, s[4:5], 0x248247; GCN3-NEXT: s_waitcnt lgkmcnt(0)8248; GCN3-NEXT: v_mov_b32_e32 v0, s08249; GCN3-NEXT: v_mov_b32_e32 v1, s18250; GCN3-NEXT: v_mov_b32_e32 v2, s28251; GCN3-NEXT: flat_store_byte v[0:1], v28252; GCN3-NEXT: s_endpgm8253;8254; GFX11-TRUE16-LABEL: atomic_store_i8:8255; GFX11-TRUE16: ; %bb.0: ; %entry8256; GFX11-TRUE16-NEXT: s_clause 0x18257; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c8258; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x248259; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)8260; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s08261; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s28262; GFX11-TRUE16-NEXT: flat_store_b8 v[1:2], v08263; GFX11-TRUE16-NEXT: s_endpgm8264;8265; GFX11-FAKE16-LABEL: atomic_store_i8:8266; GFX11-FAKE16: ; %bb.0: ; %entry8267; GFX11-FAKE16-NEXT: s_clause 0x18268; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c8269; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x248270; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)8271; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18272; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s28273; GFX11-FAKE16-NEXT: flat_store_b8 v[0:1], v28274; GFX11-FAKE16-NEXT: s_endpgm8275entry:8276 store atomic i8 %in, ptr %out seq_cst, align 18277 ret void8278}8279 8280define amdgpu_kernel void @atomic_store_i8_addr64_offset(i8 %in, ptr %out, i64 %index) {8281; GCN1-LABEL: atomic_store_i8_addr64_offset:8282; GCN1: ; %bb.0: ; %entry8283; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb8284; GCN1-NEXT: s_load_dword s4, s[4:5], 0x98285; GCN1-NEXT: s_waitcnt lgkmcnt(0)8286; GCN1-NEXT: s_add_u32 s0, s0, s28287; GCN1-NEXT: s_addc_u32 s1, s1, s38288; GCN1-NEXT: s_add_u32 s0, s0, 168289; GCN1-NEXT: s_addc_u32 s1, s1, 08290; GCN1-NEXT: v_mov_b32_e32 v0, s08291; GCN1-NEXT: v_mov_b32_e32 v1, s18292; GCN1-NEXT: v_mov_b32_e32 v2, s48293; GCN1-NEXT: flat_store_byte v[0:1], v28294; GCN1-NEXT: s_endpgm8295;8296; GCN2-LABEL: atomic_store_i8_addr64_offset:8297; GCN2: ; %bb.0: ; %entry8298; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c8299; GCN2-NEXT: s_load_dword s4, s[4:5], 0x248300; GCN2-NEXT: s_waitcnt lgkmcnt(0)8301; GCN2-NEXT: s_add_u32 s0, s0, s28302; GCN2-NEXT: s_addc_u32 s1, s1, s38303; GCN2-NEXT: s_add_u32 s0, s0, 168304; GCN2-NEXT: s_addc_u32 s1, s1, 08305; GCN2-NEXT: v_mov_b32_e32 v0, s08306; GCN2-NEXT: v_mov_b32_e32 v1, s18307; GCN2-NEXT: v_mov_b32_e32 v2, s48308; GCN2-NEXT: flat_store_byte v[0:1], v28309; GCN2-NEXT: s_endpgm8310;8311; GCN3-LABEL: atomic_store_i8_addr64_offset:8312; GCN3: ; %bb.0: ; %entry8313; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c8314; GCN3-NEXT: s_load_dword s6, s[4:5], 0x248315; GCN3-NEXT: s_waitcnt lgkmcnt(0)8316; GCN3-NEXT: s_add_u32 s0, s0, s28317; GCN3-NEXT: s_addc_u32 s1, s1, s38318; GCN3-NEXT: v_mov_b32_e32 v0, s08319; GCN3-NEXT: v_mov_b32_e32 v1, s18320; GCN3-NEXT: v_mov_b32_e32 v2, s68321; GCN3-NEXT: flat_store_byte v[0:1], v2 offset:168322; GCN3-NEXT: s_endpgm8323;8324; GFX11-TRUE16-LABEL: atomic_store_i8_addr64_offset:8325; GFX11-TRUE16: ; %bb.0: ; %entry8326; GFX11-TRUE16-NEXT: s_clause 0x18327; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c8328; GFX11-TRUE16-NEXT: s_load_b32 s4, s[4:5], 0x248329; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)8330; GFX11-TRUE16-NEXT: s_add_u32 s0, s0, s28331; GFX11-TRUE16-NEXT: s_addc_u32 s1, s1, s38332; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s48333; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s08334; GFX11-TRUE16-NEXT: flat_store_b8 v[1:2], v0 offset:168335; GFX11-TRUE16-NEXT: s_endpgm8336;8337; GFX11-FAKE16-LABEL: atomic_store_i8_addr64_offset:8338; GFX11-FAKE16: ; %bb.0: ; %entry8339; GFX11-FAKE16-NEXT: s_clause 0x18340; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c8341; GFX11-FAKE16-NEXT: s_load_b32 s4, s[4:5], 0x248342; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)8343; GFX11-FAKE16-NEXT: s_add_u32 s0, s0, s28344; GFX11-FAKE16-NEXT: s_addc_u32 s1, s1, s38345; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8346; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18347; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s48348; GFX11-FAKE16-NEXT: flat_store_b8 v[0:1], v2 offset:168349; GFX11-FAKE16-NEXT: s_endpgm8350entry:8351 %ptr = getelementptr inbounds i8, ptr %out, i64 %index8352 %gep = getelementptr inbounds i8, ptr %ptr, i64 168353 store atomic i8 %in, ptr %gep seq_cst, align 18354 ret void8355}8356 8357define amdgpu_kernel void @atomic_load_i16_offset(ptr %in, ptr %out) {8358; GCN1-LABEL: atomic_load_i16_offset:8359; GCN1: ; %bb.0: ; %entry8360; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x98361; GCN1-NEXT: s_waitcnt lgkmcnt(0)8362; GCN1-NEXT: s_add_u32 s0, s0, 168363; GCN1-NEXT: s_addc_u32 s1, s1, 08364; GCN1-NEXT: v_mov_b32_e32 v0, s08365; GCN1-NEXT: v_mov_b32_e32 v1, s18366; GCN1-NEXT: flat_load_ushort v2, v[0:1] glc8367; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8368; GCN1-NEXT: buffer_wbinvl1_vol8369; GCN1-NEXT: v_mov_b32_e32 v0, s28370; GCN1-NEXT: v_mov_b32_e32 v1, s38371; GCN1-NEXT: flat_store_short v[0:1], v28372; GCN1-NEXT: s_endpgm8373;8374; GCN2-LABEL: atomic_load_i16_offset:8375; GCN2: ; %bb.0: ; %entry8376; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x248377; GCN2-NEXT: s_waitcnt lgkmcnt(0)8378; GCN2-NEXT: s_add_u32 s0, s0, 168379; GCN2-NEXT: s_addc_u32 s1, s1, 08380; GCN2-NEXT: v_mov_b32_e32 v0, s08381; GCN2-NEXT: v_mov_b32_e32 v1, s18382; GCN2-NEXT: flat_load_ushort v2, v[0:1] glc8383; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8384; GCN2-NEXT: buffer_wbinvl1_vol8385; GCN2-NEXT: v_mov_b32_e32 v0, s28386; GCN2-NEXT: v_mov_b32_e32 v1, s38387; GCN2-NEXT: flat_store_short v[0:1], v28388; GCN2-NEXT: s_endpgm8389;8390; GCN3-LABEL: atomic_load_i16_offset:8391; GCN3: ; %bb.0: ; %entry8392; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x248393; GCN3-NEXT: s_waitcnt lgkmcnt(0)8394; GCN3-NEXT: v_mov_b32_e32 v0, s08395; GCN3-NEXT: v_mov_b32_e32 v1, s18396; GCN3-NEXT: flat_load_ushort v2, v[0:1] offset:16 glc8397; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8398; GCN3-NEXT: buffer_wbinvl1_vol8399; GCN3-NEXT: v_mov_b32_e32 v0, s28400; GCN3-NEXT: v_mov_b32_e32 v1, s38401; GCN3-NEXT: flat_store_short v[0:1], v28402; GCN3-NEXT: s_endpgm8403;8404; GFX11-TRUE16-LABEL: atomic_load_i16_offset:8405; GFX11-TRUE16: ; %bb.0: ; %entry8406; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x248407; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)8408; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18409; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s38410; GFX11-TRUE16-NEXT: flat_load_d16_b16 v0, v[0:1] offset:16 glc8411; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8412; GFX11-TRUE16-NEXT: buffer_gl1_inv8413; GFX11-TRUE16-NEXT: buffer_gl0_inv8414; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s28415; GFX11-TRUE16-NEXT: flat_store_b16 v[1:2], v08416; GFX11-TRUE16-NEXT: s_endpgm8417;8418; GFX11-FAKE16-LABEL: atomic_load_i16_offset:8419; GFX11-FAKE16: ; %bb.0: ; %entry8420; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x248421; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)8422; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18423; GFX11-FAKE16-NEXT: flat_load_u16 v2, v[0:1] offset:16 glc8424; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8425; GFX11-FAKE16-NEXT: buffer_gl1_inv8426; GFX11-FAKE16-NEXT: buffer_gl0_inv8427; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s38428; GFX11-FAKE16-NEXT: flat_store_b16 v[0:1], v28429; GFX11-FAKE16-NEXT: s_endpgm8430entry:8431 %gep = getelementptr inbounds i16, ptr %in, i64 88432 %val = load atomic i16, ptr %gep seq_cst, align 28433 store i16 %val, ptr %out8434 ret void8435}8436 8437define amdgpu_kernel void @atomic_load_i16(ptr %in, ptr %out) {8438; GCN1-LABEL: atomic_load_i16:8439; GCN1: ; %bb.0: ; %entry8440; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x98441; GCN1-NEXT: s_waitcnt lgkmcnt(0)8442; GCN1-NEXT: v_mov_b32_e32 v0, s08443; GCN1-NEXT: v_mov_b32_e32 v1, s18444; GCN1-NEXT: flat_load_ushort v2, v[0:1] glc8445; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8446; GCN1-NEXT: buffer_wbinvl1_vol8447; GCN1-NEXT: v_mov_b32_e32 v0, s28448; GCN1-NEXT: v_mov_b32_e32 v1, s38449; GCN1-NEXT: flat_store_short v[0:1], v28450; GCN1-NEXT: s_endpgm8451;8452; GCN2-LABEL: atomic_load_i16:8453; GCN2: ; %bb.0: ; %entry8454; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x248455; GCN2-NEXT: s_waitcnt lgkmcnt(0)8456; GCN2-NEXT: v_mov_b32_e32 v0, s08457; GCN2-NEXT: v_mov_b32_e32 v1, s18458; GCN2-NEXT: flat_load_ushort v2, v[0:1] glc8459; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8460; GCN2-NEXT: buffer_wbinvl1_vol8461; GCN2-NEXT: v_mov_b32_e32 v0, s28462; GCN2-NEXT: v_mov_b32_e32 v1, s38463; GCN2-NEXT: flat_store_short v[0:1], v28464; GCN2-NEXT: s_endpgm8465;8466; GCN3-LABEL: atomic_load_i16:8467; GCN3: ; %bb.0: ; %entry8468; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x248469; GCN3-NEXT: s_waitcnt lgkmcnt(0)8470; GCN3-NEXT: v_mov_b32_e32 v0, s08471; GCN3-NEXT: v_mov_b32_e32 v1, s18472; GCN3-NEXT: flat_load_ushort v2, v[0:1] glc8473; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8474; GCN3-NEXT: buffer_wbinvl1_vol8475; GCN3-NEXT: v_mov_b32_e32 v0, s28476; GCN3-NEXT: v_mov_b32_e32 v1, s38477; GCN3-NEXT: flat_store_short v[0:1], v28478; GCN3-NEXT: s_endpgm8479;8480; GFX11-TRUE16-LABEL: atomic_load_i16:8481; GFX11-TRUE16: ; %bb.0: ; %entry8482; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x248483; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)8484; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18485; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s38486; GFX11-TRUE16-NEXT: flat_load_d16_b16 v0, v[0:1] glc8487; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8488; GFX11-TRUE16-NEXT: buffer_gl1_inv8489; GFX11-TRUE16-NEXT: buffer_gl0_inv8490; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s28491; GFX11-TRUE16-NEXT: flat_store_b16 v[1:2], v08492; GFX11-TRUE16-NEXT: s_endpgm8493;8494; GFX11-FAKE16-LABEL: atomic_load_i16:8495; GFX11-FAKE16: ; %bb.0: ; %entry8496; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x248497; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)8498; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18499; GFX11-FAKE16-NEXT: flat_load_u16 v2, v[0:1] glc8500; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8501; GFX11-FAKE16-NEXT: buffer_gl1_inv8502; GFX11-FAKE16-NEXT: buffer_gl0_inv8503; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s38504; GFX11-FAKE16-NEXT: flat_store_b16 v[0:1], v28505; GFX11-FAKE16-NEXT: s_endpgm8506entry:8507 %val = load atomic i16, ptr %in seq_cst, align 28508 store i16 %val, ptr %out8509 ret void8510}8511 8512define amdgpu_kernel void @atomic_load_i16_addr64_offset(ptr %in, ptr %out, i64 %index) {8513; GCN1-LABEL: atomic_load_i16_addr64_offset:8514; GCN1: ; %bb.0: ; %entry8515; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd8516; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x98517; GCN1-NEXT: s_waitcnt lgkmcnt(0)8518; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 18519; GCN1-NEXT: s_add_u32 s0, s0, s48520; GCN1-NEXT: s_addc_u32 s1, s1, s58521; GCN1-NEXT: s_add_u32 s0, s0, 168522; GCN1-NEXT: s_addc_u32 s1, s1, 08523; GCN1-NEXT: v_mov_b32_e32 v0, s08524; GCN1-NEXT: v_mov_b32_e32 v1, s18525; GCN1-NEXT: flat_load_ushort v2, v[0:1] glc8526; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8527; GCN1-NEXT: buffer_wbinvl1_vol8528; GCN1-NEXT: v_mov_b32_e32 v0, s28529; GCN1-NEXT: v_mov_b32_e32 v1, s38530; GCN1-NEXT: flat_store_short v[0:1], v28531; GCN1-NEXT: s_endpgm8532;8533; GCN2-LABEL: atomic_load_i16_addr64_offset:8534; GCN2: ; %bb.0: ; %entry8535; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x348536; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x248537; GCN2-NEXT: s_waitcnt lgkmcnt(0)8538; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 18539; GCN2-NEXT: s_add_u32 s0, s0, s48540; GCN2-NEXT: s_addc_u32 s1, s1, s58541; GCN2-NEXT: s_add_u32 s0, s0, 168542; GCN2-NEXT: s_addc_u32 s1, s1, 08543; GCN2-NEXT: v_mov_b32_e32 v0, s08544; GCN2-NEXT: v_mov_b32_e32 v1, s18545; GCN2-NEXT: flat_load_ushort v2, v[0:1] glc8546; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8547; GCN2-NEXT: buffer_wbinvl1_vol8548; GCN2-NEXT: v_mov_b32_e32 v0, s28549; GCN2-NEXT: v_mov_b32_e32 v1, s38550; GCN2-NEXT: flat_store_short v[0:1], v28551; GCN2-NEXT: s_endpgm8552;8553; GCN3-LABEL: atomic_load_i16_addr64_offset:8554; GCN3: ; %bb.0: ; %entry8555; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x348556; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x248557; GCN3-NEXT: s_waitcnt lgkmcnt(0)8558; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 18559; GCN3-NEXT: s_add_u32 s0, s0, s48560; GCN3-NEXT: s_addc_u32 s1, s1, s58561; GCN3-NEXT: v_mov_b32_e32 v0, s08562; GCN3-NEXT: v_mov_b32_e32 v1, s18563; GCN3-NEXT: flat_load_ushort v2, v[0:1] offset:16 glc8564; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8565; GCN3-NEXT: buffer_wbinvl1_vol8566; GCN3-NEXT: v_mov_b32_e32 v0, s28567; GCN3-NEXT: v_mov_b32_e32 v1, s38568; GCN3-NEXT: flat_store_short v[0:1], v28569; GCN3-NEXT: s_endpgm8570;8571; GFX11-TRUE16-LABEL: atomic_load_i16_addr64_offset:8572; GFX11-TRUE16: ; %bb.0: ; %entry8573; GFX11-TRUE16-NEXT: s_clause 0x18574; GFX11-TRUE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x348575; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x248576; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)8577; GFX11-TRUE16-NEXT: s_lshl_b64 s[4:5], s[6:7], 18578; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s38579; GFX11-TRUE16-NEXT: s_add_u32 s0, s0, s48580; GFX11-TRUE16-NEXT: s_addc_u32 s1, s1, s58581; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8582; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18583; GFX11-TRUE16-NEXT: flat_load_d16_b16 v0, v[0:1] offset:16 glc8584; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8585; GFX11-TRUE16-NEXT: buffer_gl1_inv8586; GFX11-TRUE16-NEXT: buffer_gl0_inv8587; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s28588; GFX11-TRUE16-NEXT: flat_store_b16 v[1:2], v08589; GFX11-TRUE16-NEXT: s_endpgm8590;8591; GFX11-FAKE16-LABEL: atomic_load_i16_addr64_offset:8592; GFX11-FAKE16: ; %bb.0: ; %entry8593; GFX11-FAKE16-NEXT: s_clause 0x18594; GFX11-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x348595; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x248596; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)8597; GFX11-FAKE16-NEXT: s_lshl_b64 s[4:5], s[6:7], 18598; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)8599; GFX11-FAKE16-NEXT: s_add_u32 s0, s0, s48600; GFX11-FAKE16-NEXT: s_addc_u32 s1, s1, s58601; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18602; GFX11-FAKE16-NEXT: flat_load_u16 v2, v[0:1] offset:16 glc8603; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8604; GFX11-FAKE16-NEXT: buffer_gl1_inv8605; GFX11-FAKE16-NEXT: buffer_gl0_inv8606; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s38607; GFX11-FAKE16-NEXT: flat_store_b16 v[0:1], v28608; GFX11-FAKE16-NEXT: s_endpgm8609entry:8610 %ptr = getelementptr inbounds i16, ptr %in, i64 %index8611 %gep = getelementptr inbounds i16, ptr %ptr, i64 88612 %val = load atomic i16, ptr %gep seq_cst, align 28613 store i16 %val, ptr %out8614 ret void8615}8616 8617define amdgpu_kernel void @atomic_store_i16_offset(i16 %in, ptr %out) {8618; GCN1-LABEL: atomic_store_i16_offset:8619; GCN1: ; %bb.0: ; %entry8620; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xb8621; GCN1-NEXT: s_load_dword s2, s[4:5], 0x98622; GCN1-NEXT: s_waitcnt lgkmcnt(0)8623; GCN1-NEXT: s_add_u32 s0, s0, 168624; GCN1-NEXT: s_addc_u32 s1, s1, 08625; GCN1-NEXT: v_mov_b32_e32 v0, s08626; GCN1-NEXT: v_mov_b32_e32 v1, s18627; GCN1-NEXT: v_mov_b32_e32 v2, s28628; GCN1-NEXT: flat_store_short v[0:1], v28629; GCN1-NEXT: s_endpgm8630;8631; GCN2-LABEL: atomic_store_i16_offset:8632; GCN2: ; %bb.0: ; %entry8633; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c8634; GCN2-NEXT: s_load_dword s2, s[4:5], 0x248635; GCN2-NEXT: s_waitcnt lgkmcnt(0)8636; GCN2-NEXT: s_add_u32 s0, s0, 168637; GCN2-NEXT: s_addc_u32 s1, s1, 08638; GCN2-NEXT: v_mov_b32_e32 v0, s08639; GCN2-NEXT: v_mov_b32_e32 v1, s18640; GCN2-NEXT: v_mov_b32_e32 v2, s28641; GCN2-NEXT: flat_store_short v[0:1], v28642; GCN2-NEXT: s_endpgm8643;8644; GCN3-LABEL: atomic_store_i16_offset:8645; GCN3: ; %bb.0: ; %entry8646; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c8647; GCN3-NEXT: s_load_dword s2, s[4:5], 0x248648; GCN3-NEXT: s_waitcnt lgkmcnt(0)8649; GCN3-NEXT: v_mov_b32_e32 v0, s08650; GCN3-NEXT: v_mov_b32_e32 v1, s18651; GCN3-NEXT: v_mov_b32_e32 v2, s28652; GCN3-NEXT: flat_store_short v[0:1], v2 offset:168653; GCN3-NEXT: s_endpgm8654;8655; GFX11-TRUE16-LABEL: atomic_store_i16_offset:8656; GFX11-TRUE16: ; %bb.0: ; %entry8657; GFX11-TRUE16-NEXT: s_clause 0x18658; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c8659; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x248660; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)8661; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s08662; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s28663; GFX11-TRUE16-NEXT: flat_store_b16 v[1:2], v0 offset:168664; GFX11-TRUE16-NEXT: s_endpgm8665;8666; GFX11-FAKE16-LABEL: atomic_store_i16_offset:8667; GFX11-FAKE16: ; %bb.0: ; %entry8668; GFX11-FAKE16-NEXT: s_clause 0x18669; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c8670; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x248671; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)8672; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18673; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s28674; GFX11-FAKE16-NEXT: flat_store_b16 v[0:1], v2 offset:168675; GFX11-FAKE16-NEXT: s_endpgm8676entry:8677 %gep = getelementptr inbounds i16, ptr %out, i64 88678 store atomic i16 %in, ptr %gep seq_cst, align 28679 ret void8680}8681 8682define amdgpu_kernel void @atomic_store_i16(i16 %in, ptr %out) {8683; GCN1-LABEL: atomic_store_i16:8684; GCN1: ; %bb.0: ; %entry8685; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xb8686; GCN1-NEXT: s_load_dword s2, s[4:5], 0x98687; GCN1-NEXT: s_waitcnt lgkmcnt(0)8688; GCN1-NEXT: v_mov_b32_e32 v0, s08689; GCN1-NEXT: v_mov_b32_e32 v1, s18690; GCN1-NEXT: v_mov_b32_e32 v2, s28691; GCN1-NEXT: flat_store_short v[0:1], v28692; GCN1-NEXT: s_endpgm8693;8694; GCN2-LABEL: atomic_store_i16:8695; GCN2: ; %bb.0: ; %entry8696; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c8697; GCN2-NEXT: s_load_dword s2, s[4:5], 0x248698; GCN2-NEXT: s_waitcnt lgkmcnt(0)8699; GCN2-NEXT: v_mov_b32_e32 v0, s08700; GCN2-NEXT: v_mov_b32_e32 v1, s18701; GCN2-NEXT: v_mov_b32_e32 v2, s28702; GCN2-NEXT: flat_store_short v[0:1], v28703; GCN2-NEXT: s_endpgm8704;8705; GCN3-LABEL: atomic_store_i16:8706; GCN3: ; %bb.0: ; %entry8707; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c8708; GCN3-NEXT: s_load_dword s2, s[4:5], 0x248709; GCN3-NEXT: s_waitcnt lgkmcnt(0)8710; GCN3-NEXT: v_mov_b32_e32 v0, s08711; GCN3-NEXT: v_mov_b32_e32 v1, s18712; GCN3-NEXT: v_mov_b32_e32 v2, s28713; GCN3-NEXT: flat_store_short v[0:1], v28714; GCN3-NEXT: s_endpgm8715;8716; GFX11-TRUE16-LABEL: atomic_store_i16:8717; GFX11-TRUE16: ; %bb.0: ; %entry8718; GFX11-TRUE16-NEXT: s_clause 0x18719; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c8720; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x248721; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)8722; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s08723; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s28724; GFX11-TRUE16-NEXT: flat_store_b16 v[1:2], v08725; GFX11-TRUE16-NEXT: s_endpgm8726;8727; GFX11-FAKE16-LABEL: atomic_store_i16:8728; GFX11-FAKE16: ; %bb.0: ; %entry8729; GFX11-FAKE16-NEXT: s_clause 0x18730; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c8731; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x248732; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)8733; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18734; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s28735; GFX11-FAKE16-NEXT: flat_store_b16 v[0:1], v28736; GFX11-FAKE16-NEXT: s_endpgm8737entry:8738 store atomic i16 %in, ptr %out seq_cst, align 28739 ret void8740}8741 8742define amdgpu_kernel void @atomic_store_i16_addr64_offset(i16 %in, ptr %out, i64 %index) {8743; GCN1-LABEL: atomic_store_i16_addr64_offset:8744; GCN1: ; %bb.0: ; %entry8745; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb8746; GCN1-NEXT: s_load_dword s4, s[4:5], 0x98747; GCN1-NEXT: s_waitcnt lgkmcnt(0)8748; GCN1-NEXT: s_lshl_b64 s[2:3], s[2:3], 18749; GCN1-NEXT: s_add_u32 s0, s0, s28750; GCN1-NEXT: s_addc_u32 s1, s1, s38751; GCN1-NEXT: s_add_u32 s0, s0, 168752; GCN1-NEXT: s_addc_u32 s1, s1, 08753; GCN1-NEXT: v_mov_b32_e32 v0, s08754; GCN1-NEXT: v_mov_b32_e32 v1, s18755; GCN1-NEXT: v_mov_b32_e32 v2, s48756; GCN1-NEXT: flat_store_short v[0:1], v28757; GCN1-NEXT: s_endpgm8758;8759; GCN2-LABEL: atomic_store_i16_addr64_offset:8760; GCN2: ; %bb.0: ; %entry8761; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c8762; GCN2-NEXT: s_load_dword s4, s[4:5], 0x248763; GCN2-NEXT: s_waitcnt lgkmcnt(0)8764; GCN2-NEXT: s_lshl_b64 s[2:3], s[2:3], 18765; GCN2-NEXT: s_add_u32 s0, s0, s28766; GCN2-NEXT: s_addc_u32 s1, s1, s38767; GCN2-NEXT: s_add_u32 s0, s0, 168768; GCN2-NEXT: s_addc_u32 s1, s1, 08769; GCN2-NEXT: v_mov_b32_e32 v0, s08770; GCN2-NEXT: v_mov_b32_e32 v1, s18771; GCN2-NEXT: v_mov_b32_e32 v2, s48772; GCN2-NEXT: flat_store_short v[0:1], v28773; GCN2-NEXT: s_endpgm8774;8775; GCN3-LABEL: atomic_store_i16_addr64_offset:8776; GCN3: ; %bb.0: ; %entry8777; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c8778; GCN3-NEXT: s_load_dword s6, s[4:5], 0x248779; GCN3-NEXT: s_waitcnt lgkmcnt(0)8780; GCN3-NEXT: s_lshl_b64 s[2:3], s[2:3], 18781; GCN3-NEXT: s_add_u32 s0, s0, s28782; GCN3-NEXT: s_addc_u32 s1, s1, s38783; GCN3-NEXT: v_mov_b32_e32 v0, s08784; GCN3-NEXT: v_mov_b32_e32 v1, s18785; GCN3-NEXT: v_mov_b32_e32 v2, s68786; GCN3-NEXT: flat_store_short v[0:1], v2 offset:168787; GCN3-NEXT: s_endpgm8788;8789; GFX11-TRUE16-LABEL: atomic_store_i16_addr64_offset:8790; GFX11-TRUE16: ; %bb.0: ; %entry8791; GFX11-TRUE16-NEXT: s_clause 0x18792; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c8793; GFX11-TRUE16-NEXT: s_load_b32 s4, s[4:5], 0x248794; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)8795; GFX11-TRUE16-NEXT: s_lshl_b64 s[2:3], s[2:3], 18796; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s48797; GFX11-TRUE16-NEXT: s_add_u32 s0, s0, s28798; GFX11-TRUE16-NEXT: s_addc_u32 s1, s1, s38799; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8800; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s08801; GFX11-TRUE16-NEXT: flat_store_b16 v[1:2], v0 offset:168802; GFX11-TRUE16-NEXT: s_endpgm8803;8804; GFX11-FAKE16-LABEL: atomic_store_i16_addr64_offset:8805; GFX11-FAKE16: ; %bb.0: ; %entry8806; GFX11-FAKE16-NEXT: s_clause 0x18807; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c8808; GFX11-FAKE16-NEXT: s_load_b32 s4, s[4:5], 0x248809; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)8810; GFX11-FAKE16-NEXT: s_lshl_b64 s[2:3], s[2:3], 18811; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s48812; GFX11-FAKE16-NEXT: s_add_u32 s0, s0, s28813; GFX11-FAKE16-NEXT: s_addc_u32 s1, s1, s38814; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8815; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18816; GFX11-FAKE16-NEXT: flat_store_b16 v[0:1], v2 offset:168817; GFX11-FAKE16-NEXT: s_endpgm8818entry:8819 %ptr = getelementptr inbounds i16, ptr %out, i64 %index8820 %gep = getelementptr inbounds i16, ptr %ptr, i64 88821 store atomic i16 %in, ptr %gep seq_cst, align 28822 ret void8823}8824 8825define amdgpu_kernel void @atomic_store_f16_offset(half %in, ptr %out) {8826; GCN1-LABEL: atomic_store_f16_offset:8827; GCN1: ; %bb.0: ; %entry8828; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xb8829; GCN1-NEXT: s_load_dword s2, s[4:5], 0x98830; GCN1-NEXT: s_waitcnt lgkmcnt(0)8831; GCN1-NEXT: s_add_u32 s0, s0, 168832; GCN1-NEXT: s_addc_u32 s1, s1, 08833; GCN1-NEXT: v_mov_b32_e32 v0, s08834; GCN1-NEXT: v_mov_b32_e32 v1, s18835; GCN1-NEXT: v_mov_b32_e32 v2, s28836; GCN1-NEXT: flat_store_short v[0:1], v28837; GCN1-NEXT: s_endpgm8838;8839; GCN2-LABEL: atomic_store_f16_offset:8840; GCN2: ; %bb.0: ; %entry8841; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c8842; GCN2-NEXT: s_load_dword s2, s[4:5], 0x248843; GCN2-NEXT: s_waitcnt lgkmcnt(0)8844; GCN2-NEXT: s_add_u32 s0, s0, 168845; GCN2-NEXT: s_addc_u32 s1, s1, 08846; GCN2-NEXT: v_mov_b32_e32 v0, s08847; GCN2-NEXT: v_mov_b32_e32 v1, s18848; GCN2-NEXT: v_mov_b32_e32 v2, s28849; GCN2-NEXT: flat_store_short v[0:1], v28850; GCN2-NEXT: s_endpgm8851;8852; GCN3-LABEL: atomic_store_f16_offset:8853; GCN3: ; %bb.0: ; %entry8854; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c8855; GCN3-NEXT: s_load_dword s2, s[4:5], 0x248856; GCN3-NEXT: s_waitcnt lgkmcnt(0)8857; GCN3-NEXT: v_mov_b32_e32 v0, s08858; GCN3-NEXT: v_mov_b32_e32 v1, s18859; GCN3-NEXT: v_mov_b32_e32 v2, s28860; GCN3-NEXT: flat_store_short v[0:1], v2 offset:168861; GCN3-NEXT: s_endpgm8862;8863; GFX11-TRUE16-LABEL: atomic_store_f16_offset:8864; GFX11-TRUE16: ; %bb.0: ; %entry8865; GFX11-TRUE16-NEXT: s_clause 0x18866; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c8867; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x248868; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)8869; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s08870; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s28871; GFX11-TRUE16-NEXT: flat_store_b16 v[1:2], v0 offset:168872; GFX11-TRUE16-NEXT: s_endpgm8873;8874; GFX11-FAKE16-LABEL: atomic_store_f16_offset:8875; GFX11-FAKE16: ; %bb.0: ; %entry8876; GFX11-FAKE16-NEXT: s_clause 0x18877; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c8878; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x248879; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)8880; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18881; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s28882; GFX11-FAKE16-NEXT: flat_store_b16 v[0:1], v2 offset:168883; GFX11-FAKE16-NEXT: s_endpgm8884entry:8885 %gep = getelementptr inbounds half, ptr %out, i64 88886 store atomic half %in, ptr %gep seq_cst, align 28887 ret void8888}8889 8890define amdgpu_kernel void @atomic_store_f16(half %in, ptr %out) {8891; GCN1-LABEL: atomic_store_f16:8892; GCN1: ; %bb.0: ; %entry8893; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xb8894; GCN1-NEXT: s_load_dword s2, s[4:5], 0x98895; GCN1-NEXT: s_waitcnt lgkmcnt(0)8896; GCN1-NEXT: v_mov_b32_e32 v0, s08897; GCN1-NEXT: v_mov_b32_e32 v1, s18898; GCN1-NEXT: v_mov_b32_e32 v2, s28899; GCN1-NEXT: flat_store_short v[0:1], v28900; GCN1-NEXT: s_endpgm8901;8902; GCN2-LABEL: atomic_store_f16:8903; GCN2: ; %bb.0: ; %entry8904; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c8905; GCN2-NEXT: s_load_dword s2, s[4:5], 0x248906; GCN2-NEXT: s_waitcnt lgkmcnt(0)8907; GCN2-NEXT: v_mov_b32_e32 v0, s08908; GCN2-NEXT: v_mov_b32_e32 v1, s18909; GCN2-NEXT: v_mov_b32_e32 v2, s28910; GCN2-NEXT: flat_store_short v[0:1], v28911; GCN2-NEXT: s_endpgm8912;8913; GCN3-LABEL: atomic_store_f16:8914; GCN3: ; %bb.0: ; %entry8915; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c8916; GCN3-NEXT: s_load_dword s2, s[4:5], 0x248917; GCN3-NEXT: s_waitcnt lgkmcnt(0)8918; GCN3-NEXT: v_mov_b32_e32 v0, s08919; GCN3-NEXT: v_mov_b32_e32 v1, s18920; GCN3-NEXT: v_mov_b32_e32 v2, s28921; GCN3-NEXT: flat_store_short v[0:1], v28922; GCN3-NEXT: s_endpgm8923;8924; GFX11-TRUE16-LABEL: atomic_store_f16:8925; GFX11-TRUE16: ; %bb.0: ; %entry8926; GFX11-TRUE16-NEXT: s_clause 0x18927; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c8928; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x248929; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)8930; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s08931; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s28932; GFX11-TRUE16-NEXT: flat_store_b16 v[1:2], v08933; GFX11-TRUE16-NEXT: s_endpgm8934;8935; GFX11-FAKE16-LABEL: atomic_store_f16:8936; GFX11-FAKE16: ; %bb.0: ; %entry8937; GFX11-FAKE16-NEXT: s_clause 0x18938; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c8939; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x248940; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)8941; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s18942; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s28943; GFX11-FAKE16-NEXT: flat_store_b16 v[0:1], v28944; GFX11-FAKE16-NEXT: s_endpgm8945entry:8946 store atomic half %in, ptr %out seq_cst, align 28947 ret void8948}8949 8950define amdgpu_kernel void @atomic_store_bf16_offset(bfloat %in, ptr %out) {8951; GCN1-LABEL: atomic_store_bf16_offset:8952; GCN1: ; %bb.0:8953; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xb8954; GCN1-NEXT: s_load_dword s2, s[4:5], 0x98955; GCN1-NEXT: s_waitcnt lgkmcnt(0)8956; GCN1-NEXT: v_mov_b32_e32 v0, s08957; GCN1-NEXT: v_mov_b32_e32 v1, s18958; GCN1-NEXT: v_mov_b32_e32 v2, s28959; GCN1-NEXT: flat_store_short v[0:1], v28960; GCN1-NEXT: s_endpgm8961;8962; GCN2-LABEL: atomic_store_bf16_offset:8963; GCN2: ; %bb.0:8964; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c8965; GCN2-NEXT: s_load_dword s2, s[4:5], 0x248966; GCN2-NEXT: s_waitcnt lgkmcnt(0)8967; GCN2-NEXT: v_mov_b32_e32 v0, s08968; GCN2-NEXT: v_mov_b32_e32 v1, s18969; GCN2-NEXT: v_mov_b32_e32 v2, s28970; GCN2-NEXT: flat_store_short v[0:1], v28971; GCN2-NEXT: s_endpgm8972;8973; GCN3-LABEL: atomic_store_bf16_offset:8974; GCN3: ; %bb.0:8975; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c8976; GCN3-NEXT: s_load_dword s2, s[4:5], 0x248977; GCN3-NEXT: s_waitcnt lgkmcnt(0)8978; GCN3-NEXT: v_mov_b32_e32 v0, s08979; GCN3-NEXT: v_mov_b32_e32 v1, s18980; GCN3-NEXT: v_mov_b32_e32 v2, s28981; GCN3-NEXT: flat_store_short v[0:1], v28982; GCN3-NEXT: s_endpgm8983;8984; GFX11-TRUE16-LABEL: atomic_store_bf16_offset:8985; GFX11-TRUE16: ; %bb.0:8986; GFX11-TRUE16-NEXT: s_clause 0x18987; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c8988; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x248989; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)8990; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s08991; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s28992; GFX11-TRUE16-NEXT: flat_store_b16 v[1:2], v08993; GFX11-TRUE16-NEXT: s_endpgm8994;8995; GFX11-FAKE16-LABEL: atomic_store_bf16_offset:8996; GFX11-FAKE16: ; %bb.0:8997; GFX11-FAKE16-NEXT: s_clause 0x18998; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c8999; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x249000; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)9001; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s19002; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s29003; GFX11-FAKE16-NEXT: flat_store_b16 v[0:1], v29004; GFX11-FAKE16-NEXT: s_endpgm9005 %gep = getelementptr inbounds bfloat, ptr %out, i64 89006 store atomic bfloat %in, ptr %out seq_cst, align 29007 ret void9008}9009 9010define amdgpu_kernel void @atomic_store_bf16(bfloat %in, ptr %out) {9011; GCN1-LABEL: atomic_store_bf16:9012; GCN1: ; %bb.0:9013; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xb9014; GCN1-NEXT: s_load_dword s2, s[4:5], 0x99015; GCN1-NEXT: s_waitcnt lgkmcnt(0)9016; GCN1-NEXT: v_mov_b32_e32 v0, s09017; GCN1-NEXT: v_mov_b32_e32 v1, s19018; GCN1-NEXT: v_mov_b32_e32 v2, s29019; GCN1-NEXT: flat_store_short v[0:1], v29020; GCN1-NEXT: s_endpgm9021;9022; GCN2-LABEL: atomic_store_bf16:9023; GCN2: ; %bb.0:9024; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c9025; GCN2-NEXT: s_load_dword s2, s[4:5], 0x249026; GCN2-NEXT: s_waitcnt lgkmcnt(0)9027; GCN2-NEXT: v_mov_b32_e32 v0, s09028; GCN2-NEXT: v_mov_b32_e32 v1, s19029; GCN2-NEXT: v_mov_b32_e32 v2, s29030; GCN2-NEXT: flat_store_short v[0:1], v29031; GCN2-NEXT: s_endpgm9032;9033; GCN3-LABEL: atomic_store_bf16:9034; GCN3: ; %bb.0:9035; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c9036; GCN3-NEXT: s_load_dword s2, s[4:5], 0x249037; GCN3-NEXT: s_waitcnt lgkmcnt(0)9038; GCN3-NEXT: v_mov_b32_e32 v0, s09039; GCN3-NEXT: v_mov_b32_e32 v1, s19040; GCN3-NEXT: v_mov_b32_e32 v2, s29041; GCN3-NEXT: flat_store_short v[0:1], v29042; GCN3-NEXT: s_endpgm9043;9044; GFX11-TRUE16-LABEL: atomic_store_bf16:9045; GFX11-TRUE16: ; %bb.0:9046; GFX11-TRUE16-NEXT: s_clause 0x19047; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c9048; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x249049; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)9050; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s09051; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s29052; GFX11-TRUE16-NEXT: flat_store_b16 v[1:2], v09053; GFX11-TRUE16-NEXT: s_endpgm9054;9055; GFX11-FAKE16-LABEL: atomic_store_bf16:9056; GFX11-FAKE16: ; %bb.0:9057; GFX11-FAKE16-NEXT: s_clause 0x19058; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c9059; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x249060; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)9061; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s19062; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s29063; GFX11-FAKE16-NEXT: flat_store_b16 v[0:1], v29064; GFX11-FAKE16-NEXT: s_endpgm9065 store atomic bfloat %in, ptr %out seq_cst, align 29066 ret void9067}9068 9069define amdgpu_kernel void @atomic_inc_i32_offset(ptr %out, i32 %in) {9070; GCN1-LABEL: atomic_inc_i32_offset:9071; GCN1: ; %bb.0: ; %entry9072; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x99073; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb9074; GCN1-NEXT: s_waitcnt lgkmcnt(0)9075; GCN1-NEXT: s_add_u32 s0, s0, 169076; GCN1-NEXT: s_addc_u32 s1, s1, 09077; GCN1-NEXT: v_mov_b32_e32 v0, s09078; GCN1-NEXT: v_mov_b32_e32 v1, s19079; GCN1-NEXT: v_mov_b32_e32 v2, s29080; GCN1-NEXT: flat_atomic_inc v[0:1], v29081; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9082; GCN1-NEXT: buffer_wbinvl1_vol9083; GCN1-NEXT: s_endpgm9084;9085; GCN2-LABEL: atomic_inc_i32_offset:9086; GCN2: ; %bb.0: ; %entry9087; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x249088; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c9089; GCN2-NEXT: s_waitcnt lgkmcnt(0)9090; GCN2-NEXT: s_add_u32 s0, s0, 169091; GCN2-NEXT: s_addc_u32 s1, s1, 09092; GCN2-NEXT: v_mov_b32_e32 v0, s09093; GCN2-NEXT: v_mov_b32_e32 v1, s19094; GCN2-NEXT: v_mov_b32_e32 v2, s29095; GCN2-NEXT: flat_atomic_inc v[0:1], v29096; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9097; GCN2-NEXT: buffer_wbinvl1_vol9098; GCN2-NEXT: s_endpgm9099;9100; GCN3-LABEL: atomic_inc_i32_offset:9101; GCN3: ; %bb.0: ; %entry9102; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x249103; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c9104; GCN3-NEXT: s_waitcnt lgkmcnt(0)9105; GCN3-NEXT: v_mov_b32_e32 v0, s09106; GCN3-NEXT: v_mov_b32_e32 v1, s19107; GCN3-NEXT: v_mov_b32_e32 v2, s29108; GCN3-NEXT: flat_atomic_inc v[0:1], v2 offset:169109; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9110; GCN3-NEXT: buffer_wbinvl1_vol9111; GCN3-NEXT: s_endpgm9112;9113; GFX11-LABEL: atomic_inc_i32_offset:9114; GFX11: ; %bb.0: ; %entry9115; GFX11-NEXT: s_clause 0x19116; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x249117; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c9118; GFX11-NEXT: s_waitcnt lgkmcnt(0)9119; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s19120; GFX11-NEXT: v_mov_b32_e32 v2, s29121; GFX11-NEXT: flat_atomic_inc_u32 v[0:1], v2 offset:169122; GFX11-NEXT: s_waitcnt lgkmcnt(0)9123; GFX11-NEXT: s_waitcnt_vscnt null, 0x09124; GFX11-NEXT: buffer_gl1_inv9125; GFX11-NEXT: buffer_gl0_inv9126; GFX11-NEXT: s_endpgm9127entry:9128 %gep = getelementptr inbounds i32, ptr %out, i32 49129 %val = atomicrmw volatile uinc_wrap ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !09130 ret void9131}9132 9133define amdgpu_kernel void @atomic_inc_i32_max_offset(ptr %out, i32 %in) {9134; GCN1-LABEL: atomic_inc_i32_max_offset:9135; GCN1: ; %bb.0: ; %entry9136; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x99137; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb9138; GCN1-NEXT: s_waitcnt lgkmcnt(0)9139; GCN1-NEXT: s_add_u32 s0, s0, 0xffc9140; GCN1-NEXT: s_addc_u32 s1, s1, 09141; GCN1-NEXT: v_mov_b32_e32 v0, s09142; GCN1-NEXT: v_mov_b32_e32 v1, s19143; GCN1-NEXT: v_mov_b32_e32 v2, s29144; GCN1-NEXT: flat_atomic_inc v[0:1], v29145; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9146; GCN1-NEXT: buffer_wbinvl1_vol9147; GCN1-NEXT: s_endpgm9148;9149; GCN2-LABEL: atomic_inc_i32_max_offset:9150; GCN2: ; %bb.0: ; %entry9151; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x249152; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c9153; GCN2-NEXT: s_waitcnt lgkmcnt(0)9154; GCN2-NEXT: s_add_u32 s0, s0, 0xffc9155; GCN2-NEXT: s_addc_u32 s1, s1, 09156; GCN2-NEXT: v_mov_b32_e32 v0, s09157; GCN2-NEXT: v_mov_b32_e32 v1, s19158; GCN2-NEXT: v_mov_b32_e32 v2, s29159; GCN2-NEXT: flat_atomic_inc v[0:1], v29160; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9161; GCN2-NEXT: buffer_wbinvl1_vol9162; GCN2-NEXT: s_endpgm9163;9164; GCN3-LABEL: atomic_inc_i32_max_offset:9165; GCN3: ; %bb.0: ; %entry9166; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x249167; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c9168; GCN3-NEXT: s_waitcnt lgkmcnt(0)9169; GCN3-NEXT: v_mov_b32_e32 v0, s09170; GCN3-NEXT: v_mov_b32_e32 v1, s19171; GCN3-NEXT: v_mov_b32_e32 v2, s29172; GCN3-NEXT: flat_atomic_inc v[0:1], v2 offset:40929173; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9174; GCN3-NEXT: buffer_wbinvl1_vol9175; GCN3-NEXT: s_endpgm9176;9177; GFX11-LABEL: atomic_inc_i32_max_offset:9178; GFX11: ; %bb.0: ; %entry9179; GFX11-NEXT: s_clause 0x19180; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x249181; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c9182; GFX11-NEXT: s_waitcnt lgkmcnt(0)9183; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s19184; GFX11-NEXT: v_mov_b32_e32 v2, s29185; GFX11-NEXT: flat_atomic_inc_u32 v[0:1], v2 offset:40929186; GFX11-NEXT: s_waitcnt lgkmcnt(0)9187; GFX11-NEXT: s_waitcnt_vscnt null, 0x09188; GFX11-NEXT: buffer_gl1_inv9189; GFX11-NEXT: buffer_gl0_inv9190; GFX11-NEXT: s_endpgm9191entry:9192 %gep = getelementptr inbounds i32, ptr %out, i32 10239193 %val = atomicrmw volatile uinc_wrap ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !09194 ret void9195}9196 9197define amdgpu_kernel void @atomic_inc_i32_max_offset_p1(ptr %out, i32 %in) {9198; GCN1-LABEL: atomic_inc_i32_max_offset_p1:9199; GCN1: ; %bb.0: ; %entry9200; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x99201; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb9202; GCN1-NEXT: s_waitcnt lgkmcnt(0)9203; GCN1-NEXT: s_add_u32 s0, s0, 0x10009204; GCN1-NEXT: s_addc_u32 s1, s1, 09205; GCN1-NEXT: v_mov_b32_e32 v0, s09206; GCN1-NEXT: v_mov_b32_e32 v1, s19207; GCN1-NEXT: v_mov_b32_e32 v2, s29208; GCN1-NEXT: flat_atomic_inc v[0:1], v29209; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9210; GCN1-NEXT: buffer_wbinvl1_vol9211; GCN1-NEXT: s_endpgm9212;9213; GCN2-LABEL: atomic_inc_i32_max_offset_p1:9214; GCN2: ; %bb.0: ; %entry9215; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x249216; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c9217; GCN2-NEXT: s_waitcnt lgkmcnt(0)9218; GCN2-NEXT: s_add_u32 s0, s0, 0x10009219; GCN2-NEXT: s_addc_u32 s1, s1, 09220; GCN2-NEXT: v_mov_b32_e32 v0, s09221; GCN2-NEXT: v_mov_b32_e32 v1, s19222; GCN2-NEXT: v_mov_b32_e32 v2, s29223; GCN2-NEXT: flat_atomic_inc v[0:1], v29224; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9225; GCN2-NEXT: buffer_wbinvl1_vol9226; GCN2-NEXT: s_endpgm9227;9228; GCN3-LABEL: atomic_inc_i32_max_offset_p1:9229; GCN3: ; %bb.0: ; %entry9230; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x249231; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c9232; GCN3-NEXT: s_waitcnt lgkmcnt(0)9233; GCN3-NEXT: v_mov_b32_e32 v0, s09234; GCN3-NEXT: v_mov_b32_e32 v1, s19235; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 0x1000, v09236; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc9237; GCN3-NEXT: v_mov_b32_e32 v2, s29238; GCN3-NEXT: flat_atomic_inc v[0:1], v29239; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9240; GCN3-NEXT: buffer_wbinvl1_vol9241; GCN3-NEXT: s_endpgm9242;9243; GFX11-LABEL: atomic_inc_i32_max_offset_p1:9244; GFX11: ; %bb.0: ; %entry9245; GFX11-NEXT: s_clause 0x19246; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x249247; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c9248; GFX11-NEXT: s_waitcnt lgkmcnt(0)9249; GFX11-NEXT: v_add_co_u32 v0, s0, 0x1000, s09250; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)9251; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, s1, s09252; GFX11-NEXT: v_mov_b32_e32 v2, s29253; GFX11-NEXT: flat_atomic_inc_u32 v[0:1], v29254; GFX11-NEXT: s_waitcnt lgkmcnt(0)9255; GFX11-NEXT: s_waitcnt_vscnt null, 0x09256; GFX11-NEXT: buffer_gl1_inv9257; GFX11-NEXT: buffer_gl0_inv9258; GFX11-NEXT: s_endpgm9259entry:9260 %gep = getelementptr inbounds i32, ptr %out, i32 10249261 %val = atomicrmw volatile uinc_wrap ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !09262 ret void9263}9264 9265define amdgpu_kernel void @atomic_inc_i32_ret_offset(ptr %out, ptr %out2, i32 %in) {9266; GCN1-LABEL: atomic_inc_i32_ret_offset:9267; GCN1: ; %bb.0: ; %entry9268; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x99269; GCN1-NEXT: s_load_dword s4, s[4:5], 0xd9270; GCN1-NEXT: s_waitcnt lgkmcnt(0)9271; GCN1-NEXT: s_add_u32 s0, s0, 169272; GCN1-NEXT: s_addc_u32 s1, s1, 09273; GCN1-NEXT: v_mov_b32_e32 v0, s09274; GCN1-NEXT: v_mov_b32_e32 v1, s19275; GCN1-NEXT: v_mov_b32_e32 v2, s49276; GCN1-NEXT: flat_atomic_inc v2, v[0:1], v2 glc9277; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9278; GCN1-NEXT: buffer_wbinvl1_vol9279; GCN1-NEXT: v_mov_b32_e32 v0, s29280; GCN1-NEXT: v_mov_b32_e32 v1, s39281; GCN1-NEXT: flat_store_dword v[0:1], v29282; GCN1-NEXT: s_endpgm9283;9284; GCN2-LABEL: atomic_inc_i32_ret_offset:9285; GCN2: ; %bb.0: ; %entry9286; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249287; GCN2-NEXT: s_load_dword s4, s[4:5], 0x349288; GCN2-NEXT: s_waitcnt lgkmcnt(0)9289; GCN2-NEXT: s_add_u32 s0, s0, 169290; GCN2-NEXT: s_addc_u32 s1, s1, 09291; GCN2-NEXT: v_mov_b32_e32 v0, s09292; GCN2-NEXT: v_mov_b32_e32 v1, s19293; GCN2-NEXT: v_mov_b32_e32 v2, s49294; GCN2-NEXT: flat_atomic_inc v2, v[0:1], v2 glc9295; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9296; GCN2-NEXT: buffer_wbinvl1_vol9297; GCN2-NEXT: v_mov_b32_e32 v0, s29298; GCN2-NEXT: v_mov_b32_e32 v1, s39299; GCN2-NEXT: flat_store_dword v[0:1], v29300; GCN2-NEXT: s_endpgm9301;9302; GCN3-LABEL: atomic_inc_i32_ret_offset:9303; GCN3: ; %bb.0: ; %entry9304; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249305; GCN3-NEXT: s_load_dword s6, s[4:5], 0x349306; GCN3-NEXT: s_waitcnt lgkmcnt(0)9307; GCN3-NEXT: v_mov_b32_e32 v0, s09308; GCN3-NEXT: v_mov_b32_e32 v1, s19309; GCN3-NEXT: v_mov_b32_e32 v2, s69310; GCN3-NEXT: flat_atomic_inc v2, v[0:1], v2 offset:16 glc9311; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9312; GCN3-NEXT: buffer_wbinvl1_vol9313; GCN3-NEXT: v_mov_b32_e32 v0, s29314; GCN3-NEXT: v_mov_b32_e32 v1, s39315; GCN3-NEXT: flat_store_dword v[0:1], v29316; GCN3-NEXT: s_endpgm9317;9318; GFX11-LABEL: atomic_inc_i32_ret_offset:9319; GFX11: ; %bb.0: ; %entry9320; GFX11-NEXT: s_clause 0x19321; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x249322; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x349323; GFX11-NEXT: s_waitcnt lgkmcnt(0)9324; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s19325; GFX11-NEXT: v_mov_b32_e32 v2, s49326; GFX11-NEXT: flat_atomic_inc_u32 v2, v[0:1], v2 offset:16 glc9327; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9328; GFX11-NEXT: buffer_gl1_inv9329; GFX11-NEXT: buffer_gl0_inv9330; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s39331; GFX11-NEXT: flat_store_b32 v[0:1], v29332; GFX11-NEXT: s_endpgm9333entry:9334 %gep = getelementptr inbounds i32, ptr %out, i32 49335 %val = atomicrmw volatile uinc_wrap ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !09336 store i32 %val, ptr %out29337 ret void9338}9339 9340define amdgpu_kernel void @atomic_inc_i32_incr64_offset(ptr %out, i32 %in, i64 %index) {9341; GCN1-LABEL: atomic_inc_i32_incr64_offset:9342; GCN1: ; %bb.0: ; %entry9343; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd9344; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x99345; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb9346; GCN1-NEXT: s_waitcnt lgkmcnt(0)9347; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 29348; GCN1-NEXT: s_add_u32 s0, s2, s09349; GCN1-NEXT: s_addc_u32 s1, s3, s19350; GCN1-NEXT: s_add_u32 s0, s0, 169351; GCN1-NEXT: s_addc_u32 s1, s1, 09352; GCN1-NEXT: v_mov_b32_e32 v0, s09353; GCN1-NEXT: v_mov_b32_e32 v1, s19354; GCN1-NEXT: v_mov_b32_e32 v2, s49355; GCN1-NEXT: flat_atomic_inc v[0:1], v29356; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9357; GCN1-NEXT: buffer_wbinvl1_vol9358; GCN1-NEXT: s_endpgm9359;9360; GCN2-LABEL: atomic_inc_i32_incr64_offset:9361; GCN2: ; %bb.0: ; %entry9362; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x349363; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x249364; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c9365; GCN2-NEXT: s_waitcnt lgkmcnt(0)9366; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 29367; GCN2-NEXT: s_add_u32 s0, s2, s09368; GCN2-NEXT: s_addc_u32 s1, s3, s19369; GCN2-NEXT: s_add_u32 s0, s0, 169370; GCN2-NEXT: s_addc_u32 s1, s1, 09371; GCN2-NEXT: v_mov_b32_e32 v0, s09372; GCN2-NEXT: v_mov_b32_e32 v1, s19373; GCN2-NEXT: v_mov_b32_e32 v2, s49374; GCN2-NEXT: flat_atomic_inc v[0:1], v29375; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9376; GCN2-NEXT: buffer_wbinvl1_vol9377; GCN2-NEXT: s_endpgm9378;9379; GCN3-LABEL: atomic_inc_i32_incr64_offset:9380; GCN3: ; %bb.0: ; %entry9381; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x349382; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x249383; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c9384; GCN3-NEXT: s_waitcnt lgkmcnt(0)9385; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 29386; GCN3-NEXT: s_add_u32 s0, s2, s09387; GCN3-NEXT: s_addc_u32 s1, s3, s19388; GCN3-NEXT: v_mov_b32_e32 v0, s09389; GCN3-NEXT: v_mov_b32_e32 v1, s19390; GCN3-NEXT: v_mov_b32_e32 v2, s69391; GCN3-NEXT: flat_atomic_inc v[0:1], v2 offset:169392; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9393; GCN3-NEXT: buffer_wbinvl1_vol9394; GCN3-NEXT: s_endpgm9395;9396; GFX11-LABEL: atomic_inc_i32_incr64_offset:9397; GFX11: ; %bb.0: ; %entry9398; GFX11-NEXT: s_clause 0x29399; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x349400; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x249401; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c9402; GFX11-NEXT: s_waitcnt lgkmcnt(0)9403; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 29404; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)9405; GFX11-NEXT: s_add_u32 s0, s2, s09406; GFX11-NEXT: s_addc_u32 s1, s3, s19407; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s19408; GFX11-NEXT: v_mov_b32_e32 v2, s49409; GFX11-NEXT: flat_atomic_inc_u32 v[0:1], v2 offset:169410; GFX11-NEXT: s_waitcnt lgkmcnt(0)9411; GFX11-NEXT: s_waitcnt_vscnt null, 0x09412; GFX11-NEXT: buffer_gl1_inv9413; GFX11-NEXT: buffer_gl0_inv9414; GFX11-NEXT: s_endpgm9415entry:9416 %ptr = getelementptr inbounds i32, ptr %out, i64 %index9417 %gep = getelementptr inbounds i32, ptr %ptr, i32 49418 %val = atomicrmw volatile uinc_wrap ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !09419 ret void9420}9421 9422define amdgpu_kernel void @atomic_inc_i32_ret_incr64_offset(ptr %out, ptr %out2, i32 %in, i64 %index) {9423; GCN1-LABEL: atomic_inc_i32_ret_incr64_offset:9424; GCN1: ; %bb.0: ; %entry9425; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf9426; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x99427; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd9428; GCN1-NEXT: s_waitcnt lgkmcnt(0)9429; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 29430; GCN1-NEXT: s_add_u32 s0, s0, s49431; GCN1-NEXT: s_addc_u32 s1, s1, s59432; GCN1-NEXT: s_add_u32 s0, s0, 169433; GCN1-NEXT: s_addc_u32 s1, s1, 09434; GCN1-NEXT: v_mov_b32_e32 v0, s09435; GCN1-NEXT: v_mov_b32_e32 v1, s19436; GCN1-NEXT: v_mov_b32_e32 v2, s89437; GCN1-NEXT: flat_atomic_inc v2, v[0:1], v2 glc9438; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9439; GCN1-NEXT: buffer_wbinvl1_vol9440; GCN1-NEXT: v_mov_b32_e32 v0, s29441; GCN1-NEXT: v_mov_b32_e32 v1, s39442; GCN1-NEXT: flat_store_dword v[0:1], v29443; GCN1-NEXT: s_endpgm9444;9445; GCN2-LABEL: atomic_inc_i32_ret_incr64_offset:9446; GCN2: ; %bb.0: ; %entry9447; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c9448; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249449; GCN2-NEXT: s_load_dword s8, s[4:5], 0x349450; GCN2-NEXT: s_waitcnt lgkmcnt(0)9451; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 29452; GCN2-NEXT: s_add_u32 s0, s0, s49453; GCN2-NEXT: s_addc_u32 s1, s1, s59454; GCN2-NEXT: s_add_u32 s0, s0, 169455; GCN2-NEXT: s_addc_u32 s1, s1, 09456; GCN2-NEXT: v_mov_b32_e32 v0, s09457; GCN2-NEXT: v_mov_b32_e32 v1, s19458; GCN2-NEXT: v_mov_b32_e32 v2, s89459; GCN2-NEXT: flat_atomic_inc v2, v[0:1], v2 glc9460; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9461; GCN2-NEXT: buffer_wbinvl1_vol9462; GCN2-NEXT: v_mov_b32_e32 v0, s29463; GCN2-NEXT: v_mov_b32_e32 v1, s39464; GCN2-NEXT: flat_store_dword v[0:1], v29465; GCN2-NEXT: s_endpgm9466;9467; GCN3-LABEL: atomic_inc_i32_ret_incr64_offset:9468; GCN3: ; %bb.0: ; %entry9469; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c9470; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249471; GCN3-NEXT: s_load_dword s8, s[4:5], 0x349472; GCN3-NEXT: s_waitcnt lgkmcnt(0)9473; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 29474; GCN3-NEXT: s_add_u32 s0, s0, s49475; GCN3-NEXT: s_addc_u32 s1, s1, s59476; GCN3-NEXT: v_mov_b32_e32 v0, s09477; GCN3-NEXT: v_mov_b32_e32 v1, s19478; GCN3-NEXT: v_mov_b32_e32 v2, s89479; GCN3-NEXT: flat_atomic_inc v2, v[0:1], v2 offset:16 glc9480; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9481; GCN3-NEXT: buffer_wbinvl1_vol9482; GCN3-NEXT: v_mov_b32_e32 v0, s29483; GCN3-NEXT: v_mov_b32_e32 v1, s39484; GCN3-NEXT: flat_store_dword v[0:1], v29485; GCN3-NEXT: s_endpgm9486;9487; GFX11-LABEL: atomic_inc_i32_ret_incr64_offset:9488; GFX11: ; %bb.0: ; %entry9489; GFX11-NEXT: s_clause 0x29490; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c9491; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x249492; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x349493; GFX11-NEXT: s_waitcnt lgkmcnt(0)9494; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 29495; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)9496; GFX11-NEXT: s_add_u32 s0, s0, s49497; GFX11-NEXT: s_addc_u32 s1, s1, s59498; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s19499; GFX11-NEXT: v_mov_b32_e32 v2, s89500; GFX11-NEXT: flat_atomic_inc_u32 v2, v[0:1], v2 offset:16 glc9501; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9502; GFX11-NEXT: buffer_gl1_inv9503; GFX11-NEXT: buffer_gl0_inv9504; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s39505; GFX11-NEXT: flat_store_b32 v[0:1], v29506; GFX11-NEXT: s_endpgm9507entry:9508 %ptr = getelementptr inbounds i32, ptr %out, i64 %index9509 %gep = getelementptr inbounds i32, ptr %ptr, i32 49510 %val = atomicrmw volatile uinc_wrap ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !09511 store i32 %val, ptr %out29512 ret void9513}9514 9515define amdgpu_kernel void @atomic_inc_i32(ptr %out, i32 %in) {9516; GCN1-LABEL: atomic_inc_i32:9517; GCN1: ; %bb.0: ; %entry9518; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x99519; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb9520; GCN1-NEXT: s_waitcnt lgkmcnt(0)9521; GCN1-NEXT: v_mov_b32_e32 v0, s09522; GCN1-NEXT: v_mov_b32_e32 v1, s19523; GCN1-NEXT: v_mov_b32_e32 v2, s29524; GCN1-NEXT: flat_atomic_inc v[0:1], v29525; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9526; GCN1-NEXT: buffer_wbinvl1_vol9527; GCN1-NEXT: s_endpgm9528;9529; GCN2-LABEL: atomic_inc_i32:9530; GCN2: ; %bb.0: ; %entry9531; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x249532; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c9533; GCN2-NEXT: s_waitcnt lgkmcnt(0)9534; GCN2-NEXT: v_mov_b32_e32 v0, s09535; GCN2-NEXT: v_mov_b32_e32 v1, s19536; GCN2-NEXT: v_mov_b32_e32 v2, s29537; GCN2-NEXT: flat_atomic_inc v[0:1], v29538; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9539; GCN2-NEXT: buffer_wbinvl1_vol9540; GCN2-NEXT: s_endpgm9541;9542; GCN3-LABEL: atomic_inc_i32:9543; GCN3: ; %bb.0: ; %entry9544; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x249545; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c9546; GCN3-NEXT: s_waitcnt lgkmcnt(0)9547; GCN3-NEXT: v_mov_b32_e32 v0, s09548; GCN3-NEXT: v_mov_b32_e32 v1, s19549; GCN3-NEXT: v_mov_b32_e32 v2, s29550; GCN3-NEXT: flat_atomic_inc v[0:1], v29551; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9552; GCN3-NEXT: buffer_wbinvl1_vol9553; GCN3-NEXT: s_endpgm9554;9555; GFX11-LABEL: atomic_inc_i32:9556; GFX11: ; %bb.0: ; %entry9557; GFX11-NEXT: s_clause 0x19558; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x249559; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c9560; GFX11-NEXT: s_waitcnt lgkmcnt(0)9561; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s19562; GFX11-NEXT: v_mov_b32_e32 v2, s29563; GFX11-NEXT: flat_atomic_inc_u32 v[0:1], v29564; GFX11-NEXT: s_waitcnt lgkmcnt(0)9565; GFX11-NEXT: s_waitcnt_vscnt null, 0x09566; GFX11-NEXT: buffer_gl1_inv9567; GFX11-NEXT: buffer_gl0_inv9568; GFX11-NEXT: s_endpgm9569entry:9570 %val = atomicrmw volatile uinc_wrap ptr %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !09571 ret void9572}9573 9574define amdgpu_kernel void @atomic_inc_i32_ret(ptr %out, ptr %out2, i32 %in) {9575; GCN1-LABEL: atomic_inc_i32_ret:9576; GCN1: ; %bb.0: ; %entry9577; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x99578; GCN1-NEXT: s_load_dword s4, s[4:5], 0xd9579; GCN1-NEXT: s_waitcnt lgkmcnt(0)9580; GCN1-NEXT: v_mov_b32_e32 v0, s09581; GCN1-NEXT: v_mov_b32_e32 v1, s19582; GCN1-NEXT: v_mov_b32_e32 v2, s49583; GCN1-NEXT: flat_atomic_inc v2, v[0:1], v2 glc9584; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9585; GCN1-NEXT: buffer_wbinvl1_vol9586; GCN1-NEXT: v_mov_b32_e32 v0, s29587; GCN1-NEXT: v_mov_b32_e32 v1, s39588; GCN1-NEXT: flat_store_dword v[0:1], v29589; GCN1-NEXT: s_endpgm9590;9591; GCN2-LABEL: atomic_inc_i32_ret:9592; GCN2: ; %bb.0: ; %entry9593; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249594; GCN2-NEXT: s_load_dword s4, s[4:5], 0x349595; GCN2-NEXT: s_waitcnt lgkmcnt(0)9596; GCN2-NEXT: v_mov_b32_e32 v0, s09597; GCN2-NEXT: v_mov_b32_e32 v1, s19598; GCN2-NEXT: v_mov_b32_e32 v2, s49599; GCN2-NEXT: flat_atomic_inc v2, v[0:1], v2 glc9600; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9601; GCN2-NEXT: buffer_wbinvl1_vol9602; GCN2-NEXT: v_mov_b32_e32 v0, s29603; GCN2-NEXT: v_mov_b32_e32 v1, s39604; GCN2-NEXT: flat_store_dword v[0:1], v29605; GCN2-NEXT: s_endpgm9606;9607; GCN3-LABEL: atomic_inc_i32_ret:9608; GCN3: ; %bb.0: ; %entry9609; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249610; GCN3-NEXT: s_load_dword s6, s[4:5], 0x349611; GCN3-NEXT: s_waitcnt lgkmcnt(0)9612; GCN3-NEXT: v_mov_b32_e32 v0, s09613; GCN3-NEXT: v_mov_b32_e32 v1, s19614; GCN3-NEXT: v_mov_b32_e32 v2, s69615; GCN3-NEXT: flat_atomic_inc v2, v[0:1], v2 glc9616; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9617; GCN3-NEXT: buffer_wbinvl1_vol9618; GCN3-NEXT: v_mov_b32_e32 v0, s29619; GCN3-NEXT: v_mov_b32_e32 v1, s39620; GCN3-NEXT: flat_store_dword v[0:1], v29621; GCN3-NEXT: s_endpgm9622;9623; GFX11-LABEL: atomic_inc_i32_ret:9624; GFX11: ; %bb.0: ; %entry9625; GFX11-NEXT: s_clause 0x19626; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x249627; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x349628; GFX11-NEXT: s_waitcnt lgkmcnt(0)9629; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s19630; GFX11-NEXT: v_mov_b32_e32 v2, s49631; GFX11-NEXT: flat_atomic_inc_u32 v2, v[0:1], v2 glc9632; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9633; GFX11-NEXT: buffer_gl1_inv9634; GFX11-NEXT: buffer_gl0_inv9635; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s39636; GFX11-NEXT: flat_store_b32 v[0:1], v29637; GFX11-NEXT: s_endpgm9638entry:9639 %val = atomicrmw volatile uinc_wrap ptr %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !09640 store i32 %val, ptr %out29641 ret void9642}9643 9644define amdgpu_kernel void @atomic_inc_i32_incr64(ptr %out, i32 %in, i64 %index) {9645; GCN1-LABEL: atomic_inc_i32_incr64:9646; GCN1: ; %bb.0: ; %entry9647; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd9648; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x99649; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb9650; GCN1-NEXT: s_waitcnt lgkmcnt(0)9651; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 29652; GCN1-NEXT: s_add_u32 s0, s2, s09653; GCN1-NEXT: s_addc_u32 s1, s3, s19654; GCN1-NEXT: v_mov_b32_e32 v0, s09655; GCN1-NEXT: v_mov_b32_e32 v1, s19656; GCN1-NEXT: v_mov_b32_e32 v2, s49657; GCN1-NEXT: flat_atomic_inc v[0:1], v29658; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9659; GCN1-NEXT: buffer_wbinvl1_vol9660; GCN1-NEXT: s_endpgm9661;9662; GCN2-LABEL: atomic_inc_i32_incr64:9663; GCN2: ; %bb.0: ; %entry9664; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x349665; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x249666; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c9667; GCN2-NEXT: s_waitcnt lgkmcnt(0)9668; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 29669; GCN2-NEXT: s_add_u32 s0, s2, s09670; GCN2-NEXT: s_addc_u32 s1, s3, s19671; GCN2-NEXT: v_mov_b32_e32 v0, s09672; GCN2-NEXT: v_mov_b32_e32 v1, s19673; GCN2-NEXT: v_mov_b32_e32 v2, s49674; GCN2-NEXT: flat_atomic_inc v[0:1], v29675; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9676; GCN2-NEXT: buffer_wbinvl1_vol9677; GCN2-NEXT: s_endpgm9678;9679; GCN3-LABEL: atomic_inc_i32_incr64:9680; GCN3: ; %bb.0: ; %entry9681; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x349682; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x249683; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c9684; GCN3-NEXT: s_waitcnt lgkmcnt(0)9685; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 29686; GCN3-NEXT: s_add_u32 s0, s2, s09687; GCN3-NEXT: s_addc_u32 s1, s3, s19688; GCN3-NEXT: v_mov_b32_e32 v0, s09689; GCN3-NEXT: v_mov_b32_e32 v1, s19690; GCN3-NEXT: v_mov_b32_e32 v2, s69691; GCN3-NEXT: flat_atomic_inc v[0:1], v29692; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9693; GCN3-NEXT: buffer_wbinvl1_vol9694; GCN3-NEXT: s_endpgm9695;9696; GFX11-LABEL: atomic_inc_i32_incr64:9697; GFX11: ; %bb.0: ; %entry9698; GFX11-NEXT: s_clause 0x29699; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x349700; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x249701; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c9702; GFX11-NEXT: s_waitcnt lgkmcnt(0)9703; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 29704; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)9705; GFX11-NEXT: s_add_u32 s0, s2, s09706; GFX11-NEXT: s_addc_u32 s1, s3, s19707; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s19708; GFX11-NEXT: v_mov_b32_e32 v2, s49709; GFX11-NEXT: flat_atomic_inc_u32 v[0:1], v29710; GFX11-NEXT: s_waitcnt lgkmcnt(0)9711; GFX11-NEXT: s_waitcnt_vscnt null, 0x09712; GFX11-NEXT: buffer_gl1_inv9713; GFX11-NEXT: buffer_gl0_inv9714; GFX11-NEXT: s_endpgm9715entry:9716 %ptr = getelementptr inbounds i32, ptr %out, i64 %index9717 %val = atomicrmw volatile uinc_wrap ptr %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !09718 ret void9719}9720 9721define amdgpu_kernel void @atomic_inc_i32_ret_incr64(ptr %out, ptr %out2, i32 %in, i64 %index) {9722; GCN1-LABEL: atomic_inc_i32_ret_incr64:9723; GCN1: ; %bb.0: ; %entry9724; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf9725; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x99726; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd9727; GCN1-NEXT: s_waitcnt lgkmcnt(0)9728; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 29729; GCN1-NEXT: s_add_u32 s0, s0, s49730; GCN1-NEXT: s_addc_u32 s1, s1, s59731; GCN1-NEXT: v_mov_b32_e32 v0, s09732; GCN1-NEXT: v_mov_b32_e32 v1, s19733; GCN1-NEXT: v_mov_b32_e32 v2, s89734; GCN1-NEXT: flat_atomic_inc v2, v[0:1], v2 glc9735; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9736; GCN1-NEXT: buffer_wbinvl1_vol9737; GCN1-NEXT: v_mov_b32_e32 v0, s29738; GCN1-NEXT: v_mov_b32_e32 v1, s39739; GCN1-NEXT: flat_store_dword v[0:1], v29740; GCN1-NEXT: s_endpgm9741;9742; GCN2-LABEL: atomic_inc_i32_ret_incr64:9743; GCN2: ; %bb.0: ; %entry9744; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c9745; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249746; GCN2-NEXT: s_load_dword s8, s[4:5], 0x349747; GCN2-NEXT: s_waitcnt lgkmcnt(0)9748; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 29749; GCN2-NEXT: s_add_u32 s0, s0, s49750; GCN2-NEXT: s_addc_u32 s1, s1, s59751; GCN2-NEXT: v_mov_b32_e32 v0, s09752; GCN2-NEXT: v_mov_b32_e32 v1, s19753; GCN2-NEXT: v_mov_b32_e32 v2, s89754; GCN2-NEXT: flat_atomic_inc v2, v[0:1], v2 glc9755; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9756; GCN2-NEXT: buffer_wbinvl1_vol9757; GCN2-NEXT: v_mov_b32_e32 v0, s29758; GCN2-NEXT: v_mov_b32_e32 v1, s39759; GCN2-NEXT: flat_store_dword v[0:1], v29760; GCN2-NEXT: s_endpgm9761;9762; GCN3-LABEL: atomic_inc_i32_ret_incr64:9763; GCN3: ; %bb.0: ; %entry9764; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c9765; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249766; GCN3-NEXT: s_load_dword s8, s[4:5], 0x349767; GCN3-NEXT: s_waitcnt lgkmcnt(0)9768; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 29769; GCN3-NEXT: s_add_u32 s0, s0, s49770; GCN3-NEXT: s_addc_u32 s1, s1, s59771; GCN3-NEXT: v_mov_b32_e32 v0, s09772; GCN3-NEXT: v_mov_b32_e32 v1, s19773; GCN3-NEXT: v_mov_b32_e32 v2, s89774; GCN3-NEXT: flat_atomic_inc v2, v[0:1], v2 glc9775; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9776; GCN3-NEXT: buffer_wbinvl1_vol9777; GCN3-NEXT: v_mov_b32_e32 v0, s29778; GCN3-NEXT: v_mov_b32_e32 v1, s39779; GCN3-NEXT: flat_store_dword v[0:1], v29780; GCN3-NEXT: s_endpgm9781;9782; GFX11-LABEL: atomic_inc_i32_ret_incr64:9783; GFX11: ; %bb.0: ; %entry9784; GFX11-NEXT: s_clause 0x29785; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c9786; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x249787; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x349788; GFX11-NEXT: s_waitcnt lgkmcnt(0)9789; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 29790; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)9791; GFX11-NEXT: s_add_u32 s0, s0, s49792; GFX11-NEXT: s_addc_u32 s1, s1, s59793; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s19794; GFX11-NEXT: v_mov_b32_e32 v2, s89795; GFX11-NEXT: flat_atomic_inc_u32 v2, v[0:1], v2 glc9796; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9797; GFX11-NEXT: buffer_gl1_inv9798; GFX11-NEXT: buffer_gl0_inv9799; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s39800; GFX11-NEXT: flat_store_b32 v[0:1], v29801; GFX11-NEXT: s_endpgm9802entry:9803 %ptr = getelementptr inbounds i32, ptr %out, i64 %index9804 %val = atomicrmw volatile uinc_wrap ptr %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !09805 store i32 %val, ptr %out29806 ret void9807}9808 9809define amdgpu_kernel void @atomic_dec_i32_offset(ptr %out, i32 %in) {9810; GCN1-LABEL: atomic_dec_i32_offset:9811; GCN1: ; %bb.0: ; %entry9812; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x99813; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb9814; GCN1-NEXT: s_waitcnt lgkmcnt(0)9815; GCN1-NEXT: s_add_u32 s0, s0, 169816; GCN1-NEXT: s_addc_u32 s1, s1, 09817; GCN1-NEXT: v_mov_b32_e32 v0, s09818; GCN1-NEXT: v_mov_b32_e32 v1, s19819; GCN1-NEXT: v_mov_b32_e32 v2, s29820; GCN1-NEXT: flat_atomic_dec v[0:1], v29821; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9822; GCN1-NEXT: buffer_wbinvl1_vol9823; GCN1-NEXT: s_endpgm9824;9825; GCN2-LABEL: atomic_dec_i32_offset:9826; GCN2: ; %bb.0: ; %entry9827; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x249828; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c9829; GCN2-NEXT: s_waitcnt lgkmcnt(0)9830; GCN2-NEXT: s_add_u32 s0, s0, 169831; GCN2-NEXT: s_addc_u32 s1, s1, 09832; GCN2-NEXT: v_mov_b32_e32 v0, s09833; GCN2-NEXT: v_mov_b32_e32 v1, s19834; GCN2-NEXT: v_mov_b32_e32 v2, s29835; GCN2-NEXT: flat_atomic_dec v[0:1], v29836; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9837; GCN2-NEXT: buffer_wbinvl1_vol9838; GCN2-NEXT: s_endpgm9839;9840; GCN3-LABEL: atomic_dec_i32_offset:9841; GCN3: ; %bb.0: ; %entry9842; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x249843; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c9844; GCN3-NEXT: s_waitcnt lgkmcnt(0)9845; GCN3-NEXT: v_mov_b32_e32 v0, s09846; GCN3-NEXT: v_mov_b32_e32 v1, s19847; GCN3-NEXT: v_mov_b32_e32 v2, s29848; GCN3-NEXT: flat_atomic_dec v[0:1], v2 offset:169849; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9850; GCN3-NEXT: buffer_wbinvl1_vol9851; GCN3-NEXT: s_endpgm9852;9853; GFX11-LABEL: atomic_dec_i32_offset:9854; GFX11: ; %bb.0: ; %entry9855; GFX11-NEXT: s_clause 0x19856; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x249857; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c9858; GFX11-NEXT: s_waitcnt lgkmcnt(0)9859; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s19860; GFX11-NEXT: v_mov_b32_e32 v2, s29861; GFX11-NEXT: flat_atomic_dec_u32 v[0:1], v2 offset:169862; GFX11-NEXT: s_waitcnt lgkmcnt(0)9863; GFX11-NEXT: s_waitcnt_vscnt null, 0x09864; GFX11-NEXT: buffer_gl1_inv9865; GFX11-NEXT: buffer_gl0_inv9866; GFX11-NEXT: s_endpgm9867entry:9868 %gep = getelementptr inbounds i32, ptr %out, i32 49869 %val = atomicrmw volatile udec_wrap ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !09870 ret void9871}9872 9873define amdgpu_kernel void @atomic_dec_i32_max_offset(ptr %out, i32 %in) {9874; GCN1-LABEL: atomic_dec_i32_max_offset:9875; GCN1: ; %bb.0: ; %entry9876; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x99877; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb9878; GCN1-NEXT: s_waitcnt lgkmcnt(0)9879; GCN1-NEXT: s_add_u32 s0, s0, 0xffc9880; GCN1-NEXT: s_addc_u32 s1, s1, 09881; GCN1-NEXT: v_mov_b32_e32 v0, s09882; GCN1-NEXT: v_mov_b32_e32 v1, s19883; GCN1-NEXT: v_mov_b32_e32 v2, s29884; GCN1-NEXT: flat_atomic_dec v[0:1], v29885; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9886; GCN1-NEXT: buffer_wbinvl1_vol9887; GCN1-NEXT: s_endpgm9888;9889; GCN2-LABEL: atomic_dec_i32_max_offset:9890; GCN2: ; %bb.0: ; %entry9891; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x249892; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c9893; GCN2-NEXT: s_waitcnt lgkmcnt(0)9894; GCN2-NEXT: s_add_u32 s0, s0, 0xffc9895; GCN2-NEXT: s_addc_u32 s1, s1, 09896; GCN2-NEXT: v_mov_b32_e32 v0, s09897; GCN2-NEXT: v_mov_b32_e32 v1, s19898; GCN2-NEXT: v_mov_b32_e32 v2, s29899; GCN2-NEXT: flat_atomic_dec v[0:1], v29900; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9901; GCN2-NEXT: buffer_wbinvl1_vol9902; GCN2-NEXT: s_endpgm9903;9904; GCN3-LABEL: atomic_dec_i32_max_offset:9905; GCN3: ; %bb.0: ; %entry9906; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x249907; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c9908; GCN3-NEXT: s_waitcnt lgkmcnt(0)9909; GCN3-NEXT: v_mov_b32_e32 v0, s09910; GCN3-NEXT: v_mov_b32_e32 v1, s19911; GCN3-NEXT: v_mov_b32_e32 v2, s29912; GCN3-NEXT: flat_atomic_dec v[0:1], v2 offset:40929913; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9914; GCN3-NEXT: buffer_wbinvl1_vol9915; GCN3-NEXT: s_endpgm9916;9917; GFX11-LABEL: atomic_dec_i32_max_offset:9918; GFX11: ; %bb.0: ; %entry9919; GFX11-NEXT: s_clause 0x19920; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x249921; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c9922; GFX11-NEXT: s_waitcnt lgkmcnt(0)9923; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s19924; GFX11-NEXT: v_mov_b32_e32 v2, s29925; GFX11-NEXT: flat_atomic_dec_u32 v[0:1], v2 offset:40929926; GFX11-NEXT: s_waitcnt lgkmcnt(0)9927; GFX11-NEXT: s_waitcnt_vscnt null, 0x09928; GFX11-NEXT: buffer_gl1_inv9929; GFX11-NEXT: buffer_gl0_inv9930; GFX11-NEXT: s_endpgm9931entry:9932 %gep = getelementptr inbounds i32, ptr %out, i32 10239933 %val = atomicrmw volatile udec_wrap ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !09934 ret void9935}9936 9937define amdgpu_kernel void @atomic_dec_i32_max_offset_p1(ptr %out, i32 %in) {9938; GCN1-LABEL: atomic_dec_i32_max_offset_p1:9939; GCN1: ; %bb.0: ; %entry9940; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x99941; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb9942; GCN1-NEXT: s_waitcnt lgkmcnt(0)9943; GCN1-NEXT: s_add_u32 s0, s0, 0x10009944; GCN1-NEXT: s_addc_u32 s1, s1, 09945; GCN1-NEXT: v_mov_b32_e32 v0, s09946; GCN1-NEXT: v_mov_b32_e32 v1, s19947; GCN1-NEXT: v_mov_b32_e32 v2, s29948; GCN1-NEXT: flat_atomic_dec v[0:1], v29949; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9950; GCN1-NEXT: buffer_wbinvl1_vol9951; GCN1-NEXT: s_endpgm9952;9953; GCN2-LABEL: atomic_dec_i32_max_offset_p1:9954; GCN2: ; %bb.0: ; %entry9955; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x249956; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c9957; GCN2-NEXT: s_waitcnt lgkmcnt(0)9958; GCN2-NEXT: s_add_u32 s0, s0, 0x10009959; GCN2-NEXT: s_addc_u32 s1, s1, 09960; GCN2-NEXT: v_mov_b32_e32 v0, s09961; GCN2-NEXT: v_mov_b32_e32 v1, s19962; GCN2-NEXT: v_mov_b32_e32 v2, s29963; GCN2-NEXT: flat_atomic_dec v[0:1], v29964; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9965; GCN2-NEXT: buffer_wbinvl1_vol9966; GCN2-NEXT: s_endpgm9967;9968; GCN3-LABEL: atomic_dec_i32_max_offset_p1:9969; GCN3: ; %bb.0: ; %entry9970; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x249971; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c9972; GCN3-NEXT: s_waitcnt lgkmcnt(0)9973; GCN3-NEXT: v_mov_b32_e32 v0, s09974; GCN3-NEXT: v_mov_b32_e32 v1, s19975; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 0x1000, v09976; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc9977; GCN3-NEXT: v_mov_b32_e32 v2, s29978; GCN3-NEXT: flat_atomic_dec v[0:1], v29979; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9980; GCN3-NEXT: buffer_wbinvl1_vol9981; GCN3-NEXT: s_endpgm9982;9983; GFX11-LABEL: atomic_dec_i32_max_offset_p1:9984; GFX11: ; %bb.0: ; %entry9985; GFX11-NEXT: s_clause 0x19986; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x249987; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c9988; GFX11-NEXT: s_waitcnt lgkmcnt(0)9989; GFX11-NEXT: v_add_co_u32 v0, s0, 0x1000, s09990; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)9991; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, s1, s09992; GFX11-NEXT: v_mov_b32_e32 v2, s29993; GFX11-NEXT: flat_atomic_dec_u32 v[0:1], v29994; GFX11-NEXT: s_waitcnt lgkmcnt(0)9995; GFX11-NEXT: s_waitcnt_vscnt null, 0x09996; GFX11-NEXT: buffer_gl1_inv9997; GFX11-NEXT: buffer_gl0_inv9998; GFX11-NEXT: s_endpgm9999entry:10000 %gep = getelementptr inbounds i32, ptr %out, i32 102410001 %val = atomicrmw volatile udec_wrap ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !010002 ret void10003}10004 10005define amdgpu_kernel void @atomic_dec_i32_ret_offset(ptr %out, ptr %out2, i32 %in) {10006; GCN1-LABEL: atomic_dec_i32_ret_offset:10007; GCN1: ; %bb.0: ; %entry10008; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x910009; GCN1-NEXT: s_load_dword s4, s[4:5], 0xd10010; GCN1-NEXT: s_waitcnt lgkmcnt(0)10011; GCN1-NEXT: s_add_u32 s0, s0, 1610012; GCN1-NEXT: s_addc_u32 s1, s1, 010013; GCN1-NEXT: v_mov_b32_e32 v0, s010014; GCN1-NEXT: v_mov_b32_e32 v1, s110015; GCN1-NEXT: v_mov_b32_e32 v2, s410016; GCN1-NEXT: flat_atomic_dec v2, v[0:1], v2 glc10017; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10018; GCN1-NEXT: buffer_wbinvl1_vol10019; GCN1-NEXT: v_mov_b32_e32 v0, s210020; GCN1-NEXT: v_mov_b32_e32 v1, s310021; GCN1-NEXT: flat_store_dword v[0:1], v210022; GCN1-NEXT: s_endpgm10023;10024; GCN2-LABEL: atomic_dec_i32_ret_offset:10025; GCN2: ; %bb.0: ; %entry10026; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410027; GCN2-NEXT: s_load_dword s4, s[4:5], 0x3410028; GCN2-NEXT: s_waitcnt lgkmcnt(0)10029; GCN2-NEXT: s_add_u32 s0, s0, 1610030; GCN2-NEXT: s_addc_u32 s1, s1, 010031; GCN2-NEXT: v_mov_b32_e32 v0, s010032; GCN2-NEXT: v_mov_b32_e32 v1, s110033; GCN2-NEXT: v_mov_b32_e32 v2, s410034; GCN2-NEXT: flat_atomic_dec v2, v[0:1], v2 glc10035; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10036; GCN2-NEXT: buffer_wbinvl1_vol10037; GCN2-NEXT: v_mov_b32_e32 v0, s210038; GCN2-NEXT: v_mov_b32_e32 v1, s310039; GCN2-NEXT: flat_store_dword v[0:1], v210040; GCN2-NEXT: s_endpgm10041;10042; GCN3-LABEL: atomic_dec_i32_ret_offset:10043; GCN3: ; %bb.0: ; %entry10044; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410045; GCN3-NEXT: s_load_dword s6, s[4:5], 0x3410046; GCN3-NEXT: s_waitcnt lgkmcnt(0)10047; GCN3-NEXT: v_mov_b32_e32 v0, s010048; GCN3-NEXT: v_mov_b32_e32 v1, s110049; GCN3-NEXT: v_mov_b32_e32 v2, s610050; GCN3-NEXT: flat_atomic_dec v2, v[0:1], v2 offset:16 glc10051; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10052; GCN3-NEXT: buffer_wbinvl1_vol10053; GCN3-NEXT: v_mov_b32_e32 v0, s210054; GCN3-NEXT: v_mov_b32_e32 v1, s310055; GCN3-NEXT: flat_store_dword v[0:1], v210056; GCN3-NEXT: s_endpgm10057;10058; GFX11-LABEL: atomic_dec_i32_ret_offset:10059; GFX11: ; %bb.0: ; %entry10060; GFX11-NEXT: s_clause 0x110061; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2410062; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x3410063; GFX11-NEXT: s_waitcnt lgkmcnt(0)10064; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s110065; GFX11-NEXT: v_mov_b32_e32 v2, s410066; GFX11-NEXT: flat_atomic_dec_u32 v2, v[0:1], v2 offset:16 glc10067; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10068; GFX11-NEXT: buffer_gl1_inv10069; GFX11-NEXT: buffer_gl0_inv10070; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s310071; GFX11-NEXT: flat_store_b32 v[0:1], v210072; GFX11-NEXT: s_endpgm10073entry:10074 %gep = getelementptr inbounds i32, ptr %out, i32 410075 %val = atomicrmw volatile udec_wrap ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !010076 store i32 %val, ptr %out210077 ret void10078}10079 10080define amdgpu_kernel void @atomic_dec_i32_decr64_offset(ptr %out, i32 %in, i64 %index) {10081; GCN1-LABEL: atomic_dec_i32_decr64_offset:10082; GCN1: ; %bb.0: ; %entry10083; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd10084; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x910085; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb10086; GCN1-NEXT: s_waitcnt lgkmcnt(0)10087; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 210088; GCN1-NEXT: s_add_u32 s0, s2, s010089; GCN1-NEXT: s_addc_u32 s1, s3, s110090; GCN1-NEXT: s_add_u32 s0, s0, 1610091; GCN1-NEXT: s_addc_u32 s1, s1, 010092; GCN1-NEXT: v_mov_b32_e32 v0, s010093; GCN1-NEXT: v_mov_b32_e32 v1, s110094; GCN1-NEXT: v_mov_b32_e32 v2, s410095; GCN1-NEXT: flat_atomic_dec v[0:1], v210096; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10097; GCN1-NEXT: buffer_wbinvl1_vol10098; GCN1-NEXT: s_endpgm10099;10100; GCN2-LABEL: atomic_dec_i32_decr64_offset:10101; GCN2: ; %bb.0: ; %entry10102; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x3410103; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2410104; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c10105; GCN2-NEXT: s_waitcnt lgkmcnt(0)10106; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 210107; GCN2-NEXT: s_add_u32 s0, s2, s010108; GCN2-NEXT: s_addc_u32 s1, s3, s110109; GCN2-NEXT: s_add_u32 s0, s0, 1610110; GCN2-NEXT: s_addc_u32 s1, s1, 010111; GCN2-NEXT: v_mov_b32_e32 v0, s010112; GCN2-NEXT: v_mov_b32_e32 v1, s110113; GCN2-NEXT: v_mov_b32_e32 v2, s410114; GCN2-NEXT: flat_atomic_dec v[0:1], v210115; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10116; GCN2-NEXT: buffer_wbinvl1_vol10117; GCN2-NEXT: s_endpgm10118;10119; GCN3-LABEL: atomic_dec_i32_decr64_offset:10120; GCN3: ; %bb.0: ; %entry10121; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x3410122; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2410123; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c10124; GCN3-NEXT: s_waitcnt lgkmcnt(0)10125; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 210126; GCN3-NEXT: s_add_u32 s0, s2, s010127; GCN3-NEXT: s_addc_u32 s1, s3, s110128; GCN3-NEXT: v_mov_b32_e32 v0, s010129; GCN3-NEXT: v_mov_b32_e32 v1, s110130; GCN3-NEXT: v_mov_b32_e32 v2, s610131; GCN3-NEXT: flat_atomic_dec v[0:1], v2 offset:1610132; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10133; GCN3-NEXT: buffer_wbinvl1_vol10134; GCN3-NEXT: s_endpgm10135;10136; GFX11-LABEL: atomic_dec_i32_decr64_offset:10137; GFX11: ; %bb.0: ; %entry10138; GFX11-NEXT: s_clause 0x210139; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x3410140; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x2410141; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c10142; GFX11-NEXT: s_waitcnt lgkmcnt(0)10143; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 210144; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)10145; GFX11-NEXT: s_add_u32 s0, s2, s010146; GFX11-NEXT: s_addc_u32 s1, s3, s110147; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s110148; GFX11-NEXT: v_mov_b32_e32 v2, s410149; GFX11-NEXT: flat_atomic_dec_u32 v[0:1], v2 offset:1610150; GFX11-NEXT: s_waitcnt lgkmcnt(0)10151; GFX11-NEXT: s_waitcnt_vscnt null, 0x010152; GFX11-NEXT: buffer_gl1_inv10153; GFX11-NEXT: buffer_gl0_inv10154; GFX11-NEXT: s_endpgm10155entry:10156 %ptr = getelementptr inbounds i32, ptr %out, i64 %index10157 %gep = getelementptr inbounds i32, ptr %ptr, i32 410158 %val = atomicrmw volatile udec_wrap ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !010159 ret void10160}10161 10162define amdgpu_kernel void @atomic_dec_i32_ret_decr64_offset(ptr %out, ptr %out2, i32 %in, i64 %index) {10163; GCN1-LABEL: atomic_dec_i32_ret_decr64_offset:10164; GCN1: ; %bb.0: ; %entry10165; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf10166; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x910167; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd10168; GCN1-NEXT: s_waitcnt lgkmcnt(0)10169; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 210170; GCN1-NEXT: s_add_u32 s0, s0, s410171; GCN1-NEXT: s_addc_u32 s1, s1, s510172; GCN1-NEXT: s_add_u32 s0, s0, 1610173; GCN1-NEXT: s_addc_u32 s1, s1, 010174; GCN1-NEXT: v_mov_b32_e32 v0, s010175; GCN1-NEXT: v_mov_b32_e32 v1, s110176; GCN1-NEXT: v_mov_b32_e32 v2, s810177; GCN1-NEXT: flat_atomic_dec v2, v[0:1], v2 glc10178; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10179; GCN1-NEXT: buffer_wbinvl1_vol10180; GCN1-NEXT: v_mov_b32_e32 v0, s210181; GCN1-NEXT: v_mov_b32_e32 v1, s310182; GCN1-NEXT: flat_store_dword v[0:1], v210183; GCN1-NEXT: s_endpgm10184;10185; GCN2-LABEL: atomic_dec_i32_ret_decr64_offset:10186; GCN2: ; %bb.0: ; %entry10187; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c10188; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410189; GCN2-NEXT: s_load_dword s8, s[4:5], 0x3410190; GCN2-NEXT: s_waitcnt lgkmcnt(0)10191; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 210192; GCN2-NEXT: s_add_u32 s0, s0, s410193; GCN2-NEXT: s_addc_u32 s1, s1, s510194; GCN2-NEXT: s_add_u32 s0, s0, 1610195; GCN2-NEXT: s_addc_u32 s1, s1, 010196; GCN2-NEXT: v_mov_b32_e32 v0, s010197; GCN2-NEXT: v_mov_b32_e32 v1, s110198; GCN2-NEXT: v_mov_b32_e32 v2, s810199; GCN2-NEXT: flat_atomic_dec v2, v[0:1], v2 glc10200; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10201; GCN2-NEXT: buffer_wbinvl1_vol10202; GCN2-NEXT: v_mov_b32_e32 v0, s210203; GCN2-NEXT: v_mov_b32_e32 v1, s310204; GCN2-NEXT: flat_store_dword v[0:1], v210205; GCN2-NEXT: s_endpgm10206;10207; GCN3-LABEL: atomic_dec_i32_ret_decr64_offset:10208; GCN3: ; %bb.0: ; %entry10209; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c10210; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410211; GCN3-NEXT: s_load_dword s8, s[4:5], 0x3410212; GCN3-NEXT: s_waitcnt lgkmcnt(0)10213; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 210214; GCN3-NEXT: s_add_u32 s0, s0, s410215; GCN3-NEXT: s_addc_u32 s1, s1, s510216; GCN3-NEXT: v_mov_b32_e32 v0, s010217; GCN3-NEXT: v_mov_b32_e32 v1, s110218; GCN3-NEXT: v_mov_b32_e32 v2, s810219; GCN3-NEXT: flat_atomic_dec v2, v[0:1], v2 offset:16 glc10220; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10221; GCN3-NEXT: buffer_wbinvl1_vol10222; GCN3-NEXT: v_mov_b32_e32 v0, s210223; GCN3-NEXT: v_mov_b32_e32 v1, s310224; GCN3-NEXT: flat_store_dword v[0:1], v210225; GCN3-NEXT: s_endpgm10226;10227; GFX11-LABEL: atomic_dec_i32_ret_decr64_offset:10228; GFX11: ; %bb.0: ; %entry10229; GFX11-NEXT: s_clause 0x210230; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c10231; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2410232; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x3410233; GFX11-NEXT: s_waitcnt lgkmcnt(0)10234; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 210235; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)10236; GFX11-NEXT: s_add_u32 s0, s0, s410237; GFX11-NEXT: s_addc_u32 s1, s1, s510238; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s110239; GFX11-NEXT: v_mov_b32_e32 v2, s810240; GFX11-NEXT: flat_atomic_dec_u32 v2, v[0:1], v2 offset:16 glc10241; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10242; GFX11-NEXT: buffer_gl1_inv10243; GFX11-NEXT: buffer_gl0_inv10244; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s310245; GFX11-NEXT: flat_store_b32 v[0:1], v210246; GFX11-NEXT: s_endpgm10247entry:10248 %ptr = getelementptr inbounds i32, ptr %out, i64 %index10249 %gep = getelementptr inbounds i32, ptr %ptr, i32 410250 %val = atomicrmw volatile udec_wrap ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !010251 store i32 %val, ptr %out210252 ret void10253}10254 10255define amdgpu_kernel void @atomic_dec_i32(ptr %out, i32 %in) {10256; GCN1-LABEL: atomic_dec_i32:10257; GCN1: ; %bb.0: ; %entry10258; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x910259; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb10260; GCN1-NEXT: s_waitcnt lgkmcnt(0)10261; GCN1-NEXT: v_mov_b32_e32 v0, s010262; GCN1-NEXT: v_mov_b32_e32 v1, s110263; GCN1-NEXT: v_mov_b32_e32 v2, s210264; GCN1-NEXT: flat_atomic_dec v[0:1], v210265; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10266; GCN1-NEXT: buffer_wbinvl1_vol10267; GCN1-NEXT: s_endpgm10268;10269; GCN2-LABEL: atomic_dec_i32:10270; GCN2: ; %bb.0: ; %entry10271; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2410272; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c10273; GCN2-NEXT: s_waitcnt lgkmcnt(0)10274; GCN2-NEXT: v_mov_b32_e32 v0, s010275; GCN2-NEXT: v_mov_b32_e32 v1, s110276; GCN2-NEXT: v_mov_b32_e32 v2, s210277; GCN2-NEXT: flat_atomic_dec v[0:1], v210278; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10279; GCN2-NEXT: buffer_wbinvl1_vol10280; GCN2-NEXT: s_endpgm10281;10282; GCN3-LABEL: atomic_dec_i32:10283; GCN3: ; %bb.0: ; %entry10284; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2410285; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c10286; GCN3-NEXT: s_waitcnt lgkmcnt(0)10287; GCN3-NEXT: v_mov_b32_e32 v0, s010288; GCN3-NEXT: v_mov_b32_e32 v1, s110289; GCN3-NEXT: v_mov_b32_e32 v2, s210290; GCN3-NEXT: flat_atomic_dec v[0:1], v210291; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10292; GCN3-NEXT: buffer_wbinvl1_vol10293; GCN3-NEXT: s_endpgm10294;10295; GFX11-LABEL: atomic_dec_i32:10296; GFX11: ; %bb.0: ; %entry10297; GFX11-NEXT: s_clause 0x110298; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x2410299; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c10300; GFX11-NEXT: s_waitcnt lgkmcnt(0)10301; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s110302; GFX11-NEXT: v_mov_b32_e32 v2, s210303; GFX11-NEXT: flat_atomic_dec_u32 v[0:1], v210304; GFX11-NEXT: s_waitcnt lgkmcnt(0)10305; GFX11-NEXT: s_waitcnt_vscnt null, 0x010306; GFX11-NEXT: buffer_gl1_inv10307; GFX11-NEXT: buffer_gl0_inv10308; GFX11-NEXT: s_endpgm10309entry:10310 %val = atomicrmw volatile udec_wrap ptr %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !010311 ret void10312}10313 10314define amdgpu_kernel void @atomic_dec_i32_ret(ptr %out, ptr %out2, i32 %in) {10315; GCN1-LABEL: atomic_dec_i32_ret:10316; GCN1: ; %bb.0: ; %entry10317; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x910318; GCN1-NEXT: s_load_dword s4, s[4:5], 0xd10319; GCN1-NEXT: s_waitcnt lgkmcnt(0)10320; GCN1-NEXT: v_mov_b32_e32 v0, s010321; GCN1-NEXT: v_mov_b32_e32 v1, s110322; GCN1-NEXT: v_mov_b32_e32 v2, s410323; GCN1-NEXT: flat_atomic_dec v2, v[0:1], v2 glc10324; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10325; GCN1-NEXT: buffer_wbinvl1_vol10326; GCN1-NEXT: v_mov_b32_e32 v0, s210327; GCN1-NEXT: v_mov_b32_e32 v1, s310328; GCN1-NEXT: flat_store_dword v[0:1], v210329; GCN1-NEXT: s_endpgm10330;10331; GCN2-LABEL: atomic_dec_i32_ret:10332; GCN2: ; %bb.0: ; %entry10333; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410334; GCN2-NEXT: s_load_dword s4, s[4:5], 0x3410335; GCN2-NEXT: s_waitcnt lgkmcnt(0)10336; GCN2-NEXT: v_mov_b32_e32 v0, s010337; GCN2-NEXT: v_mov_b32_e32 v1, s110338; GCN2-NEXT: v_mov_b32_e32 v2, s410339; GCN2-NEXT: flat_atomic_dec v2, v[0:1], v2 glc10340; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10341; GCN2-NEXT: buffer_wbinvl1_vol10342; GCN2-NEXT: v_mov_b32_e32 v0, s210343; GCN2-NEXT: v_mov_b32_e32 v1, s310344; GCN2-NEXT: flat_store_dword v[0:1], v210345; GCN2-NEXT: s_endpgm10346;10347; GCN3-LABEL: atomic_dec_i32_ret:10348; GCN3: ; %bb.0: ; %entry10349; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410350; GCN3-NEXT: s_load_dword s6, s[4:5], 0x3410351; GCN3-NEXT: s_waitcnt lgkmcnt(0)10352; GCN3-NEXT: v_mov_b32_e32 v0, s010353; GCN3-NEXT: v_mov_b32_e32 v1, s110354; GCN3-NEXT: v_mov_b32_e32 v2, s610355; GCN3-NEXT: flat_atomic_dec v2, v[0:1], v2 glc10356; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10357; GCN3-NEXT: buffer_wbinvl1_vol10358; GCN3-NEXT: v_mov_b32_e32 v0, s210359; GCN3-NEXT: v_mov_b32_e32 v1, s310360; GCN3-NEXT: flat_store_dword v[0:1], v210361; GCN3-NEXT: s_endpgm10362;10363; GFX11-LABEL: atomic_dec_i32_ret:10364; GFX11: ; %bb.0: ; %entry10365; GFX11-NEXT: s_clause 0x110366; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2410367; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x3410368; GFX11-NEXT: s_waitcnt lgkmcnt(0)10369; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s110370; GFX11-NEXT: v_mov_b32_e32 v2, s410371; GFX11-NEXT: flat_atomic_dec_u32 v2, v[0:1], v2 glc10372; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10373; GFX11-NEXT: buffer_gl1_inv10374; GFX11-NEXT: buffer_gl0_inv10375; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s310376; GFX11-NEXT: flat_store_b32 v[0:1], v210377; GFX11-NEXT: s_endpgm10378entry:10379 %val = atomicrmw volatile udec_wrap ptr %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !010380 store i32 %val, ptr %out210381 ret void10382}10383 10384define amdgpu_kernel void @atomic_dec_i32_decr64(ptr %out, i32 %in, i64 %index) {10385; GCN1-LABEL: atomic_dec_i32_decr64:10386; GCN1: ; %bb.0: ; %entry10387; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd10388; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x910389; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb10390; GCN1-NEXT: s_waitcnt lgkmcnt(0)10391; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 210392; GCN1-NEXT: s_add_u32 s0, s2, s010393; GCN1-NEXT: s_addc_u32 s1, s3, s110394; GCN1-NEXT: v_mov_b32_e32 v0, s010395; GCN1-NEXT: v_mov_b32_e32 v1, s110396; GCN1-NEXT: v_mov_b32_e32 v2, s410397; GCN1-NEXT: flat_atomic_dec v[0:1], v210398; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10399; GCN1-NEXT: buffer_wbinvl1_vol10400; GCN1-NEXT: s_endpgm10401;10402; GCN2-LABEL: atomic_dec_i32_decr64:10403; GCN2: ; %bb.0: ; %entry10404; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x3410405; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2410406; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c10407; GCN2-NEXT: s_waitcnt lgkmcnt(0)10408; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 210409; GCN2-NEXT: s_add_u32 s0, s2, s010410; GCN2-NEXT: s_addc_u32 s1, s3, s110411; GCN2-NEXT: v_mov_b32_e32 v0, s010412; GCN2-NEXT: v_mov_b32_e32 v1, s110413; GCN2-NEXT: v_mov_b32_e32 v2, s410414; GCN2-NEXT: flat_atomic_dec v[0:1], v210415; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10416; GCN2-NEXT: buffer_wbinvl1_vol10417; GCN2-NEXT: s_endpgm10418;10419; GCN3-LABEL: atomic_dec_i32_decr64:10420; GCN3: ; %bb.0: ; %entry10421; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x3410422; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2410423; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c10424; GCN3-NEXT: s_waitcnt lgkmcnt(0)10425; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 210426; GCN3-NEXT: s_add_u32 s0, s2, s010427; GCN3-NEXT: s_addc_u32 s1, s3, s110428; GCN3-NEXT: v_mov_b32_e32 v0, s010429; GCN3-NEXT: v_mov_b32_e32 v1, s110430; GCN3-NEXT: v_mov_b32_e32 v2, s610431; GCN3-NEXT: flat_atomic_dec v[0:1], v210432; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10433; GCN3-NEXT: buffer_wbinvl1_vol10434; GCN3-NEXT: s_endpgm10435;10436; GFX11-LABEL: atomic_dec_i32_decr64:10437; GFX11: ; %bb.0: ; %entry10438; GFX11-NEXT: s_clause 0x210439; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x3410440; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x2410441; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c10442; GFX11-NEXT: s_waitcnt lgkmcnt(0)10443; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 210444; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)10445; GFX11-NEXT: s_add_u32 s0, s2, s010446; GFX11-NEXT: s_addc_u32 s1, s3, s110447; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s110448; GFX11-NEXT: v_mov_b32_e32 v2, s410449; GFX11-NEXT: flat_atomic_dec_u32 v[0:1], v210450; GFX11-NEXT: s_waitcnt lgkmcnt(0)10451; GFX11-NEXT: s_waitcnt_vscnt null, 0x010452; GFX11-NEXT: buffer_gl1_inv10453; GFX11-NEXT: buffer_gl0_inv10454; GFX11-NEXT: s_endpgm10455entry:10456 %ptr = getelementptr inbounds i32, ptr %out, i64 %index10457 %val = atomicrmw volatile udec_wrap ptr %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !010458 ret void10459}10460 10461define amdgpu_kernel void @atomic_dec_i32_ret_decr64(ptr %out, ptr %out2, i32 %in, i64 %index) {10462; GCN1-LABEL: atomic_dec_i32_ret_decr64:10463; GCN1: ; %bb.0: ; %entry10464; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf10465; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x910466; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd10467; GCN1-NEXT: s_waitcnt lgkmcnt(0)10468; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 210469; GCN1-NEXT: s_add_u32 s0, s0, s410470; GCN1-NEXT: s_addc_u32 s1, s1, s510471; GCN1-NEXT: v_mov_b32_e32 v0, s010472; GCN1-NEXT: v_mov_b32_e32 v1, s110473; GCN1-NEXT: v_mov_b32_e32 v2, s810474; GCN1-NEXT: flat_atomic_dec v2, v[0:1], v2 glc10475; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10476; GCN1-NEXT: buffer_wbinvl1_vol10477; GCN1-NEXT: v_mov_b32_e32 v0, s210478; GCN1-NEXT: v_mov_b32_e32 v1, s310479; GCN1-NEXT: flat_store_dword v[0:1], v210480; GCN1-NEXT: s_endpgm10481;10482; GCN2-LABEL: atomic_dec_i32_ret_decr64:10483; GCN2: ; %bb.0: ; %entry10484; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c10485; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410486; GCN2-NEXT: s_load_dword s8, s[4:5], 0x3410487; GCN2-NEXT: s_waitcnt lgkmcnt(0)10488; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 210489; GCN2-NEXT: s_add_u32 s0, s0, s410490; GCN2-NEXT: s_addc_u32 s1, s1, s510491; GCN2-NEXT: v_mov_b32_e32 v0, s010492; GCN2-NEXT: v_mov_b32_e32 v1, s110493; GCN2-NEXT: v_mov_b32_e32 v2, s810494; GCN2-NEXT: flat_atomic_dec v2, v[0:1], v2 glc10495; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10496; GCN2-NEXT: buffer_wbinvl1_vol10497; GCN2-NEXT: v_mov_b32_e32 v0, s210498; GCN2-NEXT: v_mov_b32_e32 v1, s310499; GCN2-NEXT: flat_store_dword v[0:1], v210500; GCN2-NEXT: s_endpgm10501;10502; GCN3-LABEL: atomic_dec_i32_ret_decr64:10503; GCN3: ; %bb.0: ; %entry10504; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c10505; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410506; GCN3-NEXT: s_load_dword s8, s[4:5], 0x3410507; GCN3-NEXT: s_waitcnt lgkmcnt(0)10508; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 210509; GCN3-NEXT: s_add_u32 s0, s0, s410510; GCN3-NEXT: s_addc_u32 s1, s1, s510511; GCN3-NEXT: v_mov_b32_e32 v0, s010512; GCN3-NEXT: v_mov_b32_e32 v1, s110513; GCN3-NEXT: v_mov_b32_e32 v2, s810514; GCN3-NEXT: flat_atomic_dec v2, v[0:1], v2 glc10515; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10516; GCN3-NEXT: buffer_wbinvl1_vol10517; GCN3-NEXT: v_mov_b32_e32 v0, s210518; GCN3-NEXT: v_mov_b32_e32 v1, s310519; GCN3-NEXT: flat_store_dword v[0:1], v210520; GCN3-NEXT: s_endpgm10521;10522; GFX11-LABEL: atomic_dec_i32_ret_decr64:10523; GFX11: ; %bb.0: ; %entry10524; GFX11-NEXT: s_clause 0x210525; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c10526; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2410527; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x3410528; GFX11-NEXT: s_waitcnt lgkmcnt(0)10529; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 210530; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)10531; GFX11-NEXT: s_add_u32 s0, s0, s410532; GFX11-NEXT: s_addc_u32 s1, s1, s510533; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s110534; GFX11-NEXT: v_mov_b32_e32 v2, s810535; GFX11-NEXT: flat_atomic_dec_u32 v2, v[0:1], v2 glc10536; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10537; GFX11-NEXT: buffer_gl1_inv10538; GFX11-NEXT: buffer_gl0_inv10539; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s310540; GFX11-NEXT: flat_store_b32 v[0:1], v210541; GFX11-NEXT: s_endpgm10542entry:10543 %ptr = getelementptr inbounds i32, ptr %out, i64 %index10544 %val = atomicrmw volatile udec_wrap ptr %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !010545 store i32 %val, ptr %out210546 ret void10547}10548 10549define amdgpu_kernel void @atomic_load_f16_offset(ptr %in, ptr %out) {10550; GCN1-LABEL: atomic_load_f16_offset:10551; GCN1: ; %bb.0:10552; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x910553; GCN1-NEXT: s_waitcnt lgkmcnt(0)10554; GCN1-NEXT: s_add_u32 s0, s0, 1610555; GCN1-NEXT: s_addc_u32 s1, s1, 010556; GCN1-NEXT: v_mov_b32_e32 v0, s010557; GCN1-NEXT: v_mov_b32_e32 v1, s110558; GCN1-NEXT: flat_load_ushort v2, v[0:1] glc10559; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10560; GCN1-NEXT: buffer_wbinvl1_vol10561; GCN1-NEXT: v_mov_b32_e32 v0, s210562; GCN1-NEXT: v_mov_b32_e32 v1, s310563; GCN1-NEXT: flat_store_short v[0:1], v210564; GCN1-NEXT: s_endpgm10565;10566; GCN2-LABEL: atomic_load_f16_offset:10567; GCN2: ; %bb.0:10568; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410569; GCN2-NEXT: s_waitcnt lgkmcnt(0)10570; GCN2-NEXT: s_add_u32 s0, s0, 1610571; GCN2-NEXT: s_addc_u32 s1, s1, 010572; GCN2-NEXT: v_mov_b32_e32 v0, s010573; GCN2-NEXT: v_mov_b32_e32 v1, s110574; GCN2-NEXT: flat_load_ushort v2, v[0:1] glc10575; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10576; GCN2-NEXT: buffer_wbinvl1_vol10577; GCN2-NEXT: v_mov_b32_e32 v0, s210578; GCN2-NEXT: v_mov_b32_e32 v1, s310579; GCN2-NEXT: flat_store_short v[0:1], v210580; GCN2-NEXT: s_endpgm10581;10582; GCN3-LABEL: atomic_load_f16_offset:10583; GCN3: ; %bb.0:10584; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410585; GCN3-NEXT: s_waitcnt lgkmcnt(0)10586; GCN3-NEXT: v_mov_b32_e32 v0, s010587; GCN3-NEXT: v_mov_b32_e32 v1, s110588; GCN3-NEXT: flat_load_ushort v2, v[0:1] offset:16 glc10589; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10590; GCN3-NEXT: buffer_wbinvl1_vol10591; GCN3-NEXT: v_mov_b32_e32 v0, s210592; GCN3-NEXT: v_mov_b32_e32 v1, s310593; GCN3-NEXT: flat_store_short v[0:1], v210594; GCN3-NEXT: s_endpgm10595;10596; GFX11-TRUE16-LABEL: atomic_load_f16_offset:10597; GFX11-TRUE16: ; %bb.0:10598; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2410599; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)10600; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s110601; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s310602; GFX11-TRUE16-NEXT: flat_load_d16_b16 v0, v[0:1] offset:16 glc10603; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10604; GFX11-TRUE16-NEXT: buffer_gl1_inv10605; GFX11-TRUE16-NEXT: buffer_gl0_inv10606; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s210607; GFX11-TRUE16-NEXT: flat_store_b16 v[1:2], v010608; GFX11-TRUE16-NEXT: s_endpgm10609;10610; GFX11-FAKE16-LABEL: atomic_load_f16_offset:10611; GFX11-FAKE16: ; %bb.0:10612; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2410613; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)10614; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s110615; GFX11-FAKE16-NEXT: flat_load_u16 v2, v[0:1] offset:16 glc10616; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10617; GFX11-FAKE16-NEXT: buffer_gl1_inv10618; GFX11-FAKE16-NEXT: buffer_gl0_inv10619; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s310620; GFX11-FAKE16-NEXT: flat_store_b16 v[0:1], v210621; GFX11-FAKE16-NEXT: s_endpgm10622 %gep = getelementptr inbounds half, ptr %in, i64 810623 %val = load atomic half, ptr %gep seq_cst, align 210624 store half %val, ptr %out10625 ret void10626}10627 10628define amdgpu_kernel void @atomic_load_f16(ptr %in, ptr %out) {10629; GCN1-LABEL: atomic_load_f16:10630; GCN1: ; %bb.0:10631; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x910632; GCN1-NEXT: s_waitcnt lgkmcnt(0)10633; GCN1-NEXT: v_mov_b32_e32 v0, s010634; GCN1-NEXT: v_mov_b32_e32 v1, s110635; GCN1-NEXT: flat_load_ushort v2, v[0:1] glc10636; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10637; GCN1-NEXT: buffer_wbinvl1_vol10638; GCN1-NEXT: v_mov_b32_e32 v0, s210639; GCN1-NEXT: v_mov_b32_e32 v1, s310640; GCN1-NEXT: flat_store_short v[0:1], v210641; GCN1-NEXT: s_endpgm10642;10643; GCN2-LABEL: atomic_load_f16:10644; GCN2: ; %bb.0:10645; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410646; GCN2-NEXT: s_waitcnt lgkmcnt(0)10647; GCN2-NEXT: v_mov_b32_e32 v0, s010648; GCN2-NEXT: v_mov_b32_e32 v1, s110649; GCN2-NEXT: flat_load_ushort v2, v[0:1] glc10650; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10651; GCN2-NEXT: buffer_wbinvl1_vol10652; GCN2-NEXT: v_mov_b32_e32 v0, s210653; GCN2-NEXT: v_mov_b32_e32 v1, s310654; GCN2-NEXT: flat_store_short v[0:1], v210655; GCN2-NEXT: s_endpgm10656;10657; GCN3-LABEL: atomic_load_f16:10658; GCN3: ; %bb.0:10659; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410660; GCN3-NEXT: s_waitcnt lgkmcnt(0)10661; GCN3-NEXT: v_mov_b32_e32 v0, s010662; GCN3-NEXT: v_mov_b32_e32 v1, s110663; GCN3-NEXT: flat_load_ushort v2, v[0:1] glc10664; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10665; GCN3-NEXT: buffer_wbinvl1_vol10666; GCN3-NEXT: v_mov_b32_e32 v0, s210667; GCN3-NEXT: v_mov_b32_e32 v1, s310668; GCN3-NEXT: flat_store_short v[0:1], v210669; GCN3-NEXT: s_endpgm10670;10671; GFX11-TRUE16-LABEL: atomic_load_f16:10672; GFX11-TRUE16: ; %bb.0:10673; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2410674; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)10675; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s110676; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s310677; GFX11-TRUE16-NEXT: flat_load_d16_b16 v0, v[0:1] glc10678; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10679; GFX11-TRUE16-NEXT: buffer_gl1_inv10680; GFX11-TRUE16-NEXT: buffer_gl0_inv10681; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s210682; GFX11-TRUE16-NEXT: flat_store_b16 v[1:2], v010683; GFX11-TRUE16-NEXT: s_endpgm10684;10685; GFX11-FAKE16-LABEL: atomic_load_f16:10686; GFX11-FAKE16: ; %bb.0:10687; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2410688; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)10689; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s110690; GFX11-FAKE16-NEXT: flat_load_u16 v2, v[0:1] glc10691; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10692; GFX11-FAKE16-NEXT: buffer_gl1_inv10693; GFX11-FAKE16-NEXT: buffer_gl0_inv10694; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s310695; GFX11-FAKE16-NEXT: flat_store_b16 v[0:1], v210696; GFX11-FAKE16-NEXT: s_endpgm10697 %val = load atomic half, ptr %in seq_cst, align 210698 store half %val, ptr %out10699 ret void10700}10701 10702define amdgpu_kernel void @atomic_load_bf16_offset(ptr %in, ptr %out) {10703; GCN1-LABEL: atomic_load_bf16_offset:10704; GCN1: ; %bb.0:10705; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x910706; GCN1-NEXT: s_waitcnt lgkmcnt(0)10707; GCN1-NEXT: s_add_u32 s0, s0, 1610708; GCN1-NEXT: s_addc_u32 s1, s1, 010709; GCN1-NEXT: v_mov_b32_e32 v0, s010710; GCN1-NEXT: v_mov_b32_e32 v1, s110711; GCN1-NEXT: flat_load_ushort v2, v[0:1] glc10712; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10713; GCN1-NEXT: buffer_wbinvl1_vol10714; GCN1-NEXT: v_mov_b32_e32 v0, s210715; GCN1-NEXT: v_mov_b32_e32 v1, s310716; GCN1-NEXT: flat_store_short v[0:1], v210717; GCN1-NEXT: s_endpgm10718;10719; GCN2-LABEL: atomic_load_bf16_offset:10720; GCN2: ; %bb.0:10721; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410722; GCN2-NEXT: s_waitcnt lgkmcnt(0)10723; GCN2-NEXT: s_add_u32 s0, s0, 1610724; GCN2-NEXT: s_addc_u32 s1, s1, 010725; GCN2-NEXT: v_mov_b32_e32 v0, s010726; GCN2-NEXT: v_mov_b32_e32 v1, s110727; GCN2-NEXT: flat_load_ushort v2, v[0:1] glc10728; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10729; GCN2-NEXT: buffer_wbinvl1_vol10730; GCN2-NEXT: v_mov_b32_e32 v0, s210731; GCN2-NEXT: v_mov_b32_e32 v1, s310732; GCN2-NEXT: flat_store_short v[0:1], v210733; GCN2-NEXT: s_endpgm10734;10735; GCN3-LABEL: atomic_load_bf16_offset:10736; GCN3: ; %bb.0:10737; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410738; GCN3-NEXT: s_waitcnt lgkmcnt(0)10739; GCN3-NEXT: v_mov_b32_e32 v0, s010740; GCN3-NEXT: v_mov_b32_e32 v1, s110741; GCN3-NEXT: flat_load_ushort v2, v[0:1] offset:16 glc10742; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10743; GCN3-NEXT: buffer_wbinvl1_vol10744; GCN3-NEXT: v_mov_b32_e32 v0, s210745; GCN3-NEXT: v_mov_b32_e32 v1, s310746; GCN3-NEXT: flat_store_short v[0:1], v210747; GCN3-NEXT: s_endpgm10748;10749; GFX11-TRUE16-LABEL: atomic_load_bf16_offset:10750; GFX11-TRUE16: ; %bb.0:10751; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2410752; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)10753; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s110754; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s310755; GFX11-TRUE16-NEXT: flat_load_d16_b16 v0, v[0:1] offset:16 glc10756; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10757; GFX11-TRUE16-NEXT: buffer_gl1_inv10758; GFX11-TRUE16-NEXT: buffer_gl0_inv10759; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s210760; GFX11-TRUE16-NEXT: flat_store_b16 v[1:2], v010761; GFX11-TRUE16-NEXT: s_endpgm10762;10763; GFX11-FAKE16-LABEL: atomic_load_bf16_offset:10764; GFX11-FAKE16: ; %bb.0:10765; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2410766; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)10767; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s110768; GFX11-FAKE16-NEXT: flat_load_u16 v2, v[0:1] offset:16 glc10769; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10770; GFX11-FAKE16-NEXT: buffer_gl1_inv10771; GFX11-FAKE16-NEXT: buffer_gl0_inv10772; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s310773; GFX11-FAKE16-NEXT: flat_store_b16 v[0:1], v210774; GFX11-FAKE16-NEXT: s_endpgm10775 %gep = getelementptr inbounds bfloat, ptr %in, i64 810776 %val = load atomic bfloat, ptr %gep seq_cst, align 210777 store bfloat %val, ptr %out10778 ret void10779}10780 10781define amdgpu_kernel void @atomic_load_bf16(ptr %in, ptr %out) {10782; GCN1-LABEL: atomic_load_bf16:10783; GCN1: ; %bb.0:10784; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x910785; GCN1-NEXT: s_waitcnt lgkmcnt(0)10786; GCN1-NEXT: v_mov_b32_e32 v0, s010787; GCN1-NEXT: v_mov_b32_e32 v1, s110788; GCN1-NEXT: flat_load_ushort v2, v[0:1] glc10789; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10790; GCN1-NEXT: buffer_wbinvl1_vol10791; GCN1-NEXT: v_mov_b32_e32 v0, s210792; GCN1-NEXT: v_mov_b32_e32 v1, s310793; GCN1-NEXT: flat_store_short v[0:1], v210794; GCN1-NEXT: s_endpgm10795;10796; GCN2-LABEL: atomic_load_bf16:10797; GCN2: ; %bb.0:10798; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410799; GCN2-NEXT: s_waitcnt lgkmcnt(0)10800; GCN2-NEXT: v_mov_b32_e32 v0, s010801; GCN2-NEXT: v_mov_b32_e32 v1, s110802; GCN2-NEXT: flat_load_ushort v2, v[0:1] glc10803; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10804; GCN2-NEXT: buffer_wbinvl1_vol10805; GCN2-NEXT: v_mov_b32_e32 v0, s210806; GCN2-NEXT: v_mov_b32_e32 v1, s310807; GCN2-NEXT: flat_store_short v[0:1], v210808; GCN2-NEXT: s_endpgm10809;10810; GCN3-LABEL: atomic_load_bf16:10811; GCN3: ; %bb.0:10812; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410813; GCN3-NEXT: s_waitcnt lgkmcnt(0)10814; GCN3-NEXT: v_mov_b32_e32 v0, s010815; GCN3-NEXT: v_mov_b32_e32 v1, s110816; GCN3-NEXT: flat_load_ushort v2, v[0:1] glc10817; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10818; GCN3-NEXT: buffer_wbinvl1_vol10819; GCN3-NEXT: v_mov_b32_e32 v0, s210820; GCN3-NEXT: v_mov_b32_e32 v1, s310821; GCN3-NEXT: flat_store_short v[0:1], v210822; GCN3-NEXT: s_endpgm10823;10824; GFX11-TRUE16-LABEL: atomic_load_bf16:10825; GFX11-TRUE16: ; %bb.0:10826; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2410827; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)10828; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s110829; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s310830; GFX11-TRUE16-NEXT: flat_load_d16_b16 v0, v[0:1] glc10831; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10832; GFX11-TRUE16-NEXT: buffer_gl1_inv10833; GFX11-TRUE16-NEXT: buffer_gl0_inv10834; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s210835; GFX11-TRUE16-NEXT: flat_store_b16 v[1:2], v010836; GFX11-TRUE16-NEXT: s_endpgm10837;10838; GFX11-FAKE16-LABEL: atomic_load_bf16:10839; GFX11-FAKE16: ; %bb.0:10840; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2410841; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)10842; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s110843; GFX11-FAKE16-NEXT: flat_load_u16 v2, v[0:1] glc10844; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10845; GFX11-FAKE16-NEXT: buffer_gl1_inv10846; GFX11-FAKE16-NEXT: buffer_gl0_inv10847; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s310848; GFX11-FAKE16-NEXT: flat_store_b16 v[0:1], v210849; GFX11-FAKE16-NEXT: s_endpgm10850 %val = load atomic bfloat, ptr %in seq_cst, align 210851 store bfloat %val, ptr %out10852 ret void10853}10854 10855!0 = !{}10856