1161 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx700 < %s | FileCheck --check-prefixes=GFX7 %s3; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1010 < %s | FileCheck --check-prefixes=GFX10-WGP %s4; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1010 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX10-CU %s5; RUN: llc -mtriple=amdgcn-amd-amdpal -O0 -mcpu=gfx700 -amdgcn-skip-cache-invalidations < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s6; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11-WGP %s7; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s8; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s9; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s10; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s11 12define amdgpu_kernel void @flat_nontemporal_load_0(13; GFX7-LABEL: flat_nontemporal_load_0:14; GFX7: ; %bb.0: ; %entry15; GFX7-NEXT: s_mov_b32 flat_scratch_lo, s1316; GFX7-NEXT: s_add_i32 s12, s12, s1717; GFX7-NEXT: s_lshr_b32 flat_scratch_hi, s12, 818; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x019; GFX7-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x220; GFX7-NEXT: s_waitcnt lgkmcnt(0)21; GFX7-NEXT: v_mov_b32_e32 v0, s622; GFX7-NEXT: v_mov_b32_e32 v1, s723; GFX7-NEXT: flat_load_dword v2, v[0:1] glc24; GFX7-NEXT: s_waitcnt vmcnt(0)25; GFX7-NEXT: v_mov_b32_e32 v0, s426; GFX7-NEXT: v_mov_b32_e32 v1, s527; GFX7-NEXT: s_waitcnt lgkmcnt(0)28; GFX7-NEXT: flat_store_dword v[0:1], v229; GFX7-NEXT: s_endpgm30;31; GFX10-WGP-LABEL: flat_nontemporal_load_0:32; GFX10-WGP: ; %bb.0: ; %entry33; GFX10-WGP-NEXT: s_add_u32 s12, s12, s1734; GFX10-WGP-NEXT: s_addc_u32 s13, s13, 035; GFX10-WGP-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s1236; GFX10-WGP-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1337; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x038; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x839; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)40; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s641; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s742; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc dlc43; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)44; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s445; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s546; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)47; GFX10-WGP-NEXT: flat_store_dword v[0:1], v248; GFX10-WGP-NEXT: s_endpgm49;50; GFX10-CU-LABEL: flat_nontemporal_load_0:51; GFX10-CU: ; %bb.0: ; %entry52; GFX10-CU-NEXT: s_add_u32 s12, s12, s1753; GFX10-CU-NEXT: s_addc_u32 s13, s13, 054; GFX10-CU-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s1255; GFX10-CU-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1356; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x057; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x858; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)59; GFX10-CU-NEXT: v_mov_b32_e32 v0, s660; GFX10-CU-NEXT: v_mov_b32_e32 v1, s761; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] glc dlc62; GFX10-CU-NEXT: s_waitcnt vmcnt(0)63; GFX10-CU-NEXT: v_mov_b32_e32 v0, s464; GFX10-CU-NEXT: v_mov_b32_e32 v1, s565; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)66; GFX10-CU-NEXT: flat_store_dword v[0:1], v267; GFX10-CU-NEXT: s_endpgm68;69; SKIP-CACHE-INV-LABEL: flat_nontemporal_load_0:70; SKIP-CACHE-INV: ; %bb.0: ; %entry71; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x072; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x273; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)74; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s275; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s376; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1] glc77; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)78; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s079; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s180; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)81; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v282; SKIP-CACHE-INV-NEXT: s_endpgm83;84; GFX11-WGP-LABEL: flat_nontemporal_load_0:85; GFX11-WGP: ; %bb.0: ; %entry86; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x087; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x888; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)89; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s290; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s391; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc dlc92; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)93; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s094; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s195; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)96; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v297; GFX11-WGP-NEXT: s_endpgm98;99; GFX11-CU-LABEL: flat_nontemporal_load_0:100; GFX11-CU: ; %bb.0: ; %entry101; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0102; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8103; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)104; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2105; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3106; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1] glc dlc107; GFX11-CU-NEXT: s_waitcnt vmcnt(0)108; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0109; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1110; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)111; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2112; GFX11-CU-NEXT: s_endpgm113;114; GFX12-WGP-LABEL: flat_nontemporal_load_0:115; GFX12-WGP: ; %bb.0: ; %entry116; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0117; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8118; GFX12-WGP-NEXT: s_wait_kmcnt 0x0119; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2120; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3121; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SYS122; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0123; GFX12-WGP-NEXT: s_wait_samplecnt 0x0124; GFX12-WGP-NEXT: s_wait_loadcnt 0x0125; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0126; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1127; GFX12-WGP-NEXT: s_wait_dscnt 0x0128; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2129; GFX12-WGP-NEXT: s_endpgm130;131; GFX12-CU-LABEL: flat_nontemporal_load_0:132; GFX12-CU: ; %bb.0: ; %entry133; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0134; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8135; GFX12-CU-NEXT: s_wait_kmcnt 0x0136; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2137; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3138; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SYS139; GFX12-CU-NEXT: s_wait_bvhcnt 0x0140; GFX12-CU-NEXT: s_wait_samplecnt 0x0141; GFX12-CU-NEXT: s_wait_loadcnt 0x0142; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0143; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1144; GFX12-CU-NEXT: s_wait_dscnt 0x0145; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2146; GFX12-CU-NEXT: s_endpgm147;148; GFX1250-LABEL: flat_nontemporal_load_0:149; GFX1250: ; %bb.0: ; %entry150; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1151; GFX1250-NEXT: v_mov_b32_e32 v0, 0152; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x0153; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8154; GFX1250-NEXT: s_wait_xcnt 0x0155; GFX1250-NEXT: s_wait_kmcnt 0x0156; GFX1250-NEXT: flat_load_b32 v1, v0, s[2:3] scope:SCOPE_SYS157; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0158; GFX1250-NEXT: flat_store_b32 v0, v1, s[0:1]159; GFX1250-NEXT: s_endpgm160 ptr %in, ptr %out) {161entry:162 %val = load volatile i32, ptr %in, align 4163 store i32 %val, ptr %out164 ret void165}166 167define amdgpu_kernel void @flat_nontemporal_load_1(168; GFX7-LABEL: flat_nontemporal_load_1:169; GFX7: ; %bb.0: ; %entry170; GFX7-NEXT: s_mov_b32 flat_scratch_lo, s13171; GFX7-NEXT: s_add_i32 s12, s12, s17172; GFX7-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8173; GFX7-NEXT: s_mov_b64 s[4:5], s[8:9]174; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0175; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x2176; GFX7-NEXT: s_mov_b32 s6, 2177; GFX7-NEXT: v_lshlrev_b32_e64 v1, s6, v0178; GFX7-NEXT: v_mov_b32_e32 v0, 0179; GFX7-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec180; GFX7-NEXT: v_mov_b32_e32 v2, v0181; GFX7-NEXT: s_waitcnt lgkmcnt(0)182; GFX7-NEXT: s_mov_b32 s6, s8183; GFX7-NEXT: v_mov_b32_e32 v0, v1184; GFX7-NEXT: s_mov_b32 s8, s9185; GFX7-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $vgpr1_vgpr2 killed $exec186; GFX7-NEXT: v_add_i32_e64 v0, s[6:7], s6, v0187; GFX7-NEXT: v_mov_b32_e32 v1, s8188; GFX7-NEXT: v_addc_u32_e64 v2, s[6:7], v1, v2, s[6:7]189; GFX7-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec190; GFX7-NEXT: v_mov_b32_e32 v1, v2191; GFX7-NEXT: flat_load_dword v2, v[0:1] glc192; GFX7-NEXT: s_waitcnt vmcnt(0)193; GFX7-NEXT: v_mov_b32_e32 v0, s4194; GFX7-NEXT: v_mov_b32_e32 v1, s5195; GFX7-NEXT: s_waitcnt lgkmcnt(0)196; GFX7-NEXT: flat_store_dword v[0:1], v2197; GFX7-NEXT: s_endpgm198;199; GFX10-WGP-LABEL: flat_nontemporal_load_1:200; GFX10-WGP: ; %bb.0: ; %entry201; GFX10-WGP-NEXT: s_add_u32 s12, s12, s17202; GFX10-WGP-NEXT: s_addc_u32 s13, s13, 0203; GFX10-WGP-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12204; GFX10-WGP-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13205; GFX10-WGP-NEXT: s_mov_b64 s[4:5], s[8:9]206; GFX10-WGP-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0207; GFX10-WGP-NEXT: s_nop 0208; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8209; GFX10-WGP-NEXT: s_mov_b32 s6, 2210; GFX10-WGP-NEXT: v_lshlrev_b32_e64 v1, s6, v0211; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0212; GFX10-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec213; GFX10-WGP-NEXT: v_mov_b32_e32 v2, v0214; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)215; GFX10-WGP-NEXT: s_mov_b32 s7, s8216; GFX10-WGP-NEXT: v_mov_b32_e32 v0, v1217; GFX10-WGP-NEXT: s_mov_b32 s6, s9218; GFX10-WGP-NEXT: v_mov_b32_e32 v1, v2219; GFX10-WGP-NEXT: v_add_co_u32 v0, s7, s7, v0220; GFX10-WGP-NEXT: v_add_co_ci_u32_e64 v2, s6, s6, v1, s7221; GFX10-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec222; GFX10-WGP-NEXT: v_mov_b32_e32 v1, v2223; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc dlc224; GFX10-WGP-NEXT: s_waitcnt vmcnt(0)225; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4226; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5227; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)228; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2229; GFX10-WGP-NEXT: s_endpgm230;231; GFX10-CU-LABEL: flat_nontemporal_load_1:232; GFX10-CU: ; %bb.0: ; %entry233; GFX10-CU-NEXT: s_add_u32 s12, s12, s17234; GFX10-CU-NEXT: s_addc_u32 s13, s13, 0235; GFX10-CU-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12236; GFX10-CU-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13237; GFX10-CU-NEXT: s_mov_b64 s[4:5], s[8:9]238; GFX10-CU-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0239; GFX10-CU-NEXT: s_nop 0240; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x8241; GFX10-CU-NEXT: s_mov_b32 s6, 2242; GFX10-CU-NEXT: v_lshlrev_b32_e64 v1, s6, v0243; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0244; GFX10-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec245; GFX10-CU-NEXT: v_mov_b32_e32 v2, v0246; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)247; GFX10-CU-NEXT: s_mov_b32 s7, s8248; GFX10-CU-NEXT: v_mov_b32_e32 v0, v1249; GFX10-CU-NEXT: s_mov_b32 s6, s9250; GFX10-CU-NEXT: v_mov_b32_e32 v1, v2251; GFX10-CU-NEXT: v_add_co_u32 v0, s7, s7, v0252; GFX10-CU-NEXT: v_add_co_ci_u32_e64 v2, s6, s6, v1, s7253; GFX10-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec254; GFX10-CU-NEXT: v_mov_b32_e32 v1, v2255; GFX10-CU-NEXT: flat_load_dword v2, v[0:1] glc dlc256; GFX10-CU-NEXT: s_waitcnt vmcnt(0)257; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4258; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5259; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)260; GFX10-CU-NEXT: flat_store_dword v[0:1], v2261; GFX10-CU-NEXT: s_endpgm262;263; SKIP-CACHE-INV-LABEL: flat_nontemporal_load_1:264; SKIP-CACHE-INV: ; %bb.0: ; %entry265; SKIP-CACHE-INV-NEXT: s_mov_b64 s[0:1], s[4:5]266; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0267; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x2268; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, 2269; SKIP-CACHE-INV-NEXT: v_lshlrev_b32_e64 v1, s2, v0270; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, 0271; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec272; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, v0273; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)274; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s4275; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, v1276; SKIP-CACHE-INV-NEXT: s_mov_b32 s4, s5277; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr2 killed $vgpr2 killed $vgpr1_vgpr2 killed $exec278; SKIP-CACHE-INV-NEXT: v_add_i32_e64 v0, s[2:3], s2, v0279; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s4280; SKIP-CACHE-INV-NEXT: v_addc_u32_e64 v2, s[2:3], v1, v2, s[2:3]281; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec282; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v2283; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1] glc284; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)285; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0286; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1287; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)288; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2289; SKIP-CACHE-INV-NEXT: s_endpgm290;291; GFX11-WGP-LABEL: flat_nontemporal_load_1:292; GFX11-WGP: ; %bb.0: ; %entry293; GFX11-WGP-NEXT: s_mov_b64 s[0:1], s[4:5]294; GFX11-WGP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0295; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8296; GFX11-WGP-NEXT: s_mov_b32 s2, 0x3ff297; GFX11-WGP-NEXT: v_and_b32_e64 v0, v0, s2298; GFX11-WGP-NEXT: s_mov_b32 s2, 2299; GFX11-WGP-NEXT: v_lshlrev_b32_e64 v1, s2, v0300; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0301; GFX11-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec302; GFX11-WGP-NEXT: v_mov_b32_e32 v2, v0303; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)304; GFX11-WGP-NEXT: s_mov_b32 s3, s4305; GFX11-WGP-NEXT: v_mov_b32_e32 v0, v1306; GFX11-WGP-NEXT: s_mov_b32 s2, s5307; GFX11-WGP-NEXT: v_mov_b32_e32 v1, v2308; GFX11-WGP-NEXT: v_add_co_u32 v0, s3, s3, v0309; GFX11-WGP-NEXT: v_add_co_ci_u32_e64 v2, s2, s2, v1, s3310; GFX11-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec311; GFX11-WGP-NEXT: v_mov_b32_e32 v1, v2312; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc dlc313; GFX11-WGP-NEXT: s_waitcnt vmcnt(0)314; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0315; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1316; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)317; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2318; GFX11-WGP-NEXT: s_endpgm319;320; GFX11-CU-LABEL: flat_nontemporal_load_1:321; GFX11-CU: ; %bb.0: ; %entry322; GFX11-CU-NEXT: s_mov_b64 s[0:1], s[4:5]323; GFX11-CU-NEXT: s_load_b64 s[4:5], s[0:1], 0x0324; GFX11-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8325; GFX11-CU-NEXT: s_mov_b32 s2, 0x3ff326; GFX11-CU-NEXT: v_and_b32_e64 v0, v0, s2327; GFX11-CU-NEXT: s_mov_b32 s2, 2328; GFX11-CU-NEXT: v_lshlrev_b32_e64 v1, s2, v0329; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0330; GFX11-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec331; GFX11-CU-NEXT: v_mov_b32_e32 v2, v0332; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)333; GFX11-CU-NEXT: s_mov_b32 s3, s4334; GFX11-CU-NEXT: v_mov_b32_e32 v0, v1335; GFX11-CU-NEXT: s_mov_b32 s2, s5336; GFX11-CU-NEXT: v_mov_b32_e32 v1, v2337; GFX11-CU-NEXT: v_add_co_u32 v0, s3, s3, v0338; GFX11-CU-NEXT: v_add_co_ci_u32_e64 v2, s2, s2, v1, s3339; GFX11-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec340; GFX11-CU-NEXT: v_mov_b32_e32 v1, v2341; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1] glc dlc342; GFX11-CU-NEXT: s_waitcnt vmcnt(0)343; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0344; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1345; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)346; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2347; GFX11-CU-NEXT: s_endpgm348;349; GFX12-WGP-LABEL: flat_nontemporal_load_1:350; GFX12-WGP: ; %bb.0: ; %entry351; GFX12-WGP-NEXT: s_mov_b64 s[0:1], s[4:5]352; GFX12-WGP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0353; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[0:1], 0x8354; GFX12-WGP-NEXT: s_mov_b32 s2, 0x3ff355; GFX12-WGP-NEXT: v_and_b32_e64 v0, v0, s2356; GFX12-WGP-NEXT: s_mov_b32 s2, 2357; GFX12-WGP-NEXT: s_wait_alu depctr_sa_sdst(0)358; GFX12-WGP-NEXT: v_lshlrev_b32_e64 v1, s2, v0359; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0360; GFX12-WGP-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec361; GFX12-WGP-NEXT: v_mov_b32_e32 v2, v0362; GFX12-WGP-NEXT: s_wait_kmcnt 0x0363; GFX12-WGP-NEXT: s_mov_b32 s3, s4364; GFX12-WGP-NEXT: v_mov_b32_e32 v0, v1365; GFX12-WGP-NEXT: s_mov_b32 s2, s5366; GFX12-WGP-NEXT: v_mov_b32_e32 v1, v2367; GFX12-WGP-NEXT: s_wait_alu depctr_sa_sdst(0)368; GFX12-WGP-NEXT: v_add_co_u32 v0, s3, s3, v0369; GFX12-WGP-NEXT: s_wait_alu depctr_va_sdst(0)370; GFX12-WGP-NEXT: v_add_co_ci_u32_e64 v2, s2, s2, v1, s3371; GFX12-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec372; GFX12-WGP-NEXT: v_mov_b32_e32 v1, v2373; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SYS374; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0375; GFX12-WGP-NEXT: s_wait_samplecnt 0x0376; GFX12-WGP-NEXT: s_wait_loadcnt 0x0377; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0378; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1379; GFX12-WGP-NEXT: s_wait_dscnt 0x0380; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2381; GFX12-WGP-NEXT: s_endpgm382;383; GFX12-CU-LABEL: flat_nontemporal_load_1:384; GFX12-CU: ; %bb.0: ; %entry385; GFX12-CU-NEXT: s_mov_b64 s[0:1], s[4:5]386; GFX12-CU-NEXT: s_load_b64 s[4:5], s[0:1], 0x0387; GFX12-CU-NEXT: s_load_b64 s[0:1], s[0:1], 0x8388; GFX12-CU-NEXT: s_mov_b32 s2, 0x3ff389; GFX12-CU-NEXT: v_and_b32_e64 v0, v0, s2390; GFX12-CU-NEXT: s_mov_b32 s2, 2391; GFX12-CU-NEXT: s_wait_alu depctr_sa_sdst(0)392; GFX12-CU-NEXT: v_lshlrev_b32_e64 v1, s2, v0393; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0394; GFX12-CU-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec395; GFX12-CU-NEXT: v_mov_b32_e32 v2, v0396; GFX12-CU-NEXT: s_wait_kmcnt 0x0397; GFX12-CU-NEXT: s_mov_b32 s3, s4398; GFX12-CU-NEXT: v_mov_b32_e32 v0, v1399; GFX12-CU-NEXT: s_mov_b32 s2, s5400; GFX12-CU-NEXT: v_mov_b32_e32 v1, v2401; GFX12-CU-NEXT: s_wait_alu depctr_sa_sdst(0)402; GFX12-CU-NEXT: v_add_co_u32 v0, s3, s3, v0403; GFX12-CU-NEXT: s_wait_alu depctr_va_sdst(0)404; GFX12-CU-NEXT: v_add_co_ci_u32_e64 v2, s2, s2, v1, s3405; GFX12-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec406; GFX12-CU-NEXT: v_mov_b32_e32 v1, v2407; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SYS408; GFX12-CU-NEXT: s_wait_bvhcnt 0x0409; GFX12-CU-NEXT: s_wait_samplecnt 0x0410; GFX12-CU-NEXT: s_wait_loadcnt 0x0411; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0412; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1413; GFX12-CU-NEXT: s_wait_dscnt 0x0414; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2415; GFX12-CU-NEXT: s_endpgm416;417; GFX1250-LABEL: flat_nontemporal_load_1:418; GFX1250: ; %bb.0: ; %entry419; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1420; GFX1250-NEXT: v_mov_b32_e32 v1, v0421; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x0422; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8423; GFX1250-NEXT: v_mov_b32_e32 v0, 0424; GFX1250-NEXT: s_wait_xcnt 0x0425; GFX1250-NEXT: s_mov_b32 s4, 0x3ff426; GFX1250-NEXT: v_and_b32_e64 v1, v1, s4427; GFX1250-NEXT: s_wait_xcnt 0x0428; GFX1250-NEXT: s_wait_kmcnt 0x0429; GFX1250-NEXT: flat_load_b32 v1, v1, s[2:3] scale_offset scope:SCOPE_SYS430; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0431; GFX1250-NEXT: flat_store_b32 v0, v1, s[0:1]432; GFX1250-NEXT: s_endpgm433 ptr %in, ptr %out) {434entry:435 %tid = call i32 @llvm.amdgcn.workitem.id.x()436 %val.gep = getelementptr inbounds i32, ptr %in, i32 %tid437 %val = load volatile i32, ptr %val.gep, align 4438 store i32 %val, ptr %out439 ret void440}441 442define amdgpu_kernel void @flat_nontemporal_store_0(443; GFX7-LABEL: flat_nontemporal_store_0:444; GFX7: ; %bb.0: ; %entry445; GFX7-NEXT: s_mov_b32 flat_scratch_lo, s13446; GFX7-NEXT: s_add_i32 s12, s12, s17447; GFX7-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8448; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0449; GFX7-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x2450; GFX7-NEXT: s_waitcnt lgkmcnt(0)451; GFX7-NEXT: v_mov_b32_e32 v0, s6452; GFX7-NEXT: v_mov_b32_e32 v1, s7453; GFX7-NEXT: flat_load_dword v2, v[0:1]454; GFX7-NEXT: v_mov_b32_e32 v0, s4455; GFX7-NEXT: v_mov_b32_e32 v1, s5456; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)457; GFX7-NEXT: flat_store_dword v[0:1], v2458; GFX7-NEXT: s_waitcnt vmcnt(0)459; GFX7-NEXT: s_endpgm460;461; GFX10-WGP-LABEL: flat_nontemporal_store_0:462; GFX10-WGP: ; %bb.0: ; %entry463; GFX10-WGP-NEXT: s_add_u32 s12, s12, s17464; GFX10-WGP-NEXT: s_addc_u32 s13, s13, 0465; GFX10-WGP-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12466; GFX10-WGP-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13467; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0468; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8469; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)470; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6471; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7472; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1]473; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4474; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5475; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)476; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2477; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0478; GFX10-WGP-NEXT: s_endpgm479;480; GFX10-CU-LABEL: flat_nontemporal_store_0:481; GFX10-CU: ; %bb.0: ; %entry482; GFX10-CU-NEXT: s_add_u32 s12, s12, s17483; GFX10-CU-NEXT: s_addc_u32 s13, s13, 0484; GFX10-CU-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12485; GFX10-CU-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13486; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0487; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8488; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)489; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6490; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7491; GFX10-CU-NEXT: flat_load_dword v2, v[0:1]492; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4493; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5494; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)495; GFX10-CU-NEXT: flat_store_dword v[0:1], v2496; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0497; GFX10-CU-NEXT: s_endpgm498;499; SKIP-CACHE-INV-LABEL: flat_nontemporal_store_0:500; SKIP-CACHE-INV: ; %bb.0: ; %entry501; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0502; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2503; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)504; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2505; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3506; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1]507; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0508; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1509; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)510; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2511; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)512; SKIP-CACHE-INV-NEXT: s_endpgm513;514; GFX11-WGP-LABEL: flat_nontemporal_store_0:515; GFX11-WGP: ; %bb.0: ; %entry516; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0517; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8518; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)519; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2520; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3521; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1]522; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0523; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1524; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)525; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 dlc526; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0527; GFX11-WGP-NEXT: s_endpgm528;529; GFX11-CU-LABEL: flat_nontemporal_store_0:530; GFX11-CU: ; %bb.0: ; %entry531; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0532; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8533; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)534; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2535; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3536; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1]537; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0538; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1539; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)540; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 dlc541; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0542; GFX11-CU-NEXT: s_endpgm543;544; GFX12-WGP-LABEL: flat_nontemporal_store_0:545; GFX12-WGP: ; %bb.0: ; %entry546; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0547; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8548; GFX12-WGP-NEXT: s_wait_kmcnt 0x0549; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2550; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3551; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1]552; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0553; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1554; GFX12-WGP-NEXT: s_wait_samplecnt 0x0555; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0556; GFX12-WGP-NEXT: s_wait_kmcnt 0x0557; GFX12-WGP-NEXT: s_wait_storecnt 0x0558; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0559; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SYS560; GFX12-WGP-NEXT: s_wait_storecnt 0x0561; GFX12-WGP-NEXT: s_endpgm562;563; GFX12-CU-LABEL: flat_nontemporal_store_0:564; GFX12-CU: ; %bb.0: ; %entry565; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0566; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8567; GFX12-CU-NEXT: s_wait_kmcnt 0x0568; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2569; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3570; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]571; GFX12-CU-NEXT: v_mov_b32_e32 v0, s0572; GFX12-CU-NEXT: v_mov_b32_e32 v1, s1573; GFX12-CU-NEXT: s_wait_samplecnt 0x0574; GFX12-CU-NEXT: s_wait_bvhcnt 0x0575; GFX12-CU-NEXT: s_wait_kmcnt 0x0576; GFX12-CU-NEXT: s_wait_storecnt 0x0577; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0578; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SYS579; GFX12-CU-NEXT: s_wait_storecnt 0x0580; GFX12-CU-NEXT: s_endpgm581;582; GFX1250-LABEL: flat_nontemporal_store_0:583; GFX1250: ; %bb.0: ; %entry584; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1585; GFX1250-NEXT: v_mov_b32_e32 v0, 0586; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x0587; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8588; GFX1250-NEXT: s_wait_kmcnt 0x0589; GFX1250-NEXT: flat_load_b32 v1, v0, s[2:3]590; GFX1250-NEXT: s_wait_xcnt 0x0591; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0592; GFX1250-NEXT: flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS593; GFX1250-NEXT: s_wait_storecnt 0x0594; GFX1250-NEXT: s_endpgm595 ptr %in, ptr %out) {596entry:597 %val = load i32, ptr %in, align 4598 store volatile i32 %val, ptr %out599 ret void600}601 602define amdgpu_kernel void @flat_nontemporal_store_1(603; GFX7-LABEL: flat_nontemporal_store_1:604; GFX7: ; %bb.0: ; %entry605; GFX7-NEXT: s_mov_b32 flat_scratch_lo, s13606; GFX7-NEXT: s_add_i32 s12, s12, s17607; GFX7-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8608; GFX7-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0609; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x2610; GFX7-NEXT: s_waitcnt lgkmcnt(0)611; GFX7-NEXT: v_mov_b32_e32 v1, s4612; GFX7-NEXT: v_mov_b32_e32 v2, s5613; GFX7-NEXT: flat_load_dword v2, v[1:2]614; GFX7-NEXT: s_mov_b32 s4, 2615; GFX7-NEXT: v_lshlrev_b32_e64 v3, s4, v0616; GFX7-NEXT: v_mov_b32_e32 v0, 0617; GFX7-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec618; GFX7-NEXT: v_mov_b32_e32 v4, v0619; GFX7-NEXT: s_mov_b32 s4, s6620; GFX7-NEXT: v_mov_b32_e32 v0, v3621; GFX7-NEXT: s_mov_b32 s6, s7622; GFX7-NEXT: v_mov_b32_e32 v3, v4623; GFX7-NEXT: v_add_i32_e64 v0, s[4:5], s4, v0624; GFX7-NEXT: v_mov_b32_e32 v1, s6625; GFX7-NEXT: v_addc_u32_e64 v3, s[4:5], v1, v3, s[4:5]626; GFX7-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec627; GFX7-NEXT: v_mov_b32_e32 v1, v3628; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)629; GFX7-NEXT: flat_store_dword v[0:1], v2630; GFX7-NEXT: s_waitcnt vmcnt(0)631; GFX7-NEXT: s_endpgm632;633; GFX10-WGP-LABEL: flat_nontemporal_store_1:634; GFX10-WGP: ; %bb.0: ; %entry635; GFX10-WGP-NEXT: s_add_u32 s12, s12, s17636; GFX10-WGP-NEXT: s_addc_u32 s13, s13, 0637; GFX10-WGP-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12638; GFX10-WGP-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13639; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0640; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8641; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)642; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s4643; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s5644; GFX10-WGP-NEXT: flat_load_dword v2, v[1:2]645; GFX10-WGP-NEXT: s_mov_b32 s4, 2646; GFX10-WGP-NEXT: v_lshlrev_b32_e64 v3, s4, v0647; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0648; GFX10-WGP-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec649; GFX10-WGP-NEXT: v_mov_b32_e32 v4, v0650; GFX10-WGP-NEXT: s_mov_b32 s5, s6651; GFX10-WGP-NEXT: v_mov_b32_e32 v0, v3652; GFX10-WGP-NEXT: s_mov_b32 s4, s7653; GFX10-WGP-NEXT: v_mov_b32_e32 v1, v4654; GFX10-WGP-NEXT: v_add_co_u32 v0, s5, s5, v0655; GFX10-WGP-NEXT: v_add_co_ci_u32_e64 v3, s4, s4, v1, s5656; GFX10-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec657; GFX10-WGP-NEXT: v_mov_b32_e32 v1, v3658; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)659; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2660; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x0661; GFX10-WGP-NEXT: s_endpgm662;663; GFX10-CU-LABEL: flat_nontemporal_store_1:664; GFX10-CU: ; %bb.0: ; %entry665; GFX10-CU-NEXT: s_add_u32 s12, s12, s17666; GFX10-CU-NEXT: s_addc_u32 s13, s13, 0667; GFX10-CU-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12668; GFX10-CU-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13669; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0670; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8671; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)672; GFX10-CU-NEXT: v_mov_b32_e32 v1, s4673; GFX10-CU-NEXT: v_mov_b32_e32 v2, s5674; GFX10-CU-NEXT: flat_load_dword v2, v[1:2]675; GFX10-CU-NEXT: s_mov_b32 s4, 2676; GFX10-CU-NEXT: v_lshlrev_b32_e64 v3, s4, v0677; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0678; GFX10-CU-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec679; GFX10-CU-NEXT: v_mov_b32_e32 v4, v0680; GFX10-CU-NEXT: s_mov_b32 s5, s6681; GFX10-CU-NEXT: v_mov_b32_e32 v0, v3682; GFX10-CU-NEXT: s_mov_b32 s4, s7683; GFX10-CU-NEXT: v_mov_b32_e32 v1, v4684; GFX10-CU-NEXT: v_add_co_u32 v0, s5, s5, v0685; GFX10-CU-NEXT: v_add_co_ci_u32_e64 v3, s4, s4, v1, s5686; GFX10-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec687; GFX10-CU-NEXT: v_mov_b32_e32 v1, v3688; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)689; GFX10-CU-NEXT: flat_store_dword v[0:1], v2690; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x0691; GFX10-CU-NEXT: s_endpgm692;693; SKIP-CACHE-INV-LABEL: flat_nontemporal_store_1:694; SKIP-CACHE-INV: ; %bb.0: ; %entry695; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0696; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2697; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)698; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s1699; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s0700; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[1:2]701; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, 2702; SKIP-CACHE-INV-NEXT: v_lshlrev_b32_e64 v3, s0, v0703; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, 0704; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec705; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v4, v0706; SKIP-CACHE-INV-NEXT: s_mov_b32 s0, s2707; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, v3708; SKIP-CACHE-INV-NEXT: s_mov_b32 s2, s3709; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v3, v4710; SKIP-CACHE-INV-NEXT: v_add_i32_e64 v0, s[0:1], s0, v0711; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s2712; SKIP-CACHE-INV-NEXT: v_addc_u32_e64 v3, s[0:1], v1, v3, s[0:1]713; SKIP-CACHE-INV-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec714; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, v3715; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)716; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2717; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)718; SKIP-CACHE-INV-NEXT: s_endpgm719;720; GFX11-WGP-LABEL: flat_nontemporal_store_1:721; GFX11-WGP: ; %bb.0: ; %entry722; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0723; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x8724; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)725; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s1726; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s0727; GFX11-WGP-NEXT: flat_load_b32 v2, v[1:2]728; GFX11-WGP-NEXT: s_mov_b32 s0, 0x3ff729; GFX11-WGP-NEXT: v_and_b32_e64 v0, v0, s0730; GFX11-WGP-NEXT: s_mov_b32 s0, 2731; GFX11-WGP-NEXT: v_lshlrev_b32_e64 v3, s0, v0732; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0733; GFX11-WGP-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec734; GFX11-WGP-NEXT: v_mov_b32_e32 v4, v0735; GFX11-WGP-NEXT: s_mov_b32 s1, s2736; GFX11-WGP-NEXT: v_mov_b32_e32 v0, v3737; GFX11-WGP-NEXT: s_mov_b32 s0, s3738; GFX11-WGP-NEXT: v_mov_b32_e32 v1, v4739; GFX11-WGP-NEXT: v_add_co_u32 v0, s1, s1, v0740; GFX11-WGP-NEXT: v_add_co_ci_u32_e64 v3, s0, s0, v1, s1741; GFX11-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec742; GFX11-WGP-NEXT: v_mov_b32_e32 v1, v3743; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)744; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2 dlc745; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x0746; GFX11-WGP-NEXT: s_endpgm747;748; GFX11-CU-LABEL: flat_nontemporal_store_1:749; GFX11-CU: ; %bb.0: ; %entry750; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0751; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x8752; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)753; GFX11-CU-NEXT: v_mov_b32_e32 v2, s1754; GFX11-CU-NEXT: v_mov_b32_e32 v1, s0755; GFX11-CU-NEXT: flat_load_b32 v2, v[1:2]756; GFX11-CU-NEXT: s_mov_b32 s0, 0x3ff757; GFX11-CU-NEXT: v_and_b32_e64 v0, v0, s0758; GFX11-CU-NEXT: s_mov_b32 s0, 2759; GFX11-CU-NEXT: v_lshlrev_b32_e64 v3, s0, v0760; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0761; GFX11-CU-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec762; GFX11-CU-NEXT: v_mov_b32_e32 v4, v0763; GFX11-CU-NEXT: s_mov_b32 s1, s2764; GFX11-CU-NEXT: v_mov_b32_e32 v0, v3765; GFX11-CU-NEXT: s_mov_b32 s0, s3766; GFX11-CU-NEXT: v_mov_b32_e32 v1, v4767; GFX11-CU-NEXT: v_add_co_u32 v0, s1, s1, v0768; GFX11-CU-NEXT: v_add_co_ci_u32_e64 v3, s0, s0, v1, s1769; GFX11-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec770; GFX11-CU-NEXT: v_mov_b32_e32 v1, v3771; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)772; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2 dlc773; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x0774; GFX11-CU-NEXT: s_endpgm775;776; GFX12-WGP-LABEL: flat_nontemporal_store_1:777; GFX12-WGP: ; %bb.0: ; %entry778; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0779; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x8780; GFX12-WGP-NEXT: s_wait_kmcnt 0x0781; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s1782; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s0783; GFX12-WGP-NEXT: flat_load_b32 v2, v[1:2]784; GFX12-WGP-NEXT: s_mov_b32 s0, 0x3ff785; GFX12-WGP-NEXT: s_wait_alu depctr_sa_sdst(0)786; GFX12-WGP-NEXT: v_and_b32_e64 v0, v0, s0787; GFX12-WGP-NEXT: s_mov_b32 s0, 2788; GFX12-WGP-NEXT: s_wait_alu depctr_sa_sdst(0)789; GFX12-WGP-NEXT: v_lshlrev_b32_e64 v3, s0, v0790; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0791; GFX12-WGP-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec792; GFX12-WGP-NEXT: v_mov_b32_e32 v4, v0793; GFX12-WGP-NEXT: s_mov_b32 s1, s2794; GFX12-WGP-NEXT: v_mov_b32_e32 v0, v3795; GFX12-WGP-NEXT: s_mov_b32 s0, s3796; GFX12-WGP-NEXT: v_mov_b32_e32 v1, v4797; GFX12-WGP-NEXT: s_wait_alu depctr_sa_sdst(0)798; GFX12-WGP-NEXT: v_add_co_u32 v0, s1, s1, v0799; GFX12-WGP-NEXT: s_wait_alu depctr_va_sdst(0)800; GFX12-WGP-NEXT: v_add_co_ci_u32_e64 v3, s0, s0, v1, s1801; GFX12-WGP-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec802; GFX12-WGP-NEXT: v_mov_b32_e32 v1, v3803; GFX12-WGP-NEXT: s_wait_samplecnt 0x0804; GFX12-WGP-NEXT: s_wait_bvhcnt 0x0805; GFX12-WGP-NEXT: s_wait_kmcnt 0x0806; GFX12-WGP-NEXT: s_wait_storecnt 0x0807; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0808; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SYS809; GFX12-WGP-NEXT: s_wait_storecnt 0x0810; GFX12-WGP-NEXT: s_endpgm811;812; GFX12-CU-LABEL: flat_nontemporal_store_1:813; GFX12-CU: ; %bb.0: ; %entry814; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0815; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x8816; GFX12-CU-NEXT: s_wait_kmcnt 0x0817; GFX12-CU-NEXT: v_mov_b32_e32 v2, s1818; GFX12-CU-NEXT: v_mov_b32_e32 v1, s0819; GFX12-CU-NEXT: flat_load_b32 v2, v[1:2]820; GFX12-CU-NEXT: s_mov_b32 s0, 0x3ff821; GFX12-CU-NEXT: s_wait_alu depctr_sa_sdst(0)822; GFX12-CU-NEXT: v_and_b32_e64 v0, v0, s0823; GFX12-CU-NEXT: s_mov_b32 s0, 2824; GFX12-CU-NEXT: s_wait_alu depctr_sa_sdst(0)825; GFX12-CU-NEXT: v_lshlrev_b32_e64 v3, s0, v0826; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0827; GFX12-CU-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec828; GFX12-CU-NEXT: v_mov_b32_e32 v4, v0829; GFX12-CU-NEXT: s_mov_b32 s1, s2830; GFX12-CU-NEXT: v_mov_b32_e32 v0, v3831; GFX12-CU-NEXT: s_mov_b32 s0, s3832; GFX12-CU-NEXT: v_mov_b32_e32 v1, v4833; GFX12-CU-NEXT: s_wait_alu depctr_sa_sdst(0)834; GFX12-CU-NEXT: v_add_co_u32 v0, s1, s1, v0835; GFX12-CU-NEXT: s_wait_alu depctr_va_sdst(0)836; GFX12-CU-NEXT: v_add_co_ci_u32_e64 v3, s0, s0, v1, s1837; GFX12-CU-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec838; GFX12-CU-NEXT: v_mov_b32_e32 v1, v3839; GFX12-CU-NEXT: s_wait_samplecnt 0x0840; GFX12-CU-NEXT: s_wait_bvhcnt 0x0841; GFX12-CU-NEXT: s_wait_kmcnt 0x0842; GFX12-CU-NEXT: s_wait_storecnt 0x0843; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x0844; GFX12-CU-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SYS845; GFX12-CU-NEXT: s_wait_storecnt 0x0846; GFX12-CU-NEXT: s_endpgm847;848; GFX1250-LABEL: flat_nontemporal_store_1:849; GFX1250: ; %bb.0: ; %entry850; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1851; GFX1250-NEXT: v_mov_b32_e32 v1, 0852; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x0853; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8854; GFX1250-NEXT: s_wait_kmcnt 0x0855; GFX1250-NEXT: flat_load_b32 v1, v1, s[2:3]856; GFX1250-NEXT: s_wait_xcnt 0x0857; GFX1250-NEXT: s_mov_b32 s2, 0x3ff858; GFX1250-NEXT: v_and_b32_e64 v0, v0, s2859; GFX1250-NEXT: s_wait_xcnt 0x0860; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0861; GFX1250-NEXT: flat_store_b32 v0, v1, s[0:1] scale_offset scope:SCOPE_SYS862; GFX1250-NEXT: s_wait_storecnt 0x0863; GFX1250-NEXT: s_endpgm864 ptr %in, ptr %out) {865entry:866 %tid = call i32 @llvm.amdgcn.workitem.id.x()867 %val = load i32, ptr %in, align 4868 %out.gep = getelementptr inbounds i32, ptr %out, i32 %tid869 store volatile i32 %val, ptr %out.gep870 ret void871}872 873define amdgpu_kernel void @flat_volatile_workgroup_acquire_load(874; GFX7-LABEL: flat_volatile_workgroup_acquire_load:875; GFX7: ; %bb.0: ; %entry876; GFX7-NEXT: s_mov_b32 flat_scratch_lo, s13877; GFX7-NEXT: s_add_i32 s12, s12, s17878; GFX7-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8879; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0880; GFX7-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x2881; GFX7-NEXT: s_waitcnt lgkmcnt(0)882; GFX7-NEXT: v_mov_b32_e32 v0, s6883; GFX7-NEXT: v_mov_b32_e32 v1, s7884; GFX7-NEXT: flat_load_dword v2, v[0:1]885; GFX7-NEXT: s_waitcnt lgkmcnt(0)886; GFX7-NEXT: v_mov_b32_e32 v0, s4887; GFX7-NEXT: v_mov_b32_e32 v1, s5888; GFX7-NEXT: s_waitcnt vmcnt(0)889; GFX7-NEXT: flat_store_dword v[0:1], v2890; GFX7-NEXT: s_endpgm891;892; GFX10-WGP-LABEL: flat_volatile_workgroup_acquire_load:893; GFX10-WGP: ; %bb.0: ; %entry894; GFX10-WGP-NEXT: s_add_u32 s12, s12, s17895; GFX10-WGP-NEXT: s_addc_u32 s13, s13, 0896; GFX10-WGP-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12897; GFX10-WGP-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13898; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0899; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8900; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)901; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s6902; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s7903; GFX10-WGP-NEXT: flat_load_dword v2, v[0:1] glc904; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)905; GFX10-WGP-NEXT: buffer_gl0_inv906; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4907; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s5908; GFX10-WGP-NEXT: flat_store_dword v[0:1], v2909; GFX10-WGP-NEXT: s_endpgm910;911; GFX10-CU-LABEL: flat_volatile_workgroup_acquire_load:912; GFX10-CU: ; %bb.0: ; %entry913; GFX10-CU-NEXT: s_add_u32 s12, s12, s17914; GFX10-CU-NEXT: s_addc_u32 s13, s13, 0915; GFX10-CU-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12916; GFX10-CU-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13917; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0918; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8919; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)920; GFX10-CU-NEXT: v_mov_b32_e32 v0, s6921; GFX10-CU-NEXT: v_mov_b32_e32 v1, s7922; GFX10-CU-NEXT: flat_load_dword v2, v[0:1]923; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)924; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4925; GFX10-CU-NEXT: v_mov_b32_e32 v1, s5926; GFX10-CU-NEXT: s_waitcnt vmcnt(0)927; GFX10-CU-NEXT: flat_store_dword v[0:1], v2928; GFX10-CU-NEXT: s_endpgm929;930; SKIP-CACHE-INV-LABEL: flat_volatile_workgroup_acquire_load:931; SKIP-CACHE-INV: ; %bb.0: ; %entry932; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0933; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2934; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)935; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s2936; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s3937; SKIP-CACHE-INV-NEXT: flat_load_dword v2, v[0:1]938; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)939; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s0940; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s1941; SKIP-CACHE-INV-NEXT: s_waitcnt vmcnt(0)942; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v2943; SKIP-CACHE-INV-NEXT: s_endpgm944;945; GFX11-WGP-LABEL: flat_volatile_workgroup_acquire_load:946; GFX11-WGP: ; %bb.0: ; %entry947; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0948; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8949; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)950; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s2951; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s3952; GFX11-WGP-NEXT: flat_load_b32 v2, v[0:1] glc953; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)954; GFX11-WGP-NEXT: buffer_gl0_inv955; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0956; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s1957; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v2958; GFX11-WGP-NEXT: s_endpgm959;960; GFX11-CU-LABEL: flat_volatile_workgroup_acquire_load:961; GFX11-CU: ; %bb.0: ; %entry962; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0963; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8964; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)965; GFX11-CU-NEXT: v_mov_b32_e32 v0, s2966; GFX11-CU-NEXT: v_mov_b32_e32 v1, s3967; GFX11-CU-NEXT: flat_load_b32 v2, v[0:1]968; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)969; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0970; GFX11-CU-NEXT: v_mov_b32_e32 v1, s1971; GFX11-CU-NEXT: s_waitcnt vmcnt(0)972; GFX11-CU-NEXT: flat_store_b32 v[0:1], v2973; GFX11-CU-NEXT: s_endpgm974;975; GFX12-WGP-LABEL: flat_volatile_workgroup_acquire_load:976; GFX12-WGP: ; %bb.0: ; %entry977; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0978; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8979; GFX12-WGP-NEXT: s_wait_kmcnt 0x0980; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s2981; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s3982; GFX12-WGP-NEXT: flat_load_b32 v2, v[0:1] scope:SCOPE_SE983; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x0984; GFX12-WGP-NEXT: global_inv scope:SCOPE_SE985; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s0986; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s1987; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2988; GFX12-WGP-NEXT: s_endpgm989;990; GFX12-CU-LABEL: flat_volatile_workgroup_acquire_load:991; GFX12-CU: ; %bb.0: ; %entry992; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0993; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8994; GFX12-CU-NEXT: s_wait_kmcnt 0x0995; GFX12-CU-NEXT: v_mov_b32_e32 v0, s2996; GFX12-CU-NEXT: v_mov_b32_e32 v1, s3997; GFX12-CU-NEXT: flat_load_b32 v2, v[0:1]998; GFX12-CU-NEXT: s_wait_dscnt 0x0999; GFX12-CU-NEXT: v_mov_b32_e32 v0, s01000; GFX12-CU-NEXT: v_mov_b32_e32 v1, s11001; GFX12-CU-NEXT: s_wait_loadcnt 0x01002; GFX12-CU-NEXT: flat_store_b32 v[0:1], v21003; GFX12-CU-NEXT: s_endpgm1004;1005; GFX1250-LABEL: flat_volatile_workgroup_acquire_load:1006; GFX1250: ; %bb.0: ; %entry1007; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11008; GFX1250-NEXT: v_mov_b32_e32 v0, 01009; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x01010; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x81011; GFX1250-NEXT: s_wait_kmcnt 0x01012; GFX1250-NEXT: flat_load_b32 v1, v0, s[2:3]1013; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x01014; GFX1250-NEXT: flat_store_b32 v0, v1, s[0:1]1015; GFX1250-NEXT: s_endpgm1016 ptr %in, ptr %out) {1017entry:1018 %val = load atomic volatile i32, ptr %in syncscope("workgroup") acquire, align 41019 store i32 %val, ptr %out1020 ret void1021}1022 1023define amdgpu_kernel void @flat_volatile_workgroup_release_store(1024; GFX7-LABEL: flat_volatile_workgroup_release_store:1025; GFX7: ; %bb.0: ; %entry1026; GFX7-NEXT: s_mov_b32 flat_scratch_lo, s131027; GFX7-NEXT: s_add_i32 s12, s12, s171028; GFX7-NEXT: s_lshr_b32 flat_scratch_hi, s12, 81029; GFX7-NEXT: s_load_dword s4, s[8:9], 0x01030; GFX7-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x21031; GFX7-NEXT: s_waitcnt lgkmcnt(0)1032; GFX7-NEXT: v_mov_b32_e32 v0, s61033; GFX7-NEXT: v_mov_b32_e32 v1, s71034; GFX7-NEXT: v_mov_b32_e32 v2, s41035; GFX7-NEXT: s_waitcnt lgkmcnt(0)1036; GFX7-NEXT: flat_store_dword v[0:1], v21037; GFX7-NEXT: s_endpgm1038;1039; GFX10-WGP-LABEL: flat_volatile_workgroup_release_store:1040; GFX10-WGP: ; %bb.0: ; %entry1041; GFX10-WGP-NEXT: s_add_u32 s12, s12, s171042; GFX10-WGP-NEXT: s_addc_u32 s13, s13, 01043; GFX10-WGP-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s121044; GFX10-WGP-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s131045; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x01046; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x81047; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)1048; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s61049; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s71050; GFX10-WGP-NEXT: v_mov_b32_e32 v2, s41051; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1052; GFX10-WGP-NEXT: s_waitcnt_vscnt null, 0x01053; GFX10-WGP-NEXT: flat_store_dword v[0:1], v21054; GFX10-WGP-NEXT: s_endpgm1055;1056; GFX10-CU-LABEL: flat_volatile_workgroup_release_store:1057; GFX10-CU: ; %bb.0: ; %entry1058; GFX10-CU-NEXT: s_add_u32 s12, s12, s171059; GFX10-CU-NEXT: s_addc_u32 s13, s13, 01060; GFX10-CU-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s121061; GFX10-CU-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s131062; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x01063; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x81064; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)1065; GFX10-CU-NEXT: v_mov_b32_e32 v0, s61066; GFX10-CU-NEXT: v_mov_b32_e32 v1, s71067; GFX10-CU-NEXT: v_mov_b32_e32 v2, s41068; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1069; GFX10-CU-NEXT: s_waitcnt_vscnt null, 0x01070; GFX10-CU-NEXT: flat_store_dword v[0:1], v21071; GFX10-CU-NEXT: s_endpgm1072;1073; SKIP-CACHE-INV-LABEL: flat_volatile_workgroup_release_store:1074; SKIP-CACHE-INV: ; %bb.0: ; %entry1075; SKIP-CACHE-INV-NEXT: s_load_dword s0, s[4:5], 0x01076; SKIP-CACHE-INV-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x21077; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)1078; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v0, s21079; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v1, s31080; SKIP-CACHE-INV-NEXT: v_mov_b32_e32 v2, s01081; SKIP-CACHE-INV-NEXT: s_waitcnt lgkmcnt(0)1082; SKIP-CACHE-INV-NEXT: flat_store_dword v[0:1], v21083; SKIP-CACHE-INV-NEXT: s_endpgm1084;1085; GFX11-WGP-LABEL: flat_volatile_workgroup_release_store:1086; GFX11-WGP: ; %bb.0: ; %entry1087; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x01088; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x81089; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)1090; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s21091; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s31092; GFX11-WGP-NEXT: v_mov_b32_e32 v2, s01093; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1094; GFX11-WGP-NEXT: s_waitcnt_vscnt null, 0x01095; GFX11-WGP-NEXT: flat_store_b32 v[0:1], v21096; GFX11-WGP-NEXT: s_endpgm1097;1098; GFX11-CU-LABEL: flat_volatile_workgroup_release_store:1099; GFX11-CU: ; %bb.0: ; %entry1100; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x01101; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x81102; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)1103; GFX11-CU-NEXT: v_mov_b32_e32 v0, s21104; GFX11-CU-NEXT: v_mov_b32_e32 v1, s31105; GFX11-CU-NEXT: v_mov_b32_e32 v2, s01106; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1107; GFX11-CU-NEXT: s_waitcnt_vscnt null, 0x01108; GFX11-CU-NEXT: flat_store_b32 v[0:1], v21109; GFX11-CU-NEXT: s_endpgm1110;1111; GFX12-WGP-LABEL: flat_volatile_workgroup_release_store:1112; GFX12-WGP: ; %bb.0: ; %entry1113; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x01114; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x81115; GFX12-WGP-NEXT: s_wait_kmcnt 0x01116; GFX12-WGP-NEXT: v_mov_b32_e32 v0, s21117; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s31118; GFX12-WGP-NEXT: v_mov_b32_e32 v2, s01119; GFX12-WGP-NEXT: s_wait_bvhcnt 0x01120; GFX12-WGP-NEXT: s_wait_samplecnt 0x01121; GFX12-WGP-NEXT: s_wait_storecnt 0x01122; GFX12-WGP-NEXT: s_wait_loadcnt_dscnt 0x01123; GFX12-WGP-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SE1124; GFX12-WGP-NEXT: s_endpgm1125;1126; GFX12-CU-LABEL: flat_volatile_workgroup_release_store:1127; GFX12-CU: ; %bb.0: ; %entry1128; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x01129; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x81130; GFX12-CU-NEXT: s_wait_kmcnt 0x01131; GFX12-CU-NEXT: v_mov_b32_e32 v0, s21132; GFX12-CU-NEXT: v_mov_b32_e32 v1, s31133; GFX12-CU-NEXT: v_mov_b32_e32 v2, s01134; GFX12-CU-NEXT: s_wait_bvhcnt 0x01135; GFX12-CU-NEXT: s_wait_samplecnt 0x01136; GFX12-CU-NEXT: s_wait_storecnt 0x01137; GFX12-CU-NEXT: s_wait_loadcnt_dscnt 0x01138; GFX12-CU-NEXT: flat_store_b32 v[0:1], v21139; GFX12-CU-NEXT: s_endpgm1140;1141; GFX1250-LABEL: flat_volatile_workgroup_release_store:1142; GFX1250: ; %bb.0: ; %entry1143; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11144; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x01145; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x81146; GFX1250-NEXT: v_mov_b32_e32 v0, 01147; GFX1250-NEXT: s_wait_kmcnt 0x01148; GFX1250-NEXT: v_mov_b32_e32 v1, s21149; GFX1250-NEXT: s_wait_storecnt 0x01150; GFX1250-NEXT: s_wait_xcnt 0x01151; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x01152; GFX1250-NEXT: flat_store_b32 v0, v1, s[0:1]1153; GFX1250-NEXT: s_endpgm1154 i32 %in, ptr %out) {1155entry:1156 store atomic volatile i32 %in, ptr %out syncscope("workgroup") release, align 41157 ret void1158}1159 1160declare i32 @llvm.amdgcn.workitem.id.x()1161