11529 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -mcpu=bonaire < %s | FileCheck -check-prefix=GFX7 %s3; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=GFX8 %s4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9 %s5 6; ---------------------------------------------------------------------7; atomicrmw xchg8; ---------------------------------------------------------------------9 10define void @flat_atomic_xchg_i64_noret(ptr %ptr, i64 %in) {11; GFX7-LABEL: flat_atomic_xchg_i64_noret:12; GFX7: ; %bb.0:13; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)14; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]15; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)16; GFX7-NEXT: buffer_wbinvl1_vol17; GFX7-NEXT: s_setpc_b64 s[30:31]18;19; GFX8-LABEL: flat_atomic_xchg_i64_noret:20; GFX8: ; %bb.0:21; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)22; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]23; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)24; GFX8-NEXT: buffer_wbinvl1_vol25; GFX8-NEXT: s_setpc_b64 s[30:31]26;27; GFX9-LABEL: flat_atomic_xchg_i64_noret:28; GFX9: ; %bb.0:29; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)30; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]31; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)32; GFX9-NEXT: buffer_wbinvl1_vol33; GFX9-NEXT: s_setpc_b64 s[30:31]34 %tmp0 = atomicrmw xchg ptr %ptr, i64 %in seq_cst, !noalias.addrspace !135 ret void36}37 38define void @flat_atomic_xchg_i64_noret_offset(ptr %out, i64 %in) {39; GFX7-LABEL: flat_atomic_xchg_i64_noret_offset:40; GFX7: ; %bb.0:41; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)42; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v043; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc44; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]45; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)46; GFX7-NEXT: buffer_wbinvl1_vol47; GFX7-NEXT: s_setpc_b64 s[30:31]48;49; GFX8-LABEL: flat_atomic_xchg_i64_noret_offset:50; GFX8: ; %bb.0:51; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)52; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v053; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc54; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]55; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)56; GFX8-NEXT: buffer_wbinvl1_vol57; GFX8-NEXT: s_setpc_b64 s[30:31]58;59; GFX9-LABEL: flat_atomic_xchg_i64_noret_offset:60; GFX9: ; %bb.0:61; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)62; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] offset:3263; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)64; GFX9-NEXT: buffer_wbinvl1_vol65; GFX9-NEXT: s_setpc_b64 s[30:31]66 %gep = getelementptr inbounds i64, ptr %out, i64 467 %tmp0 = atomicrmw xchg ptr %gep, i64 %in seq_cst, !noalias.addrspace !168 ret void69}70 71define i64 @flat_atomic_xchg_i64_ret(ptr %ptr, i64 %in) {72; GFX7-LABEL: flat_atomic_xchg_i64_ret:73; GFX7: ; %bb.0:74; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)75; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc76; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)77; GFX7-NEXT: buffer_wbinvl1_vol78; GFX7-NEXT: s_setpc_b64 s[30:31]79;80; GFX8-LABEL: flat_atomic_xchg_i64_ret:81; GFX8: ; %bb.0:82; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)83; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc84; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)85; GFX8-NEXT: buffer_wbinvl1_vol86; GFX8-NEXT: s_setpc_b64 s[30:31]87;88; GFX9-LABEL: flat_atomic_xchg_i64_ret:89; GFX9: ; %bb.0:90; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)91; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc92; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)93; GFX9-NEXT: buffer_wbinvl1_vol94; GFX9-NEXT: s_setpc_b64 s[30:31]95 %result = atomicrmw xchg ptr %ptr, i64 %in seq_cst, !noalias.addrspace !196 ret i64 %result97}98 99define i64 @flat_atomic_xchg_i64_ret_offset(ptr %out, i64 %in) {100; GFX7-LABEL: flat_atomic_xchg_i64_ret_offset:101; GFX7: ; %bb.0:102; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)103; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0104; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc105; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc106; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)107; GFX7-NEXT: buffer_wbinvl1_vol108; GFX7-NEXT: s_setpc_b64 s[30:31]109;110; GFX8-LABEL: flat_atomic_xchg_i64_ret_offset:111; GFX8: ; %bb.0:112; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)113; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0114; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc115; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc116; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)117; GFX8-NEXT: buffer_wbinvl1_vol118; GFX8-NEXT: s_setpc_b64 s[30:31]119;120; GFX9-LABEL: flat_atomic_xchg_i64_ret_offset:121; GFX9: ; %bb.0:122; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)123; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] offset:32 glc124; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)125; GFX9-NEXT: buffer_wbinvl1_vol126; GFX9-NEXT: s_setpc_b64 s[30:31]127 %gep = getelementptr inbounds i64, ptr %out, i64 4128 %result = atomicrmw xchg ptr %gep, i64 %in seq_cst, !noalias.addrspace !1129 ret i64 %result130}131 132define amdgpu_gfx void @flat_atomic_xchg_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {133; GFX7-LABEL: flat_atomic_xchg_i64_noret_scalar:134; GFX7: ; %bb.0:135; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)136; GFX7-NEXT: v_mov_b32_e32 v0, s6137; GFX7-NEXT: v_mov_b32_e32 v1, s7138; GFX7-NEXT: v_mov_b32_e32 v2, s4139; GFX7-NEXT: v_mov_b32_e32 v3, s5140; GFX7-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]141; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)142; GFX7-NEXT: buffer_wbinvl1_vol143; GFX7-NEXT: s_setpc_b64 s[30:31]144;145; GFX8-LABEL: flat_atomic_xchg_i64_noret_scalar:146; GFX8: ; %bb.0:147; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)148; GFX8-NEXT: v_mov_b32_e32 v0, s6149; GFX8-NEXT: v_mov_b32_e32 v1, s7150; GFX8-NEXT: v_mov_b32_e32 v2, s4151; GFX8-NEXT: v_mov_b32_e32 v3, s5152; GFX8-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]153; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)154; GFX8-NEXT: buffer_wbinvl1_vol155; GFX8-NEXT: s_setpc_b64 s[30:31]156;157; GFX9-LABEL: flat_atomic_xchg_i64_noret_scalar:158; GFX9: ; %bb.0:159; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)160; GFX9-NEXT: v_mov_b32_e32 v0, s6161; GFX9-NEXT: v_mov_b32_e32 v1, s7162; GFX9-NEXT: v_mov_b32_e32 v2, s4163; GFX9-NEXT: v_mov_b32_e32 v3, s5164; GFX9-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]165; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)166; GFX9-NEXT: buffer_wbinvl1_vol167; GFX9-NEXT: s_setpc_b64 s[30:31]168 %tmp0 = atomicrmw xchg ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1169 ret void170}171 172define amdgpu_gfx void @flat_atomic_xchg_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {173; GFX7-LABEL: flat_atomic_xchg_i64_noret_offset_scalar:174; GFX7: ; %bb.0:175; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)176; GFX7-NEXT: s_add_u32 s34, s4, 32177; GFX7-NEXT: s_addc_u32 s35, s5, 0178; GFX7-NEXT: v_mov_b32_e32 v2, s34179; GFX7-NEXT: v_mov_b32_e32 v0, s6180; GFX7-NEXT: v_mov_b32_e32 v1, s7181; GFX7-NEXT: v_mov_b32_e32 v3, s35182; GFX7-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]183; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)184; GFX7-NEXT: buffer_wbinvl1_vol185; GFX7-NEXT: s_setpc_b64 s[30:31]186;187; GFX8-LABEL: flat_atomic_xchg_i64_noret_offset_scalar:188; GFX8: ; %bb.0:189; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)190; GFX8-NEXT: s_add_u32 s34, s4, 32191; GFX8-NEXT: s_addc_u32 s35, s5, 0192; GFX8-NEXT: v_mov_b32_e32 v2, s34193; GFX8-NEXT: v_mov_b32_e32 v0, s6194; GFX8-NEXT: v_mov_b32_e32 v1, s7195; GFX8-NEXT: v_mov_b32_e32 v3, s35196; GFX8-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]197; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)198; GFX8-NEXT: buffer_wbinvl1_vol199; GFX8-NEXT: s_setpc_b64 s[30:31]200;201; GFX9-LABEL: flat_atomic_xchg_i64_noret_offset_scalar:202; GFX9: ; %bb.0:203; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)204; GFX9-NEXT: v_mov_b32_e32 v0, s6205; GFX9-NEXT: v_mov_b32_e32 v1, s7206; GFX9-NEXT: v_mov_b32_e32 v2, s4207; GFX9-NEXT: v_mov_b32_e32 v3, s5208; GFX9-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1] offset:32209; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)210; GFX9-NEXT: buffer_wbinvl1_vol211; GFX9-NEXT: s_setpc_b64 s[30:31]212 %gep = getelementptr inbounds i64, ptr %out, i64 4213 %tmp0 = atomicrmw xchg ptr %gep, i64 %in seq_cst, !noalias.addrspace !1214 ret void215}216 217define amdgpu_gfx i64 @flat_atomic_xchg_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {218; GFX7-LABEL: flat_atomic_xchg_i64_ret_scalar:219; GFX7: ; %bb.0:220; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)221; GFX7-NEXT: v_mov_b32_e32 v0, s6222; GFX7-NEXT: v_mov_b32_e32 v1, s7223; GFX7-NEXT: v_mov_b32_e32 v2, s4224; GFX7-NEXT: v_mov_b32_e32 v3, s5225; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc226; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)227; GFX7-NEXT: buffer_wbinvl1_vol228; GFX7-NEXT: s_setpc_b64 s[30:31]229;230; GFX8-LABEL: flat_atomic_xchg_i64_ret_scalar:231; GFX8: ; %bb.0:232; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)233; GFX8-NEXT: v_mov_b32_e32 v0, s6234; GFX8-NEXT: v_mov_b32_e32 v1, s7235; GFX8-NEXT: v_mov_b32_e32 v2, s4236; GFX8-NEXT: v_mov_b32_e32 v3, s5237; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc238; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)239; GFX8-NEXT: buffer_wbinvl1_vol240; GFX8-NEXT: s_setpc_b64 s[30:31]241;242; GFX9-LABEL: flat_atomic_xchg_i64_ret_scalar:243; GFX9: ; %bb.0:244; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)245; GFX9-NEXT: v_mov_b32_e32 v0, s6246; GFX9-NEXT: v_mov_b32_e32 v1, s7247; GFX9-NEXT: v_mov_b32_e32 v2, s4248; GFX9-NEXT: v_mov_b32_e32 v3, s5249; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc250; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)251; GFX9-NEXT: buffer_wbinvl1_vol252; GFX9-NEXT: s_setpc_b64 s[30:31]253 %result = atomicrmw xchg ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1254 ret i64 %result255}256 257define amdgpu_gfx i64 @flat_atomic_xchg_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {258; GFX7-LABEL: flat_atomic_xchg_i64_ret_offset_scalar:259; GFX7: ; %bb.0:260; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)261; GFX7-NEXT: s_add_u32 s34, s4, 32262; GFX7-NEXT: s_addc_u32 s35, s5, 0263; GFX7-NEXT: v_mov_b32_e32 v2, s34264; GFX7-NEXT: v_mov_b32_e32 v0, s6265; GFX7-NEXT: v_mov_b32_e32 v1, s7266; GFX7-NEXT: v_mov_b32_e32 v3, s35267; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc268; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)269; GFX7-NEXT: buffer_wbinvl1_vol270; GFX7-NEXT: s_setpc_b64 s[30:31]271;272; GFX8-LABEL: flat_atomic_xchg_i64_ret_offset_scalar:273; GFX8: ; %bb.0:274; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)275; GFX8-NEXT: s_add_u32 s34, s4, 32276; GFX8-NEXT: s_addc_u32 s35, s5, 0277; GFX8-NEXT: v_mov_b32_e32 v2, s34278; GFX8-NEXT: v_mov_b32_e32 v0, s6279; GFX8-NEXT: v_mov_b32_e32 v1, s7280; GFX8-NEXT: v_mov_b32_e32 v3, s35281; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc282; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)283; GFX8-NEXT: buffer_wbinvl1_vol284; GFX8-NEXT: s_setpc_b64 s[30:31]285;286; GFX9-LABEL: flat_atomic_xchg_i64_ret_offset_scalar:287; GFX9: ; %bb.0:288; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)289; GFX9-NEXT: v_mov_b32_e32 v0, s6290; GFX9-NEXT: v_mov_b32_e32 v1, s7291; GFX9-NEXT: v_mov_b32_e32 v2, s4292; GFX9-NEXT: v_mov_b32_e32 v3, s5293; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] offset:32 glc294; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)295; GFX9-NEXT: buffer_wbinvl1_vol296; GFX9-NEXT: s_setpc_b64 s[30:31]297 %gep = getelementptr inbounds i64, ptr %out, i64 4298 %result = atomicrmw xchg ptr %gep, i64 %in seq_cst, !noalias.addrspace !1299 ret i64 %result300}301 302define void @flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {303; GFX7-LABEL: flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory:304; GFX7: ; %bb.0:305; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)306; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0307; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc308; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]309; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)310; GFX7-NEXT: buffer_wbinvl1_vol311; GFX7-NEXT: s_setpc_b64 s[30:31]312;313; GFX8-LABEL: flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory:314; GFX8: ; %bb.0:315; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)316; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0317; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc318; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]319; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)320; GFX8-NEXT: buffer_wbinvl1_vol321; GFX8-NEXT: s_setpc_b64 s[30:31]322;323; GFX9-LABEL: flat_atomic_xchg_i64_noret_offset__amdgpu_no_remote_memory:324; GFX9: ; %bb.0:325; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)326; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] offset:32327; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)328; GFX9-NEXT: buffer_wbinvl1_vol329; GFX9-NEXT: s_setpc_b64 s[30:31]330 %gep = getelementptr inbounds i64, ptr %out, i64 4331 %tmp0 = atomicrmw xchg ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1332 ret void333}334 335define i64 @flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {336; GFX7-LABEL: flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory:337; GFX7: ; %bb.0:338; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)339; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0340; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc341; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc342; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)343; GFX7-NEXT: buffer_wbinvl1_vol344; GFX7-NEXT: s_setpc_b64 s[30:31]345;346; GFX8-LABEL: flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory:347; GFX8: ; %bb.0:348; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)349; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0350; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc351; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc352; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)353; GFX8-NEXT: buffer_wbinvl1_vol354; GFX8-NEXT: s_setpc_b64 s[30:31]355;356; GFX9-LABEL: flat_atomic_xchg_i64_ret_offset__amdgpu_no_remote_memory:357; GFX9: ; %bb.0:358; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)359; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] offset:32 glc360; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)361; GFX9-NEXT: buffer_wbinvl1_vol362; GFX9-NEXT: s_setpc_b64 s[30:31]363 %gep = getelementptr inbounds i64, ptr %out, i64 4364 %result = atomicrmw xchg ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1365 ret i64 %result366}367 368; ---------------------------------------------------------------------369; atomicrmw xchg f64370; ---------------------------------------------------------------------371 372define void @flat_atomic_xchg_f64_noret(ptr %ptr, double %in) {373; GFX7-LABEL: flat_atomic_xchg_f64_noret:374; GFX7: ; %bb.0:375; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)376; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]377; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)378; GFX7-NEXT: buffer_wbinvl1_vol379; GFX7-NEXT: s_setpc_b64 s[30:31]380;381; GFX8-LABEL: flat_atomic_xchg_f64_noret:382; GFX8: ; %bb.0:383; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)384; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]385; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)386; GFX8-NEXT: buffer_wbinvl1_vol387; GFX8-NEXT: s_setpc_b64 s[30:31]388;389; GFX9-LABEL: flat_atomic_xchg_f64_noret:390; GFX9: ; %bb.0:391; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)392; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]393; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)394; GFX9-NEXT: buffer_wbinvl1_vol395; GFX9-NEXT: s_setpc_b64 s[30:31]396 %tmp0 = atomicrmw xchg ptr %ptr, double %in seq_cst, !noalias.addrspace !1397 ret void398}399 400define void @flat_atomic_xchg_f64_noret_offset(ptr %out, double %in) {401; GFX7-LABEL: flat_atomic_xchg_f64_noret_offset:402; GFX7: ; %bb.0:403; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)404; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0405; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc406; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]407; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)408; GFX7-NEXT: buffer_wbinvl1_vol409; GFX7-NEXT: s_setpc_b64 s[30:31]410;411; GFX8-LABEL: flat_atomic_xchg_f64_noret_offset:412; GFX8: ; %bb.0:413; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)414; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0415; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc416; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]417; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)418; GFX8-NEXT: buffer_wbinvl1_vol419; GFX8-NEXT: s_setpc_b64 s[30:31]420;421; GFX9-LABEL: flat_atomic_xchg_f64_noret_offset:422; GFX9: ; %bb.0:423; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)424; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] offset:32425; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)426; GFX9-NEXT: buffer_wbinvl1_vol427; GFX9-NEXT: s_setpc_b64 s[30:31]428 %gep = getelementptr inbounds double, ptr %out, i32 4429 %tmp0 = atomicrmw xchg ptr %gep, double %in seq_cst, !noalias.addrspace !1430 ret void431}432 433define double @flat_atomic_xchg_f64_ret(ptr %ptr, double %in) {434; GFX7-LABEL: flat_atomic_xchg_f64_ret:435; GFX7: ; %bb.0:436; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)437; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc438; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)439; GFX7-NEXT: buffer_wbinvl1_vol440; GFX7-NEXT: s_setpc_b64 s[30:31]441;442; GFX8-LABEL: flat_atomic_xchg_f64_ret:443; GFX8: ; %bb.0:444; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)445; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc446; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)447; GFX8-NEXT: buffer_wbinvl1_vol448; GFX8-NEXT: s_setpc_b64 s[30:31]449;450; GFX9-LABEL: flat_atomic_xchg_f64_ret:451; GFX9: ; %bb.0:452; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)453; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc454; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)455; GFX9-NEXT: buffer_wbinvl1_vol456; GFX9-NEXT: s_setpc_b64 s[30:31]457 %result = atomicrmw xchg ptr %ptr, double %in seq_cst, !noalias.addrspace !1458 ret double %result459}460 461define double @flat_atomic_xchg_f64_ret_offset(ptr %out, double %in) {462; GFX7-LABEL: flat_atomic_xchg_f64_ret_offset:463; GFX7: ; %bb.0:464; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)465; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0466; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc467; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc468; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)469; GFX7-NEXT: buffer_wbinvl1_vol470; GFX7-NEXT: s_setpc_b64 s[30:31]471;472; GFX8-LABEL: flat_atomic_xchg_f64_ret_offset:473; GFX8: ; %bb.0:474; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)475; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0476; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc477; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc478; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)479; GFX8-NEXT: buffer_wbinvl1_vol480; GFX8-NEXT: s_setpc_b64 s[30:31]481;482; GFX9-LABEL: flat_atomic_xchg_f64_ret_offset:483; GFX9: ; %bb.0:484; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)485; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] offset:32 glc486; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)487; GFX9-NEXT: buffer_wbinvl1_vol488; GFX9-NEXT: s_setpc_b64 s[30:31]489 %gep = getelementptr inbounds double, ptr %out, i32 4490 %result = atomicrmw xchg ptr %gep, double %in seq_cst, !noalias.addrspace !1491 ret double %result492}493 494define amdgpu_gfx void @flat_atomic_xchg_f64_noret_scalar(ptr inreg %ptr, double inreg %in) {495; GFX7-LABEL: flat_atomic_xchg_f64_noret_scalar:496; GFX7: ; %bb.0:497; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)498; GFX7-NEXT: v_mov_b32_e32 v0, s6499; GFX7-NEXT: v_mov_b32_e32 v1, s7500; GFX7-NEXT: v_mov_b32_e32 v2, s4501; GFX7-NEXT: v_mov_b32_e32 v3, s5502; GFX7-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]503; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)504; GFX7-NEXT: buffer_wbinvl1_vol505; GFX7-NEXT: s_setpc_b64 s[30:31]506;507; GFX8-LABEL: flat_atomic_xchg_f64_noret_scalar:508; GFX8: ; %bb.0:509; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)510; GFX8-NEXT: v_mov_b32_e32 v0, s6511; GFX8-NEXT: v_mov_b32_e32 v1, s7512; GFX8-NEXT: v_mov_b32_e32 v2, s4513; GFX8-NEXT: v_mov_b32_e32 v3, s5514; GFX8-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]515; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)516; GFX8-NEXT: buffer_wbinvl1_vol517; GFX8-NEXT: s_setpc_b64 s[30:31]518;519; GFX9-LABEL: flat_atomic_xchg_f64_noret_scalar:520; GFX9: ; %bb.0:521; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)522; GFX9-NEXT: v_mov_b32_e32 v0, s6523; GFX9-NEXT: v_mov_b32_e32 v1, s7524; GFX9-NEXT: v_mov_b32_e32 v2, s4525; GFX9-NEXT: v_mov_b32_e32 v3, s5526; GFX9-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]527; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)528; GFX9-NEXT: buffer_wbinvl1_vol529; GFX9-NEXT: s_setpc_b64 s[30:31]530 %tmp0 = atomicrmw xchg ptr %ptr, double %in seq_cst, !noalias.addrspace !1531 ret void532}533 534define amdgpu_gfx void @flat_atomic_xchg_f64_noret_offset_scalar(ptr inreg %out, double inreg %in) {535; GFX7-LABEL: flat_atomic_xchg_f64_noret_offset_scalar:536; GFX7: ; %bb.0:537; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)538; GFX7-NEXT: s_add_u32 s34, s4, 32539; GFX7-NEXT: s_addc_u32 s35, s5, 0540; GFX7-NEXT: v_mov_b32_e32 v2, s34541; GFX7-NEXT: v_mov_b32_e32 v0, s6542; GFX7-NEXT: v_mov_b32_e32 v1, s7543; GFX7-NEXT: v_mov_b32_e32 v3, s35544; GFX7-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]545; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)546; GFX7-NEXT: buffer_wbinvl1_vol547; GFX7-NEXT: s_setpc_b64 s[30:31]548;549; GFX8-LABEL: flat_atomic_xchg_f64_noret_offset_scalar:550; GFX8: ; %bb.0:551; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)552; GFX8-NEXT: s_add_u32 s34, s4, 32553; GFX8-NEXT: s_addc_u32 s35, s5, 0554; GFX8-NEXT: v_mov_b32_e32 v2, s34555; GFX8-NEXT: v_mov_b32_e32 v0, s6556; GFX8-NEXT: v_mov_b32_e32 v1, s7557; GFX8-NEXT: v_mov_b32_e32 v3, s35558; GFX8-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1]559; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)560; GFX8-NEXT: buffer_wbinvl1_vol561; GFX8-NEXT: s_setpc_b64 s[30:31]562;563; GFX9-LABEL: flat_atomic_xchg_f64_noret_offset_scalar:564; GFX9: ; %bb.0:565; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)566; GFX9-NEXT: v_mov_b32_e32 v0, s6567; GFX9-NEXT: v_mov_b32_e32 v1, s7568; GFX9-NEXT: v_mov_b32_e32 v2, s4569; GFX9-NEXT: v_mov_b32_e32 v3, s5570; GFX9-NEXT: flat_atomic_swap_x2 v[2:3], v[0:1] offset:32571; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)572; GFX9-NEXT: buffer_wbinvl1_vol573; GFX9-NEXT: s_setpc_b64 s[30:31]574 %gep = getelementptr inbounds double, ptr %out, i32 4575 %tmp0 = atomicrmw xchg ptr %gep, double %in seq_cst, !noalias.addrspace !1576 ret void577}578 579define amdgpu_gfx double @flat_atomic_xchg_f64_ret_scalar(ptr inreg %ptr, double inreg %in) {580; GFX7-LABEL: flat_atomic_xchg_f64_ret_scalar:581; GFX7: ; %bb.0:582; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)583; GFX7-NEXT: v_mov_b32_e32 v0, s6584; GFX7-NEXT: v_mov_b32_e32 v1, s7585; GFX7-NEXT: v_mov_b32_e32 v2, s4586; GFX7-NEXT: v_mov_b32_e32 v3, s5587; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc588; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)589; GFX7-NEXT: buffer_wbinvl1_vol590; GFX7-NEXT: s_setpc_b64 s[30:31]591;592; GFX8-LABEL: flat_atomic_xchg_f64_ret_scalar:593; GFX8: ; %bb.0:594; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)595; GFX8-NEXT: v_mov_b32_e32 v0, s6596; GFX8-NEXT: v_mov_b32_e32 v1, s7597; GFX8-NEXT: v_mov_b32_e32 v2, s4598; GFX8-NEXT: v_mov_b32_e32 v3, s5599; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc600; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)601; GFX8-NEXT: buffer_wbinvl1_vol602; GFX8-NEXT: s_setpc_b64 s[30:31]603;604; GFX9-LABEL: flat_atomic_xchg_f64_ret_scalar:605; GFX9: ; %bb.0:606; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)607; GFX9-NEXT: v_mov_b32_e32 v0, s6608; GFX9-NEXT: v_mov_b32_e32 v1, s7609; GFX9-NEXT: v_mov_b32_e32 v2, s4610; GFX9-NEXT: v_mov_b32_e32 v3, s5611; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc612; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)613; GFX9-NEXT: buffer_wbinvl1_vol614; GFX9-NEXT: s_setpc_b64 s[30:31]615 %result = atomicrmw xchg ptr %ptr, double %in seq_cst, !noalias.addrspace !1616 ret double %result617}618 619define amdgpu_gfx double @flat_atomic_xchg_f64_ret_offset_scalar(ptr inreg %out, double inreg %in) {620; GFX7-LABEL: flat_atomic_xchg_f64_ret_offset_scalar:621; GFX7: ; %bb.0:622; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)623; GFX7-NEXT: s_add_u32 s34, s4, 32624; GFX7-NEXT: s_addc_u32 s35, s5, 0625; GFX7-NEXT: v_mov_b32_e32 v2, s34626; GFX7-NEXT: v_mov_b32_e32 v0, s6627; GFX7-NEXT: v_mov_b32_e32 v1, s7628; GFX7-NEXT: v_mov_b32_e32 v3, s35629; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc630; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)631; GFX7-NEXT: buffer_wbinvl1_vol632; GFX7-NEXT: s_setpc_b64 s[30:31]633;634; GFX8-LABEL: flat_atomic_xchg_f64_ret_offset_scalar:635; GFX8: ; %bb.0:636; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)637; GFX8-NEXT: s_add_u32 s34, s4, 32638; GFX8-NEXT: s_addc_u32 s35, s5, 0639; GFX8-NEXT: v_mov_b32_e32 v2, s34640; GFX8-NEXT: v_mov_b32_e32 v0, s6641; GFX8-NEXT: v_mov_b32_e32 v1, s7642; GFX8-NEXT: v_mov_b32_e32 v3, s35643; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] glc644; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)645; GFX8-NEXT: buffer_wbinvl1_vol646; GFX8-NEXT: s_setpc_b64 s[30:31]647;648; GFX9-LABEL: flat_atomic_xchg_f64_ret_offset_scalar:649; GFX9: ; %bb.0:650; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)651; GFX9-NEXT: v_mov_b32_e32 v0, s6652; GFX9-NEXT: v_mov_b32_e32 v1, s7653; GFX9-NEXT: v_mov_b32_e32 v2, s4654; GFX9-NEXT: v_mov_b32_e32 v3, s5655; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[0:1] offset:32 glc656; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)657; GFX9-NEXT: buffer_wbinvl1_vol658; GFX9-NEXT: s_setpc_b64 s[30:31]659 %gep = getelementptr inbounds double, ptr %out, i32 4660 %result = atomicrmw xchg ptr %gep, double %in seq_cst, !noalias.addrspace !1661 ret double %result662}663 664define void @flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory(ptr %out, double %in) {665; GFX7-LABEL: flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory:666; GFX7: ; %bb.0:667; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)668; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0669; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc670; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]671; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)672; GFX7-NEXT: buffer_wbinvl1_vol673; GFX7-NEXT: s_setpc_b64 s[30:31]674;675; GFX8-LABEL: flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory:676; GFX8: ; %bb.0:677; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)678; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0679; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc680; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]681; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)682; GFX8-NEXT: buffer_wbinvl1_vol683; GFX8-NEXT: s_setpc_b64 s[30:31]684;685; GFX9-LABEL: flat_atomic_xchg_f64_noret_offset__amdgpu_no_remote_memory:686; GFX9: ; %bb.0:687; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)688; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3] offset:32689; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)690; GFX9-NEXT: buffer_wbinvl1_vol691; GFX9-NEXT: s_setpc_b64 s[30:31]692 %gep = getelementptr inbounds double, ptr %out, i64 4693 %tmp0 = atomicrmw xchg ptr %gep, double %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1694 ret void695}696 697define double @flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory(ptr %out, double %in) {698; GFX7-LABEL: flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory:699; GFX7: ; %bb.0:700; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)701; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0702; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc703; GFX7-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc704; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)705; GFX7-NEXT: buffer_wbinvl1_vol706; GFX7-NEXT: s_setpc_b64 s[30:31]707;708; GFX8-LABEL: flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory:709; GFX8: ; %bb.0:710; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)711; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0712; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc713; GFX8-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] glc714; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)715; GFX8-NEXT: buffer_wbinvl1_vol716; GFX8-NEXT: s_setpc_b64 s[30:31]717;718; GFX9-LABEL: flat_atomic_xchg_f64_ret_offset__amdgpu_no_remote_memory:719; GFX9: ; %bb.0:720; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)721; GFX9-NEXT: flat_atomic_swap_x2 v[0:1], v[0:1], v[2:3] offset:32 glc722; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)723; GFX9-NEXT: buffer_wbinvl1_vol724; GFX9-NEXT: s_setpc_b64 s[30:31]725 %gep = getelementptr inbounds double, ptr %out, i64 4726 %result = atomicrmw xchg ptr %gep, double %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1727 ret double %result728}729 730; ---------------------------------------------------------------------731; atomicrmw add732; ---------------------------------------------------------------------733 734define void @flat_atomic_add_i64_noret(ptr %ptr, i64 %in) {735; GFX7-LABEL: flat_atomic_add_i64_noret:736; GFX7: ; %bb.0:737; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)738; GFX7-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]739; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)740; GFX7-NEXT: buffer_wbinvl1_vol741; GFX7-NEXT: s_setpc_b64 s[30:31]742;743; GFX8-LABEL: flat_atomic_add_i64_noret:744; GFX8: ; %bb.0:745; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)746; GFX8-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]747; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)748; GFX8-NEXT: buffer_wbinvl1_vol749; GFX8-NEXT: s_setpc_b64 s[30:31]750;751; GFX9-LABEL: flat_atomic_add_i64_noret:752; GFX9: ; %bb.0:753; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)754; GFX9-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]755; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)756; GFX9-NEXT: buffer_wbinvl1_vol757; GFX9-NEXT: s_setpc_b64 s[30:31]758 %tmp0 = atomicrmw add ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1759 ret void760}761 762define void @flat_atomic_add_i64_noret_offset(ptr %out, i64 %in) {763; GFX7-LABEL: flat_atomic_add_i64_noret_offset:764; GFX7: ; %bb.0:765; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)766; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0767; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc768; GFX7-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]769; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)770; GFX7-NEXT: buffer_wbinvl1_vol771; GFX7-NEXT: s_setpc_b64 s[30:31]772;773; GFX8-LABEL: flat_atomic_add_i64_noret_offset:774; GFX8: ; %bb.0:775; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)776; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0777; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc778; GFX8-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]779; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)780; GFX8-NEXT: buffer_wbinvl1_vol781; GFX8-NEXT: s_setpc_b64 s[30:31]782;783; GFX9-LABEL: flat_atomic_add_i64_noret_offset:784; GFX9: ; %bb.0:785; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)786; GFX9-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] offset:32787; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)788; GFX9-NEXT: buffer_wbinvl1_vol789; GFX9-NEXT: s_setpc_b64 s[30:31]790 %gep = getelementptr inbounds i64, ptr %out, i64 4791 %tmp0 = atomicrmw add ptr %gep, i64 %in seq_cst, !noalias.addrspace !1792 ret void793}794 795define i64 @flat_atomic_add_i64_ret(ptr %ptr, i64 %in) {796; GFX7-LABEL: flat_atomic_add_i64_ret:797; GFX7: ; %bb.0:798; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)799; GFX7-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc800; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)801; GFX7-NEXT: buffer_wbinvl1_vol802; GFX7-NEXT: s_setpc_b64 s[30:31]803;804; GFX8-LABEL: flat_atomic_add_i64_ret:805; GFX8: ; %bb.0:806; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)807; GFX8-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc808; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)809; GFX8-NEXT: buffer_wbinvl1_vol810; GFX8-NEXT: s_setpc_b64 s[30:31]811;812; GFX9-LABEL: flat_atomic_add_i64_ret:813; GFX9: ; %bb.0:814; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)815; GFX9-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc816; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)817; GFX9-NEXT: buffer_wbinvl1_vol818; GFX9-NEXT: s_setpc_b64 s[30:31]819 %result = atomicrmw add ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1820 ret i64 %result821}822 823define i64 @flat_atomic_add_i64_ret_offset(ptr %out, i64 %in) {824; GFX7-LABEL: flat_atomic_add_i64_ret_offset:825; GFX7: ; %bb.0:826; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)827; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v0828; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc829; GFX7-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc830; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)831; GFX7-NEXT: buffer_wbinvl1_vol832; GFX7-NEXT: s_setpc_b64 s[30:31]833;834; GFX8-LABEL: flat_atomic_add_i64_ret_offset:835; GFX8: ; %bb.0:836; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)837; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v0838; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc839; GFX8-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc840; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)841; GFX8-NEXT: buffer_wbinvl1_vol842; GFX8-NEXT: s_setpc_b64 s[30:31]843;844; GFX9-LABEL: flat_atomic_add_i64_ret_offset:845; GFX9: ; %bb.0:846; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)847; GFX9-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] offset:32 glc848; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)849; GFX9-NEXT: buffer_wbinvl1_vol850; GFX9-NEXT: s_setpc_b64 s[30:31]851 %gep = getelementptr inbounds i64, ptr %out, i64 4852 %result = atomicrmw add ptr %gep, i64 %in seq_cst, !noalias.addrspace !1853 ret i64 %result854}855 856define amdgpu_gfx void @flat_atomic_add_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {857; GFX7-LABEL: flat_atomic_add_i64_noret_scalar:858; GFX7: ; %bb.0:859; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)860; GFX7-NEXT: v_mov_b32_e32 v0, s6861; GFX7-NEXT: v_mov_b32_e32 v1, s7862; GFX7-NEXT: v_mov_b32_e32 v2, s4863; GFX7-NEXT: v_mov_b32_e32 v3, s5864; GFX7-NEXT: flat_atomic_add_x2 v[2:3], v[0:1]865; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)866; GFX7-NEXT: buffer_wbinvl1_vol867; GFX7-NEXT: s_setpc_b64 s[30:31]868;869; GFX8-LABEL: flat_atomic_add_i64_noret_scalar:870; GFX8: ; %bb.0:871; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)872; GFX8-NEXT: v_mov_b32_e32 v0, s6873; GFX8-NEXT: v_mov_b32_e32 v1, s7874; GFX8-NEXT: v_mov_b32_e32 v2, s4875; GFX8-NEXT: v_mov_b32_e32 v3, s5876; GFX8-NEXT: flat_atomic_add_x2 v[2:3], v[0:1]877; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)878; GFX8-NEXT: buffer_wbinvl1_vol879; GFX8-NEXT: s_setpc_b64 s[30:31]880;881; GFX9-LABEL: flat_atomic_add_i64_noret_scalar:882; GFX9: ; %bb.0:883; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)884; GFX9-NEXT: v_mov_b32_e32 v0, s6885; GFX9-NEXT: v_mov_b32_e32 v1, s7886; GFX9-NEXT: v_mov_b32_e32 v2, s4887; GFX9-NEXT: v_mov_b32_e32 v3, s5888; GFX9-NEXT: flat_atomic_add_x2 v[2:3], v[0:1]889; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)890; GFX9-NEXT: buffer_wbinvl1_vol891; GFX9-NEXT: s_setpc_b64 s[30:31]892 %tmp0 = atomicrmw add ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1893 ret void894}895 896define amdgpu_gfx void @flat_atomic_add_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {897; GFX7-LABEL: flat_atomic_add_i64_noret_offset_scalar:898; GFX7: ; %bb.0:899; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)900; GFX7-NEXT: s_add_u32 s34, s4, 32901; GFX7-NEXT: s_addc_u32 s35, s5, 0902; GFX7-NEXT: v_mov_b32_e32 v2, s34903; GFX7-NEXT: v_mov_b32_e32 v0, s6904; GFX7-NEXT: v_mov_b32_e32 v1, s7905; GFX7-NEXT: v_mov_b32_e32 v3, s35906; GFX7-NEXT: flat_atomic_add_x2 v[2:3], v[0:1]907; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)908; GFX7-NEXT: buffer_wbinvl1_vol909; GFX7-NEXT: s_setpc_b64 s[30:31]910;911; GFX8-LABEL: flat_atomic_add_i64_noret_offset_scalar:912; GFX8: ; %bb.0:913; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)914; GFX8-NEXT: s_add_u32 s34, s4, 32915; GFX8-NEXT: s_addc_u32 s35, s5, 0916; GFX8-NEXT: v_mov_b32_e32 v2, s34917; GFX8-NEXT: v_mov_b32_e32 v0, s6918; GFX8-NEXT: v_mov_b32_e32 v1, s7919; GFX8-NEXT: v_mov_b32_e32 v3, s35920; GFX8-NEXT: flat_atomic_add_x2 v[2:3], v[0:1]921; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)922; GFX8-NEXT: buffer_wbinvl1_vol923; GFX8-NEXT: s_setpc_b64 s[30:31]924;925; GFX9-LABEL: flat_atomic_add_i64_noret_offset_scalar:926; GFX9: ; %bb.0:927; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)928; GFX9-NEXT: v_mov_b32_e32 v0, s6929; GFX9-NEXT: v_mov_b32_e32 v1, s7930; GFX9-NEXT: v_mov_b32_e32 v2, s4931; GFX9-NEXT: v_mov_b32_e32 v3, s5932; GFX9-NEXT: flat_atomic_add_x2 v[2:3], v[0:1] offset:32933; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)934; GFX9-NEXT: buffer_wbinvl1_vol935; GFX9-NEXT: s_setpc_b64 s[30:31]936 %gep = getelementptr inbounds i64, ptr %out, i64 4937 %tmp0 = atomicrmw add ptr %gep, i64 %in seq_cst, !noalias.addrspace !1938 ret void939}940 941define amdgpu_gfx i64 @flat_atomic_add_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {942; GFX7-LABEL: flat_atomic_add_i64_ret_scalar:943; GFX7: ; %bb.0:944; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)945; GFX7-NEXT: v_mov_b32_e32 v0, s6946; GFX7-NEXT: v_mov_b32_e32 v1, s7947; GFX7-NEXT: v_mov_b32_e32 v2, s4948; GFX7-NEXT: v_mov_b32_e32 v3, s5949; GFX7-NEXT: flat_atomic_add_x2 v[0:1], v[2:3], v[0:1] glc950; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)951; GFX7-NEXT: buffer_wbinvl1_vol952; GFX7-NEXT: s_setpc_b64 s[30:31]953;954; GFX8-LABEL: flat_atomic_add_i64_ret_scalar:955; GFX8: ; %bb.0:956; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)957; GFX8-NEXT: v_mov_b32_e32 v0, s6958; GFX8-NEXT: v_mov_b32_e32 v1, s7959; GFX8-NEXT: v_mov_b32_e32 v2, s4960; GFX8-NEXT: v_mov_b32_e32 v3, s5961; GFX8-NEXT: flat_atomic_add_x2 v[0:1], v[2:3], v[0:1] glc962; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)963; GFX8-NEXT: buffer_wbinvl1_vol964; GFX8-NEXT: s_setpc_b64 s[30:31]965;966; GFX9-LABEL: flat_atomic_add_i64_ret_scalar:967; GFX9: ; %bb.0:968; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)969; GFX9-NEXT: v_mov_b32_e32 v0, s6970; GFX9-NEXT: v_mov_b32_e32 v1, s7971; GFX9-NEXT: v_mov_b32_e32 v2, s4972; GFX9-NEXT: v_mov_b32_e32 v3, s5973; GFX9-NEXT: flat_atomic_add_x2 v[0:1], v[2:3], v[0:1] glc974; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)975; GFX9-NEXT: buffer_wbinvl1_vol976; GFX9-NEXT: s_setpc_b64 s[30:31]977 %result = atomicrmw add ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1978 ret i64 %result979}980 981define amdgpu_gfx i64 @flat_atomic_add_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {982; GFX7-LABEL: flat_atomic_add_i64_ret_offset_scalar:983; GFX7: ; %bb.0:984; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)985; GFX7-NEXT: s_add_u32 s34, s4, 32986; GFX7-NEXT: s_addc_u32 s35, s5, 0987; GFX7-NEXT: v_mov_b32_e32 v2, s34988; GFX7-NEXT: v_mov_b32_e32 v0, s6989; GFX7-NEXT: v_mov_b32_e32 v1, s7990; GFX7-NEXT: v_mov_b32_e32 v3, s35991; GFX7-NEXT: flat_atomic_add_x2 v[0:1], v[2:3], v[0:1] glc992; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)993; GFX7-NEXT: buffer_wbinvl1_vol994; GFX7-NEXT: s_setpc_b64 s[30:31]995;996; GFX8-LABEL: flat_atomic_add_i64_ret_offset_scalar:997; GFX8: ; %bb.0:998; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)999; GFX8-NEXT: s_add_u32 s34, s4, 321000; GFX8-NEXT: s_addc_u32 s35, s5, 01001; GFX8-NEXT: v_mov_b32_e32 v2, s341002; GFX8-NEXT: v_mov_b32_e32 v0, s61003; GFX8-NEXT: v_mov_b32_e32 v1, s71004; GFX8-NEXT: v_mov_b32_e32 v3, s351005; GFX8-NEXT: flat_atomic_add_x2 v[0:1], v[2:3], v[0:1] glc1006; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1007; GFX8-NEXT: buffer_wbinvl1_vol1008; GFX8-NEXT: s_setpc_b64 s[30:31]1009;1010; GFX9-LABEL: flat_atomic_add_i64_ret_offset_scalar:1011; GFX9: ; %bb.0:1012; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1013; GFX9-NEXT: v_mov_b32_e32 v0, s61014; GFX9-NEXT: v_mov_b32_e32 v1, s71015; GFX9-NEXT: v_mov_b32_e32 v2, s41016; GFX9-NEXT: v_mov_b32_e32 v3, s51017; GFX9-NEXT: flat_atomic_add_x2 v[0:1], v[2:3], v[0:1] offset:32 glc1018; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1019; GFX9-NEXT: buffer_wbinvl1_vol1020; GFX9-NEXT: s_setpc_b64 s[30:31]1021 %gep = getelementptr inbounds i64, ptr %out, i64 41022 %result = atomicrmw add ptr %gep, i64 %in seq_cst, !noalias.addrspace !11023 ret i64 %result1024}1025 1026define void @flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {1027; GFX7-LABEL: flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory:1028; GFX7: ; %bb.0:1029; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1030; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v01031; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1032; GFX7-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]1033; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1034; GFX7-NEXT: buffer_wbinvl1_vol1035; GFX7-NEXT: s_setpc_b64 s[30:31]1036;1037; GFX8-LABEL: flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory:1038; GFX8: ; %bb.0:1039; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1040; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v01041; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1042; GFX8-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]1043; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1044; GFX8-NEXT: buffer_wbinvl1_vol1045; GFX8-NEXT: s_setpc_b64 s[30:31]1046;1047; GFX9-LABEL: flat_atomic_add_i64_noret_offset__amdgpu_no_remote_memory:1048; GFX9: ; %bb.0:1049; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1050; GFX9-NEXT: flat_atomic_add_x2 v[0:1], v[2:3] offset:321051; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1052; GFX9-NEXT: buffer_wbinvl1_vol1053; GFX9-NEXT: s_setpc_b64 s[30:31]1054 %gep = getelementptr inbounds i64, ptr %out, i64 41055 %tmp0 = atomicrmw add ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !11056 ret void1057}1058 1059define i64 @flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {1060; GFX7-LABEL: flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory:1061; GFX7: ; %bb.0:1062; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1063; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v01064; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1065; GFX7-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc1066; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1067; GFX7-NEXT: buffer_wbinvl1_vol1068; GFX7-NEXT: s_setpc_b64 s[30:31]1069;1070; GFX8-LABEL: flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory:1071; GFX8: ; %bb.0:1072; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1073; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v01074; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1075; GFX8-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] glc1076; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1077; GFX8-NEXT: buffer_wbinvl1_vol1078; GFX8-NEXT: s_setpc_b64 s[30:31]1079;1080; GFX9-LABEL: flat_atomic_add_i64_ret_offset__amdgpu_no_remote_memory:1081; GFX9: ; %bb.0:1082; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1083; GFX9-NEXT: flat_atomic_add_x2 v[0:1], v[0:1], v[2:3] offset:32 glc1084; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1085; GFX9-NEXT: buffer_wbinvl1_vol1086; GFX9-NEXT: s_setpc_b64 s[30:31]1087 %gep = getelementptr inbounds i64, ptr %out, i64 41088 %result = atomicrmw add ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !11089 ret i64 %result1090}1091 1092; ---------------------------------------------------------------------1093; atomicrmw sub1094; ---------------------------------------------------------------------1095 1096define void @flat_atomic_sub_i64_noret(ptr %ptr, i64 %in) {1097; GFX7-LABEL: flat_atomic_sub_i64_noret:1098; GFX7: ; %bb.0:1099; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1100; GFX7-NEXT: v_add_i32_e32 v4, vcc, 4, v01101; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc1102; GFX7-NEXT: flat_load_dword v6, v[0:1]1103; GFX7-NEXT: flat_load_dword v7, v[4:5]1104; GFX7-NEXT: s_mov_b64 s[4:5], 01105; GFX7-NEXT: .LBB30_1: ; %atomicrmw.start1106; GFX7-NEXT: ; =>This Inner Loop Header: Depth=11107; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1108; GFX7-NEXT: v_sub_i32_e32 v4, vcc, v6, v21109; GFX7-NEXT: v_subb_u32_e32 v5, vcc, v7, v3, vcc1110; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc1111; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1112; GFX7-NEXT: buffer_wbinvl1_vol1113; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]1114; GFX7-NEXT: v_mov_b32_e32 v7, v51115; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1116; GFX7-NEXT: v_mov_b32_e32 v6, v41117; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]1118; GFX7-NEXT: s_cbranch_execnz .LBB30_11119; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end1120; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]1121; GFX7-NEXT: s_setpc_b64 s[30:31]1122;1123; GFX8-LABEL: flat_atomic_sub_i64_noret:1124; GFX8: ; %bb.0:1125; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1126; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v01127; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc1128; GFX8-NEXT: flat_load_dword v6, v[0:1]1129; GFX8-NEXT: flat_load_dword v7, v[4:5]1130; GFX8-NEXT: s_mov_b64 s[4:5], 01131; GFX8-NEXT: .LBB30_1: ; %atomicrmw.start1132; GFX8-NEXT: ; =>This Inner Loop Header: Depth=11133; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1134; GFX8-NEXT: v_sub_u32_e32 v4, vcc, v6, v21135; GFX8-NEXT: v_subb_u32_e32 v5, vcc, v7, v3, vcc1136; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc1137; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1138; GFX8-NEXT: buffer_wbinvl1_vol1139; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]1140; GFX8-NEXT: v_mov_b32_e32 v7, v51141; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1142; GFX8-NEXT: v_mov_b32_e32 v6, v41143; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]1144; GFX8-NEXT: s_cbranch_execnz .LBB30_11145; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end1146; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]1147; GFX8-NEXT: s_setpc_b64 s[30:31]1148;1149; GFX9-LABEL: flat_atomic_sub_i64_noret:1150; GFX9: ; %bb.0:1151; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1152; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1]1153; GFX9-NEXT: s_mov_b64 s[4:5], 01154; GFX9-NEXT: .LBB30_1: ; %atomicrmw.start1155; GFX9-NEXT: ; =>This Inner Loop Header: Depth=11156; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1157; GFX9-NEXT: v_sub_co_u32_e32 v4, vcc, v6, v21158; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v7, v3, vcc1159; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc1160; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1161; GFX9-NEXT: buffer_wbinvl1_vol1162; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]1163; GFX9-NEXT: v_mov_b32_e32 v7, v51164; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1165; GFX9-NEXT: v_mov_b32_e32 v6, v41166; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]1167; GFX9-NEXT: s_cbranch_execnz .LBB30_11168; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end1169; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]1170; GFX9-NEXT: s_setpc_b64 s[30:31]1171 %tmp0 = atomicrmw sub ptr %ptr, i64 %in seq_cst, !noalias.addrspace !11172 ret void1173}1174 1175define void @flat_atomic_sub_i64_noret_offset(ptr %out, i64 %in) {1176; GFX7-LABEL: flat_atomic_sub_i64_noret_offset:1177; GFX7: ; %bb.0:1178; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1179; GFX7-NEXT: v_add_i32_e32 v8, vcc, 32, v01180; GFX7-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc1181; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v01182; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1183; GFX7-NEXT: flat_load_dword v7, v[0:1]1184; GFX7-NEXT: flat_load_dword v6, v[8:9]1185; GFX7-NEXT: s_mov_b64 s[4:5], 01186; GFX7-NEXT: .LBB31_1: ; %atomicrmw.start1187; GFX7-NEXT: ; =>This Inner Loop Header: Depth=11188; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1189; GFX7-NEXT: v_sub_i32_e32 v4, vcc, v6, v21190; GFX7-NEXT: v_subb_u32_e32 v5, vcc, v7, v3, vcc1191; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc1192; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1193; GFX7-NEXT: buffer_wbinvl1_vol1194; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]1195; GFX7-NEXT: v_mov_b32_e32 v7, v11196; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1197; GFX7-NEXT: v_mov_b32_e32 v6, v01198; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]1199; GFX7-NEXT: s_cbranch_execnz .LBB31_11200; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end1201; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]1202; GFX7-NEXT: s_setpc_b64 s[30:31]1203;1204; GFX8-LABEL: flat_atomic_sub_i64_noret_offset:1205; GFX8: ; %bb.0:1206; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1207; GFX8-NEXT: v_add_u32_e32 v8, vcc, 32, v01208; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc1209; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v01210; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1211; GFX8-NEXT: flat_load_dword v7, v[0:1]1212; GFX8-NEXT: flat_load_dword v6, v[8:9]1213; GFX8-NEXT: s_mov_b64 s[4:5], 01214; GFX8-NEXT: .LBB31_1: ; %atomicrmw.start1215; GFX8-NEXT: ; =>This Inner Loop Header: Depth=11216; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1217; GFX8-NEXT: v_sub_u32_e32 v4, vcc, v6, v21218; GFX8-NEXT: v_subb_u32_e32 v5, vcc, v7, v3, vcc1219; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc1220; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1221; GFX8-NEXT: buffer_wbinvl1_vol1222; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]1223; GFX8-NEXT: v_mov_b32_e32 v7, v11224; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1225; GFX8-NEXT: v_mov_b32_e32 v6, v01226; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]1227; GFX8-NEXT: s_cbranch_execnz .LBB31_11228; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end1229; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]1230; GFX8-NEXT: s_setpc_b64 s[30:31]1231;1232; GFX9-LABEL: flat_atomic_sub_i64_noret_offset:1233; GFX9: ; %bb.0:1234; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1235; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:321236; GFX9-NEXT: s_mov_b64 s[4:5], 01237; GFX9-NEXT: .LBB31_1: ; %atomicrmw.start1238; GFX9-NEXT: ; =>This Inner Loop Header: Depth=11239; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1240; GFX9-NEXT: v_sub_co_u32_e32 v4, vcc, v6, v21241; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v7, v3, vcc1242; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc1243; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1244; GFX9-NEXT: buffer_wbinvl1_vol1245; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]1246; GFX9-NEXT: v_mov_b32_e32 v7, v51247; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1248; GFX9-NEXT: v_mov_b32_e32 v6, v41249; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]1250; GFX9-NEXT: s_cbranch_execnz .LBB31_11251; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end1252; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]1253; GFX9-NEXT: s_setpc_b64 s[30:31]1254 %gep = getelementptr inbounds i64, ptr %out, i64 41255 %tmp0 = atomicrmw sub ptr %gep, i64 %in seq_cst, !noalias.addrspace !11256 ret void1257}1258 1259define i64 @flat_atomic_sub_i64_ret(ptr %ptr, i64 %in) {1260; GFX7-LABEL: flat_atomic_sub_i64_ret:1261; GFX7: ; %bb.0:1262; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1263; GFX7-NEXT: v_add_i32_e32 v5, vcc, 4, v01264; GFX7-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc1265; GFX7-NEXT: flat_load_dword v4, v[0:1]1266; GFX7-NEXT: flat_load_dword v5, v[5:6]1267; GFX7-NEXT: s_mov_b64 s[4:5], 01268; GFX7-NEXT: .LBB32_1: ; %atomicrmw.start1269; GFX7-NEXT: ; =>This Inner Loop Header: Depth=11270; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1271; GFX7-NEXT: v_mov_b32_e32 v7, v51272; GFX7-NEXT: v_mov_b32_e32 v6, v41273; GFX7-NEXT: v_sub_i32_e32 v4, vcc, v6, v21274; GFX7-NEXT: v_subb_u32_e32 v5, vcc, v7, v3, vcc1275; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc1276; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1277; GFX7-NEXT: buffer_wbinvl1_vol1278; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]1279; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1280; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]1281; GFX7-NEXT: s_cbranch_execnz .LBB32_11282; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end1283; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]1284; GFX7-NEXT: v_mov_b32_e32 v0, v41285; GFX7-NEXT: v_mov_b32_e32 v1, v51286; GFX7-NEXT: s_setpc_b64 s[30:31]1287;1288; GFX8-LABEL: flat_atomic_sub_i64_ret:1289; GFX8: ; %bb.0:1290; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1291; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v01292; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc1293; GFX8-NEXT: flat_load_dword v4, v[0:1]1294; GFX8-NEXT: flat_load_dword v5, v[5:6]1295; GFX8-NEXT: s_mov_b64 s[4:5], 01296; GFX8-NEXT: .LBB32_1: ; %atomicrmw.start1297; GFX8-NEXT: ; =>This Inner Loop Header: Depth=11298; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1299; GFX8-NEXT: v_mov_b32_e32 v7, v51300; GFX8-NEXT: v_mov_b32_e32 v6, v41301; GFX8-NEXT: v_sub_u32_e32 v4, vcc, v6, v21302; GFX8-NEXT: v_subb_u32_e32 v5, vcc, v7, v3, vcc1303; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc1304; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1305; GFX8-NEXT: buffer_wbinvl1_vol1306; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]1307; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1308; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]1309; GFX8-NEXT: s_cbranch_execnz .LBB32_11310; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end1311; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]1312; GFX8-NEXT: v_mov_b32_e32 v0, v41313; GFX8-NEXT: v_mov_b32_e32 v1, v51314; GFX8-NEXT: s_setpc_b64 s[30:31]1315;1316; GFX9-LABEL: flat_atomic_sub_i64_ret:1317; GFX9: ; %bb.0:1318; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1319; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1]1320; GFX9-NEXT: s_mov_b64 s[4:5], 01321; GFX9-NEXT: .LBB32_1: ; %atomicrmw.start1322; GFX9-NEXT: ; =>This Inner Loop Header: Depth=11323; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1324; GFX9-NEXT: v_mov_b32_e32 v7, v51325; GFX9-NEXT: v_mov_b32_e32 v6, v41326; GFX9-NEXT: v_sub_co_u32_e32 v4, vcc, v6, v21327; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v7, v3, vcc1328; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc1329; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1330; GFX9-NEXT: buffer_wbinvl1_vol1331; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]1332; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1333; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]1334; GFX9-NEXT: s_cbranch_execnz .LBB32_11335; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end1336; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]1337; GFX9-NEXT: v_mov_b32_e32 v0, v41338; GFX9-NEXT: v_mov_b32_e32 v1, v51339; GFX9-NEXT: s_setpc_b64 s[30:31]1340 %result = atomicrmw sub ptr %ptr, i64 %in seq_cst, !noalias.addrspace !11341 ret i64 %result1342}1343 1344define i64 @flat_atomic_sub_i64_ret_offset(ptr %out, i64 %in) {1345; GFX7-LABEL: flat_atomic_sub_i64_ret_offset:1346; GFX7: ; %bb.0:1347; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1348; GFX7-NEXT: v_add_i32_e32 v4, vcc, 32, v01349; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc1350; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v01351; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1352; GFX7-NEXT: flat_load_dword v1, v[0:1]1353; GFX7-NEXT: flat_load_dword v0, v[4:5]1354; GFX7-NEXT: s_mov_b64 s[4:5], 01355; GFX7-NEXT: .LBB33_1: ; %atomicrmw.start1356; GFX7-NEXT: ; =>This Inner Loop Header: Depth=11357; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1358; GFX7-NEXT: v_mov_b32_e32 v9, v11359; GFX7-NEXT: v_mov_b32_e32 v8, v01360; GFX7-NEXT: v_sub_i32_e32 v6, vcc, v8, v21361; GFX7-NEXT: v_subb_u32_e32 v7, vcc, v9, v3, vcc1362; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc1363; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1364; GFX7-NEXT: buffer_wbinvl1_vol1365; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]1366; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1367; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]1368; GFX7-NEXT: s_cbranch_execnz .LBB33_11369; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end1370; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]1371; GFX7-NEXT: s_setpc_b64 s[30:31]1372;1373; GFX8-LABEL: flat_atomic_sub_i64_ret_offset:1374; GFX8: ; %bb.0:1375; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1376; GFX8-NEXT: v_add_u32_e32 v4, vcc, 32, v01377; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc1378; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v01379; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1380; GFX8-NEXT: flat_load_dword v1, v[0:1]1381; GFX8-NEXT: flat_load_dword v0, v[4:5]1382; GFX8-NEXT: s_mov_b64 s[4:5], 01383; GFX8-NEXT: .LBB33_1: ; %atomicrmw.start1384; GFX8-NEXT: ; =>This Inner Loop Header: Depth=11385; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1386; GFX8-NEXT: v_mov_b32_e32 v9, v11387; GFX8-NEXT: v_mov_b32_e32 v8, v01388; GFX8-NEXT: v_sub_u32_e32 v6, vcc, v8, v21389; GFX8-NEXT: v_subb_u32_e32 v7, vcc, v9, v3, vcc1390; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc1391; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1392; GFX8-NEXT: buffer_wbinvl1_vol1393; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]1394; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1395; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]1396; GFX8-NEXT: s_cbranch_execnz .LBB33_11397; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end1398; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]1399; GFX8-NEXT: s_setpc_b64 s[30:31]1400;1401; GFX9-LABEL: flat_atomic_sub_i64_ret_offset:1402; GFX9: ; %bb.0:1403; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1404; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:321405; GFX9-NEXT: s_mov_b64 s[4:5], 01406; GFX9-NEXT: .LBB33_1: ; %atomicrmw.start1407; GFX9-NEXT: ; =>This Inner Loop Header: Depth=11408; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1409; GFX9-NEXT: v_mov_b32_e32 v7, v51410; GFX9-NEXT: v_mov_b32_e32 v6, v41411; GFX9-NEXT: v_sub_co_u32_e32 v4, vcc, v6, v21412; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v7, v3, vcc1413; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc1414; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1415; GFX9-NEXT: buffer_wbinvl1_vol1416; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]1417; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1418; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]1419; GFX9-NEXT: s_cbranch_execnz .LBB33_11420; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end1421; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]1422; GFX9-NEXT: v_mov_b32_e32 v0, v41423; GFX9-NEXT: v_mov_b32_e32 v1, v51424; GFX9-NEXT: s_setpc_b64 s[30:31]1425 %gep = getelementptr inbounds i64, ptr %out, i64 41426 %result = atomicrmw sub ptr %gep, i64 %in seq_cst, !noalias.addrspace !11427 ret i64 %result1428}1429 1430define amdgpu_gfx void @flat_atomic_sub_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {1431; GFX7-LABEL: flat_atomic_sub_i64_noret_scalar:1432; GFX7: ; %bb.0:1433; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1434; GFX7-NEXT: v_mov_b32_e32 v0, s41435; GFX7-NEXT: s_add_u32 s34, s4, 41436; GFX7-NEXT: v_mov_b32_e32 v1, s51437; GFX7-NEXT: s_addc_u32 s35, s5, 01438; GFX7-NEXT: v_mov_b32_e32 v3, s341439; GFX7-NEXT: v_mov_b32_e32 v4, s351440; GFX7-NEXT: flat_load_dword v2, v[0:1]1441; GFX7-NEXT: flat_load_dword v3, v[3:4]1442; GFX7-NEXT: v_mov_b32_e32 v4, s41443; GFX7-NEXT: s_mov_b64 s[34:35], 01444; GFX7-NEXT: v_mov_b32_e32 v6, s71445; GFX7-NEXT: v_mov_b32_e32 v5, s51446; GFX7-NEXT: .LBB34_1: ; %atomicrmw.start1447; GFX7-NEXT: ; =>This Inner Loop Header: Depth=11448; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1449; GFX7-NEXT: v_subrev_i32_e32 v0, vcc, s6, v21450; GFX7-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc1451; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc1452; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1453; GFX7-NEXT: buffer_wbinvl1_vol1454; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]1455; GFX7-NEXT: v_mov_b32_e32 v3, v11456; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]1457; GFX7-NEXT: v_mov_b32_e32 v2, v01458; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]1459; GFX7-NEXT: s_cbranch_execnz .LBB34_11460; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end1461; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]1462; GFX7-NEXT: s_setpc_b64 s[30:31]1463;1464; GFX8-LABEL: flat_atomic_sub_i64_noret_scalar:1465; GFX8: ; %bb.0:1466; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1467; GFX8-NEXT: v_mov_b32_e32 v0, s41468; GFX8-NEXT: s_add_u32 s34, s4, 41469; GFX8-NEXT: v_mov_b32_e32 v1, s51470; GFX8-NEXT: s_addc_u32 s35, s5, 01471; GFX8-NEXT: v_mov_b32_e32 v3, s341472; GFX8-NEXT: v_mov_b32_e32 v4, s351473; GFX8-NEXT: flat_load_dword v2, v[0:1]1474; GFX8-NEXT: flat_load_dword v3, v[3:4]1475; GFX8-NEXT: v_mov_b32_e32 v4, s41476; GFX8-NEXT: s_mov_b64 s[34:35], 01477; GFX8-NEXT: v_mov_b32_e32 v6, s71478; GFX8-NEXT: v_mov_b32_e32 v5, s51479; GFX8-NEXT: .LBB34_1: ; %atomicrmw.start1480; GFX8-NEXT: ; =>This Inner Loop Header: Depth=11481; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1482; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, s6, v21483; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc1484; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc1485; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1486; GFX8-NEXT: buffer_wbinvl1_vol1487; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]1488; GFX8-NEXT: v_mov_b32_e32 v3, v11489; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]1490; GFX8-NEXT: v_mov_b32_e32 v2, v01491; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]1492; GFX8-NEXT: s_cbranch_execnz .LBB34_11493; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end1494; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]1495; GFX8-NEXT: s_setpc_b64 s[30:31]1496;1497; GFX9-LABEL: flat_atomic_sub_i64_noret_scalar:1498; GFX9: ; %bb.0:1499; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1500; GFX9-NEXT: v_mov_b32_e32 v0, s41501; GFX9-NEXT: v_mov_b32_e32 v1, s51502; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]1503; GFX9-NEXT: v_mov_b32_e32 v4, s41504; GFX9-NEXT: s_mov_b64 s[34:35], 01505; GFX9-NEXT: v_mov_b32_e32 v6, s71506; GFX9-NEXT: v_mov_b32_e32 v5, s51507; GFX9-NEXT: .LBB34_1: ; %atomicrmw.start1508; GFX9-NEXT: ; =>This Inner Loop Header: Depth=11509; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1510; GFX9-NEXT: v_subrev_co_u32_e32 v0, vcc, s6, v21511; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v6, vcc1512; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc1513; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1514; GFX9-NEXT: buffer_wbinvl1_vol1515; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]1516; GFX9-NEXT: v_mov_b32_e32 v3, v11517; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]1518; GFX9-NEXT: v_mov_b32_e32 v2, v01519; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]1520; GFX9-NEXT: s_cbranch_execnz .LBB34_11521; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end1522; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]1523; GFX9-NEXT: s_setpc_b64 s[30:31]1524 %tmp0 = atomicrmw sub ptr %ptr, i64 %in seq_cst, !noalias.addrspace !11525 ret void1526}1527 1528define amdgpu_gfx void @flat_atomic_sub_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {1529; GFX7-LABEL: flat_atomic_sub_i64_noret_offset_scalar:1530; GFX7: ; %bb.0:1531; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1532; GFX7-NEXT: s_add_u32 s34, s4, 321533; GFX7-NEXT: s_addc_u32 s35, s5, 01534; GFX7-NEXT: s_add_u32 s36, s4, 361535; GFX7-NEXT: s_addc_u32 s37, s5, 01536; GFX7-NEXT: v_mov_b32_e32 v0, s361537; GFX7-NEXT: v_mov_b32_e32 v1, s371538; GFX7-NEXT: v_mov_b32_e32 v4, s341539; GFX7-NEXT: v_mov_b32_e32 v5, s351540; GFX7-NEXT: flat_load_dword v3, v[0:1]1541; GFX7-NEXT: flat_load_dword v2, v[4:5]1542; GFX7-NEXT: s_mov_b64 s[34:35], 01543; GFX7-NEXT: v_mov_b32_e32 v6, s71544; GFX7-NEXT: .LBB35_1: ; %atomicrmw.start1545; GFX7-NEXT: ; =>This Inner Loop Header: Depth=11546; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1547; GFX7-NEXT: v_subrev_i32_e32 v0, vcc, s6, v21548; GFX7-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc1549; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc1550; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1551; GFX7-NEXT: buffer_wbinvl1_vol1552; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]1553; GFX7-NEXT: v_mov_b32_e32 v3, v11554; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]1555; GFX7-NEXT: v_mov_b32_e32 v2, v01556; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]1557; GFX7-NEXT: s_cbranch_execnz .LBB35_11558; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end1559; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]1560; GFX7-NEXT: s_setpc_b64 s[30:31]1561;1562; GFX8-LABEL: flat_atomic_sub_i64_noret_offset_scalar:1563; GFX8: ; %bb.0:1564; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1565; GFX8-NEXT: s_add_u32 s34, s4, 321566; GFX8-NEXT: s_addc_u32 s35, s5, 01567; GFX8-NEXT: s_add_u32 s36, s4, 361568; GFX8-NEXT: s_addc_u32 s37, s5, 01569; GFX8-NEXT: v_mov_b32_e32 v0, s361570; GFX8-NEXT: v_mov_b32_e32 v1, s371571; GFX8-NEXT: v_mov_b32_e32 v4, s341572; GFX8-NEXT: v_mov_b32_e32 v5, s351573; GFX8-NEXT: flat_load_dword v3, v[0:1]1574; GFX8-NEXT: flat_load_dword v2, v[4:5]1575; GFX8-NEXT: s_mov_b64 s[34:35], 01576; GFX8-NEXT: v_mov_b32_e32 v6, s71577; GFX8-NEXT: .LBB35_1: ; %atomicrmw.start1578; GFX8-NEXT: ; =>This Inner Loop Header: Depth=11579; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1580; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, s6, v21581; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc1582; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc1583; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1584; GFX8-NEXT: buffer_wbinvl1_vol1585; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]1586; GFX8-NEXT: v_mov_b32_e32 v3, v11587; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]1588; GFX8-NEXT: v_mov_b32_e32 v2, v01589; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]1590; GFX8-NEXT: s_cbranch_execnz .LBB35_11591; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end1592; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]1593; GFX8-NEXT: s_setpc_b64 s[30:31]1594;1595; GFX9-LABEL: flat_atomic_sub_i64_noret_offset_scalar:1596; GFX9: ; %bb.0:1597; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1598; GFX9-NEXT: v_mov_b32_e32 v0, s41599; GFX9-NEXT: v_mov_b32_e32 v1, s51600; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:321601; GFX9-NEXT: v_mov_b32_e32 v4, s41602; GFX9-NEXT: s_mov_b64 s[34:35], 01603; GFX9-NEXT: v_mov_b32_e32 v6, s71604; GFX9-NEXT: v_mov_b32_e32 v5, s51605; GFX9-NEXT: .LBB35_1: ; %atomicrmw.start1606; GFX9-NEXT: ; =>This Inner Loop Header: Depth=11607; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1608; GFX9-NEXT: v_subrev_co_u32_e32 v0, vcc, s6, v21609; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v6, vcc1610; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc1611; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1612; GFX9-NEXT: buffer_wbinvl1_vol1613; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]1614; GFX9-NEXT: v_mov_b32_e32 v3, v11615; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]1616; GFX9-NEXT: v_mov_b32_e32 v2, v01617; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]1618; GFX9-NEXT: s_cbranch_execnz .LBB35_11619; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end1620; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]1621; GFX9-NEXT: s_setpc_b64 s[30:31]1622 %gep = getelementptr inbounds i64, ptr %out, i64 41623 %tmp0 = atomicrmw sub ptr %gep, i64 %in seq_cst, !noalias.addrspace !11624 ret void1625}1626 1627define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {1628; GFX7-LABEL: flat_atomic_sub_i64_ret_scalar:1629; GFX7: ; %bb.0:1630; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1631; GFX7-NEXT: v_mov_b32_e32 v0, s41632; GFX7-NEXT: s_add_u32 s34, s4, 41633; GFX7-NEXT: v_mov_b32_e32 v1, s51634; GFX7-NEXT: s_addc_u32 s35, s5, 01635; GFX7-NEXT: v_mov_b32_e32 v2, s341636; GFX7-NEXT: v_mov_b32_e32 v3, s351637; GFX7-NEXT: flat_load_dword v0, v[0:1]1638; GFX7-NEXT: flat_load_dword v1, v[2:3]1639; GFX7-NEXT: v_mov_b32_e32 v2, s41640; GFX7-NEXT: s_mov_b64 s[34:35], 01641; GFX7-NEXT: v_mov_b32_e32 v4, s71642; GFX7-NEXT: v_mov_b32_e32 v3, s51643; GFX7-NEXT: .LBB36_1: ; %atomicrmw.start1644; GFX7-NEXT: ; =>This Inner Loop Header: Depth=11645; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1646; GFX7-NEXT: v_mov_b32_e32 v8, v11647; GFX7-NEXT: v_mov_b32_e32 v7, v01648; GFX7-NEXT: v_subrev_i32_e32 v5, vcc, s6, v71649; GFX7-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc1650; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc1651; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1652; GFX7-NEXT: buffer_wbinvl1_vol1653; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]1654; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]1655; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]1656; GFX7-NEXT: s_cbranch_execnz .LBB36_11657; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end1658; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]1659; GFX7-NEXT: s_setpc_b64 s[30:31]1660;1661; GFX8-LABEL: flat_atomic_sub_i64_ret_scalar:1662; GFX8: ; %bb.0:1663; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1664; GFX8-NEXT: v_mov_b32_e32 v0, s41665; GFX8-NEXT: s_add_u32 s34, s4, 41666; GFX8-NEXT: v_mov_b32_e32 v1, s51667; GFX8-NEXT: s_addc_u32 s35, s5, 01668; GFX8-NEXT: v_mov_b32_e32 v2, s341669; GFX8-NEXT: v_mov_b32_e32 v3, s351670; GFX8-NEXT: flat_load_dword v0, v[0:1]1671; GFX8-NEXT: flat_load_dword v1, v[2:3]1672; GFX8-NEXT: v_mov_b32_e32 v2, s41673; GFX8-NEXT: s_mov_b64 s[34:35], 01674; GFX8-NEXT: v_mov_b32_e32 v4, s71675; GFX8-NEXT: v_mov_b32_e32 v3, s51676; GFX8-NEXT: .LBB36_1: ; %atomicrmw.start1677; GFX8-NEXT: ; =>This Inner Loop Header: Depth=11678; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1679; GFX8-NEXT: v_mov_b32_e32 v8, v11680; GFX8-NEXT: v_mov_b32_e32 v7, v01681; GFX8-NEXT: v_subrev_u32_e32 v5, vcc, s6, v71682; GFX8-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc1683; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc1684; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1685; GFX8-NEXT: buffer_wbinvl1_vol1686; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]1687; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]1688; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]1689; GFX8-NEXT: s_cbranch_execnz .LBB36_11690; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end1691; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]1692; GFX8-NEXT: s_setpc_b64 s[30:31]1693;1694; GFX9-LABEL: flat_atomic_sub_i64_ret_scalar:1695; GFX9: ; %bb.0:1696; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1697; GFX9-NEXT: v_mov_b32_e32 v0, s41698; GFX9-NEXT: v_mov_b32_e32 v1, s51699; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]1700; GFX9-NEXT: v_mov_b32_e32 v2, s41701; GFX9-NEXT: s_mov_b64 s[34:35], 01702; GFX9-NEXT: v_mov_b32_e32 v4, s71703; GFX9-NEXT: v_mov_b32_e32 v3, s51704; GFX9-NEXT: .LBB36_1: ; %atomicrmw.start1705; GFX9-NEXT: ; =>This Inner Loop Header: Depth=11706; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1707; GFX9-NEXT: v_mov_b32_e32 v8, v11708; GFX9-NEXT: v_mov_b32_e32 v7, v01709; GFX9-NEXT: v_subrev_co_u32_e32 v5, vcc, s6, v71710; GFX9-NEXT: v_subb_co_u32_e32 v6, vcc, v8, v4, vcc1711; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc1712; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1713; GFX9-NEXT: buffer_wbinvl1_vol1714; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]1715; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]1716; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]1717; GFX9-NEXT: s_cbranch_execnz .LBB36_11718; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end1719; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]1720; GFX9-NEXT: s_setpc_b64 s[30:31]1721 %result = atomicrmw sub ptr %ptr, i64 %in seq_cst, !noalias.addrspace !11722 ret i64 %result1723}1724 1725define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {1726; GFX7-LABEL: flat_atomic_sub_i64_ret_offset_scalar:1727; GFX7: ; %bb.0:1728; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1729; GFX7-NEXT: s_add_u32 s34, s4, 321730; GFX7-NEXT: s_addc_u32 s35, s5, 01731; GFX7-NEXT: s_add_u32 s36, s4, 361732; GFX7-NEXT: s_addc_u32 s37, s5, 01733; GFX7-NEXT: v_mov_b32_e32 v0, s361734; GFX7-NEXT: v_mov_b32_e32 v1, s371735; GFX7-NEXT: v_mov_b32_e32 v2, s341736; GFX7-NEXT: v_mov_b32_e32 v3, s351737; GFX7-NEXT: flat_load_dword v1, v[0:1]1738; GFX7-NEXT: flat_load_dword v0, v[2:3]1739; GFX7-NEXT: s_mov_b64 s[34:35], 01740; GFX7-NEXT: v_mov_b32_e32 v4, s71741; GFX7-NEXT: .LBB37_1: ; %atomicrmw.start1742; GFX7-NEXT: ; =>This Inner Loop Header: Depth=11743; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1744; GFX7-NEXT: v_mov_b32_e32 v8, v11745; GFX7-NEXT: v_mov_b32_e32 v7, v01746; GFX7-NEXT: v_subrev_i32_e32 v5, vcc, s6, v71747; GFX7-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc1748; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc1749; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1750; GFX7-NEXT: buffer_wbinvl1_vol1751; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]1752; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]1753; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]1754; GFX7-NEXT: s_cbranch_execnz .LBB37_11755; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end1756; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]1757; GFX7-NEXT: s_setpc_b64 s[30:31]1758;1759; GFX8-LABEL: flat_atomic_sub_i64_ret_offset_scalar:1760; GFX8: ; %bb.0:1761; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1762; GFX8-NEXT: s_add_u32 s34, s4, 321763; GFX8-NEXT: s_addc_u32 s35, s5, 01764; GFX8-NEXT: s_add_u32 s36, s4, 361765; GFX8-NEXT: s_addc_u32 s37, s5, 01766; GFX8-NEXT: v_mov_b32_e32 v0, s361767; GFX8-NEXT: v_mov_b32_e32 v1, s371768; GFX8-NEXT: v_mov_b32_e32 v2, s341769; GFX8-NEXT: v_mov_b32_e32 v3, s351770; GFX8-NEXT: flat_load_dword v1, v[0:1]1771; GFX8-NEXT: flat_load_dword v0, v[2:3]1772; GFX8-NEXT: s_mov_b64 s[34:35], 01773; GFX8-NEXT: v_mov_b32_e32 v4, s71774; GFX8-NEXT: .LBB37_1: ; %atomicrmw.start1775; GFX8-NEXT: ; =>This Inner Loop Header: Depth=11776; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1777; GFX8-NEXT: v_mov_b32_e32 v8, v11778; GFX8-NEXT: v_mov_b32_e32 v7, v01779; GFX8-NEXT: v_subrev_u32_e32 v5, vcc, s6, v71780; GFX8-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc1781; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc1782; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1783; GFX8-NEXT: buffer_wbinvl1_vol1784; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]1785; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]1786; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]1787; GFX8-NEXT: s_cbranch_execnz .LBB37_11788; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end1789; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]1790; GFX8-NEXT: s_setpc_b64 s[30:31]1791;1792; GFX9-LABEL: flat_atomic_sub_i64_ret_offset_scalar:1793; GFX9: ; %bb.0:1794; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1795; GFX9-NEXT: v_mov_b32_e32 v0, s41796; GFX9-NEXT: v_mov_b32_e32 v1, s51797; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:321798; GFX9-NEXT: v_mov_b32_e32 v2, s41799; GFX9-NEXT: s_mov_b64 s[34:35], 01800; GFX9-NEXT: v_mov_b32_e32 v4, s71801; GFX9-NEXT: v_mov_b32_e32 v3, s51802; GFX9-NEXT: .LBB37_1: ; %atomicrmw.start1803; GFX9-NEXT: ; =>This Inner Loop Header: Depth=11804; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1805; GFX9-NEXT: v_mov_b32_e32 v8, v11806; GFX9-NEXT: v_mov_b32_e32 v7, v01807; GFX9-NEXT: v_subrev_co_u32_e32 v5, vcc, s6, v71808; GFX9-NEXT: v_subb_co_u32_e32 v6, vcc, v8, v4, vcc1809; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] offset:32 glc1810; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1811; GFX9-NEXT: buffer_wbinvl1_vol1812; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]1813; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]1814; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]1815; GFX9-NEXT: s_cbranch_execnz .LBB37_11816; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end1817; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]1818; GFX9-NEXT: s_setpc_b64 s[30:31]1819 %gep = getelementptr inbounds i64, ptr %out, i64 41820 %result = atomicrmw sub ptr %gep, i64 %in seq_cst, !noalias.addrspace !11821 ret i64 %result1822}1823 1824define void @flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {1825; GFX7-LABEL: flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory:1826; GFX7: ; %bb.0:1827; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1828; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v01829; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1830; GFX7-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]1831; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1832; GFX7-NEXT: buffer_wbinvl1_vol1833; GFX7-NEXT: s_setpc_b64 s[30:31]1834;1835; GFX8-LABEL: flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory:1836; GFX8: ; %bb.0:1837; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1838; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v01839; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1840; GFX8-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]1841; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1842; GFX8-NEXT: buffer_wbinvl1_vol1843; GFX8-NEXT: s_setpc_b64 s[30:31]1844;1845; GFX9-LABEL: flat_atomic_sub_i64_noret_offset__amdgpu_no_remote_memory:1846; GFX9: ; %bb.0:1847; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1848; GFX9-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3] offset:321849; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1850; GFX9-NEXT: buffer_wbinvl1_vol1851; GFX9-NEXT: s_setpc_b64 s[30:31]1852 %gep = getelementptr inbounds i64, ptr %out, i64 41853 %tmp0 = atomicrmw sub ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !11854 ret void1855}1856 1857define i64 @flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {1858; GFX7-LABEL: flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory:1859; GFX7: ; %bb.0:1860; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1861; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v01862; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1863; GFX7-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc1864; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1865; GFX7-NEXT: buffer_wbinvl1_vol1866; GFX7-NEXT: s_setpc_b64 s[30:31]1867;1868; GFX8-LABEL: flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory:1869; GFX8: ; %bb.0:1870; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1871; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v01872; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1873; GFX8-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc1874; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1875; GFX8-NEXT: buffer_wbinvl1_vol1876; GFX8-NEXT: s_setpc_b64 s[30:31]1877;1878; GFX9-LABEL: flat_atomic_sub_i64_ret_offset__amdgpu_no_remote_memory:1879; GFX9: ; %bb.0:1880; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1881; GFX9-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] offset:32 glc1882; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1883; GFX9-NEXT: buffer_wbinvl1_vol1884; GFX9-NEXT: s_setpc_b64 s[30:31]1885 %gep = getelementptr inbounds i64, ptr %out, i64 41886 %result = atomicrmw sub ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !11887 ret i64 %result1888}1889 1890; ---------------------------------------------------------------------1891; atomicrmw and1892; ---------------------------------------------------------------------1893 1894define void @flat_atomic_and_i64_noret(ptr %ptr, i64 %in) {1895; GFX7-LABEL: flat_atomic_and_i64_noret:1896; GFX7: ; %bb.0:1897; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1898; GFX7-NEXT: v_add_i32_e32 v4, vcc, 4, v01899; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc1900; GFX7-NEXT: flat_load_dword v6, v[0:1]1901; GFX7-NEXT: flat_load_dword v7, v[4:5]1902; GFX7-NEXT: s_mov_b64 s[4:5], 01903; GFX7-NEXT: .LBB40_1: ; %atomicrmw.start1904; GFX7-NEXT: ; =>This Inner Loop Header: Depth=11905; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1906; GFX7-NEXT: v_and_b32_e32 v5, v7, v31907; GFX7-NEXT: v_and_b32_e32 v4, v6, v21908; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc1909; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1910; GFX7-NEXT: buffer_wbinvl1_vol1911; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]1912; GFX7-NEXT: v_mov_b32_e32 v7, v51913; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1914; GFX7-NEXT: v_mov_b32_e32 v6, v41915; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]1916; GFX7-NEXT: s_cbranch_execnz .LBB40_11917; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end1918; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]1919; GFX7-NEXT: s_setpc_b64 s[30:31]1920;1921; GFX8-LABEL: flat_atomic_and_i64_noret:1922; GFX8: ; %bb.0:1923; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1924; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v01925; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc1926; GFX8-NEXT: flat_load_dword v6, v[0:1]1927; GFX8-NEXT: flat_load_dword v7, v[4:5]1928; GFX8-NEXT: s_mov_b64 s[4:5], 01929; GFX8-NEXT: .LBB40_1: ; %atomicrmw.start1930; GFX8-NEXT: ; =>This Inner Loop Header: Depth=11931; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1932; GFX8-NEXT: v_and_b32_e32 v5, v7, v31933; GFX8-NEXT: v_and_b32_e32 v4, v6, v21934; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc1935; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1936; GFX8-NEXT: buffer_wbinvl1_vol1937; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]1938; GFX8-NEXT: v_mov_b32_e32 v7, v51939; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1940; GFX8-NEXT: v_mov_b32_e32 v6, v41941; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]1942; GFX8-NEXT: s_cbranch_execnz .LBB40_11943; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end1944; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]1945; GFX8-NEXT: s_setpc_b64 s[30:31]1946;1947; GFX9-LABEL: flat_atomic_and_i64_noret:1948; GFX9: ; %bb.0:1949; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1950; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1]1951; GFX9-NEXT: s_mov_b64 s[4:5], 01952; GFX9-NEXT: .LBB40_1: ; %atomicrmw.start1953; GFX9-NEXT: ; =>This Inner Loop Header: Depth=11954; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1955; GFX9-NEXT: v_and_b32_e32 v5, v7, v31956; GFX9-NEXT: v_and_b32_e32 v4, v6, v21957; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc1958; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1959; GFX9-NEXT: buffer_wbinvl1_vol1960; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]1961; GFX9-NEXT: v_mov_b32_e32 v7, v51962; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1963; GFX9-NEXT: v_mov_b32_e32 v6, v41964; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]1965; GFX9-NEXT: s_cbranch_execnz .LBB40_11966; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end1967; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]1968; GFX9-NEXT: s_setpc_b64 s[30:31]1969 %tmp0 = atomicrmw and ptr %ptr, i64 %in seq_cst, !noalias.addrspace !11970 ret void1971}1972 1973define void @flat_atomic_and_i64_noret_offset(ptr %out, i64 %in) {1974; GFX7-LABEL: flat_atomic_and_i64_noret_offset:1975; GFX7: ; %bb.0:1976; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1977; GFX7-NEXT: v_add_i32_e32 v8, vcc, 32, v01978; GFX7-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc1979; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v01980; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc1981; GFX7-NEXT: flat_load_dword v7, v[0:1]1982; GFX7-NEXT: flat_load_dword v6, v[8:9]1983; GFX7-NEXT: s_mov_b64 s[4:5], 01984; GFX7-NEXT: .LBB41_1: ; %atomicrmw.start1985; GFX7-NEXT: ; =>This Inner Loop Header: Depth=11986; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1987; GFX7-NEXT: v_and_b32_e32 v5, v7, v31988; GFX7-NEXT: v_and_b32_e32 v4, v6, v21989; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc1990; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1991; GFX7-NEXT: buffer_wbinvl1_vol1992; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]1993; GFX7-NEXT: v_mov_b32_e32 v7, v11994; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1995; GFX7-NEXT: v_mov_b32_e32 v6, v01996; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]1997; GFX7-NEXT: s_cbranch_execnz .LBB41_11998; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end1999; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]2000; GFX7-NEXT: s_setpc_b64 s[30:31]2001;2002; GFX8-LABEL: flat_atomic_and_i64_noret_offset:2003; GFX8: ; %bb.0:2004; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2005; GFX8-NEXT: v_add_u32_e32 v8, vcc, 32, v02006; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc2007; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v02008; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2009; GFX8-NEXT: flat_load_dword v7, v[0:1]2010; GFX8-NEXT: flat_load_dword v6, v[8:9]2011; GFX8-NEXT: s_mov_b64 s[4:5], 02012; GFX8-NEXT: .LBB41_1: ; %atomicrmw.start2013; GFX8-NEXT: ; =>This Inner Loop Header: Depth=12014; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2015; GFX8-NEXT: v_and_b32_e32 v5, v7, v32016; GFX8-NEXT: v_and_b32_e32 v4, v6, v22017; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc2018; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2019; GFX8-NEXT: buffer_wbinvl1_vol2020; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]2021; GFX8-NEXT: v_mov_b32_e32 v7, v12022; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2023; GFX8-NEXT: v_mov_b32_e32 v6, v02024; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]2025; GFX8-NEXT: s_cbranch_execnz .LBB41_12026; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end2027; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]2028; GFX8-NEXT: s_setpc_b64 s[30:31]2029;2030; GFX9-LABEL: flat_atomic_and_i64_noret_offset:2031; GFX9: ; %bb.0:2032; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2033; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:322034; GFX9-NEXT: s_mov_b64 s[4:5], 02035; GFX9-NEXT: .LBB41_1: ; %atomicrmw.start2036; GFX9-NEXT: ; =>This Inner Loop Header: Depth=12037; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2038; GFX9-NEXT: v_and_b32_e32 v5, v7, v32039; GFX9-NEXT: v_and_b32_e32 v4, v6, v22040; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc2041; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2042; GFX9-NEXT: buffer_wbinvl1_vol2043; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]2044; GFX9-NEXT: v_mov_b32_e32 v7, v52045; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2046; GFX9-NEXT: v_mov_b32_e32 v6, v42047; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]2048; GFX9-NEXT: s_cbranch_execnz .LBB41_12049; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end2050; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]2051; GFX9-NEXT: s_setpc_b64 s[30:31]2052 %gep = getelementptr inbounds i64, ptr %out, i64 42053 %tmp0 = atomicrmw and ptr %gep, i64 %in seq_cst, !noalias.addrspace !12054 ret void2055}2056 2057define i64 @flat_atomic_and_i64_ret(ptr %ptr, i64 %in) {2058; GFX7-LABEL: flat_atomic_and_i64_ret:2059; GFX7: ; %bb.0:2060; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2061; GFX7-NEXT: v_add_i32_e32 v5, vcc, 4, v02062; GFX7-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc2063; GFX7-NEXT: flat_load_dword v4, v[0:1]2064; GFX7-NEXT: flat_load_dword v5, v[5:6]2065; GFX7-NEXT: s_mov_b64 s[4:5], 02066; GFX7-NEXT: .LBB42_1: ; %atomicrmw.start2067; GFX7-NEXT: ; =>This Inner Loop Header: Depth=12068; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2069; GFX7-NEXT: v_mov_b32_e32 v7, v52070; GFX7-NEXT: v_mov_b32_e32 v6, v42071; GFX7-NEXT: v_and_b32_e32 v5, v7, v32072; GFX7-NEXT: v_and_b32_e32 v4, v6, v22073; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc2074; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2075; GFX7-NEXT: buffer_wbinvl1_vol2076; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]2077; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2078; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]2079; GFX7-NEXT: s_cbranch_execnz .LBB42_12080; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end2081; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]2082; GFX7-NEXT: v_mov_b32_e32 v0, v42083; GFX7-NEXT: v_mov_b32_e32 v1, v52084; GFX7-NEXT: s_setpc_b64 s[30:31]2085;2086; GFX8-LABEL: flat_atomic_and_i64_ret:2087; GFX8: ; %bb.0:2088; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2089; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v02090; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc2091; GFX8-NEXT: flat_load_dword v4, v[0:1]2092; GFX8-NEXT: flat_load_dword v5, v[5:6]2093; GFX8-NEXT: s_mov_b64 s[4:5], 02094; GFX8-NEXT: .LBB42_1: ; %atomicrmw.start2095; GFX8-NEXT: ; =>This Inner Loop Header: Depth=12096; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2097; GFX8-NEXT: v_mov_b32_e32 v7, v52098; GFX8-NEXT: v_mov_b32_e32 v6, v42099; GFX8-NEXT: v_and_b32_e32 v5, v7, v32100; GFX8-NEXT: v_and_b32_e32 v4, v6, v22101; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc2102; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2103; GFX8-NEXT: buffer_wbinvl1_vol2104; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]2105; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2106; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]2107; GFX8-NEXT: s_cbranch_execnz .LBB42_12108; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end2109; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]2110; GFX8-NEXT: v_mov_b32_e32 v0, v42111; GFX8-NEXT: v_mov_b32_e32 v1, v52112; GFX8-NEXT: s_setpc_b64 s[30:31]2113;2114; GFX9-LABEL: flat_atomic_and_i64_ret:2115; GFX9: ; %bb.0:2116; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2117; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1]2118; GFX9-NEXT: s_mov_b64 s[4:5], 02119; GFX9-NEXT: .LBB42_1: ; %atomicrmw.start2120; GFX9-NEXT: ; =>This Inner Loop Header: Depth=12121; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2122; GFX9-NEXT: v_mov_b32_e32 v7, v52123; GFX9-NEXT: v_mov_b32_e32 v6, v42124; GFX9-NEXT: v_and_b32_e32 v5, v7, v32125; GFX9-NEXT: v_and_b32_e32 v4, v6, v22126; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc2127; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2128; GFX9-NEXT: buffer_wbinvl1_vol2129; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]2130; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2131; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]2132; GFX9-NEXT: s_cbranch_execnz .LBB42_12133; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end2134; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]2135; GFX9-NEXT: v_mov_b32_e32 v0, v42136; GFX9-NEXT: v_mov_b32_e32 v1, v52137; GFX9-NEXT: s_setpc_b64 s[30:31]2138 %result = atomicrmw and ptr %ptr, i64 %in seq_cst, !noalias.addrspace !12139 ret i64 %result2140}2141 2142define i64 @flat_atomic_and_i64_ret_offset(ptr %out, i64 %in) {2143; GFX7-LABEL: flat_atomic_and_i64_ret_offset:2144; GFX7: ; %bb.0:2145; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2146; GFX7-NEXT: v_add_i32_e32 v4, vcc, 32, v02147; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc2148; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v02149; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2150; GFX7-NEXT: flat_load_dword v1, v[0:1]2151; GFX7-NEXT: flat_load_dword v0, v[4:5]2152; GFX7-NEXT: s_mov_b64 s[4:5], 02153; GFX7-NEXT: .LBB43_1: ; %atomicrmw.start2154; GFX7-NEXT: ; =>This Inner Loop Header: Depth=12155; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2156; GFX7-NEXT: v_mov_b32_e32 v9, v12157; GFX7-NEXT: v_mov_b32_e32 v8, v02158; GFX7-NEXT: v_and_b32_e32 v7, v9, v32159; GFX7-NEXT: v_and_b32_e32 v6, v8, v22160; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc2161; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2162; GFX7-NEXT: buffer_wbinvl1_vol2163; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]2164; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2165; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]2166; GFX7-NEXT: s_cbranch_execnz .LBB43_12167; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end2168; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]2169; GFX7-NEXT: s_setpc_b64 s[30:31]2170;2171; GFX8-LABEL: flat_atomic_and_i64_ret_offset:2172; GFX8: ; %bb.0:2173; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2174; GFX8-NEXT: v_add_u32_e32 v4, vcc, 32, v02175; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc2176; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v02177; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2178; GFX8-NEXT: flat_load_dword v1, v[0:1]2179; GFX8-NEXT: flat_load_dword v0, v[4:5]2180; GFX8-NEXT: s_mov_b64 s[4:5], 02181; GFX8-NEXT: .LBB43_1: ; %atomicrmw.start2182; GFX8-NEXT: ; =>This Inner Loop Header: Depth=12183; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2184; GFX8-NEXT: v_mov_b32_e32 v9, v12185; GFX8-NEXT: v_mov_b32_e32 v8, v02186; GFX8-NEXT: v_and_b32_e32 v7, v9, v32187; GFX8-NEXT: v_and_b32_e32 v6, v8, v22188; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc2189; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2190; GFX8-NEXT: buffer_wbinvl1_vol2191; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]2192; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2193; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]2194; GFX8-NEXT: s_cbranch_execnz .LBB43_12195; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end2196; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]2197; GFX8-NEXT: s_setpc_b64 s[30:31]2198;2199; GFX9-LABEL: flat_atomic_and_i64_ret_offset:2200; GFX9: ; %bb.0:2201; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2202; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:322203; GFX9-NEXT: s_mov_b64 s[4:5], 02204; GFX9-NEXT: .LBB43_1: ; %atomicrmw.start2205; GFX9-NEXT: ; =>This Inner Loop Header: Depth=12206; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2207; GFX9-NEXT: v_mov_b32_e32 v7, v52208; GFX9-NEXT: v_mov_b32_e32 v6, v42209; GFX9-NEXT: v_and_b32_e32 v5, v7, v32210; GFX9-NEXT: v_and_b32_e32 v4, v6, v22211; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc2212; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2213; GFX9-NEXT: buffer_wbinvl1_vol2214; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]2215; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2216; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]2217; GFX9-NEXT: s_cbranch_execnz .LBB43_12218; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end2219; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]2220; GFX9-NEXT: v_mov_b32_e32 v0, v42221; GFX9-NEXT: v_mov_b32_e32 v1, v52222; GFX9-NEXT: s_setpc_b64 s[30:31]2223 %gep = getelementptr inbounds i64, ptr %out, i64 42224 %result = atomicrmw and ptr %gep, i64 %in seq_cst, !noalias.addrspace !12225 ret i64 %result2226}2227 2228define amdgpu_gfx void @flat_atomic_and_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {2229; GFX7-LABEL: flat_atomic_and_i64_noret_scalar:2230; GFX7: ; %bb.0:2231; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2232; GFX7-NEXT: v_mov_b32_e32 v0, s42233; GFX7-NEXT: s_add_u32 s34, s4, 42234; GFX7-NEXT: v_mov_b32_e32 v1, s52235; GFX7-NEXT: s_addc_u32 s35, s5, 02236; GFX7-NEXT: v_mov_b32_e32 v3, s342237; GFX7-NEXT: v_mov_b32_e32 v4, s352238; GFX7-NEXT: flat_load_dword v2, v[0:1]2239; GFX7-NEXT: flat_load_dword v3, v[3:4]2240; GFX7-NEXT: v_mov_b32_e32 v4, s42241; GFX7-NEXT: s_mov_b64 s[34:35], 02242; GFX7-NEXT: v_mov_b32_e32 v5, s52243; GFX7-NEXT: .LBB44_1: ; %atomicrmw.start2244; GFX7-NEXT: ; =>This Inner Loop Header: Depth=12245; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2246; GFX7-NEXT: v_and_b32_e32 v1, s7, v32247; GFX7-NEXT: v_and_b32_e32 v0, s6, v22248; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc2249; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2250; GFX7-NEXT: buffer_wbinvl1_vol2251; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]2252; GFX7-NEXT: v_mov_b32_e32 v3, v12253; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]2254; GFX7-NEXT: v_mov_b32_e32 v2, v02255; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]2256; GFX7-NEXT: s_cbranch_execnz .LBB44_12257; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end2258; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]2259; GFX7-NEXT: s_setpc_b64 s[30:31]2260;2261; GFX8-LABEL: flat_atomic_and_i64_noret_scalar:2262; GFX8: ; %bb.0:2263; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2264; GFX8-NEXT: v_mov_b32_e32 v0, s42265; GFX8-NEXT: s_add_u32 s34, s4, 42266; GFX8-NEXT: v_mov_b32_e32 v1, s52267; GFX8-NEXT: s_addc_u32 s35, s5, 02268; GFX8-NEXT: v_mov_b32_e32 v3, s342269; GFX8-NEXT: v_mov_b32_e32 v4, s352270; GFX8-NEXT: flat_load_dword v2, v[0:1]2271; GFX8-NEXT: flat_load_dword v3, v[3:4]2272; GFX8-NEXT: v_mov_b32_e32 v4, s42273; GFX8-NEXT: s_mov_b64 s[34:35], 02274; GFX8-NEXT: v_mov_b32_e32 v5, s52275; GFX8-NEXT: .LBB44_1: ; %atomicrmw.start2276; GFX8-NEXT: ; =>This Inner Loop Header: Depth=12277; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2278; GFX8-NEXT: v_and_b32_e32 v1, s7, v32279; GFX8-NEXT: v_and_b32_e32 v0, s6, v22280; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc2281; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2282; GFX8-NEXT: buffer_wbinvl1_vol2283; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]2284; GFX8-NEXT: v_mov_b32_e32 v3, v12285; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]2286; GFX8-NEXT: v_mov_b32_e32 v2, v02287; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]2288; GFX8-NEXT: s_cbranch_execnz .LBB44_12289; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end2290; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]2291; GFX8-NEXT: s_setpc_b64 s[30:31]2292;2293; GFX9-LABEL: flat_atomic_and_i64_noret_scalar:2294; GFX9: ; %bb.0:2295; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2296; GFX9-NEXT: v_mov_b32_e32 v0, s42297; GFX9-NEXT: v_mov_b32_e32 v1, s52298; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]2299; GFX9-NEXT: v_mov_b32_e32 v4, s42300; GFX9-NEXT: s_mov_b64 s[34:35], 02301; GFX9-NEXT: v_mov_b32_e32 v5, s52302; GFX9-NEXT: .LBB44_1: ; %atomicrmw.start2303; GFX9-NEXT: ; =>This Inner Loop Header: Depth=12304; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2305; GFX9-NEXT: v_and_b32_e32 v1, s7, v32306; GFX9-NEXT: v_and_b32_e32 v0, s6, v22307; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc2308; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2309; GFX9-NEXT: buffer_wbinvl1_vol2310; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]2311; GFX9-NEXT: v_mov_b32_e32 v3, v12312; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]2313; GFX9-NEXT: v_mov_b32_e32 v2, v02314; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]2315; GFX9-NEXT: s_cbranch_execnz .LBB44_12316; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end2317; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]2318; GFX9-NEXT: s_setpc_b64 s[30:31]2319 %tmp0 = atomicrmw and ptr %ptr, i64 %in seq_cst, !noalias.addrspace !12320 ret void2321}2322 2323define amdgpu_gfx void @flat_atomic_and_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {2324; GFX7-LABEL: flat_atomic_and_i64_noret_offset_scalar:2325; GFX7: ; %bb.0:2326; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2327; GFX7-NEXT: s_add_u32 s34, s4, 322328; GFX7-NEXT: s_addc_u32 s35, s5, 02329; GFX7-NEXT: s_add_u32 s36, s4, 362330; GFX7-NEXT: s_addc_u32 s37, s5, 02331; GFX7-NEXT: v_mov_b32_e32 v0, s362332; GFX7-NEXT: v_mov_b32_e32 v1, s372333; GFX7-NEXT: v_mov_b32_e32 v4, s342334; GFX7-NEXT: v_mov_b32_e32 v5, s352335; GFX7-NEXT: flat_load_dword v3, v[0:1]2336; GFX7-NEXT: flat_load_dword v2, v[4:5]2337; GFX7-NEXT: s_mov_b64 s[34:35], 02338; GFX7-NEXT: .LBB45_1: ; %atomicrmw.start2339; GFX7-NEXT: ; =>This Inner Loop Header: Depth=12340; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2341; GFX7-NEXT: v_and_b32_e32 v1, s7, v32342; GFX7-NEXT: v_and_b32_e32 v0, s6, v22343; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc2344; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2345; GFX7-NEXT: buffer_wbinvl1_vol2346; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]2347; GFX7-NEXT: v_mov_b32_e32 v3, v12348; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]2349; GFX7-NEXT: v_mov_b32_e32 v2, v02350; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]2351; GFX7-NEXT: s_cbranch_execnz .LBB45_12352; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end2353; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]2354; GFX7-NEXT: s_setpc_b64 s[30:31]2355;2356; GFX8-LABEL: flat_atomic_and_i64_noret_offset_scalar:2357; GFX8: ; %bb.0:2358; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2359; GFX8-NEXT: s_add_u32 s34, s4, 322360; GFX8-NEXT: s_addc_u32 s35, s5, 02361; GFX8-NEXT: s_add_u32 s36, s4, 362362; GFX8-NEXT: s_addc_u32 s37, s5, 02363; GFX8-NEXT: v_mov_b32_e32 v0, s362364; GFX8-NEXT: v_mov_b32_e32 v1, s372365; GFX8-NEXT: v_mov_b32_e32 v4, s342366; GFX8-NEXT: v_mov_b32_e32 v5, s352367; GFX8-NEXT: flat_load_dword v3, v[0:1]2368; GFX8-NEXT: flat_load_dword v2, v[4:5]2369; GFX8-NEXT: s_mov_b64 s[34:35], 02370; GFX8-NEXT: .LBB45_1: ; %atomicrmw.start2371; GFX8-NEXT: ; =>This Inner Loop Header: Depth=12372; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2373; GFX8-NEXT: v_and_b32_e32 v1, s7, v32374; GFX8-NEXT: v_and_b32_e32 v0, s6, v22375; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc2376; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2377; GFX8-NEXT: buffer_wbinvl1_vol2378; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]2379; GFX8-NEXT: v_mov_b32_e32 v3, v12380; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]2381; GFX8-NEXT: v_mov_b32_e32 v2, v02382; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]2383; GFX8-NEXT: s_cbranch_execnz .LBB45_12384; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end2385; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]2386; GFX8-NEXT: s_setpc_b64 s[30:31]2387;2388; GFX9-LABEL: flat_atomic_and_i64_noret_offset_scalar:2389; GFX9: ; %bb.0:2390; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2391; GFX9-NEXT: v_mov_b32_e32 v0, s42392; GFX9-NEXT: v_mov_b32_e32 v1, s52393; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:322394; GFX9-NEXT: v_mov_b32_e32 v4, s42395; GFX9-NEXT: s_mov_b64 s[34:35], 02396; GFX9-NEXT: v_mov_b32_e32 v5, s52397; GFX9-NEXT: .LBB45_1: ; %atomicrmw.start2398; GFX9-NEXT: ; =>This Inner Loop Header: Depth=12399; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2400; GFX9-NEXT: v_and_b32_e32 v1, s7, v32401; GFX9-NEXT: v_and_b32_e32 v0, s6, v22402; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc2403; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2404; GFX9-NEXT: buffer_wbinvl1_vol2405; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]2406; GFX9-NEXT: v_mov_b32_e32 v3, v12407; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]2408; GFX9-NEXT: v_mov_b32_e32 v2, v02409; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]2410; GFX9-NEXT: s_cbranch_execnz .LBB45_12411; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end2412; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]2413; GFX9-NEXT: s_setpc_b64 s[30:31]2414 %gep = getelementptr inbounds i64, ptr %out, i64 42415 %tmp0 = atomicrmw and ptr %gep, i64 %in seq_cst, !noalias.addrspace !12416 ret void2417}2418 2419define amdgpu_gfx i64 @flat_atomic_and_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {2420; GFX7-LABEL: flat_atomic_and_i64_ret_scalar:2421; GFX7: ; %bb.0:2422; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2423; GFX7-NEXT: v_mov_b32_e32 v0, s42424; GFX7-NEXT: s_add_u32 s34, s4, 42425; GFX7-NEXT: v_mov_b32_e32 v1, s52426; GFX7-NEXT: s_addc_u32 s35, s5, 02427; GFX7-NEXT: v_mov_b32_e32 v2, s342428; GFX7-NEXT: v_mov_b32_e32 v3, s352429; GFX7-NEXT: flat_load_dword v0, v[0:1]2430; GFX7-NEXT: flat_load_dword v1, v[2:3]2431; GFX7-NEXT: v_mov_b32_e32 v2, s42432; GFX7-NEXT: s_mov_b64 s[34:35], 02433; GFX7-NEXT: v_mov_b32_e32 v3, s52434; GFX7-NEXT: .LBB46_1: ; %atomicrmw.start2435; GFX7-NEXT: ; =>This Inner Loop Header: Depth=12436; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2437; GFX7-NEXT: v_mov_b32_e32 v7, v12438; GFX7-NEXT: v_mov_b32_e32 v6, v02439; GFX7-NEXT: v_and_b32_e32 v5, s7, v72440; GFX7-NEXT: v_and_b32_e32 v4, s6, v62441; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc2442; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2443; GFX7-NEXT: buffer_wbinvl1_vol2444; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]2445; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]2446; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]2447; GFX7-NEXT: s_cbranch_execnz .LBB46_12448; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end2449; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]2450; GFX7-NEXT: s_setpc_b64 s[30:31]2451;2452; GFX8-LABEL: flat_atomic_and_i64_ret_scalar:2453; GFX8: ; %bb.0:2454; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2455; GFX8-NEXT: v_mov_b32_e32 v0, s42456; GFX8-NEXT: s_add_u32 s34, s4, 42457; GFX8-NEXT: v_mov_b32_e32 v1, s52458; GFX8-NEXT: s_addc_u32 s35, s5, 02459; GFX8-NEXT: v_mov_b32_e32 v2, s342460; GFX8-NEXT: v_mov_b32_e32 v3, s352461; GFX8-NEXT: flat_load_dword v0, v[0:1]2462; GFX8-NEXT: flat_load_dword v1, v[2:3]2463; GFX8-NEXT: v_mov_b32_e32 v2, s42464; GFX8-NEXT: s_mov_b64 s[34:35], 02465; GFX8-NEXT: v_mov_b32_e32 v3, s52466; GFX8-NEXT: .LBB46_1: ; %atomicrmw.start2467; GFX8-NEXT: ; =>This Inner Loop Header: Depth=12468; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2469; GFX8-NEXT: v_mov_b32_e32 v7, v12470; GFX8-NEXT: v_mov_b32_e32 v6, v02471; GFX8-NEXT: v_and_b32_e32 v5, s7, v72472; GFX8-NEXT: v_and_b32_e32 v4, s6, v62473; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc2474; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2475; GFX8-NEXT: buffer_wbinvl1_vol2476; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]2477; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]2478; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]2479; GFX8-NEXT: s_cbranch_execnz .LBB46_12480; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end2481; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]2482; GFX8-NEXT: s_setpc_b64 s[30:31]2483;2484; GFX9-LABEL: flat_atomic_and_i64_ret_scalar:2485; GFX9: ; %bb.0:2486; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2487; GFX9-NEXT: v_mov_b32_e32 v0, s42488; GFX9-NEXT: v_mov_b32_e32 v1, s52489; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]2490; GFX9-NEXT: v_mov_b32_e32 v2, s42491; GFX9-NEXT: s_mov_b64 s[34:35], 02492; GFX9-NEXT: v_mov_b32_e32 v3, s52493; GFX9-NEXT: .LBB46_1: ; %atomicrmw.start2494; GFX9-NEXT: ; =>This Inner Loop Header: Depth=12495; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2496; GFX9-NEXT: v_mov_b32_e32 v7, v12497; GFX9-NEXT: v_mov_b32_e32 v6, v02498; GFX9-NEXT: v_and_b32_e32 v5, s7, v72499; GFX9-NEXT: v_and_b32_e32 v4, s6, v62500; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc2501; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2502; GFX9-NEXT: buffer_wbinvl1_vol2503; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]2504; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]2505; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]2506; GFX9-NEXT: s_cbranch_execnz .LBB46_12507; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end2508; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]2509; GFX9-NEXT: s_setpc_b64 s[30:31]2510 %result = atomicrmw and ptr %ptr, i64 %in seq_cst, !noalias.addrspace !12511 ret i64 %result2512}2513 2514define amdgpu_gfx i64 @flat_atomic_and_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {2515; GFX7-LABEL: flat_atomic_and_i64_ret_offset_scalar:2516; GFX7: ; %bb.0:2517; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2518; GFX7-NEXT: s_add_u32 s34, s4, 322519; GFX7-NEXT: s_addc_u32 s35, s5, 02520; GFX7-NEXT: s_add_u32 s36, s4, 362521; GFX7-NEXT: s_addc_u32 s37, s5, 02522; GFX7-NEXT: v_mov_b32_e32 v0, s362523; GFX7-NEXT: v_mov_b32_e32 v1, s372524; GFX7-NEXT: v_mov_b32_e32 v2, s342525; GFX7-NEXT: v_mov_b32_e32 v3, s352526; GFX7-NEXT: flat_load_dword v1, v[0:1]2527; GFX7-NEXT: flat_load_dword v0, v[2:3]2528; GFX7-NEXT: s_mov_b64 s[34:35], 02529; GFX7-NEXT: .LBB47_1: ; %atomicrmw.start2530; GFX7-NEXT: ; =>This Inner Loop Header: Depth=12531; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2532; GFX7-NEXT: v_mov_b32_e32 v7, v12533; GFX7-NEXT: v_mov_b32_e32 v6, v02534; GFX7-NEXT: v_and_b32_e32 v5, s7, v72535; GFX7-NEXT: v_and_b32_e32 v4, s6, v62536; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc2537; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2538; GFX7-NEXT: buffer_wbinvl1_vol2539; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]2540; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]2541; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]2542; GFX7-NEXT: s_cbranch_execnz .LBB47_12543; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end2544; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]2545; GFX7-NEXT: s_setpc_b64 s[30:31]2546;2547; GFX8-LABEL: flat_atomic_and_i64_ret_offset_scalar:2548; GFX8: ; %bb.0:2549; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2550; GFX8-NEXT: s_add_u32 s34, s4, 322551; GFX8-NEXT: s_addc_u32 s35, s5, 02552; GFX8-NEXT: s_add_u32 s36, s4, 362553; GFX8-NEXT: s_addc_u32 s37, s5, 02554; GFX8-NEXT: v_mov_b32_e32 v0, s362555; GFX8-NEXT: v_mov_b32_e32 v1, s372556; GFX8-NEXT: v_mov_b32_e32 v2, s342557; GFX8-NEXT: v_mov_b32_e32 v3, s352558; GFX8-NEXT: flat_load_dword v1, v[0:1]2559; GFX8-NEXT: flat_load_dword v0, v[2:3]2560; GFX8-NEXT: s_mov_b64 s[34:35], 02561; GFX8-NEXT: .LBB47_1: ; %atomicrmw.start2562; GFX8-NEXT: ; =>This Inner Loop Header: Depth=12563; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2564; GFX8-NEXT: v_mov_b32_e32 v7, v12565; GFX8-NEXT: v_mov_b32_e32 v6, v02566; GFX8-NEXT: v_and_b32_e32 v5, s7, v72567; GFX8-NEXT: v_and_b32_e32 v4, s6, v62568; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc2569; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2570; GFX8-NEXT: buffer_wbinvl1_vol2571; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]2572; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]2573; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]2574; GFX8-NEXT: s_cbranch_execnz .LBB47_12575; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end2576; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]2577; GFX8-NEXT: s_setpc_b64 s[30:31]2578;2579; GFX9-LABEL: flat_atomic_and_i64_ret_offset_scalar:2580; GFX9: ; %bb.0:2581; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2582; GFX9-NEXT: v_mov_b32_e32 v0, s42583; GFX9-NEXT: v_mov_b32_e32 v1, s52584; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:322585; GFX9-NEXT: v_mov_b32_e32 v2, s42586; GFX9-NEXT: s_mov_b64 s[34:35], 02587; GFX9-NEXT: v_mov_b32_e32 v3, s52588; GFX9-NEXT: .LBB47_1: ; %atomicrmw.start2589; GFX9-NEXT: ; =>This Inner Loop Header: Depth=12590; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2591; GFX9-NEXT: v_mov_b32_e32 v7, v12592; GFX9-NEXT: v_mov_b32_e32 v6, v02593; GFX9-NEXT: v_and_b32_e32 v5, s7, v72594; GFX9-NEXT: v_and_b32_e32 v4, s6, v62595; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] offset:32 glc2596; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2597; GFX9-NEXT: buffer_wbinvl1_vol2598; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]2599; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]2600; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]2601; GFX9-NEXT: s_cbranch_execnz .LBB47_12602; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end2603; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]2604; GFX9-NEXT: s_setpc_b64 s[30:31]2605 %gep = getelementptr inbounds i64, ptr %out, i64 42606 %result = atomicrmw and ptr %gep, i64 %in seq_cst, !noalias.addrspace !12607 ret i64 %result2608}2609 2610define void @flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {2611; GFX7-LABEL: flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory:2612; GFX7: ; %bb.0:2613; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2614; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v02615; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2616; GFX7-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]2617; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2618; GFX7-NEXT: buffer_wbinvl1_vol2619; GFX7-NEXT: s_setpc_b64 s[30:31]2620;2621; GFX8-LABEL: flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory:2622; GFX8: ; %bb.0:2623; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2624; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v02625; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2626; GFX8-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]2627; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2628; GFX8-NEXT: buffer_wbinvl1_vol2629; GFX8-NEXT: s_setpc_b64 s[30:31]2630;2631; GFX9-LABEL: flat_atomic_and_i64_noret_offset__amdgpu_no_remote_memory:2632; GFX9: ; %bb.0:2633; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2634; GFX9-NEXT: flat_atomic_and_x2 v[0:1], v[2:3] offset:322635; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2636; GFX9-NEXT: buffer_wbinvl1_vol2637; GFX9-NEXT: s_setpc_b64 s[30:31]2638 %gep = getelementptr inbounds i64, ptr %out, i64 42639 %tmp0 = atomicrmw and ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !12640 ret void2641}2642 2643define i64 @flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {2644; GFX7-LABEL: flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory:2645; GFX7: ; %bb.0:2646; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2647; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v02648; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2649; GFX7-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc2650; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2651; GFX7-NEXT: buffer_wbinvl1_vol2652; GFX7-NEXT: s_setpc_b64 s[30:31]2653;2654; GFX8-LABEL: flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory:2655; GFX8: ; %bb.0:2656; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2657; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v02658; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2659; GFX8-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc2660; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2661; GFX8-NEXT: buffer_wbinvl1_vol2662; GFX8-NEXT: s_setpc_b64 s[30:31]2663;2664; GFX9-LABEL: flat_atomic_and_i64_ret_offset__amdgpu_no_remote_memory:2665; GFX9: ; %bb.0:2666; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2667; GFX9-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] offset:32 glc2668; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2669; GFX9-NEXT: buffer_wbinvl1_vol2670; GFX9-NEXT: s_setpc_b64 s[30:31]2671 %gep = getelementptr inbounds i64, ptr %out, i64 42672 %result = atomicrmw and ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !12673 ret i64 %result2674}2675 2676; ---------------------------------------------------------------------2677; atomicrmw nand2678; ---------------------------------------------------------------------2679 2680define void @flat_atomic_nand_i64_noret(ptr %ptr, i64 %in) {2681; GFX7-LABEL: flat_atomic_nand_i64_noret:2682; GFX7: ; %bb.0:2683; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2684; GFX7-NEXT: v_add_i32_e32 v4, vcc, 4, v02685; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc2686; GFX7-NEXT: flat_load_dword v6, v[0:1]2687; GFX7-NEXT: flat_load_dword v7, v[4:5]2688; GFX7-NEXT: s_mov_b64 s[4:5], 02689; GFX7-NEXT: .LBB50_1: ; %atomicrmw.start2690; GFX7-NEXT: ; =>This Inner Loop Header: Depth=12691; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2692; GFX7-NEXT: v_and_b32_e32 v4, v7, v32693; GFX7-NEXT: v_and_b32_e32 v8, v6, v22694; GFX7-NEXT: v_not_b32_e32 v5, v42695; GFX7-NEXT: v_not_b32_e32 v4, v82696; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc2697; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2698; GFX7-NEXT: buffer_wbinvl1_vol2699; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]2700; GFX7-NEXT: v_mov_b32_e32 v7, v52701; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2702; GFX7-NEXT: v_mov_b32_e32 v6, v42703; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]2704; GFX7-NEXT: s_cbranch_execnz .LBB50_12705; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end2706; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]2707; GFX7-NEXT: s_setpc_b64 s[30:31]2708;2709; GFX8-LABEL: flat_atomic_nand_i64_noret:2710; GFX8: ; %bb.0:2711; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2712; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v02713; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc2714; GFX8-NEXT: flat_load_dword v6, v[0:1]2715; GFX8-NEXT: flat_load_dword v7, v[4:5]2716; GFX8-NEXT: s_mov_b64 s[4:5], 02717; GFX8-NEXT: .LBB50_1: ; %atomicrmw.start2718; GFX8-NEXT: ; =>This Inner Loop Header: Depth=12719; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2720; GFX8-NEXT: v_and_b32_e32 v4, v7, v32721; GFX8-NEXT: v_and_b32_e32 v8, v6, v22722; GFX8-NEXT: v_not_b32_e32 v5, v42723; GFX8-NEXT: v_not_b32_e32 v4, v82724; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc2725; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2726; GFX8-NEXT: buffer_wbinvl1_vol2727; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]2728; GFX8-NEXT: v_mov_b32_e32 v7, v52729; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2730; GFX8-NEXT: v_mov_b32_e32 v6, v42731; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]2732; GFX8-NEXT: s_cbranch_execnz .LBB50_12733; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end2734; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]2735; GFX8-NEXT: s_setpc_b64 s[30:31]2736;2737; GFX9-LABEL: flat_atomic_nand_i64_noret:2738; GFX9: ; %bb.0:2739; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2740; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1]2741; GFX9-NEXT: s_mov_b64 s[4:5], 02742; GFX9-NEXT: .LBB50_1: ; %atomicrmw.start2743; GFX9-NEXT: ; =>This Inner Loop Header: Depth=12744; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2745; GFX9-NEXT: v_and_b32_e32 v4, v7, v32746; GFX9-NEXT: v_and_b32_e32 v8, v6, v22747; GFX9-NEXT: v_not_b32_e32 v5, v42748; GFX9-NEXT: v_not_b32_e32 v4, v82749; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc2750; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2751; GFX9-NEXT: buffer_wbinvl1_vol2752; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]2753; GFX9-NEXT: v_mov_b32_e32 v7, v52754; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2755; GFX9-NEXT: v_mov_b32_e32 v6, v42756; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]2757; GFX9-NEXT: s_cbranch_execnz .LBB50_12758; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end2759; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]2760; GFX9-NEXT: s_setpc_b64 s[30:31]2761 %tmp0 = atomicrmw nand ptr %ptr, i64 %in seq_cst, !noalias.addrspace !12762 ret void2763}2764 2765define void @flat_atomic_nand_i64_noret_offset(ptr %out, i64 %in) {2766; GFX7-LABEL: flat_atomic_nand_i64_noret_offset:2767; GFX7: ; %bb.0:2768; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2769; GFX7-NEXT: v_add_i32_e32 v8, vcc, 32, v02770; GFX7-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc2771; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v02772; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2773; GFX7-NEXT: flat_load_dword v7, v[0:1]2774; GFX7-NEXT: flat_load_dword v6, v[8:9]2775; GFX7-NEXT: s_mov_b64 s[4:5], 02776; GFX7-NEXT: .LBB51_1: ; %atomicrmw.start2777; GFX7-NEXT: ; =>This Inner Loop Header: Depth=12778; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2779; GFX7-NEXT: v_and_b32_e32 v0, v7, v32780; GFX7-NEXT: v_and_b32_e32 v1, v6, v22781; GFX7-NEXT: v_not_b32_e32 v5, v02782; GFX7-NEXT: v_not_b32_e32 v4, v12783; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc2784; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2785; GFX7-NEXT: buffer_wbinvl1_vol2786; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]2787; GFX7-NEXT: v_mov_b32_e32 v7, v12788; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2789; GFX7-NEXT: v_mov_b32_e32 v6, v02790; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]2791; GFX7-NEXT: s_cbranch_execnz .LBB51_12792; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end2793; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]2794; GFX7-NEXT: s_setpc_b64 s[30:31]2795;2796; GFX8-LABEL: flat_atomic_nand_i64_noret_offset:2797; GFX8: ; %bb.0:2798; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2799; GFX8-NEXT: v_add_u32_e32 v8, vcc, 32, v02800; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc2801; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v02802; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2803; GFX8-NEXT: flat_load_dword v7, v[0:1]2804; GFX8-NEXT: flat_load_dword v6, v[8:9]2805; GFX8-NEXT: s_mov_b64 s[4:5], 02806; GFX8-NEXT: .LBB51_1: ; %atomicrmw.start2807; GFX8-NEXT: ; =>This Inner Loop Header: Depth=12808; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2809; GFX8-NEXT: v_and_b32_e32 v0, v7, v32810; GFX8-NEXT: v_and_b32_e32 v1, v6, v22811; GFX8-NEXT: v_not_b32_e32 v5, v02812; GFX8-NEXT: v_not_b32_e32 v4, v12813; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc2814; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2815; GFX8-NEXT: buffer_wbinvl1_vol2816; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]2817; GFX8-NEXT: v_mov_b32_e32 v7, v12818; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2819; GFX8-NEXT: v_mov_b32_e32 v6, v02820; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]2821; GFX8-NEXT: s_cbranch_execnz .LBB51_12822; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end2823; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]2824; GFX8-NEXT: s_setpc_b64 s[30:31]2825;2826; GFX9-LABEL: flat_atomic_nand_i64_noret_offset:2827; GFX9: ; %bb.0:2828; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2829; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:322830; GFX9-NEXT: s_mov_b64 s[4:5], 02831; GFX9-NEXT: .LBB51_1: ; %atomicrmw.start2832; GFX9-NEXT: ; =>This Inner Loop Header: Depth=12833; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2834; GFX9-NEXT: v_and_b32_e32 v4, v7, v32835; GFX9-NEXT: v_and_b32_e32 v8, v6, v22836; GFX9-NEXT: v_not_b32_e32 v5, v42837; GFX9-NEXT: v_not_b32_e32 v4, v82838; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc2839; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2840; GFX9-NEXT: buffer_wbinvl1_vol2841; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]2842; GFX9-NEXT: v_mov_b32_e32 v7, v52843; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2844; GFX9-NEXT: v_mov_b32_e32 v6, v42845; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]2846; GFX9-NEXT: s_cbranch_execnz .LBB51_12847; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end2848; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]2849; GFX9-NEXT: s_setpc_b64 s[30:31]2850 %gep = getelementptr inbounds i64, ptr %out, i64 42851 %tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst, !noalias.addrspace !12852 ret void2853}2854 2855define i64 @flat_atomic_nand_i64_ret(ptr %ptr, i64 %in) {2856; GFX7-LABEL: flat_atomic_nand_i64_ret:2857; GFX7: ; %bb.0:2858; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2859; GFX7-NEXT: v_add_i32_e32 v5, vcc, 4, v02860; GFX7-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc2861; GFX7-NEXT: flat_load_dword v4, v[0:1]2862; GFX7-NEXT: flat_load_dword v5, v[5:6]2863; GFX7-NEXT: s_mov_b64 s[4:5], 02864; GFX7-NEXT: .LBB52_1: ; %atomicrmw.start2865; GFX7-NEXT: ; =>This Inner Loop Header: Depth=12866; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2867; GFX7-NEXT: v_mov_b32_e32 v7, v52868; GFX7-NEXT: v_mov_b32_e32 v6, v42869; GFX7-NEXT: v_and_b32_e32 v4, v7, v32870; GFX7-NEXT: v_and_b32_e32 v8, v6, v22871; GFX7-NEXT: v_not_b32_e32 v5, v42872; GFX7-NEXT: v_not_b32_e32 v4, v82873; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc2874; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2875; GFX7-NEXT: buffer_wbinvl1_vol2876; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]2877; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2878; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]2879; GFX7-NEXT: s_cbranch_execnz .LBB52_12880; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end2881; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]2882; GFX7-NEXT: v_mov_b32_e32 v0, v42883; GFX7-NEXT: v_mov_b32_e32 v1, v52884; GFX7-NEXT: s_setpc_b64 s[30:31]2885;2886; GFX8-LABEL: flat_atomic_nand_i64_ret:2887; GFX8: ; %bb.0:2888; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2889; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v02890; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc2891; GFX8-NEXT: flat_load_dword v4, v[0:1]2892; GFX8-NEXT: flat_load_dword v5, v[5:6]2893; GFX8-NEXT: s_mov_b64 s[4:5], 02894; GFX8-NEXT: .LBB52_1: ; %atomicrmw.start2895; GFX8-NEXT: ; =>This Inner Loop Header: Depth=12896; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2897; GFX8-NEXT: v_mov_b32_e32 v7, v52898; GFX8-NEXT: v_mov_b32_e32 v6, v42899; GFX8-NEXT: v_and_b32_e32 v4, v7, v32900; GFX8-NEXT: v_and_b32_e32 v8, v6, v22901; GFX8-NEXT: v_not_b32_e32 v5, v42902; GFX8-NEXT: v_not_b32_e32 v4, v82903; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc2904; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2905; GFX8-NEXT: buffer_wbinvl1_vol2906; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]2907; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2908; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]2909; GFX8-NEXT: s_cbranch_execnz .LBB52_12910; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end2911; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]2912; GFX8-NEXT: v_mov_b32_e32 v0, v42913; GFX8-NEXT: v_mov_b32_e32 v1, v52914; GFX8-NEXT: s_setpc_b64 s[30:31]2915;2916; GFX9-LABEL: flat_atomic_nand_i64_ret:2917; GFX9: ; %bb.0:2918; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2919; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1]2920; GFX9-NEXT: s_mov_b64 s[4:5], 02921; GFX9-NEXT: .LBB52_1: ; %atomicrmw.start2922; GFX9-NEXT: ; =>This Inner Loop Header: Depth=12923; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2924; GFX9-NEXT: v_mov_b32_e32 v7, v52925; GFX9-NEXT: v_mov_b32_e32 v6, v42926; GFX9-NEXT: v_and_b32_e32 v4, v7, v32927; GFX9-NEXT: v_and_b32_e32 v8, v6, v22928; GFX9-NEXT: v_not_b32_e32 v5, v42929; GFX9-NEXT: v_not_b32_e32 v4, v82930; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc2931; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2932; GFX9-NEXT: buffer_wbinvl1_vol2933; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]2934; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2935; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]2936; GFX9-NEXT: s_cbranch_execnz .LBB52_12937; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end2938; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]2939; GFX9-NEXT: v_mov_b32_e32 v0, v42940; GFX9-NEXT: v_mov_b32_e32 v1, v52941; GFX9-NEXT: s_setpc_b64 s[30:31]2942 %result = atomicrmw nand ptr %ptr, i64 %in seq_cst, !noalias.addrspace !12943 ret i64 %result2944}2945 2946define i64 @flat_atomic_nand_i64_ret_offset(ptr %out, i64 %in) {2947; GFX7-LABEL: flat_atomic_nand_i64_ret_offset:2948; GFX7: ; %bb.0:2949; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2950; GFX7-NEXT: v_add_i32_e32 v4, vcc, 32, v02951; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc2952; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v02953; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2954; GFX7-NEXT: flat_load_dword v1, v[0:1]2955; GFX7-NEXT: flat_load_dword v0, v[4:5]2956; GFX7-NEXT: s_mov_b64 s[4:5], 02957; GFX7-NEXT: .LBB53_1: ; %atomicrmw.start2958; GFX7-NEXT: ; =>This Inner Loop Header: Depth=12959; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2960; GFX7-NEXT: v_mov_b32_e32 v9, v12961; GFX7-NEXT: v_mov_b32_e32 v8, v02962; GFX7-NEXT: v_and_b32_e32 v0, v9, v32963; GFX7-NEXT: v_and_b32_e32 v1, v8, v22964; GFX7-NEXT: v_not_b32_e32 v7, v02965; GFX7-NEXT: v_not_b32_e32 v6, v12966; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc2967; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2968; GFX7-NEXT: buffer_wbinvl1_vol2969; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]2970; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2971; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]2972; GFX7-NEXT: s_cbranch_execnz .LBB53_12973; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end2974; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]2975; GFX7-NEXT: s_setpc_b64 s[30:31]2976;2977; GFX8-LABEL: flat_atomic_nand_i64_ret_offset:2978; GFX8: ; %bb.0:2979; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2980; GFX8-NEXT: v_add_u32_e32 v4, vcc, 32, v02981; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc2982; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v02983; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc2984; GFX8-NEXT: flat_load_dword v1, v[0:1]2985; GFX8-NEXT: flat_load_dword v0, v[4:5]2986; GFX8-NEXT: s_mov_b64 s[4:5], 02987; GFX8-NEXT: .LBB53_1: ; %atomicrmw.start2988; GFX8-NEXT: ; =>This Inner Loop Header: Depth=12989; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2990; GFX8-NEXT: v_mov_b32_e32 v9, v12991; GFX8-NEXT: v_mov_b32_e32 v8, v02992; GFX8-NEXT: v_and_b32_e32 v0, v9, v32993; GFX8-NEXT: v_and_b32_e32 v1, v8, v22994; GFX8-NEXT: v_not_b32_e32 v7, v02995; GFX8-NEXT: v_not_b32_e32 v6, v12996; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc2997; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2998; GFX8-NEXT: buffer_wbinvl1_vol2999; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]3000; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3001; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]3002; GFX8-NEXT: s_cbranch_execnz .LBB53_13003; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end3004; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]3005; GFX8-NEXT: s_setpc_b64 s[30:31]3006;3007; GFX9-LABEL: flat_atomic_nand_i64_ret_offset:3008; GFX9: ; %bb.0:3009; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3010; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:323011; GFX9-NEXT: s_mov_b64 s[4:5], 03012; GFX9-NEXT: .LBB53_1: ; %atomicrmw.start3013; GFX9-NEXT: ; =>This Inner Loop Header: Depth=13014; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3015; GFX9-NEXT: v_mov_b32_e32 v7, v53016; GFX9-NEXT: v_mov_b32_e32 v6, v43017; GFX9-NEXT: v_and_b32_e32 v4, v7, v33018; GFX9-NEXT: v_and_b32_e32 v8, v6, v23019; GFX9-NEXT: v_not_b32_e32 v5, v43020; GFX9-NEXT: v_not_b32_e32 v4, v83021; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc3022; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3023; GFX9-NEXT: buffer_wbinvl1_vol3024; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]3025; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3026; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]3027; GFX9-NEXT: s_cbranch_execnz .LBB53_13028; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end3029; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]3030; GFX9-NEXT: v_mov_b32_e32 v0, v43031; GFX9-NEXT: v_mov_b32_e32 v1, v53032; GFX9-NEXT: s_setpc_b64 s[30:31]3033 %gep = getelementptr inbounds i64, ptr %out, i64 43034 %result = atomicrmw nand ptr %gep, i64 %in seq_cst, !noalias.addrspace !13035 ret i64 %result3036}3037 3038define amdgpu_gfx void @flat_atomic_nand_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {3039; GFX7-LABEL: flat_atomic_nand_i64_noret_scalar:3040; GFX7: ; %bb.0:3041; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3042; GFX7-NEXT: v_mov_b32_e32 v0, s43043; GFX7-NEXT: s_add_u32 s34, s4, 43044; GFX7-NEXT: v_mov_b32_e32 v1, s53045; GFX7-NEXT: s_addc_u32 s35, s5, 03046; GFX7-NEXT: v_mov_b32_e32 v3, s343047; GFX7-NEXT: v_mov_b32_e32 v4, s353048; GFX7-NEXT: flat_load_dword v2, v[0:1]3049; GFX7-NEXT: flat_load_dword v3, v[3:4]3050; GFX7-NEXT: v_mov_b32_e32 v4, s43051; GFX7-NEXT: s_mov_b64 s[34:35], 03052; GFX7-NEXT: v_mov_b32_e32 v5, s53053; GFX7-NEXT: .LBB54_1: ; %atomicrmw.start3054; GFX7-NEXT: ; =>This Inner Loop Header: Depth=13055; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3056; GFX7-NEXT: v_and_b32_e32 v0, s7, v33057; GFX7-NEXT: v_and_b32_e32 v6, s6, v23058; GFX7-NEXT: v_not_b32_e32 v1, v03059; GFX7-NEXT: v_not_b32_e32 v0, v63060; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc3061; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3062; GFX7-NEXT: buffer_wbinvl1_vol3063; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]3064; GFX7-NEXT: v_mov_b32_e32 v3, v13065; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]3066; GFX7-NEXT: v_mov_b32_e32 v2, v03067; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]3068; GFX7-NEXT: s_cbranch_execnz .LBB54_13069; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end3070; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]3071; GFX7-NEXT: s_setpc_b64 s[30:31]3072;3073; GFX8-LABEL: flat_atomic_nand_i64_noret_scalar:3074; GFX8: ; %bb.0:3075; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3076; GFX8-NEXT: v_mov_b32_e32 v0, s43077; GFX8-NEXT: s_add_u32 s34, s4, 43078; GFX8-NEXT: v_mov_b32_e32 v1, s53079; GFX8-NEXT: s_addc_u32 s35, s5, 03080; GFX8-NEXT: v_mov_b32_e32 v3, s343081; GFX8-NEXT: v_mov_b32_e32 v4, s353082; GFX8-NEXT: flat_load_dword v2, v[0:1]3083; GFX8-NEXT: flat_load_dword v3, v[3:4]3084; GFX8-NEXT: v_mov_b32_e32 v4, s43085; GFX8-NEXT: s_mov_b64 s[34:35], 03086; GFX8-NEXT: v_mov_b32_e32 v5, s53087; GFX8-NEXT: .LBB54_1: ; %atomicrmw.start3088; GFX8-NEXT: ; =>This Inner Loop Header: Depth=13089; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3090; GFX8-NEXT: v_and_b32_e32 v0, s7, v33091; GFX8-NEXT: v_and_b32_e32 v6, s6, v23092; GFX8-NEXT: v_not_b32_e32 v1, v03093; GFX8-NEXT: v_not_b32_e32 v0, v63094; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc3095; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3096; GFX8-NEXT: buffer_wbinvl1_vol3097; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]3098; GFX8-NEXT: v_mov_b32_e32 v3, v13099; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]3100; GFX8-NEXT: v_mov_b32_e32 v2, v03101; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]3102; GFX8-NEXT: s_cbranch_execnz .LBB54_13103; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end3104; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]3105; GFX8-NEXT: s_setpc_b64 s[30:31]3106;3107; GFX9-LABEL: flat_atomic_nand_i64_noret_scalar:3108; GFX9: ; %bb.0:3109; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3110; GFX9-NEXT: v_mov_b32_e32 v0, s43111; GFX9-NEXT: v_mov_b32_e32 v1, s53112; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]3113; GFX9-NEXT: v_mov_b32_e32 v4, s43114; GFX9-NEXT: s_mov_b64 s[34:35], 03115; GFX9-NEXT: v_mov_b32_e32 v5, s53116; GFX9-NEXT: .LBB54_1: ; %atomicrmw.start3117; GFX9-NEXT: ; =>This Inner Loop Header: Depth=13118; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3119; GFX9-NEXT: v_and_b32_e32 v0, s7, v33120; GFX9-NEXT: v_and_b32_e32 v6, s6, v23121; GFX9-NEXT: v_not_b32_e32 v1, v03122; GFX9-NEXT: v_not_b32_e32 v0, v63123; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc3124; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3125; GFX9-NEXT: buffer_wbinvl1_vol3126; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]3127; GFX9-NEXT: v_mov_b32_e32 v3, v13128; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]3129; GFX9-NEXT: v_mov_b32_e32 v2, v03130; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]3131; GFX9-NEXT: s_cbranch_execnz .LBB54_13132; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end3133; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]3134; GFX9-NEXT: s_setpc_b64 s[30:31]3135 %tmp0 = atomicrmw nand ptr %ptr, i64 %in seq_cst, !noalias.addrspace !13136 ret void3137}3138 3139define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {3140; GFX7-LABEL: flat_atomic_nand_i64_noret_offset_scalar:3141; GFX7: ; %bb.0:3142; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3143; GFX7-NEXT: s_add_u32 s34, s4, 323144; GFX7-NEXT: s_addc_u32 s35, s5, 03145; GFX7-NEXT: s_add_u32 s36, s4, 363146; GFX7-NEXT: s_addc_u32 s37, s5, 03147; GFX7-NEXT: v_mov_b32_e32 v0, s363148; GFX7-NEXT: v_mov_b32_e32 v1, s373149; GFX7-NEXT: v_mov_b32_e32 v4, s343150; GFX7-NEXT: v_mov_b32_e32 v5, s353151; GFX7-NEXT: flat_load_dword v3, v[0:1]3152; GFX7-NEXT: flat_load_dword v2, v[4:5]3153; GFX7-NEXT: s_mov_b64 s[34:35], 03154; GFX7-NEXT: .LBB55_1: ; %atomicrmw.start3155; GFX7-NEXT: ; =>This Inner Loop Header: Depth=13156; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3157; GFX7-NEXT: v_and_b32_e32 v0, s7, v33158; GFX7-NEXT: v_and_b32_e32 v6, s6, v23159; GFX7-NEXT: v_not_b32_e32 v1, v03160; GFX7-NEXT: v_not_b32_e32 v0, v63161; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc3162; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3163; GFX7-NEXT: buffer_wbinvl1_vol3164; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]3165; GFX7-NEXT: v_mov_b32_e32 v3, v13166; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]3167; GFX7-NEXT: v_mov_b32_e32 v2, v03168; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]3169; GFX7-NEXT: s_cbranch_execnz .LBB55_13170; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end3171; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]3172; GFX7-NEXT: s_setpc_b64 s[30:31]3173;3174; GFX8-LABEL: flat_atomic_nand_i64_noret_offset_scalar:3175; GFX8: ; %bb.0:3176; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3177; GFX8-NEXT: s_add_u32 s34, s4, 323178; GFX8-NEXT: s_addc_u32 s35, s5, 03179; GFX8-NEXT: s_add_u32 s36, s4, 363180; GFX8-NEXT: s_addc_u32 s37, s5, 03181; GFX8-NEXT: v_mov_b32_e32 v0, s363182; GFX8-NEXT: v_mov_b32_e32 v1, s373183; GFX8-NEXT: v_mov_b32_e32 v4, s343184; GFX8-NEXT: v_mov_b32_e32 v5, s353185; GFX8-NEXT: flat_load_dword v3, v[0:1]3186; GFX8-NEXT: flat_load_dword v2, v[4:5]3187; GFX8-NEXT: s_mov_b64 s[34:35], 03188; GFX8-NEXT: .LBB55_1: ; %atomicrmw.start3189; GFX8-NEXT: ; =>This Inner Loop Header: Depth=13190; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3191; GFX8-NEXT: v_and_b32_e32 v0, s7, v33192; GFX8-NEXT: v_and_b32_e32 v6, s6, v23193; GFX8-NEXT: v_not_b32_e32 v1, v03194; GFX8-NEXT: v_not_b32_e32 v0, v63195; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc3196; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3197; GFX8-NEXT: buffer_wbinvl1_vol3198; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]3199; GFX8-NEXT: v_mov_b32_e32 v3, v13200; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]3201; GFX8-NEXT: v_mov_b32_e32 v2, v03202; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]3203; GFX8-NEXT: s_cbranch_execnz .LBB55_13204; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end3205; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]3206; GFX8-NEXT: s_setpc_b64 s[30:31]3207;3208; GFX9-LABEL: flat_atomic_nand_i64_noret_offset_scalar:3209; GFX9: ; %bb.0:3210; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3211; GFX9-NEXT: v_mov_b32_e32 v0, s43212; GFX9-NEXT: v_mov_b32_e32 v1, s53213; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:323214; GFX9-NEXT: v_mov_b32_e32 v4, s43215; GFX9-NEXT: s_mov_b64 s[34:35], 03216; GFX9-NEXT: v_mov_b32_e32 v5, s53217; GFX9-NEXT: .LBB55_1: ; %atomicrmw.start3218; GFX9-NEXT: ; =>This Inner Loop Header: Depth=13219; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3220; GFX9-NEXT: v_and_b32_e32 v0, s7, v33221; GFX9-NEXT: v_and_b32_e32 v6, s6, v23222; GFX9-NEXT: v_not_b32_e32 v1, v03223; GFX9-NEXT: v_not_b32_e32 v0, v63224; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc3225; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3226; GFX9-NEXT: buffer_wbinvl1_vol3227; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]3228; GFX9-NEXT: v_mov_b32_e32 v3, v13229; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]3230; GFX9-NEXT: v_mov_b32_e32 v2, v03231; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]3232; GFX9-NEXT: s_cbranch_execnz .LBB55_13233; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end3234; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]3235; GFX9-NEXT: s_setpc_b64 s[30:31]3236 %gep = getelementptr inbounds i64, ptr %out, i64 43237 %tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst, !noalias.addrspace !13238 ret void3239}3240 3241define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {3242; GFX7-LABEL: flat_atomic_nand_i64_ret_scalar:3243; GFX7: ; %bb.0:3244; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3245; GFX7-NEXT: v_mov_b32_e32 v0, s43246; GFX7-NEXT: s_add_u32 s34, s4, 43247; GFX7-NEXT: v_mov_b32_e32 v1, s53248; GFX7-NEXT: s_addc_u32 s35, s5, 03249; GFX7-NEXT: v_mov_b32_e32 v2, s343250; GFX7-NEXT: v_mov_b32_e32 v3, s353251; GFX7-NEXT: flat_load_dword v0, v[0:1]3252; GFX7-NEXT: flat_load_dword v1, v[2:3]3253; GFX7-NEXT: v_mov_b32_e32 v2, s43254; GFX7-NEXT: s_mov_b64 s[34:35], 03255; GFX7-NEXT: v_mov_b32_e32 v3, s53256; GFX7-NEXT: .LBB56_1: ; %atomicrmw.start3257; GFX7-NEXT: ; =>This Inner Loop Header: Depth=13258; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3259; GFX7-NEXT: v_mov_b32_e32 v7, v13260; GFX7-NEXT: v_mov_b32_e32 v6, v03261; GFX7-NEXT: v_and_b32_e32 v0, s7, v73262; GFX7-NEXT: v_and_b32_e32 v1, s6, v63263; GFX7-NEXT: v_not_b32_e32 v5, v03264; GFX7-NEXT: v_not_b32_e32 v4, v13265; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc3266; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3267; GFX7-NEXT: buffer_wbinvl1_vol3268; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]3269; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]3270; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]3271; GFX7-NEXT: s_cbranch_execnz .LBB56_13272; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end3273; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]3274; GFX7-NEXT: s_setpc_b64 s[30:31]3275;3276; GFX8-LABEL: flat_atomic_nand_i64_ret_scalar:3277; GFX8: ; %bb.0:3278; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3279; GFX8-NEXT: v_mov_b32_e32 v0, s43280; GFX8-NEXT: s_add_u32 s34, s4, 43281; GFX8-NEXT: v_mov_b32_e32 v1, s53282; GFX8-NEXT: s_addc_u32 s35, s5, 03283; GFX8-NEXT: v_mov_b32_e32 v2, s343284; GFX8-NEXT: v_mov_b32_e32 v3, s353285; GFX8-NEXT: flat_load_dword v0, v[0:1]3286; GFX8-NEXT: flat_load_dword v1, v[2:3]3287; GFX8-NEXT: v_mov_b32_e32 v2, s43288; GFX8-NEXT: s_mov_b64 s[34:35], 03289; GFX8-NEXT: v_mov_b32_e32 v3, s53290; GFX8-NEXT: .LBB56_1: ; %atomicrmw.start3291; GFX8-NEXT: ; =>This Inner Loop Header: Depth=13292; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3293; GFX8-NEXT: v_mov_b32_e32 v7, v13294; GFX8-NEXT: v_mov_b32_e32 v6, v03295; GFX8-NEXT: v_and_b32_e32 v0, s7, v73296; GFX8-NEXT: v_and_b32_e32 v1, s6, v63297; GFX8-NEXT: v_not_b32_e32 v5, v03298; GFX8-NEXT: v_not_b32_e32 v4, v13299; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc3300; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3301; GFX8-NEXT: buffer_wbinvl1_vol3302; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]3303; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]3304; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]3305; GFX8-NEXT: s_cbranch_execnz .LBB56_13306; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end3307; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]3308; GFX8-NEXT: s_setpc_b64 s[30:31]3309;3310; GFX9-LABEL: flat_atomic_nand_i64_ret_scalar:3311; GFX9: ; %bb.0:3312; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3313; GFX9-NEXT: v_mov_b32_e32 v0, s43314; GFX9-NEXT: v_mov_b32_e32 v1, s53315; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]3316; GFX9-NEXT: v_mov_b32_e32 v2, s43317; GFX9-NEXT: s_mov_b64 s[34:35], 03318; GFX9-NEXT: v_mov_b32_e32 v3, s53319; GFX9-NEXT: .LBB56_1: ; %atomicrmw.start3320; GFX9-NEXT: ; =>This Inner Loop Header: Depth=13321; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3322; GFX9-NEXT: v_mov_b32_e32 v7, v13323; GFX9-NEXT: v_mov_b32_e32 v6, v03324; GFX9-NEXT: v_and_b32_e32 v0, s7, v73325; GFX9-NEXT: v_and_b32_e32 v1, s6, v63326; GFX9-NEXT: v_not_b32_e32 v5, v03327; GFX9-NEXT: v_not_b32_e32 v4, v13328; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc3329; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3330; GFX9-NEXT: buffer_wbinvl1_vol3331; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]3332; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]3333; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]3334; GFX9-NEXT: s_cbranch_execnz .LBB56_13335; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end3336; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]3337; GFX9-NEXT: s_setpc_b64 s[30:31]3338 %result = atomicrmw nand ptr %ptr, i64 %in seq_cst, !noalias.addrspace !13339 ret i64 %result3340}3341 3342define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {3343; GFX7-LABEL: flat_atomic_nand_i64_ret_offset_scalar:3344; GFX7: ; %bb.0:3345; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3346; GFX7-NEXT: s_add_u32 s34, s4, 323347; GFX7-NEXT: s_addc_u32 s35, s5, 03348; GFX7-NEXT: s_add_u32 s36, s4, 363349; GFX7-NEXT: s_addc_u32 s37, s5, 03350; GFX7-NEXT: v_mov_b32_e32 v0, s363351; GFX7-NEXT: v_mov_b32_e32 v1, s373352; GFX7-NEXT: v_mov_b32_e32 v2, s343353; GFX7-NEXT: v_mov_b32_e32 v3, s353354; GFX7-NEXT: flat_load_dword v1, v[0:1]3355; GFX7-NEXT: flat_load_dword v0, v[2:3]3356; GFX7-NEXT: s_mov_b64 s[34:35], 03357; GFX7-NEXT: .LBB57_1: ; %atomicrmw.start3358; GFX7-NEXT: ; =>This Inner Loop Header: Depth=13359; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3360; GFX7-NEXT: v_mov_b32_e32 v7, v13361; GFX7-NEXT: v_mov_b32_e32 v6, v03362; GFX7-NEXT: v_and_b32_e32 v0, s7, v73363; GFX7-NEXT: v_and_b32_e32 v1, s6, v63364; GFX7-NEXT: v_not_b32_e32 v5, v03365; GFX7-NEXT: v_not_b32_e32 v4, v13366; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc3367; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3368; GFX7-NEXT: buffer_wbinvl1_vol3369; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]3370; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]3371; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]3372; GFX7-NEXT: s_cbranch_execnz .LBB57_13373; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end3374; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]3375; GFX7-NEXT: s_setpc_b64 s[30:31]3376;3377; GFX8-LABEL: flat_atomic_nand_i64_ret_offset_scalar:3378; GFX8: ; %bb.0:3379; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3380; GFX8-NEXT: s_add_u32 s34, s4, 323381; GFX8-NEXT: s_addc_u32 s35, s5, 03382; GFX8-NEXT: s_add_u32 s36, s4, 363383; GFX8-NEXT: s_addc_u32 s37, s5, 03384; GFX8-NEXT: v_mov_b32_e32 v0, s363385; GFX8-NEXT: v_mov_b32_e32 v1, s373386; GFX8-NEXT: v_mov_b32_e32 v2, s343387; GFX8-NEXT: v_mov_b32_e32 v3, s353388; GFX8-NEXT: flat_load_dword v1, v[0:1]3389; GFX8-NEXT: flat_load_dword v0, v[2:3]3390; GFX8-NEXT: s_mov_b64 s[34:35], 03391; GFX8-NEXT: .LBB57_1: ; %atomicrmw.start3392; GFX8-NEXT: ; =>This Inner Loop Header: Depth=13393; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3394; GFX8-NEXT: v_mov_b32_e32 v7, v13395; GFX8-NEXT: v_mov_b32_e32 v6, v03396; GFX8-NEXT: v_and_b32_e32 v0, s7, v73397; GFX8-NEXT: v_and_b32_e32 v1, s6, v63398; GFX8-NEXT: v_not_b32_e32 v5, v03399; GFX8-NEXT: v_not_b32_e32 v4, v13400; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc3401; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3402; GFX8-NEXT: buffer_wbinvl1_vol3403; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]3404; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]3405; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]3406; GFX8-NEXT: s_cbranch_execnz .LBB57_13407; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end3408; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]3409; GFX8-NEXT: s_setpc_b64 s[30:31]3410;3411; GFX9-LABEL: flat_atomic_nand_i64_ret_offset_scalar:3412; GFX9: ; %bb.0:3413; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3414; GFX9-NEXT: v_mov_b32_e32 v0, s43415; GFX9-NEXT: v_mov_b32_e32 v1, s53416; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:323417; GFX9-NEXT: v_mov_b32_e32 v2, s43418; GFX9-NEXT: s_mov_b64 s[34:35], 03419; GFX9-NEXT: v_mov_b32_e32 v3, s53420; GFX9-NEXT: .LBB57_1: ; %atomicrmw.start3421; GFX9-NEXT: ; =>This Inner Loop Header: Depth=13422; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3423; GFX9-NEXT: v_mov_b32_e32 v7, v13424; GFX9-NEXT: v_mov_b32_e32 v6, v03425; GFX9-NEXT: v_and_b32_e32 v0, s7, v73426; GFX9-NEXT: v_and_b32_e32 v1, s6, v63427; GFX9-NEXT: v_not_b32_e32 v5, v03428; GFX9-NEXT: v_not_b32_e32 v4, v13429; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] offset:32 glc3430; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3431; GFX9-NEXT: buffer_wbinvl1_vol3432; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]3433; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]3434; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]3435; GFX9-NEXT: s_cbranch_execnz .LBB57_13436; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end3437; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]3438; GFX9-NEXT: s_setpc_b64 s[30:31]3439 %gep = getelementptr inbounds i64, ptr %out, i64 43440 %result = atomicrmw nand ptr %gep, i64 %in seq_cst, !noalias.addrspace !13441 ret i64 %result3442}3443 3444define void @flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {3445; GFX7-LABEL: flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory:3446; GFX7: ; %bb.0:3447; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3448; GFX7-NEXT: v_add_i32_e32 v8, vcc, 32, v03449; GFX7-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc3450; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v03451; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3452; GFX7-NEXT: flat_load_dword v7, v[0:1]3453; GFX7-NEXT: flat_load_dword v6, v[8:9]3454; GFX7-NEXT: s_mov_b64 s[4:5], 03455; GFX7-NEXT: .LBB58_1: ; %atomicrmw.start3456; GFX7-NEXT: ; =>This Inner Loop Header: Depth=13457; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3458; GFX7-NEXT: v_and_b32_e32 v0, v7, v33459; GFX7-NEXT: v_and_b32_e32 v1, v6, v23460; GFX7-NEXT: v_not_b32_e32 v5, v03461; GFX7-NEXT: v_not_b32_e32 v4, v13462; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc3463; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3464; GFX7-NEXT: buffer_wbinvl1_vol3465; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]3466; GFX7-NEXT: v_mov_b32_e32 v7, v13467; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3468; GFX7-NEXT: v_mov_b32_e32 v6, v03469; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]3470; GFX7-NEXT: s_cbranch_execnz .LBB58_13471; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end3472; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]3473; GFX7-NEXT: s_setpc_b64 s[30:31]3474;3475; GFX8-LABEL: flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory:3476; GFX8: ; %bb.0:3477; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3478; GFX8-NEXT: v_add_u32_e32 v8, vcc, 32, v03479; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc3480; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v03481; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3482; GFX8-NEXT: flat_load_dword v7, v[0:1]3483; GFX8-NEXT: flat_load_dword v6, v[8:9]3484; GFX8-NEXT: s_mov_b64 s[4:5], 03485; GFX8-NEXT: .LBB58_1: ; %atomicrmw.start3486; GFX8-NEXT: ; =>This Inner Loop Header: Depth=13487; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3488; GFX8-NEXT: v_and_b32_e32 v0, v7, v33489; GFX8-NEXT: v_and_b32_e32 v1, v6, v23490; GFX8-NEXT: v_not_b32_e32 v5, v03491; GFX8-NEXT: v_not_b32_e32 v4, v13492; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc3493; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3494; GFX8-NEXT: buffer_wbinvl1_vol3495; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]3496; GFX8-NEXT: v_mov_b32_e32 v7, v13497; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3498; GFX8-NEXT: v_mov_b32_e32 v6, v03499; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]3500; GFX8-NEXT: s_cbranch_execnz .LBB58_13501; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end3502; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]3503; GFX8-NEXT: s_setpc_b64 s[30:31]3504;3505; GFX9-LABEL: flat_atomic_nand_i64_noret_offset__amdgpu_no_remote_memory:3506; GFX9: ; %bb.0:3507; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3508; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:323509; GFX9-NEXT: s_mov_b64 s[4:5], 03510; GFX9-NEXT: .LBB58_1: ; %atomicrmw.start3511; GFX9-NEXT: ; =>This Inner Loop Header: Depth=13512; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3513; GFX9-NEXT: v_and_b32_e32 v4, v7, v33514; GFX9-NEXT: v_and_b32_e32 v8, v6, v23515; GFX9-NEXT: v_not_b32_e32 v5, v43516; GFX9-NEXT: v_not_b32_e32 v4, v83517; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc3518; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3519; GFX9-NEXT: buffer_wbinvl1_vol3520; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]3521; GFX9-NEXT: v_mov_b32_e32 v7, v53522; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3523; GFX9-NEXT: v_mov_b32_e32 v6, v43524; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]3525; GFX9-NEXT: s_cbranch_execnz .LBB58_13526; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end3527; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]3528; GFX9-NEXT: s_setpc_b64 s[30:31]3529 %gep = getelementptr inbounds i64, ptr %out, i64 43530 %tmp0 = atomicrmw nand ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !13531 ret void3532}3533 3534define i64 @flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {3535; GFX7-LABEL: flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory:3536; GFX7: ; %bb.0:3537; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3538; GFX7-NEXT: v_add_i32_e32 v4, vcc, 32, v03539; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc3540; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v03541; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3542; GFX7-NEXT: flat_load_dword v1, v[0:1]3543; GFX7-NEXT: flat_load_dword v0, v[4:5]3544; GFX7-NEXT: s_mov_b64 s[4:5], 03545; GFX7-NEXT: .LBB59_1: ; %atomicrmw.start3546; GFX7-NEXT: ; =>This Inner Loop Header: Depth=13547; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3548; GFX7-NEXT: v_mov_b32_e32 v9, v13549; GFX7-NEXT: v_mov_b32_e32 v8, v03550; GFX7-NEXT: v_and_b32_e32 v0, v9, v33551; GFX7-NEXT: v_and_b32_e32 v1, v8, v23552; GFX7-NEXT: v_not_b32_e32 v7, v03553; GFX7-NEXT: v_not_b32_e32 v6, v13554; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc3555; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3556; GFX7-NEXT: buffer_wbinvl1_vol3557; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]3558; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3559; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]3560; GFX7-NEXT: s_cbranch_execnz .LBB59_13561; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end3562; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]3563; GFX7-NEXT: s_setpc_b64 s[30:31]3564;3565; GFX8-LABEL: flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory:3566; GFX8: ; %bb.0:3567; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3568; GFX8-NEXT: v_add_u32_e32 v4, vcc, 32, v03569; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc3570; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v03571; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3572; GFX8-NEXT: flat_load_dword v1, v[0:1]3573; GFX8-NEXT: flat_load_dword v0, v[4:5]3574; GFX8-NEXT: s_mov_b64 s[4:5], 03575; GFX8-NEXT: .LBB59_1: ; %atomicrmw.start3576; GFX8-NEXT: ; =>This Inner Loop Header: Depth=13577; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3578; GFX8-NEXT: v_mov_b32_e32 v9, v13579; GFX8-NEXT: v_mov_b32_e32 v8, v03580; GFX8-NEXT: v_and_b32_e32 v0, v9, v33581; GFX8-NEXT: v_and_b32_e32 v1, v8, v23582; GFX8-NEXT: v_not_b32_e32 v7, v03583; GFX8-NEXT: v_not_b32_e32 v6, v13584; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc3585; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3586; GFX8-NEXT: buffer_wbinvl1_vol3587; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]3588; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3589; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]3590; GFX8-NEXT: s_cbranch_execnz .LBB59_13591; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end3592; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]3593; GFX8-NEXT: s_setpc_b64 s[30:31]3594;3595; GFX9-LABEL: flat_atomic_nand_i64_ret_offset__amdgpu_no_remote_memory:3596; GFX9: ; %bb.0:3597; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3598; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:323599; GFX9-NEXT: s_mov_b64 s[4:5], 03600; GFX9-NEXT: .LBB59_1: ; %atomicrmw.start3601; GFX9-NEXT: ; =>This Inner Loop Header: Depth=13602; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3603; GFX9-NEXT: v_mov_b32_e32 v7, v53604; GFX9-NEXT: v_mov_b32_e32 v6, v43605; GFX9-NEXT: v_and_b32_e32 v4, v7, v33606; GFX9-NEXT: v_and_b32_e32 v8, v6, v23607; GFX9-NEXT: v_not_b32_e32 v5, v43608; GFX9-NEXT: v_not_b32_e32 v4, v83609; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc3610; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3611; GFX9-NEXT: buffer_wbinvl1_vol3612; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]3613; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3614; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]3615; GFX9-NEXT: s_cbranch_execnz .LBB59_13616; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end3617; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]3618; GFX9-NEXT: v_mov_b32_e32 v0, v43619; GFX9-NEXT: v_mov_b32_e32 v1, v53620; GFX9-NEXT: s_setpc_b64 s[30:31]3621 %gep = getelementptr inbounds i64, ptr %out, i64 43622 %result = atomicrmw nand ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !13623 ret i64 %result3624}3625 3626; ---------------------------------------------------------------------3627; atomicrmw or3628; ---------------------------------------------------------------------3629 3630define void @flat_atomic_or_i64_noret(ptr %ptr, i64 %in) {3631; GFX7-LABEL: flat_atomic_or_i64_noret:3632; GFX7: ; %bb.0:3633; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3634; GFX7-NEXT: v_add_i32_e32 v4, vcc, 4, v03635; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc3636; GFX7-NEXT: flat_load_dword v6, v[0:1]3637; GFX7-NEXT: flat_load_dword v7, v[4:5]3638; GFX7-NEXT: s_mov_b64 s[4:5], 03639; GFX7-NEXT: .LBB60_1: ; %atomicrmw.start3640; GFX7-NEXT: ; =>This Inner Loop Header: Depth=13641; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3642; GFX7-NEXT: v_or_b32_e32 v5, v7, v33643; GFX7-NEXT: v_or_b32_e32 v4, v6, v23644; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc3645; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3646; GFX7-NEXT: buffer_wbinvl1_vol3647; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]3648; GFX7-NEXT: v_mov_b32_e32 v7, v53649; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3650; GFX7-NEXT: v_mov_b32_e32 v6, v43651; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]3652; GFX7-NEXT: s_cbranch_execnz .LBB60_13653; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end3654; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]3655; GFX7-NEXT: s_setpc_b64 s[30:31]3656;3657; GFX8-LABEL: flat_atomic_or_i64_noret:3658; GFX8: ; %bb.0:3659; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3660; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v03661; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc3662; GFX8-NEXT: flat_load_dword v6, v[0:1]3663; GFX8-NEXT: flat_load_dword v7, v[4:5]3664; GFX8-NEXT: s_mov_b64 s[4:5], 03665; GFX8-NEXT: .LBB60_1: ; %atomicrmw.start3666; GFX8-NEXT: ; =>This Inner Loop Header: Depth=13667; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3668; GFX8-NEXT: v_or_b32_e32 v5, v7, v33669; GFX8-NEXT: v_or_b32_e32 v4, v6, v23670; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc3671; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3672; GFX8-NEXT: buffer_wbinvl1_vol3673; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]3674; GFX8-NEXT: v_mov_b32_e32 v7, v53675; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3676; GFX8-NEXT: v_mov_b32_e32 v6, v43677; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]3678; GFX8-NEXT: s_cbranch_execnz .LBB60_13679; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end3680; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]3681; GFX8-NEXT: s_setpc_b64 s[30:31]3682;3683; GFX9-LABEL: flat_atomic_or_i64_noret:3684; GFX9: ; %bb.0:3685; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3686; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1]3687; GFX9-NEXT: s_mov_b64 s[4:5], 03688; GFX9-NEXT: .LBB60_1: ; %atomicrmw.start3689; GFX9-NEXT: ; =>This Inner Loop Header: Depth=13690; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3691; GFX9-NEXT: v_or_b32_e32 v5, v7, v33692; GFX9-NEXT: v_or_b32_e32 v4, v6, v23693; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc3694; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3695; GFX9-NEXT: buffer_wbinvl1_vol3696; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]3697; GFX9-NEXT: v_mov_b32_e32 v7, v53698; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3699; GFX9-NEXT: v_mov_b32_e32 v6, v43700; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]3701; GFX9-NEXT: s_cbranch_execnz .LBB60_13702; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end3703; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]3704; GFX9-NEXT: s_setpc_b64 s[30:31]3705 %tmp0 = atomicrmw or ptr %ptr, i64 %in seq_cst, !noalias.addrspace !13706 ret void3707}3708 3709define void @flat_atomic_or_i64_noret_offset(ptr %out, i64 %in) {3710; GFX7-LABEL: flat_atomic_or_i64_noret_offset:3711; GFX7: ; %bb.0:3712; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3713; GFX7-NEXT: v_add_i32_e32 v8, vcc, 32, v03714; GFX7-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc3715; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v03716; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3717; GFX7-NEXT: flat_load_dword v7, v[0:1]3718; GFX7-NEXT: flat_load_dword v6, v[8:9]3719; GFX7-NEXT: s_mov_b64 s[4:5], 03720; GFX7-NEXT: .LBB61_1: ; %atomicrmw.start3721; GFX7-NEXT: ; =>This Inner Loop Header: Depth=13722; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3723; GFX7-NEXT: v_or_b32_e32 v5, v7, v33724; GFX7-NEXT: v_or_b32_e32 v4, v6, v23725; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc3726; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3727; GFX7-NEXT: buffer_wbinvl1_vol3728; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]3729; GFX7-NEXT: v_mov_b32_e32 v7, v13730; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3731; GFX7-NEXT: v_mov_b32_e32 v6, v03732; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]3733; GFX7-NEXT: s_cbranch_execnz .LBB61_13734; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end3735; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]3736; GFX7-NEXT: s_setpc_b64 s[30:31]3737;3738; GFX8-LABEL: flat_atomic_or_i64_noret_offset:3739; GFX8: ; %bb.0:3740; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3741; GFX8-NEXT: v_add_u32_e32 v8, vcc, 32, v03742; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc3743; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v03744; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3745; GFX8-NEXT: flat_load_dword v7, v[0:1]3746; GFX8-NEXT: flat_load_dword v6, v[8:9]3747; GFX8-NEXT: s_mov_b64 s[4:5], 03748; GFX8-NEXT: .LBB61_1: ; %atomicrmw.start3749; GFX8-NEXT: ; =>This Inner Loop Header: Depth=13750; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3751; GFX8-NEXT: v_or_b32_e32 v5, v7, v33752; GFX8-NEXT: v_or_b32_e32 v4, v6, v23753; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc3754; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3755; GFX8-NEXT: buffer_wbinvl1_vol3756; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]3757; GFX8-NEXT: v_mov_b32_e32 v7, v13758; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3759; GFX8-NEXT: v_mov_b32_e32 v6, v03760; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]3761; GFX8-NEXT: s_cbranch_execnz .LBB61_13762; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end3763; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]3764; GFX8-NEXT: s_setpc_b64 s[30:31]3765;3766; GFX9-LABEL: flat_atomic_or_i64_noret_offset:3767; GFX9: ; %bb.0:3768; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3769; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:323770; GFX9-NEXT: s_mov_b64 s[4:5], 03771; GFX9-NEXT: .LBB61_1: ; %atomicrmw.start3772; GFX9-NEXT: ; =>This Inner Loop Header: Depth=13773; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3774; GFX9-NEXT: v_or_b32_e32 v5, v7, v33775; GFX9-NEXT: v_or_b32_e32 v4, v6, v23776; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc3777; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3778; GFX9-NEXT: buffer_wbinvl1_vol3779; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]3780; GFX9-NEXT: v_mov_b32_e32 v7, v53781; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3782; GFX9-NEXT: v_mov_b32_e32 v6, v43783; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]3784; GFX9-NEXT: s_cbranch_execnz .LBB61_13785; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end3786; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]3787; GFX9-NEXT: s_setpc_b64 s[30:31]3788 %gep = getelementptr inbounds i64, ptr %out, i64 43789 %tmp0 = atomicrmw or ptr %gep, i64 %in seq_cst, !noalias.addrspace !13790 ret void3791}3792 3793define i64 @flat_atomic_or_i64_ret(ptr %ptr, i64 %in) {3794; GFX7-LABEL: flat_atomic_or_i64_ret:3795; GFX7: ; %bb.0:3796; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3797; GFX7-NEXT: v_add_i32_e32 v5, vcc, 4, v03798; GFX7-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc3799; GFX7-NEXT: flat_load_dword v4, v[0:1]3800; GFX7-NEXT: flat_load_dword v5, v[5:6]3801; GFX7-NEXT: s_mov_b64 s[4:5], 03802; GFX7-NEXT: .LBB62_1: ; %atomicrmw.start3803; GFX7-NEXT: ; =>This Inner Loop Header: Depth=13804; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3805; GFX7-NEXT: v_mov_b32_e32 v7, v53806; GFX7-NEXT: v_mov_b32_e32 v6, v43807; GFX7-NEXT: v_or_b32_e32 v5, v7, v33808; GFX7-NEXT: v_or_b32_e32 v4, v6, v23809; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc3810; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3811; GFX7-NEXT: buffer_wbinvl1_vol3812; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]3813; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3814; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]3815; GFX7-NEXT: s_cbranch_execnz .LBB62_13816; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end3817; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]3818; GFX7-NEXT: v_mov_b32_e32 v0, v43819; GFX7-NEXT: v_mov_b32_e32 v1, v53820; GFX7-NEXT: s_setpc_b64 s[30:31]3821;3822; GFX8-LABEL: flat_atomic_or_i64_ret:3823; GFX8: ; %bb.0:3824; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3825; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v03826; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc3827; GFX8-NEXT: flat_load_dword v4, v[0:1]3828; GFX8-NEXT: flat_load_dword v5, v[5:6]3829; GFX8-NEXT: s_mov_b64 s[4:5], 03830; GFX8-NEXT: .LBB62_1: ; %atomicrmw.start3831; GFX8-NEXT: ; =>This Inner Loop Header: Depth=13832; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3833; GFX8-NEXT: v_mov_b32_e32 v7, v53834; GFX8-NEXT: v_mov_b32_e32 v6, v43835; GFX8-NEXT: v_or_b32_e32 v5, v7, v33836; GFX8-NEXT: v_or_b32_e32 v4, v6, v23837; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc3838; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3839; GFX8-NEXT: buffer_wbinvl1_vol3840; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]3841; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3842; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]3843; GFX8-NEXT: s_cbranch_execnz .LBB62_13844; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end3845; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]3846; GFX8-NEXT: v_mov_b32_e32 v0, v43847; GFX8-NEXT: v_mov_b32_e32 v1, v53848; GFX8-NEXT: s_setpc_b64 s[30:31]3849;3850; GFX9-LABEL: flat_atomic_or_i64_ret:3851; GFX9: ; %bb.0:3852; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3853; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1]3854; GFX9-NEXT: s_mov_b64 s[4:5], 03855; GFX9-NEXT: .LBB62_1: ; %atomicrmw.start3856; GFX9-NEXT: ; =>This Inner Loop Header: Depth=13857; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3858; GFX9-NEXT: v_mov_b32_e32 v7, v53859; GFX9-NEXT: v_mov_b32_e32 v6, v43860; GFX9-NEXT: v_or_b32_e32 v5, v7, v33861; GFX9-NEXT: v_or_b32_e32 v4, v6, v23862; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc3863; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3864; GFX9-NEXT: buffer_wbinvl1_vol3865; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]3866; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3867; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]3868; GFX9-NEXT: s_cbranch_execnz .LBB62_13869; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end3870; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]3871; GFX9-NEXT: v_mov_b32_e32 v0, v43872; GFX9-NEXT: v_mov_b32_e32 v1, v53873; GFX9-NEXT: s_setpc_b64 s[30:31]3874 %result = atomicrmw or ptr %ptr, i64 %in seq_cst, !noalias.addrspace !13875 ret i64 %result3876}3877 3878define i64 @flat_atomic_or_i64_ret_offset(ptr %out, i64 %in) {3879; GFX7-LABEL: flat_atomic_or_i64_ret_offset:3880; GFX7: ; %bb.0:3881; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3882; GFX7-NEXT: v_add_i32_e32 v4, vcc, 32, v03883; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc3884; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v03885; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3886; GFX7-NEXT: flat_load_dword v1, v[0:1]3887; GFX7-NEXT: flat_load_dword v0, v[4:5]3888; GFX7-NEXT: s_mov_b64 s[4:5], 03889; GFX7-NEXT: .LBB63_1: ; %atomicrmw.start3890; GFX7-NEXT: ; =>This Inner Loop Header: Depth=13891; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3892; GFX7-NEXT: v_mov_b32_e32 v9, v13893; GFX7-NEXT: v_mov_b32_e32 v8, v03894; GFX7-NEXT: v_or_b32_e32 v7, v9, v33895; GFX7-NEXT: v_or_b32_e32 v6, v8, v23896; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc3897; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3898; GFX7-NEXT: buffer_wbinvl1_vol3899; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]3900; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3901; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]3902; GFX7-NEXT: s_cbranch_execnz .LBB63_13903; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end3904; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]3905; GFX7-NEXT: s_setpc_b64 s[30:31]3906;3907; GFX8-LABEL: flat_atomic_or_i64_ret_offset:3908; GFX8: ; %bb.0:3909; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3910; GFX8-NEXT: v_add_u32_e32 v4, vcc, 32, v03911; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc3912; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v03913; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc3914; GFX8-NEXT: flat_load_dword v1, v[0:1]3915; GFX8-NEXT: flat_load_dword v0, v[4:5]3916; GFX8-NEXT: s_mov_b64 s[4:5], 03917; GFX8-NEXT: .LBB63_1: ; %atomicrmw.start3918; GFX8-NEXT: ; =>This Inner Loop Header: Depth=13919; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3920; GFX8-NEXT: v_mov_b32_e32 v9, v13921; GFX8-NEXT: v_mov_b32_e32 v8, v03922; GFX8-NEXT: v_or_b32_e32 v7, v9, v33923; GFX8-NEXT: v_or_b32_e32 v6, v8, v23924; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc3925; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3926; GFX8-NEXT: buffer_wbinvl1_vol3927; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]3928; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3929; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]3930; GFX8-NEXT: s_cbranch_execnz .LBB63_13931; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end3932; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]3933; GFX8-NEXT: s_setpc_b64 s[30:31]3934;3935; GFX9-LABEL: flat_atomic_or_i64_ret_offset:3936; GFX9: ; %bb.0:3937; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3938; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:323939; GFX9-NEXT: s_mov_b64 s[4:5], 03940; GFX9-NEXT: .LBB63_1: ; %atomicrmw.start3941; GFX9-NEXT: ; =>This Inner Loop Header: Depth=13942; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3943; GFX9-NEXT: v_mov_b32_e32 v7, v53944; GFX9-NEXT: v_mov_b32_e32 v6, v43945; GFX9-NEXT: v_or_b32_e32 v5, v7, v33946; GFX9-NEXT: v_or_b32_e32 v4, v6, v23947; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc3948; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3949; GFX9-NEXT: buffer_wbinvl1_vol3950; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]3951; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3952; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]3953; GFX9-NEXT: s_cbranch_execnz .LBB63_13954; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end3955; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]3956; GFX9-NEXT: v_mov_b32_e32 v0, v43957; GFX9-NEXT: v_mov_b32_e32 v1, v53958; GFX9-NEXT: s_setpc_b64 s[30:31]3959 %gep = getelementptr inbounds i64, ptr %out, i64 43960 %result = atomicrmw or ptr %gep, i64 %in seq_cst, !noalias.addrspace !13961 ret i64 %result3962}3963 3964define amdgpu_gfx void @flat_atomic_or_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {3965; GFX7-LABEL: flat_atomic_or_i64_noret_scalar:3966; GFX7: ; %bb.0:3967; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3968; GFX7-NEXT: v_mov_b32_e32 v0, s43969; GFX7-NEXT: s_add_u32 s34, s4, 43970; GFX7-NEXT: v_mov_b32_e32 v1, s53971; GFX7-NEXT: s_addc_u32 s35, s5, 03972; GFX7-NEXT: v_mov_b32_e32 v3, s343973; GFX7-NEXT: v_mov_b32_e32 v4, s353974; GFX7-NEXT: flat_load_dword v2, v[0:1]3975; GFX7-NEXT: flat_load_dword v3, v[3:4]3976; GFX7-NEXT: v_mov_b32_e32 v4, s43977; GFX7-NEXT: s_mov_b64 s[34:35], 03978; GFX7-NEXT: v_mov_b32_e32 v5, s53979; GFX7-NEXT: .LBB64_1: ; %atomicrmw.start3980; GFX7-NEXT: ; =>This Inner Loop Header: Depth=13981; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3982; GFX7-NEXT: v_or_b32_e32 v1, s7, v33983; GFX7-NEXT: v_or_b32_e32 v0, s6, v23984; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc3985; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3986; GFX7-NEXT: buffer_wbinvl1_vol3987; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]3988; GFX7-NEXT: v_mov_b32_e32 v3, v13989; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]3990; GFX7-NEXT: v_mov_b32_e32 v2, v03991; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]3992; GFX7-NEXT: s_cbranch_execnz .LBB64_13993; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end3994; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]3995; GFX7-NEXT: s_setpc_b64 s[30:31]3996;3997; GFX8-LABEL: flat_atomic_or_i64_noret_scalar:3998; GFX8: ; %bb.0:3999; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4000; GFX8-NEXT: v_mov_b32_e32 v0, s44001; GFX8-NEXT: s_add_u32 s34, s4, 44002; GFX8-NEXT: v_mov_b32_e32 v1, s54003; GFX8-NEXT: s_addc_u32 s35, s5, 04004; GFX8-NEXT: v_mov_b32_e32 v3, s344005; GFX8-NEXT: v_mov_b32_e32 v4, s354006; GFX8-NEXT: flat_load_dword v2, v[0:1]4007; GFX8-NEXT: flat_load_dword v3, v[3:4]4008; GFX8-NEXT: v_mov_b32_e32 v4, s44009; GFX8-NEXT: s_mov_b64 s[34:35], 04010; GFX8-NEXT: v_mov_b32_e32 v5, s54011; GFX8-NEXT: .LBB64_1: ; %atomicrmw.start4012; GFX8-NEXT: ; =>This Inner Loop Header: Depth=14013; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4014; GFX8-NEXT: v_or_b32_e32 v1, s7, v34015; GFX8-NEXT: v_or_b32_e32 v0, s6, v24016; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc4017; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4018; GFX8-NEXT: buffer_wbinvl1_vol4019; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4020; GFX8-NEXT: v_mov_b32_e32 v3, v14021; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4022; GFX8-NEXT: v_mov_b32_e32 v2, v04023; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]4024; GFX8-NEXT: s_cbranch_execnz .LBB64_14025; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end4026; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]4027; GFX8-NEXT: s_setpc_b64 s[30:31]4028;4029; GFX9-LABEL: flat_atomic_or_i64_noret_scalar:4030; GFX9: ; %bb.0:4031; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4032; GFX9-NEXT: v_mov_b32_e32 v0, s44033; GFX9-NEXT: v_mov_b32_e32 v1, s54034; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]4035; GFX9-NEXT: v_mov_b32_e32 v4, s44036; GFX9-NEXT: s_mov_b64 s[34:35], 04037; GFX9-NEXT: v_mov_b32_e32 v5, s54038; GFX9-NEXT: .LBB64_1: ; %atomicrmw.start4039; GFX9-NEXT: ; =>This Inner Loop Header: Depth=14040; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4041; GFX9-NEXT: v_or_b32_e32 v1, s7, v34042; GFX9-NEXT: v_or_b32_e32 v0, s6, v24043; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc4044; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4045; GFX9-NEXT: buffer_wbinvl1_vol4046; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4047; GFX9-NEXT: v_mov_b32_e32 v3, v14048; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4049; GFX9-NEXT: v_mov_b32_e32 v2, v04050; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]4051; GFX9-NEXT: s_cbranch_execnz .LBB64_14052; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end4053; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]4054; GFX9-NEXT: s_setpc_b64 s[30:31]4055 %tmp0 = atomicrmw or ptr %ptr, i64 %in seq_cst, !noalias.addrspace !14056 ret void4057}4058 4059define amdgpu_gfx void @flat_atomic_or_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {4060; GFX7-LABEL: flat_atomic_or_i64_noret_offset_scalar:4061; GFX7: ; %bb.0:4062; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4063; GFX7-NEXT: s_add_u32 s34, s4, 324064; GFX7-NEXT: s_addc_u32 s35, s5, 04065; GFX7-NEXT: s_add_u32 s36, s4, 364066; GFX7-NEXT: s_addc_u32 s37, s5, 04067; GFX7-NEXT: v_mov_b32_e32 v0, s364068; GFX7-NEXT: v_mov_b32_e32 v1, s374069; GFX7-NEXT: v_mov_b32_e32 v4, s344070; GFX7-NEXT: v_mov_b32_e32 v5, s354071; GFX7-NEXT: flat_load_dword v3, v[0:1]4072; GFX7-NEXT: flat_load_dword v2, v[4:5]4073; GFX7-NEXT: s_mov_b64 s[34:35], 04074; GFX7-NEXT: .LBB65_1: ; %atomicrmw.start4075; GFX7-NEXT: ; =>This Inner Loop Header: Depth=14076; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4077; GFX7-NEXT: v_or_b32_e32 v1, s7, v34078; GFX7-NEXT: v_or_b32_e32 v0, s6, v24079; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc4080; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4081; GFX7-NEXT: buffer_wbinvl1_vol4082; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4083; GFX7-NEXT: v_mov_b32_e32 v3, v14084; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4085; GFX7-NEXT: v_mov_b32_e32 v2, v04086; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]4087; GFX7-NEXT: s_cbranch_execnz .LBB65_14088; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end4089; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]4090; GFX7-NEXT: s_setpc_b64 s[30:31]4091;4092; GFX8-LABEL: flat_atomic_or_i64_noret_offset_scalar:4093; GFX8: ; %bb.0:4094; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4095; GFX8-NEXT: s_add_u32 s34, s4, 324096; GFX8-NEXT: s_addc_u32 s35, s5, 04097; GFX8-NEXT: s_add_u32 s36, s4, 364098; GFX8-NEXT: s_addc_u32 s37, s5, 04099; GFX8-NEXT: v_mov_b32_e32 v0, s364100; GFX8-NEXT: v_mov_b32_e32 v1, s374101; GFX8-NEXT: v_mov_b32_e32 v4, s344102; GFX8-NEXT: v_mov_b32_e32 v5, s354103; GFX8-NEXT: flat_load_dword v3, v[0:1]4104; GFX8-NEXT: flat_load_dword v2, v[4:5]4105; GFX8-NEXT: s_mov_b64 s[34:35], 04106; GFX8-NEXT: .LBB65_1: ; %atomicrmw.start4107; GFX8-NEXT: ; =>This Inner Loop Header: Depth=14108; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4109; GFX8-NEXT: v_or_b32_e32 v1, s7, v34110; GFX8-NEXT: v_or_b32_e32 v0, s6, v24111; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc4112; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4113; GFX8-NEXT: buffer_wbinvl1_vol4114; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4115; GFX8-NEXT: v_mov_b32_e32 v3, v14116; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4117; GFX8-NEXT: v_mov_b32_e32 v2, v04118; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]4119; GFX8-NEXT: s_cbranch_execnz .LBB65_14120; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end4121; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]4122; GFX8-NEXT: s_setpc_b64 s[30:31]4123;4124; GFX9-LABEL: flat_atomic_or_i64_noret_offset_scalar:4125; GFX9: ; %bb.0:4126; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4127; GFX9-NEXT: v_mov_b32_e32 v0, s44128; GFX9-NEXT: v_mov_b32_e32 v1, s54129; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:324130; GFX9-NEXT: v_mov_b32_e32 v4, s44131; GFX9-NEXT: s_mov_b64 s[34:35], 04132; GFX9-NEXT: v_mov_b32_e32 v5, s54133; GFX9-NEXT: .LBB65_1: ; %atomicrmw.start4134; GFX9-NEXT: ; =>This Inner Loop Header: Depth=14135; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4136; GFX9-NEXT: v_or_b32_e32 v1, s7, v34137; GFX9-NEXT: v_or_b32_e32 v0, s6, v24138; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc4139; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4140; GFX9-NEXT: buffer_wbinvl1_vol4141; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4142; GFX9-NEXT: v_mov_b32_e32 v3, v14143; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4144; GFX9-NEXT: v_mov_b32_e32 v2, v04145; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]4146; GFX9-NEXT: s_cbranch_execnz .LBB65_14147; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end4148; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]4149; GFX9-NEXT: s_setpc_b64 s[30:31]4150 %gep = getelementptr inbounds i64, ptr %out, i64 44151 %tmp0 = atomicrmw or ptr %gep, i64 %in seq_cst, !noalias.addrspace !14152 ret void4153}4154 4155define amdgpu_gfx i64 @flat_atomic_or_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {4156; GFX7-LABEL: flat_atomic_or_i64_ret_scalar:4157; GFX7: ; %bb.0:4158; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4159; GFX7-NEXT: v_mov_b32_e32 v0, s44160; GFX7-NEXT: s_add_u32 s34, s4, 44161; GFX7-NEXT: v_mov_b32_e32 v1, s54162; GFX7-NEXT: s_addc_u32 s35, s5, 04163; GFX7-NEXT: v_mov_b32_e32 v2, s344164; GFX7-NEXT: v_mov_b32_e32 v3, s354165; GFX7-NEXT: flat_load_dword v0, v[0:1]4166; GFX7-NEXT: flat_load_dword v1, v[2:3]4167; GFX7-NEXT: v_mov_b32_e32 v2, s44168; GFX7-NEXT: s_mov_b64 s[34:35], 04169; GFX7-NEXT: v_mov_b32_e32 v3, s54170; GFX7-NEXT: .LBB66_1: ; %atomicrmw.start4171; GFX7-NEXT: ; =>This Inner Loop Header: Depth=14172; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4173; GFX7-NEXT: v_mov_b32_e32 v7, v14174; GFX7-NEXT: v_mov_b32_e32 v6, v04175; GFX7-NEXT: v_or_b32_e32 v5, s7, v74176; GFX7-NEXT: v_or_b32_e32 v4, s6, v64177; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc4178; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4179; GFX7-NEXT: buffer_wbinvl1_vol4180; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]4181; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4182; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]4183; GFX7-NEXT: s_cbranch_execnz .LBB66_14184; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end4185; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]4186; GFX7-NEXT: s_setpc_b64 s[30:31]4187;4188; GFX8-LABEL: flat_atomic_or_i64_ret_scalar:4189; GFX8: ; %bb.0:4190; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4191; GFX8-NEXT: v_mov_b32_e32 v0, s44192; GFX8-NEXT: s_add_u32 s34, s4, 44193; GFX8-NEXT: v_mov_b32_e32 v1, s54194; GFX8-NEXT: s_addc_u32 s35, s5, 04195; GFX8-NEXT: v_mov_b32_e32 v2, s344196; GFX8-NEXT: v_mov_b32_e32 v3, s354197; GFX8-NEXT: flat_load_dword v0, v[0:1]4198; GFX8-NEXT: flat_load_dword v1, v[2:3]4199; GFX8-NEXT: v_mov_b32_e32 v2, s44200; GFX8-NEXT: s_mov_b64 s[34:35], 04201; GFX8-NEXT: v_mov_b32_e32 v3, s54202; GFX8-NEXT: .LBB66_1: ; %atomicrmw.start4203; GFX8-NEXT: ; =>This Inner Loop Header: Depth=14204; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4205; GFX8-NEXT: v_mov_b32_e32 v7, v14206; GFX8-NEXT: v_mov_b32_e32 v6, v04207; GFX8-NEXT: v_or_b32_e32 v5, s7, v74208; GFX8-NEXT: v_or_b32_e32 v4, s6, v64209; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc4210; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4211; GFX8-NEXT: buffer_wbinvl1_vol4212; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]4213; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4214; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]4215; GFX8-NEXT: s_cbranch_execnz .LBB66_14216; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end4217; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]4218; GFX8-NEXT: s_setpc_b64 s[30:31]4219;4220; GFX9-LABEL: flat_atomic_or_i64_ret_scalar:4221; GFX9: ; %bb.0:4222; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4223; GFX9-NEXT: v_mov_b32_e32 v0, s44224; GFX9-NEXT: v_mov_b32_e32 v1, s54225; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]4226; GFX9-NEXT: v_mov_b32_e32 v2, s44227; GFX9-NEXT: s_mov_b64 s[34:35], 04228; GFX9-NEXT: v_mov_b32_e32 v3, s54229; GFX9-NEXT: .LBB66_1: ; %atomicrmw.start4230; GFX9-NEXT: ; =>This Inner Loop Header: Depth=14231; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4232; GFX9-NEXT: v_mov_b32_e32 v7, v14233; GFX9-NEXT: v_mov_b32_e32 v6, v04234; GFX9-NEXT: v_or_b32_e32 v5, s7, v74235; GFX9-NEXT: v_or_b32_e32 v4, s6, v64236; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc4237; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4238; GFX9-NEXT: buffer_wbinvl1_vol4239; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]4240; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4241; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]4242; GFX9-NEXT: s_cbranch_execnz .LBB66_14243; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end4244; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]4245; GFX9-NEXT: s_setpc_b64 s[30:31]4246 %result = atomicrmw or ptr %ptr, i64 %in seq_cst, !noalias.addrspace !14247 ret i64 %result4248}4249 4250define amdgpu_gfx i64 @flat_atomic_or_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {4251; GFX7-LABEL: flat_atomic_or_i64_ret_offset_scalar:4252; GFX7: ; %bb.0:4253; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4254; GFX7-NEXT: s_add_u32 s34, s4, 324255; GFX7-NEXT: s_addc_u32 s35, s5, 04256; GFX7-NEXT: s_add_u32 s36, s4, 364257; GFX7-NEXT: s_addc_u32 s37, s5, 04258; GFX7-NEXT: v_mov_b32_e32 v0, s364259; GFX7-NEXT: v_mov_b32_e32 v1, s374260; GFX7-NEXT: v_mov_b32_e32 v2, s344261; GFX7-NEXT: v_mov_b32_e32 v3, s354262; GFX7-NEXT: flat_load_dword v1, v[0:1]4263; GFX7-NEXT: flat_load_dword v0, v[2:3]4264; GFX7-NEXT: s_mov_b64 s[34:35], 04265; GFX7-NEXT: .LBB67_1: ; %atomicrmw.start4266; GFX7-NEXT: ; =>This Inner Loop Header: Depth=14267; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4268; GFX7-NEXT: v_mov_b32_e32 v7, v14269; GFX7-NEXT: v_mov_b32_e32 v6, v04270; GFX7-NEXT: v_or_b32_e32 v5, s7, v74271; GFX7-NEXT: v_or_b32_e32 v4, s6, v64272; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc4273; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4274; GFX7-NEXT: buffer_wbinvl1_vol4275; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]4276; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4277; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]4278; GFX7-NEXT: s_cbranch_execnz .LBB67_14279; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end4280; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]4281; GFX7-NEXT: s_setpc_b64 s[30:31]4282;4283; GFX8-LABEL: flat_atomic_or_i64_ret_offset_scalar:4284; GFX8: ; %bb.0:4285; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4286; GFX8-NEXT: s_add_u32 s34, s4, 324287; GFX8-NEXT: s_addc_u32 s35, s5, 04288; GFX8-NEXT: s_add_u32 s36, s4, 364289; GFX8-NEXT: s_addc_u32 s37, s5, 04290; GFX8-NEXT: v_mov_b32_e32 v0, s364291; GFX8-NEXT: v_mov_b32_e32 v1, s374292; GFX8-NEXT: v_mov_b32_e32 v2, s344293; GFX8-NEXT: v_mov_b32_e32 v3, s354294; GFX8-NEXT: flat_load_dword v1, v[0:1]4295; GFX8-NEXT: flat_load_dword v0, v[2:3]4296; GFX8-NEXT: s_mov_b64 s[34:35], 04297; GFX8-NEXT: .LBB67_1: ; %atomicrmw.start4298; GFX8-NEXT: ; =>This Inner Loop Header: Depth=14299; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4300; GFX8-NEXT: v_mov_b32_e32 v7, v14301; GFX8-NEXT: v_mov_b32_e32 v6, v04302; GFX8-NEXT: v_or_b32_e32 v5, s7, v74303; GFX8-NEXT: v_or_b32_e32 v4, s6, v64304; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc4305; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4306; GFX8-NEXT: buffer_wbinvl1_vol4307; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]4308; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4309; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]4310; GFX8-NEXT: s_cbranch_execnz .LBB67_14311; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end4312; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]4313; GFX8-NEXT: s_setpc_b64 s[30:31]4314;4315; GFX9-LABEL: flat_atomic_or_i64_ret_offset_scalar:4316; GFX9: ; %bb.0:4317; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4318; GFX9-NEXT: v_mov_b32_e32 v0, s44319; GFX9-NEXT: v_mov_b32_e32 v1, s54320; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:324321; GFX9-NEXT: v_mov_b32_e32 v2, s44322; GFX9-NEXT: s_mov_b64 s[34:35], 04323; GFX9-NEXT: v_mov_b32_e32 v3, s54324; GFX9-NEXT: .LBB67_1: ; %atomicrmw.start4325; GFX9-NEXT: ; =>This Inner Loop Header: Depth=14326; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4327; GFX9-NEXT: v_mov_b32_e32 v7, v14328; GFX9-NEXT: v_mov_b32_e32 v6, v04329; GFX9-NEXT: v_or_b32_e32 v5, s7, v74330; GFX9-NEXT: v_or_b32_e32 v4, s6, v64331; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] offset:32 glc4332; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4333; GFX9-NEXT: buffer_wbinvl1_vol4334; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]4335; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4336; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]4337; GFX9-NEXT: s_cbranch_execnz .LBB67_14338; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end4339; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]4340; GFX9-NEXT: s_setpc_b64 s[30:31]4341 %gep = getelementptr inbounds i64, ptr %out, i64 44342 %result = atomicrmw or ptr %gep, i64 %in seq_cst, !noalias.addrspace !14343 ret i64 %result4344}4345 4346define void @flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {4347; GFX7-LABEL: flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory:4348; GFX7: ; %bb.0:4349; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4350; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v04351; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc4352; GFX7-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]4353; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4354; GFX7-NEXT: buffer_wbinvl1_vol4355; GFX7-NEXT: s_setpc_b64 s[30:31]4356;4357; GFX8-LABEL: flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory:4358; GFX8: ; %bb.0:4359; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4360; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v04361; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc4362; GFX8-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]4363; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4364; GFX8-NEXT: buffer_wbinvl1_vol4365; GFX8-NEXT: s_setpc_b64 s[30:31]4366;4367; GFX9-LABEL: flat_atomic_or_i64_noret_offset__amdgpu_no_remote_memory:4368; GFX9: ; %bb.0:4369; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4370; GFX9-NEXT: flat_atomic_or_x2 v[0:1], v[2:3] offset:324371; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4372; GFX9-NEXT: buffer_wbinvl1_vol4373; GFX9-NEXT: s_setpc_b64 s[30:31]4374 %gep = getelementptr inbounds i64, ptr %out, i64 44375 %tmp0 = atomicrmw or ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !14376 ret void4377}4378 4379define i64 @flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {4380; GFX7-LABEL: flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory:4381; GFX7: ; %bb.0:4382; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4383; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v04384; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc4385; GFX7-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc4386; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4387; GFX7-NEXT: buffer_wbinvl1_vol4388; GFX7-NEXT: s_setpc_b64 s[30:31]4389;4390; GFX8-LABEL: flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory:4391; GFX8: ; %bb.0:4392; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4393; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v04394; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc4395; GFX8-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc4396; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4397; GFX8-NEXT: buffer_wbinvl1_vol4398; GFX8-NEXT: s_setpc_b64 s[30:31]4399;4400; GFX9-LABEL: flat_atomic_or_i64_ret_offset__amdgpu_no_remote_memory:4401; GFX9: ; %bb.0:4402; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4403; GFX9-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] offset:32 glc4404; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4405; GFX9-NEXT: buffer_wbinvl1_vol4406; GFX9-NEXT: s_setpc_b64 s[30:31]4407 %gep = getelementptr inbounds i64, ptr %out, i64 44408 %result = atomicrmw or ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !14409 ret i64 %result4410}4411 4412; ---------------------------------------------------------------------4413; atomicrmw xor4414; ---------------------------------------------------------------------4415 4416define void @flat_atomic_xor_i64_noret(ptr %ptr, i64 %in) {4417; GFX7-LABEL: flat_atomic_xor_i64_noret:4418; GFX7: ; %bb.0:4419; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4420; GFX7-NEXT: v_add_i32_e32 v4, vcc, 4, v04421; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc4422; GFX7-NEXT: flat_load_dword v6, v[0:1]4423; GFX7-NEXT: flat_load_dword v7, v[4:5]4424; GFX7-NEXT: s_mov_b64 s[4:5], 04425; GFX7-NEXT: .LBB70_1: ; %atomicrmw.start4426; GFX7-NEXT: ; =>This Inner Loop Header: Depth=14427; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4428; GFX7-NEXT: v_xor_b32_e32 v5, v7, v34429; GFX7-NEXT: v_xor_b32_e32 v4, v6, v24430; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc4431; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4432; GFX7-NEXT: buffer_wbinvl1_vol4433; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]4434; GFX7-NEXT: v_mov_b32_e32 v7, v54435; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4436; GFX7-NEXT: v_mov_b32_e32 v6, v44437; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]4438; GFX7-NEXT: s_cbranch_execnz .LBB70_14439; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end4440; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]4441; GFX7-NEXT: s_setpc_b64 s[30:31]4442;4443; GFX8-LABEL: flat_atomic_xor_i64_noret:4444; GFX8: ; %bb.0:4445; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4446; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v04447; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc4448; GFX8-NEXT: flat_load_dword v6, v[0:1]4449; GFX8-NEXT: flat_load_dword v7, v[4:5]4450; GFX8-NEXT: s_mov_b64 s[4:5], 04451; GFX8-NEXT: .LBB70_1: ; %atomicrmw.start4452; GFX8-NEXT: ; =>This Inner Loop Header: Depth=14453; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4454; GFX8-NEXT: v_xor_b32_e32 v5, v7, v34455; GFX8-NEXT: v_xor_b32_e32 v4, v6, v24456; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc4457; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4458; GFX8-NEXT: buffer_wbinvl1_vol4459; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]4460; GFX8-NEXT: v_mov_b32_e32 v7, v54461; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4462; GFX8-NEXT: v_mov_b32_e32 v6, v44463; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]4464; GFX8-NEXT: s_cbranch_execnz .LBB70_14465; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end4466; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]4467; GFX8-NEXT: s_setpc_b64 s[30:31]4468;4469; GFX9-LABEL: flat_atomic_xor_i64_noret:4470; GFX9: ; %bb.0:4471; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4472; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1]4473; GFX9-NEXT: s_mov_b64 s[4:5], 04474; GFX9-NEXT: .LBB70_1: ; %atomicrmw.start4475; GFX9-NEXT: ; =>This Inner Loop Header: Depth=14476; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4477; GFX9-NEXT: v_xor_b32_e32 v5, v7, v34478; GFX9-NEXT: v_xor_b32_e32 v4, v6, v24479; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc4480; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4481; GFX9-NEXT: buffer_wbinvl1_vol4482; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]4483; GFX9-NEXT: v_mov_b32_e32 v7, v54484; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4485; GFX9-NEXT: v_mov_b32_e32 v6, v44486; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]4487; GFX9-NEXT: s_cbranch_execnz .LBB70_14488; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end4489; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]4490; GFX9-NEXT: s_setpc_b64 s[30:31]4491 %tmp0 = atomicrmw xor ptr %ptr, i64 %in seq_cst, !noalias.addrspace !14492 ret void4493}4494 4495define void @flat_atomic_xor_i64_noret_offset(ptr %out, i64 %in) {4496; GFX7-LABEL: flat_atomic_xor_i64_noret_offset:4497; GFX7: ; %bb.0:4498; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4499; GFX7-NEXT: v_add_i32_e32 v8, vcc, 32, v04500; GFX7-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc4501; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v04502; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc4503; GFX7-NEXT: flat_load_dword v7, v[0:1]4504; GFX7-NEXT: flat_load_dword v6, v[8:9]4505; GFX7-NEXT: s_mov_b64 s[4:5], 04506; GFX7-NEXT: .LBB71_1: ; %atomicrmw.start4507; GFX7-NEXT: ; =>This Inner Loop Header: Depth=14508; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4509; GFX7-NEXT: v_xor_b32_e32 v5, v7, v34510; GFX7-NEXT: v_xor_b32_e32 v4, v6, v24511; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc4512; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4513; GFX7-NEXT: buffer_wbinvl1_vol4514; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]4515; GFX7-NEXT: v_mov_b32_e32 v7, v14516; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4517; GFX7-NEXT: v_mov_b32_e32 v6, v04518; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]4519; GFX7-NEXT: s_cbranch_execnz .LBB71_14520; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end4521; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]4522; GFX7-NEXT: s_setpc_b64 s[30:31]4523;4524; GFX8-LABEL: flat_atomic_xor_i64_noret_offset:4525; GFX8: ; %bb.0:4526; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4527; GFX8-NEXT: v_add_u32_e32 v8, vcc, 32, v04528; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc4529; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v04530; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc4531; GFX8-NEXT: flat_load_dword v7, v[0:1]4532; GFX8-NEXT: flat_load_dword v6, v[8:9]4533; GFX8-NEXT: s_mov_b64 s[4:5], 04534; GFX8-NEXT: .LBB71_1: ; %atomicrmw.start4535; GFX8-NEXT: ; =>This Inner Loop Header: Depth=14536; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4537; GFX8-NEXT: v_xor_b32_e32 v5, v7, v34538; GFX8-NEXT: v_xor_b32_e32 v4, v6, v24539; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc4540; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4541; GFX8-NEXT: buffer_wbinvl1_vol4542; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]4543; GFX8-NEXT: v_mov_b32_e32 v7, v14544; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4545; GFX8-NEXT: v_mov_b32_e32 v6, v04546; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]4547; GFX8-NEXT: s_cbranch_execnz .LBB71_14548; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end4549; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]4550; GFX8-NEXT: s_setpc_b64 s[30:31]4551;4552; GFX9-LABEL: flat_atomic_xor_i64_noret_offset:4553; GFX9: ; %bb.0:4554; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4555; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:324556; GFX9-NEXT: s_mov_b64 s[4:5], 04557; GFX9-NEXT: .LBB71_1: ; %atomicrmw.start4558; GFX9-NEXT: ; =>This Inner Loop Header: Depth=14559; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4560; GFX9-NEXT: v_xor_b32_e32 v5, v7, v34561; GFX9-NEXT: v_xor_b32_e32 v4, v6, v24562; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc4563; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4564; GFX9-NEXT: buffer_wbinvl1_vol4565; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]4566; GFX9-NEXT: v_mov_b32_e32 v7, v54567; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4568; GFX9-NEXT: v_mov_b32_e32 v6, v44569; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]4570; GFX9-NEXT: s_cbranch_execnz .LBB71_14571; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end4572; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]4573; GFX9-NEXT: s_setpc_b64 s[30:31]4574 %gep = getelementptr inbounds i64, ptr %out, i64 44575 %tmp0 = atomicrmw xor ptr %gep, i64 %in seq_cst, !noalias.addrspace !14576 ret void4577}4578 4579define i64 @flat_atomic_xor_i64_ret(ptr %ptr, i64 %in) {4580; GFX7-LABEL: flat_atomic_xor_i64_ret:4581; GFX7: ; %bb.0:4582; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4583; GFX7-NEXT: v_add_i32_e32 v5, vcc, 4, v04584; GFX7-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc4585; GFX7-NEXT: flat_load_dword v4, v[0:1]4586; GFX7-NEXT: flat_load_dword v5, v[5:6]4587; GFX7-NEXT: s_mov_b64 s[4:5], 04588; GFX7-NEXT: .LBB72_1: ; %atomicrmw.start4589; GFX7-NEXT: ; =>This Inner Loop Header: Depth=14590; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4591; GFX7-NEXT: v_mov_b32_e32 v7, v54592; GFX7-NEXT: v_mov_b32_e32 v6, v44593; GFX7-NEXT: v_xor_b32_e32 v5, v7, v34594; GFX7-NEXT: v_xor_b32_e32 v4, v6, v24595; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc4596; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4597; GFX7-NEXT: buffer_wbinvl1_vol4598; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]4599; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4600; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]4601; GFX7-NEXT: s_cbranch_execnz .LBB72_14602; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end4603; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]4604; GFX7-NEXT: v_mov_b32_e32 v0, v44605; GFX7-NEXT: v_mov_b32_e32 v1, v54606; GFX7-NEXT: s_setpc_b64 s[30:31]4607;4608; GFX8-LABEL: flat_atomic_xor_i64_ret:4609; GFX8: ; %bb.0:4610; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4611; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v04612; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc4613; GFX8-NEXT: flat_load_dword v4, v[0:1]4614; GFX8-NEXT: flat_load_dword v5, v[5:6]4615; GFX8-NEXT: s_mov_b64 s[4:5], 04616; GFX8-NEXT: .LBB72_1: ; %atomicrmw.start4617; GFX8-NEXT: ; =>This Inner Loop Header: Depth=14618; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4619; GFX8-NEXT: v_mov_b32_e32 v7, v54620; GFX8-NEXT: v_mov_b32_e32 v6, v44621; GFX8-NEXT: v_xor_b32_e32 v5, v7, v34622; GFX8-NEXT: v_xor_b32_e32 v4, v6, v24623; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc4624; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4625; GFX8-NEXT: buffer_wbinvl1_vol4626; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]4627; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4628; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]4629; GFX8-NEXT: s_cbranch_execnz .LBB72_14630; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end4631; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]4632; GFX8-NEXT: v_mov_b32_e32 v0, v44633; GFX8-NEXT: v_mov_b32_e32 v1, v54634; GFX8-NEXT: s_setpc_b64 s[30:31]4635;4636; GFX9-LABEL: flat_atomic_xor_i64_ret:4637; GFX9: ; %bb.0:4638; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4639; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1]4640; GFX9-NEXT: s_mov_b64 s[4:5], 04641; GFX9-NEXT: .LBB72_1: ; %atomicrmw.start4642; GFX9-NEXT: ; =>This Inner Loop Header: Depth=14643; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4644; GFX9-NEXT: v_mov_b32_e32 v7, v54645; GFX9-NEXT: v_mov_b32_e32 v6, v44646; GFX9-NEXT: v_xor_b32_e32 v5, v7, v34647; GFX9-NEXT: v_xor_b32_e32 v4, v6, v24648; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc4649; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4650; GFX9-NEXT: buffer_wbinvl1_vol4651; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]4652; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4653; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]4654; GFX9-NEXT: s_cbranch_execnz .LBB72_14655; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end4656; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]4657; GFX9-NEXT: v_mov_b32_e32 v0, v44658; GFX9-NEXT: v_mov_b32_e32 v1, v54659; GFX9-NEXT: s_setpc_b64 s[30:31]4660 %result = atomicrmw xor ptr %ptr, i64 %in seq_cst, !noalias.addrspace !14661 ret i64 %result4662}4663 4664define i64 @flat_atomic_xor_i64_ret_offset(ptr %out, i64 %in) {4665; GFX7-LABEL: flat_atomic_xor_i64_ret_offset:4666; GFX7: ; %bb.0:4667; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4668; GFX7-NEXT: v_add_i32_e32 v4, vcc, 32, v04669; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc4670; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v04671; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc4672; GFX7-NEXT: flat_load_dword v1, v[0:1]4673; GFX7-NEXT: flat_load_dword v0, v[4:5]4674; GFX7-NEXT: s_mov_b64 s[4:5], 04675; GFX7-NEXT: .LBB73_1: ; %atomicrmw.start4676; GFX7-NEXT: ; =>This Inner Loop Header: Depth=14677; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4678; GFX7-NEXT: v_mov_b32_e32 v9, v14679; GFX7-NEXT: v_mov_b32_e32 v8, v04680; GFX7-NEXT: v_xor_b32_e32 v7, v9, v34681; GFX7-NEXT: v_xor_b32_e32 v6, v8, v24682; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc4683; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4684; GFX7-NEXT: buffer_wbinvl1_vol4685; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]4686; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4687; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]4688; GFX7-NEXT: s_cbranch_execnz .LBB73_14689; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end4690; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]4691; GFX7-NEXT: s_setpc_b64 s[30:31]4692;4693; GFX8-LABEL: flat_atomic_xor_i64_ret_offset:4694; GFX8: ; %bb.0:4695; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4696; GFX8-NEXT: v_add_u32_e32 v4, vcc, 32, v04697; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc4698; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v04699; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc4700; GFX8-NEXT: flat_load_dword v1, v[0:1]4701; GFX8-NEXT: flat_load_dword v0, v[4:5]4702; GFX8-NEXT: s_mov_b64 s[4:5], 04703; GFX8-NEXT: .LBB73_1: ; %atomicrmw.start4704; GFX8-NEXT: ; =>This Inner Loop Header: Depth=14705; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4706; GFX8-NEXT: v_mov_b32_e32 v9, v14707; GFX8-NEXT: v_mov_b32_e32 v8, v04708; GFX8-NEXT: v_xor_b32_e32 v7, v9, v34709; GFX8-NEXT: v_xor_b32_e32 v6, v8, v24710; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc4711; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4712; GFX8-NEXT: buffer_wbinvl1_vol4713; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]4714; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4715; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]4716; GFX8-NEXT: s_cbranch_execnz .LBB73_14717; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end4718; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]4719; GFX8-NEXT: s_setpc_b64 s[30:31]4720;4721; GFX9-LABEL: flat_atomic_xor_i64_ret_offset:4722; GFX9: ; %bb.0:4723; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4724; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:324725; GFX9-NEXT: s_mov_b64 s[4:5], 04726; GFX9-NEXT: .LBB73_1: ; %atomicrmw.start4727; GFX9-NEXT: ; =>This Inner Loop Header: Depth=14728; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4729; GFX9-NEXT: v_mov_b32_e32 v7, v54730; GFX9-NEXT: v_mov_b32_e32 v6, v44731; GFX9-NEXT: v_xor_b32_e32 v5, v7, v34732; GFX9-NEXT: v_xor_b32_e32 v4, v6, v24733; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc4734; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4735; GFX9-NEXT: buffer_wbinvl1_vol4736; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]4737; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4738; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]4739; GFX9-NEXT: s_cbranch_execnz .LBB73_14740; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end4741; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]4742; GFX9-NEXT: v_mov_b32_e32 v0, v44743; GFX9-NEXT: v_mov_b32_e32 v1, v54744; GFX9-NEXT: s_setpc_b64 s[30:31]4745 %gep = getelementptr inbounds i64, ptr %out, i64 44746 %result = atomicrmw xor ptr %gep, i64 %in seq_cst, !noalias.addrspace !14747 ret i64 %result4748}4749 4750define amdgpu_gfx void @flat_atomic_xor_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {4751; GFX7-LABEL: flat_atomic_xor_i64_noret_scalar:4752; GFX7: ; %bb.0:4753; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4754; GFX7-NEXT: v_mov_b32_e32 v0, s44755; GFX7-NEXT: s_add_u32 s34, s4, 44756; GFX7-NEXT: v_mov_b32_e32 v1, s54757; GFX7-NEXT: s_addc_u32 s35, s5, 04758; GFX7-NEXT: v_mov_b32_e32 v3, s344759; GFX7-NEXT: v_mov_b32_e32 v4, s354760; GFX7-NEXT: flat_load_dword v2, v[0:1]4761; GFX7-NEXT: flat_load_dword v3, v[3:4]4762; GFX7-NEXT: v_mov_b32_e32 v4, s44763; GFX7-NEXT: s_mov_b64 s[34:35], 04764; GFX7-NEXT: v_mov_b32_e32 v5, s54765; GFX7-NEXT: .LBB74_1: ; %atomicrmw.start4766; GFX7-NEXT: ; =>This Inner Loop Header: Depth=14767; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4768; GFX7-NEXT: v_xor_b32_e32 v1, s7, v34769; GFX7-NEXT: v_xor_b32_e32 v0, s6, v24770; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc4771; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4772; GFX7-NEXT: buffer_wbinvl1_vol4773; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4774; GFX7-NEXT: v_mov_b32_e32 v3, v14775; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4776; GFX7-NEXT: v_mov_b32_e32 v2, v04777; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]4778; GFX7-NEXT: s_cbranch_execnz .LBB74_14779; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end4780; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]4781; GFX7-NEXT: s_setpc_b64 s[30:31]4782;4783; GFX8-LABEL: flat_atomic_xor_i64_noret_scalar:4784; GFX8: ; %bb.0:4785; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4786; GFX8-NEXT: v_mov_b32_e32 v0, s44787; GFX8-NEXT: s_add_u32 s34, s4, 44788; GFX8-NEXT: v_mov_b32_e32 v1, s54789; GFX8-NEXT: s_addc_u32 s35, s5, 04790; GFX8-NEXT: v_mov_b32_e32 v3, s344791; GFX8-NEXT: v_mov_b32_e32 v4, s354792; GFX8-NEXT: flat_load_dword v2, v[0:1]4793; GFX8-NEXT: flat_load_dword v3, v[3:4]4794; GFX8-NEXT: v_mov_b32_e32 v4, s44795; GFX8-NEXT: s_mov_b64 s[34:35], 04796; GFX8-NEXT: v_mov_b32_e32 v5, s54797; GFX8-NEXT: .LBB74_1: ; %atomicrmw.start4798; GFX8-NEXT: ; =>This Inner Loop Header: Depth=14799; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4800; GFX8-NEXT: v_xor_b32_e32 v1, s7, v34801; GFX8-NEXT: v_xor_b32_e32 v0, s6, v24802; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc4803; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4804; GFX8-NEXT: buffer_wbinvl1_vol4805; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4806; GFX8-NEXT: v_mov_b32_e32 v3, v14807; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4808; GFX8-NEXT: v_mov_b32_e32 v2, v04809; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]4810; GFX8-NEXT: s_cbranch_execnz .LBB74_14811; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end4812; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]4813; GFX8-NEXT: s_setpc_b64 s[30:31]4814;4815; GFX9-LABEL: flat_atomic_xor_i64_noret_scalar:4816; GFX9: ; %bb.0:4817; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4818; GFX9-NEXT: v_mov_b32_e32 v0, s44819; GFX9-NEXT: v_mov_b32_e32 v1, s54820; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]4821; GFX9-NEXT: v_mov_b32_e32 v4, s44822; GFX9-NEXT: s_mov_b64 s[34:35], 04823; GFX9-NEXT: v_mov_b32_e32 v5, s54824; GFX9-NEXT: .LBB74_1: ; %atomicrmw.start4825; GFX9-NEXT: ; =>This Inner Loop Header: Depth=14826; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4827; GFX9-NEXT: v_xor_b32_e32 v1, s7, v34828; GFX9-NEXT: v_xor_b32_e32 v0, s6, v24829; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc4830; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4831; GFX9-NEXT: buffer_wbinvl1_vol4832; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4833; GFX9-NEXT: v_mov_b32_e32 v3, v14834; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4835; GFX9-NEXT: v_mov_b32_e32 v2, v04836; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]4837; GFX9-NEXT: s_cbranch_execnz .LBB74_14838; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end4839; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]4840; GFX9-NEXT: s_setpc_b64 s[30:31]4841 %tmp0 = atomicrmw xor ptr %ptr, i64 %in seq_cst, !noalias.addrspace !14842 ret void4843}4844 4845define amdgpu_gfx void @flat_atomic_xor_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {4846; GFX7-LABEL: flat_atomic_xor_i64_noret_offset_scalar:4847; GFX7: ; %bb.0:4848; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4849; GFX7-NEXT: s_add_u32 s34, s4, 324850; GFX7-NEXT: s_addc_u32 s35, s5, 04851; GFX7-NEXT: s_add_u32 s36, s4, 364852; GFX7-NEXT: s_addc_u32 s37, s5, 04853; GFX7-NEXT: v_mov_b32_e32 v0, s364854; GFX7-NEXT: v_mov_b32_e32 v1, s374855; GFX7-NEXT: v_mov_b32_e32 v4, s344856; GFX7-NEXT: v_mov_b32_e32 v5, s354857; GFX7-NEXT: flat_load_dword v3, v[0:1]4858; GFX7-NEXT: flat_load_dword v2, v[4:5]4859; GFX7-NEXT: s_mov_b64 s[34:35], 04860; GFX7-NEXT: .LBB75_1: ; %atomicrmw.start4861; GFX7-NEXT: ; =>This Inner Loop Header: Depth=14862; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4863; GFX7-NEXT: v_xor_b32_e32 v1, s7, v34864; GFX7-NEXT: v_xor_b32_e32 v0, s6, v24865; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc4866; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4867; GFX7-NEXT: buffer_wbinvl1_vol4868; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4869; GFX7-NEXT: v_mov_b32_e32 v3, v14870; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4871; GFX7-NEXT: v_mov_b32_e32 v2, v04872; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]4873; GFX7-NEXT: s_cbranch_execnz .LBB75_14874; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end4875; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]4876; GFX7-NEXT: s_setpc_b64 s[30:31]4877;4878; GFX8-LABEL: flat_atomic_xor_i64_noret_offset_scalar:4879; GFX8: ; %bb.0:4880; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4881; GFX8-NEXT: s_add_u32 s34, s4, 324882; GFX8-NEXT: s_addc_u32 s35, s5, 04883; GFX8-NEXT: s_add_u32 s36, s4, 364884; GFX8-NEXT: s_addc_u32 s37, s5, 04885; GFX8-NEXT: v_mov_b32_e32 v0, s364886; GFX8-NEXT: v_mov_b32_e32 v1, s374887; GFX8-NEXT: v_mov_b32_e32 v4, s344888; GFX8-NEXT: v_mov_b32_e32 v5, s354889; GFX8-NEXT: flat_load_dword v3, v[0:1]4890; GFX8-NEXT: flat_load_dword v2, v[4:5]4891; GFX8-NEXT: s_mov_b64 s[34:35], 04892; GFX8-NEXT: .LBB75_1: ; %atomicrmw.start4893; GFX8-NEXT: ; =>This Inner Loop Header: Depth=14894; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4895; GFX8-NEXT: v_xor_b32_e32 v1, s7, v34896; GFX8-NEXT: v_xor_b32_e32 v0, s6, v24897; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc4898; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4899; GFX8-NEXT: buffer_wbinvl1_vol4900; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4901; GFX8-NEXT: v_mov_b32_e32 v3, v14902; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4903; GFX8-NEXT: v_mov_b32_e32 v2, v04904; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]4905; GFX8-NEXT: s_cbranch_execnz .LBB75_14906; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end4907; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]4908; GFX8-NEXT: s_setpc_b64 s[30:31]4909;4910; GFX9-LABEL: flat_atomic_xor_i64_noret_offset_scalar:4911; GFX9: ; %bb.0:4912; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4913; GFX9-NEXT: v_mov_b32_e32 v0, s44914; GFX9-NEXT: v_mov_b32_e32 v1, s54915; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:324916; GFX9-NEXT: v_mov_b32_e32 v4, s44917; GFX9-NEXT: s_mov_b64 s[34:35], 04918; GFX9-NEXT: v_mov_b32_e32 v5, s54919; GFX9-NEXT: .LBB75_1: ; %atomicrmw.start4920; GFX9-NEXT: ; =>This Inner Loop Header: Depth=14921; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4922; GFX9-NEXT: v_xor_b32_e32 v1, s7, v34923; GFX9-NEXT: v_xor_b32_e32 v0, s6, v24924; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc4925; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4926; GFX9-NEXT: buffer_wbinvl1_vol4927; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]4928; GFX9-NEXT: v_mov_b32_e32 v3, v14929; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4930; GFX9-NEXT: v_mov_b32_e32 v2, v04931; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]4932; GFX9-NEXT: s_cbranch_execnz .LBB75_14933; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end4934; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]4935; GFX9-NEXT: s_setpc_b64 s[30:31]4936 %gep = getelementptr inbounds i64, ptr %out, i64 44937 %tmp0 = atomicrmw xor ptr %gep, i64 %in seq_cst, !noalias.addrspace !14938 ret void4939}4940 4941define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {4942; GFX7-LABEL: flat_atomic_xor_i64_ret_scalar:4943; GFX7: ; %bb.0:4944; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4945; GFX7-NEXT: v_mov_b32_e32 v0, s44946; GFX7-NEXT: s_add_u32 s34, s4, 44947; GFX7-NEXT: v_mov_b32_e32 v1, s54948; GFX7-NEXT: s_addc_u32 s35, s5, 04949; GFX7-NEXT: v_mov_b32_e32 v2, s344950; GFX7-NEXT: v_mov_b32_e32 v3, s354951; GFX7-NEXT: flat_load_dword v0, v[0:1]4952; GFX7-NEXT: flat_load_dword v1, v[2:3]4953; GFX7-NEXT: v_mov_b32_e32 v2, s44954; GFX7-NEXT: s_mov_b64 s[34:35], 04955; GFX7-NEXT: v_mov_b32_e32 v3, s54956; GFX7-NEXT: .LBB76_1: ; %atomicrmw.start4957; GFX7-NEXT: ; =>This Inner Loop Header: Depth=14958; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4959; GFX7-NEXT: v_mov_b32_e32 v7, v14960; GFX7-NEXT: v_mov_b32_e32 v6, v04961; GFX7-NEXT: v_xor_b32_e32 v5, s7, v74962; GFX7-NEXT: v_xor_b32_e32 v4, s6, v64963; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc4964; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4965; GFX7-NEXT: buffer_wbinvl1_vol4966; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]4967; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]4968; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]4969; GFX7-NEXT: s_cbranch_execnz .LBB76_14970; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end4971; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]4972; GFX7-NEXT: s_setpc_b64 s[30:31]4973;4974; GFX8-LABEL: flat_atomic_xor_i64_ret_scalar:4975; GFX8: ; %bb.0:4976; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4977; GFX8-NEXT: v_mov_b32_e32 v0, s44978; GFX8-NEXT: s_add_u32 s34, s4, 44979; GFX8-NEXT: v_mov_b32_e32 v1, s54980; GFX8-NEXT: s_addc_u32 s35, s5, 04981; GFX8-NEXT: v_mov_b32_e32 v2, s344982; GFX8-NEXT: v_mov_b32_e32 v3, s354983; GFX8-NEXT: flat_load_dword v0, v[0:1]4984; GFX8-NEXT: flat_load_dword v1, v[2:3]4985; GFX8-NEXT: v_mov_b32_e32 v2, s44986; GFX8-NEXT: s_mov_b64 s[34:35], 04987; GFX8-NEXT: v_mov_b32_e32 v3, s54988; GFX8-NEXT: .LBB76_1: ; %atomicrmw.start4989; GFX8-NEXT: ; =>This Inner Loop Header: Depth=14990; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4991; GFX8-NEXT: v_mov_b32_e32 v7, v14992; GFX8-NEXT: v_mov_b32_e32 v6, v04993; GFX8-NEXT: v_xor_b32_e32 v5, s7, v74994; GFX8-NEXT: v_xor_b32_e32 v4, s6, v64995; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc4996; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4997; GFX8-NEXT: buffer_wbinvl1_vol4998; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]4999; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5000; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]5001; GFX8-NEXT: s_cbranch_execnz .LBB76_15002; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end5003; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]5004; GFX8-NEXT: s_setpc_b64 s[30:31]5005;5006; GFX9-LABEL: flat_atomic_xor_i64_ret_scalar:5007; GFX9: ; %bb.0:5008; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5009; GFX9-NEXT: v_mov_b32_e32 v0, s45010; GFX9-NEXT: v_mov_b32_e32 v1, s55011; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]5012; GFX9-NEXT: v_mov_b32_e32 v2, s45013; GFX9-NEXT: s_mov_b64 s[34:35], 05014; GFX9-NEXT: v_mov_b32_e32 v3, s55015; GFX9-NEXT: .LBB76_1: ; %atomicrmw.start5016; GFX9-NEXT: ; =>This Inner Loop Header: Depth=15017; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5018; GFX9-NEXT: v_mov_b32_e32 v7, v15019; GFX9-NEXT: v_mov_b32_e32 v6, v05020; GFX9-NEXT: v_xor_b32_e32 v5, s7, v75021; GFX9-NEXT: v_xor_b32_e32 v4, s6, v65022; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc5023; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5024; GFX9-NEXT: buffer_wbinvl1_vol5025; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]5026; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5027; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]5028; GFX9-NEXT: s_cbranch_execnz .LBB76_15029; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end5030; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]5031; GFX9-NEXT: s_setpc_b64 s[30:31]5032 %result = atomicrmw xor ptr %ptr, i64 %in seq_cst, !noalias.addrspace !15033 ret i64 %result5034}5035 5036define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {5037; GFX7-LABEL: flat_atomic_xor_i64_ret_offset_scalar:5038; GFX7: ; %bb.0:5039; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5040; GFX7-NEXT: s_add_u32 s34, s4, 325041; GFX7-NEXT: s_addc_u32 s35, s5, 05042; GFX7-NEXT: s_add_u32 s36, s4, 365043; GFX7-NEXT: s_addc_u32 s37, s5, 05044; GFX7-NEXT: v_mov_b32_e32 v0, s365045; GFX7-NEXT: v_mov_b32_e32 v1, s375046; GFX7-NEXT: v_mov_b32_e32 v2, s345047; GFX7-NEXT: v_mov_b32_e32 v3, s355048; GFX7-NEXT: flat_load_dword v1, v[0:1]5049; GFX7-NEXT: flat_load_dword v0, v[2:3]5050; GFX7-NEXT: s_mov_b64 s[34:35], 05051; GFX7-NEXT: .LBB77_1: ; %atomicrmw.start5052; GFX7-NEXT: ; =>This Inner Loop Header: Depth=15053; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5054; GFX7-NEXT: v_mov_b32_e32 v7, v15055; GFX7-NEXT: v_mov_b32_e32 v6, v05056; GFX7-NEXT: v_xor_b32_e32 v5, s7, v75057; GFX7-NEXT: v_xor_b32_e32 v4, s6, v65058; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc5059; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5060; GFX7-NEXT: buffer_wbinvl1_vol5061; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]5062; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5063; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]5064; GFX7-NEXT: s_cbranch_execnz .LBB77_15065; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end5066; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]5067; GFX7-NEXT: s_setpc_b64 s[30:31]5068;5069; GFX8-LABEL: flat_atomic_xor_i64_ret_offset_scalar:5070; GFX8: ; %bb.0:5071; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5072; GFX8-NEXT: s_add_u32 s34, s4, 325073; GFX8-NEXT: s_addc_u32 s35, s5, 05074; GFX8-NEXT: s_add_u32 s36, s4, 365075; GFX8-NEXT: s_addc_u32 s37, s5, 05076; GFX8-NEXT: v_mov_b32_e32 v0, s365077; GFX8-NEXT: v_mov_b32_e32 v1, s375078; GFX8-NEXT: v_mov_b32_e32 v2, s345079; GFX8-NEXT: v_mov_b32_e32 v3, s355080; GFX8-NEXT: flat_load_dword v1, v[0:1]5081; GFX8-NEXT: flat_load_dword v0, v[2:3]5082; GFX8-NEXT: s_mov_b64 s[34:35], 05083; GFX8-NEXT: .LBB77_1: ; %atomicrmw.start5084; GFX8-NEXT: ; =>This Inner Loop Header: Depth=15085; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5086; GFX8-NEXT: v_mov_b32_e32 v7, v15087; GFX8-NEXT: v_mov_b32_e32 v6, v05088; GFX8-NEXT: v_xor_b32_e32 v5, s7, v75089; GFX8-NEXT: v_xor_b32_e32 v4, s6, v65090; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc5091; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5092; GFX8-NEXT: buffer_wbinvl1_vol5093; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]5094; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5095; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]5096; GFX8-NEXT: s_cbranch_execnz .LBB77_15097; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end5098; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]5099; GFX8-NEXT: s_setpc_b64 s[30:31]5100;5101; GFX9-LABEL: flat_atomic_xor_i64_ret_offset_scalar:5102; GFX9: ; %bb.0:5103; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5104; GFX9-NEXT: v_mov_b32_e32 v0, s45105; GFX9-NEXT: v_mov_b32_e32 v1, s55106; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:325107; GFX9-NEXT: v_mov_b32_e32 v2, s45108; GFX9-NEXT: s_mov_b64 s[34:35], 05109; GFX9-NEXT: v_mov_b32_e32 v3, s55110; GFX9-NEXT: .LBB77_1: ; %atomicrmw.start5111; GFX9-NEXT: ; =>This Inner Loop Header: Depth=15112; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5113; GFX9-NEXT: v_mov_b32_e32 v7, v15114; GFX9-NEXT: v_mov_b32_e32 v6, v05115; GFX9-NEXT: v_xor_b32_e32 v5, s7, v75116; GFX9-NEXT: v_xor_b32_e32 v4, s6, v65117; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] offset:32 glc5118; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5119; GFX9-NEXT: buffer_wbinvl1_vol5120; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]5121; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5122; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]5123; GFX9-NEXT: s_cbranch_execnz .LBB77_15124; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end5125; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]5126; GFX9-NEXT: s_setpc_b64 s[30:31]5127 %gep = getelementptr inbounds i64, ptr %out, i64 45128 %result = atomicrmw xor ptr %gep, i64 %in seq_cst, !noalias.addrspace !15129 ret i64 %result5130}5131 5132define void @flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {5133; GFX7-LABEL: flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory:5134; GFX7: ; %bb.0:5135; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5136; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v05137; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc5138; GFX7-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]5139; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5140; GFX7-NEXT: buffer_wbinvl1_vol5141; GFX7-NEXT: s_setpc_b64 s[30:31]5142;5143; GFX8-LABEL: flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory:5144; GFX8: ; %bb.0:5145; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5146; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v05147; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc5148; GFX8-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]5149; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5150; GFX8-NEXT: buffer_wbinvl1_vol5151; GFX8-NEXT: s_setpc_b64 s[30:31]5152;5153; GFX9-LABEL: flat_atomic_xor_i64_noret_offset__amdgpu_no_remote_memory:5154; GFX9: ; %bb.0:5155; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5156; GFX9-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3] offset:325157; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5158; GFX9-NEXT: buffer_wbinvl1_vol5159; GFX9-NEXT: s_setpc_b64 s[30:31]5160 %gep = getelementptr inbounds i64, ptr %out, i64 45161 %tmp0 = atomicrmw xor ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !15162 ret void5163}5164 5165define i64 @flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {5166; GFX7-LABEL: flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory:5167; GFX7: ; %bb.0:5168; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5169; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v05170; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc5171; GFX7-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc5172; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5173; GFX7-NEXT: buffer_wbinvl1_vol5174; GFX7-NEXT: s_setpc_b64 s[30:31]5175;5176; GFX8-LABEL: flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory:5177; GFX8: ; %bb.0:5178; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5179; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v05180; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc5181; GFX8-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc5182; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5183; GFX8-NEXT: buffer_wbinvl1_vol5184; GFX8-NEXT: s_setpc_b64 s[30:31]5185;5186; GFX9-LABEL: flat_atomic_xor_i64_ret_offset__amdgpu_no_remote_memory:5187; GFX9: ; %bb.0:5188; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5189; GFX9-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] offset:32 glc5190; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5191; GFX9-NEXT: buffer_wbinvl1_vol5192; GFX9-NEXT: s_setpc_b64 s[30:31]5193 %gep = getelementptr inbounds i64, ptr %out, i64 45194 %result = atomicrmw xor ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !15195 ret i64 %result5196}5197 5198; ---------------------------------------------------------------------5199; atomicrmw max5200; ---------------------------------------------------------------------5201 5202define void @flat_atomic_max_i64_noret(ptr %ptr, i64 %in) {5203; GFX7-LABEL: flat_atomic_max_i64_noret:5204; GFX7: ; %bb.0:5205; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5206; GFX7-NEXT: v_add_i32_e32 v4, vcc, 4, v05207; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc5208; GFX7-NEXT: flat_load_dword v6, v[0:1]5209; GFX7-NEXT: flat_load_dword v7, v[4:5]5210; GFX7-NEXT: s_mov_b64 s[4:5], 05211; GFX7-NEXT: .LBB80_1: ; %atomicrmw.start5212; GFX7-NEXT: ; =>This Inner Loop Header: Depth=15213; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5214; GFX7-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]5215; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc5216; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc5217; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc5218; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5219; GFX7-NEXT: buffer_wbinvl1_vol5220; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]5221; GFX7-NEXT: v_mov_b32_e32 v7, v55222; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5223; GFX7-NEXT: v_mov_b32_e32 v6, v45224; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]5225; GFX7-NEXT: s_cbranch_execnz .LBB80_15226; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end5227; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]5228; GFX7-NEXT: s_setpc_b64 s[30:31]5229;5230; GFX8-LABEL: flat_atomic_max_i64_noret:5231; GFX8: ; %bb.0:5232; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5233; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v05234; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc5235; GFX8-NEXT: flat_load_dword v6, v[0:1]5236; GFX8-NEXT: flat_load_dword v7, v[4:5]5237; GFX8-NEXT: s_mov_b64 s[4:5], 05238; GFX8-NEXT: .LBB80_1: ; %atomicrmw.start5239; GFX8-NEXT: ; =>This Inner Loop Header: Depth=15240; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5241; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]5242; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc5243; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc5244; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc5245; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5246; GFX8-NEXT: buffer_wbinvl1_vol5247; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]5248; GFX8-NEXT: v_mov_b32_e32 v7, v55249; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5250; GFX8-NEXT: v_mov_b32_e32 v6, v45251; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]5252; GFX8-NEXT: s_cbranch_execnz .LBB80_15253; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end5254; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]5255; GFX8-NEXT: s_setpc_b64 s[30:31]5256;5257; GFX9-LABEL: flat_atomic_max_i64_noret:5258; GFX9: ; %bb.0:5259; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5260; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1]5261; GFX9-NEXT: s_mov_b64 s[4:5], 05262; GFX9-NEXT: .LBB80_1: ; %atomicrmw.start5263; GFX9-NEXT: ; =>This Inner Loop Header: Depth=15264; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5265; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]5266; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc5267; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc5268; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc5269; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5270; GFX9-NEXT: buffer_wbinvl1_vol5271; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]5272; GFX9-NEXT: v_mov_b32_e32 v7, v55273; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5274; GFX9-NEXT: v_mov_b32_e32 v6, v45275; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]5276; GFX9-NEXT: s_cbranch_execnz .LBB80_15277; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end5278; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]5279; GFX9-NEXT: s_setpc_b64 s[30:31]5280 %tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst, !noalias.addrspace !15281 ret void5282}5283 5284define void @flat_atomic_max_i64_noret_offset(ptr %out, i64 %in) {5285; GFX7-LABEL: flat_atomic_max_i64_noret_offset:5286; GFX7: ; %bb.0:5287; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5288; GFX7-NEXT: v_add_i32_e32 v8, vcc, 32, v05289; GFX7-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc5290; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v05291; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc5292; GFX7-NEXT: flat_load_dword v7, v[0:1]5293; GFX7-NEXT: flat_load_dword v6, v[8:9]5294; GFX7-NEXT: s_mov_b64 s[4:5], 05295; GFX7-NEXT: .LBB81_1: ; %atomicrmw.start5296; GFX7-NEXT: ; =>This Inner Loop Header: Depth=15297; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5298; GFX7-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]5299; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc5300; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc5301; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc5302; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5303; GFX7-NEXT: buffer_wbinvl1_vol5304; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]5305; GFX7-NEXT: v_mov_b32_e32 v7, v15306; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5307; GFX7-NEXT: v_mov_b32_e32 v6, v05308; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]5309; GFX7-NEXT: s_cbranch_execnz .LBB81_15310; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end5311; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]5312; GFX7-NEXT: s_setpc_b64 s[30:31]5313;5314; GFX8-LABEL: flat_atomic_max_i64_noret_offset:5315; GFX8: ; %bb.0:5316; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5317; GFX8-NEXT: v_add_u32_e32 v8, vcc, 32, v05318; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc5319; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v05320; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc5321; GFX8-NEXT: flat_load_dword v7, v[0:1]5322; GFX8-NEXT: flat_load_dword v6, v[8:9]5323; GFX8-NEXT: s_mov_b64 s[4:5], 05324; GFX8-NEXT: .LBB81_1: ; %atomicrmw.start5325; GFX8-NEXT: ; =>This Inner Loop Header: Depth=15326; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5327; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]5328; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc5329; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc5330; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc5331; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5332; GFX8-NEXT: buffer_wbinvl1_vol5333; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]5334; GFX8-NEXT: v_mov_b32_e32 v7, v15335; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5336; GFX8-NEXT: v_mov_b32_e32 v6, v05337; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]5338; GFX8-NEXT: s_cbranch_execnz .LBB81_15339; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end5340; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]5341; GFX8-NEXT: s_setpc_b64 s[30:31]5342;5343; GFX9-LABEL: flat_atomic_max_i64_noret_offset:5344; GFX9: ; %bb.0:5345; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5346; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:325347; GFX9-NEXT: s_mov_b64 s[4:5], 05348; GFX9-NEXT: .LBB81_1: ; %atomicrmw.start5349; GFX9-NEXT: ; =>This Inner Loop Header: Depth=15350; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5351; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]5352; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc5353; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc5354; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc5355; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5356; GFX9-NEXT: buffer_wbinvl1_vol5357; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]5358; GFX9-NEXT: v_mov_b32_e32 v7, v55359; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5360; GFX9-NEXT: v_mov_b32_e32 v6, v45361; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]5362; GFX9-NEXT: s_cbranch_execnz .LBB81_15363; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end5364; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]5365; GFX9-NEXT: s_setpc_b64 s[30:31]5366 %gep = getelementptr inbounds i64, ptr %out, i64 45367 %tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !15368 ret void5369}5370 5371define i64 @flat_atomic_max_i64_ret(ptr %ptr, i64 %in) {5372; GFX7-LABEL: flat_atomic_max_i64_ret:5373; GFX7: ; %bb.0:5374; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5375; GFX7-NEXT: v_add_i32_e32 v5, vcc, 4, v05376; GFX7-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc5377; GFX7-NEXT: flat_load_dword v4, v[0:1]5378; GFX7-NEXT: flat_load_dword v5, v[5:6]5379; GFX7-NEXT: s_mov_b64 s[4:5], 05380; GFX7-NEXT: .LBB82_1: ; %atomicrmw.start5381; GFX7-NEXT: ; =>This Inner Loop Header: Depth=15382; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5383; GFX7-NEXT: v_mov_b32_e32 v7, v55384; GFX7-NEXT: v_mov_b32_e32 v6, v45385; GFX7-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]5386; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc5387; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc5388; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc5389; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5390; GFX7-NEXT: buffer_wbinvl1_vol5391; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]5392; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5393; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]5394; GFX7-NEXT: s_cbranch_execnz .LBB82_15395; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end5396; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]5397; GFX7-NEXT: v_mov_b32_e32 v0, v45398; GFX7-NEXT: v_mov_b32_e32 v1, v55399; GFX7-NEXT: s_setpc_b64 s[30:31]5400;5401; GFX8-LABEL: flat_atomic_max_i64_ret:5402; GFX8: ; %bb.0:5403; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5404; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v05405; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc5406; GFX8-NEXT: flat_load_dword v4, v[0:1]5407; GFX8-NEXT: flat_load_dword v5, v[5:6]5408; GFX8-NEXT: s_mov_b64 s[4:5], 05409; GFX8-NEXT: .LBB82_1: ; %atomicrmw.start5410; GFX8-NEXT: ; =>This Inner Loop Header: Depth=15411; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5412; GFX8-NEXT: v_mov_b32_e32 v7, v55413; GFX8-NEXT: v_mov_b32_e32 v6, v45414; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]5415; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc5416; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc5417; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc5418; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5419; GFX8-NEXT: buffer_wbinvl1_vol5420; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]5421; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5422; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]5423; GFX8-NEXT: s_cbranch_execnz .LBB82_15424; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end5425; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]5426; GFX8-NEXT: v_mov_b32_e32 v0, v45427; GFX8-NEXT: v_mov_b32_e32 v1, v55428; GFX8-NEXT: s_setpc_b64 s[30:31]5429;5430; GFX9-LABEL: flat_atomic_max_i64_ret:5431; GFX9: ; %bb.0:5432; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5433; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1]5434; GFX9-NEXT: s_mov_b64 s[4:5], 05435; GFX9-NEXT: .LBB82_1: ; %atomicrmw.start5436; GFX9-NEXT: ; =>This Inner Loop Header: Depth=15437; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5438; GFX9-NEXT: v_mov_b32_e32 v7, v55439; GFX9-NEXT: v_mov_b32_e32 v6, v45440; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]5441; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc5442; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc5443; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc5444; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5445; GFX9-NEXT: buffer_wbinvl1_vol5446; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]5447; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5448; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]5449; GFX9-NEXT: s_cbranch_execnz .LBB82_15450; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end5451; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]5452; GFX9-NEXT: v_mov_b32_e32 v0, v45453; GFX9-NEXT: v_mov_b32_e32 v1, v55454; GFX9-NEXT: s_setpc_b64 s[30:31]5455 %result = atomicrmw max ptr %ptr, i64 %in seq_cst, !noalias.addrspace !15456 ret i64 %result5457}5458 5459define i64 @flat_atomic_max_i64_ret_offset(ptr %out, i64 %in) {5460; GFX7-LABEL: flat_atomic_max_i64_ret_offset:5461; GFX7: ; %bb.0:5462; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5463; GFX7-NEXT: v_add_i32_e32 v4, vcc, 32, v05464; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc5465; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v05466; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc5467; GFX7-NEXT: flat_load_dword v1, v[0:1]5468; GFX7-NEXT: flat_load_dword v0, v[4:5]5469; GFX7-NEXT: s_mov_b64 s[4:5], 05470; GFX7-NEXT: .LBB83_1: ; %atomicrmw.start5471; GFX7-NEXT: ; =>This Inner Loop Header: Depth=15472; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5473; GFX7-NEXT: v_mov_b32_e32 v9, v15474; GFX7-NEXT: v_mov_b32_e32 v8, v05475; GFX7-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]5476; GFX7-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc5477; GFX7-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc5478; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc5479; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5480; GFX7-NEXT: buffer_wbinvl1_vol5481; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]5482; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5483; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]5484; GFX7-NEXT: s_cbranch_execnz .LBB83_15485; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end5486; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]5487; GFX7-NEXT: s_setpc_b64 s[30:31]5488;5489; GFX8-LABEL: flat_atomic_max_i64_ret_offset:5490; GFX8: ; %bb.0:5491; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5492; GFX8-NEXT: v_add_u32_e32 v4, vcc, 32, v05493; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc5494; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v05495; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc5496; GFX8-NEXT: flat_load_dword v1, v[0:1]5497; GFX8-NEXT: flat_load_dword v0, v[4:5]5498; GFX8-NEXT: s_mov_b64 s[4:5], 05499; GFX8-NEXT: .LBB83_1: ; %atomicrmw.start5500; GFX8-NEXT: ; =>This Inner Loop Header: Depth=15501; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5502; GFX8-NEXT: v_mov_b32_e32 v9, v15503; GFX8-NEXT: v_mov_b32_e32 v8, v05504; GFX8-NEXT: v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]5505; GFX8-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc5506; GFX8-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc5507; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc5508; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5509; GFX8-NEXT: buffer_wbinvl1_vol5510; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]5511; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5512; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]5513; GFX8-NEXT: s_cbranch_execnz .LBB83_15514; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end5515; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]5516; GFX8-NEXT: s_setpc_b64 s[30:31]5517;5518; GFX9-LABEL: flat_atomic_max_i64_ret_offset:5519; GFX9: ; %bb.0:5520; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5521; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:325522; GFX9-NEXT: s_mov_b64 s[4:5], 05523; GFX9-NEXT: .LBB83_1: ; %atomicrmw.start5524; GFX9-NEXT: ; =>This Inner Loop Header: Depth=15525; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5526; GFX9-NEXT: v_mov_b32_e32 v7, v55527; GFX9-NEXT: v_mov_b32_e32 v6, v45528; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]5529; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc5530; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc5531; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc5532; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5533; GFX9-NEXT: buffer_wbinvl1_vol5534; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]5535; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5536; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]5537; GFX9-NEXT: s_cbranch_execnz .LBB83_15538; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end5539; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]5540; GFX9-NEXT: v_mov_b32_e32 v0, v45541; GFX9-NEXT: v_mov_b32_e32 v1, v55542; GFX9-NEXT: s_setpc_b64 s[30:31]5543 %gep = getelementptr inbounds i64, ptr %out, i64 45544 %result = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !15545 ret i64 %result5546}5547 5548define amdgpu_gfx void @flat_atomic_max_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {5549; GFX7-LABEL: flat_atomic_max_i64_noret_scalar:5550; GFX7: ; %bb.0:5551; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5552; GFX7-NEXT: v_mov_b32_e32 v0, s45553; GFX7-NEXT: s_add_u32 s34, s4, 45554; GFX7-NEXT: v_mov_b32_e32 v1, s55555; GFX7-NEXT: s_addc_u32 s35, s5, 05556; GFX7-NEXT: v_mov_b32_e32 v3, s345557; GFX7-NEXT: v_mov_b32_e32 v4, s355558; GFX7-NEXT: flat_load_dword v2, v[0:1]5559; GFX7-NEXT: flat_load_dword v3, v[3:4]5560; GFX7-NEXT: v_mov_b32_e32 v4, s45561; GFX7-NEXT: s_mov_b64 s[34:35], 05562; GFX7-NEXT: v_mov_b32_e32 v6, s75563; GFX7-NEXT: v_mov_b32_e32 v7, s65564; GFX7-NEXT: v_mov_b32_e32 v5, s55565; GFX7-NEXT: .LBB84_1: ; %atomicrmw.start5566; GFX7-NEXT: ; =>This Inner Loop Header: Depth=15567; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5568; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]5569; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc5570; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc5571; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc5572; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5573; GFX7-NEXT: buffer_wbinvl1_vol5574; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]5575; GFX7-NEXT: v_mov_b32_e32 v3, v15576; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5577; GFX7-NEXT: v_mov_b32_e32 v2, v05578; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]5579; GFX7-NEXT: s_cbranch_execnz .LBB84_15580; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end5581; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]5582; GFX7-NEXT: s_setpc_b64 s[30:31]5583;5584; GFX8-LABEL: flat_atomic_max_i64_noret_scalar:5585; GFX8: ; %bb.0:5586; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5587; GFX8-NEXT: v_mov_b32_e32 v0, s45588; GFX8-NEXT: s_add_u32 s34, s4, 45589; GFX8-NEXT: v_mov_b32_e32 v1, s55590; GFX8-NEXT: s_addc_u32 s35, s5, 05591; GFX8-NEXT: v_mov_b32_e32 v3, s345592; GFX8-NEXT: v_mov_b32_e32 v4, s355593; GFX8-NEXT: flat_load_dword v2, v[0:1]5594; GFX8-NEXT: flat_load_dword v3, v[3:4]5595; GFX8-NEXT: v_mov_b32_e32 v4, s45596; GFX8-NEXT: s_mov_b64 s[34:35], 05597; GFX8-NEXT: v_mov_b32_e32 v6, s75598; GFX8-NEXT: v_mov_b32_e32 v7, s65599; GFX8-NEXT: v_mov_b32_e32 v5, s55600; GFX8-NEXT: .LBB84_1: ; %atomicrmw.start5601; GFX8-NEXT: ; =>This Inner Loop Header: Depth=15602; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5603; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]5604; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc5605; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc5606; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc5607; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5608; GFX8-NEXT: buffer_wbinvl1_vol5609; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]5610; GFX8-NEXT: v_mov_b32_e32 v3, v15611; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5612; GFX8-NEXT: v_mov_b32_e32 v2, v05613; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]5614; GFX8-NEXT: s_cbranch_execnz .LBB84_15615; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end5616; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]5617; GFX8-NEXT: s_setpc_b64 s[30:31]5618;5619; GFX9-LABEL: flat_atomic_max_i64_noret_scalar:5620; GFX9: ; %bb.0:5621; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5622; GFX9-NEXT: v_mov_b32_e32 v0, s45623; GFX9-NEXT: v_mov_b32_e32 v1, s55624; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]5625; GFX9-NEXT: v_mov_b32_e32 v4, s45626; GFX9-NEXT: s_mov_b64 s[34:35], 05627; GFX9-NEXT: v_mov_b32_e32 v6, s75628; GFX9-NEXT: v_mov_b32_e32 v7, s65629; GFX9-NEXT: v_mov_b32_e32 v5, s55630; GFX9-NEXT: .LBB84_1: ; %atomicrmw.start5631; GFX9-NEXT: ; =>This Inner Loop Header: Depth=15632; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5633; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]5634; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc5635; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc5636; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc5637; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5638; GFX9-NEXT: buffer_wbinvl1_vol5639; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]5640; GFX9-NEXT: v_mov_b32_e32 v3, v15641; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5642; GFX9-NEXT: v_mov_b32_e32 v2, v05643; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]5644; GFX9-NEXT: s_cbranch_execnz .LBB84_15645; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end5646; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]5647; GFX9-NEXT: s_setpc_b64 s[30:31]5648 %tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst, !noalias.addrspace !15649 ret void5650}5651 5652define amdgpu_gfx void @flat_atomic_max_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {5653; GFX7-LABEL: flat_atomic_max_i64_noret_offset_scalar:5654; GFX7: ; %bb.0:5655; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5656; GFX7-NEXT: s_add_u32 s34, s4, 325657; GFX7-NEXT: s_addc_u32 s35, s5, 05658; GFX7-NEXT: s_add_u32 s36, s4, 365659; GFX7-NEXT: s_addc_u32 s37, s5, 05660; GFX7-NEXT: v_mov_b32_e32 v0, s365661; GFX7-NEXT: v_mov_b32_e32 v1, s375662; GFX7-NEXT: v_mov_b32_e32 v4, s345663; GFX7-NEXT: v_mov_b32_e32 v5, s355664; GFX7-NEXT: flat_load_dword v3, v[0:1]5665; GFX7-NEXT: flat_load_dword v2, v[4:5]5666; GFX7-NEXT: s_mov_b64 s[34:35], 05667; GFX7-NEXT: v_mov_b32_e32 v6, s75668; GFX7-NEXT: v_mov_b32_e32 v7, s65669; GFX7-NEXT: .LBB85_1: ; %atomicrmw.start5670; GFX7-NEXT: ; =>This Inner Loop Header: Depth=15671; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5672; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]5673; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc5674; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc5675; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc5676; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5677; GFX7-NEXT: buffer_wbinvl1_vol5678; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]5679; GFX7-NEXT: v_mov_b32_e32 v3, v15680; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5681; GFX7-NEXT: v_mov_b32_e32 v2, v05682; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]5683; GFX7-NEXT: s_cbranch_execnz .LBB85_15684; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end5685; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]5686; GFX7-NEXT: s_setpc_b64 s[30:31]5687;5688; GFX8-LABEL: flat_atomic_max_i64_noret_offset_scalar:5689; GFX8: ; %bb.0:5690; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5691; GFX8-NEXT: s_add_u32 s34, s4, 325692; GFX8-NEXT: s_addc_u32 s35, s5, 05693; GFX8-NEXT: s_add_u32 s36, s4, 365694; GFX8-NEXT: s_addc_u32 s37, s5, 05695; GFX8-NEXT: v_mov_b32_e32 v0, s365696; GFX8-NEXT: v_mov_b32_e32 v1, s375697; GFX8-NEXT: v_mov_b32_e32 v4, s345698; GFX8-NEXT: v_mov_b32_e32 v5, s355699; GFX8-NEXT: flat_load_dword v3, v[0:1]5700; GFX8-NEXT: flat_load_dword v2, v[4:5]5701; GFX8-NEXT: s_mov_b64 s[34:35], 05702; GFX8-NEXT: v_mov_b32_e32 v6, s75703; GFX8-NEXT: v_mov_b32_e32 v7, s65704; GFX8-NEXT: .LBB85_1: ; %atomicrmw.start5705; GFX8-NEXT: ; =>This Inner Loop Header: Depth=15706; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5707; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]5708; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc5709; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc5710; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc5711; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5712; GFX8-NEXT: buffer_wbinvl1_vol5713; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]5714; GFX8-NEXT: v_mov_b32_e32 v3, v15715; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5716; GFX8-NEXT: v_mov_b32_e32 v2, v05717; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]5718; GFX8-NEXT: s_cbranch_execnz .LBB85_15719; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end5720; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]5721; GFX8-NEXT: s_setpc_b64 s[30:31]5722;5723; GFX9-LABEL: flat_atomic_max_i64_noret_offset_scalar:5724; GFX9: ; %bb.0:5725; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5726; GFX9-NEXT: v_mov_b32_e32 v0, s45727; GFX9-NEXT: v_mov_b32_e32 v1, s55728; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:325729; GFX9-NEXT: v_mov_b32_e32 v4, s45730; GFX9-NEXT: s_mov_b64 s[34:35], 05731; GFX9-NEXT: v_mov_b32_e32 v6, s75732; GFX9-NEXT: v_mov_b32_e32 v7, s65733; GFX9-NEXT: v_mov_b32_e32 v5, s55734; GFX9-NEXT: .LBB85_1: ; %atomicrmw.start5735; GFX9-NEXT: ; =>This Inner Loop Header: Depth=15736; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5737; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]5738; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc5739; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc5740; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc5741; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5742; GFX9-NEXT: buffer_wbinvl1_vol5743; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]5744; GFX9-NEXT: v_mov_b32_e32 v3, v15745; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5746; GFX9-NEXT: v_mov_b32_e32 v2, v05747; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]5748; GFX9-NEXT: s_cbranch_execnz .LBB85_15749; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end5750; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]5751; GFX9-NEXT: s_setpc_b64 s[30:31]5752 %gep = getelementptr inbounds i64, ptr %out, i64 45753 %tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !15754 ret void5755}5756 5757define amdgpu_gfx i64 @flat_atomic_max_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {5758; GFX7-LABEL: flat_atomic_max_i64_ret_scalar:5759; GFX7: ; %bb.0:5760; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5761; GFX7-NEXT: v_mov_b32_e32 v0, s45762; GFX7-NEXT: s_add_u32 s34, s4, 45763; GFX7-NEXT: v_mov_b32_e32 v1, s55764; GFX7-NEXT: s_addc_u32 s35, s5, 05765; GFX7-NEXT: v_mov_b32_e32 v2, s345766; GFX7-NEXT: v_mov_b32_e32 v3, s355767; GFX7-NEXT: flat_load_dword v0, v[0:1]5768; GFX7-NEXT: flat_load_dword v1, v[2:3]5769; GFX7-NEXT: v_mov_b32_e32 v2, s45770; GFX7-NEXT: s_mov_b64 s[34:35], 05771; GFX7-NEXT: v_mov_b32_e32 v4, s75772; GFX7-NEXT: v_mov_b32_e32 v5, s65773; GFX7-NEXT: v_mov_b32_e32 v3, s55774; GFX7-NEXT: .LBB86_1: ; %atomicrmw.start5775; GFX7-NEXT: ; =>This Inner Loop Header: Depth=15776; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5777; GFX7-NEXT: v_mov_b32_e32 v9, v15778; GFX7-NEXT: v_mov_b32_e32 v8, v05779; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]5780; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc5781; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc5782; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc5783; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5784; GFX7-NEXT: buffer_wbinvl1_vol5785; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]5786; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5787; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]5788; GFX7-NEXT: s_cbranch_execnz .LBB86_15789; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end5790; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]5791; GFX7-NEXT: s_setpc_b64 s[30:31]5792;5793; GFX8-LABEL: flat_atomic_max_i64_ret_scalar:5794; GFX8: ; %bb.0:5795; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5796; GFX8-NEXT: v_mov_b32_e32 v0, s45797; GFX8-NEXT: s_add_u32 s34, s4, 45798; GFX8-NEXT: v_mov_b32_e32 v1, s55799; GFX8-NEXT: s_addc_u32 s35, s5, 05800; GFX8-NEXT: v_mov_b32_e32 v2, s345801; GFX8-NEXT: v_mov_b32_e32 v3, s355802; GFX8-NEXT: flat_load_dword v0, v[0:1]5803; GFX8-NEXT: flat_load_dword v1, v[2:3]5804; GFX8-NEXT: v_mov_b32_e32 v2, s45805; GFX8-NEXT: s_mov_b64 s[34:35], 05806; GFX8-NEXT: v_mov_b32_e32 v4, s75807; GFX8-NEXT: v_mov_b32_e32 v5, s65808; GFX8-NEXT: v_mov_b32_e32 v3, s55809; GFX8-NEXT: .LBB86_1: ; %atomicrmw.start5810; GFX8-NEXT: ; =>This Inner Loop Header: Depth=15811; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5812; GFX8-NEXT: v_mov_b32_e32 v9, v15813; GFX8-NEXT: v_mov_b32_e32 v8, v05814; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]5815; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc5816; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc5817; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc5818; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5819; GFX8-NEXT: buffer_wbinvl1_vol5820; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]5821; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5822; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]5823; GFX8-NEXT: s_cbranch_execnz .LBB86_15824; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end5825; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]5826; GFX8-NEXT: s_setpc_b64 s[30:31]5827;5828; GFX9-LABEL: flat_atomic_max_i64_ret_scalar:5829; GFX9: ; %bb.0:5830; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5831; GFX9-NEXT: v_mov_b32_e32 v0, s45832; GFX9-NEXT: v_mov_b32_e32 v1, s55833; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]5834; GFX9-NEXT: v_mov_b32_e32 v2, s45835; GFX9-NEXT: s_mov_b64 s[34:35], 05836; GFX9-NEXT: v_mov_b32_e32 v4, s75837; GFX9-NEXT: v_mov_b32_e32 v5, s65838; GFX9-NEXT: v_mov_b32_e32 v3, s55839; GFX9-NEXT: .LBB86_1: ; %atomicrmw.start5840; GFX9-NEXT: ; =>This Inner Loop Header: Depth=15841; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5842; GFX9-NEXT: v_mov_b32_e32 v9, v15843; GFX9-NEXT: v_mov_b32_e32 v8, v05844; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]5845; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc5846; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc5847; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc5848; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5849; GFX9-NEXT: buffer_wbinvl1_vol5850; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]5851; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5852; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]5853; GFX9-NEXT: s_cbranch_execnz .LBB86_15854; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end5855; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]5856; GFX9-NEXT: s_setpc_b64 s[30:31]5857 %result = atomicrmw max ptr %ptr, i64 %in seq_cst, !noalias.addrspace !15858 ret i64 %result5859}5860 5861define amdgpu_gfx i64 @flat_atomic_max_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {5862; GFX7-LABEL: flat_atomic_max_i64_ret_offset_scalar:5863; GFX7: ; %bb.0:5864; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5865; GFX7-NEXT: s_add_u32 s34, s4, 325866; GFX7-NEXT: s_addc_u32 s35, s5, 05867; GFX7-NEXT: s_add_u32 s36, s4, 365868; GFX7-NEXT: s_addc_u32 s37, s5, 05869; GFX7-NEXT: v_mov_b32_e32 v0, s365870; GFX7-NEXT: v_mov_b32_e32 v1, s375871; GFX7-NEXT: v_mov_b32_e32 v2, s345872; GFX7-NEXT: v_mov_b32_e32 v3, s355873; GFX7-NEXT: flat_load_dword v1, v[0:1]5874; GFX7-NEXT: flat_load_dword v0, v[2:3]5875; GFX7-NEXT: s_mov_b64 s[34:35], 05876; GFX7-NEXT: v_mov_b32_e32 v4, s75877; GFX7-NEXT: v_mov_b32_e32 v5, s65878; GFX7-NEXT: .LBB87_1: ; %atomicrmw.start5879; GFX7-NEXT: ; =>This Inner Loop Header: Depth=15880; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5881; GFX7-NEXT: v_mov_b32_e32 v9, v15882; GFX7-NEXT: v_mov_b32_e32 v8, v05883; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]5884; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc5885; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc5886; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc5887; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5888; GFX7-NEXT: buffer_wbinvl1_vol5889; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]5890; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5891; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]5892; GFX7-NEXT: s_cbranch_execnz .LBB87_15893; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end5894; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]5895; GFX7-NEXT: s_setpc_b64 s[30:31]5896;5897; GFX8-LABEL: flat_atomic_max_i64_ret_offset_scalar:5898; GFX8: ; %bb.0:5899; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5900; GFX8-NEXT: s_add_u32 s34, s4, 325901; GFX8-NEXT: s_addc_u32 s35, s5, 05902; GFX8-NEXT: s_add_u32 s36, s4, 365903; GFX8-NEXT: s_addc_u32 s37, s5, 05904; GFX8-NEXT: v_mov_b32_e32 v0, s365905; GFX8-NEXT: v_mov_b32_e32 v1, s375906; GFX8-NEXT: v_mov_b32_e32 v2, s345907; GFX8-NEXT: v_mov_b32_e32 v3, s355908; GFX8-NEXT: flat_load_dword v1, v[0:1]5909; GFX8-NEXT: flat_load_dword v0, v[2:3]5910; GFX8-NEXT: s_mov_b64 s[34:35], 05911; GFX8-NEXT: v_mov_b32_e32 v4, s75912; GFX8-NEXT: v_mov_b32_e32 v5, s65913; GFX8-NEXT: .LBB87_1: ; %atomicrmw.start5914; GFX8-NEXT: ; =>This Inner Loop Header: Depth=15915; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5916; GFX8-NEXT: v_mov_b32_e32 v9, v15917; GFX8-NEXT: v_mov_b32_e32 v8, v05918; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]5919; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc5920; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc5921; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc5922; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5923; GFX8-NEXT: buffer_wbinvl1_vol5924; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]5925; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5926; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]5927; GFX8-NEXT: s_cbranch_execnz .LBB87_15928; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end5929; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]5930; GFX8-NEXT: s_setpc_b64 s[30:31]5931;5932; GFX9-LABEL: flat_atomic_max_i64_ret_offset_scalar:5933; GFX9: ; %bb.0:5934; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5935; GFX9-NEXT: v_mov_b32_e32 v0, s45936; GFX9-NEXT: v_mov_b32_e32 v1, s55937; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:325938; GFX9-NEXT: v_mov_b32_e32 v2, s45939; GFX9-NEXT: s_mov_b64 s[34:35], 05940; GFX9-NEXT: v_mov_b32_e32 v4, s75941; GFX9-NEXT: v_mov_b32_e32 v5, s65942; GFX9-NEXT: v_mov_b32_e32 v3, s55943; GFX9-NEXT: .LBB87_1: ; %atomicrmw.start5944; GFX9-NEXT: ; =>This Inner Loop Header: Depth=15945; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5946; GFX9-NEXT: v_mov_b32_e32 v9, v15947; GFX9-NEXT: v_mov_b32_e32 v8, v05948; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]5949; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc5950; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc5951; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc5952; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5953; GFX9-NEXT: buffer_wbinvl1_vol5954; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]5955; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]5956; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]5957; GFX9-NEXT: s_cbranch_execnz .LBB87_15958; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end5959; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]5960; GFX9-NEXT: s_setpc_b64 s[30:31]5961 %gep = getelementptr inbounds i64, ptr %out, i64 45962 %result = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !15963 ret i64 %result5964}5965 5966define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {5967; GFX7-LABEL: atomic_max_i64_addr64_offset:5968; GFX7: ; %bb.0: ; %entry5969; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd5970; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95971; GFX7-NEXT: s_waitcnt lgkmcnt(0)5972; GFX7-NEXT: s_lshl_b64 s[4:5], s[6:7], 35973; GFX7-NEXT: s_add_u32 s0, s0, s45974; GFX7-NEXT: s_addc_u32 s1, s1, s55975; GFX7-NEXT: s_add_u32 s0, s0, 325976; GFX7-NEXT: s_addc_u32 s1, s1, 05977; GFX7-NEXT: v_mov_b32_e32 v5, s15978; GFX7-NEXT: v_mov_b32_e32 v4, s05979; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5]5980; GFX7-NEXT: s_mov_b64 s[0:1], 05981; GFX7-NEXT: v_mov_b32_e32 v6, s35982; GFX7-NEXT: v_mov_b32_e32 v7, s25983; GFX7-NEXT: .LBB88_1: ; %atomicrmw.start5984; GFX7-NEXT: ; =>This Inner Loop Header: Depth=15985; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5986; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]5987; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc5988; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc5989; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc5990; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5991; GFX7-NEXT: buffer_wbinvl1_vol5992; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]5993; GFX7-NEXT: v_mov_b32_e32 v3, v15994; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]5995; GFX7-NEXT: v_mov_b32_e32 v2, v05996; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]5997; GFX7-NEXT: s_cbranch_execnz .LBB88_15998; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end5999; GFX7-NEXT: s_endpgm6000;6001; GFX8-LABEL: atomic_max_i64_addr64_offset:6002; GFX8: ; %bb.0: ; %entry6003; GFX8-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346004; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246005; GFX8-NEXT: s_waitcnt lgkmcnt(0)6006; GFX8-NEXT: s_lshl_b64 s[4:5], s[6:7], 36007; GFX8-NEXT: s_add_u32 s0, s0, s46008; GFX8-NEXT: s_addc_u32 s1, s1, s56009; GFX8-NEXT: s_add_u32 s0, s0, 326010; GFX8-NEXT: s_addc_u32 s1, s1, 06011; GFX8-NEXT: v_mov_b32_e32 v5, s16012; GFX8-NEXT: v_mov_b32_e32 v4, s06013; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5]6014; GFX8-NEXT: s_mov_b64 s[0:1], 06015; GFX8-NEXT: v_mov_b32_e32 v6, s36016; GFX8-NEXT: v_mov_b32_e32 v7, s26017; GFX8-NEXT: .LBB88_1: ; %atomicrmw.start6018; GFX8-NEXT: ; =>This Inner Loop Header: Depth=16019; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6020; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]6021; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc6022; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc6023; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc6024; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6025; GFX8-NEXT: buffer_wbinvl1_vol6026; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]6027; GFX8-NEXT: v_mov_b32_e32 v3, v16028; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]6029; GFX8-NEXT: v_mov_b32_e32 v2, v06030; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]6031; GFX8-NEXT: s_cbranch_execnz .LBB88_16032; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end6033; GFX8-NEXT: s_endpgm6034;6035; GFX9-LABEL: atomic_max_i64_addr64_offset:6036; GFX9: ; %bb.0: ; %entry6037; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346038; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246039; GFX9-NEXT: s_waitcnt lgkmcnt(0)6040; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 36041; GFX9-NEXT: s_add_u32 s0, s0, s46042; GFX9-NEXT: s_addc_u32 s1, s1, s56043; GFX9-NEXT: v_mov_b32_e32 v5, s16044; GFX9-NEXT: v_mov_b32_e32 v4, s06045; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[4:5] offset:326046; GFX9-NEXT: s_mov_b64 s[0:1], 06047; GFX9-NEXT: v_mov_b32_e32 v6, s36048; GFX9-NEXT: v_mov_b32_e32 v7, s26049; GFX9-NEXT: .LBB88_1: ; %atomicrmw.start6050; GFX9-NEXT: ; =>This Inner Loop Header: Depth=16051; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6052; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]6053; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc6054; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc6055; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc6056; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6057; GFX9-NEXT: buffer_wbinvl1_vol6058; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]6059; GFX9-NEXT: v_mov_b32_e32 v3, v16060; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]6061; GFX9-NEXT: v_mov_b32_e32 v2, v06062; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]6063; GFX9-NEXT: s_cbranch_execnz .LBB88_16064; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end6065; GFX9-NEXT: s_endpgm6066entry:6067 %ptr = getelementptr inbounds i64, ptr %out, i64 %index6068 %gep = getelementptr inbounds i64, ptr %ptr, i64 46069 %tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !16070 ret void6071}6072 6073define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {6074; GFX7-LABEL: atomic_max_i64_ret_addr64_offset:6075; GFX7: ; %bb.0: ; %entry6076; GFX7-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x96077; GFX7-NEXT: s_waitcnt lgkmcnt(0)6078; GFX7-NEXT: s_lshl_b64 s[6:7], s[6:7], 36079; GFX7-NEXT: s_add_u32 s0, s0, s66080; GFX7-NEXT: s_addc_u32 s1, s1, s76081; GFX7-NEXT: s_add_u32 s0, s0, 326082; GFX7-NEXT: s_addc_u32 s1, s1, 06083; GFX7-NEXT: v_mov_b32_e32 v0, s06084; GFX7-NEXT: v_mov_b32_e32 v1, s16085; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]6086; GFX7-NEXT: s_mov_b64 s[0:1], 06087; GFX7-NEXT: v_mov_b32_e32 v4, s56088; GFX7-NEXT: v_mov_b32_e32 v5, s46089; GFX7-NEXT: .LBB89_1: ; %atomicrmw.start6090; GFX7-NEXT: ; =>This Inner Loop Header: Depth=16091; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6092; GFX7-NEXT: v_mov_b32_e32 v9, v36093; GFX7-NEXT: v_mov_b32_e32 v8, v26094; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]6095; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc6096; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc6097; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc6098; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6099; GFX7-NEXT: buffer_wbinvl1_vol6100; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]6101; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]6102; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]6103; GFX7-NEXT: s_cbranch_execnz .LBB89_16104; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end6105; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]6106; GFX7-NEXT: v_mov_b32_e32 v0, s26107; GFX7-NEXT: v_mov_b32_e32 v1, s36108; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]6109; GFX7-NEXT: s_endpgm6110;6111; GFX8-LABEL: atomic_max_i64_ret_addr64_offset:6112; GFX8: ; %bb.0: ; %entry6113; GFX8-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x246114; GFX8-NEXT: s_waitcnt lgkmcnt(0)6115; GFX8-NEXT: s_lshl_b64 s[6:7], s[6:7], 36116; GFX8-NEXT: s_add_u32 s0, s0, s66117; GFX8-NEXT: s_addc_u32 s1, s1, s76118; GFX8-NEXT: s_add_u32 s0, s0, 326119; GFX8-NEXT: s_addc_u32 s1, s1, 06120; GFX8-NEXT: v_mov_b32_e32 v0, s06121; GFX8-NEXT: v_mov_b32_e32 v1, s16122; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]6123; GFX8-NEXT: s_mov_b64 s[0:1], 06124; GFX8-NEXT: v_mov_b32_e32 v4, s56125; GFX8-NEXT: v_mov_b32_e32 v5, s46126; GFX8-NEXT: .LBB89_1: ; %atomicrmw.start6127; GFX8-NEXT: ; =>This Inner Loop Header: Depth=16128; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6129; GFX8-NEXT: v_mov_b32_e32 v9, v36130; GFX8-NEXT: v_mov_b32_e32 v8, v26131; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]6132; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc6133; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc6134; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc6135; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6136; GFX8-NEXT: buffer_wbinvl1_vol6137; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]6138; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]6139; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]6140; GFX8-NEXT: s_cbranch_execnz .LBB89_16141; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end6142; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]6143; GFX8-NEXT: v_mov_b32_e32 v0, s26144; GFX8-NEXT: v_mov_b32_e32 v1, s36145; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]6146; GFX8-NEXT: s_endpgm6147;6148; GFX9-LABEL: atomic_max_i64_ret_addr64_offset:6149; GFX9: ; %bb.0: ; %entry6150; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x246151; GFX9-NEXT: s_waitcnt lgkmcnt(0)6152; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 36153; GFX9-NEXT: s_add_u32 s0, s8, s06154; GFX9-NEXT: s_addc_u32 s1, s9, s16155; GFX9-NEXT: v_mov_b32_e32 v0, s06156; GFX9-NEXT: v_mov_b32_e32 v1, s16157; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:326158; GFX9-NEXT: s_mov_b64 s[0:1], 06159; GFX9-NEXT: v_mov_b32_e32 v4, s136160; GFX9-NEXT: v_mov_b32_e32 v5, s126161; GFX9-NEXT: .LBB89_1: ; %atomicrmw.start6162; GFX9-NEXT: ; =>This Inner Loop Header: Depth=16163; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6164; GFX9-NEXT: v_mov_b32_e32 v9, v36165; GFX9-NEXT: v_mov_b32_e32 v8, v26166; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9]6167; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc6168; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc6169; GFX9-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] offset:32 glc6170; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6171; GFX9-NEXT: buffer_wbinvl1_vol6172; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]6173; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]6174; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]6175; GFX9-NEXT: s_cbranch_execnz .LBB89_16176; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end6177; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]6178; GFX9-NEXT: v_mov_b32_e32 v0, s106179; GFX9-NEXT: v_mov_b32_e32 v1, s116180; GFX9-NEXT: flat_store_dwordx2 v[0:1], v[2:3]6181; GFX9-NEXT: s_endpgm6182entry:6183 %ptr = getelementptr inbounds i64, ptr %out, i64 %index6184 %gep = getelementptr inbounds i64, ptr %ptr, i64 46185 %tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst, !noalias.addrspace !16186 store i64 %tmp0, ptr %out26187 ret void6188}6189 6190define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index) {6191; GFX7-LABEL: atomic_max_i64_addr64:6192; GFX7: ; %bb.0: ; %entry6193; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd6194; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96195; GFX7-NEXT: s_waitcnt lgkmcnt(0)6196; GFX7-NEXT: s_lshl_b64 s[4:5], s[6:7], 36197; GFX7-NEXT: s_add_u32 s0, s0, s46198; GFX7-NEXT: s_addc_u32 s1, s1, s56199; GFX7-NEXT: v_mov_b32_e32 v5, s16200; GFX7-NEXT: v_mov_b32_e32 v4, s06201; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5]6202; GFX7-NEXT: s_mov_b64 s[0:1], 06203; GFX7-NEXT: v_mov_b32_e32 v6, s36204; GFX7-NEXT: v_mov_b32_e32 v7, s26205; GFX7-NEXT: .LBB90_1: ; %atomicrmw.start6206; GFX7-NEXT: ; =>This Inner Loop Header: Depth=16207; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6208; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]6209; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc6210; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc6211; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc6212; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6213; GFX7-NEXT: buffer_wbinvl1_vol6214; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]6215; GFX7-NEXT: v_mov_b32_e32 v3, v16216; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]6217; GFX7-NEXT: v_mov_b32_e32 v2, v06218; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]6219; GFX7-NEXT: s_cbranch_execnz .LBB90_16220; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end6221; GFX7-NEXT: s_endpgm6222;6223; GFX8-LABEL: atomic_max_i64_addr64:6224; GFX8: ; %bb.0: ; %entry6225; GFX8-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346226; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246227; GFX8-NEXT: s_waitcnt lgkmcnt(0)6228; GFX8-NEXT: s_lshl_b64 s[4:5], s[6:7], 36229; GFX8-NEXT: s_add_u32 s0, s0, s46230; GFX8-NEXT: s_addc_u32 s1, s1, s56231; GFX8-NEXT: v_mov_b32_e32 v5, s16232; GFX8-NEXT: v_mov_b32_e32 v4, s06233; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5]6234; GFX8-NEXT: s_mov_b64 s[0:1], 06235; GFX8-NEXT: v_mov_b32_e32 v6, s36236; GFX8-NEXT: v_mov_b32_e32 v7, s26237; GFX8-NEXT: .LBB90_1: ; %atomicrmw.start6238; GFX8-NEXT: ; =>This Inner Loop Header: Depth=16239; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6240; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]6241; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc6242; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc6243; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc6244; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6245; GFX8-NEXT: buffer_wbinvl1_vol6246; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]6247; GFX8-NEXT: v_mov_b32_e32 v3, v16248; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]6249; GFX8-NEXT: v_mov_b32_e32 v2, v06250; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]6251; GFX8-NEXT: s_cbranch_execnz .LBB90_16252; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end6253; GFX8-NEXT: s_endpgm6254;6255; GFX9-LABEL: atomic_max_i64_addr64:6256; GFX9: ; %bb.0: ; %entry6257; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x346258; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246259; GFX9-NEXT: s_waitcnt lgkmcnt(0)6260; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 36261; GFX9-NEXT: s_add_u32 s0, s0, s46262; GFX9-NEXT: s_addc_u32 s1, s1, s56263; GFX9-NEXT: v_mov_b32_e32 v5, s16264; GFX9-NEXT: v_mov_b32_e32 v4, s06265; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[4:5]6266; GFX9-NEXT: s_mov_b64 s[0:1], 06267; GFX9-NEXT: v_mov_b32_e32 v6, s36268; GFX9-NEXT: v_mov_b32_e32 v7, s26269; GFX9-NEXT: .LBB90_1: ; %atomicrmw.start6270; GFX9-NEXT: ; =>This Inner Loop Header: Depth=16271; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6272; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]6273; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc6274; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc6275; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc6276; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6277; GFX9-NEXT: buffer_wbinvl1_vol6278; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]6279; GFX9-NEXT: v_mov_b32_e32 v3, v16280; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]6281; GFX9-NEXT: v_mov_b32_e32 v2, v06282; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]6283; GFX9-NEXT: s_cbranch_execnz .LBB90_16284; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end6285; GFX9-NEXT: s_endpgm6286entry:6287 %ptr = getelementptr inbounds i64, ptr %out, i64 %index6288 %tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst, !noalias.addrspace !16289 ret void6290}6291 6292define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {6293; GFX7-LABEL: atomic_max_i64_ret_addr64:6294; GFX7: ; %bb.0: ; %entry6295; GFX7-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x96296; GFX7-NEXT: s_waitcnt lgkmcnt(0)6297; GFX7-NEXT: s_lshl_b64 s[6:7], s[6:7], 36298; GFX7-NEXT: s_add_u32 s0, s0, s66299; GFX7-NEXT: s_addc_u32 s1, s1, s76300; GFX7-NEXT: v_mov_b32_e32 v0, s06301; GFX7-NEXT: v_mov_b32_e32 v1, s16302; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]6303; GFX7-NEXT: s_mov_b64 s[0:1], 06304; GFX7-NEXT: v_mov_b32_e32 v4, s56305; GFX7-NEXT: v_mov_b32_e32 v5, s46306; GFX7-NEXT: .LBB91_1: ; %atomicrmw.start6307; GFX7-NEXT: ; =>This Inner Loop Header: Depth=16308; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6309; GFX7-NEXT: v_mov_b32_e32 v9, v36310; GFX7-NEXT: v_mov_b32_e32 v8, v26311; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]6312; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc6313; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc6314; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc6315; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6316; GFX7-NEXT: buffer_wbinvl1_vol6317; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]6318; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]6319; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]6320; GFX7-NEXT: s_cbranch_execnz .LBB91_16321; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end6322; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]6323; GFX7-NEXT: v_mov_b32_e32 v0, s26324; GFX7-NEXT: v_mov_b32_e32 v1, s36325; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]6326; GFX7-NEXT: s_endpgm6327;6328; GFX8-LABEL: atomic_max_i64_ret_addr64:6329; GFX8: ; %bb.0: ; %entry6330; GFX8-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x246331; GFX8-NEXT: s_waitcnt lgkmcnt(0)6332; GFX8-NEXT: s_lshl_b64 s[6:7], s[6:7], 36333; GFX8-NEXT: s_add_u32 s0, s0, s66334; GFX8-NEXT: s_addc_u32 s1, s1, s76335; GFX8-NEXT: v_mov_b32_e32 v0, s06336; GFX8-NEXT: v_mov_b32_e32 v1, s16337; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]6338; GFX8-NEXT: s_mov_b64 s[0:1], 06339; GFX8-NEXT: v_mov_b32_e32 v4, s56340; GFX8-NEXT: v_mov_b32_e32 v5, s46341; GFX8-NEXT: .LBB91_1: ; %atomicrmw.start6342; GFX8-NEXT: ; =>This Inner Loop Header: Depth=16343; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6344; GFX8-NEXT: v_mov_b32_e32 v9, v36345; GFX8-NEXT: v_mov_b32_e32 v8, v26346; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]6347; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc6348; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc6349; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc6350; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6351; GFX8-NEXT: buffer_wbinvl1_vol6352; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]6353; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]6354; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]6355; GFX8-NEXT: s_cbranch_execnz .LBB91_16356; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end6357; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]6358; GFX8-NEXT: v_mov_b32_e32 v0, s26359; GFX8-NEXT: v_mov_b32_e32 v1, s36360; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]6361; GFX8-NEXT: s_endpgm6362;6363; GFX9-LABEL: atomic_max_i64_ret_addr64:6364; GFX9: ; %bb.0: ; %entry6365; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x246366; GFX9-NEXT: s_waitcnt lgkmcnt(0)6367; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 36368; GFX9-NEXT: s_add_u32 s0, s8, s06369; GFX9-NEXT: s_addc_u32 s1, s9, s16370; GFX9-NEXT: v_mov_b32_e32 v0, s06371; GFX9-NEXT: v_mov_b32_e32 v1, s16372; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]6373; GFX9-NEXT: s_mov_b64 s[0:1], 06374; GFX9-NEXT: v_mov_b32_e32 v4, s136375; GFX9-NEXT: v_mov_b32_e32 v5, s126376; GFX9-NEXT: .LBB91_1: ; %atomicrmw.start6377; GFX9-NEXT: ; =>This Inner Loop Header: Depth=16378; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6379; GFX9-NEXT: v_mov_b32_e32 v9, v36380; GFX9-NEXT: v_mov_b32_e32 v8, v26381; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9]6382; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc6383; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc6384; GFX9-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc6385; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6386; GFX9-NEXT: buffer_wbinvl1_vol6387; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]6388; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]6389; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]6390; GFX9-NEXT: s_cbranch_execnz .LBB91_16391; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end6392; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]6393; GFX9-NEXT: v_mov_b32_e32 v0, s106394; GFX9-NEXT: v_mov_b32_e32 v1, s116395; GFX9-NEXT: flat_store_dwordx2 v[0:1], v[2:3]6396; GFX9-NEXT: s_endpgm6397entry:6398 %ptr = getelementptr inbounds i64, ptr %out, i64 %index6399 %tmp0 = atomicrmw max ptr %ptr, i64 %in seq_cst, !noalias.addrspace !16400 store i64 %tmp0, ptr %out26401 ret void6402}6403 6404define void @flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {6405; GFX7-LABEL: flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory:6406; GFX7: ; %bb.0:6407; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6408; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v06409; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc6410; GFX7-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3]6411; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6412; GFX7-NEXT: buffer_wbinvl1_vol6413; GFX7-NEXT: s_setpc_b64 s[30:31]6414;6415; GFX8-LABEL: flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory:6416; GFX8: ; %bb.0:6417; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6418; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v06419; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc6420; GFX8-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3]6421; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6422; GFX8-NEXT: buffer_wbinvl1_vol6423; GFX8-NEXT: s_setpc_b64 s[30:31]6424;6425; GFX9-LABEL: flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory:6426; GFX9: ; %bb.0:6427; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6428; GFX9-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3] offset:326429; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6430; GFX9-NEXT: buffer_wbinvl1_vol6431; GFX9-NEXT: s_setpc_b64 s[30:31]6432 %gep = getelementptr inbounds i64, ptr %out, i64 46433 %tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !16434 ret void6435}6436 6437define i64 @flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {6438; GFX7-LABEL: flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory:6439; GFX7: ; %bb.0:6440; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6441; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v06442; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc6443; GFX7-NEXT: flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc6444; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6445; GFX7-NEXT: buffer_wbinvl1_vol6446; GFX7-NEXT: s_setpc_b64 s[30:31]6447;6448; GFX8-LABEL: flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory:6449; GFX8: ; %bb.0:6450; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6451; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v06452; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc6453; GFX8-NEXT: flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc6454; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6455; GFX8-NEXT: buffer_wbinvl1_vol6456; GFX8-NEXT: s_setpc_b64 s[30:31]6457;6458; GFX9-LABEL: flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory:6459; GFX9: ; %bb.0:6460; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6461; GFX9-NEXT: flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] offset:32 glc6462; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6463; GFX9-NEXT: buffer_wbinvl1_vol6464; GFX9-NEXT: s_setpc_b64 s[30:31]6465 %gep = getelementptr inbounds i64, ptr %out, i64 46466 %result = atomicrmw max ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !16467 ret i64 %result6468}6469 6470; ---------------------------------------------------------------------6471; atomicrmw umax6472; ---------------------------------------------------------------------6473 6474define void @flat_atomic_umax_i64_noret(ptr %ptr, i64 %in) {6475; GFX7-LABEL: flat_atomic_umax_i64_noret:6476; GFX7: ; %bb.0:6477; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6478; GFX7-NEXT: v_add_i32_e32 v4, vcc, 4, v06479; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc6480; GFX7-NEXT: flat_load_dword v6, v[0:1]6481; GFX7-NEXT: flat_load_dword v7, v[4:5]6482; GFX7-NEXT: s_mov_b64 s[4:5], 06483; GFX7-NEXT: .LBB94_1: ; %atomicrmw.start6484; GFX7-NEXT: ; =>This Inner Loop Header: Depth=16485; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6486; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]6487; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc6488; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc6489; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc6490; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6491; GFX7-NEXT: buffer_wbinvl1_vol6492; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]6493; GFX7-NEXT: v_mov_b32_e32 v7, v56494; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6495; GFX7-NEXT: v_mov_b32_e32 v6, v46496; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]6497; GFX7-NEXT: s_cbranch_execnz .LBB94_16498; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end6499; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]6500; GFX7-NEXT: s_setpc_b64 s[30:31]6501;6502; GFX8-LABEL: flat_atomic_umax_i64_noret:6503; GFX8: ; %bb.0:6504; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6505; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v06506; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc6507; GFX8-NEXT: flat_load_dword v6, v[0:1]6508; GFX8-NEXT: flat_load_dword v7, v[4:5]6509; GFX8-NEXT: s_mov_b64 s[4:5], 06510; GFX8-NEXT: .LBB94_1: ; %atomicrmw.start6511; GFX8-NEXT: ; =>This Inner Loop Header: Depth=16512; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6513; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]6514; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc6515; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc6516; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc6517; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6518; GFX8-NEXT: buffer_wbinvl1_vol6519; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]6520; GFX8-NEXT: v_mov_b32_e32 v7, v56521; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6522; GFX8-NEXT: v_mov_b32_e32 v6, v46523; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]6524; GFX8-NEXT: s_cbranch_execnz .LBB94_16525; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end6526; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]6527; GFX8-NEXT: s_setpc_b64 s[30:31]6528;6529; GFX9-LABEL: flat_atomic_umax_i64_noret:6530; GFX9: ; %bb.0:6531; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6532; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1]6533; GFX9-NEXT: s_mov_b64 s[4:5], 06534; GFX9-NEXT: .LBB94_1: ; %atomicrmw.start6535; GFX9-NEXT: ; =>This Inner Loop Header: Depth=16536; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6537; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]6538; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc6539; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc6540; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc6541; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6542; GFX9-NEXT: buffer_wbinvl1_vol6543; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]6544; GFX9-NEXT: v_mov_b32_e32 v7, v56545; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6546; GFX9-NEXT: v_mov_b32_e32 v6, v46547; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]6548; GFX9-NEXT: s_cbranch_execnz .LBB94_16549; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end6550; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]6551; GFX9-NEXT: s_setpc_b64 s[30:31]6552 %tmp0 = atomicrmw umax ptr %ptr, i64 %in seq_cst, !noalias.addrspace !16553 ret void6554}6555 6556define void @flat_atomic_umax_i64_noret_offset(ptr %out, i64 %in) {6557; GFX7-LABEL: flat_atomic_umax_i64_noret_offset:6558; GFX7: ; %bb.0:6559; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6560; GFX7-NEXT: v_add_i32_e32 v8, vcc, 32, v06561; GFX7-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc6562; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v06563; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc6564; GFX7-NEXT: flat_load_dword v7, v[0:1]6565; GFX7-NEXT: flat_load_dword v6, v[8:9]6566; GFX7-NEXT: s_mov_b64 s[4:5], 06567; GFX7-NEXT: .LBB95_1: ; %atomicrmw.start6568; GFX7-NEXT: ; =>This Inner Loop Header: Depth=16569; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6570; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]6571; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc6572; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc6573; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc6574; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6575; GFX7-NEXT: buffer_wbinvl1_vol6576; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]6577; GFX7-NEXT: v_mov_b32_e32 v7, v16578; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6579; GFX7-NEXT: v_mov_b32_e32 v6, v06580; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]6581; GFX7-NEXT: s_cbranch_execnz .LBB95_16582; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end6583; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]6584; GFX7-NEXT: s_setpc_b64 s[30:31]6585;6586; GFX8-LABEL: flat_atomic_umax_i64_noret_offset:6587; GFX8: ; %bb.0:6588; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6589; GFX8-NEXT: v_add_u32_e32 v8, vcc, 32, v06590; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc6591; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v06592; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc6593; GFX8-NEXT: flat_load_dword v7, v[0:1]6594; GFX8-NEXT: flat_load_dword v6, v[8:9]6595; GFX8-NEXT: s_mov_b64 s[4:5], 06596; GFX8-NEXT: .LBB95_1: ; %atomicrmw.start6597; GFX8-NEXT: ; =>This Inner Loop Header: Depth=16598; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6599; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]6600; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc6601; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc6602; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc6603; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6604; GFX8-NEXT: buffer_wbinvl1_vol6605; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]6606; GFX8-NEXT: v_mov_b32_e32 v7, v16607; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6608; GFX8-NEXT: v_mov_b32_e32 v6, v06609; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]6610; GFX8-NEXT: s_cbranch_execnz .LBB95_16611; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end6612; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]6613; GFX8-NEXT: s_setpc_b64 s[30:31]6614;6615; GFX9-LABEL: flat_atomic_umax_i64_noret_offset:6616; GFX9: ; %bb.0:6617; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6618; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:326619; GFX9-NEXT: s_mov_b64 s[4:5], 06620; GFX9-NEXT: .LBB95_1: ; %atomicrmw.start6621; GFX9-NEXT: ; =>This Inner Loop Header: Depth=16622; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6623; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]6624; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc6625; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc6626; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc6627; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6628; GFX9-NEXT: buffer_wbinvl1_vol6629; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]6630; GFX9-NEXT: v_mov_b32_e32 v7, v56631; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6632; GFX9-NEXT: v_mov_b32_e32 v6, v46633; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]6634; GFX9-NEXT: s_cbranch_execnz .LBB95_16635; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end6636; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]6637; GFX9-NEXT: s_setpc_b64 s[30:31]6638 %gep = getelementptr inbounds i64, ptr %out, i64 46639 %tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !16640 ret void6641}6642 6643define i64 @flat_atomic_umax_i64_ret(ptr %ptr, i64 %in) {6644; GFX7-LABEL: flat_atomic_umax_i64_ret:6645; GFX7: ; %bb.0:6646; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6647; GFX7-NEXT: v_add_i32_e32 v5, vcc, 4, v06648; GFX7-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc6649; GFX7-NEXT: flat_load_dword v4, v[0:1]6650; GFX7-NEXT: flat_load_dword v5, v[5:6]6651; GFX7-NEXT: s_mov_b64 s[4:5], 06652; GFX7-NEXT: .LBB96_1: ; %atomicrmw.start6653; GFX7-NEXT: ; =>This Inner Loop Header: Depth=16654; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6655; GFX7-NEXT: v_mov_b32_e32 v7, v56656; GFX7-NEXT: v_mov_b32_e32 v6, v46657; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]6658; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc6659; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc6660; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc6661; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6662; GFX7-NEXT: buffer_wbinvl1_vol6663; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]6664; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6665; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]6666; GFX7-NEXT: s_cbranch_execnz .LBB96_16667; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end6668; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]6669; GFX7-NEXT: v_mov_b32_e32 v0, v46670; GFX7-NEXT: v_mov_b32_e32 v1, v56671; GFX7-NEXT: s_setpc_b64 s[30:31]6672;6673; GFX8-LABEL: flat_atomic_umax_i64_ret:6674; GFX8: ; %bb.0:6675; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6676; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v06677; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc6678; GFX8-NEXT: flat_load_dword v4, v[0:1]6679; GFX8-NEXT: flat_load_dword v5, v[5:6]6680; GFX8-NEXT: s_mov_b64 s[4:5], 06681; GFX8-NEXT: .LBB96_1: ; %atomicrmw.start6682; GFX8-NEXT: ; =>This Inner Loop Header: Depth=16683; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6684; GFX8-NEXT: v_mov_b32_e32 v7, v56685; GFX8-NEXT: v_mov_b32_e32 v6, v46686; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]6687; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc6688; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc6689; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc6690; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6691; GFX8-NEXT: buffer_wbinvl1_vol6692; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]6693; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6694; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]6695; GFX8-NEXT: s_cbranch_execnz .LBB96_16696; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end6697; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]6698; GFX8-NEXT: v_mov_b32_e32 v0, v46699; GFX8-NEXT: v_mov_b32_e32 v1, v56700; GFX8-NEXT: s_setpc_b64 s[30:31]6701;6702; GFX9-LABEL: flat_atomic_umax_i64_ret:6703; GFX9: ; %bb.0:6704; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6705; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1]6706; GFX9-NEXT: s_mov_b64 s[4:5], 06707; GFX9-NEXT: .LBB96_1: ; %atomicrmw.start6708; GFX9-NEXT: ; =>This Inner Loop Header: Depth=16709; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6710; GFX9-NEXT: v_mov_b32_e32 v7, v56711; GFX9-NEXT: v_mov_b32_e32 v6, v46712; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]6713; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc6714; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc6715; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc6716; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6717; GFX9-NEXT: buffer_wbinvl1_vol6718; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]6719; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6720; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]6721; GFX9-NEXT: s_cbranch_execnz .LBB96_16722; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end6723; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]6724; GFX9-NEXT: v_mov_b32_e32 v0, v46725; GFX9-NEXT: v_mov_b32_e32 v1, v56726; GFX9-NEXT: s_setpc_b64 s[30:31]6727 %result = atomicrmw umax ptr %ptr, i64 %in seq_cst, !noalias.addrspace !16728 ret i64 %result6729}6730 6731define i64 @flat_atomic_umax_i64_ret_offset(ptr %out, i64 %in) {6732; GFX7-LABEL: flat_atomic_umax_i64_ret_offset:6733; GFX7: ; %bb.0:6734; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6735; GFX7-NEXT: v_add_i32_e32 v4, vcc, 32, v06736; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc6737; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v06738; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc6739; GFX7-NEXT: flat_load_dword v1, v[0:1]6740; GFX7-NEXT: flat_load_dword v0, v[4:5]6741; GFX7-NEXT: s_mov_b64 s[4:5], 06742; GFX7-NEXT: .LBB97_1: ; %atomicrmw.start6743; GFX7-NEXT: ; =>This Inner Loop Header: Depth=16744; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6745; GFX7-NEXT: v_mov_b32_e32 v9, v16746; GFX7-NEXT: v_mov_b32_e32 v8, v06747; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]6748; GFX7-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc6749; GFX7-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc6750; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc6751; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6752; GFX7-NEXT: buffer_wbinvl1_vol6753; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]6754; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6755; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]6756; GFX7-NEXT: s_cbranch_execnz .LBB97_16757; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end6758; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]6759; GFX7-NEXT: s_setpc_b64 s[30:31]6760;6761; GFX8-LABEL: flat_atomic_umax_i64_ret_offset:6762; GFX8: ; %bb.0:6763; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6764; GFX8-NEXT: v_add_u32_e32 v4, vcc, 32, v06765; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc6766; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v06767; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc6768; GFX8-NEXT: flat_load_dword v1, v[0:1]6769; GFX8-NEXT: flat_load_dword v0, v[4:5]6770; GFX8-NEXT: s_mov_b64 s[4:5], 06771; GFX8-NEXT: .LBB97_1: ; %atomicrmw.start6772; GFX8-NEXT: ; =>This Inner Loop Header: Depth=16773; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6774; GFX8-NEXT: v_mov_b32_e32 v9, v16775; GFX8-NEXT: v_mov_b32_e32 v8, v06776; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]6777; GFX8-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc6778; GFX8-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc6779; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc6780; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6781; GFX8-NEXT: buffer_wbinvl1_vol6782; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]6783; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6784; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]6785; GFX8-NEXT: s_cbranch_execnz .LBB97_16786; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end6787; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]6788; GFX8-NEXT: s_setpc_b64 s[30:31]6789;6790; GFX9-LABEL: flat_atomic_umax_i64_ret_offset:6791; GFX9: ; %bb.0:6792; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6793; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:326794; GFX9-NEXT: s_mov_b64 s[4:5], 06795; GFX9-NEXT: .LBB97_1: ; %atomicrmw.start6796; GFX9-NEXT: ; =>This Inner Loop Header: Depth=16797; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6798; GFX9-NEXT: v_mov_b32_e32 v7, v56799; GFX9-NEXT: v_mov_b32_e32 v6, v46800; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]6801; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc6802; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc6803; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc6804; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6805; GFX9-NEXT: buffer_wbinvl1_vol6806; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]6807; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6808; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]6809; GFX9-NEXT: s_cbranch_execnz .LBB97_16810; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end6811; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]6812; GFX9-NEXT: v_mov_b32_e32 v0, v46813; GFX9-NEXT: v_mov_b32_e32 v1, v56814; GFX9-NEXT: s_setpc_b64 s[30:31]6815 %gep = getelementptr inbounds i64, ptr %out, i64 46816 %result = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !16817 ret i64 %result6818}6819 6820define amdgpu_gfx void @flat_atomic_umax_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {6821; GFX7-LABEL: flat_atomic_umax_i64_noret_scalar:6822; GFX7: ; %bb.0:6823; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6824; GFX7-NEXT: v_mov_b32_e32 v0, s46825; GFX7-NEXT: s_add_u32 s34, s4, 46826; GFX7-NEXT: v_mov_b32_e32 v1, s56827; GFX7-NEXT: s_addc_u32 s35, s5, 06828; GFX7-NEXT: v_mov_b32_e32 v3, s346829; GFX7-NEXT: v_mov_b32_e32 v4, s356830; GFX7-NEXT: flat_load_dword v2, v[0:1]6831; GFX7-NEXT: flat_load_dword v3, v[3:4]6832; GFX7-NEXT: v_mov_b32_e32 v4, s46833; GFX7-NEXT: s_mov_b64 s[34:35], 06834; GFX7-NEXT: v_mov_b32_e32 v6, s76835; GFX7-NEXT: v_mov_b32_e32 v7, s66836; GFX7-NEXT: v_mov_b32_e32 v5, s56837; GFX7-NEXT: .LBB98_1: ; %atomicrmw.start6838; GFX7-NEXT: ; =>This Inner Loop Header: Depth=16839; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6840; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]6841; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc6842; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc6843; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc6844; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6845; GFX7-NEXT: buffer_wbinvl1_vol6846; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]6847; GFX7-NEXT: v_mov_b32_e32 v3, v16848; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]6849; GFX7-NEXT: v_mov_b32_e32 v2, v06850; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]6851; GFX7-NEXT: s_cbranch_execnz .LBB98_16852; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end6853; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]6854; GFX7-NEXT: s_setpc_b64 s[30:31]6855;6856; GFX8-LABEL: flat_atomic_umax_i64_noret_scalar:6857; GFX8: ; %bb.0:6858; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6859; GFX8-NEXT: v_mov_b32_e32 v0, s46860; GFX8-NEXT: s_add_u32 s34, s4, 46861; GFX8-NEXT: v_mov_b32_e32 v1, s56862; GFX8-NEXT: s_addc_u32 s35, s5, 06863; GFX8-NEXT: v_mov_b32_e32 v3, s346864; GFX8-NEXT: v_mov_b32_e32 v4, s356865; GFX8-NEXT: flat_load_dword v2, v[0:1]6866; GFX8-NEXT: flat_load_dword v3, v[3:4]6867; GFX8-NEXT: v_mov_b32_e32 v4, s46868; GFX8-NEXT: s_mov_b64 s[34:35], 06869; GFX8-NEXT: v_mov_b32_e32 v6, s76870; GFX8-NEXT: v_mov_b32_e32 v7, s66871; GFX8-NEXT: v_mov_b32_e32 v5, s56872; GFX8-NEXT: .LBB98_1: ; %atomicrmw.start6873; GFX8-NEXT: ; =>This Inner Loop Header: Depth=16874; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6875; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]6876; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc6877; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc6878; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc6879; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6880; GFX8-NEXT: buffer_wbinvl1_vol6881; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]6882; GFX8-NEXT: v_mov_b32_e32 v3, v16883; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]6884; GFX8-NEXT: v_mov_b32_e32 v2, v06885; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]6886; GFX8-NEXT: s_cbranch_execnz .LBB98_16887; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end6888; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]6889; GFX8-NEXT: s_setpc_b64 s[30:31]6890;6891; GFX9-LABEL: flat_atomic_umax_i64_noret_scalar:6892; GFX9: ; %bb.0:6893; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6894; GFX9-NEXT: v_mov_b32_e32 v0, s46895; GFX9-NEXT: v_mov_b32_e32 v1, s56896; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]6897; GFX9-NEXT: v_mov_b32_e32 v4, s46898; GFX9-NEXT: s_mov_b64 s[34:35], 06899; GFX9-NEXT: v_mov_b32_e32 v6, s76900; GFX9-NEXT: v_mov_b32_e32 v7, s66901; GFX9-NEXT: v_mov_b32_e32 v5, s56902; GFX9-NEXT: .LBB98_1: ; %atomicrmw.start6903; GFX9-NEXT: ; =>This Inner Loop Header: Depth=16904; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6905; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]6906; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc6907; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc6908; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc6909; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6910; GFX9-NEXT: buffer_wbinvl1_vol6911; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]6912; GFX9-NEXT: v_mov_b32_e32 v3, v16913; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]6914; GFX9-NEXT: v_mov_b32_e32 v2, v06915; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]6916; GFX9-NEXT: s_cbranch_execnz .LBB98_16917; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end6918; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]6919; GFX9-NEXT: s_setpc_b64 s[30:31]6920 %tmp0 = atomicrmw umax ptr %ptr, i64 %in seq_cst, !noalias.addrspace !16921 ret void6922}6923 6924define amdgpu_gfx void @flat_atomic_umax_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {6925; GFX7-LABEL: flat_atomic_umax_i64_noret_offset_scalar:6926; GFX7: ; %bb.0:6927; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6928; GFX7-NEXT: s_add_u32 s34, s4, 326929; GFX7-NEXT: s_addc_u32 s35, s5, 06930; GFX7-NEXT: s_add_u32 s36, s4, 366931; GFX7-NEXT: s_addc_u32 s37, s5, 06932; GFX7-NEXT: v_mov_b32_e32 v0, s366933; GFX7-NEXT: v_mov_b32_e32 v1, s376934; GFX7-NEXT: v_mov_b32_e32 v4, s346935; GFX7-NEXT: v_mov_b32_e32 v5, s356936; GFX7-NEXT: flat_load_dword v3, v[0:1]6937; GFX7-NEXT: flat_load_dword v2, v[4:5]6938; GFX7-NEXT: s_mov_b64 s[34:35], 06939; GFX7-NEXT: v_mov_b32_e32 v6, s76940; GFX7-NEXT: v_mov_b32_e32 v7, s66941; GFX7-NEXT: .LBB99_1: ; %atomicrmw.start6942; GFX7-NEXT: ; =>This Inner Loop Header: Depth=16943; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6944; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]6945; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc6946; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc6947; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc6948; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6949; GFX7-NEXT: buffer_wbinvl1_vol6950; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]6951; GFX7-NEXT: v_mov_b32_e32 v3, v16952; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]6953; GFX7-NEXT: v_mov_b32_e32 v2, v06954; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]6955; GFX7-NEXT: s_cbranch_execnz .LBB99_16956; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end6957; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]6958; GFX7-NEXT: s_setpc_b64 s[30:31]6959;6960; GFX8-LABEL: flat_atomic_umax_i64_noret_offset_scalar:6961; GFX8: ; %bb.0:6962; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6963; GFX8-NEXT: s_add_u32 s34, s4, 326964; GFX8-NEXT: s_addc_u32 s35, s5, 06965; GFX8-NEXT: s_add_u32 s36, s4, 366966; GFX8-NEXT: s_addc_u32 s37, s5, 06967; GFX8-NEXT: v_mov_b32_e32 v0, s366968; GFX8-NEXT: v_mov_b32_e32 v1, s376969; GFX8-NEXT: v_mov_b32_e32 v4, s346970; GFX8-NEXT: v_mov_b32_e32 v5, s356971; GFX8-NEXT: flat_load_dword v3, v[0:1]6972; GFX8-NEXT: flat_load_dword v2, v[4:5]6973; GFX8-NEXT: s_mov_b64 s[34:35], 06974; GFX8-NEXT: v_mov_b32_e32 v6, s76975; GFX8-NEXT: v_mov_b32_e32 v7, s66976; GFX8-NEXT: .LBB99_1: ; %atomicrmw.start6977; GFX8-NEXT: ; =>This Inner Loop Header: Depth=16978; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6979; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]6980; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc6981; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc6982; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc6983; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6984; GFX8-NEXT: buffer_wbinvl1_vol6985; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]6986; GFX8-NEXT: v_mov_b32_e32 v3, v16987; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]6988; GFX8-NEXT: v_mov_b32_e32 v2, v06989; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]6990; GFX8-NEXT: s_cbranch_execnz .LBB99_16991; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end6992; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]6993; GFX8-NEXT: s_setpc_b64 s[30:31]6994;6995; GFX9-LABEL: flat_atomic_umax_i64_noret_offset_scalar:6996; GFX9: ; %bb.0:6997; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6998; GFX9-NEXT: v_mov_b32_e32 v0, s46999; GFX9-NEXT: v_mov_b32_e32 v1, s57000; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:327001; GFX9-NEXT: v_mov_b32_e32 v4, s47002; GFX9-NEXT: s_mov_b64 s[34:35], 07003; GFX9-NEXT: v_mov_b32_e32 v6, s77004; GFX9-NEXT: v_mov_b32_e32 v7, s67005; GFX9-NEXT: v_mov_b32_e32 v5, s57006; GFX9-NEXT: .LBB99_1: ; %atomicrmw.start7007; GFX9-NEXT: ; =>This Inner Loop Header: Depth=17008; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7009; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]7010; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc7011; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc7012; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc7013; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7014; GFX9-NEXT: buffer_wbinvl1_vol7015; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]7016; GFX9-NEXT: v_mov_b32_e32 v3, v17017; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]7018; GFX9-NEXT: v_mov_b32_e32 v2, v07019; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]7020; GFX9-NEXT: s_cbranch_execnz .LBB99_17021; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end7022; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]7023; GFX9-NEXT: s_setpc_b64 s[30:31]7024 %gep = getelementptr inbounds i64, ptr %out, i64 47025 %tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !17026 ret void7027}7028 7029define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {7030; GFX7-LABEL: flat_atomic_umax_i64_ret_scalar:7031; GFX7: ; %bb.0:7032; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7033; GFX7-NEXT: v_mov_b32_e32 v0, s47034; GFX7-NEXT: s_add_u32 s34, s4, 47035; GFX7-NEXT: v_mov_b32_e32 v1, s57036; GFX7-NEXT: s_addc_u32 s35, s5, 07037; GFX7-NEXT: v_mov_b32_e32 v2, s347038; GFX7-NEXT: v_mov_b32_e32 v3, s357039; GFX7-NEXT: flat_load_dword v0, v[0:1]7040; GFX7-NEXT: flat_load_dword v1, v[2:3]7041; GFX7-NEXT: v_mov_b32_e32 v2, s47042; GFX7-NEXT: s_mov_b64 s[34:35], 07043; GFX7-NEXT: v_mov_b32_e32 v4, s77044; GFX7-NEXT: v_mov_b32_e32 v5, s67045; GFX7-NEXT: v_mov_b32_e32 v3, s57046; GFX7-NEXT: .LBB100_1: ; %atomicrmw.start7047; GFX7-NEXT: ; =>This Inner Loop Header: Depth=17048; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7049; GFX7-NEXT: v_mov_b32_e32 v9, v17050; GFX7-NEXT: v_mov_b32_e32 v8, v07051; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]7052; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc7053; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc7054; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc7055; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7056; GFX7-NEXT: buffer_wbinvl1_vol7057; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]7058; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]7059; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]7060; GFX7-NEXT: s_cbranch_execnz .LBB100_17061; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end7062; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]7063; GFX7-NEXT: s_setpc_b64 s[30:31]7064;7065; GFX8-LABEL: flat_atomic_umax_i64_ret_scalar:7066; GFX8: ; %bb.0:7067; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7068; GFX8-NEXT: v_mov_b32_e32 v0, s47069; GFX8-NEXT: s_add_u32 s34, s4, 47070; GFX8-NEXT: v_mov_b32_e32 v1, s57071; GFX8-NEXT: s_addc_u32 s35, s5, 07072; GFX8-NEXT: v_mov_b32_e32 v2, s347073; GFX8-NEXT: v_mov_b32_e32 v3, s357074; GFX8-NEXT: flat_load_dword v0, v[0:1]7075; GFX8-NEXT: flat_load_dword v1, v[2:3]7076; GFX8-NEXT: v_mov_b32_e32 v2, s47077; GFX8-NEXT: s_mov_b64 s[34:35], 07078; GFX8-NEXT: v_mov_b32_e32 v4, s77079; GFX8-NEXT: v_mov_b32_e32 v5, s67080; GFX8-NEXT: v_mov_b32_e32 v3, s57081; GFX8-NEXT: .LBB100_1: ; %atomicrmw.start7082; GFX8-NEXT: ; =>This Inner Loop Header: Depth=17083; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7084; GFX8-NEXT: v_mov_b32_e32 v9, v17085; GFX8-NEXT: v_mov_b32_e32 v8, v07086; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]7087; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc7088; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc7089; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc7090; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7091; GFX8-NEXT: buffer_wbinvl1_vol7092; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]7093; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]7094; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]7095; GFX8-NEXT: s_cbranch_execnz .LBB100_17096; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end7097; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]7098; GFX8-NEXT: s_setpc_b64 s[30:31]7099;7100; GFX9-LABEL: flat_atomic_umax_i64_ret_scalar:7101; GFX9: ; %bb.0:7102; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7103; GFX9-NEXT: v_mov_b32_e32 v0, s47104; GFX9-NEXT: v_mov_b32_e32 v1, s57105; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]7106; GFX9-NEXT: v_mov_b32_e32 v2, s47107; GFX9-NEXT: s_mov_b64 s[34:35], 07108; GFX9-NEXT: v_mov_b32_e32 v4, s77109; GFX9-NEXT: v_mov_b32_e32 v5, s67110; GFX9-NEXT: v_mov_b32_e32 v3, s57111; GFX9-NEXT: .LBB100_1: ; %atomicrmw.start7112; GFX9-NEXT: ; =>This Inner Loop Header: Depth=17113; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7114; GFX9-NEXT: v_mov_b32_e32 v9, v17115; GFX9-NEXT: v_mov_b32_e32 v8, v07116; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]7117; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc7118; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc7119; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc7120; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7121; GFX9-NEXT: buffer_wbinvl1_vol7122; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]7123; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]7124; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]7125; GFX9-NEXT: s_cbranch_execnz .LBB100_17126; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end7127; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]7128; GFX9-NEXT: s_setpc_b64 s[30:31]7129 %result = atomicrmw umax ptr %ptr, i64 %in seq_cst, !noalias.addrspace !17130 ret i64 %result7131}7132 7133define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {7134; GFX7-LABEL: flat_atomic_umax_i64_ret_offset_scalar:7135; GFX7: ; %bb.0:7136; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7137; GFX7-NEXT: s_add_u32 s34, s4, 327138; GFX7-NEXT: s_addc_u32 s35, s5, 07139; GFX7-NEXT: s_add_u32 s36, s4, 367140; GFX7-NEXT: s_addc_u32 s37, s5, 07141; GFX7-NEXT: v_mov_b32_e32 v0, s367142; GFX7-NEXT: v_mov_b32_e32 v1, s377143; GFX7-NEXT: v_mov_b32_e32 v2, s347144; GFX7-NEXT: v_mov_b32_e32 v3, s357145; GFX7-NEXT: flat_load_dword v1, v[0:1]7146; GFX7-NEXT: flat_load_dword v0, v[2:3]7147; GFX7-NEXT: s_mov_b64 s[34:35], 07148; GFX7-NEXT: v_mov_b32_e32 v4, s77149; GFX7-NEXT: v_mov_b32_e32 v5, s67150; GFX7-NEXT: .LBB101_1: ; %atomicrmw.start7151; GFX7-NEXT: ; =>This Inner Loop Header: Depth=17152; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7153; GFX7-NEXT: v_mov_b32_e32 v9, v17154; GFX7-NEXT: v_mov_b32_e32 v8, v07155; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]7156; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc7157; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc7158; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc7159; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7160; GFX7-NEXT: buffer_wbinvl1_vol7161; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]7162; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]7163; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]7164; GFX7-NEXT: s_cbranch_execnz .LBB101_17165; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end7166; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]7167; GFX7-NEXT: s_setpc_b64 s[30:31]7168;7169; GFX8-LABEL: flat_atomic_umax_i64_ret_offset_scalar:7170; GFX8: ; %bb.0:7171; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7172; GFX8-NEXT: s_add_u32 s34, s4, 327173; GFX8-NEXT: s_addc_u32 s35, s5, 07174; GFX8-NEXT: s_add_u32 s36, s4, 367175; GFX8-NEXT: s_addc_u32 s37, s5, 07176; GFX8-NEXT: v_mov_b32_e32 v0, s367177; GFX8-NEXT: v_mov_b32_e32 v1, s377178; GFX8-NEXT: v_mov_b32_e32 v2, s347179; GFX8-NEXT: v_mov_b32_e32 v3, s357180; GFX8-NEXT: flat_load_dword v1, v[0:1]7181; GFX8-NEXT: flat_load_dword v0, v[2:3]7182; GFX8-NEXT: s_mov_b64 s[34:35], 07183; GFX8-NEXT: v_mov_b32_e32 v4, s77184; GFX8-NEXT: v_mov_b32_e32 v5, s67185; GFX8-NEXT: .LBB101_1: ; %atomicrmw.start7186; GFX8-NEXT: ; =>This Inner Loop Header: Depth=17187; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7188; GFX8-NEXT: v_mov_b32_e32 v9, v17189; GFX8-NEXT: v_mov_b32_e32 v8, v07190; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]7191; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc7192; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc7193; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc7194; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7195; GFX8-NEXT: buffer_wbinvl1_vol7196; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]7197; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]7198; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]7199; GFX8-NEXT: s_cbranch_execnz .LBB101_17200; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end7201; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]7202; GFX8-NEXT: s_setpc_b64 s[30:31]7203;7204; GFX9-LABEL: flat_atomic_umax_i64_ret_offset_scalar:7205; GFX9: ; %bb.0:7206; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7207; GFX9-NEXT: v_mov_b32_e32 v0, s47208; GFX9-NEXT: v_mov_b32_e32 v1, s57209; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:327210; GFX9-NEXT: v_mov_b32_e32 v2, s47211; GFX9-NEXT: s_mov_b64 s[34:35], 07212; GFX9-NEXT: v_mov_b32_e32 v4, s77213; GFX9-NEXT: v_mov_b32_e32 v5, s67214; GFX9-NEXT: v_mov_b32_e32 v3, s57215; GFX9-NEXT: .LBB101_1: ; %atomicrmw.start7216; GFX9-NEXT: ; =>This Inner Loop Header: Depth=17217; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7218; GFX9-NEXT: v_mov_b32_e32 v9, v17219; GFX9-NEXT: v_mov_b32_e32 v8, v07220; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]7221; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc7222; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc7223; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc7224; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7225; GFX9-NEXT: buffer_wbinvl1_vol7226; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]7227; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]7228; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]7229; GFX9-NEXT: s_cbranch_execnz .LBB101_17230; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end7231; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]7232; GFX9-NEXT: s_setpc_b64 s[30:31]7233 %gep = getelementptr inbounds i64, ptr %out, i64 47234 %result = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !17235 ret i64 %result7236}7237 7238define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {7239; GFX7-LABEL: atomic_umax_i64_addr64_offset:7240; GFX7: ; %bb.0: ; %entry7241; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd7242; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x97243; GFX7-NEXT: s_waitcnt lgkmcnt(0)7244; GFX7-NEXT: s_lshl_b64 s[4:5], s[6:7], 37245; GFX7-NEXT: s_add_u32 s0, s0, s47246; GFX7-NEXT: s_addc_u32 s1, s1, s57247; GFX7-NEXT: s_add_u32 s0, s0, 327248; GFX7-NEXT: s_addc_u32 s1, s1, 07249; GFX7-NEXT: v_mov_b32_e32 v5, s17250; GFX7-NEXT: v_mov_b32_e32 v4, s07251; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5]7252; GFX7-NEXT: s_mov_b64 s[0:1], 07253; GFX7-NEXT: v_mov_b32_e32 v6, s37254; GFX7-NEXT: v_mov_b32_e32 v7, s27255; GFX7-NEXT: .LBB102_1: ; %atomicrmw.start7256; GFX7-NEXT: ; =>This Inner Loop Header: Depth=17257; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7258; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]7259; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc7260; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc7261; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc7262; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7263; GFX7-NEXT: buffer_wbinvl1_vol7264; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]7265; GFX7-NEXT: v_mov_b32_e32 v3, v17266; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]7267; GFX7-NEXT: v_mov_b32_e32 v2, v07268; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]7269; GFX7-NEXT: s_cbranch_execnz .LBB102_17270; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end7271; GFX7-NEXT: s_endpgm7272;7273; GFX8-LABEL: atomic_umax_i64_addr64_offset:7274; GFX8: ; %bb.0: ; %entry7275; GFX8-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x347276; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247277; GFX8-NEXT: s_waitcnt lgkmcnt(0)7278; GFX8-NEXT: s_lshl_b64 s[4:5], s[6:7], 37279; GFX8-NEXT: s_add_u32 s0, s0, s47280; GFX8-NEXT: s_addc_u32 s1, s1, s57281; GFX8-NEXT: s_add_u32 s0, s0, 327282; GFX8-NEXT: s_addc_u32 s1, s1, 07283; GFX8-NEXT: v_mov_b32_e32 v5, s17284; GFX8-NEXT: v_mov_b32_e32 v4, s07285; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5]7286; GFX8-NEXT: s_mov_b64 s[0:1], 07287; GFX8-NEXT: v_mov_b32_e32 v6, s37288; GFX8-NEXT: v_mov_b32_e32 v7, s27289; GFX8-NEXT: .LBB102_1: ; %atomicrmw.start7290; GFX8-NEXT: ; =>This Inner Loop Header: Depth=17291; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7292; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]7293; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc7294; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc7295; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc7296; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7297; GFX8-NEXT: buffer_wbinvl1_vol7298; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]7299; GFX8-NEXT: v_mov_b32_e32 v3, v17300; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]7301; GFX8-NEXT: v_mov_b32_e32 v2, v07302; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]7303; GFX8-NEXT: s_cbranch_execnz .LBB102_17304; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end7305; GFX8-NEXT: s_endpgm7306;7307; GFX9-LABEL: atomic_umax_i64_addr64_offset:7308; GFX9: ; %bb.0: ; %entry7309; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x347310; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247311; GFX9-NEXT: s_waitcnt lgkmcnt(0)7312; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 37313; GFX9-NEXT: s_add_u32 s0, s0, s47314; GFX9-NEXT: s_addc_u32 s1, s1, s57315; GFX9-NEXT: v_mov_b32_e32 v5, s17316; GFX9-NEXT: v_mov_b32_e32 v4, s07317; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[4:5] offset:327318; GFX9-NEXT: s_mov_b64 s[0:1], 07319; GFX9-NEXT: v_mov_b32_e32 v6, s37320; GFX9-NEXT: v_mov_b32_e32 v7, s27321; GFX9-NEXT: .LBB102_1: ; %atomicrmw.start7322; GFX9-NEXT: ; =>This Inner Loop Header: Depth=17323; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7324; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]7325; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc7326; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc7327; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc7328; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7329; GFX9-NEXT: buffer_wbinvl1_vol7330; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]7331; GFX9-NEXT: v_mov_b32_e32 v3, v17332; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]7333; GFX9-NEXT: v_mov_b32_e32 v2, v07334; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]7335; GFX9-NEXT: s_cbranch_execnz .LBB102_17336; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end7337; GFX9-NEXT: s_endpgm7338entry:7339 %ptr = getelementptr inbounds i64, ptr %out, i64 %index7340 %gep = getelementptr inbounds i64, ptr %ptr, i64 47341 %tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !17342 ret void7343}7344 7345define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {7346; GFX7-LABEL: atomic_umax_i64_ret_addr64_offset:7347; GFX7: ; %bb.0: ; %entry7348; GFX7-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x97349; GFX7-NEXT: s_waitcnt lgkmcnt(0)7350; GFX7-NEXT: s_lshl_b64 s[6:7], s[6:7], 37351; GFX7-NEXT: s_add_u32 s0, s0, s67352; GFX7-NEXT: s_addc_u32 s1, s1, s77353; GFX7-NEXT: s_add_u32 s0, s0, 327354; GFX7-NEXT: s_addc_u32 s1, s1, 07355; GFX7-NEXT: v_mov_b32_e32 v0, s07356; GFX7-NEXT: v_mov_b32_e32 v1, s17357; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]7358; GFX7-NEXT: s_mov_b64 s[0:1], 07359; GFX7-NEXT: v_mov_b32_e32 v4, s57360; GFX7-NEXT: v_mov_b32_e32 v5, s47361; GFX7-NEXT: .LBB103_1: ; %atomicrmw.start7362; GFX7-NEXT: ; =>This Inner Loop Header: Depth=17363; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7364; GFX7-NEXT: v_mov_b32_e32 v9, v37365; GFX7-NEXT: v_mov_b32_e32 v8, v27366; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]7367; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc7368; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc7369; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc7370; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7371; GFX7-NEXT: buffer_wbinvl1_vol7372; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]7373; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]7374; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]7375; GFX7-NEXT: s_cbranch_execnz .LBB103_17376; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end7377; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]7378; GFX7-NEXT: v_mov_b32_e32 v0, s27379; GFX7-NEXT: v_mov_b32_e32 v1, s37380; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]7381; GFX7-NEXT: s_endpgm7382;7383; GFX8-LABEL: atomic_umax_i64_ret_addr64_offset:7384; GFX8: ; %bb.0: ; %entry7385; GFX8-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x247386; GFX8-NEXT: s_waitcnt lgkmcnt(0)7387; GFX8-NEXT: s_lshl_b64 s[6:7], s[6:7], 37388; GFX8-NEXT: s_add_u32 s0, s0, s67389; GFX8-NEXT: s_addc_u32 s1, s1, s77390; GFX8-NEXT: s_add_u32 s0, s0, 327391; GFX8-NEXT: s_addc_u32 s1, s1, 07392; GFX8-NEXT: v_mov_b32_e32 v0, s07393; GFX8-NEXT: v_mov_b32_e32 v1, s17394; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]7395; GFX8-NEXT: s_mov_b64 s[0:1], 07396; GFX8-NEXT: v_mov_b32_e32 v4, s57397; GFX8-NEXT: v_mov_b32_e32 v5, s47398; GFX8-NEXT: .LBB103_1: ; %atomicrmw.start7399; GFX8-NEXT: ; =>This Inner Loop Header: Depth=17400; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7401; GFX8-NEXT: v_mov_b32_e32 v9, v37402; GFX8-NEXT: v_mov_b32_e32 v8, v27403; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]7404; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc7405; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc7406; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc7407; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7408; GFX8-NEXT: buffer_wbinvl1_vol7409; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]7410; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]7411; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]7412; GFX8-NEXT: s_cbranch_execnz .LBB103_17413; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end7414; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]7415; GFX8-NEXT: v_mov_b32_e32 v0, s27416; GFX8-NEXT: v_mov_b32_e32 v1, s37417; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]7418; GFX8-NEXT: s_endpgm7419;7420; GFX9-LABEL: atomic_umax_i64_ret_addr64_offset:7421; GFX9: ; %bb.0: ; %entry7422; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x247423; GFX9-NEXT: s_waitcnt lgkmcnt(0)7424; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 37425; GFX9-NEXT: s_add_u32 s0, s8, s07426; GFX9-NEXT: s_addc_u32 s1, s9, s17427; GFX9-NEXT: v_mov_b32_e32 v0, s07428; GFX9-NEXT: v_mov_b32_e32 v1, s17429; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:327430; GFX9-NEXT: s_mov_b64 s[0:1], 07431; GFX9-NEXT: v_mov_b32_e32 v4, s137432; GFX9-NEXT: v_mov_b32_e32 v5, s127433; GFX9-NEXT: .LBB103_1: ; %atomicrmw.start7434; GFX9-NEXT: ; =>This Inner Loop Header: Depth=17435; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7436; GFX9-NEXT: v_mov_b32_e32 v9, v37437; GFX9-NEXT: v_mov_b32_e32 v8, v27438; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9]7439; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc7440; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc7441; GFX9-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] offset:32 glc7442; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7443; GFX9-NEXT: buffer_wbinvl1_vol7444; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]7445; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]7446; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]7447; GFX9-NEXT: s_cbranch_execnz .LBB103_17448; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end7449; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]7450; GFX9-NEXT: v_mov_b32_e32 v0, s107451; GFX9-NEXT: v_mov_b32_e32 v1, s117452; GFX9-NEXT: flat_store_dwordx2 v[0:1], v[2:3]7453; GFX9-NEXT: s_endpgm7454entry:7455 %ptr = getelementptr inbounds i64, ptr %out, i64 %index7456 %gep = getelementptr inbounds i64, ptr %ptr, i64 47457 %tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst, !noalias.addrspace !17458 store i64 %tmp0, ptr %out27459 ret void7460}7461 7462define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {7463; GFX7-LABEL: atomic_umax_i64_ret_addr64:7464; GFX7: ; %bb.0: ; %entry7465; GFX7-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x97466; GFX7-NEXT: s_waitcnt lgkmcnt(0)7467; GFX7-NEXT: s_lshl_b64 s[6:7], s[6:7], 37468; GFX7-NEXT: s_add_u32 s0, s0, s67469; GFX7-NEXT: s_addc_u32 s1, s1, s77470; GFX7-NEXT: v_mov_b32_e32 v0, s07471; GFX7-NEXT: v_mov_b32_e32 v1, s17472; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]7473; GFX7-NEXT: s_mov_b64 s[0:1], 07474; GFX7-NEXT: v_mov_b32_e32 v4, s57475; GFX7-NEXT: v_mov_b32_e32 v5, s47476; GFX7-NEXT: .LBB104_1: ; %atomicrmw.start7477; GFX7-NEXT: ; =>This Inner Loop Header: Depth=17478; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7479; GFX7-NEXT: v_mov_b32_e32 v9, v37480; GFX7-NEXT: v_mov_b32_e32 v8, v27481; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]7482; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc7483; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc7484; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc7485; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7486; GFX7-NEXT: buffer_wbinvl1_vol7487; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]7488; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]7489; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]7490; GFX7-NEXT: s_cbranch_execnz .LBB104_17491; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end7492; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]7493; GFX7-NEXT: v_mov_b32_e32 v0, s27494; GFX7-NEXT: v_mov_b32_e32 v1, s37495; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]7496; GFX7-NEXT: s_endpgm7497;7498; GFX8-LABEL: atomic_umax_i64_ret_addr64:7499; GFX8: ; %bb.0: ; %entry7500; GFX8-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x247501; GFX8-NEXT: s_waitcnt lgkmcnt(0)7502; GFX8-NEXT: s_lshl_b64 s[6:7], s[6:7], 37503; GFX8-NEXT: s_add_u32 s0, s0, s67504; GFX8-NEXT: s_addc_u32 s1, s1, s77505; GFX8-NEXT: v_mov_b32_e32 v0, s07506; GFX8-NEXT: v_mov_b32_e32 v1, s17507; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]7508; GFX8-NEXT: s_mov_b64 s[0:1], 07509; GFX8-NEXT: v_mov_b32_e32 v4, s57510; GFX8-NEXT: v_mov_b32_e32 v5, s47511; GFX8-NEXT: .LBB104_1: ; %atomicrmw.start7512; GFX8-NEXT: ; =>This Inner Loop Header: Depth=17513; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7514; GFX8-NEXT: v_mov_b32_e32 v9, v37515; GFX8-NEXT: v_mov_b32_e32 v8, v27516; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]7517; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc7518; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc7519; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc7520; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7521; GFX8-NEXT: buffer_wbinvl1_vol7522; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]7523; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]7524; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]7525; GFX8-NEXT: s_cbranch_execnz .LBB104_17526; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end7527; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]7528; GFX8-NEXT: v_mov_b32_e32 v0, s27529; GFX8-NEXT: v_mov_b32_e32 v1, s37530; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]7531; GFX8-NEXT: s_endpgm7532;7533; GFX9-LABEL: atomic_umax_i64_ret_addr64:7534; GFX9: ; %bb.0: ; %entry7535; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x247536; GFX9-NEXT: s_waitcnt lgkmcnt(0)7537; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 37538; GFX9-NEXT: s_add_u32 s0, s8, s07539; GFX9-NEXT: s_addc_u32 s1, s9, s17540; GFX9-NEXT: v_mov_b32_e32 v0, s07541; GFX9-NEXT: v_mov_b32_e32 v1, s17542; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]7543; GFX9-NEXT: s_mov_b64 s[0:1], 07544; GFX9-NEXT: v_mov_b32_e32 v4, s137545; GFX9-NEXT: v_mov_b32_e32 v5, s127546; GFX9-NEXT: .LBB104_1: ; %atomicrmw.start7547; GFX9-NEXT: ; =>This Inner Loop Header: Depth=17548; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7549; GFX9-NEXT: v_mov_b32_e32 v9, v37550; GFX9-NEXT: v_mov_b32_e32 v8, v27551; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9]7552; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc7553; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc7554; GFX9-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc7555; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7556; GFX9-NEXT: buffer_wbinvl1_vol7557; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]7558; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]7559; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]7560; GFX9-NEXT: s_cbranch_execnz .LBB104_17561; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end7562; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]7563; GFX9-NEXT: v_mov_b32_e32 v0, s107564; GFX9-NEXT: v_mov_b32_e32 v1, s117565; GFX9-NEXT: flat_store_dwordx2 v[0:1], v[2:3]7566; GFX9-NEXT: s_endpgm7567entry:7568 %ptr = getelementptr inbounds i64, ptr %out, i64 %index7569 %tmp0 = atomicrmw umax ptr %ptr, i64 %in seq_cst, !noalias.addrspace !17570 store i64 %tmp0, ptr %out27571 ret void7572}7573 7574define void @flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {7575; GFX7-LABEL: flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory:7576; GFX7: ; %bb.0:7577; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7578; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v07579; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc7580; GFX7-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3]7581; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7582; GFX7-NEXT: buffer_wbinvl1_vol7583; GFX7-NEXT: s_setpc_b64 s[30:31]7584;7585; GFX8-LABEL: flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory:7586; GFX8: ; %bb.0:7587; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7588; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v07589; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc7590; GFX8-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3]7591; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7592; GFX8-NEXT: buffer_wbinvl1_vol7593; GFX8-NEXT: s_setpc_b64 s[30:31]7594;7595; GFX9-LABEL: flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory:7596; GFX9: ; %bb.0:7597; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7598; GFX9-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3] offset:327599; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7600; GFX9-NEXT: buffer_wbinvl1_vol7601; GFX9-NEXT: s_setpc_b64 s[30:31]7602 %gep = getelementptr inbounds i64, ptr %out, i64 47603 %tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !17604 ret void7605}7606 7607define i64 @flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {7608; GFX7-LABEL: flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory:7609; GFX7: ; %bb.0:7610; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7611; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v07612; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc7613; GFX7-NEXT: flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc7614; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7615; GFX7-NEXT: buffer_wbinvl1_vol7616; GFX7-NEXT: s_setpc_b64 s[30:31]7617;7618; GFX8-LABEL: flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory:7619; GFX8: ; %bb.0:7620; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7621; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v07622; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc7623; GFX8-NEXT: flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc7624; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7625; GFX8-NEXT: buffer_wbinvl1_vol7626; GFX8-NEXT: s_setpc_b64 s[30:31]7627;7628; GFX9-LABEL: flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory:7629; GFX9: ; %bb.0:7630; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7631; GFX9-NEXT: flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] offset:32 glc7632; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7633; GFX9-NEXT: buffer_wbinvl1_vol7634; GFX9-NEXT: s_setpc_b64 s[30:31]7635 %gep = getelementptr inbounds i64, ptr %out, i64 47636 %result = atomicrmw umax ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !17637 ret i64 %result7638}7639 7640; ---------------------------------------------------------------------7641; atomicrmw umin7642; ---------------------------------------------------------------------7643 7644define void @flat_atomic_umin_i64_noret(ptr %ptr, i64 %in) {7645; GFX7-LABEL: flat_atomic_umin_i64_noret:7646; GFX7: ; %bb.0:7647; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7648; GFX7-NEXT: v_add_i32_e32 v4, vcc, 4, v07649; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc7650; GFX7-NEXT: flat_load_dword v6, v[0:1]7651; GFX7-NEXT: flat_load_dword v7, v[4:5]7652; GFX7-NEXT: s_mov_b64 s[4:5], 07653; GFX7-NEXT: .LBB107_1: ; %atomicrmw.start7654; GFX7-NEXT: ; =>This Inner Loop Header: Depth=17655; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7656; GFX7-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]7657; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc7658; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc7659; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc7660; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7661; GFX7-NEXT: buffer_wbinvl1_vol7662; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]7663; GFX7-NEXT: v_mov_b32_e32 v7, v57664; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7665; GFX7-NEXT: v_mov_b32_e32 v6, v47666; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]7667; GFX7-NEXT: s_cbranch_execnz .LBB107_17668; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end7669; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]7670; GFX7-NEXT: s_setpc_b64 s[30:31]7671;7672; GFX8-LABEL: flat_atomic_umin_i64_noret:7673; GFX8: ; %bb.0:7674; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7675; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v07676; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc7677; GFX8-NEXT: flat_load_dword v6, v[0:1]7678; GFX8-NEXT: flat_load_dword v7, v[4:5]7679; GFX8-NEXT: s_mov_b64 s[4:5], 07680; GFX8-NEXT: .LBB107_1: ; %atomicrmw.start7681; GFX8-NEXT: ; =>This Inner Loop Header: Depth=17682; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7683; GFX8-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]7684; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc7685; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc7686; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc7687; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7688; GFX8-NEXT: buffer_wbinvl1_vol7689; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]7690; GFX8-NEXT: v_mov_b32_e32 v7, v57691; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7692; GFX8-NEXT: v_mov_b32_e32 v6, v47693; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]7694; GFX8-NEXT: s_cbranch_execnz .LBB107_17695; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end7696; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]7697; GFX8-NEXT: s_setpc_b64 s[30:31]7698;7699; GFX9-LABEL: flat_atomic_umin_i64_noret:7700; GFX9: ; %bb.0:7701; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7702; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1]7703; GFX9-NEXT: s_mov_b64 s[4:5], 07704; GFX9-NEXT: .LBB107_1: ; %atomicrmw.start7705; GFX9-NEXT: ; =>This Inner Loop Header: Depth=17706; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7707; GFX9-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]7708; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc7709; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc7710; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc7711; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7712; GFX9-NEXT: buffer_wbinvl1_vol7713; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]7714; GFX9-NEXT: v_mov_b32_e32 v7, v57715; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7716; GFX9-NEXT: v_mov_b32_e32 v6, v47717; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]7718; GFX9-NEXT: s_cbranch_execnz .LBB107_17719; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end7720; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]7721; GFX9-NEXT: s_setpc_b64 s[30:31]7722 %tmp0 = atomicrmw umin ptr %ptr, i64 %in seq_cst, !noalias.addrspace !17723 ret void7724}7725 7726define void @flat_atomic_umin_i64_noret_offset(ptr %out, i64 %in) {7727; GFX7-LABEL: flat_atomic_umin_i64_noret_offset:7728; GFX7: ; %bb.0:7729; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7730; GFX7-NEXT: v_add_i32_e32 v8, vcc, 32, v07731; GFX7-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc7732; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v07733; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc7734; GFX7-NEXT: flat_load_dword v7, v[0:1]7735; GFX7-NEXT: flat_load_dword v6, v[8:9]7736; GFX7-NEXT: s_mov_b64 s[4:5], 07737; GFX7-NEXT: .LBB108_1: ; %atomicrmw.start7738; GFX7-NEXT: ; =>This Inner Loop Header: Depth=17739; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7740; GFX7-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]7741; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc7742; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc7743; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc7744; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7745; GFX7-NEXT: buffer_wbinvl1_vol7746; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]7747; GFX7-NEXT: v_mov_b32_e32 v7, v17748; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7749; GFX7-NEXT: v_mov_b32_e32 v6, v07750; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]7751; GFX7-NEXT: s_cbranch_execnz .LBB108_17752; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end7753; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]7754; GFX7-NEXT: s_setpc_b64 s[30:31]7755;7756; GFX8-LABEL: flat_atomic_umin_i64_noret_offset:7757; GFX8: ; %bb.0:7758; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7759; GFX8-NEXT: v_add_u32_e32 v8, vcc, 32, v07760; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc7761; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v07762; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc7763; GFX8-NEXT: flat_load_dword v7, v[0:1]7764; GFX8-NEXT: flat_load_dword v6, v[8:9]7765; GFX8-NEXT: s_mov_b64 s[4:5], 07766; GFX8-NEXT: .LBB108_1: ; %atomicrmw.start7767; GFX8-NEXT: ; =>This Inner Loop Header: Depth=17768; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7769; GFX8-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]7770; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc7771; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc7772; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc7773; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7774; GFX8-NEXT: buffer_wbinvl1_vol7775; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]7776; GFX8-NEXT: v_mov_b32_e32 v7, v17777; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7778; GFX8-NEXT: v_mov_b32_e32 v6, v07779; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]7780; GFX8-NEXT: s_cbranch_execnz .LBB108_17781; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end7782; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]7783; GFX8-NEXT: s_setpc_b64 s[30:31]7784;7785; GFX9-LABEL: flat_atomic_umin_i64_noret_offset:7786; GFX9: ; %bb.0:7787; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7788; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:327789; GFX9-NEXT: s_mov_b64 s[4:5], 07790; GFX9-NEXT: .LBB108_1: ; %atomicrmw.start7791; GFX9-NEXT: ; =>This Inner Loop Header: Depth=17792; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7793; GFX9-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]7794; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc7795; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc7796; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc7797; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7798; GFX9-NEXT: buffer_wbinvl1_vol7799; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]7800; GFX9-NEXT: v_mov_b32_e32 v7, v57801; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7802; GFX9-NEXT: v_mov_b32_e32 v6, v47803; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]7804; GFX9-NEXT: s_cbranch_execnz .LBB108_17805; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end7806; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]7807; GFX9-NEXT: s_setpc_b64 s[30:31]7808 %gep = getelementptr inbounds i64, ptr %out, i64 47809 %tmp0 = atomicrmw umin ptr %gep, i64 %in seq_cst, !noalias.addrspace !17810 ret void7811}7812 7813define i64 @flat_atomic_umin_i64_ret(ptr %ptr, i64 %in) {7814; GFX7-LABEL: flat_atomic_umin_i64_ret:7815; GFX7: ; %bb.0:7816; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7817; GFX7-NEXT: v_add_i32_e32 v5, vcc, 4, v07818; GFX7-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc7819; GFX7-NEXT: flat_load_dword v4, v[0:1]7820; GFX7-NEXT: flat_load_dword v5, v[5:6]7821; GFX7-NEXT: s_mov_b64 s[4:5], 07822; GFX7-NEXT: .LBB109_1: ; %atomicrmw.start7823; GFX7-NEXT: ; =>This Inner Loop Header: Depth=17824; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7825; GFX7-NEXT: v_mov_b32_e32 v7, v57826; GFX7-NEXT: v_mov_b32_e32 v6, v47827; GFX7-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]7828; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc7829; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc7830; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc7831; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7832; GFX7-NEXT: buffer_wbinvl1_vol7833; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]7834; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7835; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]7836; GFX7-NEXT: s_cbranch_execnz .LBB109_17837; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end7838; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]7839; GFX7-NEXT: v_mov_b32_e32 v0, v47840; GFX7-NEXT: v_mov_b32_e32 v1, v57841; GFX7-NEXT: s_setpc_b64 s[30:31]7842;7843; GFX8-LABEL: flat_atomic_umin_i64_ret:7844; GFX8: ; %bb.0:7845; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7846; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v07847; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc7848; GFX8-NEXT: flat_load_dword v4, v[0:1]7849; GFX8-NEXT: flat_load_dword v5, v[5:6]7850; GFX8-NEXT: s_mov_b64 s[4:5], 07851; GFX8-NEXT: .LBB109_1: ; %atomicrmw.start7852; GFX8-NEXT: ; =>This Inner Loop Header: Depth=17853; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7854; GFX8-NEXT: v_mov_b32_e32 v7, v57855; GFX8-NEXT: v_mov_b32_e32 v6, v47856; GFX8-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]7857; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc7858; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc7859; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc7860; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7861; GFX8-NEXT: buffer_wbinvl1_vol7862; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]7863; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7864; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]7865; GFX8-NEXT: s_cbranch_execnz .LBB109_17866; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end7867; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]7868; GFX8-NEXT: v_mov_b32_e32 v0, v47869; GFX8-NEXT: v_mov_b32_e32 v1, v57870; GFX8-NEXT: s_setpc_b64 s[30:31]7871;7872; GFX9-LABEL: flat_atomic_umin_i64_ret:7873; GFX9: ; %bb.0:7874; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7875; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1]7876; GFX9-NEXT: s_mov_b64 s[4:5], 07877; GFX9-NEXT: .LBB109_1: ; %atomicrmw.start7878; GFX9-NEXT: ; =>This Inner Loop Header: Depth=17879; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7880; GFX9-NEXT: v_mov_b32_e32 v7, v57881; GFX9-NEXT: v_mov_b32_e32 v6, v47882; GFX9-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]7883; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc7884; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc7885; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc7886; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7887; GFX9-NEXT: buffer_wbinvl1_vol7888; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]7889; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7890; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]7891; GFX9-NEXT: s_cbranch_execnz .LBB109_17892; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end7893; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]7894; GFX9-NEXT: v_mov_b32_e32 v0, v47895; GFX9-NEXT: v_mov_b32_e32 v1, v57896; GFX9-NEXT: s_setpc_b64 s[30:31]7897 %result = atomicrmw umin ptr %ptr, i64 %in seq_cst, !noalias.addrspace !17898 ret i64 %result7899}7900 7901define i64 @flat_atomic_umin_i64_ret_offset(ptr %out, i64 %in) {7902; GFX7-LABEL: flat_atomic_umin_i64_ret_offset:7903; GFX7: ; %bb.0:7904; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7905; GFX7-NEXT: v_add_i32_e32 v4, vcc, 32, v07906; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc7907; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v07908; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc7909; GFX7-NEXT: flat_load_dword v1, v[0:1]7910; GFX7-NEXT: flat_load_dword v0, v[4:5]7911; GFX7-NEXT: s_mov_b64 s[4:5], 07912; GFX7-NEXT: .LBB110_1: ; %atomicrmw.start7913; GFX7-NEXT: ; =>This Inner Loop Header: Depth=17914; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7915; GFX7-NEXT: v_mov_b32_e32 v9, v17916; GFX7-NEXT: v_mov_b32_e32 v8, v07917; GFX7-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]7918; GFX7-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc7919; GFX7-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc7920; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc7921; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7922; GFX7-NEXT: buffer_wbinvl1_vol7923; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]7924; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7925; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]7926; GFX7-NEXT: s_cbranch_execnz .LBB110_17927; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end7928; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]7929; GFX7-NEXT: s_setpc_b64 s[30:31]7930;7931; GFX8-LABEL: flat_atomic_umin_i64_ret_offset:7932; GFX8: ; %bb.0:7933; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7934; GFX8-NEXT: v_add_u32_e32 v4, vcc, 32, v07935; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc7936; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v07937; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc7938; GFX8-NEXT: flat_load_dword v1, v[0:1]7939; GFX8-NEXT: flat_load_dword v0, v[4:5]7940; GFX8-NEXT: s_mov_b64 s[4:5], 07941; GFX8-NEXT: .LBB110_1: ; %atomicrmw.start7942; GFX8-NEXT: ; =>This Inner Loop Header: Depth=17943; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7944; GFX8-NEXT: v_mov_b32_e32 v9, v17945; GFX8-NEXT: v_mov_b32_e32 v8, v07946; GFX8-NEXT: v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]7947; GFX8-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc7948; GFX8-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc7949; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc7950; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7951; GFX8-NEXT: buffer_wbinvl1_vol7952; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]7953; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7954; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]7955; GFX8-NEXT: s_cbranch_execnz .LBB110_17956; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end7957; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]7958; GFX8-NEXT: s_setpc_b64 s[30:31]7959;7960; GFX9-LABEL: flat_atomic_umin_i64_ret_offset:7961; GFX9: ; %bb.0:7962; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7963; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:327964; GFX9-NEXT: s_mov_b64 s[4:5], 07965; GFX9-NEXT: .LBB110_1: ; %atomicrmw.start7966; GFX9-NEXT: ; =>This Inner Loop Header: Depth=17967; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7968; GFX9-NEXT: v_mov_b32_e32 v7, v57969; GFX9-NEXT: v_mov_b32_e32 v6, v47970; GFX9-NEXT: v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]7971; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc7972; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc7973; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc7974; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7975; GFX9-NEXT: buffer_wbinvl1_vol7976; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]7977; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7978; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]7979; GFX9-NEXT: s_cbranch_execnz .LBB110_17980; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end7981; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]7982; GFX9-NEXT: v_mov_b32_e32 v0, v47983; GFX9-NEXT: v_mov_b32_e32 v1, v57984; GFX9-NEXT: s_setpc_b64 s[30:31]7985 %gep = getelementptr inbounds i64, ptr %out, i64 47986 %result = atomicrmw umin ptr %gep, i64 %in seq_cst, !noalias.addrspace !17987 ret i64 %result7988}7989 7990define amdgpu_gfx void @flat_atomic_umin_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {7991; GFX7-LABEL: flat_atomic_umin_i64_noret_scalar:7992; GFX7: ; %bb.0:7993; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7994; GFX7-NEXT: v_mov_b32_e32 v0, s47995; GFX7-NEXT: s_add_u32 s34, s4, 47996; GFX7-NEXT: v_mov_b32_e32 v1, s57997; GFX7-NEXT: s_addc_u32 s35, s5, 07998; GFX7-NEXT: v_mov_b32_e32 v3, s347999; GFX7-NEXT: v_mov_b32_e32 v4, s358000; GFX7-NEXT: flat_load_dword v2, v[0:1]8001; GFX7-NEXT: flat_load_dword v3, v[3:4]8002; GFX7-NEXT: v_mov_b32_e32 v4, s48003; GFX7-NEXT: s_mov_b64 s[34:35], 08004; GFX7-NEXT: v_mov_b32_e32 v6, s78005; GFX7-NEXT: v_mov_b32_e32 v7, s68006; GFX7-NEXT: v_mov_b32_e32 v5, s58007; GFX7-NEXT: .LBB111_1: ; %atomicrmw.start8008; GFX7-NEXT: ; =>This Inner Loop Header: Depth=18009; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8010; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]8011; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc8012; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc8013; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc8014; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8015; GFX7-NEXT: buffer_wbinvl1_vol8016; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8017; GFX7-NEXT: v_mov_b32_e32 v3, v18018; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]8019; GFX7-NEXT: v_mov_b32_e32 v2, v08020; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]8021; GFX7-NEXT: s_cbranch_execnz .LBB111_18022; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end8023; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]8024; GFX7-NEXT: s_setpc_b64 s[30:31]8025;8026; GFX8-LABEL: flat_atomic_umin_i64_noret_scalar:8027; GFX8: ; %bb.0:8028; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8029; GFX8-NEXT: v_mov_b32_e32 v0, s48030; GFX8-NEXT: s_add_u32 s34, s4, 48031; GFX8-NEXT: v_mov_b32_e32 v1, s58032; GFX8-NEXT: s_addc_u32 s35, s5, 08033; GFX8-NEXT: v_mov_b32_e32 v3, s348034; GFX8-NEXT: v_mov_b32_e32 v4, s358035; GFX8-NEXT: flat_load_dword v2, v[0:1]8036; GFX8-NEXT: flat_load_dword v3, v[3:4]8037; GFX8-NEXT: v_mov_b32_e32 v4, s48038; GFX8-NEXT: s_mov_b64 s[34:35], 08039; GFX8-NEXT: v_mov_b32_e32 v6, s78040; GFX8-NEXT: v_mov_b32_e32 v7, s68041; GFX8-NEXT: v_mov_b32_e32 v5, s58042; GFX8-NEXT: .LBB111_1: ; %atomicrmw.start8043; GFX8-NEXT: ; =>This Inner Loop Header: Depth=18044; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8045; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]8046; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc8047; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc8048; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc8049; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8050; GFX8-NEXT: buffer_wbinvl1_vol8051; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8052; GFX8-NEXT: v_mov_b32_e32 v3, v18053; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]8054; GFX8-NEXT: v_mov_b32_e32 v2, v08055; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]8056; GFX8-NEXT: s_cbranch_execnz .LBB111_18057; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end8058; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]8059; GFX8-NEXT: s_setpc_b64 s[30:31]8060;8061; GFX9-LABEL: flat_atomic_umin_i64_noret_scalar:8062; GFX9: ; %bb.0:8063; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8064; GFX9-NEXT: v_mov_b32_e32 v0, s48065; GFX9-NEXT: v_mov_b32_e32 v1, s58066; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]8067; GFX9-NEXT: v_mov_b32_e32 v4, s48068; GFX9-NEXT: s_mov_b64 s[34:35], 08069; GFX9-NEXT: v_mov_b32_e32 v6, s78070; GFX9-NEXT: v_mov_b32_e32 v7, s68071; GFX9-NEXT: v_mov_b32_e32 v5, s58072; GFX9-NEXT: .LBB111_1: ; %atomicrmw.start8073; GFX9-NEXT: ; =>This Inner Loop Header: Depth=18074; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8075; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]8076; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc8077; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc8078; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc8079; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8080; GFX9-NEXT: buffer_wbinvl1_vol8081; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8082; GFX9-NEXT: v_mov_b32_e32 v3, v18083; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]8084; GFX9-NEXT: v_mov_b32_e32 v2, v08085; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]8086; GFX9-NEXT: s_cbranch_execnz .LBB111_18087; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end8088; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]8089; GFX9-NEXT: s_setpc_b64 s[30:31]8090 %tmp0 = atomicrmw umin ptr %ptr, i64 %in seq_cst, !noalias.addrspace !18091 ret void8092}8093 8094define amdgpu_gfx void @flat_atomic_umin_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {8095; GFX7-LABEL: flat_atomic_umin_i64_noret_offset_scalar:8096; GFX7: ; %bb.0:8097; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8098; GFX7-NEXT: s_add_u32 s34, s4, 328099; GFX7-NEXT: s_addc_u32 s35, s5, 08100; GFX7-NEXT: s_add_u32 s36, s4, 368101; GFX7-NEXT: s_addc_u32 s37, s5, 08102; GFX7-NEXT: v_mov_b32_e32 v0, s368103; GFX7-NEXT: v_mov_b32_e32 v1, s378104; GFX7-NEXT: v_mov_b32_e32 v4, s348105; GFX7-NEXT: v_mov_b32_e32 v5, s358106; GFX7-NEXT: flat_load_dword v3, v[0:1]8107; GFX7-NEXT: flat_load_dword v2, v[4:5]8108; GFX7-NEXT: s_mov_b64 s[34:35], 08109; GFX7-NEXT: v_mov_b32_e32 v6, s78110; GFX7-NEXT: v_mov_b32_e32 v7, s68111; GFX7-NEXT: .LBB112_1: ; %atomicrmw.start8112; GFX7-NEXT: ; =>This Inner Loop Header: Depth=18113; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8114; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]8115; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc8116; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc8117; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc8118; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8119; GFX7-NEXT: buffer_wbinvl1_vol8120; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8121; GFX7-NEXT: v_mov_b32_e32 v3, v18122; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]8123; GFX7-NEXT: v_mov_b32_e32 v2, v08124; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]8125; GFX7-NEXT: s_cbranch_execnz .LBB112_18126; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end8127; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]8128; GFX7-NEXT: s_setpc_b64 s[30:31]8129;8130; GFX8-LABEL: flat_atomic_umin_i64_noret_offset_scalar:8131; GFX8: ; %bb.0:8132; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8133; GFX8-NEXT: s_add_u32 s34, s4, 328134; GFX8-NEXT: s_addc_u32 s35, s5, 08135; GFX8-NEXT: s_add_u32 s36, s4, 368136; GFX8-NEXT: s_addc_u32 s37, s5, 08137; GFX8-NEXT: v_mov_b32_e32 v0, s368138; GFX8-NEXT: v_mov_b32_e32 v1, s378139; GFX8-NEXT: v_mov_b32_e32 v4, s348140; GFX8-NEXT: v_mov_b32_e32 v5, s358141; GFX8-NEXT: flat_load_dword v3, v[0:1]8142; GFX8-NEXT: flat_load_dword v2, v[4:5]8143; GFX8-NEXT: s_mov_b64 s[34:35], 08144; GFX8-NEXT: v_mov_b32_e32 v6, s78145; GFX8-NEXT: v_mov_b32_e32 v7, s68146; GFX8-NEXT: .LBB112_1: ; %atomicrmw.start8147; GFX8-NEXT: ; =>This Inner Loop Header: Depth=18148; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8149; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]8150; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc8151; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc8152; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc8153; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8154; GFX8-NEXT: buffer_wbinvl1_vol8155; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8156; GFX8-NEXT: v_mov_b32_e32 v3, v18157; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]8158; GFX8-NEXT: v_mov_b32_e32 v2, v08159; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]8160; GFX8-NEXT: s_cbranch_execnz .LBB112_18161; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end8162; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]8163; GFX8-NEXT: s_setpc_b64 s[30:31]8164;8165; GFX9-LABEL: flat_atomic_umin_i64_noret_offset_scalar:8166; GFX9: ; %bb.0:8167; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8168; GFX9-NEXT: v_mov_b32_e32 v0, s48169; GFX9-NEXT: v_mov_b32_e32 v1, s58170; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:328171; GFX9-NEXT: v_mov_b32_e32 v4, s48172; GFX9-NEXT: s_mov_b64 s[34:35], 08173; GFX9-NEXT: v_mov_b32_e32 v6, s78174; GFX9-NEXT: v_mov_b32_e32 v7, s68175; GFX9-NEXT: v_mov_b32_e32 v5, s58176; GFX9-NEXT: .LBB112_1: ; %atomicrmw.start8177; GFX9-NEXT: ; =>This Inner Loop Header: Depth=18178; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8179; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]8180; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc8181; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc8182; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc8183; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8184; GFX9-NEXT: buffer_wbinvl1_vol8185; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8186; GFX9-NEXT: v_mov_b32_e32 v3, v18187; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]8188; GFX9-NEXT: v_mov_b32_e32 v2, v08189; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]8190; GFX9-NEXT: s_cbranch_execnz .LBB112_18191; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end8192; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]8193; GFX9-NEXT: s_setpc_b64 s[30:31]8194 %gep = getelementptr inbounds i64, ptr %out, i64 48195 %tmp0 = atomicrmw umin ptr %gep, i64 %in seq_cst, !noalias.addrspace !18196 ret void8197}8198 8199define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {8200; GFX7-LABEL: flat_atomic_umin_i64_ret_scalar:8201; GFX7: ; %bb.0:8202; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8203; GFX7-NEXT: v_mov_b32_e32 v0, s48204; GFX7-NEXT: s_add_u32 s34, s4, 48205; GFX7-NEXT: v_mov_b32_e32 v1, s58206; GFX7-NEXT: s_addc_u32 s35, s5, 08207; GFX7-NEXT: v_mov_b32_e32 v2, s348208; GFX7-NEXT: v_mov_b32_e32 v3, s358209; GFX7-NEXT: flat_load_dword v0, v[0:1]8210; GFX7-NEXT: flat_load_dword v1, v[2:3]8211; GFX7-NEXT: v_mov_b32_e32 v2, s48212; GFX7-NEXT: s_mov_b64 s[34:35], 08213; GFX7-NEXT: v_mov_b32_e32 v4, s78214; GFX7-NEXT: v_mov_b32_e32 v5, s68215; GFX7-NEXT: v_mov_b32_e32 v3, s58216; GFX7-NEXT: .LBB113_1: ; %atomicrmw.start8217; GFX7-NEXT: ; =>This Inner Loop Header: Depth=18218; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8219; GFX7-NEXT: v_mov_b32_e32 v9, v18220; GFX7-NEXT: v_mov_b32_e32 v8, v08221; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]8222; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc8223; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc8224; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc8225; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8226; GFX7-NEXT: buffer_wbinvl1_vol8227; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]8228; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]8229; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]8230; GFX7-NEXT: s_cbranch_execnz .LBB113_18231; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end8232; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]8233; GFX7-NEXT: s_setpc_b64 s[30:31]8234;8235; GFX8-LABEL: flat_atomic_umin_i64_ret_scalar:8236; GFX8: ; %bb.0:8237; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8238; GFX8-NEXT: v_mov_b32_e32 v0, s48239; GFX8-NEXT: s_add_u32 s34, s4, 48240; GFX8-NEXT: v_mov_b32_e32 v1, s58241; GFX8-NEXT: s_addc_u32 s35, s5, 08242; GFX8-NEXT: v_mov_b32_e32 v2, s348243; GFX8-NEXT: v_mov_b32_e32 v3, s358244; GFX8-NEXT: flat_load_dword v0, v[0:1]8245; GFX8-NEXT: flat_load_dword v1, v[2:3]8246; GFX8-NEXT: v_mov_b32_e32 v2, s48247; GFX8-NEXT: s_mov_b64 s[34:35], 08248; GFX8-NEXT: v_mov_b32_e32 v4, s78249; GFX8-NEXT: v_mov_b32_e32 v5, s68250; GFX8-NEXT: v_mov_b32_e32 v3, s58251; GFX8-NEXT: .LBB113_1: ; %atomicrmw.start8252; GFX8-NEXT: ; =>This Inner Loop Header: Depth=18253; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8254; GFX8-NEXT: v_mov_b32_e32 v9, v18255; GFX8-NEXT: v_mov_b32_e32 v8, v08256; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]8257; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc8258; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc8259; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc8260; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8261; GFX8-NEXT: buffer_wbinvl1_vol8262; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]8263; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]8264; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]8265; GFX8-NEXT: s_cbranch_execnz .LBB113_18266; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end8267; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]8268; GFX8-NEXT: s_setpc_b64 s[30:31]8269;8270; GFX9-LABEL: flat_atomic_umin_i64_ret_scalar:8271; GFX9: ; %bb.0:8272; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8273; GFX9-NEXT: v_mov_b32_e32 v0, s48274; GFX9-NEXT: v_mov_b32_e32 v1, s58275; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]8276; GFX9-NEXT: v_mov_b32_e32 v2, s48277; GFX9-NEXT: s_mov_b64 s[34:35], 08278; GFX9-NEXT: v_mov_b32_e32 v4, s78279; GFX9-NEXT: v_mov_b32_e32 v5, s68280; GFX9-NEXT: v_mov_b32_e32 v3, s58281; GFX9-NEXT: .LBB113_1: ; %atomicrmw.start8282; GFX9-NEXT: ; =>This Inner Loop Header: Depth=18283; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8284; GFX9-NEXT: v_mov_b32_e32 v9, v18285; GFX9-NEXT: v_mov_b32_e32 v8, v08286; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]8287; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc8288; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc8289; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc8290; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8291; GFX9-NEXT: buffer_wbinvl1_vol8292; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]8293; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]8294; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]8295; GFX9-NEXT: s_cbranch_execnz .LBB113_18296; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end8297; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]8298; GFX9-NEXT: s_setpc_b64 s[30:31]8299 %result = atomicrmw umin ptr %ptr, i64 %in seq_cst, !noalias.addrspace !18300 ret i64 %result8301}8302 8303define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {8304; GFX7-LABEL: flat_atomic_umin_i64_ret_offset_scalar:8305; GFX7: ; %bb.0:8306; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8307; GFX7-NEXT: s_add_u32 s34, s4, 328308; GFX7-NEXT: s_addc_u32 s35, s5, 08309; GFX7-NEXT: s_add_u32 s36, s4, 368310; GFX7-NEXT: s_addc_u32 s37, s5, 08311; GFX7-NEXT: v_mov_b32_e32 v0, s368312; GFX7-NEXT: v_mov_b32_e32 v1, s378313; GFX7-NEXT: v_mov_b32_e32 v2, s348314; GFX7-NEXT: v_mov_b32_e32 v3, s358315; GFX7-NEXT: flat_load_dword v1, v[0:1]8316; GFX7-NEXT: flat_load_dword v0, v[2:3]8317; GFX7-NEXT: s_mov_b64 s[34:35], 08318; GFX7-NEXT: v_mov_b32_e32 v4, s78319; GFX7-NEXT: v_mov_b32_e32 v5, s68320; GFX7-NEXT: .LBB114_1: ; %atomicrmw.start8321; GFX7-NEXT: ; =>This Inner Loop Header: Depth=18322; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8323; GFX7-NEXT: v_mov_b32_e32 v9, v18324; GFX7-NEXT: v_mov_b32_e32 v8, v08325; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]8326; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc8327; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc8328; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc8329; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8330; GFX7-NEXT: buffer_wbinvl1_vol8331; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]8332; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]8333; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]8334; GFX7-NEXT: s_cbranch_execnz .LBB114_18335; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end8336; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]8337; GFX7-NEXT: s_setpc_b64 s[30:31]8338;8339; GFX8-LABEL: flat_atomic_umin_i64_ret_offset_scalar:8340; GFX8: ; %bb.0:8341; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8342; GFX8-NEXT: s_add_u32 s34, s4, 328343; GFX8-NEXT: s_addc_u32 s35, s5, 08344; GFX8-NEXT: s_add_u32 s36, s4, 368345; GFX8-NEXT: s_addc_u32 s37, s5, 08346; GFX8-NEXT: v_mov_b32_e32 v0, s368347; GFX8-NEXT: v_mov_b32_e32 v1, s378348; GFX8-NEXT: v_mov_b32_e32 v2, s348349; GFX8-NEXT: v_mov_b32_e32 v3, s358350; GFX8-NEXT: flat_load_dword v1, v[0:1]8351; GFX8-NEXT: flat_load_dword v0, v[2:3]8352; GFX8-NEXT: s_mov_b64 s[34:35], 08353; GFX8-NEXT: v_mov_b32_e32 v4, s78354; GFX8-NEXT: v_mov_b32_e32 v5, s68355; GFX8-NEXT: .LBB114_1: ; %atomicrmw.start8356; GFX8-NEXT: ; =>This Inner Loop Header: Depth=18357; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8358; GFX8-NEXT: v_mov_b32_e32 v9, v18359; GFX8-NEXT: v_mov_b32_e32 v8, v08360; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]8361; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc8362; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc8363; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc8364; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8365; GFX8-NEXT: buffer_wbinvl1_vol8366; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]8367; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]8368; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]8369; GFX8-NEXT: s_cbranch_execnz .LBB114_18370; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end8371; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]8372; GFX8-NEXT: s_setpc_b64 s[30:31]8373;8374; GFX9-LABEL: flat_atomic_umin_i64_ret_offset_scalar:8375; GFX9: ; %bb.0:8376; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8377; GFX9-NEXT: v_mov_b32_e32 v0, s48378; GFX9-NEXT: v_mov_b32_e32 v1, s58379; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:328380; GFX9-NEXT: v_mov_b32_e32 v2, s48381; GFX9-NEXT: s_mov_b64 s[34:35], 08382; GFX9-NEXT: v_mov_b32_e32 v4, s78383; GFX9-NEXT: v_mov_b32_e32 v5, s68384; GFX9-NEXT: v_mov_b32_e32 v3, s58385; GFX9-NEXT: .LBB114_1: ; %atomicrmw.start8386; GFX9-NEXT: ; =>This Inner Loop Header: Depth=18387; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8388; GFX9-NEXT: v_mov_b32_e32 v9, v18389; GFX9-NEXT: v_mov_b32_e32 v8, v08390; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]8391; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc8392; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc8393; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc8394; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8395; GFX9-NEXT: buffer_wbinvl1_vol8396; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]8397; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]8398; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]8399; GFX9-NEXT: s_cbranch_execnz .LBB114_18400; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end8401; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]8402; GFX9-NEXT: s_setpc_b64 s[30:31]8403 %gep = getelementptr inbounds i64, ptr %out, i64 48404 %result = atomicrmw umin ptr %gep, i64 %in seq_cst, !noalias.addrspace !18405 ret i64 %result8406}8407 8408define void @flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {8409; GFX7-LABEL: flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory:8410; GFX7: ; %bb.0:8411; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8412; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v08413; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc8414; GFX7-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3]8415; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8416; GFX7-NEXT: buffer_wbinvl1_vol8417; GFX7-NEXT: s_setpc_b64 s[30:31]8418;8419; GFX8-LABEL: flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory:8420; GFX8: ; %bb.0:8421; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8422; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v08423; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc8424; GFX8-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3]8425; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8426; GFX8-NEXT: buffer_wbinvl1_vol8427; GFX8-NEXT: s_setpc_b64 s[30:31]8428;8429; GFX9-LABEL: flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory:8430; GFX9: ; %bb.0:8431; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8432; GFX9-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3] offset:328433; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8434; GFX9-NEXT: buffer_wbinvl1_vol8435; GFX9-NEXT: s_setpc_b64 s[30:31]8436 %gep = getelementptr inbounds i64, ptr %out, i64 48437 %tmp0 = atomicrmw umin ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !18438 ret void8439}8440 8441define i64 @flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {8442; GFX7-LABEL: flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory:8443; GFX7: ; %bb.0:8444; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8445; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v08446; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc8447; GFX7-NEXT: flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc8448; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8449; GFX7-NEXT: buffer_wbinvl1_vol8450; GFX7-NEXT: s_setpc_b64 s[30:31]8451;8452; GFX8-LABEL: flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory:8453; GFX8: ; %bb.0:8454; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8455; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v08456; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc8457; GFX8-NEXT: flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc8458; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8459; GFX8-NEXT: buffer_wbinvl1_vol8460; GFX8-NEXT: s_setpc_b64 s[30:31]8461;8462; GFX9-LABEL: flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory:8463; GFX9: ; %bb.0:8464; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8465; GFX9-NEXT: flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] offset:32 glc8466; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8467; GFX9-NEXT: buffer_wbinvl1_vol8468; GFX9-NEXT: s_setpc_b64 s[30:31]8469 %gep = getelementptr inbounds i64, ptr %out, i64 48470 %result = atomicrmw umin ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !18471 ret i64 %result8472}8473 8474; ---------------------------------------------------------------------8475; atomicrmw min8476; ---------------------------------------------------------------------8477 8478define void @flat_atomic_min_i64_noret(ptr %ptr, i64 %in) {8479; GFX7-LABEL: flat_atomic_min_i64_noret:8480; GFX7: ; %bb.0:8481; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8482; GFX7-NEXT: v_add_i32_e32 v4, vcc, 4, v08483; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc8484; GFX7-NEXT: flat_load_dword v6, v[0:1]8485; GFX7-NEXT: flat_load_dword v7, v[4:5]8486; GFX7-NEXT: s_mov_b64 s[4:5], 08487; GFX7-NEXT: .LBB117_1: ; %atomicrmw.start8488; GFX7-NEXT: ; =>This Inner Loop Header: Depth=18489; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8490; GFX7-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]8491; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc8492; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc8493; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc8494; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8495; GFX7-NEXT: buffer_wbinvl1_vol8496; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]8497; GFX7-NEXT: v_mov_b32_e32 v7, v58498; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8499; GFX7-NEXT: v_mov_b32_e32 v6, v48500; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]8501; GFX7-NEXT: s_cbranch_execnz .LBB117_18502; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end8503; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]8504; GFX7-NEXT: s_setpc_b64 s[30:31]8505;8506; GFX8-LABEL: flat_atomic_min_i64_noret:8507; GFX8: ; %bb.0:8508; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8509; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v08510; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc8511; GFX8-NEXT: flat_load_dword v6, v[0:1]8512; GFX8-NEXT: flat_load_dword v7, v[4:5]8513; GFX8-NEXT: s_mov_b64 s[4:5], 08514; GFX8-NEXT: .LBB117_1: ; %atomicrmw.start8515; GFX8-NEXT: ; =>This Inner Loop Header: Depth=18516; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8517; GFX8-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]8518; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc8519; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc8520; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc8521; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8522; GFX8-NEXT: buffer_wbinvl1_vol8523; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]8524; GFX8-NEXT: v_mov_b32_e32 v7, v58525; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8526; GFX8-NEXT: v_mov_b32_e32 v6, v48527; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]8528; GFX8-NEXT: s_cbranch_execnz .LBB117_18529; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end8530; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]8531; GFX8-NEXT: s_setpc_b64 s[30:31]8532;8533; GFX9-LABEL: flat_atomic_min_i64_noret:8534; GFX9: ; %bb.0:8535; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8536; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1]8537; GFX9-NEXT: s_mov_b64 s[4:5], 08538; GFX9-NEXT: .LBB117_1: ; %atomicrmw.start8539; GFX9-NEXT: ; =>This Inner Loop Header: Depth=18540; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8541; GFX9-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]8542; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc8543; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc8544; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc8545; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8546; GFX9-NEXT: buffer_wbinvl1_vol8547; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]8548; GFX9-NEXT: v_mov_b32_e32 v7, v58549; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8550; GFX9-NEXT: v_mov_b32_e32 v6, v48551; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]8552; GFX9-NEXT: s_cbranch_execnz .LBB117_18553; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end8554; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]8555; GFX9-NEXT: s_setpc_b64 s[30:31]8556 %tmp0 = atomicrmw min ptr %ptr, i64 %in seq_cst, !noalias.addrspace !18557 ret void8558}8559 8560define void @flat_atomic_min_i64_noret_offset(ptr %out, i64 %in) {8561; GFX7-LABEL: flat_atomic_min_i64_noret_offset:8562; GFX7: ; %bb.0:8563; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8564; GFX7-NEXT: v_add_i32_e32 v8, vcc, 32, v08565; GFX7-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc8566; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v08567; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc8568; GFX7-NEXT: flat_load_dword v7, v[0:1]8569; GFX7-NEXT: flat_load_dword v6, v[8:9]8570; GFX7-NEXT: s_mov_b64 s[4:5], 08571; GFX7-NEXT: .LBB118_1: ; %atomicrmw.start8572; GFX7-NEXT: ; =>This Inner Loop Header: Depth=18573; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8574; GFX7-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]8575; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc8576; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc8577; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc8578; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8579; GFX7-NEXT: buffer_wbinvl1_vol8580; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]8581; GFX7-NEXT: v_mov_b32_e32 v7, v18582; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8583; GFX7-NEXT: v_mov_b32_e32 v6, v08584; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]8585; GFX7-NEXT: s_cbranch_execnz .LBB118_18586; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end8587; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]8588; GFX7-NEXT: s_setpc_b64 s[30:31]8589;8590; GFX8-LABEL: flat_atomic_min_i64_noret_offset:8591; GFX8: ; %bb.0:8592; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8593; GFX8-NEXT: v_add_u32_e32 v8, vcc, 32, v08594; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc8595; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v08596; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc8597; GFX8-NEXT: flat_load_dword v7, v[0:1]8598; GFX8-NEXT: flat_load_dword v6, v[8:9]8599; GFX8-NEXT: s_mov_b64 s[4:5], 08600; GFX8-NEXT: .LBB118_1: ; %atomicrmw.start8601; GFX8-NEXT: ; =>This Inner Loop Header: Depth=18602; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8603; GFX8-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]8604; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc8605; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc8606; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc8607; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8608; GFX8-NEXT: buffer_wbinvl1_vol8609; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]8610; GFX8-NEXT: v_mov_b32_e32 v7, v18611; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8612; GFX8-NEXT: v_mov_b32_e32 v6, v08613; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]8614; GFX8-NEXT: s_cbranch_execnz .LBB118_18615; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end8616; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]8617; GFX8-NEXT: s_setpc_b64 s[30:31]8618;8619; GFX9-LABEL: flat_atomic_min_i64_noret_offset:8620; GFX9: ; %bb.0:8621; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8622; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:328623; GFX9-NEXT: s_mov_b64 s[4:5], 08624; GFX9-NEXT: .LBB118_1: ; %atomicrmw.start8625; GFX9-NEXT: ; =>This Inner Loop Header: Depth=18626; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8627; GFX9-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]8628; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc8629; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc8630; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc8631; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8632; GFX9-NEXT: buffer_wbinvl1_vol8633; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]8634; GFX9-NEXT: v_mov_b32_e32 v7, v58635; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8636; GFX9-NEXT: v_mov_b32_e32 v6, v48637; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]8638; GFX9-NEXT: s_cbranch_execnz .LBB118_18639; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end8640; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]8641; GFX9-NEXT: s_setpc_b64 s[30:31]8642 %gep = getelementptr inbounds i64, ptr %out, i64 48643 %tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !18644 ret void8645}8646 8647define i64 @flat_atomic_min_i64_ret(ptr %ptr, i64 %in) {8648; GFX7-LABEL: flat_atomic_min_i64_ret:8649; GFX7: ; %bb.0:8650; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8651; GFX7-NEXT: v_add_i32_e32 v5, vcc, 4, v08652; GFX7-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc8653; GFX7-NEXT: flat_load_dword v4, v[0:1]8654; GFX7-NEXT: flat_load_dword v5, v[5:6]8655; GFX7-NEXT: s_mov_b64 s[4:5], 08656; GFX7-NEXT: .LBB119_1: ; %atomicrmw.start8657; GFX7-NEXT: ; =>This Inner Loop Header: Depth=18658; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8659; GFX7-NEXT: v_mov_b32_e32 v7, v58660; GFX7-NEXT: v_mov_b32_e32 v6, v48661; GFX7-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]8662; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc8663; GFX7-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc8664; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc8665; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8666; GFX7-NEXT: buffer_wbinvl1_vol8667; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]8668; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8669; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]8670; GFX7-NEXT: s_cbranch_execnz .LBB119_18671; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end8672; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]8673; GFX7-NEXT: v_mov_b32_e32 v0, v48674; GFX7-NEXT: v_mov_b32_e32 v1, v58675; GFX7-NEXT: s_setpc_b64 s[30:31]8676;8677; GFX8-LABEL: flat_atomic_min_i64_ret:8678; GFX8: ; %bb.0:8679; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8680; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v08681; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc8682; GFX8-NEXT: flat_load_dword v4, v[0:1]8683; GFX8-NEXT: flat_load_dword v5, v[5:6]8684; GFX8-NEXT: s_mov_b64 s[4:5], 08685; GFX8-NEXT: .LBB119_1: ; %atomicrmw.start8686; GFX8-NEXT: ; =>This Inner Loop Header: Depth=18687; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8688; GFX8-NEXT: v_mov_b32_e32 v7, v58689; GFX8-NEXT: v_mov_b32_e32 v6, v48690; GFX8-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]8691; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc8692; GFX8-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc8693; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc8694; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8695; GFX8-NEXT: buffer_wbinvl1_vol8696; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]8697; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8698; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]8699; GFX8-NEXT: s_cbranch_execnz .LBB119_18700; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end8701; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]8702; GFX8-NEXT: v_mov_b32_e32 v0, v48703; GFX8-NEXT: v_mov_b32_e32 v1, v58704; GFX8-NEXT: s_setpc_b64 s[30:31]8705;8706; GFX9-LABEL: flat_atomic_min_i64_ret:8707; GFX9: ; %bb.0:8708; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8709; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1]8710; GFX9-NEXT: s_mov_b64 s[4:5], 08711; GFX9-NEXT: .LBB119_1: ; %atomicrmw.start8712; GFX9-NEXT: ; =>This Inner Loop Header: Depth=18713; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8714; GFX9-NEXT: v_mov_b32_e32 v7, v58715; GFX9-NEXT: v_mov_b32_e32 v6, v48716; GFX9-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]8717; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc8718; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc8719; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc8720; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8721; GFX9-NEXT: buffer_wbinvl1_vol8722; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]8723; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8724; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]8725; GFX9-NEXT: s_cbranch_execnz .LBB119_18726; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end8727; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]8728; GFX9-NEXT: v_mov_b32_e32 v0, v48729; GFX9-NEXT: v_mov_b32_e32 v1, v58730; GFX9-NEXT: s_setpc_b64 s[30:31]8731 %result = atomicrmw min ptr %ptr, i64 %in seq_cst, !noalias.addrspace !18732 ret i64 %result8733}8734 8735define i64 @flat_atomic_min_i64_ret_offset(ptr %out, i64 %in) {8736; GFX7-LABEL: flat_atomic_min_i64_ret_offset:8737; GFX7: ; %bb.0:8738; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8739; GFX7-NEXT: v_add_i32_e32 v4, vcc, 32, v08740; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc8741; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v08742; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc8743; GFX7-NEXT: flat_load_dword v1, v[0:1]8744; GFX7-NEXT: flat_load_dword v0, v[4:5]8745; GFX7-NEXT: s_mov_b64 s[4:5], 08746; GFX7-NEXT: .LBB120_1: ; %atomicrmw.start8747; GFX7-NEXT: ; =>This Inner Loop Header: Depth=18748; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8749; GFX7-NEXT: v_mov_b32_e32 v9, v18750; GFX7-NEXT: v_mov_b32_e32 v8, v08751; GFX7-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]8752; GFX7-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc8753; GFX7-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc8754; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc8755; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8756; GFX7-NEXT: buffer_wbinvl1_vol8757; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]8758; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8759; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]8760; GFX7-NEXT: s_cbranch_execnz .LBB120_18761; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end8762; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]8763; GFX7-NEXT: s_setpc_b64 s[30:31]8764;8765; GFX8-LABEL: flat_atomic_min_i64_ret_offset:8766; GFX8: ; %bb.0:8767; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8768; GFX8-NEXT: v_add_u32_e32 v4, vcc, 32, v08769; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc8770; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v08771; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc8772; GFX8-NEXT: flat_load_dword v1, v[0:1]8773; GFX8-NEXT: flat_load_dword v0, v[4:5]8774; GFX8-NEXT: s_mov_b64 s[4:5], 08775; GFX8-NEXT: .LBB120_1: ; %atomicrmw.start8776; GFX8-NEXT: ; =>This Inner Loop Header: Depth=18777; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8778; GFX8-NEXT: v_mov_b32_e32 v9, v18779; GFX8-NEXT: v_mov_b32_e32 v8, v08780; GFX8-NEXT: v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]8781; GFX8-NEXT: v_cndmask_b32_e32 v7, v3, v9, vcc8782; GFX8-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc8783; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc8784; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8785; GFX8-NEXT: buffer_wbinvl1_vol8786; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]8787; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8788; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]8789; GFX8-NEXT: s_cbranch_execnz .LBB120_18790; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end8791; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]8792; GFX8-NEXT: s_setpc_b64 s[30:31]8793;8794; GFX9-LABEL: flat_atomic_min_i64_ret_offset:8795; GFX9: ; %bb.0:8796; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8797; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:328798; GFX9-NEXT: s_mov_b64 s[4:5], 08799; GFX9-NEXT: .LBB120_1: ; %atomicrmw.start8800; GFX9-NEXT: ; =>This Inner Loop Header: Depth=18801; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8802; GFX9-NEXT: v_mov_b32_e32 v7, v58803; GFX9-NEXT: v_mov_b32_e32 v6, v48804; GFX9-NEXT: v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]8805; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc8806; GFX9-NEXT: v_cndmask_b32_e32 v4, v2, v6, vcc8807; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc8808; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8809; GFX9-NEXT: buffer_wbinvl1_vol8810; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]8811; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8812; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]8813; GFX9-NEXT: s_cbranch_execnz .LBB120_18814; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end8815; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]8816; GFX9-NEXT: v_mov_b32_e32 v0, v48817; GFX9-NEXT: v_mov_b32_e32 v1, v58818; GFX9-NEXT: s_setpc_b64 s[30:31]8819 %gep = getelementptr inbounds i64, ptr %out, i64 48820 %result = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !18821 ret i64 %result8822}8823 8824define amdgpu_gfx void @flat_atomic_min_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {8825; GFX7-LABEL: flat_atomic_min_i64_noret_scalar:8826; GFX7: ; %bb.0:8827; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8828; GFX7-NEXT: v_mov_b32_e32 v0, s48829; GFX7-NEXT: s_add_u32 s34, s4, 48830; GFX7-NEXT: v_mov_b32_e32 v1, s58831; GFX7-NEXT: s_addc_u32 s35, s5, 08832; GFX7-NEXT: v_mov_b32_e32 v3, s348833; GFX7-NEXT: v_mov_b32_e32 v4, s358834; GFX7-NEXT: flat_load_dword v2, v[0:1]8835; GFX7-NEXT: flat_load_dword v3, v[3:4]8836; GFX7-NEXT: v_mov_b32_e32 v4, s48837; GFX7-NEXT: s_mov_b64 s[34:35], 08838; GFX7-NEXT: v_mov_b32_e32 v6, s78839; GFX7-NEXT: v_mov_b32_e32 v7, s68840; GFX7-NEXT: v_mov_b32_e32 v5, s58841; GFX7-NEXT: .LBB121_1: ; %atomicrmw.start8842; GFX7-NEXT: ; =>This Inner Loop Header: Depth=18843; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8844; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]8845; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc8846; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc8847; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc8848; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8849; GFX7-NEXT: buffer_wbinvl1_vol8850; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8851; GFX7-NEXT: v_mov_b32_e32 v3, v18852; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]8853; GFX7-NEXT: v_mov_b32_e32 v2, v08854; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]8855; GFX7-NEXT: s_cbranch_execnz .LBB121_18856; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end8857; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]8858; GFX7-NEXT: s_setpc_b64 s[30:31]8859;8860; GFX8-LABEL: flat_atomic_min_i64_noret_scalar:8861; GFX8: ; %bb.0:8862; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8863; GFX8-NEXT: v_mov_b32_e32 v0, s48864; GFX8-NEXT: s_add_u32 s34, s4, 48865; GFX8-NEXT: v_mov_b32_e32 v1, s58866; GFX8-NEXT: s_addc_u32 s35, s5, 08867; GFX8-NEXT: v_mov_b32_e32 v3, s348868; GFX8-NEXT: v_mov_b32_e32 v4, s358869; GFX8-NEXT: flat_load_dword v2, v[0:1]8870; GFX8-NEXT: flat_load_dword v3, v[3:4]8871; GFX8-NEXT: v_mov_b32_e32 v4, s48872; GFX8-NEXT: s_mov_b64 s[34:35], 08873; GFX8-NEXT: v_mov_b32_e32 v6, s78874; GFX8-NEXT: v_mov_b32_e32 v7, s68875; GFX8-NEXT: v_mov_b32_e32 v5, s58876; GFX8-NEXT: .LBB121_1: ; %atomicrmw.start8877; GFX8-NEXT: ; =>This Inner Loop Header: Depth=18878; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8879; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]8880; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc8881; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc8882; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc8883; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8884; GFX8-NEXT: buffer_wbinvl1_vol8885; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8886; GFX8-NEXT: v_mov_b32_e32 v3, v18887; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]8888; GFX8-NEXT: v_mov_b32_e32 v2, v08889; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]8890; GFX8-NEXT: s_cbranch_execnz .LBB121_18891; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end8892; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]8893; GFX8-NEXT: s_setpc_b64 s[30:31]8894;8895; GFX9-LABEL: flat_atomic_min_i64_noret_scalar:8896; GFX9: ; %bb.0:8897; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8898; GFX9-NEXT: v_mov_b32_e32 v0, s48899; GFX9-NEXT: v_mov_b32_e32 v1, s58900; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]8901; GFX9-NEXT: v_mov_b32_e32 v4, s48902; GFX9-NEXT: s_mov_b64 s[34:35], 08903; GFX9-NEXT: v_mov_b32_e32 v6, s78904; GFX9-NEXT: v_mov_b32_e32 v7, s68905; GFX9-NEXT: v_mov_b32_e32 v5, s58906; GFX9-NEXT: .LBB121_1: ; %atomicrmw.start8907; GFX9-NEXT: ; =>This Inner Loop Header: Depth=18908; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8909; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]8910; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc8911; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc8912; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc8913; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8914; GFX9-NEXT: buffer_wbinvl1_vol8915; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8916; GFX9-NEXT: v_mov_b32_e32 v3, v18917; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]8918; GFX9-NEXT: v_mov_b32_e32 v2, v08919; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]8920; GFX9-NEXT: s_cbranch_execnz .LBB121_18921; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end8922; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]8923; GFX9-NEXT: s_setpc_b64 s[30:31]8924 %tmp0 = atomicrmw min ptr %ptr, i64 %in seq_cst, !noalias.addrspace !18925 ret void8926}8927 8928define amdgpu_gfx void @flat_atomic_min_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {8929; GFX7-LABEL: flat_atomic_min_i64_noret_offset_scalar:8930; GFX7: ; %bb.0:8931; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8932; GFX7-NEXT: s_add_u32 s34, s4, 328933; GFX7-NEXT: s_addc_u32 s35, s5, 08934; GFX7-NEXT: s_add_u32 s36, s4, 368935; GFX7-NEXT: s_addc_u32 s37, s5, 08936; GFX7-NEXT: v_mov_b32_e32 v0, s368937; GFX7-NEXT: v_mov_b32_e32 v1, s378938; GFX7-NEXT: v_mov_b32_e32 v4, s348939; GFX7-NEXT: v_mov_b32_e32 v5, s358940; GFX7-NEXT: flat_load_dword v3, v[0:1]8941; GFX7-NEXT: flat_load_dword v2, v[4:5]8942; GFX7-NEXT: s_mov_b64 s[34:35], 08943; GFX7-NEXT: v_mov_b32_e32 v6, s78944; GFX7-NEXT: v_mov_b32_e32 v7, s68945; GFX7-NEXT: .LBB122_1: ; %atomicrmw.start8946; GFX7-NEXT: ; =>This Inner Loop Header: Depth=18947; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8948; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]8949; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc8950; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc8951; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc8952; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8953; GFX7-NEXT: buffer_wbinvl1_vol8954; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8955; GFX7-NEXT: v_mov_b32_e32 v3, v18956; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]8957; GFX7-NEXT: v_mov_b32_e32 v2, v08958; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]8959; GFX7-NEXT: s_cbranch_execnz .LBB122_18960; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end8961; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]8962; GFX7-NEXT: s_setpc_b64 s[30:31]8963;8964; GFX8-LABEL: flat_atomic_min_i64_noret_offset_scalar:8965; GFX8: ; %bb.0:8966; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8967; GFX8-NEXT: s_add_u32 s34, s4, 328968; GFX8-NEXT: s_addc_u32 s35, s5, 08969; GFX8-NEXT: s_add_u32 s36, s4, 368970; GFX8-NEXT: s_addc_u32 s37, s5, 08971; GFX8-NEXT: v_mov_b32_e32 v0, s368972; GFX8-NEXT: v_mov_b32_e32 v1, s378973; GFX8-NEXT: v_mov_b32_e32 v4, s348974; GFX8-NEXT: v_mov_b32_e32 v5, s358975; GFX8-NEXT: flat_load_dword v3, v[0:1]8976; GFX8-NEXT: flat_load_dword v2, v[4:5]8977; GFX8-NEXT: s_mov_b64 s[34:35], 08978; GFX8-NEXT: v_mov_b32_e32 v6, s78979; GFX8-NEXT: v_mov_b32_e32 v7, s68980; GFX8-NEXT: .LBB122_1: ; %atomicrmw.start8981; GFX8-NEXT: ; =>This Inner Loop Header: Depth=18982; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8983; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]8984; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc8985; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc8986; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc8987; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8988; GFX8-NEXT: buffer_wbinvl1_vol8989; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]8990; GFX8-NEXT: v_mov_b32_e32 v3, v18991; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]8992; GFX8-NEXT: v_mov_b32_e32 v2, v08993; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]8994; GFX8-NEXT: s_cbranch_execnz .LBB122_18995; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end8996; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]8997; GFX8-NEXT: s_setpc_b64 s[30:31]8998;8999; GFX9-LABEL: flat_atomic_min_i64_noret_offset_scalar:9000; GFX9: ; %bb.0:9001; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9002; GFX9-NEXT: v_mov_b32_e32 v0, s49003; GFX9-NEXT: v_mov_b32_e32 v1, s59004; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:329005; GFX9-NEXT: v_mov_b32_e32 v4, s49006; GFX9-NEXT: s_mov_b64 s[34:35], 09007; GFX9-NEXT: v_mov_b32_e32 v6, s79008; GFX9-NEXT: v_mov_b32_e32 v7, s69009; GFX9-NEXT: v_mov_b32_e32 v5, s59010; GFX9-NEXT: .LBB122_1: ; %atomicrmw.start9011; GFX9-NEXT: ; =>This Inner Loop Header: Depth=19012; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9013; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]9014; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc9015; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc9016; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc9017; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9018; GFX9-NEXT: buffer_wbinvl1_vol9019; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]9020; GFX9-NEXT: v_mov_b32_e32 v3, v19021; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]9022; GFX9-NEXT: v_mov_b32_e32 v2, v09023; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]9024; GFX9-NEXT: s_cbranch_execnz .LBB122_19025; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end9026; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]9027; GFX9-NEXT: s_setpc_b64 s[30:31]9028 %gep = getelementptr inbounds i64, ptr %out, i64 49029 %tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !19030 ret void9031}9032 9033define amdgpu_gfx i64 @flat_atomic_min_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {9034; GFX7-LABEL: flat_atomic_min_i64_ret_scalar:9035; GFX7: ; %bb.0:9036; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9037; GFX7-NEXT: v_mov_b32_e32 v0, s49038; GFX7-NEXT: s_add_u32 s34, s4, 49039; GFX7-NEXT: v_mov_b32_e32 v1, s59040; GFX7-NEXT: s_addc_u32 s35, s5, 09041; GFX7-NEXT: v_mov_b32_e32 v2, s349042; GFX7-NEXT: v_mov_b32_e32 v3, s359043; GFX7-NEXT: flat_load_dword v0, v[0:1]9044; GFX7-NEXT: flat_load_dword v1, v[2:3]9045; GFX7-NEXT: v_mov_b32_e32 v2, s49046; GFX7-NEXT: s_mov_b64 s[34:35], 09047; GFX7-NEXT: v_mov_b32_e32 v4, s79048; GFX7-NEXT: v_mov_b32_e32 v5, s69049; GFX7-NEXT: v_mov_b32_e32 v3, s59050; GFX7-NEXT: .LBB123_1: ; %atomicrmw.start9051; GFX7-NEXT: ; =>This Inner Loop Header: Depth=19052; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9053; GFX7-NEXT: v_mov_b32_e32 v9, v19054; GFX7-NEXT: v_mov_b32_e32 v8, v09055; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]9056; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc9057; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc9058; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc9059; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9060; GFX7-NEXT: buffer_wbinvl1_vol9061; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]9062; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]9063; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]9064; GFX7-NEXT: s_cbranch_execnz .LBB123_19065; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end9066; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]9067; GFX7-NEXT: s_setpc_b64 s[30:31]9068;9069; GFX8-LABEL: flat_atomic_min_i64_ret_scalar:9070; GFX8: ; %bb.0:9071; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9072; GFX8-NEXT: v_mov_b32_e32 v0, s49073; GFX8-NEXT: s_add_u32 s34, s4, 49074; GFX8-NEXT: v_mov_b32_e32 v1, s59075; GFX8-NEXT: s_addc_u32 s35, s5, 09076; GFX8-NEXT: v_mov_b32_e32 v2, s349077; GFX8-NEXT: v_mov_b32_e32 v3, s359078; GFX8-NEXT: flat_load_dword v0, v[0:1]9079; GFX8-NEXT: flat_load_dword v1, v[2:3]9080; GFX8-NEXT: v_mov_b32_e32 v2, s49081; GFX8-NEXT: s_mov_b64 s[34:35], 09082; GFX8-NEXT: v_mov_b32_e32 v4, s79083; GFX8-NEXT: v_mov_b32_e32 v5, s69084; GFX8-NEXT: v_mov_b32_e32 v3, s59085; GFX8-NEXT: .LBB123_1: ; %atomicrmw.start9086; GFX8-NEXT: ; =>This Inner Loop Header: Depth=19087; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9088; GFX8-NEXT: v_mov_b32_e32 v9, v19089; GFX8-NEXT: v_mov_b32_e32 v8, v09090; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]9091; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc9092; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc9093; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc9094; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9095; GFX8-NEXT: buffer_wbinvl1_vol9096; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]9097; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]9098; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]9099; GFX8-NEXT: s_cbranch_execnz .LBB123_19100; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end9101; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]9102; GFX8-NEXT: s_setpc_b64 s[30:31]9103;9104; GFX9-LABEL: flat_atomic_min_i64_ret_scalar:9105; GFX9: ; %bb.0:9106; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9107; GFX9-NEXT: v_mov_b32_e32 v0, s49108; GFX9-NEXT: v_mov_b32_e32 v1, s59109; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]9110; GFX9-NEXT: v_mov_b32_e32 v2, s49111; GFX9-NEXT: s_mov_b64 s[34:35], 09112; GFX9-NEXT: v_mov_b32_e32 v4, s79113; GFX9-NEXT: v_mov_b32_e32 v5, s69114; GFX9-NEXT: v_mov_b32_e32 v3, s59115; GFX9-NEXT: .LBB123_1: ; %atomicrmw.start9116; GFX9-NEXT: ; =>This Inner Loop Header: Depth=19117; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9118; GFX9-NEXT: v_mov_b32_e32 v9, v19119; GFX9-NEXT: v_mov_b32_e32 v8, v09120; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]9121; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc9122; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc9123; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc9124; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9125; GFX9-NEXT: buffer_wbinvl1_vol9126; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]9127; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]9128; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]9129; GFX9-NEXT: s_cbranch_execnz .LBB123_19130; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end9131; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]9132; GFX9-NEXT: s_setpc_b64 s[30:31]9133 %result = atomicrmw min ptr %ptr, i64 %in seq_cst, !noalias.addrspace !19134 ret i64 %result9135}9136 9137define amdgpu_gfx i64 @flat_atomic_min_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {9138; GFX7-LABEL: flat_atomic_min_i64_ret_offset_scalar:9139; GFX7: ; %bb.0:9140; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9141; GFX7-NEXT: s_add_u32 s34, s4, 329142; GFX7-NEXT: s_addc_u32 s35, s5, 09143; GFX7-NEXT: s_add_u32 s36, s4, 369144; GFX7-NEXT: s_addc_u32 s37, s5, 09145; GFX7-NEXT: v_mov_b32_e32 v0, s369146; GFX7-NEXT: v_mov_b32_e32 v1, s379147; GFX7-NEXT: v_mov_b32_e32 v2, s349148; GFX7-NEXT: v_mov_b32_e32 v3, s359149; GFX7-NEXT: flat_load_dword v1, v[0:1]9150; GFX7-NEXT: flat_load_dword v0, v[2:3]9151; GFX7-NEXT: s_mov_b64 s[34:35], 09152; GFX7-NEXT: v_mov_b32_e32 v4, s79153; GFX7-NEXT: v_mov_b32_e32 v5, s69154; GFX7-NEXT: .LBB124_1: ; %atomicrmw.start9155; GFX7-NEXT: ; =>This Inner Loop Header: Depth=19156; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9157; GFX7-NEXT: v_mov_b32_e32 v9, v19158; GFX7-NEXT: v_mov_b32_e32 v8, v09159; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]9160; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc9161; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc9162; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc9163; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9164; GFX7-NEXT: buffer_wbinvl1_vol9165; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]9166; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]9167; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]9168; GFX7-NEXT: s_cbranch_execnz .LBB124_19169; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end9170; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]9171; GFX7-NEXT: s_setpc_b64 s[30:31]9172;9173; GFX8-LABEL: flat_atomic_min_i64_ret_offset_scalar:9174; GFX8: ; %bb.0:9175; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9176; GFX8-NEXT: s_add_u32 s34, s4, 329177; GFX8-NEXT: s_addc_u32 s35, s5, 09178; GFX8-NEXT: s_add_u32 s36, s4, 369179; GFX8-NEXT: s_addc_u32 s37, s5, 09180; GFX8-NEXT: v_mov_b32_e32 v0, s369181; GFX8-NEXT: v_mov_b32_e32 v1, s379182; GFX8-NEXT: v_mov_b32_e32 v2, s349183; GFX8-NEXT: v_mov_b32_e32 v3, s359184; GFX8-NEXT: flat_load_dword v1, v[0:1]9185; GFX8-NEXT: flat_load_dword v0, v[2:3]9186; GFX8-NEXT: s_mov_b64 s[34:35], 09187; GFX8-NEXT: v_mov_b32_e32 v4, s79188; GFX8-NEXT: v_mov_b32_e32 v5, s69189; GFX8-NEXT: .LBB124_1: ; %atomicrmw.start9190; GFX8-NEXT: ; =>This Inner Loop Header: Depth=19191; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9192; GFX8-NEXT: v_mov_b32_e32 v9, v19193; GFX8-NEXT: v_mov_b32_e32 v8, v09194; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]9195; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc9196; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc9197; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc9198; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9199; GFX8-NEXT: buffer_wbinvl1_vol9200; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]9201; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]9202; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]9203; GFX8-NEXT: s_cbranch_execnz .LBB124_19204; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end9205; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]9206; GFX8-NEXT: s_setpc_b64 s[30:31]9207;9208; GFX9-LABEL: flat_atomic_min_i64_ret_offset_scalar:9209; GFX9: ; %bb.0:9210; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9211; GFX9-NEXT: v_mov_b32_e32 v0, s49212; GFX9-NEXT: v_mov_b32_e32 v1, s59213; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:329214; GFX9-NEXT: v_mov_b32_e32 v2, s49215; GFX9-NEXT: s_mov_b64 s[34:35], 09216; GFX9-NEXT: v_mov_b32_e32 v4, s79217; GFX9-NEXT: v_mov_b32_e32 v5, s69218; GFX9-NEXT: v_mov_b32_e32 v3, s59219; GFX9-NEXT: .LBB124_1: ; %atomicrmw.start9220; GFX9-NEXT: ; =>This Inner Loop Header: Depth=19221; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9222; GFX9-NEXT: v_mov_b32_e32 v9, v19223; GFX9-NEXT: v_mov_b32_e32 v8, v09224; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]9225; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc9226; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc9227; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc9228; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9229; GFX9-NEXT: buffer_wbinvl1_vol9230; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]9231; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]9232; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]9233; GFX9-NEXT: s_cbranch_execnz .LBB124_19234; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end9235; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]9236; GFX9-NEXT: s_setpc_b64 s[30:31]9237 %gep = getelementptr inbounds i64, ptr %out, i64 49238 %result = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !19239 ret i64 %result9240}9241 9242define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {9243; GFX7-LABEL: atomic_min_i64_addr64_offset:9244; GFX7: ; %bb.0: ; %entry9245; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd9246; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x99247; GFX7-NEXT: s_waitcnt lgkmcnt(0)9248; GFX7-NEXT: s_lshl_b64 s[4:5], s[6:7], 39249; GFX7-NEXT: s_add_u32 s0, s0, s49250; GFX7-NEXT: s_addc_u32 s1, s1, s59251; GFX7-NEXT: s_add_u32 s0, s0, 329252; GFX7-NEXT: s_addc_u32 s1, s1, 09253; GFX7-NEXT: v_mov_b32_e32 v5, s19254; GFX7-NEXT: v_mov_b32_e32 v4, s09255; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5]9256; GFX7-NEXT: s_mov_b64 s[0:1], 09257; GFX7-NEXT: v_mov_b32_e32 v6, s39258; GFX7-NEXT: v_mov_b32_e32 v7, s29259; GFX7-NEXT: .LBB125_1: ; %atomicrmw.start9260; GFX7-NEXT: ; =>This Inner Loop Header: Depth=19261; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9262; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]9263; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc9264; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc9265; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc9266; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9267; GFX7-NEXT: buffer_wbinvl1_vol9268; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]9269; GFX7-NEXT: v_mov_b32_e32 v3, v19270; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]9271; GFX7-NEXT: v_mov_b32_e32 v2, v09272; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]9273; GFX7-NEXT: s_cbranch_execnz .LBB125_19274; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end9275; GFX7-NEXT: s_endpgm9276;9277; GFX8-LABEL: atomic_min_i64_addr64_offset:9278; GFX8: ; %bb.0: ; %entry9279; GFX8-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x349280; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249281; GFX8-NEXT: s_waitcnt lgkmcnt(0)9282; GFX8-NEXT: s_lshl_b64 s[4:5], s[6:7], 39283; GFX8-NEXT: s_add_u32 s0, s0, s49284; GFX8-NEXT: s_addc_u32 s1, s1, s59285; GFX8-NEXT: s_add_u32 s0, s0, 329286; GFX8-NEXT: s_addc_u32 s1, s1, 09287; GFX8-NEXT: v_mov_b32_e32 v5, s19288; GFX8-NEXT: v_mov_b32_e32 v4, s09289; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5]9290; GFX8-NEXT: s_mov_b64 s[0:1], 09291; GFX8-NEXT: v_mov_b32_e32 v6, s39292; GFX8-NEXT: v_mov_b32_e32 v7, s29293; GFX8-NEXT: .LBB125_1: ; %atomicrmw.start9294; GFX8-NEXT: ; =>This Inner Loop Header: Depth=19295; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9296; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]9297; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc9298; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc9299; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc9300; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9301; GFX8-NEXT: buffer_wbinvl1_vol9302; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]9303; GFX8-NEXT: v_mov_b32_e32 v3, v19304; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]9305; GFX8-NEXT: v_mov_b32_e32 v2, v09306; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]9307; GFX8-NEXT: s_cbranch_execnz .LBB125_19308; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end9309; GFX8-NEXT: s_endpgm9310;9311; GFX9-LABEL: atomic_min_i64_addr64_offset:9312; GFX9: ; %bb.0: ; %entry9313; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x349314; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249315; GFX9-NEXT: s_waitcnt lgkmcnt(0)9316; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 39317; GFX9-NEXT: s_add_u32 s0, s0, s49318; GFX9-NEXT: s_addc_u32 s1, s1, s59319; GFX9-NEXT: v_mov_b32_e32 v5, s19320; GFX9-NEXT: v_mov_b32_e32 v4, s09321; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[4:5] offset:329322; GFX9-NEXT: s_mov_b64 s[0:1], 09323; GFX9-NEXT: v_mov_b32_e32 v6, s39324; GFX9-NEXT: v_mov_b32_e32 v7, s29325; GFX9-NEXT: .LBB125_1: ; %atomicrmw.start9326; GFX9-NEXT: ; =>This Inner Loop Header: Depth=19327; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9328; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]9329; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc9330; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc9331; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc9332; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9333; GFX9-NEXT: buffer_wbinvl1_vol9334; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]9335; GFX9-NEXT: v_mov_b32_e32 v3, v19336; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]9337; GFX9-NEXT: v_mov_b32_e32 v2, v09338; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]9339; GFX9-NEXT: s_cbranch_execnz .LBB125_19340; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end9341; GFX9-NEXT: s_endpgm9342entry:9343 %ptr = getelementptr inbounds i64, ptr %out, i64 %index9344 %gep = getelementptr inbounds i64, ptr %ptr, i64 49345 %tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !19346 ret void9347}9348 9349define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {9350; GFX7-LABEL: atomic_min_i64_ret_addr64_offset:9351; GFX7: ; %bb.0: ; %entry9352; GFX7-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x99353; GFX7-NEXT: s_waitcnt lgkmcnt(0)9354; GFX7-NEXT: s_lshl_b64 s[6:7], s[6:7], 39355; GFX7-NEXT: s_add_u32 s0, s0, s69356; GFX7-NEXT: s_addc_u32 s1, s1, s79357; GFX7-NEXT: s_add_u32 s0, s0, 329358; GFX7-NEXT: s_addc_u32 s1, s1, 09359; GFX7-NEXT: v_mov_b32_e32 v0, s09360; GFX7-NEXT: v_mov_b32_e32 v1, s19361; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]9362; GFX7-NEXT: s_mov_b64 s[0:1], 09363; GFX7-NEXT: v_mov_b32_e32 v4, s59364; GFX7-NEXT: v_mov_b32_e32 v5, s49365; GFX7-NEXT: .LBB126_1: ; %atomicrmw.start9366; GFX7-NEXT: ; =>This Inner Loop Header: Depth=19367; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9368; GFX7-NEXT: v_mov_b32_e32 v9, v39369; GFX7-NEXT: v_mov_b32_e32 v8, v29370; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]9371; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc9372; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc9373; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc9374; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9375; GFX7-NEXT: buffer_wbinvl1_vol9376; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]9377; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]9378; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]9379; GFX7-NEXT: s_cbranch_execnz .LBB126_19380; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end9381; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]9382; GFX7-NEXT: v_mov_b32_e32 v0, s29383; GFX7-NEXT: v_mov_b32_e32 v1, s39384; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]9385; GFX7-NEXT: s_endpgm9386;9387; GFX8-LABEL: atomic_min_i64_ret_addr64_offset:9388; GFX8: ; %bb.0: ; %entry9389; GFX8-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x249390; GFX8-NEXT: s_waitcnt lgkmcnt(0)9391; GFX8-NEXT: s_lshl_b64 s[6:7], s[6:7], 39392; GFX8-NEXT: s_add_u32 s0, s0, s69393; GFX8-NEXT: s_addc_u32 s1, s1, s79394; GFX8-NEXT: s_add_u32 s0, s0, 329395; GFX8-NEXT: s_addc_u32 s1, s1, 09396; GFX8-NEXT: v_mov_b32_e32 v0, s09397; GFX8-NEXT: v_mov_b32_e32 v1, s19398; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]9399; GFX8-NEXT: s_mov_b64 s[0:1], 09400; GFX8-NEXT: v_mov_b32_e32 v4, s59401; GFX8-NEXT: v_mov_b32_e32 v5, s49402; GFX8-NEXT: .LBB126_1: ; %atomicrmw.start9403; GFX8-NEXT: ; =>This Inner Loop Header: Depth=19404; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9405; GFX8-NEXT: v_mov_b32_e32 v9, v39406; GFX8-NEXT: v_mov_b32_e32 v8, v29407; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]9408; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc9409; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc9410; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc9411; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9412; GFX8-NEXT: buffer_wbinvl1_vol9413; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]9414; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]9415; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]9416; GFX8-NEXT: s_cbranch_execnz .LBB126_19417; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end9418; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]9419; GFX8-NEXT: v_mov_b32_e32 v0, s29420; GFX8-NEXT: v_mov_b32_e32 v1, s39421; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]9422; GFX8-NEXT: s_endpgm9423;9424; GFX9-LABEL: atomic_min_i64_ret_addr64_offset:9425; GFX9: ; %bb.0: ; %entry9426; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x249427; GFX9-NEXT: s_waitcnt lgkmcnt(0)9428; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 39429; GFX9-NEXT: s_add_u32 s0, s8, s09430; GFX9-NEXT: s_addc_u32 s1, s9, s19431; GFX9-NEXT: v_mov_b32_e32 v0, s09432; GFX9-NEXT: v_mov_b32_e32 v1, s19433; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:329434; GFX9-NEXT: s_mov_b64 s[0:1], 09435; GFX9-NEXT: v_mov_b32_e32 v4, s139436; GFX9-NEXT: v_mov_b32_e32 v5, s129437; GFX9-NEXT: .LBB126_1: ; %atomicrmw.start9438; GFX9-NEXT: ; =>This Inner Loop Header: Depth=19439; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9440; GFX9-NEXT: v_mov_b32_e32 v9, v39441; GFX9-NEXT: v_mov_b32_e32 v8, v29442; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9]9443; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc9444; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc9445; GFX9-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] offset:32 glc9446; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9447; GFX9-NEXT: buffer_wbinvl1_vol9448; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]9449; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]9450; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]9451; GFX9-NEXT: s_cbranch_execnz .LBB126_19452; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end9453; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]9454; GFX9-NEXT: v_mov_b32_e32 v0, s109455; GFX9-NEXT: v_mov_b32_e32 v1, s119456; GFX9-NEXT: flat_store_dwordx2 v[0:1], v[2:3]9457; GFX9-NEXT: s_endpgm9458entry:9459 %ptr = getelementptr inbounds i64, ptr %out, i64 %index9460 %gep = getelementptr inbounds i64, ptr %ptr, i64 49461 %tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst, !noalias.addrspace !19462 store i64 %tmp0, ptr %out29463 ret void9464}9465 9466define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {9467; GFX7-LABEL: atomic_min_i64:9468; GFX7: ; %bb.0: ; %entry9469; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x99470; GFX7-NEXT: s_mov_b64 s[4:5], 09471; GFX7-NEXT: s_waitcnt lgkmcnt(0)9472; GFX7-NEXT: v_mov_b32_e32 v0, s09473; GFX7-NEXT: v_mov_b32_e32 v1, s19474; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]9475; GFX7-NEXT: v_mov_b32_e32 v5, s19476; GFX7-NEXT: v_mov_b32_e32 v6, s39477; GFX7-NEXT: v_mov_b32_e32 v7, s29478; GFX7-NEXT: v_mov_b32_e32 v4, s09479; GFX7-NEXT: .LBB127_1: ; %atomicrmw.start9480; GFX7-NEXT: ; =>This Inner Loop Header: Depth=19481; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9482; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]9483; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc9484; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc9485; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc9486; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9487; GFX7-NEXT: buffer_wbinvl1_vol9488; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]9489; GFX7-NEXT: v_mov_b32_e32 v3, v19490; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9491; GFX7-NEXT: v_mov_b32_e32 v2, v09492; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]9493; GFX7-NEXT: s_cbranch_execnz .LBB127_19494; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end9495; GFX7-NEXT: s_endpgm9496;9497; GFX8-LABEL: atomic_min_i64:9498; GFX8: ; %bb.0: ; %entry9499; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249500; GFX8-NEXT: s_mov_b64 s[4:5], 09501; GFX8-NEXT: s_waitcnt lgkmcnt(0)9502; GFX8-NEXT: v_mov_b32_e32 v0, s09503; GFX8-NEXT: v_mov_b32_e32 v1, s19504; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]9505; GFX8-NEXT: v_mov_b32_e32 v5, s19506; GFX8-NEXT: v_mov_b32_e32 v6, s39507; GFX8-NEXT: v_mov_b32_e32 v7, s29508; GFX8-NEXT: v_mov_b32_e32 v4, s09509; GFX8-NEXT: .LBB127_1: ; %atomicrmw.start9510; GFX8-NEXT: ; =>This Inner Loop Header: Depth=19511; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9512; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]9513; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc9514; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc9515; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc9516; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9517; GFX8-NEXT: buffer_wbinvl1_vol9518; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]9519; GFX8-NEXT: v_mov_b32_e32 v3, v19520; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9521; GFX8-NEXT: v_mov_b32_e32 v2, v09522; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]9523; GFX8-NEXT: s_cbranch_execnz .LBB127_19524; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end9525; GFX8-NEXT: s_endpgm9526;9527; GFX9-LABEL: atomic_min_i64:9528; GFX9: ; %bb.0: ; %entry9529; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249530; GFX9-NEXT: s_mov_b64 s[4:5], 09531; GFX9-NEXT: s_waitcnt lgkmcnt(0)9532; GFX9-NEXT: v_mov_b32_e32 v0, s09533; GFX9-NEXT: v_mov_b32_e32 v1, s19534; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]9535; GFX9-NEXT: v_mov_b32_e32 v5, s19536; GFX9-NEXT: v_mov_b32_e32 v6, s39537; GFX9-NEXT: v_mov_b32_e32 v7, s29538; GFX9-NEXT: v_mov_b32_e32 v4, s09539; GFX9-NEXT: .LBB127_1: ; %atomicrmw.start9540; GFX9-NEXT: ; =>This Inner Loop Header: Depth=19541; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9542; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]9543; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc9544; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc9545; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc9546; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9547; GFX9-NEXT: buffer_wbinvl1_vol9548; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]9549; GFX9-NEXT: v_mov_b32_e32 v3, v19550; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9551; GFX9-NEXT: v_mov_b32_e32 v2, v09552; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]9553; GFX9-NEXT: s_cbranch_execnz .LBB127_19554; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end9555; GFX9-NEXT: s_endpgm9556entry:9557 %tmp0 = atomicrmw min ptr %out, i64 %in seq_cst, !noalias.addrspace !19558 ret void9559}9560 9561define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %in, i64 %index) {9562; GFX7-LABEL: atomic_min_i64_ret_addr64:9563; GFX7: ; %bb.0: ; %entry9564; GFX7-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x99565; GFX7-NEXT: s_waitcnt lgkmcnt(0)9566; GFX7-NEXT: s_lshl_b64 s[6:7], s[6:7], 39567; GFX7-NEXT: s_add_u32 s0, s0, s69568; GFX7-NEXT: s_addc_u32 s1, s1, s79569; GFX7-NEXT: v_mov_b32_e32 v0, s09570; GFX7-NEXT: v_mov_b32_e32 v1, s19571; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]9572; GFX7-NEXT: s_mov_b64 s[0:1], 09573; GFX7-NEXT: v_mov_b32_e32 v4, s59574; GFX7-NEXT: v_mov_b32_e32 v5, s49575; GFX7-NEXT: .LBB128_1: ; %atomicrmw.start9576; GFX7-NEXT: ; =>This Inner Loop Header: Depth=19577; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9578; GFX7-NEXT: v_mov_b32_e32 v9, v39579; GFX7-NEXT: v_mov_b32_e32 v8, v29580; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]9581; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc9582; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc9583; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc9584; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9585; GFX7-NEXT: buffer_wbinvl1_vol9586; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]9587; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]9588; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]9589; GFX7-NEXT: s_cbranch_execnz .LBB128_19590; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end9591; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]9592; GFX7-NEXT: v_mov_b32_e32 v0, s29593; GFX7-NEXT: v_mov_b32_e32 v1, s39594; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]9595; GFX7-NEXT: s_endpgm9596;9597; GFX8-LABEL: atomic_min_i64_ret_addr64:9598; GFX8: ; %bb.0: ; %entry9599; GFX8-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x249600; GFX8-NEXT: s_waitcnt lgkmcnt(0)9601; GFX8-NEXT: s_lshl_b64 s[6:7], s[6:7], 39602; GFX8-NEXT: s_add_u32 s0, s0, s69603; GFX8-NEXT: s_addc_u32 s1, s1, s79604; GFX8-NEXT: v_mov_b32_e32 v0, s09605; GFX8-NEXT: v_mov_b32_e32 v1, s19606; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]9607; GFX8-NEXT: s_mov_b64 s[0:1], 09608; GFX8-NEXT: v_mov_b32_e32 v4, s59609; GFX8-NEXT: v_mov_b32_e32 v5, s49610; GFX8-NEXT: .LBB128_1: ; %atomicrmw.start9611; GFX8-NEXT: ; =>This Inner Loop Header: Depth=19612; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9613; GFX8-NEXT: v_mov_b32_e32 v9, v39614; GFX8-NEXT: v_mov_b32_e32 v8, v29615; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]9616; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc9617; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc9618; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc9619; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9620; GFX8-NEXT: buffer_wbinvl1_vol9621; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]9622; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]9623; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]9624; GFX8-NEXT: s_cbranch_execnz .LBB128_19625; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end9626; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]9627; GFX8-NEXT: v_mov_b32_e32 v0, s29628; GFX8-NEXT: v_mov_b32_e32 v1, s39629; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]9630; GFX8-NEXT: s_endpgm9631;9632; GFX9-LABEL: atomic_min_i64_ret_addr64:9633; GFX9: ; %bb.0: ; %entry9634; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x249635; GFX9-NEXT: s_waitcnt lgkmcnt(0)9636; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 39637; GFX9-NEXT: s_add_u32 s0, s8, s09638; GFX9-NEXT: s_addc_u32 s1, s9, s19639; GFX9-NEXT: v_mov_b32_e32 v0, s09640; GFX9-NEXT: v_mov_b32_e32 v1, s19641; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]9642; GFX9-NEXT: s_mov_b64 s[0:1], 09643; GFX9-NEXT: v_mov_b32_e32 v4, s139644; GFX9-NEXT: v_mov_b32_e32 v5, s129645; GFX9-NEXT: .LBB128_1: ; %atomicrmw.start9646; GFX9-NEXT: ; =>This Inner Loop Header: Depth=19647; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9648; GFX9-NEXT: v_mov_b32_e32 v9, v39649; GFX9-NEXT: v_mov_b32_e32 v8, v29650; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9]9651; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc9652; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc9653; GFX9-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc9654; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9655; GFX9-NEXT: buffer_wbinvl1_vol9656; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]9657; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]9658; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]9659; GFX9-NEXT: s_cbranch_execnz .LBB128_19660; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end9661; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]9662; GFX9-NEXT: v_mov_b32_e32 v0, s109663; GFX9-NEXT: v_mov_b32_e32 v1, s119664; GFX9-NEXT: flat_store_dwordx2 v[0:1], v[2:3]9665; GFX9-NEXT: s_endpgm9666entry:9667 %ptr = getelementptr inbounds i64, ptr %out, i64 %index9668 %tmp0 = atomicrmw min ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1, !noalias.addrspace !19669 store i64 %tmp0, ptr %out29670 ret void9671}9672 9673define void @flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {9674; GFX7-LABEL: flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory:9675; GFX7: ; %bb.0:9676; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9677; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v09678; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc9679; GFX7-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3]9680; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9681; GFX7-NEXT: buffer_wbinvl1_vol9682; GFX7-NEXT: s_setpc_b64 s[30:31]9683;9684; GFX8-LABEL: flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory:9685; GFX8: ; %bb.0:9686; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9687; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v09688; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc9689; GFX8-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3]9690; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9691; GFX8-NEXT: buffer_wbinvl1_vol9692; GFX8-NEXT: s_setpc_b64 s[30:31]9693;9694; GFX9-LABEL: flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory:9695; GFX9: ; %bb.0:9696; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9697; GFX9-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3] offset:329698; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9699; GFX9-NEXT: buffer_wbinvl1_vol9700; GFX9-NEXT: s_setpc_b64 s[30:31]9701 %gep = getelementptr inbounds i64, ptr %out, i64 49702 %tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !19703 ret void9704}9705 9706define i64 @flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {9707; GFX7-LABEL: flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory:9708; GFX7: ; %bb.0:9709; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9710; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v09711; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc9712; GFX7-NEXT: flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc9713; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9714; GFX7-NEXT: buffer_wbinvl1_vol9715; GFX7-NEXT: s_setpc_b64 s[30:31]9716;9717; GFX8-LABEL: flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory:9718; GFX8: ; %bb.0:9719; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9720; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v09721; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc9722; GFX8-NEXT: flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc9723; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9724; GFX8-NEXT: buffer_wbinvl1_vol9725; GFX8-NEXT: s_setpc_b64 s[30:31]9726;9727; GFX9-LABEL: flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory:9728; GFX9: ; %bb.0:9729; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9730; GFX9-NEXT: flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] offset:32 glc9731; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9732; GFX9-NEXT: buffer_wbinvl1_vol9733; GFX9-NEXT: s_setpc_b64 s[30:31]9734 %gep = getelementptr inbounds i64, ptr %out, i64 49735 %result = atomicrmw min ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !19736 ret i64 %result9737}9738 9739; ---------------------------------------------------------------------9740; atomicrmw uinc_wrap9741; ---------------------------------------------------------------------9742 9743define void @flat_atomic_uinc_wrap_i64_noret(ptr %ptr, i64 %in) {9744; GFX7-LABEL: flat_atomic_uinc_wrap_i64_noret:9745; GFX7: ; %bb.0:9746; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9747; GFX7-NEXT: v_add_i32_e32 v4, vcc, 4, v09748; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc9749; GFX7-NEXT: flat_load_dword v6, v[0:1]9750; GFX7-NEXT: flat_load_dword v7, v[4:5]9751; GFX7-NEXT: s_mov_b64 s[4:5], 09752; GFX7-NEXT: .LBB131_1: ; %atomicrmw.start9753; GFX7-NEXT: ; =>This Inner Loop Header: Depth=19754; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9755; GFX7-NEXT: v_add_i32_e32 v4, vcc, 1, v69756; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc9757; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]9758; GFX7-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc9759; GFX7-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc9760; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc9761; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9762; GFX7-NEXT: buffer_wbinvl1_vol9763; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]9764; GFX7-NEXT: v_mov_b32_e32 v7, v59765; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9766; GFX7-NEXT: v_mov_b32_e32 v6, v49767; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]9768; GFX7-NEXT: s_cbranch_execnz .LBB131_19769; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end9770; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]9771; GFX7-NEXT: s_setpc_b64 s[30:31]9772;9773; GFX8-LABEL: flat_atomic_uinc_wrap_i64_noret:9774; GFX8: ; %bb.0:9775; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9776; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v09777; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc9778; GFX8-NEXT: flat_load_dword v6, v[0:1]9779; GFX8-NEXT: flat_load_dword v7, v[4:5]9780; GFX8-NEXT: s_mov_b64 s[4:5], 09781; GFX8-NEXT: .LBB131_1: ; %atomicrmw.start9782; GFX8-NEXT: ; =>This Inner Loop Header: Depth=19783; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9784; GFX8-NEXT: v_add_u32_e32 v4, vcc, 1, v69785; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc9786; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]9787; GFX8-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc9788; GFX8-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc9789; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc9790; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9791; GFX8-NEXT: buffer_wbinvl1_vol9792; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]9793; GFX8-NEXT: v_mov_b32_e32 v7, v59794; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9795; GFX8-NEXT: v_mov_b32_e32 v6, v49796; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]9797; GFX8-NEXT: s_cbranch_execnz .LBB131_19798; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end9799; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]9800; GFX8-NEXT: s_setpc_b64 s[30:31]9801;9802; GFX9-LABEL: flat_atomic_uinc_wrap_i64_noret:9803; GFX9: ; %bb.0:9804; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9805; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1]9806; GFX9-NEXT: s_mov_b64 s[4:5], 09807; GFX9-NEXT: .LBB131_1: ; %atomicrmw.start9808; GFX9-NEXT: ; =>This Inner Loop Header: Depth=19809; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9810; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, 1, v69811; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v7, vcc9812; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]9813; GFX9-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc9814; GFX9-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc9815; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc9816; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9817; GFX9-NEXT: buffer_wbinvl1_vol9818; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]9819; GFX9-NEXT: v_mov_b32_e32 v7, v59820; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9821; GFX9-NEXT: v_mov_b32_e32 v6, v49822; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]9823; GFX9-NEXT: s_cbranch_execnz .LBB131_19824; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end9825; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]9826; GFX9-NEXT: s_setpc_b64 s[30:31]9827 %tmp0 = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !19828 ret void9829}9830 9831define void @flat_atomic_uinc_wrap_i64_noret_offset(ptr %out, i64 %in) {9832; GFX7-LABEL: flat_atomic_uinc_wrap_i64_noret_offset:9833; GFX7: ; %bb.0:9834; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9835; GFX7-NEXT: v_add_i32_e32 v8, vcc, 32, v09836; GFX7-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc9837; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v09838; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc9839; GFX7-NEXT: flat_load_dword v7, v[0:1]9840; GFX7-NEXT: flat_load_dword v6, v[8:9]9841; GFX7-NEXT: s_mov_b64 s[4:5], 09842; GFX7-NEXT: .LBB132_1: ; %atomicrmw.start9843; GFX7-NEXT: ; =>This Inner Loop Header: Depth=19844; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9845; GFX7-NEXT: v_add_i32_e32 v0, vcc, 1, v69846; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc9847; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]9848; GFX7-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc9849; GFX7-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc9850; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc9851; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9852; GFX7-NEXT: buffer_wbinvl1_vol9853; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]9854; GFX7-NEXT: v_mov_b32_e32 v7, v19855; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9856; GFX7-NEXT: v_mov_b32_e32 v6, v09857; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]9858; GFX7-NEXT: s_cbranch_execnz .LBB132_19859; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end9860; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]9861; GFX7-NEXT: s_setpc_b64 s[30:31]9862;9863; GFX8-LABEL: flat_atomic_uinc_wrap_i64_noret_offset:9864; GFX8: ; %bb.0:9865; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9866; GFX8-NEXT: v_add_u32_e32 v8, vcc, 32, v09867; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc9868; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v09869; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc9870; GFX8-NEXT: flat_load_dword v7, v[0:1]9871; GFX8-NEXT: flat_load_dword v6, v[8:9]9872; GFX8-NEXT: s_mov_b64 s[4:5], 09873; GFX8-NEXT: .LBB132_1: ; %atomicrmw.start9874; GFX8-NEXT: ; =>This Inner Loop Header: Depth=19875; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9876; GFX8-NEXT: v_add_u32_e32 v0, vcc, 1, v69877; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc9878; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]9879; GFX8-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc9880; GFX8-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc9881; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc9882; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9883; GFX8-NEXT: buffer_wbinvl1_vol9884; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]9885; GFX8-NEXT: v_mov_b32_e32 v7, v19886; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9887; GFX8-NEXT: v_mov_b32_e32 v6, v09888; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]9889; GFX8-NEXT: s_cbranch_execnz .LBB132_19890; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end9891; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]9892; GFX8-NEXT: s_setpc_b64 s[30:31]9893;9894; GFX9-LABEL: flat_atomic_uinc_wrap_i64_noret_offset:9895; GFX9: ; %bb.0:9896; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9897; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:329898; GFX9-NEXT: s_mov_b64 s[4:5], 09899; GFX9-NEXT: .LBB132_1: ; %atomicrmw.start9900; GFX9-NEXT: ; =>This Inner Loop Header: Depth=19901; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9902; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, 1, v69903; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v7, vcc9904; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]9905; GFX9-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc9906; GFX9-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc9907; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc9908; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9909; GFX9-NEXT: buffer_wbinvl1_vol9910; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]9911; GFX9-NEXT: v_mov_b32_e32 v7, v59912; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9913; GFX9-NEXT: v_mov_b32_e32 v6, v49914; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]9915; GFX9-NEXT: s_cbranch_execnz .LBB132_19916; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end9917; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]9918; GFX9-NEXT: s_setpc_b64 s[30:31]9919 %gep = getelementptr inbounds i64, ptr %out, i64 49920 %tmp0 = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !19921 ret void9922}9923 9924define i64 @flat_atomic_uinc_wrap_i64_ret(ptr %ptr, i64 %in) {9925; GFX7-LABEL: flat_atomic_uinc_wrap_i64_ret:9926; GFX7: ; %bb.0:9927; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9928; GFX7-NEXT: v_add_i32_e32 v5, vcc, 4, v09929; GFX7-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc9930; GFX7-NEXT: flat_load_dword v4, v[0:1]9931; GFX7-NEXT: flat_load_dword v5, v[5:6]9932; GFX7-NEXT: s_mov_b64 s[4:5], 09933; GFX7-NEXT: .LBB133_1: ; %atomicrmw.start9934; GFX7-NEXT: ; =>This Inner Loop Header: Depth=19935; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9936; GFX7-NEXT: v_mov_b32_e32 v7, v59937; GFX7-NEXT: v_mov_b32_e32 v6, v49938; GFX7-NEXT: v_add_i32_e32 v4, vcc, 1, v69939; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc9940; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]9941; GFX7-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc9942; GFX7-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc9943; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc9944; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9945; GFX7-NEXT: buffer_wbinvl1_vol9946; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]9947; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9948; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]9949; GFX7-NEXT: s_cbranch_execnz .LBB133_19950; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end9951; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]9952; GFX7-NEXT: v_mov_b32_e32 v0, v49953; GFX7-NEXT: v_mov_b32_e32 v1, v59954; GFX7-NEXT: s_setpc_b64 s[30:31]9955;9956; GFX8-LABEL: flat_atomic_uinc_wrap_i64_ret:9957; GFX8: ; %bb.0:9958; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9959; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v09960; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc9961; GFX8-NEXT: flat_load_dword v4, v[0:1]9962; GFX8-NEXT: flat_load_dword v5, v[5:6]9963; GFX8-NEXT: s_mov_b64 s[4:5], 09964; GFX8-NEXT: .LBB133_1: ; %atomicrmw.start9965; GFX8-NEXT: ; =>This Inner Loop Header: Depth=19966; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9967; GFX8-NEXT: v_mov_b32_e32 v7, v59968; GFX8-NEXT: v_mov_b32_e32 v6, v49969; GFX8-NEXT: v_add_u32_e32 v4, vcc, 1, v69970; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v7, vcc9971; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]9972; GFX8-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc9973; GFX8-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc9974; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc9975; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9976; GFX8-NEXT: buffer_wbinvl1_vol9977; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]9978; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9979; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]9980; GFX8-NEXT: s_cbranch_execnz .LBB133_19981; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end9982; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]9983; GFX8-NEXT: v_mov_b32_e32 v0, v49984; GFX8-NEXT: v_mov_b32_e32 v1, v59985; GFX8-NEXT: s_setpc_b64 s[30:31]9986;9987; GFX9-LABEL: flat_atomic_uinc_wrap_i64_ret:9988; GFX9: ; %bb.0:9989; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9990; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1]9991; GFX9-NEXT: s_mov_b64 s[4:5], 09992; GFX9-NEXT: .LBB133_1: ; %atomicrmw.start9993; GFX9-NEXT: ; =>This Inner Loop Header: Depth=19994; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9995; GFX9-NEXT: v_mov_b32_e32 v7, v59996; GFX9-NEXT: v_mov_b32_e32 v6, v49997; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, 1, v69998; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v7, vcc9999; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]10000; GFX9-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc10001; GFX9-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc10002; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc10003; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10004; GFX9-NEXT: buffer_wbinvl1_vol10005; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]10006; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]10007; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]10008; GFX9-NEXT: s_cbranch_execnz .LBB133_110009; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end10010; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]10011; GFX9-NEXT: v_mov_b32_e32 v0, v410012; GFX9-NEXT: v_mov_b32_e32 v1, v510013; GFX9-NEXT: s_setpc_b64 s[30:31]10014 %result = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !110015 ret i64 %result10016}10017 10018define i64 @flat_atomic_uinc_wrap_i64_ret_offset(ptr %out, i64 %in) {10019; GFX7-LABEL: flat_atomic_uinc_wrap_i64_ret_offset:10020; GFX7: ; %bb.0:10021; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10022; GFX7-NEXT: v_add_i32_e32 v4, vcc, 32, v010023; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc10024; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v010025; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc10026; GFX7-NEXT: flat_load_dword v1, v[0:1]10027; GFX7-NEXT: flat_load_dword v0, v[4:5]10028; GFX7-NEXT: s_mov_b64 s[4:5], 010029; GFX7-NEXT: .LBB134_1: ; %atomicrmw.start10030; GFX7-NEXT: ; =>This Inner Loop Header: Depth=110031; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10032; GFX7-NEXT: v_mov_b32_e32 v9, v110033; GFX7-NEXT: v_mov_b32_e32 v8, v010034; GFX7-NEXT: v_add_i32_e32 v0, vcc, 1, v810035; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v9, vcc10036; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, v[8:9], v[2:3]10037; GFX7-NEXT: v_cndmask_b32_e32 v7, 0, v1, vcc10038; GFX7-NEXT: v_cndmask_b32_e32 v6, 0, v0, vcc10039; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc10040; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10041; GFX7-NEXT: buffer_wbinvl1_vol10042; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]10043; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]10044; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]10045; GFX7-NEXT: s_cbranch_execnz .LBB134_110046; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end10047; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]10048; GFX7-NEXT: s_setpc_b64 s[30:31]10049;10050; GFX8-LABEL: flat_atomic_uinc_wrap_i64_ret_offset:10051; GFX8: ; %bb.0:10052; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10053; GFX8-NEXT: v_add_u32_e32 v4, vcc, 32, v010054; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc10055; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v010056; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc10057; GFX8-NEXT: flat_load_dword v1, v[0:1]10058; GFX8-NEXT: flat_load_dword v0, v[4:5]10059; GFX8-NEXT: s_mov_b64 s[4:5], 010060; GFX8-NEXT: .LBB134_1: ; %atomicrmw.start10061; GFX8-NEXT: ; =>This Inner Loop Header: Depth=110062; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10063; GFX8-NEXT: v_mov_b32_e32 v9, v110064; GFX8-NEXT: v_mov_b32_e32 v8, v010065; GFX8-NEXT: v_add_u32_e32 v0, vcc, 1, v810066; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v9, vcc10067; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, v[8:9], v[2:3]10068; GFX8-NEXT: v_cndmask_b32_e32 v7, 0, v1, vcc10069; GFX8-NEXT: v_cndmask_b32_e32 v6, 0, v0, vcc10070; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc10071; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10072; GFX8-NEXT: buffer_wbinvl1_vol10073; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]10074; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]10075; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]10076; GFX8-NEXT: s_cbranch_execnz .LBB134_110077; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end10078; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]10079; GFX8-NEXT: s_setpc_b64 s[30:31]10080;10081; GFX9-LABEL: flat_atomic_uinc_wrap_i64_ret_offset:10082; GFX9: ; %bb.0:10083; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10084; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:3210085; GFX9-NEXT: s_mov_b64 s[4:5], 010086; GFX9-NEXT: .LBB134_1: ; %atomicrmw.start10087; GFX9-NEXT: ; =>This Inner Loop Header: Depth=110088; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10089; GFX9-NEXT: v_mov_b32_e32 v7, v510090; GFX9-NEXT: v_mov_b32_e32 v6, v410091; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, 1, v610092; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v7, vcc10093; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]10094; GFX9-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc10095; GFX9-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc10096; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc10097; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10098; GFX9-NEXT: buffer_wbinvl1_vol10099; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]10100; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]10101; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]10102; GFX9-NEXT: s_cbranch_execnz .LBB134_110103; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end10104; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]10105; GFX9-NEXT: v_mov_b32_e32 v0, v410106; GFX9-NEXT: v_mov_b32_e32 v1, v510107; GFX9-NEXT: s_setpc_b64 s[30:31]10108 %gep = getelementptr inbounds i64, ptr %out, i64 410109 %result = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !110110 ret i64 %result10111}10112 10113define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {10114; GFX7-LABEL: flat_atomic_uinc_wrap_i64_noret_scalar:10115; GFX7: ; %bb.0:10116; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10117; GFX7-NEXT: v_mov_b32_e32 v0, s410118; GFX7-NEXT: s_add_u32 s34, s4, 410119; GFX7-NEXT: v_mov_b32_e32 v1, s510120; GFX7-NEXT: s_addc_u32 s35, s5, 010121; GFX7-NEXT: v_mov_b32_e32 v3, s3410122; GFX7-NEXT: v_mov_b32_e32 v4, s3510123; GFX7-NEXT: flat_load_dword v2, v[0:1]10124; GFX7-NEXT: flat_load_dword v3, v[3:4]10125; GFX7-NEXT: v_mov_b32_e32 v4, s410126; GFX7-NEXT: s_mov_b64 s[34:35], 010127; GFX7-NEXT: v_mov_b32_e32 v5, s510128; GFX7-NEXT: .LBB135_1: ; %atomicrmw.start10129; GFX7-NEXT: ; =>This Inner Loop Header: Depth=110130; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10131; GFX7-NEXT: v_add_i32_e32 v0, vcc, 1, v210132; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc10133; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]10134; GFX7-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc10135; GFX7-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc10136; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc10137; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10138; GFX7-NEXT: buffer_wbinvl1_vol10139; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]10140; GFX7-NEXT: v_mov_b32_e32 v3, v110141; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]10142; GFX7-NEXT: v_mov_b32_e32 v2, v010143; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]10144; GFX7-NEXT: s_cbranch_execnz .LBB135_110145; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end10146; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]10147; GFX7-NEXT: s_setpc_b64 s[30:31]10148;10149; GFX8-LABEL: flat_atomic_uinc_wrap_i64_noret_scalar:10150; GFX8: ; %bb.0:10151; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10152; GFX8-NEXT: v_mov_b32_e32 v0, s410153; GFX8-NEXT: s_add_u32 s34, s4, 410154; GFX8-NEXT: v_mov_b32_e32 v1, s510155; GFX8-NEXT: s_addc_u32 s35, s5, 010156; GFX8-NEXT: v_mov_b32_e32 v3, s3410157; GFX8-NEXT: v_mov_b32_e32 v4, s3510158; GFX8-NEXT: flat_load_dword v2, v[0:1]10159; GFX8-NEXT: flat_load_dword v3, v[3:4]10160; GFX8-NEXT: v_mov_b32_e32 v4, s410161; GFX8-NEXT: s_mov_b64 s[34:35], 010162; GFX8-NEXT: v_mov_b32_e32 v5, s510163; GFX8-NEXT: .LBB135_1: ; %atomicrmw.start10164; GFX8-NEXT: ; =>This Inner Loop Header: Depth=110165; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10166; GFX8-NEXT: v_add_u32_e32 v0, vcc, 1, v210167; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc10168; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]10169; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc10170; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc10171; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc10172; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10173; GFX8-NEXT: buffer_wbinvl1_vol10174; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]10175; GFX8-NEXT: v_mov_b32_e32 v3, v110176; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]10177; GFX8-NEXT: v_mov_b32_e32 v2, v010178; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]10179; GFX8-NEXT: s_cbranch_execnz .LBB135_110180; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end10181; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]10182; GFX8-NEXT: s_setpc_b64 s[30:31]10183;10184; GFX9-LABEL: flat_atomic_uinc_wrap_i64_noret_scalar:10185; GFX9: ; %bb.0:10186; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10187; GFX9-NEXT: v_mov_b32_e32 v0, s410188; GFX9-NEXT: v_mov_b32_e32 v1, s510189; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]10190; GFX9-NEXT: v_mov_b32_e32 v4, s410191; GFX9-NEXT: s_mov_b64 s[34:35], 010192; GFX9-NEXT: v_mov_b32_e32 v5, s510193; GFX9-NEXT: .LBB135_1: ; %atomicrmw.start10194; GFX9-NEXT: ; =>This Inner Loop Header: Depth=110195; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10196; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 1, v210197; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc10198; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]10199; GFX9-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc10200; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc10201; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc10202; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10203; GFX9-NEXT: buffer_wbinvl1_vol10204; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]10205; GFX9-NEXT: v_mov_b32_e32 v3, v110206; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]10207; GFX9-NEXT: v_mov_b32_e32 v2, v010208; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]10209; GFX9-NEXT: s_cbranch_execnz .LBB135_110210; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end10211; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]10212; GFX9-NEXT: s_setpc_b64 s[30:31]10213 %tmp0 = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !110214 ret void10215}10216 10217define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {10218; GFX7-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar:10219; GFX7: ; %bb.0:10220; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10221; GFX7-NEXT: s_add_u32 s34, s4, 3210222; GFX7-NEXT: s_addc_u32 s35, s5, 010223; GFX7-NEXT: s_add_u32 s36, s4, 3610224; GFX7-NEXT: s_addc_u32 s37, s5, 010225; GFX7-NEXT: v_mov_b32_e32 v0, s3610226; GFX7-NEXT: v_mov_b32_e32 v1, s3710227; GFX7-NEXT: v_mov_b32_e32 v4, s3410228; GFX7-NEXT: v_mov_b32_e32 v5, s3510229; GFX7-NEXT: flat_load_dword v3, v[0:1]10230; GFX7-NEXT: flat_load_dword v2, v[4:5]10231; GFX7-NEXT: s_mov_b64 s[34:35], 010232; GFX7-NEXT: .LBB136_1: ; %atomicrmw.start10233; GFX7-NEXT: ; =>This Inner Loop Header: Depth=110234; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10235; GFX7-NEXT: v_add_i32_e32 v0, vcc, 1, v210236; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc10237; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]10238; GFX7-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc10239; GFX7-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc10240; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc10241; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10242; GFX7-NEXT: buffer_wbinvl1_vol10243; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]10244; GFX7-NEXT: v_mov_b32_e32 v3, v110245; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]10246; GFX7-NEXT: v_mov_b32_e32 v2, v010247; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]10248; GFX7-NEXT: s_cbranch_execnz .LBB136_110249; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end10250; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]10251; GFX7-NEXT: s_setpc_b64 s[30:31]10252;10253; GFX8-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar:10254; GFX8: ; %bb.0:10255; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10256; GFX8-NEXT: s_add_u32 s34, s4, 3210257; GFX8-NEXT: s_addc_u32 s35, s5, 010258; GFX8-NEXT: s_add_u32 s36, s4, 3610259; GFX8-NEXT: s_addc_u32 s37, s5, 010260; GFX8-NEXT: v_mov_b32_e32 v0, s3610261; GFX8-NEXT: v_mov_b32_e32 v1, s3710262; GFX8-NEXT: v_mov_b32_e32 v4, s3410263; GFX8-NEXT: v_mov_b32_e32 v5, s3510264; GFX8-NEXT: flat_load_dword v3, v[0:1]10265; GFX8-NEXT: flat_load_dword v2, v[4:5]10266; GFX8-NEXT: s_mov_b64 s[34:35], 010267; GFX8-NEXT: .LBB136_1: ; %atomicrmw.start10268; GFX8-NEXT: ; =>This Inner Loop Header: Depth=110269; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10270; GFX8-NEXT: v_add_u32_e32 v0, vcc, 1, v210271; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc10272; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]10273; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc10274; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc10275; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc10276; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10277; GFX8-NEXT: buffer_wbinvl1_vol10278; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]10279; GFX8-NEXT: v_mov_b32_e32 v3, v110280; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]10281; GFX8-NEXT: v_mov_b32_e32 v2, v010282; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]10283; GFX8-NEXT: s_cbranch_execnz .LBB136_110284; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end10285; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]10286; GFX8-NEXT: s_setpc_b64 s[30:31]10287;10288; GFX9-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar:10289; GFX9: ; %bb.0:10290; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10291; GFX9-NEXT: v_mov_b32_e32 v0, s410292; GFX9-NEXT: v_mov_b32_e32 v1, s510293; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:3210294; GFX9-NEXT: v_mov_b32_e32 v4, s410295; GFX9-NEXT: s_mov_b64 s[34:35], 010296; GFX9-NEXT: v_mov_b32_e32 v5, s510297; GFX9-NEXT: .LBB136_1: ; %atomicrmw.start10298; GFX9-NEXT: ; =>This Inner Loop Header: Depth=110299; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10300; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 1, v210301; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc10302; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]10303; GFX9-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc10304; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc10305; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc10306; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10307; GFX9-NEXT: buffer_wbinvl1_vol10308; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]10309; GFX9-NEXT: v_mov_b32_e32 v3, v110310; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]10311; GFX9-NEXT: v_mov_b32_e32 v2, v010312; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]10313; GFX9-NEXT: s_cbranch_execnz .LBB136_110314; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end10315; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]10316; GFX9-NEXT: s_setpc_b64 s[30:31]10317 %gep = getelementptr inbounds i64, ptr %out, i64 410318 %tmp0 = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !110319 ret void10320}10321 10322define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {10323; GFX7-LABEL: flat_atomic_uinc_wrap_i64_ret_scalar:10324; GFX7: ; %bb.0:10325; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10326; GFX7-NEXT: v_mov_b32_e32 v0, s410327; GFX7-NEXT: s_add_u32 s34, s4, 410328; GFX7-NEXT: v_mov_b32_e32 v1, s510329; GFX7-NEXT: s_addc_u32 s35, s5, 010330; GFX7-NEXT: v_mov_b32_e32 v2, s3410331; GFX7-NEXT: v_mov_b32_e32 v3, s3510332; GFX7-NEXT: flat_load_dword v0, v[0:1]10333; GFX7-NEXT: flat_load_dword v1, v[2:3]10334; GFX7-NEXT: v_mov_b32_e32 v2, s410335; GFX7-NEXT: s_mov_b64 s[34:35], 010336; GFX7-NEXT: v_mov_b32_e32 v3, s510337; GFX7-NEXT: .LBB137_1: ; %atomicrmw.start10338; GFX7-NEXT: ; =>This Inner Loop Header: Depth=110339; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10340; GFX7-NEXT: v_mov_b32_e32 v7, v110341; GFX7-NEXT: v_mov_b32_e32 v6, v010342; GFX7-NEXT: v_add_i32_e32 v0, vcc, 1, v610343; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc10344; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]10345; GFX7-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc10346; GFX7-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc10347; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc10348; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10349; GFX7-NEXT: buffer_wbinvl1_vol10350; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]10351; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]10352; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]10353; GFX7-NEXT: s_cbranch_execnz .LBB137_110354; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end10355; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]10356; GFX7-NEXT: s_setpc_b64 s[30:31]10357;10358; GFX8-LABEL: flat_atomic_uinc_wrap_i64_ret_scalar:10359; GFX8: ; %bb.0:10360; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10361; GFX8-NEXT: v_mov_b32_e32 v0, s410362; GFX8-NEXT: s_add_u32 s34, s4, 410363; GFX8-NEXT: v_mov_b32_e32 v1, s510364; GFX8-NEXT: s_addc_u32 s35, s5, 010365; GFX8-NEXT: v_mov_b32_e32 v2, s3410366; GFX8-NEXT: v_mov_b32_e32 v3, s3510367; GFX8-NEXT: flat_load_dword v0, v[0:1]10368; GFX8-NEXT: flat_load_dword v1, v[2:3]10369; GFX8-NEXT: v_mov_b32_e32 v2, s410370; GFX8-NEXT: s_mov_b64 s[34:35], 010371; GFX8-NEXT: v_mov_b32_e32 v3, s510372; GFX8-NEXT: .LBB137_1: ; %atomicrmw.start10373; GFX8-NEXT: ; =>This Inner Loop Header: Depth=110374; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10375; GFX8-NEXT: v_mov_b32_e32 v7, v110376; GFX8-NEXT: v_mov_b32_e32 v6, v010377; GFX8-NEXT: v_add_u32_e32 v0, vcc, 1, v610378; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc10379; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]10380; GFX8-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc10381; GFX8-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc10382; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc10383; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10384; GFX8-NEXT: buffer_wbinvl1_vol10385; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]10386; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]10387; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]10388; GFX8-NEXT: s_cbranch_execnz .LBB137_110389; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end10390; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]10391; GFX8-NEXT: s_setpc_b64 s[30:31]10392;10393; GFX9-LABEL: flat_atomic_uinc_wrap_i64_ret_scalar:10394; GFX9: ; %bb.0:10395; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10396; GFX9-NEXT: v_mov_b32_e32 v0, s410397; GFX9-NEXT: v_mov_b32_e32 v1, s510398; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]10399; GFX9-NEXT: v_mov_b32_e32 v2, s410400; GFX9-NEXT: s_mov_b64 s[34:35], 010401; GFX9-NEXT: v_mov_b32_e32 v3, s510402; GFX9-NEXT: .LBB137_1: ; %atomicrmw.start10403; GFX9-NEXT: ; =>This Inner Loop Header: Depth=110404; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10405; GFX9-NEXT: v_mov_b32_e32 v7, v110406; GFX9-NEXT: v_mov_b32_e32 v6, v010407; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 1, v610408; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v7, vcc10409; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]10410; GFX9-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc10411; GFX9-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc10412; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc10413; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10414; GFX9-NEXT: buffer_wbinvl1_vol10415; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]10416; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]10417; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]10418; GFX9-NEXT: s_cbranch_execnz .LBB137_110419; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end10420; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]10421; GFX9-NEXT: s_setpc_b64 s[30:31]10422 %result = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !110423 ret i64 %result10424}10425 10426define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {10427; GFX7-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar:10428; GFX7: ; %bb.0:10429; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10430; GFX7-NEXT: s_add_u32 s34, s4, 3210431; GFX7-NEXT: s_addc_u32 s35, s5, 010432; GFX7-NEXT: s_add_u32 s36, s4, 3610433; GFX7-NEXT: s_addc_u32 s37, s5, 010434; GFX7-NEXT: v_mov_b32_e32 v0, s3610435; GFX7-NEXT: v_mov_b32_e32 v1, s3710436; GFX7-NEXT: v_mov_b32_e32 v2, s3410437; GFX7-NEXT: v_mov_b32_e32 v3, s3510438; GFX7-NEXT: flat_load_dword v1, v[0:1]10439; GFX7-NEXT: flat_load_dword v0, v[2:3]10440; GFX7-NEXT: s_mov_b64 s[34:35], 010441; GFX7-NEXT: .LBB138_1: ; %atomicrmw.start10442; GFX7-NEXT: ; =>This Inner Loop Header: Depth=110443; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10444; GFX7-NEXT: v_mov_b32_e32 v7, v110445; GFX7-NEXT: v_mov_b32_e32 v6, v010446; GFX7-NEXT: v_add_i32_e32 v0, vcc, 1, v610447; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc10448; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]10449; GFX7-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc10450; GFX7-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc10451; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc10452; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10453; GFX7-NEXT: buffer_wbinvl1_vol10454; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]10455; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]10456; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]10457; GFX7-NEXT: s_cbranch_execnz .LBB138_110458; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end10459; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]10460; GFX7-NEXT: s_setpc_b64 s[30:31]10461;10462; GFX8-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar:10463; GFX8: ; %bb.0:10464; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10465; GFX8-NEXT: s_add_u32 s34, s4, 3210466; GFX8-NEXT: s_addc_u32 s35, s5, 010467; GFX8-NEXT: s_add_u32 s36, s4, 3610468; GFX8-NEXT: s_addc_u32 s37, s5, 010469; GFX8-NEXT: v_mov_b32_e32 v0, s3610470; GFX8-NEXT: v_mov_b32_e32 v1, s3710471; GFX8-NEXT: v_mov_b32_e32 v2, s3410472; GFX8-NEXT: v_mov_b32_e32 v3, s3510473; GFX8-NEXT: flat_load_dword v1, v[0:1]10474; GFX8-NEXT: flat_load_dword v0, v[2:3]10475; GFX8-NEXT: s_mov_b64 s[34:35], 010476; GFX8-NEXT: .LBB138_1: ; %atomicrmw.start10477; GFX8-NEXT: ; =>This Inner Loop Header: Depth=110478; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10479; GFX8-NEXT: v_mov_b32_e32 v7, v110480; GFX8-NEXT: v_mov_b32_e32 v6, v010481; GFX8-NEXT: v_add_u32_e32 v0, vcc, 1, v610482; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc10483; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]10484; GFX8-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc10485; GFX8-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc10486; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc10487; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10488; GFX8-NEXT: buffer_wbinvl1_vol10489; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]10490; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]10491; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]10492; GFX8-NEXT: s_cbranch_execnz .LBB138_110493; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end10494; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]10495; GFX8-NEXT: s_setpc_b64 s[30:31]10496;10497; GFX9-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar:10498; GFX9: ; %bb.0:10499; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10500; GFX9-NEXT: v_mov_b32_e32 v0, s410501; GFX9-NEXT: v_mov_b32_e32 v1, s510502; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:3210503; GFX9-NEXT: v_mov_b32_e32 v2, s410504; GFX9-NEXT: s_mov_b64 s[34:35], 010505; GFX9-NEXT: v_mov_b32_e32 v3, s510506; GFX9-NEXT: .LBB138_1: ; %atomicrmw.start10507; GFX9-NEXT: ; =>This Inner Loop Header: Depth=110508; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10509; GFX9-NEXT: v_mov_b32_e32 v7, v110510; GFX9-NEXT: v_mov_b32_e32 v6, v010511; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 1, v610512; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v7, vcc10513; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]10514; GFX9-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc10515; GFX9-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc10516; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] offset:32 glc10517; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10518; GFX9-NEXT: buffer_wbinvl1_vol10519; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]10520; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]10521; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]10522; GFX9-NEXT: s_cbranch_execnz .LBB138_110523; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end10524; GFX9-NEXT: s_or_b64 exec, exec, s[34:35]10525; GFX9-NEXT: s_setpc_b64 s[30:31]10526 %gep = getelementptr inbounds i64, ptr %out, i64 410527 %result = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !110528 ret i64 %result10529}10530 10531define void @flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {10532; GFX7-LABEL: flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory:10533; GFX7: ; %bb.0:10534; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10535; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v010536; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc10537; GFX7-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]10538; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10539; GFX7-NEXT: buffer_wbinvl1_vol10540; GFX7-NEXT: s_setpc_b64 s[30:31]10541;10542; GFX8-LABEL: flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory:10543; GFX8: ; %bb.0:10544; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10545; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v010546; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc10547; GFX8-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]10548; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10549; GFX8-NEXT: buffer_wbinvl1_vol10550; GFX8-NEXT: s_setpc_b64 s[30:31]10551;10552; GFX9-LABEL: flat_atomic_uinc_wrap_i64_noret_offset__amdgpu_no_remote_memory:10553; GFX9: ; %bb.0:10554; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10555; GFX9-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3] offset:3210556; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10557; GFX9-NEXT: buffer_wbinvl1_vol10558; GFX9-NEXT: s_setpc_b64 s[30:31]10559 %gep = getelementptr inbounds i64, ptr %out, i64 410560 %tmp0 = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !110561 ret void10562}10563 10564define i64 @flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {10565; GFX7-LABEL: flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory:10566; GFX7: ; %bb.0:10567; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10568; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v010569; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc10570; GFX7-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc10571; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10572; GFX7-NEXT: buffer_wbinvl1_vol10573; GFX7-NEXT: s_setpc_b64 s[30:31]10574;10575; GFX8-LABEL: flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory:10576; GFX8: ; %bb.0:10577; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10578; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v010579; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc10580; GFX8-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc10581; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10582; GFX8-NEXT: buffer_wbinvl1_vol10583; GFX8-NEXT: s_setpc_b64 s[30:31]10584;10585; GFX9-LABEL: flat_atomic_uinc_wrap_i64_ret_offset__amdgpu_no_remote_memory:10586; GFX9: ; %bb.0:10587; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10588; GFX9-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] offset:32 glc10589; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10590; GFX9-NEXT: buffer_wbinvl1_vol10591; GFX9-NEXT: s_setpc_b64 s[30:31]10592 %gep = getelementptr inbounds i64, ptr %out, i64 410593 %result = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !110594 ret i64 %result10595}10596 10597; ---------------------------------------------------------------------10598; atomicrmw udec_wrap10599; ---------------------------------------------------------------------10600 10601define void @flat_atomic_udec_wrap_i64_noret(ptr %ptr, i64 %in) {10602; GFX7-LABEL: flat_atomic_udec_wrap_i64_noret:10603; GFX7: ; %bb.0:10604; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10605; GFX7-NEXT: v_add_i32_e32 v4, vcc, 4, v010606; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc10607; GFX7-NEXT: flat_load_dword v6, v[0:1]10608; GFX7-NEXT: flat_load_dword v7, v[4:5]10609; GFX7-NEXT: s_mov_b64 s[8:9], 010610; GFX7-NEXT: .LBB141_1: ; %atomicrmw.start10611; GFX7-NEXT: ; =>This Inner Loop Header: Depth=110612; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10613; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]10614; GFX7-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]10615; GFX7-NEXT: v_add_i32_e64 v4, s[6:7], -1, v610616; GFX7-NEXT: v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]10617; GFX7-NEXT: s_or_b64 vcc, vcc, s[4:5]10618; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc10619; GFX7-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc10620; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc10621; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10622; GFX7-NEXT: buffer_wbinvl1_vol10623; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]10624; GFX7-NEXT: v_mov_b32_e32 v7, v510625; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]10626; GFX7-NEXT: v_mov_b32_e32 v6, v410627; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]10628; GFX7-NEXT: s_cbranch_execnz .LBB141_110629; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end10630; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]10631; GFX7-NEXT: s_setpc_b64 s[30:31]10632;10633; GFX8-LABEL: flat_atomic_udec_wrap_i64_noret:10634; GFX8: ; %bb.0:10635; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10636; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v010637; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc10638; GFX8-NEXT: flat_load_dword v6, v[0:1]10639; GFX8-NEXT: flat_load_dword v7, v[4:5]10640; GFX8-NEXT: s_mov_b64 s[8:9], 010641; GFX8-NEXT: .LBB141_1: ; %atomicrmw.start10642; GFX8-NEXT: ; =>This Inner Loop Header: Depth=110643; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10644; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]10645; GFX8-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]10646; GFX8-NEXT: v_add_u32_e64 v4, s[6:7], -1, v610647; GFX8-NEXT: v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]10648; GFX8-NEXT: s_or_b64 vcc, vcc, s[4:5]10649; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc10650; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc10651; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc10652; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10653; GFX8-NEXT: buffer_wbinvl1_vol10654; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]10655; GFX8-NEXT: v_mov_b32_e32 v7, v510656; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]10657; GFX8-NEXT: v_mov_b32_e32 v6, v410658; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]10659; GFX8-NEXT: s_cbranch_execnz .LBB141_110660; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end10661; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]10662; GFX8-NEXT: s_setpc_b64 s[30:31]10663;10664; GFX9-LABEL: flat_atomic_udec_wrap_i64_noret:10665; GFX9: ; %bb.0:10666; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10667; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1]10668; GFX9-NEXT: s_mov_b64 s[8:9], 010669; GFX9-NEXT: .LBB141_1: ; %atomicrmw.start10670; GFX9-NEXT: ; =>This Inner Loop Header: Depth=110671; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10672; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]10673; GFX9-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]10674; GFX9-NEXT: v_add_co_u32_e64 v4, s[6:7], -1, v610675; GFX9-NEXT: v_addc_co_u32_e64 v5, s[6:7], -1, v7, s[6:7]10676; GFX9-NEXT: s_or_b64 vcc, vcc, s[4:5]10677; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc10678; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc10679; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc10680; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10681; GFX9-NEXT: buffer_wbinvl1_vol10682; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]10683; GFX9-NEXT: v_mov_b32_e32 v7, v510684; GFX9-NEXT: s_or_b64 s[8:9], vcc, s[8:9]10685; GFX9-NEXT: v_mov_b32_e32 v6, v410686; GFX9-NEXT: s_andn2_b64 exec, exec, s[8:9]10687; GFX9-NEXT: s_cbranch_execnz .LBB141_110688; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end10689; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]10690; GFX9-NEXT: s_setpc_b64 s[30:31]10691 %tmp0 = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !110692 ret void10693}10694 10695define void @flat_atomic_udec_wrap_i64_noret_offset(ptr %out, i64 %in) {10696; GFX7-LABEL: flat_atomic_udec_wrap_i64_noret_offset:10697; GFX7: ; %bb.0:10698; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10699; GFX7-NEXT: v_add_i32_e32 v8, vcc, 32, v010700; GFX7-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc10701; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v010702; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc10703; GFX7-NEXT: flat_load_dword v7, v[0:1]10704; GFX7-NEXT: flat_load_dword v6, v[8:9]10705; GFX7-NEXT: s_mov_b64 s[8:9], 010706; GFX7-NEXT: .LBB142_1: ; %atomicrmw.start10707; GFX7-NEXT: ; =>This Inner Loop Header: Depth=110708; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10709; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]10710; GFX7-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]10711; GFX7-NEXT: v_add_i32_e64 v0, s[6:7], -1, v610712; GFX7-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v7, s[6:7]10713; GFX7-NEXT: s_or_b64 vcc, vcc, s[4:5]10714; GFX7-NEXT: v_cndmask_b32_e32 v5, v1, v3, vcc10715; GFX7-NEXT: v_cndmask_b32_e32 v4, v0, v2, vcc10716; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc10717; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10718; GFX7-NEXT: buffer_wbinvl1_vol10719; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]10720; GFX7-NEXT: v_mov_b32_e32 v7, v110721; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]10722; GFX7-NEXT: v_mov_b32_e32 v6, v010723; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]10724; GFX7-NEXT: s_cbranch_execnz .LBB142_110725; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end10726; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]10727; GFX7-NEXT: s_setpc_b64 s[30:31]10728;10729; GFX8-LABEL: flat_atomic_udec_wrap_i64_noret_offset:10730; GFX8: ; %bb.0:10731; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10732; GFX8-NEXT: v_add_u32_e32 v8, vcc, 32, v010733; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v1, vcc10734; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v010735; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc10736; GFX8-NEXT: flat_load_dword v7, v[0:1]10737; GFX8-NEXT: flat_load_dword v6, v[8:9]10738; GFX8-NEXT: s_mov_b64 s[8:9], 010739; GFX8-NEXT: .LBB142_1: ; %atomicrmw.start10740; GFX8-NEXT: ; =>This Inner Loop Header: Depth=110741; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10742; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]10743; GFX8-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]10744; GFX8-NEXT: v_add_u32_e64 v0, s[6:7], -1, v610745; GFX8-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v7, s[6:7]10746; GFX8-NEXT: s_or_b64 vcc, vcc, s[4:5]10747; GFX8-NEXT: v_cndmask_b32_e32 v5, v1, v3, vcc10748; GFX8-NEXT: v_cndmask_b32_e32 v4, v0, v2, vcc10749; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc10750; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10751; GFX8-NEXT: buffer_wbinvl1_vol10752; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]10753; GFX8-NEXT: v_mov_b32_e32 v7, v110754; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]10755; GFX8-NEXT: v_mov_b32_e32 v6, v010756; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]10757; GFX8-NEXT: s_cbranch_execnz .LBB142_110758; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end10759; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]10760; GFX8-NEXT: s_setpc_b64 s[30:31]10761;10762; GFX9-LABEL: flat_atomic_udec_wrap_i64_noret_offset:10763; GFX9: ; %bb.0:10764; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10765; GFX9-NEXT: flat_load_dwordx2 v[6:7], v[0:1] offset:3210766; GFX9-NEXT: s_mov_b64 s[8:9], 010767; GFX9-NEXT: .LBB142_1: ; %atomicrmw.start10768; GFX9-NEXT: ; =>This Inner Loop Header: Depth=110769; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10770; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]10771; GFX9-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]10772; GFX9-NEXT: v_add_co_u32_e64 v4, s[6:7], -1, v610773; GFX9-NEXT: v_addc_co_u32_e64 v5, s[6:7], -1, v7, s[6:7]10774; GFX9-NEXT: s_or_b64 vcc, vcc, s[4:5]10775; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc10776; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc10777; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc10778; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10779; GFX9-NEXT: buffer_wbinvl1_vol10780; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]10781; GFX9-NEXT: v_mov_b32_e32 v7, v510782; GFX9-NEXT: s_or_b64 s[8:9], vcc, s[8:9]10783; GFX9-NEXT: v_mov_b32_e32 v6, v410784; GFX9-NEXT: s_andn2_b64 exec, exec, s[8:9]10785; GFX9-NEXT: s_cbranch_execnz .LBB142_110786; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end10787; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]10788; GFX9-NEXT: s_setpc_b64 s[30:31]10789 %gep = getelementptr inbounds i64, ptr %out, i64 410790 %tmp0 = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !110791 ret void10792}10793 10794define i64 @flat_atomic_udec_wrap_i64_ret(ptr %ptr, i64 %in) {10795; GFX7-LABEL: flat_atomic_udec_wrap_i64_ret:10796; GFX7: ; %bb.0:10797; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10798; GFX7-NEXT: v_add_i32_e32 v5, vcc, 4, v010799; GFX7-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc10800; GFX7-NEXT: flat_load_dword v4, v[0:1]10801; GFX7-NEXT: flat_load_dword v5, v[5:6]10802; GFX7-NEXT: s_mov_b64 s[8:9], 010803; GFX7-NEXT: .LBB143_1: ; %atomicrmw.start10804; GFX7-NEXT: ; =>This Inner Loop Header: Depth=110805; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10806; GFX7-NEXT: v_mov_b32_e32 v7, v510807; GFX7-NEXT: v_mov_b32_e32 v6, v410808; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]10809; GFX7-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]10810; GFX7-NEXT: v_add_i32_e64 v4, s[6:7], -1, v610811; GFX7-NEXT: v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]10812; GFX7-NEXT: s_or_b64 vcc, vcc, s[4:5]10813; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc10814; GFX7-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc10815; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc10816; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10817; GFX7-NEXT: buffer_wbinvl1_vol10818; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]10819; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]10820; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]10821; GFX7-NEXT: s_cbranch_execnz .LBB143_110822; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end10823; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]10824; GFX7-NEXT: v_mov_b32_e32 v0, v410825; GFX7-NEXT: v_mov_b32_e32 v1, v510826; GFX7-NEXT: s_setpc_b64 s[30:31]10827;10828; GFX8-LABEL: flat_atomic_udec_wrap_i64_ret:10829; GFX8: ; %bb.0:10830; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10831; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v010832; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc10833; GFX8-NEXT: flat_load_dword v4, v[0:1]10834; GFX8-NEXT: flat_load_dword v5, v[5:6]10835; GFX8-NEXT: s_mov_b64 s[8:9], 010836; GFX8-NEXT: .LBB143_1: ; %atomicrmw.start10837; GFX8-NEXT: ; =>This Inner Loop Header: Depth=110838; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10839; GFX8-NEXT: v_mov_b32_e32 v7, v510840; GFX8-NEXT: v_mov_b32_e32 v6, v410841; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]10842; GFX8-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]10843; GFX8-NEXT: v_add_u32_e64 v4, s[6:7], -1, v610844; GFX8-NEXT: v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]10845; GFX8-NEXT: s_or_b64 vcc, vcc, s[4:5]10846; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc10847; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc10848; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc10849; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10850; GFX8-NEXT: buffer_wbinvl1_vol10851; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]10852; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]10853; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]10854; GFX8-NEXT: s_cbranch_execnz .LBB143_110855; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end10856; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]10857; GFX8-NEXT: v_mov_b32_e32 v0, v410858; GFX8-NEXT: v_mov_b32_e32 v1, v510859; GFX8-NEXT: s_setpc_b64 s[30:31]10860;10861; GFX9-LABEL: flat_atomic_udec_wrap_i64_ret:10862; GFX9: ; %bb.0:10863; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10864; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1]10865; GFX9-NEXT: s_mov_b64 s[8:9], 010866; GFX9-NEXT: .LBB143_1: ; %atomicrmw.start10867; GFX9-NEXT: ; =>This Inner Loop Header: Depth=110868; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10869; GFX9-NEXT: v_mov_b32_e32 v7, v510870; GFX9-NEXT: v_mov_b32_e32 v6, v410871; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]10872; GFX9-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]10873; GFX9-NEXT: v_add_co_u32_e64 v4, s[6:7], -1, v610874; GFX9-NEXT: v_addc_co_u32_e64 v5, s[6:7], -1, v7, s[6:7]10875; GFX9-NEXT: s_or_b64 vcc, vcc, s[4:5]10876; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc10877; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc10878; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc10879; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10880; GFX9-NEXT: buffer_wbinvl1_vol10881; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]10882; GFX9-NEXT: s_or_b64 s[8:9], vcc, s[8:9]10883; GFX9-NEXT: s_andn2_b64 exec, exec, s[8:9]10884; GFX9-NEXT: s_cbranch_execnz .LBB143_110885; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end10886; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]10887; GFX9-NEXT: v_mov_b32_e32 v0, v410888; GFX9-NEXT: v_mov_b32_e32 v1, v510889; GFX9-NEXT: s_setpc_b64 s[30:31]10890 %result = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !110891 ret i64 %result10892}10893 10894define i64 @flat_atomic_udec_wrap_i64_ret_offset(ptr %out, i64 %in) {10895; GFX7-LABEL: flat_atomic_udec_wrap_i64_ret_offset:10896; GFX7: ; %bb.0:10897; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10898; GFX7-NEXT: v_add_i32_e32 v4, vcc, 32, v010899; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc10900; GFX7-NEXT: v_add_i32_e32 v0, vcc, 36, v010901; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc10902; GFX7-NEXT: flat_load_dword v1, v[0:1]10903; GFX7-NEXT: flat_load_dword v0, v[4:5]10904; GFX7-NEXT: s_mov_b64 s[8:9], 010905; GFX7-NEXT: .LBB144_1: ; %atomicrmw.start10906; GFX7-NEXT: ; =>This Inner Loop Header: Depth=110907; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10908; GFX7-NEXT: v_mov_b32_e32 v9, v110909; GFX7-NEXT: v_mov_b32_e32 v8, v010910; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]10911; GFX7-NEXT: v_cmp_gt_u64_e64 s[4:5], v[8:9], v[2:3]10912; GFX7-NEXT: v_add_i32_e64 v0, s[6:7], -1, v810913; GFX7-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v9, s[6:7]10914; GFX7-NEXT: s_or_b64 vcc, vcc, s[4:5]10915; GFX7-NEXT: v_cndmask_b32_e32 v7, v1, v3, vcc10916; GFX7-NEXT: v_cndmask_b32_e32 v6, v0, v2, vcc10917; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc10918; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10919; GFX7-NEXT: buffer_wbinvl1_vol10920; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]10921; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]10922; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]10923; GFX7-NEXT: s_cbranch_execnz .LBB144_110924; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end10925; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]10926; GFX7-NEXT: s_setpc_b64 s[30:31]10927;10928; GFX8-LABEL: flat_atomic_udec_wrap_i64_ret_offset:10929; GFX8: ; %bb.0:10930; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10931; GFX8-NEXT: v_add_u32_e32 v4, vcc, 32, v010932; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc10933; GFX8-NEXT: v_add_u32_e32 v0, vcc, 36, v010934; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc10935; GFX8-NEXT: flat_load_dword v1, v[0:1]10936; GFX8-NEXT: flat_load_dword v0, v[4:5]10937; GFX8-NEXT: s_mov_b64 s[8:9], 010938; GFX8-NEXT: .LBB144_1: ; %atomicrmw.start10939; GFX8-NEXT: ; =>This Inner Loop Header: Depth=110940; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10941; GFX8-NEXT: v_mov_b32_e32 v9, v110942; GFX8-NEXT: v_mov_b32_e32 v8, v010943; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]10944; GFX8-NEXT: v_cmp_gt_u64_e64 s[4:5], v[8:9], v[2:3]10945; GFX8-NEXT: v_add_u32_e64 v0, s[6:7], -1, v810946; GFX8-NEXT: v_addc_u32_e64 v1, s[6:7], -1, v9, s[6:7]10947; GFX8-NEXT: s_or_b64 vcc, vcc, s[4:5]10948; GFX8-NEXT: v_cndmask_b32_e32 v7, v1, v3, vcc10949; GFX8-NEXT: v_cndmask_b32_e32 v6, v0, v2, vcc10950; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc10951; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10952; GFX8-NEXT: buffer_wbinvl1_vol10953; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]10954; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]10955; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]10956; GFX8-NEXT: s_cbranch_execnz .LBB144_110957; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end10958; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]10959; GFX8-NEXT: s_setpc_b64 s[30:31]10960;10961; GFX9-LABEL: flat_atomic_udec_wrap_i64_ret_offset:10962; GFX9: ; %bb.0:10963; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10964; GFX9-NEXT: flat_load_dwordx2 v[4:5], v[0:1] offset:3210965; GFX9-NEXT: s_mov_b64 s[8:9], 010966; GFX9-NEXT: .LBB144_1: ; %atomicrmw.start10967; GFX9-NEXT: ; =>This Inner Loop Header: Depth=110968; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10969; GFX9-NEXT: v_mov_b32_e32 v7, v510970; GFX9-NEXT: v_mov_b32_e32 v6, v410971; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]10972; GFX9-NEXT: v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]10973; GFX9-NEXT: v_add_co_u32_e64 v4, s[6:7], -1, v610974; GFX9-NEXT: v_addc_co_u32_e64 v5, s[6:7], -1, v7, s[6:7]10975; GFX9-NEXT: s_or_b64 vcc, vcc, s[4:5]10976; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v3, vcc10977; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc10978; GFX9-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc10979; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10980; GFX9-NEXT: buffer_wbinvl1_vol10981; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]10982; GFX9-NEXT: s_or_b64 s[8:9], vcc, s[8:9]10983; GFX9-NEXT: s_andn2_b64 exec, exec, s[8:9]10984; GFX9-NEXT: s_cbranch_execnz .LBB144_110985; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end10986; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]10987; GFX9-NEXT: v_mov_b32_e32 v0, v410988; GFX9-NEXT: v_mov_b32_e32 v1, v510989; GFX9-NEXT: s_setpc_b64 s[30:31]10990 %gep = getelementptr inbounds i64, ptr %out, i64 410991 %result = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !110992 ret i64 %result10993}10994 10995define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_scalar(ptr inreg %ptr, i64 inreg %in) {10996; GFX7-LABEL: flat_atomic_udec_wrap_i64_noret_scalar:10997; GFX7: ; %bb.0:10998; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10999; GFX7-NEXT: v_mov_b32_e32 v0, s411000; GFX7-NEXT: s_add_u32 s34, s4, 411001; GFX7-NEXT: v_mov_b32_e32 v1, s511002; GFX7-NEXT: s_addc_u32 s35, s5, 011003; GFX7-NEXT: v_mov_b32_e32 v3, s3411004; GFX7-NEXT: v_mov_b32_e32 v4, s3511005; GFX7-NEXT: flat_load_dword v2, v[0:1]11006; GFX7-NEXT: flat_load_dword v3, v[3:4]11007; GFX7-NEXT: v_mov_b32_e32 v4, s411008; GFX7-NEXT: s_mov_b64 s[38:39], 011009; GFX7-NEXT: v_mov_b32_e32 v6, s711010; GFX7-NEXT: v_mov_b32_e32 v7, s611011; GFX7-NEXT: v_mov_b32_e32 v5, s511012; GFX7-NEXT: .LBB145_1: ; %atomicrmw.start11013; GFX7-NEXT: ; =>This Inner Loop Header: Depth=111014; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11015; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]11016; GFX7-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]11017; GFX7-NEXT: v_add_i32_e64 v0, s[36:37], -1, v211018; GFX7-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v3, s[36:37]11019; GFX7-NEXT: s_or_b64 vcc, vcc, s[34:35]11020; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc11021; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc11022; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc11023; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11024; GFX7-NEXT: buffer_wbinvl1_vol11025; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]11026; GFX7-NEXT: v_mov_b32_e32 v3, v111027; GFX7-NEXT: s_or_b64 s[38:39], vcc, s[38:39]11028; GFX7-NEXT: v_mov_b32_e32 v2, v011029; GFX7-NEXT: s_andn2_b64 exec, exec, s[38:39]11030; GFX7-NEXT: s_cbranch_execnz .LBB145_111031; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end11032; GFX7-NEXT: s_or_b64 exec, exec, s[38:39]11033; GFX7-NEXT: s_setpc_b64 s[30:31]11034;11035; GFX8-LABEL: flat_atomic_udec_wrap_i64_noret_scalar:11036; GFX8: ; %bb.0:11037; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11038; GFX8-NEXT: v_mov_b32_e32 v0, s411039; GFX8-NEXT: s_add_u32 s34, s4, 411040; GFX8-NEXT: v_mov_b32_e32 v1, s511041; GFX8-NEXT: s_addc_u32 s35, s5, 011042; GFX8-NEXT: v_mov_b32_e32 v3, s3411043; GFX8-NEXT: v_mov_b32_e32 v4, s3511044; GFX8-NEXT: flat_load_dword v2, v[0:1]11045; GFX8-NEXT: flat_load_dword v3, v[3:4]11046; GFX8-NEXT: v_mov_b32_e32 v4, s411047; GFX8-NEXT: s_mov_b64 s[38:39], 011048; GFX8-NEXT: v_mov_b32_e32 v6, s711049; GFX8-NEXT: v_mov_b32_e32 v7, s611050; GFX8-NEXT: v_mov_b32_e32 v5, s511051; GFX8-NEXT: .LBB145_1: ; %atomicrmw.start11052; GFX8-NEXT: ; =>This Inner Loop Header: Depth=111053; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11054; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]11055; GFX8-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]11056; GFX8-NEXT: v_add_u32_e64 v0, s[36:37], -1, v211057; GFX8-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v3, s[36:37]11058; GFX8-NEXT: s_or_b64 vcc, vcc, s[34:35]11059; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc11060; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc11061; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc11062; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11063; GFX8-NEXT: buffer_wbinvl1_vol11064; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]11065; GFX8-NEXT: v_mov_b32_e32 v3, v111066; GFX8-NEXT: s_or_b64 s[38:39], vcc, s[38:39]11067; GFX8-NEXT: v_mov_b32_e32 v2, v011068; GFX8-NEXT: s_andn2_b64 exec, exec, s[38:39]11069; GFX8-NEXT: s_cbranch_execnz .LBB145_111070; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end11071; GFX8-NEXT: s_or_b64 exec, exec, s[38:39]11072; GFX8-NEXT: s_setpc_b64 s[30:31]11073;11074; GFX9-LABEL: flat_atomic_udec_wrap_i64_noret_scalar:11075; GFX9: ; %bb.0:11076; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11077; GFX9-NEXT: v_mov_b32_e32 v0, s411078; GFX9-NEXT: v_mov_b32_e32 v1, s511079; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]11080; GFX9-NEXT: v_mov_b32_e32 v4, s411081; GFX9-NEXT: s_mov_b64 s[38:39], 011082; GFX9-NEXT: v_mov_b32_e32 v6, s711083; GFX9-NEXT: v_mov_b32_e32 v7, s611084; GFX9-NEXT: v_mov_b32_e32 v5, s511085; GFX9-NEXT: .LBB145_1: ; %atomicrmw.start11086; GFX9-NEXT: ; =>This Inner Loop Header: Depth=111087; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11088; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]11089; GFX9-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]11090; GFX9-NEXT: v_add_co_u32_e64 v0, s[36:37], -1, v211091; GFX9-NEXT: v_addc_co_u32_e64 v1, s[36:37], -1, v3, s[36:37]11092; GFX9-NEXT: s_or_b64 vcc, vcc, s[34:35]11093; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc11094; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc11095; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc11096; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11097; GFX9-NEXT: buffer_wbinvl1_vol11098; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]11099; GFX9-NEXT: v_mov_b32_e32 v3, v111100; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39]11101; GFX9-NEXT: v_mov_b32_e32 v2, v011102; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39]11103; GFX9-NEXT: s_cbranch_execnz .LBB145_111104; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end11105; GFX9-NEXT: s_or_b64 exec, exec, s[38:39]11106; GFX9-NEXT: s_setpc_b64 s[30:31]11107 %tmp0 = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !111108 ret void11109}11110 11111define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {11112; GFX7-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar:11113; GFX7: ; %bb.0:11114; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11115; GFX7-NEXT: s_add_u32 s34, s4, 3211116; GFX7-NEXT: s_addc_u32 s35, s5, 011117; GFX7-NEXT: s_add_u32 s36, s4, 3611118; GFX7-NEXT: s_addc_u32 s37, s5, 011119; GFX7-NEXT: v_mov_b32_e32 v0, s3611120; GFX7-NEXT: v_mov_b32_e32 v1, s3711121; GFX7-NEXT: v_mov_b32_e32 v4, s3411122; GFX7-NEXT: v_mov_b32_e32 v5, s3511123; GFX7-NEXT: flat_load_dword v3, v[0:1]11124; GFX7-NEXT: flat_load_dword v2, v[4:5]11125; GFX7-NEXT: s_mov_b64 s[38:39], 011126; GFX7-NEXT: v_mov_b32_e32 v6, s711127; GFX7-NEXT: v_mov_b32_e32 v7, s611128; GFX7-NEXT: .LBB146_1: ; %atomicrmw.start11129; GFX7-NEXT: ; =>This Inner Loop Header: Depth=111130; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11131; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]11132; GFX7-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]11133; GFX7-NEXT: v_add_i32_e64 v0, s[36:37], -1, v211134; GFX7-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v3, s[36:37]11135; GFX7-NEXT: s_or_b64 vcc, vcc, s[34:35]11136; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc11137; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc11138; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc11139; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11140; GFX7-NEXT: buffer_wbinvl1_vol11141; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]11142; GFX7-NEXT: v_mov_b32_e32 v3, v111143; GFX7-NEXT: s_or_b64 s[38:39], vcc, s[38:39]11144; GFX7-NEXT: v_mov_b32_e32 v2, v011145; GFX7-NEXT: s_andn2_b64 exec, exec, s[38:39]11146; GFX7-NEXT: s_cbranch_execnz .LBB146_111147; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end11148; GFX7-NEXT: s_or_b64 exec, exec, s[38:39]11149; GFX7-NEXT: s_setpc_b64 s[30:31]11150;11151; GFX8-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar:11152; GFX8: ; %bb.0:11153; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11154; GFX8-NEXT: s_add_u32 s34, s4, 3211155; GFX8-NEXT: s_addc_u32 s35, s5, 011156; GFX8-NEXT: s_add_u32 s36, s4, 3611157; GFX8-NEXT: s_addc_u32 s37, s5, 011158; GFX8-NEXT: v_mov_b32_e32 v0, s3611159; GFX8-NEXT: v_mov_b32_e32 v1, s3711160; GFX8-NEXT: v_mov_b32_e32 v4, s3411161; GFX8-NEXT: v_mov_b32_e32 v5, s3511162; GFX8-NEXT: flat_load_dword v3, v[0:1]11163; GFX8-NEXT: flat_load_dword v2, v[4:5]11164; GFX8-NEXT: s_mov_b64 s[38:39], 011165; GFX8-NEXT: v_mov_b32_e32 v6, s711166; GFX8-NEXT: v_mov_b32_e32 v7, s611167; GFX8-NEXT: .LBB146_1: ; %atomicrmw.start11168; GFX8-NEXT: ; =>This Inner Loop Header: Depth=111169; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11170; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]11171; GFX8-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]11172; GFX8-NEXT: v_add_u32_e64 v0, s[36:37], -1, v211173; GFX8-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v3, s[36:37]11174; GFX8-NEXT: s_or_b64 vcc, vcc, s[34:35]11175; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc11176; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc11177; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc11178; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11179; GFX8-NEXT: buffer_wbinvl1_vol11180; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]11181; GFX8-NEXT: v_mov_b32_e32 v3, v111182; GFX8-NEXT: s_or_b64 s[38:39], vcc, s[38:39]11183; GFX8-NEXT: v_mov_b32_e32 v2, v011184; GFX8-NEXT: s_andn2_b64 exec, exec, s[38:39]11185; GFX8-NEXT: s_cbranch_execnz .LBB146_111186; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end11187; GFX8-NEXT: s_or_b64 exec, exec, s[38:39]11188; GFX8-NEXT: s_setpc_b64 s[30:31]11189;11190; GFX9-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar:11191; GFX9: ; %bb.0:11192; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11193; GFX9-NEXT: v_mov_b32_e32 v0, s411194; GFX9-NEXT: v_mov_b32_e32 v1, s511195; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:3211196; GFX9-NEXT: v_mov_b32_e32 v4, s411197; GFX9-NEXT: s_mov_b64 s[38:39], 011198; GFX9-NEXT: v_mov_b32_e32 v6, s711199; GFX9-NEXT: v_mov_b32_e32 v7, s611200; GFX9-NEXT: v_mov_b32_e32 v5, s511201; GFX9-NEXT: .LBB146_1: ; %atomicrmw.start11202; GFX9-NEXT: ; =>This Inner Loop Header: Depth=111203; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11204; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]11205; GFX9-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]11206; GFX9-NEXT: v_add_co_u32_e64 v0, s[36:37], -1, v211207; GFX9-NEXT: v_addc_co_u32_e64 v1, s[36:37], -1, v3, s[36:37]11208; GFX9-NEXT: s_or_b64 vcc, vcc, s[34:35]11209; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc11210; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc11211; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc11212; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11213; GFX9-NEXT: buffer_wbinvl1_vol11214; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]11215; GFX9-NEXT: v_mov_b32_e32 v3, v111216; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39]11217; GFX9-NEXT: v_mov_b32_e32 v2, v011218; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39]11219; GFX9-NEXT: s_cbranch_execnz .LBB146_111220; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end11221; GFX9-NEXT: s_or_b64 exec, exec, s[38:39]11222; GFX9-NEXT: s_setpc_b64 s[30:31]11223 %gep = getelementptr inbounds i64, ptr %out, i64 411224 %tmp0 = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !111225 ret void11226}11227 11228define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_scalar(ptr inreg %ptr, i64 inreg %in) {11229; GFX7-LABEL: flat_atomic_udec_wrap_i64_ret_scalar:11230; GFX7: ; %bb.0:11231; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11232; GFX7-NEXT: v_mov_b32_e32 v0, s411233; GFX7-NEXT: s_add_u32 s34, s4, 411234; GFX7-NEXT: v_mov_b32_e32 v1, s511235; GFX7-NEXT: s_addc_u32 s35, s5, 011236; GFX7-NEXT: v_mov_b32_e32 v2, s3411237; GFX7-NEXT: v_mov_b32_e32 v3, s3511238; GFX7-NEXT: flat_load_dword v0, v[0:1]11239; GFX7-NEXT: flat_load_dword v1, v[2:3]11240; GFX7-NEXT: v_mov_b32_e32 v2, s411241; GFX7-NEXT: s_mov_b64 s[38:39], 011242; GFX7-NEXT: v_mov_b32_e32 v4, s711243; GFX7-NEXT: v_mov_b32_e32 v5, s611244; GFX7-NEXT: v_mov_b32_e32 v3, s511245; GFX7-NEXT: .LBB147_1: ; %atomicrmw.start11246; GFX7-NEXT: ; =>This Inner Loop Header: Depth=111247; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11248; GFX7-NEXT: v_mov_b32_e32 v9, v111249; GFX7-NEXT: v_mov_b32_e32 v8, v011250; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]11251; GFX7-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]11252; GFX7-NEXT: v_add_i32_e64 v0, s[36:37], -1, v811253; GFX7-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v9, s[36:37]11254; GFX7-NEXT: s_or_b64 vcc, vcc, s[34:35]11255; GFX7-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc11256; GFX7-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc11257; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc11258; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11259; GFX7-NEXT: buffer_wbinvl1_vol11260; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]11261; GFX7-NEXT: s_or_b64 s[38:39], vcc, s[38:39]11262; GFX7-NEXT: s_andn2_b64 exec, exec, s[38:39]11263; GFX7-NEXT: s_cbranch_execnz .LBB147_111264; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end11265; GFX7-NEXT: s_or_b64 exec, exec, s[38:39]11266; GFX7-NEXT: s_setpc_b64 s[30:31]11267;11268; GFX8-LABEL: flat_atomic_udec_wrap_i64_ret_scalar:11269; GFX8: ; %bb.0:11270; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11271; GFX8-NEXT: v_mov_b32_e32 v0, s411272; GFX8-NEXT: s_add_u32 s34, s4, 411273; GFX8-NEXT: v_mov_b32_e32 v1, s511274; GFX8-NEXT: s_addc_u32 s35, s5, 011275; GFX8-NEXT: v_mov_b32_e32 v2, s3411276; GFX8-NEXT: v_mov_b32_e32 v3, s3511277; GFX8-NEXT: flat_load_dword v0, v[0:1]11278; GFX8-NEXT: flat_load_dword v1, v[2:3]11279; GFX8-NEXT: v_mov_b32_e32 v2, s411280; GFX8-NEXT: s_mov_b64 s[38:39], 011281; GFX8-NEXT: v_mov_b32_e32 v4, s711282; GFX8-NEXT: v_mov_b32_e32 v5, s611283; GFX8-NEXT: v_mov_b32_e32 v3, s511284; GFX8-NEXT: .LBB147_1: ; %atomicrmw.start11285; GFX8-NEXT: ; =>This Inner Loop Header: Depth=111286; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11287; GFX8-NEXT: v_mov_b32_e32 v9, v111288; GFX8-NEXT: v_mov_b32_e32 v8, v011289; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]11290; GFX8-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]11291; GFX8-NEXT: v_add_u32_e64 v0, s[36:37], -1, v811292; GFX8-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v9, s[36:37]11293; GFX8-NEXT: s_or_b64 vcc, vcc, s[34:35]11294; GFX8-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc11295; GFX8-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc11296; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc11297; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11298; GFX8-NEXT: buffer_wbinvl1_vol11299; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]11300; GFX8-NEXT: s_or_b64 s[38:39], vcc, s[38:39]11301; GFX8-NEXT: s_andn2_b64 exec, exec, s[38:39]11302; GFX8-NEXT: s_cbranch_execnz .LBB147_111303; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end11304; GFX8-NEXT: s_or_b64 exec, exec, s[38:39]11305; GFX8-NEXT: s_setpc_b64 s[30:31]11306;11307; GFX9-LABEL: flat_atomic_udec_wrap_i64_ret_scalar:11308; GFX9: ; %bb.0:11309; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11310; GFX9-NEXT: v_mov_b32_e32 v0, s411311; GFX9-NEXT: v_mov_b32_e32 v1, s511312; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]11313; GFX9-NEXT: v_mov_b32_e32 v2, s411314; GFX9-NEXT: s_mov_b64 s[38:39], 011315; GFX9-NEXT: v_mov_b32_e32 v4, s711316; GFX9-NEXT: v_mov_b32_e32 v5, s611317; GFX9-NEXT: v_mov_b32_e32 v3, s511318; GFX9-NEXT: .LBB147_1: ; %atomicrmw.start11319; GFX9-NEXT: ; =>This Inner Loop Header: Depth=111320; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11321; GFX9-NEXT: v_mov_b32_e32 v9, v111322; GFX9-NEXT: v_mov_b32_e32 v8, v011323; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]11324; GFX9-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]11325; GFX9-NEXT: v_add_co_u32_e64 v0, s[36:37], -1, v811326; GFX9-NEXT: v_addc_co_u32_e64 v1, s[36:37], -1, v9, s[36:37]11327; GFX9-NEXT: s_or_b64 vcc, vcc, s[34:35]11328; GFX9-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc11329; GFX9-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc11330; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc11331; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11332; GFX9-NEXT: buffer_wbinvl1_vol11333; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]11334; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39]11335; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39]11336; GFX9-NEXT: s_cbranch_execnz .LBB147_111337; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end11338; GFX9-NEXT: s_or_b64 exec, exec, s[38:39]11339; GFX9-NEXT: s_setpc_b64 s[30:31]11340 %result = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !111341 ret i64 %result11342}11343 11344define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_offset_scalar(ptr inreg %out, i64 inreg %in) {11345; GFX7-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar:11346; GFX7: ; %bb.0:11347; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11348; GFX7-NEXT: s_add_u32 s34, s4, 3211349; GFX7-NEXT: s_addc_u32 s35, s5, 011350; GFX7-NEXT: s_add_u32 s36, s4, 3611351; GFX7-NEXT: s_addc_u32 s37, s5, 011352; GFX7-NEXT: v_mov_b32_e32 v0, s3611353; GFX7-NEXT: v_mov_b32_e32 v1, s3711354; GFX7-NEXT: v_mov_b32_e32 v2, s3411355; GFX7-NEXT: v_mov_b32_e32 v3, s3511356; GFX7-NEXT: flat_load_dword v1, v[0:1]11357; GFX7-NEXT: flat_load_dword v0, v[2:3]11358; GFX7-NEXT: s_mov_b64 s[38:39], 011359; GFX7-NEXT: v_mov_b32_e32 v4, s711360; GFX7-NEXT: v_mov_b32_e32 v5, s611361; GFX7-NEXT: .LBB148_1: ; %atomicrmw.start11362; GFX7-NEXT: ; =>This Inner Loop Header: Depth=111363; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11364; GFX7-NEXT: v_mov_b32_e32 v9, v111365; GFX7-NEXT: v_mov_b32_e32 v8, v011366; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]11367; GFX7-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]11368; GFX7-NEXT: v_add_i32_e64 v0, s[36:37], -1, v811369; GFX7-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v9, s[36:37]11370; GFX7-NEXT: s_or_b64 vcc, vcc, s[34:35]11371; GFX7-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc11372; GFX7-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc11373; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc11374; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11375; GFX7-NEXT: buffer_wbinvl1_vol11376; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]11377; GFX7-NEXT: s_or_b64 s[38:39], vcc, s[38:39]11378; GFX7-NEXT: s_andn2_b64 exec, exec, s[38:39]11379; GFX7-NEXT: s_cbranch_execnz .LBB148_111380; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end11381; GFX7-NEXT: s_or_b64 exec, exec, s[38:39]11382; GFX7-NEXT: s_setpc_b64 s[30:31]11383;11384; GFX8-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar:11385; GFX8: ; %bb.0:11386; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11387; GFX8-NEXT: s_add_u32 s34, s4, 3211388; GFX8-NEXT: s_addc_u32 s35, s5, 011389; GFX8-NEXT: s_add_u32 s36, s4, 3611390; GFX8-NEXT: s_addc_u32 s37, s5, 011391; GFX8-NEXT: v_mov_b32_e32 v0, s3611392; GFX8-NEXT: v_mov_b32_e32 v1, s3711393; GFX8-NEXT: v_mov_b32_e32 v2, s3411394; GFX8-NEXT: v_mov_b32_e32 v3, s3511395; GFX8-NEXT: flat_load_dword v1, v[0:1]11396; GFX8-NEXT: flat_load_dword v0, v[2:3]11397; GFX8-NEXT: s_mov_b64 s[38:39], 011398; GFX8-NEXT: v_mov_b32_e32 v4, s711399; GFX8-NEXT: v_mov_b32_e32 v5, s611400; GFX8-NEXT: .LBB148_1: ; %atomicrmw.start11401; GFX8-NEXT: ; =>This Inner Loop Header: Depth=111402; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11403; GFX8-NEXT: v_mov_b32_e32 v9, v111404; GFX8-NEXT: v_mov_b32_e32 v8, v011405; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]11406; GFX8-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]11407; GFX8-NEXT: v_add_u32_e64 v0, s[36:37], -1, v811408; GFX8-NEXT: v_addc_u32_e64 v1, s[36:37], -1, v9, s[36:37]11409; GFX8-NEXT: s_or_b64 vcc, vcc, s[34:35]11410; GFX8-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc11411; GFX8-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc11412; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc11413; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11414; GFX8-NEXT: buffer_wbinvl1_vol11415; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]11416; GFX8-NEXT: s_or_b64 s[38:39], vcc, s[38:39]11417; GFX8-NEXT: s_andn2_b64 exec, exec, s[38:39]11418; GFX8-NEXT: s_cbranch_execnz .LBB148_111419; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end11420; GFX8-NEXT: s_or_b64 exec, exec, s[38:39]11421; GFX8-NEXT: s_setpc_b64 s[30:31]11422;11423; GFX9-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar:11424; GFX9: ; %bb.0:11425; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11426; GFX9-NEXT: v_mov_b32_e32 v0, s411427; GFX9-NEXT: v_mov_b32_e32 v1, s511428; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:3211429; GFX9-NEXT: v_mov_b32_e32 v2, s411430; GFX9-NEXT: s_mov_b64 s[38:39], 011431; GFX9-NEXT: v_mov_b32_e32 v4, s711432; GFX9-NEXT: v_mov_b32_e32 v5, s611433; GFX9-NEXT: v_mov_b32_e32 v3, s511434; GFX9-NEXT: .LBB148_1: ; %atomicrmw.start11435; GFX9-NEXT: ; =>This Inner Loop Header: Depth=111436; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11437; GFX9-NEXT: v_mov_b32_e32 v9, v111438; GFX9-NEXT: v_mov_b32_e32 v8, v011439; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]11440; GFX9-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]11441; GFX9-NEXT: v_add_co_u32_e64 v0, s[36:37], -1, v811442; GFX9-NEXT: v_addc_co_u32_e64 v1, s[36:37], -1, v9, s[36:37]11443; GFX9-NEXT: s_or_b64 vcc, vcc, s[34:35]11444; GFX9-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc11445; GFX9-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc11446; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc11447; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11448; GFX9-NEXT: buffer_wbinvl1_vol11449; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]11450; GFX9-NEXT: s_or_b64 s[38:39], vcc, s[38:39]11451; GFX9-NEXT: s_andn2_b64 exec, exec, s[38:39]11452; GFX9-NEXT: s_cbranch_execnz .LBB148_111453; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end11454; GFX9-NEXT: s_or_b64 exec, exec, s[38:39]11455; GFX9-NEXT: s_setpc_b64 s[30:31]11456 %gep = getelementptr inbounds i64, ptr %out, i64 411457 %result = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !111458 ret i64 %result11459}11460 11461define void @flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {11462; GFX7-LABEL: flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory:11463; GFX7: ; %bb.0:11464; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11465; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v011466; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc11467; GFX7-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]11468; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11469; GFX7-NEXT: buffer_wbinvl1_vol11470; GFX7-NEXT: s_setpc_b64 s[30:31]11471;11472; GFX8-LABEL: flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory:11473; GFX8: ; %bb.0:11474; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11475; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v011476; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc11477; GFX8-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]11478; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11479; GFX8-NEXT: buffer_wbinvl1_vol11480; GFX8-NEXT: s_setpc_b64 s[30:31]11481;11482; GFX9-LABEL: flat_atomic_udec_wrap_i64_noret_offset__amdgpu_no_remote_memory:11483; GFX9: ; %bb.0:11484; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11485; GFX9-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3] offset:3211486; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11487; GFX9-NEXT: buffer_wbinvl1_vol11488; GFX9-NEXT: s_setpc_b64 s[30:31]11489 %gep = getelementptr inbounds i64, ptr %out, i64 411490 %tmp0 = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !111491 ret void11492}11493 11494define i64 @flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i64 %in) {11495; GFX7-LABEL: flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory:11496; GFX7: ; %bb.0:11497; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11498; GFX7-NEXT: v_add_i32_e32 v0, vcc, 32, v011499; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc11500; GFX7-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc11501; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11502; GFX7-NEXT: buffer_wbinvl1_vol11503; GFX7-NEXT: s_setpc_b64 s[30:31]11504;11505; GFX8-LABEL: flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory:11506; GFX8: ; %bb.0:11507; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11508; GFX8-NEXT: v_add_u32_e32 v0, vcc, 32, v011509; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc11510; GFX8-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc11511; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11512; GFX8-NEXT: buffer_wbinvl1_vol11513; GFX8-NEXT: s_setpc_b64 s[30:31]11514;11515; GFX9-LABEL: flat_atomic_udec_wrap_i64_ret_offset__amdgpu_no_remote_memory:11516; GFX9: ; %bb.0:11517; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11518; GFX9-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] offset:32 glc11519; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11520; GFX9-NEXT: buffer_wbinvl1_vol11521; GFX9-NEXT: s_setpc_b64 s[30:31]11522 %gep = getelementptr inbounds i64, ptr %out, i64 411523 %result = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !111524 ret i64 %result11525}11526 11527!0 = !{}11528!1 = !{i32 5, i32 6}11529