13762 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -global-isel=0 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG %s3; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s4; RUN: llc -global-isel=0 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx950 < %s | FileCheck -check-prefixes=GFX950,GFX950-SDAG %s5; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx950 < %s | FileCheck -check-prefixes=GFX950,GFX950-GISEL %s6 7; Test using saddr addressing mode of flat_* atomic instructions. Make8; sure these are not incorrectly selected before gfx1250.9 10define amdgpu_ps void @flat_xchg_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {11; GFX1250-LABEL: flat_xchg_saddr_i32_nortn:12; GFX1250: ; %bb.0:13; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 114; GFX1250-NEXT: global_wb scope:SCOPE_DEV15; GFX1250-NEXT: s_wait_storecnt 0x016; GFX1250-NEXT: flat_atomic_swap_b32 v0, v1, s[2:3] scope:SCOPE_DEV17; GFX1250-NEXT: s_wait_storecnt_dscnt 0x018; GFX1250-NEXT: global_inv scope:SCOPE_DEV19; GFX1250-NEXT: s_endpgm20;21; GFX950-SDAG-LABEL: flat_xchg_saddr_i32_nortn:22; GFX950-SDAG: ; %bb.0:23; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v124; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 025; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]26; GFX950-SDAG-NEXT: buffer_wbl2 sc127; GFX950-SDAG-NEXT: flat_atomic_swap v[0:1], v228; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)29; GFX950-SDAG-NEXT: buffer_inv sc130; GFX950-SDAG-NEXT: s_endpgm31;32; GFX950-GISEL-LABEL: flat_xchg_saddr_i32_nortn:33; GFX950-GISEL: ; %bb.0:34; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]35; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v036; GFX950-GISEL-NEXT: s_nop 137; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc38; GFX950-GISEL-NEXT: buffer_wbl2 sc139; GFX950-GISEL-NEXT: flat_atomic_swap v[2:3], v140; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)41; GFX950-GISEL-NEXT: buffer_inv sc142; GFX950-GISEL-NEXT: s_endpgm43 %zext.offset = zext i32 %voffset to i6444 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset45 %unused = atomicrmw xchg ptr %gep0, i32 %data syncscope("agent") seq_cst46 ret void47}48 49; Maximum positive offset on gfx1050define amdgpu_ps void @flat_xchg_saddr_i32_nortn_offset_2047(ptr inreg %sbase, i32 %voffset, i32 %data) {51; GFX1250-LABEL: flat_xchg_saddr_i32_nortn_offset_2047:52; GFX1250: ; %bb.0:53; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 154; GFX1250-NEXT: global_wb scope:SCOPE_DEV55; GFX1250-NEXT: s_wait_storecnt 0x056; GFX1250-NEXT: flat_atomic_swap_b32 v0, v1, s[2:3] offset:2047 scope:SCOPE_DEV57; GFX1250-NEXT: s_wait_storecnt_dscnt 0x058; GFX1250-NEXT: global_inv scope:SCOPE_DEV59; GFX1250-NEXT: s_endpgm60;61; GFX950-SDAG-LABEL: flat_xchg_saddr_i32_nortn_offset_2047:62; GFX950-SDAG: ; %bb.0:63; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v164; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 065; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]66; GFX950-SDAG-NEXT: buffer_wbl2 sc167; GFX950-SDAG-NEXT: flat_atomic_swap v[0:1], v2 offset:204768; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)69; GFX950-SDAG-NEXT: buffer_inv sc170; GFX950-SDAG-NEXT: s_endpgm71;72; GFX950-GISEL-LABEL: flat_xchg_saddr_i32_nortn_offset_2047:73; GFX950-GISEL: ; %bb.0:74; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]75; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v076; GFX950-GISEL-NEXT: s_nop 177; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc78; GFX950-GISEL-NEXT: buffer_wbl2 sc179; GFX950-GISEL-NEXT: flat_atomic_swap v[2:3], v1 offset:204780; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)81; GFX950-GISEL-NEXT: buffer_inv sc182; GFX950-GISEL-NEXT: s_endpgm83 %zext.offset = zext i32 %voffset to i6484 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset85 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 204786 %unused = atomicrmw xchg ptr %gep1, i32 %data syncscope("agent") seq_cst87 ret void88}89 90; Maximum negative offset on gfx1091define amdgpu_ps void @flat_xchg_saddr_i32_nortn_offset_neg2048(ptr inreg %sbase, i32 %voffset, i32 %data) {92; GFX1250-LABEL: flat_xchg_saddr_i32_nortn_offset_neg2048:93; GFX1250: ; %bb.0:94; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 195; GFX1250-NEXT: global_wb scope:SCOPE_DEV96; GFX1250-NEXT: s_wait_storecnt 0x097; GFX1250-NEXT: flat_atomic_swap_b32 v0, v1, s[2:3] offset:-2048 scope:SCOPE_DEV98; GFX1250-NEXT: s_wait_storecnt_dscnt 0x099; GFX1250-NEXT: global_inv scope:SCOPE_DEV100; GFX1250-NEXT: s_endpgm101;102; GFX950-SDAG-LABEL: flat_xchg_saddr_i32_nortn_offset_neg2048:103; GFX950-SDAG: ; %bb.0:104; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1105; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0106; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]107; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0108; GFX950-SDAG-NEXT: s_nop 1109; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc110; GFX950-SDAG-NEXT: buffer_wbl2 sc1111; GFX950-SDAG-NEXT: flat_atomic_swap v[0:1], v2112; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)113; GFX950-SDAG-NEXT: buffer_inv sc1114; GFX950-SDAG-NEXT: s_endpgm115;116; GFX950-GISEL-LABEL: flat_xchg_saddr_i32_nortn_offset_neg2048:117; GFX950-GISEL: ; %bb.0:118; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]119; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0120; GFX950-GISEL-NEXT: s_nop 1121; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc122; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xfffff800, v0123; GFX950-GISEL-NEXT: s_nop 1124; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc125; GFX950-GISEL-NEXT: buffer_wbl2 sc1126; GFX950-GISEL-NEXT: flat_atomic_swap v[2:3], v1127; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)128; GFX950-GISEL-NEXT: buffer_inv sc1129; GFX950-GISEL-NEXT: s_endpgm130 %zext.offset = zext i32 %voffset to i64131 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset132 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -2048133 %unused = atomicrmw xchg ptr %gep1, i32 %data syncscope("agent") seq_cst134 ret void135}136 137define amdgpu_ps float @flat_xchg_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {138; GFX1250-LABEL: flat_xchg_saddr_i32_rtn:139; GFX1250: ; %bb.0:140; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1141; GFX1250-NEXT: global_wb scope:SCOPE_DEV142; GFX1250-NEXT: s_wait_storecnt 0x0143; GFX1250-NEXT: flat_atomic_swap_b32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV144; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0145; GFX1250-NEXT: global_inv scope:SCOPE_DEV146; GFX1250-NEXT: s_wait_loadcnt 0x0147; GFX1250-NEXT: ; return to shader part epilog148;149; GFX950-SDAG-LABEL: flat_xchg_saddr_i32_rtn:150; GFX950-SDAG: ; %bb.0:151; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1152; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0153; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]154; GFX950-SDAG-NEXT: buffer_wbl2 sc1155; GFX950-SDAG-NEXT: flat_atomic_swap v0, v[0:1], v2 sc0156; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)157; GFX950-SDAG-NEXT: buffer_inv sc1158; GFX950-SDAG-NEXT: ; return to shader part epilog159;160; GFX950-GISEL-LABEL: flat_xchg_saddr_i32_rtn:161; GFX950-GISEL: ; %bb.0:162; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]163; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0164; GFX950-GISEL-NEXT: s_nop 1165; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc166; GFX950-GISEL-NEXT: buffer_wbl2 sc1167; GFX950-GISEL-NEXT: flat_atomic_swap v0, v[2:3], v1 sc0168; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)169; GFX950-GISEL-NEXT: buffer_inv sc1170; GFX950-GISEL-NEXT: ; return to shader part epilog171 %zext.offset = zext i32 %voffset to i64172 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset173 %rtn = atomicrmw xchg ptr %gep0, i32 %data syncscope("agent") seq_cst174 %cast.rtn = bitcast i32 %rtn to float175 ret float %cast.rtn176}177 178define amdgpu_ps float @flat_xchg_saddr_i32_rtn_2048(ptr inreg %sbase, i32 %voffset, i32 %data) {179; GFX1250-LABEL: flat_xchg_saddr_i32_rtn_2048:180; GFX1250: ; %bb.0:181; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1182; GFX1250-NEXT: global_wb scope:SCOPE_DEV183; GFX1250-NEXT: s_wait_storecnt 0x0184; GFX1250-NEXT: flat_atomic_swap_b32 v0, v0, v1, s[2:3] offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV185; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0186; GFX1250-NEXT: global_inv scope:SCOPE_DEV187; GFX1250-NEXT: s_wait_loadcnt 0x0188; GFX1250-NEXT: ; return to shader part epilog189;190; GFX950-SDAG-LABEL: flat_xchg_saddr_i32_rtn_2048:191; GFX950-SDAG: ; %bb.0:192; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1193; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0194; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]195; GFX950-SDAG-NEXT: buffer_wbl2 sc1196; GFX950-SDAG-NEXT: flat_atomic_swap v0, v[0:1], v2 offset:2048 sc0197; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)198; GFX950-SDAG-NEXT: buffer_inv sc1199; GFX950-SDAG-NEXT: ; return to shader part epilog200;201; GFX950-GISEL-LABEL: flat_xchg_saddr_i32_rtn_2048:202; GFX950-GISEL: ; %bb.0:203; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]204; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0205; GFX950-GISEL-NEXT: s_nop 1206; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc207; GFX950-GISEL-NEXT: buffer_wbl2 sc1208; GFX950-GISEL-NEXT: flat_atomic_swap v0, v[2:3], v1 offset:2048 sc0209; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)210; GFX950-GISEL-NEXT: buffer_inv sc1211; GFX950-GISEL-NEXT: ; return to shader part epilog212 %zext.offset = zext i32 %voffset to i64213 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset214 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 2048215 %rtn = atomicrmw xchg ptr %gep1, i32 %data syncscope("agent") seq_cst216 %cast.rtn = bitcast i32 %rtn to float217 ret float %cast.rtn218}219 220define amdgpu_ps float @flat_xchg_saddr_i32_rtn_neg2048(ptr inreg %sbase, i32 %voffset, i32 %data) {221; GFX1250-LABEL: flat_xchg_saddr_i32_rtn_neg2048:222; GFX1250: ; %bb.0:223; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1224; GFX1250-NEXT: global_wb scope:SCOPE_DEV225; GFX1250-NEXT: s_wait_storecnt 0x0226; GFX1250-NEXT: flat_atomic_swap_b32 v0, v0, v1, s[2:3] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV227; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0228; GFX1250-NEXT: global_inv scope:SCOPE_DEV229; GFX1250-NEXT: s_wait_loadcnt 0x0230; GFX1250-NEXT: ; return to shader part epilog231;232; GFX950-SDAG-LABEL: flat_xchg_saddr_i32_rtn_neg2048:233; GFX950-SDAG: ; %bb.0:234; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1235; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0236; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]237; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0238; GFX950-SDAG-NEXT: s_nop 1239; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc240; GFX950-SDAG-NEXT: buffer_wbl2 sc1241; GFX950-SDAG-NEXT: flat_atomic_swap v0, v[0:1], v2 sc0242; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)243; GFX950-SDAG-NEXT: buffer_inv sc1244; GFX950-SDAG-NEXT: ; return to shader part epilog245;246; GFX950-GISEL-LABEL: flat_xchg_saddr_i32_rtn_neg2048:247; GFX950-GISEL: ; %bb.0:248; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]249; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0250; GFX950-GISEL-NEXT: s_nop 1251; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc252; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xfffff800, v0253; GFX950-GISEL-NEXT: s_nop 1254; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc255; GFX950-GISEL-NEXT: buffer_wbl2 sc1256; GFX950-GISEL-NEXT: flat_atomic_swap v0, v[2:3], v1 sc0257; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)258; GFX950-GISEL-NEXT: buffer_inv sc1259; GFX950-GISEL-NEXT: ; return to shader part epilog260 %zext.offset = zext i32 %voffset to i64261 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset262 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -2048263 %rtn = atomicrmw xchg ptr %gep1, i32 %data syncscope("agent") seq_cst264 %cast.rtn = bitcast i32 %rtn to float265 ret float %cast.rtn266}267 268; --------------------------------------------------------------------------------269; Uniformity edge cases270; --------------------------------------------------------------------------------271 272@ptr.in.lds = internal addrspace(3) global ptr poison273 274; Base pointer is uniform, but also in VGPRs275define amdgpu_ps float @flat_xchg_saddr_uniform_ptr_in_vgprs_rtn(i32 %voffset, i32 %data) {276; GFX1250-SDAG-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_rtn:277; GFX1250-SDAG: ; %bb.0:278; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1279; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0280; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2281; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0282; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v2283; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v3284; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV285; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0286; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v0, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV287; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0288; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV289; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0290; GFX1250-SDAG-NEXT: ; return to shader part epilog291;292; GFX1250-GISEL-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_rtn:293; GFX1250-GISEL: ; %bb.0:294; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1295; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0296; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v2297; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0298; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0299; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)300; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo301; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV302; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0303; GFX1250-GISEL-NEXT: flat_atomic_swap_b32 v0, v[2:3], v1 th:TH_ATOMIC_RETURN scope:SCOPE_DEV304; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0305; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV306; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0307; GFX1250-GISEL-NEXT: ; return to shader part epilog308;309; GFX950-SDAG-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_rtn:310; GFX950-SDAG: ; %bb.0:311; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1312; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0313; GFX950-SDAG-NEXT: ds_read_b64 v[4:5], v1314; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)315; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[4:5], 0, v[0:1]316; GFX950-SDAG-NEXT: buffer_wbl2 sc1317; GFX950-SDAG-NEXT: flat_atomic_swap v0, v[0:1], v2 sc0318; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)319; GFX950-SDAG-NEXT: buffer_inv sc1320; GFX950-SDAG-NEXT: ; return to shader part epilog321;322; GFX950-GISEL-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_rtn:323; GFX950-GISEL: ; %bb.0:324; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 0325; GFX950-GISEL-NEXT: ds_read_b64 v[2:3], v2326; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)327; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0328; GFX950-GISEL-NEXT: s_nop 1329; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc330; GFX950-GISEL-NEXT: buffer_wbl2 sc1331; GFX950-GISEL-NEXT: flat_atomic_swap v0, v[2:3], v1 sc0332; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)333; GFX950-GISEL-NEXT: buffer_inv sc1334; GFX950-GISEL-NEXT: ; return to shader part epilog335 %sbase = load ptr, ptr addrspace(3) @ptr.in.lds336 %zext.offset = zext i32 %voffset to i64337 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset338 %rtn = atomicrmw xchg ptr %gep0, i32 %data syncscope("agent") seq_cst339 %cast.rtn = bitcast i32 %rtn to float340 ret float %cast.rtn341}342 343; Base pointer is uniform, but also in VGPRs, with imm offset344define amdgpu_ps float @flat_xchg_saddr_uniform_ptr_in_vgprs_rtn_immoffset(i32 %voffset, i32 %data) {345; GFX1250-SDAG-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_rtn_immoffset:346; GFX1250-SDAG: ; %bb.0:347; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1348; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0349; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2350; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0351; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v2352; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v3353; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV354; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0355; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v0, v0, v1, s[0:1] offset:42 th:TH_ATOMIC_RETURN scope:SCOPE_DEV356; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0357; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV358; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0359; GFX1250-SDAG-NEXT: ; return to shader part epilog360;361; GFX1250-GISEL-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_rtn_immoffset:362; GFX1250-GISEL: ; %bb.0:363; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1364; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0365; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v2366; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0367; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0368; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)369; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo370; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV371; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0372; GFX1250-GISEL-NEXT: flat_atomic_swap_b32 v0, v[2:3], v1 offset:42 th:TH_ATOMIC_RETURN scope:SCOPE_DEV373; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0374; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV375; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0376; GFX1250-GISEL-NEXT: ; return to shader part epilog377;378; GFX950-SDAG-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_rtn_immoffset:379; GFX950-SDAG: ; %bb.0:380; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1381; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0382; GFX950-SDAG-NEXT: ds_read_b64 v[4:5], v1383; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)384; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[4:5], 0, v[0:1]385; GFX950-SDAG-NEXT: buffer_wbl2 sc1386; GFX950-SDAG-NEXT: flat_atomic_swap v0, v[0:1], v2 offset:42 sc0387; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)388; GFX950-SDAG-NEXT: buffer_inv sc1389; GFX950-SDAG-NEXT: ; return to shader part epilog390;391; GFX950-GISEL-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_rtn_immoffset:392; GFX950-GISEL: ; %bb.0:393; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 0394; GFX950-GISEL-NEXT: ds_read_b64 v[2:3], v2395; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)396; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0397; GFX950-GISEL-NEXT: s_nop 1398; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc399; GFX950-GISEL-NEXT: buffer_wbl2 sc1400; GFX950-GISEL-NEXT: flat_atomic_swap v0, v[2:3], v1 offset:42 sc0401; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)402; GFX950-GISEL-NEXT: buffer_inv sc1403; GFX950-GISEL-NEXT: ; return to shader part epilog404 %sbase = load ptr, ptr addrspace(3) @ptr.in.lds405 %zext.offset = zext i32 %voffset to i64406 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset407 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 42408 %rtn = atomicrmw xchg ptr %gep1, i32 %data syncscope("agent") seq_cst409 %cast.rtn = bitcast i32 %rtn to float410 ret float %cast.rtn411}412 413; Base pointer is uniform, but also in VGPRs414define amdgpu_ps void @flat_xchg_saddr_uniform_ptr_in_vgprs_nortn(i32 %voffset, i32 %data) {415; GFX1250-SDAG-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_nortn:416; GFX1250-SDAG: ; %bb.0:417; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1418; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0419; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2420; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0421; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v2422; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v3423; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV424; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0425; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV426; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0427; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV428; GFX1250-SDAG-NEXT: s_endpgm429;430; GFX1250-GISEL-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_nortn:431; GFX1250-GISEL: ; %bb.0:432; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1433; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0434; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v2435; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0436; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0437; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)438; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo439; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV440; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0441; GFX1250-GISEL-NEXT: flat_atomic_swap_b32 v[2:3], v1 scope:SCOPE_DEV442; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0443; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV444; GFX1250-GISEL-NEXT: s_endpgm445;446; GFX950-SDAG-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_nortn:447; GFX950-SDAG: ; %bb.0:448; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1449; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0450; GFX950-SDAG-NEXT: ds_read_b64 v[4:5], v1451; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)452; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[4:5], 0, v[0:1]453; GFX950-SDAG-NEXT: buffer_wbl2 sc1454; GFX950-SDAG-NEXT: flat_atomic_swap v[0:1], v2455; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)456; GFX950-SDAG-NEXT: buffer_inv sc1457; GFX950-SDAG-NEXT: s_endpgm458;459; GFX950-GISEL-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_nortn:460; GFX950-GISEL: ; %bb.0:461; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 0462; GFX950-GISEL-NEXT: ds_read_b64 v[2:3], v2463; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)464; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0465; GFX950-GISEL-NEXT: s_nop 1466; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc467; GFX950-GISEL-NEXT: buffer_wbl2 sc1468; GFX950-GISEL-NEXT: flat_atomic_swap v[2:3], v1469; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)470; GFX950-GISEL-NEXT: buffer_inv sc1471; GFX950-GISEL-NEXT: s_endpgm472 %sbase = load ptr, ptr addrspace(3) @ptr.in.lds473 %zext.offset = zext i32 %voffset to i64474 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset475 %unused = atomicrmw xchg ptr %gep0, i32 %data syncscope("agent") seq_cst476 ret void477}478 479; Base pointer is uniform, but also in VGPRs, with imm offset480define amdgpu_ps void @flat_xchg_saddr_uniform_ptr_in_vgprs_nortn_immoffset(i32 %voffset, i32 %data) {481; GFX1250-SDAG-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_nortn_immoffset:482; GFX1250-SDAG: ; %bb.0:483; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1484; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0485; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2486; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0487; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v2488; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v3489; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV490; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0491; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v0, v1, s[0:1] offset:42 scope:SCOPE_DEV492; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0493; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV494; GFX1250-SDAG-NEXT: s_endpgm495;496; GFX1250-GISEL-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_nortn_immoffset:497; GFX1250-GISEL: ; %bb.0:498; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1499; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0500; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v2501; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0502; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0503; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)504; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo505; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV506; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0507; GFX1250-GISEL-NEXT: flat_atomic_swap_b32 v[2:3], v1 offset:42 scope:SCOPE_DEV508; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0509; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV510; GFX1250-GISEL-NEXT: s_endpgm511;512; GFX950-SDAG-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_nortn_immoffset:513; GFX950-SDAG: ; %bb.0:514; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1515; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0516; GFX950-SDAG-NEXT: ds_read_b64 v[4:5], v1517; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)518; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[4:5], 0, v[0:1]519; GFX950-SDAG-NEXT: buffer_wbl2 sc1520; GFX950-SDAG-NEXT: flat_atomic_swap v[0:1], v2 offset:42521; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)522; GFX950-SDAG-NEXT: buffer_inv sc1523; GFX950-SDAG-NEXT: s_endpgm524;525; GFX950-GISEL-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_nortn_immoffset:526; GFX950-GISEL: ; %bb.0:527; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 0528; GFX950-GISEL-NEXT: ds_read_b64 v[2:3], v2529; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)530; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0531; GFX950-GISEL-NEXT: s_nop 1532; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc533; GFX950-GISEL-NEXT: buffer_wbl2 sc1534; GFX950-GISEL-NEXT: flat_atomic_swap v[2:3], v1 offset:42535; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)536; GFX950-GISEL-NEXT: buffer_inv sc1537; GFX950-GISEL-NEXT: s_endpgm538 %sbase = load ptr, ptr addrspace(3) @ptr.in.lds539 %zext.offset = zext i32 %voffset to i64540 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset541 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 42542 %unused = atomicrmw xchg ptr %gep1, i32 %data syncscope("agent") seq_cst543 ret void544}545 546; --------------------------------------------------------------------------------547; All atomicrmw ops548; --------------------------------------------------------------------------------549 550; --------------------------------------------------------------------------------551; atomicrmw xchg552; --------------------------------------------------------------------------------553 554define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {555; GFX1250-SDAG-LABEL: flat_xchg_saddr_i64_rtn:556; GFX1250-SDAG: ; %bb.0:557; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1558; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1559; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0560; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)561; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]562; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v5563; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)564; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v0565; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1566; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo567; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0568; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB10_3569; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow570; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0571; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB10_4572; GFX1250-SDAG-NEXT: .LBB10_2: ; %atomicrmw.phi573; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0574; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0575; GFX1250-SDAG-NEXT: s_branch .LBB10_5576; GFX1250-SDAG-NEXT: .LBB10_3: ; %atomicrmw.global577; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV578; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0579; GFX1250-SDAG-NEXT: flat_atomic_swap_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV580; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0581; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV582; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5583; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3584; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0585; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB10_2586; GFX1250-SDAG-NEXT: .LBB10_4: ; %atomicrmw.private587; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]588; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4589; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)590; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo591; GFX1250-SDAG-NEXT: s_clause 0x1592; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off593; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off594; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0595; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0596; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0597; GFX1250-SDAG-NEXT: s_branch .LBB10_5598; GFX1250-SDAG-NEXT: .LBB10_5:599;600; GFX1250-GISEL-LABEL: flat_xchg_saddr_i64_rtn:601; GFX1250-GISEL: ; %bb.0:602; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1603; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1604; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]605; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)606; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3607; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo608; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)609; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14610; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0611; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1612; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo613; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)614; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0615; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB10_3616; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow617; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0618; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB10_4619; GFX1250-GISEL-NEXT: .LBB10_2: ; %atomicrmw.phi620; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0621; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0622; GFX1250-GISEL-NEXT: s_branch .LBB10_5623; GFX1250-GISEL-NEXT: .LBB10_3: ; %atomicrmw.global624; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV625; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0626; GFX1250-GISEL-NEXT: flat_atomic_swap_b64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV627; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0628; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV629; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6630; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5631; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0632; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB10_2633; GFX1250-GISEL-NEXT: .LBB10_4: ; %atomicrmw.private634; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]635; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6636; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)637; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo638; GFX1250-GISEL-NEXT: s_clause 0x1639; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off640; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[4:5], off641; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0642; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0643; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0644; GFX1250-GISEL-NEXT: s_branch .LBB10_5645; GFX1250-GISEL-NEXT: .LBB10_5:646;647; GFX950-SDAG-LABEL: flat_xchg_saddr_i64_rtn:648; GFX950-SDAG: ; %bb.0:649; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v2650; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1651; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0652; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base653; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1]654; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v5655; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1656; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc657; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]658; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB10_3659; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow660; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]661; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB10_4662; GFX950-SDAG-NEXT: .LBB10_2: ; %atomicrmw.phi663; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]664; GFX950-SDAG-NEXT: s_branch .LBB10_5665; GFX950-SDAG-NEXT: .LBB10_3: ; %atomicrmw.global666; GFX950-SDAG-NEXT: buffer_wbl2 sc1667; GFX950-SDAG-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] sc0668; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)669; GFX950-SDAG-NEXT: buffer_inv sc1670; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5671; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3672; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]673; GFX950-SDAG-NEXT: s_cbranch_execz .LBB10_2674; GFX950-SDAG-NEXT: .LBB10_4: ; %atomicrmw.private675; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]676; GFX950-SDAG-NEXT: s_nop 1677; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc678; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off679; GFX950-SDAG-NEXT: s_nop 0680; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off681; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]682; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)683; GFX950-SDAG-NEXT: s_branch .LBB10_5684; GFX950-SDAG-NEXT: .LBB10_5:685;686; GFX950-GISEL-LABEL: flat_xchg_saddr_i64_rtn:687; GFX950-GISEL: ; %bb.0:688; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2689; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]690; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v0691; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base692; GFX950-GISEL-NEXT: s_nop 0693; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc694; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1695; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v3696; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1697; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc698; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]699; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB10_3700; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow701; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]702; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB10_4703; GFX950-GISEL-NEXT: .LBB10_2: ; %atomicrmw.phi704; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]705; GFX950-GISEL-NEXT: s_branch .LBB10_5706; GFX950-GISEL-NEXT: .LBB10_3: ; %atomicrmw.global707; GFX950-GISEL-NEXT: buffer_wbl2 sc1708; GFX950-GISEL-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[4:5] sc0709; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)710; GFX950-GISEL-NEXT: buffer_inv sc1711; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3712; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5713; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]714; GFX950-GISEL-NEXT: s_cbranch_execz .LBB10_2715; GFX950-GISEL-NEXT: .LBB10_4: ; %atomicrmw.private716; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]717; GFX950-GISEL-NEXT: s_nop 1718; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc719; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off720; GFX950-GISEL-NEXT: s_nop 0721; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[4:5], off722; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]723; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)724; GFX950-GISEL-NEXT: s_branch .LBB10_5725; GFX950-GISEL-NEXT: .LBB10_5:726 %zext.offset = zext i32 %voffset to i64727 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset728 %rtn = atomicrmw xchg ptr %gep0, i64 %data syncscope("agent") seq_cst729 %cast.rtn = bitcast i64 %rtn to <2 x float>730 ret <2 x float> %cast.rtn731}732 733define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {734; GFX1250-SDAG-LABEL: flat_xchg_saddr_i64_rtn_neg128:735; GFX1250-SDAG: ; %bb.0:736; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1737; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1738; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0739; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80740; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)741; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]742; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]743; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)744; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v5745; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v0746; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1747; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo748; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)749; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0750; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB11_3751; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow752; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0753; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB11_4754; GFX1250-SDAG-NEXT: .LBB11_2: ; %atomicrmw.phi755; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0756; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0757; GFX1250-SDAG-NEXT: s_branch .LBB11_5758; GFX1250-SDAG-NEXT: .LBB11_3: ; %atomicrmw.global759; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV760; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0761; GFX1250-SDAG-NEXT: flat_atomic_swap_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV762; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0763; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV764; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5765; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3766; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0767; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB11_2768; GFX1250-SDAG-NEXT: .LBB11_4: ; %atomicrmw.private769; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]770; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v4771; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)772; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo773; GFX1250-SDAG-NEXT: s_clause 0x1774; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off775; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off776; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0777; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0778; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0779; GFX1250-SDAG-NEXT: s_branch .LBB11_5780; GFX1250-SDAG-NEXT: .LBB11_5:781;782; GFX1250-GISEL-LABEL: flat_xchg_saddr_i64_rtn_neg128:783; GFX1250-GISEL: ; %bb.0:784; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1785; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1786; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]787; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)788; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3789; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo790; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)791; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0792; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo793; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)794; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14795; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0796; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1797; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo798; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)799; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0800; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB11_3801; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow802; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0803; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB11_4804; GFX1250-GISEL-NEXT: .LBB11_2: ; %atomicrmw.phi805; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0806; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0807; GFX1250-GISEL-NEXT: s_branch .LBB11_5808; GFX1250-GISEL-NEXT: .LBB11_3: ; %atomicrmw.global809; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV810; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0811; GFX1250-GISEL-NEXT: flat_atomic_swap_b64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV812; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0813; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV814; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6815; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5816; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0817; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB11_2818; GFX1250-GISEL-NEXT: .LBB11_4: ; %atomicrmw.private819; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]820; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6821; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)822; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo823; GFX1250-GISEL-NEXT: s_clause 0x1824; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off825; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[4:5], off826; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0827; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0828; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0829; GFX1250-GISEL-NEXT: s_branch .LBB11_5830; GFX1250-GISEL-NEXT: .LBB11_5:831;832; GFX950-SDAG-LABEL: flat_xchg_saddr_i64_rtn_neg128:833; GFX950-SDAG: ; %bb.0:834; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v2835; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1836; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0837; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]838; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff80839; GFX950-SDAG-NEXT: s_mov_b32 s3, -1840; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base841; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]842; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v5843; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1844; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc845; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]846; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB11_3847; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow848; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]849; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB11_4850; GFX950-SDAG-NEXT: .LBB11_2: ; %atomicrmw.phi851; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]852; GFX950-SDAG-NEXT: s_branch .LBB11_5853; GFX950-SDAG-NEXT: .LBB11_3: ; %atomicrmw.global854; GFX950-SDAG-NEXT: buffer_wbl2 sc1855; GFX950-SDAG-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5], v[2:3] sc0856; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)857; GFX950-SDAG-NEXT: buffer_inv sc1858; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5859; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3860; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]861; GFX950-SDAG-NEXT: s_cbranch_execz .LBB11_2862; GFX950-SDAG-NEXT: .LBB11_4: ; %atomicrmw.private863; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]864; GFX950-SDAG-NEXT: s_nop 1865; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc866; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off867; GFX950-SDAG-NEXT: s_nop 0868; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off869; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]870; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)871; GFX950-SDAG-NEXT: s_branch .LBB11_5872; GFX950-SDAG-NEXT: .LBB11_5:873;874; GFX950-GISEL-LABEL: flat_xchg_saddr_i64_rtn_neg128:875; GFX950-GISEL: ; %bb.0:876; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v2877; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]878; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v0879; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v1880; GFX950-GISEL-NEXT: s_nop 0881; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc882; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v0883; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base884; GFX950-GISEL-NEXT: s_nop 0885; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v1, vcc886; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v3887; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1888; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc889; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]890; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB11_3891; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow892; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]893; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB11_4894; GFX950-GISEL-NEXT: .LBB11_2: ; %atomicrmw.phi895; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]896; GFX950-GISEL-NEXT: s_branch .LBB11_5897; GFX950-GISEL-NEXT: .LBB11_3: ; %atomicrmw.global898; GFX950-GISEL-NEXT: buffer_wbl2 sc1899; GFX950-GISEL-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3], v[4:5] sc0900; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)901; GFX950-GISEL-NEXT: buffer_inv sc1902; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr3903; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5904; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]905; GFX950-GISEL-NEXT: s_cbranch_execz .LBB11_2906; GFX950-GISEL-NEXT: .LBB11_4: ; %atomicrmw.private907; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]908; GFX950-GISEL-NEXT: s_nop 1909; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc910; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off911; GFX950-GISEL-NEXT: s_nop 0912; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[4:5], off913; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]914; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)915; GFX950-GISEL-NEXT: s_branch .LBB11_5916; GFX950-GISEL-NEXT: .LBB11_5:917 %zext.offset = zext i32 %voffset to i64918 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset919 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128920 %rtn = atomicrmw xchg ptr %gep1, i64 %data syncscope("agent") seq_cst921 %cast.rtn = bitcast i64 %rtn to <2 x float>922 ret <2 x float> %cast.rtn923}924 925define amdgpu_ps void @flat_xchg_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {926; GFX1250-SDAG-LABEL: flat_xchg_saddr_i64_nortn:927; GFX1250-SDAG: ; %bb.0:928; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1929; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1930; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0931; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo932; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)933; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]934; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v1935; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)936; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v4937; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s0938; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB12_3939; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow940; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0941; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB12_4942; GFX1250-SDAG-NEXT: .LBB12_2: ; %atomicrmw.phi943; GFX1250-SDAG-NEXT: s_endpgm944; GFX1250-SDAG-NEXT: .LBB12_3: ; %atomicrmw.global945; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV946; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0947; GFX1250-SDAG-NEXT: flat_atomic_swap_b64 v[0:1], v[2:3] scope:SCOPE_DEV948; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0949; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV950; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1951; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3952; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0953; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s0954; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB12_2955; GFX1250-SDAG-NEXT: .LBB12_4: ; %atomicrmw.private956; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]957; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v0958; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)959; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v0, -1, v4, vcc_lo960; GFX1250-SDAG-NEXT: scratch_store_b64 v0, v[2:3], off961; GFX1250-SDAG-NEXT: s_endpgm962;963; GFX1250-GISEL-LABEL: flat_xchg_saddr_i64_nortn:964; GFX1250-GISEL: ; %bb.0:965; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1966; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2967; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]968; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo969; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)970; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0971; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo972; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)973; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3974; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1975; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0976; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB12_3977; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow978; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0979; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB12_4980; GFX1250-GISEL-NEXT: .LBB12_2: ; %atomicrmw.phi981; GFX1250-GISEL-NEXT: s_endpgm982; GFX1250-GISEL-NEXT: .LBB12_3: ; %atomicrmw.global983; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV984; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0985; GFX1250-GISEL-NEXT: flat_atomic_swap_b64 v0, v[4:5], s[2:3] scope:SCOPE_DEV986; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0987; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV988; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2989; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5990; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0991; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0992; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB12_2993; GFX1250-GISEL-NEXT: .LBB12_4: ; %atomicrmw.private994; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]995; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2996; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)997; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc_lo998; GFX1250-GISEL-NEXT: scratch_store_b64 v0, v[4:5], off999; GFX1250-GISEL-NEXT: s_endpgm1000;1001; GFX950-SDAG-LABEL: flat_xchg_saddr_i64_nortn:1002; GFX950-SDAG: ; %bb.0:1003; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v21004; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v11005; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 01006; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base1007; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]1008; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v11009; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc1010; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]1011; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB12_31012; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow1013; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1014; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB12_41015; GFX950-SDAG-NEXT: .LBB12_2: ; %atomicrmw.phi1016; GFX950-SDAG-NEXT: s_endpgm1017; GFX950-SDAG-NEXT: .LBB12_3: ; %atomicrmw.global1018; GFX950-SDAG-NEXT: buffer_wbl2 sc11019; GFX950-SDAG-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]1020; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1021; GFX950-SDAG-NEXT: buffer_inv sc11022; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr11023; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr31024; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1025; GFX950-SDAG-NEXT: s_cbranch_execz .LBB12_21026; GFX950-SDAG-NEXT: .LBB12_4: ; %atomicrmw.private1027; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]1028; GFX950-SDAG-NEXT: s_nop 11029; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc1030; GFX950-SDAG-NEXT: scratch_store_dwordx2 v0, v[2:3], off1031; GFX950-SDAG-NEXT: s_endpgm1032;1033; GFX950-GISEL-LABEL: flat_xchg_saddr_i64_nortn:1034; GFX950-GISEL: ; %bb.0:1035; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v21036; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]1037; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v01038; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v11039; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base1040; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc1041; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v11042; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc1043; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]1044; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB12_31045; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow1046; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1047; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB12_41048; GFX950-GISEL-NEXT: .LBB12_2: ; %atomicrmw.phi1049; GFX950-GISEL-NEXT: s_endpgm1050; GFX950-GISEL-NEXT: .LBB12_3: ; %atomicrmw.global1051; GFX950-GISEL-NEXT: buffer_wbl2 sc11052; GFX950-GISEL-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5]1053; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1054; GFX950-GISEL-NEXT: buffer_inv sc11055; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr11056; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr51057; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1058; GFX950-GISEL-NEXT: s_cbranch_execz .LBB12_21059; GFX950-GISEL-NEXT: .LBB12_4: ; %atomicrmw.private1060; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]1061; GFX950-GISEL-NEXT: s_nop 11062; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc1063; GFX950-GISEL-NEXT: scratch_store_dwordx2 v0, v[4:5], off1064; GFX950-GISEL-NEXT: s_endpgm1065 %zext.offset = zext i32 %voffset to i641066 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset1067 %unused = atomicrmw xchg ptr %gep0, i64 %data syncscope("agent") seq_cst1068 ret void1069}1070 1071define amdgpu_ps void @flat_xchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {1072; GFX1250-SDAG-LABEL: flat_xchg_saddr_i64_nortn_neg128:1073; GFX1250-SDAG: ; %bb.0:1074; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11075; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v11076; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 01077; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff801078; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1079; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]1080; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]1081; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo1082; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1083; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v11084; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v41085; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s01086; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB13_31087; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow1088; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s01089; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB13_41090; GFX1250-SDAG-NEXT: .LBB13_2: ; %atomicrmw.phi1091; GFX1250-SDAG-NEXT: s_endpgm1092; GFX1250-SDAG-NEXT: .LBB13_3: ; %atomicrmw.global1093; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV1094; GFX1250-SDAG-NEXT: s_wait_storecnt 0x01095; GFX1250-SDAG-NEXT: flat_atomic_swap_b64 v[0:1], v[2:3] scope:SCOPE_DEV1096; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x01097; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV1098; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr11099; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr31100; GFX1250-SDAG-NEXT: s_wait_xcnt 0x01101; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s01102; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB13_21103; GFX1250-SDAG-NEXT: .LBB13_4: ; %atomicrmw.private1104; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]1105; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v01106; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)1107; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v0, -1, v4, vcc_lo1108; GFX1250-SDAG-NEXT: scratch_store_b64 v0, v[2:3], off1109; GFX1250-SDAG-NEXT: s_endpgm1110;1111; GFX1250-GISEL-LABEL: flat_xchg_saddr_i64_nortn_neg128:1112; GFX1250-GISEL: ; %bb.0:1113; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11114; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v21115; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]1116; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo1117; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1118; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v01119; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo1120; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)1121; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v11122; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo1123; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1124; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v31125; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v11126; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s01127; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB13_31128; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow1129; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s01130; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB13_41131; GFX1250-GISEL-NEXT: .LBB13_2: ; %atomicrmw.phi1132; GFX1250-GISEL-NEXT: s_endpgm1133; GFX1250-GISEL-NEXT: .LBB13_3: ; %atomicrmw.global1134; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV1135; GFX1250-GISEL-NEXT: s_wait_storecnt 0x01136; GFX1250-GISEL-NEXT: flat_atomic_swap_b64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV1137; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x01138; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV1139; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr21140; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr51141; GFX1250-GISEL-NEXT: s_wait_xcnt 0x01142; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s01143; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB13_21144; GFX1250-GISEL-NEXT: .LBB13_4: ; %atomicrmw.private1145; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]1146; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v21147; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)1148; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc_lo1149; GFX1250-GISEL-NEXT: scratch_store_b64 v0, v[4:5], off1150; GFX1250-GISEL-NEXT: s_endpgm1151;1152; GFX950-SDAG-LABEL: flat_xchg_saddr_i64_nortn_neg128:1153; GFX950-SDAG: ; %bb.0:1154; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v21155; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v11156; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 01157; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]1158; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff801159; GFX950-SDAG-NEXT: s_mov_b32 s3, -11160; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base1161; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]1162; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v11163; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc1164; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]1165; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB13_31166; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow1167; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1168; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB13_41169; GFX950-SDAG-NEXT: .LBB13_2: ; %atomicrmw.phi1170; GFX950-SDAG-NEXT: s_endpgm1171; GFX950-SDAG-NEXT: .LBB13_3: ; %atomicrmw.global1172; GFX950-SDAG-NEXT: buffer_wbl2 sc11173; GFX950-SDAG-NEXT: flat_atomic_swap_x2 v[0:1], v[2:3]1174; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1175; GFX950-SDAG-NEXT: buffer_inv sc11176; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr11177; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr31178; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1179; GFX950-SDAG-NEXT: s_cbranch_execz .LBB13_21180; GFX950-SDAG-NEXT: .LBB13_4: ; %atomicrmw.private1181; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]1182; GFX950-SDAG-NEXT: s_nop 11183; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc1184; GFX950-SDAG-NEXT: scratch_store_dwordx2 v0, v[2:3], off1185; GFX950-SDAG-NEXT: s_endpgm1186;1187; GFX950-GISEL-LABEL: flat_xchg_saddr_i64_nortn_neg128:1188; GFX950-GISEL: ; %bb.0:1189; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v21190; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]1191; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v01192; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v11193; GFX950-GISEL-NEXT: s_nop 01194; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc1195; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v01196; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base1197; GFX950-GISEL-NEXT: s_nop 01198; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc1199; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v11200; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc1201; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]1202; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB13_31203; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow1204; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1205; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB13_41206; GFX950-GISEL-NEXT: .LBB13_2: ; %atomicrmw.phi1207; GFX950-GISEL-NEXT: s_endpgm1208; GFX950-GISEL-NEXT: .LBB13_3: ; %atomicrmw.global1209; GFX950-GISEL-NEXT: buffer_wbl2 sc11210; GFX950-GISEL-NEXT: flat_atomic_swap_x2 v[0:1], v[4:5]1211; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1212; GFX950-GISEL-NEXT: buffer_inv sc11213; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr11214; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr51215; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1216; GFX950-GISEL-NEXT: s_cbranch_execz .LBB13_21217; GFX950-GISEL-NEXT: .LBB13_4: ; %atomicrmw.private1218; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]1219; GFX950-GISEL-NEXT: s_nop 11220; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc1221; GFX950-GISEL-NEXT: scratch_store_dwordx2 v0, v[4:5], off1222; GFX950-GISEL-NEXT: s_endpgm1223 %zext.offset = zext i32 %voffset to i641224 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset1225 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1281226 %unused = atomicrmw xchg ptr %gep1, i64 %data syncscope("agent") seq_cst1227 ret void1228}1229 1230; --------------------------------------------------------------------------------1231; atomicrmw add1232; --------------------------------------------------------------------------------1233 1234define amdgpu_ps float @flat_add_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {1235; GFX1250-LABEL: flat_add_saddr_i32_rtn:1236; GFX1250: ; %bb.0:1237; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11238; GFX1250-NEXT: global_wb scope:SCOPE_DEV1239; GFX1250-NEXT: s_wait_storecnt 0x01240; GFX1250-NEXT: flat_atomic_add_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV1241; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x01242; GFX1250-NEXT: global_inv scope:SCOPE_DEV1243; GFX1250-NEXT: s_wait_loadcnt 0x01244; GFX1250-NEXT: ; return to shader part epilog1245;1246; GFX950-SDAG-LABEL: flat_add_saddr_i32_rtn:1247; GFX950-SDAG: ; %bb.0:1248; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v11249; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 01250; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]1251; GFX950-SDAG-NEXT: buffer_wbl2 sc11252; GFX950-SDAG-NEXT: flat_atomic_add v0, v[0:1], v2 sc01253; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1254; GFX950-SDAG-NEXT: buffer_inv sc11255; GFX950-SDAG-NEXT: ; return to shader part epilog1256;1257; GFX950-GISEL-LABEL: flat_add_saddr_i32_rtn:1258; GFX950-GISEL: ; %bb.0:1259; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]1260; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v01261; GFX950-GISEL-NEXT: s_nop 11262; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc1263; GFX950-GISEL-NEXT: buffer_wbl2 sc11264; GFX950-GISEL-NEXT: flat_atomic_add v0, v[2:3], v1 sc01265; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1266; GFX950-GISEL-NEXT: buffer_inv sc11267; GFX950-GISEL-NEXT: ; return to shader part epilog1268 %zext.offset = zext i32 %voffset to i641269 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset1270 %rtn = atomicrmw add ptr %gep0, i32 %data syncscope("agent") seq_cst1271 %cast.rtn = bitcast i32 %rtn to float1272 ret float %cast.rtn1273}1274 1275define amdgpu_ps float @flat_add_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {1276; GFX1250-LABEL: flat_add_saddr_i32_rtn_neg128:1277; GFX1250: ; %bb.0:1278; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11279; GFX1250-NEXT: global_wb scope:SCOPE_DEV1280; GFX1250-NEXT: s_wait_storecnt 0x01281; GFX1250-NEXT: flat_atomic_add_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV1282; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x01283; GFX1250-NEXT: global_inv scope:SCOPE_DEV1284; GFX1250-NEXT: s_wait_loadcnt 0x01285; GFX1250-NEXT: ; return to shader part epilog1286;1287; GFX950-SDAG-LABEL: flat_add_saddr_i32_rtn_neg128:1288; GFX950-SDAG: ; %bb.0:1289; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v11290; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 01291; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]1292; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v01293; GFX950-SDAG-NEXT: s_nop 11294; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc1295; GFX950-SDAG-NEXT: buffer_wbl2 sc11296; GFX950-SDAG-NEXT: flat_atomic_add v0, v[0:1], v2 sc01297; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1298; GFX950-SDAG-NEXT: buffer_inv sc11299; GFX950-SDAG-NEXT: ; return to shader part epilog1300;1301; GFX950-GISEL-LABEL: flat_add_saddr_i32_rtn_neg128:1302; GFX950-GISEL: ; %bb.0:1303; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]1304; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v01305; GFX950-GISEL-NEXT: s_nop 11306; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc1307; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v01308; GFX950-GISEL-NEXT: s_nop 11309; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc1310; GFX950-GISEL-NEXT: buffer_wbl2 sc11311; GFX950-GISEL-NEXT: flat_atomic_add v0, v[2:3], v1 sc01312; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1313; GFX950-GISEL-NEXT: buffer_inv sc11314; GFX950-GISEL-NEXT: ; return to shader part epilog1315 %zext.offset = zext i32 %voffset to i641316 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset1317 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1281318 %rtn = atomicrmw add ptr %gep1, i32 %data syncscope("agent") seq_cst1319 %cast.rtn = bitcast i32 %rtn to float1320 ret float %cast.rtn1321}1322 1323define amdgpu_ps void @flat_add_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {1324; GFX1250-LABEL: flat_add_saddr_i32_nortn:1325; GFX1250: ; %bb.0:1326; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11327; GFX1250-NEXT: global_wb scope:SCOPE_DEV1328; GFX1250-NEXT: s_wait_storecnt 0x01329; GFX1250-NEXT: flat_atomic_add_u32 v0, v1, s[2:3] scope:SCOPE_DEV1330; GFX1250-NEXT: s_wait_storecnt_dscnt 0x01331; GFX1250-NEXT: global_inv scope:SCOPE_DEV1332; GFX1250-NEXT: s_endpgm1333;1334; GFX950-SDAG-LABEL: flat_add_saddr_i32_nortn:1335; GFX950-SDAG: ; %bb.0:1336; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v11337; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 01338; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]1339; GFX950-SDAG-NEXT: buffer_wbl2 sc11340; GFX950-SDAG-NEXT: flat_atomic_add v[0:1], v21341; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1342; GFX950-SDAG-NEXT: buffer_inv sc11343; GFX950-SDAG-NEXT: s_endpgm1344;1345; GFX950-GISEL-LABEL: flat_add_saddr_i32_nortn:1346; GFX950-GISEL: ; %bb.0:1347; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]1348; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v01349; GFX950-GISEL-NEXT: s_nop 11350; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc1351; GFX950-GISEL-NEXT: buffer_wbl2 sc11352; GFX950-GISEL-NEXT: flat_atomic_add v[2:3], v11353; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1354; GFX950-GISEL-NEXT: buffer_inv sc11355; GFX950-GISEL-NEXT: s_endpgm1356 %zext.offset = zext i32 %voffset to i641357 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset1358 %unused = atomicrmw add ptr %gep0, i32 %data syncscope("agent") seq_cst1359 ret void1360}1361 1362define amdgpu_ps void @flat_add_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {1363; GFX1250-LABEL: flat_add_saddr_i32_nortn_neg128:1364; GFX1250: ; %bb.0:1365; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11366; GFX1250-NEXT: global_wb scope:SCOPE_DEV1367; GFX1250-NEXT: s_wait_storecnt 0x01368; GFX1250-NEXT: flat_atomic_add_u32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV1369; GFX1250-NEXT: s_wait_storecnt_dscnt 0x01370; GFX1250-NEXT: global_inv scope:SCOPE_DEV1371; GFX1250-NEXT: s_endpgm1372;1373; GFX950-SDAG-LABEL: flat_add_saddr_i32_nortn_neg128:1374; GFX950-SDAG: ; %bb.0:1375; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v11376; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 01377; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]1378; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v01379; GFX950-SDAG-NEXT: s_nop 11380; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc1381; GFX950-SDAG-NEXT: buffer_wbl2 sc11382; GFX950-SDAG-NEXT: flat_atomic_add v[0:1], v21383; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1384; GFX950-SDAG-NEXT: buffer_inv sc11385; GFX950-SDAG-NEXT: s_endpgm1386;1387; GFX950-GISEL-LABEL: flat_add_saddr_i32_nortn_neg128:1388; GFX950-GISEL: ; %bb.0:1389; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]1390; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v01391; GFX950-GISEL-NEXT: s_nop 11392; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc1393; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v01394; GFX950-GISEL-NEXT: s_nop 11395; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc1396; GFX950-GISEL-NEXT: buffer_wbl2 sc11397; GFX950-GISEL-NEXT: flat_atomic_add v[2:3], v11398; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1399; GFX950-GISEL-NEXT: buffer_inv sc11400; GFX950-GISEL-NEXT: s_endpgm1401 %zext.offset = zext i32 %voffset to i641402 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset1403 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1281404 %unused = atomicrmw add ptr %gep1, i32 %data syncscope("agent") seq_cst1405 ret void1406}1407 1408define amdgpu_ps <2 x float> @flat_add_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {1409; GFX1250-SDAG-LABEL: flat_add_saddr_i64_rtn:1410; GFX1250-SDAG: ; %bb.0:1411; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11412; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v11413; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 01414; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1415; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]1416; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v51417; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)1418; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v01419; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr11420; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo1421; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s01422; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB18_31423; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow1424; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s01425; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB18_41426; GFX1250-SDAG-NEXT: .LBB18_2: ; %atomicrmw.phi1427; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s01428; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x01429; GFX1250-SDAG-NEXT: s_branch .LBB18_51430; GFX1250-SDAG-NEXT: .LBB18_3: ; %atomicrmw.global1431; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV1432; GFX1250-SDAG-NEXT: s_wait_storecnt 0x01433; GFX1250-SDAG-NEXT: flat_atomic_add_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV1434; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x01435; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV1436; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr51437; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr31438; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s01439; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB18_21440; GFX1250-SDAG-NEXT: .LBB18_4: ; %atomicrmw.private1441; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]1442; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v41443; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)1444; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo1445; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off1446; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x01447; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[2:3], v[0:1], v[2:3]1448; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off1449; GFX1250-SDAG-NEXT: s_wait_xcnt 0x01450; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s01451; GFX1250-SDAG-NEXT: s_branch .LBB18_51452; GFX1250-SDAG-NEXT: .LBB18_5:1453;1454; GFX1250-GISEL-LABEL: flat_add_saddr_i64_rtn:1455; GFX1250-GISEL: ; %bb.0:1456; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11457; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v11458; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]1459; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1460; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v31461; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo1462; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1463; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x141464; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v01465; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr11466; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo1467; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1468; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s01469; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB18_31470; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow1471; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s01472; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB18_41473; GFX1250-GISEL-NEXT: .LBB18_2: ; %atomicrmw.phi1474; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s01475; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x01476; GFX1250-GISEL-NEXT: s_branch .LBB18_51477; GFX1250-GISEL-NEXT: .LBB18_3: ; %atomicrmw.global1478; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV1479; GFX1250-GISEL-NEXT: s_wait_storecnt 0x01480; GFX1250-GISEL-NEXT: flat_atomic_add_u64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV1481; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x01482; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV1483; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr61484; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr51485; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s01486; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB18_21487; GFX1250-GISEL-NEXT: .LBB18_4: ; %atomicrmw.private1488; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]1489; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v61490; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)1491; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo1492; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off1493; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x01494; GFX1250-GISEL-NEXT: v_add_nc_u64_e32 v[2:3], v[0:1], v[4:5]1495; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off1496; GFX1250-GISEL-NEXT: s_wait_xcnt 0x01497; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s01498; GFX1250-GISEL-NEXT: s_branch .LBB18_51499; GFX1250-GISEL-NEXT: .LBB18_5:1500;1501; GFX950-SDAG-LABEL: flat_add_saddr_i64_rtn:1502; GFX950-SDAG: ; %bb.0:1503; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v21504; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v11505; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 01506; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base1507; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1]1508; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v51509; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr11510; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc1511; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]1512; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB18_31513; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow1514; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1515; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB18_41516; GFX950-SDAG-NEXT: .LBB18_2: ; %atomicrmw.phi1517; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]1518; GFX950-SDAG-NEXT: s_branch .LBB18_51519; GFX950-SDAG-NEXT: .LBB18_3: ; %atomicrmw.global1520; GFX950-SDAG-NEXT: buffer_wbl2 sc11521; GFX950-SDAG-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] sc01522; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1523; GFX950-SDAG-NEXT: buffer_inv sc11524; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr51525; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr31526; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1527; GFX950-SDAG-NEXT: s_cbranch_execz .LBB18_21528; GFX950-SDAG-NEXT: .LBB18_4: ; %atomicrmw.private1529; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]1530; GFX950-SDAG-NEXT: s_nop 11531; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc1532; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off1533; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)1534; GFX950-SDAG-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, v[2:3]1535; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off1536; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]1537; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)1538; GFX950-SDAG-NEXT: s_branch .LBB18_51539; GFX950-SDAG-NEXT: .LBB18_5:1540;1541; GFX950-GISEL-LABEL: flat_add_saddr_i64_rtn:1542; GFX950-GISEL: ; %bb.0:1543; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v21544; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]1545; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v01546; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base1547; GFX950-GISEL-NEXT: s_nop 01548; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc1549; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v11550; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v31551; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr11552; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc1553; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]1554; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB18_31555; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow1556; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1557; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB18_41558; GFX950-GISEL-NEXT: .LBB18_2: ; %atomicrmw.phi1559; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]1560; GFX950-GISEL-NEXT: s_branch .LBB18_51561; GFX950-GISEL-NEXT: .LBB18_3: ; %atomicrmw.global1562; GFX950-GISEL-NEXT: buffer_wbl2 sc11563; GFX950-GISEL-NEXT: flat_atomic_add_x2 v[0:1], v[2:3], v[4:5] sc01564; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1565; GFX950-GISEL-NEXT: buffer_inv sc11566; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr31567; GFX950-GISEL-NEXT: ; implicit-def: $vgpr41568; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1569; GFX950-GISEL-NEXT: s_cbranch_execz .LBB18_21570; GFX950-GISEL-NEXT: .LBB18_4: ; %atomicrmw.private1571; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]1572; GFX950-GISEL-NEXT: s_nop 11573; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc1574; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off1575; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)1576; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v0, v41577; GFX950-GISEL-NEXT: s_nop 11578; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v5, vcc1579; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off1580; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]1581; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)1582; GFX950-GISEL-NEXT: s_branch .LBB18_51583; GFX950-GISEL-NEXT: .LBB18_5:1584 %zext.offset = zext i32 %voffset to i641585 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset1586 %rtn = atomicrmw add ptr %gep0, i64 %data syncscope("agent") seq_cst1587 %cast.rtn = bitcast i64 %rtn to <2 x float>1588 ret <2 x float> %cast.rtn1589}1590 1591define amdgpu_ps <2 x float> @flat_add_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {1592; GFX1250-SDAG-LABEL: flat_add_saddr_i64_rtn_neg128:1593; GFX1250-SDAG: ; %bb.0:1594; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11595; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v11596; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 01597; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff801598; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1599; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]1600; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]1601; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1602; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v51603; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v01604; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr11605; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo1606; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1607; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s01608; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB19_31609; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow1610; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s01611; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB19_41612; GFX1250-SDAG-NEXT: .LBB19_2: ; %atomicrmw.phi1613; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s01614; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x01615; GFX1250-SDAG-NEXT: s_branch .LBB19_51616; GFX1250-SDAG-NEXT: .LBB19_3: ; %atomicrmw.global1617; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV1618; GFX1250-SDAG-NEXT: s_wait_storecnt 0x01619; GFX1250-SDAG-NEXT: flat_atomic_add_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV1620; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x01621; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV1622; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr51623; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr31624; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s01625; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB19_21626; GFX1250-SDAG-NEXT: .LBB19_4: ; %atomicrmw.private1627; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]1628; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v41629; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)1630; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo1631; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off1632; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x01633; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[2:3], v[0:1], v[2:3]1634; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off1635; GFX1250-SDAG-NEXT: s_wait_xcnt 0x01636; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s01637; GFX1250-SDAG-NEXT: s_branch .LBB19_51638; GFX1250-SDAG-NEXT: .LBB19_5:1639;1640; GFX1250-GISEL-LABEL: flat_add_saddr_i64_rtn_neg128:1641; GFX1250-GISEL: ; %bb.0:1642; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11643; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v11644; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]1645; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1646; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v31647; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo1648; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)1649; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v01650; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo1651; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1652; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x141653; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v01654; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr11655; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo1656; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1657; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s01658; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB19_31659; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow1660; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s01661; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB19_41662; GFX1250-GISEL-NEXT: .LBB19_2: ; %atomicrmw.phi1663; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s01664; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x01665; GFX1250-GISEL-NEXT: s_branch .LBB19_51666; GFX1250-GISEL-NEXT: .LBB19_3: ; %atomicrmw.global1667; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV1668; GFX1250-GISEL-NEXT: s_wait_storecnt 0x01669; GFX1250-GISEL-NEXT: flat_atomic_add_u64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV1670; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x01671; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV1672; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr61673; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr51674; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s01675; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB19_21676; GFX1250-GISEL-NEXT: .LBB19_4: ; %atomicrmw.private1677; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]1678; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v61679; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)1680; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo1681; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off1682; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x01683; GFX1250-GISEL-NEXT: v_add_nc_u64_e32 v[2:3], v[0:1], v[4:5]1684; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off1685; GFX1250-GISEL-NEXT: s_wait_xcnt 0x01686; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s01687; GFX1250-GISEL-NEXT: s_branch .LBB19_51688; GFX1250-GISEL-NEXT: .LBB19_5:1689;1690; GFX950-SDAG-LABEL: flat_add_saddr_i64_rtn_neg128:1691; GFX950-SDAG: ; %bb.0:1692; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v21693; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v11694; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 01695; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]1696; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff801697; GFX950-SDAG-NEXT: s_mov_b32 s3, -11698; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base1699; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]1700; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v51701; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr11702; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc1703; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]1704; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB19_31705; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow1706; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1707; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB19_41708; GFX950-SDAG-NEXT: .LBB19_2: ; %atomicrmw.phi1709; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]1710; GFX950-SDAG-NEXT: s_branch .LBB19_51711; GFX950-SDAG-NEXT: .LBB19_3: ; %atomicrmw.global1712; GFX950-SDAG-NEXT: buffer_wbl2 sc11713; GFX950-SDAG-NEXT: flat_atomic_add_x2 v[0:1], v[4:5], v[2:3] sc01714; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1715; GFX950-SDAG-NEXT: buffer_inv sc11716; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr51717; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr31718; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1719; GFX950-SDAG-NEXT: s_cbranch_execz .LBB19_21720; GFX950-SDAG-NEXT: .LBB19_4: ; %atomicrmw.private1721; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]1722; GFX950-SDAG-NEXT: s_nop 11723; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc1724; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off1725; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)1726; GFX950-SDAG-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, v[2:3]1727; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off1728; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]1729; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)1730; GFX950-SDAG-NEXT: s_branch .LBB19_51731; GFX950-SDAG-NEXT: .LBB19_5:1732;1733; GFX950-GISEL-LABEL: flat_add_saddr_i64_rtn_neg128:1734; GFX950-GISEL: ; %bb.0:1735; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v21736; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]1737; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v01738; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v11739; GFX950-GISEL-NEXT: s_nop 01740; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc1741; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v01742; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base1743; GFX950-GISEL-NEXT: s_nop 01744; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v1, vcc1745; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v31746; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr11747; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc1748; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]1749; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB19_31750; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow1751; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1752; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB19_41753; GFX950-GISEL-NEXT: .LBB19_2: ; %atomicrmw.phi1754; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]1755; GFX950-GISEL-NEXT: s_branch .LBB19_51756; GFX950-GISEL-NEXT: .LBB19_3: ; %atomicrmw.global1757; GFX950-GISEL-NEXT: buffer_wbl2 sc11758; GFX950-GISEL-NEXT: flat_atomic_add_x2 v[0:1], v[2:3], v[4:5] sc01759; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1760; GFX950-GISEL-NEXT: buffer_inv sc11761; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr31762; GFX950-GISEL-NEXT: ; implicit-def: $vgpr41763; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1764; GFX950-GISEL-NEXT: s_cbranch_execz .LBB19_21765; GFX950-GISEL-NEXT: .LBB19_4: ; %atomicrmw.private1766; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]1767; GFX950-GISEL-NEXT: s_nop 11768; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc1769; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off1770; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)1771; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v0, v41772; GFX950-GISEL-NEXT: s_nop 11773; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v5, vcc1774; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off1775; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]1776; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)1777; GFX950-GISEL-NEXT: s_branch .LBB19_51778; GFX950-GISEL-NEXT: .LBB19_5:1779 %zext.offset = zext i32 %voffset to i641780 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset1781 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1281782 %rtn = atomicrmw add ptr %gep1, i64 %data syncscope("agent") seq_cst1783 %cast.rtn = bitcast i64 %rtn to <2 x float>1784 ret <2 x float> %cast.rtn1785}1786 1787define amdgpu_ps void @flat_add_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {1788; GFX1250-SDAG-LABEL: flat_add_saddr_i64_nortn:1789; GFX1250-SDAG: ; %bb.0:1790; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11791; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v11792; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 01793; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo1794; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1795; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]1796; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v11797; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)1798; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v41799; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s01800; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB20_31801; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow1802; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s01803; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB20_41804; GFX1250-SDAG-NEXT: .LBB20_2: ; %atomicrmw.phi1805; GFX1250-SDAG-NEXT: s_endpgm1806; GFX1250-SDAG-NEXT: .LBB20_3: ; %atomicrmw.global1807; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV1808; GFX1250-SDAG-NEXT: s_wait_storecnt 0x01809; GFX1250-SDAG-NEXT: flat_atomic_add_u64 v[0:1], v[2:3] scope:SCOPE_DEV1810; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x01811; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV1812; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr11813; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr31814; GFX1250-SDAG-NEXT: s_wait_xcnt 0x01815; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s01816; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB20_21817; GFX1250-SDAG-NEXT: .LBB20_4: ; %atomicrmw.private1818; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]1819; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v01820; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)1821; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo1822; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off1823; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x01824; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[0:1], v[2:3]1825; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off1826; GFX1250-SDAG-NEXT: s_endpgm1827;1828; GFX1250-GISEL-LABEL: flat_add_saddr_i64_nortn:1829; GFX1250-GISEL: ; %bb.0:1830; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11831; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v21832; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]1833; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo1834; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1835; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v01836; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo1837; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1838; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v31839; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v11840; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s01841; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB20_31842; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow1843; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s01844; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB20_41845; GFX1250-GISEL-NEXT: .LBB20_2: ; %atomicrmw.phi1846; GFX1250-GISEL-NEXT: s_endpgm1847; GFX1250-GISEL-NEXT: .LBB20_3: ; %atomicrmw.global1848; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV1849; GFX1250-GISEL-NEXT: s_wait_storecnt 0x01850; GFX1250-GISEL-NEXT: flat_atomic_add_u64 v0, v[4:5], s[2:3] scope:SCOPE_DEV1851; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x01852; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV1853; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr21854; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr51855; GFX1250-GISEL-NEXT: s_wait_xcnt 0x01856; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s01857; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB20_21858; GFX1250-GISEL-NEXT: .LBB20_4: ; %atomicrmw.private1859; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]1860; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v21861; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)1862; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo1863; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off1864; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x01865; GFX1250-GISEL-NEXT: v_add_nc_u64_e32 v[0:1], v[0:1], v[4:5]1866; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off1867; GFX1250-GISEL-NEXT: s_endpgm1868;1869; GFX950-SDAG-LABEL: flat_add_saddr_i64_nortn:1870; GFX950-SDAG: ; %bb.0:1871; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v21872; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v11873; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 01874; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base1875; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]1876; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v11877; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc1878; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]1879; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB20_31880; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow1881; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1882; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB20_41883; GFX950-SDAG-NEXT: .LBB20_2: ; %atomicrmw.phi1884; GFX950-SDAG-NEXT: s_endpgm1885; GFX950-SDAG-NEXT: .LBB20_3: ; %atomicrmw.global1886; GFX950-SDAG-NEXT: buffer_wbl2 sc11887; GFX950-SDAG-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]1888; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1889; GFX950-SDAG-NEXT: buffer_inv sc11890; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr11891; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr31892; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1893; GFX950-SDAG-NEXT: s_cbranch_execz .LBB20_21894; GFX950-SDAG-NEXT: .LBB20_4: ; %atomicrmw.private1895; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]1896; GFX950-SDAG-NEXT: s_nop 11897; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc1898; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off1899; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)1900; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]1901; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[0:1], off1902; GFX950-SDAG-NEXT: s_endpgm1903;1904; GFX950-GISEL-LABEL: flat_add_saddr_i64_nortn:1905; GFX950-GISEL: ; %bb.0:1906; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v21907; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]1908; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v01909; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v11910; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base1911; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc1912; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v11913; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc1914; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]1915; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB20_31916; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow1917; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1918; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB20_41919; GFX950-GISEL-NEXT: .LBB20_2: ; %atomicrmw.phi1920; GFX950-GISEL-NEXT: s_endpgm1921; GFX950-GISEL-NEXT: .LBB20_3: ; %atomicrmw.global1922; GFX950-GISEL-NEXT: buffer_wbl2 sc11923; GFX950-GISEL-NEXT: flat_atomic_add_x2 v[0:1], v[4:5]1924; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1925; GFX950-GISEL-NEXT: buffer_inv sc11926; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr11927; GFX950-GISEL-NEXT: ; implicit-def: $vgpr41928; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]1929; GFX950-GISEL-NEXT: s_cbranch_execz .LBB20_21930; GFX950-GISEL-NEXT: .LBB20_4: ; %atomicrmw.private1931; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]1932; GFX950-GISEL-NEXT: s_nop 11933; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc1934; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off1935; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)1936; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v41937; GFX950-GISEL-NEXT: s_nop 11938; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v5, vcc1939; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off1940; GFX950-GISEL-NEXT: s_endpgm1941 %zext.offset = zext i32 %voffset to i641942 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset1943 %unused = atomicrmw add ptr %gep0, i64 %data syncscope("agent") seq_cst1944 ret void1945}1946 1947define amdgpu_ps void @flat_add_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {1948; GFX1250-SDAG-LABEL: flat_add_saddr_i64_nortn_neg128:1949; GFX1250-SDAG: ; %bb.0:1950; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11951; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v11952; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 01953; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff801954; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1955; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]1956; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]1957; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo1958; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1959; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v11960; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v41961; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s01962; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB21_31963; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow1964; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s01965; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB21_41966; GFX1250-SDAG-NEXT: .LBB21_2: ; %atomicrmw.phi1967; GFX1250-SDAG-NEXT: s_endpgm1968; GFX1250-SDAG-NEXT: .LBB21_3: ; %atomicrmw.global1969; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV1970; GFX1250-SDAG-NEXT: s_wait_storecnt 0x01971; GFX1250-SDAG-NEXT: flat_atomic_add_u64 v[0:1], v[2:3] scope:SCOPE_DEV1972; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x01973; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV1974; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr11975; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr31976; GFX1250-SDAG-NEXT: s_wait_xcnt 0x01977; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s01978; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB21_21979; GFX1250-SDAG-NEXT: .LBB21_4: ; %atomicrmw.private1980; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]1981; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v01982; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)1983; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo1984; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off1985; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x01986; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[0:1], v[2:3]1987; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off1988; GFX1250-SDAG-NEXT: s_endpgm1989;1990; GFX1250-GISEL-LABEL: flat_add_saddr_i64_nortn_neg128:1991; GFX1250-GISEL: ; %bb.0:1992; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 11993; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v21994; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]1995; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo1996; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1997; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v01998; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo1999; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)2000; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v12001; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo2002; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2003; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v32004; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v12005; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s02006; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB21_32007; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow2008; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s02009; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB21_42010; GFX1250-GISEL-NEXT: .LBB21_2: ; %atomicrmw.phi2011; GFX1250-GISEL-NEXT: s_endpgm2012; GFX1250-GISEL-NEXT: .LBB21_3: ; %atomicrmw.global2013; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV2014; GFX1250-GISEL-NEXT: s_wait_storecnt 0x02015; GFX1250-GISEL-NEXT: flat_atomic_add_u64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV2016; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x02017; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV2018; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr22019; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr52020; GFX1250-GISEL-NEXT: s_wait_xcnt 0x02021; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s02022; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB21_22023; GFX1250-GISEL-NEXT: .LBB21_4: ; %atomicrmw.private2024; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]2025; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v22026; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)2027; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo2028; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off2029; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x02030; GFX1250-GISEL-NEXT: v_add_nc_u64_e32 v[0:1], v[0:1], v[4:5]2031; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off2032; GFX1250-GISEL-NEXT: s_endpgm2033;2034; GFX950-SDAG-LABEL: flat_add_saddr_i64_nortn_neg128:2035; GFX950-SDAG: ; %bb.0:2036; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v22037; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v12038; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 02039; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]2040; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff802041; GFX950-SDAG-NEXT: s_mov_b32 s3, -12042; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base2043; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]2044; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v12045; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc2046; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]2047; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB21_32048; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow2049; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]2050; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB21_42051; GFX950-SDAG-NEXT: .LBB21_2: ; %atomicrmw.phi2052; GFX950-SDAG-NEXT: s_endpgm2053; GFX950-SDAG-NEXT: .LBB21_3: ; %atomicrmw.global2054; GFX950-SDAG-NEXT: buffer_wbl2 sc12055; GFX950-SDAG-NEXT: flat_atomic_add_x2 v[0:1], v[2:3]2056; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2057; GFX950-SDAG-NEXT: buffer_inv sc12058; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr12059; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr32060; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]2061; GFX950-SDAG-NEXT: s_cbranch_execz .LBB21_22062; GFX950-SDAG-NEXT: .LBB21_4: ; %atomicrmw.private2063; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]2064; GFX950-SDAG-NEXT: s_nop 12065; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc2066; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off2067; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)2068; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]2069; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[0:1], off2070; GFX950-SDAG-NEXT: s_endpgm2071;2072; GFX950-GISEL-LABEL: flat_add_saddr_i64_nortn_neg128:2073; GFX950-GISEL: ; %bb.0:2074; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v22075; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]2076; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v02077; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v12078; GFX950-GISEL-NEXT: s_nop 02079; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc2080; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v02081; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base2082; GFX950-GISEL-NEXT: s_nop 02083; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc2084; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v12085; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc2086; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]2087; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB21_32088; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow2089; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]2090; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB21_42091; GFX950-GISEL-NEXT: .LBB21_2: ; %atomicrmw.phi2092; GFX950-GISEL-NEXT: s_endpgm2093; GFX950-GISEL-NEXT: .LBB21_3: ; %atomicrmw.global2094; GFX950-GISEL-NEXT: buffer_wbl2 sc12095; GFX950-GISEL-NEXT: flat_atomic_add_x2 v[0:1], v[4:5]2096; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2097; GFX950-GISEL-NEXT: buffer_inv sc12098; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr12099; GFX950-GISEL-NEXT: ; implicit-def: $vgpr42100; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]2101; GFX950-GISEL-NEXT: s_cbranch_execz .LBB21_22102; GFX950-GISEL-NEXT: .LBB21_4: ; %atomicrmw.private2103; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]2104; GFX950-GISEL-NEXT: s_nop 12105; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc2106; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off2107; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)2108; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v42109; GFX950-GISEL-NEXT: s_nop 12110; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v5, vcc2111; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off2112; GFX950-GISEL-NEXT: s_endpgm2113 %zext.offset = zext i32 %voffset to i642114 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset2115 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1282116 %unused = atomicrmw add ptr %gep1, i64 %data syncscope("agent") seq_cst2117 ret void2118}2119 2120; --------------------------------------------------------------------------------2121; atomicrmw sub2122; --------------------------------------------------------------------------------2123 2124define amdgpu_ps float @flat_sub_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {2125; GFX1250-LABEL: flat_sub_saddr_i32_rtn:2126; GFX1250: ; %bb.0:2127; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12128; GFX1250-NEXT: global_wb scope:SCOPE_DEV2129; GFX1250-NEXT: s_wait_storecnt 0x02130; GFX1250-NEXT: flat_atomic_sub_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV2131; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x02132; GFX1250-NEXT: global_inv scope:SCOPE_DEV2133; GFX1250-NEXT: s_wait_loadcnt 0x02134; GFX1250-NEXT: ; return to shader part epilog2135;2136; GFX950-SDAG-LABEL: flat_sub_saddr_i32_rtn:2137; GFX950-SDAG: ; %bb.0:2138; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v12139; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 02140; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]2141; GFX950-SDAG-NEXT: buffer_wbl2 sc12142; GFX950-SDAG-NEXT: flat_atomic_sub v0, v[0:1], v2 sc02143; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2144; GFX950-SDAG-NEXT: buffer_inv sc12145; GFX950-SDAG-NEXT: ; return to shader part epilog2146;2147; GFX950-GISEL-LABEL: flat_sub_saddr_i32_rtn:2148; GFX950-GISEL: ; %bb.0:2149; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]2150; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v02151; GFX950-GISEL-NEXT: s_nop 12152; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc2153; GFX950-GISEL-NEXT: buffer_wbl2 sc12154; GFX950-GISEL-NEXT: flat_atomic_sub v0, v[2:3], v1 sc02155; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2156; GFX950-GISEL-NEXT: buffer_inv sc12157; GFX950-GISEL-NEXT: ; return to shader part epilog2158 %zext.offset = zext i32 %voffset to i642159 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset2160 %rtn = atomicrmw sub ptr %gep0, i32 %data syncscope("agent") seq_cst2161 %cast.rtn = bitcast i32 %rtn to float2162 ret float %cast.rtn2163}2164 2165define amdgpu_ps float @flat_sub_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {2166; GFX1250-LABEL: flat_sub_saddr_i32_rtn_neg128:2167; GFX1250: ; %bb.0:2168; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12169; GFX1250-NEXT: global_wb scope:SCOPE_DEV2170; GFX1250-NEXT: s_wait_storecnt 0x02171; GFX1250-NEXT: flat_atomic_sub_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV2172; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x02173; GFX1250-NEXT: global_inv scope:SCOPE_DEV2174; GFX1250-NEXT: s_wait_loadcnt 0x02175; GFX1250-NEXT: ; return to shader part epilog2176;2177; GFX950-SDAG-LABEL: flat_sub_saddr_i32_rtn_neg128:2178; GFX950-SDAG: ; %bb.0:2179; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v12180; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 02181; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]2182; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v02183; GFX950-SDAG-NEXT: s_nop 12184; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc2185; GFX950-SDAG-NEXT: buffer_wbl2 sc12186; GFX950-SDAG-NEXT: flat_atomic_sub v0, v[0:1], v2 sc02187; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2188; GFX950-SDAG-NEXT: buffer_inv sc12189; GFX950-SDAG-NEXT: ; return to shader part epilog2190;2191; GFX950-GISEL-LABEL: flat_sub_saddr_i32_rtn_neg128:2192; GFX950-GISEL: ; %bb.0:2193; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]2194; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v02195; GFX950-GISEL-NEXT: s_nop 12196; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc2197; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v02198; GFX950-GISEL-NEXT: s_nop 12199; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc2200; GFX950-GISEL-NEXT: buffer_wbl2 sc12201; GFX950-GISEL-NEXT: flat_atomic_sub v0, v[2:3], v1 sc02202; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2203; GFX950-GISEL-NEXT: buffer_inv sc12204; GFX950-GISEL-NEXT: ; return to shader part epilog2205 %zext.offset = zext i32 %voffset to i642206 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset2207 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1282208 %rtn = atomicrmw sub ptr %gep1, i32 %data syncscope("agent") seq_cst2209 %cast.rtn = bitcast i32 %rtn to float2210 ret float %cast.rtn2211}2212 2213define amdgpu_ps void @flat_sub_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {2214; GFX1250-LABEL: flat_sub_saddr_i32_nortn:2215; GFX1250: ; %bb.0:2216; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12217; GFX1250-NEXT: global_wb scope:SCOPE_DEV2218; GFX1250-NEXT: s_wait_storecnt 0x02219; GFX1250-NEXT: flat_atomic_sub_u32 v0, v1, s[2:3] scope:SCOPE_DEV2220; GFX1250-NEXT: s_wait_storecnt_dscnt 0x02221; GFX1250-NEXT: global_inv scope:SCOPE_DEV2222; GFX1250-NEXT: s_endpgm2223;2224; GFX950-SDAG-LABEL: flat_sub_saddr_i32_nortn:2225; GFX950-SDAG: ; %bb.0:2226; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v12227; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 02228; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]2229; GFX950-SDAG-NEXT: buffer_wbl2 sc12230; GFX950-SDAG-NEXT: flat_atomic_sub v[0:1], v22231; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2232; GFX950-SDAG-NEXT: buffer_inv sc12233; GFX950-SDAG-NEXT: s_endpgm2234;2235; GFX950-GISEL-LABEL: flat_sub_saddr_i32_nortn:2236; GFX950-GISEL: ; %bb.0:2237; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]2238; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v02239; GFX950-GISEL-NEXT: s_nop 12240; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc2241; GFX950-GISEL-NEXT: buffer_wbl2 sc12242; GFX950-GISEL-NEXT: flat_atomic_sub v[2:3], v12243; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2244; GFX950-GISEL-NEXT: buffer_inv sc12245; GFX950-GISEL-NEXT: s_endpgm2246 %zext.offset = zext i32 %voffset to i642247 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset2248 %unused = atomicrmw sub ptr %gep0, i32 %data syncscope("agent") seq_cst2249 ret void2250}2251 2252define amdgpu_ps void @flat_sub_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {2253; GFX1250-LABEL: flat_sub_saddr_i32_nortn_neg128:2254; GFX1250: ; %bb.0:2255; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12256; GFX1250-NEXT: global_wb scope:SCOPE_DEV2257; GFX1250-NEXT: s_wait_storecnt 0x02258; GFX1250-NEXT: flat_atomic_sub_u32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV2259; GFX1250-NEXT: s_wait_storecnt_dscnt 0x02260; GFX1250-NEXT: global_inv scope:SCOPE_DEV2261; GFX1250-NEXT: s_endpgm2262;2263; GFX950-SDAG-LABEL: flat_sub_saddr_i32_nortn_neg128:2264; GFX950-SDAG: ; %bb.0:2265; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v12266; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 02267; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]2268; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v02269; GFX950-SDAG-NEXT: s_nop 12270; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc2271; GFX950-SDAG-NEXT: buffer_wbl2 sc12272; GFX950-SDAG-NEXT: flat_atomic_sub v[0:1], v22273; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2274; GFX950-SDAG-NEXT: buffer_inv sc12275; GFX950-SDAG-NEXT: s_endpgm2276;2277; GFX950-GISEL-LABEL: flat_sub_saddr_i32_nortn_neg128:2278; GFX950-GISEL: ; %bb.0:2279; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]2280; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v02281; GFX950-GISEL-NEXT: s_nop 12282; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc2283; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v02284; GFX950-GISEL-NEXT: s_nop 12285; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc2286; GFX950-GISEL-NEXT: buffer_wbl2 sc12287; GFX950-GISEL-NEXT: flat_atomic_sub v[2:3], v12288; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2289; GFX950-GISEL-NEXT: buffer_inv sc12290; GFX950-GISEL-NEXT: s_endpgm2291 %zext.offset = zext i32 %voffset to i642292 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset2293 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1282294 %unused = atomicrmw sub ptr %gep1, i32 %data syncscope("agent") seq_cst2295 ret void2296}2297 2298define amdgpu_ps <2 x float> @flat_sub_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {2299; GFX1250-SDAG-LABEL: flat_sub_saddr_i64_rtn:2300; GFX1250-SDAG: ; %bb.0:2301; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12302; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v12303; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 02304; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2305; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]2306; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v52307; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)2308; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v02309; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr12310; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo2311; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s02312; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB26_32313; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow2314; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s02315; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB26_42316; GFX1250-SDAG-NEXT: .LBB26_2: ; %atomicrmw.phi2317; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s02318; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x02319; GFX1250-SDAG-NEXT: s_branch .LBB26_52320; GFX1250-SDAG-NEXT: .LBB26_3: ; %atomicrmw.global2321; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV2322; GFX1250-SDAG-NEXT: s_wait_storecnt 0x02323; GFX1250-SDAG-NEXT: flat_atomic_sub_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV2324; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x02325; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV2326; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr52327; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr32328; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s02329; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB26_22330; GFX1250-SDAG-NEXT: .LBB26_4: ; %atomicrmw.private2331; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]2332; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v42333; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)2334; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo2335; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off2336; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x02337; GFX1250-SDAG-NEXT: v_sub_nc_u64_e32 v[2:3], v[0:1], v[2:3]2338; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off2339; GFX1250-SDAG-NEXT: s_wait_xcnt 0x02340; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s02341; GFX1250-SDAG-NEXT: s_branch .LBB26_52342; GFX1250-SDAG-NEXT: .LBB26_5:2343;2344; GFX1250-GISEL-LABEL: flat_sub_saddr_i64_rtn:2345; GFX1250-GISEL: ; %bb.0:2346; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12347; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v12348; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]2349; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2350; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v32351; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo2352; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2353; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x142354; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v02355; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr12356; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo2357; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2358; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s02359; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB26_32360; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow2361; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s02362; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB26_42363; GFX1250-GISEL-NEXT: .LBB26_2: ; %atomicrmw.phi2364; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s02365; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x02366; GFX1250-GISEL-NEXT: s_branch .LBB26_52367; GFX1250-GISEL-NEXT: .LBB26_3: ; %atomicrmw.global2368; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV2369; GFX1250-GISEL-NEXT: s_wait_storecnt 0x02370; GFX1250-GISEL-NEXT: flat_atomic_sub_u64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV2371; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x02372; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV2373; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr62374; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr52375; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s02376; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB26_22377; GFX1250-GISEL-NEXT: .LBB26_4: ; %atomicrmw.private2378; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]2379; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v62380; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)2381; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo2382; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off2383; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x02384; GFX1250-GISEL-NEXT: v_sub_nc_u64_e32 v[2:3], v[0:1], v[4:5]2385; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off2386; GFX1250-GISEL-NEXT: s_wait_xcnt 0x02387; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s02388; GFX1250-GISEL-NEXT: s_branch .LBB26_52389; GFX1250-GISEL-NEXT: .LBB26_5:2390;2391; GFX950-SDAG-LABEL: flat_sub_saddr_i64_rtn:2392; GFX950-SDAG: ; %bb.0:2393; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v22394; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v12395; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 02396; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base2397; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1]2398; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v52399; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr12400; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc2401; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]2402; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB26_32403; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow2404; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]2405; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB26_42406; GFX950-SDAG-NEXT: .LBB26_2: ; %atomicrmw.phi2407; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]2408; GFX950-SDAG-NEXT: s_branch .LBB26_52409; GFX950-SDAG-NEXT: .LBB26_3: ; %atomicrmw.global2410; GFX950-SDAG-NEXT: buffer_wbl2 sc12411; GFX950-SDAG-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] sc02412; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2413; GFX950-SDAG-NEXT: buffer_inv sc12414; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr52415; GFX950-SDAG-NEXT: ; implicit-def: $vgpr22416; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]2417; GFX950-SDAG-NEXT: s_cbranch_execz .LBB26_22418; GFX950-SDAG-NEXT: .LBB26_4: ; %atomicrmw.private2419; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]2420; GFX950-SDAG-NEXT: s_nop 12421; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc2422; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off2423; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)2424; GFX950-SDAG-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v22425; GFX950-SDAG-NEXT: s_nop 12426; GFX950-SDAG-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc2427; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off2428; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]2429; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)2430; GFX950-SDAG-NEXT: s_branch .LBB26_52431; GFX950-SDAG-NEXT: .LBB26_5:2432;2433; GFX950-GISEL-LABEL: flat_sub_saddr_i64_rtn:2434; GFX950-GISEL: ; %bb.0:2435; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v22436; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]2437; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v02438; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base2439; GFX950-GISEL-NEXT: s_nop 02440; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc2441; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v12442; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v32443; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr12444; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc2445; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]2446; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB26_32447; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow2448; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]2449; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB26_42450; GFX950-GISEL-NEXT: .LBB26_2: ; %atomicrmw.phi2451; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]2452; GFX950-GISEL-NEXT: s_branch .LBB26_52453; GFX950-GISEL-NEXT: .LBB26_3: ; %atomicrmw.global2454; GFX950-GISEL-NEXT: buffer_wbl2 sc12455; GFX950-GISEL-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3], v[4:5] sc02456; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2457; GFX950-GISEL-NEXT: buffer_inv sc12458; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr32459; GFX950-GISEL-NEXT: ; implicit-def: $vgpr42460; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]2461; GFX950-GISEL-NEXT: s_cbranch_execz .LBB26_22462; GFX950-GISEL-NEXT: .LBB26_4: ; %atomicrmw.private2463; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]2464; GFX950-GISEL-NEXT: s_nop 12465; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc2466; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off2467; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)2468; GFX950-GISEL-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v42469; GFX950-GISEL-NEXT: s_nop 12470; GFX950-GISEL-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v5, vcc2471; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off2472; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]2473; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)2474; GFX950-GISEL-NEXT: s_branch .LBB26_52475; GFX950-GISEL-NEXT: .LBB26_5:2476 %zext.offset = zext i32 %voffset to i642477 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset2478 %rtn = atomicrmw sub ptr %gep0, i64 %data syncscope("agent") seq_cst2479 %cast.rtn = bitcast i64 %rtn to <2 x float>2480 ret <2 x float> %cast.rtn2481}2482 2483define amdgpu_ps <2 x float> @flat_sub_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {2484; GFX1250-SDAG-LABEL: flat_sub_saddr_i64_rtn_neg128:2485; GFX1250-SDAG: ; %bb.0:2486; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12487; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v12488; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 02489; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff802490; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2491; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]2492; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]2493; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2494; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v52495; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v02496; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr12497; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo2498; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2499; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s02500; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB27_32501; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow2502; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s02503; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB27_42504; GFX1250-SDAG-NEXT: .LBB27_2: ; %atomicrmw.phi2505; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s02506; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x02507; GFX1250-SDAG-NEXT: s_branch .LBB27_52508; GFX1250-SDAG-NEXT: .LBB27_3: ; %atomicrmw.global2509; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV2510; GFX1250-SDAG-NEXT: s_wait_storecnt 0x02511; GFX1250-SDAG-NEXT: flat_atomic_sub_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV2512; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x02513; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV2514; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr52515; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr32516; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s02517; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB27_22518; GFX1250-SDAG-NEXT: .LBB27_4: ; %atomicrmw.private2519; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]2520; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v42521; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)2522; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo2523; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off2524; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x02525; GFX1250-SDAG-NEXT: v_sub_nc_u64_e32 v[2:3], v[0:1], v[2:3]2526; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off2527; GFX1250-SDAG-NEXT: s_wait_xcnt 0x02528; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s02529; GFX1250-SDAG-NEXT: s_branch .LBB27_52530; GFX1250-SDAG-NEXT: .LBB27_5:2531;2532; GFX1250-GISEL-LABEL: flat_sub_saddr_i64_rtn_neg128:2533; GFX1250-GISEL: ; %bb.0:2534; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12535; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v12536; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]2537; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2538; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v32539; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo2540; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)2541; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v02542; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo2543; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2544; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x142545; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v02546; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr12547; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo2548; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2549; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s02550; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB27_32551; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow2552; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s02553; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB27_42554; GFX1250-GISEL-NEXT: .LBB27_2: ; %atomicrmw.phi2555; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s02556; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x02557; GFX1250-GISEL-NEXT: s_branch .LBB27_52558; GFX1250-GISEL-NEXT: .LBB27_3: ; %atomicrmw.global2559; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV2560; GFX1250-GISEL-NEXT: s_wait_storecnt 0x02561; GFX1250-GISEL-NEXT: flat_atomic_sub_u64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV2562; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x02563; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV2564; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr62565; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr52566; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s02567; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB27_22568; GFX1250-GISEL-NEXT: .LBB27_4: ; %atomicrmw.private2569; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]2570; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v62571; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)2572; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo2573; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off2574; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x02575; GFX1250-GISEL-NEXT: v_sub_nc_u64_e32 v[2:3], v[0:1], v[4:5]2576; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off2577; GFX1250-GISEL-NEXT: s_wait_xcnt 0x02578; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s02579; GFX1250-GISEL-NEXT: s_branch .LBB27_52580; GFX1250-GISEL-NEXT: .LBB27_5:2581;2582; GFX950-SDAG-LABEL: flat_sub_saddr_i64_rtn_neg128:2583; GFX950-SDAG: ; %bb.0:2584; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v22585; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v12586; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 02587; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]2588; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff802589; GFX950-SDAG-NEXT: s_mov_b32 s3, -12590; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base2591; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]2592; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v52593; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr12594; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc2595; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]2596; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB27_32597; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow2598; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]2599; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB27_42600; GFX950-SDAG-NEXT: .LBB27_2: ; %atomicrmw.phi2601; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]2602; GFX950-SDAG-NEXT: s_branch .LBB27_52603; GFX950-SDAG-NEXT: .LBB27_3: ; %atomicrmw.global2604; GFX950-SDAG-NEXT: buffer_wbl2 sc12605; GFX950-SDAG-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] sc02606; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2607; GFX950-SDAG-NEXT: buffer_inv sc12608; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr52609; GFX950-SDAG-NEXT: ; implicit-def: $vgpr22610; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]2611; GFX950-SDAG-NEXT: s_cbranch_execz .LBB27_22612; GFX950-SDAG-NEXT: .LBB27_4: ; %atomicrmw.private2613; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]2614; GFX950-SDAG-NEXT: s_nop 12615; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc2616; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off2617; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)2618; GFX950-SDAG-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v22619; GFX950-SDAG-NEXT: s_nop 12620; GFX950-SDAG-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v3, vcc2621; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off2622; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]2623; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)2624; GFX950-SDAG-NEXT: s_branch .LBB27_52625; GFX950-SDAG-NEXT: .LBB27_5:2626;2627; GFX950-GISEL-LABEL: flat_sub_saddr_i64_rtn_neg128:2628; GFX950-GISEL: ; %bb.0:2629; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v22630; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]2631; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v02632; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v12633; GFX950-GISEL-NEXT: s_nop 02634; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc2635; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v02636; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base2637; GFX950-GISEL-NEXT: s_nop 02638; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v1, vcc2639; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v32640; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr12641; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc2642; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]2643; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB27_32644; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow2645; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]2646; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB27_42647; GFX950-GISEL-NEXT: .LBB27_2: ; %atomicrmw.phi2648; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]2649; GFX950-GISEL-NEXT: s_branch .LBB27_52650; GFX950-GISEL-NEXT: .LBB27_3: ; %atomicrmw.global2651; GFX950-GISEL-NEXT: buffer_wbl2 sc12652; GFX950-GISEL-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3], v[4:5] sc02653; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2654; GFX950-GISEL-NEXT: buffer_inv sc12655; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr32656; GFX950-GISEL-NEXT: ; implicit-def: $vgpr42657; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]2658; GFX950-GISEL-NEXT: s_cbranch_execz .LBB27_22659; GFX950-GISEL-NEXT: .LBB27_4: ; %atomicrmw.private2660; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]2661; GFX950-GISEL-NEXT: s_nop 12662; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc2663; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off2664; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)2665; GFX950-GISEL-NEXT: v_sub_co_u32_e32 v2, vcc, v0, v42666; GFX950-GISEL-NEXT: s_nop 12667; GFX950-GISEL-NEXT: v_subb_co_u32_e32 v3, vcc, v1, v5, vcc2668; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off2669; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]2670; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)2671; GFX950-GISEL-NEXT: s_branch .LBB27_52672; GFX950-GISEL-NEXT: .LBB27_5:2673 %zext.offset = zext i32 %voffset to i642674 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset2675 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1282676 %rtn = atomicrmw sub ptr %gep1, i64 %data syncscope("agent") seq_cst2677 %cast.rtn = bitcast i64 %rtn to <2 x float>2678 ret <2 x float> %cast.rtn2679}2680 2681define amdgpu_ps void @flat_sub_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {2682; GFX1250-SDAG-LABEL: flat_sub_saddr_i64_nortn:2683; GFX1250-SDAG: ; %bb.0:2684; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12685; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v12686; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 02687; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo2688; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2689; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]2690; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v12691; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)2692; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v42693; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s02694; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB28_32695; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow2696; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s02697; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB28_42698; GFX1250-SDAG-NEXT: .LBB28_2: ; %atomicrmw.phi2699; GFX1250-SDAG-NEXT: s_endpgm2700; GFX1250-SDAG-NEXT: .LBB28_3: ; %atomicrmw.global2701; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV2702; GFX1250-SDAG-NEXT: s_wait_storecnt 0x02703; GFX1250-SDAG-NEXT: flat_atomic_sub_u64 v[0:1], v[2:3] scope:SCOPE_DEV2704; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x02705; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV2706; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr12707; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr32708; GFX1250-SDAG-NEXT: s_wait_xcnt 0x02709; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s02710; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB28_22711; GFX1250-SDAG-NEXT: .LBB28_4: ; %atomicrmw.private2712; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]2713; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v02714; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)2715; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo2716; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off2717; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x02718; GFX1250-SDAG-NEXT: v_sub_nc_u64_e32 v[0:1], v[0:1], v[2:3]2719; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off2720; GFX1250-SDAG-NEXT: s_endpgm2721;2722; GFX1250-GISEL-LABEL: flat_sub_saddr_i64_nortn:2723; GFX1250-GISEL: ; %bb.0:2724; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12725; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v22726; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]2727; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo2728; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2729; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v02730; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo2731; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2732; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v32733; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v12734; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s02735; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB28_32736; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow2737; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s02738; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB28_42739; GFX1250-GISEL-NEXT: .LBB28_2: ; %atomicrmw.phi2740; GFX1250-GISEL-NEXT: s_endpgm2741; GFX1250-GISEL-NEXT: .LBB28_3: ; %atomicrmw.global2742; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV2743; GFX1250-GISEL-NEXT: s_wait_storecnt 0x02744; GFX1250-GISEL-NEXT: flat_atomic_sub_u64 v0, v[4:5], s[2:3] scope:SCOPE_DEV2745; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x02746; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV2747; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr22748; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr52749; GFX1250-GISEL-NEXT: s_wait_xcnt 0x02750; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s02751; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB28_22752; GFX1250-GISEL-NEXT: .LBB28_4: ; %atomicrmw.private2753; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]2754; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v22755; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)2756; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo2757; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off2758; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x02759; GFX1250-GISEL-NEXT: v_sub_nc_u64_e32 v[0:1], v[0:1], v[4:5]2760; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off2761; GFX1250-GISEL-NEXT: s_endpgm2762;2763; GFX950-SDAG-LABEL: flat_sub_saddr_i64_nortn:2764; GFX950-SDAG: ; %bb.0:2765; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v22766; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v12767; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 02768; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base2769; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]2770; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v12771; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc2772; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]2773; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB28_32774; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow2775; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]2776; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB28_42777; GFX950-SDAG-NEXT: .LBB28_2: ; %atomicrmw.phi2778; GFX950-SDAG-NEXT: s_endpgm2779; GFX950-SDAG-NEXT: .LBB28_3: ; %atomicrmw.global2780; GFX950-SDAG-NEXT: buffer_wbl2 sc12781; GFX950-SDAG-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]2782; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2783; GFX950-SDAG-NEXT: buffer_inv sc12784; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr12785; GFX950-SDAG-NEXT: ; implicit-def: $vgpr22786; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]2787; GFX950-SDAG-NEXT: s_cbranch_execz .LBB28_22788; GFX950-SDAG-NEXT: .LBB28_4: ; %atomicrmw.private2789; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]2790; GFX950-SDAG-NEXT: s_nop 12791; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc2792; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off2793; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)2794; GFX950-SDAG-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v22795; GFX950-SDAG-NEXT: s_nop 12796; GFX950-SDAG-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v3, vcc2797; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[0:1], off2798; GFX950-SDAG-NEXT: s_endpgm2799;2800; GFX950-GISEL-LABEL: flat_sub_saddr_i64_nortn:2801; GFX950-GISEL: ; %bb.0:2802; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v22803; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]2804; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v02805; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v12806; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base2807; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc2808; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v12809; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc2810; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]2811; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB28_32812; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow2813; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]2814; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB28_42815; GFX950-GISEL-NEXT: .LBB28_2: ; %atomicrmw.phi2816; GFX950-GISEL-NEXT: s_endpgm2817; GFX950-GISEL-NEXT: .LBB28_3: ; %atomicrmw.global2818; GFX950-GISEL-NEXT: buffer_wbl2 sc12819; GFX950-GISEL-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5]2820; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2821; GFX950-GISEL-NEXT: buffer_inv sc12822; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr12823; GFX950-GISEL-NEXT: ; implicit-def: $vgpr42824; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]2825; GFX950-GISEL-NEXT: s_cbranch_execz .LBB28_22826; GFX950-GISEL-NEXT: .LBB28_4: ; %atomicrmw.private2827; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]2828; GFX950-GISEL-NEXT: s_nop 12829; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc2830; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off2831; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)2832; GFX950-GISEL-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v42833; GFX950-GISEL-NEXT: s_nop 12834; GFX950-GISEL-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v5, vcc2835; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off2836; GFX950-GISEL-NEXT: s_endpgm2837 %zext.offset = zext i32 %voffset to i642838 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset2839 %unused = atomicrmw sub ptr %gep0, i64 %data syncscope("agent") seq_cst2840 ret void2841}2842 2843define amdgpu_ps void @flat_sub_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {2844; GFX1250-SDAG-LABEL: flat_sub_saddr_i64_nortn_neg128:2845; GFX1250-SDAG: ; %bb.0:2846; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12847; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v12848; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 02849; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff802850; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2851; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]2852; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]2853; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo2854; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2855; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v12856; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v42857; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s02858; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB29_32859; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow2860; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s02861; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB29_42862; GFX1250-SDAG-NEXT: .LBB29_2: ; %atomicrmw.phi2863; GFX1250-SDAG-NEXT: s_endpgm2864; GFX1250-SDAG-NEXT: .LBB29_3: ; %atomicrmw.global2865; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV2866; GFX1250-SDAG-NEXT: s_wait_storecnt 0x02867; GFX1250-SDAG-NEXT: flat_atomic_sub_u64 v[0:1], v[2:3] scope:SCOPE_DEV2868; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x02869; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV2870; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr12871; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr32872; GFX1250-SDAG-NEXT: s_wait_xcnt 0x02873; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s02874; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB29_22875; GFX1250-SDAG-NEXT: .LBB29_4: ; %atomicrmw.private2876; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]2877; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v02878; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)2879; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo2880; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off2881; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x02882; GFX1250-SDAG-NEXT: v_sub_nc_u64_e32 v[0:1], v[0:1], v[2:3]2883; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off2884; GFX1250-SDAG-NEXT: s_endpgm2885;2886; GFX1250-GISEL-LABEL: flat_sub_saddr_i64_nortn_neg128:2887; GFX1250-GISEL: ; %bb.0:2888; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 12889; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v22890; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]2891; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo2892; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2893; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v02894; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo2895; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)2896; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v12897; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo2898; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2899; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v32900; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v12901; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s02902; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB29_32903; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow2904; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s02905; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB29_42906; GFX1250-GISEL-NEXT: .LBB29_2: ; %atomicrmw.phi2907; GFX1250-GISEL-NEXT: s_endpgm2908; GFX1250-GISEL-NEXT: .LBB29_3: ; %atomicrmw.global2909; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV2910; GFX1250-GISEL-NEXT: s_wait_storecnt 0x02911; GFX1250-GISEL-NEXT: flat_atomic_sub_u64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV2912; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x02913; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV2914; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr22915; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr52916; GFX1250-GISEL-NEXT: s_wait_xcnt 0x02917; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s02918; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB29_22919; GFX1250-GISEL-NEXT: .LBB29_4: ; %atomicrmw.private2920; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]2921; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v22922; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)2923; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo2924; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off2925; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x02926; GFX1250-GISEL-NEXT: v_sub_nc_u64_e32 v[0:1], v[0:1], v[4:5]2927; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off2928; GFX1250-GISEL-NEXT: s_endpgm2929;2930; GFX950-SDAG-LABEL: flat_sub_saddr_i64_nortn_neg128:2931; GFX950-SDAG: ; %bb.0:2932; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v22933; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v12934; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 02935; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]2936; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff802937; GFX950-SDAG-NEXT: s_mov_b32 s3, -12938; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base2939; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]2940; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v12941; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc2942; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]2943; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB29_32944; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow2945; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]2946; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB29_42947; GFX950-SDAG-NEXT: .LBB29_2: ; %atomicrmw.phi2948; GFX950-SDAG-NEXT: s_endpgm2949; GFX950-SDAG-NEXT: .LBB29_3: ; %atomicrmw.global2950; GFX950-SDAG-NEXT: buffer_wbl2 sc12951; GFX950-SDAG-NEXT: flat_atomic_sub_x2 v[0:1], v[2:3]2952; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2953; GFX950-SDAG-NEXT: buffer_inv sc12954; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr12955; GFX950-SDAG-NEXT: ; implicit-def: $vgpr22956; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]2957; GFX950-SDAG-NEXT: s_cbranch_execz .LBB29_22958; GFX950-SDAG-NEXT: .LBB29_4: ; %atomicrmw.private2959; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]2960; GFX950-SDAG-NEXT: s_nop 12961; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc2962; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off2963; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)2964; GFX950-SDAG-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v22965; GFX950-SDAG-NEXT: s_nop 12966; GFX950-SDAG-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v3, vcc2967; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[0:1], off2968; GFX950-SDAG-NEXT: s_endpgm2969;2970; GFX950-GISEL-LABEL: flat_sub_saddr_i64_nortn_neg128:2971; GFX950-GISEL: ; %bb.0:2972; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v22973; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]2974; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v02975; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v12976; GFX950-GISEL-NEXT: s_nop 02977; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc2978; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v02979; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base2980; GFX950-GISEL-NEXT: s_nop 02981; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc2982; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v12983; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc2984; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]2985; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB29_32986; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow2987; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]2988; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB29_42989; GFX950-GISEL-NEXT: .LBB29_2: ; %atomicrmw.phi2990; GFX950-GISEL-NEXT: s_endpgm2991; GFX950-GISEL-NEXT: .LBB29_3: ; %atomicrmw.global2992; GFX950-GISEL-NEXT: buffer_wbl2 sc12993; GFX950-GISEL-NEXT: flat_atomic_sub_x2 v[0:1], v[4:5]2994; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)2995; GFX950-GISEL-NEXT: buffer_inv sc12996; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr12997; GFX950-GISEL-NEXT: ; implicit-def: $vgpr42998; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]2999; GFX950-GISEL-NEXT: s_cbranch_execz .LBB29_23000; GFX950-GISEL-NEXT: .LBB29_4: ; %atomicrmw.private3001; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]3002; GFX950-GISEL-NEXT: s_nop 13003; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc3004; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off3005; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)3006; GFX950-GISEL-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v43007; GFX950-GISEL-NEXT: s_nop 13008; GFX950-GISEL-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v5, vcc3009; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off3010; GFX950-GISEL-NEXT: s_endpgm3011 %zext.offset = zext i32 %voffset to i643012 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset3013 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1283014 %unused = atomicrmw sub ptr %gep1, i64 %data syncscope("agent") seq_cst3015 ret void3016}3017 3018; --------------------------------------------------------------------------------3019; atomicrmw and3020; --------------------------------------------------------------------------------3021 3022define amdgpu_ps float @flat_and_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {3023; GFX1250-LABEL: flat_and_saddr_i32_rtn:3024; GFX1250: ; %bb.0:3025; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 13026; GFX1250-NEXT: global_wb scope:SCOPE_DEV3027; GFX1250-NEXT: s_wait_storecnt 0x03028; GFX1250-NEXT: flat_atomic_and_b32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV3029; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x03030; GFX1250-NEXT: global_inv scope:SCOPE_DEV3031; GFX1250-NEXT: s_wait_loadcnt 0x03032; GFX1250-NEXT: ; return to shader part epilog3033;3034; GFX950-SDAG-LABEL: flat_and_saddr_i32_rtn:3035; GFX950-SDAG: ; %bb.0:3036; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v13037; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 03038; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]3039; GFX950-SDAG-NEXT: buffer_wbl2 sc13040; GFX950-SDAG-NEXT: flat_atomic_and v0, v[0:1], v2 sc03041; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3042; GFX950-SDAG-NEXT: buffer_inv sc13043; GFX950-SDAG-NEXT: ; return to shader part epilog3044;3045; GFX950-GISEL-LABEL: flat_and_saddr_i32_rtn:3046; GFX950-GISEL: ; %bb.0:3047; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]3048; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v03049; GFX950-GISEL-NEXT: s_nop 13050; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc3051; GFX950-GISEL-NEXT: buffer_wbl2 sc13052; GFX950-GISEL-NEXT: flat_atomic_and v0, v[2:3], v1 sc03053; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3054; GFX950-GISEL-NEXT: buffer_inv sc13055; GFX950-GISEL-NEXT: ; return to shader part epilog3056 %zext.offset = zext i32 %voffset to i643057 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset3058 %rtn = atomicrmw and ptr %gep0, i32 %data syncscope("agent") seq_cst3059 %cast.rtn = bitcast i32 %rtn to float3060 ret float %cast.rtn3061}3062 3063define amdgpu_ps float @flat_and_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {3064; GFX1250-LABEL: flat_and_saddr_i32_rtn_neg128:3065; GFX1250: ; %bb.0:3066; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 13067; GFX1250-NEXT: global_wb scope:SCOPE_DEV3068; GFX1250-NEXT: s_wait_storecnt 0x03069; GFX1250-NEXT: flat_atomic_and_b32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV3070; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x03071; GFX1250-NEXT: global_inv scope:SCOPE_DEV3072; GFX1250-NEXT: s_wait_loadcnt 0x03073; GFX1250-NEXT: ; return to shader part epilog3074;3075; GFX950-SDAG-LABEL: flat_and_saddr_i32_rtn_neg128:3076; GFX950-SDAG: ; %bb.0:3077; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v13078; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 03079; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]3080; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v03081; GFX950-SDAG-NEXT: s_nop 13082; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc3083; GFX950-SDAG-NEXT: buffer_wbl2 sc13084; GFX950-SDAG-NEXT: flat_atomic_and v0, v[0:1], v2 sc03085; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3086; GFX950-SDAG-NEXT: buffer_inv sc13087; GFX950-SDAG-NEXT: ; return to shader part epilog3088;3089; GFX950-GISEL-LABEL: flat_and_saddr_i32_rtn_neg128:3090; GFX950-GISEL: ; %bb.0:3091; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]3092; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v03093; GFX950-GISEL-NEXT: s_nop 13094; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc3095; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v03096; GFX950-GISEL-NEXT: s_nop 13097; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc3098; GFX950-GISEL-NEXT: buffer_wbl2 sc13099; GFX950-GISEL-NEXT: flat_atomic_and v0, v[2:3], v1 sc03100; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3101; GFX950-GISEL-NEXT: buffer_inv sc13102; GFX950-GISEL-NEXT: ; return to shader part epilog3103 %zext.offset = zext i32 %voffset to i643104 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset3105 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1283106 %rtn = atomicrmw and ptr %gep1, i32 %data syncscope("agent") seq_cst3107 %cast.rtn = bitcast i32 %rtn to float3108 ret float %cast.rtn3109}3110 3111define amdgpu_ps void @flat_and_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {3112; GFX1250-LABEL: flat_and_saddr_i32_nortn:3113; GFX1250: ; %bb.0:3114; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 13115; GFX1250-NEXT: global_wb scope:SCOPE_DEV3116; GFX1250-NEXT: s_wait_storecnt 0x03117; GFX1250-NEXT: flat_atomic_and_b32 v0, v1, s[2:3] scope:SCOPE_DEV3118; GFX1250-NEXT: s_wait_storecnt_dscnt 0x03119; GFX1250-NEXT: global_inv scope:SCOPE_DEV3120; GFX1250-NEXT: s_endpgm3121;3122; GFX950-SDAG-LABEL: flat_and_saddr_i32_nortn:3123; GFX950-SDAG: ; %bb.0:3124; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v13125; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 03126; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]3127; GFX950-SDAG-NEXT: buffer_wbl2 sc13128; GFX950-SDAG-NEXT: flat_atomic_and v[0:1], v23129; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3130; GFX950-SDAG-NEXT: buffer_inv sc13131; GFX950-SDAG-NEXT: s_endpgm3132;3133; GFX950-GISEL-LABEL: flat_and_saddr_i32_nortn:3134; GFX950-GISEL: ; %bb.0:3135; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]3136; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v03137; GFX950-GISEL-NEXT: s_nop 13138; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc3139; GFX950-GISEL-NEXT: buffer_wbl2 sc13140; GFX950-GISEL-NEXT: flat_atomic_and v[2:3], v13141; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3142; GFX950-GISEL-NEXT: buffer_inv sc13143; GFX950-GISEL-NEXT: s_endpgm3144 %zext.offset = zext i32 %voffset to i643145 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset3146 %unused = atomicrmw and ptr %gep0, i32 %data syncscope("agent") seq_cst3147 ret void3148}3149 3150define amdgpu_ps void @flat_and_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {3151; GFX1250-LABEL: flat_and_saddr_i32_nortn_neg128:3152; GFX1250: ; %bb.0:3153; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 13154; GFX1250-NEXT: global_wb scope:SCOPE_DEV3155; GFX1250-NEXT: s_wait_storecnt 0x03156; GFX1250-NEXT: flat_atomic_and_b32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV3157; GFX1250-NEXT: s_wait_storecnt_dscnt 0x03158; GFX1250-NEXT: global_inv scope:SCOPE_DEV3159; GFX1250-NEXT: s_endpgm3160;3161; GFX950-SDAG-LABEL: flat_and_saddr_i32_nortn_neg128:3162; GFX950-SDAG: ; %bb.0:3163; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v13164; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 03165; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]3166; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v03167; GFX950-SDAG-NEXT: s_nop 13168; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc3169; GFX950-SDAG-NEXT: buffer_wbl2 sc13170; GFX950-SDAG-NEXT: flat_atomic_and v[0:1], v23171; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3172; GFX950-SDAG-NEXT: buffer_inv sc13173; GFX950-SDAG-NEXT: s_endpgm3174;3175; GFX950-GISEL-LABEL: flat_and_saddr_i32_nortn_neg128:3176; GFX950-GISEL: ; %bb.0:3177; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]3178; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v03179; GFX950-GISEL-NEXT: s_nop 13180; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc3181; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v03182; GFX950-GISEL-NEXT: s_nop 13183; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc3184; GFX950-GISEL-NEXT: buffer_wbl2 sc13185; GFX950-GISEL-NEXT: flat_atomic_and v[2:3], v13186; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3187; GFX950-GISEL-NEXT: buffer_inv sc13188; GFX950-GISEL-NEXT: s_endpgm3189 %zext.offset = zext i32 %voffset to i643190 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset3191 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1283192 %unused = atomicrmw and ptr %gep1, i32 %data syncscope("agent") seq_cst3193 ret void3194}3195 3196define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {3197; GFX1250-SDAG-LABEL: flat_and_saddr_i64_rtn:3198; GFX1250-SDAG: ; %bb.0:3199; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 13200; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v13201; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 03202; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3203; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]3204; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v53205; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)3206; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v03207; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr13208; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo3209; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s03210; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB34_33211; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow3212; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s03213; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB34_43214; GFX1250-SDAG-NEXT: .LBB34_2: ; %atomicrmw.phi3215; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s03216; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x03217; GFX1250-SDAG-NEXT: s_branch .LBB34_53218; GFX1250-SDAG-NEXT: .LBB34_3: ; %atomicrmw.global3219; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV3220; GFX1250-SDAG-NEXT: s_wait_storecnt 0x03221; GFX1250-SDAG-NEXT: flat_atomic_and_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV3222; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x03223; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV3224; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr53225; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr33226; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s03227; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB34_23228; GFX1250-SDAG-NEXT: .LBB34_4: ; %atomicrmw.private3229; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]3230; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v43231; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)3232; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo3233; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off3234; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x03235; GFX1250-SDAG-NEXT: v_and_b32_e32 v3, v1, v33236; GFX1250-SDAG-NEXT: v_and_b32_e32 v2, v0, v23237; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off3238; GFX1250-SDAG-NEXT: s_wait_xcnt 0x03239; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s03240; GFX1250-SDAG-NEXT: s_branch .LBB34_53241; GFX1250-SDAG-NEXT: .LBB34_5:3242;3243; GFX1250-GISEL-LABEL: flat_and_saddr_i64_rtn:3244; GFX1250-GISEL: ; %bb.0:3245; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 13246; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v13247; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]3248; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3249; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v33250; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo3251; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3252; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x143253; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v03254; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr13255; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo3256; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)3257; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s03258; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB34_33259; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow3260; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s03261; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB34_43262; GFX1250-GISEL-NEXT: .LBB34_2: ; %atomicrmw.phi3263; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s03264; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x03265; GFX1250-GISEL-NEXT: s_branch .LBB34_53266; GFX1250-GISEL-NEXT: .LBB34_3: ; %atomicrmw.global3267; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV3268; GFX1250-GISEL-NEXT: s_wait_storecnt 0x03269; GFX1250-GISEL-NEXT: flat_atomic_and_b64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV3270; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x03271; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV3272; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr63273; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr43274; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s03275; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB34_23276; GFX1250-GISEL-NEXT: .LBB34_4: ; %atomicrmw.private3277; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]3278; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v63279; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)3280; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo3281; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off3282; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x03283; GFX1250-GISEL-NEXT: v_and_b32_e32 v2, v0, v43284; GFX1250-GISEL-NEXT: v_and_b32_e32 v3, v1, v53285; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off3286; GFX1250-GISEL-NEXT: s_wait_xcnt 0x03287; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s03288; GFX1250-GISEL-NEXT: s_branch .LBB34_53289; GFX1250-GISEL-NEXT: .LBB34_5:3290;3291; GFX950-SDAG-LABEL: flat_and_saddr_i64_rtn:3292; GFX950-SDAG: ; %bb.0:3293; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v23294; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v13295; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 03296; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base3297; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1]3298; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v53299; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr13300; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc3301; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]3302; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB34_33303; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow3304; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3305; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB34_43306; GFX950-SDAG-NEXT: .LBB34_2: ; %atomicrmw.phi3307; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]3308; GFX950-SDAG-NEXT: s_branch .LBB34_53309; GFX950-SDAG-NEXT: .LBB34_3: ; %atomicrmw.global3310; GFX950-SDAG-NEXT: buffer_wbl2 sc13311; GFX950-SDAG-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] sc03312; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3313; GFX950-SDAG-NEXT: buffer_inv sc13314; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr53315; GFX950-SDAG-NEXT: ; implicit-def: $vgpr33316; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3317; GFX950-SDAG-NEXT: s_cbranch_execz .LBB34_23318; GFX950-SDAG-NEXT: .LBB34_4: ; %atomicrmw.private3319; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]3320; GFX950-SDAG-NEXT: s_nop 13321; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc3322; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off3323; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)3324; GFX950-SDAG-NEXT: v_and_b32_e32 v3, v1, v33325; GFX950-SDAG-NEXT: v_and_b32_e32 v2, v0, v23326; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off3327; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]3328; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)3329; GFX950-SDAG-NEXT: s_branch .LBB34_53330; GFX950-SDAG-NEXT: .LBB34_5:3331;3332; GFX950-GISEL-LABEL: flat_and_saddr_i64_rtn:3333; GFX950-GISEL: ; %bb.0:3334; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v23335; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]3336; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v03337; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base3338; GFX950-GISEL-NEXT: s_nop 03339; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc3340; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v13341; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v33342; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr13343; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc3344; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]3345; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB34_33346; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow3347; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3348; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB34_43349; GFX950-GISEL-NEXT: .LBB34_2: ; %atomicrmw.phi3350; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]3351; GFX950-GISEL-NEXT: s_branch .LBB34_53352; GFX950-GISEL-NEXT: .LBB34_3: ; %atomicrmw.global3353; GFX950-GISEL-NEXT: buffer_wbl2 sc13354; GFX950-GISEL-NEXT: flat_atomic_and_x2 v[0:1], v[2:3], v[4:5] sc03355; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3356; GFX950-GISEL-NEXT: buffer_inv sc13357; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr33358; GFX950-GISEL-NEXT: ; implicit-def: $vgpr43359; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3360; GFX950-GISEL-NEXT: s_cbranch_execz .LBB34_23361; GFX950-GISEL-NEXT: .LBB34_4: ; %atomicrmw.private3362; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]3363; GFX950-GISEL-NEXT: s_nop 13364; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc3365; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off3366; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)3367; GFX950-GISEL-NEXT: v_and_b32_e32 v2, v0, v43368; GFX950-GISEL-NEXT: v_and_b32_e32 v3, v1, v53369; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off3370; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]3371; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)3372; GFX950-GISEL-NEXT: s_branch .LBB34_53373; GFX950-GISEL-NEXT: .LBB34_5:3374 %zext.offset = zext i32 %voffset to i643375 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset3376 %rtn = atomicrmw and ptr %gep0, i64 %data syncscope("agent") seq_cst3377 %cast.rtn = bitcast i64 %rtn to <2 x float>3378 ret <2 x float> %cast.rtn3379}3380 3381define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {3382; GFX1250-SDAG-LABEL: flat_and_saddr_i64_rtn_neg128:3383; GFX1250-SDAG: ; %bb.0:3384; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 13385; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v13386; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 03387; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff803388; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3389; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]3390; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]3391; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3392; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v53393; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v03394; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr13395; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo3396; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)3397; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s03398; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB35_33399; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow3400; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s03401; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB35_43402; GFX1250-SDAG-NEXT: .LBB35_2: ; %atomicrmw.phi3403; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s03404; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x03405; GFX1250-SDAG-NEXT: s_branch .LBB35_53406; GFX1250-SDAG-NEXT: .LBB35_3: ; %atomicrmw.global3407; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV3408; GFX1250-SDAG-NEXT: s_wait_storecnt 0x03409; GFX1250-SDAG-NEXT: flat_atomic_and_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV3410; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x03411; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV3412; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr53413; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr33414; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s03415; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB35_23416; GFX1250-SDAG-NEXT: .LBB35_4: ; %atomicrmw.private3417; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]3418; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v43419; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)3420; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo3421; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off3422; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x03423; GFX1250-SDAG-NEXT: v_and_b32_e32 v3, v1, v33424; GFX1250-SDAG-NEXT: v_and_b32_e32 v2, v0, v23425; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off3426; GFX1250-SDAG-NEXT: s_wait_xcnt 0x03427; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s03428; GFX1250-SDAG-NEXT: s_branch .LBB35_53429; GFX1250-SDAG-NEXT: .LBB35_5:3430;3431; GFX1250-GISEL-LABEL: flat_and_saddr_i64_rtn_neg128:3432; GFX1250-GISEL: ; %bb.0:3433; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 13434; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v13435; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]3436; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3437; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v33438; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo3439; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)3440; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v03441; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo3442; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3443; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x143444; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v03445; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr13446; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo3447; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)3448; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s03449; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB35_33450; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow3451; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s03452; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB35_43453; GFX1250-GISEL-NEXT: .LBB35_2: ; %atomicrmw.phi3454; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s03455; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x03456; GFX1250-GISEL-NEXT: s_branch .LBB35_53457; GFX1250-GISEL-NEXT: .LBB35_3: ; %atomicrmw.global3458; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV3459; GFX1250-GISEL-NEXT: s_wait_storecnt 0x03460; GFX1250-GISEL-NEXT: flat_atomic_and_b64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV3461; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x03462; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV3463; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr63464; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr43465; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s03466; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB35_23467; GFX1250-GISEL-NEXT: .LBB35_4: ; %atomicrmw.private3468; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]3469; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v63470; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)3471; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo3472; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off3473; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x03474; GFX1250-GISEL-NEXT: v_and_b32_e32 v2, v0, v43475; GFX1250-GISEL-NEXT: v_and_b32_e32 v3, v1, v53476; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off3477; GFX1250-GISEL-NEXT: s_wait_xcnt 0x03478; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s03479; GFX1250-GISEL-NEXT: s_branch .LBB35_53480; GFX1250-GISEL-NEXT: .LBB35_5:3481;3482; GFX950-SDAG-LABEL: flat_and_saddr_i64_rtn_neg128:3483; GFX950-SDAG: ; %bb.0:3484; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v23485; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v13486; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 03487; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]3488; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff803489; GFX950-SDAG-NEXT: s_mov_b32 s3, -13490; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base3491; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]3492; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v53493; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr13494; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc3495; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]3496; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB35_33497; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow3498; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3499; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB35_43500; GFX950-SDAG-NEXT: .LBB35_2: ; %atomicrmw.phi3501; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]3502; GFX950-SDAG-NEXT: s_branch .LBB35_53503; GFX950-SDAG-NEXT: .LBB35_3: ; %atomicrmw.global3504; GFX950-SDAG-NEXT: buffer_wbl2 sc13505; GFX950-SDAG-NEXT: flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] sc03506; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3507; GFX950-SDAG-NEXT: buffer_inv sc13508; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr53509; GFX950-SDAG-NEXT: ; implicit-def: $vgpr33510; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3511; GFX950-SDAG-NEXT: s_cbranch_execz .LBB35_23512; GFX950-SDAG-NEXT: .LBB35_4: ; %atomicrmw.private3513; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]3514; GFX950-SDAG-NEXT: s_nop 13515; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc3516; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off3517; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)3518; GFX950-SDAG-NEXT: v_and_b32_e32 v3, v1, v33519; GFX950-SDAG-NEXT: v_and_b32_e32 v2, v0, v23520; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off3521; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]3522; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)3523; GFX950-SDAG-NEXT: s_branch .LBB35_53524; GFX950-SDAG-NEXT: .LBB35_5:3525;3526; GFX950-GISEL-LABEL: flat_and_saddr_i64_rtn_neg128:3527; GFX950-GISEL: ; %bb.0:3528; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v23529; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]3530; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v03531; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v13532; GFX950-GISEL-NEXT: s_nop 03533; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc3534; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v03535; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base3536; GFX950-GISEL-NEXT: s_nop 03537; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v1, vcc3538; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v33539; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr13540; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc3541; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]3542; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB35_33543; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow3544; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3545; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB35_43546; GFX950-GISEL-NEXT: .LBB35_2: ; %atomicrmw.phi3547; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]3548; GFX950-GISEL-NEXT: s_branch .LBB35_53549; GFX950-GISEL-NEXT: .LBB35_3: ; %atomicrmw.global3550; GFX950-GISEL-NEXT: buffer_wbl2 sc13551; GFX950-GISEL-NEXT: flat_atomic_and_x2 v[0:1], v[2:3], v[4:5] sc03552; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3553; GFX950-GISEL-NEXT: buffer_inv sc13554; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr33555; GFX950-GISEL-NEXT: ; implicit-def: $vgpr43556; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3557; GFX950-GISEL-NEXT: s_cbranch_execz .LBB35_23558; GFX950-GISEL-NEXT: .LBB35_4: ; %atomicrmw.private3559; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]3560; GFX950-GISEL-NEXT: s_nop 13561; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc3562; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off3563; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)3564; GFX950-GISEL-NEXT: v_and_b32_e32 v2, v0, v43565; GFX950-GISEL-NEXT: v_and_b32_e32 v3, v1, v53566; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off3567; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]3568; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)3569; GFX950-GISEL-NEXT: s_branch .LBB35_53570; GFX950-GISEL-NEXT: .LBB35_5:3571 %zext.offset = zext i32 %voffset to i643572 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset3573 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1283574 %rtn = atomicrmw and ptr %gep1, i64 %data syncscope("agent") seq_cst3575 %cast.rtn = bitcast i64 %rtn to <2 x float>3576 ret <2 x float> %cast.rtn3577}3578 3579define amdgpu_ps void @flat_and_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {3580; GFX1250-SDAG-LABEL: flat_and_saddr_i64_nortn:3581; GFX1250-SDAG: ; %bb.0:3582; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 13583; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v13584; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 03585; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo3586; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3587; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]3588; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v13589; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)3590; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v43591; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s03592; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB36_33593; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow3594; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s03595; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB36_43596; GFX1250-SDAG-NEXT: .LBB36_2: ; %atomicrmw.phi3597; GFX1250-SDAG-NEXT: s_endpgm3598; GFX1250-SDAG-NEXT: .LBB36_3: ; %atomicrmw.global3599; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV3600; GFX1250-SDAG-NEXT: s_wait_storecnt 0x03601; GFX1250-SDAG-NEXT: flat_atomic_and_b64 v[0:1], v[2:3] scope:SCOPE_DEV3602; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x03603; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV3604; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr13605; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr33606; GFX1250-SDAG-NEXT: s_wait_xcnt 0x03607; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s03608; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB36_23609; GFX1250-SDAG-NEXT: .LBB36_4: ; %atomicrmw.private3610; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]3611; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v03612; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)3613; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo3614; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off3615; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x03616; GFX1250-SDAG-NEXT: v_and_b32_e32 v1, v1, v33617; GFX1250-SDAG-NEXT: v_and_b32_e32 v0, v0, v23618; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off3619; GFX1250-SDAG-NEXT: s_endpgm3620;3621; GFX1250-GISEL-LABEL: flat_and_saddr_i64_nortn:3622; GFX1250-GISEL: ; %bb.0:3623; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 13624; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v23625; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]3626; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo3627; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3628; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v03629; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo3630; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3631; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v33632; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v13633; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s03634; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB36_33635; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow3636; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s03637; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB36_43638; GFX1250-GISEL-NEXT: .LBB36_2: ; %atomicrmw.phi3639; GFX1250-GISEL-NEXT: s_endpgm3640; GFX1250-GISEL-NEXT: .LBB36_3: ; %atomicrmw.global3641; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV3642; GFX1250-GISEL-NEXT: s_wait_storecnt 0x03643; GFX1250-GISEL-NEXT: flat_atomic_and_b64 v0, v[4:5], s[2:3] scope:SCOPE_DEV3644; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x03645; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV3646; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr23647; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr43648; GFX1250-GISEL-NEXT: s_wait_xcnt 0x03649; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s03650; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB36_23651; GFX1250-GISEL-NEXT: .LBB36_4: ; %atomicrmw.private3652; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]3653; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v23654; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)3655; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo3656; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off3657; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x03658; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, v0, v43659; GFX1250-GISEL-NEXT: v_and_b32_e32 v1, v1, v53660; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off3661; GFX1250-GISEL-NEXT: s_endpgm3662;3663; GFX950-SDAG-LABEL: flat_and_saddr_i64_nortn:3664; GFX950-SDAG: ; %bb.0:3665; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v23666; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v13667; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 03668; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base3669; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]3670; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v13671; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc3672; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]3673; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB36_33674; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow3675; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3676; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB36_43677; GFX950-SDAG-NEXT: .LBB36_2: ; %atomicrmw.phi3678; GFX950-SDAG-NEXT: s_endpgm3679; GFX950-SDAG-NEXT: .LBB36_3: ; %atomicrmw.global3680; GFX950-SDAG-NEXT: buffer_wbl2 sc13681; GFX950-SDAG-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]3682; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3683; GFX950-SDAG-NEXT: buffer_inv sc13684; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr13685; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr33686; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3687; GFX950-SDAG-NEXT: s_cbranch_execz .LBB36_23688; GFX950-SDAG-NEXT: .LBB36_4: ; %atomicrmw.private3689; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]3690; GFX950-SDAG-NEXT: s_nop 13691; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc3692; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off3693; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)3694; GFX950-SDAG-NEXT: v_and_b32_e32 v1, v1, v33695; GFX950-SDAG-NEXT: v_and_b32_e32 v0, v0, v23696; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[0:1], off3697; GFX950-SDAG-NEXT: s_endpgm3698;3699; GFX950-GISEL-LABEL: flat_and_saddr_i64_nortn:3700; GFX950-GISEL: ; %bb.0:3701; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v23702; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]3703; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v03704; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v13705; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base3706; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc3707; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v13708; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc3709; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]3710; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB36_33711; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow3712; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3713; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB36_43714; GFX950-GISEL-NEXT: .LBB36_2: ; %atomicrmw.phi3715; GFX950-GISEL-NEXT: s_endpgm3716; GFX950-GISEL-NEXT: .LBB36_3: ; %atomicrmw.global3717; GFX950-GISEL-NEXT: buffer_wbl2 sc13718; GFX950-GISEL-NEXT: flat_atomic_and_x2 v[0:1], v[4:5]3719; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3720; GFX950-GISEL-NEXT: buffer_inv sc13721; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr13722; GFX950-GISEL-NEXT: ; implicit-def: $vgpr43723; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3724; GFX950-GISEL-NEXT: s_cbranch_execz .LBB36_23725; GFX950-GISEL-NEXT: .LBB36_4: ; %atomicrmw.private3726; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]3727; GFX950-GISEL-NEXT: s_nop 13728; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc3729; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off3730; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)3731; GFX950-GISEL-NEXT: v_and_b32_e32 v0, v0, v43732; GFX950-GISEL-NEXT: v_and_b32_e32 v1, v1, v53733; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off3734; GFX950-GISEL-NEXT: s_endpgm3735 %zext.offset = zext i32 %voffset to i643736 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset3737 %unused = atomicrmw and ptr %gep0, i64 %data syncscope("agent") seq_cst3738 ret void3739}3740 3741define amdgpu_ps void @flat_and_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {3742; GFX1250-SDAG-LABEL: flat_and_saddr_i64_nortn_neg128:3743; GFX1250-SDAG: ; %bb.0:3744; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 13745; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v13746; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 03747; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff803748; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3749; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]3750; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]3751; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo3752; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3753; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v13754; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v43755; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s03756; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB37_33757; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow3758; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s03759; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB37_43760; GFX1250-SDAG-NEXT: .LBB37_2: ; %atomicrmw.phi3761; GFX1250-SDAG-NEXT: s_endpgm3762; GFX1250-SDAG-NEXT: .LBB37_3: ; %atomicrmw.global3763; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV3764; GFX1250-SDAG-NEXT: s_wait_storecnt 0x03765; GFX1250-SDAG-NEXT: flat_atomic_and_b64 v[0:1], v[2:3] scope:SCOPE_DEV3766; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x03767; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV3768; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr13769; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr33770; GFX1250-SDAG-NEXT: s_wait_xcnt 0x03771; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s03772; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB37_23773; GFX1250-SDAG-NEXT: .LBB37_4: ; %atomicrmw.private3774; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]3775; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v03776; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)3777; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo3778; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off3779; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x03780; GFX1250-SDAG-NEXT: v_and_b32_e32 v1, v1, v33781; GFX1250-SDAG-NEXT: v_and_b32_e32 v0, v0, v23782; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off3783; GFX1250-SDAG-NEXT: s_endpgm3784;3785; GFX1250-GISEL-LABEL: flat_and_saddr_i64_nortn_neg128:3786; GFX1250-GISEL: ; %bb.0:3787; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 13788; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v23789; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]3790; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo3791; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3792; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v03793; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo3794; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)3795; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v13796; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo3797; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3798; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v33799; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v13800; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s03801; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB37_33802; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow3803; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s03804; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB37_43805; GFX1250-GISEL-NEXT: .LBB37_2: ; %atomicrmw.phi3806; GFX1250-GISEL-NEXT: s_endpgm3807; GFX1250-GISEL-NEXT: .LBB37_3: ; %atomicrmw.global3808; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV3809; GFX1250-GISEL-NEXT: s_wait_storecnt 0x03810; GFX1250-GISEL-NEXT: flat_atomic_and_b64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV3811; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x03812; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV3813; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr23814; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr43815; GFX1250-GISEL-NEXT: s_wait_xcnt 0x03816; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s03817; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB37_23818; GFX1250-GISEL-NEXT: .LBB37_4: ; %atomicrmw.private3819; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]3820; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v23821; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)3822; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo3823; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off3824; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x03825; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, v0, v43826; GFX1250-GISEL-NEXT: v_and_b32_e32 v1, v1, v53827; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off3828; GFX1250-GISEL-NEXT: s_endpgm3829;3830; GFX950-SDAG-LABEL: flat_and_saddr_i64_nortn_neg128:3831; GFX950-SDAG: ; %bb.0:3832; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v23833; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v13834; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 03835; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]3836; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff803837; GFX950-SDAG-NEXT: s_mov_b32 s3, -13838; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base3839; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]3840; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v13841; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc3842; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]3843; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB37_33844; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow3845; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3846; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB37_43847; GFX950-SDAG-NEXT: .LBB37_2: ; %atomicrmw.phi3848; GFX950-SDAG-NEXT: s_endpgm3849; GFX950-SDAG-NEXT: .LBB37_3: ; %atomicrmw.global3850; GFX950-SDAG-NEXT: buffer_wbl2 sc13851; GFX950-SDAG-NEXT: flat_atomic_and_x2 v[0:1], v[2:3]3852; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3853; GFX950-SDAG-NEXT: buffer_inv sc13854; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr13855; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr33856; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3857; GFX950-SDAG-NEXT: s_cbranch_execz .LBB37_23858; GFX950-SDAG-NEXT: .LBB37_4: ; %atomicrmw.private3859; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]3860; GFX950-SDAG-NEXT: s_nop 13861; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc3862; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off3863; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)3864; GFX950-SDAG-NEXT: v_and_b32_e32 v1, v1, v33865; GFX950-SDAG-NEXT: v_and_b32_e32 v0, v0, v23866; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[0:1], off3867; GFX950-SDAG-NEXT: s_endpgm3868;3869; GFX950-GISEL-LABEL: flat_and_saddr_i64_nortn_neg128:3870; GFX950-GISEL: ; %bb.0:3871; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v23872; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]3873; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v03874; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v13875; GFX950-GISEL-NEXT: s_nop 03876; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc3877; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v03878; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base3879; GFX950-GISEL-NEXT: s_nop 03880; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc3881; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v13882; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc3883; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]3884; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB37_33885; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow3886; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3887; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB37_43888; GFX950-GISEL-NEXT: .LBB37_2: ; %atomicrmw.phi3889; GFX950-GISEL-NEXT: s_endpgm3890; GFX950-GISEL-NEXT: .LBB37_3: ; %atomicrmw.global3891; GFX950-GISEL-NEXT: buffer_wbl2 sc13892; GFX950-GISEL-NEXT: flat_atomic_and_x2 v[0:1], v[4:5]3893; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3894; GFX950-GISEL-NEXT: buffer_inv sc13895; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr13896; GFX950-GISEL-NEXT: ; implicit-def: $vgpr43897; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]3898; GFX950-GISEL-NEXT: s_cbranch_execz .LBB37_23899; GFX950-GISEL-NEXT: .LBB37_4: ; %atomicrmw.private3900; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]3901; GFX950-GISEL-NEXT: s_nop 13902; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc3903; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off3904; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)3905; GFX950-GISEL-NEXT: v_and_b32_e32 v0, v0, v43906; GFX950-GISEL-NEXT: v_and_b32_e32 v1, v1, v53907; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off3908; GFX950-GISEL-NEXT: s_endpgm3909 %zext.offset = zext i32 %voffset to i643910 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset3911 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1283912 %unused = atomicrmw and ptr %gep1, i64 %data syncscope("agent") seq_cst3913 ret void3914}3915 3916; --------------------------------------------------------------------------------3917; atomicrmw or3918; --------------------------------------------------------------------------------3919 3920define amdgpu_ps float @flat_or_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {3921; GFX1250-LABEL: flat_or_saddr_i32_rtn:3922; GFX1250: ; %bb.0:3923; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 13924; GFX1250-NEXT: global_wb scope:SCOPE_DEV3925; GFX1250-NEXT: s_wait_storecnt 0x03926; GFX1250-NEXT: flat_atomic_or_b32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV3927; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x03928; GFX1250-NEXT: global_inv scope:SCOPE_DEV3929; GFX1250-NEXT: s_wait_loadcnt 0x03930; GFX1250-NEXT: ; return to shader part epilog3931;3932; GFX950-SDAG-LABEL: flat_or_saddr_i32_rtn:3933; GFX950-SDAG: ; %bb.0:3934; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v13935; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 03936; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]3937; GFX950-SDAG-NEXT: buffer_wbl2 sc13938; GFX950-SDAG-NEXT: flat_atomic_or v0, v[0:1], v2 sc03939; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3940; GFX950-SDAG-NEXT: buffer_inv sc13941; GFX950-SDAG-NEXT: ; return to shader part epilog3942;3943; GFX950-GISEL-LABEL: flat_or_saddr_i32_rtn:3944; GFX950-GISEL: ; %bb.0:3945; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]3946; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v03947; GFX950-GISEL-NEXT: s_nop 13948; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc3949; GFX950-GISEL-NEXT: buffer_wbl2 sc13950; GFX950-GISEL-NEXT: flat_atomic_or v0, v[2:3], v1 sc03951; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3952; GFX950-GISEL-NEXT: buffer_inv sc13953; GFX950-GISEL-NEXT: ; return to shader part epilog3954 %zext.offset = zext i32 %voffset to i643955 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset3956 %rtn = atomicrmw or ptr %gep0, i32 %data syncscope("agent") seq_cst3957 %cast.rtn = bitcast i32 %rtn to float3958 ret float %cast.rtn3959}3960 3961define amdgpu_ps float @flat_or_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {3962; GFX1250-LABEL: flat_or_saddr_i32_rtn_neg128:3963; GFX1250: ; %bb.0:3964; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 13965; GFX1250-NEXT: global_wb scope:SCOPE_DEV3966; GFX1250-NEXT: s_wait_storecnt 0x03967; GFX1250-NEXT: flat_atomic_or_b32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV3968; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x03969; GFX1250-NEXT: global_inv scope:SCOPE_DEV3970; GFX1250-NEXT: s_wait_loadcnt 0x03971; GFX1250-NEXT: ; return to shader part epilog3972;3973; GFX950-SDAG-LABEL: flat_or_saddr_i32_rtn_neg128:3974; GFX950-SDAG: ; %bb.0:3975; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v13976; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 03977; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]3978; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v03979; GFX950-SDAG-NEXT: s_nop 13980; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc3981; GFX950-SDAG-NEXT: buffer_wbl2 sc13982; GFX950-SDAG-NEXT: flat_atomic_or v0, v[0:1], v2 sc03983; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3984; GFX950-SDAG-NEXT: buffer_inv sc13985; GFX950-SDAG-NEXT: ; return to shader part epilog3986;3987; GFX950-GISEL-LABEL: flat_or_saddr_i32_rtn_neg128:3988; GFX950-GISEL: ; %bb.0:3989; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]3990; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v03991; GFX950-GISEL-NEXT: s_nop 13992; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc3993; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v03994; GFX950-GISEL-NEXT: s_nop 13995; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc3996; GFX950-GISEL-NEXT: buffer_wbl2 sc13997; GFX950-GISEL-NEXT: flat_atomic_or v0, v[2:3], v1 sc03998; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)3999; GFX950-GISEL-NEXT: buffer_inv sc14000; GFX950-GISEL-NEXT: ; return to shader part epilog4001 %zext.offset = zext i32 %voffset to i644002 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset4003 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1284004 %rtn = atomicrmw or ptr %gep1, i32 %data syncscope("agent") seq_cst4005 %cast.rtn = bitcast i32 %rtn to float4006 ret float %cast.rtn4007}4008 4009define amdgpu_ps void @flat_or_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {4010; GFX1250-LABEL: flat_or_saddr_i32_nortn:4011; GFX1250: ; %bb.0:4012; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 14013; GFX1250-NEXT: global_wb scope:SCOPE_DEV4014; GFX1250-NEXT: s_wait_storecnt 0x04015; GFX1250-NEXT: flat_atomic_or_b32 v0, v1, s[2:3] scope:SCOPE_DEV4016; GFX1250-NEXT: s_wait_storecnt_dscnt 0x04017; GFX1250-NEXT: global_inv scope:SCOPE_DEV4018; GFX1250-NEXT: s_endpgm4019;4020; GFX950-SDAG-LABEL: flat_or_saddr_i32_nortn:4021; GFX950-SDAG: ; %bb.0:4022; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v14023; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 04024; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]4025; GFX950-SDAG-NEXT: buffer_wbl2 sc14026; GFX950-SDAG-NEXT: flat_atomic_or v[0:1], v24027; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4028; GFX950-SDAG-NEXT: buffer_inv sc14029; GFX950-SDAG-NEXT: s_endpgm4030;4031; GFX950-GISEL-LABEL: flat_or_saddr_i32_nortn:4032; GFX950-GISEL: ; %bb.0:4033; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]4034; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v04035; GFX950-GISEL-NEXT: s_nop 14036; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc4037; GFX950-GISEL-NEXT: buffer_wbl2 sc14038; GFX950-GISEL-NEXT: flat_atomic_or v[2:3], v14039; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4040; GFX950-GISEL-NEXT: buffer_inv sc14041; GFX950-GISEL-NEXT: s_endpgm4042 %zext.offset = zext i32 %voffset to i644043 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset4044 %unused = atomicrmw or ptr %gep0, i32 %data syncscope("agent") seq_cst4045 ret void4046}4047 4048define amdgpu_ps void @flat_or_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {4049; GFX1250-LABEL: flat_or_saddr_i32_nortn_neg128:4050; GFX1250: ; %bb.0:4051; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 14052; GFX1250-NEXT: global_wb scope:SCOPE_DEV4053; GFX1250-NEXT: s_wait_storecnt 0x04054; GFX1250-NEXT: flat_atomic_or_b32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV4055; GFX1250-NEXT: s_wait_storecnt_dscnt 0x04056; GFX1250-NEXT: global_inv scope:SCOPE_DEV4057; GFX1250-NEXT: s_endpgm4058;4059; GFX950-SDAG-LABEL: flat_or_saddr_i32_nortn_neg128:4060; GFX950-SDAG: ; %bb.0:4061; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v14062; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 04063; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]4064; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v04065; GFX950-SDAG-NEXT: s_nop 14066; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc4067; GFX950-SDAG-NEXT: buffer_wbl2 sc14068; GFX950-SDAG-NEXT: flat_atomic_or v[0:1], v24069; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4070; GFX950-SDAG-NEXT: buffer_inv sc14071; GFX950-SDAG-NEXT: s_endpgm4072;4073; GFX950-GISEL-LABEL: flat_or_saddr_i32_nortn_neg128:4074; GFX950-GISEL: ; %bb.0:4075; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]4076; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v04077; GFX950-GISEL-NEXT: s_nop 14078; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc4079; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v04080; GFX950-GISEL-NEXT: s_nop 14081; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc4082; GFX950-GISEL-NEXT: buffer_wbl2 sc14083; GFX950-GISEL-NEXT: flat_atomic_or v[2:3], v14084; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4085; GFX950-GISEL-NEXT: buffer_inv sc14086; GFX950-GISEL-NEXT: s_endpgm4087 %zext.offset = zext i32 %voffset to i644088 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset4089 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1284090 %unused = atomicrmw or ptr %gep1, i32 %data syncscope("agent") seq_cst4091 ret void4092}4093 4094define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {4095; GFX1250-SDAG-LABEL: flat_or_saddr_i64_rtn:4096; GFX1250-SDAG: ; %bb.0:4097; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 14098; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v14099; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 04100; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4101; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]4102; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v54103; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)4104; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v04105; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr14106; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo4107; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s04108; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB42_34109; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow4110; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s04111; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB42_44112; GFX1250-SDAG-NEXT: .LBB42_2: ; %atomicrmw.phi4113; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s04114; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x04115; GFX1250-SDAG-NEXT: s_branch .LBB42_54116; GFX1250-SDAG-NEXT: .LBB42_3: ; %atomicrmw.global4117; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV4118; GFX1250-SDAG-NEXT: s_wait_storecnt 0x04119; GFX1250-SDAG-NEXT: flat_atomic_or_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV4120; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x04121; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV4122; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr54123; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr34124; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s04125; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB42_24126; GFX1250-SDAG-NEXT: .LBB42_4: ; %atomicrmw.private4127; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]4128; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v44129; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)4130; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo4131; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off4132; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x04133; GFX1250-SDAG-NEXT: v_or_b32_e32 v3, v1, v34134; GFX1250-SDAG-NEXT: v_or_b32_e32 v2, v0, v24135; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off4136; GFX1250-SDAG-NEXT: s_wait_xcnt 0x04137; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s04138; GFX1250-SDAG-NEXT: s_branch .LBB42_54139; GFX1250-SDAG-NEXT: .LBB42_5:4140;4141; GFX1250-GISEL-LABEL: flat_or_saddr_i64_rtn:4142; GFX1250-GISEL: ; %bb.0:4143; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 14144; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v14145; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]4146; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4147; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v34148; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo4149; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4150; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x144151; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v04152; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr14153; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo4154; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4155; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s04156; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB42_34157; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow4158; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s04159; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB42_44160; GFX1250-GISEL-NEXT: .LBB42_2: ; %atomicrmw.phi4161; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s04162; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x04163; GFX1250-GISEL-NEXT: s_branch .LBB42_54164; GFX1250-GISEL-NEXT: .LBB42_3: ; %atomicrmw.global4165; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV4166; GFX1250-GISEL-NEXT: s_wait_storecnt 0x04167; GFX1250-GISEL-NEXT: flat_atomic_or_b64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV4168; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x04169; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV4170; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr64171; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr44172; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s04173; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB42_24174; GFX1250-GISEL-NEXT: .LBB42_4: ; %atomicrmw.private4175; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]4176; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v64177; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)4178; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo4179; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off4180; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x04181; GFX1250-GISEL-NEXT: v_or_b32_e32 v2, v0, v44182; GFX1250-GISEL-NEXT: v_or_b32_e32 v3, v1, v54183; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off4184; GFX1250-GISEL-NEXT: s_wait_xcnt 0x04185; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s04186; GFX1250-GISEL-NEXT: s_branch .LBB42_54187; GFX1250-GISEL-NEXT: .LBB42_5:4188;4189; GFX950-SDAG-LABEL: flat_or_saddr_i64_rtn:4190; GFX950-SDAG: ; %bb.0:4191; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v24192; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v14193; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 04194; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base4195; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1]4196; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v54197; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr14198; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc4199; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]4200; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB42_34201; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow4202; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]4203; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB42_44204; GFX950-SDAG-NEXT: .LBB42_2: ; %atomicrmw.phi4205; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]4206; GFX950-SDAG-NEXT: s_branch .LBB42_54207; GFX950-SDAG-NEXT: .LBB42_3: ; %atomicrmw.global4208; GFX950-SDAG-NEXT: buffer_wbl2 sc14209; GFX950-SDAG-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] sc04210; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4211; GFX950-SDAG-NEXT: buffer_inv sc14212; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr54213; GFX950-SDAG-NEXT: ; implicit-def: $vgpr34214; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]4215; GFX950-SDAG-NEXT: s_cbranch_execz .LBB42_24216; GFX950-SDAG-NEXT: .LBB42_4: ; %atomicrmw.private4217; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]4218; GFX950-SDAG-NEXT: s_nop 14219; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc4220; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off4221; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)4222; GFX950-SDAG-NEXT: v_or_b32_e32 v3, v1, v34223; GFX950-SDAG-NEXT: v_or_b32_e32 v2, v0, v24224; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off4225; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]4226; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)4227; GFX950-SDAG-NEXT: s_branch .LBB42_54228; GFX950-SDAG-NEXT: .LBB42_5:4229;4230; GFX950-GISEL-LABEL: flat_or_saddr_i64_rtn:4231; GFX950-GISEL: ; %bb.0:4232; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v24233; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]4234; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v04235; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base4236; GFX950-GISEL-NEXT: s_nop 04237; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc4238; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v14239; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v34240; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr14241; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc4242; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]4243; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB42_34244; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow4245; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]4246; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB42_44247; GFX950-GISEL-NEXT: .LBB42_2: ; %atomicrmw.phi4248; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]4249; GFX950-GISEL-NEXT: s_branch .LBB42_54250; GFX950-GISEL-NEXT: .LBB42_3: ; %atomicrmw.global4251; GFX950-GISEL-NEXT: buffer_wbl2 sc14252; GFX950-GISEL-NEXT: flat_atomic_or_x2 v[0:1], v[2:3], v[4:5] sc04253; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4254; GFX950-GISEL-NEXT: buffer_inv sc14255; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr34256; GFX950-GISEL-NEXT: ; implicit-def: $vgpr44257; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]4258; GFX950-GISEL-NEXT: s_cbranch_execz .LBB42_24259; GFX950-GISEL-NEXT: .LBB42_4: ; %atomicrmw.private4260; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]4261; GFX950-GISEL-NEXT: s_nop 14262; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc4263; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off4264; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)4265; GFX950-GISEL-NEXT: v_or_b32_e32 v2, v0, v44266; GFX950-GISEL-NEXT: v_or_b32_e32 v3, v1, v54267; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off4268; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]4269; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)4270; GFX950-GISEL-NEXT: s_branch .LBB42_54271; GFX950-GISEL-NEXT: .LBB42_5:4272 %zext.offset = zext i32 %voffset to i644273 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset4274 %rtn = atomicrmw or ptr %gep0, i64 %data syncscope("agent") seq_cst4275 %cast.rtn = bitcast i64 %rtn to <2 x float>4276 ret <2 x float> %cast.rtn4277}4278 4279define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {4280; GFX1250-SDAG-LABEL: flat_or_saddr_i64_rtn_neg128:4281; GFX1250-SDAG: ; %bb.0:4282; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 14283; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v14284; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 04285; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff804286; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4287; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]4288; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]4289; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4290; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v54291; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v04292; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr14293; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo4294; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4295; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s04296; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB43_34297; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow4298; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s04299; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB43_44300; GFX1250-SDAG-NEXT: .LBB43_2: ; %atomicrmw.phi4301; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s04302; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x04303; GFX1250-SDAG-NEXT: s_branch .LBB43_54304; GFX1250-SDAG-NEXT: .LBB43_3: ; %atomicrmw.global4305; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV4306; GFX1250-SDAG-NEXT: s_wait_storecnt 0x04307; GFX1250-SDAG-NEXT: flat_atomic_or_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV4308; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x04309; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV4310; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr54311; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr34312; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s04313; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB43_24314; GFX1250-SDAG-NEXT: .LBB43_4: ; %atomicrmw.private4315; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]4316; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v44317; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)4318; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo4319; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off4320; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x04321; GFX1250-SDAG-NEXT: v_or_b32_e32 v3, v1, v34322; GFX1250-SDAG-NEXT: v_or_b32_e32 v2, v0, v24323; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off4324; GFX1250-SDAG-NEXT: s_wait_xcnt 0x04325; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s04326; GFX1250-SDAG-NEXT: s_branch .LBB43_54327; GFX1250-SDAG-NEXT: .LBB43_5:4328;4329; GFX1250-GISEL-LABEL: flat_or_saddr_i64_rtn_neg128:4330; GFX1250-GISEL: ; %bb.0:4331; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 14332; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v14333; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]4334; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4335; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v34336; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo4337; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)4338; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v04339; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo4340; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4341; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x144342; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v04343; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr14344; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo4345; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4346; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s04347; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB43_34348; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow4349; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s04350; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB43_44351; GFX1250-GISEL-NEXT: .LBB43_2: ; %atomicrmw.phi4352; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s04353; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x04354; GFX1250-GISEL-NEXT: s_branch .LBB43_54355; GFX1250-GISEL-NEXT: .LBB43_3: ; %atomicrmw.global4356; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV4357; GFX1250-GISEL-NEXT: s_wait_storecnt 0x04358; GFX1250-GISEL-NEXT: flat_atomic_or_b64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV4359; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x04360; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV4361; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr64362; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr44363; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s04364; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB43_24365; GFX1250-GISEL-NEXT: .LBB43_4: ; %atomicrmw.private4366; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]4367; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v64368; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)4369; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo4370; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off4371; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x04372; GFX1250-GISEL-NEXT: v_or_b32_e32 v2, v0, v44373; GFX1250-GISEL-NEXT: v_or_b32_e32 v3, v1, v54374; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off4375; GFX1250-GISEL-NEXT: s_wait_xcnt 0x04376; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s04377; GFX1250-GISEL-NEXT: s_branch .LBB43_54378; GFX1250-GISEL-NEXT: .LBB43_5:4379;4380; GFX950-SDAG-LABEL: flat_or_saddr_i64_rtn_neg128:4381; GFX950-SDAG: ; %bb.0:4382; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v24383; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v14384; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 04385; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]4386; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff804387; GFX950-SDAG-NEXT: s_mov_b32 s3, -14388; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base4389; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]4390; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v54391; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr14392; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc4393; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]4394; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB43_34395; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow4396; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]4397; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB43_44398; GFX950-SDAG-NEXT: .LBB43_2: ; %atomicrmw.phi4399; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]4400; GFX950-SDAG-NEXT: s_branch .LBB43_54401; GFX950-SDAG-NEXT: .LBB43_3: ; %atomicrmw.global4402; GFX950-SDAG-NEXT: buffer_wbl2 sc14403; GFX950-SDAG-NEXT: flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] sc04404; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4405; GFX950-SDAG-NEXT: buffer_inv sc14406; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr54407; GFX950-SDAG-NEXT: ; implicit-def: $vgpr34408; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]4409; GFX950-SDAG-NEXT: s_cbranch_execz .LBB43_24410; GFX950-SDAG-NEXT: .LBB43_4: ; %atomicrmw.private4411; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]4412; GFX950-SDAG-NEXT: s_nop 14413; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc4414; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off4415; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)4416; GFX950-SDAG-NEXT: v_or_b32_e32 v3, v1, v34417; GFX950-SDAG-NEXT: v_or_b32_e32 v2, v0, v24418; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off4419; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]4420; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)4421; GFX950-SDAG-NEXT: s_branch .LBB43_54422; GFX950-SDAG-NEXT: .LBB43_5:4423;4424; GFX950-GISEL-LABEL: flat_or_saddr_i64_rtn_neg128:4425; GFX950-GISEL: ; %bb.0:4426; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v24427; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]4428; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v04429; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v14430; GFX950-GISEL-NEXT: s_nop 04431; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc4432; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v04433; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base4434; GFX950-GISEL-NEXT: s_nop 04435; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v1, vcc4436; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v34437; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr14438; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc4439; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]4440; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB43_34441; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow4442; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]4443; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB43_44444; GFX950-GISEL-NEXT: .LBB43_2: ; %atomicrmw.phi4445; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]4446; GFX950-GISEL-NEXT: s_branch .LBB43_54447; GFX950-GISEL-NEXT: .LBB43_3: ; %atomicrmw.global4448; GFX950-GISEL-NEXT: buffer_wbl2 sc14449; GFX950-GISEL-NEXT: flat_atomic_or_x2 v[0:1], v[2:3], v[4:5] sc04450; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4451; GFX950-GISEL-NEXT: buffer_inv sc14452; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr34453; GFX950-GISEL-NEXT: ; implicit-def: $vgpr44454; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]4455; GFX950-GISEL-NEXT: s_cbranch_execz .LBB43_24456; GFX950-GISEL-NEXT: .LBB43_4: ; %atomicrmw.private4457; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]4458; GFX950-GISEL-NEXT: s_nop 14459; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc4460; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off4461; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)4462; GFX950-GISEL-NEXT: v_or_b32_e32 v2, v0, v44463; GFX950-GISEL-NEXT: v_or_b32_e32 v3, v1, v54464; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off4465; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]4466; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)4467; GFX950-GISEL-NEXT: s_branch .LBB43_54468; GFX950-GISEL-NEXT: .LBB43_5:4469 %zext.offset = zext i32 %voffset to i644470 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset4471 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1284472 %rtn = atomicrmw or ptr %gep1, i64 %data syncscope("agent") seq_cst4473 %cast.rtn = bitcast i64 %rtn to <2 x float>4474 ret <2 x float> %cast.rtn4475}4476 4477define amdgpu_ps void @flat_or_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {4478; GFX1250-SDAG-LABEL: flat_or_saddr_i64_nortn:4479; GFX1250-SDAG: ; %bb.0:4480; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 14481; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v14482; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 04483; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo4484; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4485; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]4486; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v14487; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)4488; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v44489; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s04490; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB44_34491; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow4492; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s04493; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB44_44494; GFX1250-SDAG-NEXT: .LBB44_2: ; %atomicrmw.phi4495; GFX1250-SDAG-NEXT: s_endpgm4496; GFX1250-SDAG-NEXT: .LBB44_3: ; %atomicrmw.global4497; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV4498; GFX1250-SDAG-NEXT: s_wait_storecnt 0x04499; GFX1250-SDAG-NEXT: flat_atomic_or_b64 v[0:1], v[2:3] scope:SCOPE_DEV4500; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x04501; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV4502; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr14503; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr34504; GFX1250-SDAG-NEXT: s_wait_xcnt 0x04505; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s04506; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB44_24507; GFX1250-SDAG-NEXT: .LBB44_4: ; %atomicrmw.private4508; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]4509; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v04510; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)4511; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo4512; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off4513; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x04514; GFX1250-SDAG-NEXT: v_or_b32_e32 v1, v1, v34515; GFX1250-SDAG-NEXT: v_or_b32_e32 v0, v0, v24516; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off4517; GFX1250-SDAG-NEXT: s_endpgm4518;4519; GFX1250-GISEL-LABEL: flat_or_saddr_i64_nortn:4520; GFX1250-GISEL: ; %bb.0:4521; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 14522; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v24523; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]4524; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo4525; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4526; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v04527; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo4528; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4529; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v34530; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v14531; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s04532; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB44_34533; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow4534; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s04535; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB44_44536; GFX1250-GISEL-NEXT: .LBB44_2: ; %atomicrmw.phi4537; GFX1250-GISEL-NEXT: s_endpgm4538; GFX1250-GISEL-NEXT: .LBB44_3: ; %atomicrmw.global4539; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV4540; GFX1250-GISEL-NEXT: s_wait_storecnt 0x04541; GFX1250-GISEL-NEXT: flat_atomic_or_b64 v0, v[4:5], s[2:3] scope:SCOPE_DEV4542; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x04543; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV4544; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr24545; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr44546; GFX1250-GISEL-NEXT: s_wait_xcnt 0x04547; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s04548; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB44_24549; GFX1250-GISEL-NEXT: .LBB44_4: ; %atomicrmw.private4550; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]4551; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v24552; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)4553; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo4554; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off4555; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x04556; GFX1250-GISEL-NEXT: v_or_b32_e32 v0, v0, v44557; GFX1250-GISEL-NEXT: v_or_b32_e32 v1, v1, v54558; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off4559; GFX1250-GISEL-NEXT: s_endpgm4560;4561; GFX950-SDAG-LABEL: flat_or_saddr_i64_nortn:4562; GFX950-SDAG: ; %bb.0:4563; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v24564; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v14565; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 04566; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base4567; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]4568; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v14569; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc4570; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]4571; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB44_34572; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow4573; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]4574; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB44_44575; GFX950-SDAG-NEXT: .LBB44_2: ; %atomicrmw.phi4576; GFX950-SDAG-NEXT: s_endpgm4577; GFX950-SDAG-NEXT: .LBB44_3: ; %atomicrmw.global4578; GFX950-SDAG-NEXT: buffer_wbl2 sc14579; GFX950-SDAG-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]4580; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4581; GFX950-SDAG-NEXT: buffer_inv sc14582; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr14583; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr34584; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]4585; GFX950-SDAG-NEXT: s_cbranch_execz .LBB44_24586; GFX950-SDAG-NEXT: .LBB44_4: ; %atomicrmw.private4587; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]4588; GFX950-SDAG-NEXT: s_nop 14589; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc4590; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off4591; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)4592; GFX950-SDAG-NEXT: v_or_b32_e32 v1, v1, v34593; GFX950-SDAG-NEXT: v_or_b32_e32 v0, v0, v24594; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[0:1], off4595; GFX950-SDAG-NEXT: s_endpgm4596;4597; GFX950-GISEL-LABEL: flat_or_saddr_i64_nortn:4598; GFX950-GISEL: ; %bb.0:4599; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v24600; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]4601; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v04602; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v14603; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base4604; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc4605; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v14606; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc4607; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]4608; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB44_34609; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow4610; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]4611; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB44_44612; GFX950-GISEL-NEXT: .LBB44_2: ; %atomicrmw.phi4613; GFX950-GISEL-NEXT: s_endpgm4614; GFX950-GISEL-NEXT: .LBB44_3: ; %atomicrmw.global4615; GFX950-GISEL-NEXT: buffer_wbl2 sc14616; GFX950-GISEL-NEXT: flat_atomic_or_x2 v[0:1], v[4:5]4617; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4618; GFX950-GISEL-NEXT: buffer_inv sc14619; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr14620; GFX950-GISEL-NEXT: ; implicit-def: $vgpr44621; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]4622; GFX950-GISEL-NEXT: s_cbranch_execz .LBB44_24623; GFX950-GISEL-NEXT: .LBB44_4: ; %atomicrmw.private4624; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]4625; GFX950-GISEL-NEXT: s_nop 14626; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc4627; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off4628; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)4629; GFX950-GISEL-NEXT: v_or_b32_e32 v0, v0, v44630; GFX950-GISEL-NEXT: v_or_b32_e32 v1, v1, v54631; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off4632; GFX950-GISEL-NEXT: s_endpgm4633 %zext.offset = zext i32 %voffset to i644634 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset4635 %unused = atomicrmw or ptr %gep0, i64 %data syncscope("agent") seq_cst4636 ret void4637}4638 4639define amdgpu_ps void @flat_or_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {4640; GFX1250-SDAG-LABEL: flat_or_saddr_i64_nortn_neg128:4641; GFX1250-SDAG: ; %bb.0:4642; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 14643; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v14644; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 04645; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff804646; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4647; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]4648; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]4649; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo4650; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4651; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v14652; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v44653; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s04654; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB45_34655; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow4656; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s04657; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB45_44658; GFX1250-SDAG-NEXT: .LBB45_2: ; %atomicrmw.phi4659; GFX1250-SDAG-NEXT: s_endpgm4660; GFX1250-SDAG-NEXT: .LBB45_3: ; %atomicrmw.global4661; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV4662; GFX1250-SDAG-NEXT: s_wait_storecnt 0x04663; GFX1250-SDAG-NEXT: flat_atomic_or_b64 v[0:1], v[2:3] scope:SCOPE_DEV4664; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x04665; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV4666; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr14667; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr34668; GFX1250-SDAG-NEXT: s_wait_xcnt 0x04669; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s04670; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB45_24671; GFX1250-SDAG-NEXT: .LBB45_4: ; %atomicrmw.private4672; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]4673; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v04674; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)4675; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo4676; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off4677; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x04678; GFX1250-SDAG-NEXT: v_or_b32_e32 v1, v1, v34679; GFX1250-SDAG-NEXT: v_or_b32_e32 v0, v0, v24680; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off4681; GFX1250-SDAG-NEXT: s_endpgm4682;4683; GFX1250-GISEL-LABEL: flat_or_saddr_i64_nortn_neg128:4684; GFX1250-GISEL: ; %bb.0:4685; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 14686; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v24687; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]4688; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo4689; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4690; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v04691; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo4692; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)4693; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v14694; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo4695; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4696; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v34697; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v14698; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s04699; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB45_34700; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow4701; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s04702; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB45_44703; GFX1250-GISEL-NEXT: .LBB45_2: ; %atomicrmw.phi4704; GFX1250-GISEL-NEXT: s_endpgm4705; GFX1250-GISEL-NEXT: .LBB45_3: ; %atomicrmw.global4706; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV4707; GFX1250-GISEL-NEXT: s_wait_storecnt 0x04708; GFX1250-GISEL-NEXT: flat_atomic_or_b64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV4709; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x04710; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV4711; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr24712; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr44713; GFX1250-GISEL-NEXT: s_wait_xcnt 0x04714; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s04715; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB45_24716; GFX1250-GISEL-NEXT: .LBB45_4: ; %atomicrmw.private4717; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]4718; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v24719; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)4720; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo4721; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off4722; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x04723; GFX1250-GISEL-NEXT: v_or_b32_e32 v0, v0, v44724; GFX1250-GISEL-NEXT: v_or_b32_e32 v1, v1, v54725; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off4726; GFX1250-GISEL-NEXT: s_endpgm4727;4728; GFX950-SDAG-LABEL: flat_or_saddr_i64_nortn_neg128:4729; GFX950-SDAG: ; %bb.0:4730; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v24731; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v14732; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 04733; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]4734; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff804735; GFX950-SDAG-NEXT: s_mov_b32 s3, -14736; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base4737; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]4738; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v14739; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc4740; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]4741; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB45_34742; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow4743; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]4744; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB45_44745; GFX950-SDAG-NEXT: .LBB45_2: ; %atomicrmw.phi4746; GFX950-SDAG-NEXT: s_endpgm4747; GFX950-SDAG-NEXT: .LBB45_3: ; %atomicrmw.global4748; GFX950-SDAG-NEXT: buffer_wbl2 sc14749; GFX950-SDAG-NEXT: flat_atomic_or_x2 v[0:1], v[2:3]4750; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4751; GFX950-SDAG-NEXT: buffer_inv sc14752; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr14753; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr34754; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]4755; GFX950-SDAG-NEXT: s_cbranch_execz .LBB45_24756; GFX950-SDAG-NEXT: .LBB45_4: ; %atomicrmw.private4757; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]4758; GFX950-SDAG-NEXT: s_nop 14759; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc4760; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off4761; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)4762; GFX950-SDAG-NEXT: v_or_b32_e32 v1, v1, v34763; GFX950-SDAG-NEXT: v_or_b32_e32 v0, v0, v24764; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[0:1], off4765; GFX950-SDAG-NEXT: s_endpgm4766;4767; GFX950-GISEL-LABEL: flat_or_saddr_i64_nortn_neg128:4768; GFX950-GISEL: ; %bb.0:4769; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v24770; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]4771; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v04772; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v14773; GFX950-GISEL-NEXT: s_nop 04774; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc4775; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v04776; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base4777; GFX950-GISEL-NEXT: s_nop 04778; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc4779; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v14780; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc4781; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]4782; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB45_34783; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow4784; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]4785; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB45_44786; GFX950-GISEL-NEXT: .LBB45_2: ; %atomicrmw.phi4787; GFX950-GISEL-NEXT: s_endpgm4788; GFX950-GISEL-NEXT: .LBB45_3: ; %atomicrmw.global4789; GFX950-GISEL-NEXT: buffer_wbl2 sc14790; GFX950-GISEL-NEXT: flat_atomic_or_x2 v[0:1], v[4:5]4791; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4792; GFX950-GISEL-NEXT: buffer_inv sc14793; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr14794; GFX950-GISEL-NEXT: ; implicit-def: $vgpr44795; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]4796; GFX950-GISEL-NEXT: s_cbranch_execz .LBB45_24797; GFX950-GISEL-NEXT: .LBB45_4: ; %atomicrmw.private4798; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]4799; GFX950-GISEL-NEXT: s_nop 14800; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc4801; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off4802; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)4803; GFX950-GISEL-NEXT: v_or_b32_e32 v0, v0, v44804; GFX950-GISEL-NEXT: v_or_b32_e32 v1, v1, v54805; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off4806; GFX950-GISEL-NEXT: s_endpgm4807 %zext.offset = zext i32 %voffset to i644808 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset4809 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1284810 %unused = atomicrmw or ptr %gep1, i64 %data syncscope("agent") seq_cst4811 ret void4812}4813 4814; --------------------------------------------------------------------------------4815; atomicrmw xor4816; --------------------------------------------------------------------------------4817 4818define amdgpu_ps float @flat_xor_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {4819; GFX1250-LABEL: flat_xor_saddr_i32_rtn:4820; GFX1250: ; %bb.0:4821; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 14822; GFX1250-NEXT: global_wb scope:SCOPE_DEV4823; GFX1250-NEXT: s_wait_storecnt 0x04824; GFX1250-NEXT: flat_atomic_xor_b32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV4825; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x04826; GFX1250-NEXT: global_inv scope:SCOPE_DEV4827; GFX1250-NEXT: s_wait_loadcnt 0x04828; GFX1250-NEXT: ; return to shader part epilog4829;4830; GFX950-SDAG-LABEL: flat_xor_saddr_i32_rtn:4831; GFX950-SDAG: ; %bb.0:4832; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v14833; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 04834; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]4835; GFX950-SDAG-NEXT: buffer_wbl2 sc14836; GFX950-SDAG-NEXT: flat_atomic_xor v0, v[0:1], v2 sc04837; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4838; GFX950-SDAG-NEXT: buffer_inv sc14839; GFX950-SDAG-NEXT: ; return to shader part epilog4840;4841; GFX950-GISEL-LABEL: flat_xor_saddr_i32_rtn:4842; GFX950-GISEL: ; %bb.0:4843; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]4844; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v04845; GFX950-GISEL-NEXT: s_nop 14846; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc4847; GFX950-GISEL-NEXT: buffer_wbl2 sc14848; GFX950-GISEL-NEXT: flat_atomic_xor v0, v[2:3], v1 sc04849; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4850; GFX950-GISEL-NEXT: buffer_inv sc14851; GFX950-GISEL-NEXT: ; return to shader part epilog4852 %zext.offset = zext i32 %voffset to i644853 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset4854 %rtn = atomicrmw xor ptr %gep0, i32 %data syncscope("agent") seq_cst4855 %cast.rtn = bitcast i32 %rtn to float4856 ret float %cast.rtn4857}4858 4859define amdgpu_ps float @flat_xor_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {4860; GFX1250-LABEL: flat_xor_saddr_i32_rtn_neg128:4861; GFX1250: ; %bb.0:4862; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 14863; GFX1250-NEXT: global_wb scope:SCOPE_DEV4864; GFX1250-NEXT: s_wait_storecnt 0x04865; GFX1250-NEXT: flat_atomic_xor_b32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV4866; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x04867; GFX1250-NEXT: global_inv scope:SCOPE_DEV4868; GFX1250-NEXT: s_wait_loadcnt 0x04869; GFX1250-NEXT: ; return to shader part epilog4870;4871; GFX950-SDAG-LABEL: flat_xor_saddr_i32_rtn_neg128:4872; GFX950-SDAG: ; %bb.0:4873; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v14874; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 04875; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]4876; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v04877; GFX950-SDAG-NEXT: s_nop 14878; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc4879; GFX950-SDAG-NEXT: buffer_wbl2 sc14880; GFX950-SDAG-NEXT: flat_atomic_xor v0, v[0:1], v2 sc04881; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4882; GFX950-SDAG-NEXT: buffer_inv sc14883; GFX950-SDAG-NEXT: ; return to shader part epilog4884;4885; GFX950-GISEL-LABEL: flat_xor_saddr_i32_rtn_neg128:4886; GFX950-GISEL: ; %bb.0:4887; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]4888; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v04889; GFX950-GISEL-NEXT: s_nop 14890; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc4891; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v04892; GFX950-GISEL-NEXT: s_nop 14893; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc4894; GFX950-GISEL-NEXT: buffer_wbl2 sc14895; GFX950-GISEL-NEXT: flat_atomic_xor v0, v[2:3], v1 sc04896; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4897; GFX950-GISEL-NEXT: buffer_inv sc14898; GFX950-GISEL-NEXT: ; return to shader part epilog4899 %zext.offset = zext i32 %voffset to i644900 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset4901 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1284902 %rtn = atomicrmw xor ptr %gep1, i32 %data syncscope("agent") seq_cst4903 %cast.rtn = bitcast i32 %rtn to float4904 ret float %cast.rtn4905}4906 4907define amdgpu_ps void @flat_xor_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {4908; GFX1250-LABEL: flat_xor_saddr_i32_nortn:4909; GFX1250: ; %bb.0:4910; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 14911; GFX1250-NEXT: global_wb scope:SCOPE_DEV4912; GFX1250-NEXT: s_wait_storecnt 0x04913; GFX1250-NEXT: flat_atomic_xor_b32 v0, v1, s[2:3] scope:SCOPE_DEV4914; GFX1250-NEXT: s_wait_storecnt_dscnt 0x04915; GFX1250-NEXT: global_inv scope:SCOPE_DEV4916; GFX1250-NEXT: s_endpgm4917;4918; GFX950-SDAG-LABEL: flat_xor_saddr_i32_nortn:4919; GFX950-SDAG: ; %bb.0:4920; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v14921; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 04922; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]4923; GFX950-SDAG-NEXT: buffer_wbl2 sc14924; GFX950-SDAG-NEXT: flat_atomic_xor v[0:1], v24925; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4926; GFX950-SDAG-NEXT: buffer_inv sc14927; GFX950-SDAG-NEXT: s_endpgm4928;4929; GFX950-GISEL-LABEL: flat_xor_saddr_i32_nortn:4930; GFX950-GISEL: ; %bb.0:4931; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]4932; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v04933; GFX950-GISEL-NEXT: s_nop 14934; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc4935; GFX950-GISEL-NEXT: buffer_wbl2 sc14936; GFX950-GISEL-NEXT: flat_atomic_xor v[2:3], v14937; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4938; GFX950-GISEL-NEXT: buffer_inv sc14939; GFX950-GISEL-NEXT: s_endpgm4940 %zext.offset = zext i32 %voffset to i644941 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset4942 %unused = atomicrmw xor ptr %gep0, i32 %data syncscope("agent") seq_cst4943 ret void4944}4945 4946define amdgpu_ps void @flat_xor_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {4947; GFX1250-LABEL: flat_xor_saddr_i32_nortn_neg128:4948; GFX1250: ; %bb.0:4949; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 14950; GFX1250-NEXT: global_wb scope:SCOPE_DEV4951; GFX1250-NEXT: s_wait_storecnt 0x04952; GFX1250-NEXT: flat_atomic_xor_b32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV4953; GFX1250-NEXT: s_wait_storecnt_dscnt 0x04954; GFX1250-NEXT: global_inv scope:SCOPE_DEV4955; GFX1250-NEXT: s_endpgm4956;4957; GFX950-SDAG-LABEL: flat_xor_saddr_i32_nortn_neg128:4958; GFX950-SDAG: ; %bb.0:4959; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v14960; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 04961; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]4962; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v04963; GFX950-SDAG-NEXT: s_nop 14964; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc4965; GFX950-SDAG-NEXT: buffer_wbl2 sc14966; GFX950-SDAG-NEXT: flat_atomic_xor v[0:1], v24967; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4968; GFX950-SDAG-NEXT: buffer_inv sc14969; GFX950-SDAG-NEXT: s_endpgm4970;4971; GFX950-GISEL-LABEL: flat_xor_saddr_i32_nortn_neg128:4972; GFX950-GISEL: ; %bb.0:4973; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]4974; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v04975; GFX950-GISEL-NEXT: s_nop 14976; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc4977; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v04978; GFX950-GISEL-NEXT: s_nop 14979; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc4980; GFX950-GISEL-NEXT: buffer_wbl2 sc14981; GFX950-GISEL-NEXT: flat_atomic_xor v[2:3], v14982; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)4983; GFX950-GISEL-NEXT: buffer_inv sc14984; GFX950-GISEL-NEXT: s_endpgm4985 %zext.offset = zext i32 %voffset to i644986 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset4987 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1284988 %unused = atomicrmw xor ptr %gep1, i32 %data syncscope("agent") seq_cst4989 ret void4990}4991 4992define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {4993; GFX1250-SDAG-LABEL: flat_xor_saddr_i64_rtn:4994; GFX1250-SDAG: ; %bb.0:4995; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 14996; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v14997; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 04998; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4999; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]5000; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v55001; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)5002; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v05003; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr15004; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo5005; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s05006; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB50_35007; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow5008; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s05009; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB50_45010; GFX1250-SDAG-NEXT: .LBB50_2: ; %atomicrmw.phi5011; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s05012; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x05013; GFX1250-SDAG-NEXT: s_branch .LBB50_55014; GFX1250-SDAG-NEXT: .LBB50_3: ; %atomicrmw.global5015; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV5016; GFX1250-SDAG-NEXT: s_wait_storecnt 0x05017; GFX1250-SDAG-NEXT: flat_atomic_xor_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV5018; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x05019; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV5020; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr55021; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr35022; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s05023; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB50_25024; GFX1250-SDAG-NEXT: .LBB50_4: ; %atomicrmw.private5025; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]5026; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v45027; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)5028; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo5029; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off5030; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x05031; GFX1250-SDAG-NEXT: v_xor_b32_e32 v3, v1, v35032; GFX1250-SDAG-NEXT: v_xor_b32_e32 v2, v0, v25033; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off5034; GFX1250-SDAG-NEXT: s_wait_xcnt 0x05035; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s05036; GFX1250-SDAG-NEXT: s_branch .LBB50_55037; GFX1250-SDAG-NEXT: .LBB50_5:5038;5039; GFX1250-GISEL-LABEL: flat_xor_saddr_i64_rtn:5040; GFX1250-GISEL: ; %bb.0:5041; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 15042; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v15043; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]5044; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5045; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v35046; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo5047; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5048; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x145049; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v05050; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr15051; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo5052; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5053; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s05054; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB50_35055; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow5056; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s05057; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB50_45058; GFX1250-GISEL-NEXT: .LBB50_2: ; %atomicrmw.phi5059; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s05060; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x05061; GFX1250-GISEL-NEXT: s_branch .LBB50_55062; GFX1250-GISEL-NEXT: .LBB50_3: ; %atomicrmw.global5063; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV5064; GFX1250-GISEL-NEXT: s_wait_storecnt 0x05065; GFX1250-GISEL-NEXT: flat_atomic_xor_b64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV5066; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x05067; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV5068; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr65069; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr45070; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s05071; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB50_25072; GFX1250-GISEL-NEXT: .LBB50_4: ; %atomicrmw.private5073; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]5074; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v65075; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)5076; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo5077; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off5078; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x05079; GFX1250-GISEL-NEXT: v_xor_b32_e32 v2, v0, v45080; GFX1250-GISEL-NEXT: v_xor_b32_e32 v3, v1, v55081; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off5082; GFX1250-GISEL-NEXT: s_wait_xcnt 0x05083; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s05084; GFX1250-GISEL-NEXT: s_branch .LBB50_55085; GFX1250-GISEL-NEXT: .LBB50_5:5086;5087; GFX950-SDAG-LABEL: flat_xor_saddr_i64_rtn:5088; GFX950-SDAG: ; %bb.0:5089; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v25090; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v15091; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 05092; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base5093; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1]5094; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v55095; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr15096; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc5097; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]5098; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB50_35099; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow5100; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]5101; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB50_45102; GFX950-SDAG-NEXT: .LBB50_2: ; %atomicrmw.phi5103; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]5104; GFX950-SDAG-NEXT: s_branch .LBB50_55105; GFX950-SDAG-NEXT: .LBB50_3: ; %atomicrmw.global5106; GFX950-SDAG-NEXT: buffer_wbl2 sc15107; GFX950-SDAG-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] sc05108; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5109; GFX950-SDAG-NEXT: buffer_inv sc15110; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr55111; GFX950-SDAG-NEXT: ; implicit-def: $vgpr35112; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]5113; GFX950-SDAG-NEXT: s_cbranch_execz .LBB50_25114; GFX950-SDAG-NEXT: .LBB50_4: ; %atomicrmw.private5115; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]5116; GFX950-SDAG-NEXT: s_nop 15117; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc5118; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off5119; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)5120; GFX950-SDAG-NEXT: v_xor_b32_e32 v3, v1, v35121; GFX950-SDAG-NEXT: v_xor_b32_e32 v2, v0, v25122; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off5123; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]5124; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)5125; GFX950-SDAG-NEXT: s_branch .LBB50_55126; GFX950-SDAG-NEXT: .LBB50_5:5127;5128; GFX950-GISEL-LABEL: flat_xor_saddr_i64_rtn:5129; GFX950-GISEL: ; %bb.0:5130; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v25131; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]5132; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v05133; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base5134; GFX950-GISEL-NEXT: s_nop 05135; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc5136; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v15137; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v35138; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr15139; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc5140; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]5141; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB50_35142; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow5143; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]5144; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB50_45145; GFX950-GISEL-NEXT: .LBB50_2: ; %atomicrmw.phi5146; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]5147; GFX950-GISEL-NEXT: s_branch .LBB50_55148; GFX950-GISEL-NEXT: .LBB50_3: ; %atomicrmw.global5149; GFX950-GISEL-NEXT: buffer_wbl2 sc15150; GFX950-GISEL-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3], v[4:5] sc05151; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5152; GFX950-GISEL-NEXT: buffer_inv sc15153; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr35154; GFX950-GISEL-NEXT: ; implicit-def: $vgpr45155; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]5156; GFX950-GISEL-NEXT: s_cbranch_execz .LBB50_25157; GFX950-GISEL-NEXT: .LBB50_4: ; %atomicrmw.private5158; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]5159; GFX950-GISEL-NEXT: s_nop 15160; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc5161; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off5162; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)5163; GFX950-GISEL-NEXT: v_xor_b32_e32 v2, v0, v45164; GFX950-GISEL-NEXT: v_xor_b32_e32 v3, v1, v55165; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off5166; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]5167; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)5168; GFX950-GISEL-NEXT: s_branch .LBB50_55169; GFX950-GISEL-NEXT: .LBB50_5:5170 %zext.offset = zext i32 %voffset to i645171 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset5172 %rtn = atomicrmw xor ptr %gep0, i64 %data syncscope("agent") seq_cst5173 %cast.rtn = bitcast i64 %rtn to <2 x float>5174 ret <2 x float> %cast.rtn5175}5176 5177define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {5178; GFX1250-SDAG-LABEL: flat_xor_saddr_i64_rtn_neg128:5179; GFX1250-SDAG: ; %bb.0:5180; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 15181; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v15182; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 05183; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff805184; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5185; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]5186; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]5187; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5188; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v55189; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v05190; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr15191; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo5192; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5193; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s05194; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB51_35195; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow5196; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s05197; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB51_45198; GFX1250-SDAG-NEXT: .LBB51_2: ; %atomicrmw.phi5199; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s05200; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x05201; GFX1250-SDAG-NEXT: s_branch .LBB51_55202; GFX1250-SDAG-NEXT: .LBB51_3: ; %atomicrmw.global5203; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV5204; GFX1250-SDAG-NEXT: s_wait_storecnt 0x05205; GFX1250-SDAG-NEXT: flat_atomic_xor_b64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV5206; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x05207; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV5208; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr55209; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr35210; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s05211; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB51_25212; GFX1250-SDAG-NEXT: .LBB51_4: ; %atomicrmw.private5213; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]5214; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v45215; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)5216; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo5217; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off5218; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x05219; GFX1250-SDAG-NEXT: v_xor_b32_e32 v3, v1, v35220; GFX1250-SDAG-NEXT: v_xor_b32_e32 v2, v0, v25221; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off5222; GFX1250-SDAG-NEXT: s_wait_xcnt 0x05223; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s05224; GFX1250-SDAG-NEXT: s_branch .LBB51_55225; GFX1250-SDAG-NEXT: .LBB51_5:5226;5227; GFX1250-GISEL-LABEL: flat_xor_saddr_i64_rtn_neg128:5228; GFX1250-GISEL: ; %bb.0:5229; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 15230; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v15231; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]5232; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5233; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v35234; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo5235; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)5236; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v05237; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo5238; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5239; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x145240; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v05241; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr15242; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo5243; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5244; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s05245; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB51_35246; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow5247; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s05248; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB51_45249; GFX1250-GISEL-NEXT: .LBB51_2: ; %atomicrmw.phi5250; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s05251; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x05252; GFX1250-GISEL-NEXT: s_branch .LBB51_55253; GFX1250-GISEL-NEXT: .LBB51_3: ; %atomicrmw.global5254; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV5255; GFX1250-GISEL-NEXT: s_wait_storecnt 0x05256; GFX1250-GISEL-NEXT: flat_atomic_xor_b64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV5257; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x05258; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV5259; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr65260; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr45261; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s05262; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB51_25263; GFX1250-GISEL-NEXT: .LBB51_4: ; %atomicrmw.private5264; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]5265; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v65266; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)5267; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo5268; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off5269; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x05270; GFX1250-GISEL-NEXT: v_xor_b32_e32 v2, v0, v45271; GFX1250-GISEL-NEXT: v_xor_b32_e32 v3, v1, v55272; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off5273; GFX1250-GISEL-NEXT: s_wait_xcnt 0x05274; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s05275; GFX1250-GISEL-NEXT: s_branch .LBB51_55276; GFX1250-GISEL-NEXT: .LBB51_5:5277;5278; GFX950-SDAG-LABEL: flat_xor_saddr_i64_rtn_neg128:5279; GFX950-SDAG: ; %bb.0:5280; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v25281; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v15282; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 05283; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]5284; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff805285; GFX950-SDAG-NEXT: s_mov_b32 s3, -15286; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base5287; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]5288; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v55289; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr15290; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc5291; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]5292; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB51_35293; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow5294; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]5295; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB51_45296; GFX950-SDAG-NEXT: .LBB51_2: ; %atomicrmw.phi5297; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]5298; GFX950-SDAG-NEXT: s_branch .LBB51_55299; GFX950-SDAG-NEXT: .LBB51_3: ; %atomicrmw.global5300; GFX950-SDAG-NEXT: buffer_wbl2 sc15301; GFX950-SDAG-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] sc05302; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5303; GFX950-SDAG-NEXT: buffer_inv sc15304; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr55305; GFX950-SDAG-NEXT: ; implicit-def: $vgpr35306; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]5307; GFX950-SDAG-NEXT: s_cbranch_execz .LBB51_25308; GFX950-SDAG-NEXT: .LBB51_4: ; %atomicrmw.private5309; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]5310; GFX950-SDAG-NEXT: s_nop 15311; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc5312; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off5313; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)5314; GFX950-SDAG-NEXT: v_xor_b32_e32 v3, v1, v35315; GFX950-SDAG-NEXT: v_xor_b32_e32 v2, v0, v25316; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off5317; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]5318; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)5319; GFX950-SDAG-NEXT: s_branch .LBB51_55320; GFX950-SDAG-NEXT: .LBB51_5:5321;5322; GFX950-GISEL-LABEL: flat_xor_saddr_i64_rtn_neg128:5323; GFX950-GISEL: ; %bb.0:5324; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v25325; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]5326; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v05327; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v15328; GFX950-GISEL-NEXT: s_nop 05329; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc5330; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v05331; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base5332; GFX950-GISEL-NEXT: s_nop 05333; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v1, vcc5334; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v35335; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr15336; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc5337; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]5338; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB51_35339; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow5340; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]5341; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB51_45342; GFX950-GISEL-NEXT: .LBB51_2: ; %atomicrmw.phi5343; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]5344; GFX950-GISEL-NEXT: s_branch .LBB51_55345; GFX950-GISEL-NEXT: .LBB51_3: ; %atomicrmw.global5346; GFX950-GISEL-NEXT: buffer_wbl2 sc15347; GFX950-GISEL-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3], v[4:5] sc05348; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5349; GFX950-GISEL-NEXT: buffer_inv sc15350; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr35351; GFX950-GISEL-NEXT: ; implicit-def: $vgpr45352; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]5353; GFX950-GISEL-NEXT: s_cbranch_execz .LBB51_25354; GFX950-GISEL-NEXT: .LBB51_4: ; %atomicrmw.private5355; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]5356; GFX950-GISEL-NEXT: s_nop 15357; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc5358; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off5359; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)5360; GFX950-GISEL-NEXT: v_xor_b32_e32 v2, v0, v45361; GFX950-GISEL-NEXT: v_xor_b32_e32 v3, v1, v55362; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off5363; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]5364; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)5365; GFX950-GISEL-NEXT: s_branch .LBB51_55366; GFX950-GISEL-NEXT: .LBB51_5:5367 %zext.offset = zext i32 %voffset to i645368 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset5369 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1285370 %rtn = atomicrmw xor ptr %gep1, i64 %data syncscope("agent") seq_cst5371 %cast.rtn = bitcast i64 %rtn to <2 x float>5372 ret <2 x float> %cast.rtn5373}5374 5375define amdgpu_ps void @flat_xor_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {5376; GFX1250-SDAG-LABEL: flat_xor_saddr_i64_nortn:5377; GFX1250-SDAG: ; %bb.0:5378; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 15379; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v15380; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 05381; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo5382; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5383; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]5384; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v15385; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)5386; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v45387; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s05388; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB52_35389; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow5390; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s05391; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB52_45392; GFX1250-SDAG-NEXT: .LBB52_2: ; %atomicrmw.phi5393; GFX1250-SDAG-NEXT: s_endpgm5394; GFX1250-SDAG-NEXT: .LBB52_3: ; %atomicrmw.global5395; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV5396; GFX1250-SDAG-NEXT: s_wait_storecnt 0x05397; GFX1250-SDAG-NEXT: flat_atomic_xor_b64 v[0:1], v[2:3] scope:SCOPE_DEV5398; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x05399; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV5400; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr15401; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr35402; GFX1250-SDAG-NEXT: s_wait_xcnt 0x05403; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s05404; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB52_25405; GFX1250-SDAG-NEXT: .LBB52_4: ; %atomicrmw.private5406; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]5407; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v05408; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)5409; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo5410; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off5411; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x05412; GFX1250-SDAG-NEXT: v_xor_b32_e32 v1, v1, v35413; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, v0, v25414; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off5415; GFX1250-SDAG-NEXT: s_endpgm5416;5417; GFX1250-GISEL-LABEL: flat_xor_saddr_i64_nortn:5418; GFX1250-GISEL: ; %bb.0:5419; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 15420; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v25421; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]5422; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo5423; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5424; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v05425; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo5426; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5427; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v35428; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v15429; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s05430; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB52_35431; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow5432; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s05433; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB52_45434; GFX1250-GISEL-NEXT: .LBB52_2: ; %atomicrmw.phi5435; GFX1250-GISEL-NEXT: s_endpgm5436; GFX1250-GISEL-NEXT: .LBB52_3: ; %atomicrmw.global5437; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV5438; GFX1250-GISEL-NEXT: s_wait_storecnt 0x05439; GFX1250-GISEL-NEXT: flat_atomic_xor_b64 v0, v[4:5], s[2:3] scope:SCOPE_DEV5440; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x05441; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV5442; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr25443; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr45444; GFX1250-GISEL-NEXT: s_wait_xcnt 0x05445; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s05446; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB52_25447; GFX1250-GISEL-NEXT: .LBB52_4: ; %atomicrmw.private5448; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]5449; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v25450; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)5451; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo5452; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off5453; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x05454; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, v0, v45455; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, v1, v55456; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off5457; GFX1250-GISEL-NEXT: s_endpgm5458;5459; GFX950-SDAG-LABEL: flat_xor_saddr_i64_nortn:5460; GFX950-SDAG: ; %bb.0:5461; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v25462; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v15463; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 05464; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base5465; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]5466; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v15467; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc5468; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]5469; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB52_35470; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow5471; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]5472; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB52_45473; GFX950-SDAG-NEXT: .LBB52_2: ; %atomicrmw.phi5474; GFX950-SDAG-NEXT: s_endpgm5475; GFX950-SDAG-NEXT: .LBB52_3: ; %atomicrmw.global5476; GFX950-SDAG-NEXT: buffer_wbl2 sc15477; GFX950-SDAG-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]5478; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5479; GFX950-SDAG-NEXT: buffer_inv sc15480; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr15481; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr35482; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]5483; GFX950-SDAG-NEXT: s_cbranch_execz .LBB52_25484; GFX950-SDAG-NEXT: .LBB52_4: ; %atomicrmw.private5485; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]5486; GFX950-SDAG-NEXT: s_nop 15487; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc5488; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off5489; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)5490; GFX950-SDAG-NEXT: v_xor_b32_e32 v1, v1, v35491; GFX950-SDAG-NEXT: v_xor_b32_e32 v0, v0, v25492; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[0:1], off5493; GFX950-SDAG-NEXT: s_endpgm5494;5495; GFX950-GISEL-LABEL: flat_xor_saddr_i64_nortn:5496; GFX950-GISEL: ; %bb.0:5497; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v25498; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]5499; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v05500; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v15501; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base5502; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc5503; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v15504; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc5505; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]5506; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB52_35507; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow5508; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]5509; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB52_45510; GFX950-GISEL-NEXT: .LBB52_2: ; %atomicrmw.phi5511; GFX950-GISEL-NEXT: s_endpgm5512; GFX950-GISEL-NEXT: .LBB52_3: ; %atomicrmw.global5513; GFX950-GISEL-NEXT: buffer_wbl2 sc15514; GFX950-GISEL-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5]5515; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5516; GFX950-GISEL-NEXT: buffer_inv sc15517; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr15518; GFX950-GISEL-NEXT: ; implicit-def: $vgpr45519; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]5520; GFX950-GISEL-NEXT: s_cbranch_execz .LBB52_25521; GFX950-GISEL-NEXT: .LBB52_4: ; %atomicrmw.private5522; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]5523; GFX950-GISEL-NEXT: s_nop 15524; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc5525; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off5526; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)5527; GFX950-GISEL-NEXT: v_xor_b32_e32 v0, v0, v45528; GFX950-GISEL-NEXT: v_xor_b32_e32 v1, v1, v55529; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off5530; GFX950-GISEL-NEXT: s_endpgm5531 %zext.offset = zext i32 %voffset to i645532 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset5533 %unused = atomicrmw xor ptr %gep0, i64 %data syncscope("agent") seq_cst5534 ret void5535}5536 5537define amdgpu_ps void @flat_xor_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {5538; GFX1250-SDAG-LABEL: flat_xor_saddr_i64_nortn_neg128:5539; GFX1250-SDAG: ; %bb.0:5540; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 15541; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v15542; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 05543; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff805544; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5545; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]5546; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]5547; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo5548; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5549; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v15550; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v45551; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s05552; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB53_35553; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow5554; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s05555; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB53_45556; GFX1250-SDAG-NEXT: .LBB53_2: ; %atomicrmw.phi5557; GFX1250-SDAG-NEXT: s_endpgm5558; GFX1250-SDAG-NEXT: .LBB53_3: ; %atomicrmw.global5559; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV5560; GFX1250-SDAG-NEXT: s_wait_storecnt 0x05561; GFX1250-SDAG-NEXT: flat_atomic_xor_b64 v[0:1], v[2:3] scope:SCOPE_DEV5562; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x05563; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV5564; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr15565; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr35566; GFX1250-SDAG-NEXT: s_wait_xcnt 0x05567; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s05568; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB53_25569; GFX1250-SDAG-NEXT: .LBB53_4: ; %atomicrmw.private5570; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]5571; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v05572; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)5573; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo5574; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off5575; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x05576; GFX1250-SDAG-NEXT: v_xor_b32_e32 v1, v1, v35577; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, v0, v25578; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off5579; GFX1250-SDAG-NEXT: s_endpgm5580;5581; GFX1250-GISEL-LABEL: flat_xor_saddr_i64_nortn_neg128:5582; GFX1250-GISEL: ; %bb.0:5583; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 15584; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v25585; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]5586; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo5587; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5588; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v05589; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo5590; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)5591; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v15592; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo5593; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5594; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v35595; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v15596; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s05597; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB53_35598; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow5599; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s05600; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB53_45601; GFX1250-GISEL-NEXT: .LBB53_2: ; %atomicrmw.phi5602; GFX1250-GISEL-NEXT: s_endpgm5603; GFX1250-GISEL-NEXT: .LBB53_3: ; %atomicrmw.global5604; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV5605; GFX1250-GISEL-NEXT: s_wait_storecnt 0x05606; GFX1250-GISEL-NEXT: flat_atomic_xor_b64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV5607; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x05608; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV5609; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr25610; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr45611; GFX1250-GISEL-NEXT: s_wait_xcnt 0x05612; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s05613; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB53_25614; GFX1250-GISEL-NEXT: .LBB53_4: ; %atomicrmw.private5615; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]5616; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v25617; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)5618; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo5619; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off5620; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x05621; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, v0, v45622; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, v1, v55623; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off5624; GFX1250-GISEL-NEXT: s_endpgm5625;5626; GFX950-SDAG-LABEL: flat_xor_saddr_i64_nortn_neg128:5627; GFX950-SDAG: ; %bb.0:5628; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v25629; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v15630; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 05631; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]5632; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff805633; GFX950-SDAG-NEXT: s_mov_b32 s3, -15634; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base5635; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]5636; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v15637; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc5638; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]5639; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB53_35640; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow5641; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]5642; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB53_45643; GFX950-SDAG-NEXT: .LBB53_2: ; %atomicrmw.phi5644; GFX950-SDAG-NEXT: s_endpgm5645; GFX950-SDAG-NEXT: .LBB53_3: ; %atomicrmw.global5646; GFX950-SDAG-NEXT: buffer_wbl2 sc15647; GFX950-SDAG-NEXT: flat_atomic_xor_x2 v[0:1], v[2:3]5648; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5649; GFX950-SDAG-NEXT: buffer_inv sc15650; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr15651; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr35652; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]5653; GFX950-SDAG-NEXT: s_cbranch_execz .LBB53_25654; GFX950-SDAG-NEXT: .LBB53_4: ; %atomicrmw.private5655; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]5656; GFX950-SDAG-NEXT: s_nop 15657; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc5658; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off5659; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)5660; GFX950-SDAG-NEXT: v_xor_b32_e32 v1, v1, v35661; GFX950-SDAG-NEXT: v_xor_b32_e32 v0, v0, v25662; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[0:1], off5663; GFX950-SDAG-NEXT: s_endpgm5664;5665; GFX950-GISEL-LABEL: flat_xor_saddr_i64_nortn_neg128:5666; GFX950-GISEL: ; %bb.0:5667; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v25668; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]5669; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v05670; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v15671; GFX950-GISEL-NEXT: s_nop 05672; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc5673; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v05674; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base5675; GFX950-GISEL-NEXT: s_nop 05676; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc5677; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v15678; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc5679; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]5680; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB53_35681; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow5682; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]5683; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB53_45684; GFX950-GISEL-NEXT: .LBB53_2: ; %atomicrmw.phi5685; GFX950-GISEL-NEXT: s_endpgm5686; GFX950-GISEL-NEXT: .LBB53_3: ; %atomicrmw.global5687; GFX950-GISEL-NEXT: buffer_wbl2 sc15688; GFX950-GISEL-NEXT: flat_atomic_xor_x2 v[0:1], v[4:5]5689; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5690; GFX950-GISEL-NEXT: buffer_inv sc15691; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr15692; GFX950-GISEL-NEXT: ; implicit-def: $vgpr45693; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]5694; GFX950-GISEL-NEXT: s_cbranch_execz .LBB53_25695; GFX950-GISEL-NEXT: .LBB53_4: ; %atomicrmw.private5696; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]5697; GFX950-GISEL-NEXT: s_nop 15698; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc5699; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off5700; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)5701; GFX950-GISEL-NEXT: v_xor_b32_e32 v0, v0, v45702; GFX950-GISEL-NEXT: v_xor_b32_e32 v1, v1, v55703; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off5704; GFX950-GISEL-NEXT: s_endpgm5705 %zext.offset = zext i32 %voffset to i645706 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset5707 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1285708 %unused = atomicrmw xor ptr %gep1, i64 %data syncscope("agent") seq_cst5709 ret void5710}5711 5712; --------------------------------------------------------------------------------5713; atomicrmw max5714; --------------------------------------------------------------------------------5715 5716define amdgpu_ps float @flat_max_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {5717; GFX1250-LABEL: flat_max_saddr_i32_rtn:5718; GFX1250: ; %bb.0:5719; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 15720; GFX1250-NEXT: flat_atomic_max_i32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN5721; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x05722; GFX1250-NEXT: ; return to shader part epilog5723;5724; GFX950-SDAG-LABEL: flat_max_saddr_i32_rtn:5725; GFX950-SDAG: ; %bb.0:5726; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v15727; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 05728; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]5729; GFX950-SDAG-NEXT: flat_atomic_smax v0, v[0:1], v2 sc05730; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5731; GFX950-SDAG-NEXT: ; return to shader part epilog5732;5733; GFX950-GISEL-LABEL: flat_max_saddr_i32_rtn:5734; GFX950-GISEL: ; %bb.0:5735; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]5736; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v05737; GFX950-GISEL-NEXT: s_nop 15738; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc5739; GFX950-GISEL-NEXT: flat_atomic_smax v0, v[2:3], v1 sc05740; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5741; GFX950-GISEL-NEXT: ; return to shader part epilog5742 %zext.offset = zext i32 %voffset to i645743 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset5744 %rtn = atomicrmw max ptr %gep0, i32 %data syncscope("workgroup") seq_cst5745 %cast.rtn = bitcast i32 %rtn to float5746 ret float %cast.rtn5747}5748 5749define amdgpu_ps float @flat_max_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {5750; GFX1250-LABEL: flat_max_saddr_i32_rtn_neg128:5751; GFX1250: ; %bb.0:5752; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 15753; GFX1250-NEXT: flat_atomic_max_i32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN5754; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x05755; GFX1250-NEXT: ; return to shader part epilog5756;5757; GFX950-SDAG-LABEL: flat_max_saddr_i32_rtn_neg128:5758; GFX950-SDAG: ; %bb.0:5759; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v15760; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 05761; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]5762; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v05763; GFX950-SDAG-NEXT: s_nop 15764; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc5765; GFX950-SDAG-NEXT: flat_atomic_smax v0, v[0:1], v2 sc05766; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5767; GFX950-SDAG-NEXT: ; return to shader part epilog5768;5769; GFX950-GISEL-LABEL: flat_max_saddr_i32_rtn_neg128:5770; GFX950-GISEL: ; %bb.0:5771; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]5772; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v05773; GFX950-GISEL-NEXT: s_nop 15774; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc5775; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v05776; GFX950-GISEL-NEXT: s_nop 15777; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc5778; GFX950-GISEL-NEXT: flat_atomic_smax v0, v[2:3], v1 sc05779; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)5780; GFX950-GISEL-NEXT: ; return to shader part epilog5781 %zext.offset = zext i32 %voffset to i645782 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset5783 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1285784 %rtn = atomicrmw max ptr %gep1, i32 %data syncscope("workgroup") seq_cst5785 %cast.rtn = bitcast i32 %rtn to float5786 ret float %cast.rtn5787}5788 5789define amdgpu_ps void @flat_max_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {5790; GFX1250-LABEL: flat_max_saddr_i32_nortn:5791; GFX1250: ; %bb.0:5792; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 15793; GFX1250-NEXT: flat_atomic_max_i32 v0, v1, s[2:3]5794; GFX1250-NEXT: s_wait_storecnt_dscnt 0x05795; GFX1250-NEXT: s_endpgm5796;5797; GFX950-SDAG-LABEL: flat_max_saddr_i32_nortn:5798; GFX950-SDAG: ; %bb.0:5799; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v15800; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 05801; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]5802; GFX950-SDAG-NEXT: flat_atomic_smax v[0:1], v25803; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)5804; GFX950-SDAG-NEXT: s_endpgm5805;5806; GFX950-GISEL-LABEL: flat_max_saddr_i32_nortn:5807; GFX950-GISEL: ; %bb.0:5808; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]5809; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v05810; GFX950-GISEL-NEXT: s_nop 15811; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc5812; GFX950-GISEL-NEXT: flat_atomic_smax v[2:3], v15813; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)5814; GFX950-GISEL-NEXT: s_endpgm5815 %zext.offset = zext i32 %voffset to i645816 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset5817 %unused = atomicrmw max ptr %gep0, i32 %data syncscope("workgroup") seq_cst5818 ret void5819}5820 5821define amdgpu_ps void @flat_max_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {5822; GFX1250-LABEL: flat_max_saddr_i32_nortn_neg128:5823; GFX1250: ; %bb.0:5824; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 15825; GFX1250-NEXT: flat_atomic_max_i32 v0, v1, s[2:3] offset:-1285826; GFX1250-NEXT: s_wait_storecnt_dscnt 0x05827; GFX1250-NEXT: s_endpgm5828;5829; GFX950-SDAG-LABEL: flat_max_saddr_i32_nortn_neg128:5830; GFX950-SDAG: ; %bb.0:5831; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v15832; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 05833; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]5834; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v05835; GFX950-SDAG-NEXT: s_nop 15836; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc5837; GFX950-SDAG-NEXT: flat_atomic_smax v[0:1], v25838; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)5839; GFX950-SDAG-NEXT: s_endpgm5840;5841; GFX950-GISEL-LABEL: flat_max_saddr_i32_nortn_neg128:5842; GFX950-GISEL: ; %bb.0:5843; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]5844; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v05845; GFX950-GISEL-NEXT: s_nop 15846; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc5847; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v05848; GFX950-GISEL-NEXT: s_nop 15849; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc5850; GFX950-GISEL-NEXT: flat_atomic_smax v[2:3], v15851; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)5852; GFX950-GISEL-NEXT: s_endpgm5853 %zext.offset = zext i32 %voffset to i645854 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset5855 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1285856 %unused = atomicrmw max ptr %gep1, i32 %data syncscope("workgroup") seq_cst5857 ret void5858}5859 5860define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {5861; GFX1250-SDAG-LABEL: flat_max_saddr_i64_rtn:5862; GFX1250-SDAG: ; %bb.0:5863; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 15864; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v15865; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 05866; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5867; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]5868; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v55869; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)5870; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v05871; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr15872; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo5873; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s05874; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB58_35875; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow5876; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s05877; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB58_45878; GFX1250-SDAG-NEXT: .LBB58_2: ; %atomicrmw.phi5879; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s05880; GFX1250-SDAG-NEXT: s_branch .LBB58_55881; GFX1250-SDAG-NEXT: .LBB58_3: ; %atomicrmw.global5882; GFX1250-SDAG-NEXT: flat_atomic_max_i64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN5883; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr55884; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr35885; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x05886; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s05887; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB58_25888; GFX1250-SDAG-NEXT: .LBB58_4: ; %atomicrmw.private5889; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]5890; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v45891; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)5892; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo5893; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off5894; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x05895; GFX1250-SDAG-NEXT: v_max_i64 v[2:3], v[0:1], v[2:3]5896; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off5897; GFX1250-SDAG-NEXT: s_wait_xcnt 0x05898; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s05899; GFX1250-SDAG-NEXT: s_branch .LBB58_55900; GFX1250-SDAG-NEXT: .LBB58_5:5901;5902; GFX1250-GISEL-LABEL: flat_max_saddr_i64_rtn:5903; GFX1250-GISEL: ; %bb.0:5904; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 15905; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v15906; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]5907; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5908; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v35909; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo5910; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5911; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x145912; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v05913; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr15914; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo5915; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5916; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s05917; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB58_35918; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow5919; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s05920; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB58_45921; GFX1250-GISEL-NEXT: .LBB58_2: ; %atomicrmw.phi5922; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s05923; GFX1250-GISEL-NEXT: s_branch .LBB58_55924; GFX1250-GISEL-NEXT: .LBB58_3: ; %atomicrmw.global5925; GFX1250-GISEL-NEXT: flat_atomic_max_i64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN5926; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr65927; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr55928; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x05929; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s05930; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB58_25931; GFX1250-GISEL-NEXT: .LBB58_4: ; %atomicrmw.private5932; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]5933; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v65934; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)5935; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo5936; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off5937; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x05938; GFX1250-GISEL-NEXT: v_max_i64 v[2:3], v[0:1], v[4:5]5939; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off5940; GFX1250-GISEL-NEXT: s_wait_xcnt 0x05941; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s05942; GFX1250-GISEL-NEXT: s_branch .LBB58_55943; GFX1250-GISEL-NEXT: .LBB58_5:5944;5945; GFX950-SDAG-LABEL: flat_max_saddr_i64_rtn:5946; GFX950-SDAG: ; %bb.0:5947; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v25948; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v15949; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 05950; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base5951; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1]5952; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v55953; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr15954; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc5955; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]5956; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB58_35957; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow5958; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]5959; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB58_45960; GFX950-SDAG-NEXT: .LBB58_2: ; %atomicrmw.phi5961; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]5962; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)5963; GFX950-SDAG-NEXT: s_branch .LBB58_55964; GFX950-SDAG-NEXT: .LBB58_3: ; %atomicrmw.global5965; GFX950-SDAG-NEXT: flat_atomic_smax_x2 v[0:1], v[4:5], v[2:3] sc05966; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)5967; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr55968; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr35969; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]5970; GFX950-SDAG-NEXT: s_cbranch_execz .LBB58_25971; GFX950-SDAG-NEXT: .LBB58_4: ; %atomicrmw.private5972; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]5973; GFX950-SDAG-NEXT: s_nop 15974; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc5975; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off5976; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)5977; GFX950-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]5978; GFX950-SDAG-NEXT: s_nop 15979; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc5980; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc5981; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off5982; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]5983; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)5984; GFX950-SDAG-NEXT: s_branch .LBB58_55985; GFX950-SDAG-NEXT: .LBB58_5:5986;5987; GFX950-GISEL-LABEL: flat_max_saddr_i64_rtn:5988; GFX950-GISEL: ; %bb.0:5989; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v25990; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]5991; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v05992; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base5993; GFX950-GISEL-NEXT: s_nop 05994; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc5995; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v15996; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v35997; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr15998; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc5999; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]6000; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB58_36001; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow6002; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6003; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB58_46004; GFX950-GISEL-NEXT: .LBB58_2: ; %atomicrmw.phi6005; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]6006; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)6007; GFX950-GISEL-NEXT: s_branch .LBB58_56008; GFX950-GISEL-NEXT: .LBB58_3: ; %atomicrmw.global6009; GFX950-GISEL-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3], v[4:5] sc06010; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)6011; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr36012; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr56013; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6014; GFX950-GISEL-NEXT: s_cbranch_execz .LBB58_26015; GFX950-GISEL-NEXT: .LBB58_4: ; %atomicrmw.private6016; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]6017; GFX950-GISEL-NEXT: s_nop 16018; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc6019; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off6020; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)6021; GFX950-GISEL-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[4:5]6022; GFX950-GISEL-NEXT: s_nop 16023; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v0, vcc6024; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v1, vcc6025; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off6026; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]6027; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)6028; GFX950-GISEL-NEXT: s_branch .LBB58_56029; GFX950-GISEL-NEXT: .LBB58_5:6030 %zext.offset = zext i32 %voffset to i646031 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset6032 %rtn = atomicrmw max ptr %gep0, i64 %data syncscope("workgroup") seq_cst6033 %cast.rtn = bitcast i64 %rtn to <2 x float>6034 ret <2 x float> %cast.rtn6035}6036 6037define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {6038; GFX1250-SDAG-LABEL: flat_max_saddr_i64_rtn_neg128:6039; GFX1250-SDAG: ; %bb.0:6040; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 16041; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v16042; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 06043; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff806044; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6045; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]6046; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]6047; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6048; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v56049; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v06050; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr16051; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo6052; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6053; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s06054; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB59_36055; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow6056; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s06057; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB59_46058; GFX1250-SDAG-NEXT: .LBB59_2: ; %atomicrmw.phi6059; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s06060; GFX1250-SDAG-NEXT: s_branch .LBB59_56061; GFX1250-SDAG-NEXT: .LBB59_3: ; %atomicrmw.global6062; GFX1250-SDAG-NEXT: flat_atomic_max_i64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN6063; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr56064; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr36065; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x06066; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s06067; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB59_26068; GFX1250-SDAG-NEXT: .LBB59_4: ; %atomicrmw.private6069; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]6070; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v46071; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)6072; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo6073; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off6074; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x06075; GFX1250-SDAG-NEXT: v_max_i64 v[2:3], v[0:1], v[2:3]6076; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off6077; GFX1250-SDAG-NEXT: s_wait_xcnt 0x06078; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s06079; GFX1250-SDAG-NEXT: s_branch .LBB59_56080; GFX1250-SDAG-NEXT: .LBB59_5:6081;6082; GFX1250-GISEL-LABEL: flat_max_saddr_i64_rtn_neg128:6083; GFX1250-GISEL: ; %bb.0:6084; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 16085; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v16086; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]6087; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6088; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v36089; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo6090; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)6091; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v06092; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo6093; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6094; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x146095; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v06096; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr16097; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo6098; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6099; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s06100; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB59_36101; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow6102; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s06103; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB59_46104; GFX1250-GISEL-NEXT: .LBB59_2: ; %atomicrmw.phi6105; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s06106; GFX1250-GISEL-NEXT: s_branch .LBB59_56107; GFX1250-GISEL-NEXT: .LBB59_3: ; %atomicrmw.global6108; GFX1250-GISEL-NEXT: flat_atomic_max_i64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN6109; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr66110; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr56111; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x06112; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s06113; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB59_26114; GFX1250-GISEL-NEXT: .LBB59_4: ; %atomicrmw.private6115; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]6116; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v66117; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)6118; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo6119; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off6120; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x06121; GFX1250-GISEL-NEXT: v_max_i64 v[2:3], v[0:1], v[4:5]6122; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off6123; GFX1250-GISEL-NEXT: s_wait_xcnt 0x06124; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s06125; GFX1250-GISEL-NEXT: s_branch .LBB59_56126; GFX1250-GISEL-NEXT: .LBB59_5:6127;6128; GFX950-SDAG-LABEL: flat_max_saddr_i64_rtn_neg128:6129; GFX950-SDAG: ; %bb.0:6130; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v26131; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v16132; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 06133; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]6134; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff806135; GFX950-SDAG-NEXT: s_mov_b32 s3, -16136; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base6137; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]6138; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v56139; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr16140; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc6141; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]6142; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB59_36143; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow6144; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6145; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB59_46146; GFX950-SDAG-NEXT: .LBB59_2: ; %atomicrmw.phi6147; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]6148; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)6149; GFX950-SDAG-NEXT: s_branch .LBB59_56150; GFX950-SDAG-NEXT: .LBB59_3: ; %atomicrmw.global6151; GFX950-SDAG-NEXT: flat_atomic_smax_x2 v[0:1], v[4:5], v[2:3] sc06152; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)6153; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr56154; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr36155; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6156; GFX950-SDAG-NEXT: s_cbranch_execz .LBB59_26157; GFX950-SDAG-NEXT: .LBB59_4: ; %atomicrmw.private6158; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]6159; GFX950-SDAG-NEXT: s_nop 16160; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc6161; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off6162; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)6163; GFX950-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]6164; GFX950-SDAG-NEXT: s_nop 16165; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc6166; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc6167; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off6168; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]6169; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)6170; GFX950-SDAG-NEXT: s_branch .LBB59_56171; GFX950-SDAG-NEXT: .LBB59_5:6172;6173; GFX950-GISEL-LABEL: flat_max_saddr_i64_rtn_neg128:6174; GFX950-GISEL: ; %bb.0:6175; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v26176; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]6177; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v06178; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v16179; GFX950-GISEL-NEXT: s_nop 06180; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc6181; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v06182; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base6183; GFX950-GISEL-NEXT: s_nop 06184; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v1, vcc6185; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v36186; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr16187; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc6188; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]6189; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB59_36190; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow6191; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6192; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB59_46193; GFX950-GISEL-NEXT: .LBB59_2: ; %atomicrmw.phi6194; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]6195; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)6196; GFX950-GISEL-NEXT: s_branch .LBB59_56197; GFX950-GISEL-NEXT: .LBB59_3: ; %atomicrmw.global6198; GFX950-GISEL-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3], v[4:5] sc06199; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)6200; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr36201; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr56202; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6203; GFX950-GISEL-NEXT: s_cbranch_execz .LBB59_26204; GFX950-GISEL-NEXT: .LBB59_4: ; %atomicrmw.private6205; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]6206; GFX950-GISEL-NEXT: s_nop 16207; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc6208; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off6209; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)6210; GFX950-GISEL-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[4:5]6211; GFX950-GISEL-NEXT: s_nop 16212; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v0, vcc6213; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v1, vcc6214; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off6215; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]6216; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)6217; GFX950-GISEL-NEXT: s_branch .LBB59_56218; GFX950-GISEL-NEXT: .LBB59_5:6219 %zext.offset = zext i32 %voffset to i646220 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset6221 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1286222 %rtn = atomicrmw max ptr %gep1, i64 %data syncscope("workgroup") seq_cst6223 %cast.rtn = bitcast i64 %rtn to <2 x float>6224 ret <2 x float> %cast.rtn6225}6226 6227define amdgpu_ps void @flat_max_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {6228; GFX1250-SDAG-LABEL: flat_max_saddr_i64_nortn:6229; GFX1250-SDAG: ; %bb.0:6230; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 16231; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v16232; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 06233; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo6234; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6235; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]6236; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v16237; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)6238; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v46239; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s06240; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB60_36241; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow6242; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s06243; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB60_46244; GFX1250-SDAG-NEXT: .LBB60_2: ; %atomicrmw.phi6245; GFX1250-SDAG-NEXT: s_endpgm6246; GFX1250-SDAG-NEXT: .LBB60_3: ; %atomicrmw.global6247; GFX1250-SDAG-NEXT: flat_atomic_max_i64 v[0:1], v[2:3]6248; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr16249; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr36250; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x06251; GFX1250-SDAG-NEXT: s_wait_xcnt 0x06252; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s06253; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB60_26254; GFX1250-SDAG-NEXT: .LBB60_4: ; %atomicrmw.private6255; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]6256; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v06257; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)6258; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo6259; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off6260; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x06261; GFX1250-SDAG-NEXT: v_max_i64 v[0:1], v[0:1], v[2:3]6262; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off6263; GFX1250-SDAG-NEXT: s_endpgm6264;6265; GFX1250-GISEL-LABEL: flat_max_saddr_i64_nortn:6266; GFX1250-GISEL: ; %bb.0:6267; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 16268; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v26269; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]6270; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo6271; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6272; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v06273; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo6274; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6275; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v36276; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v16277; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s06278; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB60_36279; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow6280; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s06281; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB60_46282; GFX1250-GISEL-NEXT: .LBB60_2: ; %atomicrmw.phi6283; GFX1250-GISEL-NEXT: s_endpgm6284; GFX1250-GISEL-NEXT: .LBB60_3: ; %atomicrmw.global6285; GFX1250-GISEL-NEXT: flat_atomic_max_i64 v0, v[4:5], s[2:3]6286; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr26287; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr56288; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x06289; GFX1250-GISEL-NEXT: s_wait_xcnt 0x06290; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s06291; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB60_26292; GFX1250-GISEL-NEXT: .LBB60_4: ; %atomicrmw.private6293; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]6294; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v26295; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)6296; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo6297; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off6298; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x06299; GFX1250-GISEL-NEXT: v_max_i64 v[0:1], v[0:1], v[4:5]6300; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off6301; GFX1250-GISEL-NEXT: s_endpgm6302;6303; GFX950-SDAG-LABEL: flat_max_saddr_i64_nortn:6304; GFX950-SDAG: ; %bb.0:6305; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v26306; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v16307; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 06308; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base6309; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]6310; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v16311; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc6312; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]6313; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB60_36314; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow6315; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6316; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB60_46317; GFX950-SDAG-NEXT: .LBB60_2: ; %atomicrmw.phi6318; GFX950-SDAG-NEXT: s_endpgm6319; GFX950-SDAG-NEXT: .LBB60_3: ; %atomicrmw.global6320; GFX950-SDAG-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3]6321; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)6322; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr16323; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr36324; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6325; GFX950-SDAG-NEXT: s_cbranch_execz .LBB60_26326; GFX950-SDAG-NEXT: .LBB60_4: ; %atomicrmw.private6327; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]6328; GFX950-SDAG-NEXT: s_nop 16329; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc6330; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off6331; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)6332; GFX950-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]6333; GFX950-SDAG-NEXT: s_nop 16334; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc6335; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc6336; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[0:1], off6337; GFX950-SDAG-NEXT: s_endpgm6338;6339; GFX950-GISEL-LABEL: flat_max_saddr_i64_nortn:6340; GFX950-GISEL: ; %bb.0:6341; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v26342; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]6343; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v06344; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v16345; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base6346; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc6347; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v16348; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc6349; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]6350; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB60_36351; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow6352; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6353; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB60_46354; GFX950-GISEL-NEXT: .LBB60_2: ; %atomicrmw.phi6355; GFX950-GISEL-NEXT: s_endpgm6356; GFX950-GISEL-NEXT: .LBB60_3: ; %atomicrmw.global6357; GFX950-GISEL-NEXT: flat_atomic_smax_x2 v[0:1], v[4:5]6358; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)6359; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr16360; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr56361; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6362; GFX950-GISEL-NEXT: s_cbranch_execz .LBB60_26363; GFX950-GISEL-NEXT: .LBB60_4: ; %atomicrmw.private6364; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]6365; GFX950-GISEL-NEXT: s_nop 16366; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc6367; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off6368; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)6369; GFX950-GISEL-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[4:5]6370; GFX950-GISEL-NEXT: s_nop 16371; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc6372; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc6373; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off6374; GFX950-GISEL-NEXT: s_endpgm6375 %zext.offset = zext i32 %voffset to i646376 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset6377 %unused = atomicrmw max ptr %gep0, i64 %data syncscope("workgroup") seq_cst6378 ret void6379}6380 6381define amdgpu_ps void @flat_max_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {6382; GFX1250-SDAG-LABEL: flat_max_saddr_i64_nortn_neg128:6383; GFX1250-SDAG: ; %bb.0:6384; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 16385; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v16386; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 06387; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff806388; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6389; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]6390; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]6391; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo6392; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6393; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v16394; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v46395; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s06396; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB61_36397; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow6398; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s06399; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB61_46400; GFX1250-SDAG-NEXT: .LBB61_2: ; %atomicrmw.phi6401; GFX1250-SDAG-NEXT: s_endpgm6402; GFX1250-SDAG-NEXT: .LBB61_3: ; %atomicrmw.global6403; GFX1250-SDAG-NEXT: flat_atomic_max_i64 v[0:1], v[2:3]6404; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr16405; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr36406; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x06407; GFX1250-SDAG-NEXT: s_wait_xcnt 0x06408; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s06409; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB61_26410; GFX1250-SDAG-NEXT: .LBB61_4: ; %atomicrmw.private6411; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]6412; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v06413; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)6414; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo6415; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off6416; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x06417; GFX1250-SDAG-NEXT: v_max_i64 v[0:1], v[0:1], v[2:3]6418; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off6419; GFX1250-SDAG-NEXT: s_endpgm6420;6421; GFX1250-GISEL-LABEL: flat_max_saddr_i64_nortn_neg128:6422; GFX1250-GISEL: ; %bb.0:6423; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 16424; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v26425; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]6426; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo6427; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6428; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v06429; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo6430; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)6431; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v16432; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo6433; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6434; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v36435; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v16436; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s06437; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB61_36438; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow6439; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s06440; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB61_46441; GFX1250-GISEL-NEXT: .LBB61_2: ; %atomicrmw.phi6442; GFX1250-GISEL-NEXT: s_endpgm6443; GFX1250-GISEL-NEXT: .LBB61_3: ; %atomicrmw.global6444; GFX1250-GISEL-NEXT: flat_atomic_max_i64 v0, v[4:5], s[2:3] offset:-1286445; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr26446; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr56447; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x06448; GFX1250-GISEL-NEXT: s_wait_xcnt 0x06449; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s06450; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB61_26451; GFX1250-GISEL-NEXT: .LBB61_4: ; %atomicrmw.private6452; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]6453; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v26454; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)6455; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo6456; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off6457; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x06458; GFX1250-GISEL-NEXT: v_max_i64 v[0:1], v[0:1], v[4:5]6459; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off6460; GFX1250-GISEL-NEXT: s_endpgm6461;6462; GFX950-SDAG-LABEL: flat_max_saddr_i64_nortn_neg128:6463; GFX950-SDAG: ; %bb.0:6464; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v26465; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v16466; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 06467; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]6468; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff806469; GFX950-SDAG-NEXT: s_mov_b32 s3, -16470; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base6471; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]6472; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v16473; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc6474; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]6475; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB61_36476; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow6477; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6478; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB61_46479; GFX950-SDAG-NEXT: .LBB61_2: ; %atomicrmw.phi6480; GFX950-SDAG-NEXT: s_endpgm6481; GFX950-SDAG-NEXT: .LBB61_3: ; %atomicrmw.global6482; GFX950-SDAG-NEXT: flat_atomic_smax_x2 v[0:1], v[2:3]6483; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)6484; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr16485; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr36486; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6487; GFX950-SDAG-NEXT: s_cbranch_execz .LBB61_26488; GFX950-SDAG-NEXT: .LBB61_4: ; %atomicrmw.private6489; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]6490; GFX950-SDAG-NEXT: s_nop 16491; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc6492; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off6493; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)6494; GFX950-SDAG-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[2:3]6495; GFX950-SDAG-NEXT: s_nop 16496; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc6497; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc6498; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[0:1], off6499; GFX950-SDAG-NEXT: s_endpgm6500;6501; GFX950-GISEL-LABEL: flat_max_saddr_i64_nortn_neg128:6502; GFX950-GISEL: ; %bb.0:6503; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v26504; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]6505; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v06506; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v16507; GFX950-GISEL-NEXT: s_nop 06508; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc6509; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v06510; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base6511; GFX950-GISEL-NEXT: s_nop 06512; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc6513; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v16514; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc6515; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]6516; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB61_36517; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow6518; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6519; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB61_46520; GFX950-GISEL-NEXT: .LBB61_2: ; %atomicrmw.phi6521; GFX950-GISEL-NEXT: s_endpgm6522; GFX950-GISEL-NEXT: .LBB61_3: ; %atomicrmw.global6523; GFX950-GISEL-NEXT: flat_atomic_smax_x2 v[0:1], v[4:5]6524; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)6525; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr16526; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr56527; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6528; GFX950-GISEL-NEXT: s_cbranch_execz .LBB61_26529; GFX950-GISEL-NEXT: .LBB61_4: ; %atomicrmw.private6530; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]6531; GFX950-GISEL-NEXT: s_nop 16532; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc6533; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off6534; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)6535; GFX950-GISEL-NEXT: v_cmp_gt_i64_e32 vcc, v[0:1], v[4:5]6536; GFX950-GISEL-NEXT: s_nop 16537; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc6538; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc6539; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off6540; GFX950-GISEL-NEXT: s_endpgm6541 %zext.offset = zext i32 %voffset to i646542 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset6543 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1286544 %unused = atomicrmw max ptr %gep1, i64 %data syncscope("workgroup") seq_cst6545 ret void6546}6547 6548; --------------------------------------------------------------------------------6549; atomicrmw min6550; --------------------------------------------------------------------------------6551 6552define amdgpu_ps float @flat_min_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {6553; GFX1250-LABEL: flat_min_saddr_i32_rtn:6554; GFX1250: ; %bb.0:6555; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 16556; GFX1250-NEXT: flat_atomic_min_i32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN6557; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x06558; GFX1250-NEXT: ; return to shader part epilog6559;6560; GFX950-SDAG-LABEL: flat_min_saddr_i32_rtn:6561; GFX950-SDAG: ; %bb.0:6562; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v16563; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 06564; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]6565; GFX950-SDAG-NEXT: flat_atomic_smin v0, v[0:1], v2 sc06566; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6567; GFX950-SDAG-NEXT: ; return to shader part epilog6568;6569; GFX950-GISEL-LABEL: flat_min_saddr_i32_rtn:6570; GFX950-GISEL: ; %bb.0:6571; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]6572; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v06573; GFX950-GISEL-NEXT: s_nop 16574; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc6575; GFX950-GISEL-NEXT: flat_atomic_smin v0, v[2:3], v1 sc06576; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6577; GFX950-GISEL-NEXT: ; return to shader part epilog6578 %zext.offset = zext i32 %voffset to i646579 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset6580 %rtn = atomicrmw min ptr %gep0, i32 %data syncscope("workgroup") seq_cst6581 %cast.rtn = bitcast i32 %rtn to float6582 ret float %cast.rtn6583}6584 6585define amdgpu_ps float @flat_min_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {6586; GFX1250-LABEL: flat_min_saddr_i32_rtn_neg128:6587; GFX1250: ; %bb.0:6588; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 16589; GFX1250-NEXT: flat_atomic_min_i32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN6590; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x06591; GFX1250-NEXT: ; return to shader part epilog6592;6593; GFX950-SDAG-LABEL: flat_min_saddr_i32_rtn_neg128:6594; GFX950-SDAG: ; %bb.0:6595; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v16596; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 06597; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]6598; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v06599; GFX950-SDAG-NEXT: s_nop 16600; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc6601; GFX950-SDAG-NEXT: flat_atomic_smin v0, v[0:1], v2 sc06602; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6603; GFX950-SDAG-NEXT: ; return to shader part epilog6604;6605; GFX950-GISEL-LABEL: flat_min_saddr_i32_rtn_neg128:6606; GFX950-GISEL: ; %bb.0:6607; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]6608; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v06609; GFX950-GISEL-NEXT: s_nop 16610; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc6611; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v06612; GFX950-GISEL-NEXT: s_nop 16613; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc6614; GFX950-GISEL-NEXT: flat_atomic_smin v0, v[2:3], v1 sc06615; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)6616; GFX950-GISEL-NEXT: ; return to shader part epilog6617 %zext.offset = zext i32 %voffset to i646618 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset6619 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1286620 %rtn = atomicrmw min ptr %gep1, i32 %data syncscope("workgroup") seq_cst6621 %cast.rtn = bitcast i32 %rtn to float6622 ret float %cast.rtn6623}6624 6625define amdgpu_ps void @flat_min_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {6626; GFX1250-LABEL: flat_min_saddr_i32_nortn:6627; GFX1250: ; %bb.0:6628; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 16629; GFX1250-NEXT: flat_atomic_min_i32 v0, v1, s[2:3]6630; GFX1250-NEXT: s_wait_storecnt_dscnt 0x06631; GFX1250-NEXT: s_endpgm6632;6633; GFX950-SDAG-LABEL: flat_min_saddr_i32_nortn:6634; GFX950-SDAG: ; %bb.0:6635; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v16636; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 06637; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]6638; GFX950-SDAG-NEXT: flat_atomic_smin v[0:1], v26639; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)6640; GFX950-SDAG-NEXT: s_endpgm6641;6642; GFX950-GISEL-LABEL: flat_min_saddr_i32_nortn:6643; GFX950-GISEL: ; %bb.0:6644; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]6645; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v06646; GFX950-GISEL-NEXT: s_nop 16647; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc6648; GFX950-GISEL-NEXT: flat_atomic_smin v[2:3], v16649; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)6650; GFX950-GISEL-NEXT: s_endpgm6651 %zext.offset = zext i32 %voffset to i646652 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset6653 %unused = atomicrmw min ptr %gep0, i32 %data syncscope("workgroup") seq_cst6654 ret void6655}6656 6657define amdgpu_ps void @flat_min_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {6658; GFX1250-LABEL: flat_min_saddr_i32_nortn_neg128:6659; GFX1250: ; %bb.0:6660; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 16661; GFX1250-NEXT: flat_atomic_min_i32 v0, v1, s[2:3] offset:-1286662; GFX1250-NEXT: s_wait_storecnt_dscnt 0x06663; GFX1250-NEXT: s_endpgm6664;6665; GFX950-SDAG-LABEL: flat_min_saddr_i32_nortn_neg128:6666; GFX950-SDAG: ; %bb.0:6667; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v16668; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 06669; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]6670; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v06671; GFX950-SDAG-NEXT: s_nop 16672; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc6673; GFX950-SDAG-NEXT: flat_atomic_smin v[0:1], v26674; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)6675; GFX950-SDAG-NEXT: s_endpgm6676;6677; GFX950-GISEL-LABEL: flat_min_saddr_i32_nortn_neg128:6678; GFX950-GISEL: ; %bb.0:6679; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]6680; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v06681; GFX950-GISEL-NEXT: s_nop 16682; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc6683; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v06684; GFX950-GISEL-NEXT: s_nop 16685; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc6686; GFX950-GISEL-NEXT: flat_atomic_smin v[2:3], v16687; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)6688; GFX950-GISEL-NEXT: s_endpgm6689 %zext.offset = zext i32 %voffset to i646690 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset6691 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1286692 %unused = atomicrmw min ptr %gep1, i32 %data syncscope("workgroup") seq_cst6693 ret void6694}6695 6696define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {6697; GFX1250-SDAG-LABEL: flat_min_saddr_i64_rtn:6698; GFX1250-SDAG: ; %bb.0:6699; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 16700; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v16701; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 06702; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6703; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]6704; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v56705; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)6706; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v06707; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr16708; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo6709; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s06710; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB66_36711; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow6712; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s06713; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB66_46714; GFX1250-SDAG-NEXT: .LBB66_2: ; %atomicrmw.phi6715; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s06716; GFX1250-SDAG-NEXT: s_branch .LBB66_56717; GFX1250-SDAG-NEXT: .LBB66_3: ; %atomicrmw.global6718; GFX1250-SDAG-NEXT: flat_atomic_min_i64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN6719; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr56720; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr36721; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x06722; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s06723; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB66_26724; GFX1250-SDAG-NEXT: .LBB66_4: ; %atomicrmw.private6725; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]6726; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v46727; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)6728; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo6729; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off6730; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x06731; GFX1250-SDAG-NEXT: v_min_i64 v[2:3], v[0:1], v[2:3]6732; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off6733; GFX1250-SDAG-NEXT: s_wait_xcnt 0x06734; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s06735; GFX1250-SDAG-NEXT: s_branch .LBB66_56736; GFX1250-SDAG-NEXT: .LBB66_5:6737;6738; GFX1250-GISEL-LABEL: flat_min_saddr_i64_rtn:6739; GFX1250-GISEL: ; %bb.0:6740; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 16741; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v16742; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]6743; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6744; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v36745; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo6746; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6747; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x146748; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v06749; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr16750; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo6751; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6752; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s06753; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB66_36754; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow6755; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s06756; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB66_46757; GFX1250-GISEL-NEXT: .LBB66_2: ; %atomicrmw.phi6758; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s06759; GFX1250-GISEL-NEXT: s_branch .LBB66_56760; GFX1250-GISEL-NEXT: .LBB66_3: ; %atomicrmw.global6761; GFX1250-GISEL-NEXT: flat_atomic_min_i64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN6762; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr66763; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr56764; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x06765; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s06766; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB66_26767; GFX1250-GISEL-NEXT: .LBB66_4: ; %atomicrmw.private6768; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]6769; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v66770; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)6771; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo6772; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off6773; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x06774; GFX1250-GISEL-NEXT: v_min_i64 v[2:3], v[0:1], v[4:5]6775; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off6776; GFX1250-GISEL-NEXT: s_wait_xcnt 0x06777; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s06778; GFX1250-GISEL-NEXT: s_branch .LBB66_56779; GFX1250-GISEL-NEXT: .LBB66_5:6780;6781; GFX950-SDAG-LABEL: flat_min_saddr_i64_rtn:6782; GFX950-SDAG: ; %bb.0:6783; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v26784; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v16785; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 06786; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base6787; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1]6788; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v56789; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr16790; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc6791; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]6792; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB66_36793; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow6794; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6795; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB66_46796; GFX950-SDAG-NEXT: .LBB66_2: ; %atomicrmw.phi6797; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]6798; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)6799; GFX950-SDAG-NEXT: s_branch .LBB66_56800; GFX950-SDAG-NEXT: .LBB66_3: ; %atomicrmw.global6801; GFX950-SDAG-NEXT: flat_atomic_smin_x2 v[0:1], v[4:5], v[2:3] sc06802; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)6803; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr56804; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr36805; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6806; GFX950-SDAG-NEXT: s_cbranch_execz .LBB66_26807; GFX950-SDAG-NEXT: .LBB66_4: ; %atomicrmw.private6808; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]6809; GFX950-SDAG-NEXT: s_nop 16810; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc6811; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off6812; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)6813; GFX950-SDAG-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]6814; GFX950-SDAG-NEXT: s_nop 16815; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc6816; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc6817; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off6818; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]6819; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)6820; GFX950-SDAG-NEXT: s_branch .LBB66_56821; GFX950-SDAG-NEXT: .LBB66_5:6822;6823; GFX950-GISEL-LABEL: flat_min_saddr_i64_rtn:6824; GFX950-GISEL: ; %bb.0:6825; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v26826; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]6827; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v06828; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base6829; GFX950-GISEL-NEXT: s_nop 06830; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc6831; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v16832; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v36833; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr16834; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc6835; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]6836; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB66_36837; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow6838; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6839; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB66_46840; GFX950-GISEL-NEXT: .LBB66_2: ; %atomicrmw.phi6841; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]6842; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)6843; GFX950-GISEL-NEXT: s_branch .LBB66_56844; GFX950-GISEL-NEXT: .LBB66_3: ; %atomicrmw.global6845; GFX950-GISEL-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3], v[4:5] sc06846; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)6847; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr36848; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr56849; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6850; GFX950-GISEL-NEXT: s_cbranch_execz .LBB66_26851; GFX950-GISEL-NEXT: .LBB66_4: ; %atomicrmw.private6852; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]6853; GFX950-GISEL-NEXT: s_nop 16854; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc6855; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off6856; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)6857; GFX950-GISEL-NEXT: v_cmp_lt_i64_e32 vcc, v[0:1], v[4:5]6858; GFX950-GISEL-NEXT: s_nop 16859; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v0, vcc6860; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v1, vcc6861; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off6862; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]6863; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)6864; GFX950-GISEL-NEXT: s_branch .LBB66_56865; GFX950-GISEL-NEXT: .LBB66_5:6866 %zext.offset = zext i32 %voffset to i646867 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset6868 %rtn = atomicrmw min ptr %gep0, i64 %data syncscope("workgroup") seq_cst6869 %cast.rtn = bitcast i64 %rtn to <2 x float>6870 ret <2 x float> %cast.rtn6871}6872 6873define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {6874; GFX1250-SDAG-LABEL: flat_min_saddr_i64_rtn_neg128:6875; GFX1250-SDAG: ; %bb.0:6876; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 16877; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v16878; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 06879; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff806880; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6881; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]6882; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]6883; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6884; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v56885; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v06886; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr16887; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo6888; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6889; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s06890; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB67_36891; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow6892; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s06893; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB67_46894; GFX1250-SDAG-NEXT: .LBB67_2: ; %atomicrmw.phi6895; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s06896; GFX1250-SDAG-NEXT: s_branch .LBB67_56897; GFX1250-SDAG-NEXT: .LBB67_3: ; %atomicrmw.global6898; GFX1250-SDAG-NEXT: flat_atomic_min_i64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN6899; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr56900; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr36901; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x06902; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s06903; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB67_26904; GFX1250-SDAG-NEXT: .LBB67_4: ; %atomicrmw.private6905; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]6906; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v46907; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)6908; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo6909; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off6910; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x06911; GFX1250-SDAG-NEXT: v_min_i64 v[2:3], v[0:1], v[2:3]6912; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off6913; GFX1250-SDAG-NEXT: s_wait_xcnt 0x06914; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s06915; GFX1250-SDAG-NEXT: s_branch .LBB67_56916; GFX1250-SDAG-NEXT: .LBB67_5:6917;6918; GFX1250-GISEL-LABEL: flat_min_saddr_i64_rtn_neg128:6919; GFX1250-GISEL: ; %bb.0:6920; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 16921; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v16922; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]6923; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6924; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v36925; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo6926; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)6927; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v06928; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo6929; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6930; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x146931; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v06932; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr16933; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo6934; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6935; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s06936; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB67_36937; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow6938; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s06939; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB67_46940; GFX1250-GISEL-NEXT: .LBB67_2: ; %atomicrmw.phi6941; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s06942; GFX1250-GISEL-NEXT: s_branch .LBB67_56943; GFX1250-GISEL-NEXT: .LBB67_3: ; %atomicrmw.global6944; GFX1250-GISEL-NEXT: flat_atomic_min_i64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN6945; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr66946; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr56947; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x06948; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s06949; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB67_26950; GFX1250-GISEL-NEXT: .LBB67_4: ; %atomicrmw.private6951; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]6952; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v66953; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)6954; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo6955; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off6956; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x06957; GFX1250-GISEL-NEXT: v_min_i64 v[2:3], v[0:1], v[4:5]6958; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off6959; GFX1250-GISEL-NEXT: s_wait_xcnt 0x06960; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s06961; GFX1250-GISEL-NEXT: s_branch .LBB67_56962; GFX1250-GISEL-NEXT: .LBB67_5:6963;6964; GFX950-SDAG-LABEL: flat_min_saddr_i64_rtn_neg128:6965; GFX950-SDAG: ; %bb.0:6966; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v26967; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v16968; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 06969; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]6970; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff806971; GFX950-SDAG-NEXT: s_mov_b32 s3, -16972; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base6973; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]6974; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v56975; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr16976; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc6977; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]6978; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB67_36979; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow6980; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6981; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB67_46982; GFX950-SDAG-NEXT: .LBB67_2: ; %atomicrmw.phi6983; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]6984; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)6985; GFX950-SDAG-NEXT: s_branch .LBB67_56986; GFX950-SDAG-NEXT: .LBB67_3: ; %atomicrmw.global6987; GFX950-SDAG-NEXT: flat_atomic_smin_x2 v[0:1], v[4:5], v[2:3] sc06988; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)6989; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr56990; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr36991; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]6992; GFX950-SDAG-NEXT: s_cbranch_execz .LBB67_26993; GFX950-SDAG-NEXT: .LBB67_4: ; %atomicrmw.private6994; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]6995; GFX950-SDAG-NEXT: s_nop 16996; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc6997; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off6998; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)6999; GFX950-SDAG-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]7000; GFX950-SDAG-NEXT: s_nop 17001; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc7002; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc7003; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off7004; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]7005; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)7006; GFX950-SDAG-NEXT: s_branch .LBB67_57007; GFX950-SDAG-NEXT: .LBB67_5:7008;7009; GFX950-GISEL-LABEL: flat_min_saddr_i64_rtn_neg128:7010; GFX950-GISEL: ; %bb.0:7011; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v27012; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]7013; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v07014; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v17015; GFX950-GISEL-NEXT: s_nop 07016; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc7017; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v07018; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base7019; GFX950-GISEL-NEXT: s_nop 07020; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v1, vcc7021; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v37022; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr17023; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc7024; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]7025; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB67_37026; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow7027; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7028; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB67_47029; GFX950-GISEL-NEXT: .LBB67_2: ; %atomicrmw.phi7030; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]7031; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)7032; GFX950-GISEL-NEXT: s_branch .LBB67_57033; GFX950-GISEL-NEXT: .LBB67_3: ; %atomicrmw.global7034; GFX950-GISEL-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3], v[4:5] sc07035; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)7036; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr37037; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr57038; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7039; GFX950-GISEL-NEXT: s_cbranch_execz .LBB67_27040; GFX950-GISEL-NEXT: .LBB67_4: ; %atomicrmw.private7041; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]7042; GFX950-GISEL-NEXT: s_nop 17043; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc7044; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off7045; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)7046; GFX950-GISEL-NEXT: v_cmp_lt_i64_e32 vcc, v[0:1], v[4:5]7047; GFX950-GISEL-NEXT: s_nop 17048; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v0, vcc7049; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v1, vcc7050; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off7051; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]7052; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)7053; GFX950-GISEL-NEXT: s_branch .LBB67_57054; GFX950-GISEL-NEXT: .LBB67_5:7055 %zext.offset = zext i32 %voffset to i647056 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset7057 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1287058 %rtn = atomicrmw min ptr %gep1, i64 %data syncscope("workgroup") seq_cst7059 %cast.rtn = bitcast i64 %rtn to <2 x float>7060 ret <2 x float> %cast.rtn7061}7062 7063define amdgpu_ps void @flat_min_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {7064; GFX1250-SDAG-LABEL: flat_min_saddr_i64_nortn:7065; GFX1250-SDAG: ; %bb.0:7066; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 17067; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v17068; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 07069; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo7070; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7071; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]7072; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v17073; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)7074; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v47075; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s07076; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB68_37077; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow7078; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s07079; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB68_47080; GFX1250-SDAG-NEXT: .LBB68_2: ; %atomicrmw.phi7081; GFX1250-SDAG-NEXT: s_endpgm7082; GFX1250-SDAG-NEXT: .LBB68_3: ; %atomicrmw.global7083; GFX1250-SDAG-NEXT: flat_atomic_min_i64 v[0:1], v[2:3]7084; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr17085; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr37086; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x07087; GFX1250-SDAG-NEXT: s_wait_xcnt 0x07088; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s07089; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB68_27090; GFX1250-SDAG-NEXT: .LBB68_4: ; %atomicrmw.private7091; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]7092; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v07093; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)7094; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo7095; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off7096; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x07097; GFX1250-SDAG-NEXT: v_min_i64 v[0:1], v[0:1], v[2:3]7098; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off7099; GFX1250-SDAG-NEXT: s_endpgm7100;7101; GFX1250-GISEL-LABEL: flat_min_saddr_i64_nortn:7102; GFX1250-GISEL: ; %bb.0:7103; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 17104; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v27105; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]7106; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo7107; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7108; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v07109; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo7110; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7111; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v37112; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v17113; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s07114; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB68_37115; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow7116; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s07117; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB68_47118; GFX1250-GISEL-NEXT: .LBB68_2: ; %atomicrmw.phi7119; GFX1250-GISEL-NEXT: s_endpgm7120; GFX1250-GISEL-NEXT: .LBB68_3: ; %atomicrmw.global7121; GFX1250-GISEL-NEXT: flat_atomic_min_i64 v0, v[4:5], s[2:3]7122; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr27123; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr57124; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x07125; GFX1250-GISEL-NEXT: s_wait_xcnt 0x07126; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s07127; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB68_27128; GFX1250-GISEL-NEXT: .LBB68_4: ; %atomicrmw.private7129; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]7130; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v27131; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)7132; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo7133; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off7134; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x07135; GFX1250-GISEL-NEXT: v_min_i64 v[0:1], v[0:1], v[4:5]7136; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off7137; GFX1250-GISEL-NEXT: s_endpgm7138;7139; GFX950-SDAG-LABEL: flat_min_saddr_i64_nortn:7140; GFX950-SDAG: ; %bb.0:7141; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v27142; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v17143; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 07144; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base7145; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]7146; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v17147; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc7148; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]7149; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB68_37150; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow7151; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7152; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB68_47153; GFX950-SDAG-NEXT: .LBB68_2: ; %atomicrmw.phi7154; GFX950-SDAG-NEXT: s_endpgm7155; GFX950-SDAG-NEXT: .LBB68_3: ; %atomicrmw.global7156; GFX950-SDAG-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3]7157; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)7158; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr17159; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr37160; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7161; GFX950-SDAG-NEXT: s_cbranch_execz .LBB68_27162; GFX950-SDAG-NEXT: .LBB68_4: ; %atomicrmw.private7163; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]7164; GFX950-SDAG-NEXT: s_nop 17165; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc7166; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off7167; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)7168; GFX950-SDAG-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]7169; GFX950-SDAG-NEXT: s_nop 17170; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc7171; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc7172; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[0:1], off7173; GFX950-SDAG-NEXT: s_endpgm7174;7175; GFX950-GISEL-LABEL: flat_min_saddr_i64_nortn:7176; GFX950-GISEL: ; %bb.0:7177; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v27178; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]7179; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v07180; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v17181; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base7182; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc7183; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v17184; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc7185; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]7186; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB68_37187; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow7188; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7189; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB68_47190; GFX950-GISEL-NEXT: .LBB68_2: ; %atomicrmw.phi7191; GFX950-GISEL-NEXT: s_endpgm7192; GFX950-GISEL-NEXT: .LBB68_3: ; %atomicrmw.global7193; GFX950-GISEL-NEXT: flat_atomic_smin_x2 v[0:1], v[4:5]7194; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)7195; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr17196; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr57197; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7198; GFX950-GISEL-NEXT: s_cbranch_execz .LBB68_27199; GFX950-GISEL-NEXT: .LBB68_4: ; %atomicrmw.private7200; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]7201; GFX950-GISEL-NEXT: s_nop 17202; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc7203; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off7204; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)7205; GFX950-GISEL-NEXT: v_cmp_lt_i64_e32 vcc, v[0:1], v[4:5]7206; GFX950-GISEL-NEXT: s_nop 17207; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc7208; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc7209; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off7210; GFX950-GISEL-NEXT: s_endpgm7211 %zext.offset = zext i32 %voffset to i647212 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset7213 %unused = atomicrmw min ptr %gep0, i64 %data syncscope("workgroup") seq_cst7214 ret void7215}7216 7217define amdgpu_ps void @flat_min_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {7218; GFX1250-SDAG-LABEL: flat_min_saddr_i64_nortn_neg128:7219; GFX1250-SDAG: ; %bb.0:7220; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 17221; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v17222; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 07223; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff807224; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7225; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]7226; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]7227; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo7228; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7229; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v17230; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v47231; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s07232; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB69_37233; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow7234; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s07235; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB69_47236; GFX1250-SDAG-NEXT: .LBB69_2: ; %atomicrmw.phi7237; GFX1250-SDAG-NEXT: s_endpgm7238; GFX1250-SDAG-NEXT: .LBB69_3: ; %atomicrmw.global7239; GFX1250-SDAG-NEXT: flat_atomic_min_i64 v[0:1], v[2:3]7240; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr17241; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr37242; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x07243; GFX1250-SDAG-NEXT: s_wait_xcnt 0x07244; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s07245; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB69_27246; GFX1250-SDAG-NEXT: .LBB69_4: ; %atomicrmw.private7247; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]7248; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v07249; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)7250; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo7251; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off7252; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x07253; GFX1250-SDAG-NEXT: v_min_i64 v[0:1], v[0:1], v[2:3]7254; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off7255; GFX1250-SDAG-NEXT: s_endpgm7256;7257; GFX1250-GISEL-LABEL: flat_min_saddr_i64_nortn_neg128:7258; GFX1250-GISEL: ; %bb.0:7259; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 17260; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v27261; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]7262; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo7263; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7264; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v07265; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo7266; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)7267; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v17268; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo7269; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7270; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v37271; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v17272; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s07273; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB69_37274; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow7275; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s07276; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB69_47277; GFX1250-GISEL-NEXT: .LBB69_2: ; %atomicrmw.phi7278; GFX1250-GISEL-NEXT: s_endpgm7279; GFX1250-GISEL-NEXT: .LBB69_3: ; %atomicrmw.global7280; GFX1250-GISEL-NEXT: flat_atomic_min_i64 v0, v[4:5], s[2:3] offset:-1287281; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr27282; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr57283; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x07284; GFX1250-GISEL-NEXT: s_wait_xcnt 0x07285; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s07286; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB69_27287; GFX1250-GISEL-NEXT: .LBB69_4: ; %atomicrmw.private7288; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]7289; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v27290; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)7291; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo7292; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off7293; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x07294; GFX1250-GISEL-NEXT: v_min_i64 v[0:1], v[0:1], v[4:5]7295; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off7296; GFX1250-GISEL-NEXT: s_endpgm7297;7298; GFX950-SDAG-LABEL: flat_min_saddr_i64_nortn_neg128:7299; GFX950-SDAG: ; %bb.0:7300; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v27301; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v17302; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 07303; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]7304; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff807305; GFX950-SDAG-NEXT: s_mov_b32 s3, -17306; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base7307; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]7308; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v17309; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc7310; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]7311; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB69_37312; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow7313; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7314; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB69_47315; GFX950-SDAG-NEXT: .LBB69_2: ; %atomicrmw.phi7316; GFX950-SDAG-NEXT: s_endpgm7317; GFX950-SDAG-NEXT: .LBB69_3: ; %atomicrmw.global7318; GFX950-SDAG-NEXT: flat_atomic_smin_x2 v[0:1], v[2:3]7319; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)7320; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr17321; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr37322; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7323; GFX950-SDAG-NEXT: s_cbranch_execz .LBB69_27324; GFX950-SDAG-NEXT: .LBB69_4: ; %atomicrmw.private7325; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]7326; GFX950-SDAG-NEXT: s_nop 17327; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc7328; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off7329; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)7330; GFX950-SDAG-NEXT: v_cmp_le_i64_e32 vcc, v[0:1], v[2:3]7331; GFX950-SDAG-NEXT: s_nop 17332; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc7333; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc7334; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[0:1], off7335; GFX950-SDAG-NEXT: s_endpgm7336;7337; GFX950-GISEL-LABEL: flat_min_saddr_i64_nortn_neg128:7338; GFX950-GISEL: ; %bb.0:7339; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v27340; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]7341; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v07342; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v17343; GFX950-GISEL-NEXT: s_nop 07344; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc7345; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v07346; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base7347; GFX950-GISEL-NEXT: s_nop 07348; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc7349; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v17350; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc7351; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]7352; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB69_37353; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow7354; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7355; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB69_47356; GFX950-GISEL-NEXT: .LBB69_2: ; %atomicrmw.phi7357; GFX950-GISEL-NEXT: s_endpgm7358; GFX950-GISEL-NEXT: .LBB69_3: ; %atomicrmw.global7359; GFX950-GISEL-NEXT: flat_atomic_smin_x2 v[0:1], v[4:5]7360; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)7361; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr17362; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr57363; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7364; GFX950-GISEL-NEXT: s_cbranch_execz .LBB69_27365; GFX950-GISEL-NEXT: .LBB69_4: ; %atomicrmw.private7366; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]7367; GFX950-GISEL-NEXT: s_nop 17368; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc7369; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off7370; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)7371; GFX950-GISEL-NEXT: v_cmp_lt_i64_e32 vcc, v[0:1], v[4:5]7372; GFX950-GISEL-NEXT: s_nop 17373; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc7374; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc7375; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off7376; GFX950-GISEL-NEXT: s_endpgm7377 %zext.offset = zext i32 %voffset to i647378 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset7379 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1287380 %unused = atomicrmw min ptr %gep1, i64 %data syncscope("workgroup") seq_cst7381 ret void7382}7383 7384; --------------------------------------------------------------------------------7385; atomicrmw umax7386; --------------------------------------------------------------------------------7387 7388define amdgpu_ps float @flat_umax_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {7389; GFX1250-LABEL: flat_umax_saddr_i32_rtn:7390; GFX1250: ; %bb.0:7391; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 17392; GFX1250-NEXT: flat_atomic_max_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN7393; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x07394; GFX1250-NEXT: ; return to shader part epilog7395;7396; GFX950-SDAG-LABEL: flat_umax_saddr_i32_rtn:7397; GFX950-SDAG: ; %bb.0:7398; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v17399; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 07400; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]7401; GFX950-SDAG-NEXT: flat_atomic_umax v0, v[0:1], v2 sc07402; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7403; GFX950-SDAG-NEXT: ; return to shader part epilog7404;7405; GFX950-GISEL-LABEL: flat_umax_saddr_i32_rtn:7406; GFX950-GISEL: ; %bb.0:7407; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]7408; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v07409; GFX950-GISEL-NEXT: s_nop 17410; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc7411; GFX950-GISEL-NEXT: flat_atomic_umax v0, v[2:3], v1 sc07412; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7413; GFX950-GISEL-NEXT: ; return to shader part epilog7414 %zext.offset = zext i32 %voffset to i647415 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset7416 %rtn = atomicrmw umax ptr %gep0, i32 %data syncscope("workgroup") seq_cst7417 %cast.rtn = bitcast i32 %rtn to float7418 ret float %cast.rtn7419}7420 7421define amdgpu_ps float @flat_umax_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {7422; GFX1250-LABEL: flat_umax_saddr_i32_rtn_neg128:7423; GFX1250: ; %bb.0:7424; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 17425; GFX1250-NEXT: flat_atomic_max_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN7426; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x07427; GFX1250-NEXT: ; return to shader part epilog7428;7429; GFX950-SDAG-LABEL: flat_umax_saddr_i32_rtn_neg128:7430; GFX950-SDAG: ; %bb.0:7431; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v17432; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 07433; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]7434; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v07435; GFX950-SDAG-NEXT: s_nop 17436; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc7437; GFX950-SDAG-NEXT: flat_atomic_umax v0, v[0:1], v2 sc07438; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7439; GFX950-SDAG-NEXT: ; return to shader part epilog7440;7441; GFX950-GISEL-LABEL: flat_umax_saddr_i32_rtn_neg128:7442; GFX950-GISEL: ; %bb.0:7443; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]7444; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v07445; GFX950-GISEL-NEXT: s_nop 17446; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc7447; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v07448; GFX950-GISEL-NEXT: s_nop 17449; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc7450; GFX950-GISEL-NEXT: flat_atomic_umax v0, v[2:3], v1 sc07451; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)7452; GFX950-GISEL-NEXT: ; return to shader part epilog7453 %zext.offset = zext i32 %voffset to i647454 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset7455 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1287456 %rtn = atomicrmw umax ptr %gep1, i32 %data syncscope("workgroup") seq_cst7457 %cast.rtn = bitcast i32 %rtn to float7458 ret float %cast.rtn7459}7460 7461define amdgpu_ps void @flat_umax_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {7462; GFX1250-LABEL: flat_umax_saddr_i32_nortn:7463; GFX1250: ; %bb.0:7464; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 17465; GFX1250-NEXT: flat_atomic_max_u32 v0, v1, s[2:3]7466; GFX1250-NEXT: s_wait_storecnt_dscnt 0x07467; GFX1250-NEXT: s_endpgm7468;7469; GFX950-SDAG-LABEL: flat_umax_saddr_i32_nortn:7470; GFX950-SDAG: ; %bb.0:7471; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v17472; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 07473; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]7474; GFX950-SDAG-NEXT: flat_atomic_umax v[0:1], v27475; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)7476; GFX950-SDAG-NEXT: s_endpgm7477;7478; GFX950-GISEL-LABEL: flat_umax_saddr_i32_nortn:7479; GFX950-GISEL: ; %bb.0:7480; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]7481; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v07482; GFX950-GISEL-NEXT: s_nop 17483; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc7484; GFX950-GISEL-NEXT: flat_atomic_umax v[2:3], v17485; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)7486; GFX950-GISEL-NEXT: s_endpgm7487 %zext.offset = zext i32 %voffset to i647488 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset7489 %unused = atomicrmw umax ptr %gep0, i32 %data syncscope("workgroup") seq_cst7490 ret void7491}7492 7493define amdgpu_ps void @flat_umax_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {7494; GFX1250-LABEL: flat_umax_saddr_i32_nortn_neg128:7495; GFX1250: ; %bb.0:7496; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 17497; GFX1250-NEXT: flat_atomic_max_u32 v0, v1, s[2:3] offset:-1287498; GFX1250-NEXT: s_wait_storecnt_dscnt 0x07499; GFX1250-NEXT: s_endpgm7500;7501; GFX950-SDAG-LABEL: flat_umax_saddr_i32_nortn_neg128:7502; GFX950-SDAG: ; %bb.0:7503; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v17504; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 07505; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]7506; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v07507; GFX950-SDAG-NEXT: s_nop 17508; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc7509; GFX950-SDAG-NEXT: flat_atomic_umax v[0:1], v27510; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)7511; GFX950-SDAG-NEXT: s_endpgm7512;7513; GFX950-GISEL-LABEL: flat_umax_saddr_i32_nortn_neg128:7514; GFX950-GISEL: ; %bb.0:7515; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]7516; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v07517; GFX950-GISEL-NEXT: s_nop 17518; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc7519; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v07520; GFX950-GISEL-NEXT: s_nop 17521; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc7522; GFX950-GISEL-NEXT: flat_atomic_umax v[2:3], v17523; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)7524; GFX950-GISEL-NEXT: s_endpgm7525 %zext.offset = zext i32 %voffset to i647526 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset7527 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1287528 %unused = atomicrmw umax ptr %gep1, i32 %data syncscope("workgroup") seq_cst7529 ret void7530}7531 7532define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {7533; GFX1250-SDAG-LABEL: flat_umax_saddr_i64_rtn:7534; GFX1250-SDAG: ; %bb.0:7535; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 17536; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v17537; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 07538; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7539; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]7540; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v57541; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)7542; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v07543; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr17544; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo7545; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s07546; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB74_37547; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow7548; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s07549; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB74_47550; GFX1250-SDAG-NEXT: .LBB74_2: ; %atomicrmw.phi7551; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s07552; GFX1250-SDAG-NEXT: s_branch .LBB74_57553; GFX1250-SDAG-NEXT: .LBB74_3: ; %atomicrmw.global7554; GFX1250-SDAG-NEXT: flat_atomic_max_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN7555; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr57556; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr37557; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x07558; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s07559; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB74_27560; GFX1250-SDAG-NEXT: .LBB74_4: ; %atomicrmw.private7561; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]7562; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v47563; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)7564; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo7565; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off7566; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x07567; GFX1250-SDAG-NEXT: v_max_u64 v[2:3], v[0:1], v[2:3]7568; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off7569; GFX1250-SDAG-NEXT: s_wait_xcnt 0x07570; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s07571; GFX1250-SDAG-NEXT: s_branch .LBB74_57572; GFX1250-SDAG-NEXT: .LBB74_5:7573;7574; GFX1250-GISEL-LABEL: flat_umax_saddr_i64_rtn:7575; GFX1250-GISEL: ; %bb.0:7576; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 17577; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v17578; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]7579; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7580; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v37581; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo7582; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7583; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x147584; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v07585; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr17586; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo7587; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7588; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s07589; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB74_37590; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow7591; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s07592; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB74_47593; GFX1250-GISEL-NEXT: .LBB74_2: ; %atomicrmw.phi7594; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s07595; GFX1250-GISEL-NEXT: s_branch .LBB74_57596; GFX1250-GISEL-NEXT: .LBB74_3: ; %atomicrmw.global7597; GFX1250-GISEL-NEXT: flat_atomic_max_u64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN7598; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr67599; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr57600; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x07601; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s07602; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB74_27603; GFX1250-GISEL-NEXT: .LBB74_4: ; %atomicrmw.private7604; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]7605; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v67606; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)7607; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo7608; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off7609; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x07610; GFX1250-GISEL-NEXT: v_max_u64 v[2:3], v[0:1], v[4:5]7611; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off7612; GFX1250-GISEL-NEXT: s_wait_xcnt 0x07613; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s07614; GFX1250-GISEL-NEXT: s_branch .LBB74_57615; GFX1250-GISEL-NEXT: .LBB74_5:7616;7617; GFX950-SDAG-LABEL: flat_umax_saddr_i64_rtn:7618; GFX950-SDAG: ; %bb.0:7619; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v27620; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v17621; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 07622; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base7623; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1]7624; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v57625; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr17626; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc7627; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]7628; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB74_37629; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow7630; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7631; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB74_47632; GFX950-SDAG-NEXT: .LBB74_2: ; %atomicrmw.phi7633; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]7634; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)7635; GFX950-SDAG-NEXT: s_branch .LBB74_57636; GFX950-SDAG-NEXT: .LBB74_3: ; %atomicrmw.global7637; GFX950-SDAG-NEXT: flat_atomic_umax_x2 v[0:1], v[4:5], v[2:3] sc07638; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)7639; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr57640; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr37641; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7642; GFX950-SDAG-NEXT: s_cbranch_execz .LBB74_27643; GFX950-SDAG-NEXT: .LBB74_4: ; %atomicrmw.private7644; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]7645; GFX950-SDAG-NEXT: s_nop 17646; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc7647; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off7648; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)7649; GFX950-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]7650; GFX950-SDAG-NEXT: s_nop 17651; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc7652; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc7653; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off7654; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]7655; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)7656; GFX950-SDAG-NEXT: s_branch .LBB74_57657; GFX950-SDAG-NEXT: .LBB74_5:7658;7659; GFX950-GISEL-LABEL: flat_umax_saddr_i64_rtn:7660; GFX950-GISEL: ; %bb.0:7661; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v27662; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]7663; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v07664; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base7665; GFX950-GISEL-NEXT: s_nop 07666; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc7667; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v17668; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v37669; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr17670; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc7671; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]7672; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB74_37673; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow7674; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7675; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB74_47676; GFX950-GISEL-NEXT: .LBB74_2: ; %atomicrmw.phi7677; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]7678; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)7679; GFX950-GISEL-NEXT: s_branch .LBB74_57680; GFX950-GISEL-NEXT: .LBB74_3: ; %atomicrmw.global7681; GFX950-GISEL-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3], v[4:5] sc07682; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)7683; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr37684; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr57685; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7686; GFX950-GISEL-NEXT: s_cbranch_execz .LBB74_27687; GFX950-GISEL-NEXT: .LBB74_4: ; %atomicrmw.private7688; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]7689; GFX950-GISEL-NEXT: s_nop 17690; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc7691; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off7692; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)7693; GFX950-GISEL-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[4:5]7694; GFX950-GISEL-NEXT: s_nop 17695; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v0, vcc7696; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v1, vcc7697; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off7698; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]7699; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)7700; GFX950-GISEL-NEXT: s_branch .LBB74_57701; GFX950-GISEL-NEXT: .LBB74_5:7702 %zext.offset = zext i32 %voffset to i647703 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset7704 %rtn = atomicrmw umax ptr %gep0, i64 %data syncscope("workgroup") seq_cst7705 %cast.rtn = bitcast i64 %rtn to <2 x float>7706 ret <2 x float> %cast.rtn7707}7708 7709define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {7710; GFX1250-SDAG-LABEL: flat_umax_saddr_i64_rtn_neg128:7711; GFX1250-SDAG: ; %bb.0:7712; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 17713; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v17714; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 07715; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff807716; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7717; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]7718; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]7719; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7720; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v57721; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v07722; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr17723; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo7724; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7725; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s07726; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB75_37727; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow7728; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s07729; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB75_47730; GFX1250-SDAG-NEXT: .LBB75_2: ; %atomicrmw.phi7731; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s07732; GFX1250-SDAG-NEXT: s_branch .LBB75_57733; GFX1250-SDAG-NEXT: .LBB75_3: ; %atomicrmw.global7734; GFX1250-SDAG-NEXT: flat_atomic_max_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN7735; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr57736; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr37737; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x07738; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s07739; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB75_27740; GFX1250-SDAG-NEXT: .LBB75_4: ; %atomicrmw.private7741; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]7742; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v47743; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)7744; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo7745; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off7746; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x07747; GFX1250-SDAG-NEXT: v_max_u64 v[2:3], v[0:1], v[2:3]7748; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off7749; GFX1250-SDAG-NEXT: s_wait_xcnt 0x07750; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s07751; GFX1250-SDAG-NEXT: s_branch .LBB75_57752; GFX1250-SDAG-NEXT: .LBB75_5:7753;7754; GFX1250-GISEL-LABEL: flat_umax_saddr_i64_rtn_neg128:7755; GFX1250-GISEL: ; %bb.0:7756; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 17757; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v17758; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]7759; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7760; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v37761; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo7762; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)7763; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v07764; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo7765; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7766; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x147767; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v07768; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr17769; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo7770; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7771; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s07772; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB75_37773; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow7774; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s07775; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB75_47776; GFX1250-GISEL-NEXT: .LBB75_2: ; %atomicrmw.phi7777; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s07778; GFX1250-GISEL-NEXT: s_branch .LBB75_57779; GFX1250-GISEL-NEXT: .LBB75_3: ; %atomicrmw.global7780; GFX1250-GISEL-NEXT: flat_atomic_max_u64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN7781; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr67782; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr57783; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x07784; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s07785; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB75_27786; GFX1250-GISEL-NEXT: .LBB75_4: ; %atomicrmw.private7787; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]7788; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v67789; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)7790; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo7791; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off7792; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x07793; GFX1250-GISEL-NEXT: v_max_u64 v[2:3], v[0:1], v[4:5]7794; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off7795; GFX1250-GISEL-NEXT: s_wait_xcnt 0x07796; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s07797; GFX1250-GISEL-NEXT: s_branch .LBB75_57798; GFX1250-GISEL-NEXT: .LBB75_5:7799;7800; GFX950-SDAG-LABEL: flat_umax_saddr_i64_rtn_neg128:7801; GFX950-SDAG: ; %bb.0:7802; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v27803; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v17804; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 07805; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]7806; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff807807; GFX950-SDAG-NEXT: s_mov_b32 s3, -17808; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base7809; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]7810; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v57811; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr17812; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc7813; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]7814; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB75_37815; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow7816; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7817; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB75_47818; GFX950-SDAG-NEXT: .LBB75_2: ; %atomicrmw.phi7819; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]7820; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)7821; GFX950-SDAG-NEXT: s_branch .LBB75_57822; GFX950-SDAG-NEXT: .LBB75_3: ; %atomicrmw.global7823; GFX950-SDAG-NEXT: flat_atomic_umax_x2 v[0:1], v[4:5], v[2:3] sc07824; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)7825; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr57826; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr37827; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7828; GFX950-SDAG-NEXT: s_cbranch_execz .LBB75_27829; GFX950-SDAG-NEXT: .LBB75_4: ; %atomicrmw.private7830; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]7831; GFX950-SDAG-NEXT: s_nop 17832; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc7833; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off7834; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)7835; GFX950-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]7836; GFX950-SDAG-NEXT: s_nop 17837; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc7838; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc7839; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off7840; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]7841; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)7842; GFX950-SDAG-NEXT: s_branch .LBB75_57843; GFX950-SDAG-NEXT: .LBB75_5:7844;7845; GFX950-GISEL-LABEL: flat_umax_saddr_i64_rtn_neg128:7846; GFX950-GISEL: ; %bb.0:7847; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v27848; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]7849; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v07850; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v17851; GFX950-GISEL-NEXT: s_nop 07852; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc7853; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v07854; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base7855; GFX950-GISEL-NEXT: s_nop 07856; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v1, vcc7857; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v37858; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr17859; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc7860; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]7861; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB75_37862; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow7863; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7864; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB75_47865; GFX950-GISEL-NEXT: .LBB75_2: ; %atomicrmw.phi7866; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]7867; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)7868; GFX950-GISEL-NEXT: s_branch .LBB75_57869; GFX950-GISEL-NEXT: .LBB75_3: ; %atomicrmw.global7870; GFX950-GISEL-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3], v[4:5] sc07871; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)7872; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr37873; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr57874; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7875; GFX950-GISEL-NEXT: s_cbranch_execz .LBB75_27876; GFX950-GISEL-NEXT: .LBB75_4: ; %atomicrmw.private7877; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]7878; GFX950-GISEL-NEXT: s_nop 17879; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc7880; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off7881; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)7882; GFX950-GISEL-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[4:5]7883; GFX950-GISEL-NEXT: s_nop 17884; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v0, vcc7885; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v1, vcc7886; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off7887; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]7888; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)7889; GFX950-GISEL-NEXT: s_branch .LBB75_57890; GFX950-GISEL-NEXT: .LBB75_5:7891 %zext.offset = zext i32 %voffset to i647892 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset7893 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1287894 %rtn = atomicrmw umax ptr %gep1, i64 %data syncscope("workgroup") seq_cst7895 %cast.rtn = bitcast i64 %rtn to <2 x float>7896 ret <2 x float> %cast.rtn7897}7898 7899define amdgpu_ps void @flat_umax_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {7900; GFX1250-SDAG-LABEL: flat_umax_saddr_i64_nortn:7901; GFX1250-SDAG: ; %bb.0:7902; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 17903; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v17904; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 07905; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo7906; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7907; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]7908; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v17909; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)7910; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v47911; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s07912; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB76_37913; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow7914; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s07915; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB76_47916; GFX1250-SDAG-NEXT: .LBB76_2: ; %atomicrmw.phi7917; GFX1250-SDAG-NEXT: s_endpgm7918; GFX1250-SDAG-NEXT: .LBB76_3: ; %atomicrmw.global7919; GFX1250-SDAG-NEXT: flat_atomic_max_u64 v[0:1], v[2:3]7920; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr17921; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr37922; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x07923; GFX1250-SDAG-NEXT: s_wait_xcnt 0x07924; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s07925; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB76_27926; GFX1250-SDAG-NEXT: .LBB76_4: ; %atomicrmw.private7927; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]7928; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v07929; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)7930; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo7931; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off7932; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x07933; GFX1250-SDAG-NEXT: v_max_u64 v[0:1], v[0:1], v[2:3]7934; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off7935; GFX1250-SDAG-NEXT: s_endpgm7936;7937; GFX1250-GISEL-LABEL: flat_umax_saddr_i64_nortn:7938; GFX1250-GISEL: ; %bb.0:7939; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 17940; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v27941; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]7942; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo7943; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7944; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v07945; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo7946; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7947; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v37948; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v17949; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s07950; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB76_37951; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow7952; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s07953; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB76_47954; GFX1250-GISEL-NEXT: .LBB76_2: ; %atomicrmw.phi7955; GFX1250-GISEL-NEXT: s_endpgm7956; GFX1250-GISEL-NEXT: .LBB76_3: ; %atomicrmw.global7957; GFX1250-GISEL-NEXT: flat_atomic_max_u64 v0, v[4:5], s[2:3]7958; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr27959; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr57960; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x07961; GFX1250-GISEL-NEXT: s_wait_xcnt 0x07962; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s07963; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB76_27964; GFX1250-GISEL-NEXT: .LBB76_4: ; %atomicrmw.private7965; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]7966; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v27967; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)7968; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo7969; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off7970; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x07971; GFX1250-GISEL-NEXT: v_max_u64 v[0:1], v[0:1], v[4:5]7972; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off7973; GFX1250-GISEL-NEXT: s_endpgm7974;7975; GFX950-SDAG-LABEL: flat_umax_saddr_i64_nortn:7976; GFX950-SDAG: ; %bb.0:7977; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v27978; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v17979; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 07980; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base7981; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]7982; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v17983; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc7984; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]7985; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB76_37986; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow7987; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7988; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB76_47989; GFX950-SDAG-NEXT: .LBB76_2: ; %atomicrmw.phi7990; GFX950-SDAG-NEXT: s_endpgm7991; GFX950-SDAG-NEXT: .LBB76_3: ; %atomicrmw.global7992; GFX950-SDAG-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3]7993; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)7994; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr17995; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr37996; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]7997; GFX950-SDAG-NEXT: s_cbranch_execz .LBB76_27998; GFX950-SDAG-NEXT: .LBB76_4: ; %atomicrmw.private7999; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]8000; GFX950-SDAG-NEXT: s_nop 18001; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc8002; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off8003; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)8004; GFX950-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]8005; GFX950-SDAG-NEXT: s_nop 18006; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc8007; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc8008; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[0:1], off8009; GFX950-SDAG-NEXT: s_endpgm8010;8011; GFX950-GISEL-LABEL: flat_umax_saddr_i64_nortn:8012; GFX950-GISEL: ; %bb.0:8013; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v28014; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]8015; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v08016; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v18017; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base8018; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc8019; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v18020; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc8021; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]8022; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB76_38023; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow8024; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8025; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB76_48026; GFX950-GISEL-NEXT: .LBB76_2: ; %atomicrmw.phi8027; GFX950-GISEL-NEXT: s_endpgm8028; GFX950-GISEL-NEXT: .LBB76_3: ; %atomicrmw.global8029; GFX950-GISEL-NEXT: flat_atomic_umax_x2 v[0:1], v[4:5]8030; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)8031; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr18032; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr58033; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8034; GFX950-GISEL-NEXT: s_cbranch_execz .LBB76_28035; GFX950-GISEL-NEXT: .LBB76_4: ; %atomicrmw.private8036; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]8037; GFX950-GISEL-NEXT: s_nop 18038; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc8039; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off8040; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)8041; GFX950-GISEL-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[4:5]8042; GFX950-GISEL-NEXT: s_nop 18043; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc8044; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc8045; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off8046; GFX950-GISEL-NEXT: s_endpgm8047 %zext.offset = zext i32 %voffset to i648048 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset8049 %unused = atomicrmw umax ptr %gep0, i64 %data syncscope("workgroup") seq_cst8050 ret void8051}8052 8053define amdgpu_ps void @flat_umax_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {8054; GFX1250-SDAG-LABEL: flat_umax_saddr_i64_nortn_neg128:8055; GFX1250-SDAG: ; %bb.0:8056; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 18057; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v18058; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 08059; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff808060; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8061; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]8062; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]8063; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo8064; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8065; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v18066; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v48067; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s08068; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB77_38069; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow8070; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s08071; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB77_48072; GFX1250-SDAG-NEXT: .LBB77_2: ; %atomicrmw.phi8073; GFX1250-SDAG-NEXT: s_endpgm8074; GFX1250-SDAG-NEXT: .LBB77_3: ; %atomicrmw.global8075; GFX1250-SDAG-NEXT: flat_atomic_max_u64 v[0:1], v[2:3]8076; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr18077; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr38078; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x08079; GFX1250-SDAG-NEXT: s_wait_xcnt 0x08080; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s08081; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB77_28082; GFX1250-SDAG-NEXT: .LBB77_4: ; %atomicrmw.private8083; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]8084; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v08085; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)8086; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo8087; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off8088; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x08089; GFX1250-SDAG-NEXT: v_max_u64 v[0:1], v[0:1], v[2:3]8090; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off8091; GFX1250-SDAG-NEXT: s_endpgm8092;8093; GFX1250-GISEL-LABEL: flat_umax_saddr_i64_nortn_neg128:8094; GFX1250-GISEL: ; %bb.0:8095; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 18096; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v28097; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]8098; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo8099; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8100; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v08101; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo8102; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)8103; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v18104; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo8105; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8106; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v38107; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v18108; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s08109; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB77_38110; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow8111; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s08112; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB77_48113; GFX1250-GISEL-NEXT: .LBB77_2: ; %atomicrmw.phi8114; GFX1250-GISEL-NEXT: s_endpgm8115; GFX1250-GISEL-NEXT: .LBB77_3: ; %atomicrmw.global8116; GFX1250-GISEL-NEXT: flat_atomic_max_u64 v0, v[4:5], s[2:3] offset:-1288117; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr28118; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr58119; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x08120; GFX1250-GISEL-NEXT: s_wait_xcnt 0x08121; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s08122; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB77_28123; GFX1250-GISEL-NEXT: .LBB77_4: ; %atomicrmw.private8124; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]8125; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v28126; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)8127; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo8128; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off8129; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x08130; GFX1250-GISEL-NEXT: v_max_u64 v[0:1], v[0:1], v[4:5]8131; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off8132; GFX1250-GISEL-NEXT: s_endpgm8133;8134; GFX950-SDAG-LABEL: flat_umax_saddr_i64_nortn_neg128:8135; GFX950-SDAG: ; %bb.0:8136; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v28137; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v18138; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 08139; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]8140; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff808141; GFX950-SDAG-NEXT: s_mov_b32 s3, -18142; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base8143; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]8144; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v18145; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc8146; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]8147; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB77_38148; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow8149; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8150; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB77_48151; GFX950-SDAG-NEXT: .LBB77_2: ; %atomicrmw.phi8152; GFX950-SDAG-NEXT: s_endpgm8153; GFX950-SDAG-NEXT: .LBB77_3: ; %atomicrmw.global8154; GFX950-SDAG-NEXT: flat_atomic_umax_x2 v[0:1], v[2:3]8155; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)8156; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr18157; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr38158; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8159; GFX950-SDAG-NEXT: s_cbranch_execz .LBB77_28160; GFX950-SDAG-NEXT: .LBB77_4: ; %atomicrmw.private8161; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]8162; GFX950-SDAG-NEXT: s_nop 18163; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc8164; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off8165; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)8166; GFX950-SDAG-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[2:3]8167; GFX950-SDAG-NEXT: s_nop 18168; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc8169; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc8170; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[0:1], off8171; GFX950-SDAG-NEXT: s_endpgm8172;8173; GFX950-GISEL-LABEL: flat_umax_saddr_i64_nortn_neg128:8174; GFX950-GISEL: ; %bb.0:8175; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v28176; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]8177; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v08178; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v18179; GFX950-GISEL-NEXT: s_nop 08180; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc8181; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v08182; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base8183; GFX950-GISEL-NEXT: s_nop 08184; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc8185; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v18186; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc8187; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]8188; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB77_38189; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow8190; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8191; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB77_48192; GFX950-GISEL-NEXT: .LBB77_2: ; %atomicrmw.phi8193; GFX950-GISEL-NEXT: s_endpgm8194; GFX950-GISEL-NEXT: .LBB77_3: ; %atomicrmw.global8195; GFX950-GISEL-NEXT: flat_atomic_umax_x2 v[0:1], v[4:5]8196; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)8197; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr18198; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr58199; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8200; GFX950-GISEL-NEXT: s_cbranch_execz .LBB77_28201; GFX950-GISEL-NEXT: .LBB77_4: ; %atomicrmw.private8202; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]8203; GFX950-GISEL-NEXT: s_nop 18204; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc8205; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off8206; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)8207; GFX950-GISEL-NEXT: v_cmp_gt_u64_e32 vcc, v[0:1], v[4:5]8208; GFX950-GISEL-NEXT: s_nop 18209; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc8210; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc8211; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off8212; GFX950-GISEL-NEXT: s_endpgm8213 %zext.offset = zext i32 %voffset to i648214 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset8215 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1288216 %unused = atomicrmw umax ptr %gep1, i64 %data syncscope("workgroup") seq_cst8217 ret void8218}8219 8220; --------------------------------------------------------------------------------8221; atomicrmw umin8222; --------------------------------------------------------------------------------8223 8224define amdgpu_ps float @flat_umin_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {8225; GFX1250-LABEL: flat_umin_saddr_i32_rtn:8226; GFX1250: ; %bb.0:8227; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 18228; GFX1250-NEXT: flat_atomic_min_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN8229; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x08230; GFX1250-NEXT: ; return to shader part epilog8231;8232; GFX950-SDAG-LABEL: flat_umin_saddr_i32_rtn:8233; GFX950-SDAG: ; %bb.0:8234; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v18235; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 08236; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]8237; GFX950-SDAG-NEXT: flat_atomic_umin v0, v[0:1], v2 sc08238; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8239; GFX950-SDAG-NEXT: ; return to shader part epilog8240;8241; GFX950-GISEL-LABEL: flat_umin_saddr_i32_rtn:8242; GFX950-GISEL: ; %bb.0:8243; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]8244; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v08245; GFX950-GISEL-NEXT: s_nop 18246; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc8247; GFX950-GISEL-NEXT: flat_atomic_umin v0, v[2:3], v1 sc08248; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8249; GFX950-GISEL-NEXT: ; return to shader part epilog8250 %zext.offset = zext i32 %voffset to i648251 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset8252 %rtn = atomicrmw umin ptr %gep0, i32 %data syncscope("workgroup") seq_cst8253 %cast.rtn = bitcast i32 %rtn to float8254 ret float %cast.rtn8255}8256 8257define amdgpu_ps float @flat_umin_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {8258; GFX1250-LABEL: flat_umin_saddr_i32_rtn_neg128:8259; GFX1250: ; %bb.0:8260; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 18261; GFX1250-NEXT: flat_atomic_min_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN8262; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x08263; GFX1250-NEXT: ; return to shader part epilog8264;8265; GFX950-SDAG-LABEL: flat_umin_saddr_i32_rtn_neg128:8266; GFX950-SDAG: ; %bb.0:8267; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v18268; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 08269; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]8270; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v08271; GFX950-SDAG-NEXT: s_nop 18272; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc8273; GFX950-SDAG-NEXT: flat_atomic_umin v0, v[0:1], v2 sc08274; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8275; GFX950-SDAG-NEXT: ; return to shader part epilog8276;8277; GFX950-GISEL-LABEL: flat_umin_saddr_i32_rtn_neg128:8278; GFX950-GISEL: ; %bb.0:8279; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]8280; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v08281; GFX950-GISEL-NEXT: s_nop 18282; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc8283; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v08284; GFX950-GISEL-NEXT: s_nop 18285; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc8286; GFX950-GISEL-NEXT: flat_atomic_umin v0, v[2:3], v1 sc08287; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)8288; GFX950-GISEL-NEXT: ; return to shader part epilog8289 %zext.offset = zext i32 %voffset to i648290 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset8291 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1288292 %rtn = atomicrmw umin ptr %gep1, i32 %data syncscope("workgroup") seq_cst8293 %cast.rtn = bitcast i32 %rtn to float8294 ret float %cast.rtn8295}8296 8297define amdgpu_ps void @flat_umin_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {8298; GFX1250-LABEL: flat_umin_saddr_i32_nortn:8299; GFX1250: ; %bb.0:8300; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 18301; GFX1250-NEXT: flat_atomic_min_u32 v0, v1, s[2:3]8302; GFX1250-NEXT: s_wait_storecnt_dscnt 0x08303; GFX1250-NEXT: s_endpgm8304;8305; GFX950-SDAG-LABEL: flat_umin_saddr_i32_nortn:8306; GFX950-SDAG: ; %bb.0:8307; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v18308; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 08309; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]8310; GFX950-SDAG-NEXT: flat_atomic_umin v[0:1], v28311; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)8312; GFX950-SDAG-NEXT: s_endpgm8313;8314; GFX950-GISEL-LABEL: flat_umin_saddr_i32_nortn:8315; GFX950-GISEL: ; %bb.0:8316; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]8317; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v08318; GFX950-GISEL-NEXT: s_nop 18319; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc8320; GFX950-GISEL-NEXT: flat_atomic_umin v[2:3], v18321; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)8322; GFX950-GISEL-NEXT: s_endpgm8323 %zext.offset = zext i32 %voffset to i648324 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset8325 %unused = atomicrmw umin ptr %gep0, i32 %data syncscope("workgroup") seq_cst8326 ret void8327}8328 8329define amdgpu_ps void @flat_umin_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {8330; GFX1250-LABEL: flat_umin_saddr_i32_nortn_neg128:8331; GFX1250: ; %bb.0:8332; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 18333; GFX1250-NEXT: flat_atomic_min_u32 v0, v1, s[2:3] offset:-1288334; GFX1250-NEXT: s_wait_storecnt_dscnt 0x08335; GFX1250-NEXT: s_endpgm8336;8337; GFX950-SDAG-LABEL: flat_umin_saddr_i32_nortn_neg128:8338; GFX950-SDAG: ; %bb.0:8339; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v18340; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 08341; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]8342; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v08343; GFX950-SDAG-NEXT: s_nop 18344; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc8345; GFX950-SDAG-NEXT: flat_atomic_umin v[0:1], v28346; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)8347; GFX950-SDAG-NEXT: s_endpgm8348;8349; GFX950-GISEL-LABEL: flat_umin_saddr_i32_nortn_neg128:8350; GFX950-GISEL: ; %bb.0:8351; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]8352; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v08353; GFX950-GISEL-NEXT: s_nop 18354; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc8355; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v08356; GFX950-GISEL-NEXT: s_nop 18357; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc8358; GFX950-GISEL-NEXT: flat_atomic_umin v[2:3], v18359; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)8360; GFX950-GISEL-NEXT: s_endpgm8361 %zext.offset = zext i32 %voffset to i648362 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset8363 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1288364 %unused = atomicrmw umin ptr %gep1, i32 %data syncscope("workgroup") seq_cst8365 ret void8366}8367 8368define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {8369; GFX1250-SDAG-LABEL: flat_umin_saddr_i64_rtn:8370; GFX1250-SDAG: ; %bb.0:8371; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 18372; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v18373; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 08374; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8375; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]8376; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v58377; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)8378; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v08379; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr18380; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo8381; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s08382; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB82_38383; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow8384; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s08385; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB82_48386; GFX1250-SDAG-NEXT: .LBB82_2: ; %atomicrmw.phi8387; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s08388; GFX1250-SDAG-NEXT: s_branch .LBB82_58389; GFX1250-SDAG-NEXT: .LBB82_3: ; %atomicrmw.global8390; GFX1250-SDAG-NEXT: flat_atomic_min_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN8391; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr58392; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr38393; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x08394; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s08395; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB82_28396; GFX1250-SDAG-NEXT: .LBB82_4: ; %atomicrmw.private8397; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]8398; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v48399; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)8400; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo8401; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off8402; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x08403; GFX1250-SDAG-NEXT: v_min_u64 v[2:3], v[0:1], v[2:3]8404; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off8405; GFX1250-SDAG-NEXT: s_wait_xcnt 0x08406; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s08407; GFX1250-SDAG-NEXT: s_branch .LBB82_58408; GFX1250-SDAG-NEXT: .LBB82_5:8409;8410; GFX1250-GISEL-LABEL: flat_umin_saddr_i64_rtn:8411; GFX1250-GISEL: ; %bb.0:8412; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 18413; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v18414; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]8415; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8416; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v38417; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo8418; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8419; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x148420; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v08421; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr18422; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo8423; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8424; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s08425; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB82_38426; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow8427; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s08428; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB82_48429; GFX1250-GISEL-NEXT: .LBB82_2: ; %atomicrmw.phi8430; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s08431; GFX1250-GISEL-NEXT: s_branch .LBB82_58432; GFX1250-GISEL-NEXT: .LBB82_3: ; %atomicrmw.global8433; GFX1250-GISEL-NEXT: flat_atomic_min_u64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN8434; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr68435; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr58436; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x08437; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s08438; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB82_28439; GFX1250-GISEL-NEXT: .LBB82_4: ; %atomicrmw.private8440; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]8441; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v68442; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)8443; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo8444; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off8445; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x08446; GFX1250-GISEL-NEXT: v_min_u64 v[2:3], v[0:1], v[4:5]8447; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off8448; GFX1250-GISEL-NEXT: s_wait_xcnt 0x08449; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s08450; GFX1250-GISEL-NEXT: s_branch .LBB82_58451; GFX1250-GISEL-NEXT: .LBB82_5:8452;8453; GFX950-SDAG-LABEL: flat_umin_saddr_i64_rtn:8454; GFX950-SDAG: ; %bb.0:8455; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v28456; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v18457; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 08458; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base8459; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1]8460; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v58461; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr18462; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc8463; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]8464; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB82_38465; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow8466; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8467; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB82_48468; GFX950-SDAG-NEXT: .LBB82_2: ; %atomicrmw.phi8469; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]8470; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)8471; GFX950-SDAG-NEXT: s_branch .LBB82_58472; GFX950-SDAG-NEXT: .LBB82_3: ; %atomicrmw.global8473; GFX950-SDAG-NEXT: flat_atomic_umin_x2 v[0:1], v[4:5], v[2:3] sc08474; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)8475; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr58476; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr38477; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8478; GFX950-SDAG-NEXT: s_cbranch_execz .LBB82_28479; GFX950-SDAG-NEXT: .LBB82_4: ; %atomicrmw.private8480; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]8481; GFX950-SDAG-NEXT: s_nop 18482; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc8483; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off8484; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)8485; GFX950-SDAG-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]8486; GFX950-SDAG-NEXT: s_nop 18487; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc8488; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc8489; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off8490; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]8491; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)8492; GFX950-SDAG-NEXT: s_branch .LBB82_58493; GFX950-SDAG-NEXT: .LBB82_5:8494;8495; GFX950-GISEL-LABEL: flat_umin_saddr_i64_rtn:8496; GFX950-GISEL: ; %bb.0:8497; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v28498; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]8499; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v08500; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base8501; GFX950-GISEL-NEXT: s_nop 08502; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc8503; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v18504; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v38505; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr18506; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc8507; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]8508; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB82_38509; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow8510; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8511; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB82_48512; GFX950-GISEL-NEXT: .LBB82_2: ; %atomicrmw.phi8513; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]8514; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)8515; GFX950-GISEL-NEXT: s_branch .LBB82_58516; GFX950-GISEL-NEXT: .LBB82_3: ; %atomicrmw.global8517; GFX950-GISEL-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3], v[4:5] sc08518; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)8519; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr38520; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr58521; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8522; GFX950-GISEL-NEXT: s_cbranch_execz .LBB82_28523; GFX950-GISEL-NEXT: .LBB82_4: ; %atomicrmw.private8524; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]8525; GFX950-GISEL-NEXT: s_nop 18526; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc8527; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off8528; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)8529; GFX950-GISEL-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[4:5]8530; GFX950-GISEL-NEXT: s_nop 18531; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v0, vcc8532; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v1, vcc8533; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off8534; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]8535; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)8536; GFX950-GISEL-NEXT: s_branch .LBB82_58537; GFX950-GISEL-NEXT: .LBB82_5:8538 %zext.offset = zext i32 %voffset to i648539 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset8540 %rtn = atomicrmw umin ptr %gep0, i64 %data syncscope("workgroup") seq_cst8541 %cast.rtn = bitcast i64 %rtn to <2 x float>8542 ret <2 x float> %cast.rtn8543}8544 8545define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {8546; GFX1250-SDAG-LABEL: flat_umin_saddr_i64_rtn_neg128:8547; GFX1250-SDAG: ; %bb.0:8548; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 18549; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v18550; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 08551; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff808552; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8553; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]8554; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]8555; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8556; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v58557; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v08558; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr18559; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo8560; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8561; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s08562; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB83_38563; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow8564; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s08565; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB83_48566; GFX1250-SDAG-NEXT: .LBB83_2: ; %atomicrmw.phi8567; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s08568; GFX1250-SDAG-NEXT: s_branch .LBB83_58569; GFX1250-SDAG-NEXT: .LBB83_3: ; %atomicrmw.global8570; GFX1250-SDAG-NEXT: flat_atomic_min_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN8571; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr58572; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr38573; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x08574; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s08575; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB83_28576; GFX1250-SDAG-NEXT: .LBB83_4: ; %atomicrmw.private8577; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]8578; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v48579; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)8580; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo8581; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off8582; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x08583; GFX1250-SDAG-NEXT: v_min_u64 v[2:3], v[0:1], v[2:3]8584; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off8585; GFX1250-SDAG-NEXT: s_wait_xcnt 0x08586; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s08587; GFX1250-SDAG-NEXT: s_branch .LBB83_58588; GFX1250-SDAG-NEXT: .LBB83_5:8589;8590; GFX1250-GISEL-LABEL: flat_umin_saddr_i64_rtn_neg128:8591; GFX1250-GISEL: ; %bb.0:8592; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 18593; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v18594; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]8595; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8596; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v38597; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo8598; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)8599; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v08600; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo8601; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8602; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x148603; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v08604; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr18605; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo8606; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8607; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s08608; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB83_38609; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow8610; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s08611; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB83_48612; GFX1250-GISEL-NEXT: .LBB83_2: ; %atomicrmw.phi8613; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s08614; GFX1250-GISEL-NEXT: s_branch .LBB83_58615; GFX1250-GISEL-NEXT: .LBB83_3: ; %atomicrmw.global8616; GFX1250-GISEL-NEXT: flat_atomic_min_u64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN8617; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr68618; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr58619; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x08620; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s08621; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB83_28622; GFX1250-GISEL-NEXT: .LBB83_4: ; %atomicrmw.private8623; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]8624; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v68625; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)8626; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo8627; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off8628; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x08629; GFX1250-GISEL-NEXT: v_min_u64 v[2:3], v[0:1], v[4:5]8630; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off8631; GFX1250-GISEL-NEXT: s_wait_xcnt 0x08632; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s08633; GFX1250-GISEL-NEXT: s_branch .LBB83_58634; GFX1250-GISEL-NEXT: .LBB83_5:8635;8636; GFX950-SDAG-LABEL: flat_umin_saddr_i64_rtn_neg128:8637; GFX950-SDAG: ; %bb.0:8638; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v28639; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v18640; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 08641; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]8642; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff808643; GFX950-SDAG-NEXT: s_mov_b32 s3, -18644; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base8645; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]8646; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v58647; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr18648; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc8649; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]8650; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB83_38651; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow8652; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8653; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB83_48654; GFX950-SDAG-NEXT: .LBB83_2: ; %atomicrmw.phi8655; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]8656; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)8657; GFX950-SDAG-NEXT: s_branch .LBB83_58658; GFX950-SDAG-NEXT: .LBB83_3: ; %atomicrmw.global8659; GFX950-SDAG-NEXT: flat_atomic_umin_x2 v[0:1], v[4:5], v[2:3] sc08660; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)8661; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr58662; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr38663; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8664; GFX950-SDAG-NEXT: s_cbranch_execz .LBB83_28665; GFX950-SDAG-NEXT: .LBB83_4: ; %atomicrmw.private8666; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]8667; GFX950-SDAG-NEXT: s_nop 18668; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc8669; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off8670; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)8671; GFX950-SDAG-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]8672; GFX950-SDAG-NEXT: s_nop 18673; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc8674; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc8675; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[2:3], off8676; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]8677; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)8678; GFX950-SDAG-NEXT: s_branch .LBB83_58679; GFX950-SDAG-NEXT: .LBB83_5:8680;8681; GFX950-GISEL-LABEL: flat_umin_saddr_i64_rtn_neg128:8682; GFX950-GISEL: ; %bb.0:8683; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v28684; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]8685; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v08686; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v18687; GFX950-GISEL-NEXT: s_nop 08688; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc8689; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v08690; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base8691; GFX950-GISEL-NEXT: s_nop 08692; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v1, vcc8693; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v38694; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr18695; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc8696; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]8697; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB83_38698; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow8699; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8700; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB83_48701; GFX950-GISEL-NEXT: .LBB83_2: ; %atomicrmw.phi8702; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]8703; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)8704; GFX950-GISEL-NEXT: s_branch .LBB83_58705; GFX950-GISEL-NEXT: .LBB83_3: ; %atomicrmw.global8706; GFX950-GISEL-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3], v[4:5] sc08707; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)8708; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr38709; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr58710; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8711; GFX950-GISEL-NEXT: s_cbranch_execz .LBB83_28712; GFX950-GISEL-NEXT: .LBB83_4: ; %atomicrmw.private8713; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]8714; GFX950-GISEL-NEXT: s_nop 18715; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc8716; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off8717; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)8718; GFX950-GISEL-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[4:5]8719; GFX950-GISEL-NEXT: s_nop 18720; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, v4, v0, vcc8721; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v1, vcc8722; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off8723; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]8724; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)8725; GFX950-GISEL-NEXT: s_branch .LBB83_58726; GFX950-GISEL-NEXT: .LBB83_5:8727 %zext.offset = zext i32 %voffset to i648728 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset8729 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1288730 %rtn = atomicrmw umin ptr %gep1, i64 %data syncscope("workgroup") seq_cst8731 %cast.rtn = bitcast i64 %rtn to <2 x float>8732 ret <2 x float> %cast.rtn8733}8734 8735define amdgpu_ps void @flat_umin_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {8736; GFX1250-SDAG-LABEL: flat_umin_saddr_i64_nortn:8737; GFX1250-SDAG: ; %bb.0:8738; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 18739; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v18740; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 08741; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo8742; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8743; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]8744; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v18745; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)8746; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v48747; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s08748; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB84_38749; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow8750; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s08751; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB84_48752; GFX1250-SDAG-NEXT: .LBB84_2: ; %atomicrmw.phi8753; GFX1250-SDAG-NEXT: s_endpgm8754; GFX1250-SDAG-NEXT: .LBB84_3: ; %atomicrmw.global8755; GFX1250-SDAG-NEXT: flat_atomic_min_u64 v[0:1], v[2:3]8756; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr18757; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr38758; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x08759; GFX1250-SDAG-NEXT: s_wait_xcnt 0x08760; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s08761; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB84_28762; GFX1250-SDAG-NEXT: .LBB84_4: ; %atomicrmw.private8763; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]8764; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v08765; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)8766; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo8767; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off8768; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x08769; GFX1250-SDAG-NEXT: v_min_u64 v[0:1], v[0:1], v[2:3]8770; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off8771; GFX1250-SDAG-NEXT: s_endpgm8772;8773; GFX1250-GISEL-LABEL: flat_umin_saddr_i64_nortn:8774; GFX1250-GISEL: ; %bb.0:8775; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 18776; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v28777; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]8778; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo8779; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8780; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v08781; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo8782; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8783; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v38784; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v18785; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s08786; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB84_38787; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow8788; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s08789; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB84_48790; GFX1250-GISEL-NEXT: .LBB84_2: ; %atomicrmw.phi8791; GFX1250-GISEL-NEXT: s_endpgm8792; GFX1250-GISEL-NEXT: .LBB84_3: ; %atomicrmw.global8793; GFX1250-GISEL-NEXT: flat_atomic_min_u64 v0, v[4:5], s[2:3]8794; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr28795; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr58796; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x08797; GFX1250-GISEL-NEXT: s_wait_xcnt 0x08798; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s08799; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB84_28800; GFX1250-GISEL-NEXT: .LBB84_4: ; %atomicrmw.private8801; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]8802; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v28803; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)8804; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo8805; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off8806; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x08807; GFX1250-GISEL-NEXT: v_min_u64 v[0:1], v[0:1], v[4:5]8808; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off8809; GFX1250-GISEL-NEXT: s_endpgm8810;8811; GFX950-SDAG-LABEL: flat_umin_saddr_i64_nortn:8812; GFX950-SDAG: ; %bb.0:8813; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v28814; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v18815; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 08816; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base8817; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]8818; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v18819; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc8820; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]8821; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB84_38822; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow8823; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8824; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB84_48825; GFX950-SDAG-NEXT: .LBB84_2: ; %atomicrmw.phi8826; GFX950-SDAG-NEXT: s_endpgm8827; GFX950-SDAG-NEXT: .LBB84_3: ; %atomicrmw.global8828; GFX950-SDAG-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3]8829; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)8830; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr18831; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr38832; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8833; GFX950-SDAG-NEXT: s_cbranch_execz .LBB84_28834; GFX950-SDAG-NEXT: .LBB84_4: ; %atomicrmw.private8835; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]8836; GFX950-SDAG-NEXT: s_nop 18837; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc8838; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off8839; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)8840; GFX950-SDAG-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]8841; GFX950-SDAG-NEXT: s_nop 18842; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc8843; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc8844; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[0:1], off8845; GFX950-SDAG-NEXT: s_endpgm8846;8847; GFX950-GISEL-LABEL: flat_umin_saddr_i64_nortn:8848; GFX950-GISEL: ; %bb.0:8849; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v28850; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]8851; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v08852; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v18853; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base8854; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc8855; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v18856; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc8857; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]8858; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB84_38859; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow8860; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8861; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB84_48862; GFX950-GISEL-NEXT: .LBB84_2: ; %atomicrmw.phi8863; GFX950-GISEL-NEXT: s_endpgm8864; GFX950-GISEL-NEXT: .LBB84_3: ; %atomicrmw.global8865; GFX950-GISEL-NEXT: flat_atomic_umin_x2 v[0:1], v[4:5]8866; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)8867; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr18868; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr58869; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8870; GFX950-GISEL-NEXT: s_cbranch_execz .LBB84_28871; GFX950-GISEL-NEXT: .LBB84_4: ; %atomicrmw.private8872; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]8873; GFX950-GISEL-NEXT: s_nop 18874; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc8875; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off8876; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)8877; GFX950-GISEL-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[4:5]8878; GFX950-GISEL-NEXT: s_nop 18879; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc8880; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc8881; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off8882; GFX950-GISEL-NEXT: s_endpgm8883 %zext.offset = zext i32 %voffset to i648884 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset8885 %unused = atomicrmw umin ptr %gep0, i64 %data syncscope("workgroup") seq_cst8886 ret void8887}8888 8889define amdgpu_ps void @flat_umin_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {8890; GFX1250-SDAG-LABEL: flat_umin_saddr_i64_nortn_neg128:8891; GFX1250-SDAG: ; %bb.0:8892; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 18893; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v18894; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 08895; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff808896; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8897; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]8898; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]8899; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo8900; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8901; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v18902; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v48903; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s08904; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB85_38905; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow8906; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s08907; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB85_48908; GFX1250-SDAG-NEXT: .LBB85_2: ; %atomicrmw.phi8909; GFX1250-SDAG-NEXT: s_endpgm8910; GFX1250-SDAG-NEXT: .LBB85_3: ; %atomicrmw.global8911; GFX1250-SDAG-NEXT: flat_atomic_min_u64 v[0:1], v[2:3]8912; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr18913; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr38914; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x08915; GFX1250-SDAG-NEXT: s_wait_xcnt 0x08916; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s08917; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB85_28918; GFX1250-SDAG-NEXT: .LBB85_4: ; %atomicrmw.private8919; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]8920; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v08921; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)8922; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo8923; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off8924; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x08925; GFX1250-SDAG-NEXT: v_min_u64 v[0:1], v[0:1], v[2:3]8926; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off8927; GFX1250-SDAG-NEXT: s_endpgm8928;8929; GFX1250-GISEL-LABEL: flat_umin_saddr_i64_nortn_neg128:8930; GFX1250-GISEL: ; %bb.0:8931; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 18932; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v28933; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]8934; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo8935; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8936; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v08937; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo8938; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)8939; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v18940; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo8941; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8942; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v38943; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v18944; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s08945; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB85_38946; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow8947; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s08948; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB85_48949; GFX1250-GISEL-NEXT: .LBB85_2: ; %atomicrmw.phi8950; GFX1250-GISEL-NEXT: s_endpgm8951; GFX1250-GISEL-NEXT: .LBB85_3: ; %atomicrmw.global8952; GFX1250-GISEL-NEXT: flat_atomic_min_u64 v0, v[4:5], s[2:3] offset:-1288953; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr28954; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr58955; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x08956; GFX1250-GISEL-NEXT: s_wait_xcnt 0x08957; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s08958; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB85_28959; GFX1250-GISEL-NEXT: .LBB85_4: ; %atomicrmw.private8960; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]8961; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v28962; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)8963; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo8964; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off8965; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x08966; GFX1250-GISEL-NEXT: v_min_u64 v[0:1], v[0:1], v[4:5]8967; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off8968; GFX1250-GISEL-NEXT: s_endpgm8969;8970; GFX950-SDAG-LABEL: flat_umin_saddr_i64_nortn_neg128:8971; GFX950-SDAG: ; %bb.0:8972; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v28973; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v18974; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 08975; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]8976; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff808977; GFX950-SDAG-NEXT: s_mov_b32 s3, -18978; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base8979; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]8980; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v18981; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc8982; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]8983; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB85_38984; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow8985; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8986; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB85_48987; GFX950-SDAG-NEXT: .LBB85_2: ; %atomicrmw.phi8988; GFX950-SDAG-NEXT: s_endpgm8989; GFX950-SDAG-NEXT: .LBB85_3: ; %atomicrmw.global8990; GFX950-SDAG-NEXT: flat_atomic_umin_x2 v[0:1], v[2:3]8991; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)8992; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr18993; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr38994; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]8995; GFX950-SDAG-NEXT: s_cbranch_execz .LBB85_28996; GFX950-SDAG-NEXT: .LBB85_4: ; %atomicrmw.private8997; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]8998; GFX950-SDAG-NEXT: s_nop 18999; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc9000; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v4, off9001; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)9002; GFX950-SDAG-NEXT: v_cmp_le_u64_e32 vcc, v[0:1], v[2:3]9003; GFX950-SDAG-NEXT: s_nop 19004; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc9005; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc9006; GFX950-SDAG-NEXT: scratch_store_dwordx2 v4, v[0:1], off9007; GFX950-SDAG-NEXT: s_endpgm9008;9009; GFX950-GISEL-LABEL: flat_umin_saddr_i64_nortn_neg128:9010; GFX950-GISEL: ; %bb.0:9011; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v29012; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]9013; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v09014; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v19015; GFX950-GISEL-NEXT: s_nop 09016; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc9017; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v09018; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base9019; GFX950-GISEL-NEXT: s_nop 09020; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc9021; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v19022; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc9023; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]9024; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB85_39025; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow9026; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]9027; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB85_49028; GFX950-GISEL-NEXT: .LBB85_2: ; %atomicrmw.phi9029; GFX950-GISEL-NEXT: s_endpgm9030; GFX950-GISEL-NEXT: .LBB85_3: ; %atomicrmw.global9031; GFX950-GISEL-NEXT: flat_atomic_umin_x2 v[0:1], v[4:5]9032; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)9033; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr19034; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr59035; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]9036; GFX950-GISEL-NEXT: s_cbranch_execz .LBB85_29037; GFX950-GISEL-NEXT: .LBB85_4: ; %atomicrmw.private9038; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]9039; GFX950-GISEL-NEXT: s_nop 19040; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc9041; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off9042; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)9043; GFX950-GISEL-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[4:5]9044; GFX950-GISEL-NEXT: s_nop 19045; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc9046; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc9047; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off9048; GFX950-GISEL-NEXT: s_endpgm9049 %zext.offset = zext i32 %voffset to i649050 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset9051 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1289052 %unused = atomicrmw umin ptr %gep1, i64 %data syncscope("workgroup") seq_cst9053 ret void9054}9055 9056; --------------------------------------------------------------------------------9057; cmpxchg9058; --------------------------------------------------------------------------------9059 9060define amdgpu_ps float @flat_cmpxchg_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %cmp, i32 %data) {9061; GFX1250-LABEL: flat_cmpxchg_saddr_i32_rtn:9062; GFX1250: ; %bb.0:9063; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 19064; GFX1250-NEXT: v_mov_b32_e32 v3, v19065; GFX1250-NEXT: global_wb scope:SCOPE_SYS9066; GFX1250-NEXT: s_wait_storecnt 0x09067; GFX1250-NEXT: flat_atomic_cmpswap_b32 v0, v0, v[2:3], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS9068; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x09069; GFX1250-NEXT: global_inv scope:SCOPE_SYS9070; GFX1250-NEXT: s_wait_loadcnt 0x09071; GFX1250-NEXT: ; return to shader part epilog9072;9073; GFX950-SDAG-LABEL: flat_cmpxchg_saddr_i32_rtn:9074; GFX950-SDAG: ; %bb.0:9075; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v19076; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 09077; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]9078; GFX950-SDAG-NEXT: buffer_wbl2 sc0 sc19079; GFX950-SDAG-NEXT: flat_atomic_cmpswap v0, v[0:1], v[2:3] sc0 sc19080; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9081; GFX950-SDAG-NEXT: buffer_inv sc0 sc19082; GFX950-SDAG-NEXT: ; return to shader part epilog9083;9084; GFX950-GISEL-LABEL: flat_cmpxchg_saddr_i32_rtn:9085; GFX950-GISEL: ; %bb.0:9086; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]9087; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v4, v09088; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, v19089; GFX950-GISEL-NEXT: s_nop 09090; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v5, vcc9091; GFX950-GISEL-NEXT: buffer_wbl2 sc0 sc19092; GFX950-GISEL-NEXT: flat_atomic_cmpswap v0, v[0:1], v[2:3] sc0 sc19093; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9094; GFX950-GISEL-NEXT: buffer_inv sc0 sc19095; GFX950-GISEL-NEXT: ; return to shader part epilog9096 %zext.offset = zext i32 %voffset to i649097 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset9098 %cmpxchg = cmpxchg ptr %gep0, i32 %cmp, i32 %data seq_cst seq_cst9099 %rtn = extractvalue { i32, i1 } %cmpxchg, 09100 %cast.rtn = bitcast i32 %rtn to float9101 ret float %cast.rtn9102}9103 9104define amdgpu_ps float @flat_cmpxchg_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %cmp, i32 %data) {9105; GFX1250-LABEL: flat_cmpxchg_saddr_i32_rtn_neg128:9106; GFX1250: ; %bb.0:9107; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 19108; GFX1250-NEXT: v_mov_b32_e32 v3, v19109; GFX1250-NEXT: global_wb scope:SCOPE_SYS9110; GFX1250-NEXT: s_wait_storecnt 0x09111; GFX1250-NEXT: flat_atomic_cmpswap_b32 v0, v0, v[2:3], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_SYS9112; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x09113; GFX1250-NEXT: global_inv scope:SCOPE_SYS9114; GFX1250-NEXT: s_wait_loadcnt 0x09115; GFX1250-NEXT: ; return to shader part epilog9116;9117; GFX950-SDAG-LABEL: flat_cmpxchg_saddr_i32_rtn_neg128:9118; GFX950-SDAG: ; %bb.0:9119; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v19120; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 09121; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]9122; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v09123; GFX950-SDAG-NEXT: s_nop 19124; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc9125; GFX950-SDAG-NEXT: buffer_wbl2 sc0 sc19126; GFX950-SDAG-NEXT: flat_atomic_cmpswap v0, v[0:1], v[2:3] sc0 sc19127; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9128; GFX950-SDAG-NEXT: buffer_inv sc0 sc19129; GFX950-SDAG-NEXT: ; return to shader part epilog9130;9131; GFX950-GISEL-LABEL: flat_cmpxchg_saddr_i32_rtn_neg128:9132; GFX950-GISEL: ; %bb.0:9133; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]9134; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v4, v09135; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, v19136; GFX950-GISEL-NEXT: s_nop 09137; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v5, vcc9138; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v09139; GFX950-GISEL-NEXT: s_nop 19140; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc9141; GFX950-GISEL-NEXT: buffer_wbl2 sc0 sc19142; GFX950-GISEL-NEXT: flat_atomic_cmpswap v0, v[0:1], v[2:3] sc0 sc19143; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9144; GFX950-GISEL-NEXT: buffer_inv sc0 sc19145; GFX950-GISEL-NEXT: ; return to shader part epilog9146 %zext.offset = zext i32 %voffset to i649147 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset9148 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1289149 %cmpxchg = cmpxchg ptr %gep1, i32 %cmp, i32 %data seq_cst seq_cst9150 %rtn = extractvalue { i32, i1 } %cmpxchg, 09151 %cast.rtn = bitcast i32 %rtn to float9152 ret float %cast.rtn9153}9154 9155define amdgpu_ps void @flat_cmpxchg_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %cmp, i32 %data) {9156; GFX1250-LABEL: flat_cmpxchg_saddr_i32_nortn:9157; GFX1250: ; %bb.0:9158; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 19159; GFX1250-NEXT: v_mov_b32_e32 v3, v19160; GFX1250-NEXT: global_wb scope:SCOPE_SYS9161; GFX1250-NEXT: s_wait_storecnt 0x09162; GFX1250-NEXT: flat_atomic_cmpswap_b32 v0, v[2:3], s[2:3] scope:SCOPE_SYS9163; GFX1250-NEXT: s_wait_storecnt_dscnt 0x09164; GFX1250-NEXT: global_inv scope:SCOPE_SYS9165; GFX1250-NEXT: s_endpgm9166;9167; GFX950-SDAG-LABEL: flat_cmpxchg_saddr_i32_nortn:9168; GFX950-SDAG: ; %bb.0:9169; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v19170; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 09171; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]9172; GFX950-SDAG-NEXT: buffer_wbl2 sc0 sc19173; GFX950-SDAG-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] sc19174; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9175; GFX950-SDAG-NEXT: buffer_inv sc0 sc19176; GFX950-SDAG-NEXT: s_endpgm9177;9178; GFX950-GISEL-LABEL: flat_cmpxchg_saddr_i32_nortn:9179; GFX950-GISEL: ; %bb.0:9180; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]9181; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v4, v09182; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, v19183; GFX950-GISEL-NEXT: s_nop 09184; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v5, vcc9185; GFX950-GISEL-NEXT: buffer_wbl2 sc0 sc19186; GFX950-GISEL-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] sc19187; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9188; GFX950-GISEL-NEXT: buffer_inv sc0 sc19189; GFX950-GISEL-NEXT: s_endpgm9190 %zext.offset = zext i32 %voffset to i649191 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset9192 %unused = cmpxchg ptr %gep0, i32 %cmp, i32 %data seq_cst seq_cst9193 ret void9194}9195 9196define amdgpu_ps void @flat_cmpxchg_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %cmp, i32 %data) {9197; GFX1250-LABEL: flat_cmpxchg_saddr_i32_nortn_neg128:9198; GFX1250: ; %bb.0:9199; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 19200; GFX1250-NEXT: v_mov_b32_e32 v3, v19201; GFX1250-NEXT: global_wb scope:SCOPE_SYS9202; GFX1250-NEXT: s_wait_storecnt 0x09203; GFX1250-NEXT: flat_atomic_cmpswap_b32 v0, v[2:3], s[2:3] offset:-128 scope:SCOPE_SYS9204; GFX1250-NEXT: s_wait_storecnt_dscnt 0x09205; GFX1250-NEXT: global_inv scope:SCOPE_SYS9206; GFX1250-NEXT: s_endpgm9207;9208; GFX950-SDAG-LABEL: flat_cmpxchg_saddr_i32_nortn_neg128:9209; GFX950-SDAG: ; %bb.0:9210; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v19211; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 09212; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]9213; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v09214; GFX950-SDAG-NEXT: s_nop 19215; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc9216; GFX950-SDAG-NEXT: buffer_wbl2 sc0 sc19217; GFX950-SDAG-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] sc19218; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9219; GFX950-SDAG-NEXT: buffer_inv sc0 sc19220; GFX950-SDAG-NEXT: s_endpgm9221;9222; GFX950-GISEL-LABEL: flat_cmpxchg_saddr_i32_nortn_neg128:9223; GFX950-GISEL: ; %bb.0:9224; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]9225; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v4, v09226; GFX950-GISEL-NEXT: v_mov_b32_e32 v3, v19227; GFX950-GISEL-NEXT: s_nop 09228; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v5, vcc9229; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v09230; GFX950-GISEL-NEXT: s_nop 19231; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc9232; GFX950-GISEL-NEXT: buffer_wbl2 sc0 sc19233; GFX950-GISEL-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] sc19234; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9235; GFX950-GISEL-NEXT: buffer_inv sc0 sc19236; GFX950-GISEL-NEXT: s_endpgm9237 %zext.offset = zext i32 %voffset to i649238 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset9239 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1289240 %unused = cmpxchg ptr %gep1, i32 %cmp, i32 %data seq_cst seq_cst9241 ret void9242}9243 9244define amdgpu_ps <2 x float> @flat_cmpxchg_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %cmp, i64 %data) {9245; GFX1250-SDAG-LABEL: flat_cmpxchg_saddr_i64_rtn:9246; GFX1250-SDAG: ; %bb.0:9247; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 19248; GFX1250-SDAG-NEXT: v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v19249; GFX1250-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v5, v49250; GFX1250-SDAG-NEXT: v_mov_b32_e32 v4, v39251; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)9252; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[0:1]9253; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v39254; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)9255; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v09256; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr19257; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo9258; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s09259; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB90_39260; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow9261; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s09262; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB90_49263; GFX1250-SDAG-NEXT: .LBB90_2: ; %atomicrmw.phi9264; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s09265; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x09266; GFX1250-SDAG-NEXT: s_branch .LBB90_59267; GFX1250-SDAG-NEXT: .LBB90_3: ; %atomicrmw.global9268; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_SYS9269; GFX1250-SDAG-NEXT: s_wait_storecnt 0x09270; GFX1250-SDAG-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[2:3], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_SYS9271; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x09272; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_SYS9273; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr39274; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr6_vgpr79275; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s09276; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB90_29277; GFX1250-SDAG-NEXT: .LBB90_4: ; %atomicrmw.private9278; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]9279; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v29280; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)9281; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v8, -1, v0, vcc_lo9282; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v8, off9283; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x09284; GFX1250-SDAG-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[6:7]9285; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v1, v5 :: v_dual_cndmask_b32 v2, v0, v49286; GFX1250-SDAG-NEXT: scratch_store_b64 v8, v[2:3], off9287; GFX1250-SDAG-NEXT: s_wait_xcnt 0x09288; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s09289; GFX1250-SDAG-NEXT: s_branch .LBB90_59290; GFX1250-SDAG-NEXT: .LBB90_5:9291;9292; GFX1250-GISEL-LABEL: flat_cmpxchg_saddr_i64_rtn:9293; GFX1250-GISEL: ; %bb.0:9294; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 19295; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v8, v19296; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]9297; GFX1250-GISEL-NEXT: v_dual_mov_b32 v9, v2 :: v_dual_mov_b32 v6, v39298; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)9299; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v0, v59300; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v1, vcc_lo9301; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9302; GFX1250-GISEL-NEXT: v_dual_mov_b32 v7, v4 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v3 bitop3:0x149303; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v09304; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr19305; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo9306; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)9307; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s09308; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB90_39309; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow9310; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s09311; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB90_49312; GFX1250-GISEL-NEXT: .LBB90_2: ; %atomicrmw.phi9313; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s09314; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x09315; GFX1250-GISEL-NEXT: s_branch .LBB90_59316; GFX1250-GISEL-NEXT: .LBB90_3: ; %atomicrmw.global9317; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS9318; GFX1250-GISEL-NEXT: s_wait_storecnt 0x09319; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b64 v[0:1], v5, v[6:9], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS9320; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x09321; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_SYS9322; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr29323; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr8_vgpr99324; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s09325; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB90_29326; GFX1250-GISEL-NEXT: .LBB90_4: ; %atomicrmw.private9327; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]9328; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v29329; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)9330; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo9331; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v4, off9332; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x09333; GFX1250-GISEL-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]9334; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v2, v0, v6 :: v_dual_cndmask_b32 v3, v1, v79335; GFX1250-GISEL-NEXT: scratch_store_b64 v4, v[2:3], off9336; GFX1250-GISEL-NEXT: s_wait_xcnt 0x09337; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s09338; GFX1250-GISEL-NEXT: s_branch .LBB90_59339; GFX1250-GISEL-NEXT: .LBB90_5:9340;9341; GFX950-SDAG-LABEL: flat_cmpxchg_saddr_i64_rtn:9342; GFX950-SDAG: ; %bb.0:9343; GFX950-SDAG-NEXT: v_mov_b32_e32 v6, v19344; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 09345; GFX950-SDAG-NEXT: v_mov_b32_e32 v5, v49346; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, v39347; GFX950-SDAG-NEXT: v_mov_b32_e32 v7, v29348; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base9349; GFX950-SDAG-NEXT: v_lshl_add_u64 v[2:3], s[2:3], 0, v[0:1]9350; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v39351; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr19352; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc9353; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]9354; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB90_39355; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow9356; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]9357; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB90_49358; GFX950-SDAG-NEXT: .LBB90_2: ; %atomicrmw.phi9359; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]9360; GFX950-SDAG-NEXT: s_branch .LBB90_59361; GFX950-SDAG-NEXT: .LBB90_3: ; %atomicrmw.global9362; GFX950-SDAG-NEXT: buffer_wbl2 sc0 sc19363; GFX950-SDAG-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] sc0 sc19364; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9365; GFX950-SDAG-NEXT: buffer_inv sc0 sc19366; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr39367; GFX950-SDAG-NEXT: ; implicit-def: $vgpr6_vgpr79368; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]9369; GFX950-SDAG-NEXT: s_cbranch_execz .LBB90_29370; GFX950-SDAG-NEXT: .LBB90_4: ; %atomicrmw.private9371; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]9372; GFX950-SDAG-NEXT: s_nop 19373; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v8, -1, v2, vcc9374; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v8, off9375; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)9376; GFX950-SDAG-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]9377; GFX950-SDAG-NEXT: s_nop 19378; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc9379; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v2, v0, v4, vcc9380; GFX950-SDAG-NEXT: scratch_store_dwordx2 v8, v[2:3], off9381; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]9382; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)9383; GFX950-SDAG-NEXT: s_branch .LBB90_59384; GFX950-SDAG-NEXT: .LBB90_5:9385;9386; GFX950-GISEL-LABEL: flat_cmpxchg_saddr_i64_rtn:9387; GFX950-GISEL: ; %bb.0:9388; GFX950-GISEL-NEXT: v_mov_b32_e32 v9, v29389; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, v39390; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]9391; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v09392; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base9393; GFX950-GISEL-NEXT: s_nop 09394; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc9395; GFX950-GISEL-NEXT: v_mov_b32_e32 v8, v19396; GFX950-GISEL-NEXT: v_mov_b32_e32 v7, v49397; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v39398; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr19399; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc9400; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]9401; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB90_39402; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow9403; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]9404; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB90_49405; GFX950-GISEL-NEXT: .LBB90_2: ; %atomicrmw.phi9406; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]9407; GFX950-GISEL-NEXT: s_branch .LBB90_59408; GFX950-GISEL-NEXT: .LBB90_3: ; %atomicrmw.global9409; GFX950-GISEL-NEXT: buffer_wbl2 sc0 sc19410; GFX950-GISEL-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] sc0 sc19411; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9412; GFX950-GISEL-NEXT: buffer_inv sc0 sc19413; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr39414; GFX950-GISEL-NEXT: ; implicit-def: $vgpr8_vgpr99415; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]9416; GFX950-GISEL-NEXT: s_cbranch_execz .LBB90_29417; GFX950-GISEL-NEXT: .LBB90_4: ; %atomicrmw.private9418; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]9419; GFX950-GISEL-NEXT: s_nop 19420; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v4, -1, v2, vcc9421; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v4, off9422; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)9423; GFX950-GISEL-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]9424; GFX950-GISEL-NEXT: s_nop 19425; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, v0, v6, vcc9426; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v3, v1, v7, vcc9427; GFX950-GISEL-NEXT: scratch_store_dwordx2 v4, v[2:3], off9428; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]9429; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)9430; GFX950-GISEL-NEXT: s_branch .LBB90_59431; GFX950-GISEL-NEXT: .LBB90_5:9432 %zext.offset = zext i32 %voffset to i649433 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset9434 %cmpxchg = cmpxchg ptr %gep0, i64 %cmp, i64 %data seq_cst seq_cst9435 %rtn = extractvalue { i64, i1 } %cmpxchg, 09436 %cast.rtn = bitcast i64 %rtn to <2 x float>9437 ret <2 x float> %cast.rtn9438}9439 9440define amdgpu_ps <2 x float> @flat_cmpxchg_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %cmp, i64 %data) {9441; GFX1250-SDAG-LABEL: flat_cmpxchg_saddr_i64_rtn_neg128:9442; GFX1250-SDAG: ; %bb.0:9443; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 19444; GFX1250-SDAG-NEXT: v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v19445; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 09446; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff809447; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v39448; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)9449; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]9450; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[2:3], s[0:1], v[0:1]9451; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9452; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v39453; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v09454; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr19455; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo9456; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)9457; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s09458; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB91_39459; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow9460; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s09461; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB91_49462; GFX1250-SDAG-NEXT: .LBB91_2: ; %atomicrmw.phi9463; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s09464; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x09465; GFX1250-SDAG-NEXT: s_branch .LBB91_59466; GFX1250-SDAG-NEXT: .LBB91_3: ; %atomicrmw.global9467; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_SYS9468; GFX1250-SDAG-NEXT: s_wait_storecnt 0x09469; GFX1250-SDAG-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[2:3], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_SYS9470; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x09471; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_SYS9472; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr39473; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr6_vgpr79474; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s09475; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB91_29476; GFX1250-SDAG-NEXT: .LBB91_4: ; %atomicrmw.private9477; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]9478; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v29479; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)9480; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v8, -1, v0, vcc_lo9481; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v8, off9482; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x09483; GFX1250-SDAG-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[6:7]9484; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v1, v5 :: v_dual_cndmask_b32 v2, v0, v49485; GFX1250-SDAG-NEXT: scratch_store_b64 v8, v[2:3], off9486; GFX1250-SDAG-NEXT: s_wait_xcnt 0x09487; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s09488; GFX1250-SDAG-NEXT: s_branch .LBB91_59489; GFX1250-SDAG-NEXT: .LBB91_5:9490;9491; GFX1250-GISEL-LABEL: flat_cmpxchg_saddr_i64_rtn_neg128:9492; GFX1250-GISEL: ; %bb.0:9493; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 19494; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v8, v19495; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]9496; GFX1250-GISEL-NEXT: v_dual_mov_b32 v9, v2 :: v_dual_mov_b32 v6, v39497; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)9498; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v59499; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo9500; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)9501; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v09502; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v1, vcc_lo9503; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9504; GFX1250-GISEL-NEXT: v_dual_mov_b32 v7, v4 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v3 bitop3:0x149505; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v09506; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr19507; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo9508; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)9509; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s09510; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB91_39511; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow9512; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s09513; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB91_49514; GFX1250-GISEL-NEXT: .LBB91_2: ; %atomicrmw.phi9515; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s09516; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x09517; GFX1250-GISEL-NEXT: s_branch .LBB91_59518; GFX1250-GISEL-NEXT: .LBB91_3: ; %atomicrmw.global9519; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS9520; GFX1250-GISEL-NEXT: s_wait_storecnt 0x09521; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b64 v[0:1], v5, v[6:9], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_SYS9522; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x09523; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_SYS9524; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr29525; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr8_vgpr99526; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s09527; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB91_29528; GFX1250-GISEL-NEXT: .LBB91_4: ; %atomicrmw.private9529; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]9530; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v29531; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)9532; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo9533; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v4, off9534; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x09535; GFX1250-GISEL-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]9536; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v2, v0, v6 :: v_dual_cndmask_b32 v3, v1, v79537; GFX1250-GISEL-NEXT: scratch_store_b64 v4, v[2:3], off9538; GFX1250-GISEL-NEXT: s_wait_xcnt 0x09539; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s09540; GFX1250-GISEL-NEXT: s_branch .LBB91_59541; GFX1250-GISEL-NEXT: .LBB91_5:9542;9543; GFX950-SDAG-LABEL: flat_cmpxchg_saddr_i64_rtn_neg128:9544; GFX950-SDAG: ; %bb.0:9545; GFX950-SDAG-NEXT: v_mov_b32_e32 v6, v19546; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 09547; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]9548; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff809549; GFX950-SDAG-NEXT: s_mov_b32 s3, -19550; GFX950-SDAG-NEXT: v_mov_b32_e32 v5, v49551; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, v39552; GFX950-SDAG-NEXT: v_mov_b32_e32 v7, v29553; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base9554; GFX950-SDAG-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, s[2:3]9555; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v39556; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr19557; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc9558; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]9559; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB91_39560; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow9561; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]9562; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB91_49563; GFX950-SDAG-NEXT: .LBB91_2: ; %atomicrmw.phi9564; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]9565; GFX950-SDAG-NEXT: s_branch .LBB91_59566; GFX950-SDAG-NEXT: .LBB91_3: ; %atomicrmw.global9567; GFX950-SDAG-NEXT: buffer_wbl2 sc0 sc19568; GFX950-SDAG-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] sc0 sc19569; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9570; GFX950-SDAG-NEXT: buffer_inv sc0 sc19571; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr39572; GFX950-SDAG-NEXT: ; implicit-def: $vgpr6_vgpr79573; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]9574; GFX950-SDAG-NEXT: s_cbranch_execz .LBB91_29575; GFX950-SDAG-NEXT: .LBB91_4: ; %atomicrmw.private9576; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]9577; GFX950-SDAG-NEXT: s_nop 19578; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v8, -1, v2, vcc9579; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v8, off9580; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)9581; GFX950-SDAG-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]9582; GFX950-SDAG-NEXT: s_nop 19583; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc9584; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v2, v0, v4, vcc9585; GFX950-SDAG-NEXT: scratch_store_dwordx2 v8, v[2:3], off9586; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]9587; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)9588; GFX950-SDAG-NEXT: s_branch .LBB91_59589; GFX950-SDAG-NEXT: .LBB91_5:9590;9591; GFX950-GISEL-LABEL: flat_cmpxchg_saddr_i64_rtn_neg128:9592; GFX950-GISEL: ; %bb.0:9593; GFX950-GISEL-NEXT: v_mov_b32_e32 v9, v29594; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, v39595; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]9596; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v09597; GFX950-GISEL-NEXT: v_mov_b32_e32 v8, v19598; GFX950-GISEL-NEXT: s_nop 09599; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc9600; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v09601; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base9602; GFX950-GISEL-NEXT: s_nop 09603; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v1, vcc9604; GFX950-GISEL-NEXT: v_mov_b32_e32 v7, v49605; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v39606; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr19607; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc9608; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]9609; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB91_39610; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow9611; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]9612; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB91_49613; GFX950-GISEL-NEXT: .LBB91_2: ; %atomicrmw.phi9614; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]9615; GFX950-GISEL-NEXT: s_branch .LBB91_59616; GFX950-GISEL-NEXT: .LBB91_3: ; %atomicrmw.global9617; GFX950-GISEL-NEXT: buffer_wbl2 sc0 sc19618; GFX950-GISEL-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] sc0 sc19619; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9620; GFX950-GISEL-NEXT: buffer_inv sc0 sc19621; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr39622; GFX950-GISEL-NEXT: ; implicit-def: $vgpr8_vgpr99623; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]9624; GFX950-GISEL-NEXT: s_cbranch_execz .LBB91_29625; GFX950-GISEL-NEXT: .LBB91_4: ; %atomicrmw.private9626; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]9627; GFX950-GISEL-NEXT: s_nop 19628; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v4, -1, v2, vcc9629; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v4, off9630; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)9631; GFX950-GISEL-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]9632; GFX950-GISEL-NEXT: s_nop 19633; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, v0, v6, vcc9634; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v3, v1, v7, vcc9635; GFX950-GISEL-NEXT: scratch_store_dwordx2 v4, v[2:3], off9636; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]9637; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)9638; GFX950-GISEL-NEXT: s_branch .LBB91_59639; GFX950-GISEL-NEXT: .LBB91_5:9640 %zext.offset = zext i32 %voffset to i649641 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset9642 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -1289643 %cmpxchg = cmpxchg ptr %gep1, i64 %cmp, i64 %data seq_cst seq_cst9644 %rtn = extractvalue { i64, i1 } %cmpxchg, 09645 %cast.rtn = bitcast i64 %rtn to <2 x float>9646 ret <2 x float> %cast.rtn9647}9648 9649define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %cmp, i64 %data) {9650; GFX1250-SDAG-LABEL: flat_cmpxchg_saddr_i64_nortn:9651; GFX1250-SDAG: ; %bb.0:9652; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 19653; GFX1250-SDAG-NEXT: v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v19654; GFX1250-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v5, v49655; GFX1250-SDAG-NEXT: v_mov_b32_e32 v4, v39656; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo9657; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)9658; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]9659; GFX1250-SDAG-NEXT: v_xor_b32_e32 v2, src_flat_scratch_base_hi, v19660; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)9661; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v29662; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s09663; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB92_39664; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow9665; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s09666; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB92_49667; GFX1250-SDAG-NEXT: .LBB92_2: ; %atomicrmw.phi9668; GFX1250-SDAG-NEXT: s_endpgm9669; GFX1250-SDAG-NEXT: .LBB92_3: ; %atomicrmw.global9670; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_SYS9671; GFX1250-SDAG-NEXT: s_wait_storecnt 0x09672; GFX1250-SDAG-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[4:7] scope:SCOPE_SYS9673; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x09674; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_SYS9675; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr19676; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr6_vgpr79677; GFX1250-SDAG-NEXT: s_wait_xcnt 0x09678; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s09679; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB92_29680; GFX1250-SDAG-NEXT: .LBB92_4: ; %atomicrmw.private9681; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]9682; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v09683; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)9684; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo9685; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v2, off9686; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x09687; GFX1250-SDAG-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[6:7]9688; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v1, v5 :: v_dual_cndmask_b32 v0, v0, v49689; GFX1250-SDAG-NEXT: scratch_store_b64 v2, v[0:1], off9690; GFX1250-SDAG-NEXT: s_endpgm9691;9692; GFX1250-GISEL-LABEL: flat_cmpxchg_saddr_i64_nortn:9693; GFX1250-GISEL: ; %bb.0:9694; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 19695; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v9, v29696; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v49697; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]9698; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo9699; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9700; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v09701; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo9702; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9703; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v39704; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v19705; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s09706; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB92_39707; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow9708; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s09709; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB92_49710; GFX1250-GISEL-NEXT: .LBB92_2: ; %atomicrmw.phi9711; GFX1250-GISEL-NEXT: s_endpgm9712; GFX1250-GISEL-NEXT: .LBB92_3: ; %atomicrmw.global9713; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS9714; GFX1250-GISEL-NEXT: s_wait_storecnt 0x09715; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b64 v0, v[6:9], s[2:3] scope:SCOPE_SYS9716; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x09717; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_SYS9718; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr29719; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr8_vgpr99720; GFX1250-GISEL-NEXT: s_wait_xcnt 0x09721; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s09722; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB92_29723; GFX1250-GISEL-NEXT: .LBB92_4: ; %atomicrmw.private9724; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]9725; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v29726; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)9727; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo9728; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off9729; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x09730; GFX1250-GISEL-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]9731; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v0, v0, v6 :: v_dual_cndmask_b32 v1, v1, v79732; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off9733; GFX1250-GISEL-NEXT: s_endpgm9734;9735; GFX950-SDAG-LABEL: flat_cmpxchg_saddr_i64_nortn:9736; GFX950-SDAG: ; %bb.0:9737; GFX950-SDAG-NEXT: v_mov_b32_e32 v6, v19738; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 09739; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base9740; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]9741; GFX950-SDAG-NEXT: v_mov_b32_e32 v5, v49742; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, v39743; GFX950-SDAG-NEXT: v_mov_b32_e32 v7, v29744; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v19745; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc9746; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]9747; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB92_39748; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow9749; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]9750; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB92_49751; GFX950-SDAG-NEXT: .LBB92_2: ; %atomicrmw.phi9752; GFX950-SDAG-NEXT: s_endpgm9753; GFX950-SDAG-NEXT: .LBB92_3: ; %atomicrmw.global9754; GFX950-SDAG-NEXT: buffer_wbl2 sc0 sc19755; GFX950-SDAG-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:7] sc19756; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9757; GFX950-SDAG-NEXT: buffer_inv sc0 sc19758; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr19759; GFX950-SDAG-NEXT: ; implicit-def: $vgpr6_vgpr79760; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]9761; GFX950-SDAG-NEXT: s_cbranch_execz .LBB92_29762; GFX950-SDAG-NEXT: .LBB92_4: ; %atomicrmw.private9763; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]9764; GFX950-SDAG-NEXT: s_nop 19765; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc9766; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v2, off9767; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)9768; GFX950-SDAG-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]9769; GFX950-SDAG-NEXT: s_nop 19770; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc9771; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc9772; GFX950-SDAG-NEXT: scratch_store_dwordx2 v2, v[0:1], off9773; GFX950-SDAG-NEXT: s_endpgm9774;9775; GFX950-GISEL-LABEL: flat_cmpxchg_saddr_i64_nortn:9776; GFX950-GISEL: ; %bb.0:9777; GFX950-GISEL-NEXT: v_mov_b32_e32 v9, v29778; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, v39779; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]9780; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v09781; GFX950-GISEL-NEXT: v_mov_b32_e32 v8, v19782; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base9783; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc9784; GFX950-GISEL-NEXT: v_mov_b32_e32 v7, v49785; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v19786; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc9787; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]9788; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB92_39789; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow9790; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]9791; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB92_49792; GFX950-GISEL-NEXT: .LBB92_2: ; %atomicrmw.phi9793; GFX950-GISEL-NEXT: s_endpgm9794; GFX950-GISEL-NEXT: .LBB92_3: ; %atomicrmw.global9795; GFX950-GISEL-NEXT: buffer_wbl2 sc0 sc19796; GFX950-GISEL-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:9] sc19797; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9798; GFX950-GISEL-NEXT: buffer_inv sc0 sc19799; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr19800; GFX950-GISEL-NEXT: ; implicit-def: $vgpr8_vgpr99801; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]9802; GFX950-GISEL-NEXT: s_cbranch_execz .LBB92_29803; GFX950-GISEL-NEXT: .LBB92_4: ; %atomicrmw.private9804; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]9805; GFX950-GISEL-NEXT: s_nop 19806; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc9807; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off9808; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)9809; GFX950-GISEL-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]9810; GFX950-GISEL-NEXT: s_nop 19811; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc9812; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc9813; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off9814; GFX950-GISEL-NEXT: s_endpgm9815 %zext.offset = zext i32 %voffset to i649816 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset9817 %unused = cmpxchg ptr %gep0, i64 %cmp, i64 %data seq_cst seq_cst9818 ret void9819}9820 9821define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %cmp, i64 %data) {9822; GFX1250-SDAG-LABEL: flat_cmpxchg_saddr_i64_nortn_neg128:9823; GFX1250-SDAG: ; %bb.0:9824; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 19825; GFX1250-SDAG-NEXT: v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v19826; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 09827; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff809828; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v39829; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)9830; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]9831; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]9832; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo9833; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9834; GFX1250-SDAG-NEXT: v_xor_b32_e32 v2, src_flat_scratch_base_hi, v19835; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v29836; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s09837; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB93_39838; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow9839; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s09840; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB93_49841; GFX1250-SDAG-NEXT: .LBB93_2: ; %atomicrmw.phi9842; GFX1250-SDAG-NEXT: s_endpgm9843; GFX1250-SDAG-NEXT: .LBB93_3: ; %atomicrmw.global9844; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_SYS9845; GFX1250-SDAG-NEXT: s_wait_storecnt 0x09846; GFX1250-SDAG-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[4:7] scope:SCOPE_SYS9847; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x09848; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_SYS9849; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr19850; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr6_vgpr79851; GFX1250-SDAG-NEXT: s_wait_xcnt 0x09852; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s09853; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB93_29854; GFX1250-SDAG-NEXT: .LBB93_4: ; %atomicrmw.private9855; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]9856; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v09857; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)9858; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo9859; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v2, off9860; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x09861; GFX1250-SDAG-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[6:7]9862; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v1, v5 :: v_dual_cndmask_b32 v0, v0, v49863; GFX1250-SDAG-NEXT: scratch_store_b64 v2, v[0:1], off9864; GFX1250-SDAG-NEXT: s_endpgm9865;9866; GFX1250-GISEL-LABEL: flat_cmpxchg_saddr_i64_nortn_neg128:9867; GFX1250-GISEL: ; %bb.0:9868; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 19869; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v9, v29870; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v49871; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]9872; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo9873; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9874; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v09875; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo9876; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)9877; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v19878; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo9879; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9880; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v39881; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v19882; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s09883; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB93_39884; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow9885; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s09886; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB93_49887; GFX1250-GISEL-NEXT: .LBB93_2: ; %atomicrmw.phi9888; GFX1250-GISEL-NEXT: s_endpgm9889; GFX1250-GISEL-NEXT: .LBB93_3: ; %atomicrmw.global9890; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS9891; GFX1250-GISEL-NEXT: s_wait_storecnt 0x09892; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b64 v0, v[6:9], s[2:3] offset:-128 scope:SCOPE_SYS9893; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x09894; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_SYS9895; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr29896; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr8_vgpr99897; GFX1250-GISEL-NEXT: s_wait_xcnt 0x09898; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s09899; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB93_29900; GFX1250-GISEL-NEXT: .LBB93_4: ; %atomicrmw.private9901; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]9902; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v29903; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)9904; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo9905; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off9906; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x09907; GFX1250-GISEL-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]9908; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v0, v0, v6 :: v_dual_cndmask_b32 v1, v1, v79909; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off9910; GFX1250-GISEL-NEXT: s_endpgm9911;9912; GFX950-SDAG-LABEL: flat_cmpxchg_saddr_i64_nortn_neg128:9913; GFX950-SDAG: ; %bb.0:9914; GFX950-SDAG-NEXT: v_mov_b32_e32 v6, v19915; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 09916; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]9917; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff809918; GFX950-SDAG-NEXT: s_mov_b32 s3, -19919; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base9920; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]9921; GFX950-SDAG-NEXT: v_mov_b32_e32 v5, v49922; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, v39923; GFX950-SDAG-NEXT: v_mov_b32_e32 v7, v29924; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v19925; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc9926; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]9927; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB93_39928; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow9929; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]9930; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB93_49931; GFX950-SDAG-NEXT: .LBB93_2: ; %atomicrmw.phi9932; GFX950-SDAG-NEXT: s_endpgm9933; GFX950-SDAG-NEXT: .LBB93_3: ; %atomicrmw.global9934; GFX950-SDAG-NEXT: buffer_wbl2 sc0 sc19935; GFX950-SDAG-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:7] sc19936; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9937; GFX950-SDAG-NEXT: buffer_inv sc0 sc19938; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr19939; GFX950-SDAG-NEXT: ; implicit-def: $vgpr6_vgpr79940; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]9941; GFX950-SDAG-NEXT: s_cbranch_execz .LBB93_29942; GFX950-SDAG-NEXT: .LBB93_4: ; %atomicrmw.private9943; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]9944; GFX950-SDAG-NEXT: s_nop 19945; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc9946; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v2, off9947; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)9948; GFX950-SDAG-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]9949; GFX950-SDAG-NEXT: s_nop 19950; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc9951; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc9952; GFX950-SDAG-NEXT: scratch_store_dwordx2 v2, v[0:1], off9953; GFX950-SDAG-NEXT: s_endpgm9954;9955; GFX950-GISEL-LABEL: flat_cmpxchg_saddr_i64_nortn_neg128:9956; GFX950-GISEL: ; %bb.0:9957; GFX950-GISEL-NEXT: v_mov_b32_e32 v9, v29958; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, v39959; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]9960; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v09961; GFX950-GISEL-NEXT: v_mov_b32_e32 v8, v19962; GFX950-GISEL-NEXT: s_nop 09963; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc9964; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v09965; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base9966; GFX950-GISEL-NEXT: s_nop 09967; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc9968; GFX950-GISEL-NEXT: v_mov_b32_e32 v7, v49969; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v19970; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc9971; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]9972; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB93_39973; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow9974; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]9975; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB93_49976; GFX950-GISEL-NEXT: .LBB93_2: ; %atomicrmw.phi9977; GFX950-GISEL-NEXT: s_endpgm9978; GFX950-GISEL-NEXT: .LBB93_3: ; %atomicrmw.global9979; GFX950-GISEL-NEXT: buffer_wbl2 sc0 sc19980; GFX950-GISEL-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:9] sc19981; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)9982; GFX950-GISEL-NEXT: buffer_inv sc0 sc19983; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr19984; GFX950-GISEL-NEXT: ; implicit-def: $vgpr8_vgpr99985; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]9986; GFX950-GISEL-NEXT: s_cbranch_execz .LBB93_29987; GFX950-GISEL-NEXT: .LBB93_4: ; %atomicrmw.private9988; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]9989; GFX950-GISEL-NEXT: s_nop 19990; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc9991; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off9992; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)9993; GFX950-GISEL-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]9994; GFX950-GISEL-NEXT: s_nop 19995; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc9996; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc9997; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off9998; GFX950-GISEL-NEXT: s_endpgm9999 %zext.offset = zext i32 %voffset to i6410000 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset10001 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -12810002 %unused = cmpxchg ptr %gep1, i64 %cmp, i64 %data seq_cst seq_cst10003 ret void10004}10005 10006; --------------------------------------------------------------------------------10007; amdgcn atomic inc10008; --------------------------------------------------------------------------------10009 10010define amdgpu_ps float @flat_inc_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {10011; GFX1250-LABEL: flat_inc_saddr_i32_rtn:10012; GFX1250: ; %bb.0:10013; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 110014; GFX1250-NEXT: flat_atomic_inc_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV10015; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x010016; GFX1250-NEXT: ; return to shader part epilog10017;10018; GFX950-SDAG-LABEL: flat_inc_saddr_i32_rtn:10019; GFX950-SDAG: ; %bb.0:10020; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v110021; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 010022; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]10023; GFX950-SDAG-NEXT: flat_atomic_inc v0, v[0:1], v2 sc010024; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10025; GFX950-SDAG-NEXT: ; return to shader part epilog10026;10027; GFX950-GISEL-LABEL: flat_inc_saddr_i32_rtn:10028; GFX950-GISEL: ; %bb.0:10029; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]10030; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v010031; GFX950-GISEL-NEXT: s_nop 110032; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc10033; GFX950-GISEL-NEXT: flat_atomic_inc v0, v[2:3], v1 sc010034; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10035; GFX950-GISEL-NEXT: ; return to shader part epilog10036 %zext.offset = zext i32 %voffset to i6410037 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset10038 %rtn = atomicrmw uinc_wrap ptr %gep0, i32 %data syncscope("agent") monotonic10039 %cast.rtn = bitcast i32 %rtn to float10040 ret float %cast.rtn10041}10042 10043define amdgpu_ps float @flat_inc_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {10044; GFX1250-LABEL: flat_inc_saddr_i32_rtn_neg128:10045; GFX1250: ; %bb.0:10046; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 110047; GFX1250-NEXT: flat_atomic_inc_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV10048; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x010049; GFX1250-NEXT: ; return to shader part epilog10050;10051; GFX950-SDAG-LABEL: flat_inc_saddr_i32_rtn_neg128:10052; GFX950-SDAG: ; %bb.0:10053; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v110054; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 010055; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]10056; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v010057; GFX950-SDAG-NEXT: s_nop 110058; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc10059; GFX950-SDAG-NEXT: flat_atomic_inc v0, v[0:1], v2 sc010060; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10061; GFX950-SDAG-NEXT: ; return to shader part epilog10062;10063; GFX950-GISEL-LABEL: flat_inc_saddr_i32_rtn_neg128:10064; GFX950-GISEL: ; %bb.0:10065; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]10066; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v010067; GFX950-GISEL-NEXT: s_nop 110068; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc10069; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v010070; GFX950-GISEL-NEXT: s_nop 110071; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc10072; GFX950-GISEL-NEXT: flat_atomic_inc v0, v[2:3], v1 sc010073; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10074; GFX950-GISEL-NEXT: ; return to shader part epilog10075 %zext.offset = zext i32 %voffset to i6410076 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset10077 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -12810078 %rtn = atomicrmw uinc_wrap ptr %gep1, i32 %data syncscope("agent") monotonic10079 %cast.rtn = bitcast i32 %rtn to float10080 ret float %cast.rtn10081}10082 10083define amdgpu_ps void @flat_inc_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {10084; GFX1250-LABEL: flat_inc_saddr_i32_nortn:10085; GFX1250: ; %bb.0:10086; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 110087; GFX1250-NEXT: flat_atomic_inc_u32 v0, v1, s[2:3] scope:SCOPE_DEV10088; GFX1250-NEXT: s_endpgm10089;10090; GFX950-SDAG-LABEL: flat_inc_saddr_i32_nortn:10091; GFX950-SDAG: ; %bb.0:10092; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v110093; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 010094; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]10095; GFX950-SDAG-NEXT: flat_atomic_inc v[0:1], v210096; GFX950-SDAG-NEXT: s_endpgm10097;10098; GFX950-GISEL-LABEL: flat_inc_saddr_i32_nortn:10099; GFX950-GISEL: ; %bb.0:10100; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]10101; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v010102; GFX950-GISEL-NEXT: s_nop 110103; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc10104; GFX950-GISEL-NEXT: flat_atomic_inc v[2:3], v110105; GFX950-GISEL-NEXT: s_endpgm10106 %zext.offset = zext i32 %voffset to i6410107 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset10108 %unused = atomicrmw uinc_wrap ptr %gep0, i32 %data syncscope("agent") monotonic10109 ret void10110}10111 10112define amdgpu_ps void @flat_inc_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {10113; GFX1250-LABEL: flat_inc_saddr_i32_nortn_neg128:10114; GFX1250: ; %bb.0:10115; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 110116; GFX1250-NEXT: flat_atomic_inc_u32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV10117; GFX1250-NEXT: s_endpgm10118;10119; GFX950-SDAG-LABEL: flat_inc_saddr_i32_nortn_neg128:10120; GFX950-SDAG: ; %bb.0:10121; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v110122; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 010123; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]10124; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v010125; GFX950-SDAG-NEXT: s_nop 110126; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc10127; GFX950-SDAG-NEXT: flat_atomic_inc v[0:1], v210128; GFX950-SDAG-NEXT: s_endpgm10129;10130; GFX950-GISEL-LABEL: flat_inc_saddr_i32_nortn_neg128:10131; GFX950-GISEL: ; %bb.0:10132; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]10133; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v010134; GFX950-GISEL-NEXT: s_nop 110135; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc10136; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v010137; GFX950-GISEL-NEXT: s_nop 110138; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc10139; GFX950-GISEL-NEXT: flat_atomic_inc v[2:3], v110140; GFX950-GISEL-NEXT: s_endpgm10141 %zext.offset = zext i32 %voffset to i6410142 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset10143 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -12810144 %unused = atomicrmw uinc_wrap ptr %gep1, i32 %data syncscope("agent") monotonic10145 ret void10146}10147 10148define amdgpu_ps <2 x float> @flat_inc_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {10149; GFX1250-SDAG-LABEL: flat_inc_saddr_i64_rtn:10150; GFX1250-SDAG: ; %bb.0:10151; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 110152; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v110153; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 010154; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10155; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]10156; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v510157; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)10158; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v010159; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr110160; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo10161; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s010162; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB98_310163; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow10164; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s010165; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB98_410166; GFX1250-SDAG-NEXT: .LBB98_2: ; %atomicrmw.phi10167; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s010168; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x010169; GFX1250-SDAG-NEXT: s_branch .LBB98_510170; GFX1250-SDAG-NEXT: .LBB98_3: ; %atomicrmw.global10171; GFX1250-SDAG-NEXT: flat_atomic_inc_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV10172; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr510173; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr310174; GFX1250-SDAG-NEXT: s_wait_xcnt 0x010175; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s010176; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB98_210177; GFX1250-SDAG-NEXT: .LBB98_4: ; %atomicrmw.private10178; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]10179; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x010180; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v410181; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)10182; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo10183; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v6, off10184; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x010185; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], 1, v[0:1]10186; GFX1250-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[2:3]10187; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, 0, v5 :: v_dual_cndmask_b32 v2, 0, v410188; GFX1250-SDAG-NEXT: scratch_store_b64 v6, v[2:3], off10189; GFX1250-SDAG-NEXT: s_wait_xcnt 0x010190; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s010191; GFX1250-SDAG-NEXT: s_branch .LBB98_510192; GFX1250-SDAG-NEXT: .LBB98_5:10193;10194; GFX1250-GISEL-LABEL: flat_inc_saddr_i64_rtn:10195; GFX1250-GISEL: ; %bb.0:10196; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 110197; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v110198; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]10199; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10200; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v310201; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo10202; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10203; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x1410204; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v010205; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr110206; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo10207; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)10208; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s010209; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB98_310210; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow10211; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s010212; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB98_410213; GFX1250-GISEL-NEXT: .LBB98_2: ; %atomicrmw.phi10214; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s010215; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x010216; GFX1250-GISEL-NEXT: s_branch .LBB98_510217; GFX1250-GISEL-NEXT: .LBB98_3: ; %atomicrmw.global10218; GFX1250-GISEL-NEXT: flat_atomic_inc_u64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV10219; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr610220; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr510221; GFX1250-GISEL-NEXT: s_wait_xcnt 0x010222; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s010223; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB98_210224; GFX1250-GISEL-NEXT: .LBB98_4: ; %atomicrmw.private10225; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]10226; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x010227; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v610228; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)10229; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo10230; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off10231; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x010232; GFX1250-GISEL-NEXT: v_add_nc_u64_e32 v[2:3], 1, v[0:1]10233; GFX1250-GISEL-NEXT: v_cmp_ge_u64_e32 vcc_lo, v[0:1], v[4:5]10234; GFX1250-GISEL-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc_lo10235; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3)10236; GFX1250-GISEL-NEXT: v_cndmask_b32_e64 v3, v3, 0, vcc_lo10237; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off10238; GFX1250-GISEL-NEXT: s_wait_xcnt 0x010239; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s010240; GFX1250-GISEL-NEXT: s_branch .LBB98_510241; GFX1250-GISEL-NEXT: .LBB98_5:10242;10243; GFX950-SDAG-LABEL: flat_inc_saddr_i64_rtn:10244; GFX950-SDAG: ; %bb.0:10245; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v210246; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v110247; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 010248; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base10249; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1]10250; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v510251; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr110252; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc10253; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]10254; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB98_310255; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow10256; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10257; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB98_410258; GFX950-SDAG-NEXT: .LBB98_2: ; %atomicrmw.phi10259; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]10260; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10261; GFX950-SDAG-NEXT: s_branch .LBB98_510262; GFX950-SDAG-NEXT: .LBB98_3: ; %atomicrmw.global10263; GFX950-SDAG-NEXT: flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] sc010264; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr510265; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr310266; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10267; GFX950-SDAG-NEXT: s_cbranch_execz .LBB98_210268; GFX950-SDAG-NEXT: .LBB98_4: ; %atomicrmw.private10269; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]10270; GFX950-SDAG-NEXT: s_nop 110271; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc10272; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)10273; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v6, off10274; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)10275; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, 110276; GFX950-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]10277; GFX950-SDAG-NEXT: s_nop 110278; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v3, 0, v5, vcc10279; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v4, vcc10280; GFX950-SDAG-NEXT: scratch_store_dwordx2 v6, v[2:3], off10281; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]10282; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)10283; GFX950-SDAG-NEXT: s_branch .LBB98_510284; GFX950-SDAG-NEXT: .LBB98_5:10285;10286; GFX950-GISEL-LABEL: flat_inc_saddr_i64_rtn:10287; GFX950-GISEL: ; %bb.0:10288; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v210289; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]10290; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v010291; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base10292; GFX950-GISEL-NEXT: s_nop 010293; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc10294; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v110295; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v310296; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr110297; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc10298; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]10299; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB98_310300; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow10301; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10302; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB98_410303; GFX950-GISEL-NEXT: .LBB98_2: ; %atomicrmw.phi10304; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]10305; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10306; GFX950-GISEL-NEXT: s_branch .LBB98_510307; GFX950-GISEL-NEXT: .LBB98_3: ; %atomicrmw.global10308; GFX950-GISEL-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[4:5] sc010309; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr310310; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr510311; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10312; GFX950-GISEL-NEXT: s_cbranch_execz .LBB98_210313; GFX950-GISEL-NEXT: .LBB98_4: ; %atomicrmw.private10314; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]10315; GFX950-GISEL-NEXT: s_nop 110316; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc10317; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)10318; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off10319; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)10320; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 1, v010321; GFX950-GISEL-NEXT: s_nop 110322; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc10323; GFX950-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, v[0:1], v[4:5]10324; GFX950-GISEL-NEXT: s_nop 110325; GFX950-GISEL-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc10326; GFX950-GISEL-NEXT: v_cndmask_b32_e64 v3, v3, 0, vcc10327; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off10328; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]10329; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)10330; GFX950-GISEL-NEXT: s_branch .LBB98_510331; GFX950-GISEL-NEXT: .LBB98_5:10332 %zext.offset = zext i32 %voffset to i6410333 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset10334 %rtn = atomicrmw uinc_wrap ptr %gep0, i64 %data syncscope("agent") monotonic10335 %cast.rtn = bitcast i64 %rtn to <2 x float>10336 ret <2 x float> %cast.rtn10337}10338 10339define amdgpu_ps <2 x float> @flat_inc_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {10340; GFX1250-SDAG-LABEL: flat_inc_saddr_i64_rtn_neg128:10341; GFX1250-SDAG: ; %bb.0:10342; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 110343; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v110344; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 010345; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff8010346; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10347; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]10348; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]10349; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10350; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v510351; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v010352; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr110353; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo10354; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)10355; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s010356; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB99_310357; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow10358; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s010359; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB99_410360; GFX1250-SDAG-NEXT: .LBB99_2: ; %atomicrmw.phi10361; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s010362; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x010363; GFX1250-SDAG-NEXT: s_branch .LBB99_510364; GFX1250-SDAG-NEXT: .LBB99_3: ; %atomicrmw.global10365; GFX1250-SDAG-NEXT: flat_atomic_inc_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV10366; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr510367; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr310368; GFX1250-SDAG-NEXT: s_wait_xcnt 0x010369; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s010370; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB99_210371; GFX1250-SDAG-NEXT: .LBB99_4: ; %atomicrmw.private10372; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]10373; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x010374; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v410375; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)10376; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo10377; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v6, off10378; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x010379; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], 1, v[0:1]10380; GFX1250-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[2:3]10381; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, 0, v5 :: v_dual_cndmask_b32 v2, 0, v410382; GFX1250-SDAG-NEXT: scratch_store_b64 v6, v[2:3], off10383; GFX1250-SDAG-NEXT: s_wait_xcnt 0x010384; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s010385; GFX1250-SDAG-NEXT: s_branch .LBB99_510386; GFX1250-SDAG-NEXT: .LBB99_5:10387;10388; GFX1250-GISEL-LABEL: flat_inc_saddr_i64_rtn_neg128:10389; GFX1250-GISEL: ; %bb.0:10390; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 110391; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v110392; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]10393; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10394; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v310395; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo10396; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)10397; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v010398; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo10399; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10400; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x1410401; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v010402; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr110403; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo10404; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)10405; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s010406; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB99_310407; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow10408; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s010409; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB99_410410; GFX1250-GISEL-NEXT: .LBB99_2: ; %atomicrmw.phi10411; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s010412; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x010413; GFX1250-GISEL-NEXT: s_branch .LBB99_510414; GFX1250-GISEL-NEXT: .LBB99_3: ; %atomicrmw.global10415; GFX1250-GISEL-NEXT: flat_atomic_inc_u64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV10416; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr610417; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr510418; GFX1250-GISEL-NEXT: s_wait_xcnt 0x010419; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s010420; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB99_210421; GFX1250-GISEL-NEXT: .LBB99_4: ; %atomicrmw.private10422; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]10423; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x010424; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v610425; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)10426; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo10427; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off10428; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x010429; GFX1250-GISEL-NEXT: v_add_nc_u64_e32 v[2:3], 1, v[0:1]10430; GFX1250-GISEL-NEXT: v_cmp_ge_u64_e32 vcc_lo, v[0:1], v[4:5]10431; GFX1250-GISEL-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc_lo10432; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3)10433; GFX1250-GISEL-NEXT: v_cndmask_b32_e64 v3, v3, 0, vcc_lo10434; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off10435; GFX1250-GISEL-NEXT: s_wait_xcnt 0x010436; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s010437; GFX1250-GISEL-NEXT: s_branch .LBB99_510438; GFX1250-GISEL-NEXT: .LBB99_5:10439;10440; GFX950-SDAG-LABEL: flat_inc_saddr_i64_rtn_neg128:10441; GFX950-SDAG: ; %bb.0:10442; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v210443; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v110444; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 010445; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]10446; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff8010447; GFX950-SDAG-NEXT: s_mov_b32 s3, -110448; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base10449; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]10450; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v510451; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr110452; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc10453; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]10454; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB99_310455; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow10456; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10457; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB99_410458; GFX950-SDAG-NEXT: .LBB99_2: ; %atomicrmw.phi10459; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]10460; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10461; GFX950-SDAG-NEXT: s_branch .LBB99_510462; GFX950-SDAG-NEXT: .LBB99_3: ; %atomicrmw.global10463; GFX950-SDAG-NEXT: flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] sc010464; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr510465; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr310466; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10467; GFX950-SDAG-NEXT: s_cbranch_execz .LBB99_210468; GFX950-SDAG-NEXT: .LBB99_4: ; %atomicrmw.private10469; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]10470; GFX950-SDAG-NEXT: s_nop 110471; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc10472; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)10473; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v6, off10474; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)10475; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, 110476; GFX950-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]10477; GFX950-SDAG-NEXT: s_nop 110478; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v3, 0, v5, vcc10479; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v4, vcc10480; GFX950-SDAG-NEXT: scratch_store_dwordx2 v6, v[2:3], off10481; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]10482; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)10483; GFX950-SDAG-NEXT: s_branch .LBB99_510484; GFX950-SDAG-NEXT: .LBB99_5:10485;10486; GFX950-GISEL-LABEL: flat_inc_saddr_i64_rtn_neg128:10487; GFX950-GISEL: ; %bb.0:10488; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v210489; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]10490; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v010491; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v110492; GFX950-GISEL-NEXT: s_nop 010493; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc10494; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v010495; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base10496; GFX950-GISEL-NEXT: s_nop 010497; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v1, vcc10498; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v310499; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr110500; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc10501; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]10502; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB99_310503; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow10504; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10505; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB99_410506; GFX950-GISEL-NEXT: .LBB99_2: ; %atomicrmw.phi10507; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]10508; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10509; GFX950-GISEL-NEXT: s_branch .LBB99_510510; GFX950-GISEL-NEXT: .LBB99_3: ; %atomicrmw.global10511; GFX950-GISEL-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3], v[4:5] sc010512; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr310513; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr510514; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10515; GFX950-GISEL-NEXT: s_cbranch_execz .LBB99_210516; GFX950-GISEL-NEXT: .LBB99_4: ; %atomicrmw.private10517; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]10518; GFX950-GISEL-NEXT: s_nop 110519; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc10520; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)10521; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off10522; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)10523; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 1, v010524; GFX950-GISEL-NEXT: s_nop 110525; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc10526; GFX950-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, v[0:1], v[4:5]10527; GFX950-GISEL-NEXT: s_nop 110528; GFX950-GISEL-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc10529; GFX950-GISEL-NEXT: v_cndmask_b32_e64 v3, v3, 0, vcc10530; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off10531; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]10532; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)10533; GFX950-GISEL-NEXT: s_branch .LBB99_510534; GFX950-GISEL-NEXT: .LBB99_5:10535 %zext.offset = zext i32 %voffset to i6410536 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset10537 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -12810538 %rtn = atomicrmw uinc_wrap ptr %gep1, i64 %data syncscope("agent") monotonic10539 %cast.rtn = bitcast i64 %rtn to <2 x float>10540 ret <2 x float> %cast.rtn10541}10542 10543define amdgpu_ps void @flat_inc_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {10544; GFX1250-SDAG-LABEL: flat_inc_saddr_i64_nortn:10545; GFX1250-SDAG: ; %bb.0:10546; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 110547; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v110548; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 010549; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo10550; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10551; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]10552; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v110553; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)10554; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v410555; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s010556; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB100_310557; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow10558; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s010559; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB100_410560; GFX1250-SDAG-NEXT: .LBB100_2: ; %atomicrmw.phi10561; GFX1250-SDAG-NEXT: s_endpgm10562; GFX1250-SDAG-NEXT: .LBB100_3: ; %atomicrmw.global10563; GFX1250-SDAG-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3] scope:SCOPE_DEV10564; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr110565; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr310566; GFX1250-SDAG-NEXT: s_wait_xcnt 0x010567; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s010568; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB100_210569; GFX1250-SDAG-NEXT: .LBB100_4: ; %atomicrmw.private10570; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]10571; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v010572; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)10573; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo10574; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v6, off10575; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x010576; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], 1, v[0:1]10577; GFX1250-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[2:3]10578; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, 0, v5 :: v_dual_cndmask_b32 v0, 0, v410579; GFX1250-SDAG-NEXT: scratch_store_b64 v6, v[0:1], off10580; GFX1250-SDAG-NEXT: s_endpgm10581;10582; GFX1250-GISEL-LABEL: flat_inc_saddr_i64_nortn:10583; GFX1250-GISEL: ; %bb.0:10584; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 110585; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v210586; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]10587; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo10588; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10589; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v010590; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo10591; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10592; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v310593; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v110594; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s010595; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB100_310596; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow10597; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s010598; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB100_410599; GFX1250-GISEL-NEXT: .LBB100_2: ; %atomicrmw.phi10600; GFX1250-GISEL-NEXT: s_endpgm10601; GFX1250-GISEL-NEXT: .LBB100_3: ; %atomicrmw.global10602; GFX1250-GISEL-NEXT: flat_atomic_inc_u64 v0, v[4:5], s[2:3] scope:SCOPE_DEV10603; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr210604; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr510605; GFX1250-GISEL-NEXT: s_wait_xcnt 0x010606; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s010607; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB100_210608; GFX1250-GISEL-NEXT: .LBB100_4: ; %atomicrmw.private10609; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]10610; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v210611; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)10612; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo10613; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off10614; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x010615; GFX1250-GISEL-NEXT: v_add_nc_u64_e32 v[2:3], 1, v[0:1]10616; GFX1250-GISEL-NEXT: v_cmp_ge_u64_e32 vcc_lo, v[0:1], v[4:5]10617; GFX1250-GISEL-NEXT: v_cndmask_b32_e64 v0, v2, 0, vcc_lo10618; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3)10619; GFX1250-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0, vcc_lo10620; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[0:1], off10621; GFX1250-GISEL-NEXT: s_endpgm10622;10623; GFX950-SDAG-LABEL: flat_inc_saddr_i64_nortn:10624; GFX950-SDAG: ; %bb.0:10625; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v210626; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v110627; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 010628; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base10629; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]10630; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v110631; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc10632; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]10633; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB100_310634; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow10635; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10636; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB100_410637; GFX950-SDAG-NEXT: .LBB100_2: ; %atomicrmw.phi10638; GFX950-SDAG-NEXT: s_endpgm10639; GFX950-SDAG-NEXT: .LBB100_3: ; %atomicrmw.global10640; GFX950-SDAG-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]10641; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr110642; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr310643; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10644; GFX950-SDAG-NEXT: s_cbranch_execz .LBB100_210645; GFX950-SDAG-NEXT: .LBB100_4: ; %atomicrmw.private10646; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]10647; GFX950-SDAG-NEXT: s_nop 110648; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc10649; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v6, off10650; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)10651; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, 110652; GFX950-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]10653; GFX950-SDAG-NEXT: s_nop 110654; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v5, vcc10655; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v4, vcc10656; GFX950-SDAG-NEXT: scratch_store_dwordx2 v6, v[0:1], off10657; GFX950-SDAG-NEXT: s_endpgm10658;10659; GFX950-GISEL-LABEL: flat_inc_saddr_i64_nortn:10660; GFX950-GISEL: ; %bb.0:10661; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v210662; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]10663; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v010664; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v110665; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base10666; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc10667; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v110668; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc10669; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]10670; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB100_310671; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow10672; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10673; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB100_410674; GFX950-GISEL-NEXT: .LBB100_2: ; %atomicrmw.phi10675; GFX950-GISEL-NEXT: s_endpgm10676; GFX950-GISEL-NEXT: .LBB100_3: ; %atomicrmw.global10677; GFX950-GISEL-NEXT: flat_atomic_inc_x2 v[0:1], v[4:5]10678; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr110679; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr510680; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10681; GFX950-GISEL-NEXT: s_cbranch_execz .LBB100_210682; GFX950-GISEL-NEXT: .LBB100_4: ; %atomicrmw.private10683; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]10684; GFX950-GISEL-NEXT: s_nop 110685; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc10686; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off10687; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)10688; GFX950-GISEL-NEXT: v_add_co_u32_e32 v3, vcc, 1, v010689; GFX950-GISEL-NEXT: s_nop 110690; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v1, vcc10691; GFX950-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, v[0:1], v[4:5]10692; GFX950-GISEL-NEXT: s_nop 110693; GFX950-GISEL-NEXT: v_cndmask_b32_e64 v0, v3, 0, vcc10694; GFX950-GISEL-NEXT: v_cndmask_b32_e64 v1, v6, 0, vcc10695; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off10696; GFX950-GISEL-NEXT: s_endpgm10697 %zext.offset = zext i32 %voffset to i6410698 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset10699 %unused = atomicrmw uinc_wrap ptr %gep0, i64 %data syncscope("agent") monotonic10700 ret void10701}10702 10703define amdgpu_ps void @flat_inc_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {10704; GFX1250-SDAG-LABEL: flat_inc_saddr_i64_nortn_neg128:10705; GFX1250-SDAG: ; %bb.0:10706; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 110707; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v110708; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 010709; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff8010710; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10711; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]10712; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]10713; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo10714; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10715; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v110716; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v410717; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s010718; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB101_310719; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow10720; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s010721; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB101_410722; GFX1250-SDAG-NEXT: .LBB101_2: ; %atomicrmw.phi10723; GFX1250-SDAG-NEXT: s_endpgm10724; GFX1250-SDAG-NEXT: .LBB101_3: ; %atomicrmw.global10725; GFX1250-SDAG-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3] scope:SCOPE_DEV10726; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr110727; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr310728; GFX1250-SDAG-NEXT: s_wait_xcnt 0x010729; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s010730; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB101_210731; GFX1250-SDAG-NEXT: .LBB101_4: ; %atomicrmw.private10732; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]10733; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v010734; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)10735; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo10736; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v6, off10737; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x010738; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], 1, v[0:1]10739; GFX1250-SDAG-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[2:3]10740; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, 0, v5 :: v_dual_cndmask_b32 v0, 0, v410741; GFX1250-SDAG-NEXT: scratch_store_b64 v6, v[0:1], off10742; GFX1250-SDAG-NEXT: s_endpgm10743;10744; GFX1250-GISEL-LABEL: flat_inc_saddr_i64_nortn_neg128:10745; GFX1250-GISEL: ; %bb.0:10746; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 110747; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v210748; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]10749; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo10750; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10751; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v010752; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo10753; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)10754; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v110755; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo10756; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10757; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v310758; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v110759; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s010760; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB101_310761; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow10762; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s010763; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB101_410764; GFX1250-GISEL-NEXT: .LBB101_2: ; %atomicrmw.phi10765; GFX1250-GISEL-NEXT: s_endpgm10766; GFX1250-GISEL-NEXT: .LBB101_3: ; %atomicrmw.global10767; GFX1250-GISEL-NEXT: flat_atomic_inc_u64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV10768; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr210769; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr510770; GFX1250-GISEL-NEXT: s_wait_xcnt 0x010771; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s010772; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB101_210773; GFX1250-GISEL-NEXT: .LBB101_4: ; %atomicrmw.private10774; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]10775; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v210776; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)10777; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo10778; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off10779; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x010780; GFX1250-GISEL-NEXT: v_add_nc_u64_e32 v[2:3], 1, v[0:1]10781; GFX1250-GISEL-NEXT: v_cmp_ge_u64_e32 vcc_lo, v[0:1], v[4:5]10782; GFX1250-GISEL-NEXT: v_cndmask_b32_e64 v0, v2, 0, vcc_lo10783; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3)10784; GFX1250-GISEL-NEXT: v_cndmask_b32_e64 v1, v3, 0, vcc_lo10785; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[0:1], off10786; GFX1250-GISEL-NEXT: s_endpgm10787;10788; GFX950-SDAG-LABEL: flat_inc_saddr_i64_nortn_neg128:10789; GFX950-SDAG: ; %bb.0:10790; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v210791; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v110792; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 010793; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]10794; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff8010795; GFX950-SDAG-NEXT: s_mov_b32 s3, -110796; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base10797; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]10798; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v110799; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc10800; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]10801; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB101_310802; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow10803; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10804; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB101_410805; GFX950-SDAG-NEXT: .LBB101_2: ; %atomicrmw.phi10806; GFX950-SDAG-NEXT: s_endpgm10807; GFX950-SDAG-NEXT: .LBB101_3: ; %atomicrmw.global10808; GFX950-SDAG-NEXT: flat_atomic_inc_x2 v[0:1], v[2:3]10809; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr110810; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr310811; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10812; GFX950-SDAG-NEXT: s_cbranch_execz .LBB101_210813; GFX950-SDAG-NEXT: .LBB101_4: ; %atomicrmw.private10814; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]10815; GFX950-SDAG-NEXT: s_nop 110816; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc10817; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v6, off10818; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)10819; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, 110820; GFX950-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]10821; GFX950-SDAG-NEXT: s_nop 110822; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v5, vcc10823; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v4, vcc10824; GFX950-SDAG-NEXT: scratch_store_dwordx2 v6, v[0:1], off10825; GFX950-SDAG-NEXT: s_endpgm10826;10827; GFX950-GISEL-LABEL: flat_inc_saddr_i64_nortn_neg128:10828; GFX950-GISEL: ; %bb.0:10829; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v210830; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]10831; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v010832; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v110833; GFX950-GISEL-NEXT: s_nop 010834; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc10835; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v010836; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base10837; GFX950-GISEL-NEXT: s_nop 010838; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc10839; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v110840; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc10841; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]10842; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB101_310843; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow10844; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10845; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB101_410846; GFX950-GISEL-NEXT: .LBB101_2: ; %atomicrmw.phi10847; GFX950-GISEL-NEXT: s_endpgm10848; GFX950-GISEL-NEXT: .LBB101_3: ; %atomicrmw.global10849; GFX950-GISEL-NEXT: flat_atomic_inc_x2 v[0:1], v[4:5]10850; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr110851; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr510852; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]10853; GFX950-GISEL-NEXT: s_cbranch_execz .LBB101_210854; GFX950-GISEL-NEXT: .LBB101_4: ; %atomicrmw.private10855; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]10856; GFX950-GISEL-NEXT: s_nop 110857; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc10858; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off10859; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)10860; GFX950-GISEL-NEXT: v_add_co_u32_e32 v3, vcc, 1, v010861; GFX950-GISEL-NEXT: s_nop 110862; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v1, vcc10863; GFX950-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, v[0:1], v[4:5]10864; GFX950-GISEL-NEXT: s_nop 110865; GFX950-GISEL-NEXT: v_cndmask_b32_e64 v0, v3, 0, vcc10866; GFX950-GISEL-NEXT: v_cndmask_b32_e64 v1, v6, 0, vcc10867; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off10868; GFX950-GISEL-NEXT: s_endpgm10869 %zext.offset = zext i32 %voffset to i6410870 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset10871 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -12810872 %unused = atomicrmw uinc_wrap ptr %gep1, i64 %data syncscope("agent") monotonic10873 ret void10874}10875 10876; --------------------------------------------------------------------------------10877; amdgcn atomic dec10878; --------------------------------------------------------------------------------10879 10880 10881define amdgpu_ps float @flat_dec_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {10882; GFX1250-LABEL: flat_dec_saddr_i32_rtn:10883; GFX1250: ; %bb.0:10884; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 110885; GFX1250-NEXT: flat_atomic_dec_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV10886; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x010887; GFX1250-NEXT: ; return to shader part epilog10888;10889; GFX950-SDAG-LABEL: flat_dec_saddr_i32_rtn:10890; GFX950-SDAG: ; %bb.0:10891; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v110892; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 010893; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]10894; GFX950-SDAG-NEXT: flat_atomic_dec v0, v[0:1], v2 sc010895; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10896; GFX950-SDAG-NEXT: ; return to shader part epilog10897;10898; GFX950-GISEL-LABEL: flat_dec_saddr_i32_rtn:10899; GFX950-GISEL: ; %bb.0:10900; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]10901; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v010902; GFX950-GISEL-NEXT: s_nop 110903; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc10904; GFX950-GISEL-NEXT: flat_atomic_dec v0, v[2:3], v1 sc010905; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10906; GFX950-GISEL-NEXT: ; return to shader part epilog10907 %zext.offset = zext i32 %voffset to i6410908 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset10909 %rtn = atomicrmw udec_wrap ptr %gep0, i32 %data syncscope("agent") monotonic10910 %cast.rtn = bitcast i32 %rtn to float10911 ret float %cast.rtn10912}10913 10914define amdgpu_ps float @flat_dec_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {10915; GFX1250-LABEL: flat_dec_saddr_i32_rtn_neg128:10916; GFX1250: ; %bb.0:10917; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 110918; GFX1250-NEXT: flat_atomic_dec_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV10919; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x010920; GFX1250-NEXT: ; return to shader part epilog10921;10922; GFX950-SDAG-LABEL: flat_dec_saddr_i32_rtn_neg128:10923; GFX950-SDAG: ; %bb.0:10924; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v110925; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 010926; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]10927; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v010928; GFX950-SDAG-NEXT: s_nop 110929; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc10930; GFX950-SDAG-NEXT: flat_atomic_dec v0, v[0:1], v2 sc010931; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10932; GFX950-SDAG-NEXT: ; return to shader part epilog10933;10934; GFX950-GISEL-LABEL: flat_dec_saddr_i32_rtn_neg128:10935; GFX950-GISEL: ; %bb.0:10936; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]10937; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v010938; GFX950-GISEL-NEXT: s_nop 110939; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc10940; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v010941; GFX950-GISEL-NEXT: s_nop 110942; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc10943; GFX950-GISEL-NEXT: flat_atomic_dec v0, v[2:3], v1 sc010944; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)10945; GFX950-GISEL-NEXT: ; return to shader part epilog10946 %zext.offset = zext i32 %voffset to i6410947 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset10948 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -12810949 %rtn = atomicrmw udec_wrap ptr %gep1, i32 %data syncscope("agent") monotonic10950 %cast.rtn = bitcast i32 %rtn to float10951 ret float %cast.rtn10952}10953 10954define amdgpu_ps void @flat_dec_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {10955; GFX1250-LABEL: flat_dec_saddr_i32_nortn:10956; GFX1250: ; %bb.0:10957; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 110958; GFX1250-NEXT: flat_atomic_dec_u32 v0, v1, s[2:3] scope:SCOPE_DEV10959; GFX1250-NEXT: s_endpgm10960;10961; GFX950-SDAG-LABEL: flat_dec_saddr_i32_nortn:10962; GFX950-SDAG: ; %bb.0:10963; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v110964; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 010965; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]10966; GFX950-SDAG-NEXT: flat_atomic_dec v[0:1], v210967; GFX950-SDAG-NEXT: s_endpgm10968;10969; GFX950-GISEL-LABEL: flat_dec_saddr_i32_nortn:10970; GFX950-GISEL: ; %bb.0:10971; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]10972; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v010973; GFX950-GISEL-NEXT: s_nop 110974; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc10975; GFX950-GISEL-NEXT: flat_atomic_dec v[2:3], v110976; GFX950-GISEL-NEXT: s_endpgm10977 %zext.offset = zext i32 %voffset to i6410978 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset10979 %unused = atomicrmw udec_wrap ptr %gep0, i32 %data syncscope("agent") monotonic10980 ret void10981}10982 10983define amdgpu_ps void @flat_dec_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {10984; GFX1250-LABEL: flat_dec_saddr_i32_nortn_neg128:10985; GFX1250: ; %bb.0:10986; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 110987; GFX1250-NEXT: flat_atomic_dec_u32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV10988; GFX1250-NEXT: s_endpgm10989;10990; GFX950-SDAG-LABEL: flat_dec_saddr_i32_nortn_neg128:10991; GFX950-SDAG: ; %bb.0:10992; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v110993; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 010994; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]10995; GFX950-SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v010996; GFX950-SDAG-NEXT: s_nop 110997; GFX950-SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc10998; GFX950-SDAG-NEXT: flat_atomic_dec v[0:1], v210999; GFX950-SDAG-NEXT: s_endpgm11000;11001; GFX950-GISEL-LABEL: flat_dec_saddr_i32_nortn_neg128:11002; GFX950-GISEL: ; %bb.0:11003; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]11004; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v011005; GFX950-GISEL-NEXT: s_nop 111006; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc11007; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v011008; GFX950-GISEL-NEXT: s_nop 111009; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v3, vcc11010; GFX950-GISEL-NEXT: flat_atomic_dec v[2:3], v111011; GFX950-GISEL-NEXT: s_endpgm11012 %zext.offset = zext i32 %voffset to i6411013 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset11014 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -12811015 %unused = atomicrmw udec_wrap ptr %gep1, i32 %data syncscope("agent") monotonic11016 ret void11017}11018 11019define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {11020; GFX1250-SDAG-LABEL: flat_dec_saddr_i64_rtn:11021; GFX1250-SDAG: ; %bb.0:11022; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 111023; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v111024; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 011025; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11026; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[2:3], v[0:1]11027; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v511028; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)11029; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v011030; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr111031; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo11032; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s011033; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB106_311034; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow11035; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s1, s011036; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB106_411037; GFX1250-SDAG-NEXT: .LBB106_2: ; %atomicrmw.phi11038; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s111039; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x011040; GFX1250-SDAG-NEXT: s_branch .LBB106_511041; GFX1250-SDAG-NEXT: .LBB106_3: ; %atomicrmw.global11042; GFX1250-SDAG-NEXT: flat_atomic_dec_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV11043; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr511044; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr311045; GFX1250-SDAG-NEXT: s_wait_xcnt 0x011046; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s1, s011047; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB106_211048; GFX1250-SDAG-NEXT: .LBB106_4: ; %atomicrmw.private11049; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]11050; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x011051; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v411052; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)11053; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo11054; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v6, off11055; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x011056; GFX1250-SDAG-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]11057; GFX1250-SDAG-NEXT: v_cmp_gt_u64_e64 s0, v[0:1], v[2:3]11058; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], -1, v[0:1]11059; GFX1250-SDAG-NEXT: s_or_b32 vcc_lo, vcc_lo, s011060; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)11061; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v5, v3 :: v_dual_cndmask_b32 v2, v4, v211062; GFX1250-SDAG-NEXT: scratch_store_b64 v6, v[2:3], off11063; GFX1250-SDAG-NEXT: s_wait_xcnt 0x011064; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s111065; GFX1250-SDAG-NEXT: s_branch .LBB106_511066; GFX1250-SDAG-NEXT: .LBB106_5:11067;11068; GFX1250-GISEL-LABEL: flat_dec_saddr_i64_rtn:11069; GFX1250-GISEL: ; %bb.0:11070; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 111071; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v111072; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]11073; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11074; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v311075; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo11076; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11077; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x1411078; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v011079; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr111080; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo11081; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)11082; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s011083; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB106_311084; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow11085; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s1, s011086; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB106_411087; GFX1250-GISEL-NEXT: .LBB106_2: ; %atomicrmw.phi11088; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s111089; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x011090; GFX1250-GISEL-NEXT: s_branch .LBB106_511091; GFX1250-GISEL-NEXT: .LBB106_3: ; %atomicrmw.global11092; GFX1250-GISEL-NEXT: flat_atomic_dec_u64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV11093; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr611094; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr511095; GFX1250-GISEL-NEXT: s_wait_xcnt 0x011096; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s1, s011097; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB106_211098; GFX1250-GISEL-NEXT: .LBB106_4: ; %atomicrmw.private11099; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]11100; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x011101; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v611102; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)11103; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo11104; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off11105; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x011106; GFX1250-GISEL-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]11107; GFX1250-GISEL-NEXT: v_cmp_gt_u64_e64 s0, v[0:1], v[4:5]11108; GFX1250-GISEL-NEXT: v_add_nc_u64_e32 v[2:3], -1, v[0:1]11109; GFX1250-GISEL-NEXT: s_or_b32 vcc_lo, vcc_lo, s011110; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)11111; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v2, v2, v4 :: v_dual_cndmask_b32 v3, v3, v511112; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off11113; GFX1250-GISEL-NEXT: s_wait_xcnt 0x011114; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s111115; GFX1250-GISEL-NEXT: s_branch .LBB106_511116; GFX1250-GISEL-NEXT: .LBB106_5:11117;11118; GFX950-SDAG-LABEL: flat_dec_saddr_i64_rtn:11119; GFX950-SDAG: ; %bb.0:11120; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v211121; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v111122; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 011123; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base11124; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], s[2:3], 0, v[0:1]11125; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v511126; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr111127; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc11128; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]11129; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB106_311130; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow11131; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[2:3], s[0:1]11132; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB106_411133; GFX950-SDAG-NEXT: .LBB106_2: ; %atomicrmw.phi11134; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]11135; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11136; GFX950-SDAG-NEXT: s_branch .LBB106_511137; GFX950-SDAG-NEXT: .LBB106_3: ; %atomicrmw.global11138; GFX950-SDAG-NEXT: flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] sc011139; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr511140; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr311141; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[2:3], s[0:1]11142; GFX950-SDAG-NEXT: s_cbranch_execz .LBB106_211143; GFX950-SDAG-NEXT: .LBB106_4: ; %atomicrmw.private11144; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]11145; GFX950-SDAG-NEXT: s_nop 111146; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc11147; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)11148; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v6, off11149; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)11150; GFX950-SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]11151; GFX950-SDAG-NEXT: v_cmp_gt_u64_e64 s[0:1], v[0:1], v[2:3]11152; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, -111153; GFX950-SDAG-NEXT: s_or_b64 vcc, vcc, s[0:1]11154; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc11155; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc11156; GFX950-SDAG-NEXT: scratch_store_dwordx2 v6, v[2:3], off11157; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]11158; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)11159; GFX950-SDAG-NEXT: s_branch .LBB106_511160; GFX950-SDAG-NEXT: .LBB106_5:11161;11162; GFX950-GISEL-LABEL: flat_dec_saddr_i64_rtn:11163; GFX950-GISEL: ; %bb.0:11164; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v211165; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]11166; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, v2, v011167; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base11168; GFX950-GISEL-NEXT: s_nop 011169; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc11170; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v111171; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v311172; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr111173; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc11174; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]11175; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB106_311176; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow11177; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[2:3], s[0:1]11178; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB106_411179; GFX950-GISEL-NEXT: .LBB106_2: ; %atomicrmw.phi11180; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]11181; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11182; GFX950-GISEL-NEXT: s_branch .LBB106_511183; GFX950-GISEL-NEXT: .LBB106_3: ; %atomicrmw.global11184; GFX950-GISEL-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[4:5] sc011185; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr311186; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr511187; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[2:3], s[0:1]11188; GFX950-GISEL-NEXT: s_cbranch_execz .LBB106_211189; GFX950-GISEL-NEXT: .LBB106_4: ; %atomicrmw.private11190; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]11191; GFX950-GISEL-NEXT: s_nop 111192; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc11193; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)11194; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off11195; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)11196; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, -1, v011197; GFX950-GISEL-NEXT: s_nop 111198; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v1, vcc11199; GFX950-GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]11200; GFX950-GISEL-NEXT: v_cmp_gt_u64_e64 s[0:1], v[0:1], v[4:5]11201; GFX950-GISEL-NEXT: s_or_b64 vcc, vcc, s[0:1]11202; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc11203; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc11204; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off11205; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]11206; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)11207; GFX950-GISEL-NEXT: s_branch .LBB106_511208; GFX950-GISEL-NEXT: .LBB106_5:11209 %zext.offset = zext i32 %voffset to i6411210 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset11211 %rtn = atomicrmw udec_wrap ptr %gep0, i64 %data syncscope("agent") monotonic11212 %cast.rtn = bitcast i64 %rtn to <2 x float>11213 ret <2 x float> %cast.rtn11214}11215 11216define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {11217; GFX1250-SDAG-LABEL: flat_dec_saddr_i64_rtn_neg128:11218; GFX1250-SDAG: ; %bb.0:11219; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 111220; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v111221; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 011222; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff8011223; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11224; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]11225; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]11226; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11227; GFX1250-SDAG-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v511228; GFX1250-SDAG-NEXT: v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v011229; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr111230; GFX1250-SDAG-NEXT: s_and_saveexec_b32 s0, vcc_lo11231; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)11232; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s011233; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB107_311234; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow11235; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s1, s011236; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB107_411237; GFX1250-SDAG-NEXT: .LBB107_2: ; %atomicrmw.phi11238; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s111239; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x011240; GFX1250-SDAG-NEXT: s_branch .LBB107_511241; GFX1250-SDAG-NEXT: .LBB107_3: ; %atomicrmw.global11242; GFX1250-SDAG-NEXT: flat_atomic_dec_u64 v[0:1], v[4:5], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV11243; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr511244; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr311245; GFX1250-SDAG-NEXT: s_wait_xcnt 0x011246; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s1, s011247; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB107_211248; GFX1250-SDAG-NEXT: .LBB107_4: ; %atomicrmw.private11249; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]11250; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x011251; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v411252; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)11253; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo11254; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v6, off11255; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x011256; GFX1250-SDAG-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]11257; GFX1250-SDAG-NEXT: v_cmp_gt_u64_e64 s0, v[0:1], v[2:3]11258; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], -1, v[0:1]11259; GFX1250-SDAG-NEXT: s_or_b32 vcc_lo, vcc_lo, s011260; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)11261; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v5, v3 :: v_dual_cndmask_b32 v2, v4, v211262; GFX1250-SDAG-NEXT: scratch_store_b64 v6, v[2:3], off11263; GFX1250-SDAG-NEXT: s_wait_xcnt 0x011264; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s111265; GFX1250-SDAG-NEXT: s_branch .LBB107_511266; GFX1250-SDAG-NEXT: .LBB107_5:11267;11268; GFX1250-GISEL-LABEL: flat_dec_saddr_i64_rtn_neg128:11269; GFX1250-GISEL: ; %bb.0:11270; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 111271; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v111272; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]11273; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11274; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v311275; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo11276; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)11277; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v011278; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo11279; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11280; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x1411281; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v011282; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr111283; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo11284; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)11285; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s011286; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB107_311287; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow11288; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s1, s011289; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB107_411290; GFX1250-GISEL-NEXT: .LBB107_2: ; %atomicrmw.phi11291; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s111292; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x011293; GFX1250-GISEL-NEXT: s_branch .LBB107_511294; GFX1250-GISEL-NEXT: .LBB107_3: ; %atomicrmw.global11295; GFX1250-GISEL-NEXT: flat_atomic_dec_u64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV11296; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr611297; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr511298; GFX1250-GISEL-NEXT: s_wait_xcnt 0x011299; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s1, s011300; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB107_211301; GFX1250-GISEL-NEXT: .LBB107_4: ; %atomicrmw.private11302; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]11303; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x011304; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v611305; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)11306; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo11307; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off11308; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x011309; GFX1250-GISEL-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]11310; GFX1250-GISEL-NEXT: v_cmp_gt_u64_e64 s0, v[0:1], v[4:5]11311; GFX1250-GISEL-NEXT: v_add_nc_u64_e32 v[2:3], -1, v[0:1]11312; GFX1250-GISEL-NEXT: s_or_b32 vcc_lo, vcc_lo, s011313; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)11314; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v2, v2, v4 :: v_dual_cndmask_b32 v3, v3, v511315; GFX1250-GISEL-NEXT: scratch_store_b64 v6, v[2:3], off11316; GFX1250-GISEL-NEXT: s_wait_xcnt 0x011317; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s111318; GFX1250-GISEL-NEXT: s_branch .LBB107_511319; GFX1250-GISEL-NEXT: .LBB107_5:11320;11321; GFX950-SDAG-LABEL: flat_dec_saddr_i64_rtn_neg128:11322; GFX950-SDAG: ; %bb.0:11323; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v211324; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v111325; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 011326; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]11327; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff8011328; GFX950-SDAG-NEXT: s_mov_b32 s3, -111329; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base11330; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[2:3]11331; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v511332; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr111333; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc11334; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]11335; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB107_311336; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow11337; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[2:3], s[0:1]11338; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB107_411339; GFX950-SDAG-NEXT: .LBB107_2: ; %atomicrmw.phi11340; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]11341; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11342; GFX950-SDAG-NEXT: s_branch .LBB107_511343; GFX950-SDAG-NEXT: .LBB107_3: ; %atomicrmw.global11344; GFX950-SDAG-NEXT: flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] sc011345; GFX950-SDAG-NEXT: ; implicit-def: $vgpr4_vgpr511346; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr311347; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[2:3], s[0:1]11348; GFX950-SDAG-NEXT: s_cbranch_execz .LBB107_211349; GFX950-SDAG-NEXT: .LBB107_4: ; %atomicrmw.private11350; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]11351; GFX950-SDAG-NEXT: s_nop 111352; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc11353; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)11354; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v6, off11355; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)11356; GFX950-SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]11357; GFX950-SDAG-NEXT: v_cmp_gt_u64_e64 s[0:1], v[0:1], v[2:3]11358; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, -111359; GFX950-SDAG-NEXT: s_or_b64 vcc, vcc, s[0:1]11360; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc11361; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc11362; GFX950-SDAG-NEXT: scratch_store_dwordx2 v6, v[2:3], off11363; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]11364; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)11365; GFX950-SDAG-NEXT: s_branch .LBB107_511366; GFX950-SDAG-NEXT: .LBB107_5:11367;11368; GFX950-GISEL-LABEL: flat_dec_saddr_i64_rtn_neg128:11369; GFX950-GISEL: ; %bb.0:11370; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v211371; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]11372; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v011373; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v111374; GFX950-GISEL-NEXT: s_nop 011375; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc11376; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0xffffff80, v011377; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base11378; GFX950-GISEL-NEXT: s_nop 011379; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v1, vcc11380; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v311381; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr111382; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc11383; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]11384; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB107_311385; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow11386; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[2:3], s[0:1]11387; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB107_411388; GFX950-GISEL-NEXT: .LBB107_2: ; %atomicrmw.phi11389; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]11390; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)11391; GFX950-GISEL-NEXT: s_branch .LBB107_511392; GFX950-GISEL-NEXT: .LBB107_3: ; %atomicrmw.global11393; GFX950-GISEL-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3], v[4:5] sc011394; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr311395; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr511396; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[2:3], s[0:1]11397; GFX950-GISEL-NEXT: s_cbranch_execz .LBB107_211398; GFX950-GISEL-NEXT: .LBB107_4: ; %atomicrmw.private11399; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]11400; GFX950-GISEL-NEXT: s_nop 111401; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc11402; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)11403; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v6, off11404; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)11405; GFX950-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, -1, v011406; GFX950-GISEL-NEXT: s_nop 111407; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v1, vcc11408; GFX950-GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]11409; GFX950-GISEL-NEXT: v_cmp_gt_u64_e64 s[0:1], v[0:1], v[4:5]11410; GFX950-GISEL-NEXT: s_or_b64 vcc, vcc, s[0:1]11411; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc11412; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc11413; GFX950-GISEL-NEXT: scratch_store_dwordx2 v6, v[2:3], off11414; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]11415; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)11416; GFX950-GISEL-NEXT: s_branch .LBB107_511417; GFX950-GISEL-NEXT: .LBB107_5:11418 %zext.offset = zext i32 %voffset to i6411419 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset11420 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -12811421 %rtn = atomicrmw udec_wrap ptr %gep1, i64 %data syncscope("agent") monotonic11422 %cast.rtn = bitcast i64 %rtn to <2 x float>11423 ret <2 x float> %cast.rtn11424}11425 11426define amdgpu_ps void @flat_dec_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {11427; GFX1250-SDAG-LABEL: flat_dec_saddr_i64_nortn:11428; GFX1250-SDAG: ; %bb.0:11429; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 111430; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v111431; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 011432; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo11433; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11434; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]11435; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v111436; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)11437; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v411438; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s011439; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB108_311440; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow11441; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s011442; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB108_411443; GFX1250-SDAG-NEXT: .LBB108_2: ; %atomicrmw.phi11444; GFX1250-SDAG-NEXT: s_endpgm11445; GFX1250-SDAG-NEXT: .LBB108_3: ; %atomicrmw.global11446; GFX1250-SDAG-NEXT: flat_atomic_dec_u64 v[0:1], v[2:3] scope:SCOPE_DEV11447; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr111448; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr311449; GFX1250-SDAG-NEXT: s_wait_xcnt 0x011450; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s011451; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB108_211452; GFX1250-SDAG-NEXT: .LBB108_4: ; %atomicrmw.private11453; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]11454; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v011455; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)11456; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo11457; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off11458; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x011459; GFX1250-SDAG-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]11460; GFX1250-SDAG-NEXT: v_cmp_gt_u64_e64 s0, v[0:1], v[2:3]11461; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], -1, v[0:1]11462; GFX1250-SDAG-NEXT: s_or_b32 vcc_lo, vcc_lo, s011463; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)11464; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v1, v3 :: v_dual_cndmask_b32 v0, v0, v211465; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off11466; GFX1250-SDAG-NEXT: s_endpgm11467;11468; GFX1250-GISEL-LABEL: flat_dec_saddr_i64_nortn:11469; GFX1250-GISEL: ; %bb.0:11470; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 111471; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v211472; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]11473; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo11474; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11475; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v011476; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo11477; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11478; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v311479; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v111480; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s011481; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB108_311482; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow11483; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s011484; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB108_411485; GFX1250-GISEL-NEXT: .LBB108_2: ; %atomicrmw.phi11486; GFX1250-GISEL-NEXT: s_endpgm11487; GFX1250-GISEL-NEXT: .LBB108_3: ; %atomicrmw.global11488; GFX1250-GISEL-NEXT: flat_atomic_dec_u64 v0, v[4:5], s[2:3] scope:SCOPE_DEV11489; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr211490; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr511491; GFX1250-GISEL-NEXT: s_wait_xcnt 0x011492; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s011493; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB108_211494; GFX1250-GISEL-NEXT: .LBB108_4: ; %atomicrmw.private11495; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]11496; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v211497; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)11498; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo11499; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off11500; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x011501; GFX1250-GISEL-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]11502; GFX1250-GISEL-NEXT: v_cmp_gt_u64_e64 s0, v[0:1], v[4:5]11503; GFX1250-GISEL-NEXT: v_add_nc_u64_e32 v[0:1], -1, v[0:1]11504; GFX1250-GISEL-NEXT: s_or_b32 vcc_lo, vcc_lo, s011505; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)11506; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v0, v0, v4 :: v_dual_cndmask_b32 v1, v1, v511507; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off11508; GFX1250-GISEL-NEXT: s_endpgm11509;11510; GFX950-SDAG-LABEL: flat_dec_saddr_i64_nortn:11511; GFX950-SDAG: ; %bb.0:11512; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v211513; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v111514; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 011515; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base11516; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]11517; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v111518; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc11519; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]11520; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB108_311521; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow11522; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]11523; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB108_411524; GFX950-SDAG-NEXT: .LBB108_2: ; %atomicrmw.phi11525; GFX950-SDAG-NEXT: s_endpgm11526; GFX950-SDAG-NEXT: .LBB108_3: ; %atomicrmw.global11527; GFX950-SDAG-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]11528; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr111529; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr311530; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]11531; GFX950-SDAG-NEXT: s_cbranch_execz .LBB108_211532; GFX950-SDAG-NEXT: .LBB108_4: ; %atomicrmw.private11533; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]11534; GFX950-SDAG-NEXT: s_nop 111535; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc11536; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v6, off11537; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)11538; GFX950-SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]11539; GFX950-SDAG-NEXT: v_cmp_gt_u64_e64 s[0:1], v[0:1], v[2:3]11540; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, -111541; GFX950-SDAG-NEXT: s_or_b64 vcc, vcc, s[0:1]11542; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc11543; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc11544; GFX950-SDAG-NEXT: scratch_store_dwordx2 v6, v[0:1], off11545; GFX950-SDAG-NEXT: s_endpgm11546;11547; GFX950-GISEL-LABEL: flat_dec_saddr_i64_nortn:11548; GFX950-GISEL: ; %bb.0:11549; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v211550; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]11551; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v011552; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v111553; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base11554; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc11555; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v111556; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc11557; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]11558; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB108_311559; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow11560; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]11561; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB108_411562; GFX950-GISEL-NEXT: .LBB108_2: ; %atomicrmw.phi11563; GFX950-GISEL-NEXT: s_endpgm11564; GFX950-GISEL-NEXT: .LBB108_3: ; %atomicrmw.global11565; GFX950-GISEL-NEXT: flat_atomic_dec_x2 v[0:1], v[4:5]11566; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr111567; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr511568; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]11569; GFX950-GISEL-NEXT: s_cbranch_execz .LBB108_211570; GFX950-GISEL-NEXT: .LBB108_4: ; %atomicrmw.private11571; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]11572; GFX950-GISEL-NEXT: s_nop 111573; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc11574; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off11575; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)11576; GFX950-GISEL-NEXT: v_add_co_u32_e32 v3, vcc, -1, v011577; GFX950-GISEL-NEXT: s_nop 111578; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v6, vcc, -1, v1, vcc11579; GFX950-GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]11580; GFX950-GISEL-NEXT: v_cmp_gt_u64_e64 s[0:1], v[0:1], v[4:5]11581; GFX950-GISEL-NEXT: s_or_b64 vcc, vcc, s[0:1]11582; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc11583; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v1, v6, v5, vcc11584; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off11585; GFX950-GISEL-NEXT: s_endpgm11586 %zext.offset = zext i32 %voffset to i6411587 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset11588 %unused = atomicrmw udec_wrap ptr %gep0, i64 %data syncscope("agent") monotonic11589 ret void11590}11591 11592define amdgpu_ps void @flat_dec_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {11593; GFX1250-SDAG-LABEL: flat_dec_saddr_i64_nortn_neg128:11594; GFX1250-SDAG: ; %bb.0:11595; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 111596; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v111597; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 011598; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff8011599; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11600; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]11601; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]11602; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo11603; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11604; GFX1250-SDAG-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v111605; GFX1250-SDAG-NEXT: v_cmpx_lt_u32_e32 0x3ffffff, v411606; GFX1250-SDAG-NEXT: s_xor_b32 s0, exec_lo, s011607; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB109_311608; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow11609; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s011610; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB109_411611; GFX1250-SDAG-NEXT: .LBB109_2: ; %atomicrmw.phi11612; GFX1250-SDAG-NEXT: s_endpgm11613; GFX1250-SDAG-NEXT: .LBB109_3: ; %atomicrmw.global11614; GFX1250-SDAG-NEXT: flat_atomic_dec_u64 v[0:1], v[2:3] scope:SCOPE_DEV11615; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr111616; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr311617; GFX1250-SDAG-NEXT: s_wait_xcnt 0x011618; GFX1250-SDAG-NEXT: s_and_not1_saveexec_b32 s0, s011619; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB109_211620; GFX1250-SDAG-NEXT: .LBB109_4: ; %atomicrmw.private11621; GFX1250-SDAG-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]11622; GFX1250-SDAG-NEXT: v_subrev_nc_u32_e32 v4, src_flat_scratch_base_lo, v011623; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)11624; GFX1250-SDAG-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo11625; GFX1250-SDAG-NEXT: scratch_load_b64 v[0:1], v4, off11626; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x011627; GFX1250-SDAG-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]11628; GFX1250-SDAG-NEXT: v_cmp_gt_u64_e64 s0, v[0:1], v[2:3]11629; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], -1, v[0:1]11630; GFX1250-SDAG-NEXT: s_or_b32 vcc_lo, vcc_lo, s011631; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)11632; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v1, v3 :: v_dual_cndmask_b32 v0, v0, v211633; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off11634; GFX1250-SDAG-NEXT: s_endpgm11635;11636; GFX1250-GISEL-LABEL: flat_dec_saddr_i64_nortn_neg128:11637; GFX1250-GISEL: ; %bb.0:11638; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 111639; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v211640; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]11641; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo11642; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11643; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v011644; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo11645; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)11646; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v111647; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo11648; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11649; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v311650; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v111651; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s011652; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB109_311653; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow11654; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s011655; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB109_411656; GFX1250-GISEL-NEXT: .LBB109_2: ; %atomicrmw.phi11657; GFX1250-GISEL-NEXT: s_endpgm11658; GFX1250-GISEL-NEXT: .LBB109_3: ; %atomicrmw.global11659; GFX1250-GISEL-NEXT: flat_atomic_dec_u64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV11660; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr211661; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr511662; GFX1250-GISEL-NEXT: s_wait_xcnt 0x011663; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s011664; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB109_211665; GFX1250-GISEL-NEXT: .LBB109_4: ; %atomicrmw.private11666; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]11667; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v211668; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)11669; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo11670; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off11671; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x011672; GFX1250-GISEL-NEXT: v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]11673; GFX1250-GISEL-NEXT: v_cmp_gt_u64_e64 s0, v[0:1], v[4:5]11674; GFX1250-GISEL-NEXT: v_add_nc_u64_e32 v[0:1], -1, v[0:1]11675; GFX1250-GISEL-NEXT: s_or_b32 vcc_lo, vcc_lo, s011676; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)11677; GFX1250-GISEL-NEXT: v_dual_cndmask_b32 v0, v0, v4 :: v_dual_cndmask_b32 v1, v1, v511678; GFX1250-GISEL-NEXT: scratch_store_b64 v2, v[0:1], off11679; GFX1250-GISEL-NEXT: s_endpgm11680;11681; GFX950-SDAG-LABEL: flat_dec_saddr_i64_nortn_neg128:11682; GFX950-SDAG: ; %bb.0:11683; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, v211684; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v111685; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 011686; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], s[2:3], 0, v[0:1]11687; GFX950-SDAG-NEXT: s_movk_i32 s2, 0xff8011688; GFX950-SDAG-NEXT: s_mov_b32 s3, -111689; GFX950-SDAG-NEXT: s_mov_b64 s[0:1], src_private_base11690; GFX950-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[2:3]11691; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, s1, v111692; GFX950-SDAG-NEXT: s_and_saveexec_b64 s[0:1], vcc11693; GFX950-SDAG-NEXT: s_xor_b64 s[0:1], exec, s[0:1]11694; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB109_311695; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow11696; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]11697; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB109_411698; GFX950-SDAG-NEXT: .LBB109_2: ; %atomicrmw.phi11699; GFX950-SDAG-NEXT: s_endpgm11700; GFX950-SDAG-NEXT: .LBB109_3: ; %atomicrmw.global11701; GFX950-SDAG-NEXT: flat_atomic_dec_x2 v[0:1], v[2:3]11702; GFX950-SDAG-NEXT: ; implicit-def: $vgpr0_vgpr111703; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr311704; GFX950-SDAG-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]11705; GFX950-SDAG-NEXT: s_cbranch_execz .LBB109_211706; GFX950-SDAG-NEXT: .LBB109_4: ; %atomicrmw.private11707; GFX950-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]11708; GFX950-SDAG-NEXT: s_nop 111709; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc11710; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[0:1], v6, off11711; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)11712; GFX950-SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]11713; GFX950-SDAG-NEXT: v_cmp_gt_u64_e64 s[0:1], v[0:1], v[2:3]11714; GFX950-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, -111715; GFX950-SDAG-NEXT: s_or_b64 vcc, vcc, s[0:1]11716; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc11717; GFX950-SDAG-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc11718; GFX950-SDAG-NEXT: scratch_store_dwordx2 v6, v[0:1], off11719; GFX950-SDAG-NEXT: s_endpgm11720;11721; GFX950-GISEL-LABEL: flat_dec_saddr_i64_nortn_neg128:11722; GFX950-GISEL: ; %bb.0:11723; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v211724; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]11725; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v2, v011726; GFX950-GISEL-NEXT: v_mov_b32_e32 v4, v111727; GFX950-GISEL-NEXT: s_nop 011728; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc11729; GFX950-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff80, v011730; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], src_private_base11731; GFX950-GISEL-NEXT: s_nop 011732; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc11733; GFX950-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, s1, v111734; GFX950-GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc11735; GFX950-GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]11736; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB109_311737; GFX950-GISEL-NEXT: ; %bb.1: ; %Flow11738; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]11739; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB109_411740; GFX950-GISEL-NEXT: .LBB109_2: ; %atomicrmw.phi11741; GFX950-GISEL-NEXT: s_endpgm11742; GFX950-GISEL-NEXT: .LBB109_3: ; %atomicrmw.global11743; GFX950-GISEL-NEXT: flat_atomic_dec_x2 v[0:1], v[4:5]11744; GFX950-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr111745; GFX950-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr511746; GFX950-GISEL-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]11747; GFX950-GISEL-NEXT: s_cbranch_execz .LBB109_211748; GFX950-GISEL-NEXT: .LBB109_4: ; %atomicrmw.private11749; GFX950-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]11750; GFX950-GISEL-NEXT: s_nop 111751; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc11752; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[0:1], v2, off11753; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)11754; GFX950-GISEL-NEXT: v_add_co_u32_e32 v3, vcc, -1, v011755; GFX950-GISEL-NEXT: s_nop 111756; GFX950-GISEL-NEXT: v_addc_co_u32_e32 v6, vcc, -1, v1, vcc11757; GFX950-GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]11758; GFX950-GISEL-NEXT: v_cmp_gt_u64_e64 s[0:1], v[0:1], v[4:5]11759; GFX950-GISEL-NEXT: s_or_b64 vcc, vcc, s[0:1]11760; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc11761; GFX950-GISEL-NEXT: v_cndmask_b32_e32 v1, v6, v5, vcc11762; GFX950-GISEL-NEXT: scratch_store_dwordx2 v2, v[0:1], off11763; GFX950-GISEL-NEXT: s_endpgm11764 %zext.offset = zext i32 %voffset to i6411765 %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset11766 %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -12811767 %unused = atomicrmw udec_wrap ptr %gep1, i64 %data syncscope("agent") monotonic11768 ret void11769}11770 11771define double @flat_atomic_fadd_f64_saddr_rtn(ptr inreg %ptr, double %data) {11772; GFX1250-SDAG-LABEL: flat_atomic_fadd_f64_saddr_rtn:11773; GFX1250-SDAG: ; %bb.0:11774; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x011775; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x011776; GFX1250-SDAG-NEXT: s_mov_b64 s[2:3], src_shared_base11777; GFX1250-SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 0x5011778; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)11779; GFX1250-SDAG-NEXT: s_cmp_eq_u32 s1, s311780; GFX1250-SDAG-NEXT: s_cselect_b32 s2, -1, 011781; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s211782; GFX1250-SDAG-NEXT: s_cbranch_vccz .LBB110_311783; GFX1250-SDAG-NEXT: ; %bb.1: ; %atomicrmw.check.private11784; GFX1250-SDAG-NEXT: s_xor_b32 s2, s1, src_flat_scratch_base_hi11785; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)11786; GFX1250-SDAG-NEXT: s_cmp_lt_u32 s2, 0x400000011787; GFX1250-SDAG-NEXT: s_cselect_b32 s2, -1, 011788; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s211789; GFX1250-SDAG-NEXT: s_cbranch_vccz .LBB110_411790; GFX1250-SDAG-NEXT: ; %bb.2: ; %atomicrmw.global11791; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 011792; GFX1250-SDAG-NEXT: s_wait_storecnt 0x011793; GFX1250-SDAG-NEXT: global_atomic_add_f64 v[2:3], v2, v[0:1], s[0:1] th:TH_ATOMIC_RETURN11794; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x011795; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB110_511796; GFX1250-SDAG-NEXT: s_branch .LBB110_611797; GFX1250-SDAG-NEXT: .LBB110_3:11798; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr311799; GFX1250-SDAG-NEXT: s_branch .LBB110_711800; GFX1250-SDAG-NEXT: .LBB110_4:11801; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr311802; GFX1250-SDAG-NEXT: .LBB110_5: ; %atomicrmw.private11803; GFX1250-SDAG-NEXT: s_sub_co_i32 s2, s0, src_flat_scratch_base_lo11804; GFX1250-SDAG-NEXT: s_cmp_lg_u64 s[0:1], 011805; GFX1250-SDAG-NEXT: s_cselect_b32 s2, s2, -111806; GFX1250-SDAG-NEXT: scratch_load_b64 v[2:3], off, s211807; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x011808; GFX1250-SDAG-NEXT: v_add_f64_e32 v[4:5], v[2:3], v[0:1]11809; GFX1250-SDAG-NEXT: scratch_store_b64 off, v[4:5], s211810; GFX1250-SDAG-NEXT: .LBB110_6: ; %Flow111811; GFX1250-SDAG-NEXT: s_wait_xcnt 0x011812; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB110_811813; GFX1250-SDAG-NEXT: .LBB110_7: ; %atomicrmw.shared11814; GFX1250-SDAG-NEXT: s_cmp_lg_u64 s[0:1], 011815; GFX1250-SDAG-NEXT: s_cselect_b32 s0, s0, -111816; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)11817; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s011818; GFX1250-SDAG-NEXT: s_wait_storecnt 0x011819; GFX1250-SDAG-NEXT: ds_add_rtn_f64 v[2:3], v2, v[0:1]11820; GFX1250-SDAG-NEXT: s_wait_dscnt 0x011821; GFX1250-SDAG-NEXT: .LBB110_8: ; %atomicrmw.end11822; GFX1250-SDAG-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v311823; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]11824;11825; GFX1250-GISEL-LABEL: flat_atomic_fadd_f64_saddr_rtn:11826; GFX1250-GISEL: ; %bb.0:11827; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x011828; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x011829; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s0, 0x5011830; GFX1250-GISEL-NEXT: s_mov_b64 s[2:3], src_shared_base11831; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s1, 011832; GFX1250-GISEL-NEXT: s_mov_b32 s2, 111833; GFX1250-GISEL-NEXT: s_cmp_lg_u32 s1, s311834; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr311835; GFX1250-GISEL-NEXT: s_cbranch_scc0 .LBB110_611836; GFX1250-GISEL-NEXT: ; %bb.1: ; %atomicrmw.check.private11837; GFX1250-GISEL-NEXT: s_xor_b32 s2, s1, src_flat_scratch_base_hi11838; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr311839; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)11840; GFX1250-GISEL-NEXT: s_cmp_ge_u32 s2, 0x400000011841; GFX1250-GISEL-NEXT: s_mov_b32 s2, 111842; GFX1250-GISEL-NEXT: s_cbranch_scc0 .LBB110_311843; GFX1250-GISEL-NEXT: ; %bb.2: ; %atomicrmw.global11844; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 011845; GFX1250-GISEL-NEXT: s_mov_b32 s2, 011846; GFX1250-GISEL-NEXT: s_wait_storecnt 0x011847; GFX1250-GISEL-NEXT: global_atomic_add_f64 v[2:3], v2, v[0:1], s[0:1] th:TH_ATOMIC_RETURN11848; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x011849; GFX1250-GISEL-NEXT: .LBB110_3: ; %Flow11850; GFX1250-GISEL-NEXT: s_xor_b32 s2, s2, 111851; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)11852; GFX1250-GISEL-NEXT: s_cmp_lg_u32 s2, 011853; GFX1250-GISEL-NEXT: s_cbranch_scc1 .LBB110_511854; GFX1250-GISEL-NEXT: ; %bb.4: ; %atomicrmw.private11855; GFX1250-GISEL-NEXT: s_sub_co_i32 s2, s0, src_flat_scratch_base_lo11856; GFX1250-GISEL-NEXT: s_cmp_lg_u64 s[0:1], 011857; GFX1250-GISEL-NEXT: s_cselect_b32 s2, s2, -111858; GFX1250-GISEL-NEXT: scratch_load_b64 v[2:3], off, s211859; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x011860; GFX1250-GISEL-NEXT: v_add_f64_e32 v[4:5], v[2:3], v[0:1]11861; GFX1250-GISEL-NEXT: scratch_store_b64 off, v[4:5], s211862; GFX1250-GISEL-NEXT: .LBB110_5: ; %Flow111863; GFX1250-GISEL-NEXT: s_wait_xcnt 0x011864; GFX1250-GISEL-NEXT: s_mov_b32 s2, 011865; GFX1250-GISEL-NEXT: .LBB110_6: ; %Flow211866; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)11867; GFX1250-GISEL-NEXT: s_xor_b32 s2, s2, 111868; GFX1250-GISEL-NEXT: s_cmp_lg_u32 s2, 011869; GFX1250-GISEL-NEXT: s_cbranch_scc1 .LBB110_811870; GFX1250-GISEL-NEXT: ; %bb.7: ; %atomicrmw.shared11871; GFX1250-GISEL-NEXT: s_cmp_lg_u64 s[0:1], 011872; GFX1250-GISEL-NEXT: s_cselect_b32 s0, s0, -111873; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)11874; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, s011875; GFX1250-GISEL-NEXT: s_wait_storecnt 0x011876; GFX1250-GISEL-NEXT: ds_add_rtn_f64 v[2:3], v2, v[0:1]11877; GFX1250-GISEL-NEXT: s_wait_dscnt 0x011878; GFX1250-GISEL-NEXT: .LBB110_8: ; %atomicrmw.end11879; GFX1250-GISEL-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v311880; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]11881;11882; GFX950-SDAG-LABEL: flat_atomic_fadd_f64_saddr_rtn:11883; GFX950-SDAG: ; %bb.0:11884; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11885; GFX950-SDAG-NEXT: s_add_u32 s0, s0, 0x5011886; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], src_shared_base11887; GFX950-SDAG-NEXT: s_addc_u32 s1, s1, 011888; GFX950-SDAG-NEXT: s_cmp_eq_u32 s1, s311889; GFX950-SDAG-NEXT: s_cselect_b64 s[2:3], -1, 011890; GFX950-SDAG-NEXT: s_andn2_b64 vcc, exec, s[2:3]11891; GFX950-SDAG-NEXT: s_cbranch_vccz .LBB110_311892; GFX950-SDAG-NEXT: ; %bb.1: ; %atomicrmw.check.private11893; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], src_private_base11894; GFX950-SDAG-NEXT: s_cmp_eq_u32 s1, s311895; GFX950-SDAG-NEXT: s_cselect_b64 s[2:3], -1, 011896; GFX950-SDAG-NEXT: s_andn2_b64 vcc, exec, s[2:3]11897; GFX950-SDAG-NEXT: s_cbranch_vccz .LBB110_411898; GFX950-SDAG-NEXT: ; %bb.2: ; %atomicrmw.global11899; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, 011900; GFX950-SDAG-NEXT: global_atomic_add_f64 v[2:3], v2, v[0:1], s[0:1] sc011901; GFX950-SDAG-NEXT: s_cbranch_execz .LBB110_511902; GFX950-SDAG-NEXT: s_branch .LBB110_611903; GFX950-SDAG-NEXT: .LBB110_3:11904; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr311905; GFX950-SDAG-NEXT: s_branch .LBB110_711906; GFX950-SDAG-NEXT: .LBB110_4:11907; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr311908; GFX950-SDAG-NEXT: .LBB110_5: ; %atomicrmw.private11909; GFX950-SDAG-NEXT: s_cmp_lg_u64 s[0:1], 011910; GFX950-SDAG-NEXT: s_cselect_b32 s2, s0, -111911; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[2:3], off, s211912; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)11913; GFX950-SDAG-NEXT: v_add_f64 v[4:5], v[2:3], v[0:1]11914; GFX950-SDAG-NEXT: scratch_store_dwordx2 off, v[4:5], s211915; GFX950-SDAG-NEXT: .LBB110_6: ; %Flow111916; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB110_811917; GFX950-SDAG-NEXT: .LBB110_7: ; %atomicrmw.shared11918; GFX950-SDAG-NEXT: s_cmp_lg_u64 s[0:1], 011919; GFX950-SDAG-NEXT: s_cselect_b32 s0, s0, -111920; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)11921; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s011922; GFX950-SDAG-NEXT: ds_add_rtn_f64 v[2:3], v2, v[0:1]11923; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)11924; GFX950-SDAG-NEXT: .LBB110_8: ; %atomicrmw.end11925; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)11926; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, v211927; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, v311928; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]11929;11930; GFX950-GISEL-LABEL: flat_atomic_fadd_f64_saddr_rtn:11931; GFX950-GISEL: ; %bb.0:11932; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11933; GFX950-GISEL-NEXT: s_add_u32 s0, s0, 0x5011934; GFX950-GISEL-NEXT: s_mov_b64 s[2:3], src_shared_base11935; GFX950-GISEL-NEXT: s_addc_u32 s1, s1, 011936; GFX950-GISEL-NEXT: s_cmp_lg_u32 s1, s311937; GFX950-GISEL-NEXT: s_mov_b32 s2, 111938; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr311939; GFX950-GISEL-NEXT: s_cbranch_scc0 .LBB110_611940; GFX950-GISEL-NEXT: ; %bb.1: ; %atomicrmw.check.private11941; GFX950-GISEL-NEXT: s_mov_b64 s[2:3], src_private_base11942; GFX950-GISEL-NEXT: s_cmp_lg_u32 s1, s311943; GFX950-GISEL-NEXT: s_mov_b32 s2, 111944; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr311945; GFX950-GISEL-NEXT: s_cbranch_scc0 .LBB110_311946; GFX950-GISEL-NEXT: ; %bb.2: ; %atomicrmw.global11947; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 011948; GFX950-GISEL-NEXT: global_atomic_add_f64 v[2:3], v2, v[0:1], s[0:1] sc011949; GFX950-GISEL-NEXT: s_mov_b32 s2, 011950; GFX950-GISEL-NEXT: .LBB110_3: ; %Flow11951; GFX950-GISEL-NEXT: s_xor_b32 s2, s2, 111952; GFX950-GISEL-NEXT: s_cmp_lg_u32 s2, 011953; GFX950-GISEL-NEXT: s_cbranch_scc1 .LBB110_511954; GFX950-GISEL-NEXT: ; %bb.4: ; %atomicrmw.private11955; GFX950-GISEL-NEXT: s_cmp_lg_u64 s[0:1], 011956; GFX950-GISEL-NEXT: s_cselect_b32 s2, s0, -111957; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[2:3], off, s211958; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)11959; GFX950-GISEL-NEXT: v_add_f64 v[4:5], v[2:3], v[0:1]11960; GFX950-GISEL-NEXT: scratch_store_dwordx2 off, v[4:5], s211961; GFX950-GISEL-NEXT: .LBB110_5: ; %Flow111962; GFX950-GISEL-NEXT: s_mov_b32 s2, 011963; GFX950-GISEL-NEXT: .LBB110_6: ; %Flow211964; GFX950-GISEL-NEXT: s_xor_b32 s2, s2, 111965; GFX950-GISEL-NEXT: s_cmp_lg_u32 s2, 011966; GFX950-GISEL-NEXT: s_cbranch_scc1 .LBB110_811967; GFX950-GISEL-NEXT: ; %bb.7: ; %atomicrmw.shared11968; GFX950-GISEL-NEXT: s_cmp_lg_u64 s[0:1], 011969; GFX950-GISEL-NEXT: s_cselect_b32 s0, s0, -111970; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)11971; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s011972; GFX950-GISEL-NEXT: ds_add_rtn_f64 v[2:3], v2, v[0:1]11973; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)11974; GFX950-GISEL-NEXT: .LBB110_8: ; %atomicrmw.end11975; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)11976; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, v211977; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, v311978; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]11979 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1011980 %result = atomicrmw fadd ptr %gep.0, double %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !011981 ret double %result11982}11983 11984define void @flat_atomic_fadd_f64_saddr_nortn(ptr inreg %ptr, double %data) {11985; GFX1250-SDAG-LABEL: flat_atomic_fadd_f64_saddr_nortn:11986; GFX1250-SDAG: ; %bb.0:11987; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x011988; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x011989; GFX1250-SDAG-NEXT: s_mov_b64 s[2:3], src_shared_base11990; GFX1250-SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 0x5011991; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)11992; GFX1250-SDAG-NEXT: s_cmp_eq_u32 s1, s311993; GFX1250-SDAG-NEXT: s_cselect_b32 s2, -1, 011994; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s211995; GFX1250-SDAG-NEXT: s_mov_b32 s2, -111996; GFX1250-SDAG-NEXT: s_cbranch_vccnz .LBB111_311997; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow211998; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s211999; GFX1250-SDAG-NEXT: s_cbranch_vccz .LBB111_812000; GFX1250-SDAG-NEXT: .LBB111_2: ; %atomicrmw.phi12001; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]12002; GFX1250-SDAG-NEXT: .LBB111_3: ; %atomicrmw.check.private12003; GFX1250-SDAG-NEXT: s_xor_b32 s2, s1, src_flat_scratch_base_hi12004; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)12005; GFX1250-SDAG-NEXT: s_cmp_lt_u32 s2, 0x400000012006; GFX1250-SDAG-NEXT: s_cselect_b32 s2, -1, 012007; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s212008; GFX1250-SDAG-NEXT: s_mov_b32 s2, -112009; GFX1250-SDAG-NEXT: s_cbranch_vccz .LBB111_512010; GFX1250-SDAG-NEXT: ; %bb.4: ; %atomicrmw.global12011; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 012012; GFX1250-SDAG-NEXT: s_mov_b32 s2, 012013; GFX1250-SDAG-NEXT: s_wait_storecnt 0x012014; GFX1250-SDAG-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1]12015; GFX1250-SDAG-NEXT: s_wait_storecnt 0x012016; GFX1250-SDAG-NEXT: .LBB111_5: ; %Flow12017; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s212018; GFX1250-SDAG-NEXT: s_cbranch_vccnz .LBB111_712019; GFX1250-SDAG-NEXT: ; %bb.6: ; %atomicrmw.private12020; GFX1250-SDAG-NEXT: s_sub_co_i32 s2, s0, src_flat_scratch_base_lo12021; GFX1250-SDAG-NEXT: s_cmp_lg_u64 s[0:1], 012022; GFX1250-SDAG-NEXT: s_cselect_b32 s2, s2, -112023; GFX1250-SDAG-NEXT: scratch_load_b64 v[2:3], off, s212024; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x012025; GFX1250-SDAG-NEXT: v_add_f64_e32 v[2:3], v[2:3], v[0:1]12026; GFX1250-SDAG-NEXT: scratch_store_b64 off, v[2:3], s212027; GFX1250-SDAG-NEXT: .LBB111_7: ; %Flow112028; GFX1250-SDAG-NEXT: s_wait_xcnt 0x012029; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB111_212030; GFX1250-SDAG-NEXT: .LBB111_8: ; %atomicrmw.shared12031; GFX1250-SDAG-NEXT: s_cmp_lg_u64 s[0:1], 012032; GFX1250-SDAG-NEXT: s_cselect_b32 s0, s0, -112033; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)12034; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s012035; GFX1250-SDAG-NEXT: s_wait_storecnt 0x012036; GFX1250-SDAG-NEXT: ds_add_f64 v2, v[0:1]12037; GFX1250-SDAG-NEXT: s_wait_dscnt 0x012038; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]12039;12040; GFX1250-GISEL-LABEL: flat_atomic_fadd_f64_saddr_nortn:12041; GFX1250-GISEL: ; %bb.0:12042; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x012043; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x012044; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s0, 0x5012045; GFX1250-GISEL-NEXT: s_mov_b64 s[2:3], src_shared_base12046; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s1, 012047; GFX1250-GISEL-NEXT: s_mov_b32 s2, 112048; GFX1250-GISEL-NEXT: s_cmp_lg_u32 s1, s312049; GFX1250-GISEL-NEXT: s_cbranch_scc0 .LBB111_612050; GFX1250-GISEL-NEXT: ; %bb.1: ; %atomicrmw.check.private12051; GFX1250-GISEL-NEXT: s_xor_b32 s2, s1, src_flat_scratch_base_hi12052; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)12053; GFX1250-GISEL-NEXT: s_cmp_ge_u32 s2, 0x400000012054; GFX1250-GISEL-NEXT: s_mov_b32 s2, 112055; GFX1250-GISEL-NEXT: s_cbranch_scc0 .LBB111_312056; GFX1250-GISEL-NEXT: ; %bb.2: ; %atomicrmw.global12057; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 012058; GFX1250-GISEL-NEXT: s_mov_b32 s2, 012059; GFX1250-GISEL-NEXT: s_wait_storecnt 0x012060; GFX1250-GISEL-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1]12061; GFX1250-GISEL-NEXT: s_wait_storecnt 0x012062; GFX1250-GISEL-NEXT: .LBB111_3: ; %Flow12063; GFX1250-GISEL-NEXT: s_xor_b32 s2, s2, 112064; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)12065; GFX1250-GISEL-NEXT: s_cmp_lg_u32 s2, 012066; GFX1250-GISEL-NEXT: s_cbranch_scc1 .LBB111_512067; GFX1250-GISEL-NEXT: ; %bb.4: ; %atomicrmw.private12068; GFX1250-GISEL-NEXT: s_sub_co_i32 s2, s0, src_flat_scratch_base_lo12069; GFX1250-GISEL-NEXT: s_cmp_lg_u64 s[0:1], 012070; GFX1250-GISEL-NEXT: s_cselect_b32 s2, s2, -112071; GFX1250-GISEL-NEXT: scratch_load_b64 v[2:3], off, s212072; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x012073; GFX1250-GISEL-NEXT: v_add_f64_e32 v[2:3], v[2:3], v[0:1]12074; GFX1250-GISEL-NEXT: scratch_store_b64 off, v[2:3], s212075; GFX1250-GISEL-NEXT: .LBB111_5: ; %Flow112076; GFX1250-GISEL-NEXT: s_wait_xcnt 0x012077; GFX1250-GISEL-NEXT: s_mov_b32 s2, 012078; GFX1250-GISEL-NEXT: .LBB111_6: ; %Flow212079; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)12080; GFX1250-GISEL-NEXT: s_xor_b32 s2, s2, 112081; GFX1250-GISEL-NEXT: s_cmp_lg_u32 s2, 012082; GFX1250-GISEL-NEXT: s_cbranch_scc1 .LBB111_812083; GFX1250-GISEL-NEXT: ; %bb.7: ; %atomicrmw.shared12084; GFX1250-GISEL-NEXT: s_cmp_lg_u64 s[0:1], 012085; GFX1250-GISEL-NEXT: s_cselect_b32 s0, s0, -112086; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)12087; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, s012088; GFX1250-GISEL-NEXT: s_wait_storecnt 0x012089; GFX1250-GISEL-NEXT: ds_add_f64 v2, v[0:1]12090; GFX1250-GISEL-NEXT: s_wait_dscnt 0x012091; GFX1250-GISEL-NEXT: .LBB111_8: ; %atomicrmw.phi12092; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]12093;12094; GFX950-SDAG-LABEL: flat_atomic_fadd_f64_saddr_nortn:12095; GFX950-SDAG: ; %bb.0:12096; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12097; GFX950-SDAG-NEXT: s_add_u32 s0, s0, 0x5012098; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], src_shared_base12099; GFX950-SDAG-NEXT: s_addc_u32 s1, s1, 012100; GFX950-SDAG-NEXT: s_cmp_eq_u32 s1, s312101; GFX950-SDAG-NEXT: s_cselect_b64 s[2:3], -1, 012102; GFX950-SDAG-NEXT: s_andn2_b64 vcc, exec, s[2:3]12103; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], -112104; GFX950-SDAG-NEXT: s_cbranch_vccnz .LBB111_312105; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow212106; GFX950-SDAG-NEXT: s_andn2_b64 vcc, exec, s[2:3]12107; GFX950-SDAG-NEXT: s_cbranch_vccz .LBB111_812108; GFX950-SDAG-NEXT: .LBB111_2: ; %atomicrmw.phi12109; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)12110; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]12111; GFX950-SDAG-NEXT: .LBB111_3: ; %atomicrmw.check.private12112; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], src_private_base12113; GFX950-SDAG-NEXT: s_cmp_eq_u32 s1, s312114; GFX950-SDAG-NEXT: s_cselect_b64 s[2:3], -1, 012115; GFX950-SDAG-NEXT: s_andn2_b64 vcc, exec, s[2:3]12116; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], -112117; GFX950-SDAG-NEXT: s_cbranch_vccz .LBB111_512118; GFX950-SDAG-NEXT: ; %bb.4: ; %atomicrmw.global12119; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, 012120; GFX950-SDAG-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1]12121; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], 012122; GFX950-SDAG-NEXT: .LBB111_5: ; %Flow12123; GFX950-SDAG-NEXT: s_andn2_b64 vcc, exec, s[2:3]12124; GFX950-SDAG-NEXT: s_cbranch_vccnz .LBB111_712125; GFX950-SDAG-NEXT: ; %bb.6: ; %atomicrmw.private12126; GFX950-SDAG-NEXT: s_cmp_lg_u64 s[0:1], 012127; GFX950-SDAG-NEXT: s_cselect_b32 s2, s0, -112128; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[2:3], off, s212129; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)12130; GFX950-SDAG-NEXT: v_add_f64 v[2:3], v[2:3], v[0:1]12131; GFX950-SDAG-NEXT: scratch_store_dwordx2 off, v[2:3], s212132; GFX950-SDAG-NEXT: .LBB111_7: ; %Flow112133; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB111_212134; GFX950-SDAG-NEXT: .LBB111_8: ; %atomicrmw.shared12135; GFX950-SDAG-NEXT: s_cmp_lg_u64 s[0:1], 012136; GFX950-SDAG-NEXT: s_cselect_b32 s0, s0, -112137; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s012138; GFX950-SDAG-NEXT: ds_add_f64 v2, v[0:1]12139; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12140; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]12141;12142; GFX950-GISEL-LABEL: flat_atomic_fadd_f64_saddr_nortn:12143; GFX950-GISEL: ; %bb.0:12144; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12145; GFX950-GISEL-NEXT: s_add_u32 s0, s0, 0x5012146; GFX950-GISEL-NEXT: s_mov_b64 s[2:3], src_shared_base12147; GFX950-GISEL-NEXT: s_addc_u32 s1, s1, 012148; GFX950-GISEL-NEXT: s_cmp_lg_u32 s1, s312149; GFX950-GISEL-NEXT: s_mov_b32 s2, 112150; GFX950-GISEL-NEXT: s_cbranch_scc0 .LBB111_612151; GFX950-GISEL-NEXT: ; %bb.1: ; %atomicrmw.check.private12152; GFX950-GISEL-NEXT: s_mov_b64 s[2:3], src_private_base12153; GFX950-GISEL-NEXT: s_cmp_lg_u32 s1, s312154; GFX950-GISEL-NEXT: s_mov_b32 s2, 112155; GFX950-GISEL-NEXT: s_cbranch_scc0 .LBB111_312156; GFX950-GISEL-NEXT: ; %bb.2: ; %atomicrmw.global12157; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, 012158; GFX950-GISEL-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1]12159; GFX950-GISEL-NEXT: s_mov_b32 s2, 012160; GFX950-GISEL-NEXT: .LBB111_3: ; %Flow12161; GFX950-GISEL-NEXT: s_xor_b32 s2, s2, 112162; GFX950-GISEL-NEXT: s_cmp_lg_u32 s2, 012163; GFX950-GISEL-NEXT: s_cbranch_scc1 .LBB111_512164; GFX950-GISEL-NEXT: ; %bb.4: ; %atomicrmw.private12165; GFX950-GISEL-NEXT: s_cmp_lg_u64 s[0:1], 012166; GFX950-GISEL-NEXT: s_cselect_b32 s2, s0, -112167; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[2:3], off, s212168; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)12169; GFX950-GISEL-NEXT: v_add_f64 v[2:3], v[2:3], v[0:1]12170; GFX950-GISEL-NEXT: scratch_store_dwordx2 off, v[2:3], s212171; GFX950-GISEL-NEXT: .LBB111_5: ; %Flow112172; GFX950-GISEL-NEXT: s_mov_b32 s2, 012173; GFX950-GISEL-NEXT: .LBB111_6: ; %Flow212174; GFX950-GISEL-NEXT: s_xor_b32 s2, s2, 112175; GFX950-GISEL-NEXT: s_cmp_lg_u32 s2, 012176; GFX950-GISEL-NEXT: s_cbranch_scc1 .LBB111_812177; GFX950-GISEL-NEXT: ; %bb.7: ; %atomicrmw.shared12178; GFX950-GISEL-NEXT: s_cmp_lg_u64 s[0:1], 012179; GFX950-GISEL-NEXT: s_cselect_b32 s0, s0, -112180; GFX950-GISEL-NEXT: v_mov_b32_e32 v2, s012181; GFX950-GISEL-NEXT: ds_add_f64 v2, v[0:1]12182; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)12183; GFX950-GISEL-NEXT: .LBB111_8: ; %atomicrmw.phi12184; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)12185; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]12186 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1012187 %unused = atomicrmw fadd ptr %gep.0, double %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !012188 ret void12189}12190 12191define double @flat_atomic_fmax_f64_saddr_rtn(ptr inreg %ptr, double %data) {12192; GFX1250-SDAG-LABEL: flat_atomic_fmax_f64_saddr_rtn:12193; GFX1250-SDAG: ; %bb.0:12194; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x012195; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x012196; GFX1250-SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 0x5012197; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)12198; GFX1250-SDAG-NEXT: s_xor_b32 s2, s1, src_flat_scratch_base_hi12199; GFX1250-SDAG-NEXT: s_cmp_lt_u32 s2, 0x400000012200; GFX1250-SDAG-NEXT: s_cselect_b32 s2, -1, 012201; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)12202; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s212203; GFX1250-SDAG-NEXT: s_cbranch_vccz .LBB112_212204; GFX1250-SDAG-NEXT: ; %bb.1: ; %atomicrmw.global12205; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 012206; GFX1250-SDAG-NEXT: s_wait_storecnt 0x012207; GFX1250-SDAG-NEXT: flat_atomic_max_num_f64 v[2:3], v2, v[0:1], s[0:1] th:TH_ATOMIC_RETURN12208; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x012209; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB112_312210; GFX1250-SDAG-NEXT: s_branch .LBB112_412211; GFX1250-SDAG-NEXT: .LBB112_2:12212; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr312213; GFX1250-SDAG-NEXT: .LBB112_3: ; %atomicrmw.private12214; GFX1250-SDAG-NEXT: s_sub_co_i32 s2, s0, src_flat_scratch_base_lo12215; GFX1250-SDAG-NEXT: s_cmp_lg_u64 s[0:1], 012216; GFX1250-SDAG-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[0:1]12217; GFX1250-SDAG-NEXT: s_cselect_b32 s0, s2, -112218; GFX1250-SDAG-NEXT: scratch_load_b64 v[2:3], off, s012219; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x012220; GFX1250-SDAG-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]12221; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)12222; GFX1250-SDAG-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[0:1]12223; GFX1250-SDAG-NEXT: scratch_store_b64 off, v[0:1], s012224; GFX1250-SDAG-NEXT: .LBB112_4: ; %atomicrmw.end12225; GFX1250-SDAG-NEXT: s_wait_xcnt 0x012226; GFX1250-SDAG-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v312227; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]12228;12229; GFX1250-GISEL-LABEL: flat_atomic_fmax_f64_saddr_rtn:12230; GFX1250-GISEL: ; %bb.0:12231; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x012232; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x012233; GFX1250-GISEL-NEXT: s_add_co_u32 s2, s0, 0x5012234; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s3, s1, 012235; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr312236; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)12237; GFX1250-GISEL-NEXT: s_xor_b32 s4, s3, src_flat_scratch_base_hi12238; GFX1250-GISEL-NEXT: s_cmp_ge_u32 s4, 0x400000012239; GFX1250-GISEL-NEXT: s_mov_b32 s4, 112240; GFX1250-GISEL-NEXT: s_cbranch_scc0 .LBB112_212241; GFX1250-GISEL-NEXT: ; %bb.1: ; %atomicrmw.global12242; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 012243; GFX1250-GISEL-NEXT: s_mov_b32 s4, 012244; GFX1250-GISEL-NEXT: s_wait_storecnt 0x012245; GFX1250-GISEL-NEXT: flat_atomic_max_num_f64 v[2:3], v2, v[0:1], s[0:1] offset:80 th:TH_ATOMIC_RETURN12246; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x012247; GFX1250-GISEL-NEXT: .LBB112_2: ; %Flow12248; GFX1250-GISEL-NEXT: s_xor_b32 s0, s4, 112249; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)12250; GFX1250-GISEL-NEXT: s_cmp_lg_u32 s0, 012251; GFX1250-GISEL-NEXT: s_cbranch_scc1 .LBB112_412252; GFX1250-GISEL-NEXT: ; %bb.3: ; %atomicrmw.private12253; GFX1250-GISEL-NEXT: s_sub_co_i32 s0, s2, src_flat_scratch_base_lo12254; GFX1250-GISEL-NEXT: s_cmp_lg_u64 s[2:3], 012255; GFX1250-GISEL-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[0:1]12256; GFX1250-GISEL-NEXT: s_cselect_b32 s0, s0, -112257; GFX1250-GISEL-NEXT: scratch_load_b64 v[2:3], off, s012258; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x012259; GFX1250-GISEL-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]12260; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)12261; GFX1250-GISEL-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[0:1]12262; GFX1250-GISEL-NEXT: scratch_store_b64 off, v[0:1], s012263; GFX1250-GISEL-NEXT: .LBB112_4: ; %atomicrmw.end12264; GFX1250-GISEL-NEXT: s_wait_xcnt 0x012265; GFX1250-GISEL-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v312266; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]12267;12268; GFX950-SDAG-LABEL: flat_atomic_fmax_f64_saddr_rtn:12269; GFX950-SDAG: ; %bb.0:12270; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12271; GFX950-SDAG-NEXT: s_add_u32 s0, s0, 0x5012272; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], src_private_base12273; GFX950-SDAG-NEXT: s_addc_u32 s1, s1, 012274; GFX950-SDAG-NEXT: s_cmp_eq_u32 s1, s312275; GFX950-SDAG-NEXT: s_cselect_b64 s[2:3], -1, 012276; GFX950-SDAG-NEXT: s_andn2_b64 vcc, exec, s[2:3]12277; GFX950-SDAG-NEXT: s_cbranch_vccz .LBB112_212278; GFX950-SDAG-NEXT: ; %bb.1: ; %atomicrmw.global12279; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[0:1]12280; GFX950-SDAG-NEXT: flat_atomic_max_f64 v[2:3], v[2:3], v[0:1] sc012281; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)12282; GFX950-SDAG-NEXT: s_cbranch_execz .LBB112_312283; GFX950-SDAG-NEXT: s_branch .LBB112_412284; GFX950-SDAG-NEXT: .LBB112_2:12285; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr312286; GFX950-SDAG-NEXT: .LBB112_3: ; %atomicrmw.private12287; GFX950-SDAG-NEXT: s_cmp_lg_u64 s[0:1], 012288; GFX950-SDAG-NEXT: s_cselect_b32 s0, s0, -112289; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[2:3], off, s012290; GFX950-SDAG-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]12291; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)12292; GFX950-SDAG-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]12293; GFX950-SDAG-NEXT: v_max_f64 v[0:1], v[4:5], v[0:1]12294; GFX950-SDAG-NEXT: scratch_store_dwordx2 off, v[0:1], s012295; GFX950-SDAG-NEXT: .LBB112_4: ; %atomicrmw.end12296; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)12297; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, v212298; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, v312299; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]12300;12301; GFX950-GISEL-LABEL: flat_atomic_fmax_f64_saddr_rtn:12302; GFX950-GISEL: ; %bb.0:12303; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12304; GFX950-GISEL-NEXT: s_add_u32 s2, s0, 0x5012305; GFX950-GISEL-NEXT: s_mov_b64 s[4:5], src_private_base12306; GFX950-GISEL-NEXT: s_addc_u32 s3, s1, 012307; GFX950-GISEL-NEXT: s_cmp_lg_u32 s3, s512308; GFX950-GISEL-NEXT: s_mov_b32 s4, 112309; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr312310; GFX950-GISEL-NEXT: s_cbranch_scc0 .LBB112_212311; GFX950-GISEL-NEXT: ; %bb.1: ; %atomicrmw.global12312; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]12313; GFX950-GISEL-NEXT: flat_atomic_max_f64 v[2:3], v[2:3], v[0:1] offset:80 sc012314; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)12315; GFX950-GISEL-NEXT: s_mov_b32 s4, 012316; GFX950-GISEL-NEXT: .LBB112_2: ; %Flow12317; GFX950-GISEL-NEXT: s_xor_b32 s0, s4, 112318; GFX950-GISEL-NEXT: s_cmp_lg_u32 s0, 012319; GFX950-GISEL-NEXT: s_cbranch_scc1 .LBB112_412320; GFX950-GISEL-NEXT: ; %bb.3: ; %atomicrmw.private12321; GFX950-GISEL-NEXT: s_cmp_lg_u64 s[2:3], 012322; GFX950-GISEL-NEXT: s_cselect_b32 s0, s2, -112323; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[2:3], off, s012324; GFX950-GISEL-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]12325; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)12326; GFX950-GISEL-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]12327; GFX950-GISEL-NEXT: v_max_f64 v[0:1], v[4:5], v[0:1]12328; GFX950-GISEL-NEXT: scratch_store_dwordx2 off, v[0:1], s012329; GFX950-GISEL-NEXT: .LBB112_4: ; %atomicrmw.end12330; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)12331; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, v212332; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, v312333; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]12334 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1012335 %result = atomicrmw fmax ptr %gep.0, double %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !012336 ret double %result12337}12338 12339define void @flat_atomic_fmax_f64_saddr_nortn(ptr inreg %ptr, double %data) {12340; GFX1250-SDAG-LABEL: flat_atomic_fmax_f64_saddr_nortn:12341; GFX1250-SDAG: ; %bb.0:12342; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x012343; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x012344; GFX1250-SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 0x5012345; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)12346; GFX1250-SDAG-NEXT: s_xor_b32 s2, s1, src_flat_scratch_base_hi12347; GFX1250-SDAG-NEXT: s_cmp_lt_u32 s2, 0x400000012348; GFX1250-SDAG-NEXT: s_cselect_b32 s2, -1, 012349; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)12350; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s212351; GFX1250-SDAG-NEXT: s_mov_b32 s2, -112352; GFX1250-SDAG-NEXT: s_cbranch_vccnz .LBB113_312353; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow12354; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s212355; GFX1250-SDAG-NEXT: s_cbranch_vccz .LBB113_412356; GFX1250-SDAG-NEXT: .LBB113_2: ; %atomicrmw.phi12357; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]12358; GFX1250-SDAG-NEXT: .LBB113_3: ; %atomicrmw.global12359; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 012360; GFX1250-SDAG-NEXT: s_wait_storecnt 0x012361; GFX1250-SDAG-NEXT: flat_atomic_max_num_f64 v2, v[0:1], s[0:1]12362; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x012363; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB113_212364; GFX1250-SDAG-NEXT: .LBB113_4: ; %atomicrmw.private12365; GFX1250-SDAG-NEXT: s_sub_co_i32 s2, s0, src_flat_scratch_base_lo12366; GFX1250-SDAG-NEXT: s_cmp_lg_u64 s[0:1], 012367; GFX1250-SDAG-NEXT: s_wait_xcnt 0x012368; GFX1250-SDAG-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[0:1]12369; GFX1250-SDAG-NEXT: s_cselect_b32 s0, s2, -112370; GFX1250-SDAG-NEXT: scratch_load_b64 v[2:3], off, s012371; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x012372; GFX1250-SDAG-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]12373; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)12374; GFX1250-SDAG-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[0:1]12375; GFX1250-SDAG-NEXT: scratch_store_b64 off, v[0:1], s012376; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]12377;12378; GFX1250-GISEL-LABEL: flat_atomic_fmax_f64_saddr_nortn:12379; GFX1250-GISEL: ; %bb.0:12380; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x012381; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x012382; GFX1250-GISEL-NEXT: s_add_co_u32 s2, s0, 0x5012383; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s3, s1, 012384; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)12385; GFX1250-GISEL-NEXT: s_xor_b32 s4, s3, src_flat_scratch_base_hi12386; GFX1250-GISEL-NEXT: s_cmp_ge_u32 s4, 0x400000012387; GFX1250-GISEL-NEXT: s_mov_b32 s4, 112388; GFX1250-GISEL-NEXT: s_cbranch_scc0 .LBB113_212389; GFX1250-GISEL-NEXT: ; %bb.1: ; %atomicrmw.global12390; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 012391; GFX1250-GISEL-NEXT: s_mov_b32 s4, 012392; GFX1250-GISEL-NEXT: s_wait_storecnt 0x012393; GFX1250-GISEL-NEXT: flat_atomic_max_num_f64 v2, v[0:1], s[0:1] offset:8012394; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x012395; GFX1250-GISEL-NEXT: .LBB113_2: ; %Flow12396; GFX1250-GISEL-NEXT: s_wait_xcnt 0x012397; GFX1250-GISEL-NEXT: s_xor_b32 s0, s4, 112398; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)12399; GFX1250-GISEL-NEXT: s_cmp_lg_u32 s0, 012400; GFX1250-GISEL-NEXT: s_cbranch_scc1 .LBB113_412401; GFX1250-GISEL-NEXT: ; %bb.3: ; %atomicrmw.private12402; GFX1250-GISEL-NEXT: s_sub_co_i32 s0, s2, src_flat_scratch_base_lo12403; GFX1250-GISEL-NEXT: s_cmp_lg_u64 s[2:3], 012404; GFX1250-GISEL-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[0:1]12405; GFX1250-GISEL-NEXT: s_cselect_b32 s0, s0, -112406; GFX1250-GISEL-NEXT: scratch_load_b64 v[2:3], off, s012407; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x012408; GFX1250-GISEL-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]12409; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)12410; GFX1250-GISEL-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[0:1]12411; GFX1250-GISEL-NEXT: scratch_store_b64 off, v[0:1], s012412; GFX1250-GISEL-NEXT: .LBB113_4: ; %atomicrmw.phi12413; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]12414;12415; GFX950-SDAG-LABEL: flat_atomic_fmax_f64_saddr_nortn:12416; GFX950-SDAG: ; %bb.0:12417; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12418; GFX950-SDAG-NEXT: s_add_u32 s0, s0, 0x5012419; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], src_private_base12420; GFX950-SDAG-NEXT: s_addc_u32 s1, s1, 012421; GFX950-SDAG-NEXT: s_cmp_eq_u32 s1, s312422; GFX950-SDAG-NEXT: s_cselect_b64 s[2:3], -1, 012423; GFX950-SDAG-NEXT: s_andn2_b64 vcc, exec, s[2:3]12424; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], -112425; GFX950-SDAG-NEXT: s_cbranch_vccnz .LBB113_312426; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow12427; GFX950-SDAG-NEXT: s_andn2_b64 vcc, exec, s[2:3]12428; GFX950-SDAG-NEXT: s_cbranch_vccz .LBB113_412429; GFX950-SDAG-NEXT: .LBB113_2: ; %atomicrmw.phi12430; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)12431; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]12432; GFX950-SDAG-NEXT: .LBB113_3: ; %atomicrmw.global12433; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[0:1]12434; GFX950-SDAG-NEXT: flat_atomic_max_f64 v[2:3], v[0:1]12435; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)12436; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB113_212437; GFX950-SDAG-NEXT: .LBB113_4: ; %atomicrmw.private12438; GFX950-SDAG-NEXT: s_cmp_lg_u64 s[0:1], 012439; GFX950-SDAG-NEXT: s_cselect_b32 s0, s0, -112440; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[2:3], off, s012441; GFX950-SDAG-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]12442; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)12443; GFX950-SDAG-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]12444; GFX950-SDAG-NEXT: v_max_f64 v[0:1], v[2:3], v[0:1]12445; GFX950-SDAG-NEXT: scratch_store_dwordx2 off, v[0:1], s012446; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)12447; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]12448;12449; GFX950-GISEL-LABEL: flat_atomic_fmax_f64_saddr_nortn:12450; GFX950-GISEL: ; %bb.0:12451; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12452; GFX950-GISEL-NEXT: s_add_u32 s2, s0, 0x5012453; GFX950-GISEL-NEXT: s_mov_b64 s[4:5], src_private_base12454; GFX950-GISEL-NEXT: s_addc_u32 s3, s1, 012455; GFX950-GISEL-NEXT: s_cmp_lg_u32 s3, s512456; GFX950-GISEL-NEXT: s_mov_b32 s4, 112457; GFX950-GISEL-NEXT: s_cbranch_scc0 .LBB113_212458; GFX950-GISEL-NEXT: ; %bb.1: ; %atomicrmw.global12459; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]12460; GFX950-GISEL-NEXT: flat_atomic_max_f64 v[2:3], v[0:1] offset:8012461; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)12462; GFX950-GISEL-NEXT: s_mov_b32 s4, 012463; GFX950-GISEL-NEXT: .LBB113_2: ; %Flow12464; GFX950-GISEL-NEXT: s_xor_b32 s0, s4, 112465; GFX950-GISEL-NEXT: s_cmp_lg_u32 s0, 012466; GFX950-GISEL-NEXT: s_cbranch_scc1 .LBB113_412467; GFX950-GISEL-NEXT: ; %bb.3: ; %atomicrmw.private12468; GFX950-GISEL-NEXT: s_cmp_lg_u64 s[2:3], 012469; GFX950-GISEL-NEXT: s_cselect_b32 s0, s2, -112470; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[2:3], off, s012471; GFX950-GISEL-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]12472; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)12473; GFX950-GISEL-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]12474; GFX950-GISEL-NEXT: v_max_f64 v[0:1], v[2:3], v[0:1]12475; GFX950-GISEL-NEXT: scratch_store_dwordx2 off, v[0:1], s012476; GFX950-GISEL-NEXT: .LBB113_4: ; %atomicrmw.phi12477; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)12478; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]12479 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1012480 %unused = atomicrmw fmax ptr %gep.0, double %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !012481 ret void12482}12483 12484define double @flat_atomic_fmin_f64_saddr_rtn(ptr inreg %ptr, double %data) {12485; GFX1250-SDAG-LABEL: flat_atomic_fmin_f64_saddr_rtn:12486; GFX1250-SDAG: ; %bb.0:12487; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x012488; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x012489; GFX1250-SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 0x5012490; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)12491; GFX1250-SDAG-NEXT: s_xor_b32 s2, s1, src_flat_scratch_base_hi12492; GFX1250-SDAG-NEXT: s_cmp_lt_u32 s2, 0x400000012493; GFX1250-SDAG-NEXT: s_cselect_b32 s2, -1, 012494; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)12495; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s212496; GFX1250-SDAG-NEXT: s_cbranch_vccz .LBB114_212497; GFX1250-SDAG-NEXT: ; %bb.1: ; %atomicrmw.global12498; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 012499; GFX1250-SDAG-NEXT: s_wait_storecnt 0x012500; GFX1250-SDAG-NEXT: flat_atomic_min_num_f64 v[2:3], v2, v[0:1], s[0:1] th:TH_ATOMIC_RETURN12501; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x012502; GFX1250-SDAG-NEXT: s_cbranch_execz .LBB114_312503; GFX1250-SDAG-NEXT: s_branch .LBB114_412504; GFX1250-SDAG-NEXT: .LBB114_2:12505; GFX1250-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr312506; GFX1250-SDAG-NEXT: .LBB114_3: ; %atomicrmw.private12507; GFX1250-SDAG-NEXT: s_sub_co_i32 s2, s0, src_flat_scratch_base_lo12508; GFX1250-SDAG-NEXT: s_cmp_lg_u64 s[0:1], 012509; GFX1250-SDAG-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[0:1]12510; GFX1250-SDAG-NEXT: s_cselect_b32 s0, s2, -112511; GFX1250-SDAG-NEXT: scratch_load_b64 v[2:3], off, s012512; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x012513; GFX1250-SDAG-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]12514; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)12515; GFX1250-SDAG-NEXT: v_min_num_f64_e32 v[0:1], v[4:5], v[0:1]12516; GFX1250-SDAG-NEXT: scratch_store_b64 off, v[0:1], s012517; GFX1250-SDAG-NEXT: .LBB114_4: ; %atomicrmw.end12518; GFX1250-SDAG-NEXT: s_wait_xcnt 0x012519; GFX1250-SDAG-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v312520; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]12521;12522; GFX1250-GISEL-LABEL: flat_atomic_fmin_f64_saddr_rtn:12523; GFX1250-GISEL: ; %bb.0:12524; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x012525; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x012526; GFX1250-GISEL-NEXT: s_add_co_u32 s2, s0, 0x5012527; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s3, s1, 012528; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr312529; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)12530; GFX1250-GISEL-NEXT: s_xor_b32 s4, s3, src_flat_scratch_base_hi12531; GFX1250-GISEL-NEXT: s_cmp_ge_u32 s4, 0x400000012532; GFX1250-GISEL-NEXT: s_mov_b32 s4, 112533; GFX1250-GISEL-NEXT: s_cbranch_scc0 .LBB114_212534; GFX1250-GISEL-NEXT: ; %bb.1: ; %atomicrmw.global12535; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 012536; GFX1250-GISEL-NEXT: s_mov_b32 s4, 012537; GFX1250-GISEL-NEXT: s_wait_storecnt 0x012538; GFX1250-GISEL-NEXT: flat_atomic_min_num_f64 v[2:3], v2, v[0:1], s[0:1] offset:80 th:TH_ATOMIC_RETURN12539; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x012540; GFX1250-GISEL-NEXT: .LBB114_2: ; %Flow12541; GFX1250-GISEL-NEXT: s_xor_b32 s0, s4, 112542; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)12543; GFX1250-GISEL-NEXT: s_cmp_lg_u32 s0, 012544; GFX1250-GISEL-NEXT: s_cbranch_scc1 .LBB114_412545; GFX1250-GISEL-NEXT: ; %bb.3: ; %atomicrmw.private12546; GFX1250-GISEL-NEXT: s_sub_co_i32 s0, s2, src_flat_scratch_base_lo12547; GFX1250-GISEL-NEXT: s_cmp_lg_u64 s[2:3], 012548; GFX1250-GISEL-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[0:1]12549; GFX1250-GISEL-NEXT: s_cselect_b32 s0, s0, -112550; GFX1250-GISEL-NEXT: scratch_load_b64 v[2:3], off, s012551; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x012552; GFX1250-GISEL-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]12553; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)12554; GFX1250-GISEL-NEXT: v_min_num_f64_e32 v[0:1], v[4:5], v[0:1]12555; GFX1250-GISEL-NEXT: scratch_store_b64 off, v[0:1], s012556; GFX1250-GISEL-NEXT: .LBB114_4: ; %atomicrmw.end12557; GFX1250-GISEL-NEXT: s_wait_xcnt 0x012558; GFX1250-GISEL-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v312559; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]12560;12561; GFX950-SDAG-LABEL: flat_atomic_fmin_f64_saddr_rtn:12562; GFX950-SDAG: ; %bb.0:12563; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12564; GFX950-SDAG-NEXT: s_add_u32 s0, s0, 0x5012565; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], src_private_base12566; GFX950-SDAG-NEXT: s_addc_u32 s1, s1, 012567; GFX950-SDAG-NEXT: s_cmp_eq_u32 s1, s312568; GFX950-SDAG-NEXT: s_cselect_b64 s[2:3], -1, 012569; GFX950-SDAG-NEXT: s_andn2_b64 vcc, exec, s[2:3]12570; GFX950-SDAG-NEXT: s_cbranch_vccz .LBB114_212571; GFX950-SDAG-NEXT: ; %bb.1: ; %atomicrmw.global12572; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[0:1]12573; GFX950-SDAG-NEXT: flat_atomic_min_f64 v[2:3], v[2:3], v[0:1] sc012574; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)12575; GFX950-SDAG-NEXT: s_cbranch_execz .LBB114_312576; GFX950-SDAG-NEXT: s_branch .LBB114_412577; GFX950-SDAG-NEXT: .LBB114_2:12578; GFX950-SDAG-NEXT: ; implicit-def: $vgpr2_vgpr312579; GFX950-SDAG-NEXT: .LBB114_3: ; %atomicrmw.private12580; GFX950-SDAG-NEXT: s_cmp_lg_u64 s[0:1], 012581; GFX950-SDAG-NEXT: s_cselect_b32 s0, s0, -112582; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[2:3], off, s012583; GFX950-SDAG-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]12584; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)12585; GFX950-SDAG-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]12586; GFX950-SDAG-NEXT: v_min_f64 v[0:1], v[4:5], v[0:1]12587; GFX950-SDAG-NEXT: scratch_store_dwordx2 off, v[0:1], s012588; GFX950-SDAG-NEXT: .LBB114_4: ; %atomicrmw.end12589; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)12590; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, v212591; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, v312592; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]12593;12594; GFX950-GISEL-LABEL: flat_atomic_fmin_f64_saddr_rtn:12595; GFX950-GISEL: ; %bb.0:12596; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12597; GFX950-GISEL-NEXT: s_add_u32 s2, s0, 0x5012598; GFX950-GISEL-NEXT: s_mov_b64 s[4:5], src_private_base12599; GFX950-GISEL-NEXT: s_addc_u32 s3, s1, 012600; GFX950-GISEL-NEXT: s_cmp_lg_u32 s3, s512601; GFX950-GISEL-NEXT: s_mov_b32 s4, 112602; GFX950-GISEL-NEXT: ; implicit-def: $vgpr2_vgpr312603; GFX950-GISEL-NEXT: s_cbranch_scc0 .LBB114_212604; GFX950-GISEL-NEXT: ; %bb.1: ; %atomicrmw.global12605; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]12606; GFX950-GISEL-NEXT: flat_atomic_min_f64 v[2:3], v[2:3], v[0:1] offset:80 sc012607; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)12608; GFX950-GISEL-NEXT: s_mov_b32 s4, 012609; GFX950-GISEL-NEXT: .LBB114_2: ; %Flow12610; GFX950-GISEL-NEXT: s_xor_b32 s0, s4, 112611; GFX950-GISEL-NEXT: s_cmp_lg_u32 s0, 012612; GFX950-GISEL-NEXT: s_cbranch_scc1 .LBB114_412613; GFX950-GISEL-NEXT: ; %bb.3: ; %atomicrmw.private12614; GFX950-GISEL-NEXT: s_cmp_lg_u64 s[2:3], 012615; GFX950-GISEL-NEXT: s_cselect_b32 s0, s2, -112616; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[2:3], off, s012617; GFX950-GISEL-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]12618; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)12619; GFX950-GISEL-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]12620; GFX950-GISEL-NEXT: v_min_f64 v[0:1], v[4:5], v[0:1]12621; GFX950-GISEL-NEXT: scratch_store_dwordx2 off, v[0:1], s012622; GFX950-GISEL-NEXT: .LBB114_4: ; %atomicrmw.end12623; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)12624; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, v212625; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, v312626; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]12627 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1012628 %result = atomicrmw fmin ptr %gep.0, double %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !012629 ret double %result12630}12631 12632define void @flat_atomic_fmin_f64_saddr_nortn(ptr inreg %ptr, double %data) {12633; GFX1250-SDAG-LABEL: flat_atomic_fmin_f64_saddr_nortn:12634; GFX1250-SDAG: ; %bb.0:12635; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x012636; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x012637; GFX1250-SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], 0x5012638; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)12639; GFX1250-SDAG-NEXT: s_xor_b32 s2, s1, src_flat_scratch_base_hi12640; GFX1250-SDAG-NEXT: s_cmp_lt_u32 s2, 0x400000012641; GFX1250-SDAG-NEXT: s_cselect_b32 s2, -1, 012642; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)12643; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s212644; GFX1250-SDAG-NEXT: s_mov_b32 s2, -112645; GFX1250-SDAG-NEXT: s_cbranch_vccnz .LBB115_312646; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow12647; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s212648; GFX1250-SDAG-NEXT: s_cbranch_vccz .LBB115_412649; GFX1250-SDAG-NEXT: .LBB115_2: ; %atomicrmw.phi12650; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]12651; GFX1250-SDAG-NEXT: .LBB115_3: ; %atomicrmw.global12652; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 012653; GFX1250-SDAG-NEXT: s_wait_storecnt 0x012654; GFX1250-SDAG-NEXT: flat_atomic_min_num_f64 v2, v[0:1], s[0:1]12655; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x012656; GFX1250-SDAG-NEXT: s_cbranch_execnz .LBB115_212657; GFX1250-SDAG-NEXT: .LBB115_4: ; %atomicrmw.private12658; GFX1250-SDAG-NEXT: s_sub_co_i32 s2, s0, src_flat_scratch_base_lo12659; GFX1250-SDAG-NEXT: s_cmp_lg_u64 s[0:1], 012660; GFX1250-SDAG-NEXT: s_wait_xcnt 0x012661; GFX1250-SDAG-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[0:1]12662; GFX1250-SDAG-NEXT: s_cselect_b32 s0, s2, -112663; GFX1250-SDAG-NEXT: scratch_load_b64 v[2:3], off, s012664; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x012665; GFX1250-SDAG-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]12666; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)12667; GFX1250-SDAG-NEXT: v_min_num_f64_e32 v[0:1], v[2:3], v[0:1]12668; GFX1250-SDAG-NEXT: scratch_store_b64 off, v[0:1], s012669; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]12670;12671; GFX1250-GISEL-LABEL: flat_atomic_fmin_f64_saddr_nortn:12672; GFX1250-GISEL: ; %bb.0:12673; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x012674; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x012675; GFX1250-GISEL-NEXT: s_add_co_u32 s2, s0, 0x5012676; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s3, s1, 012677; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)12678; GFX1250-GISEL-NEXT: s_xor_b32 s4, s3, src_flat_scratch_base_hi12679; GFX1250-GISEL-NEXT: s_cmp_ge_u32 s4, 0x400000012680; GFX1250-GISEL-NEXT: s_mov_b32 s4, 112681; GFX1250-GISEL-NEXT: s_cbranch_scc0 .LBB115_212682; GFX1250-GISEL-NEXT: ; %bb.1: ; %atomicrmw.global12683; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 012684; GFX1250-GISEL-NEXT: s_mov_b32 s4, 012685; GFX1250-GISEL-NEXT: s_wait_storecnt 0x012686; GFX1250-GISEL-NEXT: flat_atomic_min_num_f64 v2, v[0:1], s[0:1] offset:8012687; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x012688; GFX1250-GISEL-NEXT: .LBB115_2: ; %Flow12689; GFX1250-GISEL-NEXT: s_wait_xcnt 0x012690; GFX1250-GISEL-NEXT: s_xor_b32 s0, s4, 112691; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)12692; GFX1250-GISEL-NEXT: s_cmp_lg_u32 s0, 012693; GFX1250-GISEL-NEXT: s_cbranch_scc1 .LBB115_412694; GFX1250-GISEL-NEXT: ; %bb.3: ; %atomicrmw.private12695; GFX1250-GISEL-NEXT: s_sub_co_i32 s0, s2, src_flat_scratch_base_lo12696; GFX1250-GISEL-NEXT: s_cmp_lg_u64 s[2:3], 012697; GFX1250-GISEL-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[0:1]12698; GFX1250-GISEL-NEXT: s_cselect_b32 s0, s0, -112699; GFX1250-GISEL-NEXT: scratch_load_b64 v[2:3], off, s012700; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x012701; GFX1250-GISEL-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]12702; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)12703; GFX1250-GISEL-NEXT: v_min_num_f64_e32 v[0:1], v[2:3], v[0:1]12704; GFX1250-GISEL-NEXT: scratch_store_b64 off, v[0:1], s012705; GFX1250-GISEL-NEXT: .LBB115_4: ; %atomicrmw.phi12706; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]12707;12708; GFX950-SDAG-LABEL: flat_atomic_fmin_f64_saddr_nortn:12709; GFX950-SDAG: ; %bb.0:12710; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12711; GFX950-SDAG-NEXT: s_add_u32 s0, s0, 0x5012712; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], src_private_base12713; GFX950-SDAG-NEXT: s_addc_u32 s1, s1, 012714; GFX950-SDAG-NEXT: s_cmp_eq_u32 s1, s312715; GFX950-SDAG-NEXT: s_cselect_b64 s[2:3], -1, 012716; GFX950-SDAG-NEXT: s_andn2_b64 vcc, exec, s[2:3]12717; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], -112718; GFX950-SDAG-NEXT: s_cbranch_vccnz .LBB115_312719; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow12720; GFX950-SDAG-NEXT: s_andn2_b64 vcc, exec, s[2:3]12721; GFX950-SDAG-NEXT: s_cbranch_vccz .LBB115_412722; GFX950-SDAG-NEXT: .LBB115_2: ; %atomicrmw.phi12723; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)12724; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]12725; GFX950-SDAG-NEXT: .LBB115_3: ; %atomicrmw.global12726; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[0:1]12727; GFX950-SDAG-NEXT: flat_atomic_min_f64 v[2:3], v[0:1]12728; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)12729; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB115_212730; GFX950-SDAG-NEXT: .LBB115_4: ; %atomicrmw.private12731; GFX950-SDAG-NEXT: s_cmp_lg_u64 s[0:1], 012732; GFX950-SDAG-NEXT: s_cselect_b32 s0, s0, -112733; GFX950-SDAG-NEXT: scratch_load_dwordx2 v[2:3], off, s012734; GFX950-SDAG-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]12735; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)12736; GFX950-SDAG-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]12737; GFX950-SDAG-NEXT: v_min_f64 v[0:1], v[2:3], v[0:1]12738; GFX950-SDAG-NEXT: scratch_store_dwordx2 off, v[0:1], s012739; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)12740; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]12741;12742; GFX950-GISEL-LABEL: flat_atomic_fmin_f64_saddr_nortn:12743; GFX950-GISEL: ; %bb.0:12744; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12745; GFX950-GISEL-NEXT: s_add_u32 s2, s0, 0x5012746; GFX950-GISEL-NEXT: s_mov_b64 s[4:5], src_private_base12747; GFX950-GISEL-NEXT: s_addc_u32 s3, s1, 012748; GFX950-GISEL-NEXT: s_cmp_lg_u32 s3, s512749; GFX950-GISEL-NEXT: s_mov_b32 s4, 112750; GFX950-GISEL-NEXT: s_cbranch_scc0 .LBB115_212751; GFX950-GISEL-NEXT: ; %bb.1: ; %atomicrmw.global12752; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]12753; GFX950-GISEL-NEXT: flat_atomic_min_f64 v[2:3], v[0:1] offset:8012754; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)12755; GFX950-GISEL-NEXT: s_mov_b32 s4, 012756; GFX950-GISEL-NEXT: .LBB115_2: ; %Flow12757; GFX950-GISEL-NEXT: s_xor_b32 s0, s4, 112758; GFX950-GISEL-NEXT: s_cmp_lg_u32 s0, 012759; GFX950-GISEL-NEXT: s_cbranch_scc1 .LBB115_412760; GFX950-GISEL-NEXT: ; %bb.3: ; %atomicrmw.private12761; GFX950-GISEL-NEXT: s_cmp_lg_u64 s[2:3], 012762; GFX950-GISEL-NEXT: s_cselect_b32 s0, s2, -112763; GFX950-GISEL-NEXT: scratch_load_dwordx2 v[2:3], off, s012764; GFX950-GISEL-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]12765; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)12766; GFX950-GISEL-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]12767; GFX950-GISEL-NEXT: v_min_f64 v[0:1], v[2:3], v[0:1]12768; GFX950-GISEL-NEXT: scratch_store_dwordx2 off, v[0:1], s012769; GFX950-GISEL-NEXT: .LBB115_4: ; %atomicrmw.phi12770; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0)12771; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]12772 %gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 1012773 %unused = atomicrmw fmin ptr %gep.0, double %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !012774 ret void12775}12776 12777define float @flat_atomic_fadd_f32_saddr_rtn(ptr inreg %ptr, float %data) {12778; GFX1250-LABEL: flat_atomic_fadd_f32_saddr_rtn:12779; GFX1250: ; %bb.0:12780; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x012781; GFX1250-NEXT: s_wait_kmcnt 0x012782; GFX1250-NEXT: v_mov_b32_e32 v1, 012783; GFX1250-NEXT: s_wait_storecnt 0x012784; GFX1250-NEXT: flat_atomic_add_f32 v0, v1, v0, s[0:1] offset:40 th:TH_ATOMIC_RETURN12785; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x012786; GFX1250-NEXT: s_set_pc_i64 s[30:31]12787;12788; GFX950-SDAG-LABEL: flat_atomic_fadd_f32_saddr_rtn:12789; GFX950-SDAG: ; %bb.0:12790; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12791; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s012792; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s112793; GFX950-SDAG-NEXT: flat_atomic_add_f32 v0, v[2:3], v0 offset:40 sc012794; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12795; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]12796;12797; GFX950-GISEL-LABEL: flat_atomic_fadd_f32_saddr_rtn:12798; GFX950-GISEL: ; %bb.0:12799; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12800; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]12801; GFX950-GISEL-NEXT: flat_atomic_add_f32 v0, v[2:3], v0 offset:40 sc012802; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12803; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]12804 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 1012805 %result = atomicrmw fadd ptr %gep.0, float %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !012806 ret float %result12807}12808 12809define void @flat_atomic_fadd_f32_saddr_nortn(ptr inreg %ptr, float %data) {12810; GFX1250-LABEL: flat_atomic_fadd_f32_saddr_nortn:12811; GFX1250: ; %bb.0:12812; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x012813; GFX1250-NEXT: s_wait_kmcnt 0x012814; GFX1250-NEXT: v_mov_b32_e32 v1, 012815; GFX1250-NEXT: s_wait_storecnt 0x012816; GFX1250-NEXT: flat_atomic_add_f32 v1, v0, s[0:1] offset:4012817; GFX1250-NEXT: s_wait_storecnt_dscnt 0x012818; GFX1250-NEXT: s_set_pc_i64 s[30:31]12819;12820; GFX950-SDAG-LABEL: flat_atomic_fadd_f32_saddr_nortn:12821; GFX950-SDAG: ; %bb.0:12822; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12823; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s012824; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s112825; GFX950-SDAG-NEXT: flat_atomic_add_f32 v[2:3], v0 offset:4012826; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12827; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]12828;12829; GFX950-GISEL-LABEL: flat_atomic_fadd_f32_saddr_nortn:12830; GFX950-GISEL: ; %bb.0:12831; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12832; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]12833; GFX950-GISEL-NEXT: flat_atomic_add_f32 v[2:3], v0 offset:4012834; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12835; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]12836 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 1012837 %unused = atomicrmw fadd ptr %gep.0, float %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !012838 ret void12839}12840 12841define float @flat_atomic_fmax_f32_saddr_rtn(ptr inreg %ptr, float %data) {12842; GFX1250-LABEL: flat_atomic_fmax_f32_saddr_rtn:12843; GFX1250: ; %bb.0:12844; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x012845; GFX1250-NEXT: s_wait_kmcnt 0x012846; GFX1250-NEXT: v_mov_b32_e32 v1, 012847; GFX1250-NEXT: s_wait_storecnt 0x012848; GFX1250-NEXT: flat_atomic_max_num_f32 v0, v1, v0, s[0:1] offset:40 th:TH_ATOMIC_RETURN12849; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x012850; GFX1250-NEXT: s_set_pc_i64 s[30:31]12851;12852; GFX950-SDAG-LABEL: flat_atomic_fmax_f32_saddr_rtn:12853; GFX950-SDAG: ; %bb.0:12854; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12855; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[0:1]12856; GFX950-SDAG-NEXT: flat_load_dword v1, v[2:3] offset:4012857; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], 012858; GFX950-SDAG-NEXT: v_max_f32_e32 v4, v0, v012859; GFX950-SDAG-NEXT: .LBB118_1: ; %atomicrmw.start12860; GFX950-SDAG-NEXT: ; =>This Inner Loop Header: Depth=112861; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12862; GFX950-SDAG-NEXT: v_max_f32_e32 v0, v1, v112863; GFX950-SDAG-NEXT: v_max_f32_e32 v0, v0, v412864; GFX950-SDAG-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc012865; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12866; GFX950-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v112867; GFX950-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]12868; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, v012869; GFX950-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]12870; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB118_112871; GFX950-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end12872; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]12873; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]12874;12875; GFX950-GISEL-LABEL: flat_atomic_fmax_f32_saddr_rtn:12876; GFX950-GISEL: ; %bb.0:12877; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12878; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]12879; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, v012880; GFX950-GISEL-NEXT: flat_load_dword v0, v[2:3] offset:4012881; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], 012882; GFX950-GISEL-NEXT: v_max_f32_e32 v1, v1, v112883; GFX950-GISEL-NEXT: .LBB118_1: ; %atomicrmw.start12884; GFX950-GISEL-NEXT: ; =>This Inner Loop Header: Depth=112885; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12886; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v012887; GFX950-GISEL-NEXT: v_max_f32_e32 v0, v5, v512888; GFX950-GISEL-NEXT: v_max_f32_e32 v4, v0, v112889; GFX950-GISEL-NEXT: flat_atomic_cmpswap v0, v[2:3], v[4:5] offset:40 sc012890; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12891; GFX950-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v512892; GFX950-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]12893; GFX950-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]12894; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB118_112895; GFX950-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end12896; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]12897; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]12898 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 1012899 %result = atomicrmw fmax ptr %gep.0, float %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !012900 ret float %result12901}12902 12903define void @flat_atomic_fmax_f32_saddr_nortn(ptr inreg %ptr, float %data) {12904; GFX1250-LABEL: flat_atomic_fmax_f32_saddr_nortn:12905; GFX1250: ; %bb.0:12906; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x012907; GFX1250-NEXT: s_wait_kmcnt 0x012908; GFX1250-NEXT: v_mov_b32_e32 v1, 012909; GFX1250-NEXT: s_wait_storecnt 0x012910; GFX1250-NEXT: flat_atomic_max_num_f32 v1, v0, s[0:1] offset:4012911; GFX1250-NEXT: s_wait_storecnt_dscnt 0x012912; GFX1250-NEXT: s_set_pc_i64 s[30:31]12913;12914; GFX950-SDAG-LABEL: flat_atomic_fmax_f32_saddr_nortn:12915; GFX950-SDAG: ; %bb.0:12916; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12917; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[0:1]12918; GFX950-SDAG-NEXT: flat_load_dword v1, v[2:3] offset:4012919; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], 012920; GFX950-SDAG-NEXT: v_max_f32_e32 v4, v0, v012921; GFX950-SDAG-NEXT: .LBB119_1: ; %atomicrmw.start12922; GFX950-SDAG-NEXT: ; =>This Inner Loop Header: Depth=112923; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12924; GFX950-SDAG-NEXT: v_max_f32_e32 v0, v1, v112925; GFX950-SDAG-NEXT: v_max_f32_e32 v0, v0, v412926; GFX950-SDAG-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc012927; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12928; GFX950-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v112929; GFX950-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]12930; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, v012931; GFX950-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]12932; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB119_112933; GFX950-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end12934; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]12935; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]12936;12937; GFX950-GISEL-LABEL: flat_atomic_fmax_f32_saddr_nortn:12938; GFX950-GISEL: ; %bb.0:12939; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12940; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]12941; GFX950-GISEL-NEXT: flat_load_dword v1, v[2:3] offset:4012942; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], 012943; GFX950-GISEL-NEXT: v_max_f32_e32 v4, v0, v012944; GFX950-GISEL-NEXT: .LBB119_1: ; %atomicrmw.start12945; GFX950-GISEL-NEXT: ; =>This Inner Loop Header: Depth=112946; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12947; GFX950-GISEL-NEXT: v_max_f32_e32 v0, v1, v112948; GFX950-GISEL-NEXT: v_max_f32_e32 v0, v0, v412949; GFX950-GISEL-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc012950; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12951; GFX950-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v112952; GFX950-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]12953; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, v012954; GFX950-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]12955; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB119_112956; GFX950-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end12957; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]12958; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]12959 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 1012960 %unused = atomicrmw fmax ptr %gep.0, float %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !012961 ret void12962}12963 12964define float @flat_atomic_fmin_f32_saddr_rtn(ptr inreg %ptr, float %data) {12965; GFX1250-LABEL: flat_atomic_fmin_f32_saddr_rtn:12966; GFX1250: ; %bb.0:12967; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x012968; GFX1250-NEXT: s_wait_kmcnt 0x012969; GFX1250-NEXT: v_mov_b32_e32 v1, 012970; GFX1250-NEXT: s_wait_storecnt 0x012971; GFX1250-NEXT: flat_atomic_min_num_f32 v0, v1, v0, s[0:1] offset:40 th:TH_ATOMIC_RETURN12972; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x012973; GFX1250-NEXT: s_set_pc_i64 s[30:31]12974;12975; GFX950-SDAG-LABEL: flat_atomic_fmin_f32_saddr_rtn:12976; GFX950-SDAG: ; %bb.0:12977; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12978; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[0:1]12979; GFX950-SDAG-NEXT: flat_load_dword v1, v[2:3] offset:4012980; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], 012981; GFX950-SDAG-NEXT: v_max_f32_e32 v4, v0, v012982; GFX950-SDAG-NEXT: .LBB120_1: ; %atomicrmw.start12983; GFX950-SDAG-NEXT: ; =>This Inner Loop Header: Depth=112984; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12985; GFX950-SDAG-NEXT: v_max_f32_e32 v0, v1, v112986; GFX950-SDAG-NEXT: v_min_f32_e32 v0, v0, v412987; GFX950-SDAG-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc012988; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)12989; GFX950-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v112990; GFX950-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]12991; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, v012992; GFX950-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]12993; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB120_112994; GFX950-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end12995; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]12996; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]12997;12998; GFX950-GISEL-LABEL: flat_atomic_fmin_f32_saddr_rtn:12999; GFX950-GISEL: ; %bb.0:13000; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13001; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]13002; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, v013003; GFX950-GISEL-NEXT: flat_load_dword v0, v[2:3] offset:4013004; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], 013005; GFX950-GISEL-NEXT: v_max_f32_e32 v1, v1, v113006; GFX950-GISEL-NEXT: .LBB120_1: ; %atomicrmw.start13007; GFX950-GISEL-NEXT: ; =>This Inner Loop Header: Depth=113008; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13009; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v013010; GFX950-GISEL-NEXT: v_max_f32_e32 v0, v5, v513011; GFX950-GISEL-NEXT: v_min_f32_e32 v4, v0, v113012; GFX950-GISEL-NEXT: flat_atomic_cmpswap v0, v[2:3], v[4:5] offset:40 sc013013; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13014; GFX950-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v513015; GFX950-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]13016; GFX950-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]13017; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB120_113018; GFX950-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end13019; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]13020; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]13021 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 1013022 %result = atomicrmw fmin ptr %gep.0, float %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !013023 ret float %result13024}13025 13026define void @flat_atomic_fmin_f32_saddr_nortn(ptr inreg %ptr, float %data) {13027; GFX1250-LABEL: flat_atomic_fmin_f32_saddr_nortn:13028; GFX1250: ; %bb.0:13029; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013030; GFX1250-NEXT: s_wait_kmcnt 0x013031; GFX1250-NEXT: v_mov_b32_e32 v1, 013032; GFX1250-NEXT: s_wait_storecnt 0x013033; GFX1250-NEXT: flat_atomic_min_num_f32 v1, v0, s[0:1] offset:4013034; GFX1250-NEXT: s_wait_storecnt_dscnt 0x013035; GFX1250-NEXT: s_set_pc_i64 s[30:31]13036;13037; GFX950-SDAG-LABEL: flat_atomic_fmin_f32_saddr_nortn:13038; GFX950-SDAG: ; %bb.0:13039; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13040; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[0:1]13041; GFX950-SDAG-NEXT: flat_load_dword v1, v[2:3] offset:4013042; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], 013043; GFX950-SDAG-NEXT: v_max_f32_e32 v4, v0, v013044; GFX950-SDAG-NEXT: .LBB121_1: ; %atomicrmw.start13045; GFX950-SDAG-NEXT: ; =>This Inner Loop Header: Depth=113046; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13047; GFX950-SDAG-NEXT: v_max_f32_e32 v0, v1, v113048; GFX950-SDAG-NEXT: v_min_f32_e32 v0, v0, v413049; GFX950-SDAG-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc013050; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13051; GFX950-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v113052; GFX950-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]13053; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, v013054; GFX950-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]13055; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB121_113056; GFX950-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end13057; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]13058; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]13059;13060; GFX950-GISEL-LABEL: flat_atomic_fmin_f32_saddr_nortn:13061; GFX950-GISEL: ; %bb.0:13062; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13063; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]13064; GFX950-GISEL-NEXT: flat_load_dword v1, v[2:3] offset:4013065; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], 013066; GFX950-GISEL-NEXT: v_max_f32_e32 v4, v0, v013067; GFX950-GISEL-NEXT: .LBB121_1: ; %atomicrmw.start13068; GFX950-GISEL-NEXT: ; =>This Inner Loop Header: Depth=113069; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13070; GFX950-GISEL-NEXT: v_max_f32_e32 v0, v1, v113071; GFX950-GISEL-NEXT: v_min_f32_e32 v0, v0, v413072; GFX950-GISEL-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc013073; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13074; GFX950-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v113075; GFX950-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]13076; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, v013077; GFX950-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]13078; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB121_113079; GFX950-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end13080; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]13081; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]13082 %gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 1013083 %unused = atomicrmw fmin ptr %gep.0, float %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !013084 ret void13085}13086 13087define <2 x half> @flat_atomic_fadd_v2f16_saddr_rtn(ptr inreg %ptr, <2 x half> %data) {13088; GFX1250-LABEL: flat_atomic_fadd_v2f16_saddr_rtn:13089; GFX1250: ; %bb.0:13090; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013091; GFX1250-NEXT: s_wait_kmcnt 0x013092; GFX1250-NEXT: v_mov_b32_e32 v1, 013093; GFX1250-NEXT: s_wait_storecnt 0x013094; GFX1250-NEXT: flat_atomic_pk_add_f16 v0, v1, v0, s[0:1] offset:40 th:TH_ATOMIC_RETURN13095; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013096; GFX1250-NEXT: s_set_pc_i64 s[30:31]13097;13098; GFX950-SDAG-LABEL: flat_atomic_fadd_v2f16_saddr_rtn:13099; GFX950-SDAG: ; %bb.0:13100; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13101; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s013102; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s113103; GFX950-SDAG-NEXT: flat_atomic_pk_add_f16 v0, v[2:3], v0 offset:40 sc013104; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13105; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]13106;13107; GFX950-GISEL-LABEL: flat_atomic_fadd_v2f16_saddr_rtn:13108; GFX950-GISEL: ; %bb.0:13109; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13110; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]13111; GFX950-GISEL-NEXT: flat_atomic_pk_add_f16 v0, v[2:3], v0 offset:40 sc013112; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13113; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]13114 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1013115 %result = atomicrmw fadd ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !013116 ret <2 x half> %result13117}13118 13119define void @flat_atomic_fadd_v2f16_saddr_nortn(ptr inreg %ptr, <2 x half> %data) {13120; GFX1250-LABEL: flat_atomic_fadd_v2f16_saddr_nortn:13121; GFX1250: ; %bb.0:13122; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013123; GFX1250-NEXT: s_wait_kmcnt 0x013124; GFX1250-NEXT: v_mov_b32_e32 v1, 013125; GFX1250-NEXT: s_wait_storecnt 0x013126; GFX1250-NEXT: flat_atomic_pk_add_f16 v1, v0, s[0:1] offset:4013127; GFX1250-NEXT: s_wait_storecnt_dscnt 0x013128; GFX1250-NEXT: s_set_pc_i64 s[30:31]13129;13130; GFX950-SDAG-LABEL: flat_atomic_fadd_v2f16_saddr_nortn:13131; GFX950-SDAG: ; %bb.0:13132; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13133; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s013134; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s113135; GFX950-SDAG-NEXT: flat_atomic_pk_add_f16 v[2:3], v0 offset:4013136; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13137; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]13138;13139; GFX950-GISEL-LABEL: flat_atomic_fadd_v2f16_saddr_nortn:13140; GFX950-GISEL: ; %bb.0:13141; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13142; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]13143; GFX950-GISEL-NEXT: flat_atomic_pk_add_f16 v[2:3], v0 offset:4013144; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13145; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]13146 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1013147 %unused = atomicrmw fadd ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !013148 ret void13149}13150 13151define <2 x half> @flat_atomic_fmax_v2f16_saddr_rtn(ptr inreg %ptr, <2 x half> %data) {13152; GFX1250-LABEL: flat_atomic_fmax_v2f16_saddr_rtn:13153; GFX1250: ; %bb.0:13154; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013155; GFX1250-NEXT: s_wait_kmcnt 0x013156; GFX1250-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, 013157; GFX1250-NEXT: s_mov_b32 s2, 013158; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)13159; GFX1250-NEXT: v_pk_max_num_f16 v1, v1, v113160; GFX1250-NEXT: flat_load_b32 v0, v2, s[0:1] offset:4013161; GFX1250-NEXT: .LBB124_1: ; %atomicrmw.start13162; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=113163; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013164; GFX1250-NEXT: v_mov_b32_e32 v5, v013165; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)13166; GFX1250-NEXT: v_pk_max_num_f16 v0, v5, v513167; GFX1250-NEXT: v_pk_max_num_f16 v4, v0, v113168; GFX1250-NEXT: s_wait_storecnt 0x013169; GFX1250-NEXT: s_wait_xcnt 0x013170; GFX1250-NEXT: flat_atomic_cmpswap_b32 v0, v2, v[4:5], s[0:1] offset:40 th:TH_ATOMIC_RETURN13171; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013172; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v513173; GFX1250-NEXT: s_or_b32 s2, vcc_lo, s213174; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)13175; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s213176; GFX1250-NEXT: s_cbranch_execnz .LBB124_113177; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end13178; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s213179; GFX1250-NEXT: s_set_pc_i64 s[30:31]13180;13181; GFX950-SDAG-LABEL: flat_atomic_fmax_v2f16_saddr_rtn:13182; GFX950-SDAG: ; %bb.0:13183; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13184; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[0:1]13185; GFX950-SDAG-NEXT: flat_load_dword v1, v[2:3] offset:4013186; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], 013187; GFX950-SDAG-NEXT: v_pk_max_f16 v4, v0, v013188; GFX950-SDAG-NEXT: .LBB124_1: ; %atomicrmw.start13189; GFX950-SDAG-NEXT: ; =>This Inner Loop Header: Depth=113190; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13191; GFX950-SDAG-NEXT: v_pk_max_f16 v0, v1, v113192; GFX950-SDAG-NEXT: s_nop 013193; GFX950-SDAG-NEXT: v_pk_max_f16 v0, v0, v413194; GFX950-SDAG-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc013195; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13196; GFX950-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v113197; GFX950-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]13198; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, v013199; GFX950-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]13200; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB124_113201; GFX950-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end13202; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]13203; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]13204;13205; GFX950-GISEL-LABEL: flat_atomic_fmax_v2f16_saddr_rtn:13206; GFX950-GISEL: ; %bb.0:13207; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13208; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]13209; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, v013210; GFX950-GISEL-NEXT: flat_load_dword v0, v[2:3] offset:4013211; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], 013212; GFX950-GISEL-NEXT: v_pk_max_f16 v1, v1, v113213; GFX950-GISEL-NEXT: .LBB124_1: ; %atomicrmw.start13214; GFX950-GISEL-NEXT: ; =>This Inner Loop Header: Depth=113215; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13216; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v013217; GFX950-GISEL-NEXT: v_pk_max_f16 v0, v5, v513218; GFX950-GISEL-NEXT: s_nop 013219; GFX950-GISEL-NEXT: v_pk_max_f16 v4, v0, v113220; GFX950-GISEL-NEXT: flat_atomic_cmpswap v0, v[2:3], v[4:5] offset:40 sc013221; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13222; GFX950-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v513223; GFX950-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]13224; GFX950-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]13225; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB124_113226; GFX950-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end13227; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]13228; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]13229 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1013230 %result = atomicrmw fmax ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !013231 ret <2 x half> %result13232}13233 13234define void @flat_atomic_fmax_v2f16_saddr_nortn(ptr inreg %ptr, <2 x half> %data) {13235; GFX1250-LABEL: flat_atomic_fmax_v2f16_saddr_nortn:13236; GFX1250: ; %bb.0:13237; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013238; GFX1250-NEXT: s_wait_kmcnt 0x013239; GFX1250-NEXT: v_mov_b32_e32 v2, 013240; GFX1250-NEXT: v_pk_max_num_f16 v3, v0, v013241; GFX1250-NEXT: s_mov_b32 s2, 013242; GFX1250-NEXT: flat_load_b32 v1, v2, s[0:1] offset:4013243; GFX1250-NEXT: .LBB125_1: ; %atomicrmw.start13244; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=113245; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013246; GFX1250-NEXT: v_pk_max_num_f16 v0, v1, v113247; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)13248; GFX1250-NEXT: v_pk_max_num_f16 v0, v0, v313249; GFX1250-NEXT: s_wait_storecnt 0x013250; GFX1250-NEXT: s_wait_xcnt 0x013251; GFX1250-NEXT: flat_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:40 th:TH_ATOMIC_RETURN13252; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013253; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v113254; GFX1250-NEXT: v_mov_b32_e32 v1, v013255; GFX1250-NEXT: s_or_b32 s2, vcc_lo, s213256; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)13257; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s213258; GFX1250-NEXT: s_cbranch_execnz .LBB125_113259; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end13260; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s213261; GFX1250-NEXT: s_set_pc_i64 s[30:31]13262;13263; GFX950-SDAG-LABEL: flat_atomic_fmax_v2f16_saddr_nortn:13264; GFX950-SDAG: ; %bb.0:13265; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13266; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[0:1]13267; GFX950-SDAG-NEXT: flat_load_dword v1, v[2:3] offset:4013268; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], 013269; GFX950-SDAG-NEXT: v_pk_max_f16 v4, v0, v013270; GFX950-SDAG-NEXT: .LBB125_1: ; %atomicrmw.start13271; GFX950-SDAG-NEXT: ; =>This Inner Loop Header: Depth=113272; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13273; GFX950-SDAG-NEXT: v_pk_max_f16 v0, v1, v113274; GFX950-SDAG-NEXT: s_nop 013275; GFX950-SDAG-NEXT: v_pk_max_f16 v0, v0, v413276; GFX950-SDAG-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc013277; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13278; GFX950-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v113279; GFX950-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]13280; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, v013281; GFX950-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]13282; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB125_113283; GFX950-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end13284; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]13285; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]13286;13287; GFX950-GISEL-LABEL: flat_atomic_fmax_v2f16_saddr_nortn:13288; GFX950-GISEL: ; %bb.0:13289; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13290; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]13291; GFX950-GISEL-NEXT: flat_load_dword v1, v[2:3] offset:4013292; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], 013293; GFX950-GISEL-NEXT: v_pk_max_f16 v4, v0, v013294; GFX950-GISEL-NEXT: .LBB125_1: ; %atomicrmw.start13295; GFX950-GISEL-NEXT: ; =>This Inner Loop Header: Depth=113296; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13297; GFX950-GISEL-NEXT: v_pk_max_f16 v0, v1, v113298; GFX950-GISEL-NEXT: s_nop 013299; GFX950-GISEL-NEXT: v_pk_max_f16 v0, v0, v413300; GFX950-GISEL-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc013301; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13302; GFX950-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v113303; GFX950-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]13304; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, v013305; GFX950-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]13306; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB125_113307; GFX950-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end13308; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]13309; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]13310 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1013311 %unused = atomicrmw fmax ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !013312 ret void13313}13314 13315define <2 x half> @flat_atomic_fmin_v2f16_saddr_rtn(ptr inreg %ptr, <2 x half> %data) {13316; GFX1250-LABEL: flat_atomic_fmin_v2f16_saddr_rtn:13317; GFX1250: ; %bb.0:13318; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013319; GFX1250-NEXT: s_wait_kmcnt 0x013320; GFX1250-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, 013321; GFX1250-NEXT: s_mov_b32 s2, 013322; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)13323; GFX1250-NEXT: v_pk_max_num_f16 v1, v1, v113324; GFX1250-NEXT: flat_load_b32 v0, v2, s[0:1] offset:4013325; GFX1250-NEXT: .LBB126_1: ; %atomicrmw.start13326; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=113327; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013328; GFX1250-NEXT: v_mov_b32_e32 v5, v013329; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)13330; GFX1250-NEXT: v_pk_max_num_f16 v0, v5, v513331; GFX1250-NEXT: v_pk_min_num_f16 v4, v0, v113332; GFX1250-NEXT: s_wait_storecnt 0x013333; GFX1250-NEXT: s_wait_xcnt 0x013334; GFX1250-NEXT: flat_atomic_cmpswap_b32 v0, v2, v[4:5], s[0:1] offset:40 th:TH_ATOMIC_RETURN13335; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013336; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v513337; GFX1250-NEXT: s_or_b32 s2, vcc_lo, s213338; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)13339; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s213340; GFX1250-NEXT: s_cbranch_execnz .LBB126_113341; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end13342; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s213343; GFX1250-NEXT: s_set_pc_i64 s[30:31]13344;13345; GFX950-SDAG-LABEL: flat_atomic_fmin_v2f16_saddr_rtn:13346; GFX950-SDAG: ; %bb.0:13347; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13348; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[0:1]13349; GFX950-SDAG-NEXT: flat_load_dword v1, v[2:3] offset:4013350; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], 013351; GFX950-SDAG-NEXT: v_pk_max_f16 v4, v0, v013352; GFX950-SDAG-NEXT: .LBB126_1: ; %atomicrmw.start13353; GFX950-SDAG-NEXT: ; =>This Inner Loop Header: Depth=113354; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13355; GFX950-SDAG-NEXT: v_pk_max_f16 v0, v1, v113356; GFX950-SDAG-NEXT: s_nop 013357; GFX950-SDAG-NEXT: v_pk_min_f16 v0, v0, v413358; GFX950-SDAG-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc013359; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13360; GFX950-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v113361; GFX950-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]13362; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, v013363; GFX950-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]13364; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB126_113365; GFX950-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end13366; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]13367; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]13368;13369; GFX950-GISEL-LABEL: flat_atomic_fmin_v2f16_saddr_rtn:13370; GFX950-GISEL: ; %bb.0:13371; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13372; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]13373; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, v013374; GFX950-GISEL-NEXT: flat_load_dword v0, v[2:3] offset:4013375; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], 013376; GFX950-GISEL-NEXT: v_pk_max_f16 v1, v1, v113377; GFX950-GISEL-NEXT: .LBB126_1: ; %atomicrmw.start13378; GFX950-GISEL-NEXT: ; =>This Inner Loop Header: Depth=113379; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13380; GFX950-GISEL-NEXT: v_mov_b32_e32 v5, v013381; GFX950-GISEL-NEXT: v_pk_max_f16 v0, v5, v513382; GFX950-GISEL-NEXT: s_nop 013383; GFX950-GISEL-NEXT: v_pk_min_f16 v4, v0, v113384; GFX950-GISEL-NEXT: flat_atomic_cmpswap v0, v[2:3], v[4:5] offset:40 sc013385; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13386; GFX950-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v513387; GFX950-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]13388; GFX950-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]13389; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB126_113390; GFX950-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end13391; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]13392; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]13393 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1013394 %result = atomicrmw fmin ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !013395 ret <2 x half> %result13396}13397 13398define void @flat_atomic_fmin_v2f16_saddr_nortn(ptr inreg %ptr, <2 x half> %data) {13399; GFX1250-LABEL: flat_atomic_fmin_v2f16_saddr_nortn:13400; GFX1250: ; %bb.0:13401; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013402; GFX1250-NEXT: s_wait_kmcnt 0x013403; GFX1250-NEXT: v_mov_b32_e32 v2, 013404; GFX1250-NEXT: v_pk_max_num_f16 v3, v0, v013405; GFX1250-NEXT: s_mov_b32 s2, 013406; GFX1250-NEXT: flat_load_b32 v1, v2, s[0:1] offset:4013407; GFX1250-NEXT: .LBB127_1: ; %atomicrmw.start13408; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=113409; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013410; GFX1250-NEXT: v_pk_max_num_f16 v0, v1, v113411; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)13412; GFX1250-NEXT: v_pk_min_num_f16 v0, v0, v313413; GFX1250-NEXT: s_wait_storecnt 0x013414; GFX1250-NEXT: s_wait_xcnt 0x013415; GFX1250-NEXT: flat_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:40 th:TH_ATOMIC_RETURN13416; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013417; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v113418; GFX1250-NEXT: v_mov_b32_e32 v1, v013419; GFX1250-NEXT: s_or_b32 s2, vcc_lo, s213420; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)13421; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s213422; GFX1250-NEXT: s_cbranch_execnz .LBB127_113423; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end13424; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s213425; GFX1250-NEXT: s_set_pc_i64 s[30:31]13426;13427; GFX950-SDAG-LABEL: flat_atomic_fmin_v2f16_saddr_nortn:13428; GFX950-SDAG: ; %bb.0:13429; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13430; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[0:1]13431; GFX950-SDAG-NEXT: flat_load_dword v1, v[2:3] offset:4013432; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], 013433; GFX950-SDAG-NEXT: v_pk_max_f16 v4, v0, v013434; GFX950-SDAG-NEXT: .LBB127_1: ; %atomicrmw.start13435; GFX950-SDAG-NEXT: ; =>This Inner Loop Header: Depth=113436; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13437; GFX950-SDAG-NEXT: v_pk_max_f16 v0, v1, v113438; GFX950-SDAG-NEXT: s_nop 013439; GFX950-SDAG-NEXT: v_pk_min_f16 v0, v0, v413440; GFX950-SDAG-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc013441; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13442; GFX950-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v113443; GFX950-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]13444; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, v013445; GFX950-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]13446; GFX950-SDAG-NEXT: s_cbranch_execnz .LBB127_113447; GFX950-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end13448; GFX950-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]13449; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]13450;13451; GFX950-GISEL-LABEL: flat_atomic_fmin_v2f16_saddr_nortn:13452; GFX950-GISEL: ; %bb.0:13453; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13454; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]13455; GFX950-GISEL-NEXT: flat_load_dword v1, v[2:3] offset:4013456; GFX950-GISEL-NEXT: s_mov_b64 s[0:1], 013457; GFX950-GISEL-NEXT: v_pk_max_f16 v4, v0, v013458; GFX950-GISEL-NEXT: .LBB127_1: ; %atomicrmw.start13459; GFX950-GISEL-NEXT: ; =>This Inner Loop Header: Depth=113460; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13461; GFX950-GISEL-NEXT: v_pk_max_f16 v0, v1, v113462; GFX950-GISEL-NEXT: s_nop 013463; GFX950-GISEL-NEXT: v_pk_min_f16 v0, v0, v413464; GFX950-GISEL-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc013465; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13466; GFX950-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v113467; GFX950-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]13468; GFX950-GISEL-NEXT: v_mov_b32_e32 v1, v013469; GFX950-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]13470; GFX950-GISEL-NEXT: s_cbranch_execnz .LBB127_113471; GFX950-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end13472; GFX950-GISEL-NEXT: s_or_b64 exec, exec, s[0:1]13473; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]13474 %gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 1013475 %unused = atomicrmw fmin ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !013476 ret void13477}13478 13479define <2 x bfloat> @flat_atomic_fadd_v2bf16_saddr_rtn(ptr inreg %ptr, <2 x bfloat> %data) {13480; GFX1250-LABEL: flat_atomic_fadd_v2bf16_saddr_rtn:13481; GFX1250: ; %bb.0:13482; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013483; GFX1250-NEXT: s_wait_kmcnt 0x013484; GFX1250-NEXT: v_mov_b32_e32 v1, 013485; GFX1250-NEXT: s_wait_storecnt 0x013486; GFX1250-NEXT: flat_atomic_pk_add_bf16 v0, v1, v0, s[0:1] offset:40 th:TH_ATOMIC_RETURN13487; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013488; GFX1250-NEXT: s_set_pc_i64 s[30:31]13489;13490; GFX950-LABEL: flat_atomic_fadd_v2bf16_saddr_rtn:13491; GFX950: ; %bb.0:13492; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13493; GFX950-NEXT: v_mov_b32_e32 v2, s013494; GFX950-NEXT: v_mov_b32_e32 v3, s113495; GFX950-NEXT: flat_atomic_pk_add_bf16 v0, v[2:3], v0 offset:40 sc013496; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13497; GFX950-NEXT: s_setpc_b64 s[30:31]13498 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1013499 %result = atomicrmw fadd ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !013500 ret <2 x bfloat> %result13501}13502 13503define void @flat_atomic_fadd_v2bf16_saddr_nortn(ptr inreg %ptr, <2 x bfloat> %data) {13504; GFX1250-LABEL: flat_atomic_fadd_v2bf16_saddr_nortn:13505; GFX1250: ; %bb.0:13506; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013507; GFX1250-NEXT: s_wait_kmcnt 0x013508; GFX1250-NEXT: v_mov_b32_e32 v1, 013509; GFX1250-NEXT: s_wait_storecnt 0x013510; GFX1250-NEXT: flat_atomic_pk_add_bf16 v1, v0, s[0:1] offset:4013511; GFX1250-NEXT: s_wait_storecnt_dscnt 0x013512; GFX1250-NEXT: s_set_pc_i64 s[30:31]13513;13514; GFX950-LABEL: flat_atomic_fadd_v2bf16_saddr_nortn:13515; GFX950: ; %bb.0:13516; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13517; GFX950-NEXT: v_mov_b32_e32 v2, s013518; GFX950-NEXT: v_mov_b32_e32 v3, s113519; GFX950-NEXT: flat_atomic_pk_add_bf16 v[2:3], v0 offset:4013520; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13521; GFX950-NEXT: s_setpc_b64 s[30:31]13522 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1013523 %unused = atomicrmw fadd ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !013524 ret void13525}13526 13527define <2 x bfloat> @flat_atomic_fmax_v2bf16_saddr_rtn(ptr inreg %ptr, <2 x bfloat> %data) {13528; GFX1250-LABEL: flat_atomic_fmax_v2bf16_saddr_rtn:13529; GFX1250: ; %bb.0:13530; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013531; GFX1250-NEXT: s_wait_kmcnt 0x013532; GFX1250-NEXT: v_mov_b32_e32 v2, 013533; GFX1250-NEXT: s_mov_b32 s2, 013534; GFX1250-NEXT: flat_load_b32 v1, v2, s[0:1] offset:4013535; GFX1250-NEXT: .LBB130_1: ; %atomicrmw.start13536; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=113537; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013538; GFX1250-NEXT: v_mov_b32_e32 v5, v113539; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)13540; GFX1250-NEXT: v_pk_max_num_bf16 v4, v5, v013541; GFX1250-NEXT: s_wait_storecnt 0x013542; GFX1250-NEXT: s_wait_xcnt 0x013543; GFX1250-NEXT: flat_atomic_cmpswap_b32 v1, v2, v[4:5], s[0:1] offset:40 th:TH_ATOMIC_RETURN13544; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013545; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v513546; GFX1250-NEXT: s_or_b32 s2, vcc_lo, s213547; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)13548; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s213549; GFX1250-NEXT: s_cbranch_execnz .LBB130_113550; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end13551; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s213552; GFX1250-NEXT: v_mov_b32_e32 v0, v113553; GFX1250-NEXT: s_set_pc_i64 s[30:31]13554;13555; GFX950-LABEL: flat_atomic_fmax_v2bf16_saddr_rtn:13556; GFX950: ; %bb.0:13557; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13558; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]13559; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:4013560; GFX950-NEXT: s_mov_b64 s[2:3], 013561; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v013562; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v013563; GFX950-NEXT: .LBB130_1: ; %atomicrmw.start13564; GFX950-NEXT: ; =>This Inner Loop Header: Depth=113565; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13566; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v113567; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v113568; GFX950-NEXT: v_max_f32_e32 v0, v0, v413569; GFX950-NEXT: v_max_f32_e32 v6, v6, v513570; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v013571; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc013572; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13573; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v113574; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]13575; GFX950-NEXT: v_mov_b32_e32 v1, v013576; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]13577; GFX950-NEXT: s_cbranch_execnz .LBB130_113578; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end13579; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]13580; GFX950-NEXT: s_setpc_b64 s[30:31]13581 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1013582 %result = atomicrmw fmax ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !013583 ret <2 x bfloat> %result13584}13585 13586define void @flat_atomic_fmax_v2bf16_saddr_nortn(ptr inreg %ptr, <2 x bfloat> %data) {13587; GFX1250-LABEL: flat_atomic_fmax_v2bf16_saddr_nortn:13588; GFX1250: ; %bb.0:13589; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013590; GFX1250-NEXT: s_wait_kmcnt 0x013591; GFX1250-NEXT: v_mov_b32_e32 v1, 013592; GFX1250-NEXT: s_mov_b32 s2, 013593; GFX1250-NEXT: flat_load_b32 v3, v1, s[0:1] offset:4013594; GFX1250-NEXT: .LBB131_1: ; %atomicrmw.start13595; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=113596; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013597; GFX1250-NEXT: v_pk_max_num_bf16 v2, v3, v013598; GFX1250-NEXT: s_wait_storecnt 0x013599; GFX1250-NEXT: s_wait_xcnt 0x013600; GFX1250-NEXT: flat_atomic_cmpswap_b32 v2, v1, v[2:3], s[0:1] offset:40 th:TH_ATOMIC_RETURN13601; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013602; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v313603; GFX1250-NEXT: v_mov_b32_e32 v3, v213604; GFX1250-NEXT: s_or_b32 s2, vcc_lo, s213605; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)13606; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s213607; GFX1250-NEXT: s_cbranch_execnz .LBB131_113608; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end13609; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s213610; GFX1250-NEXT: s_set_pc_i64 s[30:31]13611;13612; GFX950-LABEL: flat_atomic_fmax_v2bf16_saddr_nortn:13613; GFX950: ; %bb.0:13614; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13615; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]13616; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:4013617; GFX950-NEXT: s_mov_b64 s[2:3], 013618; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v013619; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v013620; GFX950-NEXT: .LBB131_1: ; %atomicrmw.start13621; GFX950-NEXT: ; =>This Inner Loop Header: Depth=113622; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13623; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v113624; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v113625; GFX950-NEXT: v_max_f32_e32 v0, v0, v413626; GFX950-NEXT: v_max_f32_e32 v6, v6, v513627; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v013628; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc013629; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13630; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v113631; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]13632; GFX950-NEXT: v_mov_b32_e32 v1, v013633; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]13634; GFX950-NEXT: s_cbranch_execnz .LBB131_113635; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end13636; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]13637; GFX950-NEXT: s_setpc_b64 s[30:31]13638 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1013639 %unused = atomicrmw fmax ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !013640 ret void13641}13642 13643define <2 x bfloat> @flat_atomic_fmin_v2bf16_saddr_rtn(ptr inreg %ptr, <2 x bfloat> %data) {13644; GFX1250-LABEL: flat_atomic_fmin_v2bf16_saddr_rtn:13645; GFX1250: ; %bb.0:13646; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013647; GFX1250-NEXT: s_wait_kmcnt 0x013648; GFX1250-NEXT: v_mov_b32_e32 v2, 013649; GFX1250-NEXT: s_mov_b32 s2, 013650; GFX1250-NEXT: flat_load_b32 v1, v2, s[0:1] offset:4013651; GFX1250-NEXT: .LBB132_1: ; %atomicrmw.start13652; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=113653; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013654; GFX1250-NEXT: v_mov_b32_e32 v5, v113655; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)13656; GFX1250-NEXT: v_pk_min_num_bf16 v4, v5, v013657; GFX1250-NEXT: s_wait_storecnt 0x013658; GFX1250-NEXT: s_wait_xcnt 0x013659; GFX1250-NEXT: flat_atomic_cmpswap_b32 v1, v2, v[4:5], s[0:1] offset:40 th:TH_ATOMIC_RETURN13660; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013661; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v513662; GFX1250-NEXT: s_or_b32 s2, vcc_lo, s213663; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)13664; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s213665; GFX1250-NEXT: s_cbranch_execnz .LBB132_113666; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end13667; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s213668; GFX1250-NEXT: v_mov_b32_e32 v0, v113669; GFX1250-NEXT: s_set_pc_i64 s[30:31]13670;13671; GFX950-LABEL: flat_atomic_fmin_v2bf16_saddr_rtn:13672; GFX950: ; %bb.0:13673; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13674; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]13675; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:4013676; GFX950-NEXT: s_mov_b64 s[2:3], 013677; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v013678; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v013679; GFX950-NEXT: .LBB132_1: ; %atomicrmw.start13680; GFX950-NEXT: ; =>This Inner Loop Header: Depth=113681; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13682; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v113683; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v113684; GFX950-NEXT: v_min_f32_e32 v0, v0, v413685; GFX950-NEXT: v_min_f32_e32 v6, v6, v513686; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v013687; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc013688; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13689; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v113690; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]13691; GFX950-NEXT: v_mov_b32_e32 v1, v013692; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]13693; GFX950-NEXT: s_cbranch_execnz .LBB132_113694; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end13695; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]13696; GFX950-NEXT: s_setpc_b64 s[30:31]13697 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1013698 %result = atomicrmw fmin ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !013699 ret <2 x bfloat> %result13700}13701 13702define void @flat_atomic_fmin_v2bf16_saddr_nortn(ptr inreg %ptr, <2 x bfloat> %data) {13703; GFX1250-LABEL: flat_atomic_fmin_v2bf16_saddr_nortn:13704; GFX1250: ; %bb.0:13705; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013706; GFX1250-NEXT: s_wait_kmcnt 0x013707; GFX1250-NEXT: v_mov_b32_e32 v1, 013708; GFX1250-NEXT: s_mov_b32 s2, 013709; GFX1250-NEXT: flat_load_b32 v3, v1, s[0:1] offset:4013710; GFX1250-NEXT: .LBB133_1: ; %atomicrmw.start13711; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=113712; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013713; GFX1250-NEXT: v_pk_min_num_bf16 v2, v3, v013714; GFX1250-NEXT: s_wait_storecnt 0x013715; GFX1250-NEXT: s_wait_xcnt 0x013716; GFX1250-NEXT: flat_atomic_cmpswap_b32 v2, v1, v[2:3], s[0:1] offset:40 th:TH_ATOMIC_RETURN13717; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x013718; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v313719; GFX1250-NEXT: v_mov_b32_e32 v3, v213720; GFX1250-NEXT: s_or_b32 s2, vcc_lo, s213721; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)13722; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s213723; GFX1250-NEXT: s_cbranch_execnz .LBB133_113724; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end13725; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s213726; GFX1250-NEXT: s_set_pc_i64 s[30:31]13727;13728; GFX950-LABEL: flat_atomic_fmin_v2bf16_saddr_nortn:13729; GFX950: ; %bb.0:13730; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)13731; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]13732; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:4013733; GFX950-NEXT: s_mov_b64 s[2:3], 013734; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v013735; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v013736; GFX950-NEXT: .LBB133_1: ; %atomicrmw.start13737; GFX950-NEXT: ; =>This Inner Loop Header: Depth=113738; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13739; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v113740; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v113741; GFX950-NEXT: v_min_f32_e32 v0, v0, v413742; GFX950-NEXT: v_min_f32_e32 v6, v6, v513743; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v013744; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc013745; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)13746; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v113747; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]13748; GFX950-NEXT: v_mov_b32_e32 v1, v013749; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]13750; GFX950-NEXT: s_cbranch_execnz .LBB133_113751; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end13752; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]13753; GFX950-NEXT: s_setpc_b64 s[30:31]13754 %gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 1013755 %unused = atomicrmw fmin ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !013756 ret void13757}13758 13759attributes #0 = { argmemonly nounwind willreturn }13760 13761!0 = !{}13762