14269 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX7LESS,GFX7LESS_ITERATIVE %s3; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX8,GFX8_ITERATIVE %s4; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9_ITERATIVE %s5; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX1064,GFX1064_ITERATIVE %s6; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize32 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX1032,GFX1032_ITERATIVE %s7; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -mattr=+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX1164,GFX1164-TRUE16,GFX1164_ITERATIVE,GFX1164_ITERATIVE-TRUE16 %s8; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -mattr=+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX1164,GFX1164-FAKE16,GFX1164_ITERATIVE,GFX1164_ITERATIVE-FAKE16 %s9; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -mattr=+wavefrontsize32 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX1132,GFX1132-TRUE16,GFX1132_ITERATIVE,GFX1132_ITERATIVE-TRUE16 %s10; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -mattr=+wavefrontsize32 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX1132,GFX1132-FAKE16,GFX1132_ITERATIVE,GFX1132_ITERATIVE-FAKE16 %s11; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -mattr=+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX1264,GFX1264-TRUE16,GFX1264_ITERATIVE,GFX1264_ITERATIVE-TRUE16 %s12; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -mattr=+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX1264,GFX1264-FAKE16,GFX1264_ITERATIVE,GFX1264_ITERATIVE-FAKE16 %s13; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -mattr=+wavefrontsize32 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX1232,GFX1232-TRUE16,GFX1232_ITERATIVE,GFX1232_ITERATIVE-TRUE16 %s14; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -mattr=+wavefrontsize32 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX1232,GFX1232-FAKE16,GFX1232_ITERATIVE,GFX1232_ITERATIVE-FAKE16 %s15; RUN: llc -mtriple=amdgcn -amdgpu-atomic-optimizer-strategy=DPP < %s | FileCheck -enable-var-scope -check-prefixes=GFX7LESS,GFX7LESS_DPP %s16; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=DPP < %s | FileCheck -enable-var-scope -check-prefixes=GFX8,GFX8_DPP %s17; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=DPP < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9_DPP %s18; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=DPP < %s | FileCheck -enable-var-scope -check-prefixes=GFX1064,GFX1064_DPP %s19; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize32 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=DPP < %s | FileCheck -enable-var-scope -check-prefixes=GFX1032,GFX1032_DPP %s20; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -mattr=+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=DPP < %s | FileCheck -enable-var-scope -check-prefixes=GFX1164,GFX1164-TRUE16,GFX1164_DPP,GFX1164_DPP-TRUE16 %s21; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -mattr=+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=DPP < %s | FileCheck -enable-var-scope -check-prefixes=GFX1164,GFX1164-FAKE16,GFX1164_DPP,GFX1164_DPP-FAKE16 %s22; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -mattr=+wavefrontsize32 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=DPP < %s | FileCheck -enable-var-scope -check-prefixes=GFX1132,GFX1132-TRUE16,GFX1132_DPP,GFX1132_DPP-TRUE16 %s23; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -mattr=+wavefrontsize32 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=DPP < %s | FileCheck -enable-var-scope -check-prefixes=GFX1132,GFX1132-FAKE16,GFX1132_DPP,GFX1132_DPP-FAKE16 %s24; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -mattr=+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=DPP < %s | FileCheck -enable-var-scope -check-prefixes=GFX1264,GFX1264-TRUE16,GFX1264_DPP,GFX1264_DPP-TRUE16 %s25; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -mattr=+wavefrontsize64 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=DPP < %s | FileCheck -enable-var-scope -check-prefixes=GFX1264,GFX1264-FAKE16,GFX1264_DPP,GFX1264_DPP-FAKE16 %s26; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -mattr=+wavefrontsize32 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=DPP < %s | FileCheck -enable-var-scope -check-prefixes=GFX1232,GFX1232-TRUE16,GFX1232_DPP,GFX1232_DPP-TRUE16 %s27; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -mattr=+wavefrontsize32 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=DPP < %s | FileCheck -enable-var-scope -check-prefixes=GFX1232,GFX1232-FAKE16,GFX1232_DPP,GFX1232_DPP-FAKE16 %s28 29declare i32 @llvm.amdgcn.workitem.id.x()30 31; Show what the atomic optimization pass will do for global pointers.32 33define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace(1) %inout) {34; GFX7LESS-LABEL: add_i32_constant:35; GFX7LESS: ; %bb.0: ; %entry36; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec37; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x938; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 039; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v040; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v041; GFX7LESS-NEXT: ; implicit-def: $vgpr142; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc43; GFX7LESS-NEXT: s_cbranch_execz .LBB0_244; GFX7LESS-NEXT: ; %bb.1:45; GFX7LESS-NEXT: s_mov_b32 s11, 0xf00046; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]47; GFX7LESS-NEXT: s_mul_i32 s6, s6, 548; GFX7LESS-NEXT: s_mov_b32 s10, -149; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)50; GFX7LESS-NEXT: s_mov_b32 s8, s251; GFX7LESS-NEXT: s_mov_b32 s9, s352; GFX7LESS-NEXT: v_mov_b32_e32 v1, s653; GFX7LESS-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc54; GFX7LESS-NEXT: s_waitcnt vmcnt(0)55; GFX7LESS-NEXT: buffer_wbinvl156; GFX7LESS-NEXT: .LBB0_2:57; GFX7LESS-NEXT: s_or_b64 exec, exec, s[4:5]58; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)59; GFX7LESS-NEXT: s_mov_b32 s3, 0xf00060; GFX7LESS-NEXT: s_mov_b32 s2, -161; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v162; GFX7LESS-NEXT: v_mad_u32_u24 v0, v0, 5, s463; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 064; GFX7LESS-NEXT: s_endpgm65;66; GFX8-LABEL: add_i32_constant:67; GFX8: ; %bb.0: ; %entry68; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2469; GFX8-NEXT: s_mov_b64 s[6:7], exec70; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 071; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v072; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v073; GFX8-NEXT: ; implicit-def: $vgpr174; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc75; GFX8-NEXT: s_cbranch_execz .LBB0_276; GFX8-NEXT: ; %bb.1:77; GFX8-NEXT: s_waitcnt lgkmcnt(0)78; GFX8-NEXT: s_mov_b32 s8, s279; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[6:7]80; GFX8-NEXT: s_mul_i32 s2, s2, 581; GFX8-NEXT: s_mov_b32 s11, 0xf00082; GFX8-NEXT: s_mov_b32 s10, -183; GFX8-NEXT: s_mov_b32 s9, s384; GFX8-NEXT: v_mov_b32_e32 v1, s285; GFX8-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc86; GFX8-NEXT: s_waitcnt vmcnt(0)87; GFX8-NEXT: buffer_wbinvl1_vol88; GFX8-NEXT: .LBB0_2:89; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]90; GFX8-NEXT: v_readfirstlane_b32 s4, v191; GFX8-NEXT: s_waitcnt lgkmcnt(0)92; GFX8-NEXT: s_mov_b32 s3, 0xf00093; GFX8-NEXT: s_mov_b32 s2, -194; GFX8-NEXT: v_mad_u32_u24 v0, v0, 5, s495; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 096; GFX8-NEXT: s_endpgm97;98; GFX9-LABEL: add_i32_constant:99; GFX9: ; %bb.0: ; %entry100; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24101; GFX9-NEXT: s_mov_b64 s[6:7], exec102; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0103; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0104; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0105; GFX9-NEXT: ; implicit-def: $vgpr1106; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc107; GFX9-NEXT: s_cbranch_execz .LBB0_2108; GFX9-NEXT: ; %bb.1:109; GFX9-NEXT: s_waitcnt lgkmcnt(0)110; GFX9-NEXT: s_mov_b32 s8, s2111; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[6:7]112; GFX9-NEXT: s_mul_i32 s2, s2, 5113; GFX9-NEXT: s_mov_b32 s11, 0xf000114; GFX9-NEXT: s_mov_b32 s10, -1115; GFX9-NEXT: s_mov_b32 s9, s3116; GFX9-NEXT: v_mov_b32_e32 v1, s2117; GFX9-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc118; GFX9-NEXT: s_waitcnt vmcnt(0)119; GFX9-NEXT: buffer_wbinvl1_vol120; GFX9-NEXT: .LBB0_2:121; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]122; GFX9-NEXT: v_readfirstlane_b32 s4, v1123; GFX9-NEXT: s_waitcnt lgkmcnt(0)124; GFX9-NEXT: s_mov_b32 s3, 0xf000125; GFX9-NEXT: s_mov_b32 s2, -1126; GFX9-NEXT: v_mad_u32_u24 v0, v0, 5, s4127; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0128; GFX9-NEXT: s_endpgm129;130; GFX1064-LABEL: add_i32_constant:131; GFX1064: ; %bb.0: ; %entry132; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24133; GFX1064-NEXT: s_mov_b64 s[6:7], exec134; GFX1064-NEXT: ; implicit-def: $vgpr1135; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0136; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0137; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0138; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc139; GFX1064-NEXT: s_cbranch_execz .LBB0_2140; GFX1064-NEXT: ; %bb.1:141; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]142; GFX1064-NEXT: s_mov_b32 s11, 0x31016000143; GFX1064-NEXT: s_mul_i32 s6, s6, 5144; GFX1064-NEXT: s_mov_b32 s10, -1145; GFX1064-NEXT: v_mov_b32_e32 v1, s6146; GFX1064-NEXT: s_waitcnt lgkmcnt(0)147; GFX1064-NEXT: s_mov_b32 s8, s2148; GFX1064-NEXT: s_mov_b32 s9, s3149; GFX1064-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc150; GFX1064-NEXT: s_waitcnt vmcnt(0)151; GFX1064-NEXT: buffer_gl1_inv152; GFX1064-NEXT: buffer_gl0_inv153; GFX1064-NEXT: .LBB0_2:154; GFX1064-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)155; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5]156; GFX1064-NEXT: s_waitcnt lgkmcnt(0)157; GFX1064-NEXT: v_readfirstlane_b32 s2, v1158; GFX1064-NEXT: s_mov_b32 s3, 0x31016000159; GFX1064-NEXT: v_mad_u32_u24 v0, v0, 5, s2160; GFX1064-NEXT: s_mov_b32 s2, -1161; GFX1064-NEXT: buffer_store_dword v0, off, s[0:3], 0162; GFX1064-NEXT: s_endpgm163;164; GFX1032-LABEL: add_i32_constant:165; GFX1032: ; %bb.0: ; %entry166; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24167; GFX1032-NEXT: s_mov_b32 s6, exec_lo168; GFX1032-NEXT: ; implicit-def: $vgpr1169; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0170; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0171; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo172; GFX1032-NEXT: s_cbranch_execz .LBB0_2173; GFX1032-NEXT: ; %bb.1:174; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s6175; GFX1032-NEXT: s_mov_b32 s11, 0x31016000176; GFX1032-NEXT: s_mul_i32 s5, s5, 5177; GFX1032-NEXT: s_mov_b32 s10, -1178; GFX1032-NEXT: v_mov_b32_e32 v1, s5179; GFX1032-NEXT: s_waitcnt lgkmcnt(0)180; GFX1032-NEXT: s_mov_b32 s8, s2181; GFX1032-NEXT: s_mov_b32 s9, s3182; GFX1032-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc183; GFX1032-NEXT: s_waitcnt vmcnt(0)184; GFX1032-NEXT: buffer_gl1_inv185; GFX1032-NEXT: buffer_gl0_inv186; GFX1032-NEXT: .LBB0_2:187; GFX1032-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)188; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4189; GFX1032-NEXT: s_waitcnt lgkmcnt(0)190; GFX1032-NEXT: v_readfirstlane_b32 s2, v1191; GFX1032-NEXT: s_mov_b32 s3, 0x31016000192; GFX1032-NEXT: v_mad_u32_u24 v0, v0, 5, s2193; GFX1032-NEXT: s_mov_b32 s2, -1194; GFX1032-NEXT: buffer_store_dword v0, off, s[0:3], 0195; GFX1032-NEXT: s_endpgm196;197; GFX1164-LABEL: add_i32_constant:198; GFX1164: ; %bb.0: ; %entry199; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24200; GFX1164-NEXT: s_mov_b64 s[6:7], exec201; GFX1164-NEXT: s_mov_b64 s[4:5], exec202; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0203; GFX1164-NEXT: ; implicit-def: $vgpr1204; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)205; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0206; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0207; GFX1164-NEXT: s_cbranch_execz .LBB0_2208; GFX1164-NEXT: ; %bb.1:209; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]210; GFX1164-NEXT: s_mov_b32 s11, 0x31016000211; GFX1164-NEXT: s_mul_i32 s6, s6, 5212; GFX1164-NEXT: s_mov_b32 s10, -1213; GFX1164-NEXT: v_mov_b32_e32 v1, s6214; GFX1164-NEXT: s_waitcnt lgkmcnt(0)215; GFX1164-NEXT: s_mov_b32 s8, s2216; GFX1164-NEXT: s_mov_b32 s9, s3217; GFX1164-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc218; GFX1164-NEXT: s_waitcnt vmcnt(0)219; GFX1164-NEXT: buffer_gl1_inv220; GFX1164-NEXT: buffer_gl0_inv221; GFX1164-NEXT: .LBB0_2:222; GFX1164-NEXT: s_or_b64 exec, exec, s[4:5]223; GFX1164-NEXT: s_waitcnt lgkmcnt(0)224; GFX1164-NEXT: v_readfirstlane_b32 s2, v1225; GFX1164-NEXT: s_mov_b32 s3, 0x31016000226; GFX1164-NEXT: v_mad_u32_u24 v0, v0, 5, s2227; GFX1164-NEXT: s_mov_b32 s2, -1228; GFX1164-NEXT: buffer_store_b32 v0, off, s[0:3], 0229; GFX1164-NEXT: s_endpgm230;231; GFX1132-LABEL: add_i32_constant:232; GFX1132: ; %bb.0: ; %entry233; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24234; GFX1132-NEXT: s_mov_b32 s6, exec_lo235; GFX1132-NEXT: s_mov_b32 s4, exec_lo236; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0237; GFX1132-NEXT: ; implicit-def: $vgpr1238; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)239; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0240; GFX1132-NEXT: s_cbranch_execz .LBB0_2241; GFX1132-NEXT: ; %bb.1:242; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s6243; GFX1132-NEXT: s_mov_b32 s11, 0x31016000244; GFX1132-NEXT: s_mul_i32 s5, s5, 5245; GFX1132-NEXT: s_mov_b32 s10, -1246; GFX1132-NEXT: v_mov_b32_e32 v1, s5247; GFX1132-NEXT: s_waitcnt lgkmcnt(0)248; GFX1132-NEXT: s_mov_b32 s8, s2249; GFX1132-NEXT: s_mov_b32 s9, s3250; GFX1132-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc251; GFX1132-NEXT: s_waitcnt vmcnt(0)252; GFX1132-NEXT: buffer_gl1_inv253; GFX1132-NEXT: buffer_gl0_inv254; GFX1132-NEXT: .LBB0_2:255; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s4256; GFX1132-NEXT: s_waitcnt lgkmcnt(0)257; GFX1132-NEXT: v_readfirstlane_b32 s2, v1258; GFX1132-NEXT: s_mov_b32 s3, 0x31016000259; GFX1132-NEXT: v_mad_u32_u24 v0, v0, 5, s2260; GFX1132-NEXT: s_mov_b32 s2, -1261; GFX1132-NEXT: buffer_store_b32 v0, off, s[0:3], 0262; GFX1132-NEXT: s_endpgm263;264; GFX1264-LABEL: add_i32_constant:265; GFX1264: ; %bb.0: ; %entry266; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24267; GFX1264-NEXT: s_mov_b64 s[6:7], exec268; GFX1264-NEXT: s_mov_b64 s[4:5], exec269; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0270; GFX1264-NEXT: ; implicit-def: $vgpr1271; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)272; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0273; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v0274; GFX1264-NEXT: s_cbranch_execz .LBB0_2275; GFX1264-NEXT: ; %bb.1:276; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[6:7]277; GFX1264-NEXT: s_mov_b32 s11, 0x31016000278; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)279; GFX1264-NEXT: s_mul_i32 s6, s6, 5280; GFX1264-NEXT: s_mov_b32 s10, -1281; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)282; GFX1264-NEXT: v_mov_b32_e32 v1, s6283; GFX1264-NEXT: s_wait_kmcnt 0x0284; GFX1264-NEXT: s_mov_b32 s8, s2285; GFX1264-NEXT: s_mov_b32 s9, s3286; GFX1264-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV287; GFX1264-NEXT: s_wait_loadcnt 0x0288; GFX1264-NEXT: global_inv scope:SCOPE_DEV289; GFX1264-NEXT: .LBB0_2:290; GFX1264-NEXT: s_or_b64 exec, exec, s[4:5]291; GFX1264-NEXT: s_wait_kmcnt 0x0292; GFX1264-NEXT: v_readfirstlane_b32 s2, v1293; GFX1264-NEXT: s_mov_b32 s3, 0x31016000294; GFX1264-NEXT: v_mad_u32_u24 v0, v0, 5, s2295; GFX1264-NEXT: s_mov_b32 s2, -1296; GFX1264-NEXT: buffer_store_b32 v0, off, s[0:3], null297; GFX1264-NEXT: s_endpgm298;299; GFX1232-LABEL: add_i32_constant:300; GFX1232: ; %bb.0: ; %entry301; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24302; GFX1232-NEXT: s_mov_b32 s6, exec_lo303; GFX1232-NEXT: s_mov_b32 s4, exec_lo304; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0305; GFX1232-NEXT: ; implicit-def: $vgpr1306; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)307; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v0308; GFX1232-NEXT: s_cbranch_execz .LBB0_2309; GFX1232-NEXT: ; %bb.1:310; GFX1232-NEXT: s_bcnt1_i32_b32 s5, s6311; GFX1232-NEXT: s_mov_b32 s11, 0x31016000312; GFX1232-NEXT: s_mul_i32 s5, s5, 5313; GFX1232-NEXT: s_mov_b32 s10, -1314; GFX1232-NEXT: v_mov_b32_e32 v1, s5315; GFX1232-NEXT: s_wait_kmcnt 0x0316; GFX1232-NEXT: s_mov_b32 s8, s2317; GFX1232-NEXT: s_mov_b32 s9, s3318; GFX1232-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV319; GFX1232-NEXT: s_wait_loadcnt 0x0320; GFX1232-NEXT: global_inv scope:SCOPE_DEV321; GFX1232-NEXT: .LBB0_2:322; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s4323; GFX1232-NEXT: s_wait_kmcnt 0x0324; GFX1232-NEXT: v_readfirstlane_b32 s2, v1325; GFX1232-NEXT: s_mov_b32 s3, 0x31016000326; GFX1232-NEXT: v_mad_u32_u24 v0, v0, 5, s2327; GFX1232-NEXT: s_mov_b32 s2, -1328; GFX1232-NEXT: buffer_store_b32 v0, off, s[0:3], null329; GFX1232-NEXT: s_endpgm330entry:331 %old = atomicrmw add ptr addrspace(1) %inout, i32 5 syncscope("agent") acq_rel332 store i32 %old, ptr addrspace(1) %out333 ret void334}335 336define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(1) %inout, i32 %additive) {337; GFX7LESS-LABEL: add_i32_uniform:338; GFX7LESS: ; %bb.0: ; %entry339; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec340; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9341; GFX7LESS-NEXT: s_load_dword s8, s[4:5], 0xd342; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0343; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0344; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0345; GFX7LESS-NEXT: ; implicit-def: $vgpr1346; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc347; GFX7LESS-NEXT: s_cbranch_execz .LBB1_2348; GFX7LESS-NEXT: ; %bb.1:349; GFX7LESS-NEXT: s_mov_b32 s15, 0xf000350; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]351; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)352; GFX7LESS-NEXT: s_mul_i32 s6, s8, s6353; GFX7LESS-NEXT: s_mov_b32 s14, -1354; GFX7LESS-NEXT: s_mov_b32 s12, s2355; GFX7LESS-NEXT: s_mov_b32 s13, s3356; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6357; GFX7LESS-NEXT: buffer_atomic_add v1, off, s[12:15], 0 glc358; GFX7LESS-NEXT: s_waitcnt vmcnt(0)359; GFX7LESS-NEXT: buffer_wbinvl1360; GFX7LESS-NEXT: .LBB1_2:361; GFX7LESS-NEXT: s_or_b64 exec, exec, s[4:5]362; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)363; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000364; GFX7LESS-NEXT: s_mov_b32 s2, -1365; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v1366; GFX7LESS-NEXT: v_mul_lo_u32 v0, s8, v0367; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s4, v0368; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 0369; GFX7LESS-NEXT: s_endpgm370;371; GFX8-LABEL: add_i32_uniform:372; GFX8: ; %bb.0: ; %entry373; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24374; GFX8-NEXT: s_load_dword s8, s[4:5], 0x34375; GFX8-NEXT: s_mov_b64 s[6:7], exec376; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0377; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0378; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0379; GFX8-NEXT: ; implicit-def: $vgpr1380; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc381; GFX8-NEXT: s_cbranch_execz .LBB1_2382; GFX8-NEXT: ; %bb.1:383; GFX8-NEXT: s_waitcnt lgkmcnt(0)384; GFX8-NEXT: s_mov_b32 s12, s2385; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[6:7]386; GFX8-NEXT: s_mul_i32 s2, s8, s2387; GFX8-NEXT: s_mov_b32 s15, 0xf000388; GFX8-NEXT: s_mov_b32 s14, -1389; GFX8-NEXT: s_mov_b32 s13, s3390; GFX8-NEXT: v_mov_b32_e32 v1, s2391; GFX8-NEXT: buffer_atomic_add v1, off, s[12:15], 0 glc392; GFX8-NEXT: s_waitcnt vmcnt(0)393; GFX8-NEXT: buffer_wbinvl1_vol394; GFX8-NEXT: .LBB1_2:395; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]396; GFX8-NEXT: s_waitcnt lgkmcnt(0)397; GFX8-NEXT: v_mul_lo_u32 v0, s8, v0398; GFX8-NEXT: v_readfirstlane_b32 s4, v1399; GFX8-NEXT: s_mov_b32 s3, 0xf000400; GFX8-NEXT: s_mov_b32 s2, -1401; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v0402; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0403; GFX8-NEXT: s_endpgm404;405; GFX9-LABEL: add_i32_uniform:406; GFX9: ; %bb.0: ; %entry407; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24408; GFX9-NEXT: s_load_dword s8, s[4:5], 0x34409; GFX9-NEXT: s_mov_b64 s[6:7], exec410; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0411; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0412; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0413; GFX9-NEXT: ; implicit-def: $vgpr1414; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc415; GFX9-NEXT: s_cbranch_execz .LBB1_2416; GFX9-NEXT: ; %bb.1:417; GFX9-NEXT: s_waitcnt lgkmcnt(0)418; GFX9-NEXT: s_mov_b32 s12, s2419; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[6:7]420; GFX9-NEXT: s_mul_i32 s2, s8, s2421; GFX9-NEXT: s_mov_b32 s15, 0xf000422; GFX9-NEXT: s_mov_b32 s14, -1423; GFX9-NEXT: s_mov_b32 s13, s3424; GFX9-NEXT: v_mov_b32_e32 v1, s2425; GFX9-NEXT: buffer_atomic_add v1, off, s[12:15], 0 glc426; GFX9-NEXT: s_waitcnt vmcnt(0)427; GFX9-NEXT: buffer_wbinvl1_vol428; GFX9-NEXT: .LBB1_2:429; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]430; GFX9-NEXT: s_waitcnt lgkmcnt(0)431; GFX9-NEXT: v_mul_lo_u32 v0, s8, v0432; GFX9-NEXT: v_readfirstlane_b32 s4, v1433; GFX9-NEXT: s_mov_b32 s3, 0xf000434; GFX9-NEXT: s_mov_b32 s2, -1435; GFX9-NEXT: v_add_u32_e32 v0, s4, v0436; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0437; GFX9-NEXT: s_endpgm438;439; GFX1064-LABEL: add_i32_uniform:440; GFX1064: ; %bb.0: ; %entry441; GFX1064-NEXT: s_clause 0x1442; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24443; GFX1064-NEXT: s_load_dword s8, s[4:5], 0x34444; GFX1064-NEXT: s_mov_b64 s[6:7], exec445; GFX1064-NEXT: ; implicit-def: $vgpr1446; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0447; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0448; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0449; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc450; GFX1064-NEXT: s_cbranch_execz .LBB1_2451; GFX1064-NEXT: ; %bb.1:452; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]453; GFX1064-NEXT: s_mov_b32 s15, 0x31016000454; GFX1064-NEXT: s_waitcnt lgkmcnt(0)455; GFX1064-NEXT: s_mul_i32 s6, s8, s6456; GFX1064-NEXT: s_mov_b32 s14, -1457; GFX1064-NEXT: v_mov_b32_e32 v1, s6458; GFX1064-NEXT: s_mov_b32 s12, s2459; GFX1064-NEXT: s_mov_b32 s13, s3460; GFX1064-NEXT: buffer_atomic_add v1, off, s[12:15], 0 glc461; GFX1064-NEXT: s_waitcnt vmcnt(0)462; GFX1064-NEXT: buffer_gl1_inv463; GFX1064-NEXT: buffer_gl0_inv464; GFX1064-NEXT: .LBB1_2:465; GFX1064-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)466; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5]467; GFX1064-NEXT: s_waitcnt lgkmcnt(0)468; GFX1064-NEXT: v_readfirstlane_b32 s2, v1469; GFX1064-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s8, v0, s[2:3]470; GFX1064-NEXT: s_mov_b32 s3, 0x31016000471; GFX1064-NEXT: s_mov_b32 s2, -1472; GFX1064-NEXT: buffer_store_dword v0, off, s[0:3], 0473; GFX1064-NEXT: s_endpgm474;475; GFX1032-LABEL: add_i32_uniform:476; GFX1032: ; %bb.0: ; %entry477; GFX1032-NEXT: s_clause 0x1478; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24479; GFX1032-NEXT: s_load_dword s6, s[4:5], 0x34480; GFX1032-NEXT: s_mov_b32 s7, exec_lo481; GFX1032-NEXT: ; implicit-def: $vgpr1482; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s7, 0483; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0484; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo485; GFX1032-NEXT: s_cbranch_execz .LBB1_2486; GFX1032-NEXT: ; %bb.1:487; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s7488; GFX1032-NEXT: s_mov_b32 s11, 0x31016000489; GFX1032-NEXT: s_waitcnt lgkmcnt(0)490; GFX1032-NEXT: s_mul_i32 s5, s6, s5491; GFX1032-NEXT: s_mov_b32 s10, -1492; GFX1032-NEXT: v_mov_b32_e32 v1, s5493; GFX1032-NEXT: s_mov_b32 s8, s2494; GFX1032-NEXT: s_mov_b32 s9, s3495; GFX1032-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc496; GFX1032-NEXT: s_waitcnt vmcnt(0)497; GFX1032-NEXT: buffer_gl1_inv498; GFX1032-NEXT: buffer_gl0_inv499; GFX1032-NEXT: .LBB1_2:500; GFX1032-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)501; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4502; GFX1032-NEXT: s_waitcnt lgkmcnt(0)503; GFX1032-NEXT: v_readfirstlane_b32 s2, v1504; GFX1032-NEXT: v_mad_u64_u32 v[0:1], s2, s6, v0, s[2:3]505; GFX1032-NEXT: s_mov_b32 s3, 0x31016000506; GFX1032-NEXT: s_mov_b32 s2, -1507; GFX1032-NEXT: buffer_store_dword v0, off, s[0:3], 0508; GFX1032-NEXT: s_endpgm509;510; GFX1164-LABEL: add_i32_uniform:511; GFX1164: ; %bb.0: ; %entry512; GFX1164-NEXT: s_clause 0x1513; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24514; GFX1164-NEXT: s_load_b32 s8, s[4:5], 0x34515; GFX1164-NEXT: s_mov_b64 s[6:7], exec516; GFX1164-NEXT: s_mov_b64 s[4:5], exec517; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0518; GFX1164-NEXT: ; implicit-def: $vgpr1519; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)520; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0521; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0522; GFX1164-NEXT: s_cbranch_execz .LBB1_2523; GFX1164-NEXT: ; %bb.1:524; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]525; GFX1164-NEXT: s_mov_b32 s15, 0x31016000526; GFX1164-NEXT: s_waitcnt lgkmcnt(0)527; GFX1164-NEXT: s_mul_i32 s6, s8, s6528; GFX1164-NEXT: s_mov_b32 s14, -1529; GFX1164-NEXT: v_mov_b32_e32 v1, s6530; GFX1164-NEXT: s_mov_b32 s12, s2531; GFX1164-NEXT: s_mov_b32 s13, s3532; GFX1164-NEXT: buffer_atomic_add_u32 v1, off, s[12:15], 0 glc533; GFX1164-NEXT: s_waitcnt vmcnt(0)534; GFX1164-NEXT: buffer_gl1_inv535; GFX1164-NEXT: buffer_gl0_inv536; GFX1164-NEXT: .LBB1_2:537; GFX1164-NEXT: s_or_b64 exec, exec, s[4:5]538; GFX1164-NEXT: s_waitcnt lgkmcnt(0)539; GFX1164-NEXT: v_readfirstlane_b32 s2, v1540; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)541; GFX1164-NEXT: v_mad_u64_u32 v[1:2], null, s8, v0, s[2:3]542; GFX1164-NEXT: s_mov_b32 s3, 0x31016000543; GFX1164-NEXT: s_mov_b32 s2, -1544; GFX1164-NEXT: buffer_store_b32 v1, off, s[0:3], 0545; GFX1164-NEXT: s_endpgm546;547; GFX1132-LABEL: add_i32_uniform:548; GFX1132: ; %bb.0: ; %entry549; GFX1132-NEXT: s_clause 0x1550; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24551; GFX1132-NEXT: s_load_b32 s4, s[4:5], 0x34552; GFX1132-NEXT: s_mov_b32 s6, exec_lo553; GFX1132-NEXT: s_mov_b32 s5, exec_lo554; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0555; GFX1132-NEXT: ; implicit-def: $vgpr1556; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)557; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0558; GFX1132-NEXT: s_cbranch_execz .LBB1_2559; GFX1132-NEXT: ; %bb.1:560; GFX1132-NEXT: s_bcnt1_i32_b32 s6, s6561; GFX1132-NEXT: s_mov_b32 s11, 0x31016000562; GFX1132-NEXT: s_waitcnt lgkmcnt(0)563; GFX1132-NEXT: s_mul_i32 s6, s4, s6564; GFX1132-NEXT: s_mov_b32 s10, -1565; GFX1132-NEXT: v_mov_b32_e32 v1, s6566; GFX1132-NEXT: s_mov_b32 s8, s2567; GFX1132-NEXT: s_mov_b32 s9, s3568; GFX1132-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc569; GFX1132-NEXT: s_waitcnt vmcnt(0)570; GFX1132-NEXT: buffer_gl1_inv571; GFX1132-NEXT: buffer_gl0_inv572; GFX1132-NEXT: .LBB1_2:573; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s5574; GFX1132-NEXT: s_waitcnt lgkmcnt(0)575; GFX1132-NEXT: v_readfirstlane_b32 s2, v1576; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)577; GFX1132-NEXT: v_mad_u64_u32 v[1:2], null, s4, v0, s[2:3]578; GFX1132-NEXT: s_mov_b32 s3, 0x31016000579; GFX1132-NEXT: s_mov_b32 s2, -1580; GFX1132-NEXT: buffer_store_b32 v1, off, s[0:3], 0581; GFX1132-NEXT: s_endpgm582;583; GFX1264-LABEL: add_i32_uniform:584; GFX1264: ; %bb.0: ; %entry585; GFX1264-NEXT: s_clause 0x1586; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24587; GFX1264-NEXT: s_load_b32 s8, s[4:5], 0x34588; GFX1264-NEXT: s_mov_b64 s[6:7], exec589; GFX1264-NEXT: s_mov_b64 s[4:5], exec590; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0591; GFX1264-NEXT: ; implicit-def: $vgpr1592; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)593; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0594; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v0595; GFX1264-NEXT: s_cbranch_execz .LBB1_2596; GFX1264-NEXT: ; %bb.1:597; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[6:7]598; GFX1264-NEXT: s_mov_b32 s15, 0x31016000599; GFX1264-NEXT: s_wait_kmcnt 0x0600; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)601; GFX1264-NEXT: s_mul_i32 s6, s8, s6602; GFX1264-NEXT: s_mov_b32 s14, -1603; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)604; GFX1264-NEXT: v_mov_b32_e32 v1, s6605; GFX1264-NEXT: s_mov_b32 s12, s2606; GFX1264-NEXT: s_mov_b32 s13, s3607; GFX1264-NEXT: buffer_atomic_add_u32 v1, off, s[12:15], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV608; GFX1264-NEXT: s_wait_loadcnt 0x0609; GFX1264-NEXT: global_inv scope:SCOPE_DEV610; GFX1264-NEXT: .LBB1_2:611; GFX1264-NEXT: s_or_b64 exec, exec, s[4:5]612; GFX1264-NEXT: s_wait_kmcnt 0x0613; GFX1264-NEXT: v_readfirstlane_b32 s2, v1614; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)615; GFX1264-NEXT: v_mad_co_u64_u32 v[0:1], null, s8, v0, s[2:3]616; GFX1264-NEXT: s_mov_b32 s3, 0x31016000617; GFX1264-NEXT: s_mov_b32 s2, -1618; GFX1264-NEXT: buffer_store_b32 v0, off, s[0:3], null619; GFX1264-NEXT: s_endpgm620;621; GFX1232-LABEL: add_i32_uniform:622; GFX1232: ; %bb.0: ; %entry623; GFX1232-NEXT: s_clause 0x1624; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24625; GFX1232-NEXT: s_load_b32 s4, s[4:5], 0x34626; GFX1232-NEXT: s_mov_b32 s6, exec_lo627; GFX1232-NEXT: s_mov_b32 s5, exec_lo628; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0629; GFX1232-NEXT: ; implicit-def: $vgpr1630; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)631; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v0632; GFX1232-NEXT: s_cbranch_execz .LBB1_2633; GFX1232-NEXT: ; %bb.1:634; GFX1232-NEXT: s_bcnt1_i32_b32 s6, s6635; GFX1232-NEXT: s_mov_b32 s11, 0x31016000636; GFX1232-NEXT: s_wait_kmcnt 0x0637; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)638; GFX1232-NEXT: s_mul_i32 s6, s4, s6639; GFX1232-NEXT: s_mov_b32 s10, -1640; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)641; GFX1232-NEXT: v_mov_b32_e32 v1, s6642; GFX1232-NEXT: s_mov_b32 s8, s2643; GFX1232-NEXT: s_mov_b32 s9, s3644; GFX1232-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV645; GFX1232-NEXT: s_wait_loadcnt 0x0646; GFX1232-NEXT: global_inv scope:SCOPE_DEV647; GFX1232-NEXT: .LBB1_2:648; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s5649; GFX1232-NEXT: s_wait_kmcnt 0x0650; GFX1232-NEXT: v_readfirstlane_b32 s2, v1651; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)652; GFX1232-NEXT: v_mad_co_u64_u32 v[0:1], null, s4, v0, s[2:3]653; GFX1232-NEXT: s_mov_b32 s3, 0x31016000654; GFX1232-NEXT: s_mov_b32 s2, -1655; GFX1232-NEXT: buffer_store_b32 v0, off, s[0:3], null656; GFX1232-NEXT: s_endpgm657entry:658 %old = atomicrmw add ptr addrspace(1) %inout, i32 %additive syncscope("agent") acq_rel659 store i32 %old, ptr addrspace(1) %out660 ret void661}662 663define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(1) %inout) {664; GFX7LESS_ITERATIVE-LABEL: add_i32_varying:665; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry666; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec667; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s6, 0668; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr1669; GFX7LESS_ITERATIVE-NEXT: .LBB2_1: ; %ComputeLoop670; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1671; GFX7LESS_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]672; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 m0, s2673; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s7, v0, s2674; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v1, s6, m0675; GFX7LESS_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s2676; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]677; GFX7LESS_ITERATIVE-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0678; GFX7LESS_ITERATIVE-NEXT: s_and_b64 vcc, exec, s[2:3]679; GFX7LESS_ITERATIVE-NEXT: s_add_i32 s6, s6, s7680; GFX7LESS_ITERATIVE-NEXT: s_cbranch_vccnz .LBB2_1681; GFX7LESS_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd682; GFX7LESS_ITERATIVE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9683; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0684; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0685; GFX7LESS_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0686; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr0687; GFX7LESS_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc688; GFX7LESS_ITERATIVE-NEXT: s_xor_b64 s[4:5], exec, s[4:5]689; GFX7LESS_ITERATIVE-NEXT: s_cbranch_execz .LBB2_4690; GFX7LESS_ITERATIVE-NEXT: ; %bb.3:691; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s11, 0xf000692; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s10, -1693; GFX7LESS_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)694; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s8, s2695; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s9, s3696; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, s6697; GFX7LESS_ITERATIVE-NEXT: buffer_atomic_add v0, off, s[8:11], 0 glc698; GFX7LESS_ITERATIVE-NEXT: s_waitcnt vmcnt(0)699; GFX7LESS_ITERATIVE-NEXT: buffer_wbinvl1700; GFX7LESS_ITERATIVE-NEXT: .LBB2_4:701; GFX7LESS_ITERATIVE-NEXT: s_or_b64 exec, exec, s[4:5]702; GFX7LESS_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)703; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s3, 0xf000704; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s2, -1705; GFX7LESS_ITERATIVE-NEXT: v_readfirstlane_b32 s4, v0706; GFX7LESS_ITERATIVE-NEXT: s_waitcnt expcnt(0)707; GFX7LESS_ITERATIVE-NEXT: v_add_i32_e32 v0, vcc, s4, v1708; GFX7LESS_ITERATIVE-NEXT: buffer_store_dword v0, off, s[0:3], 0709; GFX7LESS_ITERATIVE-NEXT: s_endpgm710;711; GFX8_ITERATIVE-LABEL: add_i32_varying:712; GFX8_ITERATIVE: ; %bb.0: ; %entry713; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec714; GFX8_ITERATIVE-NEXT: s_mov_b32 s6, 0715; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr1716; GFX8_ITERATIVE-NEXT: .LBB2_1: ; %ComputeLoop717; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1718; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]719; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s2720; GFX8_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s2721; GFX8_ITERATIVE-NEXT: v_writelane_b32 v1, s6, m0722; GFX8_ITERATIVE-NEXT: s_add_i32 s6, s6, s3723; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s2724; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]725; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB2_1726; GFX8_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd727; GFX8_ITERATIVE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24728; GFX8_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0729; GFX8_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0730; GFX8_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0731; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr0732; GFX8_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc733; GFX8_ITERATIVE-NEXT: s_xor_b64 s[4:5], exec, s[4:5]734; GFX8_ITERATIVE-NEXT: s_cbranch_execz .LBB2_4735; GFX8_ITERATIVE-NEXT: ; %bb.3:736; GFX8_ITERATIVE-NEXT: s_mov_b32 s11, 0xf000737; GFX8_ITERATIVE-NEXT: s_mov_b32 s10, -1738; GFX8_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)739; GFX8_ITERATIVE-NEXT: s_mov_b32 s8, s2740; GFX8_ITERATIVE-NEXT: s_mov_b32 s9, s3741; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, s6742; GFX8_ITERATIVE-NEXT: buffer_atomic_add v0, off, s[8:11], 0 glc743; GFX8_ITERATIVE-NEXT: s_waitcnt vmcnt(0)744; GFX8_ITERATIVE-NEXT: buffer_wbinvl1_vol745; GFX8_ITERATIVE-NEXT: .LBB2_4:746; GFX8_ITERATIVE-NEXT: s_or_b64 exec, exec, s[4:5]747; GFX8_ITERATIVE-NEXT: v_readfirstlane_b32 s4, v0748; GFX8_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)749; GFX8_ITERATIVE-NEXT: s_mov_b32 s3, 0xf000750; GFX8_ITERATIVE-NEXT: s_mov_b32 s2, -1751; GFX8_ITERATIVE-NEXT: v_add_u32_e32 v0, vcc, s4, v1752; GFX8_ITERATIVE-NEXT: buffer_store_dword v0, off, s[0:3], 0753; GFX8_ITERATIVE-NEXT: s_endpgm754;755; GFX9_ITERATIVE-LABEL: add_i32_varying:756; GFX9_ITERATIVE: ; %bb.0: ; %entry757; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec758; GFX9_ITERATIVE-NEXT: s_mov_b32 s6, 0759; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr1760; GFX9_ITERATIVE-NEXT: .LBB2_1: ; %ComputeLoop761; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1762; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]763; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s2764; GFX9_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s2765; GFX9_ITERATIVE-NEXT: v_writelane_b32 v1, s6, m0766; GFX9_ITERATIVE-NEXT: s_add_i32 s6, s6, s3767; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s2768; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]769; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB2_1770; GFX9_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd771; GFX9_ITERATIVE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24772; GFX9_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0773; GFX9_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0774; GFX9_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0775; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr0776; GFX9_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc777; GFX9_ITERATIVE-NEXT: s_xor_b64 s[4:5], exec, s[4:5]778; GFX9_ITERATIVE-NEXT: s_cbranch_execz .LBB2_4779; GFX9_ITERATIVE-NEXT: ; %bb.3:780; GFX9_ITERATIVE-NEXT: s_mov_b32 s11, 0xf000781; GFX9_ITERATIVE-NEXT: s_mov_b32 s10, -1782; GFX9_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)783; GFX9_ITERATIVE-NEXT: s_mov_b32 s8, s2784; GFX9_ITERATIVE-NEXT: s_mov_b32 s9, s3785; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, s6786; GFX9_ITERATIVE-NEXT: buffer_atomic_add v0, off, s[8:11], 0 glc787; GFX9_ITERATIVE-NEXT: s_waitcnt vmcnt(0)788; GFX9_ITERATIVE-NEXT: buffer_wbinvl1_vol789; GFX9_ITERATIVE-NEXT: .LBB2_4:790; GFX9_ITERATIVE-NEXT: s_or_b64 exec, exec, s[4:5]791; GFX9_ITERATIVE-NEXT: v_readfirstlane_b32 s4, v0792; GFX9_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)793; GFX9_ITERATIVE-NEXT: s_mov_b32 s3, 0xf000794; GFX9_ITERATIVE-NEXT: s_mov_b32 s2, -1795; GFX9_ITERATIVE-NEXT: v_add_u32_e32 v0, s4, v1796; GFX9_ITERATIVE-NEXT: buffer_store_dword v0, off, s[0:3], 0797; GFX9_ITERATIVE-NEXT: s_endpgm798;799; GFX1064_ITERATIVE-LABEL: add_i32_varying:800; GFX1064_ITERATIVE: ; %bb.0: ; %entry801; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec802; GFX1064_ITERATIVE-NEXT: s_mov_b32 s6, 0803; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr1804; GFX1064_ITERATIVE-NEXT: .LBB2_1: ; %ComputeLoop805; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1806; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]807; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s7, v0, s2808; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v1, s6, s2809; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s2810; GFX1064_ITERATIVE-NEXT: s_add_i32 s6, s6, s7811; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]812; GFX1064_ITERATIVE-NEXT: s_cbranch_scc1 .LBB2_1813; GFX1064_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd814; GFX1064_ITERATIVE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24815; GFX1064_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0816; GFX1064_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0817; GFX1064_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0818; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr0819; GFX1064_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc820; GFX1064_ITERATIVE-NEXT: s_xor_b64 s[4:5], exec, s[4:5]821; GFX1064_ITERATIVE-NEXT: s_cbranch_execz .LBB2_4822; GFX1064_ITERATIVE-NEXT: ; %bb.3:823; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v0, s6824; GFX1064_ITERATIVE-NEXT: s_mov_b32 s11, 0x31016000825; GFX1064_ITERATIVE-NEXT: s_mov_b32 s10, -1826; GFX1064_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)827; GFX1064_ITERATIVE-NEXT: s_mov_b32 s8, s2828; GFX1064_ITERATIVE-NEXT: s_mov_b32 s9, s3829; GFX1064_ITERATIVE-NEXT: buffer_atomic_add v0, off, s[8:11], 0 glc830; GFX1064_ITERATIVE-NEXT: s_waitcnt vmcnt(0)831; GFX1064_ITERATIVE-NEXT: buffer_gl1_inv832; GFX1064_ITERATIVE-NEXT: buffer_gl0_inv833; GFX1064_ITERATIVE-NEXT: .LBB2_4:834; GFX1064_ITERATIVE-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)835; GFX1064_ITERATIVE-NEXT: s_or_b64 exec, exec, s[4:5]836; GFX1064_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)837; GFX1064_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v0838; GFX1064_ITERATIVE-NEXT: s_mov_b32 s3, 0x31016000839; GFX1064_ITERATIVE-NEXT: v_add_nc_u32_e32 v0, s2, v1840; GFX1064_ITERATIVE-NEXT: s_mov_b32 s2, -1841; GFX1064_ITERATIVE-NEXT: buffer_store_dword v0, off, s[0:3], 0842; GFX1064_ITERATIVE-NEXT: s_endpgm843;844; GFX1032_ITERATIVE-LABEL: add_i32_varying:845; GFX1032_ITERATIVE: ; %bb.0: ; %entry846; GFX1032_ITERATIVE-NEXT: s_mov_b32 s0, exec_lo847; GFX1032_ITERATIVE-NEXT: s_mov_b32 s6, 0848; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr1849; GFX1032_ITERATIVE-NEXT: .LBB2_1: ; %ComputeLoop850; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1851; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s1, s0852; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s2, v0, s1853; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s6, s1854; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s1, 1, s1855; GFX1032_ITERATIVE-NEXT: s_add_i32 s6, s6, s2856; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s0, s0, s1857; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB2_1858; GFX1032_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd859; GFX1032_ITERATIVE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24860; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0861; GFX1032_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0862; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr0863; GFX1032_ITERATIVE-NEXT: s_and_saveexec_b32 s4, vcc_lo864; GFX1032_ITERATIVE-NEXT: s_xor_b32 s4, exec_lo, s4865; GFX1032_ITERATIVE-NEXT: s_cbranch_execz .LBB2_4866; GFX1032_ITERATIVE-NEXT: ; %bb.3:867; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v0, s6868; GFX1032_ITERATIVE-NEXT: s_mov_b32 s11, 0x31016000869; GFX1032_ITERATIVE-NEXT: s_mov_b32 s10, -1870; GFX1032_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)871; GFX1032_ITERATIVE-NEXT: s_mov_b32 s8, s2872; GFX1032_ITERATIVE-NEXT: s_mov_b32 s9, s3873; GFX1032_ITERATIVE-NEXT: buffer_atomic_add v0, off, s[8:11], 0 glc874; GFX1032_ITERATIVE-NEXT: s_waitcnt vmcnt(0)875; GFX1032_ITERATIVE-NEXT: buffer_gl1_inv876; GFX1032_ITERATIVE-NEXT: buffer_gl0_inv877; GFX1032_ITERATIVE-NEXT: .LBB2_4:878; GFX1032_ITERATIVE-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)879; GFX1032_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s4880; GFX1032_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)881; GFX1032_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v0882; GFX1032_ITERATIVE-NEXT: s_mov_b32 s3, 0x31016000883; GFX1032_ITERATIVE-NEXT: v_add_nc_u32_e32 v0, s2, v1884; GFX1032_ITERATIVE-NEXT: s_mov_b32 s2, -1885; GFX1032_ITERATIVE-NEXT: buffer_store_dword v0, off, s[0:3], 0886; GFX1032_ITERATIVE-NEXT: s_endpgm887;888; GFX1164_ITERATIVE-LABEL: add_i32_varying:889; GFX1164_ITERATIVE: ; %bb.0: ; %entry890; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v1, 0x3ff, v0891; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec892; GFX1164_ITERATIVE-NEXT: s_mov_b32 s6, 0893; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr0894; GFX1164_ITERATIVE-NEXT: .LBB2_1: ; %ComputeLoop895; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1896; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s2, s[0:1]897; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)898; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s7, v1, s2899; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v0, s6, s2900; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s2901; GFX1164_ITERATIVE-NEXT: s_add_i32 s6, s6, s7902; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]903; GFX1164_ITERATIVE-NEXT: s_cbranch_scc1 .LBB2_1904; GFX1164_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd905; GFX1164_ITERATIVE-NEXT: s_load_b128 s[0:3], s[4:5], 0x24906; GFX1164_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0907; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)908; GFX1164_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1909; GFX1164_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1910; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr1911; GFX1164_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc912; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)913; GFX1164_ITERATIVE-NEXT: s_xor_b64 s[4:5], exec, s[4:5]914; GFX1164_ITERATIVE-NEXT: s_cbranch_execz .LBB2_4915; GFX1164_ITERATIVE-NEXT: ; %bb.3:916; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v1, s6917; GFX1164_ITERATIVE-NEXT: s_mov_b32 s11, 0x31016000918; GFX1164_ITERATIVE-NEXT: s_mov_b32 s10, -1919; GFX1164_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)920; GFX1164_ITERATIVE-NEXT: s_mov_b32 s8, s2921; GFX1164_ITERATIVE-NEXT: s_mov_b32 s9, s3922; GFX1164_ITERATIVE-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc923; GFX1164_ITERATIVE-NEXT: s_waitcnt vmcnt(0)924; GFX1164_ITERATIVE-NEXT: buffer_gl1_inv925; GFX1164_ITERATIVE-NEXT: buffer_gl0_inv926; GFX1164_ITERATIVE-NEXT: .LBB2_4:927; GFX1164_ITERATIVE-NEXT: s_or_b64 exec, exec, s[4:5]928; GFX1164_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)929; GFX1164_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v1930; GFX1164_ITERATIVE-NEXT: s_mov_b32 s3, 0x31016000931; GFX1164_ITERATIVE-NEXT: v_add_nc_u32_e32 v0, s2, v0932; GFX1164_ITERATIVE-NEXT: s_mov_b32 s2, -1933; GFX1164_ITERATIVE-NEXT: buffer_store_b32 v0, off, s[0:3], 0934; GFX1164_ITERATIVE-NEXT: s_endpgm935;936; GFX1132_ITERATIVE-LABEL: add_i32_varying:937; GFX1132_ITERATIVE: ; %bb.0: ; %entry938; GFX1132_ITERATIVE-NEXT: v_and_b32_e32 v1, 0x3ff, v0939; GFX1132_ITERATIVE-NEXT: s_mov_b32 s0, exec_lo940; GFX1132_ITERATIVE-NEXT: s_mov_b32 s6, 0941; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr0942; GFX1132_ITERATIVE-NEXT: .LBB2_1: ; %ComputeLoop943; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1944; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s1, s0945; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)946; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s2, v1, s1947; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s6, s1948; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s1, 1, s1949; GFX1132_ITERATIVE-NEXT: s_add_i32 s6, s6, s2950; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s0, s0, s1951; GFX1132_ITERATIVE-NEXT: s_cbranch_scc1 .LBB2_1952; GFX1132_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd953; GFX1132_ITERATIVE-NEXT: s_load_b128 s[0:3], s[4:5], 0x24954; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0955; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)956; GFX1132_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1957; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr1958; GFX1132_ITERATIVE-NEXT: s_and_saveexec_b32 s4, vcc_lo959; GFX1132_ITERATIVE-NEXT: s_xor_b32 s4, exec_lo, s4960; GFX1132_ITERATIVE-NEXT: s_cbranch_execz .LBB2_4961; GFX1132_ITERATIVE-NEXT: ; %bb.3:962; GFX1132_ITERATIVE-NEXT: v_mov_b32_e32 v1, s6963; GFX1132_ITERATIVE-NEXT: s_mov_b32 s11, 0x31016000964; GFX1132_ITERATIVE-NEXT: s_mov_b32 s10, -1965; GFX1132_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)966; GFX1132_ITERATIVE-NEXT: s_mov_b32 s8, s2967; GFX1132_ITERATIVE-NEXT: s_mov_b32 s9, s3968; GFX1132_ITERATIVE-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc969; GFX1132_ITERATIVE-NEXT: s_waitcnt vmcnt(0)970; GFX1132_ITERATIVE-NEXT: buffer_gl1_inv971; GFX1132_ITERATIVE-NEXT: buffer_gl0_inv972; GFX1132_ITERATIVE-NEXT: .LBB2_4:973; GFX1132_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s4974; GFX1132_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)975; GFX1132_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v1976; GFX1132_ITERATIVE-NEXT: s_mov_b32 s3, 0x31016000977; GFX1132_ITERATIVE-NEXT: v_add_nc_u32_e32 v0, s2, v0978; GFX1132_ITERATIVE-NEXT: s_mov_b32 s2, -1979; GFX1132_ITERATIVE-NEXT: buffer_store_b32 v0, off, s[0:3], 0980; GFX1132_ITERATIVE-NEXT: s_endpgm981;982; GFX1264_ITERATIVE-LABEL: add_i32_varying:983; GFX1264_ITERATIVE: ; %bb.0: ; %entry984; GFX1264_ITERATIVE-NEXT: v_and_b32_e32 v1, 0x3ff, v0985; GFX1264_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec986; GFX1264_ITERATIVE-NEXT: s_mov_b32 s6, 0987; GFX1264_ITERATIVE-NEXT: ; implicit-def: $vgpr0988; GFX1264_ITERATIVE-NEXT: .LBB2_1: ; %ComputeLoop989; GFX1264_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1990; GFX1264_ITERATIVE-NEXT: s_ctz_i32_b64 s2, s[0:1]991; GFX1264_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)992; GFX1264_ITERATIVE-NEXT: v_readlane_b32 s7, v1, s2993; GFX1264_ITERATIVE-NEXT: v_writelane_b32 v0, s6, s2994; GFX1264_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s2995; GFX1264_ITERATIVE-NEXT: s_add_co_i32 s6, s6, s7996; GFX1264_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)997; GFX1264_ITERATIVE-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]998; GFX1264_ITERATIVE-NEXT: s_cbranch_scc1 .LBB2_1999; GFX1264_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd1000; GFX1264_ITERATIVE-NEXT: s_load_b128 s[0:3], s[4:5], 0x241001; GFX1264_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 01002; GFX1264_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1003; GFX1264_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v11004; GFX1264_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v11005; GFX1264_ITERATIVE-NEXT: ; implicit-def: $vgpr11006; GFX1264_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc1007; GFX1264_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1008; GFX1264_ITERATIVE-NEXT: s_xor_b64 s[4:5], exec, s[4:5]1009; GFX1264_ITERATIVE-NEXT: s_cbranch_execz .LBB2_41010; GFX1264_ITERATIVE-NEXT: ; %bb.3:1011; GFX1264_ITERATIVE-NEXT: v_mov_b32_e32 v1, s61012; GFX1264_ITERATIVE-NEXT: s_mov_b32 s11, 0x310160001013; GFX1264_ITERATIVE-NEXT: s_mov_b32 s10, -11014; GFX1264_ITERATIVE-NEXT: s_wait_kmcnt 0x01015; GFX1264_ITERATIVE-NEXT: s_mov_b32 s8, s21016; GFX1264_ITERATIVE-NEXT: s_mov_b32 s9, s31017; GFX1264_ITERATIVE-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV1018; GFX1264_ITERATIVE-NEXT: s_wait_loadcnt 0x01019; GFX1264_ITERATIVE-NEXT: global_inv scope:SCOPE_DEV1020; GFX1264_ITERATIVE-NEXT: .LBB2_4:1021; GFX1264_ITERATIVE-NEXT: s_or_b64 exec, exec, s[4:5]1022; GFX1264_ITERATIVE-NEXT: s_wait_kmcnt 0x01023; GFX1264_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v11024; GFX1264_ITERATIVE-NEXT: s_mov_b32 s3, 0x310160001025; GFX1264_ITERATIVE-NEXT: s_wait_alu depctr_va_sdst(0)1026; GFX1264_ITERATIVE-NEXT: v_add_nc_u32_e32 v0, s2, v01027; GFX1264_ITERATIVE-NEXT: s_mov_b32 s2, -11028; GFX1264_ITERATIVE-NEXT: buffer_store_b32 v0, off, s[0:3], null1029; GFX1264_ITERATIVE-NEXT: s_endpgm1030;1031; GFX1232_ITERATIVE-LABEL: add_i32_varying:1032; GFX1232_ITERATIVE: ; %bb.0: ; %entry1033; GFX1232_ITERATIVE-NEXT: v_and_b32_e32 v1, 0x3ff, v01034; GFX1232_ITERATIVE-NEXT: s_mov_b32 s0, exec_lo1035; GFX1232_ITERATIVE-NEXT: s_mov_b32 s6, 01036; GFX1232_ITERATIVE-NEXT: ; implicit-def: $vgpr01037; GFX1232_ITERATIVE-NEXT: .LBB2_1: ; %ComputeLoop1038; GFX1232_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=11039; GFX1232_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)1040; GFX1232_ITERATIVE-NEXT: s_ctz_i32_b32 s1, s01041; GFX1232_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)1042; GFX1232_ITERATIVE-NEXT: v_readlane_b32 s2, v1, s11043; GFX1232_ITERATIVE-NEXT: v_writelane_b32 v0, s6, s11044; GFX1232_ITERATIVE-NEXT: s_lshl_b32 s1, 1, s11045; GFX1232_ITERATIVE-NEXT: s_add_co_i32 s6, s6, s21046; GFX1232_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)1047; GFX1232_ITERATIVE-NEXT: s_and_not1_b32 s0, s0, s11048; GFX1232_ITERATIVE-NEXT: s_cbranch_scc1 .LBB2_11049; GFX1232_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd1050; GFX1232_ITERATIVE-NEXT: s_load_b128 s[0:3], s[4:5], 0x241051; GFX1232_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 01052; GFX1232_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)1053; GFX1232_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v11054; GFX1232_ITERATIVE-NEXT: ; implicit-def: $vgpr11055; GFX1232_ITERATIVE-NEXT: s_and_saveexec_b32 s4, vcc_lo1056; GFX1232_ITERATIVE-NEXT: s_xor_b32 s4, exec_lo, s41057; GFX1232_ITERATIVE-NEXT: s_cbranch_execz .LBB2_41058; GFX1232_ITERATIVE-NEXT: ; %bb.3:1059; GFX1232_ITERATIVE-NEXT: v_mov_b32_e32 v1, s61060; GFX1232_ITERATIVE-NEXT: s_mov_b32 s11, 0x310160001061; GFX1232_ITERATIVE-NEXT: s_mov_b32 s10, -11062; GFX1232_ITERATIVE-NEXT: s_wait_kmcnt 0x01063; GFX1232_ITERATIVE-NEXT: s_mov_b32 s8, s21064; GFX1232_ITERATIVE-NEXT: s_mov_b32 s9, s31065; GFX1232_ITERATIVE-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV1066; GFX1232_ITERATIVE-NEXT: s_wait_loadcnt 0x01067; GFX1232_ITERATIVE-NEXT: global_inv scope:SCOPE_DEV1068; GFX1232_ITERATIVE-NEXT: .LBB2_4:1069; GFX1232_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s41070; GFX1232_ITERATIVE-NEXT: s_wait_kmcnt 0x01071; GFX1232_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v11072; GFX1232_ITERATIVE-NEXT: s_mov_b32 s3, 0x310160001073; GFX1232_ITERATIVE-NEXT: v_add_nc_u32_e32 v0, s2, v01074; GFX1232_ITERATIVE-NEXT: s_mov_b32 s2, -11075; GFX1232_ITERATIVE-NEXT: buffer_store_b32 v0, off, s[0:3], null1076; GFX1232_ITERATIVE-NEXT: s_endpgm1077;1078; GFX7LESS_DPP-LABEL: add_i32_varying:1079; GFX7LESS_DPP: ; %bb.0: ; %entry1080; GFX7LESS_DPP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91081; GFX7LESS_DPP-NEXT: s_mov_b32 s7, 0xf0001082; GFX7LESS_DPP-NEXT: s_mov_b32 s6, -11083; GFX7LESS_DPP-NEXT: s_mov_b32 s10, s61084; GFX7LESS_DPP-NEXT: s_mov_b32 s11, s71085; GFX7LESS_DPP-NEXT: s_waitcnt lgkmcnt(0)1086; GFX7LESS_DPP-NEXT: s_mov_b32 s8, s21087; GFX7LESS_DPP-NEXT: s_mov_b32 s9, s31088; GFX7LESS_DPP-NEXT: buffer_atomic_add v0, off, s[8:11], 0 glc1089; GFX7LESS_DPP-NEXT: s_waitcnt vmcnt(0)1090; GFX7LESS_DPP-NEXT: buffer_wbinvl11091; GFX7LESS_DPP-NEXT: s_mov_b32 s4, s01092; GFX7LESS_DPP-NEXT: s_mov_b32 s5, s11093; GFX7LESS_DPP-NEXT: buffer_store_dword v0, off, s[4:7], 01094; GFX7LESS_DPP-NEXT: s_endpgm1095;1096; GFX8_DPP-LABEL: add_i32_varying:1097; GFX8_DPP: ; %bb.0: ; %entry1098; GFX8_DPP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241099; GFX8_DPP-NEXT: s_or_saveexec_b64 s[4:5], -11100; GFX8_DPP-NEXT: v_mov_b32_e32 v1, 01101; GFX8_DPP-NEXT: s_mov_b64 exec, s[4:5]1102; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 01103; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v31104; GFX8_DPP-NEXT: s_or_saveexec_b64 s[4:5], -11105; GFX8_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[4:5]1106; GFX8_DPP-NEXT: s_nop 11107; GFX8_DPP-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:11108; GFX8_DPP-NEXT: s_nop 11109; GFX8_DPP-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:11110; GFX8_DPP-NEXT: s_nop 11111; GFX8_DPP-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:11112; GFX8_DPP-NEXT: s_nop 11113; GFX8_DPP-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:11114; GFX8_DPP-NEXT: s_nop 11115; GFX8_DPP-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf1116; GFX8_DPP-NEXT: s_nop 11117; GFX8_DPP-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf1118; GFX8_DPP-NEXT: v_readlane_b32 s6, v2, 631119; GFX8_DPP-NEXT: s_nop 01120; GFX8_DPP-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf1121; GFX8_DPP-NEXT: s_mov_b64 exec, s[4:5]1122; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v31123; GFX8_DPP-NEXT: ; implicit-def: $vgpr01124; GFX8_DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc1125; GFX8_DPP-NEXT: s_cbranch_execz .LBB2_21126; GFX8_DPP-NEXT: ; %bb.1:1127; GFX8_DPP-NEXT: s_mov_b32 s11, 0xf0001128; GFX8_DPP-NEXT: s_mov_b32 s10, -11129; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0)1130; GFX8_DPP-NEXT: s_mov_b32 s8, s21131; GFX8_DPP-NEXT: s_mov_b32 s9, s31132; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s61133; GFX8_DPP-NEXT: buffer_atomic_add v0, off, s[8:11], 0 glc1134; GFX8_DPP-NEXT: s_waitcnt vmcnt(0)1135; GFX8_DPP-NEXT: buffer_wbinvl1_vol1136; GFX8_DPP-NEXT: .LBB2_2:1137; GFX8_DPP-NEXT: s_or_b64 exec, exec, s[4:5]1138; GFX8_DPP-NEXT: v_readfirstlane_b32 s4, v01139; GFX8_DPP-NEXT: v_mov_b32_e32 v0, v11140; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0)1141; GFX8_DPP-NEXT: s_mov_b32 s3, 0xf0001142; GFX8_DPP-NEXT: s_mov_b32 s2, -11143; GFX8_DPP-NEXT: v_add_u32_e32 v0, vcc, s4, v01144; GFX8_DPP-NEXT: buffer_store_dword v0, off, s[0:3], 01145; GFX8_DPP-NEXT: s_endpgm1146;1147; GFX9_DPP-LABEL: add_i32_varying:1148; GFX9_DPP: ; %bb.0: ; %entry1149; GFX9_DPP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241150; GFX9_DPP-NEXT: s_or_saveexec_b64 s[4:5], -11151; GFX9_DPP-NEXT: v_mov_b32_e32 v1, 01152; GFX9_DPP-NEXT: s_mov_b64 exec, s[4:5]1153; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 01154; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v31155; GFX9_DPP-NEXT: s_or_saveexec_b64 s[4:5], -11156; GFX9_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[4:5]1157; GFX9_DPP-NEXT: s_nop 11158; GFX9_DPP-NEXT: v_add_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:11159; GFX9_DPP-NEXT: s_nop 11160; GFX9_DPP-NEXT: v_add_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:11161; GFX9_DPP-NEXT: s_nop 11162; GFX9_DPP-NEXT: v_add_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:11163; GFX9_DPP-NEXT: s_nop 11164; GFX9_DPP-NEXT: v_add_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:11165; GFX9_DPP-NEXT: s_nop 11166; GFX9_DPP-NEXT: v_add_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf1167; GFX9_DPP-NEXT: s_nop 11168; GFX9_DPP-NEXT: v_add_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf1169; GFX9_DPP-NEXT: v_readlane_b32 s6, v2, 631170; GFX9_DPP-NEXT: s_nop 01171; GFX9_DPP-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf1172; GFX9_DPP-NEXT: s_mov_b64 exec, s[4:5]1173; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v31174; GFX9_DPP-NEXT: ; implicit-def: $vgpr01175; GFX9_DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc1176; GFX9_DPP-NEXT: s_cbranch_execz .LBB2_21177; GFX9_DPP-NEXT: ; %bb.1:1178; GFX9_DPP-NEXT: s_mov_b32 s11, 0xf0001179; GFX9_DPP-NEXT: s_mov_b32 s10, -11180; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0)1181; GFX9_DPP-NEXT: s_mov_b32 s8, s21182; GFX9_DPP-NEXT: s_mov_b32 s9, s31183; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s61184; GFX9_DPP-NEXT: buffer_atomic_add v0, off, s[8:11], 0 glc1185; GFX9_DPP-NEXT: s_waitcnt vmcnt(0)1186; GFX9_DPP-NEXT: buffer_wbinvl1_vol1187; GFX9_DPP-NEXT: .LBB2_2:1188; GFX9_DPP-NEXT: s_or_b64 exec, exec, s[4:5]1189; GFX9_DPP-NEXT: v_readfirstlane_b32 s4, v01190; GFX9_DPP-NEXT: v_mov_b32_e32 v0, v11191; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0)1192; GFX9_DPP-NEXT: s_mov_b32 s3, 0xf0001193; GFX9_DPP-NEXT: s_mov_b32 s2, -11194; GFX9_DPP-NEXT: v_add_u32_e32 v0, s4, v01195; GFX9_DPP-NEXT: buffer_store_dword v0, off, s[0:3], 01196; GFX9_DPP-NEXT: s_endpgm1197;1198; GFX1064_DPP-LABEL: add_i32_varying:1199; GFX1064_DPP: ; %bb.0: ; %entry1200; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -11201; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]1202; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 01203; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:11204; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:11205; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:11206; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:11207; GFX1064_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -11208; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf1209; GFX1064_DPP-NEXT: v_readlane_b32 s2, v1, 311210; GFX1064_DPP-NEXT: v_mov_b32_e32 v2, s21211; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf1212; GFX1064_DPP-NEXT: v_readlane_b32 s6, v1, 151213; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf1214; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1]1215; GFX1064_DPP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241216; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[4:5], -11217; GFX1064_DPP-NEXT: v_readlane_b32 s7, v1, 311218; GFX1064_DPP-NEXT: v_writelane_b32 v3, s6, 161219; GFX1064_DPP-NEXT: s_mov_b64 exec, s[4:5]1220; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 01221; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[4:5], -11222; GFX1064_DPP-NEXT: v_readlane_b32 s8, v1, 471223; GFX1064_DPP-NEXT: v_readlane_b32 s9, v1, 631224; GFX1064_DPP-NEXT: v_writelane_b32 v3, s7, 321225; GFX1064_DPP-NEXT: s_mov_b64 exec, s[4:5]1226; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v01227; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[6:7], -11228; GFX1064_DPP-NEXT: s_mov_b32 s4, s91229; GFX1064_DPP-NEXT: v_writelane_b32 v3, s8, 481230; GFX1064_DPP-NEXT: s_mov_b64 exec, s[6:7]1231; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v01232; GFX1064_DPP-NEXT: s_mov_b32 s6, -11233; GFX1064_DPP-NEXT: ; implicit-def: $vgpr01234; GFX1064_DPP-NEXT: s_and_saveexec_b64 s[8:9], vcc1235; GFX1064_DPP-NEXT: s_cbranch_execz .LBB2_21236; GFX1064_DPP-NEXT: ; %bb.1:1237; GFX1064_DPP-NEXT: v_mov_b32_e32 v0, s41238; GFX1064_DPP-NEXT: s_mov_b32 s7, 0x310160001239; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)1240; GFX1064_DPP-NEXT: s_mov_b32 s4, s21241; GFX1064_DPP-NEXT: s_mov_b32 s5, s31242; GFX1064_DPP-NEXT: buffer_atomic_add v0, off, s[4:7], 0 glc1243; GFX1064_DPP-NEXT: s_waitcnt vmcnt(0)1244; GFX1064_DPP-NEXT: buffer_gl1_inv1245; GFX1064_DPP-NEXT: buffer_gl0_inv1246; GFX1064_DPP-NEXT: .LBB2_2:1247; GFX1064_DPP-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)1248; GFX1064_DPP-NEXT: s_or_b64 exec, exec, s[8:9]1249; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)1250; GFX1064_DPP-NEXT: v_readfirstlane_b32 s2, v01251; GFX1064_DPP-NEXT: v_mov_b32_e32 v0, v31252; GFX1064_DPP-NEXT: s_mov_b32 s3, 0x310160001253; GFX1064_DPP-NEXT: v_add_nc_u32_e32 v0, s2, v01254; GFX1064_DPP-NEXT: s_mov_b32 s2, s61255; GFX1064_DPP-NEXT: buffer_store_dword v0, off, s[0:3], 01256; GFX1064_DPP-NEXT: s_endpgm1257;1258; GFX1032_DPP-LABEL: add_i32_varying:1259; GFX1032_DPP: ; %bb.0: ; %entry1260; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -11261; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s01262; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:11263; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:11264; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:11265; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:11266; GFX1032_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -11267; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s01268; GFX1032_DPP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241269; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -11270; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf1271; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 01272; GFX1032_DPP-NEXT: v_readlane_b32 s6, v1, 311273; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf1274; GFX1032_DPP-NEXT: v_readlane_b32 s5, v1, 151275; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s41276; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 01277; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -11278; GFX1032_DPP-NEXT: v_writelane_b32 v3, s5, 161279; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s41280; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v01281; GFX1032_DPP-NEXT: s_mov_b32 s4, s61282; GFX1032_DPP-NEXT: s_mov_b32 s6, -11283; GFX1032_DPP-NEXT: ; implicit-def: $vgpr01284; GFX1032_DPP-NEXT: s_and_saveexec_b32 s8, vcc_lo1285; GFX1032_DPP-NEXT: s_cbranch_execz .LBB2_21286; GFX1032_DPP-NEXT: ; %bb.1:1287; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, s41288; GFX1032_DPP-NEXT: s_mov_b32 s7, 0x310160001289; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)1290; GFX1032_DPP-NEXT: s_mov_b32 s4, s21291; GFX1032_DPP-NEXT: s_mov_b32 s5, s31292; GFX1032_DPP-NEXT: buffer_atomic_add v0, off, s[4:7], 0 glc1293; GFX1032_DPP-NEXT: s_waitcnt vmcnt(0)1294; GFX1032_DPP-NEXT: buffer_gl1_inv1295; GFX1032_DPP-NEXT: buffer_gl0_inv1296; GFX1032_DPP-NEXT: .LBB2_2:1297; GFX1032_DPP-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)1298; GFX1032_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s81299; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)1300; GFX1032_DPP-NEXT: v_readfirstlane_b32 s2, v01301; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, v31302; GFX1032_DPP-NEXT: s_mov_b32 s3, 0x310160001303; GFX1032_DPP-NEXT: v_add_nc_u32_e32 v0, s2, v01304; GFX1032_DPP-NEXT: s_mov_b32 s2, s61305; GFX1032_DPP-NEXT: buffer_store_dword v0, off, s[0:3], 01306; GFX1032_DPP-NEXT: s_endpgm1307;1308; GFX1164_DPP-LABEL: add_i32_varying:1309; GFX1164_DPP: ; %bb.0: ; %entry1310; GFX1164_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v01311; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -11312; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)1313; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]1314; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 01315; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)1316; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:11317; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:11318; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1319; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:11320; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:11321; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1322; GFX1164_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -11323; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf1324; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1325; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 311326; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, s21327; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1328; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf1329; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 151330; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf1331; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]1332; GFX1164_DPP-NEXT: s_load_b128 s[0:3], s[4:5], 0x241333; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[4:5], -11334; GFX1164_DPP-NEXT: v_readlane_b32 s7, v1, 311335; GFX1164_DPP-NEXT: v_writelane_b32 v3, s6, 161336; GFX1164_DPP-NEXT: s_mov_b64 exec, s[4:5]1337; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1338; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 01339; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[4:5], -11340; GFX1164_DPP-NEXT: v_readlane_b32 s8, v1, 471341; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 631342; GFX1164_DPP-NEXT: v_writelane_b32 v3, s7, 321343; GFX1164_DPP-NEXT: s_mov_b64 exec, s[4:5]1344; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1345; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v01346; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[6:7], -11347; GFX1164_DPP-NEXT: s_mov_b32 s4, s91348; GFX1164_DPP-NEXT: v_writelane_b32 v3, s8, 481349; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7]1350; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v01351; GFX1164_DPP-NEXT: s_mov_b32 s6, -11352; GFX1164_DPP-NEXT: ; implicit-def: $vgpr01353; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[8:9], vcc1354; GFX1164_DPP-NEXT: s_cbranch_execz .LBB2_21355; GFX1164_DPP-NEXT: ; %bb.1:1356; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, s41357; GFX1164_DPP-NEXT: s_mov_b32 s7, 0x310160001358; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)1359; GFX1164_DPP-NEXT: s_mov_b32 s4, s21360; GFX1164_DPP-NEXT: s_mov_b32 s5, s31361; GFX1164_DPP-NEXT: buffer_atomic_add_u32 v0, off, s[4:7], 0 glc1362; GFX1164_DPP-NEXT: s_waitcnt vmcnt(0)1363; GFX1164_DPP-NEXT: buffer_gl1_inv1364; GFX1164_DPP-NEXT: buffer_gl0_inv1365; GFX1164_DPP-NEXT: .LBB2_2:1366; GFX1164_DPP-NEXT: s_or_b64 exec, exec, s[8:9]1367; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)1368; GFX1164_DPP-NEXT: v_readfirstlane_b32 s2, v01369; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, v31370; GFX1164_DPP-NEXT: s_mov_b32 s3, 0x310160001371; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)1372; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v0, s2, v01373; GFX1164_DPP-NEXT: s_mov_b32 s2, s61374; GFX1164_DPP-NEXT: buffer_store_b32 v0, off, s[0:3], 01375; GFX1164_DPP-NEXT: s_endpgm1376;1377; GFX1132_DPP-LABEL: add_i32_varying:1378; GFX1132_DPP: ; %bb.0: ; %entry1379; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v01380; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -11381; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)1382; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s01383; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1384; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:11385; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:11386; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1387; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:11388; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:11389; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)1390; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -11391; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s01392; GFX1132_DPP-NEXT: s_load_b128 s[0:3], s[4:5], 0x241393; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -11394; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf1395; GFX1132_DPP-NEXT: v_mov_b32_e32 v3, 01396; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)1397; GFX1132_DPP-NEXT: v_readlane_b32 s6, v1, 311398; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf1399; GFX1132_DPP-NEXT: v_readlane_b32 s5, v1, 151400; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s41401; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)1402; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 01403; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -11404; GFX1132_DPP-NEXT: v_writelane_b32 v3, s5, 161405; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s41406; GFX1132_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v01407; GFX1132_DPP-NEXT: s_mov_b32 s4, s61408; GFX1132_DPP-NEXT: s_mov_b32 s6, -11409; GFX1132_DPP-NEXT: ; implicit-def: $vgpr01410; GFX1132_DPP-NEXT: s_and_saveexec_b32 s8, vcc_lo1411; GFX1132_DPP-NEXT: s_cbranch_execz .LBB2_21412; GFX1132_DPP-NEXT: ; %bb.1:1413; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, s41414; GFX1132_DPP-NEXT: s_mov_b32 s7, 0x310160001415; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)1416; GFX1132_DPP-NEXT: s_mov_b32 s4, s21417; GFX1132_DPP-NEXT: s_mov_b32 s5, s31418; GFX1132_DPP-NEXT: buffer_atomic_add_u32 v0, off, s[4:7], 0 glc1419; GFX1132_DPP-NEXT: s_waitcnt vmcnt(0)1420; GFX1132_DPP-NEXT: buffer_gl1_inv1421; GFX1132_DPP-NEXT: buffer_gl0_inv1422; GFX1132_DPP-NEXT: .LBB2_2:1423; GFX1132_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s81424; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)1425; GFX1132_DPP-NEXT: v_readfirstlane_b32 s2, v01426; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, v31427; GFX1132_DPP-NEXT: s_mov_b32 s3, 0x310160001428; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)1429; GFX1132_DPP-NEXT: v_add_nc_u32_e32 v0, s2, v01430; GFX1132_DPP-NEXT: s_mov_b32 s2, s61431; GFX1132_DPP-NEXT: buffer_store_b32 v0, off, s[0:3], 01432; GFX1132_DPP-NEXT: s_endpgm1433;1434; GFX1264_DPP-LABEL: add_i32_varying:1435; GFX1264_DPP: ; %bb.0: ; %entry1436; GFX1264_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v01437; GFX1264_DPP-NEXT: s_or_saveexec_b64 s[0:1], -11438; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)1439; GFX1264_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]1440; GFX1264_DPP-NEXT: v_mov_b32_e32 v3, 01441; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)1442; GFX1264_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:11443; GFX1264_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:11444; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1445; GFX1264_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:11446; GFX1264_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:11447; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1448; GFX1264_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -11449; GFX1264_DPP-NEXT: v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf1450; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1451; GFX1264_DPP-NEXT: v_readlane_b32 s2, v1, 311452; GFX1264_DPP-NEXT: v_mov_b32_e32 v2, s21453; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1454; GFX1264_DPP-NEXT: v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf1455; GFX1264_DPP-NEXT: v_readlane_b32 s6, v1, 151456; GFX1264_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf1457; GFX1264_DPP-NEXT: s_mov_b64 exec, s[0:1]1458; GFX1264_DPP-NEXT: s_load_b128 s[0:3], s[4:5], 0x241459; GFX1264_DPP-NEXT: s_or_saveexec_b64 s[4:5], -11460; GFX1264_DPP-NEXT: v_readlane_b32 s7, v1, 311461; GFX1264_DPP-NEXT: v_writelane_b32 v3, s6, 161462; GFX1264_DPP-NEXT: s_mov_b64 exec, s[4:5]1463; GFX1264_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1464; GFX1264_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 01465; GFX1264_DPP-NEXT: s_or_saveexec_b64 s[4:5], -11466; GFX1264_DPP-NEXT: v_readlane_b32 s8, v1, 471467; GFX1264_DPP-NEXT: v_readlane_b32 s9, v1, 631468; GFX1264_DPP-NEXT: v_writelane_b32 v3, s7, 321469; GFX1264_DPP-NEXT: s_mov_b64 exec, s[4:5]1470; GFX1264_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1471; GFX1264_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v01472; GFX1264_DPP-NEXT: s_or_saveexec_b64 s[6:7], -11473; GFX1264_DPP-NEXT: s_mov_b32 s4, s91474; GFX1264_DPP-NEXT: v_writelane_b32 v3, s8, 481475; GFX1264_DPP-NEXT: s_wait_alu depctr_sa_sdst(0)1476; GFX1264_DPP-NEXT: s_mov_b64 exec, s[6:7]1477; GFX1264_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v01478; GFX1264_DPP-NEXT: s_mov_b32 s6, -11479; GFX1264_DPP-NEXT: ; implicit-def: $vgpr01480; GFX1264_DPP-NEXT: s_and_saveexec_b64 s[8:9], vcc1481; GFX1264_DPP-NEXT: s_cbranch_execz .LBB2_21482; GFX1264_DPP-NEXT: ; %bb.1:1483; GFX1264_DPP-NEXT: v_mov_b32_e32 v0, s41484; GFX1264_DPP-NEXT: s_mov_b32 s7, 0x310160001485; GFX1264_DPP-NEXT: s_wait_kmcnt 0x01486; GFX1264_DPP-NEXT: s_mov_b32 s4, s21487; GFX1264_DPP-NEXT: s_mov_b32 s5, s31488; GFX1264_DPP-NEXT: buffer_atomic_add_u32 v0, off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV1489; GFX1264_DPP-NEXT: s_wait_loadcnt 0x01490; GFX1264_DPP-NEXT: global_inv scope:SCOPE_DEV1491; GFX1264_DPP-NEXT: .LBB2_2:1492; GFX1264_DPP-NEXT: s_wait_alu depctr_sa_sdst(0)1493; GFX1264_DPP-NEXT: s_or_b64 exec, exec, s[8:9]1494; GFX1264_DPP-NEXT: s_wait_kmcnt 0x01495; GFX1264_DPP-NEXT: v_readfirstlane_b32 s2, v01496; GFX1264_DPP-NEXT: v_mov_b32_e32 v0, v31497; GFX1264_DPP-NEXT: s_mov_b32 s3, 0x310160001498; GFX1264_DPP-NEXT: s_wait_alu depctr_va_sdst(0)1499; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)1500; GFX1264_DPP-NEXT: v_add_nc_u32_e32 v0, s2, v01501; GFX1264_DPP-NEXT: s_mov_b32 s2, s61502; GFX1264_DPP-NEXT: buffer_store_b32 v0, off, s[0:3], null1503; GFX1264_DPP-NEXT: s_endpgm1504;1505; GFX1232_DPP-LABEL: add_i32_varying:1506; GFX1232_DPP: ; %bb.0: ; %entry1507; GFX1232_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v01508; GFX1232_DPP-NEXT: s_or_saveexec_b32 s0, -11509; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)1510; GFX1232_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s01511; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1512; GFX1232_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:11513; GFX1232_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:11514; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1515; GFX1232_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:11516; GFX1232_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:11517; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)1518; GFX1232_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -11519; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s01520; GFX1232_DPP-NEXT: s_load_b128 s[0:3], s[4:5], 0x241521; GFX1232_DPP-NEXT: s_or_saveexec_b32 s4, -11522; GFX1232_DPP-NEXT: v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf1523; GFX1232_DPP-NEXT: v_mov_b32_e32 v3, 01524; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)1525; GFX1232_DPP-NEXT: v_readlane_b32 s6, v1, 311526; GFX1232_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf1527; GFX1232_DPP-NEXT: v_readlane_b32 s5, v1, 151528; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s41529; GFX1232_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)1530; GFX1232_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 01531; GFX1232_DPP-NEXT: s_or_saveexec_b32 s4, -11532; GFX1232_DPP-NEXT: v_writelane_b32 v3, s5, 161533; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s41534; GFX1232_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v01535; GFX1232_DPP-NEXT: s_mov_b32 s4, s61536; GFX1232_DPP-NEXT: s_mov_b32 s6, -11537; GFX1232_DPP-NEXT: ; implicit-def: $vgpr01538; GFX1232_DPP-NEXT: s_and_saveexec_b32 s8, vcc_lo1539; GFX1232_DPP-NEXT: s_cbranch_execz .LBB2_21540; GFX1232_DPP-NEXT: ; %bb.1:1541; GFX1232_DPP-NEXT: s_wait_alu depctr_sa_sdst(0)1542; GFX1232_DPP-NEXT: v_mov_b32_e32 v0, s41543; GFX1232_DPP-NEXT: s_mov_b32 s7, 0x310160001544; GFX1232_DPP-NEXT: s_wait_kmcnt 0x01545; GFX1232_DPP-NEXT: s_mov_b32 s4, s21546; GFX1232_DPP-NEXT: s_mov_b32 s5, s31547; GFX1232_DPP-NEXT: buffer_atomic_add_u32 v0, off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV1548; GFX1232_DPP-NEXT: s_wait_loadcnt 0x01549; GFX1232_DPP-NEXT: global_inv scope:SCOPE_DEV1550; GFX1232_DPP-NEXT: .LBB2_2:1551; GFX1232_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s81552; GFX1232_DPP-NEXT: s_wait_kmcnt 0x01553; GFX1232_DPP-NEXT: v_readfirstlane_b32 s2, v01554; GFX1232_DPP-NEXT: v_mov_b32_e32 v0, v31555; GFX1232_DPP-NEXT: s_mov_b32 s3, 0x310160001556; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)1557; GFX1232_DPP-NEXT: v_add_nc_u32_e32 v0, s2, v01558; GFX1232_DPP-NEXT: s_mov_b32 s2, s61559; GFX1232_DPP-NEXT: buffer_store_b32 v0, off, s[0:3], null1560; GFX1232_DPP-NEXT: s_endpgm1561entry:1562 %lane = call i32 @llvm.amdgcn.workitem.id.x()1563 %old = atomicrmw add ptr addrspace(1) %inout, i32 %lane syncscope("agent") acq_rel1564 store i32 %old, ptr addrspace(1) %out1565 ret void1566}1567 1568define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace(1) %inout) {1569; GFX7LESS-LABEL: add_i64_constant:1570; GFX7LESS: ; %bb.0: ; %entry1571; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec1572; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91573; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 01574; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s7, v01575; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v21576; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr11577; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc1578; GFX7LESS-NEXT: s_cbranch_execz .LBB3_21579; GFX7LESS-NEXT: ; %bb.1:1580; GFX7LESS-NEXT: s_mov_b32 s11, 0xf0001581; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]1582; GFX7LESS-NEXT: s_mul_i32 s6, s6, 51583; GFX7LESS-NEXT: s_mov_b32 s10, -11584; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)1585; GFX7LESS-NEXT: s_mov_b32 s8, s21586; GFX7LESS-NEXT: s_mov_b32 s9, s31587; GFX7LESS-NEXT: v_mov_b32_e32 v0, s61588; GFX7LESS-NEXT: v_mov_b32_e32 v1, 01589; GFX7LESS-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc1590; GFX7LESS-NEXT: s_waitcnt vmcnt(0)1591; GFX7LESS-NEXT: buffer_wbinvl11592; GFX7LESS-NEXT: .LBB3_2:1593; GFX7LESS-NEXT: s_or_b64 exec, exec, s[4:5]1594; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)1595; GFX7LESS-NEXT: s_mov_b32 s3, 0xf0001596; GFX7LESS-NEXT: s_mov_b32 s2, -11597; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v11598; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v01599; GFX7LESS-NEXT: s_waitcnt expcnt(0)1600; GFX7LESS-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v21601; GFX7LESS-NEXT: v_mul_u32_u24_e32 v0, 5, v21602; GFX7LESS-NEXT: v_mov_b32_e32 v2, s41603; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s5, v01604; GFX7LESS-NEXT: v_addc_u32_e32 v1, vcc, v2, v1, vcc1605; GFX7LESS-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 01606; GFX7LESS-NEXT: s_endpgm1607;1608; GFX8-LABEL: add_i64_constant:1609; GFX8: ; %bb.0: ; %entry1610; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241611; GFX8-NEXT: s_mov_b64 s[6:7], exec1612; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 01613; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v01614; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v21615; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr11616; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc1617; GFX8-NEXT: s_cbranch_execz .LBB3_21618; GFX8-NEXT: ; %bb.1:1619; GFX8-NEXT: s_waitcnt lgkmcnt(0)1620; GFX8-NEXT: s_mov_b32 s8, s21621; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[6:7]1622; GFX8-NEXT: s_mul_i32 s2, s2, 51623; GFX8-NEXT: s_mov_b32 s11, 0xf0001624; GFX8-NEXT: s_mov_b32 s10, -11625; GFX8-NEXT: s_mov_b32 s9, s31626; GFX8-NEXT: v_mov_b32_e32 v0, s21627; GFX8-NEXT: v_mov_b32_e32 v1, 01628; GFX8-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc1629; GFX8-NEXT: s_waitcnt vmcnt(0)1630; GFX8-NEXT: buffer_wbinvl1_vol1631; GFX8-NEXT: .LBB3_2:1632; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]1633; GFX8-NEXT: s_waitcnt lgkmcnt(0)1634; GFX8-NEXT: v_readfirstlane_b32 s2, v11635; GFX8-NEXT: v_readfirstlane_b32 s3, v01636; GFX8-NEXT: v_mov_b32_e32 v0, s31637; GFX8-NEXT: v_mov_b32_e32 v1, s21638; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v2, 5, v[0:1]1639; GFX8-NEXT: s_mov_b32 s3, 0xf0001640; GFX8-NEXT: s_mov_b32 s2, -11641; GFX8-NEXT: s_nop 21642; GFX8-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 01643; GFX8-NEXT: s_endpgm1644;1645; GFX9-LABEL: add_i64_constant:1646; GFX9: ; %bb.0: ; %entry1647; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241648; GFX9-NEXT: s_mov_b64 s[6:7], exec1649; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 01650; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v01651; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v21652; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr11653; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc1654; GFX9-NEXT: s_cbranch_execz .LBB3_21655; GFX9-NEXT: ; %bb.1:1656; GFX9-NEXT: s_waitcnt lgkmcnt(0)1657; GFX9-NEXT: s_mov_b32 s8, s21658; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[6:7]1659; GFX9-NEXT: s_mul_i32 s2, s2, 51660; GFX9-NEXT: s_mov_b32 s11, 0xf0001661; GFX9-NEXT: s_mov_b32 s10, -11662; GFX9-NEXT: s_mov_b32 s9, s31663; GFX9-NEXT: v_mov_b32_e32 v0, s21664; GFX9-NEXT: v_mov_b32_e32 v1, 01665; GFX9-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc1666; GFX9-NEXT: s_waitcnt vmcnt(0)1667; GFX9-NEXT: buffer_wbinvl1_vol1668; GFX9-NEXT: .LBB3_2:1669; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]1670; GFX9-NEXT: s_waitcnt lgkmcnt(0)1671; GFX9-NEXT: v_readfirstlane_b32 s2, v11672; GFX9-NEXT: v_readfirstlane_b32 s3, v01673; GFX9-NEXT: v_mov_b32_e32 v0, s31674; GFX9-NEXT: v_mov_b32_e32 v1, s21675; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v2, 5, v[0:1]1676; GFX9-NEXT: s_mov_b32 s3, 0xf0001677; GFX9-NEXT: s_mov_b32 s2, -11678; GFX9-NEXT: s_nop 21679; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 01680; GFX9-NEXT: s_endpgm1681;1682; GFX1064-LABEL: add_i64_constant:1683; GFX1064: ; %bb.0: ; %entry1684; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241685; GFX1064-NEXT: s_mov_b64 s[6:7], exec1686; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 01687; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v01688; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr11689; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v21690; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc1691; GFX1064-NEXT: s_cbranch_execz .LBB3_21692; GFX1064-NEXT: ; %bb.1:1693; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]1694; GFX1064-NEXT: v_mov_b32_e32 v1, 01695; GFX1064-NEXT: s_mul_i32 s6, s6, 51696; GFX1064-NEXT: s_mov_b32 s11, 0x310160001697; GFX1064-NEXT: v_mov_b32_e32 v0, s61698; GFX1064-NEXT: s_mov_b32 s10, -11699; GFX1064-NEXT: s_waitcnt lgkmcnt(0)1700; GFX1064-NEXT: s_mov_b32 s8, s21701; GFX1064-NEXT: s_mov_b32 s9, s31702; GFX1064-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc1703; GFX1064-NEXT: s_waitcnt vmcnt(0)1704; GFX1064-NEXT: buffer_gl1_inv1705; GFX1064-NEXT: buffer_gl0_inv1706; GFX1064-NEXT: .LBB3_2:1707; GFX1064-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)1708; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5]1709; GFX1064-NEXT: s_waitcnt lgkmcnt(0)1710; GFX1064-NEXT: v_readfirstlane_b32 s3, v11711; GFX1064-NEXT: v_readfirstlane_b32 s2, v01712; GFX1064-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v2, 5, s[2:3]1713; GFX1064-NEXT: s_mov_b32 s3, 0x310160001714; GFX1064-NEXT: s_mov_b32 s2, -11715; GFX1064-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 01716; GFX1064-NEXT: s_endpgm1717;1718; GFX1032-LABEL: add_i64_constant:1719; GFX1032: ; %bb.0: ; %entry1720; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241721; GFX1032-NEXT: s_mov_b32 s6, exec_lo1722; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr11723; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 01724; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v21725; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo1726; GFX1032-NEXT: s_cbranch_execz .LBB3_21727; GFX1032-NEXT: ; %bb.1:1728; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s61729; GFX1032-NEXT: v_mov_b32_e32 v1, 01730; GFX1032-NEXT: s_mul_i32 s5, s5, 51731; GFX1032-NEXT: s_mov_b32 s11, 0x310160001732; GFX1032-NEXT: v_mov_b32_e32 v0, s51733; GFX1032-NEXT: s_mov_b32 s10, -11734; GFX1032-NEXT: s_waitcnt lgkmcnt(0)1735; GFX1032-NEXT: s_mov_b32 s8, s21736; GFX1032-NEXT: s_mov_b32 s9, s31737; GFX1032-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc1738; GFX1032-NEXT: s_waitcnt vmcnt(0)1739; GFX1032-NEXT: buffer_gl1_inv1740; GFX1032-NEXT: buffer_gl0_inv1741; GFX1032-NEXT: .LBB3_2:1742; GFX1032-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)1743; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s41744; GFX1032-NEXT: s_waitcnt lgkmcnt(0)1745; GFX1032-NEXT: v_readfirstlane_b32 s3, v11746; GFX1032-NEXT: v_readfirstlane_b32 s2, v01747; GFX1032-NEXT: v_mad_u64_u32 v[0:1], s2, v2, 5, s[2:3]1748; GFX1032-NEXT: s_mov_b32 s3, 0x310160001749; GFX1032-NEXT: s_mov_b32 s2, -11750; GFX1032-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 01751; GFX1032-NEXT: s_endpgm1752;1753; GFX1164-LABEL: add_i64_constant:1754; GFX1164: ; %bb.0: ; %entry1755; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x241756; GFX1164-NEXT: s_mov_b64 s[6:7], exec1757; GFX1164-NEXT: s_mov_b64 s[4:5], exec1758; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 01759; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1760; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v01761; GFX1164-NEXT: ; implicit-def: $vgpr0_vgpr11762; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v21763; GFX1164-NEXT: s_cbranch_execz .LBB3_21764; GFX1164-NEXT: ; %bb.1:1765; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]1766; GFX1164-NEXT: v_mov_b32_e32 v1, 01767; GFX1164-NEXT: s_mul_i32 s6, s6, 51768; GFX1164-NEXT: s_mov_b32 s11, 0x310160001769; GFX1164-NEXT: v_mov_b32_e32 v0, s61770; GFX1164-NEXT: s_mov_b32 s10, -11771; GFX1164-NEXT: s_waitcnt lgkmcnt(0)1772; GFX1164-NEXT: s_mov_b32 s8, s21773; GFX1164-NEXT: s_mov_b32 s9, s31774; GFX1164-NEXT: buffer_atomic_add_u64 v[0:1], off, s[8:11], 0 glc1775; GFX1164-NEXT: s_waitcnt vmcnt(0)1776; GFX1164-NEXT: buffer_gl1_inv1777; GFX1164-NEXT: buffer_gl0_inv1778; GFX1164-NEXT: .LBB3_2:1779; GFX1164-NEXT: s_or_b64 exec, exec, s[4:5]1780; GFX1164-NEXT: s_waitcnt lgkmcnt(0)1781; GFX1164-NEXT: v_readfirstlane_b32 s3, v11782; GFX1164-NEXT: v_readfirstlane_b32 s2, v01783; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)1784; GFX1164-NEXT: v_mad_u64_u32 v[0:1], null, v2, 5, s[2:3]1785; GFX1164-NEXT: s_mov_b32 s3, 0x310160001786; GFX1164-NEXT: s_mov_b32 s2, -11787; GFX1164-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 01788; GFX1164-NEXT: s_endpgm1789;1790; GFX1132-LABEL: add_i64_constant:1791; GFX1132: ; %bb.0: ; %entry1792; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x241793; GFX1132-NEXT: s_mov_b32 s6, exec_lo1794; GFX1132-NEXT: s_mov_b32 s4, exec_lo1795; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 01796; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr11797; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)1798; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v21799; GFX1132-NEXT: s_cbranch_execz .LBB3_21800; GFX1132-NEXT: ; %bb.1:1801; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s61802; GFX1132-NEXT: s_mov_b32 s11, 0x310160001803; GFX1132-NEXT: s_mul_i32 s5, s5, 51804; GFX1132-NEXT: s_mov_b32 s10, -11805; GFX1132-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, 01806; GFX1132-NEXT: s_waitcnt lgkmcnt(0)1807; GFX1132-NEXT: s_mov_b32 s8, s21808; GFX1132-NEXT: s_mov_b32 s9, s31809; GFX1132-NEXT: buffer_atomic_add_u64 v[0:1], off, s[8:11], 0 glc1810; GFX1132-NEXT: s_waitcnt vmcnt(0)1811; GFX1132-NEXT: buffer_gl1_inv1812; GFX1132-NEXT: buffer_gl0_inv1813; GFX1132-NEXT: .LBB3_2:1814; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s41815; GFX1132-NEXT: s_waitcnt lgkmcnt(0)1816; GFX1132-NEXT: v_readfirstlane_b32 s3, v11817; GFX1132-NEXT: v_readfirstlane_b32 s2, v01818; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)1819; GFX1132-NEXT: v_mad_u64_u32 v[0:1], null, v2, 5, s[2:3]1820; GFX1132-NEXT: s_mov_b32 s3, 0x310160001821; GFX1132-NEXT: s_mov_b32 s2, -11822; GFX1132-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 01823; GFX1132-NEXT: s_endpgm1824;1825; GFX1264-LABEL: add_i64_constant:1826; GFX1264: ; %bb.0: ; %entry1827; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x241828; GFX1264-NEXT: s_mov_b64 s[6:7], exec1829; GFX1264-NEXT: s_mov_b64 s[4:5], exec1830; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 01831; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1832; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v01833; GFX1264-NEXT: ; implicit-def: $vgpr0_vgpr11834; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v21835; GFX1264-NEXT: s_cbranch_execz .LBB3_21836; GFX1264-NEXT: ; %bb.1:1837; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[6:7]1838; GFX1264-NEXT: v_mov_b32_e32 v1, 01839; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)1840; GFX1264-NEXT: s_mul_i32 s6, s6, 51841; GFX1264-NEXT: s_mov_b32 s11, 0x310160001842; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)1843; GFX1264-NEXT: v_mov_b32_e32 v0, s61844; GFX1264-NEXT: s_mov_b32 s10, -11845; GFX1264-NEXT: s_wait_kmcnt 0x01846; GFX1264-NEXT: s_mov_b32 s8, s21847; GFX1264-NEXT: s_mov_b32 s9, s31848; GFX1264-NEXT: buffer_atomic_add_u64 v[0:1], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV1849; GFX1264-NEXT: s_wait_loadcnt 0x01850; GFX1264-NEXT: global_inv scope:SCOPE_DEV1851; GFX1264-NEXT: .LBB3_2:1852; GFX1264-NEXT: s_or_b64 exec, exec, s[4:5]1853; GFX1264-NEXT: s_wait_kmcnt 0x01854; GFX1264-NEXT: v_readfirstlane_b32 s3, v11855; GFX1264-NEXT: v_readfirstlane_b32 s2, v01856; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)1857; GFX1264-NEXT: v_mad_co_u64_u32 v[0:1], null, v2, 5, s[2:3]1858; GFX1264-NEXT: s_mov_b32 s3, 0x310160001859; GFX1264-NEXT: s_mov_b32 s2, -11860; GFX1264-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null1861; GFX1264-NEXT: s_endpgm1862;1863; GFX1232-LABEL: add_i64_constant:1864; GFX1232: ; %bb.0: ; %entry1865; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x241866; GFX1232-NEXT: s_mov_b32 s6, exec_lo1867; GFX1232-NEXT: s_mov_b32 s4, exec_lo1868; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 01869; GFX1232-NEXT: ; implicit-def: $vgpr0_vgpr11870; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)1871; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v21872; GFX1232-NEXT: s_cbranch_execz .LBB3_21873; GFX1232-NEXT: ; %bb.1:1874; GFX1232-NEXT: s_bcnt1_i32_b32 s5, s61875; GFX1232-NEXT: s_mov_b32 s11, 0x310160001876; GFX1232-NEXT: s_mul_i32 s5, s5, 51877; GFX1232-NEXT: s_mov_b32 s10, -11878; GFX1232-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, 01879; GFX1232-NEXT: s_wait_kmcnt 0x01880; GFX1232-NEXT: s_mov_b32 s8, s21881; GFX1232-NEXT: s_mov_b32 s9, s31882; GFX1232-NEXT: buffer_atomic_add_u64 v[0:1], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV1883; GFX1232-NEXT: s_wait_loadcnt 0x01884; GFX1232-NEXT: global_inv scope:SCOPE_DEV1885; GFX1232-NEXT: .LBB3_2:1886; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s41887; GFX1232-NEXT: s_wait_kmcnt 0x01888; GFX1232-NEXT: v_readfirstlane_b32 s3, v11889; GFX1232-NEXT: v_readfirstlane_b32 s2, v01890; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)1891; GFX1232-NEXT: v_mad_co_u64_u32 v[0:1], null, v2, 5, s[2:3]1892; GFX1232-NEXT: s_mov_b32 s3, 0x310160001893; GFX1232-NEXT: s_mov_b32 s2, -11894; GFX1232-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null1895; GFX1232-NEXT: s_endpgm1896entry:1897 %old = atomicrmw add ptr addrspace(1) %inout, i64 5 syncscope("agent") acq_rel1898 store i64 %old, ptr addrspace(1) %out1899 ret void1900}1901 1902define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(1) %inout, i64 %additive) {1903; GFX7LESS-LABEL: add_i64_uniform:1904; GFX7LESS: ; %bb.0: ; %entry1905; GFX7LESS-NEXT: s_mov_b64 s[8:9], exec1906; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x91907; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd1908; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s8, 01909; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s9, v01910; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v21911; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr11912; GFX7LESS-NEXT: s_and_saveexec_b64 s[6:7], vcc1913; GFX7LESS-NEXT: s_cbranch_execz .LBB4_21914; GFX7LESS-NEXT: ; %bb.1:1915; GFX7LESS-NEXT: s_mov_b32 s15, 0xf0001916; GFX7LESS-NEXT: s_mov_b32 s14, -11917; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)1918; GFX7LESS-NEXT: s_mov_b32 s12, s21919; GFX7LESS-NEXT: s_mov_b32 s13, s31920; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[8:9]1921; GFX7LESS-NEXT: s_mul_i32 s3, s5, s21922; GFX7LESS-NEXT: v_mov_b32_e32 v0, s21923; GFX7LESS-NEXT: v_mul_hi_u32 v0, s4, v01924; GFX7LESS-NEXT: s_mul_i32 s2, s4, s21925; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, s3, v01926; GFX7LESS-NEXT: v_mov_b32_e32 v0, s21927; GFX7LESS-NEXT: buffer_atomic_add_x2 v[0:1], off, s[12:15], 0 glc1928; GFX7LESS-NEXT: s_waitcnt vmcnt(0)1929; GFX7LESS-NEXT: buffer_wbinvl11930; GFX7LESS-NEXT: .LBB4_2:1931; GFX7LESS-NEXT: s_or_b64 exec, exec, s[6:7]1932; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)1933; GFX7LESS-NEXT: s_mov_b32 s3, 0xf0001934; GFX7LESS-NEXT: s_mov_b32 s2, -11935; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v11936; GFX7LESS-NEXT: v_readfirstlane_b32 s7, v01937; GFX7LESS-NEXT: s_waitcnt expcnt(0)1938; GFX7LESS-NEXT: v_mul_lo_u32 v0, s5, v21939; GFX7LESS-NEXT: v_mul_hi_u32 v1, s4, v21940; GFX7LESS-NEXT: v_mul_lo_u32 v2, s4, v21941; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, v1, v01942; GFX7LESS-NEXT: v_mov_b32_e32 v3, s61943; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s7, v21944; GFX7LESS-NEXT: v_addc_u32_e32 v1, vcc, v3, v1, vcc1945; GFX7LESS-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 01946; GFX7LESS-NEXT: s_endpgm1947;1948; GFX8-LABEL: add_i64_uniform:1949; GFX8: ; %bb.0: ; %entry1950; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241951; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x341952; GFX8-NEXT: s_mov_b64 s[8:9], exec1953; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 01954; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v01955; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v21956; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr11957; GFX8-NEXT: s_and_saveexec_b64 s[6:7], vcc1958; GFX8-NEXT: s_cbranch_execz .LBB4_21959; GFX8-NEXT: ; %bb.1:1960; GFX8-NEXT: s_waitcnt lgkmcnt(0)1961; GFX8-NEXT: s_mov_b32 s12, s21962; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[8:9]1963; GFX8-NEXT: v_mov_b32_e32 v0, s21964; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[8:9], s4, v0, 01965; GFX8-NEXT: s_mul_i32 s2, s5, s21966; GFX8-NEXT: s_mov_b32 s15, 0xf0001967; GFX8-NEXT: s_mov_b32 s14, -11968; GFX8-NEXT: s_mov_b32 s13, s31969; GFX8-NEXT: v_add_u32_e32 v1, vcc, s2, v11970; GFX8-NEXT: buffer_atomic_add_x2 v[0:1], off, s[12:15], 0 glc1971; GFX8-NEXT: s_waitcnt vmcnt(0)1972; GFX8-NEXT: buffer_wbinvl1_vol1973; GFX8-NEXT: .LBB4_2:1974; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]1975; GFX8-NEXT: s_waitcnt lgkmcnt(0)1976; GFX8-NEXT: v_readfirstlane_b32 s2, v11977; GFX8-NEXT: v_readfirstlane_b32 s3, v01978; GFX8-NEXT: v_mov_b32_e32 v0, s31979; GFX8-NEXT: v_mov_b32_e32 v1, s21980; GFX8-NEXT: v_mul_lo_u32 v3, s5, v21981; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s4, v2, v[0:1]1982; GFX8-NEXT: s_mov_b32 s3, 0xf0001983; GFX8-NEXT: s_mov_b32 s2, -11984; GFX8-NEXT: v_add_u32_e32 v1, vcc, v3, v11985; GFX8-NEXT: s_nop 11986; GFX8-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 01987; GFX8-NEXT: s_endpgm1988;1989; GFX9-LABEL: add_i64_uniform:1990; GFX9: ; %bb.0: ; %entry1991; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x241992; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x341993; GFX9-NEXT: s_mov_b64 s[8:9], exec1994; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 01995; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v01996; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v21997; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr11998; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc1999; GFX9-NEXT: s_cbranch_execz .LBB4_22000; GFX9-NEXT: ; %bb.1:2001; GFX9-NEXT: s_waitcnt lgkmcnt(0)2002; GFX9-NEXT: s_mov_b32 s12, s22003; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[8:9]2004; GFX9-NEXT: s_mov_b32 s13, s32005; GFX9-NEXT: s_mul_i32 s3, s7, s22006; GFX9-NEXT: s_mul_hi_u32 s8, s6, s22007; GFX9-NEXT: s_add_i32 s8, s8, s32008; GFX9-NEXT: s_mul_i32 s2, s6, s22009; GFX9-NEXT: s_mov_b32 s15, 0xf0002010; GFX9-NEXT: s_mov_b32 s14, -12011; GFX9-NEXT: v_mov_b32_e32 v0, s22012; GFX9-NEXT: v_mov_b32_e32 v1, s82013; GFX9-NEXT: buffer_atomic_add_x2 v[0:1], off, s[12:15], 0 glc2014; GFX9-NEXT: s_waitcnt vmcnt(0)2015; GFX9-NEXT: buffer_wbinvl1_vol2016; GFX9-NEXT: .LBB4_2:2017; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]2018; GFX9-NEXT: s_waitcnt lgkmcnt(0)2019; GFX9-NEXT: v_readfirstlane_b32 s2, v12020; GFX9-NEXT: v_readfirstlane_b32 s3, v02021; GFX9-NEXT: v_mov_b32_e32 v0, s32022; GFX9-NEXT: v_mov_b32_e32 v1, s22023; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s6, v2, v[0:1]2024; GFX9-NEXT: v_mad_u64_u32 v[1:2], s[2:3], s7, v2, v[1:2]2025; GFX9-NEXT: s_mov_b32 s3, 0xf0002026; GFX9-NEXT: s_mov_b32 s2, -12027; GFX9-NEXT: s_nop 22028; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 02029; GFX9-NEXT: s_endpgm2030;2031; GFX1064-LABEL: add_i64_uniform:2032; GFX1064: ; %bb.0: ; %entry2033; GFX1064-NEXT: s_clause 0x12034; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242035; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342036; GFX1064-NEXT: s_mov_b64 s[8:9], exec2037; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 02038; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v02039; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr12040; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v22041; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc2042; GFX1064-NEXT: s_cbranch_execz .LBB4_22043; GFX1064-NEXT: ; %bb.1:2044; GFX1064-NEXT: s_bcnt1_i32_b64 s8, s[8:9]2045; GFX1064-NEXT: s_mov_b32 s11, 0x310160002046; GFX1064-NEXT: s_waitcnt lgkmcnt(0)2047; GFX1064-NEXT: s_mul_i32 s9, s7, s82048; GFX1064-NEXT: s_mul_hi_u32 s10, s6, s82049; GFX1064-NEXT: s_mul_i32 s8, s6, s82050; GFX1064-NEXT: s_add_i32 s10, s10, s92051; GFX1064-NEXT: v_mov_b32_e32 v0, s82052; GFX1064-NEXT: v_mov_b32_e32 v1, s102053; GFX1064-NEXT: s_mov_b32 s10, -12054; GFX1064-NEXT: s_mov_b32 s8, s22055; GFX1064-NEXT: s_mov_b32 s9, s32056; GFX1064-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc2057; GFX1064-NEXT: s_waitcnt vmcnt(0)2058; GFX1064-NEXT: buffer_gl1_inv2059; GFX1064-NEXT: buffer_gl0_inv2060; GFX1064-NEXT: .LBB4_2:2061; GFX1064-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)2062; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5]2063; GFX1064-NEXT: s_waitcnt lgkmcnt(0)2064; GFX1064-NEXT: v_readfirstlane_b32 s3, v12065; GFX1064-NEXT: v_readfirstlane_b32 s2, v02066; GFX1064-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s6, v2, s[2:3]2067; GFX1064-NEXT: v_mad_u64_u32 v[1:2], s[2:3], s7, v2, v[1:2]2068; GFX1064-NEXT: s_mov_b32 s3, 0x310160002069; GFX1064-NEXT: s_mov_b32 s2, -12070; GFX1064-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 02071; GFX1064-NEXT: s_endpgm2072;2073; GFX1032-LABEL: add_i64_uniform:2074; GFX1032: ; %bb.0: ; %entry2075; GFX1032-NEXT: s_clause 0x12076; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242077; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x342078; GFX1032-NEXT: s_mov_b32 s8, exec_lo2079; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr12080; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s8, 02081; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v22082; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo2083; GFX1032-NEXT: s_cbranch_execz .LBB4_22084; GFX1032-NEXT: ; %bb.1:2085; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s82086; GFX1032-NEXT: s_mov_b32 s11, 0x310160002087; GFX1032-NEXT: s_waitcnt lgkmcnt(0)2088; GFX1032-NEXT: s_mul_i32 s8, s7, s52089; GFX1032-NEXT: s_mul_hi_u32 s9, s6, s52090; GFX1032-NEXT: s_mul_i32 s5, s6, s52091; GFX1032-NEXT: s_add_i32 s9, s9, s82092; GFX1032-NEXT: v_mov_b32_e32 v0, s52093; GFX1032-NEXT: v_mov_b32_e32 v1, s92094; GFX1032-NEXT: s_mov_b32 s10, -12095; GFX1032-NEXT: s_mov_b32 s8, s22096; GFX1032-NEXT: s_mov_b32 s9, s32097; GFX1032-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc2098; GFX1032-NEXT: s_waitcnt vmcnt(0)2099; GFX1032-NEXT: buffer_gl1_inv2100; GFX1032-NEXT: buffer_gl0_inv2101; GFX1032-NEXT: .LBB4_2:2102; GFX1032-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)2103; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s42104; GFX1032-NEXT: s_waitcnt lgkmcnt(0)2105; GFX1032-NEXT: v_readfirstlane_b32 s3, v12106; GFX1032-NEXT: v_readfirstlane_b32 s2, v02107; GFX1032-NEXT: v_mad_u64_u32 v[0:1], s2, s6, v2, s[2:3]2108; GFX1032-NEXT: s_mov_b32 s3, 0x310160002109; GFX1032-NEXT: v_mad_u64_u32 v[1:2], s2, s7, v2, v[1:2]2110; GFX1032-NEXT: s_mov_b32 s2, -12111; GFX1032-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 02112; GFX1032-NEXT: s_endpgm2113;2114; GFX1164-LABEL: add_i64_uniform:2115; GFX1164: ; %bb.0: ; %entry2116; GFX1164-NEXT: s_clause 0x12117; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x242118; GFX1164-NEXT: s_load_b64 s[4:5], s[4:5], 0x342119; GFX1164-NEXT: s_mov_b64 s[8:9], exec2120; GFX1164-NEXT: s_mov_b64 s[6:7], exec2121; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 02122; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2123; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v02124; GFX1164-NEXT: ; implicit-def: $vgpr0_vgpr12125; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v22126; GFX1164-NEXT: s_cbranch_execz .LBB4_22127; GFX1164-NEXT: ; %bb.1:2128; GFX1164-NEXT: s_bcnt1_i32_b64 s8, s[8:9]2129; GFX1164-NEXT: s_mov_b32 s11, 0x310160002130; GFX1164-NEXT: s_waitcnt lgkmcnt(0)2131; GFX1164-NEXT: s_mul_i32 s9, s5, s82132; GFX1164-NEXT: s_mul_hi_u32 s10, s4, s82133; GFX1164-NEXT: s_mul_i32 s8, s4, s82134; GFX1164-NEXT: s_add_i32 s10, s10, s92135; GFX1164-NEXT: v_mov_b32_e32 v0, s82136; GFX1164-NEXT: v_mov_b32_e32 v1, s102137; GFX1164-NEXT: s_mov_b32 s10, -12138; GFX1164-NEXT: s_mov_b32 s8, s22139; GFX1164-NEXT: s_mov_b32 s9, s32140; GFX1164-NEXT: buffer_atomic_add_u64 v[0:1], off, s[8:11], 0 glc2141; GFX1164-NEXT: s_waitcnt vmcnt(0)2142; GFX1164-NEXT: buffer_gl1_inv2143; GFX1164-NEXT: buffer_gl0_inv2144; GFX1164-NEXT: .LBB4_2:2145; GFX1164-NEXT: s_or_b64 exec, exec, s[6:7]2146; GFX1164-NEXT: s_waitcnt lgkmcnt(0)2147; GFX1164-NEXT: v_readfirstlane_b32 s3, v12148; GFX1164-NEXT: v_readfirstlane_b32 s2, v02149; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)2150; GFX1164-NEXT: v_mad_u64_u32 v[0:1], null, s4, v2, s[2:3]2151; GFX1164-NEXT: s_mov_b32 s3, 0x310160002152; GFX1164-NEXT: s_mov_b32 s2, -12153; GFX1164-NEXT: v_mad_u64_u32 v[3:4], null, s5, v2, v[1:2]2154; GFX1164-NEXT: v_mov_b32_e32 v1, v32155; GFX1164-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 02156; GFX1164-NEXT: s_endpgm2157;2158; GFX1132-LABEL: add_i64_uniform:2159; GFX1132: ; %bb.0: ; %entry2160; GFX1132-NEXT: s_clause 0x12161; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x242162; GFX1132-NEXT: s_load_b64 s[4:5], s[4:5], 0x342163; GFX1132-NEXT: s_mov_b32 s7, exec_lo2164; GFX1132-NEXT: s_mov_b32 s6, exec_lo2165; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 02166; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr12167; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)2168; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v22169; GFX1132-NEXT: s_cbranch_execz .LBB4_22170; GFX1132-NEXT: ; %bb.1:2171; GFX1132-NEXT: s_bcnt1_i32_b32 s7, s72172; GFX1132-NEXT: s_mov_b32 s11, 0x310160002173; GFX1132-NEXT: s_waitcnt lgkmcnt(0)2174; GFX1132-NEXT: s_mul_i32 s8, s5, s72175; GFX1132-NEXT: s_mul_hi_u32 s9, s4, s72176; GFX1132-NEXT: s_mul_i32 s7, s4, s72177; GFX1132-NEXT: s_add_i32 s9, s9, s82178; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2179; GFX1132-NEXT: v_dual_mov_b32 v0, s7 :: v_dual_mov_b32 v1, s92180; GFX1132-NEXT: s_mov_b32 s10, -12181; GFX1132-NEXT: s_mov_b32 s8, s22182; GFX1132-NEXT: s_mov_b32 s9, s32183; GFX1132-NEXT: buffer_atomic_add_u64 v[0:1], off, s[8:11], 0 glc2184; GFX1132-NEXT: s_waitcnt vmcnt(0)2185; GFX1132-NEXT: buffer_gl1_inv2186; GFX1132-NEXT: buffer_gl0_inv2187; GFX1132-NEXT: .LBB4_2:2188; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s62189; GFX1132-NEXT: s_waitcnt lgkmcnt(0)2190; GFX1132-NEXT: v_readfirstlane_b32 s3, v12191; GFX1132-NEXT: v_readfirstlane_b32 s2, v02192; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)2193; GFX1132-NEXT: v_mad_u64_u32 v[0:1], null, s4, v2, s[2:3]2194; GFX1132-NEXT: s_mov_b32 s3, 0x310160002195; GFX1132-NEXT: s_mov_b32 s2, -12196; GFX1132-NEXT: v_mad_u64_u32 v[3:4], null, s5, v2, v[1:2]2197; GFX1132-NEXT: v_mov_b32_e32 v1, v32198; GFX1132-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 02199; GFX1132-NEXT: s_endpgm2200;2201; GFX1264-LABEL: add_i64_uniform:2202; GFX1264: ; %bb.0: ; %entry2203; GFX1264-NEXT: s_clause 0x12204; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x242205; GFX1264-NEXT: s_load_b64 s[4:5], s[4:5], 0x342206; GFX1264-NEXT: s_mov_b64 s[8:9], exec2207; GFX1264-NEXT: s_mov_b32 s11, 02208; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 02209; GFX1264-NEXT: s_mov_b64 s[6:7], exec2210; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2211; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v02212; GFX1264-NEXT: ; implicit-def: $vgpr0_vgpr12213; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v22214; GFX1264-NEXT: s_cbranch_execz .LBB4_22215; GFX1264-NEXT: ; %bb.1:2216; GFX1264-NEXT: s_bcnt1_i32_b64 s10, s[8:9]2217; GFX1264-NEXT: s_wait_kmcnt 0x02218; GFX1264-NEXT: s_mul_u64 s[8:9], s[4:5], s[10:11]2219; GFX1264-NEXT: s_mov_b32 s11, 0x310160002220; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)2221; GFX1264-NEXT: v_mov_b32_e32 v0, s82222; GFX1264-NEXT: v_mov_b32_e32 v1, s92223; GFX1264-NEXT: s_mov_b32 s10, -12224; GFX1264-NEXT: s_mov_b32 s8, s22225; GFX1264-NEXT: s_mov_b32 s9, s32226; GFX1264-NEXT: buffer_atomic_add_u64 v[0:1], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV2227; GFX1264-NEXT: s_wait_loadcnt 0x02228; GFX1264-NEXT: global_inv scope:SCOPE_DEV2229; GFX1264-NEXT: .LBB4_2:2230; GFX1264-NEXT: s_or_b64 exec, exec, s[6:7]2231; GFX1264-NEXT: s_wait_kmcnt 0x02232; GFX1264-NEXT: v_readfirstlane_b32 s3, v12233; GFX1264-NEXT: v_readfirstlane_b32 s2, v02234; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)2235; GFX1264-NEXT: v_mad_co_u64_u32 v[0:1], null, s4, v2, s[2:3]2236; GFX1264-NEXT: s_mov_b32 s3, 0x310160002237; GFX1264-NEXT: s_mov_b32 s2, -12238; GFX1264-NEXT: v_mad_co_u64_u32 v[1:2], null, s5, v2, v[1:2]2239; GFX1264-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null2240; GFX1264-NEXT: s_endpgm2241;2242; GFX1232-LABEL: add_i64_uniform:2243; GFX1232: ; %bb.0: ; %entry2244; GFX1232-NEXT: s_clause 0x12245; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x242246; GFX1232-NEXT: s_load_b64 s[4:5], s[4:5], 0x342247; GFX1232-NEXT: s_mov_b32 s6, exec_lo2248; GFX1232-NEXT: s_mov_b32 s7, 02249; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 02250; GFX1232-NEXT: s_mov_b32 s8, exec_lo2251; GFX1232-NEXT: ; implicit-def: $vgpr0_vgpr12252; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)2253; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v22254; GFX1232-NEXT: s_cbranch_execz .LBB4_22255; GFX1232-NEXT: ; %bb.1:2256; GFX1232-NEXT: s_bcnt1_i32_b32 s6, s62257; GFX1232-NEXT: s_mov_b32 s15, 0x310160002258; GFX1232-NEXT: s_wait_kmcnt 0x02259; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)2260; GFX1232-NEXT: s_mul_u64 s[6:7], s[4:5], s[6:7]2261; GFX1232-NEXT: s_mov_b32 s14, -12262; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)2263; GFX1232-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s72264; GFX1232-NEXT: s_mov_b32 s12, s22265; GFX1232-NEXT: s_mov_b32 s13, s32266; GFX1232-NEXT: buffer_atomic_add_u64 v[0:1], off, s[12:15], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV2267; GFX1232-NEXT: s_wait_loadcnt 0x02268; GFX1232-NEXT: global_inv scope:SCOPE_DEV2269; GFX1232-NEXT: .LBB4_2:2270; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s82271; GFX1232-NEXT: s_wait_kmcnt 0x02272; GFX1232-NEXT: v_readfirstlane_b32 s3, v12273; GFX1232-NEXT: v_readfirstlane_b32 s2, v02274; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)2275; GFX1232-NEXT: v_mad_co_u64_u32 v[0:1], null, s4, v2, s[2:3]2276; GFX1232-NEXT: s_mov_b32 s3, 0x310160002277; GFX1232-NEXT: s_mov_b32 s2, -12278; GFX1232-NEXT: v_mad_co_u64_u32 v[1:2], null, s5, v2, v[1:2]2279; GFX1232-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null2280; GFX1232-NEXT: s_endpgm2281entry:2282 %old = atomicrmw add ptr addrspace(1) %inout, i64 %additive syncscope("agent") acq_rel2283 store i64 %old, ptr addrspace(1) %out2284 ret void2285}2286 2287define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(1) %inout) {2288; GFX7LESS_ITERATIVE-LABEL: add_i64_varying:2289; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry2290; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec2291; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v3, 02292; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[6:7], 02293; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr22294; GFX7LESS_ITERATIVE-NEXT: .LBB5_1: ; %ComputeLoop2295; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=12296; GFX7LESS_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]2297; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 m0, s22298; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s3, v3, s22299; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s8, v0, s22300; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v2, s7, m02301; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v1, s6, m02302; GFX7LESS_ITERATIVE-NEXT: s_add_u32 s6, s6, s82303; GFX7LESS_ITERATIVE-NEXT: s_addc_u32 s7, s7, s32304; GFX7LESS_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s22305; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]2306; GFX7LESS_ITERATIVE-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 02307; GFX7LESS_ITERATIVE-NEXT: s_and_b64 vcc, exec, s[2:3]2308; GFX7LESS_ITERATIVE-NEXT: s_cbranch_vccnz .LBB5_12309; GFX7LESS_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd2310; GFX7LESS_ITERATIVE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92311; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 02312; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v02313; GFX7LESS_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v02314; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr3_vgpr42315; GFX7LESS_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc2316; GFX7LESS_ITERATIVE-NEXT: s_xor_b64 s[4:5], exec, s[4:5]2317; GFX7LESS_ITERATIVE-NEXT: s_cbranch_execz .LBB5_42318; GFX7LESS_ITERATIVE-NEXT: ; %bb.3:2319; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s11, 0xf0002320; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s10, -12321; GFX7LESS_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)2322; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s8, s22323; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s9, s32324; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v3, s62325; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v4, s72326; GFX7LESS_ITERATIVE-NEXT: buffer_atomic_add_x2 v[3:4], off, s[8:11], 0 glc2327; GFX7LESS_ITERATIVE-NEXT: s_waitcnt vmcnt(0)2328; GFX7LESS_ITERATIVE-NEXT: buffer_wbinvl12329; GFX7LESS_ITERATIVE-NEXT: .LBB5_4:2330; GFX7LESS_ITERATIVE-NEXT: s_or_b64 exec, exec, s[4:5]2331; GFX7LESS_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)2332; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s3, 0xf0002333; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s2, -12334; GFX7LESS_ITERATIVE-NEXT: v_readfirstlane_b32 s4, v42335; GFX7LESS_ITERATIVE-NEXT: v_readfirstlane_b32 s5, v32336; GFX7LESS_ITERATIVE-NEXT: s_waitcnt expcnt(0)2337; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v3, s42338; GFX7LESS_ITERATIVE-NEXT: v_add_i32_e32 v0, vcc, s5, v12339; GFX7LESS_ITERATIVE-NEXT: v_addc_u32_e32 v1, vcc, v3, v2, vcc2340; GFX7LESS_ITERATIVE-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 02341; GFX7LESS_ITERATIVE-NEXT: s_endpgm2342;2343; GFX8_ITERATIVE-LABEL: add_i64_varying:2344; GFX8_ITERATIVE: ; %bb.0: ; %entry2345; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec2346; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v3, 02347; GFX8_ITERATIVE-NEXT: s_mov_b64 s[6:7], 02348; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr22349; GFX8_ITERATIVE-NEXT: .LBB5_1: ; %ComputeLoop2350; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=12351; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]2352; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s22353; GFX8_ITERATIVE-NEXT: v_readlane_b32 s8, v0, s22354; GFX8_ITERATIVE-NEXT: v_readlane_b32 s3, v3, s22355; GFX8_ITERATIVE-NEXT: v_writelane_b32 v1, s6, m02356; GFX8_ITERATIVE-NEXT: s_add_u32 s6, s6, s82357; GFX8_ITERATIVE-NEXT: v_writelane_b32 v2, s7, m02358; GFX8_ITERATIVE-NEXT: s_addc_u32 s7, s7, s32359; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s22360; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]2361; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB5_12362; GFX8_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd2363; GFX8_ITERATIVE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242364; GFX8_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 02365; GFX8_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v02366; GFX8_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v02367; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr3_vgpr42368; GFX8_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc2369; GFX8_ITERATIVE-NEXT: s_xor_b64 s[4:5], exec, s[4:5]2370; GFX8_ITERATIVE-NEXT: s_cbranch_execz .LBB5_42371; GFX8_ITERATIVE-NEXT: ; %bb.3:2372; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v3, s62373; GFX8_ITERATIVE-NEXT: s_mov_b32 s11, 0xf0002374; GFX8_ITERATIVE-NEXT: s_mov_b32 s10, -12375; GFX8_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)2376; GFX8_ITERATIVE-NEXT: s_mov_b32 s8, s22377; GFX8_ITERATIVE-NEXT: s_mov_b32 s9, s32378; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, s72379; GFX8_ITERATIVE-NEXT: buffer_atomic_add_x2 v[3:4], off, s[8:11], 0 glc2380; GFX8_ITERATIVE-NEXT: s_waitcnt vmcnt(0)2381; GFX8_ITERATIVE-NEXT: buffer_wbinvl1_vol2382; GFX8_ITERATIVE-NEXT: .LBB5_4:2383; GFX8_ITERATIVE-NEXT: s_or_b64 exec, exec, s[4:5]2384; GFX8_ITERATIVE-NEXT: v_readfirstlane_b32 s4, v42385; GFX8_ITERATIVE-NEXT: v_readfirstlane_b32 s5, v32386; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v3, s42387; GFX8_ITERATIVE-NEXT: v_add_u32_e32 v0, vcc, s5, v12388; GFX8_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)2389; GFX8_ITERATIVE-NEXT: s_mov_b32 s3, 0xf0002390; GFX8_ITERATIVE-NEXT: s_mov_b32 s2, -12391; GFX8_ITERATIVE-NEXT: v_addc_u32_e32 v1, vcc, v3, v2, vcc2392; GFX8_ITERATIVE-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 02393; GFX8_ITERATIVE-NEXT: s_endpgm2394;2395; GFX9_ITERATIVE-LABEL: add_i64_varying:2396; GFX9_ITERATIVE: ; %bb.0: ; %entry2397; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec2398; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v3, 02399; GFX9_ITERATIVE-NEXT: s_mov_b64 s[6:7], 02400; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr22401; GFX9_ITERATIVE-NEXT: .LBB5_1: ; %ComputeLoop2402; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=12403; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]2404; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s22405; GFX9_ITERATIVE-NEXT: v_readlane_b32 s8, v0, s22406; GFX9_ITERATIVE-NEXT: v_readlane_b32 s3, v3, s22407; GFX9_ITERATIVE-NEXT: v_writelane_b32 v1, s6, m02408; GFX9_ITERATIVE-NEXT: s_add_u32 s6, s6, s82409; GFX9_ITERATIVE-NEXT: v_writelane_b32 v2, s7, m02410; GFX9_ITERATIVE-NEXT: s_addc_u32 s7, s7, s32411; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s22412; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]2413; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB5_12414; GFX9_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd2415; GFX9_ITERATIVE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242416; GFX9_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 02417; GFX9_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v02418; GFX9_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v02419; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr3_vgpr42420; GFX9_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc2421; GFX9_ITERATIVE-NEXT: s_xor_b64 s[4:5], exec, s[4:5]2422; GFX9_ITERATIVE-NEXT: s_cbranch_execz .LBB5_42423; GFX9_ITERATIVE-NEXT: ; %bb.3:2424; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v3, s62425; GFX9_ITERATIVE-NEXT: s_mov_b32 s11, 0xf0002426; GFX9_ITERATIVE-NEXT: s_mov_b32 s10, -12427; GFX9_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)2428; GFX9_ITERATIVE-NEXT: s_mov_b32 s8, s22429; GFX9_ITERATIVE-NEXT: s_mov_b32 s9, s32430; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, s72431; GFX9_ITERATIVE-NEXT: buffer_atomic_add_x2 v[3:4], off, s[8:11], 0 glc2432; GFX9_ITERATIVE-NEXT: s_waitcnt vmcnt(0)2433; GFX9_ITERATIVE-NEXT: buffer_wbinvl1_vol2434; GFX9_ITERATIVE-NEXT: .LBB5_4:2435; GFX9_ITERATIVE-NEXT: s_or_b64 exec, exec, s[4:5]2436; GFX9_ITERATIVE-NEXT: v_readfirstlane_b32 s4, v42437; GFX9_ITERATIVE-NEXT: v_readfirstlane_b32 s5, v32438; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v3, s42439; GFX9_ITERATIVE-NEXT: v_add_co_u32_e32 v0, vcc, s5, v12440; GFX9_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)2441; GFX9_ITERATIVE-NEXT: s_mov_b32 s3, 0xf0002442; GFX9_ITERATIVE-NEXT: s_mov_b32 s2, -12443; GFX9_ITERATIVE-NEXT: v_addc_co_u32_e32 v1, vcc, v3, v2, vcc2444; GFX9_ITERATIVE-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 02445; GFX9_ITERATIVE-NEXT: s_endpgm2446;2447; GFX1064_ITERATIVE-LABEL: add_i64_varying:2448; GFX1064_ITERATIVE: ; %bb.0: ; %entry2449; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v3, 02450; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec2451; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[6:7], 02452; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr22453; GFX1064_ITERATIVE-NEXT: .LBB5_1: ; %ComputeLoop2454; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=12455; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]2456; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s22457; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s8, v3, s22458; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v2, s7, s22459; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v1, s6, s22460; GFX1064_ITERATIVE-NEXT: s_add_u32 s6, s6, s32461; GFX1064_ITERATIVE-NEXT: s_addc_u32 s7, s7, s82462; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s22463; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]2464; GFX1064_ITERATIVE-NEXT: s_cbranch_scc1 .LBB5_12465; GFX1064_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd2466; GFX1064_ITERATIVE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242467; GFX1064_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 02468; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr3_vgpr42469; GFX1064_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v02470; GFX1064_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v02471; GFX1064_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc2472; GFX1064_ITERATIVE-NEXT: s_xor_b64 s[4:5], exec, s[4:5]2473; GFX1064_ITERATIVE-NEXT: s_cbranch_execz .LBB5_42474; GFX1064_ITERATIVE-NEXT: ; %bb.3:2475; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v3, s62476; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v4, s72477; GFX1064_ITERATIVE-NEXT: s_mov_b32 s11, 0x310160002478; GFX1064_ITERATIVE-NEXT: s_mov_b32 s10, -12479; GFX1064_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)2480; GFX1064_ITERATIVE-NEXT: s_mov_b32 s8, s22481; GFX1064_ITERATIVE-NEXT: s_mov_b32 s9, s32482; GFX1064_ITERATIVE-NEXT: buffer_atomic_add_x2 v[3:4], off, s[8:11], 0 glc2483; GFX1064_ITERATIVE-NEXT: s_waitcnt vmcnt(0)2484; GFX1064_ITERATIVE-NEXT: buffer_gl1_inv2485; GFX1064_ITERATIVE-NEXT: buffer_gl0_inv2486; GFX1064_ITERATIVE-NEXT: .LBB5_4:2487; GFX1064_ITERATIVE-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)2488; GFX1064_ITERATIVE-NEXT: s_or_b64 exec, exec, s[4:5]2489; GFX1064_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)2490; GFX1064_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v32491; GFX1064_ITERATIVE-NEXT: v_readfirstlane_b32 s3, v42492; GFX1064_ITERATIVE-NEXT: v_add_co_u32 v0, vcc, s2, v12493; GFX1064_ITERATIVE-NEXT: v_add_co_ci_u32_e32 v1, vcc, s3, v2, vcc2494; GFX1064_ITERATIVE-NEXT: s_mov_b32 s3, 0x310160002495; GFX1064_ITERATIVE-NEXT: s_mov_b32 s2, -12496; GFX1064_ITERATIVE-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 02497; GFX1064_ITERATIVE-NEXT: s_endpgm2498;2499; GFX1032_ITERATIVE-LABEL: add_i64_varying:2500; GFX1032_ITERATIVE: ; %bb.0: ; %entry2501; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v3, 02502; GFX1032_ITERATIVE-NEXT: s_mov_b32 s0, exec_lo2503; GFX1032_ITERATIVE-NEXT: s_mov_b64 s[6:7], 02504; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr22505; GFX1032_ITERATIVE-NEXT: .LBB5_1: ; %ComputeLoop2506; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=12507; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s1, s02508; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s2, v0, s12509; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s3, v3, s12510; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s7, s12511; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s6, s12512; GFX1032_ITERATIVE-NEXT: s_add_u32 s6, s6, s22513; GFX1032_ITERATIVE-NEXT: s_addc_u32 s7, s7, s32514; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s1, 1, s12515; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s0, s0, s12516; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB5_12517; GFX1032_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd2518; GFX1032_ITERATIVE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242519; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 02520; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr3_vgpr42521; GFX1032_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v02522; GFX1032_ITERATIVE-NEXT: s_and_saveexec_b32 s4, vcc_lo2523; GFX1032_ITERATIVE-NEXT: s_xor_b32 s4, exec_lo, s42524; GFX1032_ITERATIVE-NEXT: s_cbranch_execz .LBB5_42525; GFX1032_ITERATIVE-NEXT: ; %bb.3:2526; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v3, s62527; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v4, s72528; GFX1032_ITERATIVE-NEXT: s_mov_b32 s11, 0x310160002529; GFX1032_ITERATIVE-NEXT: s_mov_b32 s10, -12530; GFX1032_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)2531; GFX1032_ITERATIVE-NEXT: s_mov_b32 s8, s22532; GFX1032_ITERATIVE-NEXT: s_mov_b32 s9, s32533; GFX1032_ITERATIVE-NEXT: buffer_atomic_add_x2 v[3:4], off, s[8:11], 0 glc2534; GFX1032_ITERATIVE-NEXT: s_waitcnt vmcnt(0)2535; GFX1032_ITERATIVE-NEXT: buffer_gl1_inv2536; GFX1032_ITERATIVE-NEXT: buffer_gl0_inv2537; GFX1032_ITERATIVE-NEXT: .LBB5_4:2538; GFX1032_ITERATIVE-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)2539; GFX1032_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s42540; GFX1032_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)2541; GFX1032_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v32542; GFX1032_ITERATIVE-NEXT: v_readfirstlane_b32 s3, v42543; GFX1032_ITERATIVE-NEXT: v_add_co_u32 v0, vcc_lo, s2, v12544; GFX1032_ITERATIVE-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, s3, v2, vcc_lo2545; GFX1032_ITERATIVE-NEXT: s_mov_b32 s3, 0x310160002546; GFX1032_ITERATIVE-NEXT: s_mov_b32 s2, -12547; GFX1032_ITERATIVE-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 02548; GFX1032_ITERATIVE-NEXT: s_endpgm2549;2550; GFX1164_ITERATIVE-LABEL: add_i64_varying:2551; GFX1164_ITERATIVE: ; %bb.0: ; %entry2552; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v2, 02553; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v3, 0x3ff, v02554; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec2555; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[6:7], 02556; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr12557; GFX1164_ITERATIVE-NEXT: .LBB5_1: ; %ComputeLoop2558; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=12559; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s2, s[0:1]2560; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)2561; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s3, v3, s22562; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s8, v2, s22563; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v1, s7, s22564; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v0, s6, s22565; GFX1164_ITERATIVE-NEXT: s_add_u32 s6, s6, s32566; GFX1164_ITERATIVE-NEXT: s_addc_u32 s7, s7, s82567; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s22568; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2569; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]2570; GFX1164_ITERATIVE-NEXT: s_cbranch_scc1 .LBB5_12571; GFX1164_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd2572; GFX1164_ITERATIVE-NEXT: s_load_b128 s[0:3], s[4:5], 0x242573; GFX1164_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 02574; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2575; GFX1164_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v22576; GFX1164_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v22577; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr2_vgpr32578; GFX1164_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc2579; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2580; GFX1164_ITERATIVE-NEXT: s_xor_b64 s[4:5], exec, s[4:5]2581; GFX1164_ITERATIVE-NEXT: s_cbranch_execz .LBB5_42582; GFX1164_ITERATIVE-NEXT: ; %bb.3:2583; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v2, s62584; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v3, s72585; GFX1164_ITERATIVE-NEXT: s_mov_b32 s11, 0x310160002586; GFX1164_ITERATIVE-NEXT: s_mov_b32 s10, -12587; GFX1164_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)2588; GFX1164_ITERATIVE-NEXT: s_mov_b32 s8, s22589; GFX1164_ITERATIVE-NEXT: s_mov_b32 s9, s32590; GFX1164_ITERATIVE-NEXT: buffer_atomic_add_u64 v[2:3], off, s[8:11], 0 glc2591; GFX1164_ITERATIVE-NEXT: s_waitcnt vmcnt(0)2592; GFX1164_ITERATIVE-NEXT: buffer_gl1_inv2593; GFX1164_ITERATIVE-NEXT: buffer_gl0_inv2594; GFX1164_ITERATIVE-NEXT: .LBB5_4:2595; GFX1164_ITERATIVE-NEXT: s_or_b64 exec, exec, s[4:5]2596; GFX1164_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)2597; GFX1164_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v22598; GFX1164_ITERATIVE-NEXT: v_readfirstlane_b32 s3, v32599; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)2600; GFX1164_ITERATIVE-NEXT: v_add_co_u32 v0, vcc, s2, v02601; GFX1164_ITERATIVE-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc2602; GFX1164_ITERATIVE-NEXT: s_mov_b32 s3, 0x310160002603; GFX1164_ITERATIVE-NEXT: s_mov_b32 s2, -12604; GFX1164_ITERATIVE-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 02605; GFX1164_ITERATIVE-NEXT: s_endpgm2606;2607; GFX1132_ITERATIVE-LABEL: add_i64_varying:2608; GFX1132_ITERATIVE: ; %bb.0: ; %entry2609; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v3, 0x3ff, v02610; GFX1132_ITERATIVE-NEXT: s_mov_b32 s0, exec_lo2611; GFX1132_ITERATIVE-NEXT: s_mov_b64 s[6:7], 02612; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr12613; GFX1132_ITERATIVE-NEXT: .LBB5_1: ; %ComputeLoop2614; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=12615; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s1, s02616; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)2617; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s2, v3, s12618; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s3, v2, s12619; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v1, s7, s12620; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s6, s12621; GFX1132_ITERATIVE-NEXT: s_add_u32 s6, s6, s22622; GFX1132_ITERATIVE-NEXT: s_addc_u32 s7, s7, s32623; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s1, 1, s12624; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2625; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s0, s0, s12626; GFX1132_ITERATIVE-NEXT: s_cbranch_scc1 .LBB5_12627; GFX1132_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd2628; GFX1132_ITERATIVE-NEXT: s_load_b128 s[0:3], s[4:5], 0x242629; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 02630; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)2631; GFX1132_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v22632; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr2_vgpr32633; GFX1132_ITERATIVE-NEXT: s_and_saveexec_b32 s4, vcc_lo2634; GFX1132_ITERATIVE-NEXT: s_xor_b32 s4, exec_lo, s42635; GFX1132_ITERATIVE-NEXT: s_cbranch_execz .LBB5_42636; GFX1132_ITERATIVE-NEXT: ; %bb.3:2637; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s72638; GFX1132_ITERATIVE-NEXT: s_mov_b32 s11, 0x310160002639; GFX1132_ITERATIVE-NEXT: s_mov_b32 s10, -12640; GFX1132_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)2641; GFX1132_ITERATIVE-NEXT: s_mov_b32 s8, s22642; GFX1132_ITERATIVE-NEXT: s_mov_b32 s9, s32643; GFX1132_ITERATIVE-NEXT: buffer_atomic_add_u64 v[2:3], off, s[8:11], 0 glc2644; GFX1132_ITERATIVE-NEXT: s_waitcnt vmcnt(0)2645; GFX1132_ITERATIVE-NEXT: buffer_gl1_inv2646; GFX1132_ITERATIVE-NEXT: buffer_gl0_inv2647; GFX1132_ITERATIVE-NEXT: .LBB5_4:2648; GFX1132_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s42649; GFX1132_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)2650; GFX1132_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v22651; GFX1132_ITERATIVE-NEXT: v_readfirstlane_b32 s3, v32652; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)2653; GFX1132_ITERATIVE-NEXT: v_add_co_u32 v0, vcc_lo, s2, v02654; GFX1132_ITERATIVE-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo2655; GFX1132_ITERATIVE-NEXT: s_mov_b32 s3, 0x310160002656; GFX1132_ITERATIVE-NEXT: s_mov_b32 s2, -12657; GFX1132_ITERATIVE-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 02658; GFX1132_ITERATIVE-NEXT: s_endpgm2659;2660; GFX1264_ITERATIVE-LABEL: add_i64_varying:2661; GFX1264_ITERATIVE: ; %bb.0: ; %entry2662; GFX1264_ITERATIVE-NEXT: v_mov_b32_e32 v2, 02663; GFX1264_ITERATIVE-NEXT: v_and_b32_e32 v3, 0x3ff, v02664; GFX1264_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec2665; GFX1264_ITERATIVE-NEXT: s_mov_b64 s[6:7], 02666; GFX1264_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr12667; GFX1264_ITERATIVE-NEXT: .LBB5_1: ; %ComputeLoop2668; GFX1264_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=12669; GFX1264_ITERATIVE-NEXT: s_ctz_i32_b64 s8, s[0:1]2670; GFX1264_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)2671; GFX1264_ITERATIVE-NEXT: v_readlane_b32 s3, v2, s82672; GFX1264_ITERATIVE-NEXT: v_readlane_b32 s2, v3, s82673; GFX1264_ITERATIVE-NEXT: v_writelane_b32 v1, s7, s82674; GFX1264_ITERATIVE-NEXT: v_writelane_b32 v0, s6, s82675; GFX1264_ITERATIVE-NEXT: s_lshl_b64 s[8:9], 1, s82676; GFX1264_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)2677; GFX1264_ITERATIVE-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[8:9]2678; GFX1264_ITERATIVE-NEXT: s_add_nc_u64 s[6:7], s[6:7], s[2:3]2679; GFX1264_ITERATIVE-NEXT: s_cbranch_scc1 .LBB5_12680; GFX1264_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd2681; GFX1264_ITERATIVE-NEXT: s_load_b128 s[0:3], s[4:5], 0x242682; GFX1264_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 02683; GFX1264_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2684; GFX1264_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v22685; GFX1264_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v22686; GFX1264_ITERATIVE-NEXT: ; implicit-def: $vgpr2_vgpr32687; GFX1264_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc2688; GFX1264_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2689; GFX1264_ITERATIVE-NEXT: s_xor_b64 s[4:5], exec, s[4:5]2690; GFX1264_ITERATIVE-NEXT: s_cbranch_execz .LBB5_42691; GFX1264_ITERATIVE-NEXT: ; %bb.3:2692; GFX1264_ITERATIVE-NEXT: v_mov_b32_e32 v2, s62693; GFX1264_ITERATIVE-NEXT: v_mov_b32_e32 v3, s72694; GFX1264_ITERATIVE-NEXT: s_mov_b32 s11, 0x310160002695; GFX1264_ITERATIVE-NEXT: s_mov_b32 s10, -12696; GFX1264_ITERATIVE-NEXT: s_wait_kmcnt 0x02697; GFX1264_ITERATIVE-NEXT: s_mov_b32 s8, s22698; GFX1264_ITERATIVE-NEXT: s_mov_b32 s9, s32699; GFX1264_ITERATIVE-NEXT: buffer_atomic_add_u64 v[2:3], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV2700; GFX1264_ITERATIVE-NEXT: s_wait_loadcnt 0x02701; GFX1264_ITERATIVE-NEXT: global_inv scope:SCOPE_DEV2702; GFX1264_ITERATIVE-NEXT: .LBB5_4:2703; GFX1264_ITERATIVE-NEXT: s_or_b64 exec, exec, s[4:5]2704; GFX1264_ITERATIVE-NEXT: s_wait_kmcnt 0x02705; GFX1264_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v22706; GFX1264_ITERATIVE-NEXT: v_readfirstlane_b32 s3, v32707; GFX1264_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)2708; GFX1264_ITERATIVE-NEXT: v_add_co_u32 v0, vcc, s2, v02709; GFX1264_ITERATIVE-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc2710; GFX1264_ITERATIVE-NEXT: s_mov_b32 s3, 0x310160002711; GFX1264_ITERATIVE-NEXT: s_mov_b32 s2, -12712; GFX1264_ITERATIVE-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null2713; GFX1264_ITERATIVE-NEXT: s_endpgm2714;2715; GFX1232_ITERATIVE-LABEL: add_i64_varying:2716; GFX1232_ITERATIVE: ; %bb.0: ; %entry2717; GFX1232_ITERATIVE-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v3, 0x3ff, v02718; GFX1232_ITERATIVE-NEXT: s_mov_b32 s0, exec_lo2719; GFX1232_ITERATIVE-NEXT: s_mov_b64 s[6:7], 02720; GFX1232_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr12721; GFX1232_ITERATIVE-NEXT: .LBB5_1: ; %ComputeLoop2722; GFX1232_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=12723; GFX1232_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)2724; GFX1232_ITERATIVE-NEXT: s_ctz_i32_b32 s1, s02725; GFX1232_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)2726; GFX1232_ITERATIVE-NEXT: v_readlane_b32 s3, v2, s12727; GFX1232_ITERATIVE-NEXT: v_readlane_b32 s2, v3, s12728; GFX1232_ITERATIVE-NEXT: v_writelane_b32 v1, s7, s12729; GFX1232_ITERATIVE-NEXT: v_writelane_b32 v0, s6, s12730; GFX1232_ITERATIVE-NEXT: s_lshl_b32 s1, 1, s12731; GFX1232_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)2732; GFX1232_ITERATIVE-NEXT: s_and_not1_b32 s0, s0, s12733; GFX1232_ITERATIVE-NEXT: s_add_nc_u64 s[6:7], s[6:7], s[2:3]2734; GFX1232_ITERATIVE-NEXT: s_cbranch_scc1 .LBB5_12735; GFX1232_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd2736; GFX1232_ITERATIVE-NEXT: s_load_b128 s[0:3], s[4:5], 0x242737; GFX1232_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 02738; GFX1232_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)2739; GFX1232_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v22740; GFX1232_ITERATIVE-NEXT: ; implicit-def: $vgpr2_vgpr32741; GFX1232_ITERATIVE-NEXT: s_and_saveexec_b32 s4, vcc_lo2742; GFX1232_ITERATIVE-NEXT: s_xor_b32 s4, exec_lo, s42743; GFX1232_ITERATIVE-NEXT: s_cbranch_execz .LBB5_42744; GFX1232_ITERATIVE-NEXT: ; %bb.3:2745; GFX1232_ITERATIVE-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s72746; GFX1232_ITERATIVE-NEXT: s_mov_b32 s11, 0x310160002747; GFX1232_ITERATIVE-NEXT: s_mov_b32 s10, -12748; GFX1232_ITERATIVE-NEXT: s_wait_kmcnt 0x02749; GFX1232_ITERATIVE-NEXT: s_mov_b32 s8, s22750; GFX1232_ITERATIVE-NEXT: s_mov_b32 s9, s32751; GFX1232_ITERATIVE-NEXT: buffer_atomic_add_u64 v[2:3], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV2752; GFX1232_ITERATIVE-NEXT: s_wait_loadcnt 0x02753; GFX1232_ITERATIVE-NEXT: global_inv scope:SCOPE_DEV2754; GFX1232_ITERATIVE-NEXT: .LBB5_4:2755; GFX1232_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s42756; GFX1232_ITERATIVE-NEXT: s_wait_kmcnt 0x02757; GFX1232_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v22758; GFX1232_ITERATIVE-NEXT: v_readfirstlane_b32 s3, v32759; GFX1232_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)2760; GFX1232_ITERATIVE-NEXT: v_add_co_u32 v0, vcc_lo, s2, v02761; GFX1232_ITERATIVE-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo2762; GFX1232_ITERATIVE-NEXT: s_mov_b32 s3, 0x310160002763; GFX1232_ITERATIVE-NEXT: s_mov_b32 s2, -12764; GFX1232_ITERATIVE-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null2765; GFX1232_ITERATIVE-NEXT: s_endpgm2766;2767; GFX7LESS_DPP-LABEL: add_i64_varying:2768; GFX7LESS_DPP: ; %bb.0: ; %entry2769; GFX7LESS_DPP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x92770; GFX7LESS_DPP-NEXT: s_mov_b32 s7, 0xf0002771; GFX7LESS_DPP-NEXT: s_mov_b32 s6, -12772; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v1, 02773; GFX7LESS_DPP-NEXT: s_mov_b32 s10, s62774; GFX7LESS_DPP-NEXT: s_mov_b32 s11, s72775; GFX7LESS_DPP-NEXT: s_waitcnt lgkmcnt(0)2776; GFX7LESS_DPP-NEXT: s_mov_b32 s8, s22777; GFX7LESS_DPP-NEXT: s_mov_b32 s9, s32778; GFX7LESS_DPP-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc2779; GFX7LESS_DPP-NEXT: s_waitcnt vmcnt(0)2780; GFX7LESS_DPP-NEXT: buffer_wbinvl12781; GFX7LESS_DPP-NEXT: s_mov_b32 s4, s02782; GFX7LESS_DPP-NEXT: s_mov_b32 s5, s12783; GFX7LESS_DPP-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 02784; GFX7LESS_DPP-NEXT: s_endpgm2785;2786; GFX8_DPP-LABEL: add_i64_varying:2787; GFX8_DPP: ; %bb.0: ; %entry2788; GFX8_DPP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242789; GFX8_DPP-NEXT: s_or_saveexec_b64 s[4:5], -12790; GFX8_DPP-NEXT: v_mov_b32_e32 v1, 02791; GFX8_DPP-NEXT: s_mov_b64 exec, s[4:5]2792; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 02793; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v62794; GFX8_DPP-NEXT: s_or_saveexec_b64 s[4:5], -12795; GFX8_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[4:5]2796; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 02797; GFX8_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s[4:5]2798; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 02799; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf2800; GFX8_DPP-NEXT: v_add_u32_e32 v3, vcc, v3, v52801; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf2802; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 02803; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc2804; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 02805; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:2 row_mask:0xf bank_mask:0xf2806; GFX8_DPP-NEXT: v_add_u32_e32 v3, vcc, v3, v52807; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:2 row_mask:0xf bank_mask:0xf2808; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 02809; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc2810; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 02811; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf2812; GFX8_DPP-NEXT: v_add_u32_e32 v3, vcc, v3, v52813; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf2814; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 02815; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc2816; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 02817; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:8 row_mask:0xf bank_mask:0xf2818; GFX8_DPP-NEXT: v_add_u32_e32 v3, vcc, v3, v52819; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:8 row_mask:0xf bank_mask:0xf2820; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 02821; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc2822; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 02823; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:15 row_mask:0xa bank_mask:0xf2824; GFX8_DPP-NEXT: v_add_u32_e32 v3, vcc, v3, v52825; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:15 row_mask:0xa bank_mask:0xf2826; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 02827; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc2828; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 02829; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:31 row_mask:0xc bank_mask:0xf2830; GFX8_DPP-NEXT: v_add_u32_e32 v3, vcc, v3, v52831; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:31 row_mask:0xc bank_mask:0xf2832; GFX8_DPP-NEXT: v_addc_u32_e32 v4, vcc, v2, v4, vcc2833; GFX8_DPP-NEXT: v_mov_b32_e32 v2, 02834; GFX8_DPP-NEXT: v_readlane_b32 s7, v4, 632835; GFX8_DPP-NEXT: v_readlane_b32 s6, v3, 632836; GFX8_DPP-NEXT: v_mov_b32_dpp v2, v4 wave_shr:1 row_mask:0xf bank_mask:0xf2837; GFX8_DPP-NEXT: v_mov_b32_dpp v1, v3 wave_shr:1 row_mask:0xf bank_mask:0xf2838; GFX8_DPP-NEXT: s_mov_b64 exec, s[4:5]2839; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v62840; GFX8_DPP-NEXT: ; implicit-def: $vgpr6_vgpr72841; GFX8_DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc2842; GFX8_DPP-NEXT: s_cbranch_execz .LBB5_22843; GFX8_DPP-NEXT: ; %bb.1:2844; GFX8_DPP-NEXT: v_mov_b32_e32 v6, s62845; GFX8_DPP-NEXT: s_mov_b32 s11, 0xf0002846; GFX8_DPP-NEXT: s_mov_b32 s10, -12847; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0)2848; GFX8_DPP-NEXT: s_mov_b32 s8, s22849; GFX8_DPP-NEXT: s_mov_b32 s9, s32850; GFX8_DPP-NEXT: v_mov_b32_e32 v7, s72851; GFX8_DPP-NEXT: buffer_atomic_add_x2 v[6:7], off, s[8:11], 0 glc2852; GFX8_DPP-NEXT: s_waitcnt vmcnt(0)2853; GFX8_DPP-NEXT: buffer_wbinvl1_vol2854; GFX8_DPP-NEXT: .LBB5_2:2855; GFX8_DPP-NEXT: s_or_b64 exec, exec, s[4:5]2856; GFX8_DPP-NEXT: v_readfirstlane_b32 s4, v72857; GFX8_DPP-NEXT: v_readfirstlane_b32 s5, v62858; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v12859; GFX8_DPP-NEXT: v_mov_b32_e32 v7, v22860; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s42861; GFX8_DPP-NEXT: v_add_u32_e32 v6, vcc, s5, v62862; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0)2863; GFX8_DPP-NEXT: s_mov_b32 s3, 0xf0002864; GFX8_DPP-NEXT: s_mov_b32 s2, -12865; GFX8_DPP-NEXT: v_addc_u32_e32 v7, vcc, v0, v7, vcc2866; GFX8_DPP-NEXT: buffer_store_dwordx2 v[6:7], off, s[0:3], 02867; GFX8_DPP-NEXT: s_endpgm2868;2869; GFX9_DPP-LABEL: add_i64_varying:2870; GFX9_DPP: ; %bb.0: ; %entry2871; GFX9_DPP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x242872; GFX9_DPP-NEXT: s_or_saveexec_b64 s[4:5], -12873; GFX9_DPP-NEXT: v_mov_b32_e32 v1, 02874; GFX9_DPP-NEXT: s_mov_b64 exec, s[4:5]2875; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 02876; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v62877; GFX9_DPP-NEXT: s_or_saveexec_b64 s[4:5], -12878; GFX9_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[4:5]2879; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 02880; GFX9_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s[4:5]2881; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 02882; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf2883; GFX9_DPP-NEXT: v_add_co_u32_e32 v3, vcc, v3, v52884; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf2885; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 02886; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc2887; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 02888; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:2 row_mask:0xf bank_mask:0xf2889; GFX9_DPP-NEXT: v_add_co_u32_e32 v3, vcc, v3, v52890; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:2 row_mask:0xf bank_mask:0xf2891; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 02892; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc2893; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 02894; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf2895; GFX9_DPP-NEXT: v_add_co_u32_e32 v3, vcc, v3, v52896; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf2897; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 02898; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc2899; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 02900; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:8 row_mask:0xf bank_mask:0xf2901; GFX9_DPP-NEXT: v_add_co_u32_e32 v3, vcc, v3, v52902; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:8 row_mask:0xf bank_mask:0xf2903; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 02904; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc2905; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 02906; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:15 row_mask:0xa bank_mask:0xf2907; GFX9_DPP-NEXT: v_add_co_u32_e32 v3, vcc, v3, v52908; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:15 row_mask:0xa bank_mask:0xf2909; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 02910; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc2911; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 02912; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:31 row_mask:0xc bank_mask:0xf2913; GFX9_DPP-NEXT: v_add_co_u32_e32 v3, vcc, v3, v52914; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:31 row_mask:0xc bank_mask:0xf2915; GFX9_DPP-NEXT: v_addc_co_u32_e32 v4, vcc, v2, v4, vcc2916; GFX9_DPP-NEXT: v_mov_b32_e32 v2, 02917; GFX9_DPP-NEXT: v_readlane_b32 s7, v4, 632918; GFX9_DPP-NEXT: v_readlane_b32 s6, v3, 632919; GFX9_DPP-NEXT: v_mov_b32_dpp v2, v4 wave_shr:1 row_mask:0xf bank_mask:0xf2920; GFX9_DPP-NEXT: v_mov_b32_dpp v1, v3 wave_shr:1 row_mask:0xf bank_mask:0xf2921; GFX9_DPP-NEXT: s_mov_b64 exec, s[4:5]2922; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v62923; GFX9_DPP-NEXT: ; implicit-def: $vgpr6_vgpr72924; GFX9_DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc2925; GFX9_DPP-NEXT: s_cbranch_execz .LBB5_22926; GFX9_DPP-NEXT: ; %bb.1:2927; GFX9_DPP-NEXT: v_mov_b32_e32 v6, s62928; GFX9_DPP-NEXT: s_mov_b32 s11, 0xf0002929; GFX9_DPP-NEXT: s_mov_b32 s10, -12930; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0)2931; GFX9_DPP-NEXT: s_mov_b32 s8, s22932; GFX9_DPP-NEXT: s_mov_b32 s9, s32933; GFX9_DPP-NEXT: v_mov_b32_e32 v7, s72934; GFX9_DPP-NEXT: buffer_atomic_add_x2 v[6:7], off, s[8:11], 0 glc2935; GFX9_DPP-NEXT: s_waitcnt vmcnt(0)2936; GFX9_DPP-NEXT: buffer_wbinvl1_vol2937; GFX9_DPP-NEXT: .LBB5_2:2938; GFX9_DPP-NEXT: s_or_b64 exec, exec, s[4:5]2939; GFX9_DPP-NEXT: v_readfirstlane_b32 s4, v72940; GFX9_DPP-NEXT: v_readfirstlane_b32 s5, v62941; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v12942; GFX9_DPP-NEXT: v_mov_b32_e32 v7, v22943; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s42944; GFX9_DPP-NEXT: v_add_co_u32_e32 v6, vcc, s5, v62945; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0)2946; GFX9_DPP-NEXT: s_mov_b32 s3, 0xf0002947; GFX9_DPP-NEXT: s_mov_b32 s2, -12948; GFX9_DPP-NEXT: v_addc_co_u32_e32 v7, vcc, v0, v7, vcc2949; GFX9_DPP-NEXT: buffer_store_dwordx2 v[6:7], off, s[0:3], 02950; GFX9_DPP-NEXT: s_endpgm2951;2952; GFX1064_DPP-LABEL: add_i64_varying:2953; GFX1064_DPP: ; %bb.0: ; %entry2954; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -12955; GFX1064_DPP-NEXT: v_mov_b32_e32 v1, 02956; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]2957; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s[0:1]2958; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 02959; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 02960; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 02961; GFX1064_DPP-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf2962; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, 02963; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:1 row_mask:0xf bank_mask:0xf2964; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v2, v12965; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v3, v4, vcc2966; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 02967; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:2 row_mask:0xf bank_mask:0xf2968; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 02969; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:2 row_mask:0xf bank_mask:0xf2970; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v62971; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc2972; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 02973; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:4 row_mask:0xf bank_mask:0xf2974; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 02975; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v2 row_shr:4 row_mask:0xf bank_mask:0xf2976; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v42977; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v3, vcc2978; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 02979; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:8 row_mask:0xf bank_mask:0xf2980; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:8 row_mask:0xf bank_mask:0xf2981; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v62982; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc2983; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 02984; GFX1064_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -12985; GFX1064_DPP-NEXT: v_permlanex16_b32 v6, v2, -1, -12986; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf2987; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf2988; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v32989; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc2990; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 02991; GFX1064_DPP-NEXT: v_readlane_b32 s2, v1, 312992; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 02993; GFX1064_DPP-NEXT: v_readlane_b32 s3, v2, 312994; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, s22995; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, s32996; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf2997; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf2998; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 02999; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v33000; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc3001; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1]3002; GFX1064_DPP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243003; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 03004; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[4:5], -13005; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:1 row_mask:0xf bank_mask:0xf3006; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v2 row_shr:1 row_mask:0xf bank_mask:0xf3007; GFX1064_DPP-NEXT: v_readlane_b32 s6, v2, 153008; GFX1064_DPP-NEXT: v_readlane_b32 s7, v1, 153009; GFX1064_DPP-NEXT: v_readlane_b32 s8, v2, 313010; GFX1064_DPP-NEXT: v_readlane_b32 s9, v1, 313011; GFX1064_DPP-NEXT: v_readlane_b32 s10, v1, 473012; GFX1064_DPP-NEXT: v_writelane_b32 v7, s6, 163013; GFX1064_DPP-NEXT: v_writelane_b32 v6, s7, 163014; GFX1064_DPP-NEXT: v_readlane_b32 s6, v1, 633015; GFX1064_DPP-NEXT: v_readlane_b32 s11, v2, 473016; GFX1064_DPP-NEXT: v_readlane_b32 s7, v2, 633017; GFX1064_DPP-NEXT: v_writelane_b32 v7, s8, 323018; GFX1064_DPP-NEXT: v_writelane_b32 v6, s9, 323019; GFX1064_DPP-NEXT: s_mov_b64 exec, s[4:5]3020; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v03021; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[8:9], -13022; GFX1064_DPP-NEXT: s_mov_b64 s[4:5], s[6:7]3023; GFX1064_DPP-NEXT: v_writelane_b32 v7, s11, 483024; GFX1064_DPP-NEXT: v_writelane_b32 v6, s10, 483025; GFX1064_DPP-NEXT: s_mov_b64 exec, s[8:9]3026; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v03027; GFX1064_DPP-NEXT: s_mov_b32 s6, -13028; GFX1064_DPP-NEXT: ; implicit-def: $vgpr8_vgpr93029; GFX1064_DPP-NEXT: s_and_saveexec_b64 s[8:9], vcc3030; GFX1064_DPP-NEXT: s_cbranch_execz .LBB5_23031; GFX1064_DPP-NEXT: ; %bb.1:3032; GFX1064_DPP-NEXT: v_mov_b32_e32 v9, s53033; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, s43034; GFX1064_DPP-NEXT: s_mov_b32 s7, 0x310160003035; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)3036; GFX1064_DPP-NEXT: s_mov_b32 s4, s23037; GFX1064_DPP-NEXT: s_mov_b32 s5, s33038; GFX1064_DPP-NEXT: buffer_atomic_add_x2 v[8:9], off, s[4:7], 0 glc3039; GFX1064_DPP-NEXT: s_waitcnt vmcnt(0)3040; GFX1064_DPP-NEXT: buffer_gl1_inv3041; GFX1064_DPP-NEXT: buffer_gl0_inv3042; GFX1064_DPP-NEXT: .LBB5_2:3043; GFX1064_DPP-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)3044; GFX1064_DPP-NEXT: s_or_b64 exec, exec, s[8:9]3045; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)3046; GFX1064_DPP-NEXT: v_readfirstlane_b32 s2, v83047; GFX1064_DPP-NEXT: v_mov_b32_e32 v10, v63048; GFX1064_DPP-NEXT: v_mov_b32_e32 v11, v73049; GFX1064_DPP-NEXT: v_readfirstlane_b32 s3, v93050; GFX1064_DPP-NEXT: v_add_co_u32 v8, vcc, s2, v103051; GFX1064_DPP-NEXT: s_mov_b32 s2, s63052; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v9, vcc, s3, v11, vcc3053; GFX1064_DPP-NEXT: s_mov_b32 s3, 0x310160003054; GFX1064_DPP-NEXT: buffer_store_dwordx2 v[8:9], off, s[0:3], 03055; GFX1064_DPP-NEXT: s_endpgm3056;3057; GFX1032_DPP-LABEL: add_i64_varying:3058; GFX1032_DPP: ; %bb.0: ; %entry3059; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -13060; GFX1032_DPP-NEXT: v_mov_b32_e32 v1, 03061; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s03062; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s03063; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 03064; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, 03065; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 03066; GFX1032_DPP-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf3067; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, 03068; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:1 row_mask:0xf bank_mask:0xf3069; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, 03070; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v2, v13071; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v3, v4, vcc_lo3072; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 03073; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:2 row_mask:0xf bank_mask:0xf3074; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 03075; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:2 row_mask:0xf bank_mask:0xf3076; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v63077; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo3078; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, 03079; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:4 row_mask:0xf bank_mask:0xf3080; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 03081; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v2 row_shr:4 row_mask:0xf bank_mask:0xf3082; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v43083; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v3, vcc_lo3084; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 03085; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:8 row_mask:0xf bank_mask:0xf3086; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:8 row_mask:0xf bank_mask:0xf3087; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v63088; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo3089; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 03090; GFX1032_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -13091; GFX1032_DPP-NEXT: v_permlanex16_b32 v6, v2, -1, -13092; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf3093; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf3094; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s03095; GFX1032_DPP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243096; GFX1032_DPP-NEXT: s_or_saveexec_b32 s6, -13097; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v33098; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo3099; GFX1032_DPP-NEXT: v_readlane_b32 s4, v1, 313100; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf3101; GFX1032_DPP-NEXT: v_readlane_b32 s8, v2, 153102; GFX1032_DPP-NEXT: v_readlane_b32 s5, v2, 313103; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v2 row_shr:1 row_mask:0xf bank_mask:0xf3104; GFX1032_DPP-NEXT: v_readlane_b32 s7, v1, 153105; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s63106; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 03107; GFX1032_DPP-NEXT: s_or_saveexec_b32 s6, -13108; GFX1032_DPP-NEXT: v_writelane_b32 v8, s8, 163109; GFX1032_DPP-NEXT: v_writelane_b32 v7, s7, 163110; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s63111; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v03112; GFX1032_DPP-NEXT: s_mov_b32 s6, -13113; GFX1032_DPP-NEXT: ; implicit-def: $vgpr9_vgpr103114; GFX1032_DPP-NEXT: s_and_saveexec_b32 s8, vcc_lo3115; GFX1032_DPP-NEXT: s_cbranch_execz .LBB5_23116; GFX1032_DPP-NEXT: ; %bb.1:3117; GFX1032_DPP-NEXT: v_mov_b32_e32 v10, s53118; GFX1032_DPP-NEXT: v_mov_b32_e32 v9, s43119; GFX1032_DPP-NEXT: s_mov_b32 s7, 0x310160003120; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)3121; GFX1032_DPP-NEXT: s_mov_b32 s4, s23122; GFX1032_DPP-NEXT: s_mov_b32 s5, s33123; GFX1032_DPP-NEXT: buffer_atomic_add_x2 v[9:10], off, s[4:7], 0 glc3124; GFX1032_DPP-NEXT: s_waitcnt vmcnt(0)3125; GFX1032_DPP-NEXT: buffer_gl1_inv3126; GFX1032_DPP-NEXT: buffer_gl0_inv3127; GFX1032_DPP-NEXT: .LBB5_2:3128; GFX1032_DPP-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)3129; GFX1032_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s83130; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)3131; GFX1032_DPP-NEXT: v_readfirstlane_b32 s2, v93132; GFX1032_DPP-NEXT: v_mov_b32_e32 v11, v73133; GFX1032_DPP-NEXT: v_mov_b32_e32 v12, v83134; GFX1032_DPP-NEXT: v_readfirstlane_b32 s3, v103135; GFX1032_DPP-NEXT: v_add_co_u32 v9, vcc_lo, s2, v113136; GFX1032_DPP-NEXT: s_mov_b32 s2, s63137; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v10, vcc_lo, s3, v12, vcc_lo3138; GFX1032_DPP-NEXT: s_mov_b32 s3, 0x310160003139; GFX1032_DPP-NEXT: buffer_store_dwordx2 v[9:10], off, s[0:3], 03140; GFX1032_DPP-NEXT: s_endpgm3141;3142; GFX1164_DPP-LABEL: add_i64_varying:3143; GFX1164_DPP: ; %bb.0: ; %entry3144; GFX1164_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v03145; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -13146; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)3147; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1]3148; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 03149; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]3150; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 03151; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)3152; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf3153; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:13154; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)3155; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc3156; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:13157; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)3158; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 03159; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)3160; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf3161; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc3162; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:13163; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)3164; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 03165; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)3166; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf3167; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc3168; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:13169; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)3170; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 03171; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)3172; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf3173; GFX1164_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -13174; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)3175; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc3176; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf3177; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)3178; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, 03179; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)3180; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -13181; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf3182; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)3183; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc3184; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 03185; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 313186; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)3187; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, s23188; GFX1164_DPP-NEXT: v_readlane_b32 s2, v2, 313189; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf3190; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)3191; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v2, s2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf3192; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)3193; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 03194; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc3195; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]3196; GFX1164_DPP-NEXT: s_load_b128 s[0:3], s[4:5], 0x243197; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 03198; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[4:5], -13199; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf3200; GFX1164_DPP-NEXT: v_readlane_b32 s6, v2, 153201; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf3202; GFX1164_DPP-NEXT: v_readlane_b32 s7, v1, 153203; GFX1164_DPP-NEXT: v_readlane_b32 s8, v2, 313204; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 313205; GFX1164_DPP-NEXT: v_writelane_b32 v4, s6, 163206; GFX1164_DPP-NEXT: v_readlane_b32 s6, v2, 633207; GFX1164_DPP-NEXT: v_writelane_b32 v5, s7, 163208; GFX1164_DPP-NEXT: v_readlane_b32 s10, v2, 473209; GFX1164_DPP-NEXT: v_readlane_b32 s11, v1, 473210; GFX1164_DPP-NEXT: v_readlane_b32 s7, v1, 633211; GFX1164_DPP-NEXT: v_writelane_b32 v4, s8, 323212; GFX1164_DPP-NEXT: v_writelane_b32 v5, s9, 323213; GFX1164_DPP-NEXT: s_mov_b64 exec, s[4:5]3214; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)3215; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v03216; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[8:9], -13217; GFX1164_DPP-NEXT: s_mov_b64 s[4:5], s[6:7]3218; GFX1164_DPP-NEXT: v_writelane_b32 v4, s10, 483219; GFX1164_DPP-NEXT: v_writelane_b32 v5, s11, 483220; GFX1164_DPP-NEXT: s_mov_b64 exec, s[8:9]3221; GFX1164_DPP-NEXT: s_mov_b32 s6, -13222; GFX1164_DPP-NEXT: s_mov_b64 s[8:9], exec3223; GFX1164_DPP-NEXT: ; implicit-def: $vgpr6_vgpr73224; GFX1164_DPP-NEXT: v_cmpx_eq_u32_e32 0, v03225; GFX1164_DPP-NEXT: s_cbranch_execz .LBB5_23226; GFX1164_DPP-NEXT: ; %bb.1:3227; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, s53228; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, s43229; GFX1164_DPP-NEXT: s_mov_b32 s7, 0x310160003230; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)3231; GFX1164_DPP-NEXT: s_mov_b32 s4, s23232; GFX1164_DPP-NEXT: s_mov_b32 s5, s33233; GFX1164_DPP-NEXT: buffer_atomic_add_u64 v[6:7], off, s[4:7], 0 glc3234; GFX1164_DPP-NEXT: s_waitcnt vmcnt(0)3235; GFX1164_DPP-NEXT: buffer_gl1_inv3236; GFX1164_DPP-NEXT: buffer_gl0_inv3237; GFX1164_DPP-NEXT: .LBB5_2:3238; GFX1164_DPP-NEXT: s_or_b64 exec, exec, s[8:9]3239; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)3240; GFX1164_DPP-NEXT: v_readfirstlane_b32 s2, v63241; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, v43242; GFX1164_DPP-NEXT: v_mov_b32_e32 v9, v53243; GFX1164_DPP-NEXT: v_readfirstlane_b32 s3, v73244; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)3245; GFX1164_DPP-NEXT: v_add_co_u32 v6, vcc, s2, v83246; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)3247; GFX1164_DPP-NEXT: s_mov_b32 s2, s63248; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v7, null, s3, v9, vcc3249; GFX1164_DPP-NEXT: s_mov_b32 s3, 0x310160003250; GFX1164_DPP-NEXT: buffer_store_b64 v[6:7], off, s[0:3], 03251; GFX1164_DPP-NEXT: s_endpgm3252;3253; GFX1132_DPP-LABEL: add_i64_varying:3254; GFX1132_DPP: ; %bb.0: ; %entry3255; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v03256; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -13257; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)3258; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s03259; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, 03260; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s03261; GFX1132_DPP-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v7, 03262; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, 03263; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)3264; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf3265; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:13266; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)3267; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc_lo3268; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:13269; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, 03270; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)3271; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf3272; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo3273; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)3274; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:13275; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, 03276; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf3277; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)3278; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc_lo3279; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:13280; GFX1132_DPP-NEXT: v_mov_b32_e32 v3, 03281; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)3282; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf3283; GFX1132_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -13284; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)3285; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo3286; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf3287; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)3288; GFX1132_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -13289; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf3290; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s03291; GFX1132_DPP-NEXT: s_load_b128 s[0:3], s[4:5], 0x243292; GFX1132_DPP-NEXT: s_or_saveexec_b32 s6, -13293; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)3294; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo3295; GFX1132_DPP-NEXT: v_readlane_b32 s4, v2, 313296; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf3297; GFX1132_DPP-NEXT: v_readlane_b32 s7, v2, 153298; GFX1132_DPP-NEXT: v_readlane_b32 s8, v1, 153299; GFX1132_DPP-NEXT: v_readlane_b32 s5, v1, 313300; GFX1132_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf3301; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s63302; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)3303; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 03304; GFX1132_DPP-NEXT: s_or_saveexec_b32 s6, -13305; GFX1132_DPP-NEXT: v_writelane_b32 v6, s7, 163306; GFX1132_DPP-NEXT: v_writelane_b32 v7, s8, 163307; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s63308; GFX1132_DPP-NEXT: s_mov_b32 s6, -13309; GFX1132_DPP-NEXT: s_mov_b32 s8, exec_lo3310; GFX1132_DPP-NEXT: ; implicit-def: $vgpr8_vgpr93311; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v03312; GFX1132_DPP-NEXT: s_cbranch_execz .LBB5_23313; GFX1132_DPP-NEXT: ; %bb.1:3314; GFX1132_DPP-NEXT: v_dual_mov_b32 v9, s5 :: v_dual_mov_b32 v8, s43315; GFX1132_DPP-NEXT: s_mov_b32 s7, 0x310160003316; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)3317; GFX1132_DPP-NEXT: s_mov_b32 s4, s23318; GFX1132_DPP-NEXT: s_mov_b32 s5, s33319; GFX1132_DPP-NEXT: buffer_atomic_add_u64 v[8:9], off, s[4:7], 0 glc3320; GFX1132_DPP-NEXT: s_waitcnt vmcnt(0)3321; GFX1132_DPP-NEXT: buffer_gl1_inv3322; GFX1132_DPP-NEXT: buffer_gl0_inv3323; GFX1132_DPP-NEXT: .LBB5_2:3324; GFX1132_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s83325; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)3326; GFX1132_DPP-NEXT: v_readfirstlane_b32 s2, v83327; GFX1132_DPP-NEXT: v_mov_b32_e32 v10, v63328; GFX1132_DPP-NEXT: v_mov_b32_e32 v11, v73329; GFX1132_DPP-NEXT: v_readfirstlane_b32 s3, v93330; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)3331; GFX1132_DPP-NEXT: v_add_co_u32 v8, vcc_lo, s2, v103332; GFX1132_DPP-NEXT: s_mov_b32 s2, s63333; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v9, null, s3, v11, vcc_lo3334; GFX1132_DPP-NEXT: s_mov_b32 s3, 0x310160003335; GFX1132_DPP-NEXT: buffer_store_b64 v[8:9], off, s[0:3], 03336; GFX1132_DPP-NEXT: s_endpgm3337;3338; GFX1264_DPP-LABEL: add_i64_varying:3339; GFX1264_DPP: ; %bb.0: ; %entry3340; GFX1264_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v03341; GFX1264_DPP-NEXT: s_or_saveexec_b64 s[0:1], -13342; GFX1264_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)3343; GFX1264_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1]3344; GFX1264_DPP-NEXT: v_mov_b32_e32 v2, 03345; GFX1264_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]3346; GFX1264_DPP-NEXT: v_mov_b32_e32 v4, 03347; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)3348; GFX1264_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf3349; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:13350; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)3351; GFX1264_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc3352; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:13353; GFX1264_DPP-NEXT: v_mov_b32_e32 v2, 03354; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)3355; GFX1264_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf3356; GFX1264_DPP-NEXT: s_wait_alu depctr_va_vcc(0)3357; GFX1264_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc3358; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:13359; GFX1264_DPP-NEXT: v_mov_b32_e32 v4, 03360; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)3361; GFX1264_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf3362; GFX1264_DPP-NEXT: s_wait_alu depctr_va_vcc(0)3363; GFX1264_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc3364; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:13365; GFX1264_DPP-NEXT: v_mov_b32_e32 v3, 03366; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)3367; GFX1264_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf3368; GFX1264_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -13369; GFX1264_DPP-NEXT: s_wait_alu depctr_va_vcc(0)3370; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)3371; GFX1264_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc3372; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf3373; GFX1264_DPP-NEXT: v_mov_b32_e32 v5, 03374; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)3375; GFX1264_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -13376; GFX1264_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf3377; GFX1264_DPP-NEXT: s_wait_alu depctr_va_vcc(0)3378; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)3379; GFX1264_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc3380; GFX1264_DPP-NEXT: v_mov_b32_e32 v3, 03381; GFX1264_DPP-NEXT: v_readlane_b32 s2, v1, 313382; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)3383; GFX1264_DPP-NEXT: v_mov_b32_e32 v4, s23384; GFX1264_DPP-NEXT: v_readlane_b32 s2, v2, 313385; GFX1264_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf3386; GFX1264_DPP-NEXT: s_wait_alu depctr_va_sdst(0)3387; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)3388; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v2, s2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf3389; GFX1264_DPP-NEXT: v_mov_b32_e32 v4, 03390; GFX1264_DPP-NEXT: s_wait_alu depctr_va_vcc(0)3391; GFX1264_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc3392; GFX1264_DPP-NEXT: s_mov_b64 exec, s[0:1]3393; GFX1264_DPP-NEXT: s_load_b128 s[0:3], s[4:5], 0x243394; GFX1264_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 03395; GFX1264_DPP-NEXT: s_or_saveexec_b64 s[4:5], -13396; GFX1264_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf3397; GFX1264_DPP-NEXT: v_readlane_b32 s6, v2, 153398; GFX1264_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf3399; GFX1264_DPP-NEXT: v_readlane_b32 s7, v1, 153400; GFX1264_DPP-NEXT: v_readlane_b32 s8, v2, 313401; GFX1264_DPP-NEXT: v_readlane_b32 s9, v1, 313402; GFX1264_DPP-NEXT: v_writelane_b32 v4, s6, 163403; GFX1264_DPP-NEXT: v_readlane_b32 s6, v2, 633404; GFX1264_DPP-NEXT: v_writelane_b32 v5, s7, 163405; GFX1264_DPP-NEXT: v_readlane_b32 s10, v2, 473406; GFX1264_DPP-NEXT: v_readlane_b32 s11, v1, 473407; GFX1264_DPP-NEXT: v_readlane_b32 s7, v1, 633408; GFX1264_DPP-NEXT: v_writelane_b32 v4, s8, 323409; GFX1264_DPP-NEXT: v_writelane_b32 v5, s9, 323410; GFX1264_DPP-NEXT: s_mov_b64 exec, s[4:5]3411; GFX1264_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)3412; GFX1264_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v03413; GFX1264_DPP-NEXT: s_or_saveexec_b64 s[8:9], -13414; GFX1264_DPP-NEXT: s_mov_b64 s[4:5], s[6:7]3415; GFX1264_DPP-NEXT: v_writelane_b32 v4, s10, 483416; GFX1264_DPP-NEXT: v_writelane_b32 v5, s11, 483417; GFX1264_DPP-NEXT: s_wait_alu depctr_sa_sdst(0)3418; GFX1264_DPP-NEXT: s_mov_b64 exec, s[8:9]3419; GFX1264_DPP-NEXT: s_mov_b32 s6, -13420; GFX1264_DPP-NEXT: s_mov_b64 s[8:9], exec3421; GFX1264_DPP-NEXT: ; implicit-def: $vgpr6_vgpr73422; GFX1264_DPP-NEXT: v_cmpx_eq_u32_e32 0, v03423; GFX1264_DPP-NEXT: s_cbranch_execz .LBB5_23424; GFX1264_DPP-NEXT: ; %bb.1:3425; GFX1264_DPP-NEXT: v_mov_b32_e32 v7, s53426; GFX1264_DPP-NEXT: v_mov_b32_e32 v6, s43427; GFX1264_DPP-NEXT: s_mov_b32 s7, 0x310160003428; GFX1264_DPP-NEXT: s_wait_kmcnt 0x03429; GFX1264_DPP-NEXT: s_mov_b32 s4, s23430; GFX1264_DPP-NEXT: s_mov_b32 s5, s33431; GFX1264_DPP-NEXT: buffer_atomic_add_u64 v[6:7], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV3432; GFX1264_DPP-NEXT: s_wait_loadcnt 0x03433; GFX1264_DPP-NEXT: global_inv scope:SCOPE_DEV3434; GFX1264_DPP-NEXT: .LBB5_2:3435; GFX1264_DPP-NEXT: s_wait_alu depctr_sa_sdst(0)3436; GFX1264_DPP-NEXT: s_or_b64 exec, exec, s[8:9]3437; GFX1264_DPP-NEXT: s_wait_kmcnt 0x03438; GFX1264_DPP-NEXT: v_readfirstlane_b32 s2, v63439; GFX1264_DPP-NEXT: v_mov_b32_e32 v8, v43440; GFX1264_DPP-NEXT: v_mov_b32_e32 v9, v53441; GFX1264_DPP-NEXT: v_readfirstlane_b32 s3, v73442; GFX1264_DPP-NEXT: s_wait_alu depctr_va_sdst(0)3443; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)3444; GFX1264_DPP-NEXT: v_add_co_u32 v6, vcc, s2, v83445; GFX1264_DPP-NEXT: s_mov_b32 s2, s63446; GFX1264_DPP-NEXT: s_wait_alu depctr_va_vcc(0)3447; GFX1264_DPP-NEXT: v_add_co_ci_u32_e64 v7, null, s3, v9, vcc3448; GFX1264_DPP-NEXT: s_mov_b32 s3, 0x310160003449; GFX1264_DPP-NEXT: buffer_store_b64 v[6:7], off, s[0:3], null3450; GFX1264_DPP-NEXT: s_endpgm3451;3452; GFX1232_DPP-LABEL: add_i64_varying:3453; GFX1232_DPP: ; %bb.0: ; %entry3454; GFX1232_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v03455; GFX1232_DPP-NEXT: s_or_saveexec_b32 s0, -13456; GFX1232_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)3457; GFX1232_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s03458; GFX1232_DPP-NEXT: v_mov_b32_e32 v2, 03459; GFX1232_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s03460; GFX1232_DPP-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v7, 03461; GFX1232_DPP-NEXT: v_mov_b32_e32 v6, 03462; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)3463; GFX1232_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf3464; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:13465; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)3466; GFX1232_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc_lo3467; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:13468; GFX1232_DPP-NEXT: v_mov_b32_e32 v2, 03469; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)3470; GFX1232_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf3471; GFX1232_DPP-NEXT: s_wait_alu depctr_va_vcc(0)3472; GFX1232_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo3473; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:13474; GFX1232_DPP-NEXT: v_mov_b32_e32 v4, 03475; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)3476; GFX1232_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf3477; GFX1232_DPP-NEXT: s_wait_alu depctr_va_vcc(0)3478; GFX1232_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc_lo3479; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:13480; GFX1232_DPP-NEXT: v_mov_b32_e32 v3, 03481; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)3482; GFX1232_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf3483; GFX1232_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -13484; GFX1232_DPP-NEXT: s_wait_alu depctr_va_vcc(0)3485; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)3486; GFX1232_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo3487; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf3488; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)3489; GFX1232_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -13490; GFX1232_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf3491; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s03492; GFX1232_DPP-NEXT: s_load_b128 s[0:3], s[4:5], 0x243493; GFX1232_DPP-NEXT: s_or_saveexec_b32 s6, -13494; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)3495; GFX1232_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo3496; GFX1232_DPP-NEXT: v_readlane_b32 s4, v2, 313497; GFX1232_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf3498; GFX1232_DPP-NEXT: v_readlane_b32 s7, v2, 153499; GFX1232_DPP-NEXT: v_readlane_b32 s8, v1, 153500; GFX1232_DPP-NEXT: v_readlane_b32 s5, v1, 313501; GFX1232_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf3502; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s63503; GFX1232_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)3504; GFX1232_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 03505; GFX1232_DPP-NEXT: s_or_saveexec_b32 s6, -13506; GFX1232_DPP-NEXT: v_writelane_b32 v6, s7, 163507; GFX1232_DPP-NEXT: v_writelane_b32 v7, s8, 163508; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s63509; GFX1232_DPP-NEXT: s_mov_b32 s6, -13510; GFX1232_DPP-NEXT: s_mov_b32 s8, exec_lo3511; GFX1232_DPP-NEXT: ; implicit-def: $vgpr8_vgpr93512; GFX1232_DPP-NEXT: v_cmpx_eq_u32_e32 0, v03513; GFX1232_DPP-NEXT: s_cbranch_execz .LBB5_23514; GFX1232_DPP-NEXT: ; %bb.1:3515; GFX1232_DPP-NEXT: v_dual_mov_b32 v9, s5 :: v_dual_mov_b32 v8, s43516; GFX1232_DPP-NEXT: s_mov_b32 s7, 0x310160003517; GFX1232_DPP-NEXT: s_wait_kmcnt 0x03518; GFX1232_DPP-NEXT: s_mov_b32 s4, s23519; GFX1232_DPP-NEXT: s_mov_b32 s5, s33520; GFX1232_DPP-NEXT: buffer_atomic_add_u64 v[8:9], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV3521; GFX1232_DPP-NEXT: s_wait_loadcnt 0x03522; GFX1232_DPP-NEXT: global_inv scope:SCOPE_DEV3523; GFX1232_DPP-NEXT: .LBB5_2:3524; GFX1232_DPP-NEXT: s_wait_alu depctr_sa_sdst(0)3525; GFX1232_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s83526; GFX1232_DPP-NEXT: s_wait_kmcnt 0x03527; GFX1232_DPP-NEXT: v_readfirstlane_b32 s2, v83528; GFX1232_DPP-NEXT: v_mov_b32_e32 v10, v63529; GFX1232_DPP-NEXT: v_mov_b32_e32 v11, v73530; GFX1232_DPP-NEXT: v_readfirstlane_b32 s3, v93531; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)3532; GFX1232_DPP-NEXT: v_add_co_u32 v8, vcc_lo, s2, v103533; GFX1232_DPP-NEXT: s_mov_b32 s2, s63534; GFX1232_DPP-NEXT: s_wait_alu depctr_va_vcc(0)3535; GFX1232_DPP-NEXT: v_add_co_ci_u32_e64 v9, null, s3, v11, vcc_lo3536; GFX1232_DPP-NEXT: s_mov_b32 s3, 0x310160003537; GFX1232_DPP-NEXT: buffer_store_b64 v[8:9], off, s[0:3], null3538; GFX1232_DPP-NEXT: s_endpgm3539entry:3540 %lane = call i32 @llvm.amdgcn.workitem.id.x()3541 %zext = zext i32 %lane to i643542 %old = atomicrmw add ptr addrspace(1) %inout, i64 %zext syncscope("agent") acq_rel3543 store i64 %old, ptr addrspace(1) %out3544 ret void3545}3546 3547define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace(1) %inout) {3548; GFX7LESS-LABEL: sub_i32_constant:3549; GFX7LESS: ; %bb.0: ; %entry3550; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec3551; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93552; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 03553; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s7, v03554; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v23555; GFX7LESS-NEXT: ; implicit-def: $vgpr03556; GFX7LESS-NEXT: s_and_saveexec_b64 s[8:9], vcc3557; GFX7LESS-NEXT: s_cbranch_execz .LBB6_43558; GFX7LESS-NEXT: ; %bb.1:3559; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)3560; GFX7LESS-NEXT: s_load_dword s4, s[2:3], 0x03561; GFX7LESS-NEXT: s_bcnt1_i32_b64 s12, s[6:7]3562; GFX7LESS-NEXT: s_mov_b64 s[10:11], 03563; GFX7LESS-NEXT: s_mov_b32 s7, 0xf0003564; GFX7LESS-NEXT: s_mul_i32 s12, s12, 53565; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)3566; GFX7LESS-NEXT: v_mov_b32_e32 v0, s43567; GFX7LESS-NEXT: s_mov_b32 s6, -13568; GFX7LESS-NEXT: s_mov_b32 s4, s23569; GFX7LESS-NEXT: s_mov_b32 s5, s33570; GFX7LESS-NEXT: .LBB6_2: ; %atomicrmw.start3571; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=13572; GFX7LESS-NEXT: v_mov_b32_e32 v4, v03573; GFX7LESS-NEXT: v_subrev_i32_e32 v3, vcc, s12, v43574; GFX7LESS-NEXT: s_waitcnt expcnt(0)3575; GFX7LESS-NEXT: v_mov_b32_e32 v0, v33576; GFX7LESS-NEXT: v_mov_b32_e32 v1, v43577; GFX7LESS-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc3578; GFX7LESS-NEXT: s_waitcnt vmcnt(0)3579; GFX7LESS-NEXT: buffer_wbinvl13580; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v0, v43581; GFX7LESS-NEXT: s_or_b64 s[10:11], vcc, s[10:11]3582; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[10:11]3583; GFX7LESS-NEXT: s_cbranch_execnz .LBB6_23584; GFX7LESS-NEXT: ; %bb.3: ; %Flow3585; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]3586; GFX7LESS-NEXT: .LBB6_4: ; %Flow33587; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]3588; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)3589; GFX7LESS-NEXT: s_mov_b32 s3, 0xf0003590; GFX7LESS-NEXT: s_mov_b32 s2, -13591; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v03592; GFX7LESS-NEXT: s_waitcnt expcnt(0)3593; GFX7LESS-NEXT: v_mul_u32_u24_e32 v0, 5, v23594; GFX7LESS-NEXT: v_sub_i32_e32 v0, vcc, s4, v03595; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 03596; GFX7LESS-NEXT: s_endpgm3597;3598; GFX8-LABEL: sub_i32_constant:3599; GFX8: ; %bb.0: ; %entry3600; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243601; GFX8-NEXT: s_mov_b64 s[6:7], exec3602; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 03603; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v03604; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v23605; GFX8-NEXT: ; implicit-def: $vgpr03606; GFX8-NEXT: s_and_saveexec_b64 s[8:9], vcc3607; GFX8-NEXT: s_cbranch_execz .LBB6_43608; GFX8-NEXT: ; %bb.1:3609; GFX8-NEXT: s_waitcnt lgkmcnt(0)3610; GFX8-NEXT: s_load_dword s4, s[2:3], 0x03611; GFX8-NEXT: s_bcnt1_i32_b64 s12, s[6:7]3612; GFX8-NEXT: s_mov_b64 s[10:11], 03613; GFX8-NEXT: s_mov_b32 s7, 0xf0003614; GFX8-NEXT: s_mul_i32 s12, s12, 53615; GFX8-NEXT: s_waitcnt lgkmcnt(0)3616; GFX8-NEXT: v_mov_b32_e32 v0, s43617; GFX8-NEXT: s_mov_b32 s6, -13618; GFX8-NEXT: s_mov_b32 s4, s23619; GFX8-NEXT: s_mov_b32 s5, s33620; GFX8-NEXT: .LBB6_2: ; %atomicrmw.start3621; GFX8-NEXT: ; =>This Inner Loop Header: Depth=13622; GFX8-NEXT: v_mov_b32_e32 v4, v03623; GFX8-NEXT: v_subrev_u32_e32 v3, vcc, s12, v43624; GFX8-NEXT: v_mov_b32_e32 v0, v33625; GFX8-NEXT: v_mov_b32_e32 v1, v43626; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc3627; GFX8-NEXT: s_waitcnt vmcnt(0)3628; GFX8-NEXT: buffer_wbinvl1_vol3629; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v43630; GFX8-NEXT: s_or_b64 s[10:11], vcc, s[10:11]3631; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]3632; GFX8-NEXT: s_cbranch_execnz .LBB6_23633; GFX8-NEXT: ; %bb.3: ; %Flow3634; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]3635; GFX8-NEXT: .LBB6_4: ; %Flow33636; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]3637; GFX8-NEXT: v_readfirstlane_b32 s4, v03638; GFX8-NEXT: v_mul_u32_u24_e32 v0, 5, v23639; GFX8-NEXT: s_waitcnt lgkmcnt(0)3640; GFX8-NEXT: s_mov_b32 s3, 0xf0003641; GFX8-NEXT: s_mov_b32 s2, -13642; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s4, v03643; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 03644; GFX8-NEXT: s_endpgm3645;3646; GFX9-LABEL: sub_i32_constant:3647; GFX9: ; %bb.0: ; %entry3648; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243649; GFX9-NEXT: s_mov_b64 s[6:7], exec3650; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 03651; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v03652; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v23653; GFX9-NEXT: ; implicit-def: $vgpr03654; GFX9-NEXT: s_and_saveexec_b64 s[8:9], vcc3655; GFX9-NEXT: s_cbranch_execz .LBB6_43656; GFX9-NEXT: ; %bb.1:3657; GFX9-NEXT: s_waitcnt lgkmcnt(0)3658; GFX9-NEXT: s_load_dword s4, s[2:3], 0x03659; GFX9-NEXT: s_bcnt1_i32_b64 s12, s[6:7]3660; GFX9-NEXT: s_mov_b64 s[10:11], 03661; GFX9-NEXT: s_mov_b32 s7, 0xf0003662; GFX9-NEXT: s_mul_i32 s12, s12, 53663; GFX9-NEXT: s_waitcnt lgkmcnt(0)3664; GFX9-NEXT: v_mov_b32_e32 v0, s43665; GFX9-NEXT: s_mov_b32 s6, -13666; GFX9-NEXT: s_mov_b32 s4, s23667; GFX9-NEXT: s_mov_b32 s5, s33668; GFX9-NEXT: .LBB6_2: ; %atomicrmw.start3669; GFX9-NEXT: ; =>This Inner Loop Header: Depth=13670; GFX9-NEXT: v_mov_b32_e32 v4, v03671; GFX9-NEXT: v_subrev_u32_e32 v3, s12, v43672; GFX9-NEXT: v_mov_b32_e32 v0, v33673; GFX9-NEXT: v_mov_b32_e32 v1, v43674; GFX9-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc3675; GFX9-NEXT: s_waitcnt vmcnt(0)3676; GFX9-NEXT: buffer_wbinvl1_vol3677; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v43678; GFX9-NEXT: s_or_b64 s[10:11], vcc, s[10:11]3679; GFX9-NEXT: s_andn2_b64 exec, exec, s[10:11]3680; GFX9-NEXT: s_cbranch_execnz .LBB6_23681; GFX9-NEXT: ; %bb.3: ; %Flow3682; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]3683; GFX9-NEXT: .LBB6_4: ; %Flow33684; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]3685; GFX9-NEXT: v_readfirstlane_b32 s4, v03686; GFX9-NEXT: v_mul_u32_u24_e32 v0, 5, v23687; GFX9-NEXT: s_waitcnt lgkmcnt(0)3688; GFX9-NEXT: s_mov_b32 s3, 0xf0003689; GFX9-NEXT: s_mov_b32 s2, -13690; GFX9-NEXT: v_sub_u32_e32 v0, s4, v03691; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 03692; GFX9-NEXT: s_endpgm3693;3694; GFX1064-LABEL: sub_i32_constant:3695; GFX1064: ; %bb.0: ; %entry3696; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243697; GFX1064-NEXT: s_mov_b64 s[6:7], exec3698; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 03699; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v03700; GFX1064-NEXT: ; implicit-def: $vgpr03701; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v23702; GFX1064-NEXT: s_and_saveexec_b64 s[8:9], vcc3703; GFX1064-NEXT: s_cbranch_execz .LBB6_43704; GFX1064-NEXT: ; %bb.1:3705; GFX1064-NEXT: s_waitcnt lgkmcnt(0)3706; GFX1064-NEXT: s_load_dword s4, s[2:3], 0x03707; GFX1064-NEXT: s_bcnt1_i32_b64 s12, s[6:7]3708; GFX1064-NEXT: s_mov_b64 s[10:11], 03709; GFX1064-NEXT: s_mul_i32 s12, s12, 53710; GFX1064-NEXT: s_mov_b32 s7, 0x310160003711; GFX1064-NEXT: s_mov_b32 s6, -13712; GFX1064-NEXT: s_mov_b32 s5, s33713; GFX1064-NEXT: s_waitcnt lgkmcnt(0)3714; GFX1064-NEXT: v_mov_b32_e32 v0, s43715; GFX1064-NEXT: s_mov_b32 s4, s23716; GFX1064-NEXT: .LBB6_2: ; %atomicrmw.start3717; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=13718; GFX1064-NEXT: v_mov_b32_e32 v4, v03719; GFX1064-NEXT: v_subrev_nc_u32_e32 v3, s12, v43720; GFX1064-NEXT: v_mov_b32_e32 v0, v33721; GFX1064-NEXT: v_mov_b32_e32 v1, v43722; GFX1064-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc3723; GFX1064-NEXT: s_waitcnt vmcnt(0)3724; GFX1064-NEXT: buffer_gl1_inv3725; GFX1064-NEXT: buffer_gl0_inv3726; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v43727; GFX1064-NEXT: s_or_b64 s[10:11], vcc, s[10:11]3728; GFX1064-NEXT: s_andn2_b64 exec, exec, s[10:11]3729; GFX1064-NEXT: s_cbranch_execnz .LBB6_23730; GFX1064-NEXT: ; %bb.3: ; %Flow3731; GFX1064-NEXT: s_or_b64 exec, exec, s[10:11]3732; GFX1064-NEXT: .LBB6_4: ; %Flow33733; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]3734; GFX1064-NEXT: s_waitcnt lgkmcnt(0)3735; GFX1064-NEXT: v_readfirstlane_b32 s2, v03736; GFX1064-NEXT: v_mul_u32_u24_e32 v0, 5, v23737; GFX1064-NEXT: s_mov_b32 s3, 0x310160003738; GFX1064-NEXT: v_sub_nc_u32_e32 v0, s2, v03739; GFX1064-NEXT: s_mov_b32 s2, -13740; GFX1064-NEXT: buffer_store_dword v0, off, s[0:3], 03741; GFX1064-NEXT: s_endpgm3742;3743; GFX1032-LABEL: sub_i32_constant:3744; GFX1032: ; %bb.0: ; %entry3745; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x243746; GFX1032-NEXT: s_mov_b32 s6, exec_lo3747; GFX1032-NEXT: s_mov_b32 s9, 03748; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 03749; GFX1032-NEXT: ; implicit-def: $vgpr03750; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v23751; GFX1032-NEXT: s_and_saveexec_b32 s8, vcc_lo3752; GFX1032-NEXT: s_cbranch_execz .LBB6_43753; GFX1032-NEXT: ; %bb.1:3754; GFX1032-NEXT: s_waitcnt lgkmcnt(0)3755; GFX1032-NEXT: s_load_dword s4, s[2:3], 0x03756; GFX1032-NEXT: s_bcnt1_i32_b32 s10, s63757; GFX1032-NEXT: s_mov_b32 s7, 0x310160003758; GFX1032-NEXT: s_mul_i32 s10, s10, 53759; GFX1032-NEXT: s_mov_b32 s6, -13760; GFX1032-NEXT: s_mov_b32 s5, s33761; GFX1032-NEXT: s_waitcnt lgkmcnt(0)3762; GFX1032-NEXT: v_mov_b32_e32 v0, s43763; GFX1032-NEXT: s_mov_b32 s4, s23764; GFX1032-NEXT: .LBB6_2: ; %atomicrmw.start3765; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=13766; GFX1032-NEXT: v_mov_b32_e32 v4, v03767; GFX1032-NEXT: v_subrev_nc_u32_e32 v3, s10, v43768; GFX1032-NEXT: v_mov_b32_e32 v0, v33769; GFX1032-NEXT: v_mov_b32_e32 v1, v43770; GFX1032-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc3771; GFX1032-NEXT: s_waitcnt vmcnt(0)3772; GFX1032-NEXT: buffer_gl1_inv3773; GFX1032-NEXT: buffer_gl0_inv3774; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v43775; GFX1032-NEXT: s_or_b32 s9, vcc_lo, s93776; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s93777; GFX1032-NEXT: s_cbranch_execnz .LBB6_23778; GFX1032-NEXT: ; %bb.3: ; %Flow3779; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s93780; GFX1032-NEXT: .LBB6_4: ; %Flow33781; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s83782; GFX1032-NEXT: s_waitcnt lgkmcnt(0)3783; GFX1032-NEXT: v_readfirstlane_b32 s2, v03784; GFX1032-NEXT: v_mul_u32_u24_e32 v0, 5, v23785; GFX1032-NEXT: s_mov_b32 s3, 0x310160003786; GFX1032-NEXT: v_sub_nc_u32_e32 v0, s2, v03787; GFX1032-NEXT: s_mov_b32 s2, -13788; GFX1032-NEXT: buffer_store_dword v0, off, s[0:3], 03789; GFX1032-NEXT: s_endpgm3790;3791; GFX1164-LABEL: sub_i32_constant:3792; GFX1164: ; %bb.0: ; %entry3793; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x243794; GFX1164-NEXT: s_mov_b64 s[6:7], exec3795; GFX1164-NEXT: s_mov_b64 s[8:9], exec3796; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 03797; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3798; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v03799; GFX1164-NEXT: ; implicit-def: $vgpr03800; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v23801; GFX1164-NEXT: s_cbranch_execz .LBB6_43802; GFX1164-NEXT: ; %bb.1:3803; GFX1164-NEXT: s_waitcnt lgkmcnt(0)3804; GFX1164-NEXT: s_load_b32 s4, s[2:3], 0x03805; GFX1164-NEXT: s_bcnt1_i32_b64 s12, s[6:7]3806; GFX1164-NEXT: s_mov_b64 s[10:11], 03807; GFX1164-NEXT: s_mul_i32 s12, s12, 53808; GFX1164-NEXT: s_mov_b32 s7, 0x310160003809; GFX1164-NEXT: s_mov_b32 s6, -13810; GFX1164-NEXT: s_mov_b32 s5, s33811; GFX1164-NEXT: s_waitcnt lgkmcnt(0)3812; GFX1164-NEXT: v_mov_b32_e32 v0, s43813; GFX1164-NEXT: s_mov_b32 s4, s23814; GFX1164-NEXT: .LBB6_2: ; %atomicrmw.start3815; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=13816; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3817; GFX1164-NEXT: v_mov_b32_e32 v4, v03818; GFX1164-NEXT: v_subrev_nc_u32_e32 v3, s12, v43819; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)3820; GFX1164-NEXT: v_mov_b32_e32 v0, v33821; GFX1164-NEXT: v_mov_b32_e32 v1, v43822; GFX1164-NEXT: buffer_atomic_cmpswap_b32 v[0:1], off, s[4:7], 0 glc3823; GFX1164-NEXT: s_waitcnt vmcnt(0)3824; GFX1164-NEXT: buffer_gl1_inv3825; GFX1164-NEXT: buffer_gl0_inv3826; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v43827; GFX1164-NEXT: s_or_b64 s[10:11], vcc, s[10:11]3828; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)3829; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[10:11]3830; GFX1164-NEXT: s_cbranch_execnz .LBB6_23831; GFX1164-NEXT: ; %bb.3: ; %Flow3832; GFX1164-NEXT: s_or_b64 exec, exec, s[10:11]3833; GFX1164-NEXT: .LBB6_4: ; %Flow33834; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)3835; GFX1164-NEXT: s_or_b64 exec, exec, s[8:9]3836; GFX1164-NEXT: s_waitcnt lgkmcnt(0)3837; GFX1164-NEXT: v_readfirstlane_b32 s2, v03838; GFX1164-NEXT: v_mul_u32_u24_e32 v0, 5, v23839; GFX1164-NEXT: s_mov_b32 s3, 0x310160003840; GFX1164-NEXT: v_sub_nc_u32_e32 v0, s2, v03841; GFX1164-NEXT: s_mov_b32 s2, -13842; GFX1164-NEXT: buffer_store_b32 v0, off, s[0:3], 03843; GFX1164-NEXT: s_endpgm3844;3845; GFX1132-LABEL: sub_i32_constant:3846; GFX1132: ; %bb.0: ; %entry3847; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x243848; GFX1132-NEXT: s_mov_b32 s6, exec_lo3849; GFX1132-NEXT: s_mov_b32 s9, 03850; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 03851; GFX1132-NEXT: s_mov_b32 s8, exec_lo3852; GFX1132-NEXT: ; implicit-def: $vgpr03853; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)3854; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v23855; GFX1132-NEXT: s_cbranch_execz .LBB6_43856; GFX1132-NEXT: ; %bb.1:3857; GFX1132-NEXT: s_waitcnt lgkmcnt(0)3858; GFX1132-NEXT: s_load_b32 s4, s[2:3], 0x03859; GFX1132-NEXT: s_bcnt1_i32_b32 s10, s63860; GFX1132-NEXT: s_mov_b32 s7, 0x310160003861; GFX1132-NEXT: s_mul_i32 s10, s10, 53862; GFX1132-NEXT: s_mov_b32 s6, -13863; GFX1132-NEXT: s_mov_b32 s5, s33864; GFX1132-NEXT: s_waitcnt lgkmcnt(0)3865; GFX1132-NEXT: v_mov_b32_e32 v0, s43866; GFX1132-NEXT: s_mov_b32 s4, s23867; GFX1132-NEXT: .LBB6_2: ; %atomicrmw.start3868; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=13869; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3870; GFX1132-NEXT: v_mov_b32_e32 v4, v03871; GFX1132-NEXT: v_subrev_nc_u32_e32 v3, s10, v43872; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)3873; GFX1132-NEXT: v_dual_mov_b32 v0, v3 :: v_dual_mov_b32 v1, v43874; GFX1132-NEXT: buffer_atomic_cmpswap_b32 v[0:1], off, s[4:7], 0 glc3875; GFX1132-NEXT: s_waitcnt vmcnt(0)3876; GFX1132-NEXT: buffer_gl1_inv3877; GFX1132-NEXT: buffer_gl0_inv3878; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v43879; GFX1132-NEXT: s_or_b32 s9, vcc_lo, s93880; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)3881; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s93882; GFX1132-NEXT: s_cbranch_execnz .LBB6_23883; GFX1132-NEXT: ; %bb.3: ; %Flow3884; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s93885; GFX1132-NEXT: .LBB6_4: ; %Flow33886; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)3887; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s83888; GFX1132-NEXT: s_waitcnt lgkmcnt(0)3889; GFX1132-NEXT: v_readfirstlane_b32 s2, v03890; GFX1132-NEXT: v_mul_u32_u24_e32 v0, 5, v23891; GFX1132-NEXT: s_mov_b32 s3, 0x310160003892; GFX1132-NEXT: v_sub_nc_u32_e32 v0, s2, v03893; GFX1132-NEXT: s_mov_b32 s2, -13894; GFX1132-NEXT: buffer_store_b32 v0, off, s[0:3], 03895; GFX1132-NEXT: s_endpgm3896;3897; GFX1264-LABEL: sub_i32_constant:3898; GFX1264: ; %bb.0: ; %entry3899; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x243900; GFX1264-NEXT: s_mov_b64 s[6:7], exec3901; GFX1264-NEXT: s_mov_b64 s[4:5], exec3902; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 03903; GFX1264-NEXT: ; implicit-def: $vgpr13904; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3905; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v03906; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v03907; GFX1264-NEXT: s_cbranch_execz .LBB6_23908; GFX1264-NEXT: ; %bb.1:3909; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[6:7]3910; GFX1264-NEXT: s_mov_b32 s11, 0x310160003911; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)3912; GFX1264-NEXT: s_mul_i32 s6, s6, 53913; GFX1264-NEXT: s_mov_b32 s10, -13914; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)3915; GFX1264-NEXT: v_mov_b32_e32 v1, s63916; GFX1264-NEXT: s_wait_kmcnt 0x03917; GFX1264-NEXT: s_mov_b32 s8, s23918; GFX1264-NEXT: s_mov_b32 s9, s33919; GFX1264-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV3920; GFX1264-NEXT: s_wait_loadcnt 0x03921; GFX1264-NEXT: global_inv scope:SCOPE_DEV3922; GFX1264-NEXT: .LBB6_2:3923; GFX1264-NEXT: s_or_b64 exec, exec, s[4:5]3924; GFX1264-NEXT: s_wait_kmcnt 0x03925; GFX1264-NEXT: v_readfirstlane_b32 s2, v13926; GFX1264-NEXT: v_mul_u32_u24_e32 v0, 5, v03927; GFX1264-NEXT: s_mov_b32 s3, 0x310160003928; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)3929; GFX1264-NEXT: v_sub_nc_u32_e32 v0, s2, v03930; GFX1264-NEXT: s_mov_b32 s2, -13931; GFX1264-NEXT: buffer_store_b32 v0, off, s[0:3], null3932; GFX1264-NEXT: s_endpgm3933;3934; GFX1232-LABEL: sub_i32_constant:3935; GFX1232: ; %bb.0: ; %entry3936; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x243937; GFX1232-NEXT: s_mov_b32 s6, exec_lo3938; GFX1232-NEXT: s_mov_b32 s4, exec_lo3939; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 03940; GFX1232-NEXT: ; implicit-def: $vgpr13941; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)3942; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v03943; GFX1232-NEXT: s_cbranch_execz .LBB6_23944; GFX1232-NEXT: ; %bb.1:3945; GFX1232-NEXT: s_bcnt1_i32_b32 s5, s63946; GFX1232-NEXT: s_mov_b32 s11, 0x310160003947; GFX1232-NEXT: s_mul_i32 s5, s5, 53948; GFX1232-NEXT: s_mov_b32 s10, -13949; GFX1232-NEXT: v_mov_b32_e32 v1, s53950; GFX1232-NEXT: s_wait_kmcnt 0x03951; GFX1232-NEXT: s_mov_b32 s8, s23952; GFX1232-NEXT: s_mov_b32 s9, s33953; GFX1232-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV3954; GFX1232-NEXT: s_wait_loadcnt 0x03955; GFX1232-NEXT: global_inv scope:SCOPE_DEV3956; GFX1232-NEXT: .LBB6_2:3957; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s43958; GFX1232-NEXT: s_wait_kmcnt 0x03959; GFX1232-NEXT: v_readfirstlane_b32 s2, v13960; GFX1232-NEXT: v_mul_u32_u24_e32 v0, 5, v03961; GFX1232-NEXT: s_mov_b32 s3, 0x310160003962; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)3963; GFX1232-NEXT: v_sub_nc_u32_e32 v0, s2, v03964; GFX1232-NEXT: s_mov_b32 s2, -13965; GFX1232-NEXT: buffer_store_b32 v0, off, s[0:3], null3966; GFX1232-NEXT: s_endpgm3967entry:3968 %old = atomicrmw sub ptr addrspace(1) %inout, i32 5 syncscope("agent") acq_rel3969 store i32 %old, ptr addrspace(1) %out3970 ret void3971}3972 3973define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(1) %inout, i32 %subitive) {3974; GFX7LESS-LABEL: sub_i32_uniform:3975; GFX7LESS: ; %bb.0: ; %entry3976; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec3977; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x93978; GFX7LESS-NEXT: s_load_dword s12, s[4:5], 0xd3979; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 03980; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s7, v03981; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v23982; GFX7LESS-NEXT: ; implicit-def: $vgpr03983; GFX7LESS-NEXT: s_and_saveexec_b64 s[8:9], vcc3984; GFX7LESS-NEXT: s_cbranch_execz .LBB7_43985; GFX7LESS-NEXT: ; %bb.1:3986; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)3987; GFX7LESS-NEXT: s_load_dword s4, s[2:3], 0x03988; GFX7LESS-NEXT: s_bcnt1_i32_b64 s5, s[6:7]3989; GFX7LESS-NEXT: s_mov_b64 s[10:11], 03990; GFX7LESS-NEXT: s_mov_b32 s7, 0xf0003991; GFX7LESS-NEXT: s_mul_i32 s13, s12, s53992; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)3993; GFX7LESS-NEXT: v_mov_b32_e32 v0, s43994; GFX7LESS-NEXT: s_mov_b32 s6, -13995; GFX7LESS-NEXT: s_mov_b32 s4, s23996; GFX7LESS-NEXT: s_mov_b32 s5, s33997; GFX7LESS-NEXT: .LBB7_2: ; %atomicrmw.start3998; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=13999; GFX7LESS-NEXT: v_mov_b32_e32 v4, v04000; GFX7LESS-NEXT: v_subrev_i32_e32 v3, vcc, s13, v44001; GFX7LESS-NEXT: s_waitcnt expcnt(0)4002; GFX7LESS-NEXT: v_mov_b32_e32 v0, v34003; GFX7LESS-NEXT: v_mov_b32_e32 v1, v44004; GFX7LESS-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc4005; GFX7LESS-NEXT: s_waitcnt vmcnt(0)4006; GFX7LESS-NEXT: buffer_wbinvl14007; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v0, v44008; GFX7LESS-NEXT: s_or_b64 s[10:11], vcc, s[10:11]4009; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[10:11]4010; GFX7LESS-NEXT: s_cbranch_execnz .LBB7_24011; GFX7LESS-NEXT: ; %bb.3: ; %Flow4012; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]4013; GFX7LESS-NEXT: .LBB7_4: ; %Flow34014; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]4015; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)4016; GFX7LESS-NEXT: s_mov_b32 s3, 0xf0004017; GFX7LESS-NEXT: s_mov_b32 s2, -14018; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v04019; GFX7LESS-NEXT: s_waitcnt expcnt(0)4020; GFX7LESS-NEXT: v_mul_lo_u32 v0, s12, v24021; GFX7LESS-NEXT: v_sub_i32_e32 v0, vcc, s4, v04022; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 04023; GFX7LESS-NEXT: s_endpgm4024;4025; GFX8-LABEL: sub_i32_uniform:4026; GFX8: ; %bb.0: ; %entry4027; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244028; GFX8-NEXT: s_load_dword s12, s[4:5], 0x344029; GFX8-NEXT: s_mov_b64 s[6:7], exec4030; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 04031; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v04032; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v24033; GFX8-NEXT: ; implicit-def: $vgpr04034; GFX8-NEXT: s_and_saveexec_b64 s[8:9], vcc4035; GFX8-NEXT: s_cbranch_execz .LBB7_44036; GFX8-NEXT: ; %bb.1:4037; GFX8-NEXT: s_waitcnt lgkmcnt(0)4038; GFX8-NEXT: s_load_dword s4, s[2:3], 0x04039; GFX8-NEXT: s_bcnt1_i32_b64 s5, s[6:7]4040; GFX8-NEXT: s_mov_b64 s[10:11], 04041; GFX8-NEXT: s_mov_b32 s7, 0xf0004042; GFX8-NEXT: s_mul_i32 s13, s12, s54043; GFX8-NEXT: s_waitcnt lgkmcnt(0)4044; GFX8-NEXT: v_mov_b32_e32 v0, s44045; GFX8-NEXT: s_mov_b32 s6, -14046; GFX8-NEXT: s_mov_b32 s4, s24047; GFX8-NEXT: s_mov_b32 s5, s34048; GFX8-NEXT: .LBB7_2: ; %atomicrmw.start4049; GFX8-NEXT: ; =>This Inner Loop Header: Depth=14050; GFX8-NEXT: v_mov_b32_e32 v4, v04051; GFX8-NEXT: v_subrev_u32_e32 v3, vcc, s13, v44052; GFX8-NEXT: v_mov_b32_e32 v0, v34053; GFX8-NEXT: v_mov_b32_e32 v1, v44054; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc4055; GFX8-NEXT: s_waitcnt vmcnt(0)4056; GFX8-NEXT: buffer_wbinvl1_vol4057; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v44058; GFX8-NEXT: s_or_b64 s[10:11], vcc, s[10:11]4059; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]4060; GFX8-NEXT: s_cbranch_execnz .LBB7_24061; GFX8-NEXT: ; %bb.3: ; %Flow4062; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]4063; GFX8-NEXT: .LBB7_4: ; %Flow34064; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]4065; GFX8-NEXT: s_waitcnt lgkmcnt(0)4066; GFX8-NEXT: v_mul_lo_u32 v1, s12, v24067; GFX8-NEXT: v_readfirstlane_b32 s4, v04068; GFX8-NEXT: s_mov_b32 s3, 0xf0004069; GFX8-NEXT: s_mov_b32 s2, -14070; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s4, v14071; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 04072; GFX8-NEXT: s_endpgm4073;4074; GFX9-LABEL: sub_i32_uniform:4075; GFX9: ; %bb.0: ; %entry4076; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244077; GFX9-NEXT: s_load_dword s12, s[4:5], 0x344078; GFX9-NEXT: s_mov_b64 s[6:7], exec4079; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 04080; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v04081; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v24082; GFX9-NEXT: ; implicit-def: $vgpr04083; GFX9-NEXT: s_and_saveexec_b64 s[8:9], vcc4084; GFX9-NEXT: s_cbranch_execz .LBB7_44085; GFX9-NEXT: ; %bb.1:4086; GFX9-NEXT: s_waitcnt lgkmcnt(0)4087; GFX9-NEXT: s_load_dword s4, s[2:3], 0x04088; GFX9-NEXT: s_bcnt1_i32_b64 s5, s[6:7]4089; GFX9-NEXT: s_mov_b64 s[10:11], 04090; GFX9-NEXT: s_mov_b32 s7, 0xf0004091; GFX9-NEXT: s_mul_i32 s13, s12, s54092; GFX9-NEXT: s_waitcnt lgkmcnt(0)4093; GFX9-NEXT: v_mov_b32_e32 v0, s44094; GFX9-NEXT: s_mov_b32 s6, -14095; GFX9-NEXT: s_mov_b32 s4, s24096; GFX9-NEXT: s_mov_b32 s5, s34097; GFX9-NEXT: .LBB7_2: ; %atomicrmw.start4098; GFX9-NEXT: ; =>This Inner Loop Header: Depth=14099; GFX9-NEXT: v_mov_b32_e32 v4, v04100; GFX9-NEXT: v_subrev_u32_e32 v3, s13, v44101; GFX9-NEXT: v_mov_b32_e32 v0, v34102; GFX9-NEXT: v_mov_b32_e32 v1, v44103; GFX9-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc4104; GFX9-NEXT: s_waitcnt vmcnt(0)4105; GFX9-NEXT: buffer_wbinvl1_vol4106; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v44107; GFX9-NEXT: s_or_b64 s[10:11], vcc, s[10:11]4108; GFX9-NEXT: s_andn2_b64 exec, exec, s[10:11]4109; GFX9-NEXT: s_cbranch_execnz .LBB7_24110; GFX9-NEXT: ; %bb.3: ; %Flow4111; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]4112; GFX9-NEXT: .LBB7_4: ; %Flow34113; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]4114; GFX9-NEXT: s_waitcnt lgkmcnt(0)4115; GFX9-NEXT: v_mul_lo_u32 v1, s12, v24116; GFX9-NEXT: v_readfirstlane_b32 s4, v04117; GFX9-NEXT: s_mov_b32 s3, 0xf0004118; GFX9-NEXT: s_mov_b32 s2, -14119; GFX9-NEXT: v_sub_u32_e32 v0, s4, v14120; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 04121; GFX9-NEXT: s_endpgm4122;4123; GFX1064-LABEL: sub_i32_uniform:4124; GFX1064: ; %bb.0: ; %entry4125; GFX1064-NEXT: s_clause 0x14126; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244127; GFX1064-NEXT: s_load_dword s12, s[4:5], 0x344128; GFX1064-NEXT: s_mov_b64 s[6:7], exec4129; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 04130; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v04131; GFX1064-NEXT: ; implicit-def: $vgpr04132; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v24133; GFX1064-NEXT: s_and_saveexec_b64 s[8:9], vcc4134; GFX1064-NEXT: s_cbranch_execz .LBB7_44135; GFX1064-NEXT: ; %bb.1:4136; GFX1064-NEXT: s_waitcnt lgkmcnt(0)4137; GFX1064-NEXT: s_load_dword s4, s[2:3], 0x04138; GFX1064-NEXT: s_bcnt1_i32_b64 s5, s[6:7]4139; GFX1064-NEXT: s_mov_b64 s[10:11], 04140; GFX1064-NEXT: s_mul_i32 s13, s12, s54141; GFX1064-NEXT: s_mov_b32 s7, 0x310160004142; GFX1064-NEXT: s_mov_b32 s6, -14143; GFX1064-NEXT: s_mov_b32 s5, s34144; GFX1064-NEXT: s_waitcnt lgkmcnt(0)4145; GFX1064-NEXT: v_mov_b32_e32 v0, s44146; GFX1064-NEXT: s_mov_b32 s4, s24147; GFX1064-NEXT: .LBB7_2: ; %atomicrmw.start4148; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=14149; GFX1064-NEXT: v_mov_b32_e32 v4, v04150; GFX1064-NEXT: v_subrev_nc_u32_e32 v3, s13, v44151; GFX1064-NEXT: v_mov_b32_e32 v0, v34152; GFX1064-NEXT: v_mov_b32_e32 v1, v44153; GFX1064-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc4154; GFX1064-NEXT: s_waitcnt vmcnt(0)4155; GFX1064-NEXT: buffer_gl1_inv4156; GFX1064-NEXT: buffer_gl0_inv4157; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v44158; GFX1064-NEXT: s_or_b64 s[10:11], vcc, s[10:11]4159; GFX1064-NEXT: s_andn2_b64 exec, exec, s[10:11]4160; GFX1064-NEXT: s_cbranch_execnz .LBB7_24161; GFX1064-NEXT: ; %bb.3: ; %Flow4162; GFX1064-NEXT: s_or_b64 exec, exec, s[10:11]4163; GFX1064-NEXT: .LBB7_4: ; %Flow34164; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]4165; GFX1064-NEXT: s_waitcnt lgkmcnt(0)4166; GFX1064-NEXT: v_mul_lo_u32 v1, s12, v24167; GFX1064-NEXT: v_readfirstlane_b32 s2, v04168; GFX1064-NEXT: s_mov_b32 s3, 0x310160004169; GFX1064-NEXT: v_sub_nc_u32_e32 v0, s2, v14170; GFX1064-NEXT: s_mov_b32 s2, -14171; GFX1064-NEXT: buffer_store_dword v0, off, s[0:3], 04172; GFX1064-NEXT: s_endpgm4173;4174; GFX1032-LABEL: sub_i32_uniform:4175; GFX1032: ; %bb.0: ; %entry4176; GFX1032-NEXT: s_clause 0x14177; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244178; GFX1032-NEXT: s_load_dword s8, s[4:5], 0x344179; GFX1032-NEXT: s_mov_b32 s6, exec_lo4180; GFX1032-NEXT: s_mov_b32 s10, 04181; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 04182; GFX1032-NEXT: ; implicit-def: $vgpr04183; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v24184; GFX1032-NEXT: s_and_saveexec_b32 s9, vcc_lo4185; GFX1032-NEXT: s_cbranch_execz .LBB7_44186; GFX1032-NEXT: ; %bb.1:4187; GFX1032-NEXT: s_waitcnt lgkmcnt(0)4188; GFX1032-NEXT: s_load_dword s4, s[2:3], 0x04189; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s64190; GFX1032-NEXT: s_mov_b32 s7, 0x310160004191; GFX1032-NEXT: s_mul_i32 s11, s8, s54192; GFX1032-NEXT: s_mov_b32 s6, -14193; GFX1032-NEXT: s_mov_b32 s5, s34194; GFX1032-NEXT: s_waitcnt lgkmcnt(0)4195; GFX1032-NEXT: v_mov_b32_e32 v0, s44196; GFX1032-NEXT: s_mov_b32 s4, s24197; GFX1032-NEXT: .LBB7_2: ; %atomicrmw.start4198; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=14199; GFX1032-NEXT: v_mov_b32_e32 v4, v04200; GFX1032-NEXT: v_subrev_nc_u32_e32 v3, s11, v44201; GFX1032-NEXT: v_mov_b32_e32 v0, v34202; GFX1032-NEXT: v_mov_b32_e32 v1, v44203; GFX1032-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc4204; GFX1032-NEXT: s_waitcnt vmcnt(0)4205; GFX1032-NEXT: buffer_gl1_inv4206; GFX1032-NEXT: buffer_gl0_inv4207; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v44208; GFX1032-NEXT: s_or_b32 s10, vcc_lo, s104209; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s104210; GFX1032-NEXT: s_cbranch_execnz .LBB7_24211; GFX1032-NEXT: ; %bb.3: ; %Flow4212; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s104213; GFX1032-NEXT: .LBB7_4: ; %Flow34214; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s94215; GFX1032-NEXT: s_waitcnt lgkmcnt(0)4216; GFX1032-NEXT: v_mul_lo_u32 v1, s8, v24217; GFX1032-NEXT: v_readfirstlane_b32 s2, v04218; GFX1032-NEXT: s_mov_b32 s3, 0x310160004219; GFX1032-NEXT: v_sub_nc_u32_e32 v0, s2, v14220; GFX1032-NEXT: s_mov_b32 s2, -14221; GFX1032-NEXT: buffer_store_dword v0, off, s[0:3], 04222; GFX1032-NEXT: s_endpgm4223;4224; GFX1164-LABEL: sub_i32_uniform:4225; GFX1164: ; %bb.0: ; %entry4226; GFX1164-NEXT: s_clause 0x14227; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x244228; GFX1164-NEXT: s_load_b32 s12, s[4:5], 0x344229; GFX1164-NEXT: s_mov_b64 s[6:7], exec4230; GFX1164-NEXT: s_mov_b64 s[8:9], exec4231; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 04232; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4233; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v04234; GFX1164-NEXT: ; implicit-def: $vgpr04235; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v24236; GFX1164-NEXT: s_cbranch_execz .LBB7_44237; GFX1164-NEXT: ; %bb.1:4238; GFX1164-NEXT: s_waitcnt lgkmcnt(0)4239; GFX1164-NEXT: s_load_b32 s4, s[2:3], 0x04240; GFX1164-NEXT: s_bcnt1_i32_b64 s5, s[6:7]4241; GFX1164-NEXT: s_mov_b64 s[10:11], 04242; GFX1164-NEXT: s_mul_i32 s13, s12, s54243; GFX1164-NEXT: s_mov_b32 s7, 0x310160004244; GFX1164-NEXT: s_mov_b32 s6, -14245; GFX1164-NEXT: s_mov_b32 s5, s34246; GFX1164-NEXT: s_waitcnt lgkmcnt(0)4247; GFX1164-NEXT: v_mov_b32_e32 v0, s44248; GFX1164-NEXT: s_mov_b32 s4, s24249; GFX1164-NEXT: .LBB7_2: ; %atomicrmw.start4250; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=14251; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4252; GFX1164-NEXT: v_mov_b32_e32 v4, v04253; GFX1164-NEXT: v_subrev_nc_u32_e32 v3, s13, v44254; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)4255; GFX1164-NEXT: v_mov_b32_e32 v0, v34256; GFX1164-NEXT: v_mov_b32_e32 v1, v44257; GFX1164-NEXT: buffer_atomic_cmpswap_b32 v[0:1], off, s[4:7], 0 glc4258; GFX1164-NEXT: s_waitcnt vmcnt(0)4259; GFX1164-NEXT: buffer_gl1_inv4260; GFX1164-NEXT: buffer_gl0_inv4261; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v44262; GFX1164-NEXT: s_or_b64 s[10:11], vcc, s[10:11]4263; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4264; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[10:11]4265; GFX1164-NEXT: s_cbranch_execnz .LBB7_24266; GFX1164-NEXT: ; %bb.3: ; %Flow4267; GFX1164-NEXT: s_or_b64 exec, exec, s[10:11]4268; GFX1164-NEXT: .LBB7_4: ; %Flow34269; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4270; GFX1164-NEXT: s_or_b64 exec, exec, s[8:9]4271; GFX1164-NEXT: s_waitcnt lgkmcnt(0)4272; GFX1164-NEXT: v_mul_lo_u32 v1, s12, v24273; GFX1164-NEXT: v_readfirstlane_b32 s2, v04274; GFX1164-NEXT: s_mov_b32 s3, 0x310160004275; GFX1164-NEXT: v_sub_nc_u32_e32 v0, s2, v14276; GFX1164-NEXT: s_mov_b32 s2, -14277; GFX1164-NEXT: buffer_store_b32 v0, off, s[0:3], 04278; GFX1164-NEXT: s_endpgm4279;4280; GFX1132-LABEL: sub_i32_uniform:4281; GFX1132: ; %bb.0: ; %entry4282; GFX1132-NEXT: s_clause 0x14283; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x244284; GFX1132-NEXT: s_load_b32 s8, s[4:5], 0x344285; GFX1132-NEXT: s_mov_b32 s6, exec_lo4286; GFX1132-NEXT: s_mov_b32 s10, 04287; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 04288; GFX1132-NEXT: s_mov_b32 s9, exec_lo4289; GFX1132-NEXT: ; implicit-def: $vgpr04290; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)4291; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v24292; GFX1132-NEXT: s_cbranch_execz .LBB7_44293; GFX1132-NEXT: ; %bb.1:4294; GFX1132-NEXT: s_waitcnt lgkmcnt(0)4295; GFX1132-NEXT: s_load_b32 s4, s[2:3], 0x04296; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s64297; GFX1132-NEXT: s_mov_b32 s7, 0x310160004298; GFX1132-NEXT: s_mul_i32 s11, s8, s54299; GFX1132-NEXT: s_mov_b32 s6, -14300; GFX1132-NEXT: s_mov_b32 s5, s34301; GFX1132-NEXT: s_waitcnt lgkmcnt(0)4302; GFX1132-NEXT: v_mov_b32_e32 v0, s44303; GFX1132-NEXT: s_mov_b32 s4, s24304; GFX1132-NEXT: .LBB7_2: ; %atomicrmw.start4305; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=14306; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4307; GFX1132-NEXT: v_mov_b32_e32 v4, v04308; GFX1132-NEXT: v_subrev_nc_u32_e32 v3, s11, v44309; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)4310; GFX1132-NEXT: v_dual_mov_b32 v0, v3 :: v_dual_mov_b32 v1, v44311; GFX1132-NEXT: buffer_atomic_cmpswap_b32 v[0:1], off, s[4:7], 0 glc4312; GFX1132-NEXT: s_waitcnt vmcnt(0)4313; GFX1132-NEXT: buffer_gl1_inv4314; GFX1132-NEXT: buffer_gl0_inv4315; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v44316; GFX1132-NEXT: s_or_b32 s10, vcc_lo, s104317; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4318; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s104319; GFX1132-NEXT: s_cbranch_execnz .LBB7_24320; GFX1132-NEXT: ; %bb.3: ; %Flow4321; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s104322; GFX1132-NEXT: .LBB7_4: ; %Flow34323; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4324; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s94325; GFX1132-NEXT: s_waitcnt lgkmcnt(0)4326; GFX1132-NEXT: v_mul_lo_u32 v1, s8, v24327; GFX1132-NEXT: v_readfirstlane_b32 s2, v04328; GFX1132-NEXT: s_mov_b32 s3, 0x310160004329; GFX1132-NEXT: v_sub_nc_u32_e32 v0, s2, v14330; GFX1132-NEXT: s_mov_b32 s2, -14331; GFX1132-NEXT: buffer_store_b32 v0, off, s[0:3], 04332; GFX1132-NEXT: s_endpgm4333;4334; GFX1264-LABEL: sub_i32_uniform:4335; GFX1264: ; %bb.0: ; %entry4336; GFX1264-NEXT: s_clause 0x14337; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x244338; GFX1264-NEXT: s_load_b32 s8, s[4:5], 0x344339; GFX1264-NEXT: s_mov_b64 s[6:7], exec4340; GFX1264-NEXT: s_mov_b64 s[4:5], exec4341; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 04342; GFX1264-NEXT: ; implicit-def: $vgpr14343; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4344; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v04345; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v04346; GFX1264-NEXT: s_cbranch_execz .LBB7_24347; GFX1264-NEXT: ; %bb.1:4348; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[6:7]4349; GFX1264-NEXT: s_mov_b32 s15, 0x310160004350; GFX1264-NEXT: s_wait_kmcnt 0x04351; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)4352; GFX1264-NEXT: s_mul_i32 s6, s8, s64353; GFX1264-NEXT: s_mov_b32 s14, -14354; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)4355; GFX1264-NEXT: v_mov_b32_e32 v1, s64356; GFX1264-NEXT: s_mov_b32 s12, s24357; GFX1264-NEXT: s_mov_b32 s13, s34358; GFX1264-NEXT: buffer_atomic_sub_u32 v1, off, s[12:15], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV4359; GFX1264-NEXT: s_wait_loadcnt 0x04360; GFX1264-NEXT: global_inv scope:SCOPE_DEV4361; GFX1264-NEXT: .LBB7_2:4362; GFX1264-NEXT: s_or_b64 exec, exec, s[4:5]4363; GFX1264-NEXT: s_wait_kmcnt 0x04364; GFX1264-NEXT: v_mul_lo_u32 v0, s8, v04365; GFX1264-NEXT: v_readfirstlane_b32 s2, v14366; GFX1264-NEXT: s_mov_b32 s3, 0x310160004367; GFX1264-NEXT: v_sub_nc_u32_e32 v0, s2, v04368; GFX1264-NEXT: s_mov_b32 s2, -14369; GFX1264-NEXT: buffer_store_b32 v0, off, s[0:3], null4370; GFX1264-NEXT: s_endpgm4371;4372; GFX1232-LABEL: sub_i32_uniform:4373; GFX1232: ; %bb.0: ; %entry4374; GFX1232-NEXT: s_clause 0x14375; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x244376; GFX1232-NEXT: s_load_b32 s4, s[4:5], 0x344377; GFX1232-NEXT: s_mov_b32 s6, exec_lo4378; GFX1232-NEXT: s_mov_b32 s5, exec_lo4379; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 04380; GFX1232-NEXT: ; implicit-def: $vgpr14381; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)4382; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v04383; GFX1232-NEXT: s_cbranch_execz .LBB7_24384; GFX1232-NEXT: ; %bb.1:4385; GFX1232-NEXT: s_bcnt1_i32_b32 s6, s64386; GFX1232-NEXT: s_mov_b32 s11, 0x310160004387; GFX1232-NEXT: s_wait_kmcnt 0x04388; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)4389; GFX1232-NEXT: s_mul_i32 s6, s4, s64390; GFX1232-NEXT: s_mov_b32 s10, -14391; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)4392; GFX1232-NEXT: v_mov_b32_e32 v1, s64393; GFX1232-NEXT: s_mov_b32 s8, s24394; GFX1232-NEXT: s_mov_b32 s9, s34395; GFX1232-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV4396; GFX1232-NEXT: s_wait_loadcnt 0x04397; GFX1232-NEXT: global_inv scope:SCOPE_DEV4398; GFX1232-NEXT: .LBB7_2:4399; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s54400; GFX1232-NEXT: s_wait_kmcnt 0x04401; GFX1232-NEXT: v_mul_lo_u32 v0, s4, v04402; GFX1232-NEXT: v_readfirstlane_b32 s2, v14403; GFX1232-NEXT: s_mov_b32 s3, 0x310160004404; GFX1232-NEXT: v_sub_nc_u32_e32 v0, s2, v04405; GFX1232-NEXT: s_mov_b32 s2, -14406; GFX1232-NEXT: buffer_store_b32 v0, off, s[0:3], null4407; GFX1232-NEXT: s_endpgm4408entry:4409 %old = atomicrmw sub ptr addrspace(1) %inout, i32 %subitive syncscope("agent") acq_rel4410 store i32 %old, ptr addrspace(1) %out4411 ret void4412}4413 4414define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(1) %inout) {4415; GFX7LESS_ITERATIVE-LABEL: sub_i32_varying:4416; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry4417; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec4418; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s12, 04419; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr24420; GFX7LESS_ITERATIVE-NEXT: .LBB8_1: ; %ComputeLoop4421; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=14422; GFX7LESS_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]4423; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 m0, s24424; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s24425; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v2, s12, m04426; GFX7LESS_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s24427; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]4428; GFX7LESS_ITERATIVE-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 04429; GFX7LESS_ITERATIVE-NEXT: s_and_b64 vcc, exec, s[2:3]4430; GFX7LESS_ITERATIVE-NEXT: s_add_i32 s12, s12, s64431; GFX7LESS_ITERATIVE-NEXT: s_cbranch_vccnz .LBB8_14432; GFX7LESS_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd4433; GFX7LESS_ITERATIVE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94434; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 04435; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v04436; GFX7LESS_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v04437; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr04438; GFX7LESS_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc4439; GFX7LESS_ITERATIVE-NEXT: s_xor_b64 s[8:9], exec, s[4:5]4440; GFX7LESS_ITERATIVE-NEXT: s_cbranch_execz .LBB8_64441; GFX7LESS_ITERATIVE-NEXT: ; %bb.3:4442; GFX7LESS_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)4443; GFX7LESS_ITERATIVE-NEXT: s_load_dword s4, s[2:3], 0x04444; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[10:11], 04445; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s7, 0xf0004446; GFX7LESS_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)4447; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, s44448; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s6, -14449; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s4, s24450; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s5, s34451; GFX7LESS_ITERATIVE-NEXT: .LBB8_4: ; %atomicrmw.start4452; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=14453; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v4, v04454; GFX7LESS_ITERATIVE-NEXT: v_subrev_i32_e32 v3, vcc, s12, v44455; GFX7LESS_ITERATIVE-NEXT: s_waitcnt expcnt(0)4456; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, v34457; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, v44458; GFX7LESS_ITERATIVE-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc4459; GFX7LESS_ITERATIVE-NEXT: s_waitcnt vmcnt(0)4460; GFX7LESS_ITERATIVE-NEXT: buffer_wbinvl14461; GFX7LESS_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, v0, v44462; GFX7LESS_ITERATIVE-NEXT: s_or_b64 s[10:11], vcc, s[10:11]4463; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 exec, exec, s[10:11]4464; GFX7LESS_ITERATIVE-NEXT: s_cbranch_execnz .LBB8_44465; GFX7LESS_ITERATIVE-NEXT: ; %bb.5: ; %Flow4466; GFX7LESS_ITERATIVE-NEXT: s_or_b64 exec, exec, s[10:11]4467; GFX7LESS_ITERATIVE-NEXT: .LBB8_6: ; %Flow44468; GFX7LESS_ITERATIVE-NEXT: s_or_b64 exec, exec, s[8:9]4469; GFX7LESS_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)4470; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s3, 0xf0004471; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s2, -14472; GFX7LESS_ITERATIVE-NEXT: v_readfirstlane_b32 s4, v04473; GFX7LESS_ITERATIVE-NEXT: s_waitcnt expcnt(0)4474; GFX7LESS_ITERATIVE-NEXT: v_sub_i32_e32 v0, vcc, s4, v24475; GFX7LESS_ITERATIVE-NEXT: buffer_store_dword v0, off, s[0:3], 04476; GFX7LESS_ITERATIVE-NEXT: s_endpgm4477;4478; GFX8_ITERATIVE-LABEL: sub_i32_varying:4479; GFX8_ITERATIVE: ; %bb.0: ; %entry4480; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec4481; GFX8_ITERATIVE-NEXT: s_mov_b32 s12, 04482; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr24483; GFX8_ITERATIVE-NEXT: .LBB8_1: ; %ComputeLoop4484; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=14485; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]4486; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s24487; GFX8_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s24488; GFX8_ITERATIVE-NEXT: v_writelane_b32 v2, s12, m04489; GFX8_ITERATIVE-NEXT: s_add_i32 s12, s12, s34490; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s24491; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]4492; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB8_14493; GFX8_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd4494; GFX8_ITERATIVE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244495; GFX8_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 04496; GFX8_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v04497; GFX8_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v04498; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr04499; GFX8_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc4500; GFX8_ITERATIVE-NEXT: s_xor_b64 s[8:9], exec, s[4:5]4501; GFX8_ITERATIVE-NEXT: s_cbranch_execz .LBB8_64502; GFX8_ITERATIVE-NEXT: ; %bb.3:4503; GFX8_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)4504; GFX8_ITERATIVE-NEXT: s_load_dword s5, s[2:3], 0x04505; GFX8_ITERATIVE-NEXT: s_mov_b64 s[10:11], 04506; GFX8_ITERATIVE-NEXT: s_mov_b32 s7, 0xf0004507; GFX8_ITERATIVE-NEXT: s_mov_b32 s6, -14508; GFX8_ITERATIVE-NEXT: s_mov_b32 s4, s24509; GFX8_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)4510; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, s54511; GFX8_ITERATIVE-NEXT: s_mov_b32 s5, s34512; GFX8_ITERATIVE-NEXT: .LBB8_4: ; %atomicrmw.start4513; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=14514; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, v04515; GFX8_ITERATIVE-NEXT: v_subrev_u32_e32 v3, vcc, s12, v44516; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, v34517; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, v44518; GFX8_ITERATIVE-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc4519; GFX8_ITERATIVE-NEXT: s_waitcnt vmcnt(0)4520; GFX8_ITERATIVE-NEXT: buffer_wbinvl1_vol4521; GFX8_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, v0, v44522; GFX8_ITERATIVE-NEXT: s_or_b64 s[10:11], vcc, s[10:11]4523; GFX8_ITERATIVE-NEXT: s_andn2_b64 exec, exec, s[10:11]4524; GFX8_ITERATIVE-NEXT: s_cbranch_execnz .LBB8_44525; GFX8_ITERATIVE-NEXT: ; %bb.5: ; %Flow4526; GFX8_ITERATIVE-NEXT: s_or_b64 exec, exec, s[10:11]4527; GFX8_ITERATIVE-NEXT: .LBB8_6: ; %Flow44528; GFX8_ITERATIVE-NEXT: s_or_b64 exec, exec, s[8:9]4529; GFX8_ITERATIVE-NEXT: v_readfirstlane_b32 s4, v04530; GFX8_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)4531; GFX8_ITERATIVE-NEXT: s_mov_b32 s3, 0xf0004532; GFX8_ITERATIVE-NEXT: s_mov_b32 s2, -14533; GFX8_ITERATIVE-NEXT: v_sub_u32_e32 v0, vcc, s4, v24534; GFX8_ITERATIVE-NEXT: buffer_store_dword v0, off, s[0:3], 04535; GFX8_ITERATIVE-NEXT: s_endpgm4536;4537; GFX9_ITERATIVE-LABEL: sub_i32_varying:4538; GFX9_ITERATIVE: ; %bb.0: ; %entry4539; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec4540; GFX9_ITERATIVE-NEXT: s_mov_b32 s12, 04541; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr24542; GFX9_ITERATIVE-NEXT: .LBB8_1: ; %ComputeLoop4543; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=14544; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]4545; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s24546; GFX9_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s24547; GFX9_ITERATIVE-NEXT: v_writelane_b32 v2, s12, m04548; GFX9_ITERATIVE-NEXT: s_add_i32 s12, s12, s34549; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s24550; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]4551; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB8_14552; GFX9_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd4553; GFX9_ITERATIVE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244554; GFX9_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 04555; GFX9_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v04556; GFX9_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v04557; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr04558; GFX9_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc4559; GFX9_ITERATIVE-NEXT: s_xor_b64 s[8:9], exec, s[4:5]4560; GFX9_ITERATIVE-NEXT: s_cbranch_execz .LBB8_64561; GFX9_ITERATIVE-NEXT: ; %bb.3:4562; GFX9_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)4563; GFX9_ITERATIVE-NEXT: s_load_dword s5, s[2:3], 0x04564; GFX9_ITERATIVE-NEXT: s_mov_b64 s[10:11], 04565; GFX9_ITERATIVE-NEXT: s_mov_b32 s7, 0xf0004566; GFX9_ITERATIVE-NEXT: s_mov_b32 s6, -14567; GFX9_ITERATIVE-NEXT: s_mov_b32 s4, s24568; GFX9_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)4569; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, s54570; GFX9_ITERATIVE-NEXT: s_mov_b32 s5, s34571; GFX9_ITERATIVE-NEXT: .LBB8_4: ; %atomicrmw.start4572; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=14573; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, v04574; GFX9_ITERATIVE-NEXT: v_subrev_u32_e32 v3, s12, v44575; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, v34576; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, v44577; GFX9_ITERATIVE-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc4578; GFX9_ITERATIVE-NEXT: s_waitcnt vmcnt(0)4579; GFX9_ITERATIVE-NEXT: buffer_wbinvl1_vol4580; GFX9_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, v0, v44581; GFX9_ITERATIVE-NEXT: s_or_b64 s[10:11], vcc, s[10:11]4582; GFX9_ITERATIVE-NEXT: s_andn2_b64 exec, exec, s[10:11]4583; GFX9_ITERATIVE-NEXT: s_cbranch_execnz .LBB8_44584; GFX9_ITERATIVE-NEXT: ; %bb.5: ; %Flow4585; GFX9_ITERATIVE-NEXT: s_or_b64 exec, exec, s[10:11]4586; GFX9_ITERATIVE-NEXT: .LBB8_6: ; %Flow44587; GFX9_ITERATIVE-NEXT: s_or_b64 exec, exec, s[8:9]4588; GFX9_ITERATIVE-NEXT: v_readfirstlane_b32 s4, v04589; GFX9_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)4590; GFX9_ITERATIVE-NEXT: s_mov_b32 s3, 0xf0004591; GFX9_ITERATIVE-NEXT: s_mov_b32 s2, -14592; GFX9_ITERATIVE-NEXT: v_sub_u32_e32 v0, s4, v24593; GFX9_ITERATIVE-NEXT: buffer_store_dword v0, off, s[0:3], 04594; GFX9_ITERATIVE-NEXT: s_endpgm4595;4596; GFX1064_ITERATIVE-LABEL: sub_i32_varying:4597; GFX1064_ITERATIVE: ; %bb.0: ; %entry4598; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec4599; GFX1064_ITERATIVE-NEXT: s_mov_b32 s12, 04600; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr24601; GFX1064_ITERATIVE-NEXT: .LBB8_1: ; %ComputeLoop4602; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=14603; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]4604; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s24605; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v2, s12, s24606; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s24607; GFX1064_ITERATIVE-NEXT: s_add_i32 s12, s12, s64608; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]4609; GFX1064_ITERATIVE-NEXT: s_cbranch_scc1 .LBB8_14610; GFX1064_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd4611; GFX1064_ITERATIVE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244612; GFX1064_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 04613; GFX1064_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v04614; GFX1064_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v04615; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr04616; GFX1064_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc4617; GFX1064_ITERATIVE-NEXT: s_xor_b64 s[8:9], exec, s[4:5]4618; GFX1064_ITERATIVE-NEXT: s_cbranch_execz .LBB8_64619; GFX1064_ITERATIVE-NEXT: ; %bb.3:4620; GFX1064_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)4621; GFX1064_ITERATIVE-NEXT: s_load_dword s4, s[2:3], 0x04622; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[10:11], 04623; GFX1064_ITERATIVE-NEXT: s_mov_b32 s7, 0x310160004624; GFX1064_ITERATIVE-NEXT: s_mov_b32 s6, -14625; GFX1064_ITERATIVE-NEXT: s_mov_b32 s5, s34626; GFX1064_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)4627; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v0, s44628; GFX1064_ITERATIVE-NEXT: s_mov_b32 s4, s24629; GFX1064_ITERATIVE-NEXT: .LBB8_4: ; %atomicrmw.start4630; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=14631; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v4, v04632; GFX1064_ITERATIVE-NEXT: v_subrev_nc_u32_e32 v3, s12, v44633; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v0, v34634; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v1, v44635; GFX1064_ITERATIVE-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc4636; GFX1064_ITERATIVE-NEXT: s_waitcnt vmcnt(0)4637; GFX1064_ITERATIVE-NEXT: buffer_gl1_inv4638; GFX1064_ITERATIVE-NEXT: buffer_gl0_inv4639; GFX1064_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, v0, v44640; GFX1064_ITERATIVE-NEXT: s_or_b64 s[10:11], vcc, s[10:11]4641; GFX1064_ITERATIVE-NEXT: s_andn2_b64 exec, exec, s[10:11]4642; GFX1064_ITERATIVE-NEXT: s_cbranch_execnz .LBB8_44643; GFX1064_ITERATIVE-NEXT: ; %bb.5: ; %Flow4644; GFX1064_ITERATIVE-NEXT: s_or_b64 exec, exec, s[10:11]4645; GFX1064_ITERATIVE-NEXT: .LBB8_6: ; %Flow44646; GFX1064_ITERATIVE-NEXT: s_or_b64 exec, exec, s[8:9]4647; GFX1064_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)4648; GFX1064_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v04649; GFX1064_ITERATIVE-NEXT: s_mov_b32 s3, 0x310160004650; GFX1064_ITERATIVE-NEXT: v_sub_nc_u32_e32 v0, s2, v24651; GFX1064_ITERATIVE-NEXT: s_mov_b32 s2, -14652; GFX1064_ITERATIVE-NEXT: buffer_store_dword v0, off, s[0:3], 04653; GFX1064_ITERATIVE-NEXT: s_endpgm4654;4655; GFX1032_ITERATIVE-LABEL: sub_i32_varying:4656; GFX1032_ITERATIVE: ; %bb.0: ; %entry4657; GFX1032_ITERATIVE-NEXT: s_mov_b32 s0, exec_lo4658; GFX1032_ITERATIVE-NEXT: s_mov_b32 s8, 04659; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr24660; GFX1032_ITERATIVE-NEXT: .LBB8_1: ; %ComputeLoop4661; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=14662; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s1, s04663; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s2, v0, s14664; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s8, s14665; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s1, 1, s14666; GFX1032_ITERATIVE-NEXT: s_add_i32 s8, s8, s24667; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s0, s0, s14668; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB8_14669; GFX1032_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd4670; GFX1032_ITERATIVE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244671; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 04672; GFX1032_ITERATIVE-NEXT: s_mov_b32 s10, 04673; GFX1032_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v04674; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr04675; GFX1032_ITERATIVE-NEXT: s_and_saveexec_b32 s4, vcc_lo4676; GFX1032_ITERATIVE-NEXT: s_xor_b32 s9, exec_lo, s44677; GFX1032_ITERATIVE-NEXT: s_cbranch_execz .LBB8_64678; GFX1032_ITERATIVE-NEXT: ; %bb.3:4679; GFX1032_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)4680; GFX1032_ITERATIVE-NEXT: s_load_dword s4, s[2:3], 0x04681; GFX1032_ITERATIVE-NEXT: s_mov_b32 s7, 0x310160004682; GFX1032_ITERATIVE-NEXT: s_mov_b32 s6, -14683; GFX1032_ITERATIVE-NEXT: s_mov_b32 s5, s34684; GFX1032_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)4685; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v0, s44686; GFX1032_ITERATIVE-NEXT: s_mov_b32 s4, s24687; GFX1032_ITERATIVE-NEXT: .LBB8_4: ; %atomicrmw.start4688; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=14689; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v4, v04690; GFX1032_ITERATIVE-NEXT: v_subrev_nc_u32_e32 v3, s8, v44691; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v0, v34692; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v1, v44693; GFX1032_ITERATIVE-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc4694; GFX1032_ITERATIVE-NEXT: s_waitcnt vmcnt(0)4695; GFX1032_ITERATIVE-NEXT: buffer_gl1_inv4696; GFX1032_ITERATIVE-NEXT: buffer_gl0_inv4697; GFX1032_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v44698; GFX1032_ITERATIVE-NEXT: s_or_b32 s10, vcc_lo, s104699; GFX1032_ITERATIVE-NEXT: s_andn2_b32 exec_lo, exec_lo, s104700; GFX1032_ITERATIVE-NEXT: s_cbranch_execnz .LBB8_44701; GFX1032_ITERATIVE-NEXT: ; %bb.5: ; %Flow4702; GFX1032_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s104703; GFX1032_ITERATIVE-NEXT: .LBB8_6: ; %Flow44704; GFX1032_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s94705; GFX1032_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)4706; GFX1032_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v04707; GFX1032_ITERATIVE-NEXT: s_mov_b32 s3, 0x310160004708; GFX1032_ITERATIVE-NEXT: v_sub_nc_u32_e32 v0, s2, v24709; GFX1032_ITERATIVE-NEXT: s_mov_b32 s2, -14710; GFX1032_ITERATIVE-NEXT: buffer_store_dword v0, off, s[0:3], 04711; GFX1032_ITERATIVE-NEXT: s_endpgm4712;4713; GFX1164_ITERATIVE-LABEL: sub_i32_varying:4714; GFX1164_ITERATIVE: ; %bb.0: ; %entry4715; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v0, 0x3ff, v04716; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec4717; GFX1164_ITERATIVE-NEXT: s_mov_b32 s12, 04718; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr24719; GFX1164_ITERATIVE-NEXT: .LBB8_1: ; %ComputeLoop4720; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=14721; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s2, s[0:1]4722; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)4723; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s24724; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v2, s12, s24725; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s24726; GFX1164_ITERATIVE-NEXT: s_add_i32 s12, s12, s64727; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]4728; GFX1164_ITERATIVE-NEXT: s_cbranch_scc1 .LBB8_14729; GFX1164_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd4730; GFX1164_ITERATIVE-NEXT: s_load_b128 s[0:3], s[4:5], 0x244731; GFX1164_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 04732; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4733; GFX1164_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v04734; GFX1164_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v04735; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr04736; GFX1164_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc4737; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4738; GFX1164_ITERATIVE-NEXT: s_xor_b64 s[8:9], exec, s[4:5]4739; GFX1164_ITERATIVE-NEXT: s_cbranch_execz .LBB8_64740; GFX1164_ITERATIVE-NEXT: ; %bb.3:4741; GFX1164_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)4742; GFX1164_ITERATIVE-NEXT: s_load_b32 s4, s[2:3], 0x04743; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[10:11], 04744; GFX1164_ITERATIVE-NEXT: s_mov_b32 s7, 0x310160004745; GFX1164_ITERATIVE-NEXT: s_mov_b32 s6, -14746; GFX1164_ITERATIVE-NEXT: s_mov_b32 s5, s34747; GFX1164_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)4748; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v0, s44749; GFX1164_ITERATIVE-NEXT: s_mov_b32 s4, s24750; GFX1164_ITERATIVE-NEXT: .LBB8_4: ; %atomicrmw.start4751; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=14752; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4753; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v4, v04754; GFX1164_ITERATIVE-NEXT: v_subrev_nc_u32_e32 v3, s12, v44755; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1)4756; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v0, v34757; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v1, v44758; GFX1164_ITERATIVE-NEXT: buffer_atomic_cmpswap_b32 v[0:1], off, s[4:7], 0 glc4759; GFX1164_ITERATIVE-NEXT: s_waitcnt vmcnt(0)4760; GFX1164_ITERATIVE-NEXT: buffer_gl1_inv4761; GFX1164_ITERATIVE-NEXT: buffer_gl0_inv4762; GFX1164_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, v0, v44763; GFX1164_ITERATIVE-NEXT: s_or_b64 s[10:11], vcc, s[10:11]4764; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4765; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 exec, exec, s[10:11]4766; GFX1164_ITERATIVE-NEXT: s_cbranch_execnz .LBB8_44767; GFX1164_ITERATIVE-NEXT: ; %bb.5: ; %Flow4768; GFX1164_ITERATIVE-NEXT: s_or_b64 exec, exec, s[10:11]4769; GFX1164_ITERATIVE-NEXT: .LBB8_6: ; %Flow44770; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4771; GFX1164_ITERATIVE-NEXT: s_or_b64 exec, exec, s[8:9]4772; GFX1164_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)4773; GFX1164_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v04774; GFX1164_ITERATIVE-NEXT: s_mov_b32 s3, 0x310160004775; GFX1164_ITERATIVE-NEXT: v_sub_nc_u32_e32 v0, s2, v24776; GFX1164_ITERATIVE-NEXT: s_mov_b32 s2, -14777; GFX1164_ITERATIVE-NEXT: buffer_store_b32 v0, off, s[0:3], 04778; GFX1164_ITERATIVE-NEXT: s_endpgm4779;4780; GFX1132_ITERATIVE-LABEL: sub_i32_varying:4781; GFX1132_ITERATIVE: ; %bb.0: ; %entry4782; GFX1132_ITERATIVE-NEXT: v_and_b32_e32 v0, 0x3ff, v04783; GFX1132_ITERATIVE-NEXT: s_mov_b32 s0, exec_lo4784; GFX1132_ITERATIVE-NEXT: s_mov_b32 s8, 04785; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr24786; GFX1132_ITERATIVE-NEXT: .LBB8_1: ; %ComputeLoop4787; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=14788; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s1, s04789; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)4790; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s2, v0, s14791; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v2, s8, s14792; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s1, 1, s14793; GFX1132_ITERATIVE-NEXT: s_add_i32 s8, s8, s24794; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s0, s0, s14795; GFX1132_ITERATIVE-NEXT: s_cbranch_scc1 .LBB8_14796; GFX1132_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd4797; GFX1132_ITERATIVE-NEXT: s_load_b128 s[0:3], s[4:5], 0x244798; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 04799; GFX1132_ITERATIVE-NEXT: s_mov_b32 s10, 04800; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)4801; GFX1132_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v04802; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr04803; GFX1132_ITERATIVE-NEXT: s_and_saveexec_b32 s4, vcc_lo4804; GFX1132_ITERATIVE-NEXT: s_xor_b32 s9, exec_lo, s44805; GFX1132_ITERATIVE-NEXT: s_cbranch_execz .LBB8_64806; GFX1132_ITERATIVE-NEXT: ; %bb.3:4807; GFX1132_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)4808; GFX1132_ITERATIVE-NEXT: s_load_b32 s4, s[2:3], 0x04809; GFX1132_ITERATIVE-NEXT: s_mov_b32 s7, 0x310160004810; GFX1132_ITERATIVE-NEXT: s_mov_b32 s6, -14811; GFX1132_ITERATIVE-NEXT: s_mov_b32 s5, s34812; GFX1132_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)4813; GFX1132_ITERATIVE-NEXT: v_mov_b32_e32 v0, s44814; GFX1132_ITERATIVE-NEXT: s_mov_b32 s4, s24815; GFX1132_ITERATIVE-NEXT: .LBB8_4: ; %atomicrmw.start4816; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=14817; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4818; GFX1132_ITERATIVE-NEXT: v_mov_b32_e32 v4, v04819; GFX1132_ITERATIVE-NEXT: v_subrev_nc_u32_e32 v3, s8, v44820; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1)4821; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v0, v3 :: v_dual_mov_b32 v1, v44822; GFX1132_ITERATIVE-NEXT: buffer_atomic_cmpswap_b32 v[0:1], off, s[4:7], 0 glc4823; GFX1132_ITERATIVE-NEXT: s_waitcnt vmcnt(0)4824; GFX1132_ITERATIVE-NEXT: buffer_gl1_inv4825; GFX1132_ITERATIVE-NEXT: buffer_gl0_inv4826; GFX1132_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v44827; GFX1132_ITERATIVE-NEXT: s_or_b32 s10, vcc_lo, s104828; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4829; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 exec_lo, exec_lo, s104830; GFX1132_ITERATIVE-NEXT: s_cbranch_execnz .LBB8_44831; GFX1132_ITERATIVE-NEXT: ; %bb.5: ; %Flow4832; GFX1132_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s104833; GFX1132_ITERATIVE-NEXT: .LBB8_6: ; %Flow44834; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4835; GFX1132_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s94836; GFX1132_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)4837; GFX1132_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v04838; GFX1132_ITERATIVE-NEXT: s_mov_b32 s3, 0x310160004839; GFX1132_ITERATIVE-NEXT: v_sub_nc_u32_e32 v0, s2, v24840; GFX1132_ITERATIVE-NEXT: s_mov_b32 s2, -14841; GFX1132_ITERATIVE-NEXT: buffer_store_b32 v0, off, s[0:3], 04842; GFX1132_ITERATIVE-NEXT: s_endpgm4843;4844; GFX1264_ITERATIVE-LABEL: sub_i32_varying:4845; GFX1264_ITERATIVE: ; %bb.0: ; %entry4846; GFX1264_ITERATIVE-NEXT: v_and_b32_e32 v1, 0x3ff, v04847; GFX1264_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec4848; GFX1264_ITERATIVE-NEXT: s_mov_b32 s6, 04849; GFX1264_ITERATIVE-NEXT: ; implicit-def: $vgpr04850; GFX1264_ITERATIVE-NEXT: .LBB8_1: ; %ComputeLoop4851; GFX1264_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=14852; GFX1264_ITERATIVE-NEXT: s_ctz_i32_b64 s2, s[0:1]4853; GFX1264_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)4854; GFX1264_ITERATIVE-NEXT: v_readlane_b32 s7, v1, s24855; GFX1264_ITERATIVE-NEXT: v_writelane_b32 v0, s6, s24856; GFX1264_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s24857; GFX1264_ITERATIVE-NEXT: s_add_co_i32 s6, s6, s74858; GFX1264_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)4859; GFX1264_ITERATIVE-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]4860; GFX1264_ITERATIVE-NEXT: s_cbranch_scc1 .LBB8_14861; GFX1264_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd4862; GFX1264_ITERATIVE-NEXT: s_load_b128 s[0:3], s[4:5], 0x244863; GFX1264_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 04864; GFX1264_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4865; GFX1264_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v14866; GFX1264_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v14867; GFX1264_ITERATIVE-NEXT: ; implicit-def: $vgpr14868; GFX1264_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc4869; GFX1264_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4870; GFX1264_ITERATIVE-NEXT: s_xor_b64 s[4:5], exec, s[4:5]4871; GFX1264_ITERATIVE-NEXT: s_cbranch_execz .LBB8_44872; GFX1264_ITERATIVE-NEXT: ; %bb.3:4873; GFX1264_ITERATIVE-NEXT: v_mov_b32_e32 v1, s64874; GFX1264_ITERATIVE-NEXT: s_mov_b32 s11, 0x310160004875; GFX1264_ITERATIVE-NEXT: s_mov_b32 s10, -14876; GFX1264_ITERATIVE-NEXT: s_wait_kmcnt 0x04877; GFX1264_ITERATIVE-NEXT: s_mov_b32 s8, s24878; GFX1264_ITERATIVE-NEXT: s_mov_b32 s9, s34879; GFX1264_ITERATIVE-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV4880; GFX1264_ITERATIVE-NEXT: s_wait_loadcnt 0x04881; GFX1264_ITERATIVE-NEXT: global_inv scope:SCOPE_DEV4882; GFX1264_ITERATIVE-NEXT: .LBB8_4:4883; GFX1264_ITERATIVE-NEXT: s_or_b64 exec, exec, s[4:5]4884; GFX1264_ITERATIVE-NEXT: s_wait_kmcnt 0x04885; GFX1264_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v14886; GFX1264_ITERATIVE-NEXT: s_mov_b32 s3, 0x310160004887; GFX1264_ITERATIVE-NEXT: s_wait_alu depctr_va_sdst(0)4888; GFX1264_ITERATIVE-NEXT: v_sub_nc_u32_e32 v0, s2, v04889; GFX1264_ITERATIVE-NEXT: s_mov_b32 s2, -14890; GFX1264_ITERATIVE-NEXT: buffer_store_b32 v0, off, s[0:3], null4891; GFX1264_ITERATIVE-NEXT: s_endpgm4892;4893; GFX1232_ITERATIVE-LABEL: sub_i32_varying:4894; GFX1232_ITERATIVE: ; %bb.0: ; %entry4895; GFX1232_ITERATIVE-NEXT: v_and_b32_e32 v1, 0x3ff, v04896; GFX1232_ITERATIVE-NEXT: s_mov_b32 s0, exec_lo4897; GFX1232_ITERATIVE-NEXT: s_mov_b32 s6, 04898; GFX1232_ITERATIVE-NEXT: ; implicit-def: $vgpr04899; GFX1232_ITERATIVE-NEXT: .LBB8_1: ; %ComputeLoop4900; GFX1232_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=14901; GFX1232_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)4902; GFX1232_ITERATIVE-NEXT: s_ctz_i32_b32 s1, s04903; GFX1232_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)4904; GFX1232_ITERATIVE-NEXT: v_readlane_b32 s2, v1, s14905; GFX1232_ITERATIVE-NEXT: v_writelane_b32 v0, s6, s14906; GFX1232_ITERATIVE-NEXT: s_lshl_b32 s1, 1, s14907; GFX1232_ITERATIVE-NEXT: s_add_co_i32 s6, s6, s24908; GFX1232_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)4909; GFX1232_ITERATIVE-NEXT: s_and_not1_b32 s0, s0, s14910; GFX1232_ITERATIVE-NEXT: s_cbranch_scc1 .LBB8_14911; GFX1232_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd4912; GFX1232_ITERATIVE-NEXT: s_load_b128 s[0:3], s[4:5], 0x244913; GFX1232_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 04914; GFX1232_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)4915; GFX1232_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v14916; GFX1232_ITERATIVE-NEXT: ; implicit-def: $vgpr14917; GFX1232_ITERATIVE-NEXT: s_and_saveexec_b32 s4, vcc_lo4918; GFX1232_ITERATIVE-NEXT: s_xor_b32 s4, exec_lo, s44919; GFX1232_ITERATIVE-NEXT: s_cbranch_execz .LBB8_44920; GFX1232_ITERATIVE-NEXT: ; %bb.3:4921; GFX1232_ITERATIVE-NEXT: v_mov_b32_e32 v1, s64922; GFX1232_ITERATIVE-NEXT: s_mov_b32 s11, 0x310160004923; GFX1232_ITERATIVE-NEXT: s_mov_b32 s10, -14924; GFX1232_ITERATIVE-NEXT: s_wait_kmcnt 0x04925; GFX1232_ITERATIVE-NEXT: s_mov_b32 s8, s24926; GFX1232_ITERATIVE-NEXT: s_mov_b32 s9, s34927; GFX1232_ITERATIVE-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV4928; GFX1232_ITERATIVE-NEXT: s_wait_loadcnt 0x04929; GFX1232_ITERATIVE-NEXT: global_inv scope:SCOPE_DEV4930; GFX1232_ITERATIVE-NEXT: .LBB8_4:4931; GFX1232_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s44932; GFX1232_ITERATIVE-NEXT: s_wait_kmcnt 0x04933; GFX1232_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v14934; GFX1232_ITERATIVE-NEXT: s_mov_b32 s3, 0x310160004935; GFX1232_ITERATIVE-NEXT: v_sub_nc_u32_e32 v0, s2, v04936; GFX1232_ITERATIVE-NEXT: s_mov_b32 s2, -14937; GFX1232_ITERATIVE-NEXT: buffer_store_b32 v0, off, s[0:3], null4938; GFX1232_ITERATIVE-NEXT: s_endpgm4939;4940; GFX7LESS_DPP-LABEL: sub_i32_varying:4941; GFX7LESS_DPP: ; %bb.0: ; %entry4942; GFX7LESS_DPP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x94943; GFX7LESS_DPP-NEXT: s_waitcnt lgkmcnt(0)4944; GFX7LESS_DPP-NEXT: s_load_dword s4, s[2:3], 0x04945; GFX7LESS_DPP-NEXT: s_mov_b64 s[8:9], 04946; GFX7LESS_DPP-NEXT: s_mov_b32 s7, 0xf0004947; GFX7LESS_DPP-NEXT: s_waitcnt lgkmcnt(0)4948; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v2, s44949; GFX7LESS_DPP-NEXT: s_mov_b32 s6, -14950; GFX7LESS_DPP-NEXT: s_mov_b32 s4, s24951; GFX7LESS_DPP-NEXT: s_mov_b32 s5, s34952; GFX7LESS_DPP-NEXT: .LBB8_1: ; %atomicrmw.start4953; GFX7LESS_DPP-NEXT: ; =>This Inner Loop Header: Depth=14954; GFX7LESS_DPP-NEXT: v_sub_i32_e32 v1, vcc, v2, v04955; GFX7LESS_DPP-NEXT: s_waitcnt expcnt(0)4956; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v4, v24957; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v3, v14958; GFX7LESS_DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[4:7], 0 glc4959; GFX7LESS_DPP-NEXT: s_waitcnt vmcnt(0)4960; GFX7LESS_DPP-NEXT: buffer_wbinvl14961; GFX7LESS_DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v24962; GFX7LESS_DPP-NEXT: s_or_b64 s[8:9], vcc, s[8:9]4963; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v2, v34964; GFX7LESS_DPP-NEXT: s_andn2_b64 exec, exec, s[8:9]4965; GFX7LESS_DPP-NEXT: s_cbranch_execnz .LBB8_14966; GFX7LESS_DPP-NEXT: ; %bb.2: ; %atomicrmw.end4967; GFX7LESS_DPP-NEXT: s_or_b64 exec, exec, s[8:9]4968; GFX7LESS_DPP-NEXT: s_mov_b32 s3, 0xf0004969; GFX7LESS_DPP-NEXT: s_mov_b32 s2, -14970; GFX7LESS_DPP-NEXT: buffer_store_dword v3, off, s[0:3], 04971; GFX7LESS_DPP-NEXT: s_endpgm4972;4973; GFX8_DPP-LABEL: sub_i32_varying:4974; GFX8_DPP: ; %bb.0: ; %entry4975; GFX8_DPP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x244976; GFX8_DPP-NEXT: s_or_saveexec_b64 s[4:5], -14977; GFX8_DPP-NEXT: v_mov_b32_e32 v1, 04978; GFX8_DPP-NEXT: s_mov_b64 exec, s[4:5]4979; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 04980; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v34981; GFX8_DPP-NEXT: s_or_saveexec_b64 s[4:5], -14982; GFX8_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[4:5]4983; GFX8_DPP-NEXT: s_nop 14984; GFX8_DPP-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:14985; GFX8_DPP-NEXT: s_nop 14986; GFX8_DPP-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:14987; GFX8_DPP-NEXT: s_nop 14988; GFX8_DPP-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:14989; GFX8_DPP-NEXT: s_nop 14990; GFX8_DPP-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:14991; GFX8_DPP-NEXT: s_nop 14992; GFX8_DPP-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf4993; GFX8_DPP-NEXT: s_nop 14994; GFX8_DPP-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf4995; GFX8_DPP-NEXT: v_readlane_b32 s12, v2, 634996; GFX8_DPP-NEXT: s_nop 04997; GFX8_DPP-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf4998; GFX8_DPP-NEXT: s_mov_b64 exec, s[4:5]4999; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v35000; GFX8_DPP-NEXT: ; implicit-def: $vgpr35001; GFX8_DPP-NEXT: s_and_saveexec_b64 s[8:9], vcc5002; GFX8_DPP-NEXT: s_cbranch_execz .LBB8_45003; GFX8_DPP-NEXT: ; %bb.1:5004; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0)5005; GFX8_DPP-NEXT: s_load_dword s5, s[2:3], 0x05006; GFX8_DPP-NEXT: s_mov_b64 s[10:11], 05007; GFX8_DPP-NEXT: s_mov_b32 s7, 0xf0005008; GFX8_DPP-NEXT: s_mov_b32 s6, -15009; GFX8_DPP-NEXT: s_mov_b32 s4, s25010; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0)5011; GFX8_DPP-NEXT: v_mov_b32_e32 v3, s55012; GFX8_DPP-NEXT: s_mov_b32 s5, s35013; GFX8_DPP-NEXT: .LBB8_2: ; %atomicrmw.start5014; GFX8_DPP-NEXT: ; =>This Inner Loop Header: Depth=15015; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v35016; GFX8_DPP-NEXT: v_subrev_u32_e32 v4, vcc, s12, v55017; GFX8_DPP-NEXT: v_mov_b32_e32 v3, v45018; GFX8_DPP-NEXT: v_mov_b32_e32 v4, v55019; GFX8_DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[4:7], 0 glc5020; GFX8_DPP-NEXT: s_waitcnt vmcnt(0)5021; GFX8_DPP-NEXT: buffer_wbinvl1_vol5022; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v55023; GFX8_DPP-NEXT: s_or_b64 s[10:11], vcc, s[10:11]5024; GFX8_DPP-NEXT: s_andn2_b64 exec, exec, s[10:11]5025; GFX8_DPP-NEXT: s_cbranch_execnz .LBB8_25026; GFX8_DPP-NEXT: ; %bb.3: ; %Flow5027; GFX8_DPP-NEXT: s_or_b64 exec, exec, s[10:11]5028; GFX8_DPP-NEXT: .LBB8_4: ; %Flow35029; GFX8_DPP-NEXT: s_or_b64 exec, exec, s[8:9]5030; GFX8_DPP-NEXT: v_readfirstlane_b32 s4, v35031; GFX8_DPP-NEXT: v_mov_b32_e32 v0, v15032; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0)5033; GFX8_DPP-NEXT: s_mov_b32 s3, 0xf0005034; GFX8_DPP-NEXT: s_mov_b32 s2, -15035; GFX8_DPP-NEXT: v_sub_u32_e32 v0, vcc, s4, v05036; GFX8_DPP-NEXT: buffer_store_dword v0, off, s[0:3], 05037; GFX8_DPP-NEXT: s_endpgm5038;5039; GFX9_DPP-LABEL: sub_i32_varying:5040; GFX9_DPP: ; %bb.0: ; %entry5041; GFX9_DPP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245042; GFX9_DPP-NEXT: s_or_saveexec_b64 s[4:5], -15043; GFX9_DPP-NEXT: v_mov_b32_e32 v1, 05044; GFX9_DPP-NEXT: s_mov_b64 exec, s[4:5]5045; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 05046; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v35047; GFX9_DPP-NEXT: s_or_saveexec_b64 s[4:5], -15048; GFX9_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[4:5]5049; GFX9_DPP-NEXT: s_nop 15050; GFX9_DPP-NEXT: v_add_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:15051; GFX9_DPP-NEXT: s_nop 15052; GFX9_DPP-NEXT: v_add_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:15053; GFX9_DPP-NEXT: s_nop 15054; GFX9_DPP-NEXT: v_add_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:15055; GFX9_DPP-NEXT: s_nop 15056; GFX9_DPP-NEXT: v_add_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:15057; GFX9_DPP-NEXT: s_nop 15058; GFX9_DPP-NEXT: v_add_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf5059; GFX9_DPP-NEXT: s_nop 15060; GFX9_DPP-NEXT: v_add_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf5061; GFX9_DPP-NEXT: v_readlane_b32 s12, v2, 635062; GFX9_DPP-NEXT: s_nop 05063; GFX9_DPP-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf5064; GFX9_DPP-NEXT: s_mov_b64 exec, s[4:5]5065; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v35066; GFX9_DPP-NEXT: ; implicit-def: $vgpr35067; GFX9_DPP-NEXT: s_and_saveexec_b64 s[8:9], vcc5068; GFX9_DPP-NEXT: s_cbranch_execz .LBB8_45069; GFX9_DPP-NEXT: ; %bb.1:5070; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0)5071; GFX9_DPP-NEXT: s_load_dword s5, s[2:3], 0x05072; GFX9_DPP-NEXT: s_mov_b64 s[10:11], 05073; GFX9_DPP-NEXT: s_mov_b32 s7, 0xf0005074; GFX9_DPP-NEXT: s_mov_b32 s6, -15075; GFX9_DPP-NEXT: s_mov_b32 s4, s25076; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0)5077; GFX9_DPP-NEXT: v_mov_b32_e32 v3, s55078; GFX9_DPP-NEXT: s_mov_b32 s5, s35079; GFX9_DPP-NEXT: .LBB8_2: ; %atomicrmw.start5080; GFX9_DPP-NEXT: ; =>This Inner Loop Header: Depth=15081; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v35082; GFX9_DPP-NEXT: v_subrev_u32_e32 v4, s12, v55083; GFX9_DPP-NEXT: v_mov_b32_e32 v3, v45084; GFX9_DPP-NEXT: v_mov_b32_e32 v4, v55085; GFX9_DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[4:7], 0 glc5086; GFX9_DPP-NEXT: s_waitcnt vmcnt(0)5087; GFX9_DPP-NEXT: buffer_wbinvl1_vol5088; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v55089; GFX9_DPP-NEXT: s_or_b64 s[10:11], vcc, s[10:11]5090; GFX9_DPP-NEXT: s_andn2_b64 exec, exec, s[10:11]5091; GFX9_DPP-NEXT: s_cbranch_execnz .LBB8_25092; GFX9_DPP-NEXT: ; %bb.3: ; %Flow5093; GFX9_DPP-NEXT: s_or_b64 exec, exec, s[10:11]5094; GFX9_DPP-NEXT: .LBB8_4: ; %Flow35095; GFX9_DPP-NEXT: s_or_b64 exec, exec, s[8:9]5096; GFX9_DPP-NEXT: v_readfirstlane_b32 s4, v35097; GFX9_DPP-NEXT: v_mov_b32_e32 v0, v15098; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0)5099; GFX9_DPP-NEXT: s_mov_b32 s3, 0xf0005100; GFX9_DPP-NEXT: s_mov_b32 s2, -15101; GFX9_DPP-NEXT: v_sub_u32_e32 v0, s4, v05102; GFX9_DPP-NEXT: buffer_store_dword v0, off, s[0:3], 05103; GFX9_DPP-NEXT: s_endpgm5104;5105; GFX1064_DPP-LABEL: sub_i32_varying:5106; GFX1064_DPP: ; %bb.0: ; %entry5107; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -15108; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]5109; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 05110; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:15111; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:15112; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:15113; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:15114; GFX1064_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -15115; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf5116; GFX1064_DPP-NEXT: v_readlane_b32 s2, v1, 315117; GFX1064_DPP-NEXT: v_mov_b32_e32 v2, s25118; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf5119; GFX1064_DPP-NEXT: v_readlane_b32 s6, v1, 155120; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf5121; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1]5122; GFX1064_DPP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245123; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[4:5], -15124; GFX1064_DPP-NEXT: v_readlane_b32 s7, v1, 315125; GFX1064_DPP-NEXT: v_writelane_b32 v3, s6, 165126; GFX1064_DPP-NEXT: s_mov_b64 exec, s[4:5]5127; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 05128; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[4:5], -15129; GFX1064_DPP-NEXT: v_readlane_b32 s6, v1, 475130; GFX1064_DPP-NEXT: v_readlane_b32 s12, v1, 635131; GFX1064_DPP-NEXT: v_writelane_b32 v3, s7, 325132; GFX1064_DPP-NEXT: s_mov_b64 exec, s[4:5]5133; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v05134; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[4:5], -15135; GFX1064_DPP-NEXT: v_writelane_b32 v3, s6, 485136; GFX1064_DPP-NEXT: s_mov_b64 exec, s[4:5]5137; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v05138; GFX1064_DPP-NEXT: s_mov_b32 s6, -15139; GFX1064_DPP-NEXT: ; implicit-def: $vgpr45140; GFX1064_DPP-NEXT: s_and_saveexec_b64 s[8:9], vcc5141; GFX1064_DPP-NEXT: s_cbranch_execz .LBB8_45142; GFX1064_DPP-NEXT: ; %bb.1:5143; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)5144; GFX1064_DPP-NEXT: s_load_dword s4, s[2:3], 0x05145; GFX1064_DPP-NEXT: s_mov_b64 s[10:11], 05146; GFX1064_DPP-NEXT: s_mov_b32 s7, 0x310160005147; GFX1064_DPP-NEXT: s_mov_b32 s5, s35148; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)5149; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, s45150; GFX1064_DPP-NEXT: s_mov_b32 s4, s25151; GFX1064_DPP-NEXT: .LBB8_2: ; %atomicrmw.start5152; GFX1064_DPP-NEXT: ; =>This Inner Loop Header: Depth=15153; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v45154; GFX1064_DPP-NEXT: v_subrev_nc_u32_e32 v5, s12, v65155; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, v55156; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v65157; GFX1064_DPP-NEXT: buffer_atomic_cmpswap v[4:5], off, s[4:7], 0 glc5158; GFX1064_DPP-NEXT: s_waitcnt vmcnt(0)5159; GFX1064_DPP-NEXT: buffer_gl1_inv5160; GFX1064_DPP-NEXT: buffer_gl0_inv5161; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, v4, v65162; GFX1064_DPP-NEXT: s_or_b64 s[10:11], vcc, s[10:11]5163; GFX1064_DPP-NEXT: s_andn2_b64 exec, exec, s[10:11]5164; GFX1064_DPP-NEXT: s_cbranch_execnz .LBB8_25165; GFX1064_DPP-NEXT: ; %bb.3: ; %Flow5166; GFX1064_DPP-NEXT: s_or_b64 exec, exec, s[10:11]5167; GFX1064_DPP-NEXT: .LBB8_4: ; %Flow35168; GFX1064_DPP-NEXT: s_or_b64 exec, exec, s[8:9]5169; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)5170; GFX1064_DPP-NEXT: v_readfirstlane_b32 s2, v45171; GFX1064_DPP-NEXT: v_mov_b32_e32 v0, v35172; GFX1064_DPP-NEXT: s_mov_b32 s3, 0x310160005173; GFX1064_DPP-NEXT: v_sub_nc_u32_e32 v0, s2, v05174; GFX1064_DPP-NEXT: s_mov_b32 s2, -15175; GFX1064_DPP-NEXT: buffer_store_dword v0, off, s[0:3], 05176; GFX1064_DPP-NEXT: s_endpgm5177;5178; GFX1032_DPP-LABEL: sub_i32_varying:5179; GFX1032_DPP: ; %bb.0: ; %entry5180; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -15181; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s05182; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:15183; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:15184; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:15185; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:15186; GFX1032_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -15187; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s05188; GFX1032_DPP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245189; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -15190; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf5191; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 05192; GFX1032_DPP-NEXT: v_readlane_b32 s9, v1, 315193; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf5194; GFX1032_DPP-NEXT: v_readlane_b32 s5, v1, 155195; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s45196; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 05197; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -15198; GFX1032_DPP-NEXT: v_writelane_b32 v3, s5, 165199; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s45200; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v05201; GFX1032_DPP-NEXT: s_mov_b32 s6, -15202; GFX1032_DPP-NEXT: ; implicit-def: $vgpr45203; GFX1032_DPP-NEXT: s_and_saveexec_b32 s8, vcc_lo5204; GFX1032_DPP-NEXT: s_cbranch_execz .LBB8_45205; GFX1032_DPP-NEXT: ; %bb.1:5206; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)5207; GFX1032_DPP-NEXT: s_load_dword s4, s[2:3], 0x05208; GFX1032_DPP-NEXT: s_mov_b32 s10, 05209; GFX1032_DPP-NEXT: s_mov_b32 s7, 0x310160005210; GFX1032_DPP-NEXT: s_mov_b32 s5, s35211; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)5212; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, s45213; GFX1032_DPP-NEXT: s_mov_b32 s4, s25214; GFX1032_DPP-NEXT: .LBB8_2: ; %atomicrmw.start5215; GFX1032_DPP-NEXT: ; =>This Inner Loop Header: Depth=15216; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, v45217; GFX1032_DPP-NEXT: v_subrev_nc_u32_e32 v5, s9, v65218; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, v55219; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, v65220; GFX1032_DPP-NEXT: buffer_atomic_cmpswap v[4:5], off, s[4:7], 0 glc5221; GFX1032_DPP-NEXT: s_waitcnt vmcnt(0)5222; GFX1032_DPP-NEXT: buffer_gl1_inv5223; GFX1032_DPP-NEXT: buffer_gl0_inv5224; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v65225; GFX1032_DPP-NEXT: s_or_b32 s10, vcc_lo, s105226; GFX1032_DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s105227; GFX1032_DPP-NEXT: s_cbranch_execnz .LBB8_25228; GFX1032_DPP-NEXT: ; %bb.3: ; %Flow5229; GFX1032_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s105230; GFX1032_DPP-NEXT: .LBB8_4: ; %Flow35231; GFX1032_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s85232; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)5233; GFX1032_DPP-NEXT: v_readfirstlane_b32 s2, v45234; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, v35235; GFX1032_DPP-NEXT: s_mov_b32 s3, 0x310160005236; GFX1032_DPP-NEXT: v_sub_nc_u32_e32 v0, s2, v05237; GFX1032_DPP-NEXT: s_mov_b32 s2, -15238; GFX1032_DPP-NEXT: buffer_store_dword v0, off, s[0:3], 05239; GFX1032_DPP-NEXT: s_endpgm5240;5241; GFX1164_DPP-LABEL: sub_i32_varying:5242; GFX1164_DPP: ; %bb.0: ; %entry5243; GFX1164_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v05244; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -15245; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)5246; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]5247; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 05248; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)5249; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:15250; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:15251; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5252; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:15253; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:15254; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5255; GFX1164_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -15256; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf5257; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5258; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 315259; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, s25260; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5261; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf5262; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 155263; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf5264; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]5265; GFX1164_DPP-NEXT: s_load_b128 s[0:3], s[4:5], 0x245266; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[4:5], -15267; GFX1164_DPP-NEXT: v_readlane_b32 s7, v1, 315268; GFX1164_DPP-NEXT: v_writelane_b32 v3, s6, 165269; GFX1164_DPP-NEXT: s_mov_b64 exec, s[4:5]5270; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5271; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 05272; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[4:5], -15273; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 475274; GFX1164_DPP-NEXT: v_readlane_b32 s12, v1, 635275; GFX1164_DPP-NEXT: v_writelane_b32 v3, s7, 325276; GFX1164_DPP-NEXT: s_mov_b64 exec, s[4:5]5277; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5278; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v05279; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[4:5], -15280; GFX1164_DPP-NEXT: v_writelane_b32 v3, s6, 485281; GFX1164_DPP-NEXT: s_mov_b64 exec, s[4:5]5282; GFX1164_DPP-NEXT: s_mov_b32 s6, -15283; GFX1164_DPP-NEXT: s_mov_b64 s[8:9], exec5284; GFX1164_DPP-NEXT: ; implicit-def: $vgpr45285; GFX1164_DPP-NEXT: v_cmpx_eq_u32_e32 0, v05286; GFX1164_DPP-NEXT: s_cbranch_execz .LBB8_45287; GFX1164_DPP-NEXT: ; %bb.1:5288; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)5289; GFX1164_DPP-NEXT: s_load_b32 s4, s[2:3], 0x05290; GFX1164_DPP-NEXT: s_mov_b64 s[10:11], 05291; GFX1164_DPP-NEXT: s_mov_b32 s7, 0x310160005292; GFX1164_DPP-NEXT: s_mov_b32 s5, s35293; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)5294; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, s45295; GFX1164_DPP-NEXT: s_mov_b32 s4, s25296; GFX1164_DPP-NEXT: .LBB8_2: ; %atomicrmw.start5297; GFX1164_DPP-NEXT: ; =>This Inner Loop Header: Depth=15298; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5299; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v45300; GFX1164_DPP-NEXT: v_subrev_nc_u32_e32 v5, s12, v65301; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)5302; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, v55303; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v65304; GFX1164_DPP-NEXT: buffer_atomic_cmpswap_b32 v[4:5], off, s[4:7], 0 glc5305; GFX1164_DPP-NEXT: s_waitcnt vmcnt(0)5306; GFX1164_DPP-NEXT: buffer_gl1_inv5307; GFX1164_DPP-NEXT: buffer_gl0_inv5308; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, v4, v65309; GFX1164_DPP-NEXT: s_or_b64 s[10:11], vcc, s[10:11]5310; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5311; GFX1164_DPP-NEXT: s_and_not1_b64 exec, exec, s[10:11]5312; GFX1164_DPP-NEXT: s_cbranch_execnz .LBB8_25313; GFX1164_DPP-NEXT: ; %bb.3: ; %Flow5314; GFX1164_DPP-NEXT: s_or_b64 exec, exec, s[10:11]5315; GFX1164_DPP-NEXT: .LBB8_4: ; %Flow35316; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)5317; GFX1164_DPP-NEXT: s_or_b64 exec, exec, s[8:9]5318; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)5319; GFX1164_DPP-NEXT: v_readfirstlane_b32 s2, v45320; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, v35321; GFX1164_DPP-NEXT: s_mov_b32 s3, 0x310160005322; GFX1164_DPP-NEXT: v_sub_nc_u32_e32 v0, s2, v05323; GFX1164_DPP-NEXT: s_mov_b32 s2, -15324; GFX1164_DPP-NEXT: buffer_store_b32 v0, off, s[0:3], 05325; GFX1164_DPP-NEXT: s_endpgm5326;5327; GFX1132_DPP-LABEL: sub_i32_varying:5328; GFX1132_DPP: ; %bb.0: ; %entry5329; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v05330; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -15331; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)5332; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s05333; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5334; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:15335; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:15336; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5337; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:15338; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:15339; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)5340; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -15341; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s05342; GFX1132_DPP-NEXT: s_load_b128 s[0:3], s[4:5], 0x245343; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -15344; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf5345; GFX1132_DPP-NEXT: v_mov_b32_e32 v3, 05346; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)5347; GFX1132_DPP-NEXT: v_readlane_b32 s9, v1, 315348; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf5349; GFX1132_DPP-NEXT: v_readlane_b32 s5, v1, 155350; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s45351; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5352; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 05353; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -15354; GFX1132_DPP-NEXT: v_writelane_b32 v3, s5, 165355; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s45356; GFX1132_DPP-NEXT: s_mov_b32 s6, -15357; GFX1132_DPP-NEXT: s_mov_b32 s8, exec_lo5358; GFX1132_DPP-NEXT: ; implicit-def: $vgpr45359; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v05360; GFX1132_DPP-NEXT: s_cbranch_execz .LBB8_45361; GFX1132_DPP-NEXT: ; %bb.1:5362; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)5363; GFX1132_DPP-NEXT: s_load_b32 s4, s[2:3], 0x05364; GFX1132_DPP-NEXT: s_mov_b32 s10, 05365; GFX1132_DPP-NEXT: s_mov_b32 s7, 0x310160005366; GFX1132_DPP-NEXT: s_mov_b32 s5, s35367; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)5368; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, s45369; GFX1132_DPP-NEXT: s_mov_b32 s4, s25370; GFX1132_DPP-NEXT: .LBB8_2: ; %atomicrmw.start5371; GFX1132_DPP-NEXT: ; =>This Inner Loop Header: Depth=15372; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5373; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, v45374; GFX1132_DPP-NEXT: v_subrev_nc_u32_e32 v5, s9, v65375; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)5376; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, v55377; GFX1132_DPP-NEXT: v_mov_b32_e32 v5, v65378; GFX1132_DPP-NEXT: buffer_atomic_cmpswap_b32 v[4:5], off, s[4:7], 0 glc5379; GFX1132_DPP-NEXT: s_waitcnt vmcnt(0)5380; GFX1132_DPP-NEXT: buffer_gl1_inv5381; GFX1132_DPP-NEXT: buffer_gl0_inv5382; GFX1132_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v65383; GFX1132_DPP-NEXT: s_or_b32 s10, vcc_lo, s105384; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5385; GFX1132_DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s105386; GFX1132_DPP-NEXT: s_cbranch_execnz .LBB8_25387; GFX1132_DPP-NEXT: ; %bb.3: ; %Flow5388; GFX1132_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s105389; GFX1132_DPP-NEXT: .LBB8_4: ; %Flow35390; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)5391; GFX1132_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s85392; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)5393; GFX1132_DPP-NEXT: v_readfirstlane_b32 s2, v45394; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, v35395; GFX1132_DPP-NEXT: s_mov_b32 s3, 0x310160005396; GFX1132_DPP-NEXT: v_sub_nc_u32_e32 v0, s2, v05397; GFX1132_DPP-NEXT: s_mov_b32 s2, -15398; GFX1132_DPP-NEXT: buffer_store_b32 v0, off, s[0:3], 05399; GFX1132_DPP-NEXT: s_endpgm5400;5401; GFX1264_DPP-LABEL: sub_i32_varying:5402; GFX1264_DPP: ; %bb.0: ; %entry5403; GFX1264_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v05404; GFX1264_DPP-NEXT: s_or_saveexec_b64 s[0:1], -15405; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)5406; GFX1264_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]5407; GFX1264_DPP-NEXT: v_mov_b32_e32 v3, 05408; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)5409; GFX1264_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:15410; GFX1264_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:15411; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5412; GFX1264_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:15413; GFX1264_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:15414; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5415; GFX1264_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -15416; GFX1264_DPP-NEXT: v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf5417; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5418; GFX1264_DPP-NEXT: v_readlane_b32 s2, v1, 315419; GFX1264_DPP-NEXT: v_mov_b32_e32 v2, s25420; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5421; GFX1264_DPP-NEXT: v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf5422; GFX1264_DPP-NEXT: v_readlane_b32 s6, v1, 155423; GFX1264_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf5424; GFX1264_DPP-NEXT: s_mov_b64 exec, s[0:1]5425; GFX1264_DPP-NEXT: s_load_b128 s[0:3], s[4:5], 0x245426; GFX1264_DPP-NEXT: s_or_saveexec_b64 s[4:5], -15427; GFX1264_DPP-NEXT: v_readlane_b32 s7, v1, 315428; GFX1264_DPP-NEXT: v_writelane_b32 v3, s6, 165429; GFX1264_DPP-NEXT: s_mov_b64 exec, s[4:5]5430; GFX1264_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5431; GFX1264_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 05432; GFX1264_DPP-NEXT: s_or_saveexec_b64 s[4:5], -15433; GFX1264_DPP-NEXT: v_readlane_b32 s8, v1, 475434; GFX1264_DPP-NEXT: v_readlane_b32 s9, v1, 635435; GFX1264_DPP-NEXT: v_writelane_b32 v3, s7, 325436; GFX1264_DPP-NEXT: s_mov_b64 exec, s[4:5]5437; GFX1264_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5438; GFX1264_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v05439; GFX1264_DPP-NEXT: s_or_saveexec_b64 s[6:7], -15440; GFX1264_DPP-NEXT: s_mov_b32 s4, s95441; GFX1264_DPP-NEXT: v_writelane_b32 v3, s8, 485442; GFX1264_DPP-NEXT: s_wait_alu depctr_sa_sdst(0)5443; GFX1264_DPP-NEXT: s_mov_b64 exec, s[6:7]5444; GFX1264_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v05445; GFX1264_DPP-NEXT: s_mov_b32 s6, -15446; GFX1264_DPP-NEXT: ; implicit-def: $vgpr05447; GFX1264_DPP-NEXT: s_and_saveexec_b64 s[8:9], vcc5448; GFX1264_DPP-NEXT: s_cbranch_execz .LBB8_25449; GFX1264_DPP-NEXT: ; %bb.1:5450; GFX1264_DPP-NEXT: v_mov_b32_e32 v0, s45451; GFX1264_DPP-NEXT: s_mov_b32 s7, 0x310160005452; GFX1264_DPP-NEXT: s_wait_kmcnt 0x05453; GFX1264_DPP-NEXT: s_mov_b32 s4, s25454; GFX1264_DPP-NEXT: s_mov_b32 s5, s35455; GFX1264_DPP-NEXT: buffer_atomic_sub_u32 v0, off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV5456; GFX1264_DPP-NEXT: s_wait_loadcnt 0x05457; GFX1264_DPP-NEXT: global_inv scope:SCOPE_DEV5458; GFX1264_DPP-NEXT: .LBB8_2:5459; GFX1264_DPP-NEXT: s_wait_alu depctr_sa_sdst(0)5460; GFX1264_DPP-NEXT: s_or_b64 exec, exec, s[8:9]5461; GFX1264_DPP-NEXT: s_wait_kmcnt 0x05462; GFX1264_DPP-NEXT: v_readfirstlane_b32 s2, v05463; GFX1264_DPP-NEXT: v_mov_b32_e32 v0, v35464; GFX1264_DPP-NEXT: s_mov_b32 s3, 0x310160005465; GFX1264_DPP-NEXT: s_wait_alu depctr_va_sdst(0)5466; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)5467; GFX1264_DPP-NEXT: v_sub_nc_u32_e32 v0, s2, v05468; GFX1264_DPP-NEXT: s_mov_b32 s2, s65469; GFX1264_DPP-NEXT: buffer_store_b32 v0, off, s[0:3], null5470; GFX1264_DPP-NEXT: s_endpgm5471;5472; GFX1232_DPP-LABEL: sub_i32_varying:5473; GFX1232_DPP: ; %bb.0: ; %entry5474; GFX1232_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v05475; GFX1232_DPP-NEXT: s_or_saveexec_b32 s0, -15476; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)5477; GFX1232_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s05478; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5479; GFX1232_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:15480; GFX1232_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:15481; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5482; GFX1232_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:15483; GFX1232_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:15484; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)5485; GFX1232_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -15486; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s05487; GFX1232_DPP-NEXT: s_load_b128 s[0:3], s[4:5], 0x245488; GFX1232_DPP-NEXT: s_or_saveexec_b32 s4, -15489; GFX1232_DPP-NEXT: v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf5490; GFX1232_DPP-NEXT: v_mov_b32_e32 v3, 05491; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)5492; GFX1232_DPP-NEXT: v_readlane_b32 s6, v1, 315493; GFX1232_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf5494; GFX1232_DPP-NEXT: v_readlane_b32 s5, v1, 155495; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s45496; GFX1232_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)5497; GFX1232_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 05498; GFX1232_DPP-NEXT: s_or_saveexec_b32 s4, -15499; GFX1232_DPP-NEXT: v_writelane_b32 v3, s5, 165500; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s45501; GFX1232_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v05502; GFX1232_DPP-NEXT: s_mov_b32 s4, s65503; GFX1232_DPP-NEXT: s_mov_b32 s6, -15504; GFX1232_DPP-NEXT: ; implicit-def: $vgpr05505; GFX1232_DPP-NEXT: s_and_saveexec_b32 s8, vcc_lo5506; GFX1232_DPP-NEXT: s_cbranch_execz .LBB8_25507; GFX1232_DPP-NEXT: ; %bb.1:5508; GFX1232_DPP-NEXT: s_wait_alu depctr_sa_sdst(0)5509; GFX1232_DPP-NEXT: v_mov_b32_e32 v0, s45510; GFX1232_DPP-NEXT: s_mov_b32 s7, 0x310160005511; GFX1232_DPP-NEXT: s_wait_kmcnt 0x05512; GFX1232_DPP-NEXT: s_mov_b32 s4, s25513; GFX1232_DPP-NEXT: s_mov_b32 s5, s35514; GFX1232_DPP-NEXT: buffer_atomic_sub_u32 v0, off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV5515; GFX1232_DPP-NEXT: s_wait_loadcnt 0x05516; GFX1232_DPP-NEXT: global_inv scope:SCOPE_DEV5517; GFX1232_DPP-NEXT: .LBB8_2:5518; GFX1232_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s85519; GFX1232_DPP-NEXT: s_wait_kmcnt 0x05520; GFX1232_DPP-NEXT: v_readfirstlane_b32 s2, v05521; GFX1232_DPP-NEXT: v_mov_b32_e32 v0, v35522; GFX1232_DPP-NEXT: s_mov_b32 s3, 0x310160005523; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)5524; GFX1232_DPP-NEXT: v_sub_nc_u32_e32 v0, s2, v05525; GFX1232_DPP-NEXT: s_mov_b32 s2, s65526; GFX1232_DPP-NEXT: buffer_store_b32 v0, off, s[0:3], null5527; GFX1232_DPP-NEXT: s_endpgm5528entry:5529 %lane = call i32 @llvm.amdgcn.workitem.id.x()5530 %old = atomicrmw sub ptr addrspace(1) %inout, i32 %lane syncscope("agent") acq_rel5531 store i32 %old, ptr addrspace(1) %out5532 ret void5533}5534 5535define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace(1) %inout) {5536; GFX7LESS-LABEL: sub_i64_constant:5537; GFX7LESS: ; %bb.0: ; %entry5538; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec5539; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x95540; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 05541; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, s7, v05542; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v45543; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr15544; GFX7LESS-NEXT: s_and_saveexec_b64 s[8:9], vcc5545; GFX7LESS-NEXT: s_cbranch_execz .LBB9_45546; GFX7LESS-NEXT: ; %bb.1:5547; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)5548; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x05549; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]5550; GFX7LESS-NEXT: s_mov_b64 s[10:11], 05551; GFX7LESS-NEXT: s_mov_b32 s7, 0xf0005552; GFX7LESS-NEXT: s_mul_i32 s12, s6, 55553; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)5554; GFX7LESS-NEXT: v_mov_b32_e32 v0, s45555; GFX7LESS-NEXT: v_mov_b32_e32 v1, s55556; GFX7LESS-NEXT: s_mov_b32 s6, -15557; GFX7LESS-NEXT: s_mov_b32 s4, s25558; GFX7LESS-NEXT: s_mov_b32 s5, s35559; GFX7LESS-NEXT: .LBB9_2: ; %atomicrmw.start5560; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=15561; GFX7LESS-NEXT: v_mov_b32_e32 v8, v15562; GFX7LESS-NEXT: v_mov_b32_e32 v7, v05563; GFX7LESS-NEXT: v_subrev_i32_e32 v5, vcc, s12, v75564; GFX7LESS-NEXT: v_subbrev_u32_e32 v6, vcc, 0, v8, vcc5565; GFX7LESS-NEXT: s_waitcnt expcnt(0)5566; GFX7LESS-NEXT: v_mov_b32_e32 v0, v55567; GFX7LESS-NEXT: v_mov_b32_e32 v1, v65568; GFX7LESS-NEXT: v_mov_b32_e32 v2, v75569; GFX7LESS-NEXT: v_mov_b32_e32 v3, v85570; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc5571; GFX7LESS-NEXT: s_waitcnt vmcnt(0)5572; GFX7LESS-NEXT: buffer_wbinvl15573; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]5574; GFX7LESS-NEXT: s_or_b64 s[10:11], vcc, s[10:11]5575; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[10:11]5576; GFX7LESS-NEXT: s_cbranch_execnz .LBB9_25577; GFX7LESS-NEXT: ; %bb.3: ; %Flow5578; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]5579; GFX7LESS-NEXT: .LBB9_4: ; %Flow35580; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]5581; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)5582; GFX7LESS-NEXT: s_mov_b32 s3, 0xf0005583; GFX7LESS-NEXT: s_mov_b32 s2, -15584; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v15585; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v05586; GFX7LESS-NEXT: s_waitcnt expcnt(0)5587; GFX7LESS-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v45588; GFX7LESS-NEXT: v_mul_u32_u24_e32 v0, 5, v45589; GFX7LESS-NEXT: v_mov_b32_e32 v2, s45590; GFX7LESS-NEXT: v_sub_i32_e32 v0, vcc, s5, v05591; GFX7LESS-NEXT: v_subb_u32_e32 v1, vcc, v2, v1, vcc5592; GFX7LESS-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 05593; GFX7LESS-NEXT: s_endpgm5594;5595; GFX8-LABEL: sub_i64_constant:5596; GFX8: ; %bb.0: ; %entry5597; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245598; GFX8-NEXT: s_mov_b64 s[6:7], exec5599; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 05600; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v05601; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v45602; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr15603; GFX8-NEXT: s_and_saveexec_b64 s[8:9], vcc5604; GFX8-NEXT: s_cbranch_execz .LBB9_45605; GFX8-NEXT: ; %bb.1:5606; GFX8-NEXT: s_waitcnt lgkmcnt(0)5607; GFX8-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x05608; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7]5609; GFX8-NEXT: s_mov_b64 s[10:11], 05610; GFX8-NEXT: s_mov_b32 s7, 0xf0005611; GFX8-NEXT: s_mul_i32 s12, s6, 55612; GFX8-NEXT: s_waitcnt lgkmcnt(0)5613; GFX8-NEXT: v_mov_b32_e32 v0, s45614; GFX8-NEXT: v_mov_b32_e32 v1, s55615; GFX8-NEXT: s_mov_b32 s6, -15616; GFX8-NEXT: s_mov_b32 s4, s25617; GFX8-NEXT: s_mov_b32 s5, s35618; GFX8-NEXT: .LBB9_2: ; %atomicrmw.start5619; GFX8-NEXT: ; =>This Inner Loop Header: Depth=15620; GFX8-NEXT: v_mov_b32_e32 v8, v15621; GFX8-NEXT: v_mov_b32_e32 v7, v05622; GFX8-NEXT: v_subrev_u32_e32 v5, vcc, s12, v75623; GFX8-NEXT: v_subbrev_u32_e32 v6, vcc, 0, v8, vcc5624; GFX8-NEXT: v_mov_b32_e32 v0, v55625; GFX8-NEXT: v_mov_b32_e32 v1, v65626; GFX8-NEXT: v_mov_b32_e32 v2, v75627; GFX8-NEXT: v_mov_b32_e32 v3, v85628; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc5629; GFX8-NEXT: s_waitcnt vmcnt(0)5630; GFX8-NEXT: buffer_wbinvl1_vol5631; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]5632; GFX8-NEXT: s_or_b64 s[10:11], vcc, s[10:11]5633; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]5634; GFX8-NEXT: s_cbranch_execnz .LBB9_25635; GFX8-NEXT: ; %bb.3: ; %Flow5636; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]5637; GFX8-NEXT: .LBB9_4: ; %Flow35638; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]5639; GFX8-NEXT: v_readfirstlane_b32 s4, v15640; GFX8-NEXT: v_readfirstlane_b32 s5, v05641; GFX8-NEXT: v_mul_u32_u24_e32 v0, 5, v45642; GFX8-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v45643; GFX8-NEXT: v_mov_b32_e32 v2, s45644; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s5, v05645; GFX8-NEXT: s_waitcnt lgkmcnt(0)5646; GFX8-NEXT: s_mov_b32 s3, 0xf0005647; GFX8-NEXT: s_mov_b32 s2, -15648; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v2, v1, vcc5649; GFX8-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 05650; GFX8-NEXT: s_endpgm5651;5652; GFX9-LABEL: sub_i64_constant:5653; GFX9: ; %bb.0: ; %entry5654; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245655; GFX9-NEXT: s_mov_b64 s[6:7], exec5656; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 05657; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v05658; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v45659; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr15660; GFX9-NEXT: s_and_saveexec_b64 s[8:9], vcc5661; GFX9-NEXT: s_cbranch_execz .LBB9_45662; GFX9-NEXT: ; %bb.1:5663; GFX9-NEXT: s_waitcnt lgkmcnt(0)5664; GFX9-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x05665; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[6:7]5666; GFX9-NEXT: s_mov_b64 s[10:11], 05667; GFX9-NEXT: s_mov_b32 s7, 0xf0005668; GFX9-NEXT: s_mul_i32 s12, s6, 55669; GFX9-NEXT: s_waitcnt lgkmcnt(0)5670; GFX9-NEXT: v_mov_b32_e32 v0, s45671; GFX9-NEXT: v_mov_b32_e32 v1, s55672; GFX9-NEXT: s_mov_b32 s6, -15673; GFX9-NEXT: s_mov_b32 s4, s25674; GFX9-NEXT: s_mov_b32 s5, s35675; GFX9-NEXT: .LBB9_2: ; %atomicrmw.start5676; GFX9-NEXT: ; =>This Inner Loop Header: Depth=15677; GFX9-NEXT: v_mov_b32_e32 v8, v15678; GFX9-NEXT: v_mov_b32_e32 v7, v05679; GFX9-NEXT: v_subrev_co_u32_e32 v5, vcc, s12, v75680; GFX9-NEXT: v_subbrev_co_u32_e32 v6, vcc, 0, v8, vcc5681; GFX9-NEXT: v_mov_b32_e32 v0, v55682; GFX9-NEXT: v_mov_b32_e32 v1, v65683; GFX9-NEXT: v_mov_b32_e32 v2, v75684; GFX9-NEXT: v_mov_b32_e32 v3, v85685; GFX9-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc5686; GFX9-NEXT: s_waitcnt vmcnt(0)5687; GFX9-NEXT: buffer_wbinvl1_vol5688; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]5689; GFX9-NEXT: s_or_b64 s[10:11], vcc, s[10:11]5690; GFX9-NEXT: s_andn2_b64 exec, exec, s[10:11]5691; GFX9-NEXT: s_cbranch_execnz .LBB9_25692; GFX9-NEXT: ; %bb.3: ; %Flow5693; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]5694; GFX9-NEXT: .LBB9_4: ; %Flow35695; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]5696; GFX9-NEXT: v_readfirstlane_b32 s4, v15697; GFX9-NEXT: v_readfirstlane_b32 s5, v05698; GFX9-NEXT: v_mul_u32_u24_e32 v0, 5, v45699; GFX9-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v45700; GFX9-NEXT: v_mov_b32_e32 v2, s45701; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, s5, v05702; GFX9-NEXT: s_waitcnt lgkmcnt(0)5703; GFX9-NEXT: s_mov_b32 s3, 0xf0005704; GFX9-NEXT: s_mov_b32 s2, -15705; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v2, v1, vcc5706; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 05707; GFX9-NEXT: s_endpgm5708;5709; GFX1064-LABEL: sub_i64_constant:5710; GFX1064: ; %bb.0: ; %entry5711; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245712; GFX1064-NEXT: s_mov_b64 s[6:7], exec5713; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 05714; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v05715; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr15716; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v45717; GFX1064-NEXT: s_and_saveexec_b64 s[8:9], vcc5718; GFX1064-NEXT: s_cbranch_execz .LBB9_45719; GFX1064-NEXT: ; %bb.1:5720; GFX1064-NEXT: s_waitcnt lgkmcnt(0)5721; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x05722; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]5723; GFX1064-NEXT: s_mov_b64 s[10:11], 05724; GFX1064-NEXT: s_mul_i32 s12, s6, 55725; GFX1064-NEXT: s_mov_b32 s7, 0x310160005726; GFX1064-NEXT: s_mov_b32 s6, -15727; GFX1064-NEXT: s_waitcnt lgkmcnt(0)5728; GFX1064-NEXT: v_mov_b32_e32 v0, s45729; GFX1064-NEXT: v_mov_b32_e32 v1, s55730; GFX1064-NEXT: s_mov_b32 s4, s25731; GFX1064-NEXT: s_mov_b32 s5, s35732; GFX1064-NEXT: .LBB9_2: ; %atomicrmw.start5733; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=15734; GFX1064-NEXT: v_mov_b32_e32 v8, v15735; GFX1064-NEXT: v_mov_b32_e32 v7, v05736; GFX1064-NEXT: v_sub_co_u32 v5, vcc, v7, s125737; GFX1064-NEXT: v_subrev_co_ci_u32_e32 v6, vcc, 0, v8, vcc5738; GFX1064-NEXT: v_mov_b32_e32 v0, v55739; GFX1064-NEXT: v_mov_b32_e32 v2, v75740; GFX1064-NEXT: v_mov_b32_e32 v1, v65741; GFX1064-NEXT: v_mov_b32_e32 v3, v85742; GFX1064-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc5743; GFX1064-NEXT: s_waitcnt vmcnt(0)5744; GFX1064-NEXT: buffer_gl1_inv5745; GFX1064-NEXT: buffer_gl0_inv5746; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]5747; GFX1064-NEXT: s_or_b64 s[10:11], vcc, s[10:11]5748; GFX1064-NEXT: s_andn2_b64 exec, exec, s[10:11]5749; GFX1064-NEXT: s_cbranch_execnz .LBB9_25750; GFX1064-NEXT: ; %bb.3: ; %Flow5751; GFX1064-NEXT: s_or_b64 exec, exec, s[10:11]5752; GFX1064-NEXT: .LBB9_4: ; %Flow35753; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]5754; GFX1064-NEXT: s_waitcnt lgkmcnt(0)5755; GFX1064-NEXT: v_readfirstlane_b32 s2, v05756; GFX1064-NEXT: v_mul_u32_u24_e32 v0, 5, v45757; GFX1064-NEXT: v_readfirstlane_b32 s3, v15758; GFX1064-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v45759; GFX1064-NEXT: v_sub_co_u32 v0, vcc, s2, v05760; GFX1064-NEXT: v_sub_co_ci_u32_e32 v1, vcc, s3, v1, vcc5761; GFX1064-NEXT: s_mov_b32 s3, 0x310160005762; GFX1064-NEXT: s_mov_b32 s2, -15763; GFX1064-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 05764; GFX1064-NEXT: s_endpgm5765;5766; GFX1032-LABEL: sub_i64_constant:5767; GFX1032: ; %bb.0: ; %entry5768; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x245769; GFX1032-NEXT: s_mov_b32 s6, exec_lo5770; GFX1032-NEXT: s_mov_b32 s9, 05771; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 05772; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr15773; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v45774; GFX1032-NEXT: s_and_saveexec_b32 s8, vcc_lo5775; GFX1032-NEXT: s_cbranch_execz .LBB9_45776; GFX1032-NEXT: ; %bb.1:5777; GFX1032-NEXT: s_waitcnt lgkmcnt(0)5778; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x05779; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s65780; GFX1032-NEXT: s_mov_b32 s7, 0x310160005781; GFX1032-NEXT: s_mul_i32 s10, s6, 55782; GFX1032-NEXT: s_mov_b32 s6, -15783; GFX1032-NEXT: s_waitcnt lgkmcnt(0)5784; GFX1032-NEXT: v_mov_b32_e32 v0, s45785; GFX1032-NEXT: v_mov_b32_e32 v1, s55786; GFX1032-NEXT: s_mov_b32 s4, s25787; GFX1032-NEXT: s_mov_b32 s5, s35788; GFX1032-NEXT: .LBB9_2: ; %atomicrmw.start5789; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=15790; GFX1032-NEXT: v_mov_b32_e32 v8, v15791; GFX1032-NEXT: v_mov_b32_e32 v7, v05792; GFX1032-NEXT: v_sub_co_u32 v5, vcc_lo, v7, s105793; GFX1032-NEXT: v_subrev_co_ci_u32_e32 v6, vcc_lo, 0, v8, vcc_lo5794; GFX1032-NEXT: v_mov_b32_e32 v0, v55795; GFX1032-NEXT: v_mov_b32_e32 v2, v75796; GFX1032-NEXT: v_mov_b32_e32 v1, v65797; GFX1032-NEXT: v_mov_b32_e32 v3, v85798; GFX1032-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc5799; GFX1032-NEXT: s_waitcnt vmcnt(0)5800; GFX1032-NEXT: buffer_gl1_inv5801; GFX1032-NEXT: buffer_gl0_inv5802; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[7:8]5803; GFX1032-NEXT: s_or_b32 s9, vcc_lo, s95804; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s95805; GFX1032-NEXT: s_cbranch_execnz .LBB9_25806; GFX1032-NEXT: ; %bb.3: ; %Flow5807; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s95808; GFX1032-NEXT: .LBB9_4: ; %Flow35809; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s85810; GFX1032-NEXT: s_waitcnt lgkmcnt(0)5811; GFX1032-NEXT: v_readfirstlane_b32 s2, v05812; GFX1032-NEXT: v_mul_u32_u24_e32 v0, 5, v45813; GFX1032-NEXT: v_readfirstlane_b32 s3, v15814; GFX1032-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v45815; GFX1032-NEXT: v_sub_co_u32 v0, vcc_lo, s2, v05816; GFX1032-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, s3, v1, vcc_lo5817; GFX1032-NEXT: s_mov_b32 s3, 0x310160005818; GFX1032-NEXT: s_mov_b32 s2, -15819; GFX1032-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 05820; GFX1032-NEXT: s_endpgm5821;5822; GFX1164-LABEL: sub_i64_constant:5823; GFX1164: ; %bb.0: ; %entry5824; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x245825; GFX1164-NEXT: s_mov_b64 s[6:7], exec5826; GFX1164-NEXT: s_mov_b64 s[8:9], exec5827; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 05828; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5829; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v05830; GFX1164-NEXT: ; implicit-def: $vgpr0_vgpr15831; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v45832; GFX1164-NEXT: s_cbranch_execz .LBB9_45833; GFX1164-NEXT: ; %bb.1:5834; GFX1164-NEXT: s_waitcnt lgkmcnt(0)5835; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x05836; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]5837; GFX1164-NEXT: s_mov_b64 s[10:11], 05838; GFX1164-NEXT: s_mul_i32 s12, s6, 55839; GFX1164-NEXT: s_mov_b32 s7, 0x310160005840; GFX1164-NEXT: s_mov_b32 s6, -15841; GFX1164-NEXT: s_waitcnt lgkmcnt(0)5842; GFX1164-NEXT: v_mov_b32_e32 v0, s45843; GFX1164-NEXT: v_mov_b32_e32 v1, s55844; GFX1164-NEXT: s_mov_b32 s4, s25845; GFX1164-NEXT: s_mov_b32 s5, s35846; GFX1164-NEXT: .LBB9_2: ; %atomicrmw.start5847; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=15848; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)5849; GFX1164-NEXT: v_mov_b32_e32 v8, v15850; GFX1164-NEXT: v_mov_b32_e32 v7, v05851; GFX1164-NEXT: v_sub_co_u32 v5, vcc, v7, s125852; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)5853; GFX1164-NEXT: v_subrev_co_ci_u32_e64 v6, null, 0, v8, vcc5854; GFX1164-NEXT: v_mov_b32_e32 v0, v55855; GFX1164-NEXT: v_mov_b32_e32 v2, v75856; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_3)5857; GFX1164-NEXT: v_mov_b32_e32 v1, v65858; GFX1164-NEXT: v_mov_b32_e32 v3, v85859; GFX1164-NEXT: buffer_atomic_cmpswap_b64 v[0:3], off, s[4:7], 0 glc5860; GFX1164-NEXT: s_waitcnt vmcnt(0)5861; GFX1164-NEXT: buffer_gl1_inv5862; GFX1164-NEXT: buffer_gl0_inv5863; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]5864; GFX1164-NEXT: s_waitcnt_depctr depctr_va_vcc(0)5865; GFX1164-NEXT: s_or_b64 s[10:11], vcc, s[10:11]5866; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5867; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[10:11]5868; GFX1164-NEXT: s_cbranch_execnz .LBB9_25869; GFX1164-NEXT: ; %bb.3: ; %Flow5870; GFX1164-NEXT: s_or_b64 exec, exec, s[10:11]5871; GFX1164-NEXT: .LBB9_4: ; %Flow35872; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5873; GFX1164-NEXT: s_or_b64 exec, exec, s[8:9]5874; GFX1164-NEXT: s_waitcnt lgkmcnt(0)5875; GFX1164-NEXT: v_readfirstlane_b32 s2, v05876; GFX1164-NEXT: v_mul_u32_u24_e32 v0, 5, v45877; GFX1164-NEXT: v_readfirstlane_b32 s3, v15878; GFX1164-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v45879; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)5880; GFX1164-NEXT: v_sub_co_u32 v0, vcc, s2, v05881; GFX1164-NEXT: v_sub_co_ci_u32_e64 v1, null, s3, v1, vcc5882; GFX1164-NEXT: s_mov_b32 s3, 0x310160005883; GFX1164-NEXT: s_mov_b32 s2, -15884; GFX1164-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 05885; GFX1164-NEXT: s_endpgm5886;5887; GFX1132-LABEL: sub_i64_constant:5888; GFX1132: ; %bb.0: ; %entry5889; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x245890; GFX1132-NEXT: s_mov_b32 s6, exec_lo5891; GFX1132-NEXT: s_mov_b32 s9, 05892; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 05893; GFX1132-NEXT: s_mov_b32 s8, exec_lo5894; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr15895; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)5896; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v45897; GFX1132-NEXT: s_cbranch_execz .LBB9_45898; GFX1132-NEXT: ; %bb.1:5899; GFX1132-NEXT: s_waitcnt lgkmcnt(0)5900; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x05901; GFX1132-NEXT: s_bcnt1_i32_b32 s6, s65902; GFX1132-NEXT: s_mov_b32 s7, 0x310160005903; GFX1132-NEXT: s_mul_i32 s10, s6, 55904; GFX1132-NEXT: s_mov_b32 s6, -15905; GFX1132-NEXT: s_waitcnt lgkmcnt(0)5906; GFX1132-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s55907; GFX1132-NEXT: s_mov_b32 s4, s25908; GFX1132-NEXT: s_mov_b32 s5, s35909; GFX1132-NEXT: .LBB9_2: ; %atomicrmw.start5910; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=15911; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5912; GFX1132-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v05913; GFX1132-NEXT: v_sub_co_u32 v5, vcc_lo, v7, s105914; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)5915; GFX1132-NEXT: v_subrev_co_ci_u32_e64 v6, null, 0, v8, vcc_lo5916; GFX1132-NEXT: v_mov_b32_e32 v0, v55917; GFX1132-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v85918; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_3)5919; GFX1132-NEXT: v_mov_b32_e32 v1, v65920; GFX1132-NEXT: buffer_atomic_cmpswap_b64 v[0:3], off, s[4:7], 0 glc5921; GFX1132-NEXT: s_waitcnt vmcnt(0)5922; GFX1132-NEXT: buffer_gl1_inv5923; GFX1132-NEXT: buffer_gl0_inv5924; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[7:8]5925; GFX1132-NEXT: s_or_b32 s9, vcc_lo, s95926; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5927; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s95928; GFX1132-NEXT: s_cbranch_execnz .LBB9_25929; GFX1132-NEXT: ; %bb.3: ; %Flow5930; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s95931; GFX1132-NEXT: .LBB9_4: ; %Flow35932; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5933; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s85934; GFX1132-NEXT: s_waitcnt lgkmcnt(0)5935; GFX1132-NEXT: v_readfirstlane_b32 s2, v05936; GFX1132-NEXT: v_mul_u32_u24_e32 v0, 5, v45937; GFX1132-NEXT: v_readfirstlane_b32 s3, v15938; GFX1132-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v45939; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)5940; GFX1132-NEXT: v_sub_co_u32 v0, vcc_lo, s2, v05941; GFX1132-NEXT: v_sub_co_ci_u32_e64 v1, null, s3, v1, vcc_lo5942; GFX1132-NEXT: s_mov_b32 s3, 0x310160005943; GFX1132-NEXT: s_mov_b32 s2, -15944; GFX1132-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 05945; GFX1132-NEXT: s_endpgm5946;5947; GFX1264-LABEL: sub_i64_constant:5948; GFX1264: ; %bb.0: ; %entry5949; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x245950; GFX1264-NEXT: s_mov_b64 s[6:7], exec5951; GFX1264-NEXT: s_mov_b64 s[4:5], exec5952; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 05953; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5954; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v05955; GFX1264-NEXT: ; implicit-def: $vgpr0_vgpr15956; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v25957; GFX1264-NEXT: s_cbranch_execz .LBB9_25958; GFX1264-NEXT: ; %bb.1:5959; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[6:7]5960; GFX1264-NEXT: v_mov_b32_e32 v1, 05961; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)5962; GFX1264-NEXT: s_mul_i32 s6, s6, 55963; GFX1264-NEXT: s_mov_b32 s11, 0x310160005964; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)5965; GFX1264-NEXT: v_mov_b32_e32 v0, s65966; GFX1264-NEXT: s_mov_b32 s10, -15967; GFX1264-NEXT: s_wait_kmcnt 0x05968; GFX1264-NEXT: s_mov_b32 s8, s25969; GFX1264-NEXT: s_mov_b32 s9, s35970; GFX1264-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV5971; GFX1264-NEXT: s_wait_loadcnt 0x05972; GFX1264-NEXT: global_inv scope:SCOPE_DEV5973; GFX1264-NEXT: .LBB9_2:5974; GFX1264-NEXT: s_or_b64 exec, exec, s[4:5]5975; GFX1264-NEXT: s_wait_kmcnt 0x05976; GFX1264-NEXT: v_readfirstlane_b32 s2, v05977; GFX1264-NEXT: v_mul_u32_u24_e32 v0, 5, v25978; GFX1264-NEXT: v_readfirstlane_b32 s3, v15979; GFX1264-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v25980; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)5981; GFX1264-NEXT: v_sub_co_u32 v0, vcc, s2, v05982; GFX1264-NEXT: v_sub_co_ci_u32_e64 v1, null, s3, v1, vcc5983; GFX1264-NEXT: s_mov_b32 s3, 0x310160005984; GFX1264-NEXT: s_mov_b32 s2, -15985; GFX1264-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null5986; GFX1264-NEXT: s_endpgm5987;5988; GFX1232-LABEL: sub_i64_constant:5989; GFX1232: ; %bb.0: ; %entry5990; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x245991; GFX1232-NEXT: s_mov_b32 s6, exec_lo5992; GFX1232-NEXT: s_mov_b32 s4, exec_lo5993; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 05994; GFX1232-NEXT: ; implicit-def: $vgpr0_vgpr15995; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)5996; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v25997; GFX1232-NEXT: s_cbranch_execz .LBB9_25998; GFX1232-NEXT: ; %bb.1:5999; GFX1232-NEXT: s_bcnt1_i32_b32 s5, s66000; GFX1232-NEXT: s_mov_b32 s11, 0x310160006001; GFX1232-NEXT: s_mul_i32 s5, s5, 56002; GFX1232-NEXT: s_mov_b32 s10, -16003; GFX1232-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, 06004; GFX1232-NEXT: s_wait_kmcnt 0x06005; GFX1232-NEXT: s_mov_b32 s8, s26006; GFX1232-NEXT: s_mov_b32 s9, s36007; GFX1232-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV6008; GFX1232-NEXT: s_wait_loadcnt 0x06009; GFX1232-NEXT: global_inv scope:SCOPE_DEV6010; GFX1232-NEXT: .LBB9_2:6011; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s46012; GFX1232-NEXT: s_wait_kmcnt 0x06013; GFX1232-NEXT: v_readfirstlane_b32 s2, v06014; GFX1232-NEXT: v_mul_u32_u24_e32 v0, 5, v26015; GFX1232-NEXT: v_readfirstlane_b32 s3, v16016; GFX1232-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v26017; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)6018; GFX1232-NEXT: v_sub_co_u32 v0, vcc_lo, s2, v06019; GFX1232-NEXT: v_sub_co_ci_u32_e64 v1, null, s3, v1, vcc_lo6020; GFX1232-NEXT: s_mov_b32 s3, 0x310160006021; GFX1232-NEXT: s_mov_b32 s2, -16022; GFX1232-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null6023; GFX1232-NEXT: s_endpgm6024entry:6025 %old = atomicrmw sub ptr addrspace(1) %inout, i64 5 syncscope("agent") acq_rel6026 store i64 %old, ptr addrspace(1) %out6027 ret void6028}6029 6030define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(1) %inout, i64 %subitive) {6031; GFX7LESS-LABEL: sub_i64_uniform:6032; GFX7LESS: ; %bb.0: ; %entry6033; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec6034; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96035; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd6036; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 06037; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, s7, v06038; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v46039; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr16040; GFX7LESS-NEXT: s_and_saveexec_b64 s[10:11], vcc6041; GFX7LESS-NEXT: s_cbranch_execz .LBB10_46042; GFX7LESS-NEXT: ; %bb.1:6043; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]6044; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)6045; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x06046; GFX7LESS-NEXT: s_mov_b64 s[12:13], 06047; GFX7LESS-NEXT: s_mov_b32 s7, 0xf0006048; GFX7LESS-NEXT: s_mul_i32 s14, s9, s66049; GFX7LESS-NEXT: v_mov_b32_e32 v0, s66050; GFX7LESS-NEXT: v_mul_hi_u32 v0, s8, v06051; GFX7LESS-NEXT: s_mul_i32 s6, s8, s66052; GFX7LESS-NEXT: v_add_i32_e32 v5, vcc, s14, v06053; GFX7LESS-NEXT: v_mov_b32_e32 v6, s66054; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)6055; GFX7LESS-NEXT: v_mov_b32_e32 v0, s46056; GFX7LESS-NEXT: v_mov_b32_e32 v1, s56057; GFX7LESS-NEXT: s_mov_b32 s6, -16058; GFX7LESS-NEXT: s_mov_b32 s4, s26059; GFX7LESS-NEXT: s_mov_b32 s5, s36060; GFX7LESS-NEXT: .LBB10_2: ; %atomicrmw.start6061; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=16062; GFX7LESS-NEXT: v_mov_b32_e32 v10, v16063; GFX7LESS-NEXT: v_mov_b32_e32 v9, v06064; GFX7LESS-NEXT: v_sub_i32_e32 v7, vcc, v9, v66065; GFX7LESS-NEXT: v_subb_u32_e32 v8, vcc, v10, v5, vcc6066; GFX7LESS-NEXT: s_waitcnt expcnt(0)6067; GFX7LESS-NEXT: v_mov_b32_e32 v0, v76068; GFX7LESS-NEXT: v_mov_b32_e32 v1, v86069; GFX7LESS-NEXT: v_mov_b32_e32 v2, v96070; GFX7LESS-NEXT: v_mov_b32_e32 v3, v106071; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc6072; GFX7LESS-NEXT: s_waitcnt vmcnt(0)6073; GFX7LESS-NEXT: buffer_wbinvl16074; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]6075; GFX7LESS-NEXT: s_or_b64 s[12:13], vcc, s[12:13]6076; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[12:13]6077; GFX7LESS-NEXT: s_cbranch_execnz .LBB10_26078; GFX7LESS-NEXT: ; %bb.3: ; %Flow6079; GFX7LESS-NEXT: s_or_b64 exec, exec, s[12:13]6080; GFX7LESS-NEXT: .LBB10_4: ; %Flow46081; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]6082; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)6083; GFX7LESS-NEXT: s_mov_b32 s3, 0xf0006084; GFX7LESS-NEXT: s_mov_b32 s2, -16085; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v16086; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v06087; GFX7LESS-NEXT: s_waitcnt expcnt(0)6088; GFX7LESS-NEXT: v_mul_lo_u32 v0, s9, v46089; GFX7LESS-NEXT: v_mul_hi_u32 v1, s8, v46090; GFX7LESS-NEXT: v_mul_lo_u32 v2, s8, v46091; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, v1, v06092; GFX7LESS-NEXT: v_mov_b32_e32 v3, s46093; GFX7LESS-NEXT: v_sub_i32_e32 v0, vcc, s5, v26094; GFX7LESS-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc6095; GFX7LESS-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 06096; GFX7LESS-NEXT: s_endpgm6097;6098; GFX8-LABEL: sub_i64_uniform:6099; GFX8: ; %bb.0: ; %entry6100; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246101; GFX8-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x346102; GFX8-NEXT: s_mov_b64 s[6:7], exec6103; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 06104; GFX8-NEXT: v_mbcnt_hi_u32_b32 v6, s7, v06105; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v66106; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr16107; GFX8-NEXT: s_and_saveexec_b64 s[10:11], vcc6108; GFX8-NEXT: s_cbranch_execz .LBB10_46109; GFX8-NEXT: ; %bb.1:6110; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7]6111; GFX8-NEXT: v_mov_b32_e32 v0, s66112; GFX8-NEXT: s_waitcnt lgkmcnt(0)6113; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[4:5], s8, v0, 06114; GFX8-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x06115; GFX8-NEXT: s_mul_i32 s6, s9, s66116; GFX8-NEXT: v_add_u32_e32 v5, vcc, s6, v56117; GFX8-NEXT: s_mov_b64 s[12:13], 06118; GFX8-NEXT: s_waitcnt lgkmcnt(0)6119; GFX8-NEXT: v_mov_b32_e32 v0, s46120; GFX8-NEXT: v_mov_b32_e32 v1, s56121; GFX8-NEXT: s_mov_b32 s7, 0xf0006122; GFX8-NEXT: s_mov_b32 s6, -16123; GFX8-NEXT: s_mov_b32 s4, s26124; GFX8-NEXT: s_mov_b32 s5, s36125; GFX8-NEXT: .LBB10_2: ; %atomicrmw.start6126; GFX8-NEXT: ; =>This Inner Loop Header: Depth=16127; GFX8-NEXT: v_mov_b32_e32 v10, v16128; GFX8-NEXT: v_mov_b32_e32 v9, v06129; GFX8-NEXT: v_sub_u32_e32 v7, vcc, v9, v46130; GFX8-NEXT: v_subb_u32_e32 v8, vcc, v10, v5, vcc6131; GFX8-NEXT: v_mov_b32_e32 v0, v76132; GFX8-NEXT: v_mov_b32_e32 v1, v86133; GFX8-NEXT: v_mov_b32_e32 v2, v96134; GFX8-NEXT: v_mov_b32_e32 v3, v106135; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc6136; GFX8-NEXT: s_waitcnt vmcnt(0)6137; GFX8-NEXT: buffer_wbinvl1_vol6138; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]6139; GFX8-NEXT: s_or_b64 s[12:13], vcc, s[12:13]6140; GFX8-NEXT: s_andn2_b64 exec, exec, s[12:13]6141; GFX8-NEXT: s_cbranch_execnz .LBB10_26142; GFX8-NEXT: ; %bb.3: ; %Flow6143; GFX8-NEXT: s_or_b64 exec, exec, s[12:13]6144; GFX8-NEXT: .LBB10_4: ; %Flow46145; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]6146; GFX8-NEXT: s_waitcnt lgkmcnt(0)6147; GFX8-NEXT: v_mul_lo_u32 v4, s9, v66148; GFX8-NEXT: v_mad_u64_u32 v[2:3], s[4:5], s8, v6, 06149; GFX8-NEXT: v_readfirstlane_b32 s4, v16150; GFX8-NEXT: v_readfirstlane_b32 s5, v06151; GFX8-NEXT: v_add_u32_e32 v1, vcc, v3, v46152; GFX8-NEXT: v_mov_b32_e32 v3, s46153; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s5, v26154; GFX8-NEXT: s_mov_b32 s3, 0xf0006155; GFX8-NEXT: s_mov_b32 s2, -16156; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc6157; GFX8-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 06158; GFX8-NEXT: s_endpgm6159;6160; GFX9-LABEL: sub_i64_uniform:6161; GFX9: ; %bb.0: ; %entry6162; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246163; GFX9-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x346164; GFX9-NEXT: s_mov_b64 s[6:7], exec6165; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 06166; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v06167; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v46168; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr16169; GFX9-NEXT: s_and_saveexec_b64 s[10:11], vcc6170; GFX9-NEXT: s_cbranch_execz .LBB10_46171; GFX9-NEXT: ; %bb.1:6172; GFX9-NEXT: s_waitcnt lgkmcnt(0)6173; GFX9-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x06174; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[6:7]6175; GFX9-NEXT: s_mul_i32 s7, s9, s66176; GFX9-NEXT: s_mul_hi_u32 s12, s8, s66177; GFX9-NEXT: s_add_i32 s7, s12, s76178; GFX9-NEXT: s_waitcnt lgkmcnt(0)6179; GFX9-NEXT: v_mov_b32_e32 v0, s46180; GFX9-NEXT: s_mul_i32 s14, s8, s66181; GFX9-NEXT: s_mov_b64 s[12:13], 06182; GFX9-NEXT: v_mov_b32_e32 v1, s56183; GFX9-NEXT: v_mov_b32_e32 v5, s76184; GFX9-NEXT: s_mov_b32 s7, 0xf0006185; GFX9-NEXT: s_mov_b32 s6, -16186; GFX9-NEXT: s_mov_b32 s4, s26187; GFX9-NEXT: s_mov_b32 s5, s36188; GFX9-NEXT: .LBB10_2: ; %atomicrmw.start6189; GFX9-NEXT: ; =>This Inner Loop Header: Depth=16190; GFX9-NEXT: v_mov_b32_e32 v9, v16191; GFX9-NEXT: v_mov_b32_e32 v8, v06192; GFX9-NEXT: v_subrev_co_u32_e32 v6, vcc, s14, v86193; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, v9, v5, vcc6194; GFX9-NEXT: v_mov_b32_e32 v0, v66195; GFX9-NEXT: v_mov_b32_e32 v1, v76196; GFX9-NEXT: v_mov_b32_e32 v2, v86197; GFX9-NEXT: v_mov_b32_e32 v3, v96198; GFX9-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc6199; GFX9-NEXT: s_waitcnt vmcnt(0)6200; GFX9-NEXT: buffer_wbinvl1_vol6201; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]6202; GFX9-NEXT: s_or_b64 s[12:13], vcc, s[12:13]6203; GFX9-NEXT: s_andn2_b64 exec, exec, s[12:13]6204; GFX9-NEXT: s_cbranch_execnz .LBB10_26205; GFX9-NEXT: ; %bb.3: ; %Flow6206; GFX9-NEXT: s_or_b64 exec, exec, s[12:13]6207; GFX9-NEXT: .LBB10_4: ; %Flow46208; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]6209; GFX9-NEXT: s_waitcnt lgkmcnt(0)6210; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], s8, v4, 06211; GFX9-NEXT: v_readfirstlane_b32 s6, v16212; GFX9-NEXT: v_readfirstlane_b32 s7, v06213; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s9, v4, v[3:4]6214; GFX9-NEXT: v_mov_b32_e32 v3, s66215; GFX9-NEXT: v_sub_co_u32_e32 v1, vcc, s7, v26216; GFX9-NEXT: s_mov_b32 s3, 0xf0006217; GFX9-NEXT: s_mov_b32 s2, -16218; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v3, v0, vcc6219; GFX9-NEXT: buffer_store_dwordx2 v[1:2], off, s[0:3], 06220; GFX9-NEXT: s_endpgm6221;6222; GFX1064-LABEL: sub_i64_uniform:6223; GFX1064: ; %bb.0: ; %entry6224; GFX1064-NEXT: s_clause 0x16225; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246226; GFX1064-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x346227; GFX1064-NEXT: s_mov_b64 s[6:7], exec6228; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 06229; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v06230; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr16231; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v46232; GFX1064-NEXT: s_and_saveexec_b64 s[10:11], vcc6233; GFX1064-NEXT: s_cbranch_execz .LBB10_46234; GFX1064-NEXT: ; %bb.1:6235; GFX1064-NEXT: s_waitcnt lgkmcnt(0)6236; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x06237; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]6238; GFX1064-NEXT: s_mov_b64 s[12:13], 06239; GFX1064-NEXT: s_mul_i32 s7, s9, s66240; GFX1064-NEXT: s_mul_hi_u32 s15, s8, s66241; GFX1064-NEXT: s_mul_i32 s14, s8, s66242; GFX1064-NEXT: s_add_i32 s15, s15, s76243; GFX1064-NEXT: s_mov_b32 s7, 0x310160006244; GFX1064-NEXT: s_mov_b32 s6, -16245; GFX1064-NEXT: s_waitcnt lgkmcnt(0)6246; GFX1064-NEXT: v_mov_b32_e32 v0, s46247; GFX1064-NEXT: v_mov_b32_e32 v1, s56248; GFX1064-NEXT: s_mov_b32 s4, s26249; GFX1064-NEXT: s_mov_b32 s5, s36250; GFX1064-NEXT: .LBB10_2: ; %atomicrmw.start6251; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=16252; GFX1064-NEXT: v_mov_b32_e32 v8, v16253; GFX1064-NEXT: v_mov_b32_e32 v7, v06254; GFX1064-NEXT: v_sub_co_u32 v5, vcc, v7, s146255; GFX1064-NEXT: v_subrev_co_ci_u32_e32 v6, vcc, s15, v8, vcc6256; GFX1064-NEXT: v_mov_b32_e32 v0, v56257; GFX1064-NEXT: v_mov_b32_e32 v2, v76258; GFX1064-NEXT: v_mov_b32_e32 v1, v66259; GFX1064-NEXT: v_mov_b32_e32 v3, v86260; GFX1064-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc6261; GFX1064-NEXT: s_waitcnt vmcnt(0)6262; GFX1064-NEXT: buffer_gl1_inv6263; GFX1064-NEXT: buffer_gl0_inv6264; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]6265; GFX1064-NEXT: s_or_b64 s[12:13], vcc, s[12:13]6266; GFX1064-NEXT: s_andn2_b64 exec, exec, s[12:13]6267; GFX1064-NEXT: s_cbranch_execnz .LBB10_26268; GFX1064-NEXT: ; %bb.3: ; %Flow6269; GFX1064-NEXT: s_or_b64 exec, exec, s[12:13]6270; GFX1064-NEXT: .LBB10_4: ; %Flow46271; GFX1064-NEXT: s_or_b64 exec, exec, s[10:11]6272; GFX1064-NEXT: s_waitcnt lgkmcnt(0)6273; GFX1064-NEXT: v_mad_u64_u32 v[2:3], s[2:3], s8, v4, 06274; GFX1064-NEXT: v_mad_u64_u32 v[3:4], s[2:3], s9, v4, v[3:4]6275; GFX1064-NEXT: v_readfirstlane_b32 s2, v06276; GFX1064-NEXT: v_readfirstlane_b32 s3, v16277; GFX1064-NEXT: v_sub_co_u32 v0, vcc, s2, v26278; GFX1064-NEXT: v_sub_co_ci_u32_e32 v1, vcc, s3, v3, vcc6279; GFX1064-NEXT: s_mov_b32 s3, 0x310160006280; GFX1064-NEXT: s_mov_b32 s2, -16281; GFX1064-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 06282; GFX1064-NEXT: s_endpgm6283;6284; GFX1032-LABEL: sub_i64_uniform:6285; GFX1032: ; %bb.0: ; %entry6286; GFX1032-NEXT: s_clause 0x16287; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246288; GFX1032-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x346289; GFX1032-NEXT: s_mov_b32 s6, exec_lo6290; GFX1032-NEXT: s_mov_b32 s11, 06291; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 06292; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr16293; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v46294; GFX1032-NEXT: s_and_saveexec_b32 s10, vcc_lo6295; GFX1032-NEXT: s_cbranch_execz .LBB10_46296; GFX1032-NEXT: ; %bb.1:6297; GFX1032-NEXT: s_waitcnt lgkmcnt(0)6298; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x06299; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s66300; GFX1032-NEXT: s_mul_i32 s7, s9, s66301; GFX1032-NEXT: s_mul_hi_u32 s13, s8, s66302; GFX1032-NEXT: s_mul_i32 s12, s8, s66303; GFX1032-NEXT: s_add_i32 s13, s13, s76304; GFX1032-NEXT: s_mov_b32 s7, 0x310160006305; GFX1032-NEXT: s_mov_b32 s6, -16306; GFX1032-NEXT: s_waitcnt lgkmcnt(0)6307; GFX1032-NEXT: v_mov_b32_e32 v0, s46308; GFX1032-NEXT: v_mov_b32_e32 v1, s56309; GFX1032-NEXT: s_mov_b32 s4, s26310; GFX1032-NEXT: s_mov_b32 s5, s36311; GFX1032-NEXT: .LBB10_2: ; %atomicrmw.start6312; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=16313; GFX1032-NEXT: v_mov_b32_e32 v8, v16314; GFX1032-NEXT: v_mov_b32_e32 v7, v06315; GFX1032-NEXT: v_sub_co_u32 v5, vcc_lo, v7, s126316; GFX1032-NEXT: v_subrev_co_ci_u32_e32 v6, vcc_lo, s13, v8, vcc_lo6317; GFX1032-NEXT: v_mov_b32_e32 v0, v56318; GFX1032-NEXT: v_mov_b32_e32 v2, v76319; GFX1032-NEXT: v_mov_b32_e32 v1, v66320; GFX1032-NEXT: v_mov_b32_e32 v3, v86321; GFX1032-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc6322; GFX1032-NEXT: s_waitcnt vmcnt(0)6323; GFX1032-NEXT: buffer_gl1_inv6324; GFX1032-NEXT: buffer_gl0_inv6325; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[7:8]6326; GFX1032-NEXT: s_or_b32 s11, vcc_lo, s116327; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s116328; GFX1032-NEXT: s_cbranch_execnz .LBB10_26329; GFX1032-NEXT: ; %bb.3: ; %Flow6330; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s116331; GFX1032-NEXT: .LBB10_4: ; %Flow46332; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s106333; GFX1032-NEXT: s_waitcnt lgkmcnt(0)6334; GFX1032-NEXT: v_mad_u64_u32 v[2:3], s2, s8, v4, 06335; GFX1032-NEXT: v_readfirstlane_b32 s3, v16336; GFX1032-NEXT: v_mad_u64_u32 v[3:4], s2, s9, v4, v[3:4]6337; GFX1032-NEXT: v_readfirstlane_b32 s2, v06338; GFX1032-NEXT: v_sub_co_u32 v0, vcc_lo, s2, v26339; GFX1032-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, s3, v3, vcc_lo6340; GFX1032-NEXT: s_mov_b32 s3, 0x310160006341; GFX1032-NEXT: s_mov_b32 s2, -16342; GFX1032-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 06343; GFX1032-NEXT: s_endpgm6344;6345; GFX1164-LABEL: sub_i64_uniform:6346; GFX1164: ; %bb.0: ; %entry6347; GFX1164-NEXT: s_clause 0x16348; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x246349; GFX1164-NEXT: s_load_b64 s[8:9], s[4:5], 0x346350; GFX1164-NEXT: s_mov_b64 s[6:7], exec6351; GFX1164-NEXT: s_mov_b64 s[10:11], exec6352; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 06353; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6354; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v06355; GFX1164-NEXT: ; implicit-def: $vgpr0_vgpr16356; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v46357; GFX1164-NEXT: s_cbranch_execz .LBB10_46358; GFX1164-NEXT: ; %bb.1:6359; GFX1164-NEXT: s_waitcnt lgkmcnt(0)6360; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x06361; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]6362; GFX1164-NEXT: s_mov_b64 s[12:13], 06363; GFX1164-NEXT: s_mul_i32 s7, s9, s66364; GFX1164-NEXT: s_mul_hi_u32 s15, s8, s66365; GFX1164-NEXT: s_mul_i32 s14, s8, s66366; GFX1164-NEXT: s_add_i32 s15, s15, s76367; GFX1164-NEXT: s_mov_b32 s7, 0x310160006368; GFX1164-NEXT: s_mov_b32 s6, -16369; GFX1164-NEXT: s_waitcnt lgkmcnt(0)6370; GFX1164-NEXT: v_mov_b32_e32 v0, s46371; GFX1164-NEXT: v_mov_b32_e32 v1, s56372; GFX1164-NEXT: s_mov_b32 s4, s26373; GFX1164-NEXT: s_mov_b32 s5, s36374; GFX1164-NEXT: .LBB10_2: ; %atomicrmw.start6375; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=16376; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)6377; GFX1164-NEXT: v_mov_b32_e32 v8, v16378; GFX1164-NEXT: v_mov_b32_e32 v7, v06379; GFX1164-NEXT: v_sub_co_u32 v5, vcc, v7, s146380; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)6381; GFX1164-NEXT: v_subrev_co_ci_u32_e64 v6, null, s15, v8, vcc6382; GFX1164-NEXT: v_mov_b32_e32 v0, v56383; GFX1164-NEXT: v_mov_b32_e32 v2, v76384; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_3)6385; GFX1164-NEXT: v_mov_b32_e32 v1, v66386; GFX1164-NEXT: v_mov_b32_e32 v3, v86387; GFX1164-NEXT: buffer_atomic_cmpswap_b64 v[0:3], off, s[4:7], 0 glc6388; GFX1164-NEXT: s_waitcnt vmcnt(0)6389; GFX1164-NEXT: buffer_gl1_inv6390; GFX1164-NEXT: buffer_gl0_inv6391; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]6392; GFX1164-NEXT: s_waitcnt_depctr depctr_va_vcc(0)6393; GFX1164-NEXT: s_or_b64 s[12:13], vcc, s[12:13]6394; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6395; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[12:13]6396; GFX1164-NEXT: s_cbranch_execnz .LBB10_26397; GFX1164-NEXT: ; %bb.3: ; %Flow6398; GFX1164-NEXT: s_or_b64 exec, exec, s[12:13]6399; GFX1164-NEXT: .LBB10_4: ; %Flow46400; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6401; GFX1164-NEXT: s_or_b64 exec, exec, s[10:11]6402; GFX1164-NEXT: s_waitcnt lgkmcnt(0)6403; GFX1164-NEXT: v_mad_u64_u32 v[2:3], null, s8, v4, 06404; GFX1164-NEXT: v_readfirstlane_b32 s2, v06405; GFX1164-NEXT: v_readfirstlane_b32 s3, v16406; GFX1164-NEXT: s_waitcnt_depctr depctr_va_vdst(0)6407; GFX1164-NEXT: v_mad_u64_u32 v[5:6], null, s9, v4, v[3:4]6408; GFX1164-NEXT: v_sub_co_u32 v0, vcc, s2, v26409; GFX1164-NEXT: s_mov_b32 s2, -16410; GFX1164-NEXT: v_sub_co_ci_u32_e64 v1, null, s3, v5, vcc6411; GFX1164-NEXT: s_mov_b32 s3, 0x310160006412; GFX1164-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 06413; GFX1164-NEXT: s_endpgm6414;6415; GFX1132-LABEL: sub_i64_uniform:6416; GFX1132: ; %bb.0: ; %entry6417; GFX1132-NEXT: s_clause 0x16418; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x246419; GFX1132-NEXT: s_load_b64 s[8:9], s[4:5], 0x346420; GFX1132-NEXT: s_mov_b32 s6, exec_lo6421; GFX1132-NEXT: s_mov_b32 s11, 06422; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 06423; GFX1132-NEXT: s_mov_b32 s10, exec_lo6424; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr16425; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)6426; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v46427; GFX1132-NEXT: s_cbranch_execz .LBB10_46428; GFX1132-NEXT: ; %bb.1:6429; GFX1132-NEXT: s_waitcnt lgkmcnt(0)6430; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x06431; GFX1132-NEXT: s_bcnt1_i32_b32 s6, s66432; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6433; GFX1132-NEXT: s_mul_i32 s7, s9, s66434; GFX1132-NEXT: s_mul_hi_u32 s13, s8, s66435; GFX1132-NEXT: s_mul_i32 s12, s8, s66436; GFX1132-NEXT: s_add_i32 s13, s13, s76437; GFX1132-NEXT: s_mov_b32 s7, 0x310160006438; GFX1132-NEXT: s_mov_b32 s6, -16439; GFX1132-NEXT: s_waitcnt lgkmcnt(0)6440; GFX1132-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s56441; GFX1132-NEXT: s_mov_b32 s4, s26442; GFX1132-NEXT: s_mov_b32 s5, s36443; GFX1132-NEXT: .LBB10_2: ; %atomicrmw.start6444; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=16445; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6446; GFX1132-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v06447; GFX1132-NEXT: v_sub_co_u32 v5, vcc_lo, v7, s126448; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)6449; GFX1132-NEXT: v_subrev_co_ci_u32_e64 v6, null, s13, v8, vcc_lo6450; GFX1132-NEXT: v_mov_b32_e32 v0, v56451; GFX1132-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v86452; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_3)6453; GFX1132-NEXT: v_mov_b32_e32 v1, v66454; GFX1132-NEXT: buffer_atomic_cmpswap_b64 v[0:3], off, s[4:7], 0 glc6455; GFX1132-NEXT: s_waitcnt vmcnt(0)6456; GFX1132-NEXT: buffer_gl1_inv6457; GFX1132-NEXT: buffer_gl0_inv6458; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[7:8]6459; GFX1132-NEXT: s_or_b32 s11, vcc_lo, s116460; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6461; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s116462; GFX1132-NEXT: s_cbranch_execnz .LBB10_26463; GFX1132-NEXT: ; %bb.3: ; %Flow6464; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s116465; GFX1132-NEXT: .LBB10_4: ; %Flow46466; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)6467; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s106468; GFX1132-NEXT: s_waitcnt lgkmcnt(0)6469; GFX1132-NEXT: v_mad_u64_u32 v[2:3], null, s8, v4, 06470; GFX1132-NEXT: v_readfirstlane_b32 s2, v06471; GFX1132-NEXT: v_readfirstlane_b32 s3, v16472; GFX1132-NEXT: v_mad_u64_u32 v[5:6], null, s9, v4, v[3:4]6473; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_3)6474; GFX1132-NEXT: v_sub_co_u32 v0, vcc_lo, s2, v26475; GFX1132-NEXT: s_mov_b32 s2, -16476; GFX1132-NEXT: v_sub_co_ci_u32_e64 v1, null, s3, v5, vcc_lo6477; GFX1132-NEXT: s_mov_b32 s3, 0x310160006478; GFX1132-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 06479; GFX1132-NEXT: s_endpgm6480;6481; GFX1264-LABEL: sub_i64_uniform:6482; GFX1264: ; %bb.0: ; %entry6483; GFX1264-NEXT: s_clause 0x16484; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x246485; GFX1264-NEXT: s_load_b64 s[4:5], s[4:5], 0x346486; GFX1264-NEXT: s_mov_b64 s[8:9], exec6487; GFX1264-NEXT: s_mov_b32 s11, 06488; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 06489; GFX1264-NEXT: s_mov_b64 s[6:7], exec6490; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6491; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v06492; GFX1264-NEXT: ; implicit-def: $vgpr0_vgpr16493; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v26494; GFX1264-NEXT: s_cbranch_execz .LBB10_26495; GFX1264-NEXT: ; %bb.1:6496; GFX1264-NEXT: s_bcnt1_i32_b64 s10, s[8:9]6497; GFX1264-NEXT: s_wait_kmcnt 0x06498; GFX1264-NEXT: s_mul_u64 s[8:9], s[4:5], s[10:11]6499; GFX1264-NEXT: s_mov_b32 s11, 0x310160006500; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)6501; GFX1264-NEXT: v_mov_b32_e32 v0, s86502; GFX1264-NEXT: v_mov_b32_e32 v1, s96503; GFX1264-NEXT: s_mov_b32 s10, -16504; GFX1264-NEXT: s_mov_b32 s8, s26505; GFX1264-NEXT: s_mov_b32 s9, s36506; GFX1264-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV6507; GFX1264-NEXT: s_wait_loadcnt 0x06508; GFX1264-NEXT: global_inv scope:SCOPE_DEV6509; GFX1264-NEXT: .LBB10_2:6510; GFX1264-NEXT: s_or_b64 exec, exec, s[6:7]6511; GFX1264-NEXT: s_wait_kmcnt 0x06512; GFX1264-NEXT: v_mad_co_u64_u32 v[3:4], null, s4, v2, 06513; GFX1264-NEXT: v_readfirstlane_b32 s2, v06514; GFX1264-NEXT: v_readfirstlane_b32 s3, v16515; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)6516; GFX1264-NEXT: v_mad_co_u64_u32 v[4:5], null, s5, v2, v[4:5]6517; GFX1264-NEXT: v_sub_co_u32 v0, vcc, s2, v36518; GFX1264-NEXT: s_mov_b32 s2, -16519; GFX1264-NEXT: v_sub_co_ci_u32_e64 v1, null, s3, v4, vcc6520; GFX1264-NEXT: s_mov_b32 s3, 0x310160006521; GFX1264-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null6522; GFX1264-NEXT: s_endpgm6523;6524; GFX1232-LABEL: sub_i64_uniform:6525; GFX1232: ; %bb.0: ; %entry6526; GFX1232-NEXT: s_clause 0x16527; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x246528; GFX1232-NEXT: s_load_b64 s[4:5], s[4:5], 0x346529; GFX1232-NEXT: s_mov_b32 s6, exec_lo6530; GFX1232-NEXT: s_mov_b32 s7, 06531; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 06532; GFX1232-NEXT: s_mov_b32 s8, exec_lo6533; GFX1232-NEXT: ; implicit-def: $vgpr0_vgpr16534; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)6535; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v26536; GFX1232-NEXT: s_cbranch_execz .LBB10_26537; GFX1232-NEXT: ; %bb.1:6538; GFX1232-NEXT: s_bcnt1_i32_b32 s6, s66539; GFX1232-NEXT: s_mov_b32 s15, 0x310160006540; GFX1232-NEXT: s_wait_kmcnt 0x06541; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)6542; GFX1232-NEXT: s_mul_u64 s[6:7], s[4:5], s[6:7]6543; GFX1232-NEXT: s_mov_b32 s14, -16544; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)6545; GFX1232-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s76546; GFX1232-NEXT: s_mov_b32 s12, s26547; GFX1232-NEXT: s_mov_b32 s13, s36548; GFX1232-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[12:15], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV6549; GFX1232-NEXT: s_wait_loadcnt 0x06550; GFX1232-NEXT: global_inv scope:SCOPE_DEV6551; GFX1232-NEXT: .LBB10_2:6552; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s86553; GFX1232-NEXT: s_wait_kmcnt 0x06554; GFX1232-NEXT: v_mad_co_u64_u32 v[3:4], null, s4, v2, 06555; GFX1232-NEXT: v_readfirstlane_b32 s2, v06556; GFX1232-NEXT: v_readfirstlane_b32 s3, v16557; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)6558; GFX1232-NEXT: v_mad_co_u64_u32 v[4:5], null, s5, v2, v[4:5]6559; GFX1232-NEXT: v_sub_co_u32 v0, vcc_lo, s2, v36560; GFX1232-NEXT: s_mov_b32 s2, -16561; GFX1232-NEXT: v_sub_co_ci_u32_e64 v1, null, s3, v4, vcc_lo6562; GFX1232-NEXT: s_mov_b32 s3, 0x310160006563; GFX1232-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null6564; GFX1232-NEXT: s_endpgm6565entry:6566 %old = atomicrmw sub ptr addrspace(1) %inout, i64 %subitive syncscope("agent") acq_rel6567 store i64 %old, ptr addrspace(1) %out6568 ret void6569}6570 6571define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(1) %inout) {6572; GFX7LESS_ITERATIVE-LABEL: sub_i64_varying:6573; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry6574; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec6575; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, 06576; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[8:9], 06577; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr4_vgpr56578; GFX7LESS_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop6579; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=16580; GFX7LESS_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]6581; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 m0, s26582; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s3, v1, s26583; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s26584; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v5, s9, m06585; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v4, s8, m06586; GFX7LESS_ITERATIVE-NEXT: s_add_u32 s8, s8, s66587; GFX7LESS_ITERATIVE-NEXT: s_addc_u32 s9, s9, s36588; GFX7LESS_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s26589; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]6590; GFX7LESS_ITERATIVE-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 06591; GFX7LESS_ITERATIVE-NEXT: s_and_b64 vcc, exec, s[2:3]6592; GFX7LESS_ITERATIVE-NEXT: s_cbranch_vccnz .LBB11_16593; GFX7LESS_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd6594; GFX7LESS_ITERATIVE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x96595; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 06596; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v06597; GFX7LESS_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v06598; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr16599; GFX7LESS_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc6600; GFX7LESS_ITERATIVE-NEXT: s_xor_b64 s[10:11], exec, s[4:5]6601; GFX7LESS_ITERATIVE-NEXT: s_cbranch_execz .LBB11_66602; GFX7LESS_ITERATIVE-NEXT: ; %bb.3:6603; GFX7LESS_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)6604; GFX7LESS_ITERATIVE-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x06605; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[12:13], 06606; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v6, s96607; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s7, 0xf0006608; GFX7LESS_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)6609; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, s46610; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, s56611; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s6, -16612; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s4, s26613; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s5, s36614; GFX7LESS_ITERATIVE-NEXT: .LBB11_4: ; %atomicrmw.start6615; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=16616; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v10, v16617; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v9, v06618; GFX7LESS_ITERATIVE-NEXT: v_subrev_i32_e32 v7, vcc, s8, v96619; GFX7LESS_ITERATIVE-NEXT: v_subb_u32_e32 v8, vcc, v10, v6, vcc6620; GFX7LESS_ITERATIVE-NEXT: s_waitcnt expcnt(0)6621; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, v76622; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, v86623; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v2, v96624; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v3, v106625; GFX7LESS_ITERATIVE-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc6626; GFX7LESS_ITERATIVE-NEXT: s_waitcnt vmcnt(0)6627; GFX7LESS_ITERATIVE-NEXT: buffer_wbinvl16628; GFX7LESS_ITERATIVE-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]6629; GFX7LESS_ITERATIVE-NEXT: s_or_b64 s[12:13], vcc, s[12:13]6630; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 exec, exec, s[12:13]6631; GFX7LESS_ITERATIVE-NEXT: s_cbranch_execnz .LBB11_46632; GFX7LESS_ITERATIVE-NEXT: ; %bb.5: ; %Flow6633; GFX7LESS_ITERATIVE-NEXT: s_or_b64 exec, exec, s[12:13]6634; GFX7LESS_ITERATIVE-NEXT: .LBB11_6: ; %Flow46635; GFX7LESS_ITERATIVE-NEXT: s_or_b64 exec, exec, s[10:11]6636; GFX7LESS_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)6637; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s3, 0xf0006638; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s2, -16639; GFX7LESS_ITERATIVE-NEXT: v_readfirstlane_b32 s4, v16640; GFX7LESS_ITERATIVE-NEXT: v_readfirstlane_b32 s5, v06641; GFX7LESS_ITERATIVE-NEXT: s_waitcnt expcnt(0)6642; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, s46643; GFX7LESS_ITERATIVE-NEXT: v_sub_i32_e32 v0, vcc, s5, v46644; GFX7LESS_ITERATIVE-NEXT: v_subb_u32_e32 v1, vcc, v1, v5, vcc6645; GFX7LESS_ITERATIVE-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 06646; GFX7LESS_ITERATIVE-NEXT: s_endpgm6647;6648; GFX8_ITERATIVE-LABEL: sub_i64_varying:6649; GFX8_ITERATIVE: ; %bb.0: ; %entry6650; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec6651; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, 06652; GFX8_ITERATIVE-NEXT: s_mov_b64 s[8:9], 06653; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr4_vgpr56654; GFX8_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop6655; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=16656; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]6657; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s26658; GFX8_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s26659; GFX8_ITERATIVE-NEXT: v_readlane_b32 s3, v1, s26660; GFX8_ITERATIVE-NEXT: v_writelane_b32 v4, s8, m06661; GFX8_ITERATIVE-NEXT: s_add_u32 s8, s8, s66662; GFX8_ITERATIVE-NEXT: v_writelane_b32 v5, s9, m06663; GFX8_ITERATIVE-NEXT: s_addc_u32 s9, s9, s36664; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s26665; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]6666; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_16667; GFX8_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd6668; GFX8_ITERATIVE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246669; GFX8_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 06670; GFX8_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v06671; GFX8_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v06672; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr16673; GFX8_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc6674; GFX8_ITERATIVE-NEXT: s_xor_b64 s[10:11], exec, s[4:5]6675; GFX8_ITERATIVE-NEXT: s_cbranch_execz .LBB11_66676; GFX8_ITERATIVE-NEXT: ; %bb.3:6677; GFX8_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)6678; GFX8_ITERATIVE-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x06679; GFX8_ITERATIVE-NEXT: s_mov_b64 s[12:13], 06680; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v6, s96681; GFX8_ITERATIVE-NEXT: s_mov_b32 s7, 0xf0006682; GFX8_ITERATIVE-NEXT: s_mov_b32 s6, -16683; GFX8_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)6684; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, s46685; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, s56686; GFX8_ITERATIVE-NEXT: s_mov_b32 s4, s26687; GFX8_ITERATIVE-NEXT: s_mov_b32 s5, s36688; GFX8_ITERATIVE-NEXT: .LBB11_4: ; %atomicrmw.start6689; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=16690; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v10, v16691; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v9, v06692; GFX8_ITERATIVE-NEXT: v_subrev_u32_e32 v7, vcc, s8, v96693; GFX8_ITERATIVE-NEXT: v_subb_u32_e32 v8, vcc, v10, v6, vcc6694; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, v76695; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, v86696; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v2, v96697; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v3, v106698; GFX8_ITERATIVE-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc6699; GFX8_ITERATIVE-NEXT: s_waitcnt vmcnt(0)6700; GFX8_ITERATIVE-NEXT: buffer_wbinvl1_vol6701; GFX8_ITERATIVE-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]6702; GFX8_ITERATIVE-NEXT: s_or_b64 s[12:13], vcc, s[12:13]6703; GFX8_ITERATIVE-NEXT: s_andn2_b64 exec, exec, s[12:13]6704; GFX8_ITERATIVE-NEXT: s_cbranch_execnz .LBB11_46705; GFX8_ITERATIVE-NEXT: ; %bb.5: ; %Flow6706; GFX8_ITERATIVE-NEXT: s_or_b64 exec, exec, s[12:13]6707; GFX8_ITERATIVE-NEXT: .LBB11_6: ; %Flow46708; GFX8_ITERATIVE-NEXT: s_or_b64 exec, exec, s[10:11]6709; GFX8_ITERATIVE-NEXT: v_readfirstlane_b32 s4, v16710; GFX8_ITERATIVE-NEXT: v_readfirstlane_b32 s5, v06711; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, s46712; GFX8_ITERATIVE-NEXT: v_sub_u32_e32 v0, vcc, s5, v46713; GFX8_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)6714; GFX8_ITERATIVE-NEXT: s_mov_b32 s3, 0xf0006715; GFX8_ITERATIVE-NEXT: s_mov_b32 s2, -16716; GFX8_ITERATIVE-NEXT: v_subb_u32_e32 v1, vcc, v1, v5, vcc6717; GFX8_ITERATIVE-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 06718; GFX8_ITERATIVE-NEXT: s_endpgm6719;6720; GFX9_ITERATIVE-LABEL: sub_i64_varying:6721; GFX9_ITERATIVE: ; %bb.0: ; %entry6722; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec6723; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, 06724; GFX9_ITERATIVE-NEXT: s_mov_b64 s[8:9], 06725; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr4_vgpr56726; GFX9_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop6727; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=16728; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]6729; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s26730; GFX9_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s26731; GFX9_ITERATIVE-NEXT: v_readlane_b32 s3, v1, s26732; GFX9_ITERATIVE-NEXT: v_writelane_b32 v4, s8, m06733; GFX9_ITERATIVE-NEXT: s_add_u32 s8, s8, s66734; GFX9_ITERATIVE-NEXT: v_writelane_b32 v5, s9, m06735; GFX9_ITERATIVE-NEXT: s_addc_u32 s9, s9, s36736; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s26737; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]6738; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_16739; GFX9_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd6740; GFX9_ITERATIVE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246741; GFX9_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 06742; GFX9_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v06743; GFX9_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v06744; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr16745; GFX9_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc6746; GFX9_ITERATIVE-NEXT: s_xor_b64 s[10:11], exec, s[4:5]6747; GFX9_ITERATIVE-NEXT: s_cbranch_execz .LBB11_66748; GFX9_ITERATIVE-NEXT: ; %bb.3:6749; GFX9_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)6750; GFX9_ITERATIVE-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x06751; GFX9_ITERATIVE-NEXT: s_mov_b64 s[12:13], 06752; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v6, s96753; GFX9_ITERATIVE-NEXT: s_mov_b32 s7, 0xf0006754; GFX9_ITERATIVE-NEXT: s_mov_b32 s6, -16755; GFX9_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)6756; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, s46757; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, s56758; GFX9_ITERATIVE-NEXT: s_mov_b32 s4, s26759; GFX9_ITERATIVE-NEXT: s_mov_b32 s5, s36760; GFX9_ITERATIVE-NEXT: .LBB11_4: ; %atomicrmw.start6761; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=16762; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v10, v16763; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v9, v06764; GFX9_ITERATIVE-NEXT: v_subrev_co_u32_e32 v7, vcc, s8, v96765; GFX9_ITERATIVE-NEXT: v_subb_co_u32_e32 v8, vcc, v10, v6, vcc6766; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, v76767; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, v86768; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v2, v96769; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v3, v106770; GFX9_ITERATIVE-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc6771; GFX9_ITERATIVE-NEXT: s_waitcnt vmcnt(0)6772; GFX9_ITERATIVE-NEXT: buffer_wbinvl1_vol6773; GFX9_ITERATIVE-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]6774; GFX9_ITERATIVE-NEXT: s_or_b64 s[12:13], vcc, s[12:13]6775; GFX9_ITERATIVE-NEXT: s_andn2_b64 exec, exec, s[12:13]6776; GFX9_ITERATIVE-NEXT: s_cbranch_execnz .LBB11_46777; GFX9_ITERATIVE-NEXT: ; %bb.5: ; %Flow6778; GFX9_ITERATIVE-NEXT: s_or_b64 exec, exec, s[12:13]6779; GFX9_ITERATIVE-NEXT: .LBB11_6: ; %Flow46780; GFX9_ITERATIVE-NEXT: s_or_b64 exec, exec, s[10:11]6781; GFX9_ITERATIVE-NEXT: v_readfirstlane_b32 s4, v16782; GFX9_ITERATIVE-NEXT: v_readfirstlane_b32 s5, v06783; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, s46784; GFX9_ITERATIVE-NEXT: v_sub_co_u32_e32 v0, vcc, s5, v46785; GFX9_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)6786; GFX9_ITERATIVE-NEXT: s_mov_b32 s3, 0xf0006787; GFX9_ITERATIVE-NEXT: s_mov_b32 s2, -16788; GFX9_ITERATIVE-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v5, vcc6789; GFX9_ITERATIVE-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 06790; GFX9_ITERATIVE-NEXT: s_endpgm6791;6792; GFX1064_ITERATIVE-LABEL: sub_i64_varying:6793; GFX1064_ITERATIVE: ; %bb.0: ; %entry6794; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v1, 06795; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec6796; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[8:9], 06797; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr4_vgpr56798; GFX1064_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop6799; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=16800; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]6801; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s26802; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s6, v1, s26803; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v5, s9, s26804; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v4, s8, s26805; GFX1064_ITERATIVE-NEXT: s_add_u32 s8, s8, s36806; GFX1064_ITERATIVE-NEXT: s_addc_u32 s9, s9, s66807; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s26808; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]6809; GFX1064_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_16810; GFX1064_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd6811; GFX1064_ITERATIVE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246812; GFX1064_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 06813; GFX1064_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v06814; GFX1064_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v06815; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr16816; GFX1064_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc6817; GFX1064_ITERATIVE-NEXT: s_xor_b64 s[10:11], exec, s[4:5]6818; GFX1064_ITERATIVE-NEXT: s_cbranch_execz .LBB11_66819; GFX1064_ITERATIVE-NEXT: ; %bb.3:6820; GFX1064_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)6821; GFX1064_ITERATIVE-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x06822; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[12:13], 06823; GFX1064_ITERATIVE-NEXT: s_mov_b32 s7, 0x310160006824; GFX1064_ITERATIVE-NEXT: s_mov_b32 s6, -16825; GFX1064_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)6826; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v0, s46827; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v1, s56828; GFX1064_ITERATIVE-NEXT: s_mov_b32 s4, s26829; GFX1064_ITERATIVE-NEXT: s_mov_b32 s5, s36830; GFX1064_ITERATIVE-NEXT: .LBB11_4: ; %atomicrmw.start6831; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=16832; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v9, v16833; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v8, v06834; GFX1064_ITERATIVE-NEXT: v_sub_co_u32 v6, vcc, v8, s86835; GFX1064_ITERATIVE-NEXT: v_subrev_co_ci_u32_e32 v7, vcc, s9, v9, vcc6836; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v0, v66837; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v2, v86838; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v1, v76839; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v3, v96840; GFX1064_ITERATIVE-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc6841; GFX1064_ITERATIVE-NEXT: s_waitcnt vmcnt(0)6842; GFX1064_ITERATIVE-NEXT: buffer_gl1_inv6843; GFX1064_ITERATIVE-NEXT: buffer_gl0_inv6844; GFX1064_ITERATIVE-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]6845; GFX1064_ITERATIVE-NEXT: s_or_b64 s[12:13], vcc, s[12:13]6846; GFX1064_ITERATIVE-NEXT: s_andn2_b64 exec, exec, s[12:13]6847; GFX1064_ITERATIVE-NEXT: s_cbranch_execnz .LBB11_46848; GFX1064_ITERATIVE-NEXT: ; %bb.5: ; %Flow6849; GFX1064_ITERATIVE-NEXT: s_or_b64 exec, exec, s[12:13]6850; GFX1064_ITERATIVE-NEXT: .LBB11_6: ; %Flow46851; GFX1064_ITERATIVE-NEXT: s_or_b64 exec, exec, s[10:11]6852; GFX1064_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)6853; GFX1064_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v06854; GFX1064_ITERATIVE-NEXT: v_readfirstlane_b32 s3, v16855; GFX1064_ITERATIVE-NEXT: v_sub_co_u32 v0, vcc, s2, v46856; GFX1064_ITERATIVE-NEXT: v_sub_co_ci_u32_e32 v1, vcc, s3, v5, vcc6857; GFX1064_ITERATIVE-NEXT: s_mov_b32 s3, 0x310160006858; GFX1064_ITERATIVE-NEXT: s_mov_b32 s2, -16859; GFX1064_ITERATIVE-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 06860; GFX1064_ITERATIVE-NEXT: s_endpgm6861;6862; GFX1032_ITERATIVE-LABEL: sub_i64_varying:6863; GFX1032_ITERATIVE: ; %bb.0: ; %entry6864; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v1, 06865; GFX1032_ITERATIVE-NEXT: s_mov_b32 s0, exec_lo6866; GFX1032_ITERATIVE-NEXT: s_mov_b64 s[8:9], 06867; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr4_vgpr56868; GFX1032_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop6869; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=16870; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s1, s06871; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s2, v0, s16872; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s3, v1, s16873; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v5, s9, s16874; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v4, s8, s16875; GFX1032_ITERATIVE-NEXT: s_add_u32 s8, s8, s26876; GFX1032_ITERATIVE-NEXT: s_addc_u32 s9, s9, s36877; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s1, 1, s16878; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s0, s0, s16879; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_16880; GFX1032_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd6881; GFX1032_ITERATIVE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x246882; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 06883; GFX1032_ITERATIVE-NEXT: s_mov_b32 s11, 06884; GFX1032_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v06885; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr16886; GFX1032_ITERATIVE-NEXT: s_and_saveexec_b32 s4, vcc_lo6887; GFX1032_ITERATIVE-NEXT: s_xor_b32 s10, exec_lo, s46888; GFX1032_ITERATIVE-NEXT: s_cbranch_execz .LBB11_66889; GFX1032_ITERATIVE-NEXT: ; %bb.3:6890; GFX1032_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)6891; GFX1032_ITERATIVE-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x06892; GFX1032_ITERATIVE-NEXT: s_mov_b32 s7, 0x310160006893; GFX1032_ITERATIVE-NEXT: s_mov_b32 s6, -16894; GFX1032_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)6895; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v0, s46896; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v1, s56897; GFX1032_ITERATIVE-NEXT: s_mov_b32 s4, s26898; GFX1032_ITERATIVE-NEXT: s_mov_b32 s5, s36899; GFX1032_ITERATIVE-NEXT: .LBB11_4: ; %atomicrmw.start6900; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=16901; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v9, v16902; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v8, v06903; GFX1032_ITERATIVE-NEXT: v_sub_co_u32 v6, vcc_lo, v8, s86904; GFX1032_ITERATIVE-NEXT: v_subrev_co_ci_u32_e32 v7, vcc_lo, s9, v9, vcc_lo6905; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v0, v66906; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v2, v86907; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v1, v76908; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v3, v96909; GFX1032_ITERATIVE-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc6910; GFX1032_ITERATIVE-NEXT: s_waitcnt vmcnt(0)6911; GFX1032_ITERATIVE-NEXT: buffer_gl1_inv6912; GFX1032_ITERATIVE-NEXT: buffer_gl0_inv6913; GFX1032_ITERATIVE-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]6914; GFX1032_ITERATIVE-NEXT: s_or_b32 s11, vcc_lo, s116915; GFX1032_ITERATIVE-NEXT: s_andn2_b32 exec_lo, exec_lo, s116916; GFX1032_ITERATIVE-NEXT: s_cbranch_execnz .LBB11_46917; GFX1032_ITERATIVE-NEXT: ; %bb.5: ; %Flow6918; GFX1032_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s116919; GFX1032_ITERATIVE-NEXT: .LBB11_6: ; %Flow46920; GFX1032_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s106921; GFX1032_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)6922; GFX1032_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v06923; GFX1032_ITERATIVE-NEXT: v_readfirstlane_b32 s3, v16924; GFX1032_ITERATIVE-NEXT: v_sub_co_u32 v0, vcc_lo, s2, v46925; GFX1032_ITERATIVE-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, s3, v5, vcc_lo6926; GFX1032_ITERATIVE-NEXT: s_mov_b32 s3, 0x310160006927; GFX1032_ITERATIVE-NEXT: s_mov_b32 s2, -16928; GFX1032_ITERATIVE-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 06929; GFX1032_ITERATIVE-NEXT: s_endpgm6930;6931; GFX1164_ITERATIVE-LABEL: sub_i64_varying:6932; GFX1164_ITERATIVE: ; %bb.0: ; %entry6933; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v1, 06934; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v0, 0x3ff, v06935; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec6936; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[8:9], 06937; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr4_vgpr56938; GFX1164_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop6939; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=16940; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s2, s[0:1]6941; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)6942; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s26943; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v1, s26944; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v5, s9, s26945; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v4, s8, s26946; GFX1164_ITERATIVE-NEXT: s_add_u32 s8, s8, s36947; GFX1164_ITERATIVE-NEXT: s_addc_u32 s9, s9, s66948; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s26949; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6950; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]6951; GFX1164_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_16952; GFX1164_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd6953; GFX1164_ITERATIVE-NEXT: s_load_b128 s[0:3], s[4:5], 0x246954; GFX1164_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 06955; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6956; GFX1164_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v06957; GFX1164_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v06958; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr16959; GFX1164_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc6960; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6961; GFX1164_ITERATIVE-NEXT: s_xor_b64 s[10:11], exec, s[4:5]6962; GFX1164_ITERATIVE-NEXT: s_cbranch_execz .LBB11_66963; GFX1164_ITERATIVE-NEXT: ; %bb.3:6964; GFX1164_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)6965; GFX1164_ITERATIVE-NEXT: s_load_b64 s[4:5], s[2:3], 0x06966; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[12:13], 06967; GFX1164_ITERATIVE-NEXT: s_mov_b32 s7, 0x310160006968; GFX1164_ITERATIVE-NEXT: s_mov_b32 s6, -16969; GFX1164_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)6970; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v0, s46971; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v1, s56972; GFX1164_ITERATIVE-NEXT: s_mov_b32 s4, s26973; GFX1164_ITERATIVE-NEXT: s_mov_b32 s5, s36974; GFX1164_ITERATIVE-NEXT: .LBB11_4: ; %atomicrmw.start6975; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=16976; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)6977; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v9, v16978; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v8, v06979; GFX1164_ITERATIVE-NEXT: v_sub_co_u32 v6, vcc, v8, s86980; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)6981; GFX1164_ITERATIVE-NEXT: v_subrev_co_ci_u32_e64 v7, null, s9, v9, vcc6982; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v0, v66983; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v2, v86984; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_3)6985; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v1, v76986; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v3, v96987; GFX1164_ITERATIVE-NEXT: buffer_atomic_cmpswap_b64 v[0:3], off, s[4:7], 0 glc6988; GFX1164_ITERATIVE-NEXT: s_waitcnt vmcnt(0)6989; GFX1164_ITERATIVE-NEXT: buffer_gl1_inv6990; GFX1164_ITERATIVE-NEXT: buffer_gl0_inv6991; GFX1164_ITERATIVE-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]6992; GFX1164_ITERATIVE-NEXT: s_waitcnt_depctr depctr_va_vcc(0)6993; GFX1164_ITERATIVE-NEXT: s_or_b64 s[12:13], vcc, s[12:13]6994; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6995; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 exec, exec, s[12:13]6996; GFX1164_ITERATIVE-NEXT: s_cbranch_execnz .LBB11_46997; GFX1164_ITERATIVE-NEXT: ; %bb.5: ; %Flow6998; GFX1164_ITERATIVE-NEXT: s_or_b64 exec, exec, s[12:13]6999; GFX1164_ITERATIVE-NEXT: .LBB11_6: ; %Flow47000; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)7001; GFX1164_ITERATIVE-NEXT: s_or_b64 exec, exec, s[10:11]7002; GFX1164_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)7003; GFX1164_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v07004; GFX1164_ITERATIVE-NEXT: v_readfirstlane_b32 s3, v17005; GFX1164_ITERATIVE-NEXT: v_sub_co_u32 v0, vcc, s2, v47006; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1)7007; GFX1164_ITERATIVE-NEXT: v_sub_co_ci_u32_e64 v1, null, s3, v5, vcc7008; GFX1164_ITERATIVE-NEXT: s_mov_b32 s3, 0x310160007009; GFX1164_ITERATIVE-NEXT: s_mov_b32 s2, -17010; GFX1164_ITERATIVE-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 07011; GFX1164_ITERATIVE-NEXT: s_endpgm7012;7013; GFX1132_ITERATIVE-LABEL: sub_i64_varying:7014; GFX1132_ITERATIVE: ; %bb.0: ; %entry7015; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v07016; GFX1132_ITERATIVE-NEXT: s_mov_b32 s0, exec_lo7017; GFX1132_ITERATIVE-NEXT: s_mov_b64 s[8:9], 07018; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr4_vgpr57019; GFX1132_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop7020; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=17021; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s1, s07022; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)7023; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s2, v0, s17024; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s3, v1, s17025; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v5, s9, s17026; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v4, s8, s17027; GFX1132_ITERATIVE-NEXT: s_add_u32 s8, s8, s27028; GFX1132_ITERATIVE-NEXT: s_addc_u32 s9, s9, s37029; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s1, 1, s17030; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7031; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s0, s0, s17032; GFX1132_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_17033; GFX1132_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd7034; GFX1132_ITERATIVE-NEXT: s_load_b128 s[0:3], s[4:5], 0x247035; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 07036; GFX1132_ITERATIVE-NEXT: s_mov_b32 s11, 07037; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)7038; GFX1132_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v07039; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr17040; GFX1132_ITERATIVE-NEXT: s_and_saveexec_b32 s4, vcc_lo7041; GFX1132_ITERATIVE-NEXT: s_xor_b32 s10, exec_lo, s47042; GFX1132_ITERATIVE-NEXT: s_cbranch_execz .LBB11_67043; GFX1132_ITERATIVE-NEXT: ; %bb.3:7044; GFX1132_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)7045; GFX1132_ITERATIVE-NEXT: s_load_b64 s[4:5], s[2:3], 0x07046; GFX1132_ITERATIVE-NEXT: s_mov_b32 s7, 0x310160007047; GFX1132_ITERATIVE-NEXT: s_mov_b32 s6, -17048; GFX1132_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)7049; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s57050; GFX1132_ITERATIVE-NEXT: s_mov_b32 s4, s27051; GFX1132_ITERATIVE-NEXT: s_mov_b32 s5, s37052; GFX1132_ITERATIVE-NEXT: .LBB11_4: ; %atomicrmw.start7053; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=17054; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7055; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v07056; GFX1132_ITERATIVE-NEXT: v_sub_co_u32 v6, vcc_lo, v8, s87057; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)7058; GFX1132_ITERATIVE-NEXT: v_subrev_co_ci_u32_e64 v7, null, s9, v9, vcc_lo7059; GFX1132_ITERATIVE-NEXT: v_mov_b32_e32 v0, v67060; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v97061; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_3)7062; GFX1132_ITERATIVE-NEXT: v_mov_b32_e32 v1, v77063; GFX1132_ITERATIVE-NEXT: buffer_atomic_cmpswap_b64 v[0:3], off, s[4:7], 0 glc7064; GFX1132_ITERATIVE-NEXT: s_waitcnt vmcnt(0)7065; GFX1132_ITERATIVE-NEXT: buffer_gl1_inv7066; GFX1132_ITERATIVE-NEXT: buffer_gl0_inv7067; GFX1132_ITERATIVE-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]7068; GFX1132_ITERATIVE-NEXT: s_or_b32 s11, vcc_lo, s117069; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7070; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 exec_lo, exec_lo, s117071; GFX1132_ITERATIVE-NEXT: s_cbranch_execnz .LBB11_47072; GFX1132_ITERATIVE-NEXT: ; %bb.5: ; %Flow7073; GFX1132_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s117074; GFX1132_ITERATIVE-NEXT: .LBB11_6: ; %Flow47075; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)7076; GFX1132_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s107077; GFX1132_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)7078; GFX1132_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v07079; GFX1132_ITERATIVE-NEXT: v_readfirstlane_b32 s3, v17080; GFX1132_ITERATIVE-NEXT: v_sub_co_u32 v0, vcc_lo, s2, v47081; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1)7082; GFX1132_ITERATIVE-NEXT: v_sub_co_ci_u32_e64 v1, null, s3, v5, vcc_lo7083; GFX1132_ITERATIVE-NEXT: s_mov_b32 s3, 0x310160007084; GFX1132_ITERATIVE-NEXT: s_mov_b32 s2, -17085; GFX1132_ITERATIVE-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 07086; GFX1132_ITERATIVE-NEXT: s_endpgm7087;7088; GFX1264_ITERATIVE-LABEL: sub_i64_varying:7089; GFX1264_ITERATIVE: ; %bb.0: ; %entry7090; GFX1264_ITERATIVE-NEXT: v_mov_b32_e32 v2, 07091; GFX1264_ITERATIVE-NEXT: v_and_b32_e32 v3, 0x3ff, v07092; GFX1264_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec7093; GFX1264_ITERATIVE-NEXT: s_mov_b64 s[6:7], 07094; GFX1264_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr17095; GFX1264_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop7096; GFX1264_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=17097; GFX1264_ITERATIVE-NEXT: s_ctz_i32_b64 s8, s[0:1]7098; GFX1264_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)7099; GFX1264_ITERATIVE-NEXT: v_readlane_b32 s3, v2, s87100; GFX1264_ITERATIVE-NEXT: v_readlane_b32 s2, v3, s87101; GFX1264_ITERATIVE-NEXT: v_writelane_b32 v1, s7, s87102; GFX1264_ITERATIVE-NEXT: v_writelane_b32 v0, s6, s87103; GFX1264_ITERATIVE-NEXT: s_lshl_b64 s[8:9], 1, s87104; GFX1264_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)7105; GFX1264_ITERATIVE-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[8:9]7106; GFX1264_ITERATIVE-NEXT: s_add_nc_u64 s[6:7], s[6:7], s[2:3]7107; GFX1264_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_17108; GFX1264_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd7109; GFX1264_ITERATIVE-NEXT: s_load_b128 s[0:3], s[4:5], 0x247110; GFX1264_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 07111; GFX1264_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7112; GFX1264_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v27113; GFX1264_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v27114; GFX1264_ITERATIVE-NEXT: ; implicit-def: $vgpr2_vgpr37115; GFX1264_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc7116; GFX1264_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7117; GFX1264_ITERATIVE-NEXT: s_xor_b64 s[4:5], exec, s[4:5]7118; GFX1264_ITERATIVE-NEXT: s_cbranch_execz .LBB11_47119; GFX1264_ITERATIVE-NEXT: ; %bb.3:7120; GFX1264_ITERATIVE-NEXT: v_mov_b32_e32 v2, s67121; GFX1264_ITERATIVE-NEXT: v_mov_b32_e32 v3, s77122; GFX1264_ITERATIVE-NEXT: s_mov_b32 s11, 0x310160007123; GFX1264_ITERATIVE-NEXT: s_mov_b32 s10, -17124; GFX1264_ITERATIVE-NEXT: s_wait_kmcnt 0x07125; GFX1264_ITERATIVE-NEXT: s_mov_b32 s8, s27126; GFX1264_ITERATIVE-NEXT: s_mov_b32 s9, s37127; GFX1264_ITERATIVE-NEXT: buffer_atomic_sub_u64 v[2:3], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV7128; GFX1264_ITERATIVE-NEXT: s_wait_loadcnt 0x07129; GFX1264_ITERATIVE-NEXT: global_inv scope:SCOPE_DEV7130; GFX1264_ITERATIVE-NEXT: .LBB11_4:7131; GFX1264_ITERATIVE-NEXT: s_or_b64 exec, exec, s[4:5]7132; GFX1264_ITERATIVE-NEXT: s_wait_kmcnt 0x07133; GFX1264_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v27134; GFX1264_ITERATIVE-NEXT: v_readfirstlane_b32 s3, v37135; GFX1264_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)7136; GFX1264_ITERATIVE-NEXT: v_sub_co_u32 v0, vcc, s2, v07137; GFX1264_ITERATIVE-NEXT: v_sub_co_ci_u32_e64 v1, null, s3, v1, vcc7138; GFX1264_ITERATIVE-NEXT: s_mov_b32 s3, 0x310160007139; GFX1264_ITERATIVE-NEXT: s_mov_b32 s2, -17140; GFX1264_ITERATIVE-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null7141; GFX1264_ITERATIVE-NEXT: s_endpgm7142;7143; GFX1232_ITERATIVE-LABEL: sub_i64_varying:7144; GFX1232_ITERATIVE: ; %bb.0: ; %entry7145; GFX1232_ITERATIVE-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v3, 0x3ff, v07146; GFX1232_ITERATIVE-NEXT: s_mov_b32 s0, exec_lo7147; GFX1232_ITERATIVE-NEXT: s_mov_b64 s[6:7], 07148; GFX1232_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr17149; GFX1232_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop7150; GFX1232_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=17151; GFX1232_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)7152; GFX1232_ITERATIVE-NEXT: s_ctz_i32_b32 s1, s07153; GFX1232_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)7154; GFX1232_ITERATIVE-NEXT: v_readlane_b32 s3, v2, s17155; GFX1232_ITERATIVE-NEXT: v_readlane_b32 s2, v3, s17156; GFX1232_ITERATIVE-NEXT: v_writelane_b32 v1, s7, s17157; GFX1232_ITERATIVE-NEXT: v_writelane_b32 v0, s6, s17158; GFX1232_ITERATIVE-NEXT: s_lshl_b32 s1, 1, s17159; GFX1232_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)7160; GFX1232_ITERATIVE-NEXT: s_and_not1_b32 s0, s0, s17161; GFX1232_ITERATIVE-NEXT: s_add_nc_u64 s[6:7], s[6:7], s[2:3]7162; GFX1232_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_17163; GFX1232_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd7164; GFX1232_ITERATIVE-NEXT: s_load_b128 s[0:3], s[4:5], 0x247165; GFX1232_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 07166; GFX1232_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)7167; GFX1232_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v27168; GFX1232_ITERATIVE-NEXT: ; implicit-def: $vgpr2_vgpr37169; GFX1232_ITERATIVE-NEXT: s_and_saveexec_b32 s4, vcc_lo7170; GFX1232_ITERATIVE-NEXT: s_xor_b32 s4, exec_lo, s47171; GFX1232_ITERATIVE-NEXT: s_cbranch_execz .LBB11_47172; GFX1232_ITERATIVE-NEXT: ; %bb.3:7173; GFX1232_ITERATIVE-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s77174; GFX1232_ITERATIVE-NEXT: s_mov_b32 s11, 0x310160007175; GFX1232_ITERATIVE-NEXT: s_mov_b32 s10, -17176; GFX1232_ITERATIVE-NEXT: s_wait_kmcnt 0x07177; GFX1232_ITERATIVE-NEXT: s_mov_b32 s8, s27178; GFX1232_ITERATIVE-NEXT: s_mov_b32 s9, s37179; GFX1232_ITERATIVE-NEXT: buffer_atomic_sub_u64 v[2:3], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV7180; GFX1232_ITERATIVE-NEXT: s_wait_loadcnt 0x07181; GFX1232_ITERATIVE-NEXT: global_inv scope:SCOPE_DEV7182; GFX1232_ITERATIVE-NEXT: .LBB11_4:7183; GFX1232_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s47184; GFX1232_ITERATIVE-NEXT: s_wait_kmcnt 0x07185; GFX1232_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v27186; GFX1232_ITERATIVE-NEXT: v_readfirstlane_b32 s3, v37187; GFX1232_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)7188; GFX1232_ITERATIVE-NEXT: v_sub_co_u32 v0, vcc_lo, s2, v07189; GFX1232_ITERATIVE-NEXT: v_sub_co_ci_u32_e64 v1, null, s3, v1, vcc_lo7190; GFX1232_ITERATIVE-NEXT: s_mov_b32 s3, 0x310160007191; GFX1232_ITERATIVE-NEXT: s_mov_b32 s2, -17192; GFX1232_ITERATIVE-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null7193; GFX1232_ITERATIVE-NEXT: s_endpgm7194;7195; GFX7LESS_DPP-LABEL: sub_i64_varying:7196; GFX7LESS_DPP: ; %bb.0: ; %entry7197; GFX7LESS_DPP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x97198; GFX7LESS_DPP-NEXT: s_waitcnt lgkmcnt(0)7199; GFX7LESS_DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x07200; GFX7LESS_DPP-NEXT: s_mov_b64 s[8:9], 07201; GFX7LESS_DPP-NEXT: s_mov_b32 s7, 0xf0007202; GFX7LESS_DPP-NEXT: s_waitcnt lgkmcnt(0)7203; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v3, s47204; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v4, s57205; GFX7LESS_DPP-NEXT: s_mov_b32 s6, -17206; GFX7LESS_DPP-NEXT: s_mov_b32 s4, s27207; GFX7LESS_DPP-NEXT: s_mov_b32 s5, s37208; GFX7LESS_DPP-NEXT: .LBB11_1: ; %atomicrmw.start7209; GFX7LESS_DPP-NEXT: ; =>This Inner Loop Header: Depth=17210; GFX7LESS_DPP-NEXT: v_sub_i32_e32 v1, vcc, v3, v07211; GFX7LESS_DPP-NEXT: v_subbrev_u32_e32 v2, vcc, 0, v4, vcc7212; GFX7LESS_DPP-NEXT: s_waitcnt expcnt(0)7213; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v8, v47214; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v7, v37215; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v6, v27216; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v5, v17217; GFX7LESS_DPP-NEXT: buffer_atomic_cmpswap_x2 v[5:8], off, s[4:7], 0 glc7218; GFX7LESS_DPP-NEXT: s_waitcnt vmcnt(0)7219; GFX7LESS_DPP-NEXT: buffer_wbinvl17220; GFX7LESS_DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[5:6], v[3:4]7221; GFX7LESS_DPP-NEXT: s_or_b64 s[8:9], vcc, s[8:9]7222; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v3, v57223; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v4, v67224; GFX7LESS_DPP-NEXT: s_andn2_b64 exec, exec, s[8:9]7225; GFX7LESS_DPP-NEXT: s_cbranch_execnz .LBB11_17226; GFX7LESS_DPP-NEXT: ; %bb.2: ; %atomicrmw.end7227; GFX7LESS_DPP-NEXT: s_or_b64 exec, exec, s[8:9]7228; GFX7LESS_DPP-NEXT: s_mov_b32 s3, 0xf0007229; GFX7LESS_DPP-NEXT: s_mov_b32 s2, -17230; GFX7LESS_DPP-NEXT: buffer_store_dwordx2 v[5:6], off, s[0:3], 07231; GFX7LESS_DPP-NEXT: s_endpgm7232;7233; GFX8_DPP-LABEL: sub_i64_varying:7234; GFX8_DPP: ; %bb.0: ; %entry7235; GFX8_DPP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247236; GFX8_DPP-NEXT: s_or_saveexec_b64 s[4:5], -17237; GFX8_DPP-NEXT: v_mov_b32_e32 v1, 07238; GFX8_DPP-NEXT: s_mov_b64 exec, s[4:5]7239; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 07240; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v67241; GFX8_DPP-NEXT: s_or_saveexec_b64 s[4:5], -17242; GFX8_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[4:5]7243; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 07244; GFX8_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s[4:5]7245; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 07246; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf7247; GFX8_DPP-NEXT: v_add_u32_e32 v3, vcc, v3, v57248; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf7249; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 07250; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc7251; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 07252; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:2 row_mask:0xf bank_mask:0xf7253; GFX8_DPP-NEXT: v_add_u32_e32 v3, vcc, v3, v57254; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:2 row_mask:0xf bank_mask:0xf7255; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 07256; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc7257; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 07258; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf7259; GFX8_DPP-NEXT: v_add_u32_e32 v3, vcc, v3, v57260; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf7261; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 07262; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc7263; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 07264; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:8 row_mask:0xf bank_mask:0xf7265; GFX8_DPP-NEXT: v_add_u32_e32 v3, vcc, v3, v57266; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:8 row_mask:0xf bank_mask:0xf7267; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 07268; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc7269; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 07270; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:15 row_mask:0xa bank_mask:0xf7271; GFX8_DPP-NEXT: v_add_u32_e32 v3, vcc, v3, v57272; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:15 row_mask:0xa bank_mask:0xf7273; GFX8_DPP-NEXT: v_mov_b32_e32 v5, 07274; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc7275; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 07276; GFX8_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:31 row_mask:0xc bank_mask:0xf7277; GFX8_DPP-NEXT: v_add_u32_e32 v3, vcc, v3, v57278; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:31 row_mask:0xc bank_mask:0xf7279; GFX8_DPP-NEXT: v_addc_u32_e32 v4, vcc, v2, v4, vcc7280; GFX8_DPP-NEXT: v_mov_b32_e32 v2, 07281; GFX8_DPP-NEXT: v_readlane_b32 s11, v4, 637282; GFX8_DPP-NEXT: v_readlane_b32 s10, v3, 637283; GFX8_DPP-NEXT: v_mov_b32_dpp v2, v4 wave_shr:1 row_mask:0xf bank_mask:0xf7284; GFX8_DPP-NEXT: v_mov_b32_dpp v1, v3 wave_shr:1 row_mask:0xf bank_mask:0xf7285; GFX8_DPP-NEXT: s_mov_b64 exec, s[4:5]7286; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v67287; GFX8_DPP-NEXT: ; implicit-def: $vgpr6_vgpr77288; GFX8_DPP-NEXT: s_and_saveexec_b64 s[8:9], vcc7289; GFX8_DPP-NEXT: s_cbranch_execz .LBB11_47290; GFX8_DPP-NEXT: ; %bb.1:7291; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0)7292; GFX8_DPP-NEXT: s_load_dwordx2 s[14:15], s[2:3], 0x07293; GFX8_DPP-NEXT: s_mov_b64 s[12:13], 07294; GFX8_DPP-NEXT: s_mov_b32 s7, 0xf0007295; GFX8_DPP-NEXT: s_mov_b32 s6, -17296; GFX8_DPP-NEXT: s_mov_b32 s4, s27297; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0)7298; GFX8_DPP-NEXT: v_mov_b32_e32 v6, s147299; GFX8_DPP-NEXT: v_mov_b32_e32 v7, s157300; GFX8_DPP-NEXT: s_mov_b32 s5, s37301; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s117302; GFX8_DPP-NEXT: .LBB11_2: ; %atomicrmw.start7303; GFX8_DPP-NEXT: ; =>This Inner Loop Header: Depth=17304; GFX8_DPP-NEXT: v_mov_b32_e32 v11, v77305; GFX8_DPP-NEXT: v_mov_b32_e32 v10, v67306; GFX8_DPP-NEXT: v_subrev_u32_e32 v8, vcc, s10, v107307; GFX8_DPP-NEXT: v_subb_u32_e32 v9, vcc, v11, v0, vcc7308; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v87309; GFX8_DPP-NEXT: v_mov_b32_e32 v7, v97310; GFX8_DPP-NEXT: v_mov_b32_e32 v8, v107311; GFX8_DPP-NEXT: v_mov_b32_e32 v9, v117312; GFX8_DPP-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 glc7313; GFX8_DPP-NEXT: s_waitcnt vmcnt(0)7314; GFX8_DPP-NEXT: buffer_wbinvl1_vol7315; GFX8_DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[10:11]7316; GFX8_DPP-NEXT: s_or_b64 s[12:13], vcc, s[12:13]7317; GFX8_DPP-NEXT: s_andn2_b64 exec, exec, s[12:13]7318; GFX8_DPP-NEXT: s_cbranch_execnz .LBB11_27319; GFX8_DPP-NEXT: ; %bb.3: ; %Flow7320; GFX8_DPP-NEXT: s_or_b64 exec, exec, s[12:13]7321; GFX8_DPP-NEXT: .LBB11_4: ; %Flow37322; GFX8_DPP-NEXT: s_or_b64 exec, exec, s[8:9]7323; GFX8_DPP-NEXT: v_readfirstlane_b32 s4, v77324; GFX8_DPP-NEXT: v_readfirstlane_b32 s5, v67325; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v17326; GFX8_DPP-NEXT: v_mov_b32_e32 v7, v27327; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s47328; GFX8_DPP-NEXT: v_sub_u32_e32 v6, vcc, s5, v67329; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0)7330; GFX8_DPP-NEXT: s_mov_b32 s3, 0xf0007331; GFX8_DPP-NEXT: s_mov_b32 s2, -17332; GFX8_DPP-NEXT: v_subb_u32_e32 v7, vcc, v0, v7, vcc7333; GFX8_DPP-NEXT: buffer_store_dwordx2 v[6:7], off, s[0:3], 07334; GFX8_DPP-NEXT: s_endpgm7335;7336; GFX9_DPP-LABEL: sub_i64_varying:7337; GFX9_DPP: ; %bb.0: ; %entry7338; GFX9_DPP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247339; GFX9_DPP-NEXT: s_or_saveexec_b64 s[4:5], -17340; GFX9_DPP-NEXT: v_mov_b32_e32 v1, 07341; GFX9_DPP-NEXT: s_mov_b64 exec, s[4:5]7342; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 07343; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v67344; GFX9_DPP-NEXT: s_or_saveexec_b64 s[4:5], -17345; GFX9_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[4:5]7346; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 07347; GFX9_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s[4:5]7348; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 07349; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:1 row_mask:0xf bank_mask:0xf7350; GFX9_DPP-NEXT: v_add_co_u32_e32 v3, vcc, v3, v57351; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf7352; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 07353; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc7354; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 07355; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:2 row_mask:0xf bank_mask:0xf7356; GFX9_DPP-NEXT: v_add_co_u32_e32 v3, vcc, v3, v57357; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:2 row_mask:0xf bank_mask:0xf7358; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 07359; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc7360; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 07361; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:4 row_mask:0xf bank_mask:0xf7362; GFX9_DPP-NEXT: v_add_co_u32_e32 v3, vcc, v3, v57363; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf7364; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 07365; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc7366; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 07367; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_shr:8 row_mask:0xf bank_mask:0xf7368; GFX9_DPP-NEXT: v_add_co_u32_e32 v3, vcc, v3, v57369; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:8 row_mask:0xf bank_mask:0xf7370; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 07371; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc7372; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 07373; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:15 row_mask:0xa bank_mask:0xf7374; GFX9_DPP-NEXT: v_add_co_u32_e32 v3, vcc, v3, v57375; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:15 row_mask:0xa bank_mask:0xf7376; GFX9_DPP-NEXT: v_mov_b32_e32 v5, 07377; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc7378; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 07379; GFX9_DPP-NEXT: v_mov_b32_dpp v5, v3 row_bcast:31 row_mask:0xc bank_mask:0xf7380; GFX9_DPP-NEXT: v_add_co_u32_e32 v3, vcc, v3, v57381; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:31 row_mask:0xc bank_mask:0xf7382; GFX9_DPP-NEXT: v_addc_co_u32_e32 v4, vcc, v2, v4, vcc7383; GFX9_DPP-NEXT: v_mov_b32_e32 v2, 07384; GFX9_DPP-NEXT: v_readlane_b32 s11, v4, 637385; GFX9_DPP-NEXT: v_readlane_b32 s10, v3, 637386; GFX9_DPP-NEXT: v_mov_b32_dpp v2, v4 wave_shr:1 row_mask:0xf bank_mask:0xf7387; GFX9_DPP-NEXT: v_mov_b32_dpp v1, v3 wave_shr:1 row_mask:0xf bank_mask:0xf7388; GFX9_DPP-NEXT: s_mov_b64 exec, s[4:5]7389; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v67390; GFX9_DPP-NEXT: ; implicit-def: $vgpr6_vgpr77391; GFX9_DPP-NEXT: s_and_saveexec_b64 s[8:9], vcc7392; GFX9_DPP-NEXT: s_cbranch_execz .LBB11_47393; GFX9_DPP-NEXT: ; %bb.1:7394; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0)7395; GFX9_DPP-NEXT: s_load_dwordx2 s[14:15], s[2:3], 0x07396; GFX9_DPP-NEXT: s_mov_b64 s[12:13], 07397; GFX9_DPP-NEXT: s_mov_b32 s7, 0xf0007398; GFX9_DPP-NEXT: s_mov_b32 s6, -17399; GFX9_DPP-NEXT: s_mov_b32 s4, s27400; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0)7401; GFX9_DPP-NEXT: v_mov_b32_e32 v6, s147402; GFX9_DPP-NEXT: v_mov_b32_e32 v7, s157403; GFX9_DPP-NEXT: s_mov_b32 s5, s37404; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s117405; GFX9_DPP-NEXT: .LBB11_2: ; %atomicrmw.start7406; GFX9_DPP-NEXT: ; =>This Inner Loop Header: Depth=17407; GFX9_DPP-NEXT: v_mov_b32_e32 v11, v77408; GFX9_DPP-NEXT: v_mov_b32_e32 v10, v67409; GFX9_DPP-NEXT: v_subrev_co_u32_e32 v8, vcc, s10, v107410; GFX9_DPP-NEXT: v_subb_co_u32_e32 v9, vcc, v11, v0, vcc7411; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v87412; GFX9_DPP-NEXT: v_mov_b32_e32 v7, v97413; GFX9_DPP-NEXT: v_mov_b32_e32 v8, v107414; GFX9_DPP-NEXT: v_mov_b32_e32 v9, v117415; GFX9_DPP-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 glc7416; GFX9_DPP-NEXT: s_waitcnt vmcnt(0)7417; GFX9_DPP-NEXT: buffer_wbinvl1_vol7418; GFX9_DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[10:11]7419; GFX9_DPP-NEXT: s_or_b64 s[12:13], vcc, s[12:13]7420; GFX9_DPP-NEXT: s_andn2_b64 exec, exec, s[12:13]7421; GFX9_DPP-NEXT: s_cbranch_execnz .LBB11_27422; GFX9_DPP-NEXT: ; %bb.3: ; %Flow7423; GFX9_DPP-NEXT: s_or_b64 exec, exec, s[12:13]7424; GFX9_DPP-NEXT: .LBB11_4: ; %Flow37425; GFX9_DPP-NEXT: s_or_b64 exec, exec, s[8:9]7426; GFX9_DPP-NEXT: v_readfirstlane_b32 s4, v77427; GFX9_DPP-NEXT: v_readfirstlane_b32 s5, v67428; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v17429; GFX9_DPP-NEXT: v_mov_b32_e32 v7, v27430; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s47431; GFX9_DPP-NEXT: v_sub_co_u32_e32 v6, vcc, s5, v67432; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0)7433; GFX9_DPP-NEXT: s_mov_b32 s3, 0xf0007434; GFX9_DPP-NEXT: s_mov_b32 s2, -17435; GFX9_DPP-NEXT: v_subb_co_u32_e32 v7, vcc, v0, v7, vcc7436; GFX9_DPP-NEXT: buffer_store_dwordx2 v[6:7], off, s[0:3], 07437; GFX9_DPP-NEXT: s_endpgm7438;7439; GFX1064_DPP-LABEL: sub_i64_varying:7440; GFX1064_DPP: ; %bb.0: ; %entry7441; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -17442; GFX1064_DPP-NEXT: v_mov_b32_e32 v1, 07443; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]7444; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s[0:1]7445; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 07446; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 07447; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 07448; GFX1064_DPP-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf7449; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, 07450; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:1 row_mask:0xf bank_mask:0xf7451; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v2, v17452; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v3, v4, vcc7453; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 07454; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:2 row_mask:0xf bank_mask:0xf7455; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 07456; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:2 row_mask:0xf bank_mask:0xf7457; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v67458; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc7459; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 07460; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:4 row_mask:0xf bank_mask:0xf7461; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 07462; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v2 row_shr:4 row_mask:0xf bank_mask:0xf7463; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v47464; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v3, vcc7465; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 07466; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:8 row_mask:0xf bank_mask:0xf7467; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:8 row_mask:0xf bank_mask:0xf7468; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v67469; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc7470; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 07471; GFX1064_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -17472; GFX1064_DPP-NEXT: v_permlanex16_b32 v6, v2, -1, -17473; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf7474; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf7475; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v37476; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc7477; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 07478; GFX1064_DPP-NEXT: v_readlane_b32 s2, v1, 317479; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 07480; GFX1064_DPP-NEXT: v_readlane_b32 s3, v2, 317481; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, s27482; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, s37483; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf7484; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf7485; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 07486; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v37487; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc7488; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1]7489; GFX1064_DPP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247490; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 07491; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[4:5], -17492; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:1 row_mask:0xf bank_mask:0xf7493; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v2 row_shr:1 row_mask:0xf bank_mask:0xf7494; GFX1064_DPP-NEXT: v_readlane_b32 s6, v2, 157495; GFX1064_DPP-NEXT: v_readlane_b32 s7, v1, 157496; GFX1064_DPP-NEXT: v_readlane_b32 s10, v2, 317497; GFX1064_DPP-NEXT: v_readlane_b32 s11, v1, 317498; GFX1064_DPP-NEXT: v_readlane_b32 s9, v2, 637499; GFX1064_DPP-NEXT: v_writelane_b32 v7, s6, 167500; GFX1064_DPP-NEXT: v_writelane_b32 v6, s7, 167501; GFX1064_DPP-NEXT: v_readlane_b32 s6, v2, 477502; GFX1064_DPP-NEXT: v_readlane_b32 s8, v1, 637503; GFX1064_DPP-NEXT: v_readlane_b32 s12, v1, 477504; GFX1064_DPP-NEXT: v_writelane_b32 v7, s10, 327505; GFX1064_DPP-NEXT: v_writelane_b32 v6, s11, 327506; GFX1064_DPP-NEXT: s_mov_b64 exec, s[4:5]7507; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v07508; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[4:5], -17509; GFX1064_DPP-NEXT: v_writelane_b32 v7, s6, 487510; GFX1064_DPP-NEXT: v_writelane_b32 v6, s12, 487511; GFX1064_DPP-NEXT: s_mov_b64 exec, s[4:5]7512; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v07513; GFX1064_DPP-NEXT: s_mov_b32 s6, -17514; GFX1064_DPP-NEXT: ; implicit-def: $vgpr8_vgpr97515; GFX1064_DPP-NEXT: s_and_saveexec_b64 s[10:11], vcc7516; GFX1064_DPP-NEXT: s_cbranch_execz .LBB11_47517; GFX1064_DPP-NEXT: ; %bb.1:7518; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)7519; GFX1064_DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x07520; GFX1064_DPP-NEXT: s_mov_b64 s[12:13], 07521; GFX1064_DPP-NEXT: s_mov_b32 s7, 0x310160007522; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)7523; GFX1064_DPP-NEXT: v_mov_b32_e32 v9, s57524; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, s47525; GFX1064_DPP-NEXT: s_mov_b32 s4, s27526; GFX1064_DPP-NEXT: s_mov_b32 s5, s37527; GFX1064_DPP-NEXT: .LBB11_2: ; %atomicrmw.start7528; GFX1064_DPP-NEXT: ; =>This Inner Loop Header: Depth=17529; GFX1064_DPP-NEXT: v_mov_b32_e32 v13, v97530; GFX1064_DPP-NEXT: v_mov_b32_e32 v12, v87531; GFX1064_DPP-NEXT: v_sub_co_u32 v10, vcc, v12, s87532; GFX1064_DPP-NEXT: v_subrev_co_ci_u32_e32 v11, vcc, s9, v13, vcc7533; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, v107534; GFX1064_DPP-NEXT: v_mov_b32_e32 v9, v117535; GFX1064_DPP-NEXT: v_mov_b32_e32 v10, v127536; GFX1064_DPP-NEXT: v_mov_b32_e32 v11, v137537; GFX1064_DPP-NEXT: buffer_atomic_cmpswap_x2 v[8:11], off, s[4:7], 0 glc7538; GFX1064_DPP-NEXT: s_waitcnt vmcnt(0)7539; GFX1064_DPP-NEXT: buffer_gl1_inv7540; GFX1064_DPP-NEXT: buffer_gl0_inv7541; GFX1064_DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[12:13]7542; GFX1064_DPP-NEXT: s_or_b64 s[12:13], vcc, s[12:13]7543; GFX1064_DPP-NEXT: s_andn2_b64 exec, exec, s[12:13]7544; GFX1064_DPP-NEXT: s_cbranch_execnz .LBB11_27545; GFX1064_DPP-NEXT: ; %bb.3: ; %Flow7546; GFX1064_DPP-NEXT: s_or_b64 exec, exec, s[12:13]7547; GFX1064_DPP-NEXT: .LBB11_4: ; %Flow37548; GFX1064_DPP-NEXT: s_or_b64 exec, exec, s[10:11]7549; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)7550; GFX1064_DPP-NEXT: v_readfirstlane_b32 s2, v87551; GFX1064_DPP-NEXT: v_mov_b32_e32 v10, v67552; GFX1064_DPP-NEXT: v_mov_b32_e32 v11, v77553; GFX1064_DPP-NEXT: v_readfirstlane_b32 s3, v97554; GFX1064_DPP-NEXT: v_sub_co_u32 v8, vcc, s2, v107555; GFX1064_DPP-NEXT: s_mov_b32 s2, -17556; GFX1064_DPP-NEXT: v_sub_co_ci_u32_e32 v9, vcc, s3, v11, vcc7557; GFX1064_DPP-NEXT: s_mov_b32 s3, 0x310160007558; GFX1064_DPP-NEXT: buffer_store_dwordx2 v[8:9], off, s[0:3], 07559; GFX1064_DPP-NEXT: s_endpgm7560;7561; GFX1032_DPP-LABEL: sub_i64_varying:7562; GFX1032_DPP: ; %bb.0: ; %entry7563; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -17564; GFX1032_DPP-NEXT: v_mov_b32_e32 v1, 07565; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s07566; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s07567; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 07568; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, 07569; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 07570; GFX1032_DPP-NEXT: v_mov_b32_dpp v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf7571; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, 07572; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v3 row_shr:1 row_mask:0xf bank_mask:0xf7573; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, 07574; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v2, v17575; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v3, v4, vcc_lo7576; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 07577; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:2 row_mask:0xf bank_mask:0xf7578; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 07579; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:2 row_mask:0xf bank_mask:0xf7580; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v67581; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo7582; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, 07583; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:4 row_mask:0xf bank_mask:0xf7584; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 07585; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v2 row_shr:4 row_mask:0xf bank_mask:0xf7586; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v47587; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v3, vcc_lo7588; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 07589; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v1 row_shr:8 row_mask:0xf bank_mask:0xf7590; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:8 row_mask:0xf bank_mask:0xf7591; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v67592; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo7593; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 07594; GFX1032_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -17595; GFX1032_DPP-NEXT: v_permlanex16_b32 v6, v2, -1, -17596; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf7597; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf7598; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s07599; GFX1032_DPP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x247600; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -17601; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v37602; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo7603; GFX1032_DPP-NEXT: v_readlane_b32 s8, v1, 317604; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf7605; GFX1032_DPP-NEXT: v_readlane_b32 s6, v2, 157606; GFX1032_DPP-NEXT: v_readlane_b32 s9, v2, 317607; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v2 row_shr:1 row_mask:0xf bank_mask:0xf7608; GFX1032_DPP-NEXT: v_readlane_b32 s5, v1, 157609; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s47610; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 07611; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -17612; GFX1032_DPP-NEXT: v_writelane_b32 v8, s6, 167613; GFX1032_DPP-NEXT: v_writelane_b32 v7, s5, 167614; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s47615; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v07616; GFX1032_DPP-NEXT: s_mov_b32 s6, -17617; GFX1032_DPP-NEXT: ; implicit-def: $vgpr9_vgpr107618; GFX1032_DPP-NEXT: s_and_saveexec_b32 s10, vcc_lo7619; GFX1032_DPP-NEXT: s_cbranch_execz .LBB11_47620; GFX1032_DPP-NEXT: ; %bb.1:7621; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)7622; GFX1032_DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x07623; GFX1032_DPP-NEXT: s_mov_b32 s11, 07624; GFX1032_DPP-NEXT: s_mov_b32 s7, 0x310160007625; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)7626; GFX1032_DPP-NEXT: v_mov_b32_e32 v10, s57627; GFX1032_DPP-NEXT: v_mov_b32_e32 v9, s47628; GFX1032_DPP-NEXT: s_mov_b32 s4, s27629; GFX1032_DPP-NEXT: s_mov_b32 s5, s37630; GFX1032_DPP-NEXT: .LBB11_2: ; %atomicrmw.start7631; GFX1032_DPP-NEXT: ; =>This Inner Loop Header: Depth=17632; GFX1032_DPP-NEXT: v_mov_b32_e32 v14, v107633; GFX1032_DPP-NEXT: v_mov_b32_e32 v13, v97634; GFX1032_DPP-NEXT: v_sub_co_u32 v11, vcc_lo, v13, s87635; GFX1032_DPP-NEXT: v_subrev_co_ci_u32_e32 v12, vcc_lo, s9, v14, vcc_lo7636; GFX1032_DPP-NEXT: v_mov_b32_e32 v9, v117637; GFX1032_DPP-NEXT: v_mov_b32_e32 v10, v127638; GFX1032_DPP-NEXT: v_mov_b32_e32 v11, v137639; GFX1032_DPP-NEXT: v_mov_b32_e32 v12, v147640; GFX1032_DPP-NEXT: buffer_atomic_cmpswap_x2 v[9:12], off, s[4:7], 0 glc7641; GFX1032_DPP-NEXT: s_waitcnt vmcnt(0)7642; GFX1032_DPP-NEXT: buffer_gl1_inv7643; GFX1032_DPP-NEXT: buffer_gl0_inv7644; GFX1032_DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[9:10], v[13:14]7645; GFX1032_DPP-NEXT: s_or_b32 s11, vcc_lo, s117646; GFX1032_DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s117647; GFX1032_DPP-NEXT: s_cbranch_execnz .LBB11_27648; GFX1032_DPP-NEXT: ; %bb.3: ; %Flow7649; GFX1032_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s117650; GFX1032_DPP-NEXT: .LBB11_4: ; %Flow37651; GFX1032_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s107652; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)7653; GFX1032_DPP-NEXT: v_readfirstlane_b32 s2, v97654; GFX1032_DPP-NEXT: v_mov_b32_e32 v11, v77655; GFX1032_DPP-NEXT: v_mov_b32_e32 v12, v87656; GFX1032_DPP-NEXT: v_readfirstlane_b32 s3, v107657; GFX1032_DPP-NEXT: v_sub_co_u32 v9, vcc_lo, s2, v117658; GFX1032_DPP-NEXT: s_mov_b32 s2, -17659; GFX1032_DPP-NEXT: v_sub_co_ci_u32_e32 v10, vcc_lo, s3, v12, vcc_lo7660; GFX1032_DPP-NEXT: s_mov_b32 s3, 0x310160007661; GFX1032_DPP-NEXT: buffer_store_dwordx2 v[9:10], off, s[0:3], 07662; GFX1032_DPP-NEXT: s_endpgm7663;7664; GFX1164_DPP-LABEL: sub_i64_varying:7665; GFX1164_DPP: ; %bb.0: ; %entry7666; GFX1164_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v07667; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -17668; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)7669; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1]7670; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 07671; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]7672; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 07673; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)7674; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf7675; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:17676; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)7677; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc7678; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:17679; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)7680; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 07681; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)7682; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf7683; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc7684; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:17685; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)7686; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 07687; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)7688; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf7689; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc7690; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:17691; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)7692; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 07693; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)7694; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf7695; GFX1164_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -17696; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)7697; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc7698; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf7699; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)7700; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, 07701; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)7702; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -17703; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf7704; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)7705; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc7706; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, 07707; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 317708; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)7709; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, s27710; GFX1164_DPP-NEXT: v_readlane_b32 s2, v2, 317711; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf7712; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)7713; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v2, s2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf7714; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)7715; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 07716; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc7717; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]7718; GFX1164_DPP-NEXT: s_load_b128 s[0:3], s[4:5], 0x247719; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 07720; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[4:5], -17721; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf7722; GFX1164_DPP-NEXT: v_readlane_b32 s6, v2, 157723; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf7724; GFX1164_DPP-NEXT: v_readlane_b32 s7, v1, 157725; GFX1164_DPP-NEXT: v_readlane_b32 s10, v2, 317726; GFX1164_DPP-NEXT: v_readlane_b32 s11, v2, 477727; GFX1164_DPP-NEXT: v_writelane_b32 v4, s6, 167728; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 317729; GFX1164_DPP-NEXT: v_writelane_b32 v5, s7, 167730; GFX1164_DPP-NEXT: v_readlane_b32 s8, v2, 637731; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 637732; GFX1164_DPP-NEXT: v_readlane_b32 s7, v1, 477733; GFX1164_DPP-NEXT: v_writelane_b32 v4, s10, 327734; GFX1164_DPP-NEXT: v_writelane_b32 v5, s6, 327735; GFX1164_DPP-NEXT: s_mov_b64 exec, s[4:5]7736; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7737; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v07738; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[4:5], -17739; GFX1164_DPP-NEXT: v_writelane_b32 v4, s11, 487740; GFX1164_DPP-NEXT: v_writelane_b32 v5, s7, 487741; GFX1164_DPP-NEXT: s_mov_b64 exec, s[4:5]7742; GFX1164_DPP-NEXT: s_mov_b32 s6, -17743; GFX1164_DPP-NEXT: s_mov_b64 s[10:11], exec7744; GFX1164_DPP-NEXT: ; implicit-def: $vgpr6_vgpr77745; GFX1164_DPP-NEXT: v_cmpx_eq_u32_e32 0, v07746; GFX1164_DPP-NEXT: s_cbranch_execz .LBB11_47747; GFX1164_DPP-NEXT: ; %bb.1:7748; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)7749; GFX1164_DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x07750; GFX1164_DPP-NEXT: s_mov_b64 s[12:13], 07751; GFX1164_DPP-NEXT: s_mov_b32 s7, 0x310160007752; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)7753; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, s57754; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, s47755; GFX1164_DPP-NEXT: s_mov_b32 s4, s27756; GFX1164_DPP-NEXT: s_mov_b32 s5, s37757; GFX1164_DPP-NEXT: .LBB11_2: ; %atomicrmw.start7758; GFX1164_DPP-NEXT: ; =>This Inner Loop Header: Depth=17759; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)7760; GFX1164_DPP-NEXT: v_mov_b32_e32 v11, v77761; GFX1164_DPP-NEXT: v_mov_b32_e32 v10, v67762; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)7763; GFX1164_DPP-NEXT: v_sub_co_u32 v8, vcc, v10, s87764; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)7765; GFX1164_DPP-NEXT: v_subrev_co_ci_u32_e64 v9, null, s9, v11, vcc7766; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)7767; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v87768; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, v97769; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, v107770; GFX1164_DPP-NEXT: v_mov_b32_e32 v9, v117771; GFX1164_DPP-NEXT: buffer_atomic_cmpswap_b64 v[6:9], off, s[4:7], 0 glc7772; GFX1164_DPP-NEXT: s_waitcnt vmcnt(0)7773; GFX1164_DPP-NEXT: buffer_gl1_inv7774; GFX1164_DPP-NEXT: buffer_gl0_inv7775; GFX1164_DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[10:11]7776; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)7777; GFX1164_DPP-NEXT: s_or_b64 s[12:13], vcc, s[12:13]7778; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7779; GFX1164_DPP-NEXT: s_and_not1_b64 exec, exec, s[12:13]7780; GFX1164_DPP-NEXT: s_cbranch_execnz .LBB11_27781; GFX1164_DPP-NEXT: ; %bb.3: ; %Flow7782; GFX1164_DPP-NEXT: s_or_b64 exec, exec, s[12:13]7783; GFX1164_DPP-NEXT: .LBB11_4: ; %Flow37784; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7785; GFX1164_DPP-NEXT: s_or_b64 exec, exec, s[10:11]7786; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)7787; GFX1164_DPP-NEXT: v_readfirstlane_b32 s2, v67788; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, v47789; GFX1164_DPP-NEXT: v_mov_b32_e32 v9, v57790; GFX1164_DPP-NEXT: v_readfirstlane_b32 s3, v77791; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)7792; GFX1164_DPP-NEXT: v_sub_co_u32 v6, vcc, s2, v87793; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)7794; GFX1164_DPP-NEXT: s_mov_b32 s2, -17795; GFX1164_DPP-NEXT: v_sub_co_ci_u32_e64 v7, null, s3, v9, vcc7796; GFX1164_DPP-NEXT: s_mov_b32 s3, 0x310160007797; GFX1164_DPP-NEXT: buffer_store_b64 v[6:7], off, s[0:3], 07798; GFX1164_DPP-NEXT: s_endpgm7799;7800; GFX1132_DPP-LABEL: sub_i64_varying:7801; GFX1132_DPP: ; %bb.0: ; %entry7802; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v07803; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -17804; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)7805; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s07806; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, 07807; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s07808; GFX1132_DPP-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v7, 07809; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, 07810; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)7811; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf7812; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:17813; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)7814; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc_lo7815; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:17816; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, 07817; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)7818; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf7819; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo7820; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)7821; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:17822; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, 07823; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf7824; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)7825; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc_lo7826; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:17827; GFX1132_DPP-NEXT: v_mov_b32_e32 v3, 07828; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)7829; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf7830; GFX1132_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -17831; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)7832; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo7833; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf7834; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)7835; GFX1132_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -17836; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf7837; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s07838; GFX1132_DPP-NEXT: s_load_b128 s[0:3], s[4:5], 0x247839; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -17840; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)7841; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo7842; GFX1132_DPP-NEXT: v_readlane_b32 s8, v2, 317843; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf7844; GFX1132_DPP-NEXT: v_readlane_b32 s5, v2, 157845; GFX1132_DPP-NEXT: v_readlane_b32 s6, v1, 157846; GFX1132_DPP-NEXT: v_readlane_b32 s9, v1, 317847; GFX1132_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf7848; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s47849; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7850; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 07851; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -17852; GFX1132_DPP-NEXT: v_writelane_b32 v6, s5, 167853; GFX1132_DPP-NEXT: v_writelane_b32 v7, s6, 167854; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s47855; GFX1132_DPP-NEXT: s_mov_b32 s6, -17856; GFX1132_DPP-NEXT: s_mov_b32 s10, exec_lo7857; GFX1132_DPP-NEXT: ; implicit-def: $vgpr8_vgpr97858; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v07859; GFX1132_DPP-NEXT: s_cbranch_execz .LBB11_47860; GFX1132_DPP-NEXT: ; %bb.1:7861; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)7862; GFX1132_DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x07863; GFX1132_DPP-NEXT: s_mov_b32 s11, 07864; GFX1132_DPP-NEXT: s_mov_b32 s7, 0x310160007865; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)7866; GFX1132_DPP-NEXT: v_dual_mov_b32 v9, s5 :: v_dual_mov_b32 v8, s47867; GFX1132_DPP-NEXT: s_mov_b32 s4, s27868; GFX1132_DPP-NEXT: s_mov_b32 s5, s37869; GFX1132_DPP-NEXT: .LBB11_2: ; %atomicrmw.start7870; GFX1132_DPP-NEXT: ; =>This Inner Loop Header: Depth=17871; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7872; GFX1132_DPP-NEXT: v_dual_mov_b32 v13, v9 :: v_dual_mov_b32 v12, v87873; GFX1132_DPP-NEXT: v_sub_co_u32 v10, vcc_lo, v12, s87874; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)7875; GFX1132_DPP-NEXT: v_subrev_co_ci_u32_e64 v11, null, s9, v13, vcc_lo7876; GFX1132_DPP-NEXT: v_mov_b32_e32 v8, v107877; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)7878; GFX1132_DPP-NEXT: v_dual_mov_b32 v9, v11 :: v_dual_mov_b32 v10, v127879; GFX1132_DPP-NEXT: v_mov_b32_e32 v11, v137880; GFX1132_DPP-NEXT: buffer_atomic_cmpswap_b64 v[8:11], off, s[4:7], 0 glc7881; GFX1132_DPP-NEXT: s_waitcnt vmcnt(0)7882; GFX1132_DPP-NEXT: buffer_gl1_inv7883; GFX1132_DPP-NEXT: buffer_gl0_inv7884; GFX1132_DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[8:9], v[12:13]7885; GFX1132_DPP-NEXT: s_or_b32 s11, vcc_lo, s117886; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7887; GFX1132_DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s117888; GFX1132_DPP-NEXT: s_cbranch_execnz .LBB11_27889; GFX1132_DPP-NEXT: ; %bb.3: ; %Flow7890; GFX1132_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s117891; GFX1132_DPP-NEXT: .LBB11_4: ; %Flow37892; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7893; GFX1132_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s107894; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)7895; GFX1132_DPP-NEXT: v_readfirstlane_b32 s2, v87896; GFX1132_DPP-NEXT: v_mov_b32_e32 v10, v67897; GFX1132_DPP-NEXT: v_mov_b32_e32 v11, v77898; GFX1132_DPP-NEXT: v_readfirstlane_b32 s3, v97899; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)7900; GFX1132_DPP-NEXT: v_sub_co_u32 v8, vcc_lo, s2, v107901; GFX1132_DPP-NEXT: s_mov_b32 s2, -17902; GFX1132_DPP-NEXT: v_sub_co_ci_u32_e64 v9, null, s3, v11, vcc_lo7903; GFX1132_DPP-NEXT: s_mov_b32 s3, 0x310160007904; GFX1132_DPP-NEXT: buffer_store_b64 v[8:9], off, s[0:3], 07905; GFX1132_DPP-NEXT: s_endpgm7906;7907; GFX1264_DPP-LABEL: sub_i64_varying:7908; GFX1264_DPP: ; %bb.0: ; %entry7909; GFX1264_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v07910; GFX1264_DPP-NEXT: s_or_saveexec_b64 s[0:1], -17911; GFX1264_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)7912; GFX1264_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1]7913; GFX1264_DPP-NEXT: v_mov_b32_e32 v2, 07914; GFX1264_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]7915; GFX1264_DPP-NEXT: v_mov_b32_e32 v4, 07916; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)7917; GFX1264_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf7918; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:17919; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)7920; GFX1264_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc7921; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:17922; GFX1264_DPP-NEXT: v_mov_b32_e32 v2, 07923; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)7924; GFX1264_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf7925; GFX1264_DPP-NEXT: s_wait_alu depctr_va_vcc(0)7926; GFX1264_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc7927; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:17928; GFX1264_DPP-NEXT: v_mov_b32_e32 v4, 07929; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)7930; GFX1264_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf7931; GFX1264_DPP-NEXT: s_wait_alu depctr_va_vcc(0)7932; GFX1264_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc7933; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:17934; GFX1264_DPP-NEXT: v_mov_b32_e32 v3, 07935; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)7936; GFX1264_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf7937; GFX1264_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -17938; GFX1264_DPP-NEXT: s_wait_alu depctr_va_vcc(0)7939; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)7940; GFX1264_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc7941; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf7942; GFX1264_DPP-NEXT: v_mov_b32_e32 v5, 07943; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)7944; GFX1264_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -17945; GFX1264_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf7946; GFX1264_DPP-NEXT: s_wait_alu depctr_va_vcc(0)7947; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)7948; GFX1264_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc7949; GFX1264_DPP-NEXT: v_mov_b32_e32 v3, 07950; GFX1264_DPP-NEXT: v_readlane_b32 s2, v1, 317951; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)7952; GFX1264_DPP-NEXT: v_mov_b32_e32 v4, s27953; GFX1264_DPP-NEXT: v_readlane_b32 s2, v2, 317954; GFX1264_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf7955; GFX1264_DPP-NEXT: s_wait_alu depctr_va_sdst(0)7956; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)7957; GFX1264_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v2, s2 quad_perm:[0,1,2,3] row_mask:0xc bank_mask:0xf7958; GFX1264_DPP-NEXT: v_mov_b32_e32 v4, 07959; GFX1264_DPP-NEXT: s_wait_alu depctr_va_vcc(0)7960; GFX1264_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc7961; GFX1264_DPP-NEXT: s_mov_b64 exec, s[0:1]7962; GFX1264_DPP-NEXT: s_load_b128 s[0:3], s[4:5], 0x247963; GFX1264_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 07964; GFX1264_DPP-NEXT: s_or_saveexec_b64 s[4:5], -17965; GFX1264_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf7966; GFX1264_DPP-NEXT: v_readlane_b32 s6, v2, 157967; GFX1264_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf7968; GFX1264_DPP-NEXT: v_readlane_b32 s7, v1, 157969; GFX1264_DPP-NEXT: v_readlane_b32 s8, v2, 317970; GFX1264_DPP-NEXT: v_readlane_b32 s9, v1, 317971; GFX1264_DPP-NEXT: v_writelane_b32 v4, s6, 167972; GFX1264_DPP-NEXT: v_readlane_b32 s6, v2, 637973; GFX1264_DPP-NEXT: v_writelane_b32 v5, s7, 167974; GFX1264_DPP-NEXT: v_readlane_b32 s10, v2, 477975; GFX1264_DPP-NEXT: v_readlane_b32 s11, v1, 477976; GFX1264_DPP-NEXT: v_readlane_b32 s7, v1, 637977; GFX1264_DPP-NEXT: v_writelane_b32 v4, s8, 327978; GFX1264_DPP-NEXT: v_writelane_b32 v5, s9, 327979; GFX1264_DPP-NEXT: s_mov_b64 exec, s[4:5]7980; GFX1264_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7981; GFX1264_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v07982; GFX1264_DPP-NEXT: s_or_saveexec_b64 s[8:9], -17983; GFX1264_DPP-NEXT: s_mov_b64 s[4:5], s[6:7]7984; GFX1264_DPP-NEXT: v_writelane_b32 v4, s10, 487985; GFX1264_DPP-NEXT: v_writelane_b32 v5, s11, 487986; GFX1264_DPP-NEXT: s_wait_alu depctr_sa_sdst(0)7987; GFX1264_DPP-NEXT: s_mov_b64 exec, s[8:9]7988; GFX1264_DPP-NEXT: s_mov_b32 s6, -17989; GFX1264_DPP-NEXT: s_mov_b64 s[8:9], exec7990; GFX1264_DPP-NEXT: ; implicit-def: $vgpr6_vgpr77991; GFX1264_DPP-NEXT: v_cmpx_eq_u32_e32 0, v07992; GFX1264_DPP-NEXT: s_cbranch_execz .LBB11_27993; GFX1264_DPP-NEXT: ; %bb.1:7994; GFX1264_DPP-NEXT: v_mov_b32_e32 v7, s57995; GFX1264_DPP-NEXT: v_mov_b32_e32 v6, s47996; GFX1264_DPP-NEXT: s_mov_b32 s7, 0x310160007997; GFX1264_DPP-NEXT: s_wait_kmcnt 0x07998; GFX1264_DPP-NEXT: s_mov_b32 s4, s27999; GFX1264_DPP-NEXT: s_mov_b32 s5, s38000; GFX1264_DPP-NEXT: buffer_atomic_sub_u64 v[6:7], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV8001; GFX1264_DPP-NEXT: s_wait_loadcnt 0x08002; GFX1264_DPP-NEXT: global_inv scope:SCOPE_DEV8003; GFX1264_DPP-NEXT: .LBB11_2:8004; GFX1264_DPP-NEXT: s_wait_alu depctr_sa_sdst(0)8005; GFX1264_DPP-NEXT: s_or_b64 exec, exec, s[8:9]8006; GFX1264_DPP-NEXT: s_wait_kmcnt 0x08007; GFX1264_DPP-NEXT: v_readfirstlane_b32 s2, v68008; GFX1264_DPP-NEXT: v_mov_b32_e32 v8, v48009; GFX1264_DPP-NEXT: v_mov_b32_e32 v9, v58010; GFX1264_DPP-NEXT: v_readfirstlane_b32 s3, v78011; GFX1264_DPP-NEXT: s_wait_alu depctr_va_sdst(0)8012; GFX1264_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)8013; GFX1264_DPP-NEXT: v_sub_co_u32 v6, vcc, s2, v88014; GFX1264_DPP-NEXT: s_mov_b32 s2, s68015; GFX1264_DPP-NEXT: s_wait_alu depctr_va_vcc(0)8016; GFX1264_DPP-NEXT: v_sub_co_ci_u32_e64 v7, null, s3, v9, vcc8017; GFX1264_DPP-NEXT: s_mov_b32 s3, 0x310160008018; GFX1264_DPP-NEXT: buffer_store_b64 v[6:7], off, s[0:3], null8019; GFX1264_DPP-NEXT: s_endpgm8020;8021; GFX1232_DPP-LABEL: sub_i64_varying:8022; GFX1232_DPP: ; %bb.0: ; %entry8023; GFX1232_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v08024; GFX1232_DPP-NEXT: s_or_saveexec_b32 s0, -18025; GFX1232_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)8026; GFX1232_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s08027; GFX1232_DPP-NEXT: v_mov_b32_e32 v2, 08028; GFX1232_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s08029; GFX1232_DPP-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v7, 08030; GFX1232_DPP-NEXT: v_mov_b32_e32 v6, 08031; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)8032; GFX1232_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf8033; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:18034; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)8035; GFX1232_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc_lo8036; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:18037; GFX1232_DPP-NEXT: v_mov_b32_e32 v2, 08038; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)8039; GFX1232_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:2 row_mask:0xf bank_mask:0xf8040; GFX1232_DPP-NEXT: s_wait_alu depctr_va_vcc(0)8041; GFX1232_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo8042; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:18043; GFX1232_DPP-NEXT: v_mov_b32_e32 v4, 08044; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)8045; GFX1232_DPP-NEXT: v_mov_b32_dpp v2, v1 row_shr:4 row_mask:0xf bank_mask:0xf8046; GFX1232_DPP-NEXT: s_wait_alu depctr_va_vcc(0)8047; GFX1232_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc_lo8048; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:18049; GFX1232_DPP-NEXT: v_mov_b32_e32 v3, 08050; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)8051; GFX1232_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:8 row_mask:0xf bank_mask:0xf8052; GFX1232_DPP-NEXT: v_permlanex16_b32 v5, v2, -1, -18053; GFX1232_DPP-NEXT: s_wait_alu depctr_va_vcc(0)8054; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)8055; GFX1232_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo8056; GFX1232_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v5, v2 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf8057; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)8058; GFX1232_DPP-NEXT: v_permlanex16_b32 v4, v1, -1, -18059; GFX1232_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf8060; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s08061; GFX1232_DPP-NEXT: s_load_b128 s[0:3], s[4:5], 0x248062; GFX1232_DPP-NEXT: s_or_saveexec_b32 s6, -18063; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)8064; GFX1232_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo8065; GFX1232_DPP-NEXT: v_readlane_b32 s4, v2, 318066; GFX1232_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf8067; GFX1232_DPP-NEXT: v_readlane_b32 s7, v2, 158068; GFX1232_DPP-NEXT: v_readlane_b32 s8, v1, 158069; GFX1232_DPP-NEXT: v_readlane_b32 s5, v1, 318070; GFX1232_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf8071; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s68072; GFX1232_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8073; GFX1232_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 08074; GFX1232_DPP-NEXT: s_or_saveexec_b32 s6, -18075; GFX1232_DPP-NEXT: v_writelane_b32 v6, s7, 168076; GFX1232_DPP-NEXT: v_writelane_b32 v7, s8, 168077; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s68078; GFX1232_DPP-NEXT: s_mov_b32 s6, -18079; GFX1232_DPP-NEXT: s_mov_b32 s8, exec_lo8080; GFX1232_DPP-NEXT: ; implicit-def: $vgpr8_vgpr98081; GFX1232_DPP-NEXT: v_cmpx_eq_u32_e32 0, v08082; GFX1232_DPP-NEXT: s_cbranch_execz .LBB11_28083; GFX1232_DPP-NEXT: ; %bb.1:8084; GFX1232_DPP-NEXT: v_dual_mov_b32 v9, s5 :: v_dual_mov_b32 v8, s48085; GFX1232_DPP-NEXT: s_mov_b32 s7, 0x310160008086; GFX1232_DPP-NEXT: s_wait_kmcnt 0x08087; GFX1232_DPP-NEXT: s_mov_b32 s4, s28088; GFX1232_DPP-NEXT: s_mov_b32 s5, s38089; GFX1232_DPP-NEXT: buffer_atomic_sub_u64 v[8:9], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV8090; GFX1232_DPP-NEXT: s_wait_loadcnt 0x08091; GFX1232_DPP-NEXT: global_inv scope:SCOPE_DEV8092; GFX1232_DPP-NEXT: .LBB11_2:8093; GFX1232_DPP-NEXT: s_wait_alu depctr_sa_sdst(0)8094; GFX1232_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s88095; GFX1232_DPP-NEXT: s_wait_kmcnt 0x08096; GFX1232_DPP-NEXT: v_readfirstlane_b32 s2, v88097; GFX1232_DPP-NEXT: v_mov_b32_e32 v10, v68098; GFX1232_DPP-NEXT: v_mov_b32_e32 v11, v78099; GFX1232_DPP-NEXT: v_readfirstlane_b32 s3, v98100; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)8101; GFX1232_DPP-NEXT: v_sub_co_u32 v8, vcc_lo, s2, v108102; GFX1232_DPP-NEXT: s_mov_b32 s2, s68103; GFX1232_DPP-NEXT: s_wait_alu depctr_va_vcc(0)8104; GFX1232_DPP-NEXT: v_sub_co_ci_u32_e64 v9, null, s3, v11, vcc_lo8105; GFX1232_DPP-NEXT: s_mov_b32 s3, 0x310160008106; GFX1232_DPP-NEXT: buffer_store_b64 v[8:9], off, s[0:3], null8107; GFX1232_DPP-NEXT: s_endpgm8108entry:8109 %lane = call i32 @llvm.amdgcn.workitem.id.x()8110 %zext = zext i32 %lane to i648111 %old = atomicrmw sub ptr addrspace(1) %inout, i64 %zext syncscope("agent") acq_rel8112 store i64 %old, ptr addrspace(1) %out8113 ret void8114}8115 8116define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i8 %val) {8117; GFX7LESS-LABEL: uniform_or_i8:8118; GFX7LESS: ; %bb.0:8119; GFX7LESS-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x98120; GFX7LESS-NEXT: s_load_dword s12, s[4:5], 0xd8121; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 08122; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v08123; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v08124; GFX7LESS-NEXT: ; implicit-def: $vgpr08125; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc8126; GFX7LESS-NEXT: s_cbranch_execz .LBB12_48127; GFX7LESS-NEXT: ; %bb.1:8128; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)8129; GFX7LESS-NEXT: s_and_b32 s4, s10, -48130; GFX7LESS-NEXT: s_mov_b32 s5, s118131; GFX7LESS-NEXT: s_and_b32 s0, s10, 38132; GFX7LESS-NEXT: s_and_b32 s1, s12, 0xff8133; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0x08134; GFX7LESS-NEXT: s_mov_b64 s[10:11], 08135; GFX7LESS-NEXT: s_mov_b32 s7, 0xf0008136; GFX7LESS-NEXT: s_lshl_b32 s13, s0, 38137; GFX7LESS-NEXT: s_lshl_b32 s14, s1, s138138; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)8139; GFX7LESS-NEXT: v_mov_b32_e32 v1, s68140; GFX7LESS-NEXT: s_mov_b32 s6, -18141; GFX7LESS-NEXT: .LBB12_2: ; %atomicrmw.start8142; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=18143; GFX7LESS-NEXT: v_or_b32_e32 v0, s14, v18144; GFX7LESS-NEXT: s_waitcnt expcnt(0)8145; GFX7LESS-NEXT: v_mov_b32_e32 v3, v18146; GFX7LESS-NEXT: v_mov_b32_e32 v2, v08147; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc8148; GFX7LESS-NEXT: s_waitcnt vmcnt(0)8149; GFX7LESS-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v18150; GFX7LESS-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]8151; GFX7LESS-NEXT: v_mov_b32_e32 v1, v28152; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[10:11]8153; GFX7LESS-NEXT: s_cbranch_execnz .LBB12_28154; GFX7LESS-NEXT: ; %bb.3: ; %atomicrmw.end8155; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]8156; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s13, v28157; GFX7LESS-NEXT: .LBB12_4: ; %Flow8158; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3]8159; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)8160; GFX7LESS-NEXT: s_mov_b32 s11, 0xf0008161; GFX7LESS-NEXT: s_mov_b32 s10, -18162; GFX7LESS-NEXT: v_and_b32_e32 v0, 0xff, v08163; GFX7LESS-NEXT: v_mov_b32_e32 v1, s128164; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v08165; GFX7LESS-NEXT: v_cndmask_b32_e64 v0, v1, 0, vcc8166; GFX7LESS-NEXT: v_or_b32_e32 v0, s0, v08167; GFX7LESS-NEXT: buffer_store_byte v0, off, s[8:11], 08168; GFX7LESS-NEXT: s_endpgm8169;8170; GFX8-LABEL: uniform_or_i8:8171; GFX8: ; %bb.0:8172; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x248173; GFX8-NEXT: s_load_dword s12, s[4:5], 0x348174; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 08175; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v08176; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v08177; GFX8-NEXT: ; implicit-def: $vgpr08178; GFX8-NEXT: s_and_saveexec_b64 s[2:3], vcc8179; GFX8-NEXT: s_cbranch_execz .LBB12_48180; GFX8-NEXT: ; %bb.1:8181; GFX8-NEXT: s_waitcnt lgkmcnt(0)8182; GFX8-NEXT: s_and_b32 s4, s10, -48183; GFX8-NEXT: s_mov_b32 s5, s118184; GFX8-NEXT: s_load_dword s1, s[4:5], 0x08185; GFX8-NEXT: s_and_b32 s0, s10, 38186; GFX8-NEXT: s_lshl_b32 s13, s0, 38187; GFX8-NEXT: s_and_b32 s0, s12, 0xff8188; GFX8-NEXT: s_lshl_b32 s14, s0, s138189; GFX8-NEXT: s_mov_b64 s[10:11], 08190; GFX8-NEXT: s_waitcnt lgkmcnt(0)8191; GFX8-NEXT: v_mov_b32_e32 v1, s18192; GFX8-NEXT: s_mov_b32 s7, 0xf0008193; GFX8-NEXT: s_mov_b32 s6, -18194; GFX8-NEXT: .LBB12_2: ; %atomicrmw.start8195; GFX8-NEXT: ; =>This Inner Loop Header: Depth=18196; GFX8-NEXT: v_or_b32_e32 v0, s14, v18197; GFX8-NEXT: v_mov_b32_e32 v3, v18198; GFX8-NEXT: v_mov_b32_e32 v2, v08199; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc8200; GFX8-NEXT: s_waitcnt vmcnt(0)8201; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v18202; GFX8-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]8203; GFX8-NEXT: v_mov_b32_e32 v1, v28204; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]8205; GFX8-NEXT: s_cbranch_execnz .LBB12_28206; GFX8-NEXT: ; %bb.3: ; %atomicrmw.end8207; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]8208; GFX8-NEXT: v_lshrrev_b32_e32 v0, s13, v28209; GFX8-NEXT: .LBB12_4: ; %Flow8210; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]8211; GFX8-NEXT: v_and_b32_e32 v0, 0xff, v08212; GFX8-NEXT: v_readfirstlane_b32 s0, v08213; GFX8-NEXT: s_waitcnt lgkmcnt(0)8214; GFX8-NEXT: v_mov_b32_e32 v0, s128215; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc8216; GFX8-NEXT: s_mov_b32 s11, 0xf0008217; GFX8-NEXT: s_mov_b32 s10, -18218; GFX8-NEXT: v_or_b32_e32 v0, s0, v08219; GFX8-NEXT: buffer_store_byte v0, off, s[8:11], 08220; GFX8-NEXT: s_endpgm8221;8222; GFX9-LABEL: uniform_or_i8:8223; GFX9: ; %bb.0:8224; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x248225; GFX9-NEXT: s_load_dword s12, s[4:5], 0x348226; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 08227; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v08228; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v08229; GFX9-NEXT: ; implicit-def: $vgpr08230; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc8231; GFX9-NEXT: s_cbranch_execz .LBB12_48232; GFX9-NEXT: ; %bb.1:8233; GFX9-NEXT: s_waitcnt lgkmcnt(0)8234; GFX9-NEXT: s_and_b32 s4, s10, -48235; GFX9-NEXT: s_mov_b32 s5, s118236; GFX9-NEXT: s_load_dword s1, s[4:5], 0x08237; GFX9-NEXT: s_and_b32 s0, s10, 38238; GFX9-NEXT: s_lshl_b32 s13, s0, 38239; GFX9-NEXT: s_and_b32 s0, s12, 0xff8240; GFX9-NEXT: s_lshl_b32 s14, s0, s138241; GFX9-NEXT: s_mov_b64 s[10:11], 08242; GFX9-NEXT: s_waitcnt lgkmcnt(0)8243; GFX9-NEXT: v_mov_b32_e32 v1, s18244; GFX9-NEXT: s_mov_b32 s7, 0xf0008245; GFX9-NEXT: s_mov_b32 s6, -18246; GFX9-NEXT: .LBB12_2: ; %atomicrmw.start8247; GFX9-NEXT: ; =>This Inner Loop Header: Depth=18248; GFX9-NEXT: v_or_b32_e32 v0, s14, v18249; GFX9-NEXT: v_mov_b32_e32 v3, v18250; GFX9-NEXT: v_mov_b32_e32 v2, v08251; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc8252; GFX9-NEXT: s_waitcnt vmcnt(0)8253; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v18254; GFX9-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]8255; GFX9-NEXT: v_mov_b32_e32 v1, v28256; GFX9-NEXT: s_andn2_b64 exec, exec, s[10:11]8257; GFX9-NEXT: s_cbranch_execnz .LBB12_28258; GFX9-NEXT: ; %bb.3: ; %atomicrmw.end8259; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]8260; GFX9-NEXT: v_lshrrev_b32_e32 v0, s13, v28261; GFX9-NEXT: .LBB12_4: ; %Flow8262; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]8263; GFX9-NEXT: v_and_b32_e32 v0, 0xff, v08264; GFX9-NEXT: v_readfirstlane_b32 s0, v08265; GFX9-NEXT: s_waitcnt lgkmcnt(0)8266; GFX9-NEXT: v_mov_b32_e32 v0, s128267; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc8268; GFX9-NEXT: s_mov_b32 s11, 0xf0008269; GFX9-NEXT: s_mov_b32 s10, -18270; GFX9-NEXT: v_or_b32_e32 v0, s0, v08271; GFX9-NEXT: buffer_store_byte v0, off, s[8:11], 08272; GFX9-NEXT: s_endpgm8273;8274; GFX1064-LABEL: uniform_or_i8:8275; GFX1064: ; %bb.0:8276; GFX1064-NEXT: s_clause 0x18277; GFX1064-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x248278; GFX1064-NEXT: s_load_dword s12, s[4:5], 0x348279; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 08280; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v08281; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v08282; GFX1064-NEXT: ; implicit-def: $vgpr08283; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc8284; GFX1064-NEXT: s_cbranch_execz .LBB12_48285; GFX1064-NEXT: ; %bb.1:8286; GFX1064-NEXT: s_waitcnt lgkmcnt(0)8287; GFX1064-NEXT: s_and_b32 s4, s10, -48288; GFX1064-NEXT: s_mov_b32 s5, s118289; GFX1064-NEXT: s_and_b32 s1, s10, 38290; GFX1064-NEXT: s_load_dword s0, s[4:5], 0x08291; GFX1064-NEXT: s_lshl_b32 s13, s1, 38292; GFX1064-NEXT: s_and_b32 s1, s12, 0xff8293; GFX1064-NEXT: s_mov_b64 s[10:11], 08294; GFX1064-NEXT: s_lshl_b32 s14, s1, s138295; GFX1064-NEXT: s_mov_b32 s7, 0x310160008296; GFX1064-NEXT: s_mov_b32 s6, -18297; GFX1064-NEXT: s_waitcnt lgkmcnt(0)8298; GFX1064-NEXT: v_mov_b32_e32 v1, s08299; GFX1064-NEXT: .LBB12_2: ; %atomicrmw.start8300; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=18301; GFX1064-NEXT: v_or_b32_e32 v0, s14, v18302; GFX1064-NEXT: v_mov_b32_e32 v3, v18303; GFX1064-NEXT: v_mov_b32_e32 v2, v08304; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc8305; GFX1064-NEXT: s_waitcnt vmcnt(0)8306; GFX1064-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v18307; GFX1064-NEXT: v_mov_b32_e32 v1, v28308; GFX1064-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]8309; GFX1064-NEXT: s_andn2_b64 exec, exec, s[10:11]8310; GFX1064-NEXT: s_cbranch_execnz .LBB12_28311; GFX1064-NEXT: ; %bb.3: ; %atomicrmw.end8312; GFX1064-NEXT: s_or_b64 exec, exec, s[10:11]8313; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s13, v28314; GFX1064-NEXT: .LBB12_4: ; %Flow8315; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]8316; GFX1064-NEXT: v_and_b32_e32 v0, 0xff, v08317; GFX1064-NEXT: s_waitcnt lgkmcnt(0)8318; GFX1064-NEXT: s_mov_b32 s11, 0x310160008319; GFX1064-NEXT: s_mov_b32 s10, -18320; GFX1064-NEXT: v_readfirstlane_b32 s0, v08321; GFX1064-NEXT: v_cndmask_b32_e64 v0, s12, 0, vcc8322; GFX1064-NEXT: v_or_b32_e32 v0, s0, v08323; GFX1064-NEXT: buffer_store_byte v0, off, s[8:11], 08324; GFX1064-NEXT: s_endpgm8325;8326; GFX1032-LABEL: uniform_or_i8:8327; GFX1032: ; %bb.0:8328; GFX1032-NEXT: s_clause 0x18329; GFX1032-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x248330; GFX1032-NEXT: s_load_dword s1, s[4:5], 0x348331; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 08332; GFX1032-NEXT: s_mov_b32 s3, 08333; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v08334; GFX1032-NEXT: ; implicit-def: $vgpr08335; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo8336; GFX1032-NEXT: s_cbranch_execz .LBB12_48337; GFX1032-NEXT: ; %bb.1:8338; GFX1032-NEXT: s_waitcnt lgkmcnt(0)8339; GFX1032-NEXT: s_and_b32 s4, s10, -48340; GFX1032-NEXT: s_mov_b32 s5, s118341; GFX1032-NEXT: s_and_b32 s6, s10, 38342; GFX1032-NEXT: s_load_dword s0, s[4:5], 0x08343; GFX1032-NEXT: s_lshl_b32 s10, s6, 38344; GFX1032-NEXT: s_and_b32 s6, s1, 0xff8345; GFX1032-NEXT: s_mov_b32 s7, 0x310160008346; GFX1032-NEXT: s_lshl_b32 s11, s6, s108347; GFX1032-NEXT: s_mov_b32 s6, -18348; GFX1032-NEXT: s_waitcnt lgkmcnt(0)8349; GFX1032-NEXT: v_mov_b32_e32 v1, s08350; GFX1032-NEXT: .LBB12_2: ; %atomicrmw.start8351; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=18352; GFX1032-NEXT: v_or_b32_e32 v0, s11, v18353; GFX1032-NEXT: v_mov_b32_e32 v3, v18354; GFX1032-NEXT: v_mov_b32_e32 v2, v08355; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc8356; GFX1032-NEXT: s_waitcnt vmcnt(0)8357; GFX1032-NEXT: v_cmp_eq_u32_e64 s0, v2, v18358; GFX1032-NEXT: v_mov_b32_e32 v1, v28359; GFX1032-NEXT: s_or_b32 s3, s0, s38360; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s38361; GFX1032-NEXT: s_cbranch_execnz .LBB12_28362; GFX1032-NEXT: ; %bb.3: ; %atomicrmw.end8363; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s38364; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s10, v28365; GFX1032-NEXT: .LBB12_4: ; %Flow8366; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s28367; GFX1032-NEXT: v_and_b32_e32 v0, 0xff, v08368; GFX1032-NEXT: s_waitcnt lgkmcnt(0)8369; GFX1032-NEXT: s_mov_b32 s11, 0x310160008370; GFX1032-NEXT: s_mov_b32 s10, -18371; GFX1032-NEXT: v_readfirstlane_b32 s0, v08372; GFX1032-NEXT: v_cndmask_b32_e64 v0, s1, 0, vcc_lo8373; GFX1032-NEXT: v_or_b32_e32 v0, s0, v08374; GFX1032-NEXT: buffer_store_byte v0, off, s[8:11], 08375; GFX1032-NEXT: s_endpgm8376;8377; GFX1164-TRUE16-LABEL: uniform_or_i8:8378; GFX1164-TRUE16: ; %bb.0:8379; GFX1164-TRUE16-NEXT: s_clause 0x18380; GFX1164-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x248381; GFX1164-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x348382; GFX1164-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 08383; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8384; GFX1164-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v08385; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, 0, v08386; GFX1164-TRUE16-NEXT: ; implicit-def: $vgpr0_lo168387; GFX1164-TRUE16-NEXT: s_and_saveexec_b64 s[2:3], vcc8388; GFX1164-TRUE16-NEXT: s_cbranch_execz .LBB12_48389; GFX1164-TRUE16-NEXT: ; %bb.1:8390; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)8391; GFX1164-TRUE16-NEXT: s_and_b32 s4, s10, -48392; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s118393; GFX1164-TRUE16-NEXT: s_and_b32 s1, s10, 38394; GFX1164-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x08395; GFX1164-TRUE16-NEXT: s_lshl_b32 s13, s1, 38396; GFX1164-TRUE16-NEXT: s_and_b32 s1, s12, 0xff8397; GFX1164-TRUE16-NEXT: s_mov_b64 s[10:11], 08398; GFX1164-TRUE16-NEXT: s_lshl_b32 s14, s1, s138399; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x310160008400; GFX1164-TRUE16-NEXT: s_mov_b32 s6, -18401; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)8402; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s08403; GFX1164-TRUE16-NEXT: .LBB12_2: ; %atomicrmw.start8404; GFX1164-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=18405; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)8406; GFX1164-TRUE16-NEXT: v_or_b32_e32 v0, s14, v18407; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v3, v18408; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v2, v08409; GFX1164-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc8410; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)8411; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v18412; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v28413; GFX1164-TRUE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]8414; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8415; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]8416; GFX1164-TRUE16-NEXT: s_cbranch_execnz .LBB12_28417; GFX1164-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end8418; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[10:11]8419; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s13, v28420; GFX1164-TRUE16-NEXT: .LBB12_4: ; %Flow8421; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]8422; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)8423; GFX1164-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v08424; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)8425; GFX1164-TRUE16-NEXT: s_mov_b32 s11, 0x310160008426; GFX1164-TRUE16-NEXT: s_mov_b32 s10, -18427; GFX1164-TRUE16-NEXT: v_readfirstlane_b32 s0, v08428; GFX1164-TRUE16-NEXT: v_cndmask_b16 v0.l, s12, 0, vcc8429; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)8430; GFX1164-TRUE16-NEXT: v_or_b16 v0.l, s0, v0.l8431; GFX1164-TRUE16-NEXT: buffer_store_b8 v0, off, s[8:11], 08432; GFX1164-TRUE16-NEXT: s_endpgm8433;8434; GFX1164-FAKE16-LABEL: uniform_or_i8:8435; GFX1164-FAKE16: ; %bb.0:8436; GFX1164-FAKE16-NEXT: s_clause 0x18437; GFX1164-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x248438; GFX1164-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x348439; GFX1164-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 08440; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8441; GFX1164-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v08442; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, 0, v08443; GFX1164-FAKE16-NEXT: ; implicit-def: $vgpr08444; GFX1164-FAKE16-NEXT: s_and_saveexec_b64 s[2:3], vcc8445; GFX1164-FAKE16-NEXT: s_cbranch_execz .LBB12_48446; GFX1164-FAKE16-NEXT: ; %bb.1:8447; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)8448; GFX1164-FAKE16-NEXT: s_and_b32 s4, s10, -48449; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s118450; GFX1164-FAKE16-NEXT: s_and_b32 s1, s10, 38451; GFX1164-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x08452; GFX1164-FAKE16-NEXT: s_lshl_b32 s13, s1, 38453; GFX1164-FAKE16-NEXT: s_and_b32 s1, s12, 0xff8454; GFX1164-FAKE16-NEXT: s_mov_b64 s[10:11], 08455; GFX1164-FAKE16-NEXT: s_lshl_b32 s14, s1, s138456; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x310160008457; GFX1164-FAKE16-NEXT: s_mov_b32 s6, -18458; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)8459; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s08460; GFX1164-FAKE16-NEXT: .LBB12_2: ; %atomicrmw.start8461; GFX1164-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=18462; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)8463; GFX1164-FAKE16-NEXT: v_or_b32_e32 v0, s14, v18464; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v3, v18465; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v2, v08466; GFX1164-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc8467; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)8468; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v18469; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v28470; GFX1164-FAKE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]8471; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8472; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]8473; GFX1164-FAKE16-NEXT: s_cbranch_execnz .LBB12_28474; GFX1164-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end8475; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[10:11]8476; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s13, v28477; GFX1164-FAKE16-NEXT: .LBB12_4: ; %Flow8478; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]8479; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)8480; GFX1164-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v08481; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)8482; GFX1164-FAKE16-NEXT: s_mov_b32 s11, 0x310160008483; GFX1164-FAKE16-NEXT: s_mov_b32 s10, -18484; GFX1164-FAKE16-NEXT: v_readfirstlane_b32 s0, v08485; GFX1164-FAKE16-NEXT: v_cndmask_b32_e64 v0, s12, 0, vcc8486; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)8487; GFX1164-FAKE16-NEXT: v_or_b32_e32 v0, s0, v08488; GFX1164-FAKE16-NEXT: buffer_store_b8 v0, off, s[8:11], 08489; GFX1164-FAKE16-NEXT: s_endpgm8490;8491; GFX1132-TRUE16-LABEL: uniform_or_i8:8492; GFX1132-TRUE16: ; %bb.0:8493; GFX1132-TRUE16-NEXT: s_clause 0x18494; GFX1132-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x248495; GFX1132-TRUE16-NEXT: s_load_b32 s1, s[4:5], 0x348496; GFX1132-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 08497; GFX1132-TRUE16-NEXT: s_mov_b32 s3, 08498; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)8499; GFX1132-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v08500; GFX1132-TRUE16-NEXT: ; implicit-def: $vgpr0_lo168501; GFX1132-TRUE16-NEXT: s_and_saveexec_b32 s2, vcc_lo8502; GFX1132-TRUE16-NEXT: s_cbranch_execz .LBB12_48503; GFX1132-TRUE16-NEXT: ; %bb.1:8504; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)8505; GFX1132-TRUE16-NEXT: s_and_b32 s4, s10, -48506; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s118507; GFX1132-TRUE16-NEXT: s_and_b32 s6, s10, 38508; GFX1132-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x08509; GFX1132-TRUE16-NEXT: s_lshl_b32 s10, s6, 38510; GFX1132-TRUE16-NEXT: s_and_b32 s6, s1, 0xff8511; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x310160008512; GFX1132-TRUE16-NEXT: s_lshl_b32 s11, s6, s108513; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -18514; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)8515; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s08516; GFX1132-TRUE16-NEXT: .LBB12_2: ; %atomicrmw.start8517; GFX1132-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=18518; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8519; GFX1132-TRUE16-NEXT: v_or_b32_e32 v0, s11, v18520; GFX1132-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v08521; GFX1132-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc8522; GFX1132-TRUE16-NEXT: s_waitcnt vmcnt(0)8523; GFX1132-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v18524; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, v28525; GFX1132-TRUE16-NEXT: s_or_b32 s3, s0, s38526; GFX1132-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8527; GFX1132-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38528; GFX1132-TRUE16-NEXT: s_cbranch_execnz .LBB12_28529; GFX1132-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end8530; GFX1132-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s38531; GFX1132-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s10, v28532; GFX1132-TRUE16-NEXT: .LBB12_4: ; %Flow8533; GFX1132-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s28534; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)8535; GFX1132-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v08536; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)8537; GFX1132-TRUE16-NEXT: s_mov_b32 s11, 0x310160008538; GFX1132-TRUE16-NEXT: s_mov_b32 s10, -18539; GFX1132-TRUE16-NEXT: v_readfirstlane_b32 s0, v08540; GFX1132-TRUE16-NEXT: v_cndmask_b16 v0.l, s1, 0, vcc_lo8541; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)8542; GFX1132-TRUE16-NEXT: v_or_b16 v0.l, s0, v0.l8543; GFX1132-TRUE16-NEXT: buffer_store_b8 v0, off, s[8:11], 08544; GFX1132-TRUE16-NEXT: s_endpgm8545;8546; GFX1132-FAKE16-LABEL: uniform_or_i8:8547; GFX1132-FAKE16: ; %bb.0:8548; GFX1132-FAKE16-NEXT: s_clause 0x18549; GFX1132-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x248550; GFX1132-FAKE16-NEXT: s_load_b32 s1, s[4:5], 0x348551; GFX1132-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 08552; GFX1132-FAKE16-NEXT: s_mov_b32 s3, 08553; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)8554; GFX1132-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v08555; GFX1132-FAKE16-NEXT: ; implicit-def: $vgpr08556; GFX1132-FAKE16-NEXT: s_and_saveexec_b32 s2, vcc_lo8557; GFX1132-FAKE16-NEXT: s_cbranch_execz .LBB12_48558; GFX1132-FAKE16-NEXT: ; %bb.1:8559; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)8560; GFX1132-FAKE16-NEXT: s_and_b32 s4, s10, -48561; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s118562; GFX1132-FAKE16-NEXT: s_and_b32 s6, s10, 38563; GFX1132-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x08564; GFX1132-FAKE16-NEXT: s_lshl_b32 s10, s6, 38565; GFX1132-FAKE16-NEXT: s_and_b32 s6, s1, 0xff8566; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x310160008567; GFX1132-FAKE16-NEXT: s_lshl_b32 s11, s6, s108568; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -18569; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)8570; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s08571; GFX1132-FAKE16-NEXT: .LBB12_2: ; %atomicrmw.start8572; GFX1132-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=18573; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8574; GFX1132-FAKE16-NEXT: v_or_b32_e32 v0, s11, v18575; GFX1132-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v08576; GFX1132-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc8577; GFX1132-FAKE16-NEXT: s_waitcnt vmcnt(0)8578; GFX1132-FAKE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v18579; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, v28580; GFX1132-FAKE16-NEXT: s_or_b32 s3, s0, s38581; GFX1132-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8582; GFX1132-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38583; GFX1132-FAKE16-NEXT: s_cbranch_execnz .LBB12_28584; GFX1132-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end8585; GFX1132-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s38586; GFX1132-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s10, v28587; GFX1132-FAKE16-NEXT: .LBB12_4: ; %Flow8588; GFX1132-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s28589; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)8590; GFX1132-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v08591; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)8592; GFX1132-FAKE16-NEXT: s_mov_b32 s11, 0x310160008593; GFX1132-FAKE16-NEXT: s_mov_b32 s10, -18594; GFX1132-FAKE16-NEXT: v_readfirstlane_b32 s0, v08595; GFX1132-FAKE16-NEXT: v_cndmask_b32_e64 v0, s1, 0, vcc_lo8596; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)8597; GFX1132-FAKE16-NEXT: v_or_b32_e32 v0, s0, v08598; GFX1132-FAKE16-NEXT: buffer_store_b8 v0, off, s[8:11], 08599; GFX1132-FAKE16-NEXT: s_endpgm8600;8601; GFX1264-TRUE16-LABEL: uniform_or_i8:8602; GFX1264-TRUE16: ; %bb.0:8603; GFX1264-TRUE16-NEXT: s_clause 0x18604; GFX1264-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x248605; GFX1264-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x348606; GFX1264-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 08607; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8608; GFX1264-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v08609; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, 0, v08610; GFX1264-TRUE16-NEXT: ; implicit-def: $vgpr0_lo168611; GFX1264-TRUE16-NEXT: s_and_saveexec_b64 s[2:3], vcc8612; GFX1264-TRUE16-NEXT: s_cbranch_execz .LBB12_48613; GFX1264-TRUE16-NEXT: ; %bb.1:8614; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x08615; GFX1264-TRUE16-NEXT: s_and_b32 s4, s10, -48616; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s118617; GFX1264-TRUE16-NEXT: s_and_b32 s1, s10, 38618; GFX1264-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x08619; GFX1264-TRUE16-NEXT: s_lshl_b32 s13, s1, 38620; GFX1264-TRUE16-NEXT: s_and_b32 s1, s12, 0xff8621; GFX1264-TRUE16-NEXT: s_mov_b64 s[10:11], 08622; GFX1264-TRUE16-NEXT: s_lshl_b32 s14, s1, s138623; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x310160008624; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -18625; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x08626; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s08627; GFX1264-TRUE16-NEXT: .LBB12_2: ; %atomicrmw.start8628; GFX1264-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=18629; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)8630; GFX1264-TRUE16-NEXT: v_or_b32_e32 v0, s14, v18631; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v3, v18632; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v2, v08633; GFX1264-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS8634; GFX1264-TRUE16-NEXT: s_wait_loadcnt 0x08635; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v18636; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, v28637; GFX1264-TRUE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]8638; GFX1264-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8639; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]8640; GFX1264-TRUE16-NEXT: s_cbranch_execnz .LBB12_28641; GFX1264-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end8642; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[10:11]8643; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s13, v28644; GFX1264-TRUE16-NEXT: .LBB12_4: ; %Flow8645; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]8646; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)8647; GFX1264-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v08648; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x08649; GFX1264-TRUE16-NEXT: s_mov_b32 s11, 0x310160008650; GFX1264-TRUE16-NEXT: s_mov_b32 s10, -18651; GFX1264-TRUE16-NEXT: v_readfirstlane_b32 s0, v08652; GFX1264-TRUE16-NEXT: v_cndmask_b16 v0.l, s12, 0, vcc8653; GFX1264-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)8654; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)8655; GFX1264-TRUE16-NEXT: v_or_b16 v0.l, s0, v0.l8656; GFX1264-TRUE16-NEXT: buffer_store_b8 v0, off, s[8:11], null8657; GFX1264-TRUE16-NEXT: s_endpgm8658;8659; GFX1264-FAKE16-LABEL: uniform_or_i8:8660; GFX1264-FAKE16: ; %bb.0:8661; GFX1264-FAKE16-NEXT: s_clause 0x18662; GFX1264-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x248663; GFX1264-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x348664; GFX1264-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 08665; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8666; GFX1264-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v08667; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, 0, v08668; GFX1264-FAKE16-NEXT: ; implicit-def: $vgpr08669; GFX1264-FAKE16-NEXT: s_and_saveexec_b64 s[2:3], vcc8670; GFX1264-FAKE16-NEXT: s_cbranch_execz .LBB12_48671; GFX1264-FAKE16-NEXT: ; %bb.1:8672; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x08673; GFX1264-FAKE16-NEXT: s_and_b32 s4, s10, -48674; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s118675; GFX1264-FAKE16-NEXT: s_and_b32 s1, s10, 38676; GFX1264-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x08677; GFX1264-FAKE16-NEXT: s_lshl_b32 s13, s1, 38678; GFX1264-FAKE16-NEXT: s_and_b32 s1, s12, 0xff8679; GFX1264-FAKE16-NEXT: s_mov_b64 s[10:11], 08680; GFX1264-FAKE16-NEXT: s_lshl_b32 s14, s1, s138681; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x310160008682; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -18683; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x08684; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s08685; GFX1264-FAKE16-NEXT: .LBB12_2: ; %atomicrmw.start8686; GFX1264-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=18687; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)8688; GFX1264-FAKE16-NEXT: v_or_b32_e32 v0, s14, v18689; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v3, v18690; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v2, v08691; GFX1264-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS8692; GFX1264-FAKE16-NEXT: s_wait_loadcnt 0x08693; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v18694; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, v28695; GFX1264-FAKE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]8696; GFX1264-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8697; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]8698; GFX1264-FAKE16-NEXT: s_cbranch_execnz .LBB12_28699; GFX1264-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end8700; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[10:11]8701; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s13, v28702; GFX1264-FAKE16-NEXT: .LBB12_4: ; %Flow8703; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]8704; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)8705; GFX1264-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v08706; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x08707; GFX1264-FAKE16-NEXT: s_mov_b32 s11, 0x310160008708; GFX1264-FAKE16-NEXT: s_mov_b32 s10, -18709; GFX1264-FAKE16-NEXT: v_readfirstlane_b32 s0, v08710; GFX1264-FAKE16-NEXT: v_cndmask_b32_e64 v0, s12, 0, vcc8711; GFX1264-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)8712; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)8713; GFX1264-FAKE16-NEXT: v_or_b32_e32 v0, s0, v08714; GFX1264-FAKE16-NEXT: buffer_store_b8 v0, off, s[8:11], null8715; GFX1264-FAKE16-NEXT: s_endpgm8716;8717; GFX1232-TRUE16-LABEL: uniform_or_i8:8718; GFX1232-TRUE16: ; %bb.0:8719; GFX1232-TRUE16-NEXT: s_clause 0x18720; GFX1232-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x248721; GFX1232-TRUE16-NEXT: s_load_b32 s1, s[4:5], 0x348722; GFX1232-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 08723; GFX1232-TRUE16-NEXT: s_mov_b32 s3, 08724; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)8725; GFX1232-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v08726; GFX1232-TRUE16-NEXT: ; implicit-def: $vgpr0_lo168727; GFX1232-TRUE16-NEXT: s_and_saveexec_b32 s2, vcc_lo8728; GFX1232-TRUE16-NEXT: s_cbranch_execz .LBB12_48729; GFX1232-TRUE16-NEXT: ; %bb.1:8730; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x08731; GFX1232-TRUE16-NEXT: s_and_b32 s4, s10, -48732; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s118733; GFX1232-TRUE16-NEXT: s_and_b32 s6, s10, 38734; GFX1232-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x08735; GFX1232-TRUE16-NEXT: s_lshl_b32 s10, s6, 38736; GFX1232-TRUE16-NEXT: s_and_b32 s6, s1, 0xff8737; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x310160008738; GFX1232-TRUE16-NEXT: s_lshl_b32 s11, s6, s108739; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -18740; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x08741; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s08742; GFX1232-TRUE16-NEXT: .LBB12_2: ; %atomicrmw.start8743; GFX1232-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=18744; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8745; GFX1232-TRUE16-NEXT: v_or_b32_e32 v0, s11, v18746; GFX1232-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v08747; GFX1232-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS8748; GFX1232-TRUE16-NEXT: s_wait_loadcnt 0x08749; GFX1232-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v18750; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, v28751; GFX1232-TRUE16-NEXT: s_or_b32 s3, s0, s38752; GFX1232-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8753; GFX1232-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38754; GFX1232-TRUE16-NEXT: s_cbranch_execnz .LBB12_28755; GFX1232-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end8756; GFX1232-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s38757; GFX1232-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s10, v28758; GFX1232-TRUE16-NEXT: .LBB12_4: ; %Flow8759; GFX1232-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s28760; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)8761; GFX1232-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v08762; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x08763; GFX1232-TRUE16-NEXT: s_mov_b32 s11, 0x310160008764; GFX1232-TRUE16-NEXT: s_mov_b32 s10, -18765; GFX1232-TRUE16-NEXT: v_readfirstlane_b32 s0, v08766; GFX1232-TRUE16-NEXT: v_cndmask_b16 v0.l, s1, 0, vcc_lo8767; GFX1232-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)8768; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)8769; GFX1232-TRUE16-NEXT: v_or_b16 v0.l, s0, v0.l8770; GFX1232-TRUE16-NEXT: buffer_store_b8 v0, off, s[8:11], null8771; GFX1232-TRUE16-NEXT: s_endpgm8772;8773; GFX1232-FAKE16-LABEL: uniform_or_i8:8774; GFX1232-FAKE16: ; %bb.0:8775; GFX1232-FAKE16-NEXT: s_clause 0x18776; GFX1232-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x248777; GFX1232-FAKE16-NEXT: s_load_b32 s1, s[4:5], 0x348778; GFX1232-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 08779; GFX1232-FAKE16-NEXT: s_mov_b32 s3, 08780; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)8781; GFX1232-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v08782; GFX1232-FAKE16-NEXT: ; implicit-def: $vgpr08783; GFX1232-FAKE16-NEXT: s_and_saveexec_b32 s2, vcc_lo8784; GFX1232-FAKE16-NEXT: s_cbranch_execz .LBB12_48785; GFX1232-FAKE16-NEXT: ; %bb.1:8786; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x08787; GFX1232-FAKE16-NEXT: s_and_b32 s4, s10, -48788; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s118789; GFX1232-FAKE16-NEXT: s_and_b32 s6, s10, 38790; GFX1232-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x08791; GFX1232-FAKE16-NEXT: s_lshl_b32 s10, s6, 38792; GFX1232-FAKE16-NEXT: s_and_b32 s6, s1, 0xff8793; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x310160008794; GFX1232-FAKE16-NEXT: s_lshl_b32 s11, s6, s108795; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -18796; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x08797; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s08798; GFX1232-FAKE16-NEXT: .LBB12_2: ; %atomicrmw.start8799; GFX1232-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=18800; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8801; GFX1232-FAKE16-NEXT: v_or_b32_e32 v0, s11, v18802; GFX1232-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v08803; GFX1232-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS8804; GFX1232-FAKE16-NEXT: s_wait_loadcnt 0x08805; GFX1232-FAKE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v18806; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, v28807; GFX1232-FAKE16-NEXT: s_or_b32 s3, s0, s38808; GFX1232-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8809; GFX1232-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s38810; GFX1232-FAKE16-NEXT: s_cbranch_execnz .LBB12_28811; GFX1232-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end8812; GFX1232-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s38813; GFX1232-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s10, v28814; GFX1232-FAKE16-NEXT: .LBB12_4: ; %Flow8815; GFX1232-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s28816; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)8817; GFX1232-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v08818; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x08819; GFX1232-FAKE16-NEXT: s_mov_b32 s11, 0x310160008820; GFX1232-FAKE16-NEXT: s_mov_b32 s10, -18821; GFX1232-FAKE16-NEXT: v_readfirstlane_b32 s0, v08822; GFX1232-FAKE16-NEXT: v_cndmask_b32_e64 v0, s1, 0, vcc_lo8823; GFX1232-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)8824; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)8825; GFX1232-FAKE16-NEXT: v_or_b32_e32 v0, s0, v08826; GFX1232-FAKE16-NEXT: buffer_store_b8 v0, off, s[8:11], null8827; GFX1232-FAKE16-NEXT: s_endpgm8828 %rmw = atomicrmw or ptr addrspace(1) %uniform.ptr, i8 %val monotonic, align 18829 store i8 %rmw, ptr addrspace(1) %result8830 ret void8831}8832 8833define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i8 %val) {8834; GFX7LESS-LABEL: uniform_add_i8:8835; GFX7LESS: ; %bb.0:8836; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec8837; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x98838; GFX7LESS-NEXT: s_load_dword s10, s[4:5], 0xd8839; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 08840; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, s7, v08841; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v48842; GFX7LESS-NEXT: ; implicit-def: $vgpr08843; GFX7LESS-NEXT: s_and_saveexec_b64 s[8:9], vcc8844; GFX7LESS-NEXT: s_cbranch_execz .LBB13_48845; GFX7LESS-NEXT: ; %bb.1:8846; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]8847; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)8848; GFX7LESS-NEXT: s_and_b32 s4, s2, -48849; GFX7LESS-NEXT: s_mov_b32 s5, s38850; GFX7LESS-NEXT: s_and_b32 s11, s2, 38851; GFX7LESS-NEXT: s_mov_b64 s[2:3], 08852; GFX7LESS-NEXT: s_mov_b32 s7, 0xf0008853; GFX7LESS-NEXT: s_mul_i32 s6, s10, s68854; GFX7LESS-NEXT: s_lshl_b32 s11, s11, 38855; GFX7LESS-NEXT: s_load_dword s15, s[4:5], 0x08856; GFX7LESS-NEXT: s_lshl_b32 s12, 0xff, s118857; GFX7LESS-NEXT: s_and_b32 s6, s6, 0xff8858; GFX7LESS-NEXT: s_not_b32 s13, s128859; GFX7LESS-NEXT: s_lshl_b32 s14, s6, s118860; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)8861; GFX7LESS-NEXT: v_mov_b32_e32 v1, s158862; GFX7LESS-NEXT: s_mov_b32 s6, -18863; GFX7LESS-NEXT: .LBB13_2: ; %atomicrmw.start8864; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=18865; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s14, v18866; GFX7LESS-NEXT: v_and_b32_e32 v0, s12, v08867; GFX7LESS-NEXT: s_waitcnt expcnt(0)8868; GFX7LESS-NEXT: v_and_b32_e32 v2, s13, v18869; GFX7LESS-NEXT: v_or_b32_e32 v0, v2, v08870; GFX7LESS-NEXT: v_mov_b32_e32 v3, v18871; GFX7LESS-NEXT: v_mov_b32_e32 v2, v08872; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc8873; GFX7LESS-NEXT: s_waitcnt vmcnt(0)8874; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v18875; GFX7LESS-NEXT: s_or_b64 s[2:3], vcc, s[2:3]8876; GFX7LESS-NEXT: v_mov_b32_e32 v1, v28877; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[2:3]8878; GFX7LESS-NEXT: s_cbranch_execnz .LBB13_28879; GFX7LESS-NEXT: ; %bb.3: ; %atomicrmw.end8880; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3]8881; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s11, v28882; GFX7LESS-NEXT: .LBB13_4: ; %Flow8883; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]8884; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)8885; GFX7LESS-NEXT: s_mov_b32 s3, 0xf0008886; GFX7LESS-NEXT: s_mov_b32 s2, -18887; GFX7LESS-NEXT: v_and_b32_e32 v0, 0xff, v08888; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v08889; GFX7LESS-NEXT: s_and_b32 s5, s10, 0xff8890; GFX7LESS-NEXT: v_mov_b32_e32 v0, s48891; GFX7LESS-NEXT: v_mad_u32_u24 v0, s5, v4, v08892; GFX7LESS-NEXT: buffer_store_byte v0, off, s[0:3], 08893; GFX7LESS-NEXT: s_endpgm8894;8895; GFX8-LABEL: uniform_add_i8:8896; GFX8: ; %bb.0:8897; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x248898; GFX8-NEXT: s_load_dword s10, s[4:5], 0x348899; GFX8-NEXT: s_mov_b64 s[6:7], exec8900; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 08901; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v08902; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v48903; GFX8-NEXT: ; implicit-def: $vgpr08904; GFX8-NEXT: s_and_saveexec_b64 s[8:9], vcc8905; GFX8-NEXT: s_cbranch_execz .LBB13_48906; GFX8-NEXT: ; %bb.1:8907; GFX8-NEXT: s_bcnt1_i32_b64 s4, s[6:7]8908; GFX8-NEXT: s_waitcnt lgkmcnt(0)8909; GFX8-NEXT: s_mul_i32 s6, s10, s48910; GFX8-NEXT: s_and_b32 s4, s2, -48911; GFX8-NEXT: s_mov_b32 s5, s38912; GFX8-NEXT: s_load_dword s7, s[4:5], 0x08913; GFX8-NEXT: s_and_b32 s2, s2, 38914; GFX8-NEXT: s_lshl_b32 s11, s2, 38915; GFX8-NEXT: s_lshl_b32 s12, 0xff, s118916; GFX8-NEXT: s_and_b32 s2, s6, 0xff8917; GFX8-NEXT: s_not_b32 s13, s128918; GFX8-NEXT: s_lshl_b32 s14, s2, s118919; GFX8-NEXT: s_mov_b64 s[2:3], 08920; GFX8-NEXT: s_waitcnt lgkmcnt(0)8921; GFX8-NEXT: v_mov_b32_e32 v1, s78922; GFX8-NEXT: s_mov_b32 s7, 0xf0008923; GFX8-NEXT: s_mov_b32 s6, -18924; GFX8-NEXT: .LBB13_2: ; %atomicrmw.start8925; GFX8-NEXT: ; =>This Inner Loop Header: Depth=18926; GFX8-NEXT: v_add_u32_e32 v0, vcc, s14, v18927; GFX8-NEXT: v_and_b32_e32 v2, s13, v18928; GFX8-NEXT: v_and_b32_e32 v0, s12, v08929; GFX8-NEXT: v_or_b32_e32 v0, v2, v08930; GFX8-NEXT: v_mov_b32_e32 v3, v18931; GFX8-NEXT: v_mov_b32_e32 v2, v08932; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc8933; GFX8-NEXT: s_waitcnt vmcnt(0)8934; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v18935; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]8936; GFX8-NEXT: v_mov_b32_e32 v1, v28937; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]8938; GFX8-NEXT: s_cbranch_execnz .LBB13_28939; GFX8-NEXT: ; %bb.3: ; %atomicrmw.end8940; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]8941; GFX8-NEXT: v_lshrrev_b32_e32 v0, s11, v28942; GFX8-NEXT: .LBB13_4: ; %Flow8943; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]8944; GFX8-NEXT: v_and_b32_e32 v0, 0xff, v08945; GFX8-NEXT: v_readfirstlane_b32 s4, v08946; GFX8-NEXT: v_mov_b32_e32 v0, s48947; GFX8-NEXT: s_waitcnt lgkmcnt(0)8948; GFX8-NEXT: s_mov_b32 s3, 0xf0008949; GFX8-NEXT: s_mov_b32 s2, -18950; GFX8-NEXT: v_mad_u16 v0, s10, v4, v08951; GFX8-NEXT: buffer_store_byte v0, off, s[0:3], 08952; GFX8-NEXT: s_endpgm8953;8954; GFX9-LABEL: uniform_add_i8:8955; GFX9: ; %bb.0:8956; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x248957; GFX9-NEXT: s_load_dword s10, s[4:5], 0x348958; GFX9-NEXT: s_mov_b64 s[6:7], exec8959; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 08960; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v08961; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v48962; GFX9-NEXT: ; implicit-def: $vgpr08963; GFX9-NEXT: s_and_saveexec_b64 s[8:9], vcc8964; GFX9-NEXT: s_cbranch_execz .LBB13_48965; GFX9-NEXT: ; %bb.1:8966; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[6:7]8967; GFX9-NEXT: s_waitcnt lgkmcnt(0)8968; GFX9-NEXT: s_mul_i32 s6, s10, s48969; GFX9-NEXT: s_and_b32 s4, s2, -48970; GFX9-NEXT: s_mov_b32 s5, s38971; GFX9-NEXT: s_load_dword s7, s[4:5], 0x08972; GFX9-NEXT: s_and_b32 s2, s2, 38973; GFX9-NEXT: s_lshl_b32 s11, s2, 38974; GFX9-NEXT: s_lshl_b32 s12, 0xff, s118975; GFX9-NEXT: s_and_b32 s2, s6, 0xff8976; GFX9-NEXT: s_not_b32 s13, s128977; GFX9-NEXT: s_lshl_b32 s14, s2, s118978; GFX9-NEXT: s_mov_b64 s[2:3], 08979; GFX9-NEXT: s_waitcnt lgkmcnt(0)8980; GFX9-NEXT: v_mov_b32_e32 v1, s78981; GFX9-NEXT: s_mov_b32 s7, 0xf0008982; GFX9-NEXT: s_mov_b32 s6, -18983; GFX9-NEXT: .LBB13_2: ; %atomicrmw.start8984; GFX9-NEXT: ; =>This Inner Loop Header: Depth=18985; GFX9-NEXT: v_add_u32_e32 v0, s14, v18986; GFX9-NEXT: v_and_b32_e32 v0, s12, v08987; GFX9-NEXT: v_and_or_b32 v0, v1, s13, v08988; GFX9-NEXT: v_mov_b32_e32 v3, v18989; GFX9-NEXT: v_mov_b32_e32 v2, v08990; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc8991; GFX9-NEXT: s_waitcnt vmcnt(0)8992; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v18993; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]8994; GFX9-NEXT: v_mov_b32_e32 v1, v28995; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]8996; GFX9-NEXT: s_cbranch_execnz .LBB13_28997; GFX9-NEXT: ; %bb.3: ; %atomicrmw.end8998; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]8999; GFX9-NEXT: v_lshrrev_b32_e32 v0, s11, v29000; GFX9-NEXT: .LBB13_4: ; %Flow9001; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]9002; GFX9-NEXT: v_and_b32_e32 v0, 0xff, v09003; GFX9-NEXT: v_readfirstlane_b32 s4, v09004; GFX9-NEXT: v_mov_b32_e32 v0, s49005; GFX9-NEXT: s_waitcnt lgkmcnt(0)9006; GFX9-NEXT: s_mov_b32 s3, 0xf0009007; GFX9-NEXT: s_mov_b32 s2, -19008; GFX9-NEXT: v_mad_legacy_u16 v0, s10, v4, v09009; GFX9-NEXT: buffer_store_byte v0, off, s[0:3], 09010; GFX9-NEXT: s_endpgm9011;9012; GFX1064-LABEL: uniform_add_i8:9013; GFX1064: ; %bb.0:9014; GFX1064-NEXT: s_clause 0x19015; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249016; GFX1064-NEXT: s_load_dword s10, s[4:5], 0x349017; GFX1064-NEXT: s_mov_b64 s[6:7], exec9018; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 09019; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v09020; GFX1064-NEXT: ; implicit-def: $vgpr09021; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v49022; GFX1064-NEXT: s_and_saveexec_b64 s[8:9], vcc9023; GFX1064-NEXT: s_cbranch_execz .LBB13_49024; GFX1064-NEXT: ; %bb.1:9025; GFX1064-NEXT: s_waitcnt lgkmcnt(0)9026; GFX1064-NEXT: s_and_b32 s4, s2, -49027; GFX1064-NEXT: s_mov_b32 s5, s39028; GFX1064-NEXT: s_and_b32 s2, s2, 39029; GFX1064-NEXT: s_load_dword s3, s[4:5], 0x09030; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]9031; GFX1064-NEXT: s_lshl_b32 s11, s2, 39032; GFX1064-NEXT: s_mul_i32 s2, s10, s69033; GFX1064-NEXT: s_lshl_b32 s12, 0xff, s119034; GFX1064-NEXT: s_and_b32 s2, s2, 0xff9035; GFX1064-NEXT: s_not_b32 s13, s129036; GFX1064-NEXT: s_lshl_b32 s14, s2, s119037; GFX1064-NEXT: s_mov_b32 s7, 0x310160009038; GFX1064-NEXT: s_mov_b32 s6, -19039; GFX1064-NEXT: s_waitcnt lgkmcnt(0)9040; GFX1064-NEXT: v_mov_b32_e32 v1, s39041; GFX1064-NEXT: s_mov_b64 s[2:3], 09042; GFX1064-NEXT: .LBB13_2: ; %atomicrmw.start9043; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=19044; GFX1064-NEXT: v_add_nc_u32_e32 v0, s14, v19045; GFX1064-NEXT: v_and_b32_e32 v0, s12, v09046; GFX1064-NEXT: v_and_or_b32 v0, v1, s13, v09047; GFX1064-NEXT: v_mov_b32_e32 v3, v19048; GFX1064-NEXT: v_mov_b32_e32 v2, v09049; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc9050; GFX1064-NEXT: s_waitcnt vmcnt(0)9051; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v19052; GFX1064-NEXT: v_mov_b32_e32 v1, v29053; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]9054; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]9055; GFX1064-NEXT: s_cbranch_execnz .LBB13_29056; GFX1064-NEXT: ; %bb.3: ; %atomicrmw.end9057; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]9058; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s11, v29059; GFX1064-NEXT: .LBB13_4: ; %Flow9060; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]9061; GFX1064-NEXT: v_and_b32_e32 v0, 0xff, v09062; GFX1064-NEXT: s_waitcnt lgkmcnt(0)9063; GFX1064-NEXT: s_mov_b32 s3, 0x310160009064; GFX1064-NEXT: v_readfirstlane_b32 s2, v09065; GFX1064-NEXT: v_mad_u16 v0, s10, v4, s29066; GFX1064-NEXT: s_mov_b32 s2, -19067; GFX1064-NEXT: buffer_store_byte v0, off, s[0:3], 09068; GFX1064-NEXT: s_endpgm9069;9070; GFX1032-LABEL: uniform_add_i8:9071; GFX1032: ; %bb.0:9072; GFX1032-NEXT: s_clause 0x19073; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249074; GFX1032-NEXT: s_load_dword s8, s[4:5], 0x349075; GFX1032-NEXT: s_mov_b32 s6, exec_lo9076; GFX1032-NEXT: s_mov_b32 s10, 09077; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 09078; GFX1032-NEXT: ; implicit-def: $vgpr09079; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v49080; GFX1032-NEXT: s_and_saveexec_b32 s9, vcc_lo9081; GFX1032-NEXT: s_cbranch_execz .LBB13_49082; GFX1032-NEXT: ; %bb.1:9083; GFX1032-NEXT: s_waitcnt lgkmcnt(0)9084; GFX1032-NEXT: s_and_b32 s4, s2, -49085; GFX1032-NEXT: s_mov_b32 s5, s39086; GFX1032-NEXT: s_and_b32 s2, s2, 39087; GFX1032-NEXT: s_load_dword s7, s[4:5], 0x09088; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s69089; GFX1032-NEXT: s_lshl_b32 s2, s2, 39090; GFX1032-NEXT: s_mul_i32 s6, s8, s69091; GFX1032-NEXT: s_lshl_b32 s3, 0xff, s29092; GFX1032-NEXT: s_and_b32 s6, s6, 0xff9093; GFX1032-NEXT: s_not_b32 s11, s39094; GFX1032-NEXT: s_lshl_b32 s12, s6, s29095; GFX1032-NEXT: s_mov_b32 s6, -19096; GFX1032-NEXT: s_waitcnt lgkmcnt(0)9097; GFX1032-NEXT: v_mov_b32_e32 v1, s79098; GFX1032-NEXT: s_mov_b32 s7, 0x310160009099; GFX1032-NEXT: .LBB13_2: ; %atomicrmw.start9100; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=19101; GFX1032-NEXT: v_add_nc_u32_e32 v0, s12, v19102; GFX1032-NEXT: v_and_b32_e32 v0, s3, v09103; GFX1032-NEXT: v_and_or_b32 v0, v1, s11, v09104; GFX1032-NEXT: v_mov_b32_e32 v3, v19105; GFX1032-NEXT: v_mov_b32_e32 v2, v09106; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc9107; GFX1032-NEXT: s_waitcnt vmcnt(0)9108; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v19109; GFX1032-NEXT: v_mov_b32_e32 v1, v29110; GFX1032-NEXT: s_or_b32 s10, vcc_lo, s109111; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s109112; GFX1032-NEXT: s_cbranch_execnz .LBB13_29113; GFX1032-NEXT: ; %bb.3: ; %atomicrmw.end9114; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s109115; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s2, v29116; GFX1032-NEXT: .LBB13_4: ; %Flow9117; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s99118; GFX1032-NEXT: v_and_b32_e32 v0, 0xff, v09119; GFX1032-NEXT: s_waitcnt lgkmcnt(0)9120; GFX1032-NEXT: s_mov_b32 s3, 0x310160009121; GFX1032-NEXT: v_readfirstlane_b32 s2, v09122; GFX1032-NEXT: v_mad_u16 v0, s8, v4, s29123; GFX1032-NEXT: s_mov_b32 s2, -19124; GFX1032-NEXT: buffer_store_byte v0, off, s[0:3], 09125; GFX1032-NEXT: s_endpgm9126;9127; GFX1164-TRUE16-LABEL: uniform_add_i8:9128; GFX1164-TRUE16: ; %bb.0:9129; GFX1164-TRUE16-NEXT: s_clause 0x19130; GFX1164-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x249131; GFX1164-TRUE16-NEXT: s_load_b32 s10, s[4:5], 0x349132; GFX1164-TRUE16-NEXT: s_mov_b64 s[6:7], exec9133; GFX1164-TRUE16-NEXT: s_mov_b64 s[8:9], exec9134; GFX1164-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 09135; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9136; GFX1164-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v09137; GFX1164-TRUE16-NEXT: ; implicit-def: $vgpr0_lo169138; GFX1164-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v49139; GFX1164-TRUE16-NEXT: s_cbranch_execz .LBB13_49140; GFX1164-TRUE16-NEXT: ; %bb.1:9141; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)9142; GFX1164-TRUE16-NEXT: s_and_b32 s4, s2, -49143; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s39144; GFX1164-TRUE16-NEXT: s_and_b32 s2, s2, 39145; GFX1164-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x09146; GFX1164-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]9147; GFX1164-TRUE16-NEXT: s_lshl_b32 s11, s2, 39148; GFX1164-TRUE16-NEXT: s_mul_i32 s2, s10, s69149; GFX1164-TRUE16-NEXT: s_lshl_b32 s12, 0xff, s119150; GFX1164-TRUE16-NEXT: s_and_b32 s2, s2, 0xff9151; GFX1164-TRUE16-NEXT: s_not_b32 s13, s129152; GFX1164-TRUE16-NEXT: s_lshl_b32 s14, s2, s119153; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x310160009154; GFX1164-TRUE16-NEXT: s_mov_b32 s6, -19155; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)9156; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s39157; GFX1164-TRUE16-NEXT: s_mov_b64 s[2:3], 09158; GFX1164-TRUE16-NEXT: .LBB13_2: ; %atomicrmw.start9159; GFX1164-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=19160; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9161; GFX1164-TRUE16-NEXT: v_add_nc_u32_e32 v0, s14, v19162; GFX1164-TRUE16-NEXT: v_and_b32_e32 v0, s12, v09163; GFX1164-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)9164; GFX1164-TRUE16-NEXT: v_and_or_b32 v0, v1, s13, v09165; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v3, v19166; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)9167; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v2, v09168; GFX1164-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc9169; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)9170; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v19171; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v29172; GFX1164-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]9173; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)9174; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]9175; GFX1164-TRUE16-NEXT: s_cbranch_execnz .LBB13_29176; GFX1164-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end9177; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]9178; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s11, v29179; GFX1164-TRUE16-NEXT: .LBB13_4: ; %Flow9180; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[8:9]9181; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)9182; GFX1164-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v09183; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)9184; GFX1164-TRUE16-NEXT: s_mov_b32 s3, 0x310160009185; GFX1164-TRUE16-NEXT: v_readfirstlane_b32 s2, v09186; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)9187; GFX1164-TRUE16-NEXT: v_mad_u16 v0.l, s10, v4.l, s29188; GFX1164-TRUE16-NEXT: s_mov_b32 s2, -19189; GFX1164-TRUE16-NEXT: buffer_store_b8 v0, off, s[0:3], 09190; GFX1164-TRUE16-NEXT: s_endpgm9191;9192; GFX1164-FAKE16-LABEL: uniform_add_i8:9193; GFX1164-FAKE16: ; %bb.0:9194; GFX1164-FAKE16-NEXT: s_clause 0x19195; GFX1164-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x249196; GFX1164-FAKE16-NEXT: s_load_b32 s10, s[4:5], 0x349197; GFX1164-FAKE16-NEXT: s_mov_b64 s[6:7], exec9198; GFX1164-FAKE16-NEXT: s_mov_b64 s[8:9], exec9199; GFX1164-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 09200; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9201; GFX1164-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v09202; GFX1164-FAKE16-NEXT: ; implicit-def: $vgpr09203; GFX1164-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v49204; GFX1164-FAKE16-NEXT: s_cbranch_execz .LBB13_49205; GFX1164-FAKE16-NEXT: ; %bb.1:9206; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)9207; GFX1164-FAKE16-NEXT: s_and_b32 s4, s2, -49208; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s39209; GFX1164-FAKE16-NEXT: s_and_b32 s2, s2, 39210; GFX1164-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x09211; GFX1164-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]9212; GFX1164-FAKE16-NEXT: s_lshl_b32 s11, s2, 39213; GFX1164-FAKE16-NEXT: s_mul_i32 s2, s10, s69214; GFX1164-FAKE16-NEXT: s_lshl_b32 s12, 0xff, s119215; GFX1164-FAKE16-NEXT: s_and_b32 s2, s2, 0xff9216; GFX1164-FAKE16-NEXT: s_not_b32 s13, s129217; GFX1164-FAKE16-NEXT: s_lshl_b32 s14, s2, s119218; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x310160009219; GFX1164-FAKE16-NEXT: s_mov_b32 s6, -19220; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)9221; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s39222; GFX1164-FAKE16-NEXT: s_mov_b64 s[2:3], 09223; GFX1164-FAKE16-NEXT: .LBB13_2: ; %atomicrmw.start9224; GFX1164-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=19225; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9226; GFX1164-FAKE16-NEXT: v_add_nc_u32_e32 v0, s14, v19227; GFX1164-FAKE16-NEXT: v_and_b32_e32 v0, s12, v09228; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)9229; GFX1164-FAKE16-NEXT: v_and_or_b32 v0, v1, s13, v09230; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v3, v19231; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)9232; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v2, v09233; GFX1164-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc9234; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)9235; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v19236; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v29237; GFX1164-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]9238; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)9239; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]9240; GFX1164-FAKE16-NEXT: s_cbranch_execnz .LBB13_29241; GFX1164-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end9242; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]9243; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s11, v29244; GFX1164-FAKE16-NEXT: .LBB13_4: ; %Flow9245; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[8:9]9246; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)9247; GFX1164-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v09248; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)9249; GFX1164-FAKE16-NEXT: s_mov_b32 s3, 0x310160009250; GFX1164-FAKE16-NEXT: v_readfirstlane_b32 s2, v09251; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)9252; GFX1164-FAKE16-NEXT: v_mad_u16 v0, s10, v4, s29253; GFX1164-FAKE16-NEXT: s_mov_b32 s2, -19254; GFX1164-FAKE16-NEXT: buffer_store_b8 v0, off, s[0:3], 09255; GFX1164-FAKE16-NEXT: s_endpgm9256;9257; GFX1132-TRUE16-LABEL: uniform_add_i8:9258; GFX1132-TRUE16: ; %bb.0:9259; GFX1132-TRUE16-NEXT: s_clause 0x19260; GFX1132-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x249261; GFX1132-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x349262; GFX1132-TRUE16-NEXT: s_mov_b32 s6, exec_lo9263; GFX1132-TRUE16-NEXT: s_mov_b32 s10, 09264; GFX1132-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 09265; GFX1132-TRUE16-NEXT: s_mov_b32 s9, exec_lo9266; GFX1132-TRUE16-NEXT: ; implicit-def: $vgpr0_lo169267; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)9268; GFX1132-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v49269; GFX1132-TRUE16-NEXT: s_cbranch_execz .LBB13_49270; GFX1132-TRUE16-NEXT: ; %bb.1:9271; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)9272; GFX1132-TRUE16-NEXT: s_and_b32 s4, s2, -49273; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s39274; GFX1132-TRUE16-NEXT: s_and_b32 s2, s2, 39275; GFX1132-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x09276; GFX1132-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s69277; GFX1132-TRUE16-NEXT: s_lshl_b32 s2, s2, 39278; GFX1132-TRUE16-NEXT: s_mul_i32 s6, s8, s69279; GFX1132-TRUE16-NEXT: s_lshl_b32 s3, 0xff, s29280; GFX1132-TRUE16-NEXT: s_and_b32 s6, s6, 0xff9281; GFX1132-TRUE16-NEXT: s_not_b32 s11, s39282; GFX1132-TRUE16-NEXT: s_lshl_b32 s12, s6, s29283; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -19284; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)9285; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s79286; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x310160009287; GFX1132-TRUE16-NEXT: .LBB13_2: ; %atomicrmw.start9288; GFX1132-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=19289; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9290; GFX1132-TRUE16-NEXT: v_add_nc_u32_e32 v0, s12, v19291; GFX1132-TRUE16-NEXT: v_and_b32_e32 v0, s3, v09292; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9293; GFX1132-TRUE16-NEXT: v_and_or_b32 v0, v1, s11, v09294; GFX1132-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v09295; GFX1132-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc9296; GFX1132-TRUE16-NEXT: s_waitcnt vmcnt(0)9297; GFX1132-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v19298; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, v29299; GFX1132-TRUE16-NEXT: s_or_b32 s10, vcc_lo, s109300; GFX1132-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)9301; GFX1132-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s109302; GFX1132-TRUE16-NEXT: s_cbranch_execnz .LBB13_29303; GFX1132-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end9304; GFX1132-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s109305; GFX1132-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v29306; GFX1132-TRUE16-NEXT: .LBB13_4: ; %Flow9307; GFX1132-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s99308; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)9309; GFX1132-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v09310; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)9311; GFX1132-TRUE16-NEXT: s_mov_b32 s3, 0x310160009312; GFX1132-TRUE16-NEXT: v_readfirstlane_b32 s2, v09313; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)9314; GFX1132-TRUE16-NEXT: v_mad_u16 v0.l, s8, v4.l, s29315; GFX1132-TRUE16-NEXT: s_mov_b32 s2, -19316; GFX1132-TRUE16-NEXT: buffer_store_b8 v0, off, s[0:3], 09317; GFX1132-TRUE16-NEXT: s_endpgm9318;9319; GFX1132-FAKE16-LABEL: uniform_add_i8:9320; GFX1132-FAKE16: ; %bb.0:9321; GFX1132-FAKE16-NEXT: s_clause 0x19322; GFX1132-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x249323; GFX1132-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x349324; GFX1132-FAKE16-NEXT: s_mov_b32 s6, exec_lo9325; GFX1132-FAKE16-NEXT: s_mov_b32 s10, 09326; GFX1132-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 09327; GFX1132-FAKE16-NEXT: s_mov_b32 s9, exec_lo9328; GFX1132-FAKE16-NEXT: ; implicit-def: $vgpr09329; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)9330; GFX1132-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v49331; GFX1132-FAKE16-NEXT: s_cbranch_execz .LBB13_49332; GFX1132-FAKE16-NEXT: ; %bb.1:9333; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)9334; GFX1132-FAKE16-NEXT: s_and_b32 s4, s2, -49335; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s39336; GFX1132-FAKE16-NEXT: s_and_b32 s2, s2, 39337; GFX1132-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x09338; GFX1132-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s69339; GFX1132-FAKE16-NEXT: s_lshl_b32 s2, s2, 39340; GFX1132-FAKE16-NEXT: s_mul_i32 s6, s8, s69341; GFX1132-FAKE16-NEXT: s_lshl_b32 s3, 0xff, s29342; GFX1132-FAKE16-NEXT: s_and_b32 s6, s6, 0xff9343; GFX1132-FAKE16-NEXT: s_not_b32 s11, s39344; GFX1132-FAKE16-NEXT: s_lshl_b32 s12, s6, s29345; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -19346; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)9347; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s79348; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x310160009349; GFX1132-FAKE16-NEXT: .LBB13_2: ; %atomicrmw.start9350; GFX1132-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=19351; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9352; GFX1132-FAKE16-NEXT: v_add_nc_u32_e32 v0, s12, v19353; GFX1132-FAKE16-NEXT: v_and_b32_e32 v0, s3, v09354; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9355; GFX1132-FAKE16-NEXT: v_and_or_b32 v0, v1, s11, v09356; GFX1132-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v09357; GFX1132-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc9358; GFX1132-FAKE16-NEXT: s_waitcnt vmcnt(0)9359; GFX1132-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v19360; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, v29361; GFX1132-FAKE16-NEXT: s_or_b32 s10, vcc_lo, s109362; GFX1132-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)9363; GFX1132-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s109364; GFX1132-FAKE16-NEXT: s_cbranch_execnz .LBB13_29365; GFX1132-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end9366; GFX1132-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s109367; GFX1132-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v29368; GFX1132-FAKE16-NEXT: .LBB13_4: ; %Flow9369; GFX1132-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s99370; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)9371; GFX1132-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v09372; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)9373; GFX1132-FAKE16-NEXT: s_mov_b32 s3, 0x310160009374; GFX1132-FAKE16-NEXT: v_readfirstlane_b32 s2, v09375; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)9376; GFX1132-FAKE16-NEXT: v_mad_u16 v0, s8, v4, s29377; GFX1132-FAKE16-NEXT: s_mov_b32 s2, -19378; GFX1132-FAKE16-NEXT: buffer_store_b8 v0, off, s[0:3], 09379; GFX1132-FAKE16-NEXT: s_endpgm9380;9381; GFX1264-TRUE16-LABEL: uniform_add_i8:9382; GFX1264-TRUE16: ; %bb.0:9383; GFX1264-TRUE16-NEXT: s_clause 0x19384; GFX1264-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x249385; GFX1264-TRUE16-NEXT: s_load_b32 s10, s[4:5], 0x349386; GFX1264-TRUE16-NEXT: s_mov_b64 s[6:7], exec9387; GFX1264-TRUE16-NEXT: s_mov_b64 s[8:9], exec9388; GFX1264-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 09389; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9390; GFX1264-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v09391; GFX1264-TRUE16-NEXT: ; implicit-def: $vgpr0_lo169392; GFX1264-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v49393; GFX1264-TRUE16-NEXT: s_cbranch_execz .LBB13_49394; GFX1264-TRUE16-NEXT: ; %bb.1:9395; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x09396; GFX1264-TRUE16-NEXT: s_and_b32 s4, s2, -49397; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s39398; GFX1264-TRUE16-NEXT: s_and_b32 s2, s2, 39399; GFX1264-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x09400; GFX1264-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]9401; GFX1264-TRUE16-NEXT: s_lshl_b32 s11, s2, 39402; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)9403; GFX1264-TRUE16-NEXT: s_mul_i32 s2, s10, s69404; GFX1264-TRUE16-NEXT: s_lshl_b32 s12, 0xff, s119405; GFX1264-TRUE16-NEXT: s_and_b32 s2, s2, 0xff9406; GFX1264-TRUE16-NEXT: s_not_b32 s13, s129407; GFX1264-TRUE16-NEXT: s_lshl_b32 s14, s2, s119408; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x310160009409; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -19410; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x09411; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s39412; GFX1264-TRUE16-NEXT: s_mov_b64 s[2:3], 09413; GFX1264-TRUE16-NEXT: .LBB13_2: ; %atomicrmw.start9414; GFX1264-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=19415; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9416; GFX1264-TRUE16-NEXT: v_add_nc_u32_e32 v0, s14, v19417; GFX1264-TRUE16-NEXT: v_and_b32_e32 v0, s12, v09418; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)9419; GFX1264-TRUE16-NEXT: v_and_or_b32 v0, v1, s13, v09420; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v3, v19421; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v2, v09422; GFX1264-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS9423; GFX1264-TRUE16-NEXT: s_wait_loadcnt 0x09424; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v19425; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, v29426; GFX1264-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]9427; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)9428; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]9429; GFX1264-TRUE16-NEXT: s_cbranch_execnz .LBB13_29430; GFX1264-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end9431; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]9432; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s11, v29433; GFX1264-TRUE16-NEXT: .LBB13_4: ; %Flow9434; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[8:9]9435; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)9436; GFX1264-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v09437; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x09438; GFX1264-TRUE16-NEXT: s_mov_b32 s3, 0x310160009439; GFX1264-TRUE16-NEXT: v_readfirstlane_b32 s2, v09440; GFX1264-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)9441; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)9442; GFX1264-TRUE16-NEXT: v_mad_u16 v0.l, s10, v4.l, s29443; GFX1264-TRUE16-NEXT: s_mov_b32 s2, -19444; GFX1264-TRUE16-NEXT: buffer_store_b8 v0, off, s[0:3], null9445; GFX1264-TRUE16-NEXT: s_endpgm9446;9447; GFX1264-FAKE16-LABEL: uniform_add_i8:9448; GFX1264-FAKE16: ; %bb.0:9449; GFX1264-FAKE16-NEXT: s_clause 0x19450; GFX1264-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x249451; GFX1264-FAKE16-NEXT: s_load_b32 s10, s[4:5], 0x349452; GFX1264-FAKE16-NEXT: s_mov_b64 s[6:7], exec9453; GFX1264-FAKE16-NEXT: s_mov_b64 s[8:9], exec9454; GFX1264-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 09455; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9456; GFX1264-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v09457; GFX1264-FAKE16-NEXT: ; implicit-def: $vgpr09458; GFX1264-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v49459; GFX1264-FAKE16-NEXT: s_cbranch_execz .LBB13_49460; GFX1264-FAKE16-NEXT: ; %bb.1:9461; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x09462; GFX1264-FAKE16-NEXT: s_and_b32 s4, s2, -49463; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s39464; GFX1264-FAKE16-NEXT: s_and_b32 s2, s2, 39465; GFX1264-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x09466; GFX1264-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]9467; GFX1264-FAKE16-NEXT: s_lshl_b32 s11, s2, 39468; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)9469; GFX1264-FAKE16-NEXT: s_mul_i32 s2, s10, s69470; GFX1264-FAKE16-NEXT: s_lshl_b32 s12, 0xff, s119471; GFX1264-FAKE16-NEXT: s_and_b32 s2, s2, 0xff9472; GFX1264-FAKE16-NEXT: s_not_b32 s13, s129473; GFX1264-FAKE16-NEXT: s_lshl_b32 s14, s2, s119474; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x310160009475; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -19476; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x09477; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s39478; GFX1264-FAKE16-NEXT: s_mov_b64 s[2:3], 09479; GFX1264-FAKE16-NEXT: .LBB13_2: ; %atomicrmw.start9480; GFX1264-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=19481; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9482; GFX1264-FAKE16-NEXT: v_add_nc_u32_e32 v0, s14, v19483; GFX1264-FAKE16-NEXT: v_and_b32_e32 v0, s12, v09484; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)9485; GFX1264-FAKE16-NEXT: v_and_or_b32 v0, v1, s13, v09486; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v3, v19487; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v2, v09488; GFX1264-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS9489; GFX1264-FAKE16-NEXT: s_wait_loadcnt 0x09490; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v19491; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, v29492; GFX1264-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]9493; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)9494; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]9495; GFX1264-FAKE16-NEXT: s_cbranch_execnz .LBB13_29496; GFX1264-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end9497; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]9498; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s11, v29499; GFX1264-FAKE16-NEXT: .LBB13_4: ; %Flow9500; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[8:9]9501; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)9502; GFX1264-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v09503; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x09504; GFX1264-FAKE16-NEXT: s_mov_b32 s3, 0x310160009505; GFX1264-FAKE16-NEXT: v_readfirstlane_b32 s2, v09506; GFX1264-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)9507; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)9508; GFX1264-FAKE16-NEXT: v_mad_u16 v0, s10, v4, s29509; GFX1264-FAKE16-NEXT: s_mov_b32 s2, -19510; GFX1264-FAKE16-NEXT: buffer_store_b8 v0, off, s[0:3], null9511; GFX1264-FAKE16-NEXT: s_endpgm9512;9513; GFX1232-TRUE16-LABEL: uniform_add_i8:9514; GFX1232-TRUE16: ; %bb.0:9515; GFX1232-TRUE16-NEXT: s_clause 0x19516; GFX1232-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x249517; GFX1232-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x349518; GFX1232-TRUE16-NEXT: s_mov_b32 s6, exec_lo9519; GFX1232-TRUE16-NEXT: s_mov_b32 s10, 09520; GFX1232-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 09521; GFX1232-TRUE16-NEXT: s_mov_b32 s9, exec_lo9522; GFX1232-TRUE16-NEXT: ; implicit-def: $vgpr0_lo169523; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)9524; GFX1232-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v49525; GFX1232-TRUE16-NEXT: s_cbranch_execz .LBB13_49526; GFX1232-TRUE16-NEXT: ; %bb.1:9527; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x09528; GFX1232-TRUE16-NEXT: s_and_b32 s4, s2, -49529; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s39530; GFX1232-TRUE16-NEXT: s_and_b32 s2, s2, 39531; GFX1232-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x09532; GFX1232-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s69533; GFX1232-TRUE16-NEXT: s_lshl_b32 s2, s2, 39534; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)9535; GFX1232-TRUE16-NEXT: s_mul_i32 s6, s8, s69536; GFX1232-TRUE16-NEXT: s_lshl_b32 s3, 0xff, s29537; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)9538; GFX1232-TRUE16-NEXT: s_and_b32 s6, s6, 0xff9539; GFX1232-TRUE16-NEXT: s_not_b32 s11, s39540; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)9541; GFX1232-TRUE16-NEXT: s_lshl_b32 s12, s6, s29542; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -19543; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x09544; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s79545; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x310160009546; GFX1232-TRUE16-NEXT: .LBB13_2: ; %atomicrmw.start9547; GFX1232-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=19548; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9549; GFX1232-TRUE16-NEXT: v_add_nc_u32_e32 v0, s12, v19550; GFX1232-TRUE16-NEXT: v_and_b32_e32 v0, s3, v09551; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9552; GFX1232-TRUE16-NEXT: v_and_or_b32 v0, v1, s11, v09553; GFX1232-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v09554; GFX1232-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS9555; GFX1232-TRUE16-NEXT: s_wait_loadcnt 0x09556; GFX1232-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v19557; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, v29558; GFX1232-TRUE16-NEXT: s_or_b32 s10, vcc_lo, s109559; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)9560; GFX1232-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s109561; GFX1232-TRUE16-NEXT: s_cbranch_execnz .LBB13_29562; GFX1232-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end9563; GFX1232-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s109564; GFX1232-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v29565; GFX1232-TRUE16-NEXT: .LBB13_4: ; %Flow9566; GFX1232-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s99567; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)9568; GFX1232-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v09569; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x09570; GFX1232-TRUE16-NEXT: s_mov_b32 s3, 0x310160009571; GFX1232-TRUE16-NEXT: v_readfirstlane_b32 s2, v09572; GFX1232-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)9573; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)9574; GFX1232-TRUE16-NEXT: v_mad_u16 v0.l, s8, v4.l, s29575; GFX1232-TRUE16-NEXT: s_mov_b32 s2, -19576; GFX1232-TRUE16-NEXT: buffer_store_b8 v0, off, s[0:3], null9577; GFX1232-TRUE16-NEXT: s_endpgm9578;9579; GFX1232-FAKE16-LABEL: uniform_add_i8:9580; GFX1232-FAKE16: ; %bb.0:9581; GFX1232-FAKE16-NEXT: s_clause 0x19582; GFX1232-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x249583; GFX1232-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x349584; GFX1232-FAKE16-NEXT: s_mov_b32 s6, exec_lo9585; GFX1232-FAKE16-NEXT: s_mov_b32 s10, 09586; GFX1232-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 09587; GFX1232-FAKE16-NEXT: s_mov_b32 s9, exec_lo9588; GFX1232-FAKE16-NEXT: ; implicit-def: $vgpr09589; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)9590; GFX1232-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v49591; GFX1232-FAKE16-NEXT: s_cbranch_execz .LBB13_49592; GFX1232-FAKE16-NEXT: ; %bb.1:9593; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x09594; GFX1232-FAKE16-NEXT: s_and_b32 s4, s2, -49595; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s39596; GFX1232-FAKE16-NEXT: s_and_b32 s2, s2, 39597; GFX1232-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x09598; GFX1232-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s69599; GFX1232-FAKE16-NEXT: s_lshl_b32 s2, s2, 39600; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)9601; GFX1232-FAKE16-NEXT: s_mul_i32 s6, s8, s69602; GFX1232-FAKE16-NEXT: s_lshl_b32 s3, 0xff, s29603; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)9604; GFX1232-FAKE16-NEXT: s_and_b32 s6, s6, 0xff9605; GFX1232-FAKE16-NEXT: s_not_b32 s11, s39606; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)9607; GFX1232-FAKE16-NEXT: s_lshl_b32 s12, s6, s29608; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -19609; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x09610; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s79611; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x310160009612; GFX1232-FAKE16-NEXT: .LBB13_2: ; %atomicrmw.start9613; GFX1232-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=19614; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9615; GFX1232-FAKE16-NEXT: v_add_nc_u32_e32 v0, s12, v19616; GFX1232-FAKE16-NEXT: v_and_b32_e32 v0, s3, v09617; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9618; GFX1232-FAKE16-NEXT: v_and_or_b32 v0, v1, s11, v09619; GFX1232-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v09620; GFX1232-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS9621; GFX1232-FAKE16-NEXT: s_wait_loadcnt 0x09622; GFX1232-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v19623; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, v29624; GFX1232-FAKE16-NEXT: s_or_b32 s10, vcc_lo, s109625; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)9626; GFX1232-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s109627; GFX1232-FAKE16-NEXT: s_cbranch_execnz .LBB13_29628; GFX1232-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end9629; GFX1232-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s109630; GFX1232-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v29631; GFX1232-FAKE16-NEXT: .LBB13_4: ; %Flow9632; GFX1232-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s99633; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)9634; GFX1232-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v09635; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x09636; GFX1232-FAKE16-NEXT: s_mov_b32 s3, 0x310160009637; GFX1232-FAKE16-NEXT: v_readfirstlane_b32 s2, v09638; GFX1232-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)9639; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)9640; GFX1232-FAKE16-NEXT: v_mad_u16 v0, s8, v4, s29641; GFX1232-FAKE16-NEXT: s_mov_b32 s2, -19642; GFX1232-FAKE16-NEXT: buffer_store_b8 v0, off, s[0:3], null9643; GFX1232-FAKE16-NEXT: s_endpgm9644 %rmw = atomicrmw add ptr addrspace(1) %uniform.ptr, i8 %val monotonic, align 19645 store i8 %rmw, ptr addrspace(1) %result9646 ret void9647}9648 9649define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i8 %val) {9650; GFX7LESS-LABEL: uniform_xchg_i8:9651; GFX7LESS: ; %bb.0:9652; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x99653; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xd9654; GFX7LESS-NEXT: s_mov_b64 s[8:9], 09655; GFX7LESS-NEXT: s_mov_b32 s7, 0xf0009656; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)9657; GFX7LESS-NEXT: s_and_b32 s4, s2, -49658; GFX7LESS-NEXT: s_mov_b32 s5, s39659; GFX7LESS-NEXT: s_and_b32 s2, s2, 39660; GFX7LESS-NEXT: s_and_b32 s3, s6, 0xff9661; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0x09662; GFX7LESS-NEXT: s_lshl_b32 s10, s2, 39663; GFX7LESS-NEXT: s_lshl_b32 s11, 0xff, s109664; GFX7LESS-NEXT: s_lshl_b32 s2, s3, s109665; GFX7LESS-NEXT: s_not_b32 s3, s119666; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)9667; GFX7LESS-NEXT: v_mov_b32_e32 v1, s69668; GFX7LESS-NEXT: s_mov_b32 s6, -19669; GFX7LESS-NEXT: .LBB14_1: ; %atomicrmw.start9670; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=19671; GFX7LESS-NEXT: v_and_b32_e32 v0, s3, v19672; GFX7LESS-NEXT: v_or_b32_e32 v0, s2, v09673; GFX7LESS-NEXT: s_waitcnt expcnt(0)9674; GFX7LESS-NEXT: v_mov_b32_e32 v3, v19675; GFX7LESS-NEXT: v_mov_b32_e32 v2, v09676; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc9677; GFX7LESS-NEXT: s_waitcnt vmcnt(0)9678; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v19679; GFX7LESS-NEXT: s_or_b64 s[8:9], vcc, s[8:9]9680; GFX7LESS-NEXT: v_mov_b32_e32 v1, v29681; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[8:9]9682; GFX7LESS-NEXT: s_cbranch_execnz .LBB14_19683; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end9684; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]9685; GFX7LESS-NEXT: s_mov_b32 s3, 0xf0009686; GFX7LESS-NEXT: s_mov_b32 s2, -19687; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s10, v29688; GFX7LESS-NEXT: buffer_store_byte v0, off, s[0:3], 09689; GFX7LESS-NEXT: s_endpgm9690;9691; GFX8-LABEL: uniform_xchg_i8:9692; GFX8: ; %bb.0:9693; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249694; GFX8-NEXT: s_load_dword s6, s[4:5], 0x349695; GFX8-NEXT: s_waitcnt lgkmcnt(0)9696; GFX8-NEXT: s_and_b32 s4, s2, -49697; GFX8-NEXT: s_mov_b32 s5, s39698; GFX8-NEXT: s_load_dword s7, s[4:5], 0x09699; GFX8-NEXT: s_and_b32 s2, s2, 39700; GFX8-NEXT: s_lshl_b32 s8, s2, 39701; GFX8-NEXT: s_lshl_b32 s2, 0xff, s89702; GFX8-NEXT: s_not_b32 s9, s29703; GFX8-NEXT: s_and_b32 s2, s6, 0xff9704; GFX8-NEXT: s_lshl_b32 s10, s2, s89705; GFX8-NEXT: s_mov_b64 s[2:3], 09706; GFX8-NEXT: s_waitcnt lgkmcnt(0)9707; GFX8-NEXT: v_mov_b32_e32 v1, s79708; GFX8-NEXT: s_mov_b32 s7, 0xf0009709; GFX8-NEXT: s_mov_b32 s6, -19710; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start9711; GFX8-NEXT: ; =>This Inner Loop Header: Depth=19712; GFX8-NEXT: v_and_b32_e32 v0, s9, v19713; GFX8-NEXT: v_or_b32_e32 v0, s10, v09714; GFX8-NEXT: v_mov_b32_e32 v3, v19715; GFX8-NEXT: v_mov_b32_e32 v2, v09716; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc9717; GFX8-NEXT: s_waitcnt vmcnt(0)9718; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v19719; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]9720; GFX8-NEXT: v_mov_b32_e32 v1, v29721; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]9722; GFX8-NEXT: s_cbranch_execnz .LBB14_19723; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end9724; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]9725; GFX8-NEXT: s_mov_b32 s3, 0xf0009726; GFX8-NEXT: s_mov_b32 s2, -19727; GFX8-NEXT: v_lshrrev_b32_e32 v0, s8, v29728; GFX8-NEXT: buffer_store_byte v0, off, s[0:3], 09729; GFX8-NEXT: s_endpgm9730;9731; GFX9-LABEL: uniform_xchg_i8:9732; GFX9: ; %bb.0:9733; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249734; GFX9-NEXT: s_load_dword s6, s[4:5], 0x349735; GFX9-NEXT: s_waitcnt lgkmcnt(0)9736; GFX9-NEXT: s_and_b32 s4, s2, -49737; GFX9-NEXT: s_mov_b32 s5, s39738; GFX9-NEXT: s_load_dword s7, s[4:5], 0x09739; GFX9-NEXT: s_and_b32 s2, s2, 39740; GFX9-NEXT: s_lshl_b32 s8, s2, 39741; GFX9-NEXT: s_lshl_b32 s2, 0xff, s89742; GFX9-NEXT: s_not_b32 s9, s29743; GFX9-NEXT: s_and_b32 s2, s6, 0xff9744; GFX9-NEXT: s_lshl_b32 s10, s2, s89745; GFX9-NEXT: s_mov_b64 s[2:3], 09746; GFX9-NEXT: s_waitcnt lgkmcnt(0)9747; GFX9-NEXT: v_mov_b32_e32 v1, s79748; GFX9-NEXT: s_mov_b32 s7, 0xf0009749; GFX9-NEXT: s_mov_b32 s6, -19750; GFX9-NEXT: .LBB14_1: ; %atomicrmw.start9751; GFX9-NEXT: ; =>This Inner Loop Header: Depth=19752; GFX9-NEXT: v_and_b32_e32 v0, s9, v19753; GFX9-NEXT: v_or_b32_e32 v0, s10, v09754; GFX9-NEXT: v_mov_b32_e32 v3, v19755; GFX9-NEXT: v_mov_b32_e32 v2, v09756; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc9757; GFX9-NEXT: s_waitcnt vmcnt(0)9758; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v19759; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]9760; GFX9-NEXT: v_mov_b32_e32 v1, v29761; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]9762; GFX9-NEXT: s_cbranch_execnz .LBB14_19763; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end9764; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]9765; GFX9-NEXT: s_mov_b32 s3, 0xf0009766; GFX9-NEXT: s_mov_b32 s2, -19767; GFX9-NEXT: v_lshrrev_b32_e32 v0, s8, v29768; GFX9-NEXT: buffer_store_byte v0, off, s[0:3], 09769; GFX9-NEXT: s_endpgm9770;9771; GFX1064-LABEL: uniform_xchg_i8:9772; GFX1064: ; %bb.0:9773; GFX1064-NEXT: s_clause 0x19774; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249775; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x349776; GFX1064-NEXT: s_mov_b32 s7, 0x310160009777; GFX1064-NEXT: s_waitcnt lgkmcnt(0)9778; GFX1064-NEXT: s_and_b32 s4, s2, -49779; GFX1064-NEXT: s_mov_b32 s5, s39780; GFX1064-NEXT: s_and_b32 s2, s2, 39781; GFX1064-NEXT: s_load_dword s3, s[4:5], 0x09782; GFX1064-NEXT: s_lshl_b32 s8, s2, 39783; GFX1064-NEXT: s_and_b32 s6, s6, 0xff9784; GFX1064-NEXT: s_lshl_b32 s2, 0xff, s89785; GFX1064-NEXT: s_lshl_b32 s10, s6, s89786; GFX1064-NEXT: s_not_b32 s9, s29787; GFX1064-NEXT: s_mov_b32 s6, -19788; GFX1064-NEXT: s_waitcnt lgkmcnt(0)9789; GFX1064-NEXT: v_mov_b32_e32 v1, s39790; GFX1064-NEXT: s_mov_b64 s[2:3], 09791; GFX1064-NEXT: .LBB14_1: ; %atomicrmw.start9792; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=19793; GFX1064-NEXT: v_and_or_b32 v0, v1, s9, s109794; GFX1064-NEXT: v_mov_b32_e32 v3, v19795; GFX1064-NEXT: v_mov_b32_e32 v2, v09796; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc9797; GFX1064-NEXT: s_waitcnt vmcnt(0)9798; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v19799; GFX1064-NEXT: v_mov_b32_e32 v1, v29800; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]9801; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]9802; GFX1064-NEXT: s_cbranch_execnz .LBB14_19803; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end9804; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]9805; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s8, v29806; GFX1064-NEXT: s_mov_b32 s3, 0x310160009807; GFX1064-NEXT: s_mov_b32 s2, -19808; GFX1064-NEXT: buffer_store_byte v0, off, s[0:3], 09809; GFX1064-NEXT: s_endpgm9810;9811; GFX1032-LABEL: uniform_xchg_i8:9812; GFX1032: ; %bb.0:9813; GFX1032-NEXT: s_clause 0x19814; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x249815; GFX1032-NEXT: s_load_dword s6, s[4:5], 0x349816; GFX1032-NEXT: s_mov_b32 s9, 09817; GFX1032-NEXT: s_waitcnt lgkmcnt(0)9818; GFX1032-NEXT: s_and_b32 s4, s2, -49819; GFX1032-NEXT: s_mov_b32 s5, s39820; GFX1032-NEXT: s_and_b32 s2, s2, 39821; GFX1032-NEXT: s_load_dword s7, s[4:5], 0x09822; GFX1032-NEXT: s_lshl_b32 s2, s2, 39823; GFX1032-NEXT: s_and_b32 s6, s6, 0xff9824; GFX1032-NEXT: s_lshl_b32 s3, 0xff, s29825; GFX1032-NEXT: s_lshl_b32 s8, s6, s29826; GFX1032-NEXT: s_not_b32 s3, s39827; GFX1032-NEXT: s_mov_b32 s6, -19828; GFX1032-NEXT: s_waitcnt lgkmcnt(0)9829; GFX1032-NEXT: v_mov_b32_e32 v1, s79830; GFX1032-NEXT: s_mov_b32 s7, 0x310160009831; GFX1032-NEXT: .LBB14_1: ; %atomicrmw.start9832; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=19833; GFX1032-NEXT: v_and_or_b32 v0, v1, s3, s89834; GFX1032-NEXT: v_mov_b32_e32 v3, v19835; GFX1032-NEXT: v_mov_b32_e32 v2, v09836; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc9837; GFX1032-NEXT: s_waitcnt vmcnt(0)9838; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v19839; GFX1032-NEXT: v_mov_b32_e32 v1, v29840; GFX1032-NEXT: s_or_b32 s9, vcc_lo, s99841; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s99842; GFX1032-NEXT: s_cbranch_execnz .LBB14_19843; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end9844; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s99845; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s2, v29846; GFX1032-NEXT: s_mov_b32 s3, 0x310160009847; GFX1032-NEXT: s_mov_b32 s2, -19848; GFX1032-NEXT: buffer_store_byte v0, off, s[0:3], 09849; GFX1032-NEXT: s_endpgm9850;9851; GFX1164-LABEL: uniform_xchg_i8:9852; GFX1164: ; %bb.0:9853; GFX1164-NEXT: s_clause 0x19854; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x249855; GFX1164-NEXT: s_load_b32 s6, s[4:5], 0x349856; GFX1164-NEXT: s_mov_b32 s7, 0x310160009857; GFX1164-NEXT: s_waitcnt lgkmcnt(0)9858; GFX1164-NEXT: s_and_b32 s4, s2, -49859; GFX1164-NEXT: s_mov_b32 s5, s39860; GFX1164-NEXT: s_and_b32 s2, s2, 39861; GFX1164-NEXT: s_load_b32 s3, s[4:5], 0x09862; GFX1164-NEXT: s_lshl_b32 s8, s2, 39863; GFX1164-NEXT: s_and_b32 s6, s6, 0xff9864; GFX1164-NEXT: s_lshl_b32 s2, 0xff, s89865; GFX1164-NEXT: s_lshl_b32 s10, s6, s89866; GFX1164-NEXT: s_not_b32 s9, s29867; GFX1164-NEXT: s_mov_b32 s6, -19868; GFX1164-NEXT: s_waitcnt lgkmcnt(0)9869; GFX1164-NEXT: v_mov_b32_e32 v1, s39870; GFX1164-NEXT: s_mov_b64 s[2:3], 09871; GFX1164-NEXT: .LBB14_1: ; %atomicrmw.start9872; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=19873; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)9874; GFX1164-NEXT: v_and_or_b32 v0, v1, s9, s109875; GFX1164-NEXT: v_mov_b32_e32 v3, v19876; GFX1164-NEXT: v_mov_b32_e32 v2, v09877; GFX1164-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc9878; GFX1164-NEXT: s_waitcnt vmcnt(0)9879; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v2, v19880; GFX1164-NEXT: v_mov_b32_e32 v1, v29881; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]9882; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)9883; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]9884; GFX1164-NEXT: s_cbranch_execnz .LBB14_19885; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end9886; GFX1164-NEXT: s_or_b64 exec, exec, s[2:3]9887; GFX1164-NEXT: v_lshrrev_b32_e32 v0, s8, v29888; GFX1164-NEXT: s_mov_b32 s3, 0x310160009889; GFX1164-NEXT: s_mov_b32 s2, -19890; GFX1164-NEXT: buffer_store_b8 v0, off, s[0:3], 09891; GFX1164-NEXT: s_endpgm9892;9893; GFX1132-LABEL: uniform_xchg_i8:9894; GFX1132: ; %bb.0:9895; GFX1132-NEXT: s_clause 0x19896; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x249897; GFX1132-NEXT: s_load_b32 s6, s[4:5], 0x349898; GFX1132-NEXT: s_mov_b32 s9, 09899; GFX1132-NEXT: s_waitcnt lgkmcnt(0)9900; GFX1132-NEXT: s_and_b32 s4, s2, -49901; GFX1132-NEXT: s_mov_b32 s5, s39902; GFX1132-NEXT: s_and_b32 s2, s2, 39903; GFX1132-NEXT: s_load_b32 s7, s[4:5], 0x09904; GFX1132-NEXT: s_lshl_b32 s2, s2, 39905; GFX1132-NEXT: s_and_b32 s6, s6, 0xff9906; GFX1132-NEXT: s_lshl_b32 s3, 0xff, s29907; GFX1132-NEXT: s_lshl_b32 s8, s6, s29908; GFX1132-NEXT: s_not_b32 s3, s39909; GFX1132-NEXT: s_mov_b32 s6, -19910; GFX1132-NEXT: s_waitcnt lgkmcnt(0)9911; GFX1132-NEXT: v_mov_b32_e32 v1, s79912; GFX1132-NEXT: s_mov_b32 s7, 0x310160009913; GFX1132-NEXT: .LBB14_1: ; %atomicrmw.start9914; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=19915; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9916; GFX1132-NEXT: v_and_or_b32 v0, v1, s3, s89917; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v09918; GFX1132-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc9919; GFX1132-NEXT: s_waitcnt vmcnt(0)9920; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v19921; GFX1132-NEXT: v_mov_b32_e32 v1, v29922; GFX1132-NEXT: s_or_b32 s9, vcc_lo, s99923; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)9924; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s99925; GFX1132-NEXT: s_cbranch_execnz .LBB14_19926; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end9927; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s99928; GFX1132-NEXT: v_lshrrev_b32_e32 v0, s2, v29929; GFX1132-NEXT: s_mov_b32 s3, 0x310160009930; GFX1132-NEXT: s_mov_b32 s2, -19931; GFX1132-NEXT: buffer_store_b8 v0, off, s[0:3], 09932; GFX1132-NEXT: s_endpgm9933;9934; GFX1264-LABEL: uniform_xchg_i8:9935; GFX1264: ; %bb.0:9936; GFX1264-NEXT: s_clause 0x19937; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x249938; GFX1264-NEXT: s_load_b32 s6, s[4:5], 0x349939; GFX1264-NEXT: s_mov_b32 s7, 0x310160009940; GFX1264-NEXT: s_wait_kmcnt 0x09941; GFX1264-NEXT: s_and_b32 s4, s2, -49942; GFX1264-NEXT: s_mov_b32 s5, s39943; GFX1264-NEXT: s_and_b32 s2, s2, 39944; GFX1264-NEXT: s_load_b32 s3, s[4:5], 0x09945; GFX1264-NEXT: s_lshl_b32 s8, s2, 39946; GFX1264-NEXT: s_and_b32 s6, s6, 0xff9947; GFX1264-NEXT: s_lshl_b32 s2, 0xff, s89948; GFX1264-NEXT: s_lshl_b32 s10, s6, s89949; GFX1264-NEXT: s_not_b32 s9, s29950; GFX1264-NEXT: s_mov_b32 s6, -19951; GFX1264-NEXT: s_wait_kmcnt 0x09952; GFX1264-NEXT: v_mov_b32_e32 v1, s39953; GFX1264-NEXT: s_mov_b64 s[2:3], 09954; GFX1264-NEXT: .LBB14_1: ; %atomicrmw.start9955; GFX1264-NEXT: ; =>This Inner Loop Header: Depth=19956; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)9957; GFX1264-NEXT: v_and_or_b32 v0, v1, s9, s109958; GFX1264-NEXT: v_mov_b32_e32 v3, v19959; GFX1264-NEXT: v_mov_b32_e32 v2, v09960; GFX1264-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS9961; GFX1264-NEXT: s_wait_loadcnt 0x09962; GFX1264-NEXT: v_cmp_eq_u32_e32 vcc, v2, v19963; GFX1264-NEXT: v_mov_b32_e32 v1, v29964; GFX1264-NEXT: s_or_b64 s[2:3], vcc, s[2:3]9965; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)9966; GFX1264-NEXT: s_and_not1_b64 exec, exec, s[2:3]9967; GFX1264-NEXT: s_cbranch_execnz .LBB14_19968; GFX1264-NEXT: ; %bb.2: ; %atomicrmw.end9969; GFX1264-NEXT: s_or_b64 exec, exec, s[2:3]9970; GFX1264-NEXT: v_lshrrev_b32_e32 v0, s8, v29971; GFX1264-NEXT: s_mov_b32 s3, 0x310160009972; GFX1264-NEXT: s_mov_b32 s2, -19973; GFX1264-NEXT: buffer_store_b8 v0, off, s[0:3], null9974; GFX1264-NEXT: s_endpgm9975;9976; GFX1232-LABEL: uniform_xchg_i8:9977; GFX1232: ; %bb.0:9978; GFX1232-NEXT: s_clause 0x19979; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x249980; GFX1232-NEXT: s_load_b32 s6, s[4:5], 0x349981; GFX1232-NEXT: s_mov_b32 s9, 09982; GFX1232-NEXT: s_wait_kmcnt 0x09983; GFX1232-NEXT: s_and_b32 s4, s2, -49984; GFX1232-NEXT: s_mov_b32 s5, s39985; GFX1232-NEXT: s_and_b32 s2, s2, 39986; GFX1232-NEXT: s_load_b32 s7, s[4:5], 0x09987; GFX1232-NEXT: s_lshl_b32 s2, s2, 39988; GFX1232-NEXT: s_and_b32 s6, s6, 0xff9989; GFX1232-NEXT: s_lshl_b32 s3, 0xff, s29990; GFX1232-NEXT: s_lshl_b32 s8, s6, s29991; GFX1232-NEXT: s_not_b32 s3, s39992; GFX1232-NEXT: s_mov_b32 s6, -19993; GFX1232-NEXT: s_wait_kmcnt 0x09994; GFX1232-NEXT: v_mov_b32_e32 v1, s79995; GFX1232-NEXT: s_mov_b32 s7, 0x310160009996; GFX1232-NEXT: .LBB14_1: ; %atomicrmw.start9997; GFX1232-NEXT: ; =>This Inner Loop Header: Depth=19998; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9999; GFX1232-NEXT: v_and_or_b32 v0, v1, s3, s810000; GFX1232-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v010001; GFX1232-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS10002; GFX1232-NEXT: s_wait_loadcnt 0x010003; GFX1232-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v110004; GFX1232-NEXT: v_mov_b32_e32 v1, v210005; GFX1232-NEXT: s_or_b32 s9, vcc_lo, s910006; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)10007; GFX1232-NEXT: s_and_not1_b32 exec_lo, exec_lo, s910008; GFX1232-NEXT: s_cbranch_execnz .LBB14_110009; GFX1232-NEXT: ; %bb.2: ; %atomicrmw.end10010; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s910011; GFX1232-NEXT: v_lshrrev_b32_e32 v0, s2, v210012; GFX1232-NEXT: s_mov_b32 s3, 0x3101600010013; GFX1232-NEXT: s_mov_b32 s2, -110014; GFX1232-NEXT: buffer_store_b8 v0, off, s[0:3], null10015; GFX1232-NEXT: s_endpgm10016 %rmw = atomicrmw xchg ptr addrspace(1) %uniform.ptr, i8 %val monotonic, align 110017 store i8 %rmw, ptr addrspace(1) %result10018 ret void10019}10020 10021define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i16 %val) {10022; GFX7LESS-LABEL: uniform_or_i16:10023; GFX7LESS: ; %bb.0:10024; GFX7LESS-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x910025; GFX7LESS-NEXT: s_load_dword s12, s[4:5], 0xd10026; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 010027; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v010028; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v010029; GFX7LESS-NEXT: ; implicit-def: $vgpr010030; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc10031; GFX7LESS-NEXT: s_cbranch_execz .LBB15_410032; GFX7LESS-NEXT: ; %bb.1:10033; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)10034; GFX7LESS-NEXT: s_and_b32 s4, s10, -410035; GFX7LESS-NEXT: s_mov_b32 s5, s1110036; GFX7LESS-NEXT: s_and_b32 s0, s10, 310037; GFX7LESS-NEXT: s_and_b32 s1, s12, 0xffff10038; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0x010039; GFX7LESS-NEXT: s_mov_b64 s[10:11], 010040; GFX7LESS-NEXT: s_mov_b32 s7, 0xf00010041; GFX7LESS-NEXT: s_lshl_b32 s13, s0, 310042; GFX7LESS-NEXT: s_lshl_b32 s14, s1, s1310043; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)10044; GFX7LESS-NEXT: v_mov_b32_e32 v1, s610045; GFX7LESS-NEXT: s_mov_b32 s6, -110046; GFX7LESS-NEXT: .LBB15_2: ; %atomicrmw.start10047; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=110048; GFX7LESS-NEXT: v_or_b32_e32 v0, s14, v110049; GFX7LESS-NEXT: s_waitcnt expcnt(0)10050; GFX7LESS-NEXT: v_mov_b32_e32 v3, v110051; GFX7LESS-NEXT: v_mov_b32_e32 v2, v010052; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc10053; GFX7LESS-NEXT: s_waitcnt vmcnt(0)10054; GFX7LESS-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v110055; GFX7LESS-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]10056; GFX7LESS-NEXT: v_mov_b32_e32 v1, v210057; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[10:11]10058; GFX7LESS-NEXT: s_cbranch_execnz .LBB15_210059; GFX7LESS-NEXT: ; %bb.3: ; %atomicrmw.end10060; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]10061; GFX7LESS-NEXT: v_bfe_u32 v0, v2, s13, 1610062; GFX7LESS-NEXT: .LBB15_4: ; %Flow10063; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3]10064; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)10065; GFX7LESS-NEXT: s_mov_b32 s11, 0xf00010066; GFX7LESS-NEXT: s_mov_b32 s10, -110067; GFX7LESS-NEXT: v_and_b32_e32 v0, 0xffff, v010068; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1210069; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v010070; GFX7LESS-NEXT: v_cndmask_b32_e64 v0, v1, 0, vcc10071; GFX7LESS-NEXT: v_or_b32_e32 v0, s0, v010072; GFX7LESS-NEXT: buffer_store_short v0, off, s[8:11], 010073; GFX7LESS-NEXT: s_endpgm10074;10075; GFX8-LABEL: uniform_or_i16:10076; GFX8: ; %bb.0:10077; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x2410078; GFX8-NEXT: s_load_dword s12, s[4:5], 0x3410079; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 010080; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v010081; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v010082; GFX8-NEXT: ; implicit-def: $vgpr010083; GFX8-NEXT: s_and_saveexec_b64 s[2:3], vcc10084; GFX8-NEXT: s_cbranch_execz .LBB15_410085; GFX8-NEXT: ; %bb.1:10086; GFX8-NEXT: s_waitcnt lgkmcnt(0)10087; GFX8-NEXT: s_and_b32 s4, s10, -410088; GFX8-NEXT: s_mov_b32 s5, s1110089; GFX8-NEXT: s_load_dword s1, s[4:5], 0x010090; GFX8-NEXT: s_and_b32 s0, s10, 310091; GFX8-NEXT: s_lshl_b32 s13, s0, 310092; GFX8-NEXT: s_and_b32 s0, 0xffff, s1210093; GFX8-NEXT: s_lshl_b32 s14, s0, s1310094; GFX8-NEXT: s_mov_b64 s[10:11], 010095; GFX8-NEXT: s_waitcnt lgkmcnt(0)10096; GFX8-NEXT: v_mov_b32_e32 v1, s110097; GFX8-NEXT: s_mov_b32 s7, 0xf00010098; GFX8-NEXT: s_mov_b32 s6, -110099; GFX8-NEXT: .LBB15_2: ; %atomicrmw.start10100; GFX8-NEXT: ; =>This Inner Loop Header: Depth=110101; GFX8-NEXT: v_or_b32_e32 v0, s14, v110102; GFX8-NEXT: v_mov_b32_e32 v3, v110103; GFX8-NEXT: v_mov_b32_e32 v2, v010104; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc10105; GFX8-NEXT: s_waitcnt vmcnt(0)10106; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v110107; GFX8-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]10108; GFX8-NEXT: v_mov_b32_e32 v1, v210109; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]10110; GFX8-NEXT: s_cbranch_execnz .LBB15_210111; GFX8-NEXT: ; %bb.3: ; %atomicrmw.end10112; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]10113; GFX8-NEXT: v_lshrrev_b32_e32 v0, s13, v210114; GFX8-NEXT: .LBB15_4: ; %Flow10115; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]10116; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v010117; GFX8-NEXT: v_readfirstlane_b32 s0, v010118; GFX8-NEXT: s_waitcnt lgkmcnt(0)10119; GFX8-NEXT: v_mov_b32_e32 v0, s1210120; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc10121; GFX8-NEXT: s_mov_b32 s11, 0xf00010122; GFX8-NEXT: s_mov_b32 s10, -110123; GFX8-NEXT: v_or_b32_e32 v0, s0, v010124; GFX8-NEXT: buffer_store_short v0, off, s[8:11], 010125; GFX8-NEXT: s_endpgm10126;10127; GFX9-LABEL: uniform_or_i16:10128; GFX9: ; %bb.0:10129; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x2410130; GFX9-NEXT: s_load_dword s12, s[4:5], 0x3410131; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 010132; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v010133; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v010134; GFX9-NEXT: ; implicit-def: $vgpr010135; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc10136; GFX9-NEXT: s_cbranch_execz .LBB15_410137; GFX9-NEXT: ; %bb.1:10138; GFX9-NEXT: s_waitcnt lgkmcnt(0)10139; GFX9-NEXT: s_and_b32 s4, s10, -410140; GFX9-NEXT: s_mov_b32 s5, s1110141; GFX9-NEXT: s_load_dword s1, s[4:5], 0x010142; GFX9-NEXT: s_and_b32 s0, s10, 310143; GFX9-NEXT: s_lshl_b32 s13, s0, 310144; GFX9-NEXT: s_and_b32 s0, 0xffff, s1210145; GFX9-NEXT: s_lshl_b32 s14, s0, s1310146; GFX9-NEXT: s_mov_b64 s[10:11], 010147; GFX9-NEXT: s_waitcnt lgkmcnt(0)10148; GFX9-NEXT: v_mov_b32_e32 v1, s110149; GFX9-NEXT: s_mov_b32 s7, 0xf00010150; GFX9-NEXT: s_mov_b32 s6, -110151; GFX9-NEXT: .LBB15_2: ; %atomicrmw.start10152; GFX9-NEXT: ; =>This Inner Loop Header: Depth=110153; GFX9-NEXT: v_or_b32_e32 v0, s14, v110154; GFX9-NEXT: v_mov_b32_e32 v3, v110155; GFX9-NEXT: v_mov_b32_e32 v2, v010156; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc10157; GFX9-NEXT: s_waitcnt vmcnt(0)10158; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v110159; GFX9-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]10160; GFX9-NEXT: v_mov_b32_e32 v1, v210161; GFX9-NEXT: s_andn2_b64 exec, exec, s[10:11]10162; GFX9-NEXT: s_cbranch_execnz .LBB15_210163; GFX9-NEXT: ; %bb.3: ; %atomicrmw.end10164; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]10165; GFX9-NEXT: v_lshrrev_b32_e32 v0, s13, v210166; GFX9-NEXT: .LBB15_4: ; %Flow10167; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]10168; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v010169; GFX9-NEXT: v_readfirstlane_b32 s0, v010170; GFX9-NEXT: s_waitcnt lgkmcnt(0)10171; GFX9-NEXT: v_mov_b32_e32 v0, s1210172; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc10173; GFX9-NEXT: s_mov_b32 s11, 0xf00010174; GFX9-NEXT: s_mov_b32 s10, -110175; GFX9-NEXT: v_or_b32_e32 v0, s0, v010176; GFX9-NEXT: buffer_store_short v0, off, s[8:11], 010177; GFX9-NEXT: s_endpgm10178;10179; GFX1064-LABEL: uniform_or_i16:10180; GFX1064: ; %bb.0:10181; GFX1064-NEXT: s_clause 0x110182; GFX1064-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x2410183; GFX1064-NEXT: s_load_dword s12, s[4:5], 0x3410184; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 010185; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v010186; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v010187; GFX1064-NEXT: ; implicit-def: $vgpr010188; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc10189; GFX1064-NEXT: s_cbranch_execz .LBB15_410190; GFX1064-NEXT: ; %bb.1:10191; GFX1064-NEXT: s_waitcnt lgkmcnt(0)10192; GFX1064-NEXT: s_and_b32 s4, s10, -410193; GFX1064-NEXT: s_mov_b32 s5, s1110194; GFX1064-NEXT: s_and_b32 s1, s10, 310195; GFX1064-NEXT: s_load_dword s0, s[4:5], 0x010196; GFX1064-NEXT: s_lshl_b32 s13, s1, 310197; GFX1064-NEXT: s_and_b32 s1, 0xffff, s1210198; GFX1064-NEXT: s_mov_b64 s[10:11], 010199; GFX1064-NEXT: s_lshl_b32 s14, s1, s1310200; GFX1064-NEXT: s_mov_b32 s7, 0x3101600010201; GFX1064-NEXT: s_mov_b32 s6, -110202; GFX1064-NEXT: s_waitcnt lgkmcnt(0)10203; GFX1064-NEXT: v_mov_b32_e32 v1, s010204; GFX1064-NEXT: .LBB15_2: ; %atomicrmw.start10205; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=110206; GFX1064-NEXT: v_or_b32_e32 v0, s14, v110207; GFX1064-NEXT: v_mov_b32_e32 v3, v110208; GFX1064-NEXT: v_mov_b32_e32 v2, v010209; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc10210; GFX1064-NEXT: s_waitcnt vmcnt(0)10211; GFX1064-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v110212; GFX1064-NEXT: v_mov_b32_e32 v1, v210213; GFX1064-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]10214; GFX1064-NEXT: s_andn2_b64 exec, exec, s[10:11]10215; GFX1064-NEXT: s_cbranch_execnz .LBB15_210216; GFX1064-NEXT: ; %bb.3: ; %atomicrmw.end10217; GFX1064-NEXT: s_or_b64 exec, exec, s[10:11]10218; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s13, v210219; GFX1064-NEXT: .LBB15_4: ; %Flow10220; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]10221; GFX1064-NEXT: v_and_b32_e32 v0, 0xffff, v010222; GFX1064-NEXT: s_waitcnt lgkmcnt(0)10223; GFX1064-NEXT: s_mov_b32 s11, 0x3101600010224; GFX1064-NEXT: s_mov_b32 s10, -110225; GFX1064-NEXT: v_readfirstlane_b32 s0, v010226; GFX1064-NEXT: v_cndmask_b32_e64 v0, s12, 0, vcc10227; GFX1064-NEXT: v_or_b32_e32 v0, s0, v010228; GFX1064-NEXT: buffer_store_short v0, off, s[8:11], 010229; GFX1064-NEXT: s_endpgm10230;10231; GFX1032-LABEL: uniform_or_i16:10232; GFX1032: ; %bb.0:10233; GFX1032-NEXT: s_clause 0x110234; GFX1032-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x2410235; GFX1032-NEXT: s_load_dword s1, s[4:5], 0x3410236; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 010237; GFX1032-NEXT: s_mov_b32 s3, 010238; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v010239; GFX1032-NEXT: ; implicit-def: $vgpr010240; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo10241; GFX1032-NEXT: s_cbranch_execz .LBB15_410242; GFX1032-NEXT: ; %bb.1:10243; GFX1032-NEXT: s_waitcnt lgkmcnt(0)10244; GFX1032-NEXT: s_and_b32 s4, s10, -410245; GFX1032-NEXT: s_mov_b32 s5, s1110246; GFX1032-NEXT: s_and_b32 s6, s10, 310247; GFX1032-NEXT: s_load_dword s0, s[4:5], 0x010248; GFX1032-NEXT: s_lshl_b32 s10, s6, 310249; GFX1032-NEXT: s_and_b32 s6, 0xffff, s110250; GFX1032-NEXT: s_mov_b32 s7, 0x3101600010251; GFX1032-NEXT: s_lshl_b32 s11, s6, s1010252; GFX1032-NEXT: s_mov_b32 s6, -110253; GFX1032-NEXT: s_waitcnt lgkmcnt(0)10254; GFX1032-NEXT: v_mov_b32_e32 v1, s010255; GFX1032-NEXT: .LBB15_2: ; %atomicrmw.start10256; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=110257; GFX1032-NEXT: v_or_b32_e32 v0, s11, v110258; GFX1032-NEXT: v_mov_b32_e32 v3, v110259; GFX1032-NEXT: v_mov_b32_e32 v2, v010260; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc10261; GFX1032-NEXT: s_waitcnt vmcnt(0)10262; GFX1032-NEXT: v_cmp_eq_u32_e64 s0, v2, v110263; GFX1032-NEXT: v_mov_b32_e32 v1, v210264; GFX1032-NEXT: s_or_b32 s3, s0, s310265; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s310266; GFX1032-NEXT: s_cbranch_execnz .LBB15_210267; GFX1032-NEXT: ; %bb.3: ; %atomicrmw.end10268; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s310269; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s10, v210270; GFX1032-NEXT: .LBB15_4: ; %Flow10271; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s210272; GFX1032-NEXT: v_and_b32_e32 v0, 0xffff, v010273; GFX1032-NEXT: s_waitcnt lgkmcnt(0)10274; GFX1032-NEXT: s_mov_b32 s11, 0x3101600010275; GFX1032-NEXT: s_mov_b32 s10, -110276; GFX1032-NEXT: v_readfirstlane_b32 s0, v010277; GFX1032-NEXT: v_cndmask_b32_e64 v0, s1, 0, vcc_lo10278; GFX1032-NEXT: v_or_b32_e32 v0, s0, v010279; GFX1032-NEXT: buffer_store_short v0, off, s[8:11], 010280; GFX1032-NEXT: s_endpgm10281;10282; GFX1164-TRUE16-LABEL: uniform_or_i16:10283; GFX1164-TRUE16: ; %bb.0:10284; GFX1164-TRUE16-NEXT: s_clause 0x110285; GFX1164-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x2410286; GFX1164-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x3410287; GFX1164-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 010288; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10289; GFX1164-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v010290; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, 0, v010291; GFX1164-TRUE16-NEXT: ; implicit-def: $vgpr0_lo1610292; GFX1164-TRUE16-NEXT: s_and_saveexec_b64 s[2:3], vcc10293; GFX1164-TRUE16-NEXT: s_cbranch_execz .LBB15_410294; GFX1164-TRUE16-NEXT: ; %bb.1:10295; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)10296; GFX1164-TRUE16-NEXT: s_and_b32 s4, s10, -410297; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s1110298; GFX1164-TRUE16-NEXT: s_and_b32 s1, s10, 310299; GFX1164-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x010300; GFX1164-TRUE16-NEXT: s_lshl_b32 s13, s1, 310301; GFX1164-TRUE16-NEXT: s_and_b32 s1, 0xffff, s1210302; GFX1164-TRUE16-NEXT: s_mov_b64 s[10:11], 010303; GFX1164-TRUE16-NEXT: s_lshl_b32 s14, s1, s1310304; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x3101600010305; GFX1164-TRUE16-NEXT: s_mov_b32 s6, -110306; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)10307; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s010308; GFX1164-TRUE16-NEXT: .LBB15_2: ; %atomicrmw.start10309; GFX1164-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=110310; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)10311; GFX1164-TRUE16-NEXT: v_or_b32_e32 v0, s14, v110312; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v3, v110313; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v2, v010314; GFX1164-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc10315; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)10316; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v110317; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v210318; GFX1164-TRUE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]10319; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)10320; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]10321; GFX1164-TRUE16-NEXT: s_cbranch_execnz .LBB15_210322; GFX1164-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end10323; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[10:11]10324; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s13, v210325; GFX1164-TRUE16-NEXT: .LBB15_4: ; %Flow10326; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]10327; GFX1164-TRUE16-NEXT: v_mov_b16_e32 v0.h, 010328; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)10329; GFX1164-TRUE16-NEXT: s_mov_b32 s11, 0x3101600010330; GFX1164-TRUE16-NEXT: s_mov_b32 s10, -110331; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)10332; GFX1164-TRUE16-NEXT: v_readfirstlane_b32 s0, v010333; GFX1164-TRUE16-NEXT: v_cndmask_b16 v0.l, s12, 0, vcc10334; GFX1164-TRUE16-NEXT: v_or_b16 v0.l, s0, v0.l10335; GFX1164-TRUE16-NEXT: buffer_store_b16 v0, off, s[8:11], 010336; GFX1164-TRUE16-NEXT: s_endpgm10337;10338; GFX1164-FAKE16-LABEL: uniform_or_i16:10339; GFX1164-FAKE16: ; %bb.0:10340; GFX1164-FAKE16-NEXT: s_clause 0x110341; GFX1164-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x2410342; GFX1164-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x3410343; GFX1164-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 010344; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10345; GFX1164-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v010346; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, 0, v010347; GFX1164-FAKE16-NEXT: ; implicit-def: $vgpr010348; GFX1164-FAKE16-NEXT: s_and_saveexec_b64 s[2:3], vcc10349; GFX1164-FAKE16-NEXT: s_cbranch_execz .LBB15_410350; GFX1164-FAKE16-NEXT: ; %bb.1:10351; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)10352; GFX1164-FAKE16-NEXT: s_and_b32 s4, s10, -410353; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s1110354; GFX1164-FAKE16-NEXT: s_and_b32 s1, s10, 310355; GFX1164-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x010356; GFX1164-FAKE16-NEXT: s_lshl_b32 s13, s1, 310357; GFX1164-FAKE16-NEXT: s_and_b32 s1, 0xffff, s1210358; GFX1164-FAKE16-NEXT: s_mov_b64 s[10:11], 010359; GFX1164-FAKE16-NEXT: s_lshl_b32 s14, s1, s1310360; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x3101600010361; GFX1164-FAKE16-NEXT: s_mov_b32 s6, -110362; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)10363; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s010364; GFX1164-FAKE16-NEXT: .LBB15_2: ; %atomicrmw.start10365; GFX1164-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=110366; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)10367; GFX1164-FAKE16-NEXT: v_or_b32_e32 v0, s14, v110368; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v3, v110369; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v2, v010370; GFX1164-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc10371; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)10372; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v110373; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v210374; GFX1164-FAKE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]10375; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)10376; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]10377; GFX1164-FAKE16-NEXT: s_cbranch_execnz .LBB15_210378; GFX1164-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end10379; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[10:11]10380; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s13, v210381; GFX1164-FAKE16-NEXT: .LBB15_4: ; %Flow10382; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]10383; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)10384; GFX1164-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v010385; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)10386; GFX1164-FAKE16-NEXT: s_mov_b32 s11, 0x3101600010387; GFX1164-FAKE16-NEXT: s_mov_b32 s10, -110388; GFX1164-FAKE16-NEXT: v_readfirstlane_b32 s0, v010389; GFX1164-FAKE16-NEXT: v_cndmask_b32_e64 v0, s12, 0, vcc10390; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)10391; GFX1164-FAKE16-NEXT: v_or_b32_e32 v0, s0, v010392; GFX1164-FAKE16-NEXT: buffer_store_b16 v0, off, s[8:11], 010393; GFX1164-FAKE16-NEXT: s_endpgm10394;10395; GFX1132-TRUE16-LABEL: uniform_or_i16:10396; GFX1132-TRUE16: ; %bb.0:10397; GFX1132-TRUE16-NEXT: s_clause 0x110398; GFX1132-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x2410399; GFX1132-TRUE16-NEXT: s_load_b32 s1, s[4:5], 0x3410400; GFX1132-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 010401; GFX1132-TRUE16-NEXT: s_mov_b32 s3, 010402; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)10403; GFX1132-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v010404; GFX1132-TRUE16-NEXT: ; implicit-def: $vgpr0_lo1610405; GFX1132-TRUE16-NEXT: s_and_saveexec_b32 s2, vcc_lo10406; GFX1132-TRUE16-NEXT: s_cbranch_execz .LBB15_410407; GFX1132-TRUE16-NEXT: ; %bb.1:10408; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)10409; GFX1132-TRUE16-NEXT: s_and_b32 s4, s10, -410410; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s1110411; GFX1132-TRUE16-NEXT: s_and_b32 s6, s10, 310412; GFX1132-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x010413; GFX1132-TRUE16-NEXT: s_lshl_b32 s10, s6, 310414; GFX1132-TRUE16-NEXT: s_and_b32 s6, 0xffff, s110415; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x3101600010416; GFX1132-TRUE16-NEXT: s_lshl_b32 s11, s6, s1010417; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -110418; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)10419; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s010420; GFX1132-TRUE16-NEXT: .LBB15_2: ; %atomicrmw.start10421; GFX1132-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=110422; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10423; GFX1132-TRUE16-NEXT: v_or_b32_e32 v0, s11, v110424; GFX1132-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v010425; GFX1132-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc10426; GFX1132-TRUE16-NEXT: s_waitcnt vmcnt(0)10427; GFX1132-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v110428; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, v210429; GFX1132-TRUE16-NEXT: s_or_b32 s3, s0, s310430; GFX1132-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)10431; GFX1132-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s310432; GFX1132-TRUE16-NEXT: s_cbranch_execnz .LBB15_210433; GFX1132-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end10434; GFX1132-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s310435; GFX1132-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s10, v210436; GFX1132-TRUE16-NEXT: .LBB15_4: ; %Flow10437; GFX1132-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s210438; GFX1132-TRUE16-NEXT: v_mov_b16_e32 v0.h, 010439; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)10440; GFX1132-TRUE16-NEXT: s_mov_b32 s11, 0x3101600010441; GFX1132-TRUE16-NEXT: s_mov_b32 s10, -110442; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)10443; GFX1132-TRUE16-NEXT: v_readfirstlane_b32 s0, v010444; GFX1132-TRUE16-NEXT: v_cndmask_b16 v0.l, s1, 0, vcc_lo10445; GFX1132-TRUE16-NEXT: v_or_b16 v0.l, s0, v0.l10446; GFX1132-TRUE16-NEXT: buffer_store_b16 v0, off, s[8:11], 010447; GFX1132-TRUE16-NEXT: s_endpgm10448;10449; GFX1132-FAKE16-LABEL: uniform_or_i16:10450; GFX1132-FAKE16: ; %bb.0:10451; GFX1132-FAKE16-NEXT: s_clause 0x110452; GFX1132-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x2410453; GFX1132-FAKE16-NEXT: s_load_b32 s1, s[4:5], 0x3410454; GFX1132-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 010455; GFX1132-FAKE16-NEXT: s_mov_b32 s3, 010456; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)10457; GFX1132-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v010458; GFX1132-FAKE16-NEXT: ; implicit-def: $vgpr010459; GFX1132-FAKE16-NEXT: s_and_saveexec_b32 s2, vcc_lo10460; GFX1132-FAKE16-NEXT: s_cbranch_execz .LBB15_410461; GFX1132-FAKE16-NEXT: ; %bb.1:10462; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)10463; GFX1132-FAKE16-NEXT: s_and_b32 s4, s10, -410464; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s1110465; GFX1132-FAKE16-NEXT: s_and_b32 s6, s10, 310466; GFX1132-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x010467; GFX1132-FAKE16-NEXT: s_lshl_b32 s10, s6, 310468; GFX1132-FAKE16-NEXT: s_and_b32 s6, 0xffff, s110469; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x3101600010470; GFX1132-FAKE16-NEXT: s_lshl_b32 s11, s6, s1010471; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -110472; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)10473; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s010474; GFX1132-FAKE16-NEXT: .LBB15_2: ; %atomicrmw.start10475; GFX1132-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=110476; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10477; GFX1132-FAKE16-NEXT: v_or_b32_e32 v0, s11, v110478; GFX1132-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v010479; GFX1132-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc10480; GFX1132-FAKE16-NEXT: s_waitcnt vmcnt(0)10481; GFX1132-FAKE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v110482; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, v210483; GFX1132-FAKE16-NEXT: s_or_b32 s3, s0, s310484; GFX1132-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)10485; GFX1132-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s310486; GFX1132-FAKE16-NEXT: s_cbranch_execnz .LBB15_210487; GFX1132-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end10488; GFX1132-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s310489; GFX1132-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s10, v210490; GFX1132-FAKE16-NEXT: .LBB15_4: ; %Flow10491; GFX1132-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s210492; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)10493; GFX1132-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v010494; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)10495; GFX1132-FAKE16-NEXT: s_mov_b32 s11, 0x3101600010496; GFX1132-FAKE16-NEXT: s_mov_b32 s10, -110497; GFX1132-FAKE16-NEXT: v_readfirstlane_b32 s0, v010498; GFX1132-FAKE16-NEXT: v_cndmask_b32_e64 v0, s1, 0, vcc_lo10499; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)10500; GFX1132-FAKE16-NEXT: v_or_b32_e32 v0, s0, v010501; GFX1132-FAKE16-NEXT: buffer_store_b16 v0, off, s[8:11], 010502; GFX1132-FAKE16-NEXT: s_endpgm10503;10504; GFX1264-TRUE16-LABEL: uniform_or_i16:10505; GFX1264-TRUE16: ; %bb.0:10506; GFX1264-TRUE16-NEXT: s_clause 0x110507; GFX1264-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x2410508; GFX1264-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x3410509; GFX1264-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 010510; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10511; GFX1264-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v010512; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, 0, v010513; GFX1264-TRUE16-NEXT: ; implicit-def: $vgpr0_lo1610514; GFX1264-TRUE16-NEXT: s_and_saveexec_b64 s[2:3], vcc10515; GFX1264-TRUE16-NEXT: s_cbranch_execz .LBB15_410516; GFX1264-TRUE16-NEXT: ; %bb.1:10517; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x010518; GFX1264-TRUE16-NEXT: s_and_b32 s4, s10, -410519; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s1110520; GFX1264-TRUE16-NEXT: s_and_b32 s1, s10, 310521; GFX1264-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x010522; GFX1264-TRUE16-NEXT: s_lshl_b32 s13, s1, 310523; GFX1264-TRUE16-NEXT: s_and_b32 s1, 0xffff, s1210524; GFX1264-TRUE16-NEXT: s_mov_b64 s[10:11], 010525; GFX1264-TRUE16-NEXT: s_lshl_b32 s14, s1, s1310526; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x3101600010527; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -110528; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x010529; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s010530; GFX1264-TRUE16-NEXT: .LBB15_2: ; %atomicrmw.start10531; GFX1264-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=110532; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)10533; GFX1264-TRUE16-NEXT: v_or_b32_e32 v0, s14, v110534; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v3, v110535; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v2, v010536; GFX1264-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS10537; GFX1264-TRUE16-NEXT: s_wait_loadcnt 0x010538; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v110539; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, v210540; GFX1264-TRUE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]10541; GFX1264-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)10542; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]10543; GFX1264-TRUE16-NEXT: s_cbranch_execnz .LBB15_210544; GFX1264-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end10545; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[10:11]10546; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s13, v210547; GFX1264-TRUE16-NEXT: .LBB15_4: ; %Flow10548; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]10549; GFX1264-TRUE16-NEXT: v_mov_b16_e32 v0.h, 010550; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x010551; GFX1264-TRUE16-NEXT: s_mov_b32 s11, 0x3101600010552; GFX1264-TRUE16-NEXT: s_mov_b32 s10, -110553; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)10554; GFX1264-TRUE16-NEXT: v_readfirstlane_b32 s0, v010555; GFX1264-TRUE16-NEXT: v_cndmask_b16 v0.l, s12, 0, vcc10556; GFX1264-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)10557; GFX1264-TRUE16-NEXT: v_or_b16 v0.l, s0, v0.l10558; GFX1264-TRUE16-NEXT: buffer_store_b16 v0, off, s[8:11], null10559; GFX1264-TRUE16-NEXT: s_endpgm10560;10561; GFX1264-FAKE16-LABEL: uniform_or_i16:10562; GFX1264-FAKE16: ; %bb.0:10563; GFX1264-FAKE16-NEXT: s_clause 0x110564; GFX1264-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x2410565; GFX1264-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x3410566; GFX1264-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 010567; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10568; GFX1264-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v010569; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, 0, v010570; GFX1264-FAKE16-NEXT: ; implicit-def: $vgpr010571; GFX1264-FAKE16-NEXT: s_and_saveexec_b64 s[2:3], vcc10572; GFX1264-FAKE16-NEXT: s_cbranch_execz .LBB15_410573; GFX1264-FAKE16-NEXT: ; %bb.1:10574; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x010575; GFX1264-FAKE16-NEXT: s_and_b32 s4, s10, -410576; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s1110577; GFX1264-FAKE16-NEXT: s_and_b32 s1, s10, 310578; GFX1264-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x010579; GFX1264-FAKE16-NEXT: s_lshl_b32 s13, s1, 310580; GFX1264-FAKE16-NEXT: s_and_b32 s1, 0xffff, s1210581; GFX1264-FAKE16-NEXT: s_mov_b64 s[10:11], 010582; GFX1264-FAKE16-NEXT: s_lshl_b32 s14, s1, s1310583; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x3101600010584; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -110585; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x010586; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s010587; GFX1264-FAKE16-NEXT: .LBB15_2: ; %atomicrmw.start10588; GFX1264-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=110589; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)10590; GFX1264-FAKE16-NEXT: v_or_b32_e32 v0, s14, v110591; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v3, v110592; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v2, v010593; GFX1264-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS10594; GFX1264-FAKE16-NEXT: s_wait_loadcnt 0x010595; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v110596; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, v210597; GFX1264-FAKE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]10598; GFX1264-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)10599; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]10600; GFX1264-FAKE16-NEXT: s_cbranch_execnz .LBB15_210601; GFX1264-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end10602; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[10:11]10603; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s13, v210604; GFX1264-FAKE16-NEXT: .LBB15_4: ; %Flow10605; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]10606; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)10607; GFX1264-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v010608; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x010609; GFX1264-FAKE16-NEXT: s_mov_b32 s11, 0x3101600010610; GFX1264-FAKE16-NEXT: s_mov_b32 s10, -110611; GFX1264-FAKE16-NEXT: v_readfirstlane_b32 s0, v010612; GFX1264-FAKE16-NEXT: v_cndmask_b32_e64 v0, s12, 0, vcc10613; GFX1264-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)10614; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)10615; GFX1264-FAKE16-NEXT: v_or_b32_e32 v0, s0, v010616; GFX1264-FAKE16-NEXT: buffer_store_b16 v0, off, s[8:11], null10617; GFX1264-FAKE16-NEXT: s_endpgm10618;10619; GFX1232-TRUE16-LABEL: uniform_or_i16:10620; GFX1232-TRUE16: ; %bb.0:10621; GFX1232-TRUE16-NEXT: s_clause 0x110622; GFX1232-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x2410623; GFX1232-TRUE16-NEXT: s_load_b32 s1, s[4:5], 0x3410624; GFX1232-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 010625; GFX1232-TRUE16-NEXT: s_mov_b32 s3, 010626; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)10627; GFX1232-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v010628; GFX1232-TRUE16-NEXT: ; implicit-def: $vgpr0_lo1610629; GFX1232-TRUE16-NEXT: s_and_saveexec_b32 s2, vcc_lo10630; GFX1232-TRUE16-NEXT: s_cbranch_execz .LBB15_410631; GFX1232-TRUE16-NEXT: ; %bb.1:10632; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x010633; GFX1232-TRUE16-NEXT: s_and_b32 s4, s10, -410634; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s1110635; GFX1232-TRUE16-NEXT: s_and_b32 s6, s10, 310636; GFX1232-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x010637; GFX1232-TRUE16-NEXT: s_lshl_b32 s10, s6, 310638; GFX1232-TRUE16-NEXT: s_and_b32 s6, 0xffff, s110639; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x3101600010640; GFX1232-TRUE16-NEXT: s_lshl_b32 s11, s6, s1010641; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -110642; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x010643; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s010644; GFX1232-TRUE16-NEXT: .LBB15_2: ; %atomicrmw.start10645; GFX1232-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=110646; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10647; GFX1232-TRUE16-NEXT: v_or_b32_e32 v0, s11, v110648; GFX1232-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v010649; GFX1232-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS10650; GFX1232-TRUE16-NEXT: s_wait_loadcnt 0x010651; GFX1232-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v110652; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, v210653; GFX1232-TRUE16-NEXT: s_or_b32 s3, s0, s310654; GFX1232-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)10655; GFX1232-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s310656; GFX1232-TRUE16-NEXT: s_cbranch_execnz .LBB15_210657; GFX1232-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end10658; GFX1232-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s310659; GFX1232-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s10, v210660; GFX1232-TRUE16-NEXT: .LBB15_4: ; %Flow10661; GFX1232-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s210662; GFX1232-TRUE16-NEXT: v_mov_b16_e32 v0.h, 010663; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x010664; GFX1232-TRUE16-NEXT: s_mov_b32 s11, 0x3101600010665; GFX1232-TRUE16-NEXT: s_mov_b32 s10, -110666; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)10667; GFX1232-TRUE16-NEXT: v_readfirstlane_b32 s0, v010668; GFX1232-TRUE16-NEXT: v_cndmask_b16 v0.l, s1, 0, vcc_lo10669; GFX1232-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)10670; GFX1232-TRUE16-NEXT: v_or_b16 v0.l, s0, v0.l10671; GFX1232-TRUE16-NEXT: buffer_store_b16 v0, off, s[8:11], null10672; GFX1232-TRUE16-NEXT: s_endpgm10673;10674; GFX1232-FAKE16-LABEL: uniform_or_i16:10675; GFX1232-FAKE16: ; %bb.0:10676; GFX1232-FAKE16-NEXT: s_clause 0x110677; GFX1232-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x2410678; GFX1232-FAKE16-NEXT: s_load_b32 s1, s[4:5], 0x3410679; GFX1232-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 010680; GFX1232-FAKE16-NEXT: s_mov_b32 s3, 010681; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)10682; GFX1232-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v010683; GFX1232-FAKE16-NEXT: ; implicit-def: $vgpr010684; GFX1232-FAKE16-NEXT: s_and_saveexec_b32 s2, vcc_lo10685; GFX1232-FAKE16-NEXT: s_cbranch_execz .LBB15_410686; GFX1232-FAKE16-NEXT: ; %bb.1:10687; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x010688; GFX1232-FAKE16-NEXT: s_and_b32 s4, s10, -410689; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s1110690; GFX1232-FAKE16-NEXT: s_and_b32 s6, s10, 310691; GFX1232-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x010692; GFX1232-FAKE16-NEXT: s_lshl_b32 s10, s6, 310693; GFX1232-FAKE16-NEXT: s_and_b32 s6, 0xffff, s110694; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x3101600010695; GFX1232-FAKE16-NEXT: s_lshl_b32 s11, s6, s1010696; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -110697; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x010698; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s010699; GFX1232-FAKE16-NEXT: .LBB15_2: ; %atomicrmw.start10700; GFX1232-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=110701; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10702; GFX1232-FAKE16-NEXT: v_or_b32_e32 v0, s11, v110703; GFX1232-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v010704; GFX1232-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS10705; GFX1232-FAKE16-NEXT: s_wait_loadcnt 0x010706; GFX1232-FAKE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v110707; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, v210708; GFX1232-FAKE16-NEXT: s_or_b32 s3, s0, s310709; GFX1232-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)10710; GFX1232-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s310711; GFX1232-FAKE16-NEXT: s_cbranch_execnz .LBB15_210712; GFX1232-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end10713; GFX1232-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s310714; GFX1232-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s10, v210715; GFX1232-FAKE16-NEXT: .LBB15_4: ; %Flow10716; GFX1232-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s210717; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)10718; GFX1232-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v010719; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x010720; GFX1232-FAKE16-NEXT: s_mov_b32 s11, 0x3101600010721; GFX1232-FAKE16-NEXT: s_mov_b32 s10, -110722; GFX1232-FAKE16-NEXT: v_readfirstlane_b32 s0, v010723; GFX1232-FAKE16-NEXT: v_cndmask_b32_e64 v0, s1, 0, vcc_lo10724; GFX1232-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)10725; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)10726; GFX1232-FAKE16-NEXT: v_or_b32_e32 v0, s0, v010727; GFX1232-FAKE16-NEXT: buffer_store_b16 v0, off, s[8:11], null10728; GFX1232-FAKE16-NEXT: s_endpgm10729 %rmw = atomicrmw or ptr addrspace(1) %uniform.ptr, i16 %val monotonic, align 210730 store i16 %rmw, ptr addrspace(1) %result10731 ret void10732}10733 10734define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i16 %val) {10735; GFX7LESS-LABEL: uniform_add_i16:10736; GFX7LESS: ; %bb.0:10737; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec10738; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x910739; GFX7LESS-NEXT: s_load_dword s10, s[4:5], 0xd10740; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 010741; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, s7, v010742; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v410743; GFX7LESS-NEXT: ; implicit-def: $vgpr010744; GFX7LESS-NEXT: s_and_saveexec_b64 s[8:9], vcc10745; GFX7LESS-NEXT: s_cbranch_execz .LBB16_410746; GFX7LESS-NEXT: ; %bb.1:10747; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]10748; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)10749; GFX7LESS-NEXT: s_and_b32 s4, s2, -410750; GFX7LESS-NEXT: s_mov_b32 s5, s310751; GFX7LESS-NEXT: s_and_b32 s11, s2, 310752; GFX7LESS-NEXT: s_mov_b64 s[2:3], 010753; GFX7LESS-NEXT: s_mov_b32 s7, 0xf00010754; GFX7LESS-NEXT: s_mul_i32 s6, s10, s610755; GFX7LESS-NEXT: s_lshl_b32 s11, s11, 310756; GFX7LESS-NEXT: s_load_dword s15, s[4:5], 0x010757; GFX7LESS-NEXT: s_lshl_b32 s12, 0xffff, s1110758; GFX7LESS-NEXT: s_and_b32 s6, s6, 0xffff10759; GFX7LESS-NEXT: s_not_b32 s13, s1210760; GFX7LESS-NEXT: s_lshl_b32 s14, s6, s1110761; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)10762; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1510763; GFX7LESS-NEXT: s_mov_b32 s6, -110764; GFX7LESS-NEXT: .LBB16_2: ; %atomicrmw.start10765; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=110766; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s14, v110767; GFX7LESS-NEXT: v_and_b32_e32 v0, s12, v010768; GFX7LESS-NEXT: s_waitcnt expcnt(0)10769; GFX7LESS-NEXT: v_and_b32_e32 v2, s13, v110770; GFX7LESS-NEXT: v_or_b32_e32 v0, v2, v010771; GFX7LESS-NEXT: v_mov_b32_e32 v3, v110772; GFX7LESS-NEXT: v_mov_b32_e32 v2, v010773; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc10774; GFX7LESS-NEXT: s_waitcnt vmcnt(0)10775; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v110776; GFX7LESS-NEXT: s_or_b64 s[2:3], vcc, s[2:3]10777; GFX7LESS-NEXT: v_mov_b32_e32 v1, v210778; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[2:3]10779; GFX7LESS-NEXT: s_cbranch_execnz .LBB16_210780; GFX7LESS-NEXT: ; %bb.3: ; %atomicrmw.end10781; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3]10782; GFX7LESS-NEXT: v_bfe_u32 v0, v2, s11, 1610783; GFX7LESS-NEXT: .LBB16_4: ; %Flow10784; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]10785; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)10786; GFX7LESS-NEXT: s_mov_b32 s3, 0xf00010787; GFX7LESS-NEXT: s_mov_b32 s2, -110788; GFX7LESS-NEXT: v_and_b32_e32 v0, 0xffff, v010789; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v010790; GFX7LESS-NEXT: s_and_b32 s5, s10, 0xffff10791; GFX7LESS-NEXT: v_mov_b32_e32 v0, s410792; GFX7LESS-NEXT: v_mad_u32_u24 v0, s5, v4, v010793; GFX7LESS-NEXT: buffer_store_short v0, off, s[0:3], 010794; GFX7LESS-NEXT: s_endpgm10795;10796; GFX8-LABEL: uniform_add_i16:10797; GFX8: ; %bb.0:10798; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410799; GFX8-NEXT: s_load_dword s10, s[4:5], 0x3410800; GFX8-NEXT: s_mov_b64 s[6:7], exec10801; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 010802; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v010803; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v410804; GFX8-NEXT: ; implicit-def: $vgpr010805; GFX8-NEXT: s_and_saveexec_b64 s[8:9], vcc10806; GFX8-NEXT: s_cbranch_execz .LBB16_410807; GFX8-NEXT: ; %bb.1:10808; GFX8-NEXT: s_bcnt1_i32_b64 s4, s[6:7]10809; GFX8-NEXT: s_waitcnt lgkmcnt(0)10810; GFX8-NEXT: s_mul_i32 s6, s10, s410811; GFX8-NEXT: s_and_b32 s4, s2, -410812; GFX8-NEXT: s_mov_b32 s5, s310813; GFX8-NEXT: s_load_dword s7, s[4:5], 0x010814; GFX8-NEXT: s_and_b32 s2, s2, 310815; GFX8-NEXT: s_lshl_b32 s11, s2, 310816; GFX8-NEXT: s_lshl_b32 s12, 0xffff, s1110817; GFX8-NEXT: s_and_b32 s2, s6, 0xffff10818; GFX8-NEXT: s_not_b32 s13, s1210819; GFX8-NEXT: s_lshl_b32 s14, s2, s1110820; GFX8-NEXT: s_mov_b64 s[2:3], 010821; GFX8-NEXT: s_waitcnt lgkmcnt(0)10822; GFX8-NEXT: v_mov_b32_e32 v1, s710823; GFX8-NEXT: s_mov_b32 s7, 0xf00010824; GFX8-NEXT: s_mov_b32 s6, -110825; GFX8-NEXT: .LBB16_2: ; %atomicrmw.start10826; GFX8-NEXT: ; =>This Inner Loop Header: Depth=110827; GFX8-NEXT: v_add_u32_e32 v0, vcc, s14, v110828; GFX8-NEXT: v_and_b32_e32 v2, s13, v110829; GFX8-NEXT: v_and_b32_e32 v0, s12, v010830; GFX8-NEXT: v_or_b32_e32 v0, v2, v010831; GFX8-NEXT: v_mov_b32_e32 v3, v110832; GFX8-NEXT: v_mov_b32_e32 v2, v010833; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc10834; GFX8-NEXT: s_waitcnt vmcnt(0)10835; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v110836; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]10837; GFX8-NEXT: v_mov_b32_e32 v1, v210838; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]10839; GFX8-NEXT: s_cbranch_execnz .LBB16_210840; GFX8-NEXT: ; %bb.3: ; %atomicrmw.end10841; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]10842; GFX8-NEXT: v_lshrrev_b32_e32 v0, s11, v210843; GFX8-NEXT: .LBB16_4: ; %Flow10844; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]10845; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v010846; GFX8-NEXT: v_readfirstlane_b32 s4, v010847; GFX8-NEXT: v_mov_b32_e32 v0, s410848; GFX8-NEXT: s_waitcnt lgkmcnt(0)10849; GFX8-NEXT: s_mov_b32 s3, 0xf00010850; GFX8-NEXT: s_mov_b32 s2, -110851; GFX8-NEXT: v_mad_u16 v0, s10, v4, v010852; GFX8-NEXT: buffer_store_short v0, off, s[0:3], 010853; GFX8-NEXT: s_endpgm10854;10855; GFX9-LABEL: uniform_add_i16:10856; GFX9: ; %bb.0:10857; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410858; GFX9-NEXT: s_load_dword s10, s[4:5], 0x3410859; GFX9-NEXT: s_mov_b64 s[6:7], exec10860; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 010861; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v010862; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v410863; GFX9-NEXT: ; implicit-def: $vgpr010864; GFX9-NEXT: s_and_saveexec_b64 s[8:9], vcc10865; GFX9-NEXT: s_cbranch_execz .LBB16_410866; GFX9-NEXT: ; %bb.1:10867; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[6:7]10868; GFX9-NEXT: s_waitcnt lgkmcnt(0)10869; GFX9-NEXT: s_mul_i32 s6, s10, s410870; GFX9-NEXT: s_and_b32 s4, s2, -410871; GFX9-NEXT: s_mov_b32 s5, s310872; GFX9-NEXT: s_load_dword s7, s[4:5], 0x010873; GFX9-NEXT: s_and_b32 s2, s2, 310874; GFX9-NEXT: s_lshl_b32 s11, s2, 310875; GFX9-NEXT: s_lshl_b32 s12, 0xffff, s1110876; GFX9-NEXT: s_and_b32 s2, s6, 0xffff10877; GFX9-NEXT: s_not_b32 s13, s1210878; GFX9-NEXT: s_lshl_b32 s14, s2, s1110879; GFX9-NEXT: s_mov_b64 s[2:3], 010880; GFX9-NEXT: s_waitcnt lgkmcnt(0)10881; GFX9-NEXT: v_mov_b32_e32 v1, s710882; GFX9-NEXT: s_mov_b32 s7, 0xf00010883; GFX9-NEXT: s_mov_b32 s6, -110884; GFX9-NEXT: .LBB16_2: ; %atomicrmw.start10885; GFX9-NEXT: ; =>This Inner Loop Header: Depth=110886; GFX9-NEXT: v_add_u32_e32 v0, s14, v110887; GFX9-NEXT: v_and_b32_e32 v0, s12, v010888; GFX9-NEXT: v_and_or_b32 v0, v1, s13, v010889; GFX9-NEXT: v_mov_b32_e32 v3, v110890; GFX9-NEXT: v_mov_b32_e32 v2, v010891; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc10892; GFX9-NEXT: s_waitcnt vmcnt(0)10893; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v110894; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]10895; GFX9-NEXT: v_mov_b32_e32 v1, v210896; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]10897; GFX9-NEXT: s_cbranch_execnz .LBB16_210898; GFX9-NEXT: ; %bb.3: ; %atomicrmw.end10899; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]10900; GFX9-NEXT: v_lshrrev_b32_e32 v0, s11, v210901; GFX9-NEXT: .LBB16_4: ; %Flow10902; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]10903; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v010904; GFX9-NEXT: v_readfirstlane_b32 s4, v010905; GFX9-NEXT: v_mov_b32_e32 v0, s410906; GFX9-NEXT: s_waitcnt lgkmcnt(0)10907; GFX9-NEXT: s_mov_b32 s3, 0xf00010908; GFX9-NEXT: s_mov_b32 s2, -110909; GFX9-NEXT: v_mad_legacy_u16 v0, s10, v4, v010910; GFX9-NEXT: buffer_store_short v0, off, s[0:3], 010911; GFX9-NEXT: s_endpgm10912;10913; GFX1064-LABEL: uniform_add_i16:10914; GFX1064: ; %bb.0:10915; GFX1064-NEXT: s_clause 0x110916; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410917; GFX1064-NEXT: s_load_dword s10, s[4:5], 0x3410918; GFX1064-NEXT: s_mov_b64 s[6:7], exec10919; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 010920; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v010921; GFX1064-NEXT: ; implicit-def: $vgpr010922; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v410923; GFX1064-NEXT: s_and_saveexec_b64 s[8:9], vcc10924; GFX1064-NEXT: s_cbranch_execz .LBB16_410925; GFX1064-NEXT: ; %bb.1:10926; GFX1064-NEXT: s_waitcnt lgkmcnt(0)10927; GFX1064-NEXT: s_and_b32 s4, s2, -410928; GFX1064-NEXT: s_mov_b32 s5, s310929; GFX1064-NEXT: s_and_b32 s2, s2, 310930; GFX1064-NEXT: s_load_dword s3, s[4:5], 0x010931; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]10932; GFX1064-NEXT: s_lshl_b32 s11, s2, 310933; GFX1064-NEXT: s_mul_i32 s2, s10, s610934; GFX1064-NEXT: s_lshl_b32 s12, 0xffff, s1110935; GFX1064-NEXT: s_and_b32 s2, s2, 0xffff10936; GFX1064-NEXT: s_not_b32 s13, s1210937; GFX1064-NEXT: s_lshl_b32 s14, s2, s1110938; GFX1064-NEXT: s_mov_b32 s7, 0x3101600010939; GFX1064-NEXT: s_mov_b32 s6, -110940; GFX1064-NEXT: s_waitcnt lgkmcnt(0)10941; GFX1064-NEXT: v_mov_b32_e32 v1, s310942; GFX1064-NEXT: s_mov_b64 s[2:3], 010943; GFX1064-NEXT: .LBB16_2: ; %atomicrmw.start10944; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=110945; GFX1064-NEXT: v_add_nc_u32_e32 v0, s14, v110946; GFX1064-NEXT: v_and_b32_e32 v0, s12, v010947; GFX1064-NEXT: v_and_or_b32 v0, v1, s13, v010948; GFX1064-NEXT: v_mov_b32_e32 v3, v110949; GFX1064-NEXT: v_mov_b32_e32 v2, v010950; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc10951; GFX1064-NEXT: s_waitcnt vmcnt(0)10952; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v110953; GFX1064-NEXT: v_mov_b32_e32 v1, v210954; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]10955; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]10956; GFX1064-NEXT: s_cbranch_execnz .LBB16_210957; GFX1064-NEXT: ; %bb.3: ; %atomicrmw.end10958; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]10959; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s11, v210960; GFX1064-NEXT: .LBB16_4: ; %Flow10961; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]10962; GFX1064-NEXT: v_and_b32_e32 v0, 0xffff, v010963; GFX1064-NEXT: s_waitcnt lgkmcnt(0)10964; GFX1064-NEXT: s_mov_b32 s3, 0x3101600010965; GFX1064-NEXT: v_readfirstlane_b32 s2, v010966; GFX1064-NEXT: v_mad_u16 v0, s10, v4, s210967; GFX1064-NEXT: s_mov_b32 s2, -110968; GFX1064-NEXT: buffer_store_short v0, off, s[0:3], 010969; GFX1064-NEXT: s_endpgm10970;10971; GFX1032-LABEL: uniform_add_i16:10972; GFX1032: ; %bb.0:10973; GFX1032-NEXT: s_clause 0x110974; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2410975; GFX1032-NEXT: s_load_dword s8, s[4:5], 0x3410976; GFX1032-NEXT: s_mov_b32 s6, exec_lo10977; GFX1032-NEXT: s_mov_b32 s10, 010978; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 010979; GFX1032-NEXT: ; implicit-def: $vgpr010980; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v410981; GFX1032-NEXT: s_and_saveexec_b32 s9, vcc_lo10982; GFX1032-NEXT: s_cbranch_execz .LBB16_410983; GFX1032-NEXT: ; %bb.1:10984; GFX1032-NEXT: s_waitcnt lgkmcnt(0)10985; GFX1032-NEXT: s_and_b32 s4, s2, -410986; GFX1032-NEXT: s_mov_b32 s5, s310987; GFX1032-NEXT: s_and_b32 s2, s2, 310988; GFX1032-NEXT: s_load_dword s7, s[4:5], 0x010989; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s610990; GFX1032-NEXT: s_lshl_b32 s2, s2, 310991; GFX1032-NEXT: s_mul_i32 s6, s8, s610992; GFX1032-NEXT: s_lshl_b32 s3, 0xffff, s210993; GFX1032-NEXT: s_and_b32 s6, s6, 0xffff10994; GFX1032-NEXT: s_not_b32 s11, s310995; GFX1032-NEXT: s_lshl_b32 s12, s6, s210996; GFX1032-NEXT: s_mov_b32 s6, -110997; GFX1032-NEXT: s_waitcnt lgkmcnt(0)10998; GFX1032-NEXT: v_mov_b32_e32 v1, s710999; GFX1032-NEXT: s_mov_b32 s7, 0x3101600011000; GFX1032-NEXT: .LBB16_2: ; %atomicrmw.start11001; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=111002; GFX1032-NEXT: v_add_nc_u32_e32 v0, s12, v111003; GFX1032-NEXT: v_and_b32_e32 v0, s3, v011004; GFX1032-NEXT: v_and_or_b32 v0, v1, s11, v011005; GFX1032-NEXT: v_mov_b32_e32 v3, v111006; GFX1032-NEXT: v_mov_b32_e32 v2, v011007; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc11008; GFX1032-NEXT: s_waitcnt vmcnt(0)11009; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v111010; GFX1032-NEXT: v_mov_b32_e32 v1, v211011; GFX1032-NEXT: s_or_b32 s10, vcc_lo, s1011012; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s1011013; GFX1032-NEXT: s_cbranch_execnz .LBB16_211014; GFX1032-NEXT: ; %bb.3: ; %atomicrmw.end11015; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s1011016; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s2, v211017; GFX1032-NEXT: .LBB16_4: ; %Flow11018; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s911019; GFX1032-NEXT: v_and_b32_e32 v0, 0xffff, v011020; GFX1032-NEXT: s_waitcnt lgkmcnt(0)11021; GFX1032-NEXT: s_mov_b32 s3, 0x3101600011022; GFX1032-NEXT: v_readfirstlane_b32 s2, v011023; GFX1032-NEXT: v_mad_u16 v0, s8, v4, s211024; GFX1032-NEXT: s_mov_b32 s2, -111025; GFX1032-NEXT: buffer_store_short v0, off, s[0:3], 011026; GFX1032-NEXT: s_endpgm11027;11028; GFX1164-TRUE16-LABEL: uniform_add_i16:11029; GFX1164-TRUE16: ; %bb.0:11030; GFX1164-TRUE16-NEXT: s_clause 0x111031; GFX1164-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2411032; GFX1164-TRUE16-NEXT: s_load_b32 s10, s[4:5], 0x3411033; GFX1164-TRUE16-NEXT: s_mov_b64 s[6:7], exec11034; GFX1164-TRUE16-NEXT: s_mov_b64 s[8:9], exec11035; GFX1164-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 011036; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11037; GFX1164-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v011038; GFX1164-TRUE16-NEXT: ; implicit-def: $vgpr0_lo1611039; GFX1164-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v411040; GFX1164-TRUE16-NEXT: s_cbranch_execz .LBB16_411041; GFX1164-TRUE16-NEXT: ; %bb.1:11042; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)11043; GFX1164-TRUE16-NEXT: s_and_b32 s4, s2, -411044; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s311045; GFX1164-TRUE16-NEXT: s_and_b32 s2, s2, 311046; GFX1164-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x011047; GFX1164-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]11048; GFX1164-TRUE16-NEXT: s_lshl_b32 s11, s2, 311049; GFX1164-TRUE16-NEXT: s_mul_i32 s2, s10, s611050; GFX1164-TRUE16-NEXT: s_lshl_b32 s12, 0xffff, s1111051; GFX1164-TRUE16-NEXT: s_and_b32 s2, s2, 0xffff11052; GFX1164-TRUE16-NEXT: s_not_b32 s13, s1211053; GFX1164-TRUE16-NEXT: s_lshl_b32 s14, s2, s1111054; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x3101600011055; GFX1164-TRUE16-NEXT: s_mov_b32 s6, -111056; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)11057; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s311058; GFX1164-TRUE16-NEXT: s_mov_b64 s[2:3], 011059; GFX1164-TRUE16-NEXT: .LBB16_2: ; %atomicrmw.start11060; GFX1164-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=111061; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11062; GFX1164-TRUE16-NEXT: v_add_nc_u32_e32 v0, s14, v111063; GFX1164-TRUE16-NEXT: v_and_b32_e32 v0, s12, v011064; GFX1164-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)11065; GFX1164-TRUE16-NEXT: v_and_or_b32 v0, v1, s13, v011066; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v3, v111067; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)11068; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v2, v011069; GFX1164-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc11070; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)11071; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v111072; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v211073; GFX1164-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]11074; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)11075; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]11076; GFX1164-TRUE16-NEXT: s_cbranch_execnz .LBB16_211077; GFX1164-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end11078; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]11079; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s11, v211080; GFX1164-TRUE16-NEXT: .LBB16_4: ; %Flow11081; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[8:9]11082; GFX1164-TRUE16-NEXT: v_mov_b16_e32 v0.h, 011083; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)11084; GFX1164-TRUE16-NEXT: s_mov_b32 s3, 0x3101600011085; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11086; GFX1164-TRUE16-NEXT: v_readfirstlane_b32 s2, v011087; GFX1164-TRUE16-NEXT: v_mad_u16 v0.l, s10, v4.l, s211088; GFX1164-TRUE16-NEXT: s_mov_b32 s2, -111089; GFX1164-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 011090; GFX1164-TRUE16-NEXT: s_endpgm11091;11092; GFX1164-FAKE16-LABEL: uniform_add_i16:11093; GFX1164-FAKE16: ; %bb.0:11094; GFX1164-FAKE16-NEXT: s_clause 0x111095; GFX1164-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2411096; GFX1164-FAKE16-NEXT: s_load_b32 s10, s[4:5], 0x3411097; GFX1164-FAKE16-NEXT: s_mov_b64 s[6:7], exec11098; GFX1164-FAKE16-NEXT: s_mov_b64 s[8:9], exec11099; GFX1164-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 011100; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11101; GFX1164-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v011102; GFX1164-FAKE16-NEXT: ; implicit-def: $vgpr011103; GFX1164-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v411104; GFX1164-FAKE16-NEXT: s_cbranch_execz .LBB16_411105; GFX1164-FAKE16-NEXT: ; %bb.1:11106; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)11107; GFX1164-FAKE16-NEXT: s_and_b32 s4, s2, -411108; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s311109; GFX1164-FAKE16-NEXT: s_and_b32 s2, s2, 311110; GFX1164-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x011111; GFX1164-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]11112; GFX1164-FAKE16-NEXT: s_lshl_b32 s11, s2, 311113; GFX1164-FAKE16-NEXT: s_mul_i32 s2, s10, s611114; GFX1164-FAKE16-NEXT: s_lshl_b32 s12, 0xffff, s1111115; GFX1164-FAKE16-NEXT: s_and_b32 s2, s2, 0xffff11116; GFX1164-FAKE16-NEXT: s_not_b32 s13, s1211117; GFX1164-FAKE16-NEXT: s_lshl_b32 s14, s2, s1111118; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x3101600011119; GFX1164-FAKE16-NEXT: s_mov_b32 s6, -111120; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)11121; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s311122; GFX1164-FAKE16-NEXT: s_mov_b64 s[2:3], 011123; GFX1164-FAKE16-NEXT: .LBB16_2: ; %atomicrmw.start11124; GFX1164-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=111125; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11126; GFX1164-FAKE16-NEXT: v_add_nc_u32_e32 v0, s14, v111127; GFX1164-FAKE16-NEXT: v_and_b32_e32 v0, s12, v011128; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)11129; GFX1164-FAKE16-NEXT: v_and_or_b32 v0, v1, s13, v011130; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v3, v111131; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)11132; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v2, v011133; GFX1164-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc11134; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)11135; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v111136; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v211137; GFX1164-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]11138; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)11139; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]11140; GFX1164-FAKE16-NEXT: s_cbranch_execnz .LBB16_211141; GFX1164-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end11142; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]11143; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s11, v211144; GFX1164-FAKE16-NEXT: .LBB16_4: ; %Flow11145; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[8:9]11146; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)11147; GFX1164-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v011148; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)11149; GFX1164-FAKE16-NEXT: s_mov_b32 s3, 0x3101600011150; GFX1164-FAKE16-NEXT: v_readfirstlane_b32 s2, v011151; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)11152; GFX1164-FAKE16-NEXT: v_mad_u16 v0, s10, v4, s211153; GFX1164-FAKE16-NEXT: s_mov_b32 s2, -111154; GFX1164-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 011155; GFX1164-FAKE16-NEXT: s_endpgm11156;11157; GFX1132-TRUE16-LABEL: uniform_add_i16:11158; GFX1132-TRUE16: ; %bb.0:11159; GFX1132-TRUE16-NEXT: s_clause 0x111160; GFX1132-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2411161; GFX1132-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x3411162; GFX1132-TRUE16-NEXT: s_mov_b32 s6, exec_lo11163; GFX1132-TRUE16-NEXT: s_mov_b32 s10, 011164; GFX1132-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 011165; GFX1132-TRUE16-NEXT: s_mov_b32 s9, exec_lo11166; GFX1132-TRUE16-NEXT: ; implicit-def: $vgpr0_lo1611167; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)11168; GFX1132-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v411169; GFX1132-TRUE16-NEXT: s_cbranch_execz .LBB16_411170; GFX1132-TRUE16-NEXT: ; %bb.1:11171; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)11172; GFX1132-TRUE16-NEXT: s_and_b32 s4, s2, -411173; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s311174; GFX1132-TRUE16-NEXT: s_and_b32 s2, s2, 311175; GFX1132-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x011176; GFX1132-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s611177; GFX1132-TRUE16-NEXT: s_lshl_b32 s2, s2, 311178; GFX1132-TRUE16-NEXT: s_mul_i32 s6, s8, s611179; GFX1132-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s211180; GFX1132-TRUE16-NEXT: s_and_b32 s6, s6, 0xffff11181; GFX1132-TRUE16-NEXT: s_not_b32 s11, s311182; GFX1132-TRUE16-NEXT: s_lshl_b32 s12, s6, s211183; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -111184; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)11185; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s711186; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x3101600011187; GFX1132-TRUE16-NEXT: .LBB16_2: ; %atomicrmw.start11188; GFX1132-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=111189; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11190; GFX1132-TRUE16-NEXT: v_add_nc_u32_e32 v0, s12, v111191; GFX1132-TRUE16-NEXT: v_and_b32_e32 v0, s3, v011192; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11193; GFX1132-TRUE16-NEXT: v_and_or_b32 v0, v1, s11, v011194; GFX1132-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v011195; GFX1132-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc11196; GFX1132-TRUE16-NEXT: s_waitcnt vmcnt(0)11197; GFX1132-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v111198; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, v211199; GFX1132-TRUE16-NEXT: s_or_b32 s10, vcc_lo, s1011200; GFX1132-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)11201; GFX1132-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1011202; GFX1132-TRUE16-NEXT: s_cbranch_execnz .LBB16_211203; GFX1132-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end11204; GFX1132-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1011205; GFX1132-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v211206; GFX1132-TRUE16-NEXT: .LBB16_4: ; %Flow11207; GFX1132-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s911208; GFX1132-TRUE16-NEXT: v_mov_b16_e32 v0.h, 011209; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)11210; GFX1132-TRUE16-NEXT: s_mov_b32 s3, 0x3101600011211; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11212; GFX1132-TRUE16-NEXT: v_readfirstlane_b32 s2, v011213; GFX1132-TRUE16-NEXT: v_mad_u16 v0.l, s8, v4.l, s211214; GFX1132-TRUE16-NEXT: s_mov_b32 s2, -111215; GFX1132-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 011216; GFX1132-TRUE16-NEXT: s_endpgm11217;11218; GFX1132-FAKE16-LABEL: uniform_add_i16:11219; GFX1132-FAKE16: ; %bb.0:11220; GFX1132-FAKE16-NEXT: s_clause 0x111221; GFX1132-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2411222; GFX1132-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x3411223; GFX1132-FAKE16-NEXT: s_mov_b32 s6, exec_lo11224; GFX1132-FAKE16-NEXT: s_mov_b32 s10, 011225; GFX1132-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 011226; GFX1132-FAKE16-NEXT: s_mov_b32 s9, exec_lo11227; GFX1132-FAKE16-NEXT: ; implicit-def: $vgpr011228; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)11229; GFX1132-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v411230; GFX1132-FAKE16-NEXT: s_cbranch_execz .LBB16_411231; GFX1132-FAKE16-NEXT: ; %bb.1:11232; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)11233; GFX1132-FAKE16-NEXT: s_and_b32 s4, s2, -411234; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s311235; GFX1132-FAKE16-NEXT: s_and_b32 s2, s2, 311236; GFX1132-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x011237; GFX1132-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s611238; GFX1132-FAKE16-NEXT: s_lshl_b32 s2, s2, 311239; GFX1132-FAKE16-NEXT: s_mul_i32 s6, s8, s611240; GFX1132-FAKE16-NEXT: s_lshl_b32 s3, 0xffff, s211241; GFX1132-FAKE16-NEXT: s_and_b32 s6, s6, 0xffff11242; GFX1132-FAKE16-NEXT: s_not_b32 s11, s311243; GFX1132-FAKE16-NEXT: s_lshl_b32 s12, s6, s211244; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -111245; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)11246; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s711247; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x3101600011248; GFX1132-FAKE16-NEXT: .LBB16_2: ; %atomicrmw.start11249; GFX1132-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=111250; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11251; GFX1132-FAKE16-NEXT: v_add_nc_u32_e32 v0, s12, v111252; GFX1132-FAKE16-NEXT: v_and_b32_e32 v0, s3, v011253; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11254; GFX1132-FAKE16-NEXT: v_and_or_b32 v0, v1, s11, v011255; GFX1132-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v011256; GFX1132-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc11257; GFX1132-FAKE16-NEXT: s_waitcnt vmcnt(0)11258; GFX1132-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v111259; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, v211260; GFX1132-FAKE16-NEXT: s_or_b32 s10, vcc_lo, s1011261; GFX1132-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)11262; GFX1132-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1011263; GFX1132-FAKE16-NEXT: s_cbranch_execnz .LBB16_211264; GFX1132-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end11265; GFX1132-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1011266; GFX1132-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v211267; GFX1132-FAKE16-NEXT: .LBB16_4: ; %Flow11268; GFX1132-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s911269; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)11270; GFX1132-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v011271; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)11272; GFX1132-FAKE16-NEXT: s_mov_b32 s3, 0x3101600011273; GFX1132-FAKE16-NEXT: v_readfirstlane_b32 s2, v011274; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)11275; GFX1132-FAKE16-NEXT: v_mad_u16 v0, s8, v4, s211276; GFX1132-FAKE16-NEXT: s_mov_b32 s2, -111277; GFX1132-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 011278; GFX1132-FAKE16-NEXT: s_endpgm11279;11280; GFX1264-TRUE16-LABEL: uniform_add_i16:11281; GFX1264-TRUE16: ; %bb.0:11282; GFX1264-TRUE16-NEXT: s_clause 0x111283; GFX1264-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2411284; GFX1264-TRUE16-NEXT: s_load_b32 s10, s[4:5], 0x3411285; GFX1264-TRUE16-NEXT: s_mov_b64 s[6:7], exec11286; GFX1264-TRUE16-NEXT: s_mov_b64 s[8:9], exec11287; GFX1264-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 011288; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11289; GFX1264-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v011290; GFX1264-TRUE16-NEXT: ; implicit-def: $vgpr0_lo1611291; GFX1264-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v411292; GFX1264-TRUE16-NEXT: s_cbranch_execz .LBB16_411293; GFX1264-TRUE16-NEXT: ; %bb.1:11294; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x011295; GFX1264-TRUE16-NEXT: s_and_b32 s4, s2, -411296; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s311297; GFX1264-TRUE16-NEXT: s_and_b32 s2, s2, 311298; GFX1264-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x011299; GFX1264-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]11300; GFX1264-TRUE16-NEXT: s_lshl_b32 s11, s2, 311301; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)11302; GFX1264-TRUE16-NEXT: s_mul_i32 s2, s10, s611303; GFX1264-TRUE16-NEXT: s_lshl_b32 s12, 0xffff, s1111304; GFX1264-TRUE16-NEXT: s_and_b32 s2, s2, 0xffff11305; GFX1264-TRUE16-NEXT: s_not_b32 s13, s1211306; GFX1264-TRUE16-NEXT: s_lshl_b32 s14, s2, s1111307; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x3101600011308; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -111309; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x011310; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s311311; GFX1264-TRUE16-NEXT: s_mov_b64 s[2:3], 011312; GFX1264-TRUE16-NEXT: .LBB16_2: ; %atomicrmw.start11313; GFX1264-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=111314; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11315; GFX1264-TRUE16-NEXT: v_add_nc_u32_e32 v0, s14, v111316; GFX1264-TRUE16-NEXT: v_and_b32_e32 v0, s12, v011317; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)11318; GFX1264-TRUE16-NEXT: v_and_or_b32 v0, v1, s13, v011319; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v3, v111320; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v2, v011321; GFX1264-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS11322; GFX1264-TRUE16-NEXT: s_wait_loadcnt 0x011323; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v111324; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, v211325; GFX1264-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]11326; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)11327; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]11328; GFX1264-TRUE16-NEXT: s_cbranch_execnz .LBB16_211329; GFX1264-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end11330; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]11331; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s11, v211332; GFX1264-TRUE16-NEXT: .LBB16_4: ; %Flow11333; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[8:9]11334; GFX1264-TRUE16-NEXT: v_mov_b16_e32 v0.h, 011335; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x011336; GFX1264-TRUE16-NEXT: s_mov_b32 s3, 0x3101600011337; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)11338; GFX1264-TRUE16-NEXT: v_readfirstlane_b32 s2, v011339; GFX1264-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)11340; GFX1264-TRUE16-NEXT: v_mad_u16 v0.l, s10, v4.l, s211341; GFX1264-TRUE16-NEXT: s_mov_b32 s2, -111342; GFX1264-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null11343; GFX1264-TRUE16-NEXT: s_endpgm11344;11345; GFX1264-FAKE16-LABEL: uniform_add_i16:11346; GFX1264-FAKE16: ; %bb.0:11347; GFX1264-FAKE16-NEXT: s_clause 0x111348; GFX1264-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2411349; GFX1264-FAKE16-NEXT: s_load_b32 s10, s[4:5], 0x3411350; GFX1264-FAKE16-NEXT: s_mov_b64 s[6:7], exec11351; GFX1264-FAKE16-NEXT: s_mov_b64 s[8:9], exec11352; GFX1264-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 011353; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11354; GFX1264-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v011355; GFX1264-FAKE16-NEXT: ; implicit-def: $vgpr011356; GFX1264-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v411357; GFX1264-FAKE16-NEXT: s_cbranch_execz .LBB16_411358; GFX1264-FAKE16-NEXT: ; %bb.1:11359; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x011360; GFX1264-FAKE16-NEXT: s_and_b32 s4, s2, -411361; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s311362; GFX1264-FAKE16-NEXT: s_and_b32 s2, s2, 311363; GFX1264-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x011364; GFX1264-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]11365; GFX1264-FAKE16-NEXT: s_lshl_b32 s11, s2, 311366; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)11367; GFX1264-FAKE16-NEXT: s_mul_i32 s2, s10, s611368; GFX1264-FAKE16-NEXT: s_lshl_b32 s12, 0xffff, s1111369; GFX1264-FAKE16-NEXT: s_and_b32 s2, s2, 0xffff11370; GFX1264-FAKE16-NEXT: s_not_b32 s13, s1211371; GFX1264-FAKE16-NEXT: s_lshl_b32 s14, s2, s1111372; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x3101600011373; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -111374; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x011375; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s311376; GFX1264-FAKE16-NEXT: s_mov_b64 s[2:3], 011377; GFX1264-FAKE16-NEXT: .LBB16_2: ; %atomicrmw.start11378; GFX1264-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=111379; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11380; GFX1264-FAKE16-NEXT: v_add_nc_u32_e32 v0, s14, v111381; GFX1264-FAKE16-NEXT: v_and_b32_e32 v0, s12, v011382; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)11383; GFX1264-FAKE16-NEXT: v_and_or_b32 v0, v1, s13, v011384; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v3, v111385; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v2, v011386; GFX1264-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS11387; GFX1264-FAKE16-NEXT: s_wait_loadcnt 0x011388; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v111389; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, v211390; GFX1264-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]11391; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)11392; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]11393; GFX1264-FAKE16-NEXT: s_cbranch_execnz .LBB16_211394; GFX1264-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end11395; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]11396; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s11, v211397; GFX1264-FAKE16-NEXT: .LBB16_4: ; %Flow11398; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[8:9]11399; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)11400; GFX1264-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v011401; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x011402; GFX1264-FAKE16-NEXT: s_mov_b32 s3, 0x3101600011403; GFX1264-FAKE16-NEXT: v_readfirstlane_b32 s2, v011404; GFX1264-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)11405; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)11406; GFX1264-FAKE16-NEXT: v_mad_u16 v0, s10, v4, s211407; GFX1264-FAKE16-NEXT: s_mov_b32 s2, -111408; GFX1264-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null11409; GFX1264-FAKE16-NEXT: s_endpgm11410;11411; GFX1232-TRUE16-LABEL: uniform_add_i16:11412; GFX1232-TRUE16: ; %bb.0:11413; GFX1232-TRUE16-NEXT: s_clause 0x111414; GFX1232-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2411415; GFX1232-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x3411416; GFX1232-TRUE16-NEXT: s_mov_b32 s6, exec_lo11417; GFX1232-TRUE16-NEXT: s_mov_b32 s10, 011418; GFX1232-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 011419; GFX1232-TRUE16-NEXT: s_mov_b32 s9, exec_lo11420; GFX1232-TRUE16-NEXT: ; implicit-def: $vgpr0_lo1611421; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)11422; GFX1232-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v411423; GFX1232-TRUE16-NEXT: s_cbranch_execz .LBB16_411424; GFX1232-TRUE16-NEXT: ; %bb.1:11425; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x011426; GFX1232-TRUE16-NEXT: s_and_b32 s4, s2, -411427; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s311428; GFX1232-TRUE16-NEXT: s_and_b32 s2, s2, 311429; GFX1232-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x011430; GFX1232-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s611431; GFX1232-TRUE16-NEXT: s_lshl_b32 s2, s2, 311432; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)11433; GFX1232-TRUE16-NEXT: s_mul_i32 s6, s8, s611434; GFX1232-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s211435; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)11436; GFX1232-TRUE16-NEXT: s_and_b32 s6, s6, 0xffff11437; GFX1232-TRUE16-NEXT: s_not_b32 s11, s311438; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)11439; GFX1232-TRUE16-NEXT: s_lshl_b32 s12, s6, s211440; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -111441; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x011442; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s711443; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x3101600011444; GFX1232-TRUE16-NEXT: .LBB16_2: ; %atomicrmw.start11445; GFX1232-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=111446; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11447; GFX1232-TRUE16-NEXT: v_add_nc_u32_e32 v0, s12, v111448; GFX1232-TRUE16-NEXT: v_and_b32_e32 v0, s3, v011449; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11450; GFX1232-TRUE16-NEXT: v_and_or_b32 v0, v1, s11, v011451; GFX1232-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v011452; GFX1232-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS11453; GFX1232-TRUE16-NEXT: s_wait_loadcnt 0x011454; GFX1232-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v111455; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, v211456; GFX1232-TRUE16-NEXT: s_or_b32 s10, vcc_lo, s1011457; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)11458; GFX1232-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1011459; GFX1232-TRUE16-NEXT: s_cbranch_execnz .LBB16_211460; GFX1232-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end11461; GFX1232-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1011462; GFX1232-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v211463; GFX1232-TRUE16-NEXT: .LBB16_4: ; %Flow11464; GFX1232-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s911465; GFX1232-TRUE16-NEXT: v_mov_b16_e32 v0.h, 011466; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x011467; GFX1232-TRUE16-NEXT: s_mov_b32 s3, 0x3101600011468; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)11469; GFX1232-TRUE16-NEXT: v_readfirstlane_b32 s2, v011470; GFX1232-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)11471; GFX1232-TRUE16-NEXT: v_mad_u16 v0.l, s8, v4.l, s211472; GFX1232-TRUE16-NEXT: s_mov_b32 s2, -111473; GFX1232-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null11474; GFX1232-TRUE16-NEXT: s_endpgm11475;11476; GFX1232-FAKE16-LABEL: uniform_add_i16:11477; GFX1232-FAKE16: ; %bb.0:11478; GFX1232-FAKE16-NEXT: s_clause 0x111479; GFX1232-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2411480; GFX1232-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x3411481; GFX1232-FAKE16-NEXT: s_mov_b32 s6, exec_lo11482; GFX1232-FAKE16-NEXT: s_mov_b32 s10, 011483; GFX1232-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 011484; GFX1232-FAKE16-NEXT: s_mov_b32 s9, exec_lo11485; GFX1232-FAKE16-NEXT: ; implicit-def: $vgpr011486; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)11487; GFX1232-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v411488; GFX1232-FAKE16-NEXT: s_cbranch_execz .LBB16_411489; GFX1232-FAKE16-NEXT: ; %bb.1:11490; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x011491; GFX1232-FAKE16-NEXT: s_and_b32 s4, s2, -411492; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s311493; GFX1232-FAKE16-NEXT: s_and_b32 s2, s2, 311494; GFX1232-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x011495; GFX1232-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s611496; GFX1232-FAKE16-NEXT: s_lshl_b32 s2, s2, 311497; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)11498; GFX1232-FAKE16-NEXT: s_mul_i32 s6, s8, s611499; GFX1232-FAKE16-NEXT: s_lshl_b32 s3, 0xffff, s211500; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)11501; GFX1232-FAKE16-NEXT: s_and_b32 s6, s6, 0xffff11502; GFX1232-FAKE16-NEXT: s_not_b32 s11, s311503; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)11504; GFX1232-FAKE16-NEXT: s_lshl_b32 s12, s6, s211505; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -111506; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x011507; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s711508; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x3101600011509; GFX1232-FAKE16-NEXT: .LBB16_2: ; %atomicrmw.start11510; GFX1232-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=111511; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11512; GFX1232-FAKE16-NEXT: v_add_nc_u32_e32 v0, s12, v111513; GFX1232-FAKE16-NEXT: v_and_b32_e32 v0, s3, v011514; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11515; GFX1232-FAKE16-NEXT: v_and_or_b32 v0, v1, s11, v011516; GFX1232-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v011517; GFX1232-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS11518; GFX1232-FAKE16-NEXT: s_wait_loadcnt 0x011519; GFX1232-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v111520; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, v211521; GFX1232-FAKE16-NEXT: s_or_b32 s10, vcc_lo, s1011522; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)11523; GFX1232-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1011524; GFX1232-FAKE16-NEXT: s_cbranch_execnz .LBB16_211525; GFX1232-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end11526; GFX1232-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1011527; GFX1232-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v211528; GFX1232-FAKE16-NEXT: .LBB16_4: ; %Flow11529; GFX1232-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s911530; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)11531; GFX1232-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v011532; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x011533; GFX1232-FAKE16-NEXT: s_mov_b32 s3, 0x3101600011534; GFX1232-FAKE16-NEXT: v_readfirstlane_b32 s2, v011535; GFX1232-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)11536; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)11537; GFX1232-FAKE16-NEXT: v_mad_u16 v0, s8, v4, s211538; GFX1232-FAKE16-NEXT: s_mov_b32 s2, -111539; GFX1232-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null11540; GFX1232-FAKE16-NEXT: s_endpgm11541 %rmw = atomicrmw add ptr addrspace(1) %uniform.ptr, i16 %val monotonic, align 211542 store i16 %rmw, ptr addrspace(1) %result11543 ret void11544}11545 11546define amdgpu_kernel void @uniform_xchg_i16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i16 %val) {11547; GFX7LESS-LABEL: uniform_xchg_i16:11548; GFX7LESS: ; %bb.0:11549; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x911550; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xd11551; GFX7LESS-NEXT: s_mov_b64 s[8:9], 011552; GFX7LESS-NEXT: s_mov_b32 s7, 0xf00011553; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)11554; GFX7LESS-NEXT: s_and_b32 s4, s2, -411555; GFX7LESS-NEXT: s_mov_b32 s5, s311556; GFX7LESS-NEXT: s_and_b32 s2, s2, 311557; GFX7LESS-NEXT: s_and_b32 s3, s6, 0xffff11558; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0x011559; GFX7LESS-NEXT: s_lshl_b32 s10, s2, 311560; GFX7LESS-NEXT: s_lshl_b32 s11, 0xffff, s1011561; GFX7LESS-NEXT: s_lshl_b32 s2, s3, s1011562; GFX7LESS-NEXT: s_not_b32 s3, s1111563; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)11564; GFX7LESS-NEXT: v_mov_b32_e32 v1, s611565; GFX7LESS-NEXT: s_mov_b32 s6, -111566; GFX7LESS-NEXT: .LBB17_1: ; %atomicrmw.start11567; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=111568; GFX7LESS-NEXT: v_and_b32_e32 v0, s3, v111569; GFX7LESS-NEXT: v_or_b32_e32 v0, s2, v011570; GFX7LESS-NEXT: s_waitcnt expcnt(0)11571; GFX7LESS-NEXT: v_mov_b32_e32 v3, v111572; GFX7LESS-NEXT: v_mov_b32_e32 v2, v011573; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc11574; GFX7LESS-NEXT: s_waitcnt vmcnt(0)11575; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v111576; GFX7LESS-NEXT: s_or_b64 s[8:9], vcc, s[8:9]11577; GFX7LESS-NEXT: v_mov_b32_e32 v1, v211578; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[8:9]11579; GFX7LESS-NEXT: s_cbranch_execnz .LBB17_111580; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end11581; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]11582; GFX7LESS-NEXT: s_mov_b32 s3, 0xf00011583; GFX7LESS-NEXT: s_mov_b32 s2, -111584; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s10, v211585; GFX7LESS-NEXT: buffer_store_short v0, off, s[0:3], 011586; GFX7LESS-NEXT: s_endpgm11587;11588; GFX8-LABEL: uniform_xchg_i16:11589; GFX8: ; %bb.0:11590; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2411591; GFX8-NEXT: s_load_dword s6, s[4:5], 0x3411592; GFX8-NEXT: s_waitcnt lgkmcnt(0)11593; GFX8-NEXT: s_and_b32 s4, s2, -411594; GFX8-NEXT: s_mov_b32 s5, s311595; GFX8-NEXT: s_load_dword s7, s[4:5], 0x011596; GFX8-NEXT: s_and_b32 s2, s2, 311597; GFX8-NEXT: s_lshl_b32 s8, s2, 311598; GFX8-NEXT: s_lshl_b32 s2, 0xffff, s811599; GFX8-NEXT: s_not_b32 s9, s211600; GFX8-NEXT: s_and_b32 s2, s6, 0xffff11601; GFX8-NEXT: s_lshl_b32 s10, s2, s811602; GFX8-NEXT: s_mov_b64 s[2:3], 011603; GFX8-NEXT: s_waitcnt lgkmcnt(0)11604; GFX8-NEXT: v_mov_b32_e32 v1, s711605; GFX8-NEXT: s_mov_b32 s7, 0xf00011606; GFX8-NEXT: s_mov_b32 s6, -111607; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start11608; GFX8-NEXT: ; =>This Inner Loop Header: Depth=111609; GFX8-NEXT: v_and_b32_e32 v0, s9, v111610; GFX8-NEXT: v_or_b32_e32 v0, s10, v011611; GFX8-NEXT: v_mov_b32_e32 v3, v111612; GFX8-NEXT: v_mov_b32_e32 v2, v011613; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc11614; GFX8-NEXT: s_waitcnt vmcnt(0)11615; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v111616; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]11617; GFX8-NEXT: v_mov_b32_e32 v1, v211618; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]11619; GFX8-NEXT: s_cbranch_execnz .LBB17_111620; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end11621; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]11622; GFX8-NEXT: s_mov_b32 s3, 0xf00011623; GFX8-NEXT: s_mov_b32 s2, -111624; GFX8-NEXT: v_lshrrev_b32_e32 v0, s8, v211625; GFX8-NEXT: buffer_store_short v0, off, s[0:3], 011626; GFX8-NEXT: s_endpgm11627;11628; GFX9-LABEL: uniform_xchg_i16:11629; GFX9: ; %bb.0:11630; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2411631; GFX9-NEXT: s_load_dword s6, s[4:5], 0x3411632; GFX9-NEXT: s_waitcnt lgkmcnt(0)11633; GFX9-NEXT: s_and_b32 s4, s2, -411634; GFX9-NEXT: s_mov_b32 s5, s311635; GFX9-NEXT: s_load_dword s7, s[4:5], 0x011636; GFX9-NEXT: s_and_b32 s2, s2, 311637; GFX9-NEXT: s_lshl_b32 s8, s2, 311638; GFX9-NEXT: s_lshl_b32 s2, 0xffff, s811639; GFX9-NEXT: s_not_b32 s9, s211640; GFX9-NEXT: s_and_b32 s2, s6, 0xffff11641; GFX9-NEXT: s_lshl_b32 s10, s2, s811642; GFX9-NEXT: s_mov_b64 s[2:3], 011643; GFX9-NEXT: s_waitcnt lgkmcnt(0)11644; GFX9-NEXT: v_mov_b32_e32 v1, s711645; GFX9-NEXT: s_mov_b32 s7, 0xf00011646; GFX9-NEXT: s_mov_b32 s6, -111647; GFX9-NEXT: .LBB17_1: ; %atomicrmw.start11648; GFX9-NEXT: ; =>This Inner Loop Header: Depth=111649; GFX9-NEXT: v_and_b32_e32 v0, s9, v111650; GFX9-NEXT: v_or_b32_e32 v0, s10, v011651; GFX9-NEXT: v_mov_b32_e32 v3, v111652; GFX9-NEXT: v_mov_b32_e32 v2, v011653; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc11654; GFX9-NEXT: s_waitcnt vmcnt(0)11655; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v111656; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]11657; GFX9-NEXT: v_mov_b32_e32 v1, v211658; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]11659; GFX9-NEXT: s_cbranch_execnz .LBB17_111660; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end11661; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]11662; GFX9-NEXT: s_mov_b32 s3, 0xf00011663; GFX9-NEXT: s_mov_b32 s2, -111664; GFX9-NEXT: v_lshrrev_b32_e32 v0, s8, v211665; GFX9-NEXT: buffer_store_short v0, off, s[0:3], 011666; GFX9-NEXT: s_endpgm11667;11668; GFX1064-LABEL: uniform_xchg_i16:11669; GFX1064: ; %bb.0:11670; GFX1064-NEXT: s_clause 0x111671; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2411672; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x3411673; GFX1064-NEXT: s_mov_b32 s7, 0x3101600011674; GFX1064-NEXT: s_waitcnt lgkmcnt(0)11675; GFX1064-NEXT: s_and_b32 s4, s2, -411676; GFX1064-NEXT: s_mov_b32 s5, s311677; GFX1064-NEXT: s_and_b32 s2, s2, 311678; GFX1064-NEXT: s_load_dword s3, s[4:5], 0x011679; GFX1064-NEXT: s_lshl_b32 s8, s2, 311680; GFX1064-NEXT: s_and_b32 s6, s6, 0xffff11681; GFX1064-NEXT: s_lshl_b32 s2, 0xffff, s811682; GFX1064-NEXT: s_lshl_b32 s10, s6, s811683; GFX1064-NEXT: s_not_b32 s9, s211684; GFX1064-NEXT: s_mov_b32 s6, -111685; GFX1064-NEXT: s_waitcnt lgkmcnt(0)11686; GFX1064-NEXT: v_mov_b32_e32 v1, s311687; GFX1064-NEXT: s_mov_b64 s[2:3], 011688; GFX1064-NEXT: .LBB17_1: ; %atomicrmw.start11689; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=111690; GFX1064-NEXT: v_and_or_b32 v0, v1, s9, s1011691; GFX1064-NEXT: v_mov_b32_e32 v3, v111692; GFX1064-NEXT: v_mov_b32_e32 v2, v011693; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc11694; GFX1064-NEXT: s_waitcnt vmcnt(0)11695; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v111696; GFX1064-NEXT: v_mov_b32_e32 v1, v211697; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]11698; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]11699; GFX1064-NEXT: s_cbranch_execnz .LBB17_111700; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end11701; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]11702; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s8, v211703; GFX1064-NEXT: s_mov_b32 s3, 0x3101600011704; GFX1064-NEXT: s_mov_b32 s2, -111705; GFX1064-NEXT: buffer_store_short v0, off, s[0:3], 011706; GFX1064-NEXT: s_endpgm11707;11708; GFX1032-LABEL: uniform_xchg_i16:11709; GFX1032: ; %bb.0:11710; GFX1032-NEXT: s_clause 0x111711; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2411712; GFX1032-NEXT: s_load_dword s6, s[4:5], 0x3411713; GFX1032-NEXT: s_mov_b32 s9, 011714; GFX1032-NEXT: s_waitcnt lgkmcnt(0)11715; GFX1032-NEXT: s_and_b32 s4, s2, -411716; GFX1032-NEXT: s_mov_b32 s5, s311717; GFX1032-NEXT: s_and_b32 s2, s2, 311718; GFX1032-NEXT: s_load_dword s7, s[4:5], 0x011719; GFX1032-NEXT: s_lshl_b32 s2, s2, 311720; GFX1032-NEXT: s_and_b32 s6, s6, 0xffff11721; GFX1032-NEXT: s_lshl_b32 s3, 0xffff, s211722; GFX1032-NEXT: s_lshl_b32 s8, s6, s211723; GFX1032-NEXT: s_not_b32 s3, s311724; GFX1032-NEXT: s_mov_b32 s6, -111725; GFX1032-NEXT: s_waitcnt lgkmcnt(0)11726; GFX1032-NEXT: v_mov_b32_e32 v1, s711727; GFX1032-NEXT: s_mov_b32 s7, 0x3101600011728; GFX1032-NEXT: .LBB17_1: ; %atomicrmw.start11729; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=111730; GFX1032-NEXT: v_and_or_b32 v0, v1, s3, s811731; GFX1032-NEXT: v_mov_b32_e32 v3, v111732; GFX1032-NEXT: v_mov_b32_e32 v2, v011733; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc11734; GFX1032-NEXT: s_waitcnt vmcnt(0)11735; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v111736; GFX1032-NEXT: v_mov_b32_e32 v1, v211737; GFX1032-NEXT: s_or_b32 s9, vcc_lo, s911738; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s911739; GFX1032-NEXT: s_cbranch_execnz .LBB17_111740; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end11741; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s911742; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s2, v211743; GFX1032-NEXT: s_mov_b32 s3, 0x3101600011744; GFX1032-NEXT: s_mov_b32 s2, -111745; GFX1032-NEXT: buffer_store_short v0, off, s[0:3], 011746; GFX1032-NEXT: s_endpgm11747;11748; GFX1164-LABEL: uniform_xchg_i16:11749; GFX1164: ; %bb.0:11750; GFX1164-NEXT: s_clause 0x111751; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x2411752; GFX1164-NEXT: s_load_b32 s6, s[4:5], 0x3411753; GFX1164-NEXT: s_mov_b32 s7, 0x3101600011754; GFX1164-NEXT: s_waitcnt lgkmcnt(0)11755; GFX1164-NEXT: s_and_b32 s4, s2, -411756; GFX1164-NEXT: s_mov_b32 s5, s311757; GFX1164-NEXT: s_and_b32 s2, s2, 311758; GFX1164-NEXT: s_load_b32 s3, s[4:5], 0x011759; GFX1164-NEXT: s_lshl_b32 s8, s2, 311760; GFX1164-NEXT: s_and_b32 s6, s6, 0xffff11761; GFX1164-NEXT: s_lshl_b32 s2, 0xffff, s811762; GFX1164-NEXT: s_lshl_b32 s10, s6, s811763; GFX1164-NEXT: s_not_b32 s9, s211764; GFX1164-NEXT: s_mov_b32 s6, -111765; GFX1164-NEXT: s_waitcnt lgkmcnt(0)11766; GFX1164-NEXT: v_mov_b32_e32 v1, s311767; GFX1164-NEXT: s_mov_b64 s[2:3], 011768; GFX1164-NEXT: .LBB17_1: ; %atomicrmw.start11769; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=111770; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)11771; GFX1164-NEXT: v_and_or_b32 v0, v1, s9, s1011772; GFX1164-NEXT: v_mov_b32_e32 v3, v111773; GFX1164-NEXT: v_mov_b32_e32 v2, v011774; GFX1164-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc11775; GFX1164-NEXT: s_waitcnt vmcnt(0)11776; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v2, v111777; GFX1164-NEXT: v_mov_b32_e32 v1, v211778; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]11779; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)11780; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]11781; GFX1164-NEXT: s_cbranch_execnz .LBB17_111782; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end11783; GFX1164-NEXT: s_or_b64 exec, exec, s[2:3]11784; GFX1164-NEXT: v_lshrrev_b32_e32 v0, s8, v211785; GFX1164-NEXT: s_mov_b32 s3, 0x3101600011786; GFX1164-NEXT: s_mov_b32 s2, -111787; GFX1164-NEXT: buffer_store_b16 v0, off, s[0:3], 011788; GFX1164-NEXT: s_endpgm11789;11790; GFX1132-LABEL: uniform_xchg_i16:11791; GFX1132: ; %bb.0:11792; GFX1132-NEXT: s_clause 0x111793; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x2411794; GFX1132-NEXT: s_load_b32 s6, s[4:5], 0x3411795; GFX1132-NEXT: s_mov_b32 s9, 011796; GFX1132-NEXT: s_waitcnt lgkmcnt(0)11797; GFX1132-NEXT: s_and_b32 s4, s2, -411798; GFX1132-NEXT: s_mov_b32 s5, s311799; GFX1132-NEXT: s_and_b32 s2, s2, 311800; GFX1132-NEXT: s_load_b32 s7, s[4:5], 0x011801; GFX1132-NEXT: s_lshl_b32 s2, s2, 311802; GFX1132-NEXT: s_and_b32 s6, s6, 0xffff11803; GFX1132-NEXT: s_lshl_b32 s3, 0xffff, s211804; GFX1132-NEXT: s_lshl_b32 s8, s6, s211805; GFX1132-NEXT: s_not_b32 s3, s311806; GFX1132-NEXT: s_mov_b32 s6, -111807; GFX1132-NEXT: s_waitcnt lgkmcnt(0)11808; GFX1132-NEXT: v_mov_b32_e32 v1, s711809; GFX1132-NEXT: s_mov_b32 s7, 0x3101600011810; GFX1132-NEXT: .LBB17_1: ; %atomicrmw.start11811; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=111812; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11813; GFX1132-NEXT: v_and_or_b32 v0, v1, s3, s811814; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v011815; GFX1132-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc11816; GFX1132-NEXT: s_waitcnt vmcnt(0)11817; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v111818; GFX1132-NEXT: v_mov_b32_e32 v1, v211819; GFX1132-NEXT: s_or_b32 s9, vcc_lo, s911820; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)11821; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s911822; GFX1132-NEXT: s_cbranch_execnz .LBB17_111823; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end11824; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s911825; GFX1132-NEXT: v_lshrrev_b32_e32 v0, s2, v211826; GFX1132-NEXT: s_mov_b32 s3, 0x3101600011827; GFX1132-NEXT: s_mov_b32 s2, -111828; GFX1132-NEXT: buffer_store_b16 v0, off, s[0:3], 011829; GFX1132-NEXT: s_endpgm11830;11831; GFX1264-LABEL: uniform_xchg_i16:11832; GFX1264: ; %bb.0:11833; GFX1264-NEXT: s_clause 0x111834; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x2411835; GFX1264-NEXT: s_load_b32 s6, s[4:5], 0x3411836; GFX1264-NEXT: s_mov_b32 s7, 0x3101600011837; GFX1264-NEXT: s_wait_kmcnt 0x011838; GFX1264-NEXT: s_and_b32 s4, s2, -411839; GFX1264-NEXT: s_mov_b32 s5, s311840; GFX1264-NEXT: s_and_b32 s2, s2, 311841; GFX1264-NEXT: s_load_b32 s3, s[4:5], 0x011842; GFX1264-NEXT: s_lshl_b32 s8, s2, 311843; GFX1264-NEXT: s_and_b32 s6, s6, 0xffff11844; GFX1264-NEXT: s_lshl_b32 s2, 0xffff, s811845; GFX1264-NEXT: s_lshl_b32 s10, s6, s811846; GFX1264-NEXT: s_not_b32 s9, s211847; GFX1264-NEXT: s_mov_b32 s6, -111848; GFX1264-NEXT: s_wait_kmcnt 0x011849; GFX1264-NEXT: v_mov_b32_e32 v1, s311850; GFX1264-NEXT: s_mov_b64 s[2:3], 011851; GFX1264-NEXT: .LBB17_1: ; %atomicrmw.start11852; GFX1264-NEXT: ; =>This Inner Loop Header: Depth=111853; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)11854; GFX1264-NEXT: v_and_or_b32 v0, v1, s9, s1011855; GFX1264-NEXT: v_mov_b32_e32 v3, v111856; GFX1264-NEXT: v_mov_b32_e32 v2, v011857; GFX1264-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS11858; GFX1264-NEXT: s_wait_loadcnt 0x011859; GFX1264-NEXT: v_cmp_eq_u32_e32 vcc, v2, v111860; GFX1264-NEXT: v_mov_b32_e32 v1, v211861; GFX1264-NEXT: s_or_b64 s[2:3], vcc, s[2:3]11862; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)11863; GFX1264-NEXT: s_and_not1_b64 exec, exec, s[2:3]11864; GFX1264-NEXT: s_cbranch_execnz .LBB17_111865; GFX1264-NEXT: ; %bb.2: ; %atomicrmw.end11866; GFX1264-NEXT: s_or_b64 exec, exec, s[2:3]11867; GFX1264-NEXT: v_lshrrev_b32_e32 v0, s8, v211868; GFX1264-NEXT: s_mov_b32 s3, 0x3101600011869; GFX1264-NEXT: s_mov_b32 s2, -111870; GFX1264-NEXT: buffer_store_b16 v0, off, s[0:3], null11871; GFX1264-NEXT: s_endpgm11872;11873; GFX1232-LABEL: uniform_xchg_i16:11874; GFX1232: ; %bb.0:11875; GFX1232-NEXT: s_clause 0x111876; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x2411877; GFX1232-NEXT: s_load_b32 s6, s[4:5], 0x3411878; GFX1232-NEXT: s_mov_b32 s9, 011879; GFX1232-NEXT: s_wait_kmcnt 0x011880; GFX1232-NEXT: s_and_b32 s4, s2, -411881; GFX1232-NEXT: s_mov_b32 s5, s311882; GFX1232-NEXT: s_and_b32 s2, s2, 311883; GFX1232-NEXT: s_load_b32 s7, s[4:5], 0x011884; GFX1232-NEXT: s_lshl_b32 s2, s2, 311885; GFX1232-NEXT: s_and_b32 s6, s6, 0xffff11886; GFX1232-NEXT: s_lshl_b32 s3, 0xffff, s211887; GFX1232-NEXT: s_lshl_b32 s8, s6, s211888; GFX1232-NEXT: s_not_b32 s3, s311889; GFX1232-NEXT: s_mov_b32 s6, -111890; GFX1232-NEXT: s_wait_kmcnt 0x011891; GFX1232-NEXT: v_mov_b32_e32 v1, s711892; GFX1232-NEXT: s_mov_b32 s7, 0x3101600011893; GFX1232-NEXT: .LBB17_1: ; %atomicrmw.start11894; GFX1232-NEXT: ; =>This Inner Loop Header: Depth=111895; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11896; GFX1232-NEXT: v_and_or_b32 v0, v1, s3, s811897; GFX1232-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v011898; GFX1232-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS11899; GFX1232-NEXT: s_wait_loadcnt 0x011900; GFX1232-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v111901; GFX1232-NEXT: v_mov_b32_e32 v1, v211902; GFX1232-NEXT: s_or_b32 s9, vcc_lo, s911903; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)11904; GFX1232-NEXT: s_and_not1_b32 exec_lo, exec_lo, s911905; GFX1232-NEXT: s_cbranch_execnz .LBB17_111906; GFX1232-NEXT: ; %bb.2: ; %atomicrmw.end11907; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s911908; GFX1232-NEXT: v_lshrrev_b32_e32 v0, s2, v211909; GFX1232-NEXT: s_mov_b32 s3, 0x3101600011910; GFX1232-NEXT: s_mov_b32 s2, -111911; GFX1232-NEXT: buffer_store_b16 v0, off, s[0:3], null11912; GFX1232-NEXT: s_endpgm11913 %rmw = atomicrmw xchg ptr addrspace(1) %uniform.ptr, i16 %val monotonic, align 211914 store i16 %rmw, ptr addrspace(1) %result11915 ret void11916}11917 11918define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, half %val) {11919; GFX7LESS-LABEL: uniform_fadd_f16:11920; GFX7LESS: ; %bb.0:11921; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xd11922; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x911923; GFX7LESS-NEXT: s_mov_b64 s[8:9], 011924; GFX7LESS-NEXT: s_mov_b32 s7, 0xf00011925; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)11926; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v0, s611927; GFX7LESS-NEXT: s_and_b32 s4, s2, -411928; GFX7LESS-NEXT: s_mov_b32 s5, s311929; GFX7LESS-NEXT: s_and_b32 s2, s2, 311930; GFX7LESS-NEXT: s_lshl_b32 s10, s2, 311931; GFX7LESS-NEXT: s_load_dword s3, s[4:5], 0x011932; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v0, v011933; GFX7LESS-NEXT: s_lshl_b32 s2, 0xffff, s1011934; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v4, v011935; GFX7LESS-NEXT: s_not_b32 s2, s211936; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)11937; GFX7LESS-NEXT: v_mov_b32_e32 v1, s311938; GFX7LESS-NEXT: s_mov_b32 s6, -111939; GFX7LESS-NEXT: .LBB18_1: ; %atomicrmw.start11940; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=111941; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s10, v111942; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v0, v011943; GFX7LESS-NEXT: v_add_f32_e32 v0, v0, v411944; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v0, v011945; GFX7LESS-NEXT: v_lshlrev_b32_e32 v0, s10, v011946; GFX7LESS-NEXT: s_waitcnt expcnt(0)11947; GFX7LESS-NEXT: v_and_b32_e32 v2, s2, v111948; GFX7LESS-NEXT: v_or_b32_e32 v0, v2, v011949; GFX7LESS-NEXT: v_mov_b32_e32 v3, v111950; GFX7LESS-NEXT: v_mov_b32_e32 v2, v011951; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc11952; GFX7LESS-NEXT: s_waitcnt vmcnt(0)11953; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v111954; GFX7LESS-NEXT: s_or_b64 s[8:9], vcc, s[8:9]11955; GFX7LESS-NEXT: v_mov_b32_e32 v1, v211956; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[8:9]11957; GFX7LESS-NEXT: s_cbranch_execnz .LBB18_111958; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end11959; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]11960; GFX7LESS-NEXT: s_mov_b32 s3, 0xf00011961; GFX7LESS-NEXT: s_mov_b32 s2, -111962; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s10, v211963; GFX7LESS-NEXT: buffer_store_short v0, off, s[0:3], 011964; GFX7LESS-NEXT: s_endpgm11965;11966; GFX8-LABEL: uniform_fadd_f16:11967; GFX8: ; %bb.0:11968; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2411969; GFX8-NEXT: s_load_dword s11, s[4:5], 0x3411970; GFX8-NEXT: s_mov_b64 s[8:9], 011971; GFX8-NEXT: s_mov_b32 s7, 0xf00011972; GFX8-NEXT: s_mov_b32 s6, -111973; GFX8-NEXT: s_waitcnt lgkmcnt(0)11974; GFX8-NEXT: s_and_b32 s4, s2, -411975; GFX8-NEXT: s_mov_b32 s5, s311976; GFX8-NEXT: s_load_dword s3, s[4:5], 0x011977; GFX8-NEXT: s_and_b32 s2, s2, 311978; GFX8-NEXT: s_lshl_b32 s10, s2, 311979; GFX8-NEXT: s_lshl_b32 s2, 0xffff, s1011980; GFX8-NEXT: s_not_b32 s2, s211981; GFX8-NEXT: s_waitcnt lgkmcnt(0)11982; GFX8-NEXT: v_mov_b32_e32 v1, s311983; GFX8-NEXT: .LBB18_1: ; %atomicrmw.start11984; GFX8-NEXT: ; =>This Inner Loop Header: Depth=111985; GFX8-NEXT: v_lshrrev_b32_e32 v0, s10, v111986; GFX8-NEXT: v_add_f16_e32 v0, s11, v011987; GFX8-NEXT: v_and_b32_e32 v2, s2, v111988; GFX8-NEXT: v_lshlrev_b32_e32 v0, s10, v011989; GFX8-NEXT: v_or_b32_e32 v0, v2, v011990; GFX8-NEXT: v_mov_b32_e32 v3, v111991; GFX8-NEXT: v_mov_b32_e32 v2, v011992; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc11993; GFX8-NEXT: s_waitcnt vmcnt(0)11994; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v111995; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]11996; GFX8-NEXT: v_mov_b32_e32 v1, v211997; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]11998; GFX8-NEXT: s_cbranch_execnz .LBB18_111999; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end12000; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]12001; GFX8-NEXT: s_mov_b32 s3, 0xf00012002; GFX8-NEXT: s_mov_b32 s2, -112003; GFX8-NEXT: v_lshrrev_b32_e32 v0, s10, v212004; GFX8-NEXT: buffer_store_short v0, off, s[0:3], 012005; GFX8-NEXT: s_endpgm12006;12007; GFX9-LABEL: uniform_fadd_f16:12008; GFX9: ; %bb.0:12009; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2412010; GFX9-NEXT: s_load_dword s11, s[4:5], 0x3412011; GFX9-NEXT: s_mov_b64 s[8:9], 012012; GFX9-NEXT: s_mov_b32 s7, 0xf00012013; GFX9-NEXT: s_mov_b32 s6, -112014; GFX9-NEXT: s_waitcnt lgkmcnt(0)12015; GFX9-NEXT: s_and_b32 s4, s2, -412016; GFX9-NEXT: s_mov_b32 s5, s312017; GFX9-NEXT: s_load_dword s3, s[4:5], 0x012018; GFX9-NEXT: s_and_b32 s2, s2, 312019; GFX9-NEXT: s_lshl_b32 s10, s2, 312020; GFX9-NEXT: s_lshl_b32 s2, 0xffff, s1012021; GFX9-NEXT: s_not_b32 s2, s212022; GFX9-NEXT: s_waitcnt lgkmcnt(0)12023; GFX9-NEXT: v_mov_b32_e32 v1, s312024; GFX9-NEXT: .LBB18_1: ; %atomicrmw.start12025; GFX9-NEXT: ; =>This Inner Loop Header: Depth=112026; GFX9-NEXT: v_lshrrev_b32_e32 v0, s10, v112027; GFX9-NEXT: v_add_f16_e32 v0, s11, v012028; GFX9-NEXT: v_lshlrev_b32_e32 v0, s10, v012029; GFX9-NEXT: v_and_or_b32 v0, v1, s2, v012030; GFX9-NEXT: v_mov_b32_e32 v3, v112031; GFX9-NEXT: v_mov_b32_e32 v2, v012032; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc12033; GFX9-NEXT: s_waitcnt vmcnt(0)12034; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v112035; GFX9-NEXT: s_or_b64 s[8:9], vcc, s[8:9]12036; GFX9-NEXT: v_mov_b32_e32 v1, v212037; GFX9-NEXT: s_andn2_b64 exec, exec, s[8:9]12038; GFX9-NEXT: s_cbranch_execnz .LBB18_112039; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end12040; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]12041; GFX9-NEXT: s_mov_b32 s3, 0xf00012042; GFX9-NEXT: s_mov_b32 s2, -112043; GFX9-NEXT: v_lshrrev_b32_e32 v0, s10, v212044; GFX9-NEXT: buffer_store_short v0, off, s[0:3], 012045; GFX9-NEXT: s_endpgm12046;12047; GFX1064-LABEL: uniform_fadd_f16:12048; GFX1064: ; %bb.0:12049; GFX1064-NEXT: s_clause 0x112050; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2412051; GFX1064-NEXT: s_load_dword s8, s[4:5], 0x3412052; GFX1064-NEXT: s_mov_b32 s7, 0x3101600012053; GFX1064-NEXT: s_mov_b32 s6, -112054; GFX1064-NEXT: s_waitcnt lgkmcnt(0)12055; GFX1064-NEXT: s_and_b32 s4, s2, -412056; GFX1064-NEXT: s_mov_b32 s5, s312057; GFX1064-NEXT: s_and_b32 s2, s2, 312058; GFX1064-NEXT: s_load_dword s3, s[4:5], 0x012059; GFX1064-NEXT: s_lshl_b32 s9, s2, 312060; GFX1064-NEXT: s_lshl_b32 s2, 0xffff, s912061; GFX1064-NEXT: s_not_b32 s10, s212062; GFX1064-NEXT: s_waitcnt lgkmcnt(0)12063; GFX1064-NEXT: v_mov_b32_e32 v1, s312064; GFX1064-NEXT: s_mov_b64 s[2:3], 012065; GFX1064-NEXT: .LBB18_1: ; %atomicrmw.start12066; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=112067; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s9, v112068; GFX1064-NEXT: v_add_f16_e32 v0, s8, v012069; GFX1064-NEXT: v_lshlrev_b32_sdwa v0, s9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_012070; GFX1064-NEXT: v_and_or_b32 v0, v1, s10, v012071; GFX1064-NEXT: v_mov_b32_e32 v3, v112072; GFX1064-NEXT: v_mov_b32_e32 v2, v012073; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc12074; GFX1064-NEXT: s_waitcnt vmcnt(0)12075; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v112076; GFX1064-NEXT: v_mov_b32_e32 v1, v212077; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]12078; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]12079; GFX1064-NEXT: s_cbranch_execnz .LBB18_112080; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end12081; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]12082; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s9, v212083; GFX1064-NEXT: s_mov_b32 s3, 0x3101600012084; GFX1064-NEXT: s_mov_b32 s2, -112085; GFX1064-NEXT: buffer_store_short v0, off, s[0:3], 012086; GFX1064-NEXT: s_endpgm12087;12088; GFX1032-LABEL: uniform_fadd_f16:12089; GFX1032: ; %bb.0:12090; GFX1032-NEXT: s_clause 0x112091; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2412092; GFX1032-NEXT: s_load_dword s8, s[4:5], 0x3412093; GFX1032-NEXT: s_mov_b32 s9, 012094; GFX1032-NEXT: s_mov_b32 s7, 0x3101600012095; GFX1032-NEXT: s_waitcnt lgkmcnt(0)12096; GFX1032-NEXT: s_and_b32 s4, s2, -412097; GFX1032-NEXT: s_mov_b32 s5, s312098; GFX1032-NEXT: s_and_b32 s2, s2, 312099; GFX1032-NEXT: s_load_dword s6, s[4:5], 0x012100; GFX1032-NEXT: s_lshl_b32 s2, s2, 312101; GFX1032-NEXT: s_lshl_b32 s3, 0xffff, s212102; GFX1032-NEXT: s_not_b32 s3, s312103; GFX1032-NEXT: s_waitcnt lgkmcnt(0)12104; GFX1032-NEXT: v_mov_b32_e32 v1, s612105; GFX1032-NEXT: s_mov_b32 s6, -112106; GFX1032-NEXT: .LBB18_1: ; %atomicrmw.start12107; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=112108; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s2, v112109; GFX1032-NEXT: v_add_f16_e32 v0, s8, v012110; GFX1032-NEXT: v_lshlrev_b32_sdwa v0, s2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_012111; GFX1032-NEXT: v_and_or_b32 v0, v1, s3, v012112; GFX1032-NEXT: v_mov_b32_e32 v3, v112113; GFX1032-NEXT: v_mov_b32_e32 v2, v012114; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc12115; GFX1032-NEXT: s_waitcnt vmcnt(0)12116; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v112117; GFX1032-NEXT: v_mov_b32_e32 v1, v212118; GFX1032-NEXT: s_or_b32 s9, vcc_lo, s912119; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s912120; GFX1032-NEXT: s_cbranch_execnz .LBB18_112121; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end12122; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s912123; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s2, v212124; GFX1032-NEXT: s_mov_b32 s3, 0x3101600012125; GFX1032-NEXT: s_mov_b32 s2, -112126; GFX1032-NEXT: buffer_store_short v0, off, s[0:3], 012127; GFX1032-NEXT: s_endpgm12128;12129; GFX1164-TRUE16-LABEL: uniform_fadd_f16:12130; GFX1164-TRUE16: ; %bb.0:12131; GFX1164-TRUE16-NEXT: s_clause 0x112132; GFX1164-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2412133; GFX1164-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x3412134; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x3101600012135; GFX1164-TRUE16-NEXT: s_mov_b32 s6, -112136; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)12137; GFX1164-TRUE16-NEXT: s_and_b32 s4, s2, -412138; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s312139; GFX1164-TRUE16-NEXT: s_and_b32 s2, s2, 312140; GFX1164-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x012141; GFX1164-TRUE16-NEXT: s_lshl_b32 s9, s2, 312142; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)12143; GFX1164-TRUE16-NEXT: s_lshl_b32 s2, 0xffff, s912144; GFX1164-TRUE16-NEXT: s_not_b32 s10, s212145; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)12146; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s312147; GFX1164-TRUE16-NEXT: s_mov_b64 s[2:3], 012148; GFX1164-TRUE16-NEXT: .LBB18_1: ; %atomicrmw.start12149; GFX1164-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=112150; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)12151; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s9, v112152; GFX1164-TRUE16-NEXT: v_mov_b16_e32 v0.h, 012153; GFX1164-TRUE16-NEXT: v_add_f16_e32 v0.l, s8, v0.l12154; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12155; GFX1164-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s9, v012156; GFX1164-TRUE16-NEXT: v_and_or_b32 v0, v1, s10, v012157; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v3, v112158; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)12159; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v2, v012160; GFX1164-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc12161; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)12162; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v112163; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v212164; GFX1164-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]12165; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)12166; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]12167; GFX1164-TRUE16-NEXT: s_cbranch_execnz .LBB18_112168; GFX1164-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end12169; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]12170; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s9, v212171; GFX1164-TRUE16-NEXT: s_mov_b32 s3, 0x3101600012172; GFX1164-TRUE16-NEXT: s_mov_b32 s2, -112173; GFX1164-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 012174; GFX1164-TRUE16-NEXT: s_endpgm12175;12176; GFX1164-FAKE16-LABEL: uniform_fadd_f16:12177; GFX1164-FAKE16: ; %bb.0:12178; GFX1164-FAKE16-NEXT: s_clause 0x112179; GFX1164-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2412180; GFX1164-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x3412181; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x3101600012182; GFX1164-FAKE16-NEXT: s_mov_b32 s6, -112183; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)12184; GFX1164-FAKE16-NEXT: s_and_b32 s4, s2, -412185; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s312186; GFX1164-FAKE16-NEXT: s_and_b32 s2, s2, 312187; GFX1164-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x012188; GFX1164-FAKE16-NEXT: s_lshl_b32 s9, s2, 312189; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)12190; GFX1164-FAKE16-NEXT: s_lshl_b32 s2, 0xffff, s912191; GFX1164-FAKE16-NEXT: s_not_b32 s10, s212192; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)12193; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s312194; GFX1164-FAKE16-NEXT: s_mov_b64 s[2:3], 012195; GFX1164-FAKE16-NEXT: .LBB18_1: ; %atomicrmw.start12196; GFX1164-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=112197; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12198; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s9, v112199; GFX1164-FAKE16-NEXT: v_add_f16_e32 v0, s8, v012200; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12201; GFX1164-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v012202; GFX1164-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s9, v012203; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)12204; GFX1164-FAKE16-NEXT: v_and_or_b32 v0, v1, s10, v012205; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v3, v112206; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v2, v012207; GFX1164-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc12208; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)12209; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v112210; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v212211; GFX1164-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]12212; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)12213; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]12214; GFX1164-FAKE16-NEXT: s_cbranch_execnz .LBB18_112215; GFX1164-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end12216; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]12217; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s9, v212218; GFX1164-FAKE16-NEXT: s_mov_b32 s3, 0x3101600012219; GFX1164-FAKE16-NEXT: s_mov_b32 s2, -112220; GFX1164-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 012221; GFX1164-FAKE16-NEXT: s_endpgm12222;12223; GFX1132-TRUE16-LABEL: uniform_fadd_f16:12224; GFX1132-TRUE16: ; %bb.0:12225; GFX1132-TRUE16-NEXT: s_clause 0x112226; GFX1132-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2412227; GFX1132-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x3412228; GFX1132-TRUE16-NEXT: s_mov_b32 s9, 012229; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x3101600012230; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)12231; GFX1132-TRUE16-NEXT: s_and_b32 s4, s2, -412232; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s312233; GFX1132-TRUE16-NEXT: s_and_b32 s2, s2, 312234; GFX1132-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x012235; GFX1132-TRUE16-NEXT: s_lshl_b32 s2, s2, 312236; GFX1132-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)12237; GFX1132-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s212238; GFX1132-TRUE16-NEXT: s_not_b32 s3, s312239; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)12240; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s612241; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -112242; GFX1132-TRUE16-NEXT: .LBB18_1: ; %atomicrmw.start12243; GFX1132-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=112244; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)12245; GFX1132-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v112246; GFX1132-TRUE16-NEXT: v_mov_b16_e32 v0.h, 012247; GFX1132-TRUE16-NEXT: v_add_f16_e32 v0.l, s8, v0.l12248; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12249; GFX1132-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v012250; GFX1132-TRUE16-NEXT: v_and_or_b32 v0, v1, s3, v012251; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)12252; GFX1132-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v012253; GFX1132-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc12254; GFX1132-TRUE16-NEXT: s_waitcnt vmcnt(0)12255; GFX1132-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v112256; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, v212257; GFX1132-TRUE16-NEXT: s_or_b32 s9, vcc_lo, s912258; GFX1132-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)12259; GFX1132-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s912260; GFX1132-TRUE16-NEXT: s_cbranch_execnz .LBB18_112261; GFX1132-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end12262; GFX1132-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s912263; GFX1132-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v212264; GFX1132-TRUE16-NEXT: s_mov_b32 s3, 0x3101600012265; GFX1132-TRUE16-NEXT: s_mov_b32 s2, -112266; GFX1132-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 012267; GFX1132-TRUE16-NEXT: s_endpgm12268;12269; GFX1132-FAKE16-LABEL: uniform_fadd_f16:12270; GFX1132-FAKE16: ; %bb.0:12271; GFX1132-FAKE16-NEXT: s_clause 0x112272; GFX1132-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2412273; GFX1132-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x3412274; GFX1132-FAKE16-NEXT: s_mov_b32 s9, 012275; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x3101600012276; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)12277; GFX1132-FAKE16-NEXT: s_and_b32 s4, s2, -412278; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s312279; GFX1132-FAKE16-NEXT: s_and_b32 s2, s2, 312280; GFX1132-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x012281; GFX1132-FAKE16-NEXT: s_lshl_b32 s2, s2, 312282; GFX1132-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)12283; GFX1132-FAKE16-NEXT: s_lshl_b32 s3, 0xffff, s212284; GFX1132-FAKE16-NEXT: s_not_b32 s3, s312285; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)12286; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s612287; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -112288; GFX1132-FAKE16-NEXT: .LBB18_1: ; %atomicrmw.start12289; GFX1132-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=112290; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12291; GFX1132-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v112292; GFX1132-FAKE16-NEXT: v_add_f16_e32 v0, s8, v012293; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12294; GFX1132-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v012295; GFX1132-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s2, v012296; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12297; GFX1132-FAKE16-NEXT: v_and_or_b32 v0, v1, s3, v012298; GFX1132-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v012299; GFX1132-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc12300; GFX1132-FAKE16-NEXT: s_waitcnt vmcnt(0)12301; GFX1132-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v112302; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, v212303; GFX1132-FAKE16-NEXT: s_or_b32 s9, vcc_lo, s912304; GFX1132-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)12305; GFX1132-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s912306; GFX1132-FAKE16-NEXT: s_cbranch_execnz .LBB18_112307; GFX1132-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end12308; GFX1132-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s912309; GFX1132-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v212310; GFX1132-FAKE16-NEXT: s_mov_b32 s3, 0x3101600012311; GFX1132-FAKE16-NEXT: s_mov_b32 s2, -112312; GFX1132-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 012313; GFX1132-FAKE16-NEXT: s_endpgm12314;12315; GFX1264-TRUE16-LABEL: uniform_fadd_f16:12316; GFX1264-TRUE16: ; %bb.0:12317; GFX1264-TRUE16-NEXT: s_clause 0x112318; GFX1264-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2412319; GFX1264-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x3412320; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x3101600012321; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -112322; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x012323; GFX1264-TRUE16-NEXT: s_and_b32 s4, s2, -412324; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s312325; GFX1264-TRUE16-NEXT: s_and_b32 s2, s2, 312326; GFX1264-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x012327; GFX1264-TRUE16-NEXT: s_lshl_b32 s9, s2, 312328; GFX1264-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)12329; GFX1264-TRUE16-NEXT: s_lshl_b32 s2, 0xffff, s912330; GFX1264-TRUE16-NEXT: s_not_b32 s10, s212331; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x012332; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s312333; GFX1264-TRUE16-NEXT: s_mov_b64 s[2:3], 012334; GFX1264-TRUE16-NEXT: .LBB18_1: ; %atomicrmw.start12335; GFX1264-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=112336; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)12337; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s9, v112338; GFX1264-TRUE16-NEXT: v_mov_b16_e32 v0.h, 012339; GFX1264-TRUE16-NEXT: v_add_f16_e32 v0.l, s8, v0.l12340; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12341; GFX1264-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s9, v012342; GFX1264-TRUE16-NEXT: v_and_or_b32 v0, v1, s10, v012343; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v3, v112344; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)12345; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v2, v012346; GFX1264-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS12347; GFX1264-TRUE16-NEXT: s_wait_loadcnt 0x012348; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v112349; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, v212350; GFX1264-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]12351; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)12352; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]12353; GFX1264-TRUE16-NEXT: s_cbranch_execnz .LBB18_112354; GFX1264-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end12355; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]12356; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s9, v212357; GFX1264-TRUE16-NEXT: s_mov_b32 s3, 0x3101600012358; GFX1264-TRUE16-NEXT: s_mov_b32 s2, -112359; GFX1264-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null12360; GFX1264-TRUE16-NEXT: s_endpgm12361;12362; GFX1264-FAKE16-LABEL: uniform_fadd_f16:12363; GFX1264-FAKE16: ; %bb.0:12364; GFX1264-FAKE16-NEXT: s_clause 0x112365; GFX1264-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2412366; GFX1264-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x3412367; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x3101600012368; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -112369; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x012370; GFX1264-FAKE16-NEXT: s_and_b32 s4, s2, -412371; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s312372; GFX1264-FAKE16-NEXT: s_and_b32 s2, s2, 312373; GFX1264-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x012374; GFX1264-FAKE16-NEXT: s_lshl_b32 s9, s2, 312375; GFX1264-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)12376; GFX1264-FAKE16-NEXT: s_lshl_b32 s2, 0xffff, s912377; GFX1264-FAKE16-NEXT: s_not_b32 s10, s212378; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x012379; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s312380; GFX1264-FAKE16-NEXT: s_mov_b64 s[2:3], 012381; GFX1264-FAKE16-NEXT: .LBB18_1: ; %atomicrmw.start12382; GFX1264-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=112383; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12384; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s9, v112385; GFX1264-FAKE16-NEXT: v_add_f16_e32 v0, s8, v012386; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12387; GFX1264-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v012388; GFX1264-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s9, v012389; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)12390; GFX1264-FAKE16-NEXT: v_and_or_b32 v0, v1, s10, v012391; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v3, v112392; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v2, v012393; GFX1264-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS12394; GFX1264-FAKE16-NEXT: s_wait_loadcnt 0x012395; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v112396; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, v212397; GFX1264-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]12398; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)12399; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]12400; GFX1264-FAKE16-NEXT: s_cbranch_execnz .LBB18_112401; GFX1264-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end12402; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]12403; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s9, v212404; GFX1264-FAKE16-NEXT: s_mov_b32 s3, 0x3101600012405; GFX1264-FAKE16-NEXT: s_mov_b32 s2, -112406; GFX1264-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null12407; GFX1264-FAKE16-NEXT: s_endpgm12408;12409; GFX1232-TRUE16-LABEL: uniform_fadd_f16:12410; GFX1232-TRUE16: ; %bb.0:12411; GFX1232-TRUE16-NEXT: s_clause 0x112412; GFX1232-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2412413; GFX1232-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x3412414; GFX1232-TRUE16-NEXT: s_mov_b32 s9, 012415; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x3101600012416; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x012417; GFX1232-TRUE16-NEXT: s_and_b32 s4, s2, -412418; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s312419; GFX1232-TRUE16-NEXT: s_and_b32 s2, s2, 312420; GFX1232-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x012421; GFX1232-TRUE16-NEXT: s_lshl_b32 s2, s2, 312422; GFX1232-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)12423; GFX1232-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s212424; GFX1232-TRUE16-NEXT: s_not_b32 s3, s312425; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x012426; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s612427; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -112428; GFX1232-TRUE16-NEXT: .LBB18_1: ; %atomicrmw.start12429; GFX1232-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=112430; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)12431; GFX1232-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v112432; GFX1232-TRUE16-NEXT: v_mov_b16_e32 v0.h, 012433; GFX1232-TRUE16-NEXT: v_add_f16_e32 v0.l, s8, v0.l12434; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12435; GFX1232-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v012436; GFX1232-TRUE16-NEXT: v_and_or_b32 v0, v1, s3, v012437; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)12438; GFX1232-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v012439; GFX1232-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS12440; GFX1232-TRUE16-NEXT: s_wait_loadcnt 0x012441; GFX1232-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v112442; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, v212443; GFX1232-TRUE16-NEXT: s_or_b32 s9, vcc_lo, s912444; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)12445; GFX1232-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s912446; GFX1232-TRUE16-NEXT: s_cbranch_execnz .LBB18_112447; GFX1232-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end12448; GFX1232-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s912449; GFX1232-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v212450; GFX1232-TRUE16-NEXT: s_mov_b32 s3, 0x3101600012451; GFX1232-TRUE16-NEXT: s_mov_b32 s2, -112452; GFX1232-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null12453; GFX1232-TRUE16-NEXT: s_endpgm12454;12455; GFX1232-FAKE16-LABEL: uniform_fadd_f16:12456; GFX1232-FAKE16: ; %bb.0:12457; GFX1232-FAKE16-NEXT: s_clause 0x112458; GFX1232-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2412459; GFX1232-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x3412460; GFX1232-FAKE16-NEXT: s_mov_b32 s9, 012461; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x3101600012462; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x012463; GFX1232-FAKE16-NEXT: s_and_b32 s4, s2, -412464; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s312465; GFX1232-FAKE16-NEXT: s_and_b32 s2, s2, 312466; GFX1232-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x012467; GFX1232-FAKE16-NEXT: s_lshl_b32 s2, s2, 312468; GFX1232-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)12469; GFX1232-FAKE16-NEXT: s_lshl_b32 s3, 0xffff, s212470; GFX1232-FAKE16-NEXT: s_not_b32 s3, s312471; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x012472; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s612473; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -112474; GFX1232-FAKE16-NEXT: .LBB18_1: ; %atomicrmw.start12475; GFX1232-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=112476; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12477; GFX1232-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v112478; GFX1232-FAKE16-NEXT: v_add_f16_e32 v0, s8, v012479; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12480; GFX1232-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v012481; GFX1232-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s2, v012482; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12483; GFX1232-FAKE16-NEXT: v_and_or_b32 v0, v1, s3, v012484; GFX1232-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v012485; GFX1232-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS12486; GFX1232-FAKE16-NEXT: s_wait_loadcnt 0x012487; GFX1232-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v112488; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, v212489; GFX1232-FAKE16-NEXT: s_or_b32 s9, vcc_lo, s912490; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)12491; GFX1232-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s912492; GFX1232-FAKE16-NEXT: s_cbranch_execnz .LBB18_112493; GFX1232-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end12494; GFX1232-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s912495; GFX1232-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v212496; GFX1232-FAKE16-NEXT: s_mov_b32 s3, 0x3101600012497; GFX1232-FAKE16-NEXT: s_mov_b32 s2, -112498; GFX1232-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null12499; GFX1232-FAKE16-NEXT: s_endpgm12500 %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, half %val monotonic, align 212501 store half %rmw, ptr addrspace(1) %result12502 ret void12503}12504 12505define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, bfloat %val) {12506; GFX7LESS-LABEL: uniform_fadd_bf16:12507; GFX7LESS: ; %bb.0:12508; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xd12509; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x912510; GFX7LESS-NEXT: s_mov_b64 s[8:9], 012511; GFX7LESS-NEXT: s_mov_b32 s7, 0xf00012512; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)12513; GFX7LESS-NEXT: s_lshl_b32 s6, s6, 1612514; GFX7LESS-NEXT: s_and_b32 s4, s2, -412515; GFX7LESS-NEXT: s_mov_b32 s5, s312516; GFX7LESS-NEXT: s_and_b32 s2, s2, 312517; GFX7LESS-NEXT: s_lshl_b32 s10, s2, 312518; GFX7LESS-NEXT: s_load_dword s3, s[4:5], 0x012519; GFX7LESS-NEXT: v_mul_f32_e64 v0, 1.0, s612520; GFX7LESS-NEXT: s_lshl_b32 s2, 0xffff, s1012521; GFX7LESS-NEXT: v_and_b32_e32 v4, 0xffff0000, v012522; GFX7LESS-NEXT: s_not_b32 s2, s212523; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)12524; GFX7LESS-NEXT: v_mov_b32_e32 v1, s312525; GFX7LESS-NEXT: s_mov_b32 s6, -112526; GFX7LESS-NEXT: .LBB19_1: ; %atomicrmw.start12527; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=112528; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s10, v112529; GFX7LESS-NEXT: v_lshlrev_b32_e32 v0, 16, v012530; GFX7LESS-NEXT: v_add_f32_e32 v0, v0, v412531; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, 16, v012532; GFX7LESS-NEXT: v_lshlrev_b32_e32 v0, s10, v012533; GFX7LESS-NEXT: s_waitcnt expcnt(0)12534; GFX7LESS-NEXT: v_and_b32_e32 v2, s2, v112535; GFX7LESS-NEXT: v_or_b32_e32 v0, v2, v012536; GFX7LESS-NEXT: v_mov_b32_e32 v3, v112537; GFX7LESS-NEXT: v_mov_b32_e32 v2, v012538; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc12539; GFX7LESS-NEXT: s_waitcnt vmcnt(0)12540; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v112541; GFX7LESS-NEXT: s_or_b64 s[8:9], vcc, s[8:9]12542; GFX7LESS-NEXT: v_mov_b32_e32 v1, v212543; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[8:9]12544; GFX7LESS-NEXT: s_cbranch_execnz .LBB19_112545; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end12546; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]12547; GFX7LESS-NEXT: s_mov_b32 s3, 0xf00012548; GFX7LESS-NEXT: s_mov_b32 s2, -112549; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s10, v212550; GFX7LESS-NEXT: buffer_store_short v0, off, s[0:3], 012551; GFX7LESS-NEXT: s_endpgm12552;12553; GFX8-LABEL: uniform_fadd_bf16:12554; GFX8: ; %bb.0:12555; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2412556; GFX8-NEXT: s_load_dword s6, s[4:5], 0x3412557; GFX8-NEXT: s_mov_b64 s[8:9], 012558; GFX8-NEXT: s_waitcnt lgkmcnt(0)12559; GFX8-NEXT: s_and_b32 s4, s2, -412560; GFX8-NEXT: s_mov_b32 s5, s312561; GFX8-NEXT: s_load_dword s7, s[4:5], 0x012562; GFX8-NEXT: s_and_b32 s3, s2, 312563; GFX8-NEXT: s_lshl_b32 s10, s3, 312564; GFX8-NEXT: s_lshl_b32 s3, 0xffff, s1012565; GFX8-NEXT: s_lshl_b32 s2, s6, 1612566; GFX8-NEXT: s_not_b32 s3, s312567; GFX8-NEXT: s_waitcnt lgkmcnt(0)12568; GFX8-NEXT: v_mov_b32_e32 v1, s712569; GFX8-NEXT: s_mov_b32 s7, 0xf00012570; GFX8-NEXT: s_mov_b32 s6, -112571; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start12572; GFX8-NEXT: ; =>This Inner Loop Header: Depth=112573; GFX8-NEXT: v_mov_b32_e32 v0, s1012574; GFX8-NEXT: v_lshrrev_b32_sdwa v3, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD12575; GFX8-NEXT: v_add_f32_e32 v3, s2, v312576; GFX8-NEXT: v_bfe_u32 v4, v3, 16, 112577; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v312578; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v412579; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v312580; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v312581; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc12582; GFX8-NEXT: v_and_b32_e32 v2, s3, v112583; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_112584; GFX8-NEXT: v_or_b32_e32 v0, v2, v012585; GFX8-NEXT: v_mov_b32_e32 v3, v112586; GFX8-NEXT: v_mov_b32_e32 v2, v012587; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc12588; GFX8-NEXT: s_waitcnt vmcnt(0)12589; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v112590; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]12591; GFX8-NEXT: v_mov_b32_e32 v1, v212592; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]12593; GFX8-NEXT: s_cbranch_execnz .LBB19_112594; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end12595; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]12596; GFX8-NEXT: s_mov_b32 s3, 0xf00012597; GFX8-NEXT: s_mov_b32 s2, -112598; GFX8-NEXT: v_lshrrev_b32_e32 v0, s10, v212599; GFX8-NEXT: buffer_store_short v0, off, s[0:3], 012600; GFX8-NEXT: s_endpgm12601;12602; GFX9-LABEL: uniform_fadd_bf16:12603; GFX9: ; %bb.0:12604; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2412605; GFX9-NEXT: s_load_dword s6, s[4:5], 0x3412606; GFX9-NEXT: s_mov_b64 s[8:9], 012607; GFX9-NEXT: s_movk_i32 s11, 0x7fff12608; GFX9-NEXT: s_waitcnt lgkmcnt(0)12609; GFX9-NEXT: s_and_b32 s4, s2, -412610; GFX9-NEXT: s_mov_b32 s5, s312611; GFX9-NEXT: s_load_dword s7, s[4:5], 0x012612; GFX9-NEXT: s_and_b32 s3, s2, 312613; GFX9-NEXT: s_lshl_b32 s10, s3, 312614; GFX9-NEXT: s_lshl_b32 s3, 0xffff, s1012615; GFX9-NEXT: s_lshl_b32 s2, s6, 1612616; GFX9-NEXT: s_not_b32 s3, s312617; GFX9-NEXT: s_waitcnt lgkmcnt(0)12618; GFX9-NEXT: v_mov_b32_e32 v1, s712619; GFX9-NEXT: s_mov_b32 s7, 0xf00012620; GFX9-NEXT: s_mov_b32 s6, -112621; GFX9-NEXT: .LBB19_1: ; %atomicrmw.start12622; GFX9-NEXT: ; =>This Inner Loop Header: Depth=112623; GFX9-NEXT: v_lshrrev_b32_sdwa v0, s10, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD12624; GFX9-NEXT: v_add_f32_e32 v0, s2, v012625; GFX9-NEXT: v_bfe_u32 v2, v0, 16, 112626; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v012627; GFX9-NEXT: v_add3_u32 v2, v2, v0, s1112628; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v0, v012629; GFX9-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc12630; GFX9-NEXT: v_lshlrev_b32_sdwa v0, s10, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_112631; GFX9-NEXT: v_and_or_b32 v0, v1, s3, v012632; GFX9-NEXT: v_mov_b32_e32 v3, v112633; GFX9-NEXT: v_mov_b32_e32 v2, v012634; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc12635; GFX9-NEXT: s_waitcnt vmcnt(0)12636; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v112637; GFX9-NEXT: s_or_b64 s[8:9], vcc, s[8:9]12638; GFX9-NEXT: v_mov_b32_e32 v1, v212639; GFX9-NEXT: s_andn2_b64 exec, exec, s[8:9]12640; GFX9-NEXT: s_cbranch_execnz .LBB19_112641; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end12642; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]12643; GFX9-NEXT: s_mov_b32 s3, 0xf00012644; GFX9-NEXT: s_mov_b32 s2, -112645; GFX9-NEXT: v_lshrrev_b32_e32 v0, s10, v212646; GFX9-NEXT: buffer_store_short v0, off, s[0:3], 012647; GFX9-NEXT: s_endpgm12648;12649; GFX1064-LABEL: uniform_fadd_bf16:12650; GFX1064: ; %bb.0:12651; GFX1064-NEXT: s_clause 0x112652; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2412653; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x3412654; GFX1064-NEXT: s_waitcnt lgkmcnt(0)12655; GFX1064-NEXT: s_and_b32 s4, s2, -412656; GFX1064-NEXT: s_mov_b32 s5, s312657; GFX1064-NEXT: s_and_b32 s2, s2, 312658; GFX1064-NEXT: s_load_dword s7, s[4:5], 0x012659; GFX1064-NEXT: s_lshl_b32 s8, s2, 312660; GFX1064-NEXT: s_lshl_b32 s10, s6, 1612661; GFX1064-NEXT: s_lshl_b32 s2, 0xffff, s812662; GFX1064-NEXT: s_mov_b32 s6, -112663; GFX1064-NEXT: s_not_b32 s9, s212664; GFX1064-NEXT: s_mov_b64 s[2:3], 012665; GFX1064-NEXT: s_waitcnt lgkmcnt(0)12666; GFX1064-NEXT: v_mov_b32_e32 v1, s712667; GFX1064-NEXT: s_mov_b32 s7, 0x3101600012668; GFX1064-NEXT: .LBB19_1: ; %atomicrmw.start12669; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=112670; GFX1064-NEXT: v_lshrrev_b32_sdwa v0, s8, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD12671; GFX1064-NEXT: v_add_f32_e32 v0, s10, v012672; GFX1064-NEXT: v_bfe_u32 v2, v0, 16, 112673; GFX1064-NEXT: v_or_b32_e32 v3, 0x400000, v012674; GFX1064-NEXT: v_cmp_u_f32_e32 vcc, v0, v012675; GFX1064-NEXT: v_add3_u32 v2, v2, v0, 0x7fff12676; GFX1064-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc12677; GFX1064-NEXT: v_lshlrev_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_112678; GFX1064-NEXT: v_and_or_b32 v0, v1, s9, v012679; GFX1064-NEXT: v_mov_b32_e32 v3, v112680; GFX1064-NEXT: v_mov_b32_e32 v2, v012681; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc12682; GFX1064-NEXT: s_waitcnt vmcnt(0)12683; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v112684; GFX1064-NEXT: v_mov_b32_e32 v1, v212685; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]12686; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]12687; GFX1064-NEXT: s_cbranch_execnz .LBB19_112688; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end12689; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]12690; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s8, v212691; GFX1064-NEXT: s_mov_b32 s3, 0x3101600012692; GFX1064-NEXT: s_mov_b32 s2, -112693; GFX1064-NEXT: buffer_store_short v0, off, s[0:3], 012694; GFX1064-NEXT: s_endpgm12695;12696; GFX1032-LABEL: uniform_fadd_bf16:12697; GFX1032: ; %bb.0:12698; GFX1032-NEXT: s_clause 0x112699; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2412700; GFX1032-NEXT: s_load_dword s6, s[4:5], 0x3412701; GFX1032-NEXT: s_waitcnt lgkmcnt(0)12702; GFX1032-NEXT: s_and_b32 s4, s2, -412703; GFX1032-NEXT: s_mov_b32 s5, s312704; GFX1032-NEXT: s_and_b32 s2, s2, 312705; GFX1032-NEXT: s_load_dword s7, s[4:5], 0x012706; GFX1032-NEXT: s_lshl_b32 s2, s2, 312707; GFX1032-NEXT: s_lshl_b32 s9, s6, 1612708; GFX1032-NEXT: s_lshl_b32 s3, 0xffff, s212709; GFX1032-NEXT: s_mov_b32 s6, -112710; GFX1032-NEXT: s_not_b32 s8, s312711; GFX1032-NEXT: s_mov_b32 s3, 012712; GFX1032-NEXT: s_waitcnt lgkmcnt(0)12713; GFX1032-NEXT: v_mov_b32_e32 v1, s712714; GFX1032-NEXT: s_mov_b32 s7, 0x3101600012715; GFX1032-NEXT: .LBB19_1: ; %atomicrmw.start12716; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=112717; GFX1032-NEXT: v_lshrrev_b32_sdwa v0, s2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD12718; GFX1032-NEXT: v_add_f32_e32 v0, s9, v012719; GFX1032-NEXT: v_bfe_u32 v2, v0, 16, 112720; GFX1032-NEXT: v_or_b32_e32 v3, 0x400000, v012721; GFX1032-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v012722; GFX1032-NEXT: v_add3_u32 v2, v2, v0, 0x7fff12723; GFX1032-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo12724; GFX1032-NEXT: v_lshlrev_b32_sdwa v0, s2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_112725; GFX1032-NEXT: v_and_or_b32 v0, v1, s8, v012726; GFX1032-NEXT: v_mov_b32_e32 v3, v112727; GFX1032-NEXT: v_mov_b32_e32 v2, v012728; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc12729; GFX1032-NEXT: s_waitcnt vmcnt(0)12730; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v112731; GFX1032-NEXT: v_mov_b32_e32 v1, v212732; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s312733; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s312734; GFX1032-NEXT: s_cbranch_execnz .LBB19_112735; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end12736; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s312737; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s2, v212738; GFX1032-NEXT: s_mov_b32 s3, 0x3101600012739; GFX1032-NEXT: s_mov_b32 s2, -112740; GFX1032-NEXT: buffer_store_short v0, off, s[0:3], 012741; GFX1032-NEXT: s_endpgm12742;12743; GFX1164-TRUE16-LABEL: uniform_fadd_bf16:12744; GFX1164-TRUE16: ; %bb.0:12745; GFX1164-TRUE16-NEXT: s_clause 0x112746; GFX1164-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2412747; GFX1164-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x3412748; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)12749; GFX1164-TRUE16-NEXT: s_and_b32 s4, s2, -412750; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s312751; GFX1164-TRUE16-NEXT: s_and_b32 s2, s2, 312752; GFX1164-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x012753; GFX1164-TRUE16-NEXT: s_lshl_b32 s8, s2, 312754; GFX1164-TRUE16-NEXT: s_lshl_b32 s10, s6, 1612755; GFX1164-TRUE16-NEXT: s_lshl_b32 s2, 0xffff, s812756; GFX1164-TRUE16-NEXT: s_mov_b32 s6, -112757; GFX1164-TRUE16-NEXT: s_not_b32 s9, s212758; GFX1164-TRUE16-NEXT: s_mov_b64 s[2:3], 012759; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)12760; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s712761; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x3101600012762; GFX1164-TRUE16-NEXT: .p2align 612763; GFX1164-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start12764; GFX1164-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=112765; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12766; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s8, v112767; GFX1164-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v012768; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12769; GFX1164-TRUE16-NEXT: v_add_f32_e32 v0, s10, v012770; GFX1164-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 112771; GFX1164-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v012772; GFX1164-TRUE16-NEXT: v_cmp_u_f32_e32 vcc, v0, v012773; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)12774; GFX1164-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff12775; GFX1164-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc12776; GFX1164-TRUE16-NEXT: v_mov_b16_e32 v2.h, 012777; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)12778; GFX1164-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h12779; GFX1164-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s8, v212780; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)12781; GFX1164-TRUE16-NEXT: v_and_or_b32 v0, v1, s9, v012782; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v3, v112783; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v2, v012784; GFX1164-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc12785; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)12786; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v112787; GFX1164-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)12788; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v212789; GFX1164-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]12790; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)12791; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]12792; GFX1164-TRUE16-NEXT: s_cbranch_execnz .LBB19_112793; GFX1164-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end12794; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]12795; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s8, v212796; GFX1164-TRUE16-NEXT: s_mov_b32 s3, 0x3101600012797; GFX1164-TRUE16-NEXT: s_mov_b32 s2, -112798; GFX1164-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 012799; GFX1164-TRUE16-NEXT: s_endpgm12800;12801; GFX1164-FAKE16-LABEL: uniform_fadd_bf16:12802; GFX1164-FAKE16: ; %bb.0:12803; GFX1164-FAKE16-NEXT: s_clause 0x112804; GFX1164-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2412805; GFX1164-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x3412806; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)12807; GFX1164-FAKE16-NEXT: s_and_b32 s4, s2, -412808; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s312809; GFX1164-FAKE16-NEXT: s_and_b32 s2, s2, 312810; GFX1164-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x012811; GFX1164-FAKE16-NEXT: s_lshl_b32 s8, s2, 312812; GFX1164-FAKE16-NEXT: s_lshl_b32 s10, s6, 1612813; GFX1164-FAKE16-NEXT: s_lshl_b32 s2, 0xffff, s812814; GFX1164-FAKE16-NEXT: s_mov_b32 s6, -112815; GFX1164-FAKE16-NEXT: s_not_b32 s9, s212816; GFX1164-FAKE16-NEXT: s_mov_b64 s[2:3], 012817; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)12818; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s712819; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x3101600012820; GFX1164-FAKE16-NEXT: .p2align 612821; GFX1164-FAKE16-NEXT: .LBB19_1: ; %atomicrmw.start12822; GFX1164-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=112823; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12824; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s8, v112825; GFX1164-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v012826; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12827; GFX1164-FAKE16-NEXT: v_add_f32_e32 v0, s10, v012828; GFX1164-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 112829; GFX1164-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v012830; GFX1164-FAKE16-NEXT: v_cmp_u_f32_e32 vcc, v0, v012831; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)12832; GFX1164-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff12833; GFX1164-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc12834; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12835; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v012836; GFX1164-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s8, v012837; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)12838; GFX1164-FAKE16-NEXT: v_and_or_b32 v0, v1, s9, v012839; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v3, v112840; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v2, v012841; GFX1164-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc12842; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)12843; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v112844; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)12845; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v212846; GFX1164-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]12847; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)12848; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]12849; GFX1164-FAKE16-NEXT: s_cbranch_execnz .LBB19_112850; GFX1164-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end12851; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]12852; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s8, v212853; GFX1164-FAKE16-NEXT: s_mov_b32 s3, 0x3101600012854; GFX1164-FAKE16-NEXT: s_mov_b32 s2, -112855; GFX1164-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 012856; GFX1164-FAKE16-NEXT: s_endpgm12857;12858; GFX1132-TRUE16-LABEL: uniform_fadd_bf16:12859; GFX1132-TRUE16: ; %bb.0:12860; GFX1132-TRUE16-NEXT: s_clause 0x112861; GFX1132-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2412862; GFX1132-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x3412863; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)12864; GFX1132-TRUE16-NEXT: s_and_b32 s4, s2, -412865; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s312866; GFX1132-TRUE16-NEXT: s_and_b32 s2, s2, 312867; GFX1132-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x012868; GFX1132-TRUE16-NEXT: s_lshl_b32 s2, s2, 312869; GFX1132-TRUE16-NEXT: s_lshl_b32 s9, s6, 1612870; GFX1132-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s212871; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -112872; GFX1132-TRUE16-NEXT: s_not_b32 s8, s312873; GFX1132-TRUE16-NEXT: s_mov_b32 s3, 012874; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)12875; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s712876; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x3101600012877; GFX1132-TRUE16-NEXT: .p2align 612878; GFX1132-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start12879; GFX1132-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=112880; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12881; GFX1132-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v112882; GFX1132-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v012883; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12884; GFX1132-TRUE16-NEXT: v_add_f32_e32 v0, s9, v012885; GFX1132-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 112886; GFX1132-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v012887; GFX1132-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v012888; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)12889; GFX1132-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff12890; GFX1132-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo12891; GFX1132-TRUE16-NEXT: v_mov_b16_e32 v2.h, 012892; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)12893; GFX1132-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h12894; GFX1132-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v212895; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12896; GFX1132-TRUE16-NEXT: v_and_or_b32 v0, v1, s8, v012897; GFX1132-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v012898; GFX1132-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc12899; GFX1132-TRUE16-NEXT: s_waitcnt vmcnt(0)12900; GFX1132-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v112901; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, v212902; GFX1132-TRUE16-NEXT: s_or_b32 s3, vcc_lo, s312903; GFX1132-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)12904; GFX1132-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s312905; GFX1132-TRUE16-NEXT: s_cbranch_execnz .LBB19_112906; GFX1132-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end12907; GFX1132-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s312908; GFX1132-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v212909; GFX1132-TRUE16-NEXT: s_mov_b32 s3, 0x3101600012910; GFX1132-TRUE16-NEXT: s_mov_b32 s2, -112911; GFX1132-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 012912; GFX1132-TRUE16-NEXT: s_endpgm12913;12914; GFX1132-FAKE16-LABEL: uniform_fadd_bf16:12915; GFX1132-FAKE16: ; %bb.0:12916; GFX1132-FAKE16-NEXT: s_clause 0x112917; GFX1132-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2412918; GFX1132-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x3412919; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)12920; GFX1132-FAKE16-NEXT: s_and_b32 s4, s2, -412921; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s312922; GFX1132-FAKE16-NEXT: s_and_b32 s2, s2, 312923; GFX1132-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x012924; GFX1132-FAKE16-NEXT: s_lshl_b32 s2, s2, 312925; GFX1132-FAKE16-NEXT: s_lshl_b32 s9, s6, 1612926; GFX1132-FAKE16-NEXT: s_lshl_b32 s3, 0xffff, s212927; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -112928; GFX1132-FAKE16-NEXT: s_not_b32 s8, s312929; GFX1132-FAKE16-NEXT: s_mov_b32 s3, 012930; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)12931; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s712932; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x3101600012933; GFX1132-FAKE16-NEXT: .p2align 612934; GFX1132-FAKE16-NEXT: .LBB19_1: ; %atomicrmw.start12935; GFX1132-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=112936; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12937; GFX1132-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v112938; GFX1132-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v012939; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12940; GFX1132-FAKE16-NEXT: v_add_f32_e32 v0, s9, v012941; GFX1132-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 112942; GFX1132-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v012943; GFX1132-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v012944; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)12945; GFX1132-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff12946; GFX1132-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo12947; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12948; GFX1132-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v012949; GFX1132-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s2, v012950; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12951; GFX1132-FAKE16-NEXT: v_and_or_b32 v0, v1, s8, v012952; GFX1132-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v012953; GFX1132-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc12954; GFX1132-FAKE16-NEXT: s_waitcnt vmcnt(0)12955; GFX1132-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v112956; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, v212957; GFX1132-FAKE16-NEXT: s_or_b32 s3, vcc_lo, s312958; GFX1132-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)12959; GFX1132-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s312960; GFX1132-FAKE16-NEXT: s_cbranch_execnz .LBB19_112961; GFX1132-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end12962; GFX1132-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s312963; GFX1132-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v212964; GFX1132-FAKE16-NEXT: s_mov_b32 s3, 0x3101600012965; GFX1132-FAKE16-NEXT: s_mov_b32 s2, -112966; GFX1132-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 012967; GFX1132-FAKE16-NEXT: s_endpgm12968;12969; GFX1264-TRUE16-LABEL: uniform_fadd_bf16:12970; GFX1264-TRUE16: ; %bb.0:12971; GFX1264-TRUE16-NEXT: s_clause 0x112972; GFX1264-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2412973; GFX1264-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x3412974; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x012975; GFX1264-TRUE16-NEXT: s_and_b32 s4, s2, -412976; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s312977; GFX1264-TRUE16-NEXT: s_and_b32 s2, s2, 312978; GFX1264-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x012979; GFX1264-TRUE16-NEXT: s_lshl_b32 s8, s2, 312980; GFX1264-TRUE16-NEXT: s_lshl_b32 s10, s6, 1612981; GFX1264-TRUE16-NEXT: s_lshl_b32 s2, 0xffff, s812982; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -112983; GFX1264-TRUE16-NEXT: s_not_b32 s9, s212984; GFX1264-TRUE16-NEXT: s_mov_b64 s[2:3], 012985; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x012986; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s712987; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x3101600012988; GFX1264-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start12989; GFX1264-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=112990; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12991; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s8, v112992; GFX1264-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v012993; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12994; GFX1264-TRUE16-NEXT: v_add_f32_e32 v0, s10, v012995; GFX1264-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 112996; GFX1264-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v012997; GFX1264-TRUE16-NEXT: v_cmp_u_f32_e32 vcc, v0, v012998; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)12999; GFX1264-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff13000; GFX1264-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)13001; GFX1264-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc13002; GFX1264-TRUE16-NEXT: v_mov_b16_e32 v2.h, 013003; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)13004; GFX1264-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h13005; GFX1264-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s8, v213006; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)13007; GFX1264-TRUE16-NEXT: v_and_or_b32 v0, v1, s9, v013008; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v3, v113009; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v2, v013010; GFX1264-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS13011; GFX1264-TRUE16-NEXT: s_wait_loadcnt 0x013012; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v113013; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, v213014; GFX1264-TRUE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]13015; GFX1264-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)13016; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]13017; GFX1264-TRUE16-NEXT: s_cbranch_execnz .LBB19_113018; GFX1264-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end13019; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]13020; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s8, v213021; GFX1264-TRUE16-NEXT: s_mov_b32 s3, 0x3101600013022; GFX1264-TRUE16-NEXT: s_mov_b32 s2, -113023; GFX1264-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null13024; GFX1264-TRUE16-NEXT: s_endpgm13025;13026; GFX1264-FAKE16-LABEL: uniform_fadd_bf16:13027; GFX1264-FAKE16: ; %bb.0:13028; GFX1264-FAKE16-NEXT: s_clause 0x113029; GFX1264-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2413030; GFX1264-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x3413031; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x013032; GFX1264-FAKE16-NEXT: s_and_b32 s4, s2, -413033; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s313034; GFX1264-FAKE16-NEXT: s_and_b32 s2, s2, 313035; GFX1264-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x013036; GFX1264-FAKE16-NEXT: s_lshl_b32 s8, s2, 313037; GFX1264-FAKE16-NEXT: s_lshl_b32 s10, s6, 1613038; GFX1264-FAKE16-NEXT: s_lshl_b32 s2, 0xffff, s813039; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -113040; GFX1264-FAKE16-NEXT: s_not_b32 s9, s213041; GFX1264-FAKE16-NEXT: s_mov_b64 s[2:3], 013042; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x013043; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s713044; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x3101600013045; GFX1264-FAKE16-NEXT: .LBB19_1: ; %atomicrmw.start13046; GFX1264-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=113047; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)13048; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s8, v113049; GFX1264-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v013050; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)13051; GFX1264-FAKE16-NEXT: v_add_f32_e32 v0, s10, v013052; GFX1264-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 113053; GFX1264-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v013054; GFX1264-FAKE16-NEXT: v_cmp_u_f32_e32 vcc, v0, v013055; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)13056; GFX1264-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff13057; GFX1264-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)13058; GFX1264-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc13059; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)13060; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v013061; GFX1264-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s8, v013062; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)13063; GFX1264-FAKE16-NEXT: v_and_or_b32 v0, v1, s9, v013064; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v3, v113065; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v2, v013066; GFX1264-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS13067; GFX1264-FAKE16-NEXT: s_wait_loadcnt 0x013068; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v113069; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, v213070; GFX1264-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]13071; GFX1264-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)13072; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]13073; GFX1264-FAKE16-NEXT: s_cbranch_execnz .LBB19_113074; GFX1264-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end13075; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]13076; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s8, v213077; GFX1264-FAKE16-NEXT: s_mov_b32 s3, 0x3101600013078; GFX1264-FAKE16-NEXT: s_mov_b32 s2, -113079; GFX1264-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null13080; GFX1264-FAKE16-NEXT: s_endpgm13081;13082; GFX1232-TRUE16-LABEL: uniform_fadd_bf16:13083; GFX1232-TRUE16: ; %bb.0:13084; GFX1232-TRUE16-NEXT: s_clause 0x113085; GFX1232-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2413086; GFX1232-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x3413087; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x013088; GFX1232-TRUE16-NEXT: s_and_b32 s4, s2, -413089; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s313090; GFX1232-TRUE16-NEXT: s_and_b32 s2, s2, 313091; GFX1232-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x013092; GFX1232-TRUE16-NEXT: s_lshl_b32 s2, s2, 313093; GFX1232-TRUE16-NEXT: s_lshl_b32 s9, s6, 1613094; GFX1232-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s213095; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -113096; GFX1232-TRUE16-NEXT: s_not_b32 s8, s313097; GFX1232-TRUE16-NEXT: s_mov_b32 s3, 013098; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x013099; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s713100; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x3101600013101; GFX1232-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start13102; GFX1232-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=113103; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)13104; GFX1232-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v113105; GFX1232-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v013106; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)13107; GFX1232-TRUE16-NEXT: v_add_f32_e32 v0, s9, v013108; GFX1232-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 113109; GFX1232-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v013110; GFX1232-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v013111; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)13112; GFX1232-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff13113; GFX1232-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)13114; GFX1232-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo13115; GFX1232-TRUE16-NEXT: v_mov_b16_e32 v2.h, 013116; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)13117; GFX1232-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h13118; GFX1232-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v213119; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)13120; GFX1232-TRUE16-NEXT: v_and_or_b32 v0, v1, s8, v013121; GFX1232-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v013122; GFX1232-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS13123; GFX1232-TRUE16-NEXT: s_wait_loadcnt 0x013124; GFX1232-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v113125; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, v213126; GFX1232-TRUE16-NEXT: s_or_b32 s3, vcc_lo, s313127; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)13128; GFX1232-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s313129; GFX1232-TRUE16-NEXT: s_cbranch_execnz .LBB19_113130; GFX1232-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end13131; GFX1232-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s313132; GFX1232-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v213133; GFX1232-TRUE16-NEXT: s_mov_b32 s3, 0x3101600013134; GFX1232-TRUE16-NEXT: s_mov_b32 s2, -113135; GFX1232-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null13136; GFX1232-TRUE16-NEXT: s_endpgm13137;13138; GFX1232-FAKE16-LABEL: uniform_fadd_bf16:13139; GFX1232-FAKE16: ; %bb.0:13140; GFX1232-FAKE16-NEXT: s_clause 0x113141; GFX1232-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2413142; GFX1232-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x3413143; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x013144; GFX1232-FAKE16-NEXT: s_and_b32 s4, s2, -413145; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s313146; GFX1232-FAKE16-NEXT: s_and_b32 s2, s2, 313147; GFX1232-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x013148; GFX1232-FAKE16-NEXT: s_lshl_b32 s2, s2, 313149; GFX1232-FAKE16-NEXT: s_lshl_b32 s9, s6, 1613150; GFX1232-FAKE16-NEXT: s_lshl_b32 s3, 0xffff, s213151; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -113152; GFX1232-FAKE16-NEXT: s_not_b32 s8, s313153; GFX1232-FAKE16-NEXT: s_mov_b32 s3, 013154; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x013155; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s713156; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x3101600013157; GFX1232-FAKE16-NEXT: .LBB19_1: ; %atomicrmw.start13158; GFX1232-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=113159; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)13160; GFX1232-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v113161; GFX1232-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v013162; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)13163; GFX1232-FAKE16-NEXT: v_add_f32_e32 v0, s9, v013164; GFX1232-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 113165; GFX1232-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v013166; GFX1232-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v013167; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)13168; GFX1232-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff13169; GFX1232-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)13170; GFX1232-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo13171; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)13172; GFX1232-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v013173; GFX1232-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s2, v013174; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)13175; GFX1232-FAKE16-NEXT: v_and_or_b32 v0, v1, s8, v013176; GFX1232-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v013177; GFX1232-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS13178; GFX1232-FAKE16-NEXT: s_wait_loadcnt 0x013179; GFX1232-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v113180; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, v213181; GFX1232-FAKE16-NEXT: s_or_b32 s3, vcc_lo, s313182; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)13183; GFX1232-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s313184; GFX1232-FAKE16-NEXT: s_cbranch_execnz .LBB19_113185; GFX1232-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end13186; GFX1232-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s313187; GFX1232-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s2, v213188; GFX1232-FAKE16-NEXT: s_mov_b32 s3, 0x3101600013189; GFX1232-FAKE16-NEXT: s_mov_b32 s2, -113190; GFX1232-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null13191; GFX1232-FAKE16-NEXT: s_endpgm13192 %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, bfloat %val monotonic, align 213193 store bfloat %rmw, ptr addrspace(1) %result13194 ret void13195}13196 13197define amdgpu_kernel void @uniform_fadd_v2f16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, <2 x half> %val) {13198; GFX7LESS-LABEL: uniform_fadd_v2f16:13199; GFX7LESS: ; %bb.0:13200; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xd13201; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x913202; GFX7LESS-NEXT: s_mov_b64 s[8:9], 013203; GFX7LESS-NEXT: s_mov_b32 s7, 0xf00013204; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)13205; GFX7LESS-NEXT: s_lshr_b32 s4, s6, 1613206; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v0, s613207; GFX7LESS-NEXT: s_load_dword s5, s[2:3], 0x013208; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v1, s413209; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v0, v013210; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v1, v113211; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v0, v013212; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v1, v113213; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)13214; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v2, s513215; GFX7LESS-NEXT: s_lshr_b32 s4, s5, 1613216; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v3, s413217; GFX7LESS-NEXT: s_mov_b32 s6, -113218; GFX7LESS-NEXT: s_mov_b32 s4, s213219; GFX7LESS-NEXT: s_mov_b32 s5, s313220; GFX7LESS-NEXT: .LBB20_1: ; %atomicrmw.start13221; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=113222; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v3, v313223; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v2, v213224; GFX7LESS-NEXT: s_waitcnt expcnt(0)13225; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v4, v313226; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v5, v213227; GFX7LESS-NEXT: v_lshlrev_b32_e32 v3, 16, v313228; GFX7LESS-NEXT: v_add_f32_e32 v4, v4, v113229; GFX7LESS-NEXT: v_add_f32_e32 v5, v5, v013230; GFX7LESS-NEXT: v_or_b32_e32 v3, v2, v313231; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v2, v413232; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v4, v513233; GFX7LESS-NEXT: v_lshlrev_b32_e32 v2, 16, v213234; GFX7LESS-NEXT: v_or_b32_e32 v2, v4, v213235; GFX7LESS-NEXT: v_mov_b32_e32 v5, v313236; GFX7LESS-NEXT: v_mov_b32_e32 v4, v213237; GFX7LESS-NEXT: buffer_atomic_cmpswap v[4:5], off, s[4:7], 0 glc13238; GFX7LESS-NEXT: s_waitcnt vmcnt(0)13239; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v4, v313240; GFX7LESS-NEXT: v_lshrrev_b32_e32 v3, 16, v413241; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v2, v413242; GFX7LESS-NEXT: s_or_b64 s[8:9], vcc, s[8:9]13243; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v3, v313244; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[8:9]13245; GFX7LESS-NEXT: s_cbranch_execnz .LBB20_113246; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end13247; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]13248; GFX7LESS-NEXT: s_mov_b32 s3, 0xf00013249; GFX7LESS-NEXT: s_mov_b32 s2, -113250; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v0, v313251; GFX7LESS-NEXT: v_lshlrev_b32_e32 v0, 16, v013252; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v1, v213253; GFX7LESS-NEXT: v_or_b32_e32 v0, v1, v013254; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 013255; GFX7LESS-NEXT: s_endpgm13256;13257; GFX8-LABEL: uniform_fadd_v2f16:13258; GFX8: ; %bb.0:13259; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2413260; GFX8-NEXT: s_load_dword s10, s[4:5], 0x3413261; GFX8-NEXT: s_mov_b64 s[8:9], 013262; GFX8-NEXT: s_mov_b32 s7, 0xf00013263; GFX8-NEXT: s_mov_b32 s6, -113264; GFX8-NEXT: s_waitcnt lgkmcnt(0)13265; GFX8-NEXT: s_load_dword s5, s[2:3], 0x013266; GFX8-NEXT: s_lshr_b32 s11, s10, 1613267; GFX8-NEXT: s_mov_b32 s4, s213268; GFX8-NEXT: s_waitcnt lgkmcnt(0)13269; GFX8-NEXT: v_mov_b32_e32 v1, s513270; GFX8-NEXT: s_mov_b32 s5, s313271; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start13272; GFX8-NEXT: ; =>This Inner Loop Header: Depth=113273; GFX8-NEXT: v_mov_b32_e32 v0, s1113274; GFX8-NEXT: v_add_f16_e32 v2, s10, v113275; GFX8-NEXT: v_add_f16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD13276; GFX8-NEXT: v_or_b32_e32 v0, v2, v013277; GFX8-NEXT: v_mov_b32_e32 v3, v113278; GFX8-NEXT: v_mov_b32_e32 v2, v013279; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc13280; GFX8-NEXT: s_waitcnt vmcnt(0)13281; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v113282; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]13283; GFX8-NEXT: v_mov_b32_e32 v1, v213284; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]13285; GFX8-NEXT: s_cbranch_execnz .LBB20_113286; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end13287; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]13288; GFX8-NEXT: s_mov_b32 s3, 0xf00013289; GFX8-NEXT: s_mov_b32 s2, -113290; GFX8-NEXT: buffer_store_dword v2, off, s[0:3], 013291; GFX8-NEXT: s_endpgm13292;13293; GFX9-LABEL: uniform_fadd_v2f16:13294; GFX9: ; %bb.0:13295; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2413296; GFX9-NEXT: s_load_dword s10, s[4:5], 0x3413297; GFX9-NEXT: s_mov_b64 s[8:9], 013298; GFX9-NEXT: s_mov_b32 s7, 0xf00013299; GFX9-NEXT: s_mov_b32 s6, -113300; GFX9-NEXT: s_waitcnt lgkmcnt(0)13301; GFX9-NEXT: s_load_dword s5, s[2:3], 0x013302; GFX9-NEXT: s_mov_b32 s4, s213303; GFX9-NEXT: s_waitcnt lgkmcnt(0)13304; GFX9-NEXT: v_mov_b32_e32 v1, s513305; GFX9-NEXT: s_mov_b32 s5, s313306; GFX9-NEXT: .LBB20_1: ; %atomicrmw.start13307; GFX9-NEXT: ; =>This Inner Loop Header: Depth=113308; GFX9-NEXT: v_pk_add_f16 v0, v1, s1013309; GFX9-NEXT: v_mov_b32_e32 v3, v113310; GFX9-NEXT: v_mov_b32_e32 v2, v013311; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc13312; GFX9-NEXT: s_waitcnt vmcnt(0)13313; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v113314; GFX9-NEXT: s_or_b64 s[8:9], vcc, s[8:9]13315; GFX9-NEXT: v_mov_b32_e32 v1, v213316; GFX9-NEXT: s_andn2_b64 exec, exec, s[8:9]13317; GFX9-NEXT: s_cbranch_execnz .LBB20_113318; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end13319; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]13320; GFX9-NEXT: s_mov_b32 s3, 0xf00013321; GFX9-NEXT: s_mov_b32 s2, -113322; GFX9-NEXT: buffer_store_dword v2, off, s[0:3], 013323; GFX9-NEXT: s_endpgm13324;13325; GFX1064-LABEL: uniform_fadd_v2f16:13326; GFX1064: ; %bb.0:13327; GFX1064-NEXT: s_clause 0x113328; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2413329; GFX1064-NEXT: s_load_dword s10, s[4:5], 0x3413330; GFX1064-NEXT: s_mov_b64 s[8:9], 013331; GFX1064-NEXT: s_mov_b32 s7, 0x3101600013332; GFX1064-NEXT: s_mov_b32 s6, -113333; GFX1064-NEXT: s_waitcnt lgkmcnt(0)13334; GFX1064-NEXT: s_load_dword s4, s[2:3], 0x013335; GFX1064-NEXT: s_mov_b32 s5, s313336; GFX1064-NEXT: s_waitcnt lgkmcnt(0)13337; GFX1064-NEXT: v_mov_b32_e32 v1, s413338; GFX1064-NEXT: s_mov_b32 s4, s213339; GFX1064-NEXT: .LBB20_1: ; %atomicrmw.start13340; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=113341; GFX1064-NEXT: v_pk_add_f16 v0, v1, s1013342; GFX1064-NEXT: v_mov_b32_e32 v3, v113343; GFX1064-NEXT: v_mov_b32_e32 v2, v013344; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc13345; GFX1064-NEXT: s_waitcnt vmcnt(0)13346; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v113347; GFX1064-NEXT: v_mov_b32_e32 v1, v213348; GFX1064-NEXT: s_or_b64 s[8:9], vcc, s[8:9]13349; GFX1064-NEXT: s_andn2_b64 exec, exec, s[8:9]13350; GFX1064-NEXT: s_cbranch_execnz .LBB20_113351; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end13352; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]13353; GFX1064-NEXT: s_mov_b32 s3, 0x3101600013354; GFX1064-NEXT: s_mov_b32 s2, -113355; GFX1064-NEXT: buffer_store_dword v2, off, s[0:3], 013356; GFX1064-NEXT: s_endpgm13357;13358; GFX1032-LABEL: uniform_fadd_v2f16:13359; GFX1032: ; %bb.0:13360; GFX1032-NEXT: s_clause 0x113361; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2413362; GFX1032-NEXT: s_load_dword s8, s[4:5], 0x3413363; GFX1032-NEXT: s_mov_b32 s9, 013364; GFX1032-NEXT: s_mov_b32 s7, 0x3101600013365; GFX1032-NEXT: s_mov_b32 s6, -113366; GFX1032-NEXT: s_waitcnt lgkmcnt(0)13367; GFX1032-NEXT: s_load_dword s4, s[2:3], 0x013368; GFX1032-NEXT: s_mov_b32 s5, s313369; GFX1032-NEXT: s_waitcnt lgkmcnt(0)13370; GFX1032-NEXT: v_mov_b32_e32 v1, s413371; GFX1032-NEXT: s_mov_b32 s4, s213372; GFX1032-NEXT: .LBB20_1: ; %atomicrmw.start13373; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=113374; GFX1032-NEXT: v_pk_add_f16 v0, v1, s813375; GFX1032-NEXT: v_mov_b32_e32 v3, v113376; GFX1032-NEXT: v_mov_b32_e32 v2, v013377; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc13378; GFX1032-NEXT: s_waitcnt vmcnt(0)13379; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v113380; GFX1032-NEXT: v_mov_b32_e32 v1, v213381; GFX1032-NEXT: s_or_b32 s9, vcc_lo, s913382; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s913383; GFX1032-NEXT: s_cbranch_execnz .LBB20_113384; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end13385; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s913386; GFX1032-NEXT: s_mov_b32 s3, 0x3101600013387; GFX1032-NEXT: s_mov_b32 s2, -113388; GFX1032-NEXT: buffer_store_dword v2, off, s[0:3], 013389; GFX1032-NEXT: s_endpgm13390;13391; GFX1164-LABEL: uniform_fadd_v2f16:13392; GFX1164: ; %bb.0:13393; GFX1164-NEXT: s_clause 0x113394; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x2413395; GFX1164-NEXT: s_load_b32 s10, s[4:5], 0x3413396; GFX1164-NEXT: s_mov_b64 s[8:9], 013397; GFX1164-NEXT: s_mov_b32 s7, 0x3101600013398; GFX1164-NEXT: s_mov_b32 s6, -113399; GFX1164-NEXT: s_waitcnt lgkmcnt(0)13400; GFX1164-NEXT: s_load_b32 s4, s[2:3], 0x013401; GFX1164-NEXT: s_mov_b32 s5, s313402; GFX1164-NEXT: s_waitcnt lgkmcnt(0)13403; GFX1164-NEXT: v_mov_b32_e32 v1, s413404; GFX1164-NEXT: s_mov_b32 s4, s213405; GFX1164-NEXT: .LBB20_1: ; %atomicrmw.start13406; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=113407; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)13408; GFX1164-NEXT: v_pk_add_f16 v0, v1, s1013409; GFX1164-NEXT: v_mov_b32_e32 v3, v113410; GFX1164-NEXT: v_mov_b32_e32 v2, v013411; GFX1164-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc13412; GFX1164-NEXT: s_waitcnt vmcnt(0)13413; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v2, v113414; GFX1164-NEXT: v_mov_b32_e32 v1, v213415; GFX1164-NEXT: s_or_b64 s[8:9], vcc, s[8:9]13416; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)13417; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[8:9]13418; GFX1164-NEXT: s_cbranch_execnz .LBB20_113419; GFX1164-NEXT: ; %bb.2: ; %atomicrmw.end13420; GFX1164-NEXT: s_or_b64 exec, exec, s[8:9]13421; GFX1164-NEXT: s_mov_b32 s3, 0x3101600013422; GFX1164-NEXT: s_mov_b32 s2, -113423; GFX1164-NEXT: buffer_store_b32 v2, off, s[0:3], 013424; GFX1164-NEXT: s_endpgm13425;13426; GFX1132-LABEL: uniform_fadd_v2f16:13427; GFX1132: ; %bb.0:13428; GFX1132-NEXT: s_clause 0x113429; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x2413430; GFX1132-NEXT: s_load_b32 s8, s[4:5], 0x3413431; GFX1132-NEXT: s_mov_b32 s9, 013432; GFX1132-NEXT: s_mov_b32 s7, 0x3101600013433; GFX1132-NEXT: s_mov_b32 s6, -113434; GFX1132-NEXT: s_waitcnt lgkmcnt(0)13435; GFX1132-NEXT: s_load_b32 s4, s[2:3], 0x013436; GFX1132-NEXT: s_mov_b32 s5, s313437; GFX1132-NEXT: s_waitcnt lgkmcnt(0)13438; GFX1132-NEXT: v_mov_b32_e32 v1, s413439; GFX1132-NEXT: s_mov_b32 s4, s213440; GFX1132-NEXT: .LBB20_1: ; %atomicrmw.start13441; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=113442; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)13443; GFX1132-NEXT: v_pk_add_f16 v0, v1, s813444; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v013445; GFX1132-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc13446; GFX1132-NEXT: s_waitcnt vmcnt(0)13447; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v113448; GFX1132-NEXT: v_mov_b32_e32 v1, v213449; GFX1132-NEXT: s_or_b32 s9, vcc_lo, s913450; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)13451; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s913452; GFX1132-NEXT: s_cbranch_execnz .LBB20_113453; GFX1132-NEXT: ; %bb.2: ; %atomicrmw.end13454; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s913455; GFX1132-NEXT: s_mov_b32 s3, 0x3101600013456; GFX1132-NEXT: s_mov_b32 s2, -113457; GFX1132-NEXT: buffer_store_b32 v2, off, s[0:3], 013458; GFX1132-NEXT: s_endpgm13459;13460; GFX1264-LABEL: uniform_fadd_v2f16:13461; GFX1264: ; %bb.0:13462; GFX1264-NEXT: s_clause 0x113463; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x2413464; GFX1264-NEXT: s_load_b32 s10, s[4:5], 0x3413465; GFX1264-NEXT: s_mov_b64 s[8:9], 013466; GFX1264-NEXT: s_mov_b32 s7, 0x3101600013467; GFX1264-NEXT: s_mov_b32 s6, -113468; GFX1264-NEXT: s_wait_kmcnt 0x013469; GFX1264-NEXT: s_load_b32 s4, s[2:3], 0x013470; GFX1264-NEXT: s_mov_b32 s5, s313471; GFX1264-NEXT: s_wait_kmcnt 0x013472; GFX1264-NEXT: v_mov_b32_e32 v1, s413473; GFX1264-NEXT: s_mov_b32 s4, s213474; GFX1264-NEXT: .LBB20_1: ; %atomicrmw.start13475; GFX1264-NEXT: ; =>This Inner Loop Header: Depth=113476; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)13477; GFX1264-NEXT: v_pk_add_f16 v0, v1, s1013478; GFX1264-NEXT: v_mov_b32_e32 v3, v113479; GFX1264-NEXT: v_mov_b32_e32 v2, v013480; GFX1264-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS13481; GFX1264-NEXT: s_wait_loadcnt 0x013482; GFX1264-NEXT: v_cmp_eq_u32_e32 vcc, v2, v113483; GFX1264-NEXT: v_mov_b32_e32 v1, v213484; GFX1264-NEXT: s_or_b64 s[8:9], vcc, s[8:9]13485; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1)13486; GFX1264-NEXT: s_and_not1_b64 exec, exec, s[8:9]13487; GFX1264-NEXT: s_cbranch_execnz .LBB20_113488; GFX1264-NEXT: ; %bb.2: ; %atomicrmw.end13489; GFX1264-NEXT: s_or_b64 exec, exec, s[8:9]13490; GFX1264-NEXT: s_mov_b32 s3, 0x3101600013491; GFX1264-NEXT: s_mov_b32 s2, -113492; GFX1264-NEXT: buffer_store_b32 v2, off, s[0:3], null13493; GFX1264-NEXT: s_endpgm13494;13495; GFX1232-LABEL: uniform_fadd_v2f16:13496; GFX1232: ; %bb.0:13497; GFX1232-NEXT: s_clause 0x113498; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x2413499; GFX1232-NEXT: s_load_b32 s8, s[4:5], 0x3413500; GFX1232-NEXT: s_mov_b32 s9, 013501; GFX1232-NEXT: s_mov_b32 s7, 0x3101600013502; GFX1232-NEXT: s_mov_b32 s6, -113503; GFX1232-NEXT: s_wait_kmcnt 0x013504; GFX1232-NEXT: s_load_b32 s4, s[2:3], 0x013505; GFX1232-NEXT: s_mov_b32 s5, s313506; GFX1232-NEXT: s_wait_kmcnt 0x013507; GFX1232-NEXT: v_mov_b32_e32 v1, s413508; GFX1232-NEXT: s_mov_b32 s4, s213509; GFX1232-NEXT: .LBB20_1: ; %atomicrmw.start13510; GFX1232-NEXT: ; =>This Inner Loop Header: Depth=113511; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)13512; GFX1232-NEXT: v_pk_add_f16 v0, v1, s813513; GFX1232-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v013514; GFX1232-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS13515; GFX1232-NEXT: s_wait_loadcnt 0x013516; GFX1232-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v113517; GFX1232-NEXT: v_mov_b32_e32 v1, v213518; GFX1232-NEXT: s_or_b32 s9, vcc_lo, s913519; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)13520; GFX1232-NEXT: s_and_not1_b32 exec_lo, exec_lo, s913521; GFX1232-NEXT: s_cbranch_execnz .LBB20_113522; GFX1232-NEXT: ; %bb.2: ; %atomicrmw.end13523; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s913524; GFX1232-NEXT: s_mov_b32 s3, 0x3101600013525; GFX1232-NEXT: s_mov_b32 s2, -113526; GFX1232-NEXT: buffer_store_b32 v2, off, s[0:3], null13527; GFX1232-NEXT: s_endpgm13528 %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, <2 x half> %val monotonic, align 413529 store <2 x half> %rmw, ptr addrspace(1) %result13530 ret void13531}13532 13533define amdgpu_kernel void @uniform_fadd_v2bf16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, <2 x bfloat> %val) {13534; GFX7LESS-LABEL: uniform_fadd_v2bf16:13535; GFX7LESS: ; %bb.0:13536; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xd13537; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x913538; GFX7LESS-NEXT: s_mov_b64 s[8:9], 013539; GFX7LESS-NEXT: s_mov_b32 s7, 0xf00013540; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)13541; GFX7LESS-NEXT: s_and_b32 s4, s6, 0xffff000013542; GFX7LESS-NEXT: s_lshl_b32 s5, s6, 1613543; GFX7LESS-NEXT: s_load_dword s6, s[2:3], 0x013544; GFX7LESS-NEXT: v_mul_f32_e64 v0, 1.0, s513545; GFX7LESS-NEXT: v_mul_f32_e64 v1, 1.0, s413546; GFX7LESS-NEXT: v_and_b32_e32 v0, 0xffff0000, v013547; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)13548; GFX7LESS-NEXT: s_and_b32 s4, s6, 0xffff000013549; GFX7LESS-NEXT: s_lshl_b32 s5, s6, 1613550; GFX7LESS-NEXT: v_and_b32_e32 v1, 0xffff0000, v113551; GFX7LESS-NEXT: v_mov_b32_e32 v3, s513552; GFX7LESS-NEXT: v_mov_b32_e32 v2, s413553; GFX7LESS-NEXT: s_mov_b32 s6, -113554; GFX7LESS-NEXT: s_mov_b32 s4, s213555; GFX7LESS-NEXT: s_mov_b32 s5, s313556; GFX7LESS-NEXT: .LBB21_1: ; %atomicrmw.start13557; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=113558; GFX7LESS-NEXT: v_mul_f32_e32 v3, 1.0, v313559; GFX7LESS-NEXT: v_mul_f32_e32 v2, 1.0, v213560; GFX7LESS-NEXT: s_waitcnt expcnt(0)13561; GFX7LESS-NEXT: v_and_b32_e32 v4, 0xffff0000, v313562; GFX7LESS-NEXT: v_and_b32_e32 v5, 0xffff0000, v213563; GFX7LESS-NEXT: v_lshrrev_b32_e32 v2, 16, v213564; GFX7LESS-NEXT: v_add_f32_e32 v4, v4, v013565; GFX7LESS-NEXT: v_add_f32_e32 v5, v5, v113566; GFX7LESS-NEXT: v_alignbit_b32 v3, v2, v3, 1613567; GFX7LESS-NEXT: v_lshrrev_b32_e32 v2, 16, v513568; GFX7LESS-NEXT: v_alignbit_b32 v2, v2, v4, 1613569; GFX7LESS-NEXT: v_mov_b32_e32 v5, v313570; GFX7LESS-NEXT: v_mov_b32_e32 v4, v213571; GFX7LESS-NEXT: buffer_atomic_cmpswap v[4:5], off, s[4:7], 0 glc13572; GFX7LESS-NEXT: s_waitcnt vmcnt(0)13573; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v4, v313574; GFX7LESS-NEXT: v_and_b32_e32 v2, 0xffff0000, v413575; GFX7LESS-NEXT: s_or_b64 s[8:9], vcc, s[8:9]13576; GFX7LESS-NEXT: v_lshlrev_b32_e32 v3, 16, v413577; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[8:9]13578; GFX7LESS-NEXT: s_cbranch_execnz .LBB21_113579; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end13580; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]13581; GFX7LESS-NEXT: s_mov_b32 s3, 0xf00013582; GFX7LESS-NEXT: s_mov_b32 s2, -113583; GFX7LESS-NEXT: v_mul_f32_e32 v0, 1.0, v213584; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, 16, v013585; GFX7LESS-NEXT: v_mul_f32_e32 v1, 1.0, v313586; GFX7LESS-NEXT: v_alignbit_b32 v0, v0, v1, 1613587; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 013588; GFX7LESS-NEXT: s_endpgm13589;13590; GFX8-LABEL: uniform_fadd_v2bf16:13591; GFX8: ; %bb.0:13592; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x2413593; GFX8-NEXT: s_load_dword s0, s[4:5], 0x3413594; GFX8-NEXT: s_mov_b64 s[2:3], 013595; GFX8-NEXT: s_mov_b32 s7, 0xf00013596; GFX8-NEXT: s_mov_b32 s6, -113597; GFX8-NEXT: s_waitcnt lgkmcnt(0)13598; GFX8-NEXT: s_load_dword s1, s[10:11], 0x013599; GFX8-NEXT: s_lshl_b32 s12, s0, 1613600; GFX8-NEXT: s_and_b32 s13, s0, 0xffff000013601; GFX8-NEXT: s_mov_b32 s4, s1013602; GFX8-NEXT: s_mov_b32 s5, s1113603; GFX8-NEXT: s_waitcnt lgkmcnt(0)13604; GFX8-NEXT: v_mov_b32_e32 v1, s113605; GFX8-NEXT: .LBB21_1: ; %atomicrmw.start13606; GFX8-NEXT: ; =>This Inner Loop Header: Depth=113607; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v113608; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v113609; GFX8-NEXT: v_add_f32_e32 v0, s12, v013610; GFX8-NEXT: v_add_f32_e32 v2, s13, v213611; GFX8-NEXT: v_bfe_u32 v3, v0, 16, 113612; GFX8-NEXT: v_bfe_u32 v5, v2, 16, 113613; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v013614; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v213615; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v313616; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v513617; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v213618; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v213619; GFX8-NEXT: v_or_b32_e32 v4, 0x400000, v013620; GFX8-NEXT: v_cmp_u_f32_e64 s[0:1], v0, v013621; GFX8-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc13622; GFX8-NEXT: v_cndmask_b32_e64 v0, v3, v4, s[0:1]13623; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v213624; GFX8-NEXT: v_alignbit_b32 v0, v2, v0, 1613625; GFX8-NEXT: v_mov_b32_e32 v3, v113626; GFX8-NEXT: v_mov_b32_e32 v2, v013627; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc13628; GFX8-NEXT: s_waitcnt vmcnt(0)13629; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v113630; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]13631; GFX8-NEXT: v_mov_b32_e32 v1, v213632; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]13633; GFX8-NEXT: s_cbranch_execnz .LBB21_113634; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end13635; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]13636; GFX8-NEXT: s_mov_b32 s11, 0xf00013637; GFX8-NEXT: s_mov_b32 s10, -113638; GFX8-NEXT: buffer_store_dword v2, off, s[8:11], 013639; GFX8-NEXT: s_endpgm13640;13641; GFX9-LABEL: uniform_fadd_v2bf16:13642; GFX9: ; %bb.0:13643; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x2413644; GFX9-NEXT: s_load_dword s0, s[4:5], 0x3413645; GFX9-NEXT: s_mov_b64 s[2:3], 013646; GFX9-NEXT: s_movk_i32 s12, 0x7fff13647; GFX9-NEXT: s_mov_b32 s13, 0x706030213648; GFX9-NEXT: s_waitcnt lgkmcnt(0)13649; GFX9-NEXT: s_load_dword s1, s[10:11], 0x013650; GFX9-NEXT: s_mov_b32 s7, 0xf00013651; GFX9-NEXT: s_mov_b32 s6, -113652; GFX9-NEXT: s_lshl_b32 s14, s0, 1613653; GFX9-NEXT: s_and_b32 s15, s0, 0xffff000013654; GFX9-NEXT: s_waitcnt lgkmcnt(0)13655; GFX9-NEXT: v_mov_b32_e32 v1, s113656; GFX9-NEXT: s_mov_b32 s4, s1013657; GFX9-NEXT: s_mov_b32 s5, s1113658; GFX9-NEXT: .LBB21_1: ; %atomicrmw.start13659; GFX9-NEXT: ; =>This Inner Loop Header: Depth=113660; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v113661; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v113662; GFX9-NEXT: v_add_f32_e32 v0, s14, v013663; GFX9-NEXT: v_add_f32_e32 v2, s15, v213664; GFX9-NEXT: v_bfe_u32 v3, v0, 16, 113665; GFX9-NEXT: v_bfe_u32 v5, v2, 16, 113666; GFX9-NEXT: v_or_b32_e32 v4, 0x400000, v013667; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v213668; GFX9-NEXT: v_add3_u32 v3, v3, v0, s1213669; GFX9-NEXT: v_add3_u32 v5, v5, v2, s1213670; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v213671; GFX9-NEXT: v_cmp_u_f32_e64 s[0:1], v0, v013672; GFX9-NEXT: v_cndmask_b32_e64 v0, v3, v4, s[0:1]13673; GFX9-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc13674; GFX9-NEXT: v_perm_b32 v0, v2, v0, s1313675; GFX9-NEXT: v_mov_b32_e32 v3, v113676; GFX9-NEXT: v_mov_b32_e32 v2, v013677; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc13678; GFX9-NEXT: s_waitcnt vmcnt(0)13679; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v2, v113680; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]13681; GFX9-NEXT: v_mov_b32_e32 v1, v213682; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]13683; GFX9-NEXT: s_cbranch_execnz .LBB21_113684; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end13685; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]13686; GFX9-NEXT: s_mov_b32 s11, 0xf00013687; GFX9-NEXT: s_mov_b32 s10, -113688; GFX9-NEXT: buffer_store_dword v2, off, s[8:11], 013689; GFX9-NEXT: s_endpgm13690;13691; GFX1064-LABEL: uniform_fadd_v2bf16:13692; GFX1064: ; %bb.0:13693; GFX1064-NEXT: s_clause 0x113694; GFX1064-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x2413695; GFX1064-NEXT: s_load_dword s0, s[4:5], 0x3413696; GFX1064-NEXT: s_mov_b64 s[2:3], 013697; GFX1064-NEXT: s_mov_b32 s7, 0x3101600013698; GFX1064-NEXT: s_mov_b32 s6, -113699; GFX1064-NEXT: s_waitcnt lgkmcnt(0)13700; GFX1064-NEXT: s_load_dword s1, s[10:11], 0x013701; GFX1064-NEXT: s_lshl_b32 s12, s0, 1613702; GFX1064-NEXT: s_and_b32 s13, s0, 0xffff000013703; GFX1064-NEXT: s_mov_b32 s4, s1013704; GFX1064-NEXT: s_mov_b32 s5, s1113705; GFX1064-NEXT: s_waitcnt lgkmcnt(0)13706; GFX1064-NEXT: v_mov_b32_e32 v1, s113707; GFX1064-NEXT: .LBB21_1: ; %atomicrmw.start13708; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=113709; GFX1064-NEXT: v_lshlrev_b32_e32 v0, 16, v113710; GFX1064-NEXT: v_and_b32_e32 v2, 0xffff0000, v113711; GFX1064-NEXT: v_add_f32_e32 v0, s12, v013712; GFX1064-NEXT: v_add_f32_e32 v2, s13, v213713; GFX1064-NEXT: v_bfe_u32 v3, v0, 16, 113714; GFX1064-NEXT: v_bfe_u32 v4, v2, 16, 113715; GFX1064-NEXT: v_or_b32_e32 v5, 0x400000, v013716; GFX1064-NEXT: v_or_b32_e32 v6, 0x400000, v213717; GFX1064-NEXT: v_cmp_u_f32_e32 vcc, v2, v213718; GFX1064-NEXT: v_add3_u32 v3, v3, v0, 0x7fff13719; GFX1064-NEXT: v_add3_u32 v4, v4, v2, 0x7fff13720; GFX1064-NEXT: v_cmp_u_f32_e64 s[0:1], v0, v013721; GFX1064-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc13722; GFX1064-NEXT: v_cndmask_b32_e64 v0, v3, v5, s[0:1]13723; GFX1064-NEXT: v_perm_b32 v0, v2, v0, 0x706030213724; GFX1064-NEXT: v_mov_b32_e32 v3, v113725; GFX1064-NEXT: v_mov_b32_e32 v2, v013726; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc13727; GFX1064-NEXT: s_waitcnt vmcnt(0)13728; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v2, v113729; GFX1064-NEXT: v_mov_b32_e32 v1, v213730; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]13731; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]13732; GFX1064-NEXT: s_cbranch_execnz .LBB21_113733; GFX1064-NEXT: ; %bb.2: ; %atomicrmw.end13734; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]13735; GFX1064-NEXT: s_mov_b32 s11, 0x3101600013736; GFX1064-NEXT: s_mov_b32 s10, -113737; GFX1064-NEXT: buffer_store_dword v2, off, s[8:11], 013738; GFX1064-NEXT: s_endpgm13739;13740; GFX1032-LABEL: uniform_fadd_v2bf16:13741; GFX1032: ; %bb.0:13742; GFX1032-NEXT: s_clause 0x113743; GFX1032-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x2413744; GFX1032-NEXT: s_load_dword s0, s[4:5], 0x3413745; GFX1032-NEXT: s_mov_b32 s1, 013746; GFX1032-NEXT: s_mov_b32 s7, 0x3101600013747; GFX1032-NEXT: s_mov_b32 s6, -113748; GFX1032-NEXT: s_waitcnt lgkmcnt(0)13749; GFX1032-NEXT: s_load_dword s4, s[10:11], 0x013750; GFX1032-NEXT: s_lshl_b32 s2, s0, 1613751; GFX1032-NEXT: s_and_b32 s3, s0, 0xffff000013752; GFX1032-NEXT: s_mov_b32 s5, s1113753; GFX1032-NEXT: s_waitcnt lgkmcnt(0)13754; GFX1032-NEXT: v_mov_b32_e32 v1, s413755; GFX1032-NEXT: s_mov_b32 s4, s1013756; GFX1032-NEXT: .LBB21_1: ; %atomicrmw.start13757; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=113758; GFX1032-NEXT: v_lshlrev_b32_e32 v0, 16, v113759; GFX1032-NEXT: v_and_b32_e32 v2, 0xffff0000, v113760; GFX1032-NEXT: v_add_f32_e32 v0, s2, v013761; GFX1032-NEXT: v_add_f32_e32 v2, s3, v213762; GFX1032-NEXT: v_bfe_u32 v3, v0, 16, 113763; GFX1032-NEXT: v_bfe_u32 v4, v2, 16, 113764; GFX1032-NEXT: v_or_b32_e32 v5, 0x400000, v013765; GFX1032-NEXT: v_or_b32_e32 v6, 0x400000, v213766; GFX1032-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v213767; GFX1032-NEXT: v_add3_u32 v3, v3, v0, 0x7fff13768; GFX1032-NEXT: v_add3_u32 v4, v4, v2, 0x7fff13769; GFX1032-NEXT: v_cmp_u_f32_e64 s0, v0, v013770; GFX1032-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo13771; GFX1032-NEXT: v_cndmask_b32_e64 v0, v3, v5, s013772; GFX1032-NEXT: v_perm_b32 v0, v2, v0, 0x706030213773; GFX1032-NEXT: v_mov_b32_e32 v3, v113774; GFX1032-NEXT: v_mov_b32_e32 v2, v013775; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc13776; GFX1032-NEXT: s_waitcnt vmcnt(0)13777; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v113778; GFX1032-NEXT: v_mov_b32_e32 v1, v213779; GFX1032-NEXT: s_or_b32 s1, vcc_lo, s113780; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s113781; GFX1032-NEXT: s_cbranch_execnz .LBB21_113782; GFX1032-NEXT: ; %bb.2: ; %atomicrmw.end13783; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s113784; GFX1032-NEXT: s_mov_b32 s11, 0x3101600013785; GFX1032-NEXT: s_mov_b32 s10, -113786; GFX1032-NEXT: buffer_store_dword v2, off, s[8:11], 013787; GFX1032-NEXT: s_endpgm13788;13789; GFX1164-TRUE16-LABEL: uniform_fadd_v2bf16:13790; GFX1164-TRUE16: ; %bb.0:13791; GFX1164-TRUE16-NEXT: s_clause 0x113792; GFX1164-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2413793; GFX1164-TRUE16-NEXT: s_load_b32 s4, s[4:5], 0x3413794; GFX1164-TRUE16-NEXT: s_mov_b64 s[8:9], 013795; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x3101600013796; GFX1164-TRUE16-NEXT: s_mov_b32 s6, -113797; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)13798; GFX1164-TRUE16-NEXT: s_load_b32 s5, s[2:3], 0x013799; GFX1164-TRUE16-NEXT: s_and_b32 s10, s4, 0xffff000013800; GFX1164-TRUE16-NEXT: s_lshl_b32 s11, s4, 1613801; GFX1164-TRUE16-NEXT: s_mov_b32 s4, s213802; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)13803; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s513804; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s313805; GFX1164-TRUE16-NEXT: s_set_inst_prefetch_distance 0x113806; GFX1164-TRUE16-NEXT: .p2align 613807; GFX1164-TRUE16-NEXT: .LBB21_1: ; %atomicrmw.start13808; GFX1164-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=113809; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)13810; GFX1164-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v113811; GFX1164-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v113812; GFX1164-TRUE16-NEXT: v_add_f32_e32 v0, s11, v013813; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)13814; GFX1164-TRUE16-NEXT: v_add_f32_e32 v2, s10, v213815; GFX1164-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 113816; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)13817; GFX1164-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 113818; GFX1164-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v013819; GFX1164-TRUE16-NEXT: v_cmp_u_f32_e32 vcc, v0, v013820; GFX1164-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v213821; GFX1164-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff13822; GFX1164-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff13823; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)13824; GFX1164-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc13825; GFX1164-TRUE16-NEXT: v_cmp_u_f32_e32 vcc, v2, v213826; GFX1164-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)13827; GFX1164-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc13828; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)13829; GFX1164-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h13830; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v3, v113831; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v2, v013832; GFX1164-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc13833; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)13834; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v113835; GFX1164-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)13836; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v213837; GFX1164-TRUE16-NEXT: s_or_b64 s[8:9], vcc, s[8:9]13838; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)13839; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[8:9]13840; GFX1164-TRUE16-NEXT: s_cbranch_execnz .LBB21_113841; GFX1164-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end13842; GFX1164-TRUE16-NEXT: s_set_inst_prefetch_distance 0x213843; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[8:9]13844; GFX1164-TRUE16-NEXT: s_mov_b32 s3, 0x3101600013845; GFX1164-TRUE16-NEXT: s_mov_b32 s2, -113846; GFX1164-TRUE16-NEXT: buffer_store_b32 v2, off, s[0:3], 013847; GFX1164-TRUE16-NEXT: s_endpgm13848;13849; GFX1164-FAKE16-LABEL: uniform_fadd_v2bf16:13850; GFX1164-FAKE16: ; %bb.0:13851; GFX1164-FAKE16-NEXT: s_clause 0x113852; GFX1164-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x2413853; GFX1164-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x3413854; GFX1164-FAKE16-NEXT: s_mov_b64 s[2:3], 013855; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x3101600013856; GFX1164-FAKE16-NEXT: s_mov_b32 s6, -113857; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)13858; GFX1164-FAKE16-NEXT: s_load_b32 s1, s[10:11], 0x013859; GFX1164-FAKE16-NEXT: s_lshl_b32 s12, s0, 1613860; GFX1164-FAKE16-NEXT: s_and_b32 s13, s0, 0xffff000013861; GFX1164-FAKE16-NEXT: s_mov_b32 s4, s1013862; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s1113863; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)13864; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s113865; GFX1164-FAKE16-NEXT: s_set_inst_prefetch_distance 0x113866; GFX1164-FAKE16-NEXT: .p2align 613867; GFX1164-FAKE16-NEXT: .LBB21_1: ; %atomicrmw.start13868; GFX1164-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=113869; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)13870; GFX1164-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v113871; GFX1164-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v113872; GFX1164-FAKE16-NEXT: v_add_f32_e32 v0, s12, v013873; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)13874; GFX1164-FAKE16-NEXT: v_add_f32_e32 v2, s13, v213875; GFX1164-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 113876; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)13877; GFX1164-FAKE16-NEXT: v_bfe_u32 v4, v2, 16, 113878; GFX1164-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v013879; GFX1164-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v213880; GFX1164-FAKE16-NEXT: v_cmp_u_f32_e32 vcc, v2, v213881; GFX1164-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff13882; GFX1164-FAKE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff13883; GFX1164-FAKE16-NEXT: v_cmp_u_f32_e64 s[0:1], v0, v013884; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_sdst(0)13885; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)13886; GFX1164-FAKE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc13887; GFX1164-FAKE16-NEXT: v_cndmask_b32_e64 v0, v3, v5, s[0:1]13888; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)13889; GFX1164-FAKE16-NEXT: v_perm_b32 v0, v2, v0, 0x706030213890; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v3, v113891; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v2, v013892; GFX1164-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc13893; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)13894; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v113895; GFX1164-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vcc(0)13896; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v213897; GFX1164-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]13898; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)13899; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]13900; GFX1164-FAKE16-NEXT: s_cbranch_execnz .LBB21_113901; GFX1164-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end13902; GFX1164-FAKE16-NEXT: s_set_inst_prefetch_distance 0x213903; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]13904; GFX1164-FAKE16-NEXT: s_mov_b32 s11, 0x3101600013905; GFX1164-FAKE16-NEXT: s_mov_b32 s10, -113906; GFX1164-FAKE16-NEXT: buffer_store_b32 v2, off, s[8:11], 013907; GFX1164-FAKE16-NEXT: s_endpgm13908;13909; GFX1132-TRUE16-LABEL: uniform_fadd_v2bf16:13910; GFX1132-TRUE16: ; %bb.0:13911; GFX1132-TRUE16-NEXT: s_clause 0x113912; GFX1132-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2413913; GFX1132-TRUE16-NEXT: s_load_b32 s4, s[4:5], 0x3413914; GFX1132-TRUE16-NEXT: s_mov_b32 s8, 013915; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x3101600013916; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -113917; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)13918; GFX1132-TRUE16-NEXT: s_load_b32 s5, s[2:3], 0x013919; GFX1132-TRUE16-NEXT: s_and_b32 s9, s4, 0xffff000013920; GFX1132-TRUE16-NEXT: s_lshl_b32 s10, s4, 1613921; GFX1132-TRUE16-NEXT: s_mov_b32 s4, s213922; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)13923; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s513924; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s313925; GFX1132-TRUE16-NEXT: s_set_inst_prefetch_distance 0x113926; GFX1132-TRUE16-NEXT: .p2align 613927; GFX1132-TRUE16-NEXT: .LBB21_1: ; %atomicrmw.start13928; GFX1132-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=113929; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)13930; GFX1132-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v113931; GFX1132-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v113932; GFX1132-TRUE16-NEXT: v_add_f32_e32 v0, s10, v013933; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)13934; GFX1132-TRUE16-NEXT: v_add_f32_e32 v2, s9, v213935; GFX1132-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 113936; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)13937; GFX1132-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 113938; GFX1132-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v013939; GFX1132-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v013940; GFX1132-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v213941; GFX1132-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff13942; GFX1132-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff13943; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)13944; GFX1132-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc_lo13945; GFX1132-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v213946; GFX1132-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc_lo13947; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)13948; GFX1132-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h13949; GFX1132-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v013950; GFX1132-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc13951; GFX1132-TRUE16-NEXT: s_waitcnt vmcnt(0)13952; GFX1132-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v113953; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, v213954; GFX1132-TRUE16-NEXT: s_or_b32 s8, vcc_lo, s813955; GFX1132-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)13956; GFX1132-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s813957; GFX1132-TRUE16-NEXT: s_cbranch_execnz .LBB21_113958; GFX1132-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end13959; GFX1132-TRUE16-NEXT: s_set_inst_prefetch_distance 0x213960; GFX1132-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s813961; GFX1132-TRUE16-NEXT: s_mov_b32 s3, 0x3101600013962; GFX1132-TRUE16-NEXT: s_mov_b32 s2, -113963; GFX1132-TRUE16-NEXT: buffer_store_b32 v2, off, s[0:3], 013964; GFX1132-TRUE16-NEXT: s_endpgm13965;13966; GFX1132-FAKE16-LABEL: uniform_fadd_v2bf16:13967; GFX1132-FAKE16: ; %bb.0:13968; GFX1132-FAKE16-NEXT: s_clause 0x113969; GFX1132-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x2413970; GFX1132-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x3413971; GFX1132-FAKE16-NEXT: s_mov_b32 s1, 013972; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x3101600013973; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -113974; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)13975; GFX1132-FAKE16-NEXT: s_load_b32 s4, s[10:11], 0x013976; GFX1132-FAKE16-NEXT: s_lshl_b32 s2, s0, 1613977; GFX1132-FAKE16-NEXT: s_and_b32 s3, s0, 0xffff000013978; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s1113979; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)13980; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s413981; GFX1132-FAKE16-NEXT: s_mov_b32 s4, s1013982; GFX1132-FAKE16-NEXT: s_set_inst_prefetch_distance 0x113983; GFX1132-FAKE16-NEXT: .p2align 613984; GFX1132-FAKE16-NEXT: .LBB21_1: ; %atomicrmw.start13985; GFX1132-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=113986; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)13987; GFX1132-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v113988; GFX1132-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v113989; GFX1132-FAKE16-NEXT: v_add_f32_e32 v0, s2, v013990; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)13991; GFX1132-FAKE16-NEXT: v_add_f32_e32 v2, s3, v213992; GFX1132-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 113993; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)13994; GFX1132-FAKE16-NEXT: v_bfe_u32 v4, v2, 16, 113995; GFX1132-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v013996; GFX1132-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v213997; GFX1132-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v213998; GFX1132-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff13999; GFX1132-FAKE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff14000; GFX1132-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v0, v014001; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)14002; GFX1132-FAKE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo14003; GFX1132-FAKE16-NEXT: v_cndmask_b32_e64 v0, v3, v5, s014004; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)14005; GFX1132-FAKE16-NEXT: v_perm_b32 v0, v2, v0, 0x706030214006; GFX1132-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v014007; GFX1132-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc14008; GFX1132-FAKE16-NEXT: s_waitcnt vmcnt(0)14009; GFX1132-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v114010; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, v214011; GFX1132-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s114012; GFX1132-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)14013; GFX1132-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s114014; GFX1132-FAKE16-NEXT: s_cbranch_execnz .LBB21_114015; GFX1132-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end14016; GFX1132-FAKE16-NEXT: s_set_inst_prefetch_distance 0x214017; GFX1132-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s114018; GFX1132-FAKE16-NEXT: s_mov_b32 s11, 0x3101600014019; GFX1132-FAKE16-NEXT: s_mov_b32 s10, -114020; GFX1132-FAKE16-NEXT: buffer_store_b32 v2, off, s[8:11], 014021; GFX1132-FAKE16-NEXT: s_endpgm14022;14023; GFX1264-TRUE16-LABEL: uniform_fadd_v2bf16:14024; GFX1264-TRUE16: ; %bb.0:14025; GFX1264-TRUE16-NEXT: s_clause 0x114026; GFX1264-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2414027; GFX1264-TRUE16-NEXT: s_load_b32 s4, s[4:5], 0x3414028; GFX1264-TRUE16-NEXT: s_mov_b64 s[8:9], 014029; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x3101600014030; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -114031; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x014032; GFX1264-TRUE16-NEXT: s_load_b32 s5, s[2:3], 0x014033; GFX1264-TRUE16-NEXT: s_and_b32 s10, s4, 0xffff000014034; GFX1264-TRUE16-NEXT: s_lshl_b32 s11, s4, 1614035; GFX1264-TRUE16-NEXT: s_mov_b32 s4, s214036; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x014037; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s514038; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s314039; GFX1264-TRUE16-NEXT: .LBB21_1: ; %atomicrmw.start14040; GFX1264-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=114041; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)14042; GFX1264-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v114043; GFX1264-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v114044; GFX1264-TRUE16-NEXT: v_add_f32_e32 v0, s11, v014045; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)14046; GFX1264-TRUE16-NEXT: v_add_f32_e32 v2, s10, v214047; GFX1264-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 114048; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)14049; GFX1264-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 114050; GFX1264-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v014051; GFX1264-TRUE16-NEXT: v_cmp_u_f32_e32 vcc, v0, v014052; GFX1264-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v214053; GFX1264-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff14054; GFX1264-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff14055; GFX1264-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)14056; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)14057; GFX1264-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc14058; GFX1264-TRUE16-NEXT: v_cmp_u_f32_e32 vcc, v2, v214059; GFX1264-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)14060; GFX1264-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc14061; GFX1264-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h14062; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v3, v114063; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)14064; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v2, v014065; GFX1264-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS14066; GFX1264-TRUE16-NEXT: s_wait_loadcnt 0x014067; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v114068; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, v214069; GFX1264-TRUE16-NEXT: s_or_b64 s[8:9], vcc, s[8:9]14070; GFX1264-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)14071; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[8:9]14072; GFX1264-TRUE16-NEXT: s_cbranch_execnz .LBB21_114073; GFX1264-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end14074; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[8:9]14075; GFX1264-TRUE16-NEXT: s_mov_b32 s3, 0x3101600014076; GFX1264-TRUE16-NEXT: s_mov_b32 s2, -114077; GFX1264-TRUE16-NEXT: buffer_store_b32 v2, off, s[0:3], null14078; GFX1264-TRUE16-NEXT: s_endpgm14079;14080; GFX1264-FAKE16-LABEL: uniform_fadd_v2bf16:14081; GFX1264-FAKE16: ; %bb.0:14082; GFX1264-FAKE16-NEXT: s_clause 0x114083; GFX1264-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x2414084; GFX1264-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x3414085; GFX1264-FAKE16-NEXT: s_mov_b64 s[2:3], 014086; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x3101600014087; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -114088; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x014089; GFX1264-FAKE16-NEXT: s_load_b32 s1, s[10:11], 0x014090; GFX1264-FAKE16-NEXT: s_lshl_b32 s12, s0, 1614091; GFX1264-FAKE16-NEXT: s_and_b32 s13, s0, 0xffff000014092; GFX1264-FAKE16-NEXT: s_mov_b32 s4, s1014093; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s1114094; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x014095; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s114096; GFX1264-FAKE16-NEXT: .LBB21_1: ; %atomicrmw.start14097; GFX1264-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=114098; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)14099; GFX1264-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v114100; GFX1264-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v114101; GFX1264-FAKE16-NEXT: v_add_f32_e32 v0, s12, v014102; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)14103; GFX1264-FAKE16-NEXT: v_add_f32_e32 v2, s13, v214104; GFX1264-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 114105; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)14106; GFX1264-FAKE16-NEXT: v_bfe_u32 v4, v2, 16, 114107; GFX1264-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v014108; GFX1264-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v214109; GFX1264-FAKE16-NEXT: v_cmp_u_f32_e32 vcc, v2, v214110; GFX1264-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff14111; GFX1264-FAKE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff14112; GFX1264-FAKE16-NEXT: v_cmp_u_f32_e64 s[0:1], v0, v014113; GFX1264-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)14114; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)14115; GFX1264-FAKE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc14116; GFX1264-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)14117; GFX1264-FAKE16-NEXT: v_cndmask_b32_e64 v0, v3, v5, s[0:1]14118; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)14119; GFX1264-FAKE16-NEXT: v_perm_b32 v0, v2, v0, 0x706030214120; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v3, v114121; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v2, v014122; GFX1264-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS14123; GFX1264-FAKE16-NEXT: s_wait_loadcnt 0x014124; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v114125; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, v214126; GFX1264-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]14127; GFX1264-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)14128; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]14129; GFX1264-FAKE16-NEXT: s_cbranch_execnz .LBB21_114130; GFX1264-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end14131; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]14132; GFX1264-FAKE16-NEXT: s_mov_b32 s11, 0x3101600014133; GFX1264-FAKE16-NEXT: s_mov_b32 s10, -114134; GFX1264-FAKE16-NEXT: buffer_store_b32 v2, off, s[8:11], null14135; GFX1264-FAKE16-NEXT: s_endpgm14136;14137; GFX1232-TRUE16-LABEL: uniform_fadd_v2bf16:14138; GFX1232-TRUE16: ; %bb.0:14139; GFX1232-TRUE16-NEXT: s_clause 0x114140; GFX1232-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2414141; GFX1232-TRUE16-NEXT: s_load_b32 s4, s[4:5], 0x3414142; GFX1232-TRUE16-NEXT: s_mov_b32 s8, 014143; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x3101600014144; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -114145; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x014146; GFX1232-TRUE16-NEXT: s_load_b32 s5, s[2:3], 0x014147; GFX1232-TRUE16-NEXT: s_and_b32 s9, s4, 0xffff000014148; GFX1232-TRUE16-NEXT: s_lshl_b32 s10, s4, 1614149; GFX1232-TRUE16-NEXT: s_mov_b32 s4, s214150; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x014151; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s514152; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s314153; GFX1232-TRUE16-NEXT: .LBB21_1: ; %atomicrmw.start14154; GFX1232-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=114155; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)14156; GFX1232-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v114157; GFX1232-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v114158; GFX1232-TRUE16-NEXT: v_add_f32_e32 v0, s10, v014159; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)14160; GFX1232-TRUE16-NEXT: v_add_f32_e32 v2, s9, v214161; GFX1232-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 114162; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)14163; GFX1232-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 114164; GFX1232-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v014165; GFX1232-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v014166; GFX1232-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v214167; GFX1232-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff14168; GFX1232-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff14169; GFX1232-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)14170; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)14171; GFX1232-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc_lo14172; GFX1232-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v214173; GFX1232-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)14174; GFX1232-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc_lo14175; GFX1232-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h14176; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)14177; GFX1232-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v014178; GFX1232-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS14179; GFX1232-TRUE16-NEXT: s_wait_loadcnt 0x014180; GFX1232-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v114181; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, v214182; GFX1232-TRUE16-NEXT: s_or_b32 s8, vcc_lo, s814183; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)14184; GFX1232-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s814185; GFX1232-TRUE16-NEXT: s_cbranch_execnz .LBB21_114186; GFX1232-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end14187; GFX1232-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s814188; GFX1232-TRUE16-NEXT: s_mov_b32 s3, 0x3101600014189; GFX1232-TRUE16-NEXT: s_mov_b32 s2, -114190; GFX1232-TRUE16-NEXT: buffer_store_b32 v2, off, s[0:3], null14191; GFX1232-TRUE16-NEXT: s_endpgm14192;14193; GFX1232-FAKE16-LABEL: uniform_fadd_v2bf16:14194; GFX1232-FAKE16: ; %bb.0:14195; GFX1232-FAKE16-NEXT: s_clause 0x114196; GFX1232-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x2414197; GFX1232-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x3414198; GFX1232-FAKE16-NEXT: s_mov_b32 s1, 014199; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x3101600014200; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -114201; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x014202; GFX1232-FAKE16-NEXT: s_load_b32 s4, s[10:11], 0x014203; GFX1232-FAKE16-NEXT: s_lshl_b32 s2, s0, 1614204; GFX1232-FAKE16-NEXT: s_and_b32 s3, s0, 0xffff000014205; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s1114206; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x014207; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s414208; GFX1232-FAKE16-NEXT: s_mov_b32 s4, s1014209; GFX1232-FAKE16-NEXT: .LBB21_1: ; %atomicrmw.start14210; GFX1232-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=114211; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)14212; GFX1232-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v114213; GFX1232-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v114214; GFX1232-FAKE16-NEXT: v_add_f32_e32 v0, s2, v014215; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)14216; GFX1232-FAKE16-NEXT: v_add_f32_e32 v2, s3, v214217; GFX1232-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 114218; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)14219; GFX1232-FAKE16-NEXT: v_bfe_u32 v4, v2, 16, 114220; GFX1232-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v014221; GFX1232-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v214222; GFX1232-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v214223; GFX1232-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff14224; GFX1232-FAKE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff14225; GFX1232-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v0, v014226; GFX1232-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)14227; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)14228; GFX1232-FAKE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo14229; GFX1232-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)14230; GFX1232-FAKE16-NEXT: v_cndmask_b32_e64 v0, v3, v5, s014231; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)14232; GFX1232-FAKE16-NEXT: v_perm_b32 v0, v2, v0, 0x706030214233; GFX1232-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v014234; GFX1232-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS14235; GFX1232-FAKE16-NEXT: s_wait_loadcnt 0x014236; GFX1232-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v114237; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, v214238; GFX1232-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s114239; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)14240; GFX1232-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s114241; GFX1232-FAKE16-NEXT: s_cbranch_execnz .LBB21_114242; GFX1232-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end14243; GFX1232-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s114244; GFX1232-FAKE16-NEXT: s_mov_b32 s11, 0x3101600014245; GFX1232-FAKE16-NEXT: s_mov_b32 s10, -114246; GFX1232-FAKE16-NEXT: buffer_store_b32 v2, off, s[8:11], null14247; GFX1232-FAKE16-NEXT: s_endpgm14248 %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, <2 x bfloat> %val monotonic, align 414249 store <2 x bfloat> %rmw, ptr addrspace(1) %result14250 ret void14251}14252;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:14253; GFX1132_DPP-FAKE16: {{.*}}14254; GFX1132_DPP-TRUE16: {{.*}}14255; GFX1132_ITERATIVE-FAKE16: {{.*}}14256; GFX1132_ITERATIVE-TRUE16: {{.*}}14257; GFX1164_DPP-FAKE16: {{.*}}14258; GFX1164_DPP-TRUE16: {{.*}}14259; GFX1164_ITERATIVE-FAKE16: {{.*}}14260; GFX1164_ITERATIVE-TRUE16: {{.*}}14261; GFX1232_DPP-FAKE16: {{.*}}14262; GFX1232_DPP-TRUE16: {{.*}}14263; GFX1232_ITERATIVE-FAKE16: {{.*}}14264; GFX1232_ITERATIVE-TRUE16: {{.*}}14265; GFX1264_DPP-FAKE16: {{.*}}14266; GFX1264_DPP-TRUE16: {{.*}}14267; GFX1264_ITERATIVE-FAKE16: {{.*}}14268; GFX1264_ITERATIVE-TRUE16: {{.*}}14269