1126 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck -check-prefix=GFX908 %s3; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -check-prefix=GFX90A %s4; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -check-prefix=GFX942 %s5; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck -check-prefix=GFX1100 %s6; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX1200 %s7 8define float @syncscope_system(ptr %addr, float %val) #0 {9; GFX908-LABEL: syncscope_system:10; GFX908: ; %bb.0:11; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12; GFX908-NEXT: flat_load_dword v3, v[0:1]13; GFX908-NEXT: s_mov_b64 s[4:5], 014; GFX908-NEXT: .LBB0_1: ; %atomicrmw.start15; GFX908-NEXT: ; =>This Inner Loop Header: Depth=116; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)17; GFX908-NEXT: v_mov_b32_e32 v4, v318; GFX908-NEXT: v_add_f32_e32 v3, v4, v219; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc20; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)21; GFX908-NEXT: buffer_wbinvl1_vol22; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v423; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]24; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]25; GFX908-NEXT: s_cbranch_execnz .LBB0_126; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end27; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]28; GFX908-NEXT: v_mov_b32_e32 v0, v329; GFX908-NEXT: s_setpc_b64 s[30:31]30;31; GFX90A-LABEL: syncscope_system:32; GFX90A: ; %bb.0:33; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)34; GFX90A-NEXT: s_mov_b64 s[4:5], src_shared_base35; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v136; GFX90A-NEXT: ; implicit-def: $vgpr337; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc38; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]39; GFX90A-NEXT: s_cbranch_execz .LBB0_640; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.check.private41; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base42; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s7, v143; GFX90A-NEXT: ; implicit-def: $vgpr344; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], vcc45; GFX90A-NEXT: s_xor_b64 s[6:7], exec, s[6:7]46; GFX90A-NEXT: s_cbranch_execz .LBB0_347; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.global48; GFX90A-NEXT: buffer_wbl249; GFX90A-NEXT: global_atomic_add_f32 v3, v[0:1], v2, off glc50; GFX90A-NEXT: s_waitcnt vmcnt(0)51; GFX90A-NEXT: buffer_invl252; GFX90A-NEXT: buffer_wbinvl1_vol53; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr154; GFX90A-NEXT: ; implicit-def: $vgpr255; GFX90A-NEXT: .LBB0_3: ; %Flow56; GFX90A-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]57; GFX90A-NEXT: s_cbranch_execz .LBB0_558; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private59; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]60; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc61; GFX90A-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen62; GFX90A-NEXT: s_waitcnt vmcnt(0)63; GFX90A-NEXT: v_add_f32_e32 v1, v3, v264; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen65; GFX90A-NEXT: .LBB0_5: ; %Flow166; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]67; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr168; GFX90A-NEXT: ; implicit-def: $vgpr269; GFX90A-NEXT: .LBB0_6: ; %Flow270; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]71; GFX90A-NEXT: s_cbranch_execz .LBB0_872; GFX90A-NEXT: ; %bb.7: ; %atomicrmw.shared73; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]74; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc75; GFX90A-NEXT: ds_add_rtn_f32 v3, v0, v276; GFX90A-NEXT: s_waitcnt lgkmcnt(0)77; GFX90A-NEXT: .LBB0_8: ; %atomicrmw.phi78; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]79; GFX90A-NEXT: v_mov_b32_e32 v0, v380; GFX90A-NEXT: s_waitcnt vmcnt(0)81; GFX90A-NEXT: s_setpc_b64 s[30:31]82;83; GFX942-LABEL: syncscope_system:84; GFX942: ; %bb.0:85; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)86; GFX942-NEXT: buffer_wbl2 sc0 sc187; GFX942-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 sc0 sc188; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)89; GFX942-NEXT: buffer_inv sc0 sc190; GFX942-NEXT: s_setpc_b64 s[30:31]91;92; GFX1100-LABEL: syncscope_system:93; GFX1100: ; %bb.0:94; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)95; GFX1100-NEXT: s_waitcnt_vscnt null, 0x096; GFX1100-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 glc97; GFX1100-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)98; GFX1100-NEXT: buffer_gl1_inv99; GFX1100-NEXT: buffer_gl0_inv100; GFX1100-NEXT: s_setpc_b64 s[30:31]101;102; GFX1200-LABEL: syncscope_system:103; GFX1200: ; %bb.0:104; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0105; GFX1200-NEXT: s_wait_expcnt 0x0106; GFX1200-NEXT: s_wait_samplecnt 0x0107; GFX1200-NEXT: s_wait_bvhcnt 0x0108; GFX1200-NEXT: s_wait_kmcnt 0x0109; GFX1200-NEXT: global_wb scope:SCOPE_SYS110; GFX1200-NEXT: s_wait_storecnt 0x0111; GFX1200-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SYS112; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0113; GFX1200-NEXT: global_inv scope:SCOPE_SYS114; GFX1200-NEXT: s_wait_loadcnt 0x0115; GFX1200-NEXT: s_setpc_b64 s[30:31]116 %res = atomicrmw fadd ptr %addr, float %val seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0117 ret float %res118}119 120define float @syncscope_workgroup_rtn(ptr %addr, float %val) #0 {121; GFX908-LABEL: syncscope_workgroup_rtn:122; GFX908: ; %bb.0:123; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)124; GFX908-NEXT: flat_load_dword v3, v[0:1]125; GFX908-NEXT: s_mov_b64 s[4:5], 0126; GFX908-NEXT: .LBB1_1: ; %atomicrmw.start127; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1128; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)129; GFX908-NEXT: v_mov_b32_e32 v4, v3130; GFX908-NEXT: v_add_f32_e32 v3, v4, v2131; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc132; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)133; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4134; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]135; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]136; GFX908-NEXT: s_cbranch_execnz .LBB1_1137; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end138; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]139; GFX908-NEXT: v_mov_b32_e32 v0, v3140; GFX908-NEXT: s_setpc_b64 s[30:31]141;142; GFX90A-LABEL: syncscope_workgroup_rtn:143; GFX90A: ; %bb.0:144; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)145; GFX90A-NEXT: s_mov_b64 s[4:5], src_shared_base146; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1147; GFX90A-NEXT: ; implicit-def: $vgpr3148; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc149; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]150; GFX90A-NEXT: s_cbranch_execz .LBB1_6151; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.check.private152; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base153; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s7, v1154; GFX90A-NEXT: ; implicit-def: $vgpr3155; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], vcc156; GFX90A-NEXT: s_xor_b64 s[6:7], exec, s[6:7]157; GFX90A-NEXT: s_cbranch_execz .LBB1_3158; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.global159; GFX90A-NEXT: global_atomic_add_f32 v3, v[0:1], v2, off glc160; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1161; GFX90A-NEXT: ; implicit-def: $vgpr2162; GFX90A-NEXT: .LBB1_3: ; %Flow163; GFX90A-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]164; GFX90A-NEXT: s_cbranch_execz .LBB1_5165; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.private166; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]167; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc168; GFX90A-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen169; GFX90A-NEXT: s_waitcnt vmcnt(0)170; GFX90A-NEXT: v_add_f32_e32 v1, v3, v2171; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen172; GFX90A-NEXT: .LBB1_5: ; %Flow1173; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]174; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1175; GFX90A-NEXT: ; implicit-def: $vgpr2176; GFX90A-NEXT: .LBB1_6: ; %Flow2177; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]178; GFX90A-NEXT: s_cbranch_execz .LBB1_8179; GFX90A-NEXT: ; %bb.7: ; %atomicrmw.shared180; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]181; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc182; GFX90A-NEXT: s_waitcnt vmcnt(0)183; GFX90A-NEXT: ds_add_rtn_f32 v3, v0, v2184; GFX90A-NEXT: s_waitcnt lgkmcnt(0)185; GFX90A-NEXT: .LBB1_8: ; %atomicrmw.phi186; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]187; GFX90A-NEXT: s_waitcnt vmcnt(0)188; GFX90A-NEXT: v_mov_b32_e32 v0, v3189; GFX90A-NEXT: s_setpc_b64 s[30:31]190;191; GFX942-LABEL: syncscope_workgroup_rtn:192; GFX942: ; %bb.0:193; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)194; GFX942-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 sc0195; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)196; GFX942-NEXT: s_setpc_b64 s[30:31]197;198; GFX1100-LABEL: syncscope_workgroup_rtn:199; GFX1100: ; %bb.0:200; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)201; GFX1100-NEXT: s_waitcnt_vscnt null, 0x0202; GFX1100-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 glc203; GFX1100-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)204; GFX1100-NEXT: buffer_gl0_inv205; GFX1100-NEXT: s_setpc_b64 s[30:31]206;207; GFX1200-LABEL: syncscope_workgroup_rtn:208; GFX1200: ; %bb.0:209; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0210; GFX1200-NEXT: s_wait_expcnt 0x0211; GFX1200-NEXT: s_wait_samplecnt 0x0212; GFX1200-NEXT: s_wait_bvhcnt 0x0213; GFX1200-NEXT: s_wait_kmcnt 0x0214; GFX1200-NEXT: s_wait_storecnt 0x0215; GFX1200-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SE216; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0217; GFX1200-NEXT: global_inv scope:SCOPE_SE218; GFX1200-NEXT: s_wait_loadcnt 0x0219; GFX1200-NEXT: s_setpc_b64 s[30:31]220 %res = atomicrmw fadd ptr %addr, float %val syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0221 ret float %res222}223 224define void @syncscope_workgroup_nortn(ptr %addr, float %val) #0 {225; GFX908-LABEL: syncscope_workgroup_nortn:226; GFX908: ; %bb.0:227; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)228; GFX908-NEXT: s_mov_b64 s[4:5], src_shared_base229; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1230; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc231; GFX908-NEXT: s_xor_b64 s[4:5], exec, s[4:5]232; GFX908-NEXT: s_cbranch_execnz .LBB2_3233; GFX908-NEXT: ; %bb.1: ; %Flow2234; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]235; GFX908-NEXT: s_cbranch_execnz .LBB2_8236; GFX908-NEXT: .LBB2_2: ; %atomicrmw.phi237; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]238; GFX908-NEXT: s_waitcnt vmcnt(0)239; GFX908-NEXT: s_setpc_b64 s[30:31]240; GFX908-NEXT: .LBB2_3: ; %atomicrmw.check.private241; GFX908-NEXT: s_mov_b64 s[6:7], src_private_base242; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s7, v1243; GFX908-NEXT: s_and_saveexec_b64 s[6:7], vcc244; GFX908-NEXT: s_xor_b64 s[6:7], exec, s[6:7]245; GFX908-NEXT: s_cbranch_execz .LBB2_5246; GFX908-NEXT: ; %bb.4: ; %atomicrmw.global247; GFX908-NEXT: global_atomic_add_f32 v[0:1], v2, off248; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1249; GFX908-NEXT: ; implicit-def: $vgpr2250; GFX908-NEXT: .LBB2_5: ; %Flow251; GFX908-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]252; GFX908-NEXT: s_cbranch_execz .LBB2_7253; GFX908-NEXT: ; %bb.6: ; %atomicrmw.private254; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]255; GFX908-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc256; GFX908-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen257; GFX908-NEXT: s_waitcnt vmcnt(0)258; GFX908-NEXT: v_add_f32_e32 v1, v1, v2259; GFX908-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen260; GFX908-NEXT: .LBB2_7: ; %Flow1261; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]262; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1263; GFX908-NEXT: ; implicit-def: $vgpr2264; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]265; GFX908-NEXT: s_cbranch_execz .LBB2_2266; GFX908-NEXT: .LBB2_8: ; %atomicrmw.shared267; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]268; GFX908-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc269; GFX908-NEXT: ds_add_f32 v0, v2270; GFX908-NEXT: s_waitcnt lgkmcnt(0)271; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]272; GFX908-NEXT: s_waitcnt vmcnt(0)273; GFX908-NEXT: s_setpc_b64 s[30:31]274;275; GFX90A-LABEL: syncscope_workgroup_nortn:276; GFX90A: ; %bb.0:277; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)278; GFX90A-NEXT: s_mov_b64 s[4:5], src_shared_base279; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1280; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc281; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]282; GFX90A-NEXT: s_cbranch_execnz .LBB2_3283; GFX90A-NEXT: ; %bb.1: ; %Flow2284; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]285; GFX90A-NEXT: s_cbranch_execnz .LBB2_8286; GFX90A-NEXT: .LBB2_2: ; %atomicrmw.phi287; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]288; GFX90A-NEXT: s_waitcnt vmcnt(0)289; GFX90A-NEXT: s_setpc_b64 s[30:31]290; GFX90A-NEXT: .LBB2_3: ; %atomicrmw.check.private291; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base292; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s7, v1293; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], vcc294; GFX90A-NEXT: s_xor_b64 s[6:7], exec, s[6:7]295; GFX90A-NEXT: s_cbranch_execz .LBB2_5296; GFX90A-NEXT: ; %bb.4: ; %atomicrmw.global297; GFX90A-NEXT: global_atomic_add_f32 v[0:1], v2, off298; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1299; GFX90A-NEXT: ; implicit-def: $vgpr2300; GFX90A-NEXT: .LBB2_5: ; %Flow301; GFX90A-NEXT: s_andn2_saveexec_b64 s[6:7], s[6:7]302; GFX90A-NEXT: s_cbranch_execz .LBB2_7303; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.private304; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]305; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc306; GFX90A-NEXT: buffer_load_dword v1, v0, s[0:3], 0 offen307; GFX90A-NEXT: s_waitcnt vmcnt(0)308; GFX90A-NEXT: v_add_f32_e32 v1, v1, v2309; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen310; GFX90A-NEXT: .LBB2_7: ; %Flow1311; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]312; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1313; GFX90A-NEXT: ; implicit-def: $vgpr2314; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]315; GFX90A-NEXT: s_cbranch_execz .LBB2_2316; GFX90A-NEXT: .LBB2_8: ; %atomicrmw.shared317; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]318; GFX90A-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc319; GFX90A-NEXT: ds_add_f32 v0, v2320; GFX90A-NEXT: s_waitcnt lgkmcnt(0)321; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]322; GFX90A-NEXT: s_waitcnt vmcnt(0)323; GFX90A-NEXT: s_setpc_b64 s[30:31]324;325; GFX942-LABEL: syncscope_workgroup_nortn:326; GFX942: ; %bb.0:327; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)328; GFX942-NEXT: flat_atomic_add_f32 v[0:1], v2329; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)330; GFX942-NEXT: s_setpc_b64 s[30:31]331;332; GFX1100-LABEL: syncscope_workgroup_nortn:333; GFX1100: ; %bb.0:334; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)335; GFX1100-NEXT: s_waitcnt_vscnt null, 0x0336; GFX1100-NEXT: flat_atomic_add_f32 v[0:1], v2337; GFX1100-NEXT: s_waitcnt lgkmcnt(0)338; GFX1100-NEXT: s_waitcnt_vscnt null, 0x0339; GFX1100-NEXT: buffer_gl0_inv340; GFX1100-NEXT: s_setpc_b64 s[30:31]341;342; GFX1200-LABEL: syncscope_workgroup_nortn:343; GFX1200: ; %bb.0:344; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0345; GFX1200-NEXT: s_wait_expcnt 0x0346; GFX1200-NEXT: s_wait_samplecnt 0x0347; GFX1200-NEXT: s_wait_bvhcnt 0x0348; GFX1200-NEXT: s_wait_kmcnt 0x0349; GFX1200-NEXT: s_wait_storecnt 0x0350; GFX1200-NEXT: flat_atomic_add_f32 v[0:1], v2 scope:SCOPE_SE351; GFX1200-NEXT: s_wait_storecnt_dscnt 0x0352; GFX1200-NEXT: global_inv scope:SCOPE_SE353; GFX1200-NEXT: s_wait_loadcnt 0x0354; GFX1200-NEXT: s_setpc_b64 s[30:31]355 %res = atomicrmw fadd ptr %addr, float %val syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0356 ret void357}358 359define float @no_unsafe(ptr %addr, float %val) {360; GFX908-LABEL: no_unsafe:361; GFX908: ; %bb.0:362; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)363; GFX908-NEXT: flat_load_dword v3, v[0:1]364; GFX908-NEXT: s_mov_b64 s[4:5], 0365; GFX908-NEXT: .LBB3_1: ; %atomicrmw.start366; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1367; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)368; GFX908-NEXT: v_mov_b32_e32 v4, v3369; GFX908-NEXT: v_add_f32_e32 v3, v4, v2370; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc371; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)372; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4373; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]374; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]375; GFX908-NEXT: s_cbranch_execnz .LBB3_1376; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end377; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]378; GFX908-NEXT: v_mov_b32_e32 v0, v3379; GFX908-NEXT: s_setpc_b64 s[30:31]380;381; GFX90A-LABEL: no_unsafe:382; GFX90A: ; %bb.0:383; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)384; GFX90A-NEXT: flat_load_dword v5, v[0:1]385; GFX90A-NEXT: s_mov_b64 s[4:5], 0386; GFX90A-NEXT: .LBB3_1: ; %atomicrmw.start387; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1388; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)389; GFX90A-NEXT: v_add_f32_e32 v4, v5, v2390; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc391; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)392; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5393; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]394; GFX90A-NEXT: v_mov_b32_e32 v5, v3395; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]396; GFX90A-NEXT: s_cbranch_execnz .LBB3_1397; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end398; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]399; GFX90A-NEXT: v_mov_b32_e32 v0, v3400; GFX90A-NEXT: s_setpc_b64 s[30:31]401;402; GFX942-LABEL: no_unsafe:403; GFX942: ; %bb.0:404; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)405; GFX942-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 sc0406; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)407; GFX942-NEXT: s_setpc_b64 s[30:31]408;409; GFX1100-LABEL: no_unsafe:410; GFX1100: ; %bb.0:411; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)412; GFX1100-NEXT: flat_load_b32 v3, v[0:1]413; GFX1100-NEXT: s_mov_b32 s0, 0414; GFX1100-NEXT: .LBB3_1: ; %atomicrmw.start415; GFX1100-NEXT: ; =>This Inner Loop Header: Depth=1416; GFX1100-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)417; GFX1100-NEXT: v_mov_b32_e32 v4, v3418; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1)419; GFX1100-NEXT: v_add_f32_e32 v3, v4, v2420; GFX1100-NEXT: s_waitcnt_vscnt null, 0x0421; GFX1100-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc422; GFX1100-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)423; GFX1100-NEXT: buffer_gl0_inv424; GFX1100-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4425; GFX1100-NEXT: s_or_b32 s0, vcc_lo, s0426; GFX1100-NEXT: s_delay_alu instid0(SALU_CYCLE_1)427; GFX1100-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0428; GFX1100-NEXT: s_cbranch_execnz .LBB3_1429; GFX1100-NEXT: ; %bb.2: ; %atomicrmw.end430; GFX1100-NEXT: s_or_b32 exec_lo, exec_lo, s0431; GFX1100-NEXT: v_mov_b32_e32 v0, v3432; GFX1100-NEXT: s_setpc_b64 s[30:31]433;434; GFX1200-LABEL: no_unsafe:435; GFX1200: ; %bb.0:436; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0437; GFX1200-NEXT: s_wait_expcnt 0x0438; GFX1200-NEXT: s_wait_samplecnt 0x0439; GFX1200-NEXT: s_wait_bvhcnt 0x0440; GFX1200-NEXT: s_wait_kmcnt 0x0441; GFX1200-NEXT: s_wait_storecnt 0x0442; GFX1200-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SE443; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0444; GFX1200-NEXT: global_inv scope:SCOPE_SE445; GFX1200-NEXT: s_wait_loadcnt 0x0446; GFX1200-NEXT: s_setpc_b64 s[30:31]447 %res = atomicrmw fadd ptr %addr, float %val syncscope("workgroup") seq_cst448 ret float %res449}450 451@global = hidden addrspace(1) global i64 0, align 8452 453; Make sure there is no error on an invalid addrspacecast without optimizations454define i64 @optnone_atomicrmw_add_i64_expand(i64 %val) #1 {455; GFX908-LABEL: optnone_atomicrmw_add_i64_expand:456; GFX908: ; %bb.0:457; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)458; GFX908-NEXT: s_mov_b64 s[4:5], src_private_base459; GFX908-NEXT: s_getpc_b64 s[6:7]460; GFX908-NEXT: s_add_u32 s6, s6, global@rel32@lo+4461; GFX908-NEXT: s_addc_u32 s7, s7, global@rel32@hi+12462; GFX908-NEXT: s_cmp_eq_u32 s7, s5463; GFX908-NEXT: s_cselect_b64 s[4:5], -1, 0464; GFX908-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5]465; GFX908-NEXT: s_mov_b64 s[4:5], -1466; GFX908-NEXT: s_mov_b32 s6, 1467; GFX908-NEXT: v_cmp_ne_u32_e64 s[6:7], v2, s6468; GFX908-NEXT: s_and_b64 vcc, exec, s[6:7]469; GFX908-NEXT: ; implicit-def: $vgpr3_vgpr4470; GFX908-NEXT: s_cbranch_vccnz .LBB4_3471; GFX908-NEXT: .LBB4_1: ; %Flow472; GFX908-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5]473; GFX908-NEXT: s_mov_b32 s4, 1474; GFX908-NEXT: v_cmp_ne_u32_e64 s[4:5], v2, s4475; GFX908-NEXT: s_and_b64 vcc, exec, s[4:5]476; GFX908-NEXT: s_cbranch_vccnz .LBB4_4477; GFX908-NEXT: ; %bb.2: ; %atomicrmw.private478; GFX908-NEXT: s_waitcnt lgkmcnt(0)479; GFX908-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen480; GFX908-NEXT: s_waitcnt vmcnt(0)481; GFX908-NEXT: v_mov_b32_e32 v4, v3482; GFX908-NEXT: v_add_co_u32_e64 v0, s[4:5], v3, v0483; GFX908-NEXT: v_addc_co_u32_e64 v1, s[4:5], v4, v1, s[4:5]484; GFX908-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen485; GFX908-NEXT: buffer_store_dword v0, v0, s[0:3], 0 offen486; GFX908-NEXT: s_branch .LBB4_4487; GFX908-NEXT: .LBB4_3: ; %atomicrmw.global488; GFX908-NEXT: s_getpc_b64 s[4:5]489; GFX908-NEXT: s_add_u32 s4, s4, global@rel32@lo+4490; GFX908-NEXT: s_addc_u32 s5, s5, global@rel32@hi+12491; GFX908-NEXT: v_mov_b32_e32 v2, s4492; GFX908-NEXT: v_mov_b32_e32 v3, s5493; GFX908-NEXT: flat_atomic_add_x2 v[3:4], v[2:3], v[0:1] glc494; GFX908-NEXT: s_mov_b64 s[4:5], 0495; GFX908-NEXT: s_branch .LBB4_1496; GFX908-NEXT: .LBB4_4: ; %atomicrmw.phi497; GFX908-NEXT: ; %bb.5: ; %atomicrmw.end498; GFX908-NEXT: s_mov_b32 s4, 32499; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)500; GFX908-NEXT: v_lshrrev_b64 v[1:2], s4, v[3:4]501; GFX908-NEXT: v_mov_b32_e32 v0, v3502; GFX908-NEXT: s_setpc_b64 s[30:31]503;504; GFX90A-LABEL: optnone_atomicrmw_add_i64_expand:505; GFX90A: ; %bb.0:506; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)507; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base508; GFX90A-NEXT: s_getpc_b64 s[6:7]509; GFX90A-NEXT: s_add_u32 s6, s6, global@rel32@lo+4510; GFX90A-NEXT: s_addc_u32 s7, s7, global@rel32@hi+12511; GFX90A-NEXT: s_cmp_eq_u32 s7, s5512; GFX90A-NEXT: s_cselect_b64 s[4:5], -1, 0513; GFX90A-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5]514; GFX90A-NEXT: s_mov_b64 s[4:5], -1515; GFX90A-NEXT: s_mov_b32 s6, 1516; GFX90A-NEXT: v_cmp_ne_u32_e64 s[6:7], v2, s6517; GFX90A-NEXT: s_and_b64 vcc, exec, s[6:7]518; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3519; GFX90A-NEXT: s_cbranch_vccnz .LBB4_3520; GFX90A-NEXT: .LBB4_1: ; %Flow521; GFX90A-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[4:5]522; GFX90A-NEXT: s_mov_b32 s4, 1523; GFX90A-NEXT: v_cmp_ne_u32_e64 s[4:5], v4, s4524; GFX90A-NEXT: s_and_b64 vcc, exec, s[4:5]525; GFX90A-NEXT: s_cbranch_vccnz .LBB4_4526; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.private527; GFX90A-NEXT: s_waitcnt lgkmcnt(0)528; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen529; GFX90A-NEXT: s_waitcnt vmcnt(0)530; GFX90A-NEXT: v_mov_b32_e32 v3, v2531; GFX90A-NEXT: v_add_co_u32_e64 v0, s[4:5], v2, v0532; GFX90A-NEXT: v_addc_co_u32_e64 v1, s[4:5], v3, v1, s[4:5]533; GFX90A-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen534; GFX90A-NEXT: buffer_store_dword v0, v0, s[0:3], 0 offen535; GFX90A-NEXT: s_branch .LBB4_4536; GFX90A-NEXT: .LBB4_3: ; %atomicrmw.global537; GFX90A-NEXT: s_getpc_b64 s[4:5]538; GFX90A-NEXT: s_add_u32 s4, s4, global@rel32@lo+4539; GFX90A-NEXT: s_addc_u32 s5, s5, global@rel32@hi+12540; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]541; GFX90A-NEXT: flat_atomic_add_x2 v[2:3], v[2:3], v[0:1] glc542; GFX90A-NEXT: s_mov_b64 s[4:5], 0543; GFX90A-NEXT: s_branch .LBB4_1544; GFX90A-NEXT: .LBB4_4: ; %atomicrmw.phi545; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.end546; GFX90A-NEXT: s_mov_b32 s4, 32547; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)548; GFX90A-NEXT: v_lshrrev_b64 v[4:5], s4, v[2:3]549; GFX90A-NEXT: v_mov_b32_e32 v0, v2550; GFX90A-NEXT: v_mov_b32_e32 v1, v4551; GFX90A-NEXT: s_setpc_b64 s[30:31]552;553; GFX942-LABEL: optnone_atomicrmw_add_i64_expand:554; GFX942: ; %bb.0:555; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)556; GFX942-NEXT: s_mov_b64 s[0:1], src_private_base557; GFX942-NEXT: s_getpc_b64 s[2:3]558; GFX942-NEXT: s_add_u32 s2, s2, global@rel32@lo+4559; GFX942-NEXT: s_addc_u32 s3, s3, global@rel32@hi+12560; GFX942-NEXT: s_cmp_eq_u32 s3, s1561; GFX942-NEXT: s_cselect_b64 s[0:1], -1, 0562; GFX942-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]563; GFX942-NEXT: s_mov_b64 s[0:1], -1564; GFX942-NEXT: s_mov_b32 s2, 1565; GFX942-NEXT: v_cmp_ne_u32_e64 s[2:3], v2, s2566; GFX942-NEXT: s_and_b64 vcc, exec, s[2:3]567; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3568; GFX942-NEXT: s_cbranch_vccnz .LBB4_3569; GFX942-NEXT: .LBB4_1: ; %Flow570; GFX942-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[0:1]571; GFX942-NEXT: s_mov_b32 s0, 1572; GFX942-NEXT: v_cmp_ne_u32_e64 s[0:1], v4, s0573; GFX942-NEXT: s_and_b64 vcc, exec, s[0:1]574; GFX942-NEXT: s_cbranch_vccnz .LBB4_4575; GFX942-NEXT: ; %bb.2: ; %atomicrmw.private576; GFX942-NEXT: s_waitcnt lgkmcnt(0)577; GFX942-NEXT: s_nop 1578; GFX942-NEXT: scratch_load_dwordx2 v[2:3], off, s0579; GFX942-NEXT: s_waitcnt vmcnt(0)580; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[2:3], 0, v[0:1]581; GFX942-NEXT: scratch_store_dwordx2 off, v[0:1], s0582; GFX942-NEXT: s_branch .LBB4_4583; GFX942-NEXT: .LBB4_3: ; %atomicrmw.global584; GFX942-NEXT: s_getpc_b64 s[0:1]585; GFX942-NEXT: s_add_u32 s0, s0, global@rel32@lo+4586; GFX942-NEXT: s_addc_u32 s1, s1, global@rel32@hi+12587; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[0:1]588; GFX942-NEXT: flat_atomic_add_x2 v[2:3], v[2:3], v[0:1] sc0589; GFX942-NEXT: s_mov_b64 s[0:1], 0590; GFX942-NEXT: s_branch .LBB4_1591; GFX942-NEXT: .LBB4_4: ; %atomicrmw.phi592; GFX942-NEXT: ; %bb.5: ; %atomicrmw.end593; GFX942-NEXT: s_mov_b32 s0, 32594; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)595; GFX942-NEXT: v_lshrrev_b64 v[4:5], s0, v[2:3]596; GFX942-NEXT: v_mov_b32_e32 v0, v2597; GFX942-NEXT: v_mov_b32_e32 v1, v4598; GFX942-NEXT: s_setpc_b64 s[30:31]599;600; GFX1100-LABEL: optnone_atomicrmw_add_i64_expand:601; GFX1100: ; %bb.0:602; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)603; GFX1100-NEXT: s_mov_b64 s[0:1], src_private_base604; GFX1100-NEXT: s_getpc_b64 s[2:3]605; GFX1100-NEXT: s_add_u32 s2, s2, global@rel32@lo+4606; GFX1100-NEXT: s_addc_u32 s3, s3, global@rel32@hi+12607; GFX1100-NEXT: s_cmp_eq_u32 s3, s1608; GFX1100-NEXT: s_cselect_b32 s0, -1, 0609; GFX1100-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0610; GFX1100-NEXT: s_mov_b32 s0, -1611; GFX1100-NEXT: s_mov_b32 s1, 1612; GFX1100-NEXT: v_cmp_ne_u32_e64 s1, v2, s1613; GFX1100-NEXT: s_and_b32 vcc_lo, exec_lo, s1614; GFX1100-NEXT: ; implicit-def: $vgpr3_vgpr4615; GFX1100-NEXT: s_cbranch_vccnz .LBB4_3616; GFX1100-NEXT: .LBB4_1: ; %Flow617; GFX1100-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0618; GFX1100-NEXT: s_mov_b32 s0, 1619; GFX1100-NEXT: v_cmp_ne_u32_e64 s0, v2, s0620; GFX1100-NEXT: s_and_b32 vcc_lo, exec_lo, s0621; GFX1100-NEXT: s_cbranch_vccnz .LBB4_4622; GFX1100-NEXT: ; %bb.2: ; %atomicrmw.private623; GFX1100-NEXT: s_waitcnt lgkmcnt(0)624; GFX1100-NEXT: scratch_load_b64 v[3:4], off, s0625; GFX1100-NEXT: s_waitcnt vmcnt(0)626; GFX1100-NEXT: v_add_co_u32 v0, s0, v3, v0627; GFX1100-NEXT: v_add_co_ci_u32_e64 v1, s0, v4, v1, s0628; GFX1100-NEXT: scratch_store_b64 off, v[0:1], s0629; GFX1100-NEXT: s_branch .LBB4_4630; GFX1100-NEXT: .LBB4_3: ; %atomicrmw.global631; GFX1100-NEXT: s_getpc_b64 s[0:1]632; GFX1100-NEXT: s_add_u32 s0, s0, global@rel32@lo+4633; GFX1100-NEXT: s_addc_u32 s1, s1, global@rel32@hi+12634; GFX1100-NEXT: v_mov_b32_e32 v3, s1635; GFX1100-NEXT: v_mov_b32_e32 v2, s0636; GFX1100-NEXT: flat_atomic_add_u64 v[3:4], v[2:3], v[0:1] glc637; GFX1100-NEXT: s_mov_b32 s0, 0638; GFX1100-NEXT: s_branch .LBB4_1639; GFX1100-NEXT: .LBB4_4: ; %atomicrmw.phi640; GFX1100-NEXT: ; %bb.5: ; %atomicrmw.end641; GFX1100-NEXT: s_mov_b32 s0, 32642; GFX1100-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)643; GFX1100-NEXT: v_lshrrev_b64 v[1:2], s0, v[3:4]644; GFX1100-NEXT: v_mov_b32_e32 v0, v3645; GFX1100-NEXT: s_setpc_b64 s[30:31]646;647; GFX1200-LABEL: optnone_atomicrmw_add_i64_expand:648; GFX1200: ; %bb.0:649; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0650; GFX1200-NEXT: s_wait_expcnt 0x0651; GFX1200-NEXT: s_wait_samplecnt 0x0652; GFX1200-NEXT: s_wait_bvhcnt 0x0653; GFX1200-NEXT: s_wait_kmcnt 0x0654; GFX1200-NEXT: s_mov_b64 s[0:1], src_private_base655; GFX1200-NEXT: s_getpc_b64 s[2:3]656; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)657; GFX1200-NEXT: s_sext_i32_i16 s3, s3658; GFX1200-NEXT: s_add_co_u32 s2, s2, global@rel32@lo+12659; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)660; GFX1200-NEXT: s_add_co_ci_u32 s3, s3, global@rel32@hi+24661; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)662; GFX1200-NEXT: s_cmp_eq_u32 s3, s1663; GFX1200-NEXT: s_cselect_b32 s0, -1, 0664; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)665; GFX1200-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0666; GFX1200-NEXT: s_mov_b32 s0, -1667; GFX1200-NEXT: s_mov_b32 s1, 1668; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)669; GFX1200-NEXT: v_cmp_ne_u32_e64 s1, v2, s1670; GFX1200-NEXT: s_and_b32 vcc_lo, exec_lo, s1671; GFX1200-NEXT: ; implicit-def: $vgpr3_vgpr4672; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)673; GFX1200-NEXT: s_cbranch_vccnz .LBB4_3674; GFX1200-NEXT: .LBB4_1: ; %Flow675; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)676; GFX1200-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0677; GFX1200-NEXT: s_mov_b32 s0, 1678; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)679; GFX1200-NEXT: v_cmp_ne_u32_e64 s0, v2, s0680; GFX1200-NEXT: s_and_b32 vcc_lo, exec_lo, s0681; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)682; GFX1200-NEXT: s_cbranch_vccnz .LBB4_4683; GFX1200-NEXT: ; %bb.2: ; %atomicrmw.private684; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0685; GFX1200-NEXT: scratch_load_b64 v[3:4], off, s0686; GFX1200-NEXT: s_wait_loadcnt 0x0687; GFX1200-NEXT: v_add_co_u32 v0, s0, v3, v0688; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)689; GFX1200-NEXT: v_add_co_ci_u32_e64 v1, s0, v4, v1, s0690; GFX1200-NEXT: scratch_store_b64 off, v[0:1], s0691; GFX1200-NEXT: s_branch .LBB4_4692; GFX1200-NEXT: .LBB4_3: ; %atomicrmw.global693; GFX1200-NEXT: s_getpc_b64 s[0:1]694; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)695; GFX1200-NEXT: s_sext_i32_i16 s1, s1696; GFX1200-NEXT: s_add_co_u32 s0, s0, global@rel32@lo+12697; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)698; GFX1200-NEXT: s_add_co_ci_u32 s1, s1, global@rel32@hi+24699; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)700; GFX1200-NEXT: v_mov_b32_e32 v3, s1701; GFX1200-NEXT: v_mov_b32_e32 v2, s0702; GFX1200-NEXT: flat_atomic_add_u64 v[3:4], v[2:3], v[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV703; GFX1200-NEXT: s_mov_b32 s0, 0704; GFX1200-NEXT: s_branch .LBB4_1705; GFX1200-NEXT: .LBB4_4: ; %atomicrmw.phi706; GFX1200-NEXT: ; %bb.5: ; %atomicrmw.end707; GFX1200-NEXT: s_mov_b32 s0, 32708; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0709; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0) depctr_va_sdst(0)710; GFX1200-NEXT: v_lshrrev_b64 v[1:2], s0, v[3:4]711; GFX1200-NEXT: v_mov_b32_e32 v0, v3712; GFX1200-NEXT: s_setpc_b64 s[30:31]713 %rmw = atomicrmw add ptr addrspacecast (ptr addrspace(1) @global to ptr), i64 %val syncscope("agent") monotonic, align 8714 ret i64 %rmw715}716 717; Make sure there is no error on an invalid addrspacecast without optimizations718define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {719; GFX908-LABEL: optnone_atomicrmw_fadd_f64_expand:720; GFX908: ; %bb.0:721; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)722; GFX908-NEXT: s_mov_b64 s[4:5], src_private_base723; GFX908-NEXT: s_getpc_b64 s[6:7]724; GFX908-NEXT: s_add_u32 s6, s6, global@rel32@lo+4725; GFX908-NEXT: s_addc_u32 s7, s7, global@rel32@hi+12726; GFX908-NEXT: s_cmp_eq_u32 s7, s5727; GFX908-NEXT: s_cselect_b64 s[4:5], -1, 0728; GFX908-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5]729; GFX908-NEXT: s_mov_b64 s[4:5], -1730; GFX908-NEXT: s_mov_b32 s6, 1731; GFX908-NEXT: v_cmp_ne_u32_e64 s[6:7], v2, s6732; GFX908-NEXT: s_and_b64 vcc, exec, s[6:7]733; GFX908-NEXT: ; implicit-def: $vgpr3_vgpr4734; GFX908-NEXT: s_cbranch_vccnz .LBB5_2735; GFX908-NEXT: s_branch .LBB5_3736; GFX908-NEXT: .LBB5_1: ; %atomicrmw.private737; GFX908-NEXT: buffer_load_dword v3, v0, s[0:3], 0 offen738; GFX908-NEXT: s_waitcnt vmcnt(0)739; GFX908-NEXT: v_mov_b32_e32 v4, v3740; GFX908-NEXT: v_add_f64 v[0:1], v[3:4], v[0:1]741; GFX908-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen742; GFX908-NEXT: buffer_store_dword v0, v0, s[0:3], 0 offen743; GFX908-NEXT: s_branch .LBB5_6744; GFX908-NEXT: .LBB5_2: ; %atomicrmw.global745; GFX908-NEXT: s_getpc_b64 s[4:5]746; GFX908-NEXT: s_add_u32 s4, s4, global@rel32@lo+4747; GFX908-NEXT: s_addc_u32 s5, s5, global@rel32@hi+12748; GFX908-NEXT: v_mov_b32_e32 v2, s4749; GFX908-NEXT: v_mov_b32_e32 v3, s5750; GFX908-NEXT: flat_load_dwordx2 v[3:4], v[2:3]751; GFX908-NEXT: s_mov_b64 s[4:5], 0752; GFX908-NEXT: s_branch .LBB5_4753; GFX908-NEXT: .LBB5_3: ; %Flow754; GFX908-NEXT: s_and_b64 vcc, exec, s[4:5]755; GFX908-NEXT: s_cbranch_vccnz .LBB5_1756; GFX908-NEXT: s_branch .LBB5_6757; GFX908-NEXT: .LBB5_4: ; %atomicrmw.start758; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1759; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)760; GFX908-NEXT: v_mov_b32_e32 v6, v4761; GFX908-NEXT: v_mov_b32_e32 v5, v3762; GFX908-NEXT: v_add_f64 v[3:4], v[5:6], v[0:1]763; GFX908-NEXT: s_getpc_b64 s[6:7]764; GFX908-NEXT: s_add_u32 s6, s6, global@rel32@lo+4765; GFX908-NEXT: s_addc_u32 s7, s7, global@rel32@hi+12766; GFX908-NEXT: v_mov_b32_e32 v8, s7767; GFX908-NEXT: v_mov_b32_e32 v7, s6768; GFX908-NEXT: flat_atomic_cmpswap_x2 v[3:4], v[7:8], v[3:6] glc769; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)770; GFX908-NEXT: v_cmp_eq_u64_e64 s[6:7], v[3:4], v[5:6]771; GFX908-NEXT: s_or_b64 s[4:5], s[6:7], s[4:5]772; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]773; GFX908-NEXT: s_cbranch_execnz .LBB5_4774; GFX908-NEXT: ; %bb.5: ; %atomicrmw.end1775; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]776; GFX908-NEXT: s_mov_b64 s[4:5], 0777; GFX908-NEXT: s_branch .LBB5_3778; GFX908-NEXT: .LBB5_6: ; %atomicrmw.phi779; GFX908-NEXT: ; %bb.7: ; %atomicrmw.end780; GFX908-NEXT: s_mov_b32 s4, 32781; GFX908-NEXT: v_lshrrev_b64 v[1:2], s4, v[3:4]782; GFX908-NEXT: v_mov_b32_e32 v0, v3783; GFX908-NEXT: s_waitcnt vmcnt(0)784; GFX908-NEXT: s_setpc_b64 s[30:31]785;786; GFX90A-LABEL: optnone_atomicrmw_fadd_f64_expand:787; GFX90A: ; %bb.0:788; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)789; GFX90A-NEXT: s_mov_b64 s[4:5], src_shared_base790; GFX90A-NEXT: s_getpc_b64 s[6:7]791; GFX90A-NEXT: s_add_u32 s6, s6, global@rel32@lo+4792; GFX90A-NEXT: s_addc_u32 s7, s7, global@rel32@hi+12793; GFX90A-NEXT: s_cmp_eq_u32 s7, s5794; GFX90A-NEXT: s_cselect_b64 s[4:5], -1, 0795; GFX90A-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5]796; GFX90A-NEXT: s_mov_b64 s[4:5], -1797; GFX90A-NEXT: s_mov_b32 s6, 1798; GFX90A-NEXT: v_cmp_ne_u32_e64 s[6:7], v2, s6799; GFX90A-NEXT: s_and_b64 vcc, exec, s[6:7]800; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3801; GFX90A-NEXT: s_cbranch_vccnz .LBB5_3802; GFX90A-NEXT: .LBB5_1: ; %Flow4803; GFX90A-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[4:5]804; GFX90A-NEXT: s_mov_b32 s4, 1805; GFX90A-NEXT: v_cmp_ne_u32_e64 s[4:5], v4, s4806; GFX90A-NEXT: s_and_b64 vcc, exec, s[4:5]807; GFX90A-NEXT: s_cbranch_vccnz .LBB5_10808; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.shared809; GFX90A-NEXT: ds_add_rtn_f64 v[2:3], v0, v[0:1]810; GFX90A-NEXT: s_branch .LBB5_10811; GFX90A-NEXT: .LBB5_3: ; %atomicrmw.check.private812; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base813; GFX90A-NEXT: s_getpc_b64 s[6:7]814; GFX90A-NEXT: s_add_u32 s6, s6, global@rel32@lo+4815; GFX90A-NEXT: s_addc_u32 s7, s7, global@rel32@hi+12816; GFX90A-NEXT: s_cmp_eq_u32 s7, s5817; GFX90A-NEXT: s_cselect_b64 s[4:5], -1, 0818; GFX90A-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5]819; GFX90A-NEXT: s_mov_b64 s[4:5], -1820; GFX90A-NEXT: s_mov_b32 s6, 1821; GFX90A-NEXT: v_cmp_ne_u32_e64 s[6:7], v2, s6822; GFX90A-NEXT: s_and_b64 vcc, exec, s[6:7]823; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3824; GFX90A-NEXT: s_cbranch_vccnz .LBB5_5825; GFX90A-NEXT: s_branch .LBB5_6826; GFX90A-NEXT: .LBB5_4: ; %atomicrmw.private827; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen828; GFX90A-NEXT: s_waitcnt vmcnt(0)829; GFX90A-NEXT: v_mov_b32_e32 v3, v2830; GFX90A-NEXT: v_add_f64 v[4:5], v[2:3], v[0:1]831; GFX90A-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen832; GFX90A-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen833; GFX90A-NEXT: s_branch .LBB5_9834; GFX90A-NEXT: .LBB5_5: ; %atomicrmw.global835; GFX90A-NEXT: v_mov_b32_e32 v2, 0836; GFX90A-NEXT: s_getpc_b64 s[4:5]837; GFX90A-NEXT: s_add_u32 s4, s4, global@rel32@lo+4838; GFX90A-NEXT: s_addc_u32 s5, s5, global@rel32@hi+12839; GFX90A-NEXT: global_load_dwordx2 v[4:5], v2, s[4:5]840; GFX90A-NEXT: s_mov_b64 s[4:5], 0841; GFX90A-NEXT: s_branch .LBB5_7842; GFX90A-NEXT: .LBB5_6: ; %Flow843; GFX90A-NEXT: s_and_b64 vcc, exec, s[4:5]844; GFX90A-NEXT: s_cbranch_vccnz .LBB5_4845; GFX90A-NEXT: s_branch .LBB5_9846; GFX90A-NEXT: .LBB5_7: ; %atomicrmw.start847; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1848; GFX90A-NEXT: s_waitcnt vmcnt(0)849; GFX90A-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1]850; GFX90A-NEXT: s_getpc_b64 s[6:7]851; GFX90A-NEXT: s_add_u32 s6, s6, global@rel32@lo+4852; GFX90A-NEXT: s_addc_u32 s7, s7, global@rel32@hi+12853; GFX90A-NEXT: v_mov_b32_e32 v6, 0854; GFX90A-NEXT: global_atomic_cmpswap_x2 v[2:3], v6, v[2:5], s[6:7] glc855; GFX90A-NEXT: s_waitcnt vmcnt(0)856; GFX90A-NEXT: v_cmp_eq_u64_e64 s[6:7], v[2:3], v[4:5]857; GFX90A-NEXT: s_or_b64 s[4:5], s[6:7], s[4:5]858; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]859; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]860; GFX90A-NEXT: s_cbranch_execnz .LBB5_7861; GFX90A-NEXT: ; %bb.8: ; %atomicrmw.end1862; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]863; GFX90A-NEXT: s_mov_b64 s[4:5], 0864; GFX90A-NEXT: s_branch .LBB5_6865; GFX90A-NEXT: .LBB5_9: ; %Flow3866; GFX90A-NEXT: s_mov_b64 s[4:5], 0867; GFX90A-NEXT: s_branch .LBB5_1868; GFX90A-NEXT: .LBB5_10: ; %atomicrmw.phi869; GFX90A-NEXT: ; %bb.11: ; %atomicrmw.end870; GFX90A-NEXT: s_mov_b32 s4, 32871; GFX90A-NEXT: s_waitcnt lgkmcnt(0)872; GFX90A-NEXT: v_lshrrev_b64 v[4:5], s4, v[2:3]873; GFX90A-NEXT: v_mov_b32_e32 v0, v2874; GFX90A-NEXT: v_mov_b32_e32 v1, v4875; GFX90A-NEXT: s_waitcnt vmcnt(0)876; GFX90A-NEXT: s_setpc_b64 s[30:31]877;878; GFX942-LABEL: optnone_atomicrmw_fadd_f64_expand:879; GFX942: ; %bb.0:880; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)881; GFX942-NEXT: s_mov_b64 s[0:1], src_shared_base882; GFX942-NEXT: s_getpc_b64 s[2:3]883; GFX942-NEXT: s_add_u32 s2, s2, global@rel32@lo+4884; GFX942-NEXT: s_addc_u32 s3, s3, global@rel32@hi+12885; GFX942-NEXT: s_cmp_eq_u32 s3, s1886; GFX942-NEXT: s_cselect_b64 s[0:1], -1, 0887; GFX942-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]888; GFX942-NEXT: s_mov_b64 s[0:1], -1889; GFX942-NEXT: s_mov_b32 s2, 1890; GFX942-NEXT: v_cmp_ne_u32_e64 s[2:3], v2, s2891; GFX942-NEXT: s_and_b64 vcc, exec, s[2:3]892; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3893; GFX942-NEXT: s_cbranch_vccnz .LBB5_3894; GFX942-NEXT: .LBB5_1: ; %Flow4895; GFX942-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[0:1]896; GFX942-NEXT: s_mov_b32 s0, 1897; GFX942-NEXT: v_cmp_ne_u32_e64 s[0:1], v4, s0898; GFX942-NEXT: s_and_b64 vcc, exec, s[0:1]899; GFX942-NEXT: s_cbranch_vccnz .LBB5_10900; GFX942-NEXT: ; %bb.2: ; %atomicrmw.shared901; GFX942-NEXT: ds_add_rtn_f64 v[2:3], v0, v[0:1]902; GFX942-NEXT: s_branch .LBB5_10903; GFX942-NEXT: .LBB5_3: ; %atomicrmw.check.private904; GFX942-NEXT: s_mov_b64 s[0:1], src_private_base905; GFX942-NEXT: s_getpc_b64 s[2:3]906; GFX942-NEXT: s_add_u32 s2, s2, global@rel32@lo+4907; GFX942-NEXT: s_addc_u32 s3, s3, global@rel32@hi+12908; GFX942-NEXT: s_cmp_eq_u32 s3, s1909; GFX942-NEXT: s_cselect_b64 s[0:1], -1, 0910; GFX942-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]911; GFX942-NEXT: s_mov_b64 s[0:1], -1912; GFX942-NEXT: s_mov_b32 s2, 1913; GFX942-NEXT: v_cmp_ne_u32_e64 s[2:3], v2, s2914; GFX942-NEXT: s_and_b64 vcc, exec, s[2:3]915; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3916; GFX942-NEXT: s_cbranch_vccnz .LBB5_5917; GFX942-NEXT: s_branch .LBB5_6918; GFX942-NEXT: .LBB5_4: ; %atomicrmw.private919; GFX942-NEXT: scratch_load_dwordx2 v[2:3], off, s0920; GFX942-NEXT: s_waitcnt vmcnt(0)921; GFX942-NEXT: v_add_f64 v[4:5], v[2:3], v[0:1]922; GFX942-NEXT: scratch_store_dwordx2 off, v[4:5], s0923; GFX942-NEXT: s_branch .LBB5_9924; GFX942-NEXT: .LBB5_5: ; %atomicrmw.global925; GFX942-NEXT: v_mov_b32_e32 v2, 0926; GFX942-NEXT: s_getpc_b64 s[0:1]927; GFX942-NEXT: s_add_u32 s0, s0, global@rel32@lo+4928; GFX942-NEXT: s_addc_u32 s1, s1, global@rel32@hi+12929; GFX942-NEXT: global_load_dwordx2 v[4:5], v2, s[0:1]930; GFX942-NEXT: s_mov_b64 s[0:1], 0931; GFX942-NEXT: s_branch .LBB5_7932; GFX942-NEXT: .LBB5_6: ; %Flow933; GFX942-NEXT: s_and_b64 vcc, exec, s[0:1]934; GFX942-NEXT: s_cbranch_vccnz .LBB5_4935; GFX942-NEXT: s_branch .LBB5_9936; GFX942-NEXT: .LBB5_7: ; %atomicrmw.start937; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1938; GFX942-NEXT: s_waitcnt vmcnt(0)939; GFX942-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1]940; GFX942-NEXT: s_getpc_b64 s[2:3]941; GFX942-NEXT: s_add_u32 s2, s2, global@rel32@lo+4942; GFX942-NEXT: s_addc_u32 s3, s3, global@rel32@hi+12943; GFX942-NEXT: v_mov_b32_e32 v6, 0944; GFX942-NEXT: global_atomic_cmpswap_x2 v[2:3], v6, v[2:5], s[2:3] sc0 sc1945; GFX942-NEXT: s_waitcnt vmcnt(0)946; GFX942-NEXT: v_cmp_eq_u64_e64 s[2:3], v[2:3], v[4:5]947; GFX942-NEXT: s_or_b64 s[0:1], s[2:3], s[0:1]948; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]949; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]950; GFX942-NEXT: s_cbranch_execnz .LBB5_7951; GFX942-NEXT: ; %bb.8: ; %atomicrmw.end1952; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]953; GFX942-NEXT: s_mov_b64 s[0:1], 0954; GFX942-NEXT: s_branch .LBB5_6955; GFX942-NEXT: .LBB5_9: ; %Flow3956; GFX942-NEXT: s_mov_b64 s[0:1], 0957; GFX942-NEXT: s_branch .LBB5_1958; GFX942-NEXT: .LBB5_10: ; %atomicrmw.phi959; GFX942-NEXT: ; %bb.11: ; %atomicrmw.end960; GFX942-NEXT: s_mov_b32 s0, 32961; GFX942-NEXT: s_waitcnt lgkmcnt(0)962; GFX942-NEXT: v_lshrrev_b64 v[4:5], s0, v[2:3]963; GFX942-NEXT: v_mov_b32_e32 v0, v2964; GFX942-NEXT: v_mov_b32_e32 v1, v4965; GFX942-NEXT: s_waitcnt vmcnt(0)966; GFX942-NEXT: s_setpc_b64 s[30:31]967;968; GFX1100-LABEL: optnone_atomicrmw_fadd_f64_expand:969; GFX1100: ; %bb.0:970; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)971; GFX1100-NEXT: s_mov_b64 s[0:1], src_private_base972; GFX1100-NEXT: s_getpc_b64 s[2:3]973; GFX1100-NEXT: s_add_u32 s2, s2, global@rel32@lo+4974; GFX1100-NEXT: s_addc_u32 s3, s3, global@rel32@hi+12975; GFX1100-NEXT: s_cmp_eq_u32 s3, s1976; GFX1100-NEXT: s_cselect_b32 s0, -1, 0977; GFX1100-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0978; GFX1100-NEXT: s_mov_b32 s0, -1979; GFX1100-NEXT: s_mov_b32 s1, 1980; GFX1100-NEXT: v_cmp_ne_u32_e64 s1, v2, s1981; GFX1100-NEXT: s_and_b32 vcc_lo, exec_lo, s1982; GFX1100-NEXT: ; implicit-def: $vgpr3_vgpr4983; GFX1100-NEXT: s_cbranch_vccnz .LBB5_2984; GFX1100-NEXT: s_branch .LBB5_3985; GFX1100-NEXT: .LBB5_1: ; %atomicrmw.private986; GFX1100-NEXT: scratch_load_b64 v[3:4], off, s0987; GFX1100-NEXT: s_waitcnt vmcnt(0)988; GFX1100-NEXT: v_add_f64 v[0:1], v[3:4], v[0:1]989; GFX1100-NEXT: scratch_store_b64 off, v[0:1], s0990; GFX1100-NEXT: s_branch .LBB5_6991; GFX1100-NEXT: .LBB5_2: ; %atomicrmw.global992; GFX1100-NEXT: s_getpc_b64 s[0:1]993; GFX1100-NEXT: s_add_u32 s0, s0, global@rel32@lo+4994; GFX1100-NEXT: s_addc_u32 s1, s1, global@rel32@hi+12995; GFX1100-NEXT: v_mov_b32_e32 v3, s1996; GFX1100-NEXT: v_mov_b32_e32 v2, s0997; GFX1100-NEXT: flat_load_b64 v[3:4], v[2:3]998; GFX1100-NEXT: s_mov_b32 s0, 0999; GFX1100-NEXT: s_branch .LBB5_41000; GFX1100-NEXT: .LBB5_3: ; %Flow1001; GFX1100-NEXT: s_and_b32 vcc_lo, exec_lo, s01002; GFX1100-NEXT: s_cbranch_vccnz .LBB5_11003; GFX1100-NEXT: s_branch .LBB5_61004; GFX1100-NEXT: .LBB5_4: ; %atomicrmw.start1005; GFX1100-NEXT: ; =>This Inner Loop Header: Depth=11006; GFX1100-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1007; GFX1100-NEXT: v_mov_b32_e32 v6, v41008; GFX1100-NEXT: v_mov_b32_e32 v5, v31009; GFX1100-NEXT: v_add_f64 v[3:4], v[5:6], v[0:1]1010; GFX1100-NEXT: s_getpc_b64 s[2:3]1011; GFX1100-NEXT: s_add_u32 s2, s2, global@rel32@lo+41012; GFX1100-NEXT: s_addc_u32 s3, s3, global@rel32@hi+121013; GFX1100-NEXT: v_mov_b32_e32 v8, s31014; GFX1100-NEXT: v_mov_b32_e32 v7, s21015; GFX1100-NEXT: flat_atomic_cmpswap_b64 v[3:4], v[7:8], v[3:6] glc1016; GFX1100-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)1017; GFX1100-NEXT: v_cmp_eq_u64_e64 s1, v[3:4], v[5:6]1018; GFX1100-NEXT: s_or_b32 s0, s1, s01019; GFX1100-NEXT: s_and_not1_b32 exec_lo, exec_lo, s01020; GFX1100-NEXT: s_cbranch_execnz .LBB5_41021; GFX1100-NEXT: ; %bb.5: ; %atomicrmw.end11022; GFX1100-NEXT: s_or_b32 exec_lo, exec_lo, s01023; GFX1100-NEXT: s_mov_b32 s0, 01024; GFX1100-NEXT: s_branch .LBB5_31025; GFX1100-NEXT: .LBB5_6: ; %atomicrmw.phi1026; GFX1100-NEXT: ; %bb.7: ; %atomicrmw.end1027; GFX1100-NEXT: s_mov_b32 s0, 321028; GFX1100-NEXT: v_lshrrev_b64 v[1:2], s0, v[3:4]1029; GFX1100-NEXT: v_mov_b32_e32 v0, v31030; GFX1100-NEXT: s_setpc_b64 s[30:31]1031;1032; GFX1200-LABEL: optnone_atomicrmw_fadd_f64_expand:1033; GFX1200: ; %bb.0:1034; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x01035; GFX1200-NEXT: s_wait_expcnt 0x01036; GFX1200-NEXT: s_wait_samplecnt 0x01037; GFX1200-NEXT: s_wait_bvhcnt 0x01038; GFX1200-NEXT: s_wait_kmcnt 0x01039; GFX1200-NEXT: s_mov_b64 s[0:1], src_private_base1040; GFX1200-NEXT: s_getpc_b64 s[2:3]1041; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)1042; GFX1200-NEXT: s_sext_i32_i16 s3, s31043; GFX1200-NEXT: s_add_co_u32 s2, s2, global@rel32@lo+121044; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)1045; GFX1200-NEXT: s_add_co_ci_u32 s3, s3, global@rel32@hi+241046; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)1047; GFX1200-NEXT: s_cmp_eq_u32 s3, s11048; GFX1200-NEXT: s_cselect_b32 s0, -1, 01049; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)1050; GFX1200-NEXT: v_cndmask_b32_e64 v2, 0, 1, s01051; GFX1200-NEXT: s_mov_b32 s0, -11052; GFX1200-NEXT: s_mov_b32 s1, 11053; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)1054; GFX1200-NEXT: v_cmp_ne_u32_e64 s1, v2, s11055; GFX1200-NEXT: s_and_b32 vcc_lo, exec_lo, s11056; GFX1200-NEXT: ; implicit-def: $vgpr3_vgpr41057; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)1058; GFX1200-NEXT: s_cbranch_vccnz .LBB5_21059; GFX1200-NEXT: s_branch .LBB5_31060; GFX1200-NEXT: .LBB5_1: ; %atomicrmw.private1061; GFX1200-NEXT: scratch_load_b64 v[3:4], off, s01062; GFX1200-NEXT: s_wait_loadcnt 0x01063; GFX1200-NEXT: v_add_f64_e64 v[0:1], v[3:4], v[0:1]1064; GFX1200-NEXT: scratch_store_b64 off, v[0:1], s01065; GFX1200-NEXT: s_branch .LBB5_61066; GFX1200-NEXT: .LBB5_2: ; %atomicrmw.global1067; GFX1200-NEXT: s_getpc_b64 s[0:1]1068; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)1069; GFX1200-NEXT: s_sext_i32_i16 s1, s11070; GFX1200-NEXT: s_add_co_u32 s0, s0, global@rel32@lo+121071; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)1072; GFX1200-NEXT: s_add_co_ci_u32 s1, s1, global@rel32@hi+241073; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)1074; GFX1200-NEXT: v_mov_b32_e32 v3, s11075; GFX1200-NEXT: v_mov_b32_e32 v2, s01076; GFX1200-NEXT: flat_load_b64 v[3:4], v[2:3]1077; GFX1200-NEXT: s_mov_b32 s0, 01078; GFX1200-NEXT: s_branch .LBB5_41079; GFX1200-NEXT: .LBB5_3: ; %Flow1080; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)1081; GFX1200-NEXT: s_and_b32 vcc_lo, exec_lo, s01082; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)1083; GFX1200-NEXT: s_cbranch_vccnz .LBB5_11084; GFX1200-NEXT: s_branch .LBB5_61085; GFX1200-NEXT: .LBB5_4: ; %atomicrmw.start1086; GFX1200-NEXT: ; =>This Inner Loop Header: Depth=11087; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x01088; GFX1200-NEXT: v_mov_b32_e32 v6, v41089; GFX1200-NEXT: v_mov_b32_e32 v5, v31090; GFX1200-NEXT: v_add_f64_e64 v[3:4], v[5:6], v[0:1]1091; GFX1200-NEXT: s_getpc_b64 s[2:3]1092; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)1093; GFX1200-NEXT: s_sext_i32_i16 s3, s31094; GFX1200-NEXT: s_add_co_u32 s2, s2, global@rel32@lo+121095; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)1096; GFX1200-NEXT: s_add_co_ci_u32 s3, s3, global@rel32@hi+241097; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)1098; GFX1200-NEXT: v_mov_b32_e32 v8, s31099; GFX1200-NEXT: v_mov_b32_e32 v7, s21100; GFX1200-NEXT: flat_atomic_cmpswap_b64 v[3:4], v[7:8], v[3:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS1101; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x01102; GFX1200-NEXT: v_cmp_eq_u64_e64 s1, v[3:4], v[5:6]1103; GFX1200-NEXT: s_or_b32 s0, s1, s01104; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)1105; GFX1200-NEXT: s_and_not1_b32 exec_lo, exec_lo, s01106; GFX1200-NEXT: s_cbranch_execnz .LBB5_41107; GFX1200-NEXT: ; %bb.5: ; %atomicrmw.end11108; GFX1200-NEXT: s_or_b32 exec_lo, exec_lo, s01109; GFX1200-NEXT: s_mov_b32 s0, 01110; GFX1200-NEXT: s_branch .LBB5_31111; GFX1200-NEXT: .LBB5_6: ; %atomicrmw.phi1112; GFX1200-NEXT: ; %bb.7: ; %atomicrmw.end1113; GFX1200-NEXT: s_mov_b32 s0, 321114; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)1115; GFX1200-NEXT: v_lshrrev_b64 v[1:2], s0, v[3:4]1116; GFX1200-NEXT: v_mov_b32_e32 v0, v31117; GFX1200-NEXT: s_setpc_b64 s[30:31]1118 %rmw = atomicrmw fadd ptr addrspacecast (ptr addrspace(1) @global to ptr), double %val monotonic, align 81119 ret double %rmw1120}1121 1122attributes #0 = { nounwind }1123attributes #1 = { noinline nounwind optnone }1124 1125!0 = !{}1126