86 lines · cpp
1//===--- AMDGPUBarrierLatency.cpp - AMDGPU Barrier Latency ----------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9/// \file This file contains a DAG scheduling mutation to add latency to10/// barrier edges between ATOMIC_FENCE instructions and preceding11/// memory accesses potentially affected by the fence.12/// This encourages the scheduling of more instructions before13/// ATOMIC_FENCE instructions. ATOMIC_FENCE instructions may14/// introduce wait counting or indicate an impending S_BARRIER15/// wait. Having more instructions in-flight across these16/// constructs improves latency hiding.17//18//===----------------------------------------------------------------------===//19 20#include "AMDGPUBarrierLatency.h"21#include "MCTargetDesc/AMDGPUMCTargetDesc.h"22#include "SIInstrInfo.h"23#include "llvm/CodeGen/ScheduleDAGInstrs.h"24 25using namespace llvm;26 27namespace {28 29class BarrierLatency : public ScheduleDAGMutation {30private:31 SmallSet<SyncScope::ID, 4> IgnoredScopes;32 33public:34 BarrierLatency(MachineFunction *MF) {35 LLVMContext &Context = MF->getFunction().getContext();36 IgnoredScopes.insert(SyncScope::SingleThread);37 IgnoredScopes.insert(Context.getOrInsertSyncScopeID("wavefront"));38 IgnoredScopes.insert(Context.getOrInsertSyncScopeID("wavefront-one-as"));39 IgnoredScopes.insert(Context.getOrInsertSyncScopeID("singlethread-one-as"));40 }41 void apply(ScheduleDAGInstrs *DAG) override;42};43 44void BarrierLatency::apply(ScheduleDAGInstrs *DAG) {45 constexpr unsigned SyntheticLatency = 2000;46 for (SUnit &SU : DAG->SUnits) {47 const MachineInstr *MI = SU.getInstr();48 if (MI->getOpcode() != AMDGPU::ATOMIC_FENCE)49 continue;50 51 // Update latency on barrier edges of ATOMIC_FENCE.52 // Ignore scopes not expected to have any latency.53 SyncScope::ID SSID = static_cast<SyncScope::ID>(MI->getOperand(1).getImm());54 if (IgnoredScopes.contains(SSID))55 continue;56 57 for (SDep &PredDep : SU.Preds) {58 if (!PredDep.isBarrier())59 continue;60 SUnit *PredSU = PredDep.getSUnit();61 MachineInstr *MI = PredSU->getInstr();62 // Only consider memory loads63 if (!MI->mayLoad() || MI->mayStore())64 continue;65 SDep ForwardD = PredDep;66 ForwardD.setSUnit(&SU);67 for (SDep &SuccDep : PredSU->Succs) {68 if (SuccDep == ForwardD) {69 SuccDep.setLatency(SuccDep.getLatency() + SyntheticLatency);70 break;71 }72 }73 PredDep.setLatency(PredDep.getLatency() + SyntheticLatency);74 PredSU->setDepthDirty();75 SU.setDepthDirty();76 }77 }78}79 80} // end namespace81 82std::unique_ptr<ScheduleDAGMutation>83llvm::createAMDGPUBarrierLatencyDAGMutation(MachineFunction *MF) {84 return std::make_unique<BarrierLatency>(MF);85}86