165 lines · cpp
1//===-- AMDGPUMarkLastScratchLoad.cpp -------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Mark scratch load/spill instructions which are guaranteed to be the last time10// this scratch slot is used so it can be evicted from caches.11//12// TODO: Handle general stack accesses not just spilling.13//14//===----------------------------------------------------------------------===//15 16#include "AMDGPU.h"17#include "GCNSubtarget.h"18#include "llvm/CodeGen/LiveIntervals.h"19#include "llvm/CodeGen/LiveStacks.h"20#include "llvm/CodeGen/MachineOperand.h"21 22using namespace llvm;23 24#define DEBUG_TYPE "amdgpu-mark-last-scratch-load"25 26namespace {27 28class AMDGPUMarkLastScratchLoad {29private:30 LiveStacks *LS = nullptr;31 LiveIntervals *LIS = nullptr;32 SlotIndexes *SI = nullptr;33 const SIInstrInfo *SII = nullptr;34 35public:36 AMDGPUMarkLastScratchLoad(LiveStacks *LS, LiveIntervals *LIS, SlotIndexes *SI)37 : LS(LS), LIS(LIS), SI(SI) {}38 bool run(MachineFunction &MF);39};40 41class AMDGPUMarkLastScratchLoadLegacy : public MachineFunctionPass {42public:43 static char ID;44 45 AMDGPUMarkLastScratchLoadLegacy() : MachineFunctionPass(ID) {}46 47 bool runOnMachineFunction(MachineFunction &MF) override;48 49 void getAnalysisUsage(AnalysisUsage &AU) const override {50 AU.addRequired<SlotIndexesWrapperPass>();51 AU.addRequired<LiveIntervalsWrapperPass>();52 AU.addRequired<LiveStacksWrapperLegacy>();53 AU.setPreservesAll();54 MachineFunctionPass::getAnalysisUsage(AU);55 }56 57 StringRef getPassName() const override {58 return "AMDGPU Mark Last Scratch Load";59 }60};61 62} // end anonymous namespace63 64bool AMDGPUMarkLastScratchLoadLegacy::runOnMachineFunction(65 MachineFunction &MF) {66 if (skipFunction(MF.getFunction()))67 return false;68 69 auto &LS = getAnalysis<LiveStacksWrapperLegacy>().getLS();70 auto &LIS = getAnalysis<LiveIntervalsWrapperPass>().getLIS();71 auto &SI = getAnalysis<SlotIndexesWrapperPass>().getSI();72 73 return AMDGPUMarkLastScratchLoad(&LS, &LIS, &SI).run(MF);74}75 76PreservedAnalyses77AMDGPUMarkLastScratchLoadPass::run(MachineFunction &MF,78 MachineFunctionAnalysisManager &MFAM) {79 auto &LS = MFAM.getResult<LiveStacksAnalysis>(MF);80 auto &LIS = MFAM.getResult<LiveIntervalsAnalysis>(MF);81 auto &SI = MFAM.getResult<SlotIndexesAnalysis>(MF);82 83 AMDGPUMarkLastScratchLoad(&LS, &LIS, &SI).run(MF);84 return PreservedAnalyses::all();85}86 87bool AMDGPUMarkLastScratchLoad::run(MachineFunction &MF) {88 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();89 if (ST.getGeneration() < AMDGPUSubtarget::GFX12)90 return false;91 92 SII = ST.getInstrInfo();93 SlotIndexes &Slots = *LIS->getSlotIndexes();94 95 const unsigned NumSlots = LS->getNumIntervals();96 if (NumSlots == 0) {97 LLVM_DEBUG(dbgs() << "No live slots, skipping\n");98 return false;99 }100 101 LLVM_DEBUG(dbgs() << LS->getNumIntervals() << " intervals\n");102 103 bool Changed = false;104 105 for (auto &[SS, LI] : *LS) {106 for (const LiveRange::Segment &Segment : LI.segments) {107 108 // Ignore segments that run to the end of basic block because in this case109 // slot is still live at the end of it.110 if (Segment.end.isBlock())111 continue;112 113 const int FrameIndex = LI.reg().stackSlotIndex();114 MachineInstr *LastLoad = nullptr;115 116 MachineInstr *MISegmentEnd = SI->getInstructionFromIndex(Segment.end);117 118 // If there is no instruction at this slot because it was deleted take the119 // instruction from the next slot.120 if (!MISegmentEnd) {121 SlotIndex NextSlot = Slots.getNextNonNullIndex(Segment.end);122 MISegmentEnd = SI->getInstructionFromIndex(NextSlot);123 }124 125 MachineInstr *MISegmentStart = SI->getInstructionFromIndex(Segment.start);126 MachineBasicBlock *BB = MISegmentEnd->getParent();127 128 // Start iteration backwards from segment end until the start of basic129 // block or start of segment if it is in the same basic block.130 auto End = BB->rend();131 if (MISegmentStart && MISegmentStart->getParent() == BB)132 End = MISegmentStart->getReverseIterator();133 134 for (auto MI = MISegmentEnd->getReverseIterator(); MI != End; ++MI) {135 int LoadFI = 0;136 137 if (SII->isLoadFromStackSlot(*MI, LoadFI) && LoadFI == FrameIndex) {138 LastLoad = &*MI;139 break;140 }141 }142 143 if (LastLoad && !LastLoad->memoperands_empty()) {144 MachineMemOperand *MMO = *LastLoad->memoperands_begin();145 MMO->setFlags(MOLastUse);146 Changed = true;147 LLVM_DEBUG(dbgs() << " Found last load: " << *LastLoad);148 }149 }150 }151 152 return Changed;153}154 155char AMDGPUMarkLastScratchLoadLegacy::ID = 0;156 157char &llvm::AMDGPUMarkLastScratchLoadID = AMDGPUMarkLastScratchLoadLegacy::ID;158 159INITIALIZE_PASS_BEGIN(AMDGPUMarkLastScratchLoadLegacy, DEBUG_TYPE,160 "AMDGPU Mark last scratch load", false, false)161INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)162INITIALIZE_PASS_DEPENDENCY(LiveStacksWrapperLegacy)163INITIALIZE_PASS_END(AMDGPUMarkLastScratchLoadLegacy, DEBUG_TYPE,164 "AMDGPU Mark last scratch load", false, false)165