185 lines · cpp
1//===--- AArch64StorePairSuppress.cpp --- Suppress store pair formation ---===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This pass identifies floating point stores that should not be combined into10// store pairs. Later we may do the same for floating point loads.11// ===---------------------------------------------------------------------===//12 13#include "AArch64InstrInfo.h"14#include "AArch64Subtarget.h"15#include "llvm/CodeGen/MachineFunction.h"16#include "llvm/CodeGen/MachineFunctionPass.h"17#include "llvm/CodeGen/MachineInstr.h"18#include "llvm/CodeGen/MachineTraceMetrics.h"19#include "llvm/CodeGen/TargetInstrInfo.h"20#include "llvm/CodeGen/TargetSchedule.h"21#include "llvm/Support/Debug.h"22#include "llvm/Support/raw_ostream.h"23 24using namespace llvm;25 26#define DEBUG_TYPE "aarch64-stp-suppress"27 28#define STPSUPPRESS_PASS_NAME "AArch64 Store Pair Suppression"29 30namespace {31class AArch64StorePairSuppress : public MachineFunctionPass {32 const AArch64InstrInfo *TII;33 const TargetRegisterInfo *TRI;34 const MachineRegisterInfo *MRI;35 TargetSchedModel SchedModel;36 MachineTraceMetrics *Traces;37 MachineTraceMetrics::Ensemble *MinInstr;38 39public:40 static char ID;41 AArch64StorePairSuppress() : MachineFunctionPass(ID) {}42 43 StringRef getPassName() const override { return STPSUPPRESS_PASS_NAME; }44 45 bool runOnMachineFunction(MachineFunction &F) override;46 47private:48 bool shouldAddSTPToBlock(const MachineBasicBlock *BB);49 50 bool isNarrowFPStore(const MachineInstr &MI);51 52 void getAnalysisUsage(AnalysisUsage &AU) const override {53 AU.setPreservesCFG();54 AU.addRequired<MachineTraceMetricsWrapperPass>();55 AU.addPreserved<MachineTraceMetricsWrapperPass>();56 MachineFunctionPass::getAnalysisUsage(AU);57 }58};59char AArch64StorePairSuppress::ID = 0;60} // anonymous61 62INITIALIZE_PASS(AArch64StorePairSuppress, "aarch64-stp-suppress",63 STPSUPPRESS_PASS_NAME, false, false)64 65FunctionPass *llvm::createAArch64StorePairSuppressPass() {66 return new AArch64StorePairSuppress();67}68 69/// Return true if an STP can be added to this block without increasing the70/// critical resource height. STP is good to form in Ld/St limited blocks and71/// bad to form in float-point limited blocks. This is true independent of the72/// critical path. If the critical path is longer than the resource height, the73/// extra vector ops can limit physreg renaming. Otherwise, it could simply74/// oversaturate the vector units.75bool AArch64StorePairSuppress::shouldAddSTPToBlock(const MachineBasicBlock *BB) {76 if (!MinInstr)77 MinInstr = Traces->getEnsemble(MachineTraceStrategy::TS_MinInstrCount);78 79 MachineTraceMetrics::Trace BBTrace = MinInstr->getTrace(BB);80 unsigned ResLength = BBTrace.getResourceLength();81 82 // Get the machine model's scheduling class for STPDi and STRDui.83 // Bypass TargetSchedule's SchedClass resolution since we only have an opcode.84 unsigned SCIdx = TII->get(AArch64::STPDi).getSchedClass();85 const MCSchedClassDesc *PairSCDesc =86 SchedModel.getMCSchedModel()->getSchedClassDesc(SCIdx);87 88 unsigned SCIdx2 = TII->get(AArch64::STRDui).getSchedClass();89 const MCSchedClassDesc *SingleSCDesc =90 SchedModel.getMCSchedModel()->getSchedClassDesc(SCIdx2);91 92 // If a subtarget does not define resources for STPDi, bail here.93 if (PairSCDesc->isValid() && !PairSCDesc->isVariant() &&94 SingleSCDesc->isValid() && !SingleSCDesc->isVariant()) {95 // Compute the new critical resource length after replacing 2 separate96 // STRDui with one STPDi.97 unsigned ResLenWithSTP =98 BBTrace.getResourceLength({}, PairSCDesc, {SingleSCDesc, SingleSCDesc});99 if (ResLenWithSTP > ResLength) {100 LLVM_DEBUG(dbgs() << " Suppress STP in BB: " << BB->getNumber()101 << " resources " << ResLength << " -> " << ResLenWithSTP102 << "\n");103 return false;104 }105 }106 return true;107}108 109/// Return true if this is a floating-point store smaller than the V reg. On110/// cyclone, these require a vector shuffle before storing a pair.111/// Ideally we would call getMatchingPairOpcode() and have the machine model112/// tell us if it's profitable with no cpu knowledge here.113///114/// FIXME: We plan to develop a decent Target abstraction for simple loads and115/// stores. Until then use a nasty switch similar to AArch64LoadStoreOptimizer.116bool AArch64StorePairSuppress::isNarrowFPStore(const MachineInstr &MI) {117 switch (MI.getOpcode()) {118 default:119 return false;120 case AArch64::STRSui:121 case AArch64::STRDui:122 case AArch64::STURSi:123 case AArch64::STURDi:124 return true;125 }126}127 128bool AArch64StorePairSuppress::runOnMachineFunction(MachineFunction &MF) {129 if (skipFunction(MF.getFunction()) || MF.getFunction().hasOptSize())130 return false;131 132 const AArch64Subtarget &ST = MF.getSubtarget<AArch64Subtarget>();133 if (!ST.enableStorePairSuppress())134 return false;135 136 TII = ST.getInstrInfo();137 TRI = ST.getRegisterInfo();138 MRI = &MF.getRegInfo();139 SchedModel.init(&ST);140 Traces = &getAnalysis<MachineTraceMetricsWrapperPass>().getMTM();141 MinInstr = nullptr;142 143 LLVM_DEBUG(dbgs() << "*** " << getPassName() << ": " << MF.getName() << '\n');144 145 if (!SchedModel.hasInstrSchedModel()) {146 LLVM_DEBUG(dbgs() << " Skipping pass: no machine model present.\n");147 return false;148 }149 150 // Check for a sequence of stores to the same base address. We don't need to151 // precisely determine whether a store pair can be formed. But we do want to152 // filter out most situations where we can't form store pairs to avoid153 // computing trace metrics in those cases.154 for (auto &MBB : MF) {155 bool SuppressSTP = false;156 unsigned PrevBaseReg = 0;157 for (auto &MI : MBB) {158 if (!isNarrowFPStore(MI))159 continue;160 const MachineOperand *BaseOp;161 int64_t Offset;162 bool OffsetIsScalable;163 if (TII->getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable,164 TRI) &&165 BaseOp->isReg()) {166 Register BaseReg = BaseOp->getReg();167 if (PrevBaseReg == BaseReg) {168 // If this block can take STPs, skip ahead to the next block.169 if (!SuppressSTP && shouldAddSTPToBlock(MI.getParent()))170 break;171 // Otherwise, continue unpairing the stores in this block.172 LLVM_DEBUG(dbgs() << "Unpairing store " << MI << "\n");173 SuppressSTP = true;174 TII->suppressLdStPair(MI);175 }176 PrevBaseReg = BaseReg;177 } else178 PrevBaseReg = 0;179 }180 }181 // This pass just sets some internal MachineMemOperand flags. It can't really182 // invalidate anything.183 return false;184}185