326 lines · cpp
1//===----- X86DynAllocaExpander.cpp - Expand DynAlloca pseudo instruction -===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file defines a pass that expands DynAlloca pseudo-instructions.10//11// It performs a conservative analysis to determine whether each allocation12// falls within a region of the stack that is safe to use, or whether stack13// probes must be emitted.14//15//===----------------------------------------------------------------------===//16 17#include "X86.h"18#include "X86InstrInfo.h"19#include "X86MachineFunctionInfo.h"20#include "X86Subtarget.h"21#include "llvm/ADT/MapVector.h"22#include "llvm/ADT/PostOrderIterator.h"23#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"24#include "llvm/CodeGen/MachineFunctionPass.h"25#include "llvm/CodeGen/MachineInstrBuilder.h"26#include "llvm/CodeGen/MachineRegisterInfo.h"27#include "llvm/CodeGen/Passes.h"28#include "llvm/CodeGen/TargetInstrInfo.h"29#include "llvm/IR/Analysis.h"30#include "llvm/IR/Function.h"31 32using namespace llvm;33 34namespace {35 36class X86DynAllocaExpander {37public:38 bool run(MachineFunction &MF);39 40private:41 /// Strategies for lowering a DynAlloca.42 enum Lowering { TouchAndSub, Sub, Probe };43 44 /// Deterministic-order map from DynAlloca instruction to desired lowering.45 typedef MapVector<MachineInstr*, Lowering> LoweringMap;46 47 /// Compute which lowering to use for each DynAlloca instruction.48 void computeLowerings(MachineFunction &MF, LoweringMap& Lowerings);49 50 /// Get the appropriate lowering based on current offset and amount.51 Lowering getLowering(int64_t CurrentOffset, int64_t AllocaAmount);52 53 /// Lower a DynAlloca instruction.54 void lower(MachineInstr* MI, Lowering L);55 56 MachineRegisterInfo *MRI = nullptr;57 const X86Subtarget *STI = nullptr;58 const TargetInstrInfo *TII = nullptr;59 const X86RegisterInfo *TRI = nullptr;60 Register StackPtr;61 unsigned SlotSize = 0;62 int64_t StackProbeSize = 0;63 bool NoStackArgProbe = false;64};65 66class X86DynAllocaExpanderLegacy : public MachineFunctionPass {67public:68 X86DynAllocaExpanderLegacy() : MachineFunctionPass(ID) {}69 70 bool runOnMachineFunction(MachineFunction &MF) override;71 72private:73 StringRef getPassName() const override { return "X86 DynAlloca Expander"; }74 75public:76 static char ID;77};78 79char X86DynAllocaExpanderLegacy::ID = 0;80 81} // end anonymous namespace82 83INITIALIZE_PASS(X86DynAllocaExpanderLegacy, "x86-dyn-alloca-expander",84 "X86 DynAlloca Expander", false, false)85 86FunctionPass *llvm::createX86DynAllocaExpanderLegacyPass() {87 return new X86DynAllocaExpanderLegacy();88}89 90/// Return the allocation amount for a DynAlloca instruction, or -1 if unknown.91static int64_t getDynAllocaAmount(MachineInstr *MI, MachineRegisterInfo *MRI) {92 assert(MI->getOpcode() == X86::DYN_ALLOCA_32 ||93 MI->getOpcode() == X86::DYN_ALLOCA_64);94 assert(MI->getOperand(0).isReg());95 96 Register AmountReg = MI->getOperand(0).getReg();97 MachineInstr *Def = MRI->getUniqueVRegDef(AmountReg);98 99 if (!Def ||100 (Def->getOpcode() != X86::MOV32ri && Def->getOpcode() != X86::MOV64ri) ||101 !Def->getOperand(1).isImm())102 return -1;103 104 return Def->getOperand(1).getImm();105}106 107X86DynAllocaExpander::Lowering108X86DynAllocaExpander::getLowering(int64_t CurrentOffset,109 int64_t AllocaAmount) {110 // For a non-constant amount or a large amount, we have to probe.111 if (AllocaAmount < 0 || AllocaAmount > StackProbeSize)112 return Probe;113 114 // If it fits within the safe region of the stack, just subtract.115 if (CurrentOffset + AllocaAmount <= StackProbeSize)116 return Sub;117 118 // Otherwise, touch the current tip of the stack, then subtract.119 return TouchAndSub;120}121 122static bool isPushPop(const MachineInstr &MI) {123 switch (MI.getOpcode()) {124 case X86::PUSH32r:125 case X86::PUSH32rmm:126 case X86::PUSH32rmr:127 case X86::PUSH32i:128 case X86::PUSH64r:129 case X86::PUSH64rmm:130 case X86::PUSH64rmr:131 case X86::PUSH64i32:132 case X86::POP32r:133 case X86::POP64r:134 return true;135 default:136 return false;137 }138}139 140void X86DynAllocaExpander::computeLowerings(MachineFunction &MF,141 LoweringMap &Lowerings) {142 // Do a one-pass reverse post-order walk of the CFG to conservatively estimate143 // the offset between the stack pointer and the lowest touched part of the144 // stack, and use that to decide how to lower each DynAlloca instruction.145 146 // Initialize OutOffset[B], the stack offset at exit from B, to something big.147 DenseMap<MachineBasicBlock *, int64_t> OutOffset;148 for (MachineBasicBlock &MBB : MF)149 OutOffset[&MBB] = INT32_MAX;150 151 // Note: we don't know the offset at the start of the entry block since the152 // prologue hasn't been inserted yet, and how much that will adjust the stack153 // pointer depends on register spills, which have not been computed yet.154 155 // Compute the reverse post-order.156 ReversePostOrderTraversal<MachineFunction*> RPO(&MF);157 158 for (MachineBasicBlock *MBB : RPO) {159 int64_t Offset = -1;160 for (MachineBasicBlock *Pred : MBB->predecessors())161 Offset = std::max(Offset, OutOffset[Pred]);162 if (Offset == -1) Offset = INT32_MAX;163 164 for (MachineInstr &MI : *MBB) {165 if (MI.getOpcode() == X86::DYN_ALLOCA_32 ||166 MI.getOpcode() == X86::DYN_ALLOCA_64) {167 // A DynAlloca moves StackPtr, and potentially touches it.168 int64_t Amount = getDynAllocaAmount(&MI, MRI);169 Lowering L = getLowering(Offset, Amount);170 Lowerings[&MI] = L;171 switch (L) {172 case Sub:173 Offset += Amount;174 break;175 case TouchAndSub:176 Offset = Amount;177 break;178 case Probe:179 Offset = 0;180 break;181 }182 } else if (MI.isCall() || isPushPop(MI)) {183 // Calls, pushes and pops touch the tip of the stack.184 Offset = 0;185 } else if (MI.getOpcode() == X86::ADJCALLSTACKUP32 ||186 MI.getOpcode() == X86::ADJCALLSTACKUP64) {187 Offset -= MI.getOperand(0).getImm();188 } else if (MI.getOpcode() == X86::ADJCALLSTACKDOWN32 ||189 MI.getOpcode() == X86::ADJCALLSTACKDOWN64) {190 Offset += MI.getOperand(0).getImm();191 } else if (MI.modifiesRegister(StackPtr, TRI)) {192 // Any other modification of SP means we've lost track of it.193 Offset = INT32_MAX;194 }195 }196 197 OutOffset[MBB] = Offset;198 }199}200 201static unsigned getSubOpcode(bool Is64Bit) {202 if (Is64Bit)203 return X86::SUB64ri32;204 return X86::SUB32ri;205}206 207void X86DynAllocaExpander::lower(MachineInstr *MI, Lowering L) {208 const DebugLoc &DL = MI->getDebugLoc();209 MachineBasicBlock *MBB = MI->getParent();210 MachineBasicBlock::iterator I = *MI;211 212 int64_t Amount = getDynAllocaAmount(MI, MRI);213 if (Amount == 0) {214 MI->eraseFromParent();215 return;216 }217 218 // These two variables differ on x32, which is a 64-bit target with a219 // 32-bit alloca.220 bool Is64Bit = STI->is64Bit();221 bool Is64BitAlloca = MI->getOpcode() == X86::DYN_ALLOCA_64;222 assert(SlotSize == 4 || SlotSize == 8);223 224 std::optional<MachineFunction::DebugInstrOperandPair> InstrNum;225 if (unsigned Num = MI->peekDebugInstrNum()) {226 // Operand 2 of DYN_ALLOCAs contains the stack def.227 InstrNum = {Num, 2};228 }229 230 switch (L) {231 case TouchAndSub: {232 assert(Amount >= SlotSize);233 234 // Use a push to touch the top of the stack.235 unsigned RegA = Is64Bit ? X86::RAX : X86::EAX;236 BuildMI(*MBB, I, DL, TII->get(Is64Bit ? X86::PUSH64r : X86::PUSH32r))237 .addReg(RegA, RegState::Undef);238 Amount -= SlotSize;239 if (!Amount)240 break;241 242 // Fall through to make any remaining adjustment.243 [[fallthrough]];244 }245 case Sub:246 assert(Amount > 0);247 if (Amount == SlotSize) {248 // Use push to save size.249 unsigned RegA = Is64Bit ? X86::RAX : X86::EAX;250 BuildMI(*MBB, I, DL, TII->get(Is64Bit ? X86::PUSH64r : X86::PUSH32r))251 .addReg(RegA, RegState::Undef);252 } else {253 // Sub.254 BuildMI(*MBB, I, DL, TII->get(getSubOpcode(Is64BitAlloca)), StackPtr)255 .addReg(StackPtr)256 .addImm(Amount);257 }258 break;259 case Probe:260 if (!NoStackArgProbe) {261 // The probe lowering expects the amount in RAX/EAX.262 unsigned RegA = Is64BitAlloca ? X86::RAX : X86::EAX;263 BuildMI(*MBB, MI, DL, TII->get(TargetOpcode::COPY), RegA)264 .addReg(MI->getOperand(0).getReg());265 266 // Do the probe.267 STI->getFrameLowering()->emitStackProbe(*MBB->getParent(), *MBB, MI, DL,268 /*InProlog=*/false, InstrNum);269 } else {270 // Sub271 BuildMI(*MBB, I, DL,272 TII->get(Is64BitAlloca ? X86::SUB64rr : X86::SUB32rr), StackPtr)273 .addReg(StackPtr)274 .addReg(MI->getOperand(0).getReg());275 }276 break;277 }278 279 Register AmountReg = MI->getOperand(0).getReg();280 MI->eraseFromParent();281 282 // Delete the definition of AmountReg.283 if (MRI->use_empty(AmountReg))284 if (MachineInstr *AmountDef = MRI->getUniqueVRegDef(AmountReg))285 AmountDef->eraseFromParent();286}287 288bool X86DynAllocaExpander::run(MachineFunction &MF) {289 if (!MF.getInfo<X86MachineFunctionInfo>()->hasDynAlloca())290 return false;291 292 MRI = &MF.getRegInfo();293 STI = &MF.getSubtarget<X86Subtarget>();294 TII = STI->getInstrInfo();295 TRI = STI->getRegisterInfo();296 StackPtr = TRI->getStackRegister();297 SlotSize = TRI->getSlotSize();298 StackProbeSize = STI->getTargetLowering()->getStackProbeSize(MF);299 NoStackArgProbe = MF.getFunction().hasFnAttribute("no-stack-arg-probe");300 if (NoStackArgProbe)301 StackProbeSize = INT64_MAX;302 303 LoweringMap Lowerings;304 computeLowerings(MF, Lowerings);305 for (auto &P : Lowerings)306 lower(P.first, P.second);307 308 return true;309}310 311bool X86DynAllocaExpanderLegacy::runOnMachineFunction(MachineFunction &MF) {312 return X86DynAllocaExpander().run(MF);313}314 315PreservedAnalyses316X86DynAllocaExpanderPass::run(MachineFunction &MF,317 MachineFunctionAnalysisManager &MFAM) {318 bool Changed = X86DynAllocaExpander().run(MF);319 if (!Changed)320 return PreservedAnalyses::all();321 322 PreservedAnalyses PA = PreservedAnalyses::none();323 PA.preserveSet<CFGAnalyses>();324 return PA;325}326