//===----------------------------------------------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "AArch64PrologueEpilogue.h" #include "AArch64FrameLowering.h" #include "AArch64MachineFunctionInfo.h" #include "AArch64Subtarget.h" #include "MCTargetDesc/AArch64AddressingModes.h" #include "llvm/ADT/Statistic.h" #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/CodeGen/CFIInstBuilder.h" #include "llvm/MC/MCContext.h" #define DEBUG_TYPE "frame-info" STATISTIC(NumRedZoneFunctions, "Number of functions using red zone"); namespace llvm { static bool matchLibcall(const TargetLowering &TLI, const MachineOperand &MO, RTLIB::Libcall LC) { return MO.isSymbol() && StringRef(TLI.getLibcallName(LC)) == MO.getSymbolName(); } bool AArch64PrologueEpilogueCommon::requiresGetVGCall() const { return AFI->hasStreamingModeChanges() && !MF.getSubtarget().hasSVE(); } bool AArch64PrologueEpilogueCommon::isVGInstruction( MachineBasicBlock::iterator MBBI, const TargetLowering &TLI) const { unsigned Opc = MBBI->getOpcode(); if (Opc == AArch64::CNTD_XPiI) return true; if (!requiresGetVGCall()) return false; if (Opc == AArch64::BL) return matchLibcall(TLI, MBBI->getOperand(0), RTLIB::SMEABI_GET_CURRENT_VG); return Opc == TargetOpcode::COPY; } // Convenience function to determine whether I is part of the ZPR callee saves. static bool isPartOfZPRCalleeSaves(MachineBasicBlock::iterator I) { switch (I->getOpcode()) { default: return false; case AArch64::LD1B_2Z_IMM: case AArch64::ST1B_2Z_IMM: case AArch64::STR_ZXI: case AArch64::LDR_ZXI: case AArch64::PTRUE_C_B: return I->getFlag(MachineInstr::FrameSetup) || I->getFlag(MachineInstr::FrameDestroy); case AArch64::SEH_SaveZReg: return true; } } // Convenience function to determine whether I is part of the PPR callee saves. static bool isPartOfPPRCalleeSaves(MachineBasicBlock::iterator I) { switch (I->getOpcode()) { default: return false; case AArch64::STR_PXI: case AArch64::LDR_PXI: return I->getFlag(MachineInstr::FrameSetup) || I->getFlag(MachineInstr::FrameDestroy); case AArch64::SEH_SavePReg: return true; } } // Convenience function to determine whether I is part of the SVE callee saves. static bool isPartOfSVECalleeSaves(MachineBasicBlock::iterator I) { return isPartOfZPRCalleeSaves(I) || isPartOfPPRCalleeSaves(I); } AArch64PrologueEpilogueCommon::AArch64PrologueEpilogueCommon( MachineFunction &MF, MachineBasicBlock &MBB, const AArch64FrameLowering &AFL) : MF(MF), MBB(MBB), MFI(MF.getFrameInfo()), Subtarget(MF.getSubtarget()), AFL(AFL), RegInfo(*Subtarget.getRegisterInfo()) { TII = Subtarget.getInstrInfo(); AFI = MF.getInfo(); HasFP = AFL.hasFP(MF); NeedsWinCFI = AFL.needsWinCFI(MF); // Windows unwind can't represent the required stack adjustments if we have // both SVE callee-saves and dynamic stack allocations, and the frame pointer // is before the SVE spills. The allocation of the frame pointer must be the // last instruction in the prologue so the unwinder can restore the stack // pointer correctly. (And there isn't any unwind opcode for `addvl sp, x29, // -17`.) // // Because of this, we do spills in the opposite order on Windows: first SVE, // then GPRs. The main side-effect of this is that it makes accessing // parameters passed on the stack more expensive. // // We could consider rearranging the spills for simpler cases. if (Subtarget.isTargetWindows() && AFI->getSVECalleeSavedStackSize()) { if (AFI->hasStackHazardSlotIndex()) reportFatalUsageError("SME hazard padding is not supported on Windows"); SVELayout = SVEStackLayout::CalleeSavesAboveFrameRecord; } else if (AFI->hasSplitSVEObjects()) { SVELayout = SVEStackLayout::Split; } } MachineBasicBlock::iterator AArch64PrologueEpilogueCommon::convertCalleeSaveRestoreToSPPrePostIncDec( MachineBasicBlock::iterator MBBI, const DebugLoc &DL, int CSStackSizeInc, bool EmitCFI, MachineInstr::MIFlag FrameFlag, int CFAOffset) const { unsigned NewOpc; // If the function contains streaming mode changes, we expect instructions // to calculate the value of VG before spilling. Move past these instructions // if necessary. if (AFL.requiresSaveVG(MF)) { auto &TLI = *Subtarget.getTargetLowering(); while (isVGInstruction(MBBI, TLI)) ++MBBI; } switch (MBBI->getOpcode()) { default: llvm_unreachable("Unexpected callee-save save/restore opcode!"); case AArch64::STPXi: NewOpc = AArch64::STPXpre; break; case AArch64::STPDi: NewOpc = AArch64::STPDpre; break; case AArch64::STPQi: NewOpc = AArch64::STPQpre; break; case AArch64::STRXui: NewOpc = AArch64::STRXpre; break; case AArch64::STRDui: NewOpc = AArch64::STRDpre; break; case AArch64::STRQui: NewOpc = AArch64::STRQpre; break; case AArch64::LDPXi: NewOpc = AArch64::LDPXpost; break; case AArch64::LDPDi: NewOpc = AArch64::LDPDpost; break; case AArch64::LDPQi: NewOpc = AArch64::LDPQpost; break; case AArch64::LDRXui: NewOpc = AArch64::LDRXpost; break; case AArch64::LDRDui: NewOpc = AArch64::LDRDpost; break; case AArch64::LDRQui: NewOpc = AArch64::LDRQpost; break; } TypeSize Scale = TypeSize::getFixed(1), Width = TypeSize::getFixed(0); int64_t MinOffset, MaxOffset; bool Success = static_cast(TII)->getMemOpInfo( NewOpc, Scale, Width, MinOffset, MaxOffset); (void)Success; assert(Success && "unknown load/store opcode"); // If the first store isn't right where we want SP then we can't fold the // update in so create a normal arithmetic instruction instead. if (MBBI->getOperand(MBBI->getNumOperands() - 1).getImm() != 0 || CSStackSizeInc < MinOffset * (int64_t)Scale.getFixedValue() || CSStackSizeInc > MaxOffset * (int64_t)Scale.getFixedValue()) { // If we are destroying the frame, make sure we add the increment after the // last frame operation. if (FrameFlag == MachineInstr::FrameDestroy) { ++MBBI; // Also skip the SEH instruction, if needed if (NeedsWinCFI && AArch64InstrInfo::isSEHInstruction(*MBBI)) ++MBBI; } emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, StackOffset::getFixed(CSStackSizeInc), TII, FrameFlag, false, NeedsWinCFI, &HasWinCFI, EmitCFI, StackOffset::getFixed(CFAOffset)); return std::prev(MBBI); } // Get rid of the SEH code associated with the old instruction. if (NeedsWinCFI) { auto SEH = std::next(MBBI); if (AArch64InstrInfo::isSEHInstruction(*SEH)) SEH->eraseFromParent(); } MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc)); MIB.addReg(AArch64::SP, RegState::Define); // Copy all operands other than the immediate offset. unsigned OpndIdx = 0; for (unsigned OpndEnd = MBBI->getNumOperands() - 1; OpndIdx < OpndEnd; ++OpndIdx) MIB.add(MBBI->getOperand(OpndIdx)); assert(MBBI->getOperand(OpndIdx).getImm() == 0 && "Unexpected immediate offset in first/last callee-save save/restore " "instruction!"); assert(MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP && "Unexpected base register in callee-save save/restore instruction!"); assert(CSStackSizeInc % Scale == 0); MIB.addImm(CSStackSizeInc / (int)Scale); MIB.setMIFlags(MBBI->getFlags()); MIB.setMemRefs(MBBI->memoperands()); // Generate a new SEH code that corresponds to the new instruction. if (NeedsWinCFI) { HasWinCFI = true; AFL.insertSEH(*MIB, *TII, FrameFlag); } if (EmitCFI) CFIInstBuilder(MBB, MBBI, FrameFlag) .buildDefCFAOffset(CFAOffset - CSStackSizeInc); return std::prev(MBB.erase(MBBI)); } // Fix up the SEH opcode associated with the save/restore instruction. static void fixupSEHOpcode(MachineBasicBlock::iterator MBBI, unsigned LocalStackSize) { MachineOperand *ImmOpnd = nullptr; unsigned ImmIdx = MBBI->getNumOperands() - 1; switch (MBBI->getOpcode()) { default: llvm_unreachable("Fix the offset in the SEH instruction"); case AArch64::SEH_SaveFPLR: case AArch64::SEH_SaveRegP: case AArch64::SEH_SaveReg: case AArch64::SEH_SaveFRegP: case AArch64::SEH_SaveFReg: case AArch64::SEH_SaveAnyRegI: case AArch64::SEH_SaveAnyRegIP: case AArch64::SEH_SaveAnyRegQP: case AArch64::SEH_SaveAnyRegQPX: ImmOpnd = &MBBI->getOperand(ImmIdx); break; } if (ImmOpnd) ImmOpnd->setImm(ImmOpnd->getImm() + LocalStackSize); } void AArch64PrologueEpilogueCommon::fixupCalleeSaveRestoreStackOffset( MachineInstr &MI, uint64_t LocalStackSize) const { if (AArch64InstrInfo::isSEHInstruction(MI)) return; unsigned Opc = MI.getOpcode(); unsigned Scale; switch (Opc) { case AArch64::STPXi: case AArch64::STRXui: case AArch64::STPDi: case AArch64::STRDui: case AArch64::LDPXi: case AArch64::LDRXui: case AArch64::LDPDi: case AArch64::LDRDui: Scale = 8; break; case AArch64::STPQi: case AArch64::STRQui: case AArch64::LDPQi: case AArch64::LDRQui: Scale = 16; break; default: llvm_unreachable("Unexpected callee-save save/restore opcode!"); } unsigned OffsetIdx = MI.getNumExplicitOperands() - 1; assert(MI.getOperand(OffsetIdx - 1).getReg() == AArch64::SP && "Unexpected base register in callee-save save/restore instruction!"); // Last operand is immediate offset that needs fixing. MachineOperand &OffsetOpnd = MI.getOperand(OffsetIdx); // All generated opcodes have scaled offsets. assert(LocalStackSize % Scale == 0); OffsetOpnd.setImm(OffsetOpnd.getImm() + LocalStackSize / Scale); if (NeedsWinCFI) { HasWinCFI = true; auto MBBI = std::next(MachineBasicBlock::iterator(MI)); assert(MBBI != MI.getParent()->end() && "Expecting a valid instruction"); assert(AArch64InstrInfo::isSEHInstruction(*MBBI) && "Expecting a SEH instruction"); fixupSEHOpcode(MBBI, LocalStackSize); } } bool AArch64PrologueEpilogueCommon::shouldCombineCSRLocalStackBump( uint64_t StackBumpBytes) const { if (AFL.homogeneousPrologEpilog(MF)) return false; if (AFI->getLocalStackSize() == 0) return false; // For WinCFI, if optimizing for size, prefer to not combine the stack bump // (to force a stp with predecrement) to match the packed unwind format, // provided that there actually are any callee saved registers to merge the // decrement with. // This is potentially marginally slower, but allows using the packed // unwind format for functions that both have a local area and callee saved // registers. Using the packed unwind format notably reduces the size of // the unwind info. if (AFL.needsWinCFI(MF) && AFI->getCalleeSavedStackSize() > 0 && MF.getFunction().hasOptSize()) return false; // 512 is the maximum immediate for stp/ldp that will be used for // callee-save save/restores if (StackBumpBytes >= 512 || AFL.windowsRequiresStackProbe(MF, StackBumpBytes)) return false; if (MFI.hasVarSizedObjects()) return false; if (RegInfo.hasStackRealignment(MF)) return false; // This isn't strictly necessary, but it simplifies things a bit since the // current RedZone handling code assumes the SP is adjusted by the // callee-save save/restore code. if (AFL.canUseRedZone(MF)) return false; // When there is an SVE area on the stack, always allocate the // callee-saves and spills/locals separately. if (AFI->hasSVEStackSize()) return false; return true; } SVEFrameSizes AArch64PrologueEpilogueCommon::getSVEStackFrameSizes() const { StackOffset PPRCalleeSavesSize = StackOffset::getScalable(AFI->getPPRCalleeSavedStackSize()); StackOffset ZPRCalleeSavesSize = StackOffset::getScalable(AFI->getZPRCalleeSavedStackSize()); StackOffset PPRLocalsSize = AFL.getPPRStackSize(MF) - PPRCalleeSavesSize; StackOffset ZPRLocalsSize = AFL.getZPRStackSize(MF) - ZPRCalleeSavesSize; if (SVELayout == SVEStackLayout::Split) return {{PPRCalleeSavesSize, PPRLocalsSize}, {ZPRCalleeSavesSize, ZPRLocalsSize}}; // For simplicity, attribute all locals to ZPRs when split SVE is disabled. return {{PPRCalleeSavesSize, StackOffset{}}, {ZPRCalleeSavesSize, PPRLocalsSize + ZPRLocalsSize}}; } SVEStackAllocations AArch64PrologueEpilogueCommon::getSVEStackAllocations( SVEFrameSizes const &SVE) { StackOffset AfterZPRs = SVE.ZPR.LocalsSize; StackOffset BeforePPRs = SVE.ZPR.CalleeSavesSize + SVE.PPR.CalleeSavesSize; StackOffset AfterPPRs = {}; if (SVELayout == SVEStackLayout::Split) { BeforePPRs = SVE.PPR.CalleeSavesSize; // If there are no ZPR CSRs, place all local allocations after the ZPRs. if (SVE.ZPR.CalleeSavesSize) AfterPPRs += SVE.PPR.LocalsSize + SVE.ZPR.CalleeSavesSize; else AfterZPRs += SVE.PPR.LocalsSize; // Group allocation of locals. } return {BeforePPRs, AfterPPRs, AfterZPRs}; } struct SVEPartitions { struct { MachineBasicBlock::iterator Begin, End; } PPR, ZPR; }; static SVEPartitions partitionSVECS(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, StackOffset PPRCalleeSavesSize, StackOffset ZPRCalleeSavesSize, bool IsEpilogue) { MachineBasicBlock::iterator PPRsI = MBBI; MachineBasicBlock::iterator End = IsEpilogue ? MBB.begin() : MBB.getFirstTerminator(); auto AdjustI = [&](auto MBBI) { return IsEpilogue ? std::prev(MBBI) : MBBI; }; // Process the SVE CS to find the starts/ends of the ZPR and PPR areas. if (PPRCalleeSavesSize) { PPRsI = AdjustI(PPRsI); assert(isPartOfPPRCalleeSaves(*PPRsI) && "Unexpected instruction"); while (PPRsI != End && isPartOfPPRCalleeSaves(AdjustI(PPRsI))) IsEpilogue ? (--PPRsI) : (++PPRsI); } MachineBasicBlock::iterator ZPRsI = PPRsI; if (ZPRCalleeSavesSize) { ZPRsI = AdjustI(ZPRsI); assert(isPartOfZPRCalleeSaves(*ZPRsI) && "Unexpected instruction"); while (ZPRsI != End && isPartOfZPRCalleeSaves(AdjustI(ZPRsI))) IsEpilogue ? (--ZPRsI) : (++ZPRsI); } if (IsEpilogue) return {{PPRsI, MBBI}, {ZPRsI, PPRsI}}; return {{MBBI, PPRsI}, {PPRsI, ZPRsI}}; } AArch64PrologueEmitter::AArch64PrologueEmitter(MachineFunction &MF, MachineBasicBlock &MBB, const AArch64FrameLowering &AFL) : AArch64PrologueEpilogueCommon(MF, MBB, AFL), F(MF.getFunction()) { EmitCFI = AFI->needsDwarfUnwindInfo(MF); EmitAsyncCFI = AFI->needsAsyncDwarfUnwindInfo(MF); IsFunclet = MBB.isEHFuncletEntry(); HomPrologEpilog = AFL.homogeneousPrologEpilog(MF); #ifndef NDEBUG collectBlockLiveins(); #endif } #ifndef NDEBUG /// Collect live registers from the end of \p MI's parent up to (including) \p /// MI in \p LiveRegs. static void getLivePhysRegsUpTo(MachineInstr &MI, const TargetRegisterInfo &TRI, LivePhysRegs &LiveRegs) { MachineBasicBlock &MBB = *MI.getParent(); LiveRegs.addLiveOuts(MBB); for (const MachineInstr &MI : reverse(make_range(MI.getIterator(), MBB.instr_end()))) LiveRegs.stepBackward(MI); } void AArch64PrologueEmitter::collectBlockLiveins() { // Collect live register from the end of MBB up to the start of the existing // frame setup instructions. PrologueEndI = MBB.begin(); while (PrologueEndI != MBB.end() && PrologueEndI->getFlag(MachineInstr::FrameSetup)) ++PrologueEndI; if (PrologueEndI != MBB.end()) { getLivePhysRegsUpTo(*PrologueEndI, RegInfo, LiveRegs); // Ignore registers used for stack management for now. LiveRegs.removeReg(AArch64::SP); LiveRegs.removeReg(AArch64::X19); LiveRegs.removeReg(AArch64::FP); LiveRegs.removeReg(AArch64::LR); // X0 will be clobbered by a call to __arm_get_current_vg in the prologue. // This is necessary to spill VG if required where SVE is unavailable, but // X0 is preserved around this call. if (requiresGetVGCall()) LiveRegs.removeReg(AArch64::X0); } } void AArch64PrologueEmitter::verifyPrologueClobbers() const { if (PrologueEndI == MBB.end()) return; // Check if any of the newly instructions clobber any of the live registers. for (MachineInstr &MI : make_range(MBB.instr_begin(), PrologueEndI->getIterator())) { for (auto &Op : MI.operands()) if (Op.isReg() && Op.isDef()) assert(!LiveRegs.contains(Op.getReg()) && "live register clobbered by inserted prologue instructions"); } } #endif void AArch64PrologueEmitter::determineLocalsStackSize( uint64_t StackSize, uint64_t PrologueSaveSize) { AFI->setLocalStackSize(StackSize - PrologueSaveSize); CombineSPBump = shouldCombineCSRLocalStackBump(StackSize); } // Return the maximum possible number of bytes for `Size` due to the // architectural limit on the size of a SVE register. static int64_t upperBound(StackOffset Size) { static const int64_t MAX_BYTES_PER_SCALABLE_BYTE = 16; return Size.getScalable() * MAX_BYTES_PER_SCALABLE_BYTE + Size.getFixed(); } void AArch64PrologueEmitter::allocateStackSpace( MachineBasicBlock::iterator MBBI, int64_t RealignmentPadding, StackOffset AllocSize, bool EmitCFI, StackOffset InitialOffset, bool FollowupAllocs) { if (!AllocSize) return; DebugLoc DL; const int64_t MaxAlign = MFI.getMaxAlign().value(); const uint64_t AndMask = ~(MaxAlign - 1); if (!Subtarget.getTargetLowering()->hasInlineStackProbe(MF)) { Register TargetReg = RealignmentPadding ? AFL.findScratchNonCalleeSaveRegister(&MBB) : AArch64::SP; // SUB Xd/SP, SP, AllocSize emitFrameOffset(MBB, MBBI, DL, TargetReg, AArch64::SP, -AllocSize, TII, MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI, EmitCFI, InitialOffset); if (RealignmentPadding) { // AND SP, X9, 0b11111...0000 BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP) .addReg(TargetReg, RegState::Kill) .addImm(AArch64_AM::encodeLogicalImmediate(AndMask, 64)) .setMIFlags(MachineInstr::FrameSetup); AFI->setStackRealigned(true); // No need for SEH instructions here; if we're realigning the stack, // we've set a frame pointer and already finished the SEH prologue. assert(!NeedsWinCFI); } return; } // // Stack probing allocation. // // Fixed length allocation. If we don't need to re-align the stack and don't // have SVE objects, we can use a more efficient sequence for stack probing. if (AllocSize.getScalable() == 0 && RealignmentPadding == 0) { Register ScratchReg = AFL.findScratchNonCalleeSaveRegister(&MBB); assert(ScratchReg != AArch64::NoRegister); BuildMI(MBB, MBBI, DL, TII->get(AArch64::PROBED_STACKALLOC)) .addDef(ScratchReg) .addImm(AllocSize.getFixed()) .addImm(InitialOffset.getFixed()) .addImm(InitialOffset.getScalable()); // The fixed allocation may leave unprobed bytes at the top of the // stack. If we have subsequent allocation (e.g. if we have variable-sized // objects), we need to issue an extra probe, so these allocations start in // a known state. if (FollowupAllocs) { // STR XZR, [SP] BuildMI(MBB, MBBI, DL, TII->get(AArch64::STRXui)) .addReg(AArch64::XZR) .addReg(AArch64::SP) .addImm(0) .setMIFlags(MachineInstr::FrameSetup); } return; } // Variable length allocation. // If the (unknown) allocation size cannot exceed the probe size, decrement // the stack pointer right away. int64_t ProbeSize = AFI->getStackProbeSize(); if (upperBound(AllocSize) + RealignmentPadding <= ProbeSize) { Register ScratchReg = RealignmentPadding ? AFL.findScratchNonCalleeSaveRegister(&MBB) : AArch64::SP; assert(ScratchReg != AArch64::NoRegister); // SUB Xd, SP, AllocSize emitFrameOffset(MBB, MBBI, DL, ScratchReg, AArch64::SP, -AllocSize, TII, MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI, EmitCFI, InitialOffset); if (RealignmentPadding) { // AND SP, Xn, 0b11111...0000 BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP) .addReg(ScratchReg, RegState::Kill) .addImm(AArch64_AM::encodeLogicalImmediate(AndMask, 64)) .setMIFlags(MachineInstr::FrameSetup); AFI->setStackRealigned(true); } if (FollowupAllocs || upperBound(AllocSize) + RealignmentPadding > AArch64::StackProbeMaxUnprobedStack) { // STR XZR, [SP] BuildMI(MBB, MBBI, DL, TII->get(AArch64::STRXui)) .addReg(AArch64::XZR) .addReg(AArch64::SP) .addImm(0) .setMIFlags(MachineInstr::FrameSetup); } return; } // Emit a variable-length allocation probing loop. // TODO: As an optimisation, the loop can be "unrolled" into a few parts, // each of them guaranteed to adjust the stack by less than the probe size. Register TargetReg = AFL.findScratchNonCalleeSaveRegister(&MBB); assert(TargetReg != AArch64::NoRegister); // SUB Xd, SP, AllocSize emitFrameOffset(MBB, MBBI, DL, TargetReg, AArch64::SP, -AllocSize, TII, MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI, EmitCFI, InitialOffset); if (RealignmentPadding) { // AND Xn, Xn, 0b11111...0000 BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), TargetReg) .addReg(TargetReg, RegState::Kill) .addImm(AArch64_AM::encodeLogicalImmediate(AndMask, 64)) .setMIFlags(MachineInstr::FrameSetup); } BuildMI(MBB, MBBI, DL, TII->get(AArch64::PROBED_STACKALLOC_VAR)) .addReg(TargetReg); if (EmitCFI) { // Set the CFA register back to SP. CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup) .buildDefCFARegister(AArch64::SP); } if (RealignmentPadding) AFI->setStackRealigned(true); } void AArch64PrologueEmitter::emitPrologue() { const MachineBasicBlock::iterator PrologueBeginI = MBB.begin(); const MachineBasicBlock::iterator EndI = MBB.end(); // At this point, we're going to decide whether or not the function uses a // redzone. In most cases, the function doesn't have a redzone so let's // assume that's false and set it to true in the case that there's a redzone. AFI->setHasRedZone(false); // Debug location must be unknown since the first debug location is used // to determine the end of the prologue. DebugLoc DL; // In some cases, particularly with CallingConv::SwiftTail, it is possible to // have a tail-call where the caller only needs to adjust the stack pointer in // the epilogue. In this case, we still need to emit a SEH prologue sequence. // See `seh-minimal-prologue-epilogue.ll` test cases. if (AFI->getArgumentStackToRestore()) HasWinCFI = true; if (AFI->shouldSignReturnAddress(MF)) { // If pac-ret+leaf is in effect, PAUTH_PROLOGUE pseudo instructions // are inserted by emitPacRetPlusLeafHardening(). if (!AFL.shouldSignReturnAddressEverywhere(MF)) { BuildMI(MBB, PrologueBeginI, DL, TII->get(AArch64::PAUTH_PROLOGUE)) .setMIFlag(MachineInstr::FrameSetup); } // AArch64PointerAuth pass will insert SEH_PACSignLR HasWinCFI |= NeedsWinCFI; } if (AFI->needsShadowCallStackPrologueEpilogue(MF)) { emitShadowCallStackPrologue(PrologueBeginI, DL); HasWinCFI |= NeedsWinCFI; } if (EmitCFI && AFI->isMTETagged()) BuildMI(MBB, PrologueBeginI, DL, TII->get(AArch64::EMITMTETAGGED)) .setMIFlag(MachineInstr::FrameSetup); // We signal the presence of a Swift extended frame to external tools by // storing FP with 0b0001 in bits 63:60. In normal userland operation a simple // ORR is sufficient, it is assumed a Swift kernel would initialize the TBI // bits so that is still true. if (HasFP && AFI->hasSwiftAsyncContext()) emitSwiftAsyncContextFramePointer(PrologueBeginI, DL); // All calls are tail calls in GHC calling conv, and functions have no // prologue/epilogue. if (MF.getFunction().getCallingConv() == CallingConv::GHC) return; // Set tagged base pointer to the requested stack slot. Ideally it should // match SP value after prologue. if (std::optional TBPI = AFI->getTaggedBasePointerIndex()) AFI->setTaggedBasePointerOffset(-MFI.getObjectOffset(*TBPI)); else AFI->setTaggedBasePointerOffset(MFI.getStackSize()); // getStackSize() includes all the locals in its size calculation. We don't // include these locals when computing the stack size of a funclet, as they // are allocated in the parent's stack frame and accessed via the frame // pointer from the funclet. We only save the callee saved registers in the // funclet, which are really the callee saved registers of the parent // function, including the funclet. int64_t NumBytes = IsFunclet ? AFL.getWinEHFuncletFrameSize(MF) : MFI.getStackSize(); if (!AFI->hasStackFrame() && !AFL.windowsRequiresStackProbe(MF, NumBytes)) return emitEmptyStackFramePrologue(NumBytes, PrologueBeginI, DL); bool IsWin64 = Subtarget.isCallingConvWin64(F.getCallingConv(), F.isVarArg()); unsigned FixedObject = AFL.getFixedObjectSize(MF, AFI, IsWin64, IsFunclet); auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject; // All of the remaining stack allocations are for locals. determineLocalsStackSize(NumBytes, PrologueSaveSize); auto [PPR, ZPR] = getSVEStackFrameSizes(); SVEStackAllocations SVEAllocs = getSVEStackAllocations({PPR, ZPR}); MachineBasicBlock::iterator FirstGPRSaveI = PrologueBeginI; if (SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord) { assert(!SVEAllocs.AfterPPRs && "unexpected SVE allocs after PPRs with CalleeSavesAboveFrameRecord"); // If we're doing SVE saves first, we need to immediately allocate space // for fixed objects, then space for the SVE callee saves. // // Windows unwind requires that the scalable size is a multiple of 16; // that's handled when the callee-saved size is computed. auto SaveSize = SVEAllocs.BeforePPRs + StackOffset::getFixed(FixedObject); allocateStackSpace(PrologueBeginI, 0, SaveSize, false, StackOffset{}, /*FollowupAllocs=*/true); NumBytes -= FixedObject; // Now allocate space for the GPR callee saves. MachineBasicBlock::iterator MBBI = PrologueBeginI; while (MBBI != EndI && isPartOfSVECalleeSaves(MBBI)) ++MBBI; FirstGPRSaveI = convertCalleeSaveRestoreToSPPrePostIncDec( MBBI, DL, -AFI->getCalleeSavedStackSize(), EmitAsyncCFI); NumBytes -= AFI->getCalleeSavedStackSize(); } else if (CombineSPBump) { assert(!AFL.getSVEStackSize(MF) && "Cannot combine SP bump with SVE"); emitFrameOffset(MBB, PrologueBeginI, DL, AArch64::SP, AArch64::SP, StackOffset::getFixed(-NumBytes), TII, MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI, EmitAsyncCFI); NumBytes = 0; } else if (HomPrologEpilog) { // Stack has been already adjusted. NumBytes -= PrologueSaveSize; } else if (PrologueSaveSize != 0) { FirstGPRSaveI = convertCalleeSaveRestoreToSPPrePostIncDec( PrologueBeginI, DL, -PrologueSaveSize, EmitAsyncCFI); NumBytes -= PrologueSaveSize; } assert(NumBytes >= 0 && "Negative stack allocation size!?"); // Move past the saves of the callee-saved registers, fixing up the offsets // and pre-inc if we decided to combine the callee-save and local stack // pointer bump above. auto &TLI = *Subtarget.getTargetLowering(); MachineBasicBlock::iterator AfterGPRSavesI = FirstGPRSaveI; while (AfterGPRSavesI != EndI && AfterGPRSavesI->getFlag(MachineInstr::FrameSetup) && !isPartOfSVECalleeSaves(AfterGPRSavesI)) { if (CombineSPBump && // Only fix-up frame-setup load/store instructions. (!AFL.requiresSaveVG(MF) || !isVGInstruction(AfterGPRSavesI, TLI))) fixupCalleeSaveRestoreStackOffset(*AfterGPRSavesI, AFI->getLocalStackSize()); ++AfterGPRSavesI; } // For funclets the FP belongs to the containing function. Only set up FP if // we actually need to. if (!IsFunclet && HasFP) emitFramePointerSetup(AfterGPRSavesI, DL, FixedObject); // Now emit the moves for whatever callee saved regs we have (including FP, // LR if those are saved). Frame instructions for SVE register are emitted // later, after the instruction which actually save SVE regs. if (EmitAsyncCFI) emitCalleeSavedGPRLocations(AfterGPRSavesI); // Alignment is required for the parent frame, not the funclet const bool NeedsRealignment = NumBytes && !IsFunclet && RegInfo.hasStackRealignment(MF); const int64_t RealignmentPadding = (NeedsRealignment && MFI.getMaxAlign() > Align(16)) ? MFI.getMaxAlign().value() - 16 : 0; if (AFL.windowsRequiresStackProbe(MF, NumBytes + RealignmentPadding)) emitWindowsStackProbe(AfterGPRSavesI, DL, NumBytes, RealignmentPadding); StackOffset NonSVELocalsSize = StackOffset::getFixed(NumBytes); SVEAllocs.AfterZPRs += NonSVELocalsSize; StackOffset CFAOffset = StackOffset::getFixed(MFI.getStackSize()) - NonSVELocalsSize; MachineBasicBlock::iterator AfterSVESavesI = AfterGPRSavesI; // Allocate space for the callee saves and PPR locals (if any). if (SVELayout != SVEStackLayout::CalleeSavesAboveFrameRecord) { auto [PPRRange, ZPRRange] = partitionSVECS(MBB, AfterGPRSavesI, PPR.CalleeSavesSize, ZPR.CalleeSavesSize, /*IsEpilogue=*/false); AfterSVESavesI = ZPRRange.End; if (EmitAsyncCFI) emitCalleeSavedSVELocations(AfterSVESavesI); allocateStackSpace(PPRRange.Begin, 0, SVEAllocs.BeforePPRs, EmitAsyncCFI && !HasFP, CFAOffset, MFI.hasVarSizedObjects() || SVEAllocs.AfterPPRs || SVEAllocs.AfterZPRs); CFAOffset += SVEAllocs.BeforePPRs; assert(PPRRange.End == ZPRRange.Begin && "Expected ZPR callee saves after PPR locals"); allocateStackSpace(PPRRange.End, 0, SVEAllocs.AfterPPRs, EmitAsyncCFI && !HasFP, CFAOffset, MFI.hasVarSizedObjects() || SVEAllocs.AfterZPRs); CFAOffset += SVEAllocs.AfterPPRs; } else { assert(SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord); // Note: With CalleeSavesAboveFrameRecord, the SVE CS (BeforePPRs) have // already been allocated. PPR locals (included in AfterPPRs) are not // supported (note: this is asserted above). CFAOffset += SVEAllocs.BeforePPRs; } // Allocate space for the rest of the frame including ZPR locals. Align the // stack as necessary. assert(!(AFL.canUseRedZone(MF) && NeedsRealignment) && "Cannot use redzone with stack realignment"); if (!AFL.canUseRedZone(MF)) { // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have the // correct value here, as NumBytes also includes padding bytes, which // shouldn't be counted here. allocateStackSpace(AfterSVESavesI, RealignmentPadding, SVEAllocs.AfterZPRs, EmitAsyncCFI && !HasFP, CFAOffset, MFI.hasVarSizedObjects()); } // If we need a base pointer, set it up here. It's whatever the value of the // stack pointer is at this point. Any variable size objects will be // allocated after this, so we can still use the base pointer to reference // locals. // // FIXME: Clarify FrameSetup flags here. // Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is // needed. // For funclets the BP belongs to the containing function. if (!IsFunclet && RegInfo.hasBasePointer(MF)) { TII->copyPhysReg(MBB, AfterSVESavesI, DL, RegInfo.getBaseRegister(), AArch64::SP, false); if (NeedsWinCFI) { HasWinCFI = true; BuildMI(MBB, AfterSVESavesI, DL, TII->get(AArch64::SEH_Nop)) .setMIFlag(MachineInstr::FrameSetup); } } // The very last FrameSetup instruction indicates the end of prologue. Emit a // SEH opcode indicating the prologue end. if (NeedsWinCFI && HasWinCFI) { BuildMI(MBB, AfterSVESavesI, DL, TII->get(AArch64::SEH_PrologEnd)) .setMIFlag(MachineInstr::FrameSetup); } // SEH funclets are passed the frame pointer in X1. If the parent // function uses the base register, then the base register is used // directly, and is not retrieved from X1. if (IsFunclet && F.hasPersonalityFn()) { EHPersonality Per = classifyEHPersonality(F.getPersonalityFn()); if (isAsynchronousEHPersonality(Per)) { BuildMI(MBB, AfterSVESavesI, DL, TII->get(TargetOpcode::COPY), AArch64::FP) .addReg(AArch64::X1) .setMIFlag(MachineInstr::FrameSetup); MBB.addLiveIn(AArch64::X1); } } if (EmitCFI && !EmitAsyncCFI) { if (HasFP) { emitDefineCFAWithFP(AfterSVESavesI, FixedObject); } else { StackOffset TotalSize = AFL.getSVEStackSize(MF) + StackOffset::getFixed((int64_t)MFI.getStackSize()); CFIInstBuilder CFIBuilder(MBB, AfterSVESavesI, MachineInstr::FrameSetup); CFIBuilder.insertCFIInst( createDefCFA(RegInfo, /*FrameReg=*/AArch64::SP, /*Reg=*/AArch64::SP, TotalSize, /*LastAdjustmentWasScalable=*/false)); } emitCalleeSavedGPRLocations(AfterSVESavesI); emitCalleeSavedSVELocations(AfterSVESavesI); } } void AArch64PrologueEmitter::emitShadowCallStackPrologue( MachineBasicBlock::iterator MBBI, const DebugLoc &DL) const { // Shadow call stack prolog: str x30, [x18], #8 BuildMI(MBB, MBBI, DL, TII->get(AArch64::STRXpost)) .addReg(AArch64::X18, RegState::Define) .addReg(AArch64::LR) .addReg(AArch64::X18) .addImm(8) .setMIFlag(MachineInstr::FrameSetup); // This instruction also makes x18 live-in to the entry block. MBB.addLiveIn(AArch64::X18); if (NeedsWinCFI) BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) .setMIFlag(MachineInstr::FrameSetup); if (EmitCFI) { // Emit a CFI instruction that causes 8 to be subtracted from the value of // x18 when unwinding past this frame. static const char CFIInst[] = { dwarf::DW_CFA_val_expression, 18, // register 2, // length static_cast(unsigned(dwarf::DW_OP_breg18)), static_cast(-8) & 0x7f, // addend (sleb128) }; CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup) .buildEscape(StringRef(CFIInst, sizeof(CFIInst))); } } void AArch64PrologueEmitter::emitSwiftAsyncContextFramePointer( MachineBasicBlock::iterator MBBI, const DebugLoc &DL) const { switch (MF.getTarget().Options.SwiftAsyncFramePointer) { case SwiftAsyncFramePointerMode::DeploymentBased: if (Subtarget.swiftAsyncContextIsDynamicallySet()) { // The special symbol below is absolute and has a *value* that can be // combined with the frame pointer to signal an extended frame. BuildMI(MBB, MBBI, DL, TII->get(AArch64::LOADgot), AArch64::X16) .addExternalSymbol("swift_async_extendedFramePointerFlags", AArch64II::MO_GOT); if (NeedsWinCFI) { BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) .setMIFlags(MachineInstr::FrameSetup); HasWinCFI = true; } BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrs), AArch64::FP) .addUse(AArch64::FP) .addUse(AArch64::X16) .addImm(Subtarget.isTargetILP32() ? 32 : 0); if (NeedsWinCFI) { BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) .setMIFlags(MachineInstr::FrameSetup); HasWinCFI = true; } break; } [[fallthrough]]; case SwiftAsyncFramePointerMode::Always: // ORR x29, x29, #0x1000_0000_0000_0000 BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXri), AArch64::FP) .addUse(AArch64::FP) .addImm(0x1100) .setMIFlag(MachineInstr::FrameSetup); if (NeedsWinCFI) { BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) .setMIFlags(MachineInstr::FrameSetup); HasWinCFI = true; } break; case SwiftAsyncFramePointerMode::Never: break; } } void AArch64PrologueEmitter::emitEmptyStackFramePrologue( int64_t NumBytes, MachineBasicBlock::iterator MBBI, const DebugLoc &DL) const { assert(!HasFP && "unexpected function without stack frame but with FP"); assert(!AFL.getSVEStackSize(MF) && "unexpected function without stack frame but with SVE objects"); // All of the stack allocation is for locals. AFI->setLocalStackSize(NumBytes); if (!NumBytes) { if (NeedsWinCFI && HasWinCFI) { BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd)) .setMIFlag(MachineInstr::FrameSetup); } return; } // REDZONE: If the stack size is less than 128 bytes, we don't need // to actually allocate. if (AFL.canUseRedZone(MF)) { AFI->setHasRedZone(true); ++NumRedZoneFunctions; } else { emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP, StackOffset::getFixed(-NumBytes), TII, MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI); if (EmitCFI) { // Label used to tie together the PROLOG_LABEL and the MachineMoves. MCSymbol *FrameLabel = MF.getContext().createTempSymbol(); // Encode the stack size of the leaf function. CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup) .buildDefCFAOffset(NumBytes, FrameLabel); } } if (NeedsWinCFI) { HasWinCFI = true; BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd)) .setMIFlag(MachineInstr::FrameSetup); } } void AArch64PrologueEmitter::emitFramePointerSetup( MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned FixedObject) { int64_t FPOffset = AFI->getCalleeSaveBaseToFrameRecordOffset(); if (CombineSPBump) FPOffset += AFI->getLocalStackSize(); if (AFI->hasSwiftAsyncContext()) { // Before we update the live FP we have to ensure there's a valid (or // null) asynchronous context in its slot just before FP in the frame // record, so store it now. const auto &Attrs = MF.getFunction().getAttributes(); bool HaveInitialContext = Attrs.hasAttrSomewhere(Attribute::SwiftAsync); if (HaveInitialContext) MBB.addLiveIn(AArch64::X22); Register Reg = HaveInitialContext ? AArch64::X22 : AArch64::XZR; BuildMI(MBB, MBBI, DL, TII->get(AArch64::StoreSwiftAsyncContext)) .addUse(Reg) .addUse(AArch64::SP) .addImm(FPOffset - 8) .setMIFlags(MachineInstr::FrameSetup); if (NeedsWinCFI) { // WinCFI and arm64e, where StoreSwiftAsyncContext is expanded // to multiple instructions, should be mutually-exclusive. assert(Subtarget.getTargetTriple().getArchName() != "arm64e"); BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) .setMIFlags(MachineInstr::FrameSetup); HasWinCFI = true; } } if (HomPrologEpilog) { auto Prolog = MBBI; --Prolog; assert(Prolog->getOpcode() == AArch64::HOM_Prolog); Prolog->addOperand(MachineOperand::CreateImm(FPOffset)); } else { // Issue sub fp, sp, FPOffset or // mov fp,sp when FPOffset is zero. // Note: All stores of callee-saved registers are marked as "FrameSetup". // This code marks the instruction(s) that set the FP also. emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP, StackOffset::getFixed(FPOffset), TII, MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI); if (NeedsWinCFI && HasWinCFI) { BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd)) .setMIFlag(MachineInstr::FrameSetup); // After setting up the FP, the rest of the prolog doesn't need to be // included in the SEH unwind info. NeedsWinCFI = false; } } if (EmitAsyncCFI) emitDefineCFAWithFP(MBBI, FixedObject); } // Define the current CFA rule to use the provided FP. void AArch64PrologueEmitter::emitDefineCFAWithFP( MachineBasicBlock::iterator MBBI, unsigned FixedObject) const { const int OffsetToFirstCalleeSaveFromFP = AFI->getCalleeSaveBaseToFrameRecordOffset() - AFI->getCalleeSavedStackSize(); Register FramePtr = RegInfo.getFrameRegister(MF); CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup) .buildDefCFA(FramePtr, FixedObject - OffsetToFirstCalleeSaveFromFP); } void AArch64PrologueEmitter::emitWindowsStackProbe( MachineBasicBlock::iterator MBBI, const DebugLoc &DL, int64_t &NumBytes, int64_t RealignmentPadding) const { if (AFI->getSVECalleeSavedStackSize()) report_fatal_error("SVE callee saves not yet supported with stack probing"); // Find an available register to spill the value of X15 to, if X15 is being // used already for nest. unsigned X15Scratch = AArch64::NoRegister; if (llvm::any_of(MBB.liveins(), [this](const MachineBasicBlock::RegisterMaskPair &LiveIn) { return RegInfo.isSuperOrSubRegisterEq(AArch64::X15, LiveIn.PhysReg); })) { X15Scratch = AFL.findScratchNonCalleeSaveRegister(&MBB, /*HasCall=*/true); assert(X15Scratch != AArch64::NoRegister && (X15Scratch < AArch64::X15 || X15Scratch > AArch64::X17)); #ifndef NDEBUG LiveRegs.removeReg(AArch64::X15); // ignore X15 since we restore it #endif BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrr), X15Scratch) .addReg(AArch64::XZR) .addReg(AArch64::X15, RegState::Undef) .addReg(AArch64::X15, RegState::Implicit) .setMIFlag(MachineInstr::FrameSetup); } uint64_t NumWords = (NumBytes + RealignmentPadding) >> 4; if (NeedsWinCFI) { HasWinCFI = true; // alloc_l can hold at most 256MB, so assume that NumBytes doesn't // exceed this amount. We need to move at most 2^24 - 1 into x15. // This is at most two instructions, MOVZ followed by MOVK. // TODO: Fix to use multiple stack alloc unwind codes for stacks // exceeding 256MB in size. if (NumBytes >= (1 << 28)) report_fatal_error("Stack size cannot exceed 256MB for stack " "unwinding purposes"); uint32_t LowNumWords = NumWords & 0xFFFF; BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVZXi), AArch64::X15) .addImm(LowNumWords) .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0)) .setMIFlag(MachineInstr::FrameSetup); BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) .setMIFlag(MachineInstr::FrameSetup); if ((NumWords & 0xFFFF0000) != 0) { BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVKXi), AArch64::X15) .addReg(AArch64::X15) .addImm((NumWords & 0xFFFF0000) >> 16) // High half .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 16)) .setMIFlag(MachineInstr::FrameSetup); BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) .setMIFlag(MachineInstr::FrameSetup); } } else { BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm), AArch64::X15) .addImm(NumWords) .setMIFlags(MachineInstr::FrameSetup); } const char *ChkStk = Subtarget.getChkStkName(); switch (MF.getTarget().getCodeModel()) { case CodeModel::Tiny: case CodeModel::Small: case CodeModel::Medium: case CodeModel::Kernel: BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL)) .addExternalSymbol(ChkStk) .addReg(AArch64::X15, RegState::Implicit) .addReg(AArch64::X16, RegState::Implicit | RegState::Define | RegState::Dead) .addReg(AArch64::X17, RegState::Implicit | RegState::Define | RegState::Dead) .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define | RegState::Dead) .setMIFlags(MachineInstr::FrameSetup); if (NeedsWinCFI) { HasWinCFI = true; BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) .setMIFlag(MachineInstr::FrameSetup); } break; case CodeModel::Large: BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVaddrEXT)) .addReg(AArch64::X16, RegState::Define) .addExternalSymbol(ChkStk) .addExternalSymbol(ChkStk) .setMIFlags(MachineInstr::FrameSetup); if (NeedsWinCFI) { HasWinCFI = true; BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) .setMIFlag(MachineInstr::FrameSetup); } BuildMI(MBB, MBBI, DL, TII->get(getBLRCallOpcode(MF))) .addReg(AArch64::X16, RegState::Kill) .addReg(AArch64::X15, RegState::Implicit | RegState::Define) .addReg(AArch64::X16, RegState::Implicit | RegState::Define | RegState::Dead) .addReg(AArch64::X17, RegState::Implicit | RegState::Define | RegState::Dead) .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define | RegState::Dead) .setMIFlags(MachineInstr::FrameSetup); if (NeedsWinCFI) { HasWinCFI = true; BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) .setMIFlag(MachineInstr::FrameSetup); } break; } BuildMI(MBB, MBBI, DL, TII->get(AArch64::SUBXrx64), AArch64::SP) .addReg(AArch64::SP, RegState::Kill) .addReg(AArch64::X15, RegState::Kill) .addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 4)) .setMIFlags(MachineInstr::FrameSetup); if (NeedsWinCFI) { HasWinCFI = true; BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc)) .addImm(NumBytes) .setMIFlag(MachineInstr::FrameSetup); } NumBytes = 0; if (RealignmentPadding > 0) { if (RealignmentPadding >= 4096) { BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm)) .addReg(AArch64::X16, RegState::Define) .addImm(RealignmentPadding) .setMIFlags(MachineInstr::FrameSetup); BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDXrx64), AArch64::X15) .addReg(AArch64::SP) .addReg(AArch64::X16, RegState::Kill) .addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 0)) .setMIFlag(MachineInstr::FrameSetup); } else { BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDXri), AArch64::X15) .addReg(AArch64::SP) .addImm(RealignmentPadding) .addImm(0) .setMIFlag(MachineInstr::FrameSetup); } uint64_t AndMask = ~(MFI.getMaxAlign().value() - 1); BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP) .addReg(AArch64::X15, RegState::Kill) .addImm(AArch64_AM::encodeLogicalImmediate(AndMask, 64)); AFI->setStackRealigned(true); // No need for SEH instructions here; if we're realigning the stack, // we've set a frame pointer and already finished the SEH prologue. assert(!NeedsWinCFI); } if (X15Scratch != AArch64::NoRegister) { BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrr), AArch64::X15) .addReg(AArch64::XZR) .addReg(X15Scratch, RegState::Undef) .addReg(X15Scratch, RegState::Implicit) .setMIFlag(MachineInstr::FrameSetup); } } void AArch64PrologueEmitter::emitCalleeSavedGPRLocations( MachineBasicBlock::iterator MBBI) const { const std::vector &CSI = MFI.getCalleeSavedInfo(); if (CSI.empty()) return; CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup); for (const auto &Info : CSI) { unsigned FrameIdx = Info.getFrameIdx(); if (MFI.hasScalableStackID(FrameIdx)) continue; assert(!Info.isSpilledToReg() && "Spilling to registers not implemented"); int64_t Offset = MFI.getObjectOffset(FrameIdx) - AFL.getOffsetOfLocalArea(); CFIBuilder.buildOffset(Info.getReg(), Offset); } } void AArch64PrologueEmitter::emitCalleeSavedSVELocations( MachineBasicBlock::iterator MBBI) const { // Add callee saved registers to move list. const std::vector &CSI = MFI.getCalleeSavedInfo(); if (CSI.empty()) return; CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup); std::optional IncomingVGOffsetFromDefCFA; if (AFL.requiresSaveVG(MF)) { auto IncomingVG = *find_if( reverse(CSI), [](auto &Info) { return Info.getReg() == AArch64::VG; }); IncomingVGOffsetFromDefCFA = MFI.getObjectOffset(IncomingVG.getFrameIdx()) - AFL.getOffsetOfLocalArea(); } StackOffset PPRStackSize = AFL.getPPRStackSize(MF); for (const auto &Info : CSI) { int FI = Info.getFrameIdx(); if (!MFI.hasScalableStackID(FI)) continue; // Not all unwinders may know about SVE registers, so assume the lowest // common denominator. assert(!Info.isSpilledToReg() && "Spilling to registers not implemented"); MCRegister Reg = Info.getReg(); if (!RegInfo.regNeedsCFI(Reg, Reg)) continue; StackOffset Offset = StackOffset::getScalable(MFI.getObjectOffset(FI)) - StackOffset::getFixed(AFI->getCalleeSavedStackSize(MFI)); // The scalable vectors are below (lower address) the scalable predicates // with split SVE objects, so we must subtract the size of the predicates. if (SVELayout == SVEStackLayout::Split && MFI.getStackID(FI) == TargetStackID::ScalableVector) Offset -= PPRStackSize; CFIBuilder.insertCFIInst( createCFAOffset(RegInfo, Reg, Offset, IncomingVGOffsetFromDefCFA)); } } static bool isFuncletReturnInstr(const MachineInstr &MI) { switch (MI.getOpcode()) { default: return false; case AArch64::CATCHRET: case AArch64::CLEANUPRET: return true; } } AArch64EpilogueEmitter::AArch64EpilogueEmitter(MachineFunction &MF, MachineBasicBlock &MBB, const AArch64FrameLowering &AFL) : AArch64PrologueEpilogueCommon(MF, MBB, AFL) { EmitCFI = AFI->needsAsyncDwarfUnwindInfo(MF); HomPrologEpilog = AFL.homogeneousPrologEpilog(MF, &MBB); SEHEpilogueStartI = MBB.end(); } void AArch64EpilogueEmitter::moveSPBelowFP(MachineBasicBlock::iterator MBBI, StackOffset Offset) { // Other combinations could be supported, but are not currently needed. assert(Offset.getScalable() < 0 && Offset.getFixed() <= 0 && "expected negative offset (with optional fixed portion)"); Register Base = AArch64::FP; if (int64_t FixedOffset = Offset.getFixed()) { // If we have a negative fixed offset, we need to first subtract it in a // temporary register first (to avoid briefly deallocating the scalable // portion of the offset). Base = MF.getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass); emitFrameOffset(MBB, MBBI, DL, Base, AArch64::FP, StackOffset::getFixed(FixedOffset), TII, MachineInstr::FrameDestroy); } emitFrameOffset(MBB, MBBI, DL, AArch64::SP, Base, StackOffset::getScalable(Offset.getScalable()), TII, MachineInstr::FrameDestroy); } void AArch64EpilogueEmitter::emitEpilogue() { MachineBasicBlock::iterator EpilogueEndI = MBB.getLastNonDebugInstr(); if (MBB.end() != EpilogueEndI) { DL = EpilogueEndI->getDebugLoc(); IsFunclet = isFuncletReturnInstr(*EpilogueEndI); } int64_t NumBytes = IsFunclet ? AFL.getWinEHFuncletFrameSize(MF) : MFI.getStackSize(); // All calls are tail calls in GHC calling conv, and functions have no // prologue/epilogue. if (MF.getFunction().getCallingConv() == CallingConv::GHC) return; // How much of the stack used by incoming arguments this function is expected // to restore in this particular epilogue. int64_t ArgumentStackToRestore = AFL.getArgumentStackToRestore(MF, MBB); bool IsWin64 = Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv(), MF.getFunction().isVarArg()); unsigned FixedObject = AFL.getFixedObjectSize(MF, AFI, IsWin64, IsFunclet); int64_t AfterCSRPopSize = ArgumentStackToRestore; auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject; // We cannot rely on the local stack size set in emitPrologue if the function // has funclets, as funclets have different local stack size requirements, and // the current value set in emitPrologue may be that of the containing // function. if (MF.hasEHFunclets()) AFI->setLocalStackSize(NumBytes - PrologueSaveSize); if (HomPrologEpilog) { assert(!NeedsWinCFI); auto FirstHomogenousEpilogI = MBB.getFirstTerminator(); if (FirstHomogenousEpilogI != MBB.begin()) { auto HomogeneousEpilog = std::prev(FirstHomogenousEpilogI); if (HomogeneousEpilog->getOpcode() == AArch64::HOM_Epilog) FirstHomogenousEpilogI = HomogeneousEpilog; } // Adjust local stack emitFrameOffset(MBB, FirstHomogenousEpilogI, DL, AArch64::SP, AArch64::SP, StackOffset::getFixed(AFI->getLocalStackSize()), TII, MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI); // SP has been already adjusted while restoring callee save regs. // We've bailed-out the case with adjusting SP for arguments. assert(AfterCSRPopSize == 0); return; } bool CombineSPBump = shouldCombineCSRLocalStackBump(NumBytes); unsigned ProloguePopSize = PrologueSaveSize; if (SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord) { // With CalleeSavesAboveFrameRecord ProloguePopSize is the amount of stack // that needs to be popped until we reach the start of the SVE save area. // The "FixedObject" stack occurs after the SVE area and must be popped // later. ProloguePopSize -= FixedObject; AfterCSRPopSize += FixedObject; } // Assume we can't combine the last pop with the sp restore. if (!CombineSPBump && ProloguePopSize != 0) { MachineBasicBlock::iterator Pop = std::prev(MBB.getFirstTerminator()); while (Pop->getOpcode() == TargetOpcode::CFI_INSTRUCTION || AArch64InstrInfo::isSEHInstruction(*Pop) || (SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord && isPartOfSVECalleeSaves(Pop))) Pop = std::prev(Pop); // Converting the last ldp to a post-index ldp is valid only if the last // ldp's offset is 0. const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1); // If the offset is 0 and the AfterCSR pop is not actually trying to // allocate more stack for arguments (in space that an untimely interrupt // may clobber), convert it to a post-index ldp. if (OffsetOp.getImm() == 0 && AfterCSRPopSize >= 0) { convertCalleeSaveRestoreToSPPrePostIncDec( Pop, DL, ProloguePopSize, EmitCFI, MachineInstr::FrameDestroy, ProloguePopSize); } else if (SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord) { MachineBasicBlock::iterator AfterLastPop = std::next(Pop); if (AArch64InstrInfo::isSEHInstruction(*AfterLastPop)) ++AfterLastPop; // If not, and CalleeSavesAboveFrameRecord is enabled, deallocate // callee-save non-SVE registers to move the stack pointer to the start of // the SVE area. emitFrameOffset(MBB, AfterLastPop, DL, AArch64::SP, AArch64::SP, StackOffset::getFixed(ProloguePopSize), TII, MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI); } else { // Otherwise, make sure to emit an add after the last ldp. // We're doing this by transferring the size to be restored from the // adjustment *before* the CSR pops to the adjustment *after* the CSR // pops. AfterCSRPopSize += ProloguePopSize; } } // Move past the restores of the callee-saved registers. // If we plan on combining the sp bump of the local stack size and the callee // save stack size, we might need to adjust the CSR save and restore offsets. MachineBasicBlock::iterator FirstGPRRestoreI = MBB.getFirstTerminator(); MachineBasicBlock::iterator Begin = MBB.begin(); while (FirstGPRRestoreI != Begin) { --FirstGPRRestoreI; if (!FirstGPRRestoreI->getFlag(MachineInstr::FrameDestroy) || (SVELayout != SVEStackLayout::CalleeSavesAboveFrameRecord && isPartOfSVECalleeSaves(FirstGPRRestoreI))) { ++FirstGPRRestoreI; break; } else if (CombineSPBump) fixupCalleeSaveRestoreStackOffset(*FirstGPRRestoreI, AFI->getLocalStackSize()); } if (NeedsWinCFI) { // Note that there are cases where we insert SEH opcodes in the // epilogue when we had no SEH opcodes in the prologue. For // example, when there is no stack frame but there are stack // arguments. Insert the SEH_EpilogStart and remove it later if it // we didn't emit any SEH opcodes to avoid generating WinCFI for // functions that don't need it. BuildMI(MBB, FirstGPRRestoreI, DL, TII->get(AArch64::SEH_EpilogStart)) .setMIFlag(MachineInstr::FrameDestroy); SEHEpilogueStartI = FirstGPRRestoreI; --SEHEpilogueStartI; } // Determine the ranges of SVE callee-saves. This is done before emitting any // code at the end of the epilogue (for Swift async), which can get in the way // of finding SVE callee-saves with CalleeSavesAboveFrameRecord. auto [PPR, ZPR] = getSVEStackFrameSizes(); auto [PPRRange, ZPRRange] = partitionSVECS( MBB, SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord ? MBB.getFirstTerminator() : FirstGPRRestoreI, PPR.CalleeSavesSize, ZPR.CalleeSavesSize, /*IsEpilogue=*/true); if (HasFP && AFI->hasSwiftAsyncContext()) emitSwiftAsyncContextFramePointer(EpilogueEndI, DL); // If there is a single SP update, insert it before the ret and we're done. if (CombineSPBump) { assert(!AFI->hasSVEStackSize() && "Cannot combine SP bump with SVE"); // When we are about to restore the CSRs, the CFA register is SP again. if (EmitCFI && HasFP) CFIInstBuilder(MBB, FirstGPRRestoreI, MachineInstr::FrameDestroy) .buildDefCFA(AArch64::SP, NumBytes); emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP, StackOffset::getFixed(NumBytes + AfterCSRPopSize), TII, MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI, EmitCFI, StackOffset::getFixed(NumBytes)); return; } NumBytes -= PrologueSaveSize; assert(NumBytes >= 0 && "Negative stack allocation size!?"); StackOffset SVECalleeSavesSize = ZPR.CalleeSavesSize + PPR.CalleeSavesSize; SVEStackAllocations SVEAllocs = getSVEStackAllocations({PPR, ZPR}); // Deallocate the SVE area. if (SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord) { assert(!SVEAllocs.AfterPPRs && "unexpected SVE allocs after PPRs with CalleeSavesAboveFrameRecord"); // If the callee-save area is before FP, restoring the FP implicitly // deallocates non-callee-save SVE allocations. Otherwise, deallocate them // explicitly. if (!AFI->isStackRealigned() && !MFI.hasVarSizedObjects()) { emitFrameOffset(MBB, FirstGPRRestoreI, DL, AArch64::SP, AArch64::SP, SVEAllocs.AfterZPRs, TII, MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI); } // Deallocate callee-save SVE registers. emitFrameOffset(MBB, PPRRange.End, DL, AArch64::SP, AArch64::SP, SVEAllocs.BeforePPRs, TII, MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI); } else if (AFI->hasSVEStackSize()) { // If we have stack realignment or variable-sized objects we must use the FP // to restore SVE callee saves (as there is an unknown amount of // data/padding between the SP and SVE CS area). Register BaseForSVEDealloc = (AFI->isStackRealigned() || MFI.hasVarSizedObjects()) ? AArch64::FP : AArch64::SP; if (SVECalleeSavesSize && BaseForSVEDealloc == AArch64::FP) { if (ZPR.CalleeSavesSize || SVELayout != SVEStackLayout::Split) { // The offset from the frame-pointer to the start of the ZPR saves. StackOffset FPOffsetZPR = -SVECalleeSavesSize - PPR.LocalsSize - StackOffset::getFixed(AFI->getCalleeSaveBaseToFrameRecordOffset()); // Deallocate the stack space space by moving the SP to the start of the // ZPR/PPR callee-save area. moveSPBelowFP(ZPRRange.Begin, FPOffsetZPR); } // With split SVE, the predicates are stored in a separate area above the // ZPR saves, so we must adjust the stack to the start of the PPRs. if (PPR.CalleeSavesSize && SVELayout == SVEStackLayout::Split) { // The offset from the frame-pointer to the start of the PPR saves. StackOffset FPOffsetPPR = -PPR.CalleeSavesSize; // Move to the start of the PPR area. assert(!FPOffsetPPR.getFixed() && "expected only scalable offset"); emitFrameOffset(MBB, ZPRRange.End, DL, AArch64::SP, AArch64::FP, FPOffsetPPR, TII, MachineInstr::FrameDestroy); } } else if (BaseForSVEDealloc == AArch64::SP) { auto NonSVELocals = StackOffset::getFixed(NumBytes); auto CFAOffset = NonSVELocals + StackOffset::getFixed(PrologueSaveSize) + SVEAllocs.totalSize(); if (SVECalleeSavesSize || SVELayout == SVEStackLayout::Split) { // Deallocate non-SVE locals now. This is needed to reach the SVE callee // saves, but may also allow combining stack hazard bumps for split SVE. SVEAllocs.AfterZPRs += NonSVELocals; NumBytes -= NonSVELocals.getFixed(); } // To deallocate the SVE stack adjust by the allocations in reverse. emitFrameOffset(MBB, ZPRRange.Begin, DL, AArch64::SP, AArch64::SP, SVEAllocs.AfterZPRs, TII, MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI, EmitCFI && !HasFP, CFAOffset); CFAOffset -= SVEAllocs.AfterZPRs; assert(PPRRange.Begin == ZPRRange.End && "Expected PPR restores after ZPR"); emitFrameOffset(MBB, PPRRange.Begin, DL, AArch64::SP, AArch64::SP, SVEAllocs.AfterPPRs, TII, MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI, EmitCFI && !HasFP, CFAOffset); CFAOffset -= SVEAllocs.AfterPPRs; emitFrameOffset(MBB, PPRRange.End, DL, AArch64::SP, AArch64::SP, SVEAllocs.BeforePPRs, TII, MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI, EmitCFI && !HasFP, CFAOffset); } if (EmitCFI) emitCalleeSavedSVERestores( SVELayout == SVEStackLayout::Split ? ZPRRange.End : PPRRange.End); } if (!HasFP) { bool RedZone = AFL.canUseRedZone(MF); // If this was a redzone leaf function, we don't need to restore the // stack pointer (but we may need to pop stack args for fastcc). if (RedZone && AfterCSRPopSize == 0) return; // Pop the local variables off the stack. If there are no callee-saved // registers, it means we are actually positioned at the terminator and can // combine stack increment for the locals and the stack increment for // callee-popped arguments into (possibly) a single instruction and be done. bool NoCalleeSaveRestore = PrologueSaveSize == 0; int64_t StackRestoreBytes = RedZone ? 0 : NumBytes; if (NoCalleeSaveRestore) StackRestoreBytes += AfterCSRPopSize; emitFrameOffset( MBB, FirstGPRRestoreI, DL, AArch64::SP, AArch64::SP, StackOffset::getFixed(StackRestoreBytes), TII, MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI, EmitCFI, StackOffset::getFixed((RedZone ? 0 : NumBytes) + PrologueSaveSize)); // If we were able to combine the local stack pop with the argument pop, // then we're done. if (NoCalleeSaveRestore || AfterCSRPopSize == 0) return; NumBytes = 0; } // Restore the original stack pointer. // FIXME: Rather than doing the math here, we should instead just use // non-post-indexed loads for the restores if we aren't actually going to // be able to save any instructions. if (!IsFunclet && (MFI.hasVarSizedObjects() || AFI->isStackRealigned())) { emitFrameOffset( MBB, FirstGPRRestoreI, DL, AArch64::SP, AArch64::FP, StackOffset::getFixed(-AFI->getCalleeSaveBaseToFrameRecordOffset()), TII, MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI); } else if (NumBytes) emitFrameOffset(MBB, FirstGPRRestoreI, DL, AArch64::SP, AArch64::SP, StackOffset::getFixed(NumBytes), TII, MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI); // When we are about to restore the CSRs, the CFA register is SP again. if (EmitCFI && HasFP) CFIInstBuilder(MBB, FirstGPRRestoreI, MachineInstr::FrameDestroy) .buildDefCFA(AArch64::SP, PrologueSaveSize); // This must be placed after the callee-save restore code because that code // assumes the SP is at the same location as it was after the callee-save save // code in the prologue. if (AfterCSRPopSize) { assert(AfterCSRPopSize > 0 && "attempting to reallocate arg stack that an " "interrupt may have clobbered"); emitFrameOffset( MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP, StackOffset::getFixed(AfterCSRPopSize), TII, MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI, EmitCFI, StackOffset::getFixed(AfterCSRPopSize - ArgumentStackToRestore)); } } bool AArch64EpilogueEmitter::shouldCombineCSRLocalStackBump( uint64_t StackBumpBytes) const { if (!AArch64PrologueEpilogueCommon::shouldCombineCSRLocalStackBump( StackBumpBytes)) return false; if (MBB.empty()) return true; // Disable combined SP bump if the last instruction is an MTE tag store. It // is almost always better to merge SP adjustment into those instructions. MachineBasicBlock::iterator LastI = MBB.getFirstTerminator(); MachineBasicBlock::iterator Begin = MBB.begin(); while (LastI != Begin) { --LastI; if (LastI->isTransient()) continue; if (!LastI->getFlag(MachineInstr::FrameDestroy)) break; } switch (LastI->getOpcode()) { case AArch64::STGloop: case AArch64::STZGloop: case AArch64::STGi: case AArch64::STZGi: case AArch64::ST2Gi: case AArch64::STZ2Gi: return false; default: return true; } llvm_unreachable("unreachable"); } void AArch64EpilogueEmitter::emitSwiftAsyncContextFramePointer( MachineBasicBlock::iterator MBBI, const DebugLoc &DL) const { switch (MF.getTarget().Options.SwiftAsyncFramePointer) { case SwiftAsyncFramePointerMode::DeploymentBased: // Avoid the reload as it is GOT relative, and instead fall back to the // hardcoded value below. This allows a mismatch between the OS and // application without immediately terminating on the difference. [[fallthrough]]; case SwiftAsyncFramePointerMode::Always: // We need to reset FP to its untagged state on return. Bit 60 is // currently used to show the presence of an extended frame. // BIC x29, x29, #0x1000_0000_0000_0000 BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::ANDXri), AArch64::FP) .addUse(AArch64::FP) .addImm(0x10fe) .setMIFlag(MachineInstr::FrameDestroy); if (NeedsWinCFI) { BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) .setMIFlags(MachineInstr::FrameDestroy); HasWinCFI = true; } break; case SwiftAsyncFramePointerMode::Never: break; } } void AArch64EpilogueEmitter::emitShadowCallStackEpilogue( MachineBasicBlock::iterator MBBI, const DebugLoc &DL) const { // Shadow call stack epilog: ldr x30, [x18, #-8]! BuildMI(MBB, MBBI, DL, TII->get(AArch64::LDRXpre)) .addReg(AArch64::X18, RegState::Define) .addReg(AArch64::LR, RegState::Define) .addReg(AArch64::X18) .addImm(-8) .setMIFlag(MachineInstr::FrameDestroy); if (NeedsWinCFI) BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop)) .setMIFlag(MachineInstr::FrameDestroy); if (AFI->needsAsyncDwarfUnwindInfo(MF)) CFIInstBuilder(MBB, MBBI, MachineInstr::FrameDestroy) .buildRestore(AArch64::X18); } void AArch64EpilogueEmitter::emitCalleeSavedRestores( MachineBasicBlock::iterator MBBI, bool SVE) const { const std::vector &CSI = MFI.getCalleeSavedInfo(); if (CSI.empty()) return; CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameDestroy); for (const auto &Info : CSI) { if (SVE != MFI.hasScalableStackID(Info.getFrameIdx())) continue; MCRegister Reg = Info.getReg(); if (SVE && !RegInfo.regNeedsCFI(Reg, Reg)) continue; CFIBuilder.buildRestore(Info.getReg()); } } void AArch64EpilogueEmitter::finalizeEpilogue() const { if (AFI->needsShadowCallStackPrologueEpilogue(MF)) { emitShadowCallStackEpilogue(MBB.getFirstTerminator(), DL); HasWinCFI |= NeedsWinCFI; } if (EmitCFI) emitCalleeSavedGPRRestores(MBB.getFirstTerminator()); if (AFI->shouldSignReturnAddress(MF)) { // If pac-ret+leaf is in effect, PAUTH_EPILOGUE pseudo instructions // are inserted by emitPacRetPlusLeafHardening(). if (!AFL.shouldSignReturnAddressEverywhere(MF)) { BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::PAUTH_EPILOGUE)) .setMIFlag(MachineInstr::FrameDestroy); } // AArch64PointerAuth pass will insert SEH_PACSignLR HasWinCFI |= NeedsWinCFI; } if (HasWinCFI) { BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::SEH_EpilogEnd)) .setMIFlag(MachineInstr::FrameDestroy); if (!MF.hasWinCFI()) MF.setHasWinCFI(true); } if (NeedsWinCFI) { assert(SEHEpilogueStartI != MBB.end()); if (!HasWinCFI) MBB.erase(SEHEpilogueStartI); } } } // namespace llvm