2844 lines · cpp
1//===- lib/MC/MCWin64EH.cpp - MCWin64EH implementation --------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "llvm/MC/MCWin64EH.h"10#include "llvm/ADT/Twine.h"11#include "llvm/MC/MCAssembler.h"12#include "llvm/MC/MCContext.h"13#include "llvm/MC/MCExpr.h"14#include "llvm/MC/MCObjectStreamer.h"15#include "llvm/MC/MCStreamer.h"16#include "llvm/MC/MCSymbol.h"17#include "llvm/MC/MCValue.h"18#include "llvm/Support/Win64EH.h"19 20namespace llvm {21class MCSection;22}23 24using namespace llvm;25 26namespace {27/// MCExpr that represents the epilog unwind code in an unwind table.28class MCUnwindV2EpilogTargetExpr final : public MCTargetExpr {29 const MCSymbol *Function;30 const MCSymbol *FunctionEnd;31 const MCSymbol *UnwindV2Start;32 const MCSymbol *EpilogEnd;33 uint8_t EpilogSize;34 SMLoc Loc;35 36 MCUnwindV2EpilogTargetExpr(const WinEH::FrameInfo &FrameInfo,37 const WinEH::FrameInfo::Epilog &Epilog,38 uint8_t EpilogSize_)39 : Function(FrameInfo.Function), FunctionEnd(FrameInfo.FuncletOrFuncEnd),40 UnwindV2Start(Epilog.UnwindV2Start), EpilogEnd(Epilog.End),41 EpilogSize(EpilogSize_), Loc(Epilog.Loc) {}42 43public:44 static MCUnwindV2EpilogTargetExpr *45 create(const WinEH::FrameInfo &FrameInfo,46 const WinEH::FrameInfo::Epilog &Epilog, uint8_t EpilogSize_,47 MCContext &Ctx) {48 return new (Ctx) MCUnwindV2EpilogTargetExpr(FrameInfo, Epilog, EpilogSize_);49 }50 51 void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override {52 OS << ":epilog:";53 UnwindV2Start->print(OS, MAI);54 }55 56 bool evaluateAsRelocatableImpl(MCValue &Res,57 const MCAssembler *Asm) const override;58 59 void visitUsedExpr(MCStreamer &Streamer) const override {60 // Contains no sub-expressions.61 }62 63 MCFragment *findAssociatedFragment() const override {64 return UnwindV2Start->getFragment();65 }66};67} // namespace68 69// NOTE: All relocations generated here are 4-byte image-relative.70 71static uint8_t CountOfUnwindCodes(std::vector<WinEH::Instruction> &Insns) {72 uint8_t Count = 0;73 for (const auto &I : Insns) {74 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) {75 default:76 llvm_unreachable("Unsupported unwind code");77 case Win64EH::UOP_PushNonVol:78 case Win64EH::UOP_AllocSmall:79 case Win64EH::UOP_SetFPReg:80 case Win64EH::UOP_PushMachFrame:81 Count += 1;82 break;83 case Win64EH::UOP_SaveNonVol:84 case Win64EH::UOP_SaveXMM128:85 Count += 2;86 break;87 case Win64EH::UOP_SaveNonVolBig:88 case Win64EH::UOP_SaveXMM128Big:89 Count += 3;90 break;91 case Win64EH::UOP_AllocLarge:92 Count += (I.Offset > 512 * 1024 - 8) ? 3 : 2;93 break;94 }95 }96 return Count;97}98 99static void EmitAbsDifference(MCStreamer &Streamer, const MCSymbol *LHS,100 const MCSymbol *RHS) {101 MCContext &Context = Streamer.getContext();102 const MCExpr *Diff =103 MCBinaryExpr::createSub(MCSymbolRefExpr::create(LHS, Context),104 MCSymbolRefExpr::create(RHS, Context), Context);105 Streamer.emitValue(Diff, 1);106}107 108static void EmitUnwindCode(MCStreamer &streamer, const MCSymbol *begin,109 WinEH::Instruction &inst) {110 uint8_t b2;111 uint16_t w;112 b2 = (inst.Operation & 0x0F);113 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) {114 default:115 llvm_unreachable("Unsupported unwind code");116 case Win64EH::UOP_PushNonVol:117 EmitAbsDifference(streamer, inst.Label, begin);118 b2 |= (inst.Register & 0x0F) << 4;119 streamer.emitInt8(b2);120 break;121 case Win64EH::UOP_AllocLarge:122 EmitAbsDifference(streamer, inst.Label, begin);123 if (inst.Offset > 512 * 1024 - 8) {124 b2 |= 0x10;125 streamer.emitInt8(b2);126 w = inst.Offset & 0xFFF8;127 streamer.emitInt16(w);128 w = inst.Offset >> 16;129 } else {130 streamer.emitInt8(b2);131 w = inst.Offset >> 3;132 }133 streamer.emitInt16(w);134 break;135 case Win64EH::UOP_AllocSmall:136 b2 |= (((inst.Offset - 8) >> 3) & 0x0F) << 4;137 EmitAbsDifference(streamer, inst.Label, begin);138 streamer.emitInt8(b2);139 break;140 case Win64EH::UOP_SetFPReg:141 EmitAbsDifference(streamer, inst.Label, begin);142 streamer.emitInt8(b2);143 break;144 case Win64EH::UOP_SaveNonVol:145 case Win64EH::UOP_SaveXMM128:146 b2 |= (inst.Register & 0x0F) << 4;147 EmitAbsDifference(streamer, inst.Label, begin);148 streamer.emitInt8(b2);149 w = inst.Offset >> 3;150 if (inst.Operation == Win64EH::UOP_SaveXMM128)151 w >>= 1;152 streamer.emitInt16(w);153 break;154 case Win64EH::UOP_SaveNonVolBig:155 case Win64EH::UOP_SaveXMM128Big:156 b2 |= (inst.Register & 0x0F) << 4;157 EmitAbsDifference(streamer, inst.Label, begin);158 streamer.emitInt8(b2);159 if (inst.Operation == Win64EH::UOP_SaveXMM128Big)160 w = inst.Offset & 0xFFF0;161 else162 w = inst.Offset & 0xFFF8;163 streamer.emitInt16(w);164 w = inst.Offset >> 16;165 streamer.emitInt16(w);166 break;167 case Win64EH::UOP_PushMachFrame:168 if (inst.Offset == 1)169 b2 |= 0x10;170 EmitAbsDifference(streamer, inst.Label, begin);171 streamer.emitInt8(b2);172 break;173 }174}175 176static void EmitSymbolRefWithOfs(MCStreamer &streamer,177 const MCSymbol *Base,178 int64_t Offset) {179 MCContext &Context = streamer.getContext();180 const MCConstantExpr *OffExpr = MCConstantExpr::create(Offset, Context);181 const MCSymbolRefExpr *BaseRefRel = MCSymbolRefExpr::create(Base,182 MCSymbolRefExpr::VK_COFF_IMGREL32,183 Context);184 streamer.emitValue(MCBinaryExpr::createAdd(BaseRefRel, OffExpr, Context), 4);185}186 187static void EmitSymbolRefWithOfs(MCStreamer &streamer,188 const MCSymbol *Base,189 const MCSymbol *Other) {190 MCContext &Context = streamer.getContext();191 const MCSymbolRefExpr *BaseRef = MCSymbolRefExpr::create(Base, Context);192 const MCSymbolRefExpr *OtherRef = MCSymbolRefExpr::create(Other, Context);193 const MCExpr *Ofs = MCBinaryExpr::createSub(OtherRef, BaseRef, Context);194 const MCSymbolRefExpr *BaseRefRel = MCSymbolRefExpr::create(Base,195 MCSymbolRefExpr::VK_COFF_IMGREL32,196 Context);197 streamer.emitValue(MCBinaryExpr::createAdd(BaseRefRel, Ofs, Context), 4);198}199 200static void EmitRuntimeFunction(MCStreamer &streamer,201 const WinEH::FrameInfo *info) {202 MCContext &context = streamer.getContext();203 204 streamer.emitValueToAlignment(Align(4));205 EmitSymbolRefWithOfs(streamer, info->Begin, info->Begin);206 EmitSymbolRefWithOfs(streamer, info->Begin, info->End);207 streamer.emitValue(MCSymbolRefExpr::create(info->Symbol,208 MCSymbolRefExpr::VK_COFF_IMGREL32,209 context), 4);210}211 212static std::optional<int64_t>213GetOptionalAbsDifference(const MCAssembler &Assembler, const MCSymbol *LHS,214 const MCSymbol *RHS) {215 MCContext &Context = Assembler.getContext();216 const MCExpr *Diff =217 MCBinaryExpr::createSub(MCSymbolRefExpr::create(LHS, Context),218 MCSymbolRefExpr::create(RHS, Context), Context);219 // It should normally be possible to calculate the length of a function220 // at this point, but it might not be possible in the presence of certain221 // unusual constructs, like an inline asm with an alignment directive.222 int64_t value;223 if (!Diff->evaluateAsAbsolute(value, Assembler))224 return std::nullopt;225 return value;226}227 228static void EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info) {229 // If this UNWIND_INFO already has a symbol, it's already been emitted.230 if (info->Symbol)231 return;232 233 MCContext &context = streamer.getContext();234 MCObjectStreamer *OS = (MCObjectStreamer *)(&streamer);235 MCSymbol *Label = context.createTempSymbol();236 237 streamer.emitValueToAlignment(Align(4));238 streamer.emitLabel(Label);239 info->Symbol = Label;240 241 uint8_t numCodes = CountOfUnwindCodes(info->Instructions);242 bool LastEpilogIsAtEnd = false;243 bool AddPaddingEpilogCode = false;244 uint8_t EpilogSize = 0;245 bool EnableUnwindV2 = (info->Version >= 2) && !info->EpilogMap.empty();246 if (EnableUnwindV2) {247 auto &LastEpilog = info->EpilogMap.back().second;248 249 // Calculate the size of the epilogs. Note that we +1 to the size so that250 // the terminator instruction is also included in the epilog (the Windows251 // unwinder does a simple range check versus the current instruction pointer252 // so, although there are terminators that are large than 1 byte, the253 // starting address of the terminator instruction will always be considered254 // inside the epilog).255 auto MaybeSize = GetOptionalAbsDifference(256 OS->getAssembler(), LastEpilog.End, LastEpilog.UnwindV2Start);257 if (!MaybeSize) {258 context.reportError(LastEpilog.Loc,259 "Failed to evaluate epilog size for Unwind v2 in " +260 info->Function->getName());261 return;262 }263 assert(*MaybeSize >= 0);264 if (*MaybeSize >= (int64_t)UINT8_MAX) {265 context.reportError(LastEpilog.Loc,266 "Epilog size is too large for Unwind v2 in " +267 info->Function->getName());268 return;269 }270 EpilogSize = *MaybeSize + 1;271 272 // If the last epilog is at the end of the function, we can use a special273 // encoding for it. Because of our +1 trick for the size, this will only274 // work where that final terminator instruction is 1 byte long.275 auto LastEpilogToFuncEnd = GetOptionalAbsDifference(276 OS->getAssembler(), info->FuncletOrFuncEnd, LastEpilog.UnwindV2Start);277 LastEpilogIsAtEnd = (LastEpilogToFuncEnd == EpilogSize);278 279 // If we have an odd number of epilog codes, we need to add a padding code.280 size_t numEpilogCodes =281 info->EpilogMap.size() + (LastEpilogIsAtEnd ? 0 : 1);282 if ((numEpilogCodes % 2) != 0) {283 AddPaddingEpilogCode = true;284 numEpilogCodes++;285 }286 287 // Too many epilogs to handle.288 if ((size_t)numCodes + numEpilogCodes > UINT8_MAX) {289 context.reportError(info->FunctionLoc,290 "Too many unwind codes with Unwind v2 enabled in " +291 info->Function->getName());292 return;293 }294 295 numCodes += numEpilogCodes;296 }297 298 // Upper 3 bits are the version number.299 uint8_t flags = info->Version;300 if (info->ChainedParent)301 flags |= Win64EH::UNW_ChainInfo << 3;302 else {303 if (info->HandlesUnwind)304 flags |= Win64EH::UNW_TerminateHandler << 3;305 if (info->HandlesExceptions)306 flags |= Win64EH::UNW_ExceptionHandler << 3;307 }308 streamer.emitInt8(flags);309 310 if (info->PrologEnd)311 EmitAbsDifference(streamer, info->PrologEnd, info->Begin);312 else313 streamer.emitInt8(0);314 315 streamer.emitInt8(numCodes);316 317 uint8_t frame = 0;318 if (info->LastFrameInst >= 0) {319 WinEH::Instruction &frameInst = info->Instructions[info->LastFrameInst];320 assert(frameInst.Operation == Win64EH::UOP_SetFPReg);321 frame = (frameInst.Register & 0x0F) | (frameInst.Offset & 0xF0);322 }323 streamer.emitInt8(frame);324 325 // Emit the epilog instructions.326 if (EnableUnwindV2) {327 // Ensure the fixups and appended content apply to the same fragment.328 OS->ensureHeadroom(info->EpilogMap.size() * 2);329 330 bool IsLast = true;331 for (const auto &Epilog : llvm::reverse(info->EpilogMap)) {332 if (IsLast) {333 IsLast = false;334 uint8_t Flags = LastEpilogIsAtEnd ? 0x01 : 0;335 OS->emitInt8(EpilogSize);336 OS->emitInt8((Flags << 4) | Win64EH::UOP_Epilog);337 338 if (LastEpilogIsAtEnd)339 continue;340 }341 342 // Each epilog is emitted as a fixup, since we can't measure the distance343 // between the start of the epilog and the end of the function until344 // layout has been completed.345 auto *MCE = MCUnwindV2EpilogTargetExpr::create(*info, Epilog.second,346 EpilogSize, context);347 OS->addFixup(MCE, FK_Data_2);348 OS->appendContents(2, 0);349 }350 }351 if (AddPaddingEpilogCode)352 streamer.emitInt16(Win64EH::UOP_Epilog << 8);353 354 // Emit unwind instructions (in reverse order).355 uint8_t numInst = info->Instructions.size();356 for (uint8_t c = 0; c < numInst; ++c) {357 WinEH::Instruction inst = info->Instructions.back();358 info->Instructions.pop_back();359 EmitUnwindCode(streamer, info->Begin, inst);360 }361 362 // For alignment purposes, the instruction array will always have an even363 // number of entries, with the final entry potentially unused (in which case364 // the array will be one longer than indicated by the count of unwind codes365 // field).366 if (numCodes & 1) {367 streamer.emitInt16(0);368 }369 370 if (flags & (Win64EH::UNW_ChainInfo << 3))371 EmitRuntimeFunction(streamer, info->ChainedParent);372 else if (flags &373 ((Win64EH::UNW_TerminateHandler|Win64EH::UNW_ExceptionHandler) << 3))374 streamer.emitValue(MCSymbolRefExpr::create(info->ExceptionHandler,375 MCSymbolRefExpr::VK_COFF_IMGREL32,376 context), 4);377 else if (numCodes == 0) {378 // The minimum size of an UNWIND_INFO struct is 8 bytes. If we're not379 // a chained unwind info, if there is no handler, and if there are fewer380 // than 2 slots used in the unwind code array, we have to pad to 8 bytes.381 streamer.emitInt32(0);382 }383}384 385bool MCUnwindV2EpilogTargetExpr::evaluateAsRelocatableImpl(386 MCValue &Res, const MCAssembler *Asm) const {387 // Calculate the offset to this epilog, and validate it's within the allowed388 // range.389 auto Offset = GetOptionalAbsDifference(*Asm, FunctionEnd, UnwindV2Start);390 if (!Offset) {391 Asm->getContext().reportError(392 Loc, "Failed to evaluate epilog offset for Unwind v2 in " +393 Function->getName());394 return false;395 }396 assert(*Offset > 0);397 constexpr uint16_t MaxEpilogOffset = 0x0fff;398 if (*Offset > MaxEpilogOffset) {399 Asm->getContext().reportError(400 Loc,401 "Epilog offset is too large for Unwind v2 in " + Function->getName());402 return false;403 }404 405 // Sanity check that all epilogs are the same size.406 auto Size = GetOptionalAbsDifference(*Asm, EpilogEnd, UnwindV2Start);407 if (Size != (EpilogSize - 1)) {408 Asm->getContext().reportError(409 Loc, "Size of this epilog does not match size of last epilog in " +410 Function->getName());411 return false;412 }413 414 auto HighBits = *Offset >> 8;415 Res = MCValue::get((HighBits << 12) | (Win64EH::UOP_Epilog << 8) |416 (*Offset & 0xFF));417 return true;418}419 420void llvm::Win64EH::UnwindEmitter::Emit(MCStreamer &Streamer) const {421 // Emit the unwind info structs first.422 for (const auto &CFI : Streamer.getWinFrameInfos()) {423 MCSection *XData = Streamer.getAssociatedXDataSection(CFI->TextSection);424 Streamer.switchSection(XData);425 ::EmitUnwindInfo(Streamer, CFI.get());426 }427 428 // Now emit RUNTIME_FUNCTION entries.429 for (const auto &CFI : Streamer.getWinFrameInfos()) {430 MCSection *PData = Streamer.getAssociatedPDataSection(CFI->TextSection);431 Streamer.switchSection(PData);432 EmitRuntimeFunction(Streamer, CFI.get());433 }434}435 436void llvm::Win64EH::UnwindEmitter::EmitUnwindInfo(MCStreamer &Streamer,437 WinEH::FrameInfo *info,438 bool HandlerData) const {439 // Switch sections (the static function above is meant to be called from440 // here and from Emit().441 MCSection *XData = Streamer.getAssociatedXDataSection(info->TextSection);442 Streamer.switchSection(XData);443 444 ::EmitUnwindInfo(Streamer, info);445}446 447static const MCExpr *GetSubDivExpr(MCStreamer &Streamer, const MCSymbol *LHS,448 const MCSymbol *RHS, int Div) {449 MCContext &Context = Streamer.getContext();450 const MCExpr *Expr =451 MCBinaryExpr::createSub(MCSymbolRefExpr::create(LHS, Context),452 MCSymbolRefExpr::create(RHS, Context), Context);453 if (Div != 1)454 Expr = MCBinaryExpr::createDiv(Expr, MCConstantExpr::create(Div, Context),455 Context);456 return Expr;457}458 459static std::optional<int64_t> GetOptionalAbsDifference(MCStreamer &Streamer,460 const MCSymbol *LHS,461 const MCSymbol *RHS) {462 MCObjectStreamer *OS = (MCObjectStreamer *)(&Streamer);463 return GetOptionalAbsDifference(OS->getAssembler(), LHS, RHS);464}465 466static int64_t GetAbsDifference(MCStreamer &Streamer, const MCSymbol *LHS,467 const MCSymbol *RHS) {468 std::optional<int64_t> MaybeDiff =469 GetOptionalAbsDifference(Streamer, LHS, RHS);470 if (!MaybeDiff)471 report_fatal_error("Failed to evaluate function length in SEH unwind info");472 return *MaybeDiff;473}474 475static void checkARM64Instructions(MCStreamer &Streamer,476 ArrayRef<WinEH::Instruction> Insns,477 const MCSymbol *Begin, const MCSymbol *End,478 StringRef Name, StringRef Type) {479 if (!End)480 return;481 std::optional<int64_t> MaybeDistance =482 GetOptionalAbsDifference(Streamer, End, Begin);483 if (!MaybeDistance)484 return;485 uint32_t Distance = (uint32_t)*MaybeDistance;486 487 for (const auto &I : Insns) {488 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) {489 default:490 break;491 case Win64EH::UOP_TrapFrame:492 case Win64EH::UOP_PushMachFrame:493 case Win64EH::UOP_Context:494 case Win64EH::UOP_ECContext:495 case Win64EH::UOP_ClearUnwoundToCall:496 // Can't reason about these opcodes and how they map to actual497 // instructions.498 return;499 }500 }501 // Exclude the end opcode which doesn't map to an instruction.502 uint32_t InstructionBytes = 4 * (Insns.size() - 1);503 if (Distance != InstructionBytes) {504 Streamer.getContext().reportError(505 SMLoc(), "Incorrect size for " + Name + " " + Type + ": " +506 Twine(Distance) +507 " bytes of instructions in range, but .seh directives "508 "corresponding to " +509 Twine(InstructionBytes) + " bytes\n");510 }511}512 513static uint32_t ARM64CountOfUnwindCodes(ArrayRef<WinEH::Instruction> Insns) {514 uint32_t Count = 0;515 for (const auto &I : Insns) {516 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) {517 default:518 llvm_unreachable("Unsupported ARM64 unwind code");519 case Win64EH::UOP_AllocSmall:520 Count += 1;521 break;522 case Win64EH::UOP_AllocMedium:523 Count += 2;524 break;525 case Win64EH::UOP_AllocLarge:526 Count += 4;527 break;528 case Win64EH::UOP_SaveR19R20X:529 Count += 1;530 break;531 case Win64EH::UOP_SaveFPLRX:532 Count += 1;533 break;534 case Win64EH::UOP_SaveFPLR:535 Count += 1;536 break;537 case Win64EH::UOP_SaveReg:538 Count += 2;539 break;540 case Win64EH::UOP_SaveRegP:541 Count += 2;542 break;543 case Win64EH::UOP_SaveRegPX:544 Count += 2;545 break;546 case Win64EH::UOP_SaveRegX:547 Count += 2;548 break;549 case Win64EH::UOP_SaveLRPair:550 Count += 2;551 break;552 case Win64EH::UOP_SaveFReg:553 Count += 2;554 break;555 case Win64EH::UOP_SaveFRegP:556 Count += 2;557 break;558 case Win64EH::UOP_SaveFRegX:559 Count += 2;560 break;561 case Win64EH::UOP_SaveFRegPX:562 Count += 2;563 break;564 case Win64EH::UOP_SetFP:565 Count += 1;566 break;567 case Win64EH::UOP_AddFP:568 Count += 2;569 break;570 case Win64EH::UOP_Nop:571 Count += 1;572 break;573 case Win64EH::UOP_End:574 Count += 1;575 break;576 case Win64EH::UOP_SaveNext:577 Count += 1;578 break;579 case Win64EH::UOP_TrapFrame:580 Count += 1;581 break;582 case Win64EH::UOP_PushMachFrame:583 Count += 1;584 break;585 case Win64EH::UOP_Context:586 Count += 1;587 break;588 case Win64EH::UOP_ECContext:589 Count += 1;590 break;591 case Win64EH::UOP_ClearUnwoundToCall:592 Count += 1;593 break;594 case Win64EH::UOP_PACSignLR:595 Count += 1;596 break;597 case Win64EH::UOP_AllocZ:598 Count += 2;599 break;600 case Win64EH::UOP_SaveAnyRegI:601 case Win64EH::UOP_SaveAnyRegIP:602 case Win64EH::UOP_SaveAnyRegD:603 case Win64EH::UOP_SaveAnyRegDP:604 case Win64EH::UOP_SaveAnyRegQ:605 case Win64EH::UOP_SaveAnyRegQP:606 case Win64EH::UOP_SaveAnyRegIX:607 case Win64EH::UOP_SaveAnyRegIPX:608 case Win64EH::UOP_SaveAnyRegDX:609 case Win64EH::UOP_SaveAnyRegDPX:610 case Win64EH::UOP_SaveAnyRegQX:611 case Win64EH::UOP_SaveAnyRegQPX:612 case Win64EH::UOP_SaveZReg:613 case Win64EH::UOP_SavePReg:614 Count += 3;615 break;616 }617 }618 return Count;619}620 621// Unwind opcode encodings and restrictions are documented at622// https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling623static void ARM64EmitUnwindCode(MCStreamer &streamer,624 const WinEH::Instruction &inst) {625 uint8_t b, reg;626 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) {627 default:628 llvm_unreachable("Unsupported ARM64 unwind code");629 case Win64EH::UOP_AllocSmall:630 b = (inst.Offset >> 4) & 0x1F;631 streamer.emitInt8(b);632 break;633 case Win64EH::UOP_AllocMedium: {634 uint16_t hw = (inst.Offset >> 4) & 0x7FF;635 b = 0xC0;636 b |= (hw >> 8);637 streamer.emitInt8(b);638 b = hw & 0xFF;639 streamer.emitInt8(b);640 break;641 }642 case Win64EH::UOP_AllocLarge: {643 uint32_t w;644 b = 0xE0;645 streamer.emitInt8(b);646 w = inst.Offset >> 4;647 b = (w & 0x00FF0000) >> 16;648 streamer.emitInt8(b);649 b = (w & 0x0000FF00) >> 8;650 streamer.emitInt8(b);651 b = w & 0x000000FF;652 streamer.emitInt8(b);653 break;654 }655 case Win64EH::UOP_SetFP:656 b = 0xE1;657 streamer.emitInt8(b);658 break;659 case Win64EH::UOP_AddFP:660 b = 0xE2;661 streamer.emitInt8(b);662 b = (inst.Offset >> 3);663 streamer.emitInt8(b);664 break;665 case Win64EH::UOP_Nop:666 b = 0xE3;667 streamer.emitInt8(b);668 break;669 case Win64EH::UOP_SaveR19R20X:670 b = 0x20;671 b |= (inst.Offset >> 3) & 0x1F;672 streamer.emitInt8(b);673 break;674 case Win64EH::UOP_SaveFPLRX:675 b = 0x80;676 b |= ((inst.Offset >> 3) - 1) & 0x3F;677 streamer.emitInt8(b);678 break;679 case Win64EH::UOP_SaveFPLR:680 b = 0x40;681 b |= (inst.Offset >> 3) & 0x3F;682 streamer.emitInt8(b);683 break;684 case Win64EH::UOP_SaveReg:685 assert(inst.Register >= 19 && "Saved reg must be >= 19");686 reg = inst.Register - 19;687 b = 0xD0 | ((reg & 0xC) >> 2);688 streamer.emitInt8(b);689 b = ((reg & 0x3) << 6) | (inst.Offset >> 3);690 streamer.emitInt8(b);691 break;692 case Win64EH::UOP_SaveRegX:693 assert(inst.Register >= 19 && "Saved reg must be >= 19");694 reg = inst.Register - 19;695 b = 0xD4 | ((reg & 0x8) >> 3);696 streamer.emitInt8(b);697 b = ((reg & 0x7) << 5) | ((inst.Offset >> 3) - 1);698 streamer.emitInt8(b);699 break;700 case Win64EH::UOP_SaveRegP:701 assert(inst.Register >= 19 && "Saved registers must be >= 19");702 reg = inst.Register - 19;703 b = 0xC8 | ((reg & 0xC) >> 2);704 streamer.emitInt8(b);705 b = ((reg & 0x3) << 6) | (inst.Offset >> 3);706 streamer.emitInt8(b);707 break;708 case Win64EH::UOP_SaveRegPX:709 assert(inst.Register >= 19 && "Saved registers must be >= 19");710 reg = inst.Register - 19;711 b = 0xCC | ((reg & 0xC) >> 2);712 streamer.emitInt8(b);713 b = ((reg & 0x3) << 6) | ((inst.Offset >> 3) - 1);714 streamer.emitInt8(b);715 break;716 case Win64EH::UOP_SaveLRPair:717 assert(inst.Register >= 19 && "Saved reg must be >= 19");718 reg = inst.Register - 19;719 assert((reg % 2) == 0 && "Saved reg must be 19+2*X");720 reg /= 2;721 b = 0xD6 | ((reg & 0x7) >> 2);722 streamer.emitInt8(b);723 b = ((reg & 0x3) << 6) | (inst.Offset >> 3);724 streamer.emitInt8(b);725 break;726 case Win64EH::UOP_SaveFReg:727 assert(inst.Register >= 8 && "Saved dreg must be >= 8");728 reg = inst.Register - 8;729 b = 0xDC | ((reg & 0x4) >> 2);730 streamer.emitInt8(b);731 b = ((reg & 0x3) << 6) | (inst.Offset >> 3);732 streamer.emitInt8(b);733 break;734 case Win64EH::UOP_SaveFRegX:735 assert(inst.Register >= 8 && "Saved dreg must be >= 8");736 reg = inst.Register - 8;737 b = 0xDE;738 streamer.emitInt8(b);739 b = ((reg & 0x7) << 5) | ((inst.Offset >> 3) - 1);740 streamer.emitInt8(b);741 break;742 case Win64EH::UOP_SaveFRegP:743 assert(inst.Register >= 8 && "Saved dregs must be >= 8");744 reg = inst.Register - 8;745 b = 0xD8 | ((reg & 0x4) >> 2);746 streamer.emitInt8(b);747 b = ((reg & 0x3) << 6) | (inst.Offset >> 3);748 streamer.emitInt8(b);749 break;750 case Win64EH::UOP_SaveFRegPX:751 assert(inst.Register >= 8 && "Saved dregs must be >= 8");752 reg = inst.Register - 8;753 b = 0xDA | ((reg & 0x4) >> 2);754 streamer.emitInt8(b);755 b = ((reg & 0x3) << 6) | ((inst.Offset >> 3) - 1);756 streamer.emitInt8(b);757 break;758 case Win64EH::UOP_End:759 b = 0xE4;760 streamer.emitInt8(b);761 break;762 case Win64EH::UOP_SaveNext:763 b = 0xE6;764 streamer.emitInt8(b);765 break;766 case Win64EH::UOP_TrapFrame:767 b = 0xE8;768 streamer.emitInt8(b);769 break;770 case Win64EH::UOP_PushMachFrame:771 b = 0xE9;772 streamer.emitInt8(b);773 break;774 case Win64EH::UOP_Context:775 b = 0xEA;776 streamer.emitInt8(b);777 break;778 case Win64EH::UOP_ECContext:779 b = 0xEB;780 streamer.emitInt8(b);781 break;782 case Win64EH::UOP_ClearUnwoundToCall:783 b = 0xEC;784 streamer.emitInt8(b);785 break;786 case Win64EH::UOP_PACSignLR:787 b = 0xFC;788 streamer.emitInt8(b);789 break;790 case Win64EH::UOP_SaveAnyRegI:791 case Win64EH::UOP_SaveAnyRegIP:792 case Win64EH::UOP_SaveAnyRegD:793 case Win64EH::UOP_SaveAnyRegDP:794 case Win64EH::UOP_SaveAnyRegQ:795 case Win64EH::UOP_SaveAnyRegQP:796 case Win64EH::UOP_SaveAnyRegIX:797 case Win64EH::UOP_SaveAnyRegIPX:798 case Win64EH::UOP_SaveAnyRegDX:799 case Win64EH::UOP_SaveAnyRegDPX:800 case Win64EH::UOP_SaveAnyRegQX:801 case Win64EH::UOP_SaveAnyRegQPX: {802 // This assumes the opcodes are listed in the enum in a particular order.803 int Op = inst.Operation - Win64EH::UOP_SaveAnyRegI;804 int Writeback = Op / 6;805 int Paired = Op % 2;806 int Mode = (Op / 2) % 3;807 int Offset = inst.Offset >> 3;808 if (Writeback || Paired || Mode == 2)809 Offset >>= 1;810 if (Writeback)811 --Offset;812 b = 0xE7;813 streamer.emitInt8(b);814 assert(inst.Register < 32);815 b = inst.Register | (Writeback << 5) | (Paired << 6);816 streamer.emitInt8(b);817 b = Offset | (Mode << 6);818 streamer.emitInt8(b);819 break;820 }821 case Win64EH::UOP_AllocZ: {822 b = 0xDF;823 streamer.emitInt8(b);824 b = inst.Offset;825 streamer.emitInt8(b);826 break;827 }828 case Win64EH::UOP_SaveZReg: {829 assert(inst.Register >= 8 && inst.Register <= 23);830 assert(inst.Offset < 256);831 b = 0xE7;832 streamer.emitInt8(b);833 reg = inst.Register - 8;834 b = ((inst.Offset & 0xC0) >> 1) | reg;835 streamer.emitInt8(b);836 b = 0xC0 | (inst.Offset & 0x3F);837 streamer.emitInt8(b);838 break;839 }840 case Win64EH::UOP_SavePReg: {841 assert(inst.Register >= 4 && inst.Register <= 15);842 assert(inst.Offset < 256);843 b = 0xE7;844 streamer.emitInt8(b);845 reg = inst.Register;846 b = ((inst.Offset & 0xC0) >> 1) | 0x10 | reg;847 streamer.emitInt8(b);848 b = 0xC0 | (inst.Offset & 0x3F);849 streamer.emitInt8(b);850 break;851 }852 }853}854 855// Returns the epilog symbol of an epilog with the exact same unwind code856// sequence, if it exists. Otherwise, returns nullptr.857// EpilogInstrs - Unwind codes for the current epilog.858// Epilogs - Epilogs that potentialy match the current epilog.859static MCSymbol*860FindMatchingEpilog(const std::vector<WinEH::Instruction>& EpilogInstrs,861 const std::vector<MCSymbol *>& Epilogs,862 const WinEH::FrameInfo *info) {863 for (auto *EpilogStart : Epilogs) {864 auto InstrsIter = info->EpilogMap.find(EpilogStart);865 assert(InstrsIter != info->EpilogMap.end() &&866 "Epilog not found in EpilogMap");867 const auto &Instrs = InstrsIter->second.Instructions;868 869 if (Instrs.size() != EpilogInstrs.size())870 continue;871 872 bool Match = true;873 for (unsigned i = 0; i < Instrs.size(); ++i)874 if (Instrs[i] != EpilogInstrs[i]) {875 Match = false;876 break;877 }878 879 if (Match)880 return EpilogStart;881 }882 return nullptr;883}884 885static void simplifyARM64Opcodes(std::vector<WinEH::Instruction> &Instructions,886 bool Reverse) {887 unsigned PrevOffset = -1;888 unsigned PrevRegister = -1;889 890 auto VisitInstruction = [&](WinEH::Instruction &Inst) {891 // Convert 2-byte opcodes into equivalent 1-byte ones.892 if (Inst.Operation == Win64EH::UOP_SaveRegP && Inst.Register == 29) {893 Inst.Operation = Win64EH::UOP_SaveFPLR;894 Inst.Register = -1;895 } else if (Inst.Operation == Win64EH::UOP_SaveRegPX &&896 Inst.Register == 29) {897 Inst.Operation = Win64EH::UOP_SaveFPLRX;898 Inst.Register = -1;899 } else if (Inst.Operation == Win64EH::UOP_SaveRegPX &&900 Inst.Register == 19 && Inst.Offset <= 248) {901 Inst.Operation = Win64EH::UOP_SaveR19R20X;902 Inst.Register = -1;903 } else if (Inst.Operation == Win64EH::UOP_AddFP && Inst.Offset == 0) {904 Inst.Operation = Win64EH::UOP_SetFP;905 } else if (Inst.Operation == Win64EH::UOP_SaveRegP &&906 Inst.Register == PrevRegister + 2 &&907 Inst.Offset == PrevOffset + 16) {908 Inst.Operation = Win64EH::UOP_SaveNext;909 Inst.Register = -1;910 Inst.Offset = 0;911 // Intentionally not creating UOP_SaveNext for float register pairs,912 // as current versions of Windows (up to at least 20.04) is buggy913 // regarding SaveNext for float pairs.914 }915 // Update info about the previous instruction, for detecting if916 // the next one can be made a UOP_SaveNext917 if (Inst.Operation == Win64EH::UOP_SaveR19R20X) {918 PrevOffset = 0;919 PrevRegister = 19;920 } else if (Inst.Operation == Win64EH::UOP_SaveRegPX) {921 PrevOffset = 0;922 PrevRegister = Inst.Register;923 } else if (Inst.Operation == Win64EH::UOP_SaveRegP) {924 PrevOffset = Inst.Offset;925 PrevRegister = Inst.Register;926 } else if (Inst.Operation == Win64EH::UOP_SaveNext) {927 PrevRegister += 2;928 PrevOffset += 16;929 } else {930 PrevRegister = -1;931 PrevOffset = -1;932 }933 };934 935 // Iterate over instructions in a forward order (for prologues),936 // backwards for epilogues (i.e. always reverse compared to how the937 // opcodes are stored).938 if (Reverse) {939 for (auto It = Instructions.rbegin(); It != Instructions.rend(); It++)940 VisitInstruction(*It);941 } else {942 for (WinEH::Instruction &Inst : Instructions)943 VisitInstruction(Inst);944 }945}946 947// Check if an epilog exists as a subset of the end of a prolog (backwards).948static int949getARM64OffsetInProlog(const std::vector<WinEH::Instruction> &Prolog,950 const std::vector<WinEH::Instruction> &Epilog) {951 // Can't find an epilog as a subset if it is longer than the prolog.952 if (Epilog.size() > Prolog.size())953 return -1;954 955 // Check that the epilog actually is a perfect match for the end (backwrds)956 // of the prolog.957 for (int I = Epilog.size() - 1; I >= 0; I--) {958 if (Prolog[I] != Epilog[Epilog.size() - 1 - I])959 return -1;960 }961 962 if (Epilog.size() == Prolog.size())963 return 0;964 965 // If the epilog was a subset of the prolog, find its offset.966 return ARM64CountOfUnwindCodes(ArrayRef<WinEH::Instruction>(967 &Prolog[Epilog.size()], Prolog.size() - Epilog.size()));968}969 970static int checkARM64PackedEpilog(MCStreamer &streamer, WinEH::FrameInfo *info,971 WinEH::FrameInfo::Segment *Seg,972 int PrologCodeBytes) {973 // Can only pack if there's one single epilog974 if (Seg->Epilogs.size() != 1)975 return -1;976 977 MCSymbol *Sym = Seg->Epilogs.begin()->first;978 const std::vector<WinEH::Instruction> &Epilog =979 info->EpilogMap[Sym].Instructions;980 981 // Check that the epilog actually is at the very end of the function,982 // otherwise it can't be packed.983 uint32_t DistanceFromEnd =984 (uint32_t)(Seg->Offset + Seg->Length - Seg->Epilogs.begin()->second);985 if (DistanceFromEnd / 4 != Epilog.size())986 return -1;987 988 int RetVal = -1;989 // Even if we don't end up sharing opcodes with the prolog, we can still990 // write the offset as a packed offset, if the single epilog is located at991 // the end of the function and the offset (pointing after the prolog) fits992 // as a packed offset.993 if (PrologCodeBytes <= 31 &&994 PrologCodeBytes + ARM64CountOfUnwindCodes(Epilog) <= 124)995 RetVal = PrologCodeBytes;996 997 int Offset = getARM64OffsetInProlog(info->Instructions, Epilog);998 if (Offset < 0)999 return RetVal;1000 1001 // Check that the offset and prolog size fits in the first word; it's1002 // unclear whether the epilog count in the extension word can be taken1003 // as packed epilog offset.1004 if (Offset > 31 || PrologCodeBytes > 124)1005 return RetVal;1006 1007 // As we choose to express the epilog as part of the prolog, remove the1008 // epilog from the map, so we don't try to emit its opcodes.1009 info->EpilogMap.erase(Sym);1010 return Offset;1011}1012 1013static bool tryARM64PackedUnwind(WinEH::FrameInfo *info, uint32_t FuncLength,1014 int PackedEpilogOffset) {1015 if (PackedEpilogOffset == 0) {1016 // Fully symmetric prolog and epilog, should be ok for packed format.1017 // For CR=3, the corresponding synthesized epilog actually lacks the1018 // SetFP opcode, but unwinding should work just fine despite that1019 // (if at the SetFP opcode, the unwinder considers it as part of the1020 // function body and just unwinds the full prolog instead).1021 } else if (PackedEpilogOffset == 1) {1022 // One single case of differences between prolog and epilog is allowed:1023 // The epilog can lack a single SetFP that is the last opcode in the1024 // prolog, for the CR=3 case.1025 if (info->Instructions.back().Operation != Win64EH::UOP_SetFP)1026 return false;1027 } else {1028 // Too much difference between prolog and epilog.1029 return false;1030 }1031 unsigned RegI = 0, RegF = 0;1032 int Predecrement = 0;1033 enum {1034 Start,1035 Start2,1036 Start3,1037 IntRegs,1038 FloatRegs,1039 InputArgs,1040 StackAdjust,1041 FrameRecord,1042 End1043 } Location = Start;1044 bool StandaloneLR = false, FPLRPair = false;1045 bool PAC = false;1046 int StackOffset = 0;1047 int Nops = 0;1048 // Iterate over the prolog and check that all opcodes exactly match1049 // the canonical order and form. A more lax check could verify that1050 // all saved registers are in the expected locations, but not enforce1051 // the order - that would work fine when unwinding from within1052 // functions, but not be exactly right if unwinding happens within1053 // prologs/epilogs.1054 for (const WinEH::Instruction &Inst : info->Instructions) {1055 switch (Inst.Operation) {1056 case Win64EH::UOP_End:1057 if (Location != Start)1058 return false;1059 Location = Start2;1060 break;1061 case Win64EH::UOP_PACSignLR:1062 if (Location != Start2)1063 return false;1064 PAC = true;1065 Location = Start3;1066 break;1067 case Win64EH::UOP_SaveR19R20X:1068 if (Location != Start2 && Location != Start3)1069 return false;1070 Predecrement = Inst.Offset;1071 RegI = 2;1072 Location = IntRegs;1073 break;1074 case Win64EH::UOP_SaveRegX:1075 if (Location != Start2 && Location != Start3)1076 return false;1077 Predecrement = Inst.Offset;1078 if (Inst.Register == 19)1079 RegI += 1;1080 else if (Inst.Register == 30)1081 StandaloneLR = true;1082 else1083 return false;1084 // Odd register; can't be any further int registers.1085 Location = FloatRegs;1086 break;1087 case Win64EH::UOP_SaveRegPX:1088 // Can't have this in a canonical prologue. Either this has been1089 // canonicalized into SaveR19R20X or SaveFPLRX, or it's an unsupported1090 // register pair.1091 // It can't be canonicalized into SaveR19R20X if the offset is1092 // larger than 248 bytes, but even with the maximum case with1093 // RegI=10/RegF=8/CR=1/H=1, we end up with SavSZ = 216, which should1094 // fit into SaveR19R20X.1095 // The unwinding opcodes can't describe the otherwise seemingly valid1096 // case for RegI=1 CR=1, that would start with a1097 // "stp x19, lr, [sp, #-...]!" as that fits neither SaveRegPX nor1098 // SaveLRPair.1099 return false;1100 case Win64EH::UOP_SaveRegP:1101 if (Location != IntRegs || Inst.Offset != 8 * RegI ||1102 Inst.Register != 19 + RegI)1103 return false;1104 RegI += 2;1105 break;1106 case Win64EH::UOP_SaveReg:1107 if (Location != IntRegs || Inst.Offset != 8 * RegI)1108 return false;1109 if (Inst.Register == 19 + RegI)1110 RegI += 1;1111 else if (Inst.Register == 30)1112 StandaloneLR = true;1113 else1114 return false;1115 // Odd register; can't be any further int registers.1116 Location = FloatRegs;1117 break;1118 case Win64EH::UOP_SaveLRPair:1119 if (Location != IntRegs || Inst.Offset != 8 * RegI ||1120 Inst.Register != 19 + RegI)1121 return false;1122 RegI += 1;1123 StandaloneLR = true;1124 Location = FloatRegs;1125 break;1126 case Win64EH::UOP_SaveFRegX:1127 // Packed unwind can't handle prologs that only save one single1128 // float register.1129 return false;1130 case Win64EH::UOP_SaveFReg:1131 if (Location != FloatRegs || RegF == 0 || Inst.Register != 8 + RegF ||1132 Inst.Offset != 8 * (RegI + (StandaloneLR ? 1 : 0) + RegF))1133 return false;1134 RegF += 1;1135 Location = InputArgs;1136 break;1137 case Win64EH::UOP_SaveFRegPX:1138 if ((Location != Start2 && Location != Start3) || Inst.Register != 8)1139 return false;1140 Predecrement = Inst.Offset;1141 RegF = 2;1142 Location = FloatRegs;1143 break;1144 case Win64EH::UOP_SaveFRegP:1145 if ((Location != IntRegs && Location != FloatRegs) ||1146 Inst.Register != 8 + RegF ||1147 Inst.Offset != 8 * (RegI + (StandaloneLR ? 1 : 0) + RegF))1148 return false;1149 RegF += 2;1150 Location = FloatRegs;1151 break;1152 case Win64EH::UOP_SaveNext:1153 if (Location == IntRegs)1154 RegI += 2;1155 else if (Location == FloatRegs)1156 RegF += 2;1157 else1158 return false;1159 break;1160 case Win64EH::UOP_Nop:1161 if (Location != IntRegs && Location != FloatRegs && Location != InputArgs)1162 return false;1163 Location = InputArgs;1164 Nops++;1165 break;1166 case Win64EH::UOP_AllocSmall:1167 case Win64EH::UOP_AllocMedium:1168 if (Location != Start2 && Location != Start3 && Location != IntRegs &&1169 Location != FloatRegs && Location != InputArgs &&1170 Location != StackAdjust)1171 return false;1172 // Can have either a single decrement, or a pair of decrements with1173 // 4080 and another decrement.1174 if (StackOffset == 0)1175 StackOffset = Inst.Offset;1176 else if (StackOffset != 4080)1177 return false;1178 else1179 StackOffset += Inst.Offset;1180 Location = StackAdjust;1181 break;1182 case Win64EH::UOP_SaveFPLRX:1183 // Not allowing FPLRX after StackAdjust; if a StackAdjust is used, it1184 // should be followed by a FPLR instead.1185 if (Location != Start2 && Location != Start3 && Location != IntRegs &&1186 Location != FloatRegs && Location != InputArgs)1187 return false;1188 StackOffset = Inst.Offset;1189 Location = FrameRecord;1190 FPLRPair = true;1191 break;1192 case Win64EH::UOP_SaveFPLR:1193 // This can only follow after a StackAdjust1194 if (Location != StackAdjust || Inst.Offset != 0)1195 return false;1196 Location = FrameRecord;1197 FPLRPair = true;1198 break;1199 case Win64EH::UOP_SetFP:1200 if (Location != FrameRecord)1201 return false;1202 Location = End;1203 break;1204 case Win64EH::UOP_SaveAnyRegI:1205 case Win64EH::UOP_SaveAnyRegIP:1206 case Win64EH::UOP_SaveAnyRegD:1207 case Win64EH::UOP_SaveAnyRegDP:1208 case Win64EH::UOP_SaveAnyRegQ:1209 case Win64EH::UOP_SaveAnyRegQP:1210 case Win64EH::UOP_SaveAnyRegIX:1211 case Win64EH::UOP_SaveAnyRegIPX:1212 case Win64EH::UOP_SaveAnyRegDX:1213 case Win64EH::UOP_SaveAnyRegDPX:1214 case Win64EH::UOP_SaveAnyRegQX:1215 case Win64EH::UOP_SaveAnyRegQPX:1216 // These are never canonical; they don't show up with the usual Arm641217 // calling convention.1218 return false;1219 case Win64EH::UOP_AllocLarge:1220 // Allocations this large can't be represented in packed unwind (and1221 // usually don't fit the canonical form anyway because we need to use1222 // __chkstk to allocate the stack space).1223 return false;1224 case Win64EH::UOP_AddFP:1225 // "add x29, sp, #N" doesn't show up in the canonical pattern (except for1226 // N=0, which is UOP_SetFP).1227 return false;1228 case Win64EH::UOP_AllocZ:1229 case Win64EH::UOP_SaveZReg:1230 case Win64EH::UOP_SavePReg:1231 // Canonical prologues don't support spilling SVE registers.1232 return false;1233 case Win64EH::UOP_TrapFrame:1234 case Win64EH::UOP_Context:1235 case Win64EH::UOP_ECContext:1236 case Win64EH::UOP_ClearUnwoundToCall:1237 case Win64EH::UOP_PushMachFrame:1238 // These are special opcodes that aren't normally generated.1239 return false;1240 default:1241 report_fatal_error("Unknown Arm64 unwind opcode");1242 }1243 }1244 if (RegI > 10 || RegF > 8)1245 return false;1246 if (StandaloneLR && FPLRPair)1247 return false;1248 if (FPLRPair && Location != End)1249 return false;1250 if (Nops != 0 && Nops != 4)1251 return false;1252 if (PAC && !FPLRPair)1253 return false;1254 int H = Nops == 4;1255 // There's an inconsistency regarding packed unwind info with homed1256 // parameters; according to the documentation, the epilog shouldn't have1257 // the same corresponding nops (and thus, to set the H bit, we should1258 // require an epilog which isn't exactly symmetrical - we shouldn't accept1259 // an exact mirrored epilog for those cases), but in practice,1260 // RtlVirtualUnwind behaves as if it does expect the epilogue to contain1261 // the same nops. See https://github.com/llvm/llvm-project/issues/54879.1262 // To play it safe, don't produce packed unwind info with homed parameters.1263 if (H)1264 return false;1265 int IntSZ = 8 * RegI;1266 if (StandaloneLR)1267 IntSZ += 8;1268 int FpSZ = 8 * RegF; // RegF not yet decremented1269 int SavSZ = (IntSZ + FpSZ + 8 * 8 * H + 0xF) & ~0xF;1270 if (Predecrement != SavSZ)1271 return false;1272 if (FPLRPair && StackOffset < 16)1273 return false;1274 if (StackOffset % 16)1275 return false;1276 uint32_t FrameSize = (StackOffset + SavSZ) / 16;1277 if (FrameSize > 0x1FF)1278 return false;1279 assert(RegF != 1 && "One single float reg not allowed");1280 if (RegF > 0)1281 RegF--; // Convert from actual number of registers, to value stored1282 assert(FuncLength <= 0x7FF && "FuncLength should have been checked earlier");1283 int Flag = 0x01; // Function segments not supported yet1284 int CR = PAC ? 2 : FPLRPair ? 3 : StandaloneLR ? 1 : 0;1285 info->PackedInfo |= Flag << 0;1286 info->PackedInfo |= (FuncLength & 0x7FF) << 2;1287 info->PackedInfo |= (RegF & 0x7) << 13;1288 info->PackedInfo |= (RegI & 0xF) << 16;1289 info->PackedInfo |= (H & 0x1) << 20;1290 info->PackedInfo |= (CR & 0x3) << 21;1291 info->PackedInfo |= (FrameSize & 0x1FF) << 23;1292 return true;1293}1294 1295static void ARM64ProcessEpilogs(WinEH::FrameInfo *info,1296 WinEH::FrameInfo::Segment *Seg,1297 uint32_t &TotalCodeBytes,1298 MapVector<MCSymbol *, uint32_t> &EpilogInfo) {1299 1300 std::vector<MCSymbol *> EpilogStarts;1301 for (auto &I : Seg->Epilogs)1302 EpilogStarts.push_back(I.first);1303 1304 // Epilogs processed so far.1305 std::vector<MCSymbol *> AddedEpilogs;1306 for (auto *S : EpilogStarts) {1307 MCSymbol *EpilogStart = S;1308 auto &EpilogInstrs = info->EpilogMap[S].Instructions;1309 uint32_t CodeBytes = ARM64CountOfUnwindCodes(EpilogInstrs);1310 1311 MCSymbol* MatchingEpilog =1312 FindMatchingEpilog(EpilogInstrs, AddedEpilogs, info);1313 int PrologOffset;1314 if (MatchingEpilog) {1315 assert(EpilogInfo.contains(MatchingEpilog) &&1316 "Duplicate epilog not found");1317 EpilogInfo[EpilogStart] = EpilogInfo.lookup(MatchingEpilog);1318 // Clear the unwind codes in the EpilogMap, so that they don't get output1319 // in ARM64EmitUnwindInfoForSegment().1320 EpilogInstrs.clear();1321 } else if ((PrologOffset = getARM64OffsetInProlog(info->Instructions,1322 EpilogInstrs)) >= 0) {1323 EpilogInfo[EpilogStart] = PrologOffset;1324 // If the segment doesn't have a prolog, an end_c will be emitted before1325 // prolog opcodes. So epilog start index in opcodes array is moved by 1.1326 if (!Seg->HasProlog)1327 EpilogInfo[EpilogStart] += 1;1328 // Clear the unwind codes in the EpilogMap, so that they don't get output1329 // in ARM64EmitUnwindInfoForSegment().1330 EpilogInstrs.clear();1331 } else {1332 EpilogInfo[EpilogStart] = TotalCodeBytes;1333 TotalCodeBytes += CodeBytes;1334 AddedEpilogs.push_back(EpilogStart);1335 }1336 }1337}1338 1339static void ARM64FindSegmentsInFunction(MCStreamer &streamer,1340 WinEH::FrameInfo *info,1341 int64_t RawFuncLength) {1342 if (info->PrologEnd)1343 checkARM64Instructions(streamer, info->Instructions, info->Begin,1344 info->PrologEnd, info->Function->getName(),1345 "prologue");1346 struct EpilogStartEnd {1347 MCSymbol *Start;1348 int64_t Offset;1349 int64_t End;1350 };1351 // Record Start and End of each epilog.1352 SmallVector<struct EpilogStartEnd, 4> Epilogs;1353 for (auto &I : info->EpilogMap) {1354 MCSymbol *Start = I.first;1355 auto &Instrs = I.second.Instructions;1356 int64_t Offset = GetAbsDifference(streamer, Start, info->Begin);1357 checkARM64Instructions(streamer, Instrs, Start, I.second.End,1358 info->Function->getName(), "epilogue");1359 assert((Epilogs.size() == 0 || Offset >= Epilogs.back().End) &&1360 "Epilogs should be monotonically ordered");1361 // Exclue the end opcode from Instrs.size() when calculating the end of the1362 // epilog.1363 Epilogs.push_back({Start, Offset, Offset + (int64_t)(Instrs.size() - 1) * 4});1364 }1365 1366 unsigned E = 0;1367 int64_t SegLimit = 0xFFFFC;1368 int64_t SegOffset = 0;1369 1370 if (RawFuncLength > SegLimit) {1371 1372 int64_t RemainingLength = RawFuncLength;1373 1374 while (RemainingLength > SegLimit) {1375 // Try divide the function into segments, requirements:1376 // 1. Segment length <= 0xFFFFC;1377 // 2. Each Prologue or Epilogue must be fully within a segment.1378 int64_t SegLength = SegLimit;1379 int64_t SegEnd = SegOffset + SegLength;1380 // Keep record on symbols and offsets of epilogs in this segment.1381 MapVector<MCSymbol *, int64_t> EpilogsInSegment;1382 1383 while (E < Epilogs.size() && Epilogs[E].End < SegEnd) {1384 // Epilogs within current segment.1385 EpilogsInSegment[Epilogs[E].Start] = Epilogs[E].Offset;1386 ++E;1387 }1388 1389 // At this point, we have:1390 // 1. Put all epilogs in segments already. No action needed here; or1391 // 2. Found an epilog that will cross segments boundry. We need to1392 // move back current segment's end boundry, so the epilog is entirely1393 // in the next segment; or1394 // 3. Left at least one epilog that is entirely after this segment.1395 // It'll be handled by the next iteration, or the last segment.1396 if (E < Epilogs.size() && Epilogs[E].Offset <= SegEnd)1397 // Move back current Segment's end boundry.1398 SegLength = Epilogs[E].Offset - SegOffset;1399 1400 auto Seg = WinEH::FrameInfo::Segment(1401 SegOffset, SegLength, /* HasProlog */!SegOffset);1402 Seg.Epilogs = std::move(EpilogsInSegment);1403 info->Segments.push_back(Seg);1404 1405 SegOffset += SegLength;1406 RemainingLength -= SegLength;1407 }1408 }1409 1410 // Add the last segment when RawFuncLength > 0xFFFFC,1411 // or the only segment otherwise.1412 auto LastSeg =1413 WinEH::FrameInfo::Segment(SegOffset, RawFuncLength - SegOffset,1414 /* HasProlog */!SegOffset);1415 for (; E < Epilogs.size(); ++E)1416 LastSeg.Epilogs[Epilogs[E].Start] = Epilogs[E].Offset;1417 info->Segments.push_back(LastSeg);1418}1419 1420static void ARM64EmitUnwindInfoForSegment(MCStreamer &streamer,1421 WinEH::FrameInfo *info,1422 WinEH::FrameInfo::Segment &Seg,1423 bool TryPacked = true) {1424 MCContext &context = streamer.getContext();1425 MCSymbol *Label = context.createTempSymbol();1426 1427 streamer.emitValueToAlignment(Align(4));1428 streamer.emitLabel(Label);1429 Seg.Symbol = Label;1430 // Use the 1st segemnt's label as function's.1431 if (Seg.Offset == 0)1432 info->Symbol = Label;1433 1434 bool HasProlog = Seg.HasProlog;1435 bool HasEpilogs = (Seg.Epilogs.size() != 0);1436 1437 uint32_t SegLength = (uint32_t)Seg.Length / 4;1438 uint32_t PrologCodeBytes = info->PrologCodeBytes;1439 1440 int PackedEpilogOffset = HasEpilogs ?1441 checkARM64PackedEpilog(streamer, info, &Seg, PrologCodeBytes) : -1;1442 1443 // TODO:1444 // 1. Enable packed unwind info (.pdata only) for multi-segment functions.1445 // 2. Emit packed unwind info (.pdata only) for segments that have neithor1446 // prolog nor epilog.1447 if (info->Segments.size() == 1 && PackedEpilogOffset >= 0 &&1448 uint32_t(PackedEpilogOffset) < PrologCodeBytes &&1449 !info->HandlesExceptions && SegLength <= 0x7ff && TryPacked) {1450 // Matching prolog/epilog and no exception handlers; check if the1451 // prolog matches the patterns that can be described by the packed1452 // format.1453 1454 // info->Symbol was already set even if we didn't actually write any1455 // unwind info there. Keep using that as indicator that this unwind1456 // info has been generated already.1457 if (tryARM64PackedUnwind(info, SegLength, PackedEpilogOffset))1458 return;1459 }1460 1461 // If the prolog is not in this segment, we need to emit an end_c, which takes1462 // 1 byte, before prolog unwind ops.1463 if (!HasProlog) {1464 PrologCodeBytes += 1;1465 if (PackedEpilogOffset >= 0)1466 PackedEpilogOffset += 1;1467 // If a segment has neither prolog nor epilog, "With full .xdata record,1468 // Epilog Count = 1. Epilog Start Index points to end_c."1469 // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling#function-fragments1470 // TODO: We can remove this if testing shows zero epilog scope is ok with1471 // MS unwinder.1472 if (!HasEpilogs)1473 // Pack the fake epilog into phantom prolog.1474 PackedEpilogOffset = 0;1475 }1476 1477 uint32_t TotalCodeBytes = PrologCodeBytes;1478 1479 // Process epilogs.1480 MapVector<MCSymbol *, uint32_t> EpilogInfo;1481 ARM64ProcessEpilogs(info, &Seg, TotalCodeBytes, EpilogInfo);1482 1483 // Code Words, Epilog count, E, X, Vers, Function Length1484 uint32_t row1 = 0x0;1485 uint32_t CodeWords = TotalCodeBytes / 4;1486 uint32_t CodeWordsMod = TotalCodeBytes % 4;1487 if (CodeWordsMod)1488 CodeWords++;1489 uint32_t EpilogCount =1490 PackedEpilogOffset >= 0 ? PackedEpilogOffset : Seg.Epilogs.size();1491 bool ExtensionWord = EpilogCount > 31 || TotalCodeBytes > 124;1492 if (!ExtensionWord) {1493 row1 |= (EpilogCount & 0x1F) << 22;1494 row1 |= (CodeWords & 0x1F) << 27;1495 }1496 if (info->HandlesExceptions) // X1497 row1 |= 1 << 20;1498 if (PackedEpilogOffset >= 0) // E1499 row1 |= 1 << 21;1500 row1 |= SegLength & 0x3FFFF;1501 streamer.emitInt32(row1);1502 1503 // Extended Code Words, Extended Epilog Count1504 if (ExtensionWord) {1505 // FIXME: We should be able to split unwind info into multiple sections.1506 if (CodeWords > 0xFF || EpilogCount > 0xFFFF)1507 report_fatal_error(1508 "SEH unwind data splitting is only implemented for large functions, "1509 "cases of too many code words or too many epilogs will be done "1510 "later");1511 uint32_t row2 = 0x0;1512 row2 |= (CodeWords & 0xFF) << 16;1513 row2 |= (EpilogCount & 0xFFFF);1514 streamer.emitInt32(row2);1515 }1516 1517 if (PackedEpilogOffset < 0) {1518 // Epilog Start Index, Epilog Start Offset1519 for (auto &I : EpilogInfo) {1520 MCSymbol *EpilogStart = I.first;1521 uint32_t EpilogIndex = I.second;1522 // Epilog offset within the Segment.1523 uint32_t EpilogOffset = (uint32_t)(Seg.Epilogs[EpilogStart] - Seg.Offset);1524 if (EpilogOffset)1525 EpilogOffset /= 4;1526 uint32_t row3 = EpilogOffset;1527 row3 |= (EpilogIndex & 0x3FF) << 22;1528 streamer.emitInt32(row3);1529 }1530 }1531 1532 // Note that even for segments that have no prolog, we still need to emit1533 // prolog unwinding opcodes so that the unwinder knows how to unwind from1534 // such a segment.1535 // The end_c opcode at the start indicates to the unwinder that the actual1536 // prolog is outside of the current segment, and the unwinder shouldn't try1537 // to check for unwinding from a partial prolog.1538 if (!HasProlog)1539 // Emit an end_c.1540 streamer.emitInt8((uint8_t)0xE5);1541 1542 // Emit prolog unwind instructions (in reverse order).1543 for (auto Inst : llvm::reverse(info->Instructions))1544 ARM64EmitUnwindCode(streamer, Inst);1545 1546 // Emit epilog unwind instructions1547 for (auto &I : Seg.Epilogs) {1548 auto &EpilogInstrs = info->EpilogMap[I.first].Instructions;1549 for (const WinEH::Instruction &inst : EpilogInstrs)1550 ARM64EmitUnwindCode(streamer, inst);1551 }1552 1553 int32_t BytesMod = CodeWords * 4 - TotalCodeBytes;1554 assert(BytesMod >= 0);1555 for (int i = 0; i < BytesMod; i++)1556 streamer.emitInt8(0xE3);1557 1558 if (info->HandlesExceptions)1559 streamer.emitValue(1560 MCSymbolRefExpr::create(info->ExceptionHandler,1561 MCSymbolRefExpr::VK_COFF_IMGREL32, context),1562 4);1563}1564 1565// Populate the .xdata section. The format of .xdata on ARM64 is documented at1566// https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling1567static void ARM64EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info,1568 bool TryPacked = true) {1569 // If this UNWIND_INFO already has a symbol, it's already been emitted.1570 if (info->Symbol)1571 return;1572 // If there's no unwind info here (not even a terminating UOP_End), the1573 // unwind info is considered bogus and skipped. If this was done in1574 // response to an explicit .seh_handlerdata, the associated trailing1575 // handler data is left orphaned in the xdata section.1576 if (info->empty()) {1577 info->EmitAttempted = true;1578 return;1579 }1580 if (info->EmitAttempted) {1581 // If we tried to emit unwind info before (due to an explicit1582 // .seh_handlerdata directive), but skipped it (because there was no1583 // valid information to emit at the time), and it later got valid unwind1584 // opcodes, we can't emit it here, because the trailing handler data1585 // was already emitted elsewhere in the xdata section.1586 streamer.getContext().reportError(1587 SMLoc(), "Earlier .seh_handlerdata for " + info->Function->getName() +1588 " skipped due to no unwind info at the time "1589 "(.seh_handlerdata too early?), but the function later "1590 "did get unwind info that can't be emitted");1591 return;1592 }1593 1594 simplifyARM64Opcodes(info->Instructions, false);1595 for (auto &I : info->EpilogMap)1596 simplifyARM64Opcodes(I.second.Instructions, true);1597 1598 int64_t RawFuncLength;1599 if (!info->FuncletOrFuncEnd) {1600 report_fatal_error("FuncletOrFuncEnd not set");1601 } else {1602 // FIXME: GetAbsDifference tries to compute the length of the function1603 // immediately, before the whole file is emitted, but in general1604 // that's impossible: the size in bytes of certain assembler directives1605 // like .align and .fill is not known until the whole file is parsed and1606 // relaxations are applied. Currently, GetAbsDifference fails with a fatal1607 // error in that case. (We mostly don't hit this because inline assembly1608 // specifying those directives is rare, and we don't normally try to1609 // align loops on AArch64.)1610 //1611 // There are two potential approaches to delaying the computation. One,1612 // we could emit something like ".word (endfunc-beginfunc)/4+0x10800000",1613 // as long as we have some conservative estimate we could use to prove1614 // that we don't need to split the unwind data. Emitting the constant1615 // is straightforward, but there's no existing code for estimating the1616 // size of the function.1617 //1618 // The other approach would be to use a dedicated, relaxable fragment,1619 // which could grow to accommodate splitting the unwind data if1620 // necessary. This is more straightforward, since it automatically works1621 // without any new infrastructure, and it's consistent with how we handle1622 // relaxation in other contexts. But it would require some refactoring1623 // to move parts of the pdata/xdata emission into the implementation of1624 // a fragment. We could probably continue to encode the unwind codes1625 // here, but we'd have to emit the pdata, the xdata header, and the1626 // epilogue scopes later, since they depend on whether the we need to1627 // split the unwind data.1628 //1629 // If this is fixed, remove code in AArch64ISelLowering.cpp that1630 // disables loop alignment on Windows.1631 RawFuncLength = GetAbsDifference(streamer, info->FuncletOrFuncEnd,1632 info->Begin);1633 }1634 1635 ARM64FindSegmentsInFunction(streamer, info, RawFuncLength);1636 1637 info->PrologCodeBytes = ARM64CountOfUnwindCodes(info->Instructions);1638 for (auto &S : info->Segments)1639 ARM64EmitUnwindInfoForSegment(streamer, info, S, TryPacked);1640 1641 // Clear prolog instructions after unwind info is emitted for all segments.1642 info->Instructions.clear();1643}1644 1645static uint32_t ARMCountOfUnwindCodes(ArrayRef<WinEH::Instruction> Insns) {1646 uint32_t Count = 0;1647 for (const auto &I : Insns) {1648 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) {1649 default:1650 llvm_unreachable("Unsupported ARM unwind code");1651 case Win64EH::UOP_AllocSmall:1652 Count += 1;1653 break;1654 case Win64EH::UOP_AllocLarge:1655 Count += 3;1656 break;1657 case Win64EH::UOP_AllocHuge:1658 Count += 4;1659 break;1660 case Win64EH::UOP_WideAllocMedium:1661 Count += 2;1662 break;1663 case Win64EH::UOP_WideAllocLarge:1664 Count += 3;1665 break;1666 case Win64EH::UOP_WideAllocHuge:1667 Count += 4;1668 break;1669 case Win64EH::UOP_WideSaveRegMask:1670 Count += 2;1671 break;1672 case Win64EH::UOP_SaveSP:1673 Count += 1;1674 break;1675 case Win64EH::UOP_SaveRegsR4R7LR:1676 Count += 1;1677 break;1678 case Win64EH::UOP_WideSaveRegsR4R11LR:1679 Count += 1;1680 break;1681 case Win64EH::UOP_SaveFRegD8D15:1682 Count += 1;1683 break;1684 case Win64EH::UOP_SaveRegMask:1685 Count += 2;1686 break;1687 case Win64EH::UOP_SaveLR:1688 Count += 2;1689 break;1690 case Win64EH::UOP_SaveFRegD0D15:1691 Count += 2;1692 break;1693 case Win64EH::UOP_SaveFRegD16D31:1694 Count += 2;1695 break;1696 case Win64EH::UOP_Nop:1697 case Win64EH::UOP_WideNop:1698 case Win64EH::UOP_End:1699 case Win64EH::UOP_EndNop:1700 case Win64EH::UOP_WideEndNop:1701 Count += 1;1702 break;1703 case Win64EH::UOP_Custom: {1704 int J;1705 for (J = 3; J > 0; J--)1706 if (I.Offset & (0xffu << (8 * J)))1707 break;1708 Count += J + 1;1709 break;1710 }1711 }1712 }1713 return Count;1714}1715 1716static uint32_t ARMCountOfInstructionBytes(ArrayRef<WinEH::Instruction> Insns,1717 bool *HasCustom = nullptr) {1718 uint32_t Count = 0;1719 for (const auto &I : Insns) {1720 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) {1721 default:1722 llvm_unreachable("Unsupported ARM unwind code");1723 case Win64EH::UOP_AllocSmall:1724 case Win64EH::UOP_AllocLarge:1725 case Win64EH::UOP_AllocHuge:1726 Count += 2;1727 break;1728 case Win64EH::UOP_WideAllocMedium:1729 case Win64EH::UOP_WideAllocLarge:1730 case Win64EH::UOP_WideAllocHuge:1731 Count += 4;1732 break;1733 case Win64EH::UOP_WideSaveRegMask:1734 case Win64EH::UOP_WideSaveRegsR4R11LR:1735 Count += 4;1736 break;1737 case Win64EH::UOP_SaveSP:1738 Count += 2;1739 break;1740 case Win64EH::UOP_SaveRegMask:1741 case Win64EH::UOP_SaveRegsR4R7LR:1742 Count += 2;1743 break;1744 case Win64EH::UOP_SaveFRegD8D15:1745 case Win64EH::UOP_SaveFRegD0D15:1746 case Win64EH::UOP_SaveFRegD16D31:1747 Count += 4;1748 break;1749 case Win64EH::UOP_SaveLR:1750 Count += 4;1751 break;1752 case Win64EH::UOP_Nop:1753 case Win64EH::UOP_EndNop:1754 Count += 2;1755 break;1756 case Win64EH::UOP_WideNop:1757 case Win64EH::UOP_WideEndNop:1758 Count += 4;1759 break;1760 case Win64EH::UOP_End:1761 // This doesn't map to any instruction1762 break;1763 case Win64EH::UOP_Custom:1764 // We can't reason about what instructions this maps to; return a1765 // phony number to make sure we don't accidentally do epilog packing.1766 Count += 1000;1767 if (HasCustom)1768 *HasCustom = true;1769 break;1770 }1771 }1772 return Count;1773}1774 1775static void checkARMInstructions(MCStreamer &Streamer,1776 ArrayRef<WinEH::Instruction> Insns,1777 const MCSymbol *Begin, const MCSymbol *End,1778 StringRef Name, StringRef Type) {1779 if (!End)1780 return;1781 std::optional<int64_t> MaybeDistance =1782 GetOptionalAbsDifference(Streamer, End, Begin);1783 if (!MaybeDistance)1784 return;1785 uint32_t Distance = (uint32_t)*MaybeDistance;1786 bool HasCustom = false;1787 uint32_t InstructionBytes = ARMCountOfInstructionBytes(Insns, &HasCustom);1788 if (HasCustom)1789 return;1790 if (Distance != InstructionBytes) {1791 Streamer.getContext().reportError(1792 SMLoc(), "Incorrect size for " + Name + " " + Type + ": " +1793 Twine(Distance) +1794 " bytes of instructions in range, but .seh directives "1795 "corresponding to " +1796 Twine(InstructionBytes) + " bytes\n");1797 }1798}1799 1800static bool isARMTerminator(const WinEH::Instruction &inst) {1801 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) {1802 case Win64EH::UOP_End:1803 case Win64EH::UOP_EndNop:1804 case Win64EH::UOP_WideEndNop:1805 return true;1806 default:1807 return false;1808 }1809}1810 1811// Unwind opcode encodings and restrictions are documented at1812// https://docs.microsoft.com/en-us/cpp/build/arm-exception-handling1813static void ARMEmitUnwindCode(MCStreamer &streamer,1814 const WinEH::Instruction &inst) {1815 uint32_t w, lr;1816 int i;1817 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) {1818 default:1819 llvm_unreachable("Unsupported ARM unwind code");1820 case Win64EH::UOP_AllocSmall:1821 assert((inst.Offset & 3) == 0);1822 assert(inst.Offset / 4 <= 0x7f);1823 streamer.emitInt8(inst.Offset / 4);1824 break;1825 case Win64EH::UOP_WideSaveRegMask:1826 assert((inst.Register & ~0x5fff) == 0);1827 lr = (inst.Register >> 14) & 1;1828 w = 0x8000 | (inst.Register & 0x1fff) | (lr << 13);1829 streamer.emitInt8((w >> 8) & 0xff);1830 streamer.emitInt8((w >> 0) & 0xff);1831 break;1832 case Win64EH::UOP_SaveSP:1833 assert(inst.Register <= 0x0f);1834 streamer.emitInt8(0xc0 | inst.Register);1835 break;1836 case Win64EH::UOP_SaveRegsR4R7LR:1837 assert(inst.Register >= 4 && inst.Register <= 7);1838 assert(inst.Offset <= 1);1839 streamer.emitInt8(0xd0 | (inst.Register - 4) | (inst.Offset << 2));1840 break;1841 case Win64EH::UOP_WideSaveRegsR4R11LR:1842 assert(inst.Register >= 8 && inst.Register <= 11);1843 assert(inst.Offset <= 1);1844 streamer.emitInt8(0xd8 | (inst.Register - 8) | (inst.Offset << 2));1845 break;1846 case Win64EH::UOP_SaveFRegD8D15:1847 assert(inst.Register >= 8 && inst.Register <= 15);1848 streamer.emitInt8(0xe0 | (inst.Register - 8));1849 break;1850 case Win64EH::UOP_WideAllocMedium:1851 assert((inst.Offset & 3) == 0);1852 assert(inst.Offset / 4 <= 0x3ff);1853 w = 0xe800 | (inst.Offset / 4);1854 streamer.emitInt8((w >> 8) & 0xff);1855 streamer.emitInt8((w >> 0) & 0xff);1856 break;1857 case Win64EH::UOP_SaveRegMask:1858 assert((inst.Register & ~0x40ff) == 0);1859 lr = (inst.Register >> 14) & 1;1860 w = 0xec00 | (inst.Register & 0x0ff) | (lr << 8);1861 streamer.emitInt8((w >> 8) & 0xff);1862 streamer.emitInt8((w >> 0) & 0xff);1863 break;1864 case Win64EH::UOP_SaveLR:1865 assert((inst.Offset & 3) == 0);1866 assert(inst.Offset / 4 <= 0x0f);1867 streamer.emitInt8(0xef);1868 streamer.emitInt8(inst.Offset / 4);1869 break;1870 case Win64EH::UOP_SaveFRegD0D15:1871 assert(inst.Register <= 15);1872 assert(inst.Offset <= 15);1873 assert(inst.Register <= inst.Offset);1874 streamer.emitInt8(0xf5);1875 streamer.emitInt8((inst.Register << 4) | inst.Offset);1876 break;1877 case Win64EH::UOP_SaveFRegD16D31:1878 assert(inst.Register >= 16 && inst.Register <= 31);1879 assert(inst.Offset >= 16 && inst.Offset <= 31);1880 assert(inst.Register <= inst.Offset);1881 streamer.emitInt8(0xf6);1882 streamer.emitInt8(((inst.Register - 16) << 4) | (inst.Offset - 16));1883 break;1884 case Win64EH::UOP_AllocLarge:1885 assert((inst.Offset & 3) == 0);1886 assert(inst.Offset / 4 <= 0xffff);1887 w = inst.Offset / 4;1888 streamer.emitInt8(0xf7);1889 streamer.emitInt8((w >> 8) & 0xff);1890 streamer.emitInt8((w >> 0) & 0xff);1891 break;1892 case Win64EH::UOP_AllocHuge:1893 assert((inst.Offset & 3) == 0);1894 assert(inst.Offset / 4 <= 0xffffff);1895 w = inst.Offset / 4;1896 streamer.emitInt8(0xf8);1897 streamer.emitInt8((w >> 16) & 0xff);1898 streamer.emitInt8((w >> 8) & 0xff);1899 streamer.emitInt8((w >> 0) & 0xff);1900 break;1901 case Win64EH::UOP_WideAllocLarge:1902 assert((inst.Offset & 3) == 0);1903 assert(inst.Offset / 4 <= 0xffff);1904 w = inst.Offset / 4;1905 streamer.emitInt8(0xf9);1906 streamer.emitInt8((w >> 8) & 0xff);1907 streamer.emitInt8((w >> 0) & 0xff);1908 break;1909 case Win64EH::UOP_WideAllocHuge:1910 assert((inst.Offset & 3) == 0);1911 assert(inst.Offset / 4 <= 0xffffff);1912 w = inst.Offset / 4;1913 streamer.emitInt8(0xfa);1914 streamer.emitInt8((w >> 16) & 0xff);1915 streamer.emitInt8((w >> 8) & 0xff);1916 streamer.emitInt8((w >> 0) & 0xff);1917 break;1918 case Win64EH::UOP_Nop:1919 streamer.emitInt8(0xfb);1920 break;1921 case Win64EH::UOP_WideNop:1922 streamer.emitInt8(0xfc);1923 break;1924 case Win64EH::UOP_EndNop:1925 streamer.emitInt8(0xfd);1926 break;1927 case Win64EH::UOP_WideEndNop:1928 streamer.emitInt8(0xfe);1929 break;1930 case Win64EH::UOP_End:1931 streamer.emitInt8(0xff);1932 break;1933 case Win64EH::UOP_Custom:1934 for (i = 3; i > 0; i--)1935 if (inst.Offset & (0xffu << (8 * i)))1936 break;1937 for (; i >= 0; i--)1938 streamer.emitInt8((inst.Offset >> (8 * i)) & 0xff);1939 break;1940 }1941}1942 1943// Check if an epilog exists as a subset of the end of a prolog (backwards).1944// An epilog may end with one out of three different end opcodes; if this1945// is the first epilog that shares opcodes with the prolog, we can tolerate1946// that this opcode differs (and the caller will update the prolog to use1947// the same end opcode as the epilog). If another epilog already shares1948// opcodes with the prolog, the ending opcode must be a strict match.1949static int getARMOffsetInProlog(const std::vector<WinEH::Instruction> &Prolog,1950 const std::vector<WinEH::Instruction> &Epilog,1951 bool CanTweakProlog) {1952 // Can't find an epilog as a subset if it is longer than the prolog.1953 if (Epilog.size() > Prolog.size())1954 return -1;1955 1956 // Check that the epilog actually is a perfect match for the end (backwrds)1957 // of the prolog.1958 // If we can adjust the prolog afterwards, don't check that the end opcodes1959 // match.1960 int EndIdx = CanTweakProlog ? 1 : 0;1961 for (int I = Epilog.size() - 1; I >= EndIdx; I--) {1962 // TODO: Could also allow minor mismatches, e.g. "add sp, #16" vs1963 // "push {r0-r3}".1964 if (Prolog[I] != Epilog[Epilog.size() - 1 - I])1965 return -1;1966 }1967 1968 if (CanTweakProlog) {1969 // Check that both prolog and epilog end with an expected end opcode.1970 if (Prolog.front().Operation != Win64EH::UOP_End)1971 return -1;1972 if (Epilog.back().Operation != Win64EH::UOP_End &&1973 Epilog.back().Operation != Win64EH::UOP_EndNop &&1974 Epilog.back().Operation != Win64EH::UOP_WideEndNop)1975 return -1;1976 }1977 1978 // If the epilog was a subset of the prolog, find its offset.1979 if (Epilog.size() == Prolog.size())1980 return 0;1981 return ARMCountOfUnwindCodes(ArrayRef<WinEH::Instruction>(1982 &Prolog[Epilog.size()], Prolog.size() - Epilog.size()));1983}1984 1985static int checkARMPackedEpilog(MCStreamer &streamer, WinEH::FrameInfo *info,1986 int PrologCodeBytes) {1987 // Can only pack if there's one single epilog1988 if (info->EpilogMap.size() != 1)1989 return -1;1990 1991 const WinEH::FrameInfo::Epilog &EpilogInfo = info->EpilogMap.begin()->second;1992 // Can only pack if the epilog is unconditional1993 if (EpilogInfo.Condition != 0xe) // ARMCC::AL1994 return -1;1995 1996 const std::vector<WinEH::Instruction> &Epilog = EpilogInfo.Instructions;1997 // Make sure we have at least the trailing end opcode1998 if (info->Instructions.empty() || Epilog.empty())1999 return -1;2000 2001 // Check that the epilog actually is at the very end of the function,2002 // otherwise it can't be packed.2003 std::optional<int64_t> MaybeDistance = GetOptionalAbsDifference(2004 streamer, info->FuncletOrFuncEnd, info->EpilogMap.begin()->first);2005 if (!MaybeDistance)2006 return -1;2007 uint32_t DistanceFromEnd = (uint32_t)*MaybeDistance;2008 uint32_t InstructionBytes = ARMCountOfInstructionBytes(Epilog);2009 if (DistanceFromEnd != InstructionBytes)2010 return -1;2011 2012 int RetVal = -1;2013 // Even if we don't end up sharing opcodes with the prolog, we can still2014 // write the offset as a packed offset, if the single epilog is located at2015 // the end of the function and the offset (pointing after the prolog) fits2016 // as a packed offset.2017 if (PrologCodeBytes <= 31 &&2018 PrologCodeBytes + ARMCountOfUnwindCodes(Epilog) <= 63)2019 RetVal = PrologCodeBytes;2020 2021 int Offset =2022 getARMOffsetInProlog(info->Instructions, Epilog, /*CanTweakProlog=*/true);2023 if (Offset < 0)2024 return RetVal;2025 2026 // Check that the offset and prolog size fits in the first word; it's2027 // unclear whether the epilog count in the extension word can be taken2028 // as packed epilog offset.2029 if (Offset > 31 || PrologCodeBytes > 63)2030 return RetVal;2031 2032 // Replace the regular end opcode of the prolog with the one from the2033 // epilog.2034 info->Instructions.front() = Epilog.back();2035 2036 // As we choose to express the epilog as part of the prolog, remove the2037 // epilog from the map, so we don't try to emit its opcodes.2038 info->EpilogMap.clear();2039 return Offset;2040}2041 2042static bool parseRegMask(unsigned Mask, bool &HasLR, bool &HasR11,2043 unsigned &Folded, int &IntRegs) {2044 if (Mask & (1 << 14)) {2045 HasLR = true;2046 Mask &= ~(1 << 14);2047 }2048 if (Mask & (1 << 11)) {2049 HasR11 = true;2050 Mask &= ~(1 << 11);2051 }2052 Folded = 0;2053 IntRegs = -1;2054 if (!Mask)2055 return true;2056 int First = 0;2057 // Shift right until we have the bits at the bottom2058 while ((Mask & 1) == 0) {2059 First++;2060 Mask >>= 1;2061 }2062 if ((Mask & (Mask + 1)) != 0)2063 return false; // Not a consecutive series of bits? Can't be packed.2064 // Count the bits2065 int N = 0;2066 while (Mask & (1 << N))2067 N++;2068 if (First < 4) {2069 if (First + N < 4)2070 return false;2071 Folded = 4 - First;2072 N -= Folded;2073 First = 4;2074 }2075 if (First > 4)2076 return false; // Can't be packed2077 if (N >= 1)2078 IntRegs = N - 1;2079 return true;2080}2081 2082static bool tryARMPackedUnwind(MCStreamer &streamer, WinEH::FrameInfo *info,2083 uint32_t FuncLength) {2084 int Step = 0;2085 bool Homing = false;2086 bool HasR11 = false;2087 bool HasChain = false;2088 bool HasLR = false;2089 int IntRegs = -1; // r4 - r(4+N)2090 int FloatRegs = -1; // d8 - d(8+N)2091 unsigned PF = 0; // Number of extra pushed registers2092 unsigned StackAdjust = 0;2093 // Iterate over the prolog and check that all opcodes exactly match2094 // the canonical order and form.2095 for (const WinEH::Instruction &Inst : info->Instructions) {2096 switch (Inst.Operation) {2097 default:2098 llvm_unreachable("Unsupported ARM unwind code");2099 case Win64EH::UOP_Custom:2100 case Win64EH::UOP_AllocLarge:2101 case Win64EH::UOP_AllocHuge:2102 case Win64EH::UOP_WideAllocLarge:2103 case Win64EH::UOP_WideAllocHuge:2104 case Win64EH::UOP_SaveFRegD0D15:2105 case Win64EH::UOP_SaveFRegD16D31:2106 // Can't be packed2107 return false;2108 case Win64EH::UOP_SaveSP:2109 // Can't be packed; we can't rely on restoring sp from r11 when2110 // unwinding a packed prologue.2111 return false;2112 case Win64EH::UOP_SaveLR:2113 // Can't be present in a packed prologue2114 return false;2115 2116 case Win64EH::UOP_End:2117 case Win64EH::UOP_EndNop:2118 case Win64EH::UOP_WideEndNop:2119 if (Step != 0)2120 return false;2121 Step = 1;2122 break;2123 2124 case Win64EH::UOP_SaveRegsR4R7LR:2125 case Win64EH::UOP_WideSaveRegsR4R11LR:2126 // push {r4-r11,lr}2127 if (Step != 1 && Step != 2)2128 return false;2129 assert(Inst.Register >= 4 && Inst.Register <= 11); // r4-rX2130 assert(Inst.Offset <= 1); // Lr2131 IntRegs = Inst.Register - 4;2132 if (Inst.Register == 11) {2133 HasR11 = true;2134 IntRegs--;2135 }2136 if (Inst.Offset)2137 HasLR = true;2138 Step = 3;2139 break;2140 2141 case Win64EH::UOP_SaveRegMask:2142 if (Step == 1 && Inst.Register == 0x0f) {2143 // push {r0-r3}2144 Homing = true;2145 Step = 2;2146 break;2147 }2148 [[fallthrough]];2149 case Win64EH::UOP_WideSaveRegMask:2150 if (Step != 1 && Step != 2)2151 return false;2152 // push {r4-r9,r11,lr}2153 // push {r11,lr}2154 // push {r1-r5}2155 if (!parseRegMask(Inst.Register, HasLR, HasR11, PF, IntRegs))2156 return false;2157 Step = 3;2158 break;2159 2160 case Win64EH::UOP_Nop:2161 // mov r11, sp2162 if (Step != 3 || !HasR11 || IntRegs >= 0 || PF > 0)2163 return false;2164 HasChain = true;2165 Step = 4;2166 break;2167 case Win64EH::UOP_WideNop:2168 // add.w r11, sp, #xx2169 if (Step != 3 || !HasR11 || (IntRegs < 0 && PF == 0))2170 return false;2171 HasChain = true;2172 Step = 4;2173 break;2174 2175 case Win64EH::UOP_SaveFRegD8D15:2176 if (Step != 1 && Step != 2 && Step != 3 && Step != 4)2177 return false;2178 assert(Inst.Register >= 8 && Inst.Register <= 15);2179 if (Inst.Register == 15)2180 return false; // Can't pack this case, R==7 means no IntRegs2181 if (IntRegs >= 0)2182 return false;2183 FloatRegs = Inst.Register - 8;2184 Step = 5;2185 break;2186 2187 case Win64EH::UOP_AllocSmall:2188 case Win64EH::UOP_WideAllocMedium:2189 if (Step != 1 && Step != 2 && Step != 3 && Step != 4 && Step != 5)2190 return false;2191 if (PF > 0) // Can't have both folded and explicit stack allocation2192 return false;2193 if (Inst.Offset / 4 >= 0x3f4)2194 return false;2195 StackAdjust = Inst.Offset / 4;2196 Step = 6;2197 break;2198 }2199 }2200 if (HasR11 && !HasChain) {2201 if (IntRegs + 4 == 10) {2202 // r11 stored, but not chaining; can be packed if already saving r4-r102203 // and we can fit r11 into this range.2204 IntRegs++;2205 HasR11 = false;2206 } else2207 return false;2208 }2209 if (HasChain && !HasLR)2210 return false;2211 2212 // Packed uneind info can't express multiple epilogues.2213 if (info->EpilogMap.size() > 1)2214 return false;2215 2216 unsigned EF = 0;2217 int Ret = 0;2218 if (info->EpilogMap.size() == 0) {2219 Ret = 3; // No epilogue2220 } else {2221 // As the prologue and epilogue aren't exact mirrors of each other,2222 // we have to check the epilogue too and see if it matches what we've2223 // concluded from the prologue.2224 const WinEH::FrameInfo::Epilog &EpilogInfo =2225 info->EpilogMap.begin()->second;2226 if (EpilogInfo.Condition != 0xe) // ARMCC::AL2227 return false;2228 const std::vector<WinEH::Instruction> &Epilog = EpilogInfo.Instructions;2229 std::optional<int64_t> MaybeDistance = GetOptionalAbsDifference(2230 streamer, info->FuncletOrFuncEnd, info->EpilogMap.begin()->first);2231 if (!MaybeDistance)2232 return false;2233 uint32_t DistanceFromEnd = (uint32_t)*MaybeDistance;2234 uint32_t InstructionBytes = ARMCountOfInstructionBytes(Epilog);2235 if (DistanceFromEnd != InstructionBytes)2236 return false;2237 2238 bool GotStackAdjust = false;2239 bool GotFloatRegs = false;2240 bool GotIntRegs = false;2241 bool GotHomingRestore = false;2242 bool GotLRRestore = false;2243 bool NeedsReturn = false;2244 bool GotReturn = false;2245 2246 Step = 6;2247 for (const WinEH::Instruction &Inst : Epilog) {2248 switch (Inst.Operation) {2249 default:2250 llvm_unreachable("Unsupported ARM unwind code");2251 case Win64EH::UOP_Custom:2252 case Win64EH::UOP_AllocLarge:2253 case Win64EH::UOP_AllocHuge:2254 case Win64EH::UOP_WideAllocLarge:2255 case Win64EH::UOP_WideAllocHuge:2256 case Win64EH::UOP_SaveFRegD0D15:2257 case Win64EH::UOP_SaveFRegD16D31:2258 case Win64EH::UOP_SaveSP:2259 case Win64EH::UOP_Nop:2260 case Win64EH::UOP_WideNop:2261 // Can't be packed in an epilogue2262 return false;2263 2264 case Win64EH::UOP_AllocSmall:2265 case Win64EH::UOP_WideAllocMedium:2266 if (Inst.Offset / 4 >= 0x3f4)2267 return false;2268 if (Step == 6) {2269 if (Homing && FloatRegs < 0 && IntRegs < 0 && StackAdjust == 0 &&2270 PF == 0 && Inst.Offset == 16) {2271 GotHomingRestore = true;2272 Step = 10;2273 } else {2274 if (StackAdjust > 0) {2275 // Got stack adjust in prologue too; must match.2276 if (StackAdjust != Inst.Offset / 4)2277 return false;2278 GotStackAdjust = true;2279 } else if (PF == Inst.Offset / 4) {2280 // Folded prologue, non-folded epilogue2281 StackAdjust = Inst.Offset / 4;2282 GotStackAdjust = true;2283 } else {2284 // StackAdjust == 0 in prologue, mismatch2285 return false;2286 }2287 Step = 7;2288 }2289 } else if (Step == 7 || Step == 8 || Step == 9) {2290 if (!Homing || Inst.Offset != 16)2291 return false;2292 GotHomingRestore = true;2293 Step = 10;2294 } else2295 return false;2296 break;2297 2298 case Win64EH::UOP_SaveFRegD8D15:2299 if (Step != 6 && Step != 7)2300 return false;2301 assert(Inst.Register >= 8 && Inst.Register <= 15);2302 if (FloatRegs != (int)(Inst.Register - 8))2303 return false;2304 GotFloatRegs = true;2305 Step = 8;2306 break;2307 2308 case Win64EH::UOP_SaveRegsR4R7LR:2309 case Win64EH::UOP_WideSaveRegsR4R11LR: {2310 // push {r4-r11,lr}2311 if (Step != 6 && Step != 7 && Step != 8)2312 return false;2313 assert(Inst.Register >= 4 && Inst.Register <= 11); // r4-rX2314 assert(Inst.Offset <= 1); // Lr2315 if (Homing && HasLR) {2316 // If homing and LR is backed up, we can either restore LR here2317 // and return with Ret == 1 or 2, or return with SaveLR below2318 if (Inst.Offset) {2319 GotLRRestore = true;2320 NeedsReturn = true;2321 } else {2322 // Expecting a separate SaveLR below2323 }2324 } else {2325 if (HasLR != (Inst.Offset == 1))2326 return false;2327 }2328 GotLRRestore = Inst.Offset == 1;2329 if (IntRegs < 0) // This opcode must include r42330 return false;2331 int Expected = IntRegs;2332 if (HasChain) {2333 // Can't express r11 here unless IntRegs describe r4-r102334 if (IntRegs != 6)2335 return false;2336 Expected++;2337 }2338 if (Expected != (int)(Inst.Register - 4))2339 return false;2340 GotIntRegs = true;2341 Step = 9;2342 break;2343 }2344 2345 case Win64EH::UOP_SaveRegMask:2346 case Win64EH::UOP_WideSaveRegMask: {2347 if (Step != 6 && Step != 7 && Step != 8)2348 return false;2349 // push {r4-r9,r11,lr}2350 // push {r11,lr}2351 // push {r1-r5}2352 bool CurHasLR = false, CurHasR11 = false;2353 int Regs;2354 if (!parseRegMask(Inst.Register, CurHasLR, CurHasR11, EF, Regs))2355 return false;2356 if (EF > 0) {2357 if (EF != PF && EF != StackAdjust)2358 return false;2359 }2360 if (Homing && HasLR) {2361 // If homing and LR is backed up, we can either restore LR here2362 // and return with Ret == 1 or 2, or return with SaveLR below2363 if (CurHasLR) {2364 GotLRRestore = true;2365 NeedsReturn = true;2366 } else {2367 // Expecting a separate SaveLR below2368 }2369 } else {2370 if (CurHasLR != HasLR)2371 return false;2372 GotLRRestore = CurHasLR;2373 }2374 int Expected = IntRegs;2375 if (HasChain) {2376 // If we have chaining, the mask must have included r11.2377 if (!CurHasR11)2378 return false;2379 } else if (Expected == 7) {2380 // If we don't have chaining, the mask could still include r11,2381 // expressed as part of IntRegs Instead.2382 Expected--;2383 if (!CurHasR11)2384 return false;2385 } else {2386 // Neither HasChain nor r11 included in IntRegs, must not have r112387 // here either.2388 if (CurHasR11)2389 return false;2390 }2391 if (Expected != Regs)2392 return false;2393 GotIntRegs = true;2394 Step = 9;2395 break;2396 }2397 2398 case Win64EH::UOP_SaveLR:2399 if (Step != 6 && Step != 7 && Step != 8 && Step != 9)2400 return false;2401 if (!Homing || Inst.Offset != 20 || GotLRRestore)2402 return false;2403 GotLRRestore = true;2404 GotHomingRestore = true;2405 Step = 10;2406 break;2407 2408 case Win64EH::UOP_EndNop:2409 case Win64EH::UOP_WideEndNop:2410 GotReturn = true;2411 Ret = (Inst.Operation == Win64EH::UOP_EndNop) ? 1 : 2;2412 [[fallthrough]];2413 case Win64EH::UOP_End:2414 if (Step != 6 && Step != 7 && Step != 8 && Step != 9 && Step != 10)2415 return false;2416 Step = 11;2417 break;2418 }2419 }2420 2421 if (Step != 11)2422 return false;2423 if (StackAdjust > 0 && !GotStackAdjust && EF == 0)2424 return false;2425 if (FloatRegs >= 0 && !GotFloatRegs)2426 return false;2427 if (IntRegs >= 0 && !GotIntRegs)2428 return false;2429 if (Homing && !GotHomingRestore)2430 return false;2431 if (HasLR && !GotLRRestore)2432 return false;2433 if (NeedsReturn && !GotReturn)2434 return false;2435 }2436 2437 assert(PF == 0 || EF == 0 ||2438 StackAdjust == 0); // Can't have adjust in all three2439 if (PF > 0 || EF > 0) {2440 StackAdjust = PF > 0 ? (PF - 1) : (EF - 1);2441 assert(StackAdjust <= 3);2442 StackAdjust |= 0x3f0;2443 if (PF > 0)2444 StackAdjust |= 1 << 2;2445 if (EF > 0)2446 StackAdjust |= 1 << 3;2447 }2448 2449 assert(FuncLength <= 0x7FF && "FuncLength should have been checked earlier");2450 int Flag = info->Fragment ? 0x02 : 0x01;2451 int H = Homing ? 1 : 0;2452 int L = HasLR ? 1 : 0;2453 int C = HasChain ? 1 : 0;2454 assert(IntRegs < 0 || FloatRegs < 0);2455 unsigned Reg, R;2456 if (IntRegs >= 0) {2457 Reg = IntRegs;2458 assert(Reg <= 7);2459 R = 0;2460 } else if (FloatRegs >= 0) {2461 Reg = FloatRegs;2462 assert(Reg < 7);2463 R = 1;2464 } else {2465 // No int or float regs stored (except possibly R11,LR)2466 Reg = 7;2467 R = 1;2468 }2469 info->PackedInfo |= Flag << 0;2470 info->PackedInfo |= (FuncLength & 0x7FF) << 2;2471 info->PackedInfo |= (Ret & 0x3) << 13;2472 info->PackedInfo |= H << 15;2473 info->PackedInfo |= Reg << 16;2474 info->PackedInfo |= R << 19;2475 info->PackedInfo |= L << 20;2476 info->PackedInfo |= C << 21;2477 assert(StackAdjust <= 0x3ff);2478 info->PackedInfo |= StackAdjust << 22;2479 return true;2480}2481 2482// Populate the .xdata section. The format of .xdata on ARM is documented at2483// https://docs.microsoft.com/en-us/cpp/build/arm-exception-handling2484static void ARMEmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info,2485 bool TryPacked = true) {2486 // If this UNWIND_INFO already has a symbol, it's already been emitted.2487 if (info->Symbol)2488 return;2489 // If there's no unwind info here (not even a terminating UOP_End), the2490 // unwind info is considered bogus and skipped. If this was done in2491 // response to an explicit .seh_handlerdata, the associated trailing2492 // handler data is left orphaned in the xdata section.2493 if (info->empty()) {2494 info->EmitAttempted = true;2495 return;2496 }2497 if (info->EmitAttempted) {2498 // If we tried to emit unwind info before (due to an explicit2499 // .seh_handlerdata directive), but skipped it (because there was no2500 // valid information to emit at the time), and it later got valid unwind2501 // opcodes, we can't emit it here, because the trailing handler data2502 // was already emitted elsewhere in the xdata section.2503 streamer.getContext().reportError(2504 SMLoc(), "Earlier .seh_handlerdata for " + info->Function->getName() +2505 " skipped due to no unwind info at the time "2506 "(.seh_handlerdata too early?), but the function later "2507 "did get unwind info that can't be emitted");2508 return;2509 }2510 2511 MCContext &context = streamer.getContext();2512 MCSymbol *Label = context.createTempSymbol();2513 2514 streamer.emitValueToAlignment(Align(4));2515 streamer.emitLabel(Label);2516 info->Symbol = Label;2517 2518 if (!info->PrologEnd)2519 streamer.getContext().reportError(SMLoc(), "Prologue in " +2520 info->Function->getName() +2521 " not correctly terminated");2522 2523 if (info->PrologEnd && !info->Fragment)2524 checkARMInstructions(streamer, info->Instructions, info->Begin,2525 info->PrologEnd, info->Function->getName(),2526 "prologue");2527 for (auto &I : info->EpilogMap) {2528 MCSymbol *EpilogStart = I.first;2529 auto &Epilog = I.second;2530 checkARMInstructions(streamer, Epilog.Instructions, EpilogStart, Epilog.End,2531 info->Function->getName(), "epilogue");2532 if (Epilog.Instructions.empty() ||2533 !isARMTerminator(Epilog.Instructions.back()))2534 streamer.getContext().reportError(2535 SMLoc(), "Epilogue in " + info->Function->getName() +2536 " not correctly terminated");2537 }2538 2539 std::optional<int64_t> RawFuncLength;2540 const MCExpr *FuncLengthExpr = nullptr;2541 if (!info->FuncletOrFuncEnd) {2542 report_fatal_error("FuncletOrFuncEnd not set");2543 } else {2544 // As the size of many thumb2 instructions isn't known until later,2545 // we can't always rely on being able to calculate the absolute2546 // length of the function here. If we can't calculate it, defer it2547 // to a relocation.2548 //2549 // In such a case, we won't know if the function is too long so that2550 // the unwind info would need to be split (but this isn't implemented2551 // anyway).2552 RawFuncLength =2553 GetOptionalAbsDifference(streamer, info->FuncletOrFuncEnd, info->Begin);2554 if (!RawFuncLength)2555 FuncLengthExpr =2556 GetSubDivExpr(streamer, info->FuncletOrFuncEnd, info->Begin, 2);2557 }2558 uint32_t FuncLength = 0;2559 if (RawFuncLength)2560 FuncLength = (uint32_t)*RawFuncLength / 2;2561 if (FuncLength > 0x3FFFF)2562 report_fatal_error("SEH unwind data splitting not yet implemented");2563 uint32_t PrologCodeBytes = ARMCountOfUnwindCodes(info->Instructions);2564 uint32_t TotalCodeBytes = PrologCodeBytes;2565 2566 if (!info->HandlesExceptions && RawFuncLength && FuncLength <= 0x7ff &&2567 TryPacked) {2568 // No exception handlers; check if the prolog and epilog matches the2569 // patterns that can be described by the packed format. If we don't2570 // know the exact function length yet, we can't do this.2571 2572 // info->Symbol was already set even if we didn't actually write any2573 // unwind info there. Keep using that as indicator that this unwind2574 // info has been generated already.2575 2576 if (tryARMPackedUnwind(streamer, info, FuncLength))2577 return;2578 }2579 2580 int PackedEpilogOffset =2581 checkARMPackedEpilog(streamer, info, PrologCodeBytes);2582 2583 // Process epilogs.2584 MapVector<MCSymbol *, uint32_t> EpilogInfo;2585 // Epilogs processed so far.2586 std::vector<MCSymbol *> AddedEpilogs;2587 2588 bool CanTweakProlog = true;2589 for (auto &I : info->EpilogMap) {2590 MCSymbol *EpilogStart = I.first;2591 auto &EpilogInstrs = I.second.Instructions;2592 uint32_t CodeBytes = ARMCountOfUnwindCodes(EpilogInstrs);2593 2594 MCSymbol *MatchingEpilog =2595 FindMatchingEpilog(EpilogInstrs, AddedEpilogs, info);2596 int PrologOffset;2597 if (MatchingEpilog) {2598 assert(EpilogInfo.contains(MatchingEpilog) &&2599 "Duplicate epilog not found");2600 EpilogInfo[EpilogStart] = EpilogInfo.lookup(MatchingEpilog);2601 // Clear the unwind codes in the EpilogMap, so that they don't get output2602 // in the logic below.2603 EpilogInstrs.clear();2604 } else if ((PrologOffset = getARMOffsetInProlog(2605 info->Instructions, EpilogInstrs, CanTweakProlog)) >= 0) {2606 if (CanTweakProlog) {2607 // Replace the regular end opcode of the prolog with the one from the2608 // epilog.2609 info->Instructions.front() = EpilogInstrs.back();2610 // Later epilogs need a strict match for the end opcode.2611 CanTweakProlog = false;2612 }2613 EpilogInfo[EpilogStart] = PrologOffset;2614 // Clear the unwind codes in the EpilogMap, so that they don't get output2615 // in the logic below.2616 EpilogInstrs.clear();2617 } else {2618 EpilogInfo[EpilogStart] = TotalCodeBytes;2619 TotalCodeBytes += CodeBytes;2620 AddedEpilogs.push_back(EpilogStart);2621 }2622 }2623 2624 // Code Words, Epilog count, F, E, X, Vers, Function Length2625 uint32_t row1 = 0x0;2626 uint32_t CodeWords = TotalCodeBytes / 4;2627 uint32_t CodeWordsMod = TotalCodeBytes % 4;2628 if (CodeWordsMod)2629 CodeWords++;2630 uint32_t EpilogCount =2631 PackedEpilogOffset >= 0 ? PackedEpilogOffset : info->EpilogMap.size();2632 bool ExtensionWord = EpilogCount > 31 || CodeWords > 15;2633 if (!ExtensionWord) {2634 row1 |= (EpilogCount & 0x1F) << 23;2635 row1 |= (CodeWords & 0x0F) << 28;2636 }2637 if (info->HandlesExceptions) // X2638 row1 |= 1 << 20;2639 if (PackedEpilogOffset >= 0) // E2640 row1 |= 1 << 21;2641 if (info->Fragment) // F2642 row1 |= 1 << 22;2643 row1 |= FuncLength & 0x3FFFF;2644 if (RawFuncLength)2645 streamer.emitInt32(row1);2646 else2647 streamer.emitValue(2648 MCBinaryExpr::createOr(FuncLengthExpr,2649 MCConstantExpr::create(row1, context), context),2650 4);2651 2652 // Extended Code Words, Extended Epilog Count2653 if (ExtensionWord) {2654 // FIXME: We should be able to split unwind info into multiple sections.2655 if (CodeWords > 0xFF || EpilogCount > 0xFFFF)2656 report_fatal_error("SEH unwind data splitting not yet implemented");2657 uint32_t row2 = 0x0;2658 row2 |= (CodeWords & 0xFF) << 16;2659 row2 |= (EpilogCount & 0xFFFF);2660 streamer.emitInt32(row2);2661 }2662 2663 if (PackedEpilogOffset < 0) {2664 // Epilog Start Index, Epilog Start Offset2665 for (auto &I : EpilogInfo) {2666 MCSymbol *EpilogStart = I.first;2667 uint32_t EpilogIndex = I.second;2668 2669 std::optional<int64_t> MaybeEpilogOffset =2670 GetOptionalAbsDifference(streamer, EpilogStart, info->Begin);2671 const MCExpr *OffsetExpr = nullptr;2672 uint32_t EpilogOffset = 0;2673 if (MaybeEpilogOffset)2674 EpilogOffset = *MaybeEpilogOffset / 2;2675 else2676 OffsetExpr = GetSubDivExpr(streamer, EpilogStart, info->Begin, 2);2677 2678 assert(info->EpilogMap.contains(EpilogStart));2679 unsigned Condition = info->EpilogMap[EpilogStart].Condition;2680 assert(Condition <= 0xf);2681 2682 uint32_t row3 = EpilogOffset;2683 row3 |= Condition << 20;2684 row3 |= (EpilogIndex & 0x3FF) << 24;2685 if (MaybeEpilogOffset)2686 streamer.emitInt32(row3);2687 else2688 streamer.emitValue(2689 MCBinaryExpr::createOr(2690 OffsetExpr, MCConstantExpr::create(row3, context), context),2691 4);2692 }2693 }2694 2695 // Emit prolog unwind instructions (in reverse order).2696 uint8_t numInst = info->Instructions.size();2697 for (uint8_t c = 0; c < numInst; ++c) {2698 WinEH::Instruction inst = info->Instructions.back();2699 info->Instructions.pop_back();2700 ARMEmitUnwindCode(streamer, inst);2701 }2702 2703 // Emit epilog unwind instructions2704 for (auto &I : info->EpilogMap) {2705 auto &EpilogInstrs = I.second.Instructions;2706 for (const WinEH::Instruction &inst : EpilogInstrs)2707 ARMEmitUnwindCode(streamer, inst);2708 }2709 2710 int32_t BytesMod = CodeWords * 4 - TotalCodeBytes;2711 assert(BytesMod >= 0);2712 for (int i = 0; i < BytesMod; i++)2713 streamer.emitInt8(0xFB);2714 2715 if (info->HandlesExceptions)2716 streamer.emitValue(2717 MCSymbolRefExpr::create(info->ExceptionHandler,2718 MCSymbolRefExpr::VK_COFF_IMGREL32, context),2719 4);2720}2721 2722static void ARM64EmitRuntimeFunction(MCStreamer &streamer,2723 const WinEH::FrameInfo *info) {2724 MCContext &context = streamer.getContext();2725 2726 streamer.emitValueToAlignment(Align(4));2727 for (const auto &S : info->Segments) {2728 EmitSymbolRefWithOfs(streamer, info->Begin, S.Offset);2729 if (info->PackedInfo)2730 streamer.emitInt32(info->PackedInfo);2731 else2732 streamer.emitValue(2733 MCSymbolRefExpr::create(S.Symbol, MCSymbolRefExpr::VK_COFF_IMGREL32,2734 context),2735 4);2736 }2737}2738 2739 2740static void ARMEmitRuntimeFunction(MCStreamer &streamer,2741 const WinEH::FrameInfo *info) {2742 MCContext &context = streamer.getContext();2743 2744 streamer.emitValueToAlignment(Align(4));2745 EmitSymbolRefWithOfs(streamer, info->Begin, info->Begin);2746 if (info->PackedInfo)2747 streamer.emitInt32(info->PackedInfo);2748 else2749 streamer.emitValue(2750 MCSymbolRefExpr::create(info->Symbol, MCSymbolRefExpr::VK_COFF_IMGREL32,2751 context),2752 4);2753}2754 2755void llvm::Win64EH::ARM64UnwindEmitter::Emit(MCStreamer &Streamer) const {2756 // Emit the unwind info structs first.2757 for (const auto &CFI : Streamer.getWinFrameInfos()) {2758 WinEH::FrameInfo *Info = CFI.get();2759 if (Info->empty())2760 continue;2761 MCSection *XData = Streamer.getAssociatedXDataSection(CFI->TextSection);2762 Streamer.switchSection(XData);2763 ARM64EmitUnwindInfo(Streamer, Info);2764 }2765 2766 // Now emit RUNTIME_FUNCTION entries.2767 for (const auto &CFI : Streamer.getWinFrameInfos()) {2768 WinEH::FrameInfo *Info = CFI.get();2769 // ARM64EmitUnwindInfo above clears the info struct, so we can't check2770 // empty here. But if a Symbol is set, we should create the corresponding2771 // pdata entry.2772 if (!Info->Symbol)2773 continue;2774 MCSection *PData = Streamer.getAssociatedPDataSection(CFI->TextSection);2775 Streamer.switchSection(PData);2776 ARM64EmitRuntimeFunction(Streamer, Info);2777 }2778}2779 2780void llvm::Win64EH::ARM64UnwindEmitter::EmitUnwindInfo(MCStreamer &Streamer,2781 WinEH::FrameInfo *info,2782 bool HandlerData) const {2783 // Called if there's an .seh_handlerdata directive before the end of the2784 // function. This forces writing the xdata record already here - and2785 // in this case, the function isn't actually ended already, but the xdata2786 // record needs to know the function length. In these cases, if the funclet2787 // end hasn't been marked yet, the xdata function length won't cover the2788 // whole function, only up to this point.2789 if (!info->FuncletOrFuncEnd) {2790 Streamer.switchSection(info->TextSection);2791 info->FuncletOrFuncEnd = Streamer.emitCFILabel();2792 }2793 // Switch sections (the static function above is meant to be called from2794 // here and from Emit().2795 MCSection *XData = Streamer.getAssociatedXDataSection(info->TextSection);2796 Streamer.switchSection(XData);2797 ARM64EmitUnwindInfo(Streamer, info, /* TryPacked = */ !HandlerData);2798}2799 2800void llvm::Win64EH::ARMUnwindEmitter::Emit(MCStreamer &Streamer) const {2801 // Emit the unwind info structs first.2802 for (const auto &CFI : Streamer.getWinFrameInfos()) {2803 WinEH::FrameInfo *Info = CFI.get();2804 if (Info->empty())2805 continue;2806 MCSection *XData = Streamer.getAssociatedXDataSection(CFI->TextSection);2807 Streamer.switchSection(XData);2808 ARMEmitUnwindInfo(Streamer, Info);2809 }2810 2811 // Now emit RUNTIME_FUNCTION entries.2812 for (const auto &CFI : Streamer.getWinFrameInfos()) {2813 WinEH::FrameInfo *Info = CFI.get();2814 // ARMEmitUnwindInfo above clears the info struct, so we can't check2815 // empty here. But if a Symbol is set, we should create the corresponding2816 // pdata entry.2817 if (!Info->Symbol)2818 continue;2819 MCSection *PData = Streamer.getAssociatedPDataSection(CFI->TextSection);2820 Streamer.switchSection(PData);2821 ARMEmitRuntimeFunction(Streamer, Info);2822 }2823}2824 2825void llvm::Win64EH::ARMUnwindEmitter::EmitUnwindInfo(MCStreamer &Streamer,2826 WinEH::FrameInfo *info,2827 bool HandlerData) const {2828 // Called if there's an .seh_handlerdata directive before the end of the2829 // function. This forces writing the xdata record already here - and2830 // in this case, the function isn't actually ended already, but the xdata2831 // record needs to know the function length. In these cases, if the funclet2832 // end hasn't been marked yet, the xdata function length won't cover the2833 // whole function, only up to this point.2834 if (!info->FuncletOrFuncEnd) {2835 Streamer.switchSection(info->TextSection);2836 info->FuncletOrFuncEnd = Streamer.emitCFILabel();2837 }2838 // Switch sections (the static function above is meant to be called from2839 // here and from Emit().2840 MCSection *XData = Streamer.getAssociatedXDataSection(info->TextSection);2841 Streamer.switchSection(XData);2842 ARMEmitUnwindInfo(Streamer, info, /* TryPacked = */ !HandlerData);2843}2844