3051 lines · cpp
1//===- bolt/Target/AArch64/AArch64MCPlusBuilder.cpp -----------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file provides AArch64-specific MCPlus builder.10//11//===----------------------------------------------------------------------===//12 13#include "AArch64InstrInfo.h"14#include "AArch64MCSymbolizer.h"15#include "MCTargetDesc/AArch64AddressingModes.h"16#include "MCTargetDesc/AArch64FixupKinds.h"17#include "MCTargetDesc/AArch64MCAsmInfo.h"18#include "MCTargetDesc/AArch64MCTargetDesc.h"19#include "Utils/AArch64BaseInfo.h"20#include "bolt/Core/BinaryBasicBlock.h"21#include "bolt/Core/BinaryFunction.h"22#include "bolt/Core/MCInstUtils.h"23#include "bolt/Core/MCPlusBuilder.h"24#include "llvm/BinaryFormat/ELF.h"25#include "llvm/MC/MCContext.h"26#include "llvm/MC/MCInstBuilder.h"27#include "llvm/MC/MCInstrInfo.h"28#include "llvm/MC/MCRegister.h"29#include "llvm/MC/MCRegisterInfo.h"30#include "llvm/Support/CommandLine.h"31#include "llvm/Support/DataExtractor.h"32#include "llvm/Support/Debug.h"33#include "llvm/Support/ErrorHandling.h"34 35#define DEBUG_TYPE "mcplus"36 37using namespace llvm;38using namespace bolt;39 40namespace opts {41extern cl::OptionCategory BoltInstrCategory;42static cl::opt<bool> NoLSEAtomics(43 "no-lse-atomics",44 cl::desc("generate instrumentation code sequence without using LSE atomic "45 "instruction"),46 cl::init(false), cl::Optional, cl::cat(BoltInstrCategory));47} // namespace opts48 49namespace {50 51[[maybe_unused]] static void getSystemFlag(MCInst &Inst, MCPhysReg RegName) {52 Inst.setOpcode(AArch64::MRS);53 Inst.clear();54 Inst.addOperand(MCOperand::createReg(RegName));55 Inst.addOperand(MCOperand::createImm(AArch64SysReg::NZCV));56}57 58[[maybe_unused]] static void setSystemFlag(MCInst &Inst, MCPhysReg RegName) {59 Inst.setOpcode(AArch64::MSR);60 Inst.clear();61 Inst.addOperand(MCOperand::createImm(AArch64SysReg::NZCV));62 Inst.addOperand(MCOperand::createReg(RegName));63}64 65static void createPushRegisters(MCInst &Inst, MCPhysReg Reg1, MCPhysReg Reg2) {66 Inst.clear();67 unsigned NewOpcode = AArch64::STPXpre;68 Inst.setOpcode(NewOpcode);69 Inst.addOperand(MCOperand::createReg(AArch64::SP));70 Inst.addOperand(MCOperand::createReg(Reg1));71 Inst.addOperand(MCOperand::createReg(Reg2));72 Inst.addOperand(MCOperand::createReg(AArch64::SP));73 Inst.addOperand(MCOperand::createImm(-2));74}75 76static void createPopRegisters(MCInst &Inst, MCPhysReg Reg1, MCPhysReg Reg2) {77 Inst.clear();78 unsigned NewOpcode = AArch64::LDPXpost;79 Inst.setOpcode(NewOpcode);80 Inst.addOperand(MCOperand::createReg(AArch64::SP));81 Inst.addOperand(MCOperand::createReg(Reg1));82 Inst.addOperand(MCOperand::createReg(Reg2));83 Inst.addOperand(MCOperand::createReg(AArch64::SP));84 Inst.addOperand(MCOperand::createImm(2));85}86 87static void loadReg(MCInst &Inst, MCPhysReg To, MCPhysReg From) {88 Inst.setOpcode(AArch64::LDRXui);89 Inst.clear();90 if (From == AArch64::SP) {91 Inst.setOpcode(AArch64::LDRXpost);92 Inst.addOperand(MCOperand::createReg(From));93 Inst.addOperand(MCOperand::createReg(To));94 Inst.addOperand(MCOperand::createReg(From));95 Inst.addOperand(MCOperand::createImm(16));96 } else {97 Inst.addOperand(MCOperand::createReg(To));98 Inst.addOperand(MCOperand::createReg(From));99 Inst.addOperand(MCOperand::createImm(0));100 }101}102 103static void storeReg(MCInst &Inst, MCPhysReg From, MCPhysReg To) {104 Inst.setOpcode(AArch64::STRXui);105 Inst.clear();106 if (To == AArch64::SP) {107 Inst.setOpcode(AArch64::STRXpre);108 Inst.addOperand(MCOperand::createReg(To));109 Inst.addOperand(MCOperand::createReg(From));110 Inst.addOperand(MCOperand::createReg(To));111 Inst.addOperand(MCOperand::createImm(-16));112 } else {113 Inst.addOperand(MCOperand::createReg(From));114 Inst.addOperand(MCOperand::createReg(To));115 Inst.addOperand(MCOperand::createImm(0));116 }117}118 119static void atomicAdd(MCInst &Inst, MCPhysReg RegTo, MCPhysReg RegCnt) {120 assert(!opts::NoLSEAtomics && "Supports only ARM with LSE extension");121 Inst.setOpcode(AArch64::LDADDX);122 Inst.clear();123 Inst.addOperand(MCOperand::createReg(AArch64::XZR));124 Inst.addOperand(MCOperand::createReg(RegCnt));125 Inst.addOperand(MCOperand::createReg(RegTo));126}127 128static void createMovz(MCInst &Inst, MCPhysReg Reg, uint64_t Imm) {129 assert(Imm <= UINT16_MAX && "Invalid Imm size");130 Inst.clear();131 Inst.setOpcode(AArch64::MOVZXi);132 Inst.addOperand(MCOperand::createReg(Reg));133 Inst.addOperand(MCOperand::createImm(Imm & 0xFFFF));134 Inst.addOperand(MCOperand::createImm(0));135}136 137static InstructionListType createIncMemory(MCPhysReg RegTo, MCPhysReg RegTmp) {138 InstructionListType Insts;139 Insts.emplace_back();140 createMovz(Insts.back(), RegTmp, 1);141 Insts.emplace_back();142 atomicAdd(Insts.back(), RegTo, RegTmp);143 return Insts;144}145 146class AArch64MCPlusBuilder : public MCPlusBuilder {147public:148 using MCPlusBuilder::MCPlusBuilder;149 150 BinaryFunction *InstrCounterIncrFunc{nullptr};151 152 std::unique_ptr<MCSymbolizer>153 createTargetSymbolizer(BinaryFunction &Function,154 bool CreateNewSymbols) const override {155 return std::make_unique<AArch64MCSymbolizer>(Function, CreateNewSymbols);156 }157 158 MCPhysReg getStackPointer() const override { return AArch64::SP; }159 MCPhysReg getFramePointer() const override { return AArch64::FP; }160 161 bool isBreakpoint(const MCInst &Inst) const override {162 return Inst.getOpcode() == AArch64::BRK;163 }164 165 bool isPush(const MCInst &Inst) const override {166 return isStoreToStack(Inst);167 }168 169 bool isPop(const MCInst &Inst) const override {170 return isLoadFromStack(Inst);171 }172 173 // We look for instructions that load from stack or make stack pointer174 // adjustment, and assume the basic block is an epilogue if and only if175 // such instructions are present and also immediately precede the branch176 // instruction that ends the basic block.177 bool isEpilogue(const BinaryBasicBlock &BB) const override {178 if (BB.succ_size())179 return false;180 181 bool SeenLoadFromStack = false;182 bool SeenStackPointerAdjustment = false;183 for (const MCInst &Instr : BB) {184 // Skip CFI pseudo instruction.185 if (isCFI(Instr))186 continue;187 188 bool IsPop = isPop(Instr);189 // A load from stack instruction could do SP adjustment in pre-index or190 // post-index form, which we can skip to check for epilogue recognition191 // purpose.192 bool IsSPAdj = (isADD(Instr) || isMOVW(Instr)) &&193 Instr.getOperand(0).isReg() &&194 Instr.getOperand(0).getReg() == AArch64::SP;195 SeenLoadFromStack |= IsPop;196 SeenStackPointerAdjustment |= IsSPAdj;197 198 if (!SeenLoadFromStack && !SeenStackPointerAdjustment)199 continue;200 if (IsPop || IsSPAdj || isPAuthOnLR(Instr))201 continue;202 if (isReturn(Instr))203 return true;204 if (isBranch(Instr))205 break;206 207 // Any previously seen load from stack or stack adjustment instruction208 // is definitely not part of epilogue code sequence, so reset these two.209 SeenLoadFromStack = false;210 SeenStackPointerAdjustment = false;211 }212 return SeenLoadFromStack || SeenStackPointerAdjustment;213 }214 215 void createCall(MCInst &Inst, const MCSymbol *Target,216 MCContext *Ctx) override {217 createDirectCall(Inst, Target, Ctx, false);218 }219 220 bool convertTailCallToCall(MCInst &Inst) override {221 int NewOpcode;222 switch (Inst.getOpcode()) {223 default:224 return false;225 case AArch64::B:226 NewOpcode = AArch64::BL;227 break;228 case AArch64::BR:229 NewOpcode = AArch64::BLR;230 break;231 }232 233 Inst.setOpcode(NewOpcode);234 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall);235 clearOffset(Inst);236 return true;237 }238 239 bool equals(const MCSpecifierExpr &A, const MCSpecifierExpr &B,240 CompFuncTy Comp) const override {241 if (A.getSpecifier() != B.getSpecifier())242 return false;243 244 return MCPlusBuilder::equals(*A.getSubExpr(), *B.getSubExpr(), Comp);245 }246 247 bool shortenInstruction(MCInst &, const MCSubtargetInfo &) const override {248 return false;249 }250 251 SmallVector<MCPhysReg> getTrustedLiveInRegs() const override {252 return {AArch64::LR};253 }254 255 std::optional<MCPhysReg>256 getWrittenAuthenticatedReg(const MCInst &Inst,257 bool &IsChecked) const override {258 IsChecked = false;259 switch (Inst.getOpcode()) {260 case AArch64::AUTIAZ:261 case AArch64::AUTIBZ:262 case AArch64::AUTIASP:263 case AArch64::AUTIBSP:264 case AArch64::AUTIASPPCi:265 case AArch64::AUTIBSPPCi:266 case AArch64::AUTIASPPCr:267 case AArch64::AUTIBSPPCr:268 return AArch64::LR;269 case AArch64::AUTIA1716:270 case AArch64::AUTIB1716:271 case AArch64::AUTIA171615:272 case AArch64::AUTIB171615:273 return AArch64::X17;274 case AArch64::AUTIA:275 case AArch64::AUTIB:276 case AArch64::AUTDA:277 case AArch64::AUTDB:278 case AArch64::AUTIZA:279 case AArch64::AUTIZB:280 case AArch64::AUTDZA:281 case AArch64::AUTDZB:282 return Inst.getOperand(0).getReg();283 case AArch64::LDRAAwriteback:284 case AArch64::LDRABwriteback:285 // Note that LDRA(A|B)indexed are not listed here, as they do not write286 // an authenticated pointer back to the register.287 IsChecked = true;288 return Inst.getOperand(2).getReg();289 default:290 return std::nullopt;291 }292 }293 294 bool isPSignOnLR(const MCInst &Inst) const override {295 std::optional<MCPhysReg> SignReg = getSignedReg(Inst);296 return SignReg && *SignReg == AArch64::LR;297 }298 299 bool isPAuthOnLR(const MCInst &Inst) const override {300 // LDR(A|B) should not be covered.301 bool IsChecked;302 std::optional<MCPhysReg> AuthReg =303 getWrittenAuthenticatedReg(Inst, IsChecked);304 return !IsChecked && AuthReg && *AuthReg == AArch64::LR;305 }306 307 bool isPAuthAndRet(const MCInst &Inst) const override {308 return Inst.getOpcode() == AArch64::RETAA ||309 Inst.getOpcode() == AArch64::RETAB ||310 Inst.getOpcode() == AArch64::RETAASPPCi ||311 Inst.getOpcode() == AArch64::RETABSPPCi ||312 Inst.getOpcode() == AArch64::RETAASPPCr ||313 Inst.getOpcode() == AArch64::RETABSPPCr;314 }315 316 void createMatchingAuth(const MCInst &AuthAndRet, MCInst &Auth) override {317 Auth.clear();318 Auth.setOperands(AuthAndRet.getOperands());319 switch (AuthAndRet.getOpcode()) {320 case AArch64::RETAA:321 Auth.setOpcode(AArch64::AUTIASP);322 break;323 case AArch64::RETAB:324 Auth.setOpcode(AArch64::AUTIBSP);325 break;326 case AArch64::RETAASPPCi:327 Auth.setOpcode(AArch64::AUTIASPPCi);328 break;329 case AArch64::RETABSPPCi:330 Auth.setOpcode(AArch64::AUTIBSPPCi);331 break;332 case AArch64::RETAASPPCr:333 Auth.setOpcode(AArch64::AUTIASPPCr);334 break;335 case AArch64::RETABSPPCr:336 Auth.setOpcode(AArch64::AUTIBSPPCr);337 break;338 default:339 llvm_unreachable("Unhandled fused pauth-and-return instruction");340 }341 }342 343 std::optional<MCPhysReg> getSignedReg(const MCInst &Inst) const override {344 switch (Inst.getOpcode()) {345 case AArch64::PACIA:346 case AArch64::PACIB:347 case AArch64::PACDA:348 case AArch64::PACDB:349 case AArch64::PACIZA:350 case AArch64::PACIZB:351 case AArch64::PACDZA:352 case AArch64::PACDZB:353 return Inst.getOperand(0).getReg();354 case AArch64::PACIAZ:355 case AArch64::PACIBZ:356 case AArch64::PACIASP:357 case AArch64::PACIBSP:358 case AArch64::PACIASPPC:359 case AArch64::PACIBSPPC:360 case AArch64::PACNBIASPPC:361 case AArch64::PACNBIBSPPC:362 return AArch64::LR;363 case AArch64::PACIA1716:364 case AArch64::PACIB1716:365 case AArch64::PACIA171615:366 case AArch64::PACIB171615:367 return AArch64::X17;368 default:369 return std::nullopt;370 }371 }372 373 std::optional<MCPhysReg>374 getRegUsedAsRetDest(const MCInst &Inst,375 bool &IsAuthenticatedInternally) const override {376 assert(isReturn(Inst));377 switch (Inst.getOpcode()) {378 case AArch64::RET:379 IsAuthenticatedInternally = false;380 return Inst.getOperand(0).getReg();381 382 case AArch64::RETAA:383 case AArch64::RETAB:384 case AArch64::RETAASPPCi:385 case AArch64::RETABSPPCi:386 case AArch64::RETAASPPCr:387 case AArch64::RETABSPPCr:388 IsAuthenticatedInternally = true;389 return AArch64::LR;390 case AArch64::ERET:391 case AArch64::ERETAA:392 case AArch64::ERETAB:393 // The ERET* instructions use either register ELR_EL1, ELR_EL2 or394 // ELR_EL3, depending on the current Exception Level at run-time.395 //396 // Furthermore, these registers are not modelled by LLVM as a regular397 // MCPhysReg, so there is no way to indicate that through the current API.398 return std::nullopt;399 default:400 llvm_unreachable("Unhandled return instruction");401 }402 }403 404 MCPhysReg getRegUsedAsIndirectBranchDest(405 const MCInst &Inst, bool &IsAuthenticatedInternally) const override {406 assert(isIndirectCall(Inst) || isIndirectBranch(Inst));407 408 switch (Inst.getOpcode()) {409 case AArch64::BR:410 case AArch64::BLR:411 IsAuthenticatedInternally = false;412 return Inst.getOperand(0).getReg();413 case AArch64::BRAA:414 case AArch64::BRAB:415 case AArch64::BRAAZ:416 case AArch64::BRABZ:417 case AArch64::BLRAA:418 case AArch64::BLRAB:419 case AArch64::BLRAAZ:420 case AArch64::BLRABZ:421 IsAuthenticatedInternally = true;422 return Inst.getOperand(0).getReg();423 default:424 llvm_unreachable("Unhandled indirect branch or call");425 }426 }427 428 std::optional<MCPhysReg>429 getMaterializedAddressRegForPtrAuth(const MCInst &Inst) const override {430 switch (Inst.getOpcode()) {431 case AArch64::ADR:432 case AArch64::ADRP:433 // These instructions produce an address value based on the information434 // encoded into the instruction itself (which should reside in a read-only435 // code memory) and the value of PC register (that is, the location of436 // this instruction), so the produced value is not attacker-controlled.437 return Inst.getOperand(0).getReg();438 default:439 return std::nullopt;440 }441 }442 443 std::optional<std::pair<MCPhysReg, MCPhysReg>>444 analyzeAddressArithmeticsForPtrAuth(const MCInst &Inst) const override {445 switch (Inst.getOpcode()) {446 default:447 return std::nullopt;448 case AArch64::ADDXri:449 case AArch64::SUBXri:450 // The immediate addend is encoded into the instruction itself, so it is451 // not attacker-controlled under Pointer Authentication threat model.452 return std::make_pair(Inst.getOperand(0).getReg(),453 Inst.getOperand(1).getReg());454 case AArch64::ORRXrs:455 // "mov Xd, Xm" is equivalent to "orr Xd, XZR, Xm, lsl #0"456 if (Inst.getOperand(1).getReg() != AArch64::XZR ||457 Inst.getOperand(3).getImm() != 0)458 return std::nullopt;459 460 return std::make_pair(Inst.getOperand(0).getReg(),461 Inst.getOperand(2).getReg());462 }463 }464 465 std::optional<std::pair<MCPhysReg, MCInst *>>466 getAuthCheckedReg(BinaryBasicBlock &BB) const override {467 // Match several possible hard-coded sequences of instructions which can be468 // emitted by LLVM backend to check that the authenticated pointer is469 // correct (see AArch64AsmPrinter::emitPtrauthCheckAuthenticatedValue).470 //471 // This function only matches sequences involving branch instructions.472 // All these sequences have the form:473 //474 // (0) ... regular code that authenticates a pointer in Xn ...475 // (1) analyze Xn476 // (2) branch to .Lon_success if the pointer is correct477 // (3) BRK #imm (fall-through basic block)478 //479 // In the above pseudocode, (1) + (2) is one of the following sequences:480 //481 // - eor Xtmp, Xn, Xn, lsl #1482 // tbz Xtmp, #62, .Lon_success483 //484 // - mov Xtmp, Xn485 // xpac(i|d) Xn (or xpaclri if Xn is LR)486 // cmp Xtmp, Xn487 // b.eq .Lon_success488 //489 // Note that any branch destination operand is accepted as .Lon_success -490 // it is the responsibility of the caller of getAuthCheckedReg to inspect491 // the list of successors of this basic block as appropriate.492 493 // Any of the above code sequences assume the fall-through basic block494 // is a dead-end trap instruction.495 const BinaryBasicBlock *BreakBB = BB.getFallthrough();496 if (!BreakBB || BreakBB->empty() || !isTrap(BreakBB->front()))497 return std::nullopt;498 499 // Iterate over the instructions of BB in reverse order, matching opcodes500 // and operands.501 502 auto It = BB.end();503 auto StepBack = [&]() {504 while (It != BB.begin()) {505 --It;506 // Skip any CFI instructions, but no other pseudos are expected here.507 if (!isCFI(*It))508 return true;509 }510 return false;511 };512 // Step to the last non-CFI instruction.513 if (!StepBack())514 return std::nullopt;515 516 using namespace llvm::bolt::LowLevelInstMatcherDSL;517 Reg TestedReg;518 Reg ScratchReg;519 520 if (matchInst(*It, AArch64::Bcc, Imm(AArch64CC::EQ) /*, .Lon_success*/)) {521 if (!StepBack() || !matchInst(*It, AArch64::SUBSXrs, Reg(AArch64::XZR),522 TestedReg, ScratchReg, Imm(0)))523 return std::nullopt;524 525 // Either XPAC(I|D) ScratchReg, ScratchReg526 // or XPACLRI527 if (!StepBack())528 return std::nullopt;529 if (matchInst(*It, AArch64::XPACLRI)) {530 // No operands to check, but using XPACLRI forces TestedReg to be X30.531 if (TestedReg.get() != AArch64::LR)532 return std::nullopt;533 } else if (!matchInst(*It, AArch64::XPACI, ScratchReg, ScratchReg) &&534 !matchInst(*It, AArch64::XPACD, ScratchReg, ScratchReg)) {535 return std::nullopt;536 }537 538 if (!StepBack() || !matchInst(*It, AArch64::ORRXrs, ScratchReg,539 Reg(AArch64::XZR), TestedReg, Imm(0)))540 return std::nullopt;541 542 return std::make_pair(TestedReg.get(), &*It);543 }544 545 if (matchInst(*It, AArch64::TBZX, ScratchReg, Imm(62) /*, .Lon_success*/)) {546 if (!StepBack() || !matchInst(*It, AArch64::EORXrs, ScratchReg, TestedReg,547 TestedReg, Imm(1)))548 return std::nullopt;549 550 return std::make_pair(TestedReg.get(), &*It);551 }552 553 return std::nullopt;554 }555 556 std::optional<MCPhysReg> getAuthCheckedReg(const MCInst &Inst,557 bool MayOverwrite) const override {558 // Cannot trivially reuse AArch64InstrInfo::getMemOperandWithOffsetWidth()559 // method as it accepts an instance of MachineInstr, not MCInst.560 const MCInstrDesc &Desc = Info->get(Inst.getOpcode());561 562 // If signing oracles are considered, the particular value left in the base563 // register after this instruction is important. This function checks that564 // if the base register was overwritten, it is due to address write-back:565 //566 // ; good:567 // autdza x1 ; x1 is authenticated (may fail)568 // ldr x0, [x1, #8] ; x1 is checked and not changed569 // pacdzb x1570 //571 // ; also good:572 // autdza x1573 // ldr x0, [x1, #8]! ; x1 is checked and incremented by 8574 // pacdzb x1575 //576 // ; bad (the value being signed is not the authenticated one):577 // autdza x1578 // ldr x1, [x1, #8] ; x1 is overwritten with an unrelated value579 // pacdzb x1580 //581 // ; also bad:582 // autdza x1583 // pacdzb x1 ; possibly signing the result of failed authentication584 //585 // Note that this function is not needed for authentication oracles, as the586 // particular value left in the register after a successful memory access587 // is not important.588 auto ClobbersBaseRegExceptWriteback = [&](unsigned BaseRegUseIndex) {589 // FIXME: Compute the indices of address operands (base reg and written-590 // back result) in AArch64InstrInfo instead of this ad-hoc code.591 MCPhysReg BaseReg = Inst.getOperand(BaseRegUseIndex).getReg();592 unsigned WrittenBackDefIndex = Desc.getOperandConstraint(593 BaseRegUseIndex, MCOI::OperandConstraint::TIED_TO);594 595 for (unsigned DefIndex = 0; DefIndex < Desc.getNumDefs(); ++DefIndex) {596 // Address write-back is permitted:597 //598 // autda x0, x2599 // ; x0 is authenticated600 // ldr x1, [x0, #8]!601 // ; x0 is trusted (as authenticated and checked)602 if (DefIndex == WrittenBackDefIndex)603 continue;604 605 // Any other overwriting is not permitted:606 //607 // autda x0, x2608 // ; x0 is authenticated609 // ldr w0, [x0]610 // ; x0 is not authenticated anymore611 if (RegInfo->regsOverlap(Inst.getOperand(DefIndex).getReg(), BaseReg))612 return true;613 }614 615 return false;616 };617 618 // FIXME: Not all load instructions are handled by this->mayLoad(Inst).619 // On the other hand, MCInstrDesc::mayLoad() is permitted to return620 // true for non-load instructions (such as AArch64::HINT) which621 // would result in false negatives.622 if (mayLoad(Inst)) {623 // The first Use operand is the base address register.624 unsigned BaseRegIndex = Desc.getNumDefs();625 626 // Reject non-immediate offsets, as adding a 64-bit register can change627 // the resulting address arbitrarily.628 for (unsigned I = BaseRegIndex + 1, E = Desc.getNumOperands(); I < E; ++I)629 if (Inst.getOperand(I).isReg())630 return std::nullopt;631 632 if (!MayOverwrite && ClobbersBaseRegExceptWriteback(BaseRegIndex))633 return std::nullopt;634 635 return Inst.getOperand(BaseRegIndex).getReg();636 }637 638 // Store instructions are not handled yet, as they are not important for639 // pauthtest ABI. Though, they could be handled similar to loads, if needed.640 641 return std::nullopt;642 }643 644 bool isADRP(const MCInst &Inst) const override {645 return Inst.getOpcode() == AArch64::ADRP;646 }647 648 bool isADR(const MCInst &Inst) const override {649 return Inst.getOpcode() == AArch64::ADR;650 }651 652 bool isAddXri(const MCInst &Inst) const override {653 return Inst.getOpcode() == AArch64::ADDXri;654 }655 656 bool isLDRWl(const MCInst &Inst) const override {657 return Inst.getOpcode() == AArch64::LDRWl;658 }659 660 bool isLDRXl(const MCInst &Inst) const override {661 return Inst.getOpcode() == AArch64::LDRXl;662 }663 664 MCPhysReg getADRReg(const MCInst &Inst) const {665 assert((isADR(Inst) || isADRP(Inst)) && "Not an ADR instruction");666 assert(MCPlus::getNumPrimeOperands(Inst) != 0 &&667 "No operands for ADR instruction");668 assert(Inst.getOperand(0).isReg() &&669 "Unexpected operand in ADR instruction");670 return Inst.getOperand(0).getReg();671 }672 673 InstructionListType undoAdrpAddRelaxation(const MCInst &ADRInst,674 MCContext *Ctx) const override {675 assert(isADR(ADRInst) && "ADR instruction expected");676 677 const MCPhysReg Reg = getADRReg(ADRInst);678 const MCSymbol *Target = getTargetSymbol(ADRInst);679 const uint64_t Addend = getTargetAddend(ADRInst);680 return materializeAddress(Target, Ctx, Reg, Addend);681 }682 683 InstructionListType createAdrpLdr(const MCInst &LDRInst,684 MCContext *Ctx) const override {685 assert((isLDRXl(LDRInst) || isLDRWl(LDRInst)) &&686 "LDR (literal, 32 or 64-bit integer load) instruction expected");687 assert(LDRInst.getOperand(0).isReg() &&688 "unexpected operand in LDR instruction");689 const MCPhysReg DataReg = LDRInst.getOperand(0).getReg();690 const MCPhysReg AddrReg =691 isLDRXl(LDRInst) ? DataReg692 : (MCPhysReg)RegInfo->getMatchingSuperReg(693 DataReg, AArch64::sub_32,694 &RegInfo->getRegClass(AArch64::GPR64RegClassID));695 const MCSymbol *Target = getTargetSymbol(LDRInst, 1);696 assert(Target && "missing target symbol in LDR instruction");697 698 InstructionListType Insts(2);699 Insts[0].setOpcode(AArch64::ADRP);700 Insts[0].clear();701 Insts[0].addOperand(MCOperand::createReg(AddrReg));702 Insts[0].addOperand(MCOperand::createImm(0));703 setOperandToSymbolRef(Insts[0], /* OpNum */ 1, Target, 0, Ctx,704 ELF::R_AARCH64_NONE);705 Insts[1].setOpcode(isLDRXl(LDRInst) ? AArch64::LDRXui : AArch64::LDRWui);706 Insts[1].clear();707 Insts[1].addOperand(MCOperand::createReg(DataReg));708 Insts[1].addOperand(MCOperand::createReg(AddrReg));709 Insts[1].addOperand(MCOperand::createImm(0));710 Insts[1].addOperand(MCOperand::createImm(0));711 setOperandToSymbolRef(Insts[1], /* OpNum */ 2, Target, 0, Ctx,712 isLDRXl(LDRInst) ? ELF::R_AARCH64_LDST64_ABS_LO12_NC713 : ELF::R_AARCH64_LDST32_ABS_LO12_NC);714 return Insts;715 }716 717 bool isTB(const MCInst &Inst) const {718 return (Inst.getOpcode() == AArch64::TBNZW ||719 Inst.getOpcode() == AArch64::TBNZX ||720 Inst.getOpcode() == AArch64::TBZW ||721 Inst.getOpcode() == AArch64::TBZX);722 }723 724 bool isCB(const MCInst &Inst) const {725 return (Inst.getOpcode() == AArch64::CBNZW ||726 Inst.getOpcode() == AArch64::CBNZX ||727 Inst.getOpcode() == AArch64::CBZW ||728 Inst.getOpcode() == AArch64::CBZX);729 }730 731 bool isMOVW(const MCInst &Inst) const override {732 return (Inst.getOpcode() == AArch64::MOVKWi ||733 Inst.getOpcode() == AArch64::MOVKXi ||734 Inst.getOpcode() == AArch64::MOVNWi ||735 Inst.getOpcode() == AArch64::MOVNXi ||736 Inst.getOpcode() == AArch64::MOVZXi ||737 Inst.getOpcode() == AArch64::MOVZWi);738 }739 740 bool isADD(const MCInst &Inst) const {741 return (Inst.getOpcode() == AArch64::ADDSWri ||742 Inst.getOpcode() == AArch64::ADDSWrr ||743 Inst.getOpcode() == AArch64::ADDSWrs ||744 Inst.getOpcode() == AArch64::ADDSWrx ||745 Inst.getOpcode() == AArch64::ADDSXri ||746 Inst.getOpcode() == AArch64::ADDSXrr ||747 Inst.getOpcode() == AArch64::ADDSXrs ||748 Inst.getOpcode() == AArch64::ADDSXrx ||749 Inst.getOpcode() == AArch64::ADDSXrx64 ||750 Inst.getOpcode() == AArch64::ADDWri ||751 Inst.getOpcode() == AArch64::ADDWrr ||752 Inst.getOpcode() == AArch64::ADDWrs ||753 Inst.getOpcode() == AArch64::ADDWrx ||754 Inst.getOpcode() == AArch64::ADDXri ||755 Inst.getOpcode() == AArch64::ADDXrr ||756 Inst.getOpcode() == AArch64::ADDXrs ||757 Inst.getOpcode() == AArch64::ADDXrx ||758 Inst.getOpcode() == AArch64::ADDXrx64);759 }760 761 bool isLDRB(const MCInst &Inst) const {762 const unsigned opcode = Inst.getOpcode();763 switch (opcode) {764 case AArch64::LDRBpost:765 case AArch64::LDRBBpost:766 case AArch64::LDRBBpre:767 case AArch64::LDRBBroW:768 case AArch64::LDRBroW:769 case AArch64::LDRBroX:770 case AArch64::LDRBBroX:771 case AArch64::LDRBBui:772 case AArch64::LDRBui:773 case AArch64::LDRBpre:774 case AArch64::LDRSBWpost:775 case AArch64::LDRSBWpre:776 case AArch64::LDRSBWroW:777 case AArch64::LDRSBWroX:778 case AArch64::LDRSBWui:779 case AArch64::LDRSBXpost:780 case AArch64::LDRSBXpre:781 case AArch64::LDRSBXroW:782 case AArch64::LDRSBXroX:783 case AArch64::LDRSBXui:784 case AArch64::LDURBi:785 case AArch64::LDURBBi:786 case AArch64::LDURSBWi:787 case AArch64::LDURSBXi:788 case AArch64::LDTRBi:789 case AArch64::LDTRSBWi:790 case AArch64::LDTRSBXi:791 return true;792 default:793 break;794 }795 796 return false;797 }798 799 bool isLDRH(const MCInst &Inst) const {800 const unsigned opcode = Inst.getOpcode();801 switch (opcode) {802 case AArch64::LDRHpost:803 case AArch64::LDRHHpost:804 case AArch64::LDRHHpre:805 case AArch64::LDRHroW:806 case AArch64::LDRHHroW:807 case AArch64::LDRHroX:808 case AArch64::LDRHHroX:809 case AArch64::LDRHHui:810 case AArch64::LDRHui:811 case AArch64::LDRHpre:812 case AArch64::LDRSHWpost:813 case AArch64::LDRSHWpre:814 case AArch64::LDRSHWroW:815 case AArch64::LDRSHWroX:816 case AArch64::LDRSHWui:817 case AArch64::LDRSHXpost:818 case AArch64::LDRSHXpre:819 case AArch64::LDRSHXroW:820 case AArch64::LDRSHXroX:821 case AArch64::LDRSHXui:822 case AArch64::LDURHi:823 case AArch64::LDURHHi:824 case AArch64::LDURSHWi:825 case AArch64::LDURSHXi:826 case AArch64::LDTRHi:827 case AArch64::LDTRSHWi:828 case AArch64::LDTRSHXi:829 return true;830 default:831 break;832 }833 834 return false;835 }836 837 bool isLDRW(const MCInst &Inst) const {838 const unsigned opcode = Inst.getOpcode();839 switch (opcode) {840 case AArch64::LDRWpost:841 case AArch64::LDRWpre:842 case AArch64::LDRWroW:843 case AArch64::LDRWroX:844 case AArch64::LDRWui:845 case AArch64::LDRWl:846 case AArch64::LDRSWl:847 case AArch64::LDURWi:848 case AArch64::LDRSWpost:849 case AArch64::LDRSWpre:850 case AArch64::LDRSWroW:851 case AArch64::LDRSWroX:852 case AArch64::LDRSWui:853 case AArch64::LDURSWi:854 case AArch64::LDTRWi:855 case AArch64::LDTRSWi:856 case AArch64::LDPWi:857 case AArch64::LDPWpost:858 case AArch64::LDPWpre:859 case AArch64::LDPSWi:860 case AArch64::LDPSWpost:861 case AArch64::LDPSWpre:862 case AArch64::LDNPWi:863 return true;864 default:865 break;866 }867 868 return false;869 }870 871 bool isLDRX(const MCInst &Inst) const {872 const unsigned opcode = Inst.getOpcode();873 switch (opcode) {874 case AArch64::LDRXpost:875 case AArch64::LDRXpre:876 case AArch64::LDRXroW:877 case AArch64::LDRXroX:878 case AArch64::LDRXui:879 case AArch64::LDRXl:880 case AArch64::LDURXi:881 case AArch64::LDTRXi:882 case AArch64::LDNPXi:883 case AArch64::LDPXi:884 case AArch64::LDPXpost:885 case AArch64::LDPXpre:886 return true;887 default:888 break;889 }890 891 return false;892 }893 894 bool isLDRS(const MCInst &Inst) const {895 const unsigned opcode = Inst.getOpcode();896 switch (opcode) {897 case AArch64::LDRSl:898 case AArch64::LDRSui:899 case AArch64::LDRSroW:900 case AArch64::LDRSroX:901 case AArch64::LDURSi:902 case AArch64::LDPSi:903 case AArch64::LDNPSi:904 case AArch64::LDRSpre:905 case AArch64::LDRSpost:906 case AArch64::LDPSpost:907 case AArch64::LDPSpre:908 return true;909 default:910 break;911 }912 913 return false;914 }915 916 bool isLDRD(const MCInst &Inst) const {917 const unsigned opcode = Inst.getOpcode();918 switch (opcode) {919 case AArch64::LDRDl:920 case AArch64::LDRDui:921 case AArch64::LDRDpre:922 case AArch64::LDRDpost:923 case AArch64::LDRDroW:924 case AArch64::LDRDroX:925 case AArch64::LDURDi:926 case AArch64::LDPDi:927 case AArch64::LDNPDi:928 case AArch64::LDPDpost:929 case AArch64::LDPDpre:930 return true;931 default:932 break;933 }934 935 return false;936 }937 938 bool isLDRQ(const MCInst &Inst) const {939 const unsigned opcode = Inst.getOpcode();940 switch (opcode) {941 case AArch64::LDRQui:942 case AArch64::LDRQl:943 case AArch64::LDRQpre:944 case AArch64::LDRQpost:945 case AArch64::LDRQroW:946 case AArch64::LDRQroX:947 case AArch64::LDURQi:948 case AArch64::LDPQi:949 case AArch64::LDNPQi:950 case AArch64::LDPQpost:951 case AArch64::LDPQpre:952 return true;953 default:954 break;955 }956 957 return false;958 }959 960 bool isBRA(const MCInst &Inst) const {961 switch (Inst.getOpcode()) {962 case AArch64::BRAA:963 case AArch64::BRAB:964 case AArch64::BRAAZ:965 case AArch64::BRABZ:966 return true;967 default:968 return false;969 }970 }971 972 bool mayLoad(const MCInst &Inst) const override {973 // FIXME: Probably this could be tablegen-erated not to miss any existing974 // or future opcodes.975 return isLDRB(Inst) || isLDRH(Inst) || isLDRW(Inst) || isLDRX(Inst) ||976 isLDRQ(Inst) || isLDRD(Inst) || isLDRS(Inst);977 }978 979 bool isAArch64ExclusiveLoad(const MCInst &Inst) const override {980 return (Inst.getOpcode() == AArch64::LDXPX ||981 Inst.getOpcode() == AArch64::LDXPW ||982 Inst.getOpcode() == AArch64::LDXRX ||983 Inst.getOpcode() == AArch64::LDXRW ||984 Inst.getOpcode() == AArch64::LDXRH ||985 Inst.getOpcode() == AArch64::LDXRB ||986 Inst.getOpcode() == AArch64::LDAXPX ||987 Inst.getOpcode() == AArch64::LDAXPW ||988 Inst.getOpcode() == AArch64::LDAXRX ||989 Inst.getOpcode() == AArch64::LDAXRW ||990 Inst.getOpcode() == AArch64::LDAXRH ||991 Inst.getOpcode() == AArch64::LDAXRB);992 }993 994 bool isAArch64ExclusiveStore(const MCInst &Inst) const override {995 return (Inst.getOpcode() == AArch64::STXPX ||996 Inst.getOpcode() == AArch64::STXPW ||997 Inst.getOpcode() == AArch64::STXRX ||998 Inst.getOpcode() == AArch64::STXRW ||999 Inst.getOpcode() == AArch64::STXRH ||1000 Inst.getOpcode() == AArch64::STXRB ||1001 Inst.getOpcode() == AArch64::STLXPX ||1002 Inst.getOpcode() == AArch64::STLXPW ||1003 Inst.getOpcode() == AArch64::STLXRX ||1004 Inst.getOpcode() == AArch64::STLXRW ||1005 Inst.getOpcode() == AArch64::STLXRH ||1006 Inst.getOpcode() == AArch64::STLXRB);1007 }1008 1009 bool isAArch64ExclusiveClear(const MCInst &Inst) const override {1010 return (Inst.getOpcode() == AArch64::CLREX);1011 }1012 1013 bool isLoadFromStack(const MCInst &Inst) const {1014 if (!mayLoad(Inst))1015 return false;1016 for (const MCOperand &Operand : useOperands(Inst)) {1017 if (!Operand.isReg())1018 continue;1019 unsigned Reg = Operand.getReg();1020 if (Reg == AArch64::SP || Reg == AArch64::WSP)1021 return true;1022 }1023 return false;1024 }1025 1026 bool isRegToRegMove(const MCInst &Inst, MCPhysReg &From,1027 MCPhysReg &To) const override {1028 if (Inst.getOpcode() == AArch64::FMOVDXr) {1029 From = Inst.getOperand(1).getReg();1030 To = Inst.getOperand(0).getReg();1031 return true;1032 }1033 1034 if (Inst.getOpcode() != AArch64::ORRXrs)1035 return false;1036 if (Inst.getOperand(1).getReg() != AArch64::XZR)1037 return false;1038 if (Inst.getOperand(3).getImm() != 0)1039 return false;1040 From = Inst.getOperand(2).getReg();1041 To = Inst.getOperand(0).getReg();1042 return true;1043 }1044 1045 bool isIndirectCall(const MCInst &Inst) const override {1046 return isIndirectCallOpcode(Inst.getOpcode());1047 }1048 1049 MCPhysReg getSpRegister(int Size) const {1050 switch (Size) {1051 case 4:1052 return AArch64::WSP;1053 case 8:1054 return AArch64::SP;1055 default:1056 llvm_unreachable("Unexpected size");1057 }1058 }1059 1060 MCPhysReg getIntArgRegister(unsigned ArgNo) const override {1061 switch (ArgNo) {1062 case 0:1063 return AArch64::X0;1064 case 1:1065 return AArch64::X1;1066 case 2:1067 return AArch64::X2;1068 case 3:1069 return AArch64::X3;1070 case 4:1071 return AArch64::X4;1072 case 5:1073 return AArch64::X5;1074 case 6:1075 return AArch64::X6;1076 case 7:1077 return AArch64::X7;1078 default:1079 return getNoRegister();1080 }1081 }1082 1083 bool hasPCRelOperand(const MCInst &Inst) const override {1084 // ADRP is blacklisted and is an exception. Even though it has a1085 // PC-relative operand, this operand is not a complete symbol reference1086 // and BOLT shouldn't try to process it in isolation.1087 if (isADRP(Inst))1088 return false;1089 1090 if (isADR(Inst))1091 return true;1092 1093 // Look for literal addressing mode (see C1-143 ARM DDI 0487B.a)1094 const MCInstrDesc &MCII = Info->get(Inst.getOpcode());1095 for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I)1096 if (MCII.operands()[I].OperandType == MCOI::OPERAND_PCREL)1097 return true;1098 1099 return false;1100 }1101 1102 bool evaluateADR(const MCInst &Inst, int64_t &Imm,1103 const MCExpr **DispExpr) const {1104 assert((isADR(Inst) || isADRP(Inst)) && "Not an ADR instruction");1105 1106 const MCOperand &Label = Inst.getOperand(1);1107 if (!Label.isImm()) {1108 assert(Label.isExpr() && "Unexpected ADR operand");1109 assert(DispExpr && "DispExpr must be set");1110 *DispExpr = Label.getExpr();1111 return false;1112 }1113 1114 if (Inst.getOpcode() == AArch64::ADR) {1115 Imm = Label.getImm();1116 return true;1117 }1118 Imm = Label.getImm() << 12;1119 return true;1120 }1121 1122 bool evaluateAArch64MemoryOperand(const MCInst &Inst, int64_t &DispImm,1123 const MCExpr **DispExpr = nullptr) const {1124 if (isADR(Inst) || isADRP(Inst))1125 return evaluateADR(Inst, DispImm, DispExpr);1126 1127 // Literal addressing mode1128 const MCInstrDesc &MCII = Info->get(Inst.getOpcode());1129 for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I) {1130 if (MCII.operands()[I].OperandType != MCOI::OPERAND_PCREL)1131 continue;1132 1133 if (!Inst.getOperand(I).isImm()) {1134 assert(Inst.getOperand(I).isExpr() && "Unexpected PCREL operand");1135 assert(DispExpr && "DispExpr must be set");1136 *DispExpr = Inst.getOperand(I).getExpr();1137 return true;1138 }1139 1140 DispImm = Inst.getOperand(I).getImm() * 4;1141 return true;1142 }1143 return false;1144 }1145 1146 bool evaluateMemOperandTarget(const MCInst &Inst, uint64_t &Target,1147 uint64_t Address,1148 uint64_t Size) const override {1149 int64_t DispValue;1150 const MCExpr *DispExpr = nullptr;1151 if (!evaluateAArch64MemoryOperand(Inst, DispValue, &DispExpr))1152 return false;1153 1154 // Make sure it's a well-formed addressing we can statically evaluate.1155 if (DispExpr)1156 return false;1157 1158 Target = DispValue;1159 if (Inst.getOpcode() == AArch64::ADRP)1160 Target += Address & ~0xFFFULL;1161 else1162 Target += Address;1163 return true;1164 }1165 1166 MCInst::iterator getMemOperandDisp(MCInst &Inst) const override {1167 MCInst::iterator OI = Inst.begin();1168 if (isADR(Inst) || isADRP(Inst)) {1169 assert(MCPlus::getNumPrimeOperands(Inst) >= 2 &&1170 "Unexpected number of operands");1171 return ++OI;1172 }1173 const MCInstrDesc &MCII = Info->get(Inst.getOpcode());1174 for (unsigned I = 0, E = MCII.getNumOperands(); I != E; ++I) {1175 if (MCII.operands()[I].OperandType == MCOI::OPERAND_PCREL)1176 break;1177 ++OI;1178 }1179 assert(OI != Inst.end() && "Literal operand not found");1180 return OI;1181 }1182 1183 bool replaceMemOperandDisp(MCInst &Inst, MCOperand Operand) const override {1184 MCInst::iterator OI = getMemOperandDisp(Inst);1185 *OI = Operand;1186 return true;1187 }1188 1189 void getCalleeSavedRegs(BitVector &Regs) const override {1190 Regs |= getAliases(AArch64::X18);1191 Regs |= getAliases(AArch64::X19);1192 Regs |= getAliases(AArch64::X20);1193 Regs |= getAliases(AArch64::X21);1194 Regs |= getAliases(AArch64::X22);1195 Regs |= getAliases(AArch64::X23);1196 Regs |= getAliases(AArch64::X24);1197 Regs |= getAliases(AArch64::X25);1198 Regs |= getAliases(AArch64::X26);1199 Regs |= getAliases(AArch64::X27);1200 Regs |= getAliases(AArch64::X28);1201 Regs |= getAliases(AArch64::LR);1202 Regs |= getAliases(AArch64::FP);1203 }1204 1205 const MCExpr *getTargetExprFor(MCInst &Inst, const MCExpr *Expr,1206 MCContext &Ctx,1207 uint32_t RelType) const override {1208 1209 if (isADR(Inst) || RelType == ELF::R_AARCH64_ADR_PREL_LO21 ||1210 RelType == ELF::R_AARCH64_TLSDESC_ADR_PREL21) {1211 return MCSpecifierExpr::create(Expr, AArch64::S_ABS, Ctx);1212 } else if (isADRP(Inst) || RelType == ELF::R_AARCH64_ADR_PREL_PG_HI21 ||1213 RelType == ELF::R_AARCH64_ADR_PREL_PG_HI21_NC ||1214 RelType == ELF::R_AARCH64_TLSDESC_ADR_PAGE21 ||1215 RelType == ELF::R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21 ||1216 RelType == ELF::R_AARCH64_ADR_GOT_PAGE) {1217 // Never emit a GOT reloc, we handled this in1218 // RewriteInstance::readRelocations().1219 return MCSpecifierExpr::create(Expr, AArch64::S_ABS_PAGE, Ctx);1220 } else {1221 switch (RelType) {1222 case ELF::R_AARCH64_ADD_ABS_LO12_NC:1223 case ELF::R_AARCH64_LD64_GOT_LO12_NC:1224 case ELF::R_AARCH64_LDST8_ABS_LO12_NC:1225 case ELF::R_AARCH64_LDST16_ABS_LO12_NC:1226 case ELF::R_AARCH64_LDST32_ABS_LO12_NC:1227 case ELF::R_AARCH64_LDST64_ABS_LO12_NC:1228 case ELF::R_AARCH64_LDST128_ABS_LO12_NC:1229 case ELF::R_AARCH64_TLSDESC_ADD_LO12:1230 case ELF::R_AARCH64_TLSDESC_LD64_LO12:1231 case ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:1232 case ELF::R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:1233 return MCSpecifierExpr::create(Expr, AArch64::S_LO12, Ctx);1234 case ELF::R_AARCH64_MOVW_UABS_G3:1235 return MCSpecifierExpr::create(Expr, AArch64::S_ABS_G3, Ctx);1236 case ELF::R_AARCH64_MOVW_UABS_G2:1237 case ELF::R_AARCH64_MOVW_UABS_G2_NC:1238 return MCSpecifierExpr::create(Expr, AArch64::S_ABS_G2_NC, Ctx);1239 case ELF::R_AARCH64_MOVW_UABS_G1:1240 case ELF::R_AARCH64_MOVW_UABS_G1_NC:1241 return MCSpecifierExpr::create(Expr, AArch64::S_ABS_G1_NC, Ctx);1242 case ELF::R_AARCH64_MOVW_UABS_G0:1243 case ELF::R_AARCH64_MOVW_UABS_G0_NC:1244 return MCSpecifierExpr::create(Expr, AArch64::S_ABS_G0_NC, Ctx);1245 default:1246 break;1247 }1248 }1249 return Expr;1250 }1251 1252 bool getSymbolRefOperandNum(const MCInst &Inst, unsigned &OpNum) const {1253 if (OpNum >= MCPlus::getNumPrimeOperands(Inst))1254 return false;1255 1256 // Auto-select correct operand number1257 if (OpNum == 0) {1258 if (isConditionalBranch(Inst) || isADR(Inst) || isADRP(Inst) ||1259 isMOVW(Inst))1260 OpNum = 1;1261 if (isTB(Inst) || isAddXri(Inst))1262 OpNum = 2;1263 }1264 1265 return true;1266 }1267 1268 const MCSymbol *getTargetSymbol(const MCExpr *Expr) const override {1269 auto *AArchExpr = dyn_cast<MCSpecifierExpr>(Expr);1270 if (AArchExpr && AArchExpr->getSubExpr())1271 return getTargetSymbol(AArchExpr->getSubExpr());1272 1273 return MCPlusBuilder::getTargetSymbol(Expr);1274 }1275 1276 const MCSymbol *getTargetSymbol(const MCInst &Inst,1277 unsigned OpNum = 0) const override {1278 if (!OpNum && !getSymbolRefOperandNum(Inst, OpNum))1279 return nullptr;1280 1281 const MCOperand &Op = Inst.getOperand(OpNum);1282 if (!Op.isExpr())1283 return nullptr;1284 1285 return getTargetSymbol(Op.getExpr());1286 }1287 1288 int64_t getTargetAddend(const MCExpr *Expr) const override {1289 auto *AArchExpr = dyn_cast<MCSpecifierExpr>(Expr);1290 if (AArchExpr && AArchExpr->getSubExpr())1291 return getTargetAddend(AArchExpr->getSubExpr());1292 1293 auto *BinExpr = dyn_cast<MCBinaryExpr>(Expr);1294 if (BinExpr && BinExpr->getOpcode() == MCBinaryExpr::Add)1295 return getTargetAddend(BinExpr->getRHS());1296 1297 auto *ConstExpr = dyn_cast<MCConstantExpr>(Expr);1298 if (ConstExpr)1299 return ConstExpr->getValue();1300 1301 return 0;1302 }1303 1304 int64_t getTargetAddend(const MCInst &Inst,1305 unsigned OpNum = 0) const override {1306 if (!getSymbolRefOperandNum(Inst, OpNum))1307 return 0;1308 1309 const MCOperand &Op = Inst.getOperand(OpNum);1310 if (!Op.isExpr())1311 return 0;1312 1313 return getTargetAddend(Op.getExpr());1314 }1315 1316 void replaceBranchTarget(MCInst &Inst, const MCSymbol *TBB,1317 MCContext *Ctx) const override {1318 assert((isCall(Inst) || isBranch(Inst)) && !isIndirectBranch(Inst) &&1319 "Invalid instruction");1320 assert(MCPlus::getNumPrimeOperands(Inst) >= 1 &&1321 "Invalid number of operands");1322 MCInst::iterator OI = Inst.begin();1323 1324 if (isConditionalBranch(Inst)) {1325 assert(MCPlus::getNumPrimeOperands(Inst) >= 2 &&1326 "Invalid number of operands");1327 ++OI;1328 }1329 1330 if (isTB(Inst)) {1331 assert(MCPlus::getNumPrimeOperands(Inst) >= 3 &&1332 "Invalid number of operands");1333 OI = Inst.begin() + 2;1334 }1335 1336 *OI = MCOperand::createExpr(MCSymbolRefExpr::create(TBB, *Ctx));1337 }1338 1339 /// Matches indirect branch patterns in AArch64 related to a jump table (JT),1340 /// helping us to build the complete CFG. A typical indirect branch to1341 /// a jump table entry in AArch64 looks like the following:1342 ///1343 /// adrp x1, #-7585792 # Get JT Page location1344 /// add x1, x1, #692 # Complement with JT Page offset1345 /// ldrh w0, [x1, w0, uxtw #1] # Loads JT entry1346 /// adr x1, #12 # Get PC + 12 (end of this BB) used next1347 /// add x0, x1, w0, sxth #2 # Finish building branch target1348 /// # (entries in JT are relative to the end1349 /// # of this BB)1350 /// br x0 # Indirect jump instruction1351 ///1352 /// Return true on successful jump table instruction sequence match, false1353 /// otherwise.1354 bool analyzeIndirectBranchFragment(1355 const MCInst &Inst,1356 DenseMap<const MCInst *, SmallVector<MCInst *, 4>> &UDChain,1357 const MCExpr *&JumpTable, int64_t &Offset, int64_t &ScaleValue,1358 MCInst *&PCRelBase) const {1359 // The only kind of indirect branches we match is jump table, thus ignore1360 // authenticating branch instructions early.1361 if (isBRA(Inst))1362 return false;1363 1364 // Expect AArch64 BR1365 assert(Inst.getOpcode() == AArch64::BR && "Unexpected opcode");1366 1367 JumpTable = nullptr;1368 1369 // Match the indirect branch pattern for aarch641370 SmallVector<MCInst *, 4> &UsesRoot = UDChain[&Inst];1371 if (UsesRoot.size() == 0 || UsesRoot[0] == nullptr)1372 return false;1373 1374 const MCInst *DefAdd = UsesRoot[0];1375 1376 // Now we match an ADD1377 if (!isADD(*DefAdd)) {1378 // If the address is not broken up in two parts, this is not branching1379 // according to a jump table entry. Fail.1380 return false;1381 }1382 if (DefAdd->getOpcode() == AArch64::ADDXri) {1383 // This can happen when there is no offset, but a direct jump that was1384 // transformed into an indirect one (indirect tail call) :1385 // ADRP x2, Perl_re_compiler1386 // ADD x2, x2, :lo12:Perl_re_compiler1387 // BR x21388 return false;1389 }1390 if (DefAdd->getOpcode() == AArch64::ADDXrs) {1391 // Covers the less common pattern where JT entries are relative to1392 // the JT itself (like x86). Seems less efficient since we can't1393 // assume the JT is aligned at 4B boundary and thus drop 2 bits from1394 // JT values.1395 // cde264:1396 // adrp x12, #21544960 ; 216a0001397 // add x12, x12, #1696 ; 216a6a0 (JT object in .rodata)1398 // ldrsw x8, [x12, x8, lsl #2] --> loads e.g. 0xfeb73bd81399 // * add x8, x8, x12 --> = cde278, next block1400 // br x81401 // cde278:1402 //1403 // Parsed as ADDXrs reg:x8 reg:x8 reg:x12 imm:01404 return false;1405 }1406 if (DefAdd->getOpcode() != AArch64::ADDXrx)1407 return false;1408 1409 // Validate ADD operands1410 int64_t OperandExtension = DefAdd->getOperand(3).getImm();1411 unsigned ShiftVal = AArch64_AM::getArithShiftValue(OperandExtension);1412 AArch64_AM::ShiftExtendType ExtendType =1413 AArch64_AM::getArithExtendType(OperandExtension);1414 if (ShiftVal != 2) {1415 // TODO: Handle the pattern where ShiftVal != 2.1416 // The following code sequence below has no shift amount,1417 // the range could be 0 to 4.1418 // The pattern comes from libc, it occurs when the binary is static.1419 // adr x6, 0x219fb0 <sigall_set+0x88>1420 // add x6, x6, x14, lsl #21421 // ldr w7, [x6]1422 // add x6, x6, w7, sxtw => no shift amount1423 // br x61424 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: "1425 "failed to match indirect branch: ShiftVAL != 2\n");1426 return false;1427 }1428 1429 if (ExtendType == AArch64_AM::SXTB)1430 ScaleValue = 1LL;1431 else if (ExtendType == AArch64_AM::SXTH)1432 ScaleValue = 2LL;1433 else if (ExtendType == AArch64_AM::SXTW)1434 ScaleValue = 4LL;1435 else1436 return false;1437 1438 // Match an ADR to load base address to be used when addressing JT targets1439 SmallVector<MCInst *, 4> &UsesAdd = UDChain[DefAdd];1440 if (UsesAdd.size() <= 1 || UsesAdd[1] == nullptr || UsesAdd[2] == nullptr) {1441 // This happens when we don't have enough context about this jump table1442 // because the jumping code sequence was split in multiple basic blocks.1443 // This was observed in the wild in HHVM code (dispatchImpl).1444 return false;1445 }1446 MCInst *DefBaseAddr = UsesAdd[1];1447 if (DefBaseAddr->getOpcode() != AArch64::ADR)1448 return false;1449 1450 PCRelBase = DefBaseAddr;1451 // Match LOAD to load the jump table (relative) target1452 const MCInst *DefLoad = UsesAdd[2];1453 if (!mayLoad(*DefLoad) || (ScaleValue == 1LL && !isLDRB(*DefLoad)) ||1454 (ScaleValue == 2LL && !isLDRH(*DefLoad)))1455 return false;1456 1457 // Match ADD that calculates the JumpTable Base Address (not the offset)1458 SmallVector<MCInst *, 4> &UsesLoad = UDChain[DefLoad];1459 const MCInst *DefJTBaseAdd = UsesLoad[1];1460 MCPhysReg From, To;1461 if (DefJTBaseAdd == nullptr || isLoadFromStack(*DefJTBaseAdd) ||1462 isRegToRegMove(*DefJTBaseAdd, From, To)) {1463 // Sometimes base address may have been defined in another basic block1464 // (hoisted). Return with no jump table info.1465 return true;1466 }1467 1468 if (DefJTBaseAdd->getOpcode() == AArch64::ADR) {1469 // TODO: Handle the pattern where there is no adrp/add pair.1470 // It also occurs when the binary is static.1471 // adr x13, 0x215a18 <_nl_value_type_LC_COLLATE+0x50>1472 // ldrh w13, [x13, w12, uxtw #1]1473 // adr x12, 0x247b30 <__gettextparse+0x5b0>1474 // add x13, x12, w13, sxth #21475 // br x131476 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: failed to match indirect branch: "1477 "nop/adr instead of adrp/add\n");1478 return false;1479 }1480 1481 if (DefJTBaseAdd->getOpcode() != AArch64::ADDXri) {1482 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: failed to match jump table base "1483 "address pattern! (1)\n");1484 return false;1485 }1486 1487 if (DefJTBaseAdd->getOperand(2).isImm())1488 Offset = DefJTBaseAdd->getOperand(2).getImm();1489 SmallVector<MCInst *, 4> &UsesJTBaseAdd = UDChain[DefJTBaseAdd];1490 const MCInst *DefJTBasePage = UsesJTBaseAdd[1];1491 if (DefJTBasePage == nullptr || isLoadFromStack(*DefJTBasePage)) {1492 return true;1493 }1494 if (DefJTBasePage->getOpcode() != AArch64::ADRP)1495 return false;1496 1497 if (DefJTBasePage->getOperand(1).isExpr())1498 JumpTable = DefJTBasePage->getOperand(1).getExpr();1499 return true;1500 }1501 1502 DenseMap<const MCInst *, SmallVector<MCInst *, 4>>1503 computeLocalUDChain(const MCInst *CurInstr, InstructionIterator Begin,1504 InstructionIterator End) const {1505 DenseMap<int, MCInst *> RegAliasTable;1506 DenseMap<const MCInst *, SmallVector<MCInst *, 4>> Uses;1507 1508 auto addInstrOperands = [&](const MCInst &Instr) {1509 // Update Uses table1510 for (const MCOperand &Operand : MCPlus::primeOperands(Instr)) {1511 if (!Operand.isReg())1512 continue;1513 unsigned Reg = Operand.getReg();1514 MCInst *AliasInst = RegAliasTable[Reg];1515 Uses[&Instr].push_back(AliasInst);1516 LLVM_DEBUG({1517 dbgs() << "Adding reg operand " << Reg << " refs ";1518 if (AliasInst != nullptr)1519 AliasInst->dump();1520 else1521 dbgs() << "\n";1522 });1523 }1524 };1525 1526 LLVM_DEBUG(dbgs() << "computeLocalUDChain\n");1527 bool TerminatorSeen = false;1528 for (auto II = Begin; II != End; ++II) {1529 MCInst &Instr = *II;1530 // Ignore nops and CFIs1531 if (isPseudo(Instr) || isNoop(Instr))1532 continue;1533 if (TerminatorSeen) {1534 RegAliasTable.clear();1535 Uses.clear();1536 }1537 1538 LLVM_DEBUG(dbgs() << "Now updating for:\n ");1539 LLVM_DEBUG(Instr.dump());1540 addInstrOperands(Instr);1541 1542 BitVector Regs = BitVector(RegInfo->getNumRegs(), false);1543 getWrittenRegs(Instr, Regs);1544 1545 // Update register definitions after this point1546 for (int Idx : Regs.set_bits()) {1547 RegAliasTable[Idx] = &Instr;1548 LLVM_DEBUG(dbgs() << "Setting reg " << Idx1549 << " def to current instr.\n");1550 }1551 1552 TerminatorSeen = isTerminator(Instr);1553 }1554 1555 // Process the last instruction, which is not currently added into the1556 // instruction stream1557 if (CurInstr)1558 addInstrOperands(*CurInstr);1559 1560 return Uses;1561 }1562 1563 IndirectBranchType1564 analyzeIndirectBranch(MCInst &Instruction, InstructionIterator Begin,1565 InstructionIterator End, const unsigned PtrSize,1566 MCInst *&MemLocInstrOut, unsigned &BaseRegNumOut,1567 unsigned &IndexRegNumOut, int64_t &DispValueOut,1568 const MCExpr *&DispExprOut, MCInst *&PCRelBaseOut,1569 MCInst *&FixedEntryLoadInstr) const override {1570 MemLocInstrOut = nullptr;1571 BaseRegNumOut = AArch64::NoRegister;1572 IndexRegNumOut = AArch64::NoRegister;1573 DispValueOut = 0;1574 DispExprOut = nullptr;1575 FixedEntryLoadInstr = nullptr;1576 1577 // An instruction referencing memory used by jump instruction (directly or1578 // via register). This location could be an array of function pointers1579 // in case of indirect tail call, or a jump table.1580 MCInst *MemLocInstr = nullptr;1581 1582 // Analyze the memory location.1583 int64_t ScaleValue, DispValue;1584 const MCExpr *DispExpr;1585 1586 DenseMap<const MCInst *, SmallVector<llvm::MCInst *, 4>> UDChain =1587 computeLocalUDChain(&Instruction, Begin, End);1588 MCInst *PCRelBase;1589 if (!analyzeIndirectBranchFragment(Instruction, UDChain, DispExpr,1590 DispValue, ScaleValue, PCRelBase))1591 return IndirectBranchType::UNKNOWN;1592 1593 MemLocInstrOut = MemLocInstr;1594 DispValueOut = DispValue;1595 DispExprOut = DispExpr;1596 PCRelBaseOut = PCRelBase;1597 return IndirectBranchType::POSSIBLE_PIC_JUMP_TABLE;1598 }1599 1600 /// Matches PLT entry pattern and returns the associated GOT entry address.1601 /// Typical PLT entry looks like the following:1602 ///1603 /// adrp x16, 2300001604 /// ldr x17, [x16, #3040]1605 /// add x16, x16, #0xbe01606 /// br x171607 ///1608 /// The other type of trampolines are located in .plt.got, that are used for1609 /// non-lazy bindings so doesn't use x16 arg to transfer .got entry address:1610 ///1611 /// adrp x16, 2300001612 /// ldr x17, [x16, #3040]1613 /// br x171614 /// nop1615 ///1616 uint64_t analyzePLTEntry(MCInst &Instruction, InstructionIterator Begin,1617 InstructionIterator End,1618 uint64_t BeginPC) const override {1619 // Check branch instruction1620 MCInst *Branch = &Instruction;1621 assert(Branch->getOpcode() == AArch64::BR && "Unexpected opcode");1622 1623 DenseMap<const MCInst *, SmallVector<llvm::MCInst *, 4>> UDChain =1624 computeLocalUDChain(Branch, Begin, End);1625 1626 // Match ldr instruction1627 SmallVector<MCInst *, 4> &BranchUses = UDChain[Branch];1628 if (BranchUses.size() < 1 || BranchUses[0] == nullptr)1629 return 0;1630 1631 // Check ldr instruction1632 const MCInst *Ldr = BranchUses[0];1633 if (Ldr->getOpcode() != AArch64::LDRXui)1634 return 0;1635 1636 // Get ldr value1637 const unsigned ScaleLdr = 8; // LDRX operates on 8 bytes segments1638 assert(Ldr->getOperand(2).isImm() && "Unexpected ldr operand");1639 const uint64_t Offset = Ldr->getOperand(2).getImm() * ScaleLdr;1640 1641 // Match adrp instruction1642 SmallVector<MCInst *, 4> &LdrUses = UDChain[Ldr];1643 if (LdrUses.size() < 2 || LdrUses[1] == nullptr)1644 return 0;1645 1646 // Check adrp instruction1647 MCInst *Adrp = LdrUses[1];1648 if (Adrp->getOpcode() != AArch64::ADRP)1649 return 0;1650 1651 // Get adrp instruction PC1652 const unsigned InstSize = 4;1653 uint64_t AdrpPC = BeginPC;1654 for (InstructionIterator It = Begin; It != End; ++It) {1655 if (&(*It) == Adrp)1656 break;1657 AdrpPC += InstSize;1658 }1659 1660 // Get adrp value1661 uint64_t Base;1662 assert(Adrp->getOperand(1).isImm() && "Unexpected adrp operand");1663 bool Ret = evaluateMemOperandTarget(*Adrp, Base, AdrpPC, InstSize);1664 assert(Ret && "Failed to evaluate adrp");1665 (void)Ret;1666 1667 return Base + Offset;1668 }1669 1670 unsigned getInvertedBranchOpcode(unsigned Opcode) const {1671 switch (Opcode) {1672 default:1673 llvm_unreachable("Failed to invert branch opcode");1674 return Opcode;1675 case AArch64::TBZW: return AArch64::TBNZW;1676 case AArch64::TBZX: return AArch64::TBNZX;1677 case AArch64::TBNZW: return AArch64::TBZW;1678 case AArch64::TBNZX: return AArch64::TBZX;1679 case AArch64::CBZW: return AArch64::CBNZW;1680 case AArch64::CBZX: return AArch64::CBNZX;1681 case AArch64::CBNZW: return AArch64::CBZW;1682 case AArch64::CBNZX: return AArch64::CBZX;1683 }1684 }1685 1686 unsigned getCondCode(const MCInst &Inst) const override {1687 // AArch64 does not use conditional codes, so we just return the opcode1688 // of the conditional branch here.1689 return Inst.getOpcode();1690 }1691 1692 unsigned getCanonicalBranchCondCode(unsigned Opcode) const override {1693 switch (Opcode) {1694 default:1695 return Opcode;1696 case AArch64::TBNZW: return AArch64::TBZW;1697 case AArch64::TBNZX: return AArch64::TBZX;1698 case AArch64::CBNZW: return AArch64::CBZW;1699 case AArch64::CBNZX: return AArch64::CBZX;1700 }1701 }1702 1703 void reverseBranchCondition(MCInst &Inst, const MCSymbol *TBB,1704 MCContext *Ctx) const override {1705 if (isTB(Inst) || isCB(Inst)) {1706 Inst.setOpcode(getInvertedBranchOpcode(Inst.getOpcode()));1707 assert(Inst.getOpcode() != 0 && "Invalid branch instruction");1708 } else if (Inst.getOpcode() == AArch64::Bcc) {1709 Inst.getOperand(0).setImm(AArch64CC::getInvertedCondCode(1710 static_cast<AArch64CC::CondCode>(Inst.getOperand(0).getImm())));1711 assert(Inst.getOperand(0).getImm() != AArch64CC::AL &&1712 Inst.getOperand(0).getImm() != AArch64CC::NV &&1713 "Can't reverse ALWAYS cond code");1714 } else {1715 LLVM_DEBUG(Inst.dump());1716 llvm_unreachable("Unrecognized branch instruction");1717 }1718 replaceBranchTarget(Inst, TBB, Ctx);1719 }1720 1721 int getPCRelEncodingSize(const MCInst &Inst) const override {1722 switch (Inst.getOpcode()) {1723 default:1724 llvm_unreachable("Failed to get pcrel encoding size");1725 return 0;1726 case AArch64::TBZW: return 16;1727 case AArch64::TBZX: return 16;1728 case AArch64::TBNZW: return 16;1729 case AArch64::TBNZX: return 16;1730 case AArch64::CBZW: return 21;1731 case AArch64::CBZX: return 21;1732 case AArch64::CBNZW: return 21;1733 case AArch64::CBNZX: return 21;1734 case AArch64::B: return 28;1735 case AArch64::BL: return 28;1736 case AArch64::Bcc: return 21;1737 }1738 }1739 1740 int getShortJmpEncodingSize() const override { return 33; }1741 1742 int getUncondBranchEncodingSize() const override { return 28; }1743 1744 // This helper function creates the snippet of code that compares a register1745 // RegNo with an immediate Imm, and jumps to Target if they are equal.1746 // cmp RegNo, #Imm1747 // b.eq Target1748 // where cmp is an alias for subs, which results in the code below:1749 // subs xzr, RegNo, #Imm1750 // b.eq Target.1751 InstructionListType createCmpJE(MCPhysReg RegNo, int64_t Imm,1752 const MCSymbol *Target,1753 MCContext *Ctx) const override {1754 InstructionListType Code;1755 Code.emplace_back(MCInstBuilder(AArch64::SUBSXri)1756 .addReg(AArch64::XZR)1757 .addReg(RegNo)1758 .addImm(Imm)1759 .addImm(0));1760 Code.emplace_back(MCInstBuilder(AArch64::Bcc)1761 .addImm(AArch64CC::EQ)1762 .addExpr(MCSymbolRefExpr::create(Target, *Ctx)));1763 return Code;1764 }1765 1766 // This helper function creates the snippet of code that compares a register1767 // RegNo with an immediate Imm, and jumps to Target if they are not equal.1768 // cmp RegNo, #Imm1769 // b.ne Target1770 // where cmp is an alias for subs, which results in the code below:1771 // subs xzr, RegNo, #Imm1772 // b.ne Target.1773 InstructionListType createCmpJNE(MCPhysReg RegNo, int64_t Imm,1774 const MCSymbol *Target,1775 MCContext *Ctx) const override {1776 InstructionListType Code;1777 Code.emplace_back(MCInstBuilder(AArch64::SUBSXri)1778 .addReg(AArch64::XZR)1779 .addReg(RegNo)1780 .addImm(Imm)1781 .addImm(0));1782 Code.emplace_back(MCInstBuilder(AArch64::Bcc)1783 .addImm(AArch64CC::NE)1784 .addExpr(MCSymbolRefExpr::create(Target, *Ctx)));1785 return Code;1786 }1787 1788 void createTailCall(MCInst &Inst, const MCSymbol *Target,1789 MCContext *Ctx) override {1790 return createDirectCall(Inst, Target, Ctx, /*IsTailCall*/ true);1791 }1792 1793 void createLongTailCall(InstructionListType &Seq, const MCSymbol *Target,1794 MCContext *Ctx) override {1795 createShortJmp(Seq, Target, Ctx, /*IsTailCall*/ true);1796 }1797 1798 void createTrap(MCInst &Inst) const override {1799 Inst.clear();1800 Inst.setOpcode(AArch64::BRK);1801 Inst.addOperand(MCOperand::createImm(1));1802 }1803 1804 bool convertJmpToTailCall(MCInst &Inst) override {1805 setTailCall(Inst);1806 return true;1807 }1808 1809 bool convertTailCallToJmp(MCInst &Inst) override {1810 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall);1811 clearOffset(Inst);1812 if (getConditionalTailCall(Inst))1813 unsetConditionalTailCall(Inst);1814 return true;1815 }1816 1817 InstructionListType createIndirectPLTCall(MCInst &&DirectCall,1818 const MCSymbol *TargetLocation,1819 MCContext *Ctx) override {1820 const bool IsTailCall = isTailCall(DirectCall);1821 assert((DirectCall.getOpcode() == AArch64::BL ||1822 (DirectCall.getOpcode() == AArch64::B && IsTailCall)) &&1823 "64-bit direct (tail) call instruction expected");1824 1825 InstructionListType Code;1826 // Code sequence for indirect plt call:1827 // adrp x16 <symbol>1828 // ldr x17, [x16, #<offset>]1829 // blr x17 ; or 'br' for tail calls1830 1831 MCInst InstAdrp;1832 InstAdrp.setOpcode(AArch64::ADRP);1833 InstAdrp.addOperand(MCOperand::createReg(AArch64::X16));1834 InstAdrp.addOperand(MCOperand::createImm(0));1835 setOperandToSymbolRef(InstAdrp, /* OpNum */ 1, TargetLocation,1836 /* Addend */ 0, Ctx, ELF::R_AARCH64_ADR_GOT_PAGE);1837 Code.emplace_back(InstAdrp);1838 1839 MCInst InstLoad;1840 InstLoad.setOpcode(AArch64::LDRXui);1841 InstLoad.addOperand(MCOperand::createReg(AArch64::X17));1842 InstLoad.addOperand(MCOperand::createReg(AArch64::X16));1843 InstLoad.addOperand(MCOperand::createImm(0));1844 setOperandToSymbolRef(InstLoad, /* OpNum */ 2, TargetLocation,1845 /* Addend */ 0, Ctx, ELF::R_AARCH64_LD64_GOT_LO12_NC);1846 Code.emplace_back(InstLoad);1847 1848 MCInst InstCall;1849 InstCall.setOpcode(IsTailCall ? AArch64::BR : AArch64::BLR);1850 InstCall.addOperand(MCOperand::createReg(AArch64::X17));1851 moveAnnotations(std::move(DirectCall), InstCall);1852 Code.emplace_back(InstCall);1853 1854 return Code;1855 }1856 1857 bool lowerTailCall(MCInst &Inst) override {1858 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall);1859 if (getConditionalTailCall(Inst))1860 unsetConditionalTailCall(Inst);1861 return true;1862 }1863 1864 bool isNoop(const MCInst &Inst) const override {1865 return Inst.getOpcode() == AArch64::HINT &&1866 Inst.getOperand(0).getImm() == 0;1867 }1868 1869 void createNoop(MCInst &Inst) const override {1870 Inst.setOpcode(AArch64::HINT);1871 Inst.clear();1872 Inst.addOperand(MCOperand::createImm(0));1873 }1874 1875 bool isTrap(const MCInst &Inst) const override {1876 if (Inst.getOpcode() != AArch64::BRK)1877 return false;1878 // Only match the immediate values that are likely to indicate this BRK1879 // instruction is emitted to terminate the program immediately and not to1880 // be handled by a SIGTRAP handler, for example.1881 switch (Inst.getOperand(0).getImm()) {1882 case 0xc470:1883 case 0xc471:1884 case 0xc472:1885 case 0xc473:1886 // Explicit Pointer Authentication check failed, see1887 // AArch64AsmPrinter::emitPtrauthCheckAuthenticatedValue().1888 return true;1889 case 0x1:1890 // __builtin_trap(), as emitted by Clang.1891 return true;1892 case 0x3e8: // decimal 10001893 // __builtin_trap(), as emitted by GCC.1894 return true;1895 default:1896 // Some constants may indicate intentionally recoverable break-points.1897 // This is the case at least for 0xf000, which is used by1898 // __builtin_debugtrap() supported by Clang.1899 return false;1900 }1901 }1902 1903 bool isStorePair(const MCInst &Inst) const {1904 const unsigned opcode = Inst.getOpcode();1905 1906 auto isStorePairImmOffset = [&]() {1907 switch (opcode) {1908 case AArch64::STPWi:1909 case AArch64::STPXi:1910 case AArch64::STPSi:1911 case AArch64::STPDi:1912 case AArch64::STPQi:1913 case AArch64::STNPWi:1914 case AArch64::STNPXi:1915 case AArch64::STNPSi:1916 case AArch64::STNPDi:1917 case AArch64::STNPQi:1918 return true;1919 default:1920 break;1921 }1922 1923 return false;1924 };1925 1926 auto isStorePairPostIndex = [&]() {1927 switch (opcode) {1928 case AArch64::STPWpost:1929 case AArch64::STPXpost:1930 case AArch64::STPSpost:1931 case AArch64::STPDpost:1932 case AArch64::STPQpost:1933 return true;1934 default:1935 break;1936 }1937 1938 return false;1939 };1940 1941 auto isStorePairPreIndex = [&]() {1942 switch (opcode) {1943 case AArch64::STPWpre:1944 case AArch64::STPXpre:1945 case AArch64::STPSpre:1946 case AArch64::STPDpre:1947 case AArch64::STPQpre:1948 return true;1949 default:1950 break;1951 }1952 1953 return false;1954 };1955 1956 return isStorePairImmOffset() || isStorePairPostIndex() ||1957 isStorePairPreIndex();1958 }1959 1960 bool isStoreReg(const MCInst &Inst) const {1961 const unsigned opcode = Inst.getOpcode();1962 1963 auto isStoreRegUnscaleImm = [&]() {1964 switch (opcode) {1965 case AArch64::STURBi:1966 case AArch64::STURBBi:1967 case AArch64::STURHi:1968 case AArch64::STURHHi:1969 case AArch64::STURWi:1970 case AArch64::STURXi:1971 case AArch64::STURSi:1972 case AArch64::STURDi:1973 case AArch64::STURQi:1974 return true;1975 default:1976 break;1977 }1978 1979 return false;1980 };1981 1982 auto isStoreRegScaledImm = [&]() {1983 switch (opcode) {1984 case AArch64::STRBui:1985 case AArch64::STRBBui:1986 case AArch64::STRHui:1987 case AArch64::STRHHui:1988 case AArch64::STRWui:1989 case AArch64::STRXui:1990 case AArch64::STRSui:1991 case AArch64::STRDui:1992 case AArch64::STRQui:1993 return true;1994 default:1995 break;1996 }1997 1998 return false;1999 };2000 2001 auto isStoreRegImmPostIndexed = [&]() {2002 switch (opcode) {2003 case AArch64::STRBpost:2004 case AArch64::STRBBpost:2005 case AArch64::STRHpost:2006 case AArch64::STRHHpost:2007 case AArch64::STRWpost:2008 case AArch64::STRXpost:2009 case AArch64::STRSpost:2010 case AArch64::STRDpost:2011 case AArch64::STRQpost:2012 return true;2013 default:2014 break;2015 }2016 2017 return false;2018 };2019 2020 auto isStoreRegImmPreIndexed = [&]() {2021 switch (opcode) {2022 case AArch64::STRBpre:2023 case AArch64::STRBBpre:2024 case AArch64::STRHpre:2025 case AArch64::STRHHpre:2026 case AArch64::STRWpre:2027 case AArch64::STRXpre:2028 case AArch64::STRSpre:2029 case AArch64::STRDpre:2030 case AArch64::STRQpre:2031 return true;2032 default:2033 break;2034 }2035 2036 return false;2037 };2038 2039 auto isStoreRegUnscaleUnpriv = [&]() {2040 switch (opcode) {2041 case AArch64::STTRBi:2042 case AArch64::STTRHi:2043 case AArch64::STTRWi:2044 case AArch64::STTRXi:2045 return true;2046 default:2047 break;2048 }2049 2050 return false;2051 };2052 2053 auto isStoreRegTrunc = [&]() {2054 switch (opcode) {2055 case AArch64::STRBBroW:2056 case AArch64::STRBBroX:2057 case AArch64::STRBroW:2058 case AArch64::STRBroX:2059 case AArch64::STRDroW:2060 case AArch64::STRDroX:2061 case AArch64::STRHHroW:2062 case AArch64::STRHHroX:2063 case AArch64::STRHroW:2064 case AArch64::STRHroX:2065 case AArch64::STRQroW:2066 case AArch64::STRQroX:2067 case AArch64::STRSroW:2068 case AArch64::STRSroX:2069 case AArch64::STRWroW:2070 case AArch64::STRWroX:2071 case AArch64::STRXroW:2072 case AArch64::STRXroX:2073 return true;2074 default:2075 break;2076 }2077 2078 return false;2079 };2080 2081 return isStoreRegUnscaleImm() || isStoreRegScaledImm() ||2082 isStoreRegImmPreIndexed() || isStoreRegImmPostIndexed() ||2083 isStoreRegUnscaleUnpriv() || isStoreRegTrunc();2084 }2085 2086 bool mayStore(const MCInst &Inst) const override {2087 return isStorePair(Inst) || isStoreReg(Inst) ||2088 isAArch64ExclusiveStore(Inst);2089 }2090 2091 bool isStoreToStack(const MCInst &Inst) const {2092 if (!mayStore(Inst))2093 return false;2094 2095 for (const MCOperand &Operand : useOperands(Inst)) {2096 if (!Operand.isReg())2097 continue;2098 2099 unsigned Reg = Operand.getReg();2100 if (Reg == AArch64::SP || Reg == AArch64::WSP)2101 return true;2102 }2103 2104 return false;2105 }2106 2107 void createDirectCall(MCInst &Inst, const MCSymbol *Target, MCContext *Ctx,2108 bool IsTailCall) override {2109 Inst.setOpcode(IsTailCall ? AArch64::B : AArch64::BL);2110 Inst.clear();2111 Inst.addOperand(MCOperand::createExpr(getTargetExprFor(2112 Inst, MCSymbolRefExpr::create(Target, *Ctx), *Ctx, 0)));2113 if (IsTailCall)2114 convertJmpToTailCall(Inst);2115 }2116 2117 void createDirectBranch(MCInst &Inst, const MCSymbol *Target,2118 MCContext *Ctx) override {2119 Inst.setOpcode(AArch64::B);2120 Inst.clear();2121 Inst.addOperand(MCOperand::createExpr(getTargetExprFor(2122 Inst, MCSymbolRefExpr::create(Target, *Ctx), *Ctx, 0)));2123 }2124 2125 bool analyzeBranch(InstructionIterator Begin, InstructionIterator End,2126 const MCSymbol *&TBB, const MCSymbol *&FBB,2127 MCInst *&CondBranch,2128 MCInst *&UncondBranch) const override {2129 auto I = End;2130 2131 while (I != Begin) {2132 --I;2133 2134 // Ignore nops and CFIs2135 if (isPseudo(*I) || isNoop(*I))2136 continue;2137 2138 // Stop when we find the first non-terminator2139 if (!isTerminator(*I) || isTailCall(*I) || !isBranch(*I))2140 break;2141 2142 // Handle unconditional branches.2143 if (isUnconditionalBranch(*I)) {2144 // If any code was seen after this unconditional branch, we've seen2145 // unreachable code. Ignore them.2146 CondBranch = nullptr;2147 UncondBranch = &*I;2148 const MCSymbol *Sym = getTargetSymbol(*I);2149 assert(Sym != nullptr &&2150 "Couldn't extract BB symbol from jump operand");2151 TBB = Sym;2152 continue;2153 }2154 2155 // Handle conditional branches and ignore indirect branches2156 if (isIndirectBranch(*I))2157 return false;2158 2159 if (CondBranch == nullptr) {2160 const MCSymbol *TargetBB = getTargetSymbol(*I);2161 if (TargetBB == nullptr) {2162 // Unrecognized branch target2163 return false;2164 }2165 FBB = TBB;2166 TBB = TargetBB;2167 CondBranch = &*I;2168 continue;2169 }2170 2171 llvm_unreachable("multiple conditional branches in one BB");2172 }2173 return true;2174 }2175 2176 void createLongJmp(InstructionListType &Seq, const MCSymbol *Target,2177 MCContext *Ctx, bool IsTailCall) override {2178 // ip0 (r16) is reserved to the linker (refer to 5.3.1.1 of "Procedure Call2179 // Standard for the ARM 64-bit Architecture (AArch64)".2180 // The sequence of instructions we create here is the following:2181 // movz ip0, #:abs_g3:<addr>2182 // movk ip0, #:abs_g2_nc:<addr>2183 // movk ip0, #:abs_g1_nc:<addr>2184 // movk ip0, #:abs_g0_nc:<addr>2185 // br ip02186 MCInst Inst;2187 Inst.setOpcode(AArch64::MOVZXi);2188 Inst.addOperand(MCOperand::createReg(AArch64::X16));2189 Inst.addOperand(MCOperand::createExpr(2190 MCSpecifierExpr::create(Target, AArch64::S_ABS_G3, *Ctx)));2191 Inst.addOperand(MCOperand::createImm(0x30));2192 Seq.emplace_back(Inst);2193 2194 Inst.clear();2195 Inst.setOpcode(AArch64::MOVKXi);2196 Inst.addOperand(MCOperand::createReg(AArch64::X16));2197 Inst.addOperand(MCOperand::createReg(AArch64::X16));2198 Inst.addOperand(MCOperand::createExpr(2199 MCSpecifierExpr::create(Target, AArch64::S_ABS_G2_NC, *Ctx)));2200 Inst.addOperand(MCOperand::createImm(0x20));2201 Seq.emplace_back(Inst);2202 2203 Inst.clear();2204 Inst.setOpcode(AArch64::MOVKXi);2205 Inst.addOperand(MCOperand::createReg(AArch64::X16));2206 Inst.addOperand(MCOperand::createReg(AArch64::X16));2207 Inst.addOperand(MCOperand::createExpr(2208 MCSpecifierExpr::create(Target, AArch64::S_ABS_G1_NC, *Ctx)));2209 Inst.addOperand(MCOperand::createImm(0x10));2210 Seq.emplace_back(Inst);2211 2212 Inst.clear();2213 Inst.setOpcode(AArch64::MOVKXi);2214 Inst.addOperand(MCOperand::createReg(AArch64::X16));2215 Inst.addOperand(MCOperand::createReg(AArch64::X16));2216 Inst.addOperand(MCOperand::createExpr(2217 MCSpecifierExpr::create(Target, AArch64::S_ABS_G0_NC, *Ctx)));2218 Inst.addOperand(MCOperand::createImm(0));2219 Seq.emplace_back(Inst);2220 2221 Inst.clear();2222 Inst.setOpcode(AArch64::BR);2223 Inst.addOperand(MCOperand::createReg(AArch64::X16));2224 if (IsTailCall)2225 setTailCall(Inst);2226 Seq.emplace_back(Inst);2227 }2228 2229 void createShortJmp(InstructionListType &Seq, const MCSymbol *Target,2230 MCContext *Ctx, bool IsTailCall) override {2231 // ip0 (r16) is reserved to the linker (refer to 5.3.1.1 of "Procedure Call2232 // Standard for the ARM 64-bit Architecture (AArch64)".2233 // The sequence of instructions we create here is the following:2234 // adrp ip0, imm2235 // add ip0, ip0, imm2236 // br ip02237 MCPhysReg Reg = AArch64::X16;2238 InstructionListType Insts = materializeAddress(Target, Ctx, Reg);2239 Insts.emplace_back();2240 MCInst &Inst = Insts.back();2241 Inst.clear();2242 Inst.setOpcode(AArch64::BR);2243 Inst.addOperand(MCOperand::createReg(Reg));2244 if (IsTailCall)2245 setTailCall(Inst);2246 Seq.swap(Insts);2247 }2248 2249 /// Matching pattern here is2250 ///2251 /// ADRP x16, imm2252 /// ADD x16, x16, imm2253 /// BR x162254 ///2255 uint64_t matchLinkerVeneer(InstructionIterator Begin, InstructionIterator End,2256 uint64_t Address, const MCInst &CurInst,2257 MCInst *&TargetHiBits, MCInst *&TargetLowBits,2258 uint64_t &Target) const override {2259 if (CurInst.getOpcode() != AArch64::BR || !CurInst.getOperand(0).isReg() ||2260 CurInst.getOperand(0).getReg() != AArch64::X16)2261 return 0;2262 2263 auto I = End;2264 if (I == Begin)2265 return 0;2266 2267 --I;2268 Address -= 4;2269 if (I == Begin || I->getOpcode() != AArch64::ADDXri ||2270 MCPlus::getNumPrimeOperands(*I) < 3 || !I->getOperand(0).isReg() ||2271 !I->getOperand(1).isReg() ||2272 I->getOperand(0).getReg() != AArch64::X16 ||2273 I->getOperand(1).getReg() != AArch64::X16 || !I->getOperand(2).isImm())2274 return 0;2275 TargetLowBits = &*I;2276 uint64_t Addr = I->getOperand(2).getImm() & 0xFFF;2277 2278 --I;2279 Address -= 4;2280 if (I != Begin || I->getOpcode() != AArch64::ADRP ||2281 MCPlus::getNumPrimeOperands(*I) < 2 || !I->getOperand(0).isReg() ||2282 !I->getOperand(1).isImm() || I->getOperand(0).getReg() != AArch64::X16)2283 return 0;2284 TargetHiBits = &*I;2285 Addr |= (Address + ((int64_t)I->getOperand(1).getImm() << 12)) &2286 0xFFFFFFFFFFFFF000ULL;2287 Target = Addr;2288 return 3;2289 }2290 2291 /// Match the following pattern:2292 ///2293 /// LDR x16, .L12294 /// BR x162295 /// L1:2296 /// .quad Target2297 ///2298 /// Populate \p TargetAddress with the Target value on successful match.2299 bool matchAbsLongVeneer(const BinaryFunction &BF,2300 uint64_t &TargetAddress) const override {2301 if (BF.size() != 1 || BF.getMaxSize() < 16)2302 return false;2303 2304 if (!BF.hasConstantIsland())2305 return false;2306 2307 const BinaryBasicBlock &BB = BF.front();2308 if (BB.size() != 2)2309 return false;2310 2311 const MCInst &LDRInst = BB.getInstructionAtIndex(0);2312 if (LDRInst.getOpcode() != AArch64::LDRXl)2313 return false;2314 2315 if (!LDRInst.getOperand(0).isReg() ||2316 LDRInst.getOperand(0).getReg() != AArch64::X16)2317 return false;2318 2319 const MCSymbol *TargetSym = getTargetSymbol(LDRInst, 1);2320 if (!TargetSym)2321 return false;2322 2323 const MCInst &BRInst = BB.getInstructionAtIndex(1);2324 if (BRInst.getOpcode() != AArch64::BR)2325 return false;2326 if (!BRInst.getOperand(0).isReg() ||2327 BRInst.getOperand(0).getReg() != AArch64::X16)2328 return false;2329 2330 const BinaryFunction::IslandInfo &IInfo = BF.getIslandInfo();2331 if (IInfo.HasDynamicRelocations)2332 return false;2333 2334 auto Iter = IInfo.Offsets.find(8);2335 if (Iter == IInfo.Offsets.end() || Iter->second != TargetSym)2336 return false;2337 2338 // Extract the absolute value stored inside the island.2339 StringRef SectionContents = BF.getOriginSection()->getContents();2340 StringRef FunctionContents = SectionContents.substr(2341 BF.getAddress() - BF.getOriginSection()->getAddress(), BF.getMaxSize());2342 2343 const BinaryContext &BC = BF.getBinaryContext();2344 DataExtractor DE(FunctionContents, BC.AsmInfo->isLittleEndian(),2345 BC.AsmInfo->getCodePointerSize());2346 uint64_t Offset = 8;2347 TargetAddress = DE.getAddress(&Offset);2348 2349 return true;2350 }2351 2352 bool matchAdrpAddPair(const MCInst &Adrp, const MCInst &Add) const override {2353 if (!isADRP(Adrp) || !isAddXri(Add))2354 return false;2355 2356 assert(Adrp.getOperand(0).isReg() &&2357 "Unexpected operand in ADRP instruction");2358 MCPhysReg AdrpReg = Adrp.getOperand(0).getReg();2359 assert(Add.getOperand(1).isReg() &&2360 "Unexpected operand in ADDXri instruction");2361 MCPhysReg AddReg = Add.getOperand(1).getReg();2362 return AdrpReg == AddReg;2363 }2364 2365 bool replaceImmWithSymbolRef(MCInst &Inst, const MCSymbol *Symbol,2366 int64_t Addend, MCContext *Ctx, int64_t &Value,2367 uint32_t RelType) const override {2368 unsigned ImmOpNo = -1U;2369 for (unsigned Index = 0; Index < MCPlus::getNumPrimeOperands(Inst);2370 ++Index) {2371 if (Inst.getOperand(Index).isImm()) {2372 ImmOpNo = Index;2373 break;2374 }2375 }2376 if (ImmOpNo == -1U)2377 return false;2378 2379 Value = Inst.getOperand(ImmOpNo).getImm();2380 2381 setOperandToSymbolRef(Inst, ImmOpNo, Symbol, Addend, Ctx, RelType);2382 2383 return true;2384 }2385 2386 void createUncondBranch(MCInst &Inst, const MCSymbol *TBB,2387 MCContext *Ctx) const override {2388 Inst.setOpcode(AArch64::B);2389 Inst.clear();2390 Inst.addOperand(MCOperand::createExpr(2391 getTargetExprFor(Inst, MCSymbolRefExpr::create(TBB, *Ctx), *Ctx, 0)));2392 }2393 2394 bool shouldRecordCodeRelocation(uint32_t RelType) const override {2395 switch (RelType) {2396 case ELF::R_AARCH64_ABS64:2397 case ELF::R_AARCH64_ABS32:2398 case ELF::R_AARCH64_ABS16:2399 case ELF::R_AARCH64_ADD_ABS_LO12_NC:2400 case ELF::R_AARCH64_ADR_GOT_PAGE:2401 case ELF::R_AARCH64_ADR_PREL_LO21:2402 case ELF::R_AARCH64_ADR_PREL_PG_HI21:2403 case ELF::R_AARCH64_ADR_PREL_PG_HI21_NC:2404 case ELF::R_AARCH64_LD64_GOT_LO12_NC:2405 case ELF::R_AARCH64_LDST8_ABS_LO12_NC:2406 case ELF::R_AARCH64_LDST16_ABS_LO12_NC:2407 case ELF::R_AARCH64_LDST32_ABS_LO12_NC:2408 case ELF::R_AARCH64_LDST64_ABS_LO12_NC:2409 case ELF::R_AARCH64_LDST128_ABS_LO12_NC:2410 case ELF::R_AARCH64_TLSDESC_ADD_LO12:2411 case ELF::R_AARCH64_TLSDESC_ADR_PAGE21:2412 case ELF::R_AARCH64_TLSDESC_ADR_PREL21:2413 case ELF::R_AARCH64_TLSDESC_LD64_LO12:2414 case ELF::R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21:2415 case ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:2416 case ELF::R_AARCH64_TLSLE_MOVW_TPREL_G0:2417 case ELF::R_AARCH64_TLSLE_MOVW_TPREL_G0_NC:2418 case ELF::R_AARCH64_MOVW_UABS_G0:2419 case ELF::R_AARCH64_MOVW_UABS_G0_NC:2420 case ELF::R_AARCH64_MOVW_UABS_G1:2421 case ELF::R_AARCH64_MOVW_UABS_G1_NC:2422 case ELF::R_AARCH64_MOVW_UABS_G2:2423 case ELF::R_AARCH64_MOVW_UABS_G2_NC:2424 case ELF::R_AARCH64_MOVW_UABS_G3:2425 case ELF::R_AARCH64_PREL16:2426 case ELF::R_AARCH64_PREL32:2427 case ELF::R_AARCH64_PREL64:2428 return true;2429 case ELF::R_AARCH64_CALL26:2430 case ELF::R_AARCH64_JUMP26:2431 case ELF::R_AARCH64_TSTBR14:2432 case ELF::R_AARCH64_CONDBR19:2433 case ELF::R_AARCH64_TLSDESC_CALL:2434 case ELF::R_AARCH64_TLSLE_ADD_TPREL_HI12:2435 case ELF::R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:2436 return false;2437 default:2438 llvm_unreachable("Unexpected AArch64 relocation type in code");2439 }2440 }2441 2442 StringRef getTrapFillValue() const override {2443 return StringRef("\0\0\0\0", 4);2444 }2445 2446 void createReturn(MCInst &Inst) const override {2447 Inst.setOpcode(AArch64::RET);2448 Inst.clear();2449 Inst.addOperand(MCOperand::createReg(AArch64::LR));2450 }2451 2452 void createStackPointerIncrement(2453 MCInst &Inst, int Size,2454 bool NoFlagsClobber = false /*unused for AArch64*/) const override {2455 Inst.setOpcode(AArch64::SUBXri);2456 Inst.clear();2457 Inst.addOperand(MCOperand::createReg(AArch64::SP));2458 Inst.addOperand(MCOperand::createReg(AArch64::SP));2459 Inst.addOperand(MCOperand::createImm(Size));2460 Inst.addOperand(MCOperand::createImm(0));2461 }2462 2463 void createStackPointerDecrement(2464 MCInst &Inst, int Size,2465 bool NoFlagsClobber = false /*unused for AArch64*/) const override {2466 Inst.setOpcode(AArch64::ADDXri);2467 Inst.clear();2468 Inst.addOperand(MCOperand::createReg(AArch64::SP));2469 Inst.addOperand(MCOperand::createReg(AArch64::SP));2470 Inst.addOperand(MCOperand::createImm(Size));2471 Inst.addOperand(MCOperand::createImm(0));2472 }2473 2474 void createIndirectBranch(MCInst &Inst, MCPhysReg MemBaseReg,2475 int64_t Disp) const {2476 Inst.setOpcode(AArch64::BR);2477 Inst.clear();2478 Inst.addOperand(MCOperand::createReg(MemBaseReg));2479 }2480 2481 InstructionListType createInstrumentedIndCallHandlerExitBB() const override {2482 // Code sequence for instrumented indirect call handler:2483 // ret2484 2485 InstructionListType Insts;2486 2487 Insts.emplace_back();2488 createReturn(Insts.back());2489 2490 return Insts;2491 }2492 2493 InstructionListType2494 createInstrumentedIndTailCallHandlerExitBB() const override {2495 return createInstrumentedIndCallHandlerExitBB();2496 }2497 2498 InstructionListType createGetter(MCContext *Ctx, const char *name) const {2499 InstructionListType Insts(4);2500 MCSymbol *Locs = Ctx->getOrCreateSymbol(name);2501 InstructionListType Addr = materializeAddress(Locs, Ctx, AArch64::X0);2502 std::copy(Addr.begin(), Addr.end(), Insts.begin());2503 assert(Addr.size() == 2 && "Invalid Addr size");2504 loadReg(Insts[2], AArch64::X0, AArch64::X0);2505 createReturn(Insts[3]);2506 return Insts;2507 }2508 2509 InstructionListType createNumCountersGetter(MCContext *Ctx) const override {2510 return createGetter(Ctx, "__bolt_num_counters");2511 }2512 2513 InstructionListType2514 createInstrLocationsGetter(MCContext *Ctx) const override {2515 return createGetter(Ctx, "__bolt_instr_locations");2516 }2517 2518 InstructionListType createInstrTablesGetter(MCContext *Ctx) const override {2519 return createGetter(Ctx, "__bolt_instr_tables");2520 }2521 2522 InstructionListType createInstrNumFuncsGetter(MCContext *Ctx) const override {2523 return createGetter(Ctx, "__bolt_instr_num_funcs");2524 }2525 2526 void convertIndirectCallToLoad(MCInst &Inst, MCPhysReg Reg) override {2527 bool IsTailCall = isTailCall(Inst);2528 if (IsTailCall)2529 removeAnnotation(Inst, MCPlus::MCAnnotation::kTailCall);2530 if (Inst.getOpcode() == AArch64::BR || Inst.getOpcode() == AArch64::BLR) {2531 Inst.setOpcode(AArch64::ORRXrs);2532 Inst.insert(Inst.begin(), MCOperand::createReg(Reg));2533 Inst.insert(Inst.begin() + 1, MCOperand::createReg(AArch64::XZR));2534 Inst.insert(Inst.begin() + 3, MCOperand::createImm(0));2535 return;2536 }2537 llvm_unreachable("not implemented");2538 }2539 2540 InstructionListType createLoadImmediate(const MCPhysReg Dest,2541 uint64_t Imm) const override {2542 InstructionListType Insts(4);2543 int Shift = 48;2544 for (int I = 0; I < 4; I++, Shift -= 16) {2545 Insts[I].setOpcode(AArch64::MOVKXi);2546 Insts[I].addOperand(MCOperand::createReg(Dest));2547 Insts[I].addOperand(MCOperand::createReg(Dest));2548 Insts[I].addOperand(MCOperand::createImm((Imm >> Shift) & 0xFFFF));2549 Insts[I].addOperand(MCOperand::createImm(Shift));2550 }2551 return Insts;2552 }2553 2554 void createIndirectCallInst(MCInst &Inst, bool IsTailCall,2555 MCPhysReg Reg) const {2556 Inst.clear();2557 Inst.setOpcode(IsTailCall ? AArch64::BR : AArch64::BLR);2558 Inst.addOperand(MCOperand::createReg(Reg));2559 }2560 2561 InstructionListType createInstrumentedIndirectCall(MCInst &&CallInst,2562 MCSymbol *HandlerFuncAddr,2563 int CallSiteID,2564 MCContext *Ctx) override {2565 // Code sequence used to enter indirect call instrumentation helper:2566 // stp x0, x1, [sp, #-16]! createPushRegisters (1)2567 // mov target, x0 convertIndirectCallToLoad -> orr x0 target xzr2568 // mov x1 CallSiteID createLoadImmediate ->2569 // movk x1, #0x0, lsl #482570 // movk x1, #0x0, lsl #322571 // movk x1, #0x0, lsl #162572 // movk x1, #0x02573 // stp x0, x30, [sp, #-16]! (2)2574 // adr x0 *HandlerFuncAddr -> adrp + add2575 // blr x0 (__bolt_instr_ind_call_handler_func)2576 // ldp x0, x30, [sp], #16 (2)2577 // mov x0, target ; move target address to used register2578 // ldp x0, x1, [sp], #16 (1)2579 2580 InstructionListType Insts;2581 Insts.emplace_back();2582 createPushRegisters(Insts.back(), getIntArgRegister(0),2583 getIntArgRegister(1));2584 Insts.emplace_back(CallInst);2585 convertIndirectCallToLoad(Insts.back(), getIntArgRegister(0));2586 InstructionListType LoadImm =2587 createLoadImmediate(getIntArgRegister(1), CallSiteID);2588 Insts.insert(Insts.end(), LoadImm.begin(), LoadImm.end());2589 Insts.emplace_back();2590 createPushRegisters(Insts.back(), getIntArgRegister(0), AArch64::LR);2591 Insts.resize(Insts.size() + 2);2592 InstructionListType Addr = materializeAddress(2593 HandlerFuncAddr, Ctx, CallInst.getOperand(0).getReg());2594 assert(Addr.size() == 2 && "Invalid Addr size");2595 std::copy(Addr.begin(), Addr.end(), Insts.end() - Addr.size());2596 2597 Insts.emplace_back();2598 createIndirectCallInst(Insts.back(), false,2599 CallInst.getOperand(0).getReg());2600 2601 Insts.emplace_back();2602 createPopRegisters(Insts.back(), getIntArgRegister(0), AArch64::LR);2603 2604 // move x0 to indirect call register2605 Insts.emplace_back();2606 Insts.back().setOpcode(AArch64::ORRXrs);2607 Insts.back().insert(Insts.back().begin(),2608 MCOperand::createReg(CallInst.getOperand(0).getReg()));2609 Insts.back().insert(Insts.back().begin() + 1,2610 MCOperand::createReg(AArch64::XZR));2611 Insts.back().insert(Insts.back().begin() + 2,2612 MCOperand::createReg(getIntArgRegister(0)));2613 Insts.back().insert(Insts.back().begin() + 3, MCOperand::createImm(0));2614 2615 Insts.emplace_back();2616 createPopRegisters(Insts.back(), getIntArgRegister(0),2617 getIntArgRegister(1));2618 2619 return Insts;2620 }2621 2622 InstructionListType2623 createInstrumentedIndCallHandlerEntryBB(const MCSymbol *InstrTrampoline,2624 const MCSymbol *IndCallHandler,2625 MCContext *Ctx) override {2626 // Code sequence used to check whether InstrTrampoline was initialized2627 // and call it if so, returns via IndCallHandler2628 // adrp x0, InstrTrampoline2629 // ldr x0, [x0, #lo12:InstrTrampoline]2630 // subs x0, x0, #0x02631 // b.eq IndCallHandler2632 // str x30, [sp, #-16]!2633 // blr x02634 // ldr x30, [sp], #162635 // b IndCallHandler2636 InstructionListType Insts;2637 2638 // load handler address2639 MCInst InstAdrp;2640 InstAdrp.setOpcode(AArch64::ADRP);2641 InstAdrp.addOperand(MCOperand::createReg(getIntArgRegister(0)));2642 InstAdrp.addOperand(MCOperand::createImm(0));2643 setOperandToSymbolRef(InstAdrp, /* OpNum */ 1, InstrTrampoline,2644 /* Addend */ 0, Ctx, ELF::R_AARCH64_ADR_GOT_PAGE);2645 Insts.emplace_back(InstAdrp);2646 2647 MCInst InstLoad;2648 InstLoad.setOpcode(AArch64::LDRXui);2649 InstLoad.addOperand(MCOperand::createReg(getIntArgRegister(0)));2650 InstLoad.addOperand(MCOperand::createReg(getIntArgRegister(0)));2651 InstLoad.addOperand(MCOperand::createImm(0));2652 setOperandToSymbolRef(InstLoad, /* OpNum */ 2, InstrTrampoline,2653 /* Addend */ 0, Ctx, ELF::R_AARCH64_LD64_GOT_LO12_NC);2654 Insts.emplace_back(InstLoad);2655 2656 InstructionListType CmpJmp =2657 createCmpJE(getIntArgRegister(0), 0, IndCallHandler, Ctx);2658 Insts.insert(Insts.end(), CmpJmp.begin(), CmpJmp.end());2659 2660 Insts.emplace_back();2661 storeReg(Insts.back(), AArch64::LR, getSpRegister(/*Size*/ 8));2662 2663 Insts.emplace_back();2664 Insts.back().setOpcode(AArch64::BLR);2665 Insts.back().addOperand(MCOperand::createReg(getIntArgRegister(0)));2666 2667 Insts.emplace_back();2668 loadReg(Insts.back(), AArch64::LR, getSpRegister(/*Size*/ 8));2669 2670 Insts.emplace_back();2671 createDirectBranch(Insts.back(), IndCallHandler, Ctx);2672 2673 return Insts;2674 }2675 2676 // Instrumentation code sequence using LSE atomic instruction has a total of2677 // 6 instructions:2678 //2679 // stp x0, x1, [sp, #-0x10]!2680 // adrp x0, page_address(counter)2681 // add x0, x0, page_offset(counter)2682 // mov x1, #0x12683 // stadd x1, [x0]2684 // ldp x0, x1, [sp], #0x102685 //2686 // Instrumentation code sequence without using LSE atomic instruction has2687 // 8 instructions at instrumentation place, with 6 instructions in the helper:2688 //2689 // stp x0, x30, [sp, #-0x10]!2690 // stp x1, x2, [sp, #-0x10]!2691 // adrp x0, page_address(counter)2692 // add x0, x0, page_offset(counter)2693 // adrp x1, page_address(helper)2694 // add x1, x1, page_offset(helper)2695 // blr x12696 // ldp x0, x30, [sp], #0x102697 //2698 // <helper>:2699 // ldaxr x1, [x0]2700 // add x1, x1, #0x12701 // stlxr w2, x1, [x0]2702 // cbnz w2, <helper>2703 // ldp x1, x2, [sp], #0x102704 // ret2705 2706 void createInstrCounterIncrFunc(BinaryContext &BC) override {2707 assert(InstrCounterIncrFunc == nullptr &&2708 "helper function of counter increment for instrumentation "2709 "has already been created");2710 2711 if (!opts::NoLSEAtomics)2712 return;2713 2714 MCContext *Ctx = BC.Ctx.get();2715 InstrCounterIncrFunc = BC.createInjectedBinaryFunction(2716 "__bolt_instr_counter_incr", /*IsSimple*/ false);2717 std::vector<std::unique_ptr<BinaryBasicBlock>> BBs;2718 2719 BBs.emplace_back(InstrCounterIncrFunc->createBasicBlock());2720 InstructionListType Instrs(4);2721 Instrs[0].setOpcode(AArch64::LDAXRX);2722 Instrs[0].clear();2723 Instrs[0].addOperand(MCOperand::createReg(AArch64::X1));2724 Instrs[0].addOperand(MCOperand::createReg(AArch64::X0));2725 Instrs[1].setOpcode(AArch64::ADDXri);2726 Instrs[1].clear();2727 Instrs[1].addOperand(MCOperand::createReg(AArch64::X1));2728 Instrs[1].addOperand(MCOperand::createReg(AArch64::X1));2729 Instrs[1].addOperand(MCOperand::createImm(1));2730 Instrs[1].addOperand(MCOperand::createImm(0));2731 Instrs[2].setOpcode(AArch64::STLXRX);2732 Instrs[2].clear();2733 Instrs[2].addOperand(MCOperand::createReg(AArch64::W2));2734 Instrs[2].addOperand(MCOperand::createReg(AArch64::X1));2735 Instrs[2].addOperand(MCOperand::createReg(AArch64::X0));2736 Instrs[3].setOpcode(AArch64::CBNZW);2737 Instrs[3].clear();2738 Instrs[3].addOperand(MCOperand::createReg(AArch64::W2));2739 Instrs[3].addOperand(MCOperand::createExpr(2740 MCSymbolRefExpr::create(BBs.back()->getLabel(), *Ctx)));2741 BBs.back()->addInstructions(Instrs.begin(), Instrs.end());2742 BBs.back()->setCFIState(0);2743 2744 BBs.emplace_back(InstrCounterIncrFunc->createBasicBlock());2745 InstructionListType InstrsEpilog(2);2746 createPopRegisters(InstrsEpilog[0], AArch64::X1, AArch64::X2);2747 createReturn(InstrsEpilog[1]);2748 BBs.back()->addInstructions(InstrsEpilog.begin(), InstrsEpilog.end());2749 BBs.back()->setCFIState(0);2750 2751 BBs[0]->addSuccessor(BBs[0].get());2752 BBs[0]->addSuccessor(BBs[1].get());2753 2754 InstrCounterIncrFunc->insertBasicBlocks(nullptr, std::move(BBs),2755 /*UpdateLayout*/ true,2756 /*UpdateCFIState*/ false);2757 InstrCounterIncrFunc->updateState(BinaryFunction::State::CFG_Finalized);2758 2759 LLVM_DEBUG({2760 dbgs() << "BOLT-DEBUG: instrumentation counter increment helper:\n";2761 InstrCounterIncrFunc->dump();2762 });2763 }2764 2765 InstructionListType createInstrIncMemory(const MCSymbol *Target,2766 MCContext *Ctx, bool IsLeaf,2767 unsigned CodePointerSize) override {2768 unsigned int I = 0;2769 InstructionListType Instrs(opts::NoLSEAtomics ? 8 : 6);2770 2771 if (opts::NoLSEAtomics) {2772 createPushRegisters(Instrs[I++], AArch64::X0, AArch64::LR);2773 createPushRegisters(Instrs[I++], AArch64::X1, AArch64::X2);2774 } else {2775 createPushRegisters(Instrs[I++], AArch64::X0, AArch64::X1);2776 }2777 2778 InstructionListType Addr = materializeAddress(Target, Ctx, AArch64::X0);2779 assert(Addr.size() == 2 && "Invalid Addr size");2780 std::copy(Addr.begin(), Addr.end(), Instrs.begin() + I);2781 I += Addr.size();2782 2783 if (opts::NoLSEAtomics) {2784 const MCSymbol *Helper = InstrCounterIncrFunc->getSymbol();2785 InstructionListType HelperAddr =2786 materializeAddress(Helper, Ctx, AArch64::X1);2787 assert(HelperAddr.size() == 2 && "Invalid HelperAddr size");2788 std::copy(HelperAddr.begin(), HelperAddr.end(), Instrs.begin() + I);2789 I += HelperAddr.size();2790 createIndirectCallInst(Instrs[I++], /*IsTailCall*/ false, AArch64::X1);2791 } else {2792 InstructionListType Insts = createIncMemory(AArch64::X0, AArch64::X1);2793 assert(Insts.size() == 2 && "Invalid Insts size");2794 std::copy(Insts.begin(), Insts.end(), Instrs.begin() + I);2795 I += Insts.size();2796 }2797 createPopRegisters(Instrs[I++], AArch64::X0,2798 opts::NoLSEAtomics ? AArch64::LR : AArch64::X1);2799 return Instrs;2800 }2801 2802 std::vector<MCInst> createSymbolTrampoline(const MCSymbol *TgtSym,2803 MCContext *Ctx) override {2804 std::vector<MCInst> Insts;2805 createShortJmp(Insts, TgtSym, Ctx, /*IsTailCall*/ true);2806 return Insts;2807 }2808 2809 void createBTI(MCInst &Inst, bool CallTarget,2810 bool JumpTarget) const override {2811 Inst.setOpcode(AArch64::HINT);2812 unsigned HintNum = getBTIHintNum(CallTarget, JumpTarget);2813 Inst.addOperand(MCOperand::createImm(HintNum));2814 }2815 2816 bool isBTILandingPad(MCInst &Inst, bool CallTarget,2817 bool JumpTarget) const override {2818 unsigned HintNum = getBTIHintNum(CallTarget, JumpTarget);2819 bool IsExplicitBTI =2820 Inst.getOpcode() == AArch64::HINT && Inst.getNumOperands() == 1 &&2821 Inst.getOperand(0).isImm() && Inst.getOperand(0).getImm() == HintNum;2822 2823 bool IsImplicitBTI = HintNum == 34 && isImplicitBTIC(Inst);2824 return IsExplicitBTI || IsImplicitBTI;2825 }2826 2827 bool isImplicitBTIC(MCInst &Inst) const override {2828 // PACI[AB]SP are always implicitly BTI C, independently of2829 // SCTLR_EL1.BT[01].2830 return Inst.getOpcode() == AArch64::PACIASP ||2831 Inst.getOpcode() == AArch64::PACIBSP;2832 }2833 2834 void updateBTIVariant(MCInst &Inst, bool CallTarget,2835 bool JumpTarget) const override {2836 assert(Inst.getOpcode() == AArch64::HINT && "Not a BTI instruction.");2837 unsigned HintNum = getBTIHintNum(CallTarget, JumpTarget);2838 Inst.clear();2839 Inst.addOperand(MCOperand::createImm(HintNum));2840 }2841 2842 InstructionListType materializeAddress(const MCSymbol *Target, MCContext *Ctx,2843 MCPhysReg RegName,2844 int64_t Addend = 0) const override {2845 // Get page-aligned address and add page offset2846 InstructionListType Insts(2);2847 Insts[0].setOpcode(AArch64::ADRP);2848 Insts[0].clear();2849 Insts[0].addOperand(MCOperand::createReg(RegName));2850 Insts[0].addOperand(MCOperand::createImm(0));2851 setOperandToSymbolRef(Insts[0], /* OpNum */ 1, Target, Addend, Ctx,2852 ELF::R_AARCH64_NONE);2853 Insts[1].setOpcode(AArch64::ADDXri);2854 Insts[1].clear();2855 Insts[1].addOperand(MCOperand::createReg(RegName));2856 Insts[1].addOperand(MCOperand::createReg(RegName));2857 Insts[1].addOperand(MCOperand::createImm(0));2858 Insts[1].addOperand(MCOperand::createImm(0));2859 setOperandToSymbolRef(Insts[1], /* OpNum */ 2, Target, Addend, Ctx,2860 ELF::R_AARCH64_ADD_ABS_LO12_NC);2861 return Insts;2862 }2863 2864 std::optional<Relocation>2865 createRelocation(const MCFixup &Fixup,2866 const MCAsmBackend &MAB) const override {2867 MCFixupKindInfo FKI = MAB.getFixupKindInfo(Fixup.getKind());2868 2869 assert(FKI.TargetOffset == 0 && "0-bit relocation offset expected");2870 const uint64_t RelOffset = Fixup.getOffset();2871 2872 uint32_t RelType;2873 if (Fixup.getKind() == MCFixupKind(AArch64::fixup_aarch64_pcrel_call26))2874 RelType = ELF::R_AARCH64_CALL26;2875 else if (Fixup.getKind() ==2876 MCFixupKind(AArch64::fixup_aarch64_pcrel_branch26))2877 RelType = ELF::R_AARCH64_JUMP26;2878 else if (Fixup.isPCRel()) {2879 switch (FKI.TargetSize) {2880 default:2881 return std::nullopt;2882 case 16:2883 RelType = ELF::R_AARCH64_PREL16;2884 break;2885 case 32:2886 RelType = ELF::R_AARCH64_PREL32;2887 break;2888 case 64:2889 RelType = ELF::R_AARCH64_PREL64;2890 break;2891 }2892 } else {2893 switch (FKI.TargetSize) {2894 default:2895 return std::nullopt;2896 case 16:2897 RelType = ELF::R_AARCH64_ABS16;2898 break;2899 case 32:2900 RelType = ELF::R_AARCH64_ABS32;2901 break;2902 case 64:2903 RelType = ELF::R_AARCH64_ABS64;2904 break;2905 }2906 }2907 2908 auto [RelSymbol, RelAddend] = extractFixupExpr(Fixup);2909 2910 return Relocation({RelOffset, RelSymbol, RelType, RelAddend, 0});2911 }2912 2913 uint16_t getMinFunctionAlignment() const override { return 4; }2914 2915 std::optional<uint32_t>2916 getInstructionSize(const MCInst &Inst) const override {2917 return 4;2918 }2919 2920 std::optional<uint64_t>2921 extractMoveImmediate(const MCInst &Inst, MCPhysReg TargetReg) const override {2922 // Match MOVZ instructions (both X and W register variants) with no shift.2923 if ((Inst.getOpcode() == AArch64::MOVZXi ||2924 Inst.getOpcode() == AArch64::MOVZWi) &&2925 Inst.getOperand(2).getImm() == 0 &&2926 getAliases(TargetReg)[Inst.getOperand(0).getReg()])2927 return Inst.getOperand(1).getImm();2928 return std::nullopt;2929 }2930 2931 std::optional<uint64_t>2932 findMemcpySizeInBytes(const BinaryBasicBlock &BB,2933 BinaryBasicBlock::iterator CallInst) const override {2934 MCPhysReg SizeReg = getIntArgRegister(2);2935 if (SizeReg == getNoRegister())2936 return std::nullopt;2937 2938 BitVector WrittenRegs(RegInfo->getNumRegs());2939 const BitVector &SizeRegAliases = getAliases(SizeReg);2940 2941 for (auto InstIt = CallInst; InstIt != BB.begin(); --InstIt) {2942 const MCInst &Inst = *InstIt;2943 WrittenRegs.reset();2944 getWrittenRegs(Inst, WrittenRegs);2945 2946 if (WrittenRegs.anyCommon(SizeRegAliases))2947 return extractMoveImmediate(Inst, SizeReg);2948 }2949 return std::nullopt;2950 }2951 2952 InstructionListType2953 createInlineMemcpy(bool ReturnEnd,2954 std::optional<uint64_t> KnownSize) const override {2955 assert(KnownSize.has_value() &&2956 "AArch64 memcpy inlining requires known size");2957 InstructionListType Code;2958 uint64_t Size = *KnownSize;2959 2960 generateSizeSpecificMemcpy(Code, Size);2961 2962 // If _memcpy8, adjust X0 to return dest+size instead of dest.2963 if (ReturnEnd)2964 Code.emplace_back(MCInstBuilder(AArch64::ADDXri)2965 .addReg(AArch64::X0)2966 .addReg(AArch64::X0)2967 .addImm(Size)2968 .addImm(0));2969 return Code;2970 }2971 2972 InstructionListType generateSizeSpecificMemcpy(InstructionListType &Code,2973 uint64_t Size) const {2974 auto AddLoadStorePair = [&](unsigned LoadOpc, unsigned StoreOpc,2975 unsigned Reg, unsigned Offset = 0) {2976 Code.emplace_back(MCInstBuilder(LoadOpc)2977 .addReg(Reg)2978 .addReg(AArch64::X1)2979 .addImm(Offset));2980 Code.emplace_back(MCInstBuilder(StoreOpc)2981 .addReg(Reg)2982 .addReg(AArch64::X0)2983 .addImm(Offset));2984 };2985 2986 // Generate optimal instruction sequences based on exact size.2987 switch (Size) {2988 case 1:2989 AddLoadStorePair(AArch64::LDRBBui, AArch64::STRBBui, AArch64::W9);2990 break;2991 case 2:2992 AddLoadStorePair(AArch64::LDRHHui, AArch64::STRHHui, AArch64::W9);2993 break;2994 case 4:2995 AddLoadStorePair(AArch64::LDRWui, AArch64::STRWui, AArch64::W9);2996 break;2997 case 8:2998 AddLoadStorePair(AArch64::LDRXui, AArch64::STRXui, AArch64::X9);2999 break;3000 case 16:3001 AddLoadStorePair(AArch64::LDRQui, AArch64::STRQui, AArch64::Q16);3002 break;3003 case 32:3004 AddLoadStorePair(AArch64::LDRQui, AArch64::STRQui, AArch64::Q16, 0);3005 AddLoadStorePair(AArch64::LDRQui, AArch64::STRQui, AArch64::Q17, 1);3006 break;3007 3008 default:3009 // For sizes up to 64 bytes, greedily use the largest possible loads.3010 // Caller should have already filtered out sizes > 64 bytes.3011 assert(Size <= 64 &&3012 "Size should be <= 64 bytes for AArch64 memcpy inlining");3013 3014 uint64_t Remaining = Size;3015 uint64_t Offset = 0;3016 3017 const std::array<std::tuple<uint64_t, unsigned, unsigned, unsigned>, 5>3018 LoadStoreOps = {3019 {{16, AArch64::LDRQui, AArch64::STRQui, AArch64::Q16},3020 {8, AArch64::LDRXui, AArch64::STRXui, AArch64::X9},3021 {4, AArch64::LDRWui, AArch64::STRWui, AArch64::W9},3022 {2, AArch64::LDRHHui, AArch64::STRHHui, AArch64::W9},3023 {1, AArch64::LDRBBui, AArch64::STRBBui, AArch64::W9}}};3024 3025 for (const auto &[OpSize, LoadOp, StoreOp, TempReg] : LoadStoreOps)3026 while (Remaining >= OpSize) {3027 AddLoadStorePair(LoadOp, StoreOp, TempReg, Offset / OpSize);3028 Remaining -= OpSize;3029 Offset += OpSize;3030 }3031 break;3032 }3033 return Code;3034 }3035};3036 3037} // end anonymous namespace3038 3039namespace llvm {3040namespace bolt {3041 3042MCPlusBuilder *createAArch64MCPlusBuilder(const MCInstrAnalysis *Analysis,3043 const MCInstrInfo *Info,3044 const MCRegisterInfo *RegInfo,3045 const MCSubtargetInfo *STI) {3046 return new AArch64MCPlusBuilder(Analysis, Info, RegInfo, STI);3047}3048 3049} // namespace bolt3050} // namespace llvm3051