642 lines · cpp
1//===- AArch64ErrataFix.cpp -----------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8// This file implements Section Patching for the purpose of working around9// the AArch64 Cortex-53 errata 843419 that affects r0p0, r0p1, r0p2 and r0p410// versions of the core.11//12// The general principle is that an erratum sequence of one or13// more instructions is detected in the instruction stream, one of the14// instructions in the sequence is replaced with a branch to a patch sequence15// of replacement instructions. At the end of the replacement sequence the16// patch branches back to the instruction stream.17 18// This technique is only suitable for fixing an erratum when:19// - There is a set of necessary conditions required to trigger the erratum that20// can be detected at static link time.21// - There is a set of replacement instructions that can be used to remove at22// least one of the necessary conditions that trigger the erratum.23// - We can overwrite an instruction in the erratum sequence with a branch to24// the replacement sequence.25// - We can place the replacement sequence within range of the branch.26//===----------------------------------------------------------------------===//27 28#include "AArch64ErrataFix.h"29#include "InputFiles.h"30#include "LinkerScript.h"31#include "OutputSections.h"32#include "Relocations.h"33#include "Symbols.h"34#include "SyntheticSections.h"35#include "Target.h"36#include "llvm/ADT/StringExtras.h"37#include "llvm/Support/Endian.h"38#include <algorithm>39 40using namespace llvm;41using namespace llvm::ELF;42using namespace llvm::object;43using namespace llvm::support;44using namespace llvm::support::endian;45using namespace lld;46using namespace lld::elf;47 48// Helper functions to identify instructions and conditions needed to trigger49// the Cortex-A53-843419 erratum.50 51// ADRP52// | 1 | immlo (2) | 1 | 0 0 0 0 | immhi (19) | Rd (5) |53static bool isADRP(uint32_t instr) {54 return (instr & 0x9f000000) == 0x90000000;55}56 57// Load and store bit patterns from ARMv8-A.58// Instructions appear in order of appearance starting from table in59// C4.1.3 Loads and Stores.60 61// All loads and stores have 1 (at bit position 27), (0 at bit position 25).62// | op0 x op1 (2) | 1 op2 0 op3 (2) | x | op4 (5) | xxxx | op5 (2) | x (10) |63static bool isLoadStoreClass(uint32_t instr) {64 return (instr & 0x0a000000) == 0x08000000;65}66 67// LDN/STN multiple no offset68// | 0 Q 00 | 1100 | 0 L 00 | 0000 | opcode (4) | size (2) | Rn (5) | Rt (5) |69// LDN/STN multiple post-indexed70// | 0 Q 00 | 1100 | 1 L 0 | Rm (5)| opcode (4) | size (2) | Rn (5) | Rt (5) |71// L == 0 for stores.72 73// Utility routine to decode opcode field of LDN/STN multiple structure74// instructions to find the ST1 instructions.75// opcode == 0010 ST1 4 registers.76// opcode == 0110 ST1 3 registers.77// opcode == 0111 ST1 1 register.78// opcode == 1010 ST1 2 registers.79static bool isST1MultipleOpcode(uint32_t instr) {80 return (instr & 0x0000f000) == 0x00002000 ||81 (instr & 0x0000f000) == 0x00006000 ||82 (instr & 0x0000f000) == 0x00007000 ||83 (instr & 0x0000f000) == 0x0000a000;84}85 86static bool isST1Multiple(uint32_t instr) {87 return (instr & 0xbfff0000) == 0x0c000000 && isST1MultipleOpcode(instr);88}89 90// Writes to Rn (writeback).91static bool isST1MultiplePost(uint32_t instr) {92 return (instr & 0xbfe00000) == 0x0c800000 && isST1MultipleOpcode(instr);93}94 95// LDN/STN single no offset96// | 0 Q 00 | 1101 | 0 L R 0 | 0000 | opc (3) S | size (2) | Rn (5) | Rt (5)|97// LDN/STN single post-indexed98// | 0 Q 00 | 1101 | 1 L R | Rm (5) | opc (3) S | size (2) | Rn (5) | Rt (5)|99// L == 0 for stores100 101// Utility routine to decode opcode field of LDN/STN single structure102// instructions to find the ST1 instructions.103// R == 0 for ST1 and ST3, R == 1 for ST2 and ST4.104// opcode == 000 ST1 8-bit.105// opcode == 010 ST1 16-bit.106// opcode == 100 ST1 32 or 64-bit (Size determines which).107static bool isST1SingleOpcode(uint32_t instr) {108 return (instr & 0x0040e000) == 0x00000000 ||109 (instr & 0x0040e000) == 0x00004000 ||110 (instr & 0x0040e000) == 0x00008000;111}112 113static bool isST1Single(uint32_t instr) {114 return (instr & 0xbfff0000) == 0x0d000000 && isST1SingleOpcode(instr);115}116 117// Writes to Rn (writeback).118static bool isST1SinglePost(uint32_t instr) {119 return (instr & 0xbfe00000) == 0x0d800000 && isST1SingleOpcode(instr);120}121 122static bool isST1(uint32_t instr) {123 return isST1Multiple(instr) || isST1MultiplePost(instr) ||124 isST1Single(instr) || isST1SinglePost(instr);125}126 127// Load/store exclusive128// | size (2) 00 | 1000 | o2 L o1 | Rs (5) | o0 | Rt2 (5) | Rn (5) | Rt (5) |129// L == 0 for Stores.130static bool isLoadStoreExclusive(uint32_t instr) {131 return (instr & 0x3f000000) == 0x08000000;132}133 134static bool isLoadExclusive(uint32_t instr) {135 return (instr & 0x3f400000) == 0x08400000;136}137 138// Load register literal139// | opc (2) 01 | 1 V 00 | imm19 | Rt (5) |140static bool isLoadLiteral(uint32_t instr) {141 return (instr & 0x3b000000) == 0x18000000;142}143 144// Load/store no-allocate pair145// (offset)146// | opc (2) 10 | 1 V 00 | 0 L | imm7 | Rt2 (5) | Rn (5) | Rt (5) |147// L == 0 for stores.148// Never writes to register149static bool isSTNP(uint32_t instr) {150 return (instr & 0x3bc00000) == 0x28000000;151}152 153// Load/store register pair154// (post-indexed)155// | opc (2) 10 | 1 V 00 | 1 L | imm7 | Rt2 (5) | Rn (5) | Rt (5) |156// L == 0 for stores, V == 0 for Scalar, V == 1 for Simd/FP157// Writes to Rn.158static bool isSTPPost(uint32_t instr) {159 return (instr & 0x3bc00000) == 0x28800000;160}161 162// (offset)163// | opc (2) 10 | 1 V 01 | 0 L | imm7 | Rt2 (5) | Rn (5) | Rt (5) |164static bool isSTPOffset(uint32_t instr) {165 return (instr & 0x3bc00000) == 0x29000000;166}167 168// (pre-index)169// | opc (2) 10 | 1 V 01 | 1 L | imm7 | Rt2 (5) | Rn (5) | Rt (5) |170// Writes to Rn.171static bool isSTPPre(uint32_t instr) {172 return (instr & 0x3bc00000) == 0x29800000;173}174 175static bool isSTP(uint32_t instr) {176 return isSTPPost(instr) || isSTPOffset(instr) || isSTPPre(instr);177}178 179// Load/store register (unscaled immediate)180// | size (2) 11 | 1 V 00 | opc (2) 0 | imm9 | 00 | Rn (5) | Rt (5) |181// V == 0 for Scalar, V == 1 for Simd/FP.182static bool isLoadStoreUnscaled(uint32_t instr) {183 return (instr & 0x3b000c00) == 0x38000000;184}185 186// Load/store register (immediate post-indexed)187// | size (2) 11 | 1 V 00 | opc (2) 0 | imm9 | 01 | Rn (5) | Rt (5) |188static bool isLoadStoreImmediatePost(uint32_t instr) {189 return (instr & 0x3b200c00) == 0x38000400;190}191 192// Load/store register (unprivileged)193// | size (2) 11 | 1 V 00 | opc (2) 0 | imm9 | 10 | Rn (5) | Rt (5) |194static bool isLoadStoreUnpriv(uint32_t instr) {195 return (instr & 0x3b200c00) == 0x38000800;196}197 198// Load/store register (immediate pre-indexed)199// | size (2) 11 | 1 V 00 | opc (2) 0 | imm9 | 11 | Rn (5) | Rt (5) |200static bool isLoadStoreImmediatePre(uint32_t instr) {201 return (instr & 0x3b200c00) == 0x38000c00;202}203 204// Load/store register (register offset)205// | size (2) 11 | 1 V 00 | opc (2) 1 | Rm (5) | option (3) S | 10 | Rn | Rt |206static bool isLoadStoreRegisterOff(uint32_t instr) {207 return (instr & 0x3b200c00) == 0x38200800;208}209 210// Load/store register (unsigned immediate)211// | size (2) 11 | 1 V 01 | opc (2) | imm12 | Rn (5) | Rt (5) |212static bool isLoadStoreRegisterUnsigned(uint32_t instr) {213 return (instr & 0x3b000000) == 0x39000000;214}215 216// Rt is always in bit position 0 - 4.217static uint32_t getRt(uint32_t instr) { return (instr & 0x1f); }218 219// Rn is always in bit position 5 - 9.220static uint32_t getRn(uint32_t instr) { return (instr >> 5) & 0x1f; }221 222// C4.1.2 Branches, Exception Generating and System instructions223// | op0 (3) 1 | 01 op1 (4) | x (22) |224// op0 == 010 101 op1 == 0xxx Conditional Branch.225// op0 == 110 101 op1 == 1xxx Unconditional Branch Register.226// op0 == x00 101 op1 == xxxx Unconditional Branch immediate.227// op0 == x01 101 op1 == 0xxx Compare and branch immediate.228// op0 == x01 101 op1 == 1xxx Test and branch immediate.229static bool isBranch(uint32_t instr) {230 return ((instr & 0xfe000000) == 0xd6000000) || // Cond branch.231 ((instr & 0xfe000000) == 0x54000000) || // Uncond branch reg.232 ((instr & 0x7c000000) == 0x14000000) || // Uncond branch imm.233 ((instr & 0x7c000000) == 0x34000000); // Compare and test branch.234}235 236static bool isV8SingleRegisterNonStructureLoadStore(uint32_t instr) {237 return isLoadStoreUnscaled(instr) || isLoadStoreImmediatePost(instr) ||238 isLoadStoreUnpriv(instr) || isLoadStoreImmediatePre(instr) ||239 isLoadStoreRegisterOff(instr) || isLoadStoreRegisterUnsigned(instr);240}241 242// Note that this function refers to v8.0 only and does not include the243// additional load and store instructions added for in later revisions of244// the architecture such as the Atomic memory operations introduced245// in v8.1.246static bool isV8NonStructureLoad(uint32_t instr) {247 if (isLoadExclusive(instr))248 return true;249 if (isLoadLiteral(instr))250 return true;251 else if (isV8SingleRegisterNonStructureLoadStore(instr)) {252 // For Load and Store single register, Loads are derived from a253 // combination of the Size, V and Opc fields.254 uint32_t size = (instr >> 30) & 0xff;255 uint32_t v = (instr >> 26) & 0x1;256 uint32_t opc = (instr >> 22) & 0x3;257 // For the load and store instructions that we are decoding.258 // Opc == 0 are all stores.259 // Opc == 1 with a couple of exceptions are loads. The exceptions are:260 // Size == 00 (0), V == 1, Opc == 10 (2) which is a store and261 // Size == 11 (3), V == 0, Opc == 10 (2) which is a prefetch.262 return opc != 0 && !(size == 0 && v == 1 && opc == 2) &&263 !(size == 3 && v == 0 && opc == 2);264 }265 return false;266}267 268// The following decode instructions are only complete up to the instructions269// needed for errata 843419.270 271// Instruction with writeback updates the index register after the load/store.272static bool hasWriteback(uint32_t instr) {273 return isLoadStoreImmediatePre(instr) || isLoadStoreImmediatePost(instr) ||274 isSTPPre(instr) || isSTPPost(instr) || isST1SinglePost(instr) ||275 isST1MultiplePost(instr);276}277 278// For the load and store class of instructions, a load can write to the279// destination register, a load and a store can write to the base register when280// the instruction has writeback.281static bool doesLoadStoreWriteToReg(uint32_t instr, uint32_t reg) {282 return (isV8NonStructureLoad(instr) && getRt(instr) == reg) ||283 (hasWriteback(instr) && getRn(instr) == reg);284}285 286// Scanner for Cortex-A53 errata 843419287// Full details are available in the Cortex A53 MPCore revision 0 Software288// Developers Errata Notice (ARM-EPM-048406).289//290// The instruction sequence that triggers the erratum is common in compiled291// AArch64 code, however it is sensitive to the offset of the sequence within292// a 4k page. This means that by scanning and fixing the patch after we have293// assigned addresses we only need to disassemble and fix instances of the294// sequence in the range of affected offsets.295//296// In summary the erratum conditions are a series of 4 instructions:297// 1.) An ADRP instruction that writes to register Rn with low 12 bits of298// address of instruction either 0xff8 or 0xffc.299// 2.) A load or store instruction that can be:300// - A single register load or store, of either integer or vector registers.301// - An STP or STNP, of either integer or vector registers.302// - An Advanced SIMD ST1 store instruction.303// - Must not write to Rn, but may optionally read from it.304// 3.) An optional instruction that is not a branch and does not write to Rn.305// 4.) A load or store from the Load/store register (unsigned immediate) class306// that uses Rn as the base address register.307//308// Note that we do not attempt to scan for Sequence 2 as described in the309// Software Developers Errata Notice as this has been assessed to be extremely310// unlikely to occur in compiled code. This matches gold and ld.bfd behavior.311 312// Return true if the Instruction sequence Adrp, Instr2, and Instr4 match313// the erratum sequence. The Adrp, Instr2 and Instr4 correspond to 1.), 2.),314// and 4.) in the Scanner for Cortex-A53 errata comment above.315static bool is843419ErratumSequence(uint32_t instr1, uint32_t instr2,316 uint32_t instr4) {317 if (!isADRP(instr1))318 return false;319 320 uint32_t rn = getRt(instr1);321 return isLoadStoreClass(instr2) &&322 (isLoadStoreExclusive(instr2) || isLoadLiteral(instr2) ||323 isV8SingleRegisterNonStructureLoadStore(instr2) || isSTP(instr2) ||324 isSTNP(instr2) || isST1(instr2)) &&325 !doesLoadStoreWriteToReg(instr2, rn) &&326 isLoadStoreRegisterUnsigned(instr4) && getRn(instr4) == rn;327}328 329// Scan the instruction sequence starting at Offset Off from the base of330// InputSection isec. We update Off in this function rather than in the caller331// as we can skip ahead much further into the section when we know how many332// instructions we've scanned.333// Return the offset of the load or store instruction in isec that we want to334// patch or 0 if no patch required.335static uint64_t scanCortexA53Errata843419(InputSection *isec, uint64_t &off,336 uint64_t limit) {337 uint64_t isecAddr = isec->getVA(0);338 339 // Advance Off so that (isecAddr + Off) modulo 0x1000 is at least 0xff8.340 uint64_t initialPageOff = (isecAddr + off) & 0xfff;341 if (initialPageOff < 0xff8)342 off += 0xff8 - initialPageOff;343 344 bool optionalAllowed = limit - off > 12;345 if (off >= limit || limit - off < 12) {346 // Need at least 3 4-byte sized instructions to trigger erratum.347 off = limit;348 return 0;349 }350 351 uint64_t patchOff = 0;352 const uint8_t *buf = isec->content().begin();353 const ulittle32_t *instBuf = reinterpret_cast<const ulittle32_t *>(buf + off);354 uint32_t instr1 = *instBuf++;355 uint32_t instr2 = *instBuf++;356 uint32_t instr3 = *instBuf++;357 if (is843419ErratumSequence(instr1, instr2, instr3)) {358 patchOff = off + 8;359 } else if (optionalAllowed && !isBranch(instr3)) {360 uint32_t instr4 = *instBuf++;361 if (is843419ErratumSequence(instr1, instr2, instr4))362 patchOff = off + 12;363 }364 if (((isecAddr + off) & 0xfff) == 0xff8)365 off += 4;366 else367 off += 0xffc;368 return patchOff;369}370 371class elf::Patch843419Section final : public SyntheticSection {372public:373 Patch843419Section(Ctx &, InputSection *p, uint64_t off);374 375 void writeTo(uint8_t *buf) override;376 377 size_t getSize() const override { return 8; }378 379 uint64_t getLDSTAddr() const;380 381 static bool classof(const SectionBase *d) {382 return d->kind() == InputSectionBase::Synthetic && d->name == ".text.patch";383 }384 385 // The Section we are patching.386 const InputSection *patchee;387 // The offset of the instruction in the patchee section we are patching.388 uint64_t patcheeOffset;389 // A label for the start of the Patch that we can use as a relocation target.390 Symbol *patchSym;391};392 393Patch843419Section::Patch843419Section(Ctx &ctx, InputSection *p, uint64_t off)394 : SyntheticSection(ctx, ".text.patch", SHT_PROGBITS,395 SHF_ALLOC | SHF_EXECINSTR, 4),396 patchee(p), patcheeOffset(off) {397 this->parent = p->getParent();398 patchSym = addSyntheticLocal(399 ctx, ctx.saver.save("__CortexA53843419_" + utohexstr(getLDSTAddr())),400 STT_FUNC, 0, getSize(), *this);401 addSyntheticLocal(ctx, ctx.saver.save("$x"), STT_NOTYPE, 0, 0, *this);402}403 404uint64_t Patch843419Section::getLDSTAddr() const {405 return patchee->getVA(patcheeOffset);406}407 408void Patch843419Section::writeTo(uint8_t *buf) {409 // Copy the instruction that we will be replacing with a branch in the410 // patchee Section.411 write32le(buf, read32le(patchee->content().begin() + patcheeOffset));412 413 // Apply any relocation transferred from the original patchee section.414 ctx.target->relocateAlloc(*this, buf);415 416 // Return address is the next instruction after the one we have just copied.417 uint64_t s = getLDSTAddr() + 4;418 uint64_t p = patchSym->getVA(ctx) + 4;419 ctx.target->relocateNoSym(buf + 4, R_AARCH64_JUMP26, s - p);420}421 422void AArch64Err843419Patcher::init() {423 // The AArch64 ABI permits data in executable sections. We must avoid scanning424 // this data as if it were instructions to avoid false matches. We use the425 // mapping symbols in the InputObjects to identify this data, caching the426 // results in sectionMap so we don't have to recalculate it each pass.427 428 // The ABI Section 4.5.4 Mapping symbols; defines local symbols that describe429 // half open intervals [Symbol Value, Next Symbol Value) of code and data430 // within sections. If there is no next symbol then the half open interval is431 // [Symbol Value, End of section). The type, code or data, is determined by432 // the mapping symbol name, $x for code, $d for data.433 auto isCodeMapSymbol = [](const Symbol *b) {434 return b->getName() == "$x" || b->getName().starts_with("$x.");435 };436 auto isDataMapSymbol = [](const Symbol *b) {437 return b->getName() == "$d" || b->getName().starts_with("$d.");438 };439 440 // Collect mapping symbols for every executable InputSection.441 for (ELFFileBase *file : ctx.objectFiles) {442 for (Symbol *b : file->getLocalSymbols()) {443 auto *def = dyn_cast<Defined>(b);444 if (!def)445 continue;446 if (!isCodeMapSymbol(def) && !isDataMapSymbol(def))447 continue;448 if (auto *sec = dyn_cast_or_null<InputSection>(def->section))449 if (sec->flags & SHF_EXECINSTR)450 sectionMap[sec].push_back(def);451 }452 }453 // For each InputSection make sure the mapping symbols are in sorted in454 // ascending order and free from consecutive runs of mapping symbols with455 // the same type. For example we must remove the redundant $d.1 from $x.0456 // $d.0 $d.1 $x.1.457 for (auto &kv : sectionMap) {458 std::vector<const Defined *> &mapSyms = kv.second;459 llvm::stable_sort(mapSyms, [](const Defined *a, const Defined *b) {460 return a->value < b->value;461 });462 mapSyms.erase(llvm::unique(mapSyms,463 [=](const Defined *a, const Defined *b) {464 return isCodeMapSymbol(a) ==465 isCodeMapSymbol(b);466 }),467 mapSyms.end());468 // Always start with a Code Mapping Symbol.469 if (!mapSyms.empty() && !isCodeMapSymbol(mapSyms.front()))470 mapSyms.erase(mapSyms.begin());471 }472 initialized = true;473}474 475// Insert the PatchSections we have created back into the476// InputSectionDescription. As inserting patches alters the addresses of477// InputSections that follow them, we try and place the patches after all the478// executable sections, although we may need to insert them earlier if the479// InputSectionDescription is larger than the maximum branch range.480void AArch64Err843419Patcher::insertPatches(481 InputSectionDescription &isd, std::vector<Patch843419Section *> &patches) {482 uint64_t isecLimit;483 uint64_t prevIsecLimit = isd.sections.front()->outSecOff;484 uint64_t patchUpperBound =485 prevIsecLimit + ctx.target->getThunkSectionSpacing();486 uint64_t outSecAddr = isd.sections.front()->getParent()->addr;487 488 // Set the outSecOff of patches to the place where we want to insert them.489 // We use a similar strategy to Thunk placement. Place patches roughly490 // every multiple of maximum branch range.491 auto patchIt = patches.begin();492 auto patchEnd = patches.end();493 for (const InputSection *isec : isd.sections) {494 isecLimit = isec->outSecOff + isec->getSize();495 if (isecLimit > patchUpperBound) {496 while (patchIt != patchEnd) {497 if ((*patchIt)->getLDSTAddr() - outSecAddr >= prevIsecLimit)498 break;499 (*patchIt)->outSecOff = prevIsecLimit;500 ++patchIt;501 }502 patchUpperBound = prevIsecLimit + ctx.target->getThunkSectionSpacing();503 }504 prevIsecLimit = isecLimit;505 }506 for (; patchIt != patchEnd; ++patchIt) {507 (*patchIt)->outSecOff = isecLimit;508 }509 510 // Merge all patch sections. We use the outSecOff assigned above to511 // determine the insertion point. This is ok as we only merge into an512 // InputSectionDescription once per pass, and at the end of the pass513 // assignAddresses() will recalculate all the outSecOff values.514 SmallVector<InputSection *, 0> tmp;515 tmp.reserve(isd.sections.size() + patches.size());516 auto mergeCmp = [](const InputSection *a, const InputSection *b) {517 if (a->outSecOff != b->outSecOff)518 return a->outSecOff < b->outSecOff;519 return isa<Patch843419Section>(a) && !isa<Patch843419Section>(b);520 };521 std::merge(isd.sections.begin(), isd.sections.end(), patches.begin(),522 patches.end(), std::back_inserter(tmp), mergeCmp);523 isd.sections = std::move(tmp);524}525 526// Given an erratum sequence that starts at address adrpAddr, with an527// instruction that we need to patch at patcheeOffset from the start of528// InputSection isec, create a Patch843419 Section and add it to the529// Patches that we need to insert.530static void implementPatch(Ctx &ctx, uint64_t adrpAddr, uint64_t patcheeOffset,531 InputSection *isec,532 std::vector<Patch843419Section *> &patches) {533 // There may be a relocation at the same offset that we are patching. There534 // are four cases that we need to consider.535 // Case 1: R_AARCH64_JUMP26 branch relocation. We have already patched this536 // instance of the erratum on a previous patch and altered the relocation. We537 // have nothing more to do.538 // Case 2: A TLS Relaxation R_RELAX_TLS_IE_TO_LE. In this case the ADRP that539 // we read will be transformed into a MOVZ later so we actually don't match540 // the sequence and have nothing more to do.541 // Case 3: A load/store register (unsigned immediate) class relocation. There542 // are two of these R_AARCH_LD64_ABS_LO12_NC and R_AARCH_LD64_GOT_LO12_NC and543 // they are both absolute. We need to add the same relocation to the patch,544 // and replace the relocation with a R_AARCH_JUMP26 branch relocation.545 // Case 4: No relocation. We must create a new R_AARCH64_JUMP26 branch546 // relocation at the offset.547 auto relIt = llvm::find_if(isec->relocs(), [=](const Relocation &r) {548 return r.offset == patcheeOffset;549 });550 if (relIt != isec->relocs().end() &&551 (relIt->type == R_AARCH64_JUMP26 || relIt->expr == R_RELAX_TLS_IE_TO_LE))552 return;553 554 Log(ctx) << "detected cortex-a53-843419 erratum sequence starting at " <<555 utohexstr(adrpAddr) << " in unpatched output.";556 557 auto *ps = make<Patch843419Section>(ctx, isec, patcheeOffset);558 patches.push_back(ps);559 560 auto makeRelToPatch = [](uint64_t offset, Symbol *patchSym) {561 return Relocation{R_PC, R_AARCH64_JUMP26, offset, 0, patchSym};562 };563 564 if (relIt != isec->relocs().end()) {565 ps->addReloc({relIt->expr, relIt->type, 0, relIt->addend, relIt->sym});566 *relIt = makeRelToPatch(patcheeOffset, ps->patchSym);567 } else568 isec->addReloc(makeRelToPatch(patcheeOffset, ps->patchSym));569}570 571// Scan all the instructions in InputSectionDescription, for each instance of572// the erratum sequence create a Patch843419Section. We return the list of573// Patch843419Sections that need to be applied to the InputSectionDescription.574std::vector<Patch843419Section *>575AArch64Err843419Patcher::patchInputSectionDescription(576 InputSectionDescription &isd) {577 std::vector<Patch843419Section *> patches;578 for (InputSection *isec : isd.sections) {579 // LLD doesn't use the erratum sequence in SyntheticSections.580 if (isa<SyntheticSection>(isec))581 continue;582 // Use sectionMap to make sure we only scan code and not inline data.583 // We have already sorted MapSyms in ascending order and removed consecutive584 // mapping symbols of the same type. Our range of executable instructions to585 // scan is therefore [codeSym->value, dataSym->value) or [codeSym->value,586 // section size).587 std::vector<const Defined *> &mapSyms = sectionMap[isec];588 589 auto codeSym = mapSyms.begin();590 while (codeSym != mapSyms.end()) {591 auto dataSym = std::next(codeSym);592 uint64_t off = (*codeSym)->value;593 uint64_t limit = (dataSym == mapSyms.end()) ? isec->content().size()594 : (*dataSym)->value;595 596 while (off < limit) {597 uint64_t startAddr = isec->getVA(off);598 if (uint64_t patcheeOffset =599 scanCortexA53Errata843419(isec, off, limit))600 implementPatch(ctx, startAddr, patcheeOffset, isec, patches);601 }602 if (dataSym == mapSyms.end())603 break;604 codeSym = std::next(dataSym);605 }606 }607 return patches;608}609 610// For each InputSectionDescription make one pass over the executable sections611// looking for the erratum sequence; creating a synthetic Patch843419Section612// for each instance found. We insert these synthetic patch sections after the613// executable code in each InputSectionDescription.614//615// PreConditions:616// The Output and Input Sections have had their final addresses assigned.617//618// PostConditions:619// Returns true if at least one patch was added. The addresses of the620// Output and Input Sections may have been changed.621// Returns false if no patches were required and no changes were made.622bool AArch64Err843419Patcher::createFixes() {623 if (!initialized)624 init();625 626 bool addressesChanged = false;627 for (OutputSection *os : ctx.outputSections) {628 if (!(os->flags & SHF_ALLOC) || !(os->flags & SHF_EXECINSTR))629 continue;630 for (SectionCommand *cmd : os->commands)631 if (auto *isd = dyn_cast<InputSectionDescription>(cmd)) {632 std::vector<Patch843419Section *> patches =633 patchInputSectionDescription(*isd);634 if (!patches.empty()) {635 insertPatches(*isd, patches);636 addressesChanged = true;637 }638 }639 }640 return addressesChanged;641}642