543 lines · cpp
1//===- ARMErrataFix.cpp ---------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8// This file implements Section Patching for the purpose of working around the9// Cortex-a8 erratum 657417 "A 32bit branch instruction that spans 2 4K regions10// can result in an incorrect instruction fetch or processor deadlock." The11// erratum affects all but r1p7, r2p5, r2p6, r3p1 and r3p2 revisions of the12// Cortex-A8. A high level description of the patching technique is given in13// the opening comment of AArch64ErrataFix.cpp.14//===----------------------------------------------------------------------===//15 16#include "ARMErrataFix.h"17#include "InputFiles.h"18#include "LinkerScript.h"19#include "OutputSections.h"20#include "Relocations.h"21#include "Symbols.h"22#include "SyntheticSections.h"23#include "Target.h"24#include "llvm/Support/Endian.h"25#include <algorithm>26 27using namespace llvm;28using namespace llvm::ELF;29using namespace llvm::object;30using namespace llvm::support;31using namespace llvm::support::endian;32using namespace lld;33using namespace lld::elf;34 35// The documented title for Erratum 657417 is:36// "A 32bit branch instruction that spans two 4K regions can result in an37// incorrect instruction fetch or processor deadlock". Graphically using a38// 32-bit B.w instruction encoded as a pair of halfwords 0xf7fe 0xbfff39// xxxxxx000 // Memory region 1 start40// target:41// ...42// xxxxxxffe f7fe // First halfword of branch to target:43// xxxxxx000 // Memory region 2 start44// xxxxxx002 bfff // Second halfword of branch to target:45//46// The specific trigger conditions that can be detected at link time are:47// - There is a 32-bit Thumb-2 branch instruction with an address of the form48// xxxxxxFFE. The first 2 bytes of the instruction are in 4KiB region 1, the49// second 2 bytes are in region 2.50// - The branch instruction is one of BLX, BL, B.w BCC.w51// - The instruction preceding the branch is a 32-bit non-branch instruction.52// - The target of the branch is in region 1.53//54// The linker mitigation for the fix is to redirect any branch that meets the55// erratum conditions to a patch section containing a branch to the target.56//57// As adding patch sections may move branches onto region boundaries the patch58// must iterate until no more patches are added.59//60// Example, before:61// 00000FFA func: NOP.w // 32-bit Thumb function62// 00000FFE B.W func // 32-bit branch spanning 2 regions, dest in 1st.63// Example, after:64// 00000FFA func: NOP.w // 32-bit Thumb function65// 00000FFE B.w __CortexA8657417_00000FFE66// 00001002 2 - bytes padding67// 00001004 __CortexA8657417_00000FFE: B.w func68 69class elf::Patch657417Section final : public SyntheticSection {70public:71 Patch657417Section(Ctx &, InputSection *p, uint64_t off, uint32_t instr,72 bool isARM);73 74 void writeTo(uint8_t *buf) override;75 76 size_t getSize() const override { return 4; }77 78 // Get the virtual address of the branch instruction at patcheeOffset.79 uint64_t getBranchAddr() const;80 81 static bool classof(const SectionBase *d) {82 return d->kind() == InputSectionBase::Synthetic && d->name ==".text.patch";83 }84 85 // The Section we are patching.86 const InputSection *patchee;87 // The offset of the instruction in the Patchee section we are patching.88 uint64_t patcheeOffset;89 // A label for the start of the Patch that we can use as a relocation target.90 Symbol *patchSym;91 // A decoding of the branch instruction at patcheeOffset.92 uint32_t instr;93 // True If the patch is to be written in ARM state, otherwise the patch will94 // be written in Thumb state.95 bool isARM;96};97 98// Return true if the half-word, when taken as the first of a pair of halfwords99// is the first half of a 32-bit instruction.100// Reference from ARM Architecture Reference Manual ARMv7-A and ARMv7-R edition101// section A6.3: 32-bit Thumb instruction encoding102// | HW1 | HW2 |103// | 1 1 1 | op1 (2) | op2 (7) | x (4) |op| x (15) |104// With op1 == 0b00, a 16-bit instruction is encoded.105//106// We test only the first halfword, looking for op != 0b00.107static bool is32bitInstruction(uint16_t hw) {108 return (hw & 0xe000) == 0xe000 && (hw & 0x1800) != 0x0000;109}110 111// Reference from ARM Architecture Reference Manual ARMv7-A and ARMv7-R edition112// section A6.3.4 Branches and miscellaneous control.113// | HW1 | HW2 |114// | 1 1 1 | 1 0 | op (7) | x (4) | 1 | op1 (3) | op2 (4) | imm8 (8) |115// op1 == 0x0 op != x111xxx | Conditional branch (Bcc.W)116// op1 == 0x1 | Branch (B.W)117// op1 == 1x0 | Branch with Link and Exchange (BLX.w)118// op1 == 1x1 | Branch with Link (BL.W)119 120static bool isBcc(uint32_t instr) {121 return (instr & 0xf800d000) == 0xf0008000 &&122 (instr & 0x03800000) != 0x03800000;123}124 125static bool isB(uint32_t instr) { return (instr & 0xf800d000) == 0xf0009000; }126 127static bool isBLX(uint32_t instr) { return (instr & 0xf800d000) == 0xf000c000; }128 129static bool isBL(uint32_t instr) { return (instr & 0xf800d000) == 0xf000d000; }130 131static bool is32bitBranch(uint32_t instr) {132 return isBcc(instr) || isB(instr) || isBL(instr) || isBLX(instr);133}134 135Patch657417Section::Patch657417Section(Ctx &ctx, InputSection *p, uint64_t off,136 uint32_t instr, bool isARM)137 : SyntheticSection(ctx, ".text.patch", SHT_PROGBITS,138 SHF_ALLOC | SHF_EXECINSTR, 4),139 patchee(p), patcheeOffset(off), instr(instr), isARM(isARM) {140 parent = p->getParent();141 patchSym = addSyntheticLocal(142 ctx, ctx.saver.save("__CortexA8657417_" + utohexstr(getBranchAddr())),143 STT_FUNC, isARM ? 0 : 1, getSize(), *this);144 addSyntheticLocal(ctx, ctx.saver.save(isARM ? "$a" : "$t"), STT_NOTYPE, 0, 0,145 *this);146}147 148uint64_t Patch657417Section::getBranchAddr() const {149 return patchee->getVA(patcheeOffset);150}151 152// Given a branch instruction instr at sourceAddr work out its destination153// address. This is only used when the branch instruction has no relocation.154static uint64_t getThumbDestAddr(Ctx &ctx, uint64_t sourceAddr,155 uint32_t instr) {156 uint8_t buf[4];157 write16le(buf, instr >> 16);158 write16le(buf + 2, instr & 0x0000ffff);159 int64_t offset;160 if (isBcc(instr))161 offset = ctx.target->getImplicitAddend(buf, R_ARM_THM_JUMP19);162 else if (isB(instr))163 offset = ctx.target->getImplicitAddend(buf, R_ARM_THM_JUMP24);164 else165 offset = ctx.target->getImplicitAddend(buf, R_ARM_THM_CALL);166 // A BLX instruction from Thumb to Arm may have an address that is167 // not 4-byte aligned. As Arm instructions are always 4-byte aligned168 // the instruction is calculated (from Arm ARM):169 // targetAddress = Align(PC, 4) + imm32170 // where171 // Align(x, y) = y * (x Div y)172 // which corresponds to alignDown.173 if (isBLX(instr))174 sourceAddr = alignDown(sourceAddr, 4);175 return sourceAddr + offset + 4;176}177 178void Patch657417Section::writeTo(uint8_t *buf) {179 // The base instruction of the patch is always a 32-bit unconditional branch.180 if (isARM)181 write32le(buf, 0xea000000);182 else183 write32le(buf, 0x9000f000);184 // If we have a relocation then apply it.185 if (!relocs().empty()) {186 ctx.target->relocateAlloc(*this, buf);187 return;188 }189 190 // If we don't have a relocation then we must calculate and write the offset191 // ourselves.192 // Get the destination offset from the addend in the branch instruction.193 // We cannot use the instruction in the patchee section as this will have194 // been altered to point to us!195 uint64_t s = getThumbDestAddr(ctx, getBranchAddr(), instr);196 // A BLX changes the state of the branch in the patch to Arm state, which197 // has a PC Bias of 8, whereas in all other cases the branch is in Thumb198 // state with a PC Bias of 4.199 uint64_t pcBias = isBLX(instr) ? 8 : 4;200 uint64_t p = getVA(pcBias);201 ctx.target->relocateNoSym(buf, isARM ? R_ARM_JUMP24 : R_ARM_THM_JUMP24,202 s - p);203}204 205// Given a branch instruction spanning two 4KiB regions, at offset off from the206// start of isec, return true if the destination of the branch is within the207// first of the two 4Kib regions.208static bool branchDestInFirstRegion(Ctx &ctx, const InputSection *isec,209 uint64_t off, uint32_t instr,210 const Relocation *r) {211 uint64_t sourceAddr = isec->getVA(0) + off;212 assert((sourceAddr & 0xfff) == 0xffe);213 uint64_t destAddr;214 // If there is a branch relocation at the same offset we must use this to215 // find the destination address as the branch could be indirected via a thunk216 // or the PLT.217 if (r) {218 uint64_t dst =219 r->expr == R_PLT_PC ? r->sym->getPltVA(ctx) : r->sym->getVA(ctx);220 // Account for Thumb PC bias, usually cancelled to 0 by addend of -4.221 destAddr = dst + r->addend + 4;222 } else {223 // If there is no relocation, we must have an intra-section branch224 // We must extract the offset from the addend manually.225 destAddr = getThumbDestAddr(ctx, sourceAddr, instr);226 }227 228 return (destAddr & 0xfffff000) == (sourceAddr & 0xfffff000);229}230 231// Return true if a branch can reach a patch section placed after isec.232// The Bcc.w instruction has a range of 1 MiB, all others have 16 MiB.233static bool patchInRange(Ctx &ctx, const InputSection *isec, uint64_t off,234 uint32_t instr) {235 236 // We need the branch at source to reach a patch section placed immediately237 // after isec. As there can be more than one patch in the patch section we238 // add 0x100 as contingency to account for worst case of 1 branch every 4KiB239 // for a 1 MiB range.240 return ctx.target->inBranchRange(241 isBcc(instr) ? R_ARM_THM_JUMP19 : R_ARM_THM_JUMP24, isec->getVA(off),242 isec->getVA() + isec->getSize() + 0x100);243}244 245struct ScanResult {246 // Offset of branch within its InputSection.247 uint64_t off;248 // Cached decoding of the branch instruction.249 uint32_t instr;250 // Branch relocation at off. Will be nullptr if no relocation exists.251 Relocation *rel;252};253 254// Detect the erratum sequence, returning the offset of the branch instruction255// and a decoding of the branch. If the erratum sequence is not found then256// return an offset of 0 for the branch. 0 is a safe value to use for no patch257// as there must be at least one 32-bit non-branch instruction before the258// branch so the minimum offset for a patch is 4.259static ScanResult scanCortexA8Errata657417(InputSection *isec, uint64_t &off,260 uint64_t limit) {261 Ctx &ctx = isec->getCtx();262 uint64_t isecAddr = isec->getVA(0);263 // Advance Off so that (isecAddr + off) modulo 0x1000 is at least 0xffa. We264 // need to check for a 32-bit instruction immediately before a 32-bit branch265 // at 0xffe modulo 0x1000.266 off = alignTo(isecAddr + off, 0x1000, 0xffa) - isecAddr;267 if (off >= limit || limit - off < 8) {268 // Need at least 2 4-byte sized instructions to trigger erratum.269 off = limit;270 return {0, 0, nullptr};271 }272 273 ScanResult scanRes = {0, 0, nullptr};274 const uint8_t *buf = isec->content().begin();275 // ARMv7-A Thumb 32-bit instructions are encoded 2 consecutive276 // little-endian halfwords.277 const ulittle16_t *instBuf = reinterpret_cast<const ulittle16_t *>(buf + off);278 uint16_t hw11 = *instBuf++;279 uint16_t hw12 = *instBuf++;280 uint16_t hw21 = *instBuf++;281 uint16_t hw22 = *instBuf++;282 if (is32bitInstruction(hw11) && is32bitInstruction(hw21)) {283 uint32_t instr1 = (hw11 << 16) | hw12;284 uint32_t instr2 = (hw21 << 16) | hw22;285 if (!is32bitBranch(instr1) && is32bitBranch(instr2)) {286 // Find a relocation for the branch if it exists. This will be used287 // to determine the target.288 uint64_t branchOff = off + 4;289 auto relIt = llvm::find_if(isec->relocs(), [=](const Relocation &r) {290 return r.offset == branchOff &&291 (r.type == R_ARM_THM_JUMP19 || r.type == R_ARM_THM_JUMP24 ||292 r.type == R_ARM_THM_CALL);293 });294 if (relIt != isec->relocs().end())295 scanRes.rel = &(*relIt);296 if (branchDestInFirstRegion(ctx, isec, branchOff, instr2, scanRes.rel)) {297 if (patchInRange(ctx, isec, branchOff, instr2)) {298 scanRes.off = branchOff;299 scanRes.instr = instr2;300 } else {301 Warn(ctx) << isec->file302 << ": skipping cortex-a8 657417 erratum sequence, section "303 << isec->name << " is too large to patch";304 }305 }306 }307 }308 off += 0x1000;309 return scanRes;310}311 312void ARMErr657417Patcher::init() {313 // The Arm ABI permits a mix of ARM, Thumb and Data in the same314 // InputSection. We must only scan Thumb instructions to avoid false315 // matches. We use the mapping symbols in the InputObjects to identify this316 // data, caching the results in sectionMap so we don't have to recalculate317 // it each pass.318 319 // The ABI Section 4.5.5 Mapping symbols; defines local symbols that describe320 // half open intervals [Symbol Value, Next Symbol Value) of code and data321 // within sections. If there is no next symbol then the half open interval is322 // [Symbol Value, End of section). The type, code or data, is determined by323 // the mapping symbol name, $a for Arm code, $t for Thumb code, $d for data.324 auto isArmMapSymbol = [](const Symbol *s) {325 return s->getName() == "$a" || s->getName().starts_with("$a.");326 };327 auto isThumbMapSymbol = [](const Symbol *s) {328 return s->getName() == "$t" || s->getName().starts_with("$t.");329 };330 auto isDataMapSymbol = [](const Symbol *s) {331 return s->getName() == "$d" || s->getName().starts_with("$d.");332 };333 334 // Collect mapping symbols for every executable InputSection.335 for (ELFFileBase *file : ctx.objectFiles) {336 for (Symbol *s : file->getLocalSymbols()) {337 auto *def = dyn_cast<Defined>(s);338 if (!def)339 continue;340 if (!isArmMapSymbol(def) && !isThumbMapSymbol(def) &&341 !isDataMapSymbol(def))342 continue;343 if (auto *sec = dyn_cast_or_null<InputSection>(def->section))344 if (sec->flags & SHF_EXECINSTR)345 sectionMap[sec].push_back(def);346 }347 }348 // For each InputSection make sure the mapping symbols are in sorted in349 // ascending order and are in alternating Thumb, non-Thumb order.350 for (auto &kv : sectionMap) {351 std::vector<const Defined *> &mapSyms = kv.second;352 llvm::stable_sort(mapSyms, [](const Defined *a, const Defined *b) {353 return a->value < b->value;354 });355 mapSyms.erase(llvm::unique(mapSyms,356 [=](const Defined *a, const Defined *b) {357 return (isThumbMapSymbol(a) ==358 isThumbMapSymbol(b));359 }),360 mapSyms.end());361 // Always start with a Thumb Mapping Symbol362 if (!mapSyms.empty() && !isThumbMapSymbol(mapSyms.front()))363 mapSyms.erase(mapSyms.begin());364 }365 initialized = true;366}367 368void ARMErr657417Patcher::insertPatches(369 InputSectionDescription &isd, std::vector<Patch657417Section *> &patches) {370 uint64_t spacing = 0x100000 - 0x7500;371 uint64_t isecLimit;372 uint64_t prevIsecLimit = isd.sections.front()->outSecOff;373 uint64_t patchUpperBound = prevIsecLimit + spacing;374 uint64_t outSecAddr = isd.sections.front()->getParent()->addr;375 376 // Set the outSecOff of patches to the place where we want to insert them.377 // We use a similar strategy to initial thunk placement, using 1 MiB as the378 // range of the Thumb-2 conditional branch with a contingency accounting for379 // thunk generation.380 auto patchIt = patches.begin();381 auto patchEnd = patches.end();382 for (const InputSection *isec : isd.sections) {383 isecLimit = isec->outSecOff + isec->getSize();384 if (isecLimit > patchUpperBound) {385 for (; patchIt != patchEnd; ++patchIt) {386 if ((*patchIt)->getBranchAddr() - outSecAddr >= prevIsecLimit)387 break;388 (*patchIt)->outSecOff = prevIsecLimit;389 }390 patchUpperBound = prevIsecLimit + spacing;391 }392 prevIsecLimit = isecLimit;393 }394 for (; patchIt != patchEnd; ++patchIt)395 (*patchIt)->outSecOff = isecLimit;396 397 // Merge all patch sections. We use the outSecOff assigned above to398 // determine the insertion point. This is ok as we only merge into an399 // InputSectionDescription once per pass, and at the end of the pass400 // assignAddresses() will recalculate all the outSecOff values.401 SmallVector<InputSection *, 0> tmp;402 tmp.reserve(isd.sections.size() + patches.size());403 auto mergeCmp = [](const InputSection *a, const InputSection *b) {404 if (a->outSecOff != b->outSecOff)405 return a->outSecOff < b->outSecOff;406 return isa<Patch657417Section>(a) && !isa<Patch657417Section>(b);407 };408 std::merge(isd.sections.begin(), isd.sections.end(), patches.begin(),409 patches.end(), std::back_inserter(tmp), mergeCmp);410 isd.sections = std::move(tmp);411}412 413// Given a branch instruction described by ScanRes redirect it to a patch414// section containing an unconditional branch instruction to the target.415// Ensure that this patch section is 4-byte aligned so that the branch cannot416// span two 4 KiB regions. Place the patch section so that it is always after417// isec so the branch we are patching always goes forwards.418static void implementPatch(ScanResult sr, InputSection *isec,419 std::vector<Patch657417Section *> &patches) {420 Ctx &ctx = isec->getCtx();421 Log(ctx) << "detected cortex-a8-657419 erratum sequence starting at " <<422 utohexstr(isec->getVA(sr.off)) << " in unpatched output";423 Patch657417Section *psec;424 // We have two cases to deal with.425 // Case 1. There is a relocation at patcheeOffset to a symbol. The426 // unconditional branch in the patch must have a relocation so that any427 // further redirection via the PLT or a Thunk happens as normal. At428 // patcheeOffset we redirect the existing relocation to a Symbol defined at429 // the start of the patch section.430 //431 // Case 2. There is no relocation at patcheeOffset. We are unlikely to have432 // a symbol that we can use as a target for a relocation in the patch section.433 // Luckily we know that the destination cannot be indirected via the PLT or434 // a Thunk so we can just write the destination directly.435 if (sr.rel) {436 // Case 1. We have an existing relocation to redirect to patch and a437 // Symbol target.438 439 // Create a branch relocation for the unconditional branch in the patch.440 // This can be redirected via the PLT or Thunks.441 RelType patchRelType = R_ARM_THM_JUMP24;442 int64_t patchRelAddend = sr.rel->addend;443 bool destIsARM = false;444 if (isBL(sr.instr) || isBLX(sr.instr)) {445 // The final target of the branch may be ARM or Thumb, if the target446 // is ARM then we write the patch in ARM state to avoid a state change447 // Thunk from the patch to the target.448 uint64_t dstSymAddr = (sr.rel->expr == R_PLT_PC)449 ? sr.rel->sym->getPltVA(ctx)450 : sr.rel->sym->getVA(ctx);451 destIsARM = (dstSymAddr & 1) == 0;452 }453 psec = make<Patch657417Section>(ctx, isec, sr.off, sr.instr, destIsARM);454 if (destIsARM) {455 // The patch will be in ARM state. Use an ARM relocation and account for456 // the larger ARM PC-bias of 8 rather than Thumb's 4.457 patchRelType = R_ARM_JUMP24;458 patchRelAddend -= 4;459 }460 psec->addReloc(461 Relocation{sr.rel->expr, patchRelType, 0, patchRelAddend, sr.rel->sym});462 // Redirect the existing branch relocation to the patch.463 sr.rel->expr = R_PC;464 sr.rel->addend = -4;465 sr.rel->sym = psec->patchSym;466 } else {467 // Case 2. We do not have a relocation to the patch. Add a relocation of the468 // appropriate type to the patch at patcheeOffset.469 470 // The destination is ARM if we have a BLX.471 psec =472 make<Patch657417Section>(ctx, isec, sr.off, sr.instr, isBLX(sr.instr));473 RelType type;474 if (isBcc(sr.instr))475 type = R_ARM_THM_JUMP19;476 else if (isB(sr.instr))477 type = R_ARM_THM_JUMP24;478 else479 type = R_ARM_THM_CALL;480 isec->addReloc(Relocation{R_PC, type, sr.off, -4, psec->patchSym});481 }482 patches.push_back(psec);483}484 485// Scan all the instructions in InputSectionDescription, for each instance of486// the erratum sequence create a Patch657417Section. We return the list of487// Patch657417Sections that need to be applied to the InputSectionDescription.488std::vector<Patch657417Section *>489ARMErr657417Patcher::patchInputSectionDescription(490 InputSectionDescription &isd) {491 std::vector<Patch657417Section *> patches;492 for (InputSection *isec : isd.sections) {493 // LLD doesn't use the erratum sequence in SyntheticSections.494 if (isa<SyntheticSection>(isec))495 continue;496 // Use sectionMap to make sure we only scan Thumb code and not Arm or inline497 // data. We have already sorted mapSyms in ascending order and removed498 // consecutive mapping symbols of the same type. Our range of executable499 // instructions to scan is therefore [thumbSym->value, nonThumbSym->value)500 // or [thumbSym->value, section size).501 std::vector<const Defined *> &mapSyms = sectionMap[isec];502 503 auto thumbSym = mapSyms.begin();504 while (thumbSym != mapSyms.end()) {505 auto nonThumbSym = std::next(thumbSym);506 uint64_t off = (*thumbSym)->value;507 uint64_t limit = nonThumbSym == mapSyms.end() ? isec->content().size()508 : (*nonThumbSym)->value;509 510 while (off < limit) {511 ScanResult sr = scanCortexA8Errata657417(isec, off, limit);512 if (sr.off)513 implementPatch(sr, isec, patches);514 }515 if (nonThumbSym == mapSyms.end())516 break;517 thumbSym = std::next(nonThumbSym);518 }519 }520 return patches;521}522 523bool ARMErr657417Patcher::createFixes() {524 if (!initialized)525 init();526 527 bool addressesChanged = false;528 for (OutputSection *os : ctx.outputSections) {529 if (!(os->flags & SHF_ALLOC) || !(os->flags & SHF_EXECINSTR))530 continue;531 for (SectionCommand *cmd : os->commands)532 if (auto *isd = dyn_cast<InputSectionDescription>(cmd)) {533 std::vector<Patch657417Section *> patches =534 patchInputSectionDescription(*isd);535 if (!patches.empty()) {536 insertPatches(*isd, patches);537 addressesChanged = true;538 }539 }540 }541 return addressesChanged;542}543