1329 lines · cpp
1//===- AArch64.cpp --------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "InputFiles.h"10#include "OutputSections.h"11#include "Symbols.h"12#include "SyntheticSections.h"13#include "Target.h"14#include "TargetImpl.h"15#include "llvm/BinaryFormat/ELF.h"16#include "llvm/Support/Endian.h"17 18using namespace llvm;19using namespace llvm::support::endian;20using namespace llvm::ELF;21using namespace lld;22using namespace lld::elf;23 24// Page(Expr) is the page address of the expression Expr, defined25// as (Expr & ~0xFFF). (This applies even if the machine page size26// supported by the platform has a different value.)27uint64_t elf::getAArch64Page(uint64_t expr) {28 return expr & ~static_cast<uint64_t>(0xFFF);29}30 31// A BTI landing pad is a valid target for an indirect branch when the Branch32// Target Identification has been enabled. As linker generated branches are33// via x16 the BTI landing pads are defined as: BTI C, BTI J, BTI JC, PACIASP,34// PACIBSP.35bool elf::isAArch64BTILandingPad(Ctx &ctx, Symbol &s, int64_t a) {36 // PLT entries accessed indirectly have a BTI c.37 if (s.isInPlt(ctx))38 return true;39 Defined *d = dyn_cast<Defined>(&s);40 if (!isa_and_nonnull<InputSection>(d->section))41 // All places that we cannot disassemble are responsible for making42 // the target a BTI landing pad.43 return true;44 InputSection *isec = cast<InputSection>(d->section);45 uint64_t off = d->value + a;46 // Likely user error, but protect ourselves against out of bounds47 // access.48 if (off >= isec->getSize())49 return true;50 const uint8_t *buf = isec->content().begin();51 const uint32_t instr = read32le(buf + off);52 // All BTI instructions are HINT instructions which all have same encoding53 // apart from bits [11:5]54 if ((instr & 0xd503201f) == 0xd503201f &&55 is_contained({/*PACIASP*/ 0xd503233f, /*PACIBSP*/ 0xd503237f,56 /*BTI C*/ 0xd503245f, /*BTI J*/ 0xd503249f,57 /*BTI JC*/ 0xd50324df},58 instr))59 return true;60 return false;61}62 63namespace {64class AArch64 : public TargetInfo {65public:66 AArch64(Ctx &);67 RelExpr getRelExpr(RelType type, const Symbol &s,68 const uint8_t *loc) const override;69 RelType getDynRel(RelType type) const override;70 int64_t getImplicitAddend(const uint8_t *buf, RelType type) const override;71 void writeGotPlt(uint8_t *buf, const Symbol &s) const override;72 void writeIgotPlt(uint8_t *buf, const Symbol &s) const override;73 void writePltHeader(uint8_t *buf) const override;74 void writePlt(uint8_t *buf, const Symbol &sym,75 uint64_t pltEntryAddr) const override;76 bool needsThunk(RelExpr expr, RelType type, const InputFile *file,77 uint64_t branchAddr, const Symbol &s,78 int64_t a) const override;79 uint32_t getThunkSectionSpacing() const override;80 bool inBranchRange(RelType type, uint64_t src, uint64_t dst) const override;81 bool usesOnlyLowPageBits(RelType type) const override;82 void relocate(uint8_t *loc, const Relocation &rel,83 uint64_t val) const override;84 RelExpr adjustTlsExpr(RelType type, RelExpr expr) const override;85 void relocateAlloc(InputSection &sec, uint8_t *buf) const override;86 void applyBranchToBranchOpt() const override;87 88private:89 void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const;90 void relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, uint64_t val) const;91 void relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const;92};93 94struct AArch64Relaxer {95 Ctx &ctx;96 bool safeToRelaxAdrpLdr = false;97 98 AArch64Relaxer(Ctx &ctx, ArrayRef<Relocation> relocs);99 bool tryRelaxAdrpAdd(const Relocation &adrpRel, const Relocation &addRel,100 uint64_t secAddr, uint8_t *buf) const;101 bool tryRelaxAdrpLdr(const Relocation &adrpRel, const Relocation &ldrRel,102 uint64_t secAddr, uint8_t *buf) const;103};104} // namespace105 106// Return the bits [Start, End] from Val shifted Start bits.107// For instance, getBits(0xF0, 4, 8) returns 0xF.108static uint64_t getBits(uint64_t val, int start, int end) {109 uint64_t mask = ((uint64_t)1 << (end + 1 - start)) - 1;110 return (val >> start) & mask;111}112 113AArch64::AArch64(Ctx &ctx) : TargetInfo(ctx) {114 copyRel = R_AARCH64_COPY;115 relativeRel = R_AARCH64_RELATIVE;116 iRelativeRel = R_AARCH64_IRELATIVE;117 iRelSymbolicRel = R_AARCH64_FUNCINIT64;118 gotRel = R_AARCH64_GLOB_DAT;119 pltRel = R_AARCH64_JUMP_SLOT;120 symbolicRel = R_AARCH64_ABS64;121 tlsDescRel = R_AARCH64_TLSDESC;122 tlsGotRel = R_AARCH64_TLS_TPREL64;123 pltHeaderSize = 32;124 pltEntrySize = 16;125 ipltEntrySize = 16;126 defaultMaxPageSize = 65536;127 128 // Align to the 2 MiB page size (known as a superpage or huge page).129 // FreeBSD automatically promotes 2 MiB-aligned allocations.130 defaultImageBase = 0x200000;131 132 needsThunks = true;133}134 135RelExpr AArch64::getRelExpr(RelType type, const Symbol &s,136 const uint8_t *loc) const {137 switch (type) {138 case R_AARCH64_ABS16:139 case R_AARCH64_ABS32:140 case R_AARCH64_ABS64:141 case R_AARCH64_FUNCINIT64:142 case R_AARCH64_ADD_ABS_LO12_NC:143 case R_AARCH64_LDST128_ABS_LO12_NC:144 case R_AARCH64_LDST16_ABS_LO12_NC:145 case R_AARCH64_LDST32_ABS_LO12_NC:146 case R_AARCH64_LDST64_ABS_LO12_NC:147 case R_AARCH64_LDST8_ABS_LO12_NC:148 case R_AARCH64_MOVW_SABS_G0:149 case R_AARCH64_MOVW_SABS_G1:150 case R_AARCH64_MOVW_SABS_G2:151 case R_AARCH64_MOVW_UABS_G0:152 case R_AARCH64_MOVW_UABS_G0_NC:153 case R_AARCH64_MOVW_UABS_G1:154 case R_AARCH64_MOVW_UABS_G1_NC:155 case R_AARCH64_MOVW_UABS_G2:156 case R_AARCH64_MOVW_UABS_G2_NC:157 case R_AARCH64_MOVW_UABS_G3:158 return R_ABS;159 case R_AARCH64_PATCHINST:160 if (!isAbsolute(s))161 Err(ctx) << getErrorLoc(ctx, loc)162 << "R_AARCH64_PATCHINST relocation against non-absolute symbol "163 << &s;164 return R_ABS;165 case R_AARCH64_AUTH_ABS64:166 return RE_AARCH64_AUTH;167 case R_AARCH64_TLSDESC_ADR_PAGE21:168 return RE_AARCH64_TLSDESC_PAGE;169 case R_AARCH64_AUTH_TLSDESC_ADR_PAGE21:170 return RE_AARCH64_AUTH_TLSDESC_PAGE;171 case R_AARCH64_TLSDESC_LD64_LO12:172 case R_AARCH64_TLSDESC_ADD_LO12:173 return R_TLSDESC;174 case R_AARCH64_AUTH_TLSDESC_LD64_LO12:175 case R_AARCH64_AUTH_TLSDESC_ADD_LO12:176 return RE_AARCH64_AUTH_TLSDESC;177 case R_AARCH64_TLSDESC_CALL:178 return R_TLSDESC_CALL;179 case R_AARCH64_TLSLE_ADD_TPREL_HI12:180 case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:181 case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC:182 case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC:183 case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC:184 case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC:185 case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC:186 case R_AARCH64_TLSLE_MOVW_TPREL_G0:187 case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC:188 case R_AARCH64_TLSLE_MOVW_TPREL_G1:189 case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC:190 case R_AARCH64_TLSLE_MOVW_TPREL_G2:191 return R_TPREL;192 case R_AARCH64_CALL26:193 case R_AARCH64_CONDBR19:194 case R_AARCH64_JUMP26:195 case R_AARCH64_TSTBR14:196 return R_PLT_PC;197 case R_AARCH64_PLT32:198 const_cast<Symbol &>(s).thunkAccessed = true;199 return R_PLT_PC;200 case R_AARCH64_PREL16:201 case R_AARCH64_PREL32:202 case R_AARCH64_PREL64:203 case R_AARCH64_ADR_PREL_LO21:204 case R_AARCH64_LD_PREL_LO19:205 case R_AARCH64_MOVW_PREL_G0:206 case R_AARCH64_MOVW_PREL_G0_NC:207 case R_AARCH64_MOVW_PREL_G1:208 case R_AARCH64_MOVW_PREL_G1_NC:209 case R_AARCH64_MOVW_PREL_G2:210 case R_AARCH64_MOVW_PREL_G2_NC:211 case R_AARCH64_MOVW_PREL_G3:212 return R_PC;213 case R_AARCH64_ADR_PREL_PG_HI21:214 case R_AARCH64_ADR_PREL_PG_HI21_NC:215 return RE_AARCH64_PAGE_PC;216 case R_AARCH64_LD64_GOT_LO12_NC:217 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:218 return R_GOT;219 case R_AARCH64_AUTH_LD64_GOT_LO12_NC:220 case R_AARCH64_AUTH_GOT_ADD_LO12_NC:221 return RE_AARCH64_AUTH_GOT;222 case R_AARCH64_AUTH_GOT_LD_PREL19:223 case R_AARCH64_AUTH_GOT_ADR_PREL_LO21:224 return RE_AARCH64_AUTH_GOT_PC;225 case R_AARCH64_LD64_GOTPAGE_LO15:226 return RE_AARCH64_GOT_PAGE;227 case R_AARCH64_ADR_GOT_PAGE:228 case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21:229 return RE_AARCH64_GOT_PAGE_PC;230 case R_AARCH64_AUTH_ADR_GOT_PAGE:231 return RE_AARCH64_AUTH_GOT_PAGE_PC;232 case R_AARCH64_GOTPCREL32:233 case R_AARCH64_GOT_LD_PREL19:234 return R_GOT_PC;235 case R_AARCH64_NONE:236 return R_NONE;237 default:238 Err(ctx) << getErrorLoc(ctx, loc) << "unknown relocation (" << type.v239 << ") against symbol " << &s;240 return R_NONE;241 }242}243 244RelExpr AArch64::adjustTlsExpr(RelType type, RelExpr expr) const {245 if (expr == R_RELAX_TLS_GD_TO_IE) {246 if (type == R_AARCH64_TLSDESC_ADR_PAGE21)247 return RE_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC;248 return R_RELAX_TLS_GD_TO_IE_ABS;249 }250 return expr;251}252 253bool AArch64::usesOnlyLowPageBits(RelType type) const {254 switch (type) {255 default:256 return false;257 case R_AARCH64_ADD_ABS_LO12_NC:258 case R_AARCH64_LD64_GOT_LO12_NC:259 case R_AARCH64_LDST128_ABS_LO12_NC:260 case R_AARCH64_LDST16_ABS_LO12_NC:261 case R_AARCH64_LDST32_ABS_LO12_NC:262 case R_AARCH64_LDST64_ABS_LO12_NC:263 case R_AARCH64_LDST8_ABS_LO12_NC:264 case R_AARCH64_TLSDESC_ADD_LO12:265 case R_AARCH64_TLSDESC_LD64_LO12:266 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:267 return true;268 }269}270 271RelType AArch64::getDynRel(RelType type) const {272 if (type == R_AARCH64_ABS64 || type == R_AARCH64_AUTH_ABS64 ||273 type == R_AARCH64_FUNCINIT64)274 return type;275 return R_AARCH64_NONE;276}277 278int64_t AArch64::getImplicitAddend(const uint8_t *buf, RelType type) const {279 switch (type) {280 case R_AARCH64_TLSDESC:281 return read64(ctx, buf + 8);282 case R_AARCH64_NONE:283 case R_AARCH64_GLOB_DAT:284 case R_AARCH64_AUTH_GLOB_DAT:285 case R_AARCH64_JUMP_SLOT:286 return 0;287 case R_AARCH64_ABS16:288 case R_AARCH64_PREL16:289 return SignExtend64<16>(read16(ctx, buf));290 case R_AARCH64_ABS32:291 case R_AARCH64_PREL32:292 return SignExtend64<32>(read32(ctx, buf));293 case R_AARCH64_ABS64:294 case R_AARCH64_PREL64:295 case R_AARCH64_RELATIVE:296 case R_AARCH64_IRELATIVE:297 case R_AARCH64_TLS_TPREL64:298 return read64(ctx, buf);299 300 // The following relocation types all point at instructions, and301 // relocate an immediate field in the instruction.302 //303 // The general rule, from AAELF64 §5.7.2 "Addends and PC-bias",304 // says: "If the relocation relocates an instruction the immediate305 // field of the instruction is extracted, scaled as required by306 // the instruction field encoding, and sign-extended to 64 bits".307 308 // The R_AARCH64_MOVW family operates on wide MOV/MOVK/MOVZ309 // instructions, which have a 16-bit immediate field with its low310 // bit in bit 5 of the instruction encoding. When the immediate311 // field is used as an implicit addend for REL-type relocations,312 // it is treated as added to the low bits of the output value, not313 // shifted depending on the relocation type.314 //315 // This allows REL relocations to express the requirement 'please316 // add 12345 to this symbol value and give me the four 16-bit317 // chunks of the result', by putting the same addend 12345 in all318 // four instructions. Carries between the 16-bit chunks are319 // handled correctly, because the whole 64-bit addition is done320 // once per relocation.321 case R_AARCH64_MOVW_UABS_G0:322 case R_AARCH64_MOVW_UABS_G0_NC:323 case R_AARCH64_MOVW_UABS_G1:324 case R_AARCH64_MOVW_UABS_G1_NC:325 case R_AARCH64_MOVW_UABS_G2:326 case R_AARCH64_MOVW_UABS_G2_NC:327 case R_AARCH64_MOVW_UABS_G3:328 return SignExtend64<16>(getBits(read32le(buf), 5, 20));329 330 // R_AARCH64_TSTBR14 points at a TBZ or TBNZ instruction, which331 // has a 14-bit offset measured in instructions, i.e. shifted left332 // by 2.333 case R_AARCH64_TSTBR14:334 return SignExtend64<16>(getBits(read32le(buf), 5, 18) << 2);335 336 // R_AARCH64_CONDBR19 operates on the ordinary B.cond instruction,337 // which has a 19-bit offset measured in instructions.338 //339 // R_AARCH64_LD_PREL_LO19 operates on the LDR (literal)340 // instruction, which also has a 19-bit offset, measured in 4-byte341 // chunks. So the calculation is the same as for342 // R_AARCH64_CONDBR19.343 case R_AARCH64_CONDBR19:344 case R_AARCH64_LD_PREL_LO19:345 return SignExtend64<21>(getBits(read32le(buf), 5, 23) << 2);346 347 // R_AARCH64_ADD_ABS_LO12_NC operates on ADD (immediate). The348 // immediate can optionally be shifted left by 12 bits, but this349 // relocation is intended for the case where it is not.350 case R_AARCH64_ADD_ABS_LO12_NC:351 return SignExtend64<12>(getBits(read32le(buf), 10, 21));352 353 // R_AARCH64_ADR_PREL_LO21 operates on an ADR instruction, whose354 // 21-bit immediate is split between two bits high up in the word355 // (in fact the two _lowest_ order bits of the value) and 19 bits356 // lower down.357 //358 // R_AARCH64_ADR_PREL_PG_HI21[_NC] operate on an ADRP instruction,359 // which encodes the immediate in the same way, but will shift it360 // left by 12 bits when the instruction executes. For the same361 // reason as the MOVW family, we don't apply that left shift here.362 case R_AARCH64_ADR_PREL_LO21:363 case R_AARCH64_ADR_PREL_PG_HI21:364 case R_AARCH64_ADR_PREL_PG_HI21_NC:365 return SignExtend64<21>((getBits(read32le(buf), 5, 23) << 2) |366 getBits(read32le(buf), 29, 30));367 368 // R_AARCH64_{JUMP,CALL}26 operate on B and BL, which have a369 // 26-bit offset measured in instructions.370 case R_AARCH64_JUMP26:371 case R_AARCH64_CALL26:372 return SignExtend64<28>(getBits(read32le(buf), 0, 25) << 2);373 374 default:375 InternalErr(ctx, buf) << "cannot read addend for relocation " << type;376 return 0;377 }378}379 380void AArch64::writeGotPlt(uint8_t *buf, const Symbol &) const {381 write64(ctx, buf, ctx.in.plt->getVA());382}383 384void AArch64::writeIgotPlt(uint8_t *buf, const Symbol &s) const {385 if (ctx.arg.writeAddends)386 write64(ctx, buf, s.getVA(ctx));387}388 389void AArch64::writePltHeader(uint8_t *buf) const {390 const uint8_t pltData[] = {391 0xf0, 0x7b, 0xbf, 0xa9, // stp x16, x30, [sp,#-16]!392 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.got.plt[2]))393 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.got.plt[2]))]394 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.got.plt[2]))395 0x20, 0x02, 0x1f, 0xd6, // br x17396 0x1f, 0x20, 0x03, 0xd5, // nop397 0x1f, 0x20, 0x03, 0xd5, // nop398 0x1f, 0x20, 0x03, 0xd5 // nop399 };400 memcpy(buf, pltData, sizeof(pltData));401 402 uint64_t got = ctx.in.gotPlt->getVA();403 uint64_t plt = ctx.in.plt->getVA();404 relocateNoSym(buf + 4, R_AARCH64_ADR_PREL_PG_HI21,405 getAArch64Page(got + 16) - getAArch64Page(plt + 4));406 relocateNoSym(buf + 8, R_AARCH64_LDST64_ABS_LO12_NC, got + 16);407 relocateNoSym(buf + 12, R_AARCH64_ADD_ABS_LO12_NC, got + 16);408}409 410void AArch64::writePlt(uint8_t *buf, const Symbol &sym,411 uint64_t pltEntryAddr) const {412 const uint8_t inst[] = {413 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.got.plt[n]))414 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.got.plt[n]))]415 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.got.plt[n]))416 0x20, 0x02, 0x1f, 0xd6 // br x17417 };418 memcpy(buf, inst, sizeof(inst));419 420 uint64_t gotPltEntryAddr = sym.getGotPltVA(ctx);421 relocateNoSym(buf, R_AARCH64_ADR_PREL_PG_HI21,422 getAArch64Page(gotPltEntryAddr) - getAArch64Page(pltEntryAddr));423 relocateNoSym(buf + 4, R_AARCH64_LDST64_ABS_LO12_NC, gotPltEntryAddr);424 relocateNoSym(buf + 8, R_AARCH64_ADD_ABS_LO12_NC, gotPltEntryAddr);425}426 427bool AArch64::needsThunk(RelExpr expr, RelType type, const InputFile *file,428 uint64_t branchAddr, const Symbol &s,429 int64_t a) const {430 // If s is an undefined weak symbol and does not have a PLT entry then it will431 // be resolved as a branch to the next instruction. If it is hidden, its432 // binding has been converted to local, so we just check isUndefined() here. A433 // undefined non-weak symbol will have been errored.434 if (s.isUndefined() && !s.isInPlt(ctx))435 return false;436 // ELF for the ARM 64-bit architecture, section Call and Jump relocations437 // only permits range extension thunks for R_AARCH64_CALL26 and438 // R_AARCH64_JUMP26 relocation types.439 if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26 &&440 type != R_AARCH64_PLT32)441 return false;442 uint64_t dst = expr == R_PLT_PC ? s.getPltVA(ctx) : s.getVA(ctx, a);443 return !inBranchRange(type, branchAddr, dst);444}445 446uint32_t AArch64::getThunkSectionSpacing() const {447 // See comment in Arch/ARM.cpp for a more detailed explanation of448 // getThunkSectionSpacing(). For AArch64 the only branches we are permitted to449 // Thunk have a range of +/- 128 MiB450 return (128 * 1024 * 1024) - 0x30000;451}452 453bool AArch64::inBranchRange(RelType type, uint64_t src, uint64_t dst) const {454 if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26 &&455 type != R_AARCH64_PLT32)456 return true;457 // The AArch64 call and unconditional branch instructions have a range of458 // +/- 128 MiB. The PLT32 relocation supports a range up to +/- 2 GiB.459 uint64_t range =460 type == R_AARCH64_PLT32 ? (UINT64_C(1) << 31) : (128 * 1024 * 1024);461 if (dst > src) {462 // Immediate of branch is signed.463 range -= 4;464 return dst - src <= range;465 }466 return src - dst <= range;467}468 469static void write32AArch64Addr(uint8_t *l, uint64_t imm) {470 uint32_t immLo = (imm & 0x3) << 29;471 uint32_t immHi = (imm & 0x1FFFFC) << 3;472 uint64_t mask = (0x3 << 29) | (0x1FFFFC << 3);473 write32le(l, (read32le(l) & ~mask) | immLo | immHi);474}475 476static void writeMaskedBits32le(uint8_t *p, int32_t v, uint32_t mask) {477 write32le(p, (read32le(p) & ~mask) | v);478}479 480// Update the immediate field in a AARCH64 ldr, str, and add instruction.481static void write32Imm12(uint8_t *l, uint64_t imm) {482 writeMaskedBits32le(l, (imm & 0xFFF) << 10, 0xFFF << 10);483}484 485// Update the immediate field in an AArch64 movk, movn or movz instruction486// for a signed relocation, and update the opcode of a movn or movz instruction487// to match the sign of the operand.488static void writeSMovWImm(uint8_t *loc, uint32_t imm) {489 uint32_t inst = read32le(loc);490 // Opcode field is bits 30, 29, with 10 = movz, 00 = movn and 11 = movk.491 if (!(inst & (1 << 29))) {492 // movn or movz.493 if (imm & 0x10000) {494 // Change opcode to movn, which takes an inverted operand.495 imm ^= 0xFFFF;496 inst &= ~(1 << 30);497 } else {498 // Change opcode to movz.499 inst |= 1 << 30;500 }501 }502 write32le(loc, inst | ((imm & 0xFFFF) << 5));503}504 505void AArch64::relocate(uint8_t *loc, const Relocation &rel,506 uint64_t val) const {507 switch (rel.type) {508 case R_AARCH64_ABS16:509 case R_AARCH64_PREL16:510 checkIntUInt(ctx, loc, val, 16, rel);511 write16(ctx, loc, val);512 break;513 case R_AARCH64_ABS32:514 case R_AARCH64_PREL32:515 checkIntUInt(ctx, loc, val, 32, rel);516 write32(ctx, loc, val);517 break;518 case R_AARCH64_PATCHINST:519 if (!rel.sym->isUndefined()) {520 checkUInt(ctx, loc, val, 32, rel);521 write32le(loc, val);522 }523 break;524 case R_AARCH64_PLT32:525 case R_AARCH64_GOTPCREL32:526 checkInt(ctx, loc, val, 32, rel);527 write32(ctx, loc, val);528 break;529 case R_AARCH64_ABS64:530 // AArch64 relocations to tagged symbols have extended semantics, as531 // described here:532 // https://github.com/ARM-software/abi-aa/blob/main/memtagabielf64/memtagabielf64.rst#841extended-semantics-of-r_aarch64_relative.533 // tl;dr: encode the symbol's special addend in the place, which is an534 // offset to the point where the logical tag is derived from. Quick hack, if535 // the addend is within the symbol's bounds, no need to encode the tag536 // derivation offset.537 if (rel.sym && rel.sym->isTagged() &&538 (rel.addend < 0 ||539 rel.addend >= static_cast<int64_t>(rel.sym->getSize())))540 write64(ctx, loc, -rel.addend);541 else542 write64(ctx, loc, val);543 break;544 case R_AARCH64_PREL64:545 write64(ctx, loc, val);546 break;547 case R_AARCH64_AUTH_ABS64:548 // If val is wider than 32 bits, the relocation must have been moved from549 // .relr.auth.dyn to .rela.dyn, and the addend write is not needed.550 //551 // If val fits in 32 bits, we have two potential scenarios:552 // * True RELR: Write the 32-bit `val`.553 // * RELA: Even if the value now fits in 32 bits, it might have been554 // converted from RELR during an iteration in555 // finalizeAddressDependentContent(). Writing the value is harmless556 // because dynamic linking ignores it.557 if (isInt<32>(val))558 write32(ctx, loc, val);559 break;560 case R_AARCH64_ADD_ABS_LO12_NC:561 case R_AARCH64_AUTH_GOT_ADD_LO12_NC:562 write32Imm12(loc, val);563 break;564 case R_AARCH64_ADR_GOT_PAGE:565 case R_AARCH64_AUTH_ADR_GOT_PAGE:566 case R_AARCH64_ADR_PREL_PG_HI21:567 case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21:568 case R_AARCH64_TLSDESC_ADR_PAGE21:569 case R_AARCH64_AUTH_TLSDESC_ADR_PAGE21:570 checkInt(ctx, loc, val, 33, rel);571 [[fallthrough]];572 case R_AARCH64_ADR_PREL_PG_HI21_NC:573 write32AArch64Addr(loc, val >> 12);574 break;575 case R_AARCH64_ADR_PREL_LO21:576 case R_AARCH64_AUTH_GOT_ADR_PREL_LO21:577 checkInt(ctx, loc, val, 21, rel);578 write32AArch64Addr(loc, val);579 break;580 case R_AARCH64_JUMP26:581 // Normally we would just write the bits of the immediate field, however582 // when patching instructions for the cpu errata fix -fix-cortex-a53-843419583 // we want to replace a non-branch instruction with a branch immediate584 // instruction. By writing all the bits of the instruction including the585 // opcode and the immediate (0 001 | 01 imm26) we can do this586 // transformation by placing a R_AARCH64_JUMP26 relocation at the offset of587 // the instruction we want to patch.588 write32le(loc, 0x14000000);589 [[fallthrough]];590 case R_AARCH64_CALL26:591 checkInt(ctx, loc, val, 28, rel);592 writeMaskedBits32le(loc, (val & 0x0FFFFFFC) >> 2, 0x0FFFFFFC >> 2);593 break;594 case R_AARCH64_CONDBR19:595 case R_AARCH64_LD_PREL_LO19:596 case R_AARCH64_GOT_LD_PREL19:597 case R_AARCH64_AUTH_GOT_LD_PREL19:598 checkAlignment(ctx, loc, val, 4, rel);599 checkInt(ctx, loc, val, 21, rel);600 writeMaskedBits32le(loc, (val & 0x1FFFFC) << 3, 0x1FFFFC << 3);601 break;602 case R_AARCH64_LDST8_ABS_LO12_NC:603 case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC:604 write32Imm12(loc, getBits(val, 0, 11));605 break;606 case R_AARCH64_LDST16_ABS_LO12_NC:607 case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC:608 checkAlignment(ctx, loc, val, 2, rel);609 write32Imm12(loc, getBits(val, 1, 11));610 break;611 case R_AARCH64_LDST32_ABS_LO12_NC:612 case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC:613 checkAlignment(ctx, loc, val, 4, rel);614 write32Imm12(loc, getBits(val, 2, 11));615 break;616 case R_AARCH64_LDST64_ABS_LO12_NC:617 case R_AARCH64_LD64_GOT_LO12_NC:618 case R_AARCH64_AUTH_LD64_GOT_LO12_NC:619 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:620 case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC:621 case R_AARCH64_TLSDESC_LD64_LO12:622 case R_AARCH64_AUTH_TLSDESC_LD64_LO12:623 checkAlignment(ctx, loc, val, 8, rel);624 write32Imm12(loc, getBits(val, 3, 11));625 break;626 case R_AARCH64_LDST128_ABS_LO12_NC:627 case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC:628 checkAlignment(ctx, loc, val, 16, rel);629 write32Imm12(loc, getBits(val, 4, 11));630 break;631 case R_AARCH64_LD64_GOTPAGE_LO15:632 checkAlignment(ctx, loc, val, 8, rel);633 write32Imm12(loc, getBits(val, 3, 14));634 break;635 case R_AARCH64_MOVW_UABS_G0:636 checkUInt(ctx, loc, val, 16, rel);637 [[fallthrough]];638 case R_AARCH64_MOVW_UABS_G0_NC:639 writeMaskedBits32le(loc, (val & 0xFFFF) << 5, 0xFFFF << 5);640 break;641 case R_AARCH64_MOVW_UABS_G1:642 checkUInt(ctx, loc, val, 32, rel);643 [[fallthrough]];644 case R_AARCH64_MOVW_UABS_G1_NC:645 writeMaskedBits32le(loc, (val & 0xFFFF0000) >> 11, 0xFFFF0000 >> 11);646 break;647 case R_AARCH64_MOVW_UABS_G2:648 checkUInt(ctx, loc, val, 48, rel);649 [[fallthrough]];650 case R_AARCH64_MOVW_UABS_G2_NC:651 writeMaskedBits32le(loc, (val & 0xFFFF00000000) >> 27,652 0xFFFF00000000 >> 27);653 break;654 case R_AARCH64_MOVW_UABS_G3:655 writeMaskedBits32le(loc, (val & 0xFFFF000000000000) >> 43,656 0xFFFF000000000000 >> 43);657 break;658 case R_AARCH64_MOVW_PREL_G0:659 case R_AARCH64_MOVW_SABS_G0:660 case R_AARCH64_TLSLE_MOVW_TPREL_G0:661 checkInt(ctx, loc, val, 17, rel);662 [[fallthrough]];663 case R_AARCH64_MOVW_PREL_G0_NC:664 case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC:665 writeSMovWImm(loc, val);666 break;667 case R_AARCH64_MOVW_PREL_G1:668 case R_AARCH64_MOVW_SABS_G1:669 case R_AARCH64_TLSLE_MOVW_TPREL_G1:670 checkInt(ctx, loc, val, 33, rel);671 [[fallthrough]];672 case R_AARCH64_MOVW_PREL_G1_NC:673 case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC:674 writeSMovWImm(loc, val >> 16);675 break;676 case R_AARCH64_MOVW_PREL_G2:677 case R_AARCH64_MOVW_SABS_G2:678 case R_AARCH64_TLSLE_MOVW_TPREL_G2:679 checkInt(ctx, loc, val, 49, rel);680 [[fallthrough]];681 case R_AARCH64_MOVW_PREL_G2_NC:682 writeSMovWImm(loc, val >> 32);683 break;684 case R_AARCH64_MOVW_PREL_G3:685 writeSMovWImm(loc, val >> 48);686 break;687 case R_AARCH64_TSTBR14:688 checkInt(ctx, loc, val, 16, rel);689 writeMaskedBits32le(loc, (val & 0xFFFC) << 3, 0xFFFC << 3);690 break;691 case R_AARCH64_TLSLE_ADD_TPREL_HI12:692 checkUInt(ctx, loc, val, 24, rel);693 write32Imm12(loc, val >> 12);694 break;695 case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:696 case R_AARCH64_TLSDESC_ADD_LO12:697 case R_AARCH64_AUTH_TLSDESC_ADD_LO12:698 write32Imm12(loc, val);699 break;700 case R_AARCH64_TLSDESC:701 // For R_AARCH64_TLSDESC the addend is stored in the second 64-bit word.702 write64(ctx, loc + 8, val);703 break;704 default:705 llvm_unreachable("unknown relocation");706 }707}708 709void AArch64::relaxTlsGdToLe(uint8_t *loc, const Relocation &rel,710 uint64_t val) const {711 // TLSDESC Global-Dynamic relocation are in the form:712 // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21]713 // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12]714 // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12]715 // .tlsdesccall [R_AARCH64_TLSDESC_CALL]716 // blr x1717 // And it can optimized to:718 // movz x0, #0x0, lsl #16719 // movk x0, #0x10720 // nop721 // nop722 checkUInt(ctx, loc, val, 32, rel);723 724 switch (rel.type) {725 case R_AARCH64_TLSDESC_ADD_LO12:726 case R_AARCH64_TLSDESC_CALL:727 write32le(loc, 0xd503201f); // nop728 return;729 case R_AARCH64_TLSDESC_ADR_PAGE21:730 write32le(loc, 0xd2a00000 | (((val >> 16) & 0xffff) << 5)); // movz731 return;732 case R_AARCH64_TLSDESC_LD64_LO12:733 write32le(loc, 0xf2800000 | ((val & 0xffff) << 5)); // movk734 return;735 default:736 llvm_unreachable("unsupported relocation for TLS GD to LE relaxation");737 }738}739 740void AArch64::relaxTlsGdToIe(uint8_t *loc, const Relocation &rel,741 uint64_t val) const {742 // TLSDESC Global-Dynamic relocation are in the form:743 // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21]744 // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12]745 // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12]746 // .tlsdesccall [R_AARCH64_TLSDESC_CALL]747 // blr x1748 // And it can optimized to:749 // adrp x0, :gottprel:v750 // ldr x0, [x0, :gottprel_lo12:v]751 // nop752 // nop753 754 switch (rel.type) {755 case R_AARCH64_TLSDESC_ADD_LO12:756 case R_AARCH64_TLSDESC_CALL:757 write32le(loc, 0xd503201f); // nop758 break;759 case R_AARCH64_TLSDESC_ADR_PAGE21:760 write32le(loc, 0x90000000); // adrp761 relocateNoSym(loc, R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21, val);762 break;763 case R_AARCH64_TLSDESC_LD64_LO12:764 write32le(loc, 0xf9400000); // ldr765 relocateNoSym(loc, R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC, val);766 break;767 default:768 llvm_unreachable("unsupported relocation for TLS GD to IE relaxation");769 }770}771 772void AArch64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel,773 uint64_t val) const {774 checkUInt(ctx, loc, val, 32, rel);775 776 if (rel.type == R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21) {777 // Generate MOVZ.778 uint32_t regNo = read32le(loc) & 0x1f;779 write32le(loc, (0xd2a00000 | regNo) | (((val >> 16) & 0xffff) << 5));780 return;781 }782 if (rel.type == R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC) {783 // Generate MOVK.784 uint32_t regNo = read32le(loc) & 0x1f;785 write32le(loc, (0xf2800000 | regNo) | ((val & 0xffff) << 5));786 return;787 }788 llvm_unreachable("invalid relocation for TLS IE to LE relaxation");789}790 791AArch64Relaxer::AArch64Relaxer(Ctx &ctx, ArrayRef<Relocation> relocs)792 : ctx(ctx) {793 if (!ctx.arg.relax)794 return;795 // Check if R_AARCH64_ADR_GOT_PAGE and R_AARCH64_LD64_GOT_LO12_NC796 // always appear in pairs.797 size_t i = 0;798 const size_t size = relocs.size();799 for (; i != size; ++i) {800 if (relocs[i].type == R_AARCH64_ADR_GOT_PAGE) {801 if (i + 1 < size && relocs[i + 1].type == R_AARCH64_LD64_GOT_LO12_NC) {802 ++i;803 continue;804 }805 break;806 } else if (relocs[i].type == R_AARCH64_LD64_GOT_LO12_NC) {807 break;808 }809 }810 safeToRelaxAdrpLdr = i == size;811}812 813bool AArch64Relaxer::tryRelaxAdrpAdd(const Relocation &adrpRel,814 const Relocation &addRel, uint64_t secAddr,815 uint8_t *buf) const {816 // When the address of sym is within the range of ADR then817 // we may relax818 // ADRP xn, sym819 // ADD xn, xn, :lo12: sym820 // to821 // NOP822 // ADR xn, sym823 if (!ctx.arg.relax || adrpRel.type != R_AARCH64_ADR_PREL_PG_HI21 ||824 addRel.type != R_AARCH64_ADD_ABS_LO12_NC)825 return false;826 // Check if the relocations apply to consecutive instructions.827 if (adrpRel.offset + 4 != addRel.offset)828 return false;829 if (adrpRel.sym != addRel.sym)830 return false;831 if (adrpRel.addend != 0 || addRel.addend != 0)832 return false;833 834 uint32_t adrpInstr = read32le(buf + adrpRel.offset);835 uint32_t addInstr = read32le(buf + addRel.offset);836 // Check if the first instruction is ADRP and the second instruction is ADD.837 if ((adrpInstr & 0x9f000000) != 0x90000000 ||838 (addInstr & 0xffc00000) != 0x91000000)839 return false;840 uint32_t adrpDestReg = adrpInstr & 0x1f;841 uint32_t addDestReg = addInstr & 0x1f;842 uint32_t addSrcReg = (addInstr >> 5) & 0x1f;843 if (adrpDestReg != addDestReg || adrpDestReg != addSrcReg)844 return false;845 846 Symbol &sym = *adrpRel.sym;847 // Check if the address difference is within 1MiB range.848 int64_t val = sym.getVA(ctx) - (secAddr + addRel.offset);849 if (val < -1024 * 1024 || val >= 1024 * 1024)850 return false;851 852 Relocation adrRel = {R_ABS, R_AARCH64_ADR_PREL_LO21, addRel.offset,853 /*addend=*/0, &sym};854 // nop855 write32le(buf + adrpRel.offset, 0xd503201f);856 // adr x_<dest_reg>857 write32le(buf + adrRel.offset, 0x10000000 | adrpDestReg);858 ctx.target->relocate(buf + adrRel.offset, adrRel, val);859 return true;860}861 862bool AArch64Relaxer::tryRelaxAdrpLdr(const Relocation &adrpRel,863 const Relocation &ldrRel, uint64_t secAddr,864 uint8_t *buf) const {865 if (!safeToRelaxAdrpLdr)866 return false;867 868 // When the definition of sym is not preemptible then we may869 // be able to relax870 // ADRP xn, :got: sym871 // LDR xn, [ xn :got_lo12: sym]872 // to873 // ADRP xn, sym874 // ADD xn, xn, :lo_12: sym875 876 if (adrpRel.type != R_AARCH64_ADR_GOT_PAGE ||877 ldrRel.type != R_AARCH64_LD64_GOT_LO12_NC)878 return false;879 // Check if the relocations apply to consecutive instructions.880 if (adrpRel.offset + 4 != ldrRel.offset)881 return false;882 // Check if the relocations reference the same symbol and883 // skip undefined, preemptible and STT_GNU_IFUNC symbols.884 if (!adrpRel.sym || adrpRel.sym != ldrRel.sym || !adrpRel.sym->isDefined() ||885 adrpRel.sym->isPreemptible || adrpRel.sym->isGnuIFunc())886 return false;887 // Check if the addends of the both relocations are zero.888 if (adrpRel.addend != 0 || ldrRel.addend != 0)889 return false;890 uint32_t adrpInstr = read32le(buf + adrpRel.offset);891 uint32_t ldrInstr = read32le(buf + ldrRel.offset);892 // Check if the first instruction is ADRP and the second instruction is LDR.893 if ((adrpInstr & 0x9f000000) != 0x90000000 ||894 (ldrInstr & 0x3b000000) != 0x39000000)895 return false;896 // Check the value of the sf bit.897 if (!(ldrInstr >> 31))898 return false;899 uint32_t adrpDestReg = adrpInstr & 0x1f;900 uint32_t ldrDestReg = ldrInstr & 0x1f;901 uint32_t ldrSrcReg = (ldrInstr >> 5) & 0x1f;902 // Check if ADPR and LDR use the same register.903 if (adrpDestReg != ldrDestReg || adrpDestReg != ldrSrcReg)904 return false;905 906 Symbol &sym = *adrpRel.sym;907 // GOT references to absolute symbols can't be relaxed to use ADRP/ADD in908 // position-independent code because these instructions produce a relative909 // address.910 if (ctx.arg.isPic && !cast<Defined>(sym).section)911 return false;912 // Check if the address difference is within 4GB range.913 int64_t val =914 getAArch64Page(sym.getVA(ctx)) - getAArch64Page(secAddr + adrpRel.offset);915 if (val != llvm::SignExtend64(val, 33))916 return false;917 918 Relocation adrpSymRel = {RE_AARCH64_PAGE_PC, R_AARCH64_ADR_PREL_PG_HI21,919 adrpRel.offset, /*addend=*/0, &sym};920 Relocation addRel = {R_ABS, R_AARCH64_ADD_ABS_LO12_NC, ldrRel.offset,921 /*addend=*/0, &sym};922 923 // adrp x_<dest_reg>924 write32le(buf + adrpSymRel.offset, 0x90000000 | adrpDestReg);925 // add x_<dest reg>, x_<dest reg>926 write32le(buf + addRel.offset, 0x91000000 | adrpDestReg | (adrpDestReg << 5));927 928 ctx.target->relocate(929 buf + adrpSymRel.offset, adrpSymRel,930 SignExtend64(getAArch64Page(sym.getVA(ctx)) -931 getAArch64Page(secAddr + adrpSymRel.offset),932 64));933 ctx.target->relocate(buf + addRel.offset, addRel,934 SignExtend64(sym.getVA(ctx), 64));935 tryRelaxAdrpAdd(adrpSymRel, addRel, secAddr, buf);936 return true;937}938 939// Tagged symbols have upper address bits that are added by the dynamic loader,940// and thus need the full 64-bit GOT entry. Do not relax such symbols.941static bool needsGotForMemtag(const Relocation &rel) {942 return rel.sym->isTagged() && needsGot(rel.expr);943}944 945void AArch64::relocateAlloc(InputSection &sec, uint8_t *buf) const {946 uint64_t secAddr = sec.getOutputSection()->addr + sec.outSecOff;947 AArch64Relaxer relaxer(ctx, sec.relocs());948 for (size_t i = 0, size = sec.relocs().size(); i != size; ++i) {949 const Relocation &rel = sec.relocs()[i];950 uint8_t *loc = buf + rel.offset;951 const uint64_t val = sec.getRelocTargetVA(ctx, rel, secAddr + rel.offset);952 953 if (needsGotForMemtag(rel)) {954 relocate(loc, rel, val);955 continue;956 }957 958 switch (rel.expr) {959 case RE_AARCH64_GOT_PAGE_PC:960 if (i + 1 < size &&961 relaxer.tryRelaxAdrpLdr(rel, sec.relocs()[i + 1], secAddr, buf)) {962 ++i;963 continue;964 }965 break;966 case RE_AARCH64_PAGE_PC:967 if (i + 1 < size &&968 relaxer.tryRelaxAdrpAdd(rel, sec.relocs()[i + 1], secAddr, buf)) {969 ++i;970 continue;971 }972 break;973 case RE_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC:974 case R_RELAX_TLS_GD_TO_IE_ABS:975 relaxTlsGdToIe(loc, rel, val);976 continue;977 case R_RELAX_TLS_GD_TO_LE:978 relaxTlsGdToLe(loc, rel, val);979 continue;980 case R_RELAX_TLS_IE_TO_LE:981 relaxTlsIeToLe(loc, rel, val);982 continue;983 default:984 break;985 }986 relocate(loc, rel, val);987 }988}989 990static std::optional<uint64_t> getControlTransferAddend(InputSection &is,991 Relocation &r) {992 // Identify a control transfer relocation for the branch-to-branch993 // optimization. A "control transfer relocation" means a B or BL994 // target but it also includes relative vtable relocations for example.995 //996 // We require the relocation type to be JUMP26, CALL26 or PLT32. With a997 // relocation type of PLT32 the value may be assumed to be used for branching998 // directly to the symbol and the addend is only used to produce the relocated999 // value (hence the effective addend is always 0). This is because if a PLT is1000 // needed the addend will be added to the address of the PLT, and it doesn't1001 // make sense to branch into the middle of a PLT. For example, relative vtable1002 // relocations use PLT32 and 0 or a positive value as the addend but still are1003 // used to branch to the symbol.1004 //1005 // With JUMP26 or CALL26 the only reasonable interpretation of a non-zero1006 // addend is that we are branching to symbol+addend so that becomes the1007 // effective addend.1008 if (r.type == R_AARCH64_PLT32)1009 return 0;1010 if (r.type == R_AARCH64_JUMP26 || r.type == R_AARCH64_CALL26)1011 return r.addend;1012 return std::nullopt;1013}1014 1015static std::pair<Relocation *, uint64_t>1016getBranchInfoAtTarget(InputSection &is, uint64_t offset) {1017 auto *i = llvm::partition_point(1018 is.relocations, [&](Relocation &r) { return r.offset < offset; });1019 if (i != is.relocations.end() && i->offset == offset &&1020 i->type == R_AARCH64_JUMP26) {1021 return {i, i->addend};1022 }1023 return {nullptr, 0};1024}1025 1026static void redirectControlTransferRelocations(Relocation &r1,1027 const Relocation &r2) {1028 r1.expr = r2.expr;1029 r1.sym = r2.sym;1030 // With PLT32 we must respect the original addend as that affects the value's1031 // interpretation. With the other relocation types the original addend is1032 // irrelevant because it referred to an offset within the original target1033 // section so we overwrite it.1034 if (r1.type == R_AARCH64_PLT32)1035 r1.addend += r2.addend;1036 else1037 r1.addend = r2.addend;1038}1039 1040void AArch64::applyBranchToBranchOpt() const {1041 applyBranchToBranchOptImpl(ctx, getControlTransferAddend,1042 getBranchInfoAtTarget,1043 redirectControlTransferRelocations);1044}1045 1046// AArch64 may use security features in variant PLT sequences. These are:1047// Pointer Authentication (PAC), introduced in armv8.3-a and Branch Target1048// Indicator (BTI) introduced in armv8.5-a. The additional instructions used1049// in the variant Plt sequences are encoded in the Hint space so they can be1050// deployed on older architectures, which treat the instructions as a nop.1051// PAC and BTI can be combined leading to the following combinations:1052// writePltHeader1053// writePltHeaderBti (no PAC Header needed)1054// writePlt1055// writePltBti (BTI only)1056// writePltPac (PAC only)1057// writePltBtiPac (BTI and PAC)1058//1059// When PAC is enabled the dynamic loader encrypts the address that it places1060// in the .got.plt using the pacia1716 instruction which encrypts the value in1061// x17 using the modifier in x16. The static linker places autia1716 before the1062// indirect branch to x17 to authenticate the address in x17 with the modifier1063// in x16. This makes it more difficult for an attacker to modify the value in1064// the .got.plt.1065//1066// When BTI is enabled all indirect branches must land on a bti instruction.1067// The static linker must place a bti instruction at the start of any PLT entry1068// that may be the target of an indirect branch. As the PLT entries call the1069// lazy resolver indirectly this must have a bti instruction at start. In1070// general a bti instruction is not needed for a PLT entry as indirect calls1071// are resolved to the function address and not the PLT entry for the function.1072// There are a small number of cases where the PLT address can escape, such as1073// taking the address of a function or ifunc via a non got-generating1074// relocation, and a shared library refers to that symbol.1075//1076// We use the bti c variant of the instruction which permits indirect branches1077// (br) via x16/x17 and indirect function calls (blr) via any register. The ABI1078// guarantees that all indirect branches from code requiring BTI protection1079// will go via x16/x171080 1081namespace {1082class AArch64BtiPac final : public AArch64 {1083public:1084 AArch64BtiPac(Ctx &);1085 void writePltHeader(uint8_t *buf) const override;1086 void writePlt(uint8_t *buf, const Symbol &sym,1087 uint64_t pltEntryAddr) const override;1088 1089private:1090 bool btiHeader; // bti instruction needed in PLT Header and Entry1091 enum {1092 PEK_NoAuth,1093 PEK_AuthHint, // use autia1716 instr for authenticated branch in PLT entry1094 PEK_Auth, // use braa instr for authenticated branch in PLT entry1095 } pacEntryKind;1096};1097} // namespace1098 1099AArch64BtiPac::AArch64BtiPac(Ctx &ctx) : AArch64(ctx) {1100 btiHeader = (ctx.arg.andFeatures & GNU_PROPERTY_AARCH64_FEATURE_1_BTI);1101 // A BTI (Branch Target Indicator) Plt Entry is only required if the1102 // address of the PLT entry can be taken by the program, which permits an1103 // indirect jump to the PLT entry. This can happen when the address1104 // of the PLT entry for a function is canonicalised due to the address of1105 // the function in an executable being taken by a shared library, or1106 // non-preemptible ifunc referenced by non-GOT-generating, non-PLT-generating1107 // relocations.1108 // The PAC PLT entries require dynamic loader support and this isn't known1109 // from properties in the objects, so we use the command line flag.1110 // By default we only use hint-space instructions, but if we detect the1111 // PAuthABI, which requires v8.3-A, we can use the non-hint space1112 // instructions.1113 1114 if (ctx.arg.zPacPlt) {1115 if (ctx.aarch64PauthAbiCoreInfo && ctx.aarch64PauthAbiCoreInfo->isValid())1116 pacEntryKind = PEK_Auth;1117 else1118 pacEntryKind = PEK_AuthHint;1119 } else {1120 pacEntryKind = PEK_NoAuth;1121 }1122 1123 if (btiHeader || (pacEntryKind != PEK_NoAuth)) {1124 pltEntrySize = 24;1125 ipltEntrySize = 24;1126 }1127}1128 1129void AArch64BtiPac::writePltHeader(uint8_t *buf) const {1130 const uint8_t btiData[] = { 0x5f, 0x24, 0x03, 0xd5 }; // bti c1131 const uint8_t pltData[] = {1132 0xf0, 0x7b, 0xbf, 0xa9, // stp x16, x30, [sp,#-16]!1133 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.got.plt[2]))1134 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.got.plt[2]))]1135 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.got.plt[2]))1136 0x20, 0x02, 0x1f, 0xd6, // br x171137 0x1f, 0x20, 0x03, 0xd5, // nop1138 0x1f, 0x20, 0x03, 0xd5 // nop1139 };1140 const uint8_t nopData[] = { 0x1f, 0x20, 0x03, 0xd5 }; // nop1141 1142 uint64_t got = ctx.in.gotPlt->getVA();1143 uint64_t plt = ctx.in.plt->getVA();1144 1145 if (btiHeader) {1146 // PltHeader is called indirectly by plt[N]. Prefix pltData with a BTI C1147 // instruction.1148 memcpy(buf, btiData, sizeof(btiData));1149 buf += sizeof(btiData);1150 plt += sizeof(btiData);1151 }1152 memcpy(buf, pltData, sizeof(pltData));1153 1154 relocateNoSym(buf + 4, R_AARCH64_ADR_PREL_PG_HI21,1155 getAArch64Page(got + 16) - getAArch64Page(plt + 4));1156 relocateNoSym(buf + 8, R_AARCH64_LDST64_ABS_LO12_NC, got + 16);1157 relocateNoSym(buf + 12, R_AARCH64_ADD_ABS_LO12_NC, got + 16);1158 if (!btiHeader)1159 // We didn't add the BTI c instruction so round out size with NOP.1160 memcpy(buf + sizeof(pltData), nopData, sizeof(nopData));1161}1162 1163void AArch64BtiPac::writePlt(uint8_t *buf, const Symbol &sym,1164 uint64_t pltEntryAddr) const {1165 // The PLT entry is of the form:1166 // [btiData] addrInst (pacBr | stdBr) [nopData]1167 const uint8_t btiData[] = { 0x5f, 0x24, 0x03, 0xd5 }; // bti c1168 const uint8_t addrInst[] = {1169 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.got.plt[n]))1170 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.got.plt[n]))]1171 0x10, 0x02, 0x00, 0x91 // add x16, x16, Offset(&(.got.plt[n]))1172 };1173 const uint8_t pacHintBr[] = {1174 0x9f, 0x21, 0x03, 0xd5, // autia17161175 0x20, 0x02, 0x1f, 0xd6 // br x171176 };1177 const uint8_t pacBr[] = {1178 0x30, 0x0a, 0x1f, 0xd7, // braa x17, x161179 0x1f, 0x20, 0x03, 0xd5 // nop1180 };1181 const uint8_t stdBr[] = {1182 0x20, 0x02, 0x1f, 0xd6, // br x171183 0x1f, 0x20, 0x03, 0xd5 // nop1184 };1185 const uint8_t nopData[] = { 0x1f, 0x20, 0x03, 0xd5 }; // nop1186 1187 // NEEDS_COPY indicates a non-ifunc canonical PLT entry whose address may1188 // escape to shared objects. isInIplt indicates a non-preemptible ifunc. Its1189 // address may escape if referenced by a direct relocation. If relative1190 // vtables are used then if the vtable is in a shared object the offsets will1191 // be to the PLT entry. The condition is conservative.1192 bool hasBti = btiHeader &&1193 (sym.hasFlag(NEEDS_COPY) || sym.isInIplt || sym.thunkAccessed);1194 if (hasBti) {1195 memcpy(buf, btiData, sizeof(btiData));1196 buf += sizeof(btiData);1197 pltEntryAddr += sizeof(btiData);1198 }1199 1200 uint64_t gotPltEntryAddr = sym.getGotPltVA(ctx);1201 memcpy(buf, addrInst, sizeof(addrInst));1202 relocateNoSym(buf, R_AARCH64_ADR_PREL_PG_HI21,1203 getAArch64Page(gotPltEntryAddr) - getAArch64Page(pltEntryAddr));1204 relocateNoSym(buf + 4, R_AARCH64_LDST64_ABS_LO12_NC, gotPltEntryAddr);1205 relocateNoSym(buf + 8, R_AARCH64_ADD_ABS_LO12_NC, gotPltEntryAddr);1206 1207 if (pacEntryKind != PEK_NoAuth)1208 memcpy(buf + sizeof(addrInst),1209 pacEntryKind == PEK_AuthHint ? pacHintBr : pacBr,1210 sizeof(pacEntryKind == PEK_AuthHint ? pacHintBr : pacBr));1211 else1212 memcpy(buf + sizeof(addrInst), stdBr, sizeof(stdBr));1213 if (!hasBti)1214 // We didn't add the BTI c instruction so round out size with NOP.1215 memcpy(buf + sizeof(addrInst) + sizeof(stdBr), nopData, sizeof(nopData));1216}1217 1218template <class ELFT>1219static void1220addTaggedSymbolReferences(Ctx &ctx, InputSectionBase &sec,1221 DenseMap<Symbol *, unsigned> &referenceCount) {1222 assert(sec.type == SHT_AARCH64_MEMTAG_GLOBALS_STATIC);1223 1224 const RelsOrRelas<ELFT> rels = sec.relsOrRelas<ELFT>();1225 if (rels.areRelocsRel())1226 ErrAlways(ctx)1227 << "non-RELA relocations are not allowed with memtag globals";1228 1229 for (const typename ELFT::Rela &rel : rels.relas) {1230 Symbol &sym = sec.file->getRelocTargetSym(rel);1231 // Linker-synthesized symbols such as __executable_start may be referenced1232 // as tagged in input objfiles, and we don't want them to be tagged. A1233 // cheap way to exclude them is the type check, but their type is1234 // STT_NOTYPE. In addition, this save us from checking untaggable symbols,1235 // like functions or TLS symbols.1236 if (sym.type != STT_OBJECT)1237 continue;1238 // STB_LOCAL symbols can't be referenced from outside the object file, and1239 // thus don't need to be checked for references from other object files.1240 if (sym.binding == STB_LOCAL) {1241 sym.setIsTagged(true);1242 continue;1243 }1244 ++referenceCount[&sym];1245 }1246 sec.markDead();1247}1248 1249// A tagged symbol must be denoted as being tagged by all references and the1250// chosen definition. For simplicity, here, it must also be denoted as tagged1251// for all definitions. Otherwise:1252//1253// 1. A tagged definition can be used by an untagged declaration, in which case1254// the untagged access may be PC-relative, causing a tag mismatch at1255// runtime.1256// 2. An untagged definition can be used by a tagged declaration, where the1257// compiler has taken advantage of the increased alignment of the tagged1258// declaration, but the alignment at runtime is wrong, causing a fault.1259//1260// Ideally, this isn't a problem, as any TU that imports or exports tagged1261// symbols should also be built with tagging. But, to handle these cases, we1262// demote the symbol to be untagged.1263void elf::createTaggedSymbols(Ctx &ctx) {1264 assert(hasMemtag(ctx));1265 1266 // First, collect all symbols that are marked as tagged, and count how many1267 // times they're marked as tagged.1268 DenseMap<Symbol *, unsigned> taggedSymbolReferenceCount;1269 for (InputFile *file : ctx.objectFiles) {1270 if (file->kind() != InputFile::ObjKind)1271 continue;1272 for (InputSectionBase *section : file->getSections()) {1273 if (!section || section->type != SHT_AARCH64_MEMTAG_GLOBALS_STATIC ||1274 section == &InputSection::discarded)1275 continue;1276 invokeELFT(addTaggedSymbolReferences, ctx, *section,1277 taggedSymbolReferenceCount);1278 }1279 }1280 1281 // Now, go through all the symbols. If the number of declarations +1282 // definitions to a symbol exceeds the amount of times they're marked as1283 // tagged, it means we have an objfile that uses the untagged variant of the1284 // symbol.1285 for (InputFile *file : ctx.objectFiles) {1286 if (file->kind() != InputFile::BinaryKind &&1287 file->kind() != InputFile::ObjKind)1288 continue;1289 1290 for (Symbol *symbol : file->getSymbols()) {1291 // See `addTaggedSymbolReferences` for more details.1292 if (symbol->type != STT_OBJECT ||1293 symbol->binding == STB_LOCAL)1294 continue;1295 auto it = taggedSymbolReferenceCount.find(symbol);1296 if (it == taggedSymbolReferenceCount.end()) continue;1297 unsigned &remainingAllowedTaggedRefs = it->second;1298 if (remainingAllowedTaggedRefs == 0) {1299 taggedSymbolReferenceCount.erase(it);1300 continue;1301 }1302 --remainingAllowedTaggedRefs;1303 }1304 }1305 1306 // `addTaggedSymbolReferences` has already checked that we have RELA1307 // relocations, the only other way to get written addends is with1308 // --apply-dynamic-relocs.1309 if (!taggedSymbolReferenceCount.empty() && ctx.arg.writeAddends)1310 ErrAlways(ctx) << "--apply-dynamic-relocs cannot be used with MTE globals";1311 1312 // Now, `taggedSymbolReferenceCount` should only contain symbols that are1313 // defined as tagged exactly the same amount as it's referenced, meaning all1314 // uses are tagged.1315 for (auto &[symbol, remainingTaggedRefs] : taggedSymbolReferenceCount) {1316 assert(remainingTaggedRefs == 0 &&1317 "Symbol is defined as tagged more times than it's used");1318 symbol->setIsTagged(true);1319 }1320}1321 1322void elf::setAArch64TargetInfo(Ctx &ctx) {1323 if ((ctx.arg.andFeatures & GNU_PROPERTY_AARCH64_FEATURE_1_BTI) ||1324 ctx.arg.zPacPlt)1325 ctx.target.reset(new AArch64BtiPac(ctx));1326 else1327 ctx.target.reset(new AArch64(ctx));1328}1329