410 lines · cpp
1//===---- aarch64.cpp - Generic JITLink aarch64 edge kinds, utilities -----===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Generic utilities for graphs representing aarch64 objects.10//11//===----------------------------------------------------------------------===//12 13#include "llvm/ExecutionEngine/JITLink/aarch64.h"14 15#include "llvm/Support/BinaryStreamWriter.h"16 17#define DEBUG_TYPE "jitlink"18 19namespace llvm {20namespace jitlink {21namespace aarch64 {22 23const char NullPointerContent[8] = {0x00, 0x00, 0x00, 0x00,24 0x00, 0x00, 0x00, 0x00};25 26const char PointerJumpStubContent[12] = {27 0x10, 0x00, 0x00, (char)0x90u, // ADRP x16, <imm>@page2128 0x10, 0x02, 0x40, (char)0xf9u, // LDR x16, [x16, <imm>@pageoff12]29 0x00, 0x02, 0x1f, (char)0xd6u // BR x1630};31 32const char ReentryTrampolineContent[8] = {33 (char)0xfd, 0x7b, (char)0xbf, (char)0xa9, // STP x30, [sp, #-8]34 0x00, 0x00, 0x00, (char)0x94 // BL35};36 37const char *getEdgeKindName(Edge::Kind R) {38 switch (R) {39 case Pointer64:40 return "Pointer64";41 case Pointer64Authenticated:42 return "Pointer64Authenticated";43 case Pointer32:44 return "Pointer32";45 case Delta64:46 return "Delta64";47 case Delta32:48 return "Delta32";49 case NegDelta64:50 return "NegDelta64";51 case NegDelta32:52 return "NegDelta32";53 case Branch26PCRel:54 return "Branch26PCRel";55 case MoveWide16:56 return "MoveWide16";57 case LDRLiteral19:58 return "LDRLiteral19";59 case TestAndBranch14PCRel:60 return "TestAndBranch14PCRel";61 case CondBranch19PCRel:62 return "CondBranch19PCRel";63 case ADRLiteral21:64 return "ADRLiteral21";65 case Page21:66 return "Page21";67 case PageOffset12:68 return "PageOffset12";69 case GotPageOffset15:70 return "GotPageOffset15";71 case RequestGOTAndTransformToPage21:72 return "RequestGOTAndTransformToPage21";73 case RequestGOTAndTransformToPageOffset12:74 return "RequestGOTAndTransformToPageOffset12";75 case RequestGOTAndTransformToPageOffset15:76 return "RequestGOTAndTransformToPageOffset15";77 case RequestGOTAndTransformToDelta32:78 return "RequestGOTAndTransformToDelta32";79 case RequestTLVPAndTransformToPage21:80 return "RequestTLVPAndTransformToPage21";81 case RequestTLVPAndTransformToPageOffset12:82 return "RequestTLVPAndTransformToPageOffset12";83 case RequestTLSDescEntryAndTransformToPage21:84 return "RequestTLSDescEntryAndTransformToPage21";85 case RequestTLSDescEntryAndTransformToPageOffset12:86 return "RequestTLSDescEntryAndTransformToPageOffset12";87 default:88 return getGenericEdgeKindName(R);89 }90}91 92// Write a 64-bit GPR -> GPR move.93template <typename AppendFtor>94static Error writeMovRegRegSeq(AppendFtor &Append, uint64_t DstReg,95 uint64_t SrcReg) {96 assert(DstReg < 32 && "Dst reg out of range");97 assert(SrcReg < 32 && "Src reg out of range");98 99 if (DstReg == SrcReg)100 return Error::success();101 102 constexpr uint32_t MOVGPR64Template = 0xaa0003e0;103 constexpr uint32_t DstRegIndex = 0;104 constexpr uint32_t SrcRegIndex = 16;105 uint32_t Instr = MOVGPR64Template;106 Instr |= DstReg << DstRegIndex;107 Instr |= SrcReg << SrcRegIndex;108 return Append(Instr);109}110 111// Generate a sequence of imm writes to assign the given value.112template <typename AppendFtor>113static Error writeMovRegImm64Seq(AppendFtor &Append, uint64_t Reg,114 uint64_t Imm) {115 assert(Reg < 32 && "Invalid register number");116 117 constexpr uint32_t MovRegImm64Template = 0xd2800000;118 constexpr unsigned PreserveBitIndex = 29;119 constexpr unsigned ShiftBitsIndex = 21;120 constexpr unsigned ImmBitsIndex = 5;121 122 bool PreserveRegValue = false;123 for (unsigned I = 0; I != 4; ++I) {124 uint32_t ImmBits = Imm & 0xffff;125 Imm >>= 16;126 127 // Skip any all-zero immediates after the first one.128 if (PreserveRegValue && !ImmBits)129 continue;130 131 uint32_t Instr = MovRegImm64Template;132 Instr |= PreserveRegValue << PreserveBitIndex;133 Instr |= (I << ShiftBitsIndex);134 Instr |= ImmBits << ImmBitsIndex;135 Instr |= Reg;136 if (auto Err = Append(Instr))137 return Err;138 PreserveRegValue = true;139 }140 141 return Error::success();142}143 144template <typename AppendFtor>145static Error146writePACSignSeq(AppendFtor &Append, unsigned DstReg, orc::ExecutorAddr RawAddr,147 unsigned RawAddrReg, unsigned DiscriminatorReg, unsigned Key,148 uint64_t EncodedDiscriminator, bool AddressDiversify) {149 assert(DstReg < 32 && "DstReg out of range");150 assert(RawAddrReg < 32 && "AddrReg out of range");151 assert(DiscriminatorReg < 32 && "DiscriminatorReg out of range");152 assert(EncodedDiscriminator < 0x10000 && "EncodedDiscriminator out of range");153 154 if (AddressDiversify) {155 // Move the address into the discriminator register.156 if (auto Err = writeMovRegRegSeq(Append, DiscriminatorReg, RawAddrReg))157 return Err;158 // Blend encoded discriminator if there is one.159 if (EncodedDiscriminator) {160 constexpr uint32_t MOVKTemplate = 0xf2e00000;161 constexpr unsigned ImmIndex = 5;162 uint32_t BlendInstr = MOVKTemplate;163 BlendInstr |= EncodedDiscriminator << ImmIndex;164 BlendInstr |= DiscriminatorReg;165 if (auto Err = Append(BlendInstr))166 return Err;167 }168 } else if (EncodedDiscriminator) {169 // Move the encoded discriminator into the discriminator register.170 if (auto Err =171 writeMovRegImm64Seq(Append, DiscriminatorReg, EncodedDiscriminator))172 return Err;173 } else174 DiscriminatorReg = 31; // WZR175 176 constexpr uint32_t PACTemplate = 0xdac10000;177 constexpr unsigned ZBitIndex = 13;178 constexpr unsigned KeyIndex = 10;179 constexpr unsigned DiscriminatorRegIndex = 5;180 181 uint32_t Instr = PACTemplate;182 Instr |= (DiscriminatorReg == 31) << ZBitIndex;183 Instr |= Key << KeyIndex;184 Instr |= DiscriminatorReg << DiscriminatorRegIndex;185 Instr |= DstReg;186 187 return Append(Instr);188}189 190template <typename AppendFtor>191static Error writeStoreRegSeq(AppendFtor &Append, unsigned DstLocReg,192 unsigned SrcReg) {193 assert(DstLocReg < 32 && "DstLocReg out of range");194 assert(SrcReg < 32 && "SrcReg out of range");195 196 constexpr uint32_t STRTemplate = 0xf9000000;197 constexpr unsigned DstLocRegIndex = 5;198 constexpr unsigned SrcRegIndex = 0;199 200 uint32_t Instr = STRTemplate;201 Instr |= DstLocReg << DstLocRegIndex;202 Instr |= SrcReg << SrcRegIndex;203 204 return Append(Instr);205}206 207void GOTTableManager::registerExistingEntries() {208 for (auto *EntrySym : GOTSection->symbols()) {209 assert(EntrySym->getBlock().edges_size() == 1 &&210 "GOT block edge count != 1");211 registerPreExistingEntry(EntrySym->getBlock().edges().begin()->getTarget(),212 *EntrySym);213 }214}215 216void PLTTableManager::registerExistingEntries() {217 for (auto *EntrySym : StubsSection->symbols()) {218 assert(EntrySym->getBlock().edges_size() == 2 &&219 "PLT block edge count != 2");220 auto &GOTSym = EntrySym->getBlock().edges().begin()->getTarget();221 assert(GOTSym.getBlock().edges_size() == 1 && "GOT block edge count != 1");222 registerPreExistingEntry(GOTSym.getBlock().edges().begin()->getTarget(),223 *EntrySym);224 }225}226 227const char *getPointerSigningFunctionSectionName() { return "$__ptrauth_sign"; }228 229/// Creates a pointer signing function section, block, and symbol to reserve230/// space for a signing function for this LinkGraph. Clients should insert this231/// pass in the post-prune phase, and add the paired232/// lowerPointer64AuthEdgesToSigningFunction pass to the pre-fixup phase.233Error createEmptyPointerSigningFunction(LinkGraph &G) {234 LLVM_DEBUG({235 dbgs() << "Creating empty pointer signing function for " << G.getName()236 << "\n";237 });238 239 // FIXME: We could put a tighter bound on this if we inspected the ptrauth240 // info encoded in the addend -- the only actually unknown quantity is the241 // fixup location, and we can probably put constraints even on that.242 size_t NumPtrAuthFixupLocations = 0;243 for (auto &Sec : G.sections()) {244 245 // No-alloc sections can't have ptrauth edges. We don't need to error out246 // here: applyFixup will catch these edges if any make it to the fixup247 // stage.248 if (Sec.getMemLifetime() == orc::MemLifetime::NoAlloc)249 continue;250 251 for (auto *B : Sec.blocks()) {252 for (auto &E : B->edges())253 NumPtrAuthFixupLocations +=254 E.getKind() == aarch64::Pointer64Authenticated;255 }256 }257 258 constexpr size_t MaxPtrSignSeqLength =259 4 + // To materialize the value to sign.260 4 + // To materialize the fixup location.261 3 + // To copy, blend discriminator, and sign262 1; // To store the result.263 264 // The maximum number of signing instructions required is the maximum per265 // location, times the number of locations, plus three instructions to266 // materialize the return value and return.267 size_t NumSigningInstrs = NumPtrAuthFixupLocations * MaxPtrSignSeqLength + 3;268 269 // Create signing function section.270 auto &SigningSection =271 G.createSection(getPointerSigningFunctionSectionName(),272 orc::MemProt::Read | orc::MemProt::Exec);273 SigningSection.setMemLifetime(orc::MemLifetime::Finalize);274 275 size_t SigningFunctionSize = NumSigningInstrs * 4;276 auto &SigningFunctionBlock = G.createMutableContentBlock(277 SigningSection, G.allocateBuffer(SigningFunctionSize),278 orc::ExecutorAddr(), 4, 0);279 G.addAnonymousSymbol(SigningFunctionBlock, 0, SigningFunctionBlock.getSize(),280 true, true);281 282 LLVM_DEBUG({283 dbgs() << " " << NumPtrAuthFixupLocations << " location(s) to sign, up to "284 << NumSigningInstrs << " instructions required ("285 << formatv("{0:x}", SigningFunctionBlock.getSize()) << " bytes)\n";286 });287 288 return Error::success();289}290 291/// Given a LinkGraph containing Pointer64Auth edges, transform those edges to292/// Pointer64 and add code to sign the pointers in the executor.293///294/// This function will add a $__ptrauth_sign section with finalization-lifetime295/// containing an anonymous function that will sign all pointers in the graph.296/// An allocation action will be added to run this function during finalization.297Error lowerPointer64AuthEdgesToSigningFunction(LinkGraph &G) {298 LLVM_DEBUG({299 dbgs() << "Writing pointer signing function for " << G.getName() << "\n";300 });301 302 constexpr unsigned Reg1 = 8; // Holds pointer value to sign.303 constexpr unsigned Reg2 = 9; // Holds fixup address.304 constexpr unsigned Reg3 = 10; // Temporary for discriminator value if needed.305 306 // Find the signing function.307 auto *SigningSection =308 G.findSectionByName(getPointerSigningFunctionSectionName());309 assert(SigningSection && "Siging section missing");310 assert(SigningSection->blocks_size() == 1 &&311 "Unexpected number of blocks in signing section");312 assert(SigningSection->symbols_size() == 1 &&313 "Unexpected number of symbols in signing section");314 315 auto &SigningFunctionSym = **SigningSection->symbols().begin();316 auto &SigningFunctionBlock = SigningFunctionSym.getBlock();317 auto SigningFunctionBuf = SigningFunctionBlock.getAlreadyMutableContent();318 319 // Write the instructions to the block content.320 BinaryStreamWriter InstrWriter(321 {reinterpret_cast<uint8_t *>(SigningFunctionBuf.data()),322 SigningFunctionBuf.size()},323 G.getEndianness());324 325 auto AppendInstr = [&](uint32_t Instr) {326 return InstrWriter.writeInteger(Instr);327 };328 329 for (auto &Sec : G.sections()) {330 331 if (Sec.getMemLifetime() == orc::MemLifetime::NoAlloc)332 continue;333 334 for (auto *B : Sec.blocks()) {335 for (auto &E : B->edges()) {336 // We're only concerned with Pointer64Authenticated edges here.337 if (E.getKind() != aarch64::Pointer64Authenticated)338 continue;339 340 uint64_t EncodedInfo = E.getAddend();341 int32_t RealAddend = (uint32_t)(EncodedInfo & 0xffffffff);342 auto ValueToSign = E.getTarget().getAddress() + RealAddend;343 if (!ValueToSign) {344 LLVM_DEBUG(dbgs() << " " << B->getFixupAddress(E) << " <- null\n");345 E.setAddend(RealAddend);346 E.setKind(aarch64::Pointer64);347 continue;348 }349 350 uint32_t InitialDiscriminator = (EncodedInfo >> 32) & 0xffff;351 bool AddressDiversify = (EncodedInfo >> 48) & 0x1;352 uint32_t Key = (EncodedInfo >> 49) & 0x3;353 uint32_t HighBits = EncodedInfo >> 51;354 355 if (HighBits != 0x1000)356 return make_error<JITLinkError>(357 "Pointer64Auth edge at " +358 formatv("{0:x}", B->getFixupAddress(E).getValue()) +359 " has invalid encoded addend " + formatv("{0:x}", EncodedInfo));360 361 LLVM_DEBUG({362 const char *const KeyNames[] = {"IA", "IB", "DA", "DB"};363 dbgs() << " " << B->getFixupAddress(E) << " <- " << ValueToSign364 << " : key = " << KeyNames[Key] << ", discriminator = "365 << formatv("{0:x4}", InitialDiscriminator)366 << ", address diversified = "367 << (AddressDiversify ? "yes" : "no") << "\n";368 });369 370 // Materialize pointer value.371 cantFail(372 writeMovRegImm64Seq(AppendInstr, Reg1, ValueToSign.getValue()));373 374 // Materialize fixup pointer.375 cantFail(writeMovRegImm64Seq(AppendInstr, Reg2,376 B->getFixupAddress(E).getValue()));377 378 // Write signing instruction(s).379 cantFail(writePACSignSeq(AppendInstr, Reg1, ValueToSign, Reg2, Reg3,380 Key, InitialDiscriminator, AddressDiversify));381 382 // Store signed pointer.383 cantFail(writeStoreRegSeq(AppendInstr, Reg2, Reg1));384 385 // Replace edge with a keep-alive to preserve dependence info.386 E.setKind(Edge::KeepAlive);387 }388 }389 }390 391 // Write epilogue. x0 = 0, x1 = 1 is an SPS serialized Error::success value.392 constexpr uint32_t RETInstr = 0xd65f03c0;393 cantFail(writeMovRegImm64Seq(AppendInstr, 0, 0)); // mov x0, #0394 cantFail(writeMovRegImm64Seq(AppendInstr, 1, 1)); // mov x1, #1395 cantFail(AppendInstr(RETInstr)); // ret396 397 // Add an allocation action to call the signing function.398 using namespace orc::shared;399 G.allocActions().push_back(400 {cantFail(WrapperFunctionCall::Create<SPSArgList<>>(401 SigningFunctionSym.getAddress())),402 {}});403 404 return Error::success();405}406 407} // namespace aarch64408} // namespace jitlink409} // namespace llvm410