3021 lines · cpp
1//===-- RuntimeDyldELF.cpp - Run-time dynamic linker for MC-JIT -*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Implementation of ELF support for the MC-JIT runtime dynamic linker.10//11//===----------------------------------------------------------------------===//12 13#include "RuntimeDyldELF.h"14#include "Targets/RuntimeDyldELFMips.h"15#include "llvm/ADT/StringRef.h"16#include "llvm/BinaryFormat/ELF.h"17#include "llvm/ExecutionEngine/Orc/SymbolStringPool.h"18#include "llvm/Object/ELFObjectFile.h"19#include "llvm/Object/ObjectFile.h"20#include "llvm/Support/Endian.h"21#include "llvm/Support/MemoryBuffer.h"22#include "llvm/TargetParser/Triple.h"23 24using namespace llvm;25using namespace llvm::object;26using namespace llvm::support::endian;27 28#define DEBUG_TYPE "dyld"29 30static void or32le(void *P, int32_t V) { write32le(P, read32le(P) | V); }31 32static void or32AArch64Imm(void *L, uint64_t Imm) {33 or32le(L, (Imm & 0xFFF) << 10);34}35 36template <class T> static void write(bool isBE, void *P, T V) {37 isBE ? write<T, llvm::endianness::big>(P, V)38 : write<T, llvm::endianness::little>(P, V);39}40 41static void write32AArch64Addr(void *L, uint64_t Imm) {42 uint32_t ImmLo = (Imm & 0x3) << 29;43 uint32_t ImmHi = (Imm & 0x1FFFFC) << 3;44 uint64_t Mask = (0x3 << 29) | (0x1FFFFC << 3);45 write32le(L, (read32le(L) & ~Mask) | ImmLo | ImmHi);46}47 48// Return the bits [Start, End] from Val shifted Start bits.49// For instance, getBits(0xF0, 4, 8) returns 0xF.50static uint64_t getBits(uint64_t Val, int Start, int End) {51 uint64_t Mask = ((uint64_t)1 << (End + 1 - Start)) - 1;52 return (Val >> Start) & Mask;53}54 55namespace {56 57template <class ELFT> class DyldELFObject : public ELFObjectFile<ELFT> {58 LLVM_ELF_IMPORT_TYPES_ELFT(ELFT)59 60 typedef typename ELFT::uint addr_type;61 62 DyldELFObject(ELFObjectFile<ELFT> &&Obj);63 64public:65 static Expected<std::unique_ptr<DyldELFObject>>66 create(MemoryBufferRef Wrapper);67 68 void updateSectionAddress(const SectionRef &Sec, uint64_t Addr);69 70 void updateSymbolAddress(const SymbolRef &SymRef, uint64_t Addr);71 72 // Methods for type inquiry through isa, cast and dyn_cast73 static bool classof(const Binary *v) {74 return (isa<ELFObjectFile<ELFT>>(v) &&75 classof(cast<ELFObjectFile<ELFT>>(v)));76 }77 static bool classof(const ELFObjectFile<ELFT> *v) {78 return v->isDyldType();79 }80};81 82 83 84// The MemoryBuffer passed into this constructor is just a wrapper around the85// actual memory. Ultimately, the Binary parent class will take ownership of86// this MemoryBuffer object but not the underlying memory.87template <class ELFT>88DyldELFObject<ELFT>::DyldELFObject(ELFObjectFile<ELFT> &&Obj)89 : ELFObjectFile<ELFT>(std::move(Obj)) {90 this->isDyldELFObject = true;91}92 93template <class ELFT>94Expected<std::unique_ptr<DyldELFObject<ELFT>>>95DyldELFObject<ELFT>::create(MemoryBufferRef Wrapper) {96 auto Obj = ELFObjectFile<ELFT>::create(Wrapper);97 if (auto E = Obj.takeError())98 return std::move(E);99 std::unique_ptr<DyldELFObject<ELFT>> Ret(100 new DyldELFObject<ELFT>(std::move(*Obj)));101 return std::move(Ret);102}103 104template <class ELFT>105void DyldELFObject<ELFT>::updateSectionAddress(const SectionRef &Sec,106 uint64_t Addr) {107 DataRefImpl ShdrRef = Sec.getRawDataRefImpl();108 Elf_Shdr *shdr =109 const_cast<Elf_Shdr *>(reinterpret_cast<const Elf_Shdr *>(ShdrRef.p));110 111 // This assumes the address passed in matches the target address bitness112 // The template-based type cast handles everything else.113 shdr->sh_addr = static_cast<addr_type>(Addr);114}115 116template <class ELFT>117void DyldELFObject<ELFT>::updateSymbolAddress(const SymbolRef &SymRef,118 uint64_t Addr) {119 120 Elf_Sym *sym = const_cast<Elf_Sym *>(121 ELFObjectFile<ELFT>::getSymbol(SymRef.getRawDataRefImpl()));122 123 // This assumes the address passed in matches the target address bitness124 // The template-based type cast handles everything else.125 sym->st_value = static_cast<addr_type>(Addr);126}127 128class LoadedELFObjectInfo final129 : public LoadedObjectInfoHelper<LoadedELFObjectInfo,130 RuntimeDyld::LoadedObjectInfo> {131public:132 LoadedELFObjectInfo(RuntimeDyldImpl &RTDyld, ObjSectionToIDMap ObjSecToIDMap)133 : LoadedObjectInfoHelper(RTDyld, std::move(ObjSecToIDMap)) {}134 135 OwningBinary<ObjectFile>136 getObjectForDebug(const ObjectFile &Obj) const override;137};138 139template <typename ELFT>140static Expected<std::unique_ptr<DyldELFObject<ELFT>>>141createRTDyldELFObject(MemoryBufferRef Buffer, const ObjectFile &SourceObject,142 const LoadedELFObjectInfo &L) {143 typedef typename ELFT::Shdr Elf_Shdr;144 typedef typename ELFT::uint addr_type;145 146 Expected<std::unique_ptr<DyldELFObject<ELFT>>> ObjOrErr =147 DyldELFObject<ELFT>::create(Buffer);148 if (Error E = ObjOrErr.takeError())149 return std::move(E);150 151 std::unique_ptr<DyldELFObject<ELFT>> Obj = std::move(*ObjOrErr);152 153 // Iterate over all sections in the object.154 auto SI = SourceObject.section_begin();155 for (const auto &Sec : Obj->sections()) {156 Expected<StringRef> NameOrErr = Sec.getName();157 if (!NameOrErr) {158 consumeError(NameOrErr.takeError());159 continue;160 }161 162 if (*NameOrErr != "") {163 DataRefImpl ShdrRef = Sec.getRawDataRefImpl();164 Elf_Shdr *shdr = const_cast<Elf_Shdr *>(165 reinterpret_cast<const Elf_Shdr *>(ShdrRef.p));166 167 if (uint64_t SecLoadAddr = L.getSectionLoadAddress(*SI)) {168 // This assumes that the address passed in matches the target address169 // bitness. The template-based type cast handles everything else.170 shdr->sh_addr = static_cast<addr_type>(SecLoadAddr);171 }172 }173 ++SI;174 }175 176 return std::move(Obj);177}178 179static OwningBinary<ObjectFile>180createELFDebugObject(const ObjectFile &Obj, const LoadedELFObjectInfo &L) {181 assert(Obj.isELF() && "Not an ELF object file.");182 183 std::unique_ptr<MemoryBuffer> Buffer =184 MemoryBuffer::getMemBufferCopy(Obj.getData(), Obj.getFileName());185 186 Expected<std::unique_ptr<ObjectFile>> DebugObj(nullptr);187 handleAllErrors(DebugObj.takeError());188 if (Obj.getBytesInAddress() == 4 && Obj.isLittleEndian())189 DebugObj =190 createRTDyldELFObject<ELF32LE>(Buffer->getMemBufferRef(), Obj, L);191 else if (Obj.getBytesInAddress() == 4 && !Obj.isLittleEndian())192 DebugObj =193 createRTDyldELFObject<ELF32BE>(Buffer->getMemBufferRef(), Obj, L);194 else if (Obj.getBytesInAddress() == 8 && !Obj.isLittleEndian())195 DebugObj =196 createRTDyldELFObject<ELF64BE>(Buffer->getMemBufferRef(), Obj, L);197 else if (Obj.getBytesInAddress() == 8 && Obj.isLittleEndian())198 DebugObj =199 createRTDyldELFObject<ELF64LE>(Buffer->getMemBufferRef(), Obj, L);200 else201 llvm_unreachable("Unexpected ELF format");202 203 handleAllErrors(DebugObj.takeError());204 return OwningBinary<ObjectFile>(std::move(*DebugObj), std::move(Buffer));205}206 207OwningBinary<ObjectFile>208LoadedELFObjectInfo::getObjectForDebug(const ObjectFile &Obj) const {209 return createELFDebugObject(Obj, *this);210}211 212} // anonymous namespace213 214namespace llvm {215 216RuntimeDyldELF::RuntimeDyldELF(RuntimeDyld::MemoryManager &MemMgr,217 JITSymbolResolver &Resolver)218 : RuntimeDyldImpl(MemMgr, Resolver), GOTSectionID(0), CurrentGOTIndex(0) {}219RuntimeDyldELF::~RuntimeDyldELF() = default;220 221void RuntimeDyldELF::registerEHFrames() {222 for (SID EHFrameSID : UnregisteredEHFrameSections) {223 uint8_t *EHFrameAddr = Sections[EHFrameSID].getAddress();224 uint64_t EHFrameLoadAddr = Sections[EHFrameSID].getLoadAddress();225 size_t EHFrameSize = Sections[EHFrameSID].getSize();226 MemMgr.registerEHFrames(EHFrameAddr, EHFrameLoadAddr, EHFrameSize);227 }228 UnregisteredEHFrameSections.clear();229}230 231std::unique_ptr<RuntimeDyldELF>232llvm::RuntimeDyldELF::create(Triple::ArchType Arch,233 RuntimeDyld::MemoryManager &MemMgr,234 JITSymbolResolver &Resolver) {235 switch (Arch) {236 default:237 return std::make_unique<RuntimeDyldELF>(MemMgr, Resolver);238 case Triple::mips:239 case Triple::mipsel:240 case Triple::mips64:241 case Triple::mips64el:242 return std::make_unique<RuntimeDyldELFMips>(MemMgr, Resolver);243 }244}245 246std::unique_ptr<RuntimeDyld::LoadedObjectInfo>247RuntimeDyldELF::loadObject(const object::ObjectFile &O) {248 if (auto ObjSectionToIDOrErr = loadObjectImpl(O))249 return std::make_unique<LoadedELFObjectInfo>(*this, *ObjSectionToIDOrErr);250 else {251 HasError = true;252 raw_string_ostream ErrStream(ErrorStr);253 logAllUnhandledErrors(ObjSectionToIDOrErr.takeError(), ErrStream);254 return nullptr;255 }256}257 258void RuntimeDyldELF::resolveX86_64Relocation(const SectionEntry &Section,259 uint64_t Offset, uint64_t Value,260 uint32_t Type, int64_t Addend,261 uint64_t SymOffset) {262 switch (Type) {263 default:264 report_fatal_error("Relocation type not implemented yet!");265 break;266 case ELF::R_X86_64_NONE:267 break;268 case ELF::R_X86_64_8: {269 Value += Addend;270 assert((int64_t)Value <= INT8_MAX && (int64_t)Value >= INT8_MIN);271 uint8_t TruncatedAddr = (Value & 0xFF);272 *Section.getAddressWithOffset(Offset) = TruncatedAddr;273 LLVM_DEBUG(dbgs() << "Writing " << format("%p", TruncatedAddr) << " at "274 << format("%p\n", Section.getAddressWithOffset(Offset)));275 break;276 }277 case ELF::R_X86_64_16: {278 Value += Addend;279 assert((int64_t)Value <= INT16_MAX && (int64_t)Value >= INT16_MIN);280 uint16_t TruncatedAddr = (Value & 0xFFFF);281 support::ulittle16_t::ref(Section.getAddressWithOffset(Offset)) =282 TruncatedAddr;283 LLVM_DEBUG(dbgs() << "Writing " << format("%p", TruncatedAddr) << " at "284 << format("%p\n", Section.getAddressWithOffset(Offset)));285 break;286 }287 case ELF::R_X86_64_64: {288 support::ulittle64_t::ref(Section.getAddressWithOffset(Offset)) =289 Value + Addend;290 LLVM_DEBUG(dbgs() << "Writing " << format("%p", (Value + Addend)) << " at "291 << format("%p\n", Section.getAddressWithOffset(Offset)));292 break;293 }294 case ELF::R_X86_64_32:295 case ELF::R_X86_64_32S: {296 Value += Addend;297 assert((Type == ELF::R_X86_64_32 && (Value <= UINT32_MAX)) ||298 (Type == ELF::R_X86_64_32S &&299 ((int64_t)Value <= INT32_MAX && (int64_t)Value >= INT32_MIN)));300 uint32_t TruncatedAddr = (Value & 0xFFFFFFFF);301 support::ulittle32_t::ref(Section.getAddressWithOffset(Offset)) =302 TruncatedAddr;303 LLVM_DEBUG(dbgs() << "Writing " << format("%p", TruncatedAddr) << " at "304 << format("%p\n", Section.getAddressWithOffset(Offset)));305 break;306 }307 case ELF::R_X86_64_PC8: {308 uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset);309 int64_t RealOffset = Value + Addend - FinalAddress;310 assert(isInt<8>(RealOffset));311 int8_t TruncOffset = (RealOffset & 0xFF);312 Section.getAddress()[Offset] = TruncOffset;313 break;314 }315 case ELF::R_X86_64_PC32: {316 uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset);317 int64_t RealOffset = Value + Addend - FinalAddress;318 assert(isInt<32>(RealOffset));319 int32_t TruncOffset = (RealOffset & 0xFFFFFFFF);320 support::ulittle32_t::ref(Section.getAddressWithOffset(Offset)) =321 TruncOffset;322 break;323 }324 case ELF::R_X86_64_PC64: {325 uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset);326 int64_t RealOffset = Value + Addend - FinalAddress;327 support::ulittle64_t::ref(Section.getAddressWithOffset(Offset)) =328 RealOffset;329 LLVM_DEBUG(dbgs() << "Writing " << format("%p", RealOffset) << " at "330 << format("%p\n", FinalAddress));331 break;332 }333 case ELF::R_X86_64_GOTOFF64: {334 // Compute Value - GOTBase.335 uint64_t GOTBase = 0;336 for (const auto &Section : Sections) {337 if (Section.getName() == ".got") {338 GOTBase = Section.getLoadAddressWithOffset(0);339 break;340 }341 }342 assert(GOTBase != 0 && "missing GOT");343 int64_t GOTOffset = Value - GOTBase + Addend;344 support::ulittle64_t::ref(Section.getAddressWithOffset(Offset)) = GOTOffset;345 break;346 }347 case ELF::R_X86_64_DTPMOD64: {348 // We only have one DSO, so the module id is always 1.349 support::ulittle64_t::ref(Section.getAddressWithOffset(Offset)) = 1;350 break;351 }352 case ELF::R_X86_64_DTPOFF64:353 case ELF::R_X86_64_TPOFF64: {354 // DTPOFF64 should resolve to the offset in the TLS block, TPOFF64 to the355 // offset in the *initial* TLS block. Since we are statically linking, all356 // TLS blocks already exist in the initial block, so resolve both357 // relocations equally.358 support::ulittle64_t::ref(Section.getAddressWithOffset(Offset)) =359 Value + Addend;360 break;361 }362 case ELF::R_X86_64_DTPOFF32:363 case ELF::R_X86_64_TPOFF32: {364 // As for the (D)TPOFF64 relocations above, both DTPOFF32 and TPOFF32 can365 // be resolved equally.366 int64_t RealValue = Value + Addend;367 assert(RealValue >= INT32_MIN && RealValue <= INT32_MAX);368 int32_t TruncValue = RealValue;369 support::ulittle32_t::ref(Section.getAddressWithOffset(Offset)) =370 TruncValue;371 break;372 }373 }374}375 376void RuntimeDyldELF::resolveX86Relocation(const SectionEntry &Section,377 uint64_t Offset, uint32_t Value,378 uint32_t Type, int32_t Addend) {379 switch (Type) {380 case ELF::R_386_32: {381 support::ulittle32_t::ref(Section.getAddressWithOffset(Offset)) =382 Value + Addend;383 break;384 }385 // Handle R_386_PLT32 like R_386_PC32 since it should be able to386 // reach any 32 bit address.387 case ELF::R_386_PLT32:388 case ELF::R_386_PC32: {389 uint32_t FinalAddress =390 Section.getLoadAddressWithOffset(Offset) & 0xFFFFFFFF;391 uint32_t RealOffset = Value + Addend - FinalAddress;392 support::ulittle32_t::ref(Section.getAddressWithOffset(Offset)) =393 RealOffset;394 break;395 }396 default:397 // There are other relocation types, but it appears these are the398 // only ones currently used by the LLVM ELF object writer399 report_fatal_error("Relocation type not implemented yet!");400 break;401 }402}403 404void RuntimeDyldELF::resolveAArch64Relocation(const SectionEntry &Section,405 uint64_t Offset, uint64_t Value,406 uint32_t Type, int64_t Addend) {407 uint32_t *TargetPtr =408 reinterpret_cast<uint32_t *>(Section.getAddressWithOffset(Offset));409 uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset);410 // Data should use target endian. Code should always use little endian.411 bool isBE = Arch == Triple::aarch64_be;412 413 LLVM_DEBUG(dbgs() << "resolveAArch64Relocation, LocalAddress: 0x"414 << format("%llx", Section.getAddressWithOffset(Offset))415 << " FinalAddress: 0x" << format("%llx", FinalAddress)416 << " Value: 0x" << format("%llx", Value) << " Type: 0x"417 << format("%x", Type) << " Addend: 0x"418 << format("%llx", Addend) << "\n");419 420 switch (Type) {421 default:422 report_fatal_error("Relocation type not implemented yet!");423 break;424 case ELF::R_AARCH64_NONE:425 break;426 case ELF::R_AARCH64_ABS16: {427 uint64_t Result = Value + Addend;428 assert(Result == static_cast<uint64_t>(llvm::SignExtend64(Result, 16)) ||429 (Result >> 16) == 0);430 write(isBE, TargetPtr, static_cast<uint16_t>(Result & 0xffffU));431 break;432 }433 case ELF::R_AARCH64_ABS32: {434 uint64_t Result = Value + Addend;435 assert(Result == static_cast<uint64_t>(llvm::SignExtend64(Result, 32)) ||436 (Result >> 32) == 0);437 write(isBE, TargetPtr, static_cast<uint32_t>(Result & 0xffffffffU));438 break;439 }440 case ELF::R_AARCH64_ABS64:441 write(isBE, TargetPtr, Value + Addend);442 break;443 case ELF::R_AARCH64_PLT32: {444 uint64_t Result = Value + Addend - FinalAddress;445 assert(static_cast<int64_t>(Result) >= INT32_MIN &&446 static_cast<int64_t>(Result) <= INT32_MAX);447 write(isBE, TargetPtr, static_cast<uint32_t>(Result));448 break;449 }450 case ELF::R_AARCH64_PREL16: {451 uint64_t Result = Value + Addend - FinalAddress;452 assert(static_cast<int64_t>(Result) >= INT16_MIN &&453 static_cast<int64_t>(Result) <= UINT16_MAX);454 write(isBE, TargetPtr, static_cast<uint16_t>(Result & 0xffffU));455 break;456 }457 case ELF::R_AARCH64_PREL32: {458 uint64_t Result = Value + Addend - FinalAddress;459 assert(static_cast<int64_t>(Result) >= INT32_MIN &&460 static_cast<int64_t>(Result) <= UINT32_MAX);461 write(isBE, TargetPtr, static_cast<uint32_t>(Result & 0xffffffffU));462 break;463 }464 case ELF::R_AARCH64_PREL64:465 write(isBE, TargetPtr, Value + Addend - FinalAddress);466 break;467 case ELF::R_AARCH64_CONDBR19: {468 uint64_t BranchImm = Value + Addend - FinalAddress;469 470 assert(isInt<21>(BranchImm));471 *TargetPtr &= 0xff00001fU;472 // Immediate:20:2 goes in bits 23:5 of Bcc, CBZ, CBNZ473 or32le(TargetPtr, (BranchImm & 0x001FFFFC) << 3);474 break;475 }476 case ELF::R_AARCH64_TSTBR14: {477 uint64_t BranchImm = Value + Addend - FinalAddress;478 479 assert(isInt<16>(BranchImm));480 481 uint32_t RawInstr = *(support::little32_t *)TargetPtr;482 *(support::little32_t *)TargetPtr = RawInstr & 0xfff8001fU;483 484 // Immediate:15:2 goes in bits 18:5 of TBZ, TBNZ485 or32le(TargetPtr, (BranchImm & 0x0000FFFC) << 3);486 break;487 }488 case ELF::R_AARCH64_CALL26: // fallthrough489 case ELF::R_AARCH64_JUMP26: {490 // Operation: S+A-P. Set Call or B immediate value to bits fff_fffc of the491 // calculation.492 uint64_t BranchImm = Value + Addend - FinalAddress;493 494 // "Check that -2^27 <= result < 2^27".495 assert(isInt<28>(BranchImm));496 or32le(TargetPtr, (BranchImm & 0x0FFFFFFC) >> 2);497 break;498 }499 case ELF::R_AARCH64_MOVW_UABS_G3:500 or32le(TargetPtr, ((Value + Addend) & 0xFFFF000000000000) >> 43);501 break;502 case ELF::R_AARCH64_MOVW_UABS_G2_NC:503 or32le(TargetPtr, ((Value + Addend) & 0xFFFF00000000) >> 27);504 break;505 case ELF::R_AARCH64_MOVW_UABS_G1_NC:506 or32le(TargetPtr, ((Value + Addend) & 0xFFFF0000) >> 11);507 break;508 case ELF::R_AARCH64_MOVW_UABS_G0_NC:509 or32le(TargetPtr, ((Value + Addend) & 0xFFFF) << 5);510 break;511 case ELF::R_AARCH64_ADR_PREL_PG_HI21: {512 // Operation: Page(S+A) - Page(P)513 uint64_t Result =514 ((Value + Addend) & ~0xfffULL) - (FinalAddress & ~0xfffULL);515 516 // Check that -2^32 <= X < 2^32517 assert(isInt<33>(Result) && "overflow check failed for relocation");518 519 // Immediate goes in bits 30:29 + 5:23 of ADRP instruction, taken520 // from bits 32:12 of X.521 write32AArch64Addr(TargetPtr, Result >> 12);522 break;523 }524 case ELF::R_AARCH64_ADD_ABS_LO12_NC:525 // Operation: S + A526 // Immediate goes in bits 21:10 of LD/ST instruction, taken527 // from bits 11:0 of X528 or32AArch64Imm(TargetPtr, Value + Addend);529 break;530 case ELF::R_AARCH64_LDST8_ABS_LO12_NC:531 // Operation: S + A532 // Immediate goes in bits 21:10 of LD/ST instruction, taken533 // from bits 11:0 of X534 or32AArch64Imm(TargetPtr, getBits(Value + Addend, 0, 11));535 break;536 case ELF::R_AARCH64_LDST16_ABS_LO12_NC:537 // Operation: S + A538 // Immediate goes in bits 21:10 of LD/ST instruction, taken539 // from bits 11:1 of X540 or32AArch64Imm(TargetPtr, getBits(Value + Addend, 1, 11));541 break;542 case ELF::R_AARCH64_LDST32_ABS_LO12_NC:543 // Operation: S + A544 // Immediate goes in bits 21:10 of LD/ST instruction, taken545 // from bits 11:2 of X546 or32AArch64Imm(TargetPtr, getBits(Value + Addend, 2, 11));547 break;548 case ELF::R_AARCH64_LDST64_ABS_LO12_NC:549 // Operation: S + A550 // Immediate goes in bits 21:10 of LD/ST instruction, taken551 // from bits 11:3 of X552 or32AArch64Imm(TargetPtr, getBits(Value + Addend, 3, 11));553 break;554 case ELF::R_AARCH64_LDST128_ABS_LO12_NC:555 // Operation: S + A556 // Immediate goes in bits 21:10 of LD/ST instruction, taken557 // from bits 11:4 of X558 or32AArch64Imm(TargetPtr, getBits(Value + Addend, 4, 11));559 break;560 case ELF::R_AARCH64_LD_PREL_LO19: {561 // Operation: S + A - P562 uint64_t Result = Value + Addend - FinalAddress;563 564 // "Check that -2^20 <= result < 2^20".565 assert(isInt<21>(Result));566 567 *TargetPtr &= 0xff00001fU;568 // Immediate goes in bits 23:5 of LD imm instruction, taken569 // from bits 20:2 of X570 *TargetPtr |= ((Result & 0xffc) << (5 - 2));571 break;572 }573 case ELF::R_AARCH64_ADR_PREL_LO21: {574 // Operation: S + A - P575 uint64_t Result = Value + Addend - FinalAddress;576 577 // "Check that -2^20 <= result < 2^20".578 assert(isInt<21>(Result));579 580 *TargetPtr &= 0x9f00001fU;581 // Immediate goes in bits 23:5, 30:29 of ADR imm instruction, taken582 // from bits 20:0 of X583 *TargetPtr |= ((Result & 0xffc) << (5 - 2));584 *TargetPtr |= (Result & 0x3) << 29;585 break;586 }587 }588}589 590void RuntimeDyldELF::resolveARMRelocation(const SectionEntry &Section,591 uint64_t Offset, uint32_t Value,592 uint32_t Type, int32_t Addend) {593 // TODO: Add Thumb relocations.594 uint32_t *TargetPtr =595 reinterpret_cast<uint32_t *>(Section.getAddressWithOffset(Offset));596 uint32_t FinalAddress = Section.getLoadAddressWithOffset(Offset) & 0xFFFFFFFF;597 Value += Addend;598 599 LLVM_DEBUG(dbgs() << "resolveARMRelocation, LocalAddress: "600 << Section.getAddressWithOffset(Offset)601 << " FinalAddress: " << format("%p", FinalAddress)602 << " Value: " << format("%x", Value)603 << " Type: " << format("%x", Type)604 << " Addend: " << format("%x", Addend) << "\n");605 606 switch (Type) {607 default:608 llvm_unreachable("Not implemented relocation type!");609 610 case ELF::R_ARM_NONE:611 break;612 // Write a 31bit signed offset613 case ELF::R_ARM_PREL31:614 support::ulittle32_t::ref{TargetPtr} =615 (support::ulittle32_t::ref{TargetPtr} & 0x80000000) |616 ((Value - FinalAddress) & ~0x80000000);617 break;618 case ELF::R_ARM_TARGET1:619 case ELF::R_ARM_ABS32:620 support::ulittle32_t::ref{TargetPtr} = Value;621 break;622 // Write first 16 bit of 32 bit value to the mov instruction.623 // Last 4 bit should be shifted.624 case ELF::R_ARM_MOVW_ABS_NC:625 case ELF::R_ARM_MOVT_ABS:626 if (Type == ELF::R_ARM_MOVW_ABS_NC)627 Value = Value & 0xFFFF;628 else if (Type == ELF::R_ARM_MOVT_ABS)629 Value = (Value >> 16) & 0xFFFF;630 support::ulittle32_t::ref{TargetPtr} =631 (support::ulittle32_t::ref{TargetPtr} & ~0x000F0FFF) | (Value & 0xFFF) |632 (((Value >> 12) & 0xF) << 16);633 break;634 // Write 24 bit relative value to the branch instruction.635 case ELF::R_ARM_PC24: // Fall through.636 case ELF::R_ARM_CALL: // Fall through.637 case ELF::R_ARM_JUMP24:638 int32_t RelValue = static_cast<int32_t>(Value - FinalAddress - 8);639 RelValue = (RelValue & 0x03FFFFFC) >> 2;640 assert((support::ulittle32_t::ref{TargetPtr} & 0xFFFFFF) == 0xFFFFFE);641 support::ulittle32_t::ref{TargetPtr} =642 (support::ulittle32_t::ref{TargetPtr} & 0xFF000000) | RelValue;643 break;644 }645}646 647bool RuntimeDyldELF::resolveLoongArch64ShortBranch(648 unsigned SectionID, relocation_iterator RelI,649 const RelocationValueRef &Value) {650 uint64_t Address;651 if (Value.SymbolName) {652 auto Loc = GlobalSymbolTable.find(Value.SymbolName);653 // Don't create direct branch for external symbols.654 if (Loc == GlobalSymbolTable.end())655 return false;656 const auto &SymInfo = Loc->second;657 Address = Sections[SymInfo.getSectionID()].getLoadAddressWithOffset(658 SymInfo.getOffset());659 } else {660 Address = Sections[Value.SectionID].getLoadAddress();661 }662 uint64_t Offset = RelI->getOffset();663 uint64_t SourceAddress = Sections[SectionID].getLoadAddressWithOffset(Offset);664 uint64_t Delta = Address + Value.Addend - SourceAddress;665 // Normal call666 if (RelI->getType() == ELF::R_LARCH_B26) {667 if (!isInt<28>(Delta))668 return false;669 resolveRelocation(Sections[SectionID], Offset, Address, RelI->getType(),670 Value.Addend);671 return true;672 }673 // Medium call: R_LARCH_CALL36674 // Range: [-128G - 0x20000, +128G - 0x20000)675 if (((int64_t)Delta + 0x20000) != llvm::SignExtend64(Delta + 0x20000, 38))676 return false;677 resolveRelocation(Sections[SectionID], Offset, Address, RelI->getType(),678 Value.Addend);679 return true;680}681 682void RuntimeDyldELF::resolveLoongArch64Branch(unsigned SectionID,683 const RelocationValueRef &Value,684 relocation_iterator RelI,685 StubMap &Stubs) {686 LLVM_DEBUG(dbgs() << "\t\tThis is an LoongArch64 branch relocation.\n");687 688 if (resolveLoongArch64ShortBranch(SectionID, RelI, Value))689 return;690 691 SectionEntry &Section = Sections[SectionID];692 uint64_t Offset = RelI->getOffset();693 unsigned RelType = RelI->getType();694 // Look for an existing stub.695 auto [It, Inserted] = Stubs.try_emplace(Value);696 if (!Inserted) {697 resolveRelocation(Section, Offset,698 (uint64_t)Section.getAddressWithOffset(It->second),699 RelType, 0);700 LLVM_DEBUG(dbgs() << " Stub function found\n");701 return;702 }703 // Create a new stub function.704 LLVM_DEBUG(dbgs() << " Create a new stub function\n");705 It->second = Section.getStubOffset();706 uint8_t *StubTargetAddr =707 createStubFunction(Section.getAddressWithOffset(Section.getStubOffset()));708 RelocationEntry LU12I_W(SectionID, StubTargetAddr - Section.getAddress(),709 ELF::R_LARCH_ABS_HI20, Value.Addend);710 RelocationEntry ORI(SectionID, StubTargetAddr - Section.getAddress() + 4,711 ELF::R_LARCH_ABS_LO12, Value.Addend);712 RelocationEntry LU32I_D(SectionID, StubTargetAddr - Section.getAddress() + 8,713 ELF::R_LARCH_ABS64_LO20, Value.Addend);714 RelocationEntry LU52I_D(SectionID, StubTargetAddr - Section.getAddress() + 12,715 ELF::R_LARCH_ABS64_HI12, Value.Addend);716 if (Value.SymbolName) {717 addRelocationForSymbol(LU12I_W, Value.SymbolName);718 addRelocationForSymbol(ORI, Value.SymbolName);719 addRelocationForSymbol(LU32I_D, Value.SymbolName);720 addRelocationForSymbol(LU52I_D, Value.SymbolName);721 } else {722 addRelocationForSection(LU12I_W, Value.SectionID);723 addRelocationForSection(ORI, Value.SectionID);724 addRelocationForSection(LU32I_D, Value.SectionID);725 726 addRelocationForSection(LU52I_D, Value.SectionID);727 }728 resolveRelocation(Section, Offset,729 reinterpret_cast<uint64_t>(730 Section.getAddressWithOffset(Section.getStubOffset())),731 RelType, 0);732 Section.advanceStubOffset(getMaxStubSize());733}734 735// Returns extract bits Val[Hi:Lo].736static inline uint32_t extractBits(uint64_t Val, uint32_t Hi, uint32_t Lo) {737 return Hi == 63 ? Val >> Lo : (Val & (((1ULL << (Hi + 1)) - 1))) >> Lo;738}739 740// Calculate the adjusted page delta between dest and PC. The code is copied741// from lld and see comments there for more details.742static uint64_t getLoongArchPageDelta(uint64_t dest, uint64_t pc,743 uint32_t type) {744 uint64_t pcalau12i_pc;745 switch (type) {746 case ELF::R_LARCH_PCALA64_LO20:747 case ELF::R_LARCH_GOT64_PC_LO20:748 pcalau12i_pc = pc - 8;749 break;750 case ELF::R_LARCH_PCALA64_HI12:751 case ELF::R_LARCH_GOT64_PC_HI12:752 pcalau12i_pc = pc - 12;753 break;754 default:755 pcalau12i_pc = pc;756 break;757 }758 uint64_t result = (dest & ~0xfffULL) - (pcalau12i_pc & ~0xfffULL);759 if (dest & 0x800)760 result += 0x1000 - 0x1'0000'0000;761 if (result & 0x8000'0000)762 result += 0x1'0000'0000;763 return result;764}765 766void RuntimeDyldELF::resolveLoongArch64Relocation(const SectionEntry &Section,767 uint64_t Offset,768 uint64_t Value, uint32_t Type,769 int64_t Addend) {770 auto *TargetPtr = Section.getAddressWithOffset(Offset);771 uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset);772 773 LLVM_DEBUG(dbgs() << "resolveLoongArch64Relocation, LocalAddress: 0x"774 << format("%llx", Section.getAddressWithOffset(Offset))775 << " FinalAddress: 0x" << format("%llx", FinalAddress)776 << " Value: 0x" << format("%llx", Value) << " Type: 0x"777 << format("%x", Type) << " Addend: 0x"778 << format("%llx", Addend) << "\n");779 780 switch (Type) {781 default:782 report_fatal_error("Relocation type not implemented yet!");783 break;784 case ELF::R_LARCH_MARK_LA:785 // ignore786 break;787 case ELF::R_LARCH_32:788 support::ulittle32_t::ref{TargetPtr} =789 static_cast<uint32_t>(Value + Addend);790 break;791 case ELF::R_LARCH_64:792 support::ulittle64_t::ref{TargetPtr} = Value + Addend;793 break;794 case ELF::R_LARCH_32_PCREL:795 support::ulittle32_t::ref{TargetPtr} =796 static_cast<uint32_t>(Value + Addend - FinalAddress);797 break;798 case ELF::R_LARCH_B26: {799 uint64_t B26 = (Value + Addend - FinalAddress) >> 2;800 auto Instr = support::ulittle32_t::ref(TargetPtr);801 uint32_t Imm15_0 = extractBits(B26, /*Hi=*/15, /*Lo=*/0) << 10;802 uint32_t Imm25_16 = extractBits(B26, /*Hi=*/25, /*Lo=*/16);803 Instr = (Instr & 0xfc000000) | Imm15_0 | Imm25_16;804 break;805 }806 case ELF::R_LARCH_CALL36: {807 uint64_t Call36 = (Value + Addend - FinalAddress) >> 2;808 auto Pcaddu18i = support::ulittle32_t::ref(TargetPtr);809 uint32_t Imm35_16 =810 extractBits((Call36 + (1UL << 15)), /*Hi=*/35, /*Lo=*/16) << 5;811 Pcaddu18i = (Pcaddu18i & 0xfe00001f) | Imm35_16;812 auto Jirl = support::ulittle32_t::ref(TargetPtr + 4);813 uint32_t Imm15_0 = extractBits(Call36, /*Hi=*/15, /*Lo=*/0) << 10;814 Jirl = (Jirl & 0xfc0003ff) | Imm15_0;815 break;816 }817 case ELF::R_LARCH_GOT_PC_HI20:818 case ELF::R_LARCH_PCALA_HI20: {819 uint64_t Target = Value + Addend;820 int64_t PageDelta = getLoongArchPageDelta(Target, FinalAddress, Type);821 auto Instr = support::ulittle32_t::ref(TargetPtr);822 uint32_t Imm31_12 = extractBits(PageDelta, /*Hi=*/31, /*Lo=*/12) << 5;823 Instr = (Instr & 0xfe00001f) | Imm31_12;824 break;825 }826 case ELF::R_LARCH_GOT_PC_LO12:827 case ELF::R_LARCH_PCALA_LO12: {828 uint64_t TargetOffset = (Value + Addend) & 0xfff;829 auto Instr = support::ulittle32_t::ref(TargetPtr);830 uint32_t Imm11_0 = TargetOffset << 10;831 Instr = (Instr & 0xffc003ff) | Imm11_0;832 break;833 }834 case ELF::R_LARCH_GOT64_PC_LO20:835 case ELF::R_LARCH_PCALA64_LO20: {836 uint64_t Target = Value + Addend;837 int64_t PageDelta = getLoongArchPageDelta(Target, FinalAddress, Type);838 auto Instr = support::ulittle32_t::ref(TargetPtr);839 uint32_t Imm51_32 = extractBits(PageDelta, /*Hi=*/51, /*Lo=*/32) << 5;840 Instr = (Instr & 0xfe00001f) | Imm51_32;841 break;842 }843 case ELF::R_LARCH_GOT64_PC_HI12:844 case ELF::R_LARCH_PCALA64_HI12: {845 uint64_t Target = Value + Addend;846 int64_t PageDelta = getLoongArchPageDelta(Target, FinalAddress, Type);847 auto Instr = support::ulittle32_t::ref(TargetPtr);848 uint32_t Imm63_52 = extractBits(PageDelta, /*Hi=*/63, /*Lo=*/52) << 10;849 Instr = (Instr & 0xffc003ff) | Imm63_52;850 break;851 }852 case ELF::R_LARCH_ABS_HI20: {853 uint64_t Target = Value + Addend;854 auto Instr = support::ulittle32_t::ref(TargetPtr);855 uint32_t Imm31_12 = extractBits(Target, /*Hi=*/31, /*Lo=*/12) << 5;856 Instr = (Instr & 0xfe00001f) | Imm31_12;857 break;858 }859 case ELF::R_LARCH_ABS_LO12: {860 uint64_t Target = Value + Addend;861 auto Instr = support::ulittle32_t::ref(TargetPtr);862 uint32_t Imm11_0 = extractBits(Target, /*Hi=*/11, /*Lo=*/0) << 10;863 Instr = (Instr & 0xffc003ff) | Imm11_0;864 break;865 }866 case ELF::R_LARCH_ABS64_LO20: {867 uint64_t Target = Value + Addend;868 auto Instr = support::ulittle32_t::ref(TargetPtr);869 uint32_t Imm51_32 = extractBits(Target, /*Hi=*/51, /*Lo=*/32) << 5;870 Instr = (Instr & 0xfe00001f) | Imm51_32;871 break;872 }873 case ELF::R_LARCH_ABS64_HI12: {874 uint64_t Target = Value + Addend;875 auto Instr = support::ulittle32_t::ref(TargetPtr);876 uint32_t Imm63_52 = extractBits(Target, /*Hi=*/63, /*Lo=*/52) << 10;877 Instr = (Instr & 0xffc003ff) | Imm63_52;878 break;879 }880 case ELF::R_LARCH_ADD32:881 support::ulittle32_t::ref{TargetPtr} =882 (support::ulittle32_t::ref{TargetPtr} +883 static_cast<uint32_t>(Value + Addend));884 break;885 case ELF::R_LARCH_SUB32:886 support::ulittle32_t::ref{TargetPtr} =887 (support::ulittle32_t::ref{TargetPtr} -888 static_cast<uint32_t>(Value + Addend));889 break;890 case ELF::R_LARCH_ADD64:891 support::ulittle64_t::ref{TargetPtr} =892 (support::ulittle64_t::ref{TargetPtr} + Value + Addend);893 break;894 case ELF::R_LARCH_SUB64:895 support::ulittle64_t::ref{TargetPtr} =896 (support::ulittle64_t::ref{TargetPtr} - Value - Addend);897 break;898 }899}900 901void RuntimeDyldELF::setMipsABI(const ObjectFile &Obj) {902 if (Arch == Triple::UnknownArch ||903 Triple::getArchTypePrefix(Arch) != "mips") {904 IsMipsO32ABI = false;905 IsMipsN32ABI = false;906 IsMipsN64ABI = false;907 return;908 }909 if (auto *E = dyn_cast<ELFObjectFileBase>(&Obj)) {910 unsigned AbiVariant = E->getPlatformFlags();911 IsMipsO32ABI = AbiVariant & ELF::EF_MIPS_ABI_O32;912 IsMipsN32ABI = AbiVariant & ELF::EF_MIPS_ABI2;913 }914 IsMipsN64ABI = Obj.getFileFormatName() == "elf64-mips";915}916 917// Return the .TOC. section and offset.918Error RuntimeDyldELF::findPPC64TOCSection(const ELFObjectFileBase &Obj,919 ObjSectionToIDMap &LocalSections,920 RelocationValueRef &Rel) {921 // Set a default SectionID in case we do not find a TOC section below.922 // This may happen for references to TOC base base (sym@toc, .odp923 // relocation) without a .toc directive. In this case just use the924 // first section (which is usually the .odp) since the code won't925 // reference the .toc base directly.926 Rel.SymbolName = nullptr;927 Rel.SectionID = 0;928 929 // The TOC consists of sections .got, .toc, .tocbss, .plt in that930 // order. The TOC starts where the first of these sections starts.931 for (auto &Section : Obj.sections()) {932 Expected<StringRef> NameOrErr = Section.getName();933 if (!NameOrErr)934 return NameOrErr.takeError();935 StringRef SectionName = *NameOrErr;936 937 if (SectionName == ".got"938 || SectionName == ".toc"939 || SectionName == ".tocbss"940 || SectionName == ".plt") {941 if (auto SectionIDOrErr =942 findOrEmitSection(Obj, Section, false, LocalSections))943 Rel.SectionID = *SectionIDOrErr;944 else945 return SectionIDOrErr.takeError();946 break;947 }948 }949 950 // Per the ppc64-elf-linux ABI, The TOC base is TOC value plus 0x8000951 // thus permitting a full 64 Kbytes segment.952 Rel.Addend = 0x8000;953 954 return Error::success();955}956 957// Returns the sections and offset associated with the ODP entry referenced958// by Symbol.959Error RuntimeDyldELF::findOPDEntrySection(const ELFObjectFileBase &Obj,960 ObjSectionToIDMap &LocalSections,961 RelocationValueRef &Rel) {962 // Get the ELF symbol value (st_value) to compare with Relocation offset in963 // .opd entries964 for (section_iterator si = Obj.section_begin(), se = Obj.section_end();965 si != se; ++si) {966 967 Expected<section_iterator> RelSecOrErr = si->getRelocatedSection();968 if (!RelSecOrErr)969 report_fatal_error(Twine(toString(RelSecOrErr.takeError())));970 971 section_iterator RelSecI = *RelSecOrErr;972 if (RelSecI == Obj.section_end())973 continue;974 975 Expected<StringRef> NameOrErr = RelSecI->getName();976 if (!NameOrErr)977 return NameOrErr.takeError();978 StringRef RelSectionName = *NameOrErr;979 980 if (RelSectionName != ".opd")981 continue;982 983 for (elf_relocation_iterator i = si->relocation_begin(),984 e = si->relocation_end();985 i != e;) {986 // The R_PPC64_ADDR64 relocation indicates the first field987 // of a .opd entry988 uint64_t TypeFunc = i->getType();989 if (TypeFunc != ELF::R_PPC64_ADDR64) {990 ++i;991 continue;992 }993 994 uint64_t TargetSymbolOffset = i->getOffset();995 symbol_iterator TargetSymbol = i->getSymbol();996 int64_t Addend;997 if (auto AddendOrErr = i->getAddend())998 Addend = *AddendOrErr;999 else1000 return AddendOrErr.takeError();1001 1002 ++i;1003 if (i == e)1004 break;1005 1006 // Just check if following relocation is a R_PPC64_TOC1007 uint64_t TypeTOC = i->getType();1008 if (TypeTOC != ELF::R_PPC64_TOC)1009 continue;1010 1011 // Finally compares the Symbol value and the target symbol offset1012 // to check if this .opd entry refers to the symbol the relocation1013 // points to.1014 if (Rel.Addend != (int64_t)TargetSymbolOffset)1015 continue;1016 1017 section_iterator TSI = Obj.section_end();1018 if (auto TSIOrErr = TargetSymbol->getSection())1019 TSI = *TSIOrErr;1020 else1021 return TSIOrErr.takeError();1022 assert(TSI != Obj.section_end() && "TSI should refer to a valid section");1023 1024 bool IsCode = TSI->isText();1025 if (auto SectionIDOrErr = findOrEmitSection(Obj, *TSI, IsCode,1026 LocalSections))1027 Rel.SectionID = *SectionIDOrErr;1028 else1029 return SectionIDOrErr.takeError();1030 Rel.Addend = (intptr_t)Addend;1031 return Error::success();1032 }1033 }1034 llvm_unreachable("Attempting to get address of ODP entry!");1035}1036 1037// Relocation masks following the #lo(value), #hi(value), #ha(value),1038// #higher(value), #highera(value), #highest(value), and #highesta(value)1039// macros defined in section 4.5.1. Relocation Types of the PPC-elf64abi1040// document.1041 1042static inline uint16_t applyPPClo(uint64_t value) { return value & 0xffff; }1043 1044static inline uint16_t applyPPChi(uint64_t value) {1045 return (value >> 16) & 0xffff;1046}1047 1048static inline uint16_t applyPPCha (uint64_t value) {1049 return ((value + 0x8000) >> 16) & 0xffff;1050}1051 1052static inline uint16_t applyPPChigher(uint64_t value) {1053 return (value >> 32) & 0xffff;1054}1055 1056static inline uint16_t applyPPChighera (uint64_t value) {1057 return ((value + 0x8000) >> 32) & 0xffff;1058}1059 1060static inline uint16_t applyPPChighest(uint64_t value) {1061 return (value >> 48) & 0xffff;1062}1063 1064static inline uint16_t applyPPChighesta (uint64_t value) {1065 return ((value + 0x8000) >> 48) & 0xffff;1066}1067 1068void RuntimeDyldELF::resolvePPC32Relocation(const SectionEntry &Section,1069 uint64_t Offset, uint64_t Value,1070 uint32_t Type, int64_t Addend) {1071 uint8_t *LocalAddress = Section.getAddressWithOffset(Offset);1072 switch (Type) {1073 default:1074 report_fatal_error("Relocation type not implemented yet!");1075 break;1076 case ELF::R_PPC_ADDR16_LO:1077 writeInt16BE(LocalAddress, applyPPClo(Value + Addend));1078 break;1079 case ELF::R_PPC_ADDR16_HI:1080 writeInt16BE(LocalAddress, applyPPChi(Value + Addend));1081 break;1082 case ELF::R_PPC_ADDR16_HA:1083 writeInt16BE(LocalAddress, applyPPCha(Value + Addend));1084 break;1085 }1086}1087 1088void RuntimeDyldELF::resolvePPC64Relocation(const SectionEntry &Section,1089 uint64_t Offset, uint64_t Value,1090 uint32_t Type, int64_t Addend) {1091 uint8_t *LocalAddress = Section.getAddressWithOffset(Offset);1092 switch (Type) {1093 default:1094 report_fatal_error("Relocation type not implemented yet!");1095 break;1096 case ELF::R_PPC64_ADDR16:1097 writeInt16BE(LocalAddress, applyPPClo(Value + Addend));1098 break;1099 case ELF::R_PPC64_ADDR16_DS:1100 writeInt16BE(LocalAddress, applyPPClo(Value + Addend) & ~3);1101 break;1102 case ELF::R_PPC64_ADDR16_LO:1103 writeInt16BE(LocalAddress, applyPPClo(Value + Addend));1104 break;1105 case ELF::R_PPC64_ADDR16_LO_DS:1106 writeInt16BE(LocalAddress, applyPPClo(Value + Addend) & ~3);1107 break;1108 case ELF::R_PPC64_ADDR16_HI:1109 case ELF::R_PPC64_ADDR16_HIGH:1110 writeInt16BE(LocalAddress, applyPPChi(Value + Addend));1111 break;1112 case ELF::R_PPC64_ADDR16_HA:1113 case ELF::R_PPC64_ADDR16_HIGHA:1114 writeInt16BE(LocalAddress, applyPPCha(Value + Addend));1115 break;1116 case ELF::R_PPC64_ADDR16_HIGHER:1117 writeInt16BE(LocalAddress, applyPPChigher(Value + Addend));1118 break;1119 case ELF::R_PPC64_ADDR16_HIGHERA:1120 writeInt16BE(LocalAddress, applyPPChighera(Value + Addend));1121 break;1122 case ELF::R_PPC64_ADDR16_HIGHEST:1123 writeInt16BE(LocalAddress, applyPPChighest(Value + Addend));1124 break;1125 case ELF::R_PPC64_ADDR16_HIGHESTA:1126 writeInt16BE(LocalAddress, applyPPChighesta(Value + Addend));1127 break;1128 case ELF::R_PPC64_ADDR14: {1129 assert(((Value + Addend) & 3) == 0);1130 // Preserve the AA/LK bits in the branch instruction1131 uint8_t aalk = *(LocalAddress + 3);1132 writeInt16BE(LocalAddress + 2, (aalk & 3) | ((Value + Addend) & 0xfffc));1133 } break;1134 case ELF::R_PPC64_REL16_LO: {1135 uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset);1136 uint64_t Delta = Value - FinalAddress + Addend;1137 writeInt16BE(LocalAddress, applyPPClo(Delta));1138 } break;1139 case ELF::R_PPC64_REL16_HI: {1140 uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset);1141 uint64_t Delta = Value - FinalAddress + Addend;1142 writeInt16BE(LocalAddress, applyPPChi(Delta));1143 } break;1144 case ELF::R_PPC64_REL16_HA: {1145 uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset);1146 uint64_t Delta = Value - FinalAddress + Addend;1147 writeInt16BE(LocalAddress, applyPPCha(Delta));1148 } break;1149 case ELF::R_PPC64_ADDR32: {1150 int64_t Result = static_cast<int64_t>(Value + Addend);1151 if (SignExtend64<32>(Result) != Result)1152 llvm_unreachable("Relocation R_PPC64_ADDR32 overflow");1153 writeInt32BE(LocalAddress, Result);1154 } break;1155 case ELF::R_PPC64_REL24: {1156 uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset);1157 int64_t delta = static_cast<int64_t>(Value - FinalAddress + Addend);1158 if (SignExtend64<26>(delta) != delta)1159 llvm_unreachable("Relocation R_PPC64_REL24 overflow");1160 // We preserve bits other than LI field, i.e. PO and AA/LK fields.1161 uint32_t Inst = readBytesUnaligned(LocalAddress, 4);1162 writeInt32BE(LocalAddress, (Inst & 0xFC000003) | (delta & 0x03FFFFFC));1163 } break;1164 case ELF::R_PPC64_REL32: {1165 uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset);1166 int64_t delta = static_cast<int64_t>(Value - FinalAddress + Addend);1167 if (SignExtend64<32>(delta) != delta)1168 llvm_unreachable("Relocation R_PPC64_REL32 overflow");1169 writeInt32BE(LocalAddress, delta);1170 } break;1171 case ELF::R_PPC64_REL64: {1172 uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset);1173 uint64_t Delta = Value - FinalAddress + Addend;1174 writeInt64BE(LocalAddress, Delta);1175 } break;1176 case ELF::R_PPC64_ADDR64:1177 writeInt64BE(LocalAddress, Value + Addend);1178 break;1179 }1180}1181 1182void RuntimeDyldELF::resolveSystemZRelocation(const SectionEntry &Section,1183 uint64_t Offset, uint64_t Value,1184 uint32_t Type, int64_t Addend) {1185 uint8_t *LocalAddress = Section.getAddressWithOffset(Offset);1186 switch (Type) {1187 default:1188 report_fatal_error("Relocation type not implemented yet!");1189 break;1190 case ELF::R_390_PC16DBL:1191 case ELF::R_390_PLT16DBL: {1192 int64_t Delta = (Value + Addend) - Section.getLoadAddressWithOffset(Offset);1193 assert(int16_t(Delta / 2) * 2 == Delta && "R_390_PC16DBL overflow");1194 writeInt16BE(LocalAddress, Delta / 2);1195 break;1196 }1197 case ELF::R_390_PC32DBL:1198 case ELF::R_390_PLT32DBL: {1199 int64_t Delta = (Value + Addend) - Section.getLoadAddressWithOffset(Offset);1200 assert(int32_t(Delta / 2) * 2 == Delta && "R_390_PC32DBL overflow");1201 writeInt32BE(LocalAddress, Delta / 2);1202 break;1203 }1204 case ELF::R_390_PC16: {1205 int64_t Delta = (Value + Addend) - Section.getLoadAddressWithOffset(Offset);1206 assert(int16_t(Delta) == Delta && "R_390_PC16 overflow");1207 writeInt16BE(LocalAddress, Delta);1208 break;1209 }1210 case ELF::R_390_PC32: {1211 int64_t Delta = (Value + Addend) - Section.getLoadAddressWithOffset(Offset);1212 assert(int32_t(Delta) == Delta && "R_390_PC32 overflow");1213 writeInt32BE(LocalAddress, Delta);1214 break;1215 }1216 case ELF::R_390_PC64: {1217 int64_t Delta = (Value + Addend) - Section.getLoadAddressWithOffset(Offset);1218 writeInt64BE(LocalAddress, Delta);1219 break;1220 }1221 case ELF::R_390_8:1222 *LocalAddress = (uint8_t)(Value + Addend);1223 break;1224 case ELF::R_390_16:1225 writeInt16BE(LocalAddress, Value + Addend);1226 break;1227 case ELF::R_390_32:1228 writeInt32BE(LocalAddress, Value + Addend);1229 break;1230 case ELF::R_390_64:1231 writeInt64BE(LocalAddress, Value + Addend);1232 break;1233 }1234}1235 1236void RuntimeDyldELF::resolveBPFRelocation(const SectionEntry &Section,1237 uint64_t Offset, uint64_t Value,1238 uint32_t Type, int64_t Addend) {1239 bool isBE = Arch == Triple::bpfeb;1240 1241 switch (Type) {1242 default:1243 report_fatal_error("Relocation type not implemented yet!");1244 break;1245 case ELF::R_BPF_NONE:1246 case ELF::R_BPF_64_64:1247 case ELF::R_BPF_64_32:1248 case ELF::R_BPF_64_NODYLD32:1249 break;1250 case ELF::R_BPF_64_ABS64: {1251 write(isBE, Section.getAddressWithOffset(Offset), Value + Addend);1252 LLVM_DEBUG(dbgs() << "Writing " << format("%p", (Value + Addend)) << " at "1253 << format("%p\n", Section.getAddressWithOffset(Offset)));1254 break;1255 }1256 case ELF::R_BPF_64_ABS32: {1257 Value += Addend;1258 assert(Value <= UINT32_MAX);1259 write(isBE, Section.getAddressWithOffset(Offset), static_cast<uint32_t>(Value));1260 LLVM_DEBUG(dbgs() << "Writing " << format("%p", Value) << " at "1261 << format("%p\n", Section.getAddressWithOffset(Offset)));1262 break;1263 }1264 }1265}1266 1267static void applyUTypeImmRISCV(uint8_t *InstrAddr, uint32_t Imm) {1268 uint32_t UpperImm = (Imm + 0x800) & 0xfffff000;1269 auto Instr = support::ulittle32_t::ref(InstrAddr);1270 Instr = (Instr & 0xfff) | UpperImm;1271}1272 1273static void applyITypeImmRISCV(uint8_t *InstrAddr, uint32_t Imm) {1274 uint32_t LowerImm = Imm & 0xfff;1275 auto Instr = support::ulittle32_t::ref(InstrAddr);1276 Instr = (Instr & 0xfffff) | (LowerImm << 20);1277}1278 1279void RuntimeDyldELF::resolveRISCVRelocation(const SectionEntry &Section,1280 uint64_t Offset, uint64_t Value,1281 uint32_t Type, int64_t Addend,1282 SID SectionID) {1283 switch (Type) {1284 default: {1285 std::string Err = "Unimplemented reloc type: " + std::to_string(Type);1286 llvm::report_fatal_error(Err.c_str());1287 }1288 // 32-bit PC-relative function call, macros call, tail (PIC)1289 // Write first 20 bits of 32 bit value to the auipc instruction1290 // Last 12 bits to the jalr instruction1291 case ELF::R_RISCV_CALL:1292 case ELF::R_RISCV_CALL_PLT: {1293 uint64_t P = Section.getLoadAddressWithOffset(Offset);1294 uint64_t PCOffset = Value + Addend - P;1295 applyUTypeImmRISCV(Section.getAddressWithOffset(Offset), PCOffset);1296 applyITypeImmRISCV(Section.getAddressWithOffset(Offset + 4), PCOffset);1297 break;1298 }1299 // High 20 bits of 32-bit absolute address, %hi(symbol)1300 case ELF::R_RISCV_HI20: {1301 uint64_t PCOffset = Value + Addend;1302 applyUTypeImmRISCV(Section.getAddressWithOffset(Offset), PCOffset);1303 break;1304 }1305 // Low 12 bits of 32-bit absolute address, %lo(symbol)1306 case ELF::R_RISCV_LO12_I: {1307 uint64_t PCOffset = Value + Addend;1308 applyITypeImmRISCV(Section.getAddressWithOffset(Offset), PCOffset);1309 break;1310 }1311 // High 20 bits of 32-bit PC-relative reference, %pcrel_hi(symbol)1312 case ELF::R_RISCV_GOT_HI20:1313 case ELF::R_RISCV_PCREL_HI20: {1314 uint64_t P = Section.getLoadAddressWithOffset(Offset);1315 uint64_t PCOffset = Value + Addend - P;1316 applyUTypeImmRISCV(Section.getAddressWithOffset(Offset), PCOffset);1317 break;1318 }1319 1320 // label:1321 // auipc a0, %pcrel_hi(symbol) // R_RISCV_PCREL_HI201322 // addi a0, a0, %pcrel_lo(label) // R_RISCV_PCREL_LO12_I1323 //1324 // The low 12 bits of relative address between pc and symbol.1325 // The symbol is related to the high part instruction which is marked by1326 // label.1327 case ELF::R_RISCV_PCREL_LO12_I: {1328 for (auto &&PendingReloc : PendingRelocs) {1329 const RelocationValueRef &MatchingValue = PendingReloc.first;1330 RelocationEntry &Reloc = PendingReloc.second;1331 uint64_t HIRelocPC =1332 getSectionLoadAddress(Reloc.SectionID) + Reloc.Offset;1333 if (Value + Addend == HIRelocPC) {1334 uint64_t Symbol = getSectionLoadAddress(MatchingValue.SectionID) +1335 MatchingValue.Addend;1336 auto PCOffset = Symbol - HIRelocPC;1337 applyITypeImmRISCV(Section.getAddressWithOffset(Offset), PCOffset);1338 return;1339 }1340 }1341 1342 llvm::report_fatal_error(1343 "R_RISCV_PCREL_LO12_I without matching R_RISCV_PCREL_HI20");1344 }1345 case ELF::R_RISCV_32_PCREL: {1346 uint64_t FinalAddress = Section.getLoadAddressWithOffset(Offset);1347 int64_t RealOffset = Value + Addend - FinalAddress;1348 int32_t TruncOffset = Lo_32(RealOffset);1349 support::ulittle32_t::ref(Section.getAddressWithOffset(Offset)) =1350 TruncOffset;1351 break;1352 }1353 case ELF::R_RISCV_32: {1354 auto Ref = support::ulittle32_t::ref(Section.getAddressWithOffset(Offset));1355 Ref = Value + Addend;1356 break;1357 }1358 case ELF::R_RISCV_64: {1359 auto Ref = support::ulittle64_t::ref(Section.getAddressWithOffset(Offset));1360 Ref = Value + Addend;1361 break;1362 }1363 case ELF::R_RISCV_ADD8: {1364 auto Ref = support::ulittle8_t::ref(Section.getAddressWithOffset(Offset));1365 Ref = Ref + Value + Addend;1366 break;1367 }1368 case ELF::R_RISCV_ADD16: {1369 auto Ref = support::ulittle16_t::ref(Section.getAddressWithOffset(Offset));1370 Ref = Ref + Value + Addend;1371 break;1372 }1373 case ELF::R_RISCV_ADD32: {1374 auto Ref = support::ulittle32_t::ref(Section.getAddressWithOffset(Offset));1375 Ref = Ref + Value + Addend;1376 break;1377 }1378 case ELF::R_RISCV_ADD64: {1379 auto Ref = support::ulittle64_t::ref(Section.getAddressWithOffset(Offset));1380 Ref = Ref + Value + Addend;1381 break;1382 }1383 case ELF::R_RISCV_SUB8: {1384 auto Ref = support::ulittle8_t::ref(Section.getAddressWithOffset(Offset));1385 Ref = Ref - Value - Addend;1386 break;1387 }1388 case ELF::R_RISCV_SUB16: {1389 auto Ref = support::ulittle16_t::ref(Section.getAddressWithOffset(Offset));1390 Ref = Ref - Value - Addend;1391 break;1392 }1393 case ELF::R_RISCV_SUB32: {1394 auto Ref = support::ulittle32_t::ref(Section.getAddressWithOffset(Offset));1395 Ref = Ref - Value - Addend;1396 break;1397 }1398 case ELF::R_RISCV_SUB64: {1399 auto Ref = support::ulittle64_t::ref(Section.getAddressWithOffset(Offset));1400 Ref = Ref - Value - Addend;1401 break;1402 }1403 case ELF::R_RISCV_SET8: {1404 auto Ref = support::ulittle8_t::ref(Section.getAddressWithOffset(Offset));1405 Ref = Value + Addend;1406 break;1407 }1408 case ELF::R_RISCV_SET16: {1409 auto Ref = support::ulittle16_t::ref(Section.getAddressWithOffset(Offset));1410 Ref = Value + Addend;1411 break;1412 }1413 case ELF::R_RISCV_SET32: {1414 auto Ref = support::ulittle32_t::ref(Section.getAddressWithOffset(Offset));1415 Ref = Value + Addend;1416 break;1417 }1418 }1419}1420 1421// The target location for the relocation is described by RE.SectionID and1422// RE.Offset. RE.SectionID can be used to find the SectionEntry. Each1423// SectionEntry has three members describing its location.1424// SectionEntry::Address is the address at which the section has been loaded1425// into memory in the current (host) process. SectionEntry::LoadAddress is the1426// address that the section will have in the target process.1427// SectionEntry::ObjAddress is the address of the bits for this section in the1428// original emitted object image (also in the current address space).1429//1430// Relocations will be applied as if the section were loaded at1431// SectionEntry::LoadAddress, but they will be applied at an address based1432// on SectionEntry::Address. SectionEntry::ObjAddress will be used to refer to1433// Target memory contents if they are required for value calculations.1434//1435// The Value parameter here is the load address of the symbol for the1436// relocation to be applied. For relocations which refer to symbols in the1437// current object Value will be the LoadAddress of the section in which1438// the symbol resides (RE.Addend provides additional information about the1439// symbol location). For external symbols, Value will be the address of the1440// symbol in the target address space.1441void RuntimeDyldELF::resolveRelocation(const RelocationEntry &RE,1442 uint64_t Value) {1443 const SectionEntry &Section = Sections[RE.SectionID];1444 return resolveRelocation(Section, RE.Offset, Value, RE.RelType, RE.Addend,1445 RE.SymOffset, RE.SectionID);1446}1447 1448void RuntimeDyldELF::resolveRelocation(const SectionEntry &Section,1449 uint64_t Offset, uint64_t Value,1450 uint32_t Type, int64_t Addend,1451 uint64_t SymOffset, SID SectionID) {1452 switch (Arch) {1453 case Triple::x86_64:1454 resolveX86_64Relocation(Section, Offset, Value, Type, Addend, SymOffset);1455 break;1456 case Triple::x86:1457 resolveX86Relocation(Section, Offset, (uint32_t)(Value & 0xffffffffL), Type,1458 (uint32_t)(Addend & 0xffffffffL));1459 break;1460 case Triple::aarch64:1461 case Triple::aarch64_be:1462 resolveAArch64Relocation(Section, Offset, Value, Type, Addend);1463 break;1464 case Triple::arm: // Fall through.1465 case Triple::armeb:1466 case Triple::thumb:1467 case Triple::thumbeb:1468 resolveARMRelocation(Section, Offset, (uint32_t)(Value & 0xffffffffL), Type,1469 (uint32_t)(Addend & 0xffffffffL));1470 break;1471 case Triple::loongarch64:1472 resolveLoongArch64Relocation(Section, Offset, Value, Type, Addend);1473 break;1474 case Triple::ppc: // Fall through.1475 case Triple::ppcle:1476 resolvePPC32Relocation(Section, Offset, Value, Type, Addend);1477 break;1478 case Triple::ppc64: // Fall through.1479 case Triple::ppc64le:1480 resolvePPC64Relocation(Section, Offset, Value, Type, Addend);1481 break;1482 case Triple::systemz:1483 resolveSystemZRelocation(Section, Offset, Value, Type, Addend);1484 break;1485 case Triple::bpfel:1486 case Triple::bpfeb:1487 resolveBPFRelocation(Section, Offset, Value, Type, Addend);1488 break;1489 case Triple::riscv32: // Fall through.1490 case Triple::riscv64:1491 resolveRISCVRelocation(Section, Offset, Value, Type, Addend, SectionID);1492 break;1493 default:1494 llvm_unreachable("Unsupported CPU type!");1495 }1496}1497 1498void *RuntimeDyldELF::computePlaceholderAddress(unsigned SectionID,1499 uint64_t Offset) const {1500 return (void *)(Sections[SectionID].getObjAddress() + Offset);1501}1502 1503void RuntimeDyldELF::processSimpleRelocation(unsigned SectionID, uint64_t Offset, unsigned RelType, RelocationValueRef Value) {1504 RelocationEntry RE(SectionID, Offset, RelType, Value.Addend, Value.Offset);1505 if (Value.SymbolName)1506 addRelocationForSymbol(RE, Value.SymbolName);1507 else1508 addRelocationForSection(RE, Value.SectionID);1509}1510 1511uint32_t RuntimeDyldELF::getMatchingLoRelocation(uint32_t RelType,1512 bool IsLocal) const {1513 switch (RelType) {1514 case ELF::R_MICROMIPS_GOT16:1515 if (IsLocal)1516 return ELF::R_MICROMIPS_LO16;1517 break;1518 case ELF::R_MICROMIPS_HI16:1519 return ELF::R_MICROMIPS_LO16;1520 case ELF::R_MIPS_GOT16:1521 if (IsLocal)1522 return ELF::R_MIPS_LO16;1523 break;1524 case ELF::R_MIPS_HI16:1525 return ELF::R_MIPS_LO16;1526 case ELF::R_MIPS_PCHI16:1527 return ELF::R_MIPS_PCLO16;1528 default:1529 break;1530 }1531 return ELF::R_MIPS_NONE;1532}1533 1534// Sometimes we don't need to create thunk for a branch.1535// This typically happens when branch target is located1536// in the same object file. In such case target is either1537// a weak symbol or symbol in a different executable section.1538// This function checks if branch target is located in the1539// same object file and if distance between source and target1540// fits R_AARCH64_CALL26 relocation. If both conditions are1541// met, it emits direct jump to the target and returns true.1542// Otherwise false is returned and thunk is created.1543bool RuntimeDyldELF::resolveAArch64ShortBranch(1544 unsigned SectionID, relocation_iterator RelI,1545 const RelocationValueRef &Value) {1546 uint64_t TargetOffset;1547 unsigned TargetSectionID;1548 if (Value.SymbolName) {1549 auto Loc = GlobalSymbolTable.find(Value.SymbolName);1550 1551 // Don't create direct branch for external symbols.1552 if (Loc == GlobalSymbolTable.end())1553 return false;1554 1555 const auto &SymInfo = Loc->second;1556 1557 TargetSectionID = SymInfo.getSectionID();1558 TargetOffset = SymInfo.getOffset();1559 } else {1560 TargetSectionID = Value.SectionID;1561 TargetOffset = 0;1562 }1563 1564 // We don't actually know the load addresses at this point, so if the1565 // branch is cross-section, we don't know exactly how far away it is.1566 if (TargetSectionID != SectionID)1567 return false;1568 1569 uint64_t SourceOffset = RelI->getOffset();1570 1571 // R_AARCH64_CALL26 requires immediate to be in range -2^27 <= imm < 2^271572 // If distance between source and target is out of range then we should1573 // create thunk.1574 if (!isInt<28>(TargetOffset + Value.Addend - SourceOffset))1575 return false;1576 1577 RelocationEntry RE(SectionID, SourceOffset, RelI->getType(), Value.Addend);1578 if (Value.SymbolName)1579 addRelocationForSymbol(RE, Value.SymbolName);1580 else1581 addRelocationForSection(RE, Value.SectionID);1582 1583 return true;1584}1585 1586void RuntimeDyldELF::resolveAArch64Branch(unsigned SectionID,1587 const RelocationValueRef &Value,1588 relocation_iterator RelI,1589 StubMap &Stubs) {1590 1591 LLVM_DEBUG(dbgs() << "\t\tThis is an AArch64 branch relocation.");1592 SectionEntry &Section = Sections[SectionID];1593 1594 uint64_t Offset = RelI->getOffset();1595 unsigned RelType = RelI->getType();1596 // Look for an existing stub.1597 StubMap::const_iterator i = Stubs.find(Value);1598 if (i != Stubs.end()) {1599 resolveRelocation(Section, Offset,1600 Section.getLoadAddressWithOffset(i->second), RelType, 0);1601 LLVM_DEBUG(dbgs() << " Stub function found\n");1602 } else if (!resolveAArch64ShortBranch(SectionID, RelI, Value)) {1603 // Create a new stub function.1604 LLVM_DEBUG(dbgs() << " Create a new stub function\n");1605 Stubs[Value] = Section.getStubOffset();1606 uint8_t *StubTargetAddr = createStubFunction(1607 Section.getAddressWithOffset(Section.getStubOffset()));1608 1609 RelocationEntry REmovz_g3(SectionID, StubTargetAddr - Section.getAddress(),1610 ELF::R_AARCH64_MOVW_UABS_G3, Value.Addend);1611 RelocationEntry REmovk_g2(SectionID,1612 StubTargetAddr - Section.getAddress() + 4,1613 ELF::R_AARCH64_MOVW_UABS_G2_NC, Value.Addend);1614 RelocationEntry REmovk_g1(SectionID,1615 StubTargetAddr - Section.getAddress() + 8,1616 ELF::R_AARCH64_MOVW_UABS_G1_NC, Value.Addend);1617 RelocationEntry REmovk_g0(SectionID,1618 StubTargetAddr - Section.getAddress() + 12,1619 ELF::R_AARCH64_MOVW_UABS_G0_NC, Value.Addend);1620 1621 if (Value.SymbolName) {1622 addRelocationForSymbol(REmovz_g3, Value.SymbolName);1623 addRelocationForSymbol(REmovk_g2, Value.SymbolName);1624 addRelocationForSymbol(REmovk_g1, Value.SymbolName);1625 addRelocationForSymbol(REmovk_g0, Value.SymbolName);1626 } else {1627 addRelocationForSection(REmovz_g3, Value.SectionID);1628 addRelocationForSection(REmovk_g2, Value.SectionID);1629 addRelocationForSection(REmovk_g1, Value.SectionID);1630 addRelocationForSection(REmovk_g0, Value.SectionID);1631 }1632 resolveRelocation(Section, Offset,1633 Section.getLoadAddressWithOffset(Section.getStubOffset()),1634 RelType, 0);1635 Section.advanceStubOffset(getMaxStubSize());1636 }1637}1638 1639Expected<relocation_iterator>1640RuntimeDyldELF::processRelocationRef(1641 unsigned SectionID, relocation_iterator RelI, const ObjectFile &O,1642 ObjSectionToIDMap &ObjSectionToID, StubMap &Stubs) {1643 const auto &Obj = cast<ELFObjectFileBase>(O);1644 uint64_t RelType = RelI->getType();1645 int64_t Addend = 0;1646 if (Expected<int64_t> AddendOrErr = ELFRelocationRef(*RelI).getAddend())1647 Addend = *AddendOrErr;1648 else1649 consumeError(AddendOrErr.takeError());1650 elf_symbol_iterator Symbol = RelI->getSymbol();1651 1652 // Obtain the symbol name which is referenced in the relocation1653 StringRef TargetName;1654 if (Symbol != Obj.symbol_end()) {1655 if (auto TargetNameOrErr = Symbol->getName())1656 TargetName = *TargetNameOrErr;1657 else1658 return TargetNameOrErr.takeError();1659 }1660 LLVM_DEBUG(dbgs() << "\t\tRelType: " << RelType << " Addend: " << Addend1661 << " TargetName: " << TargetName << "\n");1662 RelocationValueRef Value;1663 // First search for the symbol in the local symbol table1664 SymbolRef::Type SymType = SymbolRef::ST_Unknown;1665 1666 // Search for the symbol in the global symbol table1667 RTDyldSymbolTable::const_iterator gsi = GlobalSymbolTable.end();1668 if (Symbol != Obj.symbol_end()) {1669 gsi = GlobalSymbolTable.find(TargetName.data());1670 Expected<SymbolRef::Type> SymTypeOrErr = Symbol->getType();1671 if (!SymTypeOrErr) {1672 std::string Buf;1673 raw_string_ostream OS(Buf);1674 logAllUnhandledErrors(SymTypeOrErr.takeError(), OS);1675 report_fatal_error(Twine(Buf));1676 }1677 SymType = *SymTypeOrErr;1678 }1679 if (gsi != GlobalSymbolTable.end()) {1680 const auto &SymInfo = gsi->second;1681 Value.SectionID = SymInfo.getSectionID();1682 Value.Offset = SymInfo.getOffset();1683 Value.Addend = SymInfo.getOffset() + Addend;1684 } else {1685 switch (SymType) {1686 case SymbolRef::ST_Debug: {1687 // TODO: Now ELF SymbolRef::ST_Debug = STT_SECTION, it's not obviously1688 // and can be changed by another developers. Maybe best way is add1689 // a new symbol type ST_Section to SymbolRef and use it.1690 auto SectionOrErr = Symbol->getSection();1691 if (!SectionOrErr) {1692 std::string Buf;1693 raw_string_ostream OS(Buf);1694 logAllUnhandledErrors(SectionOrErr.takeError(), OS);1695 report_fatal_error(Twine(Buf));1696 }1697 section_iterator si = *SectionOrErr;1698 if (si == Obj.section_end())1699 llvm_unreachable("Symbol section not found, bad object file format!");1700 LLVM_DEBUG(dbgs() << "\t\tThis is section symbol\n");1701 bool isCode = si->isText();1702 if (auto SectionIDOrErr = findOrEmitSection(Obj, (*si), isCode,1703 ObjSectionToID))1704 Value.SectionID = *SectionIDOrErr;1705 else1706 return SectionIDOrErr.takeError();1707 Value.Addend = Addend;1708 break;1709 }1710 case SymbolRef::ST_Data:1711 case SymbolRef::ST_Function:1712 case SymbolRef::ST_Other:1713 case SymbolRef::ST_Unknown: {1714 Value.SymbolName = TargetName.data();1715 Value.Addend = Addend;1716 1717 // Absolute relocations will have a zero symbol ID (STN_UNDEF), which1718 // will manifest here as a NULL symbol name.1719 // We can set this as a valid (but empty) symbol name, and rely1720 // on addRelocationForSymbol to handle this.1721 if (!Value.SymbolName)1722 Value.SymbolName = "";1723 break;1724 }1725 default:1726 llvm_unreachable("Unresolved symbol type!");1727 break;1728 }1729 }1730 1731 uint64_t Offset = RelI->getOffset();1732 1733 LLVM_DEBUG(dbgs() << "\t\tSectionID: " << SectionID << " Offset: " << Offset1734 << "\n");1735 if ((Arch == Triple::aarch64 || Arch == Triple::aarch64_be)) {1736 if ((RelType == ELF::R_AARCH64_CALL26 ||1737 RelType == ELF::R_AARCH64_JUMP26) &&1738 MemMgr.allowStubAllocation()) {1739 resolveAArch64Branch(SectionID, Value, RelI, Stubs);1740 } else if (RelType == ELF::R_AARCH64_ADR_GOT_PAGE) {1741 // Create new GOT entry or find existing one. If GOT entry is1742 // to be created, then we also emit ABS64 relocation for it.1743 uint64_t GOTOffset = findOrAllocGOTEntry(Value, ELF::R_AARCH64_ABS64);1744 resolveGOTOffsetRelocation(SectionID, Offset, GOTOffset + Addend,1745 ELF::R_AARCH64_ADR_PREL_PG_HI21);1746 1747 } else if (RelType == ELF::R_AARCH64_LD64_GOT_LO12_NC) {1748 uint64_t GOTOffset = findOrAllocGOTEntry(Value, ELF::R_AARCH64_ABS64);1749 resolveGOTOffsetRelocation(SectionID, Offset, GOTOffset + Addend,1750 ELF::R_AARCH64_LDST64_ABS_LO12_NC);1751 } else {1752 processSimpleRelocation(SectionID, Offset, RelType, Value);1753 }1754 } else if (Arch == Triple::arm) {1755 if (RelType == ELF::R_ARM_PC24 || RelType == ELF::R_ARM_CALL ||1756 RelType == ELF::R_ARM_JUMP24) {1757 // This is an ARM branch relocation, need to use a stub function.1758 LLVM_DEBUG(dbgs() << "\t\tThis is an ARM branch relocation.\n");1759 SectionEntry &Section = Sections[SectionID];1760 1761 // Look for an existing stub.1762 auto [It, Inserted] = Stubs.try_emplace(Value);1763 if (!Inserted) {1764 resolveRelocation(Section, Offset,1765 Section.getLoadAddressWithOffset(It->second), RelType,1766 0);1767 LLVM_DEBUG(dbgs() << " Stub function found\n");1768 } else {1769 // Create a new stub function.1770 LLVM_DEBUG(dbgs() << " Create a new stub function\n");1771 It->second = Section.getStubOffset();1772 uint8_t *StubTargetAddr = createStubFunction(1773 Section.getAddressWithOffset(Section.getStubOffset()));1774 RelocationEntry RE(SectionID, StubTargetAddr - Section.getAddress(),1775 ELF::R_ARM_ABS32, Value.Addend);1776 if (Value.SymbolName)1777 addRelocationForSymbol(RE, Value.SymbolName);1778 else1779 addRelocationForSection(RE, Value.SectionID);1780 1781 resolveRelocation(1782 Section, Offset,1783 Section.getLoadAddressWithOffset(Section.getStubOffset()), RelType,1784 0);1785 Section.advanceStubOffset(getMaxStubSize());1786 }1787 } else {1788 uint32_t *Placeholder =1789 reinterpret_cast<uint32_t*>(computePlaceholderAddress(SectionID, Offset));1790 if (RelType == ELF::R_ARM_PREL31 || RelType == ELF::R_ARM_TARGET1 ||1791 RelType == ELF::R_ARM_ABS32) {1792 Value.Addend += *Placeholder;1793 } else if (RelType == ELF::R_ARM_MOVW_ABS_NC || RelType == ELF::R_ARM_MOVT_ABS) {1794 // See ELF for ARM documentation1795 Value.Addend += (int16_t)((*Placeholder & 0xFFF) | (((*Placeholder >> 16) & 0xF) << 12));1796 }1797 processSimpleRelocation(SectionID, Offset, RelType, Value);1798 }1799 } else if (Arch == Triple::loongarch64) {1800 if ((RelType == ELF::R_LARCH_B26 || RelType == ELF::R_LARCH_CALL36) &&1801 MemMgr.allowStubAllocation()) {1802 resolveLoongArch64Branch(SectionID, Value, RelI, Stubs);1803 } else if (RelType == ELF::R_LARCH_GOT_PC_HI20 ||1804 RelType == ELF::R_LARCH_GOT_PC_LO12 ||1805 RelType == ELF::R_LARCH_GOT64_PC_HI12 ||1806 RelType == ELF::R_LARCH_GOT64_PC_LO20) {1807 uint64_t GOTOffset = findOrAllocGOTEntry(Value, ELF::R_LARCH_64);1808 resolveGOTOffsetRelocation(SectionID, Offset, GOTOffset + Addend,1809 RelType);1810 } else {1811 processSimpleRelocation(SectionID, Offset, RelType, Value);1812 }1813 } else if (IsMipsO32ABI) {1814 uint8_t *Placeholder = reinterpret_cast<uint8_t *>(1815 computePlaceholderAddress(SectionID, Offset));1816 uint32_t Opcode = readBytesUnaligned(Placeholder, 4);1817 if (RelType == ELF::R_MIPS_26) {1818 // This is an Mips branch relocation, need to use a stub function.1819 LLVM_DEBUG(dbgs() << "\t\tThis is a Mips branch relocation.");1820 SectionEntry &Section = Sections[SectionID];1821 1822 // Extract the addend from the instruction.1823 // We shift up by two since the Value will be down shifted again1824 // when applying the relocation.1825 uint32_t Addend = (Opcode & 0x03ffffff) << 2;1826 1827 Value.Addend += Addend;1828 1829 // Look up for existing stub.1830 auto [It, Inserted] = Stubs.try_emplace(Value);1831 if (!Inserted) {1832 RelocationEntry RE(SectionID, Offset, RelType, It->second);1833 addRelocationForSection(RE, SectionID);1834 LLVM_DEBUG(dbgs() << " Stub function found\n");1835 } else {1836 // Create a new stub function.1837 LLVM_DEBUG(dbgs() << " Create a new stub function\n");1838 It->second = Section.getStubOffset();1839 1840 unsigned AbiVariant = Obj.getPlatformFlags();1841 1842 uint8_t *StubTargetAddr = createStubFunction(1843 Section.getAddressWithOffset(Section.getStubOffset()), AbiVariant);1844 1845 // Creating Hi and Lo relocations for the filled stub instructions.1846 RelocationEntry REHi(SectionID, StubTargetAddr - Section.getAddress(),1847 ELF::R_MIPS_HI16, Value.Addend);1848 RelocationEntry RELo(SectionID,1849 StubTargetAddr - Section.getAddress() + 4,1850 ELF::R_MIPS_LO16, Value.Addend);1851 1852 if (Value.SymbolName) {1853 addRelocationForSymbol(REHi, Value.SymbolName);1854 addRelocationForSymbol(RELo, Value.SymbolName);1855 } else {1856 addRelocationForSection(REHi, Value.SectionID);1857 addRelocationForSection(RELo, Value.SectionID);1858 }1859 1860 RelocationEntry RE(SectionID, Offset, RelType, Section.getStubOffset());1861 addRelocationForSection(RE, SectionID);1862 Section.advanceStubOffset(getMaxStubSize());1863 }1864 } else if (RelType == ELF::R_MIPS_HI16 || RelType == ELF::R_MIPS_PCHI16) {1865 int64_t Addend = (Opcode & 0x0000ffff) << 16;1866 RelocationEntry RE(SectionID, Offset, RelType, Addend);1867 PendingRelocs.push_back(std::make_pair(Value, RE));1868 } else if (RelType == ELF::R_MIPS_LO16 || RelType == ELF::R_MIPS_PCLO16) {1869 int64_t Addend = Value.Addend + SignExtend32<16>(Opcode & 0x0000ffff);1870 for (auto I = PendingRelocs.begin(); I != PendingRelocs.end();) {1871 const RelocationValueRef &MatchingValue = I->first;1872 RelocationEntry &Reloc = I->second;1873 if (MatchingValue == Value &&1874 RelType == getMatchingLoRelocation(Reloc.RelType) &&1875 SectionID == Reloc.SectionID) {1876 Reloc.Addend += Addend;1877 if (Value.SymbolName)1878 addRelocationForSymbol(Reloc, Value.SymbolName);1879 else1880 addRelocationForSection(Reloc, Value.SectionID);1881 I = PendingRelocs.erase(I);1882 } else1883 ++I;1884 }1885 RelocationEntry RE(SectionID, Offset, RelType, Addend);1886 if (Value.SymbolName)1887 addRelocationForSymbol(RE, Value.SymbolName);1888 else1889 addRelocationForSection(RE, Value.SectionID);1890 } else {1891 if (RelType == ELF::R_MIPS_32)1892 Value.Addend += Opcode;1893 else if (RelType == ELF::R_MIPS_PC16)1894 Value.Addend += SignExtend32<18>((Opcode & 0x0000ffff) << 2);1895 else if (RelType == ELF::R_MIPS_PC19_S2)1896 Value.Addend += SignExtend32<21>((Opcode & 0x0007ffff) << 2);1897 else if (RelType == ELF::R_MIPS_PC21_S2)1898 Value.Addend += SignExtend32<23>((Opcode & 0x001fffff) << 2);1899 else if (RelType == ELF::R_MIPS_PC26_S2)1900 Value.Addend += SignExtend32<28>((Opcode & 0x03ffffff) << 2);1901 processSimpleRelocation(SectionID, Offset, RelType, Value);1902 }1903 } else if (IsMipsN32ABI || IsMipsN64ABI) {1904 uint32_t r_type = RelType & 0xff;1905 RelocationEntry RE(SectionID, Offset, RelType, Value.Addend);1906 if (r_type == ELF::R_MIPS_CALL16 || r_type == ELF::R_MIPS_GOT_PAGE1907 || r_type == ELF::R_MIPS_GOT_DISP) {1908 auto [I, Inserted] = GOTSymbolOffsets.try_emplace(TargetName);1909 if (Inserted)1910 I->second = allocateGOTEntries(1);1911 RE.SymOffset = I->second;1912 if (Value.SymbolName)1913 addRelocationForSymbol(RE, Value.SymbolName);1914 else1915 addRelocationForSection(RE, Value.SectionID);1916 } else if (RelType == ELF::R_MIPS_26) {1917 // This is an Mips branch relocation, need to use a stub function.1918 LLVM_DEBUG(dbgs() << "\t\tThis is a Mips branch relocation.");1919 SectionEntry &Section = Sections[SectionID];1920 1921 // Look up for existing stub.1922 StubMap::const_iterator i = Stubs.find(Value);1923 if (i != Stubs.end()) {1924 RelocationEntry RE(SectionID, Offset, RelType, i->second);1925 addRelocationForSection(RE, SectionID);1926 LLVM_DEBUG(dbgs() << " Stub function found\n");1927 } else {1928 // Create a new stub function.1929 LLVM_DEBUG(dbgs() << " Create a new stub function\n");1930 Stubs[Value] = Section.getStubOffset();1931 1932 unsigned AbiVariant = Obj.getPlatformFlags();1933 1934 uint8_t *StubTargetAddr = createStubFunction(1935 Section.getAddressWithOffset(Section.getStubOffset()), AbiVariant);1936 1937 if (IsMipsN32ABI) {1938 // Creating Hi and Lo relocations for the filled stub instructions.1939 RelocationEntry REHi(SectionID, StubTargetAddr - Section.getAddress(),1940 ELF::R_MIPS_HI16, Value.Addend);1941 RelocationEntry RELo(SectionID,1942 StubTargetAddr - Section.getAddress() + 4,1943 ELF::R_MIPS_LO16, Value.Addend);1944 if (Value.SymbolName) {1945 addRelocationForSymbol(REHi, Value.SymbolName);1946 addRelocationForSymbol(RELo, Value.SymbolName);1947 } else {1948 addRelocationForSection(REHi, Value.SectionID);1949 addRelocationForSection(RELo, Value.SectionID);1950 }1951 } else {1952 // Creating Highest, Higher, Hi and Lo relocations for the filled stub1953 // instructions.1954 RelocationEntry REHighest(SectionID,1955 StubTargetAddr - Section.getAddress(),1956 ELF::R_MIPS_HIGHEST, Value.Addend);1957 RelocationEntry REHigher(SectionID,1958 StubTargetAddr - Section.getAddress() + 4,1959 ELF::R_MIPS_HIGHER, Value.Addend);1960 RelocationEntry REHi(SectionID,1961 StubTargetAddr - Section.getAddress() + 12,1962 ELF::R_MIPS_HI16, Value.Addend);1963 RelocationEntry RELo(SectionID,1964 StubTargetAddr - Section.getAddress() + 20,1965 ELF::R_MIPS_LO16, Value.Addend);1966 if (Value.SymbolName) {1967 addRelocationForSymbol(REHighest, Value.SymbolName);1968 addRelocationForSymbol(REHigher, Value.SymbolName);1969 addRelocationForSymbol(REHi, Value.SymbolName);1970 addRelocationForSymbol(RELo, Value.SymbolName);1971 } else {1972 addRelocationForSection(REHighest, Value.SectionID);1973 addRelocationForSection(REHigher, Value.SectionID);1974 addRelocationForSection(REHi, Value.SectionID);1975 addRelocationForSection(RELo, Value.SectionID);1976 }1977 }1978 RelocationEntry RE(SectionID, Offset, RelType, Section.getStubOffset());1979 addRelocationForSection(RE, SectionID);1980 Section.advanceStubOffset(getMaxStubSize());1981 }1982 } else {1983 processSimpleRelocation(SectionID, Offset, RelType, Value);1984 }1985 1986 } else if (Arch == Triple::ppc64 || Arch == Triple::ppc64le) {1987 if (RelType == ELF::R_PPC64_REL24) {1988 // Determine ABI variant in use for this object.1989 unsigned AbiVariant = Obj.getPlatformFlags();1990 AbiVariant &= ELF::EF_PPC64_ABI;1991 // A PPC branch relocation will need a stub function if the target is1992 // an external symbol (either Value.SymbolName is set, or SymType is1993 // Symbol::ST_Unknown) or if the target address is not within the1994 // signed 24-bits branch address.1995 SectionEntry &Section = Sections[SectionID];1996 uint8_t *Target = Section.getAddressWithOffset(Offset);1997 bool RangeOverflow = false;1998 bool IsExtern = Value.SymbolName || SymType == SymbolRef::ST_Unknown;1999 if (!IsExtern) {2000 if (AbiVariant != 2) {2001 // In the ELFv1 ABI, a function call may point to the .opd entry,2002 // so the final symbol value is calculated based on the relocation2003 // values in the .opd section.2004 if (auto Err = findOPDEntrySection(Obj, ObjSectionToID, Value))2005 return std::move(Err);2006 } else {2007 // In the ELFv2 ABI, a function symbol may provide a local entry2008 // point, which must be used for direct calls.2009 if (Value.SectionID == SectionID){2010 uint8_t SymOther = Symbol->getOther();2011 Value.Addend += ELF::decodePPC64LocalEntryOffset(SymOther);2012 }2013 }2014 uint8_t *RelocTarget =2015 Sections[Value.SectionID].getAddressWithOffset(Value.Addend);2016 int64_t delta = static_cast<int64_t>(Target - RelocTarget);2017 // If it is within 26-bits branch range, just set the branch target2018 if (SignExtend64<26>(delta) != delta) {2019 RangeOverflow = true;2020 } else if ((AbiVariant != 2) ||2021 (AbiVariant == 2 && Value.SectionID == SectionID)) {2022 RelocationEntry RE(SectionID, Offset, RelType, Value.Addend);2023 addRelocationForSection(RE, Value.SectionID);2024 }2025 }2026 if (IsExtern || (AbiVariant == 2 && Value.SectionID != SectionID) ||2027 RangeOverflow) {2028 // It is an external symbol (either Value.SymbolName is set, or2029 // SymType is SymbolRef::ST_Unknown) or out of range.2030 auto [It, Inserted] = Stubs.try_emplace(Value);2031 if (!Inserted) {2032 // Symbol function stub already created, just relocate to it2033 resolveRelocation(Section, Offset,2034 Section.getLoadAddressWithOffset(It->second),2035 RelType, 0);2036 LLVM_DEBUG(dbgs() << " Stub function found\n");2037 } else {2038 // Create a new stub function.2039 LLVM_DEBUG(dbgs() << " Create a new stub function\n");2040 It->second = Section.getStubOffset();2041 uint8_t *StubTargetAddr = createStubFunction(2042 Section.getAddressWithOffset(Section.getStubOffset()),2043 AbiVariant);2044 RelocationEntry RE(SectionID, StubTargetAddr - Section.getAddress(),2045 ELF::R_PPC64_ADDR64, Value.Addend);2046 2047 // Generates the 64-bits address loads as exemplified in section2048 // 4.5.1 in PPC64 ELF ABI. Note that the relocations need to2049 // apply to the low part of the instructions, so we have to update2050 // the offset according to the target endianness.2051 uint64_t StubRelocOffset = StubTargetAddr - Section.getAddress();2052 if (!IsTargetLittleEndian)2053 StubRelocOffset += 2;2054 2055 RelocationEntry REhst(SectionID, StubRelocOffset + 0,2056 ELF::R_PPC64_ADDR16_HIGHEST, Value.Addend);2057 RelocationEntry REhr(SectionID, StubRelocOffset + 4,2058 ELF::R_PPC64_ADDR16_HIGHER, Value.Addend);2059 RelocationEntry REh(SectionID, StubRelocOffset + 12,2060 ELF::R_PPC64_ADDR16_HI, Value.Addend);2061 RelocationEntry REl(SectionID, StubRelocOffset + 16,2062 ELF::R_PPC64_ADDR16_LO, Value.Addend);2063 2064 if (Value.SymbolName) {2065 addRelocationForSymbol(REhst, Value.SymbolName);2066 addRelocationForSymbol(REhr, Value.SymbolName);2067 addRelocationForSymbol(REh, Value.SymbolName);2068 addRelocationForSymbol(REl, Value.SymbolName);2069 } else {2070 addRelocationForSection(REhst, Value.SectionID);2071 addRelocationForSection(REhr, Value.SectionID);2072 addRelocationForSection(REh, Value.SectionID);2073 addRelocationForSection(REl, Value.SectionID);2074 }2075 2076 resolveRelocation(2077 Section, Offset,2078 Section.getLoadAddressWithOffset(Section.getStubOffset()),2079 RelType, 0);2080 Section.advanceStubOffset(getMaxStubSize());2081 }2082 if (IsExtern || (AbiVariant == 2 && Value.SectionID != SectionID)) {2083 // Restore the TOC for external calls2084 if (AbiVariant == 2)2085 writeInt32BE(Target + 4, 0xE8410018); // ld r2,24(r1)2086 else2087 writeInt32BE(Target + 4, 0xE8410028); // ld r2,40(r1)2088 }2089 }2090 } else if (RelType == ELF::R_PPC64_TOC16 ||2091 RelType == ELF::R_PPC64_TOC16_DS ||2092 RelType == ELF::R_PPC64_TOC16_LO ||2093 RelType == ELF::R_PPC64_TOC16_LO_DS ||2094 RelType == ELF::R_PPC64_TOC16_HI ||2095 RelType == ELF::R_PPC64_TOC16_HA) {2096 // These relocations are supposed to subtract the TOC address from2097 // the final value. This does not fit cleanly into the RuntimeDyld2098 // scheme, since there may be *two* sections involved in determining2099 // the relocation value (the section of the symbol referred to by the2100 // relocation, and the TOC section associated with the current module).2101 //2102 // Fortunately, these relocations are currently only ever generated2103 // referring to symbols that themselves reside in the TOC, which means2104 // that the two sections are actually the same. Thus they cancel out2105 // and we can immediately resolve the relocation right now.2106 switch (RelType) {2107 case ELF::R_PPC64_TOC16: RelType = ELF::R_PPC64_ADDR16; break;2108 case ELF::R_PPC64_TOC16_DS: RelType = ELF::R_PPC64_ADDR16_DS; break;2109 case ELF::R_PPC64_TOC16_LO: RelType = ELF::R_PPC64_ADDR16_LO; break;2110 case ELF::R_PPC64_TOC16_LO_DS: RelType = ELF::R_PPC64_ADDR16_LO_DS; break;2111 case ELF::R_PPC64_TOC16_HI: RelType = ELF::R_PPC64_ADDR16_HI; break;2112 case ELF::R_PPC64_TOC16_HA: RelType = ELF::R_PPC64_ADDR16_HA; break;2113 default: llvm_unreachable("Wrong relocation type.");2114 }2115 2116 RelocationValueRef TOCValue;2117 if (auto Err = findPPC64TOCSection(Obj, ObjSectionToID, TOCValue))2118 return std::move(Err);2119 if (Value.SymbolName || Value.SectionID != TOCValue.SectionID)2120 llvm_unreachable("Unsupported TOC relocation.");2121 Value.Addend -= TOCValue.Addend;2122 resolveRelocation(Sections[SectionID], Offset, Value.Addend, RelType, 0);2123 } else {2124 // There are two ways to refer to the TOC address directly: either2125 // via a ELF::R_PPC64_TOC relocation (where both symbol and addend are2126 // ignored), or via any relocation that refers to the magic ".TOC."2127 // symbols (in which case the addend is respected).2128 if (RelType == ELF::R_PPC64_TOC) {2129 RelType = ELF::R_PPC64_ADDR64;2130 if (auto Err = findPPC64TOCSection(Obj, ObjSectionToID, Value))2131 return std::move(Err);2132 } else if (TargetName == ".TOC.") {2133 if (auto Err = findPPC64TOCSection(Obj, ObjSectionToID, Value))2134 return std::move(Err);2135 Value.Addend += Addend;2136 }2137 2138 RelocationEntry RE(SectionID, Offset, RelType, Value.Addend);2139 2140 if (Value.SymbolName)2141 addRelocationForSymbol(RE, Value.SymbolName);2142 else2143 addRelocationForSection(RE, Value.SectionID);2144 }2145 } else if (Arch == Triple::systemz &&2146 (RelType == ELF::R_390_PLT32DBL || RelType == ELF::R_390_GOTENT)) {2147 // Create function stubs for both PLT and GOT references, regardless of2148 // whether the GOT reference is to data or code. The stub contains the2149 // full address of the symbol, as needed by GOT references, and the2150 // executable part only adds an overhead of 8 bytes.2151 //2152 // We could try to conserve space by allocating the code and data2153 // parts of the stub separately. However, as things stand, we allocate2154 // a stub for every relocation, so using a GOT in JIT code should be2155 // no less space efficient than using an explicit constant pool.2156 LLVM_DEBUG(dbgs() << "\t\tThis is a SystemZ indirect relocation.");2157 SectionEntry &Section = Sections[SectionID];2158 2159 // Look for an existing stub.2160 StubMap::const_iterator i = Stubs.find(Value);2161 uintptr_t StubAddress;2162 if (i != Stubs.end()) {2163 StubAddress = uintptr_t(Section.getAddressWithOffset(i->second));2164 LLVM_DEBUG(dbgs() << " Stub function found\n");2165 } else {2166 // Create a new stub function.2167 LLVM_DEBUG(dbgs() << " Create a new stub function\n");2168 2169 uintptr_t BaseAddress = uintptr_t(Section.getAddress());2170 StubAddress =2171 alignTo(BaseAddress + Section.getStubOffset(), getStubAlignment());2172 unsigned StubOffset = StubAddress - BaseAddress;2173 2174 Stubs[Value] = StubOffset;2175 createStubFunction((uint8_t *)StubAddress);2176 RelocationEntry RE(SectionID, StubOffset + 8, ELF::R_390_64,2177 Value.Offset);2178 if (Value.SymbolName)2179 addRelocationForSymbol(RE, Value.SymbolName);2180 else2181 addRelocationForSection(RE, Value.SectionID);2182 Section.advanceStubOffset(getMaxStubSize());2183 }2184 2185 if (RelType == ELF::R_390_GOTENT)2186 resolveRelocation(Section, Offset, StubAddress + 8, ELF::R_390_PC32DBL,2187 Addend);2188 else2189 resolveRelocation(Section, Offset, StubAddress, RelType, Addend);2190 } else if (Arch == Triple::x86_64) {2191 if (RelType == ELF::R_X86_64_PLT32) {2192 // The way the PLT relocations normally work is that the linker allocates2193 // the2194 // PLT and this relocation makes a PC-relative call into the PLT. The PLT2195 // entry will then jump to an address provided by the GOT. On first call,2196 // the2197 // GOT address will point back into PLT code that resolves the symbol. After2198 // the first call, the GOT entry points to the actual function.2199 //2200 // For local functions we're ignoring all of that here and just replacing2201 // the PLT32 relocation type with PC32, which will translate the relocation2202 // into a PC-relative call directly to the function. For external symbols we2203 // can't be sure the function will be within 2^32 bytes of the call site, so2204 // we need to create a stub, which calls into the GOT. This case is2205 // equivalent to the usual PLT implementation except that we use the stub2206 // mechanism in RuntimeDyld (which puts stubs at the end of the section)2207 // rather than allocating a PLT section.2208 if (Value.SymbolName && MemMgr.allowStubAllocation()) {2209 // This is a call to an external function.2210 // Look for an existing stub.2211 SectionEntry *Section = &Sections[SectionID];2212 auto [It, Inserted] = Stubs.try_emplace(Value);2213 uintptr_t StubAddress;2214 if (!Inserted) {2215 StubAddress = uintptr_t(Section->getAddress()) + It->second;2216 LLVM_DEBUG(dbgs() << " Stub function found\n");2217 } else {2218 // Create a new stub function (equivalent to a PLT entry).2219 LLVM_DEBUG(dbgs() << " Create a new stub function\n");2220 2221 uintptr_t BaseAddress = uintptr_t(Section->getAddress());2222 StubAddress = alignTo(BaseAddress + Section->getStubOffset(),2223 getStubAlignment());2224 unsigned StubOffset = StubAddress - BaseAddress;2225 It->second = StubOffset;2226 createStubFunction((uint8_t *)StubAddress);2227 2228 // Bump our stub offset counter2229 Section->advanceStubOffset(getMaxStubSize());2230 2231 // Allocate a GOT Entry2232 uint64_t GOTOffset = allocateGOTEntries(1);2233 // This potentially creates a new Section which potentially2234 // invalidates the Section pointer, so reload it.2235 Section = &Sections[SectionID];2236 2237 // The load of the GOT address has an addend of -42238 resolveGOTOffsetRelocation(SectionID, StubOffset + 2, GOTOffset - 4,2239 ELF::R_X86_64_PC32);2240 2241 // Fill in the value of the symbol we're targeting into the GOT2242 addRelocationForSymbol(2243 computeGOTOffsetRE(GOTOffset, 0, ELF::R_X86_64_64),2244 Value.SymbolName);2245 }2246 2247 // Make the target call a call into the stub table.2248 resolveRelocation(*Section, Offset, StubAddress, ELF::R_X86_64_PC32,2249 Addend);2250 } else {2251 Value.Addend += support::ulittle32_t::ref(2252 computePlaceholderAddress(SectionID, Offset));2253 processSimpleRelocation(SectionID, Offset, ELF::R_X86_64_PC32, Value);2254 }2255 } else if (RelType == ELF::R_X86_64_GOTPCREL ||2256 RelType == ELF::R_X86_64_GOTPCRELX ||2257 RelType == ELF::R_X86_64_REX_GOTPCRELX) {2258 uint64_t GOTOffset = allocateGOTEntries(1);2259 resolveGOTOffsetRelocation(SectionID, Offset, GOTOffset + Addend,2260 ELF::R_X86_64_PC32);2261 2262 // Fill in the value of the symbol we're targeting into the GOT2263 RelocationEntry RE =2264 computeGOTOffsetRE(GOTOffset, Value.Offset, ELF::R_X86_64_64);2265 if (Value.SymbolName)2266 addRelocationForSymbol(RE, Value.SymbolName);2267 else2268 addRelocationForSection(RE, Value.SectionID);2269 } else if (RelType == ELF::R_X86_64_GOT64) {2270 // Fill in a 64-bit GOT offset.2271 uint64_t GOTOffset = allocateGOTEntries(1);2272 resolveRelocation(Sections[SectionID], Offset, GOTOffset,2273 ELF::R_X86_64_64, 0);2274 2275 // Fill in the value of the symbol we're targeting into the GOT2276 RelocationEntry RE =2277 computeGOTOffsetRE(GOTOffset, Value.Offset, ELF::R_X86_64_64);2278 if (Value.SymbolName)2279 addRelocationForSymbol(RE, Value.SymbolName);2280 else2281 addRelocationForSection(RE, Value.SectionID);2282 } else if (RelType == ELF::R_X86_64_GOTPC32) {2283 // Materialize the address of the base of the GOT relative to the PC.2284 // This doesn't create a GOT entry, but it does mean we need a GOT2285 // section.2286 (void)allocateGOTEntries(0);2287 resolveGOTOffsetRelocation(SectionID, Offset, Addend, ELF::R_X86_64_PC32);2288 } else if (RelType == ELF::R_X86_64_GOTPC64) {2289 (void)allocateGOTEntries(0);2290 resolveGOTOffsetRelocation(SectionID, Offset, Addend, ELF::R_X86_64_PC64);2291 } else if (RelType == ELF::R_X86_64_GOTOFF64) {2292 // GOTOFF relocations ultimately require a section difference relocation.2293 (void)allocateGOTEntries(0);2294 processSimpleRelocation(SectionID, Offset, RelType, Value);2295 } else if (RelType == ELF::R_X86_64_PC32) {2296 Value.Addend += support::ulittle32_t::ref(computePlaceholderAddress(SectionID, Offset));2297 processSimpleRelocation(SectionID, Offset, RelType, Value);2298 } else if (RelType == ELF::R_X86_64_PC64) {2299 Value.Addend += support::ulittle64_t::ref(2300 computePlaceholderAddress(SectionID, Offset));2301 processSimpleRelocation(SectionID, Offset, RelType, Value);2302 } else if (RelType == ELF::R_X86_64_GOTTPOFF) {2303 processX86_64GOTTPOFFRelocation(SectionID, Offset, Value, Addend);2304 } else if (RelType == ELF::R_X86_64_TLSGD ||2305 RelType == ELF::R_X86_64_TLSLD) {2306 // The next relocation must be the relocation for __tls_get_addr.2307 ++RelI;2308 auto &GetAddrRelocation = *RelI;2309 processX86_64TLSRelocation(SectionID, Offset, RelType, Value, Addend,2310 GetAddrRelocation);2311 } else {2312 processSimpleRelocation(SectionID, Offset, RelType, Value);2313 }2314 } else if (Arch == Triple::riscv32 || Arch == Triple::riscv64) {2315 // *_LO12 relocation receive information about a symbol from the2316 // corresponding *_HI20 relocation, so we have to collect this information2317 // before resolving2318 if (RelType == ELF::R_RISCV_GOT_HI20 ||2319 RelType == ELF::R_RISCV_PCREL_HI20 ||2320 RelType == ELF::R_RISCV_TPREL_HI20 ||2321 RelType == ELF::R_RISCV_TLS_GD_HI20 ||2322 RelType == ELF::R_RISCV_TLS_GOT_HI20) {2323 RelocationEntry RE(SectionID, Offset, RelType, Addend);2324 PendingRelocs.push_back({Value, RE});2325 }2326 processSimpleRelocation(SectionID, Offset, RelType, Value);2327 } else {2328 if (Arch == Triple::x86) {2329 Value.Addend += support::ulittle32_t::ref(2330 computePlaceholderAddress(SectionID, Offset));2331 }2332 processSimpleRelocation(SectionID, Offset, RelType, Value);2333 }2334 return ++RelI;2335}2336 2337void RuntimeDyldELF::processX86_64GOTTPOFFRelocation(unsigned SectionID,2338 uint64_t Offset,2339 RelocationValueRef Value,2340 int64_t Addend) {2341 // Use the approach from "x86-64 Linker Optimizations" from the TLS spec2342 // to replace the GOTTPOFF relocation with a TPOFF relocation. The spec2343 // only mentions one optimization even though there are two different2344 // code sequences for the Initial Exec TLS Model. We match the code to2345 // find out which one was used.2346 2347 // A possible TLS code sequence and its replacement2348 struct CodeSequence {2349 // The expected code sequence2350 ArrayRef<uint8_t> ExpectedCodeSequence;2351 // The negative offset of the GOTTPOFF relocation to the beginning of2352 // the sequence2353 uint64_t TLSSequenceOffset;2354 // The new code sequence2355 ArrayRef<uint8_t> NewCodeSequence;2356 // The offset of the new TPOFF relocation2357 uint64_t TpoffRelocationOffset;2358 };2359 2360 std::array<CodeSequence, 2> CodeSequences;2361 2362 // Initial Exec Code Model Sequence2363 {2364 static const std::initializer_list<uint8_t> ExpectedCodeSequenceList = {2365 0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00,2366 0x00, // mov %fs:0, %rax2367 0x48, 0x03, 0x05, 0x00, 0x00, 0x00, 0x00 // add x@gotpoff(%rip),2368 // %rax2369 };2370 CodeSequences[0].ExpectedCodeSequence =2371 ArrayRef<uint8_t>(ExpectedCodeSequenceList);2372 CodeSequences[0].TLSSequenceOffset = 12;2373 2374 static const std::initializer_list<uint8_t> NewCodeSequenceList = {2375 0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00, 0x00, // mov %fs:0, %rax2376 0x48, 0x8d, 0x80, 0x00, 0x00, 0x00, 0x00 // lea x@tpoff(%rax), %rax2377 };2378 CodeSequences[0].NewCodeSequence = ArrayRef<uint8_t>(NewCodeSequenceList);2379 CodeSequences[0].TpoffRelocationOffset = 12;2380 }2381 2382 // Initial Exec Code Model Sequence, II2383 {2384 static const std::initializer_list<uint8_t> ExpectedCodeSequenceList = {2385 0x48, 0x8b, 0x05, 0x00, 0x00, 0x00, 0x00, // mov x@gotpoff(%rip), %rax2386 0x64, 0x48, 0x8b, 0x00, 0x00, 0x00, 0x00 // mov %fs:(%rax), %rax2387 };2388 CodeSequences[1].ExpectedCodeSequence =2389 ArrayRef<uint8_t>(ExpectedCodeSequenceList);2390 CodeSequences[1].TLSSequenceOffset = 3;2391 2392 static const std::initializer_list<uint8_t> NewCodeSequenceList = {2393 0x66, 0x0f, 0x1f, 0x44, 0x00, 0x00, // 6 byte nop2394 0x64, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00, 0x00, // mov %fs:x@tpoff, %rax2395 };2396 CodeSequences[1].NewCodeSequence = ArrayRef<uint8_t>(NewCodeSequenceList);2397 CodeSequences[1].TpoffRelocationOffset = 10;2398 }2399 2400 bool Resolved = false;2401 auto &Section = Sections[SectionID];2402 for (const auto &C : CodeSequences) {2403 assert(C.ExpectedCodeSequence.size() == C.NewCodeSequence.size() &&2404 "Old and new code sequences must have the same size");2405 2406 if (Offset < C.TLSSequenceOffset ||2407 (Offset - C.TLSSequenceOffset + C.NewCodeSequence.size()) >2408 Section.getSize()) {2409 // This can't be a matching sequence as it doesn't fit in the current2410 // section2411 continue;2412 }2413 2414 auto TLSSequenceStartOffset = Offset - C.TLSSequenceOffset;2415 auto *TLSSequence = Section.getAddressWithOffset(TLSSequenceStartOffset);2416 if (ArrayRef<uint8_t>(TLSSequence, C.ExpectedCodeSequence.size()) !=2417 C.ExpectedCodeSequence) {2418 continue;2419 }2420 2421 memcpy(TLSSequence, C.NewCodeSequence.data(), C.NewCodeSequence.size());2422 2423 // The original GOTTPOFF relocation has an addend as it is PC relative,2424 // so it needs to be corrected. The TPOFF32 relocation is used as an2425 // absolute value (which is an offset from %fs:0), so remove the addend2426 // again.2427 RelocationEntry RE(SectionID,2428 TLSSequenceStartOffset + C.TpoffRelocationOffset,2429 ELF::R_X86_64_TPOFF32, Value.Addend - Addend);2430 2431 if (Value.SymbolName)2432 addRelocationForSymbol(RE, Value.SymbolName);2433 else2434 addRelocationForSection(RE, Value.SectionID);2435 2436 Resolved = true;2437 break;2438 }2439 2440 if (!Resolved) {2441 // The GOTTPOFF relocation was not used in one of the sequences2442 // described in the spec, so we can't optimize it to a TPOFF2443 // relocation.2444 uint64_t GOTOffset = allocateGOTEntries(1);2445 resolveGOTOffsetRelocation(SectionID, Offset, GOTOffset + Addend,2446 ELF::R_X86_64_PC32);2447 RelocationEntry RE =2448 computeGOTOffsetRE(GOTOffset, Value.Offset, ELF::R_X86_64_TPOFF64);2449 if (Value.SymbolName)2450 addRelocationForSymbol(RE, Value.SymbolName);2451 else2452 addRelocationForSection(RE, Value.SectionID);2453 }2454}2455 2456void RuntimeDyldELF::processX86_64TLSRelocation(2457 unsigned SectionID, uint64_t Offset, uint64_t RelType,2458 RelocationValueRef Value, int64_t Addend,2459 const RelocationRef &GetAddrRelocation) {2460 // Since we are statically linking and have no additional DSOs, we can resolve2461 // the relocation directly without using __tls_get_addr.2462 // Use the approach from "x86-64 Linker Optimizations" from the TLS spec2463 // to replace it with the Local Exec relocation variant.2464 2465 // Find out whether the code was compiled with the large or small memory2466 // model. For this we look at the next relocation which is the relocation2467 // for the __tls_get_addr function. If it's a 32 bit relocation, it's the2468 // small code model, with a 64 bit relocation it's the large code model.2469 bool IsSmallCodeModel;2470 // Is the relocation for the __tls_get_addr a PC-relative GOT relocation?2471 bool IsGOTPCRel = false;2472 2473 switch (GetAddrRelocation.getType()) {2474 case ELF::R_X86_64_GOTPCREL:2475 case ELF::R_X86_64_REX_GOTPCRELX:2476 case ELF::R_X86_64_GOTPCRELX:2477 IsGOTPCRel = true;2478 [[fallthrough]];2479 case ELF::R_X86_64_PLT32:2480 IsSmallCodeModel = true;2481 break;2482 case ELF::R_X86_64_PLTOFF64:2483 IsSmallCodeModel = false;2484 break;2485 default:2486 report_fatal_error(2487 "invalid TLS relocations for General/Local Dynamic TLS Model: "2488 "expected PLT or GOT relocation for __tls_get_addr function");2489 }2490 2491 // The negative offset to the start of the TLS code sequence relative to2492 // the offset of the TLSGD/TLSLD relocation2493 uint64_t TLSSequenceOffset;2494 // The expected start of the code sequence2495 ArrayRef<uint8_t> ExpectedCodeSequence;2496 // The new TLS code sequence that will replace the existing code2497 ArrayRef<uint8_t> NewCodeSequence;2498 2499 if (RelType == ELF::R_X86_64_TLSGD) {2500 // The offset of the new TPOFF32 relocation (offset starting from the2501 // beginning of the whole TLS sequence)2502 uint64_t TpoffRelocOffset;2503 2504 if (IsSmallCodeModel) {2505 if (!IsGOTPCRel) {2506 static const std::initializer_list<uint8_t> CodeSequence = {2507 0x66, // data16 (no-op prefix)2508 0x48, 0x8d, 0x3d, 0x00, 0x00,2509 0x00, 0x00, // lea <disp32>(%rip), %rdi2510 0x66, 0x66, // two data16 prefixes2511 0x48, // rex64 (no-op prefix)2512 0xe8, 0x00, 0x00, 0x00, 0x00 // call __tls_get_addr@plt2513 };2514 ExpectedCodeSequence = ArrayRef<uint8_t>(CodeSequence);2515 TLSSequenceOffset = 4;2516 } else {2517 // This code sequence is not described in the TLS spec but gcc2518 // generates it sometimes.2519 static const std::initializer_list<uint8_t> CodeSequence = {2520 0x66, // data16 (no-op prefix)2521 0x48, 0x8d, 0x3d, 0x00, 0x00,2522 0x00, 0x00, // lea <disp32>(%rip), %rdi2523 0x66, // data16 prefix (no-op prefix)2524 0x48, // rex64 (no-op prefix)2525 0xff, 0x15, 0x00, 0x00, 0x00,2526 0x00 // call *__tls_get_addr@gotpcrel(%rip)2527 };2528 ExpectedCodeSequence = ArrayRef<uint8_t>(CodeSequence);2529 TLSSequenceOffset = 4;2530 }2531 2532 // The replacement code for the small code model. It's the same for2533 // both sequences.2534 static const std::initializer_list<uint8_t> SmallSequence = {2535 0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00,2536 0x00, // mov %fs:0, %rax2537 0x48, 0x8d, 0x80, 0x00, 0x00, 0x00, 0x00 // lea x@tpoff(%rax),2538 // %rax2539 };2540 NewCodeSequence = ArrayRef<uint8_t>(SmallSequence);2541 TpoffRelocOffset = 12;2542 } else {2543 static const std::initializer_list<uint8_t> CodeSequence = {2544 0x48, 0x8d, 0x3d, 0x00, 0x00, 0x00, 0x00, // lea <disp32>(%rip),2545 // %rdi2546 0x48, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,2547 0x00, // movabs $__tls_get_addr@pltoff, %rax2548 0x48, 0x01, 0xd8, // add %rbx, %rax2549 0xff, 0xd0 // call *%rax2550 };2551 ExpectedCodeSequence = ArrayRef<uint8_t>(CodeSequence);2552 TLSSequenceOffset = 3;2553 2554 // The replacement code for the large code model2555 static const std::initializer_list<uint8_t> LargeSequence = {2556 0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00,2557 0x00, // mov %fs:0, %rax2558 0x48, 0x8d, 0x80, 0x00, 0x00, 0x00, 0x00, // lea x@tpoff(%rax),2559 // %rax2560 0x66, 0x0f, 0x1f, 0x44, 0x00, 0x00 // nopw 0x0(%rax,%rax,1)2561 };2562 NewCodeSequence = ArrayRef<uint8_t>(LargeSequence);2563 TpoffRelocOffset = 12;2564 }2565 2566 // The TLSGD/TLSLD relocations are PC-relative, so they have an addend.2567 // The new TPOFF32 relocations is used as an absolute offset from2568 // %fs:0, so remove the TLSGD/TLSLD addend again.2569 RelocationEntry RE(SectionID, Offset - TLSSequenceOffset + TpoffRelocOffset,2570 ELF::R_X86_64_TPOFF32, Value.Addend - Addend);2571 if (Value.SymbolName)2572 addRelocationForSymbol(RE, Value.SymbolName);2573 else2574 addRelocationForSection(RE, Value.SectionID);2575 } else if (RelType == ELF::R_X86_64_TLSLD) {2576 if (IsSmallCodeModel) {2577 if (!IsGOTPCRel) {2578 static const std::initializer_list<uint8_t> CodeSequence = {2579 0x48, 0x8d, 0x3d, 0x00, 0x00, 0x00, // leaq <disp32>(%rip), %rdi2580 0x00, 0xe8, 0x00, 0x00, 0x00, 0x00 // call __tls_get_addr@plt2581 };2582 ExpectedCodeSequence = ArrayRef<uint8_t>(CodeSequence);2583 TLSSequenceOffset = 3;2584 2585 // The replacement code for the small code model2586 static const std::initializer_list<uint8_t> SmallSequence = {2587 0x66, 0x66, 0x66, // three data16 prefixes (no-op)2588 0x64, 0x48, 0x8b, 0x04, 0x25,2589 0x00, 0x00, 0x00, 0x00 // mov %fs:0, %rax2590 };2591 NewCodeSequence = ArrayRef<uint8_t>(SmallSequence);2592 } else {2593 // This code sequence is not described in the TLS spec but gcc2594 // generates it sometimes.2595 static const std::initializer_list<uint8_t> CodeSequence = {2596 0x48, 0x8d, 0x3d, 0x00,2597 0x00, 0x00, 0x00, // leaq <disp32>(%rip), %rdi2598 0xff, 0x15, 0x00, 0x00,2599 0x00, 0x00 // call2600 // *__tls_get_addr@gotpcrel(%rip)2601 };2602 ExpectedCodeSequence = ArrayRef<uint8_t>(CodeSequence);2603 TLSSequenceOffset = 3;2604 2605 // The replacement is code is just like above but it needs to be2606 // one byte longer.2607 static const std::initializer_list<uint8_t> SmallSequence = {2608 0x0f, 0x1f, 0x40, 0x00, // 4 byte nop2609 0x64, 0x48, 0x8b, 0x04, 0x25,2610 0x00, 0x00, 0x00, 0x00 // mov %fs:0, %rax2611 };2612 NewCodeSequence = ArrayRef<uint8_t>(SmallSequence);2613 }2614 } else {2615 // This is the same sequence as for the TLSGD sequence with the large2616 // memory model above2617 static const std::initializer_list<uint8_t> CodeSequence = {2618 0x48, 0x8d, 0x3d, 0x00, 0x00, 0x00, 0x00, // lea <disp32>(%rip),2619 // %rdi2620 0x48, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,2621 0x48, // movabs $__tls_get_addr@pltoff, %rax2622 0x01, 0xd8, // add %rbx, %rax2623 0xff, 0xd0 // call *%rax2624 };2625 ExpectedCodeSequence = ArrayRef<uint8_t>(CodeSequence);2626 TLSSequenceOffset = 3;2627 2628 // The replacement code for the large code model2629 static const std::initializer_list<uint8_t> LargeSequence = {2630 0x66, 0x66, 0x66, // three data16 prefixes (no-op)2631 0x66, 0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00,2632 0x00, // 10 byte nop2633 0x64, 0x48, 0x8b, 0x04, 0x25, 0x00, 0x00, 0x00, 0x00 // mov %fs:0,%rax2634 };2635 NewCodeSequence = ArrayRef<uint8_t>(LargeSequence);2636 }2637 } else {2638 llvm_unreachable("both TLS relocations handled above");2639 }2640 2641 assert(ExpectedCodeSequence.size() == NewCodeSequence.size() &&2642 "Old and new code sequences must have the same size");2643 2644 auto &Section = Sections[SectionID];2645 if (Offset < TLSSequenceOffset ||2646 (Offset - TLSSequenceOffset + NewCodeSequence.size()) >2647 Section.getSize()) {2648 report_fatal_error("unexpected end of section in TLS sequence");2649 }2650 2651 auto *TLSSequence = Section.getAddressWithOffset(Offset - TLSSequenceOffset);2652 if (ArrayRef<uint8_t>(TLSSequence, ExpectedCodeSequence.size()) !=2653 ExpectedCodeSequence) {2654 report_fatal_error(2655 "invalid TLS sequence for Global/Local Dynamic TLS Model");2656 }2657 2658 memcpy(TLSSequence, NewCodeSequence.data(), NewCodeSequence.size());2659}2660 2661size_t RuntimeDyldELF::getGOTEntrySize() {2662 // We don't use the GOT in all of these cases, but it's essentially free2663 // to put them all here.2664 size_t Result = 0;2665 switch (Arch) {2666 case Triple::x86_64:2667 case Triple::aarch64:2668 case Triple::aarch64_be:2669 case Triple::loongarch64:2670 case Triple::ppc64:2671 case Triple::ppc64le:2672 case Triple::systemz:2673 Result = sizeof(uint64_t);2674 break;2675 case Triple::x86:2676 case Triple::arm:2677 case Triple::thumb:2678 Result = sizeof(uint32_t);2679 break;2680 case Triple::mips:2681 case Triple::mipsel:2682 case Triple::mips64:2683 case Triple::mips64el:2684 if (IsMipsO32ABI || IsMipsN32ABI)2685 Result = sizeof(uint32_t);2686 else if (IsMipsN64ABI)2687 Result = sizeof(uint64_t);2688 else2689 llvm_unreachable("Mips ABI not handled");2690 break;2691 default:2692 llvm_unreachable("Unsupported CPU type!");2693 }2694 return Result;2695}2696 2697uint64_t RuntimeDyldELF::allocateGOTEntries(unsigned no) {2698 if (GOTSectionID == 0) {2699 GOTSectionID = Sections.size();2700 // Reserve a section id. We'll allocate the section later2701 // once we know the total size2702 Sections.push_back(SectionEntry(".got", nullptr, 0, 0, 0));2703 }2704 uint64_t StartOffset = CurrentGOTIndex * getGOTEntrySize();2705 CurrentGOTIndex += no;2706 return StartOffset;2707}2708 2709uint64_t RuntimeDyldELF::findOrAllocGOTEntry(const RelocationValueRef &Value,2710 unsigned GOTRelType) {2711 auto E = GOTOffsetMap.insert({Value, 0});2712 if (E.second) {2713 uint64_t GOTOffset = allocateGOTEntries(1);2714 2715 // Create relocation for newly created GOT entry2716 RelocationEntry RE =2717 computeGOTOffsetRE(GOTOffset, Value.Offset, GOTRelType);2718 if (Value.SymbolName)2719 addRelocationForSymbol(RE, Value.SymbolName);2720 else2721 addRelocationForSection(RE, Value.SectionID);2722 2723 E.first->second = GOTOffset;2724 }2725 2726 return E.first->second;2727}2728 2729void RuntimeDyldELF::resolveGOTOffsetRelocation(unsigned SectionID,2730 uint64_t Offset,2731 uint64_t GOTOffset,2732 uint32_t Type) {2733 // Fill in the relative address of the GOT Entry into the stub2734 RelocationEntry GOTRE(SectionID, Offset, Type, GOTOffset);2735 addRelocationForSection(GOTRE, GOTSectionID);2736}2737 2738RelocationEntry RuntimeDyldELF::computeGOTOffsetRE(uint64_t GOTOffset,2739 uint64_t SymbolOffset,2740 uint32_t Type) {2741 return RelocationEntry(GOTSectionID, GOTOffset, Type, SymbolOffset);2742}2743 2744void RuntimeDyldELF::processNewSymbol(const SymbolRef &ObjSymbol, SymbolTableEntry& Symbol) {2745 // This should never return an error as `processNewSymbol` wouldn't have been2746 // called if getFlags() returned an error before.2747 auto ObjSymbolFlags = cantFail(ObjSymbol.getFlags());2748 2749 if (ObjSymbolFlags & SymbolRef::SF_Indirect) {2750 if (IFuncStubSectionID == 0) {2751 // Create a dummy section for the ifunc stubs. It will be actually2752 // allocated in finalizeLoad() below.2753 IFuncStubSectionID = Sections.size();2754 Sections.push_back(2755 SectionEntry(".text.__llvm_IFuncStubs", nullptr, 0, 0, 0));2756 // First 64B are reserverd for the IFunc resolver2757 IFuncStubOffset = 64;2758 }2759 2760 IFuncStubs.push_back(IFuncStub{IFuncStubOffset, Symbol});2761 // Modify the symbol so that it points to the ifunc stub instead of to the2762 // resolver function.2763 Symbol = SymbolTableEntry(IFuncStubSectionID, IFuncStubOffset,2764 Symbol.getFlags());2765 IFuncStubOffset += getMaxIFuncStubSize();2766 }2767}2768 2769Error RuntimeDyldELF::finalizeLoad(const ObjectFile &Obj,2770 ObjSectionToIDMap &SectionMap) {2771 if (IsMipsO32ABI)2772 if (!PendingRelocs.empty())2773 return make_error<RuntimeDyldError>("Can't find matching LO16 reloc");2774 2775 // Create the IFunc stubs if necessary. This must be done before processing2776 // the GOT entries, as the IFunc stubs may create some.2777 if (IFuncStubSectionID != 0) {2778 uint8_t *IFuncStubsAddr = MemMgr.allocateCodeSection(2779 IFuncStubOffset, 1, IFuncStubSectionID, ".text.__llvm_IFuncStubs");2780 if (!IFuncStubsAddr)2781 return make_error<RuntimeDyldError>(2782 "Unable to allocate memory for IFunc stubs!");2783 Sections[IFuncStubSectionID] =2784 SectionEntry(".text.__llvm_IFuncStubs", IFuncStubsAddr, IFuncStubOffset,2785 IFuncStubOffset, 0);2786 2787 createIFuncResolver(IFuncStubsAddr);2788 2789 LLVM_DEBUG(dbgs() << "Creating IFunc stubs SectionID: "2790 << IFuncStubSectionID << " Addr: "2791 << Sections[IFuncStubSectionID].getAddress() << '\n');2792 for (auto &IFuncStub : IFuncStubs) {2793 auto &Symbol = IFuncStub.OriginalSymbol;2794 LLVM_DEBUG(dbgs() << "\tSectionID: " << Symbol.getSectionID()2795 << " Offset: " << format("%p", Symbol.getOffset())2796 << " IFuncStubOffset: "2797 << format("%p\n", IFuncStub.StubOffset));2798 createIFuncStub(IFuncStubSectionID, 0, IFuncStub.StubOffset,2799 Symbol.getSectionID(), Symbol.getOffset());2800 }2801 2802 IFuncStubSectionID = 0;2803 IFuncStubOffset = 0;2804 IFuncStubs.clear();2805 }2806 2807 // If necessary, allocate the global offset table2808 if (GOTSectionID != 0) {2809 // Allocate memory for the section2810 size_t TotalSize = CurrentGOTIndex * getGOTEntrySize();2811 uint8_t *Addr = MemMgr.allocateDataSection(TotalSize, getGOTEntrySize(),2812 GOTSectionID, ".got", false);2813 if (!Addr)2814 return make_error<RuntimeDyldError>("Unable to allocate memory for GOT!");2815 2816 Sections[GOTSectionID] =2817 SectionEntry(".got", Addr, TotalSize, TotalSize, 0);2818 2819 // For now, initialize all GOT entries to zero. We'll fill them in as2820 // needed when GOT-based relocations are applied.2821 memset(Addr, 0, TotalSize);2822 if (IsMipsN32ABI || IsMipsN64ABI) {2823 // To correctly resolve Mips GOT relocations, we need a mapping from2824 // object's sections to GOTs.2825 for (section_iterator SI = Obj.section_begin(), SE = Obj.section_end();2826 SI != SE; ++SI) {2827 if (!SI->relocations().empty()) {2828 Expected<section_iterator> RelSecOrErr = SI->getRelocatedSection();2829 if (!RelSecOrErr)2830 return make_error<RuntimeDyldError>(2831 toString(RelSecOrErr.takeError()));2832 2833 section_iterator RelocatedSection = *RelSecOrErr;2834 ObjSectionToIDMap::iterator i = SectionMap.find(*RelocatedSection);2835 assert(i != SectionMap.end());2836 SectionToGOTMap[i->second] = GOTSectionID;2837 }2838 }2839 GOTSymbolOffsets.clear();2840 }2841 }2842 2843 // Look for and record the EH frame section.2844 ObjSectionToIDMap::iterator i, e;2845 for (i = SectionMap.begin(), e = SectionMap.end(); i != e; ++i) {2846 const SectionRef &Section = i->first;2847 2848 StringRef Name;2849 Expected<StringRef> NameOrErr = Section.getName();2850 if (NameOrErr)2851 Name = *NameOrErr;2852 else2853 consumeError(NameOrErr.takeError());2854 2855 if (Name == ".eh_frame") {2856 UnregisteredEHFrameSections.push_back(i->second);2857 break;2858 }2859 }2860 2861 GOTOffsetMap.clear();2862 GOTSectionID = 0;2863 CurrentGOTIndex = 0;2864 2865 return Error::success();2866}2867 2868bool RuntimeDyldELF::isCompatibleFile(const object::ObjectFile &Obj) const {2869 return Obj.isELF();2870}2871 2872void RuntimeDyldELF::createIFuncResolver(uint8_t *Addr) const {2873 if (Arch == Triple::x86_64) {2874 // The adddres of the GOT1 entry is in %r11, the GOT2 entry is in %r11+82875 // (see createIFuncStub() for details)2876 // The following code first saves all registers that contain the original2877 // function arguments as those registers are not saved by the resolver2878 // function. %r11 is saved as well so that the GOT2 entry can be updated2879 // afterwards. Then it calls the actual IFunc resolver function whose2880 // address is stored in GOT2. After the resolver function returns, all2881 // saved registers are restored and the return value is written to GOT1.2882 // Finally, jump to the now resolved function.2883 // clang-format off2884 const uint8_t StubCode[] = {2885 0x57, // push %rdi2886 0x56, // push %rsi2887 0x52, // push %rdx2888 0x51, // push %rcx2889 0x41, 0x50, // push %r82890 0x41, 0x51, // push %r92891 0x41, 0x53, // push %r112892 0x41, 0xff, 0x53, 0x08, // call *0x8(%r11)2893 0x41, 0x5b, // pop %r112894 0x41, 0x59, // pop %r92895 0x41, 0x58, // pop %r82896 0x59, // pop %rcx2897 0x5a, // pop %rdx2898 0x5e, // pop %rsi2899 0x5f, // pop %rdi2900 0x49, 0x89, 0x03, // mov %rax,(%r11)2901 0xff, 0xe0 // jmp *%rax2902 };2903 // clang-format on2904 static_assert(sizeof(StubCode) <= 64,2905 "maximum size of the IFunc resolver is 64B");2906 memcpy(Addr, StubCode, sizeof(StubCode));2907 } else {2908 report_fatal_error(2909 "IFunc resolver is not supported for target architecture");2910 }2911}2912 2913void RuntimeDyldELF::createIFuncStub(unsigned IFuncStubSectionID,2914 uint64_t IFuncResolverOffset,2915 uint64_t IFuncStubOffset,2916 unsigned IFuncSectionID,2917 uint64_t IFuncOffset) {2918 auto &IFuncStubSection = Sections[IFuncStubSectionID];2919 auto *Addr = IFuncStubSection.getAddressWithOffset(IFuncStubOffset);2920 2921 if (Arch == Triple::x86_64) {2922 // The first instruction loads a PC-relative address into %r11 which is a2923 // GOT entry for this stub. This initially contains the address to the2924 // IFunc resolver. We can use %r11 here as it's caller saved but not used2925 // to pass any arguments. In fact, x86_64 ABI even suggests using %r11 for2926 // code in the PLT. The IFunc resolver will use %r11 to update the GOT2927 // entry.2928 //2929 // The next instruction just jumps to the address contained in the GOT2930 // entry. As mentioned above, we do this two-step jump by first setting2931 // %r11 so that the IFunc resolver has access to it.2932 //2933 // The IFunc resolver of course also needs to know the actual address of2934 // the actual IFunc resolver function. This will be stored in a GOT entry2935 // right next to the first one for this stub. So, the IFunc resolver will2936 // be able to call it with %r11+8.2937 //2938 // In total, two adjacent GOT entries (+relocation) and one additional2939 // relocation are required:2940 // GOT1: Address of the IFunc resolver.2941 // GOT2: Address of the IFunc resolver function.2942 // IFuncStubOffset+3: 32-bit PC-relative address of GOT1.2943 uint64_t GOT1 = allocateGOTEntries(2);2944 uint64_t GOT2 = GOT1 + getGOTEntrySize();2945 2946 RelocationEntry RE1(GOTSectionID, GOT1, ELF::R_X86_64_64,2947 IFuncResolverOffset, {});2948 addRelocationForSection(RE1, IFuncStubSectionID);2949 RelocationEntry RE2(GOTSectionID, GOT2, ELF::R_X86_64_64, IFuncOffset, {});2950 addRelocationForSection(RE2, IFuncSectionID);2951 2952 const uint8_t StubCode[] = {2953 0x4c, 0x8d, 0x1d, 0x00, 0x00, 0x00, 0x00, // leaq 0x0(%rip),%r112954 0x41, 0xff, 0x23 // jmpq *(%r11)2955 };2956 assert(sizeof(StubCode) <= getMaxIFuncStubSize() &&2957 "IFunc stub size must not exceed getMaxIFuncStubSize()");2958 memcpy(Addr, StubCode, sizeof(StubCode));2959 2960 // The PC-relative value starts 4 bytes from the end of the leaq2961 // instruction, so the addend is -4.2962 resolveGOTOffsetRelocation(IFuncStubSectionID, IFuncStubOffset + 3,2963 GOT1 - 4, ELF::R_X86_64_PC32);2964 } else {2965 report_fatal_error("IFunc stub is not supported for target architecture");2966 }2967}2968 2969unsigned RuntimeDyldELF::getMaxIFuncStubSize() const {2970 if (Arch == Triple::x86_64) {2971 return 10;2972 }2973 return 0;2974}2975 2976bool RuntimeDyldELF::relocationNeedsGot(const RelocationRef &R) const {2977 unsigned RelTy = R.getType();2978 if (Arch == Triple::aarch64 || Arch == Triple::aarch64_be)2979 return RelTy == ELF::R_AARCH64_ADR_GOT_PAGE ||2980 RelTy == ELF::R_AARCH64_LD64_GOT_LO12_NC;2981 2982 if (Arch == Triple::loongarch64)2983 return RelTy == ELF::R_LARCH_GOT_PC_HI20 ||2984 RelTy == ELF::R_LARCH_GOT_PC_LO12 ||2985 RelTy == ELF::R_LARCH_GOT64_PC_HI12 ||2986 RelTy == ELF::R_LARCH_GOT64_PC_LO20;2987 2988 if (Arch == Triple::x86_64)2989 return RelTy == ELF::R_X86_64_GOTPCREL ||2990 RelTy == ELF::R_X86_64_GOTPCRELX ||2991 RelTy == ELF::R_X86_64_GOT64 ||2992 RelTy == ELF::R_X86_64_REX_GOTPCRELX;2993 return false;2994}2995 2996bool RuntimeDyldELF::relocationNeedsStub(const RelocationRef &R) const {2997 if (Arch != Triple::x86_64)2998 return true; // Conservative answer2999 3000 switch (R.getType()) {3001 default:3002 return true; // Conservative answer3003 3004 3005 case ELF::R_X86_64_GOTPCREL:3006 case ELF::R_X86_64_GOTPCRELX:3007 case ELF::R_X86_64_REX_GOTPCRELX:3008 case ELF::R_X86_64_GOTPC64:3009 case ELF::R_X86_64_GOT64:3010 case ELF::R_X86_64_GOTOFF64:3011 case ELF::R_X86_64_PC32:3012 case ELF::R_X86_64_PC64:3013 case ELF::R_X86_64_64:3014 // We know that these reloation types won't need a stub function. This list3015 // can be extended as needed.3016 return false;3017 }3018}3019 3020} // namespace llvm3021