507 lines · cpp
1//===--- ELF_loongarch.cpp - JIT linker implementation for ELF/loongarch --===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// ELF/loongarch jit-link implementation.10//11//===----------------------------------------------------------------------===//12 13#include "llvm/ExecutionEngine/JITLink/ELF_loongarch.h"14#include "llvm/BinaryFormat/ELF.h"15#include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h"16#include "llvm/ExecutionEngine/JITLink/JITLink.h"17#include "llvm/ExecutionEngine/JITLink/loongarch.h"18#include "llvm/Object/ELF.h"19#include "llvm/Object/ELFObjectFile.h"20 21#include "EHFrameSupportImpl.h"22#include "ELFLinkGraphBuilder.h"23#include "JITLinkGeneric.h"24 25#define DEBUG_TYPE "jitlink"26 27using namespace llvm;28using namespace llvm::jitlink;29using namespace llvm::jitlink::loongarch;30 31namespace {32 33class ELFJITLinker_loongarch : public JITLinker<ELFJITLinker_loongarch> {34 friend class JITLinker<ELFJITLinker_loongarch>;35 36public:37 ELFJITLinker_loongarch(std::unique_ptr<JITLinkContext> Ctx,38 std::unique_ptr<LinkGraph> G,39 PassConfiguration PassConfig)40 : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {}41 42private:43 Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const {44 return loongarch::applyFixup(G, B, E);45 }46};47 48namespace {49 50struct SymbolAnchor {51 uint64_t Offset;52 Symbol *Sym;53 bool End; // true for the anchor of getOffset() + getSize()54};55 56struct BlockRelaxAux {57 // This records symbol start and end offsets which will be adjusted according58 // to the nearest RelocDeltas element.59 SmallVector<SymbolAnchor, 0> Anchors;60 // All edges that either 1) are R_LARCH_ALIGN or 2) have a R_LARCH_RELAX edge61 // at the same offset.62 SmallVector<Edge *, 0> RelaxEdges;63 // For RelaxEdges[I], the actual offset is RelaxEdges[I]->getOffset() - (I ?64 // RelocDeltas[I - 1] : 0).65 SmallVector<uint32_t, 0> RelocDeltas;66 // For RelaxEdges[I], the actual type is EdgeKinds[I].67 SmallVector<Edge::Kind, 0> EdgeKinds;68 // List of rewritten instructions. Contains one raw encoded instruction per69 // element in EdgeKinds that isn't Invalid or R_LARCH_ALIGN.70 SmallVector<uint32_t, 0> Writes;71};72 73struct RelaxAux {74 DenseMap<Block *, BlockRelaxAux> Blocks;75};76 77} // namespace78 79static bool shouldRelax(const Section &S) {80 return (S.getMemProt() & orc::MemProt::Exec) != orc::MemProt::None;81}82 83static bool isRelaxable(const Edge &E) {84 switch (E.getKind()) {85 default:86 return false;87 case AlignRelaxable:88 return true;89 }90}91 92static RelaxAux initRelaxAux(LinkGraph &G) {93 RelaxAux Aux;94 for (auto &S : G.sections()) {95 if (!shouldRelax(S))96 continue;97 for (auto *B : S.blocks()) {98 auto BlockEmplaceResult = Aux.Blocks.try_emplace(B);99 assert(BlockEmplaceResult.second && "Block encountered twice");100 auto &BlockAux = BlockEmplaceResult.first->second;101 102 for (auto &E : B->edges())103 if (isRelaxable(E))104 BlockAux.RelaxEdges.push_back(&E);105 106 if (BlockAux.RelaxEdges.empty()) {107 Aux.Blocks.erase(BlockEmplaceResult.first);108 continue;109 }110 111 const auto NumEdges = BlockAux.RelaxEdges.size();112 BlockAux.RelocDeltas.resize(NumEdges, 0);113 BlockAux.EdgeKinds.resize_for_overwrite(NumEdges);114 115 // Store anchors (offset and offset+size) for symbols.116 for (auto *Sym : S.symbols()) {117 if (!Sym->isDefined() || &Sym->getBlock() != B)118 continue;119 120 BlockAux.Anchors.push_back({Sym->getOffset(), Sym, false});121 BlockAux.Anchors.push_back(122 {Sym->getOffset() + Sym->getSize(), Sym, true});123 }124 }125 }126 127 // Sort anchors by offset so that we can find the closest relocation128 // efficiently. For a zero size symbol, ensure that its start anchor precedes129 // its end anchor. For two symbols with anchors at the same offset, their130 // order does not matter.131 for (auto &BlockAuxIter : Aux.Blocks) {132 llvm::sort(BlockAuxIter.second.Anchors, [](auto &A, auto &B) {133 return std::make_pair(A.Offset, A.End) < std::make_pair(B.Offset, B.End);134 });135 }136 137 return Aux;138}139 140static void relaxAlign(orc::ExecutorAddr Loc, const Edge &E, uint32_t &Remove,141 Edge::Kind &NewEdgeKind) {142 const uint64_t Addend =143 !E.getTarget().isDefined() ? Log2_64(E.getAddend()) + 1 : E.getAddend();144 const uint64_t AllBytes = (1ULL << (Addend & 0xff)) - 4;145 const uint64_t Align = 1ULL << (Addend & 0xff);146 const uint64_t MaxBytes = Addend >> 8;147 const uint64_t Off = Loc.getValue() & (Align - 1);148 const uint64_t CurBytes = Off == 0 ? 0 : Align - Off;149 // All bytes beyond the alignment boundary should be removed.150 // If emit bytes more than max bytes to emit, remove all.151 if (MaxBytes != 0 && CurBytes > MaxBytes)152 Remove = AllBytes;153 else154 Remove = AllBytes - CurBytes;155 156 assert(static_cast<int32_t>(Remove) >= 0 &&157 "R_LARCH_ALIGN needs expanding the content");158 NewEdgeKind = AlignRelaxable;159}160 161static bool relaxBlock(LinkGraph &G, Block &Block, BlockRelaxAux &Aux) {162 const auto BlockAddr = Block.getAddress();163 bool Changed = false;164 ArrayRef<SymbolAnchor> SA = ArrayRef(Aux.Anchors);165 uint32_t Delta = 0;166 167 Aux.EdgeKinds.assign(Aux.EdgeKinds.size(), Edge::Invalid);168 Aux.Writes.clear();169 170 for (auto [I, E] : llvm::enumerate(Aux.RelaxEdges)) {171 const auto Loc = BlockAddr + E->getOffset() - Delta;172 auto &Cur = Aux.RelocDeltas[I];173 uint32_t Remove = 0;174 switch (E->getKind()) {175 case AlignRelaxable:176 relaxAlign(Loc, *E, Remove, Aux.EdgeKinds[I]);177 break;178 default:179 llvm_unreachable("Unexpected relaxable edge kind");180 }181 182 // For all anchors whose offsets are <= E->getOffset(), they are preceded by183 // the previous relocation whose RelocDeltas value equals Delta.184 // Decrease their offset and update their size.185 for (; SA.size() && SA[0].Offset <= E->getOffset(); SA = SA.slice(1)) {186 if (SA[0].End)187 SA[0].Sym->setSize(SA[0].Offset - Delta - SA[0].Sym->getOffset());188 else189 SA[0].Sym->setOffset(SA[0].Offset - Delta);190 }191 192 Delta += Remove;193 if (Delta != Cur) {194 Cur = Delta;195 Changed = true;196 }197 }198 199 for (const SymbolAnchor &A : SA) {200 if (A.End)201 A.Sym->setSize(A.Offset - Delta - A.Sym->getOffset());202 else203 A.Sym->setOffset(A.Offset - Delta);204 }205 206 return Changed;207}208 209static bool relaxOnce(LinkGraph &G, RelaxAux &Aux) {210 bool Changed = false;211 212 for (auto &[B, BlockAux] : Aux.Blocks)213 Changed |= relaxBlock(G, *B, BlockAux);214 215 return Changed;216}217 218static void finalizeBlockRelax(LinkGraph &G, Block &Block, BlockRelaxAux &Aux) {219 auto Contents = Block.getAlreadyMutableContent();220 auto *Dest = Contents.data();221 uint32_t Offset = 0;222 uint32_t Delta = 0;223 224 // Update section content: remove NOPs for R_LARCH_ALIGN and rewrite225 // instructions for relaxed relocations.226 for (auto [I, E] : llvm::enumerate(Aux.RelaxEdges)) {227 uint32_t Remove = Aux.RelocDeltas[I] - Delta;228 Delta = Aux.RelocDeltas[I];229 if (Remove == 0 && Aux.EdgeKinds[I] == Edge::Invalid)230 continue;231 232 // Copy from last location to the current relocated location.233 const auto Size = E->getOffset() - Offset;234 std::memmove(Dest, Contents.data() + Offset, Size);235 Dest += Size;236 Offset = E->getOffset() + Remove;237 }238 239 std::memmove(Dest, Contents.data() + Offset, Contents.size() - Offset);240 241 // Fixup edge offsets and kinds.242 Delta = 0;243 size_t I = 0;244 for (auto &E : Block.edges()) {245 E.setOffset(E.getOffset() - Delta);246 247 if (I < Aux.RelaxEdges.size() && Aux.RelaxEdges[I] == &E) {248 if (Aux.EdgeKinds[I] != Edge::Invalid)249 E.setKind(Aux.EdgeKinds[I]);250 251 Delta = Aux.RelocDeltas[I];252 ++I;253 }254 }255 256 // Remove AlignRelaxable edges: all other relaxable edges got modified and257 // will be used later while linking. Alignment is entirely handled here so we258 // don't need these edges anymore.259 for (auto IE = Block.edges().begin(); IE != Block.edges().end();) {260 if (IE->getKind() == AlignRelaxable)261 IE = Block.removeEdge(IE);262 else263 ++IE;264 }265}266 267static void finalizeRelax(LinkGraph &G, RelaxAux &Aux) {268 for (auto &[B, BlockAux] : Aux.Blocks)269 finalizeBlockRelax(G, *B, BlockAux);270}271 272static Error relax(LinkGraph &G) {273 auto Aux = initRelaxAux(G);274 while (relaxOnce(G, Aux)) {275 }276 finalizeRelax(G, Aux);277 return Error::success();278}279 280template <typename ELFT>281class ELFLinkGraphBuilder_loongarch : public ELFLinkGraphBuilder<ELFT> {282private:283 static Expected<loongarch::EdgeKind_loongarch>284 getRelocationKind(const uint32_t Type) {285 using namespace loongarch;286 switch (Type) {287 case ELF::R_LARCH_64:288 return Pointer64;289 case ELF::R_LARCH_32:290 return Pointer32;291 case ELF::R_LARCH_32_PCREL:292 return Delta32;293 case ELF::R_LARCH_B16:294 return Branch16PCRel;295 case ELF::R_LARCH_B21:296 return Branch21PCRel;297 case ELF::R_LARCH_B26:298 return Branch26PCRel;299 case ELF::R_LARCH_PCALA_HI20:300 return Page20;301 case ELF::R_LARCH_PCALA_LO12:302 return PageOffset12;303 case ELF::R_LARCH_GOT_PC_HI20:304 return RequestGOTAndTransformToPage20;305 case ELF::R_LARCH_GOT_PC_LO12:306 return RequestGOTAndTransformToPageOffset12;307 case ELF::R_LARCH_CALL36:308 return Call36PCRel;309 case ELF::R_LARCH_ADD6:310 return Add6;311 case ELF::R_LARCH_ADD8:312 return Add8;313 case ELF::R_LARCH_ADD16:314 return Add16;315 case ELF::R_LARCH_ADD32:316 return Add32;317 case ELF::R_LARCH_ADD64:318 return Add64;319 case ELF::R_LARCH_ADD_ULEB128:320 return AddUleb128;321 case ELF::R_LARCH_SUB6:322 return Sub6;323 case ELF::R_LARCH_SUB8:324 return Sub8;325 case ELF::R_LARCH_SUB16:326 return Sub16;327 case ELF::R_LARCH_SUB32:328 return Sub32;329 case ELF::R_LARCH_SUB64:330 return Sub64;331 case ELF::R_LARCH_SUB_ULEB128:332 return SubUleb128;333 case ELF::R_LARCH_ALIGN:334 return AlignRelaxable;335 }336 337 return make_error<JITLinkError>(338 "Unsupported loongarch relocation:" + formatv("{0:d}: ", Type) +339 object::getELFRelocationTypeName(ELF::EM_LOONGARCH, Type));340 }341 342 EdgeKind_loongarch getRelaxableRelocationKind(EdgeKind_loongarch Kind) {343 // TODO: Implement more. Just ignore all relaxations now.344 return Kind;345 }346 347 Error addRelocations() override {348 LLVM_DEBUG(dbgs() << "Processing relocations:\n");349 350 using Base = ELFLinkGraphBuilder<ELFT>;351 using Self = ELFLinkGraphBuilder_loongarch<ELFT>;352 for (const auto &RelSect : Base::Sections)353 if (Error Err = Base::forEachRelaRelocation(RelSect, this,354 &Self::addSingleRelocation))355 return Err;356 357 return Error::success();358 }359 360 Error addSingleRelocation(const typename ELFT::Rela &Rel,361 const typename ELFT::Shdr &FixupSect,362 Block &BlockToFix) {363 using Base = ELFLinkGraphBuilder<ELFT>;364 365 uint32_t Type = Rel.getType(false);366 int64_t Addend = Rel.r_addend;367 368 // ignore369 if (Type == ELF::R_LARCH_MARK_LA)370 return Error::success();371 372 if (Type == ELF::R_LARCH_RELAX) {373 if (BlockToFix.edges_empty())374 return make_error<StringError>(375 "R_LARCH_RELAX without preceding relocation",376 inconvertibleErrorCode());377 378 auto &PrevEdge = *std::prev(BlockToFix.edges().end());379 auto Kind = static_cast<EdgeKind_loongarch>(PrevEdge.getKind());380 PrevEdge.setKind(getRelaxableRelocationKind(Kind));381 return Error::success();382 }383 384 Expected<loongarch::EdgeKind_loongarch> Kind = getRelocationKind(Type);385 if (!Kind)386 return Kind.takeError();387 388 uint32_t SymbolIndex = Rel.getSymbol(false);389 auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec);390 if (!ObjSymbol)391 return ObjSymbol.takeError();392 393 Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex);394 if (!GraphSymbol)395 return make_error<StringError>(396 formatv("Could not find symbol at given index, did you add it to "397 "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}",398 SymbolIndex, (*ObjSymbol)->st_shndx,399 Base::GraphSymbols.size()),400 inconvertibleErrorCode());401 402 auto FixupAddress = orc::ExecutorAddr(FixupSect.sh_addr) + Rel.r_offset;403 Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress();404 Edge GE(*Kind, Offset, *GraphSymbol, Addend);405 LLVM_DEBUG({406 dbgs() << " ";407 printEdge(dbgs(), BlockToFix, GE, loongarch::getEdgeKindName(*Kind));408 dbgs() << "\n";409 });410 411 BlockToFix.addEdge(std::move(GE));412 413 return Error::success();414 }415 416public:417 ELFLinkGraphBuilder_loongarch(StringRef FileName,418 const object::ELFFile<ELFT> &Obj,419 std::shared_ptr<orc::SymbolStringPool> SSP,420 Triple TT, SubtargetFeatures Features)421 : ELFLinkGraphBuilder<ELFT>(Obj, std::move(SSP), std::move(TT),422 std::move(Features), FileName,423 loongarch::getEdgeKindName) {}424};425 426Error buildTables_ELF_loongarch(LinkGraph &G) {427 LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n");428 429 GOTTableManager GOT;430 PLTTableManager PLT(GOT);431 visitExistingEdges(G, GOT, PLT);432 return Error::success();433}434 435} // namespace436 437namespace llvm {438namespace jitlink {439 440Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromELFObject_loongarch(441 MemoryBufferRef ObjectBuffer, std::shared_ptr<orc::SymbolStringPool> SSP) {442 LLVM_DEBUG({443 dbgs() << "Building jitlink graph for new input "444 << ObjectBuffer.getBufferIdentifier() << "...\n";445 });446 447 auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer);448 if (!ELFObj)449 return ELFObj.takeError();450 451 auto Features = (*ELFObj)->getFeatures();452 if (!Features)453 return Features.takeError();454 455 if ((*ELFObj)->getArch() == Triple::loongarch64) {456 auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);457 return ELFLinkGraphBuilder_loongarch<object::ELF64LE>(458 (*ELFObj)->getFileName(), ELFObjFile.getELFFile(),459 std::move(SSP), (*ELFObj)->makeTriple(), std::move(*Features))460 .buildGraph();461 }462 463 assert((*ELFObj)->getArch() == Triple::loongarch32 &&464 "Invalid triple for LoongArch ELF object file");465 auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF32LE>>(**ELFObj);466 return ELFLinkGraphBuilder_loongarch<object::ELF32LE>(467 (*ELFObj)->getFileName(), ELFObjFile.getELFFile(), std::move(SSP),468 (*ELFObj)->makeTriple(), std::move(*Features))469 .buildGraph();470}471 472void link_ELF_loongarch(std::unique_ptr<LinkGraph> G,473 std::unique_ptr<JITLinkContext> Ctx) {474 PassConfiguration Config;475 const Triple &TT = G->getTargetTriple();476 if (Ctx->shouldAddDefaultTargetPasses(TT)) {477 // Add eh-frame passes.478 Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame"));479 Config.PrePrunePasses.push_back(480 EHFrameEdgeFixer(".eh_frame", G->getPointerSize(), Pointer32, Pointer64,481 Delta32, Delta64, NegDelta32));482 Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame"));483 484 // Add a mark-live pass.485 if (auto MarkLive = Ctx->getMarkLivePass(TT))486 Config.PrePrunePasses.push_back(std::move(MarkLive));487 else488 Config.PrePrunePasses.push_back(markAllSymbolsLive);489 490 // Add an in-place GOT/PLTStubs build pass.491 Config.PostPrunePasses.push_back(buildTables_ELF_loongarch);492 493 // Add a linker relaxation pass.494 Config.PostAllocationPasses.push_back(relax);495 }496 497 if (auto Err = Ctx->modifyPassConfig(*G, Config))498 return Ctx->notifyFailed(std::move(Err));499 500 ELFJITLinker_loongarch::link(std::move(Ctx), std::move(G), std::move(Config));501}502 503LinkGraphPassFunction createRelaxationPass_ELF_loongarch() { return relax; }504 505} // namespace jitlink506} // namespace llvm507