453 lines · cpp
1//===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"10#include "llvm/ExecutionEngine/JITLink/x86_64.h"11#include "llvm/ExecutionEngine/Orc/OrcABISupport.h"12#include "llvm/IR/IRBuilder.h"13#include "llvm/IR/Module.h"14#include "llvm/MC/MCDisassembler/MCDisassembler.h"15#include "llvm/MC/MCInstrAnalysis.h"16#include "llvm/TargetParser/Triple.h"17#include "llvm/Transforms/Utils/Cloning.h"18 19#define DEBUG_TYPE "orc"20 21using namespace llvm;22using namespace llvm::orc;23 24namespace {25 26class CompileCallbackMaterializationUnit : public orc::MaterializationUnit {27public:28 using CompileFunction = JITCompileCallbackManager::CompileFunction;29 30 CompileCallbackMaterializationUnit(SymbolStringPtr Name,31 CompileFunction Compile)32 : MaterializationUnit(Interface(33 SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), nullptr)),34 Name(std::move(Name)), Compile(std::move(Compile)) {}35 36 StringRef getName() const override { return "<Compile Callbacks>"; }37 38private:39 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {40 SymbolMap Result;41 Result[Name] = {Compile(), JITSymbolFlags::Exported};42 // No dependencies, so these calls cannot fail.43 cantFail(R->notifyResolved(Result));44 cantFail(R->notifyEmitted({}));45 }46 47 void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {48 llvm_unreachable("Discard should never occur on a LMU?");49 }50 51 SymbolStringPtr Name;52 CompileFunction Compile;53};54 55} // namespace56 57namespace llvm {58namespace orc {59 60TrampolinePool::~TrampolinePool() = default;61void IndirectStubsManager::anchor() {}62 63Expected<ExecutorAddr>64JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) {65 if (auto TrampolineAddr = TP->getTrampoline()) {66 auto CallbackName =67 ES.intern(std::string("cc") + std::to_string(++NextCallbackId));68 69 std::lock_guard<std::mutex> Lock(CCMgrMutex);70 AddrToSymbol[*TrampolineAddr] = CallbackName;71 cantFail(72 CallbacksJD.define(std::make_unique<CompileCallbackMaterializationUnit>(73 std::move(CallbackName), std::move(Compile))));74 return *TrampolineAddr;75 } else76 return TrampolineAddr.takeError();77}78 79ExecutorAddr80JITCompileCallbackManager::executeCompileCallback(ExecutorAddr TrampolineAddr) {81 SymbolStringPtr Name;82 83 {84 std::unique_lock<std::mutex> Lock(CCMgrMutex);85 auto I = AddrToSymbol.find(TrampolineAddr);86 87 // If this address is not associated with a compile callback then report an88 // error to the execution session and return ErrorHandlerAddress to the89 // callee.90 if (I == AddrToSymbol.end()) {91 Lock.unlock();92 ES.reportError(93 make_error<StringError>("No compile callback for trampoline at " +94 formatv("{0:x}", TrampolineAddr),95 inconvertibleErrorCode()));96 return ErrorHandlerAddress;97 } else98 Name = I->second;99 }100 101 if (auto Sym =102 ES.lookup(makeJITDylibSearchOrder(103 &CallbacksJD, JITDylibLookupFlags::MatchAllSymbols),104 Name))105 return Sym->getAddress();106 else {107 llvm::dbgs() << "Didn't find callback.\n";108 // If anything goes wrong materializing Sym then report it to the session109 // and return the ErrorHandlerAddress;110 ES.reportError(Sym.takeError());111 return ErrorHandlerAddress;112 }113}114 115Error IndirectStubsManager::redirect(JITDylib &JD, const SymbolMap &NewDests) {116 for (auto &[Name, Dest] : NewDests)117 if (auto Err = updatePointer(*Name, Dest.getAddress()))118 return Err;119 return Error::success();120}121 122void IndirectStubsManager::emitRedirectableSymbols(123 std::unique_ptr<MaterializationResponsibility> MR, SymbolMap InitialDests) {124 StubInitsMap StubInits;125 for (auto &[Name, Dest] : InitialDests)126 StubInits[*Name] = {Dest.getAddress(), Dest.getFlags()};127 if (auto Err = createStubs(StubInits)) {128 MR->getExecutionSession().reportError(std::move(Err));129 return MR->failMaterialization();130 }131 SymbolMap Stubs;132 for (auto &[Name, Dest] : InitialDests) {133 auto StubSym = findStub(*Name, false);134 assert(StubSym.getAddress() && "Stub symbol should be present");135 Stubs[Name] = StubSym;136 }137 if (auto Err = MR->notifyResolved(Stubs)) {138 MR->getExecutionSession().reportError(std::move(Err));139 return MR->failMaterialization();140 }141 if (auto Err = MR->notifyEmitted({})) {142 MR->getExecutionSession().reportError(std::move(Err));143 return MR->failMaterialization();144 }145}146 147Expected<std::unique_ptr<JITCompileCallbackManager>>148createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,149 ExecutorAddr ErrorHandlerAddress) {150 switch (T.getArch()) {151 default:152 return make_error<StringError>(153 std::string("No callback manager available for ") + T.str(),154 inconvertibleErrorCode());155 case Triple::aarch64:156 case Triple::aarch64_32: {157 typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT;158 return CCMgrT::Create(ES, ErrorHandlerAddress);159 }160 161 case Triple::x86: {162 typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT;163 return CCMgrT::Create(ES, ErrorHandlerAddress);164 }165 166 case Triple::loongarch64: {167 typedef orc::LocalJITCompileCallbackManager<orc::OrcLoongArch64> CCMgrT;168 return CCMgrT::Create(ES, ErrorHandlerAddress);169 }170 171 case Triple::mips: {172 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT;173 return CCMgrT::Create(ES, ErrorHandlerAddress);174 }175 case Triple::mipsel: {176 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Le> CCMgrT;177 return CCMgrT::Create(ES, ErrorHandlerAddress);178 }179 180 case Triple::mips64:181 case Triple::mips64el: {182 typedef orc::LocalJITCompileCallbackManager<orc::OrcMips64> CCMgrT;183 return CCMgrT::Create(ES, ErrorHandlerAddress);184 }185 186 case Triple::riscv64: {187 typedef orc::LocalJITCompileCallbackManager<orc::OrcRiscv64> CCMgrT;188 return CCMgrT::Create(ES, ErrorHandlerAddress);189 }190 191 case Triple::x86_64: {192 if (T.getOS() == Triple::OSType::Win32) {193 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT;194 return CCMgrT::Create(ES, ErrorHandlerAddress);195 } else {196 typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT;197 return CCMgrT::Create(ES, ErrorHandlerAddress);198 }199 }200 201 }202}203 204std::function<std::unique_ptr<IndirectStubsManager>()>205createLocalIndirectStubsManagerBuilder(const Triple &T) {206 switch (T.getArch()) {207 default:208 return [](){209 return std::make_unique<210 orc::LocalIndirectStubsManager<orc::OrcGenericABI>>();211 };212 213 case Triple::aarch64:214 case Triple::aarch64_32:215 return [](){216 return std::make_unique<217 orc::LocalIndirectStubsManager<orc::OrcAArch64>>();218 };219 220 case Triple::x86:221 return [](){222 return std::make_unique<223 orc::LocalIndirectStubsManager<orc::OrcI386>>();224 };225 226 case Triple::loongarch64:227 return []() {228 return std::make_unique<229 orc::LocalIndirectStubsManager<orc::OrcLoongArch64>>();230 };231 232 case Triple::mips:233 return [](){234 return std::make_unique<235 orc::LocalIndirectStubsManager<orc::OrcMips32Be>>();236 };237 238 case Triple::mipsel:239 return [](){240 return std::make_unique<241 orc::LocalIndirectStubsManager<orc::OrcMips32Le>>();242 };243 244 case Triple::mips64:245 case Triple::mips64el:246 return [](){247 return std::make_unique<248 orc::LocalIndirectStubsManager<orc::OrcMips64>>();249 };250 251 case Triple::riscv64:252 return []() {253 return std::make_unique<254 orc::LocalIndirectStubsManager<orc::OrcRiscv64>>();255 };256 257 case Triple::x86_64:258 if (T.getOS() == Triple::OSType::Win32) {259 return [](){260 return std::make_unique<261 orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>();262 };263 } else {264 return [](){265 return std::make_unique<266 orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>();267 };268 }269 270 }271}272 273Constant* createIRTypedAddress(FunctionType &FT, ExecutorAddr Addr) {274 Constant *AddrIntVal =275 ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr.getValue());276 Constant *AddrPtrVal = ConstantExpr::getIntToPtr(277 AddrIntVal, PointerType::get(FT.getContext(), 0));278 return AddrPtrVal;279}280 281GlobalVariable* createImplPointer(PointerType &PT, Module &M,282 const Twine &Name, Constant *Initializer) {283 auto IP = new GlobalVariable(M, &PT, false, GlobalValue::ExternalLinkage,284 Initializer, Name, nullptr,285 GlobalValue::NotThreadLocal, 0, true);286 IP->setVisibility(GlobalValue::HiddenVisibility);287 return IP;288}289 290void makeStub(Function &F, Value &ImplPointer) {291 assert(F.isDeclaration() && "Can't turn a definition into a stub.");292 assert(F.getParent() && "Function isn't in a module.");293 Module &M = *F.getParent();294 BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F);295 IRBuilder<> Builder(EntryBlock);296 LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer);297 std::vector<Value*> CallArgs;298 for (auto &A : F.args())299 CallArgs.push_back(&A);300 CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs);301 Call->setTailCall();302 Call->setAttributes(F.getAttributes());303 if (F.getReturnType()->isVoidTy())304 Builder.CreateRetVoid();305 else306 Builder.CreateRet(Call);307}308 309std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) {310 std::vector<GlobalValue *> PromotedGlobals;311 312 for (auto &GV : M.global_values()) {313 bool Promoted = true;314 315 // Rename if necessary.316 if (!GV.hasName())317 GV.setName("__orc_anon." + Twine(NextId++));318 else if (GV.getName().starts_with("\01L"))319 GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++));320 else if (GV.hasLocalLinkage())321 GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++));322 else323 Promoted = false;324 325 if (GV.hasLocalLinkage()) {326 GV.setLinkage(GlobalValue::ExternalLinkage);327 GV.setVisibility(GlobalValue::HiddenVisibility);328 Promoted = true;329 }330 GV.setUnnamedAddr(GlobalValue::UnnamedAddr::None);331 332 if (Promoted)333 PromotedGlobals.push_back(&GV);334 }335 336 return PromotedGlobals;337}338 339Function* cloneFunctionDecl(Module &Dst, const Function &F,340 ValueToValueMapTy *VMap) {341 Function *NewF =342 Function::Create(cast<FunctionType>(F.getValueType()),343 F.getLinkage(), F.getName(), &Dst);344 NewF->copyAttributesFrom(&F);345 346 if (VMap) {347 (*VMap)[&F] = NewF;348 auto NewArgI = NewF->arg_begin();349 for (auto ArgI = F.arg_begin(), ArgE = F.arg_end(); ArgI != ArgE;350 ++ArgI, ++NewArgI)351 (*VMap)[&*ArgI] = &*NewArgI;352 }353 354 return NewF;355}356 357GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,358 ValueToValueMapTy *VMap) {359 GlobalVariable *NewGV = new GlobalVariable(360 Dst, GV.getValueType(), GV.isConstant(),361 GV.getLinkage(), nullptr, GV.getName(), nullptr,362 GV.getThreadLocalMode(), GV.getType()->getAddressSpace());363 NewGV->copyAttributesFrom(&GV);364 if (VMap)365 (*VMap)[&GV] = NewGV;366 return NewGV;367}368 369GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,370 ValueToValueMapTy &VMap) {371 assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?");372 auto *NewA = GlobalAlias::create(OrigA.getValueType(),373 OrigA.getType()->getPointerAddressSpace(),374 OrigA.getLinkage(), OrigA.getName(), &Dst);375 NewA->copyAttributesFrom(&OrigA);376 VMap[&OrigA] = NewA;377 return NewA;378}379 380Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym,381 jitlink::LinkGraph &G,382 MCDisassembler &Disassembler,383 MCInstrAnalysis &MIA) {384 // AArch64 appears to already come with the necessary relocations. Among other385 // architectures, only x86_64 is currently implemented here.386 if (G.getTargetTriple().getArch() != Triple::x86_64)387 return Error::success();388 389 raw_null_ostream CommentStream;390 auto &STI = Disassembler.getSubtargetInfo();391 392 // Determine the function bounds393 auto &B = Sym.getBlock();394 assert(!B.isZeroFill() && "expected content block");395 auto SymAddress = Sym.getAddress();396 auto SymStartInBlock =397 (const uint8_t *)B.getContent().data() + Sym.getOffset();398 auto SymSize = Sym.getSize() ? Sym.getSize() : B.getSize() - Sym.getOffset();399 auto Content = ArrayRef(SymStartInBlock, SymSize);400 401 LLVM_DEBUG(dbgs() << "Adding self-relocations to " << Sym.getName() << "\n");402 403 SmallDenseSet<uintptr_t, 8> ExistingRelocations;404 for (auto &E : B.edges()) {405 if (E.isRelocation())406 ExistingRelocations.insert(E.getOffset());407 }408 409 size_t I = 0;410 while (I < Content.size()) {411 MCInst Instr;412 uint64_t InstrSize = 0;413 uint64_t InstrStart = SymAddress.getValue() + I;414 auto DecodeStatus = Disassembler.getInstruction(415 Instr, InstrSize, Content.drop_front(I), InstrStart, CommentStream);416 if (DecodeStatus != MCDisassembler::Success) {417 LLVM_DEBUG(dbgs() << "Aborting due to disassembly failure at address "418 << InstrStart);419 return make_error<StringError>(420 formatv("failed to disassemble at address {0:x16}", InstrStart),421 inconvertibleErrorCode());422 }423 // Advance to the next instruction.424 I += InstrSize;425 426 // Check for a PC-relative address equal to the symbol itself.427 auto PCRelAddr =428 MIA.evaluateMemoryOperandAddress(Instr, &STI, InstrStart, InstrSize);429 if (!PCRelAddr || *PCRelAddr != SymAddress.getValue())430 continue;431 432 auto RelocOffInInstr =433 MIA.getMemoryOperandRelocationOffset(Instr, InstrSize);434 if (!RelocOffInInstr || InstrSize - *RelocOffInInstr != 4) {435 LLVM_DEBUG(dbgs() << "Skipping unknown self-relocation at "436 << InstrStart);437 continue;438 }439 440 auto RelocOffInBlock = orc::ExecutorAddr(InstrStart) + *RelocOffInInstr -441 SymAddress + Sym.getOffset();442 if (ExistingRelocations.contains(RelocOffInBlock))443 continue;444 445 LLVM_DEBUG(dbgs() << "Adding delta32 self-relocation at " << InstrStart);446 B.addEdge(jitlink::x86_64::Delta32, RelocOffInBlock, Sym, /*Addend=*/-4);447 }448 return Error::success();449}450 451} // End namespace orc.452} // End namespace llvm.453