230 lines · cpp
1//===- bolt/Rewrite/ExecutableFileMemoryManager.cpp -----------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "bolt/Rewrite/ExecutableFileMemoryManager.h"10#include "bolt/Rewrite/JITLinkLinker.h"11#include "bolt/Rewrite/RewriteInstance.h"12#include "llvm/ExecutionEngine/JITLink/JITLink.h"13#include "llvm/Support/MemAlloc.h"14 15#undef DEBUG_TYPE16#define DEBUG_TYPE "efmm"17 18using namespace llvm;19using namespace object;20using namespace bolt;21 22namespace llvm {23 24namespace bolt {25 26namespace {27 28SmallVector<jitlink::Section *> orderedSections(jitlink::LinkGraph &G) {29 SmallVector<jitlink::Section *> Sections(30 llvm::map_range(G.sections(), [](auto &S) { return &S; }));31 llvm::sort(Sections, [](const auto *LHS, const auto *RHS) {32 return LHS->getOrdinal() < RHS->getOrdinal();33 });34 return Sections;35}36 37size_t sectionAlignment(const jitlink::Section &Section) {38 assert(!Section.empty() && "Cannot get alignment for empty section");39 return JITLinkLinker::orderedBlocks(Section).front()->getAlignment();40}41 42StringRef sectionName(const jitlink::Section &Section,43 const BinaryContext &BC) {44 auto Name = Section.getName();45 46 if (BC.isMachO()) {47 // JITLink "normalizes" section names as "SegmentName,SectionName" on48 // Mach-O. BOLT internally refers to sections just by the section name so49 // strip-off the segment name.50 auto SegmentEnd = Name.find(',');51 assert(SegmentEnd != StringRef::npos && "Mach-O segment not found");52 Name = Name.substr(SegmentEnd + 1);53 }54 55 return Name;56}57 58struct SectionAllocInfo {59 void *Address;60 size_t Size;61 size_t Alignment;62};63 64struct AllocInfo {65 SmallVector<SectionAllocInfo, 8> AllocatedSections;66 67 ~AllocInfo() {68 for (auto &Section : AllocatedSections)69 deallocate_buffer(Section.Address, Section.Size, Section.Alignment);70 }71 72 SectionAllocInfo allocateSection(const jitlink::Section &Section) {73 auto Size = JITLinkLinker::sectionSize(Section);74 auto Alignment = sectionAlignment(Section);75 auto *Buf = allocate_buffer(Size, Alignment);76 SectionAllocInfo Alloc{Buf, Size, Alignment};77 AllocatedSections.push_back(Alloc);78 return Alloc;79 }80};81 82struct BOLTInFlightAlloc : ExecutableFileMemoryManager::InFlightAlloc {83 // Even though this is passed using a raw pointer in FinalizedAlloc, we keep84 // it in a unique_ptr as long as possible to enjoy automatic cleanup when85 // something goes wrong.86 std::unique_ptr<AllocInfo> Alloc;87 88public:89 BOLTInFlightAlloc(std::unique_ptr<AllocInfo> Alloc)90 : Alloc(std::move(Alloc)) {}91 92 virtual void abandon(OnAbandonedFunction OnAbandoned) override {93 OnAbandoned(Error::success());94 }95 96 virtual void finalize(OnFinalizedFunction OnFinalized) override {97 OnFinalized(ExecutableFileMemoryManager::FinalizedAlloc(98 orc::ExecutorAddr::fromPtr(Alloc.release())));99 }100};101 102} // anonymous namespace103 104void ExecutableFileMemoryManager::updateSection(105 const jitlink::Section &JLSection, uint8_t *Contents, size_t Size,106 size_t Alignment) {107 auto SectionID = JLSection.getName();108 auto SectionName = sectionName(JLSection, BC);109 auto Prot = JLSection.getMemProt();110 auto IsCode = (Prot & orc::MemProt::Exec) != orc::MemProt::None;111 auto IsReadOnly = (Prot & orc::MemProt::Write) == orc::MemProt::None;112 113 // Register a debug section as a note section.114 if (!ObjectsLoaded && RewriteInstance::isDebugSection(SectionName)) {115 BinarySection &Section =116 BC.registerOrUpdateNoteSection(SectionName, Contents, Size, Alignment);117 Section.setSectionID(SectionID);118 assert(!Section.isAllocatable() && "note sections cannot be allocatable");119 return;120 }121 122 if (!IsCode && (SectionName == ".strtab" || SectionName == ".symtab" ||123 SectionName == "" || SectionName.starts_with(".rela.")))124 return;125 126 SmallVector<char, 256> Buf;127 if (ObjectsLoaded > 0) {128 if (BC.isELF()) {129 SectionName = (Twine(SectionName) + ".bolt.extra." + Twine(ObjectsLoaded))130 .toStringRef(Buf);131 } else if (BC.isMachO()) {132 assert((SectionName == "__text" || SectionName == "__data" ||133 SectionName == "__fini" || SectionName == "__setup" ||134 SectionName == "__cstring" || SectionName == "__literal16") &&135 "Unexpected section in the instrumentation library");136 // Sections coming from the instrumentation runtime are prefixed with "I".137 SectionName = ("I" + Twine(SectionName)).toStringRef(Buf);138 }139 }140 141 BinarySection *Section = nullptr;142 if (!OrgSecPrefix.empty() && SectionName.starts_with(OrgSecPrefix)) {143 // Update the original section contents.144 ErrorOr<BinarySection &> OrgSection =145 BC.getUniqueSectionByName(SectionName.substr(OrgSecPrefix.length()));146 assert(OrgSection && OrgSection->isAllocatable() &&147 "Original section must exist and be allocatable.");148 149 Section = &OrgSection.get();150 Section->updateContents(Contents, Size);151 } else {152 // If the input contains a section with the section name, rename it in the153 // output file to avoid the section name conflict and emit the new section154 // under a unique internal name.155 ErrorOr<BinarySection &> OrgSection =156 BC.getUniqueSectionByName(SectionName);157 bool UsePrefix = false;158 if (OrgSection && OrgSection->hasSectionRef()) {159 OrgSection->setOutputName(OrgSecPrefix + SectionName);160 UsePrefix = true;161 }162 163 // Register the new section under a unique name to avoid name collision with164 // sections in the input file.165 BinarySection &NewSection = BC.registerOrUpdateSection(166 UsePrefix ? NewSecPrefix + SectionName : SectionName, ELF::SHT_PROGBITS,167 BinarySection::getFlags(IsReadOnly, IsCode, true), Contents, Size,168 Alignment);169 if (UsePrefix)170 NewSection.setOutputName(SectionName);171 Section = &NewSection;172 }173 174 LLVM_DEBUG({175 dbgs() << "BOLT: allocating "176 << (IsCode ? "code" : (IsReadOnly ? "read-only data" : "data"))177 << " section : " << Section->getOutputName() << " ("178 << Section->getName() << ")"179 << " with size " << Size << ", alignment " << Alignment << " at "180 << Contents << ", ID = " << SectionID << "\n";181 });182 183 Section->setSectionID(SectionID);184}185 186void ExecutableFileMemoryManager::allocate(const jitlink::JITLinkDylib *JD,187 jitlink::LinkGraph &G,188 OnAllocatedFunction OnAllocated) {189 auto Alloc = std::make_unique<AllocInfo>();190 191 for (auto *Section : orderedSections(G)) {192 if (Section->empty())193 continue;194 195 auto SectionAlloc = Alloc->allocateSection(*Section);196 updateSection(*Section, static_cast<uint8_t *>(SectionAlloc.Address),197 SectionAlloc.Size, SectionAlloc.Alignment);198 199 size_t CurrentOffset = 0;200 auto *Buf = static_cast<char *>(SectionAlloc.Address);201 for (auto *Block : JITLinkLinker::orderedBlocks(*Section)) {202 CurrentOffset = jitlink::alignToBlock(CurrentOffset, *Block);203 auto BlockSize = Block->getSize();204 auto *BlockBuf = Buf + CurrentOffset;205 206 if (Block->isZeroFill())207 std::memset(BlockBuf, 0, BlockSize);208 else209 std::memcpy(BlockBuf, Block->getContent().data(), BlockSize);210 211 Block->setMutableContent({BlockBuf, Block->getSize()});212 CurrentOffset += BlockSize;213 }214 }215 216 OnAllocated(std::make_unique<BOLTInFlightAlloc>(std::move(Alloc)));217}218 219void ExecutableFileMemoryManager::deallocate(220 std::vector<FinalizedAlloc> Allocs, OnDeallocatedFunction OnDeallocated) {221 for (auto &Alloc : Allocs)222 delete Alloc.release().toPtr<AllocInfo *>();223 224 OnDeallocated(Error::success());225}226 227} // namespace bolt228 229} // namespace llvm230