591 lines · cpp
1//===- bolt/Rewrite/MachORewriteInstance.cpp - MachO rewriter -------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "bolt/Rewrite/MachORewriteInstance.h"10#include "bolt/Core/BinaryContext.h"11#include "bolt/Core/BinaryEmitter.h"12#include "bolt/Core/BinaryFunction.h"13#include "bolt/Core/JumpTable.h"14#include "bolt/Core/MCPlusBuilder.h"15#include "bolt/Passes/Instrumentation.h"16#include "bolt/Passes/PatchEntries.h"17#include "bolt/Profile/DataReader.h"18#include "bolt/Rewrite/BinaryPassManager.h"19#include "bolt/Rewrite/ExecutableFileMemoryManager.h"20#include "bolt/Rewrite/JITLinkLinker.h"21#include "bolt/Rewrite/RewriteInstance.h"22#include "bolt/RuntimeLibs/InstrumentationRuntimeLibrary.h"23#include "bolt/Utils/CommandLineOpts.h"24#include "bolt/Utils/Utils.h"25#include "llvm/MC/MCObjectStreamer.h"26#include "llvm/Support/Errc.h"27#include "llvm/Support/FileSystem.h"28#include "llvm/Support/ToolOutputFile.h"29#include <memory>30#include <optional>31 32namespace opts {33 34using namespace llvm;35extern cl::opt<unsigned> AlignText;36// FIXME! Upstream change37// extern cl::opt<bool> CheckOverlappingElements;38extern cl::opt<bool> Instrument;39extern cl::opt<bool> InstrumentCalls;40extern cl::opt<bolt::JumpTableSupportLevel> JumpTables;41extern cl::opt<bool> KeepTmp;42extern cl::opt<bool> NeverPrint;43extern cl::opt<std::string> OutputFilename;44extern cl::opt<bool> PrintAfterBranchFixup;45extern cl::opt<bool> PrintFinalized;46extern cl::opt<bool> PrintNormalized;47extern cl::opt<bool> PrintReordered;48extern cl::opt<bool> PrintSections;49extern cl::opt<bool> PrintDisasm;50extern cl::opt<bool> PrintCFG;51extern cl::opt<std::string> RuntimeInstrumentationLib;52extern cl::opt<unsigned> Verbosity;53} // namespace opts54 55namespace llvm {56namespace bolt {57 58#define DEBUG_TYPE "bolt"59 60Expected<std::unique_ptr<MachORewriteInstance>>61MachORewriteInstance::create(object::MachOObjectFile *InputFile,62 StringRef ToolPath) {63 Error Err = Error::success();64 auto MachORI =65 std::make_unique<MachORewriteInstance>(InputFile, ToolPath, Err);66 if (Err)67 return std::move(Err);68 return std::move(MachORI);69}70 71MachORewriteInstance::MachORewriteInstance(object::MachOObjectFile *InputFile,72 StringRef ToolPath, Error &Err)73 : InputFile(InputFile), ToolPath(ToolPath) {74 ErrorAsOutParameter EAO(&Err);75 Relocation::Arch = InputFile->makeTriple().getArch();76 auto BCOrErr = BinaryContext::createBinaryContext(77 InputFile->makeTriple(), std::make_shared<orc::SymbolStringPool>(),78 InputFile->getFileName(), nullptr,79 /* IsPIC */ true, DWARFContext::create(*InputFile),80 {llvm::outs(), llvm::errs()});81 if (Error E = BCOrErr.takeError()) {82 Err = std::move(E);83 return;84 }85 BC = std::move(BCOrErr.get());86 BC->initializeTarget(std::unique_ptr<MCPlusBuilder>(87 createMCPlusBuilder(BC->TheTriple->getArch(), BC->MIA.get(),88 BC->MII.get(), BC->MRI.get(), BC->STI.get())));89 if (opts::Instrument)90 BC->setRuntimeLibrary(std::make_unique<InstrumentationRuntimeLibrary>());91}92 93Error MachORewriteInstance::setProfile(StringRef Filename) {94 if (!sys::fs::exists(Filename))95 return errorCodeToError(make_error_code(errc::no_such_file_or_directory));96 97 if (ProfileReader) {98 // Already exists99 return make_error<StringError>(100 Twine("multiple profiles specified: ") + ProfileReader->getFilename() +101 " and " + Filename, inconvertibleErrorCode());102 }103 104 ProfileReader = std::make_unique<DataReader>(Filename);105 return Error::success();106}107 108void MachORewriteInstance::preprocessProfileData() {109 if (!ProfileReader)110 return;111 if (Error E = ProfileReader->preprocessProfile(*BC))112 report_error("cannot pre-process profile", std::move(E));113}114 115void MachORewriteInstance::processProfileDataPreCFG() {116 if (!ProfileReader)117 return;118 if (Error E = ProfileReader->readProfilePreCFG(*BC))119 report_error("cannot read profile pre-CFG", std::move(E));120}121 122void MachORewriteInstance::processProfileData() {123 if (!ProfileReader)124 return;125 if (Error E = ProfileReader->readProfile(*BC))126 report_error("cannot read profile", std::move(E));127}128 129void MachORewriteInstance::readSpecialSections() {130 for (const object::SectionRef &Section : InputFile->sections()) {131 Expected<StringRef> SectionName = Section.getName();;132 check_error(SectionName.takeError(), "cannot get section name");133 // Only register sections with names.134 if (!SectionName->empty()) {135 BC->registerSection(Section);136 LLVM_DEBUG(137 dbgs() << "BOLT-DEBUG: registering section " << *SectionName138 << " @ 0x" << Twine::utohexstr(Section.getAddress()) << ":0x"139 << Twine::utohexstr(Section.getAddress() + Section.getSize())140 << "\n");141 }142 }143 144 if (opts::PrintSections) {145 outs() << "BOLT-INFO: Sections from original binary:\n";146 BC->printSections(outs());147 }148}149 150namespace {151 152struct DataInCodeRegion {153 explicit DataInCodeRegion(DiceRef D) {154 D.getOffset(Offset);155 D.getLength(Length);156 D.getKind(Kind);157 }158 159 uint32_t Offset;160 uint16_t Length;161 uint16_t Kind;162};163 164std::vector<DataInCodeRegion> readDataInCode(const MachOObjectFile &O) {165 const MachO::linkedit_data_command DataInCodeLC =166 O.getDataInCodeLoadCommand();167 const uint32_t NumberOfEntries =168 DataInCodeLC.datasize / sizeof(MachO::data_in_code_entry);169 std::vector<DataInCodeRegion> DataInCode;170 DataInCode.reserve(NumberOfEntries);171 for (auto I = O.begin_dices(), E = O.end_dices(); I != E; ++I)172 DataInCode.emplace_back(*I);173 llvm::stable_sort(DataInCode, [](DataInCodeRegion LHS, DataInCodeRegion RHS) {174 return LHS.Offset < RHS.Offset;175 });176 return DataInCode;177}178 179std::optional<uint64_t> readStartAddress(const MachOObjectFile &O) {180 std::optional<uint64_t> StartOffset;181 std::optional<uint64_t> TextVMAddr;182 for (const object::MachOObjectFile::LoadCommandInfo &LC : O.load_commands()) {183 switch (LC.C.cmd) {184 case MachO::LC_MAIN: {185 MachO::entry_point_command LCMain = O.getEntryPointCommand(LC);186 StartOffset = LCMain.entryoff;187 break;188 }189 case MachO::LC_SEGMENT: {190 MachO::segment_command LCSeg = O.getSegmentLoadCommand(LC);191 StringRef SegmentName(LCSeg.segname,192 strnlen(LCSeg.segname, sizeof(LCSeg.segname)));193 if (SegmentName == "__TEXT")194 TextVMAddr = LCSeg.vmaddr;195 break;196 }197 case MachO::LC_SEGMENT_64: {198 MachO::segment_command_64 LCSeg = O.getSegment64LoadCommand(LC);199 StringRef SegmentName(LCSeg.segname,200 strnlen(LCSeg.segname, sizeof(LCSeg.segname)));201 if (SegmentName == "__TEXT")202 TextVMAddr = LCSeg.vmaddr;203 break;204 }205 default:206 continue;207 }208 }209 return (TextVMAddr && StartOffset)210 ? std::optional<uint64_t>(*TextVMAddr + *StartOffset)211 : std::nullopt;212}213 214} // anonymous namespace215 216void MachORewriteInstance::discoverFileObjects() {217 std::vector<SymbolRef> FunctionSymbols;218 for (const SymbolRef &S : InputFile->symbols()) {219 SymbolRef::Type Type = cantFail(S.getType(), "cannot get symbol type");220 if (Type == SymbolRef::ST_Function)221 FunctionSymbols.push_back(S);222 }223 if (FunctionSymbols.empty())224 return;225 llvm::stable_sort(226 FunctionSymbols, [](const SymbolRef &LHS, const SymbolRef &RHS) {227 return cantFail(LHS.getValue()) < cantFail(RHS.getValue());228 });229 for (size_t Index = 0; Index < FunctionSymbols.size(); ++Index) {230 const uint64_t Address = cantFail(FunctionSymbols[Index].getValue());231 ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address);232 // TODO: It happens for some symbols (e.g. __mh_execute_header).233 // Add proper logic to handle them correctly.234 if (!Section) {235 errs() << "BOLT-WARNING: no section found for address " << Address236 << "\n";237 continue;238 }239 240 std::string SymbolName =241 cantFail(FunctionSymbols[Index].getName(), "cannot get symbol name")242 .str();243 // Uniquify names of local symbols.244 if (!(cantFail(FunctionSymbols[Index].getFlags()) & SymbolRef::SF_Global))245 SymbolName = NR.uniquify(SymbolName);246 247 section_iterator S = cantFail(FunctionSymbols[Index].getSection());248 uint64_t EndAddress = S->getAddress() + S->getSize();249 250 size_t NFIndex = Index + 1;251 // Skip aliases.252 while (NFIndex < FunctionSymbols.size() &&253 cantFail(FunctionSymbols[NFIndex].getValue()) == Address)254 ++NFIndex;255 if (NFIndex < FunctionSymbols.size() &&256 S == cantFail(FunctionSymbols[NFIndex].getSection()))257 EndAddress = cantFail(FunctionSymbols[NFIndex].getValue());258 259 const uint64_t SymbolSize = EndAddress - Address;260 const auto It = BC->getBinaryFunctions().find(Address);261 if (It == BC->getBinaryFunctions().end()) {262 BinaryFunction *Function = BC->createBinaryFunction(263 std::move(SymbolName), *Section, Address, SymbolSize);264 if (!opts::Instrument)265 Function->setOutputAddress(Function->getAddress());266 267 } else {268 It->second.addAlternativeName(std::move(SymbolName));269 }270 }271 272 const std::vector<DataInCodeRegion> DataInCode = readDataInCode(*InputFile);273 274 for (auto &BFI : BC->getBinaryFunctions()) {275 BinaryFunction &Function = BFI.second;276 Function.setMaxSize(Function.getSize());277 278 ErrorOr<ArrayRef<uint8_t>> FunctionData = Function.getData();279 if (!FunctionData) {280 errs() << "BOLT-ERROR: corresponding section is non-executable or "281 << "empty for function " << Function << '\n';282 continue;283 }284 285 // Treat zero-sized functions as non-simple ones.286 if (Function.getSize() == 0) {287 Function.setSimple(false);288 continue;289 }290 291 // Offset of the function in the file.292 const auto *FileBegin =293 reinterpret_cast<const uint8_t *>(InputFile->getData().data());294 Function.setFileOffset(FunctionData->begin() - FileBegin);295 296 // Treat functions which contain data in code as non-simple ones.297 const auto It = std::lower_bound(298 DataInCode.cbegin(), DataInCode.cend(), Function.getFileOffset(),299 [](DataInCodeRegion D, uint64_t Offset) { return D.Offset < Offset; });300 if (It != DataInCode.cend() &&301 It->Offset + It->Length <=302 Function.getFileOffset() + Function.getMaxSize())303 Function.setSimple(false);304 }305 306 BC->StartFunctionAddress = readStartAddress(*InputFile);307}308 309void MachORewriteInstance::disassembleFunctions() {310 for (auto &BFI : BC->getBinaryFunctions()) {311 BinaryFunction &Function = BFI.second;312 if (!Function.isSimple())313 continue;314 BC->logBOLTErrorsAndQuitOnFatal(Function.disassemble());315 if (opts::PrintDisasm)316 Function.print(outs(), "after disassembly");317 }318}319 320void MachORewriteInstance::buildFunctionsCFG() {321 for (auto &BFI : BC->getBinaryFunctions()) {322 BinaryFunction &Function = BFI.second;323 if (!Function.isSimple())324 continue;325 BC->logBOLTErrorsAndQuitOnFatal(Function.buildCFG(/*AllocId*/ 0));326 }327}328 329void MachORewriteInstance::postProcessFunctions() {330 for (auto &BFI : BC->getBinaryFunctions()) {331 BinaryFunction &Function = BFI.second;332 if (Function.empty())333 continue;334 Function.postProcessCFG();335 if (opts::PrintCFG)336 Function.print(outs(), "after building cfg");337 }338}339 340void MachORewriteInstance::runOptimizationPasses() {341 BinaryFunctionPassManager Manager(*BC);342 if (opts::Instrument) {343 Manager.registerPass(std::make_unique<PatchEntries>());344 Manager.registerPass(std::make_unique<Instrumentation>(opts::NeverPrint));345 }346 347 Manager.registerPass(std::make_unique<ShortenInstructions>(opts::NeverPrint));348 349 Manager.registerPass(std::make_unique<RemoveNops>(opts::NeverPrint));350 351 Manager.registerPass(std::make_unique<NormalizeCFG>(opts::PrintNormalized));352 353 Manager.registerPass(354 std::make_unique<ReorderBasicBlocks>(opts::PrintReordered));355 Manager.registerPass(356 std::make_unique<FixupBranches>(opts::PrintAfterBranchFixup));357 // This pass should always run last.*358 Manager.registerPass(359 std::make_unique<FinalizeFunctions>(opts::PrintFinalized));360 361 BC->logBOLTErrorsAndQuitOnFatal(Manager.runPasses());362}363 364void MachORewriteInstance::mapInstrumentationSection(365 StringRef SectionName, BOLTLinker::SectionMapper MapSection) {366 if (!opts::Instrument)367 return;368 ErrorOr<BinarySection &> Section = BC->getUniqueSectionByName(SectionName);369 if (!Section) {370 llvm::errs() << "Cannot find " + SectionName + " section\n";371 exit(1);372 }373 if (!Section->hasValidSectionID())374 return;375 MapSection(*Section, Section->getAddress());376}377 378void MachORewriteInstance::mapCodeSections(379 BOLTLinker::SectionMapper MapSection) {380 for (BinaryFunction *Function : BC->getAllBinaryFunctions()) {381 if (!Function->isEmitted())382 continue;383 if (Function->getOutputAddress() == 0)384 continue;385 ErrorOr<BinarySection &> FuncSection = Function->getCodeSection();386 if (!FuncSection)387 report_error(388 (Twine("Cannot find section for function ") + Function->getOneName())389 .str(),390 FuncSection.getError());391 392 FuncSection->setOutputAddress(Function->getOutputAddress());393 LLVM_DEBUG(dbgs() << "BOLT: mapping 0x"394 << Twine::utohexstr(FuncSection->getAllocAddress()) << " to 0x"395 << Twine::utohexstr(Function->getOutputAddress()) << '\n');396 MapSection(*FuncSection, Function->getOutputAddress());397 Function->setImageAddress(FuncSection->getAllocAddress());398 Function->setImageSize(FuncSection->getOutputSize());399 }400 401 if (opts::Instrument) {402 ErrorOr<BinarySection &> BOLT = BC->getUniqueSectionByName("__bolt");403 if (!BOLT) {404 llvm::errs() << "Cannot find __bolt section\n";405 exit(1);406 }407 uint64_t Addr = BOLT->getAddress();408 for (BinaryFunction *Function : BC->getAllBinaryFunctions()) {409 if (!Function->isEmitted())410 continue;411 if (Function->getOutputAddress() != 0)412 continue;413 ErrorOr<BinarySection &> FuncSection = Function->getCodeSection();414 assert(FuncSection && "cannot find section for function");415 Addr = llvm::alignTo(Addr, 4);416 FuncSection->setOutputAddress(Addr);417 MapSection(*FuncSection, Addr);418 Function->setFileOffset(Addr - BOLT->getAddress() +419 BOLT->getInputFileOffset());420 Function->setImageAddress(FuncSection->getAllocAddress());421 Function->setImageSize(FuncSection->getOutputSize());422 BC->registerNameAtAddress(Function->getOneName(), Addr, 0, 0);423 Addr += FuncSection->getOutputSize();424 }425 }426}427 428void MachORewriteInstance::emitAndLink() {429 std::error_code EC;430 std::unique_ptr<::llvm::ToolOutputFile> TempOut =431 std::make_unique<::llvm::ToolOutputFile>(432 opts::OutputFilename + ".bolt.o", EC, sys::fs::OF_None);433 check_error(EC, "cannot create output object file");434 435 if (opts::KeepTmp)436 TempOut->keep();437 438 std::unique_ptr<buffer_ostream> BOS =439 std::make_unique<buffer_ostream>(TempOut->os());440 raw_pwrite_stream *OS = BOS.get();441 auto Streamer = BC->createStreamer(*OS);442 443 emitBinaryContext(*Streamer, *BC, getOrgSecPrefix());444 Streamer->finish();445 446 std::unique_ptr<MemoryBuffer> ObjectMemBuffer =447 MemoryBuffer::getMemBuffer(BOS->str(), "in-memory object file", false);448 std::unique_ptr<object::ObjectFile> Obj = cantFail(449 object::ObjectFile::createObjectFile(ObjectMemBuffer->getMemBufferRef()),450 "error creating in-memory object");451 assert(Obj && "createObjectFile cannot return nullptr");452 453 auto EFMM = std::make_unique<ExecutableFileMemoryManager>(*BC);454 EFMM->setNewSecPrefix(getNewSecPrefix());455 EFMM->setOrgSecPrefix(getOrgSecPrefix());456 457 Linker = std::make_unique<JITLinkLinker>(*BC, std::move(EFMM));458 Linker->loadObject(ObjectMemBuffer->getMemBufferRef(),459 [this](auto MapSection) {460 // Assign addresses to all sections. If key corresponds461 // to the object created by ourselves, call our regular462 // mapping function. If we are loading additional objects463 // as part of runtime libraries for instrumentation,464 // treat them as extra sections.465 mapCodeSections(MapSection);466 mapInstrumentationSection("__counters", MapSection);467 mapInstrumentationSection("__tables", MapSection);468 });469 470 // TODO: Refactor addRuntimeLibSections to work properly on Mach-O471 // and use it here.472 // if (auto *RtLibrary = BC->getRuntimeLibrary()) {473 // RtLibrary->link(*BC, ToolPath, *Linker, [this](auto MapSection) {474 // mapInstrumentationSection("I__setup", MapSection);475 // mapInstrumentationSection("I__fini", MapSection);476 // mapInstrumentationSection("I__data", MapSection);477 // mapInstrumentationSection("I__text", MapSection);478 // mapInstrumentationSection("I__cstring", MapSection);479 // mapInstrumentationSection("I__literal16", MapSection);480 // });481 // }482}483 484void MachORewriteInstance::writeInstrumentationSection(StringRef SectionName,485 raw_pwrite_stream &OS) {486 if (!opts::Instrument)487 return;488 ErrorOr<BinarySection &> Section = BC->getUniqueSectionByName(SectionName);489 if (!Section) {490 llvm::errs() << "Cannot find " + SectionName + " section\n";491 exit(1);492 }493 if (!Section->hasValidSectionID())494 return;495 assert(Section->getInputFileOffset() &&496 "Section input offset cannot be zero");497 assert(Section->getAllocAddress() && "Section alloc address cannot be zero");498 assert(Section->getOutputSize() && "Section output size cannot be zero");499 OS.pwrite(reinterpret_cast<char *>(Section->getAllocAddress()),500 Section->getOutputSize(), Section->getInputFileOffset());501}502 503void MachORewriteInstance::rewriteFile() {504 std::error_code EC;505 Out = std::make_unique<ToolOutputFile>(opts::OutputFilename, EC,506 sys::fs::OF_None);507 check_error(EC, "cannot create output executable file");508 raw_fd_ostream &OS = Out->os();509 OS << InputFile->getData();510 511 for (auto &BFI : BC->getBinaryFunctions()) {512 BinaryFunction &Function = BFI.second;513 if (!Function.isSimple())514 continue;515 assert(Function.isEmitted() && "Simple function has not been emitted");516 if (!opts::Instrument && (Function.getImageSize() > Function.getMaxSize()))517 continue;518 if (opts::Verbosity >= 2)519 outs() << "BOLT: rewriting function \"" << Function << "\"\n";520 OS.pwrite(reinterpret_cast<char *>(Function.getImageAddress()),521 Function.getImageSize(), Function.getFileOffset());522 }523 524 for (const BinaryFunction *Function : BC->getInjectedBinaryFunctions()) {525 OS.pwrite(reinterpret_cast<char *>(Function->getImageAddress()),526 Function->getImageSize(), Function->getFileOffset());527 }528 529 writeInstrumentationSection("__counters", OS);530 writeInstrumentationSection("__tables", OS);531 532 // TODO: Refactor addRuntimeLibSections to work properly on Mach-O and533 // use it here.534 writeInstrumentationSection("I__setup", OS);535 writeInstrumentationSection("I__fini", OS);536 writeInstrumentationSection("I__data", OS);537 writeInstrumentationSection("I__text", OS);538 writeInstrumentationSection("I__cstring", OS);539 writeInstrumentationSection("I__literal16", OS);540 541 Out->keep();542 EC = sys::fs::setPermissions(543 opts::OutputFilename,544 static_cast<sys::fs::perms>(sys::fs::perms::all_all &545 ~sys::fs::getUmask()));546 check_error(EC, "cannot set permissions of output file");547}548 549void MachORewriteInstance::adjustCommandLineOptions() {550//FIXME! Upstream change551// opts::CheckOverlappingElements = false;552 if (!opts::AlignText.getNumOccurrences())553 opts::AlignText = BC->PageAlign;554 if (opts::Instrument.getNumOccurrences())555 opts::ForcePatch = true;556 opts::JumpTables = JTS_MOVE;557 opts::InstrumentCalls = false;558 opts::RuntimeInstrumentationLib = "libbolt_rt_instr_osx.a";559}560 561void MachORewriteInstance::run() {562 adjustCommandLineOptions();563 564 readSpecialSections();565 566 discoverFileObjects();567 568 preprocessProfileData();569 570 disassembleFunctions();571 572 processProfileDataPreCFG();573 574 buildFunctionsCFG();575 576 processProfileData();577 578 postProcessFunctions();579 580 runOptimizationPasses();581 582 emitAndLink();583 584 rewriteFile();585}586 587MachORewriteInstance::~MachORewriteInstance() {}588 589} // namespace bolt590} // namespace llvm591