788 lines · cpp
1//===-LTOBackend.cpp - LLVM Link Time Optimizer Backend -------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the "backend" phase of LTO, i.e. it performs10// optimization and code generation on a loaded module. It is generally used11// internally by the LTO class but can also be used independently, for example12// to implement a standalone ThinLTO backend.13//14//===----------------------------------------------------------------------===//15 16#include "llvm/LTO/LTOBackend.h"17#include "llvm/Analysis/AliasAnalysis.h"18#include "llvm/Analysis/CGSCCPassManager.h"19#include "llvm/Analysis/ModuleSummaryAnalysis.h"20#include "llvm/Analysis/TargetLibraryInfo.h"21#include "llvm/Bitcode/BitcodeReader.h"22#include "llvm/Bitcode/BitcodeWriter.h"23#include "llvm/CGData/CodeGenData.h"24#include "llvm/IR/LLVMRemarkStreamer.h"25#include "llvm/IR/LegacyPassManager.h"26#include "llvm/IR/PassManager.h"27#include "llvm/IR/Verifier.h"28#include "llvm/LTO/LTO.h"29#include "llvm/MC/TargetRegistry.h"30#include "llvm/Object/ModuleSymbolTable.h"31#include "llvm/Passes/PassBuilder.h"32#include "llvm/Passes/PassPlugin.h"33#include "llvm/Passes/StandardInstrumentations.h"34#include "llvm/Support/Error.h"35#include "llvm/Support/FileSystem.h"36#include "llvm/Support/MemoryBuffer.h"37#include "llvm/Support/Path.h"38#include "llvm/Support/ThreadPool.h"39#include "llvm/Support/ToolOutputFile.h"40#include "llvm/Support/VirtualFileSystem.h"41#include "llvm/Support/raw_ostream.h"42#include "llvm/Target/TargetMachine.h"43#include "llvm/TargetParser/SubtargetFeature.h"44#include "llvm/Transforms/IPO/WholeProgramDevirt.h"45#include "llvm/Transforms/Utils/FunctionImportUtils.h"46#include "llvm/Transforms/Utils/SplitModule.h"47#include <optional>48 49using namespace llvm;50using namespace lto;51 52#define DEBUG_TYPE "lto-backend"53 54enum class LTOBitcodeEmbedding {55 DoNotEmbed = 0,56 EmbedOptimized = 1,57 EmbedPostMergePreOptimized = 258};59 60static cl::opt<LTOBitcodeEmbedding> EmbedBitcode(61 "lto-embed-bitcode", cl::init(LTOBitcodeEmbedding::DoNotEmbed),62 cl::values(clEnumValN(LTOBitcodeEmbedding::DoNotEmbed, "none",63 "Do not embed"),64 clEnumValN(LTOBitcodeEmbedding::EmbedOptimized, "optimized",65 "Embed after all optimization passes"),66 clEnumValN(LTOBitcodeEmbedding::EmbedPostMergePreOptimized,67 "post-merge-pre-opt",68 "Embed post merge, but before optimizations")),69 cl::desc("Embed LLVM bitcode in object files produced by LTO"));70 71static cl::opt<bool> ThinLTOAssumeMerged(72 "thinlto-assume-merged", cl::init(false),73 cl::desc("Assume the input has already undergone ThinLTO function "74 "importing and the other pre-optimization pipeline changes."));75 76namespace llvm {77extern cl::opt<bool> NoPGOWarnMismatch;78}79 80[[noreturn]] static void reportOpenError(StringRef Path, Twine Msg) {81 errs() << "failed to open " << Path << ": " << Msg << '\n';82 errs().flush();83 exit(1);84}85 86Error Config::addSaveTemps(std::string OutputFileName, bool UseInputModulePath,87 const DenseSet<StringRef> &SaveTempsArgs) {88 ShouldDiscardValueNames = false;89 90 std::error_code EC;91 if (SaveTempsArgs.empty() || SaveTempsArgs.contains("resolution")) {92 ResolutionFile =93 std::make_unique<raw_fd_ostream>(OutputFileName + "resolution.txt", EC,94 sys::fs::OpenFlags::OF_TextWithCRLF);95 if (EC) {96 ResolutionFile.reset();97 return errorCodeToError(EC);98 }99 }100 101 auto setHook = [&](std::string PathSuffix, ModuleHookFn &Hook) {102 // Keep track of the hook provided by the linker, which also needs to run.103 ModuleHookFn LinkerHook = Hook;104 Hook = [=](unsigned Task, const Module &M) {105 // If the linker's hook returned false, we need to pass that result106 // through.107 if (LinkerHook && !LinkerHook(Task, M))108 return false;109 110 std::string PathPrefix;111 // If this is the combined module (not a ThinLTO backend compile) or the112 // user hasn't requested using the input module's path, emit to a file113 // named from the provided OutputFileName with the Task ID appended.114 if (M.getModuleIdentifier() == "ld-temp.o" || !UseInputModulePath) {115 PathPrefix = OutputFileName;116 if (Task != (unsigned)-1)117 PathPrefix += utostr(Task) + ".";118 } else119 PathPrefix = M.getModuleIdentifier() + ".";120 std::string Path = PathPrefix + PathSuffix + ".bc";121 std::error_code EC;122 raw_fd_ostream OS(Path, EC, sys::fs::OpenFlags::OF_None);123 // Because -save-temps is a debugging feature, we report the error124 // directly and exit.125 if (EC)126 reportOpenError(Path, EC.message());127 WriteBitcodeToFile(M, OS, /*ShouldPreserveUseListOrder=*/false);128 return true;129 };130 };131 132 auto SaveCombinedIndex =133 [=](const ModuleSummaryIndex &Index,134 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {135 std::string Path = OutputFileName + "index.bc";136 std::error_code EC;137 raw_fd_ostream OS(Path, EC, sys::fs::OpenFlags::OF_None);138 // Because -save-temps is a debugging feature, we report the error139 // directly and exit.140 if (EC)141 reportOpenError(Path, EC.message());142 writeIndexToFile(Index, OS);143 144 Path = OutputFileName + "index.dot";145 raw_fd_ostream OSDot(Path, EC, sys::fs::OpenFlags::OF_Text);146 if (EC)147 reportOpenError(Path, EC.message());148 Index.exportToDot(OSDot, GUIDPreservedSymbols);149 return true;150 };151 152 if (SaveTempsArgs.empty()) {153 setHook("0.preopt", PreOptModuleHook);154 setHook("1.promote", PostPromoteModuleHook);155 setHook("2.internalize", PostInternalizeModuleHook);156 setHook("3.import", PostImportModuleHook);157 setHook("4.opt", PostOptModuleHook);158 setHook("5.precodegen", PreCodeGenModuleHook);159 CombinedIndexHook = SaveCombinedIndex;160 } else {161 if (SaveTempsArgs.contains("preopt"))162 setHook("0.preopt", PreOptModuleHook);163 if (SaveTempsArgs.contains("promote"))164 setHook("1.promote", PostPromoteModuleHook);165 if (SaveTempsArgs.contains("internalize"))166 setHook("2.internalize", PostInternalizeModuleHook);167 if (SaveTempsArgs.contains("import"))168 setHook("3.import", PostImportModuleHook);169 if (SaveTempsArgs.contains("opt"))170 setHook("4.opt", PostOptModuleHook);171 if (SaveTempsArgs.contains("precodegen"))172 setHook("5.precodegen", PreCodeGenModuleHook);173 if (SaveTempsArgs.contains("combinedindex"))174 CombinedIndexHook = SaveCombinedIndex;175 }176 177 return Error::success();178}179 180#define HANDLE_EXTENSION(Ext) \181 llvm::PassPluginLibraryInfo get##Ext##PluginInfo();182#include "llvm/Support/Extension.def"183#undef HANDLE_EXTENSION184 185static void RegisterPassPlugins(ArrayRef<std::string> PassPlugins,186 PassBuilder &PB) {187#define HANDLE_EXTENSION(Ext) \188 get##Ext##PluginInfo().RegisterPassBuilderCallbacks(PB);189#include "llvm/Support/Extension.def"190#undef HANDLE_EXTENSION191 192 // Load requested pass plugins and let them register pass builder callbacks193 for (auto &PluginFN : PassPlugins) {194 auto PassPlugin = PassPlugin::Load(PluginFN);195 if (!PassPlugin)196 reportFatalUsageError(PassPlugin.takeError());197 PassPlugin->registerPassBuilderCallbacks(PB);198 }199}200 201static std::unique_ptr<TargetMachine>202createTargetMachine(const Config &Conf, const Target *TheTarget, Module &M) {203 const Triple &TheTriple = M.getTargetTriple();204 SubtargetFeatures Features;205 Features.getDefaultSubtargetFeatures(TheTriple);206 for (const std::string &A : Conf.MAttrs)207 Features.AddFeature(A);208 209 std::optional<Reloc::Model> RelocModel;210 if (Conf.RelocModel)211 RelocModel = *Conf.RelocModel;212 else if (M.getModuleFlag("PIC Level"))213 RelocModel =214 M.getPICLevel() == PICLevel::NotPIC ? Reloc::Static : Reloc::PIC_;215 216 std::optional<CodeModel::Model> CodeModel;217 if (Conf.CodeModel)218 CodeModel = *Conf.CodeModel;219 else220 CodeModel = M.getCodeModel();221 222 TargetOptions TargetOpts = Conf.Options;223 if (TargetOpts.MCOptions.ABIName.empty()) {224 TargetOpts.MCOptions.ABIName = M.getTargetABIFromMD();225 }226 227 std::unique_ptr<TargetMachine> TM(TheTarget->createTargetMachine(228 TheTriple, Conf.CPU, Features.getString(), TargetOpts, RelocModel,229 CodeModel, Conf.CGOptLevel));230 231 assert(TM && "Failed to create target machine");232 233 if (std::optional<uint64_t> LargeDataThreshold = M.getLargeDataThreshold())234 TM->setLargeDataThreshold(*LargeDataThreshold);235 236 return TM;237}238 239static void runNewPMPasses(const Config &Conf, Module &Mod, TargetMachine *TM,240 unsigned OptLevel, bool IsThinLTO,241 ModuleSummaryIndex *ExportSummary,242 const ModuleSummaryIndex *ImportSummary) {243 std::optional<PGOOptions> PGOOpt;244 if (!Conf.SampleProfile.empty())245 PGOOpt = PGOOptions(Conf.SampleProfile, "", Conf.ProfileRemapping,246 /*MemoryProfile=*/"", PGOOptions::SampleUse,247 PGOOptions::NoCSAction,248 PGOOptions::ColdFuncOpt::Default, true);249 else if (Conf.RunCSIRInstr) {250 PGOOpt = PGOOptions("", Conf.CSIRProfile, Conf.ProfileRemapping,251 /*MemoryProfile=*/"", PGOOptions::IRUse,252 PGOOptions::CSIRInstr, PGOOptions::ColdFuncOpt::Default,253 Conf.AddFSDiscriminator);254 } else if (!Conf.CSIRProfile.empty()) {255 PGOOpt =256 PGOOptions(Conf.CSIRProfile, "", Conf.ProfileRemapping,257 /*MemoryProfile=*/"", PGOOptions::IRUse, PGOOptions::CSIRUse,258 PGOOptions::ColdFuncOpt::Default, Conf.AddFSDiscriminator);259 NoPGOWarnMismatch = !Conf.PGOWarnMismatch;260 } else if (Conf.AddFSDiscriminator) {261 PGOOpt = PGOOptions("", "", "", /*MemoryProfile=*/"", PGOOptions::NoAction,262 PGOOptions::NoCSAction,263 PGOOptions::ColdFuncOpt::Default, true);264 }265 TM->setPGOOption(PGOOpt);266 267 LoopAnalysisManager LAM;268 FunctionAnalysisManager FAM;269 CGSCCAnalysisManager CGAM;270 ModuleAnalysisManager MAM;271 272 PassInstrumentationCallbacks PIC;273 StandardInstrumentations SI(Mod.getContext(), Conf.DebugPassManager,274 Conf.VerifyEach);275 SI.registerCallbacks(PIC, &MAM);276 PassBuilder PB(TM, Conf.PTO, PGOOpt, &PIC);277 278 RegisterPassPlugins(Conf.PassPlugins, PB);279 280 std::unique_ptr<TargetLibraryInfoImpl> TLII(281 new TargetLibraryInfoImpl(TM->getTargetTriple()));282 if (Conf.Freestanding)283 TLII->disableAllFunctions();284 FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });285 286 // Parse a custom AA pipeline if asked to.287 if (!Conf.AAPipeline.empty()) {288 AAManager AA;289 if (auto Err = PB.parseAAPipeline(AA, Conf.AAPipeline)) {290 report_fatal_error(Twine("unable to parse AA pipeline description '") +291 Conf.AAPipeline + "': " + toString(std::move(Err)));292 }293 // Register the AA manager first so that our version is the one used.294 FAM.registerPass([&] { return std::move(AA); });295 }296 297 // Register all the basic analyses with the managers.298 PB.registerModuleAnalyses(MAM);299 PB.registerCGSCCAnalyses(CGAM);300 PB.registerFunctionAnalyses(FAM);301 PB.registerLoopAnalyses(LAM);302 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);303 304 ModulePassManager MPM;305 306 if (!Conf.DisableVerify)307 MPM.addPass(VerifierPass());308 309 OptimizationLevel OL;310 311 switch (OptLevel) {312 default:313 llvm_unreachable("Invalid optimization level");314 case 0:315 OL = OptimizationLevel::O0;316 break;317 case 1:318 OL = OptimizationLevel::O1;319 break;320 case 2:321 OL = OptimizationLevel::O2;322 break;323 case 3:324 OL = OptimizationLevel::O3;325 break;326 }327 328 // Parse a custom pipeline if asked to.329 if (!Conf.OptPipeline.empty()) {330 if (auto Err = PB.parsePassPipeline(MPM, Conf.OptPipeline)) {331 report_fatal_error(Twine("unable to parse pass pipeline description '") +332 Conf.OptPipeline + "': " + toString(std::move(Err)));333 }334 } else if (IsThinLTO) {335 MPM.addPass(PB.buildThinLTODefaultPipeline(OL, ImportSummary));336 } else {337 MPM.addPass(PB.buildLTODefaultPipeline(OL, ExportSummary));338 }339 340 if (!Conf.DisableVerify)341 MPM.addPass(VerifierPass());342 343 if (PrintPipelinePasses) {344 std::string PipelineStr;345 raw_string_ostream OS(PipelineStr);346 MPM.printPipeline(OS, [&PIC](StringRef ClassName) {347 auto PassName = PIC.getPassNameForClassName(ClassName);348 return PassName.empty() ? ClassName : PassName;349 });350 outs() << "pipeline-passes: " << PipelineStr << '\n';351 }352 353 MPM.run(Mod, MAM);354}355 356static bool isEmptyModule(const Module &Mod) {357 // Module is empty if it has no functions, no globals, no inline asm and no358 // named metadata (aliases and ifuncs require functions or globals so we359 // don't need to check those explicitly).360 return Mod.empty() && Mod.global_empty() && Mod.named_metadata_empty() &&361 Mod.getModuleInlineAsm().empty();362}363 364bool lto::opt(const Config &Conf, TargetMachine *TM, unsigned Task, Module &Mod,365 bool IsThinLTO, ModuleSummaryIndex *ExportSummary,366 const ModuleSummaryIndex *ImportSummary,367 const std::vector<uint8_t> &CmdArgs) {368 llvm::TimeTraceScope timeScope("opt");369 if (EmbedBitcode == LTOBitcodeEmbedding::EmbedPostMergePreOptimized) {370 // FIXME: the motivation for capturing post-merge bitcode and command line371 // is replicating the compilation environment from bitcode, without needing372 // to understand the dependencies (the functions to be imported). This373 // assumes a clang - based invocation, case in which we have the command374 // line.375 // It's not very clear how the above motivation would map in the376 // linker-based case, so we currently don't plumb the command line args in377 // that case.378 if (CmdArgs.empty())379 LLVM_DEBUG(380 dbgs() << "Post-(Thin)LTO merge bitcode embedding was requested, but "381 "command line arguments are not available");382 llvm::embedBitcodeInModule(Mod, llvm::MemoryBufferRef(),383 /*EmbedBitcode*/ true, /*EmbedCmdline*/ true,384 /*Cmdline*/ CmdArgs);385 }386 // No need to run any opt passes if the module is empty.387 // In theory these passes should take almost no time for an empty388 // module, however, this guards against doing any unnecessary summary-based389 // analysis in the case of a ThinLTO build where this might be an empty390 // regular LTO combined module, with a large combined index from ThinLTO.391 if (!isEmptyModule(Mod)) {392 // FIXME: Plumb the combined index into the new pass manager.393 runNewPMPasses(Conf, Mod, TM, Conf.OptLevel, IsThinLTO, ExportSummary,394 ImportSummary);395 }396 return !Conf.PostOptModuleHook || Conf.PostOptModuleHook(Task, Mod);397}398 399static void codegen(const Config &Conf, TargetMachine *TM,400 AddStreamFn AddStream, unsigned Task, Module &Mod,401 const ModuleSummaryIndex &CombinedIndex) {402 llvm::TimeTraceScope timeScope("codegen");403 if (Conf.PreCodeGenModuleHook && !Conf.PreCodeGenModuleHook(Task, Mod))404 return;405 406 if (EmbedBitcode == LTOBitcodeEmbedding::EmbedOptimized)407 llvm::embedBitcodeInModule(Mod, llvm::MemoryBufferRef(),408 /*EmbedBitcode*/ true,409 /*EmbedCmdline*/ false,410 /*CmdArgs*/ std::vector<uint8_t>());411 412 std::unique_ptr<ToolOutputFile> DwoOut;413 SmallString<1024> DwoFile(Conf.SplitDwarfOutput);414 if (!Conf.DwoDir.empty()) {415 std::error_code EC;416 if (auto EC = llvm::sys::fs::create_directories(Conf.DwoDir))417 report_fatal_error(Twine("Failed to create directory ") + Conf.DwoDir +418 ": " + EC.message());419 420 DwoFile = Conf.DwoDir;421 sys::path::append(DwoFile, std::to_string(Task) + ".dwo");422 TM->Options.MCOptions.SplitDwarfFile = std::string(DwoFile);423 } else424 TM->Options.MCOptions.SplitDwarfFile = Conf.SplitDwarfFile;425 426 if (!DwoFile.empty()) {427 std::error_code EC;428 DwoOut = std::make_unique<ToolOutputFile>(DwoFile, EC, sys::fs::OF_None);429 if (EC)430 report_fatal_error(Twine("Failed to open ") + DwoFile + ": " +431 EC.message());432 }433 434 Expected<std::unique_ptr<CachedFileStream>> StreamOrErr =435 AddStream(Task, Mod.getModuleIdentifier());436 if (Error Err = StreamOrErr.takeError())437 report_fatal_error(std::move(Err));438 std::unique_ptr<CachedFileStream> &Stream = *StreamOrErr;439 TM->Options.ObjectFilenameForDebug = Stream->ObjectPathName;440 441 // Create the codegen pipeline in its own scope so it gets deleted before442 // Stream->commit() is called. The commit function of CacheStream deletes443 // the raw stream, which is too early as streamers (e.g. MCAsmStreamer)444 // keep the pointer and may use it until their destruction. See #138194.445 {446 legacy::PassManager CodeGenPasses;447 TargetLibraryInfoImpl TLII(Mod.getTargetTriple());448 CodeGenPasses.add(new TargetLibraryInfoWrapperPass(TLII));449 // No need to make index available if the module is empty.450 // In theory these passes should not use the index for an empty451 // module, however, this guards against doing any unnecessary summary-based452 // analysis in the case of a ThinLTO build where this might be an empty453 // regular LTO combined module, with a large combined index from ThinLTO.454 if (!isEmptyModule(Mod))455 CodeGenPasses.add(456 createImmutableModuleSummaryIndexWrapperPass(&CombinedIndex));457 if (Conf.PreCodeGenPassesHook)458 Conf.PreCodeGenPassesHook(CodeGenPasses);459 if (TM->addPassesToEmitFile(CodeGenPasses, *Stream->OS,460 DwoOut ? &DwoOut->os() : nullptr,461 Conf.CGFileType))462 report_fatal_error("Failed to setup codegen");463 CodeGenPasses.run(Mod);464 465 if (DwoOut)466 DwoOut->keep();467 }468 469 if (Error Err = Stream->commit())470 report_fatal_error(std::move(Err));471}472 473static void splitCodeGen(const Config &C, TargetMachine *TM,474 AddStreamFn AddStream,475 unsigned ParallelCodeGenParallelismLevel, Module &Mod,476 const ModuleSummaryIndex &CombinedIndex) {477 DefaultThreadPool CodegenThreadPool(478 heavyweight_hardware_concurrency(ParallelCodeGenParallelismLevel));479 unsigned ThreadCount = 0;480 const Target *T = &TM->getTarget();481 482 const auto HandleModulePartition =483 [&](std::unique_ptr<Module> MPart) {484 // We want to clone the module in a new context to multi-thread the485 // codegen. We do it by serializing partition modules to bitcode486 // (while still on the main thread, in order to avoid data races) and487 // spinning up new threads which deserialize the partitions into488 // separate contexts.489 // FIXME: Provide a more direct way to do this in LLVM.490 SmallString<0> BC;491 raw_svector_ostream BCOS(BC);492 WriteBitcodeToFile(*MPart, BCOS);493 494 // Enqueue the task495 CodegenThreadPool.async(496 [&](const SmallString<0> &BC, unsigned ThreadId) {497 LTOLLVMContext Ctx(C);498 Expected<std::unique_ptr<Module>> MOrErr =499 parseBitcodeFile(MemoryBufferRef(BC.str(), "ld-temp.o"), Ctx);500 if (!MOrErr)501 report_fatal_error("Failed to read bitcode");502 std::unique_ptr<Module> MPartInCtx = std::move(MOrErr.get());503 504 std::unique_ptr<TargetMachine> TM =505 createTargetMachine(C, T, *MPartInCtx);506 507 codegen(C, TM.get(), AddStream, ThreadId, *MPartInCtx,508 CombinedIndex);509 },510 // Pass BC using std::move to ensure that it get moved rather than511 // copied into the thread's context.512 std::move(BC), ThreadCount++);513 };514 515 // Try target-specific module splitting first, then fallback to the default.516 if (!TM->splitModule(Mod, ParallelCodeGenParallelismLevel,517 HandleModulePartition)) {518 SplitModule(Mod, ParallelCodeGenParallelismLevel, HandleModulePartition,519 false);520 }521 522 // Because the inner lambda (which runs in a worker thread) captures our local523 // variables, we need to wait for the worker threads to terminate before we524 // can leave the function scope.525 CodegenThreadPool.wait();526}527 528static Expected<const Target *> initAndLookupTarget(const Config &C,529 Module &Mod) {530 if (!C.OverrideTriple.empty())531 Mod.setTargetTriple(Triple(C.OverrideTriple));532 else if (Mod.getTargetTriple().empty())533 Mod.setTargetTriple(Triple(C.DefaultTriple));534 535 std::string Msg;536 const Target *T = TargetRegistry::lookupTarget(Mod.getTargetTriple(), Msg);537 if (!T)538 return make_error<StringError>(Msg, inconvertibleErrorCode());539 return T;540}541 542Error lto::finalizeOptimizationRemarks(LLVMRemarkFileHandle DiagOutputFile) {543 // Make sure we flush the diagnostic remarks file in case the linker doesn't544 // call the global destructors before exiting.545 if (!DiagOutputFile)546 return Error::success();547 DiagOutputFile.finalize();548 DiagOutputFile->keep();549 DiagOutputFile->os().flush();550 return Error::success();551}552 553Error lto::backend(const Config &C, AddStreamFn AddStream,554 unsigned ParallelCodeGenParallelismLevel, Module &Mod,555 ModuleSummaryIndex &CombinedIndex) {556 llvm::TimeTraceScope timeScope("LTO backend");557 Expected<const Target *> TOrErr = initAndLookupTarget(C, Mod);558 if (!TOrErr)559 return TOrErr.takeError();560 561 std::unique_ptr<TargetMachine> TM = createTargetMachine(C, *TOrErr, Mod);562 563 LLVM_DEBUG(dbgs() << "Running regular LTO\n");564 if (!C.CodeGenOnly) {565 if (!opt(C, TM.get(), 0, Mod, /*IsThinLTO=*/false,566 /*ExportSummary=*/&CombinedIndex, /*ImportSummary=*/nullptr,567 /*CmdArgs*/ std::vector<uint8_t>()))568 return Error::success();569 }570 571 if (ParallelCodeGenParallelismLevel == 1) {572 codegen(C, TM.get(), AddStream, 0, Mod, CombinedIndex);573 } else {574 splitCodeGen(C, TM.get(), AddStream, ParallelCodeGenParallelismLevel, Mod,575 CombinedIndex);576 }577 return Error::success();578}579 580static void dropDeadSymbols(Module &Mod, const GVSummaryMapTy &DefinedGlobals,581 const ModuleSummaryIndex &Index) {582 llvm::TimeTraceScope timeScope("Drop dead symbols");583 std::vector<GlobalValue*> DeadGVs;584 for (auto &GV : Mod.global_values())585 if (GlobalValueSummary *GVS = DefinedGlobals.lookup(GV.getGUID()))586 if (!Index.isGlobalValueLive(GVS)) {587 DeadGVs.push_back(&GV);588 convertToDeclaration(GV);589 }590 591 // Now that all dead bodies have been dropped, delete the actual objects592 // themselves when possible.593 for (GlobalValue *GV : DeadGVs) {594 GV->removeDeadConstantUsers();595 // Might reference something defined in native object (i.e. dropped a596 // non-prevailing IR def, but we need to keep the declaration).597 if (GV->use_empty())598 GV->eraseFromParent();599 }600}601 602Error lto::thinBackend(const Config &Conf, unsigned Task, AddStreamFn AddStream,603 Module &Mod, const ModuleSummaryIndex &CombinedIndex,604 const FunctionImporter::ImportMapTy &ImportList,605 const GVSummaryMapTy &DefinedGlobals,606 MapVector<StringRef, BitcodeModule> *ModuleMap,607 bool CodeGenOnly, AddStreamFn IRAddStream,608 const std::vector<uint8_t> &CmdArgs) {609 llvm::TimeTraceScope timeScope("Thin backend", Mod.getModuleIdentifier());610 Expected<const Target *> TOrErr = initAndLookupTarget(Conf, Mod);611 if (!TOrErr)612 return TOrErr.takeError();613 614 std::unique_ptr<TargetMachine> TM = createTargetMachine(Conf, *TOrErr, Mod);615 616 // Setup optimization remarks.617 auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(618 Mod.getContext(), Conf.RemarksFilename, Conf.RemarksPasses,619 Conf.RemarksFormat, Conf.RemarksWithHotness, Conf.RemarksHotnessThreshold,620 Task);621 if (!DiagFileOrErr)622 return DiagFileOrErr.takeError();623 auto DiagnosticOutputFile = std::move(*DiagFileOrErr);624 625 // Set the partial sample profile ratio in the profile summary module flag of626 // the module, if applicable.627 Mod.setPartialSampleProfileRatio(CombinedIndex);628 629 LLVM_DEBUG(dbgs() << "Running ThinLTO\n");630 if (CodeGenOnly) {631 // If CodeGenOnly is set, we only perform code generation and skip632 // optimization. This value may differ from Conf.CodeGenOnly.633 codegen(Conf, TM.get(), AddStream, Task, Mod, CombinedIndex);634 return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));635 }636 637 if (Conf.PreOptModuleHook && !Conf.PreOptModuleHook(Task, Mod))638 return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));639 640 auto OptimizeAndCodegen =641 [&](Module &Mod, TargetMachine *TM,642 LLVMRemarkFileHandle DiagnosticOutputFile) {643 // Perform optimization and code generation for ThinLTO.644 if (!opt(Conf, TM, Task, Mod, /*IsThinLTO=*/true,645 /*ExportSummary=*/nullptr, /*ImportSummary=*/&CombinedIndex,646 CmdArgs))647 return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));648 649 // Save the current module before the first codegen round.650 // Note that the second codegen round runs only `codegen()` without651 // running `opt()`. We're not reaching here as it's bailed out earlier652 // with `CodeGenOnly` which has been set in `SecondRoundThinBackend`.653 if (IRAddStream)654 cgdata::saveModuleForTwoRounds(Mod, Task, IRAddStream);655 656 codegen(Conf, TM, AddStream, Task, Mod, CombinedIndex);657 return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));658 };659 660 if (ThinLTOAssumeMerged)661 return OptimizeAndCodegen(Mod, TM.get(), std::move(DiagnosticOutputFile));662 663 // When linking an ELF shared object, dso_local should be dropped. We664 // conservatively do this for -fpic.665 bool ClearDSOLocalOnDeclarations =666 TM->getTargetTriple().isOSBinFormatELF() &&667 TM->getRelocationModel() != Reloc::Static &&668 Mod.getPIELevel() == PIELevel::Default;669 renameModuleForThinLTO(Mod, CombinedIndex, ClearDSOLocalOnDeclarations);670 671 dropDeadSymbols(Mod, DefinedGlobals, CombinedIndex);672 673 thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true);674 675 if (Conf.PostPromoteModuleHook && !Conf.PostPromoteModuleHook(Task, Mod))676 return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));677 678 if (!DefinedGlobals.empty())679 thinLTOInternalizeModule(Mod, DefinedGlobals);680 681 if (Conf.PostInternalizeModuleHook &&682 !Conf.PostInternalizeModuleHook(Task, Mod))683 return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));684 685 auto ModuleLoader = [&](StringRef Identifier) {686 llvm::TimeTraceScope moduleLoaderScope("Module loader", Identifier);687 assert(Mod.getContext().isODRUniquingDebugTypes() &&688 "ODR Type uniquing should be enabled on the context");689 if (ModuleMap) {690 auto I = ModuleMap->find(Identifier);691 assert(I != ModuleMap->end());692 return I->second.getLazyModule(Mod.getContext(),693 /*ShouldLazyLoadMetadata=*/true,694 /*IsImporting*/ true);695 }696 697 ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> MBOrErr =698 llvm::MemoryBuffer::getFile(Identifier);699 if (!MBOrErr)700 return Expected<std::unique_ptr<llvm::Module>>(make_error<StringError>(701 Twine("Error loading imported file ") + Identifier + " : ",702 MBOrErr.getError()));703 704 Expected<BitcodeModule> BMOrErr = findThinLTOModule(**MBOrErr);705 if (!BMOrErr)706 return Expected<std::unique_ptr<llvm::Module>>(make_error<StringError>(707 Twine("Error loading imported file ") + Identifier + " : " +708 toString(BMOrErr.takeError()),709 inconvertibleErrorCode()));710 711 Expected<std::unique_ptr<Module>> MOrErr =712 BMOrErr->getLazyModule(Mod.getContext(),713 /*ShouldLazyLoadMetadata=*/true,714 /*IsImporting*/ true);715 if (MOrErr)716 (*MOrErr)->setOwnedMemoryBuffer(std::move(*MBOrErr));717 return MOrErr;718 };719 720 {721 llvm::TimeTraceScope importScope("Import functions");722 FunctionImporter Importer(CombinedIndex, ModuleLoader,723 ClearDSOLocalOnDeclarations);724 if (Error Err = Importer.importFunctions(Mod, ImportList).takeError())725 return Err;726 }727 728 // Do this after any importing so that imported code is updated.729 updatePublicTypeTestCalls(Mod, CombinedIndex.withWholeProgramVisibility());730 731 if (Conf.PostImportModuleHook && !Conf.PostImportModuleHook(Task, Mod))732 return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));733 734 return OptimizeAndCodegen(Mod, TM.get(), std::move(DiagnosticOutputFile));735}736 737BitcodeModule *lto::findThinLTOModule(MutableArrayRef<BitcodeModule> BMs) {738 if (ThinLTOAssumeMerged && BMs.size() == 1)739 return BMs.begin();740 741 for (BitcodeModule &BM : BMs) {742 Expected<BitcodeLTOInfo> LTOInfo = BM.getLTOInfo();743 if (LTOInfo && LTOInfo->IsThinLTO)744 return &BM;745 }746 return nullptr;747}748 749Expected<BitcodeModule> lto::findThinLTOModule(MemoryBufferRef MBRef) {750 Expected<std::vector<BitcodeModule>> BMsOrErr = getBitcodeModuleList(MBRef);751 if (!BMsOrErr)752 return BMsOrErr.takeError();753 754 // The bitcode file may contain multiple modules, we want the one that is755 // marked as being the ThinLTO module.756 if (const BitcodeModule *Bm = lto::findThinLTOModule(*BMsOrErr))757 return *Bm;758 759 return make_error<StringError>("Could not find module summary",760 inconvertibleErrorCode());761}762 763bool lto::initImportList(const Module &M,764 const ModuleSummaryIndex &CombinedIndex,765 FunctionImporter::ImportMapTy &ImportList) {766 if (ThinLTOAssumeMerged)767 return true;768 // We can simply import the values mentioned in the combined index, since769 // we should only invoke this using the individual indexes written out770 // via a WriteIndexesThinBackend.771 for (const auto &GlobalList : CombinedIndex) {772 // Ignore entries for undefined references.773 if (GlobalList.second.getSummaryList().empty())774 continue;775 776 auto GUID = GlobalList.first;777 for (const auto &Summary : GlobalList.second.getSummaryList()) {778 // Skip the summaries for the importing module. These are included to779 // e.g. record required linkage changes.780 if (Summary->modulePath() == M.getModuleIdentifier())781 continue;782 // Add an entry to provoke importing by thinBackend.783 ImportList.addGUID(Summary->modulePath(), GUID, Summary->importType());784 }785 }786 return true;787}788