brintos / llvm-project-archived public Read only

1//===-LTO.cpp - LLVM Link Time Optimizer ----------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements functions and classes used to support LTO.10//11//===----------------------------------------------------------------------===//12 13#include "llvm/LTO/LTO.h"14#include "llvm/ADT/ArrayRef.h"15#include "llvm/ADT/ScopeExit.h"16#include "llvm/ADT/SmallSet.h"17#include "llvm/ADT/StableHashing.h"18#include "llvm/ADT/Statistic.h"19#include "llvm/ADT/StringExtras.h"20#include "llvm/Analysis/OptimizationRemarkEmitter.h"21#include "llvm/Analysis/StackSafetyAnalysis.h"22#include "llvm/Analysis/TargetLibraryInfo.h"23#include "llvm/Analysis/TargetTransformInfo.h"24#include "llvm/Bitcode/BitcodeReader.h"25#include "llvm/Bitcode/BitcodeWriter.h"26#include "llvm/CGData/CodeGenData.h"27#include "llvm/CodeGen/Analysis.h"28#include "llvm/Config/llvm-config.h"29#include "llvm/IR/AutoUpgrade.h"30#include "llvm/IR/DiagnosticPrinter.h"31#include "llvm/IR/Intrinsics.h"32#include "llvm/IR/LLVMRemarkStreamer.h"33#include "llvm/IR/LegacyPassManager.h"34#include "llvm/IR/Mangler.h"35#include "llvm/IR/Metadata.h"36#include "llvm/IR/RuntimeLibcalls.h"37#include "llvm/LTO/LTOBackend.h"38#include "llvm/Linker/IRMover.h"39#include "llvm/MC/TargetRegistry.h"40#include "llvm/Object/IRObjectFile.h"41#include "llvm/Support/Caching.h"42#include "llvm/Support/CommandLine.h"43#include "llvm/Support/Compiler.h"44#include "llvm/Support/Error.h"45#include "llvm/Support/FileSystem.h"46#include "llvm/Support/JSON.h"47#include "llvm/Support/MemoryBuffer.h"48#include "llvm/Support/Path.h"49#include "llvm/Support/Process.h"50#include "llvm/Support/SHA1.h"51#include "llvm/Support/SourceMgr.h"52#include "llvm/Support/ThreadPool.h"53#include "llvm/Support/Threading.h"54#include "llvm/Support/TimeProfiler.h"55#include "llvm/Support/ToolOutputFile.h"56#include "llvm/Support/VCSRevision.h"57#include "llvm/Support/raw_ostream.h"58#include "llvm/Target/TargetOptions.h"59#include "llvm/Transforms/IPO.h"60#include "llvm/Transforms/IPO/MemProfContextDisambiguation.h"61#include "llvm/Transforms/IPO/WholeProgramDevirt.h"62#include "llvm/Transforms/Utils/FunctionImportUtils.h"63#include "llvm/Transforms/Utils/SplitModule.h"64 65#include <optional>66#include <set>67 68using namespace llvm;69using namespace lto;70using namespace object;71 72#define DEBUG_TYPE "lto"73 74static cl::opt<bool>75    DumpThinCGSCCs("dump-thin-cg-sccs", cl::init(false), cl::Hidden,76                   cl::desc("Dump the SCCs in the ThinLTO index's callgraph"));77 78namespace llvm {79extern cl::opt<bool> CodeGenDataThinLTOTwoRounds;80extern cl::opt<bool> ForceImportAll;81} // end namespace llvm82 83namespace llvm {84/// Enable global value internalization in LTO.85cl::opt<bool> EnableLTOInternalization(86    "enable-lto-internalization", cl::init(true), cl::Hidden,87    cl::desc("Enable global value internalization in LTO"));88 89static cl::opt<bool>90    LTOKeepSymbolCopies("lto-keep-symbol-copies", cl::init(false), cl::Hidden,91                        cl::desc("Keep copies of symbols in LTO indexing"));92 93/// Indicate we are linking with an allocator that supports hot/cold operator94/// new interfaces.95extern cl::opt<bool> SupportsHotColdNew;96 97/// Enable MemProf context disambiguation for thin link.98extern cl::opt<bool> EnableMemProfContextDisambiguation;99} // namespace llvm100 101// Computes a unique hash for the Module considering the current list of102// export/import and other global analysis results.103// Returns the hash in its hexadecimal representation.104std::string llvm::computeLTOCacheKey(105    const Config &Conf, const ModuleSummaryIndex &Index, StringRef ModuleID,106    const FunctionImporter::ImportMapTy &ImportList,107    const FunctionImporter::ExportSetTy &ExportList,108    const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,109    const GVSummaryMapTy &DefinedGlobals,110    const DenseSet<GlobalValue::GUID> &CfiFunctionDefs,111    const DenseSet<GlobalValue::GUID> &CfiFunctionDecls) {112  // Compute the unique hash for this entry.113  // This is based on the current compiler version, the module itself, the114  // export list, the hash for every single module in the import list, the115  // list of ResolvedODR for the module, and the list of preserved symbols.116  SHA1 Hasher;117 118  // Start with the compiler revision119  Hasher.update(LLVM_VERSION_STRING);120#ifdef LLVM_REVISION121  Hasher.update(LLVM_REVISION);122#endif123 124  // Include the parts of the LTO configuration that affect code generation.125  auto AddString = [&](StringRef Str) {126    Hasher.update(Str);127    Hasher.update(ArrayRef<uint8_t>{0});128  };129  auto AddUnsigned = [&](unsigned I) {130    uint8_t Data[4];131    support::endian::write32le(Data, I);132    Hasher.update(Data);133  };134  auto AddUint64 = [&](uint64_t I) {135    uint8_t Data[8];136    support::endian::write64le(Data, I);137    Hasher.update(Data);138  };139  auto AddUint8 = [&](const uint8_t I) {140    Hasher.update(ArrayRef<uint8_t>(&I, 1));141  };142  AddString(Conf.CPU);143  // FIXME: Hash more of Options. For now all clients initialize Options from144  // command-line flags (which is unsupported in production), but may set145  // X86RelaxRelocations. The clang driver can also pass FunctionSections,146  // DataSections and DebuggerTuning via command line flags.147  AddUnsigned(Conf.Options.MCOptions.X86RelaxRelocations);148  AddUnsigned(Conf.Options.FunctionSections);149  AddUnsigned(Conf.Options.DataSections);150  AddUnsigned((unsigned)Conf.Options.DebuggerTuning);151  for (auto &A : Conf.MAttrs)152    AddString(A);153  if (Conf.RelocModel)154    AddUnsigned(*Conf.RelocModel);155  else156    AddUnsigned(-1);157  if (Conf.CodeModel)158    AddUnsigned(*Conf.CodeModel);159  else160    AddUnsigned(-1);161  for (const auto &S : Conf.MllvmArgs)162    AddString(S);163  AddUnsigned(static_cast<int>(Conf.CGOptLevel));164  AddUnsigned(static_cast<int>(Conf.CGFileType));165  AddUnsigned(Conf.OptLevel);166  AddUnsigned(Conf.Freestanding);167  AddString(Conf.OptPipeline);168  AddString(Conf.AAPipeline);169  AddString(Conf.OverrideTriple);170  AddString(Conf.DefaultTriple);171  AddString(Conf.DwoDir);172  AddUint8(Conf.Dtlto);173 174  // Include the hash for the current module175  auto ModHash = Index.getModuleHash(ModuleID);176  Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));177 178  // TODO: `ExportList` is determined by `ImportList`. Since `ImportList` is179  // used to compute cache key, we could omit hashing `ExportList` here.180  std::vector<uint64_t> ExportsGUID;181  ExportsGUID.reserve(ExportList.size());182  for (const auto &VI : ExportList)183    ExportsGUID.push_back(VI.getGUID());184 185  // Sort the export list elements GUIDs.186  llvm::sort(ExportsGUID);187  for (auto GUID : ExportsGUID)188    Hasher.update(ArrayRef<uint8_t>((uint8_t *)&GUID, sizeof(GUID)));189 190  // Order using module hash, to be both independent of module name and191  // module order.192  auto Comp = [&](const std::pair<StringRef, GlobalValue::GUID> &L,193                  const std::pair<StringRef, GlobalValue::GUID> &R) {194    return std::make_pair(Index.getModule(L.first)->second, L.second) <195           std::make_pair(Index.getModule(R.first)->second, R.second);196  };197  FunctionImporter::SortedImportList SortedImportList(ImportList, Comp);198 199  // Count the number of imports for each source module.200  DenseMap<StringRef, unsigned> ModuleToNumImports;201  for (const auto &[FromModule, GUID, Type] : SortedImportList)202    ++ModuleToNumImports[FromModule];203 204  std::optional<StringRef> LastModule;205  for (const auto &[FromModule, GUID, Type] : SortedImportList) {206    if (LastModule != FromModule) {207      // Include the hash for every module we import functions from. The set of208      // imported symbols for each module may affect code generation and is209      // sensitive to link order, so include that as well.210      LastModule = FromModule;211      auto ModHash = Index.getModule(FromModule)->second;212      Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));213      AddUint64(ModuleToNumImports[FromModule]);214    }215    AddUint64(GUID);216    AddUint8(Type);217  }218 219  // Include the hash for the resolved ODR.220  for (auto &Entry : ResolvedODR) {221    Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&Entry.first,222                                    sizeof(GlobalValue::GUID)));223    Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&Entry.second,224                                    sizeof(GlobalValue::LinkageTypes)));225  }226 227  // Members of CfiFunctionDefs and CfiFunctionDecls that are referenced or228  // defined in this module.229  std::set<GlobalValue::GUID> UsedCfiDefs;230  std::set<GlobalValue::GUID> UsedCfiDecls;231 232  // Typeids used in this module.233  std::set<GlobalValue::GUID> UsedTypeIds;234 235  auto AddUsedCfiGlobal = [&](GlobalValue::GUID ValueGUID) {236    if (CfiFunctionDefs.contains(ValueGUID))237      UsedCfiDefs.insert(ValueGUID);238    if (CfiFunctionDecls.contains(ValueGUID))239      UsedCfiDecls.insert(ValueGUID);240  };241 242  auto AddUsedThings = [&](GlobalValueSummary *GS) {243    if (!GS) return;244    AddUnsigned(GS->getVisibility());245    AddUnsigned(GS->isLive());246    AddUnsigned(GS->canAutoHide());247    for (const ValueInfo &VI : GS->refs()) {248      AddUnsigned(VI.isDSOLocal(Index.withDSOLocalPropagation()));249      AddUsedCfiGlobal(VI.getGUID());250    }251    if (auto *GVS = dyn_cast<GlobalVarSummary>(GS)) {252      AddUnsigned(GVS->maybeReadOnly());253      AddUnsigned(GVS->maybeWriteOnly());254    }255    if (auto *FS = dyn_cast<FunctionSummary>(GS)) {256      for (auto &TT : FS->type_tests())257        UsedTypeIds.insert(TT);258      for (auto &TT : FS->type_test_assume_vcalls())259        UsedTypeIds.insert(TT.GUID);260      for (auto &TT : FS->type_checked_load_vcalls())261        UsedTypeIds.insert(TT.GUID);262      for (auto &TT : FS->type_test_assume_const_vcalls())263        UsedTypeIds.insert(TT.VFunc.GUID);264      for (auto &TT : FS->type_checked_load_const_vcalls())265        UsedTypeIds.insert(TT.VFunc.GUID);266      for (auto &ET : FS->calls()) {267        AddUnsigned(ET.first.isDSOLocal(Index.withDSOLocalPropagation()));268        AddUsedCfiGlobal(ET.first.getGUID());269      }270    }271  };272 273  // Include the hash for the linkage type to reflect internalization and weak274  // resolution, and collect any used type identifier resolutions.275  for (auto &GS : DefinedGlobals) {276    GlobalValue::LinkageTypes Linkage = GS.second->linkage();277    Hasher.update(278        ArrayRef<uint8_t>((const uint8_t *)&Linkage, sizeof(Linkage)));279    AddUsedCfiGlobal(GS.first);280    AddUsedThings(GS.second);281  }282 283  // Imported functions may introduce new uses of type identifier resolutions,284  // so we need to collect their used resolutions as well.285  for (const auto &[FromModule, GUID, Type] : SortedImportList) {286    GlobalValueSummary *S = Index.findSummaryInModule(GUID, FromModule);287    AddUsedThings(S);288    // If this is an alias, we also care about any types/etc. that the aliasee289    // may reference.290    if (auto *AS = dyn_cast_or_null<AliasSummary>(S))291      AddUsedThings(AS->getBaseObject());292  }293 294  auto AddTypeIdSummary = [&](StringRef TId, const TypeIdSummary &S) {295    AddString(TId);296 297    AddUnsigned(S.TTRes.TheKind);298    AddUnsigned(S.TTRes.SizeM1BitWidth);299 300    AddUint64(S.TTRes.AlignLog2);301    AddUint64(S.TTRes.SizeM1);302    AddUint64(S.TTRes.BitMask);303    AddUint64(S.TTRes.InlineBits);304 305    AddUint64(S.WPDRes.size());306    for (auto &WPD : S.WPDRes) {307      AddUnsigned(WPD.first);308      AddUnsigned(WPD.second.TheKind);309      AddString(WPD.second.SingleImplName);310 311      AddUint64(WPD.second.ResByArg.size());312      for (auto &ByArg : WPD.second.ResByArg) {313        AddUint64(ByArg.first.size());314        for (uint64_t Arg : ByArg.first)315          AddUint64(Arg);316        AddUnsigned(ByArg.second.TheKind);317        AddUint64(ByArg.second.Info);318        AddUnsigned(ByArg.second.Byte);319        AddUnsigned(ByArg.second.Bit);320      }321    }322  };323 324  // Include the hash for all type identifiers used by this module.325  for (GlobalValue::GUID TId : UsedTypeIds) {326    auto TidIter = Index.typeIds().equal_range(TId);327    for (const auto &I : make_range(TidIter))328      AddTypeIdSummary(I.second.first, I.second.second);329  }330 331  AddUnsigned(UsedCfiDefs.size());332  for (auto &V : UsedCfiDefs)333    AddUint64(V);334 335  AddUnsigned(UsedCfiDecls.size());336  for (auto &V : UsedCfiDecls)337    AddUint64(V);338 339  if (!Conf.SampleProfile.empty()) {340    auto FileOrErr = MemoryBuffer::getFile(Conf.SampleProfile);341    if (FileOrErr) {342      Hasher.update(FileOrErr.get()->getBuffer());343 344      if (!Conf.ProfileRemapping.empty()) {345        FileOrErr = MemoryBuffer::getFile(Conf.ProfileRemapping);346        if (FileOrErr)347          Hasher.update(FileOrErr.get()->getBuffer());348      }349    }350  }351 352  return toHex(Hasher.result());353}354 355std::string llvm::recomputeLTOCacheKey(const std::string &Key,356                                       StringRef ExtraID) {357  SHA1 Hasher;358 359  auto AddString = [&](StringRef Str) {360    Hasher.update(Str);361    Hasher.update(ArrayRef<uint8_t>{0});362  };363  AddString(Key);364  AddString(ExtraID);365 366  return toHex(Hasher.result());367}368 369static void thinLTOResolvePrevailingGUID(370    const Config &C, ValueInfo VI,371    DenseSet<GlobalValueSummary *> &GlobalInvolvedWithAlias,372    function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>373        isPrevailing,374    function_ref<void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)>375        recordNewLinkage,376    const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {377  GlobalValue::VisibilityTypes Visibility =378      C.VisibilityScheme == Config::ELF ? VI.getELFVisibility()379                                        : GlobalValue::DefaultVisibility;380  for (auto &S : VI.getSummaryList()) {381    GlobalValue::LinkageTypes OriginalLinkage = S->linkage();382    // Ignore local and appending linkage values since the linker383    // doesn't resolve them.384    if (GlobalValue::isLocalLinkage(OriginalLinkage) ||385        GlobalValue::isAppendingLinkage(S->linkage()))386      continue;387    // We need to emit only one of these. The prevailing module will keep it,388    // but turned into a weak, while the others will drop it when possible.389    // This is both a compile-time optimization and a correctness390    // transformation. This is necessary for correctness when we have exported391    // a reference - we need to convert the linkonce to weak to392    // ensure a copy is kept to satisfy the exported reference.393    // FIXME: We may want to split the compile time and correctness394    // aspects into separate routines.395    if (isPrevailing(VI.getGUID(), S.get())) {396      if (GlobalValue::isLinkOnceLinkage(OriginalLinkage)) {397        S->setLinkage(GlobalValue::getWeakLinkage(398            GlobalValue::isLinkOnceODRLinkage(OriginalLinkage)));399        // The kept copy is eligible for auto-hiding (hidden visibility) if all400        // copies were (i.e. they were all linkonce_odr global unnamed addr).401        // If any copy is not (e.g. it was originally weak_odr), then the symbol402        // must remain externally available (e.g. a weak_odr from an explicitly403        // instantiated template). Additionally, if it is in the404        // GUIDPreservedSymbols set, that means that it is visibile outside405        // the summary (e.g. in a native object or a bitcode file without406        // summary), and in that case we cannot hide it as it isn't possible to407        // check all copies.408        S->setCanAutoHide(VI.canAutoHide() &&409                          !GUIDPreservedSymbols.count(VI.getGUID()));410      }411      if (C.VisibilityScheme == Config::FromPrevailing)412        Visibility = S->getVisibility();413    }414    // Alias and aliasee can't be turned into available_externally.415    // When force-import-all is used, it indicates that object linking is not416    // supported by the target. In this case, we can't change the linkage as417    // well in case the global is converted to declaration.418    else if (!isa<AliasSummary>(S.get()) &&419             !GlobalInvolvedWithAlias.count(S.get()) && !ForceImportAll)420      S->setLinkage(GlobalValue::AvailableExternallyLinkage);421 422    // For ELF, set visibility to the computed visibility from summaries. We423    // don't track visibility from declarations so this may be more relaxed than424    // the most constraining one.425    if (C.VisibilityScheme == Config::ELF)426      S->setVisibility(Visibility);427 428    if (S->linkage() != OriginalLinkage)429      recordNewLinkage(S->modulePath(), VI.getGUID(), S->linkage());430  }431 432  if (C.VisibilityScheme == Config::FromPrevailing) {433    for (auto &S : VI.getSummaryList()) {434      GlobalValue::LinkageTypes OriginalLinkage = S->linkage();435      if (GlobalValue::isLocalLinkage(OriginalLinkage) ||436          GlobalValue::isAppendingLinkage(S->linkage()))437        continue;438      S->setVisibility(Visibility);439    }440  }441}442 443/// Resolve linkage for prevailing symbols in the \p Index.444//445// We'd like to drop these functions if they are no longer referenced in the446// current module. However there is a chance that another module is still447// referencing them because of the import. We make sure we always emit at least448// one copy.449void llvm::thinLTOResolvePrevailingInIndex(450    const Config &C, ModuleSummaryIndex &Index,451    function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>452        isPrevailing,453    function_ref<void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)>454        recordNewLinkage,455    const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {456  // We won't optimize the globals that are referenced by an alias for now457  // Ideally we should turn the alias into a global and duplicate the definition458  // when needed.459  DenseSet<GlobalValueSummary *> GlobalInvolvedWithAlias;460  for (auto &I : Index)461    for (auto &S : I.second.getSummaryList())462      if (auto AS = dyn_cast<AliasSummary>(S.get()))463        GlobalInvolvedWithAlias.insert(&AS->getAliasee());464 465  for (auto &I : Index)466    thinLTOResolvePrevailingGUID(C, Index.getValueInfo(I),467                                 GlobalInvolvedWithAlias, isPrevailing,468                                 recordNewLinkage, GUIDPreservedSymbols);469}470 471static void thinLTOInternalizeAndPromoteGUID(472    ValueInfo VI, function_ref<bool(StringRef, ValueInfo)> isExported,473    function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>474        isPrevailing) {475  // Before performing index-based internalization and promotion for this GUID,476  // the local flag should be consistent with the summary list linkage types.477  VI.verifyLocal();478 479  const bool SingleExternallyVisibleCopy =480      VI.getSummaryList().size() == 1 &&481      !GlobalValue::isLocalLinkage(VI.getSummaryList().front()->linkage());482 483  for (auto &S : VI.getSummaryList()) {484    // First see if we need to promote an internal value because it is not485    // exported.486    if (isExported(S->modulePath(), VI)) {487      if (GlobalValue::isLocalLinkage(S->linkage()))488        S->setLinkage(GlobalValue::ExternalLinkage);489      continue;490    }491 492    // Otherwise, see if we can internalize.493    if (!EnableLTOInternalization)494      continue;495 496    // Non-exported values with external linkage can be internalized.497    if (GlobalValue::isExternalLinkage(S->linkage())) {498      S->setLinkage(GlobalValue::InternalLinkage);499      continue;500    }501 502    // Non-exported function and variable definitions with a weak-for-linker503    // linkage can be internalized in certain cases. The minimum legality504    // requirements would be that they are not address taken to ensure that we505    // don't break pointer equality checks, and that variables are either read-506    // or write-only. For functions, this is the case if either all copies are507    // [local_]unnamed_addr, or we can propagate reference edge attributes508    // (which is how this is guaranteed for variables, when analyzing whether509    // they are read or write-only).510    //511    // However, we only get to this code for weak-for-linkage values in one of512    // two cases:513    // 1) The prevailing copy is not in IR (it is in native code).514    // 2) The prevailing copy in IR is not exported from its module.515    // Additionally, at least for the new LTO API, case 2 will only happen if516    // there is exactly one definition of the value (i.e. in exactly one517    // module), as duplicate defs are result in the value being marked exported.518    // Likely, users of the legacy LTO API are similar, however, currently there519    // are llvm-lto based tests of the legacy LTO API that do not mark520    // duplicate linkonce_odr copies as exported via the tool, so we need521    // to handle that case below by checking the number of copies.522    //523    // Generally, we only want to internalize a weak-for-linker value in case524    // 2, because in case 1 we cannot see how the value is used to know if it525    // is read or write-only. We also don't want to bloat the binary with526    // multiple internalized copies of non-prevailing linkonce/weak functions.527    // Note if we don't internalize, we will convert non-prevailing copies to528    // available_externally anyway, so that we drop them after inlining. The529    // only reason to internalize such a function is if we indeed have a single530    // copy, because internalizing it won't increase binary size, and enables531    // use of inliner heuristics that are more aggressive in the face of a532    // single call to a static (local). For variables, internalizing a read or533    // write only variable can enable more aggressive optimization. However, we534    // already perform this elsewhere in the ThinLTO backend handling for535    // read or write-only variables (processGlobalForThinLTO).536    //537    // Therefore, only internalize linkonce/weak if there is a single copy, that538    // is prevailing in this IR module. We can do so aggressively, without539    // requiring the address to be insignificant, or that a variable be read or540    // write-only.541    if (!GlobalValue::isWeakForLinker(S->linkage()) ||542        GlobalValue::isExternalWeakLinkage(S->linkage()))543      continue;544 545    // We may have a single summary copy that is externally visible but not546    // prevailing if the prevailing copy is in a native object.547    if (SingleExternallyVisibleCopy && isPrevailing(VI.getGUID(), S.get()))548      S->setLinkage(GlobalValue::InternalLinkage);549  }550}551 552// Update the linkages in the given \p Index to mark exported values553// as external and non-exported values as internal.554void llvm::thinLTOInternalizeAndPromoteInIndex(555    ModuleSummaryIndex &Index,556    function_ref<bool(StringRef, ValueInfo)> isExported,557    function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>558        isPrevailing) {559  assert(!Index.withInternalizeAndPromote());560  for (auto &I : Index)561    thinLTOInternalizeAndPromoteGUID(Index.getValueInfo(I), isExported,562                                     isPrevailing);563  Index.setWithInternalizeAndPromote();564}565 566// Requires a destructor for std::vector<InputModule>.567InputFile::~InputFile() = default;568 569Expected<std::unique_ptr<InputFile>> InputFile::create(MemoryBufferRef Object) {570  std::unique_ptr<InputFile> File(new InputFile);571 572  Expected<IRSymtabFile> FOrErr = readIRSymtab(Object);573  if (!FOrErr)574    return FOrErr.takeError();575 576  File->TargetTriple = FOrErr->TheReader.getTargetTriple();577  File->SourceFileName = FOrErr->TheReader.getSourceFileName();578  File->COFFLinkerOpts = FOrErr->TheReader.getCOFFLinkerOpts();579  File->DependentLibraries = FOrErr->TheReader.getDependentLibraries();580  File->ComdatTable = FOrErr->TheReader.getComdatTable();581 582  for (unsigned I = 0; I != FOrErr->Mods.size(); ++I) {583    size_t Begin = File->Symbols.size();584    for (const irsymtab::Reader::SymbolRef &Sym :585         FOrErr->TheReader.module_symbols(I))586      // Skip symbols that are irrelevant to LTO. Note that this condition needs587      // to match the one in Skip() in LTO::addRegularLTO().588      if (Sym.isGlobal() && !Sym.isFormatSpecific())589        File->Symbols.push_back(Sym);590    File->ModuleSymIndices.push_back({Begin, File->Symbols.size()});591  }592 593  File->Mods = FOrErr->Mods;594  File->Strtab = std::move(FOrErr->Strtab);595  return std::move(File);596}597 598StringRef InputFile::getName() const {599  return Mods[0].getModuleIdentifier();600}601 602BitcodeModule &InputFile::getSingleBitcodeModule() {603  assert(Mods.size() == 1 && "Expect only one bitcode module");604  return Mods[0];605}606 607LTO::RegularLTOState::RegularLTOState(unsigned ParallelCodeGenParallelismLevel,608                                      const Config &Conf)609    : ParallelCodeGenParallelismLevel(ParallelCodeGenParallelismLevel),610      Ctx(Conf), CombinedModule(std::make_unique<Module>("ld-temp.o", Ctx)),611      Mover(std::make_unique<IRMover>(*CombinedModule)) {}612 613LTO::ThinLTOState::ThinLTOState(ThinBackend BackendParam)614    : Backend(std::move(BackendParam)), CombinedIndex(/*HaveGVs*/ false) {615  if (!Backend.isValid())616    Backend =617        createInProcessThinBackend(llvm::heavyweight_hardware_concurrency());618}619 620LTO::LTO(Config Conf, ThinBackend Backend,621         unsigned ParallelCodeGenParallelismLevel, LTOKind LTOMode)622    : Conf(std::move(Conf)),623      RegularLTO(ParallelCodeGenParallelismLevel, this->Conf),624      ThinLTO(std::move(Backend)),625      GlobalResolutions(626          std::make_unique<DenseMap<StringRef, GlobalResolution>>()),627      LTOMode(LTOMode) {628  if (Conf.KeepSymbolNameCopies || LTOKeepSymbolCopies) {629    Alloc = std::make_unique<BumpPtrAllocator>();630    GlobalResolutionSymbolSaver = std::make_unique<llvm::StringSaver>(*Alloc);631  }632}633 634// Requires a destructor for MapVector<BitcodeModule>.635LTO::~LTO() = default;636 637// Add the symbols in the given module to the GlobalResolutions map, and resolve638// their partitions.639void LTO::addModuleToGlobalRes(ArrayRef<InputFile::Symbol> Syms,640                               ArrayRef<SymbolResolution> Res,641                               unsigned Partition, bool InSummary) {642  llvm::TimeTraceScope timeScope("LTO add module to global resolution");643  auto *ResI = Res.begin();644  auto *ResE = Res.end();645  (void)ResE;646  for (const InputFile::Symbol &Sym : Syms) {647    assert(ResI != ResE);648    SymbolResolution Res = *ResI++;649 650    StringRef SymbolName = Sym.getName();651    // Keep copies of symbols if the client of LTO says so.652    if (GlobalResolutionSymbolSaver && !GlobalResolutions->contains(SymbolName))653      SymbolName = GlobalResolutionSymbolSaver->save(SymbolName);654 655    auto &GlobalRes = (*GlobalResolutions)[SymbolName];656    GlobalRes.UnnamedAddr &= Sym.isUnnamedAddr();657    if (Res.Prevailing) {658      assert(!GlobalRes.Prevailing &&659             "Multiple prevailing defs are not allowed");660      GlobalRes.Prevailing = true;661      GlobalRes.IRName = std::string(Sym.getIRName());662    } else if (!GlobalRes.Prevailing && GlobalRes.IRName.empty()) {663      // Sometimes it can be two copies of symbol in a module and prevailing664      // symbol can have no IR name. That might happen if symbol is defined in665      // module level inline asm block. In case we have multiple modules with666      // the same symbol we want to use IR name of the prevailing symbol.667      // Otherwise, if we haven't seen a prevailing symbol, set the name so that668      // we can later use it to check if there is any prevailing copy in IR.669      GlobalRes.IRName = std::string(Sym.getIRName());670    }671 672    // In rare occasion, the symbol used to initialize GlobalRes has a different673    // IRName from the inspected Symbol. This can happen on macOS + iOS, when a674    // symbol is referenced through its mangled name, say @"\01_symbol" while675    // the IRName is @symbol (the prefix underscore comes from MachO mangling).676    // In that case, we have the same actual Symbol that can get two different677    // GUID, leading to some invalid internalization. Workaround this by marking678    // the GlobalRes external.679 680    // FIXME: instead of this check, it would be desirable to compute GUIDs681    // based on mangled name, but this requires an access to the Target Triple682    // and would be relatively invasive on the codebase.683    if (GlobalRes.IRName != Sym.getIRName()) {684      GlobalRes.Partition = GlobalResolution::External;685      GlobalRes.VisibleOutsideSummary = true;686    }687 688    // Set the partition to external if we know it is re-defined by the linker689    // with -defsym or -wrap options, used elsewhere, e.g. it is visible to a690    // regular object, is referenced from llvm.compiler.used/llvm.used, or was691    // already recorded as being referenced from a different partition.692    if (Res.LinkerRedefined || Res.VisibleToRegularObj || Sym.isUsed() ||693        (GlobalRes.Partition != GlobalResolution::Unknown &&694         GlobalRes.Partition != Partition)) {695      GlobalRes.Partition = GlobalResolution::External;696    } else697      // First recorded reference, save the current partition.698      GlobalRes.Partition = Partition;699 700    // Flag as visible outside of summary if visible from a regular object or701    // from a module that does not have a summary.702    GlobalRes.VisibleOutsideSummary |=703        (Res.VisibleToRegularObj || Sym.isUsed() || !InSummary);704 705    GlobalRes.ExportDynamic |= Res.ExportDynamic;706  }707}708 709void LTO::releaseGlobalResolutionsMemory() {710  // Release GlobalResolutions dense-map itself.711  GlobalResolutions.reset();712  // Release the string saver memory.713  GlobalResolutionSymbolSaver.reset();714  Alloc.reset();715}716 717static void writeToResolutionFile(raw_ostream &OS, InputFile *Input,718                                  ArrayRef<SymbolResolution> Res) {719  StringRef Path = Input->getName();720  OS << Path << '\n';721  auto ResI = Res.begin();722  for (const InputFile::Symbol &Sym : Input->symbols()) {723    assert(ResI != Res.end());724    SymbolResolution Res = *ResI++;725 726    OS << "-r=" << Path << ',' << Sym.getName() << ',';727    if (Res.Prevailing)728      OS << 'p';729    if (Res.FinalDefinitionInLinkageUnit)730      OS << 'l';731    if (Res.VisibleToRegularObj)732      OS << 'x';733    if (Res.LinkerRedefined)734      OS << 'r';735    OS << '\n';736  }737  OS.flush();738  assert(ResI == Res.end());739}740 741Error LTO::add(std::unique_ptr<InputFile> Input,742               ArrayRef<SymbolResolution> Res) {743  llvm::TimeTraceScope timeScope("LTO add input", Input->getName());744  assert(!CalledGetMaxTasks);745 746  if (Conf.ResolutionFile)747    writeToResolutionFile(*Conf.ResolutionFile, Input.get(), Res);748 749  if (RegularLTO.CombinedModule->getTargetTriple().empty()) {750    Triple InputTriple(Input->getTargetTriple());751    RegularLTO.CombinedModule->setTargetTriple(InputTriple);752    if (InputTriple.isOSBinFormatELF())753      Conf.VisibilityScheme = Config::ELF;754  }755 756  ArrayRef<SymbolResolution> InputRes = Res;757  for (unsigned I = 0; I != Input->Mods.size(); ++I) {758    if (auto Err = addModule(*Input, InputRes, I, Res).moveInto(Res))759      return Err;760  }761 762  assert(Res.empty());763  return Error::success();764}765 766Expected<ArrayRef<SymbolResolution>>767LTO::addModule(InputFile &Input, ArrayRef<SymbolResolution> InputRes,768               unsigned ModI, ArrayRef<SymbolResolution> Res) {769  llvm::TimeTraceScope timeScope("LTO add module", Input.getName());770  Expected<BitcodeLTOInfo> LTOInfo = Input.Mods[ModI].getLTOInfo();771  if (!LTOInfo)772    return LTOInfo.takeError();773 774  if (EnableSplitLTOUnit) {775    // If only some modules were split, flag this in the index so that776    // we can skip or error on optimizations that need consistently split777    // modules (whole program devirt and lower type tests).778    if (*EnableSplitLTOUnit != LTOInfo->EnableSplitLTOUnit)779      ThinLTO.CombinedIndex.setPartiallySplitLTOUnits();780  } else781    EnableSplitLTOUnit = LTOInfo->EnableSplitLTOUnit;782 783  BitcodeModule BM = Input.Mods[ModI];784 785  if ((LTOMode == LTOK_UnifiedRegular || LTOMode == LTOK_UnifiedThin) &&786      !LTOInfo->UnifiedLTO)787    return make_error<StringError>(788        "unified LTO compilation must use "789        "compatible bitcode modules (use -funified-lto)",790        inconvertibleErrorCode());791 792  if (LTOInfo->UnifiedLTO && LTOMode == LTOK_Default)793    LTOMode = LTOK_UnifiedThin;794 795  bool IsThinLTO = LTOInfo->IsThinLTO && (LTOMode != LTOK_UnifiedRegular);796 797  auto ModSyms = Input.module_symbols(ModI);798  addModuleToGlobalRes(ModSyms, Res,799                       IsThinLTO ? ThinLTO.ModuleMap.size() + 1 : 0,800                       LTOInfo->HasSummary);801 802  if (IsThinLTO)803    return addThinLTO(BM, ModSyms, Res);804 805  RegularLTO.EmptyCombinedModule = false;806  auto ModOrErr = addRegularLTO(Input, InputRes, BM, ModSyms, Res);807  if (!ModOrErr)808    return ModOrErr.takeError();809  Res = ModOrErr->second;810 811  if (!LTOInfo->HasSummary) {812    if (Error Err = linkRegularLTO(std::move(ModOrErr->first),813                                   /*LivenessFromIndex=*/false))814      return Err;815    return Res;816  }817 818  // Regular LTO module summaries are added to a dummy module that represents819  // the combined regular LTO module.820  if (Error Err = BM.readSummary(ThinLTO.CombinedIndex, ""))821    return Err;822  RegularLTO.ModsWithSummaries.push_back(std::move(ModOrErr->first));823  return Res;824}825 826// Checks whether the given global value is in a non-prevailing comdat827// (comdat containing values the linker indicated were not prevailing,828// which we then dropped to available_externally), and if so, removes829// it from the comdat. This is called for all global values to ensure the830// comdat is empty rather than leaving an incomplete comdat. It is needed for831// regular LTO modules, in case we are in a mixed-LTO mode (both regular832// and thin LTO modules) compilation. Since the regular LTO module will be833// linked first in the final native link, we want to make sure the linker834// doesn't select any of these incomplete comdats that would be left835// in the regular LTO module without this cleanup.836static void837handleNonPrevailingComdat(GlobalValue &GV,838                          std::set<const Comdat *> &NonPrevailingComdats) {839  Comdat *C = GV.getComdat();840  if (!C)841    return;842 843  if (!NonPrevailingComdats.count(C))844    return;845 846  // Additionally need to drop all global values from the comdat to847  // available_externally, to satisfy the COMDAT requirement that all members848  // are discarded as a unit. The non-local linkage global values avoid849  // duplicate definition linker errors.850  GV.setLinkage(GlobalValue::AvailableExternallyLinkage);851 852  if (auto GO = dyn_cast<GlobalObject>(&GV))853    GO->setComdat(nullptr);854}855 856// Add a regular LTO object to the link.857// The resulting module needs to be linked into the combined LTO module with858// linkRegularLTO.859Expected<860    std::pair<LTO::RegularLTOState::AddedModule, ArrayRef<SymbolResolution>>>861LTO::addRegularLTO(InputFile &Input, ArrayRef<SymbolResolution> InputRes,862                   BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,863                   ArrayRef<SymbolResolution> Res) {864  llvm::TimeTraceScope timeScope("LTO add regular LTO");865  RegularLTOState::AddedModule Mod;866  Expected<std::unique_ptr<Module>> MOrErr =867      BM.getLazyModule(RegularLTO.Ctx, /*ShouldLazyLoadMetadata*/ true,868                       /*IsImporting*/ false);869  if (!MOrErr)870    return MOrErr.takeError();871  Module &M = **MOrErr;872  Mod.M = std::move(*MOrErr);873 874  if (Error Err = M.materializeMetadata())875    return std::move(Err);876 877  if (LTOMode == LTOK_UnifiedRegular) {878    // cfi.functions metadata is intended to be used with ThinLTO and may879    // trigger invalid IR transformations if they are present when doing regular880    // LTO, so delete it.881    if (NamedMDNode *CfiFunctionsMD = M.getNamedMetadata("cfi.functions"))882      M.eraseNamedMetadata(CfiFunctionsMD);883  } else if (NamedMDNode *AliasesMD = M.getNamedMetadata("aliases")) {884    // Delete aliases entries for non-prevailing symbols on the ThinLTO side of885    // this input file.886    DenseSet<StringRef> Prevailing;887    for (auto [I, R] : zip(Input.symbols(), InputRes))888      if (R.Prevailing && !I.getIRName().empty())889        Prevailing.insert(I.getIRName());890    std::vector<MDNode *> AliasGroups;891    for (MDNode *AliasGroup : AliasesMD->operands()) {892      std::vector<Metadata *> Aliases;893      for (Metadata *Alias : AliasGroup->operands()) {894        if (isa<MDString>(Alias) &&895            Prevailing.count(cast<MDString>(Alias)->getString()))896          Aliases.push_back(Alias);897      }898      if (Aliases.size() > 1)899        AliasGroups.push_back(MDTuple::get(RegularLTO.Ctx, Aliases));900    }901    AliasesMD->clearOperands();902    for (MDNode *G : AliasGroups)903      AliasesMD->addOperand(G);904  }905 906  UpgradeDebugInfo(M);907 908  ModuleSymbolTable SymTab;909  SymTab.addModule(&M);910 911  for (GlobalVariable &GV : M.globals())912    if (GV.hasAppendingLinkage())913      Mod.Keep.push_back(&GV);914 915  DenseSet<GlobalObject *> AliasedGlobals;916  for (auto &GA : M.aliases())917    if (GlobalObject *GO = GA.getAliaseeObject())918      AliasedGlobals.insert(GO);919 920  // In this function we need IR GlobalValues matching the symbols in Syms921  // (which is not backed by a module), so we need to enumerate them in the same922  // order. The symbol enumeration order of a ModuleSymbolTable intentionally923  // matches the order of an irsymtab, but when we read the irsymtab in924  // InputFile::create we omit some symbols that are irrelevant to LTO. The925  // Skip() function skips the same symbols from the module as InputFile does926  // from the symbol table.927  auto MsymI = SymTab.symbols().begin(), MsymE = SymTab.symbols().end();928  auto Skip = [&]() {929    while (MsymI != MsymE) {930      auto Flags = SymTab.getSymbolFlags(*MsymI);931      if ((Flags & object::BasicSymbolRef::SF_Global) &&932          !(Flags & object::BasicSymbolRef::SF_FormatSpecific))933        return;934      ++MsymI;935    }936  };937  Skip();938 939  std::set<const Comdat *> NonPrevailingComdats;940  SmallSet<StringRef, 2> NonPrevailingAsmSymbols;941  for (const InputFile::Symbol &Sym : Syms) {942    assert(!Res.empty());943    const SymbolResolution &R = Res.consume_front();944 945    assert(MsymI != MsymE);946    ModuleSymbolTable::Symbol Msym = *MsymI++;947    Skip();948 949    if (GlobalValue *GV = dyn_cast_if_present<GlobalValue *>(Msym)) {950      if (R.Prevailing) {951        if (Sym.isUndefined())952          continue;953        Mod.Keep.push_back(GV);954        // For symbols re-defined with linker -wrap and -defsym options,955        // set the linkage to weak to inhibit IPO. The linkage will be956        // restored by the linker.957        if (R.LinkerRedefined)958          GV->setLinkage(GlobalValue::WeakAnyLinkage);959 960        GlobalValue::LinkageTypes OriginalLinkage = GV->getLinkage();961        if (GlobalValue::isLinkOnceLinkage(OriginalLinkage))962          GV->setLinkage(GlobalValue::getWeakLinkage(963              GlobalValue::isLinkOnceODRLinkage(OriginalLinkage)));964      } else if (isa<GlobalObject>(GV) &&965                 (GV->hasLinkOnceODRLinkage() || GV->hasWeakODRLinkage() ||966                  GV->hasAvailableExternallyLinkage()) &&967                 !AliasedGlobals.count(cast<GlobalObject>(GV))) {968        // Any of the above three types of linkage indicates that the969        // chosen prevailing symbol will have the same semantics as this copy of970        // the symbol, so we may be able to link it with available_externally971        // linkage. We will decide later whether to do that when we link this972        // module (in linkRegularLTO), based on whether it is undefined.973        Mod.Keep.push_back(GV);974        GV->setLinkage(GlobalValue::AvailableExternallyLinkage);975        if (GV->hasComdat())976          NonPrevailingComdats.insert(GV->getComdat());977        cast<GlobalObject>(GV)->setComdat(nullptr);978      }979 980      // Set the 'local' flag based on the linker resolution for this symbol.981      if (R.FinalDefinitionInLinkageUnit) {982        GV->setDSOLocal(true);983        if (GV->hasDLLImportStorageClass())984          GV->setDLLStorageClass(GlobalValue::DLLStorageClassTypes::985                                 DefaultStorageClass);986      }987    } else if (auto *AS =988                   dyn_cast_if_present<ModuleSymbolTable::AsmSymbol *>(Msym)) {989      // Collect non-prevailing symbols.990      if (!R.Prevailing)991        NonPrevailingAsmSymbols.insert(AS->first);992    } else {993      llvm_unreachable("unknown symbol type");994    }995 996    // Common resolution: collect the maximum size/alignment over all commons.997    // We also record if we see an instance of a common as prevailing, so that998    // if none is prevailing we can ignore it later.999    if (Sym.isCommon()) {1000      // FIXME: We should figure out what to do about commons defined by asm.1001      // For now they aren't reported correctly by ModuleSymbolTable.1002      auto &CommonRes = RegularLTO.Commons[std::string(Sym.getIRName())];1003      CommonRes.Size = std::max(CommonRes.Size, Sym.getCommonSize());1004      if (uint32_t SymAlignValue = Sym.getCommonAlignment()) {1005        CommonRes.Alignment =1006            std::max(Align(SymAlignValue), CommonRes.Alignment);1007      }1008      CommonRes.Prevailing |= R.Prevailing;1009    }1010  }1011 1012  if (!M.getComdatSymbolTable().empty())1013    for (GlobalValue &GV : M.global_values())1014      handleNonPrevailingComdat(GV, NonPrevailingComdats);1015 1016  // Prepend ".lto_discard <sym>, <sym>*" directive to each module inline asm1017  // block.1018  if (!M.getModuleInlineAsm().empty()) {1019    std::string NewIA = ".lto_discard";1020    if (!NonPrevailingAsmSymbols.empty()) {1021      // Don't dicard a symbol if there is a live .symver for it.1022      ModuleSymbolTable::CollectAsmSymvers(1023          M, [&](StringRef Name, StringRef Alias) {1024            if (!NonPrevailingAsmSymbols.count(Alias))1025              NonPrevailingAsmSymbols.erase(Name);1026          });1027      NewIA += " " + llvm::join(NonPrevailingAsmSymbols, ", ");1028    }1029    NewIA += "\n";1030    M.setModuleInlineAsm(NewIA + M.getModuleInlineAsm());1031  }1032 1033  assert(MsymI == MsymE);1034  return std::make_pair(std::move(Mod), Res);1035}1036 1037Error LTO::linkRegularLTO(RegularLTOState::AddedModule Mod,1038                          bool LivenessFromIndex) {1039  llvm::TimeTraceScope timeScope("LTO link regular LTO");1040  std::vector<GlobalValue *> Keep;1041  for (GlobalValue *GV : Mod.Keep) {1042    if (LivenessFromIndex && !ThinLTO.CombinedIndex.isGUIDLive(GV->getGUID())) {1043      if (Function *F = dyn_cast<Function>(GV)) {1044        if (DiagnosticOutputFile) {1045          if (Error Err = F->materialize())1046            return Err;1047          OptimizationRemarkEmitter ORE(F, nullptr);1048          ORE.emit(OptimizationRemark(DEBUG_TYPE, "deadfunction", F)1049                   << ore::NV("Function", F)1050                   << " not added to the combined module ");1051        }1052      }1053      continue;1054    }1055 1056    if (!GV->hasAvailableExternallyLinkage()) {1057      Keep.push_back(GV);1058      continue;1059    }1060 1061    // Only link available_externally definitions if we don't already have a1062    // definition.1063    GlobalValue *CombinedGV =1064        RegularLTO.CombinedModule->getNamedValue(GV->getName());1065    if (CombinedGV && !CombinedGV->isDeclaration())1066      continue;1067 1068    Keep.push_back(GV);1069  }1070 1071  return RegularLTO.Mover->move(std::move(Mod.M), Keep, nullptr,1072                                /* IsPerformingImport */ false);1073}1074 1075// Add a ThinLTO module to the link.1076Expected<ArrayRef<SymbolResolution>>1077LTO::addThinLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,1078                ArrayRef<SymbolResolution> Res) {1079  llvm::TimeTraceScope timeScope("LTO add thin LTO");1080  const auto BMID = BM.getModuleIdentifier();1081  ArrayRef<SymbolResolution> ResTmp = Res;1082  for (const InputFile::Symbol &Sym : Syms) {1083    assert(!ResTmp.empty());1084    const SymbolResolution &R = ResTmp.consume_front();1085 1086    if (!Sym.getIRName().empty() && R.Prevailing) {1087      auto GUID = GlobalValue::getGUIDAssumingExternalLinkage(1088          GlobalValue::getGlobalIdentifier(Sym.getIRName(),1089                                           GlobalValue::ExternalLinkage, ""));1090      ThinLTO.setPrevailingModuleForGUID(GUID, BMID);1091    }1092  }1093 1094  if (Error Err = BM.readSummary(1095          ThinLTO.CombinedIndex, BMID, [&](GlobalValue::GUID GUID) {1096            return ThinLTO.isPrevailingModuleForGUID(GUID, BMID);1097          }))1098    return Err;1099  LLVM_DEBUG(dbgs() << "Module " << BMID << "\n");1100 1101  for (const InputFile::Symbol &Sym : Syms) {1102    assert(!Res.empty());1103    const SymbolResolution &R = Res.consume_front();1104 1105    if (!Sym.getIRName().empty() &&1106        (R.Prevailing || R.FinalDefinitionInLinkageUnit)) {1107      auto GUID = GlobalValue::getGUIDAssumingExternalLinkage(1108          GlobalValue::getGlobalIdentifier(Sym.getIRName(),1109                                           GlobalValue::ExternalLinkage, ""));1110      if (R.Prevailing) {1111        assert(ThinLTO.isPrevailingModuleForGUID(GUID, BMID));1112 1113        // For linker redefined symbols (via --wrap or --defsym) we want to1114        // switch the linkage to `weak` to prevent IPOs from happening.1115        // Find the summary in the module for this very GV and record the new1116        // linkage so that we can switch it when we import the GV.1117        if (R.LinkerRedefined)1118          if (auto S = ThinLTO.CombinedIndex.findSummaryInModule(GUID, BMID))1119            S->setLinkage(GlobalValue::WeakAnyLinkage);1120      }1121 1122      // If the linker resolved the symbol to a local definition then mark it1123      // as local in the summary for the module we are adding.1124      if (R.FinalDefinitionInLinkageUnit) {1125        if (auto S = ThinLTO.CombinedIndex.findSummaryInModule(GUID, BMID)) {1126          S->setDSOLocal(true);1127        }1128      }1129    }1130  }1131 1132  if (!ThinLTO.ModuleMap.insert({BMID, BM}).second)1133    return make_error<StringError>(1134        "Expected at most one ThinLTO module per bitcode file",1135        inconvertibleErrorCode());1136 1137  if (!Conf.ThinLTOModulesToCompile.empty()) {1138    if (!ThinLTO.ModulesToCompile)1139      ThinLTO.ModulesToCompile = ModuleMapType();1140    // This is a fuzzy name matching where only modules with name containing the1141    // specified switch values are going to be compiled.1142    for (const std::string &Name : Conf.ThinLTOModulesToCompile) {1143      if (BMID.contains(Name)) {1144        ThinLTO.ModulesToCompile->insert({BMID, BM});1145        LLVM_DEBUG(dbgs() << "[ThinLTO] Selecting " << BMID << " to compile\n");1146        break;1147      }1148    }1149  }1150 1151  return Res;1152}1153 1154unsigned LTO::getMaxTasks() const {1155  CalledGetMaxTasks = true;1156  auto ModuleCount = ThinLTO.ModulesToCompile ? ThinLTO.ModulesToCompile->size()1157                                              : ThinLTO.ModuleMap.size();1158  return RegularLTO.ParallelCodeGenParallelismLevel + ModuleCount;1159}1160 1161// If only some of the modules were split, we cannot correctly handle1162// code that contains type tests or type checked loads.1163Error LTO::checkPartiallySplit() {1164  if (!ThinLTO.CombinedIndex.partiallySplitLTOUnits())1165    return Error::success();1166 1167  const Module *Combined = RegularLTO.CombinedModule.get();1168  Function *TypeTestFunc =1169      Intrinsic::getDeclarationIfExists(Combined, Intrinsic::type_test);1170  Function *TypeCheckedLoadFunc =1171      Intrinsic::getDeclarationIfExists(Combined, Intrinsic::type_checked_load);1172  Function *TypeCheckedLoadRelativeFunc = Intrinsic::getDeclarationIfExists(1173      Combined, Intrinsic::type_checked_load_relative);1174 1175  // First check if there are type tests / type checked loads in the1176  // merged regular LTO module IR.1177  if ((TypeTestFunc && !TypeTestFunc->use_empty()) ||1178      (TypeCheckedLoadFunc && !TypeCheckedLoadFunc->use_empty()) ||1179      (TypeCheckedLoadRelativeFunc &&1180       !TypeCheckedLoadRelativeFunc->use_empty()))1181    return make_error<StringError>(1182        "inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)",1183        inconvertibleErrorCode());1184 1185  // Otherwise check if there are any recorded in the combined summary from the1186  // ThinLTO modules.1187  for (auto &P : ThinLTO.CombinedIndex) {1188    for (auto &S : P.second.getSummaryList()) {1189      auto *FS = dyn_cast<FunctionSummary>(S.get());1190      if (!FS)1191        continue;1192      if (!FS->type_test_assume_vcalls().empty() ||1193          !FS->type_checked_load_vcalls().empty() ||1194          !FS->type_test_assume_const_vcalls().empty() ||1195          !FS->type_checked_load_const_vcalls().empty() ||1196          !FS->type_tests().empty())1197        return make_error<StringError>(1198            "inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)",1199            inconvertibleErrorCode());1200    }1201  }1202  return Error::success();1203}1204 1205Error LTO::run(AddStreamFn AddStream, FileCache Cache) {1206  // Compute "dead" symbols, we don't want to import/export these!1207  DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;1208  DenseMap<GlobalValue::GUID, PrevailingType> GUIDPrevailingResolutions;1209  for (auto &Res : *GlobalResolutions) {1210    // Normally resolution have IR name of symbol. We can do nothing here1211    // otherwise. See comments in GlobalResolution struct for more details.1212    if (Res.second.IRName.empty())1213      continue;1214 1215    GlobalValue::GUID GUID = GlobalValue::getGUIDAssumingExternalLinkage(1216        GlobalValue::dropLLVMManglingEscape(Res.second.IRName));1217 1218    if (Res.second.VisibleOutsideSummary && Res.second.Prevailing)1219      GUIDPreservedSymbols.insert(GUID);1220 1221    if (Res.second.ExportDynamic)1222      DynamicExportSymbols.insert(GUID);1223 1224    GUIDPrevailingResolutions[GUID] =1225        Res.second.Prevailing ? PrevailingType::Yes : PrevailingType::No;1226  }1227 1228  auto isPrevailing = [&](GlobalValue::GUID G) {1229    auto It = GUIDPrevailingResolutions.find(G);1230    if (It == GUIDPrevailingResolutions.end())1231      return PrevailingType::Unknown;1232    return It->second;1233  };1234  computeDeadSymbolsWithConstProp(ThinLTO.CombinedIndex, GUIDPreservedSymbols,1235                                  isPrevailing, Conf.OptLevel > 0);1236 1237  // Setup output file to emit statistics.1238  auto StatsFileOrErr = setupStatsFile(Conf.StatsFile);1239  if (!StatsFileOrErr)1240    return StatsFileOrErr.takeError();1241  std::unique_ptr<ToolOutputFile> StatsFile = std::move(StatsFileOrErr.get());1242 1243  // TODO: Ideally this would be controlled automatically by detecting that we1244  // are linking with an allocator that supports these interfaces, rather than1245  // an internal option (which would still be needed for tests, however). For1246  // example, if the library exported a symbol like __malloc_hot_cold the linker1247  // could recognize that and set a flag in the lto::Config.1248  if (SupportsHotColdNew)1249    ThinLTO.CombinedIndex.setWithSupportsHotColdNew();1250 1251  Error Result = runRegularLTO(AddStream);1252  if (!Result)1253    // This will reset the GlobalResolutions optional once done with it to1254    // reduce peak memory before importing.1255    Result = runThinLTO(AddStream, Cache, GUIDPreservedSymbols);1256 1257  if (StatsFile)1258    PrintStatisticsJSON(StatsFile->os());1259 1260  return Result;1261}1262 1263Error LTO::runRegularLTO(AddStreamFn AddStream) {1264  llvm::TimeTraceScope timeScope("Run regular LTO");1265  LLVMContext &CombinedCtx = RegularLTO.CombinedModule->getContext();1266  // Setup optimization remarks.1267  auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(1268      CombinedCtx, Conf.RemarksFilename, Conf.RemarksPasses, Conf.RemarksFormat,1269      Conf.RemarksWithHotness, Conf.RemarksHotnessThreshold);1270  LLVM_DEBUG(dbgs() << "Running regular LTO\n");1271  if (!DiagFileOrErr)1272    return DiagFileOrErr.takeError();1273  DiagnosticOutputFile = std::move(*DiagFileOrErr);1274 1275  // Finalize linking of regular LTO modules containing summaries now that1276  // we have computed liveness information.1277  {1278    llvm::TimeTraceScope timeScope("Link regular LTO");1279    for (auto &M : RegularLTO.ModsWithSummaries)1280      if (Error Err = linkRegularLTO(std::move(M), /*LivenessFromIndex=*/true))1281        return Err;1282  }1283 1284  // Ensure we don't have inconsistently split LTO units with type tests.1285  // FIXME: this checks both LTO and ThinLTO. It happens to work as we take1286  // this path both cases but eventually this should be split into two and1287  // do the ThinLTO checks in `runThinLTO`.1288  if (Error Err = checkPartiallySplit())1289    return Err;1290 1291  // Make sure commons have the right size/alignment: we kept the largest from1292  // all the prevailing when adding the inputs, and we apply it here.1293  const DataLayout &DL = RegularLTO.CombinedModule->getDataLayout();1294  for (auto &I : RegularLTO.Commons) {1295    if (!I.second.Prevailing)1296      // Don't do anything if no instance of this common was prevailing.1297      continue;1298    GlobalVariable *OldGV = RegularLTO.CombinedModule->getNamedGlobal(I.first);1299    if (OldGV && DL.getTypeAllocSize(OldGV->getValueType()) == I.second.Size) {1300      // Don't create a new global if the type is already correct, just make1301      // sure the alignment is correct.1302      OldGV->setAlignment(I.second.Alignment);1303      continue;1304    }1305    ArrayType *Ty =1306        ArrayType::get(Type::getInt8Ty(RegularLTO.Ctx), I.second.Size);1307    auto *GV = new GlobalVariable(*RegularLTO.CombinedModule, Ty, false,1308                                  GlobalValue::CommonLinkage,1309                                  ConstantAggregateZero::get(Ty), "");1310    GV->setAlignment(I.second.Alignment);1311    if (OldGV) {1312      OldGV->replaceAllUsesWith(GV);1313      GV->takeName(OldGV);1314      OldGV->eraseFromParent();1315    } else {1316      GV->setName(I.first);1317    }1318  }1319 1320  bool WholeProgramVisibilityEnabledInLTO =1321      Conf.HasWholeProgramVisibility &&1322      // If validation is enabled, upgrade visibility only when all vtables1323      // have typeinfos.1324      (!Conf.ValidateAllVtablesHaveTypeInfos || Conf.AllVtablesHaveTypeInfos);1325 1326  // This returns true when the name is local or not defined. Locals are1327  // expected to be handled separately.1328  auto IsVisibleToRegularObj = [&](StringRef name) {1329    auto It = GlobalResolutions->find(name);1330    return (It == GlobalResolutions->end() ||1331            It->second.VisibleOutsideSummary || !It->second.Prevailing);1332  };1333 1334  // If allowed, upgrade public vcall visibility metadata to linkage unit1335  // visibility before whole program devirtualization in the optimizer.1336  updateVCallVisibilityInModule(1337      *RegularLTO.CombinedModule, WholeProgramVisibilityEnabledInLTO,1338      DynamicExportSymbols, Conf.ValidateAllVtablesHaveTypeInfos,1339      IsVisibleToRegularObj);1340  updatePublicTypeTestCalls(*RegularLTO.CombinedModule,1341                            WholeProgramVisibilityEnabledInLTO);1342 1343  if (Conf.PreOptModuleHook &&1344      !Conf.PreOptModuleHook(0, *RegularLTO.CombinedModule))1345    return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));1346 1347  if (!Conf.CodeGenOnly) {1348    for (const auto &R : *GlobalResolutions) {1349      GlobalValue *GV =1350          RegularLTO.CombinedModule->getNamedValue(R.second.IRName);1351      if (!R.second.isPrevailingIRSymbol())1352        continue;1353      if (R.second.Partition != 0 &&1354          R.second.Partition != GlobalResolution::External)1355        continue;1356 1357      // Ignore symbols defined in other partitions.1358      // Also skip declarations, which are not allowed to have internal linkage.1359      if (!GV || GV->hasLocalLinkage() || GV->isDeclaration())1360        continue;1361 1362      // Symbols that are marked DLLImport or DLLExport should not be1363      // internalized, as they are either externally visible or referencing1364      // external symbols. Symbols that have AvailableExternally or Appending1365      // linkage might be used by future passes and should be kept as is.1366      // These linkages are seen in Unified regular LTO, because the process1367      // of creating split LTO units introduces symbols with that linkage into1368      // one of the created modules. Normally, only the ThinLTO backend would1369      // compile this module, but Unified Regular LTO processes both1370      // modules created by the splitting process as regular LTO modules.1371      if ((LTOMode == LTOKind::LTOK_UnifiedRegular) &&1372          ((GV->getDLLStorageClass() != GlobalValue::DefaultStorageClass) ||1373           GV->hasAvailableExternallyLinkage() || GV->hasAppendingLinkage()))1374        continue;1375 1376      GV->setUnnamedAddr(R.second.UnnamedAddr ? GlobalValue::UnnamedAddr::Global1377                                              : GlobalValue::UnnamedAddr::None);1378      if (EnableLTOInternalization && R.second.Partition == 0)1379        GV->setLinkage(GlobalValue::InternalLinkage);1380    }1381 1382    if (Conf.PostInternalizeModuleHook &&1383        !Conf.PostInternalizeModuleHook(0, *RegularLTO.CombinedModule))1384      return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));1385  }1386 1387  if (!RegularLTO.EmptyCombinedModule || Conf.AlwaysEmitRegularLTOObj) {1388    if (Error Err =1389            backend(Conf, AddStream, RegularLTO.ParallelCodeGenParallelismLevel,1390                    *RegularLTO.CombinedModule, ThinLTO.CombinedIndex))1391      return Err;1392  }1393 1394  return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));1395}1396 1397SmallVector<const char *> LTO::getRuntimeLibcallSymbols(const Triple &TT) {1398  RTLIB::RuntimeLibcallsInfo Libcalls(TT);1399  SmallVector<const char *> LibcallSymbols;1400  LibcallSymbols.reserve(Libcalls.getNumAvailableLibcallImpls());1401 1402  for (RTLIB::LibcallImpl Impl : RTLIB::libcall_impls()) {1403    if (Libcalls.isAvailable(Impl))1404      LibcallSymbols.push_back(Libcalls.getLibcallImplName(Impl).data());1405  }1406 1407  return LibcallSymbols;1408}1409 1410Error ThinBackendProc::emitFiles(1411    const FunctionImporter::ImportMapTy &ImportList, llvm::StringRef ModulePath,1412    const std::string &NewModulePath) const {1413  return emitFiles(ImportList, ModulePath, NewModulePath,1414                   NewModulePath + ".thinlto.bc",1415                   /*ImportsFiles=*/std::nullopt);1416}1417 1418Error ThinBackendProc::emitFiles(1419    const FunctionImporter::ImportMapTy &ImportList, llvm::StringRef ModulePath,1420    const std::string &NewModulePath, StringRef SummaryPath,1421    std::optional<std::reference_wrapper<ImportsFilesContainer>> ImportsFiles)1422    const {1423  ModuleToSummariesForIndexTy ModuleToSummariesForIndex;1424  GVSummaryPtrSet DeclarationSummaries;1425 1426  std::error_code EC;1427  gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries,1428                                   ImportList, ModuleToSummariesForIndex,1429                                   DeclarationSummaries);1430 1431  raw_fd_ostream OS(SummaryPath, EC, sys::fs::OpenFlags::OF_None);1432  if (EC)1433    return createFileError("cannot open " + Twine(SummaryPath), EC);1434 1435  writeIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex,1436                   &DeclarationSummaries);1437 1438  if (ShouldEmitImportsFiles) {1439    Error ImportsFilesError = EmitImportsFiles(1440        ModulePath, NewModulePath + ".imports", ModuleToSummariesForIndex);1441    if (ImportsFilesError)1442      return ImportsFilesError;1443  }1444 1445  // Optionally, store the imports files.1446  if (ImportsFiles)1447    processImportsFiles(1448        ModulePath, ModuleToSummariesForIndex,1449        [&](StringRef M) { ImportsFiles->get().push_back(M.str()); });1450 1451  return Error::success();1452}1453 1454namespace {1455/// Base class for ThinLTO backends that perform code generation and insert the1456/// generated files back into the link.1457class CGThinBackend : public ThinBackendProc {1458protected:1459  AddStreamFn AddStream;1460  DenseSet<GlobalValue::GUID> CfiFunctionDefs;1461  DenseSet<GlobalValue::GUID> CfiFunctionDecls;1462  bool ShouldEmitIndexFiles;1463 1464public:1465  CGThinBackend(1466      const Config &Conf, ModuleSummaryIndex &CombinedIndex,1467      const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,1468      AddStreamFn AddStream, lto::IndexWriteCallback OnWrite,1469      bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles,1470      ThreadPoolStrategy ThinLTOParallelism)1471      : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries,1472                        OnWrite, ShouldEmitImportsFiles, ThinLTOParallelism),1473        AddStream(std::move(AddStream)),1474        ShouldEmitIndexFiles(ShouldEmitIndexFiles) {1475    auto &Defs = CombinedIndex.cfiFunctionDefs();1476    CfiFunctionDefs.insert_range(Defs.guids());1477    auto &Decls = CombinedIndex.cfiFunctionDecls();1478    CfiFunctionDecls.insert_range(Decls.guids());1479  }1480};1481 1482/// This backend performs code generation by scheduling a job to run on1483/// an in-process thread when invoked for each task.1484class InProcessThinBackend : public CGThinBackend {1485protected:1486  FileCache Cache;1487 1488public:1489  InProcessThinBackend(1490      const Config &Conf, ModuleSummaryIndex &CombinedIndex,1491      ThreadPoolStrategy ThinLTOParallelism,1492      const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,1493      AddStreamFn AddStream, FileCache Cache, lto::IndexWriteCallback OnWrite,1494      bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles)1495      : CGThinBackend(Conf, CombinedIndex, ModuleToDefinedGVSummaries,1496                      AddStream, OnWrite, ShouldEmitIndexFiles,1497                      ShouldEmitImportsFiles, ThinLTOParallelism),1498        Cache(std::move(Cache)) {}1499 1500  virtual Error runThinLTOBackendThread(1501      AddStreamFn AddStream, FileCache Cache, unsigned Task, BitcodeModule BM,1502      ModuleSummaryIndex &CombinedIndex,1503      const FunctionImporter::ImportMapTy &ImportList,1504      const FunctionImporter::ExportSetTy &ExportList,1505      const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,1506      const GVSummaryMapTy &DefinedGlobals,1507      MapVector<StringRef, BitcodeModule> &ModuleMap) {1508    auto ModuleID = BM.getModuleIdentifier();1509    llvm::TimeTraceScope timeScope("Run ThinLTO backend thread (in-process)",1510                                   ModuleID);1511    auto RunThinBackend = [&](AddStreamFn AddStream) {1512      LTOLLVMContext BackendContext(Conf);1513      Expected<std::unique_ptr<Module>> MOrErr = BM.parseModule(BackendContext);1514      if (!MOrErr)1515        return MOrErr.takeError();1516 1517      return thinBackend(Conf, Task, AddStream, **MOrErr, CombinedIndex,1518                         ImportList, DefinedGlobals, &ModuleMap,1519                         Conf.CodeGenOnly);1520    };1521    if (ShouldEmitIndexFiles) {1522      if (auto E = emitFiles(ImportList, ModuleID, ModuleID.str()))1523        return E;1524    }1525 1526    if (!Cache.isValid() || !CombinedIndex.modulePaths().count(ModuleID) ||1527        all_of(CombinedIndex.getModuleHash(ModuleID),1528               [](uint32_t V) { return V == 0; }))1529      // Cache disabled or no entry for this module in the combined index or1530      // no module hash.1531      return RunThinBackend(AddStream);1532 1533    // The module may be cached, this helps handling it.1534    std::string Key = computeLTOCacheKey(1535        Conf, CombinedIndex, ModuleID, ImportList, ExportList, ResolvedODR,1536        DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls);1537    Expected<AddStreamFn> CacheAddStreamOrErr = Cache(Task, Key, ModuleID);1538    if (Error Err = CacheAddStreamOrErr.takeError())1539      return Err;1540    AddStreamFn &CacheAddStream = *CacheAddStreamOrErr;1541    if (CacheAddStream)1542      return RunThinBackend(CacheAddStream);1543 1544    return Error::success();1545  }1546 1547  Error start(1548      unsigned Task, BitcodeModule BM,1549      const FunctionImporter::ImportMapTy &ImportList,1550      const FunctionImporter::ExportSetTy &ExportList,1551      const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,1552      MapVector<StringRef, BitcodeModule> &ModuleMap) override {1553    StringRef ModulePath = BM.getModuleIdentifier();1554    assert(ModuleToDefinedGVSummaries.count(ModulePath));1555    const GVSummaryMapTy &DefinedGlobals =1556        ModuleToDefinedGVSummaries.find(ModulePath)->second;1557    BackendThreadPool.async(1558        [=](BitcodeModule BM, ModuleSummaryIndex &CombinedIndex,1559            const FunctionImporter::ImportMapTy &ImportList,1560            const FunctionImporter::ExportSetTy &ExportList,1561            const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>1562                &ResolvedODR,1563            const GVSummaryMapTy &DefinedGlobals,1564            MapVector<StringRef, BitcodeModule> &ModuleMap) {1565          if (LLVM_ENABLE_THREADS && Conf.TimeTraceEnabled)1566            timeTraceProfilerInitialize(Conf.TimeTraceGranularity,1567                                        "thin backend");1568          Error E = runThinLTOBackendThread(1569              AddStream, Cache, Task, BM, CombinedIndex, ImportList, ExportList,1570              ResolvedODR, DefinedGlobals, ModuleMap);1571          if (E) {1572            std::unique_lock<std::mutex> L(ErrMu);1573            if (Err)1574              Err = joinErrors(std::move(*Err), std::move(E));1575            else1576              Err = std::move(E);1577          }1578          if (LLVM_ENABLE_THREADS && Conf.TimeTraceEnabled)1579            timeTraceProfilerFinishThread();1580        },1581        BM, std::ref(CombinedIndex), std::ref(ImportList), std::ref(ExportList),1582        std::ref(ResolvedODR), std::ref(DefinedGlobals), std::ref(ModuleMap));1583 1584    if (OnWrite)1585      OnWrite(std::string(ModulePath));1586    return Error::success();1587  }1588};1589 1590/// This backend is utilized in the first round of a two-codegen round process.1591/// It first saves optimized bitcode files to disk before the codegen process1592/// begins. After codegen, it stores the resulting object files in a scratch1593/// buffer. Note the codegen data stored in the scratch buffer will be extracted1594/// and merged in the subsequent step.1595class FirstRoundThinBackend : public InProcessThinBackend {1596  AddStreamFn IRAddStream;1597  FileCache IRCache;1598 1599public:1600  FirstRoundThinBackend(1601      const Config &Conf, ModuleSummaryIndex &CombinedIndex,1602      ThreadPoolStrategy ThinLTOParallelism,1603      const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,1604      AddStreamFn CGAddStream, FileCache CGCache, AddStreamFn IRAddStream,1605      FileCache IRCache)1606      : InProcessThinBackend(Conf, CombinedIndex, ThinLTOParallelism,1607                             ModuleToDefinedGVSummaries, std::move(CGAddStream),1608                             std::move(CGCache), /*OnWrite=*/nullptr,1609                             /*ShouldEmitIndexFiles=*/false,1610                             /*ShouldEmitImportsFiles=*/false),1611        IRAddStream(std::move(IRAddStream)), IRCache(std::move(IRCache)) {}1612 1613  Error runThinLTOBackendThread(1614      AddStreamFn CGAddStream, FileCache CGCache, unsigned Task,1615      BitcodeModule BM, ModuleSummaryIndex &CombinedIndex,1616      const FunctionImporter::ImportMapTy &ImportList,1617      const FunctionImporter::ExportSetTy &ExportList,1618      const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,1619      const GVSummaryMapTy &DefinedGlobals,1620      MapVector<StringRef, BitcodeModule> &ModuleMap) override {1621    auto ModuleID = BM.getModuleIdentifier();1622    llvm::TimeTraceScope timeScope("Run ThinLTO backend thread (first round)",1623                                   ModuleID);1624    auto RunThinBackend = [&](AddStreamFn CGAddStream,1625                              AddStreamFn IRAddStream) {1626      LTOLLVMContext BackendContext(Conf);1627      Expected<std::unique_ptr<Module>> MOrErr = BM.parseModule(BackendContext);1628      if (!MOrErr)1629        return MOrErr.takeError();1630 1631      return thinBackend(Conf, Task, CGAddStream, **MOrErr, CombinedIndex,1632                         ImportList, DefinedGlobals, &ModuleMap,1633                         Conf.CodeGenOnly, IRAddStream);1634    };1635    // Like InProcessThinBackend, we produce index files as needed for1636    // FirstRoundThinBackend. However, these files are not generated for1637    // SecondRoundThinBackend.1638    if (ShouldEmitIndexFiles) {1639      if (auto E = emitFiles(ImportList, ModuleID, ModuleID.str()))1640        return E;1641    }1642 1643    assert((CGCache.isValid() == IRCache.isValid()) &&1644           "Both caches for CG and IR should have matching availability");1645    if (!CGCache.isValid() || !CombinedIndex.modulePaths().count(ModuleID) ||1646        all_of(CombinedIndex.getModuleHash(ModuleID),1647               [](uint32_t V) { return V == 0; }))1648      // Cache disabled or no entry for this module in the combined index or1649      // no module hash.1650      return RunThinBackend(CGAddStream, IRAddStream);1651 1652    // Get CGKey for caching object in CGCache.1653    std::string CGKey = computeLTOCacheKey(1654        Conf, CombinedIndex, ModuleID, ImportList, ExportList, ResolvedODR,1655        DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls);1656    Expected<AddStreamFn> CacheCGAddStreamOrErr =1657        CGCache(Task, CGKey, ModuleID);1658    if (Error Err = CacheCGAddStreamOrErr.takeError())1659      return Err;1660    AddStreamFn &CacheCGAddStream = *CacheCGAddStreamOrErr;1661 1662    // Get IRKey for caching (optimized) IR in IRCache with an extra ID.1663    std::string IRKey = recomputeLTOCacheKey(CGKey, /*ExtraID=*/"IR");1664    Expected<AddStreamFn> CacheIRAddStreamOrErr =1665        IRCache(Task, IRKey, ModuleID);1666    if (Error Err = CacheIRAddStreamOrErr.takeError())1667      return Err;1668    AddStreamFn &CacheIRAddStream = *CacheIRAddStreamOrErr;1669 1670    // Ideally, both CG and IR caching should be synchronized. However, in1671    // practice, their availability may differ due to different expiration1672    // times. Therefore, if either cache is missing, the backend process is1673    // triggered.1674    if (CacheCGAddStream || CacheIRAddStream) {1675      LLVM_DEBUG(dbgs() << "[FirstRound] Cache Miss for "1676                        << BM.getModuleIdentifier() << "\n");1677      return RunThinBackend(CacheCGAddStream ? CacheCGAddStream : CGAddStream,1678                            CacheIRAddStream ? CacheIRAddStream : IRAddStream);1679    }1680 1681    return Error::success();1682  }1683};1684 1685/// This backend operates in the second round of a two-codegen round process.1686/// It starts by reading the optimized bitcode files that were saved during the1687/// first round. The backend then executes the codegen only to further optimize1688/// the code, utilizing the codegen data merged from the first round. Finally,1689/// it writes the resulting object files as usual.1690class SecondRoundThinBackend : public InProcessThinBackend {1691  std::unique_ptr<SmallVector<StringRef>> IRFiles;1692  stable_hash CombinedCGDataHash;1693 1694public:1695  SecondRoundThinBackend(1696      const Config &Conf, ModuleSummaryIndex &CombinedIndex,1697      ThreadPoolStrategy ThinLTOParallelism,1698      const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,1699      AddStreamFn AddStream, FileCache Cache,1700      std::unique_ptr<SmallVector<StringRef>> IRFiles,1701      stable_hash CombinedCGDataHash)1702      : InProcessThinBackend(Conf, CombinedIndex, ThinLTOParallelism,1703                             ModuleToDefinedGVSummaries, std::move(AddStream),1704                             std::move(Cache),1705                             /*OnWrite=*/nullptr,1706                             /*ShouldEmitIndexFiles=*/false,1707                             /*ShouldEmitImportsFiles=*/false),1708        IRFiles(std::move(IRFiles)), CombinedCGDataHash(CombinedCGDataHash) {}1709 1710  Error runThinLTOBackendThread(1711      AddStreamFn AddStream, FileCache Cache, unsigned Task, BitcodeModule BM,1712      ModuleSummaryIndex &CombinedIndex,1713      const FunctionImporter::ImportMapTy &ImportList,1714      const FunctionImporter::ExportSetTy &ExportList,1715      const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,1716      const GVSummaryMapTy &DefinedGlobals,1717      MapVector<StringRef, BitcodeModule> &ModuleMap) override {1718    auto ModuleID = BM.getModuleIdentifier();1719    llvm::TimeTraceScope timeScope("Run ThinLTO backend thread (second round)",1720                                   ModuleID);1721    auto RunThinBackend = [&](AddStreamFn AddStream) {1722      LTOLLVMContext BackendContext(Conf);1723      std::unique_ptr<Module> LoadedModule =1724          cgdata::loadModuleForTwoRounds(BM, Task, BackendContext, *IRFiles);1725 1726      return thinBackend(Conf, Task, AddStream, *LoadedModule, CombinedIndex,1727                         ImportList, DefinedGlobals, &ModuleMap,1728                         /*CodeGenOnly=*/true);1729    };1730    if (!Cache.isValid() || !CombinedIndex.modulePaths().count(ModuleID) ||1731        all_of(CombinedIndex.getModuleHash(ModuleID),1732               [](uint32_t V) { return V == 0; }))1733      // Cache disabled or no entry for this module in the combined index or1734      // no module hash.1735      return RunThinBackend(AddStream);1736 1737    // Get Key for caching the final object file in Cache with the combined1738    // CGData hash.1739    std::string Key = computeLTOCacheKey(1740        Conf, CombinedIndex, ModuleID, ImportList, ExportList, ResolvedODR,1741        DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls);1742    Key = recomputeLTOCacheKey(Key,1743                               /*ExtraID=*/std::to_string(CombinedCGDataHash));1744    Expected<AddStreamFn> CacheAddStreamOrErr = Cache(Task, Key, ModuleID);1745    if (Error Err = CacheAddStreamOrErr.takeError())1746      return Err;1747    AddStreamFn &CacheAddStream = *CacheAddStreamOrErr;1748 1749    if (CacheAddStream) {1750      LLVM_DEBUG(dbgs() << "[SecondRound] Cache Miss for "1751                        << BM.getModuleIdentifier() << "\n");1752      return RunThinBackend(CacheAddStream);1753    }1754 1755    return Error::success();1756  }1757};1758} // end anonymous namespace1759 1760ThinBackend lto::createInProcessThinBackend(ThreadPoolStrategy Parallelism,1761                                            lto::IndexWriteCallback OnWrite,1762                                            bool ShouldEmitIndexFiles,1763                                            bool ShouldEmitImportsFiles) {1764  auto Func =1765      [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,1766          const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,1767          AddStreamFn AddStream, FileCache Cache) {1768        return std::make_unique<InProcessThinBackend>(1769            Conf, CombinedIndex, Parallelism, ModuleToDefinedGVSummaries,1770            AddStream, Cache, OnWrite, ShouldEmitIndexFiles,1771            ShouldEmitImportsFiles);1772      };1773  return ThinBackend(Func, Parallelism);1774}1775 1776StringLiteral lto::getThinLTODefaultCPU(const Triple &TheTriple) {1777  if (!TheTriple.isOSDarwin())1778    return "";1779  if (TheTriple.getArch() == Triple::x86_64)1780    return "core2";1781  if (TheTriple.getArch() == Triple::x86)1782    return "yonah";1783  if (TheTriple.isArm64e())1784    return "apple-a12";1785  if (TheTriple.getArch() == Triple::aarch64 ||1786      TheTriple.getArch() == Triple::aarch64_32)1787    return "cyclone";1788  return "";1789}1790 1791// Given the original \p Path to an output file, replace any path1792// prefix matching \p OldPrefix with \p NewPrefix. Also, create the1793// resulting directory if it does not yet exist.1794std::string lto::getThinLTOOutputFile(StringRef Path, StringRef OldPrefix,1795                                      StringRef NewPrefix) {1796  if (OldPrefix.empty() && NewPrefix.empty())1797    return std::string(Path);1798  SmallString<128> NewPath(Path);1799  llvm::sys::path::replace_path_prefix(NewPath, OldPrefix, NewPrefix);1800  StringRef ParentPath = llvm::sys::path::parent_path(NewPath.str());1801  if (!ParentPath.empty()) {1802    // Make sure the new directory exists, creating it if necessary.1803    if (std::error_code EC = llvm::sys::fs::create_directories(ParentPath))1804      llvm::errs() << "warning: could not create directory '" << ParentPath1805                   << "': " << EC.message() << '\n';1806  }1807  return std::string(NewPath);1808}1809 1810namespace {1811class WriteIndexesThinBackend : public ThinBackendProc {1812  std::string OldPrefix, NewPrefix, NativeObjectPrefix;1813  raw_fd_ostream *LinkedObjectsFile;1814 1815public:1816  WriteIndexesThinBackend(1817      const Config &Conf, ModuleSummaryIndex &CombinedIndex,1818      ThreadPoolStrategy ThinLTOParallelism,1819      const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,1820      std::string OldPrefix, std::string NewPrefix,1821      std::string NativeObjectPrefix, bool ShouldEmitImportsFiles,1822      raw_fd_ostream *LinkedObjectsFile, lto::IndexWriteCallback OnWrite)1823      : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries,1824                        OnWrite, ShouldEmitImportsFiles, ThinLTOParallelism),1825        OldPrefix(OldPrefix), NewPrefix(NewPrefix),1826        NativeObjectPrefix(NativeObjectPrefix),1827        LinkedObjectsFile(LinkedObjectsFile) {}1828 1829  Error start(1830      unsigned Task, BitcodeModule BM,1831      const FunctionImporter::ImportMapTy &ImportList,1832      const FunctionImporter::ExportSetTy &ExportList,1833      const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,1834      MapVector<StringRef, BitcodeModule> &ModuleMap) override {1835    StringRef ModulePath = BM.getModuleIdentifier();1836 1837    // The contents of this file may be used as input to a native link, and must1838    // therefore contain the processed modules in a determinstic order that1839    // match the order they are provided on the command line. For that reason,1840    // we cannot include this in the asynchronously executed lambda below.1841    if (LinkedObjectsFile) {1842      std::string ObjectPrefix =1843          NativeObjectPrefix.empty() ? NewPrefix : NativeObjectPrefix;1844      std::string LinkedObjectsFilePath =1845          getThinLTOOutputFile(ModulePath, OldPrefix, ObjectPrefix);1846      *LinkedObjectsFile << LinkedObjectsFilePath << '\n';1847    }1848 1849    BackendThreadPool.async(1850        [this](const StringRef ModulePath,1851               const FunctionImporter::ImportMapTy &ImportList,1852               const std::string &OldPrefix, const std::string &NewPrefix) {1853          std::string NewModulePath =1854              getThinLTOOutputFile(ModulePath, OldPrefix, NewPrefix);1855          auto E = emitFiles(ImportList, ModulePath, NewModulePath);1856          if (E) {1857            std::unique_lock<std::mutex> L(ErrMu);1858            if (Err)1859              Err = joinErrors(std::move(*Err), std::move(E));1860            else1861              Err = std::move(E);1862            return;1863          }1864        },1865        ModulePath, ImportList, OldPrefix, NewPrefix);1866 1867    if (OnWrite)1868      OnWrite(std::string(ModulePath));1869    return Error::success();1870  }1871 1872  bool isSensitiveToInputOrder() override {1873    // The order which modules are written to LinkedObjectsFile should be1874    // deterministic and match the order they are passed on the command line.1875    return true;1876  }1877};1878} // end anonymous namespace1879 1880ThinBackend lto::createWriteIndexesThinBackend(1881    ThreadPoolStrategy Parallelism, std::string OldPrefix,1882    std::string NewPrefix, std::string NativeObjectPrefix,1883    bool ShouldEmitImportsFiles, raw_fd_ostream *LinkedObjectsFile,1884    IndexWriteCallback OnWrite) {1885  auto Func =1886      [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,1887          const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,1888          AddStreamFn AddStream, FileCache Cache) {1889        return std::make_unique<WriteIndexesThinBackend>(1890            Conf, CombinedIndex, Parallelism, ModuleToDefinedGVSummaries,1891            OldPrefix, NewPrefix, NativeObjectPrefix, ShouldEmitImportsFiles,1892            LinkedObjectsFile, OnWrite);1893      };1894  return ThinBackend(Func, Parallelism);1895}1896 1897Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache,1898                      const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {1899  llvm::TimeTraceScope timeScope("Run ThinLTO");1900  LLVM_DEBUG(dbgs() << "Running ThinLTO\n");1901  ThinLTO.CombinedIndex.releaseTemporaryMemory();1902  timeTraceProfilerBegin("ThinLink", StringRef(""));1903  auto TimeTraceScopeExit = llvm::make_scope_exit([]() {1904    if (llvm::timeTraceProfilerEnabled())1905      llvm::timeTraceProfilerEnd();1906  });1907  if (ThinLTO.ModuleMap.empty())1908    return Error::success();1909 1910  if (ThinLTO.ModulesToCompile && ThinLTO.ModulesToCompile->empty()) {1911    llvm::errs() << "warning: [ThinLTO] No module compiled\n";1912    return Error::success();1913  }1914 1915  if (Conf.CombinedIndexHook &&1916      !Conf.CombinedIndexHook(ThinLTO.CombinedIndex, GUIDPreservedSymbols))1917    return Error::success();1918 1919  // Collect for each module the list of function it defines (GUID ->1920  // Summary).1921  DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(1922      ThinLTO.ModuleMap.size());1923  ThinLTO.CombinedIndex.collectDefinedGVSummariesPerModule(1924      ModuleToDefinedGVSummaries);1925  // Create entries for any modules that didn't have any GV summaries1926  // (either they didn't have any GVs to start with, or we suppressed1927  // generation of the summaries because they e.g. had inline assembly1928  // uses that couldn't be promoted/renamed on export). This is so1929  // InProcessThinBackend::start can still launch a backend thread, which1930  // is passed the map of summaries for the module, without any special1931  // handling for this case.1932  for (auto &Mod : ThinLTO.ModuleMap)1933    if (!ModuleToDefinedGVSummaries.count(Mod.first))1934      ModuleToDefinedGVSummaries.try_emplace(Mod.first);1935 1936  FunctionImporter::ImportListsTy ImportLists(ThinLTO.ModuleMap.size());1937  DenseMap<StringRef, FunctionImporter::ExportSetTy> ExportLists(1938      ThinLTO.ModuleMap.size());1939  StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;1940 1941  if (DumpThinCGSCCs)1942    ThinLTO.CombinedIndex.dumpSCCs(outs());1943 1944  std::set<GlobalValue::GUID> ExportedGUIDs;1945 1946  bool WholeProgramVisibilityEnabledInLTO =1947      Conf.HasWholeProgramVisibility &&1948      // If validation is enabled, upgrade visibility only when all vtables1949      // have typeinfos.1950      (!Conf.ValidateAllVtablesHaveTypeInfos || Conf.AllVtablesHaveTypeInfos);1951  if (hasWholeProgramVisibility(WholeProgramVisibilityEnabledInLTO))1952    ThinLTO.CombinedIndex.setWithWholeProgramVisibility();1953 1954  // If we're validating, get the vtable symbols that should not be1955  // upgraded because they correspond to typeIDs outside of index-based1956  // WPD info.1957  DenseSet<GlobalValue::GUID> VisibleToRegularObjSymbols;1958  if (WholeProgramVisibilityEnabledInLTO &&1959      Conf.ValidateAllVtablesHaveTypeInfos) {1960    // This returns true when the name is local or not defined. Locals are1961    // expected to be handled separately.1962    auto IsVisibleToRegularObj = [&](StringRef name) {1963      auto It = GlobalResolutions->find(name);1964      return (It == GlobalResolutions->end() ||1965              It->second.VisibleOutsideSummary || !It->second.Prevailing);1966    };1967 1968    getVisibleToRegularObjVtableGUIDs(ThinLTO.CombinedIndex,1969                                      VisibleToRegularObjSymbols,1970                                      IsVisibleToRegularObj);1971  }1972 1973  // If allowed, upgrade public vcall visibility to linkage unit visibility in1974  // the summaries before whole program devirtualization below.1975  updateVCallVisibilityInIndex(1976      ThinLTO.CombinedIndex, WholeProgramVisibilityEnabledInLTO,1977      DynamicExportSymbols, VisibleToRegularObjSymbols);1978 1979  // Perform index-based WPD. This will return immediately if there are1980  // no index entries in the typeIdMetadata map (e.g. if we are instead1981  // performing IR-based WPD in hybrid regular/thin LTO mode).1982  std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap;1983  runWholeProgramDevirtOnIndex(ThinLTO.CombinedIndex, ExportedGUIDs,1984                               LocalWPDTargetsMap);1985 1986  auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {1987    return ThinLTO.isPrevailingModuleForGUID(GUID, S->modulePath());1988  };1989  if (EnableMemProfContextDisambiguation) {1990    MemProfContextDisambiguation ContextDisambiguation;1991    ContextDisambiguation.run(ThinLTO.CombinedIndex, isPrevailing);1992  }1993 1994  // Figure out which symbols need to be internalized. This also needs to happen1995  // at -O0 because summary-based DCE is implemented using internalization, and1996  // we must apply DCE consistently with the full LTO module in order to avoid1997  // undefined references during the final link.1998  for (auto &Res : *GlobalResolutions) {1999    // If the symbol does not have external references or it is not prevailing,2000    // then not need to mark it as exported from a ThinLTO partition.2001    if (Res.second.Partition != GlobalResolution::External ||2002        !Res.second.isPrevailingIRSymbol())2003      continue;2004    auto GUID = GlobalValue::getGUIDAssumingExternalLinkage(2005        GlobalValue::dropLLVMManglingEscape(Res.second.IRName));2006    // Mark exported unless index-based analysis determined it to be dead.2007    if (ThinLTO.CombinedIndex.isGUIDLive(GUID))2008      ExportedGUIDs.insert(GUID);2009  }2010 2011  // Reset the GlobalResolutions to deallocate the associated memory, as there2012  // are no further accesses. We specifically want to do this before computing2013  // cross module importing, which adds to peak memory via the computed import2014  // and export lists.2015  releaseGlobalResolutionsMemory();2016 2017  if (Conf.OptLevel > 0)2018    ComputeCrossModuleImport(ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,2019                             isPrevailing, ImportLists, ExportLists);2020 2021  // Any functions referenced by the jump table in the regular LTO object must2022  // be exported.2023  auto &Defs = ThinLTO.CombinedIndex.cfiFunctionDefs();2024  ExportedGUIDs.insert(Defs.guid_begin(), Defs.guid_end());2025  auto &Decls = ThinLTO.CombinedIndex.cfiFunctionDecls();2026  ExportedGUIDs.insert(Decls.guid_begin(), Decls.guid_end());2027 2028  auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {2029    const auto &ExportList = ExportLists.find(ModuleIdentifier);2030    return (ExportList != ExportLists.end() && ExportList->second.count(VI)) ||2031           ExportedGUIDs.count(VI.getGUID());2032  };2033 2034  // Update local devirtualized targets that were exported by cross-module2035  // importing or by other devirtualizations marked in the ExportedGUIDs set.2036  updateIndexWPDForExports(ThinLTO.CombinedIndex, isExported,2037                           LocalWPDTargetsMap);2038 2039  thinLTOInternalizeAndPromoteInIndex(ThinLTO.CombinedIndex, isExported,2040                                      isPrevailing);2041 2042  auto recordNewLinkage = [&](StringRef ModuleIdentifier,2043                              GlobalValue::GUID GUID,2044                              GlobalValue::LinkageTypes NewLinkage) {2045    ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;2046  };2047  thinLTOResolvePrevailingInIndex(Conf, ThinLTO.CombinedIndex, isPrevailing,2048                                  recordNewLinkage, GUIDPreservedSymbols);2049 2050  thinLTOPropagateFunctionAttrs(ThinLTO.CombinedIndex, isPrevailing);2051 2052  generateParamAccessSummary(ThinLTO.CombinedIndex);2053 2054  if (llvm::timeTraceProfilerEnabled())2055    llvm::timeTraceProfilerEnd();2056 2057  TimeTraceScopeExit.release();2058 2059  auto &ModuleMap =2060      ThinLTO.ModulesToCompile ? *ThinLTO.ModulesToCompile : ThinLTO.ModuleMap;2061 2062  auto RunBackends = [&](ThinBackendProc *BackendProcess) -> Error {2063    auto ProcessOneModule = [&](int I) -> Error {2064      auto &Mod = *(ModuleMap.begin() + I);2065      // Tasks 0 through ParallelCodeGenParallelismLevel-1 are reserved for2066      // combined module and parallel code generation partitions.2067      return BackendProcess->start(2068          RegularLTO.ParallelCodeGenParallelismLevel + I, Mod.second,2069          ImportLists[Mod.first], ExportLists[Mod.first],2070          ResolvedODR[Mod.first], ThinLTO.ModuleMap);2071    };2072 2073    BackendProcess->setup(ModuleMap.size(),2074                          RegularLTO.ParallelCodeGenParallelismLevel,2075                          RegularLTO.CombinedModule->getTargetTriple());2076 2077    if (BackendProcess->getThreadCount() == 1 ||2078        BackendProcess->isSensitiveToInputOrder()) {2079      // Process the modules in the order they were provided on the2080      // command-line. It is important for this codepath to be used for2081      // WriteIndexesThinBackend, to ensure the emitted LinkedObjectsFile lists2082      // ThinLTO objects in the same order as the inputs, which otherwise would2083      // affect the final link order.2084      for (int I = 0, E = ModuleMap.size(); I != E; ++I)2085        if (Error E = ProcessOneModule(I))2086          return E;2087    } else {2088      // When executing in parallel, process largest bitsize modules first to2089      // improve parallelism, and avoid starving the thread pool near the end.2090      // This saves about 15 sec on a 36-core machine while link `clang.exe`2091      // (out of 100 sec).2092      std::vector<BitcodeModule *> ModulesVec;2093      ModulesVec.reserve(ModuleMap.size());2094      for (auto &Mod : ModuleMap)2095        ModulesVec.push_back(&Mod.second);2096      for (int I : generateModulesOrdering(ModulesVec))2097        if (Error E = ProcessOneModule(I))2098          return E;2099    }2100    return BackendProcess->wait();2101  };2102 2103  if (!CodeGenDataThinLTOTwoRounds) {2104    std::unique_ptr<ThinBackendProc> BackendProc =2105        ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,2106                        AddStream, Cache);2107    return RunBackends(BackendProc.get());2108  }2109 2110  // Perform two rounds of code generation for ThinLTO:2111  // 1. First round: Perform optimization and code generation, outputting to2112  // temporary scratch objects.2113  // 2. Merge code generation data extracted from the temporary scratch objects.2114  // 3. Second round: Execute code generation again using the merged data.2115  LLVM_DEBUG(dbgs() << "[TwoRounds] Initializing ThinLTO two-codegen rounds\n");2116 2117  unsigned MaxTasks = getMaxTasks();2118  auto Parallelism = ThinLTO.Backend.getParallelism();2119  // Set up two additional streams and caches for storing temporary scratch2120  // objects and optimized IRs, using the same cache directory as the original.2121  cgdata::StreamCacheData CG(MaxTasks, Cache, "CG"), IR(MaxTasks, Cache, "IR");2122 2123  // First round: Execute optimization and code generation, outputting to2124  // temporary scratch objects. Serialize the optimized IRs before initiating2125  // code generation.2126  LLVM_DEBUG(dbgs() << "[TwoRounds] Running the first round of codegen\n");2127  auto FirstRoundLTO = std::make_unique<FirstRoundThinBackend>(2128      Conf, ThinLTO.CombinedIndex, Parallelism, ModuleToDefinedGVSummaries,2129      CG.AddStream, CG.Cache, IR.AddStream, IR.Cache);2130  if (Error E = RunBackends(FirstRoundLTO.get()))2131    return E;2132 2133  LLVM_DEBUG(dbgs() << "[TwoRounds] Merging codegen data\n");2134  auto CombinedHashOrErr = cgdata::mergeCodeGenData(*CG.getResult());2135  if (Error E = CombinedHashOrErr.takeError())2136    return E;2137  auto CombinedHash = *CombinedHashOrErr;2138  LLVM_DEBUG(dbgs() << "[TwoRounds] CGData hash: " << CombinedHash << "\n");2139 2140  // Second round: Read the optimized IRs and execute code generation using the2141  // merged data.2142  LLVM_DEBUG(dbgs() << "[TwoRounds] Running the second round of codegen\n");2143  auto SecondRoundLTO = std::make_unique<SecondRoundThinBackend>(2144      Conf, ThinLTO.CombinedIndex, Parallelism, ModuleToDefinedGVSummaries,2145      AddStream, Cache, IR.getResult(), CombinedHash);2146  return RunBackends(SecondRoundLTO.get());2147}2148 2149Expected<LLVMRemarkFileHandle> lto::setupLLVMOptimizationRemarks(2150    LLVMContext &Context, StringRef RemarksFilename, StringRef RemarksPasses,2151    StringRef RemarksFormat, bool RemarksWithHotness,2152    std::optional<uint64_t> RemarksHotnessThreshold, int Count) {2153  std::string Filename = std::string(RemarksFilename);2154  // For ThinLTO, file.opt.<format> becomes2155  // file.opt.<format>.thin.<num>.<format>.2156  if (!Filename.empty() && Count != -1)2157    Filename =2158        (Twine(Filename) + ".thin." + llvm::utostr(Count) + "." + RemarksFormat)2159            .str();2160 2161  auto ResultOrErr = llvm::setupLLVMOptimizationRemarks(2162      Context, Filename, RemarksPasses, RemarksFormat, RemarksWithHotness,2163      RemarksHotnessThreshold);2164  if (Error E = ResultOrErr.takeError())2165    return std::move(E);2166 2167  if (*ResultOrErr)2168    (*ResultOrErr)->keep();2169 2170  return ResultOrErr;2171}2172 2173Expected<std::unique_ptr<ToolOutputFile>>2174lto::setupStatsFile(StringRef StatsFilename) {2175  // Setup output file to emit statistics.2176  if (StatsFilename.empty())2177    return nullptr;2178 2179  llvm::EnableStatistics(false);2180  std::error_code EC;2181  auto StatsFile =2182      std::make_unique<ToolOutputFile>(StatsFilename, EC, sys::fs::OF_None);2183  if (EC)2184    return errorCodeToError(EC);2185 2186  StatsFile->keep();2187  return std::move(StatsFile);2188}2189 2190// Compute the ordering we will process the inputs: the rough heuristic here2191// is to sort them per size so that the largest module get schedule as soon as2192// possible. This is purely a compile-time optimization.2193std::vector<int> lto::generateModulesOrdering(ArrayRef<BitcodeModule *> R) {2194  auto Seq = llvm::seq<int>(0, R.size());2195  std::vector<int> ModulesOrdering(Seq.begin(), Seq.end());2196  llvm::sort(ModulesOrdering, [&](int LeftIndex, int RightIndex) {2197    auto LSize = R[LeftIndex]->getBuffer().size();2198    auto RSize = R[RightIndex]->getBuffer().size();2199    return LSize > RSize;2200  });2201  return ModulesOrdering;2202}2203 2204namespace {2205/// This out-of-process backend does not perform code generation when invoked2206/// for each task. Instead, it generates the necessary information (e.g., the2207/// summary index shard, import list, etc.) to enable code generation to be2208/// performed externally, similar to WriteIndexesThinBackend. The backend's2209/// `wait` function then invokes an external distributor process to carry out2210/// the backend compilations.2211class OutOfProcessThinBackend : public CGThinBackend {2212  using SString = SmallString<128>;2213 2214  BumpPtrAllocator Alloc;2215  StringSaver Saver{Alloc};2216 2217  SString LinkerOutputFile;2218 2219  SString DistributorPath;2220  ArrayRef<StringRef> DistributorArgs;2221 2222  SString RemoteCompiler;2223  ArrayRef<StringRef> RemoteCompilerPrependArgs;2224  ArrayRef<StringRef> RemoteCompilerArgs;2225 2226  bool SaveTemps;2227 2228  SmallVector<StringRef, 0> CodegenOptions;2229  DenseSet<StringRef> CommonInputs;2230  // Number of the object files that have been already cached.2231  std::atomic<size_t> CachedJobs{0};2232  // Information specific to individual backend compilation job.2233  struct Job {2234    unsigned Task;2235    StringRef ModuleID;2236    StringRef NativeObjectPath;2237    StringRef SummaryIndexPath;2238    ImportsFilesContainer ImportsFiles;2239    std::string CacheKey;2240    AddStreamFn CacheAddStream;2241    bool Cached = false;2242  };2243  // The set of backend compilations jobs.2244  SmallVector<Job> Jobs;2245 2246  // A unique string to identify the current link.2247  SmallString<8> UID;2248 2249  // The offset to the first ThinLTO task.2250  unsigned ThinLTOTaskOffset;2251 2252  // The target triple to supply for backend compilations.2253  llvm::Triple Triple;2254 2255  // Cache2256  FileCache Cache;2257 2258public:2259  OutOfProcessThinBackend(2260      const Config &Conf, ModuleSummaryIndex &CombinedIndex,2261      ThreadPoolStrategy ThinLTOParallelism,2262      const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,2263      AddStreamFn AddStream, FileCache CacheFn, lto::IndexWriteCallback OnWrite,2264      bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles,2265      StringRef LinkerOutputFile, StringRef Distributor,2266      ArrayRef<StringRef> DistributorArgs, StringRef RemoteCompiler,2267      ArrayRef<StringRef> RemoteCompilerPrependArgs,2268      ArrayRef<StringRef> RemoteCompilerArgs, bool SaveTemps)2269      : CGThinBackend(Conf, CombinedIndex, ModuleToDefinedGVSummaries,2270                      AddStream, OnWrite, ShouldEmitIndexFiles,2271                      ShouldEmitImportsFiles, ThinLTOParallelism),2272        LinkerOutputFile(LinkerOutputFile), DistributorPath(Distributor),2273        DistributorArgs(DistributorArgs), RemoteCompiler(RemoteCompiler),2274        RemoteCompilerPrependArgs(RemoteCompilerPrependArgs),2275        RemoteCompilerArgs(RemoteCompilerArgs), SaveTemps(SaveTemps),2276        Cache(std::move(CacheFn)) {}2277 2278  void setup(unsigned ThinLTONumTasks, unsigned ThinLTOTaskOffset,2279             llvm::Triple Triple) override {2280    UID = itostr(sys::Process::getProcessId());2281    Jobs.resize((size_t)ThinLTONumTasks);2282    this->ThinLTOTaskOffset = ThinLTOTaskOffset;2283    this->Triple = Triple;2284    this->Conf.Dtlto = 1;2285  }2286 2287  virtual Error runThinLTOBackendThread(2288      Job &J, const FunctionImporter::ImportMapTy &ImportList,2289      const FunctionImporter::ExportSetTy &ExportList,2290      const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>2291          &ResolvedODR) {2292 2293    llvm::TimeTraceScope timeScope(2294        "Run ThinLTO backend thread (out-of-process)", J.ModuleID);2295 2296    if (auto E = emitFiles(ImportList, J.ModuleID, J.ModuleID.str(),2297                           J.SummaryIndexPath, J.ImportsFiles))2298      return E;2299 2300    if (!Cache.isValid() || !CombinedIndex.modulePaths().count(J.ModuleID) ||2301        all_of(CombinedIndex.getModuleHash(J.ModuleID),2302               [](uint32_t V) { return V == 0; }))2303      // Cache disabled or no entry for this module in the combined index or2304      // no module hash.2305      return Error::success();2306 2307    const GVSummaryMapTy &DefinedGlobals =2308        ModuleToDefinedGVSummaries.find(J.ModuleID)->second;2309 2310    // The module may be cached, this helps handling it.2311    J.CacheKey = computeLTOCacheKey(Conf, CombinedIndex, J.ModuleID, ImportList,2312                                    ExportList, ResolvedODR, DefinedGlobals,2313                                    CfiFunctionDefs, CfiFunctionDecls);2314 2315    // The module may be cached, this helps handling it.2316    auto CacheAddStreamExp = Cache(J.Task, J.CacheKey, J.ModuleID);2317    if (Error Err = CacheAddStreamExp.takeError())2318      return Err;2319    AddStreamFn &CacheAddStream = *CacheAddStreamExp;2320    // If CacheAddStream is null, we have a cache hit and at this point2321    // object file is already passed back to the linker.2322    if (!CacheAddStream) {2323      J.Cached = true; // Cache hit, mark the job as cached.2324      CachedJobs.fetch_add(1);2325    } else {2326      // If CacheAddStream is not null, we have a cache miss and we need to2327      // run the backend for codegen. Save cache 'add stream'2328      // function for a later use.2329      J.CacheAddStream = std::move(CacheAddStream);2330    }2331    return Error::success();2332  }2333 2334  Error start(2335      unsigned Task, BitcodeModule BM,2336      const FunctionImporter::ImportMapTy &ImportList,2337      const FunctionImporter::ExportSetTy &ExportList,2338      const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,2339      MapVector<StringRef, BitcodeModule> &ModuleMap) override {2340 2341    StringRef ModulePath = BM.getModuleIdentifier();2342 2343    SString ObjFilePath = sys::path::parent_path(LinkerOutputFile);2344    sys::path::append(ObjFilePath, sys::path::stem(ModulePath) + "." +2345                                       itostr(Task) + "." + UID + ".native.o");2346 2347    Job &J = Jobs[Task - ThinLTOTaskOffset];2348    J = {Task,2349         ModulePath,2350         Saver.save(ObjFilePath.str()),2351         Saver.save(ObjFilePath.str() + ".thinlto.bc"),2352         {}, // Filled in by emitFiles below.2353         "", /*CacheKey=*/2354         nullptr,2355         false};2356 2357    assert(ModuleToDefinedGVSummaries.count(ModulePath));2358 2359    // The BackendThreadPool is only used here to write the sharded index files2360    // (similar to WriteIndexesThinBackend).2361    BackendThreadPool.async(2362        [=](Job &J, const FunctionImporter::ImportMapTy &ImportList,2363            const FunctionImporter::ExportSetTy &ExportList,2364            const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>2365                &ResolvedODR) {2366          Error E =2367              runThinLTOBackendThread(J, ImportList, ExportList, ResolvedODR);2368          if (E) {2369            std::unique_lock<std::mutex> L(ErrMu);2370            if (Err)2371              Err = joinErrors(std::move(*Err), std::move(E));2372            else2373              Err = std::move(E);2374          }2375        },2376        std::ref(J), std::ref(ImportList), std::ref(ExportList),2377        std::ref(ResolvedODR));2378 2379    return Error::success();2380  }2381 2382  // Derive a set of Clang options that will be shared/common for all DTLTO2383  // backend compilations. We are intentionally minimal here as these options2384  // must remain synchronized with the behavior of Clang. DTLTO does not support2385  // all the features available with in-process LTO. More features are expected2386  // to be added over time. Users can specify Clang options directly if a2387  // feature is not supported. Note that explicitly specified options that imply2388  // additional input or output file dependencies must be communicated to the2389  // distribution system, potentially by setting extra options on the2390  // distributor program.2391  void buildCommonRemoteCompilerOptions() {2392    const lto::Config &C = Conf;2393    auto &Ops = CodegenOptions;2394 2395    Ops.push_back(Saver.save("-O" + Twine(C.OptLevel)));2396 2397    if (C.Options.EmitAddrsig)2398      Ops.push_back("-faddrsig");2399    if (C.Options.FunctionSections)2400      Ops.push_back("-ffunction-sections");2401    if (C.Options.DataSections)2402      Ops.push_back("-fdata-sections");2403 2404    if (C.RelocModel == Reloc::PIC_)2405      // Clang doesn't have -fpic for all triples.2406      if (!Triple.isOSBinFormatCOFF())2407        Ops.push_back("-fpic");2408 2409    // Turn on/off warnings about profile cfg mismatch (default on)2410    // --lto-pgo-warn-mismatch.2411    if (!C.PGOWarnMismatch) {2412      Ops.push_back("-mllvm");2413      Ops.push_back("-no-pgo-warn-mismatch");2414    }2415 2416    // Enable sample-based profile guided optimizations.2417    // Sample profile file path --lto-sample-profile=<value>.2418    if (!C.SampleProfile.empty()) {2419      Ops.push_back(2420          Saver.save("-fprofile-sample-use=" + Twine(C.SampleProfile)));2421      CommonInputs.insert(C.SampleProfile);2422    }2423 2424    // We don't know which of options will be used by Clang.2425    Ops.push_back("-Wno-unused-command-line-argument");2426 2427    // Forward any supplied options.2428    if (!RemoteCompilerArgs.empty())2429      for (auto &a : RemoteCompilerArgs)2430        Ops.push_back(a);2431  }2432 2433  // Generates a JSON file describing the backend compilations, for the2434  // distributor.2435  bool emitDistributorJson(StringRef DistributorJson) {2436    using json::Array;2437    std::error_code EC;2438    raw_fd_ostream OS(DistributorJson, EC);2439    if (EC)2440      return false;2441 2442    json::OStream JOS(OS);2443    JOS.object([&]() {2444      // Information common to all jobs.2445      JOS.attributeObject("common", [&]() {2446        JOS.attribute("linker_output", LinkerOutputFile);2447 2448        JOS.attributeArray("args", [&]() {2449          JOS.value(RemoteCompiler);2450 2451          // Forward any supplied prepend options.2452          if (!RemoteCompilerPrependArgs.empty())2453            for (auto &A : RemoteCompilerPrependArgs)2454              JOS.value(A);2455 2456          JOS.value("-c");2457 2458          JOS.value(Saver.save("--target=" + Triple.str()));2459 2460          for (const auto &A : CodegenOptions)2461            JOS.value(A);2462        });2463 2464        JOS.attribute("inputs", Array(CommonInputs));2465      });2466 2467      // Per-compilation-job information.2468      JOS.attributeArray("jobs", [&]() {2469        for (const auto &J : Jobs) {2470          assert(J.Task != 0);2471          if (J.Cached) {2472            assert(!Cache.getCacheDirectoryPath().empty());2473            continue;2474          }2475 2476          SmallVector<StringRef, 2> Inputs;2477          SmallVector<StringRef, 1> Outputs;2478 2479          JOS.object([&]() {2480            JOS.attributeArray("args", [&]() {2481              JOS.value(J.ModuleID);2482              Inputs.push_back(J.ModuleID);2483 2484              JOS.value(2485                  Saver.save("-fthinlto-index=" + Twine(J.SummaryIndexPath)));2486              Inputs.push_back(J.SummaryIndexPath);2487 2488              JOS.value("-o");2489              JOS.value(J.NativeObjectPath);2490              Outputs.push_back(J.NativeObjectPath);2491            });2492 2493            // Add the bitcode files from which imports will be made. These do2494            // not explicitly appear on the backend compilation command lines2495            // but are recorded in the summary index shards.2496            llvm::append_range(Inputs, J.ImportsFiles);2497            JOS.attribute("inputs", Array(Inputs));2498 2499            JOS.attribute("outputs", Array(Outputs));2500          });2501        }2502      });2503    });2504 2505    return true;2506  }2507 2508  void removeFile(StringRef FileName) {2509    std::error_code EC = sys::fs::remove(FileName, true);2510    if (EC && EC != std::make_error_code(std::errc::no_such_file_or_directory))2511      errs() << "warning: could not remove the file '" << FileName2512             << "': " << EC.message() << "\n";2513  }2514 2515  Error wait() override {2516    // Wait for the information on the required backend compilations to be2517    // gathered.2518    BackendThreadPool.wait();2519    if (Err)2520      return std::move(*Err);2521 2522    auto CleanPerJobFiles = llvm::make_scope_exit([&] {2523      if (!SaveTemps)2524        for (auto &Job : Jobs) {2525          removeFile(Job.NativeObjectPath);2526          if (!ShouldEmitIndexFiles)2527            removeFile(Job.SummaryIndexPath);2528        }2529    });2530 2531    const StringRef BCError = "DTLTO backend compilation: ";2532 2533    buildCommonRemoteCompilerOptions();2534 2535    SString JsonFile = sys::path::parent_path(LinkerOutputFile);2536    sys::path::append(JsonFile, sys::path::stem(LinkerOutputFile) + "." + UID +2537                                    ".dist-file.json");2538    if (!emitDistributorJson(JsonFile))2539      return make_error<StringError>(2540          BCError + "failed to generate distributor JSON script: " + JsonFile,2541          inconvertibleErrorCode());2542    auto CleanJson = llvm::make_scope_exit([&] {2543      if (!SaveTemps)2544        removeFile(JsonFile);2545    });2546 2547    // Checks if we have any jobs that don't have corresponding cache entries.2548    if (CachedJobs.load() < Jobs.size()) {2549      SmallVector<StringRef, 3> Args = {DistributorPath};2550      llvm::append_range(Args, DistributorArgs);2551      Args.push_back(JsonFile);2552      std::string ErrMsg;2553      if (sys::ExecuteAndWait(Args[0], Args,2554                              /*Env=*/std::nullopt, /*Redirects=*/{},2555                              /*SecondsToWait=*/0, /*MemoryLimit=*/0,2556                              &ErrMsg)) {2557        return make_error<StringError>(2558            BCError + "distributor execution failed" +2559                (!ErrMsg.empty() ? ": " + ErrMsg + Twine(".") : Twine(".")),2560            inconvertibleErrorCode());2561      }2562    }2563 2564    for (auto &Job : Jobs) {2565      if (!Job.CacheKey.empty() && Job.Cached) {2566        assert(Cache.isValid());2567        continue;2568      }2569      // Load the native object from a file into a memory buffer2570      // and store its contents in the output buffer.2571      auto ObjFileMbOrErr =2572          MemoryBuffer::getFile(Job.NativeObjectPath, /*IsText=*/false,2573                                /*RequiresNullTerminator=*/false);2574      if (std::error_code EC = ObjFileMbOrErr.getError())2575        return make_error<StringError>(2576            BCError + "cannot open native object file: " +2577                Job.NativeObjectPath + ": " + EC.message(),2578            inconvertibleErrorCode());2579 2580      MemoryBufferRef ObjFileMbRef = ObjFileMbOrErr->get()->getMemBufferRef();2581      if (Cache.isValid()) {2582        // Cache hits are taken care of earlier. At this point, we could only2583        // have cache misses.2584        assert(Job.CacheAddStream);2585        // Obtain a file stream for a storing a cache entry.2586        auto CachedFileStreamOrErr = Job.CacheAddStream(Job.Task, Job.ModuleID);2587        if (!CachedFileStreamOrErr)2588          return joinErrors(2589              CachedFileStreamOrErr.takeError(),2590              createStringError(inconvertibleErrorCode(),2591                                "Cannot get a cache file stream: %s",2592                                Job.NativeObjectPath.data()));2593        // Store a file buffer into the cache stream.2594        auto &CacheStream = *(CachedFileStreamOrErr->get());2595        *(CacheStream.OS) << ObjFileMbRef.getBuffer();2596        if (Error Err = CacheStream.commit())2597          return Err;2598      } else {2599        auto StreamOrErr = AddStream(Job.Task, Job.ModuleID);2600        if (Error Err = StreamOrErr.takeError())2601          report_fatal_error(std::move(Err));2602        auto &Stream = *StreamOrErr->get();2603        *Stream.OS << ObjFileMbRef.getBuffer();2604        if (Error Err = Stream.commit())2605          report_fatal_error(std::move(Err));2606      }2607    }2608    return Error::success();2609  }2610};2611} // end anonymous namespace2612 2613ThinBackend lto::createOutOfProcessThinBackend(2614    ThreadPoolStrategy Parallelism, lto::IndexWriteCallback OnWrite,2615    bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles,2616    StringRef LinkerOutputFile, StringRef Distributor,2617    ArrayRef<StringRef> DistributorArgs, StringRef RemoteCompiler,2618    ArrayRef<StringRef> RemoteCompilerPrependArgs,2619    ArrayRef<StringRef> RemoteCompilerArgs, bool SaveTemps) {2620  auto Func =2621      [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,2622          const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,2623          AddStreamFn AddStream, FileCache Cache) {2624        return std::make_unique<OutOfProcessThinBackend>(2625            Conf, CombinedIndex, Parallelism, ModuleToDefinedGVSummaries,2626            AddStream, Cache, OnWrite, ShouldEmitIndexFiles,2627            ShouldEmitImportsFiles, LinkerOutputFile, Distributor,2628            DistributorArgs, RemoteCompiler, RemoteCompilerPrependArgs,2629            RemoteCompilerArgs, SaveTemps);2630      };2631  return ThinBackend(Func, Parallelism);2632}2633