1697 lines · cpp
1//===- lib/Linker/IRMover.cpp ---------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "llvm/Linker/IRMover.h"10#include "LinkDiagnosticInfo.h"11#include "llvm/ADT/DenseMap.h"12#include "llvm/ADT/DenseSet.h"13#include "llvm/ADT/SetVector.h"14#include "llvm/ADT/SmallString.h"15#include "llvm/IR/AutoUpgrade.h"16#include "llvm/IR/Constants.h"17#include "llvm/IR/DebugInfoMetadata.h"18#include "llvm/IR/DiagnosticPrinter.h"19#include "llvm/IR/Function.h"20#include "llvm/IR/GVMaterializer.h"21#include "llvm/IR/GlobalValue.h"22#include "llvm/IR/Instruction.h"23#include "llvm/IR/Instructions.h"24#include "llvm/IR/Intrinsics.h"25#include "llvm/IR/Module.h"26#include "llvm/IR/PseudoProbe.h"27#include "llvm/IR/TypeFinder.h"28#include "llvm/Object/ModuleSymbolTable.h"29#include "llvm/Support/Error.h"30#include "llvm/TargetParser/Triple.h"31#include "llvm/Transforms/Utils/ValueMapper.h"32#include <optional>33#include <utility>34using namespace llvm;35 36/// Most of the errors produced by this module are inconvertible StringErrors.37/// This convenience function lets us return one of those more easily.38static Error stringErr(const Twine &T) {39 return make_error<StringError>(T, inconvertibleErrorCode());40}41 42//===----------------------------------------------------------------------===//43// TypeMap implementation.44//===----------------------------------------------------------------------===//45 46namespace {47class TypeMapTy : public ValueMapTypeRemapper {48 /// This is a mapping from a source type to a destination type to use.49 DenseMap<Type *, Type *> MappedTypes;50 51public:52 TypeMapTy(IRMover::IdentifiedStructTypeSet &DstStructTypesSet)53 : DstStructTypesSet(DstStructTypesSet) {}54 55 IRMover::IdentifiedStructTypeSet &DstStructTypesSet;56 /// Indicate that the specified type in the destination module is conceptually57 /// equivalent to the specified type in the source module.58 void addTypeMapping(Type *DstTy, Type *SrcTy);59 60 /// Return the mapped type to use for the specified input type from the61 /// source module.62 Type *get(Type *SrcTy);63 64 FunctionType *get(FunctionType *T) {65 return cast<FunctionType>(get((Type *)T));66 }67 68private:69 Type *remapType(Type *SrcTy) override { return get(SrcTy); }70 71 bool recursivelyAddMappingIfTypesAreIsomorphic(Type *DstTy, Type *SrcTy);72};73}74 75void TypeMapTy::addTypeMapping(Type *DstTy, Type *SrcTy) {76 recursivelyAddMappingIfTypesAreIsomorphic(DstTy, SrcTy);77}78 79/// Recursively walk this pair of types, returning true if they are isomorphic,80/// false if they are not. Types that were determined to be isomorphic are81/// added to MappedTypes.82bool TypeMapTy::recursivelyAddMappingIfTypesAreIsomorphic(Type *DstTy,83 Type *SrcTy) {84 // Two types with differing kinds are clearly not isomorphic.85 if (DstTy->getTypeID() != SrcTy->getTypeID())86 return false;87 88 // If we have an entry in the MappedTypes table, then we have our answer.89 Type *&Entry = MappedTypes[SrcTy];90 if (Entry)91 return Entry == DstTy;92 93 // Two identical types are clearly isomorphic. Remember this94 // non-speculatively.95 if (DstTy == SrcTy) {96 Entry = DstTy;97 return true;98 }99 100 // Okay, we have two types with identical kinds that we haven't seen before.101 102 // Always consider opaque struct types non-isomorphic.103 if (StructType *SSTy = dyn_cast<StructType>(SrcTy)) {104 if (SSTy->isOpaque() || cast<StructType>(DstTy)->isOpaque())105 return false;106 }107 108 // If the number of subtypes disagree between the two types, then we fail.109 if (SrcTy->getNumContainedTypes() != DstTy->getNumContainedTypes())110 return false;111 112 // Fail if any of the extra properties (e.g. array size) of the type disagree.113 if (isa<IntegerType>(DstTy))114 return false; // bitwidth disagrees.115 if (PointerType *PT = dyn_cast<PointerType>(DstTy)) {116 if (PT->getAddressSpace() != cast<PointerType>(SrcTy)->getAddressSpace())117 return false;118 } else if (FunctionType *FT = dyn_cast<FunctionType>(DstTy)) {119 if (FT->isVarArg() != cast<FunctionType>(SrcTy)->isVarArg())120 return false;121 } else if (StructType *DSTy = dyn_cast<StructType>(DstTy)) {122 StructType *SSTy = cast<StructType>(SrcTy);123 if (DSTy->isLiteral() != SSTy->isLiteral() ||124 DSTy->isPacked() != SSTy->isPacked())125 return false;126 } else if (auto *DArrTy = dyn_cast<ArrayType>(DstTy)) {127 if (DArrTy->getNumElements() != cast<ArrayType>(SrcTy)->getNumElements())128 return false;129 } else if (auto *DVecTy = dyn_cast<VectorType>(DstTy)) {130 if (DVecTy->getElementCount() != cast<VectorType>(SrcTy)->getElementCount())131 return false;132 }133 134 // Recursively check the subelements.135 for (unsigned I = 0, E = SrcTy->getNumContainedTypes(); I != E; ++I)136 if (!recursivelyAddMappingIfTypesAreIsomorphic(DstTy->getContainedType(I),137 SrcTy->getContainedType(I)))138 return false;139 140 // If everything seems to have lined up, then everything is great.141 [[maybe_unused]] auto Res = MappedTypes.insert({SrcTy, DstTy});142 assert(!Res.second && "Recursive type?");143 144 if (auto *STy = dyn_cast<StructType>(SrcTy)) {145 // We clear name of SrcTy to lower amount of renaming in LLVM context.146 // Renaming occurs because we load all source modules to the same context147 // and declaration with existing name gets renamed (i.e Foo -> Foo.42).148 // As a result we may get several different types in the destination149 // module, which are in fact the same.150 if (STy->hasName())151 STy->setName("");152 }153 154 return true;155}156 157Type *TypeMapTy::get(Type *Ty) {158 // If we already have an entry for this type, return it.159 Type **Entry = &MappedTypes[Ty];160 if (*Entry)161 return *Entry;162 163 // These are types that LLVM itself will unique.164 bool IsUniqued = !isa<StructType>(Ty) || cast<StructType>(Ty)->isLiteral();165 166 if (!IsUniqued) {167#ifndef NDEBUG168 for (auto &Pair : MappedTypes) {169 assert(!(Pair.first != Ty && Pair.second == Ty) &&170 "mapping to a source type");171 }172#endif173 }174 175 // If this is not a recursive type, then just map all of the elements and176 // then rebuild the type from inside out.177 SmallVector<Type *, 4> ElementTypes;178 179 // If there are no element types to map, then the type is itself. This is180 // true for the anonymous {} struct, things like 'float', integers, etc.181 if (Ty->getNumContainedTypes() == 0 && IsUniqued)182 return *Entry = Ty;183 184 // Remap all of the elements, keeping track of whether any of them change.185 bool AnyChange = false;186 ElementTypes.resize(Ty->getNumContainedTypes());187 for (unsigned I = 0, E = Ty->getNumContainedTypes(); I != E; ++I) {188 ElementTypes[I] = get(Ty->getContainedType(I));189 AnyChange |= ElementTypes[I] != Ty->getContainedType(I);190 }191 192 // Refresh Entry after recursively processing stuff.193 Entry = &MappedTypes[Ty];194 assert(!*Entry && "Recursive type!");195 196 // If all of the element types mapped directly over and the type is not197 // a named struct, then the type is usable as-is.198 if (!AnyChange && IsUniqued)199 return *Entry = Ty;200 201 // Otherwise, rebuild a modified type.202 switch (Ty->getTypeID()) {203 default:204 llvm_unreachable("unknown derived type to remap");205 case Type::ArrayTyID:206 return *Entry = ArrayType::get(ElementTypes[0],207 cast<ArrayType>(Ty)->getNumElements());208 case Type::ScalableVectorTyID:209 case Type::FixedVectorTyID:210 return *Entry = VectorType::get(ElementTypes[0],211 cast<VectorType>(Ty)->getElementCount());212 case Type::FunctionTyID:213 return *Entry = FunctionType::get(ElementTypes[0],214 ArrayRef(ElementTypes).slice(1),215 cast<FunctionType>(Ty)->isVarArg());216 case Type::StructTyID: {217 auto *STy = cast<StructType>(Ty);218 bool IsPacked = STy->isPacked();219 if (IsUniqued)220 return *Entry = StructType::get(Ty->getContext(), ElementTypes, IsPacked);221 222 // If the type is opaque, we can just use it directly.223 if (STy->isOpaque()) {224 DstStructTypesSet.addOpaque(STy);225 return *Entry = Ty;226 }227 228 if (StructType *OldT =229 DstStructTypesSet.findNonOpaque(ElementTypes, IsPacked)) {230 STy->setName("");231 return *Entry = OldT;232 }233 234 if (!AnyChange) {235 DstStructTypesSet.addNonOpaque(STy);236 return *Entry = Ty;237 }238 239 StructType *DTy =240 StructType::create(Ty->getContext(), ElementTypes, "", STy->isPacked());241 242 // Steal STy's name.243 if (STy->hasName()) {244 SmallString<16> TmpName = STy->getName();245 STy->setName("");246 DTy->setName(TmpName);247 }248 249 DstStructTypesSet.addNonOpaque(DTy);250 return *Entry = DTy;251 }252 }253}254 255LinkDiagnosticInfo::LinkDiagnosticInfo(DiagnosticSeverity Severity,256 const Twine &Msg)257 : DiagnosticInfo(DK_Linker, Severity), Msg(Msg) {}258void LinkDiagnosticInfo::print(DiagnosticPrinter &DP) const { DP << Msg; }259 260//===----------------------------------------------------------------------===//261// IRLinker implementation.262//===----------------------------------------------------------------------===//263 264namespace {265class IRLinker;266 267/// Creates prototypes for functions that are lazily linked on the fly. This268/// speeds up linking for modules with many/ lazily linked functions of which269/// few get used.270class GlobalValueMaterializer final : public ValueMaterializer {271 IRLinker &TheIRLinker;272 273public:274 GlobalValueMaterializer(IRLinker &TheIRLinker) : TheIRLinker(TheIRLinker) {}275 Value *materialize(Value *V) override;276};277 278class LocalValueMaterializer final : public ValueMaterializer {279 IRLinker &TheIRLinker;280 281public:282 LocalValueMaterializer(IRLinker &TheIRLinker) : TheIRLinker(TheIRLinker) {}283 Value *materialize(Value *V) override;284};285 286/// Type of the Metadata map in \a ValueToValueMapTy.287typedef DenseMap<const Metadata *, TrackingMDRef> MDMapT;288 289/// This is responsible for keeping track of the state used for moving data290/// from SrcM to DstM.291class IRLinker {292 Module &DstM;293 std::unique_ptr<Module> SrcM;294 295 // Lookup table to optimize IRMover::linkNamedMDNodes().296 IRMover::NamedMDNodesT &NamedMDNodes;297 298 /// See IRMover::move().299 IRMover::LazyCallback AddLazyFor;300 301 TypeMapTy TypeMap;302 GlobalValueMaterializer GValMaterializer;303 LocalValueMaterializer LValMaterializer;304 305 /// A metadata map that's shared between IRLinker instances.306 MDMapT &SharedMDs;307 308 /// Mapping of values from what they used to be in Src, to what they are now309 /// in DstM. ValueToValueMapTy is a ValueMap, which involves some overhead310 /// due to the use of Value handles which the Linker doesn't actually need,311 /// but this allows us to reuse the ValueMapper code.312 ValueToValueMapTy ValueMap;313 ValueToValueMapTy IndirectSymbolValueMap;314 315 DenseSet<GlobalValue *> ValuesToLink;316 std::vector<GlobalValue *> Worklist;317 std::vector<std::pair<GlobalValue *, Value*>> RAUWWorklist;318 319 /// Set of globals with eagerly copied metadata that may require remapping.320 /// This remapping is performed after metadata linking.321 DenseSet<GlobalObject *> UnmappedMetadata;322 323 void maybeAdd(GlobalValue *GV) {324 if (ValuesToLink.insert(GV).second)325 Worklist.push_back(GV);326 }327 328 /// Whether we are importing globals for ThinLTO, as opposed to linking the329 /// source module. If this flag is set, it means that we can rely on some330 /// other object file to define any non-GlobalValue entities defined by the331 /// source module. This currently causes us to not link retained types in332 /// debug info metadata and module inline asm.333 bool IsPerformingImport;334 335 /// Set to true when all global value body linking is complete (including336 /// lazy linking). Used to prevent metadata linking from creating new337 /// references.338 bool DoneLinkingBodies = false;339 340 /// The Error encountered during materialization. We use an Optional here to341 /// avoid needing to manage an unconsumed success value.342 std::optional<Error> FoundError;343 void setError(Error E) {344 if (E)345 FoundError = std::move(E);346 }347 348 /// Entry point for mapping values and alternate context for mapping aliases.349 ValueMapper Mapper;350 unsigned IndirectSymbolMCID;351 352 /// Handles cloning of a global values from the source module into353 /// the destination module, including setting the attributes and visibility.354 GlobalValue *copyGlobalValueProto(const GlobalValue *SGV, bool ForDefinition);355 356 void emitWarning(const Twine &Message) {357 SrcM->getContext().diagnose(LinkDiagnosticInfo(DS_Warning, Message));358 }359 360 /// Given a global in the source module, return the global in the361 /// destination module that is being linked to, if any.362 GlobalValue *getLinkedToGlobal(const GlobalValue *SrcGV) {363 // If the source has no name it can't link. If it has local linkage,364 // there is no name match-up going on.365 if (!SrcGV->hasName() || SrcGV->hasLocalLinkage())366 return nullptr;367 368 // Otherwise see if we have a match in the destination module's symtab.369 GlobalValue *DGV = DstM.getNamedValue(SrcGV->getName());370 if (!DGV)371 return nullptr;372 373 // If we found a global with the same name in the dest module, but it has374 // internal linkage, we are really not doing any linkage here.375 if (DGV->hasLocalLinkage())376 return nullptr;377 378 // If we found an intrinsic declaration with mismatching prototypes, we379 // probably had a nameclash. Don't use that version.380 if (auto *FDGV = dyn_cast<Function>(DGV))381 if (FDGV->isIntrinsic())382 if (const auto *FSrcGV = dyn_cast<Function>(SrcGV))383 if (FDGV->getFunctionType() != TypeMap.get(FSrcGV->getFunctionType()))384 return nullptr;385 386 // Otherwise, we do in fact link to the destination global.387 return DGV;388 }389 390 void computeTypeMapping();391 392 Expected<Constant *> linkAppendingVarProto(GlobalVariable *DstGV,393 const GlobalVariable *SrcGV);394 395 /// Given the GlobaValue \p SGV in the source module, and the matching396 /// GlobalValue \p DGV (if any), return true if the linker will pull \p SGV397 /// into the destination module.398 ///399 /// Note this code may call the client-provided \p AddLazyFor.400 bool shouldLink(GlobalValue *DGV, GlobalValue &SGV);401 Expected<Constant *> linkGlobalValueProto(GlobalValue *GV,402 bool ForIndirectSymbol);403 404 Error linkModuleFlagsMetadata();405 406 void linkGlobalVariable(GlobalVariable &Dst, GlobalVariable &Src);407 Error linkFunctionBody(Function &Dst, Function &Src);408 void linkAliasAliasee(GlobalAlias &Dst, GlobalAlias &Src);409 void linkIFuncResolver(GlobalIFunc &Dst, GlobalIFunc &Src);410 Error linkGlobalValueBody(GlobalValue &Dst, GlobalValue &Src);411 412 /// Replace all types in the source AttributeList with the413 /// corresponding destination type.414 AttributeList mapAttributeTypes(LLVMContext &C, AttributeList Attrs);415 416 /// Functions that take care of cloning a specific global value type417 /// into the destination module.418 GlobalVariable *copyGlobalVariableProto(const GlobalVariable *SGVar);419 Function *copyFunctionProto(const Function *SF);420 GlobalValue *copyIndirectSymbolProto(const GlobalValue *SGV);421 422 /// Perform "replace all uses with" operations. These work items need to be423 /// performed as part of materialization, but we postpone them to happen after424 /// materialization is done. The materializer called by ValueMapper is not425 /// expected to delete constants, as ValueMapper is holding pointers to some426 /// of them, but constant destruction may be indirectly triggered by RAUW.427 /// Hence, the need to move this out of the materialization call chain.428 void flushRAUWWorklist();429 430 /// When importing for ThinLTO, prevent importing of types listed on431 /// the DICompileUnit that we don't need a copy of in the importing432 /// module.433 void prepareCompileUnitsForImport();434 void linkNamedMDNodes();435 436 /// Update attributes while linking.437 void updateAttributes(GlobalValue &GV);438 439public:440 IRLinker(Module &DstM, MDMapT &SharedMDs,441 IRMover::IdentifiedStructTypeSet &Set, std::unique_ptr<Module> SrcM,442 ArrayRef<GlobalValue *> ValuesToLink,443 IRMover::LazyCallback AddLazyFor, bool IsPerformingImport,444 IRMover::NamedMDNodesT &NamedMDNodes)445 : DstM(DstM), SrcM(std::move(SrcM)), NamedMDNodes(NamedMDNodes),446 AddLazyFor(std::move(AddLazyFor)), TypeMap(Set),447 GValMaterializer(*this), LValMaterializer(*this), SharedMDs(SharedMDs),448 IsPerformingImport(IsPerformingImport),449 Mapper(ValueMap, RF_ReuseAndMutateDistinctMDs | RF_IgnoreMissingLocals,450 &TypeMap, &GValMaterializer),451 IndirectSymbolMCID(Mapper.registerAlternateMappingContext(452 IndirectSymbolValueMap, &LValMaterializer)) {453 ValueMap.getMDMap() = std::move(SharedMDs);454 for (GlobalValue *GV : ValuesToLink)455 maybeAdd(GV);456 if (IsPerformingImport)457 prepareCompileUnitsForImport();458 }459 ~IRLinker() { SharedMDs = std::move(*ValueMap.getMDMap()); }460 461 Error run();462 Value *materialize(Value *V, bool ForIndirectSymbol);463};464}465 466/// The LLVM SymbolTable class autorenames globals that conflict in the symbol467/// table. This is good for all clients except for us. Go through the trouble468/// to force this back.469static void forceRenaming(GlobalValue *GV, StringRef Name) {470 // If the global doesn't force its name or if it already has the right name,471 // there is nothing for us to do.472 if (GV->hasLocalLinkage() || GV->getName() == Name)473 return;474 475 Module *M = GV->getParent();476 477 // If there is a conflict, rename the conflict.478 if (GlobalValue *ConflictGV = M->getNamedValue(Name)) {479 GV->takeName(ConflictGV);480 ConflictGV->setName(Name); // This will cause ConflictGV to get renamed481 assert(ConflictGV->getName() != Name && "forceRenaming didn't work");482 } else {483 GV->setName(Name); // Force the name back484 }485}486 487Value *GlobalValueMaterializer::materialize(Value *SGV) {488 return TheIRLinker.materialize(SGV, false);489}490 491Value *LocalValueMaterializer::materialize(Value *SGV) {492 return TheIRLinker.materialize(SGV, true);493}494 495Value *IRLinker::materialize(Value *V, bool ForIndirectSymbol) {496 auto *SGV = dyn_cast<GlobalValue>(V);497 if (!SGV)498 return nullptr;499 500 // If SGV is from dest, it was already materialized when dest was loaded.501 if (SGV->getParent() == &DstM)502 return nullptr;503 504 // When linking a global from other modules than source & dest, skip505 // materializing it because it would be mapped later when its containing506 // module is linked. Linking it now would potentially pull in many types that507 // may not be mapped properly.508 if (SGV->getParent() != SrcM.get())509 return nullptr;510 511 Expected<Constant *> NewProto = linkGlobalValueProto(SGV, ForIndirectSymbol);512 if (!NewProto) {513 setError(NewProto.takeError());514 return nullptr;515 }516 if (!*NewProto)517 return nullptr;518 519 GlobalValue *New = dyn_cast<GlobalValue>(*NewProto);520 if (!New)521 return *NewProto;522 523 // If we already created the body, just return.524 if (auto *F = dyn_cast<Function>(New)) {525 if (!F->isDeclaration())526 return New;527 } else if (auto *V = dyn_cast<GlobalVariable>(New)) {528 if (V->hasInitializer() || V->hasAppendingLinkage())529 return New;530 } else if (auto *GA = dyn_cast<GlobalAlias>(New)) {531 if (GA->getAliasee())532 return New;533 } else if (auto *GI = dyn_cast<GlobalIFunc>(New)) {534 if (GI->getResolver())535 return New;536 } else {537 llvm_unreachable("Invalid GlobalValue type");538 }539 540 // If the global is being linked for an indirect symbol, it may have already541 // been scheduled to satisfy a regular symbol. Similarly, a global being linked542 // for a regular symbol may have already been scheduled for an indirect543 // symbol. Check for these cases by looking in the other value map and544 // confirming the same value has been scheduled. If there is an entry in the545 // ValueMap but the value is different, it means that the value already had a546 // definition in the destination module (linkonce for instance), but we need a547 // new definition for the indirect symbol ("New" will be different).548 if ((ForIndirectSymbol && ValueMap.lookup(SGV) == New) ||549 (!ForIndirectSymbol && IndirectSymbolValueMap.lookup(SGV) == New))550 return New;551 552 if (ForIndirectSymbol || shouldLink(New, *SGV))553 setError(linkGlobalValueBody(*New, *SGV));554 555 updateAttributes(*New);556 return New;557}558 559/// Loop through the global variables in the src module and merge them into the560/// dest module.561GlobalVariable *IRLinker::copyGlobalVariableProto(const GlobalVariable *SGVar) {562 // No linking to be performed or linking from the source: simply create an563 // identical version of the symbol over in the dest module... the564 // initializer will be filled in later by LinkGlobalInits.565 GlobalVariable *NewDGV =566 new GlobalVariable(DstM, TypeMap.get(SGVar->getValueType()),567 SGVar->isConstant(), GlobalValue::ExternalLinkage,568 /*init*/ nullptr, SGVar->getName(),569 /*insertbefore*/ nullptr, SGVar->getThreadLocalMode(),570 SGVar->getAddressSpace());571 NewDGV->setAlignment(SGVar->getAlign());572 NewDGV->copyAttributesFrom(SGVar);573 return NewDGV;574}575 576AttributeList IRLinker::mapAttributeTypes(LLVMContext &C, AttributeList Attrs) {577 for (unsigned i = 0; i < Attrs.getNumAttrSets(); ++i) {578 for (int AttrIdx = Attribute::FirstTypeAttr;579 AttrIdx <= Attribute::LastTypeAttr; AttrIdx++) {580 Attribute::AttrKind TypedAttr = (Attribute::AttrKind)AttrIdx;581 if (Attrs.hasAttributeAtIndex(i, TypedAttr)) {582 if (Type *Ty =583 Attrs.getAttributeAtIndex(i, TypedAttr).getValueAsType()) {584 Attrs = Attrs.replaceAttributeTypeAtIndex(C, i, TypedAttr,585 TypeMap.get(Ty));586 break;587 }588 }589 }590 }591 return Attrs;592}593 594/// Link the function in the source module into the destination module if595/// needed, setting up mapping information.596Function *IRLinker::copyFunctionProto(const Function *SF) {597 // If there is no linkage to be performed or we are linking from the source,598 // bring SF over.599 auto *F = Function::Create(TypeMap.get(SF->getFunctionType()),600 GlobalValue::ExternalLinkage,601 SF->getAddressSpace(), SF->getName(), &DstM);602 F->copyAttributesFrom(SF);603 F->setAttributes(mapAttributeTypes(F->getContext(), F->getAttributes()));604 return F;605}606 607/// Set up prototypes for any indirect symbols that come over from the source608/// module.609GlobalValue *IRLinker::copyIndirectSymbolProto(const GlobalValue *SGV) {610 // If there is no linkage to be performed or we're linking from the source,611 // bring over SGA.612 auto *Ty = TypeMap.get(SGV->getValueType());613 614 if (auto *GA = dyn_cast<GlobalAlias>(SGV)) {615 auto *DGA = GlobalAlias::create(Ty, SGV->getAddressSpace(),616 GlobalValue::ExternalLinkage,617 SGV->getName(), &DstM);618 DGA->copyAttributesFrom(GA);619 return DGA;620 }621 622 if (auto *GI = dyn_cast<GlobalIFunc>(SGV)) {623 auto *DGI = GlobalIFunc::create(Ty, SGV->getAddressSpace(),624 GlobalValue::ExternalLinkage,625 SGV->getName(), nullptr, &DstM);626 DGI->copyAttributesFrom(GI);627 return DGI;628 }629 630 llvm_unreachable("Invalid source global value type");631}632 633GlobalValue *IRLinker::copyGlobalValueProto(const GlobalValue *SGV,634 bool ForDefinition) {635 GlobalValue *NewGV;636 if (auto *SGVar = dyn_cast<GlobalVariable>(SGV)) {637 NewGV = copyGlobalVariableProto(SGVar);638 } else if (auto *SF = dyn_cast<Function>(SGV)) {639 NewGV = copyFunctionProto(SF);640 } else {641 if (ForDefinition)642 NewGV = copyIndirectSymbolProto(SGV);643 else if (SGV->getValueType()->isFunctionTy())644 NewGV =645 Function::Create(cast<FunctionType>(TypeMap.get(SGV->getValueType())),646 GlobalValue::ExternalLinkage, SGV->getAddressSpace(),647 SGV->getName(), &DstM);648 else649 NewGV =650 new GlobalVariable(DstM, TypeMap.get(SGV->getValueType()),651 /*isConstant*/ false, GlobalValue::ExternalLinkage,652 /*init*/ nullptr, SGV->getName(),653 /*insertbefore*/ nullptr,654 SGV->getThreadLocalMode(), SGV->getAddressSpace());655 }656 657 if (ForDefinition)658 NewGV->setLinkage(SGV->getLinkage());659 else if (SGV->hasExternalWeakLinkage())660 NewGV->setLinkage(GlobalValue::ExternalWeakLinkage);661 662 if (auto *NewGO = dyn_cast<GlobalObject>(NewGV)) {663 // Metadata for global variables and function declarations is copied eagerly.664 if (isa<GlobalVariable>(SGV) || SGV->isDeclaration()) {665 NewGO->copyMetadata(cast<GlobalObject>(SGV), 0);666 if (SGV->isDeclaration() && NewGO->hasMetadata())667 UnmappedMetadata.insert(NewGO);668 }669 }670 671 // Remove these copied constants in case this stays a declaration, since672 // they point to the source module. If the def is linked the values will673 // be mapped in during linkFunctionBody.674 if (auto *NewF = dyn_cast<Function>(NewGV)) {675 NewF->setPersonalityFn(nullptr);676 NewF->setPrefixData(nullptr);677 NewF->setPrologueData(nullptr);678 }679 680 return NewGV;681}682 683static StringRef getTypeNamePrefix(StringRef Name) {684 size_t DotPos = Name.rfind('.');685 return (DotPos == 0 || DotPos == StringRef::npos || Name.back() == '.' ||686 !isdigit(static_cast<unsigned char>(Name[DotPos + 1])))687 ? Name688 : Name.substr(0, DotPos);689}690 691/// Loop over all of the linked values to compute type mappings. For example,692/// if we link "extern Foo *x" and "Foo *x = NULL", then we have two struct693/// types 'Foo' but one got renamed when the module was loaded into the same694/// LLVMContext.695void IRLinker::computeTypeMapping() {696 for (GlobalValue &SGV : SrcM->globals()) {697 GlobalValue *DGV = getLinkedToGlobal(&SGV);698 if (!DGV)699 continue;700 701 if (!DGV->hasAppendingLinkage() || !SGV.hasAppendingLinkage()) {702 TypeMap.addTypeMapping(DGV->getType(), SGV.getType());703 continue;704 }705 706 // Unify the element type of appending arrays.707 ArrayType *DAT = cast<ArrayType>(DGV->getValueType());708 ArrayType *SAT = cast<ArrayType>(SGV.getValueType());709 TypeMap.addTypeMapping(DAT->getElementType(), SAT->getElementType());710 }711 712 for (GlobalValue &SGV : *SrcM)713 if (GlobalValue *DGV = getLinkedToGlobal(&SGV)) {714 if (DGV->getType() == SGV.getType()) {715 // If the types of DGV and SGV are the same, it means that DGV is from716 // the source module and got added to DstM from a shared metadata. We717 // shouldn't map this type to itself in case the type's components get718 // remapped to a new type from DstM (for instance, during the loop over719 // SrcM->getIdentifiedStructTypes() below).720 continue;721 }722 723 TypeMap.addTypeMapping(DGV->getType(), SGV.getType());724 }725 726 for (GlobalValue &SGV : SrcM->aliases())727 if (GlobalValue *DGV = getLinkedToGlobal(&SGV))728 TypeMap.addTypeMapping(DGV->getType(), SGV.getType());729 730 // Incorporate types by name, scanning all the types in the source module.731 // At this point, the destination module may have a type "%foo = { i32 }" for732 // example. When the source module got loaded into the same LLVMContext, if733 // it had the same type, it would have been renamed to "%foo.42 = { i32 }".734 std::vector<StructType *> Types = SrcM->getIdentifiedStructTypes();735 for (StructType *ST : Types) {736 if (!ST->hasName())737 continue;738 739 if (TypeMap.DstStructTypesSet.hasType(ST)) {740 // This is actually a type from the destination module.741 // getIdentifiedStructTypes() can have found it by walking debug info742 // metadata nodes, some of which get linked by name when ODR Type Uniquing743 // is enabled on the Context, from the source to the destination module.744 continue;745 }746 747 auto STTypePrefix = getTypeNamePrefix(ST->getName());748 if (STTypePrefix.size() == ST->getName().size())749 continue;750 751 // Check to see if the destination module has a struct with the prefix name.752 StructType *DST = StructType::getTypeByName(ST->getContext(), STTypePrefix);753 if (!DST)754 continue;755 756 // Don't use it if this actually came from the source module. They're in757 // the same LLVMContext after all. Also don't use it unless the type is758 // actually used in the destination module. This can happen in situations759 // like this:760 //761 // Module A Module B762 // -------- --------763 // %Z = type { %A } %B = type { %C.1 }764 // %A = type { %B.1, [7 x i8] } %C.1 = type { i8* }765 // %B.1 = type { %C } %A.2 = type { %B.3, [5 x i8] }766 // %C = type { i8* } %B.3 = type { %C.1 }767 //768 // When we link Module B with Module A, the '%B' in Module B is769 // used. However, that would then use '%C.1'. But when we process '%C.1',770 // we prefer to take the '%C' version. So we are then left with both771 // '%C.1' and '%C' being used for the same types. This leads to some772 // variables using one type and some using the other.773 if (TypeMap.DstStructTypesSet.hasType(DST))774 TypeMap.addTypeMapping(DST, ST);775 }776}777 778static void getArrayElements(const Constant *C,779 SmallVectorImpl<Constant *> &Dest) {780 unsigned NumElements = cast<ArrayType>(C->getType())->getNumElements();781 782 for (unsigned i = 0; i != NumElements; ++i)783 Dest.push_back(C->getAggregateElement(i));784}785 786/// If there were any appending global variables, link them together now.787Expected<Constant *>788IRLinker::linkAppendingVarProto(GlobalVariable *DstGV,789 const GlobalVariable *SrcGV) {790 // Check that both variables have compatible properties.791 if (DstGV && !DstGV->isDeclaration() && !SrcGV->isDeclaration()) {792 if (!SrcGV->hasAppendingLinkage() || !DstGV->hasAppendingLinkage())793 return stringErr(794 "Linking globals named '" + SrcGV->getName() +795 "': can only link appending global with another appending "796 "global!");797 798 if (DstGV->isConstant() != SrcGV->isConstant())799 return stringErr("Appending variables linked with different const'ness!");800 801 if (DstGV->getAlign() != SrcGV->getAlign())802 return stringErr(803 "Appending variables with different alignment need to be linked!");804 805 if (DstGV->getVisibility() != SrcGV->getVisibility())806 return stringErr(807 "Appending variables with different visibility need to be linked!");808 809 if (DstGV->hasGlobalUnnamedAddr() != SrcGV->hasGlobalUnnamedAddr())810 return stringErr(811 "Appending variables with different unnamed_addr need to be linked!");812 813 if (DstGV->getSection() != SrcGV->getSection())814 return stringErr(815 "Appending variables with different section name need to be linked!");816 817 if (DstGV->getAddressSpace() != SrcGV->getAddressSpace())818 return stringErr("Appending variables with different address spaces need "819 "to be linked!");820 }821 822 // Do not need to do anything if source is a declaration.823 if (SrcGV->isDeclaration())824 return DstGV;825 826 Type *EltTy = cast<ArrayType>(TypeMap.get(SrcGV->getValueType()))827 ->getElementType();828 829 // FIXME: This upgrade is done during linking to support the C API. Once the830 // old form is deprecated, we should move this upgrade to831 // llvm::UpgradeGlobalVariable() and simplify the logic here and in832 // Mapper::mapAppendingVariable() in ValueMapper.cpp.833 StringRef Name = SrcGV->getName();834 bool IsNewStructor = false;835 bool IsOldStructor = false;836 if (Name == "llvm.global_ctors" || Name == "llvm.global_dtors") {837 if (cast<StructType>(EltTy)->getNumElements() == 3)838 IsNewStructor = true;839 else840 IsOldStructor = true;841 }842 843 PointerType *VoidPtrTy = PointerType::get(SrcGV->getContext(), 0);844 if (IsOldStructor) {845 auto &ST = *cast<StructType>(EltTy);846 Type *Tys[3] = {ST.getElementType(0), ST.getElementType(1), VoidPtrTy};847 EltTy = StructType::get(SrcGV->getContext(), Tys, false);848 }849 850 uint64_t DstNumElements = 0;851 if (DstGV && !DstGV->isDeclaration()) {852 ArrayType *DstTy = cast<ArrayType>(DstGV->getValueType());853 DstNumElements = DstTy->getNumElements();854 855 // Check to see that they two arrays agree on type.856 if (EltTy != DstTy->getElementType())857 return stringErr("Appending variables with different element types!");858 }859 860 SmallVector<Constant *, 16> SrcElements;861 getArrayElements(SrcGV->getInitializer(), SrcElements);862 863 if (IsNewStructor) {864 erase_if(SrcElements, [this](Constant *E) {865 auto *Key =866 dyn_cast<GlobalValue>(E->getAggregateElement(2)->stripPointerCasts());867 if (!Key)868 return false;869 GlobalValue *DGV = getLinkedToGlobal(Key);870 return !shouldLink(DGV, *Key);871 });872 }873 uint64_t NewSize = DstNumElements + SrcElements.size();874 ArrayType *NewType = ArrayType::get(EltTy, NewSize);875 876 // Create the new global variable.877 GlobalVariable *NG = new GlobalVariable(878 DstM, NewType, SrcGV->isConstant(), SrcGV->getLinkage(),879 /*init*/ nullptr, /*name*/ "", DstGV, SrcGV->getThreadLocalMode(),880 SrcGV->getAddressSpace());881 882 NG->copyAttributesFrom(SrcGV);883 forceRenaming(NG, SrcGV->getName());884 885 Mapper.scheduleMapAppendingVariable(*NG, DstGV, IsOldStructor, SrcElements);886 887 // Replace any uses of the two global variables with uses of the new888 // global.889 if (DstGV) {890 RAUWWorklist.push_back(std::make_pair(DstGV, NG));891 }892 893 return NG;894}895 896bool IRLinker::shouldLink(GlobalValue *DGV, GlobalValue &SGV) {897 if (ValuesToLink.count(&SGV) || SGV.hasLocalLinkage())898 return true;899 900 if (DGV && !DGV->isDeclarationForLinker())901 return false;902 903 if (SGV.isDeclaration() || DoneLinkingBodies)904 return false;905 906 // Callback to the client to give a chance to lazily add the Global to the907 // list of value to link.908 bool LazilyAdded = false;909 if (AddLazyFor)910 AddLazyFor(SGV, [this, &LazilyAdded](GlobalValue &GV) {911 maybeAdd(&GV);912 LazilyAdded = true;913 });914 return LazilyAdded;915}916 917Expected<Constant *> IRLinker::linkGlobalValueProto(GlobalValue *SGV,918 bool ForIndirectSymbol) {919 GlobalValue *DGV = getLinkedToGlobal(SGV);920 921 bool ShouldLink = shouldLink(DGV, *SGV);922 923 // just missing from map924 if (ShouldLink) {925 auto I = ValueMap.find(SGV);926 if (I != ValueMap.end())927 return cast<Constant>(I->second);928 929 I = IndirectSymbolValueMap.find(SGV);930 if (I != IndirectSymbolValueMap.end())931 return cast<Constant>(I->second);932 }933 934 if (!ShouldLink && ForIndirectSymbol)935 DGV = nullptr;936 937 // Handle the ultra special appending linkage case first.938 if (SGV->hasAppendingLinkage() || (DGV && DGV->hasAppendingLinkage()))939 return linkAppendingVarProto(cast_or_null<GlobalVariable>(DGV),940 cast<GlobalVariable>(SGV));941 942 bool NeedsRenaming = false;943 GlobalValue *NewGV;944 if (DGV && !ShouldLink) {945 NewGV = DGV;946 } else {947 // If we are done linking global value bodies (i.e. we are performing948 // metadata linking), don't link in the global value due to this949 // reference, simply map it to null.950 if (DoneLinkingBodies)951 return nullptr;952 953 NewGV = copyGlobalValueProto(SGV, ShouldLink || ForIndirectSymbol);954 if (ShouldLink || !ForIndirectSymbol)955 NeedsRenaming = true;956 }957 958 // Overloaded intrinsics have overloaded types names as part of their959 // names. If we renamed overloaded types we should rename the intrinsic960 // as well.961 if (Function *F = dyn_cast<Function>(NewGV))962 if (auto Remangled = Intrinsic::remangleIntrinsicFunction(F)) {963 // Note: remangleIntrinsicFunction does not copy metadata and as such964 // F should not occur in the set of objects with unmapped metadata.965 // If this assertion fails then remangleIntrinsicFunction needs updating.966 assert(!UnmappedMetadata.count(F) && "intrinsic has unmapped metadata");967 NewGV->eraseFromParent();968 NewGV = *Remangled;969 NeedsRenaming = false;970 }971 972 if (NeedsRenaming)973 forceRenaming(NewGV, SGV->getName());974 975 if (ShouldLink || ForIndirectSymbol) {976 if (const Comdat *SC = SGV->getComdat()) {977 if (auto *GO = dyn_cast<GlobalObject>(NewGV)) {978 Comdat *DC = DstM.getOrInsertComdat(SC->getName());979 DC->setSelectionKind(SC->getSelectionKind());980 GO->setComdat(DC);981 }982 }983 }984 985 if (!ShouldLink && ForIndirectSymbol)986 NewGV->setLinkage(GlobalValue::InternalLinkage);987 988 Constant *C = NewGV;989 // Only create a bitcast if necessary. In particular, with990 // DebugTypeODRUniquing we may reach metadata in the destination module991 // containing a GV from the source module, in which case SGV will be992 // the same as DGV and NewGV, and TypeMap.get() will assert since it993 // assumes it is being invoked on a type in the source module.994 if (DGV && NewGV != SGV) {995 C = ConstantExpr::getPointerBitCastOrAddrSpaceCast(996 NewGV, TypeMap.get(SGV->getType()));997 }998 999 if (DGV && NewGV != DGV) {1000 // Schedule "replace all uses with" to happen after materializing is1001 // done. It is not safe to do it now, since ValueMapper may be holding1002 // pointers to constants that will get deleted if RAUW runs.1003 RAUWWorklist.push_back(std::make_pair(1004 DGV,1005 ConstantExpr::getPointerBitCastOrAddrSpaceCast(NewGV, DGV->getType())));1006 }1007 1008 return C;1009}1010 1011/// Update the initializers in the Dest module now that all globals that may be1012/// referenced are in Dest.1013void IRLinker::linkGlobalVariable(GlobalVariable &Dst, GlobalVariable &Src) {1014 // Figure out what the initializer looks like in the dest module.1015 Mapper.scheduleMapGlobalInitializer(Dst, *Src.getInitializer());1016}1017 1018/// Copy the source function over into the dest function and fix up references1019/// to values. At this point we know that Dest is an external function, and1020/// that Src is not.1021Error IRLinker::linkFunctionBody(Function &Dst, Function &Src) {1022 assert(Dst.isDeclaration() && !Src.isDeclaration());1023 1024 // Materialize if needed.1025 if (Error Err = Src.materialize())1026 return Err;1027 1028 // Link in the operands without remapping.1029 if (Src.hasPrefixData())1030 Dst.setPrefixData(Src.getPrefixData());1031 if (Src.hasPrologueData())1032 Dst.setPrologueData(Src.getPrologueData());1033 if (Src.hasPersonalityFn())1034 Dst.setPersonalityFn(Src.getPersonalityFn());1035 1036 // Copy over the metadata attachments without remapping.1037 Dst.copyMetadata(&Src, 0);1038 1039 // Steal arguments and splice the body of Src into Dst.1040 Dst.stealArgumentListFrom(Src);1041 Dst.splice(Dst.end(), &Src);1042 1043 // Everything has been moved over. Remap it.1044 Mapper.scheduleRemapFunction(Dst);1045 return Error::success();1046}1047 1048void IRLinker::linkAliasAliasee(GlobalAlias &Dst, GlobalAlias &Src) {1049 Mapper.scheduleMapGlobalAlias(Dst, *Src.getAliasee(), IndirectSymbolMCID);1050}1051 1052void IRLinker::linkIFuncResolver(GlobalIFunc &Dst, GlobalIFunc &Src) {1053 Mapper.scheduleMapGlobalIFunc(Dst, *Src.getResolver(), IndirectSymbolMCID);1054}1055 1056Error IRLinker::linkGlobalValueBody(GlobalValue &Dst, GlobalValue &Src) {1057 if (auto *F = dyn_cast<Function>(&Src))1058 return linkFunctionBody(cast<Function>(Dst), *F);1059 if (auto *GVar = dyn_cast<GlobalVariable>(&Src)) {1060 linkGlobalVariable(cast<GlobalVariable>(Dst), *GVar);1061 return Error::success();1062 }1063 if (auto *GA = dyn_cast<GlobalAlias>(&Src)) {1064 linkAliasAliasee(cast<GlobalAlias>(Dst), *GA);1065 return Error::success();1066 }1067 linkIFuncResolver(cast<GlobalIFunc>(Dst), cast<GlobalIFunc>(Src));1068 return Error::success();1069}1070 1071void IRLinker::flushRAUWWorklist() {1072 for (const auto &Elem : RAUWWorklist) {1073 GlobalValue *Old;1074 Value *New;1075 std::tie(Old, New) = Elem;1076 1077 Old->replaceAllUsesWith(New);1078 Old->eraseFromParent();1079 }1080 RAUWWorklist.clear();1081}1082 1083void IRLinker::prepareCompileUnitsForImport() {1084 NamedMDNode *SrcCompileUnits = SrcM->getNamedMetadata("llvm.dbg.cu");1085 if (!SrcCompileUnits)1086 return;1087 // When importing for ThinLTO, prevent importing of types listed on1088 // the DICompileUnit that we don't need a copy of in the importing1089 // module. They will be emitted by the originating module.1090 for (MDNode *N : SrcCompileUnits->operands()) {1091 auto *CU = cast<DICompileUnit>(N);1092 assert(CU && "Expected valid compile unit");1093 // Enums, macros, and retained types don't need to be listed on the1094 // imported DICompileUnit. This means they will only be imported1095 // if reached from the mapped IR.1096 CU->replaceEnumTypes(nullptr);1097 CU->replaceMacros(nullptr);1098 CU->replaceRetainedTypes(nullptr);1099 1100 // The original definition (or at least its debug info - if the variable is1101 // internalized and optimized away) will remain in the source module, so1102 // there's no need to import them.1103 // If LLVM ever does more advanced optimizations on global variables1104 // (removing/localizing write operations, for instance) that can track1105 // through debug info, this decision may need to be revisited - but do so1106 // with care when it comes to debug info size. Emitting small CUs containing1107 // only a few imported entities into every destination module may be very1108 // size inefficient.1109 CU->replaceGlobalVariables(nullptr);1110 1111 CU->replaceImportedEntities(nullptr);1112 }1113}1114 1115/// Insert all of the named MDNodes in Src into the Dest module.1116void IRLinker::linkNamedMDNodes() {1117 const NamedMDNode *SrcModFlags = SrcM->getModuleFlagsMetadata();1118 for (const NamedMDNode &NMD : SrcM->named_metadata()) {1119 // Don't link module flags here. Do them separately.1120 if (&NMD == SrcModFlags)1121 continue;1122 // Don't import pseudo probe descriptors here for thinLTO. They will be1123 // emitted by the originating module.1124 if (IsPerformingImport && NMD.getName() == PseudoProbeDescMetadataName) {1125 if (!DstM.getNamedMetadata(NMD.getName()))1126 emitWarning("Pseudo-probe ignored: source module '" +1127 SrcM->getModuleIdentifier() +1128 "' is compiled with -fpseudo-probe-for-profiling while "1129 "destination module '" +1130 DstM.getModuleIdentifier() + "' is not\n");1131 continue;1132 }1133 // The stats are computed per module and will all be merged in the binary.1134 // Importing the metadata will cause duplication of the stats.1135 if (IsPerformingImport && NMD.getName() == "llvm.stats")1136 continue;1137 1138 NamedMDNode *DestNMD = DstM.getOrInsertNamedMetadata(NMD.getName());1139 1140 auto &Inserted = NamedMDNodes[DestNMD];1141 if (Inserted.empty()) {1142 // Must be the first module, copy everything from DestNMD.1143 Inserted.insert(DestNMD->operands().begin(), DestNMD->operands().end());1144 }1145 1146 // Add Src elements into Dest node.1147 for (const MDNode *Op : NMD.operands()) {1148 MDNode *MD = Mapper.mapMDNode(*Op);1149 if (Inserted.insert(MD).second)1150 DestNMD->addOperand(MD);1151 }1152 }1153}1154 1155/// Merge the linker flags in Src into the Dest module.1156Error IRLinker::linkModuleFlagsMetadata() {1157 // If the source module has no module flags, we are done.1158 const NamedMDNode *SrcModFlags = SrcM->getModuleFlagsMetadata();1159 if (!SrcModFlags)1160 return Error::success();1161 1162 // Check for module flag for updates before do anything.1163 UpgradeModuleFlags(*SrcM);1164 UpgradeNVVMAnnotations(*SrcM);1165 1166 // If the destination module doesn't have module flags yet, then just copy1167 // over the source module's flags.1168 NamedMDNode *DstModFlags = DstM.getOrInsertModuleFlagsMetadata();1169 if (DstModFlags->getNumOperands() == 0) {1170 for (unsigned I = 0, E = SrcModFlags->getNumOperands(); I != E; ++I)1171 DstModFlags->addOperand(SrcModFlags->getOperand(I));1172 1173 return Error::success();1174 }1175 1176 // First build a map of the existing module flags and requirements.1177 DenseMap<MDString *, std::pair<MDNode *, unsigned>> Flags;1178 SmallSetVector<MDNode *, 16> Requirements;1179 SmallVector<unsigned, 0> Mins;1180 DenseSet<MDString *> SeenMin;1181 for (unsigned I = 0, E = DstModFlags->getNumOperands(); I != E; ++I) {1182 MDNode *Op = DstModFlags->getOperand(I);1183 uint64_t Behavior =1184 mdconst::extract<ConstantInt>(Op->getOperand(0))->getZExtValue();1185 MDString *ID = cast<MDString>(Op->getOperand(1));1186 1187 if (Behavior == Module::Require) {1188 Requirements.insert(cast<MDNode>(Op->getOperand(2)));1189 } else {1190 if (Behavior == Module::Min)1191 Mins.push_back(I);1192 Flags[ID] = std::make_pair(Op, I);1193 }1194 }1195 1196 // Merge in the flags from the source module, and also collect its set of1197 // requirements.1198 for (unsigned I = 0, E = SrcModFlags->getNumOperands(); I != E; ++I) {1199 MDNode *SrcOp = SrcModFlags->getOperand(I);1200 ConstantInt *SrcBehavior =1201 mdconst::extract<ConstantInt>(SrcOp->getOperand(0));1202 MDString *ID = cast<MDString>(SrcOp->getOperand(1));1203 MDNode *DstOp;1204 unsigned DstIndex;1205 std::tie(DstOp, DstIndex) = Flags.lookup(ID);1206 unsigned SrcBehaviorValue = SrcBehavior->getZExtValue();1207 SeenMin.insert(ID);1208 1209 // If this is a requirement, add it and continue.1210 if (SrcBehaviorValue == Module::Require) {1211 // If the destination module does not already have this requirement, add1212 // it.1213 if (Requirements.insert(cast<MDNode>(SrcOp->getOperand(2)))) {1214 DstModFlags->addOperand(SrcOp);1215 }1216 continue;1217 }1218 1219 // If there is no existing flag with this ID, just add it.1220 if (!DstOp) {1221 if (SrcBehaviorValue == Module::Min) {1222 Mins.push_back(DstModFlags->getNumOperands());1223 SeenMin.erase(ID);1224 }1225 Flags[ID] = std::make_pair(SrcOp, DstModFlags->getNumOperands());1226 DstModFlags->addOperand(SrcOp);1227 continue;1228 }1229 1230 // Otherwise, perform a merge.1231 ConstantInt *DstBehavior =1232 mdconst::extract<ConstantInt>(DstOp->getOperand(0));1233 unsigned DstBehaviorValue = DstBehavior->getZExtValue();1234 1235 auto overrideDstValue = [&]() {1236 DstModFlags->setOperand(DstIndex, SrcOp);1237 Flags[ID].first = SrcOp;1238 };1239 1240 // If either flag has override behavior, handle it first.1241 if (DstBehaviorValue == Module::Override) {1242 // Diagnose inconsistent flags which both have override behavior.1243 if (SrcBehaviorValue == Module::Override &&1244 SrcOp->getOperand(2) != DstOp->getOperand(2))1245 return stringErr("linking module flags '" + ID->getString() +1246 "': IDs have conflicting override values in '" +1247 SrcM->getModuleIdentifier() + "' and '" +1248 DstM.getModuleIdentifier() + "'");1249 continue;1250 } else if (SrcBehaviorValue == Module::Override) {1251 // Update the destination flag to that of the source.1252 overrideDstValue();1253 continue;1254 }1255 1256 // Diagnose inconsistent merge behavior types.1257 if (SrcBehaviorValue != DstBehaviorValue) {1258 bool MinAndWarn = (SrcBehaviorValue == Module::Min &&1259 DstBehaviorValue == Module::Warning) ||1260 (DstBehaviorValue == Module::Min &&1261 SrcBehaviorValue == Module::Warning);1262 bool MaxAndWarn = (SrcBehaviorValue == Module::Max &&1263 DstBehaviorValue == Module::Warning) ||1264 (DstBehaviorValue == Module::Max &&1265 SrcBehaviorValue == Module::Warning);1266 if (!(MaxAndWarn || MinAndWarn))1267 return stringErr("linking module flags '" + ID->getString() +1268 "': IDs have conflicting behaviors in '" +1269 SrcM->getModuleIdentifier() + "' and '" +1270 DstM.getModuleIdentifier() + "'");1271 }1272 1273 auto ensureDistinctOp = [&](MDNode *DstValue) {1274 assert(isa<MDTuple>(DstValue) &&1275 "Expected MDTuple when appending module flags");1276 if (DstValue->isDistinct())1277 return dyn_cast<MDTuple>(DstValue);1278 ArrayRef<MDOperand> DstOperands = DstValue->operands();1279 MDTuple *New = MDTuple::getDistinct(1280 DstM.getContext(), SmallVector<Metadata *, 4>(DstOperands));1281 Metadata *FlagOps[] = {DstOp->getOperand(0), ID, New};1282 MDNode *Flag = MDTuple::getDistinct(DstM.getContext(), FlagOps);1283 DstModFlags->setOperand(DstIndex, Flag);1284 Flags[ID].first = Flag;1285 return New;1286 };1287 1288 // Emit a warning if the values differ and either source or destination1289 // request Warning behavior.1290 if ((DstBehaviorValue == Module::Warning ||1291 SrcBehaviorValue == Module::Warning) &&1292 SrcOp->getOperand(2) != DstOp->getOperand(2)) {1293 std::string Str;1294 raw_string_ostream(Str)1295 << "linking module flags '" << ID->getString()1296 << "': IDs have conflicting values ('" << *SrcOp->getOperand(2)1297 << "' from " << SrcM->getModuleIdentifier() << " with '"1298 << *DstOp->getOperand(2) << "' from " << DstM.getModuleIdentifier()1299 << ')';1300 emitWarning(Str);1301 }1302 1303 // Choose the minimum if either source or destination request Min behavior.1304 if (DstBehaviorValue == Module::Min || SrcBehaviorValue == Module::Min) {1305 ConstantInt *DstValue =1306 mdconst::extract<ConstantInt>(DstOp->getOperand(2));1307 ConstantInt *SrcValue =1308 mdconst::extract<ConstantInt>(SrcOp->getOperand(2));1309 1310 // The resulting flag should have a Min behavior, and contain the minimum1311 // value from between the source and destination values.1312 Metadata *FlagOps[] = {1313 (DstBehaviorValue != Module::Min ? SrcOp : DstOp)->getOperand(0), ID,1314 (SrcValue->getZExtValue() < DstValue->getZExtValue() ? SrcOp : DstOp)1315 ->getOperand(2)};1316 MDNode *Flag = MDNode::get(DstM.getContext(), FlagOps);1317 DstModFlags->setOperand(DstIndex, Flag);1318 Flags[ID].first = Flag;1319 continue;1320 }1321 1322 // Choose the maximum if either source or destination request Max behavior.1323 if (DstBehaviorValue == Module::Max || SrcBehaviorValue == Module::Max) {1324 ConstantInt *DstValue =1325 mdconst::extract<ConstantInt>(DstOp->getOperand(2));1326 ConstantInt *SrcValue =1327 mdconst::extract<ConstantInt>(SrcOp->getOperand(2));1328 1329 // The resulting flag should have a Max behavior, and contain the maximum1330 // value from between the source and destination values.1331 Metadata *FlagOps[] = {1332 (DstBehaviorValue != Module::Max ? SrcOp : DstOp)->getOperand(0), ID,1333 (SrcValue->getZExtValue() > DstValue->getZExtValue() ? SrcOp : DstOp)1334 ->getOperand(2)};1335 MDNode *Flag = MDNode::get(DstM.getContext(), FlagOps);1336 DstModFlags->setOperand(DstIndex, Flag);1337 Flags[ID].first = Flag;1338 continue;1339 }1340 1341 // Perform the merge for standard behavior types.1342 switch (SrcBehaviorValue) {1343 case Module::Require:1344 case Module::Override:1345 llvm_unreachable("not possible");1346 case Module::Error: {1347 // Emit an error if the values differ.1348 if (SrcOp->getOperand(2) != DstOp->getOperand(2)) {1349 std::string Str;1350 raw_string_ostream(Str)1351 << "linking module flags '" << ID->getString()1352 << "': IDs have conflicting values: '" << *SrcOp->getOperand(2)1353 << "' from " << SrcM->getModuleIdentifier() << ", and '"1354 << *DstOp->getOperand(2) << "' from " + DstM.getModuleIdentifier();1355 return stringErr(Str);1356 }1357 continue;1358 }1359 case Module::Warning: {1360 break;1361 }1362 case Module::Max: {1363 break;1364 }1365 case Module::Append: {1366 MDTuple *DstValue = ensureDistinctOp(cast<MDNode>(DstOp->getOperand(2)));1367 MDNode *SrcValue = cast<MDNode>(SrcOp->getOperand(2));1368 for (const auto &O : SrcValue->operands())1369 DstValue->push_back(O);1370 break;1371 }1372 case Module::AppendUnique: {1373 SmallSetVector<Metadata *, 16> Elts;1374 MDTuple *DstValue = ensureDistinctOp(cast<MDNode>(DstOp->getOperand(2)));1375 MDNode *SrcValue = cast<MDNode>(SrcOp->getOperand(2));1376 Elts.insert(DstValue->op_begin(), DstValue->op_end());1377 Elts.insert(SrcValue->op_begin(), SrcValue->op_end());1378 for (auto I = DstValue->getNumOperands(); I < Elts.size(); I++)1379 DstValue->push_back(Elts[I]);1380 break;1381 }1382 }1383 1384 }1385 1386 // For the Min behavior, set the value to 0 if either module does not have the1387 // flag.1388 for (auto Idx : Mins) {1389 MDNode *Op = DstModFlags->getOperand(Idx);1390 MDString *ID = cast<MDString>(Op->getOperand(1));1391 if (!SeenMin.count(ID)) {1392 ConstantInt *V = mdconst::extract<ConstantInt>(Op->getOperand(2));1393 Metadata *FlagOps[] = {1394 Op->getOperand(0), ID,1395 ConstantAsMetadata::get(ConstantInt::get(V->getType(), 0))};1396 DstModFlags->setOperand(Idx, MDNode::get(DstM.getContext(), FlagOps));1397 }1398 }1399 1400 // Check all of the requirements.1401 for (MDNode *Requirement : Requirements) {1402 MDString *Flag = cast<MDString>(Requirement->getOperand(0));1403 Metadata *ReqValue = Requirement->getOperand(1);1404 1405 MDNode *Op = Flags[Flag].first;1406 if (!Op || Op->getOperand(2) != ReqValue)1407 return stringErr("linking module flags '" + Flag->getString() +1408 "': does not have the required value");1409 }1410 return Error::success();1411}1412 1413/// Return InlineAsm adjusted with target-specific directives if required.1414/// For ARM and Thumb, we have to add directives to select the appropriate ISA1415/// to support mixing module-level inline assembly from ARM and Thumb modules.1416static std::string adjustInlineAsm(const std::string &InlineAsm,1417 const Triple &Triple) {1418 if (Triple.getArch() == Triple::thumb || Triple.getArch() == Triple::thumbeb)1419 return ".text\n.balign 2\n.thumb\n" + InlineAsm;1420 if (Triple.getArch() == Triple::arm || Triple.getArch() == Triple::armeb)1421 return ".text\n.balign 4\n.arm\n" + InlineAsm;1422 return InlineAsm;1423}1424 1425void IRLinker::updateAttributes(GlobalValue &GV) {1426 /// Remove nocallback attribute while linking, because nocallback attribute1427 /// indicates that the function is only allowed to jump back into caller's1428 /// module only by a return or an exception. When modules are linked, this1429 /// property cannot be guaranteed anymore. For example, the nocallback1430 /// function may contain a call to another module. But if we merge its caller1431 /// and callee module here, and not the module containing the nocallback1432 /// function definition itself, the nocallback property will be violated1433 /// (since the nocallback function will call back into the newly merged module1434 /// containing both its caller and callee). This could happen if the module1435 /// containing the nocallback function definition is native code, so it does1436 /// not participate in the LTO link. Note if the nocallback function does1437 /// participate in the LTO link, and thus ends up in the merged module1438 /// containing its caller and callee, removing the attribute doesn't hurt as1439 /// it has no effect on definitions in the same module.1440 if (auto *F = dyn_cast<Function>(&GV)) {1441 if (!F->isIntrinsic())1442 F->removeFnAttr(llvm::Attribute::NoCallback);1443 1444 // Remove nocallback attribute when it is on a call-site.1445 for (BasicBlock &BB : *F)1446 for (Instruction &I : BB)1447 if (CallBase *CI = dyn_cast<CallBase>(&I))1448 CI->removeFnAttr(Attribute::NoCallback);1449 }1450}1451 1452Error IRLinker::run() {1453 // Ensure metadata materialized before value mapping.1454 if (SrcM->getMaterializer())1455 if (Error Err = SrcM->getMaterializer()->materializeMetadata())1456 return Err;1457 1458 // Inherit the target data from the source module if the destination1459 // module doesn't have one already.1460 if (DstM.getDataLayout().isDefault())1461 DstM.setDataLayout(SrcM->getDataLayout());1462 1463 // Copy the target triple from the source to dest if the dest's is empty.1464 if (DstM.getTargetTriple().empty() && !SrcM->getTargetTriple().empty())1465 DstM.setTargetTriple(SrcM->getTargetTriple());1466 1467 Triple SrcTriple(SrcM->getTargetTriple()), DstTriple(DstM.getTargetTriple());1468 1469 // During CUDA compilation we have to link with the bitcode supplied with1470 // CUDA. libdevice bitcode either has no data layout set (pre-CUDA-11), or has1471 // the layout that is different from the one used by LLVM/clang (it does not1472 // include i128). Issuing a warning is not very helpful as there's not much1473 // the user can do about it.1474 bool EnableDLWarning = true;1475 bool EnableTripleWarning = true;1476 if (SrcTriple.isNVPTX() && DstTriple.isNVPTX()) {1477 bool SrcHasLibDeviceDL =1478 (SrcM->getDataLayoutStr().empty() ||1479 SrcM->getDataLayoutStr() == "e-i64:64-v16:16-v32:32-n16:32:64");1480 // libdevice bitcode uses nvptx64-nvidia-gpulibs or just1481 // 'nvptx-unknown-unknown' triple (before CUDA-10.x) and is compatible with1482 // all NVPTX variants.1483 bool SrcHasLibDeviceTriple = (SrcTriple.getVendor() == Triple::NVIDIA &&1484 SrcTriple.getOSName() == "gpulibs") ||1485 (SrcTriple.getVendorName() == "unknown" &&1486 SrcTriple.getOSName() == "unknown");1487 EnableTripleWarning = !SrcHasLibDeviceTriple;1488 EnableDLWarning = !(SrcHasLibDeviceTriple && SrcHasLibDeviceDL);1489 }1490 1491 if (EnableDLWarning && (SrcM->getDataLayout() != DstM.getDataLayout())) {1492 emitWarning("Linking two modules of different data layouts: '" +1493 SrcM->getModuleIdentifier() + "' is '" +1494 SrcM->getDataLayoutStr() + "' whereas '" +1495 DstM.getModuleIdentifier() + "' is '" +1496 DstM.getDataLayoutStr() + "'\n");1497 }1498 1499 if (EnableTripleWarning && !SrcM->getTargetTriple().empty() &&1500 !SrcTriple.isCompatibleWith(DstTriple))1501 emitWarning("Linking two modules of different target triples: '" +1502 SrcM->getModuleIdentifier() + "' is '" +1503 SrcM->getTargetTriple().str() + "' whereas '" +1504 DstM.getModuleIdentifier() + "' is '" +1505 DstM.getTargetTriple().str() + "'\n");1506 1507 DstM.setTargetTriple(Triple(SrcTriple.merge(DstTriple)));1508 1509 // Loop over all of the linked values to compute type mappings.1510 computeTypeMapping();1511 1512 // Convert module level attributes to function level attributes because1513 // after merging modules the attributes might change and would have different1514 // effect on the functions as the original module would have.1515 copyModuleAttrToFunctions(*SrcM);1516 1517 std::reverse(Worklist.begin(), Worklist.end());1518 while (!Worklist.empty()) {1519 GlobalValue *GV = Worklist.back();1520 Worklist.pop_back();1521 1522 // Already mapped.1523 if (ValueMap.find(GV) != ValueMap.end() ||1524 IndirectSymbolValueMap.find(GV) != IndirectSymbolValueMap.end())1525 continue;1526 1527 assert(!GV->isDeclaration());1528 Mapper.mapValue(*GV);1529 if (FoundError)1530 return std::move(*FoundError);1531 flushRAUWWorklist();1532 }1533 1534 // Note that we are done linking global value bodies. This prevents1535 // metadata linking from creating new references.1536 DoneLinkingBodies = true;1537 Mapper.addFlags(RF_NullMapMissingGlobalValues);1538 1539 // Remap all of the named MDNodes in Src into the DstM module. We do this1540 // after linking GlobalValues so that MDNodes that reference GlobalValues1541 // are properly remapped.1542 linkNamedMDNodes();1543 1544 // Clean up any global objects with potentially unmapped metadata.1545 // Specifically declarations which did not become definitions.1546 for (GlobalObject *NGO : UnmappedMetadata) {1547 if (NGO->isDeclaration())1548 Mapper.remapGlobalObjectMetadata(*NGO);1549 }1550 1551 if (!IsPerformingImport && !SrcM->getModuleInlineAsm().empty()) {1552 // Append the module inline asm string.1553 DstM.appendModuleInlineAsm(adjustInlineAsm(SrcM->getModuleInlineAsm(),1554 SrcTriple));1555 } else if (IsPerformingImport) {1556 // Import any symver directives for symbols in DstM.1557 ModuleSymbolTable::CollectAsmSymvers(*SrcM,1558 [&](StringRef Name, StringRef Alias) {1559 if (DstM.getNamedValue(Name)) {1560 SmallString<256> S(".symver ");1561 S += Name;1562 S += ", ";1563 S += Alias;1564 DstM.appendModuleInlineAsm(S);1565 }1566 });1567 }1568 1569 // Reorder the globals just added to the destination module to match their1570 // original order in the source module.1571 for (GlobalVariable &GV : SrcM->globals()) {1572 if (GV.hasAppendingLinkage())1573 continue;1574 Value *NewValue = Mapper.mapValue(GV);1575 if (FoundError)1576 return std::move(*FoundError);1577 if (NewValue) {1578 auto *NewGV = dyn_cast<GlobalVariable>(NewValue->stripPointerCasts());1579 if (NewGV) {1580 NewGV->removeFromParent();1581 DstM.insertGlobalVariable(NewGV);1582 }1583 }1584 }1585 1586 // Merge the module flags into the DstM module.1587 return linkModuleFlagsMetadata();1588}1589 1590IRMover::StructTypeKeyInfo::KeyTy::KeyTy(ArrayRef<Type *> E, bool P)1591 : ETypes(E), IsPacked(P) {}1592 1593IRMover::StructTypeKeyInfo::KeyTy::KeyTy(const StructType *ST)1594 : ETypes(ST->elements()), IsPacked(ST->isPacked()) {}1595 1596bool IRMover::StructTypeKeyInfo::KeyTy::operator==(const KeyTy &That) const {1597 return IsPacked == That.IsPacked && ETypes == That.ETypes;1598}1599 1600bool IRMover::StructTypeKeyInfo::KeyTy::operator!=(const KeyTy &That) const {1601 return !this->operator==(That);1602}1603 1604StructType *IRMover::StructTypeKeyInfo::getEmptyKey() {1605 return DenseMapInfo<StructType *>::getEmptyKey();1606}1607 1608StructType *IRMover::StructTypeKeyInfo::getTombstoneKey() {1609 return DenseMapInfo<StructType *>::getTombstoneKey();1610}1611 1612unsigned IRMover::StructTypeKeyInfo::getHashValue(const KeyTy &Key) {1613 return hash_combine(hash_combine_range(Key.ETypes), Key.IsPacked);1614}1615 1616unsigned IRMover::StructTypeKeyInfo::getHashValue(const StructType *ST) {1617 return getHashValue(KeyTy(ST));1618}1619 1620bool IRMover::StructTypeKeyInfo::isEqual(const KeyTy &LHS,1621 const StructType *RHS) {1622 if (RHS == getEmptyKey() || RHS == getTombstoneKey())1623 return false;1624 return LHS == KeyTy(RHS);1625}1626 1627bool IRMover::StructTypeKeyInfo::isEqual(const StructType *LHS,1628 const StructType *RHS) {1629 if (RHS == getEmptyKey() || RHS == getTombstoneKey())1630 return LHS == RHS;1631 return KeyTy(LHS) == KeyTy(RHS);1632}1633 1634void IRMover::IdentifiedStructTypeSet::addNonOpaque(StructType *Ty) {1635 assert(!Ty->isOpaque());1636 NonOpaqueStructTypes.insert(Ty);1637}1638 1639void IRMover::IdentifiedStructTypeSet::switchToNonOpaque(StructType *Ty) {1640 assert(!Ty->isOpaque());1641 NonOpaqueStructTypes.insert(Ty);1642 bool Removed = OpaqueStructTypes.erase(Ty);1643 (void)Removed;1644 assert(Removed);1645}1646 1647void IRMover::IdentifiedStructTypeSet::addOpaque(StructType *Ty) {1648 assert(Ty->isOpaque());1649 OpaqueStructTypes.insert(Ty);1650}1651 1652StructType *1653IRMover::IdentifiedStructTypeSet::findNonOpaque(ArrayRef<Type *> ETypes,1654 bool IsPacked) {1655 IRMover::StructTypeKeyInfo::KeyTy Key(ETypes, IsPacked);1656 auto I = NonOpaqueStructTypes.find_as(Key);1657 return I == NonOpaqueStructTypes.end() ? nullptr : *I;1658}1659 1660bool IRMover::IdentifiedStructTypeSet::hasType(StructType *Ty) {1661 if (Ty->isOpaque())1662 return OpaqueStructTypes.count(Ty);1663 auto I = NonOpaqueStructTypes.find(Ty);1664 return I == NonOpaqueStructTypes.end() ? false : *I == Ty;1665}1666 1667IRMover::IRMover(Module &M) : Composite(M) {1668 TypeFinder StructTypes;1669 StructTypes.run(M, /* OnlyNamed */ false);1670 for (StructType *Ty : StructTypes) {1671 if (Ty->isOpaque())1672 IdentifiedStructTypes.addOpaque(Ty);1673 else1674 IdentifiedStructTypes.addNonOpaque(Ty);1675 }1676 // Self-map metadatas in the destination module. This is needed when1677 // DebugTypeODRUniquing is enabled on the LLVMContext, since metadata in the1678 // destination module may be reached from the source module.1679 for (const auto *MD : StructTypes.getVisitedMetadata()) {1680 SharedMDs[MD].reset(const_cast<MDNode *>(MD));1681 }1682 1683 // Convert module level attributes to function level attributes because1684 // after merging modules the attributes might change and would have different1685 // effect on the functions as the original module would have.1686 copyModuleAttrToFunctions(M);1687}1688 1689Error IRMover::move(std::unique_ptr<Module> Src,1690 ArrayRef<GlobalValue *> ValuesToLink,1691 LazyCallback AddLazyFor, bool IsPerformingImport) {1692 IRLinker TheIRLinker(Composite, SharedMDs, IdentifiedStructTypes,1693 std::move(Src), ValuesToLink, std::move(AddLazyFor),1694 IsPerformingImport, NamedMDNodes);1695 return TheIRLinker.run();1696}1697