1255 lines · cpp
1//===- CloneFunction.cpp - Clone a function into another function ---------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the CloneFunctionInto interface, which is used as the10// low-level function cloner. This is used by the CloneFunction and function11// inliner to do the dirty work of copying the body of a function around.12//13//===----------------------------------------------------------------------===//14 15#include "llvm/ADT/SmallVector.h"16#include "llvm/ADT/Statistic.h"17#include "llvm/Analysis/ConstantFolding.h"18#include "llvm/Analysis/DomTreeUpdater.h"19#include "llvm/Analysis/InstructionSimplify.h"20#include "llvm/Analysis/LoopInfo.h"21#include "llvm/IR/AttributeMask.h"22#include "llvm/IR/CFG.h"23#include "llvm/IR/Constants.h"24#include "llvm/IR/DebugInfo.h"25#include "llvm/IR/DerivedTypes.h"26#include "llvm/IR/Function.h"27#include "llvm/IR/InstIterator.h"28#include "llvm/IR/Instructions.h"29#include "llvm/IR/IntrinsicInst.h"30#include "llvm/IR/LLVMContext.h"31#include "llvm/IR/MDBuilder.h"32#include "llvm/IR/Metadata.h"33#include "llvm/IR/Module.h"34#include "llvm/Transforms/Utils/BasicBlockUtils.h"35#include "llvm/Transforms/Utils/Cloning.h"36#include "llvm/Transforms/Utils/Local.h"37#include "llvm/Transforms/Utils/ValueMapper.h"38#include <map>39#include <optional>40using namespace llvm;41 42#define DEBUG_TYPE "clone-function"43 44STATISTIC(RemappedAtomMax, "Highest global NextAtomGroup (after mapping)");45 46void llvm::mapAtomInstance(const DebugLoc &DL, ValueToValueMapTy &VMap) {47 uint64_t CurGroup = DL->getAtomGroup();48 if (!CurGroup)49 return;50 51 // Try inserting a new entry. If there's already a mapping for this atom52 // then there's nothing to do.53 auto [It, Inserted] = VMap.AtomMap.insert({{DL.getInlinedAt(), CurGroup}, 0});54 if (!Inserted)55 return;56 57 // Map entry to a new atom group.58 uint64_t NewGroup = DL->getContext().incNextDILocationAtomGroup();59 assert(NewGroup > CurGroup && "Next should always be greater than current");60 It->second = NewGroup;61 62 RemappedAtomMax = std::max<uint64_t>(NewGroup, RemappedAtomMax);63}64 65static void collectDebugInfoFromInstructions(const Function &F,66 DebugInfoFinder &DIFinder) {67 const Module *M = F.getParent();68 if (!M)69 return;70 // Inspect instructions to process e.g. DILexicalBlocks of inlined functions71 for (const Instruction &I : instructions(F))72 DIFinder.processInstruction(*M, I);73}74 75// Create a predicate that matches the metadata that should be identity mapped76// during function cloning.77static MetadataPredicate78createIdentityMDPredicate(const Function &F, CloneFunctionChangeType Changes) {79 if (Changes >= CloneFunctionChangeType::DifferentModule)80 return [](const Metadata *MD) { return false; };81 82 DISubprogram *SPClonedWithinModule = F.getSubprogram();83 84 // Don't clone inlined subprograms.85 auto ShouldKeep = [SPClonedWithinModule](const DISubprogram *SP) -> bool {86 return SP != SPClonedWithinModule;87 };88 89 return [=](const Metadata *MD) {90 // Avoid cloning types, compile units, and (other) subprograms.91 if (isa<DICompileUnit>(MD) || isa<DIType>(MD))92 return true;93 94 if (auto *SP = dyn_cast<DISubprogram>(MD))95 return ShouldKeep(SP);96 97 // If a subprogram isn't going to be cloned skip its lexical blocks as well.98 if (auto *LScope = dyn_cast<DILocalScope>(MD))99 return ShouldKeep(LScope->getSubprogram());100 101 // Avoid cloning local variables of subprograms that won't be cloned.102 if (auto *DV = dyn_cast<DILocalVariable>(MD))103 if (auto *S = dyn_cast_or_null<DILocalScope>(DV->getScope()))104 return ShouldKeep(S->getSubprogram());105 106 return false;107 };108}109 110/// See comments in Cloning.h.111BasicBlock *llvm::CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap,112 const Twine &NameSuffix, Function *F,113 ClonedCodeInfo *CodeInfo, bool MapAtoms) {114 BasicBlock *NewBB = BasicBlock::Create(BB->getContext(), "", F);115 if (BB->hasName())116 NewBB->setName(BB->getName() + NameSuffix);117 118 bool hasCalls = false, hasDynamicAllocas = false, hasMemProfMetadata = false;119 120 // Loop over all instructions, and copy them over.121 for (const Instruction &I : *BB) {122 Instruction *NewInst = I.clone();123 if (I.hasName())124 NewInst->setName(I.getName() + NameSuffix);125 126 NewInst->insertBefore(*NewBB, NewBB->end());127 NewInst->cloneDebugInfoFrom(&I);128 129 VMap[&I] = NewInst; // Add instruction map to value.130 131 if (MapAtoms) {132 if (const DebugLoc &DL = NewInst->getDebugLoc())133 mapAtomInstance(DL.get(), VMap);134 }135 136 if (isa<CallInst>(I) && !I.isDebugOrPseudoInst()) {137 hasCalls = true;138 hasMemProfMetadata |= I.hasMetadata(LLVMContext::MD_memprof);139 hasMemProfMetadata |= I.hasMetadata(LLVMContext::MD_callsite);140 }141 if (const AllocaInst *AI = dyn_cast<AllocaInst>(&I)) {142 if (!AI->isStaticAlloca()) {143 hasDynamicAllocas = true;144 }145 }146 }147 148 if (CodeInfo) {149 CodeInfo->ContainsCalls |= hasCalls;150 CodeInfo->ContainsMemProfMetadata |= hasMemProfMetadata;151 CodeInfo->ContainsDynamicAllocas |= hasDynamicAllocas;152 }153 return NewBB;154}155 156void llvm::CloneFunctionAttributesInto(Function *NewFunc,157 const Function *OldFunc,158 ValueToValueMapTy &VMap,159 bool ModuleLevelChanges,160 ValueMapTypeRemapper *TypeMapper,161 ValueMaterializer *Materializer) {162 // Copy all attributes other than those stored in Function's AttributeList163 // which holds e.g. parameters and return value attributes.164 AttributeList NewAttrs = NewFunc->getAttributes();165 NewFunc->copyAttributesFrom(OldFunc);166 NewFunc->setAttributes(NewAttrs);167 168 const RemapFlags FuncGlobalRefFlags =169 ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges;170 171 // Fix up the personality function that got copied over.172 if (OldFunc->hasPersonalityFn())173 NewFunc->setPersonalityFn(MapValue(OldFunc->getPersonalityFn(), VMap,174 FuncGlobalRefFlags, TypeMapper,175 Materializer));176 177 if (OldFunc->hasPrefixData()) {178 NewFunc->setPrefixData(MapValue(OldFunc->getPrefixData(), VMap,179 FuncGlobalRefFlags, TypeMapper,180 Materializer));181 }182 183 if (OldFunc->hasPrologueData()) {184 NewFunc->setPrologueData(MapValue(OldFunc->getPrologueData(), VMap,185 FuncGlobalRefFlags, TypeMapper,186 Materializer));187 }188 189 SmallVector<AttributeSet, 4> NewArgAttrs(NewFunc->arg_size());190 AttributeList OldAttrs = OldFunc->getAttributes();191 192 // Clone any argument attributes that are present in the VMap.193 for (const Argument &OldArg : OldFunc->args()) {194 if (Argument *NewArg = dyn_cast<Argument>(VMap[&OldArg])) {195 // Remap the parameter indices.196 NewArgAttrs[NewArg->getArgNo()] =197 OldAttrs.getParamAttrs(OldArg.getArgNo());198 }199 }200 201 NewFunc->setAttributes(202 AttributeList::get(NewFunc->getContext(), OldAttrs.getFnAttrs(),203 OldAttrs.getRetAttrs(), NewArgAttrs));204}205 206void llvm::CloneFunctionMetadataInto(Function &NewFunc, const Function &OldFunc,207 ValueToValueMapTy &VMap,208 RemapFlags RemapFlag,209 ValueMapTypeRemapper *TypeMapper,210 ValueMaterializer *Materializer,211 const MetadataPredicate *IdentityMD) {212 SmallVector<std::pair<unsigned, MDNode *>, 1> MDs;213 OldFunc.getAllMetadata(MDs);214 for (const auto &[Kind, MD] : MDs) {215 NewFunc.addMetadata(Kind, *MapMetadata(MD, VMap, RemapFlag, TypeMapper,216 Materializer, IdentityMD));217 }218}219 220void llvm::CloneFunctionBodyInto(Function &NewFunc, const Function &OldFunc,221 ValueToValueMapTy &VMap, RemapFlags RemapFlag,222 SmallVectorImpl<ReturnInst *> &Returns,223 const char *NameSuffix,224 ClonedCodeInfo *CodeInfo,225 ValueMapTypeRemapper *TypeMapper,226 ValueMaterializer *Materializer,227 const MetadataPredicate *IdentityMD) {228 if (OldFunc.isDeclaration())229 return;230 231 // Loop over all of the basic blocks in the function, cloning them as232 // appropriate. Note that we save BE this way in order to handle cloning of233 // recursive functions into themselves.234 for (const BasicBlock &BB : OldFunc) {235 // Create a new basic block and copy instructions into it!236 BasicBlock *CBB =237 CloneBasicBlock(&BB, VMap, NameSuffix, &NewFunc, CodeInfo);238 239 // Add basic block mapping.240 VMap[&BB] = CBB;241 242 // It is only legal to clone a function if a block address within that243 // function is never referenced outside of the function. Given that, we244 // want to map block addresses from the old function to block addresses in245 // the clone. (This is different from the generic ValueMapper246 // implementation, which generates an invalid blockaddress when247 // cloning a function.)248 if (BB.hasAddressTaken()) {249 Constant *OldBBAddr = BlockAddress::get(const_cast<Function *>(&OldFunc),250 const_cast<BasicBlock *>(&BB));251 VMap[OldBBAddr] = BlockAddress::get(&NewFunc, CBB);252 }253 254 // Note return instructions for the caller.255 if (ReturnInst *RI = dyn_cast<ReturnInst>(CBB->getTerminator()))256 Returns.push_back(RI);257 }258 259 // Loop over all of the instructions in the new function, fixing up operand260 // references as we go. This uses VMap to do all the hard work.261 for (Function::iterator262 BB = cast<BasicBlock>(VMap[&OldFunc.front()])->getIterator(),263 BE = NewFunc.end();264 BB != BE; ++BB)265 // Loop over all instructions, fixing each one as we find it, and any266 // attached debug-info records.267 for (Instruction &II : *BB) {268 RemapInstruction(&II, VMap, RemapFlag, TypeMapper, Materializer,269 IdentityMD);270 RemapDbgRecordRange(II.getModule(), II.getDbgRecordRange(), VMap,271 RemapFlag, TypeMapper, Materializer, IdentityMD);272 }273}274 275// Clone OldFunc into NewFunc, transforming the old arguments into references to276// VMap values.277void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,278 ValueToValueMapTy &VMap,279 CloneFunctionChangeType Changes,280 SmallVectorImpl<ReturnInst *> &Returns,281 const char *NameSuffix, ClonedCodeInfo *CodeInfo,282 ValueMapTypeRemapper *TypeMapper,283 ValueMaterializer *Materializer) {284 assert(NameSuffix && "NameSuffix cannot be null!");285 286#ifndef NDEBUG287 for (const Argument &I : OldFunc->args())288 assert(VMap.count(&I) && "No mapping from source argument specified!");289#endif290 291 bool ModuleLevelChanges = Changes > CloneFunctionChangeType::LocalChangesOnly;292 293 CloneFunctionAttributesInto(NewFunc, OldFunc, VMap, ModuleLevelChanges,294 TypeMapper, Materializer);295 296 // Everything else beyond this point deals with function instructions,297 // so if we are dealing with a function declaration, we're done.298 if (OldFunc->isDeclaration())299 return;300 301 if (Changes < CloneFunctionChangeType::DifferentModule) {302 assert((NewFunc->getParent() == nullptr ||303 NewFunc->getParent() == OldFunc->getParent()) &&304 "Expected NewFunc to have the same parent, or no parent");305 } else {306 assert((NewFunc->getParent() == nullptr ||307 NewFunc->getParent() != OldFunc->getParent()) &&308 "Expected NewFunc to have different parents, or no parent");309 310 if (Changes == CloneFunctionChangeType::DifferentModule) {311 assert(NewFunc->getParent() &&312 "Need parent of new function to maintain debug info invariants");313 }314 }315 316 MetadataPredicate IdentityMD = createIdentityMDPredicate(*OldFunc, Changes);317 318 // Cloning is always a Module level operation, since Metadata needs to be319 // cloned.320 const RemapFlags RemapFlag = RF_None;321 322 CloneFunctionMetadataInto(*NewFunc, *OldFunc, VMap, RemapFlag, TypeMapper,323 Materializer, &IdentityMD);324 325 CloneFunctionBodyInto(*NewFunc, *OldFunc, VMap, RemapFlag, Returns,326 NameSuffix, CodeInfo, TypeMapper, Materializer,327 &IdentityMD);328 329 // Only update !llvm.dbg.cu for DifferentModule (not CloneModule). In the330 // same module, the compile unit will already be listed (or not). When331 // cloning a module, CloneModule() will handle creating the named metadata.332 if (Changes != CloneFunctionChangeType::DifferentModule)333 return;334 335 // Update !llvm.dbg.cu with compile units added to the new module if this336 // function is being cloned in isolation.337 //338 // FIXME: This is making global / module-level changes, which doesn't seem339 // like the right encapsulation Consider dropping the requirement to update340 // !llvm.dbg.cu (either obsoleting the node, or restricting it to341 // non-discardable compile units) instead of discovering compile units by342 // visiting the metadata attached to global values, which would allow this343 // code to be deleted. Alternatively, perhaps give responsibility for this344 // update to CloneFunctionInto's callers.345 Module *NewModule = NewFunc->getParent();346 NamedMDNode *NMD = NewModule->getOrInsertNamedMetadata("llvm.dbg.cu");347 // Avoid multiple insertions of the same DICompileUnit to NMD.348 SmallPtrSet<const void *, 8> Visited(llvm::from_range, NMD->operands());349 350 // Collect and clone all the compile units referenced from the instructions in351 // the function (e.g. as instructions' scope).352 DebugInfoFinder DIFinder;353 collectDebugInfoFromInstructions(*OldFunc, DIFinder);354 for (DICompileUnit *Unit : DIFinder.compile_units()) {355 MDNode *MappedUnit =356 MapMetadata(Unit, VMap, RF_None, TypeMapper, Materializer);357 if (Visited.insert(MappedUnit).second)358 NMD->addOperand(MappedUnit);359 }360}361 362/// Return a copy of the specified function and add it to that function's363/// module. Also, any references specified in the VMap are changed to refer to364/// their mapped value instead of the original one. If any of the arguments to365/// the function are in the VMap, the arguments are deleted from the resultant366/// function. The VMap is updated to include mappings from all of the367/// instructions and basicblocks in the function from their old to new values.368///369Function *llvm::CloneFunction(Function *F, ValueToValueMapTy &VMap,370 ClonedCodeInfo *CodeInfo) {371 std::vector<Type *> ArgTypes;372 373 // The user might be deleting arguments to the function by specifying them in374 // the VMap. If so, we need to not add the arguments to the arg ty vector375 //376 for (const Argument &I : F->args())377 if (VMap.count(&I) == 0) // Haven't mapped the argument to anything yet?378 ArgTypes.push_back(I.getType());379 380 // Create a new function type...381 FunctionType *FTy =382 FunctionType::get(F->getFunctionType()->getReturnType(), ArgTypes,383 F->getFunctionType()->isVarArg());384 385 // Create the new function...386 Function *NewF = Function::Create(FTy, F->getLinkage(), F->getAddressSpace(),387 F->getName(), F->getParent());388 389 // Loop over the arguments, copying the names of the mapped arguments over...390 Function::arg_iterator DestI = NewF->arg_begin();391 for (const Argument &I : F->args())392 if (VMap.count(&I) == 0) { // Is this argument preserved?393 DestI->setName(I.getName()); // Copy the name over...394 VMap[&I] = &*DestI++; // Add mapping to VMap395 }396 397 SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned.398 CloneFunctionInto(NewF, F, VMap, CloneFunctionChangeType::LocalChangesOnly,399 Returns, "", CodeInfo);400 401 return NewF;402}403 404namespace {405/// This is a private class used to implement CloneAndPruneFunctionInto.406struct PruningFunctionCloner {407 Function *NewFunc;408 const Function *OldFunc;409 ValueToValueMapTy &VMap;410 bool ModuleLevelChanges;411 const char *NameSuffix;412 ClonedCodeInfo *CodeInfo;413 bool HostFuncIsStrictFP;414 415 Instruction *cloneInstruction(BasicBlock::const_iterator II);416 417public:418 PruningFunctionCloner(Function *newFunc, const Function *oldFunc,419 ValueToValueMapTy &valueMap, bool moduleLevelChanges,420 const char *nameSuffix, ClonedCodeInfo *codeInfo)421 : NewFunc(newFunc), OldFunc(oldFunc), VMap(valueMap),422 ModuleLevelChanges(moduleLevelChanges), NameSuffix(nameSuffix),423 CodeInfo(codeInfo) {424 HostFuncIsStrictFP =425 newFunc->getAttributes().hasFnAttr(Attribute::StrictFP);426 }427 428 /// The specified block is found to be reachable, clone it and429 /// anything that it can reach.430 void CloneBlock(const BasicBlock *BB, BasicBlock::const_iterator StartingInst,431 std::vector<const BasicBlock *> &ToClone);432};433} // namespace434 435Instruction *436PruningFunctionCloner::cloneInstruction(BasicBlock::const_iterator II) {437 const Instruction &OldInst = *II;438 Instruction *NewInst = nullptr;439 if (HostFuncIsStrictFP) {440 Intrinsic::ID CIID = getConstrainedIntrinsicID(OldInst);441 if (CIID != Intrinsic::not_intrinsic) {442 // Instead of cloning the instruction, a call to constrained intrinsic443 // should be created.444 // Assume the first arguments of constrained intrinsics are the same as445 // the operands of original instruction.446 447 // Determine overloaded types of the intrinsic.448 SmallVector<Type *, 2> TParams;449 SmallVector<Intrinsic::IITDescriptor, 8> Descriptor;450 getIntrinsicInfoTableEntries(CIID, Descriptor);451 for (unsigned I = 0, E = Descriptor.size(); I != E; ++I) {452 Intrinsic::IITDescriptor Operand = Descriptor[I];453 switch (Operand.Kind) {454 case Intrinsic::IITDescriptor::Argument:455 if (Operand.getArgumentKind() !=456 Intrinsic::IITDescriptor::AK_MatchType) {457 if (I == 0)458 TParams.push_back(OldInst.getType());459 else460 TParams.push_back(OldInst.getOperand(I - 1)->getType());461 }462 break;463 case Intrinsic::IITDescriptor::SameVecWidthArgument:464 ++I;465 break;466 default:467 break;468 }469 }470 471 // Create intrinsic call.472 LLVMContext &Ctx = NewFunc->getContext();473 Function *IFn = Intrinsic::getOrInsertDeclaration(NewFunc->getParent(),474 CIID, TParams);475 SmallVector<Value *, 4> Args;476 unsigned NumOperands = OldInst.getNumOperands();477 if (isa<CallInst>(OldInst))478 --NumOperands;479 for (unsigned I = 0; I < NumOperands; ++I) {480 Value *Op = OldInst.getOperand(I);481 Args.push_back(Op);482 }483 if (const auto *CmpI = dyn_cast<FCmpInst>(&OldInst)) {484 FCmpInst::Predicate Pred = CmpI->getPredicate();485 StringRef PredName = FCmpInst::getPredicateName(Pred);486 Args.push_back(MetadataAsValue::get(Ctx, MDString::get(Ctx, PredName)));487 }488 489 // The last arguments of a constrained intrinsic are metadata that490 // represent rounding mode (absents in some intrinsics) and exception491 // behavior. The inlined function uses default settings.492 if (Intrinsic::hasConstrainedFPRoundingModeOperand(CIID))493 Args.push_back(494 MetadataAsValue::get(Ctx, MDString::get(Ctx, "round.tonearest")));495 Args.push_back(496 MetadataAsValue::get(Ctx, MDString::get(Ctx, "fpexcept.ignore")));497 498 NewInst = CallInst::Create(IFn, Args, OldInst.getName() + ".strict");499 }500 }501 if (!NewInst)502 NewInst = II->clone();503 return NewInst;504}505 506/// The specified block is found to be reachable, clone it and507/// anything that it can reach.508void PruningFunctionCloner::CloneBlock(509 const BasicBlock *BB, BasicBlock::const_iterator StartingInst,510 std::vector<const BasicBlock *> &ToClone) {511 WeakTrackingVH &BBEntry = VMap[BB];512 513 // Have we already cloned this block?514 if (BBEntry)515 return;516 517 // Nope, clone it now.518 BasicBlock *NewBB;519 Twine NewName(BB->hasName() ? Twine(BB->getName()) + NameSuffix : "");520 BBEntry = NewBB = BasicBlock::Create(BB->getContext(), NewName, NewFunc);521 522 // It is only legal to clone a function if a block address within that523 // function is never referenced outside of the function. Given that, we524 // want to map block addresses from the old function to block addresses in525 // the clone. (This is different from the generic ValueMapper526 // implementation, which generates an invalid blockaddress when527 // cloning a function.)528 //529 // Note that we don't need to fix the mapping for unreachable blocks;530 // the default mapping there is safe.531 if (BB->hasAddressTaken()) {532 Constant *OldBBAddr = BlockAddress::get(const_cast<Function *>(OldFunc),533 const_cast<BasicBlock *>(BB));534 VMap[OldBBAddr] = BlockAddress::get(NewFunc, NewBB);535 }536 537 bool hasCalls = false, hasDynamicAllocas = false, hasStaticAllocas = false;538 bool hasMemProfMetadata = false;539 540 // Keep a cursor pointing at the last place we cloned debug-info records from.541 BasicBlock::const_iterator DbgCursor = StartingInst;542 auto CloneDbgRecordsToHere =543 [&DbgCursor](Instruction *NewInst, BasicBlock::const_iterator II) {544 // Clone debug-info records onto this instruction. Iterate through any545 // source-instructions we've cloned and then subsequently optimised546 // away, so that their debug-info doesn't go missing.547 for (; DbgCursor != II; ++DbgCursor)548 NewInst->cloneDebugInfoFrom(&*DbgCursor, std::nullopt, false);549 NewInst->cloneDebugInfoFrom(&*II);550 DbgCursor = std::next(II);551 };552 553 // Loop over all instructions, and copy them over, DCE'ing as we go. This554 // loop doesn't include the terminator.555 for (BasicBlock::const_iterator II = StartingInst, IE = --BB->end(); II != IE;556 ++II) {557 558 // Don't clone fake_use as it may suppress many optimizations559 // due to inlining, especially SROA.560 if (auto *IntrInst = dyn_cast<IntrinsicInst>(II))561 if (IntrInst->getIntrinsicID() == Intrinsic::fake_use)562 continue;563 564 Instruction *NewInst = cloneInstruction(II);565 NewInst->insertInto(NewBB, NewBB->end());566 567 if (HostFuncIsStrictFP) {568 // All function calls in the inlined function must get 'strictfp'569 // attribute to prevent undesirable optimizations.570 if (auto *Call = dyn_cast<CallInst>(NewInst))571 Call->addFnAttr(Attribute::StrictFP);572 }573 574 // Eagerly remap operands to the newly cloned instruction, except for PHI575 // nodes for which we defer processing until we update the CFG.576 if (!isa<PHINode>(NewInst)) {577 RemapInstruction(NewInst, VMap,578 ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges);579 580 // Eagerly constant fold the newly cloned instruction. If successful, add581 // a mapping to the new value. Non-constant operands may be incomplete at582 // this stage, thus instruction simplification is performed after583 // processing phi-nodes.584 if (Value *V = ConstantFoldInstruction(585 NewInst, BB->getDataLayout())) {586 if (isInstructionTriviallyDead(NewInst)) {587 VMap[&*II] = V;588 NewInst->eraseFromParent();589 continue;590 }591 }592 }593 594 if (II->hasName())595 NewInst->setName(II->getName() + NameSuffix);596 VMap[&*II] = NewInst; // Add instruction map to value.597 if (isa<CallInst>(II) && !II->isDebugOrPseudoInst()) {598 hasCalls = true;599 hasMemProfMetadata |= II->hasMetadata(LLVMContext::MD_memprof);600 hasMemProfMetadata |= II->hasMetadata(LLVMContext::MD_callsite);601 }602 603 CloneDbgRecordsToHere(NewInst, II);604 605 if (CodeInfo) {606 CodeInfo->OrigVMap[&*II] = NewInst;607 if (auto *CB = dyn_cast<CallBase>(&*II))608 if (CB->hasOperandBundles())609 CodeInfo->OperandBundleCallSites.push_back(NewInst);610 }611 612 if (const AllocaInst *AI = dyn_cast<AllocaInst>(II)) {613 if (isa<ConstantInt>(AI->getArraySize()))614 hasStaticAllocas = true;615 else616 hasDynamicAllocas = true;617 }618 }619 620 // Finally, clone over the terminator.621 const Instruction *OldTI = BB->getTerminator();622 bool TerminatorDone = false;623 if (const BranchInst *BI = dyn_cast<BranchInst>(OldTI)) {624 if (BI->isConditional()) {625 // If the condition was a known constant in the callee...626 ConstantInt *Cond = dyn_cast<ConstantInt>(BI->getCondition());627 // Or is a known constant in the caller...628 if (!Cond) {629 Value *V = VMap.lookup(BI->getCondition());630 Cond = dyn_cast_or_null<ConstantInt>(V);631 }632 633 // Constant fold to uncond branch!634 if (Cond) {635 BasicBlock *Dest = BI->getSuccessor(!Cond->getZExtValue());636 auto *NewBI = BranchInst::Create(Dest, NewBB);637 NewBI->setDebugLoc(BI->getDebugLoc());638 VMap[OldTI] = NewBI;639 ToClone.push_back(Dest);640 TerminatorDone = true;641 }642 }643 } else if (const SwitchInst *SI = dyn_cast<SwitchInst>(OldTI)) {644 // If switching on a value known constant in the caller.645 ConstantInt *Cond = dyn_cast<ConstantInt>(SI->getCondition());646 if (!Cond) { // Or known constant after constant prop in the callee...647 Value *V = VMap.lookup(SI->getCondition());648 Cond = dyn_cast_or_null<ConstantInt>(V);649 }650 if (Cond) { // Constant fold to uncond branch!651 SwitchInst::ConstCaseHandle Case = *SI->findCaseValue(Cond);652 BasicBlock *Dest = const_cast<BasicBlock *>(Case.getCaseSuccessor());653 auto *NewBI = BranchInst::Create(Dest, NewBB);654 NewBI->setDebugLoc(SI->getDebugLoc());655 VMap[OldTI] = NewBI;656 ToClone.push_back(Dest);657 TerminatorDone = true;658 }659 }660 661 if (!TerminatorDone) {662 Instruction *NewInst = OldTI->clone();663 if (OldTI->hasName())664 NewInst->setName(OldTI->getName() + NameSuffix);665 NewInst->insertInto(NewBB, NewBB->end());666 667 CloneDbgRecordsToHere(NewInst, OldTI->getIterator());668 669 VMap[OldTI] = NewInst; // Add instruction map to value.670 671 if (CodeInfo) {672 CodeInfo->OrigVMap[OldTI] = NewInst;673 if (auto *CB = dyn_cast<CallBase>(OldTI))674 if (CB->hasOperandBundles())675 CodeInfo->OperandBundleCallSites.push_back(NewInst);676 }677 678 // Recursively clone any reachable successor blocks.679 append_range(ToClone, successors(BB->getTerminator()));680 } else {681 // If we didn't create a new terminator, clone DbgVariableRecords from the682 // old terminator onto the new terminator.683 Instruction *NewInst = NewBB->getTerminator();684 assert(NewInst);685 686 CloneDbgRecordsToHere(NewInst, OldTI->getIterator());687 }688 689 if (CodeInfo) {690 CodeInfo->ContainsCalls |= hasCalls;691 CodeInfo->ContainsMemProfMetadata |= hasMemProfMetadata;692 CodeInfo->ContainsDynamicAllocas |= hasDynamicAllocas;693 CodeInfo->ContainsDynamicAllocas |=694 hasStaticAllocas && BB != &BB->getParent()->front();695 }696}697 698/// This works like CloneAndPruneFunctionInto, except that it does not clone the699/// entire function. Instead it starts at an instruction provided by the caller700/// and copies (and prunes) only the code reachable from that instruction.701void llvm::CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc,702 const Instruction *StartingInst,703 ValueToValueMapTy &VMap,704 bool ModuleLevelChanges,705 SmallVectorImpl<ReturnInst *> &Returns,706 const char *NameSuffix,707 ClonedCodeInfo *CodeInfo) {708 assert(NameSuffix && "NameSuffix cannot be null!");709 710 ValueMapTypeRemapper *TypeMapper = nullptr;711 ValueMaterializer *Materializer = nullptr;712 713#ifndef NDEBUG714 // If the cloning starts at the beginning of the function, verify that715 // the function arguments are mapped.716 if (!StartingInst)717 for (const Argument &II : OldFunc->args())718 assert(VMap.count(&II) && "No mapping from source argument specified!");719#endif720 721 PruningFunctionCloner PFC(NewFunc, OldFunc, VMap, ModuleLevelChanges,722 NameSuffix, CodeInfo);723 const BasicBlock *StartingBB;724 if (StartingInst)725 StartingBB = StartingInst->getParent();726 else {727 StartingBB = &OldFunc->getEntryBlock();728 StartingInst = &StartingBB->front();729 }730 731 // Clone the entry block, and anything recursively reachable from it.732 std::vector<const BasicBlock *> CloneWorklist;733 PFC.CloneBlock(StartingBB, StartingInst->getIterator(), CloneWorklist);734 while (!CloneWorklist.empty()) {735 const BasicBlock *BB = CloneWorklist.back();736 CloneWorklist.pop_back();737 PFC.CloneBlock(BB, BB->begin(), CloneWorklist);738 }739 740 // Loop over all of the basic blocks in the old function. If the block was741 // reachable, we have cloned it and the old block is now in the value map:742 // insert it into the new function in the right order. If not, ignore it.743 //744 // Defer PHI resolution until rest of function is resolved.745 SmallVector<const PHINode *, 16> PHIToResolve;746 for (const BasicBlock &BI : *OldFunc) {747 Value *V = VMap.lookup(&BI);748 BasicBlock *NewBB = cast_or_null<BasicBlock>(V);749 if (!NewBB)750 continue; // Dead block.751 752 // Move the new block to preserve the order in the original function.753 NewBB->moveBefore(NewFunc->end());754 755 // Handle PHI nodes specially, as we have to remove references to dead756 // blocks.757 for (const PHINode &PN : BI.phis()) {758 // PHI nodes may have been remapped to non-PHI nodes by the caller or759 // during the cloning process.760 if (isa<PHINode>(VMap[&PN]))761 PHIToResolve.push_back(&PN);762 else763 break;764 }765 766 // Finally, remap the terminator instructions, as those can't be remapped767 // until all BBs are mapped.768 RemapInstruction(NewBB->getTerminator(), VMap,769 ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges,770 TypeMapper, Materializer);771 }772 773 // Defer PHI resolution until rest of function is resolved, PHI resolution774 // requires the CFG to be up-to-date.775 for (unsigned phino = 0, e = PHIToResolve.size(); phino != e;) {776 const PHINode *OPN = PHIToResolve[phino];777 unsigned NumPreds = OPN->getNumIncomingValues();778 const BasicBlock *OldBB = OPN->getParent();779 BasicBlock *NewBB = cast<BasicBlock>(VMap[OldBB]);780 781 // Map operands for blocks that are live and remove operands for blocks782 // that are dead.783 for (; phino != PHIToResolve.size() &&784 PHIToResolve[phino]->getParent() == OldBB;785 ++phino) {786 OPN = PHIToResolve[phino];787 PHINode *PN = cast<PHINode>(VMap[OPN]);788 for (unsigned pred = 0, e = NumPreds; pred != e; ++pred) {789 Value *V = VMap.lookup(PN->getIncomingBlock(pred));790 if (BasicBlock *MappedBlock = cast_or_null<BasicBlock>(V)) {791 Value *InVal =792 MapValue(PN->getIncomingValue(pred), VMap,793 ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges);794 assert(InVal && "Unknown input value?");795 PN->setIncomingValue(pred, InVal);796 PN->setIncomingBlock(pred, MappedBlock);797 } else {798 PN->removeIncomingValue(pred, false);799 --pred; // Revisit the next entry.800 --e;801 }802 }803 }804 805 // The loop above has removed PHI entries for those blocks that are dead806 // and has updated others. However, if a block is live (i.e. copied over)807 // but its terminator has been changed to not go to this block, then our808 // phi nodes will have invalid entries. Update the PHI nodes in this809 // case.810 PHINode *PN = cast<PHINode>(NewBB->begin());811 NumPreds = pred_size(NewBB);812 if (NumPreds != PN->getNumIncomingValues()) {813 assert(NumPreds < PN->getNumIncomingValues());814 // Count how many times each predecessor comes to this block.815 std::map<BasicBlock *, unsigned> PredCount;816 for (BasicBlock *Pred : predecessors(NewBB))817 --PredCount[Pred];818 819 // Figure out how many entries to remove from each PHI.820 for (BasicBlock *Pred : PN->blocks())821 ++PredCount[Pred];822 823 // At this point, the excess predecessor entries are positive in the824 // map. Loop over all of the PHIs and remove excess predecessor825 // entries.826 BasicBlock::iterator I = NewBB->begin();827 for (; (PN = dyn_cast<PHINode>(I)); ++I) {828 for (const auto &[Pred, Count] : PredCount) {829 for ([[maybe_unused]] unsigned _ : llvm::seq<unsigned>(Count))830 PN->removeIncomingValue(Pred, false);831 }832 }833 }834 835 // If the loops above have made these phi nodes have 0 or 1 operand,836 // replace them with poison or the input value. We must do this for837 // correctness, because 0-operand phis are not valid.838 PN = cast<PHINode>(NewBB->begin());839 if (PN->getNumIncomingValues() == 0) {840 BasicBlock::iterator I = NewBB->begin();841 BasicBlock::const_iterator OldI = OldBB->begin();842 while ((PN = dyn_cast<PHINode>(I++))) {843 Value *NV = PoisonValue::get(PN->getType());844 PN->replaceAllUsesWith(NV);845 assert(VMap[&*OldI] == PN && "VMap mismatch");846 VMap[&*OldI] = NV;847 PN->eraseFromParent();848 ++OldI;849 }850 }851 }852 853 // Drop all incompatible return attributes that cannot be applied to NewFunc854 // during cloning, so as to allow instruction simplification to reason on the855 // old state of the function. The original attributes are restored later.856 AttributeList Attrs = NewFunc->getAttributes();857 AttributeMask IncompatibleAttrs = AttributeFuncs::typeIncompatible(858 OldFunc->getReturnType(), Attrs.getRetAttrs());859 NewFunc->removeRetAttrs(IncompatibleAttrs);860 861 // As phi-nodes have been now remapped, allow incremental simplification of862 // newly-cloned instructions.863 const DataLayout &DL = NewFunc->getDataLayout();864 for (const BasicBlock &BB : *OldFunc) {865 for (const Instruction &I : BB) {866 auto *NewI = dyn_cast_or_null<Instruction>(VMap.lookup(&I));867 if (!NewI)868 continue;869 870 if (Value *V = simplifyInstruction(NewI, DL)) {871 NewI->replaceAllUsesWith(V);872 873 if (isInstructionTriviallyDead(NewI)) {874 NewI->eraseFromParent();875 } else {876 // Did not erase it? Restore the new instruction into VMap previously877 // dropped by `ValueIsRAUWd`.878 VMap[&I] = NewI;879 }880 }881 }882 }883 884 // Restore attributes.885 NewFunc->setAttributes(Attrs);886 887 // Remap debug records operands now that all values have been mapped.888 // Doing this now (late) preserves use-before-defs in debug records. If889 // we didn't do this, ValueAsMetadata(use-before-def) operands would be890 // replaced by empty metadata. This would signal later cleanup passes to891 // remove the debug records, potentially causing incorrect locations.892 Function::iterator Begin = cast<BasicBlock>(VMap[StartingBB])->getIterator();893 for (BasicBlock &BB : make_range(Begin, NewFunc->end())) {894 for (Instruction &I : BB) {895 RemapDbgRecordRange(I.getModule(), I.getDbgRecordRange(), VMap,896 ModuleLevelChanges ? RF_None897 : RF_NoModuleLevelChanges,898 TypeMapper, Materializer);899 }900 }901 902 // Simplify conditional branches and switches with a constant operand. We try903 // to prune these out when cloning, but if the simplification required904 // looking through PHI nodes, those are only available after forming the full905 // basic block. That may leave some here, and we still want to prune the dead906 // code as early as possible.907 for (BasicBlock &BB : make_range(Begin, NewFunc->end()))908 ConstantFoldTerminator(&BB);909 910 // Some blocks may have become unreachable as a result. Find and delete them.911 {912 SmallPtrSet<BasicBlock *, 16> ReachableBlocks;913 SmallVector<BasicBlock *, 16> Worklist;914 Worklist.push_back(&*Begin);915 while (!Worklist.empty()) {916 BasicBlock *BB = Worklist.pop_back_val();917 if (ReachableBlocks.insert(BB).second)918 append_range(Worklist, successors(BB));919 }920 921 SmallVector<BasicBlock *, 16> UnreachableBlocks;922 for (BasicBlock &BB : make_range(Begin, NewFunc->end()))923 if (!ReachableBlocks.contains(&BB))924 UnreachableBlocks.push_back(&BB);925 DeleteDeadBlocks(UnreachableBlocks);926 }927 928 // Now that the inlined function body has been fully constructed, go through929 // and zap unconditional fall-through branches. This happens all the time when930 // specializing code: code specialization turns conditional branches into931 // uncond branches, and this code folds them.932 Function::iterator I = Begin;933 while (I != NewFunc->end()) {934 BranchInst *BI = dyn_cast<BranchInst>(I->getTerminator());935 if (!BI || BI->isConditional()) {936 ++I;937 continue;938 }939 940 BasicBlock *Dest = BI->getSuccessor(0);941 if (!Dest->getSinglePredecessor() || Dest->hasAddressTaken()) {942 ++I;943 continue;944 }945 946 // We shouldn't be able to get single-entry PHI nodes here, as instsimplify947 // above should have zapped all of them..948 assert(!isa<PHINode>(Dest->begin()));949 950 // We know all single-entry PHI nodes in the inlined function have been951 // removed, so we just need to splice the blocks.952 BI->eraseFromParent();953 954 // Make all PHI nodes that referred to Dest now refer to I as their source.955 Dest->replaceAllUsesWith(&*I);956 957 // Move all the instructions in the succ to the pred.958 I->splice(I->end(), Dest);959 960 // Remove the dest block.961 Dest->eraseFromParent();962 963 // Do not increment I, iteratively merge all things this block branches to.964 }965 966 // Make a final pass over the basic blocks from the old function to gather967 // any return instructions which survived folding. We have to do this here968 // because we can iteratively remove and merge returns above.969 for (Function::iterator I = cast<BasicBlock>(VMap[StartingBB])->getIterator(),970 E = NewFunc->end();971 I != E; ++I)972 if (ReturnInst *RI = dyn_cast<ReturnInst>(I->getTerminator()))973 Returns.push_back(RI);974}975 976/// This works exactly like CloneFunctionInto,977/// except that it does some simple constant prop and DCE on the fly. The978/// effect of this is to copy significantly less code in cases where (for979/// example) a function call with constant arguments is inlined, and those980/// constant arguments cause a significant amount of code in the callee to be981/// dead. Since this doesn't produce an exact copy of the input, it can't be982/// used for things like CloneFunction or CloneModule.983void llvm::CloneAndPruneFunctionInto(984 Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap,985 bool ModuleLevelChanges, SmallVectorImpl<ReturnInst *> &Returns,986 const char *NameSuffix, ClonedCodeInfo *CodeInfo) {987 CloneAndPruneIntoFromInst(NewFunc, OldFunc, &OldFunc->front().front(), VMap,988 ModuleLevelChanges, Returns, NameSuffix, CodeInfo);989}990 991/// Remaps instructions in \p Blocks using the mapping in \p VMap.992void llvm::remapInstructionsInBlocks(ArrayRef<BasicBlock *> Blocks,993 ValueToValueMapTy &VMap) {994 // Rewrite the code to refer to itself.995 for (BasicBlock *BB : Blocks) {996 for (Instruction &Inst : *BB) {997 RemapDbgRecordRange(Inst.getModule(), Inst.getDbgRecordRange(), VMap,998 RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);999 RemapInstruction(&Inst, VMap,1000 RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);1001 }1002 }1003}1004 1005/// Clones a loop \p OrigLoop. Returns the loop and the blocks in \p1006/// Blocks.1007///1008/// Updates LoopInfo and DominatorTree assuming the loop is dominated by block1009/// \p LoopDomBB. Insert the new blocks before block specified in \p Before.1010Loop *llvm::cloneLoopWithPreheader(BasicBlock *Before, BasicBlock *LoopDomBB,1011 Loop *OrigLoop, ValueToValueMapTy &VMap,1012 const Twine &NameSuffix, LoopInfo *LI,1013 DominatorTree *DT,1014 SmallVectorImpl<BasicBlock *> &Blocks) {1015 Function *F = OrigLoop->getHeader()->getParent();1016 Loop *ParentLoop = OrigLoop->getParentLoop();1017 DenseMap<Loop *, Loop *> LMap;1018 1019 Loop *NewLoop = LI->AllocateLoop();1020 LMap[OrigLoop] = NewLoop;1021 if (ParentLoop)1022 ParentLoop->addChildLoop(NewLoop);1023 else1024 LI->addTopLevelLoop(NewLoop);1025 1026 BasicBlock *OrigPH = OrigLoop->getLoopPreheader();1027 assert(OrigPH && "No preheader");1028 BasicBlock *NewPH = CloneBasicBlock(OrigPH, VMap, NameSuffix, F);1029 // To rename the loop PHIs.1030 VMap[OrigPH] = NewPH;1031 Blocks.push_back(NewPH);1032 1033 // Update LoopInfo.1034 if (ParentLoop)1035 ParentLoop->addBasicBlockToLoop(NewPH, *LI);1036 1037 // Update DominatorTree.1038 DT->addNewBlock(NewPH, LoopDomBB);1039 1040 for (Loop *CurLoop : OrigLoop->getLoopsInPreorder()) {1041 Loop *&NewLoop = LMap[CurLoop];1042 if (!NewLoop) {1043 NewLoop = LI->AllocateLoop();1044 1045 // Establish the parent/child relationship.1046 Loop *OrigParent = CurLoop->getParentLoop();1047 assert(OrigParent && "Could not find the original parent loop");1048 Loop *NewParentLoop = LMap[OrigParent];1049 assert(NewParentLoop && "Could not find the new parent loop");1050 1051 NewParentLoop->addChildLoop(NewLoop);1052 }1053 }1054 1055 for (BasicBlock *BB : OrigLoop->getBlocks()) {1056 Loop *CurLoop = LI->getLoopFor(BB);1057 Loop *&NewLoop = LMap[CurLoop];1058 assert(NewLoop && "Expecting new loop to be allocated");1059 1060 BasicBlock *NewBB = CloneBasicBlock(BB, VMap, NameSuffix, F);1061 VMap[BB] = NewBB;1062 1063 // Update LoopInfo.1064 NewLoop->addBasicBlockToLoop(NewBB, *LI);1065 1066 // Add DominatorTree node. After seeing all blocks, update to correct1067 // IDom.1068 DT->addNewBlock(NewBB, NewPH);1069 1070 Blocks.push_back(NewBB);1071 }1072 1073 for (BasicBlock *BB : OrigLoop->getBlocks()) {1074 // Update loop headers.1075 Loop *CurLoop = LI->getLoopFor(BB);1076 if (BB == CurLoop->getHeader())1077 LMap[CurLoop]->moveToHeader(cast<BasicBlock>(VMap[BB]));1078 1079 // Update DominatorTree.1080 BasicBlock *IDomBB = DT->getNode(BB)->getIDom()->getBlock();1081 DT->changeImmediateDominator(cast<BasicBlock>(VMap[BB]),1082 cast<BasicBlock>(VMap[IDomBB]));1083 }1084 1085 // Move them physically from the end of the block list.1086 F->splice(Before->getIterator(), F, NewPH->getIterator());1087 F->splice(Before->getIterator(), F, NewLoop->getHeader()->getIterator(),1088 F->end());1089 1090 return NewLoop;1091}1092 1093/// Duplicate non-Phi instructions from the beginning of block up to1094/// StopAt instruction into a split block between BB and its predecessor.1095BasicBlock *llvm::DuplicateInstructionsInSplitBetween(1096 BasicBlock *BB, BasicBlock *PredBB, Instruction *StopAt,1097 ValueToValueMapTy &ValueMapping, DomTreeUpdater &DTU) {1098 1099 assert(count(successors(PredBB), BB) == 1 &&1100 "There must be a single edge between PredBB and BB!");1101 // We are going to have to map operands from the original BB block to the new1102 // copy of the block 'NewBB'. If there are PHI nodes in BB, evaluate them to1103 // account for entry from PredBB.1104 BasicBlock::iterator BI = BB->begin();1105 for (; PHINode *PN = dyn_cast<PHINode>(BI); ++BI)1106 ValueMapping[PN] = PN->getIncomingValueForBlock(PredBB);1107 1108 BasicBlock *NewBB = SplitEdge(PredBB, BB);1109 NewBB->setName(PredBB->getName() + ".split");1110 Instruction *NewTerm = NewBB->getTerminator();1111 1112 // FIXME: SplitEdge does not yet take a DTU, so we include the split edge1113 // in the update set here.1114 DTU.applyUpdates({{DominatorTree::Delete, PredBB, BB},1115 {DominatorTree::Insert, PredBB, NewBB},1116 {DominatorTree::Insert, NewBB, BB}});1117 1118 // Clone the non-phi instructions of BB into NewBB, keeping track of the1119 // mapping and using it to remap operands in the cloned instructions.1120 // Stop once we see the terminator too. This covers the case where BB's1121 // terminator gets replaced and StopAt == BB's terminator.1122 for (; StopAt != &*BI && BB->getTerminator() != &*BI; ++BI) {1123 Instruction *New = BI->clone();1124 New->setName(BI->getName());1125 New->insertBefore(NewTerm->getIterator());1126 New->cloneDebugInfoFrom(&*BI);1127 ValueMapping[&*BI] = New;1128 1129 // Remap operands to patch up intra-block references.1130 for (unsigned i = 0, e = New->getNumOperands(); i != e; ++i)1131 if (Instruction *Inst = dyn_cast<Instruction>(New->getOperand(i))) {1132 auto I = ValueMapping.find(Inst);1133 if (I != ValueMapping.end())1134 New->setOperand(i, I->second);1135 }1136 1137 // Remap debug variable operands.1138 remapDebugVariable(ValueMapping, New);1139 }1140 1141 return NewBB;1142}1143 1144void llvm::cloneNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes,1145 DenseMap<MDNode *, MDNode *> &ClonedScopes,1146 StringRef Ext, LLVMContext &Context) {1147 MDBuilder MDB(Context);1148 1149 for (MDNode *ScopeList : NoAliasDeclScopes) {1150 for (const MDOperand &MDOp : ScopeList->operands()) {1151 if (MDNode *MD = dyn_cast<MDNode>(MDOp)) {1152 AliasScopeNode SNANode(MD);1153 1154 std::string Name;1155 auto ScopeName = SNANode.getName();1156 if (!ScopeName.empty())1157 Name = (Twine(ScopeName) + ":" + Ext).str();1158 else1159 Name = std::string(Ext);1160 1161 MDNode *NewScope = MDB.createAnonymousAliasScope(1162 const_cast<MDNode *>(SNANode.getDomain()), Name);1163 ClonedScopes.insert(std::make_pair(MD, NewScope));1164 }1165 }1166 }1167}1168 1169void llvm::adaptNoAliasScopes(Instruction *I,1170 const DenseMap<MDNode *, MDNode *> &ClonedScopes,1171 LLVMContext &Context) {1172 auto CloneScopeList = [&](const MDNode *ScopeList) -> MDNode * {1173 bool NeedsReplacement = false;1174 SmallVector<Metadata *, 8> NewScopeList;1175 for (const MDOperand &MDOp : ScopeList->operands()) {1176 if (MDNode *MD = dyn_cast<MDNode>(MDOp)) {1177 if (auto *NewMD = ClonedScopes.lookup(MD)) {1178 NewScopeList.push_back(NewMD);1179 NeedsReplacement = true;1180 continue;1181 }1182 NewScopeList.push_back(MD);1183 }1184 }1185 if (NeedsReplacement)1186 return MDNode::get(Context, NewScopeList);1187 return nullptr;1188 };1189 1190 if (auto *Decl = dyn_cast<NoAliasScopeDeclInst>(I))1191 if (MDNode *NewScopeList = CloneScopeList(Decl->getScopeList()))1192 Decl->setScopeList(NewScopeList);1193 1194 auto replaceWhenNeeded = [&](unsigned MD_ID) {1195 if (const MDNode *CSNoAlias = I->getMetadata(MD_ID))1196 if (MDNode *NewScopeList = CloneScopeList(CSNoAlias))1197 I->setMetadata(MD_ID, NewScopeList);1198 };1199 replaceWhenNeeded(LLVMContext::MD_noalias);1200 replaceWhenNeeded(LLVMContext::MD_alias_scope);1201}1202 1203void llvm::cloneAndAdaptNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes,1204 ArrayRef<BasicBlock *> NewBlocks,1205 LLVMContext &Context, StringRef Ext) {1206 if (NoAliasDeclScopes.empty())1207 return;1208 1209 DenseMap<MDNode *, MDNode *> ClonedScopes;1210 LLVM_DEBUG(dbgs() << "cloneAndAdaptNoAliasScopes: cloning "1211 << NoAliasDeclScopes.size() << " node(s)\n");1212 1213 cloneNoAliasScopes(NoAliasDeclScopes, ClonedScopes, Ext, Context);1214 // Identify instructions using metadata that needs adaptation1215 for (BasicBlock *NewBlock : NewBlocks)1216 for (Instruction &I : *NewBlock)1217 adaptNoAliasScopes(&I, ClonedScopes, Context);1218}1219 1220void llvm::cloneAndAdaptNoAliasScopes(ArrayRef<MDNode *> NoAliasDeclScopes,1221 Instruction *IStart, Instruction *IEnd,1222 LLVMContext &Context, StringRef Ext) {1223 if (NoAliasDeclScopes.empty())1224 return;1225 1226 DenseMap<MDNode *, MDNode *> ClonedScopes;1227 LLVM_DEBUG(dbgs() << "cloneAndAdaptNoAliasScopes: cloning "1228 << NoAliasDeclScopes.size() << " node(s)\n");1229 1230 cloneNoAliasScopes(NoAliasDeclScopes, ClonedScopes, Ext, Context);1231 // Identify instructions using metadata that needs adaptation1232 assert(IStart->getParent() == IEnd->getParent() && "different basic block ?");1233 auto ItStart = IStart->getIterator();1234 auto ItEnd = IEnd->getIterator();1235 ++ItEnd; // IEnd is included, increment ItEnd to get the end of the range1236 for (auto &I : llvm::make_range(ItStart, ItEnd))1237 adaptNoAliasScopes(&I, ClonedScopes, Context);1238}1239 1240void llvm::identifyNoAliasScopesToClone(1241 ArrayRef<BasicBlock *> BBs, SmallVectorImpl<MDNode *> &NoAliasDeclScopes) {1242 for (BasicBlock *BB : BBs)1243 for (Instruction &I : *BB)1244 if (auto *Decl = dyn_cast<NoAliasScopeDeclInst>(&I))1245 NoAliasDeclScopes.push_back(Decl->getScopeList());1246}1247 1248void llvm::identifyNoAliasScopesToClone(1249 BasicBlock::iterator Start, BasicBlock::iterator End,1250 SmallVectorImpl<MDNode *> &NoAliasDeclScopes) {1251 for (Instruction &I : make_range(Start, End))1252 if (auto *Decl = dyn_cast<NoAliasScopeDeclInst>(&I))1253 NoAliasDeclScopes.push_back(Decl->getScopeList());1254}1255