brintos / llvm-project-archived public Read only

1//===- GlobalOpt.cpp - Optimize Global Variables --------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This pass transforms simple global variables that never have their address10// taken.  If obviously true, it marks read/write globals as constant, deletes11// variables only stored to, etc.12//13//===----------------------------------------------------------------------===//14 15#include "llvm/Transforms/IPO/GlobalOpt.h"16#include "llvm/ADT/DenseMap.h"17#include "llvm/ADT/STLExtras.h"18#include "llvm/ADT/SmallPtrSet.h"19#include "llvm/ADT/SmallVector.h"20#include "llvm/ADT/Statistic.h"21#include "llvm/ADT/Twine.h"22#include "llvm/ADT/iterator_range.h"23#include "llvm/Analysis/BlockFrequencyInfo.h"24#include "llvm/Analysis/ConstantFolding.h"25#include "llvm/Analysis/MemoryBuiltins.h"26#include "llvm/Analysis/TargetLibraryInfo.h"27#include "llvm/Analysis/TargetTransformInfo.h"28#include "llvm/Analysis/ValueTracking.h"29#include "llvm/BinaryFormat/Dwarf.h"30#include "llvm/IR/Attributes.h"31#include "llvm/IR/BasicBlock.h"32#include "llvm/IR/CallingConv.h"33#include "llvm/IR/Constant.h"34#include "llvm/IR/Constants.h"35#include "llvm/IR/DataLayout.h"36#include "llvm/IR/DebugInfoMetadata.h"37#include "llvm/IR/DerivedTypes.h"38#include "llvm/IR/Dominators.h"39#include "llvm/IR/Function.h"40#include "llvm/IR/GlobalAlias.h"41#include "llvm/IR/GlobalValue.h"42#include "llvm/IR/GlobalVariable.h"43#include "llvm/IR/IRBuilder.h"44#include "llvm/IR/InstrTypes.h"45#include "llvm/IR/Instruction.h"46#include "llvm/IR/Instructions.h"47#include "llvm/IR/IntrinsicInst.h"48#include "llvm/IR/Module.h"49#include "llvm/IR/Operator.h"50#include "llvm/IR/Type.h"51#include "llvm/IR/Use.h"52#include "llvm/IR/User.h"53#include "llvm/IR/Value.h"54#include "llvm/IR/ValueHandle.h"55#include "llvm/Support/AtomicOrdering.h"56#include "llvm/Support/Casting.h"57#include "llvm/Support/CommandLine.h"58#include "llvm/Support/Debug.h"59#include "llvm/Support/ErrorHandling.h"60#include "llvm/Support/raw_ostream.h"61#include "llvm/Transforms/IPO.h"62#include "llvm/Transforms/Utils/CtorUtils.h"63#include "llvm/Transforms/Utils/Evaluator.h"64#include "llvm/Transforms/Utils/GlobalStatus.h"65#include "llvm/Transforms/Utils/Local.h"66#include <cassert>67#include <cstdint>68#include <optional>69#include <utility>70#include <vector>71 72using namespace llvm;73 74#define DEBUG_TYPE "globalopt"75 76STATISTIC(NumMarked    , "Number of globals marked constant");77STATISTIC(NumUnnamed   , "Number of globals marked unnamed_addr");78STATISTIC(NumSRA       , "Number of aggregate globals broken into scalars");79STATISTIC(NumSubstitute,"Number of globals with initializers stored into them");80STATISTIC(NumDeleted   , "Number of globals deleted");81STATISTIC(NumGlobUses  , "Number of global uses devirtualized");82STATISTIC(NumLocalized , "Number of globals localized");83STATISTIC(NumShrunkToBool  , "Number of global vars shrunk to booleans");84STATISTIC(NumFastCallFns   , "Number of functions converted to fastcc");85STATISTIC(NumCtorsEvaluated, "Number of static ctors evaluated");86STATISTIC(NumNestRemoved   , "Number of nest attributes removed");87STATISTIC(NumAliasesResolved, "Number of global aliases resolved");88STATISTIC(NumAliasesRemoved, "Number of global aliases eliminated");89STATISTIC(NumCXXDtorsRemoved, "Number of global C++ destructors removed");90STATISTIC(NumAtExitRemoved, "Number of atexit handlers removed");91STATISTIC(NumInternalFunc, "Number of internal functions");92STATISTIC(NumColdCC, "Number of functions marked coldcc");93STATISTIC(NumIFuncsResolved, "Number of statically resolved IFuncs");94STATISTIC(NumIFuncsDeleted, "Number of IFuncs removed");95 96static cl::opt<bool>97    OptimizeNonFMVCallers("optimize-non-fmv-callers",98                          cl::desc("Statically resolve calls to versioned "99                                   "functions from non-versioned callers."),100                          cl::init(true), cl::Hidden);101 102static cl::opt<bool>103    EnableColdCCStressTest("enable-coldcc-stress-test",104                           cl::desc("Enable stress test of coldcc by adding "105                                    "calling conv to all internal functions."),106                           cl::init(false), cl::Hidden);107 108static cl::opt<int> ColdCCRelFreq(109    "coldcc-rel-freq", cl::Hidden, cl::init(2),110    cl::desc(111        "Maximum block frequency, expressed as a percentage of caller's "112        "entry frequency, for a call site to be considered cold for enabling "113        "coldcc"));114 115/// Is this global variable possibly used by a leak checker as a root?  If so,116/// we might not really want to eliminate the stores to it.117static bool isLeakCheckerRoot(GlobalVariable *GV) {118  // A global variable is a root if it is a pointer, or could plausibly contain119  // a pointer.  There are two challenges; one is that we could have a struct120  // the has an inner member which is a pointer.  We recurse through the type to121  // detect these (up to a point).  The other is that we may actually be a union122  // of a pointer and another type, and so our LLVM type is an integer which123  // gets converted into a pointer, or our type is an [i8 x #] with a pointer124  // potentially contained here.125 126  if (GV->hasPrivateLinkage())127    return false;128 129  SmallVector<Type *, 4> Types;130  Types.push_back(GV->getValueType());131 132  unsigned Limit = 20;133  do {134    Type *Ty = Types.pop_back_val();135    switch (Ty->getTypeID()) {136      default: break;137      case Type::PointerTyID:138        return true;139      case Type::FixedVectorTyID:140      case Type::ScalableVectorTyID:141        if (cast<VectorType>(Ty)->getElementType()->isPointerTy())142          return true;143        break;144      case Type::ArrayTyID:145        Types.push_back(cast<ArrayType>(Ty)->getElementType());146        break;147      case Type::StructTyID: {148        StructType *STy = cast<StructType>(Ty);149        if (STy->isOpaque()) return true;150        for (Type *InnerTy : STy->elements()) {151          if (isa<PointerType>(InnerTy)) return true;152          if (isa<StructType>(InnerTy) || isa<ArrayType>(InnerTy) ||153              isa<VectorType>(InnerTy))154            Types.push_back(InnerTy);155        }156        break;157      }158    }159    if (--Limit == 0) return true;160  } while (!Types.empty());161  return false;162}163 164/// Given a value that is stored to a global but never read, determine whether165/// it's safe to remove the store and the chain of computation that feeds the166/// store.167static bool IsSafeComputationToRemove(168    Value *V, function_ref<TargetLibraryInfo &(Function &)> GetTLI) {169  do {170    if (isa<Constant>(V))171      return true;172    if (!V->hasOneUse())173      return false;174    if (isa<LoadInst>(V) || isa<InvokeInst>(V) || isa<Argument>(V) ||175        isa<GlobalValue>(V))176      return false;177    if (isAllocationFn(V, GetTLI))178      return true;179 180    Instruction *I = cast<Instruction>(V);181    if (I->mayHaveSideEffects())182      return false;183    if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(I)) {184      if (!GEP->hasAllConstantIndices())185        return false;186    } else if (I->getNumOperands() != 1) {187      return false;188    }189 190    V = I->getOperand(0);191  } while (true);192}193 194/// This GV is a pointer root.  Loop over all users of the global and clean up195/// any that obviously don't assign the global a value that isn't dynamically196/// allocated.197static bool198CleanupPointerRootUsers(GlobalVariable *GV,199                        function_ref<TargetLibraryInfo &(Function &)> GetTLI) {200  // A brief explanation of leak checkers.  The goal is to find bugs where201  // pointers are forgotten, causing an accumulating growth in memory202  // usage over time.  The common strategy for leak checkers is to explicitly203  // allow the memory pointed to by globals at exit.  This is popular because it204  // also solves another problem where the main thread of a C++ program may shut205  // down before other threads that are still expecting to use those globals. To206  // handle that case, we expect the program may create a singleton and never207  // destroy it.208 209  bool Changed = false;210 211  // If Dead[n].first is the only use of a malloc result, we can delete its212  // chain of computation and the store to the global in Dead[n].second.213  SmallVector<std::pair<Instruction *, Instruction *>, 32> Dead;214 215  SmallVector<User *> Worklist(GV->users());216  // Constants can't be pointers to dynamically allocated memory.217  while (!Worklist.empty()) {218    User *U = Worklist.pop_back_val();219    if (StoreInst *SI = dyn_cast<StoreInst>(U)) {220      Value *V = SI->getValueOperand();221      if (isa<Constant>(V)) {222        Changed = true;223        SI->eraseFromParent();224      } else if (Instruction *I = dyn_cast<Instruction>(V)) {225        if (I->hasOneUse())226          Dead.push_back(std::make_pair(I, SI));227      }228    } else if (MemSetInst *MSI = dyn_cast<MemSetInst>(U)) {229      if (isa<Constant>(MSI->getValue())) {230        Changed = true;231        MSI->eraseFromParent();232      } else if (Instruction *I = dyn_cast<Instruction>(MSI->getValue())) {233        if (I->hasOneUse())234          Dead.push_back(std::make_pair(I, MSI));235      }236    } else if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(U)) {237      GlobalVariable *MemSrc = dyn_cast<GlobalVariable>(MTI->getSource());238      if (MemSrc && MemSrc->isConstant()) {239        Changed = true;240        MTI->eraseFromParent();241      } else if (Instruction *I = dyn_cast<Instruction>(MTI->getSource())) {242        if (I->hasOneUse())243          Dead.push_back(std::make_pair(I, MTI));244      }245    } else if (ConstantExpr *CE = dyn_cast<ConstantExpr>(U)) {246      if (isa<GEPOperator>(CE))247        append_range(Worklist, CE->users());248    }249  }250 251  for (const auto &[Inst, Store] : Dead) {252    if (IsSafeComputationToRemove(Inst, GetTLI)) {253      Store->eraseFromParent();254      Instruction *I = Inst;255      do {256        if (isAllocationFn(I, GetTLI))257          break;258        Instruction *J = dyn_cast<Instruction>(I->getOperand(0));259        if (!J)260          break;261        I->eraseFromParent();262        I = J;263      } while (true);264      I->eraseFromParent();265      Changed = true;266    }267  }268 269  GV->removeDeadConstantUsers();270  return Changed;271}272 273/// We just marked GV constant.  Loop over all users of the global, cleaning up274/// the obvious ones.  This is largely just a quick scan over the use list to275/// clean up the easy and obvious cruft.  This returns true if it made a change.276static bool CleanupConstantGlobalUsers(GlobalVariable *GV,277                                       const DataLayout &DL) {278  Constant *Init = GV->getInitializer();279  SmallVector<User *, 8> WorkList(GV->users());280  SmallPtrSet<User *, 8> Visited;281  bool Changed = false;282 283  SmallVector<WeakTrackingVH> MaybeDeadInsts;284  auto EraseFromParent = [&](Instruction *I) {285    for (Value *Op : I->operands())286      if (auto *OpI = dyn_cast<Instruction>(Op))287        MaybeDeadInsts.push_back(OpI);288    I->eraseFromParent();289    Changed = true;290  };291  while (!WorkList.empty()) {292    User *U = WorkList.pop_back_val();293    if (!Visited.insert(U).second)294      continue;295 296    if (auto *BO = dyn_cast<BitCastOperator>(U))297      append_range(WorkList, BO->users());298    if (auto *ASC = dyn_cast<AddrSpaceCastOperator>(U))299      append_range(WorkList, ASC->users());300    else if (auto *GEP = dyn_cast<GEPOperator>(U))301      append_range(WorkList, GEP->users());302    else if (auto *LI = dyn_cast<LoadInst>(U)) {303      // A load from a uniform value is always the same, regardless of any304      // applied offset.305      Type *Ty = LI->getType();306      if (Constant *Res = ConstantFoldLoadFromUniformValue(Init, Ty, DL)) {307        LI->replaceAllUsesWith(Res);308        EraseFromParent(LI);309        continue;310      }311 312      Value *PtrOp = LI->getPointerOperand();313      APInt Offset(DL.getIndexTypeSizeInBits(PtrOp->getType()), 0);314      PtrOp = PtrOp->stripAndAccumulateConstantOffsets(315          DL, Offset, /* AllowNonInbounds */ true);316      if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(PtrOp)) {317        if (II->getIntrinsicID() == Intrinsic::threadlocal_address)318          PtrOp = II->getArgOperand(0);319      }320      if (PtrOp == GV) {321        if (auto *Value = ConstantFoldLoadFromConst(Init, Ty, Offset, DL)) {322          LI->replaceAllUsesWith(Value);323          EraseFromParent(LI);324        }325      }326    } else if (StoreInst *SI = dyn_cast<StoreInst>(U)) {327      // Store must be unreachable or storing Init into the global.328      EraseFromParent(SI);329    } else if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(U)) { // memset/cpy/mv330      if (getUnderlyingObject(MI->getRawDest()) == GV)331        EraseFromParent(MI);332    } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U)) {333      if (II->getIntrinsicID() == Intrinsic::threadlocal_address)334        append_range(WorkList, II->users());335    }336  }337 338  Changed |=339      RecursivelyDeleteTriviallyDeadInstructionsPermissive(MaybeDeadInsts);340  GV->removeDeadConstantUsers();341  return Changed;342}343 344/// Part of the global at a specific offset, which is only accessed through345/// loads and stores with the given type.346struct GlobalPart {347  Type *Ty;348  Constant *Initializer = nullptr;349  bool IsLoaded = false;350  bool IsStored = false;351};352 353/// Look at all uses of the global and determine which (offset, type) pairs it354/// can be split into.355static bool collectSRATypes(DenseMap<uint64_t, GlobalPart> &Parts,356                            GlobalVariable *GV, const DataLayout &DL) {357  SmallVector<Use *, 16> Worklist;358  SmallPtrSet<Use *, 16> Visited;359  auto AppendUses = [&](Value *V) {360    for (Use &U : V->uses())361      if (Visited.insert(&U).second)362        Worklist.push_back(&U);363  };364  AppendUses(GV);365  while (!Worklist.empty()) {366    Use *U = Worklist.pop_back_val();367    User *V = U->getUser();368 369    auto *GEP = dyn_cast<GEPOperator>(V);370    if (isa<BitCastOperator>(V) || isa<AddrSpaceCastOperator>(V) ||371        (GEP && GEP->hasAllConstantIndices())) {372      AppendUses(V);373      continue;374    }375 376    if (Value *Ptr = getLoadStorePointerOperand(V)) {377      // This is storing the global address into somewhere, not storing into378      // the global.379      if (isa<StoreInst>(V) && U->getOperandNo() == 0)380        return false;381 382      APInt Offset(DL.getIndexTypeSizeInBits(Ptr->getType()), 0);383      Ptr = Ptr->stripAndAccumulateConstantOffsets(DL, Offset,384                                                   /* AllowNonInbounds */ true);385      if (Ptr != GV || Offset.getActiveBits() >= 64)386        return false;387 388      // TODO: We currently require that all accesses at a given offset must389      // use the same type. This could be relaxed.390      Type *Ty = getLoadStoreType(V);391      const auto &[It, Inserted] =392          Parts.try_emplace(Offset.getZExtValue(), GlobalPart{Ty});393      if (Ty != It->second.Ty)394        return false;395 396      if (Inserted) {397        It->second.Initializer =398            ConstantFoldLoadFromConst(GV->getInitializer(), Ty, Offset, DL);399        if (!It->second.Initializer) {400          LLVM_DEBUG(dbgs() << "Global SRA: Failed to evaluate initializer of "401                            << *GV << " with type " << *Ty << " at offset "402                            << Offset.getZExtValue());403          return false;404        }405      }406 407      // Scalable types not currently supported.408      if (Ty->isScalableTy())409        return false;410 411      auto IsStored = [](Value *V, Constant *Initializer) {412        auto *SI = dyn_cast<StoreInst>(V);413        if (!SI)414          return false;415 416        Constant *StoredConst = dyn_cast<Constant>(SI->getOperand(0));417        if (!StoredConst)418          return true;419 420        // Don't consider stores that only write the initializer value.421        return Initializer != StoredConst;422      };423 424      It->second.IsLoaded |= isa<LoadInst>(V);425      It->second.IsStored |= IsStored(V, It->second.Initializer);426      continue;427    }428 429    // Ignore dead constant users.430    if (auto *C = dyn_cast<Constant>(V)) {431      if (!isSafeToDestroyConstant(C))432        return false;433      continue;434    }435 436    // Unknown user.437    return false;438  }439 440  return true;441}442 443/// Copy over the debug info for a variable to its SRA replacements.444static void transferSRADebugInfo(GlobalVariable *GV, GlobalVariable *NGV,445                                 uint64_t FragmentOffsetInBits,446                                 uint64_t FragmentSizeInBits,447                                 uint64_t VarSize) {448  SmallVector<DIGlobalVariableExpression *, 1> GVs;449  GV->getDebugInfo(GVs);450  for (auto *GVE : GVs) {451    DIVariable *Var = GVE->getVariable();452    DIExpression *Expr = GVE->getExpression();453    int64_t CurVarOffsetInBytes = 0;454    uint64_t CurVarOffsetInBits = 0;455    uint64_t FragmentEndInBits = FragmentOffsetInBits + FragmentSizeInBits;456 457    // Calculate the offset (Bytes), Continue if unknown.458    if (!Expr->extractIfOffset(CurVarOffsetInBytes))459      continue;460 461    // Ignore negative offset.462    if (CurVarOffsetInBytes < 0)463      continue;464 465    // Convert offset to bits.466    CurVarOffsetInBits = CHAR_BIT * (uint64_t)CurVarOffsetInBytes;467 468    // Current var starts after the fragment, ignore.469    if (CurVarOffsetInBits >= FragmentEndInBits)470      continue;471 472    uint64_t CurVarSize = Var->getType()->getSizeInBits();473    uint64_t CurVarEndInBits = CurVarOffsetInBits + CurVarSize;474    // Current variable ends before start of fragment, ignore.475    if (CurVarSize != 0 && /* CurVarSize is known */476        CurVarEndInBits <= FragmentOffsetInBits)477      continue;478 479    // Current variable fits in (not greater than) the fragment,480    // does not need fragment expression.481    if (CurVarSize != 0 && /* CurVarSize is known */482        CurVarOffsetInBits >= FragmentOffsetInBits &&483        CurVarEndInBits <= FragmentEndInBits) {484      uint64_t CurVarOffsetInFragment =485          (CurVarOffsetInBits - FragmentOffsetInBits) / 8;486      if (CurVarOffsetInFragment != 0)487        Expr = DIExpression::get(Expr->getContext(), {dwarf::DW_OP_plus_uconst,488                                                      CurVarOffsetInFragment});489      else490        Expr = DIExpression::get(Expr->getContext(), {});491      auto *NGVE =492          DIGlobalVariableExpression::get(GVE->getContext(), Var, Expr);493      NGV->addDebugInfo(NGVE);494      continue;495    }496    // Current variable does not fit in single fragment,497    // emit a fragment expression.498    if (FragmentSizeInBits < VarSize) {499      if (CurVarOffsetInBits > FragmentOffsetInBits)500        continue;501      uint64_t CurVarFragmentOffsetInBits =502          FragmentOffsetInBits - CurVarOffsetInBits;503      uint64_t CurVarFragmentSizeInBits = FragmentSizeInBits;504      if (CurVarSize != 0 && CurVarEndInBits < FragmentEndInBits)505        CurVarFragmentSizeInBits -= (FragmentEndInBits - CurVarEndInBits);506      if (CurVarOffsetInBits)507        Expr = DIExpression::get(Expr->getContext(), {});508      if (auto E = DIExpression::createFragmentExpression(509              Expr, CurVarFragmentOffsetInBits, CurVarFragmentSizeInBits))510        Expr = *E;511      else512        continue;513    }514    auto *NGVE = DIGlobalVariableExpression::get(GVE->getContext(), Var, Expr);515    NGV->addDebugInfo(NGVE);516  }517}518 519/// Perform scalar replacement of aggregates on the specified global variable.520/// This opens the door for other optimizations by exposing the behavior of the521/// program in a more fine-grained way.  We have determined that this522/// transformation is safe already.  We return the first global variable we523/// insert so that the caller can reprocess it.524static GlobalVariable *SRAGlobal(GlobalVariable *GV, const DataLayout &DL) {525  assert(GV->hasLocalLinkage());526 527  // Collect types to split into.528  DenseMap<uint64_t, GlobalPart> Parts;529  if (!collectSRATypes(Parts, GV, DL) || Parts.empty())530    return nullptr;531 532  // Make sure we don't SRA back to the same type.533  if (Parts.size() == 1 && Parts.begin()->second.Ty == GV->getValueType())534    return nullptr;535 536  // Don't perform SRA if we would have to split into many globals. Ignore537  // parts that are either only loaded or only stored, because we expect them538  // to be optimized away.539  unsigned NumParts = count_if(Parts, [](const auto &Pair) {540    return Pair.second.IsLoaded && Pair.second.IsStored;541  });542  if (NumParts > 16)543    return nullptr;544 545  // Sort by offset.546  SmallVector<std::tuple<uint64_t, Type *, Constant *>, 16> TypesVector;547  for (const auto &Pair : Parts) {548    TypesVector.push_back(549        {Pair.first, Pair.second.Ty, Pair.second.Initializer});550  }551  sort(TypesVector, llvm::less_first());552 553  // Check that the types are non-overlapping.554  uint64_t Offset = 0;555  for (const auto &[OffsetForTy, Ty, _] : TypesVector) {556    // Overlaps with previous type.557    if (OffsetForTy < Offset)558      return nullptr;559 560    Offset = OffsetForTy + DL.getTypeAllocSize(Ty);561  }562 563  // Some accesses go beyond the end of the global, don't bother.564  if (Offset > DL.getTypeAllocSize(GV->getValueType()))565    return nullptr;566 567  LLVM_DEBUG(dbgs() << "PERFORMING GLOBAL SRA ON: " << *GV << "\n");568 569  // Get the alignment of the global, either explicit or target-specific.570  Align StartAlignment =571      DL.getValueOrABITypeAlignment(GV->getAlign(), GV->getValueType());572  uint64_t VarSize = DL.getTypeSizeInBits(GV->getValueType());573 574  // Create replacement globals.575  DenseMap<uint64_t, GlobalVariable *> NewGlobals;576  unsigned NameSuffix = 0;577  for (auto &[OffsetForTy, Ty, Initializer] : TypesVector) {578    GlobalVariable *NGV = new GlobalVariable(579        *GV->getParent(), Ty, false, GlobalVariable::InternalLinkage,580        Initializer, GV->getName() + "." + Twine(NameSuffix++), GV,581        GV->getThreadLocalMode(), GV->getAddressSpace());582    // Start out by copying attributes from the original, including alignment.583    NGV->copyAttributesFrom(GV);584    NewGlobals.insert({OffsetForTy, NGV});585 586    // Calculate the known alignment of the field.  If the original aggregate587    // had 256 byte alignment for example, then the element at a given offset588    // may also have a known alignment, and something might depend on that:589    // propagate info to each field.590    Align NewAlign = commonAlignment(StartAlignment, OffsetForTy);591    NGV->setAlignment(NewAlign);592 593    // Copy over the debug info for the variable.594    transferSRADebugInfo(GV, NGV, OffsetForTy * 8,595                         DL.getTypeAllocSizeInBits(Ty), VarSize);596  }597 598  // Replace uses of the original global with uses of the new global.599  SmallVector<Value *, 16> Worklist;600  SmallPtrSet<Value *, 16> Visited;601  SmallVector<WeakTrackingVH, 16> DeadInsts;602  auto AppendUsers = [&](Value *V) {603    for (User *U : V->users())604      if (Visited.insert(U).second)605        Worklist.push_back(U);606  };607  AppendUsers(GV);608  while (!Worklist.empty()) {609    Value *V = Worklist.pop_back_val();610    if (isa<BitCastOperator>(V) || isa<AddrSpaceCastOperator>(V) ||611        isa<GEPOperator>(V)) {612      AppendUsers(V);613      if (isa<Instruction>(V))614        DeadInsts.push_back(V);615      continue;616    }617 618    if (Value *Ptr = getLoadStorePointerOperand(V)) {619      APInt Offset(DL.getIndexTypeSizeInBits(Ptr->getType()), 0);620      Ptr = Ptr->stripAndAccumulateConstantOffsets(DL, Offset,621                                                   /* AllowNonInbounds */ true);622      assert(Ptr == GV && "Load/store must be from/to global");623      GlobalVariable *NGV = NewGlobals[Offset.getZExtValue()];624      assert(NGV && "Must have replacement global for this offset");625 626      // Update the pointer operand and recalculate alignment.627      Align PrefAlign = DL.getPrefTypeAlign(getLoadStoreType(V));628      Align NewAlign =629          getOrEnforceKnownAlignment(NGV, PrefAlign, DL, cast<Instruction>(V));630 631      if (auto *LI = dyn_cast<LoadInst>(V)) {632        LI->setOperand(0, NGV);633        LI->setAlignment(NewAlign);634      } else {635        auto *SI = cast<StoreInst>(V);636        SI->setOperand(1, NGV);637        SI->setAlignment(NewAlign);638      }639      continue;640    }641 642    assert(isa<Constant>(V) && isSafeToDestroyConstant(cast<Constant>(V)) &&643           "Other users can only be dead constants");644  }645 646  // Delete old instructions and global.647  RecursivelyDeleteTriviallyDeadInstructions(DeadInsts);648  GV->removeDeadConstantUsers();649  GV->eraseFromParent();650  ++NumSRA;651 652  assert(NewGlobals.size() > 0);653  return NewGlobals.begin()->second;654}655 656/// Return true if all users of the specified value will trap if the value is657/// dynamically null.  PHIs keeps track of any phi nodes we've seen to avoid658/// reprocessing them.659static bool AllUsesOfValueWillTrapIfNull(const Value *V,660                                        SmallPtrSetImpl<const PHINode*> &PHIs) {661  for (const User *U : V->users()) {662    if (const Instruction *I = dyn_cast<Instruction>(U)) {663      // If null pointer is considered valid, then all uses are non-trapping.664      // Non address-space 0 globals have already been pruned by the caller.665      if (NullPointerIsDefined(I->getFunction()))666        return false;667    }668    if (isa<LoadInst>(U)) {669      // Will trap.670    } else if (const StoreInst *SI = dyn_cast<StoreInst>(U)) {671      if (SI->getOperand(0) == V) {672        return false;  // Storing the value.673      }674    } else if (const CallInst *CI = dyn_cast<CallInst>(U)) {675      if (CI->getCalledOperand() != V) {676        return false;  // Not calling the ptr677      }678    } else if (const InvokeInst *II = dyn_cast<InvokeInst>(U)) {679      if (II->getCalledOperand() != V) {680        return false;  // Not calling the ptr681      }682    } else if (const AddrSpaceCastInst *CI = dyn_cast<AddrSpaceCastInst>(U)) {683      if (!AllUsesOfValueWillTrapIfNull(CI, PHIs))684        return false;685    } else if (const GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(U)) {686      if (!AllUsesOfValueWillTrapIfNull(GEPI, PHIs)) return false;687    } else if (const PHINode *PN = dyn_cast<PHINode>(U)) {688      // If we've already seen this phi node, ignore it, it has already been689      // checked.690      if (PHIs.insert(PN).second && !AllUsesOfValueWillTrapIfNull(PN, PHIs))691        return false;692    } else if (isa<ICmpInst>(U) &&693               !ICmpInst::isSigned(cast<ICmpInst>(U)->getPredicate()) &&694               isa<LoadInst>(U->getOperand(0)) &&695               isa<ConstantPointerNull>(U->getOperand(1))) {696      assert(isa<GlobalValue>(cast<LoadInst>(U->getOperand(0))697                                  ->getPointerOperand()698                                  ->stripPointerCasts()) &&699             "Should be GlobalVariable");700      // This and only this kind of non-signed ICmpInst is to be replaced with701      // the comparing of the value of the created global init bool later in702      // optimizeGlobalAddressOfAllocation for the global variable.703    } else {704      return false;705    }706  }707  return true;708}709 710/// Return true if all uses of any loads from GV will trap if the loaded value711/// is null.  Note that this also permits comparisons of the loaded value712/// against null, as a special case.713static bool allUsesOfLoadedValueWillTrapIfNull(const GlobalVariable *GV) {714  SmallVector<const Value *, 4> Worklist;715  Worklist.push_back(GV);716  while (!Worklist.empty()) {717    const Value *P = Worklist.pop_back_val();718    for (const auto *U : P->users()) {719      if (auto *LI = dyn_cast<LoadInst>(U)) {720        if (!LI->isSimple())721          return false;722        SmallPtrSet<const PHINode *, 8> PHIs;723        if (!AllUsesOfValueWillTrapIfNull(LI, PHIs))724          return false;725      } else if (auto *SI = dyn_cast<StoreInst>(U)) {726        if (!SI->isSimple())727          return false;728        // Ignore stores to the global.729        if (SI->getPointerOperand() != P)730          return false;731      } else if (auto *CE = dyn_cast<ConstantExpr>(U)) {732        if (CE->stripPointerCasts() != GV)733          return false;734        // Check further the ConstantExpr.735        Worklist.push_back(CE);736      } else {737        // We don't know or understand this user, bail out.738        return false;739      }740    }741  }742 743  return true;744}745 746/// Get all the loads/store uses for global variable \p GV.747static void allUsesOfLoadAndStores(GlobalVariable *GV,748                                   SmallVector<Value *, 4> &Uses) {749  SmallVector<Value *, 4> Worklist;750  Worklist.push_back(GV);751  while (!Worklist.empty()) {752    auto *P = Worklist.pop_back_val();753    for (auto *U : P->users()) {754      if (auto *CE = dyn_cast<ConstantExpr>(U)) {755        Worklist.push_back(CE);756        continue;757      }758 759      assert((isa<LoadInst>(U) || isa<StoreInst>(U)) &&760             "Expect only load or store instructions");761      Uses.push_back(U);762    }763  }764}765 766static bool OptimizeAwayTrappingUsesOfValue(Value *V, Constant *NewV) {767  bool Changed = false;768  for (auto UI = V->user_begin(), E = V->user_end(); UI != E; ) {769    Instruction *I = cast<Instruction>(*UI++);770    // Uses are non-trapping if null pointer is considered valid.771    // Non address-space 0 globals are already pruned by the caller.772    if (NullPointerIsDefined(I->getFunction()))773      return false;774    if (LoadInst *LI = dyn_cast<LoadInst>(I)) {775      LI->setOperand(0, NewV);776      Changed = true;777    } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {778      if (SI->getOperand(1) == V) {779        SI->setOperand(1, NewV);780        Changed = true;781      }782    } else if (isa<CallInst>(I) || isa<InvokeInst>(I)) {783      CallBase *CB = cast<CallBase>(I);784      if (CB->getCalledOperand() == V) {785        // Calling through the pointer!  Turn into a direct call, but be careful786        // that the pointer is not also being passed as an argument.787        CB->setCalledOperand(NewV);788        Changed = true;789        bool PassedAsArg = false;790        for (unsigned i = 0, e = CB->arg_size(); i != e; ++i)791          if (CB->getArgOperand(i) == V) {792            PassedAsArg = true;793            CB->setArgOperand(i, NewV);794          }795 796        if (PassedAsArg) {797          // Being passed as an argument also.  Be careful to not invalidate UI!798          UI = V->user_begin();799        }800      }801    } else if (AddrSpaceCastInst *CI = dyn_cast<AddrSpaceCastInst>(I)) {802      Changed |= OptimizeAwayTrappingUsesOfValue(803          CI, ConstantExpr::getAddrSpaceCast(NewV, CI->getType()));804      if (CI->use_empty()) {805        Changed = true;806        CI->eraseFromParent();807      }808    } else if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(I)) {809      // Should handle GEP here.810      SmallVector<Constant*, 8> Idxs;811      Idxs.reserve(GEPI->getNumOperands()-1);812      for (User::op_iterator i = GEPI->op_begin() + 1, e = GEPI->op_end();813           i != e; ++i)814        if (Constant *C = dyn_cast<Constant>(*i))815          Idxs.push_back(C);816        else817          break;818      if (Idxs.size() == GEPI->getNumOperands()-1)819        Changed |= OptimizeAwayTrappingUsesOfValue(820            GEPI, ConstantExpr::getGetElementPtr(GEPI->getSourceElementType(),821                                                 NewV, Idxs));822      if (GEPI->use_empty()) {823        Changed = true;824        GEPI->eraseFromParent();825      }826    }827  }828 829  return Changed;830}831 832/// The specified global has only one non-null value stored into it.  If there833/// are uses of the loaded value that would trap if the loaded value is834/// dynamically null, then we know that they cannot be reachable with a null835/// optimize away the load.836static bool OptimizeAwayTrappingUsesOfLoads(837    GlobalVariable *GV, Constant *LV, const DataLayout &DL,838    function_ref<TargetLibraryInfo &(Function &)> GetTLI) {839  bool Changed = false;840 841  // Keep track of whether we are able to remove all the uses of the global842  // other than the store that defines it.843  bool AllNonStoreUsesGone = true;844 845  // Replace all uses of loads with uses of uses of the stored value.846  for (User *GlobalUser : llvm::make_early_inc_range(GV->users())) {847    if (LoadInst *LI = dyn_cast<LoadInst>(GlobalUser)) {848      Changed |= OptimizeAwayTrappingUsesOfValue(LI, LV);849      // If we were able to delete all uses of the loads850      if (LI->use_empty()) {851        LI->eraseFromParent();852        Changed = true;853      } else {854        AllNonStoreUsesGone = false;855      }856    } else if (isa<StoreInst>(GlobalUser)) {857      // Ignore the store that stores "LV" to the global.858      assert(GlobalUser->getOperand(1) == GV &&859             "Must be storing *to* the global");860    } else {861      AllNonStoreUsesGone = false;862 863      // If we get here we could have other crazy uses that are transitively864      // loaded.865      assert((isa<PHINode>(GlobalUser) || isa<SelectInst>(GlobalUser) ||866              isa<ConstantExpr>(GlobalUser) || isa<CmpInst>(GlobalUser) ||867              isa<BitCastInst>(GlobalUser) ||868              isa<GetElementPtrInst>(GlobalUser) ||869              isa<AddrSpaceCastInst>(GlobalUser)) &&870             "Only expect load and stores!");871    }872  }873 874  if (Changed) {875    LLVM_DEBUG(dbgs() << "OPTIMIZED LOADS FROM STORED ONCE POINTER: " << *GV876                      << "\n");877    ++NumGlobUses;878  }879 880  // If we nuked all of the loads, then none of the stores are needed either,881  // nor is the global.882  if (AllNonStoreUsesGone) {883    if (isLeakCheckerRoot(GV)) {884      Changed |= CleanupPointerRootUsers(GV, GetTLI);885    } else {886      Changed = true;887      CleanupConstantGlobalUsers(GV, DL);888    }889    if (GV->use_empty()) {890      LLVM_DEBUG(dbgs() << "  *** GLOBAL NOW DEAD!\n");891      Changed = true;892      GV->eraseFromParent();893      ++NumDeleted;894    }895  }896  return Changed;897}898 899/// Walk the use list of V, constant folding all of the instructions that are900/// foldable.901static void ConstantPropUsersOf(Value *V, const DataLayout &DL,902                                TargetLibraryInfo *TLI) {903  for (Value::user_iterator UI = V->user_begin(), E = V->user_end(); UI != E; )904    if (Instruction *I = dyn_cast<Instruction>(*UI++))905      if (Constant *NewC = ConstantFoldInstruction(I, DL, TLI)) {906        I->replaceAllUsesWith(NewC);907 908        // Advance UI to the next non-I use to avoid invalidating it!909        // Instructions could multiply use V.910        while (UI != E && *UI == I)911          ++UI;912        if (isInstructionTriviallyDead(I, TLI))913          I->eraseFromParent();914      }915}916 917/// This function takes the specified global variable, and transforms the918/// program as if it always contained the result of the specified malloc.919/// Because it is always the result of the specified malloc, there is no reason920/// to actually DO the malloc.  Instead, turn the malloc into a global, and any921/// loads of GV as uses of the new global.922static GlobalVariable *923OptimizeGlobalAddressOfAllocation(GlobalVariable *GV, CallInst *CI,924                                  uint64_t AllocSize, Constant *InitVal,925                                  const DataLayout &DL,926                                  TargetLibraryInfo *TLI) {927  LLVM_DEBUG(errs() << "PROMOTING GLOBAL: " << *GV << "  CALL = " << *CI928                    << '\n');929 930  // Create global of type [AllocSize x i8].931  Type *GlobalType = ArrayType::get(Type::getInt8Ty(GV->getContext()),932                                    AllocSize);933 934  // Create the new global variable.  The contents of the allocated memory is935  // undefined initially, so initialize with an undef value.936  GlobalVariable *NewGV = new GlobalVariable(937      *GV->getParent(), GlobalType, false, GlobalValue::InternalLinkage,938      UndefValue::get(GlobalType), GV->getName() + ".body", nullptr,939      GV->getThreadLocalMode());940 941  // Initialize the global at the point of the original call.  Note that this942  // is a different point from the initialization referred to below for the943  // nullability handling.  Sublety: We have not proven the original global was944  // only initialized once.  As such, we can not fold this into the initializer945  // of the new global as may need to re-init the storage multiple times.946  if (!isa<UndefValue>(InitVal)) {947    IRBuilder<> Builder(CI->getNextNode());948    // TODO: Use alignment above if align!=1949    Builder.CreateMemSet(NewGV, InitVal, AllocSize, std::nullopt);950  }951 952  // Update users of the allocation to use the new global instead.953  CI->replaceAllUsesWith(NewGV);954 955  // If there is a comparison against null, we will insert a global bool to956  // keep track of whether the global was initialized yet or not.957  GlobalVariable *InitBool = new GlobalVariable(958      Type::getInt1Ty(GV->getContext()), false, GlobalValue::InternalLinkage,959      ConstantInt::getFalse(GV->getContext()), GV->getName() + ".init",960      GV->getThreadLocalMode(), GV->getAddressSpace());961  bool InitBoolUsed = false;962 963  // Loop over all instruction uses of GV, processing them in turn.964  SmallVector<Value *, 4> Guses;965  allUsesOfLoadAndStores(GV, Guses);966  for (auto *U : Guses) {967    if (StoreInst *SI = dyn_cast<StoreInst>(U)) {968      // The global is initialized when the store to it occurs. If the stored969      // value is null value, the global bool is set to false, otherwise true.970      auto *NewSI = new StoreInst(971          ConstantInt::getBool(GV->getContext(), !isa<ConstantPointerNull>(972                                                     SI->getValueOperand())),973          InitBool, false, Align(1), SI->getOrdering(), SI->getSyncScopeID(),974          SI->getIterator());975      NewSI->setDebugLoc(SI->getDebugLoc());976      SI->eraseFromParent();977      continue;978    }979 980    LoadInst *LI = cast<LoadInst>(U);981    while (!LI->use_empty()) {982      Use &LoadUse = *LI->use_begin();983      ICmpInst *ICI = dyn_cast<ICmpInst>(LoadUse.getUser());984      if (!ICI) {985        LoadUse.set(NewGV);986        continue;987      }988 989      // Replace the cmp X, 0 with a use of the bool value.990      Value *LV = new LoadInst(InitBool->getValueType(), InitBool,991                               InitBool->getName() + ".val", false, Align(1),992                               LI->getOrdering(), LI->getSyncScopeID(),993                               LI->getIterator());994      // FIXME: Should we use the DebugLoc of the load used by the predicate, or995      // the predicate? The load seems most appropriate, but there's an argument996      // that the new load does not represent the old load, but is simply a997      // component of recomputing the predicate.998      cast<LoadInst>(LV)->setDebugLoc(LI->getDebugLoc());999      InitBoolUsed = true;1000      switch (ICI->getPredicate()) {1001      default: llvm_unreachable("Unknown ICmp Predicate!");1002      case ICmpInst::ICMP_ULT: // X < null -> always false1003        LV = ConstantInt::getFalse(GV->getContext());1004        break;1005      case ICmpInst::ICMP_UGE: // X >= null -> always true1006        LV = ConstantInt::getTrue(GV->getContext());1007        break;1008      case ICmpInst::ICMP_ULE:1009      case ICmpInst::ICMP_EQ:1010        LV = BinaryOperator::CreateNot(LV, "notinit", ICI->getIterator());1011        cast<BinaryOperator>(LV)->setDebugLoc(ICI->getDebugLoc());1012        break;1013      case ICmpInst::ICMP_NE:1014      case ICmpInst::ICMP_UGT:1015        break;  // no change.1016      }1017      ICI->replaceAllUsesWith(LV);1018      ICI->eraseFromParent();1019    }1020    LI->eraseFromParent();1021  }1022 1023  // If the initialization boolean was used, insert it, otherwise delete it.1024  if (!InitBoolUsed) {1025    while (!InitBool->use_empty())  // Delete initializations1026      cast<StoreInst>(InitBool->user_back())->eraseFromParent();1027    delete InitBool;1028  } else1029    GV->getParent()->insertGlobalVariable(GV->getIterator(), InitBool);1030 1031  // Now the GV is dead, nuke it and the allocation..1032  GV->eraseFromParent();1033  CI->eraseFromParent();1034 1035  // To further other optimizations, loop over all users of NewGV and try to1036  // constant prop them.  This will promote GEP instructions with constant1037  // indices into GEP constant-exprs, which will allow global-opt to hack on it.1038  ConstantPropUsersOf(NewGV, DL, TLI);1039 1040  return NewGV;1041}1042 1043/// Scan the use-list of GV checking to make sure that there are no complex uses1044/// of GV.  We permit simple things like dereferencing the pointer, but not1045/// storing through the address, unless it is to the specified global.1046static bool1047valueIsOnlyUsedLocallyOrStoredToOneGlobal(const CallInst *CI,1048                                          const GlobalVariable *GV) {1049  SmallPtrSet<const Value *, 4> Visited;1050  SmallVector<const Value *, 4> Worklist;1051  Worklist.push_back(CI);1052 1053  while (!Worklist.empty()) {1054    const Value *V = Worklist.pop_back_val();1055    if (!Visited.insert(V).second)1056      continue;1057 1058    for (const Use &VUse : V->uses()) {1059      const User *U = VUse.getUser();1060      if (isa<LoadInst>(U) || isa<CmpInst>(U))1061        continue; // Fine, ignore.1062 1063      if (auto *SI = dyn_cast<StoreInst>(U)) {1064        if (SI->getValueOperand() == V &&1065            SI->getPointerOperand()->stripPointerCasts() != GV)1066          return false; // Storing the pointer not into GV... bad.1067        continue; // Otherwise, storing through it, or storing into GV... fine.1068      }1069 1070      if (auto *GEPI = dyn_cast<GetElementPtrInst>(U)) {1071        Worklist.push_back(GEPI);1072        continue;1073      }1074 1075      return false;1076    }1077  }1078 1079  return true;1080}1081 1082/// If we have a global that is only initialized with a fixed size allocation1083/// try to transform the program to use global memory instead of heap1084/// allocated memory. This eliminates dynamic allocation, avoids an indirection1085/// accessing the data, and exposes the resultant global to further GlobalOpt.1086static bool tryToOptimizeStoreOfAllocationToGlobal(GlobalVariable *GV,1087                                                   CallInst *CI,1088                                                   const DataLayout &DL,1089                                                   TargetLibraryInfo *TLI) {1090  if (!isRemovableAlloc(CI, TLI))1091    // Must be able to remove the call when we get done..1092    return false;1093 1094  Type *Int8Ty = Type::getInt8Ty(CI->getFunction()->getContext());1095  Constant *InitVal = getInitialValueOfAllocation(CI, TLI, Int8Ty);1096  if (!InitVal)1097    // Must be able to emit a memset for initialization1098    return false;1099 1100  uint64_t AllocSize;1101  if (!getObjectSize(CI, AllocSize, DL, TLI, ObjectSizeOpts()))1102    return false;1103 1104  // Restrict this transformation to only working on small allocations1105  // (2048 bytes currently), as we don't want to introduce a 16M global or1106  // something.1107  if (AllocSize >= 2048)1108    return false;1109 1110  // We can't optimize this global unless all uses of it are *known* to be1111  // of the malloc value, not of the null initializer value (consider a use1112  // that compares the global's value against zero to see if the malloc has1113  // been reached).  To do this, we check to see if all uses of the global1114  // would trap if the global were null: this proves that they must all1115  // happen after the malloc.1116  if (!allUsesOfLoadedValueWillTrapIfNull(GV))1117    return false;1118 1119  // We can't optimize this if the malloc itself is used in a complex way,1120  // for example, being stored into multiple globals.  This allows the1121  // malloc to be stored into the specified global, loaded, gep, icmp'd.1122  // These are all things we could transform to using the global for.1123  if (!valueIsOnlyUsedLocallyOrStoredToOneGlobal(CI, GV))1124    return false;1125 1126  OptimizeGlobalAddressOfAllocation(GV, CI, AllocSize, InitVal, DL, TLI);1127  return true;1128}1129 1130// Try to optimize globals based on the knowledge that only one value (besides1131// its initializer) is ever stored to the global.1132static bool1133optimizeOnceStoredGlobal(GlobalVariable *GV, Value *StoredOnceVal,1134                         const DataLayout &DL,1135                         function_ref<TargetLibraryInfo &(Function &)> GetTLI) {1136  // If we are dealing with a pointer global that is initialized to null and1137  // only has one (non-null) value stored into it, then we can optimize any1138  // users of the loaded value (often calls and loads) that would trap if the1139  // value was null.1140  if (GV->getInitializer()->getType()->isPointerTy() &&1141      GV->getInitializer()->isNullValue() &&1142      StoredOnceVal->getType()->isPointerTy() &&1143      !NullPointerIsDefined(1144          nullptr /* F */,1145          GV->getInitializer()->getType()->getPointerAddressSpace())) {1146    if (Constant *SOVC = dyn_cast<Constant>(StoredOnceVal)) {1147      // Optimize away any trapping uses of the loaded value.1148      if (OptimizeAwayTrappingUsesOfLoads(GV, SOVC, DL, GetTLI))1149        return true;1150    } else if (isAllocationFn(StoredOnceVal, GetTLI)) {1151      if (auto *CI = dyn_cast<CallInst>(StoredOnceVal)) {1152        auto *TLI = &GetTLI(*CI->getFunction());1153        if (tryToOptimizeStoreOfAllocationToGlobal(GV, CI, DL, TLI))1154          return true;1155      }1156    }1157  }1158 1159  return false;1160}1161 1162/// At this point, we have learned that the only two values ever stored into GV1163/// are its initializer and OtherVal.  See if we can shrink the global into a1164/// boolean and select between the two values whenever it is used.  This exposes1165/// the values to other scalar optimizations.1166static bool TryToShrinkGlobalToBoolean(GlobalVariable *GV, Constant *OtherVal) {1167  Type *GVElType = GV->getValueType();1168 1169  // If GVElType is already i1, it is already shrunk.  If the type of the GV is1170  // an FP value, pointer or vector, don't do this optimization because a select1171  // between them is very expensive and unlikely to lead to later1172  // simplification.  In these cases, we typically end up with "cond ? v1 : v2"1173  // where v1 and v2 both require constant pool loads, a big loss.1174  if (GVElType == Type::getInt1Ty(GV->getContext()) ||1175      GVElType->isFloatingPointTy() ||1176      GVElType->isPointerTy() || GVElType->isVectorTy())1177    return false;1178 1179  // Walk the use list of the global seeing if all the uses are load or store.1180  // If there is anything else, bail out.1181  for (User *U : GV->users()) {1182    if (!isa<LoadInst>(U) && !isa<StoreInst>(U))1183      return false;1184    if (getLoadStoreType(U) != GVElType)1185      return false;1186  }1187 1188  LLVM_DEBUG(dbgs() << "   *** SHRINKING TO BOOL: " << *GV << "\n");1189 1190  // Create the new global, initializing it to false.1191  GlobalVariable *NewGV = new GlobalVariable(Type::getInt1Ty(GV->getContext()),1192                                             false,1193                                             GlobalValue::InternalLinkage,1194                                        ConstantInt::getFalse(GV->getContext()),1195                                             GV->getName()+".b",1196                                             GV->getThreadLocalMode(),1197                                             GV->getType()->getAddressSpace());1198  NewGV->copyAttributesFrom(GV);1199  GV->getParent()->insertGlobalVariable(GV->getIterator(), NewGV);1200 1201  Constant *InitVal = GV->getInitializer();1202  assert(InitVal->getType() != Type::getInt1Ty(GV->getContext()) &&1203         "No reason to shrink to bool!");1204 1205  SmallVector<DIGlobalVariableExpression *, 1> GVs;1206  GV->getDebugInfo(GVs);1207 1208  // If initialized to zero and storing one into the global, we can use a cast1209  // instead of a select to synthesize the desired value.1210  bool IsOneZero = false;1211  bool EmitOneOrZero = true;1212  auto *CI = dyn_cast<ConstantInt>(OtherVal);1213  if (CI && CI->getValue().getActiveBits() <= 64) {1214    IsOneZero = InitVal->isNullValue() && CI->isOne();1215 1216    auto *CIInit = dyn_cast<ConstantInt>(GV->getInitializer());1217    if (CIInit && CIInit->getValue().getActiveBits() <= 64) {1218      uint64_t ValInit = CIInit->getZExtValue();1219      uint64_t ValOther = CI->getZExtValue();1220      uint64_t ValMinus = ValOther - ValInit;1221 1222      for(auto *GVe : GVs){1223        DIGlobalVariable *DGV = GVe->getVariable();1224        DIExpression *E = GVe->getExpression();1225        const DataLayout &DL = GV->getDataLayout();1226        unsigned SizeInOctets =1227            DL.getTypeAllocSizeInBits(NewGV->getValueType()) / 8;1228 1229        // It is expected that the address of global optimized variable is on1230        // top of the stack. After optimization, value of that variable will1231        // be ether 0 for initial value or 1 for other value. The following1232        // expression should return constant integer value depending on the1233        // value at global object address:1234        // val * (ValOther - ValInit) + ValInit:1235        // DW_OP_deref DW_OP_constu <ValMinus>1236        // DW_OP_mul DW_OP_constu <ValInit> DW_OP_plus DW_OP_stack_value1237        SmallVector<uint64_t, 12> Ops = {1238            dwarf::DW_OP_deref_size, SizeInOctets,1239            dwarf::DW_OP_constu, ValMinus,1240            dwarf::DW_OP_mul, dwarf::DW_OP_constu, ValInit,1241            dwarf::DW_OP_plus};1242        bool WithStackValue = true;1243        E = DIExpression::prependOpcodes(E, Ops, WithStackValue);1244        DIGlobalVariableExpression *DGVE =1245          DIGlobalVariableExpression::get(NewGV->getContext(), DGV, E);1246        NewGV->addDebugInfo(DGVE);1247     }1248     EmitOneOrZero = false;1249    }1250  }1251 1252  if (EmitOneOrZero) {1253     // FIXME: This will only emit address for debugger on which will1254     // be written only 0 or 1.1255     for(auto *GV : GVs)1256       NewGV->addDebugInfo(GV);1257   }1258 1259  while (!GV->use_empty()) {1260    Instruction *UI = cast<Instruction>(GV->user_back());1261    if (StoreInst *SI = dyn_cast<StoreInst>(UI)) {1262      // Change the store into a boolean store.1263      bool StoringOther = SI->getOperand(0) == OtherVal;1264      // Only do this if we weren't storing a loaded value.1265      Value *StoreVal;1266      if (StoringOther || SI->getOperand(0) == InitVal) {1267        StoreVal = ConstantInt::get(Type::getInt1Ty(GV->getContext()),1268                                    StoringOther);1269      } else {1270        // Otherwise, we are storing a previously loaded copy.  To do this,1271        // change the copy from copying the original value to just copying the1272        // bool.1273        Instruction *StoredVal = cast<Instruction>(SI->getOperand(0));1274 1275        // If we've already replaced the input, StoredVal will be a cast or1276        // select instruction.  If not, it will be a load of the original1277        // global.1278        if (LoadInst *LI = dyn_cast<LoadInst>(StoredVal)) {1279          assert(LI->getOperand(0) == GV && "Not a copy!");1280          // Insert a new load, to preserve the saved value.1281          StoreVal =1282              new LoadInst(NewGV->getValueType(), NewGV, LI->getName() + ".b",1283                           false, Align(1), LI->getOrdering(),1284                           LI->getSyncScopeID(), LI->getIterator());1285          cast<LoadInst>(StoreVal)->setDebugLoc(LI->getDebugLoc());1286        } else {1287          assert((isa<CastInst>(StoredVal) || isa<SelectInst>(StoredVal)) &&1288                 "This is not a form that we understand!");1289          StoreVal = StoredVal->getOperand(0);1290          assert(isa<LoadInst>(StoreVal) && "Not a load of NewGV!");1291        }1292      }1293      StoreInst *NSI =1294          new StoreInst(StoreVal, NewGV, false, Align(1), SI->getOrdering(),1295                        SI->getSyncScopeID(), SI->getIterator());1296      NSI->setDebugLoc(SI->getDebugLoc());1297    } else {1298      // Change the load into a load of bool then a select.1299      LoadInst *LI = cast<LoadInst>(UI);1300      LoadInst *NLI = new LoadInst(1301          NewGV->getValueType(), NewGV, LI->getName() + ".b", false, Align(1),1302          LI->getOrdering(), LI->getSyncScopeID(), LI->getIterator());1303      Instruction *NSI;1304      if (IsOneZero)1305        NSI = new ZExtInst(NLI, LI->getType(), "", LI->getIterator());1306      else1307        NSI = SelectInst::Create(NLI, OtherVal, InitVal, "", LI->getIterator());1308      NSI->takeName(LI);1309      // Since LI is split into two instructions, NLI and NSI both inherit the1310      // same DebugLoc1311      NLI->setDebugLoc(LI->getDebugLoc());1312      NSI->setDebugLoc(LI->getDebugLoc());1313      LI->replaceAllUsesWith(NSI);1314    }1315    UI->eraseFromParent();1316  }1317 1318  // Retain the name of the old global variable. People who are debugging their1319  // programs may expect these variables to be named the same.1320  NewGV->takeName(GV);1321  GV->eraseFromParent();1322  return true;1323}1324 1325static bool1326deleteIfDead(GlobalValue &GV,1327             SmallPtrSetImpl<const Comdat *> &NotDiscardableComdats,1328             function_ref<void(Function &)> DeleteFnCallback = nullptr) {1329  GV.removeDeadConstantUsers();1330 1331  if (!GV.isDiscardableIfUnused() && !GV.isDeclaration())1332    return false;1333 1334  if (const Comdat *C = GV.getComdat())1335    if (!GV.hasLocalLinkage() && NotDiscardableComdats.count(C))1336      return false;1337 1338  bool Dead;1339  if (auto *F = dyn_cast<Function>(&GV))1340    Dead = (F->isDeclaration() && F->use_empty()) || F->isDefTriviallyDead();1341  else1342    Dead = GV.use_empty();1343  if (!Dead)1344    return false;1345 1346  LLVM_DEBUG(dbgs() << "GLOBAL DEAD: " << GV << "\n");1347  if (auto *F = dyn_cast<Function>(&GV)) {1348    if (DeleteFnCallback)1349      DeleteFnCallback(*F);1350  }1351  ReplaceableMetadataImpl::SalvageDebugInfo(GV);1352  GV.eraseFromParent();1353  ++NumDeleted;1354  return true;1355}1356 1357static bool isPointerValueDeadOnEntryToFunction(1358    const Function *F, GlobalValue *GV,1359    function_ref<DominatorTree &(Function &)> LookupDomTree) {1360  // Find all uses of GV. We expect them all to be in F, and if we can't1361  // identify any of the uses we bail out.1362  //1363  // On each of these uses, identify if the memory that GV points to is1364  // used/required/live at the start of the function. If it is not, for example1365  // if the first thing the function does is store to the GV, the GV can1366  // possibly be demoted.1367  //1368  // We don't do an exhaustive search for memory operations - simply look1369  // through bitcasts as they're quite common and benign.1370  const DataLayout &DL = GV->getDataLayout();1371  SmallVector<LoadInst *, 4> Loads;1372  SmallVector<StoreInst *, 4> Stores;1373  for (auto *U : GV->users()) {1374    Instruction *I = dyn_cast<Instruction>(U);1375    if (!I)1376      return false;1377    assert(I->getParent()->getParent() == F);1378 1379    if (auto *LI = dyn_cast<LoadInst>(I))1380      Loads.push_back(LI);1381    else if (auto *SI = dyn_cast<StoreInst>(I))1382      Stores.push_back(SI);1383    else1384      return false;1385  }1386 1387  // We have identified all uses of GV into loads and stores. Now check if all1388  // of them are known not to depend on the value of the global at the function1389  // entry point. We do this by ensuring that every load is dominated by at1390  // least one store.1391  auto &DT = LookupDomTree(*const_cast<Function *>(F));1392 1393  // The below check is quadratic. Check we're not going to do too many tests.1394  // FIXME: Even though this will always have worst-case quadratic time, we1395  // could put effort into minimizing the average time by putting stores that1396  // have been shown to dominate at least one load at the beginning of the1397  // Stores array, making subsequent dominance checks more likely to succeed1398  // early.1399  //1400  // The threshold here is fairly large because global->local demotion is a1401  // very powerful optimization should it fire.1402  const unsigned Threshold = 100;1403  if (Loads.size() * Stores.size() > Threshold)1404    return false;1405 1406  for (auto *L : Loads) {1407    auto *LTy = L->getType();1408    if (none_of(Stores, [&](const StoreInst *S) {1409          auto *STy = S->getValueOperand()->getType();1410          // The load is only dominated by the store if DomTree says so1411          // and the number of bits loaded in L is less than or equal to1412          // the number of bits stored in S.1413          return DT.dominates(S, L) &&1414                 DL.getTypeStoreSize(LTy).getFixedValue() <=1415                     DL.getTypeStoreSize(STy).getFixedValue();1416        }))1417      return false;1418  }1419  // All loads have known dependences inside F, so the global can be localized.1420  return true;1421}1422 1423// For a global variable with one store, if the store dominates any loads,1424// those loads will always load the stored value (as opposed to the1425// initializer), even in the presence of recursion.1426static bool forwardStoredOnceStore(1427    GlobalVariable *GV, const StoreInst *StoredOnceStore,1428    function_ref<DominatorTree &(Function &)> LookupDomTree) {1429  const Value *StoredOnceValue = StoredOnceStore->getValueOperand();1430  // We can do this optimization for non-constants in nosync + norecurse1431  // functions, but globals used in exactly one norecurse functions are already1432  // promoted to an alloca.1433  if (!isa<Constant>(StoredOnceValue))1434    return false;1435  const Function *F = StoredOnceStore->getFunction();1436  SmallVector<LoadInst *> Loads;1437  for (User *U : GV->users()) {1438    if (auto *LI = dyn_cast<LoadInst>(U)) {1439      if (LI->getFunction() == F &&1440          LI->getType() == StoredOnceValue->getType() && LI->isSimple())1441        Loads.push_back(LI);1442    }1443  }1444  // Only compute DT if we have any loads to examine.1445  bool MadeChange = false;1446  if (!Loads.empty()) {1447    auto &DT = LookupDomTree(*const_cast<Function *>(F));1448    for (auto *LI : Loads) {1449      if (DT.dominates(StoredOnceStore, LI)) {1450        LI->replaceAllUsesWith(const_cast<Value *>(StoredOnceValue));1451        LI->eraseFromParent();1452        MadeChange = true;1453      }1454    }1455  }1456  return MadeChange;1457}1458 1459/// Analyze the specified global variable and optimize1460/// it if possible.  If we make a change, return true.1461static bool1462processInternalGlobal(GlobalVariable *GV, const GlobalStatus &GS,1463                      function_ref<TargetTransformInfo &(Function &)> GetTTI,1464                      function_ref<TargetLibraryInfo &(Function &)> GetTLI,1465                      function_ref<DominatorTree &(Function &)> LookupDomTree) {1466  auto &DL = GV->getDataLayout();1467  // If this is a first class global and has only one accessing function and1468  // this function is non-recursive, we replace the global with a local alloca1469  // in this function.1470  //1471  // NOTE: It doesn't make sense to promote non-single-value types since we1472  // are just replacing static memory to stack memory.1473  //1474  // If the global is in different address space, don't bring it to stack.1475  if (!GS.HasMultipleAccessingFunctions &&1476      GS.AccessingFunction &&1477      GV->getValueType()->isSingleValueType() &&1478      GV->getType()->getAddressSpace() == DL.getAllocaAddrSpace() &&1479      !GV->isExternallyInitialized() &&1480      GS.AccessingFunction->doesNotRecurse() &&1481      isPointerValueDeadOnEntryToFunction(GS.AccessingFunction, GV,1482                                          LookupDomTree)) {1483    const DataLayout &DL = GV->getDataLayout();1484 1485    LLVM_DEBUG(dbgs() << "LOCALIZING GLOBAL: " << *GV << "\n");1486    BasicBlock::iterator FirstI =1487        GS.AccessingFunction->getEntryBlock().begin().getNonConst();1488    Type *ElemTy = GV->getValueType();1489    // FIXME: Pass Global's alignment when globals have alignment1490    AllocaInst *Alloca = new AllocaInst(ElemTy, DL.getAllocaAddrSpace(),1491                                        nullptr, GV->getName(), FirstI);1492    Alloca->setDebugLoc(DebugLoc::getCompilerGenerated());1493    if (!isa<UndefValue>(GV->getInitializer())) {1494      auto *SI = new StoreInst(GV->getInitializer(), Alloca, FirstI);1495      // FIXME: We're localizing a global and creating a store instruction for1496      // the initial value of that global. Could we logically use the global1497      // variable's (if one exists) line for this?1498      SI->setDebugLoc(DebugLoc::getCompilerGenerated());1499    }1500 1501    GV->replaceAllUsesWith(Alloca);1502    GV->eraseFromParent();1503    ++NumLocalized;1504    return true;1505  }1506 1507  bool Changed = false;1508 1509  // If the global is never loaded (but may be stored to), it is dead.1510  // Delete it now.1511  if (!GS.IsLoaded) {1512    LLVM_DEBUG(dbgs() << "GLOBAL NEVER LOADED: " << *GV << "\n");1513 1514    if (isLeakCheckerRoot(GV)) {1515      // Delete any constant stores to the global.1516      Changed = CleanupPointerRootUsers(GV, GetTLI);1517    } else {1518      // Delete any stores we can find to the global.  We may not be able to1519      // make it completely dead though.1520      Changed = CleanupConstantGlobalUsers(GV, DL);1521    }1522 1523    // If the global is dead now, delete it.1524    if (GV->use_empty()) {1525      GV->eraseFromParent();1526      ++NumDeleted;1527      Changed = true;1528    }1529    return Changed;1530 1531  }1532  if (GS.StoredType <= GlobalStatus::InitializerStored) {1533    LLVM_DEBUG(dbgs() << "MARKING CONSTANT: " << *GV << "\n");1534 1535    // Don't actually mark a global constant if it's atomic because atomic loads1536    // are implemented by a trivial cmpxchg in some edge-cases and that usually1537    // requires write access to the variable even if it's not actually changed.1538    if (GS.Ordering == AtomicOrdering::NotAtomic) {1539      assert(!GV->isConstant() && "Expected a non-constant global");1540      GV->setConstant(true);1541      Changed = true;1542    }1543 1544    // Clean up any obviously simplifiable users now.1545    Changed |= CleanupConstantGlobalUsers(GV, DL);1546 1547    // If the global is dead now, just nuke it.1548    if (GV->use_empty()) {1549      LLVM_DEBUG(dbgs() << "   *** Marking constant allowed us to simplify "1550                        << "all users and delete global!\n");1551      GV->eraseFromParent();1552      ++NumDeleted;1553      return true;1554    }1555 1556    // Fall through to the next check; see if we can optimize further.1557    ++NumMarked;1558  }1559  if (!GV->getInitializer()->getType()->isSingleValueType()) {1560    const DataLayout &DL = GV->getDataLayout();1561    if (SRAGlobal(GV, DL))1562      return true;1563  }1564  Value *StoredOnceValue = GS.getStoredOnceValue();1565  if (GS.StoredType == GlobalStatus::StoredOnce && StoredOnceValue) {1566    Function &StoreFn =1567        const_cast<Function &>(*GS.StoredOnceStore->getFunction());1568    bool CanHaveNonUndefGlobalInitializer =1569        GetTTI(StoreFn).canHaveNonUndefGlobalInitializerInAddressSpace(1570            GV->getType()->getAddressSpace());1571    // If the initial value for the global was an undef value, and if only1572    // one other value was stored into it, we can just change the1573    // initializer to be the stored value, then delete all stores to the1574    // global.  This allows us to mark it constant.1575    // This is restricted to address spaces that allow globals to have1576    // initializers. NVPTX, for example, does not support initializers for1577    // shared memory (AS 3).1578    auto *SOVConstant = dyn_cast<Constant>(StoredOnceValue);1579    if (SOVConstant && isa<UndefValue>(GV->getInitializer()) &&1580        DL.getTypeAllocSize(SOVConstant->getType()) ==1581            DL.getTypeAllocSize(GV->getValueType()) &&1582        CanHaveNonUndefGlobalInitializer) {1583      if (SOVConstant->getType() == GV->getValueType()) {1584        // Change the initializer in place.1585        GV->setInitializer(SOVConstant);1586      } else {1587        // Create a new global with adjusted type.1588        auto *NGV = new GlobalVariable(1589            *GV->getParent(), SOVConstant->getType(), GV->isConstant(),1590            GV->getLinkage(), SOVConstant, "", GV, GV->getThreadLocalMode(),1591            GV->getAddressSpace());1592        NGV->takeName(GV);1593        NGV->copyAttributesFrom(GV);1594        GV->replaceAllUsesWith(NGV);1595        GV->eraseFromParent();1596        GV = NGV;1597      }1598 1599      // Clean up any obviously simplifiable users now.1600      CleanupConstantGlobalUsers(GV, DL);1601 1602      if (GV->use_empty()) {1603        LLVM_DEBUG(dbgs() << "   *** Substituting initializer allowed us to "1604                          << "simplify all users and delete global!\n");1605        GV->eraseFromParent();1606        ++NumDeleted;1607      }1608      ++NumSubstitute;1609      return true;1610    }1611 1612    // Try to optimize globals based on the knowledge that only one value1613    // (besides its initializer) is ever stored to the global.1614    if (optimizeOnceStoredGlobal(GV, StoredOnceValue, DL, GetTLI))1615      return true;1616 1617    // Try to forward the store to any loads. If we have more than one store, we1618    // may have a store of the initializer between StoredOnceStore and a load.1619    if (GS.NumStores == 1)1620      if (forwardStoredOnceStore(GV, GS.StoredOnceStore, LookupDomTree))1621        return true;1622 1623    // Otherwise, if the global was not a boolean, we can shrink it to be a1624    // boolean. Skip this optimization for AS that doesn't allow an initializer.1625    if (SOVConstant && GS.Ordering == AtomicOrdering::NotAtomic &&1626        (!isa<UndefValue>(GV->getInitializer()) ||1627         CanHaveNonUndefGlobalInitializer)) {1628      if (TryToShrinkGlobalToBoolean(GV, SOVConstant)) {1629        ++NumShrunkToBool;1630        return true;1631      }1632    }1633  }1634 1635  return Changed;1636}1637 1638/// Analyze the specified global variable and optimize it if possible.  If we1639/// make a change, return true.1640static bool1641processGlobal(GlobalValue &GV,1642              function_ref<TargetTransformInfo &(Function &)> GetTTI,1643              function_ref<TargetLibraryInfo &(Function &)> GetTLI,1644              function_ref<DominatorTree &(Function &)> LookupDomTree) {1645  if (GV.getName().starts_with("llvm."))1646    return false;1647 1648  GlobalStatus GS;1649 1650  if (GlobalStatus::analyzeGlobal(&GV, GS))1651    return false;1652 1653  bool Changed = false;1654  if (!GS.IsCompared && !GV.hasGlobalUnnamedAddr()) {1655    auto NewUnnamedAddr = GV.hasLocalLinkage() ? GlobalValue::UnnamedAddr::Global1656                                               : GlobalValue::UnnamedAddr::Local;1657    if (NewUnnamedAddr != GV.getUnnamedAddr()) {1658      GV.setUnnamedAddr(NewUnnamedAddr);1659      NumUnnamed++;1660      Changed = true;1661    }1662  }1663 1664  // Do more involved optimizations if the global is internal.1665  if (!GV.hasLocalLinkage())1666    return Changed;1667 1668  auto *GVar = dyn_cast<GlobalVariable>(&GV);1669  if (!GVar)1670    return Changed;1671 1672  if (GVar->isConstant() || !GVar->hasInitializer())1673    return Changed;1674 1675  return processInternalGlobal(GVar, GS, GetTTI, GetTLI, LookupDomTree) ||1676         Changed;1677}1678 1679/// Walk all of the direct calls of the specified function, changing them to1680/// FastCC.1681static void ChangeCalleesToFastCall(Function *F) {1682  for (User *U : F->users())1683    if (auto *Call = dyn_cast<CallBase>(U))1684      if (Call->getCalledOperand() == F)1685        Call->setCallingConv(CallingConv::Fast);1686}1687 1688static AttributeList StripAttr(LLVMContext &C, AttributeList Attrs,1689                               Attribute::AttrKind A) {1690  unsigned AttrIndex;1691  if (Attrs.hasAttrSomewhere(A, &AttrIndex))1692    return Attrs.removeAttributeAtIndex(C, AttrIndex, A);1693  return Attrs;1694}1695 1696static void RemoveAttribute(Function *F, Attribute::AttrKind A) {1697  F->setAttributes(StripAttr(F->getContext(), F->getAttributes(), A));1698  for (User *U : F->users()) {1699    CallBase *CB = cast<CallBase>(U);1700    CB->setAttributes(StripAttr(F->getContext(), CB->getAttributes(), A));1701  }1702}1703 1704/// Return true if this is a calling convention that we'd like to change.  The1705/// idea here is that we don't want to mess with the convention if the user1706/// explicitly requested something with performance implications like coldcc,1707/// GHC, or anyregcc.1708static bool hasChangeableCCImpl(Function *F) {1709  CallingConv::ID CC = F->getCallingConv();1710 1711  // FIXME: Is it worth transforming x86_stdcallcc and x86_fastcallcc?1712  if (CC != CallingConv::C && CC != CallingConv::X86_ThisCall)1713    return false;1714 1715  if (F->isVarArg())1716    return false;1717 1718  // FIXME: Change CC for the whole chain of musttail calls when possible.1719  //1720  // Can't change CC of the function that either has musttail calls, or is a1721  // musttail callee itself1722  for (User *U : F->users()) {1723    CallInst* CI = dyn_cast<CallInst>(U);1724    if (!CI)1725      continue;1726 1727    if (CI->isMustTailCall())1728      return false;1729  }1730 1731  for (BasicBlock &BB : *F)1732    if (BB.getTerminatingMustTailCall())1733      return false;1734 1735  return !F->hasAddressTaken();1736}1737 1738using ChangeableCCCacheTy = SmallDenseMap<Function *, bool, 8>;1739static bool hasChangeableCC(Function *F,1740                            ChangeableCCCacheTy &ChangeableCCCache) {1741  auto Res = ChangeableCCCache.try_emplace(F, false);1742  if (Res.second)1743    Res.first->second = hasChangeableCCImpl(F);1744  return Res.first->second;1745}1746 1747/// Return true if the block containing the call site has a BlockFrequency of1748/// less than ColdCCRelFreq% of the entry block.1749static bool isColdCallSite(CallBase &CB, BlockFrequencyInfo &CallerBFI) {1750  const BranchProbability ColdProb(ColdCCRelFreq, 100);1751  auto *CallSiteBB = CB.getParent();1752  auto CallSiteFreq = CallerBFI.getBlockFreq(CallSiteBB);1753  auto CallerEntryFreq =1754      CallerBFI.getBlockFreq(&(CB.getCaller()->getEntryBlock()));1755  return CallSiteFreq < CallerEntryFreq * ColdProb;1756}1757 1758// This function checks if the input function F is cold at all call sites. It1759// also looks each call site's containing function, returning false if the1760// caller function contains other non cold calls. The input vector AllCallsCold1761// contains a list of functions that only have call sites in cold blocks.1762static bool1763isValidCandidateForColdCC(Function &F,1764                          function_ref<BlockFrequencyInfo &(Function &)> GetBFI,1765                          const std::vector<Function *> &AllCallsCold) {1766 1767  if (F.user_empty())1768    return false;1769 1770  for (User *U : F.users()) {1771    CallBase *CB = dyn_cast<CallBase>(U);1772    if (!CB || CB->getCalledOperand() != &F)1773      continue;1774    Function *CallerFunc = CB->getParent()->getParent();1775    BlockFrequencyInfo &CallerBFI = GetBFI(*CallerFunc);1776    if (!isColdCallSite(*CB, CallerBFI))1777      return false;1778    if (!llvm::is_contained(AllCallsCold, CallerFunc))1779      return false;1780  }1781  return true;1782}1783 1784static void changeCallSitesToColdCC(Function *F) {1785  for (User *U : F->users())1786    if (auto *Call = dyn_cast<CallBase>(U))1787      if (Call->getCalledOperand() == F)1788        Call->setCallingConv(CallingConv::Cold);1789}1790 1791// This function iterates over all the call instructions in the input Function1792// and checks that all call sites are in cold blocks and are allowed to use the1793// coldcc calling convention.1794static bool1795hasOnlyColdCalls(Function &F,1796                 function_ref<BlockFrequencyInfo &(Function &)> GetBFI,1797                 ChangeableCCCacheTy &ChangeableCCCache) {1798  for (BasicBlock &BB : F) {1799    for (Instruction &I : BB) {1800      if (CallInst *CI = dyn_cast<CallInst>(&I)) {1801        // Skip over isline asm instructions since they aren't function calls.1802        if (CI->isInlineAsm())1803          continue;1804        Function *CalledFn = CI->getCalledFunction();1805        if (!CalledFn)1806          return false;1807        // Skip over intrinsics since they won't remain as function calls.1808        // Important to do this check before the linkage check below so we1809        // won't bail out on debug intrinsics, possibly making the generated1810        // code dependent on the presence of debug info.1811        if (CalledFn->getIntrinsicID() != Intrinsic::not_intrinsic)1812          continue;1813        if (!CalledFn->hasLocalLinkage())1814          return false;1815        // Check if it's valid to use coldcc calling convention.1816        if (!hasChangeableCC(CalledFn, ChangeableCCCache))1817          return false;1818        BlockFrequencyInfo &CallerBFI = GetBFI(F);1819        if (!isColdCallSite(*CI, CallerBFI))1820          return false;1821      }1822    }1823  }1824  return true;1825}1826 1827static bool hasMustTailCallers(Function *F) {1828  for (User *U : F->users()) {1829    CallBase *CB = cast<CallBase>(U);1830    if (CB->isMustTailCall())1831      return true;1832  }1833  return false;1834}1835 1836static bool hasInvokeCallers(Function *F) {1837  for (User *U : F->users())1838    if (isa<InvokeInst>(U))1839      return true;1840  return false;1841}1842 1843static void RemovePreallocated(Function *F) {1844  RemoveAttribute(F, Attribute::Preallocated);1845 1846  auto *M = F->getParent();1847 1848  IRBuilder<> Builder(M->getContext());1849 1850  // Cannot modify users() while iterating over it, so make a copy.1851  SmallVector<User *, 4> PreallocatedCalls(F->users());1852  for (User *U : PreallocatedCalls) {1853    CallBase *CB = dyn_cast<CallBase>(U);1854    if (!CB)1855      continue;1856 1857    assert(1858        !CB->isMustTailCall() &&1859        "Shouldn't call RemotePreallocated() on a musttail preallocated call");1860    // Create copy of call without "preallocated" operand bundle.1861    SmallVector<OperandBundleDef, 1> OpBundles;1862    CB->getOperandBundlesAsDefs(OpBundles);1863    CallBase *PreallocatedSetup = nullptr;1864    for (auto *It = OpBundles.begin(); It != OpBundles.end(); ++It) {1865      if (It->getTag() == "preallocated") {1866        PreallocatedSetup = cast<CallBase>(*It->input_begin());1867        OpBundles.erase(It);1868        break;1869      }1870    }1871    assert(PreallocatedSetup && "Did not find preallocated bundle");1872    uint64_t ArgCount =1873        cast<ConstantInt>(PreallocatedSetup->getArgOperand(0))->getZExtValue();1874 1875    assert((isa<CallInst>(CB) || isa<InvokeInst>(CB)) &&1876           "Unknown indirect call type");1877    CallBase *NewCB = CallBase::Create(CB, OpBundles, CB->getIterator());1878    CB->replaceAllUsesWith(NewCB);1879    NewCB->takeName(CB);1880    CB->eraseFromParent();1881 1882    Builder.SetInsertPoint(PreallocatedSetup);1883    auto *StackSave = Builder.CreateStackSave();1884    Builder.SetInsertPoint(NewCB->getNextNode());1885    Builder.CreateStackRestore(StackSave);1886 1887    // Replace @llvm.call.preallocated.arg() with alloca.1888    // Cannot modify users() while iterating over it, so make a copy.1889    // @llvm.call.preallocated.arg() can be called with the same index multiple1890    // times. So for each @llvm.call.preallocated.arg(), we see if we have1891    // already created a Value* for the index, and if not, create an alloca and1892    // bitcast right after the @llvm.call.preallocated.setup() so that it1893    // dominates all uses.1894    SmallVector<Value *, 2> ArgAllocas(ArgCount);1895    SmallVector<User *, 2> PreallocatedArgs(PreallocatedSetup->users());1896    for (auto *User : PreallocatedArgs) {1897      auto *UseCall = cast<CallBase>(User);1898      assert(UseCall->getCalledFunction()->getIntrinsicID() ==1899                 Intrinsic::call_preallocated_arg &&1900             "preallocated token use was not a llvm.call.preallocated.arg");1901      uint64_t AllocArgIndex =1902          cast<ConstantInt>(UseCall->getArgOperand(1))->getZExtValue();1903      Value *AllocaReplacement = ArgAllocas[AllocArgIndex];1904      if (!AllocaReplacement) {1905        auto AddressSpace = UseCall->getType()->getPointerAddressSpace();1906        auto *ArgType =1907            UseCall->getFnAttr(Attribute::Preallocated).getValueAsType();1908        auto *InsertBefore = PreallocatedSetup->getNextNode();1909        Builder.SetInsertPoint(InsertBefore);1910        auto *Alloca =1911            Builder.CreateAlloca(ArgType, AddressSpace, nullptr, "paarg");1912        ArgAllocas[AllocArgIndex] = Alloca;1913        AllocaReplacement = Alloca;1914      }1915 1916      UseCall->replaceAllUsesWith(AllocaReplacement);1917      UseCall->eraseFromParent();1918    }1919    // Remove @llvm.call.preallocated.setup().1920    cast<Instruction>(PreallocatedSetup)->eraseFromParent();1921  }1922}1923 1924static bool1925OptimizeFunctions(Module &M,1926                  function_ref<TargetLibraryInfo &(Function &)> GetTLI,1927                  function_ref<TargetTransformInfo &(Function &)> GetTTI,1928                  function_ref<BlockFrequencyInfo &(Function &)> GetBFI,1929                  function_ref<DominatorTree &(Function &)> LookupDomTree,1930                  SmallPtrSetImpl<const Comdat *> &NotDiscardableComdats,1931                  function_ref<void(Function &F)> ChangedCFGCallback,1932                  function_ref<void(Function &F)> DeleteFnCallback) {1933 1934  bool Changed = false;1935 1936  ChangeableCCCacheTy ChangeableCCCache;1937  std::vector<Function *> AllCallsCold;1938  for (Function &F : llvm::make_early_inc_range(M))1939    if (hasOnlyColdCalls(F, GetBFI, ChangeableCCCache))1940      AllCallsCold.push_back(&F);1941 1942  // Optimize functions.1943  for (Function &F : llvm::make_early_inc_range(M)) {1944    // Don't perform global opt pass on naked functions; we don't want fast1945    // calling conventions for naked functions.1946    if (F.hasFnAttribute(Attribute::Naked))1947      continue;1948 1949    // Functions without names cannot be referenced outside this module.1950    if (!F.hasName() && !F.isDeclaration() && !F.hasLocalLinkage())1951      F.setLinkage(GlobalValue::InternalLinkage);1952 1953    if (deleteIfDead(F, NotDiscardableComdats, DeleteFnCallback)) {1954      Changed = true;1955      continue;1956    }1957 1958    // LLVM's definition of dominance allows instructions that are cyclic1959    // in unreachable blocks, e.g.:1960    // %pat = select i1 %condition, @global, i16* %pat1961    // because any instruction dominates an instruction in a block that's1962    // not reachable from entry.1963    // So, remove unreachable blocks from the function, because a) there's1964    // no point in analyzing them and b) GlobalOpt should otherwise grow1965    // some more complicated logic to break these cycles.1966    // Notify the analysis manager that we've modified the function's CFG.1967    if (!F.isDeclaration()) {1968      if (removeUnreachableBlocks(F)) {1969        Changed = true;1970        ChangedCFGCallback(F);1971      }1972    }1973 1974    Changed |= processGlobal(F, GetTTI, GetTLI, LookupDomTree);1975 1976    if (!F.hasLocalLinkage())1977      continue;1978 1979    // If we have an inalloca parameter that we can safely remove the1980    // inalloca attribute from, do so. This unlocks optimizations that1981    // wouldn't be safe in the presence of inalloca.1982    // FIXME: We should also hoist alloca affected by this to the entry1983    // block if possible.1984    if (F.getAttributes().hasAttrSomewhere(Attribute::InAlloca) &&1985        !F.hasAddressTaken() && !hasMustTailCallers(&F) && !F.isVarArg()) {1986      RemoveAttribute(&F, Attribute::InAlloca);1987      Changed = true;1988    }1989 1990    // FIXME: handle invokes1991    // FIXME: handle musttail1992    if (F.getAttributes().hasAttrSomewhere(Attribute::Preallocated)) {1993      if (!F.hasAddressTaken() && !hasMustTailCallers(&F) &&1994          !hasInvokeCallers(&F)) {1995        RemovePreallocated(&F);1996        Changed = true;1997      }1998      continue;1999    }2000 2001    if (hasChangeableCC(&F, ChangeableCCCache)) {2002      NumInternalFunc++;2003      TargetTransformInfo &TTI = GetTTI(F);2004      // Change the calling convention to coldcc if either stress testing is2005      // enabled or the target would like to use coldcc on functions which are2006      // cold at all call sites and the callers contain no other non coldcc2007      // calls.2008      if (EnableColdCCStressTest ||2009          (TTI.useColdCCForColdCall(F) &&2010           isValidCandidateForColdCC(F, GetBFI, AllCallsCold))) {2011        ChangeableCCCache.erase(&F);2012        F.setCallingConv(CallingConv::Cold);2013        changeCallSitesToColdCC(&F);2014        Changed = true;2015        NumColdCC++;2016      }2017    }2018 2019    if (hasChangeableCC(&F, ChangeableCCCache)) {2020      // If this function has a calling convention worth changing, is not a2021      // varargs function, is only called directly, and is supported by the2022      // target, promote it to use the Fast calling convention.2023      TargetTransformInfo &TTI = GetTTI(F);2024      if (TTI.useFastCCForInternalCall(F)) {2025        F.setCallingConv(CallingConv::Fast);2026        ChangeCalleesToFastCall(&F);2027        ++NumFastCallFns;2028        Changed = true;2029      }2030    }2031 2032    if (F.getAttributes().hasAttrSomewhere(Attribute::Nest) &&2033        !F.hasAddressTaken()) {2034      // The function is not used by a trampoline intrinsic, so it is safe2035      // to remove the 'nest' attribute.2036      RemoveAttribute(&F, Attribute::Nest);2037      ++NumNestRemoved;2038      Changed = true;2039    }2040  }2041  return Changed;2042}2043 2044static bool2045OptimizeGlobalVars(Module &M,2046                   function_ref<TargetTransformInfo &(Function &)> GetTTI,2047                   function_ref<TargetLibraryInfo &(Function &)> GetTLI,2048                   function_ref<DominatorTree &(Function &)> LookupDomTree,2049                   SmallPtrSetImpl<const Comdat *> &NotDiscardableComdats) {2050  bool Changed = false;2051 2052  for (GlobalVariable &GV : llvm::make_early_inc_range(M.globals())) {2053    // Global variables without names cannot be referenced outside this module.2054    if (!GV.hasName() && !GV.isDeclaration() && !GV.hasLocalLinkage())2055      GV.setLinkage(GlobalValue::InternalLinkage);2056    // Simplify the initializer.2057    if (GV.hasInitializer())2058      if (auto *C = dyn_cast<Constant>(GV.getInitializer())) {2059        auto &DL = M.getDataLayout();2060        // TLI is not used in the case of a Constant, so use default nullptr2061        // for that optional parameter, since we don't have a Function to2062        // provide GetTLI anyway.2063        Constant *New = ConstantFoldConstant(C, DL, /*TLI*/ nullptr);2064        if (New != C)2065          GV.setInitializer(New);2066      }2067 2068    if (deleteIfDead(GV, NotDiscardableComdats)) {2069      Changed = true;2070      continue;2071    }2072 2073    Changed |= processGlobal(GV, GetTTI, GetTLI, LookupDomTree);2074  }2075  return Changed;2076}2077 2078/// Evaluate static constructors in the function, if we can.  Return true if we2079/// can, false otherwise.2080static bool EvaluateStaticConstructor(Function *F, const DataLayout &DL,2081                                      TargetLibraryInfo *TLI) {2082  // Skip external functions.2083  if (F->isDeclaration())2084    return false;2085  // Call the function.2086  Evaluator Eval(DL, TLI);2087  Constant *RetValDummy;2088  bool EvalSuccess = Eval.EvaluateFunction(F, RetValDummy,2089                                           SmallVector<Constant*, 0>());2090 2091  if (EvalSuccess) {2092    ++NumCtorsEvaluated;2093 2094    // We succeeded at evaluation: commit the result.2095    auto NewInitializers = Eval.getMutatedInitializers();2096    LLVM_DEBUG(dbgs() << "FULLY EVALUATED GLOBAL CTOR FUNCTION '"2097                      << F->getName() << "' to " << NewInitializers.size()2098                      << " stores.\n");2099    for (const auto &Pair : NewInitializers)2100      Pair.first->setInitializer(Pair.second);2101    for (GlobalVariable *GV : Eval.getInvariants())2102      GV->setConstant(true);2103  }2104 2105  return EvalSuccess;2106}2107 2108static int compareNames(Constant *const *A, Constant *const *B) {2109  Value *AStripped = (*A)->stripPointerCasts();2110  Value *BStripped = (*B)->stripPointerCasts();2111  return AStripped->getName().compare(BStripped->getName());2112}2113 2114static void setUsedInitializer(GlobalVariable &V,2115                               const SmallPtrSetImpl<GlobalValue *> &Init) {2116  if (Init.empty()) {2117    V.eraseFromParent();2118    return;2119  }2120 2121  // Get address space of pointers in the array of pointers.2122  const Type *UsedArrayType = V.getValueType();2123  const auto *VAT = cast<ArrayType>(UsedArrayType);2124  const auto *VEPT = cast<PointerType>(VAT->getArrayElementType());2125 2126  // Type of pointer to the array of pointers.2127  PointerType *PtrTy =2128      PointerType::get(V.getContext(), VEPT->getAddressSpace());2129 2130  SmallVector<Constant *, 8> UsedArray;2131  for (GlobalValue *GV : Init) {2132    Constant *Cast = ConstantExpr::getPointerBitCastOrAddrSpaceCast(GV, PtrTy);2133    UsedArray.push_back(Cast);2134  }2135 2136  // Sort to get deterministic order.2137  array_pod_sort(UsedArray.begin(), UsedArray.end(), compareNames);2138  ArrayType *ATy = ArrayType::get(PtrTy, UsedArray.size());2139 2140  Module *M = V.getParent();2141  V.removeFromParent();2142  GlobalVariable *NV =2143      new GlobalVariable(*M, ATy, false, GlobalValue::AppendingLinkage,2144                         ConstantArray::get(ATy, UsedArray), "");2145  NV->takeName(&V);2146  NV->setSection("llvm.metadata");2147  delete &V;2148}2149 2150namespace {2151 2152/// An easy to access representation of llvm.used and llvm.compiler.used.2153class LLVMUsed {2154  SmallPtrSet<GlobalValue *, 4> Used;2155  SmallPtrSet<GlobalValue *, 4> CompilerUsed;2156  GlobalVariable *UsedV;2157  GlobalVariable *CompilerUsedV;2158 2159public:2160  LLVMUsed(Module &M) {2161    SmallVector<GlobalValue *, 4> Vec;2162    UsedV = collectUsedGlobalVariables(M, Vec, false);2163    Used = {llvm::from_range, Vec};2164    Vec.clear();2165    CompilerUsedV = collectUsedGlobalVariables(M, Vec, true);2166    CompilerUsed = {llvm::from_range, Vec};2167  }2168 2169  using iterator = SmallPtrSet<GlobalValue *, 4>::iterator;2170  using used_iterator_range = iterator_range<iterator>;2171 2172  iterator usedBegin() { return Used.begin(); }2173  iterator usedEnd() { return Used.end(); }2174 2175  used_iterator_range used() {2176    return used_iterator_range(usedBegin(), usedEnd());2177  }2178 2179  iterator compilerUsedBegin() { return CompilerUsed.begin(); }2180  iterator compilerUsedEnd() { return CompilerUsed.end(); }2181 2182  used_iterator_range compilerUsed() {2183    return used_iterator_range(compilerUsedBegin(), compilerUsedEnd());2184  }2185 2186  bool usedCount(GlobalValue *GV) const { return Used.count(GV); }2187 2188  bool compilerUsedCount(GlobalValue *GV) const {2189    return CompilerUsed.count(GV);2190  }2191 2192  bool usedErase(GlobalValue *GV) { return Used.erase(GV); }2193  bool compilerUsedErase(GlobalValue *GV) { return CompilerUsed.erase(GV); }2194  bool usedInsert(GlobalValue *GV) { return Used.insert(GV).second; }2195 2196  bool compilerUsedInsert(GlobalValue *GV) {2197    return CompilerUsed.insert(GV).second;2198  }2199 2200  void syncVariablesAndSets() {2201    if (UsedV)2202      setUsedInitializer(*UsedV, Used);2203    if (CompilerUsedV)2204      setUsedInitializer(*CompilerUsedV, CompilerUsed);2205  }2206};2207 2208} // end anonymous namespace2209 2210static bool hasUseOtherThanLLVMUsed(GlobalAlias &GA, const LLVMUsed &U) {2211  if (GA.use_empty()) // No use at all.2212    return false;2213 2214  assert((!U.usedCount(&GA) || !U.compilerUsedCount(&GA)) &&2215         "We should have removed the duplicated "2216         "element from llvm.compiler.used");2217  if (!GA.hasOneUse())2218    // Strictly more than one use. So at least one is not in llvm.used and2219    // llvm.compiler.used.2220    return true;2221 2222  // Exactly one use. Check if it is in llvm.used or llvm.compiler.used.2223  return !U.usedCount(&GA) && !U.compilerUsedCount(&GA);2224}2225 2226static bool mayHaveOtherReferences(GlobalValue &GV, const LLVMUsed &U) {2227  if (!GV.hasLocalLinkage())2228    return true;2229 2230  return U.usedCount(&GV) || U.compilerUsedCount(&GV);2231}2232 2233static bool hasUsesToReplace(GlobalAlias &GA, const LLVMUsed &U,2234                             bool &RenameTarget) {2235  if (GA.isWeakForLinker())2236    return false;2237 2238  RenameTarget = false;2239  bool Ret = false;2240  if (hasUseOtherThanLLVMUsed(GA, U))2241    Ret = true;2242 2243  // If the alias is externally visible, we may still be able to simplify it.2244  if (!mayHaveOtherReferences(GA, U))2245    return Ret;2246 2247  // If the aliasee has internal linkage and no other references (e.g.,2248  // @llvm.used, @llvm.compiler.used), give it the name and linkage of the2249  // alias, and delete the alias. This turns:2250  //   define internal ... @f(...)2251  //   @a = alias ... @f2252  // into:2253  //   define ... @a(...)2254  Constant *Aliasee = GA.getAliasee();2255  GlobalValue *Target = cast<GlobalValue>(Aliasee->stripPointerCasts());2256  if (mayHaveOtherReferences(*Target, U))2257    return Ret;2258 2259  RenameTarget = true;2260  return true;2261}2262 2263static bool2264OptimizeGlobalAliases(Module &M,2265                      SmallPtrSetImpl<const Comdat *> &NotDiscardableComdats) {2266  bool Changed = false;2267  LLVMUsed Used(M);2268 2269  for (GlobalValue *GV : Used.used())2270    Used.compilerUsedErase(GV);2271 2272  // Return whether GV is explicitly or implicitly dso_local and not replaceable2273  // by another definition in the current linkage unit.2274  auto IsModuleLocal = [](GlobalValue &GV) {2275    return !GlobalValue::isInterposableLinkage(GV.getLinkage()) &&2276           (GV.isDSOLocal() || GV.isImplicitDSOLocal());2277  };2278 2279  for (GlobalAlias &J : llvm::make_early_inc_range(M.aliases())) {2280    // Aliases without names cannot be referenced outside this module.2281    if (!J.hasName() && !J.isDeclaration() && !J.hasLocalLinkage())2282      J.setLinkage(GlobalValue::InternalLinkage);2283 2284    if (deleteIfDead(J, NotDiscardableComdats)) {2285      Changed = true;2286      continue;2287    }2288 2289    // If the alias can change at link time, nothing can be done - bail out.2290    if (!IsModuleLocal(J))2291      continue;2292 2293    Constant *Aliasee = J.getAliasee();2294    GlobalValue *Target = dyn_cast<GlobalValue>(Aliasee->stripPointerCasts());2295    // We can't trivially replace the alias with the aliasee if the aliasee is2296    // non-trivial in some way. We also can't replace the alias with the aliasee2297    // if the aliasee may be preemptible at runtime. On ELF, a non-preemptible2298    // alias can be used to access the definition as if preemption did not2299    // happen.2300    // TODO: Try to handle non-zero GEPs of local aliasees.2301    if (!Target || !IsModuleLocal(*Target))2302      continue;2303 2304    Target->removeDeadConstantUsers();2305 2306    // Make all users of the alias use the aliasee instead.2307    bool RenameTarget;2308    if (!hasUsesToReplace(J, Used, RenameTarget))2309      continue;2310 2311    J.replaceAllUsesWith(Aliasee);2312    ++NumAliasesResolved;2313    Changed = true;2314 2315    if (RenameTarget) {2316      // Give the aliasee the name, linkage and other attributes of the alias.2317      Target->takeName(&J);2318      Target->setLinkage(J.getLinkage());2319      Target->setDSOLocal(J.isDSOLocal());2320      Target->setVisibility(J.getVisibility());2321      Target->setDLLStorageClass(J.getDLLStorageClass());2322 2323      if (Used.usedErase(&J))2324        Used.usedInsert(Target);2325 2326      if (Used.compilerUsedErase(&J))2327        Used.compilerUsedInsert(Target);2328    } else if (mayHaveOtherReferences(J, Used))2329      continue;2330 2331    // Delete the alias.2332    M.eraseAlias(&J);2333    ++NumAliasesRemoved;2334    Changed = true;2335  }2336 2337  Used.syncVariablesAndSets();2338 2339  return Changed;2340}2341 2342static Function *2343FindAtExitLibFunc(Module &M,2344                  function_ref<TargetLibraryInfo &(Function &)> GetTLI,2345                  LibFunc Func) {2346  // Hack to get a default TLI before we have actual Function.2347  auto FuncIter = M.begin();2348  if (FuncIter == M.end())2349    return nullptr;2350  auto *TLI = &GetTLI(*FuncIter);2351 2352  if (!TLI->has(Func))2353    return nullptr;2354 2355  Function *Fn = M.getFunction(TLI->getName(Func));2356  if (!Fn)2357    return nullptr;2358 2359  // Now get the actual TLI for Fn.2360  TLI = &GetTLI(*Fn);2361 2362  // Make sure that the function has the correct prototype.2363  LibFunc F;2364  if (!TLI->getLibFunc(*Fn, F) || F != Func)2365    return nullptr;2366 2367  return Fn;2368}2369 2370/// Returns whether the given function is an empty C++ destructor or atexit2371/// handler and can therefore be eliminated. Note that we assume that other2372/// optimization passes have already simplified the code so we simply check for2373/// 'ret'.2374static bool IsEmptyAtExitFunction(const Function &Fn) {2375  // FIXME: We could eliminate C++ destructors if they're readonly/readnone and2376  // nounwind, but that doesn't seem worth doing.2377  if (Fn.isDeclaration())2378    return false;2379 2380  for (const auto &I : Fn.getEntryBlock()) {2381    if (I.isDebugOrPseudoInst())2382      continue;2383    if (isa<ReturnInst>(I))2384      return true;2385    break;2386  }2387  return false;2388}2389 2390static bool OptimizeEmptyGlobalAtExitDtors(Function *CXAAtExitFn, bool isCXX) {2391  /// Itanium C++ ABI p3.3.5:2392  ///2393  ///   After constructing a global (or local static) object, that will require2394  ///   destruction on exit, a termination function is registered as follows:2395  ///2396  ///   extern "C" int __cxa_atexit ( void (*f)(void *), void *p, void *d );2397  ///2398  ///   This registration, e.g. __cxa_atexit(f,p,d), is intended to cause the2399  ///   call f(p) when DSO d is unloaded, before all such termination calls2400  ///   registered before this one. It returns zero if registration is2401  ///   successful, nonzero on failure.2402 2403  // This pass will look for calls to __cxa_atexit or atexit where the function2404  // is trivial and remove them.2405  bool Changed = false;2406 2407  for (User *U : llvm::make_early_inc_range(CXAAtExitFn->users())) {2408    // We're only interested in calls. Theoretically, we could handle invoke2409    // instructions as well, but neither llvm-gcc nor clang generate invokes2410    // to __cxa_atexit.2411    CallInst *CI = dyn_cast<CallInst>(U);2412    if (!CI)2413      continue;2414 2415    Function *DtorFn =2416      dyn_cast<Function>(CI->getArgOperand(0)->stripPointerCasts());2417    if (!DtorFn || !IsEmptyAtExitFunction(*DtorFn))2418      continue;2419 2420    // Just remove the call.2421    CI->replaceAllUsesWith(Constant::getNullValue(CI->getType()));2422    CI->eraseFromParent();2423 2424    if (isCXX)2425      ++NumCXXDtorsRemoved;2426    else2427      ++NumAtExitRemoved;2428 2429    Changed |= true;2430  }2431 2432  return Changed;2433}2434 2435static Function *hasSideeffectFreeStaticResolution(GlobalIFunc &IF) {2436  if (IF.isInterposable())2437    return nullptr;2438 2439  Function *Resolver = IF.getResolverFunction();2440  if (!Resolver)2441    return nullptr;2442 2443  if (Resolver->isInterposable())2444    return nullptr;2445 2446  // Only handle functions that have been optimized into a single basic block.2447  auto It = Resolver->begin();2448  if (++It != Resolver->end())2449    return nullptr;2450 2451  BasicBlock &BB = Resolver->getEntryBlock();2452 2453  if (any_of(BB, [](Instruction &I) { return I.mayHaveSideEffects(); }))2454    return nullptr;2455 2456  auto *Ret = dyn_cast<ReturnInst>(BB.getTerminator());2457  if (!Ret)2458    return nullptr;2459 2460  return dyn_cast<Function>(Ret->getReturnValue());2461}2462 2463/// Find IFuncs that have resolvers that always point at the same statically2464/// known callee, and replace their callers with a direct call.2465static bool OptimizeStaticIFuncs(Module &M) {2466  bool Changed = false;2467  for (GlobalIFunc &IF : M.ifuncs())2468    if (Function *Callee = hasSideeffectFreeStaticResolution(IF))2469      if (!IF.use_empty() &&2470          (!Callee->isDeclaration() ||2471           none_of(IF.users(), [](User *U) { return isa<GlobalAlias>(U); }))) {2472        IF.replaceAllUsesWith(Callee);2473        NumIFuncsResolved++;2474        Changed = true;2475      }2476  return Changed;2477}2478 2479static bool2480DeleteDeadIFuncs(Module &M,2481                 SmallPtrSetImpl<const Comdat *> &NotDiscardableComdats) {2482  bool Changed = false;2483  for (GlobalIFunc &IF : make_early_inc_range(M.ifuncs()))2484    if (deleteIfDead(IF, NotDiscardableComdats)) {2485      NumIFuncsDeleted++;2486      Changed = true;2487    }2488  return Changed;2489}2490 2491// Follows the use-def chain of \p V backwards until it finds a Function,2492// in which case it collects in \p Versions. Return true on successful2493// use-def chain traversal, false otherwise.2494static bool2495collectVersions(Value *V, SmallVectorImpl<Function *> &Versions,2496                function_ref<TargetTransformInfo &(Function &)> GetTTI) {2497  if (auto *F = dyn_cast<Function>(V)) {2498    if (!GetTTI(*F).isMultiversionedFunction(*F))2499      return false;2500    Versions.push_back(F);2501  } else if (auto *Sel = dyn_cast<SelectInst>(V)) {2502    if (!collectVersions(Sel->getTrueValue(), Versions, GetTTI))2503      return false;2504    if (!collectVersions(Sel->getFalseValue(), Versions, GetTTI))2505      return false;2506  } else if (auto *Phi = dyn_cast<PHINode>(V)) {2507    for (unsigned I = 0, E = Phi->getNumIncomingValues(); I != E; ++I)2508      if (!collectVersions(Phi->getIncomingValue(I), Versions, GetTTI))2509        return false;2510  } else {2511    // Unknown instruction type. Bail.2512    return false;2513  }2514  return true;2515}2516 2517// Try to statically resolve calls to versioned functions when possible. First2518// we identify the function versions which are associated with an IFUNC symbol.2519// We do that by examining the resolver function of the IFUNC. Once we have2520// collected all the function versions, we sort them in decreasing priority2521// order. This is necessary for determining the most suitable callee version2522// for each caller version. We then collect all the callsites to versioned2523// functions. The static resolution is performed by comparing the feature sets2524// between callers and callees. Specifically:2525// * Start a walk over caller and callee lists simultaneously in order of2526//   decreasing priority.2527// * Statically resolve calls from the current caller to the current callee,2528//   iff the caller feature bits are a superset of the callee feature bits.2529// * For FMV callers, as long as the caller feature bits are a subset of the2530//   callee feature bits, advance to the next callee. This effectively prevents2531//   considering the current callee as a candidate for static resolution by2532//   following callers (explanation: preceding callers would not have been2533//   selected in a hypothetical runtime execution).2534// * Advance to the next caller.2535//2536// Presentation in EuroLLVM2025:2537// https://www.youtube.com/watch?v=k54MFimPz-A&t=867s2538static bool OptimizeNonTrivialIFuncs(2539    Module &M, function_ref<TargetTransformInfo &(Function &)> GetTTI) {2540  bool Changed = false;2541 2542  // Map containing the feature bits for a given function.2543  DenseMap<Function *, APInt> FeatureMask;2544  // Map containing all the function versions corresponding to an IFunc symbol.2545  DenseMap<GlobalIFunc *, SmallVector<Function *>> VersionedFuncs;2546  // Map containing the IFunc symbol a function is version of.2547  DenseMap<Function *, GlobalIFunc *> VersionOf;2548  // List of all the interesting IFuncs found in the module.2549  SmallVector<GlobalIFunc *> IFuncs;2550 2551  for (GlobalIFunc &IF : M.ifuncs()) {2552    LLVM_DEBUG(dbgs() << "Examining IFUNC " << IF.getName() << "\n");2553 2554    if (IF.isInterposable())2555      continue;2556 2557    Function *Resolver = IF.getResolverFunction();2558    if (!Resolver)2559      continue;2560 2561    if (Resolver->isInterposable())2562      continue;2563 2564    SmallVector<Function *> Versions;2565    // Discover the versioned functions.2566    if (any_of(*Resolver, [&](BasicBlock &BB) {2567          if (auto *Ret = dyn_cast_or_null<ReturnInst>(BB.getTerminator()))2568            if (!collectVersions(Ret->getReturnValue(), Versions, GetTTI))2569              return true;2570          return false;2571        }))2572      continue;2573 2574    if (Versions.empty())2575      continue;2576 2577    for (Function *V : Versions) {2578      VersionOf.insert({V, &IF});2579      auto [It, Inserted] = FeatureMask.try_emplace(V);2580      if (Inserted)2581        It->second = GetTTI(*V).getFeatureMask(*V);2582    }2583 2584    // Sort function versions in decreasing priority order.2585    sort(Versions, [&](auto *LHS, auto *RHS) {2586      return FeatureMask[LHS].ugt(FeatureMask[RHS]);2587    });2588 2589    IFuncs.push_back(&IF);2590    VersionedFuncs.try_emplace(&IF, std::move(Versions));2591  }2592 2593  for (GlobalIFunc *CalleeIF : IFuncs) {2594    SmallVector<Function *> NonFMVCallers;2595    DenseSet<GlobalIFunc *> CallerIFuncs;2596    DenseMap<Function *, SmallVector<CallBase *>> CallSites;2597 2598    // Find the callsites.2599    for (User *U : CalleeIF->users()) {2600      if (auto *CB = dyn_cast<CallBase>(U)) {2601        if (CB->getCalledOperand() == CalleeIF) {2602          Function *Caller = CB->getFunction();2603          GlobalIFunc *CallerIF = nullptr;2604          TargetTransformInfo &TTI = GetTTI(*Caller);2605          bool CallerIsFMV = TTI.isMultiversionedFunction(*Caller);2606          // The caller is a version of a known IFunc.2607          if (auto It = VersionOf.find(Caller); It != VersionOf.end())2608            CallerIF = It->second;2609          else if (!CallerIsFMV && OptimizeNonFMVCallers) {2610            // The caller is non-FMV.2611            auto [It, Inserted] = FeatureMask.try_emplace(Caller);2612            if (Inserted)2613              It->second = TTI.getFeatureMask(*Caller);2614          } else2615            // The caller is none of the above, skip.2616            continue;2617          auto [It, Inserted] = CallSites.try_emplace(Caller);2618          if (Inserted) {2619            if (CallerIsFMV)2620              CallerIFuncs.insert(CallerIF);2621            else2622              NonFMVCallers.push_back(Caller);2623          }2624          It->second.push_back(CB);2625        }2626      }2627    }2628 2629    if (CallSites.empty())2630      continue;2631 2632    LLVM_DEBUG(dbgs() << "Statically resolving calls to function "2633                      << CalleeIF->getResolverFunction()->getName() << "\n");2634 2635    // The complexity of this algorithm is linear: O(NumCallers + NumCallees).2636    // TODO2637    // A limitation it has is that we are not using information about the2638    // current caller to deduce why an earlier caller of higher priority was2639    // skipped. For example let's say the current caller is aes+sve2 and a2640    // previous caller was mops+sve2. Knowing that sve2 is available we could2641    // infer that mops is unavailable. This would allow us to skip callee2642    // versions which depend on mops. I tried implementing this but the2643    // complexity was cubic :/2644    auto staticallyResolveCalls = [&](ArrayRef<Function *> Callers,2645                                      ArrayRef<Function *> Callees,2646                                      bool CallerIsFMV) {2647      // Index to the highest callee candidate.2648      unsigned I = 0;2649 2650      for (Function *const &Caller : Callers) {2651        if (I == Callees.size())2652          break;2653 2654        LLVM_DEBUG(dbgs() << "   Examining "2655                          << (CallerIsFMV ? "FMV" : "regular") << " caller "2656                          << Caller->getName() << "\n");2657 2658        Function *Callee = Callees[I];2659        APInt CallerBits = FeatureMask[Caller];2660        APInt CalleeBits = FeatureMask[Callee];2661 2662        // Statically resolve calls from the current caller to the current2663        // callee, iff the caller feature bits are a superset of the callee2664        // feature bits.2665        if (CalleeBits.isSubsetOf(CallerBits)) {2666          // Not all caller versions are necessarily users of the callee IFUNC.2667          if (auto It = CallSites.find(Caller); It != CallSites.end()) {2668            for (CallBase *CS : It->second) {2669              LLVM_DEBUG(dbgs() << "   Redirecting call " << Caller->getName()2670                                << " -> " << Callee->getName() << "\n");2671              CS->setCalledOperand(Callee);2672            }2673            Changed = true;2674          }2675        }2676 2677        // Nothing else to do about non-FMV callers.2678        if (!CallerIsFMV)2679          continue;2680 2681        // For FMV callers, as long as the caller feature bits are a subset of2682        // the callee feature bits, advance to the next callee. This effectively2683        // prevents considering the current callee as a candidate for static2684        // resolution by following callers.2685        while (CallerBits.isSubsetOf(FeatureMask[Callees[I]]) &&2686               ++I < Callees.size())2687          ;2688      }2689    };2690 2691    auto &Callees = VersionedFuncs[CalleeIF];2692 2693    // Optimize non-FMV calls.2694    if (OptimizeNonFMVCallers)2695      staticallyResolveCalls(NonFMVCallers, Callees, /*CallerIsFMV=*/false);2696 2697    // Optimize FMV calls.2698    for (GlobalIFunc *CallerIF : CallerIFuncs) {2699      auto &Callers = VersionedFuncs[CallerIF];2700      staticallyResolveCalls(Callers, Callees, /*CallerIsFMV=*/true);2701    }2702 2703    if (CalleeIF->use_empty() ||2704        all_of(CalleeIF->users(), [](User *U) { return isa<GlobalAlias>(U); }))2705      NumIFuncsResolved++;2706  }2707  return Changed;2708}2709 2710static bool2711optimizeGlobalsInModule(Module &M, const DataLayout &DL,2712                        function_ref<TargetLibraryInfo &(Function &)> GetTLI,2713                        function_ref<TargetTransformInfo &(Function &)> GetTTI,2714                        function_ref<BlockFrequencyInfo &(Function &)> GetBFI,2715                        function_ref<DominatorTree &(Function &)> LookupDomTree,2716                        function_ref<void(Function &F)> ChangedCFGCallback,2717                        function_ref<void(Function &F)> DeleteFnCallback) {2718  SmallPtrSet<const Comdat *, 8> NotDiscardableComdats;2719  bool Changed = false;2720  bool LocalChange = true;2721  std::optional<uint32_t> FirstNotFullyEvaluatedPriority;2722 2723  while (LocalChange) {2724    LocalChange = false;2725 2726    NotDiscardableComdats.clear();2727    for (const GlobalVariable &GV : M.globals())2728      if (const Comdat *C = GV.getComdat())2729        if (!GV.isDiscardableIfUnused() || !GV.use_empty())2730          NotDiscardableComdats.insert(C);2731    for (Function &F : M)2732      if (const Comdat *C = F.getComdat())2733        if (!F.isDefTriviallyDead())2734          NotDiscardableComdats.insert(C);2735    for (GlobalAlias &GA : M.aliases())2736      if (const Comdat *C = GA.getComdat())2737        if (!GA.isDiscardableIfUnused() || !GA.use_empty())2738          NotDiscardableComdats.insert(C);2739 2740    // Delete functions that are trivially dead, ccc -> fastcc2741    LocalChange |= OptimizeFunctions(M, GetTLI, GetTTI, GetBFI, LookupDomTree,2742                                     NotDiscardableComdats, ChangedCFGCallback,2743                                     DeleteFnCallback);2744 2745    // Optimize global_ctors list.2746    LocalChange |=2747        optimizeGlobalCtorsList(M, [&](uint32_t Priority, Function *F) {2748          if (FirstNotFullyEvaluatedPriority &&2749              *FirstNotFullyEvaluatedPriority != Priority)2750            return false;2751          bool Evaluated = EvaluateStaticConstructor(F, DL, &GetTLI(*F));2752          if (!Evaluated)2753            FirstNotFullyEvaluatedPriority = Priority;2754          return Evaluated;2755        });2756 2757    // Optimize non-address-taken globals.2758    LocalChange |= OptimizeGlobalVars(M, GetTTI, GetTLI, LookupDomTree,2759                                      NotDiscardableComdats);2760 2761    // Resolve aliases, when possible.2762    LocalChange |= OptimizeGlobalAliases(M, NotDiscardableComdats);2763 2764    // Try to remove trivial global destructors if they are not removed2765    // already.2766    if (Function *CXAAtExitFn =2767            FindAtExitLibFunc(M, GetTLI, LibFunc_cxa_atexit))2768      LocalChange |= OptimizeEmptyGlobalAtExitDtors(CXAAtExitFn, true);2769 2770    if (Function *AtExitFn = FindAtExitLibFunc(M, GetTLI, LibFunc_atexit))2771      LocalChange |= OptimizeEmptyGlobalAtExitDtors(AtExitFn, false);2772 2773    // Optimize IFuncs whose callee's are statically known.2774    LocalChange |= OptimizeStaticIFuncs(M);2775 2776    // Optimize IFuncs based on the target features of the caller.2777    LocalChange |= OptimizeNonTrivialIFuncs(M, GetTTI);2778 2779    // Remove any IFuncs that are now dead.2780    LocalChange |= DeleteDeadIFuncs(M, NotDiscardableComdats);2781 2782    Changed |= LocalChange;2783  }2784 2785  // TODO: Move all global ctors functions to the end of the module for code2786  // layout.2787 2788  return Changed;2789}2790 2791PreservedAnalyses GlobalOptPass::run(Module &M, ModuleAnalysisManager &AM) {2792    auto &DL = M.getDataLayout();2793    auto &FAM =2794        AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();2795    auto LookupDomTree = [&FAM](Function &F) -> DominatorTree &{2796      return FAM.getResult<DominatorTreeAnalysis>(F);2797    };2798    auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {2799      return FAM.getResult<TargetLibraryAnalysis>(F);2800    };2801    auto GetTTI = [&FAM](Function &F) -> TargetTransformInfo & {2802      return FAM.getResult<TargetIRAnalysis>(F);2803    };2804 2805    auto GetBFI = [&FAM](Function &F) -> BlockFrequencyInfo & {2806      return FAM.getResult<BlockFrequencyAnalysis>(F);2807    };2808    auto ChangedCFGCallback = [&FAM](Function &F) {2809      FAM.invalidate(F, PreservedAnalyses::none());2810    };2811    auto DeleteFnCallback = [&FAM](Function &F) { FAM.clear(F, F.getName()); };2812 2813    if (!optimizeGlobalsInModule(M, DL, GetTLI, GetTTI, GetBFI, LookupDomTree,2814                                 ChangedCFGCallback, DeleteFnCallback))2815      return PreservedAnalyses::all();2816 2817    PreservedAnalyses PA = PreservedAnalyses::none();2818    // We made sure to clear analyses for deleted functions.2819    PA.preserve<FunctionAnalysisManagerModuleProxy>();2820    // The only place we modify the CFG is when calling2821    // removeUnreachableBlocks(), but there we make sure to invalidate analyses2822    // for modified functions.2823    PA.preserveSet<CFGAnalyses>();2824    return PA;2825}2826