brintos / llvm-project-archived public Read only

1//===- AttributorAttributes.cpp - Attributes for Attributor deduction -----===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// See the Attributor.h file comment and the class descriptions in that file for10// more information.11//12//===----------------------------------------------------------------------===//13 14#include "llvm/Transforms/IPO/Attributor.h"15 16#include "llvm/ADT/APInt.h"17#include "llvm/ADT/ArrayRef.h"18#include "llvm/ADT/DenseMapInfo.h"19#include "llvm/ADT/MapVector.h"20#include "llvm/ADT/SCCIterator.h"21#include "llvm/ADT/STLExtras.h"22#include "llvm/ADT/SetOperations.h"23#include "llvm/ADT/SetVector.h"24#include "llvm/ADT/SmallPtrSet.h"25#include "llvm/ADT/SmallVector.h"26#include "llvm/ADT/Statistic.h"27#include "llvm/ADT/StringExtras.h"28#include "llvm/Analysis/AliasAnalysis.h"29#include "llvm/Analysis/AssumeBundleQueries.h"30#include "llvm/Analysis/AssumptionCache.h"31#include "llvm/Analysis/CaptureTracking.h"32#include "llvm/Analysis/CycleAnalysis.h"33#include "llvm/Analysis/InstructionSimplify.h"34#include "llvm/Analysis/LazyValueInfo.h"35#include "llvm/Analysis/MemoryBuiltins.h"36#include "llvm/Analysis/ScalarEvolution.h"37#include "llvm/Analysis/TargetTransformInfo.h"38#include "llvm/Analysis/ValueTracking.h"39#include "llvm/IR/Argument.h"40#include "llvm/IR/Assumptions.h"41#include "llvm/IR/Attributes.h"42#include "llvm/IR/BasicBlock.h"43#include "llvm/IR/Constant.h"44#include "llvm/IR/Constants.h"45#include "llvm/IR/DataLayout.h"46#include "llvm/IR/DerivedTypes.h"47#include "llvm/IR/GlobalValue.h"48#include "llvm/IR/IRBuilder.h"49#include "llvm/IR/InlineAsm.h"50#include "llvm/IR/InstrTypes.h"51#include "llvm/IR/Instruction.h"52#include "llvm/IR/Instructions.h"53#include "llvm/IR/IntrinsicInst.h"54#include "llvm/IR/IntrinsicsAMDGPU.h"55#include "llvm/IR/IntrinsicsNVPTX.h"56#include "llvm/IR/LLVMContext.h"57#include "llvm/IR/MDBuilder.h"58#include "llvm/IR/NoFolder.h"59#include "llvm/IR/Value.h"60#include "llvm/IR/ValueHandle.h"61#include "llvm/Support/Alignment.h"62#include "llvm/Support/Casting.h"63#include "llvm/Support/CommandLine.h"64#include "llvm/Support/ErrorHandling.h"65#include "llvm/Support/GraphWriter.h"66#include "llvm/Support/InterleavedRange.h"67#include "llvm/Support/KnownFPClass.h"68#include "llvm/Support/MathExtras.h"69#include "llvm/Support/TypeSize.h"70#include "llvm/Support/raw_ostream.h"71#include "llvm/Transforms/Utils/BasicBlockUtils.h"72#include "llvm/Transforms/Utils/CallPromotionUtils.h"73#include "llvm/Transforms/Utils/Local.h"74#include "llvm/Transforms/Utils/ValueMapper.h"75#include <cassert>76#include <numeric>77#include <optional>78#include <string>79 80using namespace llvm;81 82#define DEBUG_TYPE "attributor"83 84static cl::opt<bool> ManifestInternal(85    "attributor-manifest-internal", cl::Hidden,86    cl::desc("Manifest Attributor internal string attributes."),87    cl::init(false));88 89static cl::opt<int> MaxHeapToStackSize("max-heap-to-stack-size", cl::init(128),90                                       cl::Hidden);91 92template <>93unsigned llvm::PotentialConstantIntValuesState::MaxPotentialValues = 0;94 95template <> unsigned llvm::PotentialLLVMValuesState::MaxPotentialValues = -1;96 97static cl::opt<unsigned, true> MaxPotentialValues(98    "attributor-max-potential-values", cl::Hidden,99    cl::desc("Maximum number of potential values to be "100             "tracked for each position."),101    cl::location(llvm::PotentialConstantIntValuesState::MaxPotentialValues),102    cl::init(7));103 104static cl::opt<int> MaxPotentialValuesIterations(105    "attributor-max-potential-values-iterations", cl::Hidden,106    cl::desc(107        "Maximum number of iterations we keep dismantling potential values."),108    cl::init(64));109 110STATISTIC(NumAAs, "Number of abstract attributes created");111STATISTIC(NumIndirectCallsPromoted, "Number of indirect calls promoted");112 113// Some helper macros to deal with statistics tracking.114//115// Usage:116// For simple IR attribute tracking overload trackStatistics in the abstract117// attribute and choose the right STATS_DECLTRACK_********* macro,118// e.g.,:119//  void trackStatistics() const override {120//    STATS_DECLTRACK_ARG_ATTR(returned)121//  }122// If there is a single "increment" side one can use the macro123// STATS_DECLTRACK with a custom message. If there are multiple increment124// sides, STATS_DECL and STATS_TRACK can also be used separately.125//126#define BUILD_STAT_MSG_IR_ATTR(TYPE, NAME)                                     \127  ("Number of " #TYPE " marked '" #NAME "'")128#define BUILD_STAT_NAME(NAME, TYPE) NumIR##TYPE##_##NAME129#define STATS_DECL_(NAME, MSG) STATISTIC(NAME, MSG);130#define STATS_DECL(NAME, TYPE, MSG)                                            \131  STATS_DECL_(BUILD_STAT_NAME(NAME, TYPE), MSG);132#define STATS_TRACK(NAME, TYPE) ++(BUILD_STAT_NAME(NAME, TYPE));133#define STATS_DECLTRACK(NAME, TYPE, MSG)                                       \134  {STATS_DECL(NAME, TYPE, MSG) STATS_TRACK(NAME, TYPE)}135#define STATS_DECLTRACK_ARG_ATTR(NAME)                                         \136  STATS_DECLTRACK(NAME, Arguments, BUILD_STAT_MSG_IR_ATTR(arguments, NAME))137#define STATS_DECLTRACK_CSARG_ATTR(NAME)                                       \138  STATS_DECLTRACK(NAME, CSArguments,                                           \139                  BUILD_STAT_MSG_IR_ATTR(call site arguments, NAME))140#define STATS_DECLTRACK_FN_ATTR(NAME)                                          \141  STATS_DECLTRACK(NAME, Function, BUILD_STAT_MSG_IR_ATTR(functions, NAME))142#define STATS_DECLTRACK_CS_ATTR(NAME)                                          \143  STATS_DECLTRACK(NAME, CS, BUILD_STAT_MSG_IR_ATTR(call site, NAME))144#define STATS_DECLTRACK_FNRET_ATTR(NAME)                                       \145  STATS_DECLTRACK(NAME, FunctionReturn,                                        \146                  BUILD_STAT_MSG_IR_ATTR(function returns, NAME))147#define STATS_DECLTRACK_CSRET_ATTR(NAME)                                       \148  STATS_DECLTRACK(NAME, CSReturn,                                              \149                  BUILD_STAT_MSG_IR_ATTR(call site returns, NAME))150#define STATS_DECLTRACK_FLOATING_ATTR(NAME)                                    \151  STATS_DECLTRACK(NAME, Floating,                                              \152                  ("Number of floating values known to be '" #NAME "'"))153 154// Specialization of the operator<< for abstract attributes subclasses. This155// disambiguates situations where multiple operators are applicable.156namespace llvm {157#define PIPE_OPERATOR(CLASS)                                                   \158  raw_ostream &operator<<(raw_ostream &OS, const CLASS &AA) {                  \159    return OS << static_cast<const AbstractAttribute &>(AA);                   \160  }161 162PIPE_OPERATOR(AAIsDead)163PIPE_OPERATOR(AANoUnwind)164PIPE_OPERATOR(AANoSync)165PIPE_OPERATOR(AANoRecurse)166PIPE_OPERATOR(AANonConvergent)167PIPE_OPERATOR(AAWillReturn)168PIPE_OPERATOR(AANoReturn)169PIPE_OPERATOR(AANonNull)170PIPE_OPERATOR(AAMustProgress)171PIPE_OPERATOR(AANoAlias)172PIPE_OPERATOR(AADereferenceable)173PIPE_OPERATOR(AAAlign)174PIPE_OPERATOR(AAInstanceInfo)175PIPE_OPERATOR(AANoCapture)176PIPE_OPERATOR(AAValueSimplify)177PIPE_OPERATOR(AANoFree)178PIPE_OPERATOR(AAHeapToStack)179PIPE_OPERATOR(AAIntraFnReachability)180PIPE_OPERATOR(AAMemoryBehavior)181PIPE_OPERATOR(AAMemoryLocation)182PIPE_OPERATOR(AAValueConstantRange)183PIPE_OPERATOR(AAPrivatizablePtr)184PIPE_OPERATOR(AAUndefinedBehavior)185PIPE_OPERATOR(AAPotentialConstantValues)186PIPE_OPERATOR(AAPotentialValues)187PIPE_OPERATOR(AANoUndef)188PIPE_OPERATOR(AANoFPClass)189PIPE_OPERATOR(AACallEdges)190PIPE_OPERATOR(AAInterFnReachability)191PIPE_OPERATOR(AAPointerInfo)192PIPE_OPERATOR(AAAssumptionInfo)193PIPE_OPERATOR(AAUnderlyingObjects)194PIPE_OPERATOR(AAInvariantLoadPointer)195PIPE_OPERATOR(AAAddressSpace)196PIPE_OPERATOR(AANoAliasAddrSpace)197PIPE_OPERATOR(AAAllocationInfo)198PIPE_OPERATOR(AAIndirectCallInfo)199PIPE_OPERATOR(AAGlobalValueInfo)200PIPE_OPERATOR(AADenormalFPMath)201 202#undef PIPE_OPERATOR203 204template <>205ChangeStatus clampStateAndIndicateChange<DerefState>(DerefState &S,206                                                     const DerefState &R) {207  ChangeStatus CS0 =208      clampStateAndIndicateChange(S.DerefBytesState, R.DerefBytesState);209  ChangeStatus CS1 = clampStateAndIndicateChange(S.GlobalState, R.GlobalState);210  return CS0 | CS1;211}212 213} // namespace llvm214 215static bool mayBeInCycle(const CycleInfo *CI, const Instruction *I,216                         bool HeaderOnly, Cycle **CPtr = nullptr) {217  if (!CI)218    return true;219  auto *BB = I->getParent();220  auto *C = CI->getCycle(BB);221  if (!C)222    return false;223  if (CPtr)224    *CPtr = C;225  return !HeaderOnly || BB == C->getHeader();226}227 228/// Checks if a type could have padding bytes.229static bool isDenselyPacked(Type *Ty, const DataLayout &DL) {230  // There is no size information, so be conservative.231  if (!Ty->isSized())232    return false;233 234  // If the alloc size is not equal to the storage size, then there are padding235  // bytes. For x86_fp80 on x86-64, size: 80 alloc size: 128.236  if (DL.getTypeSizeInBits(Ty) != DL.getTypeAllocSizeInBits(Ty))237    return false;238 239  // FIXME: This isn't the right way to check for padding in vectors with240  // non-byte-size elements.241  if (VectorType *SeqTy = dyn_cast<VectorType>(Ty))242    return isDenselyPacked(SeqTy->getElementType(), DL);243 244  // For array types, check for padding within members.245  if (ArrayType *SeqTy = dyn_cast<ArrayType>(Ty))246    return isDenselyPacked(SeqTy->getElementType(), DL);247 248  if (!isa<StructType>(Ty))249    return true;250 251  // Check for padding within and between elements of a struct.252  StructType *StructTy = cast<StructType>(Ty);253  const StructLayout *Layout = DL.getStructLayout(StructTy);254  uint64_t StartPos = 0;255  for (unsigned I = 0, E = StructTy->getNumElements(); I < E; ++I) {256    Type *ElTy = StructTy->getElementType(I);257    if (!isDenselyPacked(ElTy, DL))258      return false;259    if (StartPos != Layout->getElementOffsetInBits(I))260      return false;261    StartPos += DL.getTypeAllocSizeInBits(ElTy);262  }263 264  return true;265}266 267/// Get pointer operand of memory accessing instruction. If \p I is268/// not a memory accessing instruction, return nullptr. If \p AllowVolatile,269/// is set to false and the instruction is volatile, return nullptr.270static const Value *getPointerOperand(const Instruction *I,271                                      bool AllowVolatile) {272  if (!AllowVolatile && I->isVolatile())273    return nullptr;274 275  if (auto *LI = dyn_cast<LoadInst>(I)) {276    return LI->getPointerOperand();277  }278 279  if (auto *SI = dyn_cast<StoreInst>(I)) {280    return SI->getPointerOperand();281  }282 283  if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(I)) {284    return CXI->getPointerOperand();285  }286 287  if (auto *RMWI = dyn_cast<AtomicRMWInst>(I)) {288    return RMWI->getPointerOperand();289  }290 291  return nullptr;292}293 294/// Helper function to create a pointer based on \p Ptr, and advanced by \p295/// Offset bytes.296static Value *constructPointer(Value *Ptr, int64_t Offset,297                               IRBuilder<NoFolder> &IRB) {298  LLVM_DEBUG(dbgs() << "Construct pointer: " << *Ptr << " + " << Offset299                    << "-bytes\n");300 301  if (Offset)302    Ptr = IRB.CreatePtrAdd(Ptr, IRB.getInt64(Offset),303                           Ptr->getName() + ".b" + Twine(Offset));304  return Ptr;305}306 307static const Value *308stripAndAccumulateOffsets(Attributor &A, const AbstractAttribute &QueryingAA,309                          const Value *Val, const DataLayout &DL, APInt &Offset,310                          bool GetMinOffset, bool AllowNonInbounds,311                          bool UseAssumed = false) {312 313  auto AttributorAnalysis = [&](Value &V, APInt &ROffset) -> bool {314    const IRPosition &Pos = IRPosition::value(V);315    // Only track dependence if we are going to use the assumed info.316    const AAValueConstantRange *ValueConstantRangeAA =317        A.getAAFor<AAValueConstantRange>(QueryingAA, Pos,318                                         UseAssumed ? DepClassTy::OPTIONAL319                                                    : DepClassTy::NONE);320    if (!ValueConstantRangeAA)321      return false;322    ConstantRange Range = UseAssumed ? ValueConstantRangeAA->getAssumed()323                                     : ValueConstantRangeAA->getKnown();324    if (Range.isFullSet())325      return false;326 327    // We can only use the lower part of the range because the upper part can328    // be higher than what the value can really be.329    if (GetMinOffset)330      ROffset = Range.getSignedMin();331    else332      ROffset = Range.getSignedMax();333    return true;334  };335 336  return Val->stripAndAccumulateConstantOffsets(DL, Offset, AllowNonInbounds,337                                                /* AllowInvariant */ true,338                                                AttributorAnalysis);339}340 341static const Value *342getMinimalBaseOfPointer(Attributor &A, const AbstractAttribute &QueryingAA,343                        const Value *Ptr, int64_t &BytesOffset,344                        const DataLayout &DL, bool AllowNonInbounds = false) {345  APInt OffsetAPInt(DL.getIndexTypeSizeInBits(Ptr->getType()), 0);346  const Value *Base =347      stripAndAccumulateOffsets(A, QueryingAA, Ptr, DL, OffsetAPInt,348                                /* GetMinOffset */ true, AllowNonInbounds);349 350  BytesOffset = OffsetAPInt.getSExtValue();351  return Base;352}353 354/// Clamp the information known for all returned values of a function355/// (identified by \p QueryingAA) into \p S.356template <typename AAType, typename StateType = typename AAType::StateType,357          Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,358          bool RecurseForSelectAndPHI = true>359static void clampReturnedValueStates(360    Attributor &A, const AAType &QueryingAA, StateType &S,361    const IRPosition::CallBaseContext *CBContext = nullptr) {362  LLVM_DEBUG(dbgs() << "[Attributor] Clamp return value states for "363                    << QueryingAA << " into " << S << "\n");364 365  assert((QueryingAA.getIRPosition().getPositionKind() ==366              IRPosition::IRP_RETURNED ||367          QueryingAA.getIRPosition().getPositionKind() ==368              IRPosition::IRP_CALL_SITE_RETURNED) &&369         "Can only clamp returned value states for a function returned or call "370         "site returned position!");371 372  // Use an optional state as there might not be any return values and we want373  // to join (IntegerState::operator&) the state of all there are.374  std::optional<StateType> T;375 376  // Callback for each possibly returned value.377  auto CheckReturnValue = [&](Value &RV) -> bool {378    const IRPosition &RVPos = IRPosition::value(RV, CBContext);379    // If possible, use the hasAssumedIRAttr interface.380    if (Attribute::isEnumAttrKind(IRAttributeKind)) {381      bool IsKnown;382      return AA::hasAssumedIRAttr<IRAttributeKind>(383          A, &QueryingAA, RVPos, DepClassTy::REQUIRED, IsKnown);384    }385 386    const AAType *AA =387        A.getAAFor<AAType>(QueryingAA, RVPos, DepClassTy::REQUIRED);388    if (!AA)389      return false;390    LLVM_DEBUG(dbgs() << "[Attributor] RV: " << RV391                      << " AA: " << AA->getAsStr(&A) << " @ " << RVPos << "\n");392    const StateType &AAS = AA->getState();393    if (!T)394      T = StateType::getBestState(AAS);395    *T &= AAS;396    LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " RV State: " << T397                      << "\n");398    return T->isValidState();399  };400 401  if (!A.checkForAllReturnedValues(CheckReturnValue, QueryingAA,402                                   AA::ValueScope::Intraprocedural,403                                   RecurseForSelectAndPHI))404    S.indicatePessimisticFixpoint();405  else if (T)406    S ^= *T;407}408 409namespace {410/// Helper class for generic deduction: return value -> returned position.411template <typename AAType, typename BaseType,412          typename StateType = typename BaseType::StateType,413          bool PropagateCallBaseContext = false,414          Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,415          bool RecurseForSelectAndPHI = true>416struct AAReturnedFromReturnedValues : public BaseType {417  AAReturnedFromReturnedValues(const IRPosition &IRP, Attributor &A)418      : BaseType(IRP, A) {}419 420  /// See AbstractAttribute::updateImpl(...).421  ChangeStatus updateImpl(Attributor &A) override {422    StateType S(StateType::getBestState(this->getState()));423    clampReturnedValueStates<AAType, StateType, IRAttributeKind,424                             RecurseForSelectAndPHI>(425        A, *this, S,426        PropagateCallBaseContext ? this->getCallBaseContext() : nullptr);427    // TODO: If we know we visited all returned values, thus no are assumed428    // dead, we can take the known information from the state T.429    return clampStateAndIndicateChange<StateType>(this->getState(), S);430  }431};432 433/// Clamp the information known at all call sites for a given argument434/// (identified by \p QueryingAA) into \p S.435template <typename AAType, typename StateType = typename AAType::StateType,436          Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>437static void clampCallSiteArgumentStates(Attributor &A, const AAType &QueryingAA,438                                        StateType &S) {439  LLVM_DEBUG(dbgs() << "[Attributor] Clamp call site argument states for "440                    << QueryingAA << " into " << S << "\n");441 442  assert(QueryingAA.getIRPosition().getPositionKind() ==443             IRPosition::IRP_ARGUMENT &&444         "Can only clamp call site argument states for an argument position!");445 446  // Use an optional state as there might not be any return values and we want447  // to join (IntegerState::operator&) the state of all there are.448  std::optional<StateType> T;449 450  // The argument number which is also the call site argument number.451  unsigned ArgNo = QueryingAA.getIRPosition().getCallSiteArgNo();452 453  auto CallSiteCheck = [&](AbstractCallSite ACS) {454    const IRPosition &ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);455    // Check if a coresponding argument was found or if it is on not associated456    // (which can happen for callback calls).457    if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)458      return false;459 460    // If possible, use the hasAssumedIRAttr interface.461    if (Attribute::isEnumAttrKind(IRAttributeKind)) {462      bool IsKnown;463      return AA::hasAssumedIRAttr<IRAttributeKind>(464          A, &QueryingAA, ACSArgPos, DepClassTy::REQUIRED, IsKnown);465    }466 467    const AAType *AA =468        A.getAAFor<AAType>(QueryingAA, ACSArgPos, DepClassTy::REQUIRED);469    if (!AA)470      return false;471    LLVM_DEBUG(dbgs() << "[Attributor] ACS: " << *ACS.getInstruction()472                      << " AA: " << AA->getAsStr(&A) << " @" << ACSArgPos473                      << "\n");474    const StateType &AAS = AA->getState();475    if (!T)476      T = StateType::getBestState(AAS);477    *T &= AAS;478    LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " CSA State: " << T479                      << "\n");480    return T->isValidState();481  };482 483  bool UsedAssumedInformation = false;484  if (!A.checkForAllCallSites(CallSiteCheck, QueryingAA, true,485                              UsedAssumedInformation))486    S.indicatePessimisticFixpoint();487  else if (T)488    S ^= *T;489}490 491/// This function is the bridge between argument position and the call base492/// context.493template <typename AAType, typename BaseType,494          typename StateType = typename AAType::StateType,495          Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>496bool getArgumentStateFromCallBaseContext(Attributor &A,497                                         BaseType &QueryingAttribute,498                                         IRPosition &Pos, StateType &State) {499  assert((Pos.getPositionKind() == IRPosition::IRP_ARGUMENT) &&500         "Expected an 'argument' position !");501  const CallBase *CBContext = Pos.getCallBaseContext();502  if (!CBContext)503    return false;504 505  int ArgNo = Pos.getCallSiteArgNo();506  assert(ArgNo >= 0 && "Invalid Arg No!");507  const IRPosition CBArgPos = IRPosition::callsite_argument(*CBContext, ArgNo);508 509  // If possible, use the hasAssumedIRAttr interface.510  if (Attribute::isEnumAttrKind(IRAttributeKind)) {511    bool IsKnown;512    return AA::hasAssumedIRAttr<IRAttributeKind>(513        A, &QueryingAttribute, CBArgPos, DepClassTy::REQUIRED, IsKnown);514  }515 516  const auto *AA =517      A.getAAFor<AAType>(QueryingAttribute, CBArgPos, DepClassTy::REQUIRED);518  if (!AA)519    return false;520  const StateType &CBArgumentState =521      static_cast<const StateType &>(AA->getState());522 523  LLVM_DEBUG(dbgs() << "[Attributor] Briding Call site context to argument"524                    << "Position:" << Pos << "CB Arg state:" << CBArgumentState525                    << "\n");526 527  // NOTE: If we want to do call site grouping it should happen here.528  State ^= CBArgumentState;529  return true;530}531 532/// Helper class for generic deduction: call site argument -> argument position.533template <typename AAType, typename BaseType,534          typename StateType = typename AAType::StateType,535          bool BridgeCallBaseContext = false,536          Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>537struct AAArgumentFromCallSiteArguments : public BaseType {538  AAArgumentFromCallSiteArguments(const IRPosition &IRP, Attributor &A)539      : BaseType(IRP, A) {}540 541  /// See AbstractAttribute::updateImpl(...).542  ChangeStatus updateImpl(Attributor &A) override {543    StateType S = StateType::getBestState(this->getState());544 545    if (BridgeCallBaseContext) {546      bool Success =547          getArgumentStateFromCallBaseContext<AAType, BaseType, StateType,548                                              IRAttributeKind>(549              A, *this, this->getIRPosition(), S);550      if (Success)551        return clampStateAndIndicateChange<StateType>(this->getState(), S);552    }553    clampCallSiteArgumentStates<AAType, StateType, IRAttributeKind>(A, *this,554                                                                    S);555 556    // TODO: If we know we visited all incoming values, thus no are assumed557    // dead, we can take the known information from the state T.558    return clampStateAndIndicateChange<StateType>(this->getState(), S);559  }560};561 562/// Helper class for generic replication: function returned -> cs returned.563template <typename AAType, typename BaseType,564          typename StateType = typename BaseType::StateType,565          bool IntroduceCallBaseContext = false,566          Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>567struct AACalleeToCallSite : public BaseType {568  AACalleeToCallSite(const IRPosition &IRP, Attributor &A) : BaseType(IRP, A) {}569 570  /// See AbstractAttribute::updateImpl(...).571  ChangeStatus updateImpl(Attributor &A) override {572    auto IRPKind = this->getIRPosition().getPositionKind();573    assert((IRPKind == IRPosition::IRP_CALL_SITE_RETURNED ||574            IRPKind == IRPosition::IRP_CALL_SITE) &&575           "Can only wrap function returned positions for call site "576           "returned positions!");577    auto &S = this->getState();578 579    CallBase &CB = cast<CallBase>(this->getAnchorValue());580    if (IntroduceCallBaseContext)581      LLVM_DEBUG(dbgs() << "[Attributor] Introducing call base context:" << CB582                        << "\n");583 584    ChangeStatus Changed = ChangeStatus::UNCHANGED;585    auto CalleePred = [&](ArrayRef<const Function *> Callees) {586      for (const Function *Callee : Callees) {587        IRPosition FnPos =588            IRPKind == llvm::IRPosition::IRP_CALL_SITE_RETURNED589                ? IRPosition::returned(*Callee,590                                       IntroduceCallBaseContext ? &CB : nullptr)591                : IRPosition::function(592                      *Callee, IntroduceCallBaseContext ? &CB : nullptr);593        // If possible, use the hasAssumedIRAttr interface.594        if (Attribute::isEnumAttrKind(IRAttributeKind)) {595          bool IsKnown;596          if (!AA::hasAssumedIRAttr<IRAttributeKind>(597                  A, this, FnPos, DepClassTy::REQUIRED, IsKnown))598            return false;599          continue;600        }601 602        const AAType *AA =603            A.getAAFor<AAType>(*this, FnPos, DepClassTy::REQUIRED);604        if (!AA)605          return false;606        Changed |= clampStateAndIndicateChange(S, AA->getState());607        if (S.isAtFixpoint())608          return S.isValidState();609      }610      return true;611    };612    if (!A.checkForAllCallees(CalleePred, *this, CB))613      return S.indicatePessimisticFixpoint();614    return Changed;615  }616};617 618/// Helper function to accumulate uses.619template <class AAType, typename StateType = typename AAType::StateType>620static void followUsesInContext(AAType &AA, Attributor &A,621                                MustBeExecutedContextExplorer &Explorer,622                                const Instruction *CtxI,623                                SetVector<const Use *> &Uses,624                                StateType &State) {625  auto EIt = Explorer.begin(CtxI), EEnd = Explorer.end(CtxI);626  for (unsigned u = 0; u < Uses.size(); ++u) {627    const Use *U = Uses[u];628    if (const Instruction *UserI = dyn_cast<Instruction>(U->getUser())) {629      bool Found = Explorer.findInContextOf(UserI, EIt, EEnd);630      if (Found && AA.followUseInMBEC(A, U, UserI, State))631        Uses.insert_range(llvm::make_pointer_range(UserI->uses()));632    }633  }634}635 636/// Use the must-be-executed-context around \p I to add information into \p S.637/// The AAType class is required to have `followUseInMBEC` method with the638/// following signature and behaviour:639///640/// bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I)641/// U - Underlying use.642/// I - The user of the \p U.643/// Returns true if the value should be tracked transitively.644///645template <class AAType, typename StateType = typename AAType::StateType>646static void followUsesInMBEC(AAType &AA, Attributor &A, StateType &S,647                             Instruction &CtxI) {648  const Value &Val = AA.getIRPosition().getAssociatedValue();649  if (isa<ConstantData>(Val))650    return;651 652  MustBeExecutedContextExplorer *Explorer =653      A.getInfoCache().getMustBeExecutedContextExplorer();654  if (!Explorer)655    return;656 657  // Container for (transitive) uses of the associated value.658  SetVector<const Use *> Uses;659  for (const Use &U : Val.uses())660    Uses.insert(&U);661 662  followUsesInContext<AAType>(AA, A, *Explorer, &CtxI, Uses, S);663 664  if (S.isAtFixpoint())665    return;666 667  SmallVector<const BranchInst *, 4> BrInsts;668  SmallPtrSet<const Instruction *, 16> Visited;669  auto Pred = [&](const Instruction *I) {670    if (!Visited.insert(I).second)671      return false;672    if (const BranchInst *Br = dyn_cast<BranchInst>(I))673      if (Br->isConditional())674        BrInsts.push_back(Br);675    return true;676  };677 678  // Here, accumulate conditional branch instructions in the context. We679  // explore the child paths and collect the known states. The disjunction of680  // those states can be merged to its own state. Let ParentState_i be a state681  // to indicate the known information for an i-th branch instruction in the682  // context. ChildStates are created for its successors respectively.683  //684  // ParentS_1 = ChildS_{1, 1} /\ ChildS_{1, 2} /\ ... /\ ChildS_{1, n_1}685  // ParentS_2 = ChildS_{2, 1} /\ ChildS_{2, 2} /\ ... /\ ChildS_{2, n_2}686  //      ...687  // ParentS_m = ChildS_{m, 1} /\ ChildS_{m, 2} /\ ... /\ ChildS_{m, n_m}688  //689  // Known State |= ParentS_1 \/ ParentS_2 \/... \/ ParentS_m690 691  Explorer->checkForAllContext(&CtxI, Pred);692  while (!BrInsts.empty()) {693    const BranchInst *Br = BrInsts.pop_back_val();694    StateType ParentState;695 696    // The known state of the parent state is a conjunction of children's697    // known states so it is initialized with a best state.698    ParentState.indicateOptimisticFixpoint();699 700    for (const BasicBlock *BB : Br->successors()) {701      StateType ChildState;702      size_t BeforeSize = Uses.size();703      const Instruction *I = &BB->front();704      followUsesInContext(AA, A, *Explorer, I, Uses, ChildState);705 706      // Erase uses which only appear in the child.707      for (auto It = Uses.begin() + BeforeSize; It != Uses.end();)708        It = Uses.erase(It);709 710      ParentState &= ChildState;711 712      // Check for recursive conditional branches.713      Explorer->checkForAllContext(I, Pred);714    }715 716    // Use only known state.717    S += ParentState;718  }719}720} // namespace721 722/// ------------------------ PointerInfo ---------------------------------------723 724namespace llvm {725namespace AA {726namespace PointerInfo {727 728struct State;729 730} // namespace PointerInfo731} // namespace AA732 733/// Helper for AA::PointerInfo::Access DenseMap/Set usage.734template <>735struct DenseMapInfo<AAPointerInfo::Access> : DenseMapInfo<Instruction *> {736  using Access = AAPointerInfo::Access;737  static inline Access getEmptyKey();738  static inline Access getTombstoneKey();739  static unsigned getHashValue(const Access &A);740  static bool isEqual(const Access &LHS, const Access &RHS);741};742 743/// Helper that allows RangeTy as a key in a DenseMap.744template <> struct DenseMapInfo<AA::RangeTy> {745  static inline AA::RangeTy getEmptyKey() {746    auto EmptyKey = DenseMapInfo<int64_t>::getEmptyKey();747    return AA::RangeTy{EmptyKey, EmptyKey};748  }749 750  static inline AA::RangeTy getTombstoneKey() {751    auto TombstoneKey = DenseMapInfo<int64_t>::getTombstoneKey();752    return AA::RangeTy{TombstoneKey, TombstoneKey};753  }754 755  static unsigned getHashValue(const AA::RangeTy &Range) {756    return detail::combineHashValue(757        DenseMapInfo<int64_t>::getHashValue(Range.Offset),758        DenseMapInfo<int64_t>::getHashValue(Range.Size));759  }760 761  static bool isEqual(const AA::RangeTy &A, const AA::RangeTy B) {762    return A == B;763  }764};765 766/// Helper for AA::PointerInfo::Access DenseMap/Set usage ignoring everythign767/// but the instruction768struct AccessAsInstructionInfo : DenseMapInfo<Instruction *> {769  using Base = DenseMapInfo<Instruction *>;770  using Access = AAPointerInfo::Access;771  static inline Access getEmptyKey();772  static inline Access getTombstoneKey();773  static unsigned getHashValue(const Access &A);774  static bool isEqual(const Access &LHS, const Access &RHS);775};776 777} // namespace llvm778 779/// A type to track pointer/struct usage and accesses for AAPointerInfo.780struct AA::PointerInfo::State : public AbstractState {781  /// Return the best possible representable state.782  static State getBestState(const State &SIS) { return State(); }783 784  /// Return the worst possible representable state.785  static State getWorstState(const State &SIS) {786    State R;787    R.indicatePessimisticFixpoint();788    return R;789  }790 791  State() = default;792  State(State &&SIS) = default;793 794  const State &getAssumed() const { return *this; }795 796  /// See AbstractState::isValidState().797  bool isValidState() const override { return BS.isValidState(); }798 799  /// See AbstractState::isAtFixpoint().800  bool isAtFixpoint() const override { return BS.isAtFixpoint(); }801 802  /// See AbstractState::indicateOptimisticFixpoint().803  ChangeStatus indicateOptimisticFixpoint() override {804    BS.indicateOptimisticFixpoint();805    return ChangeStatus::UNCHANGED;806  }807 808  /// See AbstractState::indicatePessimisticFixpoint().809  ChangeStatus indicatePessimisticFixpoint() override {810    BS.indicatePessimisticFixpoint();811    return ChangeStatus::CHANGED;812  }813 814  State &operator=(const State &R) {815    if (this == &R)816      return *this;817    BS = R.BS;818    AccessList = R.AccessList;819    OffsetBins = R.OffsetBins;820    RemoteIMap = R.RemoteIMap;821    ReturnedOffsets = R.ReturnedOffsets;822    return *this;823  }824 825  State &operator=(State &&R) {826    if (this == &R)827      return *this;828    std::swap(BS, R.BS);829    std::swap(AccessList, R.AccessList);830    std::swap(OffsetBins, R.OffsetBins);831    std::swap(RemoteIMap, R.RemoteIMap);832    std::swap(ReturnedOffsets, R.ReturnedOffsets);833    return *this;834  }835 836  /// Add a new Access to the state at offset \p Offset and with size \p Size.837  /// The access is associated with \p I, writes \p Content (if anything), and838  /// is of kind \p Kind. If an Access already exists for the same \p I and same839  /// \p RemoteI, the two are combined, potentially losing information about840  /// offset and size. The resulting access must now be moved from its original841  /// OffsetBin to the bin for its new offset.842  ///843  /// \Returns CHANGED, if the state changed, UNCHANGED otherwise.844  ChangeStatus addAccess(Attributor &A, const AAPointerInfo::RangeList &Ranges,845                         Instruction &I, std::optional<Value *> Content,846                         AAPointerInfo::AccessKind Kind, Type *Ty,847                         Instruction *RemoteI = nullptr);848 849  AAPointerInfo::const_bin_iterator begin() const { return OffsetBins.begin(); }850  AAPointerInfo::const_bin_iterator end() const { return OffsetBins.end(); }851  int64_t numOffsetBins() const { return OffsetBins.size(); }852 853  const AAPointerInfo::Access &getAccess(unsigned Index) const {854    return AccessList[Index];855  }856 857protected:858  // Every memory instruction results in an Access object. We maintain a list of859  // all Access objects that we own, along with the following maps:860  //861  // - OffsetBins: RangeTy -> { Access }862  // - RemoteIMap: RemoteI x LocalI -> Access863  //864  // A RemoteI is any instruction that accesses memory. RemoteI is different865  // from LocalI if and only if LocalI is a call; then RemoteI is some866  // instruction in the callgraph starting from LocalI. Multiple paths in the867  // callgraph from LocalI to RemoteI may produce multiple accesses, but these868  // are all combined into a single Access object. This may result in loss of869  // information in RangeTy in the Access object.870  SmallVector<AAPointerInfo::Access> AccessList;871  AAPointerInfo::OffsetBinsTy OffsetBins;872  DenseMap<const Instruction *, SmallVector<unsigned>> RemoteIMap;873 874  /// Flag to determine if the underlying pointer is reaching a return statement875  /// in the associated function or not. Returns in other functions cause876  /// invalidation.877  AAPointerInfo::OffsetInfo ReturnedOffsets;878 879  /// See AAPointerInfo::forallInterferingAccesses.880  template <typename F>881  bool forallInterferingAccesses(AA::RangeTy Range, F CB) const {882    if (!isValidState() || !ReturnedOffsets.isUnassigned())883      return false;884 885    for (const auto &It : OffsetBins) {886      AA::RangeTy ItRange = It.getFirst();887      if (!Range.mayOverlap(ItRange))888        continue;889      bool IsExact = Range == ItRange && !Range.offsetOrSizeAreUnknown();890      for (auto Index : It.getSecond()) {891        auto &Access = AccessList[Index];892        if (!CB(Access, IsExact))893          return false;894      }895    }896    return true;897  }898 899  /// See AAPointerInfo::forallInterferingAccesses.900  template <typename F>901  bool forallInterferingAccesses(Instruction &I, F CB,902                                 AA::RangeTy &Range) const {903    if (!isValidState() || !ReturnedOffsets.isUnassigned())904      return false;905 906    auto LocalList = RemoteIMap.find(&I);907    if (LocalList == RemoteIMap.end()) {908      return true;909    }910 911    for (unsigned Index : LocalList->getSecond()) {912      for (auto &R : AccessList[Index]) {913        Range &= R;914        if (Range.offsetAndSizeAreUnknown())915          break;916      }917    }918    return forallInterferingAccesses(Range, CB);919  }920 921private:922  /// State to track fixpoint and validity.923  BooleanState BS;924};925 926ChangeStatus AA::PointerInfo::State::addAccess(927    Attributor &A, const AAPointerInfo::RangeList &Ranges, Instruction &I,928    std::optional<Value *> Content, AAPointerInfo::AccessKind Kind, Type *Ty,929    Instruction *RemoteI) {930  RemoteI = RemoteI ? RemoteI : &I;931 932  // Check if we have an access for this instruction, if not, simply add it.933  auto &LocalList = RemoteIMap[RemoteI];934  bool AccExists = false;935  unsigned AccIndex = AccessList.size();936  for (auto Index : LocalList) {937    auto &A = AccessList[Index];938    if (A.getLocalInst() == &I) {939      AccExists = true;940      AccIndex = Index;941      break;942    }943  }944 945  auto AddToBins = [&](const AAPointerInfo::RangeList &ToAdd) {946    LLVM_DEBUG(if (ToAdd.size()) dbgs()947                   << "[AAPointerInfo] Inserting access in new offset bins\n";);948 949    for (auto Key : ToAdd) {950      LLVM_DEBUG(dbgs() << "    key " << Key << "\n");951      OffsetBins[Key].insert(AccIndex);952    }953  };954 955  if (!AccExists) {956    AccessList.emplace_back(&I, RemoteI, Ranges, Content, Kind, Ty);957    assert((AccessList.size() == AccIndex + 1) &&958           "New Access should have been at AccIndex");959    LocalList.push_back(AccIndex);960    AddToBins(AccessList[AccIndex].getRanges());961    return ChangeStatus::CHANGED;962  }963 964  // Combine the new Access with the existing Access, and then update the965  // mapping in the offset bins.966  AAPointerInfo::Access Acc(&I, RemoteI, Ranges, Content, Kind, Ty);967  auto &Current = AccessList[AccIndex];968  auto Before = Current;969  Current &= Acc;970  if (Current == Before)971    return ChangeStatus::UNCHANGED;972 973  auto &ExistingRanges = Before.getRanges();974  auto &NewRanges = Current.getRanges();975 976  // Ranges that are in the old access but not the new access need to be removed977  // from the offset bins.978  AAPointerInfo::RangeList ToRemove;979  AAPointerInfo::RangeList::set_difference(ExistingRanges, NewRanges, ToRemove);980  LLVM_DEBUG(if (ToRemove.size()) dbgs()981                 << "[AAPointerInfo] Removing access from old offset bins\n";);982 983  for (auto Key : ToRemove) {984    LLVM_DEBUG(dbgs() << "    key " << Key << "\n");985    assert(OffsetBins.count(Key) && "Existing Access must be in some bin.");986    auto &Bin = OffsetBins[Key];987    assert(Bin.count(AccIndex) &&988           "Expected bin to actually contain the Access.");989    Bin.erase(AccIndex);990  }991 992  // Ranges that are in the new access but not the old access need to be added993  // to the offset bins.994  AAPointerInfo::RangeList ToAdd;995  AAPointerInfo::RangeList::set_difference(NewRanges, ExistingRanges, ToAdd);996  AddToBins(ToAdd);997  return ChangeStatus::CHANGED;998}999 1000namespace {1001 1002#ifndef NDEBUG1003static raw_ostream &operator<<(raw_ostream &OS,1004                               const AAPointerInfo::OffsetInfo &OI) {1005  OS << llvm::interleaved_array(OI);1006  return OS;1007}1008#endif // NDEBUG1009 1010struct AAPointerInfoImpl1011    : public StateWrapper<AA::PointerInfo::State, AAPointerInfo> {1012  using BaseTy = StateWrapper<AA::PointerInfo::State, AAPointerInfo>;1013  AAPointerInfoImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}1014 1015  /// See AbstractAttribute::getAsStr().1016  const std::string getAsStr(Attributor *A) const override {1017    return std::string("PointerInfo ") +1018           (isValidState() ? (std::string("#") +1019                              std::to_string(OffsetBins.size()) + " bins")1020                           : "<invalid>") +1021           (reachesReturn()1022                ? (" (returned:" +1023                   join(map_range(ReturnedOffsets,1024                                  [](int64_t O) { return std::to_string(O); }),1025                        ", ") +1026                   ")")1027                : "");1028  }1029 1030  /// See AbstractAttribute::manifest(...).1031  ChangeStatus manifest(Attributor &A) override {1032    return AAPointerInfo::manifest(A);1033  }1034 1035  const_bin_iterator begin() const override { return State::begin(); }1036  const_bin_iterator end() const override { return State::end(); }1037  int64_t numOffsetBins() const override { return State::numOffsetBins(); }1038  bool reachesReturn() const override {1039    return !ReturnedOffsets.isUnassigned();1040  }1041  void addReturnedOffsetsTo(OffsetInfo &OI) const override {1042    if (ReturnedOffsets.isUnknown()) {1043      OI.setUnknown();1044      return;1045    }1046 1047    OffsetInfo MergedOI;1048    for (auto Offset : ReturnedOffsets) {1049      OffsetInfo TmpOI = OI;1050      TmpOI.addToAll(Offset);1051      MergedOI.merge(TmpOI);1052    }1053    OI = std::move(MergedOI);1054  }1055 1056  ChangeStatus setReachesReturn(const OffsetInfo &ReachedReturnedOffsets) {1057    if (ReturnedOffsets.isUnknown())1058      return ChangeStatus::UNCHANGED;1059    if (ReachedReturnedOffsets.isUnknown()) {1060      ReturnedOffsets.setUnknown();1061      return ChangeStatus::CHANGED;1062    }1063    if (ReturnedOffsets.merge(ReachedReturnedOffsets))1064      return ChangeStatus::CHANGED;1065    return ChangeStatus::UNCHANGED;1066  }1067 1068  bool forallInterferingAccesses(1069      AA::RangeTy Range,1070      function_ref<bool(const AAPointerInfo::Access &, bool)> CB)1071      const override {1072    return State::forallInterferingAccesses(Range, CB);1073  }1074 1075  bool forallInterferingAccesses(1076      Attributor &A, const AbstractAttribute &QueryingAA, Instruction &I,1077      bool FindInterferingWrites, bool FindInterferingReads,1078      function_ref<bool(const Access &, bool)> UserCB, bool &HasBeenWrittenTo,1079      AA::RangeTy &Range,1080      function_ref<bool(const Access &)> SkipCB) const override {1081    HasBeenWrittenTo = false;1082 1083    SmallPtrSet<const Access *, 8> DominatingWrites;1084    SmallVector<std::pair<const Access *, bool>, 8> InterferingAccesses;1085 1086    Function &Scope = *I.getFunction();1087    bool IsKnownNoSync;1088    bool IsAssumedNoSync = AA::hasAssumedIRAttr<Attribute::NoSync>(1089        A, &QueryingAA, IRPosition::function(Scope), DepClassTy::OPTIONAL,1090        IsKnownNoSync);1091    const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(1092        IRPosition::function(Scope), &QueryingAA, DepClassTy::NONE);1093    bool AllInSameNoSyncFn = IsAssumedNoSync;1094    bool InstIsExecutedByInitialThreadOnly =1095        ExecDomainAA && ExecDomainAA->isExecutedByInitialThreadOnly(I);1096 1097    // If the function is not ending in aligned barriers, we need the stores to1098    // be in aligned barriers. The load being in one is not sufficient since the1099    // store might be executed by a thread that disappears after, causing the1100    // aligned barrier guarding the load to unblock and the load to read a value1101    // that has no CFG path to the load.1102    bool InstIsExecutedInAlignedRegion =1103        FindInterferingReads && ExecDomainAA &&1104        ExecDomainAA->isExecutedInAlignedRegion(A, I);1105 1106    if (InstIsExecutedInAlignedRegion || InstIsExecutedByInitialThreadOnly)1107      A.recordDependence(*ExecDomainAA, QueryingAA, DepClassTy::OPTIONAL);1108 1109    InformationCache &InfoCache = A.getInfoCache();1110    bool IsThreadLocalObj =1111        AA::isAssumedThreadLocalObject(A, getAssociatedValue(), *this);1112 1113    // Helper to determine if we need to consider threading, which we cannot1114    // right now. However, if the function is (assumed) nosync or the thread1115    // executing all instructions is the main thread only we can ignore1116    // threading. Also, thread-local objects do not require threading reasoning.1117    // Finally, we can ignore threading if either access is executed in an1118    // aligned region.1119    auto CanIgnoreThreadingForInst = [&](const Instruction &I) -> bool {1120      if (IsThreadLocalObj || AllInSameNoSyncFn)1121        return true;1122      const auto *FnExecDomainAA =1123          I.getFunction() == &Scope1124              ? ExecDomainAA1125              : A.lookupAAFor<AAExecutionDomain>(1126                    IRPosition::function(*I.getFunction()), &QueryingAA,1127                    DepClassTy::NONE);1128      if (!FnExecDomainAA)1129        return false;1130      if (InstIsExecutedInAlignedRegion ||1131          (FindInterferingWrites &&1132           FnExecDomainAA->isExecutedInAlignedRegion(A, I))) {1133        A.recordDependence(*FnExecDomainAA, QueryingAA, DepClassTy::OPTIONAL);1134        return true;1135      }1136      if (InstIsExecutedByInitialThreadOnly &&1137          FnExecDomainAA->isExecutedByInitialThreadOnly(I)) {1138        A.recordDependence(*FnExecDomainAA, QueryingAA, DepClassTy::OPTIONAL);1139        return true;1140      }1141      return false;1142    };1143 1144    // Helper to determine if the access is executed by the same thread as the1145    // given instruction, for now it is sufficient to avoid any potential1146    // threading effects as we cannot deal with them anyway.1147    auto CanIgnoreThreading = [&](const Access &Acc) -> bool {1148      return CanIgnoreThreadingForInst(*Acc.getRemoteInst()) ||1149             (Acc.getRemoteInst() != Acc.getLocalInst() &&1150              CanIgnoreThreadingForInst(*Acc.getLocalInst()));1151    };1152 1153    // TODO: Use inter-procedural reachability and dominance.1154    bool IsKnownNoRecurse;1155    AA::hasAssumedIRAttr<Attribute::NoRecurse>(1156        A, this, IRPosition::function(Scope), DepClassTy::OPTIONAL,1157        IsKnownNoRecurse);1158 1159    // TODO: Use reaching kernels from AAKernelInfo (or move it to1160    // AAExecutionDomain) such that we allow scopes other than kernels as long1161    // as the reaching kernels are disjoint.1162    bool InstInKernel = A.getInfoCache().isKernel(Scope);1163    bool ObjHasKernelLifetime = false;1164    const bool UseDominanceReasoning =1165        FindInterferingWrites && IsKnownNoRecurse;1166    const DominatorTree *DT =1167        InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(Scope);1168 1169    // Helper to check if a value has "kernel lifetime", that is it will not1170    // outlive a GPU kernel. This is true for shared, constant, and local1171    // globals on AMD and NVIDIA GPUs.1172    auto HasKernelLifetime = [&](Value *V, Module &M) {1173      if (!AA::isGPU(M))1174        return false;1175      switch (AA::GPUAddressSpace(V->getType()->getPointerAddressSpace())) {1176      case AA::GPUAddressSpace::Shared:1177      case AA::GPUAddressSpace::Constant:1178      case AA::GPUAddressSpace::Local:1179        return true;1180      default:1181        return false;1182      };1183    };1184 1185    // The IsLiveInCalleeCB will be used by the AA::isPotentiallyReachable query1186    // to determine if we should look at reachability from the callee. For1187    // certain pointers we know the lifetime and we do not have to step into the1188    // callee to determine reachability as the pointer would be dead in the1189    // callee. See the conditional initialization below.1190    std::function<bool(const Function &)> IsLiveInCalleeCB;1191 1192    if (auto *AI = dyn_cast<AllocaInst>(&getAssociatedValue())) {1193      // If the alloca containing function is not recursive the alloca1194      // must be dead in the callee.1195      const Function *AIFn = AI->getFunction();1196      ObjHasKernelLifetime = A.getInfoCache().isKernel(*AIFn);1197      bool IsKnownNoRecurse;1198      if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(1199              A, this, IRPosition::function(*AIFn), DepClassTy::OPTIONAL,1200              IsKnownNoRecurse)) {1201        IsLiveInCalleeCB = [AIFn](const Function &Fn) { return AIFn != &Fn; };1202      }1203    } else if (auto *GV = dyn_cast<GlobalValue>(&getAssociatedValue())) {1204      // If the global has kernel lifetime we can stop if we reach a kernel1205      // as it is "dead" in the (unknown) callees.1206      ObjHasKernelLifetime = HasKernelLifetime(GV, *GV->getParent());1207      if (ObjHasKernelLifetime)1208        IsLiveInCalleeCB = [&A](const Function &Fn) {1209          return !A.getInfoCache().isKernel(Fn);1210        };1211    }1212 1213    // Set of accesses/instructions that will overwrite the result and are1214    // therefore blockers in the reachability traversal.1215    AA::InstExclusionSetTy ExclusionSet;1216 1217    auto AccessCB = [&](const Access &Acc, bool Exact) {1218      Function *AccScope = Acc.getRemoteInst()->getFunction();1219      bool AccInSameScope = AccScope == &Scope;1220 1221      // If the object has kernel lifetime we can ignore accesses only reachable1222      // by other kernels. For now we only skip accesses *in* other kernels.1223      if (InstInKernel && ObjHasKernelLifetime && !AccInSameScope &&1224          A.getInfoCache().isKernel(*AccScope))1225        return true;1226 1227      if (Exact && Acc.isMustAccess() && Acc.getRemoteInst() != &I) {1228        if (Acc.isWrite() || (isa<LoadInst>(I) && Acc.isWriteOrAssumption()))1229          ExclusionSet.insert(Acc.getRemoteInst());1230      }1231 1232      if ((!FindInterferingWrites || !Acc.isWriteOrAssumption()) &&1233          (!FindInterferingReads || !Acc.isRead()))1234        return true;1235 1236      bool Dominates = FindInterferingWrites && DT && Exact &&1237                       Acc.isMustAccess() && AccInSameScope &&1238                       DT->dominates(Acc.getRemoteInst(), &I);1239      if (Dominates)1240        DominatingWrites.insert(&Acc);1241 1242      // Track if all interesting accesses are in the same `nosync` function as1243      // the given instruction.1244      AllInSameNoSyncFn &= Acc.getRemoteInst()->getFunction() == &Scope;1245 1246      InterferingAccesses.push_back({&Acc, Exact});1247      return true;1248    };1249    if (!State::forallInterferingAccesses(I, AccessCB, Range))1250      return false;1251 1252    HasBeenWrittenTo = !DominatingWrites.empty();1253 1254    // Dominating writes form a chain, find the least/lowest member.1255    Instruction *LeastDominatingWriteInst = nullptr;1256    for (const Access *Acc : DominatingWrites) {1257      if (!LeastDominatingWriteInst) {1258        LeastDominatingWriteInst = Acc->getRemoteInst();1259      } else if (DT->dominates(LeastDominatingWriteInst,1260                               Acc->getRemoteInst())) {1261        LeastDominatingWriteInst = Acc->getRemoteInst();1262      }1263    }1264 1265    // Helper to determine if we can skip a specific write access.1266    auto CanSkipAccess = [&](const Access &Acc, bool Exact) {1267      if (SkipCB && SkipCB(Acc))1268        return true;1269      if (!CanIgnoreThreading(Acc))1270        return false;1271 1272      // Check read (RAW) dependences and write (WAR) dependences as necessary.1273      // If we successfully excluded all effects we are interested in, the1274      // access can be skipped.1275      bool ReadChecked = !FindInterferingReads;1276      bool WriteChecked = !FindInterferingWrites;1277 1278      // If the instruction cannot reach the access, the former does not1279      // interfere with what the access reads.1280      if (!ReadChecked) {1281        if (!AA::isPotentiallyReachable(A, I, *Acc.getRemoteInst(), QueryingAA,1282                                        &ExclusionSet, IsLiveInCalleeCB))1283          ReadChecked = true;1284      }1285      // If the instruction cannot be reach from the access, the latter does not1286      // interfere with what the instruction reads.1287      if (!WriteChecked) {1288        if (!AA::isPotentiallyReachable(A, *Acc.getRemoteInst(), I, QueryingAA,1289                                        &ExclusionSet, IsLiveInCalleeCB))1290          WriteChecked = true;1291      }1292 1293      // If we still might be affected by the write of the access but there are1294      // dominating writes in the function of the instruction1295      // (HasBeenWrittenTo), we can try to reason that the access is overwritten1296      // by them. This would have happend above if they are all in the same1297      // function, so we only check the inter-procedural case. Effectively, we1298      // want to show that there is no call after the dominting write that might1299      // reach the access, and when it returns reach the instruction with the1300      // updated value. To this end, we iterate all call sites, check if they1301      // might reach the instruction without going through another access1302      // (ExclusionSet) and at the same time might reach the access. However,1303      // that is all part of AAInterFnReachability.1304      if (!WriteChecked && HasBeenWrittenTo &&1305          Acc.getRemoteInst()->getFunction() != &Scope) {1306 1307        const auto *FnReachabilityAA = A.getAAFor<AAInterFnReachability>(1308            QueryingAA, IRPosition::function(Scope), DepClassTy::OPTIONAL);1309        if (FnReachabilityAA) {1310          // Without going backwards in the call tree, can we reach the access1311          // from the least dominating write. Do not allow to pass the1312          // instruction itself either.1313          bool Inserted = ExclusionSet.insert(&I).second;1314 1315          if (!FnReachabilityAA->instructionCanReach(1316                  A, *LeastDominatingWriteInst,1317                  *Acc.getRemoteInst()->getFunction(), &ExclusionSet))1318            WriteChecked = true;1319 1320          if (Inserted)1321            ExclusionSet.erase(&I);1322        }1323      }1324 1325      if (ReadChecked && WriteChecked)1326        return true;1327 1328      if (!DT || !UseDominanceReasoning)1329        return false;1330      if (!DominatingWrites.count(&Acc))1331        return false;1332      return LeastDominatingWriteInst != Acc.getRemoteInst();1333    };1334 1335    // Run the user callback on all accesses we cannot skip and return if1336    // that succeeded for all or not.1337    for (auto &It : InterferingAccesses) {1338      if ((!AllInSameNoSyncFn && !IsThreadLocalObj && !ExecDomainAA) ||1339          !CanSkipAccess(*It.first, It.second)) {1340        if (!UserCB(*It.first, It.second))1341          return false;1342      }1343    }1344    return true;1345  }1346 1347  ChangeStatus translateAndAddStateFromCallee(Attributor &A,1348                                              const AAPointerInfo &OtherAA,1349                                              CallBase &CB) {1350    using namespace AA::PointerInfo;1351    if (!OtherAA.getState().isValidState() || !isValidState())1352      return indicatePessimisticFixpoint();1353 1354    ChangeStatus Changed = ChangeStatus::UNCHANGED;1355    const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);1356    bool IsByval = OtherAAImpl.getAssociatedArgument()->hasByValAttr();1357    Changed |= setReachesReturn(OtherAAImpl.ReturnedOffsets);1358 1359    // Combine the accesses bin by bin.1360    const auto &State = OtherAAImpl.getState();1361    for (const auto &It : State) {1362      for (auto Index : It.getSecond()) {1363        const auto &RAcc = State.getAccess(Index);1364        if (IsByval && !RAcc.isRead())1365          continue;1366        bool UsedAssumedInformation = false;1367        AccessKind AK = RAcc.getKind();1368        auto Content = A.translateArgumentToCallSiteContent(1369            RAcc.getContent(), CB, *this, UsedAssumedInformation);1370        AK = AccessKind(AK & (IsByval ? AccessKind::AK_R : AccessKind::AK_RW));1371        AK = AccessKind(AK | (RAcc.isMayAccess() ? AK_MAY : AK_MUST));1372 1373        Changed |= addAccess(A, RAcc.getRanges(), CB, Content, AK,1374                             RAcc.getType(), RAcc.getRemoteInst());1375      }1376    }1377    return Changed;1378  }1379 1380  ChangeStatus translateAndAddState(Attributor &A, const AAPointerInfo &OtherAA,1381                                    const OffsetInfo &Offsets, CallBase &CB,1382                                    bool IsMustAcc) {1383    using namespace AA::PointerInfo;1384    if (!OtherAA.getState().isValidState() || !isValidState())1385      return indicatePessimisticFixpoint();1386 1387    const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);1388 1389    // Combine the accesses bin by bin.1390    ChangeStatus Changed = ChangeStatus::UNCHANGED;1391    const auto &State = OtherAAImpl.getState();1392    for (const auto &It : State) {1393      for (auto Index : It.getSecond()) {1394        const auto &RAcc = State.getAccess(Index);1395        if (!IsMustAcc && RAcc.isAssumption())1396          continue;1397        for (auto Offset : Offsets) {1398          auto NewRanges = Offset == AA::RangeTy::Unknown1399                               ? AA::RangeTy::getUnknown()1400                               : RAcc.getRanges();1401          if (!NewRanges.isUnknown()) {1402            NewRanges.addToAllOffsets(Offset);1403          }1404          AccessKind AK = RAcc.getKind();1405          if (!IsMustAcc)1406            AK = AccessKind((AK & ~AK_MUST) | AK_MAY);1407          Changed |= addAccess(A, NewRanges, CB, RAcc.getContent(), AK,1408                               RAcc.getType(), RAcc.getRemoteInst());1409        }1410      }1411    }1412    return Changed;1413  }1414 1415  /// Statistic tracking for all AAPointerInfo implementations.1416  /// See AbstractAttribute::trackStatistics().1417  void trackPointerInfoStatistics(const IRPosition &IRP) const {}1418 1419  /// Dump the state into \p O.1420  void dumpState(raw_ostream &O) {1421    for (auto &It : OffsetBins) {1422      O << "[" << It.first.Offset << "-" << It.first.Offset + It.first.Size1423        << "] : " << It.getSecond().size() << "\n";1424      for (auto AccIndex : It.getSecond()) {1425        auto &Acc = AccessList[AccIndex];1426        O << "     - " << Acc.getKind() << " - " << *Acc.getLocalInst() << "\n";1427        if (Acc.getLocalInst() != Acc.getRemoteInst())1428          O << "     -->                         " << *Acc.getRemoteInst()1429            << "\n";1430        if (!Acc.isWrittenValueYetUndetermined()) {1431          if (isa_and_nonnull<Function>(Acc.getWrittenValue()))1432            O << "       - c: func " << Acc.getWrittenValue()->getName()1433              << "\n";1434          else if (Acc.getWrittenValue())1435            O << "       - c: " << *Acc.getWrittenValue() << "\n";1436          else1437            O << "       - c: <unknown>\n";1438        }1439      }1440    }1441  }1442};1443 1444struct AAPointerInfoFloating : public AAPointerInfoImpl {1445  using AccessKind = AAPointerInfo::AccessKind;1446  AAPointerInfoFloating(const IRPosition &IRP, Attributor &A)1447      : AAPointerInfoImpl(IRP, A) {}1448 1449  /// Deal with an access and signal if it was handled successfully.1450  bool handleAccess(Attributor &A, Instruction &I,1451                    std::optional<Value *> Content, AccessKind Kind,1452                    OffsetInfo::VecTy &Offsets, ChangeStatus &Changed,1453                    Type &Ty) {1454    using namespace AA::PointerInfo;1455    auto Size = AA::RangeTy::Unknown;1456    const DataLayout &DL = A.getDataLayout();1457    TypeSize AccessSize = DL.getTypeStoreSize(&Ty);1458    if (!AccessSize.isScalable())1459      Size = AccessSize.getFixedValue();1460 1461    // Make a strictly ascending list of offsets as required by addAccess()1462    SmallVector<int64_t> OffsetsSorted(Offsets.begin(), Offsets.end());1463    llvm::sort(OffsetsSorted);1464 1465    VectorType *VT = dyn_cast<VectorType>(&Ty);1466    if (!VT || VT->getElementCount().isScalable() ||1467        !Content.value_or(nullptr) || !isa<Constant>(*Content) ||1468        (*Content)->getType() != VT ||1469        DL.getTypeStoreSize(VT->getElementType()).isScalable()) {1470      Changed =1471          Changed | addAccess(A, {OffsetsSorted, Size}, I, Content, Kind, &Ty);1472    } else {1473      // Handle vector stores with constant content element-wise.1474      // TODO: We could look for the elements or create instructions1475      //       representing them.1476      // TODO: We need to push the Content into the range abstraction1477      //       (AA::RangeTy) to allow different content values for different1478      //       ranges. ranges. Hence, support vectors storing different values.1479      Type *ElementType = VT->getElementType();1480      int64_t ElementSize = DL.getTypeStoreSize(ElementType).getFixedValue();1481      auto *ConstContent = cast<Constant>(*Content);1482      Type *Int32Ty = Type::getInt32Ty(ElementType->getContext());1483      SmallVector<int64_t> ElementOffsets(Offsets.begin(), Offsets.end());1484 1485      for (int i = 0, e = VT->getElementCount().getFixedValue(); i != e; ++i) {1486        Value *ElementContent = ConstantExpr::getExtractElement(1487            ConstContent, ConstantInt::get(Int32Ty, i));1488 1489        // Add the element access.1490        Changed = Changed | addAccess(A, {ElementOffsets, ElementSize}, I,1491                                      ElementContent, Kind, ElementType);1492 1493        // Advance the offsets for the next element.1494        for (auto &ElementOffset : ElementOffsets)1495          ElementOffset += ElementSize;1496      }1497    }1498    return true;1499  };1500 1501  /// See AbstractAttribute::updateImpl(...).1502  ChangeStatus updateImpl(Attributor &A) override;1503 1504  /// If the indices to \p GEP can be traced to constants, incorporate all1505  /// of these into \p UsrOI.1506  ///1507  /// \return true iff \p UsrOI is updated.1508  bool collectConstantsForGEP(Attributor &A, const DataLayout &DL,1509                              OffsetInfo &UsrOI, const OffsetInfo &PtrOI,1510                              const GEPOperator *GEP);1511 1512  /// See AbstractAttribute::trackStatistics()1513  void trackStatistics() const override {1514    AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());1515  }1516};1517 1518bool AAPointerInfoFloating::collectConstantsForGEP(Attributor &A,1519                                                   const DataLayout &DL,1520                                                   OffsetInfo &UsrOI,1521                                                   const OffsetInfo &PtrOI,1522                                                   const GEPOperator *GEP) {1523  unsigned BitWidth = DL.getIndexTypeSizeInBits(GEP->getType());1524  SmallMapVector<Value *, APInt, 4> VariableOffsets;1525  APInt ConstantOffset(BitWidth, 0);1526 1527  assert(!UsrOI.isUnknown() && !PtrOI.isUnknown() &&1528         "Don't look for constant values if the offset has already been "1529         "determined to be unknown.");1530 1531  if (!GEP->collectOffset(DL, BitWidth, VariableOffsets, ConstantOffset)) {1532    UsrOI.setUnknown();1533    return true;1534  }1535 1536  LLVM_DEBUG(dbgs() << "[AAPointerInfo] GEP offset is "1537                    << (VariableOffsets.empty() ? "" : "not") << " constant "1538                    << *GEP << "\n");1539 1540  auto Union = PtrOI;1541  Union.addToAll(ConstantOffset.getSExtValue());1542 1543  // Each VI in VariableOffsets has a set of potential constant values. Every1544  // combination of elements, picked one each from these sets, is separately1545  // added to the original set of offsets, thus resulting in more offsets.1546  for (const auto &VI : VariableOffsets) {1547    auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(1548        *this, IRPosition::value(*VI.first), DepClassTy::OPTIONAL);1549    if (!PotentialConstantsAA || !PotentialConstantsAA->isValidState()) {1550      UsrOI.setUnknown();1551      return true;1552    }1553 1554    // UndefValue is treated as a zero, which leaves Union as is.1555    if (PotentialConstantsAA->undefIsContained())1556      continue;1557 1558    // We need at least one constant in every set to compute an actual offset.1559    // Otherwise, we end up pessimizing AAPointerInfo by respecting offsets that1560    // don't actually exist. In other words, the absence of constant values1561    // implies that the operation can be assumed dead for now.1562    auto &AssumedSet = PotentialConstantsAA->getAssumedSet();1563    if (AssumedSet.empty())1564      return false;1565 1566    OffsetInfo Product;1567    for (const auto &ConstOffset : AssumedSet) {1568      auto CopyPerOffset = Union;1569      CopyPerOffset.addToAll(ConstOffset.getSExtValue() *1570                             VI.second.getZExtValue());1571      Product.merge(CopyPerOffset);1572    }1573    Union = Product;1574  }1575 1576  UsrOI = std::move(Union);1577  return true;1578}1579 1580ChangeStatus AAPointerInfoFloating::updateImpl(Attributor &A) {1581  using namespace AA::PointerInfo;1582  ChangeStatus Changed = ChangeStatus::UNCHANGED;1583  const DataLayout &DL = A.getDataLayout();1584  Value &AssociatedValue = getAssociatedValue();1585 1586  DenseMap<Value *, OffsetInfo> OffsetInfoMap;1587  OffsetInfoMap[&AssociatedValue].insert(0);1588 1589  auto HandlePassthroughUser = [&](Value *Usr, Value *CurPtr, bool &Follow) {1590    // One does not simply walk into a map and assign a reference to a possibly1591    // new location. That can cause an invalidation before the assignment1592    // happens, like so:1593    //1594    //   OffsetInfoMap[Usr] = OffsetInfoMap[CurPtr]; /* bad idea! */1595    //1596    // The RHS is a reference that may be invalidated by an insertion caused by1597    // the LHS. So we ensure that the side-effect of the LHS happens first.1598 1599    assert(OffsetInfoMap.contains(CurPtr) &&1600           "CurPtr does not exist in the map!");1601 1602    auto &UsrOI = OffsetInfoMap[Usr];1603    auto &PtrOI = OffsetInfoMap[CurPtr];1604    assert(!PtrOI.isUnassigned() &&1605           "Cannot pass through if the input Ptr was not visited!");1606    UsrOI.merge(PtrOI);1607    Follow = true;1608    return true;1609  };1610 1611  auto UsePred = [&](const Use &U, bool &Follow) -> bool {1612    Value *CurPtr = U.get();1613    User *Usr = U.getUser();1614    LLVM_DEBUG(dbgs() << "[AAPointerInfo] Analyze " << *CurPtr << " in " << *Usr1615                      << "\n");1616    assert(OffsetInfoMap.count(CurPtr) &&1617           "The current pointer offset should have been seeded!");1618    assert(!OffsetInfoMap[CurPtr].isUnassigned() &&1619           "Current pointer should be assigned");1620 1621    if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Usr)) {1622      if (CE->isCast())1623        return HandlePassthroughUser(Usr, CurPtr, Follow);1624      if (!isa<GEPOperator>(CE)) {1625        LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled constant user " << *CE1626                          << "\n");1627        return false;1628      }1629    }1630    if (auto *GEP = dyn_cast<GEPOperator>(Usr)) {1631      // Note the order here, the Usr access might change the map, CurPtr is1632      // already in it though.1633      auto &UsrOI = OffsetInfoMap[Usr];1634      auto &PtrOI = OffsetInfoMap[CurPtr];1635 1636      if (UsrOI.isUnknown())1637        return true;1638 1639      if (PtrOI.isUnknown()) {1640        Follow = true;1641        UsrOI.setUnknown();1642        return true;1643      }1644 1645      Follow = collectConstantsForGEP(A, DL, UsrOI, PtrOI, GEP);1646      return true;1647    }1648    if (isa<PtrToIntInst>(Usr))1649      return false;1650    if (isa<CastInst>(Usr) || isa<SelectInst>(Usr))1651      return HandlePassthroughUser(Usr, CurPtr, Follow);1652    // Returns are allowed if they are in the associated functions. Users can1653    // then check the call site return. Returns from other functions can't be1654    // tracked and are cause for invalidation.1655    if (auto *RI = dyn_cast<ReturnInst>(Usr)) {1656      if (RI->getFunction() == getAssociatedFunction()) {1657        auto &PtrOI = OffsetInfoMap[CurPtr];1658        Changed |= setReachesReturn(PtrOI);1659        return true;1660      }1661      return false;1662    }1663 1664    // For PHIs we need to take care of the recurrence explicitly as the value1665    // might change while we iterate through a loop. For now, we give up if1666    // the PHI is not invariant.1667    if (auto *PHI = dyn_cast<PHINode>(Usr)) {1668      // Note the order here, the Usr access might change the map, CurPtr is1669      // already in it though.1670      auto [PhiIt, IsFirstPHIUser] = OffsetInfoMap.try_emplace(PHI);1671      auto &UsrOI = PhiIt->second;1672      auto &PtrOI = OffsetInfoMap[CurPtr];1673 1674      // Check if the PHI operand has already an unknown offset as we can't1675      // improve on that anymore.1676      if (PtrOI.isUnknown()) {1677        LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand offset unknown "1678                          << *CurPtr << " in " << *PHI << "\n");1679        Follow = !UsrOI.isUnknown();1680        UsrOI.setUnknown();1681        return true;1682      }1683 1684      // Check if the PHI is invariant (so far).1685      if (UsrOI == PtrOI) {1686        assert(!PtrOI.isUnassigned() &&1687               "Cannot assign if the current Ptr was not visited!");1688        LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant (so far)");1689        return true;1690      }1691 1692      // Check if the PHI operand can be traced back to AssociatedValue.1693      APInt Offset(1694          DL.getIndexSizeInBits(CurPtr->getType()->getPointerAddressSpace()),1695          0);1696      Value *CurPtrBase = CurPtr->stripAndAccumulateConstantOffsets(1697          DL, Offset, /* AllowNonInbounds */ true);1698      auto It = OffsetInfoMap.find(CurPtrBase);1699      if (It == OffsetInfoMap.end()) {1700        LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand is too complex "1701                          << *CurPtr << " in " << *PHI1702                          << " (base: " << *CurPtrBase << ")\n");1703        UsrOI.setUnknown();1704        Follow = true;1705        return true;1706      }1707 1708      // Check if the PHI operand is not dependent on the PHI itself. Every1709      // recurrence is a cyclic net of PHIs in the data flow, and has an1710      // equivalent Cycle in the control flow. One of those PHIs must be in the1711      // header of that control flow Cycle. This is independent of the choice of1712      // Cycles reported by CycleInfo. It is sufficient to check the PHIs in1713      // every Cycle header; if such a node is marked unknown, this will1714      // eventually propagate through the whole net of PHIs in the recurrence.1715      const auto *CI =1716          A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(1717              *PHI->getFunction());1718      if (mayBeInCycle(CI, cast<Instruction>(Usr), /* HeaderOnly */ true)) {1719        auto BaseOI = It->getSecond();1720        BaseOI.addToAll(Offset.getZExtValue());1721        if (IsFirstPHIUser || BaseOI == UsrOI) {1722          LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant " << *CurPtr1723                            << " in " << *Usr << "\n");1724          return HandlePassthroughUser(Usr, CurPtr, Follow);1725        }1726 1727        LLVM_DEBUG(1728            dbgs() << "[AAPointerInfo] PHI operand pointer offset mismatch "1729                   << *CurPtr << " in " << *PHI << "\n");1730        UsrOI.setUnknown();1731        Follow = true;1732        return true;1733      }1734 1735      UsrOI.merge(PtrOI);1736      Follow = true;1737      return true;1738    }1739 1740    if (auto *LoadI = dyn_cast<LoadInst>(Usr)) {1741      // If the access is to a pointer that may or may not be the associated1742      // value, e.g. due to a PHI, we cannot assume it will be read.1743      AccessKind AK = AccessKind::AK_R;1744      if (getUnderlyingObject(CurPtr) == &AssociatedValue)1745        AK = AccessKind(AK | AccessKind::AK_MUST);1746      else1747        AK = AccessKind(AK | AccessKind::AK_MAY);1748      if (!handleAccess(A, *LoadI, /* Content */ nullptr, AK,1749                        OffsetInfoMap[CurPtr].Offsets, Changed,1750                        *LoadI->getType()))1751        return false;1752 1753      auto IsAssumption = [](Instruction &I) {1754        if (auto *II = dyn_cast<IntrinsicInst>(&I))1755          return II->isAssumeLikeIntrinsic();1756        return false;1757      };1758 1759      auto IsImpactedInRange = [&](Instruction *FromI, Instruction *ToI) {1760        // Check if the assumption and the load are executed together without1761        // memory modification.1762        do {1763          if (FromI->mayWriteToMemory() && !IsAssumption(*FromI))1764            return true;1765          FromI = FromI->getNextNode();1766        } while (FromI && FromI != ToI);1767        return false;1768      };1769 1770      BasicBlock *BB = LoadI->getParent();1771      auto IsValidAssume = [&](IntrinsicInst &IntrI) {1772        if (IntrI.getIntrinsicID() != Intrinsic::assume)1773          return false;1774        BasicBlock *IntrBB = IntrI.getParent();1775        if (IntrI.getParent() == BB) {1776          if (IsImpactedInRange(LoadI->getNextNode(), &IntrI))1777            return false;1778        } else {1779          auto PredIt = pred_begin(IntrBB);1780          if (PredIt == pred_end(IntrBB))1781            return false;1782          if ((*PredIt) != BB)1783            return false;1784          if (++PredIt != pred_end(IntrBB))1785            return false;1786          for (auto *SuccBB : successors(BB)) {1787            if (SuccBB == IntrBB)1788              continue;1789            if (isa<UnreachableInst>(SuccBB->getTerminator()))1790              continue;1791            return false;1792          }1793          if (IsImpactedInRange(LoadI->getNextNode(), BB->getTerminator()))1794            return false;1795          if (IsImpactedInRange(&IntrBB->front(), &IntrI))1796            return false;1797        }1798        return true;1799      };1800 1801      std::pair<Value *, IntrinsicInst *> Assumption;1802      for (const Use &LoadU : LoadI->uses()) {1803        if (auto *CmpI = dyn_cast<CmpInst>(LoadU.getUser())) {1804          if (!CmpI->isEquality() || !CmpI->isTrueWhenEqual())1805            continue;1806          for (const Use &CmpU : CmpI->uses()) {1807            if (auto *IntrI = dyn_cast<IntrinsicInst>(CmpU.getUser())) {1808              if (!IsValidAssume(*IntrI))1809                continue;1810              int Idx = CmpI->getOperandUse(0) == LoadU;1811              Assumption = {CmpI->getOperand(Idx), IntrI};1812              break;1813            }1814          }1815        }1816        if (Assumption.first)1817          break;1818      }1819 1820      // Check if we found an assumption associated with this load.1821      if (!Assumption.first || !Assumption.second)1822        return true;1823 1824      LLVM_DEBUG(dbgs() << "[AAPointerInfo] Assumption found "1825                        << *Assumption.second << ": " << *LoadI1826                        << " == " << *Assumption.first << "\n");1827      bool UsedAssumedInformation = false;1828      std::optional<Value *> Content = nullptr;1829      if (Assumption.first)1830        Content =1831            A.getAssumedSimplified(*Assumption.first, *this,1832                                   UsedAssumedInformation, AA::Interprocedural);1833      return handleAccess(1834          A, *Assumption.second, Content, AccessKind::AK_ASSUMPTION,1835          OffsetInfoMap[CurPtr].Offsets, Changed, *LoadI->getType());1836    }1837 1838    auto HandleStoreLike = [&](Instruction &I, Value *ValueOp, Type &ValueTy,1839                               ArrayRef<Value *> OtherOps, AccessKind AK) {1840      for (auto *OtherOp : OtherOps) {1841        if (OtherOp == CurPtr) {1842          LLVM_DEBUG(1843              dbgs()1844              << "[AAPointerInfo] Escaping use in store like instruction " << I1845              << "\n");1846          return false;1847        }1848      }1849 1850      // If the access is to a pointer that may or may not be the associated1851      // value, e.g. due to a PHI, we cannot assume it will be written.1852      if (getUnderlyingObject(CurPtr) == &AssociatedValue)1853        AK = AccessKind(AK | AccessKind::AK_MUST);1854      else1855        AK = AccessKind(AK | AccessKind::AK_MAY);1856      bool UsedAssumedInformation = false;1857      std::optional<Value *> Content = nullptr;1858      if (ValueOp)1859        Content = A.getAssumedSimplified(1860            *ValueOp, *this, UsedAssumedInformation, AA::Interprocedural);1861      return handleAccess(A, I, Content, AK, OffsetInfoMap[CurPtr].Offsets,1862                          Changed, ValueTy);1863    };1864 1865    if (auto *StoreI = dyn_cast<StoreInst>(Usr))1866      return HandleStoreLike(*StoreI, StoreI->getValueOperand(),1867                             *StoreI->getValueOperand()->getType(),1868                             {StoreI->getValueOperand()}, AccessKind::AK_W);1869    if (auto *RMWI = dyn_cast<AtomicRMWInst>(Usr))1870      return HandleStoreLike(*RMWI, nullptr, *RMWI->getValOperand()->getType(),1871                             {RMWI->getValOperand()}, AccessKind::AK_RW);1872    if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Usr))1873      return HandleStoreLike(1874          *CXI, nullptr, *CXI->getNewValOperand()->getType(),1875          {CXI->getCompareOperand(), CXI->getNewValOperand()},1876          AccessKind::AK_RW);1877 1878    if (auto *CB = dyn_cast<CallBase>(Usr)) {1879      if (CB->isLifetimeStartOrEnd())1880        return true;1881      const auto *TLI =1882          A.getInfoCache().getTargetLibraryInfoForFunction(*CB->getFunction());1883      if (getFreedOperand(CB, TLI) == U)1884        return true;1885      if (CB->isArgOperand(&U)) {1886        unsigned ArgNo = CB->getArgOperandNo(&U);1887        const auto *CSArgPI = A.getAAFor<AAPointerInfo>(1888            *this, IRPosition::callsite_argument(*CB, ArgNo),1889            DepClassTy::REQUIRED);1890        if (!CSArgPI)1891          return false;1892        bool IsArgMustAcc = (getUnderlyingObject(CurPtr) == &AssociatedValue);1893        Changed = translateAndAddState(A, *CSArgPI, OffsetInfoMap[CurPtr], *CB,1894                                       IsArgMustAcc) |1895                  Changed;1896        if (!CSArgPI->reachesReturn())1897          return isValidState();1898 1899        Function *Callee = CB->getCalledFunction();1900        if (!Callee || Callee->arg_size() <= ArgNo)1901          return false;1902        bool UsedAssumedInformation = false;1903        auto ReturnedValue = A.getAssumedSimplified(1904            IRPosition::returned(*Callee), *this, UsedAssumedInformation,1905            AA::ValueScope::Intraprocedural);1906        auto *ReturnedArg =1907            dyn_cast_or_null<Argument>(ReturnedValue.value_or(nullptr));1908        auto *Arg = Callee->getArg(ArgNo);1909        if (ReturnedArg && Arg != ReturnedArg)1910          return true;1911        bool IsRetMustAcc = IsArgMustAcc && (ReturnedArg == Arg);1912        const auto *CSRetPI = A.getAAFor<AAPointerInfo>(1913            *this, IRPosition::callsite_returned(*CB), DepClassTy::REQUIRED);1914        if (!CSRetPI)1915          return false;1916        OffsetInfo OI = OffsetInfoMap[CurPtr];1917        CSArgPI->addReturnedOffsetsTo(OI);1918        Changed =1919            translateAndAddState(A, *CSRetPI, OI, *CB, IsRetMustAcc) | Changed;1920        return isValidState();1921      }1922      LLVM_DEBUG(dbgs() << "[AAPointerInfo] Call user not handled " << *CB1923                        << "\n");1924      return false;1925    }1926 1927    LLVM_DEBUG(dbgs() << "[AAPointerInfo] User not handled " << *Usr << "\n");1928    return false;1929  };1930  auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {1931    assert(OffsetInfoMap.count(OldU) && "Old use should be known already!");1932    assert(!OffsetInfoMap[OldU].isUnassigned() && "Old use should be assinged");1933    if (OffsetInfoMap.count(NewU)) {1934      LLVM_DEBUG({1935        if (!(OffsetInfoMap[NewU] == OffsetInfoMap[OldU])) {1936          dbgs() << "[AAPointerInfo] Equivalent use callback failed: "1937                 << OffsetInfoMap[NewU] << " vs " << OffsetInfoMap[OldU]1938                 << "\n";1939        }1940      });1941      return OffsetInfoMap[NewU] == OffsetInfoMap[OldU];1942    }1943    bool Unused;1944    return HandlePassthroughUser(NewU.get(), OldU.get(), Unused);1945  };1946  if (!A.checkForAllUses(UsePred, *this, AssociatedValue,1947                         /* CheckBBLivenessOnly */ true, DepClassTy::OPTIONAL,1948                         /* IgnoreDroppableUses */ true, EquivalentUseCB)) {1949    LLVM_DEBUG(dbgs() << "[AAPointerInfo] Check for all uses failed, abort!\n");1950    return indicatePessimisticFixpoint();1951  }1952 1953  LLVM_DEBUG({1954    dbgs() << "Accesses by bin after update:\n";1955    dumpState(dbgs());1956  });1957 1958  return Changed;1959}1960 1961struct AAPointerInfoReturned final : AAPointerInfoImpl {1962  AAPointerInfoReturned(const IRPosition &IRP, Attributor &A)1963      : AAPointerInfoImpl(IRP, A) {}1964 1965  /// See AbstractAttribute::updateImpl(...).1966  ChangeStatus updateImpl(Attributor &A) override {1967    return indicatePessimisticFixpoint();1968  }1969 1970  /// See AbstractAttribute::trackStatistics()1971  void trackStatistics() const override {1972    AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());1973  }1974};1975 1976struct AAPointerInfoArgument final : AAPointerInfoFloating {1977  AAPointerInfoArgument(const IRPosition &IRP, Attributor &A)1978      : AAPointerInfoFloating(IRP, A) {}1979 1980  /// See AbstractAttribute::trackStatistics()1981  void trackStatistics() const override {1982    AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());1983  }1984};1985 1986struct AAPointerInfoCallSiteArgument final : AAPointerInfoFloating {1987  AAPointerInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)1988      : AAPointerInfoFloating(IRP, A) {}1989 1990  /// See AbstractAttribute::updateImpl(...).1991  ChangeStatus updateImpl(Attributor &A) override {1992    using namespace AA::PointerInfo;1993    // We handle memory intrinsics explicitly, at least the first (=1994    // destination) and second (=source) arguments as we know how they are1995    // accessed.1996    if (auto *MI = dyn_cast_or_null<MemIntrinsic>(getCtxI())) {1997      int64_t LengthVal = AA::RangeTy::Unknown;1998      if (auto Length = MI->getLengthInBytes())1999        LengthVal = Length->getSExtValue();2000      unsigned ArgNo = getIRPosition().getCallSiteArgNo();2001      ChangeStatus Changed = ChangeStatus::UNCHANGED;2002      if (ArgNo > 1) {2003        LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled memory intrinsic "2004                          << *MI << "\n");2005        return indicatePessimisticFixpoint();2006      } else {2007        auto Kind =2008            ArgNo == 0 ? AccessKind::AK_MUST_WRITE : AccessKind::AK_MUST_READ;2009        Changed =2010            Changed | addAccess(A, {0, LengthVal}, *MI, nullptr, Kind, nullptr);2011      }2012      LLVM_DEBUG({2013        dbgs() << "Accesses by bin after update:\n";2014        dumpState(dbgs());2015      });2016 2017      return Changed;2018    }2019 2020    // TODO: Once we have call site specific value information we can provide2021    //       call site specific liveness information and then it makes2022    //       sense to specialize attributes for call sites arguments instead of2023    //       redirecting requests to the callee argument.2024    Argument *Arg = getAssociatedArgument();2025    if (Arg) {2026      const IRPosition &ArgPos = IRPosition::argument(*Arg);2027      auto *ArgAA =2028          A.getAAFor<AAPointerInfo>(*this, ArgPos, DepClassTy::REQUIRED);2029      if (ArgAA && ArgAA->getState().isValidState())2030        return translateAndAddStateFromCallee(A, *ArgAA,2031                                              *cast<CallBase>(getCtxI()));2032      if (!Arg->getParent()->isDeclaration())2033        return indicatePessimisticFixpoint();2034    }2035 2036    bool IsKnownNoCapture;2037    if (!AA::hasAssumedIRAttr<Attribute::Captures>(2038            A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNoCapture))2039      return indicatePessimisticFixpoint();2040 2041    bool IsKnown = false;2042    if (AA::isAssumedReadNone(A, getIRPosition(), *this, IsKnown))2043      return ChangeStatus::UNCHANGED;2044    bool ReadOnly = AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown);2045    auto Kind =2046        ReadOnly ? AccessKind::AK_MAY_READ : AccessKind::AK_MAY_READ_WRITE;2047    return addAccess(A, AA::RangeTy::getUnknown(), *getCtxI(), nullptr, Kind,2048                     nullptr);2049  }2050 2051  /// See AbstractAttribute::trackStatistics()2052  void trackStatistics() const override {2053    AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());2054  }2055};2056 2057struct AAPointerInfoCallSiteReturned final : AAPointerInfoFloating {2058  AAPointerInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)2059      : AAPointerInfoFloating(IRP, A) {}2060 2061  /// See AbstractAttribute::trackStatistics()2062  void trackStatistics() const override {2063    AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());2064  }2065};2066} // namespace2067 2068/// -----------------------NoUnwind Function Attribute--------------------------2069 2070namespace {2071struct AANoUnwindImpl : AANoUnwind {2072  AANoUnwindImpl(const IRPosition &IRP, Attributor &A) : AANoUnwind(IRP, A) {}2073 2074  /// See AbstractAttribute::initialize(...).2075  void initialize(Attributor &A) override {2076    bool IsKnown;2077    assert(!AA::hasAssumedIRAttr<Attribute::NoUnwind>(2078        A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));2079    (void)IsKnown;2080  }2081 2082  const std::string getAsStr(Attributor *A) const override {2083    return getAssumed() ? "nounwind" : "may-unwind";2084  }2085 2086  /// See AbstractAttribute::updateImpl(...).2087  ChangeStatus updateImpl(Attributor &A) override {2088    auto Opcodes = {2089        (unsigned)Instruction::Invoke,      (unsigned)Instruction::CallBr,2090        (unsigned)Instruction::Call,        (unsigned)Instruction::CleanupRet,2091        (unsigned)Instruction::CatchSwitch, (unsigned)Instruction::Resume};2092 2093    auto CheckForNoUnwind = [&](Instruction &I) {2094      if (!I.mayThrow(/* IncludePhaseOneUnwind */ true))2095        return true;2096 2097      if (const auto *CB = dyn_cast<CallBase>(&I)) {2098        bool IsKnownNoUnwind;2099        return AA::hasAssumedIRAttr<Attribute::NoUnwind>(2100            A, this, IRPosition::callsite_function(*CB), DepClassTy::REQUIRED,2101            IsKnownNoUnwind);2102      }2103      return false;2104    };2105 2106    bool UsedAssumedInformation = false;2107    if (!A.checkForAllInstructions(CheckForNoUnwind, *this, Opcodes,2108                                   UsedAssumedInformation))2109      return indicatePessimisticFixpoint();2110 2111    return ChangeStatus::UNCHANGED;2112  }2113};2114 2115struct AANoUnwindFunction final : public AANoUnwindImpl {2116  AANoUnwindFunction(const IRPosition &IRP, Attributor &A)2117      : AANoUnwindImpl(IRP, A) {}2118 2119  /// See AbstractAttribute::trackStatistics()2120  void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nounwind) }2121};2122 2123/// NoUnwind attribute deduction for a call sites.2124struct AANoUnwindCallSite final2125    : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl> {2126  AANoUnwindCallSite(const IRPosition &IRP, Attributor &A)2127      : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl>(IRP, A) {}2128 2129  /// See AbstractAttribute::trackStatistics()2130  void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nounwind); }2131};2132} // namespace2133 2134/// ------------------------ NoSync Function Attribute -------------------------2135 2136bool AANoSync::isAlignedBarrier(const CallBase &CB, bool ExecutedAligned) {2137  switch (CB.getIntrinsicID()) {2138  case Intrinsic::nvvm_barrier_cta_sync_aligned_all:2139  case Intrinsic::nvvm_barrier_cta_sync_aligned_count:2140  case Intrinsic::nvvm_barrier0_and:2141  case Intrinsic::nvvm_barrier0_or:2142  case Intrinsic::nvvm_barrier0_popc:2143    return true;2144  case Intrinsic::amdgcn_s_barrier:2145    if (ExecutedAligned)2146      return true;2147    break;2148  default:2149    break;2150  }2151  return hasAssumption(CB, KnownAssumptionString("ompx_aligned_barrier"));2152}2153 2154bool AANoSync::isNonRelaxedAtomic(const Instruction *I) {2155  if (!I->isAtomic())2156    return false;2157 2158  if (auto *FI = dyn_cast<FenceInst>(I))2159    // All legal orderings for fence are stronger than monotonic.2160    return FI->getSyncScopeID() != SyncScope::SingleThread;2161  if (auto *AI = dyn_cast<AtomicCmpXchgInst>(I)) {2162    // Unordered is not a legal ordering for cmpxchg.2163    return (AI->getSuccessOrdering() != AtomicOrdering::Monotonic ||2164            AI->getFailureOrdering() != AtomicOrdering::Monotonic);2165  }2166 2167  AtomicOrdering Ordering;2168  switch (I->getOpcode()) {2169  case Instruction::AtomicRMW:2170    Ordering = cast<AtomicRMWInst>(I)->getOrdering();2171    break;2172  case Instruction::Store:2173    Ordering = cast<StoreInst>(I)->getOrdering();2174    break;2175  case Instruction::Load:2176    Ordering = cast<LoadInst>(I)->getOrdering();2177    break;2178  default:2179    llvm_unreachable(2180        "New atomic operations need to be known in the attributor.");2181  }2182 2183  return (Ordering != AtomicOrdering::Unordered &&2184          Ordering != AtomicOrdering::Monotonic);2185}2186 2187/// Return true if this intrinsic is nosync.  This is only used for intrinsics2188/// which would be nosync except that they have a volatile flag.  All other2189/// intrinsics are simply annotated with the nosync attribute in Intrinsics.td.2190bool AANoSync::isNoSyncIntrinsic(const Instruction *I) {2191  if (auto *MI = dyn_cast<MemIntrinsic>(I))2192    return !MI->isVolatile();2193  return false;2194}2195 2196namespace {2197struct AANoSyncImpl : AANoSync {2198  AANoSyncImpl(const IRPosition &IRP, Attributor &A) : AANoSync(IRP, A) {}2199 2200  /// See AbstractAttribute::initialize(...).2201  void initialize(Attributor &A) override {2202    bool IsKnown;2203    assert(!AA::hasAssumedIRAttr<Attribute::NoSync>(A, nullptr, getIRPosition(),2204                                                    DepClassTy::NONE, IsKnown));2205    (void)IsKnown;2206  }2207 2208  const std::string getAsStr(Attributor *A) const override {2209    return getAssumed() ? "nosync" : "may-sync";2210  }2211 2212  /// See AbstractAttribute::updateImpl(...).2213  ChangeStatus updateImpl(Attributor &A) override;2214};2215 2216ChangeStatus AANoSyncImpl::updateImpl(Attributor &A) {2217 2218  auto CheckRWInstForNoSync = [&](Instruction &I) {2219    return AA::isNoSyncInst(A, I, *this);2220  };2221 2222  auto CheckForNoSync = [&](Instruction &I) {2223    // At this point we handled all read/write effects and they are all2224    // nosync, so they can be skipped.2225    if (I.mayReadOrWriteMemory())2226      return true;2227 2228    bool IsKnown;2229    CallBase &CB = cast<CallBase>(I);2230    if (AA::hasAssumedIRAttr<Attribute::NoSync>(2231            A, this, IRPosition::callsite_function(CB), DepClassTy::OPTIONAL,2232            IsKnown))2233      return true;2234 2235    // non-convergent and readnone imply nosync.2236    return !CB.isConvergent();2237  };2238 2239  bool UsedAssumedInformation = false;2240  if (!A.checkForAllReadWriteInstructions(CheckRWInstForNoSync, *this,2241                                          UsedAssumedInformation) ||2242      !A.checkForAllCallLikeInstructions(CheckForNoSync, *this,2243                                         UsedAssumedInformation))2244    return indicatePessimisticFixpoint();2245 2246  return ChangeStatus::UNCHANGED;2247}2248 2249struct AANoSyncFunction final : public AANoSyncImpl {2250  AANoSyncFunction(const IRPosition &IRP, Attributor &A)2251      : AANoSyncImpl(IRP, A) {}2252 2253  /// See AbstractAttribute::trackStatistics()2254  void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nosync) }2255};2256 2257/// NoSync attribute deduction for a call sites.2258struct AANoSyncCallSite final : AACalleeToCallSite<AANoSync, AANoSyncImpl> {2259  AANoSyncCallSite(const IRPosition &IRP, Attributor &A)2260      : AACalleeToCallSite<AANoSync, AANoSyncImpl>(IRP, A) {}2261 2262  /// See AbstractAttribute::trackStatistics()2263  void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nosync); }2264};2265} // namespace2266 2267/// ------------------------ No-Free Attributes ----------------------------2268 2269namespace {2270struct AANoFreeImpl : public AANoFree {2271  AANoFreeImpl(const IRPosition &IRP, Attributor &A) : AANoFree(IRP, A) {}2272 2273  /// See AbstractAttribute::initialize(...).2274  void initialize(Attributor &A) override {2275    bool IsKnown;2276    assert(!AA::hasAssumedIRAttr<Attribute::NoFree>(A, nullptr, getIRPosition(),2277                                                    DepClassTy::NONE, IsKnown));2278    (void)IsKnown;2279  }2280 2281  /// See AbstractAttribute::updateImpl(...).2282  ChangeStatus updateImpl(Attributor &A) override {2283    auto CheckForNoFree = [&](Instruction &I) {2284      bool IsKnown;2285      return AA::hasAssumedIRAttr<Attribute::NoFree>(2286          A, this, IRPosition::callsite_function(cast<CallBase>(I)),2287          DepClassTy::REQUIRED, IsKnown);2288    };2289 2290    bool UsedAssumedInformation = false;2291    if (!A.checkForAllCallLikeInstructions(CheckForNoFree, *this,2292                                           UsedAssumedInformation))2293      return indicatePessimisticFixpoint();2294    return ChangeStatus::UNCHANGED;2295  }2296 2297  /// See AbstractAttribute::getAsStr().2298  const std::string getAsStr(Attributor *A) const override {2299    return getAssumed() ? "nofree" : "may-free";2300  }2301};2302 2303struct AANoFreeFunction final : public AANoFreeImpl {2304  AANoFreeFunction(const IRPosition &IRP, Attributor &A)2305      : AANoFreeImpl(IRP, A) {}2306 2307  /// See AbstractAttribute::trackStatistics()2308  void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nofree) }2309};2310 2311/// NoFree attribute deduction for a call sites.2312struct AANoFreeCallSite final : AACalleeToCallSite<AANoFree, AANoFreeImpl> {2313  AANoFreeCallSite(const IRPosition &IRP, Attributor &A)2314      : AACalleeToCallSite<AANoFree, AANoFreeImpl>(IRP, A) {}2315 2316  /// See AbstractAttribute::trackStatistics()2317  void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nofree); }2318};2319 2320/// NoFree attribute for floating values.2321struct AANoFreeFloating : AANoFreeImpl {2322  AANoFreeFloating(const IRPosition &IRP, Attributor &A)2323      : AANoFreeImpl(IRP, A) {}2324 2325  /// See AbstractAttribute::trackStatistics()2326  void trackStatistics() const override{STATS_DECLTRACK_FLOATING_ATTR(nofree)}2327 2328  /// See Abstract Attribute::updateImpl(...).2329  ChangeStatus updateImpl(Attributor &A) override {2330    const IRPosition &IRP = getIRPosition();2331 2332    bool IsKnown;2333    if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, this,2334                                                IRPosition::function_scope(IRP),2335                                                DepClassTy::OPTIONAL, IsKnown))2336      return ChangeStatus::UNCHANGED;2337 2338    Value &AssociatedValue = getIRPosition().getAssociatedValue();2339    auto Pred = [&](const Use &U, bool &Follow) -> bool {2340      Instruction *UserI = cast<Instruction>(U.getUser());2341      if (auto *CB = dyn_cast<CallBase>(UserI)) {2342        if (CB->isBundleOperand(&U))2343          return false;2344        if (!CB->isArgOperand(&U))2345          return true;2346        unsigned ArgNo = CB->getArgOperandNo(&U);2347 2348        bool IsKnown;2349        return AA::hasAssumedIRAttr<Attribute::NoFree>(2350            A, this, IRPosition::callsite_argument(*CB, ArgNo),2351            DepClassTy::REQUIRED, IsKnown);2352      }2353 2354      if (isa<GetElementPtrInst>(UserI) || isa<PHINode>(UserI) ||2355          isa<SelectInst>(UserI)) {2356        Follow = true;2357        return true;2358      }2359      if (isa<StoreInst>(UserI) || isa<LoadInst>(UserI))2360        return true;2361 2362      if (isa<ReturnInst>(UserI) && getIRPosition().isArgumentPosition())2363        return true;2364 2365      // Unknown user.2366      return false;2367    };2368    if (!A.checkForAllUses(Pred, *this, AssociatedValue))2369      return indicatePessimisticFixpoint();2370 2371    return ChangeStatus::UNCHANGED;2372  }2373};2374 2375/// NoFree attribute for a call site argument.2376struct AANoFreeArgument final : AANoFreeFloating {2377  AANoFreeArgument(const IRPosition &IRP, Attributor &A)2378      : AANoFreeFloating(IRP, A) {}2379 2380  /// See AbstractAttribute::trackStatistics()2381  void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofree) }2382};2383 2384/// NoFree attribute for call site arguments.2385struct AANoFreeCallSiteArgument final : AANoFreeFloating {2386  AANoFreeCallSiteArgument(const IRPosition &IRP, Attributor &A)2387      : AANoFreeFloating(IRP, A) {}2388 2389  /// See AbstractAttribute::updateImpl(...).2390  ChangeStatus updateImpl(Attributor &A) override {2391    // TODO: Once we have call site specific value information we can provide2392    //       call site specific liveness information and then it makes2393    //       sense to specialize attributes for call sites arguments instead of2394    //       redirecting requests to the callee argument.2395    Argument *Arg = getAssociatedArgument();2396    if (!Arg)2397      return indicatePessimisticFixpoint();2398    const IRPosition &ArgPos = IRPosition::argument(*Arg);2399    bool IsKnown;2400    if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, this, ArgPos,2401                                                DepClassTy::REQUIRED, IsKnown))2402      return ChangeStatus::UNCHANGED;2403    return indicatePessimisticFixpoint();2404  }2405 2406  /// See AbstractAttribute::trackStatistics()2407  void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(nofree) };2408};2409 2410/// NoFree attribute for function return value.2411struct AANoFreeReturned final : AANoFreeFloating {2412  AANoFreeReturned(const IRPosition &IRP, Attributor &A)2413      : AANoFreeFloating(IRP, A) {2414    llvm_unreachable("NoFree is not applicable to function returns!");2415  }2416 2417  /// See AbstractAttribute::initialize(...).2418  void initialize(Attributor &A) override {2419    llvm_unreachable("NoFree is not applicable to function returns!");2420  }2421 2422  /// See AbstractAttribute::updateImpl(...).2423  ChangeStatus updateImpl(Attributor &A) override {2424    llvm_unreachable("NoFree is not applicable to function returns!");2425  }2426 2427  /// See AbstractAttribute::trackStatistics()2428  void trackStatistics() const override {}2429};2430 2431/// NoFree attribute deduction for a call site return value.2432struct AANoFreeCallSiteReturned final : AANoFreeFloating {2433  AANoFreeCallSiteReturned(const IRPosition &IRP, Attributor &A)2434      : AANoFreeFloating(IRP, A) {}2435 2436  ChangeStatus manifest(Attributor &A) override {2437    return ChangeStatus::UNCHANGED;2438  }2439  /// See AbstractAttribute::trackStatistics()2440  void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nofree) }2441};2442} // namespace2443 2444/// ------------------------ NonNull Argument Attribute ------------------------2445 2446bool AANonNull::isImpliedByIR(Attributor &A, const IRPosition &IRP,2447                              Attribute::AttrKind ImpliedAttributeKind,2448                              bool IgnoreSubsumingPositions) {2449  SmallVector<Attribute::AttrKind, 2> AttrKinds;2450  AttrKinds.push_back(Attribute::NonNull);2451  if (!NullPointerIsDefined(IRP.getAnchorScope(),2452                            IRP.getAssociatedType()->getPointerAddressSpace()))2453    AttrKinds.push_back(Attribute::Dereferenceable);2454  if (A.hasAttr(IRP, AttrKinds, IgnoreSubsumingPositions, Attribute::NonNull))2455    return true;2456 2457  DominatorTree *DT = nullptr;2458  AssumptionCache *AC = nullptr;2459  InformationCache &InfoCache = A.getInfoCache();2460  if (const Function *Fn = IRP.getAnchorScope()) {2461    if (!Fn->isDeclaration()) {2462      DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(*Fn);2463      AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(*Fn);2464    }2465  }2466 2467  SmallVector<AA::ValueAndContext> Worklist;2468  if (IRP.getPositionKind() != IRP_RETURNED) {2469    Worklist.push_back({IRP.getAssociatedValue(), IRP.getCtxI()});2470  } else {2471    bool UsedAssumedInformation = false;2472    if (!A.checkForAllInstructions(2473            [&](Instruction &I) {2474              Worklist.push_back({*cast<ReturnInst>(I).getReturnValue(), &I});2475              return true;2476            },2477            IRP.getAssociatedFunction(), nullptr, {Instruction::Ret},2478            UsedAssumedInformation, false, /*CheckPotentiallyDead=*/true))2479      return false;2480  }2481 2482  if (llvm::any_of(Worklist, [&](AA::ValueAndContext VAC) {2483        return !isKnownNonZero(2484            VAC.getValue(),2485            SimplifyQuery(A.getDataLayout(), DT, AC, VAC.getCtxI()));2486      }))2487    return false;2488 2489  A.manifestAttrs(IRP, {Attribute::get(IRP.getAnchorValue().getContext(),2490                                       Attribute::NonNull)});2491  return true;2492}2493 2494namespace {2495static int64_t getKnownNonNullAndDerefBytesForUse(2496    Attributor &A, const AbstractAttribute &QueryingAA, Value &AssociatedValue,2497    const Use *U, const Instruction *I, bool &IsNonNull, bool &TrackUse) {2498  TrackUse = false;2499 2500  const Value *UseV = U->get();2501  if (!UseV->getType()->isPointerTy())2502    return 0;2503 2504  // We need to follow common pointer manipulation uses to the accesses they2505  // feed into. We can try to be smart to avoid looking through things we do not2506  // like for now, e.g., non-inbounds GEPs.2507  if (isa<CastInst>(I)) {2508    TrackUse = true;2509    return 0;2510  }2511 2512  if (isa<GetElementPtrInst>(I)) {2513    TrackUse = true;2514    return 0;2515  }2516 2517  Type *PtrTy = UseV->getType();2518  const Function *F = I->getFunction();2519  bool NullPointerIsDefined =2520      F ? llvm::NullPointerIsDefined(F, PtrTy->getPointerAddressSpace()) : true;2521  const DataLayout &DL = A.getInfoCache().getDL();2522  if (const auto *CB = dyn_cast<CallBase>(I)) {2523    if (CB->isBundleOperand(U)) {2524      if (RetainedKnowledge RK = getKnowledgeFromUse(2525              U, {Attribute::NonNull, Attribute::Dereferenceable})) {2526        IsNonNull |=2527            (RK.AttrKind == Attribute::NonNull || !NullPointerIsDefined);2528        return RK.ArgValue;2529      }2530      return 0;2531    }2532 2533    if (CB->isCallee(U)) {2534      IsNonNull |= !NullPointerIsDefined;2535      return 0;2536    }2537 2538    unsigned ArgNo = CB->getArgOperandNo(U);2539    IRPosition IRP = IRPosition::callsite_argument(*CB, ArgNo);2540    // As long as we only use known information there is no need to track2541    // dependences here.2542    bool IsKnownNonNull;2543    AA::hasAssumedIRAttr<Attribute::NonNull>(A, &QueryingAA, IRP,2544                                             DepClassTy::NONE, IsKnownNonNull);2545    IsNonNull |= IsKnownNonNull;2546    auto *DerefAA =2547        A.getAAFor<AADereferenceable>(QueryingAA, IRP, DepClassTy::NONE);2548    return DerefAA ? DerefAA->getKnownDereferenceableBytes() : 0;2549  }2550 2551  std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(I);2552  if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() ||2553      Loc->Size.isScalable() || I->isVolatile())2554    return 0;2555 2556  int64_t Offset;2557  const Value *Base =2558      getMinimalBaseOfPointer(A, QueryingAA, Loc->Ptr, Offset, DL);2559  if (Base && Base == &AssociatedValue) {2560    int64_t DerefBytes = Loc->Size.getValue() + Offset;2561    IsNonNull |= !NullPointerIsDefined;2562    return std::max(int64_t(0), DerefBytes);2563  }2564 2565  /// Corner case when an offset is 0.2566  Base = GetPointerBaseWithConstantOffset(Loc->Ptr, Offset, DL,2567                                          /*AllowNonInbounds*/ true);2568  if (Base && Base == &AssociatedValue && Offset == 0) {2569    int64_t DerefBytes = Loc->Size.getValue();2570    IsNonNull |= !NullPointerIsDefined;2571    return std::max(int64_t(0), DerefBytes);2572  }2573 2574  return 0;2575}2576 2577struct AANonNullImpl : AANonNull {2578  AANonNullImpl(const IRPosition &IRP, Attributor &A) : AANonNull(IRP, A) {}2579 2580  /// See AbstractAttribute::initialize(...).2581  void initialize(Attributor &A) override {2582    Value &V = *getAssociatedValue().stripPointerCasts();2583    if (isa<ConstantPointerNull>(V)) {2584      indicatePessimisticFixpoint();2585      return;2586    }2587 2588    if (Instruction *CtxI = getCtxI())2589      followUsesInMBEC(*this, A, getState(), *CtxI);2590  }2591 2592  /// See followUsesInMBEC2593  bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,2594                       AANonNull::StateType &State) {2595    bool IsNonNull = false;2596    bool TrackUse = false;2597    getKnownNonNullAndDerefBytesForUse(A, *this, getAssociatedValue(), U, I,2598                                       IsNonNull, TrackUse);2599    State.setKnown(IsNonNull);2600    return TrackUse;2601  }2602 2603  /// See AbstractAttribute::getAsStr().2604  const std::string getAsStr(Attributor *A) const override {2605    return getAssumed() ? "nonnull" : "may-null";2606  }2607};2608 2609/// NonNull attribute for a floating value.2610struct AANonNullFloating : public AANonNullImpl {2611  AANonNullFloating(const IRPosition &IRP, Attributor &A)2612      : AANonNullImpl(IRP, A) {}2613 2614  /// See AbstractAttribute::updateImpl(...).2615  ChangeStatus updateImpl(Attributor &A) override {2616    auto CheckIRP = [&](const IRPosition &IRP) {2617      bool IsKnownNonNull;2618      return AA::hasAssumedIRAttr<Attribute::NonNull>(2619          A, *this, IRP, DepClassTy::OPTIONAL, IsKnownNonNull);2620    };2621 2622    bool Stripped;2623    bool UsedAssumedInformation = false;2624    Value *AssociatedValue = &getAssociatedValue();2625    SmallVector<AA::ValueAndContext> Values;2626    if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,2627                                      AA::AnyScope, UsedAssumedInformation))2628      Stripped = false;2629    else2630      Stripped =2631          Values.size() != 1 || Values.front().getValue() != AssociatedValue;2632 2633    if (!Stripped) {2634      bool IsKnown;2635      if (auto *PHI = dyn_cast<PHINode>(AssociatedValue))2636        if (llvm::all_of(PHI->incoming_values(), [&](Value *Op) {2637              return AA::hasAssumedIRAttr<Attribute::NonNull>(2638                  A, this, IRPosition::value(*Op), DepClassTy::OPTIONAL,2639                  IsKnown);2640            }))2641          return ChangeStatus::UNCHANGED;2642      if (auto *Select = dyn_cast<SelectInst>(AssociatedValue))2643        if (AA::hasAssumedIRAttr<Attribute::NonNull>(2644                A, this, IRPosition::value(*Select->getFalseValue()),2645                DepClassTy::OPTIONAL, IsKnown) &&2646            AA::hasAssumedIRAttr<Attribute::NonNull>(2647                A, this, IRPosition::value(*Select->getTrueValue()),2648                DepClassTy::OPTIONAL, IsKnown))2649          return ChangeStatus::UNCHANGED;2650 2651      // If we haven't stripped anything we might still be able to use a2652      // different AA, but only if the IRP changes. Effectively when we2653      // interpret this not as a call site value but as a floating/argument2654      // value.2655      const IRPosition AVIRP = IRPosition::value(*AssociatedValue);2656      if (AVIRP == getIRPosition() || !CheckIRP(AVIRP))2657        return indicatePessimisticFixpoint();2658      return ChangeStatus::UNCHANGED;2659    }2660 2661    for (const auto &VAC : Values)2662      if (!CheckIRP(IRPosition::value(*VAC.getValue())))2663        return indicatePessimisticFixpoint();2664 2665    return ChangeStatus::UNCHANGED;2666  }2667 2668  /// See AbstractAttribute::trackStatistics()2669  void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }2670};2671 2672/// NonNull attribute for function return value.2673struct AANonNullReturned final2674    : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,2675                                   false, AANonNull::IRAttributeKind, false> {2676  AANonNullReturned(const IRPosition &IRP, Attributor &A)2677      : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,2678                                     false, Attribute::NonNull, false>(IRP, A) {2679  }2680 2681  /// See AbstractAttribute::getAsStr().2682  const std::string getAsStr(Attributor *A) const override {2683    return getAssumed() ? "nonnull" : "may-null";2684  }2685 2686  /// See AbstractAttribute::trackStatistics()2687  void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }2688};2689 2690/// NonNull attribute for function argument.2691struct AANonNullArgument final2692    : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl> {2693  AANonNullArgument(const IRPosition &IRP, Attributor &A)2694      : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl>(IRP, A) {}2695 2696  /// See AbstractAttribute::trackStatistics()2697  void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nonnull) }2698};2699 2700struct AANonNullCallSiteArgument final : AANonNullFloating {2701  AANonNullCallSiteArgument(const IRPosition &IRP, Attributor &A)2702      : AANonNullFloating(IRP, A) {}2703 2704  /// See AbstractAttribute::trackStatistics()2705  void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(nonnull) }2706};2707 2708/// NonNull attribute for a call site return position.2709struct AANonNullCallSiteReturned final2710    : AACalleeToCallSite<AANonNull, AANonNullImpl> {2711  AANonNullCallSiteReturned(const IRPosition &IRP, Attributor &A)2712      : AACalleeToCallSite<AANonNull, AANonNullImpl>(IRP, A) {}2713 2714  /// See AbstractAttribute::trackStatistics()2715  void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nonnull) }2716};2717} // namespace2718 2719/// ------------------------ Must-Progress Attributes --------------------------2720namespace {2721struct AAMustProgressImpl : public AAMustProgress {2722  AAMustProgressImpl(const IRPosition &IRP, Attributor &A)2723      : AAMustProgress(IRP, A) {}2724 2725  /// See AbstractAttribute::initialize(...).2726  void initialize(Attributor &A) override {2727    bool IsKnown;2728    assert(!AA::hasAssumedIRAttr<Attribute::MustProgress>(2729        A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));2730    (void)IsKnown;2731  }2732 2733  /// See AbstractAttribute::getAsStr()2734  const std::string getAsStr(Attributor *A) const override {2735    return getAssumed() ? "mustprogress" : "may-not-progress";2736  }2737};2738 2739struct AAMustProgressFunction final : AAMustProgressImpl {2740  AAMustProgressFunction(const IRPosition &IRP, Attributor &A)2741      : AAMustProgressImpl(IRP, A) {}2742 2743  /// See AbstractAttribute::updateImpl(...).2744  ChangeStatus updateImpl(Attributor &A) override {2745    bool IsKnown;2746    if (AA::hasAssumedIRAttr<Attribute::WillReturn>(2747            A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnown)) {2748      if (IsKnown)2749        return indicateOptimisticFixpoint();2750      return ChangeStatus::UNCHANGED;2751    }2752 2753    auto CheckForMustProgress = [&](AbstractCallSite ACS) {2754      IRPosition IPos = IRPosition::callsite_function(*ACS.getInstruction());2755      bool IsKnownMustProgress;2756      return AA::hasAssumedIRAttr<Attribute::MustProgress>(2757          A, this, IPos, DepClassTy::REQUIRED, IsKnownMustProgress,2758          /* IgnoreSubsumingPositions */ true);2759    };2760 2761    bool AllCallSitesKnown = true;2762    if (!A.checkForAllCallSites(CheckForMustProgress, *this,2763                                /* RequireAllCallSites */ true,2764                                AllCallSitesKnown))2765      return indicatePessimisticFixpoint();2766 2767    return ChangeStatus::UNCHANGED;2768  }2769 2770  /// See AbstractAttribute::trackStatistics()2771  void trackStatistics() const override {2772    STATS_DECLTRACK_FN_ATTR(mustprogress)2773  }2774};2775 2776/// MustProgress attribute deduction for a call sites.2777struct AAMustProgressCallSite final : AAMustProgressImpl {2778  AAMustProgressCallSite(const IRPosition &IRP, Attributor &A)2779      : AAMustProgressImpl(IRP, A) {}2780 2781  /// See AbstractAttribute::updateImpl(...).2782  ChangeStatus updateImpl(Attributor &A) override {2783    // TODO: Once we have call site specific value information we can provide2784    //       call site specific liveness information and then it makes2785    //       sense to specialize attributes for call sites arguments instead of2786    //       redirecting requests to the callee argument.2787    const IRPosition &FnPos = IRPosition::function(*getAnchorScope());2788    bool IsKnownMustProgress;2789    if (!AA::hasAssumedIRAttr<Attribute::MustProgress>(2790            A, this, FnPos, DepClassTy::REQUIRED, IsKnownMustProgress))2791      return indicatePessimisticFixpoint();2792    return ChangeStatus::UNCHANGED;2793  }2794 2795  /// See AbstractAttribute::trackStatistics()2796  void trackStatistics() const override {2797    STATS_DECLTRACK_CS_ATTR(mustprogress);2798  }2799};2800} // namespace2801 2802/// ------------------------ No-Recurse Attributes ----------------------------2803 2804namespace {2805struct AANoRecurseImpl : public AANoRecurse {2806  AANoRecurseImpl(const IRPosition &IRP, Attributor &A) : AANoRecurse(IRP, A) {}2807 2808  /// See AbstractAttribute::initialize(...).2809  void initialize(Attributor &A) override {2810    bool IsKnown;2811    assert(!AA::hasAssumedIRAttr<Attribute::NoRecurse>(2812        A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));2813    (void)IsKnown;2814  }2815 2816  /// See AbstractAttribute::getAsStr()2817  const std::string getAsStr(Attributor *A) const override {2818    return getAssumed() ? "norecurse" : "may-recurse";2819  }2820};2821 2822struct AANoRecurseFunction final : AANoRecurseImpl {2823  AANoRecurseFunction(const IRPosition &IRP, Attributor &A)2824      : AANoRecurseImpl(IRP, A) {}2825 2826  /// See AbstractAttribute::updateImpl(...).2827  ChangeStatus updateImpl(Attributor &A) override {2828 2829    // If all live call sites are known to be no-recurse, we are as well.2830    auto CallSitePred = [&](AbstractCallSite ACS) {2831      bool IsKnownNoRecurse;2832      if (!AA::hasAssumedIRAttr<Attribute::NoRecurse>(2833              A, this,2834              IRPosition::function(*ACS.getInstruction()->getFunction()),2835              DepClassTy::NONE, IsKnownNoRecurse))2836        return false;2837      return IsKnownNoRecurse;2838    };2839    bool UsedAssumedInformation = false;2840    if (A.checkForAllCallSites(CallSitePred, *this, true,2841                               UsedAssumedInformation)) {2842      // If we know all call sites and all are known no-recurse, we are done.2843      // If all known call sites, which might not be all that exist, are known2844      // to be no-recurse, we are not done but we can continue to assume2845      // no-recurse. If one of the call sites we have not visited will become2846      // live, another update is triggered.2847      if (!UsedAssumedInformation)2848        indicateOptimisticFixpoint();2849      return ChangeStatus::UNCHANGED;2850    }2851 2852    const AAInterFnReachability *EdgeReachability =2853        A.getAAFor<AAInterFnReachability>(*this, getIRPosition(),2854                                          DepClassTy::REQUIRED);2855    if (EdgeReachability && EdgeReachability->canReach(A, *getAnchorScope()))2856      return indicatePessimisticFixpoint();2857    return ChangeStatus::UNCHANGED;2858  }2859 2860  void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(norecurse) }2861};2862 2863/// NoRecurse attribute deduction for a call sites.2864struct AANoRecurseCallSite final2865    : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl> {2866  AANoRecurseCallSite(const IRPosition &IRP, Attributor &A)2867      : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl>(IRP, A) {}2868 2869  /// See AbstractAttribute::trackStatistics()2870  void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(norecurse); }2871};2872} // namespace2873 2874/// ------------------------ No-Convergent Attribute --------------------------2875 2876namespace {2877struct AANonConvergentImpl : public AANonConvergent {2878  AANonConvergentImpl(const IRPosition &IRP, Attributor &A)2879      : AANonConvergent(IRP, A) {}2880 2881  /// See AbstractAttribute::getAsStr()2882  const std::string getAsStr(Attributor *A) const override {2883    return getAssumed() ? "non-convergent" : "may-be-convergent";2884  }2885};2886 2887struct AANonConvergentFunction final : AANonConvergentImpl {2888  AANonConvergentFunction(const IRPosition &IRP, Attributor &A)2889      : AANonConvergentImpl(IRP, A) {}2890 2891  /// See AbstractAttribute::updateImpl(...).2892  ChangeStatus updateImpl(Attributor &A) override {2893    // If all function calls are known to not be convergent, we are not2894    // convergent.2895    auto CalleeIsNotConvergent = [&](Instruction &Inst) {2896      CallBase &CB = cast<CallBase>(Inst);2897      auto *Callee = dyn_cast_if_present<Function>(CB.getCalledOperand());2898      if (!Callee || Callee->isIntrinsic()) {2899        return false;2900      }2901      if (Callee->isDeclaration()) {2902        return !Callee->hasFnAttribute(Attribute::Convergent);2903      }2904      const auto *ConvergentAA = A.getAAFor<AANonConvergent>(2905          *this, IRPosition::function(*Callee), DepClassTy::REQUIRED);2906      return ConvergentAA && ConvergentAA->isAssumedNotConvergent();2907    };2908 2909    bool UsedAssumedInformation = false;2910    if (!A.checkForAllCallLikeInstructions(CalleeIsNotConvergent, *this,2911                                           UsedAssumedInformation)) {2912      return indicatePessimisticFixpoint();2913    }2914    return ChangeStatus::UNCHANGED;2915  }2916 2917  ChangeStatus manifest(Attributor &A) override {2918    if (isKnownNotConvergent() &&2919        A.hasAttr(getIRPosition(), Attribute::Convergent)) {2920      A.removeAttrs(getIRPosition(), {Attribute::Convergent});2921      return ChangeStatus::CHANGED;2922    }2923    return ChangeStatus::UNCHANGED;2924  }2925 2926  void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(convergent) }2927};2928} // namespace2929 2930/// -------------------- Undefined-Behavior Attributes ------------------------2931 2932namespace {2933struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior {2934  AAUndefinedBehaviorImpl(const IRPosition &IRP, Attributor &A)2935      : AAUndefinedBehavior(IRP, A) {}2936 2937  /// See AbstractAttribute::updateImpl(...).2938  // through a pointer (i.e. also branches etc.)2939  ChangeStatus updateImpl(Attributor &A) override {2940    const size_t UBPrevSize = KnownUBInsts.size();2941    const size_t NoUBPrevSize = AssumedNoUBInsts.size();2942 2943    auto InspectMemAccessInstForUB = [&](Instruction &I) {2944      // Lang ref now states volatile store is not UB, let's skip them.2945      if (I.isVolatile() && I.mayWriteToMemory())2946        return true;2947 2948      // Skip instructions that are already saved.2949      if (AssumedNoUBInsts.count(&I) || KnownUBInsts.count(&I))2950        return true;2951 2952      // If we reach here, we know we have an instruction2953      // that accesses memory through a pointer operand,2954      // for which getPointerOperand() should give it to us.2955      Value *PtrOp =2956          const_cast<Value *>(getPointerOperand(&I, /* AllowVolatile */ true));2957      assert(PtrOp &&2958             "Expected pointer operand of memory accessing instruction");2959 2960      // Either we stopped and the appropriate action was taken,2961      // or we got back a simplified value to continue.2962      std::optional<Value *> SimplifiedPtrOp =2963          stopOnUndefOrAssumed(A, PtrOp, &I);2964      if (!SimplifiedPtrOp || !*SimplifiedPtrOp)2965        return true;2966      const Value *PtrOpVal = *SimplifiedPtrOp;2967 2968      // A memory access through a pointer is considered UB2969      // only if the pointer has constant null value.2970      // TODO: Expand it to not only check constant values.2971      if (!isa<ConstantPointerNull>(PtrOpVal)) {2972        AssumedNoUBInsts.insert(&I);2973        return true;2974      }2975      const Type *PtrTy = PtrOpVal->getType();2976 2977      // Because we only consider instructions inside functions,2978      // assume that a parent function exists.2979      const Function *F = I.getFunction();2980 2981      // A memory access using constant null pointer is only considered UB2982      // if null pointer is _not_ defined for the target platform.2983      if (llvm::NullPointerIsDefined(F, PtrTy->getPointerAddressSpace()))2984        AssumedNoUBInsts.insert(&I);2985      else2986        KnownUBInsts.insert(&I);2987      return true;2988    };2989 2990    auto InspectBrInstForUB = [&](Instruction &I) {2991      // A conditional branch instruction is considered UB if it has `undef`2992      // condition.2993 2994      // Skip instructions that are already saved.2995      if (AssumedNoUBInsts.count(&I) || KnownUBInsts.count(&I))2996        return true;2997 2998      // We know we have a branch instruction.2999      auto *BrInst = cast<BranchInst>(&I);3000 3001      // Unconditional branches are never considered UB.3002      if (BrInst->isUnconditional())3003        return true;3004 3005      // Either we stopped and the appropriate action was taken,3006      // or we got back a simplified value to continue.3007      std::optional<Value *> SimplifiedCond =3008          stopOnUndefOrAssumed(A, BrInst->getCondition(), BrInst);3009      if (!SimplifiedCond || !*SimplifiedCond)3010        return true;3011      AssumedNoUBInsts.insert(&I);3012      return true;3013    };3014 3015    auto InspectCallSiteForUB = [&](Instruction &I) {3016      // Check whether a callsite always cause UB or not3017 3018      // Skip instructions that are already saved.3019      if (AssumedNoUBInsts.count(&I) || KnownUBInsts.count(&I))3020        return true;3021 3022      // Check nonnull and noundef argument attribute violation for each3023      // callsite.3024      CallBase &CB = cast<CallBase>(I);3025      auto *Callee = dyn_cast_if_present<Function>(CB.getCalledOperand());3026      if (!Callee)3027        return true;3028      for (unsigned idx = 0; idx < CB.arg_size(); idx++) {3029        // If current argument is known to be simplified to null pointer and the3030        // corresponding argument position is known to have nonnull attribute,3031        // the argument is poison. Furthermore, if the argument is poison and3032        // the position is known to have noundef attriubte, this callsite is3033        // considered UB.3034        if (idx >= Callee->arg_size())3035          break;3036        Value *ArgVal = CB.getArgOperand(idx);3037        if (!ArgVal)3038          continue;3039        // Here, we handle three cases.3040        //   (1) Not having a value means it is dead. (we can replace the value3041        //       with undef)3042        //   (2) Simplified to undef. The argument violate noundef attriubte.3043        //   (3) Simplified to null pointer where known to be nonnull.3044        //       The argument is a poison value and violate noundef attribute.3045        IRPosition CalleeArgumentIRP = IRPosition::callsite_argument(CB, idx);3046        bool IsKnownNoUndef;3047        AA::hasAssumedIRAttr<Attribute::NoUndef>(3048            A, this, CalleeArgumentIRP, DepClassTy::NONE, IsKnownNoUndef);3049        if (!IsKnownNoUndef)3050          continue;3051        bool UsedAssumedInformation = false;3052        std::optional<Value *> SimplifiedVal =3053            A.getAssumedSimplified(IRPosition::value(*ArgVal), *this,3054                                   UsedAssumedInformation, AA::Interprocedural);3055        if (UsedAssumedInformation)3056          continue;3057        if (SimplifiedVal && !*SimplifiedVal)3058          return true;3059        if (!SimplifiedVal || isa<UndefValue>(**SimplifiedVal)) {3060          KnownUBInsts.insert(&I);3061          continue;3062        }3063        if (!ArgVal->getType()->isPointerTy() ||3064            !isa<ConstantPointerNull>(**SimplifiedVal))3065          continue;3066        bool IsKnownNonNull;3067        AA::hasAssumedIRAttr<Attribute::NonNull>(3068            A, this, CalleeArgumentIRP, DepClassTy::NONE, IsKnownNonNull);3069        if (IsKnownNonNull)3070          KnownUBInsts.insert(&I);3071      }3072      return true;3073    };3074 3075    auto InspectReturnInstForUB = [&](Instruction &I) {3076      auto &RI = cast<ReturnInst>(I);3077      // Either we stopped and the appropriate action was taken,3078      // or we got back a simplified return value to continue.3079      std::optional<Value *> SimplifiedRetValue =3080          stopOnUndefOrAssumed(A, RI.getReturnValue(), &I);3081      if (!SimplifiedRetValue || !*SimplifiedRetValue)3082        return true;3083 3084      // Check if a return instruction always cause UB or not3085      // Note: It is guaranteed that the returned position of the anchor3086      //       scope has noundef attribute when this is called.3087      //       We also ensure the return position is not "assumed dead"3088      //       because the returned value was then potentially simplified to3089      //       `undef` in AAReturnedValues without removing the `noundef`3090      //       attribute yet.3091 3092      // When the returned position has noundef attriubte, UB occurs in the3093      // following cases.3094      //   (1) Returned value is known to be undef.3095      //   (2) The value is known to be a null pointer and the returned3096      //       position has nonnull attribute (because the returned value is3097      //       poison).3098      if (isa<ConstantPointerNull>(*SimplifiedRetValue)) {3099        bool IsKnownNonNull;3100        AA::hasAssumedIRAttr<Attribute::NonNull>(3101            A, this, IRPosition::returned(*getAnchorScope()), DepClassTy::NONE,3102            IsKnownNonNull);3103        if (IsKnownNonNull)3104          KnownUBInsts.insert(&I);3105      }3106 3107      return true;3108    };3109 3110    bool UsedAssumedInformation = false;3111    A.checkForAllInstructions(InspectMemAccessInstForUB, *this,3112                              {Instruction::Load, Instruction::Store,3113                               Instruction::AtomicCmpXchg,3114                               Instruction::AtomicRMW},3115                              UsedAssumedInformation,3116                              /* CheckBBLivenessOnly */ true);3117    A.checkForAllInstructions(InspectBrInstForUB, *this, {Instruction::Br},3118                              UsedAssumedInformation,3119                              /* CheckBBLivenessOnly */ true);3120    A.checkForAllCallLikeInstructions(InspectCallSiteForUB, *this,3121                                      UsedAssumedInformation);3122 3123    // If the returned position of the anchor scope has noundef attriubte, check3124    // all returned instructions.3125    if (!getAnchorScope()->getReturnType()->isVoidTy()) {3126      const IRPosition &ReturnIRP = IRPosition::returned(*getAnchorScope());3127      if (!A.isAssumedDead(ReturnIRP, this, nullptr, UsedAssumedInformation)) {3128        bool IsKnownNoUndef;3129        AA::hasAssumedIRAttr<Attribute::NoUndef>(3130            A, this, ReturnIRP, DepClassTy::NONE, IsKnownNoUndef);3131        if (IsKnownNoUndef)3132          A.checkForAllInstructions(InspectReturnInstForUB, *this,3133                                    {Instruction::Ret}, UsedAssumedInformation,3134                                    /* CheckBBLivenessOnly */ true);3135      }3136    }3137 3138    if (NoUBPrevSize != AssumedNoUBInsts.size() ||3139        UBPrevSize != KnownUBInsts.size())3140      return ChangeStatus::CHANGED;3141    return ChangeStatus::UNCHANGED;3142  }3143 3144  bool isKnownToCauseUB(Instruction *I) const override {3145    return KnownUBInsts.count(I);3146  }3147 3148  bool isAssumedToCauseUB(Instruction *I) const override {3149    // In simple words, if an instruction is not in the assumed to _not_3150    // cause UB, then it is assumed UB (that includes those3151    // in the KnownUBInsts set). The rest is boilerplate3152    // is to ensure that it is one of the instructions we test3153    // for UB.3154 3155    switch (I->getOpcode()) {3156    case Instruction::Load:3157    case Instruction::Store:3158    case Instruction::AtomicCmpXchg:3159    case Instruction::AtomicRMW:3160      return !AssumedNoUBInsts.count(I);3161    case Instruction::Br: {3162      auto *BrInst = cast<BranchInst>(I);3163      if (BrInst->isUnconditional())3164        return false;3165      return !AssumedNoUBInsts.count(I);3166    } break;3167    default:3168      return false;3169    }3170    return false;3171  }3172 3173  ChangeStatus manifest(Attributor &A) override {3174    if (KnownUBInsts.empty())3175      return ChangeStatus::UNCHANGED;3176    for (Instruction *I : KnownUBInsts)3177      A.changeToUnreachableAfterManifest(I);3178    return ChangeStatus::CHANGED;3179  }3180 3181  /// See AbstractAttribute::getAsStr()3182  const std::string getAsStr(Attributor *A) const override {3183    return getAssumed() ? "undefined-behavior" : "no-ub";3184  }3185 3186  /// Note: The correctness of this analysis depends on the fact that the3187  /// following 2 sets will stop changing after some point.3188  /// "Change" here means that their size changes.3189  /// The size of each set is monotonically increasing3190  /// (we only add items to them) and it is upper bounded by the number of3191  /// instructions in the processed function (we can never save more3192  /// elements in either set than this number). Hence, at some point,3193  /// they will stop increasing.3194  /// Consequently, at some point, both sets will have stopped3195  /// changing, effectively making the analysis reach a fixpoint.3196 3197  /// Note: These 2 sets are disjoint and an instruction can be considered3198  /// one of 3 things:3199  /// 1) Known to cause UB (AAUndefinedBehavior could prove it) and put it in3200  ///    the KnownUBInsts set.3201  /// 2) Assumed to cause UB (in every updateImpl, AAUndefinedBehavior3202  ///    has a reason to assume it).3203  /// 3) Assumed to not cause UB. very other instruction - AAUndefinedBehavior3204  ///    could not find a reason to assume or prove that it can cause UB,3205  ///    hence it assumes it doesn't. We have a set for these instructions3206  ///    so that we don't reprocess them in every update.3207  ///    Note however that instructions in this set may cause UB.3208 3209protected:3210  /// A set of all live instructions _known_ to cause UB.3211  SmallPtrSet<Instruction *, 8> KnownUBInsts;3212 3213private:3214  /// A set of all the (live) instructions that are assumed to _not_ cause UB.3215  SmallPtrSet<Instruction *, 8> AssumedNoUBInsts;3216 3217  // Should be called on updates in which if we're processing an instruction3218  // \p I that depends on a value \p V, one of the following has to happen:3219  // - If the value is assumed, then stop.3220  // - If the value is known but undef, then consider it UB.3221  // - Otherwise, do specific processing with the simplified value.3222  // We return std::nullopt in the first 2 cases to signify that an appropriate3223  // action was taken and the caller should stop.3224  // Otherwise, we return the simplified value that the caller should3225  // use for specific processing.3226  std::optional<Value *> stopOnUndefOrAssumed(Attributor &A, Value *V,3227                                              Instruction *I) {3228    bool UsedAssumedInformation = false;3229    std::optional<Value *> SimplifiedV =3230        A.getAssumedSimplified(IRPosition::value(*V), *this,3231                               UsedAssumedInformation, AA::Interprocedural);3232    if (!UsedAssumedInformation) {3233      // Don't depend on assumed values.3234      if (!SimplifiedV) {3235        // If it is known (which we tested above) but it doesn't have a value,3236        // then we can assume `undef` and hence the instruction is UB.3237        KnownUBInsts.insert(I);3238        return std::nullopt;3239      }3240      if (!*SimplifiedV)3241        return nullptr;3242      V = *SimplifiedV;3243    }3244    if (isa<UndefValue>(V)) {3245      KnownUBInsts.insert(I);3246      return std::nullopt;3247    }3248    return V;3249  }3250};3251 3252struct AAUndefinedBehaviorFunction final : AAUndefinedBehaviorImpl {3253  AAUndefinedBehaviorFunction(const IRPosition &IRP, Attributor &A)3254      : AAUndefinedBehaviorImpl(IRP, A) {}3255 3256  /// See AbstractAttribute::trackStatistics()3257  void trackStatistics() const override {3258    STATS_DECL(UndefinedBehaviorInstruction, Instruction,3259               "Number of instructions known to have UB");3260    BUILD_STAT_NAME(UndefinedBehaviorInstruction, Instruction) +=3261        KnownUBInsts.size();3262  }3263};3264} // namespace3265 3266/// ------------------------ Will-Return Attributes ----------------------------3267 3268namespace {3269// Helper function that checks whether a function has any cycle which we don't3270// know if it is bounded or not.3271// Loops with maximum trip count are considered bounded, any other cycle not.3272static bool mayContainUnboundedCycle(Function &F, Attributor &A) {3273  ScalarEvolution *SE =3274      A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(F);3275  LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(F);3276  // If either SCEV or LoopInfo is not available for the function then we assume3277  // any cycle to be unbounded cycle.3278  // We use scc_iterator which uses Tarjan algorithm to find all the maximal3279  // SCCs.To detect if there's a cycle, we only need to find the maximal ones.3280  if (!SE || !LI) {3281    for (scc_iterator<Function *> SCCI = scc_begin(&F); !SCCI.isAtEnd(); ++SCCI)3282      if (SCCI.hasCycle())3283        return true;3284    return false;3285  }3286 3287  // If there's irreducible control, the function may contain non-loop cycles.3288  if (mayContainIrreducibleControl(F, LI))3289    return true;3290 3291  // Any loop that does not have a max trip count is considered unbounded cycle.3292  for (auto *L : LI->getLoopsInPreorder()) {3293    if (!SE->getSmallConstantMaxTripCount(L))3294      return true;3295  }3296  return false;3297}3298 3299struct AAWillReturnImpl : public AAWillReturn {3300  AAWillReturnImpl(const IRPosition &IRP, Attributor &A)3301      : AAWillReturn(IRP, A) {}3302 3303  /// See AbstractAttribute::initialize(...).3304  void initialize(Attributor &A) override {3305    bool IsKnown;3306    assert(!AA::hasAssumedIRAttr<Attribute::WillReturn>(3307        A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));3308    (void)IsKnown;3309  }3310 3311  /// Check for `mustprogress` and `readonly` as they imply `willreturn`.3312  bool isImpliedByMustprogressAndReadonly(Attributor &A, bool KnownOnly) {3313    if (!A.hasAttr(getIRPosition(), {Attribute::MustProgress}))3314      return false;3315 3316    bool IsKnown;3317    if (AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown))3318      return IsKnown || !KnownOnly;3319    return false;3320  }3321 3322  /// See AbstractAttribute::updateImpl(...).3323  ChangeStatus updateImpl(Attributor &A) override {3324    if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))3325      return ChangeStatus::UNCHANGED;3326 3327    auto CheckForWillReturn = [&](Instruction &I) {3328      IRPosition IPos = IRPosition::callsite_function(cast<CallBase>(I));3329      bool IsKnown;3330      if (AA::hasAssumedIRAttr<Attribute::WillReturn>(3331              A, this, IPos, DepClassTy::REQUIRED, IsKnown)) {3332        if (IsKnown)3333          return true;3334      } else {3335        return false;3336      }3337      bool IsKnownNoRecurse;3338      return AA::hasAssumedIRAttr<Attribute::NoRecurse>(3339          A, this, IPos, DepClassTy::REQUIRED, IsKnownNoRecurse);3340    };3341 3342    bool UsedAssumedInformation = false;3343    if (!A.checkForAllCallLikeInstructions(CheckForWillReturn, *this,3344                                           UsedAssumedInformation))3345      return indicatePessimisticFixpoint();3346 3347    return ChangeStatus::UNCHANGED;3348  }3349 3350  /// See AbstractAttribute::getAsStr()3351  const std::string getAsStr(Attributor *A) const override {3352    return getAssumed() ? "willreturn" : "may-noreturn";3353  }3354};3355 3356struct AAWillReturnFunction final : AAWillReturnImpl {3357  AAWillReturnFunction(const IRPosition &IRP, Attributor &A)3358      : AAWillReturnImpl(IRP, A) {}3359 3360  /// See AbstractAttribute::initialize(...).3361  void initialize(Attributor &A) override {3362    AAWillReturnImpl::initialize(A);3363 3364    Function *F = getAnchorScope();3365    assert(F && "Did expect an anchor function");3366    if (F->isDeclaration() || mayContainUnboundedCycle(*F, A))3367      indicatePessimisticFixpoint();3368  }3369 3370  /// See AbstractAttribute::trackStatistics()3371  void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(willreturn) }3372};3373 3374/// WillReturn attribute deduction for a call sites.3375struct AAWillReturnCallSite final3376    : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl> {3377  AAWillReturnCallSite(const IRPosition &IRP, Attributor &A)3378      : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl>(IRP, A) {}3379 3380  /// See AbstractAttribute::updateImpl(...).3381  ChangeStatus updateImpl(Attributor &A) override {3382    if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))3383      return ChangeStatus::UNCHANGED;3384 3385    return AACalleeToCallSite::updateImpl(A);3386  }3387 3388  /// See AbstractAttribute::trackStatistics()3389  void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(willreturn); }3390};3391} // namespace3392 3393/// -------------------AAIntraFnReachability Attribute--------------------------3394 3395/// All information associated with a reachability query. This boilerplate code3396/// is used by both AAIntraFnReachability and AAInterFnReachability, with3397/// different \p ToTy values.3398template <typename ToTy> struct ReachabilityQueryInfo {3399  enum class Reachable {3400    No,3401    Yes,3402  };3403 3404  /// Start here,3405  const Instruction *From = nullptr;3406  /// reach this place,3407  const ToTy *To = nullptr;3408  /// without going through any of these instructions,3409  const AA::InstExclusionSetTy *ExclusionSet = nullptr;3410  /// and remember if it worked:3411  Reachable Result = Reachable::No;3412 3413  /// Precomputed hash for this RQI.3414  unsigned Hash = 0;3415 3416  unsigned computeHashValue() const {3417    assert(Hash == 0 && "Computed hash twice!");3418    using InstSetDMI = DenseMapInfo<const AA::InstExclusionSetTy *>;3419    using PairDMI = DenseMapInfo<std::pair<const Instruction *, const ToTy *>>;3420    return const_cast<ReachabilityQueryInfo<ToTy> *>(this)->Hash =3421               detail::combineHashValue(PairDMI ::getHashValue({From, To}),3422                                        InstSetDMI::getHashValue(ExclusionSet));3423  }3424 3425  ReachabilityQueryInfo(const Instruction *From, const ToTy *To)3426      : From(From), To(To) {}3427 3428  /// Constructor replacement to ensure unique and stable sets are used for the3429  /// cache.3430  ReachabilityQueryInfo(Attributor &A, const Instruction &From, const ToTy &To,3431                        const AA::InstExclusionSetTy *ES, bool MakeUnique)3432      : From(&From), To(&To), ExclusionSet(ES) {3433 3434    if (!ES || ES->empty()) {3435      ExclusionSet = nullptr;3436    } else if (MakeUnique) {3437      ExclusionSet = A.getInfoCache().getOrCreateUniqueBlockExecutionSet(ES);3438    }3439  }3440 3441  ReachabilityQueryInfo(const ReachabilityQueryInfo &RQI)3442      : From(RQI.From), To(RQI.To), ExclusionSet(RQI.ExclusionSet) {}3443};3444 3445namespace llvm {3446template <typename ToTy> struct DenseMapInfo<ReachabilityQueryInfo<ToTy> *> {3447  using InstSetDMI = DenseMapInfo<const AA::InstExclusionSetTy *>;3448  using PairDMI = DenseMapInfo<std::pair<const Instruction *, const ToTy *>>;3449 3450  static ReachabilityQueryInfo<ToTy> EmptyKey;3451  static ReachabilityQueryInfo<ToTy> TombstoneKey;3452 3453  static inline ReachabilityQueryInfo<ToTy> *getEmptyKey() { return &EmptyKey; }3454  static inline ReachabilityQueryInfo<ToTy> *getTombstoneKey() {3455    return &TombstoneKey;3456  }3457  static unsigned getHashValue(const ReachabilityQueryInfo<ToTy> *RQI) {3458    return RQI->Hash ? RQI->Hash : RQI->computeHashValue();3459  }3460  static bool isEqual(const ReachabilityQueryInfo<ToTy> *LHS,3461                      const ReachabilityQueryInfo<ToTy> *RHS) {3462    if (!PairDMI::isEqual({LHS->From, LHS->To}, {RHS->From, RHS->To}))3463      return false;3464    return InstSetDMI::isEqual(LHS->ExclusionSet, RHS->ExclusionSet);3465  }3466};3467 3468#define DefineKeys(ToTy)                                                       \3469  template <>                                                                  \3470  ReachabilityQueryInfo<ToTy>                                                  \3471      DenseMapInfo<ReachabilityQueryInfo<ToTy> *>::EmptyKey =                  \3472          ReachabilityQueryInfo<ToTy>(                                         \3473              DenseMapInfo<const Instruction *>::getEmptyKey(),                \3474              DenseMapInfo<const ToTy *>::getEmptyKey());                      \3475  template <>                                                                  \3476  ReachabilityQueryInfo<ToTy>                                                  \3477      DenseMapInfo<ReachabilityQueryInfo<ToTy> *>::TombstoneKey =              \3478          ReachabilityQueryInfo<ToTy>(                                         \3479              DenseMapInfo<const Instruction *>::getTombstoneKey(),            \3480              DenseMapInfo<const ToTy *>::getTombstoneKey());3481 3482DefineKeys(Instruction) DefineKeys(Function)3483#undef DefineKeys3484 3485} // namespace llvm3486 3487namespace {3488 3489template <typename BaseTy, typename ToTy>3490struct CachedReachabilityAA : public BaseTy {3491  using RQITy = ReachabilityQueryInfo<ToTy>;3492 3493  CachedReachabilityAA(const IRPosition &IRP, Attributor &A) : BaseTy(IRP, A) {}3494 3495  /// See AbstractAttribute::isQueryAA.3496  bool isQueryAA() const override { return true; }3497 3498  /// See AbstractAttribute::updateImpl(...).3499  ChangeStatus updateImpl(Attributor &A) override {3500    ChangeStatus Changed = ChangeStatus::UNCHANGED;3501    for (unsigned u = 0, e = QueryVector.size(); u < e; ++u) {3502      RQITy *RQI = QueryVector[u];3503      if (RQI->Result == RQITy::Reachable::No &&3504          isReachableImpl(A, *RQI, /*IsTemporaryRQI=*/false))3505        Changed = ChangeStatus::CHANGED;3506    }3507    return Changed;3508  }3509 3510  virtual bool isReachableImpl(Attributor &A, RQITy &RQI,3511                               bool IsTemporaryRQI) = 0;3512 3513  bool rememberResult(Attributor &A, typename RQITy::Reachable Result,3514                      RQITy &RQI, bool UsedExclusionSet, bool IsTemporaryRQI) {3515    RQI.Result = Result;3516 3517    // Remove the temporary RQI from the cache.3518    if (IsTemporaryRQI)3519      QueryCache.erase(&RQI);3520 3521    // Insert a plain RQI (w/o exclusion set) if that makes sense. Two options:3522    // 1) If it is reachable, it doesn't matter if we have an exclusion set for3523    // this query. 2) We did not use the exclusion set, potentially because3524    // there is none.3525    if (Result == RQITy::Reachable::Yes || !UsedExclusionSet) {3526      RQITy PlainRQI(RQI.From, RQI.To);3527      if (!QueryCache.count(&PlainRQI)) {3528        RQITy *RQIPtr = new (A.Allocator) RQITy(RQI.From, RQI.To);3529        RQIPtr->Result = Result;3530        QueryVector.push_back(RQIPtr);3531        QueryCache.insert(RQIPtr);3532      }3533    }3534 3535    // Check if we need to insert a new permanent RQI with the exclusion set.3536    if (IsTemporaryRQI && Result != RQITy::Reachable::Yes && UsedExclusionSet) {3537      assert((!RQI.ExclusionSet || !RQI.ExclusionSet->empty()) &&3538             "Did not expect empty set!");3539      RQITy *RQIPtr = new (A.Allocator)3540          RQITy(A, *RQI.From, *RQI.To, RQI.ExclusionSet, true);3541      assert(RQIPtr->Result == RQITy::Reachable::No && "Already reachable?");3542      RQIPtr->Result = Result;3543      assert(!QueryCache.count(RQIPtr));3544      QueryVector.push_back(RQIPtr);3545      QueryCache.insert(RQIPtr);3546    }3547 3548    if (Result == RQITy::Reachable::No && IsTemporaryRQI)3549      A.registerForUpdate(*this);3550    return Result == RQITy::Reachable::Yes;3551  }3552 3553  const std::string getAsStr(Attributor *A) const override {3554    // TODO: Return the number of reachable queries.3555    return "#queries(" + std::to_string(QueryVector.size()) + ")";3556  }3557 3558  bool checkQueryCache(Attributor &A, RQITy &StackRQI,3559                       typename RQITy::Reachable &Result) {3560    if (!this->getState().isValidState()) {3561      Result = RQITy::Reachable::Yes;3562      return true;3563    }3564 3565    // If we have an exclusion set we might be able to find our answer by3566    // ignoring it first.3567    if (StackRQI.ExclusionSet) {3568      RQITy PlainRQI(StackRQI.From, StackRQI.To);3569      auto It = QueryCache.find(&PlainRQI);3570      if (It != QueryCache.end() && (*It)->Result == RQITy::Reachable::No) {3571        Result = RQITy::Reachable::No;3572        return true;3573      }3574    }3575 3576    auto It = QueryCache.find(&StackRQI);3577    if (It != QueryCache.end()) {3578      Result = (*It)->Result;3579      return true;3580    }3581 3582    // Insert a temporary for recursive queries. We will replace it with a3583    // permanent entry later.3584    QueryCache.insert(&StackRQI);3585    return false;3586  }3587 3588private:3589  SmallVector<RQITy *> QueryVector;3590  DenseSet<RQITy *> QueryCache;3591};3592 3593struct AAIntraFnReachabilityFunction final3594    : public CachedReachabilityAA<AAIntraFnReachability, Instruction> {3595  using Base = CachedReachabilityAA<AAIntraFnReachability, Instruction>;3596  AAIntraFnReachabilityFunction(const IRPosition &IRP, Attributor &A)3597      : Base(IRP, A) {3598    DT = A.getInfoCache().getAnalysisResultForFunction<DominatorTreeAnalysis>(3599        *IRP.getAssociatedFunction());3600  }3601 3602  bool isAssumedReachable(3603      Attributor &A, const Instruction &From, const Instruction &To,3604      const AA::InstExclusionSetTy *ExclusionSet) const override {3605    auto *NonConstThis = const_cast<AAIntraFnReachabilityFunction *>(this);3606    if (&From == &To)3607      return true;3608 3609    RQITy StackRQI(A, From, To, ExclusionSet, false);3610    RQITy::Reachable Result;3611    if (!NonConstThis->checkQueryCache(A, StackRQI, Result))3612      return NonConstThis->isReachableImpl(A, StackRQI,3613                                           /*IsTemporaryRQI=*/true);3614    return Result == RQITy::Reachable::Yes;3615  }3616 3617  ChangeStatus updateImpl(Attributor &A) override {3618    // We only depend on liveness. DeadEdges is all we care about, check if any3619    // of them changed.3620    auto *LivenessAA =3621        A.getAAFor<AAIsDead>(*this, getIRPosition(), DepClassTy::OPTIONAL);3622    if (LivenessAA &&3623        llvm::all_of(DeadEdges,3624                     [&](const auto &DeadEdge) {3625                       return LivenessAA->isEdgeDead(DeadEdge.first,3626                                                     DeadEdge.second);3627                     }) &&3628        llvm::all_of(DeadBlocks, [&](const BasicBlock *BB) {3629          return LivenessAA->isAssumedDead(BB);3630        })) {3631      return ChangeStatus::UNCHANGED;3632    }3633    DeadEdges.clear();3634    DeadBlocks.clear();3635    return Base::updateImpl(A);3636  }3637 3638  bool isReachableImpl(Attributor &A, RQITy &RQI,3639                       bool IsTemporaryRQI) override {3640    const Instruction *Origin = RQI.From;3641    bool UsedExclusionSet = false;3642 3643    auto WillReachInBlock = [&](const Instruction &From, const Instruction &To,3644                                const AA::InstExclusionSetTy *ExclusionSet) {3645      const Instruction *IP = &From;3646      while (IP && IP != &To) {3647        if (ExclusionSet && IP != Origin && ExclusionSet->count(IP)) {3648          UsedExclusionSet = true;3649          break;3650        }3651        IP = IP->getNextNode();3652      }3653      return IP == &To;3654    };3655 3656    const BasicBlock *FromBB = RQI.From->getParent();3657    const BasicBlock *ToBB = RQI.To->getParent();3658    assert(FromBB->getParent() == ToBB->getParent() &&3659           "Not an intra-procedural query!");3660 3661    // Check intra-block reachability, however, other reaching paths are still3662    // possible.3663    if (FromBB == ToBB &&3664        WillReachInBlock(*RQI.From, *RQI.To, RQI.ExclusionSet))3665      return rememberResult(A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,3666                            IsTemporaryRQI);3667 3668    // Check if reaching the ToBB block is sufficient or if even that would not3669    // ensure reaching the target. In the latter case we are done.3670    if (!WillReachInBlock(ToBB->front(), *RQI.To, RQI.ExclusionSet))3671      return rememberResult(A, RQITy::Reachable::No, RQI, UsedExclusionSet,3672                            IsTemporaryRQI);3673 3674    const Function *Fn = FromBB->getParent();3675    SmallPtrSet<const BasicBlock *, 16> ExclusionBlocks;3676    if (RQI.ExclusionSet)3677      for (auto *I : *RQI.ExclusionSet)3678        if (I->getFunction() == Fn)3679          ExclusionBlocks.insert(I->getParent());3680 3681    // Check if we make it out of the FromBB block at all.3682    if (ExclusionBlocks.count(FromBB) &&3683        !WillReachInBlock(*RQI.From, *FromBB->getTerminator(),3684                          RQI.ExclusionSet))3685      return rememberResult(A, RQITy::Reachable::No, RQI, true, IsTemporaryRQI);3686 3687    auto *LivenessAA =3688        A.getAAFor<AAIsDead>(*this, getIRPosition(), DepClassTy::OPTIONAL);3689    if (LivenessAA && LivenessAA->isAssumedDead(ToBB)) {3690      DeadBlocks.insert(ToBB);3691      return rememberResult(A, RQITy::Reachable::No, RQI, UsedExclusionSet,3692                            IsTemporaryRQI);3693    }3694 3695    SmallPtrSet<const BasicBlock *, 16> Visited;3696    SmallVector<const BasicBlock *, 16> Worklist;3697    Worklist.push_back(FromBB);3698 3699    DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> LocalDeadEdges;3700    while (!Worklist.empty()) {3701      const BasicBlock *BB = Worklist.pop_back_val();3702      if (!Visited.insert(BB).second)3703        continue;3704      for (const BasicBlock *SuccBB : successors(BB)) {3705        if (LivenessAA && LivenessAA->isEdgeDead(BB, SuccBB)) {3706          LocalDeadEdges.insert({BB, SuccBB});3707          continue;3708        }3709        // We checked before if we just need to reach the ToBB block.3710        if (SuccBB == ToBB)3711          return rememberResult(A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,3712                                IsTemporaryRQI);3713        if (DT && ExclusionBlocks.empty() && DT->dominates(BB, ToBB))3714          return rememberResult(A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,3715                                IsTemporaryRQI);3716 3717        if (ExclusionBlocks.count(SuccBB)) {3718          UsedExclusionSet = true;3719          continue;3720        }3721        Worklist.push_back(SuccBB);3722      }3723    }3724 3725    DeadEdges.insert_range(LocalDeadEdges);3726    return rememberResult(A, RQITy::Reachable::No, RQI, UsedExclusionSet,3727                          IsTemporaryRQI);3728  }3729 3730  /// See AbstractAttribute::trackStatistics()3731  void trackStatistics() const override {}3732 3733private:3734  // Set of assumed dead blocks we used in the last query. If any changes we3735  // update the state.3736  DenseSet<const BasicBlock *> DeadBlocks;3737 3738  // Set of assumed dead edges we used in the last query. If any changes we3739  // update the state.3740  DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> DeadEdges;3741 3742  /// The dominator tree of the function to short-circuit reasoning.3743  const DominatorTree *DT = nullptr;3744};3745} // namespace3746 3747/// ------------------------ NoAlias Argument Attribute ------------------------3748 3749bool AANoAlias::isImpliedByIR(Attributor &A, const IRPosition &IRP,3750                              Attribute::AttrKind ImpliedAttributeKind,3751                              bool IgnoreSubsumingPositions) {3752  assert(ImpliedAttributeKind == Attribute::NoAlias &&3753         "Unexpected attribute kind");3754  Value *Val = &IRP.getAssociatedValue();3755  if (IRP.getPositionKind() != IRP_CALL_SITE_ARGUMENT) {3756    if (isa<AllocaInst>(Val))3757      return true;3758  } else {3759    IgnoreSubsumingPositions = true;3760  }3761 3762  if (isa<UndefValue>(Val))3763    return true;3764 3765  if (isa<ConstantPointerNull>(Val) &&3766      !NullPointerIsDefined(IRP.getAnchorScope(),3767                            Val->getType()->getPointerAddressSpace()))3768    return true;3769 3770  if (A.hasAttr(IRP, {Attribute::ByVal, Attribute::NoAlias},3771                IgnoreSubsumingPositions, Attribute::NoAlias))3772    return true;3773 3774  return false;3775}3776 3777namespace {3778struct AANoAliasImpl : AANoAlias {3779  AANoAliasImpl(const IRPosition &IRP, Attributor &A) : AANoAlias(IRP, A) {3780    assert(getAssociatedType()->isPointerTy() &&3781           "Noalias is a pointer attribute");3782  }3783 3784  const std::string getAsStr(Attributor *A) const override {3785    return getAssumed() ? "noalias" : "may-alias";3786  }3787};3788 3789/// NoAlias attribute for a floating value.3790struct AANoAliasFloating final : AANoAliasImpl {3791  AANoAliasFloating(const IRPosition &IRP, Attributor &A)3792      : AANoAliasImpl(IRP, A) {}3793 3794  /// See AbstractAttribute::updateImpl(...).3795  ChangeStatus updateImpl(Attributor &A) override {3796    // TODO: Implement this.3797    return indicatePessimisticFixpoint();3798  }3799 3800  /// See AbstractAttribute::trackStatistics()3801  void trackStatistics() const override {3802    STATS_DECLTRACK_FLOATING_ATTR(noalias)3803  }3804};3805 3806/// NoAlias attribute for an argument.3807struct AANoAliasArgument final3808    : AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl> {3809  using Base = AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl>;3810  AANoAliasArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}3811 3812  /// See AbstractAttribute::update(...).3813  ChangeStatus updateImpl(Attributor &A) override {3814    // We have to make sure no-alias on the argument does not break3815    // synchronization when this is a callback argument, see also [1] below.3816    // If synchronization cannot be affected, we delegate to the base updateImpl3817    // function, otherwise we give up for now.3818 3819    // If the function is no-sync, no-alias cannot break synchronization.3820    bool IsKnownNoSycn;3821    if (AA::hasAssumedIRAttr<Attribute::NoSync>(3822            A, this, IRPosition::function_scope(getIRPosition()),3823            DepClassTy::OPTIONAL, IsKnownNoSycn))3824      return Base::updateImpl(A);3825 3826    // If the argument is read-only, no-alias cannot break synchronization.3827    bool IsKnown;3828    if (AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown))3829      return Base::updateImpl(A);3830 3831    // If the argument is never passed through callbacks, no-alias cannot break3832    // synchronization.3833    bool UsedAssumedInformation = false;3834    if (A.checkForAllCallSites(3835            [](AbstractCallSite ACS) { return !ACS.isCallbackCall(); }, *this,3836            true, UsedAssumedInformation))3837      return Base::updateImpl(A);3838 3839    // TODO: add no-alias but make sure it doesn't break synchronization by3840    // introducing fake uses. See:3841    // [1] Compiler Optimizations for OpenMP, J. Doerfert and H. Finkel,3842    //     International Workshop on OpenMP 2018,3843    //     http://compilers.cs.uni-saarland.de/people/doerfert/par_opt18.pdf3844 3845    return indicatePessimisticFixpoint();3846  }3847 3848  /// See AbstractAttribute::trackStatistics()3849  void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noalias) }3850};3851 3852struct AANoAliasCallSiteArgument final : AANoAliasImpl {3853  AANoAliasCallSiteArgument(const IRPosition &IRP, Attributor &A)3854      : AANoAliasImpl(IRP, A) {}3855 3856  /// Determine if the underlying value may alias with the call site argument3857  /// \p OtherArgNo of \p ICS (= the underlying call site).3858  bool mayAliasWithArgument(Attributor &A, AAResults *&AAR,3859                            const AAMemoryBehavior &MemBehaviorAA,3860                            const CallBase &CB, unsigned OtherArgNo) {3861    // We do not need to worry about aliasing with the underlying IRP.3862    if (this->getCalleeArgNo() == (int)OtherArgNo)3863      return false;3864 3865    // If it is not a pointer or pointer vector we do not alias.3866    const Value *ArgOp = CB.getArgOperand(OtherArgNo);3867    if (!ArgOp->getType()->isPtrOrPtrVectorTy())3868      return false;3869 3870    auto *CBArgMemBehaviorAA = A.getAAFor<AAMemoryBehavior>(3871        *this, IRPosition::callsite_argument(CB, OtherArgNo), DepClassTy::NONE);3872 3873    // If the argument is readnone, there is no read-write aliasing.3874    if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadNone()) {3875      A.recordDependence(*CBArgMemBehaviorAA, *this, DepClassTy::OPTIONAL);3876      return false;3877    }3878 3879    // If the argument is readonly and the underlying value is readonly, there3880    // is no read-write aliasing.3881    bool IsReadOnly = MemBehaviorAA.isAssumedReadOnly();3882    if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadOnly() &&3883        IsReadOnly) {3884      A.recordDependence(MemBehaviorAA, *this, DepClassTy::OPTIONAL);3885      A.recordDependence(*CBArgMemBehaviorAA, *this, DepClassTy::OPTIONAL);3886      return false;3887    }3888 3889    // We have to utilize actual alias analysis queries so we need the object.3890    if (!AAR)3891      AAR = A.getInfoCache().getAnalysisResultForFunction<AAManager>(3892          *getAnchorScope());3893 3894    // Try to rule it out at the call site.3895    bool IsAliasing = !AAR || !AAR->isNoAlias(&getAssociatedValue(), ArgOp);3896    LLVM_DEBUG(dbgs() << "[NoAliasCSArg] Check alias between "3897                         "callsite arguments: "3898                      << getAssociatedValue() << " " << *ArgOp << " => "3899                      << (IsAliasing ? "" : "no-") << "alias \n");3900 3901    return IsAliasing;3902  }3903 3904  bool isKnownNoAliasDueToNoAliasPreservation(3905      Attributor &A, AAResults *&AAR, const AAMemoryBehavior &MemBehaviorAA) {3906    // We can deduce "noalias" if the following conditions hold.3907    // (i)   Associated value is assumed to be noalias in the definition.3908    // (ii)  Associated value is assumed to be no-capture in all the uses3909    //       possibly executed before this callsite.3910    // (iii) There is no other pointer argument which could alias with the3911    //       value.3912 3913    const IRPosition &VIRP = IRPosition::value(getAssociatedValue());3914    const Function *ScopeFn = VIRP.getAnchorScope();3915    // Check whether the value is captured in the scope using AANoCapture.3916    // Look at CFG and check only uses possibly executed before this3917    // callsite.3918    auto UsePred = [&](const Use &U, bool &Follow) -> bool {3919      Instruction *UserI = cast<Instruction>(U.getUser());3920 3921      // If UserI is the curr instruction and there is a single potential use of3922      // the value in UserI we allow the use.3923      // TODO: We should inspect the operands and allow those that cannot alias3924      //       with the value.3925      if (UserI == getCtxI() && UserI->getNumOperands() == 1)3926        return true;3927 3928      if (ScopeFn) {3929        if (auto *CB = dyn_cast<CallBase>(UserI)) {3930          if (CB->isArgOperand(&U)) {3931 3932            unsigned ArgNo = CB->getArgOperandNo(&U);3933 3934            bool IsKnownNoCapture;3935            if (AA::hasAssumedIRAttr<Attribute::Captures>(3936                    A, this, IRPosition::callsite_argument(*CB, ArgNo),3937                    DepClassTy::OPTIONAL, IsKnownNoCapture))3938              return true;3939          }3940        }3941 3942        if (!AA::isPotentiallyReachable(3943                A, *UserI, *getCtxI(), *this, /* ExclusionSet */ nullptr,3944                [ScopeFn](const Function &Fn) { return &Fn != ScopeFn; }))3945          return true;3946      }3947 3948      // TODO: We should track the capturing uses in AANoCapture but the problem3949      //       is CGSCC runs. For those we would need to "allow" AANoCapture for3950      //       a value in the module slice.3951      // TODO(captures): Make this more precise.3952      UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);3953      if (capturesNothing(CI))3954        return true;3955      if (CI.isPassthrough()) {3956        Follow = true;3957        return true;3958      }3959      LLVM_DEBUG(dbgs() << "[AANoAliasCSArg] Unknown user: " << *UserI << "\n");3960      return false;3961    };3962 3963    bool IsKnownNoCapture;3964    const AANoCapture *NoCaptureAA = nullptr;3965    bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(3966        A, this, VIRP, DepClassTy::NONE, IsKnownNoCapture, false, &NoCaptureAA);3967    if (!IsAssumedNoCapture &&3968        (!NoCaptureAA || !NoCaptureAA->isAssumedNoCaptureMaybeReturned())) {3969      if (!A.checkForAllUses(UsePred, *this, getAssociatedValue())) {3970        LLVM_DEBUG(3971            dbgs() << "[AANoAliasCSArg] " << getAssociatedValue()3972                   << " cannot be noalias as it is potentially captured\n");3973        return false;3974      }3975    }3976    if (NoCaptureAA)3977      A.recordDependence(*NoCaptureAA, *this, DepClassTy::OPTIONAL);3978 3979    // Check there is no other pointer argument which could alias with the3980    // value passed at this call site.3981    // TODO: AbstractCallSite3982    const auto &CB = cast<CallBase>(getAnchorValue());3983    for (unsigned OtherArgNo = 0; OtherArgNo < CB.arg_size(); OtherArgNo++)3984      if (mayAliasWithArgument(A, AAR, MemBehaviorAA, CB, OtherArgNo))3985        return false;3986 3987    return true;3988  }3989 3990  /// See AbstractAttribute::updateImpl(...).3991  ChangeStatus updateImpl(Attributor &A) override {3992    // If the argument is readnone we are done as there are no accesses via the3993    // argument.3994    auto *MemBehaviorAA =3995        A.getAAFor<AAMemoryBehavior>(*this, getIRPosition(), DepClassTy::NONE);3996    if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {3997      A.recordDependence(*MemBehaviorAA, *this, DepClassTy::OPTIONAL);3998      return ChangeStatus::UNCHANGED;3999    }4000 4001    bool IsKnownNoAlias;4002    const IRPosition &VIRP = IRPosition::value(getAssociatedValue());4003    if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(4004            A, this, VIRP, DepClassTy::REQUIRED, IsKnownNoAlias)) {4005      LLVM_DEBUG(dbgs() << "[AANoAlias] " << getAssociatedValue()4006                        << " is not no-alias at the definition\n");4007      return indicatePessimisticFixpoint();4008    }4009 4010    AAResults *AAR = nullptr;4011    if (MemBehaviorAA &&4012        isKnownNoAliasDueToNoAliasPreservation(A, AAR, *MemBehaviorAA)) {4013      LLVM_DEBUG(4014          dbgs() << "[AANoAlias] No-Alias deduced via no-alias preservation\n");4015      return ChangeStatus::UNCHANGED;4016    }4017 4018    return indicatePessimisticFixpoint();4019  }4020 4021  /// See AbstractAttribute::trackStatistics()4022  void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noalias) }4023};4024 4025/// NoAlias attribute for function return value.4026struct AANoAliasReturned final : AANoAliasImpl {4027  AANoAliasReturned(const IRPosition &IRP, Attributor &A)4028      : AANoAliasImpl(IRP, A) {}4029 4030  /// See AbstractAttribute::updateImpl(...).4031  ChangeStatus updateImpl(Attributor &A) override {4032 4033    auto CheckReturnValue = [&](Value &RV) -> bool {4034      if (Constant *C = dyn_cast<Constant>(&RV))4035        if (C->isNullValue() || isa<UndefValue>(C))4036          return true;4037 4038      /// For now, we can only deduce noalias if we have call sites.4039      /// FIXME: add more support.4040      if (!isa<CallBase>(&RV))4041        return false;4042 4043      const IRPosition &RVPos = IRPosition::value(RV);4044      bool IsKnownNoAlias;4045      if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(4046              A, this, RVPos, DepClassTy::REQUIRED, IsKnownNoAlias))4047        return false;4048 4049      bool IsKnownNoCapture;4050      const AANoCapture *NoCaptureAA = nullptr;4051      bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(4052          A, this, RVPos, DepClassTy::REQUIRED, IsKnownNoCapture, false,4053          &NoCaptureAA);4054      return IsAssumedNoCapture ||4055             (NoCaptureAA && NoCaptureAA->isAssumedNoCaptureMaybeReturned());4056    };4057 4058    if (!A.checkForAllReturnedValues(CheckReturnValue, *this))4059      return indicatePessimisticFixpoint();4060 4061    return ChangeStatus::UNCHANGED;4062  }4063 4064  /// See AbstractAttribute::trackStatistics()4065  void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noalias) }4066};4067 4068/// NoAlias attribute deduction for a call site return value.4069struct AANoAliasCallSiteReturned final4070    : AACalleeToCallSite<AANoAlias, AANoAliasImpl> {4071  AANoAliasCallSiteReturned(const IRPosition &IRP, Attributor &A)4072      : AACalleeToCallSite<AANoAlias, AANoAliasImpl>(IRP, A) {}4073 4074  /// See AbstractAttribute::trackStatistics()4075  void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noalias); }4076};4077} // namespace4078 4079/// -------------------AAIsDead Function Attribute-----------------------4080 4081namespace {4082struct AAIsDeadValueImpl : public AAIsDead {4083  AAIsDeadValueImpl(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}4084 4085  /// See AAIsDead::isAssumedDead().4086  bool isAssumedDead() const override { return isAssumed(IS_DEAD); }4087 4088  /// See AAIsDead::isKnownDead().4089  bool isKnownDead() const override { return isKnown(IS_DEAD); }4090 4091  /// See AAIsDead::isAssumedDead(BasicBlock *).4092  bool isAssumedDead(const BasicBlock *BB) const override { return false; }4093 4094  /// See AAIsDead::isKnownDead(BasicBlock *).4095  bool isKnownDead(const BasicBlock *BB) const override { return false; }4096 4097  /// See AAIsDead::isAssumedDead(Instruction *I).4098  bool isAssumedDead(const Instruction *I) const override {4099    return I == getCtxI() && isAssumedDead();4100  }4101 4102  /// See AAIsDead::isKnownDead(Instruction *I).4103  bool isKnownDead(const Instruction *I) const override {4104    return isAssumedDead(I) && isKnownDead();4105  }4106 4107  /// See AbstractAttribute::getAsStr().4108  const std::string getAsStr(Attributor *A) const override {4109    return isAssumedDead() ? "assumed-dead" : "assumed-live";4110  }4111 4112  /// Check if all uses are assumed dead.4113  bool areAllUsesAssumedDead(Attributor &A, Value &V) {4114    // Callers might not check the type, void has no uses.4115    if (V.getType()->isVoidTy() || V.use_empty())4116      return true;4117 4118    // If we replace a value with a constant there are no uses left afterwards.4119    if (!isa<Constant>(V)) {4120      if (auto *I = dyn_cast<Instruction>(&V))4121        if (!A.isRunOn(*I->getFunction()))4122          return false;4123      bool UsedAssumedInformation = false;4124      std::optional<Constant *> C =4125          A.getAssumedConstant(V, *this, UsedAssumedInformation);4126      if (!C || *C)4127        return true;4128    }4129 4130    auto UsePred = [&](const Use &U, bool &Follow) { return false; };4131    // Explicitly set the dependence class to required because we want a long4132    // chain of N dependent instructions to be considered live as soon as one is4133    // without going through N update cycles. This is not required for4134    // correctness.4135    return A.checkForAllUses(UsePred, *this, V, /* CheckBBLivenessOnly */ false,4136                             DepClassTy::REQUIRED,4137                             /* IgnoreDroppableUses */ false);4138  }4139 4140  /// Determine if \p I is assumed to be side-effect free.4141  bool isAssumedSideEffectFree(Attributor &A, Instruction *I) {4142    if (!I || wouldInstructionBeTriviallyDead(I))4143      return true;4144 4145    auto *CB = dyn_cast<CallBase>(I);4146    if (!CB || isa<IntrinsicInst>(CB))4147      return false;4148 4149    const IRPosition &CallIRP = IRPosition::callsite_function(*CB);4150 4151    bool IsKnownNoUnwind;4152    if (!AA::hasAssumedIRAttr<Attribute::NoUnwind>(4153            A, this, CallIRP, DepClassTy::OPTIONAL, IsKnownNoUnwind))4154      return false;4155 4156    bool IsKnown;4157    return AA::isAssumedReadOnly(A, CallIRP, *this, IsKnown);4158  }4159};4160 4161struct AAIsDeadFloating : public AAIsDeadValueImpl {4162  AAIsDeadFloating(const IRPosition &IRP, Attributor &A)4163      : AAIsDeadValueImpl(IRP, A) {}4164 4165  /// See AbstractAttribute::initialize(...).4166  void initialize(Attributor &A) override {4167    AAIsDeadValueImpl::initialize(A);4168 4169    if (isa<UndefValue>(getAssociatedValue())) {4170      indicatePessimisticFixpoint();4171      return;4172    }4173 4174    Instruction *I = dyn_cast<Instruction>(&getAssociatedValue());4175    if (!isAssumedSideEffectFree(A, I)) {4176      if (!isa_and_nonnull<StoreInst>(I) && !isa_and_nonnull<FenceInst>(I))4177        indicatePessimisticFixpoint();4178      else4179        removeAssumedBits(HAS_NO_EFFECT);4180    }4181  }4182 4183  bool isDeadFence(Attributor &A, FenceInst &FI) {4184    const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(4185        IRPosition::function(*FI.getFunction()), *this, DepClassTy::NONE);4186    if (!ExecDomainAA || !ExecDomainAA->isNoOpFence(FI))4187      return false;4188    A.recordDependence(*ExecDomainAA, *this, DepClassTy::OPTIONAL);4189    return true;4190  }4191 4192  bool isDeadStore(Attributor &A, StoreInst &SI,4193                   SmallSetVector<Instruction *, 8> *AssumeOnlyInst = nullptr) {4194    // Lang ref now states volatile store is not UB/dead, let's skip them.4195    if (SI.isVolatile())4196      return false;4197 4198    // If we are collecting assumes to be deleted we are in the manifest stage.4199    // It's problematic to collect the potential copies again now so we use the4200    // cached ones.4201    bool UsedAssumedInformation = false;4202    if (!AssumeOnlyInst) {4203      PotentialCopies.clear();4204      if (!AA::getPotentialCopiesOfStoredValue(A, SI, PotentialCopies, *this,4205                                               UsedAssumedInformation)) {4206        LLVM_DEBUG(4207            dbgs()4208            << "[AAIsDead] Could not determine potential copies of store!\n");4209        return false;4210      }4211    }4212    LLVM_DEBUG(dbgs() << "[AAIsDead] Store has " << PotentialCopies.size()4213                      << " potential copies.\n");4214 4215    InformationCache &InfoCache = A.getInfoCache();4216    return llvm::all_of(PotentialCopies, [&](Value *V) {4217      if (A.isAssumedDead(IRPosition::value(*V), this, nullptr,4218                          UsedAssumedInformation))4219        return true;4220      if (auto *LI = dyn_cast<LoadInst>(V)) {4221        if (llvm::all_of(LI->uses(), [&](const Use &U) {4222              auto &UserI = cast<Instruction>(*U.getUser());4223              if (InfoCache.isOnlyUsedByAssume(UserI)) {4224                if (AssumeOnlyInst)4225                  AssumeOnlyInst->insert(&UserI);4226                return true;4227              }4228              return A.isAssumedDead(U, this, nullptr, UsedAssumedInformation);4229            })) {4230          return true;4231        }4232      }4233      LLVM_DEBUG(dbgs() << "[AAIsDead] Potential copy " << *V4234                        << " is assumed live!\n");4235      return false;4236    });4237  }4238 4239  /// See AbstractAttribute::getAsStr().4240  const std::string getAsStr(Attributor *A) const override {4241    Instruction *I = dyn_cast<Instruction>(&getAssociatedValue());4242    if (isa_and_nonnull<StoreInst>(I))4243      if (isValidState())4244        return "assumed-dead-store";4245    if (isa_and_nonnull<FenceInst>(I))4246      if (isValidState())4247        return "assumed-dead-fence";4248    return AAIsDeadValueImpl::getAsStr(A);4249  }4250 4251  /// See AbstractAttribute::updateImpl(...).4252  ChangeStatus updateImpl(Attributor &A) override {4253    Instruction *I = dyn_cast<Instruction>(&getAssociatedValue());4254    if (auto *SI = dyn_cast_or_null<StoreInst>(I)) {4255      if (!isDeadStore(A, *SI))4256        return indicatePessimisticFixpoint();4257    } else if (auto *FI = dyn_cast_or_null<FenceInst>(I)) {4258      if (!isDeadFence(A, *FI))4259        return indicatePessimisticFixpoint();4260    } else {4261      if (!isAssumedSideEffectFree(A, I))4262        return indicatePessimisticFixpoint();4263      if (!areAllUsesAssumedDead(A, getAssociatedValue()))4264        return indicatePessimisticFixpoint();4265    }4266    return ChangeStatus::UNCHANGED;4267  }4268 4269  bool isRemovableStore() const override {4270    return isAssumed(IS_REMOVABLE) && isa<StoreInst>(&getAssociatedValue());4271  }4272 4273  /// See AbstractAttribute::manifest(...).4274  ChangeStatus manifest(Attributor &A) override {4275    Value &V = getAssociatedValue();4276    if (auto *I = dyn_cast<Instruction>(&V)) {4277      // If we get here we basically know the users are all dead. We check if4278      // isAssumedSideEffectFree returns true here again because it might not be4279      // the case and only the users are dead but the instruction (=call) is4280      // still needed.4281      if (auto *SI = dyn_cast<StoreInst>(I)) {4282        SmallSetVector<Instruction *, 8> AssumeOnlyInst;4283        bool IsDead = isDeadStore(A, *SI, &AssumeOnlyInst);4284        (void)IsDead;4285        assert(IsDead && "Store was assumed to be dead!");4286        A.deleteAfterManifest(*I);4287        for (size_t i = 0; i < AssumeOnlyInst.size(); ++i) {4288          Instruction *AOI = AssumeOnlyInst[i];4289          for (auto *Usr : AOI->users())4290            AssumeOnlyInst.insert(cast<Instruction>(Usr));4291          A.deleteAfterManifest(*AOI);4292        }4293        return ChangeStatus::CHANGED;4294      }4295      if (auto *FI = dyn_cast<FenceInst>(I)) {4296        assert(isDeadFence(A, *FI));4297        A.deleteAfterManifest(*FI);4298        return ChangeStatus::CHANGED;4299      }4300      if (isAssumedSideEffectFree(A, I) && !isa<InvokeInst>(I)) {4301        A.deleteAfterManifest(*I);4302        return ChangeStatus::CHANGED;4303      }4304    }4305    return ChangeStatus::UNCHANGED;4306  }4307 4308  /// See AbstractAttribute::trackStatistics()4309  void trackStatistics() const override {4310    STATS_DECLTRACK_FLOATING_ATTR(IsDead)4311  }4312 4313private:4314  // The potential copies of a dead store, used for deletion during manifest.4315  SmallSetVector<Value *, 4> PotentialCopies;4316};4317 4318struct AAIsDeadArgument : public AAIsDeadFloating {4319  AAIsDeadArgument(const IRPosition &IRP, Attributor &A)4320      : AAIsDeadFloating(IRP, A) {}4321 4322  /// See AbstractAttribute::manifest(...).4323  ChangeStatus manifest(Attributor &A) override {4324    Argument &Arg = *getAssociatedArgument();4325    if (A.isValidFunctionSignatureRewrite(Arg, /* ReplacementTypes */ {}))4326      if (A.registerFunctionSignatureRewrite(4327              Arg, /* ReplacementTypes */ {},4328              Attributor::ArgumentReplacementInfo::CalleeRepairCBTy{},4329              Attributor::ArgumentReplacementInfo::ACSRepairCBTy{})) {4330        return ChangeStatus::CHANGED;4331      }4332    return ChangeStatus::UNCHANGED;4333  }4334 4335  /// See AbstractAttribute::trackStatistics()4336  void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(IsDead) }4337};4338 4339struct AAIsDeadCallSiteArgument : public AAIsDeadValueImpl {4340  AAIsDeadCallSiteArgument(const IRPosition &IRP, Attributor &A)4341      : AAIsDeadValueImpl(IRP, A) {}4342 4343  /// See AbstractAttribute::initialize(...).4344  void initialize(Attributor &A) override {4345    AAIsDeadValueImpl::initialize(A);4346    if (isa<UndefValue>(getAssociatedValue()))4347      indicatePessimisticFixpoint();4348  }4349 4350  /// See AbstractAttribute::updateImpl(...).4351  ChangeStatus updateImpl(Attributor &A) override {4352    // TODO: Once we have call site specific value information we can provide4353    //       call site specific liveness information and then it makes4354    //       sense to specialize attributes for call sites arguments instead of4355    //       redirecting requests to the callee argument.4356    Argument *Arg = getAssociatedArgument();4357    if (!Arg)4358      return indicatePessimisticFixpoint();4359    const IRPosition &ArgPos = IRPosition::argument(*Arg);4360    auto *ArgAA = A.getAAFor<AAIsDead>(*this, ArgPos, DepClassTy::REQUIRED);4361    if (!ArgAA)4362      return indicatePessimisticFixpoint();4363    return clampStateAndIndicateChange(getState(), ArgAA->getState());4364  }4365 4366  /// See AbstractAttribute::manifest(...).4367  ChangeStatus manifest(Attributor &A) override {4368    CallBase &CB = cast<CallBase>(getAnchorValue());4369    Use &U = CB.getArgOperandUse(getCallSiteArgNo());4370    assert(!isa<UndefValue>(U.get()) &&4371           "Expected undef values to be filtered out!");4372    UndefValue &UV = *UndefValue::get(U->getType());4373    if (A.changeUseAfterManifest(U, UV))4374      return ChangeStatus::CHANGED;4375    return ChangeStatus::UNCHANGED;4376  }4377 4378  /// See AbstractAttribute::trackStatistics()4379  void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(IsDead) }4380};4381 4382struct AAIsDeadCallSiteReturned : public AAIsDeadFloating {4383  AAIsDeadCallSiteReturned(const IRPosition &IRP, Attributor &A)4384      : AAIsDeadFloating(IRP, A) {}4385 4386  /// See AAIsDead::isAssumedDead().4387  bool isAssumedDead() const override {4388    return AAIsDeadFloating::isAssumedDead() && IsAssumedSideEffectFree;4389  }4390 4391  /// See AbstractAttribute::initialize(...).4392  void initialize(Attributor &A) override {4393    AAIsDeadFloating::initialize(A);4394    if (isa<UndefValue>(getAssociatedValue())) {4395      indicatePessimisticFixpoint();4396      return;4397    }4398 4399    // We track this separately as a secondary state.4400    IsAssumedSideEffectFree = isAssumedSideEffectFree(A, getCtxI());4401  }4402 4403  /// See AbstractAttribute::updateImpl(...).4404  ChangeStatus updateImpl(Attributor &A) override {4405    ChangeStatus Changed = ChangeStatus::UNCHANGED;4406    if (IsAssumedSideEffectFree && !isAssumedSideEffectFree(A, getCtxI())) {4407      IsAssumedSideEffectFree = false;4408      Changed = ChangeStatus::CHANGED;4409    }4410    if (!areAllUsesAssumedDead(A, getAssociatedValue()))4411      return indicatePessimisticFixpoint();4412    return Changed;4413  }4414 4415  /// See AbstractAttribute::trackStatistics()4416  void trackStatistics() const override {4417    if (IsAssumedSideEffectFree)4418      STATS_DECLTRACK_CSRET_ATTR(IsDead)4419    else4420      STATS_DECLTRACK_CSRET_ATTR(UnusedResult)4421  }4422 4423  /// See AbstractAttribute::getAsStr().4424  const std::string getAsStr(Attributor *A) const override {4425    return isAssumedDead()4426               ? "assumed-dead"4427               : (getAssumed() ? "assumed-dead-users" : "assumed-live");4428  }4429 4430private:4431  bool IsAssumedSideEffectFree = true;4432};4433 4434struct AAIsDeadReturned : public AAIsDeadValueImpl {4435  AAIsDeadReturned(const IRPosition &IRP, Attributor &A)4436      : AAIsDeadValueImpl(IRP, A) {}4437 4438  /// See AbstractAttribute::updateImpl(...).4439  ChangeStatus updateImpl(Attributor &A) override {4440 4441    bool UsedAssumedInformation = false;4442    A.checkForAllInstructions([](Instruction &) { return true; }, *this,4443                              {Instruction::Ret}, UsedAssumedInformation);4444 4445    auto PredForCallSite = [&](AbstractCallSite ACS) {4446      if (ACS.isCallbackCall() || !ACS.getInstruction())4447        return false;4448      return areAllUsesAssumedDead(A, *ACS.getInstruction());4449    };4450 4451    if (!A.checkForAllCallSites(PredForCallSite, *this, true,4452                                UsedAssumedInformation))4453      return indicatePessimisticFixpoint();4454 4455    return ChangeStatus::UNCHANGED;4456  }4457 4458  /// See AbstractAttribute::manifest(...).4459  ChangeStatus manifest(Attributor &A) override {4460    // TODO: Rewrite the signature to return void?4461    bool AnyChange = false;4462    UndefValue &UV = *UndefValue::get(getAssociatedFunction()->getReturnType());4463    auto RetInstPred = [&](Instruction &I) {4464      ReturnInst &RI = cast<ReturnInst>(I);4465      if (!isa<UndefValue>(RI.getReturnValue()))4466        AnyChange |= A.changeUseAfterManifest(RI.getOperandUse(0), UV);4467      return true;4468    };4469    bool UsedAssumedInformation = false;4470    A.checkForAllInstructions(RetInstPred, *this, {Instruction::Ret},4471                              UsedAssumedInformation);4472    return AnyChange ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;4473  }4474 4475  /// See AbstractAttribute::trackStatistics()4476  void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(IsDead) }4477};4478 4479struct AAIsDeadFunction : public AAIsDead {4480  AAIsDeadFunction(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}4481 4482  /// See AbstractAttribute::initialize(...).4483  void initialize(Attributor &A) override {4484    Function *F = getAnchorScope();4485    assert(F && "Did expect an anchor function");4486    if (!isAssumedDeadInternalFunction(A)) {4487      ToBeExploredFrom.insert(&F->getEntryBlock().front());4488      assumeLive(A, F->getEntryBlock());4489    }4490  }4491 4492  bool isAssumedDeadInternalFunction(Attributor &A) {4493    if (!getAnchorScope()->hasLocalLinkage())4494      return false;4495    bool UsedAssumedInformation = false;4496    return A.checkForAllCallSites([](AbstractCallSite) { return false; }, *this,4497                                  true, UsedAssumedInformation);4498  }4499 4500  /// See AbstractAttribute::getAsStr().4501  const std::string getAsStr(Attributor *A) const override {4502    return "Live[#BB " + std::to_string(AssumedLiveBlocks.size()) + "/" +4503           std::to_string(getAnchorScope()->size()) + "][#TBEP " +4504           std::to_string(ToBeExploredFrom.size()) + "][#KDE " +4505           std::to_string(KnownDeadEnds.size()) + "]";4506  }4507 4508  /// See AbstractAttribute::manifest(...).4509  ChangeStatus manifest(Attributor &A) override {4510    assert(getState().isValidState() &&4511           "Attempted to manifest an invalid state!");4512 4513    ChangeStatus HasChanged = ChangeStatus::UNCHANGED;4514    Function &F = *getAnchorScope();4515 4516    if (AssumedLiveBlocks.empty()) {4517      A.deleteAfterManifest(F);4518      return ChangeStatus::CHANGED;4519    }4520 4521    // Flag to determine if we can change an invoke to a call assuming the4522    // callee is nounwind. This is not possible if the personality of the4523    // function allows to catch asynchronous exceptions.4524    bool Invoke2CallAllowed = !mayCatchAsynchronousExceptions(F);4525 4526    KnownDeadEnds.set_union(ToBeExploredFrom);4527    for (const Instruction *DeadEndI : KnownDeadEnds) {4528      auto *CB = dyn_cast<CallBase>(DeadEndI);4529      if (!CB)4530        continue;4531      bool IsKnownNoReturn;4532      bool MayReturn = !AA::hasAssumedIRAttr<Attribute::NoReturn>(4533          A, this, IRPosition::callsite_function(*CB), DepClassTy::OPTIONAL,4534          IsKnownNoReturn);4535      if (MayReturn && (!Invoke2CallAllowed || !isa<InvokeInst>(CB)))4536        continue;4537 4538      if (auto *II = dyn_cast<InvokeInst>(DeadEndI))4539        A.registerInvokeWithDeadSuccessor(const_cast<InvokeInst &>(*II));4540      else4541        A.changeToUnreachableAfterManifest(4542            const_cast<Instruction *>(DeadEndI->getNextNode()));4543      HasChanged = ChangeStatus::CHANGED;4544    }4545 4546    STATS_DECL(AAIsDead, BasicBlock, "Number of dead basic blocks deleted.");4547    for (BasicBlock &BB : F)4548      if (!AssumedLiveBlocks.count(&BB)) {4549        A.deleteAfterManifest(BB);4550        ++BUILD_STAT_NAME(AAIsDead, BasicBlock);4551        HasChanged = ChangeStatus::CHANGED;4552      }4553 4554    return HasChanged;4555  }4556 4557  /// See AbstractAttribute::updateImpl(...).4558  ChangeStatus updateImpl(Attributor &A) override;4559 4560  bool isEdgeDead(const BasicBlock *From, const BasicBlock *To) const override {4561    assert(From->getParent() == getAnchorScope() &&4562           To->getParent() == getAnchorScope() &&4563           "Used AAIsDead of the wrong function");4564    return isValidState() && !AssumedLiveEdges.count(std::make_pair(From, To));4565  }4566 4567  /// See AbstractAttribute::trackStatistics()4568  void trackStatistics() const override {}4569 4570  /// Returns true if the function is assumed dead.4571  bool isAssumedDead() const override { return false; }4572 4573  /// See AAIsDead::isKnownDead().4574  bool isKnownDead() const override { return false; }4575 4576  /// See AAIsDead::isAssumedDead(BasicBlock *).4577  bool isAssumedDead(const BasicBlock *BB) const override {4578    assert(BB->getParent() == getAnchorScope() &&4579           "BB must be in the same anchor scope function.");4580 4581    if (!getAssumed())4582      return false;4583    return !AssumedLiveBlocks.count(BB);4584  }4585 4586  /// See AAIsDead::isKnownDead(BasicBlock *).4587  bool isKnownDead(const BasicBlock *BB) const override {4588    return getKnown() && isAssumedDead(BB);4589  }4590 4591  /// See AAIsDead::isAssumed(Instruction *I).4592  bool isAssumedDead(const Instruction *I) const override {4593    assert(I->getParent()->getParent() == getAnchorScope() &&4594           "Instruction must be in the same anchor scope function.");4595 4596    if (!getAssumed())4597      return false;4598 4599    // If it is not in AssumedLiveBlocks then it for sure dead.4600    // Otherwise, it can still be after noreturn call in a live block.4601    if (!AssumedLiveBlocks.count(I->getParent()))4602      return true;4603 4604    // If it is not after a liveness barrier it is live.4605    const Instruction *PrevI = I->getPrevNode();4606    while (PrevI) {4607      if (KnownDeadEnds.count(PrevI) || ToBeExploredFrom.count(PrevI))4608        return true;4609      PrevI = PrevI->getPrevNode();4610    }4611    return false;4612  }4613 4614  /// See AAIsDead::isKnownDead(Instruction *I).4615  bool isKnownDead(const Instruction *I) const override {4616    return getKnown() && isAssumedDead(I);4617  }4618 4619  /// Assume \p BB is (partially) live now and indicate to the Attributor \p A4620  /// that internal function called from \p BB should now be looked at.4621  bool assumeLive(Attributor &A, const BasicBlock &BB) {4622    if (!AssumedLiveBlocks.insert(&BB).second)4623      return false;4624 4625    // We assume that all of BB is (probably) live now and if there are calls to4626    // internal functions we will assume that those are now live as well. This4627    // is a performance optimization for blocks with calls to a lot of internal4628    // functions. It can however cause dead functions to be treated as live.4629    for (const Instruction &I : BB)4630      if (const auto *CB = dyn_cast<CallBase>(&I))4631        if (auto *F = dyn_cast_if_present<Function>(CB->getCalledOperand()))4632          if (F->hasLocalLinkage())4633            A.markLiveInternalFunction(*F);4634    return true;4635  }4636 4637  /// Collection of instructions that need to be explored again, e.g., we4638  /// did assume they do not transfer control to (one of their) successors.4639  SmallSetVector<const Instruction *, 8> ToBeExploredFrom;4640 4641  /// Collection of instructions that are known to not transfer control.4642  SmallSetVector<const Instruction *, 8> KnownDeadEnds;4643 4644  /// Collection of all assumed live edges4645  DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> AssumedLiveEdges;4646 4647  /// Collection of all assumed live BasicBlocks.4648  DenseSet<const BasicBlock *> AssumedLiveBlocks;4649};4650 4651static bool4652identifyAliveSuccessors(Attributor &A, const CallBase &CB,4653                        AbstractAttribute &AA,4654                        SmallVectorImpl<const Instruction *> &AliveSuccessors) {4655  const IRPosition &IPos = IRPosition::callsite_function(CB);4656 4657  bool IsKnownNoReturn;4658  if (AA::hasAssumedIRAttr<Attribute::NoReturn>(4659          A, &AA, IPos, DepClassTy::OPTIONAL, IsKnownNoReturn))4660    return !IsKnownNoReturn;4661  if (CB.isTerminator())4662    AliveSuccessors.push_back(&CB.getSuccessor(0)->front());4663  else4664    AliveSuccessors.push_back(CB.getNextNode());4665  return false;4666}4667 4668static bool4669identifyAliveSuccessors(Attributor &A, const InvokeInst &II,4670                        AbstractAttribute &AA,4671                        SmallVectorImpl<const Instruction *> &AliveSuccessors) {4672  bool UsedAssumedInformation =4673      identifyAliveSuccessors(A, cast<CallBase>(II), AA, AliveSuccessors);4674 4675  // First, determine if we can change an invoke to a call assuming the4676  // callee is nounwind. This is not possible if the personality of the4677  // function allows to catch asynchronous exceptions.4678  if (AAIsDeadFunction::mayCatchAsynchronousExceptions(*II.getFunction())) {4679    AliveSuccessors.push_back(&II.getUnwindDest()->front());4680  } else {4681    const IRPosition &IPos = IRPosition::callsite_function(II);4682 4683    bool IsKnownNoUnwind;4684    if (AA::hasAssumedIRAttr<Attribute::NoUnwind>(4685            A, &AA, IPos, DepClassTy::OPTIONAL, IsKnownNoUnwind)) {4686      UsedAssumedInformation |= !IsKnownNoUnwind;4687    } else {4688      AliveSuccessors.push_back(&II.getUnwindDest()->front());4689    }4690  }4691  return UsedAssumedInformation;4692}4693 4694static bool4695identifyAliveSuccessors(Attributor &A, const BranchInst &BI,4696                        AbstractAttribute &AA,4697                        SmallVectorImpl<const Instruction *> &AliveSuccessors) {4698  bool UsedAssumedInformation = false;4699  if (BI.getNumSuccessors() == 1) {4700    AliveSuccessors.push_back(&BI.getSuccessor(0)->front());4701  } else {4702    std::optional<Constant *> C =4703        A.getAssumedConstant(*BI.getCondition(), AA, UsedAssumedInformation);4704    if (!C || isa_and_nonnull<UndefValue>(*C)) {4705      // No value yet, assume both edges are dead.4706    } else if (isa_and_nonnull<ConstantInt>(*C)) {4707      const BasicBlock *SuccBB =4708          BI.getSuccessor(1 - cast<ConstantInt>(*C)->getValue().getZExtValue());4709      AliveSuccessors.push_back(&SuccBB->front());4710    } else {4711      AliveSuccessors.push_back(&BI.getSuccessor(0)->front());4712      AliveSuccessors.push_back(&BI.getSuccessor(1)->front());4713      UsedAssumedInformation = false;4714    }4715  }4716  return UsedAssumedInformation;4717}4718 4719static bool4720identifyAliveSuccessors(Attributor &A, const SwitchInst &SI,4721                        AbstractAttribute &AA,4722                        SmallVectorImpl<const Instruction *> &AliveSuccessors) {4723  bool UsedAssumedInformation = false;4724  SmallVector<AA::ValueAndContext> Values;4725  if (!A.getAssumedSimplifiedValues(IRPosition::value(*SI.getCondition()), &AA,4726                                    Values, AA::AnyScope,4727                                    UsedAssumedInformation)) {4728    // Something went wrong, assume all successors are live.4729    for (const BasicBlock *SuccBB : successors(SI.getParent()))4730      AliveSuccessors.push_back(&SuccBB->front());4731    return false;4732  }4733 4734  if (Values.empty() ||4735      (Values.size() == 1 &&4736       isa_and_nonnull<UndefValue>(Values.front().getValue()))) {4737    // No valid value yet, assume all edges are dead.4738    return UsedAssumedInformation;4739  }4740 4741  Type &Ty = *SI.getCondition()->getType();4742  SmallPtrSet<ConstantInt *, 8> Constants;4743  auto CheckForConstantInt = [&](Value *V) {4744    if (auto *CI = dyn_cast_if_present<ConstantInt>(AA::getWithType(*V, Ty))) {4745      Constants.insert(CI);4746      return true;4747    }4748    return false;4749  };4750 4751  if (!all_of(Values, [&](AA::ValueAndContext &VAC) {4752        return CheckForConstantInt(VAC.getValue());4753      })) {4754    for (const BasicBlock *SuccBB : successors(SI.getParent()))4755      AliveSuccessors.push_back(&SuccBB->front());4756    return UsedAssumedInformation;4757  }4758 4759  unsigned MatchedCases = 0;4760  for (const auto &CaseIt : SI.cases()) {4761    if (Constants.count(CaseIt.getCaseValue())) {4762      ++MatchedCases;4763      AliveSuccessors.push_back(&CaseIt.getCaseSuccessor()->front());4764    }4765  }4766 4767  // If all potential values have been matched, we will not visit the default4768  // case.4769  if (MatchedCases < Constants.size())4770    AliveSuccessors.push_back(&SI.getDefaultDest()->front());4771  return UsedAssumedInformation;4772}4773 4774ChangeStatus AAIsDeadFunction::updateImpl(Attributor &A) {4775  ChangeStatus Change = ChangeStatus::UNCHANGED;4776 4777  if (AssumedLiveBlocks.empty()) {4778    if (isAssumedDeadInternalFunction(A))4779      return ChangeStatus::UNCHANGED;4780 4781    Function *F = getAnchorScope();4782    ToBeExploredFrom.insert(&F->getEntryBlock().front());4783    assumeLive(A, F->getEntryBlock());4784    Change = ChangeStatus::CHANGED;4785  }4786 4787  LLVM_DEBUG(dbgs() << "[AAIsDead] Live [" << AssumedLiveBlocks.size() << "/"4788                    << getAnchorScope()->size() << "] BBs and "4789                    << ToBeExploredFrom.size() << " exploration points and "4790                    << KnownDeadEnds.size() << " known dead ends\n");4791 4792  // Copy and clear the list of instructions we need to explore from. It is4793  // refilled with instructions the next update has to look at.4794  SmallVector<const Instruction *, 8> Worklist(ToBeExploredFrom.begin(),4795                                               ToBeExploredFrom.end());4796  decltype(ToBeExploredFrom) NewToBeExploredFrom;4797 4798  SmallVector<const Instruction *, 8> AliveSuccessors;4799  while (!Worklist.empty()) {4800    const Instruction *I = Worklist.pop_back_val();4801    LLVM_DEBUG(dbgs() << "[AAIsDead] Exploration inst: " << *I << "\n");4802 4803    // Fast forward for uninteresting instructions. We could look for UB here4804    // though.4805    while (!I->isTerminator() && !isa<CallBase>(I))4806      I = I->getNextNode();4807 4808    AliveSuccessors.clear();4809 4810    bool UsedAssumedInformation = false;4811    switch (I->getOpcode()) {4812    // TODO: look for (assumed) UB to backwards propagate "deadness".4813    default:4814      assert(I->isTerminator() &&4815             "Expected non-terminators to be handled already!");4816      for (const BasicBlock *SuccBB : successors(I->getParent()))4817        AliveSuccessors.push_back(&SuccBB->front());4818      break;4819    case Instruction::Call:4820      UsedAssumedInformation = identifyAliveSuccessors(A, cast<CallInst>(*I),4821                                                       *this, AliveSuccessors);4822      break;4823    case Instruction::Invoke:4824      UsedAssumedInformation = identifyAliveSuccessors(A, cast<InvokeInst>(*I),4825                                                       *this, AliveSuccessors);4826      break;4827    case Instruction::Br:4828      UsedAssumedInformation = identifyAliveSuccessors(A, cast<BranchInst>(*I),4829                                                       *this, AliveSuccessors);4830      break;4831    case Instruction::Switch:4832      UsedAssumedInformation = identifyAliveSuccessors(A, cast<SwitchInst>(*I),4833                                                       *this, AliveSuccessors);4834      break;4835    }4836 4837    if (UsedAssumedInformation) {4838      NewToBeExploredFrom.insert(I);4839    } else if (AliveSuccessors.empty() ||4840               (I->isTerminator() &&4841                AliveSuccessors.size() < I->getNumSuccessors())) {4842      if (KnownDeadEnds.insert(I))4843        Change = ChangeStatus::CHANGED;4844    }4845 4846    LLVM_DEBUG(dbgs() << "[AAIsDead] #AliveSuccessors: "4847                      << AliveSuccessors.size() << " UsedAssumedInformation: "4848                      << UsedAssumedInformation << "\n");4849 4850    for (const Instruction *AliveSuccessor : AliveSuccessors) {4851      if (!I->isTerminator()) {4852        assert(AliveSuccessors.size() == 1 &&4853               "Non-terminator expected to have a single successor!");4854        Worklist.push_back(AliveSuccessor);4855      } else {4856        // record the assumed live edge4857        auto Edge = std::make_pair(I->getParent(), AliveSuccessor->getParent());4858        if (AssumedLiveEdges.insert(Edge).second)4859          Change = ChangeStatus::CHANGED;4860        if (assumeLive(A, *AliveSuccessor->getParent()))4861          Worklist.push_back(AliveSuccessor);4862      }4863    }4864  }4865 4866  // Check if the content of ToBeExploredFrom changed, ignore the order.4867  if (NewToBeExploredFrom.size() != ToBeExploredFrom.size() ||4868      llvm::any_of(NewToBeExploredFrom, [&](const Instruction *I) {4869        return !ToBeExploredFrom.count(I);4870      })) {4871    Change = ChangeStatus::CHANGED;4872    ToBeExploredFrom = std::move(NewToBeExploredFrom);4873  }4874 4875  // If we know everything is live there is no need to query for liveness.4876  // Instead, indicating a pessimistic fixpoint will cause the state to be4877  // "invalid" and all queries to be answered conservatively without lookups.4878  // To be in this state we have to (1) finished the exploration and (3) not4879  // discovered any non-trivial dead end and (2) not ruled unreachable code4880  // dead.4881  if (ToBeExploredFrom.empty() &&4882      getAnchorScope()->size() == AssumedLiveBlocks.size() &&4883      llvm::all_of(KnownDeadEnds, [](const Instruction *DeadEndI) {4884        return DeadEndI->isTerminator() && DeadEndI->getNumSuccessors() == 0;4885      }))4886    return indicatePessimisticFixpoint();4887  return Change;4888}4889 4890/// Liveness information for a call sites.4891struct AAIsDeadCallSite final : AAIsDeadFunction {4892  AAIsDeadCallSite(const IRPosition &IRP, Attributor &A)4893      : AAIsDeadFunction(IRP, A) {}4894 4895  /// See AbstractAttribute::initialize(...).4896  void initialize(Attributor &A) override {4897    // TODO: Once we have call site specific value information we can provide4898    //       call site specific liveness information and then it makes4899    //       sense to specialize attributes for call sites instead of4900    //       redirecting requests to the callee.4901    llvm_unreachable("Abstract attributes for liveness are not "4902                     "supported for call sites yet!");4903  }4904 4905  /// See AbstractAttribute::updateImpl(...).4906  ChangeStatus updateImpl(Attributor &A) override {4907    return indicatePessimisticFixpoint();4908  }4909 4910  /// See AbstractAttribute::trackStatistics()4911  void trackStatistics() const override {}4912};4913} // namespace4914 4915/// -------------------- Dereferenceable Argument Attribute --------------------4916 4917namespace {4918struct AADereferenceableImpl : AADereferenceable {4919  AADereferenceableImpl(const IRPosition &IRP, Attributor &A)4920      : AADereferenceable(IRP, A) {}4921  using StateType = DerefState;4922 4923  /// See AbstractAttribute::initialize(...).4924  void initialize(Attributor &A) override {4925    Value &V = *getAssociatedValue().stripPointerCasts();4926    SmallVector<Attribute, 4> Attrs;4927    A.getAttrs(getIRPosition(),4928               {Attribute::Dereferenceable, Attribute::DereferenceableOrNull},4929               Attrs, /* IgnoreSubsumingPositions */ false);4930    for (const Attribute &Attr : Attrs)4931      takeKnownDerefBytesMaximum(Attr.getValueAsInt());4932 4933    // Ensure we initialize the non-null AA (if necessary).4934    bool IsKnownNonNull;4935    AA::hasAssumedIRAttr<Attribute::NonNull>(4936        A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNonNull);4937 4938    bool CanBeNull, CanBeFreed;4939    takeKnownDerefBytesMaximum(V.getPointerDereferenceableBytes(4940        A.getDataLayout(), CanBeNull, CanBeFreed));4941 4942    if (Instruction *CtxI = getCtxI())4943      followUsesInMBEC(*this, A, getState(), *CtxI);4944  }4945 4946  /// See AbstractAttribute::getState()4947  /// {4948  StateType &getState() override { return *this; }4949  const StateType &getState() const override { return *this; }4950  /// }4951 4952  /// Helper function for collecting accessed bytes in must-be-executed-context4953  void addAccessedBytesForUse(Attributor &A, const Use *U, const Instruction *I,4954                              DerefState &State) {4955    const Value *UseV = U->get();4956    if (!UseV->getType()->isPointerTy())4957      return;4958 4959    std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(I);4960    if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() || I->isVolatile())4961      return;4962 4963    int64_t Offset;4964    const Value *Base = GetPointerBaseWithConstantOffset(4965        Loc->Ptr, Offset, A.getDataLayout(), /*AllowNonInbounds*/ true);4966    if (Base && Base == &getAssociatedValue())4967      State.addAccessedBytes(Offset, Loc->Size.getValue());4968  }4969 4970  /// See followUsesInMBEC4971  bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,4972                       AADereferenceable::StateType &State) {4973    bool IsNonNull = false;4974    bool TrackUse = false;4975    int64_t DerefBytes = getKnownNonNullAndDerefBytesForUse(4976        A, *this, getAssociatedValue(), U, I, IsNonNull, TrackUse);4977    LLVM_DEBUG(dbgs() << "[AADereferenceable] Deref bytes: " << DerefBytes4978                      << " for instruction " << *I << "\n");4979 4980    addAccessedBytesForUse(A, U, I, State);4981    State.takeKnownDerefBytesMaximum(DerefBytes);4982    return TrackUse;4983  }4984 4985  /// See AbstractAttribute::manifest(...).4986  ChangeStatus manifest(Attributor &A) override {4987    ChangeStatus Change = AADereferenceable::manifest(A);4988    bool IsKnownNonNull;4989    bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(4990        A, this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull);4991    if (IsAssumedNonNull &&4992        A.hasAttr(getIRPosition(), Attribute::DereferenceableOrNull)) {4993      A.removeAttrs(getIRPosition(), {Attribute::DereferenceableOrNull});4994      return ChangeStatus::CHANGED;4995    }4996    return Change;4997  }4998 4999  void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,5000                            SmallVectorImpl<Attribute> &Attrs) const override {5001    // TODO: Add *_globally support5002    bool IsKnownNonNull;5003    bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(5004        A, this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull);5005    if (IsAssumedNonNull)5006      Attrs.emplace_back(Attribute::getWithDereferenceableBytes(5007          Ctx, getAssumedDereferenceableBytes()));5008    else5009      Attrs.emplace_back(Attribute::getWithDereferenceableOrNullBytes(5010          Ctx, getAssumedDereferenceableBytes()));5011  }5012 5013  /// See AbstractAttribute::getAsStr().5014  const std::string getAsStr(Attributor *A) const override {5015    if (!getAssumedDereferenceableBytes())5016      return "unknown-dereferenceable";5017    bool IsKnownNonNull;5018    bool IsAssumedNonNull = false;5019    if (A)5020      IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(5021          *A, this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull);5022    return std::string("dereferenceable") +5023           (IsAssumedNonNull ? "" : "_or_null") +5024           (isAssumedGlobal() ? "_globally" : "") + "<" +5025           std::to_string(getKnownDereferenceableBytes()) + "-" +5026           std::to_string(getAssumedDereferenceableBytes()) + ">" +5027           (!A ? " [non-null is unknown]" : "");5028  }5029};5030 5031/// Dereferenceable attribute for a floating value.5032struct AADereferenceableFloating : AADereferenceableImpl {5033  AADereferenceableFloating(const IRPosition &IRP, Attributor &A)5034      : AADereferenceableImpl(IRP, A) {}5035 5036  /// See AbstractAttribute::updateImpl(...).5037  ChangeStatus updateImpl(Attributor &A) override {5038    bool Stripped;5039    bool UsedAssumedInformation = false;5040    SmallVector<AA::ValueAndContext> Values;5041    if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,5042                                      AA::AnyScope, UsedAssumedInformation)) {5043      Values.push_back({getAssociatedValue(), getCtxI()});5044      Stripped = false;5045    } else {5046      Stripped = Values.size() != 1 ||5047                 Values.front().getValue() != &getAssociatedValue();5048    }5049 5050    const DataLayout &DL = A.getDataLayout();5051    DerefState T;5052 5053    auto VisitValueCB = [&](const Value &V) -> bool {5054      unsigned IdxWidth =5055          DL.getIndexSizeInBits(V.getType()->getPointerAddressSpace());5056      APInt Offset(IdxWidth, 0);5057      const Value *Base = stripAndAccumulateOffsets(5058          A, *this, &V, DL, Offset, /* GetMinOffset */ false,5059          /* AllowNonInbounds */ true);5060 5061      const auto *AA = A.getAAFor<AADereferenceable>(5062          *this, IRPosition::value(*Base), DepClassTy::REQUIRED);5063      int64_t DerefBytes = 0;5064      if (!AA || (!Stripped && this == AA)) {5065        // Use IR information if we did not strip anything.5066        // TODO: track globally.5067        bool CanBeNull, CanBeFreed;5068        DerefBytes =5069            Base->getPointerDereferenceableBytes(DL, CanBeNull, CanBeFreed);5070        T.GlobalState.indicatePessimisticFixpoint();5071      } else {5072        const DerefState &DS = AA->getState();5073        DerefBytes = DS.DerefBytesState.getAssumed();5074        T.GlobalState &= DS.GlobalState;5075      }5076 5077      // For now we do not try to "increase" dereferenceability due to negative5078      // indices as we first have to come up with code to deal with loops and5079      // for overflows of the dereferenceable bytes.5080      int64_t OffsetSExt = Offset.getSExtValue();5081      if (OffsetSExt < 0)5082        OffsetSExt = 0;5083 5084      T.takeAssumedDerefBytesMinimum(5085          std::max(int64_t(0), DerefBytes - OffsetSExt));5086 5087      if (this == AA) {5088        if (!Stripped) {5089          // If nothing was stripped IR information is all we got.5090          T.takeKnownDerefBytesMaximum(5091              std::max(int64_t(0), DerefBytes - OffsetSExt));5092          T.indicatePessimisticFixpoint();5093        } else if (OffsetSExt > 0) {5094          // If something was stripped but there is circular reasoning we look5095          // for the offset. If it is positive we basically decrease the5096          // dereferenceable bytes in a circular loop now, which will simply5097          // drive them down to the known value in a very slow way which we5098          // can accelerate.5099          T.indicatePessimisticFixpoint();5100        }5101      }5102 5103      return T.isValidState();5104    };5105 5106    for (const auto &VAC : Values)5107      if (!VisitValueCB(*VAC.getValue()))5108        return indicatePessimisticFixpoint();5109 5110    return clampStateAndIndicateChange(getState(), T);5111  }5112 5113  /// See AbstractAttribute::trackStatistics()5114  void trackStatistics() const override {5115    STATS_DECLTRACK_FLOATING_ATTR(dereferenceable)5116  }5117};5118 5119/// Dereferenceable attribute for a return value.5120struct AADereferenceableReturned final5121    : AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl> {5122  using Base =5123      AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl>;5124  AADereferenceableReturned(const IRPosition &IRP, Attributor &A)5125      : Base(IRP, A) {}5126 5127  /// See AbstractAttribute::trackStatistics()5128  void trackStatistics() const override {5129    STATS_DECLTRACK_FNRET_ATTR(dereferenceable)5130  }5131};5132 5133/// Dereferenceable attribute for an argument5134struct AADereferenceableArgument final5135    : AAArgumentFromCallSiteArguments<AADereferenceable,5136                                      AADereferenceableImpl> {5137  using Base =5138      AAArgumentFromCallSiteArguments<AADereferenceable, AADereferenceableImpl>;5139  AADereferenceableArgument(const IRPosition &IRP, Attributor &A)5140      : Base(IRP, A) {}5141 5142  /// See AbstractAttribute::trackStatistics()5143  void trackStatistics() const override {5144    STATS_DECLTRACK_ARG_ATTR(dereferenceable)5145  }5146};5147 5148/// Dereferenceable attribute for a call site argument.5149struct AADereferenceableCallSiteArgument final : AADereferenceableFloating {5150  AADereferenceableCallSiteArgument(const IRPosition &IRP, Attributor &A)5151      : AADereferenceableFloating(IRP, A) {}5152 5153  /// See AbstractAttribute::trackStatistics()5154  void trackStatistics() const override {5155    STATS_DECLTRACK_CSARG_ATTR(dereferenceable)5156  }5157};5158 5159/// Dereferenceable attribute deduction for a call site return value.5160struct AADereferenceableCallSiteReturned final5161    : AACalleeToCallSite<AADereferenceable, AADereferenceableImpl> {5162  using Base = AACalleeToCallSite<AADereferenceable, AADereferenceableImpl>;5163  AADereferenceableCallSiteReturned(const IRPosition &IRP, Attributor &A)5164      : Base(IRP, A) {}5165 5166  /// See AbstractAttribute::trackStatistics()5167  void trackStatistics() const override {5168    STATS_DECLTRACK_CS_ATTR(dereferenceable);5169  }5170};5171} // namespace5172 5173// ------------------------ Align Argument Attribute ------------------------5174 5175namespace {5176 5177static unsigned getKnownAlignForUse(Attributor &A, AAAlign &QueryingAA,5178                                    Value &AssociatedValue, const Use *U,5179                                    const Instruction *I, bool &TrackUse) {5180  // We need to follow common pointer manipulation uses to the accesses they5181  // feed into.5182  if (isa<CastInst>(I)) {5183    // Follow all but ptr2int casts.5184    TrackUse = !isa<PtrToIntInst>(I);5185    return 0;5186  }5187  if (auto *GEP = dyn_cast<GetElementPtrInst>(I)) {5188    if (GEP->hasAllConstantIndices())5189      TrackUse = true;5190    return 0;5191  }5192  if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I))5193    switch (II->getIntrinsicID()) {5194    case Intrinsic::ptrmask: {5195      // Is it appropriate to pull attribute in initialization?5196      const auto *ConstVals = A.getAAFor<AAPotentialConstantValues>(5197          QueryingAA, IRPosition::value(*II->getOperand(1)), DepClassTy::NONE);5198      const auto *AlignAA = A.getAAFor<AAAlign>(5199          QueryingAA, IRPosition::value(*II), DepClassTy::NONE);5200      if (ConstVals && ConstVals->isValidState() && ConstVals->isAtFixpoint()) {5201        unsigned ShiftValue = std::min(ConstVals->getAssumedMinTrailingZeros(),5202                                       Value::MaxAlignmentExponent);5203        Align ConstAlign(UINT64_C(1) << ShiftValue);5204        if (ConstAlign >= AlignAA->getKnownAlign())5205          return Align(1).value();5206      }5207      if (AlignAA)5208        return AlignAA->getKnownAlign().value();5209      break;5210    }5211    default:5212      break;5213    }5214 5215  MaybeAlign MA;5216  if (const auto *CB = dyn_cast<CallBase>(I)) {5217    if (CB->isBundleOperand(U) || CB->isCallee(U))5218      return 0;5219 5220    unsigned ArgNo = CB->getArgOperandNo(U);5221    IRPosition IRP = IRPosition::callsite_argument(*CB, ArgNo);5222    // As long as we only use known information there is no need to track5223    // dependences here.5224    auto *AlignAA = A.getAAFor<AAAlign>(QueryingAA, IRP, DepClassTy::NONE);5225    if (AlignAA)5226      MA = MaybeAlign(AlignAA->getKnownAlign());5227  }5228 5229  const DataLayout &DL = A.getDataLayout();5230  const Value *UseV = U->get();5231  if (auto *SI = dyn_cast<StoreInst>(I)) {5232    if (SI->getPointerOperand() == UseV)5233      MA = SI->getAlign();5234  } else if (auto *LI = dyn_cast<LoadInst>(I)) {5235    if (LI->getPointerOperand() == UseV)5236      MA = LI->getAlign();5237  } else if (auto *AI = dyn_cast<AtomicRMWInst>(I)) {5238    if (AI->getPointerOperand() == UseV)5239      MA = AI->getAlign();5240  } else if (auto *AI = dyn_cast<AtomicCmpXchgInst>(I)) {5241    if (AI->getPointerOperand() == UseV)5242      MA = AI->getAlign();5243  }5244 5245  if (!MA || *MA <= QueryingAA.getKnownAlign())5246    return 0;5247 5248  unsigned Alignment = MA->value();5249  int64_t Offset;5250 5251  if (const Value *Base = GetPointerBaseWithConstantOffset(UseV, Offset, DL)) {5252    if (Base == &AssociatedValue) {5253      // BasePointerAddr + Offset = Alignment * Q for some integer Q.5254      // So we can say that the maximum power of two which is a divisor of5255      // gcd(Offset, Alignment) is an alignment.5256 5257      uint32_t gcd = std::gcd(uint32_t(abs((int32_t)Offset)), Alignment);5258      Alignment = llvm::bit_floor(gcd);5259    }5260  }5261 5262  return Alignment;5263}5264 5265struct AAAlignImpl : AAAlign {5266  AAAlignImpl(const IRPosition &IRP, Attributor &A) : AAAlign(IRP, A) {}5267 5268  /// See AbstractAttribute::initialize(...).5269  void initialize(Attributor &A) override {5270    SmallVector<Attribute, 4> Attrs;5271    A.getAttrs(getIRPosition(), {Attribute::Alignment}, Attrs);5272    for (const Attribute &Attr : Attrs)5273      takeKnownMaximum(Attr.getValueAsInt());5274 5275    Value &V = *getAssociatedValue().stripPointerCasts();5276    takeKnownMaximum(V.getPointerAlignment(A.getDataLayout()).value());5277 5278    if (Instruction *CtxI = getCtxI())5279      followUsesInMBEC(*this, A, getState(), *CtxI);5280  }5281 5282  /// See AbstractAttribute::manifest(...).5283  ChangeStatus manifest(Attributor &A) override {5284    ChangeStatus InstrChanged = ChangeStatus::UNCHANGED;5285 5286    // Check for users that allow alignment annotations.5287    Value &AssociatedValue = getAssociatedValue();5288    if (isa<ConstantData>(AssociatedValue))5289      return ChangeStatus::UNCHANGED;5290 5291    for (const Use &U : AssociatedValue.uses()) {5292      if (auto *SI = dyn_cast<StoreInst>(U.getUser())) {5293        if (SI->getPointerOperand() == &AssociatedValue)5294          if (SI->getAlign() < getAssumedAlign()) {5295            STATS_DECLTRACK(AAAlign, Store,5296                            "Number of times alignment added to a store");5297            SI->setAlignment(getAssumedAlign());5298            InstrChanged = ChangeStatus::CHANGED;5299          }5300      } else if (auto *LI = dyn_cast<LoadInst>(U.getUser())) {5301        if (LI->getPointerOperand() == &AssociatedValue)5302          if (LI->getAlign() < getAssumedAlign()) {5303            LI->setAlignment(getAssumedAlign());5304            STATS_DECLTRACK(AAAlign, Load,5305                            "Number of times alignment added to a load");5306            InstrChanged = ChangeStatus::CHANGED;5307          }5308      } else if (auto *RMW = dyn_cast<AtomicRMWInst>(U.getUser())) {5309        if (RMW->getPointerOperand() == &AssociatedValue) {5310          if (RMW->getAlign() < getAssumedAlign()) {5311            STATS_DECLTRACK(AAAlign, AtomicRMW,5312                            "Number of times alignment added to atomicrmw");5313 5314            RMW->setAlignment(getAssumedAlign());5315            InstrChanged = ChangeStatus::CHANGED;5316          }5317        }5318      } else if (auto *CAS = dyn_cast<AtomicCmpXchgInst>(U.getUser())) {5319        if (CAS->getPointerOperand() == &AssociatedValue) {5320          if (CAS->getAlign() < getAssumedAlign()) {5321            STATS_DECLTRACK(AAAlign, AtomicCmpXchg,5322                            "Number of times alignment added to cmpxchg");5323            CAS->setAlignment(getAssumedAlign());5324            InstrChanged = ChangeStatus::CHANGED;5325          }5326        }5327      }5328    }5329 5330    ChangeStatus Changed = AAAlign::manifest(A);5331 5332    Align InheritAlign =5333        getAssociatedValue().getPointerAlignment(A.getDataLayout());5334    if (InheritAlign >= getAssumedAlign())5335      return InstrChanged;5336    return Changed | InstrChanged;5337  }5338 5339  // TODO: Provide a helper to determine the implied ABI alignment and check in5340  //       the existing manifest method and a new one for AAAlignImpl that value5341  //       to avoid making the alignment explicit if it did not improve.5342 5343  /// See AbstractAttribute::getDeducedAttributes5344  void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,5345                            SmallVectorImpl<Attribute> &Attrs) const override {5346    if (getAssumedAlign() > 1)5347      Attrs.emplace_back(5348          Attribute::getWithAlignment(Ctx, Align(getAssumedAlign())));5349  }5350 5351  /// See followUsesInMBEC5352  bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,5353                       AAAlign::StateType &State) {5354    bool TrackUse = false;5355 5356    unsigned int KnownAlign =5357        getKnownAlignForUse(A, *this, getAssociatedValue(), U, I, TrackUse);5358    State.takeKnownMaximum(KnownAlign);5359 5360    return TrackUse;5361  }5362 5363  /// See AbstractAttribute::getAsStr().5364  const std::string getAsStr(Attributor *A) const override {5365    return "align<" + std::to_string(getKnownAlign().value()) + "-" +5366           std::to_string(getAssumedAlign().value()) + ">";5367  }5368};5369 5370/// Align attribute for a floating value.5371struct AAAlignFloating : AAAlignImpl {5372  AAAlignFloating(const IRPosition &IRP, Attributor &A) : AAAlignImpl(IRP, A) {}5373 5374  /// See AbstractAttribute::updateImpl(...).5375  ChangeStatus updateImpl(Attributor &A) override {5376    const DataLayout &DL = A.getDataLayout();5377 5378    bool Stripped;5379    bool UsedAssumedInformation = false;5380    SmallVector<AA::ValueAndContext> Values;5381    if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,5382                                      AA::AnyScope, UsedAssumedInformation)) {5383      Values.push_back({getAssociatedValue(), getCtxI()});5384      Stripped = false;5385    } else {5386      Stripped = Values.size() != 1 ||5387                 Values.front().getValue() != &getAssociatedValue();5388    }5389 5390    StateType T;5391    auto VisitValueCB = [&](Value &V) -> bool {5392      if (isa<UndefValue>(V) || isa<ConstantPointerNull>(V))5393        return true;5394      const auto *AA = A.getAAFor<AAAlign>(*this, IRPosition::value(V),5395                                           DepClassTy::REQUIRED);5396      if (!AA || (!Stripped && this == AA)) {5397        int64_t Offset;5398        unsigned Alignment = 1;5399        if (const Value *Base =5400                GetPointerBaseWithConstantOffset(&V, Offset, DL)) {5401          // TODO: Use AAAlign for the base too.5402          Align PA = Base->getPointerAlignment(DL);5403          // BasePointerAddr + Offset = Alignment * Q for some integer Q.5404          // So we can say that the maximum power of two which is a divisor of5405          // gcd(Offset, Alignment) is an alignment.5406 5407          uint32_t gcd =5408              std::gcd(uint32_t(abs((int32_t)Offset)), uint32_t(PA.value()));5409          Alignment = llvm::bit_floor(gcd);5410        } else {5411          Alignment = V.getPointerAlignment(DL).value();5412        }5413        // Use only IR information if we did not strip anything.5414        T.takeKnownMaximum(Alignment);5415        T.indicatePessimisticFixpoint();5416      } else {5417        // Use abstract attribute information.5418        const AAAlign::StateType &DS = AA->getState();5419        T ^= DS;5420      }5421      return T.isValidState();5422    };5423 5424    for (const auto &VAC : Values) {5425      if (!VisitValueCB(*VAC.getValue()))5426        return indicatePessimisticFixpoint();5427    }5428 5429    //  TODO: If we know we visited all incoming values, thus no are assumed5430    //  dead, we can take the known information from the state T.5431    return clampStateAndIndicateChange(getState(), T);5432  }5433 5434  /// See AbstractAttribute::trackStatistics()5435  void trackStatistics() const override { STATS_DECLTRACK_FLOATING_ATTR(align) }5436};5437 5438/// Align attribute for function return value.5439struct AAAlignReturned final5440    : AAReturnedFromReturnedValues<AAAlign, AAAlignImpl> {5441  using Base = AAReturnedFromReturnedValues<AAAlign, AAAlignImpl>;5442  AAAlignReturned(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}5443 5444  /// See AbstractAttribute::trackStatistics()5445  void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(aligned) }5446};5447 5448/// Align attribute for function argument.5449struct AAAlignArgument final5450    : AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl> {5451  using Base = AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl>;5452  AAAlignArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}5453 5454  /// See AbstractAttribute::manifest(...).5455  ChangeStatus manifest(Attributor &A) override {5456    // If the associated argument is involved in a must-tail call we give up5457    // because we would need to keep the argument alignments of caller and5458    // callee in-sync. Just does not seem worth the trouble right now.5459    if (A.getInfoCache().isInvolvedInMustTailCall(*getAssociatedArgument()))5460      return ChangeStatus::UNCHANGED;5461    return Base::manifest(A);5462  }5463 5464  /// See AbstractAttribute::trackStatistics()5465  void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(aligned) }5466};5467 5468struct AAAlignCallSiteArgument final : AAAlignFloating {5469  AAAlignCallSiteArgument(const IRPosition &IRP, Attributor &A)5470      : AAAlignFloating(IRP, A) {}5471 5472  /// See AbstractAttribute::manifest(...).5473  ChangeStatus manifest(Attributor &A) override {5474    // If the associated argument is involved in a must-tail call we give up5475    // because we would need to keep the argument alignments of caller and5476    // callee in-sync. Just does not seem worth the trouble right now.5477    if (Argument *Arg = getAssociatedArgument())5478      if (A.getInfoCache().isInvolvedInMustTailCall(*Arg))5479        return ChangeStatus::UNCHANGED;5480    ChangeStatus Changed = AAAlignImpl::manifest(A);5481    Align InheritAlign =5482        getAssociatedValue().getPointerAlignment(A.getDataLayout());5483    if (InheritAlign >= getAssumedAlign())5484      Changed = ChangeStatus::UNCHANGED;5485    return Changed;5486  }5487 5488  /// See AbstractAttribute::updateImpl(Attributor &A).5489  ChangeStatus updateImpl(Attributor &A) override {5490    ChangeStatus Changed = AAAlignFloating::updateImpl(A);5491    if (Argument *Arg = getAssociatedArgument()) {5492      // We only take known information from the argument5493      // so we do not need to track a dependence.5494      const auto *ArgAlignAA = A.getAAFor<AAAlign>(5495          *this, IRPosition::argument(*Arg), DepClassTy::NONE);5496      if (ArgAlignAA)5497        takeKnownMaximum(ArgAlignAA->getKnownAlign().value());5498    }5499    return Changed;5500  }5501 5502  /// See AbstractAttribute::trackStatistics()5503  void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(aligned) }5504};5505 5506/// Align attribute deduction for a call site return value.5507struct AAAlignCallSiteReturned final5508    : AACalleeToCallSite<AAAlign, AAAlignImpl> {5509  using Base = AACalleeToCallSite<AAAlign, AAAlignImpl>;5510  AAAlignCallSiteReturned(const IRPosition &IRP, Attributor &A)5511      : Base(IRP, A) {}5512 5513  ChangeStatus updateImpl(Attributor &A) override {5514    Instruction *I = getIRPosition().getCtxI();5515    if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {5516      switch (II->getIntrinsicID()) {5517      case Intrinsic::ptrmask: {5518        Align Alignment;5519        bool Valid = false;5520 5521        const auto *ConstVals = A.getAAFor<AAPotentialConstantValues>(5522            *this, IRPosition::value(*II->getOperand(1)), DepClassTy::REQUIRED);5523        if (ConstVals && ConstVals->isValidState()) {5524          unsigned ShiftValue =5525              std::min(ConstVals->getAssumedMinTrailingZeros(),5526                       Value::MaxAlignmentExponent);5527          Alignment = Align(UINT64_C(1) << ShiftValue);5528          Valid = true;5529        }5530 5531        const auto *AlignAA =5532            A.getAAFor<AAAlign>(*this, IRPosition::value(*(II->getOperand(0))),5533                                DepClassTy::REQUIRED);5534        if (AlignAA && AlignAA->isValidState()) {5535          Alignment = std::max(AlignAA->getAssumedAlign(), Alignment);5536          Valid = true;5537        }5538 5539        if (Valid)5540          return clampStateAndIndicateChange<StateType>(5541              this->getState(),5542              std::min(this->getAssumedAlign(), Alignment).value());5543        break;5544      }5545      default:5546        break;5547      }5548    }5549    return Base::updateImpl(A);5550  };5551  /// See AbstractAttribute::trackStatistics()5552  void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(align); }5553};5554} // namespace5555 5556/// ------------------ Function No-Return Attribute ----------------------------5557namespace {5558struct AANoReturnImpl : public AANoReturn {5559  AANoReturnImpl(const IRPosition &IRP, Attributor &A) : AANoReturn(IRP, A) {}5560 5561  /// See AbstractAttribute::initialize(...).5562  void initialize(Attributor &A) override {5563    bool IsKnown;5564    assert(!AA::hasAssumedIRAttr<Attribute::NoReturn>(5565        A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));5566    (void)IsKnown;5567  }5568 5569  /// See AbstractAttribute::getAsStr().5570  const std::string getAsStr(Attributor *A) const override {5571    return getAssumed() ? "noreturn" : "may-return";5572  }5573 5574  /// See AbstractAttribute::updateImpl(Attributor &A).5575  ChangeStatus updateImpl(Attributor &A) override {5576    auto CheckForNoReturn = [](Instruction &) { return false; };5577    bool UsedAssumedInformation = false;5578    if (!A.checkForAllInstructions(CheckForNoReturn, *this,5579                                   {(unsigned)Instruction::Ret},5580                                   UsedAssumedInformation))5581      return indicatePessimisticFixpoint();5582    return ChangeStatus::UNCHANGED;5583  }5584};5585 5586struct AANoReturnFunction final : AANoReturnImpl {5587  AANoReturnFunction(const IRPosition &IRP, Attributor &A)5588      : AANoReturnImpl(IRP, A) {}5589 5590  /// See AbstractAttribute::trackStatistics()5591  void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(noreturn) }5592};5593 5594/// NoReturn attribute deduction for a call sites.5595struct AANoReturnCallSite final5596    : AACalleeToCallSite<AANoReturn, AANoReturnImpl> {5597  AANoReturnCallSite(const IRPosition &IRP, Attributor &A)5598      : AACalleeToCallSite<AANoReturn, AANoReturnImpl>(IRP, A) {}5599 5600  /// See AbstractAttribute::trackStatistics()5601  void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(noreturn); }5602};5603} // namespace5604 5605/// ----------------------- Instance Info ---------------------------------5606 5607namespace {5608/// A class to hold the state of for no-capture attributes.5609struct AAInstanceInfoImpl : public AAInstanceInfo {5610  AAInstanceInfoImpl(const IRPosition &IRP, Attributor &A)5611      : AAInstanceInfo(IRP, A) {}5612 5613  /// See AbstractAttribute::initialize(...).5614  void initialize(Attributor &A) override {5615    Value &V = getAssociatedValue();5616    if (auto *C = dyn_cast<Constant>(&V)) {5617      if (C->isThreadDependent())5618        indicatePessimisticFixpoint();5619      else5620        indicateOptimisticFixpoint();5621      return;5622    }5623    if (auto *CB = dyn_cast<CallBase>(&V))5624      if (CB->arg_size() == 0 && !CB->mayHaveSideEffects() &&5625          !CB->mayReadFromMemory()) {5626        indicateOptimisticFixpoint();5627        return;5628      }5629    if (auto *I = dyn_cast<Instruction>(&V)) {5630      const auto *CI =5631          A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(5632              *I->getFunction());5633      if (mayBeInCycle(CI, I, /* HeaderOnly */ false)) {5634        indicatePessimisticFixpoint();5635        return;5636      }5637    }5638  }5639 5640  /// See AbstractAttribute::updateImpl(...).5641  ChangeStatus updateImpl(Attributor &A) override {5642    ChangeStatus Changed = ChangeStatus::UNCHANGED;5643 5644    Value &V = getAssociatedValue();5645    const Function *Scope = nullptr;5646    if (auto *I = dyn_cast<Instruction>(&V))5647      Scope = I->getFunction();5648    if (auto *A = dyn_cast<Argument>(&V)) {5649      Scope = A->getParent();5650      if (!Scope->hasLocalLinkage())5651        return Changed;5652    }5653    if (!Scope)5654      return indicateOptimisticFixpoint();5655 5656    bool IsKnownNoRecurse;5657    if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(5658            A, this, IRPosition::function(*Scope), DepClassTy::OPTIONAL,5659            IsKnownNoRecurse))5660      return Changed;5661 5662    auto UsePred = [&](const Use &U, bool &Follow) {5663      const Instruction *UserI = dyn_cast<Instruction>(U.getUser());5664      if (!UserI || isa<GetElementPtrInst>(UserI) || isa<CastInst>(UserI) ||5665          isa<PHINode>(UserI) || isa<SelectInst>(UserI)) {5666        Follow = true;5667        return true;5668      }5669      if (isa<LoadInst>(UserI) || isa<CmpInst>(UserI) ||5670          (isa<StoreInst>(UserI) &&5671           cast<StoreInst>(UserI)->getValueOperand() != U.get()))5672        return true;5673      if (auto *CB = dyn_cast<CallBase>(UserI)) {5674        // This check is not guaranteeing uniqueness but for now that we cannot5675        // end up with two versions of \p U thinking it was one.5676        auto *Callee = dyn_cast_if_present<Function>(CB->getCalledOperand());5677        if (!Callee || !Callee->hasLocalLinkage())5678          return true;5679        if (!CB->isArgOperand(&U))5680          return false;5681        const auto *ArgInstanceInfoAA = A.getAAFor<AAInstanceInfo>(5682            *this, IRPosition::callsite_argument(*CB, CB->getArgOperandNo(&U)),5683            DepClassTy::OPTIONAL);5684        if (!ArgInstanceInfoAA ||5685            !ArgInstanceInfoAA->isAssumedUniqueForAnalysis())5686          return false;5687        // If this call base might reach the scope again we might forward the5688        // argument back here. This is very conservative.5689        if (AA::isPotentiallyReachable(5690                A, *CB, *Scope, *this, /* ExclusionSet */ nullptr,5691                [Scope](const Function &Fn) { return &Fn != Scope; }))5692          return false;5693        return true;5694      }5695      return false;5696    };5697 5698    auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {5699      if (auto *SI = dyn_cast<StoreInst>(OldU.getUser())) {5700        auto *Ptr = SI->getPointerOperand()->stripPointerCasts();5701        if ((isa<AllocaInst>(Ptr) || isNoAliasCall(Ptr)) &&5702            AA::isDynamicallyUnique(A, *this, *Ptr))5703          return true;5704      }5705      return false;5706    };5707 5708    if (!A.checkForAllUses(UsePred, *this, V, /* CheckBBLivenessOnly */ true,5709                           DepClassTy::OPTIONAL,5710                           /* IgnoreDroppableUses */ true, EquivalentUseCB))5711      return indicatePessimisticFixpoint();5712 5713    return Changed;5714  }5715 5716  /// See AbstractState::getAsStr().5717  const std::string getAsStr(Attributor *A) const override {5718    return isAssumedUniqueForAnalysis() ? "<unique [fAa]>" : "<unknown>";5719  }5720 5721  /// See AbstractAttribute::trackStatistics()5722  void trackStatistics() const override {}5723};5724 5725/// InstanceInfo attribute for floating values.5726struct AAInstanceInfoFloating : AAInstanceInfoImpl {5727  AAInstanceInfoFloating(const IRPosition &IRP, Attributor &A)5728      : AAInstanceInfoImpl(IRP, A) {}5729};5730 5731/// NoCapture attribute for function arguments.5732struct AAInstanceInfoArgument final : AAInstanceInfoFloating {5733  AAInstanceInfoArgument(const IRPosition &IRP, Attributor &A)5734      : AAInstanceInfoFloating(IRP, A) {}5735};5736 5737/// InstanceInfo attribute for call site arguments.5738struct AAInstanceInfoCallSiteArgument final : AAInstanceInfoImpl {5739  AAInstanceInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)5740      : AAInstanceInfoImpl(IRP, A) {}5741 5742  /// See AbstractAttribute::updateImpl(...).5743  ChangeStatus updateImpl(Attributor &A) override {5744    // TODO: Once we have call site specific value information we can provide5745    //       call site specific liveness information and then it makes5746    //       sense to specialize attributes for call sites arguments instead of5747    //       redirecting requests to the callee argument.5748    Argument *Arg = getAssociatedArgument();5749    if (!Arg)5750      return indicatePessimisticFixpoint();5751    const IRPosition &ArgPos = IRPosition::argument(*Arg);5752    auto *ArgAA =5753        A.getAAFor<AAInstanceInfo>(*this, ArgPos, DepClassTy::REQUIRED);5754    if (!ArgAA)5755      return indicatePessimisticFixpoint();5756    return clampStateAndIndicateChange(getState(), ArgAA->getState());5757  }5758};5759 5760/// InstanceInfo attribute for function return value.5761struct AAInstanceInfoReturned final : AAInstanceInfoImpl {5762  AAInstanceInfoReturned(const IRPosition &IRP, Attributor &A)5763      : AAInstanceInfoImpl(IRP, A) {5764    llvm_unreachable("InstanceInfo is not applicable to function returns!");5765  }5766 5767  /// See AbstractAttribute::initialize(...).5768  void initialize(Attributor &A) override {5769    llvm_unreachable("InstanceInfo is not applicable to function returns!");5770  }5771 5772  /// See AbstractAttribute::updateImpl(...).5773  ChangeStatus updateImpl(Attributor &A) override {5774    llvm_unreachable("InstanceInfo is not applicable to function returns!");5775  }5776};5777 5778/// InstanceInfo attribute deduction for a call site return value.5779struct AAInstanceInfoCallSiteReturned final : AAInstanceInfoFloating {5780  AAInstanceInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)5781      : AAInstanceInfoFloating(IRP, A) {}5782};5783} // namespace5784 5785/// ----------------------- Variable Capturing ---------------------------------5786bool AANoCapture::isImpliedByIR(Attributor &A, const IRPosition &IRP,5787                                Attribute::AttrKind ImpliedAttributeKind,5788                                bool IgnoreSubsumingPositions) {5789  assert(ImpliedAttributeKind == Attribute::Captures &&5790         "Unexpected attribute kind");5791  Value &V = IRP.getAssociatedValue();5792  if (!isa<Constant>(V) && !IRP.isArgumentPosition())5793    return V.use_empty();5794 5795  // You cannot "capture" null in the default address space.5796  //5797  // FIXME: This should use NullPointerIsDefined to account for the function5798  // attribute.5799  if (isa<UndefValue>(V) || (isa<ConstantPointerNull>(V) &&5800                             V.getType()->getPointerAddressSpace() == 0)) {5801    return true;5802  }5803 5804  SmallVector<Attribute, 1> Attrs;5805  A.getAttrs(IRP, {Attribute::Captures}, Attrs,5806             /* IgnoreSubsumingPositions */ true);5807  for (const Attribute &Attr : Attrs)5808    if (capturesNothing(Attr.getCaptureInfo()))5809      return true;5810 5811  if (IRP.getPositionKind() == IRP_CALL_SITE_ARGUMENT)5812    if (Argument *Arg = IRP.getAssociatedArgument()) {5813      SmallVector<Attribute, 1> Attrs;5814      A.getAttrs(IRPosition::argument(*Arg),5815                 {Attribute::Captures, Attribute::ByVal}, Attrs,5816                 /* IgnoreSubsumingPositions */ true);5817      bool ArgNoCapture = any_of(Attrs, [](Attribute Attr) {5818        return Attr.getKindAsEnum() == Attribute::ByVal ||5819               capturesNothing(Attr.getCaptureInfo());5820      });5821      if (ArgNoCapture) {5822        A.manifestAttrs(IRP, Attribute::getWithCaptureInfo(5823                                 V.getContext(), CaptureInfo::none()));5824        return true;5825      }5826    }5827 5828  if (const Function *F = IRP.getAssociatedFunction()) {5829    // Check what state the associated function can actually capture.5830    AANoCapture::StateType State;5831    determineFunctionCaptureCapabilities(IRP, *F, State);5832    if (State.isKnown(NO_CAPTURE)) {5833      A.manifestAttrs(IRP, Attribute::getWithCaptureInfo(V.getContext(),5834                                                         CaptureInfo::none()));5835      return true;5836    }5837  }5838 5839  return false;5840}5841 5842/// Set the NOT_CAPTURED_IN_MEM and NOT_CAPTURED_IN_RET bits in \p Known5843/// depending on the ability of the function associated with \p IRP to capture5844/// state in memory and through "returning/throwing", respectively.5845void AANoCapture::determineFunctionCaptureCapabilities(const IRPosition &IRP,5846                                                       const Function &F,5847                                                       BitIntegerState &State) {5848  // TODO: Once we have memory behavior attributes we should use them here.5849 5850  // If we know we cannot communicate or write to memory, we do not care about5851  // ptr2int anymore.5852  bool ReadOnly = F.onlyReadsMemory();5853  bool NoThrow = F.doesNotThrow();5854  bool IsVoidReturn = F.getReturnType()->isVoidTy();5855  if (ReadOnly && NoThrow && IsVoidReturn) {5856    State.addKnownBits(NO_CAPTURE);5857    return;5858  }5859 5860  // A function cannot capture state in memory if it only reads memory, it can5861  // however return/throw state and the state might be influenced by the5862  // pointer value, e.g., loading from a returned pointer might reveal a bit.5863  if (ReadOnly)5864    State.addKnownBits(NOT_CAPTURED_IN_MEM);5865 5866  // A function cannot communicate state back if it does not through5867  // exceptions and doesn not return values.5868  if (NoThrow && IsVoidReturn)5869    State.addKnownBits(NOT_CAPTURED_IN_RET);5870 5871  // Check existing "returned" attributes.5872  int ArgNo = IRP.getCalleeArgNo();5873  if (!NoThrow || ArgNo < 0 ||5874      !F.getAttributes().hasAttrSomewhere(Attribute::Returned))5875    return;5876 5877  for (unsigned U = 0, E = F.arg_size(); U < E; ++U)5878    if (F.hasParamAttribute(U, Attribute::Returned)) {5879      if (U == unsigned(ArgNo))5880        State.removeAssumedBits(NOT_CAPTURED_IN_RET);5881      else if (ReadOnly)5882        State.addKnownBits(NO_CAPTURE);5883      else5884        State.addKnownBits(NOT_CAPTURED_IN_RET);5885      break;5886    }5887}5888 5889namespace {5890/// A class to hold the state of for no-capture attributes.5891struct AANoCaptureImpl : public AANoCapture {5892  AANoCaptureImpl(const IRPosition &IRP, Attributor &A) : AANoCapture(IRP, A) {}5893 5894  /// See AbstractAttribute::initialize(...).5895  void initialize(Attributor &A) override {5896    bool IsKnown;5897    assert(!AA::hasAssumedIRAttr<Attribute::Captures>(5898        A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));5899    (void)IsKnown;5900  }5901 5902  /// See AbstractAttribute::updateImpl(...).5903  ChangeStatus updateImpl(Attributor &A) override;5904 5905  /// see AbstractAttribute::isAssumedNoCaptureMaybeReturned(...).5906  void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,5907                            SmallVectorImpl<Attribute> &Attrs) const override {5908    if (!isAssumedNoCaptureMaybeReturned())5909      return;5910 5911    if (isArgumentPosition()) {5912      if (isAssumedNoCapture())5913        Attrs.emplace_back(Attribute::get(Ctx, Attribute::Captures));5914      else if (ManifestInternal)5915        Attrs.emplace_back(Attribute::get(Ctx, "no-capture-maybe-returned"));5916    }5917  }5918 5919  /// See AbstractState::getAsStr().5920  const std::string getAsStr(Attributor *A) const override {5921    if (isKnownNoCapture())5922      return "known not-captured";5923    if (isAssumedNoCapture())5924      return "assumed not-captured";5925    if (isKnownNoCaptureMaybeReturned())5926      return "known not-captured-maybe-returned";5927    if (isAssumedNoCaptureMaybeReturned())5928      return "assumed not-captured-maybe-returned";5929    return "assumed-captured";5930  }5931 5932  /// Check the use \p U and update \p State accordingly. Return true if we5933  /// should continue to update the state.5934  bool checkUse(Attributor &A, AANoCapture::StateType &State, const Use &U,5935                bool &Follow) {5936    Instruction *UInst = cast<Instruction>(U.getUser());5937    LLVM_DEBUG(dbgs() << "[AANoCapture] Check use: " << *U.get() << " in "5938                      << *UInst << "\n");5939 5940    // Deal with ptr2int by following uses.5941    if (isa<PtrToIntInst>(UInst)) {5942      LLVM_DEBUG(dbgs() << " - ptr2int assume the worst!\n");5943      return isCapturedIn(State, /* Memory */ true, /* Integer */ true,5944                          /* Return */ true);5945    }5946 5947    // For stores we already checked if we can follow them, if they make it5948    // here we give up.5949    if (isa<StoreInst>(UInst))5950      return isCapturedIn(State, /* Memory */ true, /* Integer */ true,5951                          /* Return */ true);5952 5953    // Explicitly catch return instructions.5954    if (isa<ReturnInst>(UInst)) {5955      if (UInst->getFunction() == getAnchorScope())5956        return isCapturedIn(State, /* Memory */ false, /* Integer */ false,5957                            /* Return */ true);5958      return isCapturedIn(State, /* Memory */ true, /* Integer */ true,5959                          /* Return */ true);5960    }5961 5962    // For now we only use special logic for call sites. However, the tracker5963    // itself knows about a lot of other non-capturing cases already.5964    auto *CB = dyn_cast<CallBase>(UInst);5965    if (!CB || !CB->isArgOperand(&U))5966      return isCapturedIn(State, /* Memory */ true, /* Integer */ true,5967                          /* Return */ true);5968 5969    unsigned ArgNo = CB->getArgOperandNo(&U);5970    const IRPosition &CSArgPos = IRPosition::callsite_argument(*CB, ArgNo);5971    // If we have a abstract no-capture attribute for the argument we can use5972    // it to justify a non-capture attribute here. This allows recursion!5973    bool IsKnownNoCapture;5974    const AANoCapture *ArgNoCaptureAA = nullptr;5975    bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(5976        A, this, CSArgPos, DepClassTy::REQUIRED, IsKnownNoCapture, false,5977        &ArgNoCaptureAA);5978    if (IsAssumedNoCapture)5979      return isCapturedIn(State, /* Memory */ false, /* Integer */ false,5980                          /* Return */ false);5981    if (ArgNoCaptureAA && ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned()) {5982      Follow = true;5983      return isCapturedIn(State, /* Memory */ false, /* Integer */ false,5984                          /* Return */ false);5985    }5986 5987    // Lastly, we could not find a reason no-capture can be assumed so we don't.5988    return isCapturedIn(State, /* Memory */ true, /* Integer */ true,5989                        /* Return */ true);5990  }5991 5992  /// Update \p State according to \p CapturedInMem, \p CapturedInInt, and5993  /// \p CapturedInRet, then return true if we should continue updating the5994  /// state.5995  static bool isCapturedIn(AANoCapture::StateType &State, bool CapturedInMem,5996                           bool CapturedInInt, bool CapturedInRet) {5997    LLVM_DEBUG(dbgs() << " - captures [Mem " << CapturedInMem << "|Int "5998                      << CapturedInInt << "|Ret " << CapturedInRet << "]\n");5999    if (CapturedInMem)6000      State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_MEM);6001    if (CapturedInInt)6002      State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_INT);6003    if (CapturedInRet)6004      State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_RET);6005    return State.isAssumed(AANoCapture::NO_CAPTURE_MAYBE_RETURNED);6006  }6007};6008 6009ChangeStatus AANoCaptureImpl::updateImpl(Attributor &A) {6010  const IRPosition &IRP = getIRPosition();6011  Value *V = isArgumentPosition() ? IRP.getAssociatedArgument()6012                                  : &IRP.getAssociatedValue();6013  if (!V)6014    return indicatePessimisticFixpoint();6015 6016  const Function *F =6017      isArgumentPosition() ? IRP.getAssociatedFunction() : IRP.getAnchorScope();6018 6019  // TODO: Is the checkForAllUses below useful for constants?6020  if (!F)6021    return indicatePessimisticFixpoint();6022 6023  AANoCapture::StateType T;6024  const IRPosition &FnPos = IRPosition::function(*F);6025 6026  // Readonly means we cannot capture through memory.6027  bool IsKnown;6028  if (AA::isAssumedReadOnly(A, FnPos, *this, IsKnown)) {6029    T.addKnownBits(NOT_CAPTURED_IN_MEM);6030    if (IsKnown)6031      addKnownBits(NOT_CAPTURED_IN_MEM);6032  }6033 6034  // Make sure all returned values are different than the underlying value.6035  // TODO: we could do this in a more sophisticated way inside6036  //       AAReturnedValues, e.g., track all values that escape through returns6037  //       directly somehow.6038  auto CheckReturnedArgs = [&](bool &UsedAssumedInformation) {6039    SmallVector<AA::ValueAndContext> Values;6040    if (!A.getAssumedSimplifiedValues(IRPosition::returned(*F), this, Values,6041                                      AA::ValueScope::Intraprocedural,6042                                      UsedAssumedInformation))6043      return false;6044    bool SeenConstant = false;6045    for (const AA::ValueAndContext &VAC : Values) {6046      if (isa<Constant>(VAC.getValue())) {6047        if (SeenConstant)6048          return false;6049        SeenConstant = true;6050      } else if (!isa<Argument>(VAC.getValue()) ||6051                 VAC.getValue() == getAssociatedArgument())6052        return false;6053    }6054    return true;6055  };6056 6057  bool IsKnownNoUnwind;6058  if (AA::hasAssumedIRAttr<Attribute::NoUnwind>(6059          A, this, FnPos, DepClassTy::OPTIONAL, IsKnownNoUnwind)) {6060    bool IsVoidTy = F->getReturnType()->isVoidTy();6061    bool UsedAssumedInformation = false;6062    if (IsVoidTy || CheckReturnedArgs(UsedAssumedInformation)) {6063      T.addKnownBits(NOT_CAPTURED_IN_RET);6064      if (T.isKnown(NOT_CAPTURED_IN_MEM))6065        return ChangeStatus::UNCHANGED;6066      if (IsKnownNoUnwind && (IsVoidTy || !UsedAssumedInformation)) {6067        addKnownBits(NOT_CAPTURED_IN_RET);6068        if (isKnown(NOT_CAPTURED_IN_MEM))6069          return indicateOptimisticFixpoint();6070      }6071    }6072  }6073 6074  auto UseCheck = [&](const Use &U, bool &Follow) -> bool {6075    // TODO(captures): Make this more precise.6076    UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);6077    if (capturesNothing(CI))6078      return true;6079    if (CI.isPassthrough()) {6080      Follow = true;6081      return true;6082    }6083    return checkUse(A, T, U, Follow);6084  };6085 6086  if (!A.checkForAllUses(UseCheck, *this, *V))6087    return indicatePessimisticFixpoint();6088 6089  AANoCapture::StateType &S = getState();6090  auto Assumed = S.getAssumed();6091  S.intersectAssumedBits(T.getAssumed());6092  if (!isAssumedNoCaptureMaybeReturned())6093    return indicatePessimisticFixpoint();6094  return Assumed == S.getAssumed() ? ChangeStatus::UNCHANGED6095                                   : ChangeStatus::CHANGED;6096}6097 6098/// NoCapture attribute for function arguments.6099struct AANoCaptureArgument final : AANoCaptureImpl {6100  AANoCaptureArgument(const IRPosition &IRP, Attributor &A)6101      : AANoCaptureImpl(IRP, A) {}6102 6103  /// See AbstractAttribute::trackStatistics()6104  void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nocapture) }6105};6106 6107/// NoCapture attribute for call site arguments.6108struct AANoCaptureCallSiteArgument final : AANoCaptureImpl {6109  AANoCaptureCallSiteArgument(const IRPosition &IRP, Attributor &A)6110      : AANoCaptureImpl(IRP, A) {}6111 6112  /// See AbstractAttribute::updateImpl(...).6113  ChangeStatus updateImpl(Attributor &A) override {6114    // TODO: Once we have call site specific value information we can provide6115    //       call site specific liveness information and then it makes6116    //       sense to specialize attributes for call sites arguments instead of6117    //       redirecting requests to the callee argument.6118    Argument *Arg = getAssociatedArgument();6119    if (!Arg)6120      return indicatePessimisticFixpoint();6121    const IRPosition &ArgPos = IRPosition::argument(*Arg);6122    bool IsKnownNoCapture;6123    const AANoCapture *ArgAA = nullptr;6124    if (AA::hasAssumedIRAttr<Attribute::Captures>(6125            A, this, ArgPos, DepClassTy::REQUIRED, IsKnownNoCapture, false,6126            &ArgAA))6127      return ChangeStatus::UNCHANGED;6128    if (!ArgAA || !ArgAA->isAssumedNoCaptureMaybeReturned())6129      return indicatePessimisticFixpoint();6130    return clampStateAndIndicateChange(getState(), ArgAA->getState());6131  }6132 6133  /// See AbstractAttribute::trackStatistics()6134  void trackStatistics() const override {6135    STATS_DECLTRACK_CSARG_ATTR(nocapture)6136  };6137};6138 6139/// NoCapture attribute for floating values.6140struct AANoCaptureFloating final : AANoCaptureImpl {6141  AANoCaptureFloating(const IRPosition &IRP, Attributor &A)6142      : AANoCaptureImpl(IRP, A) {}6143 6144  /// See AbstractAttribute::trackStatistics()6145  void trackStatistics() const override {6146    STATS_DECLTRACK_FLOATING_ATTR(nocapture)6147  }6148};6149 6150/// NoCapture attribute for function return value.6151struct AANoCaptureReturned final : AANoCaptureImpl {6152  AANoCaptureReturned(const IRPosition &IRP, Attributor &A)6153      : AANoCaptureImpl(IRP, A) {6154    llvm_unreachable("NoCapture is not applicable to function returns!");6155  }6156 6157  /// See AbstractAttribute::initialize(...).6158  void initialize(Attributor &A) override {6159    llvm_unreachable("NoCapture is not applicable to function returns!");6160  }6161 6162  /// See AbstractAttribute::updateImpl(...).6163  ChangeStatus updateImpl(Attributor &A) override {6164    llvm_unreachable("NoCapture is not applicable to function returns!");6165  }6166 6167  /// See AbstractAttribute::trackStatistics()6168  void trackStatistics() const override {}6169};6170 6171/// NoCapture attribute deduction for a call site return value.6172struct AANoCaptureCallSiteReturned final : AANoCaptureImpl {6173  AANoCaptureCallSiteReturned(const IRPosition &IRP, Attributor &A)6174      : AANoCaptureImpl(IRP, A) {}6175 6176  /// See AbstractAttribute::initialize(...).6177  void initialize(Attributor &A) override {6178    const Function *F = getAnchorScope();6179    // Check what state the associated function can actually capture.6180    determineFunctionCaptureCapabilities(getIRPosition(), *F, *this);6181  }6182 6183  /// See AbstractAttribute::trackStatistics()6184  void trackStatistics() const override {6185    STATS_DECLTRACK_CSRET_ATTR(nocapture)6186  }6187};6188} // namespace6189 6190/// ------------------ Value Simplify Attribute ----------------------------6191 6192bool ValueSimplifyStateType::unionAssumed(std::optional<Value *> Other) {6193  // FIXME: Add a typecast support.6194  SimplifiedAssociatedValue = AA::combineOptionalValuesInAAValueLatice(6195      SimplifiedAssociatedValue, Other, Ty);6196  if (SimplifiedAssociatedValue == std::optional<Value *>(nullptr))6197    return false;6198 6199  LLVM_DEBUG({6200    if (SimplifiedAssociatedValue)6201      dbgs() << "[ValueSimplify] is assumed to be "6202             << **SimplifiedAssociatedValue << "\n";6203    else6204      dbgs() << "[ValueSimplify] is assumed to be <none>\n";6205  });6206  return true;6207}6208 6209namespace {6210struct AAValueSimplifyImpl : AAValueSimplify {6211  AAValueSimplifyImpl(const IRPosition &IRP, Attributor &A)6212      : AAValueSimplify(IRP, A) {}6213 6214  /// See AbstractAttribute::initialize(...).6215  void initialize(Attributor &A) override {6216    if (getAssociatedValue().getType()->isVoidTy())6217      indicatePessimisticFixpoint();6218    if (A.hasSimplificationCallback(getIRPosition()))6219      indicatePessimisticFixpoint();6220  }6221 6222  /// See AbstractAttribute::getAsStr().6223  const std::string getAsStr(Attributor *A) const override {6224    LLVM_DEBUG({6225      dbgs() << "SAV: " << (bool)SimplifiedAssociatedValue << " ";6226      if (SimplifiedAssociatedValue && *SimplifiedAssociatedValue)6227        dbgs() << "SAV: " << **SimplifiedAssociatedValue << " ";6228    });6229    return isValidState() ? (isAtFixpoint() ? "simplified" : "maybe-simple")6230                          : "not-simple";6231  }6232 6233  /// See AbstractAttribute::trackStatistics()6234  void trackStatistics() const override {}6235 6236  /// See AAValueSimplify::getAssumedSimplifiedValue()6237  std::optional<Value *>6238  getAssumedSimplifiedValue(Attributor &A) const override {6239    return SimplifiedAssociatedValue;6240  }6241 6242  /// Ensure the return value is \p V with type \p Ty, if not possible return6243  /// nullptr. If \p Check is true we will only verify such an operation would6244  /// suceed and return a non-nullptr value if that is the case. No IR is6245  /// generated or modified.6246  static Value *ensureType(Attributor &A, Value &V, Type &Ty, Instruction *CtxI,6247                           bool Check) {6248    if (auto *TypedV = AA::getWithType(V, Ty))6249      return TypedV;6250    if (CtxI && V.getType()->canLosslesslyBitCastTo(&Ty))6251      return Check ? &V6252                   : BitCastInst::CreatePointerBitCastOrAddrSpaceCast(6253                         &V, &Ty, "", CtxI->getIterator());6254    return nullptr;6255  }6256 6257  /// Reproduce \p I with type \p Ty or return nullptr if that is not posisble.6258  /// If \p Check is true we will only verify such an operation would suceed and6259  /// return a non-nullptr value if that is the case. No IR is generated or6260  /// modified.6261  static Value *reproduceInst(Attributor &A,6262                              const AbstractAttribute &QueryingAA,6263                              Instruction &I, Type &Ty, Instruction *CtxI,6264                              bool Check, ValueToValueMapTy &VMap) {6265    assert(CtxI && "Cannot reproduce an instruction without context!");6266    if (Check && (I.mayReadFromMemory() ||6267                  !isSafeToSpeculativelyExecute(&I, CtxI, /* DT */ nullptr,6268                                                /* TLI */ nullptr)))6269      return nullptr;6270    for (Value *Op : I.operands()) {6271      Value *NewOp = reproduceValue(A, QueryingAA, *Op, Ty, CtxI, Check, VMap);6272      if (!NewOp) {6273        assert(Check && "Manifest of new value unexpectedly failed!");6274        return nullptr;6275      }6276      if (!Check)6277        VMap[Op] = NewOp;6278    }6279    if (Check)6280      return &I;6281 6282    Instruction *CloneI = I.clone();6283    // TODO: Try to salvage debug information here.6284    CloneI->setDebugLoc(DebugLoc());6285    VMap[&I] = CloneI;6286    CloneI->insertBefore(CtxI->getIterator());6287    RemapInstruction(CloneI, VMap);6288    return CloneI;6289  }6290 6291  /// Reproduce \p V with type \p Ty or return nullptr if that is not posisble.6292  /// If \p Check is true we will only verify such an operation would suceed and6293  /// return a non-nullptr value if that is the case. No IR is generated or6294  /// modified.6295  static Value *reproduceValue(Attributor &A,6296                               const AbstractAttribute &QueryingAA, Value &V,6297                               Type &Ty, Instruction *CtxI, bool Check,6298                               ValueToValueMapTy &VMap) {6299    if (const auto &NewV = VMap.lookup(&V))6300      return NewV;6301    bool UsedAssumedInformation = false;6302    std::optional<Value *> SimpleV = A.getAssumedSimplified(6303        V, QueryingAA, UsedAssumedInformation, AA::Interprocedural);6304    if (!SimpleV.has_value())6305      return PoisonValue::get(&Ty);6306    Value *EffectiveV = &V;6307    if (*SimpleV)6308      EffectiveV = *SimpleV;6309    if (auto *C = dyn_cast<Constant>(EffectiveV))6310      return C;6311    if (CtxI && AA::isValidAtPosition(AA::ValueAndContext(*EffectiveV, *CtxI),6312                                      A.getInfoCache()))6313      return ensureType(A, *EffectiveV, Ty, CtxI, Check);6314    if (auto *I = dyn_cast<Instruction>(EffectiveV))6315      if (Value *NewV = reproduceInst(A, QueryingAA, *I, Ty, CtxI, Check, VMap))6316        return ensureType(A, *NewV, Ty, CtxI, Check);6317    return nullptr;6318  }6319 6320  /// Return a value we can use as replacement for the associated one, or6321  /// nullptr if we don't have one that makes sense.6322  Value *manifestReplacementValue(Attributor &A, Instruction *CtxI) const {6323    Value *NewV = SimplifiedAssociatedValue6324                      ? *SimplifiedAssociatedValue6325                      : UndefValue::get(getAssociatedType());6326    if (NewV && NewV != &getAssociatedValue()) {6327      ValueToValueMapTy VMap;6328      // First verify we can reprduce the value with the required type at the6329      // context location before we actually start modifying the IR.6330      if (reproduceValue(A, *this, *NewV, *getAssociatedType(), CtxI,6331                         /* CheckOnly */ true, VMap))6332        return reproduceValue(A, *this, *NewV, *getAssociatedType(), CtxI,6333                              /* CheckOnly */ false, VMap);6334    }6335    return nullptr;6336  }6337 6338  /// Helper function for querying AAValueSimplify and updating candidate.6339  /// \param IRP The value position we are trying to unify with SimplifiedValue6340  bool checkAndUpdate(Attributor &A, const AbstractAttribute &QueryingAA,6341                      const IRPosition &IRP, bool Simplify = true) {6342    bool UsedAssumedInformation = false;6343    std::optional<Value *> QueryingValueSimplified = &IRP.getAssociatedValue();6344    if (Simplify)6345      QueryingValueSimplified = A.getAssumedSimplified(6346          IRP, QueryingAA, UsedAssumedInformation, AA::Interprocedural);6347    return unionAssumed(QueryingValueSimplified);6348  }6349 6350  /// Returns a candidate is found or not6351  template <typename AAType> bool askSimplifiedValueFor(Attributor &A) {6352    if (!getAssociatedValue().getType()->isIntegerTy())6353      return false;6354 6355    // This will also pass the call base context.6356    const auto *AA =6357        A.getAAFor<AAType>(*this, getIRPosition(), DepClassTy::NONE);6358    if (!AA)6359      return false;6360 6361    std::optional<Constant *> COpt = AA->getAssumedConstant(A);6362 6363    if (!COpt) {6364      SimplifiedAssociatedValue = std::nullopt;6365      A.recordDependence(*AA, *this, DepClassTy::OPTIONAL);6366      return true;6367    }6368    if (auto *C = *COpt) {6369      SimplifiedAssociatedValue = C;6370      A.recordDependence(*AA, *this, DepClassTy::OPTIONAL);6371      return true;6372    }6373    return false;6374  }6375 6376  bool askSimplifiedValueForOtherAAs(Attributor &A) {6377    if (askSimplifiedValueFor<AAValueConstantRange>(A))6378      return true;6379    if (askSimplifiedValueFor<AAPotentialConstantValues>(A))6380      return true;6381    return false;6382  }6383 6384  /// See AbstractAttribute::manifest(...).6385  ChangeStatus manifest(Attributor &A) override {6386    ChangeStatus Changed = ChangeStatus::UNCHANGED;6387    for (auto &U : getAssociatedValue().uses()) {6388      // Check if we need to adjust the insertion point to make sure the IR is6389      // valid.6390      Instruction *IP = dyn_cast<Instruction>(U.getUser());6391      if (auto *PHI = dyn_cast_or_null<PHINode>(IP))6392        IP = PHI->getIncomingBlock(U)->getTerminator();6393      if (auto *NewV = manifestReplacementValue(A, IP)) {6394        LLVM_DEBUG(dbgs() << "[ValueSimplify] " << getAssociatedValue()6395                          << " -> " << *NewV << " :: " << *this << "\n");6396        if (A.changeUseAfterManifest(U, *NewV))6397          Changed = ChangeStatus::CHANGED;6398      }6399    }6400 6401    return Changed | AAValueSimplify::manifest(A);6402  }6403 6404  /// See AbstractState::indicatePessimisticFixpoint(...).6405  ChangeStatus indicatePessimisticFixpoint() override {6406    SimplifiedAssociatedValue = &getAssociatedValue();6407    return AAValueSimplify::indicatePessimisticFixpoint();6408  }6409};6410 6411struct AAValueSimplifyArgument final : AAValueSimplifyImpl {6412  AAValueSimplifyArgument(const IRPosition &IRP, Attributor &A)6413      : AAValueSimplifyImpl(IRP, A) {}6414 6415  void initialize(Attributor &A) override {6416    AAValueSimplifyImpl::initialize(A);6417    if (A.hasAttr(getIRPosition(),6418                  {Attribute::InAlloca, Attribute::Preallocated,6419                   Attribute::StructRet, Attribute::Nest, Attribute::ByVal},6420                  /* IgnoreSubsumingPositions */ true))6421      indicatePessimisticFixpoint();6422  }6423 6424  /// See AbstractAttribute::updateImpl(...).6425  ChangeStatus updateImpl(Attributor &A) override {6426    // Byval is only replacable if it is readonly otherwise we would write into6427    // the replaced value and not the copy that byval creates implicitly.6428    Argument *Arg = getAssociatedArgument();6429    if (Arg->hasByValAttr()) {6430      // TODO: We probably need to verify synchronization is not an issue, e.g.,6431      //       there is no race by not copying a constant byval.6432      bool IsKnown;6433      if (!AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown))6434        return indicatePessimisticFixpoint();6435    }6436 6437    auto Before = SimplifiedAssociatedValue;6438 6439    auto PredForCallSite = [&](AbstractCallSite ACS) {6440      const IRPosition &ACSArgPos =6441          IRPosition::callsite_argument(ACS, getCallSiteArgNo());6442      // Check if a coresponding argument was found or if it is on not6443      // associated (which can happen for callback calls).6444      if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)6445        return false;6446 6447      // Simplify the argument operand explicitly and check if the result is6448      // valid in the current scope. This avoids refering to simplified values6449      // in other functions, e.g., we don't want to say a an argument in a6450      // static function is actually an argument in a different function.6451      bool UsedAssumedInformation = false;6452      std::optional<Constant *> SimpleArgOp =6453          A.getAssumedConstant(ACSArgPos, *this, UsedAssumedInformation);6454      if (!SimpleArgOp)6455        return true;6456      if (!*SimpleArgOp)6457        return false;6458      if (!AA::isDynamicallyUnique(A, *this, **SimpleArgOp))6459        return false;6460      return unionAssumed(*SimpleArgOp);6461    };6462 6463    // Generate a answer specific to a call site context.6464    bool Success;6465    bool UsedAssumedInformation = false;6466    if (hasCallBaseContext() &&6467        getCallBaseContext()->getCalledOperand() == Arg->getParent())6468      Success = PredForCallSite(6469          AbstractCallSite(&getCallBaseContext()->getCalledOperandUse()));6470    else6471      Success = A.checkForAllCallSites(PredForCallSite, *this, true,6472                                       UsedAssumedInformation);6473 6474    if (!Success)6475      if (!askSimplifiedValueForOtherAAs(A))6476        return indicatePessimisticFixpoint();6477 6478    // If a candidate was found in this update, return CHANGED.6479    return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED6480                                               : ChangeStatus ::CHANGED;6481  }6482 6483  /// See AbstractAttribute::trackStatistics()6484  void trackStatistics() const override {6485    STATS_DECLTRACK_ARG_ATTR(value_simplify)6486  }6487};6488 6489struct AAValueSimplifyReturned : AAValueSimplifyImpl {6490  AAValueSimplifyReturned(const IRPosition &IRP, Attributor &A)6491      : AAValueSimplifyImpl(IRP, A) {}6492 6493  /// See AAValueSimplify::getAssumedSimplifiedValue()6494  std::optional<Value *>6495  getAssumedSimplifiedValue(Attributor &A) const override {6496    if (!isValidState())6497      return nullptr;6498    return SimplifiedAssociatedValue;6499  }6500 6501  /// See AbstractAttribute::updateImpl(...).6502  ChangeStatus updateImpl(Attributor &A) override {6503    auto Before = SimplifiedAssociatedValue;6504 6505    auto ReturnInstCB = [&](Instruction &I) {6506      auto &RI = cast<ReturnInst>(I);6507      return checkAndUpdate(6508          A, *this,6509          IRPosition::value(*RI.getReturnValue(), getCallBaseContext()));6510    };6511 6512    bool UsedAssumedInformation = false;6513    if (!A.checkForAllInstructions(ReturnInstCB, *this, {Instruction::Ret},6514                                   UsedAssumedInformation))6515      if (!askSimplifiedValueForOtherAAs(A))6516        return indicatePessimisticFixpoint();6517 6518    // If a candidate was found in this update, return CHANGED.6519    return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED6520                                               : ChangeStatus ::CHANGED;6521  }6522 6523  ChangeStatus manifest(Attributor &A) override {6524    // We queried AAValueSimplify for the returned values so they will be6525    // replaced if a simplified form was found. Nothing to do here.6526    return ChangeStatus::UNCHANGED;6527  }6528 6529  /// See AbstractAttribute::trackStatistics()6530  void trackStatistics() const override {6531    STATS_DECLTRACK_FNRET_ATTR(value_simplify)6532  }6533};6534 6535struct AAValueSimplifyFloating : AAValueSimplifyImpl {6536  AAValueSimplifyFloating(const IRPosition &IRP, Attributor &A)6537      : AAValueSimplifyImpl(IRP, A) {}6538 6539  /// See AbstractAttribute::initialize(...).6540  void initialize(Attributor &A) override {6541    AAValueSimplifyImpl::initialize(A);6542    Value &V = getAnchorValue();6543 6544    // TODO: add other stuffs6545    if (isa<Constant>(V))6546      indicatePessimisticFixpoint();6547  }6548 6549  /// See AbstractAttribute::updateImpl(...).6550  ChangeStatus updateImpl(Attributor &A) override {6551    auto Before = SimplifiedAssociatedValue;6552    if (!askSimplifiedValueForOtherAAs(A))6553      return indicatePessimisticFixpoint();6554 6555    // If a candidate was found in this update, return CHANGED.6556    return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED6557                                               : ChangeStatus ::CHANGED;6558  }6559 6560  /// See AbstractAttribute::trackStatistics()6561  void trackStatistics() const override {6562    STATS_DECLTRACK_FLOATING_ATTR(value_simplify)6563  }6564};6565 6566struct AAValueSimplifyFunction : AAValueSimplifyImpl {6567  AAValueSimplifyFunction(const IRPosition &IRP, Attributor &A)6568      : AAValueSimplifyImpl(IRP, A) {}6569 6570  /// See AbstractAttribute::initialize(...).6571  void initialize(Attributor &A) override {6572    SimplifiedAssociatedValue = nullptr;6573    indicateOptimisticFixpoint();6574  }6575  /// See AbstractAttribute::initialize(...).6576  ChangeStatus updateImpl(Attributor &A) override {6577    llvm_unreachable(6578        "AAValueSimplify(Function|CallSite)::updateImpl will not be called");6579  }6580  /// See AbstractAttribute::trackStatistics()6581  void trackStatistics() const override {6582    STATS_DECLTRACK_FN_ATTR(value_simplify)6583  }6584};6585 6586struct AAValueSimplifyCallSite : AAValueSimplifyFunction {6587  AAValueSimplifyCallSite(const IRPosition &IRP, Attributor &A)6588      : AAValueSimplifyFunction(IRP, A) {}6589  /// See AbstractAttribute::trackStatistics()6590  void trackStatistics() const override {6591    STATS_DECLTRACK_CS_ATTR(value_simplify)6592  }6593};6594 6595struct AAValueSimplifyCallSiteReturned : AAValueSimplifyImpl {6596  AAValueSimplifyCallSiteReturned(const IRPosition &IRP, Attributor &A)6597      : AAValueSimplifyImpl(IRP, A) {}6598 6599  void initialize(Attributor &A) override {6600    AAValueSimplifyImpl::initialize(A);6601    Function *Fn = getAssociatedFunction();6602    assert(Fn && "Did expect an associted function");6603    for (Argument &Arg : Fn->args()) {6604      if (Arg.hasReturnedAttr()) {6605        auto IRP = IRPosition::callsite_argument(*cast<CallBase>(getCtxI()),6606                                                 Arg.getArgNo());6607        if (IRP.getPositionKind() == IRPosition::IRP_CALL_SITE_ARGUMENT &&6608            checkAndUpdate(A, *this, IRP))6609          indicateOptimisticFixpoint();6610        else6611          indicatePessimisticFixpoint();6612        return;6613      }6614    }6615  }6616 6617  /// See AbstractAttribute::updateImpl(...).6618  ChangeStatus updateImpl(Attributor &A) override {6619    return indicatePessimisticFixpoint();6620  }6621 6622  void trackStatistics() const override {6623    STATS_DECLTRACK_CSRET_ATTR(value_simplify)6624  }6625};6626 6627struct AAValueSimplifyCallSiteArgument : AAValueSimplifyFloating {6628  AAValueSimplifyCallSiteArgument(const IRPosition &IRP, Attributor &A)6629      : AAValueSimplifyFloating(IRP, A) {}6630 6631  /// See AbstractAttribute::manifest(...).6632  ChangeStatus manifest(Attributor &A) override {6633    ChangeStatus Changed = ChangeStatus::UNCHANGED;6634    // TODO: We should avoid simplification duplication to begin with.6635    auto *FloatAA = A.lookupAAFor<AAValueSimplify>(6636        IRPosition::value(getAssociatedValue()), this, DepClassTy::NONE);6637    if (FloatAA && FloatAA->getState().isValidState())6638      return Changed;6639 6640    if (auto *NewV = manifestReplacementValue(A, getCtxI())) {6641      Use &U = cast<CallBase>(&getAnchorValue())6642                   ->getArgOperandUse(getCallSiteArgNo());6643      if (A.changeUseAfterManifest(U, *NewV))6644        Changed = ChangeStatus::CHANGED;6645    }6646 6647    return Changed | AAValueSimplify::manifest(A);6648  }6649 6650  void trackStatistics() const override {6651    STATS_DECLTRACK_CSARG_ATTR(value_simplify)6652  }6653};6654} // namespace6655 6656/// ----------------------- Heap-To-Stack Conversion ---------------------------6657namespace {6658struct AAHeapToStackFunction final : public AAHeapToStack {6659 6660  struct AllocationInfo {6661    /// The call that allocates the memory.6662    CallBase *const CB;6663 6664    /// The library function id for the allocation.6665    LibFunc LibraryFunctionId = NotLibFunc;6666 6667    /// The status wrt. a rewrite.6668    enum {6669      STACK_DUE_TO_USE,6670      STACK_DUE_TO_FREE,6671      INVALID,6672    } Status = STACK_DUE_TO_USE;6673 6674    /// Flag to indicate if we encountered a use that might free this allocation6675    /// but which is not in the deallocation infos.6676    bool HasPotentiallyFreeingUnknownUses = false;6677 6678    /// Flag to indicate that we should place the new alloca in the function6679    /// entry block rather than where the call site (CB) is.6680    bool MoveAllocaIntoEntry = true;6681 6682    /// The set of free calls that use this allocation.6683    SmallSetVector<CallBase *, 1> PotentialFreeCalls{};6684  };6685 6686  struct DeallocationInfo {6687    /// The call that deallocates the memory.6688    CallBase *const CB;6689    /// The value freed by the call.6690    Value *FreedOp;6691 6692    /// Flag to indicate if we don't know all objects this deallocation might6693    /// free.6694    bool MightFreeUnknownObjects = false;6695 6696    /// The set of allocation calls that are potentially freed.6697    SmallSetVector<CallBase *, 1> PotentialAllocationCalls{};6698  };6699 6700  AAHeapToStackFunction(const IRPosition &IRP, Attributor &A)6701      : AAHeapToStack(IRP, A) {}6702 6703  ~AAHeapToStackFunction() override {6704    // Ensure we call the destructor so we release any memory allocated in the6705    // sets.6706    for (auto &It : AllocationInfos)6707      It.second->~AllocationInfo();6708    for (auto &It : DeallocationInfos)6709      It.second->~DeallocationInfo();6710  }6711 6712  void initialize(Attributor &A) override {6713    AAHeapToStack::initialize(A);6714 6715    const Function *F = getAnchorScope();6716    const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);6717 6718    auto AllocationIdentifierCB = [&](Instruction &I) {6719      CallBase *CB = dyn_cast<CallBase>(&I);6720      if (!CB)6721        return true;6722      if (Value *FreedOp = getFreedOperand(CB, TLI)) {6723        DeallocationInfos[CB] = new (A.Allocator) DeallocationInfo{CB, FreedOp};6724        return true;6725      }6726      // To do heap to stack, we need to know that the allocation itself is6727      // removable once uses are rewritten, and that we can initialize the6728      // alloca to the same pattern as the original allocation result.6729      if (isRemovableAlloc(CB, TLI)) {6730        auto *I8Ty = Type::getInt8Ty(CB->getParent()->getContext());6731        if (nullptr != getInitialValueOfAllocation(CB, TLI, I8Ty)) {6732          AllocationInfo *AI = new (A.Allocator) AllocationInfo{CB};6733          AllocationInfos[CB] = AI;6734          if (TLI)6735            TLI->getLibFunc(*CB, AI->LibraryFunctionId);6736        }6737      }6738      return true;6739    };6740 6741    bool UsedAssumedInformation = false;6742    bool Success = A.checkForAllCallLikeInstructions(6743        AllocationIdentifierCB, *this, UsedAssumedInformation,6744        /* CheckBBLivenessOnly */ false,6745        /* CheckPotentiallyDead */ true);6746    (void)Success;6747    assert(Success && "Did not expect the call base visit callback to fail!");6748 6749    Attributor::SimplifictionCallbackTy SCB =6750        [](const IRPosition &, const AbstractAttribute *,6751           bool &) -> std::optional<Value *> { return nullptr; };6752    for (const auto &It : AllocationInfos)6753      A.registerSimplificationCallback(IRPosition::callsite_returned(*It.first),6754                                       SCB);6755    for (const auto &It : DeallocationInfos)6756      A.registerSimplificationCallback(IRPosition::callsite_returned(*It.first),6757                                       SCB);6758  }6759 6760  const std::string getAsStr(Attributor *A) const override {6761    unsigned NumH2SMallocs = 0, NumInvalidMallocs = 0;6762    for (const auto &It : AllocationInfos) {6763      if (It.second->Status == AllocationInfo::INVALID)6764        ++NumInvalidMallocs;6765      else6766        ++NumH2SMallocs;6767    }6768    return "[H2S] Mallocs Good/Bad: " + std::to_string(NumH2SMallocs) + "/" +6769           std::to_string(NumInvalidMallocs);6770  }6771 6772  /// See AbstractAttribute::trackStatistics().6773  void trackStatistics() const override {6774    STATS_DECL(6775        MallocCalls, Function,6776        "Number of malloc/calloc/aligned_alloc calls converted to allocas");6777    for (const auto &It : AllocationInfos)6778      if (It.second->Status != AllocationInfo::INVALID)6779        ++BUILD_STAT_NAME(MallocCalls, Function);6780  }6781 6782  bool isAssumedHeapToStack(const CallBase &CB) const override {6783    if (isValidState())6784      if (AllocationInfo *AI =6785              AllocationInfos.lookup(const_cast<CallBase *>(&CB)))6786        return AI->Status != AllocationInfo::INVALID;6787    return false;6788  }6789 6790  bool isAssumedHeapToStackRemovedFree(CallBase &CB) const override {6791    if (!isValidState())6792      return false;6793 6794    for (const auto &It : AllocationInfos) {6795      AllocationInfo &AI = *It.second;6796      if (AI.Status == AllocationInfo::INVALID)6797        continue;6798 6799      if (AI.PotentialFreeCalls.count(&CB))6800        return true;6801    }6802 6803    return false;6804  }6805 6806  ChangeStatus manifest(Attributor &A) override {6807    assert(getState().isValidState() &&6808           "Attempted to manifest an invalid state!");6809 6810    ChangeStatus HasChanged = ChangeStatus::UNCHANGED;6811    Function *F = getAnchorScope();6812    const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);6813 6814    for (auto &It : AllocationInfos) {6815      AllocationInfo &AI = *It.second;6816      if (AI.Status == AllocationInfo::INVALID)6817        continue;6818 6819      for (CallBase *FreeCall : AI.PotentialFreeCalls) {6820        LLVM_DEBUG(dbgs() << "H2S: Removing free call: " << *FreeCall << "\n");6821        A.deleteAfterManifest(*FreeCall);6822        HasChanged = ChangeStatus::CHANGED;6823      }6824 6825      LLVM_DEBUG(dbgs() << "H2S: Removing malloc-like call: " << *AI.CB6826                        << "\n");6827 6828      auto Remark = [&](OptimizationRemark OR) {6829        LibFunc IsAllocShared;6830        if (TLI->getLibFunc(*AI.CB, IsAllocShared))6831          if (IsAllocShared == LibFunc___kmpc_alloc_shared)6832            return OR << "Moving globalized variable to the stack.";6833        return OR << "Moving memory allocation from the heap to the stack.";6834      };6835      if (AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared)6836        A.emitRemark<OptimizationRemark>(AI.CB, "OMP110", Remark);6837      else6838        A.emitRemark<OptimizationRemark>(AI.CB, "HeapToStack", Remark);6839 6840      const DataLayout &DL = A.getInfoCache().getDL();6841      Value *Size;6842      std::optional<APInt> SizeAPI = getSize(A, *this, AI);6843      if (SizeAPI) {6844        Size = ConstantInt::get(AI.CB->getContext(), *SizeAPI);6845      } else {6846        LLVMContext &Ctx = AI.CB->getContext();6847        ObjectSizeOpts Opts;6848        ObjectSizeOffsetEvaluator Eval(DL, TLI, Ctx, Opts);6849        SizeOffsetValue SizeOffsetPair = Eval.compute(AI.CB);6850        assert(SizeOffsetPair != ObjectSizeOffsetEvaluator::unknown() &&6851               cast<ConstantInt>(SizeOffsetPair.Offset)->isZero());6852        Size = SizeOffsetPair.Size;6853      }6854 6855      BasicBlock::iterator IP = AI.MoveAllocaIntoEntry6856                                    ? F->getEntryBlock().begin()6857                                    : AI.CB->getIterator();6858 6859      Align Alignment(1);6860      if (MaybeAlign RetAlign = AI.CB->getRetAlign())6861        Alignment = std::max(Alignment, *RetAlign);6862      if (Value *Align = getAllocAlignment(AI.CB, TLI)) {6863        std::optional<APInt> AlignmentAPI = getAPInt(A, *this, *Align);6864        assert(AlignmentAPI && AlignmentAPI->getZExtValue() > 0 &&6865               "Expected an alignment during manifest!");6866        Alignment =6867            std::max(Alignment, assumeAligned(AlignmentAPI->getZExtValue()));6868      }6869 6870      // TODO: Hoist the alloca towards the function entry.6871      unsigned AS = DL.getAllocaAddrSpace();6872      Instruction *Alloca =6873          new AllocaInst(Type::getInt8Ty(F->getContext()), AS, Size, Alignment,6874                         AI.CB->getName() + ".h2s", IP);6875 6876      if (Alloca->getType() != AI.CB->getType())6877        Alloca = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(6878            Alloca, AI.CB->getType(), "malloc_cast", AI.CB->getIterator());6879 6880      auto *I8Ty = Type::getInt8Ty(F->getContext());6881      auto *InitVal = getInitialValueOfAllocation(AI.CB, TLI, I8Ty);6882      assert(InitVal &&6883             "Must be able to materialize initial memory state of allocation");6884 6885      A.changeAfterManifest(IRPosition::inst(*AI.CB), *Alloca);6886 6887      if (auto *II = dyn_cast<InvokeInst>(AI.CB)) {6888        auto *NBB = II->getNormalDest();6889        BranchInst::Create(NBB, AI.CB->getParent());6890        A.deleteAfterManifest(*AI.CB);6891      } else {6892        A.deleteAfterManifest(*AI.CB);6893      }6894 6895      // Initialize the alloca with the same value as used by the allocation6896      // function.  We can skip undef as the initial value of an alloc is6897      // undef, and the memset would simply end up being DSEd.6898      if (!isa<UndefValue>(InitVal)) {6899        IRBuilder<> Builder(Alloca->getNextNode());6900        // TODO: Use alignment above if align!=16901        Builder.CreateMemSet(Alloca, InitVal, Size, std::nullopt);6902      }6903      HasChanged = ChangeStatus::CHANGED;6904    }6905 6906    return HasChanged;6907  }6908 6909  std::optional<APInt> getAPInt(Attributor &A, const AbstractAttribute &AA,6910                                Value &V) {6911    bool UsedAssumedInformation = false;6912    std::optional<Constant *> SimpleV =6913        A.getAssumedConstant(V, AA, UsedAssumedInformation);6914    if (!SimpleV)6915      return APInt(64, 0);6916    if (auto *CI = dyn_cast_or_null<ConstantInt>(*SimpleV))6917      return CI->getValue();6918    return std::nullopt;6919  }6920 6921  std::optional<APInt> getSize(Attributor &A, const AbstractAttribute &AA,6922                               AllocationInfo &AI) {6923    auto Mapper = [&](const Value *V) -> const Value * {6924      bool UsedAssumedInformation = false;6925      if (std::optional<Constant *> SimpleV =6926              A.getAssumedConstant(*V, AA, UsedAssumedInformation))6927        if (*SimpleV)6928          return *SimpleV;6929      return V;6930    };6931 6932    const Function *F = getAnchorScope();6933    const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);6934    return getAllocSize(AI.CB, TLI, Mapper);6935  }6936 6937  /// Collection of all malloc-like calls in a function with associated6938  /// information.6939  MapVector<CallBase *, AllocationInfo *> AllocationInfos;6940 6941  /// Collection of all free-like calls in a function with associated6942  /// information.6943  MapVector<CallBase *, DeallocationInfo *> DeallocationInfos;6944 6945  ChangeStatus updateImpl(Attributor &A) override;6946};6947 6948ChangeStatus AAHeapToStackFunction::updateImpl(Attributor &A) {6949  ChangeStatus Changed = ChangeStatus::UNCHANGED;6950  const Function *F = getAnchorScope();6951  const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);6952 6953  const auto *LivenessAA =6954      A.getAAFor<AAIsDead>(*this, IRPosition::function(*F), DepClassTy::NONE);6955 6956  MustBeExecutedContextExplorer *Explorer =6957      A.getInfoCache().getMustBeExecutedContextExplorer();6958 6959  bool StackIsAccessibleByOtherThreads =6960      A.getInfoCache().stackIsAccessibleByOtherThreads();6961 6962  LoopInfo *LI =6963      A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(*F);6964  std::optional<bool> MayContainIrreducibleControl;6965  auto IsInLoop = [&](BasicBlock &BB) {6966    if (&F->getEntryBlock() == &BB)6967      return false;6968    if (!MayContainIrreducibleControl.has_value())6969      MayContainIrreducibleControl = mayContainIrreducibleControl(*F, LI);6970    if (*MayContainIrreducibleControl)6971      return true;6972    if (!LI)6973      return true;6974    return LI->getLoopFor(&BB) != nullptr;6975  };6976 6977  // Flag to ensure we update our deallocation information at most once per6978  // updateImpl call and only if we use the free check reasoning.6979  bool HasUpdatedFrees = false;6980 6981  auto UpdateFrees = [&]() {6982    HasUpdatedFrees = true;6983 6984    for (auto &It : DeallocationInfos) {6985      DeallocationInfo &DI = *It.second;6986      // For now we cannot use deallocations that have unknown inputs, skip6987      // them.6988      if (DI.MightFreeUnknownObjects)6989        continue;6990 6991      // No need to analyze dead calls, ignore them instead.6992      bool UsedAssumedInformation = false;6993      if (A.isAssumedDead(*DI.CB, this, LivenessAA, UsedAssumedInformation,6994                          /* CheckBBLivenessOnly */ true))6995        continue;6996 6997      // Use the non-optimistic version to get the freed object.6998      Value *Obj = getUnderlyingObject(DI.FreedOp);6999      if (!Obj) {7000        LLVM_DEBUG(dbgs() << "[H2S] Unknown underlying object for free!\n");7001        DI.MightFreeUnknownObjects = true;7002        continue;7003      }7004 7005      // Free of null and undef can be ignored as no-ops (or UB in the latter7006      // case).7007      if (isa<ConstantPointerNull>(Obj) || isa<UndefValue>(Obj))7008        continue;7009 7010      CallBase *ObjCB = dyn_cast<CallBase>(Obj);7011      if (!ObjCB) {7012        LLVM_DEBUG(dbgs() << "[H2S] Free of a non-call object: " << *Obj7013                          << "\n");7014        DI.MightFreeUnknownObjects = true;7015        continue;7016      }7017 7018      AllocationInfo *AI = AllocationInfos.lookup(ObjCB);7019      if (!AI) {7020        LLVM_DEBUG(dbgs() << "[H2S] Free of a non-allocation object: " << *Obj7021                          << "\n");7022        DI.MightFreeUnknownObjects = true;7023        continue;7024      }7025 7026      DI.PotentialAllocationCalls.insert(ObjCB);7027    }7028  };7029 7030  auto FreeCheck = [&](AllocationInfo &AI) {7031    // If the stack is not accessible by other threads, the "must-free" logic7032    // doesn't apply as the pointer could be shared and needs to be places in7033    // "shareable" memory.7034    if (!StackIsAccessibleByOtherThreads) {7035      bool IsKnownNoSycn;7036      if (!AA::hasAssumedIRAttr<Attribute::NoSync>(7037              A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNoSycn)) {7038        LLVM_DEBUG(7039            dbgs() << "[H2S] found an escaping use, stack is not accessible by "7040                      "other threads and function is not nosync:\n");7041        return false;7042      }7043    }7044    if (!HasUpdatedFrees)7045      UpdateFrees();7046 7047    // TODO: Allow multi exit functions that have different free calls.7048    if (AI.PotentialFreeCalls.size() != 1) {7049      LLVM_DEBUG(dbgs() << "[H2S] did not find one free call but "7050                        << AI.PotentialFreeCalls.size() << "\n");7051      return false;7052    }7053    CallBase *UniqueFree = *AI.PotentialFreeCalls.begin();7054    DeallocationInfo *DI = DeallocationInfos.lookup(UniqueFree);7055    if (!DI) {7056      LLVM_DEBUG(7057          dbgs() << "[H2S] unique free call was not known as deallocation call "7058                 << *UniqueFree << "\n");7059      return false;7060    }7061    if (DI->MightFreeUnknownObjects) {7062      LLVM_DEBUG(7063          dbgs() << "[H2S] unique free call might free unknown allocations\n");7064      return false;7065    }7066    if (DI->PotentialAllocationCalls.empty())7067      return true;7068    if (DI->PotentialAllocationCalls.size() > 1) {7069      LLVM_DEBUG(dbgs() << "[H2S] unique free call might free "7070                        << DI->PotentialAllocationCalls.size()7071                        << " different allocations\n");7072      return false;7073    }7074    if (*DI->PotentialAllocationCalls.begin() != AI.CB) {7075      LLVM_DEBUG(7076          dbgs()7077          << "[H2S] unique free call not known to free this allocation but "7078          << **DI->PotentialAllocationCalls.begin() << "\n");7079      return false;7080    }7081 7082    // __kmpc_alloc_shared and __kmpc_alloc_free are by construction matched.7083    if (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared) {7084      Instruction *CtxI = isa<InvokeInst>(AI.CB) ? AI.CB : AI.CB->getNextNode();7085      if (!Explorer || !Explorer->findInContextOf(UniqueFree, CtxI)) {7086        LLVM_DEBUG(dbgs() << "[H2S] unique free call might not be executed "7087                             "with the allocation "7088                          << *UniqueFree << "\n");7089        return false;7090      }7091    }7092    return true;7093  };7094 7095  auto UsesCheck = [&](AllocationInfo &AI) {7096    bool ValidUsesOnly = true;7097 7098    auto Pred = [&](const Use &U, bool &Follow) -> bool {7099      Instruction *UserI = cast<Instruction>(U.getUser());7100      if (isa<LoadInst>(UserI))7101        return true;7102      if (auto *SI = dyn_cast<StoreInst>(UserI)) {7103        if (SI->getValueOperand() == U.get()) {7104          LLVM_DEBUG(dbgs()7105                     << "[H2S] escaping store to memory: " << *UserI << "\n");7106          ValidUsesOnly = false;7107        } else {7108          // A store into the malloc'ed memory is fine.7109        }7110        return true;7111      }7112      if (auto *CB = dyn_cast<CallBase>(UserI)) {7113        if (!CB->isArgOperand(&U) || CB->isLifetimeStartOrEnd())7114          return true;7115        if (DeallocationInfos.count(CB)) {7116          AI.PotentialFreeCalls.insert(CB);7117          return true;7118        }7119 7120        unsigned ArgNo = CB->getArgOperandNo(&U);7121        auto CBIRP = IRPosition::callsite_argument(*CB, ArgNo);7122 7123        bool IsKnownNoCapture;7124        bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(7125            A, this, CBIRP, DepClassTy::OPTIONAL, IsKnownNoCapture);7126 7127        // If a call site argument use is nofree, we are fine.7128        bool IsKnownNoFree;7129        bool IsAssumedNoFree = AA::hasAssumedIRAttr<Attribute::NoFree>(7130            A, this, CBIRP, DepClassTy::OPTIONAL, IsKnownNoFree);7131 7132        if (!IsAssumedNoCapture ||7133            (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared &&7134             !IsAssumedNoFree)) {7135          AI.HasPotentiallyFreeingUnknownUses |= !IsAssumedNoFree;7136 7137          // Emit a missed remark if this is missed OpenMP globalization.7138          auto Remark = [&](OptimizationRemarkMissed ORM) {7139            return ORM7140                   << "Could not move globalized variable to the stack. "7141                      "Variable is potentially captured in call. Mark "7142                      "parameter as `__attribute__((noescape))` to override.";7143          };7144 7145          if (ValidUsesOnly &&7146              AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared)7147            A.emitRemark<OptimizationRemarkMissed>(CB, "OMP113", Remark);7148 7149          LLVM_DEBUG(dbgs() << "[H2S] Bad user: " << *UserI << "\n");7150          ValidUsesOnly = false;7151        }7152        return true;7153      }7154 7155      if (isa<GetElementPtrInst>(UserI) || isa<BitCastInst>(UserI) ||7156          isa<PHINode>(UserI) || isa<SelectInst>(UserI)) {7157        Follow = true;7158        return true;7159      }7160      // Unknown user for which we can not track uses further (in a way that7161      // makes sense).7162      LLVM_DEBUG(dbgs() << "[H2S] Unknown user: " << *UserI << "\n");7163      ValidUsesOnly = false;7164      return true;7165    };7166    if (!A.checkForAllUses(Pred, *this, *AI.CB, /* CheckBBLivenessOnly */ false,7167                           DepClassTy::OPTIONAL, /* IgnoreDroppableUses */ true,7168                           [&](const Use &OldU, const Use &NewU) {7169                             auto *SI = dyn_cast<StoreInst>(OldU.getUser());7170                             return !SI || StackIsAccessibleByOtherThreads ||7171                                    AA::isAssumedThreadLocalObject(7172                                        A, *SI->getPointerOperand(), *this);7173                           }))7174      return false;7175    return ValidUsesOnly;7176  };7177 7178  // The actual update starts here. We look at all allocations and depending on7179  // their status perform the appropriate check(s).7180  for (auto &It : AllocationInfos) {7181    AllocationInfo &AI = *It.second;7182    if (AI.Status == AllocationInfo::INVALID)7183      continue;7184 7185    if (Value *Align = getAllocAlignment(AI.CB, TLI)) {7186      std::optional<APInt> APAlign = getAPInt(A, *this, *Align);7187      if (!APAlign) {7188        // Can't generate an alloca which respects the required alignment7189        // on the allocation.7190        LLVM_DEBUG(dbgs() << "[H2S] Unknown allocation alignment: " << *AI.CB7191                          << "\n");7192        AI.Status = AllocationInfo::INVALID;7193        Changed = ChangeStatus::CHANGED;7194        continue;7195      }7196      if (APAlign->ugt(llvm::Value::MaximumAlignment) ||7197          !APAlign->isPowerOf2()) {7198        LLVM_DEBUG(dbgs() << "[H2S] Invalid allocation alignment: " << APAlign7199                          << "\n");7200        AI.Status = AllocationInfo::INVALID;7201        Changed = ChangeStatus::CHANGED;7202        continue;7203      }7204    }7205 7206    std::optional<APInt> Size = getSize(A, *this, AI);7207    if (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared &&7208        MaxHeapToStackSize != -1) {7209      if (!Size || Size->ugt(MaxHeapToStackSize)) {7210        LLVM_DEBUG({7211          if (!Size)7212            dbgs() << "[H2S] Unknown allocation size: " << *AI.CB << "\n";7213          else7214            dbgs() << "[H2S] Allocation size too large: " << *AI.CB << " vs. "7215                   << MaxHeapToStackSize << "\n";7216        });7217 7218        AI.Status = AllocationInfo::INVALID;7219        Changed = ChangeStatus::CHANGED;7220        continue;7221      }7222    }7223 7224    switch (AI.Status) {7225    case AllocationInfo::STACK_DUE_TO_USE:7226      if (UsesCheck(AI))7227        break;7228      AI.Status = AllocationInfo::STACK_DUE_TO_FREE;7229      [[fallthrough]];7230    case AllocationInfo::STACK_DUE_TO_FREE:7231      if (FreeCheck(AI))7232        break;7233      AI.Status = AllocationInfo::INVALID;7234      Changed = ChangeStatus::CHANGED;7235      break;7236    case AllocationInfo::INVALID:7237      llvm_unreachable("Invalid allocations should never reach this point!");7238    };7239 7240    // Check if we still think we can move it into the entry block. If the7241    // alloca comes from a converted __kmpc_alloc_shared then we can usually7242    // ignore the potential compilations associated with loops.7243    bool IsGlobalizedLocal =7244        AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared;7245    if (AI.MoveAllocaIntoEntry &&7246        (!Size.has_value() ||7247         (!IsGlobalizedLocal && IsInLoop(*AI.CB->getParent()))))7248      AI.MoveAllocaIntoEntry = false;7249  }7250 7251  return Changed;7252}7253} // namespace7254 7255/// ----------------------- Privatizable Pointers ------------------------------7256namespace {7257struct AAPrivatizablePtrImpl : public AAPrivatizablePtr {7258  AAPrivatizablePtrImpl(const IRPosition &IRP, Attributor &A)7259      : AAPrivatizablePtr(IRP, A), PrivatizableType(std::nullopt) {}7260 7261  ChangeStatus indicatePessimisticFixpoint() override {7262    AAPrivatizablePtr::indicatePessimisticFixpoint();7263    PrivatizableType = nullptr;7264    return ChangeStatus::CHANGED;7265  }7266 7267  /// Identify the type we can chose for a private copy of the underlying7268  /// argument. std::nullopt means it is not clear yet, nullptr means there is7269  /// none.7270  virtual std::optional<Type *> identifyPrivatizableType(Attributor &A) = 0;7271 7272  /// Return a privatizable type that encloses both T0 and T1.7273  /// TODO: This is merely a stub for now as we should manage a mapping as well.7274  std::optional<Type *> combineTypes(std::optional<Type *> T0,7275                                     std::optional<Type *> T1) {7276    if (!T0)7277      return T1;7278    if (!T1)7279      return T0;7280    if (T0 == T1)7281      return T0;7282    return nullptr;7283  }7284 7285  std::optional<Type *> getPrivatizableType() const override {7286    return PrivatizableType;7287  }7288 7289  const std::string getAsStr(Attributor *A) const override {7290    return isAssumedPrivatizablePtr() ? "[priv]" : "[no-priv]";7291  }7292 7293protected:7294  std::optional<Type *> PrivatizableType;7295};7296 7297// TODO: Do this for call site arguments (probably also other values) as well.7298 7299struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl {7300  AAPrivatizablePtrArgument(const IRPosition &IRP, Attributor &A)7301      : AAPrivatizablePtrImpl(IRP, A) {}7302 7303  /// See AAPrivatizablePtrImpl::identifyPrivatizableType(...)7304  std::optional<Type *> identifyPrivatizableType(Attributor &A) override {7305    // If this is a byval argument and we know all the call sites (so we can7306    // rewrite them), there is no need to check them explicitly.7307    bool UsedAssumedInformation = false;7308    SmallVector<Attribute, 1> Attrs;7309    A.getAttrs(getIRPosition(), {Attribute::ByVal}, Attrs,7310               /* IgnoreSubsumingPositions */ true);7311    if (!Attrs.empty() &&7312        A.checkForAllCallSites([](AbstractCallSite ACS) { return true; }, *this,7313                               true, UsedAssumedInformation))7314      return Attrs[0].getValueAsType();7315 7316    std::optional<Type *> Ty;7317    unsigned ArgNo = getIRPosition().getCallSiteArgNo();7318 7319    // Make sure the associated call site argument has the same type at all call7320    // sites and it is an allocation we know is safe to privatize, for now that7321    // means we only allow alloca instructions.7322    // TODO: We can additionally analyze the accesses in the callee to  create7323    //       the type from that information instead. That is a little more7324    //       involved and will be done in a follow up patch.7325    auto CallSiteCheck = [&](AbstractCallSite ACS) {7326      IRPosition ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);7327      // Check if a coresponding argument was found or if it is one not7328      // associated (which can happen for callback calls).7329      if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)7330        return false;7331 7332      // Check that all call sites agree on a type.7333      auto *PrivCSArgAA =7334          A.getAAFor<AAPrivatizablePtr>(*this, ACSArgPos, DepClassTy::REQUIRED);7335      if (!PrivCSArgAA)7336        return false;7337      std::optional<Type *> CSTy = PrivCSArgAA->getPrivatizableType();7338 7339      LLVM_DEBUG({7340        dbgs() << "[AAPrivatizablePtr] ACSPos: " << ACSArgPos << ", CSTy: ";7341        if (CSTy && *CSTy)7342          (*CSTy)->print(dbgs());7343        else if (CSTy)7344          dbgs() << "<nullptr>";7345        else7346          dbgs() << "<none>";7347      });7348 7349      Ty = combineTypes(Ty, CSTy);7350 7351      LLVM_DEBUG({7352        dbgs() << " : New Type: ";7353        if (Ty && *Ty)7354          (*Ty)->print(dbgs());7355        else if (Ty)7356          dbgs() << "<nullptr>";7357        else7358          dbgs() << "<none>";7359        dbgs() << "\n";7360      });7361 7362      return !Ty || *Ty;7363    };7364 7365    if (!A.checkForAllCallSites(CallSiteCheck, *this, true,7366                                UsedAssumedInformation))7367      return nullptr;7368    return Ty;7369  }7370 7371  /// See AbstractAttribute::updateImpl(...).7372  ChangeStatus updateImpl(Attributor &A) override {7373    PrivatizableType = identifyPrivatizableType(A);7374    if (!PrivatizableType)7375      return ChangeStatus::UNCHANGED;7376    if (!*PrivatizableType)7377      return indicatePessimisticFixpoint();7378 7379    // The dependence is optional so we don't give up once we give up on the7380    // alignment.7381    A.getAAFor<AAAlign>(*this, IRPosition::value(getAssociatedValue()),7382                        DepClassTy::OPTIONAL);7383 7384    // Avoid arguments with padding for now.7385    if (!A.hasAttr(getIRPosition(), Attribute::ByVal) &&7386        !isDenselyPacked(*PrivatizableType, A.getInfoCache().getDL())) {7387      LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Padding detected\n");7388      return indicatePessimisticFixpoint();7389    }7390 7391    // Collect the types that will replace the privatizable type in the function7392    // signature.7393    SmallVector<Type *, 16> ReplacementTypes;7394    identifyReplacementTypes(*PrivatizableType, ReplacementTypes);7395 7396    // Verify callee and caller agree on how the promoted argument would be7397    // passed.7398    Function &Fn = *getIRPosition().getAnchorScope();7399    const auto *TTI =7400        A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(Fn);7401    if (!TTI) {7402      LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Missing TTI for function "7403                        << Fn.getName() << "\n");7404      return indicatePessimisticFixpoint();7405    }7406 7407    auto CallSiteCheck = [&](AbstractCallSite ACS) {7408      CallBase *CB = ACS.getInstruction();7409      return TTI->areTypesABICompatible(7410          CB->getCaller(),7411          dyn_cast_if_present<Function>(CB->getCalledOperand()),7412          ReplacementTypes);7413    };7414    bool UsedAssumedInformation = false;7415    if (!A.checkForAllCallSites(CallSiteCheck, *this, true,7416                                UsedAssumedInformation)) {7417      LLVM_DEBUG(7418          dbgs() << "[AAPrivatizablePtr] ABI incompatibility detected for "7419                 << Fn.getName() << "\n");7420      return indicatePessimisticFixpoint();7421    }7422 7423    // Register a rewrite of the argument.7424    Argument *Arg = getAssociatedArgument();7425    if (!A.isValidFunctionSignatureRewrite(*Arg, ReplacementTypes)) {7426      LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Rewrite not valid\n");7427      return indicatePessimisticFixpoint();7428    }7429 7430    unsigned ArgNo = Arg->getArgNo();7431 7432    // Helper to check if for the given call site the associated argument is7433    // passed to a callback where the privatization would be different.7434    auto IsCompatiblePrivArgOfCallback = [&](CallBase &CB) {7435      SmallVector<const Use *, 4> CallbackUses;7436      AbstractCallSite::getCallbackUses(CB, CallbackUses);7437      for (const Use *U : CallbackUses) {7438        AbstractCallSite CBACS(U);7439        assert(CBACS && CBACS.isCallbackCall());7440        for (Argument &CBArg : CBACS.getCalledFunction()->args()) {7441          int CBArgNo = CBACS.getCallArgOperandNo(CBArg);7442 7443          LLVM_DEBUG({7444            dbgs()7445                << "[AAPrivatizablePtr] Argument " << *Arg7446                << "check if can be privatized in the context of its parent ("7447                << Arg->getParent()->getName()7448                << ")\n[AAPrivatizablePtr] because it is an argument in a "7449                   "callback ("7450                << CBArgNo << "@" << CBACS.getCalledFunction()->getName()7451                << ")\n[AAPrivatizablePtr] " << CBArg << " : "7452                << CBACS.getCallArgOperand(CBArg) << " vs "7453                << CB.getArgOperand(ArgNo) << "\n"7454                << "[AAPrivatizablePtr] " << CBArg << " : "7455                << CBACS.getCallArgOperandNo(CBArg) << " vs " << ArgNo << "\n";7456          });7457 7458          if (CBArgNo != int(ArgNo))7459            continue;7460          const auto *CBArgPrivAA = A.getAAFor<AAPrivatizablePtr>(7461              *this, IRPosition::argument(CBArg), DepClassTy::REQUIRED);7462          if (CBArgPrivAA && CBArgPrivAA->isValidState()) {7463            auto CBArgPrivTy = CBArgPrivAA->getPrivatizableType();7464            if (!CBArgPrivTy)7465              continue;7466            if (*CBArgPrivTy == PrivatizableType)7467              continue;7468          }7469 7470          LLVM_DEBUG({7471            dbgs() << "[AAPrivatizablePtr] Argument " << *Arg7472                   << " cannot be privatized in the context of its parent ("7473                   << Arg->getParent()->getName()7474                   << ")\n[AAPrivatizablePtr] because it is an argument in a "7475                      "callback ("7476                   << CBArgNo << "@" << CBACS.getCalledFunction()->getName()7477                   << ").\n[AAPrivatizablePtr] for which the argument "7478                      "privatization is not compatible.\n";7479          });7480          return false;7481        }7482      }7483      return true;7484    };7485 7486    // Helper to check if for the given call site the associated argument is7487    // passed to a direct call where the privatization would be different.7488    auto IsCompatiblePrivArgOfDirectCS = [&](AbstractCallSite ACS) {7489      CallBase *DC = cast<CallBase>(ACS.getInstruction());7490      int DCArgNo = ACS.getCallArgOperandNo(ArgNo);7491      assert(DCArgNo >= 0 && unsigned(DCArgNo) < DC->arg_size() &&7492             "Expected a direct call operand for callback call operand");7493 7494      Function *DCCallee =7495          dyn_cast_if_present<Function>(DC->getCalledOperand());7496      LLVM_DEBUG({7497        dbgs() << "[AAPrivatizablePtr] Argument " << *Arg7498               << " check if be privatized in the context of its parent ("7499               << Arg->getParent()->getName()7500               << ")\n[AAPrivatizablePtr] because it is an argument in a "7501                  "direct call of ("7502               << DCArgNo << "@" << DCCallee->getName() << ").\n";7503      });7504 7505      if (unsigned(DCArgNo) < DCCallee->arg_size()) {7506        const auto *DCArgPrivAA = A.getAAFor<AAPrivatizablePtr>(7507            *this, IRPosition::argument(*DCCallee->getArg(DCArgNo)),7508            DepClassTy::REQUIRED);7509        if (DCArgPrivAA && DCArgPrivAA->isValidState()) {7510          auto DCArgPrivTy = DCArgPrivAA->getPrivatizableType();7511          if (!DCArgPrivTy)7512            return true;7513          if (*DCArgPrivTy == PrivatizableType)7514            return true;7515        }7516      }7517 7518      LLVM_DEBUG({7519        dbgs() << "[AAPrivatizablePtr] Argument " << *Arg7520               << " cannot be privatized in the context of its parent ("7521               << Arg->getParent()->getName()7522               << ")\n[AAPrivatizablePtr] because it is an argument in a "7523                  "direct call of ("7524               << ACS.getInstruction()->getCalledOperand()->getName()7525               << ").\n[AAPrivatizablePtr] for which the argument "7526                  "privatization is not compatible.\n";7527      });7528      return false;7529    };7530 7531    // Helper to check if the associated argument is used at the given abstract7532    // call site in a way that is incompatible with the privatization assumed7533    // here.7534    auto IsCompatiblePrivArgOfOtherCallSite = [&](AbstractCallSite ACS) {7535      if (ACS.isDirectCall())7536        return IsCompatiblePrivArgOfCallback(*ACS.getInstruction());7537      if (ACS.isCallbackCall())7538        return IsCompatiblePrivArgOfDirectCS(ACS);7539      return false;7540    };7541 7542    if (!A.checkForAllCallSites(IsCompatiblePrivArgOfOtherCallSite, *this, true,7543                                UsedAssumedInformation))7544      return indicatePessimisticFixpoint();7545 7546    return ChangeStatus::UNCHANGED;7547  }7548 7549  /// Given a type to private \p PrivType, collect the constituates (which are7550  /// used) in \p ReplacementTypes.7551  static void7552  identifyReplacementTypes(Type *PrivType,7553                           SmallVectorImpl<Type *> &ReplacementTypes) {7554    // TODO: For now we expand the privatization type to the fullest which can7555    //       lead to dead arguments that need to be removed later.7556    assert(PrivType && "Expected privatizable type!");7557 7558    // Traverse the type, extract constituate types on the outermost level.7559    if (auto *PrivStructType = dyn_cast<StructType>(PrivType)) {7560      for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++)7561        ReplacementTypes.push_back(PrivStructType->getElementType(u));7562    } else if (auto *PrivArrayType = dyn_cast<ArrayType>(PrivType)) {7563      ReplacementTypes.append(PrivArrayType->getNumElements(),7564                              PrivArrayType->getElementType());7565    } else {7566      ReplacementTypes.push_back(PrivType);7567    }7568  }7569 7570  /// Initialize \p Base according to the type \p PrivType at position \p IP.7571  /// The values needed are taken from the arguments of \p F starting at7572  /// position \p ArgNo.7573  static void createInitialization(Type *PrivType, Value &Base, Function &F,7574                                   unsigned ArgNo, BasicBlock::iterator IP) {7575    assert(PrivType && "Expected privatizable type!");7576 7577    IRBuilder<NoFolder> IRB(IP->getParent(), IP);7578    const DataLayout &DL = F.getDataLayout();7579 7580    // Traverse the type, build GEPs and stores.7581    if (auto *PrivStructType = dyn_cast<StructType>(PrivType)) {7582      const StructLayout *PrivStructLayout = DL.getStructLayout(PrivStructType);7583      for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {7584        Value *Ptr =7585            constructPointer(&Base, PrivStructLayout->getElementOffset(u), IRB);7586        new StoreInst(F.getArg(ArgNo + u), Ptr, IP);7587      }7588    } else if (auto *PrivArrayType = dyn_cast<ArrayType>(PrivType)) {7589      Type *PointeeTy = PrivArrayType->getElementType();7590      uint64_t PointeeTySize = DL.getTypeStoreSize(PointeeTy);7591      for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {7592        Value *Ptr = constructPointer(&Base, u * PointeeTySize, IRB);7593        new StoreInst(F.getArg(ArgNo + u), Ptr, IP);7594      }7595    } else {7596      new StoreInst(F.getArg(ArgNo), &Base, IP);7597    }7598  }7599 7600  /// Extract values from \p Base according to the type \p PrivType at the7601  /// call position \p ACS. The values are appended to \p ReplacementValues.7602  void createReplacementValues(Align Alignment, Type *PrivType,7603                               AbstractCallSite ACS, Value *Base,7604                               SmallVectorImpl<Value *> &ReplacementValues) {7605    assert(Base && "Expected base value!");7606    assert(PrivType && "Expected privatizable type!");7607    Instruction *IP = ACS.getInstruction();7608 7609    IRBuilder<NoFolder> IRB(IP);7610    const DataLayout &DL = IP->getDataLayout();7611 7612    // Traverse the type, build GEPs and loads.7613    if (auto *PrivStructType = dyn_cast<StructType>(PrivType)) {7614      const StructLayout *PrivStructLayout = DL.getStructLayout(PrivStructType);7615      for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {7616        Type *PointeeTy = PrivStructType->getElementType(u);7617        Value *Ptr =7618            constructPointer(Base, PrivStructLayout->getElementOffset(u), IRB);7619        LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());7620        L->setAlignment(Alignment);7621        ReplacementValues.push_back(L);7622      }7623    } else if (auto *PrivArrayType = dyn_cast<ArrayType>(PrivType)) {7624      Type *PointeeTy = PrivArrayType->getElementType();7625      uint64_t PointeeTySize = DL.getTypeStoreSize(PointeeTy);7626      for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {7627        Value *Ptr = constructPointer(Base, u * PointeeTySize, IRB);7628        LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());7629        L->setAlignment(Alignment);7630        ReplacementValues.push_back(L);7631      }7632    } else {7633      LoadInst *L = new LoadInst(PrivType, Base, "", IP->getIterator());7634      L->setAlignment(Alignment);7635      ReplacementValues.push_back(L);7636    }7637  }7638 7639  /// See AbstractAttribute::manifest(...)7640  ChangeStatus manifest(Attributor &A) override {7641    if (!PrivatizableType)7642      return ChangeStatus::UNCHANGED;7643    assert(*PrivatizableType && "Expected privatizable type!");7644 7645    // Collect all tail calls in the function as we cannot allow new allocas to7646    // escape into tail recursion.7647    // TODO: Be smarter about new allocas escaping into tail calls.7648    SmallVector<CallInst *, 16> TailCalls;7649    bool UsedAssumedInformation = false;7650    if (!A.checkForAllInstructions(7651            [&](Instruction &I) {7652              CallInst &CI = cast<CallInst>(I);7653              if (CI.isTailCall())7654                TailCalls.push_back(&CI);7655              return true;7656            },7657            *this, {Instruction::Call}, UsedAssumedInformation))7658      return ChangeStatus::UNCHANGED;7659 7660    Argument *Arg = getAssociatedArgument();7661    // Query AAAlign attribute for alignment of associated argument to7662    // determine the best alignment of loads.7663    const auto *AlignAA =7664        A.getAAFor<AAAlign>(*this, IRPosition::value(*Arg), DepClassTy::NONE);7665 7666    // Callback to repair the associated function. A new alloca is placed at the7667    // beginning and initialized with the values passed through arguments. The7668    // new alloca replaces the use of the old pointer argument.7669    Attributor::ArgumentReplacementInfo::CalleeRepairCBTy FnRepairCB =7670        [=](const Attributor::ArgumentReplacementInfo &ARI,7671            Function &ReplacementFn, Function::arg_iterator ArgIt) {7672          BasicBlock &EntryBB = ReplacementFn.getEntryBlock();7673          BasicBlock::iterator IP = EntryBB.getFirstInsertionPt();7674          const DataLayout &DL = IP->getDataLayout();7675          unsigned AS = DL.getAllocaAddrSpace();7676          Instruction *AI = new AllocaInst(*PrivatizableType, AS,7677                                           Arg->getName() + ".priv", IP);7678          createInitialization(*PrivatizableType, *AI, ReplacementFn,7679                               ArgIt->getArgNo(), IP);7680 7681          if (AI->getType() != Arg->getType())7682            AI = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(7683                AI, Arg->getType(), "", IP);7684          Arg->replaceAllUsesWith(AI);7685 7686          for (CallInst *CI : TailCalls)7687            CI->setTailCall(false);7688        };7689 7690    // Callback to repair a call site of the associated function. The elements7691    // of the privatizable type are loaded prior to the call and passed to the7692    // new function version.7693    Attributor::ArgumentReplacementInfo::ACSRepairCBTy ACSRepairCB =7694        [=](const Attributor::ArgumentReplacementInfo &ARI,7695            AbstractCallSite ACS, SmallVectorImpl<Value *> &NewArgOperands) {7696          // When no alignment is specified for the load instruction,7697          // natural alignment is assumed.7698          createReplacementValues(7699              AlignAA ? AlignAA->getAssumedAlign() : Align(0),7700              *PrivatizableType, ACS,7701              ACS.getCallArgOperand(ARI.getReplacedArg().getArgNo()),7702              NewArgOperands);7703        };7704 7705    // Collect the types that will replace the privatizable type in the function7706    // signature.7707    SmallVector<Type *, 16> ReplacementTypes;7708    identifyReplacementTypes(*PrivatizableType, ReplacementTypes);7709 7710    // Register a rewrite of the argument.7711    if (A.registerFunctionSignatureRewrite(*Arg, ReplacementTypes,7712                                           std::move(FnRepairCB),7713                                           std::move(ACSRepairCB)))7714      return ChangeStatus::CHANGED;7715    return ChangeStatus::UNCHANGED;7716  }7717 7718  /// See AbstractAttribute::trackStatistics()7719  void trackStatistics() const override {7720    STATS_DECLTRACK_ARG_ATTR(privatizable_ptr);7721  }7722};7723 7724struct AAPrivatizablePtrFloating : public AAPrivatizablePtrImpl {7725  AAPrivatizablePtrFloating(const IRPosition &IRP, Attributor &A)7726      : AAPrivatizablePtrImpl(IRP, A) {}7727 7728  /// See AbstractAttribute::initialize(...).7729  void initialize(Attributor &A) override {7730    // TODO: We can privatize more than arguments.7731    indicatePessimisticFixpoint();7732  }7733 7734  ChangeStatus updateImpl(Attributor &A) override {7735    llvm_unreachable("AAPrivatizablePtr(Floating|Returned|CallSiteReturned)::"7736                     "updateImpl will not be called");7737  }7738 7739  /// See AAPrivatizablePtrImpl::identifyPrivatizableType(...)7740  std::optional<Type *> identifyPrivatizableType(Attributor &A) override {7741    Value *Obj = getUnderlyingObject(&getAssociatedValue());7742    if (!Obj) {7743      LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] No underlying object found!\n");7744      return nullptr;7745    }7746 7747    if (auto *AI = dyn_cast<AllocaInst>(Obj))7748      if (auto *CI = dyn_cast<ConstantInt>(AI->getArraySize()))7749        if (CI->isOne())7750          return AI->getAllocatedType();7751    if (auto *Arg = dyn_cast<Argument>(Obj)) {7752      auto *PrivArgAA = A.getAAFor<AAPrivatizablePtr>(7753          *this, IRPosition::argument(*Arg), DepClassTy::REQUIRED);7754      if (PrivArgAA && PrivArgAA->isAssumedPrivatizablePtr())7755        return PrivArgAA->getPrivatizableType();7756    }7757 7758    LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Underlying object neither valid "7759                         "alloca nor privatizable argument: "7760                      << *Obj << "!\n");7761    return nullptr;7762  }7763 7764  /// See AbstractAttribute::trackStatistics()7765  void trackStatistics() const override {7766    STATS_DECLTRACK_FLOATING_ATTR(privatizable_ptr);7767  }7768};7769 7770struct AAPrivatizablePtrCallSiteArgument final7771    : public AAPrivatizablePtrFloating {7772  AAPrivatizablePtrCallSiteArgument(const IRPosition &IRP, Attributor &A)7773      : AAPrivatizablePtrFloating(IRP, A) {}7774 7775  /// See AbstractAttribute::initialize(...).7776  void initialize(Attributor &A) override {7777    if (A.hasAttr(getIRPosition(), Attribute::ByVal))7778      indicateOptimisticFixpoint();7779  }7780 7781  /// See AbstractAttribute::updateImpl(...).7782  ChangeStatus updateImpl(Attributor &A) override {7783    PrivatizableType = identifyPrivatizableType(A);7784    if (!PrivatizableType)7785      return ChangeStatus::UNCHANGED;7786    if (!*PrivatizableType)7787      return indicatePessimisticFixpoint();7788 7789    const IRPosition &IRP = getIRPosition();7790    bool IsKnownNoCapture;7791    bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(7792        A, this, IRP, DepClassTy::REQUIRED, IsKnownNoCapture);7793    if (!IsAssumedNoCapture) {7794      LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might be captured!\n");7795      return indicatePessimisticFixpoint();7796    }7797 7798    bool IsKnownNoAlias;7799    if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(7800            A, this, IRP, DepClassTy::REQUIRED, IsKnownNoAlias)) {7801      LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might alias!\n");7802      return indicatePessimisticFixpoint();7803    }7804 7805    bool IsKnown;7806    if (!AA::isAssumedReadOnly(A, IRP, *this, IsKnown)) {7807      LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer is written!\n");7808      return indicatePessimisticFixpoint();7809    }7810 7811    return ChangeStatus::UNCHANGED;7812  }7813 7814  /// See AbstractAttribute::trackStatistics()7815  void trackStatistics() const override {7816    STATS_DECLTRACK_CSARG_ATTR(privatizable_ptr);7817  }7818};7819 7820struct AAPrivatizablePtrCallSiteReturned final7821    : public AAPrivatizablePtrFloating {7822  AAPrivatizablePtrCallSiteReturned(const IRPosition &IRP, Attributor &A)7823      : AAPrivatizablePtrFloating(IRP, A) {}7824 7825  /// See AbstractAttribute::initialize(...).7826  void initialize(Attributor &A) override {7827    // TODO: We can privatize more than arguments.7828    indicatePessimisticFixpoint();7829  }7830 7831  /// See AbstractAttribute::trackStatistics()7832  void trackStatistics() const override {7833    STATS_DECLTRACK_CSRET_ATTR(privatizable_ptr);7834  }7835};7836 7837struct AAPrivatizablePtrReturned final : public AAPrivatizablePtrFloating {7838  AAPrivatizablePtrReturned(const IRPosition &IRP, Attributor &A)7839      : AAPrivatizablePtrFloating(IRP, A) {}7840 7841  /// See AbstractAttribute::initialize(...).7842  void initialize(Attributor &A) override {7843    // TODO: We can privatize more than arguments.7844    indicatePessimisticFixpoint();7845  }7846 7847  /// See AbstractAttribute::trackStatistics()7848  void trackStatistics() const override {7849    STATS_DECLTRACK_FNRET_ATTR(privatizable_ptr);7850  }7851};7852} // namespace7853 7854/// -------------------- Memory Behavior Attributes ----------------------------7855/// Includes read-none, read-only, and write-only.7856/// ----------------------------------------------------------------------------7857namespace {7858struct AAMemoryBehaviorImpl : public AAMemoryBehavior {7859  AAMemoryBehaviorImpl(const IRPosition &IRP, Attributor &A)7860      : AAMemoryBehavior(IRP, A) {}7861 7862  /// See AbstractAttribute::initialize(...).7863  void initialize(Attributor &A) override {7864    intersectAssumedBits(BEST_STATE);7865    getKnownStateFromValue(A, getIRPosition(), getState());7866    AAMemoryBehavior::initialize(A);7867  }7868 7869  /// Return the memory behavior information encoded in the IR for \p IRP.7870  static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,7871                                     BitIntegerState &State,7872                                     bool IgnoreSubsumingPositions = false) {7873    SmallVector<Attribute, 2> Attrs;7874    A.getAttrs(IRP, AttrKinds, Attrs, IgnoreSubsumingPositions);7875    for (const Attribute &Attr : Attrs) {7876      switch (Attr.getKindAsEnum()) {7877      case Attribute::ReadNone:7878        State.addKnownBits(NO_ACCESSES);7879        break;7880      case Attribute::ReadOnly:7881        State.addKnownBits(NO_WRITES);7882        break;7883      case Attribute::WriteOnly:7884        State.addKnownBits(NO_READS);7885        break;7886      default:7887        llvm_unreachable("Unexpected attribute!");7888      }7889    }7890 7891    if (auto *I = dyn_cast<Instruction>(&IRP.getAnchorValue())) {7892      if (!I->mayReadFromMemory())7893        State.addKnownBits(NO_READS);7894      if (!I->mayWriteToMemory())7895        State.addKnownBits(NO_WRITES);7896    }7897  }7898 7899  /// See AbstractAttribute::getDeducedAttributes(...).7900  void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,7901                            SmallVectorImpl<Attribute> &Attrs) const override {7902    assert(Attrs.size() == 0);7903    if (isAssumedReadNone())7904      Attrs.push_back(Attribute::get(Ctx, Attribute::ReadNone));7905    else if (isAssumedReadOnly())7906      Attrs.push_back(Attribute::get(Ctx, Attribute::ReadOnly));7907    else if (isAssumedWriteOnly())7908      Attrs.push_back(Attribute::get(Ctx, Attribute::WriteOnly));7909    assert(Attrs.size() <= 1);7910  }7911 7912  /// See AbstractAttribute::manifest(...).7913  ChangeStatus manifest(Attributor &A) override {7914    const IRPosition &IRP = getIRPosition();7915 7916    if (A.hasAttr(IRP, Attribute::ReadNone,7917                  /* IgnoreSubsumingPositions */ true))7918      return ChangeStatus::UNCHANGED;7919 7920    // Check if we would improve the existing attributes first.7921    SmallVector<Attribute, 4> DeducedAttrs;7922    getDeducedAttributes(A, IRP.getAnchorValue().getContext(), DeducedAttrs);7923    if (llvm::all_of(DeducedAttrs, [&](const Attribute &Attr) {7924          return A.hasAttr(IRP, Attr.getKindAsEnum(),7925                           /* IgnoreSubsumingPositions */ true);7926        }))7927      return ChangeStatus::UNCHANGED;7928 7929    // Clear existing attributes.7930    A.removeAttrs(IRP, AttrKinds);7931    // Clear conflicting writable attribute.7932    if (isAssumedReadOnly())7933      A.removeAttrs(IRP, Attribute::Writable);7934 7935    // Use the generic manifest method.7936    return IRAttribute::manifest(A);7937  }7938 7939  /// See AbstractState::getAsStr().7940  const std::string getAsStr(Attributor *A) const override {7941    if (isAssumedReadNone())7942      return "readnone";7943    if (isAssumedReadOnly())7944      return "readonly";7945    if (isAssumedWriteOnly())7946      return "writeonly";7947    return "may-read/write";7948  }7949 7950  /// The set of IR attributes AAMemoryBehavior deals with.7951  static const Attribute::AttrKind AttrKinds[3];7952};7953 7954const Attribute::AttrKind AAMemoryBehaviorImpl::AttrKinds[] = {7955    Attribute::ReadNone, Attribute::ReadOnly, Attribute::WriteOnly};7956 7957/// Memory behavior attribute for a floating value.7958struct AAMemoryBehaviorFloating : AAMemoryBehaviorImpl {7959  AAMemoryBehaviorFloating(const IRPosition &IRP, Attributor &A)7960      : AAMemoryBehaviorImpl(IRP, A) {}7961 7962  /// See AbstractAttribute::updateImpl(...).7963  ChangeStatus updateImpl(Attributor &A) override;7964 7965  /// See AbstractAttribute::trackStatistics()7966  void trackStatistics() const override {7967    if (isAssumedReadNone())7968      STATS_DECLTRACK_FLOATING_ATTR(readnone)7969    else if (isAssumedReadOnly())7970      STATS_DECLTRACK_FLOATING_ATTR(readonly)7971    else if (isAssumedWriteOnly())7972      STATS_DECLTRACK_FLOATING_ATTR(writeonly)7973  }7974 7975private:7976  /// Return true if users of \p UserI might access the underlying7977  /// variable/location described by \p U and should therefore be analyzed.7978  bool followUsersOfUseIn(Attributor &A, const Use &U,7979                          const Instruction *UserI);7980 7981  /// Update the state according to the effect of use \p U in \p UserI.7982  void analyzeUseIn(Attributor &A, const Use &U, const Instruction *UserI);7983};7984 7985/// Memory behavior attribute for function argument.7986struct AAMemoryBehaviorArgument : AAMemoryBehaviorFloating {7987  AAMemoryBehaviorArgument(const IRPosition &IRP, Attributor &A)7988      : AAMemoryBehaviorFloating(IRP, A) {}7989 7990  /// See AbstractAttribute::initialize(...).7991  void initialize(Attributor &A) override {7992    intersectAssumedBits(BEST_STATE);7993    const IRPosition &IRP = getIRPosition();7994    // TODO: Make IgnoreSubsumingPositions a property of an IRAttribute so we7995    // can query it when we use has/getAttr. That would allow us to reuse the7996    // initialize of the base class here.7997    bool HasByVal = A.hasAttr(IRP, {Attribute::ByVal},7998                              /* IgnoreSubsumingPositions */ true);7999    getKnownStateFromValue(A, IRP, getState(),8000                           /* IgnoreSubsumingPositions */ HasByVal);8001  }8002 8003  ChangeStatus manifest(Attributor &A) override {8004    // TODO: Pointer arguments are not supported on vectors of pointers yet.8005    if (!getAssociatedValue().getType()->isPointerTy())8006      return ChangeStatus::UNCHANGED;8007 8008    // TODO: From readattrs.ll: "inalloca parameters are always8009    //                           considered written"8010    if (A.hasAttr(getIRPosition(),8011                  {Attribute::InAlloca, Attribute::Preallocated})) {8012      removeKnownBits(NO_WRITES);8013      removeAssumedBits(NO_WRITES);8014    }8015    A.removeAttrs(getIRPosition(), AttrKinds);8016    return AAMemoryBehaviorFloating::manifest(A);8017  }8018 8019  /// See AbstractAttribute::trackStatistics()8020  void trackStatistics() const override {8021    if (isAssumedReadNone())8022      STATS_DECLTRACK_ARG_ATTR(readnone)8023    else if (isAssumedReadOnly())8024      STATS_DECLTRACK_ARG_ATTR(readonly)8025    else if (isAssumedWriteOnly())8026      STATS_DECLTRACK_ARG_ATTR(writeonly)8027  }8028};8029 8030struct AAMemoryBehaviorCallSiteArgument final : AAMemoryBehaviorArgument {8031  AAMemoryBehaviorCallSiteArgument(const IRPosition &IRP, Attributor &A)8032      : AAMemoryBehaviorArgument(IRP, A) {}8033 8034  /// See AbstractAttribute::initialize(...).8035  void initialize(Attributor &A) override {8036    // If we don't have an associated attribute this is either a variadic call8037    // or an indirect call, either way, nothing to do here.8038    Argument *Arg = getAssociatedArgument();8039    if (!Arg) {8040      indicatePessimisticFixpoint();8041      return;8042    }8043    if (Arg->hasByValAttr()) {8044      addKnownBits(NO_WRITES);8045      removeKnownBits(NO_READS);8046      removeAssumedBits(NO_READS);8047    }8048    AAMemoryBehaviorArgument::initialize(A);8049    if (getAssociatedFunction()->isDeclaration())8050      indicatePessimisticFixpoint();8051  }8052 8053  /// See AbstractAttribute::updateImpl(...).8054  ChangeStatus updateImpl(Attributor &A) override {8055    // TODO: Once we have call site specific value information we can provide8056    //       call site specific liveness liveness information and then it makes8057    //       sense to specialize attributes for call sites arguments instead of8058    //       redirecting requests to the callee argument.8059    Argument *Arg = getAssociatedArgument();8060    const IRPosition &ArgPos = IRPosition::argument(*Arg);8061    auto *ArgAA =8062        A.getAAFor<AAMemoryBehavior>(*this, ArgPos, DepClassTy::REQUIRED);8063    if (!ArgAA)8064      return indicatePessimisticFixpoint();8065    return clampStateAndIndicateChange(getState(), ArgAA->getState());8066  }8067 8068  /// See AbstractAttribute::trackStatistics()8069  void trackStatistics() const override {8070    if (isAssumedReadNone())8071      STATS_DECLTRACK_CSARG_ATTR(readnone)8072    else if (isAssumedReadOnly())8073      STATS_DECLTRACK_CSARG_ATTR(readonly)8074    else if (isAssumedWriteOnly())8075      STATS_DECLTRACK_CSARG_ATTR(writeonly)8076  }8077};8078 8079/// Memory behavior attribute for a call site return position.8080struct AAMemoryBehaviorCallSiteReturned final : AAMemoryBehaviorFloating {8081  AAMemoryBehaviorCallSiteReturned(const IRPosition &IRP, Attributor &A)8082      : AAMemoryBehaviorFloating(IRP, A) {}8083 8084  /// See AbstractAttribute::initialize(...).8085  void initialize(Attributor &A) override {8086    AAMemoryBehaviorImpl::initialize(A);8087  }8088  /// See AbstractAttribute::manifest(...).8089  ChangeStatus manifest(Attributor &A) override {8090    // We do not annotate returned values.8091    return ChangeStatus::UNCHANGED;8092  }8093 8094  /// See AbstractAttribute::trackStatistics()8095  void trackStatistics() const override {}8096};8097 8098/// An AA to represent the memory behavior function attributes.8099struct AAMemoryBehaviorFunction final : public AAMemoryBehaviorImpl {8100  AAMemoryBehaviorFunction(const IRPosition &IRP, Attributor &A)8101      : AAMemoryBehaviorImpl(IRP, A) {}8102 8103  /// See AbstractAttribute::updateImpl(Attributor &A).8104  ChangeStatus updateImpl(Attributor &A) override;8105 8106  /// See AbstractAttribute::manifest(...).8107  ChangeStatus manifest(Attributor &A) override {8108    // TODO: It would be better to merge this with AAMemoryLocation, so that8109    // we could determine read/write per location. This would also have the8110    // benefit of only one place trying to manifest the memory attribute.8111    Function &F = cast<Function>(getAnchorValue());8112    MemoryEffects ME = MemoryEffects::unknown();8113    if (isAssumedReadNone())8114      ME = MemoryEffects::none();8115    else if (isAssumedReadOnly())8116      ME = MemoryEffects::readOnly();8117    else if (isAssumedWriteOnly())8118      ME = MemoryEffects::writeOnly();8119 8120    A.removeAttrs(getIRPosition(), AttrKinds);8121    // Clear conflicting writable attribute.8122    if (ME.onlyReadsMemory())8123      for (Argument &Arg : F.args())8124        A.removeAttrs(IRPosition::argument(Arg), Attribute::Writable);8125    return A.manifestAttrs(getIRPosition(),8126                           Attribute::getWithMemoryEffects(F.getContext(), ME));8127  }8128 8129  /// See AbstractAttribute::trackStatistics()8130  void trackStatistics() const override {8131    if (isAssumedReadNone())8132      STATS_DECLTRACK_FN_ATTR(readnone)8133    else if (isAssumedReadOnly())8134      STATS_DECLTRACK_FN_ATTR(readonly)8135    else if (isAssumedWriteOnly())8136      STATS_DECLTRACK_FN_ATTR(writeonly)8137  }8138};8139 8140/// AAMemoryBehavior attribute for call sites.8141struct AAMemoryBehaviorCallSite final8142    : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl> {8143  AAMemoryBehaviorCallSite(const IRPosition &IRP, Attributor &A)8144      : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl>(IRP, A) {}8145 8146  /// See AbstractAttribute::manifest(...).8147  ChangeStatus manifest(Attributor &A) override {8148    // TODO: Deduplicate this with AAMemoryBehaviorFunction.8149    CallBase &CB = cast<CallBase>(getAnchorValue());8150    MemoryEffects ME = MemoryEffects::unknown();8151    if (isAssumedReadNone())8152      ME = MemoryEffects::none();8153    else if (isAssumedReadOnly())8154      ME = MemoryEffects::readOnly();8155    else if (isAssumedWriteOnly())8156      ME = MemoryEffects::writeOnly();8157 8158    A.removeAttrs(getIRPosition(), AttrKinds);8159    // Clear conflicting writable attribute.8160    if (ME.onlyReadsMemory())8161      for (Use &U : CB.args())8162        A.removeAttrs(IRPosition::callsite_argument(CB, U.getOperandNo()),8163                      Attribute::Writable);8164    return A.manifestAttrs(8165        getIRPosition(), Attribute::getWithMemoryEffects(CB.getContext(), ME));8166  }8167 8168  /// See AbstractAttribute::trackStatistics()8169  void trackStatistics() const override {8170    if (isAssumedReadNone())8171      STATS_DECLTRACK_CS_ATTR(readnone)8172    else if (isAssumedReadOnly())8173      STATS_DECLTRACK_CS_ATTR(readonly)8174    else if (isAssumedWriteOnly())8175      STATS_DECLTRACK_CS_ATTR(writeonly)8176  }8177};8178 8179ChangeStatus AAMemoryBehaviorFunction::updateImpl(Attributor &A) {8180 8181  // The current assumed state used to determine a change.8182  auto AssumedState = getAssumed();8183 8184  auto CheckRWInst = [&](Instruction &I) {8185    // If the instruction has an own memory behavior state, use it to restrict8186    // the local state. No further analysis is required as the other memory8187    // state is as optimistic as it gets.8188    if (const auto *CB = dyn_cast<CallBase>(&I)) {8189      const auto *MemBehaviorAA = A.getAAFor<AAMemoryBehavior>(8190          *this, IRPosition::callsite_function(*CB), DepClassTy::REQUIRED);8191      if (MemBehaviorAA) {8192        intersectAssumedBits(MemBehaviorAA->getAssumed());8193        return !isAtFixpoint();8194      }8195    }8196 8197    // Remove access kind modifiers if necessary.8198    if (I.mayReadFromMemory())8199      removeAssumedBits(NO_READS);8200    if (I.mayWriteToMemory())8201      removeAssumedBits(NO_WRITES);8202    return !isAtFixpoint();8203  };8204 8205  bool UsedAssumedInformation = false;8206  if (!A.checkForAllReadWriteInstructions(CheckRWInst, *this,8207                                          UsedAssumedInformation))8208    return indicatePessimisticFixpoint();8209 8210  return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED8211                                        : ChangeStatus::UNCHANGED;8212}8213 8214ChangeStatus AAMemoryBehaviorFloating::updateImpl(Attributor &A) {8215 8216  const IRPosition &IRP = getIRPosition();8217  const IRPosition &FnPos = IRPosition::function_scope(IRP);8218  AAMemoryBehavior::StateType &S = getState();8219 8220  // First, check the function scope. We take the known information and we avoid8221  // work if the assumed information implies the current assumed information for8222  // this attribute. This is a valid for all but byval arguments.8223  Argument *Arg = IRP.getAssociatedArgument();8224  AAMemoryBehavior::base_t FnMemAssumedState =8225      AAMemoryBehavior::StateType::getWorstState();8226  if (!Arg || !Arg->hasByValAttr()) {8227    const auto *FnMemAA =8228        A.getAAFor<AAMemoryBehavior>(*this, FnPos, DepClassTy::OPTIONAL);8229    if (FnMemAA) {8230      FnMemAssumedState = FnMemAA->getAssumed();8231      S.addKnownBits(FnMemAA->getKnown());8232      if ((S.getAssumed() & FnMemAA->getAssumed()) == S.getAssumed())8233        return ChangeStatus::UNCHANGED;8234    }8235  }8236 8237  // The current assumed state used to determine a change.8238  auto AssumedState = S.getAssumed();8239 8240  // Make sure the value is not captured (except through "return"), if8241  // it is, any information derived would be irrelevant anyway as we cannot8242  // check the potential aliases introduced by the capture. However, no need8243  // to fall back to anythign less optimistic than the function state.8244  bool IsKnownNoCapture;8245  const AANoCapture *ArgNoCaptureAA = nullptr;8246  bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(8247      A, this, IRP, DepClassTy::OPTIONAL, IsKnownNoCapture, false,8248      &ArgNoCaptureAA);8249 8250  if (!IsAssumedNoCapture &&8251      (!ArgNoCaptureAA || !ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned())) {8252    S.intersectAssumedBits(FnMemAssumedState);8253    return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED8254                                          : ChangeStatus::UNCHANGED;8255  }8256 8257  // Visit and expand uses until all are analyzed or a fixpoint is reached.8258  auto UsePred = [&](const Use &U, bool &Follow) -> bool {8259    Instruction *UserI = cast<Instruction>(U.getUser());8260    LLVM_DEBUG(dbgs() << "[AAMemoryBehavior] Use: " << *U << " in " << *UserI8261                      << " \n");8262 8263    // Droppable users, e.g., llvm::assume does not actually perform any action.8264    if (UserI->isDroppable())8265      return true;8266 8267    // Check if the users of UserI should also be visited.8268    Follow = followUsersOfUseIn(A, U, UserI);8269 8270    // If UserI might touch memory we analyze the use in detail.8271    if (UserI->mayReadOrWriteMemory())8272      analyzeUseIn(A, U, UserI);8273 8274    return !isAtFixpoint();8275  };8276 8277  if (!A.checkForAllUses(UsePred, *this, getAssociatedValue()))8278    return indicatePessimisticFixpoint();8279 8280  return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED8281                                        : ChangeStatus::UNCHANGED;8282}8283 8284bool AAMemoryBehaviorFloating::followUsersOfUseIn(Attributor &A, const Use &U,8285                                                  const Instruction *UserI) {8286  // The loaded value is unrelated to the pointer argument, no need to8287  // follow the users of the load.8288  if (isa<LoadInst>(UserI) || isa<ReturnInst>(UserI))8289    return false;8290 8291  // By default we follow all uses assuming UserI might leak information on U,8292  // we have special handling for call sites operands though.8293  const auto *CB = dyn_cast<CallBase>(UserI);8294  if (!CB || !CB->isArgOperand(&U))8295    return true;8296 8297  // If the use is a call argument known not to be captured, the users of8298  // the call do not need to be visited because they have to be unrelated to8299  // the input. Note that this check is not trivial even though we disallow8300  // general capturing of the underlying argument. The reason is that the8301  // call might the argument "through return", which we allow and for which we8302  // need to check call users.8303  if (U.get()->getType()->isPointerTy()) {8304    unsigned ArgNo = CB->getArgOperandNo(&U);8305    bool IsKnownNoCapture;8306    return !AA::hasAssumedIRAttr<Attribute::Captures>(8307        A, this, IRPosition::callsite_argument(*CB, ArgNo),8308        DepClassTy::OPTIONAL, IsKnownNoCapture);8309  }8310 8311  return true;8312}8313 8314void AAMemoryBehaviorFloating::analyzeUseIn(Attributor &A, const Use &U,8315                                            const Instruction *UserI) {8316  assert(UserI->mayReadOrWriteMemory());8317 8318  switch (UserI->getOpcode()) {8319  default:8320    // TODO: Handle all atomics and other side-effect operations we know of.8321    break;8322  case Instruction::Load:8323    // Loads cause the NO_READS property to disappear.8324    removeAssumedBits(NO_READS);8325    return;8326 8327  case Instruction::Store:8328    // Stores cause the NO_WRITES property to disappear if the use is the8329    // pointer operand. Note that while capturing was taken care of somewhere8330    // else we need to deal with stores of the value that is not looked through.8331    if (cast<StoreInst>(UserI)->getPointerOperand() == U.get())8332      removeAssumedBits(NO_WRITES);8333    else8334      indicatePessimisticFixpoint();8335    return;8336 8337  case Instruction::Call:8338  case Instruction::CallBr:8339  case Instruction::Invoke: {8340    // For call sites we look at the argument memory behavior attribute (this8341    // could be recursive!) in order to restrict our own state.8342    const auto *CB = cast<CallBase>(UserI);8343 8344    // Give up on operand bundles.8345    if (CB->isBundleOperand(&U)) {8346      indicatePessimisticFixpoint();8347      return;8348    }8349 8350    // Calling a function does read the function pointer, maybe write it if the8351    // function is self-modifying.8352    if (CB->isCallee(&U)) {8353      removeAssumedBits(NO_READS);8354      break;8355    }8356 8357    // Adjust the possible access behavior based on the information on the8358    // argument.8359    IRPosition Pos;8360    if (U.get()->getType()->isPointerTy())8361      Pos = IRPosition::callsite_argument(*CB, CB->getArgOperandNo(&U));8362    else8363      Pos = IRPosition::callsite_function(*CB);8364    const auto *MemBehaviorAA =8365        A.getAAFor<AAMemoryBehavior>(*this, Pos, DepClassTy::OPTIONAL);8366    if (!MemBehaviorAA)8367      break;8368    // "assumed" has at most the same bits as the MemBehaviorAA assumed8369    // and at least "known".8370    intersectAssumedBits(MemBehaviorAA->getAssumed());8371    return;8372  }8373  };8374 8375  // Generally, look at the "may-properties" and adjust the assumed state if we8376  // did not trigger special handling before.8377  if (UserI->mayReadFromMemory())8378    removeAssumedBits(NO_READS);8379  if (UserI->mayWriteToMemory())8380    removeAssumedBits(NO_WRITES);8381}8382} // namespace8383 8384/// -------------------- Memory Locations Attributes ---------------------------8385/// Includes read-none, argmemonly, inaccessiblememonly,8386/// inaccessiblememorargmemonly8387/// ----------------------------------------------------------------------------8388 8389std::string AAMemoryLocation::getMemoryLocationsAsStr(8390    AAMemoryLocation::MemoryLocationsKind MLK) {8391  if (0 == (MLK & AAMemoryLocation::NO_LOCATIONS))8392    return "all memory";8393  if (MLK == AAMemoryLocation::NO_LOCATIONS)8394    return "no memory";8395  std::string S = "memory:";8396  if (0 == (MLK & AAMemoryLocation::NO_LOCAL_MEM))8397    S += "stack,";8398  if (0 == (MLK & AAMemoryLocation::NO_CONST_MEM))8399    S += "constant,";8400  if (0 == (MLK & AAMemoryLocation::NO_GLOBAL_INTERNAL_MEM))8401    S += "internal global,";8402  if (0 == (MLK & AAMemoryLocation::NO_GLOBAL_EXTERNAL_MEM))8403    S += "external global,";8404  if (0 == (MLK & AAMemoryLocation::NO_ARGUMENT_MEM))8405    S += "argument,";8406  if (0 == (MLK & AAMemoryLocation::NO_INACCESSIBLE_MEM))8407    S += "inaccessible,";8408  if (0 == (MLK & AAMemoryLocation::NO_MALLOCED_MEM))8409    S += "malloced,";8410  if (0 == (MLK & AAMemoryLocation::NO_UNKOWN_MEM))8411    S += "unknown,";8412  S.pop_back();8413  return S;8414}8415 8416namespace {8417struct AAMemoryLocationImpl : public AAMemoryLocation {8418 8419  AAMemoryLocationImpl(const IRPosition &IRP, Attributor &A)8420      : AAMemoryLocation(IRP, A), Allocator(A.Allocator) {8421    AccessKind2Accesses.fill(nullptr);8422  }8423 8424  ~AAMemoryLocationImpl() override {8425    // The AccessSets are allocated via a BumpPtrAllocator, we call8426    // the destructor manually.8427    for (AccessSet *AS : AccessKind2Accesses)8428      if (AS)8429        AS->~AccessSet();8430  }8431 8432  /// See AbstractAttribute::initialize(...).8433  void initialize(Attributor &A) override {8434    intersectAssumedBits(BEST_STATE);8435    getKnownStateFromValue(A, getIRPosition(), getState());8436    AAMemoryLocation::initialize(A);8437  }8438 8439  /// Return the memory behavior information encoded in the IR for \p IRP.8440  static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,8441                                     BitIntegerState &State,8442                                     bool IgnoreSubsumingPositions = false) {8443    // For internal functions we ignore `argmemonly` and8444    // `inaccessiblememorargmemonly` as we might break it via interprocedural8445    // constant propagation. It is unclear if this is the best way but it is8446    // unlikely this will cause real performance problems. If we are deriving8447    // attributes for the anchor function we even remove the attribute in8448    // addition to ignoring it.8449    // TODO: A better way to handle this would be to add ~NO_GLOBAL_MEM /8450    // MemoryEffects::Other as a possible location.8451    bool UseArgMemOnly = true;8452    Function *AnchorFn = IRP.getAnchorScope();8453    if (AnchorFn && A.isRunOn(*AnchorFn))8454      UseArgMemOnly = !AnchorFn->hasLocalLinkage();8455 8456    SmallVector<Attribute, 2> Attrs;8457    A.getAttrs(IRP, {Attribute::Memory}, Attrs, IgnoreSubsumingPositions);8458    for (const Attribute &Attr : Attrs) {8459      // TODO: We can map MemoryEffects to Attributor locations more precisely.8460      MemoryEffects ME = Attr.getMemoryEffects();8461      if (ME.doesNotAccessMemory()) {8462        State.addKnownBits(NO_LOCAL_MEM | NO_CONST_MEM);8463        continue;8464      }8465      if (ME.onlyAccessesInaccessibleMem()) {8466        State.addKnownBits(inverseLocation(NO_INACCESSIBLE_MEM, true, true));8467        continue;8468      }8469      if (ME.onlyAccessesArgPointees()) {8470        if (UseArgMemOnly)8471          State.addKnownBits(inverseLocation(NO_ARGUMENT_MEM, true, true));8472        else {8473          // Remove location information, only keep read/write info.8474          ME = MemoryEffects(ME.getModRef());8475          A.manifestAttrs(IRP,8476                          Attribute::getWithMemoryEffects(8477                              IRP.getAnchorValue().getContext(), ME),8478                          /*ForceReplace*/ true);8479        }8480        continue;8481      }8482      if (ME.onlyAccessesInaccessibleOrArgMem()) {8483        if (UseArgMemOnly)8484          State.addKnownBits(inverseLocation(8485              NO_INACCESSIBLE_MEM | NO_ARGUMENT_MEM, true, true));8486        else {8487          // Remove location information, only keep read/write info.8488          ME = MemoryEffects(ME.getModRef());8489          A.manifestAttrs(IRP,8490                          Attribute::getWithMemoryEffects(8491                              IRP.getAnchorValue().getContext(), ME),8492                          /*ForceReplace*/ true);8493        }8494        continue;8495      }8496    }8497  }8498 8499  /// See AbstractAttribute::getDeducedAttributes(...).8500  void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,8501                            SmallVectorImpl<Attribute> &Attrs) const override {8502    // TODO: We can map Attributor locations to MemoryEffects more precisely.8503    assert(Attrs.size() == 0);8504    if (getIRPosition().getPositionKind() == IRPosition::IRP_FUNCTION) {8505      if (isAssumedReadNone())8506        Attrs.push_back(8507            Attribute::getWithMemoryEffects(Ctx, MemoryEffects::none()));8508      else if (isAssumedInaccessibleMemOnly())8509        Attrs.push_back(Attribute::getWithMemoryEffects(8510            Ctx, MemoryEffects::inaccessibleMemOnly()));8511      else if (isAssumedArgMemOnly())8512        Attrs.push_back(8513            Attribute::getWithMemoryEffects(Ctx, MemoryEffects::argMemOnly()));8514      else if (isAssumedInaccessibleOrArgMemOnly())8515        Attrs.push_back(Attribute::getWithMemoryEffects(8516            Ctx, MemoryEffects::inaccessibleOrArgMemOnly()));8517    }8518    assert(Attrs.size() <= 1);8519  }8520 8521  /// See AbstractAttribute::manifest(...).8522  ChangeStatus manifest(Attributor &A) override {8523    // TODO: If AAMemoryLocation and AAMemoryBehavior are merged, we could8524    // provide per-location modref information here.8525    const IRPosition &IRP = getIRPosition();8526 8527    SmallVector<Attribute, 1> DeducedAttrs;8528    getDeducedAttributes(A, IRP.getAnchorValue().getContext(), DeducedAttrs);8529    if (DeducedAttrs.size() != 1)8530      return ChangeStatus::UNCHANGED;8531    MemoryEffects ME = DeducedAttrs[0].getMemoryEffects();8532 8533    return A.manifestAttrs(IRP, Attribute::getWithMemoryEffects(8534                                    IRP.getAnchorValue().getContext(), ME));8535  }8536 8537  /// See AAMemoryLocation::checkForAllAccessesToMemoryKind(...).8538  bool checkForAllAccessesToMemoryKind(8539      function_ref<bool(const Instruction *, const Value *, AccessKind,8540                        MemoryLocationsKind)>8541          Pred,8542      MemoryLocationsKind RequestedMLK) const override {8543    if (!isValidState())8544      return false;8545 8546    MemoryLocationsKind AssumedMLK = getAssumedNotAccessedLocation();8547    if (AssumedMLK == NO_LOCATIONS)8548      return true;8549 8550    unsigned Idx = 0;8551    for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS;8552         CurMLK *= 2, ++Idx) {8553      if (CurMLK & RequestedMLK)8554        continue;8555 8556      if (const AccessSet *Accesses = AccessKind2Accesses[Idx])8557        for (const AccessInfo &AI : *Accesses)8558          if (!Pred(AI.I, AI.Ptr, AI.Kind, CurMLK))8559            return false;8560    }8561 8562    return true;8563  }8564 8565  ChangeStatus indicatePessimisticFixpoint() override {8566    // If we give up and indicate a pessimistic fixpoint this instruction will8567    // become an access for all potential access kinds:8568    // TODO: Add pointers for argmemonly and globals to improve the results of8569    //       checkForAllAccessesToMemoryKind.8570    bool Changed = false;8571    MemoryLocationsKind KnownMLK = getKnown();8572    Instruction *I = dyn_cast<Instruction>(&getAssociatedValue());8573    for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2)8574      if (!(CurMLK & KnownMLK))8575        updateStateAndAccessesMap(getState(), CurMLK, I, nullptr, Changed,8576                                  getAccessKindFromInst(I));8577    return AAMemoryLocation::indicatePessimisticFixpoint();8578  }8579 8580protected:8581  /// Helper struct to tie together an instruction that has a read or write8582  /// effect with the pointer it accesses (if any).8583  struct AccessInfo {8584 8585    /// The instruction that caused the access.8586    const Instruction *I;8587 8588    /// The base pointer that is accessed, or null if unknown.8589    const Value *Ptr;8590 8591    /// The kind of access (read/write/read+write).8592    AccessKind Kind;8593 8594    bool operator==(const AccessInfo &RHS) const {8595      return I == RHS.I && Ptr == RHS.Ptr && Kind == RHS.Kind;8596    }8597    bool operator()(const AccessInfo &LHS, const AccessInfo &RHS) const {8598      if (LHS.I != RHS.I)8599        return LHS.I < RHS.I;8600      if (LHS.Ptr != RHS.Ptr)8601        return LHS.Ptr < RHS.Ptr;8602      if (LHS.Kind != RHS.Kind)8603        return LHS.Kind < RHS.Kind;8604      return false;8605    }8606  };8607 8608  /// Mapping from *single* memory location kinds, e.g., LOCAL_MEM with the8609  /// value of NO_LOCAL_MEM, to the accesses encountered for this memory kind.8610  using AccessSet = SmallSet<AccessInfo, 2, AccessInfo>;8611  std::array<AccessSet *, llvm::ConstantLog2<VALID_STATE>()>8612      AccessKind2Accesses;8613 8614  /// Categorize the pointer arguments of CB that might access memory in8615  /// AccessedLoc and update the state and access map accordingly.8616  void8617  categorizeArgumentPointerLocations(Attributor &A, CallBase &CB,8618                                     AAMemoryLocation::StateType &AccessedLocs,8619                                     bool &Changed);8620 8621  /// Return the kind(s) of location that may be accessed by \p V.8622  AAMemoryLocation::MemoryLocationsKind8623  categorizeAccessedLocations(Attributor &A, Instruction &I, bool &Changed);8624 8625  /// Return the access kind as determined by \p I.8626  AccessKind getAccessKindFromInst(const Instruction *I) {8627    AccessKind AK = READ_WRITE;8628    if (I) {8629      AK = I->mayReadFromMemory() ? READ : NONE;8630      AK = AccessKind(AK | (I->mayWriteToMemory() ? WRITE : NONE));8631    }8632    return AK;8633  }8634 8635  /// Update the state \p State and the AccessKind2Accesses given that \p I is8636  /// an access of kind \p AK to a \p MLK memory location with the access8637  /// pointer \p Ptr.8638  void updateStateAndAccessesMap(AAMemoryLocation::StateType &State,8639                                 MemoryLocationsKind MLK, const Instruction *I,8640                                 const Value *Ptr, bool &Changed,8641                                 AccessKind AK = READ_WRITE) {8642 8643    assert(isPowerOf2_32(MLK) && "Expected a single location set!");8644    auto *&Accesses = AccessKind2Accesses[llvm::Log2_32(MLK)];8645    if (!Accesses)8646      Accesses = new (Allocator) AccessSet();8647    Changed |= Accesses->insert(AccessInfo{I, Ptr, AK}).second;8648    if (MLK == NO_UNKOWN_MEM)8649      MLK = NO_LOCATIONS;8650    State.removeAssumedBits(MLK);8651  }8652 8653  /// Determine the underlying locations kinds for \p Ptr, e.g., globals or8654  /// arguments, and update the state and access map accordingly.8655  void categorizePtrValue(Attributor &A, const Instruction &I, const Value &Ptr,8656                          AAMemoryLocation::StateType &State, bool &Changed,8657                          unsigned AccessAS = 0);8658 8659  /// Used to allocate access sets.8660  BumpPtrAllocator &Allocator;8661};8662 8663void AAMemoryLocationImpl::categorizePtrValue(8664    Attributor &A, const Instruction &I, const Value &Ptr,8665    AAMemoryLocation::StateType &State, bool &Changed, unsigned AccessAS) {8666  LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize pointer locations for "8667                    << Ptr << " ["8668                    << getMemoryLocationsAsStr(State.getAssumed()) << "]\n");8669 8670  auto Pred = [&](Value &Obj) {8671    unsigned ObjectAS = Obj.getType()->getPointerAddressSpace();8672    // TODO: recognize the TBAA used for constant accesses.8673    MemoryLocationsKind MLK = NO_LOCATIONS;8674 8675    // Filter accesses to constant (GPU) memory if we have an AS at the access8676    // site or the object is known to actually have the associated AS.8677    if ((AccessAS == (unsigned)AA::GPUAddressSpace::Constant ||8678         (ObjectAS == (unsigned)AA::GPUAddressSpace::Constant &&8679          isIdentifiedObject(&Obj))) &&8680        AA::isGPU(*I.getModule()))8681      return true;8682 8683    if (isa<UndefValue>(&Obj))8684      return true;8685    if (isa<Argument>(&Obj)) {8686      // TODO: For now we do not treat byval arguments as local copies performed8687      // on the call edge, though, we should. To make that happen we need to8688      // teach various passes, e.g., DSE, about the copy effect of a byval. That8689      // would also allow us to mark functions only accessing byval arguments as8690      // readnone again, arguably their accesses have no effect outside of the8691      // function, like accesses to allocas.8692      MLK = NO_ARGUMENT_MEM;8693    } else if (auto *GV = dyn_cast<GlobalValue>(&Obj)) {8694      // Reading constant memory is not treated as a read "effect" by the8695      // function attr pass so we won't neither. Constants defined by TBAA are8696      // similar. (We know we do not write it because it is constant.)8697      if (auto *GVar = dyn_cast<GlobalVariable>(GV))8698        if (GVar->isConstant())8699          return true;8700 8701      if (GV->hasLocalLinkage())8702        MLK = NO_GLOBAL_INTERNAL_MEM;8703      else8704        MLK = NO_GLOBAL_EXTERNAL_MEM;8705    } else if (isa<ConstantPointerNull>(&Obj) &&8706               (!NullPointerIsDefined(getAssociatedFunction(), AccessAS) ||8707                !NullPointerIsDefined(getAssociatedFunction(), ObjectAS))) {8708      return true;8709    } else if (isa<AllocaInst>(&Obj)) {8710      MLK = NO_LOCAL_MEM;8711    } else if (const auto *CB = dyn_cast<CallBase>(&Obj)) {8712      bool IsKnownNoAlias;8713      if (AA::hasAssumedIRAttr<Attribute::NoAlias>(8714              A, this, IRPosition::callsite_returned(*CB), DepClassTy::OPTIONAL,8715              IsKnownNoAlias))8716        MLK = NO_MALLOCED_MEM;8717      else8718        MLK = NO_UNKOWN_MEM;8719    } else {8720      MLK = NO_UNKOWN_MEM;8721    }8722 8723    assert(MLK != NO_LOCATIONS && "No location specified!");8724    LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Ptr value can be categorized: "8725                      << Obj << " -> " << getMemoryLocationsAsStr(MLK) << "\n");8726    updateStateAndAccessesMap(State, MLK, &I, &Obj, Changed,8727                              getAccessKindFromInst(&I));8728 8729    return true;8730  };8731 8732  const auto *AA = A.getAAFor<AAUnderlyingObjects>(8733      *this, IRPosition::value(Ptr), DepClassTy::OPTIONAL);8734  if (!AA || !AA->forallUnderlyingObjects(Pred, AA::Intraprocedural)) {8735    LLVM_DEBUG(8736        dbgs() << "[AAMemoryLocation] Pointer locations not categorized\n");8737    updateStateAndAccessesMap(State, NO_UNKOWN_MEM, &I, nullptr, Changed,8738                              getAccessKindFromInst(&I));8739    return;8740  }8741 8742  LLVM_DEBUG(8743      dbgs() << "[AAMemoryLocation] Accessed locations with pointer locations: "8744             << getMemoryLocationsAsStr(State.getAssumed()) << "\n");8745}8746 8747void AAMemoryLocationImpl::categorizeArgumentPointerLocations(8748    Attributor &A, CallBase &CB, AAMemoryLocation::StateType &AccessedLocs,8749    bool &Changed) {8750  for (unsigned ArgNo = 0, E = CB.arg_size(); ArgNo < E; ++ArgNo) {8751 8752    // Skip non-pointer arguments.8753    const Value *ArgOp = CB.getArgOperand(ArgNo);8754    if (!ArgOp->getType()->isPtrOrPtrVectorTy())8755      continue;8756 8757    // Skip readnone arguments.8758    const IRPosition &ArgOpIRP = IRPosition::callsite_argument(CB, ArgNo);8759    const auto *ArgOpMemLocationAA =8760        A.getAAFor<AAMemoryBehavior>(*this, ArgOpIRP, DepClassTy::OPTIONAL);8761 8762    if (ArgOpMemLocationAA && ArgOpMemLocationAA->isAssumedReadNone())8763      continue;8764 8765    // Categorize potentially accessed pointer arguments as if there was an8766    // access instruction with them as pointer.8767    categorizePtrValue(A, CB, *ArgOp, AccessedLocs, Changed);8768  }8769}8770 8771AAMemoryLocation::MemoryLocationsKind8772AAMemoryLocationImpl::categorizeAccessedLocations(Attributor &A, Instruction &I,8773                                                  bool &Changed) {8774  LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize accessed locations for "8775                    << I << "\n");8776 8777  AAMemoryLocation::StateType AccessedLocs;8778  AccessedLocs.intersectAssumedBits(NO_LOCATIONS);8779 8780  if (auto *CB = dyn_cast<CallBase>(&I)) {8781 8782    // First check if we assume any memory is access is visible.8783    const auto *CBMemLocationAA = A.getAAFor<AAMemoryLocation>(8784        *this, IRPosition::callsite_function(*CB), DepClassTy::OPTIONAL);8785    LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize call site: " << I8786                      << " [" << CBMemLocationAA << "]\n");8787    if (!CBMemLocationAA) {8788      updateStateAndAccessesMap(AccessedLocs, NO_UNKOWN_MEM, &I, nullptr,8789                                Changed, getAccessKindFromInst(&I));8790      return NO_UNKOWN_MEM;8791    }8792 8793    if (CBMemLocationAA->isAssumedReadNone())8794      return NO_LOCATIONS;8795 8796    if (CBMemLocationAA->isAssumedInaccessibleMemOnly()) {8797      updateStateAndAccessesMap(AccessedLocs, NO_INACCESSIBLE_MEM, &I, nullptr,8798                                Changed, getAccessKindFromInst(&I));8799      return AccessedLocs.getAssumed();8800    }8801 8802    uint32_t CBAssumedNotAccessedLocs =8803        CBMemLocationAA->getAssumedNotAccessedLocation();8804 8805    // Set the argmemonly and global bit as we handle them separately below.8806    uint32_t CBAssumedNotAccessedLocsNoArgMem =8807        CBAssumedNotAccessedLocs | NO_ARGUMENT_MEM | NO_GLOBAL_MEM;8808 8809    for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2) {8810      if (CBAssumedNotAccessedLocsNoArgMem & CurMLK)8811        continue;8812      updateStateAndAccessesMap(AccessedLocs, CurMLK, &I, nullptr, Changed,8813                                getAccessKindFromInst(&I));8814    }8815 8816    // Now handle global memory if it might be accessed. This is slightly tricky8817    // as NO_GLOBAL_MEM has multiple bits set.8818    bool HasGlobalAccesses = ((~CBAssumedNotAccessedLocs) & NO_GLOBAL_MEM);8819    if (HasGlobalAccesses) {8820      auto AccessPred = [&](const Instruction *, const Value *Ptr,8821                            AccessKind Kind, MemoryLocationsKind MLK) {8822        updateStateAndAccessesMap(AccessedLocs, MLK, &I, Ptr, Changed,8823                                  getAccessKindFromInst(&I));8824        return true;8825      };8826      if (!CBMemLocationAA->checkForAllAccessesToMemoryKind(8827              AccessPred, inverseLocation(NO_GLOBAL_MEM, false, false)))8828        return AccessedLocs.getWorstState();8829    }8830 8831    LLVM_DEBUG(8832        dbgs() << "[AAMemoryLocation] Accessed state before argument handling: "8833               << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");8834 8835    // Now handle argument memory if it might be accessed.8836    bool HasArgAccesses = ((~CBAssumedNotAccessedLocs) & NO_ARGUMENT_MEM);8837    if (HasArgAccesses)8838      categorizeArgumentPointerLocations(A, *CB, AccessedLocs, Changed);8839 8840    LLVM_DEBUG(8841        dbgs() << "[AAMemoryLocation] Accessed state after argument handling: "8842               << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");8843 8844    return AccessedLocs.getAssumed();8845  }8846 8847  if (const Value *Ptr = getPointerOperand(&I, /* AllowVolatile */ true)) {8848    LLVM_DEBUG(8849        dbgs() << "[AAMemoryLocation] Categorize memory access with pointer: "8850               << I << " [" << *Ptr << "]\n");8851    categorizePtrValue(A, I, *Ptr, AccessedLocs, Changed,8852                       Ptr->getType()->getPointerAddressSpace());8853    return AccessedLocs.getAssumed();8854  }8855 8856  LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Failed to categorize instruction: "8857                    << I << "\n");8858  updateStateAndAccessesMap(AccessedLocs, NO_UNKOWN_MEM, &I, nullptr, Changed,8859                            getAccessKindFromInst(&I));8860  return AccessedLocs.getAssumed();8861}8862 8863/// An AA to represent the memory behavior function attributes.8864struct AAMemoryLocationFunction final : public AAMemoryLocationImpl {8865  AAMemoryLocationFunction(const IRPosition &IRP, Attributor &A)8866      : AAMemoryLocationImpl(IRP, A) {}8867 8868  /// See AbstractAttribute::updateImpl(Attributor &A).8869  ChangeStatus updateImpl(Attributor &A) override {8870 8871    const auto *MemBehaviorAA =8872        A.getAAFor<AAMemoryBehavior>(*this, getIRPosition(), DepClassTy::NONE);8873    if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {8874      if (MemBehaviorAA->isKnownReadNone())8875        return indicateOptimisticFixpoint();8876      assert(isAssumedReadNone() &&8877             "AAMemoryLocation was not read-none but AAMemoryBehavior was!");8878      A.recordDependence(*MemBehaviorAA, *this, DepClassTy::OPTIONAL);8879      return ChangeStatus::UNCHANGED;8880    }8881 8882    // The current assumed state used to determine a change.8883    auto AssumedState = getAssumed();8884    bool Changed = false;8885 8886    auto CheckRWInst = [&](Instruction &I) {8887      MemoryLocationsKind MLK = categorizeAccessedLocations(A, I, Changed);8888      LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Accessed locations for " << I8889                        << ": " << getMemoryLocationsAsStr(MLK) << "\n");8890      removeAssumedBits(inverseLocation(MLK, false, false));8891      // Stop once only the valid bit set in the *not assumed location*, thus8892      // once we don't actually exclude any memory locations in the state.8893      return getAssumedNotAccessedLocation() != VALID_STATE;8894    };8895 8896    bool UsedAssumedInformation = false;8897    if (!A.checkForAllReadWriteInstructions(CheckRWInst, *this,8898                                            UsedAssumedInformation))8899      return indicatePessimisticFixpoint();8900 8901    Changed |= AssumedState != getAssumed();8902    return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;8903  }8904 8905  /// See AbstractAttribute::trackStatistics()8906  void trackStatistics() const override {8907    if (isAssumedReadNone())8908      STATS_DECLTRACK_FN_ATTR(readnone)8909    else if (isAssumedArgMemOnly())8910      STATS_DECLTRACK_FN_ATTR(argmemonly)8911    else if (isAssumedInaccessibleMemOnly())8912      STATS_DECLTRACK_FN_ATTR(inaccessiblememonly)8913    else if (isAssumedInaccessibleOrArgMemOnly())8914      STATS_DECLTRACK_FN_ATTR(inaccessiblememorargmemonly)8915  }8916};8917 8918/// AAMemoryLocation attribute for call sites.8919struct AAMemoryLocationCallSite final : AAMemoryLocationImpl {8920  AAMemoryLocationCallSite(const IRPosition &IRP, Attributor &A)8921      : AAMemoryLocationImpl(IRP, A) {}8922 8923  /// See AbstractAttribute::updateImpl(...).8924  ChangeStatus updateImpl(Attributor &A) override {8925    // TODO: Once we have call site specific value information we can provide8926    //       call site specific liveness liveness information and then it makes8927    //       sense to specialize attributes for call sites arguments instead of8928    //       redirecting requests to the callee argument.8929    Function *F = getAssociatedFunction();8930    const IRPosition &FnPos = IRPosition::function(*F);8931    auto *FnAA =8932        A.getAAFor<AAMemoryLocation>(*this, FnPos, DepClassTy::REQUIRED);8933    if (!FnAA)8934      return indicatePessimisticFixpoint();8935    bool Changed = false;8936    auto AccessPred = [&](const Instruction *I, const Value *Ptr,8937                          AccessKind Kind, MemoryLocationsKind MLK) {8938      updateStateAndAccessesMap(getState(), MLK, I, Ptr, Changed,8939                                getAccessKindFromInst(I));8940      return true;8941    };8942    if (!FnAA->checkForAllAccessesToMemoryKind(AccessPred, ALL_LOCATIONS))8943      return indicatePessimisticFixpoint();8944    return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;8945  }8946 8947  /// See AbstractAttribute::trackStatistics()8948  void trackStatistics() const override {8949    if (isAssumedReadNone())8950      STATS_DECLTRACK_CS_ATTR(readnone)8951  }8952};8953} // namespace8954 8955/// ------------------ denormal-fp-math Attribute -------------------------8956 8957namespace {8958struct AADenormalFPMathImpl : public AADenormalFPMath {8959  AADenormalFPMathImpl(const IRPosition &IRP, Attributor &A)8960      : AADenormalFPMath(IRP, A) {}8961 8962  const std::string getAsStr(Attributor *A) const override {8963    std::string Str("AADenormalFPMath[");8964    raw_string_ostream OS(Str);8965 8966    DenormalState Known = getKnown();8967    if (Known.Mode.isValid())8968      OS << "denormal-fp-math=" << Known.Mode;8969    else8970      OS << "invalid";8971 8972    if (Known.ModeF32.isValid())8973      OS << " denormal-fp-math-f32=" << Known.ModeF32;8974    OS << ']';8975    return Str;8976  }8977};8978 8979struct AADenormalFPMathFunction final : AADenormalFPMathImpl {8980  AADenormalFPMathFunction(const IRPosition &IRP, Attributor &A)8981      : AADenormalFPMathImpl(IRP, A) {}8982 8983  void initialize(Attributor &A) override {8984    const Function *F = getAnchorScope();8985    DenormalMode Mode = F->getDenormalModeRaw();8986    DenormalMode ModeF32 = F->getDenormalModeF32Raw();8987 8988    // TODO: Handling this here prevents handling the case where a callee has a8989    // fixed denormal-fp-math with dynamic denormal-fp-math-f32, but called from8990    // a function with a fully fixed mode.8991    if (ModeF32 == DenormalMode::getInvalid())8992      ModeF32 = Mode;8993    Known = DenormalState{Mode, ModeF32};8994    if (isModeFixed())8995      indicateFixpoint();8996  }8997 8998  ChangeStatus updateImpl(Attributor &A) override {8999    ChangeStatus Change = ChangeStatus::UNCHANGED;9000 9001    auto CheckCallSite = [=, &Change, &A](AbstractCallSite CS) {9002      Function *Caller = CS.getInstruction()->getFunction();9003      LLVM_DEBUG(dbgs() << "[AADenormalFPMath] Call " << Caller->getName()9004                        << "->" << getAssociatedFunction()->getName() << '\n');9005 9006      const auto *CallerInfo = A.getAAFor<AADenormalFPMath>(9007          *this, IRPosition::function(*Caller), DepClassTy::REQUIRED);9008      if (!CallerInfo)9009        return false;9010 9011      Change = Change | clampStateAndIndicateChange(this->getState(),9012                                                    CallerInfo->getState());9013      return true;9014    };9015 9016    bool AllCallSitesKnown = true;9017    if (!A.checkForAllCallSites(CheckCallSite, *this, true, AllCallSitesKnown))9018      return indicatePessimisticFixpoint();9019 9020    if (Change == ChangeStatus::CHANGED && isModeFixed())9021      indicateFixpoint();9022    return Change;9023  }9024 9025  ChangeStatus manifest(Attributor &A) override {9026    LLVMContext &Ctx = getAssociatedFunction()->getContext();9027 9028    SmallVector<Attribute, 2> AttrToAdd;9029    SmallVector<StringRef, 2> AttrToRemove;9030    if (Known.Mode == DenormalMode::getDefault()) {9031      AttrToRemove.push_back("denormal-fp-math");9032    } else {9033      AttrToAdd.push_back(9034          Attribute::get(Ctx, "denormal-fp-math", Known.Mode.str()));9035    }9036 9037    if (Known.ModeF32 != Known.Mode) {9038      AttrToAdd.push_back(9039          Attribute::get(Ctx, "denormal-fp-math-f32", Known.ModeF32.str()));9040    } else {9041      AttrToRemove.push_back("denormal-fp-math-f32");9042    }9043 9044    auto &IRP = getIRPosition();9045 9046    // TODO: There should be a combined add and remove API.9047    return A.removeAttrs(IRP, AttrToRemove) |9048           A.manifestAttrs(IRP, AttrToAdd, /*ForceReplace=*/true);9049  }9050 9051  void trackStatistics() const override {9052    STATS_DECLTRACK_FN_ATTR(denormal_fp_math)9053  }9054};9055} // namespace9056 9057/// ------------------ Value Constant Range Attribute -------------------------9058 9059namespace {9060struct AAValueConstantRangeImpl : AAValueConstantRange {9061  using StateType = IntegerRangeState;9062  AAValueConstantRangeImpl(const IRPosition &IRP, Attributor &A)9063      : AAValueConstantRange(IRP, A) {}9064 9065  /// See AbstractAttribute::initialize(..).9066  void initialize(Attributor &A) override {9067    if (A.hasSimplificationCallback(getIRPosition())) {9068      indicatePessimisticFixpoint();9069      return;9070    }9071 9072    // Intersect a range given by SCEV.9073    intersectKnown(getConstantRangeFromSCEV(A, getCtxI()));9074 9075    // Intersect a range given by LVI.9076    intersectKnown(getConstantRangeFromLVI(A, getCtxI()));9077  }9078 9079  /// See AbstractAttribute::getAsStr().9080  const std::string getAsStr(Attributor *A) const override {9081    std::string Str;9082    llvm::raw_string_ostream OS(Str);9083    OS << "range(" << getBitWidth() << ")<";9084    getKnown().print(OS);9085    OS << " / ";9086    getAssumed().print(OS);9087    OS << ">";9088    return Str;9089  }9090 9091  /// Helper function to get a SCEV expr for the associated value at program9092  /// point \p I.9093  const SCEV *getSCEV(Attributor &A, const Instruction *I = nullptr) const {9094    if (!getAnchorScope())9095      return nullptr;9096 9097    ScalarEvolution *SE =9098        A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(9099            *getAnchorScope());9100 9101    LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(9102        *getAnchorScope());9103 9104    if (!SE || !LI)9105      return nullptr;9106 9107    const SCEV *S = SE->getSCEV(&getAssociatedValue());9108    if (!I)9109      return S;9110 9111    return SE->getSCEVAtScope(S, LI->getLoopFor(I->getParent()));9112  }9113 9114  /// Helper function to get a range from SCEV for the associated value at9115  /// program point \p I.9116  ConstantRange getConstantRangeFromSCEV(Attributor &A,9117                                         const Instruction *I = nullptr) const {9118    if (!getAnchorScope())9119      return getWorstState(getBitWidth());9120 9121    ScalarEvolution *SE =9122        A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(9123            *getAnchorScope());9124 9125    const SCEV *S = getSCEV(A, I);9126    if (!SE || !S)9127      return getWorstState(getBitWidth());9128 9129    return SE->getUnsignedRange(S);9130  }9131 9132  /// Helper function to get a range from LVI for the associated value at9133  /// program point \p I.9134  ConstantRange9135  getConstantRangeFromLVI(Attributor &A,9136                          const Instruction *CtxI = nullptr) const {9137    if (!getAnchorScope())9138      return getWorstState(getBitWidth());9139 9140    LazyValueInfo *LVI =9141        A.getInfoCache().getAnalysisResultForFunction<LazyValueAnalysis>(9142            *getAnchorScope());9143 9144    if (!LVI || !CtxI)9145      return getWorstState(getBitWidth());9146    return LVI->getConstantRange(&getAssociatedValue(),9147                                 const_cast<Instruction *>(CtxI),9148                                 /*UndefAllowed*/ false);9149  }9150 9151  /// Return true if \p CtxI is valid for querying outside analyses.9152  /// This basically makes sure we do not ask intra-procedural analysis9153  /// about a context in the wrong function or a context that violates9154  /// dominance assumptions they might have. The \p AllowAACtxI flag indicates9155  /// if the original context of this AA is OK or should be considered invalid.9156  bool isValidCtxInstructionForOutsideAnalysis(Attributor &A,9157                                               const Instruction *CtxI,9158                                               bool AllowAACtxI) const {9159    if (!CtxI || (!AllowAACtxI && CtxI == getCtxI()))9160      return false;9161 9162    // Our context might be in a different function, neither intra-procedural9163    // analysis (ScalarEvolution nor LazyValueInfo) can handle that.9164    if (!AA::isValidInScope(getAssociatedValue(), CtxI->getFunction()))9165      return false;9166 9167    // If the context is not dominated by the value there are paths to the9168    // context that do not define the value. This cannot be handled by9169    // LazyValueInfo so we need to bail.9170    if (auto *I = dyn_cast<Instruction>(&getAssociatedValue())) {9171      InformationCache &InfoCache = A.getInfoCache();9172      const DominatorTree *DT =9173          InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(9174              *I->getFunction());9175      return DT && DT->dominates(I, CtxI);9176    }9177 9178    return true;9179  }9180 9181  /// See AAValueConstantRange::getKnownConstantRange(..).9182  ConstantRange9183  getKnownConstantRange(Attributor &A,9184                        const Instruction *CtxI = nullptr) const override {9185    if (!isValidCtxInstructionForOutsideAnalysis(A, CtxI,9186                                                 /* AllowAACtxI */ false))9187      return getKnown();9188 9189    ConstantRange LVIR = getConstantRangeFromLVI(A, CtxI);9190    ConstantRange SCEVR = getConstantRangeFromSCEV(A, CtxI);9191    return getKnown().intersectWith(SCEVR).intersectWith(LVIR);9192  }9193 9194  /// See AAValueConstantRange::getAssumedConstantRange(..).9195  ConstantRange9196  getAssumedConstantRange(Attributor &A,9197                          const Instruction *CtxI = nullptr) const override {9198    // TODO: Make SCEV use Attributor assumption.9199    //       We may be able to bound a variable range via assumptions in9200    //       Attributor. ex.) If x is assumed to be in [1, 3] and y is known to9201    //       evolve to x^2 + x, then we can say that y is in [2, 12].9202    if (!isValidCtxInstructionForOutsideAnalysis(A, CtxI,9203                                                 /* AllowAACtxI */ false))9204      return getAssumed();9205 9206    ConstantRange LVIR = getConstantRangeFromLVI(A, CtxI);9207    ConstantRange SCEVR = getConstantRangeFromSCEV(A, CtxI);9208    return getAssumed().intersectWith(SCEVR).intersectWith(LVIR);9209  }9210 9211  /// Helper function to create MDNode for range metadata.9212  static MDNode *9213  getMDNodeForConstantRange(Type *Ty, LLVMContext &Ctx,9214                            const ConstantRange &AssumedConstantRange) {9215    Metadata *LowAndHigh[] = {ConstantAsMetadata::get(ConstantInt::get(9216                                  Ty, AssumedConstantRange.getLower())),9217                              ConstantAsMetadata::get(ConstantInt::get(9218                                  Ty, AssumedConstantRange.getUpper()))};9219    return MDNode::get(Ctx, LowAndHigh);9220  }9221 9222  /// Return true if \p Assumed is included in ranges from instruction \p I.9223  static bool isBetterRange(const ConstantRange &Assumed,9224                            const Instruction &I) {9225    if (Assumed.isFullSet())9226      return false;9227 9228    std::optional<ConstantRange> Known;9229 9230    if (const auto *CB = dyn_cast<CallBase>(&I)) {9231      Known = CB->getRange();9232    } else if (MDNode *KnownRanges = I.getMetadata(LLVMContext::MD_range)) {9233      // If multiple ranges are annotated in IR, we give up to annotate assumed9234      // range for now.9235 9236      // TODO:  If there exists a known range which containts assumed range, we9237      // can say assumed range is better.9238      if (KnownRanges->getNumOperands() > 2)9239        return false;9240 9241      ConstantInt *Lower =9242          mdconst::extract<ConstantInt>(KnownRanges->getOperand(0));9243      ConstantInt *Upper =9244          mdconst::extract<ConstantInt>(KnownRanges->getOperand(1));9245 9246      Known.emplace(Lower->getValue(), Upper->getValue());9247    }9248    return !Known || (*Known != Assumed && Known->contains(Assumed));9249  }9250 9251  /// Helper function to set range metadata.9252  static bool9253  setRangeMetadataIfisBetterRange(Instruction *I,9254                                  const ConstantRange &AssumedConstantRange) {9255    if (isBetterRange(AssumedConstantRange, *I)) {9256      I->setMetadata(LLVMContext::MD_range,9257                     getMDNodeForConstantRange(I->getType(), I->getContext(),9258                                               AssumedConstantRange));9259      return true;9260    }9261    return false;9262  }9263  /// Helper function to set range return attribute.9264  static bool9265  setRangeRetAttrIfisBetterRange(Attributor &A, const IRPosition &IRP,9266                                 Instruction *I,9267                                 const ConstantRange &AssumedConstantRange) {9268    if (isBetterRange(AssumedConstantRange, *I)) {9269      A.manifestAttrs(IRP,9270                      Attribute::get(I->getContext(), Attribute::Range,9271                                     AssumedConstantRange),9272                      /*ForceReplace*/ true);9273      return true;9274    }9275    return false;9276  }9277 9278  /// See AbstractAttribute::manifest()9279  ChangeStatus manifest(Attributor &A) override {9280    ChangeStatus Changed = ChangeStatus::UNCHANGED;9281    ConstantRange AssumedConstantRange = getAssumedConstantRange(A);9282    assert(!AssumedConstantRange.isFullSet() && "Invalid state");9283 9284    auto &V = getAssociatedValue();9285    if (!AssumedConstantRange.isEmptySet() &&9286        !AssumedConstantRange.isSingleElement()) {9287      if (Instruction *I = dyn_cast<Instruction>(&V)) {9288        assert(I == getCtxI() && "Should not annotate an instruction which is "9289                                 "not the context instruction");9290        if (isa<LoadInst>(I))9291          if (setRangeMetadataIfisBetterRange(I, AssumedConstantRange))9292            Changed = ChangeStatus::CHANGED;9293        if (isa<CallInst>(I))9294          if (setRangeRetAttrIfisBetterRange(A, getIRPosition(), I,9295                                             AssumedConstantRange))9296            Changed = ChangeStatus::CHANGED;9297      }9298    }9299 9300    return Changed;9301  }9302};9303 9304struct AAValueConstantRangeArgument final9305    : AAArgumentFromCallSiteArguments<9306          AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,9307          true /* BridgeCallBaseContext */> {9308  using Base = AAArgumentFromCallSiteArguments<9309      AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,9310      true /* BridgeCallBaseContext */>;9311  AAValueConstantRangeArgument(const IRPosition &IRP, Attributor &A)9312      : Base(IRP, A) {}9313 9314  /// See AbstractAttribute::trackStatistics()9315  void trackStatistics() const override {9316    STATS_DECLTRACK_ARG_ATTR(value_range)9317  }9318};9319 9320struct AAValueConstantRangeReturned9321    : AAReturnedFromReturnedValues<AAValueConstantRange,9322                                   AAValueConstantRangeImpl,9323                                   AAValueConstantRangeImpl::StateType,9324                                   /* PropagateCallBaseContext */ true> {9325  using Base =9326      AAReturnedFromReturnedValues<AAValueConstantRange,9327                                   AAValueConstantRangeImpl,9328                                   AAValueConstantRangeImpl::StateType,9329                                   /* PropagateCallBaseContext */ true>;9330  AAValueConstantRangeReturned(const IRPosition &IRP, Attributor &A)9331      : Base(IRP, A) {}9332 9333  /// See AbstractAttribute::initialize(...).9334  void initialize(Attributor &A) override {9335    if (!A.isFunctionIPOAmendable(*getAssociatedFunction()))9336      indicatePessimisticFixpoint();9337  }9338 9339  /// See AbstractAttribute::trackStatistics()9340  void trackStatistics() const override {9341    STATS_DECLTRACK_FNRET_ATTR(value_range)9342  }9343};9344 9345struct AAValueConstantRangeFloating : AAValueConstantRangeImpl {9346  AAValueConstantRangeFloating(const IRPosition &IRP, Attributor &A)9347      : AAValueConstantRangeImpl(IRP, A) {}9348 9349  /// See AbstractAttribute::initialize(...).9350  void initialize(Attributor &A) override {9351    AAValueConstantRangeImpl::initialize(A);9352    if (isAtFixpoint())9353      return;9354 9355    Value &V = getAssociatedValue();9356 9357    if (auto *C = dyn_cast<ConstantInt>(&V)) {9358      unionAssumed(ConstantRange(C->getValue()));9359      indicateOptimisticFixpoint();9360      return;9361    }9362 9363    if (isa<UndefValue>(&V)) {9364      // Collapse the undef state to 0.9365      unionAssumed(ConstantRange(APInt(getBitWidth(), 0)));9366      indicateOptimisticFixpoint();9367      return;9368    }9369 9370    if (isa<CallBase>(&V))9371      return;9372 9373    if (isa<BinaryOperator>(&V) || isa<CmpInst>(&V) || isa<CastInst>(&V))9374      return;9375 9376    // If it is a load instruction with range metadata, use it.9377    if (LoadInst *LI = dyn_cast<LoadInst>(&V))9378      if (auto *RangeMD = LI->getMetadata(LLVMContext::MD_range)) {9379        intersectKnown(getConstantRangeFromMetadata(*RangeMD));9380        return;9381      }9382 9383    // We can work with PHI and select instruction as we traverse their operands9384    // during update.9385    if (isa<SelectInst>(V) || isa<PHINode>(V))9386      return;9387 9388    // Otherwise we give up.9389    indicatePessimisticFixpoint();9390 9391    LLVM_DEBUG(dbgs() << "[AAValueConstantRange] We give up: "9392                      << getAssociatedValue() << "\n");9393  }9394 9395  bool calculateBinaryOperator(9396      Attributor &A, BinaryOperator *BinOp, IntegerRangeState &T,9397      const Instruction *CtxI,9398      SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {9399    Value *LHS = BinOp->getOperand(0);9400    Value *RHS = BinOp->getOperand(1);9401 9402    // Simplify the operands first.9403    bool UsedAssumedInformation = false;9404    const auto &SimplifiedLHS = A.getAssumedSimplified(9405        IRPosition::value(*LHS, getCallBaseContext()), *this,9406        UsedAssumedInformation, AA::Interprocedural);9407    if (!SimplifiedLHS.has_value())9408      return true;9409    if (!*SimplifiedLHS)9410      return false;9411    LHS = *SimplifiedLHS;9412 9413    const auto &SimplifiedRHS = A.getAssumedSimplified(9414        IRPosition::value(*RHS, getCallBaseContext()), *this,9415        UsedAssumedInformation, AA::Interprocedural);9416    if (!SimplifiedRHS.has_value())9417      return true;9418    if (!*SimplifiedRHS)9419      return false;9420    RHS = *SimplifiedRHS;9421 9422    // TODO: Allow non integers as well.9423    if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())9424      return false;9425 9426    auto *LHSAA = A.getAAFor<AAValueConstantRange>(9427        *this, IRPosition::value(*LHS, getCallBaseContext()),9428        DepClassTy::REQUIRED);9429    if (!LHSAA)9430      return false;9431    QuerriedAAs.push_back(LHSAA);9432    auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);9433 9434    auto *RHSAA = A.getAAFor<AAValueConstantRange>(9435        *this, IRPosition::value(*RHS, getCallBaseContext()),9436        DepClassTy::REQUIRED);9437    if (!RHSAA)9438      return false;9439    QuerriedAAs.push_back(RHSAA);9440    auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);9441 9442    auto AssumedRange = LHSAARange.binaryOp(BinOp->getOpcode(), RHSAARange);9443 9444    T.unionAssumed(AssumedRange);9445 9446    // TODO: Track a known state too.9447 9448    return T.isValidState();9449  }9450 9451  bool calculateCastInst(9452      Attributor &A, CastInst *CastI, IntegerRangeState &T,9453      const Instruction *CtxI,9454      SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {9455    assert(CastI->getNumOperands() == 1 && "Expected cast to be unary!");9456    // TODO: Allow non integers as well.9457    Value *OpV = CastI->getOperand(0);9458 9459    // Simplify the operand first.9460    bool UsedAssumedInformation = false;9461    const auto &SimplifiedOpV = A.getAssumedSimplified(9462        IRPosition::value(*OpV, getCallBaseContext()), *this,9463        UsedAssumedInformation, AA::Interprocedural);9464    if (!SimplifiedOpV.has_value())9465      return true;9466    if (!*SimplifiedOpV)9467      return false;9468    OpV = *SimplifiedOpV;9469 9470    if (!OpV->getType()->isIntegerTy())9471      return false;9472 9473    auto *OpAA = A.getAAFor<AAValueConstantRange>(9474        *this, IRPosition::value(*OpV, getCallBaseContext()),9475        DepClassTy::REQUIRED);9476    if (!OpAA)9477      return false;9478    QuerriedAAs.push_back(OpAA);9479    T.unionAssumed(OpAA->getAssumed().castOp(CastI->getOpcode(),9480                                             getState().getBitWidth()));9481    return T.isValidState();9482  }9483 9484  bool9485  calculateCmpInst(Attributor &A, CmpInst *CmpI, IntegerRangeState &T,9486                   const Instruction *CtxI,9487                   SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {9488    Value *LHS = CmpI->getOperand(0);9489    Value *RHS = CmpI->getOperand(1);9490 9491    // Simplify the operands first.9492    bool UsedAssumedInformation = false;9493    const auto &SimplifiedLHS = A.getAssumedSimplified(9494        IRPosition::value(*LHS, getCallBaseContext()), *this,9495        UsedAssumedInformation, AA::Interprocedural);9496    if (!SimplifiedLHS.has_value())9497      return true;9498    if (!*SimplifiedLHS)9499      return false;9500    LHS = *SimplifiedLHS;9501 9502    const auto &SimplifiedRHS = A.getAssumedSimplified(9503        IRPosition::value(*RHS, getCallBaseContext()), *this,9504        UsedAssumedInformation, AA::Interprocedural);9505    if (!SimplifiedRHS.has_value())9506      return true;9507    if (!*SimplifiedRHS)9508      return false;9509    RHS = *SimplifiedRHS;9510 9511    // TODO: Allow non integers as well.9512    if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())9513      return false;9514 9515    auto *LHSAA = A.getAAFor<AAValueConstantRange>(9516        *this, IRPosition::value(*LHS, getCallBaseContext()),9517        DepClassTy::REQUIRED);9518    if (!LHSAA)9519      return false;9520    QuerriedAAs.push_back(LHSAA);9521    auto *RHSAA = A.getAAFor<AAValueConstantRange>(9522        *this, IRPosition::value(*RHS, getCallBaseContext()),9523        DepClassTy::REQUIRED);9524    if (!RHSAA)9525      return false;9526    QuerriedAAs.push_back(RHSAA);9527    auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);9528    auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);9529 9530    // If one of them is empty set, we can't decide.9531    if (LHSAARange.isEmptySet() || RHSAARange.isEmptySet())9532      return true;9533 9534    bool MustTrue = false, MustFalse = false;9535 9536    auto AllowedRegion =9537        ConstantRange::makeAllowedICmpRegion(CmpI->getPredicate(), RHSAARange);9538 9539    if (AllowedRegion.intersectWith(LHSAARange).isEmptySet())9540      MustFalse = true;9541 9542    if (LHSAARange.icmp(CmpI->getPredicate(), RHSAARange))9543      MustTrue = true;9544 9545    assert((!MustTrue || !MustFalse) &&9546           "Either MustTrue or MustFalse should be false!");9547 9548    if (MustTrue)9549      T.unionAssumed(ConstantRange(APInt(/* numBits */ 1, /* val */ 1)));9550    else if (MustFalse)9551      T.unionAssumed(ConstantRange(APInt(/* numBits */ 1, /* val */ 0)));9552    else9553      T.unionAssumed(ConstantRange(/* BitWidth */ 1, /* isFullSet */ true));9554 9555    LLVM_DEBUG(dbgs() << "[AAValueConstantRange] " << *CmpI << " after "9556                      << (MustTrue ? "true" : (MustFalse ? "false" : "unknown"))9557                      << ": " << T << "\n\t" << *LHSAA << "\t<op>\n\t"9558                      << *RHSAA);9559 9560    // TODO: Track a known state too.9561    return T.isValidState();9562  }9563 9564  /// See AbstractAttribute::updateImpl(...).9565  ChangeStatus updateImpl(Attributor &A) override {9566 9567    IntegerRangeState T(getBitWidth());9568    auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {9569      Instruction *I = dyn_cast<Instruction>(&V);9570      if (!I || isa<CallBase>(I)) {9571 9572        // Simplify the operand first.9573        bool UsedAssumedInformation = false;9574        const auto &SimplifiedOpV = A.getAssumedSimplified(9575            IRPosition::value(V, getCallBaseContext()), *this,9576            UsedAssumedInformation, AA::Interprocedural);9577        if (!SimplifiedOpV.has_value())9578          return true;9579        if (!*SimplifiedOpV)9580          return false;9581        Value *VPtr = *SimplifiedOpV;9582 9583        // If the value is not instruction, we query AA to Attributor.9584        const auto *AA = A.getAAFor<AAValueConstantRange>(9585            *this, IRPosition::value(*VPtr, getCallBaseContext()),9586            DepClassTy::REQUIRED);9587 9588        // Clamp operator is not used to utilize a program point CtxI.9589        if (AA)9590          T.unionAssumed(AA->getAssumedConstantRange(A, CtxI));9591        else9592          return false;9593 9594        return T.isValidState();9595      }9596 9597      SmallVector<const AAValueConstantRange *, 4> QuerriedAAs;9598      if (auto *BinOp = dyn_cast<BinaryOperator>(I)) {9599        if (!calculateBinaryOperator(A, BinOp, T, CtxI, QuerriedAAs))9600          return false;9601      } else if (auto *CmpI = dyn_cast<CmpInst>(I)) {9602        if (!calculateCmpInst(A, CmpI, T, CtxI, QuerriedAAs))9603          return false;9604      } else if (auto *CastI = dyn_cast<CastInst>(I)) {9605        if (!calculateCastInst(A, CastI, T, CtxI, QuerriedAAs))9606          return false;9607      } else {9608        // Give up with other instructions.9609        // TODO: Add other instructions9610 9611        T.indicatePessimisticFixpoint();9612        return false;9613      }9614 9615      // Catch circular reasoning in a pessimistic way for now.9616      // TODO: Check how the range evolves and if we stripped anything, see also9617      //       AADereferenceable or AAAlign for similar situations.9618      for (const AAValueConstantRange *QueriedAA : QuerriedAAs) {9619        if (QueriedAA != this)9620          continue;9621        // If we are in a stady state we do not need to worry.9622        if (T.getAssumed() == getState().getAssumed())9623          continue;9624        T.indicatePessimisticFixpoint();9625      }9626 9627      return T.isValidState();9628    };9629 9630    if (!VisitValueCB(getAssociatedValue(), getCtxI()))9631      return indicatePessimisticFixpoint();9632 9633    // Ensure that long def-use chains can't cause circular reasoning either by9634    // introducing a cutoff below.9635    if (clampStateAndIndicateChange(getState(), T) == ChangeStatus::UNCHANGED)9636      return ChangeStatus::UNCHANGED;9637    if (++NumChanges > MaxNumChanges) {9638      LLVM_DEBUG(dbgs() << "[AAValueConstantRange] performed " << NumChanges9639                        << " but only " << MaxNumChanges9640                        << " are allowed to avoid cyclic reasoning.");9641      return indicatePessimisticFixpoint();9642    }9643    return ChangeStatus::CHANGED;9644  }9645 9646  /// See AbstractAttribute::trackStatistics()9647  void trackStatistics() const override {9648    STATS_DECLTRACK_FLOATING_ATTR(value_range)9649  }9650 9651  /// Tracker to bail after too many widening steps of the constant range.9652  int NumChanges = 0;9653 9654  /// Upper bound for the number of allowed changes (=widening steps) for the9655  /// constant range before we give up.9656  static constexpr int MaxNumChanges = 5;9657};9658 9659struct AAValueConstantRangeFunction : AAValueConstantRangeImpl {9660  AAValueConstantRangeFunction(const IRPosition &IRP, Attributor &A)9661      : AAValueConstantRangeImpl(IRP, A) {}9662 9663  /// See AbstractAttribute::initialize(...).9664  ChangeStatus updateImpl(Attributor &A) override {9665    llvm_unreachable("AAValueConstantRange(Function|CallSite)::updateImpl will "9666                     "not be called");9667  }9668 9669  /// See AbstractAttribute::trackStatistics()9670  void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(value_range) }9671};9672 9673struct AAValueConstantRangeCallSite : AAValueConstantRangeFunction {9674  AAValueConstantRangeCallSite(const IRPosition &IRP, Attributor &A)9675      : AAValueConstantRangeFunction(IRP, A) {}9676 9677  /// See AbstractAttribute::trackStatistics()9678  void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(value_range) }9679};9680 9681struct AAValueConstantRangeCallSiteReturned9682    : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,9683                         AAValueConstantRangeImpl::StateType,9684                         /* IntroduceCallBaseContext */ true> {9685  AAValueConstantRangeCallSiteReturned(const IRPosition &IRP, Attributor &A)9686      : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,9687                           AAValueConstantRangeImpl::StateType,9688                           /* IntroduceCallBaseContext */ true>(IRP, A) {}9689 9690  /// See AbstractAttribute::initialize(...).9691  void initialize(Attributor &A) override {9692    // If it is a call instruction with range attribute, use the range.9693    if (CallInst *CI = dyn_cast<CallInst>(&getAssociatedValue())) {9694      if (std::optional<ConstantRange> Range = CI->getRange())9695        intersectKnown(*Range);9696    }9697 9698    AAValueConstantRangeImpl::initialize(A);9699  }9700 9701  /// See AbstractAttribute::trackStatistics()9702  void trackStatistics() const override {9703    STATS_DECLTRACK_CSRET_ATTR(value_range)9704  }9705};9706struct AAValueConstantRangeCallSiteArgument : AAValueConstantRangeFloating {9707  AAValueConstantRangeCallSiteArgument(const IRPosition &IRP, Attributor &A)9708      : AAValueConstantRangeFloating(IRP, A) {}9709 9710  /// See AbstractAttribute::manifest()9711  ChangeStatus manifest(Attributor &A) override {9712    return ChangeStatus::UNCHANGED;9713  }9714 9715  /// See AbstractAttribute::trackStatistics()9716  void trackStatistics() const override {9717    STATS_DECLTRACK_CSARG_ATTR(value_range)9718  }9719};9720} // namespace9721 9722/// ------------------ Potential Values Attribute -------------------------9723 9724namespace {9725struct AAPotentialConstantValuesImpl : AAPotentialConstantValues {9726  using StateType = PotentialConstantIntValuesState;9727 9728  AAPotentialConstantValuesImpl(const IRPosition &IRP, Attributor &A)9729      : AAPotentialConstantValues(IRP, A) {}9730 9731  /// See AbstractAttribute::initialize(..).9732  void initialize(Attributor &A) override {9733    if (A.hasSimplificationCallback(getIRPosition()))9734      indicatePessimisticFixpoint();9735    else9736      AAPotentialConstantValues::initialize(A);9737  }9738 9739  bool fillSetWithConstantValues(Attributor &A, const IRPosition &IRP, SetTy &S,9740                                 bool &ContainsUndef, bool ForSelf) {9741    SmallVector<AA::ValueAndContext> Values;9742    bool UsedAssumedInformation = false;9743    if (!A.getAssumedSimplifiedValues(IRP, *this, Values, AA::Interprocedural,9744                                      UsedAssumedInformation)) {9745      // Avoid recursion when the caller is computing constant values for this9746      // IRP itself.9747      if (ForSelf)9748        return false;9749      if (!IRP.getAssociatedType()->isIntegerTy())9750        return false;9751      auto *PotentialValuesAA = A.getAAFor<AAPotentialConstantValues>(9752          *this, IRP, DepClassTy::REQUIRED);9753      if (!PotentialValuesAA || !PotentialValuesAA->getState().isValidState())9754        return false;9755      ContainsUndef = PotentialValuesAA->getState().undefIsContained();9756      S = PotentialValuesAA->getState().getAssumedSet();9757      return true;9758    }9759 9760    // Copy all the constant values, except UndefValue. ContainsUndef is true9761    // iff Values contains only UndefValue instances. If there are other known9762    // constants, then UndefValue is dropped.9763    ContainsUndef = false;9764    for (auto &It : Values) {9765      if (isa<UndefValue>(It.getValue())) {9766        ContainsUndef = true;9767        continue;9768      }9769      auto *CI = dyn_cast<ConstantInt>(It.getValue());9770      if (!CI)9771        return false;9772      S.insert(CI->getValue());9773    }9774    ContainsUndef &= S.empty();9775 9776    return true;9777  }9778 9779  /// See AbstractAttribute::getAsStr().9780  const std::string getAsStr(Attributor *A) const override {9781    std::string Str;9782    llvm::raw_string_ostream OS(Str);9783    OS << getState();9784    return Str;9785  }9786 9787  /// See AbstractAttribute::updateImpl(...).9788  ChangeStatus updateImpl(Attributor &A) override {9789    return indicatePessimisticFixpoint();9790  }9791};9792 9793struct AAPotentialConstantValuesArgument final9794    : AAArgumentFromCallSiteArguments<AAPotentialConstantValues,9795                                      AAPotentialConstantValuesImpl,9796                                      PotentialConstantIntValuesState> {9797  using Base = AAArgumentFromCallSiteArguments<AAPotentialConstantValues,9798                                               AAPotentialConstantValuesImpl,9799                                               PotentialConstantIntValuesState>;9800  AAPotentialConstantValuesArgument(const IRPosition &IRP, Attributor &A)9801      : Base(IRP, A) {}9802 9803  /// See AbstractAttribute::trackStatistics()9804  void trackStatistics() const override {9805    STATS_DECLTRACK_ARG_ATTR(potential_values)9806  }9807};9808 9809struct AAPotentialConstantValuesReturned9810    : AAReturnedFromReturnedValues<AAPotentialConstantValues,9811                                   AAPotentialConstantValuesImpl> {9812  using Base = AAReturnedFromReturnedValues<AAPotentialConstantValues,9813                                            AAPotentialConstantValuesImpl>;9814  AAPotentialConstantValuesReturned(const IRPosition &IRP, Attributor &A)9815      : Base(IRP, A) {}9816 9817  void initialize(Attributor &A) override {9818    if (!A.isFunctionIPOAmendable(*getAssociatedFunction()))9819      indicatePessimisticFixpoint();9820    Base::initialize(A);9821  }9822 9823  /// See AbstractAttribute::trackStatistics()9824  void trackStatistics() const override {9825    STATS_DECLTRACK_FNRET_ATTR(potential_values)9826  }9827};9828 9829struct AAPotentialConstantValuesFloating : AAPotentialConstantValuesImpl {9830  AAPotentialConstantValuesFloating(const IRPosition &IRP, Attributor &A)9831      : AAPotentialConstantValuesImpl(IRP, A) {}9832 9833  /// See AbstractAttribute::initialize(..).9834  void initialize(Attributor &A) override {9835    AAPotentialConstantValuesImpl::initialize(A);9836    if (isAtFixpoint())9837      return;9838 9839    Value &V = getAssociatedValue();9840 9841    if (auto *C = dyn_cast<ConstantInt>(&V)) {9842      unionAssumed(C->getValue());9843      indicateOptimisticFixpoint();9844      return;9845    }9846 9847    if (isa<UndefValue>(&V)) {9848      unionAssumedWithUndef();9849      indicateOptimisticFixpoint();9850      return;9851    }9852 9853    if (isa<BinaryOperator>(&V) || isa<ICmpInst>(&V) || isa<CastInst>(&V))9854      return;9855 9856    if (isa<SelectInst>(V) || isa<PHINode>(V) || isa<LoadInst>(V))9857      return;9858 9859    indicatePessimisticFixpoint();9860 9861    LLVM_DEBUG(dbgs() << "[AAPotentialConstantValues] We give up: "9862                      << getAssociatedValue() << "\n");9863  }9864 9865  static bool calculateICmpInst(const ICmpInst *ICI, const APInt &LHS,9866                                const APInt &RHS) {9867    return ICmpInst::compare(LHS, RHS, ICI->getPredicate());9868  }9869 9870  static APInt calculateCastInst(const CastInst *CI, const APInt &Src,9871                                 uint32_t ResultBitWidth) {9872    Instruction::CastOps CastOp = CI->getOpcode();9873    switch (CastOp) {9874    default:9875      llvm_unreachable("unsupported or not integer cast");9876    case Instruction::Trunc:9877      return Src.trunc(ResultBitWidth);9878    case Instruction::SExt:9879      return Src.sext(ResultBitWidth);9880    case Instruction::ZExt:9881      return Src.zext(ResultBitWidth);9882    case Instruction::BitCast:9883      return Src;9884    }9885  }9886 9887  static APInt calculateBinaryOperator(const BinaryOperator *BinOp,9888                                       const APInt &LHS, const APInt &RHS,9889                                       bool &SkipOperation, bool &Unsupported) {9890    Instruction::BinaryOps BinOpcode = BinOp->getOpcode();9891    // Unsupported is set to true when the binary operator is not supported.9892    // SkipOperation is set to true when UB occur with the given operand pair9893    // (LHS, RHS).9894    // TODO: we should look at nsw and nuw keywords to handle operations9895    //       that create poison or undef value.9896    switch (BinOpcode) {9897    default:9898      Unsupported = true;9899      return LHS;9900    case Instruction::Add:9901      return LHS + RHS;9902    case Instruction::Sub:9903      return LHS - RHS;9904    case Instruction::Mul:9905      return LHS * RHS;9906    case Instruction::UDiv:9907      if (RHS.isZero()) {9908        SkipOperation = true;9909        return LHS;9910      }9911      return LHS.udiv(RHS);9912    case Instruction::SDiv:9913      if (RHS.isZero()) {9914        SkipOperation = true;9915        return LHS;9916      }9917      return LHS.sdiv(RHS);9918    case Instruction::URem:9919      if (RHS.isZero()) {9920        SkipOperation = true;9921        return LHS;9922      }9923      return LHS.urem(RHS);9924    case Instruction::SRem:9925      if (RHS.isZero()) {9926        SkipOperation = true;9927        return LHS;9928      }9929      return LHS.srem(RHS);9930    case Instruction::Shl:9931      return LHS.shl(RHS);9932    case Instruction::LShr:9933      return LHS.lshr(RHS);9934    case Instruction::AShr:9935      return LHS.ashr(RHS);9936    case Instruction::And:9937      return LHS & RHS;9938    case Instruction::Or:9939      return LHS | RHS;9940    case Instruction::Xor:9941      return LHS ^ RHS;9942    }9943  }9944 9945  bool calculateBinaryOperatorAndTakeUnion(const BinaryOperator *BinOp,9946                                           const APInt &LHS, const APInt &RHS) {9947    bool SkipOperation = false;9948    bool Unsupported = false;9949    APInt Result =9950        calculateBinaryOperator(BinOp, LHS, RHS, SkipOperation, Unsupported);9951    if (Unsupported)9952      return false;9953    // If SkipOperation is true, we can ignore this operand pair (L, R).9954    if (!SkipOperation)9955      unionAssumed(Result);9956    return isValidState();9957  }9958 9959  ChangeStatus updateWithICmpInst(Attributor &A, ICmpInst *ICI) {9960    auto AssumedBefore = getAssumed();9961    Value *LHS = ICI->getOperand(0);9962    Value *RHS = ICI->getOperand(1);9963 9964    bool LHSContainsUndef = false, RHSContainsUndef = false;9965    SetTy LHSAAPVS, RHSAAPVS;9966    if (!fillSetWithConstantValues(A, IRPosition::value(*LHS), LHSAAPVS,9967                                   LHSContainsUndef, /* ForSelf */ false) ||9968        !fillSetWithConstantValues(A, IRPosition::value(*RHS), RHSAAPVS,9969                                   RHSContainsUndef, /* ForSelf */ false))9970      return indicatePessimisticFixpoint();9971 9972    // TODO: make use of undef flag to limit potential values aggressively.9973    bool MaybeTrue = false, MaybeFalse = false;9974    const APInt Zero(RHS->getType()->getIntegerBitWidth(), 0);9975    if (LHSContainsUndef && RHSContainsUndef) {9976      // The result of any comparison between undefs can be soundly replaced9977      // with undef.9978      unionAssumedWithUndef();9979    } else if (LHSContainsUndef) {9980      for (const APInt &R : RHSAAPVS) {9981        bool CmpResult = calculateICmpInst(ICI, Zero, R);9982        MaybeTrue |= CmpResult;9983        MaybeFalse |= !CmpResult;9984        if (MaybeTrue & MaybeFalse)9985          return indicatePessimisticFixpoint();9986      }9987    } else if (RHSContainsUndef) {9988      for (const APInt &L : LHSAAPVS) {9989        bool CmpResult = calculateICmpInst(ICI, L, Zero);9990        MaybeTrue |= CmpResult;9991        MaybeFalse |= !CmpResult;9992        if (MaybeTrue & MaybeFalse)9993          return indicatePessimisticFixpoint();9994      }9995    } else {9996      for (const APInt &L : LHSAAPVS) {9997        for (const APInt &R : RHSAAPVS) {9998          bool CmpResult = calculateICmpInst(ICI, L, R);9999          MaybeTrue |= CmpResult;10000          MaybeFalse |= !CmpResult;10001          if (MaybeTrue & MaybeFalse)10002            return indicatePessimisticFixpoint();10003        }10004      }10005    }10006    if (MaybeTrue)10007      unionAssumed(APInt(/* numBits */ 1, /* val */ 1));10008    if (MaybeFalse)10009      unionAssumed(APInt(/* numBits */ 1, /* val */ 0));10010    return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED10011                                         : ChangeStatus::CHANGED;10012  }10013 10014  ChangeStatus updateWithSelectInst(Attributor &A, SelectInst *SI) {10015    auto AssumedBefore = getAssumed();10016    Value *LHS = SI->getTrueValue();10017    Value *RHS = SI->getFalseValue();10018 10019    bool UsedAssumedInformation = false;10020    std::optional<Constant *> C = A.getAssumedConstant(10021        *SI->getCondition(), *this, UsedAssumedInformation);10022 10023    // Check if we only need one operand.10024    bool OnlyLeft = false, OnlyRight = false;10025    if (C && *C && (*C)->isOneValue())10026      OnlyLeft = true;10027    else if (C && *C && (*C)->isZeroValue())10028      OnlyRight = true;10029 10030    bool LHSContainsUndef = false, RHSContainsUndef = false;10031    SetTy LHSAAPVS, RHSAAPVS;10032    if (!OnlyRight &&10033        !fillSetWithConstantValues(A, IRPosition::value(*LHS), LHSAAPVS,10034                                   LHSContainsUndef, /* ForSelf */ false))10035      return indicatePessimisticFixpoint();10036 10037    if (!OnlyLeft &&10038        !fillSetWithConstantValues(A, IRPosition::value(*RHS), RHSAAPVS,10039                                   RHSContainsUndef, /* ForSelf */ false))10040      return indicatePessimisticFixpoint();10041 10042    if (OnlyLeft || OnlyRight) {10043      // select (true/false), lhs, rhs10044      auto *OpAA = OnlyLeft ? &LHSAAPVS : &RHSAAPVS;10045      auto Undef = OnlyLeft ? LHSContainsUndef : RHSContainsUndef;10046 10047      if (Undef)10048        unionAssumedWithUndef();10049      else {10050        for (const auto &It : *OpAA)10051          unionAssumed(It);10052      }10053 10054    } else if (LHSContainsUndef && RHSContainsUndef) {10055      // select i1 *, undef , undef => undef10056      unionAssumedWithUndef();10057    } else {10058      for (const auto &It : LHSAAPVS)10059        unionAssumed(It);10060      for (const auto &It : RHSAAPVS)10061        unionAssumed(It);10062    }10063    return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED10064                                         : ChangeStatus::CHANGED;10065  }10066 10067  ChangeStatus updateWithCastInst(Attributor &A, CastInst *CI) {10068    auto AssumedBefore = getAssumed();10069    if (!CI->isIntegerCast())10070      return indicatePessimisticFixpoint();10071    assert(CI->getNumOperands() == 1 && "Expected cast to be unary!");10072    uint32_t ResultBitWidth = CI->getDestTy()->getIntegerBitWidth();10073    Value *Src = CI->getOperand(0);10074 10075    bool SrcContainsUndef = false;10076    SetTy SrcPVS;10077    if (!fillSetWithConstantValues(A, IRPosition::value(*Src), SrcPVS,10078                                   SrcContainsUndef, /* ForSelf */ false))10079      return indicatePessimisticFixpoint();10080 10081    if (SrcContainsUndef)10082      unionAssumedWithUndef();10083    else {10084      for (const APInt &S : SrcPVS) {10085        APInt T = calculateCastInst(CI, S, ResultBitWidth);10086        unionAssumed(T);10087      }10088    }10089    return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED10090                                         : ChangeStatus::CHANGED;10091  }10092 10093  ChangeStatus updateWithBinaryOperator(Attributor &A, BinaryOperator *BinOp) {10094    auto AssumedBefore = getAssumed();10095    Value *LHS = BinOp->getOperand(0);10096    Value *RHS = BinOp->getOperand(1);10097 10098    bool LHSContainsUndef = false, RHSContainsUndef = false;10099    SetTy LHSAAPVS, RHSAAPVS;10100    if (!fillSetWithConstantValues(A, IRPosition::value(*LHS), LHSAAPVS,10101                                   LHSContainsUndef, /* ForSelf */ false) ||10102        !fillSetWithConstantValues(A, IRPosition::value(*RHS), RHSAAPVS,10103                                   RHSContainsUndef, /* ForSelf */ false))10104      return indicatePessimisticFixpoint();10105 10106    const APInt Zero = APInt(LHS->getType()->getIntegerBitWidth(), 0);10107 10108    // TODO: make use of undef flag to limit potential values aggressively.10109    if (LHSContainsUndef && RHSContainsUndef) {10110      if (!calculateBinaryOperatorAndTakeUnion(BinOp, Zero, Zero))10111        return indicatePessimisticFixpoint();10112    } else if (LHSContainsUndef) {10113      for (const APInt &R : RHSAAPVS) {10114        if (!calculateBinaryOperatorAndTakeUnion(BinOp, Zero, R))10115          return indicatePessimisticFixpoint();10116      }10117    } else if (RHSContainsUndef) {10118      for (const APInt &L : LHSAAPVS) {10119        if (!calculateBinaryOperatorAndTakeUnion(BinOp, L, Zero))10120          return indicatePessimisticFixpoint();10121      }10122    } else {10123      for (const APInt &L : LHSAAPVS) {10124        for (const APInt &R : RHSAAPVS) {10125          if (!calculateBinaryOperatorAndTakeUnion(BinOp, L, R))10126            return indicatePessimisticFixpoint();10127        }10128      }10129    }10130    return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED10131                                         : ChangeStatus::CHANGED;10132  }10133 10134  ChangeStatus updateWithInstruction(Attributor &A, Instruction *Inst) {10135    auto AssumedBefore = getAssumed();10136    SetTy Incoming;10137    bool ContainsUndef;10138    if (!fillSetWithConstantValues(A, IRPosition::value(*Inst), Incoming,10139                                   ContainsUndef, /* ForSelf */ true))10140      return indicatePessimisticFixpoint();10141    if (ContainsUndef) {10142      unionAssumedWithUndef();10143    } else {10144      for (const auto &It : Incoming)10145        unionAssumed(It);10146    }10147    return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED10148                                         : ChangeStatus::CHANGED;10149  }10150 10151  /// See AbstractAttribute::updateImpl(...).10152  ChangeStatus updateImpl(Attributor &A) override {10153    Value &V = getAssociatedValue();10154    Instruction *I = dyn_cast<Instruction>(&V);10155 10156    if (auto *ICI = dyn_cast<ICmpInst>(I))10157      return updateWithICmpInst(A, ICI);10158 10159    if (auto *SI = dyn_cast<SelectInst>(I))10160      return updateWithSelectInst(A, SI);10161 10162    if (auto *CI = dyn_cast<CastInst>(I))10163      return updateWithCastInst(A, CI);10164 10165    if (auto *BinOp = dyn_cast<BinaryOperator>(I))10166      return updateWithBinaryOperator(A, BinOp);10167 10168    if (isa<PHINode>(I) || isa<LoadInst>(I))10169      return updateWithInstruction(A, I);10170 10171    return indicatePessimisticFixpoint();10172  }10173 10174  /// See AbstractAttribute::trackStatistics()10175  void trackStatistics() const override {10176    STATS_DECLTRACK_FLOATING_ATTR(potential_values)10177  }10178};10179 10180struct AAPotentialConstantValuesFunction : AAPotentialConstantValuesImpl {10181  AAPotentialConstantValuesFunction(const IRPosition &IRP, Attributor &A)10182      : AAPotentialConstantValuesImpl(IRP, A) {}10183 10184  /// See AbstractAttribute::initialize(...).10185  ChangeStatus updateImpl(Attributor &A) override {10186    llvm_unreachable(10187        "AAPotentialConstantValues(Function|CallSite)::updateImpl will "10188        "not be called");10189  }10190 10191  /// See AbstractAttribute::trackStatistics()10192  void trackStatistics() const override {10193    STATS_DECLTRACK_FN_ATTR(potential_values)10194  }10195};10196 10197struct AAPotentialConstantValuesCallSite : AAPotentialConstantValuesFunction {10198  AAPotentialConstantValuesCallSite(const IRPosition &IRP, Attributor &A)10199      : AAPotentialConstantValuesFunction(IRP, A) {}10200 10201  /// See AbstractAttribute::trackStatistics()10202  void trackStatistics() const override {10203    STATS_DECLTRACK_CS_ATTR(potential_values)10204  }10205};10206 10207struct AAPotentialConstantValuesCallSiteReturned10208    : AACalleeToCallSite<AAPotentialConstantValues,10209                         AAPotentialConstantValuesImpl> {10210  AAPotentialConstantValuesCallSiteReturned(const IRPosition &IRP,10211                                            Attributor &A)10212      : AACalleeToCallSite<AAPotentialConstantValues,10213                           AAPotentialConstantValuesImpl>(IRP, A) {}10214 10215  /// See AbstractAttribute::trackStatistics()10216  void trackStatistics() const override {10217    STATS_DECLTRACK_CSRET_ATTR(potential_values)10218  }10219};10220 10221struct AAPotentialConstantValuesCallSiteArgument10222    : AAPotentialConstantValuesFloating {10223  AAPotentialConstantValuesCallSiteArgument(const IRPosition &IRP,10224                                            Attributor &A)10225      : AAPotentialConstantValuesFloating(IRP, A) {}10226 10227  /// See AbstractAttribute::initialize(..).10228  void initialize(Attributor &A) override {10229    AAPotentialConstantValuesImpl::initialize(A);10230    if (isAtFixpoint())10231      return;10232 10233    Value &V = getAssociatedValue();10234 10235    if (auto *C = dyn_cast<ConstantInt>(&V)) {10236      unionAssumed(C->getValue());10237      indicateOptimisticFixpoint();10238      return;10239    }10240 10241    if (isa<UndefValue>(&V)) {10242      unionAssumedWithUndef();10243      indicateOptimisticFixpoint();10244      return;10245    }10246  }10247 10248  /// See AbstractAttribute::updateImpl(...).10249  ChangeStatus updateImpl(Attributor &A) override {10250    Value &V = getAssociatedValue();10251    auto AssumedBefore = getAssumed();10252    auto *AA = A.getAAFor<AAPotentialConstantValues>(10253        *this, IRPosition::value(V), DepClassTy::REQUIRED);10254    if (!AA)10255      return indicatePessimisticFixpoint();10256    const auto &S = AA->getAssumed();10257    unionAssumed(S);10258    return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED10259                                         : ChangeStatus::CHANGED;10260  }10261 10262  /// See AbstractAttribute::trackStatistics()10263  void trackStatistics() const override {10264    STATS_DECLTRACK_CSARG_ATTR(potential_values)10265  }10266};10267} // namespace10268 10269/// ------------------------ NoUndef Attribute ---------------------------------10270bool AANoUndef::isImpliedByIR(Attributor &A, const IRPosition &IRP,10271                              Attribute::AttrKind ImpliedAttributeKind,10272                              bool IgnoreSubsumingPositions) {10273  assert(ImpliedAttributeKind == Attribute::NoUndef &&10274         "Unexpected attribute kind");10275  if (A.hasAttr(IRP, {Attribute::NoUndef}, IgnoreSubsumingPositions,10276                Attribute::NoUndef))10277    return true;10278 10279  Value &Val = IRP.getAssociatedValue();10280  if (IRP.getPositionKind() != IRPosition::IRP_RETURNED &&10281      isGuaranteedNotToBeUndefOrPoison(&Val)) {10282    LLVMContext &Ctx = Val.getContext();10283    A.manifestAttrs(IRP, Attribute::get(Ctx, Attribute::NoUndef));10284    return true;10285  }10286 10287  return false;10288}10289 10290namespace {10291struct AANoUndefImpl : AANoUndef {10292  AANoUndefImpl(const IRPosition &IRP, Attributor &A) : AANoUndef(IRP, A) {}10293 10294  /// See AbstractAttribute::initialize(...).10295  void initialize(Attributor &A) override {10296    Value &V = getAssociatedValue();10297    if (isa<UndefValue>(V))10298      indicatePessimisticFixpoint();10299    assert(!isImpliedByIR(A, getIRPosition(), Attribute::NoUndef));10300  }10301 10302  /// See followUsesInMBEC10303  bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,10304                       AANoUndef::StateType &State) {10305    const Value *UseV = U->get();10306    const DominatorTree *DT = nullptr;10307    AssumptionCache *AC = nullptr;10308    InformationCache &InfoCache = A.getInfoCache();10309    if (Function *F = getAnchorScope()) {10310      DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(*F);10311      AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(*F);10312    }10313    State.setKnown(isGuaranteedNotToBeUndefOrPoison(UseV, AC, I, DT));10314    bool TrackUse = false;10315    // Track use for instructions which must produce undef or poison bits when10316    // at least one operand contains such bits.10317    if (isa<CastInst>(*I) || isa<GetElementPtrInst>(*I))10318      TrackUse = true;10319    return TrackUse;10320  }10321 10322  /// See AbstractAttribute::getAsStr().10323  const std::string getAsStr(Attributor *A) const override {10324    return getAssumed() ? "noundef" : "may-undef-or-poison";10325  }10326 10327  ChangeStatus manifest(Attributor &A) override {10328    // We don't manifest noundef attribute for dead positions because the10329    // associated values with dead positions would be replaced with undef10330    // values.10331    bool UsedAssumedInformation = false;10332    if (A.isAssumedDead(getIRPosition(), nullptr, nullptr,10333                        UsedAssumedInformation))10334      return ChangeStatus::UNCHANGED;10335    // A position whose simplified value does not have any value is10336    // considered to be dead. We don't manifest noundef in such positions for10337    // the same reason above.10338    if (!A.getAssumedSimplified(getIRPosition(), *this, UsedAssumedInformation,10339                                AA::Interprocedural)10340             .has_value())10341      return ChangeStatus::UNCHANGED;10342    return AANoUndef::manifest(A);10343  }10344};10345 10346struct AANoUndefFloating : public AANoUndefImpl {10347  AANoUndefFloating(const IRPosition &IRP, Attributor &A)10348      : AANoUndefImpl(IRP, A) {}10349 10350  /// See AbstractAttribute::initialize(...).10351  void initialize(Attributor &A) override {10352    AANoUndefImpl::initialize(A);10353    if (!getState().isAtFixpoint() && getAnchorScope() &&10354        !getAnchorScope()->isDeclaration())10355      if (Instruction *CtxI = getCtxI())10356        followUsesInMBEC(*this, A, getState(), *CtxI);10357  }10358 10359  /// See AbstractAttribute::updateImpl(...).10360  ChangeStatus updateImpl(Attributor &A) override {10361    auto VisitValueCB = [&](const IRPosition &IRP) -> bool {10362      bool IsKnownNoUndef;10363      return AA::hasAssumedIRAttr<Attribute::NoUndef>(10364          A, this, IRP, DepClassTy::REQUIRED, IsKnownNoUndef);10365    };10366 10367    bool Stripped;10368    bool UsedAssumedInformation = false;10369    Value *AssociatedValue = &getAssociatedValue();10370    SmallVector<AA::ValueAndContext> Values;10371    if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,10372                                      AA::AnyScope, UsedAssumedInformation))10373      Stripped = false;10374    else10375      Stripped =10376          Values.size() != 1 || Values.front().getValue() != AssociatedValue;10377 10378    if (!Stripped) {10379      // If we haven't stripped anything we might still be able to use a10380      // different AA, but only if the IRP changes. Effectively when we10381      // interpret this not as a call site value but as a floating/argument10382      // value.10383      const IRPosition AVIRP = IRPosition::value(*AssociatedValue);10384      if (AVIRP == getIRPosition() || !VisitValueCB(AVIRP))10385        return indicatePessimisticFixpoint();10386      return ChangeStatus::UNCHANGED;10387    }10388 10389    for (const auto &VAC : Values)10390      if (!VisitValueCB(IRPosition::value(*VAC.getValue())))10391        return indicatePessimisticFixpoint();10392 10393    return ChangeStatus::UNCHANGED;10394  }10395 10396  /// See AbstractAttribute::trackStatistics()10397  void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }10398};10399 10400struct AANoUndefReturned final10401    : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl> {10402  AANoUndefReturned(const IRPosition &IRP, Attributor &A)10403      : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl>(IRP, A) {}10404 10405  /// See AbstractAttribute::trackStatistics()10406  void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }10407};10408 10409struct AANoUndefArgument final10410    : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl> {10411  AANoUndefArgument(const IRPosition &IRP, Attributor &A)10412      : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl>(IRP, A) {}10413 10414  /// See AbstractAttribute::trackStatistics()10415  void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noundef) }10416};10417 10418struct AANoUndefCallSiteArgument final : AANoUndefFloating {10419  AANoUndefCallSiteArgument(const IRPosition &IRP, Attributor &A)10420      : AANoUndefFloating(IRP, A) {}10421 10422  /// See AbstractAttribute::trackStatistics()10423  void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noundef) }10424};10425 10426struct AANoUndefCallSiteReturned final10427    : AACalleeToCallSite<AANoUndef, AANoUndefImpl> {10428  AANoUndefCallSiteReturned(const IRPosition &IRP, Attributor &A)10429      : AACalleeToCallSite<AANoUndef, AANoUndefImpl>(IRP, A) {}10430 10431  /// See AbstractAttribute::trackStatistics()10432  void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noundef) }10433};10434 10435/// ------------------------ NoFPClass Attribute -------------------------------10436 10437struct AANoFPClassImpl : AANoFPClass {10438  AANoFPClassImpl(const IRPosition &IRP, Attributor &A) : AANoFPClass(IRP, A) {}10439 10440  void initialize(Attributor &A) override {10441    const IRPosition &IRP = getIRPosition();10442 10443    Value &V = IRP.getAssociatedValue();10444    if (isa<UndefValue>(V)) {10445      indicateOptimisticFixpoint();10446      return;10447    }10448 10449    SmallVector<Attribute> Attrs;10450    A.getAttrs(getIRPosition(), {Attribute::NoFPClass}, Attrs, false);10451    for (const auto &Attr : Attrs) {10452      addKnownBits(Attr.getNoFPClass());10453    }10454 10455    const DataLayout &DL = A.getDataLayout();10456    if (getPositionKind() != IRPosition::IRP_RETURNED) {10457      KnownFPClass KnownFPClass = computeKnownFPClass(&V, DL);10458      addKnownBits(~KnownFPClass.KnownFPClasses);10459    }10460 10461    if (Instruction *CtxI = getCtxI())10462      followUsesInMBEC(*this, A, getState(), *CtxI);10463  }10464 10465  /// See followUsesInMBEC10466  bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,10467                       AANoFPClass::StateType &State) {10468    // TODO: Determine what instructions can be looked through.10469    auto *CB = dyn_cast<CallBase>(I);10470    if (!CB)10471      return false;10472 10473    if (!CB->isArgOperand(U))10474      return false;10475 10476    unsigned ArgNo = CB->getArgOperandNo(U);10477    IRPosition IRP = IRPosition::callsite_argument(*CB, ArgNo);10478    if (auto *NoFPAA = A.getAAFor<AANoFPClass>(*this, IRP, DepClassTy::NONE))10479      State.addKnownBits(NoFPAA->getState().getKnown());10480    return false;10481  }10482 10483  const std::string getAsStr(Attributor *A) const override {10484    std::string Result = "nofpclass";10485    raw_string_ostream OS(Result);10486    OS << getKnownNoFPClass() << '/' << getAssumedNoFPClass();10487    return Result;10488  }10489 10490  void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,10491                            SmallVectorImpl<Attribute> &Attrs) const override {10492    Attrs.emplace_back(Attribute::getWithNoFPClass(Ctx, getAssumedNoFPClass()));10493  }10494};10495 10496struct AANoFPClassFloating : public AANoFPClassImpl {10497  AANoFPClassFloating(const IRPosition &IRP, Attributor &A)10498      : AANoFPClassImpl(IRP, A) {}10499 10500  /// See AbstractAttribute::updateImpl(...).10501  ChangeStatus updateImpl(Attributor &A) override {10502    SmallVector<AA::ValueAndContext> Values;10503    bool UsedAssumedInformation = false;10504    if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,10505                                      AA::AnyScope, UsedAssumedInformation)) {10506      Values.push_back({getAssociatedValue(), getCtxI()});10507    }10508 10509    StateType T;10510    auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {10511      const auto *AA = A.getAAFor<AANoFPClass>(*this, IRPosition::value(V),10512                                               DepClassTy::REQUIRED);10513      if (!AA || this == AA) {10514        T.indicatePessimisticFixpoint();10515      } else {10516        const AANoFPClass::StateType &S =10517            static_cast<const AANoFPClass::StateType &>(AA->getState());10518        T ^= S;10519      }10520      return T.isValidState();10521    };10522 10523    for (const auto &VAC : Values)10524      if (!VisitValueCB(*VAC.getValue(), VAC.getCtxI()))10525        return indicatePessimisticFixpoint();10526 10527    return clampStateAndIndicateChange(getState(), T);10528  }10529 10530  /// See AbstractAttribute::trackStatistics()10531  void trackStatistics() const override {10532    STATS_DECLTRACK_FNRET_ATTR(nofpclass)10533  }10534};10535 10536struct AANoFPClassReturned final10537    : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,10538                                   AANoFPClassImpl::StateType, false,10539                                   Attribute::None, false> {10540  AANoFPClassReturned(const IRPosition &IRP, Attributor &A)10541      : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,10542                                     AANoFPClassImpl::StateType, false,10543                                     Attribute::None, false>(IRP, A) {}10544 10545  /// See AbstractAttribute::trackStatistics()10546  void trackStatistics() const override {10547    STATS_DECLTRACK_FNRET_ATTR(nofpclass)10548  }10549};10550 10551struct AANoFPClassArgument final10552    : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl> {10553  AANoFPClassArgument(const IRPosition &IRP, Attributor &A)10554      : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl>(IRP, A) {}10555 10556  /// See AbstractAttribute::trackStatistics()10557  void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofpclass) }10558};10559 10560struct AANoFPClassCallSiteArgument final : AANoFPClassFloating {10561  AANoFPClassCallSiteArgument(const IRPosition &IRP, Attributor &A)10562      : AANoFPClassFloating(IRP, A) {}10563 10564  /// See AbstractAttribute::trackStatistics()10565  void trackStatistics() const override {10566    STATS_DECLTRACK_CSARG_ATTR(nofpclass)10567  }10568};10569 10570struct AANoFPClassCallSiteReturned final10571    : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl> {10572  AANoFPClassCallSiteReturned(const IRPosition &IRP, Attributor &A)10573      : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl>(IRP, A) {}10574 10575  /// See AbstractAttribute::trackStatistics()10576  void trackStatistics() const override {10577    STATS_DECLTRACK_CSRET_ATTR(nofpclass)10578  }10579};10580 10581struct AACallEdgesImpl : public AACallEdges {10582  AACallEdgesImpl(const IRPosition &IRP, Attributor &A) : AACallEdges(IRP, A) {}10583 10584  const SetVector<Function *> &getOptimisticEdges() const override {10585    return CalledFunctions;10586  }10587 10588  bool hasUnknownCallee() const override { return HasUnknownCallee; }10589 10590  bool hasNonAsmUnknownCallee() const override {10591    return HasUnknownCalleeNonAsm;10592  }10593 10594  const std::string getAsStr(Attributor *A) const override {10595    return "CallEdges[" + std::to_string(HasUnknownCallee) + "," +10596           std::to_string(CalledFunctions.size()) + "]";10597  }10598 10599  void trackStatistics() const override {}10600 10601protected:10602  void addCalledFunction(Function *Fn, ChangeStatus &Change) {10603    if (CalledFunctions.insert(Fn)) {10604      Change = ChangeStatus::CHANGED;10605      LLVM_DEBUG(dbgs() << "[AACallEdges] New call edge: " << Fn->getName()10606                        << "\n");10607    }10608  }10609 10610  void setHasUnknownCallee(bool NonAsm, ChangeStatus &Change) {10611    if (!HasUnknownCallee)10612      Change = ChangeStatus::CHANGED;10613    if (NonAsm && !HasUnknownCalleeNonAsm)10614      Change = ChangeStatus::CHANGED;10615    HasUnknownCalleeNonAsm |= NonAsm;10616    HasUnknownCallee = true;10617  }10618 10619private:10620  /// Optimistic set of functions that might be called by this position.10621  SetVector<Function *> CalledFunctions;10622 10623  /// Is there any call with a unknown callee.10624  bool HasUnknownCallee = false;10625 10626  /// Is there any call with a unknown callee, excluding any inline asm.10627  bool HasUnknownCalleeNonAsm = false;10628};10629 10630struct AACallEdgesCallSite : public AACallEdgesImpl {10631  AACallEdgesCallSite(const IRPosition &IRP, Attributor &A)10632      : AACallEdgesImpl(IRP, A) {}10633  /// See AbstractAttribute::updateImpl(...).10634  ChangeStatus updateImpl(Attributor &A) override {10635    ChangeStatus Change = ChangeStatus::UNCHANGED;10636 10637    auto VisitValue = [&](Value &V, const Instruction *CtxI) -> bool {10638      if (Function *Fn = dyn_cast<Function>(&V)) {10639        addCalledFunction(Fn, Change);10640      } else {10641        LLVM_DEBUG(dbgs() << "[AACallEdges] Unrecognized value: " << V << "\n");10642        setHasUnknownCallee(true, Change);10643      }10644 10645      // Explore all values.10646      return true;10647    };10648 10649    SmallVector<AA::ValueAndContext> Values;10650    // Process any value that we might call.10651    auto ProcessCalledOperand = [&](Value *V, Instruction *CtxI) {10652      if (isa<Constant>(V)) {10653        VisitValue(*V, CtxI);10654        return;10655      }10656 10657      bool UsedAssumedInformation = false;10658      Values.clear();10659      if (!A.getAssumedSimplifiedValues(IRPosition::value(*V), *this, Values,10660                                        AA::AnyScope, UsedAssumedInformation)) {10661        Values.push_back({*V, CtxI});10662      }10663      for (auto &VAC : Values)10664        VisitValue(*VAC.getValue(), VAC.getCtxI());10665    };10666 10667    CallBase *CB = cast<CallBase>(getCtxI());10668 10669    if (auto *IA = dyn_cast<InlineAsm>(CB->getCalledOperand())) {10670      if (IA->hasSideEffects() &&10671          !hasAssumption(*CB->getCaller(), "ompx_no_call_asm") &&10672          !hasAssumption(*CB, "ompx_no_call_asm")) {10673        setHasUnknownCallee(false, Change);10674      }10675      return Change;10676    }10677 10678    if (CB->isIndirectCall())10679      if (auto *IndirectCallAA = A.getAAFor<AAIndirectCallInfo>(10680              *this, getIRPosition(), DepClassTy::OPTIONAL))10681        if (IndirectCallAA->foreachCallee(10682                [&](Function *Fn) { return VisitValue(*Fn, CB); }))10683          return Change;10684 10685    // The most simple case.10686    ProcessCalledOperand(CB->getCalledOperand(), CB);10687 10688    // Process callback functions.10689    SmallVector<const Use *, 4u> CallbackUses;10690    AbstractCallSite::getCallbackUses(*CB, CallbackUses);10691    for (const Use *U : CallbackUses)10692      ProcessCalledOperand(U->get(), CB);10693 10694    return Change;10695  }10696};10697 10698struct AACallEdgesFunction : public AACallEdgesImpl {10699  AACallEdgesFunction(const IRPosition &IRP, Attributor &A)10700      : AACallEdgesImpl(IRP, A) {}10701 10702  /// See AbstractAttribute::updateImpl(...).10703  ChangeStatus updateImpl(Attributor &A) override {10704    ChangeStatus Change = ChangeStatus::UNCHANGED;10705 10706    auto ProcessCallInst = [&](Instruction &Inst) {10707      CallBase &CB = cast<CallBase>(Inst);10708 10709      auto *CBEdges = A.getAAFor<AACallEdges>(10710          *this, IRPosition::callsite_function(CB), DepClassTy::REQUIRED);10711      if (!CBEdges)10712        return false;10713      if (CBEdges->hasNonAsmUnknownCallee())10714        setHasUnknownCallee(true, Change);10715      if (CBEdges->hasUnknownCallee())10716        setHasUnknownCallee(false, Change);10717 10718      for (Function *F : CBEdges->getOptimisticEdges())10719        addCalledFunction(F, Change);10720 10721      return true;10722    };10723 10724    // Visit all callable instructions.10725    bool UsedAssumedInformation = false;10726    if (!A.checkForAllCallLikeInstructions(ProcessCallInst, *this,10727                                           UsedAssumedInformation,10728                                           /* CheckBBLivenessOnly */ true)) {10729      // If we haven't looked at all call like instructions, assume that there10730      // are unknown callees.10731      setHasUnknownCallee(true, Change);10732    }10733 10734    return Change;10735  }10736};10737 10738/// -------------------AAInterFnReachability Attribute--------------------------10739 10740struct AAInterFnReachabilityFunction10741    : public CachedReachabilityAA<AAInterFnReachability, Function> {10742  using Base = CachedReachabilityAA<AAInterFnReachability, Function>;10743  AAInterFnReachabilityFunction(const IRPosition &IRP, Attributor &A)10744      : Base(IRP, A) {}10745 10746  bool instructionCanReach(10747      Attributor &A, const Instruction &From, const Function &To,10748      const AA::InstExclusionSetTy *ExclusionSet) const override {10749    assert(From.getFunction() == getAnchorScope() && "Queried the wrong AA!");10750    auto *NonConstThis = const_cast<AAInterFnReachabilityFunction *>(this);10751 10752    RQITy StackRQI(A, From, To, ExclusionSet, false);10753    RQITy::Reachable Result;10754    if (!NonConstThis->checkQueryCache(A, StackRQI, Result))10755      return NonConstThis->isReachableImpl(A, StackRQI,10756                                           /*IsTemporaryRQI=*/true);10757    return Result == RQITy::Reachable::Yes;10758  }10759 10760  bool isReachableImpl(Attributor &A, RQITy &RQI,10761                       bool IsTemporaryRQI) override {10762    const Instruction *EntryI =10763        &RQI.From->getFunction()->getEntryBlock().front();10764    if (EntryI != RQI.From &&10765        !instructionCanReach(A, *EntryI, *RQI.To, nullptr))10766      return rememberResult(A, RQITy::Reachable::No, RQI, false,10767                            IsTemporaryRQI);10768 10769    auto CheckReachableCallBase = [&](CallBase *CB) {10770      auto *CBEdges = A.getAAFor<AACallEdges>(10771          *this, IRPosition::callsite_function(*CB), DepClassTy::OPTIONAL);10772      if (!CBEdges || !CBEdges->getState().isValidState())10773        return false;10774      // TODO Check To backwards in this case.10775      if (CBEdges->hasUnknownCallee())10776        return false;10777 10778      for (Function *Fn : CBEdges->getOptimisticEdges()) {10779        if (Fn == RQI.To)10780          return false;10781 10782        if (Fn->isDeclaration()) {10783          if (Fn->hasFnAttribute(Attribute::NoCallback))10784            continue;10785          // TODO Check To backwards in this case.10786          return false;10787        }10788 10789        if (Fn == getAnchorScope()) {10790          if (EntryI == RQI.From)10791            continue;10792          return false;10793        }10794 10795        const AAInterFnReachability *InterFnReachability =10796            A.getAAFor<AAInterFnReachability>(*this, IRPosition::function(*Fn),10797                                              DepClassTy::OPTIONAL);10798 10799        const Instruction &FnFirstInst = Fn->getEntryBlock().front();10800        if (!InterFnReachability ||10801            InterFnReachability->instructionCanReach(A, FnFirstInst, *RQI.To,10802                                                     RQI.ExclusionSet))10803          return false;10804      }10805      return true;10806    };10807 10808    const auto *IntraFnReachability = A.getAAFor<AAIntraFnReachability>(10809        *this, IRPosition::function(*RQI.From->getFunction()),10810        DepClassTy::OPTIONAL);10811 10812    // Determine call like instructions that we can reach from the inst.10813    auto CheckCallBase = [&](Instruction &CBInst) {10814      // There are usually less nodes in the call graph, check inter function10815      // reachability first.10816      if (CheckReachableCallBase(cast<CallBase>(&CBInst)))10817        return true;10818      return IntraFnReachability && !IntraFnReachability->isAssumedReachable(10819                                        A, *RQI.From, CBInst, RQI.ExclusionSet);10820    };10821 10822    bool UsedExclusionSet = /* conservative */ true;10823    bool UsedAssumedInformation = false;10824    if (!A.checkForAllCallLikeInstructions(CheckCallBase, *this,10825                                           UsedAssumedInformation,10826                                           /* CheckBBLivenessOnly */ true))10827      return rememberResult(A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,10828                            IsTemporaryRQI);10829 10830    return rememberResult(A, RQITy::Reachable::No, RQI, UsedExclusionSet,10831                          IsTemporaryRQI);10832  }10833 10834  void trackStatistics() const override {}10835};10836} // namespace10837 10838template <typename AAType>10839static std::optional<Constant *>10840askForAssumedConstant(Attributor &A, const AbstractAttribute &QueryingAA,10841                      const IRPosition &IRP, Type &Ty) {10842  if (!Ty.isIntegerTy())10843    return nullptr;10844 10845  // This will also pass the call base context.10846  const auto *AA = A.getAAFor<AAType>(QueryingAA, IRP, DepClassTy::NONE);10847  if (!AA)10848    return nullptr;10849 10850  std::optional<Constant *> COpt = AA->getAssumedConstant(A);10851 10852  if (!COpt.has_value()) {10853    A.recordDependence(*AA, QueryingAA, DepClassTy::OPTIONAL);10854    return std::nullopt;10855  }10856  if (auto *C = *COpt) {10857    A.recordDependence(*AA, QueryingAA, DepClassTy::OPTIONAL);10858    return C;10859  }10860  return nullptr;10861}10862 10863Value *AAPotentialValues::getSingleValue(10864    Attributor &A, const AbstractAttribute &AA, const IRPosition &IRP,10865    SmallVectorImpl<AA::ValueAndContext> &Values) {10866  Type &Ty = *IRP.getAssociatedType();10867  std::optional<Value *> V;10868  for (auto &It : Values) {10869    V = AA::combineOptionalValuesInAAValueLatice(V, It.getValue(), &Ty);10870    if (V.has_value() && !*V)10871      break;10872  }10873  if (!V.has_value())10874    return UndefValue::get(&Ty);10875  return *V;10876}10877 10878namespace {10879struct AAPotentialValuesImpl : AAPotentialValues {10880  using StateType = PotentialLLVMValuesState;10881 10882  AAPotentialValuesImpl(const IRPosition &IRP, Attributor &A)10883      : AAPotentialValues(IRP, A) {}10884 10885  /// See AbstractAttribute::initialize(..).10886  void initialize(Attributor &A) override {10887    if (A.hasSimplificationCallback(getIRPosition())) {10888      indicatePessimisticFixpoint();10889      return;10890    }10891    Value *Stripped = getAssociatedValue().stripPointerCasts();10892    if (isa<Constant>(Stripped) && !isa<ConstantExpr>(Stripped)) {10893      addValue(A, getState(), *Stripped, getCtxI(), AA::AnyScope,10894               getAnchorScope());10895      indicateOptimisticFixpoint();10896      return;10897    }10898    AAPotentialValues::initialize(A);10899  }10900 10901  /// See AbstractAttribute::getAsStr().10902  const std::string getAsStr(Attributor *A) const override {10903    std::string Str;10904    llvm::raw_string_ostream OS(Str);10905    OS << getState();10906    return Str;10907  }10908 10909  template <typename AAType>10910  static std::optional<Value *> askOtherAA(Attributor &A,10911                                           const AbstractAttribute &AA,10912                                           const IRPosition &IRP, Type &Ty) {10913    if (isa<Constant>(IRP.getAssociatedValue()))10914      return &IRP.getAssociatedValue();10915    std::optional<Constant *> C = askForAssumedConstant<AAType>(A, AA, IRP, Ty);10916    if (!C)10917      return std::nullopt;10918    if (*C)10919      if (auto *CC = AA::getWithType(**C, Ty))10920        return CC;10921    return nullptr;10922  }10923 10924  virtual void addValue(Attributor &A, StateType &State, Value &V,10925                        const Instruction *CtxI, AA::ValueScope S,10926                        Function *AnchorScope) const {10927 10928    IRPosition ValIRP = IRPosition::value(V);10929    if (auto *CB = dyn_cast_or_null<CallBase>(CtxI)) {10930      for (const auto &U : CB->args()) {10931        if (U.get() != &V)10932          continue;10933        ValIRP = IRPosition::callsite_argument(*CB, CB->getArgOperandNo(&U));10934        break;10935      }10936    }10937 10938    Value *VPtr = &V;10939    if (ValIRP.getAssociatedType()->isIntegerTy()) {10940      Type &Ty = *getAssociatedType();10941      std::optional<Value *> SimpleV =10942          askOtherAA<AAValueConstantRange>(A, *this, ValIRP, Ty);10943      if (SimpleV.has_value() && !*SimpleV) {10944        auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(10945            *this, ValIRP, DepClassTy::OPTIONAL);10946        if (PotentialConstantsAA && PotentialConstantsAA->isValidState()) {10947          for (const auto &It : PotentialConstantsAA->getAssumedSet())10948            State.unionAssumed({{*ConstantInt::get(&Ty, It), nullptr}, S});10949          if (PotentialConstantsAA->undefIsContained())10950            State.unionAssumed({{*UndefValue::get(&Ty), nullptr}, S});10951          return;10952        }10953      }10954      if (!SimpleV.has_value())10955        return;10956 10957      if (*SimpleV)10958        VPtr = *SimpleV;10959    }10960 10961    if (isa<ConstantInt>(VPtr))10962      CtxI = nullptr;10963    if (!AA::isValidInScope(*VPtr, AnchorScope))10964      S = AA::ValueScope(S | AA::Interprocedural);10965 10966    State.unionAssumed({{*VPtr, CtxI}, S});10967  }10968 10969  /// Helper struct to tie a value+context pair together with the scope for10970  /// which this is the simplified version.10971  struct ItemInfo {10972    AA::ValueAndContext I;10973    AA::ValueScope S;10974 10975    bool operator==(const ItemInfo &II) const {10976      return II.I == I && II.S == S;10977    };10978    bool operator<(const ItemInfo &II) const {10979      return std::tie(I, S) < std::tie(II.I, II.S);10980    };10981  };10982 10983  bool recurseForValue(Attributor &A, const IRPosition &IRP, AA::ValueScope S) {10984    SmallMapVector<AA::ValueAndContext, int, 8> ValueScopeMap;10985    for (auto CS : {AA::Intraprocedural, AA::Interprocedural}) {10986      if (!(CS & S))10987        continue;10988 10989      bool UsedAssumedInformation = false;10990      SmallVector<AA::ValueAndContext> Values;10991      if (!A.getAssumedSimplifiedValues(IRP, this, Values, CS,10992                                        UsedAssumedInformation))10993        return false;10994 10995      for (auto &It : Values)10996        ValueScopeMap[It] += CS;10997    }10998    for (auto &It : ValueScopeMap)10999      addValue(A, getState(), *It.first.getValue(), It.first.getCtxI(),11000               AA::ValueScope(It.second), getAnchorScope());11001 11002    return true;11003  }11004 11005  void giveUpOnIntraprocedural(Attributor &A) {11006    auto NewS = StateType::getBestState(getState());11007    for (const auto &It : getAssumedSet()) {11008      if (It.second == AA::Intraprocedural)11009        continue;11010      addValue(A, NewS, *It.first.getValue(), It.first.getCtxI(),11011               AA::Interprocedural, getAnchorScope());11012    }11013    assert(!undefIsContained() && "Undef should be an explicit value!");11014    addValue(A, NewS, getAssociatedValue(), getCtxI(), AA::Intraprocedural,11015             getAnchorScope());11016    getState() = NewS;11017  }11018 11019  /// See AbstractState::indicatePessimisticFixpoint(...).11020  ChangeStatus indicatePessimisticFixpoint() override {11021    getState() = StateType::getBestState(getState());11022    getState().unionAssumed({{getAssociatedValue(), getCtxI()}, AA::AnyScope});11023    AAPotentialValues::indicateOptimisticFixpoint();11024    return ChangeStatus::CHANGED;11025  }11026 11027  /// See AbstractAttribute::updateImpl(...).11028  ChangeStatus updateImpl(Attributor &A) override {11029    return indicatePessimisticFixpoint();11030  }11031 11032  /// See AbstractAttribute::manifest(...).11033  ChangeStatus manifest(Attributor &A) override {11034    SmallVector<AA::ValueAndContext> Values;11035    for (AA::ValueScope S : {AA::Interprocedural, AA::Intraprocedural}) {11036      Values.clear();11037      if (!getAssumedSimplifiedValues(A, Values, S))11038        continue;11039      Value &OldV = getAssociatedValue();11040      if (isa<UndefValue>(OldV))11041        continue;11042      Value *NewV = getSingleValue(A, *this, getIRPosition(), Values);11043      if (!NewV || NewV == &OldV)11044        continue;11045      if (getCtxI() &&11046          !AA::isValidAtPosition({*NewV, *getCtxI()}, A.getInfoCache()))11047        continue;11048      if (A.changeAfterManifest(getIRPosition(), *NewV))11049        return ChangeStatus::CHANGED;11050    }11051    return ChangeStatus::UNCHANGED;11052  }11053 11054  bool getAssumedSimplifiedValues(11055      Attributor &A, SmallVectorImpl<AA::ValueAndContext> &Values,11056      AA::ValueScope S, bool RecurseForSelectAndPHI = false) const override {11057    if (!isValidState())11058      return false;11059    bool UsedAssumedInformation = false;11060    for (const auto &It : getAssumedSet())11061      if (It.second & S) {11062        if (RecurseForSelectAndPHI && (isa<PHINode>(It.first.getValue()) ||11063                                       isa<SelectInst>(It.first.getValue()))) {11064          if (A.getAssumedSimplifiedValues(11065                  IRPosition::inst(*cast<Instruction>(It.first.getValue())),11066                  this, Values, S, UsedAssumedInformation))11067            continue;11068        }11069        Values.push_back(It.first);11070      }11071    assert(!undefIsContained() && "Undef should be an explicit value!");11072    return true;11073  }11074};11075 11076struct AAPotentialValuesFloating : AAPotentialValuesImpl {11077  AAPotentialValuesFloating(const IRPosition &IRP, Attributor &A)11078      : AAPotentialValuesImpl(IRP, A) {}11079 11080  /// See AbstractAttribute::updateImpl(...).11081  ChangeStatus updateImpl(Attributor &A) override {11082    auto AssumedBefore = getAssumed();11083 11084    genericValueTraversal(A, &getAssociatedValue());11085 11086    return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED11087                                           : ChangeStatus::CHANGED;11088  }11089 11090  /// Helper struct to remember which AAIsDead instances we actually used.11091  struct LivenessInfo {11092    const AAIsDead *LivenessAA = nullptr;11093    bool AnyDead = false;11094  };11095 11096  /// Check if \p Cmp is a comparison we can simplify.11097  ///11098  /// We handle multiple cases, one in which at least one operand is an11099  /// (assumed) nullptr. If so, try to simplify it using AANonNull on the other11100  /// operand. Return true if successful, in that case Worklist will be updated.11101  bool handleCmp(Attributor &A, Value &Cmp, Value *LHS, Value *RHS,11102                 CmpInst::Predicate Pred, ItemInfo II,11103                 SmallVectorImpl<ItemInfo> &Worklist) {11104 11105    // Simplify the operands first.11106    bool UsedAssumedInformation = false;11107    SmallVector<AA::ValueAndContext> LHSValues, RHSValues;11108    auto GetSimplifiedValues = [&](Value &V,11109                                   SmallVector<AA::ValueAndContext> &Values) {11110      if (!A.getAssumedSimplifiedValues(11111              IRPosition::value(V, getCallBaseContext()), this, Values,11112              AA::Intraprocedural, UsedAssumedInformation)) {11113        Values.clear();11114        Values.push_back(AA::ValueAndContext{V, II.I.getCtxI()});11115      }11116      return Values.empty();11117    };11118    if (GetSimplifiedValues(*LHS, LHSValues))11119      return true;11120    if (GetSimplifiedValues(*RHS, RHSValues))11121      return true;11122 11123    LLVMContext &Ctx = LHS->getContext();11124 11125    InformationCache &InfoCache = A.getInfoCache();11126    Instruction *CmpI = dyn_cast<Instruction>(&Cmp);11127    Function *F = CmpI ? CmpI->getFunction() : nullptr;11128    const auto *DT =11129        F ? InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(*F)11130          : nullptr;11131    const auto *TLI =11132        F ? A.getInfoCache().getTargetLibraryInfoForFunction(*F) : nullptr;11133    auto *AC =11134        F ? InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(*F)11135          : nullptr;11136 11137    const DataLayout &DL = A.getDataLayout();11138    SimplifyQuery Q(DL, TLI, DT, AC, CmpI);11139 11140    auto CheckPair = [&](Value &LHSV, Value &RHSV) {11141      if (isa<UndefValue>(LHSV) || isa<UndefValue>(RHSV)) {11142        addValue(A, getState(), *UndefValue::get(Cmp.getType()),11143                 /* CtxI */ nullptr, II.S, getAnchorScope());11144        return true;11145      }11146 11147      // Handle the trivial case first in which we don't even need to think11148      // about null or non-null.11149      if (&LHSV == &RHSV &&11150          (CmpInst::isTrueWhenEqual(Pred) || CmpInst::isFalseWhenEqual(Pred))) {11151        Constant *NewV = ConstantInt::get(Type::getInt1Ty(Ctx),11152                                          CmpInst::isTrueWhenEqual(Pred));11153        addValue(A, getState(), *NewV, /* CtxI */ nullptr, II.S,11154                 getAnchorScope());11155        return true;11156      }11157 11158      auto *TypedLHS = AA::getWithType(LHSV, *LHS->getType());11159      auto *TypedRHS = AA::getWithType(RHSV, *RHS->getType());11160      if (TypedLHS && TypedRHS) {11161        Value *NewV = simplifyCmpInst(Pred, TypedLHS, TypedRHS, Q);11162        if (NewV && NewV != &Cmp) {11163          addValue(A, getState(), *NewV, /* CtxI */ nullptr, II.S,11164                   getAnchorScope());11165          return true;11166        }11167      }11168 11169      // From now on we only handle equalities (==, !=).11170      if (!CmpInst::isEquality(Pred))11171        return false;11172 11173      bool LHSIsNull = isa<ConstantPointerNull>(LHSV);11174      bool RHSIsNull = isa<ConstantPointerNull>(RHSV);11175      if (!LHSIsNull && !RHSIsNull)11176        return false;11177 11178      // Left is the nullptr ==/!= non-nullptr case. We'll use AANonNull on the11179      // non-nullptr operand and if we assume it's non-null we can conclude the11180      // result of the comparison.11181      assert((LHSIsNull || RHSIsNull) &&11182             "Expected nullptr versus non-nullptr comparison at this point");11183 11184      // The index is the operand that we assume is not null.11185      unsigned PtrIdx = LHSIsNull;11186      bool IsKnownNonNull;11187      bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(11188          A, this, IRPosition::value(*(PtrIdx ? &RHSV : &LHSV)),11189          DepClassTy::REQUIRED, IsKnownNonNull);11190      if (!IsAssumedNonNull)11191        return false;11192 11193      // The new value depends on the predicate, true for != and false for ==.11194      Constant *NewV =11195          ConstantInt::get(Type::getInt1Ty(Ctx), Pred == CmpInst::ICMP_NE);11196      addValue(A, getState(), *NewV, /* CtxI */ nullptr, II.S,11197               getAnchorScope());11198      return true;11199    };11200 11201    for (auto &LHSValue : LHSValues)11202      for (auto &RHSValue : RHSValues)11203        if (!CheckPair(*LHSValue.getValue(), *RHSValue.getValue()))11204          return false;11205    return true;11206  }11207 11208  bool handleSelectInst(Attributor &A, SelectInst &SI, ItemInfo II,11209                        SmallVectorImpl<ItemInfo> &Worklist) {11210    const Instruction *CtxI = II.I.getCtxI();11211    bool UsedAssumedInformation = false;11212 11213    std::optional<Constant *> C =11214        A.getAssumedConstant(*SI.getCondition(), *this, UsedAssumedInformation);11215    bool NoValueYet = !C.has_value();11216    if (NoValueYet || isa_and_nonnull<UndefValue>(*C))11217      return true;11218    if (auto *CI = dyn_cast_or_null<ConstantInt>(*C)) {11219      if (CI->isZero())11220        Worklist.push_back({{*SI.getFalseValue(), CtxI}, II.S});11221      else11222        Worklist.push_back({{*SI.getTrueValue(), CtxI}, II.S});11223    } else if (&SI == &getAssociatedValue()) {11224      // We could not simplify the condition, assume both values.11225      Worklist.push_back({{*SI.getTrueValue(), CtxI}, II.S});11226      Worklist.push_back({{*SI.getFalseValue(), CtxI}, II.S});11227    } else {11228      std::optional<Value *> SimpleV = A.getAssumedSimplified(11229          IRPosition::inst(SI), *this, UsedAssumedInformation, II.S);11230      if (!SimpleV.has_value())11231        return true;11232      if (*SimpleV) {11233        addValue(A, getState(), **SimpleV, CtxI, II.S, getAnchorScope());11234        return true;11235      }11236      return false;11237    }11238    return true;11239  }11240 11241  bool handleLoadInst(Attributor &A, LoadInst &LI, ItemInfo II,11242                      SmallVectorImpl<ItemInfo> &Worklist) {11243    SmallSetVector<Value *, 4> PotentialCopies;11244    SmallSetVector<Instruction *, 4> PotentialValueOrigins;11245    bool UsedAssumedInformation = false;11246    if (!AA::getPotentiallyLoadedValues(A, LI, PotentialCopies,11247                                        PotentialValueOrigins, *this,11248                                        UsedAssumedInformation,11249                                        /* OnlyExact */ true)) {11250      LLVM_DEBUG(dbgs() << "[AAPotentialValues] Failed to get potentially "11251                           "loaded values for load instruction "11252                        << LI << "\n");11253      return false;11254    }11255 11256    // Do not simplify loads that are only used in llvm.assume if we cannot also11257    // remove all stores that may feed into the load. The reason is that the11258    // assume is probably worth something as long as the stores are around.11259    InformationCache &InfoCache = A.getInfoCache();11260    if (InfoCache.isOnlyUsedByAssume(LI)) {11261      if (!llvm::all_of(PotentialValueOrigins, [&](Instruction *I) {11262            if (!I || isa<AssumeInst>(I))11263              return true;11264            if (auto *SI = dyn_cast<StoreInst>(I))11265              return A.isAssumedDead(SI->getOperandUse(0), this,11266                                     /* LivenessAA */ nullptr,11267                                     UsedAssumedInformation,11268                                     /* CheckBBLivenessOnly */ false);11269            return A.isAssumedDead(*I, this, /* LivenessAA */ nullptr,11270                                   UsedAssumedInformation,11271                                   /* CheckBBLivenessOnly */ false);11272          })) {11273        LLVM_DEBUG(dbgs() << "[AAPotentialValues] Load is onl used by assumes "11274                             "and we cannot delete all the stores: "11275                          << LI << "\n");11276        return false;11277      }11278    }11279 11280    // Values have to be dynamically unique or we loose the fact that a11281    // single llvm::Value might represent two runtime values (e.g.,11282    // stack locations in different recursive calls).11283    const Instruction *CtxI = II.I.getCtxI();11284    bool ScopeIsLocal = (II.S & AA::Intraprocedural);11285    bool AllLocal = ScopeIsLocal;11286    bool DynamicallyUnique = llvm::all_of(PotentialCopies, [&](Value *PC) {11287      AllLocal &= AA::isValidInScope(*PC, getAnchorScope());11288      return AA::isDynamicallyUnique(A, *this, *PC);11289    });11290    if (!DynamicallyUnique) {11291      LLVM_DEBUG(dbgs() << "[AAPotentialValues] Not all potentially loaded "11292                           "values are dynamically unique: "11293                        << LI << "\n");11294      return false;11295    }11296 11297    for (auto *PotentialCopy : PotentialCopies) {11298      if (AllLocal) {11299        Worklist.push_back({{*PotentialCopy, CtxI}, II.S});11300      } else {11301        Worklist.push_back({{*PotentialCopy, CtxI}, AA::Interprocedural});11302      }11303    }11304    if (!AllLocal && ScopeIsLocal)11305      addValue(A, getState(), LI, CtxI, AA::Intraprocedural, getAnchorScope());11306    return true;11307  }11308 11309  bool handlePHINode(11310      Attributor &A, PHINode &PHI, ItemInfo II,11311      SmallVectorImpl<ItemInfo> &Worklist,11312      SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) {11313    auto GetLivenessInfo = [&](const Function &F) -> LivenessInfo & {11314      LivenessInfo &LI = LivenessAAs[&F];11315      if (!LI.LivenessAA)11316        LI.LivenessAA = A.getAAFor<AAIsDead>(*this, IRPosition::function(F),11317                                             DepClassTy::NONE);11318      return LI;11319    };11320 11321    if (&PHI == &getAssociatedValue()) {11322      LivenessInfo &LI = GetLivenessInfo(*PHI.getFunction());11323      const auto *CI =11324          A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(11325              *PHI.getFunction());11326 11327      Cycle *C = nullptr;11328      bool CyclePHI = mayBeInCycle(CI, &PHI, /* HeaderOnly */ true, &C);11329      for (unsigned u = 0, e = PHI.getNumIncomingValues(); u < e; u++) {11330        BasicBlock *IncomingBB = PHI.getIncomingBlock(u);11331        if (LI.LivenessAA &&11332            LI.LivenessAA->isEdgeDead(IncomingBB, PHI.getParent())) {11333          LI.AnyDead = true;11334          continue;11335        }11336        Value *V = PHI.getIncomingValue(u);11337        if (V == &PHI)11338          continue;11339 11340        // If the incoming value is not the PHI but an instruction in the same11341        // cycle we might have multiple versions of it flying around.11342        if (CyclePHI && isa<Instruction>(V) &&11343            (!C || C->contains(cast<Instruction>(V)->getParent())))11344          return false;11345 11346        Worklist.push_back({{*V, IncomingBB->getTerminator()}, II.S});11347      }11348      return true;11349    }11350 11351    bool UsedAssumedInformation = false;11352    std::optional<Value *> SimpleV = A.getAssumedSimplified(11353        IRPosition::inst(PHI), *this, UsedAssumedInformation, II.S);11354    if (!SimpleV.has_value())11355      return true;11356    if (!(*SimpleV))11357      return false;11358    addValue(A, getState(), **SimpleV, &PHI, II.S, getAnchorScope());11359    return true;11360  }11361 11362  /// Use the generic, non-optimistic InstSimplfy functionality if we managed to11363  /// simplify any operand of the instruction \p I. Return true if successful,11364  /// in that case Worklist will be updated.11365  bool handleGenericInst(Attributor &A, Instruction &I, ItemInfo II,11366                         SmallVectorImpl<ItemInfo> &Worklist) {11367    bool SomeSimplified = false;11368    bool UsedAssumedInformation = false;11369 11370    SmallVector<Value *, 8> NewOps(I.getNumOperands());11371    int Idx = 0;11372    for (Value *Op : I.operands()) {11373      const auto &SimplifiedOp = A.getAssumedSimplified(11374          IRPosition::value(*Op, getCallBaseContext()), *this,11375          UsedAssumedInformation, AA::Intraprocedural);11376      // If we are not sure about any operand we are not sure about the entire11377      // instruction, we'll wait.11378      if (!SimplifiedOp.has_value())11379        return true;11380 11381      if (*SimplifiedOp)11382        NewOps[Idx] = *SimplifiedOp;11383      else11384        NewOps[Idx] = Op;11385 11386      SomeSimplified |= (NewOps[Idx] != Op);11387      ++Idx;11388    }11389 11390    // We won't bother with the InstSimplify interface if we didn't simplify any11391    // operand ourselves.11392    if (!SomeSimplified)11393      return false;11394 11395    InformationCache &InfoCache = A.getInfoCache();11396    Function *F = I.getFunction();11397    const auto *DT =11398        InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(*F);11399    const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);11400    auto *AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(*F);11401 11402    const DataLayout &DL = I.getDataLayout();11403    SimplifyQuery Q(DL, TLI, DT, AC, &I);11404    Value *NewV = simplifyInstructionWithOperands(&I, NewOps, Q);11405    if (!NewV || NewV == &I)11406      return false;11407 11408    LLVM_DEBUG(dbgs() << "Generic inst " << I << " assumed simplified to "11409                      << *NewV << "\n");11410    Worklist.push_back({{*NewV, II.I.getCtxI()}, II.S});11411    return true;11412  }11413 11414  bool simplifyInstruction(11415      Attributor &A, Instruction &I, ItemInfo II,11416      SmallVectorImpl<ItemInfo> &Worklist,11417      SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) {11418    if (auto *CI = dyn_cast<CmpInst>(&I))11419      return handleCmp(A, *CI, CI->getOperand(0), CI->getOperand(1),11420                       CI->getPredicate(), II, Worklist);11421 11422    switch (I.getOpcode()) {11423    case Instruction::Select:11424      return handleSelectInst(A, cast<SelectInst>(I), II, Worklist);11425    case Instruction::PHI:11426      return handlePHINode(A, cast<PHINode>(I), II, Worklist, LivenessAAs);11427    case Instruction::Load:11428      return handleLoadInst(A, cast<LoadInst>(I), II, Worklist);11429    default:11430      return handleGenericInst(A, I, II, Worklist);11431    };11432    return false;11433  }11434 11435  void genericValueTraversal(Attributor &A, Value *InitialV) {11436    SmallMapVector<const Function *, LivenessInfo, 4> LivenessAAs;11437 11438    SmallSet<ItemInfo, 16> Visited;11439    SmallVector<ItemInfo, 16> Worklist;11440    Worklist.push_back({{*InitialV, getCtxI()}, AA::AnyScope});11441 11442    int Iteration = 0;11443    do {11444      ItemInfo II = Worklist.pop_back_val();11445      Value *V = II.I.getValue();11446      assert(V);11447      const Instruction *CtxI = II.I.getCtxI();11448      AA::ValueScope S = II.S;11449 11450      // Check if we should process the current value. To prevent endless11451      // recursion keep a record of the values we followed!11452      if (!Visited.insert(II).second)11453        continue;11454 11455      // Make sure we limit the compile time for complex expressions.11456      if (Iteration++ >= MaxPotentialValuesIterations) {11457        LLVM_DEBUG(dbgs() << "Generic value traversal reached iteration limit: "11458                          << Iteration << "!\n");11459        addValue(A, getState(), *V, CtxI, S, getAnchorScope());11460        continue;11461      }11462 11463      // Explicitly look through calls with a "returned" attribute if we do11464      // not have a pointer as stripPointerCasts only works on them.11465      Value *NewV = nullptr;11466      if (V->getType()->isPointerTy()) {11467        NewV = AA::getWithType(*V->stripPointerCasts(), *V->getType());11468      } else {11469        if (auto *CB = dyn_cast<CallBase>(V))11470          if (auto *Callee =11471                  dyn_cast_if_present<Function>(CB->getCalledOperand())) {11472            for (Argument &Arg : Callee->args())11473              if (Arg.hasReturnedAttr()) {11474                NewV = CB->getArgOperand(Arg.getArgNo());11475                break;11476              }11477          }11478      }11479      if (NewV && NewV != V) {11480        Worklist.push_back({{*NewV, CtxI}, S});11481        continue;11482      }11483 11484      if (auto *I = dyn_cast<Instruction>(V)) {11485        if (simplifyInstruction(A, *I, II, Worklist, LivenessAAs))11486          continue;11487      }11488 11489      if (V != InitialV || isa<Argument>(V))11490        if (recurseForValue(A, IRPosition::value(*V), II.S))11491          continue;11492 11493      // If we haven't stripped anything we give up.11494      if (V == InitialV && CtxI == getCtxI()) {11495        indicatePessimisticFixpoint();11496        return;11497      }11498 11499      addValue(A, getState(), *V, CtxI, S, getAnchorScope());11500    } while (!Worklist.empty());11501 11502    // If we actually used liveness information so we have to record a11503    // dependence.11504    for (auto &It : LivenessAAs)11505      if (It.second.AnyDead)11506        A.recordDependence(*It.second.LivenessAA, *this, DepClassTy::OPTIONAL);11507  }11508 11509  /// See AbstractAttribute::trackStatistics()11510  void trackStatistics() const override {11511    STATS_DECLTRACK_FLOATING_ATTR(potential_values)11512  }11513};11514 11515struct AAPotentialValuesArgument final : AAPotentialValuesImpl {11516  using Base = AAPotentialValuesImpl;11517  AAPotentialValuesArgument(const IRPosition &IRP, Attributor &A)11518      : Base(IRP, A) {}11519 11520  /// See AbstractAttribute::initialize(..).11521  void initialize(Attributor &A) override {11522    auto &Arg = cast<Argument>(getAssociatedValue());11523    if (Arg.hasPointeeInMemoryValueAttr())11524      indicatePessimisticFixpoint();11525  }11526 11527  /// See AbstractAttribute::updateImpl(...).11528  ChangeStatus updateImpl(Attributor &A) override {11529    auto AssumedBefore = getAssumed();11530 11531    unsigned ArgNo = getCalleeArgNo();11532 11533    bool UsedAssumedInformation = false;11534    SmallVector<AA::ValueAndContext> Values;11535    auto CallSitePred = [&](AbstractCallSite ACS) {11536      const auto CSArgIRP = IRPosition::callsite_argument(ACS, ArgNo);11537      if (CSArgIRP.getPositionKind() == IRP_INVALID)11538        return false;11539 11540      if (!A.getAssumedSimplifiedValues(CSArgIRP, this, Values,11541                                        AA::Interprocedural,11542                                        UsedAssumedInformation))11543        return false;11544 11545      return isValidState();11546    };11547 11548    if (!A.checkForAllCallSites(CallSitePred, *this,11549                                /* RequireAllCallSites */ true,11550                                UsedAssumedInformation))11551      return indicatePessimisticFixpoint();11552 11553    Function *Fn = getAssociatedFunction();11554    bool AnyNonLocal = false;11555    for (auto &It : Values) {11556      if (isa<Constant>(It.getValue())) {11557        addValue(A, getState(), *It.getValue(), It.getCtxI(), AA::AnyScope,11558                 getAnchorScope());11559        continue;11560      }11561      if (!AA::isDynamicallyUnique(A, *this, *It.getValue()))11562        return indicatePessimisticFixpoint();11563 11564      if (auto *Arg = dyn_cast<Argument>(It.getValue()))11565        if (Arg->getParent() == Fn) {11566          addValue(A, getState(), *It.getValue(), It.getCtxI(), AA::AnyScope,11567                   getAnchorScope());11568          continue;11569        }11570      addValue(A, getState(), *It.getValue(), It.getCtxI(), AA::Interprocedural,11571               getAnchorScope());11572      AnyNonLocal = true;11573    }11574    assert(!undefIsContained() && "Undef should be an explicit value!");11575    if (AnyNonLocal)11576      giveUpOnIntraprocedural(A);11577 11578    return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED11579                                           : ChangeStatus::CHANGED;11580  }11581 11582  /// See AbstractAttribute::trackStatistics()11583  void trackStatistics() const override {11584    STATS_DECLTRACK_ARG_ATTR(potential_values)11585  }11586};11587 11588struct AAPotentialValuesReturned : public AAPotentialValuesFloating {11589  using Base = AAPotentialValuesFloating;11590  AAPotentialValuesReturned(const IRPosition &IRP, Attributor &A)11591      : Base(IRP, A) {}11592 11593  /// See AbstractAttribute::initialize(..).11594  void initialize(Attributor &A) override {11595    Function *F = getAssociatedFunction();11596    if (!F || F->isDeclaration() || F->getReturnType()->isVoidTy()) {11597      indicatePessimisticFixpoint();11598      return;11599    }11600 11601    for (Argument &Arg : F->args())11602      if (Arg.hasReturnedAttr()) {11603        addValue(A, getState(), Arg, nullptr, AA::AnyScope, F);11604        ReturnedArg = &Arg;11605        break;11606      }11607    if (!A.isFunctionIPOAmendable(*F) ||11608        A.hasSimplificationCallback(getIRPosition())) {11609      if (!ReturnedArg)11610        indicatePessimisticFixpoint();11611      else11612        indicateOptimisticFixpoint();11613    }11614  }11615 11616  /// See AbstractAttribute::updateImpl(...).11617  ChangeStatus updateImpl(Attributor &A) override {11618    auto AssumedBefore = getAssumed();11619    bool UsedAssumedInformation = false;11620 11621    SmallVector<AA::ValueAndContext> Values;11622    Function *AnchorScope = getAnchorScope();11623    auto HandleReturnedValue = [&](Value &V, Instruction *CtxI,11624                                   bool AddValues) {11625      for (AA::ValueScope S : {AA::Interprocedural, AA::Intraprocedural}) {11626        Values.clear();11627        if (!A.getAssumedSimplifiedValues(IRPosition::value(V), this, Values, S,11628                                          UsedAssumedInformation,11629                                          /* RecurseForSelectAndPHI */ true))11630          return false;11631        if (!AddValues)11632          continue;11633 11634        bool AllInterAreIntra = false;11635        if (S == AA::Interprocedural)11636          AllInterAreIntra =11637              llvm::all_of(Values, [&](const AA::ValueAndContext &VAC) {11638                return AA::isValidInScope(*VAC.getValue(), AnchorScope);11639              });11640 11641        for (const AA::ValueAndContext &VAC : Values) {11642          addValue(A, getState(), *VAC.getValue(),11643                   VAC.getCtxI() ? VAC.getCtxI() : CtxI,11644                   AllInterAreIntra ? AA::AnyScope : S, AnchorScope);11645        }11646        if (AllInterAreIntra)11647          break;11648      }11649      return true;11650    };11651 11652    if (ReturnedArg) {11653      HandleReturnedValue(*ReturnedArg, nullptr, true);11654    } else {11655      auto RetInstPred = [&](Instruction &RetI) {11656        bool AddValues = true;11657        if (isa<PHINode>(RetI.getOperand(0)) ||11658            isa<SelectInst>(RetI.getOperand(0))) {11659          addValue(A, getState(), *RetI.getOperand(0), &RetI, AA::AnyScope,11660                   AnchorScope);11661          AddValues = false;11662        }11663        return HandleReturnedValue(*RetI.getOperand(0), &RetI, AddValues);11664      };11665 11666      if (!A.checkForAllInstructions(RetInstPred, *this, {Instruction::Ret},11667                                     UsedAssumedInformation,11668                                     /* CheckBBLivenessOnly */ true))11669        return indicatePessimisticFixpoint();11670    }11671 11672    return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED11673                                           : ChangeStatus::CHANGED;11674  }11675 11676  ChangeStatus manifest(Attributor &A) override {11677    if (ReturnedArg)11678      return ChangeStatus::UNCHANGED;11679    SmallVector<AA::ValueAndContext> Values;11680    if (!getAssumedSimplifiedValues(A, Values, AA::ValueScope::Intraprocedural,11681                                    /* RecurseForSelectAndPHI */ true))11682      return ChangeStatus::UNCHANGED;11683    Value *NewVal = getSingleValue(A, *this, getIRPosition(), Values);11684    if (!NewVal)11685      return ChangeStatus::UNCHANGED;11686 11687    ChangeStatus Changed = ChangeStatus::UNCHANGED;11688    if (auto *Arg = dyn_cast<Argument>(NewVal)) {11689      STATS_DECLTRACK(UniqueReturnValue, FunctionReturn,11690                      "Number of function with unique return");11691      Changed |= A.manifestAttrs(11692          IRPosition::argument(*Arg),11693          {Attribute::get(Arg->getContext(), Attribute::Returned)});11694      STATS_DECLTRACK_ARG_ATTR(returned);11695    }11696 11697    auto RetInstPred = [&](Instruction &RetI) {11698      Value *RetOp = RetI.getOperand(0);11699      if (isa<UndefValue>(RetOp) || RetOp == NewVal)11700        return true;11701      if (AA::isValidAtPosition({*NewVal, RetI}, A.getInfoCache()))11702        if (A.changeUseAfterManifest(RetI.getOperandUse(0), *NewVal))11703          Changed = ChangeStatus::CHANGED;11704      return true;11705    };11706    bool UsedAssumedInformation = false;11707    (void)A.checkForAllInstructions(RetInstPred, *this, {Instruction::Ret},11708                                    UsedAssumedInformation,11709                                    /* CheckBBLivenessOnly */ true);11710    return Changed;11711  }11712 11713  ChangeStatus indicatePessimisticFixpoint() override {11714    return AAPotentialValues::indicatePessimisticFixpoint();11715  }11716 11717  /// See AbstractAttribute::trackStatistics()11718  void trackStatistics() const override{11719      STATS_DECLTRACK_FNRET_ATTR(potential_values)}11720 11721  /// The argumented with an existing `returned` attribute.11722  Argument *ReturnedArg = nullptr;11723};11724 11725struct AAPotentialValuesFunction : AAPotentialValuesImpl {11726  AAPotentialValuesFunction(const IRPosition &IRP, Attributor &A)11727      : AAPotentialValuesImpl(IRP, A) {}11728 11729  /// See AbstractAttribute::updateImpl(...).11730  ChangeStatus updateImpl(Attributor &A) override {11731    llvm_unreachable("AAPotentialValues(Function|CallSite)::updateImpl will "11732                     "not be called");11733  }11734 11735  /// See AbstractAttribute::trackStatistics()11736  void trackStatistics() const override {11737    STATS_DECLTRACK_FN_ATTR(potential_values)11738  }11739};11740 11741struct AAPotentialValuesCallSite : AAPotentialValuesFunction {11742  AAPotentialValuesCallSite(const IRPosition &IRP, Attributor &A)11743      : AAPotentialValuesFunction(IRP, A) {}11744 11745  /// See AbstractAttribute::trackStatistics()11746  void trackStatistics() const override {11747    STATS_DECLTRACK_CS_ATTR(potential_values)11748  }11749};11750 11751struct AAPotentialValuesCallSiteReturned : AAPotentialValuesImpl {11752  AAPotentialValuesCallSiteReturned(const IRPosition &IRP, Attributor &A)11753      : AAPotentialValuesImpl(IRP, A) {}11754 11755  /// See AbstractAttribute::updateImpl(...).11756  ChangeStatus updateImpl(Attributor &A) override {11757    auto AssumedBefore = getAssumed();11758 11759    Function *Callee = getAssociatedFunction();11760    if (!Callee)11761      return indicatePessimisticFixpoint();11762 11763    bool UsedAssumedInformation = false;11764    auto *CB = cast<CallBase>(getCtxI());11765    if (CB->isMustTailCall() &&11766        !A.isAssumedDead(IRPosition::inst(*CB), this, nullptr,11767                         UsedAssumedInformation))11768      return indicatePessimisticFixpoint();11769 11770    Function *Caller = CB->getCaller();11771 11772    auto AddScope = [&](AA::ValueScope S) {11773      SmallVector<AA::ValueAndContext> Values;11774      if (!A.getAssumedSimplifiedValues(IRPosition::returned(*Callee), this,11775                                        Values, S, UsedAssumedInformation))11776        return false;11777 11778      for (auto &It : Values) {11779        Value *V = It.getValue();11780        std::optional<Value *> CallerV = A.translateArgumentToCallSiteContent(11781            V, *CB, *this, UsedAssumedInformation);11782        if (!CallerV.has_value()) {11783          // Nothing to do as long as no value was determined.11784          continue;11785        }11786        V = *CallerV ? *CallerV : V;11787        if (*CallerV && AA::isDynamicallyUnique(A, *this, *V)) {11788          if (recurseForValue(A, IRPosition::value(*V), S))11789            continue;11790        }11791        if (S == AA::Intraprocedural && !AA::isValidInScope(*V, Caller)) {11792          giveUpOnIntraprocedural(A);11793          return true;11794        }11795        addValue(A, getState(), *V, CB, S, getAnchorScope());11796      }11797      return true;11798    };11799    if (!AddScope(AA::Intraprocedural))11800      return indicatePessimisticFixpoint();11801    if (!AddScope(AA::Interprocedural))11802      return indicatePessimisticFixpoint();11803    return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED11804                                           : ChangeStatus::CHANGED;11805  }11806 11807  ChangeStatus indicatePessimisticFixpoint() override {11808    return AAPotentialValues::indicatePessimisticFixpoint();11809  }11810 11811  /// See AbstractAttribute::trackStatistics()11812  void trackStatistics() const override {11813    STATS_DECLTRACK_CSRET_ATTR(potential_values)11814  }11815};11816 11817struct AAPotentialValuesCallSiteArgument : AAPotentialValuesFloating {11818  AAPotentialValuesCallSiteArgument(const IRPosition &IRP, Attributor &A)11819      : AAPotentialValuesFloating(IRP, A) {}11820 11821  /// See AbstractAttribute::trackStatistics()11822  void trackStatistics() const override {11823    STATS_DECLTRACK_CSARG_ATTR(potential_values)11824  }11825};11826} // namespace11827 11828/// ---------------------- Assumption Propagation ------------------------------11829namespace {11830struct AAAssumptionInfoImpl : public AAAssumptionInfo {11831  AAAssumptionInfoImpl(const IRPosition &IRP, Attributor &A,11832                       const DenseSet<StringRef> &Known)11833      : AAAssumptionInfo(IRP, A, Known) {}11834 11835  /// See AbstractAttribute::manifest(...).11836  ChangeStatus manifest(Attributor &A) override {11837    // Don't manifest a universal set if it somehow made it here.11838    if (getKnown().isUniversal())11839      return ChangeStatus::UNCHANGED;11840 11841    const IRPosition &IRP = getIRPosition();11842    SmallVector<StringRef, 0> Set(getAssumed().getSet().begin(),11843                                  getAssumed().getSet().end());11844    llvm::sort(Set);11845    return A.manifestAttrs(IRP,11846                           Attribute::get(IRP.getAnchorValue().getContext(),11847                                          AssumptionAttrKey,11848                                          llvm::join(Set, ",")),11849                           /*ForceReplace=*/true);11850  }11851 11852  bool hasAssumption(const StringRef Assumption) const override {11853    return isValidState() && setContains(Assumption);11854  }11855 11856  /// See AbstractAttribute::getAsStr()11857  const std::string getAsStr(Attributor *A) const override {11858    const SetContents &Known = getKnown();11859    const SetContents &Assumed = getAssumed();11860 11861    SmallVector<StringRef, 0> Set(Known.getSet().begin(), Known.getSet().end());11862    llvm::sort(Set);11863    const std::string KnownStr = llvm::join(Set, ",");11864 11865    std::string AssumedStr = "Universal";11866    if (!Assumed.isUniversal()) {11867      Set.assign(Assumed.getSet().begin(), Assumed.getSet().end());11868      AssumedStr = llvm::join(Set, ",");11869    }11870    return "Known [" + KnownStr + "]," + " Assumed [" + AssumedStr + "]";11871  }11872};11873 11874/// Propagates assumption information from parent functions to all of their11875/// successors. An assumption can be propagated if the containing function11876/// dominates the called function.11877///11878/// We start with a "known" set of assumptions already valid for the associated11879/// function and an "assumed" set that initially contains all possible11880/// assumptions. The assumed set is inter-procedurally updated by narrowing its11881/// contents as concrete values are known. The concrete values are seeded by the11882/// first nodes that are either entries into the call graph, or contains no11883/// assumptions. Each node is updated as the intersection of the assumed state11884/// with all of its predecessors.11885struct AAAssumptionInfoFunction final : AAAssumptionInfoImpl {11886  AAAssumptionInfoFunction(const IRPosition &IRP, Attributor &A)11887      : AAAssumptionInfoImpl(IRP, A,11888                             getAssumptions(*IRP.getAssociatedFunction())) {}11889 11890  /// See AbstractAttribute::updateImpl(...).11891  ChangeStatus updateImpl(Attributor &A) override {11892    bool Changed = false;11893 11894    auto CallSitePred = [&](AbstractCallSite ACS) {11895      const auto *AssumptionAA = A.getAAFor<AAAssumptionInfo>(11896          *this, IRPosition::callsite_function(*ACS.getInstruction()),11897          DepClassTy::REQUIRED);11898      if (!AssumptionAA)11899        return false;11900      // Get the set of assumptions shared by all of this function's callers.11901      Changed |= getIntersection(AssumptionAA->getAssumed());11902      return !getAssumed().empty() || !getKnown().empty();11903    };11904 11905    bool UsedAssumedInformation = false;11906    // Get the intersection of all assumptions held by this node's predecessors.11907    // If we don't know all the call sites then this is either an entry into the11908    // call graph or an empty node. This node is known to only contain its own11909    // assumptions and can be propagated to its successors.11910    if (!A.checkForAllCallSites(CallSitePred, *this, true,11911                                UsedAssumedInformation))11912      return indicatePessimisticFixpoint();11913 11914    return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;11915  }11916 11917  void trackStatistics() const override {}11918};11919 11920/// Assumption Info defined for call sites.11921struct AAAssumptionInfoCallSite final : AAAssumptionInfoImpl {11922 11923  AAAssumptionInfoCallSite(const IRPosition &IRP, Attributor &A)11924      : AAAssumptionInfoImpl(IRP, A, getInitialAssumptions(IRP)) {}11925 11926  /// See AbstractAttribute::initialize(...).11927  void initialize(Attributor &A) override {11928    const IRPosition &FnPos = IRPosition::function(*getAnchorScope());11929    A.getAAFor<AAAssumptionInfo>(*this, FnPos, DepClassTy::REQUIRED);11930  }11931 11932  /// See AbstractAttribute::updateImpl(...).11933  ChangeStatus updateImpl(Attributor &A) override {11934    const IRPosition &FnPos = IRPosition::function(*getAnchorScope());11935    auto *AssumptionAA =11936        A.getAAFor<AAAssumptionInfo>(*this, FnPos, DepClassTy::REQUIRED);11937    if (!AssumptionAA)11938      return indicatePessimisticFixpoint();11939    bool Changed = getIntersection(AssumptionAA->getAssumed());11940    return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;11941  }11942 11943  /// See AbstractAttribute::trackStatistics()11944  void trackStatistics() const override {}11945 11946private:11947  /// Helper to initialized the known set as all the assumptions this call and11948  /// the callee contain.11949  DenseSet<StringRef> getInitialAssumptions(const IRPosition &IRP) {11950    const CallBase &CB = cast<CallBase>(IRP.getAssociatedValue());11951    auto Assumptions = getAssumptions(CB);11952    if (const Function *F = CB.getCaller())11953      set_union(Assumptions, getAssumptions(*F));11954    if (Function *F = IRP.getAssociatedFunction())11955      set_union(Assumptions, getAssumptions(*F));11956    return Assumptions;11957  }11958};11959} // namespace11960 11961AACallGraphNode *AACallEdgeIterator::operator*() const {11962  return static_cast<AACallGraphNode *>(const_cast<AACallEdges *>(11963      A.getOrCreateAAFor<AACallEdges>(IRPosition::function(**I))));11964}11965 11966void AttributorCallGraph::print() { llvm::WriteGraph(outs(), this); }11967 11968/// ------------------------ UnderlyingObjects ---------------------------------11969 11970namespace {11971struct AAUnderlyingObjectsImpl11972    : StateWrapper<BooleanState, AAUnderlyingObjects> {11973  using BaseTy = StateWrapper<BooleanState, AAUnderlyingObjects>;11974  AAUnderlyingObjectsImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}11975 11976  /// See AbstractAttribute::getAsStr().11977  const std::string getAsStr(Attributor *A) const override {11978    if (!isValidState())11979      return "<invalid>";11980    std::string Str;11981    llvm::raw_string_ostream OS(Str);11982    OS << "underlying objects: inter " << InterAssumedUnderlyingObjects.size()11983       << " objects, intra " << IntraAssumedUnderlyingObjects.size()11984       << " objects.\n";11985    if (!InterAssumedUnderlyingObjects.empty()) {11986      OS << "inter objects:\n";11987      for (auto *Obj : InterAssumedUnderlyingObjects)11988        OS << *Obj << '\n';11989    }11990    if (!IntraAssumedUnderlyingObjects.empty()) {11991      OS << "intra objects:\n";11992      for (auto *Obj : IntraAssumedUnderlyingObjects)11993        OS << *Obj << '\n';11994    }11995    return Str;11996  }11997 11998  /// See AbstractAttribute::trackStatistics()11999  void trackStatistics() const override {}12000 12001  /// See AbstractAttribute::updateImpl(...).12002  ChangeStatus updateImpl(Attributor &A) override {12003    auto &Ptr = getAssociatedValue();12004 12005    bool UsedAssumedInformation = false;12006    auto DoUpdate = [&](SmallSetVector<Value *, 8> &UnderlyingObjects,12007                        AA::ValueScope Scope) {12008      SmallPtrSet<Value *, 8> SeenObjects;12009      SmallVector<AA::ValueAndContext> Values;12010 12011      if (!A.getAssumedSimplifiedValues(IRPosition::value(Ptr), *this, Values,12012                                        Scope, UsedAssumedInformation))12013        return UnderlyingObjects.insert(&Ptr);12014 12015      bool Changed = false;12016 12017      for (unsigned I = 0; I < Values.size(); ++I) {12018        auto &VAC = Values[I];12019        auto *Obj = VAC.getValue();12020        Value *UO = getUnderlyingObject(Obj);12021        if (!SeenObjects.insert(UO ? UO : Obj).second)12022          continue;12023        if (UO && UO != Obj) {12024          if (isa<AllocaInst>(UO) || isa<GlobalValue>(UO)) {12025            Changed |= UnderlyingObjects.insert(UO);12026            continue;12027          }12028 12029          const auto *OtherAA = A.getAAFor<AAUnderlyingObjects>(12030              *this, IRPosition::value(*UO), DepClassTy::OPTIONAL);12031          auto Pred = [&](Value &V) {12032            if (&V == UO)12033              Changed |= UnderlyingObjects.insert(UO);12034            else12035              Values.emplace_back(V, nullptr);12036            return true;12037          };12038 12039          if (!OtherAA || !OtherAA->forallUnderlyingObjects(Pred, Scope))12040            llvm_unreachable(12041                "The forall call should not return false at this position");12042          UsedAssumedInformation |= !OtherAA->getState().isAtFixpoint();12043          continue;12044        }12045 12046        if (isa<SelectInst>(Obj)) {12047          Changed |= handleIndirect(A, *Obj, UnderlyingObjects, Scope,12048                                    UsedAssumedInformation);12049          continue;12050        }12051        if (auto *PHI = dyn_cast<PHINode>(Obj)) {12052          // Explicitly look through PHIs as we do not care about dynamically12053          // uniqueness.12054          for (unsigned u = 0, e = PHI->getNumIncomingValues(); u < e; u++) {12055            Changed |=12056                handleIndirect(A, *PHI->getIncomingValue(u), UnderlyingObjects,12057                               Scope, UsedAssumedInformation);12058          }12059          continue;12060        }12061 12062        Changed |= UnderlyingObjects.insert(Obj);12063      }12064 12065      return Changed;12066    };12067 12068    bool Changed = false;12069    Changed |= DoUpdate(IntraAssumedUnderlyingObjects, AA::Intraprocedural);12070    Changed |= DoUpdate(InterAssumedUnderlyingObjects, AA::Interprocedural);12071    if (!UsedAssumedInformation)12072      indicateOptimisticFixpoint();12073    return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;12074  }12075 12076  bool forallUnderlyingObjects(12077      function_ref<bool(Value &)> Pred,12078      AA::ValueScope Scope = AA::Interprocedural) const override {12079    if (!isValidState())12080      return Pred(getAssociatedValue());12081 12082    auto &AssumedUnderlyingObjects = Scope == AA::Intraprocedural12083                                         ? IntraAssumedUnderlyingObjects12084                                         : InterAssumedUnderlyingObjects;12085    for (Value *Obj : AssumedUnderlyingObjects)12086      if (!Pred(*Obj))12087        return false;12088 12089    return true;12090  }12091 12092private:12093  /// Handle the case where the value is not the actual underlying value, such12094  /// as a phi node or a select instruction.12095  bool handleIndirect(Attributor &A, Value &V,12096                      SmallSetVector<Value *, 8> &UnderlyingObjects,12097                      AA::ValueScope Scope, bool &UsedAssumedInformation) {12098    bool Changed = false;12099    const auto *AA = A.getAAFor<AAUnderlyingObjects>(12100        *this, IRPosition::value(V), DepClassTy::OPTIONAL);12101    auto Pred = [&](Value &V) {12102      Changed |= UnderlyingObjects.insert(&V);12103      return true;12104    };12105    if (!AA || !AA->forallUnderlyingObjects(Pred, Scope))12106      llvm_unreachable(12107          "The forall call should not return false at this position");12108    UsedAssumedInformation |= !AA->getState().isAtFixpoint();12109    return Changed;12110  }12111 12112  /// All the underlying objects collected so far via intra procedural scope.12113  SmallSetVector<Value *, 8> IntraAssumedUnderlyingObjects;12114  /// All the underlying objects collected so far via inter procedural scope.12115  SmallSetVector<Value *, 8> InterAssumedUnderlyingObjects;12116};12117 12118struct AAUnderlyingObjectsFloating final : AAUnderlyingObjectsImpl {12119  AAUnderlyingObjectsFloating(const IRPosition &IRP, Attributor &A)12120      : AAUnderlyingObjectsImpl(IRP, A) {}12121};12122 12123struct AAUnderlyingObjectsArgument final : AAUnderlyingObjectsImpl {12124  AAUnderlyingObjectsArgument(const IRPosition &IRP, Attributor &A)12125      : AAUnderlyingObjectsImpl(IRP, A) {}12126};12127 12128struct AAUnderlyingObjectsCallSite final : AAUnderlyingObjectsImpl {12129  AAUnderlyingObjectsCallSite(const IRPosition &IRP, Attributor &A)12130      : AAUnderlyingObjectsImpl(IRP, A) {}12131};12132 12133struct AAUnderlyingObjectsCallSiteArgument final : AAUnderlyingObjectsImpl {12134  AAUnderlyingObjectsCallSiteArgument(const IRPosition &IRP, Attributor &A)12135      : AAUnderlyingObjectsImpl(IRP, A) {}12136};12137 12138struct AAUnderlyingObjectsReturned final : AAUnderlyingObjectsImpl {12139  AAUnderlyingObjectsReturned(const IRPosition &IRP, Attributor &A)12140      : AAUnderlyingObjectsImpl(IRP, A) {}12141};12142 12143struct AAUnderlyingObjectsCallSiteReturned final : AAUnderlyingObjectsImpl {12144  AAUnderlyingObjectsCallSiteReturned(const IRPosition &IRP, Attributor &A)12145      : AAUnderlyingObjectsImpl(IRP, A) {}12146};12147 12148struct AAUnderlyingObjectsFunction final : AAUnderlyingObjectsImpl {12149  AAUnderlyingObjectsFunction(const IRPosition &IRP, Attributor &A)12150      : AAUnderlyingObjectsImpl(IRP, A) {}12151};12152} // namespace12153 12154/// ------------------------ Global Value Info  -------------------------------12155namespace {12156struct AAGlobalValueInfoFloating : public AAGlobalValueInfo {12157  AAGlobalValueInfoFloating(const IRPosition &IRP, Attributor &A)12158      : AAGlobalValueInfo(IRP, A) {}12159 12160  /// See AbstractAttribute::initialize(...).12161  void initialize(Attributor &A) override {}12162 12163  bool checkUse(Attributor &A, const Use &U, bool &Follow,12164                SmallVectorImpl<const Value *> &Worklist) {12165    Instruction *UInst = dyn_cast<Instruction>(U.getUser());12166    if (!UInst) {12167      Follow = true;12168      return true;12169    }12170 12171    LLVM_DEBUG(dbgs() << "[AAGlobalValueInfo] Check use: " << *U.get() << " in "12172                      << *UInst << "\n");12173 12174    if (auto *Cmp = dyn_cast<ICmpInst>(U.getUser())) {12175      int Idx = &Cmp->getOperandUse(0) == &U;12176      if (isa<Constant>(Cmp->getOperand(Idx)))12177        return true;12178      return U == &getAnchorValue();12179    }12180 12181    // Explicitly catch return instructions.12182    if (isa<ReturnInst>(UInst)) {12183      auto CallSitePred = [&](AbstractCallSite ACS) {12184        Worklist.push_back(ACS.getInstruction());12185        return true;12186      };12187      bool UsedAssumedInformation = false;12188      // TODO: We should traverse the uses or add a "non-call-site" CB.12189      if (!A.checkForAllCallSites(CallSitePred, *UInst->getFunction(),12190                                  /*RequireAllCallSites=*/true, this,12191                                  UsedAssumedInformation))12192        return false;12193      return true;12194    }12195 12196    // For now we only use special logic for call sites. However, the tracker12197    // itself knows about a lot of other non-capturing cases already.12198    auto *CB = dyn_cast<CallBase>(UInst);12199    if (!CB)12200      return false;12201    // Direct calls are OK uses.12202    if (CB->isCallee(&U))12203      return true;12204    // Non-argument uses are scary.12205    if (!CB->isArgOperand(&U))12206      return false;12207    // TODO: Iterate callees.12208    auto *Fn = dyn_cast<Function>(CB->getCalledOperand());12209    if (!Fn || !A.isFunctionIPOAmendable(*Fn))12210      return false;12211 12212    unsigned ArgNo = CB->getArgOperandNo(&U);12213    Worklist.push_back(Fn->getArg(ArgNo));12214    return true;12215  }12216 12217  ChangeStatus updateImpl(Attributor &A) override {12218    unsigned NumUsesBefore = Uses.size();12219 12220    SmallPtrSet<const Value *, 8> Visited;12221    SmallVector<const Value *> Worklist;12222    Worklist.push_back(&getAnchorValue());12223 12224    auto UsePred = [&](const Use &U, bool &Follow) -> bool {12225      Uses.insert(&U);12226      // TODO(captures): Make this more precise.12227      UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);12228      if (CI.isPassthrough()) {12229        Follow = true;12230        return true;12231      }12232      return checkUse(A, U, Follow, Worklist);12233    };12234    auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {12235      Uses.insert(&OldU);12236      return true;12237    };12238 12239    while (!Worklist.empty()) {12240      const Value *V = Worklist.pop_back_val();12241      if (!Visited.insert(V).second)12242        continue;12243      if (!A.checkForAllUses(UsePred, *this, *V,12244                             /* CheckBBLivenessOnly */ true,12245                             DepClassTy::OPTIONAL,12246                             /* IgnoreDroppableUses */ true, EquivalentUseCB)) {12247        return indicatePessimisticFixpoint();12248      }12249    }12250 12251    return Uses.size() == NumUsesBefore ? ChangeStatus::UNCHANGED12252                                        : ChangeStatus::CHANGED;12253  }12254 12255  bool isPotentialUse(const Use &U) const override {12256    return !isValidState() || Uses.contains(&U);12257  }12258 12259  /// See AbstractAttribute::manifest(...).12260  ChangeStatus manifest(Attributor &A) override {12261    return ChangeStatus::UNCHANGED;12262  }12263 12264  /// See AbstractAttribute::getAsStr().12265  const std::string getAsStr(Attributor *A) const override {12266    return "[" + std::to_string(Uses.size()) + " uses]";12267  }12268 12269  void trackStatistics() const override {12270    STATS_DECLTRACK_FLOATING_ATTR(GlobalValuesTracked);12271  }12272 12273private:12274  /// Set of (transitive) uses of this GlobalValue.12275  SmallPtrSet<const Use *, 8> Uses;12276};12277} // namespace12278 12279/// ------------------------ Indirect Call Info  -------------------------------12280namespace {12281struct AAIndirectCallInfoCallSite : public AAIndirectCallInfo {12282  AAIndirectCallInfoCallSite(const IRPosition &IRP, Attributor &A)12283      : AAIndirectCallInfo(IRP, A) {}12284 12285  /// See AbstractAttribute::initialize(...).12286  void initialize(Attributor &A) override {12287    auto *MD = getCtxI()->getMetadata(LLVMContext::MD_callees);12288    if (!MD && !A.isClosedWorldModule())12289      return;12290 12291    if (MD) {12292      for (const auto &Op : MD->operands())12293        if (Function *Callee = mdconst::dyn_extract_or_null<Function>(Op))12294          PotentialCallees.insert(Callee);12295    } else if (A.isClosedWorldModule()) {12296      ArrayRef<Function *> IndirectlyCallableFunctions =12297          A.getInfoCache().getIndirectlyCallableFunctions(A);12298      PotentialCallees.insert_range(IndirectlyCallableFunctions);12299    }12300 12301    if (PotentialCallees.empty())12302      indicateOptimisticFixpoint();12303  }12304 12305  ChangeStatus updateImpl(Attributor &A) override {12306    CallBase *CB = cast<CallBase>(getCtxI());12307    const Use &CalleeUse = CB->getCalledOperandUse();12308    Value *FP = CB->getCalledOperand();12309 12310    SmallSetVector<Function *, 4> AssumedCalleesNow;12311    bool AllCalleesKnownNow = AllCalleesKnown;12312 12313    auto CheckPotentialCalleeUse = [&](Function &PotentialCallee,12314                                       bool &UsedAssumedInformation) {12315      const auto *GIAA = A.getAAFor<AAGlobalValueInfo>(12316          *this, IRPosition::value(PotentialCallee), DepClassTy::OPTIONAL);12317      if (!GIAA || GIAA->isPotentialUse(CalleeUse))12318        return true;12319      UsedAssumedInformation = !GIAA->isAtFixpoint();12320      return false;12321    };12322 12323    auto AddPotentialCallees = [&]() {12324      for (auto *PotentialCallee : PotentialCallees) {12325        bool UsedAssumedInformation = false;12326        if (CheckPotentialCalleeUse(*PotentialCallee, UsedAssumedInformation))12327          AssumedCalleesNow.insert(PotentialCallee);12328      }12329    };12330 12331    // Use simplification to find potential callees, if !callees was present,12332    // fallback to that set if necessary.12333    bool UsedAssumedInformation = false;12334    SmallVector<AA::ValueAndContext> Values;12335    if (!A.getAssumedSimplifiedValues(IRPosition::value(*FP), this, Values,12336                                      AA::ValueScope::AnyScope,12337                                      UsedAssumedInformation)) {12338      if (PotentialCallees.empty())12339        return indicatePessimisticFixpoint();12340      AddPotentialCallees();12341    }12342 12343    // Try to find a reason for \p Fn not to be a potential callee. If none was12344    // found, add it to the assumed callees set.12345    auto CheckPotentialCallee = [&](Function &Fn) {12346      if (!PotentialCallees.empty() && !PotentialCallees.count(&Fn))12347        return false;12348 12349      auto &CachedResult = FilterResults[&Fn];12350      if (CachedResult.has_value())12351        return CachedResult.value();12352 12353      bool UsedAssumedInformation = false;12354      if (!CheckPotentialCalleeUse(Fn, UsedAssumedInformation)) {12355        if (!UsedAssumedInformation)12356          CachedResult = false;12357        return false;12358      }12359 12360      int NumFnArgs = Fn.arg_size();12361      int NumCBArgs = CB->arg_size();12362 12363      // Check if any excess argument (which we fill up with poison) is known to12364      // be UB on undef.12365      for (int I = NumCBArgs; I < NumFnArgs; ++I) {12366        bool IsKnown = false;12367        if (AA::hasAssumedIRAttr<Attribute::NoUndef>(12368                A, this, IRPosition::argument(*Fn.getArg(I)),12369                DepClassTy::OPTIONAL, IsKnown)) {12370          if (IsKnown)12371            CachedResult = false;12372          return false;12373        }12374      }12375 12376      CachedResult = true;12377      return true;12378    };12379 12380    // Check simplification result, prune known UB callees, also restrict it to12381    // the !callees set, if present.12382    for (auto &VAC : Values) {12383      if (isa<UndefValue>(VAC.getValue()))12384        continue;12385      if (isa<ConstantPointerNull>(VAC.getValue()) &&12386          VAC.getValue()->getType()->getPointerAddressSpace() == 0)12387        continue;12388      // TODO: Check for known UB, e.g., poison + noundef.12389      if (auto *VACFn = dyn_cast<Function>(VAC.getValue())) {12390        if (CheckPotentialCallee(*VACFn))12391          AssumedCalleesNow.insert(VACFn);12392        continue;12393      }12394      if (!PotentialCallees.empty()) {12395        AddPotentialCallees();12396        break;12397      }12398      AllCalleesKnownNow = false;12399    }12400 12401    if (AssumedCalleesNow == AssumedCallees &&12402        AllCalleesKnown == AllCalleesKnownNow)12403      return ChangeStatus::UNCHANGED;12404 12405    std::swap(AssumedCallees, AssumedCalleesNow);12406    AllCalleesKnown = AllCalleesKnownNow;12407    return ChangeStatus::CHANGED;12408  }12409 12410  /// See AbstractAttribute::manifest(...).12411  ChangeStatus manifest(Attributor &A) override {12412    // If we can't specialize at all, give up now.12413    if (!AllCalleesKnown && AssumedCallees.empty())12414      return ChangeStatus::UNCHANGED;12415 12416    CallBase *CB = cast<CallBase>(getCtxI());12417    bool UsedAssumedInformation = false;12418    if (A.isAssumedDead(*CB, this, /*LivenessAA=*/nullptr,12419                        UsedAssumedInformation))12420      return ChangeStatus::UNCHANGED;12421 12422    ChangeStatus Changed = ChangeStatus::UNCHANGED;12423    Value *FP = CB->getCalledOperand();12424    if (FP->getType()->getPointerAddressSpace())12425      FP = new AddrSpaceCastInst(FP, PointerType::get(FP->getContext(), 0),12426                                 FP->getName() + ".as0", CB->getIterator());12427 12428    bool CBIsVoid = CB->getType()->isVoidTy();12429    BasicBlock::iterator IP = CB->getIterator();12430    FunctionType *CSFT = CB->getFunctionType();12431    SmallVector<Value *> CSArgs(CB->args());12432 12433    // If we know all callees and there are none, the call site is (effectively)12434    // dead (or UB).12435    if (AssumedCallees.empty()) {12436      assert(AllCalleesKnown &&12437             "Expected all callees to be known if there are none.");12438      A.changeToUnreachableAfterManifest(CB);12439      return ChangeStatus::CHANGED;12440    }12441 12442    // Special handling for the single callee case.12443    if (AllCalleesKnown && AssumedCallees.size() == 1) {12444      auto *NewCallee = AssumedCallees.front();12445      if (isLegalToPromote(*CB, NewCallee)) {12446        promoteCall(*CB, NewCallee, nullptr);12447        NumIndirectCallsPromoted++;12448        return ChangeStatus::CHANGED;12449      }12450      Instruction *NewCall =12451          CallInst::Create(FunctionCallee(CSFT, NewCallee), CSArgs,12452                           CB->getName(), CB->getIterator());12453      if (!CBIsVoid)12454        A.changeAfterManifest(IRPosition::callsite_returned(*CB), *NewCall);12455      A.deleteAfterManifest(*CB);12456      return ChangeStatus::CHANGED;12457    }12458 12459    // For each potential value we create a conditional12460    //12461    // ```12462    // if (ptr == value) value(args);12463    // else ...12464    // ```12465    //12466    bool SpecializedForAnyCallees = false;12467    bool SpecializedForAllCallees = AllCalleesKnown;12468    ICmpInst *LastCmp = nullptr;12469    SmallVector<Function *, 8> SkippedAssumedCallees;12470    SmallVector<std::pair<CallInst *, Instruction *>> NewCalls;12471    for (Function *NewCallee : AssumedCallees) {12472      if (!A.shouldSpecializeCallSiteForCallee(*this, *CB, *NewCallee,12473                                               AssumedCallees.size())) {12474        SkippedAssumedCallees.push_back(NewCallee);12475        SpecializedForAllCallees = false;12476        continue;12477      }12478      SpecializedForAnyCallees = true;12479 12480      LastCmp = new ICmpInst(IP, llvm::CmpInst::ICMP_EQ, FP, NewCallee);12481      Instruction *ThenTI =12482          SplitBlockAndInsertIfThen(LastCmp, IP, /* Unreachable */ false);12483      BasicBlock *CBBB = CB->getParent();12484      A.registerManifestAddedBasicBlock(*ThenTI->getParent());12485      A.registerManifestAddedBasicBlock(*IP->getParent());12486      auto *SplitTI = cast<BranchInst>(LastCmp->getNextNode());12487      BasicBlock *ElseBB;12488      if (&*IP == CB) {12489        ElseBB = BasicBlock::Create(ThenTI->getContext(), "",12490                                    ThenTI->getFunction(), CBBB);12491        A.registerManifestAddedBasicBlock(*ElseBB);12492        IP = BranchInst::Create(CBBB, ElseBB)->getIterator();12493        SplitTI->replaceUsesOfWith(CBBB, ElseBB);12494      } else {12495        ElseBB = IP->getParent();12496        ThenTI->replaceUsesOfWith(ElseBB, CBBB);12497      }12498      CastInst *RetBC = nullptr;12499      CallInst *NewCall = nullptr;12500      if (isLegalToPromote(*CB, NewCallee)) {12501        auto *CBClone = cast<CallBase>(CB->clone());12502        CBClone->insertBefore(ThenTI->getIterator());12503        NewCall = &cast<CallInst>(promoteCall(*CBClone, NewCallee, &RetBC));12504        NumIndirectCallsPromoted++;12505      } else {12506        NewCall = CallInst::Create(FunctionCallee(CSFT, NewCallee), CSArgs,12507                                   CB->getName(), ThenTI->getIterator());12508      }12509      NewCalls.push_back({NewCall, RetBC});12510    }12511 12512    auto AttachCalleeMetadata = [&](CallBase &IndirectCB) {12513      if (!AllCalleesKnown)12514        return ChangeStatus::UNCHANGED;12515      MDBuilder MDB(IndirectCB.getContext());12516      MDNode *Callees = MDB.createCallees(SkippedAssumedCallees);12517      IndirectCB.setMetadata(LLVMContext::MD_callees, Callees);12518      return ChangeStatus::CHANGED;12519    };12520 12521    if (!SpecializedForAnyCallees)12522      return AttachCalleeMetadata(*CB);12523 12524    // Check if we need the fallback indirect call still.12525    if (SpecializedForAllCallees) {12526      LastCmp->replaceAllUsesWith(ConstantInt::getTrue(LastCmp->getContext()));12527      LastCmp->eraseFromParent();12528      new UnreachableInst(IP->getContext(), IP);12529      IP->eraseFromParent();12530    } else {12531      auto *CBClone = cast<CallInst>(CB->clone());12532      CBClone->setName(CB->getName());12533      CBClone->insertBefore(*IP->getParent(), IP);12534      NewCalls.push_back({CBClone, nullptr});12535      AttachCalleeMetadata(*CBClone);12536    }12537 12538    // Check if we need a PHI to merge the results.12539    if (!CBIsVoid) {12540      auto *PHI = PHINode::Create(CB->getType(), NewCalls.size(),12541                                  CB->getName() + ".phi",12542                                  CB->getParent()->getFirstInsertionPt());12543      for (auto &It : NewCalls) {12544        CallBase *NewCall = It.first;12545        Instruction *CallRet = It.second ? It.second : It.first;12546        if (CallRet->getType() == CB->getType())12547          PHI->addIncoming(CallRet, CallRet->getParent());12548        else if (NewCall->getType()->isVoidTy())12549          PHI->addIncoming(PoisonValue::get(CB->getType()),12550                           NewCall->getParent());12551        else12552          llvm_unreachable("Call return should match or be void!");12553      }12554      A.changeAfterManifest(IRPosition::callsite_returned(*CB), *PHI);12555    }12556 12557    A.deleteAfterManifest(*CB);12558    Changed = ChangeStatus::CHANGED;12559 12560    return Changed;12561  }12562 12563  /// See AbstractAttribute::getAsStr().12564  const std::string getAsStr(Attributor *A) const override {12565    return std::string(AllCalleesKnown ? "eliminate" : "specialize") +12566           " indirect call site with " + std::to_string(AssumedCallees.size()) +12567           " functions";12568  }12569 12570  void trackStatistics() const override {12571    if (AllCalleesKnown) {12572      STATS_DECLTRACK(12573          Eliminated, CallSites,12574          "Number of indirect call sites eliminated via specialization")12575    } else {12576      STATS_DECLTRACK(Specialized, CallSites,12577                      "Number of indirect call sites specialized")12578    }12579  }12580 12581  bool foreachCallee(function_ref<bool(Function *)> CB) const override {12582    return isValidState() && AllCalleesKnown && all_of(AssumedCallees, CB);12583  }12584 12585private:12586  /// Map to remember filter results.12587  DenseMap<Function *, std::optional<bool>> FilterResults;12588 12589  /// If the !callee metadata was present, this set will contain all potential12590  /// callees (superset).12591  SmallSetVector<Function *, 4> PotentialCallees;12592 12593  /// This set contains all currently assumed calllees, which might grow over12594  /// time.12595  SmallSetVector<Function *, 4> AssumedCallees;12596 12597  /// Flag to indicate if all possible callees are in the AssumedCallees set or12598  /// if there could be others.12599  bool AllCalleesKnown = true;12600};12601} // namespace12602 12603/// --------------------- Invariant Load Pointer -------------------------------12604namespace {12605 12606struct AAInvariantLoadPointerImpl12607    : public StateWrapper<BitIntegerState<uint8_t, 15>,12608                          AAInvariantLoadPointer> {12609 12610  enum {12611    // pointer does not alias within the bounds of the function12612    IS_NOALIAS = 1 << 0,12613    // pointer is not involved in any effectful instructions within the bounds12614    // of the function12615    IS_NOEFFECT = 1 << 1,12616    // loads are invariant within the bounds of the function12617    IS_LOCALLY_INVARIANT = 1 << 2,12618    // memory lifetime is constrained within the bounds of the function12619    IS_LOCALLY_CONSTRAINED = 1 << 3,12620 12621    IS_BEST_STATE = IS_NOALIAS | IS_NOEFFECT | IS_LOCALLY_INVARIANT |12622                    IS_LOCALLY_CONSTRAINED,12623  };12624  static_assert(getBestState() == IS_BEST_STATE, "Unexpected best state");12625 12626  using Base =12627      StateWrapper<BitIntegerState<uint8_t, 15>, AAInvariantLoadPointer>;12628 12629  // the BitIntegerState is optimistic about IS_NOALIAS and IS_NOEFFECT, but12630  // pessimistic about IS_KNOWN_INVARIANT12631  AAInvariantLoadPointerImpl(const IRPosition &IRP, Attributor &A)12632      : Base(IRP) {}12633 12634  bool isKnownInvariant() const final {12635    return isKnownLocallyInvariant() && isKnown(IS_LOCALLY_CONSTRAINED);12636  }12637 12638  bool isKnownLocallyInvariant() const final {12639    if (isKnown(IS_LOCALLY_INVARIANT))12640      return true;12641    return isKnown(IS_NOALIAS | IS_NOEFFECT);12642  }12643 12644  bool isAssumedInvariant() const final {12645    return isAssumedLocallyInvariant() && isAssumed(IS_LOCALLY_CONSTRAINED);12646  }12647 12648  bool isAssumedLocallyInvariant() const final {12649    if (isAssumed(IS_LOCALLY_INVARIANT))12650      return true;12651    return isAssumed(IS_NOALIAS | IS_NOEFFECT);12652  }12653 12654  ChangeStatus updateImpl(Attributor &A) override {12655    ChangeStatus Changed = ChangeStatus::UNCHANGED;12656 12657    Changed |= updateNoAlias(A);12658    if (requiresNoAlias() && !isAssumed(IS_NOALIAS))12659      return indicatePessimisticFixpoint();12660 12661    Changed |= updateNoEffect(A);12662 12663    Changed |= updateLocalInvariance(A);12664 12665    return Changed;12666  }12667 12668  ChangeStatus manifest(Attributor &A) override {12669    if (!isKnownInvariant())12670      return ChangeStatus::UNCHANGED;12671 12672    ChangeStatus Changed = ChangeStatus::UNCHANGED;12673    const Value *Ptr = &getAssociatedValue();12674    const auto TagInvariantLoads = [&](const Use &U, bool &) {12675      if (U.get() != Ptr)12676        return true;12677      auto *I = dyn_cast<Instruction>(U.getUser());12678      if (!I)12679        return true;12680 12681      // Ensure that we are only changing uses from the corresponding callgraph12682      // SSC in the case that the AA isn't run on the entire module12683      if (!A.isRunOn(I->getFunction()))12684        return true;12685 12686      if (I->hasMetadata(LLVMContext::MD_invariant_load))12687        return true;12688 12689      if (auto *LI = dyn_cast<LoadInst>(I)) {12690        LI->setMetadata(LLVMContext::MD_invariant_load,12691                        MDNode::get(LI->getContext(), {}));12692        Changed = ChangeStatus::CHANGED;12693      }12694      return true;12695    };12696 12697    (void)A.checkForAllUses(TagInvariantLoads, *this, *Ptr);12698    return Changed;12699  }12700 12701  /// See AbstractAttribute::getAsStr().12702  const std::string getAsStr(Attributor *) const override {12703    if (isKnownInvariant())12704      return "load-invariant pointer";12705    return "non-invariant pointer";12706  }12707 12708  /// See AbstractAttribute::trackStatistics().12709  void trackStatistics() const override {}12710 12711private:12712  /// Indicate that noalias is required for the pointer to be invariant.12713  bool requiresNoAlias() const {12714    switch (getPositionKind()) {12715    default:12716      // Conservatively default to require noalias.12717      return true;12718    case IRP_FLOAT:12719    case IRP_RETURNED:12720    case IRP_CALL_SITE:12721      return false;12722    case IRP_CALL_SITE_RETURNED: {12723      const auto &CB = cast<CallBase>(getAnchorValue());12724      return !isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(12725          &CB, /*MustPreserveNullness=*/false);12726    }12727    case IRP_ARGUMENT: {12728      const Function *F = getAssociatedFunction();12729      assert(F && "no associated function for argument");12730      return !isCallableCC(F->getCallingConv());12731    }12732    }12733  }12734 12735  bool isExternal() const {12736    const Function *F = getAssociatedFunction();12737    if (!F)12738      return true;12739    return isCallableCC(F->getCallingConv()) &&12740           getPositionKind() != IRP_CALL_SITE_RETURNED;12741  }12742 12743  ChangeStatus updateNoAlias(Attributor &A) {12744    if (isKnown(IS_NOALIAS) || !isAssumed(IS_NOALIAS))12745      return ChangeStatus::UNCHANGED;12746 12747    // Try to use AANoAlias.12748    if (const auto *ANoAlias = A.getOrCreateAAFor<AANoAlias>(12749            getIRPosition(), this, DepClassTy::REQUIRED)) {12750      if (ANoAlias->isKnownNoAlias()) {12751        addKnownBits(IS_NOALIAS);12752        return ChangeStatus::CHANGED;12753      }12754 12755      if (!ANoAlias->isAssumedNoAlias()) {12756        removeAssumedBits(IS_NOALIAS);12757        return ChangeStatus::CHANGED;12758      }12759 12760      return ChangeStatus::UNCHANGED;12761    }12762 12763    // Try to infer noalias from argument attribute, since it is applicable for12764    // the duration of the function.12765    if (const Argument *Arg = getAssociatedArgument()) {12766      if (Arg->hasNoAliasAttr()) {12767        addKnownBits(IS_NOALIAS);12768        return ChangeStatus::UNCHANGED;12769      }12770 12771      // Noalias information is not provided, and cannot be inferred,12772      // so we conservatively assume the pointer aliases.12773      removeAssumedBits(IS_NOALIAS);12774      return ChangeStatus::CHANGED;12775    }12776 12777    return ChangeStatus::UNCHANGED;12778  }12779 12780  ChangeStatus updateNoEffect(Attributor &A) {12781    if (isKnown(IS_NOEFFECT) || !isAssumed(IS_NOEFFECT))12782      return ChangeStatus::UNCHANGED;12783 12784    if (!getAssociatedFunction())12785      return indicatePessimisticFixpoint();12786 12787    if (isa<AllocaInst>(&getAssociatedValue()))12788      return indicatePessimisticFixpoint();12789 12790    const auto HasNoEffectLoads = [&](const Use &U, bool &) {12791      const auto *LI = dyn_cast<LoadInst>(U.getUser());12792      return !LI || !LI->mayHaveSideEffects();12793    };12794    if (!A.checkForAllUses(HasNoEffectLoads, *this, getAssociatedValue()))12795      return indicatePessimisticFixpoint();12796 12797    if (const auto *AMemoryBehavior = A.getOrCreateAAFor<AAMemoryBehavior>(12798            getIRPosition(), this, DepClassTy::REQUIRED)) {12799      // For non-instructions, try to use AAMemoryBehavior to infer the readonly12800      // attribute12801      if (!AMemoryBehavior->isAssumedReadOnly())12802        return indicatePessimisticFixpoint();12803 12804      if (AMemoryBehavior->isKnownReadOnly()) {12805        addKnownBits(IS_NOEFFECT);12806        return ChangeStatus::UNCHANGED;12807      }12808 12809      return ChangeStatus::UNCHANGED;12810    }12811 12812    if (const Argument *Arg = getAssociatedArgument()) {12813      if (Arg->onlyReadsMemory()) {12814        addKnownBits(IS_NOEFFECT);12815        return ChangeStatus::UNCHANGED;12816      }12817 12818      // Readonly information is not provided, and cannot be inferred from12819      // AAMemoryBehavior.12820      return indicatePessimisticFixpoint();12821    }12822 12823    return ChangeStatus::UNCHANGED;12824  }12825 12826  ChangeStatus updateLocalInvariance(Attributor &A) {12827    if (isKnown(IS_LOCALLY_INVARIANT) || !isAssumed(IS_LOCALLY_INVARIANT))12828      return ChangeStatus::UNCHANGED;12829 12830    // try to infer invariance from underlying objects12831    const auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(12832        getIRPosition(), this, DepClassTy::REQUIRED);12833    if (!AUO)12834      return ChangeStatus::UNCHANGED;12835 12836    bool UsedAssumedInformation = false;12837    const auto IsLocallyInvariantLoadIfPointer = [&](const Value &V) {12838      if (!V.getType()->isPointerTy())12839        return true;12840      const auto *IsInvariantLoadPointer =12841          A.getOrCreateAAFor<AAInvariantLoadPointer>(IRPosition::value(V), this,12842                                                     DepClassTy::REQUIRED);12843      // Conservatively fail if invariance cannot be inferred.12844      if (!IsInvariantLoadPointer)12845        return false;12846 12847      if (IsInvariantLoadPointer->isKnownLocallyInvariant())12848        return true;12849      if (!IsInvariantLoadPointer->isAssumedLocallyInvariant())12850        return false;12851 12852      UsedAssumedInformation = true;12853      return true;12854    };12855    if (!AUO->forallUnderlyingObjects(IsLocallyInvariantLoadIfPointer))12856      return indicatePessimisticFixpoint();12857 12858    if (const auto *CB = dyn_cast<CallBase>(&getAnchorValue())) {12859      if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(12860              CB, /*MustPreserveNullness=*/false)) {12861        for (const Value *Arg : CB->args()) {12862          if (!IsLocallyInvariantLoadIfPointer(*Arg))12863            return indicatePessimisticFixpoint();12864        }12865      }12866    }12867 12868    if (!UsedAssumedInformation) {12869      // Pointer is known and not just assumed to be locally invariant.12870      addKnownBits(IS_LOCALLY_INVARIANT);12871      return ChangeStatus::CHANGED;12872    }12873 12874    return ChangeStatus::UNCHANGED;12875  }12876};12877 12878struct AAInvariantLoadPointerFloating final : AAInvariantLoadPointerImpl {12879  AAInvariantLoadPointerFloating(const IRPosition &IRP, Attributor &A)12880      : AAInvariantLoadPointerImpl(IRP, A) {}12881};12882 12883struct AAInvariantLoadPointerReturned final : AAInvariantLoadPointerImpl {12884  AAInvariantLoadPointerReturned(const IRPosition &IRP, Attributor &A)12885      : AAInvariantLoadPointerImpl(IRP, A) {}12886 12887  void initialize(Attributor &) override {12888    removeAssumedBits(IS_LOCALLY_CONSTRAINED);12889  }12890};12891 12892struct AAInvariantLoadPointerCallSiteReturned final12893    : AAInvariantLoadPointerImpl {12894  AAInvariantLoadPointerCallSiteReturned(const IRPosition &IRP, Attributor &A)12895      : AAInvariantLoadPointerImpl(IRP, A) {}12896 12897  void initialize(Attributor &A) override {12898    const Function *F = getAssociatedFunction();12899    assert(F && "no associated function for return from call");12900 12901    if (!F->isDeclaration() && !F->isIntrinsic())12902      return AAInvariantLoadPointerImpl::initialize(A);12903 12904    const auto &CB = cast<CallBase>(getAnchorValue());12905    if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(12906            &CB, /*MustPreserveNullness=*/false))12907      return AAInvariantLoadPointerImpl::initialize(A);12908 12909    if (F->onlyReadsMemory() && F->hasNoSync())12910      return AAInvariantLoadPointerImpl::initialize(A);12911 12912    // At this point, the function is opaque, so we conservatively assume12913    // non-invariance.12914    indicatePessimisticFixpoint();12915  }12916};12917 12918struct AAInvariantLoadPointerArgument final : AAInvariantLoadPointerImpl {12919  AAInvariantLoadPointerArgument(const IRPosition &IRP, Attributor &A)12920      : AAInvariantLoadPointerImpl(IRP, A) {}12921 12922  void initialize(Attributor &) override {12923    const Function *F = getAssociatedFunction();12924    assert(F && "no associated function for argument");12925 12926    if (!isCallableCC(F->getCallingConv())) {12927      addKnownBits(IS_LOCALLY_CONSTRAINED);12928      return;12929    }12930 12931    if (!F->hasLocalLinkage())12932      removeAssumedBits(IS_LOCALLY_CONSTRAINED);12933  }12934};12935 12936struct AAInvariantLoadPointerCallSiteArgument final12937    : AAInvariantLoadPointerImpl {12938  AAInvariantLoadPointerCallSiteArgument(const IRPosition &IRP, Attributor &A)12939      : AAInvariantLoadPointerImpl(IRP, A) {}12940};12941} // namespace12942 12943/// ------------------------ Address Space  ------------------------------------12944namespace {12945 12946template <typename InstType>12947static bool makeChange(Attributor &A, InstType *MemInst, const Use &U,12948                       Value *OriginalValue, PointerType *NewPtrTy,12949                       bool UseOriginalValue) {12950  if (U.getOperandNo() != InstType::getPointerOperandIndex())12951    return false;12952 12953  if (MemInst->isVolatile()) {12954    auto *TTI = A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(12955        *MemInst->getFunction());12956    unsigned NewAS = NewPtrTy->getPointerAddressSpace();12957    if (!TTI || !TTI->hasVolatileVariant(MemInst, NewAS))12958      return false;12959  }12960 12961  if (UseOriginalValue) {12962    A.changeUseAfterManifest(const_cast<Use &>(U), *OriginalValue);12963    return true;12964  }12965 12966  Instruction *CastInst = new AddrSpaceCastInst(OriginalValue, NewPtrTy);12967  CastInst->insertBefore(MemInst->getIterator());12968  A.changeUseAfterManifest(const_cast<Use &>(U), *CastInst);12969  return true;12970}12971 12972struct AAAddressSpaceImpl : public AAAddressSpace {12973  AAAddressSpaceImpl(const IRPosition &IRP, Attributor &A)12974      : AAAddressSpace(IRP, A) {}12975 12976  uint32_t getAddressSpace() const override {12977    assert(isValidState() && "the AA is invalid");12978    return AssumedAddressSpace;12979  }12980 12981  /// See AbstractAttribute::initialize(...).12982  void initialize(Attributor &A) override {12983    assert(getAssociatedType()->isPtrOrPtrVectorTy() &&12984           "Associated value is not a pointer");12985 12986    if (!A.getInfoCache().getFlatAddressSpace().has_value()) {12987      indicatePessimisticFixpoint();12988      return;12989    }12990 12991    unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();12992    unsigned AS = getAssociatedType()->getPointerAddressSpace();12993    if (AS != FlatAS) {12994      [[maybe_unused]] bool R = takeAddressSpace(AS);12995      assert(R && "The take should happen");12996      indicateOptimisticFixpoint();12997    }12998  }12999 13000  ChangeStatus updateImpl(Attributor &A) override {13001    uint32_t OldAddressSpace = AssumedAddressSpace;13002    unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();13003 13004    auto CheckAddressSpace = [&](Value &Obj) {13005      // Ignore undef.13006      if (isa<UndefValue>(&Obj))13007        return true;13008 13009      // If the object already has a non-flat address space, we simply take it.13010      unsigned ObjAS = Obj.getType()->getPointerAddressSpace();13011      if (ObjAS != FlatAS)13012        return takeAddressSpace(ObjAS);13013 13014      // At this point, we know Obj is in the flat address space. For a final13015      // attempt, we want to use getAssumedAddrSpace, but first we must get the13016      // associated function, if possible.13017      Function *F = nullptr;13018      if (auto *Arg = dyn_cast<Argument>(&Obj))13019        F = Arg->getParent();13020      else if (auto *I = dyn_cast<Instruction>(&Obj))13021        F = I->getFunction();13022 13023      // Use getAssumedAddrSpace if the associated function exists.13024      if (F) {13025        auto *TTI =13026            A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(*F);13027        unsigned AssumedAS = TTI->getAssumedAddrSpace(&Obj);13028        if (AssumedAS != ~0U)13029          return takeAddressSpace(AssumedAS);13030      }13031 13032      // Now we can't do anything else but to take the flat AS.13033      return takeAddressSpace(FlatAS);13034    };13035 13036    auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(getIRPosition(), this,13037                                                        DepClassTy::REQUIRED);13038    if (!AUO->forallUnderlyingObjects(CheckAddressSpace))13039      return indicatePessimisticFixpoint();13040 13041    return OldAddressSpace == AssumedAddressSpace ? ChangeStatus::UNCHANGED13042                                                  : ChangeStatus::CHANGED;13043  }13044 13045  /// See AbstractAttribute::manifest(...).13046  ChangeStatus manifest(Attributor &A) override {13047    unsigned NewAS = getAddressSpace();13048 13049    if (NewAS == InvalidAddressSpace ||13050        NewAS == getAssociatedType()->getPointerAddressSpace())13051      return ChangeStatus::UNCHANGED;13052 13053    unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();13054 13055    Value *AssociatedValue = &getAssociatedValue();13056    Value *OriginalValue = peelAddrspacecast(AssociatedValue, FlatAS);13057 13058    PointerType *NewPtrTy =13059        PointerType::get(getAssociatedType()->getContext(), NewAS);13060    bool UseOriginalValue =13061        OriginalValue->getType()->getPointerAddressSpace() == NewAS;13062 13063    bool Changed = false;13064 13065    auto Pred = [&](const Use &U, bool &) {13066      if (U.get() != AssociatedValue)13067        return true;13068      auto *Inst = dyn_cast<Instruction>(U.getUser());13069      if (!Inst)13070        return true;13071      // This is a WA to make sure we only change uses from the corresponding13072      // CGSCC if the AA is run on CGSCC instead of the entire module.13073      if (!A.isRunOn(Inst->getFunction()))13074        return true;13075      if (auto *LI = dyn_cast<LoadInst>(Inst)) {13076        Changed |=13077            makeChange(A, LI, U, OriginalValue, NewPtrTy, UseOriginalValue);13078      } else if (auto *SI = dyn_cast<StoreInst>(Inst)) {13079        Changed |=13080            makeChange(A, SI, U, OriginalValue, NewPtrTy, UseOriginalValue);13081      } else if (auto *RMW = dyn_cast<AtomicRMWInst>(Inst)) {13082        Changed |=13083            makeChange(A, RMW, U, OriginalValue, NewPtrTy, UseOriginalValue);13084      } else if (auto *CmpX = dyn_cast<AtomicCmpXchgInst>(Inst)) {13085        Changed |=13086            makeChange(A, CmpX, U, OriginalValue, NewPtrTy, UseOriginalValue);13087      }13088      return true;13089    };13090 13091    // It doesn't matter if we can't check all uses as we can simply13092    // conservatively ignore those that can not be visited.13093    (void)A.checkForAllUses(Pred, *this, getAssociatedValue(),13094                            /* CheckBBLivenessOnly */ true);13095 13096    return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;13097  }13098 13099  /// See AbstractAttribute::getAsStr().13100  const std::string getAsStr(Attributor *A) const override {13101    if (!isValidState())13102      return "addrspace(<invalid>)";13103    return "addrspace(" +13104           (AssumedAddressSpace == InvalidAddressSpace13105                ? "none"13106                : std::to_string(AssumedAddressSpace)) +13107           ")";13108  }13109 13110private:13111  uint32_t AssumedAddressSpace = InvalidAddressSpace;13112 13113  bool takeAddressSpace(uint32_t AS) {13114    if (AssumedAddressSpace == InvalidAddressSpace) {13115      AssumedAddressSpace = AS;13116      return true;13117    }13118    return AssumedAddressSpace == AS;13119  }13120 13121  static Value *peelAddrspacecast(Value *V, unsigned FlatAS) {13122    if (auto *I = dyn_cast<AddrSpaceCastInst>(V)) {13123      assert(I->getSrcAddressSpace() != FlatAS &&13124             "there should not be flat AS -> non-flat AS");13125      return I->getPointerOperand();13126    }13127    if (auto *C = dyn_cast<ConstantExpr>(V))13128      if (C->getOpcode() == Instruction::AddrSpaceCast) {13129        assert(C->getOperand(0)->getType()->getPointerAddressSpace() !=13130                   FlatAS &&13131               "there should not be flat AS -> non-flat AS X");13132        return C->getOperand(0);13133      }13134    return V;13135  }13136};13137 13138struct AAAddressSpaceFloating final : AAAddressSpaceImpl {13139  AAAddressSpaceFloating(const IRPosition &IRP, Attributor &A)13140      : AAAddressSpaceImpl(IRP, A) {}13141 13142  void trackStatistics() const override {13143    STATS_DECLTRACK_FLOATING_ATTR(addrspace);13144  }13145};13146 13147struct AAAddressSpaceReturned final : AAAddressSpaceImpl {13148  AAAddressSpaceReturned(const IRPosition &IRP, Attributor &A)13149      : AAAddressSpaceImpl(IRP, A) {}13150 13151  /// See AbstractAttribute::initialize(...).13152  void initialize(Attributor &A) override {13153    // TODO: we don't rewrite function argument for now because it will need to13154    // rewrite the function signature and all call sites.13155    (void)indicatePessimisticFixpoint();13156  }13157 13158  void trackStatistics() const override {13159    STATS_DECLTRACK_FNRET_ATTR(addrspace);13160  }13161};13162 13163struct AAAddressSpaceCallSiteReturned final : AAAddressSpaceImpl {13164  AAAddressSpaceCallSiteReturned(const IRPosition &IRP, Attributor &A)13165      : AAAddressSpaceImpl(IRP, A) {}13166 13167  void trackStatistics() const override {13168    STATS_DECLTRACK_CSRET_ATTR(addrspace);13169  }13170};13171 13172struct AAAddressSpaceArgument final : AAAddressSpaceImpl {13173  AAAddressSpaceArgument(const IRPosition &IRP, Attributor &A)13174      : AAAddressSpaceImpl(IRP, A) {}13175 13176  void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(addrspace); }13177};13178 13179struct AAAddressSpaceCallSiteArgument final : AAAddressSpaceImpl {13180  AAAddressSpaceCallSiteArgument(const IRPosition &IRP, Attributor &A)13181      : AAAddressSpaceImpl(IRP, A) {}13182 13183  /// See AbstractAttribute::initialize(...).13184  void initialize(Attributor &A) override {13185    // TODO: we don't rewrite call site argument for now because it will need to13186    // rewrite the function signature of the callee.13187    (void)indicatePessimisticFixpoint();13188  }13189 13190  void trackStatistics() const override {13191    STATS_DECLTRACK_CSARG_ATTR(addrspace);13192  }13193};13194} // namespace13195 13196/// ------------------------ No Alias Address Space  ---------------------------13197// This attribute assumes flat address space can alias all other address space13198 13199// TODO: this is similar to AAAddressSpace, most of the code should be merged.13200// But merging it created failing cased on gateway test that cannot be13201// reproduced locally. So should open a seperated PR to hande the merge of13202// AANoAliasAddrSpace and AAAddressSpace attribute13203 13204namespace {13205struct AANoAliasAddrSpaceImpl : public AANoAliasAddrSpace {13206  AANoAliasAddrSpaceImpl(const IRPosition &IRP, Attributor &A)13207      : AANoAliasAddrSpace(IRP, A) {}13208 13209  void initialize(Attributor &A) override {13210    assert(getAssociatedType()->isPtrOrPtrVectorTy() &&13211           "Associated value is not a pointer");13212 13213    resetASRanges(A);13214 13215    std::optional<unsigned> FlatAS = A.getInfoCache().getFlatAddressSpace();13216    if (!FlatAS.has_value()) {13217      indicatePessimisticFixpoint();13218      return;13219    }13220 13221    removeAS(*FlatAS);13222 13223    unsigned AS = getAssociatedType()->getPointerAddressSpace();13224    if (AS != *FlatAS) {13225      removeAS(AS);13226      indicateOptimisticFixpoint();13227    }13228  }13229 13230  ChangeStatus updateImpl(Attributor &A) override {13231    unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();13232    uint32_t OldAssumed = getAssumed();13233 13234    auto CheckAddressSpace = [&](Value &Obj) {13235      if (isa<PoisonValue>(&Obj))13236        return true;13237 13238      unsigned AS = Obj.getType()->getPointerAddressSpace();13239      if (AS == FlatAS)13240        return false;13241 13242      removeAS(Obj.getType()->getPointerAddressSpace());13243      return true;13244    };13245 13246    const AAUnderlyingObjects *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(13247        getIRPosition(), this, DepClassTy::REQUIRED);13248    if (!AUO->forallUnderlyingObjects(CheckAddressSpace))13249      return indicatePessimisticFixpoint();13250 13251    return OldAssumed == getAssumed() ? ChangeStatus::UNCHANGED13252                                      : ChangeStatus::CHANGED;13253  }13254 13255  /// See AbstractAttribute::manifest(...).13256  ChangeStatus manifest(Attributor &A) override {13257    unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();13258 13259    unsigned AS = getAssociatedType()->getPointerAddressSpace();13260    if (AS != FlatAS || Map.empty())13261      return ChangeStatus::UNCHANGED;13262 13263    LLVMContext &Ctx = getAssociatedValue().getContext();13264    MDNode *NoAliasASNode = nullptr;13265    MDBuilder MDB(Ctx);13266    // Has to use iterator to get the range info.13267    for (RangeMap::const_iterator I = Map.begin(); I != Map.end(); I++) {13268      if (!I.value())13269        continue;13270      unsigned Upper = I.stop();13271      unsigned Lower = I.start();13272      if (!NoAliasASNode) {13273        NoAliasASNode = MDB.createRange(APInt(32, Lower), APInt(32, Upper + 1));13274        continue;13275      }13276      MDNode *ASRange = MDB.createRange(APInt(32, Lower), APInt(32, Upper + 1));13277      NoAliasASNode = MDNode::getMostGenericRange(NoAliasASNode, ASRange);13278    }13279 13280    Value *AssociatedValue = &getAssociatedValue();13281    bool Changed = false;13282 13283    auto AddNoAliasAttr = [&](const Use &U, bool &) {13284      if (U.get() != AssociatedValue)13285        return true;13286      Instruction *Inst = dyn_cast<Instruction>(U.getUser());13287      if (!Inst || Inst->hasMetadata(LLVMContext::MD_noalias_addrspace))13288        return true;13289      if (!isa<LoadInst>(Inst) && !isa<StoreInst>(Inst) &&13290          !isa<AtomicCmpXchgInst>(Inst) && !isa<AtomicRMWInst>(Inst))13291        return true;13292      if (!A.isRunOn(Inst->getFunction()))13293        return true;13294      Inst->setMetadata(LLVMContext::MD_noalias_addrspace, NoAliasASNode);13295      Changed = true;13296      return true;13297    };13298    (void)A.checkForAllUses(AddNoAliasAttr, *this, *AssociatedValue,13299                            /*CheckBBLivenessOnly=*/true);13300    return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;13301  }13302 13303  /// See AbstractAttribute::getAsStr().13304  const std::string getAsStr(Attributor *A) const override {13305    if (!isValidState())13306      return "<invalid>";13307    std::string Str;13308    raw_string_ostream OS(Str);13309    OS << "CanNotBeAddrSpace(";13310    for (RangeMap::const_iterator I = Map.begin(); I != Map.end(); I++) {13311      unsigned Upper = I.stop();13312      unsigned Lower = I.start();13313      OS << ' ' << '[' << Upper << ',' << Lower + 1 << ')';13314    }13315    OS << " )";13316    return OS.str();13317  }13318 13319private:13320  void removeAS(unsigned AS) {13321    RangeMap::iterator I = Map.find(AS);13322 13323    if (I != Map.end()) {13324      unsigned Upper = I.stop();13325      unsigned Lower = I.start();13326      I.erase();13327      if (Upper == Lower)13328        return;13329      if (AS != ~((unsigned)0) && AS + 1 <= Upper)13330        Map.insert(AS + 1, Upper, /*what ever this variable name is=*/true);13331      if (AS != 0 && Lower <= AS - 1)13332        Map.insert(Lower, AS - 1, true);13333    }13334  }13335 13336  void resetASRanges(Attributor &A) {13337    Map.clear();13338    Map.insert(0, A.getInfoCache().getMaxAddrSpace(), true);13339  }13340};13341 13342struct AANoAliasAddrSpaceFloating final : AANoAliasAddrSpaceImpl {13343  AANoAliasAddrSpaceFloating(const IRPosition &IRP, Attributor &A)13344      : AANoAliasAddrSpaceImpl(IRP, A) {}13345 13346  void trackStatistics() const override {13347    STATS_DECLTRACK_FLOATING_ATTR(noaliasaddrspace);13348  }13349};13350 13351struct AANoAliasAddrSpaceReturned final : AANoAliasAddrSpaceImpl {13352  AANoAliasAddrSpaceReturned(const IRPosition &IRP, Attributor &A)13353      : AANoAliasAddrSpaceImpl(IRP, A) {}13354 13355  void trackStatistics() const override {13356    STATS_DECLTRACK_FNRET_ATTR(noaliasaddrspace);13357  }13358};13359 13360struct AANoAliasAddrSpaceCallSiteReturned final : AANoAliasAddrSpaceImpl {13361  AANoAliasAddrSpaceCallSiteReturned(const IRPosition &IRP, Attributor &A)13362      : AANoAliasAddrSpaceImpl(IRP, A) {}13363 13364  void trackStatistics() const override {13365    STATS_DECLTRACK_CSRET_ATTR(noaliasaddrspace);13366  }13367};13368 13369struct AANoAliasAddrSpaceArgument final : AANoAliasAddrSpaceImpl {13370  AANoAliasAddrSpaceArgument(const IRPosition &IRP, Attributor &A)13371      : AANoAliasAddrSpaceImpl(IRP, A) {}13372 13373  void trackStatistics() const override {13374    STATS_DECLTRACK_ARG_ATTR(noaliasaddrspace);13375  }13376};13377 13378struct AANoAliasAddrSpaceCallSiteArgument final : AANoAliasAddrSpaceImpl {13379  AANoAliasAddrSpaceCallSiteArgument(const IRPosition &IRP, Attributor &A)13380      : AANoAliasAddrSpaceImpl(IRP, A) {}13381 13382  void trackStatistics() const override {13383    STATS_DECLTRACK_CSARG_ATTR(noaliasaddrspace);13384  }13385};13386} // namespace13387/// ----------- Allocation Info ----------13388namespace {13389struct AAAllocationInfoImpl : public AAAllocationInfo {13390  AAAllocationInfoImpl(const IRPosition &IRP, Attributor &A)13391      : AAAllocationInfo(IRP, A) {}13392 13393  std::optional<TypeSize> getAllocatedSize() const override {13394    assert(isValidState() && "the AA is invalid");13395    return AssumedAllocatedSize;13396  }13397 13398  std::optional<TypeSize> findInitialAllocationSize(Instruction *I,13399                                                    const DataLayout &DL) {13400 13401    // TODO: implement case for malloc like instructions13402    switch (I->getOpcode()) {13403    case Instruction::Alloca: {13404      AllocaInst *AI = cast<AllocaInst>(I);13405      return AI->getAllocationSize(DL);13406    }13407    default:13408      return std::nullopt;13409    }13410  }13411 13412  ChangeStatus updateImpl(Attributor &A) override {13413 13414    const IRPosition &IRP = getIRPosition();13415    Instruction *I = IRP.getCtxI();13416 13417    // TODO: update check for malloc like calls13418    if (!isa<AllocaInst>(I))13419      return indicatePessimisticFixpoint();13420 13421    bool IsKnownNoCapture;13422    if (!AA::hasAssumedIRAttr<Attribute::Captures>(13423            A, this, IRP, DepClassTy::OPTIONAL, IsKnownNoCapture))13424      return indicatePessimisticFixpoint();13425 13426    const AAPointerInfo *PI =13427        A.getOrCreateAAFor<AAPointerInfo>(IRP, *this, DepClassTy::REQUIRED);13428 13429    if (!PI)13430      return indicatePessimisticFixpoint();13431 13432    if (!PI->getState().isValidState() || PI->reachesReturn())13433      return indicatePessimisticFixpoint();13434 13435    const DataLayout &DL = A.getDataLayout();13436    const auto AllocationSize = findInitialAllocationSize(I, DL);13437 13438    // If allocation size is nullopt, we give up.13439    if (!AllocationSize)13440      return indicatePessimisticFixpoint();13441 13442    // For zero sized allocations, we give up.13443    // Since we can't reduce further13444    if (*AllocationSize == 0)13445      return indicatePessimisticFixpoint();13446 13447    int64_t BinSize = PI->numOffsetBins();13448 13449    // TODO: implement for multiple bins13450    if (BinSize > 1)13451      return indicatePessimisticFixpoint();13452 13453    if (BinSize == 0) {13454      auto NewAllocationSize = std::make_optional<TypeSize>(0, false);13455      if (!changeAllocationSize(NewAllocationSize))13456        return ChangeStatus::UNCHANGED;13457      return ChangeStatus::CHANGED;13458    }13459 13460    // TODO: refactor this to be part of multiple bin case13461    const auto &It = PI->begin();13462 13463    // TODO: handle if Offset is not zero13464    if (It->first.Offset != 0)13465      return indicatePessimisticFixpoint();13466 13467    uint64_t SizeOfBin = It->first.Offset + It->first.Size;13468 13469    if (SizeOfBin >= *AllocationSize)13470      return indicatePessimisticFixpoint();13471 13472    auto NewAllocationSize = std::make_optional<TypeSize>(SizeOfBin * 8, false);13473 13474    if (!changeAllocationSize(NewAllocationSize))13475      return ChangeStatus::UNCHANGED;13476 13477    return ChangeStatus::CHANGED;13478  }13479 13480  /// See AbstractAttribute::manifest(...).13481  ChangeStatus manifest(Attributor &A) override {13482 13483    assert(isValidState() &&13484           "Manifest should only be called if the state is valid.");13485 13486    Instruction *I = getIRPosition().getCtxI();13487 13488    auto FixedAllocatedSizeInBits = getAllocatedSize()->getFixedValue();13489 13490    unsigned long NumBytesToAllocate = (FixedAllocatedSizeInBits + 7) / 8;13491 13492    switch (I->getOpcode()) {13493    // TODO: add case for malloc like calls13494    case Instruction::Alloca: {13495 13496      AllocaInst *AI = cast<AllocaInst>(I);13497 13498      Type *CharType = Type::getInt8Ty(I->getContext());13499 13500      auto *NumBytesToValue =13501          ConstantInt::get(I->getContext(), APInt(32, NumBytesToAllocate));13502 13503      BasicBlock::iterator insertPt = AI->getIterator();13504      insertPt = std::next(insertPt);13505      AllocaInst *NewAllocaInst =13506          new AllocaInst(CharType, AI->getAddressSpace(), NumBytesToValue,13507                         AI->getAlign(), AI->getName(), insertPt);13508 13509      if (A.changeAfterManifest(IRPosition::inst(*AI), *NewAllocaInst))13510        return ChangeStatus::CHANGED;13511 13512      break;13513    }13514    default:13515      break;13516    }13517 13518    return ChangeStatus::UNCHANGED;13519  }13520 13521  /// See AbstractAttribute::getAsStr().13522  const std::string getAsStr(Attributor *A) const override {13523    if (!isValidState())13524      return "allocationinfo(<invalid>)";13525    return "allocationinfo(" +13526           (AssumedAllocatedSize == HasNoAllocationSize13527                ? "none"13528                : std::to_string(AssumedAllocatedSize->getFixedValue())) +13529           ")";13530  }13531 13532private:13533  std::optional<TypeSize> AssumedAllocatedSize = HasNoAllocationSize;13534 13535  // Maintain the computed allocation size of the object.13536  // Returns (bool) weather the size of the allocation was modified or not.13537  bool changeAllocationSize(std::optional<TypeSize> Size) {13538    if (AssumedAllocatedSize == HasNoAllocationSize ||13539        AssumedAllocatedSize != Size) {13540      AssumedAllocatedSize = Size;13541      return true;13542    }13543    return false;13544  }13545};13546 13547struct AAAllocationInfoFloating : AAAllocationInfoImpl {13548  AAAllocationInfoFloating(const IRPosition &IRP, Attributor &A)13549      : AAAllocationInfoImpl(IRP, A) {}13550 13551  void trackStatistics() const override {13552    STATS_DECLTRACK_FLOATING_ATTR(allocationinfo);13553  }13554};13555 13556struct AAAllocationInfoReturned : AAAllocationInfoImpl {13557  AAAllocationInfoReturned(const IRPosition &IRP, Attributor &A)13558      : AAAllocationInfoImpl(IRP, A) {}13559 13560  /// See AbstractAttribute::initialize(...).13561  void initialize(Attributor &A) override {13562    // TODO: we don't rewrite function argument for now because it will need to13563    // rewrite the function signature and all call sites13564    (void)indicatePessimisticFixpoint();13565  }13566 13567  void trackStatistics() const override {13568    STATS_DECLTRACK_FNRET_ATTR(allocationinfo);13569  }13570};13571 13572struct AAAllocationInfoCallSiteReturned : AAAllocationInfoImpl {13573  AAAllocationInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)13574      : AAAllocationInfoImpl(IRP, A) {}13575 13576  void trackStatistics() const override {13577    STATS_DECLTRACK_CSRET_ATTR(allocationinfo);13578  }13579};13580 13581struct AAAllocationInfoArgument : AAAllocationInfoImpl {13582  AAAllocationInfoArgument(const IRPosition &IRP, Attributor &A)13583      : AAAllocationInfoImpl(IRP, A) {}13584 13585  void trackStatistics() const override {13586    STATS_DECLTRACK_ARG_ATTR(allocationinfo);13587  }13588};13589 13590struct AAAllocationInfoCallSiteArgument : AAAllocationInfoImpl {13591  AAAllocationInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)13592      : AAAllocationInfoImpl(IRP, A) {}13593 13594  /// See AbstractAttribute::initialize(...).13595  void initialize(Attributor &A) override {13596 13597    (void)indicatePessimisticFixpoint();13598  }13599 13600  void trackStatistics() const override {13601    STATS_DECLTRACK_CSARG_ATTR(allocationinfo);13602  }13603};13604} // namespace13605 13606const char AANoUnwind::ID = 0;13607const char AANoSync::ID = 0;13608const char AANoFree::ID = 0;13609const char AANonNull::ID = 0;13610const char AAMustProgress::ID = 0;13611const char AANoRecurse::ID = 0;13612const char AANonConvergent::ID = 0;13613const char AAWillReturn::ID = 0;13614const char AAUndefinedBehavior::ID = 0;13615const char AANoAlias::ID = 0;13616const char AAIntraFnReachability::ID = 0;13617const char AANoReturn::ID = 0;13618const char AAIsDead::ID = 0;13619const char AADereferenceable::ID = 0;13620const char AAAlign::ID = 0;13621const char AAInstanceInfo::ID = 0;13622const char AANoCapture::ID = 0;13623const char AAValueSimplify::ID = 0;13624const char AAHeapToStack::ID = 0;13625const char AAPrivatizablePtr::ID = 0;13626const char AAMemoryBehavior::ID = 0;13627const char AAMemoryLocation::ID = 0;13628const char AAValueConstantRange::ID = 0;13629const char AAPotentialConstantValues::ID = 0;13630const char AAPotentialValues::ID = 0;13631const char AANoUndef::ID = 0;13632const char AANoFPClass::ID = 0;13633const char AACallEdges::ID = 0;13634const char AAInterFnReachability::ID = 0;13635const char AAPointerInfo::ID = 0;13636const char AAAssumptionInfo::ID = 0;13637const char AAUnderlyingObjects::ID = 0;13638const char AAInvariantLoadPointer::ID = 0;13639const char AAAddressSpace::ID = 0;13640const char AANoAliasAddrSpace::ID = 0;13641const char AAAllocationInfo::ID = 0;13642const char AAIndirectCallInfo::ID = 0;13643const char AAGlobalValueInfo::ID = 0;13644const char AADenormalFPMath::ID = 0;13645 13646// Macro magic to create the static generator function for attributes that13647// follow the naming scheme.13648 13649#define SWITCH_PK_INV(CLASS, PK, POS_NAME)                                     \13650  case IRPosition::PK:                                                         \13651    llvm_unreachable("Cannot create " #CLASS " for a " POS_NAME " position!");13652 13653#define SWITCH_PK_CREATE(CLASS, IRP, PK, SUFFIX)                               \13654  case IRPosition::PK:                                                         \13655    AA = new (A.Allocator) CLASS##SUFFIX(IRP, A);                              \13656    ++NumAAs;                                                                  \13657    break;13658 13659#define CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)                 \13660  CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) {      \13661    CLASS *AA = nullptr;                                                       \13662    switch (IRP.getPositionKind()) {                                           \13663      SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid")                             \13664      SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating")                              \13665      SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument")                           \13666      SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned")                           \13667      SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned")       \13668      SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument")       \13669      SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function)                     \13670      SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite)                    \13671    }                                                                          \13672    return *AA;                                                                \13673  }13674 13675#define CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)                    \13676  CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) {      \13677    CLASS *AA = nullptr;                                                       \13678    switch (IRP.getPositionKind()) {                                           \13679      SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid")                             \13680      SWITCH_PK_INV(CLASS, IRP_FUNCTION, "function")                           \13681      SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site")                         \13682      SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating)                        \13683      SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument)                     \13684      SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned)                     \13685      SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned)   \13686      SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument)   \13687    }                                                                          \13688    return *AA;                                                                \13689  }13690 13691#define CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(POS, SUFFIX, CLASS)         \13692  CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) {      \13693    CLASS *AA = nullptr;                                                       \13694    switch (IRP.getPositionKind()) {                                           \13695      SWITCH_PK_CREATE(CLASS, IRP, POS, SUFFIX)                                \13696    default:                                                                   \13697      llvm_unreachable("Cannot create " #CLASS " for position otherthan " #POS \13698                       " position!");                                          \13699    }                                                                          \13700    return *AA;                                                                \13701  }13702 13703#define CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)                      \13704  CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) {      \13705    CLASS *AA = nullptr;                                                       \13706    switch (IRP.getPositionKind()) {                                           \13707      SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid")                             \13708      SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function)                     \13709      SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite)                    \13710      SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating)                        \13711      SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument)                     \13712      SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned)                     \13713      SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned)   \13714      SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument)   \13715    }                                                                          \13716    return *AA;                                                                \13717  }13718 13719#define CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)            \13720  CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) {      \13721    CLASS *AA = nullptr;                                                       \13722    switch (IRP.getPositionKind()) {                                           \13723      SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid")                             \13724      SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument")                           \13725      SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating")                              \13726      SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned")                           \13727      SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned")       \13728      SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument")       \13729      SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site")                         \13730      SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function)                     \13731    }                                                                          \13732    return *AA;                                                                \13733  }13734 13735#define CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)                  \13736  CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) {      \13737    CLASS *AA = nullptr;                                                       \13738    switch (IRP.getPositionKind()) {                                           \13739      SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid")                             \13740      SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned")                           \13741      SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function)                     \13742      SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite)                    \13743      SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating)                        \13744      SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument)                     \13745      SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned)   \13746      SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument)   \13747    }                                                                          \13748    return *AA;                                                                \13749  }13750 13751CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUnwind)13752CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoSync)13753CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoRecurse)13754CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAWillReturn)13755CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoReturn)13756CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMemoryLocation)13757CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AACallEdges)13758CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAssumptionInfo)13759CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMustProgress)13760 13761CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANonNull)13762CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoAlias)13763CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPrivatizablePtr)13764CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADereferenceable)13765CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAlign)13766CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInstanceInfo)13767CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoCapture)13768CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueConstantRange)13769CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialConstantValues)13770CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialValues)13771CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUndef)13772CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFPClass)13773CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPointerInfo)13774CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInvariantLoadPointer)13775CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAddressSpace)13776CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoAliasAddrSpace)13777CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAllocationInfo)13778 13779CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueSimplify)13780CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAIsDead)13781CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFree)13782CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAUnderlyingObjects)13783 13784CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(IRP_CALL_SITE, CallSite,13785                                           AAIndirectCallInfo)13786CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(IRP_FLOAT, Floating,13787                                           AAGlobalValueInfo)13788 13789CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAHeapToStack)13790CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAUndefinedBehavior)13791CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANonConvergent)13792CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAIntraFnReachability)13793CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInterFnReachability)13794CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADenormalFPMath)13795 13796CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMemoryBehavior)13797 13798#undef CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION13799#undef CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION13800#undef CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION13801#undef CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION13802#undef CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION13803#undef CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION13804#undef SWITCH_PK_CREATE13805#undef SWITCH_PK_INV13806