brintos / llvm-project-archived public Read only

1//===- CodeGenPrepare.cpp - Prepare a function for code generation --------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This pass munges the code in the input function to better prepare it for10// SelectionDAG-based code generation. This works around limitations in it's11// basic-block-at-a-time approach. It should eventually be removed.12//13//===----------------------------------------------------------------------===//14 15#include "llvm/CodeGen/CodeGenPrepare.h"16#include "llvm/ADT/APInt.h"17#include "llvm/ADT/ArrayRef.h"18#include "llvm/ADT/DenseMap.h"19#include "llvm/ADT/MapVector.h"20#include "llvm/ADT/PointerIntPair.h"21#include "llvm/ADT/STLExtras.h"22#include "llvm/ADT/SmallPtrSet.h"23#include "llvm/ADT/SmallVector.h"24#include "llvm/ADT/Statistic.h"25#include "llvm/Analysis/BlockFrequencyInfo.h"26#include "llvm/Analysis/BranchProbabilityInfo.h"27#include "llvm/Analysis/FloatingPointPredicateUtils.h"28#include "llvm/Analysis/InstructionSimplify.h"29#include "llvm/Analysis/LoopInfo.h"30#include "llvm/Analysis/ProfileSummaryInfo.h"31#include "llvm/Analysis/ScalarEvolutionExpressions.h"32#include "llvm/Analysis/TargetLibraryInfo.h"33#include "llvm/Analysis/TargetTransformInfo.h"34#include "llvm/Analysis/ValueTracking.h"35#include "llvm/Analysis/VectorUtils.h"36#include "llvm/CodeGen/Analysis.h"37#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"38#include "llvm/CodeGen/ISDOpcodes.h"39#include "llvm/CodeGen/SelectionDAGNodes.h"40#include "llvm/CodeGen/TargetLowering.h"41#include "llvm/CodeGen/TargetPassConfig.h"42#include "llvm/CodeGen/TargetSubtargetInfo.h"43#include "llvm/CodeGen/ValueTypes.h"44#include "llvm/CodeGenTypes/MachineValueType.h"45#include "llvm/Config/llvm-config.h"46#include "llvm/IR/Argument.h"47#include "llvm/IR/Attributes.h"48#include "llvm/IR/BasicBlock.h"49#include "llvm/IR/Constant.h"50#include "llvm/IR/Constants.h"51#include "llvm/IR/DataLayout.h"52#include "llvm/IR/DebugInfo.h"53#include "llvm/IR/DerivedTypes.h"54#include "llvm/IR/Dominators.h"55#include "llvm/IR/Function.h"56#include "llvm/IR/GetElementPtrTypeIterator.h"57#include "llvm/IR/GlobalValue.h"58#include "llvm/IR/GlobalVariable.h"59#include "llvm/IR/IRBuilder.h"60#include "llvm/IR/InlineAsm.h"61#include "llvm/IR/InstrTypes.h"62#include "llvm/IR/Instruction.h"63#include "llvm/IR/Instructions.h"64#include "llvm/IR/IntrinsicInst.h"65#include "llvm/IR/Intrinsics.h"66#include "llvm/IR/IntrinsicsAArch64.h"67#include "llvm/IR/LLVMContext.h"68#include "llvm/IR/MDBuilder.h"69#include "llvm/IR/Module.h"70#include "llvm/IR/Operator.h"71#include "llvm/IR/PatternMatch.h"72#include "llvm/IR/ProfDataUtils.h"73#include "llvm/IR/Statepoint.h"74#include "llvm/IR/Type.h"75#include "llvm/IR/Use.h"76#include "llvm/IR/User.h"77#include "llvm/IR/Value.h"78#include "llvm/IR/ValueHandle.h"79#include "llvm/IR/ValueMap.h"80#include "llvm/InitializePasses.h"81#include "llvm/Pass.h"82#include "llvm/Support/BlockFrequency.h"83#include "llvm/Support/BranchProbability.h"84#include "llvm/Support/Casting.h"85#include "llvm/Support/CommandLine.h"86#include "llvm/Support/Compiler.h"87#include "llvm/Support/Debug.h"88#include "llvm/Support/ErrorHandling.h"89#include "llvm/Support/raw_ostream.h"90#include "llvm/Target/TargetMachine.h"91#include "llvm/Target/TargetOptions.h"92#include "llvm/Transforms/Utils/BasicBlockUtils.h"93#include "llvm/Transforms/Utils/BypassSlowDivision.h"94#include "llvm/Transforms/Utils/Local.h"95#include "llvm/Transforms/Utils/SimplifyLibCalls.h"96#include "llvm/Transforms/Utils/SizeOpts.h"97#include <algorithm>98#include <cassert>99#include <cstdint>100#include <iterator>101#include <limits>102#include <memory>103#include <optional>104#include <utility>105#include <vector>106 107using namespace llvm;108using namespace llvm::PatternMatch;109 110#define DEBUG_TYPE "codegenprepare"111 112STATISTIC(NumBlocksElim, "Number of blocks eliminated");113STATISTIC(NumPHIsElim, "Number of trivial PHIs eliminated");114STATISTIC(NumGEPsElim, "Number of GEPs converted to casts");115STATISTIC(NumCmpUses, "Number of uses of Cmp expressions replaced with uses of "116                      "sunken Cmps");117STATISTIC(NumCastUses, "Number of uses of Cast expressions replaced with uses "118                       "of sunken Casts");119STATISTIC(NumMemoryInsts, "Number of memory instructions whose address "120                          "computations were sunk");121STATISTIC(NumMemoryInstsPhiCreated,122          "Number of phis created when address "123          "computations were sunk to memory instructions");124STATISTIC(NumMemoryInstsSelectCreated,125          "Number of select created when address "126          "computations were sunk to memory instructions");127STATISTIC(NumExtsMoved, "Number of [s|z]ext instructions combined with loads");128STATISTIC(NumExtUses, "Number of uses of [s|z]ext instructions optimized");129STATISTIC(NumAndsAdded,130          "Number of and mask instructions added to form ext loads");131STATISTIC(NumAndUses, "Number of uses of and mask instructions optimized");132STATISTIC(NumRetsDup, "Number of return instructions duplicated");133STATISTIC(NumDbgValueMoved, "Number of debug value instructions moved");134STATISTIC(NumSelectsExpanded, "Number of selects turned into branches");135STATISTIC(NumStoreExtractExposed, "Number of store(extractelement) exposed");136 137static cl::opt<bool> DisableBranchOpts(138    "disable-cgp-branch-opts", cl::Hidden, cl::init(false),139    cl::desc("Disable branch optimizations in CodeGenPrepare"));140 141static cl::opt<bool>142    DisableGCOpts("disable-cgp-gc-opts", cl::Hidden, cl::init(false),143                  cl::desc("Disable GC optimizations in CodeGenPrepare"));144 145static cl::opt<bool>146    DisableSelectToBranch("disable-cgp-select2branch", cl::Hidden,147                          cl::init(false),148                          cl::desc("Disable select to branch conversion."));149 150static cl::opt<bool>151    AddrSinkUsingGEPs("addr-sink-using-gep", cl::Hidden, cl::init(true),152                      cl::desc("Address sinking in CGP using GEPs."));153 154static cl::opt<bool>155    EnableAndCmpSinking("enable-andcmp-sinking", cl::Hidden, cl::init(true),156                        cl::desc("Enable sinking and/cmp into branches."));157 158static cl::opt<bool> DisableStoreExtract(159    "disable-cgp-store-extract", cl::Hidden, cl::init(false),160    cl::desc("Disable store(extract) optimizations in CodeGenPrepare"));161 162static cl::opt<bool> StressStoreExtract(163    "stress-cgp-store-extract", cl::Hidden, cl::init(false),164    cl::desc("Stress test store(extract) optimizations in CodeGenPrepare"));165 166static cl::opt<bool> DisableExtLdPromotion(167    "disable-cgp-ext-ld-promotion", cl::Hidden, cl::init(false),168    cl::desc("Disable ext(promotable(ld)) -> promoted(ext(ld)) optimization in "169             "CodeGenPrepare"));170 171static cl::opt<bool> StressExtLdPromotion(172    "stress-cgp-ext-ld-promotion", cl::Hidden, cl::init(false),173    cl::desc("Stress test ext(promotable(ld)) -> promoted(ext(ld)) "174             "optimization in CodeGenPrepare"));175 176static cl::opt<bool> DisablePreheaderProtect(177    "disable-preheader-prot", cl::Hidden, cl::init(false),178    cl::desc("Disable protection against removing loop preheaders"));179 180static cl::opt<bool> ProfileGuidedSectionPrefix(181    "profile-guided-section-prefix", cl::Hidden, cl::init(true),182    cl::desc("Use profile info to add section prefix for hot/cold functions"));183 184static cl::opt<bool> ProfileUnknownInSpecialSection(185    "profile-unknown-in-special-section", cl::Hidden,186    cl::desc("In profiling mode like sampleFDO, if a function doesn't have "187             "profile, we cannot tell the function is cold for sure because "188             "it may be a function newly added without ever being sampled. "189             "With the flag enabled, compiler can put such profile unknown "190             "functions into a special section, so runtime system can choose "191             "to handle it in a different way than .text section, to save "192             "RAM for example. "));193 194static cl::opt<bool> BBSectionsGuidedSectionPrefix(195    "bbsections-guided-section-prefix", cl::Hidden, cl::init(true),196    cl::desc("Use the basic-block-sections profile to determine the text "197             "section prefix for hot functions. Functions with "198             "basic-block-sections profile will be placed in `.text.hot` "199             "regardless of their FDO profile info. Other functions won't be "200             "impacted, i.e., their prefixes will be decided by FDO/sampleFDO "201             "profiles."));202 203static cl::opt<uint64_t> FreqRatioToSkipMerge(204    "cgp-freq-ratio-to-skip-merge", cl::Hidden, cl::init(2),205    cl::desc("Skip merging empty blocks if (frequency of empty block) / "206             "(frequency of destination block) is greater than this ratio"));207 208static cl::opt<bool> ForceSplitStore(209    "force-split-store", cl::Hidden, cl::init(false),210    cl::desc("Force store splitting no matter what the target query says."));211 212static cl::opt<bool> EnableTypePromotionMerge(213    "cgp-type-promotion-merge", cl::Hidden,214    cl::desc("Enable merging of redundant sexts when one is dominating"215             " the other."),216    cl::init(true));217 218static cl::opt<bool> DisableComplexAddrModes(219    "disable-complex-addr-modes", cl::Hidden, cl::init(false),220    cl::desc("Disables combining addressing modes with different parts "221             "in optimizeMemoryInst."));222 223static cl::opt<bool>224    AddrSinkNewPhis("addr-sink-new-phis", cl::Hidden, cl::init(false),225                    cl::desc("Allow creation of Phis in Address sinking."));226 227static cl::opt<bool> AddrSinkNewSelects(228    "addr-sink-new-select", cl::Hidden, cl::init(true),229    cl::desc("Allow creation of selects in Address sinking."));230 231static cl::opt<bool> AddrSinkCombineBaseReg(232    "addr-sink-combine-base-reg", cl::Hidden, cl::init(true),233    cl::desc("Allow combining of BaseReg field in Address sinking."));234 235static cl::opt<bool> AddrSinkCombineBaseGV(236    "addr-sink-combine-base-gv", cl::Hidden, cl::init(true),237    cl::desc("Allow combining of BaseGV field in Address sinking."));238 239static cl::opt<bool> AddrSinkCombineBaseOffs(240    "addr-sink-combine-base-offs", cl::Hidden, cl::init(true),241    cl::desc("Allow combining of BaseOffs field in Address sinking."));242 243static cl::opt<bool> AddrSinkCombineScaledReg(244    "addr-sink-combine-scaled-reg", cl::Hidden, cl::init(true),245    cl::desc("Allow combining of ScaledReg field in Address sinking."));246 247static cl::opt<bool>248    EnableGEPOffsetSplit("cgp-split-large-offset-gep", cl::Hidden,249                         cl::init(true),250                         cl::desc("Enable splitting large offset of GEP."));251 252static cl::opt<bool> EnableICMP_EQToICMP_ST(253    "cgp-icmp-eq2icmp-st", cl::Hidden, cl::init(false),254    cl::desc("Enable ICMP_EQ to ICMP_S(L|G)T conversion."));255 256static cl::opt<bool>257    VerifyBFIUpdates("cgp-verify-bfi-updates", cl::Hidden, cl::init(false),258                     cl::desc("Enable BFI update verification for "259                              "CodeGenPrepare."));260 261static cl::opt<bool>262    OptimizePhiTypes("cgp-optimize-phi-types", cl::Hidden, cl::init(true),263                     cl::desc("Enable converting phi types in CodeGenPrepare"));264 265static cl::opt<unsigned>266    HugeFuncThresholdInCGPP("cgpp-huge-func", cl::init(10000), cl::Hidden,267                            cl::desc("Least BB number of huge function."));268 269static cl::opt<unsigned>270    MaxAddressUsersToScan("cgp-max-address-users-to-scan", cl::init(100),271                          cl::Hidden,272                          cl::desc("Max number of address users to look at"));273 274static cl::opt<bool>275    DisableDeletePHIs("disable-cgp-delete-phis", cl::Hidden, cl::init(false),276                      cl::desc("Disable elimination of dead PHI nodes."));277 278namespace {279 280enum ExtType {281  ZeroExtension, // Zero extension has been seen.282  SignExtension, // Sign extension has been seen.283  BothExtension  // This extension type is used if we saw sext after284                 // ZeroExtension had been set, or if we saw zext after285                 // SignExtension had been set. It makes the type286                 // information of a promoted instruction invalid.287};288 289enum ModifyDT {290  NotModifyDT, // Not Modify any DT.291  ModifyBBDT,  // Modify the Basic Block Dominator Tree.292  ModifyInstDT // Modify the Instruction Dominator in a Basic Block,293               // This usually means we move/delete/insert instruction294               // in a Basic Block. So we should re-iterate instructions295               // in such Basic Block.296};297 298using SetOfInstrs = SmallPtrSet<Instruction *, 16>;299using TypeIsSExt = PointerIntPair<Type *, 2, ExtType>;300using InstrToOrigTy = DenseMap<Instruction *, TypeIsSExt>;301using SExts = SmallVector<Instruction *, 16>;302using ValueToSExts = MapVector<Value *, SExts>;303 304class TypePromotionTransaction;305 306class CodeGenPrepare {307  friend class CodeGenPrepareLegacyPass;308  const TargetMachine *TM = nullptr;309  const TargetSubtargetInfo *SubtargetInfo = nullptr;310  const TargetLowering *TLI = nullptr;311  const TargetRegisterInfo *TRI = nullptr;312  const TargetTransformInfo *TTI = nullptr;313  const BasicBlockSectionsProfileReader *BBSectionsProfileReader = nullptr;314  const TargetLibraryInfo *TLInfo = nullptr;315  LoopInfo *LI = nullptr;316  std::unique_ptr<BlockFrequencyInfo> BFI;317  std::unique_ptr<BranchProbabilityInfo> BPI;318  ProfileSummaryInfo *PSI = nullptr;319 320  /// As we scan instructions optimizing them, this is the next instruction321  /// to optimize. Transforms that can invalidate this should update it.322  BasicBlock::iterator CurInstIterator;323 324  /// Keeps track of non-local addresses that have been sunk into a block.325  /// This allows us to avoid inserting duplicate code for blocks with326  /// multiple load/stores of the same address. The usage of WeakTrackingVH327  /// enables SunkAddrs to be treated as a cache whose entries can be328  /// invalidated if a sunken address computation has been erased.329  ValueMap<Value *, WeakTrackingVH> SunkAddrs;330 331  /// Keeps track of all instructions inserted for the current function.332  SetOfInstrs InsertedInsts;333 334  /// Keeps track of the type of the related instruction before their335  /// promotion for the current function.336  InstrToOrigTy PromotedInsts;337 338  /// Keep track of instructions removed during promotion.339  SetOfInstrs RemovedInsts;340 341  /// Keep track of sext chains based on their initial value.342  DenseMap<Value *, Instruction *> SeenChainsForSExt;343 344  /// Keep track of GEPs accessing the same data structures such as structs or345  /// arrays that are candidates to be split later because of their large346  /// size.347  MapVector<AssertingVH<Value>,348            SmallVector<std::pair<AssertingVH<GetElementPtrInst>, int64_t>, 32>>349      LargeOffsetGEPMap;350 351  /// Keep track of new GEP base after splitting the GEPs having large offset.352  SmallSet<AssertingVH<Value>, 2> NewGEPBases;353 354  /// Map serial numbers to Large offset GEPs.355  DenseMap<AssertingVH<GetElementPtrInst>, int> LargeOffsetGEPID;356 357  /// Keep track of SExt promoted.358  ValueToSExts ValToSExtendedUses;359 360  /// True if the function has the OptSize attribute.361  bool OptSize;362 363  /// DataLayout for the Function being processed.364  const DataLayout *DL = nullptr;365 366  /// Building the dominator tree can be expensive, so we only build it367  /// lazily and update it when required.368  std::unique_ptr<DominatorTree> DT;369 370public:371  CodeGenPrepare() = default;372  CodeGenPrepare(const TargetMachine *TM) : TM(TM){};373  /// If encounter huge function, we need to limit the build time.374  bool IsHugeFunc = false;375 376  /// FreshBBs is like worklist, it collected the updated BBs which need377  /// to be optimized again.378  /// Note: Consider building time in this pass, when a BB updated, we need379  /// to insert such BB into FreshBBs for huge function.380  SmallPtrSet<BasicBlock *, 32> FreshBBs;381 382  void releaseMemory() {383    // Clear per function information.384    InsertedInsts.clear();385    PromotedInsts.clear();386    FreshBBs.clear();387    BPI.reset();388    BFI.reset();389  }390 391  bool run(Function &F, FunctionAnalysisManager &AM);392 393private:394  template <typename F>395  void resetIteratorIfInvalidatedWhileCalling(BasicBlock *BB, F f) {396    // Substituting can cause recursive simplifications, which can invalidate397    // our iterator.  Use a WeakTrackingVH to hold onto it in case this398    // happens.399    Value *CurValue = &*CurInstIterator;400    WeakTrackingVH IterHandle(CurValue);401 402    f();403 404    // If the iterator instruction was recursively deleted, start over at the405    // start of the block.406    if (IterHandle != CurValue) {407      CurInstIterator = BB->begin();408      SunkAddrs.clear();409    }410  }411 412  // Get the DominatorTree, building if necessary.413  DominatorTree &getDT(Function &F) {414    if (!DT)415      DT = std::make_unique<DominatorTree>(F);416    return *DT;417  }418 419  void removeAllAssertingVHReferences(Value *V);420  bool eliminateAssumptions(Function &F);421  bool eliminateFallThrough(Function &F, DominatorTree *DT = nullptr);422  bool eliminateMostlyEmptyBlocks(Function &F);423  BasicBlock *findDestBlockOfMergeableEmptyBlock(BasicBlock *BB);424  bool canMergeBlocks(const BasicBlock *BB, const BasicBlock *DestBB) const;425  void eliminateMostlyEmptyBlock(BasicBlock *BB);426  bool isMergingEmptyBlockProfitable(BasicBlock *BB, BasicBlock *DestBB,427                                     bool isPreheader);428  bool makeBitReverse(Instruction &I);429  bool optimizeBlock(BasicBlock &BB, ModifyDT &ModifiedDT);430  bool optimizeInst(Instruction *I, ModifyDT &ModifiedDT);431  bool optimizeMemoryInst(Instruction *MemoryInst, Value *Addr, Type *AccessTy,432                          unsigned AddrSpace);433  bool optimizeGatherScatterInst(Instruction *MemoryInst, Value *Ptr);434  bool optimizeMulWithOverflow(Instruction *I, bool IsSigned,435                               ModifyDT &ModifiedDT);436  bool optimizeInlineAsmInst(CallInst *CS);437  bool optimizeCallInst(CallInst *CI, ModifyDT &ModifiedDT);438  bool optimizeExt(Instruction *&I);439  bool optimizeExtUses(Instruction *I);440  bool optimizeLoadExt(LoadInst *Load);441  bool optimizeShiftInst(BinaryOperator *BO);442  bool optimizeFunnelShift(IntrinsicInst *Fsh);443  bool optimizeSelectInst(SelectInst *SI);444  bool optimizeShuffleVectorInst(ShuffleVectorInst *SVI);445  bool optimizeSwitchType(SwitchInst *SI);446  bool optimizeSwitchPhiConstants(SwitchInst *SI);447  bool optimizeSwitchInst(SwitchInst *SI);448  bool optimizeExtractElementInst(Instruction *Inst);449  bool dupRetToEnableTailCallOpts(BasicBlock *BB, ModifyDT &ModifiedDT);450  bool fixupDbgVariableRecord(DbgVariableRecord &I);451  bool fixupDbgVariableRecordsOnInst(Instruction &I);452  bool placeDbgValues(Function &F);453  bool placePseudoProbes(Function &F);454  bool canFormExtLd(const SmallVectorImpl<Instruction *> &MovedExts,455                    LoadInst *&LI, Instruction *&Inst, bool HasPromoted);456  bool tryToPromoteExts(TypePromotionTransaction &TPT,457                        const SmallVectorImpl<Instruction *> &Exts,458                        SmallVectorImpl<Instruction *> &ProfitablyMovedExts,459                        unsigned CreatedInstsCost = 0);460  bool mergeSExts(Function &F);461  bool splitLargeGEPOffsets();462  bool optimizePhiType(PHINode *Inst, SmallPtrSetImpl<PHINode *> &Visited,463                       SmallPtrSetImpl<Instruction *> &DeletedInstrs);464  bool optimizePhiTypes(Function &F);465  bool performAddressTypePromotion(466      Instruction *&Inst, bool AllowPromotionWithoutCommonHeader,467      bool HasPromoted, TypePromotionTransaction &TPT,468      SmallVectorImpl<Instruction *> &SpeculativelyMovedExts);469  bool splitBranchCondition(Function &F, ModifyDT &ModifiedDT);470  bool simplifyOffsetableRelocate(GCStatepointInst &I);471 472  bool tryToSinkFreeOperands(Instruction *I);473  bool replaceMathCmpWithIntrinsic(BinaryOperator *BO, Value *Arg0, Value *Arg1,474                                   CmpInst *Cmp, Intrinsic::ID IID);475  bool optimizeCmp(CmpInst *Cmp, ModifyDT &ModifiedDT);476  bool optimizeURem(Instruction *Rem);477  bool combineToUSubWithOverflow(CmpInst *Cmp, ModifyDT &ModifiedDT);478  bool combineToUAddWithOverflow(CmpInst *Cmp, ModifyDT &ModifiedDT);479  bool unfoldPowerOf2Test(CmpInst *Cmp);480  void verifyBFIUpdates(Function &F);481  bool _run(Function &F);482};483 484class CodeGenPrepareLegacyPass : public FunctionPass {485public:486  static char ID; // Pass identification, replacement for typeid487 488  CodeGenPrepareLegacyPass() : FunctionPass(ID) {489    initializeCodeGenPrepareLegacyPassPass(*PassRegistry::getPassRegistry());490  }491 492  bool runOnFunction(Function &F) override;493 494  StringRef getPassName() const override { return "CodeGen Prepare"; }495 496  void getAnalysisUsage(AnalysisUsage &AU) const override {497    // FIXME: When we can selectively preserve passes, preserve the domtree.498    AU.addRequired<ProfileSummaryInfoWrapperPass>();499    AU.addRequired<TargetLibraryInfoWrapperPass>();500    AU.addRequired<TargetPassConfig>();501    AU.addRequired<TargetTransformInfoWrapperPass>();502    AU.addRequired<LoopInfoWrapperPass>();503    AU.addUsedIfAvailable<BasicBlockSectionsProfileReaderWrapperPass>();504  }505};506 507} // end anonymous namespace508 509char CodeGenPrepareLegacyPass::ID = 0;510 511bool CodeGenPrepareLegacyPass::runOnFunction(Function &F) {512  if (skipFunction(F))513    return false;514  auto TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();515  CodeGenPrepare CGP(TM);516  CGP.DL = &F.getDataLayout();517  CGP.SubtargetInfo = TM->getSubtargetImpl(F);518  CGP.TLI = CGP.SubtargetInfo->getTargetLowering();519  CGP.TRI = CGP.SubtargetInfo->getRegisterInfo();520  CGP.TLInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);521  CGP.TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);522  CGP.LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();523  CGP.BPI.reset(new BranchProbabilityInfo(F, *CGP.LI));524  CGP.BFI.reset(new BlockFrequencyInfo(F, *CGP.BPI, *CGP.LI));525  CGP.PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();526  auto BBSPRWP =527      getAnalysisIfAvailable<BasicBlockSectionsProfileReaderWrapperPass>();528  CGP.BBSectionsProfileReader = BBSPRWP ? &BBSPRWP->getBBSPR() : nullptr;529 530  return CGP._run(F);531}532 533INITIALIZE_PASS_BEGIN(CodeGenPrepareLegacyPass, DEBUG_TYPE,534                      "Optimize for code generation", false, false)535INITIALIZE_PASS_DEPENDENCY(BasicBlockSectionsProfileReaderWrapperPass)536INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)537INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)538INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)539INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)540INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)541INITIALIZE_PASS_END(CodeGenPrepareLegacyPass, DEBUG_TYPE,542                    "Optimize for code generation", false, false)543 544FunctionPass *llvm::createCodeGenPrepareLegacyPass() {545  return new CodeGenPrepareLegacyPass();546}547 548PreservedAnalyses CodeGenPreparePass::run(Function &F,549                                          FunctionAnalysisManager &AM) {550  CodeGenPrepare CGP(TM);551 552  bool Changed = CGP.run(F, AM);553  if (!Changed)554    return PreservedAnalyses::all();555 556  PreservedAnalyses PA;557  PA.preserve<TargetLibraryAnalysis>();558  PA.preserve<TargetIRAnalysis>();559  PA.preserve<LoopAnalysis>();560  return PA;561}562 563bool CodeGenPrepare::run(Function &F, FunctionAnalysisManager &AM) {564  DL = &F.getDataLayout();565  SubtargetInfo = TM->getSubtargetImpl(F);566  TLI = SubtargetInfo->getTargetLowering();567  TRI = SubtargetInfo->getRegisterInfo();568  TLInfo = &AM.getResult<TargetLibraryAnalysis>(F);569  TTI = &AM.getResult<TargetIRAnalysis>(F);570  LI = &AM.getResult<LoopAnalysis>(F);571  BPI.reset(new BranchProbabilityInfo(F, *LI));572  BFI.reset(new BlockFrequencyInfo(F, *BPI, *LI));573  auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(F);574  PSI = MAMProxy.getCachedResult<ProfileSummaryAnalysis>(*F.getParent());575  BBSectionsProfileReader =576      AM.getCachedResult<BasicBlockSectionsProfileReaderAnalysis>(F);577  return _run(F);578}579 580bool CodeGenPrepare::_run(Function &F) {581  bool EverMadeChange = false;582 583  OptSize = F.hasOptSize();584  // Use the basic-block-sections profile to promote hot functions to .text.hot585  // if requested.586  if (BBSectionsGuidedSectionPrefix && BBSectionsProfileReader &&587      BBSectionsProfileReader->isFunctionHot(F.getName())) {588    (void)F.setSectionPrefix("hot");589  } else if (ProfileGuidedSectionPrefix) {590    // The hot attribute overwrites profile count based hotness while profile591    // counts based hotness overwrite the cold attribute.592    // This is a conservative behabvior.593    if (F.hasFnAttribute(Attribute::Hot) ||594        PSI->isFunctionHotInCallGraph(&F, *BFI))595      (void)F.setSectionPrefix("hot");596    // If PSI shows this function is not hot, we will placed the function597    // into unlikely section if (1) PSI shows this is a cold function, or598    // (2) the function has a attribute of cold.599    else if (PSI->isFunctionColdInCallGraph(&F, *BFI) ||600             F.hasFnAttribute(Attribute::Cold))601      (void)F.setSectionPrefix("unlikely");602    else if (ProfileUnknownInSpecialSection && PSI->hasPartialSampleProfile() &&603             PSI->isFunctionHotnessUnknown(F))604      (void)F.setSectionPrefix("unknown");605  }606 607  /// This optimization identifies DIV instructions that can be608  /// profitably bypassed and carried out with a shorter, faster divide.609  if (!OptSize && !PSI->hasHugeWorkingSetSize() && TLI->isSlowDivBypassed()) {610    const DenseMap<unsigned int, unsigned int> &BypassWidths =611        TLI->getBypassSlowDivWidths();612    BasicBlock *BB = &*F.begin();613    while (BB != nullptr) {614      // bypassSlowDivision may create new BBs, but we don't want to reapply the615      // optimization to those blocks.616      BasicBlock *Next = BB->getNextNode();617      if (!llvm::shouldOptimizeForSize(BB, PSI, BFI.get()))618        EverMadeChange |= bypassSlowDivision(BB, BypassWidths);619      BB = Next;620    }621  }622 623  // Get rid of @llvm.assume builtins before attempting to eliminate empty624  // blocks, since there might be blocks that only contain @llvm.assume calls625  // (plus arguments that we can get rid of).626  EverMadeChange |= eliminateAssumptions(F);627 628  // Eliminate blocks that contain only PHI nodes and an629  // unconditional branch.630  EverMadeChange |= eliminateMostlyEmptyBlocks(F);631 632  ModifyDT ModifiedDT = ModifyDT::NotModifyDT;633  if (!DisableBranchOpts)634    EverMadeChange |= splitBranchCondition(F, ModifiedDT);635 636  // Split some critical edges where one of the sources is an indirect branch,637  // to help generate sane code for PHIs involving such edges.638  EverMadeChange |=639      SplitIndirectBrCriticalEdges(F, /*IgnoreBlocksWithoutPHI=*/true);640 641  // If we are optimzing huge function, we need to consider the build time.642  // Because the basic algorithm's complex is near O(N!).643  IsHugeFunc = F.size() > HugeFuncThresholdInCGPP;644 645  // Transformations above may invalidate dominator tree and/or loop info.646  DT.reset();647  LI->releaseMemory();648  LI->analyze(getDT(F));649 650  bool MadeChange = true;651  bool FuncIterated = false;652  while (MadeChange) {653    MadeChange = false;654 655    for (BasicBlock &BB : llvm::make_early_inc_range(F)) {656      if (FuncIterated && !FreshBBs.contains(&BB))657        continue;658 659      ModifyDT ModifiedDTOnIteration = ModifyDT::NotModifyDT;660      bool Changed = optimizeBlock(BB, ModifiedDTOnIteration);661 662      if (ModifiedDTOnIteration == ModifyDT::ModifyBBDT)663        DT.reset();664 665      MadeChange |= Changed;666      if (IsHugeFunc) {667        // If the BB is updated, it may still has chance to be optimized.668        // This usually happen at sink optimization.669        // For example:670        //671        // bb0：672        // %and = and i32 %a, 4673        // %cmp = icmp eq i32 %and, 0674        //675        // If the %cmp sink to other BB, the %and will has chance to sink.676        if (Changed)677          FreshBBs.insert(&BB);678        else if (FuncIterated)679          FreshBBs.erase(&BB);680      } else {681        // For small/normal functions, we restart BB iteration if the dominator682        // tree of the Function was changed.683        if (ModifiedDTOnIteration != ModifyDT::NotModifyDT)684          break;685      }686    }687    // We have iterated all the BB in the (only work for huge) function.688    FuncIterated = IsHugeFunc;689 690    if (EnableTypePromotionMerge && !ValToSExtendedUses.empty())691      MadeChange |= mergeSExts(F);692    if (!LargeOffsetGEPMap.empty())693      MadeChange |= splitLargeGEPOffsets();694    MadeChange |= optimizePhiTypes(F);695 696    if (MadeChange)697      eliminateFallThrough(F, DT.get());698 699#ifndef NDEBUG700    if (MadeChange && VerifyLoopInfo)701      LI->verify(getDT(F));702#endif703 704    // Really free removed instructions during promotion.705    for (Instruction *I : RemovedInsts)706      I->deleteValue();707 708    EverMadeChange |= MadeChange;709    SeenChainsForSExt.clear();710    ValToSExtendedUses.clear();711    RemovedInsts.clear();712    LargeOffsetGEPMap.clear();713    LargeOffsetGEPID.clear();714  }715 716  NewGEPBases.clear();717  SunkAddrs.clear();718 719  if (!DisableBranchOpts) {720    MadeChange = false;721    // Use a set vector to get deterministic iteration order. The order the722    // blocks are removed may affect whether or not PHI nodes in successors723    // are removed.724    SmallSetVector<BasicBlock *, 8> WorkList;725    for (BasicBlock &BB : F) {726      SmallVector<BasicBlock *, 2> Successors(successors(&BB));727      MadeChange |= ConstantFoldTerminator(&BB, true);728      if (!MadeChange)729        continue;730 731      for (BasicBlock *Succ : Successors)732        if (pred_empty(Succ))733          WorkList.insert(Succ);734    }735 736    // Delete the dead blocks and any of their dead successors.737    MadeChange |= !WorkList.empty();738    while (!WorkList.empty()) {739      BasicBlock *BB = WorkList.pop_back_val();740      SmallVector<BasicBlock *, 2> Successors(successors(BB));741 742      DeleteDeadBlock(BB);743 744      for (BasicBlock *Succ : Successors)745        if (pred_empty(Succ))746          WorkList.insert(Succ);747    }748 749    // Merge pairs of basic blocks with unconditional branches, connected by750    // a single edge.751    if (EverMadeChange || MadeChange)752      MadeChange |= eliminateFallThrough(F);753 754    EverMadeChange |= MadeChange;755  }756 757  if (!DisableGCOpts) {758    SmallVector<GCStatepointInst *, 2> Statepoints;759    for (BasicBlock &BB : F)760      for (Instruction &I : BB)761        if (auto *SP = dyn_cast<GCStatepointInst>(&I))762          Statepoints.push_back(SP);763    for (auto &I : Statepoints)764      EverMadeChange |= simplifyOffsetableRelocate(*I);765  }766 767  // Do this last to clean up use-before-def scenarios introduced by other768  // preparatory transforms.769  EverMadeChange |= placeDbgValues(F);770  EverMadeChange |= placePseudoProbes(F);771 772#ifndef NDEBUG773  if (VerifyBFIUpdates)774    verifyBFIUpdates(F);775#endif776 777  return EverMadeChange;778}779 780bool CodeGenPrepare::eliminateAssumptions(Function &F) {781  bool MadeChange = false;782  for (BasicBlock &BB : F) {783    CurInstIterator = BB.begin();784    while (CurInstIterator != BB.end()) {785      Instruction *I = &*(CurInstIterator++);786      if (auto *Assume = dyn_cast<AssumeInst>(I)) {787        MadeChange = true;788        Value *Operand = Assume->getOperand(0);789        Assume->eraseFromParent();790 791        resetIteratorIfInvalidatedWhileCalling(&BB, [&]() {792          RecursivelyDeleteTriviallyDeadInstructions(Operand, TLInfo, nullptr);793        });794      }795    }796  }797  return MadeChange;798}799 800/// An instruction is about to be deleted, so remove all references to it in our801/// GEP-tracking data strcutures.802void CodeGenPrepare::removeAllAssertingVHReferences(Value *V) {803  LargeOffsetGEPMap.erase(V);804  NewGEPBases.erase(V);805 806  auto GEP = dyn_cast<GetElementPtrInst>(V);807  if (!GEP)808    return;809 810  LargeOffsetGEPID.erase(GEP);811 812  auto VecI = LargeOffsetGEPMap.find(GEP->getPointerOperand());813  if (VecI == LargeOffsetGEPMap.end())814    return;815 816  auto &GEPVector = VecI->second;817  llvm::erase_if(GEPVector, [=](auto &Elt) { return Elt.first == GEP; });818 819  if (GEPVector.empty())820    LargeOffsetGEPMap.erase(VecI);821}822 823// Verify BFI has been updated correctly by recomputing BFI and comparing them.824[[maybe_unused]] void CodeGenPrepare::verifyBFIUpdates(Function &F) {825  DominatorTree NewDT(F);826  LoopInfo NewLI(NewDT);827  BranchProbabilityInfo NewBPI(F, NewLI, TLInfo);828  BlockFrequencyInfo NewBFI(F, NewBPI, NewLI);829  NewBFI.verifyMatch(*BFI);830}831 832/// Merge basic blocks which are connected by a single edge, where one of the833/// basic blocks has a single successor pointing to the other basic block,834/// which has a single predecessor.835bool CodeGenPrepare::eliminateFallThrough(Function &F, DominatorTree *DT) {836  bool Changed = false;837  // Scan all of the blocks in the function, except for the entry block.838  // Use a temporary array to avoid iterator being invalidated when839  // deleting blocks.840  SmallVector<WeakTrackingVH, 16> Blocks(841      llvm::make_pointer_range(llvm::drop_begin(F)));842 843  SmallSet<WeakTrackingVH, 16> Preds;844  for (auto &Block : Blocks) {845    auto *BB = cast_or_null<BasicBlock>(Block);846    if (!BB)847      continue;848    // If the destination block has a single pred, then this is a trivial849    // edge, just collapse it.850    BasicBlock *SinglePred = BB->getSinglePredecessor();851 852    // Don't merge if BB's address is taken.853    if (!SinglePred || SinglePred == BB || BB->hasAddressTaken())854      continue;855 856    // Make an effort to skip unreachable blocks.857    if (DT && !DT->isReachableFromEntry(BB))858      continue;859 860    BranchInst *Term = dyn_cast<BranchInst>(SinglePred->getTerminator());861    if (Term && !Term->isConditional()) {862      Changed = true;863      LLVM_DEBUG(dbgs() << "To merge:\n" << *BB << "\n\n\n");864 865      // Merge BB into SinglePred and delete it.866      MergeBlockIntoPredecessor(BB, /* DTU */ nullptr, LI, /* MSSAU */ nullptr,867                                /* MemDep */ nullptr,868                                /* PredecessorWithTwoSuccessors */ false, DT);869      Preds.insert(SinglePred);870 871      if (IsHugeFunc) {872        // Update FreshBBs to optimize the merged BB.873        FreshBBs.insert(SinglePred);874        FreshBBs.erase(BB);875      }876    }877  }878 879  // (Repeatedly) merging blocks into their predecessors can create redundant880  // debug intrinsics.881  for (const auto &Pred : Preds)882    if (auto *BB = cast_or_null<BasicBlock>(Pred))883      RemoveRedundantDbgInstrs(BB);884 885  return Changed;886}887 888/// Find a destination block from BB if BB is mergeable empty block.889BasicBlock *CodeGenPrepare::findDestBlockOfMergeableEmptyBlock(BasicBlock *BB) {890  // If this block doesn't end with an uncond branch, ignore it.891  BranchInst *BI = dyn_cast<BranchInst>(BB->getTerminator());892  if (!BI || !BI->isUnconditional())893    return nullptr;894 895  // If the instruction before the branch (skipping debug info) isn't a phi896  // node, then other stuff is happening here.897  BasicBlock::iterator BBI = BI->getIterator();898  if (BBI != BB->begin()) {899    --BBI;900    if (!isa<PHINode>(BBI))901      return nullptr;902  }903 904  // Do not break infinite loops.905  BasicBlock *DestBB = BI->getSuccessor(0);906  if (DestBB == BB)907    return nullptr;908 909  if (!canMergeBlocks(BB, DestBB))910    DestBB = nullptr;911 912  return DestBB;913}914 915/// Eliminate blocks that contain only PHI nodes, debug info directives, and an916/// unconditional branch. Passes before isel (e.g. LSR/loopsimplify) often split917/// edges in ways that are non-optimal for isel. Start by eliminating these918/// blocks so we can split them the way we want them.919bool CodeGenPrepare::eliminateMostlyEmptyBlocks(Function &F) {920  SmallPtrSet<BasicBlock *, 16> Preheaders;921  SmallVector<Loop *, 16> LoopList(LI->begin(), LI->end());922  while (!LoopList.empty()) {923    Loop *L = LoopList.pop_back_val();924    llvm::append_range(LoopList, *L);925    if (BasicBlock *Preheader = L->getLoopPreheader())926      Preheaders.insert(Preheader);927  }928 929  bool MadeChange = false;930  // Copy blocks into a temporary array to avoid iterator invalidation issues931  // as we remove them.932  // Note that this intentionally skips the entry block.933  SmallVector<WeakTrackingVH, 16> Blocks;934  for (auto &Block : llvm::drop_begin(F)) {935    // Delete phi nodes that could block deleting other empty blocks.936    if (!DisableDeletePHIs)937      MadeChange |= DeleteDeadPHIs(&Block, TLInfo);938    Blocks.push_back(&Block);939  }940 941  for (auto &Block : Blocks) {942    BasicBlock *BB = cast_or_null<BasicBlock>(Block);943    if (!BB)944      continue;945    BasicBlock *DestBB = findDestBlockOfMergeableEmptyBlock(BB);946    if (!DestBB ||947        !isMergingEmptyBlockProfitable(BB, DestBB, Preheaders.count(BB)))948      continue;949 950    eliminateMostlyEmptyBlock(BB);951    MadeChange = true;952  }953  return MadeChange;954}955 956bool CodeGenPrepare::isMergingEmptyBlockProfitable(BasicBlock *BB,957                                                   BasicBlock *DestBB,958                                                   bool isPreheader) {959  // Do not delete loop preheaders if doing so would create a critical edge.960  // Loop preheaders can be good locations to spill registers. If the961  // preheader is deleted and we create a critical edge, registers may be962  // spilled in the loop body instead.963  if (!DisablePreheaderProtect && isPreheader &&964      !(BB->getSinglePredecessor() &&965        BB->getSinglePredecessor()->getSingleSuccessor()))966    return false;967 968  // Skip merging if the block's successor is also a successor to any callbr969  // that leads to this block.970  // FIXME: Is this really needed? Is this a correctness issue?971  for (BasicBlock *Pred : predecessors(BB)) {972    if (isa<CallBrInst>(Pred->getTerminator()) &&973        llvm::is_contained(successors(Pred), DestBB))974      return false;975  }976 977  // Try to skip merging if the unique predecessor of BB is terminated by a978  // switch or indirect branch instruction, and BB is used as an incoming block979  // of PHIs in DestBB. In such case, merging BB and DestBB would cause ISel to980  // add COPY instructions in the predecessor of BB instead of BB (if it is not981  // merged). Note that the critical edge created by merging such blocks wont be982  // split in MachineSink because the jump table is not analyzable. By keeping983  // such empty block (BB), ISel will place COPY instructions in BB, not in the984  // predecessor of BB.985  BasicBlock *Pred = BB->getUniquePredecessor();986  if (!Pred || !(isa<SwitchInst>(Pred->getTerminator()) ||987                 isa<IndirectBrInst>(Pred->getTerminator())))988    return true;989 990  if (BB->getTerminator() != &*BB->getFirstNonPHIOrDbg())991    return true;992 993  // We use a simple cost heuristic which determine skipping merging is994  // profitable if the cost of skipping merging is less than the cost of995  // merging : Cost(skipping merging) < Cost(merging BB), where the996  // Cost(skipping merging) is Freq(BB) * (Cost(Copy) + Cost(Branch)), and997  // the Cost(merging BB) is Freq(Pred) * Cost(Copy).998  // Assuming Cost(Copy) == Cost(Branch), we could simplify it to :999  //   Freq(Pred) / Freq(BB) > 2.1000  // Note that if there are multiple empty blocks sharing the same incoming1001  // value for the PHIs in the DestBB, we consider them together. In such1002  // case, Cost(merging BB) will be the sum of their frequencies.1003 1004  if (!isa<PHINode>(DestBB->begin()))1005    return true;1006 1007  SmallPtrSet<BasicBlock *, 16> SameIncomingValueBBs;1008 1009  // Find all other incoming blocks from which incoming values of all PHIs in1010  // DestBB are the same as the ones from BB.1011  for (BasicBlock *DestBBPred : predecessors(DestBB)) {1012    if (DestBBPred == BB)1013      continue;1014 1015    if (llvm::all_of(DestBB->phis(), [&](const PHINode &DestPN) {1016          return DestPN.getIncomingValueForBlock(BB) ==1017                 DestPN.getIncomingValueForBlock(DestBBPred);1018        }))1019      SameIncomingValueBBs.insert(DestBBPred);1020  }1021 1022  // See if all BB's incoming values are same as the value from Pred. In this1023  // case, no reason to skip merging because COPYs are expected to be place in1024  // Pred already.1025  if (SameIncomingValueBBs.count(Pred))1026    return true;1027 1028  BlockFrequency PredFreq = BFI->getBlockFreq(Pred);1029  BlockFrequency BBFreq = BFI->getBlockFreq(BB);1030 1031  for (auto *SameValueBB : SameIncomingValueBBs)1032    if (SameValueBB->getUniquePredecessor() == Pred &&1033        DestBB == findDestBlockOfMergeableEmptyBlock(SameValueBB))1034      BBFreq += BFI->getBlockFreq(SameValueBB);1035 1036  std::optional<BlockFrequency> Limit = BBFreq.mul(FreqRatioToSkipMerge);1037  return !Limit || PredFreq <= *Limit;1038}1039 1040/// Return true if we can merge BB into DestBB if there is a single1041/// unconditional branch between them, and BB contains no other non-phi1042/// instructions.1043bool CodeGenPrepare::canMergeBlocks(const BasicBlock *BB,1044                                    const BasicBlock *DestBB) const {1045  // We only want to eliminate blocks whose phi nodes are used by phi nodes in1046  // the successor.  If there are more complex condition (e.g. preheaders),1047  // don't mess around with them.1048  for (const PHINode &PN : BB->phis()) {1049    for (const User *U : PN.users()) {1050      const Instruction *UI = cast<Instruction>(U);1051      if (UI->getParent() != DestBB || !isa<PHINode>(UI))1052        return false;1053      // If User is inside DestBB block and it is a PHINode then check1054      // incoming value. If incoming value is not from BB then this is1055      // a complex condition (e.g. preheaders) we want to avoid here.1056      if (UI->getParent() == DestBB) {1057        if (const PHINode *UPN = dyn_cast<PHINode>(UI))1058          for (unsigned I = 0, E = UPN->getNumIncomingValues(); I != E; ++I) {1059            Instruction *Insn = dyn_cast<Instruction>(UPN->getIncomingValue(I));1060            if (Insn && Insn->getParent() == BB &&1061                Insn->getParent() != UPN->getIncomingBlock(I))1062              return false;1063          }1064      }1065    }1066  }1067 1068  // If BB and DestBB contain any common predecessors, then the phi nodes in BB1069  // and DestBB may have conflicting incoming values for the block.  If so, we1070  // can't merge the block.1071  const PHINode *DestBBPN = dyn_cast<PHINode>(DestBB->begin());1072  if (!DestBBPN)1073    return true; // no conflict.1074 1075  // Collect the preds of BB.1076  SmallPtrSet<const BasicBlock *, 16> BBPreds;1077  if (const PHINode *BBPN = dyn_cast<PHINode>(BB->begin())) {1078    // It is faster to get preds from a PHI than with pred_iterator.1079    for (unsigned i = 0, e = BBPN->getNumIncomingValues(); i != e; ++i)1080      BBPreds.insert(BBPN->getIncomingBlock(i));1081  } else {1082    BBPreds.insert_range(predecessors(BB));1083  }1084 1085  // Walk the preds of DestBB.1086  for (unsigned i = 0, e = DestBBPN->getNumIncomingValues(); i != e; ++i) {1087    BasicBlock *Pred = DestBBPN->getIncomingBlock(i);1088    if (BBPreds.count(Pred)) { // Common predecessor?1089      for (const PHINode &PN : DestBB->phis()) {1090        const Value *V1 = PN.getIncomingValueForBlock(Pred);1091        const Value *V2 = PN.getIncomingValueForBlock(BB);1092 1093        // If V2 is a phi node in BB, look up what the mapped value will be.1094        if (const PHINode *V2PN = dyn_cast<PHINode>(V2))1095          if (V2PN->getParent() == BB)1096            V2 = V2PN->getIncomingValueForBlock(Pred);1097 1098        // If there is a conflict, bail out.1099        if (V1 != V2)1100          return false;1101      }1102    }1103  }1104 1105  return true;1106}1107 1108/// Replace all old uses with new ones, and push the updated BBs into FreshBBs.1109static void replaceAllUsesWith(Value *Old, Value *New,1110                               SmallPtrSet<BasicBlock *, 32> &FreshBBs,1111                               bool IsHuge) {1112  auto *OldI = dyn_cast<Instruction>(Old);1113  if (OldI) {1114    for (Value::user_iterator UI = OldI->user_begin(), E = OldI->user_end();1115         UI != E; ++UI) {1116      Instruction *User = cast<Instruction>(*UI);1117      if (IsHuge)1118        FreshBBs.insert(User->getParent());1119    }1120  }1121  Old->replaceAllUsesWith(New);1122}1123 1124/// Eliminate a basic block that has only phi's and an unconditional branch in1125/// it.1126void CodeGenPrepare::eliminateMostlyEmptyBlock(BasicBlock *BB) {1127  BranchInst *BI = cast<BranchInst>(BB->getTerminator());1128  BasicBlock *DestBB = BI->getSuccessor(0);1129 1130  LLVM_DEBUG(dbgs() << "MERGING MOSTLY EMPTY BLOCKS - BEFORE:\n"1131                    << *BB << *DestBB);1132 1133  // If the destination block has a single pred, then this is a trivial edge,1134  // just collapse it.1135  if (BasicBlock *SinglePred = DestBB->getSinglePredecessor()) {1136    if (SinglePred != DestBB) {1137      assert(SinglePred == BB &&1138             "Single predecessor not the same as predecessor");1139      // Merge DestBB into SinglePred/BB and delete it.1140      MergeBlockIntoPredecessor(DestBB);1141      // Note: BB(=SinglePred) will not be deleted on this path.1142      // DestBB(=its single successor) is the one that was deleted.1143      LLVM_DEBUG(dbgs() << "AFTER:\n" << *SinglePred << "\n\n\n");1144 1145      if (IsHugeFunc) {1146        // Update FreshBBs to optimize the merged BB.1147        FreshBBs.insert(SinglePred);1148        FreshBBs.erase(DestBB);1149      }1150      return;1151    }1152  }1153 1154  // Otherwise, we have multiple predecessors of BB.  Update the PHIs in DestBB1155  // to handle the new incoming edges it is about to have.1156  for (PHINode &PN : DestBB->phis()) {1157    // Remove the incoming value for BB, and remember it.1158    Value *InVal = PN.removeIncomingValue(BB, false);1159 1160    // Two options: either the InVal is a phi node defined in BB or it is some1161    // value that dominates BB.1162    PHINode *InValPhi = dyn_cast<PHINode>(InVal);1163    if (InValPhi && InValPhi->getParent() == BB) {1164      // Add all of the input values of the input PHI as inputs of this phi.1165      for (unsigned i = 0, e = InValPhi->getNumIncomingValues(); i != e; ++i)1166        PN.addIncoming(InValPhi->getIncomingValue(i),1167                       InValPhi->getIncomingBlock(i));1168    } else {1169      // Otherwise, add one instance of the dominating value for each edge that1170      // we will be adding.1171      if (PHINode *BBPN = dyn_cast<PHINode>(BB->begin())) {1172        for (unsigned i = 0, e = BBPN->getNumIncomingValues(); i != e; ++i)1173          PN.addIncoming(InVal, BBPN->getIncomingBlock(i));1174      } else {1175        for (BasicBlock *Pred : predecessors(BB))1176          PN.addIncoming(InVal, Pred);1177      }1178    }1179  }1180 1181  // Preserve loop Metadata.1182  if (BI->hasMetadata(LLVMContext::MD_loop)) {1183    for (auto *Pred : predecessors(BB))1184      Pred->getTerminator()->copyMetadata(*BI, LLVMContext::MD_loop);1185  }1186 1187  // The PHIs are now updated, change everything that refers to BB to use1188  // DestBB and remove BB.1189  BB->replaceAllUsesWith(DestBB);1190  BB->eraseFromParent();1191  ++NumBlocksElim;1192 1193  LLVM_DEBUG(dbgs() << "AFTER:\n" << *DestBB << "\n\n\n");1194}1195 1196// Computes a map of base pointer relocation instructions to corresponding1197// derived pointer relocation instructions given a vector of all relocate calls1198static void computeBaseDerivedRelocateMap(1199    const SmallVectorImpl<GCRelocateInst *> &AllRelocateCalls,1200    MapVector<GCRelocateInst *, SmallVector<GCRelocateInst *, 0>>1201        &RelocateInstMap) {1202  // Collect information in two maps: one primarily for locating the base object1203  // while filling the second map; the second map is the final structure holding1204  // a mapping between Base and corresponding Derived relocate calls1205  MapVector<std::pair<unsigned, unsigned>, GCRelocateInst *> RelocateIdxMap;1206  for (auto *ThisRelocate : AllRelocateCalls) {1207    auto K = std::make_pair(ThisRelocate->getBasePtrIndex(),1208                            ThisRelocate->getDerivedPtrIndex());1209    RelocateIdxMap.insert(std::make_pair(K, ThisRelocate));1210  }1211  for (auto &Item : RelocateIdxMap) {1212    std::pair<unsigned, unsigned> Key = Item.first;1213    if (Key.first == Key.second)1214      // Base relocation: nothing to insert1215      continue;1216 1217    GCRelocateInst *I = Item.second;1218    auto BaseKey = std::make_pair(Key.first, Key.first);1219 1220    // We're iterating over RelocateIdxMap so we cannot modify it.1221    auto MaybeBase = RelocateIdxMap.find(BaseKey);1222    if (MaybeBase == RelocateIdxMap.end())1223      // TODO: We might want to insert a new base object relocate and gep off1224      // that, if there are enough derived object relocates.1225      continue;1226 1227    RelocateInstMap[MaybeBase->second].push_back(I);1228  }1229}1230 1231// Accepts a GEP and extracts the operands into a vector provided they're all1232// small integer constants1233static bool getGEPSmallConstantIntOffsetV(GetElementPtrInst *GEP,1234                                          SmallVectorImpl<Value *> &OffsetV) {1235  for (unsigned i = 1; i < GEP->getNumOperands(); i++) {1236    // Only accept small constant integer operands1237    auto *Op = dyn_cast<ConstantInt>(GEP->getOperand(i));1238    if (!Op || Op->getZExtValue() > 20)1239      return false;1240  }1241 1242  for (unsigned i = 1; i < GEP->getNumOperands(); i++)1243    OffsetV.push_back(GEP->getOperand(i));1244  return true;1245}1246 1247// Takes a RelocatedBase (base pointer relocation instruction) and Targets to1248// replace, computes a replacement, and affects it.1249static bool1250simplifyRelocatesOffABase(GCRelocateInst *RelocatedBase,1251                          const SmallVectorImpl<GCRelocateInst *> &Targets) {1252  bool MadeChange = false;1253  // We must ensure the relocation of derived pointer is defined after1254  // relocation of base pointer. If we find a relocation corresponding to base1255  // defined earlier than relocation of base then we move relocation of base1256  // right before found relocation. We consider only relocation in the same1257  // basic block as relocation of base. Relocations from other basic block will1258  // be skipped by optimization and we do not care about them.1259  for (auto R = RelocatedBase->getParent()->getFirstInsertionPt();1260       &*R != RelocatedBase; ++R)1261    if (auto *RI = dyn_cast<GCRelocateInst>(R))1262      if (RI->getStatepoint() == RelocatedBase->getStatepoint())1263        if (RI->getBasePtrIndex() == RelocatedBase->getBasePtrIndex()) {1264          RelocatedBase->moveBefore(RI->getIterator());1265          MadeChange = true;1266          break;1267        }1268 1269  for (GCRelocateInst *ToReplace : Targets) {1270    assert(ToReplace->getBasePtrIndex() == RelocatedBase->getBasePtrIndex() &&1271           "Not relocating a derived object of the original base object");1272    if (ToReplace->getBasePtrIndex() == ToReplace->getDerivedPtrIndex()) {1273      // A duplicate relocate call. TODO: coalesce duplicates.1274      continue;1275    }1276 1277    if (RelocatedBase->getParent() != ToReplace->getParent()) {1278      // Base and derived relocates are in different basic blocks.1279      // In this case transform is only valid when base dominates derived1280      // relocate. However it would be too expensive to check dominance1281      // for each such relocate, so we skip the whole transformation.1282      continue;1283    }1284 1285    Value *Base = ToReplace->getBasePtr();1286    auto *Derived = dyn_cast<GetElementPtrInst>(ToReplace->getDerivedPtr());1287    if (!Derived || Derived->getPointerOperand() != Base)1288      continue;1289 1290    SmallVector<Value *, 2> OffsetV;1291    if (!getGEPSmallConstantIntOffsetV(Derived, OffsetV))1292      continue;1293 1294    // Create a Builder and replace the target callsite with a gep1295    assert(RelocatedBase->getNextNode() &&1296           "Should always have one since it's not a terminator");1297 1298    // Insert after RelocatedBase1299    IRBuilder<> Builder(RelocatedBase->getNextNode());1300    Builder.SetCurrentDebugLocation(ToReplace->getDebugLoc());1301 1302    // If gc_relocate does not match the actual type, cast it to the right type.1303    // In theory, there must be a bitcast after gc_relocate if the type does not1304    // match, and we should reuse it to get the derived pointer. But it could be1305    // cases like this:1306    // bb1:1307    //  ...1308    //  %g1 = call coldcc i8 addrspace(1)*1309    //  @llvm.experimental.gc.relocate.p1i8(...) br label %merge1310    //1311    // bb2:1312    //  ...1313    //  %g2 = call coldcc i8 addrspace(1)*1314    //  @llvm.experimental.gc.relocate.p1i8(...) br label %merge1315    //1316    // merge:1317    //  %p1 = phi i8 addrspace(1)* [ %g1, %bb1 ], [ %g2, %bb2 ]1318    //  %cast = bitcast i8 addrspace(1)* %p1 in to i32 addrspace(1)*1319    //1320    // In this case, we can not find the bitcast any more. So we insert a new1321    // bitcast no matter there is already one or not. In this way, we can handle1322    // all cases, and the extra bitcast should be optimized away in later1323    // passes.1324    Value *ActualRelocatedBase = RelocatedBase;1325    if (RelocatedBase->getType() != Base->getType()) {1326      ActualRelocatedBase =1327          Builder.CreateBitCast(RelocatedBase, Base->getType());1328    }1329    Value *Replacement =1330        Builder.CreateGEP(Derived->getSourceElementType(), ActualRelocatedBase,1331                          ArrayRef(OffsetV));1332    Replacement->takeName(ToReplace);1333    // If the newly generated derived pointer's type does not match the original1334    // derived pointer's type, cast the new derived pointer to match it. Same1335    // reasoning as above.1336    Value *ActualReplacement = Replacement;1337    if (Replacement->getType() != ToReplace->getType()) {1338      ActualReplacement =1339          Builder.CreateBitCast(Replacement, ToReplace->getType());1340    }1341    ToReplace->replaceAllUsesWith(ActualReplacement);1342    ToReplace->eraseFromParent();1343 1344    MadeChange = true;1345  }1346  return MadeChange;1347}1348 1349// Turns this:1350//1351// %base = ...1352// %ptr = gep %base + 151353// %tok = statepoint (%fun, i32 0, i32 0, i32 0, %base, %ptr)1354// %base' = relocate(%tok, i32 4, i32 4)1355// %ptr' = relocate(%tok, i32 4, i32 5)1356// %val = load %ptr'1357//1358// into this:1359//1360// %base = ...1361// %ptr = gep %base + 151362// %tok = statepoint (%fun, i32 0, i32 0, i32 0, %base, %ptr)1363// %base' = gc.relocate(%tok, i32 4, i32 4)1364// %ptr' = gep %base' + 151365// %val = load %ptr'1366bool CodeGenPrepare::simplifyOffsetableRelocate(GCStatepointInst &I) {1367  bool MadeChange = false;1368  SmallVector<GCRelocateInst *, 2> AllRelocateCalls;1369  for (auto *U : I.users())1370    if (GCRelocateInst *Relocate = dyn_cast<GCRelocateInst>(U))1371      // Collect all the relocate calls associated with a statepoint1372      AllRelocateCalls.push_back(Relocate);1373 1374  // We need at least one base pointer relocation + one derived pointer1375  // relocation to mangle1376  if (AllRelocateCalls.size() < 2)1377    return false;1378 1379  // RelocateInstMap is a mapping from the base relocate instruction to the1380  // corresponding derived relocate instructions1381  MapVector<GCRelocateInst *, SmallVector<GCRelocateInst *, 0>> RelocateInstMap;1382  computeBaseDerivedRelocateMap(AllRelocateCalls, RelocateInstMap);1383  if (RelocateInstMap.empty())1384    return false;1385 1386  for (auto &Item : RelocateInstMap)1387    // Item.first is the RelocatedBase to offset against1388    // Item.second is the vector of Targets to replace1389    MadeChange = simplifyRelocatesOffABase(Item.first, Item.second);1390  return MadeChange;1391}1392 1393/// Sink the specified cast instruction into its user blocks.1394static bool SinkCast(CastInst *CI) {1395  BasicBlock *DefBB = CI->getParent();1396 1397  /// InsertedCasts - Only insert a cast in each block once.1398  DenseMap<BasicBlock *, CastInst *> InsertedCasts;1399 1400  bool MadeChange = false;1401  for (Value::user_iterator UI = CI->user_begin(), E = CI->user_end();1402       UI != E;) {1403    Use &TheUse = UI.getUse();1404    Instruction *User = cast<Instruction>(*UI);1405 1406    // Figure out which BB this cast is used in.  For PHI's this is the1407    // appropriate predecessor block.1408    BasicBlock *UserBB = User->getParent();1409    if (PHINode *PN = dyn_cast<PHINode>(User)) {1410      UserBB = PN->getIncomingBlock(TheUse);1411    }1412 1413    // Preincrement use iterator so we don't invalidate it.1414    ++UI;1415 1416    // The first insertion point of a block containing an EH pad is after the1417    // pad.  If the pad is the user, we cannot sink the cast past the pad.1418    if (User->isEHPad())1419      continue;1420 1421    // If the block selected to receive the cast is an EH pad that does not1422    // allow non-PHI instructions before the terminator, we can't sink the1423    // cast.1424    if (UserBB->getTerminator()->isEHPad())1425      continue;1426 1427    // If this user is in the same block as the cast, don't change the cast.1428    if (UserBB == DefBB)1429      continue;1430 1431    // If we have already inserted a cast into this block, use it.1432    CastInst *&InsertedCast = InsertedCasts[UserBB];1433 1434    if (!InsertedCast) {1435      BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();1436      assert(InsertPt != UserBB->end());1437      InsertedCast = cast<CastInst>(CI->clone());1438      InsertedCast->insertBefore(*UserBB, InsertPt);1439    }1440 1441    // Replace a use of the cast with a use of the new cast.1442    TheUse = InsertedCast;1443    MadeChange = true;1444    ++NumCastUses;1445  }1446 1447  // If we removed all uses, nuke the cast.1448  if (CI->use_empty()) {1449    salvageDebugInfo(*CI);1450    CI->eraseFromParent();1451    MadeChange = true;1452  }1453 1454  return MadeChange;1455}1456 1457/// If the specified cast instruction is a noop copy (e.g. it's casting from1458/// one pointer type to another, i32->i8 on PPC), sink it into user blocks to1459/// reduce the number of virtual registers that must be created and coalesced.1460///1461/// Return true if any changes are made.1462static bool OptimizeNoopCopyExpression(CastInst *CI, const TargetLowering &TLI,1463                                       const DataLayout &DL) {1464  // Sink only "cheap" (or nop) address-space casts.  This is a weaker condition1465  // than sinking only nop casts, but is helpful on some platforms.1466  if (auto *ASC = dyn_cast<AddrSpaceCastInst>(CI)) {1467    if (!TLI.isFreeAddrSpaceCast(ASC->getSrcAddressSpace(),1468                                 ASC->getDestAddressSpace()))1469      return false;1470  }1471 1472  // If this is a noop copy,1473  EVT SrcVT = TLI.getValueType(DL, CI->getOperand(0)->getType());1474  EVT DstVT = TLI.getValueType(DL, CI->getType());1475 1476  // This is an fp<->int conversion?1477  if (SrcVT.isInteger() != DstVT.isInteger())1478    return false;1479 1480  // If this is an extension, it will be a zero or sign extension, which1481  // isn't a noop.1482  if (SrcVT.bitsLT(DstVT))1483    return false;1484 1485  // If these values will be promoted, find out what they will be promoted1486  // to.  This helps us consider truncates on PPC as noop copies when they1487  // are.1488  if (TLI.getTypeAction(CI->getContext(), SrcVT) ==1489      TargetLowering::TypePromoteInteger)1490    SrcVT = TLI.getTypeToTransformTo(CI->getContext(), SrcVT);1491  if (TLI.getTypeAction(CI->getContext(), DstVT) ==1492      TargetLowering::TypePromoteInteger)1493    DstVT = TLI.getTypeToTransformTo(CI->getContext(), DstVT);1494 1495  // If, after promotion, these are the same types, this is a noop copy.1496  if (SrcVT != DstVT)1497    return false;1498 1499  return SinkCast(CI);1500}1501 1502// Match a simple increment by constant operation.  Note that if a sub is1503// matched, the step is negated (as if the step had been canonicalized to1504// an add, even though we leave the instruction alone.)1505static bool matchIncrement(const Instruction *IVInc, Instruction *&LHS,1506                           Constant *&Step) {1507  if (match(IVInc, m_Add(m_Instruction(LHS), m_Constant(Step))) ||1508      match(IVInc, m_ExtractValue<0>(m_Intrinsic<Intrinsic::uadd_with_overflow>(1509                       m_Instruction(LHS), m_Constant(Step)))))1510    return true;1511  if (match(IVInc, m_Sub(m_Instruction(LHS), m_Constant(Step))) ||1512      match(IVInc, m_ExtractValue<0>(m_Intrinsic<Intrinsic::usub_with_overflow>(1513                       m_Instruction(LHS), m_Constant(Step))))) {1514    Step = ConstantExpr::getNeg(Step);1515    return true;1516  }1517  return false;1518}1519 1520/// If given \p PN is an inductive variable with value IVInc coming from the1521/// backedge, and on each iteration it gets increased by Step, return pair1522/// <IVInc, Step>. Otherwise, return std::nullopt.1523static std::optional<std::pair<Instruction *, Constant *>>1524getIVIncrement(const PHINode *PN, const LoopInfo *LI) {1525  const Loop *L = LI->getLoopFor(PN->getParent());1526  if (!L || L->getHeader() != PN->getParent() || !L->getLoopLatch())1527    return std::nullopt;1528  auto *IVInc =1529      dyn_cast<Instruction>(PN->getIncomingValueForBlock(L->getLoopLatch()));1530  if (!IVInc || LI->getLoopFor(IVInc->getParent()) != L)1531    return std::nullopt;1532  Instruction *LHS = nullptr;1533  Constant *Step = nullptr;1534  if (matchIncrement(IVInc, LHS, Step) && LHS == PN)1535    return std::make_pair(IVInc, Step);1536  return std::nullopt;1537}1538 1539static bool isIVIncrement(const Value *V, const LoopInfo *LI) {1540  auto *I = dyn_cast<Instruction>(V);1541  if (!I)1542    return false;1543  Instruction *LHS = nullptr;1544  Constant *Step = nullptr;1545  if (!matchIncrement(I, LHS, Step))1546    return false;1547  if (auto *PN = dyn_cast<PHINode>(LHS))1548    if (auto IVInc = getIVIncrement(PN, LI))1549      return IVInc->first == I;1550  return false;1551}1552 1553bool CodeGenPrepare::replaceMathCmpWithIntrinsic(BinaryOperator *BO,1554                                                 Value *Arg0, Value *Arg1,1555                                                 CmpInst *Cmp,1556                                                 Intrinsic::ID IID) {1557  auto IsReplacableIVIncrement = [this, &Cmp](BinaryOperator *BO) {1558    if (!isIVIncrement(BO, LI))1559      return false;1560    const Loop *L = LI->getLoopFor(BO->getParent());1561    assert(L && "L should not be null after isIVIncrement()");1562    // Do not risk on moving increment into a child loop.1563    if (LI->getLoopFor(Cmp->getParent()) != L)1564      return false;1565 1566    // Finally, we need to ensure that the insert point will dominate all1567    // existing uses of the increment.1568 1569    auto &DT = getDT(*BO->getParent()->getParent());1570    if (DT.dominates(Cmp->getParent(), BO->getParent()))1571      // If we're moving up the dom tree, all uses are trivially dominated.1572      // (This is the common case for code produced by LSR.)1573      return true;1574 1575    // Otherwise, special case the single use in the phi recurrence.1576    return BO->hasOneUse() && DT.dominates(Cmp->getParent(), L->getLoopLatch());1577  };1578  if (BO->getParent() != Cmp->getParent() && !IsReplacableIVIncrement(BO)) {1579    // We used to use a dominator tree here to allow multi-block optimization.1580    // But that was problematic because:1581    // 1. It could cause a perf regression by hoisting the math op into the1582    //    critical path.1583    // 2. It could cause a perf regression by creating a value that was live1584    //    across multiple blocks and increasing register pressure.1585    // 3. Use of a dominator tree could cause large compile-time regression.1586    //    This is because we recompute the DT on every change in the main CGP1587    //    run-loop. The recomputing is probably unnecessary in many cases, so if1588    //    that was fixed, using a DT here would be ok.1589    //1590    // There is one important particular case we still want to handle: if BO is1591    // the IV increment. Important properties that make it profitable:1592    // - We can speculate IV increment anywhere in the loop (as long as the1593    //   indvar Phi is its only user);1594    // - Upon computing Cmp, we effectively compute something equivalent to the1595    //   IV increment (despite it loops differently in the IR). So moving it up1596    //   to the cmp point does not really increase register pressure.1597    return false;1598  }1599 1600  // We allow matching the canonical IR (add X, C) back to (usubo X, -C).1601  if (BO->getOpcode() == Instruction::Add &&1602      IID == Intrinsic::usub_with_overflow) {1603    assert(isa<Constant>(Arg1) && "Unexpected input for usubo");1604    Arg1 = ConstantExpr::getNeg(cast<Constant>(Arg1));1605  }1606 1607  // Insert at the first instruction of the pair.1608  Instruction *InsertPt = nullptr;1609  for (Instruction &Iter : *Cmp->getParent()) {1610    // If BO is an XOR, it is not guaranteed that it comes after both inputs to1611    // the overflow intrinsic are defined.1612    if ((BO->getOpcode() != Instruction::Xor && &Iter == BO) || &Iter == Cmp) {1613      InsertPt = &Iter;1614      break;1615    }1616  }1617  assert(InsertPt != nullptr && "Parent block did not contain cmp or binop");1618 1619  IRBuilder<> Builder(InsertPt);1620  Value *MathOV = Builder.CreateBinaryIntrinsic(IID, Arg0, Arg1);1621  if (BO->getOpcode() != Instruction::Xor) {1622    Value *Math = Builder.CreateExtractValue(MathOV, 0, "math");1623    replaceAllUsesWith(BO, Math, FreshBBs, IsHugeFunc);1624  } else1625    assert(BO->hasOneUse() &&1626           "Patterns with XOr should use the BO only in the compare");1627  Value *OV = Builder.CreateExtractValue(MathOV, 1, "ov");1628  replaceAllUsesWith(Cmp, OV, FreshBBs, IsHugeFunc);1629  Cmp->eraseFromParent();1630  BO->eraseFromParent();1631  return true;1632}1633 1634/// Match special-case patterns that check for unsigned add overflow.1635static bool matchUAddWithOverflowConstantEdgeCases(CmpInst *Cmp,1636                                                   BinaryOperator *&Add) {1637  // Add = add A, 1; Cmp = icmp eq A,-1 (overflow if A is max val)1638  // Add = add A,-1; Cmp = icmp ne A, 0 (overflow if A is non-zero)1639  Value *A = Cmp->getOperand(0), *B = Cmp->getOperand(1);1640 1641  // We are not expecting non-canonical/degenerate code. Just bail out.1642  if (isa<Constant>(A))1643    return false;1644 1645  ICmpInst::Predicate Pred = Cmp->getPredicate();1646  if (Pred == ICmpInst::ICMP_EQ && match(B, m_AllOnes()))1647    B = ConstantInt::get(B->getType(), 1);1648  else if (Pred == ICmpInst::ICMP_NE && match(B, m_ZeroInt()))1649    B = Constant::getAllOnesValue(B->getType());1650  else1651    return false;1652 1653  // Check the users of the variable operand of the compare looking for an add1654  // with the adjusted constant.1655  for (User *U : A->users()) {1656    if (match(U, m_Add(m_Specific(A), m_Specific(B)))) {1657      Add = cast<BinaryOperator>(U);1658      return true;1659    }1660  }1661  return false;1662}1663 1664/// Try to combine the compare into a call to the llvm.uadd.with.overflow1665/// intrinsic. Return true if any changes were made.1666bool CodeGenPrepare::combineToUAddWithOverflow(CmpInst *Cmp,1667                                               ModifyDT &ModifiedDT) {1668  bool EdgeCase = false;1669  Value *A, *B;1670  BinaryOperator *Add;1671  if (!match(Cmp, m_UAddWithOverflow(m_Value(A), m_Value(B), m_BinOp(Add)))) {1672    if (!matchUAddWithOverflowConstantEdgeCases(Cmp, Add))1673      return false;1674    // Set A and B in case we match matchUAddWithOverflowConstantEdgeCases.1675    A = Add->getOperand(0);1676    B = Add->getOperand(1);1677    EdgeCase = true;1678  }1679 1680  if (!TLI->shouldFormOverflowOp(ISD::UADDO,1681                                 TLI->getValueType(*DL, Add->getType()),1682                                 Add->hasNUsesOrMore(EdgeCase ? 1 : 2)))1683    return false;1684 1685  // We don't want to move around uses of condition values this late, so we1686  // check if it is legal to create the call to the intrinsic in the basic1687  // block containing the icmp.1688  if (Add->getParent() != Cmp->getParent() && !Add->hasOneUse())1689    return false;1690 1691  if (!replaceMathCmpWithIntrinsic(Add, A, B, Cmp,1692                                   Intrinsic::uadd_with_overflow))1693    return false;1694 1695  // Reset callers - do not crash by iterating over a dead instruction.1696  ModifiedDT = ModifyDT::ModifyInstDT;1697  return true;1698}1699 1700bool CodeGenPrepare::combineToUSubWithOverflow(CmpInst *Cmp,1701                                               ModifyDT &ModifiedDT) {1702  // We are not expecting non-canonical/degenerate code. Just bail out.1703  Value *A = Cmp->getOperand(0), *B = Cmp->getOperand(1);1704  if (isa<Constant>(A) && isa<Constant>(B))1705    return false;1706 1707  // Convert (A u> B) to (A u< B) to simplify pattern matching.1708  ICmpInst::Predicate Pred = Cmp->getPredicate();1709  if (Pred == ICmpInst::ICMP_UGT) {1710    std::swap(A, B);1711    Pred = ICmpInst::ICMP_ULT;1712  }1713  // Convert special-case: (A == 0) is the same as (A u< 1).1714  if (Pred == ICmpInst::ICMP_EQ && match(B, m_ZeroInt())) {1715    B = ConstantInt::get(B->getType(), 1);1716    Pred = ICmpInst::ICMP_ULT;1717  }1718  // Convert special-case: (A != 0) is the same as (0 u< A).1719  if (Pred == ICmpInst::ICMP_NE && match(B, m_ZeroInt())) {1720    std::swap(A, B);1721    Pred = ICmpInst::ICMP_ULT;1722  }1723  if (Pred != ICmpInst::ICMP_ULT)1724    return false;1725 1726  // Walk the users of a variable operand of a compare looking for a subtract or1727  // add with that same operand. Also match the 2nd operand of the compare to1728  // the add/sub, but that may be a negated constant operand of an add.1729  Value *CmpVariableOperand = isa<Constant>(A) ? B : A;1730  BinaryOperator *Sub = nullptr;1731  for (User *U : CmpVariableOperand->users()) {1732    // A - B, A u< B --> usubo(A, B)1733    if (match(U, m_Sub(m_Specific(A), m_Specific(B)))) {1734      Sub = cast<BinaryOperator>(U);1735      break;1736    }1737 1738    // A + (-C), A u< C (canonicalized form of (sub A, C))1739    const APInt *CmpC, *AddC;1740    if (match(U, m_Add(m_Specific(A), m_APInt(AddC))) &&1741        match(B, m_APInt(CmpC)) && *AddC == -(*CmpC)) {1742      Sub = cast<BinaryOperator>(U);1743      break;1744    }1745  }1746  if (!Sub)1747    return false;1748 1749  if (!TLI->shouldFormOverflowOp(ISD::USUBO,1750                                 TLI->getValueType(*DL, Sub->getType()),1751                                 Sub->hasNUsesOrMore(1)))1752    return false;1753 1754  // We don't want to move around uses of condition values this late, so we1755  // check if it is legal to create the call to the intrinsic in the basic1756  // block containing the icmp.1757  if (Sub->getParent() != Cmp->getParent() && !Sub->hasOneUse())1758    return false;1759 1760  if (!replaceMathCmpWithIntrinsic(Sub, Sub->getOperand(0), Sub->getOperand(1),1761                                   Cmp, Intrinsic::usub_with_overflow))1762    return false;1763 1764  // Reset callers - do not crash by iterating over a dead instruction.1765  ModifiedDT = ModifyDT::ModifyInstDT;1766  return true;1767}1768 1769// Decanonicalizes icmp+ctpop power-of-two test if ctpop is slow.1770// The same transformation exists in DAG combiner, but we repeat it here because1771// DAG builder can break the pattern by moving icmp into a successor block.1772bool CodeGenPrepare::unfoldPowerOf2Test(CmpInst *Cmp) {1773  CmpPredicate Pred;1774  Value *X;1775  const APInt *C;1776 1777  // (icmp (ctpop x), c)1778  if (!match(Cmp, m_ICmp(Pred, m_Intrinsic<Intrinsic::ctpop>(m_Value(X)),1779                         m_APIntAllowPoison(C))))1780    return false;1781 1782  // We're only interested in "is power of 2 [or zero]" patterns.1783  bool IsStrictlyPowerOf2Test = ICmpInst::isEquality(Pred) && *C == 1;1784  bool IsPowerOf2OrZeroTest = (Pred == CmpInst::ICMP_ULT && *C == 2) ||1785                              (Pred == CmpInst::ICMP_UGT && *C == 1);1786  if (!IsStrictlyPowerOf2Test && !IsPowerOf2OrZeroTest)1787    return false;1788 1789  // Some targets have better codegen for `ctpop(x) u</u>= 2/1`than for1790  // `ctpop(x) ==/!= 1`. If ctpop is fast, only try changing the comparison,1791  // and otherwise expand ctpop into a few simple instructions.1792  Type *OpTy = X->getType();1793  if (TLI->isCtpopFast(TLI->getValueType(*DL, OpTy))) {1794    // Look for `ctpop(x) ==/!= 1`, where `ctpop(x)` is known to be non-zero.1795    if (!IsStrictlyPowerOf2Test || !isKnownNonZero(Cmp->getOperand(0), *DL))1796      return false;1797 1798    // ctpop(x) == 1 -> ctpop(x) u< 21799    // ctpop(x) != 1 -> ctpop(x) u> 11800    if (Pred == ICmpInst::ICMP_EQ) {1801      Cmp->setOperand(1, ConstantInt::get(OpTy, 2));1802      Cmp->setPredicate(ICmpInst::ICMP_ULT);1803    } else {1804      Cmp->setPredicate(ICmpInst::ICMP_UGT);1805    }1806    return true;1807  }1808 1809  Value *NewCmp;1810  if (IsPowerOf2OrZeroTest ||1811      (IsStrictlyPowerOf2Test && isKnownNonZero(Cmp->getOperand(0), *DL))) {1812    // ctpop(x) u< 2 -> (x & (x - 1)) == 01813    // ctpop(x) u> 1 -> (x & (x - 1)) != 01814    IRBuilder<> Builder(Cmp);1815    Value *Sub = Builder.CreateAdd(X, Constant::getAllOnesValue(OpTy));1816    Value *And = Builder.CreateAnd(X, Sub);1817    CmpInst::Predicate NewPred =1818        (Pred == CmpInst::ICMP_ULT || Pred == CmpInst::ICMP_EQ)1819            ? CmpInst::ICMP_EQ1820            : CmpInst::ICMP_NE;1821    NewCmp = Builder.CreateICmp(NewPred, And, ConstantInt::getNullValue(OpTy));1822  } else {1823    // ctpop(x) == 1 -> (x ^ (x - 1)) u> (x - 1)1824    // ctpop(x) != 1 -> (x ^ (x - 1)) u<= (x - 1)1825    IRBuilder<> Builder(Cmp);1826    Value *Sub = Builder.CreateAdd(X, Constant::getAllOnesValue(OpTy));1827    Value *Xor = Builder.CreateXor(X, Sub);1828    CmpInst::Predicate NewPred =1829        Pred == CmpInst::ICMP_EQ ? CmpInst::ICMP_UGT : CmpInst::ICMP_ULE;1830    NewCmp = Builder.CreateICmp(NewPred, Xor, Sub);1831  }1832 1833  Cmp->replaceAllUsesWith(NewCmp);1834  RecursivelyDeleteTriviallyDeadInstructions(Cmp);1835  return true;1836}1837 1838/// Sink the given CmpInst into user blocks to reduce the number of virtual1839/// registers that must be created and coalesced. This is a clear win except on1840/// targets with multiple condition code registers (PowerPC), where it might1841/// lose; some adjustment may be wanted there.1842///1843/// Return true if any changes are made.1844static bool sinkCmpExpression(CmpInst *Cmp, const TargetLowering &TLI,1845                              const DataLayout &DL) {1846  if (TLI.hasMultipleConditionRegisters(EVT::getEVT(Cmp->getType())))1847    return false;1848 1849  // Avoid sinking soft-FP comparisons, since this can move them into a loop.1850  if (TLI.useSoftFloat() && isa<FCmpInst>(Cmp))1851    return false;1852 1853  bool UsedInPhiOrCurrentBlock = any_of(Cmp->users(), [Cmp](User *U) {1854    return isa<PHINode>(U) ||1855           cast<Instruction>(U)->getParent() == Cmp->getParent();1856  });1857 1858  // Avoid sinking larger than legal integer comparisons unless its ONLY used in1859  // another BB.1860  if (UsedInPhiOrCurrentBlock && Cmp->getOperand(0)->getType()->isIntegerTy() &&1861      Cmp->getOperand(0)->getType()->getScalarSizeInBits() >1862          DL.getLargestLegalIntTypeSizeInBits())1863    return false;1864 1865  // Only insert a cmp in each block once.1866  DenseMap<BasicBlock *, CmpInst *> InsertedCmps;1867 1868  bool MadeChange = false;1869  for (Value::user_iterator UI = Cmp->user_begin(), E = Cmp->user_end();1870       UI != E;) {1871    Use &TheUse = UI.getUse();1872    Instruction *User = cast<Instruction>(*UI);1873 1874    // Preincrement use iterator so we don't invalidate it.1875    ++UI;1876 1877    // Don't bother for PHI nodes.1878    if (isa<PHINode>(User))1879      continue;1880 1881    // Figure out which BB this cmp is used in.1882    BasicBlock *UserBB = User->getParent();1883    BasicBlock *DefBB = Cmp->getParent();1884 1885    // If this user is in the same block as the cmp, don't change the cmp.1886    if (UserBB == DefBB)1887      continue;1888 1889    // If we have already inserted a cmp into this block, use it.1890    CmpInst *&InsertedCmp = InsertedCmps[UserBB];1891 1892    if (!InsertedCmp) {1893      BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();1894      assert(InsertPt != UserBB->end());1895      InsertedCmp = CmpInst::Create(Cmp->getOpcode(), Cmp->getPredicate(),1896                                    Cmp->getOperand(0), Cmp->getOperand(1), "");1897      InsertedCmp->insertBefore(*UserBB, InsertPt);1898      // Propagate the debug info.1899      InsertedCmp->setDebugLoc(Cmp->getDebugLoc());1900    }1901 1902    // Replace a use of the cmp with a use of the new cmp.1903    TheUse = InsertedCmp;1904    MadeChange = true;1905    ++NumCmpUses;1906  }1907 1908  // If we removed all uses, nuke the cmp.1909  if (Cmp->use_empty()) {1910    Cmp->eraseFromParent();1911    MadeChange = true;1912  }1913 1914  return MadeChange;1915}1916 1917/// For pattern like:1918///1919///   DomCond = icmp sgt/slt CmpOp0, CmpOp1 (might not be in DomBB)1920///   ...1921/// DomBB:1922///   ...1923///   br DomCond, TrueBB, CmpBB1924/// CmpBB: (with DomBB being the single predecessor)1925///   ...1926///   Cmp = icmp eq CmpOp0, CmpOp11927///   ...1928///1929/// It would use two comparison on targets that lowering of icmp sgt/slt is1930/// different from lowering of icmp eq (PowerPC). This function try to convert1931/// 'Cmp = icmp eq CmpOp0, CmpOp1' to ' Cmp = icmp slt/sgt CmpOp0, CmpOp1'.1932/// After that, DomCond and Cmp can use the same comparison so reduce one1933/// comparison.1934///1935/// Return true if any changes are made.1936static bool foldICmpWithDominatingICmp(CmpInst *Cmp,1937                                       const TargetLowering &TLI) {1938  if (!EnableICMP_EQToICMP_ST && TLI.isEqualityCmpFoldedWithSignedCmp())1939    return false;1940 1941  ICmpInst::Predicate Pred = Cmp->getPredicate();1942  if (Pred != ICmpInst::ICMP_EQ)1943    return false;1944 1945  // If icmp eq has users other than BranchInst and SelectInst, converting it to1946  // icmp slt/sgt would introduce more redundant LLVM IR.1947  for (User *U : Cmp->users()) {1948    if (isa<BranchInst>(U))1949      continue;1950    if (isa<SelectInst>(U) && cast<SelectInst>(U)->getCondition() == Cmp)1951      continue;1952    return false;1953  }1954 1955  // This is a cheap/incomplete check for dominance - just match a single1956  // predecessor with a conditional branch.1957  BasicBlock *CmpBB = Cmp->getParent();1958  BasicBlock *DomBB = CmpBB->getSinglePredecessor();1959  if (!DomBB)1960    return false;1961 1962  // We want to ensure that the only way control gets to the comparison of1963  // interest is that a less/greater than comparison on the same operands is1964  // false.1965  Value *DomCond;1966  BasicBlock *TrueBB, *FalseBB;1967  if (!match(DomBB->getTerminator(), m_Br(m_Value(DomCond), TrueBB, FalseBB)))1968    return false;1969  if (CmpBB != FalseBB)1970    return false;1971 1972  Value *CmpOp0 = Cmp->getOperand(0), *CmpOp1 = Cmp->getOperand(1);1973  CmpPredicate DomPred;1974  if (!match(DomCond, m_ICmp(DomPred, m_Specific(CmpOp0), m_Specific(CmpOp1))))1975    return false;1976  if (DomPred != ICmpInst::ICMP_SGT && DomPred != ICmpInst::ICMP_SLT)1977    return false;1978 1979  // Convert the equality comparison to the opposite of the dominating1980  // comparison and swap the direction for all branch/select users.1981  // We have conceptually converted:1982  // Res = (a < b) ? <LT_RES> : (a == b) ? <EQ_RES> : <GT_RES>;1983  // to1984  // Res = (a < b) ? <LT_RES> : (a > b)  ? <GT_RES> : <EQ_RES>;1985  // And similarly for branches.1986  for (User *U : Cmp->users()) {1987    if (auto *BI = dyn_cast<BranchInst>(U)) {1988      assert(BI->isConditional() && "Must be conditional");1989      BI->swapSuccessors();1990      continue;1991    }1992    if (auto *SI = dyn_cast<SelectInst>(U)) {1993      // Swap operands1994      SI->swapValues();1995      SI->swapProfMetadata();1996      continue;1997    }1998    llvm_unreachable("Must be a branch or a select");1999  }2000  Cmp->setPredicate(CmpInst::getSwappedPredicate(DomPred));2001  return true;2002}2003 2004/// Many architectures use the same instruction for both subtract and cmp. Try2005/// to swap cmp operands to match subtract operations to allow for CSE.2006static bool swapICmpOperandsToExposeCSEOpportunities(CmpInst *Cmp) {2007  Value *Op0 = Cmp->getOperand(0);2008  Value *Op1 = Cmp->getOperand(1);2009  if (!Op0->getType()->isIntegerTy() || isa<Constant>(Op0) ||2010      isa<Constant>(Op1) || Op0 == Op1)2011    return false;2012 2013  // If a subtract already has the same operands as a compare, swapping would be2014  // bad. If a subtract has the same operands as a compare but in reverse order,2015  // then swapping is good.2016  int GoodToSwap = 0;2017  unsigned NumInspected = 0;2018  for (const User *U : Op0->users()) {2019    // Avoid walking many users.2020    if (++NumInspected > 128)2021      return false;2022    if (match(U, m_Sub(m_Specific(Op1), m_Specific(Op0))))2023      GoodToSwap++;2024    else if (match(U, m_Sub(m_Specific(Op0), m_Specific(Op1))))2025      GoodToSwap--;2026  }2027 2028  if (GoodToSwap > 0) {2029    Cmp->swapOperands();2030    return true;2031  }2032  return false;2033}2034 2035static bool foldFCmpToFPClassTest(CmpInst *Cmp, const TargetLowering &TLI,2036                                  const DataLayout &DL) {2037  FCmpInst *FCmp = dyn_cast<FCmpInst>(Cmp);2038  if (!FCmp)2039    return false;2040 2041  // Don't fold if the target offers free fabs and the predicate is legal.2042  EVT VT = TLI.getValueType(DL, Cmp->getOperand(0)->getType());2043  if (TLI.isFAbsFree(VT) &&2044      TLI.isCondCodeLegal(getFCmpCondCode(FCmp->getPredicate()),2045                          VT.getSimpleVT()))2046    return false;2047 2048  // Reverse the canonicalization if it is a FP class test2049  auto ShouldReverseTransform = [](FPClassTest ClassTest) {2050    return ClassTest == fcInf || ClassTest == (fcInf | fcNan);2051  };2052  auto [ClassVal, ClassTest] =2053      fcmpToClassTest(FCmp->getPredicate(), *FCmp->getParent()->getParent(),2054                      FCmp->getOperand(0), FCmp->getOperand(1));2055  if (!ClassVal)2056    return false;2057 2058  if (!ShouldReverseTransform(ClassTest) && !ShouldReverseTransform(~ClassTest))2059    return false;2060 2061  IRBuilder<> Builder(Cmp);2062  Value *IsFPClass = Builder.createIsFPClass(ClassVal, ClassTest);2063  Cmp->replaceAllUsesWith(IsFPClass);2064  RecursivelyDeleteTriviallyDeadInstructions(Cmp);2065  return true;2066}2067 2068static bool isRemOfLoopIncrementWithLoopInvariant(2069    Instruction *Rem, const LoopInfo *LI, Value *&RemAmtOut, Value *&AddInstOut,2070    Value *&AddOffsetOut, PHINode *&LoopIncrPNOut) {2071  Value *Incr, *RemAmt;2072  // NB: If RemAmt is a power of 2 it *should* have been transformed by now.2073  if (!match(Rem, m_URem(m_Value(Incr), m_Value(RemAmt))))2074    return false;2075 2076  Value *AddInst, *AddOffset;2077  // Find out loop increment PHI.2078  auto *PN = dyn_cast<PHINode>(Incr);2079  if (PN != nullptr) {2080    AddInst = nullptr;2081    AddOffset = nullptr;2082  } else {2083    // Search through a NUW add on top of the loop increment.2084    Value *V0, *V1;2085    if (!match(Incr, m_NUWAdd(m_Value(V0), m_Value(V1))))2086      return false;2087 2088    AddInst = Incr;2089    PN = dyn_cast<PHINode>(V0);2090    if (PN != nullptr) {2091      AddOffset = V1;2092    } else {2093      PN = dyn_cast<PHINode>(V1);2094      AddOffset = V0;2095    }2096  }2097 2098  if (!PN)2099    return false;2100 2101  // This isn't strictly necessary, what we really need is one increment and any2102  // amount of initial values all being the same.2103  if (PN->getNumIncomingValues() != 2)2104    return false;2105 2106  // Only trivially analyzable loops.2107  Loop *L = LI->getLoopFor(PN->getParent());2108  if (!L || !L->getLoopPreheader() || !L->getLoopLatch())2109    return false;2110 2111  // Req that the remainder is in the loop2112  if (!L->contains(Rem))2113    return false;2114 2115  // Only works if the remainder amount is a loop invaraint2116  if (!L->isLoopInvariant(RemAmt))2117    return false;2118 2119  // Only works if the AddOffset is a loop invaraint2120  if (AddOffset && !L->isLoopInvariant(AddOffset))2121    return false;2122 2123  // Is the PHI a loop increment?2124  auto LoopIncrInfo = getIVIncrement(PN, LI);2125  if (!LoopIncrInfo)2126    return false;2127 2128  // We need remainder_amount % increment_amount to be zero. Increment of one2129  // satisfies that without any special logic and is overwhelmingly the common2130  // case.2131  if (!match(LoopIncrInfo->second, m_One()))2132    return false;2133 2134  // Need the increment to not overflow.2135  if (!match(LoopIncrInfo->first, m_c_NUWAdd(m_Specific(PN), m_Value())))2136    return false;2137 2138  // Set output variables.2139  RemAmtOut = RemAmt;2140  LoopIncrPNOut = PN;2141  AddInstOut = AddInst;2142  AddOffsetOut = AddOffset;2143 2144  return true;2145}2146 2147// Try to transform:2148//2149// for(i = Start; i < End; ++i)2150//    Rem = (i nuw+ IncrLoopInvariant) u% RemAmtLoopInvariant;2151//2152// ->2153//2154// Rem = (Start nuw+ IncrLoopInvariant) % RemAmtLoopInvariant;2155// for(i = Start; i < End; ++i, ++rem)2156//    Rem = rem == RemAmtLoopInvariant ? 0 : Rem;2157static bool foldURemOfLoopIncrement(Instruction *Rem, const DataLayout *DL,2158                                    const LoopInfo *LI,2159                                    SmallPtrSet<BasicBlock *, 32> &FreshBBs,2160                                    bool IsHuge) {2161  Value *AddOffset, *RemAmt, *AddInst;2162  PHINode *LoopIncrPN;2163  if (!isRemOfLoopIncrementWithLoopInvariant(Rem, LI, RemAmt, AddInst,2164                                             AddOffset, LoopIncrPN))2165    return false;2166 2167  // Only non-constant remainder as the extra IV is probably not profitable2168  // in that case.2169  //2170  // Potential TODO(1): `urem` of a const ends up as `mul` + `shift` + `add`. If2171  // we can rule out register pressure and ensure this `urem` is executed each2172  // iteration, its probably profitable to handle the const case as well.2173  //2174  // Potential TODO(2): Should we have a check for how "nested" this remainder2175  // operation is? The new code runs every iteration so if the remainder is2176  // guarded behind unlikely conditions this might not be worth it.2177  if (match(RemAmt, m_ImmConstant()))2178    return false;2179 2180  Loop *L = LI->getLoopFor(LoopIncrPN->getParent());2181  Value *Start = LoopIncrPN->getIncomingValueForBlock(L->getLoopPreheader());2182  // If we have add create initial value for remainder.2183  // The logic here is:2184  // (urem (add nuw Start, IncrLoopInvariant), RemAmtLoopInvariant2185  //2186  // Only proceed if the expression simplifies (otherwise we can't fully2187  // optimize out the urem).2188  if (AddInst) {2189    assert(AddOffset && "We found an add but missing values");2190    // Without dom-condition/assumption cache we aren't likely to get much out2191    // of a context instruction.2192    Start = simplifyAddInst(Start, AddOffset,2193                            match(AddInst, m_NSWAdd(m_Value(), m_Value())),2194                            /*IsNUW=*/true, *DL);2195    if (!Start)2196      return false;2197  }2198 2199  // If we can't fully optimize out the `rem`, skip this transform.2200  Start = simplifyURemInst(Start, RemAmt, *DL);2201  if (!Start)2202    return false;2203 2204  // Create new remainder with induction variable.2205  Type *Ty = Rem->getType();2206  IRBuilder<> Builder(Rem->getContext());2207 2208  Builder.SetInsertPoint(LoopIncrPN);2209  PHINode *NewRem = Builder.CreatePHI(Ty, 2);2210 2211  Builder.SetInsertPoint(cast<Instruction>(2212      LoopIncrPN->getIncomingValueForBlock(L->getLoopLatch())));2213  // `(add (urem x, y), 1)` is always nuw.2214  Value *RemAdd = Builder.CreateNUWAdd(NewRem, ConstantInt::get(Ty, 1));2215  Value *RemCmp = Builder.CreateICmp(ICmpInst::ICMP_EQ, RemAdd, RemAmt);2216  Value *RemSel =2217      Builder.CreateSelect(RemCmp, Constant::getNullValue(Ty), RemAdd);2218 2219  NewRem->addIncoming(Start, L->getLoopPreheader());2220  NewRem->addIncoming(RemSel, L->getLoopLatch());2221 2222  // Insert all touched BBs.2223  FreshBBs.insert(LoopIncrPN->getParent());2224  FreshBBs.insert(L->getLoopLatch());2225  FreshBBs.insert(Rem->getParent());2226  if (AddInst)2227    FreshBBs.insert(cast<Instruction>(AddInst)->getParent());2228  replaceAllUsesWith(Rem, NewRem, FreshBBs, IsHuge);2229  Rem->eraseFromParent();2230  if (AddInst && AddInst->use_empty())2231    cast<Instruction>(AddInst)->eraseFromParent();2232  return true;2233}2234 2235bool CodeGenPrepare::optimizeURem(Instruction *Rem) {2236  if (foldURemOfLoopIncrement(Rem, DL, LI, FreshBBs, IsHugeFunc))2237    return true;2238  return false;2239}2240 2241bool CodeGenPrepare::optimizeCmp(CmpInst *Cmp, ModifyDT &ModifiedDT) {2242  if (sinkCmpExpression(Cmp, *TLI, *DL))2243    return true;2244 2245  if (combineToUAddWithOverflow(Cmp, ModifiedDT))2246    return true;2247 2248  if (combineToUSubWithOverflow(Cmp, ModifiedDT))2249    return true;2250 2251  if (unfoldPowerOf2Test(Cmp))2252    return true;2253 2254  if (foldICmpWithDominatingICmp(Cmp, *TLI))2255    return true;2256 2257  if (swapICmpOperandsToExposeCSEOpportunities(Cmp))2258    return true;2259 2260  if (foldFCmpToFPClassTest(Cmp, *TLI, *DL))2261    return true;2262 2263  return false;2264}2265 2266/// Duplicate and sink the given 'and' instruction into user blocks where it is2267/// used in a compare to allow isel to generate better code for targets where2268/// this operation can be combined.2269///2270/// Return true if any changes are made.2271static bool sinkAndCmp0Expression(Instruction *AndI, const TargetLowering &TLI,2272                                  SetOfInstrs &InsertedInsts) {2273  // Double-check that we're not trying to optimize an instruction that was2274  // already optimized by some other part of this pass.2275  assert(!InsertedInsts.count(AndI) &&2276         "Attempting to optimize already optimized and instruction");2277  (void)InsertedInsts;2278 2279  // Nothing to do for single use in same basic block.2280  if (AndI->hasOneUse() &&2281      AndI->getParent() == cast<Instruction>(*AndI->user_begin())->getParent())2282    return false;2283 2284  // Try to avoid cases where sinking/duplicating is likely to increase register2285  // pressure.2286  if (!isa<ConstantInt>(AndI->getOperand(0)) &&2287      !isa<ConstantInt>(AndI->getOperand(1)) &&2288      AndI->getOperand(0)->hasOneUse() && AndI->getOperand(1)->hasOneUse())2289    return false;2290 2291  for (auto *U : AndI->users()) {2292    Instruction *User = cast<Instruction>(U);2293 2294    // Only sink 'and' feeding icmp with 0.2295    if (!isa<ICmpInst>(User))2296      return false;2297 2298    auto *CmpC = dyn_cast<ConstantInt>(User->getOperand(1));2299    if (!CmpC || !CmpC->isZero())2300      return false;2301  }2302 2303  if (!TLI.isMaskAndCmp0FoldingBeneficial(*AndI))2304    return false;2305 2306  LLVM_DEBUG(dbgs() << "found 'and' feeding only icmp 0;\n");2307  LLVM_DEBUG(AndI->getParent()->dump());2308 2309  // Push the 'and' into the same block as the icmp 0.  There should only be2310  // one (icmp (and, 0)) in each block, since CSE/GVN should have removed any2311  // others, so we don't need to keep track of which BBs we insert into.2312  for (Value::user_iterator UI = AndI->user_begin(), E = AndI->user_end();2313       UI != E;) {2314    Use &TheUse = UI.getUse();2315    Instruction *User = cast<Instruction>(*UI);2316 2317    // Preincrement use iterator so we don't invalidate it.2318    ++UI;2319 2320    LLVM_DEBUG(dbgs() << "sinking 'and' use: " << *User << "\n");2321 2322    // Keep the 'and' in the same place if the use is already in the same block.2323    Instruction *InsertPt =2324        User->getParent() == AndI->getParent() ? AndI : User;2325    Instruction *InsertedAnd = BinaryOperator::Create(2326        Instruction::And, AndI->getOperand(0), AndI->getOperand(1), "",2327        InsertPt->getIterator());2328    // Propagate the debug info.2329    InsertedAnd->setDebugLoc(AndI->getDebugLoc());2330 2331    // Replace a use of the 'and' with a use of the new 'and'.2332    TheUse = InsertedAnd;2333    ++NumAndUses;2334    LLVM_DEBUG(User->getParent()->dump());2335  }2336 2337  // We removed all uses, nuke the and.2338  AndI->eraseFromParent();2339  return true;2340}2341 2342/// Check if the candidates could be combined with a shift instruction, which2343/// includes:2344/// 1. Truncate instruction2345/// 2. And instruction and the imm is a mask of the low bits:2346/// imm & (imm+1) == 02347static bool isExtractBitsCandidateUse(Instruction *User) {2348  if (!isa<TruncInst>(User)) {2349    if (User->getOpcode() != Instruction::And ||2350        !isa<ConstantInt>(User->getOperand(1)))2351      return false;2352 2353    const APInt &Cimm = cast<ConstantInt>(User->getOperand(1))->getValue();2354 2355    if ((Cimm & (Cimm + 1)).getBoolValue())2356      return false;2357  }2358  return true;2359}2360 2361/// Sink both shift and truncate instruction to the use of truncate's BB.2362static bool2363SinkShiftAndTruncate(BinaryOperator *ShiftI, Instruction *User, ConstantInt *CI,2364                     DenseMap<BasicBlock *, BinaryOperator *> &InsertedShifts,2365                     const TargetLowering &TLI, const DataLayout &DL) {2366  BasicBlock *UserBB = User->getParent();2367  DenseMap<BasicBlock *, CastInst *> InsertedTruncs;2368  auto *TruncI = cast<TruncInst>(User);2369  bool MadeChange = false;2370 2371  for (Value::user_iterator TruncUI = TruncI->user_begin(),2372                            TruncE = TruncI->user_end();2373       TruncUI != TruncE;) {2374 2375    Use &TruncTheUse = TruncUI.getUse();2376    Instruction *TruncUser = cast<Instruction>(*TruncUI);2377    // Preincrement use iterator so we don't invalidate it.2378 2379    ++TruncUI;2380 2381    int ISDOpcode = TLI.InstructionOpcodeToISD(TruncUser->getOpcode());2382    if (!ISDOpcode)2383      continue;2384 2385    // If the use is actually a legal node, there will not be an2386    // implicit truncate.2387    // FIXME: always querying the result type is just an2388    // approximation; some nodes' legality is determined by the2389    // operand or other means. There's no good way to find out though.2390    if (TLI.isOperationLegalOrCustom(2391            ISDOpcode, TLI.getValueType(DL, TruncUser->getType(), true)))2392      continue;2393 2394    // Don't bother for PHI nodes.2395    if (isa<PHINode>(TruncUser))2396      continue;2397 2398    BasicBlock *TruncUserBB = TruncUser->getParent();2399 2400    if (UserBB == TruncUserBB)2401      continue;2402 2403    BinaryOperator *&InsertedShift = InsertedShifts[TruncUserBB];2404    CastInst *&InsertedTrunc = InsertedTruncs[TruncUserBB];2405 2406    if (!InsertedShift && !InsertedTrunc) {2407      BasicBlock::iterator InsertPt = TruncUserBB->getFirstInsertionPt();2408      assert(InsertPt != TruncUserBB->end());2409      // Sink the shift2410      if (ShiftI->getOpcode() == Instruction::AShr)2411        InsertedShift =2412            BinaryOperator::CreateAShr(ShiftI->getOperand(0), CI, "");2413      else2414        InsertedShift =2415            BinaryOperator::CreateLShr(ShiftI->getOperand(0), CI, "");2416      InsertedShift->setDebugLoc(ShiftI->getDebugLoc());2417      InsertedShift->insertBefore(*TruncUserBB, InsertPt);2418 2419      // Sink the trunc2420      BasicBlock::iterator TruncInsertPt = TruncUserBB->getFirstInsertionPt();2421      TruncInsertPt++;2422      // It will go ahead of any debug-info.2423      TruncInsertPt.setHeadBit(true);2424      assert(TruncInsertPt != TruncUserBB->end());2425 2426      InsertedTrunc = CastInst::Create(TruncI->getOpcode(), InsertedShift,2427                                       TruncI->getType(), "");2428      InsertedTrunc->insertBefore(*TruncUserBB, TruncInsertPt);2429      InsertedTrunc->setDebugLoc(TruncI->getDebugLoc());2430 2431      MadeChange = true;2432 2433      TruncTheUse = InsertedTrunc;2434    }2435  }2436  return MadeChange;2437}2438 2439/// Sink the shift *right* instruction into user blocks if the uses could2440/// potentially be combined with this shift instruction and generate BitExtract2441/// instruction. It will only be applied if the architecture supports BitExtract2442/// instruction. Here is an example:2443/// BB1:2444///   %x.extract.shift = lshr i64 %arg1, 322445/// BB2:2446///   %x.extract.trunc = trunc i64 %x.extract.shift to i162447/// ==>2448///2449/// BB2:2450///   %x.extract.shift.1 = lshr i64 %arg1, 322451///   %x.extract.trunc = trunc i64 %x.extract.shift.1 to i162452///2453/// CodeGen will recognize the pattern in BB2 and generate BitExtract2454/// instruction.2455/// Return true if any changes are made.2456static bool OptimizeExtractBits(BinaryOperator *ShiftI, ConstantInt *CI,2457                                const TargetLowering &TLI,2458                                const DataLayout &DL) {2459  BasicBlock *DefBB = ShiftI->getParent();2460 2461  /// Only insert instructions in each block once.2462  DenseMap<BasicBlock *, BinaryOperator *> InsertedShifts;2463 2464  bool shiftIsLegal = TLI.isTypeLegal(TLI.getValueType(DL, ShiftI->getType()));2465 2466  bool MadeChange = false;2467  for (Value::user_iterator UI = ShiftI->user_begin(), E = ShiftI->user_end();2468       UI != E;) {2469    Use &TheUse = UI.getUse();2470    Instruction *User = cast<Instruction>(*UI);2471    // Preincrement use iterator so we don't invalidate it.2472    ++UI;2473 2474    // Don't bother for PHI nodes.2475    if (isa<PHINode>(User))2476      continue;2477 2478    if (!isExtractBitsCandidateUse(User))2479      continue;2480 2481    BasicBlock *UserBB = User->getParent();2482 2483    if (UserBB == DefBB) {2484      // If the shift and truncate instruction are in the same BB. The use of2485      // the truncate(TruncUse) may still introduce another truncate if not2486      // legal. In this case, we would like to sink both shift and truncate2487      // instruction to the BB of TruncUse.2488      // for example:2489      // BB1:2490      // i64 shift.result = lshr i64 opnd, imm2491      // trunc.result = trunc shift.result to i162492      //2493      // BB2:2494      //   ----> We will have an implicit truncate here if the architecture does2495      //   not have i16 compare.2496      // cmp i16 trunc.result, opnd22497      //2498      if (isa<TruncInst>(User) &&2499          shiftIsLegal2500          // If the type of the truncate is legal, no truncate will be2501          // introduced in other basic blocks.2502          && (!TLI.isTypeLegal(TLI.getValueType(DL, User->getType()))))2503        MadeChange =2504            SinkShiftAndTruncate(ShiftI, User, CI, InsertedShifts, TLI, DL);2505 2506      continue;2507    }2508    // If we have already inserted a shift into this block, use it.2509    BinaryOperator *&InsertedShift = InsertedShifts[UserBB];2510 2511    if (!InsertedShift) {2512      BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();2513      assert(InsertPt != UserBB->end());2514 2515      if (ShiftI->getOpcode() == Instruction::AShr)2516        InsertedShift =2517            BinaryOperator::CreateAShr(ShiftI->getOperand(0), CI, "");2518      else2519        InsertedShift =2520            BinaryOperator::CreateLShr(ShiftI->getOperand(0), CI, "");2521      InsertedShift->insertBefore(*UserBB, InsertPt);2522      InsertedShift->setDebugLoc(ShiftI->getDebugLoc());2523 2524      MadeChange = true;2525    }2526 2527    // Replace a use of the shift with a use of the new shift.2528    TheUse = InsertedShift;2529  }2530 2531  // If we removed all uses, or there are none, nuke the shift.2532  if (ShiftI->use_empty()) {2533    salvageDebugInfo(*ShiftI);2534    ShiftI->eraseFromParent();2535    MadeChange = true;2536  }2537 2538  return MadeChange;2539}2540 2541/// If counting leading or trailing zeros is an expensive operation and a zero2542/// input is defined, add a check for zero to avoid calling the intrinsic.2543///2544/// We want to transform:2545///     %z = call i64 @llvm.cttz.i64(i64 %A, i1 false)2546///2547/// into:2548///   entry:2549///     %cmpz = icmp eq i64 %A, 02550///     br i1 %cmpz, label %cond.end, label %cond.false2551///   cond.false:2552///     %z = call i64 @llvm.cttz.i64(i64 %A, i1 true)2553///     br label %cond.end2554///   cond.end:2555///     %ctz = phi i64 [ 64, %entry ], [ %z, %cond.false ]2556///2557/// If the transform is performed, return true and set ModifiedDT to true.2558static bool despeculateCountZeros(IntrinsicInst *CountZeros, LoopInfo &LI,2559                                  const TargetLowering *TLI,2560                                  const DataLayout *DL, ModifyDT &ModifiedDT,2561                                  SmallPtrSet<BasicBlock *, 32> &FreshBBs,2562                                  bool IsHugeFunc) {2563  // If a zero input is undefined, it doesn't make sense to despeculate that.2564  if (match(CountZeros->getOperand(1), m_One()))2565    return false;2566 2567  // If it's cheap to speculate, there's nothing to do.2568  Type *Ty = CountZeros->getType();2569  auto IntrinsicID = CountZeros->getIntrinsicID();2570  if ((IntrinsicID == Intrinsic::cttz && TLI->isCheapToSpeculateCttz(Ty)) ||2571      (IntrinsicID == Intrinsic::ctlz && TLI->isCheapToSpeculateCtlz(Ty)))2572    return false;2573 2574  // Only handle scalar cases. Anything else requires too much work.2575  unsigned SizeInBits = Ty->getScalarSizeInBits();2576  if (Ty->isVectorTy())2577    return false;2578 2579  // Bail if the value is never zero.2580  Use &Op = CountZeros->getOperandUse(0);2581  if (isKnownNonZero(Op, *DL))2582    return false;2583 2584  // The intrinsic will be sunk behind a compare against zero and branch.2585  BasicBlock *StartBlock = CountZeros->getParent();2586  BasicBlock *CallBlock = StartBlock->splitBasicBlock(CountZeros, "cond.false");2587  if (IsHugeFunc)2588    FreshBBs.insert(CallBlock);2589 2590  // Create another block after the count zero intrinsic. A PHI will be added2591  // in this block to select the result of the intrinsic or the bit-width2592  // constant if the input to the intrinsic is zero.2593  BasicBlock::iterator SplitPt = std::next(BasicBlock::iterator(CountZeros));2594  // Any debug-info after CountZeros should not be included.2595  SplitPt.setHeadBit(true);2596  BasicBlock *EndBlock = CallBlock->splitBasicBlock(SplitPt, "cond.end");2597  if (IsHugeFunc)2598    FreshBBs.insert(EndBlock);2599 2600  // Update the LoopInfo. The new blocks are in the same loop as the start2601  // block.2602  if (Loop *L = LI.getLoopFor(StartBlock)) {2603    L->addBasicBlockToLoop(CallBlock, LI);2604    L->addBasicBlockToLoop(EndBlock, LI);2605  }2606 2607  // Set up a builder to create a compare, conditional branch, and PHI.2608  IRBuilder<> Builder(CountZeros->getContext());2609  Builder.SetInsertPoint(StartBlock->getTerminator());2610  Builder.SetCurrentDebugLocation(CountZeros->getDebugLoc());2611 2612  // Replace the unconditional branch that was created by the first split with2613  // a compare against zero and a conditional branch.2614  Value *Zero = Constant::getNullValue(Ty);2615  // Avoid introducing branch on poison. This also replaces the ctz operand.2616  if (!isGuaranteedNotToBeUndefOrPoison(Op))2617    Op = Builder.CreateFreeze(Op, Op->getName() + ".fr");2618  Value *Cmp = Builder.CreateICmpEQ(Op, Zero, "cmpz");2619  Builder.CreateCondBr(Cmp, EndBlock, CallBlock);2620  StartBlock->getTerminator()->eraseFromParent();2621 2622  // Create a PHI in the end block to select either the output of the intrinsic2623  // or the bit width of the operand.2624  Builder.SetInsertPoint(EndBlock, EndBlock->begin());2625  PHINode *PN = Builder.CreatePHI(Ty, 2, "ctz");2626  replaceAllUsesWith(CountZeros, PN, FreshBBs, IsHugeFunc);2627  Value *BitWidth = Builder.getInt(APInt(SizeInBits, SizeInBits));2628  PN->addIncoming(BitWidth, StartBlock);2629  PN->addIncoming(CountZeros, CallBlock);2630 2631  // We are explicitly handling the zero case, so we can set the intrinsic's2632  // undefined zero argument to 'true'. This will also prevent reprocessing the2633  // intrinsic; we only despeculate when a zero input is defined.2634  CountZeros->setArgOperand(1, Builder.getTrue());2635  ModifiedDT = ModifyDT::ModifyBBDT;2636  return true;2637}2638 2639bool CodeGenPrepare::optimizeCallInst(CallInst *CI, ModifyDT &ModifiedDT) {2640  BasicBlock *BB = CI->getParent();2641 2642  // Sink address computing for memory operands into the block.2643  if (CI->isInlineAsm() && optimizeInlineAsmInst(CI))2644    return true;2645 2646  // Align the pointer arguments to this call if the target thinks it's a good2647  // idea2648  unsigned MinSize;2649  Align PrefAlign;2650  if (TLI->shouldAlignPointerArgs(CI, MinSize, PrefAlign)) {2651    for (auto &Arg : CI->args()) {2652      // We want to align both objects whose address is used directly and2653      // objects whose address is used in casts and GEPs, though it only makes2654      // sense for GEPs if the offset is a multiple of the desired alignment and2655      // if size - offset meets the size threshold.2656      if (!Arg->getType()->isPointerTy())2657        continue;2658      APInt Offset(DL->getIndexSizeInBits(2659                       cast<PointerType>(Arg->getType())->getAddressSpace()),2660                   0);2661      Value *Val = Arg->stripAndAccumulateInBoundsConstantOffsets(*DL, Offset);2662      uint64_t Offset2 = Offset.getLimitedValue();2663      if (!isAligned(PrefAlign, Offset2))2664        continue;2665      AllocaInst *AI;2666      if ((AI = dyn_cast<AllocaInst>(Val)) && AI->getAlign() < PrefAlign &&2667          DL->getTypeAllocSize(AI->getAllocatedType()) >= MinSize + Offset2)2668        AI->setAlignment(PrefAlign);2669      // Global variables can only be aligned if they are defined in this2670      // object (i.e. they are uniquely initialized in this object), and2671      // over-aligning global variables that have an explicit section is2672      // forbidden.2673      GlobalVariable *GV;2674      if ((GV = dyn_cast<GlobalVariable>(Val)) && GV->canIncreaseAlignment() &&2675          GV->getPointerAlignment(*DL) < PrefAlign &&2676          DL->getTypeAllocSize(GV->getValueType()) >= MinSize + Offset2)2677        GV->setAlignment(PrefAlign);2678    }2679  }2680  // If this is a memcpy (or similar) then we may be able to improve the2681  // alignment.2682  if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(CI)) {2683    Align DestAlign = getKnownAlignment(MI->getDest(), *DL);2684    MaybeAlign MIDestAlign = MI->getDestAlign();2685    if (!MIDestAlign || DestAlign > *MIDestAlign)2686      MI->setDestAlignment(DestAlign);2687    if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(MI)) {2688      MaybeAlign MTISrcAlign = MTI->getSourceAlign();2689      Align SrcAlign = getKnownAlignment(MTI->getSource(), *DL);2690      if (!MTISrcAlign || SrcAlign > *MTISrcAlign)2691        MTI->setSourceAlignment(SrcAlign);2692    }2693  }2694 2695  // If we have a cold call site, try to sink addressing computation into the2696  // cold block.  This interacts with our handling for loads and stores to2697  // ensure that we can fold all uses of a potential addressing computation2698  // into their uses.  TODO: generalize this to work over profiling data2699  if (CI->hasFnAttr(Attribute::Cold) &&2700      !llvm::shouldOptimizeForSize(BB, PSI, BFI.get()))2701    for (auto &Arg : CI->args()) {2702      if (!Arg->getType()->isPointerTy())2703        continue;2704      unsigned AS = Arg->getType()->getPointerAddressSpace();2705      if (optimizeMemoryInst(CI, Arg, Arg->getType(), AS))2706        return true;2707    }2708 2709  IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI);2710  if (II) {2711    switch (II->getIntrinsicID()) {2712    default:2713      break;2714    case Intrinsic::assume:2715      llvm_unreachable("llvm.assume should have been removed already");2716    case Intrinsic::allow_runtime_check:2717    case Intrinsic::allow_ubsan_check:2718    case Intrinsic::experimental_widenable_condition: {2719      // Give up on future widening opportunities so that we can fold away dead2720      // paths and merge blocks before going into block-local instruction2721      // selection.2722      if (II->use_empty()) {2723        II->eraseFromParent();2724        return true;2725      }2726      Constant *RetVal = ConstantInt::getTrue(II->getContext());2727      resetIteratorIfInvalidatedWhileCalling(BB, [&]() {2728        replaceAndRecursivelySimplify(CI, RetVal, TLInfo, nullptr);2729      });2730      return true;2731    }2732    case Intrinsic::objectsize:2733      llvm_unreachable("llvm.objectsize.* should have been lowered already");2734    case Intrinsic::is_constant:2735      llvm_unreachable("llvm.is.constant.* should have been lowered already");2736    case Intrinsic::aarch64_stlxr:2737    case Intrinsic::aarch64_stxr: {2738      ZExtInst *ExtVal = dyn_cast<ZExtInst>(CI->getArgOperand(0));2739      if (!ExtVal || !ExtVal->hasOneUse() ||2740          ExtVal->getParent() == CI->getParent())2741        return false;2742      // Sink a zext feeding stlxr/stxr before it, so it can be folded into it.2743      ExtVal->moveBefore(CI->getIterator());2744      // Mark this instruction as "inserted by CGP", so that other2745      // optimizations don't touch it.2746      InsertedInsts.insert(ExtVal);2747      return true;2748    }2749 2750    case Intrinsic::launder_invariant_group:2751    case Intrinsic::strip_invariant_group: {2752      Value *ArgVal = II->getArgOperand(0);2753      auto it = LargeOffsetGEPMap.find(II);2754      if (it != LargeOffsetGEPMap.end()) {2755        // Merge entries in LargeOffsetGEPMap to reflect the RAUW.2756        // Make sure not to have to deal with iterator invalidation2757        // after possibly adding ArgVal to LargeOffsetGEPMap.2758        auto GEPs = std::move(it->second);2759        LargeOffsetGEPMap[ArgVal].append(GEPs.begin(), GEPs.end());2760        LargeOffsetGEPMap.erase(II);2761      }2762 2763      replaceAllUsesWith(II, ArgVal, FreshBBs, IsHugeFunc);2764      II->eraseFromParent();2765      return true;2766    }2767    case Intrinsic::cttz:2768    case Intrinsic::ctlz:2769      // If counting zeros is expensive, try to avoid it.2770      return despeculateCountZeros(II, *LI, TLI, DL, ModifiedDT, FreshBBs,2771                                   IsHugeFunc);2772    case Intrinsic::fshl:2773    case Intrinsic::fshr:2774      return optimizeFunnelShift(II);2775    case Intrinsic::masked_gather:2776      return optimizeGatherScatterInst(II, II->getArgOperand(0));2777    case Intrinsic::masked_scatter:2778      return optimizeGatherScatterInst(II, II->getArgOperand(1));2779    case Intrinsic::masked_load:2780      // Treat v1X masked load as load X type.2781      if (auto *VT = dyn_cast<FixedVectorType>(II->getType())) {2782        if (VT->getNumElements() == 1) {2783          Value *PtrVal = II->getArgOperand(0);2784          unsigned AS = PtrVal->getType()->getPointerAddressSpace();2785          if (optimizeMemoryInst(II, PtrVal, VT->getElementType(), AS))2786            return true;2787        }2788      }2789      return false;2790    case Intrinsic::masked_store:2791      // Treat v1X masked store as store X type.2792      if (auto *VT =2793              dyn_cast<FixedVectorType>(II->getArgOperand(0)->getType())) {2794        if (VT->getNumElements() == 1) {2795          Value *PtrVal = II->getArgOperand(1);2796          unsigned AS = PtrVal->getType()->getPointerAddressSpace();2797          if (optimizeMemoryInst(II, PtrVal, VT->getElementType(), AS))2798            return true;2799        }2800      }2801      return false;2802    case Intrinsic::umul_with_overflow:2803      return optimizeMulWithOverflow(II, /*IsSigned=*/false, ModifiedDT);2804    case Intrinsic::smul_with_overflow:2805      return optimizeMulWithOverflow(II, /*IsSigned=*/true, ModifiedDT);2806    }2807 2808    SmallVector<Value *, 2> PtrOps;2809    Type *AccessTy;2810    if (TLI->getAddrModeArguments(II, PtrOps, AccessTy))2811      while (!PtrOps.empty()) {2812        Value *PtrVal = PtrOps.pop_back_val();2813        unsigned AS = PtrVal->getType()->getPointerAddressSpace();2814        if (optimizeMemoryInst(II, PtrVal, AccessTy, AS))2815          return true;2816      }2817  }2818 2819  // From here on out we're working with named functions.2820  auto *Callee = CI->getCalledFunction();2821  if (!Callee)2822    return false;2823 2824  // Lower all default uses of _chk calls.  This is very similar2825  // to what InstCombineCalls does, but here we are only lowering calls2826  // to fortified library functions (e.g. __memcpy_chk) that have the default2827  // "don't know" as the objectsize.  Anything else should be left alone.2828  FortifiedLibCallSimplifier Simplifier(TLInfo, true);2829  IRBuilder<> Builder(CI);2830  if (Value *V = Simplifier.optimizeCall(CI, Builder)) {2831    replaceAllUsesWith(CI, V, FreshBBs, IsHugeFunc);2832    CI->eraseFromParent();2833    return true;2834  }2835 2836  // SCCP may have propagated, among other things, C++ static variables across2837  // calls. If this happens to be the case, we may want to undo it in order to2838  // avoid redundant pointer computation of the constant, as the function method2839  // returning the constant needs to be executed anyways.2840  auto GetUniformReturnValue = [](const Function *F) -> GlobalVariable * {2841    if (!F->getReturnType()->isPointerTy())2842      return nullptr;2843 2844    GlobalVariable *UniformValue = nullptr;2845    for (auto &BB : *F) {2846      if (auto *RI = dyn_cast<ReturnInst>(BB.getTerminator())) {2847        if (auto *V = dyn_cast<GlobalVariable>(RI->getReturnValue())) {2848          if (!UniformValue)2849            UniformValue = V;2850          else if (V != UniformValue)2851            return nullptr;2852        } else {2853          return nullptr;2854        }2855      }2856    }2857 2858    return UniformValue;2859  };2860 2861  if (Callee->hasExactDefinition()) {2862    if (GlobalVariable *RV = GetUniformReturnValue(Callee)) {2863      bool MadeChange = false;2864      for (Use &U : make_early_inc_range(RV->uses())) {2865        auto *I = dyn_cast<Instruction>(U.getUser());2866        if (!I || I->getParent() != CI->getParent()) {2867          // Limit to the same basic block to avoid extending the call-site live2868          // range, which otherwise could increase register pressure.2869          continue;2870        }2871        if (CI->comesBefore(I)) {2872          U.set(CI);2873          MadeChange = true;2874        }2875      }2876 2877      return MadeChange;2878    }2879  }2880 2881  return false;2882}2883 2884static bool isIntrinsicOrLFToBeTailCalled(const TargetLibraryInfo *TLInfo,2885                                          const CallInst *CI) {2886  assert(CI && CI->use_empty());2887 2888  if (const auto *II = dyn_cast<IntrinsicInst>(CI))2889    switch (II->getIntrinsicID()) {2890    case Intrinsic::memset:2891    case Intrinsic::memcpy:2892    case Intrinsic::memmove:2893      return true;2894    default:2895      return false;2896    }2897 2898  LibFunc LF;2899  Function *Callee = CI->getCalledFunction();2900  if (Callee && TLInfo && TLInfo->getLibFunc(*Callee, LF))2901    switch (LF) {2902    case LibFunc_strcpy:2903    case LibFunc_strncpy:2904    case LibFunc_strcat:2905    case LibFunc_strncat:2906      return true;2907    default:2908      return false;2909    }2910 2911  return false;2912}2913 2914/// Look for opportunities to duplicate return instructions to the predecessor2915/// to enable tail call optimizations. The case it is currently looking for is2916/// the following one. Known intrinsics or library function that may be tail2917/// called are taken into account as well.2918/// @code2919/// bb0:2920///   %tmp0 = tail call i32 @f0()2921///   br label %return2922/// bb1:2923///   %tmp1 = tail call i32 @f1()2924///   br label %return2925/// bb2:2926///   %tmp2 = tail call i32 @f2()2927///   br label %return2928/// return:2929///   %retval = phi i32 [ %tmp0, %bb0 ], [ %tmp1, %bb1 ], [ %tmp2, %bb2 ]2930///   ret i32 %retval2931/// @endcode2932///2933/// =>2934///2935/// @code2936/// bb0:2937///   %tmp0 = tail call i32 @f0()2938///   ret i32 %tmp02939/// bb1:2940///   %tmp1 = tail call i32 @f1()2941///   ret i32 %tmp12942/// bb2:2943///   %tmp2 = tail call i32 @f2()2944///   ret i32 %tmp22945/// @endcode2946bool CodeGenPrepare::dupRetToEnableTailCallOpts(BasicBlock *BB,2947                                                ModifyDT &ModifiedDT) {2948  if (!BB->getTerminator())2949    return false;2950 2951  ReturnInst *RetI = dyn_cast<ReturnInst>(BB->getTerminator());2952  if (!RetI)2953    return false;2954 2955  assert(LI->getLoopFor(BB) == nullptr && "A return block cannot be in a loop");2956 2957  PHINode *PN = nullptr;2958  ExtractValueInst *EVI = nullptr;2959  BitCastInst *BCI = nullptr;2960  Value *V = RetI->getReturnValue();2961  if (V) {2962    BCI = dyn_cast<BitCastInst>(V);2963    if (BCI)2964      V = BCI->getOperand(0);2965 2966    EVI = dyn_cast<ExtractValueInst>(V);2967    if (EVI) {2968      V = EVI->getOperand(0);2969      if (!llvm::all_of(EVI->indices(), [](unsigned idx) { return idx == 0; }))2970        return false;2971    }2972 2973    PN = dyn_cast<PHINode>(V);2974  }2975 2976  if (PN && PN->getParent() != BB)2977    return false;2978 2979  auto isLifetimeEndOrBitCastFor = [](const Instruction *Inst) {2980    const BitCastInst *BC = dyn_cast<BitCastInst>(Inst);2981    if (BC && BC->hasOneUse())2982      Inst = BC->user_back();2983 2984    if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst))2985      return II->getIntrinsicID() == Intrinsic::lifetime_end;2986    return false;2987  };2988 2989  SmallVector<const IntrinsicInst *, 4> FakeUses;2990 2991  auto isFakeUse = [&FakeUses](const Instruction *Inst) {2992    if (auto *II = dyn_cast<IntrinsicInst>(Inst);2993        II && II->getIntrinsicID() == Intrinsic::fake_use) {2994      // Record the instruction so it can be preserved when the exit block is2995      // removed. Do not preserve the fake use that uses the result of the2996      // PHI instruction.2997      // Do not copy fake uses that use the result of a PHI node.2998      // FIXME: If we do want to copy the fake use into the return blocks, we2999      // have to figure out which of the PHI node operands to use for each3000      // copy.3001      if (!isa<PHINode>(II->getOperand(0))) {3002        FakeUses.push_back(II);3003      }3004      return true;3005    }3006 3007    return false;3008  };3009 3010  // Make sure there are no instructions between the first instruction3011  // and return.3012  BasicBlock::const_iterator BI = BB->getFirstNonPHIIt();3013  // Skip over pseudo-probes and the bitcast.3014  while (&*BI == BCI || &*BI == EVI || isa<PseudoProbeInst>(BI) ||3015         isLifetimeEndOrBitCastFor(&*BI) || isFakeUse(&*BI))3016    BI = std::next(BI);3017  if (&*BI != RetI)3018    return false;3019 3020  /// Only dup the ReturnInst if the CallInst is likely to be emitted as a tail3021  /// call.3022  const Function *F = BB->getParent();3023  SmallVector<BasicBlock *, 4> TailCallBBs;3024  // Record the call instructions so we can insert any fake uses3025  // that need to be preserved before them.3026  SmallVector<CallInst *, 4> CallInsts;3027  if (PN) {3028    for (unsigned I = 0, E = PN->getNumIncomingValues(); I != E; ++I) {3029      // Look through bitcasts.3030      Value *IncomingVal = PN->getIncomingValue(I)->stripPointerCasts();3031      CallInst *CI = dyn_cast<CallInst>(IncomingVal);3032      BasicBlock *PredBB = PN->getIncomingBlock(I);3033      // Make sure the phi value is indeed produced by the tail call.3034      if (CI && CI->hasOneUse() && CI->getParent() == PredBB &&3035          TLI->mayBeEmittedAsTailCall(CI) &&3036          attributesPermitTailCall(F, CI, RetI, *TLI)) {3037        TailCallBBs.push_back(PredBB);3038        CallInsts.push_back(CI);3039      } else {3040        // Consider the cases in which the phi value is indirectly produced by3041        // the tail call, for example when encountering memset(), memmove(),3042        // strcpy(), whose return value may have been optimized out. In such3043        // cases, the value needs to be the first function argument.3044        //3045        // bb0:3046        //   tail call void @llvm.memset.p0.i64(ptr %0, i8 0, i64 %1)3047        //   br label %return3048        // return:3049        //   %phi = phi ptr [ %0, %bb0 ], [ %2, %entry ]3050        if (PredBB && PredBB->getSingleSuccessor() == BB)3051          CI = dyn_cast_or_null<CallInst>(3052              PredBB->getTerminator()->getPrevNode());3053 3054        if (CI && CI->use_empty() &&3055            isIntrinsicOrLFToBeTailCalled(TLInfo, CI) &&3056            IncomingVal == CI->getArgOperand(0) &&3057            TLI->mayBeEmittedAsTailCall(CI) &&3058            attributesPermitTailCall(F, CI, RetI, *TLI)) {3059          TailCallBBs.push_back(PredBB);3060          CallInsts.push_back(CI);3061        }3062      }3063    }3064  } else {3065    SmallPtrSet<BasicBlock *, 4> VisitedBBs;3066    for (BasicBlock *Pred : predecessors(BB)) {3067      if (!VisitedBBs.insert(Pred).second)3068        continue;3069      if (Instruction *I = Pred->rbegin()->getPrevNode()) {3070        CallInst *CI = dyn_cast<CallInst>(I);3071        if (CI && CI->use_empty() && TLI->mayBeEmittedAsTailCall(CI) &&3072            attributesPermitTailCall(F, CI, RetI, *TLI)) {3073          // Either we return void or the return value must be the first3074          // argument of a known intrinsic or library function.3075          if (!V || isa<UndefValue>(V) ||3076              (isIntrinsicOrLFToBeTailCalled(TLInfo, CI) &&3077               V == CI->getArgOperand(0))) {3078            TailCallBBs.push_back(Pred);3079            CallInsts.push_back(CI);3080          }3081        }3082      }3083    }3084  }3085 3086  bool Changed = false;3087  for (auto const &TailCallBB : TailCallBBs) {3088    // Make sure the call instruction is followed by an unconditional branch to3089    // the return block.3090    BranchInst *BI = dyn_cast<BranchInst>(TailCallBB->getTerminator());3091    if (!BI || !BI->isUnconditional() || BI->getSuccessor(0) != BB)3092      continue;3093 3094    // Duplicate the return into TailCallBB.3095    (void)FoldReturnIntoUncondBranch(RetI, BB, TailCallBB);3096    assert(!VerifyBFIUpdates ||3097           BFI->getBlockFreq(BB) >= BFI->getBlockFreq(TailCallBB));3098    BFI->setBlockFreq(BB,3099                      (BFI->getBlockFreq(BB) - BFI->getBlockFreq(TailCallBB)));3100    ModifiedDT = ModifyDT::ModifyBBDT;3101    Changed = true;3102    ++NumRetsDup;3103  }3104 3105  // If we eliminated all predecessors of the block, delete the block now.3106  if (Changed && !BB->hasAddressTaken() && pred_empty(BB)) {3107    // Copy the fake uses found in the original return block to all blocks3108    // that contain tail calls.3109    for (auto *CI : CallInsts) {3110      for (auto const *FakeUse : FakeUses) {3111        auto *ClonedInst = FakeUse->clone();3112        ClonedInst->insertBefore(CI->getIterator());3113      }3114    }3115    BB->eraseFromParent();3116  }3117 3118  return Changed;3119}3120 3121//===----------------------------------------------------------------------===//3122// Memory Optimization3123//===----------------------------------------------------------------------===//3124 3125namespace {3126 3127/// This is an extended version of TargetLowering::AddrMode3128/// which holds actual Value*'s for register values.3129struct ExtAddrMode : public TargetLowering::AddrMode {3130  Value *BaseReg = nullptr;3131  Value *ScaledReg = nullptr;3132  Value *OriginalValue = nullptr;3133  bool InBounds = true;3134 3135  enum FieldName {3136    NoField = 0x00,3137    BaseRegField = 0x01,3138    BaseGVField = 0x02,3139    BaseOffsField = 0x04,3140    ScaledRegField = 0x08,3141    ScaleField = 0x10,3142    MultipleFields = 0xff3143  };3144 3145  ExtAddrMode() = default;3146 3147  void print(raw_ostream &OS) const;3148  void dump() const;3149 3150  // Replace From in ExtAddrMode with To.3151  // E.g., SExt insts may be promoted and deleted. We should replace them with3152  // the promoted values.3153  void replaceWith(Value *From, Value *To) {3154    if (ScaledReg == From)3155      ScaledReg = To;3156  }3157 3158  FieldName compare(const ExtAddrMode &other) {3159    // First check that the types are the same on each field, as differing types3160    // is something we can't cope with later on.3161    if (BaseReg && other.BaseReg &&3162        BaseReg->getType() != other.BaseReg->getType())3163      return MultipleFields;3164    if (BaseGV && other.BaseGV && BaseGV->getType() != other.BaseGV->getType())3165      return MultipleFields;3166    if (ScaledReg && other.ScaledReg &&3167        ScaledReg->getType() != other.ScaledReg->getType())3168      return MultipleFields;3169 3170    // Conservatively reject 'inbounds' mismatches.3171    if (InBounds != other.InBounds)3172      return MultipleFields;3173 3174    // Check each field to see if it differs.3175    unsigned Result = NoField;3176    if (BaseReg != other.BaseReg)3177      Result |= BaseRegField;3178    if (BaseGV != other.BaseGV)3179      Result |= BaseGVField;3180    if (BaseOffs != other.BaseOffs)3181      Result |= BaseOffsField;3182    if (ScaledReg != other.ScaledReg)3183      Result |= ScaledRegField;3184    // Don't count 0 as being a different scale, because that actually means3185    // unscaled (which will already be counted by having no ScaledReg).3186    if (Scale && other.Scale && Scale != other.Scale)3187      Result |= ScaleField;3188 3189    if (llvm::popcount(Result) > 1)3190      return MultipleFields;3191    else3192      return static_cast<FieldName>(Result);3193  }3194 3195  // An AddrMode is trivial if it involves no calculation i.e. it is just a base3196  // with no offset.3197  bool isTrivial() {3198    // An AddrMode is (BaseGV + BaseReg + BaseOffs + ScaleReg * Scale) so it is3199    // trivial if at most one of these terms is nonzero, except that BaseGV and3200    // BaseReg both being zero actually means a null pointer value, which we3201    // consider to be 'non-zero' here.3202    return !BaseOffs && !Scale && !(BaseGV && BaseReg);3203  }3204 3205  Value *GetFieldAsValue(FieldName Field, Type *IntPtrTy) {3206    switch (Field) {3207    default:3208      return nullptr;3209    case BaseRegField:3210      return BaseReg;3211    case BaseGVField:3212      return BaseGV;3213    case ScaledRegField:3214      return ScaledReg;3215    case BaseOffsField:3216      return ConstantInt::getSigned(IntPtrTy, BaseOffs);3217    }3218  }3219 3220  void SetCombinedField(FieldName Field, Value *V,3221                        const SmallVectorImpl<ExtAddrMode> &AddrModes) {3222    switch (Field) {3223    default:3224      llvm_unreachable("Unhandled fields are expected to be rejected earlier");3225      break;3226    case ExtAddrMode::BaseRegField:3227      BaseReg = V;3228      break;3229    case ExtAddrMode::BaseGVField:3230      // A combined BaseGV is an Instruction, not a GlobalValue, so it goes3231      // in the BaseReg field.3232      assert(BaseReg == nullptr);3233      BaseReg = V;3234      BaseGV = nullptr;3235      break;3236    case ExtAddrMode::ScaledRegField:3237      ScaledReg = V;3238      // If we have a mix of scaled and unscaled addrmodes then we want scale3239      // to be the scale and not zero.3240      if (!Scale)3241        for (const ExtAddrMode &AM : AddrModes)3242          if (AM.Scale) {3243            Scale = AM.Scale;3244            break;3245          }3246      break;3247    case ExtAddrMode::BaseOffsField:3248      // The offset is no longer a constant, so it goes in ScaledReg with a3249      // scale of 1.3250      assert(ScaledReg == nullptr);3251      ScaledReg = V;3252      Scale = 1;3253      BaseOffs = 0;3254      break;3255    }3256  }3257};3258 3259#ifndef NDEBUG3260static inline raw_ostream &operator<<(raw_ostream &OS, const ExtAddrMode &AM) {3261  AM.print(OS);3262  return OS;3263}3264#endif3265 3266#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)3267void ExtAddrMode::print(raw_ostream &OS) const {3268  bool NeedPlus = false;3269  OS << "[";3270  if (InBounds)3271    OS << "inbounds ";3272  if (BaseGV) {3273    OS << "GV:";3274    BaseGV->printAsOperand(OS, /*PrintType=*/false);3275    NeedPlus = true;3276  }3277 3278  if (BaseOffs) {3279    OS << (NeedPlus ? " + " : "") << BaseOffs;3280    NeedPlus = true;3281  }3282 3283  if (BaseReg) {3284    OS << (NeedPlus ? " + " : "") << "Base:";3285    BaseReg->printAsOperand(OS, /*PrintType=*/false);3286    NeedPlus = true;3287  }3288  if (Scale) {3289    OS << (NeedPlus ? " + " : "") << Scale << "*";3290    ScaledReg->printAsOperand(OS, /*PrintType=*/false);3291  }3292 3293  OS << ']';3294}3295 3296LLVM_DUMP_METHOD void ExtAddrMode::dump() const {3297  print(dbgs());3298  dbgs() << '\n';3299}3300#endif3301 3302} // end anonymous namespace3303 3304namespace {3305 3306/// This class provides transaction based operation on the IR.3307/// Every change made through this class is recorded in the internal state and3308/// can be undone (rollback) until commit is called.3309/// CGP does not check if instructions could be speculatively executed when3310/// moved. Preserving the original location would pessimize the debugging3311/// experience, as well as negatively impact the quality of sample PGO.3312class TypePromotionTransaction {3313  /// This represents the common interface of the individual transaction.3314  /// Each class implements the logic for doing one specific modification on3315  /// the IR via the TypePromotionTransaction.3316  class TypePromotionAction {3317  protected:3318    /// The Instruction modified.3319    Instruction *Inst;3320 3321  public:3322    /// Constructor of the action.3323    /// The constructor performs the related action on the IR.3324    TypePromotionAction(Instruction *Inst) : Inst(Inst) {}3325 3326    virtual ~TypePromotionAction() = default;3327 3328    /// Undo the modification done by this action.3329    /// When this method is called, the IR must be in the same state as it was3330    /// before this action was applied.3331    /// \pre Undoing the action works if and only if the IR is in the exact same3332    /// state as it was directly after this action was applied.3333    virtual void undo() = 0;3334 3335    /// Advocate every change made by this action.3336    /// When the results on the IR of the action are to be kept, it is important3337    /// to call this function, otherwise hidden information may be kept forever.3338    virtual void commit() {3339      // Nothing to be done, this action is not doing anything.3340    }3341  };3342 3343  /// Utility to remember the position of an instruction.3344  class InsertionHandler {3345    /// Position of an instruction.3346    /// Either an instruction:3347    /// - Is the first in a basic block: BB is used.3348    /// - Has a previous instruction: PrevInst is used.3349    struct {3350      BasicBlock::iterator PrevInst;3351      BasicBlock *BB;3352    } Point;3353    std::optional<DbgRecord::self_iterator> BeforeDbgRecord = std::nullopt;3354 3355    /// Remember whether or not the instruction had a previous instruction.3356    bool HasPrevInstruction;3357 3358  public:3359    /// Record the position of \p Inst.3360    InsertionHandler(Instruction *Inst) {3361      HasPrevInstruction = (Inst != &*(Inst->getParent()->begin()));3362      BasicBlock *BB = Inst->getParent();3363 3364      // Record where we would have to re-insert the instruction in the sequence3365      // of DbgRecords, if we ended up reinserting.3366      BeforeDbgRecord = Inst->getDbgReinsertionPosition();3367 3368      if (HasPrevInstruction) {3369        Point.PrevInst = std::prev(Inst->getIterator());3370      } else {3371        Point.BB = BB;3372      }3373    }3374 3375    /// Insert \p Inst at the recorded position.3376    void insert(Instruction *Inst) {3377      if (HasPrevInstruction) {3378        if (Inst->getParent())3379          Inst->removeFromParent();3380        Inst->insertAfter(Point.PrevInst);3381      } else {3382        BasicBlock::iterator Position = Point.BB->getFirstInsertionPt();3383        if (Inst->getParent())3384          Inst->moveBefore(*Point.BB, Position);3385        else3386          Inst->insertBefore(*Point.BB, Position);3387      }3388 3389      Inst->getParent()->reinsertInstInDbgRecords(Inst, BeforeDbgRecord);3390    }3391  };3392 3393  /// Move an instruction before another.3394  class InstructionMoveBefore : public TypePromotionAction {3395    /// Original position of the instruction.3396    InsertionHandler Position;3397 3398  public:3399    /// Move \p Inst before \p Before.3400    InstructionMoveBefore(Instruction *Inst, BasicBlock::iterator Before)3401        : TypePromotionAction(Inst), Position(Inst) {3402      LLVM_DEBUG(dbgs() << "Do: move: " << *Inst << "\nbefore: " << *Before3403                        << "\n");3404      Inst->moveBefore(Before);3405    }3406 3407    /// Move the instruction back to its original position.3408    void undo() override {3409      LLVM_DEBUG(dbgs() << "Undo: moveBefore: " << *Inst << "\n");3410      Position.insert(Inst);3411    }3412  };3413 3414  /// Set the operand of an instruction with a new value.3415  class OperandSetter : public TypePromotionAction {3416    /// Original operand of the instruction.3417    Value *Origin;3418 3419    /// Index of the modified instruction.3420    unsigned Idx;3421 3422  public:3423    /// Set \p Idx operand of \p Inst with \p NewVal.3424    OperandSetter(Instruction *Inst, unsigned Idx, Value *NewVal)3425        : TypePromotionAction(Inst), Idx(Idx) {3426      LLVM_DEBUG(dbgs() << "Do: setOperand: " << Idx << "\n"3427                        << "for:" << *Inst << "\n"3428                        << "with:" << *NewVal << "\n");3429      Origin = Inst->getOperand(Idx);3430      Inst->setOperand(Idx, NewVal);3431    }3432 3433    /// Restore the original value of the instruction.3434    void undo() override {3435      LLVM_DEBUG(dbgs() << "Undo: setOperand:" << Idx << "\n"3436                        << "for: " << *Inst << "\n"3437                        << "with: " << *Origin << "\n");3438      Inst->setOperand(Idx, Origin);3439    }3440  };3441 3442  /// Hide the operands of an instruction.3443  /// Do as if this instruction was not using any of its operands.3444  class OperandsHider : public TypePromotionAction {3445    /// The list of original operands.3446    SmallVector<Value *, 4> OriginalValues;3447 3448  public:3449    /// Remove \p Inst from the uses of the operands of \p Inst.3450    OperandsHider(Instruction *Inst) : TypePromotionAction(Inst) {3451      LLVM_DEBUG(dbgs() << "Do: OperandsHider: " << *Inst << "\n");3452      unsigned NumOpnds = Inst->getNumOperands();3453      OriginalValues.reserve(NumOpnds);3454      for (unsigned It = 0; It < NumOpnds; ++It) {3455        // Save the current operand.3456        Value *Val = Inst->getOperand(It);3457        OriginalValues.push_back(Val);3458        // Set a dummy one.3459        // We could use OperandSetter here, but that would imply an overhead3460        // that we are not willing to pay.3461        Inst->setOperand(It, PoisonValue::get(Val->getType()));3462      }3463    }3464 3465    /// Restore the original list of uses.3466    void undo() override {3467      LLVM_DEBUG(dbgs() << "Undo: OperandsHider: " << *Inst << "\n");3468      for (unsigned It = 0, EndIt = OriginalValues.size(); It != EndIt; ++It)3469        Inst->setOperand(It, OriginalValues[It]);3470    }3471  };3472 3473  /// Build a truncate instruction.3474  class TruncBuilder : public TypePromotionAction {3475    Value *Val;3476 3477  public:3478    /// Build a truncate instruction of \p Opnd producing a \p Ty3479    /// result.3480    /// trunc Opnd to Ty.3481    TruncBuilder(Instruction *Opnd, Type *Ty) : TypePromotionAction(Opnd) {3482      IRBuilder<> Builder(Opnd);3483      Builder.SetCurrentDebugLocation(DebugLoc());3484      Val = Builder.CreateTrunc(Opnd, Ty, "promoted");3485      LLVM_DEBUG(dbgs() << "Do: TruncBuilder: " << *Val << "\n");3486    }3487 3488    /// Get the built value.3489    Value *getBuiltValue() { return Val; }3490 3491    /// Remove the built instruction.3492    void undo() override {3493      LLVM_DEBUG(dbgs() << "Undo: TruncBuilder: " << *Val << "\n");3494      if (Instruction *IVal = dyn_cast<Instruction>(Val))3495        IVal->eraseFromParent();3496    }3497  };3498 3499  /// Build a sign extension instruction.3500  class SExtBuilder : public TypePromotionAction {3501    Value *Val;3502 3503  public:3504    /// Build a sign extension instruction of \p Opnd producing a \p Ty3505    /// result.3506    /// sext Opnd to Ty.3507    SExtBuilder(Instruction *InsertPt, Value *Opnd, Type *Ty)3508        : TypePromotionAction(InsertPt) {3509      IRBuilder<> Builder(InsertPt);3510      Val = Builder.CreateSExt(Opnd, Ty, "promoted");3511      LLVM_DEBUG(dbgs() << "Do: SExtBuilder: " << *Val << "\n");3512    }3513 3514    /// Get the built value.3515    Value *getBuiltValue() { return Val; }3516 3517    /// Remove the built instruction.3518    void undo() override {3519      LLVM_DEBUG(dbgs() << "Undo: SExtBuilder: " << *Val << "\n");3520      if (Instruction *IVal = dyn_cast<Instruction>(Val))3521        IVal->eraseFromParent();3522    }3523  };3524 3525  /// Build a zero extension instruction.3526  class ZExtBuilder : public TypePromotionAction {3527    Value *Val;3528 3529  public:3530    /// Build a zero extension instruction of \p Opnd producing a \p Ty3531    /// result.3532    /// zext Opnd to Ty.3533    ZExtBuilder(Instruction *InsertPt, Value *Opnd, Type *Ty)3534        : TypePromotionAction(InsertPt) {3535      IRBuilder<> Builder(InsertPt);3536      Builder.SetCurrentDebugLocation(DebugLoc());3537      Val = Builder.CreateZExt(Opnd, Ty, "promoted");3538      LLVM_DEBUG(dbgs() << "Do: ZExtBuilder: " << *Val << "\n");3539    }3540 3541    /// Get the built value.3542    Value *getBuiltValue() { return Val; }3543 3544    /// Remove the built instruction.3545    void undo() override {3546      LLVM_DEBUG(dbgs() << "Undo: ZExtBuilder: " << *Val << "\n");3547      if (Instruction *IVal = dyn_cast<Instruction>(Val))3548        IVal->eraseFromParent();3549    }3550  };3551 3552  /// Mutate an instruction to another type.3553  class TypeMutator : public TypePromotionAction {3554    /// Record the original type.3555    Type *OrigTy;3556 3557  public:3558    /// Mutate the type of \p Inst into \p NewTy.3559    TypeMutator(Instruction *Inst, Type *NewTy)3560        : TypePromotionAction(Inst), OrigTy(Inst->getType()) {3561      LLVM_DEBUG(dbgs() << "Do: MutateType: " << *Inst << " with " << *NewTy3562                        << "\n");3563      Inst->mutateType(NewTy);3564    }3565 3566    /// Mutate the instruction back to its original type.3567    void undo() override {3568      LLVM_DEBUG(dbgs() << "Undo: MutateType: " << *Inst << " with " << *OrigTy3569                        << "\n");3570      Inst->mutateType(OrigTy);3571    }3572  };3573 3574  /// Replace the uses of an instruction by another instruction.3575  class UsesReplacer : public TypePromotionAction {3576    /// Helper structure to keep track of the replaced uses.3577    struct InstructionAndIdx {3578      /// The instruction using the instruction.3579      Instruction *Inst;3580 3581      /// The index where this instruction is used for Inst.3582      unsigned Idx;3583 3584      InstructionAndIdx(Instruction *Inst, unsigned Idx)3585          : Inst(Inst), Idx(Idx) {}3586    };3587 3588    /// Keep track of the original uses (pair Instruction, Index).3589    SmallVector<InstructionAndIdx, 4> OriginalUses;3590    /// Keep track of the debug users.3591    SmallVector<DbgVariableRecord *, 1> DbgVariableRecords;3592 3593    /// Keep track of the new value so that we can undo it by replacing3594    /// instances of the new value with the original value.3595    Value *New;3596 3597    using use_iterator = SmallVectorImpl<InstructionAndIdx>::iterator;3598 3599  public:3600    /// Replace all the use of \p Inst by \p New.3601    UsesReplacer(Instruction *Inst, Value *New)3602        : TypePromotionAction(Inst), New(New) {3603      LLVM_DEBUG(dbgs() << "Do: UsersReplacer: " << *Inst << " with " << *New3604                        << "\n");3605      // Record the original uses.3606      for (Use &U : Inst->uses()) {3607        Instruction *UserI = cast<Instruction>(U.getUser());3608        OriginalUses.push_back(InstructionAndIdx(UserI, U.getOperandNo()));3609      }3610      // Record the debug uses separately. They are not in the instruction's3611      // use list, but they are replaced by RAUW.3612      findDbgValues(Inst, DbgVariableRecords);3613 3614      // Now, we can replace the uses.3615      Inst->replaceAllUsesWith(New);3616    }3617 3618    /// Reassign the original uses of Inst to Inst.3619    void undo() override {3620      LLVM_DEBUG(dbgs() << "Undo: UsersReplacer: " << *Inst << "\n");3621      for (InstructionAndIdx &Use : OriginalUses)3622        Use.Inst->setOperand(Use.Idx, Inst);3623      // RAUW has replaced all original uses with references to the new value,3624      // including the debug uses. Since we are undoing the replacements,3625      // the original debug uses must also be reinstated to maintain the3626      // correctness and utility of debug value records.3627      for (DbgVariableRecord *DVR : DbgVariableRecords)3628        DVR->replaceVariableLocationOp(New, Inst);3629    }3630  };3631 3632  /// Remove an instruction from the IR.3633  class InstructionRemover : public TypePromotionAction {3634    /// Original position of the instruction.3635    InsertionHandler Inserter;3636 3637    /// Helper structure to hide all the link to the instruction. In other3638    /// words, this helps to do as if the instruction was removed.3639    OperandsHider Hider;3640 3641    /// Keep track of the uses replaced, if any.3642    UsesReplacer *Replacer = nullptr;3643 3644    /// Keep track of instructions removed.3645    SetOfInstrs &RemovedInsts;3646 3647  public:3648    /// Remove all reference of \p Inst and optionally replace all its3649    /// uses with New.3650    /// \p RemovedInsts Keep track of the instructions removed by this Action.3651    /// \pre If !Inst->use_empty(), then New != nullptr3652    InstructionRemover(Instruction *Inst, SetOfInstrs &RemovedInsts,3653                       Value *New = nullptr)3654        : TypePromotionAction(Inst), Inserter(Inst), Hider(Inst),3655          RemovedInsts(RemovedInsts) {3656      if (New)3657        Replacer = new UsesReplacer(Inst, New);3658      LLVM_DEBUG(dbgs() << "Do: InstructionRemover: " << *Inst << "\n");3659      RemovedInsts.insert(Inst);3660      /// The instructions removed here will be freed after completing3661      /// optimizeBlock() for all blocks as we need to keep track of the3662      /// removed instructions during promotion.3663      Inst->removeFromParent();3664    }3665 3666    ~InstructionRemover() override { delete Replacer; }3667 3668    InstructionRemover &operator=(const InstructionRemover &other) = delete;3669    InstructionRemover(const InstructionRemover &other) = delete;3670 3671    /// Resurrect the instruction and reassign it to the proper uses if3672    /// new value was provided when build this action.3673    void undo() override {3674      LLVM_DEBUG(dbgs() << "Undo: InstructionRemover: " << *Inst << "\n");3675      Inserter.insert(Inst);3676      if (Replacer)3677        Replacer->undo();3678      Hider.undo();3679      RemovedInsts.erase(Inst);3680    }3681  };3682 3683public:3684  /// Restoration point.3685  /// The restoration point is a pointer to an action instead of an iterator3686  /// because the iterator may be invalidated but not the pointer.3687  using ConstRestorationPt = const TypePromotionAction *;3688 3689  TypePromotionTransaction(SetOfInstrs &RemovedInsts)3690      : RemovedInsts(RemovedInsts) {}3691 3692  /// Advocate every changes made in that transaction. Return true if any change3693  /// happen.3694  bool commit();3695 3696  /// Undo all the changes made after the given point.3697  void rollback(ConstRestorationPt Point);3698 3699  /// Get the current restoration point.3700  ConstRestorationPt getRestorationPoint() const;3701 3702  /// \name API for IR modification with state keeping to support rollback.3703  /// @{3704  /// Same as Instruction::setOperand.3705  void setOperand(Instruction *Inst, unsigned Idx, Value *NewVal);3706 3707  /// Same as Instruction::eraseFromParent.3708  void eraseInstruction(Instruction *Inst, Value *NewVal = nullptr);3709 3710  /// Same as Value::replaceAllUsesWith.3711  void replaceAllUsesWith(Instruction *Inst, Value *New);3712 3713  /// Same as Value::mutateType.3714  void mutateType(Instruction *Inst, Type *NewTy);3715 3716  /// Same as IRBuilder::createTrunc.3717  Value *createTrunc(Instruction *Opnd, Type *Ty);3718 3719  /// Same as IRBuilder::createSExt.3720  Value *createSExt(Instruction *Inst, Value *Opnd, Type *Ty);3721 3722  /// Same as IRBuilder::createZExt.3723  Value *createZExt(Instruction *Inst, Value *Opnd, Type *Ty);3724 3725private:3726  /// The ordered list of actions made so far.3727  SmallVector<std::unique_ptr<TypePromotionAction>, 16> Actions;3728 3729  using CommitPt =3730      SmallVectorImpl<std::unique_ptr<TypePromotionAction>>::iterator;3731 3732  SetOfInstrs &RemovedInsts;3733};3734 3735} // end anonymous namespace3736 3737void TypePromotionTransaction::setOperand(Instruction *Inst, unsigned Idx,3738                                          Value *NewVal) {3739  Actions.push_back(std::make_unique<TypePromotionTransaction::OperandSetter>(3740      Inst, Idx, NewVal));3741}3742 3743void TypePromotionTransaction::eraseInstruction(Instruction *Inst,3744                                                Value *NewVal) {3745  Actions.push_back(3746      std::make_unique<TypePromotionTransaction::InstructionRemover>(3747          Inst, RemovedInsts, NewVal));3748}3749 3750void TypePromotionTransaction::replaceAllUsesWith(Instruction *Inst,3751                                                  Value *New) {3752  Actions.push_back(3753      std::make_unique<TypePromotionTransaction::UsesReplacer>(Inst, New));3754}3755 3756void TypePromotionTransaction::mutateType(Instruction *Inst, Type *NewTy) {3757  Actions.push_back(3758      std::make_unique<TypePromotionTransaction::TypeMutator>(Inst, NewTy));3759}3760 3761Value *TypePromotionTransaction::createTrunc(Instruction *Opnd, Type *Ty) {3762  std::unique_ptr<TruncBuilder> Ptr(new TruncBuilder(Opnd, Ty));3763  Value *Val = Ptr->getBuiltValue();3764  Actions.push_back(std::move(Ptr));3765  return Val;3766}3767 3768Value *TypePromotionTransaction::createSExt(Instruction *Inst, Value *Opnd,3769                                            Type *Ty) {3770  std::unique_ptr<SExtBuilder> Ptr(new SExtBuilder(Inst, Opnd, Ty));3771  Value *Val = Ptr->getBuiltValue();3772  Actions.push_back(std::move(Ptr));3773  return Val;3774}3775 3776Value *TypePromotionTransaction::createZExt(Instruction *Inst, Value *Opnd,3777                                            Type *Ty) {3778  std::unique_ptr<ZExtBuilder> Ptr(new ZExtBuilder(Inst, Opnd, Ty));3779  Value *Val = Ptr->getBuiltValue();3780  Actions.push_back(std::move(Ptr));3781  return Val;3782}3783 3784TypePromotionTransaction::ConstRestorationPt3785TypePromotionTransaction::getRestorationPoint() const {3786  return !Actions.empty() ? Actions.back().get() : nullptr;3787}3788 3789bool TypePromotionTransaction::commit() {3790  for (std::unique_ptr<TypePromotionAction> &Action : Actions)3791    Action->commit();3792  bool Modified = !Actions.empty();3793  Actions.clear();3794  return Modified;3795}3796 3797void TypePromotionTransaction::rollback(3798    TypePromotionTransaction::ConstRestorationPt Point) {3799  while (!Actions.empty() && Point != Actions.back().get()) {3800    std::unique_ptr<TypePromotionAction> Curr = Actions.pop_back_val();3801    Curr->undo();3802  }3803}3804 3805namespace {3806 3807/// A helper class for matching addressing modes.3808///3809/// This encapsulates the logic for matching the target-legal addressing modes.3810class AddressingModeMatcher {3811  SmallVectorImpl<Instruction *> &AddrModeInsts;3812  const TargetLowering &TLI;3813  const TargetRegisterInfo &TRI;3814  const DataLayout &DL;3815  const LoopInfo &LI;3816  const std::function<const DominatorTree &()> getDTFn;3817 3818  /// AccessTy/MemoryInst - This is the type for the access (e.g. double) and3819  /// the memory instruction that we're computing this address for.3820  Type *AccessTy;3821  unsigned AddrSpace;3822  Instruction *MemoryInst;3823 3824  /// This is the addressing mode that we're building up. This is3825  /// part of the return value of this addressing mode matching stuff.3826  ExtAddrMode &AddrMode;3827 3828  /// The instructions inserted by other CodeGenPrepare optimizations.3829  const SetOfInstrs &InsertedInsts;3830 3831  /// A map from the instructions to their type before promotion.3832  InstrToOrigTy &PromotedInsts;3833 3834  /// The ongoing transaction where every action should be registered.3835  TypePromotionTransaction &TPT;3836 3837  // A GEP which has too large offset to be folded into the addressing mode.3838  std::pair<AssertingVH<GetElementPtrInst>, int64_t> &LargeOffsetGEP;3839 3840  /// This is set to true when we should not do profitability checks.3841  /// When true, IsProfitableToFoldIntoAddressingMode always returns true.3842  bool IgnoreProfitability;3843 3844  /// True if we are optimizing for size.3845  bool OptSize = false;3846 3847  ProfileSummaryInfo *PSI;3848  BlockFrequencyInfo *BFI;3849 3850  AddressingModeMatcher(3851      SmallVectorImpl<Instruction *> &AMI, const TargetLowering &TLI,3852      const TargetRegisterInfo &TRI, const LoopInfo &LI,3853      const std::function<const DominatorTree &()> getDTFn, Type *AT,3854      unsigned AS, Instruction *MI, ExtAddrMode &AM,3855      const SetOfInstrs &InsertedInsts, InstrToOrigTy &PromotedInsts,3856      TypePromotionTransaction &TPT,3857      std::pair<AssertingVH<GetElementPtrInst>, int64_t> &LargeOffsetGEP,3858      bool OptSize, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI)3859      : AddrModeInsts(AMI), TLI(TLI), TRI(TRI),3860        DL(MI->getDataLayout()), LI(LI), getDTFn(getDTFn),3861        AccessTy(AT), AddrSpace(AS), MemoryInst(MI), AddrMode(AM),3862        InsertedInsts(InsertedInsts), PromotedInsts(PromotedInsts), TPT(TPT),3863        LargeOffsetGEP(LargeOffsetGEP), OptSize(OptSize), PSI(PSI), BFI(BFI) {3864    IgnoreProfitability = false;3865  }3866 3867public:3868  /// Find the maximal addressing mode that a load/store of V can fold,3869  /// give an access type of AccessTy.  This returns a list of involved3870  /// instructions in AddrModeInsts.3871  /// \p InsertedInsts The instructions inserted by other CodeGenPrepare3872  /// optimizations.3873  /// \p PromotedInsts maps the instructions to their type before promotion.3874  /// \p The ongoing transaction where every action should be registered.3875  static ExtAddrMode3876  Match(Value *V, Type *AccessTy, unsigned AS, Instruction *MemoryInst,3877        SmallVectorImpl<Instruction *> &AddrModeInsts,3878        const TargetLowering &TLI, const LoopInfo &LI,3879        const std::function<const DominatorTree &()> getDTFn,3880        const TargetRegisterInfo &TRI, const SetOfInstrs &InsertedInsts,3881        InstrToOrigTy &PromotedInsts, TypePromotionTransaction &TPT,3882        std::pair<AssertingVH<GetElementPtrInst>, int64_t> &LargeOffsetGEP,3883        bool OptSize, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI) {3884    ExtAddrMode Result;3885 3886    bool Success = AddressingModeMatcher(AddrModeInsts, TLI, TRI, LI, getDTFn,3887                                         AccessTy, AS, MemoryInst, Result,3888                                         InsertedInsts, PromotedInsts, TPT,3889                                         LargeOffsetGEP, OptSize, PSI, BFI)3890                       .matchAddr(V, 0);3891    (void)Success;3892    assert(Success && "Couldn't select *anything*?");3893    return Result;3894  }3895 3896private:3897  bool matchScaledValue(Value *ScaleReg, int64_t Scale, unsigned Depth);3898  bool matchAddr(Value *Addr, unsigned Depth);3899  bool matchOperationAddr(User *AddrInst, unsigned Opcode, unsigned Depth,3900                          bool *MovedAway = nullptr);3901  bool isProfitableToFoldIntoAddressingMode(Instruction *I,3902                                            ExtAddrMode &AMBefore,3903                                            ExtAddrMode &AMAfter);3904  bool valueAlreadyLiveAtInst(Value *Val, Value *KnownLive1, Value *KnownLive2);3905  bool isPromotionProfitable(unsigned NewCost, unsigned OldCost,3906                             Value *PromotedOperand) const;3907};3908 3909class PhiNodeSet;3910 3911/// An iterator for PhiNodeSet.3912class PhiNodeSetIterator {3913  PhiNodeSet *const Set;3914  size_t CurrentIndex = 0;3915 3916public:3917  /// The constructor. Start should point to either a valid element, or be equal3918  /// to the size of the underlying SmallVector of the PhiNodeSet.3919  PhiNodeSetIterator(PhiNodeSet *const Set, size_t Start);3920  PHINode *operator*() const;3921  PhiNodeSetIterator &operator++();3922  bool operator==(const PhiNodeSetIterator &RHS) const;3923  bool operator!=(const PhiNodeSetIterator &RHS) const;3924};3925 3926/// Keeps a set of PHINodes.3927///3928/// This is a minimal set implementation for a specific use case:3929/// It is very fast when there are very few elements, but also provides good3930/// performance when there are many. It is similar to SmallPtrSet, but also3931/// provides iteration by insertion order, which is deterministic and stable3932/// across runs. It is also similar to SmallSetVector, but provides removing3933/// elements in O(1) time. This is achieved by not actually removing the element3934/// from the underlying vector, so comes at the cost of using more memory, but3935/// that is fine, since PhiNodeSets are used as short lived objects.3936class PhiNodeSet {3937  friend class PhiNodeSetIterator;3938 3939  using MapType = SmallDenseMap<PHINode *, size_t, 32>;3940  using iterator = PhiNodeSetIterator;3941 3942  /// Keeps the elements in the order of their insertion in the underlying3943  /// vector. To achieve constant time removal, it never deletes any element.3944  SmallVector<PHINode *, 32> NodeList;3945 3946  /// Keeps the elements in the underlying set implementation. This (and not the3947  /// NodeList defined above) is the source of truth on whether an element3948  /// is actually in the collection.3949  MapType NodeMap;3950 3951  /// Points to the first valid (not deleted) element when the set is not empty3952  /// and the value is not zero. Equals to the size of the underlying vector3953  /// when the set is empty. When the value is 0, as in the beginning, the3954  /// first element may or may not be valid.3955  size_t FirstValidElement = 0;3956 3957public:3958  /// Inserts a new element to the collection.3959  /// \returns true if the element is actually added, i.e. was not in the3960  /// collection before the operation.3961  bool insert(PHINode *Ptr) {3962    if (NodeMap.insert(std::make_pair(Ptr, NodeList.size())).second) {3963      NodeList.push_back(Ptr);3964      return true;3965    }3966    return false;3967  }3968 3969  /// Removes the element from the collection.3970  /// \returns whether the element is actually removed, i.e. was in the3971  /// collection before the operation.3972  bool erase(PHINode *Ptr) {3973    if (NodeMap.erase(Ptr)) {3974      SkipRemovedElements(FirstValidElement);3975      return true;3976    }3977    return false;3978  }3979 3980  /// Removes all elements and clears the collection.3981  void clear() {3982    NodeMap.clear();3983    NodeList.clear();3984    FirstValidElement = 0;3985  }3986 3987  /// \returns an iterator that will iterate the elements in the order of3988  /// insertion.3989  iterator begin() {3990    if (FirstValidElement == 0)3991      SkipRemovedElements(FirstValidElement);3992    return PhiNodeSetIterator(this, FirstValidElement);3993  }3994 3995  /// \returns an iterator that points to the end of the collection.3996  iterator end() { return PhiNodeSetIterator(this, NodeList.size()); }3997 3998  /// Returns the number of elements in the collection.3999  size_t size() const { return NodeMap.size(); }4000 4001  /// \returns 1 if the given element is in the collection, and 0 if otherwise.4002  size_t count(PHINode *Ptr) const { return NodeMap.count(Ptr); }4003 4004private:4005  /// Updates the CurrentIndex so that it will point to a valid element.4006  ///4007  /// If the element of NodeList at CurrentIndex is valid, it does not4008  /// change it. If there are no more valid elements, it updates CurrentIndex4009  /// to point to the end of the NodeList.4010  void SkipRemovedElements(size_t &CurrentIndex) {4011    while (CurrentIndex < NodeList.size()) {4012      auto it = NodeMap.find(NodeList[CurrentIndex]);4013      // If the element has been deleted and added again later, NodeMap will4014      // point to a different index, so CurrentIndex will still be invalid.4015      if (it != NodeMap.end() && it->second == CurrentIndex)4016        break;4017      ++CurrentIndex;4018    }4019  }4020};4021 4022PhiNodeSetIterator::PhiNodeSetIterator(PhiNodeSet *const Set, size_t Start)4023    : Set(Set), CurrentIndex(Start) {}4024 4025PHINode *PhiNodeSetIterator::operator*() const {4026  assert(CurrentIndex < Set->NodeList.size() &&4027         "PhiNodeSet access out of range");4028  return Set->NodeList[CurrentIndex];4029}4030 4031PhiNodeSetIterator &PhiNodeSetIterator::operator++() {4032  assert(CurrentIndex < Set->NodeList.size() &&4033         "PhiNodeSet access out of range");4034  ++CurrentIndex;4035  Set->SkipRemovedElements(CurrentIndex);4036  return *this;4037}4038 4039bool PhiNodeSetIterator::operator==(const PhiNodeSetIterator &RHS) const {4040  return CurrentIndex == RHS.CurrentIndex;4041}4042 4043bool PhiNodeSetIterator::operator!=(const PhiNodeSetIterator &RHS) const {4044  return !((*this) == RHS);4045}4046 4047/// Keep track of simplification of Phi nodes.4048/// Accept the set of all phi nodes and erase phi node from this set4049/// if it is simplified.4050class SimplificationTracker {4051  DenseMap<Value *, Value *> Storage;4052  // Tracks newly created Phi nodes. The elements are iterated by insertion4053  // order.4054  PhiNodeSet AllPhiNodes;4055  // Tracks newly created Select nodes.4056  SmallPtrSet<SelectInst *, 32> AllSelectNodes;4057 4058public:4059  Value *Get(Value *V) {4060    do {4061      auto SV = Storage.find(V);4062      if (SV == Storage.end())4063        return V;4064      V = SV->second;4065    } while (true);4066  }4067 4068  void Put(Value *From, Value *To) { Storage.insert({From, To}); }4069 4070  void ReplacePhi(PHINode *From, PHINode *To) {4071    Value *OldReplacement = Get(From);4072    while (OldReplacement != From) {4073      From = To;4074      To = dyn_cast<PHINode>(OldReplacement);4075      OldReplacement = Get(From);4076    }4077    assert(To && Get(To) == To && "Replacement PHI node is already replaced.");4078    Put(From, To);4079    From->replaceAllUsesWith(To);4080    AllPhiNodes.erase(From);4081    From->eraseFromParent();4082  }4083 4084  PhiNodeSet &newPhiNodes() { return AllPhiNodes; }4085 4086  void insertNewPhi(PHINode *PN) { AllPhiNodes.insert(PN); }4087 4088  void insertNewSelect(SelectInst *SI) { AllSelectNodes.insert(SI); }4089 4090  unsigned countNewPhiNodes() const { return AllPhiNodes.size(); }4091 4092  unsigned countNewSelectNodes() const { return AllSelectNodes.size(); }4093 4094  void destroyNewNodes(Type *CommonType) {4095    // For safe erasing, replace the uses with dummy value first.4096    auto *Dummy = PoisonValue::get(CommonType);4097    for (auto *I : AllPhiNodes) {4098      I->replaceAllUsesWith(Dummy);4099      I->eraseFromParent();4100    }4101    AllPhiNodes.clear();4102    for (auto *I : AllSelectNodes) {4103      I->replaceAllUsesWith(Dummy);4104      I->eraseFromParent();4105    }4106    AllSelectNodes.clear();4107  }4108};4109 4110/// A helper class for combining addressing modes.4111class AddressingModeCombiner {4112  typedef DenseMap<Value *, Value *> FoldAddrToValueMapping;4113  typedef std::pair<PHINode *, PHINode *> PHIPair;4114 4115private:4116  /// The addressing modes we've collected.4117  SmallVector<ExtAddrMode, 16> AddrModes;4118 4119  /// The field in which the AddrModes differ, when we have more than one.4120  ExtAddrMode::FieldName DifferentField = ExtAddrMode::NoField;4121 4122  /// Are the AddrModes that we have all just equal to their original values?4123  bool AllAddrModesTrivial = true;4124 4125  /// Common Type for all different fields in addressing modes.4126  Type *CommonType = nullptr;4127 4128  const DataLayout &DL;4129 4130  /// Original Address.4131  Value *Original;4132 4133  /// Common value among addresses4134  Value *CommonValue = nullptr;4135 4136public:4137  AddressingModeCombiner(const DataLayout &DL, Value *OriginalValue)4138      : DL(DL), Original(OriginalValue) {}4139 4140  ~AddressingModeCombiner() { eraseCommonValueIfDead(); }4141 4142  /// Get the combined AddrMode4143  const ExtAddrMode &getAddrMode() const { return AddrModes[0]; }4144 4145  /// Add a new AddrMode if it's compatible with the AddrModes we already4146  /// have.4147  /// \return True iff we succeeded in doing so.4148  bool addNewAddrMode(ExtAddrMode &NewAddrMode) {4149    // Take note of if we have any non-trivial AddrModes, as we need to detect4150    // when all AddrModes are trivial as then we would introduce a phi or select4151    // which just duplicates what's already there.4152    AllAddrModesTrivial = AllAddrModesTrivial && NewAddrMode.isTrivial();4153 4154    // If this is the first addrmode then everything is fine.4155    if (AddrModes.empty()) {4156      AddrModes.emplace_back(NewAddrMode);4157      return true;4158    }4159 4160    // Figure out how different this is from the other address modes, which we4161    // can do just by comparing against the first one given that we only care4162    // about the cumulative difference.4163    ExtAddrMode::FieldName ThisDifferentField =4164        AddrModes[0].compare(NewAddrMode);4165    if (DifferentField == ExtAddrMode::NoField)4166      DifferentField = ThisDifferentField;4167    else if (DifferentField != ThisDifferentField)4168      DifferentField = ExtAddrMode::MultipleFields;4169 4170    // If NewAddrMode differs in more than one dimension we cannot handle it.4171    bool CanHandle = DifferentField != ExtAddrMode::MultipleFields;4172 4173    // If Scale Field is different then we reject.4174    CanHandle = CanHandle && DifferentField != ExtAddrMode::ScaleField;4175 4176    // We also must reject the case when base offset is different and4177    // scale reg is not null, we cannot handle this case due to merge of4178    // different offsets will be used as ScaleReg.4179    CanHandle = CanHandle && (DifferentField != ExtAddrMode::BaseOffsField ||4180                              !NewAddrMode.ScaledReg);4181 4182    // We also must reject the case when GV is different and BaseReg installed4183    // due to we want to use base reg as a merge of GV values.4184    CanHandle = CanHandle && (DifferentField != ExtAddrMode::BaseGVField ||4185                              !NewAddrMode.HasBaseReg);4186 4187    // Even if NewAddMode is the same we still need to collect it due to4188    // original value is different. And later we will need all original values4189    // as anchors during finding the common Phi node.4190    if (CanHandle)4191      AddrModes.emplace_back(NewAddrMode);4192    else4193      AddrModes.clear();4194 4195    return CanHandle;4196  }4197 4198  /// Combine the addressing modes we've collected into a single4199  /// addressing mode.4200  /// \return True iff we successfully combined them or we only had one so4201  /// didn't need to combine them anyway.4202  bool combineAddrModes() {4203    // If we have no AddrModes then they can't be combined.4204    if (AddrModes.size() == 0)4205      return false;4206 4207    // A single AddrMode can trivially be combined.4208    if (AddrModes.size() == 1 || DifferentField == ExtAddrMode::NoField)4209      return true;4210 4211    // If the AddrModes we collected are all just equal to the value they are4212    // derived from then combining them wouldn't do anything useful.4213    if (AllAddrModesTrivial)4214      return false;4215 4216    if (!addrModeCombiningAllowed())4217      return false;4218 4219    // Build a map between <original value, basic block where we saw it> to4220    // value of base register.4221    // Bail out if there is no common type.4222    FoldAddrToValueMapping Map;4223    if (!initializeMap(Map))4224      return false;4225 4226    CommonValue = findCommon(Map);4227    if (CommonValue)4228      AddrModes[0].SetCombinedField(DifferentField, CommonValue, AddrModes);4229    return CommonValue != nullptr;4230  }4231 4232private:4233  /// `CommonValue` may be a placeholder inserted by us.4234  /// If the placeholder is not used, we should remove this dead instruction.4235  void eraseCommonValueIfDead() {4236    if (CommonValue && CommonValue->use_empty())4237      if (Instruction *CommonInst = dyn_cast<Instruction>(CommonValue))4238        CommonInst->eraseFromParent();4239  }4240 4241  /// Initialize Map with anchor values. For address seen4242  /// we set the value of different field saw in this address.4243  /// At the same time we find a common type for different field we will4244  /// use to create new Phi/Select nodes. Keep it in CommonType field.4245  /// Return false if there is no common type found.4246  bool initializeMap(FoldAddrToValueMapping &Map) {4247    // Keep track of keys where the value is null. We will need to replace it4248    // with constant null when we know the common type.4249    SmallVector<Value *, 2> NullValue;4250    Type *IntPtrTy = DL.getIntPtrType(AddrModes[0].OriginalValue->getType());4251    for (auto &AM : AddrModes) {4252      Value *DV = AM.GetFieldAsValue(DifferentField, IntPtrTy);4253      if (DV) {4254        auto *Type = DV->getType();4255        if (CommonType && CommonType != Type)4256          return false;4257        CommonType = Type;4258        Map[AM.OriginalValue] = DV;4259      } else {4260        NullValue.push_back(AM.OriginalValue);4261      }4262    }4263    assert(CommonType && "At least one non-null value must be!");4264    for (auto *V : NullValue)4265      Map[V] = Constant::getNullValue(CommonType);4266    return true;4267  }4268 4269  /// We have mapping between value A and other value B where B was a field in4270  /// addressing mode represented by A. Also we have an original value C4271  /// representing an address we start with. Traversing from C through phi and4272  /// selects we ended up with A's in a map. This utility function tries to find4273  /// a value V which is a field in addressing mode C and traversing through phi4274  /// nodes and selects we will end up in corresponded values B in a map.4275  /// The utility will create a new Phi/Selects if needed.4276  // The simple example looks as follows:4277  // BB1:4278  //   p1 = b1 + 404279  //   br cond BB2, BB34280  // BB2:4281  //   p2 = b2 + 404282  //   br BB34283  // BB3:4284  //   p = phi [p1, BB1], [p2, BB2]4285  //   v = load p4286  // Map is4287  //   p1 -> b14288  //   p2 -> b24289  // Request is4290  //   p -> ?4291  // The function tries to find or build phi [b1, BB1], [b2, BB2] in BB3.4292  Value *findCommon(FoldAddrToValueMapping &Map) {4293    // Tracks the simplification of newly created phi nodes. The reason we use4294    // this mapping is because we will add new created Phi nodes in AddrToBase.4295    // Simplification of Phi nodes is recursive, so some Phi node may4296    // be simplified after we added it to AddrToBase. In reality this4297    // simplification is possible only if original phi/selects were not4298    // simplified yet.4299    // Using this mapping we can find the current value in AddrToBase.4300    SimplificationTracker ST;4301 4302    // First step, DFS to create PHI nodes for all intermediate blocks.4303    // Also fill traverse order for the second step.4304    SmallVector<Value *, 32> TraverseOrder;4305    InsertPlaceholders(Map, TraverseOrder, ST);4306 4307    // Second Step, fill new nodes by merged values and simplify if possible.4308    FillPlaceholders(Map, TraverseOrder, ST);4309 4310    if (!AddrSinkNewSelects && ST.countNewSelectNodes() > 0) {4311      ST.destroyNewNodes(CommonType);4312      return nullptr;4313    }4314 4315    // Now we'd like to match New Phi nodes to existed ones.4316    unsigned PhiNotMatchedCount = 0;4317    if (!MatchPhiSet(ST, AddrSinkNewPhis, PhiNotMatchedCount)) {4318      ST.destroyNewNodes(CommonType);4319      return nullptr;4320    }4321 4322    auto *Result = ST.Get(Map.find(Original)->second);4323    if (Result) {4324      NumMemoryInstsPhiCreated += ST.countNewPhiNodes() + PhiNotMatchedCount;4325      NumMemoryInstsSelectCreated += ST.countNewSelectNodes();4326    }4327    return Result;4328  }4329 4330  /// Try to match PHI node to Candidate.4331  /// Matcher tracks the matched Phi nodes.4332  bool MatchPhiNode(PHINode *PHI, PHINode *Candidate,4333                    SmallSetVector<PHIPair, 8> &Matcher,4334                    PhiNodeSet &PhiNodesToMatch) {4335    SmallVector<PHIPair, 8> WorkList;4336    Matcher.insert({PHI, Candidate});4337    SmallPtrSet<PHINode *, 8> MatchedPHIs;4338    MatchedPHIs.insert(PHI);4339    WorkList.push_back({PHI, Candidate});4340    SmallSet<PHIPair, 8> Visited;4341    while (!WorkList.empty()) {4342      auto Item = WorkList.pop_back_val();4343      if (!Visited.insert(Item).second)4344        continue;4345      // We iterate over all incoming values to Phi to compare them.4346      // If values are different and both of them Phi and the first one is a4347      // Phi we added (subject to match) and both of them is in the same basic4348      // block then we can match our pair if values match. So we state that4349      // these values match and add it to work list to verify that.4350      for (auto *B : Item.first->blocks()) {4351        Value *FirstValue = Item.first->getIncomingValueForBlock(B);4352        Value *SecondValue = Item.second->getIncomingValueForBlock(B);4353        if (FirstValue == SecondValue)4354          continue;4355 4356        PHINode *FirstPhi = dyn_cast<PHINode>(FirstValue);4357        PHINode *SecondPhi = dyn_cast<PHINode>(SecondValue);4358 4359        // One of them is not Phi or4360        // The first one is not Phi node from the set we'd like to match or4361        // Phi nodes from different basic blocks then4362        // we will not be able to match.4363        if (!FirstPhi || !SecondPhi || !PhiNodesToMatch.count(FirstPhi) ||4364            FirstPhi->getParent() != SecondPhi->getParent())4365          return false;4366 4367        // If we already matched them then continue.4368        if (Matcher.count({FirstPhi, SecondPhi}))4369          continue;4370        // So the values are different and does not match. So we need them to4371        // match. (But we register no more than one match per PHI node, so that4372        // we won't later try to replace them twice.)4373        if (MatchedPHIs.insert(FirstPhi).second)4374          Matcher.insert({FirstPhi, SecondPhi});4375        // But me must check it.4376        WorkList.push_back({FirstPhi, SecondPhi});4377      }4378    }4379    return true;4380  }4381 4382  /// For the given set of PHI nodes (in the SimplificationTracker) try4383  /// to find their equivalents.4384  /// Returns false if this matching fails and creation of new Phi is disabled.4385  bool MatchPhiSet(SimplificationTracker &ST, bool AllowNewPhiNodes,4386                   unsigned &PhiNotMatchedCount) {4387    // Matched and PhiNodesToMatch iterate their elements in a deterministic4388    // order, so the replacements (ReplacePhi) are also done in a deterministic4389    // order.4390    SmallSetVector<PHIPair, 8> Matched;4391    SmallPtrSet<PHINode *, 8> WillNotMatch;4392    PhiNodeSet &PhiNodesToMatch = ST.newPhiNodes();4393    while (PhiNodesToMatch.size()) {4394      PHINode *PHI = *PhiNodesToMatch.begin();4395 4396      // Add us, if no Phi nodes in the basic block we do not match.4397      WillNotMatch.clear();4398      WillNotMatch.insert(PHI);4399 4400      // Traverse all Phis until we found equivalent or fail to do that.4401      bool IsMatched = false;4402      for (auto &P : PHI->getParent()->phis()) {4403        // Skip new Phi nodes.4404        if (PhiNodesToMatch.count(&P))4405          continue;4406        if ((IsMatched = MatchPhiNode(PHI, &P, Matched, PhiNodesToMatch)))4407          break;4408        // If it does not match, collect all Phi nodes from matcher.4409        // if we end up with no match, them all these Phi nodes will not match4410        // later.4411        WillNotMatch.insert_range(llvm::make_first_range(Matched));4412        Matched.clear();4413      }4414      if (IsMatched) {4415        // Replace all matched values and erase them.4416        for (auto MV : Matched)4417          ST.ReplacePhi(MV.first, MV.second);4418        Matched.clear();4419        continue;4420      }4421      // If we are not allowed to create new nodes then bail out.4422      if (!AllowNewPhiNodes)4423        return false;4424      // Just remove all seen values in matcher. They will not match anything.4425      PhiNotMatchedCount += WillNotMatch.size();4426      for (auto *P : WillNotMatch)4427        PhiNodesToMatch.erase(P);4428    }4429    return true;4430  }4431  /// Fill the placeholders with values from predecessors and simplify them.4432  void FillPlaceholders(FoldAddrToValueMapping &Map,4433                        SmallVectorImpl<Value *> &TraverseOrder,4434                        SimplificationTracker &ST) {4435    while (!TraverseOrder.empty()) {4436      Value *Current = TraverseOrder.pop_back_val();4437      assert(Map.contains(Current) && "No node to fill!!!");4438      Value *V = Map[Current];4439 4440      if (SelectInst *Select = dyn_cast<SelectInst>(V)) {4441        // CurrentValue also must be Select.4442        auto *CurrentSelect = cast<SelectInst>(Current);4443        auto *TrueValue = CurrentSelect->getTrueValue();4444        assert(Map.contains(TrueValue) && "No True Value!");4445        Select->setTrueValue(ST.Get(Map[TrueValue]));4446        auto *FalseValue = CurrentSelect->getFalseValue();4447        assert(Map.contains(FalseValue) && "No False Value!");4448        Select->setFalseValue(ST.Get(Map[FalseValue]));4449      } else {4450        // Must be a Phi node then.4451        auto *PHI = cast<PHINode>(V);4452        // Fill the Phi node with values from predecessors.4453        for (auto *B : predecessors(PHI->getParent())) {4454          Value *PV = cast<PHINode>(Current)->getIncomingValueForBlock(B);4455          assert(Map.contains(PV) && "No predecessor Value!");4456          PHI->addIncoming(ST.Get(Map[PV]), B);4457        }4458      }4459    }4460  }4461 4462  /// Starting from original value recursively iterates over def-use chain up to4463  /// known ending values represented in a map. For each traversed phi/select4464  /// inserts a placeholder Phi or Select.4465  /// Reports all new created Phi/Select nodes by adding them to set.4466  /// Also reports and order in what values have been traversed.4467  void InsertPlaceholders(FoldAddrToValueMapping &Map,4468                          SmallVectorImpl<Value *> &TraverseOrder,4469                          SimplificationTracker &ST) {4470    SmallVector<Value *, 32> Worklist;4471    assert((isa<PHINode>(Original) || isa<SelectInst>(Original)) &&4472           "Address must be a Phi or Select node");4473    auto *Dummy = PoisonValue::get(CommonType);4474    Worklist.push_back(Original);4475    while (!Worklist.empty()) {4476      Value *Current = Worklist.pop_back_val();4477      // if it is already visited or it is an ending value then skip it.4478      if (Map.contains(Current))4479        continue;4480      TraverseOrder.push_back(Current);4481 4482      // CurrentValue must be a Phi node or select. All others must be covered4483      // by anchors.4484      if (SelectInst *CurrentSelect = dyn_cast<SelectInst>(Current)) {4485        // Is it OK to get metadata from OrigSelect?!4486        // Create a Select placeholder with dummy value.4487        SelectInst *Select =4488            SelectInst::Create(CurrentSelect->getCondition(), Dummy, Dummy,4489                               CurrentSelect->getName(),4490                               CurrentSelect->getIterator(), CurrentSelect);4491        Map[Current] = Select;4492        ST.insertNewSelect(Select);4493        // We are interested in True and False values.4494        Worklist.push_back(CurrentSelect->getTrueValue());4495        Worklist.push_back(CurrentSelect->getFalseValue());4496      } else {4497        // It must be a Phi node then.4498        PHINode *CurrentPhi = cast<PHINode>(Current);4499        unsigned PredCount = CurrentPhi->getNumIncomingValues();4500        PHINode *PHI =4501            PHINode::Create(CommonType, PredCount, "sunk_phi", CurrentPhi->getIterator());4502        Map[Current] = PHI;4503        ST.insertNewPhi(PHI);4504        append_range(Worklist, CurrentPhi->incoming_values());4505      }4506    }4507  }4508 4509  bool addrModeCombiningAllowed() {4510    if (DisableComplexAddrModes)4511      return false;4512    switch (DifferentField) {4513    default:4514      return false;4515    case ExtAddrMode::BaseRegField:4516      return AddrSinkCombineBaseReg;4517    case ExtAddrMode::BaseGVField:4518      return AddrSinkCombineBaseGV;4519    case ExtAddrMode::BaseOffsField:4520      return AddrSinkCombineBaseOffs;4521    case ExtAddrMode::ScaledRegField:4522      return AddrSinkCombineScaledReg;4523    }4524  }4525};4526} // end anonymous namespace4527 4528/// Try adding ScaleReg*Scale to the current addressing mode.4529/// Return true and update AddrMode if this addr mode is legal for the target,4530/// false if not.4531bool AddressingModeMatcher::matchScaledValue(Value *ScaleReg, int64_t Scale,4532                                             unsigned Depth) {4533  // If Scale is 1, then this is the same as adding ScaleReg to the addressing4534  // mode.  Just process that directly.4535  if (Scale == 1)4536    return matchAddr(ScaleReg, Depth);4537 4538  // If the scale is 0, it takes nothing to add this.4539  if (Scale == 0)4540    return true;4541 4542  // If we already have a scale of this value, we can add to it, otherwise, we4543  // need an available scale field.4544  if (AddrMode.Scale != 0 && AddrMode.ScaledReg != ScaleReg)4545    return false;4546 4547  ExtAddrMode TestAddrMode = AddrMode;4548 4549  // Add scale to turn X*4+X*3 -> X*7.  This could also do things like4550  // [A+B + A*7] -> [B+A*8].4551  TestAddrMode.Scale += Scale;4552  TestAddrMode.ScaledReg = ScaleReg;4553 4554  // If the new address isn't legal, bail out.4555  if (!TLI.isLegalAddressingMode(DL, TestAddrMode, AccessTy, AddrSpace))4556    return false;4557 4558  // It was legal, so commit it.4559  AddrMode = TestAddrMode;4560 4561  // Okay, we decided that we can add ScaleReg+Scale to AddrMode.  Check now4562  // to see if ScaleReg is actually X+C.  If so, we can turn this into adding4563  // X*Scale + C*Scale to addr mode. If we found available IV increment, do not4564  // go any further: we can reuse it and cannot eliminate it.4565  ConstantInt *CI = nullptr;4566  Value *AddLHS = nullptr;4567  if (isa<Instruction>(ScaleReg) && // not a constant expr.4568      match(ScaleReg, m_Add(m_Value(AddLHS), m_ConstantInt(CI))) &&4569      !isIVIncrement(ScaleReg, &LI) && CI->getValue().isSignedIntN(64)) {4570    TestAddrMode.InBounds = false;4571    TestAddrMode.ScaledReg = AddLHS;4572    TestAddrMode.BaseOffs += CI->getSExtValue() * TestAddrMode.Scale;4573 4574    // If this addressing mode is legal, commit it and remember that we folded4575    // this instruction.4576    if (TLI.isLegalAddressingMode(DL, TestAddrMode, AccessTy, AddrSpace)) {4577      AddrModeInsts.push_back(cast<Instruction>(ScaleReg));4578      AddrMode = TestAddrMode;4579      return true;4580    }4581    // Restore status quo.4582    TestAddrMode = AddrMode;4583  }4584 4585  // If this is an add recurrence with a constant step, return the increment4586  // instruction and the canonicalized step.4587  auto GetConstantStep =4588      [this](const Value *V) -> std::optional<std::pair<Instruction *, APInt>> {4589    auto *PN = dyn_cast<PHINode>(V);4590    if (!PN)4591      return std::nullopt;4592    auto IVInc = getIVIncrement(PN, &LI);4593    if (!IVInc)4594      return std::nullopt;4595    // TODO: The result of the intrinsics above is two-complement. However when4596    // IV inc is expressed as add or sub, iv.next is potentially a poison value.4597    // If it has nuw or nsw flags, we need to make sure that these flags are4598    // inferrable at the point of memory instruction. Otherwise we are replacing4599    // well-defined two-complement computation with poison. Currently, to avoid4600    // potentially complex analysis needed to prove this, we reject such cases.4601    if (auto *OIVInc = dyn_cast<OverflowingBinaryOperator>(IVInc->first))4602      if (OIVInc->hasNoSignedWrap() || OIVInc->hasNoUnsignedWrap())4603        return std::nullopt;4604    if (auto *ConstantStep = dyn_cast<ConstantInt>(IVInc->second))4605      return std::make_pair(IVInc->first, ConstantStep->getValue());4606    return std::nullopt;4607  };4608 4609  // Try to account for the following special case:4610  // 1. ScaleReg is an inductive variable;4611  // 2. We use it with non-zero offset;4612  // 3. IV's increment is available at the point of memory instruction.4613  //4614  // In this case, we may reuse the IV increment instead of the IV Phi to4615  // achieve the following advantages:4616  // 1. If IV step matches the offset, we will have no need in the offset;4617  // 2. Even if they don't match, we will reduce the overlap of living IV4618  //    and IV increment, that will potentially lead to better register4619  //    assignment.4620  if (AddrMode.BaseOffs) {4621    if (auto IVStep = GetConstantStep(ScaleReg)) {4622      Instruction *IVInc = IVStep->first;4623      // The following assert is important to ensure a lack of infinite loops.4624      // This transforms is (intentionally) the inverse of the one just above.4625      // If they don't agree on the definition of an increment, we'd alternate4626      // back and forth indefinitely.4627      assert(isIVIncrement(IVInc, &LI) && "implied by GetConstantStep");4628      APInt Step = IVStep->second;4629      APInt Offset = Step * AddrMode.Scale;4630      if (Offset.isSignedIntN(64)) {4631        TestAddrMode.InBounds = false;4632        TestAddrMode.ScaledReg = IVInc;4633        TestAddrMode.BaseOffs -= Offset.getLimitedValue();4634        // If this addressing mode is legal, commit it..4635        // (Note that we defer the (expensive) domtree base legality check4636        // to the very last possible point.)4637        if (TLI.isLegalAddressingMode(DL, TestAddrMode, AccessTy, AddrSpace) &&4638            getDTFn().dominates(IVInc, MemoryInst)) {4639          AddrModeInsts.push_back(cast<Instruction>(IVInc));4640          AddrMode = TestAddrMode;4641          return true;4642        }4643        // Restore status quo.4644        TestAddrMode = AddrMode;4645      }4646    }4647  }4648 4649  // Otherwise, just return what we have.4650  return true;4651}4652 4653/// This is a little filter, which returns true if an addressing computation4654/// involving I might be folded into a load/store accessing it.4655/// This doesn't need to be perfect, but needs to accept at least4656/// the set of instructions that MatchOperationAddr can.4657static bool MightBeFoldableInst(Instruction *I) {4658  switch (I->getOpcode()) {4659  case Instruction::BitCast:4660  case Instruction::AddrSpaceCast:4661    // Don't touch identity bitcasts.4662    if (I->getType() == I->getOperand(0)->getType())4663      return false;4664    return I->getType()->isIntOrPtrTy();4665  case Instruction::PtrToInt:4666    // PtrToInt is always a noop, as we know that the int type is pointer sized.4667    return true;4668  case Instruction::IntToPtr:4669    // We know the input is intptr_t, so this is foldable.4670    return true;4671  case Instruction::Add:4672    return true;4673  case Instruction::Mul:4674  case Instruction::Shl:4675    // Can only handle X*C and X << C.4676    return isa<ConstantInt>(I->getOperand(1));4677  case Instruction::GetElementPtr:4678    return true;4679  default:4680    return false;4681  }4682}4683 4684/// Check whether or not \p Val is a legal instruction for \p TLI.4685/// \note \p Val is assumed to be the product of some type promotion.4686/// Therefore if \p Val has an undefined state in \p TLI, this is assumed4687/// to be legal, as the non-promoted value would have had the same state.4688static bool isPromotedInstructionLegal(const TargetLowering &TLI,4689                                       const DataLayout &DL, Value *Val) {4690  Instruction *PromotedInst = dyn_cast<Instruction>(Val);4691  if (!PromotedInst)4692    return false;4693  int ISDOpcode = TLI.InstructionOpcodeToISD(PromotedInst->getOpcode());4694  // If the ISDOpcode is undefined, it was undefined before the promotion.4695  if (!ISDOpcode)4696    return true;4697  // Otherwise, check if the promoted instruction is legal or not.4698  return TLI.isOperationLegalOrCustom(4699      ISDOpcode, TLI.getValueType(DL, PromotedInst->getType()));4700}4701 4702namespace {4703 4704/// Hepler class to perform type promotion.4705class TypePromotionHelper {4706  /// Utility function to add a promoted instruction \p ExtOpnd to4707  /// \p PromotedInsts and record the type of extension we have seen.4708  static void addPromotedInst(InstrToOrigTy &PromotedInsts,4709                              Instruction *ExtOpnd, bool IsSExt) {4710    ExtType ExtTy = IsSExt ? SignExtension : ZeroExtension;4711    auto [It, Inserted] = PromotedInsts.try_emplace(ExtOpnd);4712    if (!Inserted) {4713      // If the new extension is same as original, the information in4714      // PromotedInsts[ExtOpnd] is still correct.4715      if (It->second.getInt() == ExtTy)4716        return;4717 4718      // Now the new extension is different from old extension, we make4719      // the type information invalid by setting extension type to4720      // BothExtension.4721      ExtTy = BothExtension;4722    }4723    It->second = TypeIsSExt(ExtOpnd->getType(), ExtTy);4724  }4725 4726  /// Utility function to query the original type of instruction \p Opnd4727  /// with a matched extension type. If the extension doesn't match, we4728  /// cannot use the information we had on the original type.4729  /// BothExtension doesn't match any extension type.4730  static const Type *getOrigType(const InstrToOrigTy &PromotedInsts,4731                                 Instruction *Opnd, bool IsSExt) {4732    ExtType ExtTy = IsSExt ? SignExtension : ZeroExtension;4733    InstrToOrigTy::const_iterator It = PromotedInsts.find(Opnd);4734    if (It != PromotedInsts.end() && It->second.getInt() == ExtTy)4735      return It->second.getPointer();4736    return nullptr;4737  }4738 4739  /// Utility function to check whether or not a sign or zero extension4740  /// of \p Inst with \p ConsideredExtType can be moved through \p Inst by4741  /// either using the operands of \p Inst or promoting \p Inst.4742  /// The type of the extension is defined by \p IsSExt.4743  /// In other words, check if:4744  /// ext (Ty Inst opnd1 opnd2 ... opndN) to ConsideredExtType.4745  /// #1 Promotion applies:4746  /// ConsideredExtType Inst (ext opnd1 to ConsideredExtType, ...).4747  /// #2 Operand reuses:4748  /// ext opnd1 to ConsideredExtType.4749  /// \p PromotedInsts maps the instructions to their type before promotion.4750  static bool canGetThrough(const Instruction *Inst, Type *ConsideredExtType,4751                            const InstrToOrigTy &PromotedInsts, bool IsSExt);4752 4753  /// Utility function to determine if \p OpIdx should be promoted when4754  /// promoting \p Inst.4755  static bool shouldExtOperand(const Instruction *Inst, int OpIdx) {4756    return !(isa<SelectInst>(Inst) && OpIdx == 0);4757  }4758 4759  /// Utility function to promote the operand of \p Ext when this4760  /// operand is a promotable trunc or sext or zext.4761  /// \p PromotedInsts maps the instructions to their type before promotion.4762  /// \p CreatedInstsCost[out] contains the cost of all instructions4763  /// created to promote the operand of Ext.4764  /// Newly added extensions are inserted in \p Exts.4765  /// Newly added truncates are inserted in \p Truncs.4766  /// Should never be called directly.4767  /// \return The promoted value which is used instead of Ext.4768  static Value *promoteOperandForTruncAndAnyExt(4769      Instruction *Ext, TypePromotionTransaction &TPT,4770      InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,4771      SmallVectorImpl<Instruction *> *Exts,4772      SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI);4773 4774  /// Utility function to promote the operand of \p Ext when this4775  /// operand is promotable and is not a supported trunc or sext.4776  /// \p PromotedInsts maps the instructions to their type before promotion.4777  /// \p CreatedInstsCost[out] contains the cost of all the instructions4778  /// created to promote the operand of Ext.4779  /// Newly added extensions are inserted in \p Exts.4780  /// Newly added truncates are inserted in \p Truncs.4781  /// Should never be called directly.4782  /// \return The promoted value which is used instead of Ext.4783  static Value *promoteOperandForOther(Instruction *Ext,4784                                       TypePromotionTransaction &TPT,4785                                       InstrToOrigTy &PromotedInsts,4786                                       unsigned &CreatedInstsCost,4787                                       SmallVectorImpl<Instruction *> *Exts,4788                                       SmallVectorImpl<Instruction *> *Truncs,4789                                       const TargetLowering &TLI, bool IsSExt);4790 4791  /// \see promoteOperandForOther.4792  static Value *signExtendOperandForOther(4793      Instruction *Ext, TypePromotionTransaction &TPT,4794      InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,4795      SmallVectorImpl<Instruction *> *Exts,4796      SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI) {4797    return promoteOperandForOther(Ext, TPT, PromotedInsts, CreatedInstsCost,4798                                  Exts, Truncs, TLI, true);4799  }4800 4801  /// \see promoteOperandForOther.4802  static Value *zeroExtendOperandForOther(4803      Instruction *Ext, TypePromotionTransaction &TPT,4804      InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,4805      SmallVectorImpl<Instruction *> *Exts,4806      SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI) {4807    return promoteOperandForOther(Ext, TPT, PromotedInsts, CreatedInstsCost,4808                                  Exts, Truncs, TLI, false);4809  }4810 4811public:4812  /// Type for the utility function that promotes the operand of Ext.4813  using Action = Value *(*)(Instruction *Ext, TypePromotionTransaction &TPT,4814                            InstrToOrigTy &PromotedInsts,4815                            unsigned &CreatedInstsCost,4816                            SmallVectorImpl<Instruction *> *Exts,4817                            SmallVectorImpl<Instruction *> *Truncs,4818                            const TargetLowering &TLI);4819 4820  /// Given a sign/zero extend instruction \p Ext, return the appropriate4821  /// action to promote the operand of \p Ext instead of using Ext.4822  /// \return NULL if no promotable action is possible with the current4823  /// sign extension.4824  /// \p InsertedInsts keeps track of all the instructions inserted by the4825  /// other CodeGenPrepare optimizations. This information is important4826  /// because we do not want to promote these instructions as CodeGenPrepare4827  /// will reinsert them later. Thus creating an infinite loop: create/remove.4828  /// \p PromotedInsts maps the instructions to their type before promotion.4829  static Action getAction(Instruction *Ext, const SetOfInstrs &InsertedInsts,4830                          const TargetLowering &TLI,4831                          const InstrToOrigTy &PromotedInsts);4832};4833 4834} // end anonymous namespace4835 4836bool TypePromotionHelper::canGetThrough(const Instruction *Inst,4837                                        Type *ConsideredExtType,4838                                        const InstrToOrigTy &PromotedInsts,4839                                        bool IsSExt) {4840  // The promotion helper does not know how to deal with vector types yet.4841  // To be able to fix that, we would need to fix the places where we4842  // statically extend, e.g., constants and such.4843  if (Inst->getType()->isVectorTy())4844    return false;4845 4846  // We can always get through zext.4847  if (isa<ZExtInst>(Inst))4848    return true;4849 4850  // sext(sext) is ok too.4851  if (IsSExt && isa<SExtInst>(Inst))4852    return true;4853 4854  // We can get through binary operator, if it is legal. In other words, the4855  // binary operator must have a nuw or nsw flag.4856  if (const auto *BinOp = dyn_cast<BinaryOperator>(Inst))4857    if (isa<OverflowingBinaryOperator>(BinOp) &&4858        ((!IsSExt && BinOp->hasNoUnsignedWrap()) ||4859         (IsSExt && BinOp->hasNoSignedWrap())))4860      return true;4861 4862  // ext(and(opnd, cst)) --> and(ext(opnd), ext(cst))4863  if ((Inst->getOpcode() == Instruction::And ||4864       Inst->getOpcode() == Instruction::Or))4865    return true;4866 4867  // ext(xor(opnd, cst)) --> xor(ext(opnd), ext(cst))4868  if (Inst->getOpcode() == Instruction::Xor) {4869    // Make sure it is not a NOT.4870    if (const auto *Cst = dyn_cast<ConstantInt>(Inst->getOperand(1)))4871      if (!Cst->getValue().isAllOnes())4872        return true;4873  }4874 4875  // zext(shrl(opnd, cst)) --> shrl(zext(opnd), zext(cst))4876  // It may change a poisoned value into a regular value, like4877  //     zext i32 (shrl i8 %val, 12)  -->  shrl i32 (zext i8 %val), 124878  //          poisoned value                    regular value4879  // It should be OK since undef covers valid value.4880  if (Inst->getOpcode() == Instruction::LShr && !IsSExt)4881    return true;4882 4883  // and(ext(shl(opnd, cst)), cst) --> and(shl(ext(opnd), ext(cst)), cst)4884  // It may change a poisoned value into a regular value, like4885  //     zext i32 (shl i8 %val, 12)  -->  shl i32 (zext i8 %val), 124886  //          poisoned value                    regular value4887  // It should be OK since undef covers valid value.4888  if (Inst->getOpcode() == Instruction::Shl && Inst->hasOneUse()) {4889    const auto *ExtInst = cast<const Instruction>(*Inst->user_begin());4890    if (ExtInst->hasOneUse()) {4891      const auto *AndInst = dyn_cast<const Instruction>(*ExtInst->user_begin());4892      if (AndInst && AndInst->getOpcode() == Instruction::And) {4893        const auto *Cst = dyn_cast<ConstantInt>(AndInst->getOperand(1));4894        if (Cst &&4895            Cst->getValue().isIntN(Inst->getType()->getIntegerBitWidth()))4896          return true;4897      }4898    }4899  }4900 4901  // Check if we can do the following simplification.4902  // ext(trunc(opnd)) --> ext(opnd)4903  if (!isa<TruncInst>(Inst))4904    return false;4905 4906  Value *OpndVal = Inst->getOperand(0);4907  // Check if we can use this operand in the extension.4908  // If the type is larger than the result type of the extension, we cannot.4909  if (!OpndVal->getType()->isIntegerTy() ||4910      OpndVal->getType()->getIntegerBitWidth() >4911          ConsideredExtType->getIntegerBitWidth())4912    return false;4913 4914  // If the operand of the truncate is not an instruction, we will not have4915  // any information on the dropped bits.4916  // (Actually we could for constant but it is not worth the extra logic).4917  Instruction *Opnd = dyn_cast<Instruction>(OpndVal);4918  if (!Opnd)4919    return false;4920 4921  // Check if the source of the type is narrow enough.4922  // I.e., check that trunc just drops extended bits of the same kind of4923  // the extension.4924  // #1 get the type of the operand and check the kind of the extended bits.4925  const Type *OpndType = getOrigType(PromotedInsts, Opnd, IsSExt);4926  if (OpndType)4927    ;4928  else if ((IsSExt && isa<SExtInst>(Opnd)) || (!IsSExt && isa<ZExtInst>(Opnd)))4929    OpndType = Opnd->getOperand(0)->getType();4930  else4931    return false;4932 4933  // #2 check that the truncate just drops extended bits.4934  return Inst->getType()->getIntegerBitWidth() >=4935         OpndType->getIntegerBitWidth();4936}4937 4938TypePromotionHelper::Action TypePromotionHelper::getAction(4939    Instruction *Ext, const SetOfInstrs &InsertedInsts,4940    const TargetLowering &TLI, const InstrToOrigTy &PromotedInsts) {4941  assert((isa<SExtInst>(Ext) || isa<ZExtInst>(Ext)) &&4942         "Unexpected instruction type");4943  Instruction *ExtOpnd = dyn_cast<Instruction>(Ext->getOperand(0));4944  Type *ExtTy = Ext->getType();4945  bool IsSExt = isa<SExtInst>(Ext);4946  // If the operand of the extension is not an instruction, we cannot4947  // get through.4948  // If it, check we can get through.4949  if (!ExtOpnd || !canGetThrough(ExtOpnd, ExtTy, PromotedInsts, IsSExt))4950    return nullptr;4951 4952  // Do not promote if the operand has been added by codegenprepare.4953  // Otherwise, it means we are undoing an optimization that is likely to be4954  // redone, thus causing potential infinite loop.4955  if (isa<TruncInst>(ExtOpnd) && InsertedInsts.count(ExtOpnd))4956    return nullptr;4957 4958  // SExt or Trunc instructions.4959  // Return the related handler.4960  if (isa<SExtInst>(ExtOpnd) || isa<TruncInst>(ExtOpnd) ||4961      isa<ZExtInst>(ExtOpnd))4962    return promoteOperandForTruncAndAnyExt;4963 4964  // Regular instruction.4965  // Abort early if we will have to insert non-free instructions.4966  if (!ExtOpnd->hasOneUse() && !TLI.isTruncateFree(ExtTy, ExtOpnd->getType()))4967    return nullptr;4968  return IsSExt ? signExtendOperandForOther : zeroExtendOperandForOther;4969}4970 4971Value *TypePromotionHelper::promoteOperandForTruncAndAnyExt(4972    Instruction *SExt, TypePromotionTransaction &TPT,4973    InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,4974    SmallVectorImpl<Instruction *> *Exts,4975    SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI) {4976  // By construction, the operand of SExt is an instruction. Otherwise we cannot4977  // get through it and this method should not be called.4978  Instruction *SExtOpnd = cast<Instruction>(SExt->getOperand(0));4979  Value *ExtVal = SExt;4980  bool HasMergedNonFreeExt = false;4981  if (isa<ZExtInst>(SExtOpnd)) {4982    // Replace s|zext(zext(opnd))4983    // => zext(opnd).4984    HasMergedNonFreeExt = !TLI.isExtFree(SExtOpnd);4985    Value *ZExt =4986        TPT.createZExt(SExt, SExtOpnd->getOperand(0), SExt->getType());4987    TPT.replaceAllUsesWith(SExt, ZExt);4988    TPT.eraseInstruction(SExt);4989    ExtVal = ZExt;4990  } else {4991    // Replace z|sext(trunc(opnd)) or sext(sext(opnd))4992    // => z|sext(opnd).4993    TPT.setOperand(SExt, 0, SExtOpnd->getOperand(0));4994  }4995  CreatedInstsCost = 0;4996 4997  // Remove dead code.4998  if (SExtOpnd->use_empty())4999    TPT.eraseInstruction(SExtOpnd);5000 5001  // Check if the extension is still needed.5002  Instruction *ExtInst = dyn_cast<Instruction>(ExtVal);5003  if (!ExtInst || ExtInst->getType() != ExtInst->getOperand(0)->getType()) {5004    if (ExtInst) {5005      if (Exts)5006        Exts->push_back(ExtInst);5007      CreatedInstsCost = !TLI.isExtFree(ExtInst) && !HasMergedNonFreeExt;5008    }5009    return ExtVal;5010  }5011 5012  // At this point we have: ext ty opnd to ty.5013  // Reassign the uses of ExtInst to the opnd and remove ExtInst.5014  Value *NextVal = ExtInst->getOperand(0);5015  TPT.eraseInstruction(ExtInst, NextVal);5016  return NextVal;5017}5018 5019Value *TypePromotionHelper::promoteOperandForOther(5020    Instruction *Ext, TypePromotionTransaction &TPT,5021    InstrToOrigTy &PromotedInsts, unsigned &CreatedInstsCost,5022    SmallVectorImpl<Instruction *> *Exts,5023    SmallVectorImpl<Instruction *> *Truncs, const TargetLowering &TLI,5024    bool IsSExt) {5025  // By construction, the operand of Ext is an instruction. Otherwise we cannot5026  // get through it and this method should not be called.5027  Instruction *ExtOpnd = cast<Instruction>(Ext->getOperand(0));5028  CreatedInstsCost = 0;5029  if (!ExtOpnd->hasOneUse()) {5030    // ExtOpnd will be promoted.5031    // All its uses, but Ext, will need to use a truncated value of the5032    // promoted version.5033    // Create the truncate now.5034    Value *Trunc = TPT.createTrunc(Ext, ExtOpnd->getType());5035    if (Instruction *ITrunc = dyn_cast<Instruction>(Trunc)) {5036      // Insert it just after the definition.5037      ITrunc->moveAfter(ExtOpnd);5038      if (Truncs)5039        Truncs->push_back(ITrunc);5040    }5041 5042    TPT.replaceAllUsesWith(ExtOpnd, Trunc);5043    // Restore the operand of Ext (which has been replaced by the previous call5044    // to replaceAllUsesWith) to avoid creating a cycle trunc <-> sext.5045    TPT.setOperand(Ext, 0, ExtOpnd);5046  }5047 5048  // Get through the Instruction:5049  // 1. Update its type.5050  // 2. Replace the uses of Ext by Inst.5051  // 3. Extend each operand that needs to be extended.5052 5053  // Remember the original type of the instruction before promotion.5054  // This is useful to know that the high bits are sign extended bits.5055  addPromotedInst(PromotedInsts, ExtOpnd, IsSExt);5056  // Step #1.5057  TPT.mutateType(ExtOpnd, Ext->getType());5058  // Step #2.5059  TPT.replaceAllUsesWith(Ext, ExtOpnd);5060  // Step #3.5061  LLVM_DEBUG(dbgs() << "Propagate Ext to operands\n");5062  for (int OpIdx = 0, EndOpIdx = ExtOpnd->getNumOperands(); OpIdx != EndOpIdx;5063       ++OpIdx) {5064    LLVM_DEBUG(dbgs() << "Operand:\n" << *(ExtOpnd->getOperand(OpIdx)) << '\n');5065    if (ExtOpnd->getOperand(OpIdx)->getType() == Ext->getType() ||5066        !shouldExtOperand(ExtOpnd, OpIdx)) {5067      LLVM_DEBUG(dbgs() << "No need to propagate\n");5068      continue;5069    }5070    // Check if we can statically extend the operand.5071    Value *Opnd = ExtOpnd->getOperand(OpIdx);5072    if (const ConstantInt *Cst = dyn_cast<ConstantInt>(Opnd)) {5073      LLVM_DEBUG(dbgs() << "Statically extend\n");5074      unsigned BitWidth = Ext->getType()->getIntegerBitWidth();5075      APInt CstVal = IsSExt ? Cst->getValue().sext(BitWidth)5076                            : Cst->getValue().zext(BitWidth);5077      TPT.setOperand(ExtOpnd, OpIdx, ConstantInt::get(Ext->getType(), CstVal));5078      continue;5079    }5080    // UndefValue are typed, so we have to statically sign extend them.5081    if (isa<UndefValue>(Opnd)) {5082      LLVM_DEBUG(dbgs() << "Statically extend\n");5083      TPT.setOperand(ExtOpnd, OpIdx, UndefValue::get(Ext->getType()));5084      continue;5085    }5086 5087    // Otherwise we have to explicitly sign extend the operand.5088    Value *ValForExtOpnd = IsSExt5089                               ? TPT.createSExt(ExtOpnd, Opnd, Ext->getType())5090                               : TPT.createZExt(ExtOpnd, Opnd, Ext->getType());5091    TPT.setOperand(ExtOpnd, OpIdx, ValForExtOpnd);5092    Instruction *InstForExtOpnd = dyn_cast<Instruction>(ValForExtOpnd);5093    if (!InstForExtOpnd)5094      continue;5095 5096    if (Exts)5097      Exts->push_back(InstForExtOpnd);5098 5099    CreatedInstsCost += !TLI.isExtFree(InstForExtOpnd);5100  }5101  LLVM_DEBUG(dbgs() << "Extension is useless now\n");5102  TPT.eraseInstruction(Ext);5103  return ExtOpnd;5104}5105 5106/// Check whether or not promoting an instruction to a wider type is profitable.5107/// \p NewCost gives the cost of extension instructions created by the5108/// promotion.5109/// \p OldCost gives the cost of extension instructions before the promotion5110/// plus the number of instructions that have been5111/// matched in the addressing mode the promotion.5112/// \p PromotedOperand is the value that has been promoted.5113/// \return True if the promotion is profitable, false otherwise.5114bool AddressingModeMatcher::isPromotionProfitable(5115    unsigned NewCost, unsigned OldCost, Value *PromotedOperand) const {5116  LLVM_DEBUG(dbgs() << "OldCost: " << OldCost << "\tNewCost: " << NewCost5117                    << '\n');5118  // The cost of the new extensions is greater than the cost of the5119  // old extension plus what we folded.5120  // This is not profitable.5121  if (NewCost > OldCost)5122    return false;5123  if (NewCost < OldCost)5124    return true;5125  // The promotion is neutral but it may help folding the sign extension in5126  // loads for instance.5127  // Check that we did not create an illegal instruction.5128  return isPromotedInstructionLegal(TLI, DL, PromotedOperand);5129}5130 5131/// Given an instruction or constant expr, see if we can fold the operation5132/// into the addressing mode. If so, update the addressing mode and return5133/// true, otherwise return false without modifying AddrMode.5134/// If \p MovedAway is not NULL, it contains the information of whether or5135/// not AddrInst has to be folded into the addressing mode on success.5136/// If \p MovedAway == true, \p AddrInst will not be part of the addressing5137/// because it has been moved away.5138/// Thus AddrInst must not be added in the matched instructions.5139/// This state can happen when AddrInst is a sext, since it may be moved away.5140/// Therefore, AddrInst may not be valid when MovedAway is true and it must5141/// not be referenced anymore.5142bool AddressingModeMatcher::matchOperationAddr(User *AddrInst, unsigned Opcode,5143                                               unsigned Depth,5144                                               bool *MovedAway) {5145  // Avoid exponential behavior on extremely deep expression trees.5146  if (Depth >= 5)5147    return false;5148 5149  // By default, all matched instructions stay in place.5150  if (MovedAway)5151    *MovedAway = false;5152 5153  switch (Opcode) {5154  case Instruction::PtrToInt:5155    // PtrToInt is always a noop, as we know that the int type is pointer sized.5156    return matchAddr(AddrInst->getOperand(0), Depth);5157  case Instruction::IntToPtr: {5158    auto AS = AddrInst->getType()->getPointerAddressSpace();5159    auto PtrTy = MVT::getIntegerVT(DL.getPointerSizeInBits(AS));5160    // This inttoptr is a no-op if the integer type is pointer sized.5161    if (TLI.getValueType(DL, AddrInst->getOperand(0)->getType()) == PtrTy)5162      return matchAddr(AddrInst->getOperand(0), Depth);5163    return false;5164  }5165  case Instruction::BitCast:5166    // BitCast is always a noop, and we can handle it as long as it is5167    // int->int or pointer->pointer (we don't want int<->fp or something).5168    if (AddrInst->getOperand(0)->getType()->isIntOrPtrTy() &&5169        // Don't touch identity bitcasts.  These were probably put here by LSR,5170        // and we don't want to mess around with them.  Assume it knows what it5171        // is doing.5172        AddrInst->getOperand(0)->getType() != AddrInst->getType())5173      return matchAddr(AddrInst->getOperand(0), Depth);5174    return false;5175  case Instruction::AddrSpaceCast: {5176    unsigned SrcAS =5177        AddrInst->getOperand(0)->getType()->getPointerAddressSpace();5178    unsigned DestAS = AddrInst->getType()->getPointerAddressSpace();5179    if (TLI.getTargetMachine().isNoopAddrSpaceCast(SrcAS, DestAS))5180      return matchAddr(AddrInst->getOperand(0), Depth);5181    return false;5182  }5183  case Instruction::Add: {5184    // Check to see if we can merge in one operand, then the other.  If so, we5185    // win.5186    ExtAddrMode BackupAddrMode = AddrMode;5187    unsigned OldSize = AddrModeInsts.size();5188    // Start a transaction at this point.5189    // The LHS may match but not the RHS.5190    // Therefore, we need a higher level restoration point to undo partially5191    // matched operation.5192    TypePromotionTransaction::ConstRestorationPt LastKnownGood =5193        TPT.getRestorationPoint();5194 5195    // Try to match an integer constant second to increase its chance of ending5196    // up in `BaseOffs`, resp. decrease its chance of ending up in `BaseReg`.5197    int First = 0, Second = 1;5198    if (isa<ConstantInt>(AddrInst->getOperand(First))5199      && !isa<ConstantInt>(AddrInst->getOperand(Second)))5200        std::swap(First, Second);5201    AddrMode.InBounds = false;5202    if (matchAddr(AddrInst->getOperand(First), Depth + 1) &&5203        matchAddr(AddrInst->getOperand(Second), Depth + 1))5204      return true;5205 5206    // Restore the old addr mode info.5207    AddrMode = BackupAddrMode;5208    AddrModeInsts.resize(OldSize);5209    TPT.rollback(LastKnownGood);5210 5211    // Otherwise this was over-aggressive.  Try merging operands in the opposite5212    // order.5213    if (matchAddr(AddrInst->getOperand(Second), Depth + 1) &&5214        matchAddr(AddrInst->getOperand(First), Depth + 1))5215      return true;5216 5217    // Otherwise we definitely can't merge the ADD in.5218    AddrMode = BackupAddrMode;5219    AddrModeInsts.resize(OldSize);5220    TPT.rollback(LastKnownGood);5221    break;5222  }5223  // case Instruction::Or:5224  //  TODO: We can handle "Or Val, Imm" iff this OR is equivalent to an ADD.5225  // break;5226  case Instruction::Mul:5227  case Instruction::Shl: {5228    // Can only handle X*C and X << C.5229    AddrMode.InBounds = false;5230    ConstantInt *RHS = dyn_cast<ConstantInt>(AddrInst->getOperand(1));5231    if (!RHS || RHS->getBitWidth() > 64)5232      return false;5233    int64_t Scale = Opcode == Instruction::Shl5234                        ? 1LL << RHS->getLimitedValue(RHS->getBitWidth() - 1)5235                        : RHS->getSExtValue();5236 5237    return matchScaledValue(AddrInst->getOperand(0), Scale, Depth);5238  }5239  case Instruction::GetElementPtr: {5240    // Scan the GEP.  We check it if it contains constant offsets and at most5241    // one variable offset.5242    int VariableOperand = -1;5243    unsigned VariableScale = 0;5244 5245    int64_t ConstantOffset = 0;5246    gep_type_iterator GTI = gep_type_begin(AddrInst);5247    for (unsigned i = 1, e = AddrInst->getNumOperands(); i != e; ++i, ++GTI) {5248      if (StructType *STy = GTI.getStructTypeOrNull()) {5249        const StructLayout *SL = DL.getStructLayout(STy);5250        unsigned Idx =5251            cast<ConstantInt>(AddrInst->getOperand(i))->getZExtValue();5252        ConstantOffset += SL->getElementOffset(Idx);5253      } else {5254        TypeSize TS = GTI.getSequentialElementStride(DL);5255        if (TS.isNonZero()) {5256          // The optimisations below currently only work for fixed offsets.5257          if (TS.isScalable())5258            return false;5259          int64_t TypeSize = TS.getFixedValue();5260          if (ConstantInt *CI =5261                  dyn_cast<ConstantInt>(AddrInst->getOperand(i))) {5262            const APInt &CVal = CI->getValue();5263            if (CVal.getSignificantBits() <= 64) {5264              ConstantOffset += CVal.getSExtValue() * TypeSize;5265              continue;5266            }5267          }5268          // We only allow one variable index at the moment.5269          if (VariableOperand != -1)5270            return false;5271 5272          // Remember the variable index.5273          VariableOperand = i;5274          VariableScale = TypeSize;5275        }5276      }5277    }5278 5279    // A common case is for the GEP to only do a constant offset.  In this case,5280    // just add it to the disp field and check validity.5281    if (VariableOperand == -1) {5282      AddrMode.BaseOffs += ConstantOffset;5283      if (matchAddr(AddrInst->getOperand(0), Depth + 1)) {5284          if (!cast<GEPOperator>(AddrInst)->isInBounds())5285            AddrMode.InBounds = false;5286          return true;5287      }5288      AddrMode.BaseOffs -= ConstantOffset;5289 5290      if (EnableGEPOffsetSplit && isa<GetElementPtrInst>(AddrInst) &&5291          TLI.shouldConsiderGEPOffsetSplit() && Depth == 0 &&5292          ConstantOffset > 0) {5293          // Record GEPs with non-zero offsets as candidates for splitting in5294          // the event that the offset cannot fit into the r+i addressing mode.5295          // Simple and common case that only one GEP is used in calculating the5296          // address for the memory access.5297          Value *Base = AddrInst->getOperand(0);5298          auto *BaseI = dyn_cast<Instruction>(Base);5299          auto *GEP = cast<GetElementPtrInst>(AddrInst);5300          if (isa<Argument>(Base) || isa<GlobalValue>(Base) ||5301              (BaseI && !isa<CastInst>(BaseI) &&5302               !isa<GetElementPtrInst>(BaseI))) {5303            // Make sure the parent block allows inserting non-PHI instructions5304            // before the terminator.5305            BasicBlock *Parent = BaseI ? BaseI->getParent()5306                                       : &GEP->getFunction()->getEntryBlock();5307            if (!Parent->getTerminator()->isEHPad())5308            LargeOffsetGEP = std::make_pair(GEP, ConstantOffset);5309          }5310      }5311 5312      return false;5313    }5314 5315    // Save the valid addressing mode in case we can't match.5316    ExtAddrMode BackupAddrMode = AddrMode;5317    unsigned OldSize = AddrModeInsts.size();5318 5319    // See if the scale and offset amount is valid for this target.5320    AddrMode.BaseOffs += ConstantOffset;5321    if (!cast<GEPOperator>(AddrInst)->isInBounds())5322      AddrMode.InBounds = false;5323 5324    // Match the base operand of the GEP.5325    if (!matchAddr(AddrInst->getOperand(0), Depth + 1)) {5326      // If it couldn't be matched, just stuff the value in a register.5327      if (AddrMode.HasBaseReg) {5328        AddrMode = BackupAddrMode;5329        AddrModeInsts.resize(OldSize);5330        return false;5331      }5332      AddrMode.HasBaseReg = true;5333      AddrMode.BaseReg = AddrInst->getOperand(0);5334    }5335 5336    // Match the remaining variable portion of the GEP.5337    if (!matchScaledValue(AddrInst->getOperand(VariableOperand), VariableScale,5338                          Depth)) {5339      // If it couldn't be matched, try stuffing the base into a register5340      // instead of matching it, and retrying the match of the scale.5341      AddrMode = BackupAddrMode;5342      AddrModeInsts.resize(OldSize);5343      if (AddrMode.HasBaseReg)5344        return false;5345      AddrMode.HasBaseReg = true;5346      AddrMode.BaseReg = AddrInst->getOperand(0);5347      AddrMode.BaseOffs += ConstantOffset;5348      if (!matchScaledValue(AddrInst->getOperand(VariableOperand),5349                            VariableScale, Depth)) {5350        // If even that didn't work, bail.5351        AddrMode = BackupAddrMode;5352        AddrModeInsts.resize(OldSize);5353        return false;5354      }5355    }5356 5357    return true;5358  }5359  case Instruction::SExt:5360  case Instruction::ZExt: {5361    Instruction *Ext = dyn_cast<Instruction>(AddrInst);5362    if (!Ext)5363      return false;5364 5365    // Try to move this ext out of the way of the addressing mode.5366    // Ask for a method for doing so.5367    TypePromotionHelper::Action TPH =5368        TypePromotionHelper::getAction(Ext, InsertedInsts, TLI, PromotedInsts);5369    if (!TPH)5370      return false;5371 5372    TypePromotionTransaction::ConstRestorationPt LastKnownGood =5373        TPT.getRestorationPoint();5374    unsigned CreatedInstsCost = 0;5375    unsigned ExtCost = !TLI.isExtFree(Ext);5376    Value *PromotedOperand =5377        TPH(Ext, TPT, PromotedInsts, CreatedInstsCost, nullptr, nullptr, TLI);5378    // SExt has been moved away.5379    // Thus either it will be rematched later in the recursive calls or it is5380    // gone. Anyway, we must not fold it into the addressing mode at this point.5381    // E.g.,5382    // op = add opnd, 15383    // idx = ext op5384    // addr = gep base, idx5385    // is now:5386    // promotedOpnd = ext opnd            <- no match here5387    // op = promoted_add promotedOpnd, 1  <- match (later in recursive calls)5388    // addr = gep base, op                <- match5389    if (MovedAway)5390      *MovedAway = true;5391 5392    assert(PromotedOperand &&5393           "TypePromotionHelper should have filtered out those cases");5394 5395    ExtAddrMode BackupAddrMode = AddrMode;5396    unsigned OldSize = AddrModeInsts.size();5397 5398    if (!matchAddr(PromotedOperand, Depth) ||5399        // The total of the new cost is equal to the cost of the created5400        // instructions.5401        // The total of the old cost is equal to the cost of the extension plus5402        // what we have saved in the addressing mode.5403        !isPromotionProfitable(CreatedInstsCost,5404                               ExtCost + (AddrModeInsts.size() - OldSize),5405                               PromotedOperand)) {5406      AddrMode = BackupAddrMode;5407      AddrModeInsts.resize(OldSize);5408      LLVM_DEBUG(dbgs() << "Sign extension does not pay off: rollback\n");5409      TPT.rollback(LastKnownGood);5410      return false;5411    }5412 5413    // SExt has been deleted. Make sure it is not referenced by the AddrMode.5414    AddrMode.replaceWith(Ext, PromotedOperand);5415    return true;5416  }5417  case Instruction::Call:5418    if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(AddrInst)) {5419      if (II->getIntrinsicID() == Intrinsic::threadlocal_address) {5420        GlobalValue &GV = cast<GlobalValue>(*II->getArgOperand(0));5421        if (TLI.addressingModeSupportsTLS(GV))5422          return matchAddr(AddrInst->getOperand(0), Depth);5423      }5424    }5425    break;5426  }5427  return false;5428}5429 5430/// If we can, try to add the value of 'Addr' into the current addressing mode.5431/// If Addr can't be added to AddrMode this returns false and leaves AddrMode5432/// unmodified. This assumes that Addr is either a pointer type or intptr_t5433/// for the target.5434///5435bool AddressingModeMatcher::matchAddr(Value *Addr, unsigned Depth) {5436  // Start a transaction at this point that we will rollback if the matching5437  // fails.5438  TypePromotionTransaction::ConstRestorationPt LastKnownGood =5439      TPT.getRestorationPoint();5440  if (ConstantInt *CI = dyn_cast<ConstantInt>(Addr)) {5441    if (CI->getValue().isSignedIntN(64)) {5442      // Check if the addition would result in a signed overflow.5443      int64_t Result;5444      bool Overflow =5445          AddOverflow(AddrMode.BaseOffs, CI->getSExtValue(), Result);5446      if (!Overflow) {5447        // Fold in immediates if legal for the target.5448        AddrMode.BaseOffs = Result;5449        if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))5450          return true;5451        AddrMode.BaseOffs -= CI->getSExtValue();5452      }5453    }5454  } else if (GlobalValue *GV = dyn_cast<GlobalValue>(Addr)) {5455    // If this is a global variable, try to fold it into the addressing mode.5456    if (!AddrMode.BaseGV) {5457      AddrMode.BaseGV = GV;5458      if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))5459        return true;5460      AddrMode.BaseGV = nullptr;5461    }5462  } else if (Instruction *I = dyn_cast<Instruction>(Addr)) {5463    ExtAddrMode BackupAddrMode = AddrMode;5464    unsigned OldSize = AddrModeInsts.size();5465 5466    // Check to see if it is possible to fold this operation.5467    bool MovedAway = false;5468    if (matchOperationAddr(I, I->getOpcode(), Depth, &MovedAway)) {5469      // This instruction may have been moved away. If so, there is nothing5470      // to check here.5471      if (MovedAway)5472        return true;5473      // Okay, it's possible to fold this.  Check to see if it is actually5474      // *profitable* to do so.  We use a simple cost model to avoid increasing5475      // register pressure too much.5476      if (I->hasOneUse() ||5477          isProfitableToFoldIntoAddressingMode(I, BackupAddrMode, AddrMode)) {5478        AddrModeInsts.push_back(I);5479        return true;5480      }5481 5482      // It isn't profitable to do this, roll back.5483      AddrMode = BackupAddrMode;5484      AddrModeInsts.resize(OldSize);5485      TPT.rollback(LastKnownGood);5486    }5487  } else if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Addr)) {5488    if (matchOperationAddr(CE, CE->getOpcode(), Depth))5489      return true;5490    TPT.rollback(LastKnownGood);5491  } else if (isa<ConstantPointerNull>(Addr)) {5492    // Null pointer gets folded without affecting the addressing mode.5493    return true;5494  }5495 5496  // Worse case, the target should support [reg] addressing modes. :)5497  if (!AddrMode.HasBaseReg) {5498    AddrMode.HasBaseReg = true;5499    AddrMode.BaseReg = Addr;5500    // Still check for legality in case the target supports [imm] but not [i+r].5501    if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))5502      return true;5503    AddrMode.HasBaseReg = false;5504    AddrMode.BaseReg = nullptr;5505  }5506 5507  // If the base register is already taken, see if we can do [r+r].5508  if (AddrMode.Scale == 0) {5509    AddrMode.Scale = 1;5510    AddrMode.ScaledReg = Addr;5511    if (TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace))5512      return true;5513    AddrMode.Scale = 0;5514    AddrMode.ScaledReg = nullptr;5515  }5516  // Couldn't match.5517  TPT.rollback(LastKnownGood);5518  return false;5519}5520 5521/// Check to see if all uses of OpVal by the specified inline asm call are due5522/// to memory operands. If so, return true, otherwise return false.5523static bool IsOperandAMemoryOperand(CallInst *CI, InlineAsm *IA, Value *OpVal,5524                                    const TargetLowering &TLI,5525                                    const TargetRegisterInfo &TRI) {5526  const Function *F = CI->getFunction();5527  TargetLowering::AsmOperandInfoVector TargetConstraints =5528      TLI.ParseConstraints(F->getDataLayout(), &TRI, *CI);5529 5530  for (TargetLowering::AsmOperandInfo &OpInfo : TargetConstraints) {5531    // Compute the constraint code and ConstraintType to use.5532    TLI.ComputeConstraintToUse(OpInfo, SDValue());5533 5534    // If this asm operand is our Value*, and if it isn't an indirect memory5535    // operand, we can't fold it!  TODO: Also handle C_Address?5536    if (OpInfo.CallOperandVal == OpVal &&5537        (OpInfo.ConstraintType != TargetLowering::C_Memory ||5538         !OpInfo.isIndirect))5539      return false;5540  }5541 5542  return true;5543}5544 5545/// Recursively walk all the uses of I until we find a memory use.5546/// If we find an obviously non-foldable instruction, return true.5547/// Add accessed addresses and types to MemoryUses.5548static bool FindAllMemoryUses(5549    Instruction *I, SmallVectorImpl<std::pair<Use *, Type *>> &MemoryUses,5550    SmallPtrSetImpl<Instruction *> &ConsideredInsts, const TargetLowering &TLI,5551    const TargetRegisterInfo &TRI, bool OptSize, ProfileSummaryInfo *PSI,5552    BlockFrequencyInfo *BFI, unsigned &SeenInsts) {5553  // If we already considered this instruction, we're done.5554  if (!ConsideredInsts.insert(I).second)5555    return false;5556 5557  // If this is an obviously unfoldable instruction, bail out.5558  if (!MightBeFoldableInst(I))5559    return true;5560 5561  // Loop over all the uses, recursively processing them.5562  for (Use &U : I->uses()) {5563    // Conservatively return true if we're seeing a large number or a deep chain5564    // of users. This avoids excessive compilation times in pathological cases.5565    if (SeenInsts++ >= MaxAddressUsersToScan)5566      return true;5567 5568    Instruction *UserI = cast<Instruction>(U.getUser());5569    if (LoadInst *LI = dyn_cast<LoadInst>(UserI)) {5570      MemoryUses.push_back({&U, LI->getType()});5571      continue;5572    }5573 5574    if (StoreInst *SI = dyn_cast<StoreInst>(UserI)) {5575      if (U.getOperandNo() != StoreInst::getPointerOperandIndex())5576        return true; // Storing addr, not into addr.5577      MemoryUses.push_back({&U, SI->getValueOperand()->getType()});5578      continue;5579    }5580 5581    if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(UserI)) {5582      if (U.getOperandNo() != AtomicRMWInst::getPointerOperandIndex())5583        return true; // Storing addr, not into addr.5584      MemoryUses.push_back({&U, RMW->getValOperand()->getType()});5585      continue;5586    }5587 5588    if (AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(UserI)) {5589      if (U.getOperandNo() != AtomicCmpXchgInst::getPointerOperandIndex())5590        return true; // Storing addr, not into addr.5591      MemoryUses.push_back({&U, CmpX->getCompareOperand()->getType()});5592      continue;5593    }5594 5595    if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(UserI)) {5596      SmallVector<Value *, 2> PtrOps;5597      Type *AccessTy;5598      if (!TLI.getAddrModeArguments(II, PtrOps, AccessTy))5599        return true;5600 5601      if (!find(PtrOps, U.get()))5602        return true;5603 5604      MemoryUses.push_back({&U, AccessTy});5605      continue;5606    }5607 5608    if (CallInst *CI = dyn_cast<CallInst>(UserI)) {5609      if (CI->hasFnAttr(Attribute::Cold)) {5610        // If this is a cold call, we can sink the addressing calculation into5611        // the cold path.  See optimizeCallInst5612        if (!llvm::shouldOptimizeForSize(CI->getParent(), PSI, BFI))5613          continue;5614      }5615 5616      InlineAsm *IA = dyn_cast<InlineAsm>(CI->getCalledOperand());5617      if (!IA)5618        return true;5619 5620      // If this is a memory operand, we're cool, otherwise bail out.5621      if (!IsOperandAMemoryOperand(CI, IA, I, TLI, TRI))5622        return true;5623      continue;5624    }5625 5626    if (FindAllMemoryUses(UserI, MemoryUses, ConsideredInsts, TLI, TRI, OptSize,5627                          PSI, BFI, SeenInsts))5628      return true;5629  }5630 5631  return false;5632}5633 5634static bool FindAllMemoryUses(5635    Instruction *I, SmallVectorImpl<std::pair<Use *, Type *>> &MemoryUses,5636    const TargetLowering &TLI, const TargetRegisterInfo &TRI, bool OptSize,5637    ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI) {5638  unsigned SeenInsts = 0;5639  SmallPtrSet<Instruction *, 16> ConsideredInsts;5640  return FindAllMemoryUses(I, MemoryUses, ConsideredInsts, TLI, TRI, OptSize,5641                           PSI, BFI, SeenInsts);5642}5643 5644 5645/// Return true if Val is already known to be live at the use site that we're5646/// folding it into. If so, there is no cost to include it in the addressing5647/// mode. KnownLive1 and KnownLive2 are two values that we know are live at the5648/// instruction already.5649bool AddressingModeMatcher::valueAlreadyLiveAtInst(Value *Val,5650                                                   Value *KnownLive1,5651                                                   Value *KnownLive2) {5652  // If Val is either of the known-live values, we know it is live!5653  if (Val == nullptr || Val == KnownLive1 || Val == KnownLive2)5654    return true;5655 5656  // All values other than instructions and arguments (e.g. constants) are live.5657  if (!isa<Instruction>(Val) && !isa<Argument>(Val))5658    return true;5659 5660  // If Val is a constant sized alloca in the entry block, it is live, this is5661  // true because it is just a reference to the stack/frame pointer, which is5662  // live for the whole function.5663  if (AllocaInst *AI = dyn_cast<AllocaInst>(Val))5664    if (AI->isStaticAlloca())5665      return true;5666 5667  // Check to see if this value is already used in the memory instruction's5668  // block.  If so, it's already live into the block at the very least, so we5669  // can reasonably fold it.5670  return Val->isUsedInBasicBlock(MemoryInst->getParent());5671}5672 5673/// It is possible for the addressing mode of the machine to fold the specified5674/// instruction into a load or store that ultimately uses it.5675/// However, the specified instruction has multiple uses.5676/// Given this, it may actually increase register pressure to fold it5677/// into the load. For example, consider this code:5678///5679///     X = ...5680///     Y = X+15681///     use(Y)   -> nonload/store5682///     Z = Y+15683///     load Z5684///5685/// In this case, Y has multiple uses, and can be folded into the load of Z5686/// (yielding load [X+2]).  However, doing this will cause both "X" and "X+1" to5687/// be live at the use(Y) line.  If we don't fold Y into load Z, we use one5688/// fewer register.  Since Y can't be folded into "use(Y)" we don't increase the5689/// number of computations either.5690///5691/// Note that this (like most of CodeGenPrepare) is just a rough heuristic.  If5692/// X was live across 'load Z' for other reasons, we actually *would* want to5693/// fold the addressing mode in the Z case.  This would make Y die earlier.5694bool AddressingModeMatcher::isProfitableToFoldIntoAddressingMode(5695    Instruction *I, ExtAddrMode &AMBefore, ExtAddrMode &AMAfter) {5696  if (IgnoreProfitability)5697    return true;5698 5699  // AMBefore is the addressing mode before this instruction was folded into it,5700  // and AMAfter is the addressing mode after the instruction was folded.  Get5701  // the set of registers referenced by AMAfter and subtract out those5702  // referenced by AMBefore: this is the set of values which folding in this5703  // address extends the lifetime of.5704  //5705  // Note that there are only two potential values being referenced here,5706  // BaseReg and ScaleReg (global addresses are always available, as are any5707  // folded immediates).5708  Value *BaseReg = AMAfter.BaseReg, *ScaledReg = AMAfter.ScaledReg;5709 5710  // If the BaseReg or ScaledReg was referenced by the previous addrmode, their5711  // lifetime wasn't extended by adding this instruction.5712  if (valueAlreadyLiveAtInst(BaseReg, AMBefore.BaseReg, AMBefore.ScaledReg))5713    BaseReg = nullptr;5714  if (valueAlreadyLiveAtInst(ScaledReg, AMBefore.BaseReg, AMBefore.ScaledReg))5715    ScaledReg = nullptr;5716 5717  // If folding this instruction (and it's subexprs) didn't extend any live5718  // ranges, we're ok with it.5719  if (!BaseReg && !ScaledReg)5720    return true;5721 5722  // If all uses of this instruction can have the address mode sunk into them,5723  // we can remove the addressing mode and effectively trade one live register5724  // for another (at worst.)  In this context, folding an addressing mode into5725  // the use is just a particularly nice way of sinking it.5726  SmallVector<std::pair<Use *, Type *>, 16> MemoryUses;5727  if (FindAllMemoryUses(I, MemoryUses, TLI, TRI, OptSize, PSI, BFI))5728    return false; // Has a non-memory, non-foldable use!5729 5730  // Now that we know that all uses of this instruction are part of a chain of5731  // computation involving only operations that could theoretically be folded5732  // into a memory use, loop over each of these memory operation uses and see5733  // if they could  *actually* fold the instruction.  The assumption is that5734  // addressing modes are cheap and that duplicating the computation involved5735  // many times is worthwhile, even on a fastpath. For sinking candidates5736  // (i.e. cold call sites), this serves as a way to prevent excessive code5737  // growth since most architectures have some reasonable small and fast way to5738  // compute an effective address.  (i.e LEA on x86)5739  SmallVector<Instruction *, 32> MatchedAddrModeInsts;5740  for (const std::pair<Use *, Type *> &Pair : MemoryUses) {5741    Value *Address = Pair.first->get();5742    Instruction *UserI = cast<Instruction>(Pair.first->getUser());5743    Type *AddressAccessTy = Pair.second;5744    unsigned AS = Address->getType()->getPointerAddressSpace();5745 5746    // Do a match against the root of this address, ignoring profitability. This5747    // will tell us if the addressing mode for the memory operation will5748    // *actually* cover the shared instruction.5749    ExtAddrMode Result;5750    std::pair<AssertingVH<GetElementPtrInst>, int64_t> LargeOffsetGEP(nullptr,5751                                                                      0);5752    TypePromotionTransaction::ConstRestorationPt LastKnownGood =5753        TPT.getRestorationPoint();5754    AddressingModeMatcher Matcher(MatchedAddrModeInsts, TLI, TRI, LI, getDTFn,5755                                  AddressAccessTy, AS, UserI, Result,5756                                  InsertedInsts, PromotedInsts, TPT,5757                                  LargeOffsetGEP, OptSize, PSI, BFI);5758    Matcher.IgnoreProfitability = true;5759    bool Success = Matcher.matchAddr(Address, 0);5760    (void)Success;5761    assert(Success && "Couldn't select *anything*?");5762 5763    // The match was to check the profitability, the changes made are not5764    // part of the original matcher. Therefore, they should be dropped5765    // otherwise the original matcher will not present the right state.5766    TPT.rollback(LastKnownGood);5767 5768    // If the match didn't cover I, then it won't be shared by it.5769    if (!is_contained(MatchedAddrModeInsts, I))5770      return false;5771 5772    MatchedAddrModeInsts.clear();5773  }5774 5775  return true;5776}5777 5778/// Return true if the specified values are defined in a5779/// different basic block than BB.5780static bool IsNonLocalValue(Value *V, BasicBlock *BB) {5781  if (Instruction *I = dyn_cast<Instruction>(V))5782    return I->getParent() != BB;5783  return false;5784}5785 5786// Find an insert position of Addr for MemoryInst. We can't guarantee MemoryInst5787// is the first instruction that will use Addr. So we need to find the first5788// user of Addr in current BB.5789static BasicBlock::iterator findInsertPos(Value *Addr, Instruction *MemoryInst,5790                                          Value *SunkAddr) {5791  if (Addr->hasOneUse())5792    return MemoryInst->getIterator();5793 5794  // We already have a SunkAddr in current BB, but we may need to insert cast5795  // instruction after it.5796  if (SunkAddr) {5797    if (Instruction *AddrInst = dyn_cast<Instruction>(SunkAddr))5798      return std::next(AddrInst->getIterator());5799  }5800 5801  // Find the first user of Addr in current BB.5802  Instruction *Earliest = MemoryInst;5803  for (User *U : Addr->users()) {5804    Instruction *UserInst = dyn_cast<Instruction>(U);5805    if (UserInst && UserInst->getParent() == MemoryInst->getParent()) {5806      if (isa<PHINode>(UserInst) || UserInst->isDebugOrPseudoInst())5807        continue;5808      if (UserInst->comesBefore(Earliest))5809        Earliest = UserInst;5810    }5811  }5812  return Earliest->getIterator();5813}5814 5815/// Sink addressing mode computation immediate before MemoryInst if doing so5816/// can be done without increasing register pressure.  The need for the5817/// register pressure constraint means this can end up being an all or nothing5818/// decision for all uses of the same addressing computation.5819///5820/// Load and Store Instructions often have addressing modes that can do5821/// significant amounts of computation. As such, instruction selection will try5822/// to get the load or store to do as much computation as possible for the5823/// program. The problem is that isel can only see within a single block. As5824/// such, we sink as much legal addressing mode work into the block as possible.5825///5826/// This method is used to optimize both load/store and inline asms with memory5827/// operands.  It's also used to sink addressing computations feeding into cold5828/// call sites into their (cold) basic block.5829///5830/// The motivation for handling sinking into cold blocks is that doing so can5831/// both enable other address mode sinking (by satisfying the register pressure5832/// constraint above), and reduce register pressure globally (by removing the5833/// addressing mode computation from the fast path entirely.).5834bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr,5835                                        Type *AccessTy, unsigned AddrSpace) {5836  Value *Repl = Addr;5837 5838  // Try to collapse single-value PHI nodes.  This is necessary to undo5839  // unprofitable PRE transformations.5840  SmallVector<Value *, 8> worklist;5841  SmallPtrSet<Value *, 16> Visited;5842  worklist.push_back(Addr);5843 5844  // Use a worklist to iteratively look through PHI and select nodes, and5845  // ensure that the addressing mode obtained from the non-PHI/select roots of5846  // the graph are compatible.5847  bool PhiOrSelectSeen = false;5848  SmallVector<Instruction *, 16> AddrModeInsts;5849  AddressingModeCombiner AddrModes(*DL, Addr);5850  TypePromotionTransaction TPT(RemovedInsts);5851  TypePromotionTransaction::ConstRestorationPt LastKnownGood =5852      TPT.getRestorationPoint();5853  while (!worklist.empty()) {5854    Value *V = worklist.pop_back_val();5855 5856    // We allow traversing cyclic Phi nodes.5857    // In case of success after this loop we ensure that traversing through5858    // Phi nodes ends up with all cases to compute address of the form5859    //    BaseGV + Base + Scale * Index + Offset5860    // where Scale and Offset are constans and BaseGV, Base and Index5861    // are exactly the same Values in all cases.5862    // It means that BaseGV, Scale and Offset dominate our memory instruction5863    // and have the same value as they had in address computation represented5864    // as Phi. So we can safely sink address computation to memory instruction.5865    if (!Visited.insert(V).second)5866      continue;5867 5868    // For a PHI node, push all of its incoming values.5869    if (PHINode *P = dyn_cast<PHINode>(V)) {5870      append_range(worklist, P->incoming_values());5871      PhiOrSelectSeen = true;5872      continue;5873    }5874    // Similar for select.5875    if (SelectInst *SI = dyn_cast<SelectInst>(V)) {5876      worklist.push_back(SI->getFalseValue());5877      worklist.push_back(SI->getTrueValue());5878      PhiOrSelectSeen = true;5879      continue;5880    }5881 5882    // For non-PHIs, determine the addressing mode being computed.  Note that5883    // the result may differ depending on what other uses our candidate5884    // addressing instructions might have.5885    AddrModeInsts.clear();5886    std::pair<AssertingVH<GetElementPtrInst>, int64_t> LargeOffsetGEP(nullptr,5887                                                                      0);5888    // Defer the query (and possible computation of) the dom tree to point of5889    // actual use.  It's expected that most address matches don't actually need5890    // the domtree.5891    auto getDTFn = [MemoryInst, this]() -> const DominatorTree & {5892      Function *F = MemoryInst->getParent()->getParent();5893      return this->getDT(*F);5894    };5895    ExtAddrMode NewAddrMode = AddressingModeMatcher::Match(5896        V, AccessTy, AddrSpace, MemoryInst, AddrModeInsts, *TLI, *LI, getDTFn,5897        *TRI, InsertedInsts, PromotedInsts, TPT, LargeOffsetGEP, OptSize, PSI,5898        BFI.get());5899 5900    GetElementPtrInst *GEP = LargeOffsetGEP.first;5901    if (GEP && !NewGEPBases.count(GEP)) {5902      // If splitting the underlying data structure can reduce the offset of a5903      // GEP, collect the GEP.  Skip the GEPs that are the new bases of5904      // previously split data structures.5905      LargeOffsetGEPMap[GEP->getPointerOperand()].push_back(LargeOffsetGEP);5906      LargeOffsetGEPID.insert(std::make_pair(GEP, LargeOffsetGEPID.size()));5907    }5908 5909    NewAddrMode.OriginalValue = V;5910    if (!AddrModes.addNewAddrMode(NewAddrMode))5911      break;5912  }5913 5914  // Try to combine the AddrModes we've collected. If we couldn't collect any,5915  // or we have multiple but either couldn't combine them or combining them5916  // wouldn't do anything useful, bail out now.5917  if (!AddrModes.combineAddrModes()) {5918    TPT.rollback(LastKnownGood);5919    return false;5920  }5921  bool Modified = TPT.commit();5922 5923  // Get the combined AddrMode (or the only AddrMode, if we only had one).5924  ExtAddrMode AddrMode = AddrModes.getAddrMode();5925 5926  // If all the instructions matched are already in this BB, don't do anything.5927  // If we saw a Phi node then it is not local definitely, and if we saw a5928  // select then we want to push the address calculation past it even if it's5929  // already in this BB.5930  if (!PhiOrSelectSeen && none_of(AddrModeInsts, [&](Value *V) {5931        return IsNonLocalValue(V, MemoryInst->getParent());5932      })) {5933    LLVM_DEBUG(dbgs() << "CGP: Found      local addrmode: " << AddrMode5934                      << "\n");5935    return Modified;5936  }5937 5938  // Now that we determined the addressing expression we want to use and know5939  // that we have to sink it into this block.  Check to see if we have already5940  // done this for some other load/store instr in this block.  If so, reuse5941  // the computation.  Before attempting reuse, check if the address is valid5942  // as it may have been erased.5943 5944  WeakTrackingVH SunkAddrVH = SunkAddrs[Addr];5945 5946  Value *SunkAddr = SunkAddrVH.pointsToAliveValue() ? SunkAddrVH : nullptr;5947  Type *IntPtrTy = DL->getIntPtrType(Addr->getType());5948 5949  // The current BB may be optimized multiple times, we can't guarantee the5950  // reuse of Addr happens later, call findInsertPos to find an appropriate5951  // insert position.5952  auto InsertPos = findInsertPos(Addr, MemoryInst, SunkAddr);5953 5954  // TODO: Adjust insert point considering (Base|Scaled)Reg if possible.5955  if (!SunkAddr) {5956    auto &DT = getDT(*MemoryInst->getFunction());5957    if ((AddrMode.BaseReg && !DT.dominates(AddrMode.BaseReg, &*InsertPos)) ||5958        (AddrMode.ScaledReg && !DT.dominates(AddrMode.ScaledReg, &*InsertPos)))5959      return Modified;5960  }5961 5962  IRBuilder<> Builder(MemoryInst->getParent(), InsertPos);5963 5964  if (SunkAddr) {5965    LLVM_DEBUG(dbgs() << "CGP: Reusing nonlocal addrmode: " << AddrMode5966                      << " for " << *MemoryInst << "\n");5967    if (SunkAddr->getType() != Addr->getType()) {5968      if (SunkAddr->getType()->getPointerAddressSpace() !=5969              Addr->getType()->getPointerAddressSpace() &&5970          !DL->isNonIntegralPointerType(Addr->getType())) {5971        // There are two reasons the address spaces might not match: a no-op5972        // addrspacecast, or a ptrtoint/inttoptr pair. Either way, we emit a5973        // ptrtoint/inttoptr pair to ensure we match the original semantics.5974        // TODO: allow bitcast between different address space pointers with the5975        // same size.5976        SunkAddr = Builder.CreatePtrToInt(SunkAddr, IntPtrTy, "sunkaddr");5977        SunkAddr =5978            Builder.CreateIntToPtr(SunkAddr, Addr->getType(), "sunkaddr");5979      } else5980        SunkAddr = Builder.CreatePointerCast(SunkAddr, Addr->getType());5981    }5982  } else if (AddrSinkUsingGEPs || (!AddrSinkUsingGEPs.getNumOccurrences() &&5983                                   SubtargetInfo->addrSinkUsingGEPs())) {5984    // By default, we use the GEP-based method when AA is used later. This5985    // prevents new inttoptr/ptrtoint pairs from degrading AA capabilities.5986    LLVM_DEBUG(dbgs() << "CGP: SINKING nonlocal addrmode: " << AddrMode5987                      << " for " << *MemoryInst << "\n");5988    Value *ResultPtr = nullptr, *ResultIndex = nullptr;5989 5990    // First, find the pointer.5991    if (AddrMode.BaseReg && AddrMode.BaseReg->getType()->isPointerTy()) {5992      ResultPtr = AddrMode.BaseReg;5993      AddrMode.BaseReg = nullptr;5994    }5995 5996    if (AddrMode.Scale && AddrMode.ScaledReg->getType()->isPointerTy()) {5997      // We can't add more than one pointer together, nor can we scale a5998      // pointer (both of which seem meaningless).5999      if (ResultPtr || AddrMode.Scale != 1)6000        return Modified;6001 6002      ResultPtr = AddrMode.ScaledReg;6003      AddrMode.Scale = 0;6004    }6005 6006    // It is only safe to sign extend the BaseReg if we know that the math6007    // required to create it did not overflow before we extend it. Since6008    // the original IR value was tossed in favor of a constant back when6009    // the AddrMode was created we need to bail out gracefully if widths6010    // do not match instead of extending it.6011    //6012    // (See below for code to add the scale.)6013    if (AddrMode.Scale) {6014      Type *ScaledRegTy = AddrMode.ScaledReg->getType();6015      if (cast<IntegerType>(IntPtrTy)->getBitWidth() >6016          cast<IntegerType>(ScaledRegTy)->getBitWidth())6017        return Modified;6018    }6019 6020    GlobalValue *BaseGV = AddrMode.BaseGV;6021    if (BaseGV != nullptr) {6022      if (ResultPtr)6023        return Modified;6024 6025      if (BaseGV->isThreadLocal()) {6026        ResultPtr = Builder.CreateThreadLocalAddress(BaseGV);6027      } else {6028        ResultPtr = BaseGV;6029      }6030    }6031 6032    // If the real base value actually came from an inttoptr, then the matcher6033    // will look through it and provide only the integer value. In that case,6034    // use it here.6035    if (!DL->isNonIntegralPointerType(Addr->getType())) {6036      if (!ResultPtr && AddrMode.BaseReg) {6037        ResultPtr = Builder.CreateIntToPtr(AddrMode.BaseReg, Addr->getType(),6038                                           "sunkaddr");6039        AddrMode.BaseReg = nullptr;6040      } else if (!ResultPtr && AddrMode.Scale == 1) {6041        ResultPtr = Builder.CreateIntToPtr(AddrMode.ScaledReg, Addr->getType(),6042                                           "sunkaddr");6043        AddrMode.Scale = 0;6044      }6045    }6046 6047    if (!ResultPtr && !AddrMode.BaseReg && !AddrMode.Scale &&6048        !AddrMode.BaseOffs) {6049      SunkAddr = Constant::getNullValue(Addr->getType());6050    } else if (!ResultPtr) {6051      return Modified;6052    } else {6053      Type *I8PtrTy =6054          Builder.getPtrTy(Addr->getType()->getPointerAddressSpace());6055 6056      // Start with the base register. Do this first so that subsequent address6057      // matching finds it last, which will prevent it from trying to match it6058      // as the scaled value in case it happens to be a mul. That would be6059      // problematic if we've sunk a different mul for the scale, because then6060      // we'd end up sinking both muls.6061      if (AddrMode.BaseReg) {6062        Value *V = AddrMode.BaseReg;6063        if (V->getType() != IntPtrTy)6064          V = Builder.CreateIntCast(V, IntPtrTy, /*isSigned=*/true, "sunkaddr");6065 6066        ResultIndex = V;6067      }6068 6069      // Add the scale value.6070      if (AddrMode.Scale) {6071        Value *V = AddrMode.ScaledReg;6072        if (V->getType() == IntPtrTy) {6073          // done.6074        } else {6075          assert(cast<IntegerType>(IntPtrTy)->getBitWidth() <6076                     cast<IntegerType>(V->getType())->getBitWidth() &&6077                 "We can't transform if ScaledReg is too narrow");6078          V = Builder.CreateTrunc(V, IntPtrTy, "sunkaddr");6079        }6080 6081        if (AddrMode.Scale != 1)6082          V = Builder.CreateMul(V, ConstantInt::get(IntPtrTy, AddrMode.Scale),6083                                "sunkaddr");6084        if (ResultIndex)6085          ResultIndex = Builder.CreateAdd(ResultIndex, V, "sunkaddr");6086        else6087          ResultIndex = V;6088      }6089 6090      // Add in the Base Offset if present.6091      if (AddrMode.BaseOffs) {6092        Value *V = ConstantInt::getSigned(IntPtrTy, AddrMode.BaseOffs);6093        if (ResultIndex) {6094          // We need to add this separately from the scale above to help with6095          // SDAG consecutive load/store merging.6096          if (ResultPtr->getType() != I8PtrTy)6097            ResultPtr = Builder.CreatePointerCast(ResultPtr, I8PtrTy);6098          ResultPtr = Builder.CreatePtrAdd(ResultPtr, ResultIndex, "sunkaddr",6099                                           AddrMode.InBounds);6100        }6101 6102        ResultIndex = V;6103      }6104 6105      if (!ResultIndex) {6106        auto PtrInst = dyn_cast<Instruction>(ResultPtr);6107        // We know that we have a pointer without any offsets. If this pointer6108        // originates from a different basic block than the current one, we6109        // must be able to recreate it in the current basic block.6110        // We do not support the recreation of any instructions yet.6111        if (PtrInst && PtrInst->getParent() != MemoryInst->getParent())6112          return Modified;6113        SunkAddr = ResultPtr;6114      } else {6115        if (ResultPtr->getType() != I8PtrTy)6116          ResultPtr = Builder.CreatePointerCast(ResultPtr, I8PtrTy);6117        SunkAddr = Builder.CreatePtrAdd(ResultPtr, ResultIndex, "sunkaddr",6118                                        AddrMode.InBounds);6119      }6120 6121      if (SunkAddr->getType() != Addr->getType()) {6122        if (SunkAddr->getType()->getPointerAddressSpace() !=6123                Addr->getType()->getPointerAddressSpace() &&6124            !DL->isNonIntegralPointerType(Addr->getType())) {6125          // There are two reasons the address spaces might not match: a no-op6126          // addrspacecast, or a ptrtoint/inttoptr pair. Either way, we emit a6127          // ptrtoint/inttoptr pair to ensure we match the original semantics.6128          // TODO: allow bitcast between different address space pointers with6129          // the same size.6130          SunkAddr = Builder.CreatePtrToInt(SunkAddr, IntPtrTy, "sunkaddr");6131          SunkAddr =6132              Builder.CreateIntToPtr(SunkAddr, Addr->getType(), "sunkaddr");6133        } else6134          SunkAddr = Builder.CreatePointerCast(SunkAddr, Addr->getType());6135      }6136    }6137  } else {6138    // We'd require a ptrtoint/inttoptr down the line, which we can't do for6139    // non-integral pointers, so in that case bail out now.6140    Type *BaseTy = AddrMode.BaseReg ? AddrMode.BaseReg->getType() : nullptr;6141    Type *ScaleTy = AddrMode.Scale ? AddrMode.ScaledReg->getType() : nullptr;6142    PointerType *BasePtrTy = dyn_cast_or_null<PointerType>(BaseTy);6143    PointerType *ScalePtrTy = dyn_cast_or_null<PointerType>(ScaleTy);6144    if (DL->isNonIntegralPointerType(Addr->getType()) ||6145        (BasePtrTy && DL->isNonIntegralPointerType(BasePtrTy)) ||6146        (ScalePtrTy && DL->isNonIntegralPointerType(ScalePtrTy)) ||6147        (AddrMode.BaseGV &&6148         DL->isNonIntegralPointerType(AddrMode.BaseGV->getType())))6149      return Modified;6150 6151    LLVM_DEBUG(dbgs() << "CGP: SINKING nonlocal addrmode: " << AddrMode6152                      << " for " << *MemoryInst << "\n");6153    Type *IntPtrTy = DL->getIntPtrType(Addr->getType());6154    Value *Result = nullptr;6155 6156    // Start with the base register. Do this first so that subsequent address6157    // matching finds it last, which will prevent it from trying to match it6158    // as the scaled value in case it happens to be a mul. That would be6159    // problematic if we've sunk a different mul for the scale, because then6160    // we'd end up sinking both muls.6161    if (AddrMode.BaseReg) {6162      Value *V = AddrMode.BaseReg;6163      if (V->getType()->isPointerTy())6164        V = Builder.CreatePtrToInt(V, IntPtrTy, "sunkaddr");6165      if (V->getType() != IntPtrTy)6166        V = Builder.CreateIntCast(V, IntPtrTy, /*isSigned=*/true, "sunkaddr");6167      Result = V;6168    }6169 6170    // Add the scale value.6171    if (AddrMode.Scale) {6172      Value *V = AddrMode.ScaledReg;6173      if (V->getType() == IntPtrTy) {6174        // done.6175      } else if (V->getType()->isPointerTy()) {6176        V = Builder.CreatePtrToInt(V, IntPtrTy, "sunkaddr");6177      } else if (cast<IntegerType>(IntPtrTy)->getBitWidth() <6178                 cast<IntegerType>(V->getType())->getBitWidth()) {6179        V = Builder.CreateTrunc(V, IntPtrTy, "sunkaddr");6180      } else {6181        // It is only safe to sign extend the BaseReg if we know that the math6182        // required to create it did not overflow before we extend it. Since6183        // the original IR value was tossed in favor of a constant back when6184        // the AddrMode was created we need to bail out gracefully if widths6185        // do not match instead of extending it.6186        Instruction *I = dyn_cast_or_null<Instruction>(Result);6187        if (I && (Result != AddrMode.BaseReg))6188          I->eraseFromParent();6189        return Modified;6190      }6191      if (AddrMode.Scale != 1)6192        V = Builder.CreateMul(V, ConstantInt::get(IntPtrTy, AddrMode.Scale),6193                              "sunkaddr");6194      if (Result)6195        Result = Builder.CreateAdd(Result, V, "sunkaddr");6196      else6197        Result = V;6198    }6199 6200    // Add in the BaseGV if present.6201    GlobalValue *BaseGV = AddrMode.BaseGV;6202    if (BaseGV != nullptr) {6203      Value *BaseGVPtr;6204      if (BaseGV->isThreadLocal()) {6205        BaseGVPtr = Builder.CreateThreadLocalAddress(BaseGV);6206      } else {6207        BaseGVPtr = BaseGV;6208      }6209      Value *V = Builder.CreatePtrToInt(BaseGVPtr, IntPtrTy, "sunkaddr");6210      if (Result)6211        Result = Builder.CreateAdd(Result, V, "sunkaddr");6212      else6213        Result = V;6214    }6215 6216    // Add in the Base Offset if present.6217    if (AddrMode.BaseOffs) {6218      Value *V = ConstantInt::getSigned(IntPtrTy, AddrMode.BaseOffs);6219      if (Result)6220        Result = Builder.CreateAdd(Result, V, "sunkaddr");6221      else6222        Result = V;6223    }6224 6225    if (!Result)6226      SunkAddr = Constant::getNullValue(Addr->getType());6227    else6228      SunkAddr = Builder.CreateIntToPtr(Result, Addr->getType(), "sunkaddr");6229  }6230 6231  MemoryInst->replaceUsesOfWith(Repl, SunkAddr);6232  // Store the newly computed address into the cache. In the case we reused a6233  // value, this should be idempotent.6234  SunkAddrs[Addr] = WeakTrackingVH(SunkAddr);6235 6236  // If we have no uses, recursively delete the value and all dead instructions6237  // using it.6238  if (Repl->use_empty()) {6239    resetIteratorIfInvalidatedWhileCalling(CurInstIterator->getParent(), [&]() {6240      RecursivelyDeleteTriviallyDeadInstructions(6241          Repl, TLInfo, nullptr,6242          [&](Value *V) { removeAllAssertingVHReferences(V); });6243    });6244  }6245  ++NumMemoryInsts;6246  return true;6247}6248 6249/// Rewrite GEP input to gather/scatter to enable SelectionDAGBuilder to find6250/// a uniform base to use for ISD::MGATHER/MSCATTER. SelectionDAGBuilder can6251/// only handle a 2 operand GEP in the same basic block or a splat constant6252/// vector. The 2 operands to the GEP must have a scalar pointer and a vector6253/// index.6254///6255/// If the existing GEP has a vector base pointer that is splat, we can look6256/// through the splat to find the scalar pointer. If we can't find a scalar6257/// pointer there's nothing we can do.6258///6259/// If we have a GEP with more than 2 indices where the middle indices are all6260/// zeroes, we can replace it with 2 GEPs where the second has 2 operands.6261///6262/// If the final index isn't a vector or is a splat, we can emit a scalar GEP6263/// followed by a GEP with an all zeroes vector index. This will enable6264/// SelectionDAGBuilder to use the scalar GEP as the uniform base and have a6265/// zero index.6266bool CodeGenPrepare::optimizeGatherScatterInst(Instruction *MemoryInst,6267                                               Value *Ptr) {6268  Value *NewAddr;6269 6270  if (const auto *GEP = dyn_cast<GetElementPtrInst>(Ptr)) {6271    // Don't optimize GEPs that don't have indices.6272    if (!GEP->hasIndices())6273      return false;6274 6275    // If the GEP and the gather/scatter aren't in the same BB, don't optimize.6276    // FIXME: We should support this by sinking the GEP.6277    if (MemoryInst->getParent() != GEP->getParent())6278      return false;6279 6280    SmallVector<Value *, 2> Ops(GEP->operands());6281 6282    bool RewriteGEP = false;6283 6284    if (Ops[0]->getType()->isVectorTy()) {6285      Ops[0] = getSplatValue(Ops[0]);6286      if (!Ops[0])6287        return false;6288      RewriteGEP = true;6289    }6290 6291    unsigned FinalIndex = Ops.size() - 1;6292 6293    // Ensure all but the last index is 0.6294    // FIXME: This isn't strictly required. All that's required is that they are6295    // all scalars or splats.6296    for (unsigned i = 1; i < FinalIndex; ++i) {6297      auto *C = dyn_cast<Constant>(Ops[i]);6298      if (!C)6299        return false;6300      if (isa<VectorType>(C->getType()))6301        C = C->getSplatValue();6302      auto *CI = dyn_cast_or_null<ConstantInt>(C);6303      if (!CI || !CI->isZero())6304        return false;6305      // Scalarize the index if needed.6306      Ops[i] = CI;6307    }6308 6309    // Try to scalarize the final index.6310    if (Ops[FinalIndex]->getType()->isVectorTy()) {6311      if (Value *V = getSplatValue(Ops[FinalIndex])) {6312        auto *C = dyn_cast<ConstantInt>(V);6313        // Don't scalarize all zeros vector.6314        if (!C || !C->isZero()) {6315          Ops[FinalIndex] = V;6316          RewriteGEP = true;6317        }6318      }6319    }6320 6321    // If we made any changes or the we have extra operands, we need to generate6322    // new instructions.6323    if (!RewriteGEP && Ops.size() == 2)6324      return false;6325 6326    auto NumElts = cast<VectorType>(Ptr->getType())->getElementCount();6327 6328    IRBuilder<> Builder(MemoryInst);6329 6330    Type *SourceTy = GEP->getSourceElementType();6331    Type *ScalarIndexTy = DL->getIndexType(Ops[0]->getType()->getScalarType());6332 6333    // If the final index isn't a vector, emit a scalar GEP containing all ops6334    // and a vector GEP with all zeroes final index.6335    if (!Ops[FinalIndex]->getType()->isVectorTy()) {6336      NewAddr = Builder.CreateGEP(SourceTy, Ops[0], ArrayRef(Ops).drop_front());6337      auto *IndexTy = VectorType::get(ScalarIndexTy, NumElts);6338      auto *SecondTy = GetElementPtrInst::getIndexedType(6339          SourceTy, ArrayRef(Ops).drop_front());6340      NewAddr =6341          Builder.CreateGEP(SecondTy, NewAddr, Constant::getNullValue(IndexTy));6342    } else {6343      Value *Base = Ops[0];6344      Value *Index = Ops[FinalIndex];6345 6346      // Create a scalar GEP if there are more than 2 operands.6347      if (Ops.size() != 2) {6348        // Replace the last index with 0.6349        Ops[FinalIndex] =6350            Constant::getNullValue(Ops[FinalIndex]->getType()->getScalarType());6351        Base = Builder.CreateGEP(SourceTy, Base, ArrayRef(Ops).drop_front());6352        SourceTy = GetElementPtrInst::getIndexedType(6353            SourceTy, ArrayRef(Ops).drop_front());6354      }6355 6356      // Now create the GEP with scalar pointer and vector index.6357      NewAddr = Builder.CreateGEP(SourceTy, Base, Index);6358    }6359  } else if (!isa<Constant>(Ptr)) {6360    // Not a GEP, maybe its a splat and we can create a GEP to enable6361    // SelectionDAGBuilder to use it as a uniform base.6362    Value *V = getSplatValue(Ptr);6363    if (!V)6364      return false;6365 6366    auto NumElts = cast<VectorType>(Ptr->getType())->getElementCount();6367 6368    IRBuilder<> Builder(MemoryInst);6369 6370    // Emit a vector GEP with a scalar pointer and all 0s vector index.6371    Type *ScalarIndexTy = DL->getIndexType(V->getType()->getScalarType());6372    auto *IndexTy = VectorType::get(ScalarIndexTy, NumElts);6373    Type *ScalarTy;6374    if (cast<IntrinsicInst>(MemoryInst)->getIntrinsicID() ==6375        Intrinsic::masked_gather) {6376      ScalarTy = MemoryInst->getType()->getScalarType();6377    } else {6378      assert(cast<IntrinsicInst>(MemoryInst)->getIntrinsicID() ==6379             Intrinsic::masked_scatter);6380      ScalarTy = MemoryInst->getOperand(0)->getType()->getScalarType();6381    }6382    NewAddr = Builder.CreateGEP(ScalarTy, V, Constant::getNullValue(IndexTy));6383  } else {6384    // Constant, SelectionDAGBuilder knows to check if its a splat.6385    return false;6386  }6387 6388  MemoryInst->replaceUsesOfWith(Ptr, NewAddr);6389 6390  // If we have no uses, recursively delete the value and all dead instructions6391  // using it.6392  if (Ptr->use_empty())6393    RecursivelyDeleteTriviallyDeadInstructions(6394        Ptr, TLInfo, nullptr,6395        [&](Value *V) { removeAllAssertingVHReferences(V); });6396 6397  return true;6398}6399 6400// This is a helper for CodeGenPrepare::optimizeMulWithOverflow.6401// Check the pattern we are interested in where there are maximum 2 uses6402// of the intrinsic which are the extract instructions.6403static bool matchOverflowPattern(Instruction *&I, ExtractValueInst *&MulExtract,6404                                 ExtractValueInst *&OverflowExtract) {6405  // Bail out if it's more than 2 users:6406  if (I->hasNUsesOrMore(3))6407    return false;6408 6409  for (User *U : I->users()) {6410    auto *Extract = dyn_cast<ExtractValueInst>(U);6411    if (!Extract || Extract->getNumIndices() != 1)6412      return false;6413 6414    unsigned Index = Extract->getIndices()[0];6415    if (Index == 0)6416      MulExtract = Extract;6417    else if (Index == 1)6418      OverflowExtract = Extract;6419    else6420      return false;6421  }6422  return true;6423}6424 6425// Rewrite the mul_with_overflow intrinsic by checking if both of the6426// operands' value ranges are within the legal type. If so, we can optimize the6427// multiplication algorithm. This code is supposed to be written during the step6428// of type legalization, but given that we need to reconstruct the IR which is6429// not doable there, we do it here.6430// The IR after the optimization will look like:6431// entry:6432//   if signed:6433//     ( (lhs_lo>>BW-1) ^ lhs_hi) || ( (rhs_lo>>BW-1) ^ rhs_hi) ? overflow,6434//     overflow_no6435//   else:6436//     (lhs_hi != 0) || (rhs_hi != 0) ? overflow, overflow_no6437// overflow_no:6438// overflow:6439// overflow.res:6440// \returns true if optimization was applied6441// TODO: This optimization can be further improved to optimize branching on6442// overflow where the 'overflow_no' BB can branch directly to the false6443// successor of overflow, but that would add additional complexity so we leave6444// it for future work.6445bool CodeGenPrepare::optimizeMulWithOverflow(Instruction *I, bool IsSigned,6446                                             ModifyDT &ModifiedDT) {6447  // Check if target supports this optimization.6448  if (!TLI->shouldOptimizeMulOverflowWithZeroHighBits(6449          I->getContext(),6450          TLI->getValueType(*DL, I->getType()->getContainedType(0))))6451    return false;6452 6453  ExtractValueInst *MulExtract = nullptr, *OverflowExtract = nullptr;6454  if (!matchOverflowPattern(I, MulExtract, OverflowExtract))6455    return false;6456 6457  // Keep track of the instruction to stop reoptimizing it again.6458  InsertedInsts.insert(I);6459 6460  Value *LHS = I->getOperand(0);6461  Value *RHS = I->getOperand(1);6462  Type *Ty = LHS->getType();6463  unsigned VTHalfBitWidth = Ty->getScalarSizeInBits() / 2;6464  Type *LegalTy = Ty->getWithNewBitWidth(VTHalfBitWidth);6465 6466  // New BBs:6467  BasicBlock *OverflowEntryBB =6468      I->getParent()->splitBasicBlock(I, "", /*Before*/ true);6469  OverflowEntryBB->takeName(I->getParent());6470  // Keep the 'br' instruction that is generated as a result of the split to be6471  // erased/replaced later.6472  Instruction *OldTerminator = OverflowEntryBB->getTerminator();6473  BasicBlock *NoOverflowBB =6474      BasicBlock::Create(I->getContext(), "overflow.no", I->getFunction());6475  NoOverflowBB->moveAfter(OverflowEntryBB);6476  BasicBlock *OverflowBB =6477      BasicBlock::Create(I->getContext(), "overflow", I->getFunction());6478  OverflowBB->moveAfter(NoOverflowBB);6479 6480  // BB overflow.entry:6481  IRBuilder<> Builder(OverflowEntryBB);6482  // Extract low and high halves of LHS:6483  Value *LoLHS = Builder.CreateTrunc(LHS, LegalTy, "lo.lhs");6484  Value *HiLHS = Builder.CreateLShr(LHS, VTHalfBitWidth, "lhs.lsr");6485  HiLHS = Builder.CreateTrunc(HiLHS, LegalTy, "hi.lhs");6486 6487  // Extract low and high halves of RHS:6488  Value *LoRHS = Builder.CreateTrunc(RHS, LegalTy, "lo.rhs");6489  Value *HiRHS = Builder.CreateLShr(RHS, VTHalfBitWidth, "rhs.lsr");6490  HiRHS = Builder.CreateTrunc(HiRHS, LegalTy, "hi.rhs");6491 6492  Value *IsAnyBitTrue;6493  if (IsSigned) {6494    Value *SignLoLHS =6495        Builder.CreateAShr(LoLHS, VTHalfBitWidth - 1, "sign.lo.lhs");6496    Value *SignLoRHS =6497        Builder.CreateAShr(LoRHS, VTHalfBitWidth - 1, "sign.lo.rhs");6498    Value *XorLHS = Builder.CreateXor(HiLHS, SignLoLHS);6499    Value *XorRHS = Builder.CreateXor(HiRHS, SignLoRHS);6500    Value *Or = Builder.CreateOr(XorLHS, XorRHS, "or.lhs.rhs");6501    IsAnyBitTrue = Builder.CreateCmp(ICmpInst::ICMP_NE, Or,6502                                     ConstantInt::getNullValue(Or->getType()));6503  } else {6504    Value *CmpLHS = Builder.CreateCmp(ICmpInst::ICMP_NE, HiLHS,6505                                      ConstantInt::getNullValue(LegalTy));6506    Value *CmpRHS = Builder.CreateCmp(ICmpInst::ICMP_NE, HiRHS,6507                                      ConstantInt::getNullValue(LegalTy));6508    IsAnyBitTrue = Builder.CreateOr(CmpLHS, CmpRHS, "or.lhs.rhs");6509  }6510  Builder.CreateCondBr(IsAnyBitTrue, OverflowBB, NoOverflowBB);6511 6512  // BB overflow.no:6513  Builder.SetInsertPoint(NoOverflowBB);6514  Value *ExtLoLHS, *ExtLoRHS;6515  if (IsSigned) {6516    ExtLoLHS = Builder.CreateSExt(LoLHS, Ty, "lo.lhs.ext");6517    ExtLoRHS = Builder.CreateSExt(LoRHS, Ty, "lo.rhs.ext");6518  } else {6519    ExtLoLHS = Builder.CreateZExt(LoLHS, Ty, "lo.lhs.ext");6520    ExtLoRHS = Builder.CreateZExt(LoRHS, Ty, "lo.rhs.ext");6521  }6522 6523  Value *Mul = Builder.CreateMul(ExtLoLHS, ExtLoRHS, "mul.overflow.no");6524 6525  // Create the 'overflow.res' BB to merge the results of6526  // the two paths:6527  BasicBlock *OverflowResBB = I->getParent();6528  OverflowResBB->setName("overflow.res");6529 6530  // BB overflow.no: jump to overflow.res BB6531  Builder.CreateBr(OverflowResBB);6532  // No we don't need the old terminator in overflow.entry BB, erase it:6533  OldTerminator->eraseFromParent();6534 6535  // BB overflow.res:6536  Builder.SetInsertPoint(OverflowResBB, OverflowResBB->getFirstInsertionPt());6537  // Create PHI nodes to merge results from no.overflow BB and overflow BB to6538  // replace the extract instructions.6539  PHINode *OverflowResPHI = Builder.CreatePHI(Ty, 2),6540          *OverflowFlagPHI =6541              Builder.CreatePHI(IntegerType::getInt1Ty(I->getContext()), 2);6542 6543  // Add the incoming values from no.overflow BB and later from overflow BB.6544  OverflowResPHI->addIncoming(Mul, NoOverflowBB);6545  OverflowFlagPHI->addIncoming(ConstantInt::getFalse(I->getContext()),6546                               NoOverflowBB);6547 6548  // Replace all users of MulExtract and OverflowExtract to use the PHI nodes.6549  if (MulExtract) {6550    MulExtract->replaceAllUsesWith(OverflowResPHI);6551    MulExtract->eraseFromParent();6552  }6553  if (OverflowExtract) {6554    OverflowExtract->replaceAllUsesWith(OverflowFlagPHI);6555    OverflowExtract->eraseFromParent();6556  }6557 6558  // Remove the intrinsic from parent (overflow.res BB) as it will be part of6559  // overflow BB6560  I->removeFromParent();6561  // BB overflow:6562  I->insertInto(OverflowBB, OverflowBB->end());6563  Builder.SetInsertPoint(OverflowBB, OverflowBB->end());6564  Value *MulOverflow = Builder.CreateExtractValue(I, {0}, "mul.overflow");6565  Value *OverflowFlag = Builder.CreateExtractValue(I, {1}, "overflow.flag");6566  Builder.CreateBr(OverflowResBB);6567 6568  // Add The Extracted values to the PHINodes in the overflow.res BB.6569  OverflowResPHI->addIncoming(MulOverflow, OverflowBB);6570  OverflowFlagPHI->addIncoming(OverflowFlag, OverflowBB);6571 6572  ModifiedDT = ModifyDT::ModifyBBDT;6573  return true;6574}6575 6576/// If there are any memory operands, use OptimizeMemoryInst to sink their6577/// address computing into the block when possible / profitable.6578bool CodeGenPrepare::optimizeInlineAsmInst(CallInst *CS) {6579  bool MadeChange = false;6580 6581  const TargetRegisterInfo *TRI =6582      TM->getSubtargetImpl(*CS->getFunction())->getRegisterInfo();6583  TargetLowering::AsmOperandInfoVector TargetConstraints =6584      TLI->ParseConstraints(*DL, TRI, *CS);6585  unsigned ArgNo = 0;6586  for (TargetLowering::AsmOperandInfo &OpInfo : TargetConstraints) {6587    // Compute the constraint code and ConstraintType to use.6588    TLI->ComputeConstraintToUse(OpInfo, SDValue());6589 6590    // TODO: Also handle C_Address?6591    if (OpInfo.ConstraintType == TargetLowering::C_Memory &&6592        OpInfo.isIndirect) {6593      Value *OpVal = CS->getArgOperand(ArgNo++);6594      MadeChange |= optimizeMemoryInst(CS, OpVal, OpVal->getType(), ~0u);6595    } else if (OpInfo.Type == InlineAsm::isInput)6596      ArgNo++;6597  }6598 6599  return MadeChange;6600}6601 6602/// Check if all the uses of \p Val are equivalent (or free) zero or6603/// sign extensions.6604static bool hasSameExtUse(Value *Val, const TargetLowering &TLI) {6605  assert(!Val->use_empty() && "Input must have at least one use");6606  const Instruction *FirstUser = cast<Instruction>(*Val->user_begin());6607  bool IsSExt = isa<SExtInst>(FirstUser);6608  Type *ExtTy = FirstUser->getType();6609  for (const User *U : Val->users()) {6610    const Instruction *UI = cast<Instruction>(U);6611    if ((IsSExt && !isa<SExtInst>(UI)) || (!IsSExt && !isa<ZExtInst>(UI)))6612      return false;6613    Type *CurTy = UI->getType();6614    // Same input and output types: Same instruction after CSE.6615    if (CurTy == ExtTy)6616      continue;6617 6618    // If IsSExt is true, we are in this situation:6619    // a = Val6620    // b = sext ty1 a to ty26621    // c = sext ty1 a to ty36622    // Assuming ty2 is shorter than ty3, this could be turned into:6623    // a = Val6624    // b = sext ty1 a to ty26625    // c = sext ty2 b to ty36626    // However, the last sext is not free.6627    if (IsSExt)6628      return false;6629 6630    // This is a ZExt, maybe this is free to extend from one type to another.6631    // In that case, we would not account for a different use.6632    Type *NarrowTy;6633    Type *LargeTy;6634    if (ExtTy->getScalarType()->getIntegerBitWidth() >6635        CurTy->getScalarType()->getIntegerBitWidth()) {6636      NarrowTy = CurTy;6637      LargeTy = ExtTy;6638    } else {6639      NarrowTy = ExtTy;6640      LargeTy = CurTy;6641    }6642 6643    if (!TLI.isZExtFree(NarrowTy, LargeTy))6644      return false;6645  }6646  // All uses are the same or can be derived from one another for free.6647  return true;6648}6649 6650/// Try to speculatively promote extensions in \p Exts and continue6651/// promoting through newly promoted operands recursively as far as doing so is6652/// profitable. Save extensions profitably moved up, in \p ProfitablyMovedExts.6653/// When some promotion happened, \p TPT contains the proper state to revert6654/// them.6655///6656/// \return true if some promotion happened, false otherwise.6657bool CodeGenPrepare::tryToPromoteExts(6658    TypePromotionTransaction &TPT, const SmallVectorImpl<Instruction *> &Exts,6659    SmallVectorImpl<Instruction *> &ProfitablyMovedExts,6660    unsigned CreatedInstsCost) {6661  bool Promoted = false;6662 6663  // Iterate over all the extensions to try to promote them.6664  for (auto *I : Exts) {6665    // Early check if we directly have ext(load).6666    if (isa<LoadInst>(I->getOperand(0))) {6667      ProfitablyMovedExts.push_back(I);6668      continue;6669    }6670 6671    // Check whether or not we want to do any promotion.  The reason we have6672    // this check inside the for loop is to catch the case where an extension6673    // is directly fed by a load because in such case the extension can be moved6674    // up without any promotion on its operands.6675    if (!TLI->enableExtLdPromotion() || DisableExtLdPromotion)6676      return false;6677 6678    // Get the action to perform the promotion.6679    TypePromotionHelper::Action TPH =6680        TypePromotionHelper::getAction(I, InsertedInsts, *TLI, PromotedInsts);6681    // Check if we can promote.6682    if (!TPH) {6683      // Save the current extension as we cannot move up through its operand.6684      ProfitablyMovedExts.push_back(I);6685      continue;6686    }6687 6688    // Save the current state.6689    TypePromotionTransaction::ConstRestorationPt LastKnownGood =6690        TPT.getRestorationPoint();6691    SmallVector<Instruction *, 4> NewExts;6692    unsigned NewCreatedInstsCost = 0;6693    unsigned ExtCost = !TLI->isExtFree(I);6694    // Promote.6695    Value *PromotedVal = TPH(I, TPT, PromotedInsts, NewCreatedInstsCost,6696                             &NewExts, nullptr, *TLI);6697    assert(PromotedVal &&6698           "TypePromotionHelper should have filtered out those cases");6699 6700    // We would be able to merge only one extension in a load.6701    // Therefore, if we have more than 1 new extension we heuristically6702    // cut this search path, because it means we degrade the code quality.6703    // With exactly 2, the transformation is neutral, because we will merge6704    // one extension but leave one. However, we optimistically keep going,6705    // because the new extension may be removed too. Also avoid replacing a6706    // single free extension with multiple extensions, as this increases the6707    // number of IR instructions while not providing any savings.6708    long long TotalCreatedInstsCost = CreatedInstsCost + NewCreatedInstsCost;6709    // FIXME: It would be possible to propagate a negative value instead of6710    // conservatively ceiling it to 0.6711    TotalCreatedInstsCost =6712        std::max((long long)0, (TotalCreatedInstsCost - ExtCost));6713    if (!StressExtLdPromotion &&6714        (TotalCreatedInstsCost > 1 ||6715         !isPromotedInstructionLegal(*TLI, *DL, PromotedVal) ||6716         (ExtCost == 0 && NewExts.size() > 1))) {6717      // This promotion is not profitable, rollback to the previous state, and6718      // save the current extension in ProfitablyMovedExts as the latest6719      // speculative promotion turned out to be unprofitable.6720      TPT.rollback(LastKnownGood);6721      ProfitablyMovedExts.push_back(I);6722      continue;6723    }6724    // Continue promoting NewExts as far as doing so is profitable.6725    SmallVector<Instruction *, 2> NewlyMovedExts;6726    (void)tryToPromoteExts(TPT, NewExts, NewlyMovedExts, TotalCreatedInstsCost);6727    bool NewPromoted = false;6728    for (auto *ExtInst : NewlyMovedExts) {6729      Instruction *MovedExt = cast<Instruction>(ExtInst);6730      Value *ExtOperand = MovedExt->getOperand(0);6731      // If we have reached to a load, we need this extra profitability check6732      // as it could potentially be merged into an ext(load).6733      if (isa<LoadInst>(ExtOperand) &&6734          !(StressExtLdPromotion || NewCreatedInstsCost <= ExtCost ||6735            (ExtOperand->hasOneUse() || hasSameExtUse(ExtOperand, *TLI))))6736        continue;6737 6738      ProfitablyMovedExts.push_back(MovedExt);6739      NewPromoted = true;6740    }6741 6742    // If none of speculative promotions for NewExts is profitable, rollback6743    // and save the current extension (I) as the last profitable extension.6744    if (!NewPromoted) {6745      TPT.rollback(LastKnownGood);6746      ProfitablyMovedExts.push_back(I);6747      continue;6748    }6749    // The promotion is profitable.6750    Promoted = true;6751  }6752  return Promoted;6753}6754 6755/// Merging redundant sexts when one is dominating the other.6756bool CodeGenPrepare::mergeSExts(Function &F) {6757  bool Changed = false;6758  for (auto &Entry : ValToSExtendedUses) {6759    SExts &Insts = Entry.second;6760    SExts CurPts;6761    for (Instruction *Inst : Insts) {6762      if (RemovedInsts.count(Inst) || !isa<SExtInst>(Inst) ||6763          Inst->getOperand(0) != Entry.first)6764        continue;6765      bool inserted = false;6766      for (auto &Pt : CurPts) {6767        if (getDT(F).dominates(Inst, Pt)) {6768          replaceAllUsesWith(Pt, Inst, FreshBBs, IsHugeFunc);6769          RemovedInsts.insert(Pt);6770          Pt->removeFromParent();6771          Pt = Inst;6772          inserted = true;6773          Changed = true;6774          break;6775        }6776        if (!getDT(F).dominates(Pt, Inst))6777          // Give up if we need to merge in a common dominator as the6778          // experiments show it is not profitable.6779          continue;6780        replaceAllUsesWith(Inst, Pt, FreshBBs, IsHugeFunc);6781        RemovedInsts.insert(Inst);6782        Inst->removeFromParent();6783        inserted = true;6784        Changed = true;6785        break;6786      }6787      if (!inserted)6788        CurPts.push_back(Inst);6789    }6790  }6791  return Changed;6792}6793 6794// Splitting large data structures so that the GEPs accessing them can have6795// smaller offsets so that they can be sunk to the same blocks as their users.6796// For example, a large struct starting from %base is split into two parts6797// where the second part starts from %new_base.6798//6799// Before:6800// BB0:6801//   %base     =6802//6803// BB1:6804//   %gep0     = gep %base, off06805//   %gep1     = gep %base, off16806//   %gep2     = gep %base, off26807//6808// BB2:6809//   %load1    = load %gep06810//   %load2    = load %gep16811//   %load3    = load %gep26812//6813// After:6814// BB0:6815//   %base     =6816//   %new_base = gep %base, off06817//6818// BB1:6819//   %new_gep0 = %new_base6820//   %new_gep1 = gep %new_base, off1 - off06821//   %new_gep2 = gep %new_base, off2 - off06822//6823// BB2:6824//   %load1    = load i32, i32* %new_gep06825//   %load2    = load i32, i32* %new_gep16826//   %load3    = load i32, i32* %new_gep26827//6828// %new_gep1 and %new_gep2 can be sunk to BB2 now after the splitting because6829// their offsets are smaller enough to fit into the addressing mode.6830bool CodeGenPrepare::splitLargeGEPOffsets() {6831  bool Changed = false;6832  for (auto &Entry : LargeOffsetGEPMap) {6833    Value *OldBase = Entry.first;6834    SmallVectorImpl<std::pair<AssertingVH<GetElementPtrInst>, int64_t>>6835        &LargeOffsetGEPs = Entry.second;6836    auto compareGEPOffset =6837        [&](const std::pair<GetElementPtrInst *, int64_t> &LHS,6838            const std::pair<GetElementPtrInst *, int64_t> &RHS) {6839          if (LHS.first == RHS.first)6840            return false;6841          if (LHS.second != RHS.second)6842            return LHS.second < RHS.second;6843          return LargeOffsetGEPID[LHS.first] < LargeOffsetGEPID[RHS.first];6844        };6845    // Sorting all the GEPs of the same data structures based on the offsets.6846    llvm::sort(LargeOffsetGEPs, compareGEPOffset);6847    LargeOffsetGEPs.erase(llvm::unique(LargeOffsetGEPs), LargeOffsetGEPs.end());6848    // Skip if all the GEPs have the same offsets.6849    if (LargeOffsetGEPs.front().second == LargeOffsetGEPs.back().second)6850      continue;6851    GetElementPtrInst *BaseGEP = LargeOffsetGEPs.begin()->first;6852    int64_t BaseOffset = LargeOffsetGEPs.begin()->second;6853    Value *NewBaseGEP = nullptr;6854 6855    auto createNewBase = [&](int64_t BaseOffset, Value *OldBase,6856                             GetElementPtrInst *GEP) {6857      LLVMContext &Ctx = GEP->getContext();6858      Type *PtrIdxTy = DL->getIndexType(GEP->getType());6859      Type *I8PtrTy =6860          PointerType::get(Ctx, GEP->getType()->getPointerAddressSpace());6861 6862      BasicBlock::iterator NewBaseInsertPt;6863      BasicBlock *NewBaseInsertBB;6864      if (auto *BaseI = dyn_cast<Instruction>(OldBase)) {6865        // If the base of the struct is an instruction, the new base will be6866        // inserted close to it.6867        NewBaseInsertBB = BaseI->getParent();6868        if (isa<PHINode>(BaseI))6869          NewBaseInsertPt = NewBaseInsertBB->getFirstInsertionPt();6870        else if (InvokeInst *Invoke = dyn_cast<InvokeInst>(BaseI)) {6871          NewBaseInsertBB =6872              SplitEdge(NewBaseInsertBB, Invoke->getNormalDest(), DT.get(), LI);6873          NewBaseInsertPt = NewBaseInsertBB->getFirstInsertionPt();6874        } else6875          NewBaseInsertPt = std::next(BaseI->getIterator());6876      } else {6877        // If the current base is an argument or global value, the new base6878        // will be inserted to the entry block.6879        NewBaseInsertBB = &BaseGEP->getFunction()->getEntryBlock();6880        NewBaseInsertPt = NewBaseInsertBB->getFirstInsertionPt();6881      }6882      IRBuilder<> NewBaseBuilder(NewBaseInsertBB, NewBaseInsertPt);6883      // Create a new base.6884      Value *BaseIndex = ConstantInt::get(PtrIdxTy, BaseOffset);6885      NewBaseGEP = OldBase;6886      if (NewBaseGEP->getType() != I8PtrTy)6887        NewBaseGEP = NewBaseBuilder.CreatePointerCast(NewBaseGEP, I8PtrTy);6888      NewBaseGEP =6889          NewBaseBuilder.CreatePtrAdd(NewBaseGEP, BaseIndex, "splitgep");6890      NewGEPBases.insert(NewBaseGEP);6891      return;6892    };6893 6894    // Check whether all the offsets can be encoded with prefered common base.6895    if (int64_t PreferBase = TLI->getPreferredLargeGEPBaseOffset(6896            LargeOffsetGEPs.front().second, LargeOffsetGEPs.back().second)) {6897      BaseOffset = PreferBase;6898      // Create a new base if the offset of the BaseGEP can be decoded with one6899      // instruction.6900      createNewBase(BaseOffset, OldBase, BaseGEP);6901    }6902 6903    auto *LargeOffsetGEP = LargeOffsetGEPs.begin();6904    while (LargeOffsetGEP != LargeOffsetGEPs.end()) {6905      GetElementPtrInst *GEP = LargeOffsetGEP->first;6906      int64_t Offset = LargeOffsetGEP->second;6907      if (Offset != BaseOffset) {6908        TargetLowering::AddrMode AddrMode;6909        AddrMode.HasBaseReg = true;6910        AddrMode.BaseOffs = Offset - BaseOffset;6911        // The result type of the GEP might not be the type of the memory6912        // access.6913        if (!TLI->isLegalAddressingMode(*DL, AddrMode,6914                                        GEP->getResultElementType(),6915                                        GEP->getAddressSpace())) {6916          // We need to create a new base if the offset to the current base is6917          // too large to fit into the addressing mode. So, a very large struct6918          // may be split into several parts.6919          BaseGEP = GEP;6920          BaseOffset = Offset;6921          NewBaseGEP = nullptr;6922        }6923      }6924 6925      // Generate a new GEP to replace the current one.6926      Type *PtrIdxTy = DL->getIndexType(GEP->getType());6927 6928      if (!NewBaseGEP) {6929        // Create a new base if we don't have one yet.  Find the insertion6930        // pointer for the new base first.6931        createNewBase(BaseOffset, OldBase, GEP);6932      }6933 6934      IRBuilder<> Builder(GEP);6935      Value *NewGEP = NewBaseGEP;6936      if (Offset != BaseOffset) {6937        // Calculate the new offset for the new GEP.6938        Value *Index = ConstantInt::get(PtrIdxTy, Offset - BaseOffset);6939        NewGEP = Builder.CreatePtrAdd(NewBaseGEP, Index);6940      }6941      replaceAllUsesWith(GEP, NewGEP, FreshBBs, IsHugeFunc);6942      LargeOffsetGEPID.erase(GEP);6943      LargeOffsetGEP = LargeOffsetGEPs.erase(LargeOffsetGEP);6944      GEP->eraseFromParent();6945      Changed = true;6946    }6947  }6948  return Changed;6949}6950 6951bool CodeGenPrepare::optimizePhiType(6952    PHINode *I, SmallPtrSetImpl<PHINode *> &Visited,6953    SmallPtrSetImpl<Instruction *> &DeletedInstrs) {6954  // We are looking for a collection on interconnected phi nodes that together6955  // only use loads/bitcasts and are used by stores/bitcasts, and the bitcasts6956  // are of the same type. Convert the whole set of nodes to the type of the6957  // bitcast.6958  Type *PhiTy = I->getType();6959  Type *ConvertTy = nullptr;6960  if (Visited.count(I) ||6961      (!I->getType()->isIntegerTy() && !I->getType()->isFloatingPointTy()))6962    return false;6963 6964  SmallVector<Instruction *, 4> Worklist;6965  Worklist.push_back(cast<Instruction>(I));6966  SmallPtrSet<PHINode *, 4> PhiNodes;6967  SmallPtrSet<ConstantData *, 4> Constants;6968  PhiNodes.insert(I);6969  Visited.insert(I);6970  SmallPtrSet<Instruction *, 4> Defs;6971  SmallPtrSet<Instruction *, 4> Uses;6972  // This works by adding extra bitcasts between load/stores and removing6973  // existing bicasts. If we have a phi(bitcast(load)) or a store(bitcast(phi))6974  // we can get in the situation where we remove a bitcast in one iteration6975  // just to add it again in the next. We need to ensure that at least one6976  // bitcast we remove are anchored to something that will not change back.6977  bool AnyAnchored = false;6978 6979  while (!Worklist.empty()) {6980    Instruction *II = Worklist.pop_back_val();6981 6982    if (auto *Phi = dyn_cast<PHINode>(II)) {6983      // Handle Defs, which might also be PHI's6984      for (Value *V : Phi->incoming_values()) {6985        if (auto *OpPhi = dyn_cast<PHINode>(V)) {6986          if (!PhiNodes.count(OpPhi)) {6987            if (!Visited.insert(OpPhi).second)6988              return false;6989            PhiNodes.insert(OpPhi);6990            Worklist.push_back(OpPhi);6991          }6992        } else if (auto *OpLoad = dyn_cast<LoadInst>(V)) {6993          if (!OpLoad->isSimple())6994            return false;6995          if (Defs.insert(OpLoad).second)6996            Worklist.push_back(OpLoad);6997        } else if (auto *OpEx = dyn_cast<ExtractElementInst>(V)) {6998          if (Defs.insert(OpEx).second)6999            Worklist.push_back(OpEx);7000        } else if (auto *OpBC = dyn_cast<BitCastInst>(V)) {7001          if (!ConvertTy)7002            ConvertTy = OpBC->getOperand(0)->getType();7003          if (OpBC->getOperand(0)->getType() != ConvertTy)7004            return false;7005          if (Defs.insert(OpBC).second) {7006            Worklist.push_back(OpBC);7007            AnyAnchored |= !isa<LoadInst>(OpBC->getOperand(0)) &&7008                           !isa<ExtractElementInst>(OpBC->getOperand(0));7009          }7010        } else if (auto *OpC = dyn_cast<ConstantData>(V))7011          Constants.insert(OpC);7012        else7013          return false;7014      }7015    }7016 7017    // Handle uses which might also be phi's7018    for (User *V : II->users()) {7019      if (auto *OpPhi = dyn_cast<PHINode>(V)) {7020        if (!PhiNodes.count(OpPhi)) {7021          if (Visited.count(OpPhi))7022            return false;7023          PhiNodes.insert(OpPhi);7024          Visited.insert(OpPhi);7025          Worklist.push_back(OpPhi);7026        }7027      } else if (auto *OpStore = dyn_cast<StoreInst>(V)) {7028        if (!OpStore->isSimple() || OpStore->getOperand(0) != II)7029          return false;7030        Uses.insert(OpStore);7031      } else if (auto *OpBC = dyn_cast<BitCastInst>(V)) {7032        if (!ConvertTy)7033          ConvertTy = OpBC->getType();7034        if (OpBC->getType() != ConvertTy)7035          return false;7036        Uses.insert(OpBC);7037        AnyAnchored |=7038            any_of(OpBC->users(), [](User *U) { return !isa<StoreInst>(U); });7039      } else {7040        return false;7041      }7042    }7043  }7044 7045  if (!ConvertTy || !AnyAnchored ||7046      !TLI->shouldConvertPhiType(PhiTy, ConvertTy))7047    return false;7048 7049  LLVM_DEBUG(dbgs() << "Converting " << *I << "\n  and connected nodes to "7050                    << *ConvertTy << "\n");7051 7052  // Create all the new phi nodes of the new type, and bitcast any loads to the7053  // correct type.7054  ValueToValueMap ValMap;7055  for (ConstantData *C : Constants)7056    ValMap[C] = ConstantExpr::getBitCast(C, ConvertTy);7057  for (Instruction *D : Defs) {7058    if (isa<BitCastInst>(D)) {7059      ValMap[D] = D->getOperand(0);7060      DeletedInstrs.insert(D);7061    } else {7062      BasicBlock::iterator insertPt = std::next(D->getIterator());7063      ValMap[D] = new BitCastInst(D, ConvertTy, D->getName() + ".bc", insertPt);7064    }7065  }7066  for (PHINode *Phi : PhiNodes)7067    ValMap[Phi] = PHINode::Create(ConvertTy, Phi->getNumIncomingValues(),7068                                  Phi->getName() + ".tc", Phi->getIterator());7069  // Pipe together all the PhiNodes.7070  for (PHINode *Phi : PhiNodes) {7071    PHINode *NewPhi = cast<PHINode>(ValMap[Phi]);7072    for (int i = 0, e = Phi->getNumIncomingValues(); i < e; i++)7073      NewPhi->addIncoming(ValMap[Phi->getIncomingValue(i)],7074                          Phi->getIncomingBlock(i));7075    Visited.insert(NewPhi);7076  }7077  // And finally pipe up the stores and bitcasts7078  for (Instruction *U : Uses) {7079    if (isa<BitCastInst>(U)) {7080      DeletedInstrs.insert(U);7081      replaceAllUsesWith(U, ValMap[U->getOperand(0)], FreshBBs, IsHugeFunc);7082    } else {7083      U->setOperand(0, new BitCastInst(ValMap[U->getOperand(0)], PhiTy, "bc",7084                                       U->getIterator()));7085    }7086  }7087 7088  // Save the removed phis to be deleted later.7089  DeletedInstrs.insert_range(PhiNodes);7090  return true;7091}7092 7093bool CodeGenPrepare::optimizePhiTypes(Function &F) {7094  if (!OptimizePhiTypes)7095    return false;7096 7097  bool Changed = false;7098  SmallPtrSet<PHINode *, 4> Visited;7099  SmallPtrSet<Instruction *, 4> DeletedInstrs;7100 7101  // Attempt to optimize all the phis in the functions to the correct type.7102  for (auto &BB : F)7103    for (auto &Phi : BB.phis())7104      Changed |= optimizePhiType(&Phi, Visited, DeletedInstrs);7105 7106  // Remove any old phi's that have been converted.7107  for (auto *I : DeletedInstrs) {7108    replaceAllUsesWith(I, PoisonValue::get(I->getType()), FreshBBs, IsHugeFunc);7109    I->eraseFromParent();7110  }7111 7112  return Changed;7113}7114 7115/// Return true, if an ext(load) can be formed from an extension in7116/// \p MovedExts.7117bool CodeGenPrepare::canFormExtLd(7118    const SmallVectorImpl<Instruction *> &MovedExts, LoadInst *&LI,7119    Instruction *&Inst, bool HasPromoted) {7120  for (auto *MovedExtInst : MovedExts) {7121    if (isa<LoadInst>(MovedExtInst->getOperand(0))) {7122      LI = cast<LoadInst>(MovedExtInst->getOperand(0));7123      Inst = MovedExtInst;7124      break;7125    }7126  }7127  if (!LI)7128    return false;7129 7130  // If they're already in the same block, there's nothing to do.7131  // Make the cheap checks first if we did not promote.7132  // If we promoted, we need to check if it is indeed profitable.7133  if (!HasPromoted && LI->getParent() == Inst->getParent())7134    return false;7135 7136  return TLI->isExtLoad(LI, Inst, *DL);7137}7138 7139/// Move a zext or sext fed by a load into the same basic block as the load,7140/// unless conditions are unfavorable. This allows SelectionDAG to fold the7141/// extend into the load.7142///7143/// E.g.,7144/// \code7145/// %ld = load i32* %addr7146/// %add = add nuw i32 %ld, 47147/// %zext = zext i32 %add to i647148// \endcode7149/// =>7150/// \code7151/// %ld = load i32* %addr7152/// %zext = zext i32 %ld to i647153/// %add = add nuw i64 %zext, 47154/// \encode7155/// Note that the promotion in %add to i64 is done in tryToPromoteExts(), which7156/// allow us to match zext(load i32*) to i64.7157///7158/// Also, try to promote the computations used to obtain a sign extended7159/// value used into memory accesses.7160/// E.g.,7161/// \code7162/// a = add nsw i32 b, 37163/// d = sext i32 a to i647164/// e = getelementptr ..., i64 d7165/// \endcode7166/// =>7167/// \code7168/// f = sext i32 b to i647169/// a = add nsw i64 f, 37170/// e = getelementptr ..., i64 a7171/// \endcode7172///7173/// \p Inst[in/out] the extension may be modified during the process if some7174/// promotions apply.7175bool CodeGenPrepare::optimizeExt(Instruction *&Inst) {7176  bool AllowPromotionWithoutCommonHeader = false;7177  /// See if it is an interesting sext operations for the address type7178  /// promotion before trying to promote it, e.g., the ones with the right7179  /// type and used in memory accesses.7180  bool ATPConsiderable = TTI->shouldConsiderAddressTypePromotion(7181      *Inst, AllowPromotionWithoutCommonHeader);7182  TypePromotionTransaction TPT(RemovedInsts);7183  TypePromotionTransaction::ConstRestorationPt LastKnownGood =7184      TPT.getRestorationPoint();7185  SmallVector<Instruction *, 1> Exts;7186  SmallVector<Instruction *, 2> SpeculativelyMovedExts;7187  Exts.push_back(Inst);7188 7189  bool HasPromoted = tryToPromoteExts(TPT, Exts, SpeculativelyMovedExts);7190 7191  // Look for a load being extended.7192  LoadInst *LI = nullptr;7193  Instruction *ExtFedByLoad;7194 7195  // Try to promote a chain of computation if it allows to form an extended7196  // load.7197  if (canFormExtLd(SpeculativelyMovedExts, LI, ExtFedByLoad, HasPromoted)) {7198    assert(LI && ExtFedByLoad && "Expect a valid load and extension");7199    TPT.commit();7200    // Move the extend into the same block as the load.7201    ExtFedByLoad->moveAfter(LI);7202    ++NumExtsMoved;7203    Inst = ExtFedByLoad;7204    return true;7205  }7206 7207  // Continue promoting SExts if known as considerable depending on targets.7208  if (ATPConsiderable &&7209      performAddressTypePromotion(Inst, AllowPromotionWithoutCommonHeader,7210                                  HasPromoted, TPT, SpeculativelyMovedExts))7211    return true;7212 7213  TPT.rollback(LastKnownGood);7214  return false;7215}7216 7217// Perform address type promotion if doing so is profitable.7218// If AllowPromotionWithoutCommonHeader == false, we should find other sext7219// instructions that sign extended the same initial value. However, if7220// AllowPromotionWithoutCommonHeader == true, we expect promoting the7221// extension is just profitable.7222bool CodeGenPrepare::performAddressTypePromotion(7223    Instruction *&Inst, bool AllowPromotionWithoutCommonHeader,7224    bool HasPromoted, TypePromotionTransaction &TPT,7225    SmallVectorImpl<Instruction *> &SpeculativelyMovedExts) {7226  bool Promoted = false;7227  SmallPtrSet<Instruction *, 1> UnhandledExts;7228  bool AllSeenFirst = true;7229  for (auto *I : SpeculativelyMovedExts) {7230    Value *HeadOfChain = I->getOperand(0);7231    DenseMap<Value *, Instruction *>::iterator AlreadySeen =7232        SeenChainsForSExt.find(HeadOfChain);7233    // If there is an unhandled SExt which has the same header, try to promote7234    // it as well.7235    if (AlreadySeen != SeenChainsForSExt.end()) {7236      if (AlreadySeen->second != nullptr)7237        UnhandledExts.insert(AlreadySeen->second);7238      AllSeenFirst = false;7239    }7240  }7241 7242  if (!AllSeenFirst || (AllowPromotionWithoutCommonHeader &&7243                        SpeculativelyMovedExts.size() == 1)) {7244    TPT.commit();7245    if (HasPromoted)7246      Promoted = true;7247    for (auto *I : SpeculativelyMovedExts) {7248      Value *HeadOfChain = I->getOperand(0);7249      SeenChainsForSExt[HeadOfChain] = nullptr;7250      ValToSExtendedUses[HeadOfChain].push_back(I);7251    }7252    // Update Inst as promotion happen.7253    Inst = SpeculativelyMovedExts.pop_back_val();7254  } else {7255    // This is the first chain visited from the header, keep the current chain7256    // as unhandled. Defer to promote this until we encounter another SExt7257    // chain derived from the same header.7258    for (auto *I : SpeculativelyMovedExts) {7259      Value *HeadOfChain = I->getOperand(0);7260      SeenChainsForSExt[HeadOfChain] = Inst;7261    }7262    return false;7263  }7264 7265  if (!AllSeenFirst && !UnhandledExts.empty())7266    for (auto *VisitedSExt : UnhandledExts) {7267      if (RemovedInsts.count(VisitedSExt))7268        continue;7269      TypePromotionTransaction TPT(RemovedInsts);7270      SmallVector<Instruction *, 1> Exts;7271      SmallVector<Instruction *, 2> Chains;7272      Exts.push_back(VisitedSExt);7273      bool HasPromoted = tryToPromoteExts(TPT, Exts, Chains);7274      TPT.commit();7275      if (HasPromoted)7276        Promoted = true;7277      for (auto *I : Chains) {7278        Value *HeadOfChain = I->getOperand(0);7279        // Mark this as handled.7280        SeenChainsForSExt[HeadOfChain] = nullptr;7281        ValToSExtendedUses[HeadOfChain].push_back(I);7282      }7283    }7284  return Promoted;7285}7286 7287bool CodeGenPrepare::optimizeExtUses(Instruction *I) {7288  BasicBlock *DefBB = I->getParent();7289 7290  // If the result of a {s|z}ext and its source are both live out, rewrite all7291  // other uses of the source with result of extension.7292  Value *Src = I->getOperand(0);7293  if (Src->hasOneUse())7294    return false;7295 7296  // Only do this xform if truncating is free.7297  if (!TLI->isTruncateFree(I->getType(), Src->getType()))7298    return false;7299 7300  // Only safe to perform the optimization if the source is also defined in7301  // this block.7302  if (!isa<Instruction>(Src) || DefBB != cast<Instruction>(Src)->getParent())7303    return false;7304 7305  bool DefIsLiveOut = false;7306  for (User *U : I->users()) {7307    Instruction *UI = cast<Instruction>(U);7308 7309    // Figure out which BB this ext is used in.7310    BasicBlock *UserBB = UI->getParent();7311    if (UserBB == DefBB)7312      continue;7313    DefIsLiveOut = true;7314    break;7315  }7316  if (!DefIsLiveOut)7317    return false;7318 7319  // Make sure none of the uses are PHI nodes.7320  for (User *U : Src->users()) {7321    Instruction *UI = cast<Instruction>(U);7322    BasicBlock *UserBB = UI->getParent();7323    if (UserBB == DefBB)7324      continue;7325    // Be conservative. We don't want this xform to end up introducing7326    // reloads just before load / store instructions.7327    if (isa<PHINode>(UI) || isa<LoadInst>(UI) || isa<StoreInst>(UI))7328      return false;7329  }7330 7331  // InsertedTruncs - Only insert one trunc in each block once.7332  DenseMap<BasicBlock *, Instruction *> InsertedTruncs;7333 7334  bool MadeChange = false;7335  for (Use &U : Src->uses()) {7336    Instruction *User = cast<Instruction>(U.getUser());7337 7338    // Figure out which BB this ext is used in.7339    BasicBlock *UserBB = User->getParent();7340    if (UserBB == DefBB)7341      continue;7342 7343    // Both src and def are live in this block. Rewrite the use.7344    Instruction *&InsertedTrunc = InsertedTruncs[UserBB];7345 7346    if (!InsertedTrunc) {7347      BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();7348      assert(InsertPt != UserBB->end());7349      InsertedTrunc = new TruncInst(I, Src->getType(), "");7350      InsertedTrunc->insertBefore(*UserBB, InsertPt);7351      InsertedInsts.insert(InsertedTrunc);7352    }7353 7354    // Replace a use of the {s|z}ext source with a use of the result.7355    U = InsertedTrunc;7356    ++NumExtUses;7357    MadeChange = true;7358  }7359 7360  return MadeChange;7361}7362 7363// Find loads whose uses only use some of the loaded value's bits.  Add an "and"7364// just after the load if the target can fold this into one extload instruction,7365// with the hope of eliminating some of the other later "and" instructions using7366// the loaded value.  "and"s that are made trivially redundant by the insertion7367// of the new "and" are removed by this function, while others (e.g. those whose7368// path from the load goes through a phi) are left for isel to potentially7369// remove.7370//7371// For example:7372//7373// b0:7374//   x = load i327375//   ...7376// b1:7377//   y = and x, 0xff7378//   z = use y7379//7380// becomes:7381//7382// b0:7383//   x = load i327384//   x' = and x, 0xff7385//   ...7386// b1:7387//   z = use x'7388//7389// whereas:7390//7391// b0:7392//   x1 = load i327393//   ...7394// b1:7395//   x2 = load i327396//   ...7397// b2:7398//   x = phi x1, x27399//   y = and x, 0xff7400//7401// becomes (after a call to optimizeLoadExt for each load):7402//7403// b0:7404//   x1 = load i327405//   x1' = and x1, 0xff7406//   ...7407// b1:7408//   x2 = load i327409//   x2' = and x2, 0xff7410//   ...7411// b2:7412//   x = phi x1', x2'7413//   y = and x, 0xff7414bool CodeGenPrepare::optimizeLoadExt(LoadInst *Load) {7415  if (!Load->isSimple() || !Load->getType()->isIntOrPtrTy())7416    return false;7417 7418  // Skip loads we've already transformed.7419  if (Load->hasOneUse() &&7420      InsertedInsts.count(cast<Instruction>(*Load->user_begin())))7421    return false;7422 7423  // Look at all uses of Load, looking through phis, to determine how many bits7424  // of the loaded value are needed.7425  SmallVector<Instruction *, 8> WorkList;7426  SmallPtrSet<Instruction *, 16> Visited;7427  SmallVector<Instruction *, 8> AndsToMaybeRemove;7428  SmallVector<Instruction *, 8> DropFlags;7429  for (auto *U : Load->users())7430    WorkList.push_back(cast<Instruction>(U));7431 7432  EVT LoadResultVT = TLI->getValueType(*DL, Load->getType());7433  unsigned BitWidth = LoadResultVT.getSizeInBits();7434  // If the BitWidth is 0, do not try to optimize the type7435  if (BitWidth == 0)7436    return false;7437 7438  APInt DemandBits(BitWidth, 0);7439  APInt WidestAndBits(BitWidth, 0);7440 7441  while (!WorkList.empty()) {7442    Instruction *I = WorkList.pop_back_val();7443 7444    // Break use-def graph loops.7445    if (!Visited.insert(I).second)7446      continue;7447 7448    // For a PHI node, push all of its users.7449    if (auto *Phi = dyn_cast<PHINode>(I)) {7450      for (auto *U : Phi->users())7451        WorkList.push_back(cast<Instruction>(U));7452      continue;7453    }7454 7455    switch (I->getOpcode()) {7456    case Instruction::And: {7457      auto *AndC = dyn_cast<ConstantInt>(I->getOperand(1));7458      if (!AndC)7459        return false;7460      APInt AndBits = AndC->getValue();7461      DemandBits |= AndBits;7462      // Keep track of the widest and mask we see.7463      if (AndBits.ugt(WidestAndBits))7464        WidestAndBits = AndBits;7465      if (AndBits == WidestAndBits && I->getOperand(0) == Load)7466        AndsToMaybeRemove.push_back(I);7467      break;7468    }7469 7470    case Instruction::Shl: {7471      auto *ShlC = dyn_cast<ConstantInt>(I->getOperand(1));7472      if (!ShlC)7473        return false;7474      uint64_t ShiftAmt = ShlC->getLimitedValue(BitWidth - 1);7475      DemandBits.setLowBits(BitWidth - ShiftAmt);7476      DropFlags.push_back(I);7477      break;7478    }7479 7480    case Instruction::Trunc: {7481      EVT TruncVT = TLI->getValueType(*DL, I->getType());7482      unsigned TruncBitWidth = TruncVT.getSizeInBits();7483      DemandBits.setLowBits(TruncBitWidth);7484      DropFlags.push_back(I);7485      break;7486    }7487 7488    default:7489      return false;7490    }7491  }7492 7493  uint32_t ActiveBits = DemandBits.getActiveBits();7494  // Avoid hoisting (and (load x) 1) since it is unlikely to be folded by the7495  // target even if isLoadExtLegal says an i1 EXTLOAD is valid.  For example,7496  // for the AArch64 target isLoadExtLegal(ZEXTLOAD, i32, i1) returns true, but7497  // (and (load x) 1) is not matched as a single instruction, rather as a LDR7498  // followed by an AND.7499  // TODO: Look into removing this restriction by fixing backends to either7500  // return false for isLoadExtLegal for i1 or have them select this pattern to7501  // a single instruction.7502  //7503  // Also avoid hoisting if we didn't see any ands with the exact DemandBits7504  // mask, since these are the only ands that will be removed by isel.7505  if (ActiveBits <= 1 || !DemandBits.isMask(ActiveBits) ||7506      WidestAndBits != DemandBits)7507    return false;7508 7509  LLVMContext &Ctx = Load->getType()->getContext();7510  Type *TruncTy = Type::getIntNTy(Ctx, ActiveBits);7511  EVT TruncVT = TLI->getValueType(*DL, TruncTy);7512 7513  // Reject cases that won't be matched as extloads.7514  if (!LoadResultVT.bitsGT(TruncVT) || !TruncVT.isRound() ||7515      !TLI->isLoadExtLegal(ISD::ZEXTLOAD, LoadResultVT, TruncVT))7516    return false;7517 7518  IRBuilder<> Builder(Load->getNextNode());7519  auto *NewAnd = cast<Instruction>(7520      Builder.CreateAnd(Load, ConstantInt::get(Ctx, DemandBits)));7521  // Mark this instruction as "inserted by CGP", so that other7522  // optimizations don't touch it.7523  InsertedInsts.insert(NewAnd);7524 7525  // Replace all uses of load with new and (except for the use of load in the7526  // new and itself).7527  replaceAllUsesWith(Load, NewAnd, FreshBBs, IsHugeFunc);7528  NewAnd->setOperand(0, Load);7529 7530  // Remove any and instructions that are now redundant.7531  for (auto *And : AndsToMaybeRemove)7532    // Check that the and mask is the same as the one we decided to put on the7533    // new and.7534    if (cast<ConstantInt>(And->getOperand(1))->getValue() == DemandBits) {7535      replaceAllUsesWith(And, NewAnd, FreshBBs, IsHugeFunc);7536      if (&*CurInstIterator == And)7537        CurInstIterator = std::next(And->getIterator());7538      And->eraseFromParent();7539      ++NumAndUses;7540    }7541 7542  // NSW flags may not longer hold.7543  for (auto *Inst : DropFlags)7544    Inst->setHasNoSignedWrap(false);7545 7546  ++NumAndsAdded;7547  return true;7548}7549 7550/// Check if V (an operand of a select instruction) is an expensive instruction7551/// that is only used once.7552static bool sinkSelectOperand(const TargetTransformInfo *TTI, Value *V) {7553  auto *I = dyn_cast<Instruction>(V);7554  // If it's safe to speculatively execute, then it should not have side7555  // effects; therefore, it's safe to sink and possibly *not* execute.7556  return I && I->hasOneUse() && isSafeToSpeculativelyExecute(I) &&7557         TTI->isExpensiveToSpeculativelyExecute(I);7558}7559 7560/// Returns true if a SelectInst should be turned into an explicit branch.7561static bool isFormingBranchFromSelectProfitable(const TargetTransformInfo *TTI,7562                                                const TargetLowering *TLI,7563                                                SelectInst *SI) {7564  // If even a predictable select is cheap, then a branch can't be cheaper.7565  if (!TLI->isPredictableSelectExpensive())7566    return false;7567 7568  // FIXME: This should use the same heuristics as IfConversion to determine7569  // whether a select is better represented as a branch.7570 7571  // If metadata tells us that the select condition is obviously predictable,7572  // then we want to replace the select with a branch.7573  uint64_t TrueWeight, FalseWeight;7574  if (extractBranchWeights(*SI, TrueWeight, FalseWeight)) {7575    uint64_t Max = std::max(TrueWeight, FalseWeight);7576    uint64_t Sum = TrueWeight + FalseWeight;7577    if (Sum != 0) {7578      auto Probability = BranchProbability::getBranchProbability(Max, Sum);7579      if (Probability > TTI->getPredictableBranchThreshold())7580        return true;7581    }7582  }7583 7584  CmpInst *Cmp = dyn_cast<CmpInst>(SI->getCondition());7585 7586  // If a branch is predictable, an out-of-order CPU can avoid blocking on its7587  // comparison condition. If the compare has more than one use, there's7588  // probably another cmov or setcc around, so it's not worth emitting a branch.7589  if (!Cmp || !Cmp->hasOneUse())7590    return false;7591 7592  // If either operand of the select is expensive and only needed on one side7593  // of the select, we should form a branch.7594  if (sinkSelectOperand(TTI, SI->getTrueValue()) ||7595      sinkSelectOperand(TTI, SI->getFalseValue()))7596    return true;7597 7598  return false;7599}7600 7601/// If \p isTrue is true, return the true value of \p SI, otherwise return7602/// false value of \p SI. If the true/false value of \p SI is defined by any7603/// select instructions in \p Selects, look through the defining select7604/// instruction until the true/false value is not defined in \p Selects.7605static Value *7606getTrueOrFalseValue(SelectInst *SI, bool isTrue,7607                    const SmallPtrSet<const Instruction *, 2> &Selects) {7608  Value *V = nullptr;7609 7610  for (SelectInst *DefSI = SI; DefSI != nullptr && Selects.count(DefSI);7611       DefSI = dyn_cast<SelectInst>(V)) {7612    assert(DefSI->getCondition() == SI->getCondition() &&7613           "The condition of DefSI does not match with SI");7614    V = (isTrue ? DefSI->getTrueValue() : DefSI->getFalseValue());7615  }7616 7617  assert(V && "Failed to get select true/false value");7618  return V;7619}7620 7621bool CodeGenPrepare::optimizeShiftInst(BinaryOperator *Shift) {7622  assert(Shift->isShift() && "Expected a shift");7623 7624  // If this is (1) a vector shift, (2) shifts by scalars are cheaper than7625  // general vector shifts, and (3) the shift amount is a select-of-splatted7626  // values, hoist the shifts before the select:7627  //   shift Op0, (select Cond, TVal, FVal) -->7628  //   select Cond, (shift Op0, TVal), (shift Op0, FVal)7629  //7630  // This is inverting a generic IR transform when we know that the cost of a7631  // general vector shift is more than the cost of 2 shift-by-scalars.7632  // We can't do this effectively in SDAG because we may not be able to7633  // determine if the select operands are splats from within a basic block.7634  Type *Ty = Shift->getType();7635  if (!Ty->isVectorTy() || !TTI->isVectorShiftByScalarCheap(Ty))7636    return false;7637  Value *Cond, *TVal, *FVal;7638  if (!match(Shift->getOperand(1),7639             m_OneUse(m_Select(m_Value(Cond), m_Value(TVal), m_Value(FVal)))))7640    return false;7641  if (!isSplatValue(TVal) || !isSplatValue(FVal))7642    return false;7643 7644  IRBuilder<> Builder(Shift);7645  BinaryOperator::BinaryOps Opcode = Shift->getOpcode();7646  Value *NewTVal = Builder.CreateBinOp(Opcode, Shift->getOperand(0), TVal);7647  Value *NewFVal = Builder.CreateBinOp(Opcode, Shift->getOperand(0), FVal);7648  Value *NewSel = Builder.CreateSelect(Cond, NewTVal, NewFVal);7649  replaceAllUsesWith(Shift, NewSel, FreshBBs, IsHugeFunc);7650  Shift->eraseFromParent();7651  return true;7652}7653 7654bool CodeGenPrepare::optimizeFunnelShift(IntrinsicInst *Fsh) {7655  Intrinsic::ID Opcode = Fsh->getIntrinsicID();7656  assert((Opcode == Intrinsic::fshl || Opcode == Intrinsic::fshr) &&7657         "Expected a funnel shift");7658 7659  // If this is (1) a vector funnel shift, (2) shifts by scalars are cheaper7660  // than general vector shifts, and (3) the shift amount is select-of-splatted7661  // values, hoist the funnel shifts before the select:7662  //   fsh Op0, Op1, (select Cond, TVal, FVal) -->7663  //   select Cond, (fsh Op0, Op1, TVal), (fsh Op0, Op1, FVal)7664  //7665  // This is inverting a generic IR transform when we know that the cost of a7666  // general vector shift is more than the cost of 2 shift-by-scalars.7667  // We can't do this effectively in SDAG because we may not be able to7668  // determine if the select operands are splats from within a basic block.7669  Type *Ty = Fsh->getType();7670  if (!Ty->isVectorTy() || !TTI->isVectorShiftByScalarCheap(Ty))7671    return false;7672  Value *Cond, *TVal, *FVal;7673  if (!match(Fsh->getOperand(2),7674             m_OneUse(m_Select(m_Value(Cond), m_Value(TVal), m_Value(FVal)))))7675    return false;7676  if (!isSplatValue(TVal) || !isSplatValue(FVal))7677    return false;7678 7679  IRBuilder<> Builder(Fsh);7680  Value *X = Fsh->getOperand(0), *Y = Fsh->getOperand(1);7681  Value *NewTVal = Builder.CreateIntrinsic(Opcode, Ty, {X, Y, TVal});7682  Value *NewFVal = Builder.CreateIntrinsic(Opcode, Ty, {X, Y, FVal});7683  Value *NewSel = Builder.CreateSelect(Cond, NewTVal, NewFVal);7684  replaceAllUsesWith(Fsh, NewSel, FreshBBs, IsHugeFunc);7685  Fsh->eraseFromParent();7686  return true;7687}7688 7689/// If we have a SelectInst that will likely profit from branch prediction,7690/// turn it into a branch.7691bool CodeGenPrepare::optimizeSelectInst(SelectInst *SI) {7692  if (DisableSelectToBranch)7693    return false;7694 7695  // If the SelectOptimize pass is enabled, selects have already been optimized.7696  if (!getCGPassBuilderOption().DisableSelectOptimize)7697    return false;7698 7699  // Find all consecutive select instructions that share the same condition.7700  SmallVector<SelectInst *, 2> ASI;7701  ASI.push_back(SI);7702  for (BasicBlock::iterator It = ++BasicBlock::iterator(SI);7703       It != SI->getParent()->end(); ++It) {7704    SelectInst *I = dyn_cast<SelectInst>(&*It);7705    if (I && SI->getCondition() == I->getCondition()) {7706      ASI.push_back(I);7707    } else {7708      break;7709    }7710  }7711 7712  SelectInst *LastSI = ASI.back();7713  // Increment the current iterator to skip all the rest of select instructions7714  // because they will be either "not lowered" or "all lowered" to branch.7715  CurInstIterator = std::next(LastSI->getIterator());7716  // Examine debug-info attached to the consecutive select instructions. They7717  // won't be individually optimised by optimizeInst, so we need to perform7718  // DbgVariableRecord maintenence here instead.7719  for (SelectInst *SI : ArrayRef(ASI).drop_front())7720    fixupDbgVariableRecordsOnInst(*SI);7721 7722  bool VectorCond = !SI->getCondition()->getType()->isIntegerTy(1);7723 7724  // Can we convert the 'select' to CF ?7725  if (VectorCond || SI->getMetadata(LLVMContext::MD_unpredictable))7726    return false;7727 7728  TargetLowering::SelectSupportKind SelectKind;7729  if (SI->getType()->isVectorTy())7730    SelectKind = TargetLowering::ScalarCondVectorVal;7731  else7732    SelectKind = TargetLowering::ScalarValSelect;7733 7734  if (TLI->isSelectSupported(SelectKind) &&7735      (!isFormingBranchFromSelectProfitable(TTI, TLI, SI) ||7736       llvm::shouldOptimizeForSize(SI->getParent(), PSI, BFI.get())))7737    return false;7738 7739  // The DominatorTree needs to be rebuilt by any consumers after this7740  // transformation. We simply reset here rather than setting the ModifiedDT7741  // flag to avoid restarting the function walk in runOnFunction for each7742  // select optimized.7743  DT.reset();7744 7745  // Transform a sequence like this:7746  //    start:7747  //       %cmp = cmp uge i32 %a, %b7748  //       %sel = select i1 %cmp, i32 %c, i32 %d7749  //7750  // Into:7751  //    start:7752  //       %cmp = cmp uge i32 %a, %b7753  //       %cmp.frozen = freeze %cmp7754  //       br i1 %cmp.frozen, label %select.true, label %select.false7755  //    select.true:7756  //       br label %select.end7757  //    select.false:7758  //       br label %select.end7759  //    select.end:7760  //       %sel = phi i32 [ %c, %select.true ], [ %d, %select.false ]7761  //7762  // %cmp should be frozen, otherwise it may introduce undefined behavior.7763  // In addition, we may sink instructions that produce %c or %d from7764  // the entry block into the destination(s) of the new branch.7765  // If the true or false blocks do not contain a sunken instruction, that7766  // block and its branch may be optimized away. In that case, one side of the7767  // first branch will point directly to select.end, and the corresponding PHI7768  // predecessor block will be the start block.7769 7770  // Collect values that go on the true side and the values that go on the false7771  // side.7772  SmallVector<Instruction *> TrueInstrs, FalseInstrs;7773  for (SelectInst *SI : ASI) {7774    if (Value *V = SI->getTrueValue(); sinkSelectOperand(TTI, V))7775      TrueInstrs.push_back(cast<Instruction>(V));7776    if (Value *V = SI->getFalseValue(); sinkSelectOperand(TTI, V))7777      FalseInstrs.push_back(cast<Instruction>(V));7778  }7779 7780  // Split the select block, according to how many (if any) values go on each7781  // side.7782  BasicBlock *StartBlock = SI->getParent();7783  BasicBlock::iterator SplitPt = std::next(BasicBlock::iterator(LastSI));7784  // We should split before any debug-info.7785  SplitPt.setHeadBit(true);7786 7787  IRBuilder<> IB(SI);7788  auto *CondFr = IB.CreateFreeze(SI->getCondition(), SI->getName() + ".frozen");7789 7790  BasicBlock *TrueBlock = nullptr;7791  BasicBlock *FalseBlock = nullptr;7792  BasicBlock *EndBlock = nullptr;7793  BranchInst *TrueBranch = nullptr;7794  BranchInst *FalseBranch = nullptr;7795  if (TrueInstrs.size() == 0) {7796    FalseBranch = cast<BranchInst>(SplitBlockAndInsertIfElse(7797        CondFr, SplitPt, false, nullptr, nullptr, LI));7798    FalseBlock = FalseBranch->getParent();7799    EndBlock = cast<BasicBlock>(FalseBranch->getOperand(0));7800  } else if (FalseInstrs.size() == 0) {7801    TrueBranch = cast<BranchInst>(SplitBlockAndInsertIfThen(7802        CondFr, SplitPt, false, nullptr, nullptr, LI));7803    TrueBlock = TrueBranch->getParent();7804    EndBlock = cast<BasicBlock>(TrueBranch->getOperand(0));7805  } else {7806    Instruction *ThenTerm = nullptr;7807    Instruction *ElseTerm = nullptr;7808    SplitBlockAndInsertIfThenElse(CondFr, SplitPt, &ThenTerm, &ElseTerm,7809                                  nullptr, nullptr, LI);7810    TrueBranch = cast<BranchInst>(ThenTerm);7811    FalseBranch = cast<BranchInst>(ElseTerm);7812    TrueBlock = TrueBranch->getParent();7813    FalseBlock = FalseBranch->getParent();7814    EndBlock = cast<BasicBlock>(TrueBranch->getOperand(0));7815  }7816 7817  EndBlock->setName("select.end");7818  if (TrueBlock)7819    TrueBlock->setName("select.true.sink");7820  if (FalseBlock)7821    FalseBlock->setName(FalseInstrs.size() == 0 ? "select.false"7822                                                : "select.false.sink");7823 7824  if (IsHugeFunc) {7825    if (TrueBlock)7826      FreshBBs.insert(TrueBlock);7827    if (FalseBlock)7828      FreshBBs.insert(FalseBlock);7829    FreshBBs.insert(EndBlock);7830  }7831 7832  BFI->setBlockFreq(EndBlock, BFI->getBlockFreq(StartBlock));7833 7834  static const unsigned MD[] = {7835      LLVMContext::MD_prof, LLVMContext::MD_unpredictable,7836      LLVMContext::MD_make_implicit, LLVMContext::MD_dbg};7837  StartBlock->getTerminator()->copyMetadata(*SI, MD);7838 7839  // Sink expensive instructions into the conditional blocks to avoid executing7840  // them speculatively.7841  for (Instruction *I : TrueInstrs)7842    I->moveBefore(TrueBranch->getIterator());7843  for (Instruction *I : FalseInstrs)7844    I->moveBefore(FalseBranch->getIterator());7845 7846  // If we did not create a new block for one of the 'true' or 'false' paths7847  // of the condition, it means that side of the branch goes to the end block7848  // directly and the path originates from the start block from the point of7849  // view of the new PHI.7850  if (TrueBlock == nullptr)7851    TrueBlock = StartBlock;7852  else if (FalseBlock == nullptr)7853    FalseBlock = StartBlock;7854 7855  SmallPtrSet<const Instruction *, 2> INS(llvm::from_range, ASI);7856  // Use reverse iterator because later select may use the value of the7857  // earlier select, and we need to propagate value through earlier select7858  // to get the PHI operand.7859  for (SelectInst *SI : llvm::reverse(ASI)) {7860    // The select itself is replaced with a PHI Node.7861    PHINode *PN = PHINode::Create(SI->getType(), 2, "");7862    PN->insertBefore(EndBlock->begin());7863    PN->takeName(SI);7864    PN->addIncoming(getTrueOrFalseValue(SI, true, INS), TrueBlock);7865    PN->addIncoming(getTrueOrFalseValue(SI, false, INS), FalseBlock);7866    PN->setDebugLoc(SI->getDebugLoc());7867 7868    replaceAllUsesWith(SI, PN, FreshBBs, IsHugeFunc);7869    SI->eraseFromParent();7870    INS.erase(SI);7871    ++NumSelectsExpanded;7872  }7873 7874  // Instruct OptimizeBlock to skip to the next block.7875  CurInstIterator = StartBlock->end();7876  return true;7877}7878 7879/// Some targets only accept certain types for splat inputs. For example a VDUP7880/// in MVE takes a GPR (integer) register, and the instruction that incorporate7881/// a VDUP (such as a VADD qd, qm, rm) also require a gpr register.7882bool CodeGenPrepare::optimizeShuffleVectorInst(ShuffleVectorInst *SVI) {7883  // Accept shuf(insertelem(undef/poison, val, 0), undef/poison, <0,0,..>) only7884  if (!match(SVI, m_Shuffle(m_InsertElt(m_Undef(), m_Value(), m_ZeroInt()),7885                            m_Undef(), m_ZeroMask())))7886    return false;7887  Type *NewType = TLI->shouldConvertSplatType(SVI);7888  if (!NewType)7889    return false;7890 7891  auto *SVIVecType = cast<FixedVectorType>(SVI->getType());7892  assert(!NewType->isVectorTy() && "Expected a scalar type!");7893  assert(NewType->getScalarSizeInBits() == SVIVecType->getScalarSizeInBits() &&7894         "Expected a type of the same size!");7895  auto *NewVecType =7896      FixedVectorType::get(NewType, SVIVecType->getNumElements());7897 7898  // Create a bitcast (shuffle (insert (bitcast(..))))7899  IRBuilder<> Builder(SVI->getContext());7900  Builder.SetInsertPoint(SVI);7901  Value *BC1 = Builder.CreateBitCast(7902      cast<Instruction>(SVI->getOperand(0))->getOperand(1), NewType);7903  Value *Shuffle = Builder.CreateVectorSplat(NewVecType->getNumElements(), BC1);7904  Value *BC2 = Builder.CreateBitCast(Shuffle, SVIVecType);7905 7906  replaceAllUsesWith(SVI, BC2, FreshBBs, IsHugeFunc);7907  RecursivelyDeleteTriviallyDeadInstructions(7908      SVI, TLInfo, nullptr,7909      [&](Value *V) { removeAllAssertingVHReferences(V); });7910 7911  // Also hoist the bitcast up to its operand if it they are not in the same7912  // block.7913  if (auto *BCI = dyn_cast<Instruction>(BC1))7914    if (auto *Op = dyn_cast<Instruction>(BCI->getOperand(0)))7915      if (BCI->getParent() != Op->getParent() && !isa<PHINode>(Op) &&7916          !Op->isTerminator() && !Op->isEHPad())7917        BCI->moveAfter(Op);7918 7919  return true;7920}7921 7922bool CodeGenPrepare::tryToSinkFreeOperands(Instruction *I) {7923  // If the operands of I can be folded into a target instruction together with7924  // I, duplicate and sink them.7925  SmallVector<Use *, 4> OpsToSink;7926  if (!TTI->isProfitableToSinkOperands(I, OpsToSink))7927    return false;7928 7929  // OpsToSink can contain multiple uses in a use chain (e.g.7930  // (%u1 with %u1 = shufflevector), (%u2 with %u2 = zext %u1)). The dominating7931  // uses must come first, so we process the ops in reverse order so as to not7932  // create invalid IR.7933  BasicBlock *TargetBB = I->getParent();7934  bool Changed = false;7935  SmallVector<Use *, 4> ToReplace;7936  Instruction *InsertPoint = I;7937  DenseMap<const Instruction *, unsigned long> InstOrdering;7938  unsigned long InstNumber = 0;7939  for (const auto &I : *TargetBB)7940    InstOrdering[&I] = InstNumber++;7941 7942  for (Use *U : reverse(OpsToSink)) {7943    auto *UI = cast<Instruction>(U->get());7944    if (isa<PHINode>(UI))7945      continue;7946    if (UI->getParent() == TargetBB) {7947      if (InstOrdering[UI] < InstOrdering[InsertPoint])7948        InsertPoint = UI;7949      continue;7950    }7951    ToReplace.push_back(U);7952  }7953 7954  SetVector<Instruction *> MaybeDead;7955  DenseMap<Instruction *, Instruction *> NewInstructions;7956  for (Use *U : ToReplace) {7957    auto *UI = cast<Instruction>(U->get());7958    Instruction *NI = UI->clone();7959 7960    if (IsHugeFunc) {7961      // Now we clone an instruction, its operands' defs may sink to this BB7962      // now. So we put the operands defs' BBs into FreshBBs to do optimization.7963      for (Value *Op : NI->operands())7964        if (auto *OpDef = dyn_cast<Instruction>(Op))7965          FreshBBs.insert(OpDef->getParent());7966    }7967 7968    NewInstructions[UI] = NI;7969    MaybeDead.insert(UI);7970    LLVM_DEBUG(dbgs() << "Sinking " << *UI << " to user " << *I << "\n");7971    NI->insertBefore(InsertPoint->getIterator());7972    InsertPoint = NI;7973    InsertedInsts.insert(NI);7974 7975    // Update the use for the new instruction, making sure that we update the7976    // sunk instruction uses, if it is part of a chain that has already been7977    // sunk.7978    Instruction *OldI = cast<Instruction>(U->getUser());7979    if (auto It = NewInstructions.find(OldI); It != NewInstructions.end())7980      It->second->setOperand(U->getOperandNo(), NI);7981    else7982      U->set(NI);7983    Changed = true;7984  }7985 7986  // Remove instructions that are dead after sinking.7987  for (auto *I : MaybeDead) {7988    if (!I->hasNUsesOrMore(1)) {7989      LLVM_DEBUG(dbgs() << "Removing dead instruction: " << *I << "\n");7990      I->eraseFromParent();7991    }7992  }7993 7994  return Changed;7995}7996 7997bool CodeGenPrepare::optimizeSwitchType(SwitchInst *SI) {7998  Value *Cond = SI->getCondition();7999  Type *OldType = Cond->getType();8000  LLVMContext &Context = Cond->getContext();8001  EVT OldVT = TLI->getValueType(*DL, OldType);8002  MVT RegType = TLI->getPreferredSwitchConditionType(Context, OldVT);8003  unsigned RegWidth = RegType.getSizeInBits();8004 8005  if (RegWidth <= cast<IntegerType>(OldType)->getBitWidth())8006    return false;8007 8008  // If the register width is greater than the type width, expand the condition8009  // of the switch instruction and each case constant to the width of the8010  // register. By widening the type of the switch condition, subsequent8011  // comparisons (for case comparisons) will not need to be extended to the8012  // preferred register width, so we will potentially eliminate N-1 extends,8013  // where N is the number of cases in the switch.8014  auto *NewType = Type::getIntNTy(Context, RegWidth);8015 8016  // Extend the switch condition and case constants using the target preferred8017  // extend unless the switch condition is a function argument with an extend8018  // attribute. In that case, we can avoid an unnecessary mask/extension by8019  // matching the argument extension instead.8020  Instruction::CastOps ExtType = Instruction::ZExt;8021  // Some targets prefer SExt over ZExt.8022  if (TLI->isSExtCheaperThanZExt(OldVT, RegType))8023    ExtType = Instruction::SExt;8024 8025  if (auto *Arg = dyn_cast<Argument>(Cond)) {8026    if (Arg->hasSExtAttr())8027      ExtType = Instruction::SExt;8028    if (Arg->hasZExtAttr())8029      ExtType = Instruction::ZExt;8030  }8031 8032  auto *ExtInst = CastInst::Create(ExtType, Cond, NewType);8033  ExtInst->insertBefore(SI->getIterator());8034  ExtInst->setDebugLoc(SI->getDebugLoc());8035  SI->setCondition(ExtInst);8036  for (auto Case : SI->cases()) {8037    const APInt &NarrowConst = Case.getCaseValue()->getValue();8038    APInt WideConst = (ExtType == Instruction::ZExt)8039                          ? NarrowConst.zext(RegWidth)8040                          : NarrowConst.sext(RegWidth);8041    Case.setValue(ConstantInt::get(Context, WideConst));8042  }8043 8044  return true;8045}8046 8047bool CodeGenPrepare::optimizeSwitchPhiConstants(SwitchInst *SI) {8048  // The SCCP optimization tends to produce code like this:8049  //   switch(x) { case 42: phi(42, ...) }8050  // Materializing the constant for the phi-argument needs instructions; So we8051  // change the code to:8052  //   switch(x) { case 42: phi(x, ...) }8053 8054  Value *Condition = SI->getCondition();8055  // Avoid endless loop in degenerate case.8056  if (isa<ConstantInt>(*Condition))8057    return false;8058 8059  bool Changed = false;8060  BasicBlock *SwitchBB = SI->getParent();8061  Type *ConditionType = Condition->getType();8062 8063  for (const SwitchInst::CaseHandle &Case : SI->cases()) {8064    ConstantInt *CaseValue = Case.getCaseValue();8065    BasicBlock *CaseBB = Case.getCaseSuccessor();8066    // Set to true if we previously checked that `CaseBB` is only reached by8067    // a single case from this switch.8068    bool CheckedForSinglePred = false;8069    for (PHINode &PHI : CaseBB->phis()) {8070      Type *PHIType = PHI.getType();8071      // If ZExt is free then we can also catch patterns like this:8072      //   switch((i32)x) { case 42: phi((i64)42, ...); }8073      // and replace `(i64)42` with `zext i32 %x to i64`.8074      bool TryZExt =8075          PHIType->isIntegerTy() &&8076          PHIType->getIntegerBitWidth() > ConditionType->getIntegerBitWidth() &&8077          TLI->isZExtFree(ConditionType, PHIType);8078      if (PHIType == ConditionType || TryZExt) {8079        // Set to true to skip this case because of multiple preds.8080        bool SkipCase = false;8081        Value *Replacement = nullptr;8082        for (unsigned I = 0, E = PHI.getNumIncomingValues(); I != E; I++) {8083          Value *PHIValue = PHI.getIncomingValue(I);8084          if (PHIValue != CaseValue) {8085            if (!TryZExt)8086              continue;8087            ConstantInt *PHIValueInt = dyn_cast<ConstantInt>(PHIValue);8088            if (!PHIValueInt ||8089                PHIValueInt->getValue() !=8090                    CaseValue->getValue().zext(PHIType->getIntegerBitWidth()))8091              continue;8092          }8093          if (PHI.getIncomingBlock(I) != SwitchBB)8094            continue;8095          // We cannot optimize if there are multiple case labels jumping to8096          // this block.  This check may get expensive when there are many8097          // case labels so we test for it last.8098          if (!CheckedForSinglePred) {8099            CheckedForSinglePred = true;8100            if (SI->findCaseDest(CaseBB) == nullptr) {8101              SkipCase = true;8102              break;8103            }8104          }8105 8106          if (Replacement == nullptr) {8107            if (PHIValue == CaseValue) {8108              Replacement = Condition;8109            } else {8110              IRBuilder<> Builder(SI);8111              Replacement = Builder.CreateZExt(Condition, PHIType);8112            }8113          }8114          PHI.setIncomingValue(I, Replacement);8115          Changed = true;8116        }8117        if (SkipCase)8118          break;8119      }8120    }8121  }8122  return Changed;8123}8124 8125bool CodeGenPrepare::optimizeSwitchInst(SwitchInst *SI) {8126  bool Changed = optimizeSwitchType(SI);8127  Changed |= optimizeSwitchPhiConstants(SI);8128  return Changed;8129}8130 8131namespace {8132 8133/// Helper class to promote a scalar operation to a vector one.8134/// This class is used to move downward extractelement transition.8135/// E.g.,8136/// a = vector_op <2 x i32>8137/// b = extractelement <2 x i32> a, i32 08138/// c = scalar_op b8139/// store c8140///8141/// =>8142/// a = vector_op <2 x i32>8143/// c = vector_op a (equivalent to scalar_op on the related lane)8144/// * d = extractelement <2 x i32> c, i32 08145/// * store d8146/// Assuming both extractelement and store can be combine, we get rid of the8147/// transition.8148class VectorPromoteHelper {8149  /// DataLayout associated with the current module.8150  const DataLayout &DL;8151 8152  /// Used to perform some checks on the legality of vector operations.8153  const TargetLowering &TLI;8154 8155  /// Used to estimated the cost of the promoted chain.8156  const TargetTransformInfo &TTI;8157 8158  /// The transition being moved downwards.8159  Instruction *Transition;8160 8161  /// The sequence of instructions to be promoted.8162  SmallVector<Instruction *, 4> InstsToBePromoted;8163 8164  /// Cost of combining a store and an extract.8165  unsigned StoreExtractCombineCost;8166 8167  /// Instruction that will be combined with the transition.8168  Instruction *CombineInst = nullptr;8169 8170  /// The instruction that represents the current end of the transition.8171  /// Since we are faking the promotion until we reach the end of the chain8172  /// of computation, we need a way to get the current end of the transition.8173  Instruction *getEndOfTransition() const {8174    if (InstsToBePromoted.empty())8175      return Transition;8176    return InstsToBePromoted.back();8177  }8178 8179  /// Return the index of the original value in the transition.8180  /// E.g., for "extractelement <2 x i32> c, i32 1" the original value,8181  /// c, is at index 0.8182  unsigned getTransitionOriginalValueIdx() const {8183    assert(isa<ExtractElementInst>(Transition) &&8184           "Other kind of transitions are not supported yet");8185    return 0;8186  }8187 8188  /// Return the index of the index in the transition.8189  /// E.g., for "extractelement <2 x i32> c, i32 0" the index8190  /// is at index 1.8191  unsigned getTransitionIdx() const {8192    assert(isa<ExtractElementInst>(Transition) &&8193           "Other kind of transitions are not supported yet");8194    return 1;8195  }8196 8197  /// Get the type of the transition.8198  /// This is the type of the original value.8199  /// E.g., for "extractelement <2 x i32> c, i32 1" the type of the8200  /// transition is <2 x i32>.8201  Type *getTransitionType() const {8202    return Transition->getOperand(getTransitionOriginalValueIdx())->getType();8203  }8204 8205  /// Promote \p ToBePromoted by moving \p Def downward through.8206  /// I.e., we have the following sequence:8207  /// Def = Transition <ty1> a to <ty2>8208  /// b = ToBePromoted <ty2> Def, ...8209  /// =>8210  /// b = ToBePromoted <ty1> a, ...8211  /// Def = Transition <ty1> ToBePromoted to <ty2>8212  void promoteImpl(Instruction *ToBePromoted);8213 8214  /// Check whether or not it is profitable to promote all the8215  /// instructions enqueued to be promoted.8216  bool isProfitableToPromote() {8217    Value *ValIdx = Transition->getOperand(getTransitionOriginalValueIdx());8218    unsigned Index = isa<ConstantInt>(ValIdx)8219                         ? cast<ConstantInt>(ValIdx)->getZExtValue()8220                         : -1;8221    Type *PromotedType = getTransitionType();8222 8223    StoreInst *ST = cast<StoreInst>(CombineInst);8224    unsigned AS = ST->getPointerAddressSpace();8225    // Check if this store is supported.8226    if (!TLI.allowsMisalignedMemoryAccesses(8227            TLI.getValueType(DL, ST->getValueOperand()->getType()), AS,8228            ST->getAlign())) {8229      // If this is not supported, there is no way we can combine8230      // the extract with the store.8231      return false;8232    }8233 8234    // The scalar chain of computation has to pay for the transition8235    // scalar to vector.8236    // The vector chain has to account for the combining cost.8237    enum TargetTransformInfo::TargetCostKind CostKind =8238        TargetTransformInfo::TCK_RecipThroughput;8239    InstructionCost ScalarCost =8240        TTI.getVectorInstrCost(*Transition, PromotedType, CostKind, Index);8241    InstructionCost VectorCost = StoreExtractCombineCost;8242    for (const auto &Inst : InstsToBePromoted) {8243      // Compute the cost.8244      // By construction, all instructions being promoted are arithmetic ones.8245      // Moreover, one argument is a constant that can be viewed as a splat8246      // constant.8247      Value *Arg0 = Inst->getOperand(0);8248      bool IsArg0Constant = isa<UndefValue>(Arg0) || isa<ConstantInt>(Arg0) ||8249                            isa<ConstantFP>(Arg0);8250      TargetTransformInfo::OperandValueInfo Arg0Info, Arg1Info;8251      if (IsArg0Constant)8252        Arg0Info.Kind = TargetTransformInfo::OK_UniformConstantValue;8253      else8254        Arg1Info.Kind = TargetTransformInfo::OK_UniformConstantValue;8255 8256      ScalarCost += TTI.getArithmeticInstrCost(8257          Inst->getOpcode(), Inst->getType(), CostKind, Arg0Info, Arg1Info);8258      VectorCost += TTI.getArithmeticInstrCost(Inst->getOpcode(), PromotedType,8259                                               CostKind, Arg0Info, Arg1Info);8260    }8261    LLVM_DEBUG(8262        dbgs() << "Estimated cost of computation to be promoted:\nScalar: "8263               << ScalarCost << "\nVector: " << VectorCost << '\n');8264    return ScalarCost > VectorCost;8265  }8266 8267  /// Generate a constant vector with \p Val with the same8268  /// number of elements as the transition.8269  /// \p UseSplat defines whether or not \p Val should be replicated8270  /// across the whole vector.8271  /// In other words, if UseSplat == true, we generate <Val, Val, ..., Val>,8272  /// otherwise we generate a vector with as many poison as possible:8273  /// <poison, ..., poison, Val, poison, ..., poison> where \p Val is only8274  /// used at the index of the extract.8275  Value *getConstantVector(Constant *Val, bool UseSplat) const {8276    unsigned ExtractIdx = std::numeric_limits<unsigned>::max();8277    if (!UseSplat) {8278      // If we cannot determine where the constant must be, we have to8279      // use a splat constant.8280      Value *ValExtractIdx = Transition->getOperand(getTransitionIdx());8281      if (ConstantInt *CstVal = dyn_cast<ConstantInt>(ValExtractIdx))8282        ExtractIdx = CstVal->getSExtValue();8283      else8284        UseSplat = true;8285    }8286 8287    ElementCount EC = cast<VectorType>(getTransitionType())->getElementCount();8288    if (UseSplat)8289      return ConstantVector::getSplat(EC, Val);8290 8291    if (!EC.isScalable()) {8292      SmallVector<Constant *, 4> ConstVec;8293      PoisonValue *PoisonVal = PoisonValue::get(Val->getType());8294      for (unsigned Idx = 0; Idx != EC.getKnownMinValue(); ++Idx) {8295        if (Idx == ExtractIdx)8296          ConstVec.push_back(Val);8297        else8298          ConstVec.push_back(PoisonVal);8299      }8300      return ConstantVector::get(ConstVec);8301    } else8302      llvm_unreachable(8303          "Generate scalable vector for non-splat is unimplemented");8304  }8305 8306  /// Check if promoting to a vector type an operand at \p OperandIdx8307  /// in \p Use can trigger undefined behavior.8308  static bool canCauseUndefinedBehavior(const Instruction *Use,8309                                        unsigned OperandIdx) {8310    // This is not safe to introduce undef when the operand is on8311    // the right hand side of a division-like instruction.8312    if (OperandIdx != 1)8313      return false;8314    switch (Use->getOpcode()) {8315    default:8316      return false;8317    case Instruction::SDiv:8318    case Instruction::UDiv:8319    case Instruction::SRem:8320    case Instruction::URem:8321      return true;8322    case Instruction::FDiv:8323    case Instruction::FRem:8324      return !Use->hasNoNaNs();8325    }8326    llvm_unreachable(nullptr);8327  }8328 8329public:8330  VectorPromoteHelper(const DataLayout &DL, const TargetLowering &TLI,8331                      const TargetTransformInfo &TTI, Instruction *Transition,8332                      unsigned CombineCost)8333      : DL(DL), TLI(TLI), TTI(TTI), Transition(Transition),8334        StoreExtractCombineCost(CombineCost) {8335    assert(Transition && "Do not know how to promote null");8336  }8337 8338  /// Check if we can promote \p ToBePromoted to \p Type.8339  bool canPromote(const Instruction *ToBePromoted) const {8340    // We could support CastInst too.8341    return isa<BinaryOperator>(ToBePromoted);8342  }8343 8344  /// Check if it is profitable to promote \p ToBePromoted8345  /// by moving downward the transition through.8346  bool shouldPromote(const Instruction *ToBePromoted) const {8347    // Promote only if all the operands can be statically expanded.8348    // Indeed, we do not want to introduce any new kind of transitions.8349    for (const Use &U : ToBePromoted->operands()) {8350      const Value *Val = U.get();8351      if (Val == getEndOfTransition()) {8352        // If the use is a division and the transition is on the rhs,8353        // we cannot promote the operation, otherwise we may create a8354        // division by zero.8355        if (canCauseUndefinedBehavior(ToBePromoted, U.getOperandNo()))8356          return false;8357        continue;8358      }8359      if (!isa<ConstantInt>(Val) && !isa<UndefValue>(Val) &&8360          !isa<ConstantFP>(Val))8361        return false;8362    }8363    // Check that the resulting operation is legal.8364    int ISDOpcode = TLI.InstructionOpcodeToISD(ToBePromoted->getOpcode());8365    if (!ISDOpcode)8366      return false;8367    return StressStoreExtract ||8368           TLI.isOperationLegalOrCustom(8369               ISDOpcode, TLI.getValueType(DL, getTransitionType(), true));8370  }8371 8372  /// Check whether or not \p Use can be combined8373  /// with the transition.8374  /// I.e., is it possible to do Use(Transition) => AnotherUse?8375  bool canCombine(const Instruction *Use) { return isa<StoreInst>(Use); }8376 8377  /// Record \p ToBePromoted as part of the chain to be promoted.8378  void enqueueForPromotion(Instruction *ToBePromoted) {8379    InstsToBePromoted.push_back(ToBePromoted);8380  }8381 8382  /// Set the instruction that will be combined with the transition.8383  void recordCombineInstruction(Instruction *ToBeCombined) {8384    assert(canCombine(ToBeCombined) && "Unsupported instruction to combine");8385    CombineInst = ToBeCombined;8386  }8387 8388  /// Promote all the instructions enqueued for promotion if it is8389  /// is profitable.8390  /// \return True if the promotion happened, false otherwise.8391  bool promote() {8392    // Check if there is something to promote.8393    // Right now, if we do not have anything to combine with,8394    // we assume the promotion is not profitable.8395    if (InstsToBePromoted.empty() || !CombineInst)8396      return false;8397 8398    // Check cost.8399    if (!StressStoreExtract && !isProfitableToPromote())8400      return false;8401 8402    // Promote.8403    for (auto &ToBePromoted : InstsToBePromoted)8404      promoteImpl(ToBePromoted);8405    InstsToBePromoted.clear();8406    return true;8407  }8408};8409 8410} // end anonymous namespace8411 8412void VectorPromoteHelper::promoteImpl(Instruction *ToBePromoted) {8413  // At this point, we know that all the operands of ToBePromoted but Def8414  // can be statically promoted.8415  // For Def, we need to use its parameter in ToBePromoted:8416  // b = ToBePromoted ty1 a8417  // Def = Transition ty1 b to ty28418  // Move the transition down.8419  // 1. Replace all uses of the promoted operation by the transition.8420  // = ... b => = ... Def.8421  assert(ToBePromoted->getType() == Transition->getType() &&8422         "The type of the result of the transition does not match "8423         "the final type");8424  ToBePromoted->replaceAllUsesWith(Transition);8425  // 2. Update the type of the uses.8426  // b = ToBePromoted ty2 Def => b = ToBePromoted ty1 Def.8427  Type *TransitionTy = getTransitionType();8428  ToBePromoted->mutateType(TransitionTy);8429  // 3. Update all the operands of the promoted operation with promoted8430  // operands.8431  // b = ToBePromoted ty1 Def => b = ToBePromoted ty1 a.8432  for (Use &U : ToBePromoted->operands()) {8433    Value *Val = U.get();8434    Value *NewVal = nullptr;8435    if (Val == Transition)8436      NewVal = Transition->getOperand(getTransitionOriginalValueIdx());8437    else if (isa<UndefValue>(Val) || isa<ConstantInt>(Val) ||8438             isa<ConstantFP>(Val)) {8439      // Use a splat constant if it is not safe to use undef.8440      NewVal = getConstantVector(8441          cast<Constant>(Val),8442          isa<UndefValue>(Val) ||8443              canCauseUndefinedBehavior(ToBePromoted, U.getOperandNo()));8444    } else8445      llvm_unreachable("Did you modified shouldPromote and forgot to update "8446                       "this?");8447    ToBePromoted->setOperand(U.getOperandNo(), NewVal);8448  }8449  Transition->moveAfter(ToBePromoted);8450  Transition->setOperand(getTransitionOriginalValueIdx(), ToBePromoted);8451}8452 8453/// Some targets can do store(extractelement) with one instruction.8454/// Try to push the extractelement towards the stores when the target8455/// has this feature and this is profitable.8456bool CodeGenPrepare::optimizeExtractElementInst(Instruction *Inst) {8457  unsigned CombineCost = std::numeric_limits<unsigned>::max();8458  if (DisableStoreExtract ||8459      (!StressStoreExtract &&8460       !TLI->canCombineStoreAndExtract(Inst->getOperand(0)->getType(),8461                                       Inst->getOperand(1), CombineCost)))8462    return false;8463 8464  // At this point we know that Inst is a vector to scalar transition.8465  // Try to move it down the def-use chain, until:8466  // - We can combine the transition with its single use8467  //   => we got rid of the transition.8468  // - We escape the current basic block8469  //   => we would need to check that we are moving it at a cheaper place and8470  //      we do not do that for now.8471  BasicBlock *Parent = Inst->getParent();8472  LLVM_DEBUG(dbgs() << "Found an interesting transition: " << *Inst << '\n');8473  VectorPromoteHelper VPH(*DL, *TLI, *TTI, Inst, CombineCost);8474  // If the transition has more than one use, assume this is not going to be8475  // beneficial.8476  while (Inst->hasOneUse()) {8477    Instruction *ToBePromoted = cast<Instruction>(*Inst->user_begin());8478    LLVM_DEBUG(dbgs() << "Use: " << *ToBePromoted << '\n');8479 8480    if (ToBePromoted->getParent() != Parent) {8481      LLVM_DEBUG(dbgs() << "Instruction to promote is in a different block ("8482                        << ToBePromoted->getParent()->getName()8483                        << ") than the transition (" << Parent->getName()8484                        << ").\n");8485      return false;8486    }8487 8488    if (VPH.canCombine(ToBePromoted)) {8489      LLVM_DEBUG(dbgs() << "Assume " << *Inst << '\n'8490                        << "will be combined with: " << *ToBePromoted << '\n');8491      VPH.recordCombineInstruction(ToBePromoted);8492      bool Changed = VPH.promote();8493      NumStoreExtractExposed += Changed;8494      return Changed;8495    }8496 8497    LLVM_DEBUG(dbgs() << "Try promoting.\n");8498    if (!VPH.canPromote(ToBePromoted) || !VPH.shouldPromote(ToBePromoted))8499      return false;8500 8501    LLVM_DEBUG(dbgs() << "Promoting is possible... Enqueue for promotion!\n");8502 8503    VPH.enqueueForPromotion(ToBePromoted);8504    Inst = ToBePromoted;8505  }8506  return false;8507}8508 8509/// For the instruction sequence of store below, F and I values8510/// are bundled together as an i64 value before being stored into memory.8511/// Sometimes it is more efficient to generate separate stores for F and I,8512/// which can remove the bitwise instructions or sink them to colder places.8513///8514///   (store (or (zext (bitcast F to i32) to i64),8515///              (shl (zext I to i64), 32)), addr)  -->8516///   (store F, addr) and (store I, addr+4)8517///8518/// Similarly, splitting for other merged store can also be beneficial, like:8519/// For pair of {i32, i32}, i64 store --> two i32 stores.8520/// For pair of {i32, i16}, i64 store --> two i32 stores.8521/// For pair of {i16, i16}, i32 store --> two i16 stores.8522/// For pair of {i16, i8},  i32 store --> two i16 stores.8523/// For pair of {i8, i8},   i16 store --> two i8 stores.8524///8525/// We allow each target to determine specifically which kind of splitting is8526/// supported.8527///8528/// The store patterns are commonly seen from the simple code snippet below8529/// if only std::make_pair(...) is sroa transformed before inlined into hoo.8530///   void goo(const std::pair<int, float> &);8531///   hoo() {8532///     ...8533///     goo(std::make_pair(tmp, ftmp));8534///     ...8535///   }8536///8537/// Although we already have similar splitting in DAG Combine, we duplicate8538/// it in CodeGenPrepare to catch the case in which pattern is across8539/// multiple BBs. The logic in DAG Combine is kept to catch case generated8540/// during code expansion.8541static bool splitMergedValStore(StoreInst &SI, const DataLayout &DL,8542                                const TargetLowering &TLI) {8543  // Handle simple but common cases only.8544  Type *StoreType = SI.getValueOperand()->getType();8545 8546  // The code below assumes shifting a value by <number of bits>,8547  // whereas scalable vectors would have to be shifted by8548  // <2log(vscale) + number of bits> in order to store the8549  // low/high parts. Bailing out for now.8550  if (StoreType->isScalableTy())8551    return false;8552 8553  if (!DL.typeSizeEqualsStoreSize(StoreType) ||8554      DL.getTypeSizeInBits(StoreType) == 0)8555    return false;8556 8557  unsigned HalfValBitSize = DL.getTypeSizeInBits(StoreType) / 2;8558  Type *SplitStoreType = Type::getIntNTy(SI.getContext(), HalfValBitSize);8559  if (!DL.typeSizeEqualsStoreSize(SplitStoreType))8560    return false;8561 8562  // Don't split the store if it is volatile.8563  if (SI.isVolatile())8564    return false;8565 8566  // Match the following patterns:8567  // (store (or (zext LValue to i64),8568  //            (shl (zext HValue to i64), 32)), HalfValBitSize)8569  //  or8570  // (store (or (shl (zext HValue to i64), 32)), HalfValBitSize)8571  //            (zext LValue to i64),8572  // Expect both operands of OR and the first operand of SHL have only8573  // one use.8574  Value *LValue, *HValue;8575  if (!match(SI.getValueOperand(),8576             m_c_Or(m_OneUse(m_ZExt(m_Value(LValue))),8577                    m_OneUse(m_Shl(m_OneUse(m_ZExt(m_Value(HValue))),8578                                   m_SpecificInt(HalfValBitSize))))))8579    return false;8580 8581  // Check LValue and HValue are int with size less or equal than 32.8582  if (!LValue->getType()->isIntegerTy() ||8583      DL.getTypeSizeInBits(LValue->getType()) > HalfValBitSize ||8584      !HValue->getType()->isIntegerTy() ||8585      DL.getTypeSizeInBits(HValue->getType()) > HalfValBitSize)8586    return false;8587 8588  // If LValue/HValue is a bitcast instruction, use the EVT before bitcast8589  // as the input of target query.8590  auto *LBC = dyn_cast<BitCastInst>(LValue);8591  auto *HBC = dyn_cast<BitCastInst>(HValue);8592  EVT LowTy = LBC ? EVT::getEVT(LBC->getOperand(0)->getType())8593                  : EVT::getEVT(LValue->getType());8594  EVT HighTy = HBC ? EVT::getEVT(HBC->getOperand(0)->getType())8595                   : EVT::getEVT(HValue->getType());8596  if (!ForceSplitStore && !TLI.isMultiStoresCheaperThanBitsMerge(LowTy, HighTy))8597    return false;8598 8599  // Start to split store.8600  IRBuilder<> Builder(SI.getContext());8601  Builder.SetInsertPoint(&SI);8602 8603  // If LValue/HValue is a bitcast in another BB, create a new one in current8604  // BB so it may be merged with the splitted stores by dag combiner.8605  if (LBC && LBC->getParent() != SI.getParent())8606    LValue = Builder.CreateBitCast(LBC->getOperand(0), LBC->getType());8607  if (HBC && HBC->getParent() != SI.getParent())8608    HValue = Builder.CreateBitCast(HBC->getOperand(0), HBC->getType());8609 8610  bool IsLE = SI.getDataLayout().isLittleEndian();8611  auto CreateSplitStore = [&](Value *V, bool Upper) {8612    V = Builder.CreateZExtOrBitCast(V, SplitStoreType);8613    Value *Addr = SI.getPointerOperand();8614    Align Alignment = SI.getAlign();8615    const bool IsOffsetStore = (IsLE && Upper) || (!IsLE && !Upper);8616    if (IsOffsetStore) {8617      Addr = Builder.CreateGEP(8618          SplitStoreType, Addr,8619          ConstantInt::get(Type::getInt32Ty(SI.getContext()), 1));8620 8621      // When splitting the store in half, naturally one half will retain the8622      // alignment of the original wider store, regardless of whether it was8623      // over-aligned or not, while the other will require adjustment.8624      Alignment = commonAlignment(Alignment, HalfValBitSize / 8);8625    }8626    Builder.CreateAlignedStore(V, Addr, Alignment);8627  };8628 8629  CreateSplitStore(LValue, false);8630  CreateSplitStore(HValue, true);8631 8632  // Delete the old store.8633  SI.eraseFromParent();8634  return true;8635}8636 8637// Return true if the GEP has two operands, the first operand is of a sequential8638// type, and the second operand is a constant.8639static bool GEPSequentialConstIndexed(GetElementPtrInst *GEP) {8640  gep_type_iterator I = gep_type_begin(*GEP);8641  return GEP->getNumOperands() == 2 && I.isSequential() &&8642         isa<ConstantInt>(GEP->getOperand(1));8643}8644 8645// Try unmerging GEPs to reduce liveness interference (register pressure) across8646// IndirectBr edges. Since IndirectBr edges tend to touch on many blocks,8647// reducing liveness interference across those edges benefits global register8648// allocation. Currently handles only certain cases.8649//8650// For example, unmerge %GEPI and %UGEPI as below.8651//8652// ---------- BEFORE ----------8653// SrcBlock:8654//   ...8655//   %GEPIOp = ...8656//   ...8657//   %GEPI = gep %GEPIOp, Idx8658//   ...8659//   indirectbr ... [ label %DstB0, label %DstB1, ... label %DstBi ... ]8660//   (* %GEPI is alive on the indirectbr edges due to other uses ahead)8661//   (* %GEPIOp is alive on the indirectbr edges only because of it's used by8662//   %UGEPI)8663//8664// DstB0: ... (there may be a gep similar to %UGEPI to be unmerged)8665// DstB1: ... (there may be a gep similar to %UGEPI to be unmerged)8666// ...8667//8668// DstBi:8669//   ...8670//   %UGEPI = gep %GEPIOp, UIdx8671// ...8672// ---------------------------8673//8674// ---------- AFTER ----------8675// SrcBlock:8676//   ... (same as above)8677//    (* %GEPI is still alive on the indirectbr edges)8678//    (* %GEPIOp is no longer alive on the indirectbr edges as a result of the8679//    unmerging)8680// ...8681//8682// DstBi:8683//   ...8684//   %UGEPI = gep %GEPI, (UIdx-Idx)8685//   ...8686// ---------------------------8687//8688// The register pressure on the IndirectBr edges is reduced because %GEPIOp is8689// no longer alive on them.8690//8691// We try to unmerge GEPs here in CodGenPrepare, as opposed to limiting merging8692// of GEPs in the first place in InstCombiner::visitGetElementPtrInst() so as8693// not to disable further simplications and optimizations as a result of GEP8694// merging.8695//8696// Note this unmerging may increase the length of the data flow critical path8697// (the path from %GEPIOp to %UGEPI would go through %GEPI), which is a tradeoff8698// between the register pressure and the length of data-flow critical8699// path. Restricting this to the uncommon IndirectBr case would minimize the8700// impact of potentially longer critical path, if any, and the impact on compile8701// time.8702static bool tryUnmergingGEPsAcrossIndirectBr(GetElementPtrInst *GEPI,8703                                             const TargetTransformInfo *TTI) {8704  BasicBlock *SrcBlock = GEPI->getParent();8705  // Check that SrcBlock ends with an IndirectBr. If not, give up. The common8706  // (non-IndirectBr) cases exit early here.8707  if (!isa<IndirectBrInst>(SrcBlock->getTerminator()))8708    return false;8709  // Check that GEPI is a simple gep with a single constant index.8710  if (!GEPSequentialConstIndexed(GEPI))8711    return false;8712  ConstantInt *GEPIIdx = cast<ConstantInt>(GEPI->getOperand(1));8713  // Check that GEPI is a cheap one.8714  if (TTI->getIntImmCost(GEPIIdx->getValue(), GEPIIdx->getType(),8715                         TargetTransformInfo::TCK_SizeAndLatency) >8716      TargetTransformInfo::TCC_Basic)8717    return false;8718  Value *GEPIOp = GEPI->getOperand(0);8719  // Check that GEPIOp is an instruction that's also defined in SrcBlock.8720  if (!isa<Instruction>(GEPIOp))8721    return false;8722  auto *GEPIOpI = cast<Instruction>(GEPIOp);8723  if (GEPIOpI->getParent() != SrcBlock)8724    return false;8725  // Check that GEP is used outside the block, meaning it's alive on the8726  // IndirectBr edge(s).8727  if (llvm::none_of(GEPI->users(), [&](User *Usr) {8728        if (auto *I = dyn_cast<Instruction>(Usr)) {8729          if (I->getParent() != SrcBlock) {8730            return true;8731          }8732        }8733        return false;8734      }))8735    return false;8736  // The second elements of the GEP chains to be unmerged.8737  std::vector<GetElementPtrInst *> UGEPIs;8738  // Check each user of GEPIOp to check if unmerging would make GEPIOp not alive8739  // on IndirectBr edges.8740  for (User *Usr : GEPIOp->users()) {8741    if (Usr == GEPI)8742      continue;8743    // Check if Usr is an Instruction. If not, give up.8744    if (!isa<Instruction>(Usr))8745      return false;8746    auto *UI = cast<Instruction>(Usr);8747    // Check if Usr in the same block as GEPIOp, which is fine, skip.8748    if (UI->getParent() == SrcBlock)8749      continue;8750    // Check if Usr is a GEP. If not, give up.8751    if (!isa<GetElementPtrInst>(Usr))8752      return false;8753    auto *UGEPI = cast<GetElementPtrInst>(Usr);8754    // Check if UGEPI is a simple gep with a single constant index and GEPIOp is8755    // the pointer operand to it. If so, record it in the vector. If not, give8756    // up.8757    if (!GEPSequentialConstIndexed(UGEPI))8758      return false;8759    if (UGEPI->getOperand(0) != GEPIOp)8760      return false;8761    if (UGEPI->getSourceElementType() != GEPI->getSourceElementType())8762      return false;8763    if (GEPIIdx->getType() !=8764        cast<ConstantInt>(UGEPI->getOperand(1))->getType())8765      return false;8766    ConstantInt *UGEPIIdx = cast<ConstantInt>(UGEPI->getOperand(1));8767    if (TTI->getIntImmCost(UGEPIIdx->getValue(), UGEPIIdx->getType(),8768                           TargetTransformInfo::TCK_SizeAndLatency) >8769        TargetTransformInfo::TCC_Basic)8770      return false;8771    UGEPIs.push_back(UGEPI);8772  }8773  if (UGEPIs.size() == 0)8774    return false;8775  // Check the materializing cost of (Uidx-Idx).8776  for (GetElementPtrInst *UGEPI : UGEPIs) {8777    ConstantInt *UGEPIIdx = cast<ConstantInt>(UGEPI->getOperand(1));8778    APInt NewIdx = UGEPIIdx->getValue() - GEPIIdx->getValue();8779    InstructionCost ImmCost = TTI->getIntImmCost(8780        NewIdx, GEPIIdx->getType(), TargetTransformInfo::TCK_SizeAndLatency);8781    if (ImmCost > TargetTransformInfo::TCC_Basic)8782      return false;8783  }8784  // Now unmerge between GEPI and UGEPIs.8785  for (GetElementPtrInst *UGEPI : UGEPIs) {8786    UGEPI->setOperand(0, GEPI);8787    ConstantInt *UGEPIIdx = cast<ConstantInt>(UGEPI->getOperand(1));8788    Constant *NewUGEPIIdx = ConstantInt::get(8789        GEPIIdx->getType(), UGEPIIdx->getValue() - GEPIIdx->getValue());8790    UGEPI->setOperand(1, NewUGEPIIdx);8791    // If GEPI is not inbounds but UGEPI is inbounds, change UGEPI to not8792    // inbounds to avoid UB.8793    if (!GEPI->isInBounds()) {8794      UGEPI->setIsInBounds(false);8795    }8796  }8797  // After unmerging, verify that GEPIOp is actually only used in SrcBlock (not8798  // alive on IndirectBr edges).8799  assert(llvm::none_of(GEPIOp->users(),8800                       [&](User *Usr) {8801                         return cast<Instruction>(Usr)->getParent() != SrcBlock;8802                       }) &&8803         "GEPIOp is used outside SrcBlock");8804  return true;8805}8806 8807static bool optimizeBranch(BranchInst *Branch, const TargetLowering &TLI,8808                           SmallPtrSet<BasicBlock *, 32> &FreshBBs,8809                           bool IsHugeFunc) {8810  // Try and convert8811  //  %c = icmp ult %x, 88812  //  br %c, bla, blb8813  //  %tc = lshr %x, 38814  // to8815  //  %tc = lshr %x, 38816  //  %c = icmp eq %tc, 08817  //  br %c, bla, blb8818  // Creating the cmp to zero can be better for the backend, especially if the8819  // lshr produces flags that can be used automatically.8820  if (!TLI.preferZeroCompareBranch() || !Branch->isConditional())8821    return false;8822 8823  ICmpInst *Cmp = dyn_cast<ICmpInst>(Branch->getCondition());8824  if (!Cmp || !isa<ConstantInt>(Cmp->getOperand(1)) || !Cmp->hasOneUse())8825    return false;8826 8827  Value *X = Cmp->getOperand(0);8828  if (!X->hasUseList())8829    return false;8830 8831  APInt CmpC = cast<ConstantInt>(Cmp->getOperand(1))->getValue();8832 8833  for (auto *U : X->users()) {8834    Instruction *UI = dyn_cast<Instruction>(U);8835    // A quick dominance check8836    if (!UI ||8837        (UI->getParent() != Branch->getParent() &&8838         UI->getParent() != Branch->getSuccessor(0) &&8839         UI->getParent() != Branch->getSuccessor(1)) ||8840        (UI->getParent() != Branch->getParent() &&8841         !UI->getParent()->getSinglePredecessor()))8842      continue;8843 8844    if (CmpC.isPowerOf2() && Cmp->getPredicate() == ICmpInst::ICMP_ULT &&8845        match(UI, m_Shr(m_Specific(X), m_SpecificInt(CmpC.logBase2())))) {8846      IRBuilder<> Builder(Branch);8847      if (UI->getParent() != Branch->getParent())8848        UI->moveBefore(Branch->getIterator());8849      UI->dropPoisonGeneratingFlags();8850      Value *NewCmp = Builder.CreateCmp(ICmpInst::ICMP_EQ, UI,8851                                        ConstantInt::get(UI->getType(), 0));8852      LLVM_DEBUG(dbgs() << "Converting " << *Cmp << "\n");8853      LLVM_DEBUG(dbgs() << " to compare on zero: " << *NewCmp << "\n");8854      replaceAllUsesWith(Cmp, NewCmp, FreshBBs, IsHugeFunc);8855      return true;8856    }8857    if (Cmp->isEquality() &&8858        (match(UI, m_Add(m_Specific(X), m_SpecificInt(-CmpC))) ||8859         match(UI, m_Sub(m_Specific(X), m_SpecificInt(CmpC))) ||8860         match(UI, m_Xor(m_Specific(X), m_SpecificInt(CmpC))))) {8861      IRBuilder<> Builder(Branch);8862      if (UI->getParent() != Branch->getParent())8863        UI->moveBefore(Branch->getIterator());8864      UI->dropPoisonGeneratingFlags();8865      Value *NewCmp = Builder.CreateCmp(Cmp->getPredicate(), UI,8866                                        ConstantInt::get(UI->getType(), 0));8867      LLVM_DEBUG(dbgs() << "Converting " << *Cmp << "\n");8868      LLVM_DEBUG(dbgs() << " to compare on zero: " << *NewCmp << "\n");8869      replaceAllUsesWith(Cmp, NewCmp, FreshBBs, IsHugeFunc);8870      return true;8871    }8872  }8873  return false;8874}8875 8876bool CodeGenPrepare::optimizeInst(Instruction *I, ModifyDT &ModifiedDT) {8877  bool AnyChange = false;8878  AnyChange = fixupDbgVariableRecordsOnInst(*I);8879 8880  // Bail out if we inserted the instruction to prevent optimizations from8881  // stepping on each other's toes.8882  if (InsertedInsts.count(I))8883    return AnyChange;8884 8885  // TODO: Move into the switch on opcode below here.8886  if (PHINode *P = dyn_cast<PHINode>(I)) {8887    // It is possible for very late stage optimizations (such as SimplifyCFG)8888    // to introduce PHI nodes too late to be cleaned up.  If we detect such a8889    // trivial PHI, go ahead and zap it here.8890    if (Value *V = simplifyInstruction(P, {*DL, TLInfo})) {8891      LargeOffsetGEPMap.erase(P);8892      replaceAllUsesWith(P, V, FreshBBs, IsHugeFunc);8893      P->eraseFromParent();8894      ++NumPHIsElim;8895      return true;8896    }8897    return AnyChange;8898  }8899 8900  if (CastInst *CI = dyn_cast<CastInst>(I)) {8901    // If the source of the cast is a constant, then this should have8902    // already been constant folded.  The only reason NOT to constant fold8903    // it is if something (e.g. LSR) was careful to place the constant8904    // evaluation in a block other than then one that uses it (e.g. to hoist8905    // the address of globals out of a loop).  If this is the case, we don't8906    // want to forward-subst the cast.8907    if (isa<Constant>(CI->getOperand(0)))8908      return AnyChange;8909 8910    if (OptimizeNoopCopyExpression(CI, *TLI, *DL))8911      return true;8912 8913    if ((isa<UIToFPInst>(I) || isa<SIToFPInst>(I) || isa<FPToUIInst>(I) ||8914         isa<TruncInst>(I)) &&8915        TLI->optimizeExtendOrTruncateConversion(8916            I, LI->getLoopFor(I->getParent()), *TTI))8917      return true;8918 8919    if (isa<ZExtInst>(I) || isa<SExtInst>(I)) {8920      /// Sink a zext or sext into its user blocks if the target type doesn't8921      /// fit in one register8922      if (TLI->getTypeAction(CI->getContext(),8923                             TLI->getValueType(*DL, CI->getType())) ==8924          TargetLowering::TypeExpandInteger) {8925        return SinkCast(CI);8926      } else {8927        if (TLI->optimizeExtendOrTruncateConversion(8928                I, LI->getLoopFor(I->getParent()), *TTI))8929          return true;8930 8931        bool MadeChange = optimizeExt(I);8932        return MadeChange | optimizeExtUses(I);8933      }8934    }8935    return AnyChange;8936  }8937 8938  if (auto *Cmp = dyn_cast<CmpInst>(I))8939    if (optimizeCmp(Cmp, ModifiedDT))8940      return true;8941 8942  if (match(I, m_URem(m_Value(), m_Value())))8943    if (optimizeURem(I))8944      return true;8945 8946  if (LoadInst *LI = dyn_cast<LoadInst>(I)) {8947    LI->setMetadata(LLVMContext::MD_invariant_group, nullptr);8948    bool Modified = optimizeLoadExt(LI);8949    unsigned AS = LI->getPointerAddressSpace();8950    Modified |= optimizeMemoryInst(I, I->getOperand(0), LI->getType(), AS);8951    return Modified;8952  }8953 8954  if (StoreInst *SI = dyn_cast<StoreInst>(I)) {8955    if (splitMergedValStore(*SI, *DL, *TLI))8956      return true;8957    SI->setMetadata(LLVMContext::MD_invariant_group, nullptr);8958    unsigned AS = SI->getPointerAddressSpace();8959    return optimizeMemoryInst(I, SI->getOperand(1),8960                              SI->getOperand(0)->getType(), AS);8961  }8962 8963  if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {8964    unsigned AS = RMW->getPointerAddressSpace();8965    return optimizeMemoryInst(I, RMW->getPointerOperand(), RMW->getType(), AS);8966  }8967 8968  if (AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(I)) {8969    unsigned AS = CmpX->getPointerAddressSpace();8970    return optimizeMemoryInst(I, CmpX->getPointerOperand(),8971                              CmpX->getCompareOperand()->getType(), AS);8972  }8973 8974  BinaryOperator *BinOp = dyn_cast<BinaryOperator>(I);8975 8976  if (BinOp && BinOp->getOpcode() == Instruction::And && EnableAndCmpSinking &&8977      sinkAndCmp0Expression(BinOp, *TLI, InsertedInsts))8978    return true;8979 8980  // TODO: Move this into the switch on opcode - it handles shifts already.8981  if (BinOp && (BinOp->getOpcode() == Instruction::AShr ||8982                BinOp->getOpcode() == Instruction::LShr)) {8983    ConstantInt *CI = dyn_cast<ConstantInt>(BinOp->getOperand(1));8984    if (CI && TLI->hasExtractBitsInsn())8985      if (OptimizeExtractBits(BinOp, CI, *TLI, *DL))8986        return true;8987  }8988 8989  if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(I)) {8990    if (GEPI->hasAllZeroIndices()) {8991      /// The GEP operand must be a pointer, so must its result -> BitCast8992      Instruction *NC = new BitCastInst(GEPI->getOperand(0), GEPI->getType(),8993                                        GEPI->getName(), GEPI->getIterator());8994      NC->setDebugLoc(GEPI->getDebugLoc());8995      replaceAllUsesWith(GEPI, NC, FreshBBs, IsHugeFunc);8996      RecursivelyDeleteTriviallyDeadInstructions(8997          GEPI, TLInfo, nullptr,8998          [&](Value *V) { removeAllAssertingVHReferences(V); });8999      ++NumGEPsElim;9000      optimizeInst(NC, ModifiedDT);9001      return true;9002    }9003    if (tryUnmergingGEPsAcrossIndirectBr(GEPI, TTI)) {9004      return true;9005    }9006  }9007 9008  if (FreezeInst *FI = dyn_cast<FreezeInst>(I)) {9009    // freeze(icmp a, const)) -> icmp (freeze a), const9010    // This helps generate efficient conditional jumps.9011    Instruction *CmpI = nullptr;9012    if (ICmpInst *II = dyn_cast<ICmpInst>(FI->getOperand(0)))9013      CmpI = II;9014    else if (FCmpInst *F = dyn_cast<FCmpInst>(FI->getOperand(0)))9015      CmpI = F->getFastMathFlags().none() ? F : nullptr;9016 9017    if (CmpI && CmpI->hasOneUse()) {9018      auto Op0 = CmpI->getOperand(0), Op1 = CmpI->getOperand(1);9019      bool Const0 = isa<ConstantInt>(Op0) || isa<ConstantFP>(Op0) ||9020                    isa<ConstantPointerNull>(Op0);9021      bool Const1 = isa<ConstantInt>(Op1) || isa<ConstantFP>(Op1) ||9022                    isa<ConstantPointerNull>(Op1);9023      if (Const0 || Const1) {9024        if (!Const0 || !Const1) {9025          auto *F = new FreezeInst(Const0 ? Op1 : Op0, "", CmpI->getIterator());9026          F->takeName(FI);9027          CmpI->setOperand(Const0 ? 1 : 0, F);9028        }9029        replaceAllUsesWith(FI, CmpI, FreshBBs, IsHugeFunc);9030        FI->eraseFromParent();9031        return true;9032      }9033    }9034    return AnyChange;9035  }9036 9037  if (tryToSinkFreeOperands(I))9038    return true;9039 9040  switch (I->getOpcode()) {9041  case Instruction::Shl:9042  case Instruction::LShr:9043  case Instruction::AShr:9044    return optimizeShiftInst(cast<BinaryOperator>(I));9045  case Instruction::Call:9046    return optimizeCallInst(cast<CallInst>(I), ModifiedDT);9047  case Instruction::Select:9048    return optimizeSelectInst(cast<SelectInst>(I));9049  case Instruction::ShuffleVector:9050    return optimizeShuffleVectorInst(cast<ShuffleVectorInst>(I));9051  case Instruction::Switch:9052    return optimizeSwitchInst(cast<SwitchInst>(I));9053  case Instruction::ExtractElement:9054    return optimizeExtractElementInst(cast<ExtractElementInst>(I));9055  case Instruction::Br:9056    return optimizeBranch(cast<BranchInst>(I), *TLI, FreshBBs, IsHugeFunc);9057  }9058 9059  return AnyChange;9060}9061 9062/// Given an OR instruction, check to see if this is a bitreverse9063/// idiom. If so, insert the new intrinsic and return true.9064bool CodeGenPrepare::makeBitReverse(Instruction &I) {9065  if (!I.getType()->isIntegerTy() ||9066      !TLI->isOperationLegalOrCustom(ISD::BITREVERSE,9067                                     TLI->getValueType(*DL, I.getType(), true)))9068    return false;9069 9070  SmallVector<Instruction *, 4> Insts;9071  if (!recognizeBSwapOrBitReverseIdiom(&I, false, true, Insts))9072    return false;9073  Instruction *LastInst = Insts.back();9074  replaceAllUsesWith(&I, LastInst, FreshBBs, IsHugeFunc);9075  RecursivelyDeleteTriviallyDeadInstructions(9076      &I, TLInfo, nullptr,9077      [&](Value *V) { removeAllAssertingVHReferences(V); });9078  return true;9079}9080 9081// In this pass we look for GEP and cast instructions that are used9082// across basic blocks and rewrite them to improve basic-block-at-a-time9083// selection.9084bool CodeGenPrepare::optimizeBlock(BasicBlock &BB, ModifyDT &ModifiedDT) {9085  SunkAddrs.clear();9086  bool MadeChange = false;9087 9088  do {9089    CurInstIterator = BB.begin();9090    ModifiedDT = ModifyDT::NotModifyDT;9091    while (CurInstIterator != BB.end()) {9092      MadeChange |= optimizeInst(&*CurInstIterator++, ModifiedDT);9093      if (ModifiedDT != ModifyDT::NotModifyDT) {9094        // For huge function we tend to quickly go though the inner optmization9095        // opportunities in the BB. So we go back to the BB head to re-optimize9096        // each instruction instead of go back to the function head.9097        if (IsHugeFunc) {9098          DT.reset();9099          getDT(*BB.getParent());9100          break;9101        } else {9102          return true;9103        }9104      }9105    }9106  } while (ModifiedDT == ModifyDT::ModifyInstDT);9107 9108  bool MadeBitReverse = true;9109  while (MadeBitReverse) {9110    MadeBitReverse = false;9111    for (auto &I : reverse(BB)) {9112      if (makeBitReverse(I)) {9113        MadeBitReverse = MadeChange = true;9114        break;9115      }9116    }9117  }9118  MadeChange |= dupRetToEnableTailCallOpts(&BB, ModifiedDT);9119 9120  return MadeChange;9121}9122 9123bool CodeGenPrepare::fixupDbgVariableRecordsOnInst(Instruction &I) {9124  bool AnyChange = false;9125  for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))9126    AnyChange |= fixupDbgVariableRecord(DVR);9127  return AnyChange;9128}9129 9130// FIXME: should updating debug-info really cause the "changed" flag to fire,9131// which can cause a function to be reprocessed?9132bool CodeGenPrepare::fixupDbgVariableRecord(DbgVariableRecord &DVR) {9133  if (DVR.Type != DbgVariableRecord::LocationType::Value &&9134      DVR.Type != DbgVariableRecord::LocationType::Assign)9135    return false;9136 9137  // Does this DbgVariableRecord refer to a sunk address calculation?9138  bool AnyChange = false;9139  SmallDenseSet<Value *> LocationOps(DVR.location_ops().begin(),9140                                     DVR.location_ops().end());9141  for (Value *Location : LocationOps) {9142    WeakTrackingVH SunkAddrVH = SunkAddrs[Location];9143    Value *SunkAddr = SunkAddrVH.pointsToAliveValue() ? SunkAddrVH : nullptr;9144    if (SunkAddr) {9145      // Point dbg.value at locally computed address, which should give the best9146      // opportunity to be accurately lowered. This update may change the type9147      // of pointer being referred to; however this makes no difference to9148      // debugging information, and we can't generate bitcasts that may affect9149      // codegen.9150      DVR.replaceVariableLocationOp(Location, SunkAddr);9151      AnyChange = true;9152    }9153  }9154  return AnyChange;9155}9156 9157static void DbgInserterHelper(DbgVariableRecord *DVR, BasicBlock::iterator VI) {9158  DVR->removeFromParent();9159  BasicBlock *VIBB = VI->getParent();9160  if (isa<PHINode>(VI))9161    VIBB->insertDbgRecordBefore(DVR, VIBB->getFirstInsertionPt());9162  else9163    VIBB->insertDbgRecordAfter(DVR, &*VI);9164}9165 9166// A llvm.dbg.value may be using a value before its definition, due to9167// optimizations in this pass and others. Scan for such dbg.values, and rescue9168// them by moving the dbg.value to immediately after the value definition.9169// FIXME: Ideally this should never be necessary, and this has the potential9170// to re-order dbg.value intrinsics.9171bool CodeGenPrepare::placeDbgValues(Function &F) {9172  bool MadeChange = false;9173  DominatorTree DT(F);9174 9175  auto DbgProcessor = [&](auto *DbgItem, Instruction *Position) {9176    SmallVector<Instruction *, 4> VIs;9177    for (Value *V : DbgItem->location_ops())9178      if (Instruction *VI = dyn_cast_or_null<Instruction>(V))9179        VIs.push_back(VI);9180 9181    // This item may depend on multiple instructions, complicating any9182    // potential sink. This block takes the defensive approach, opting to9183    // "undef" the item if it has more than one instruction and any of them do9184    // not dominate iem.9185    for (Instruction *VI : VIs) {9186      if (VI->isTerminator())9187        continue;9188 9189      // If VI is a phi in a block with an EHPad terminator, we can't insert9190      // after it.9191      if (isa<PHINode>(VI) && VI->getParent()->getTerminator()->isEHPad())9192        continue;9193 9194      // If the defining instruction dominates the dbg.value, we do not need9195      // to move the dbg.value.9196      if (DT.dominates(VI, Position))9197        continue;9198 9199      // If we depend on multiple instructions and any of them doesn't9200      // dominate this DVI, we probably can't salvage it: moving it to9201      // after any of the instructions could cause us to lose the others.9202      if (VIs.size() > 1) {9203        LLVM_DEBUG(9204            dbgs()9205            << "Unable to find valid location for Debug Value, undefing:\n"9206            << *DbgItem);9207        DbgItem->setKillLocation();9208        break;9209      }9210 9211      LLVM_DEBUG(dbgs() << "Moving Debug Value before :\n"9212                        << *DbgItem << ' ' << *VI);9213      DbgInserterHelper(DbgItem, VI->getIterator());9214      MadeChange = true;9215      ++NumDbgValueMoved;9216    }9217  };9218 9219  for (BasicBlock &BB : F) {9220    for (Instruction &Insn : llvm::make_early_inc_range(BB)) {9221      // Process any DbgVariableRecord records attached to this9222      // instruction.9223      for (DbgVariableRecord &DVR : llvm::make_early_inc_range(9224               filterDbgVars(Insn.getDbgRecordRange()))) {9225        if (DVR.Type != DbgVariableRecord::LocationType::Value)9226          continue;9227        DbgProcessor(&DVR, &Insn);9228      }9229    }9230  }9231 9232  return MadeChange;9233}9234 9235// Group scattered pseudo probes in a block to favor SelectionDAG. Scattered9236// probes can be chained dependencies of other regular DAG nodes and block DAG9237// combine optimizations.9238bool CodeGenPrepare::placePseudoProbes(Function &F) {9239  bool MadeChange = false;9240  for (auto &Block : F) {9241    // Move the rest probes to the beginning of the block.9242    auto FirstInst = Block.getFirstInsertionPt();9243    while (FirstInst != Block.end() && FirstInst->isDebugOrPseudoInst())9244      ++FirstInst;9245    BasicBlock::iterator I(FirstInst);9246    I++;9247    while (I != Block.end()) {9248      if (auto *II = dyn_cast<PseudoProbeInst>(I++)) {9249        II->moveBefore(FirstInst);9250        MadeChange = true;9251      }9252    }9253  }9254  return MadeChange;9255}9256 9257/// Scale down both weights to fit into uint32_t.9258static void scaleWeights(uint64_t &NewTrue, uint64_t &NewFalse) {9259  uint64_t NewMax = (NewTrue > NewFalse) ? NewTrue : NewFalse;9260  uint32_t Scale = (NewMax / std::numeric_limits<uint32_t>::max()) + 1;9261  NewTrue = NewTrue / Scale;9262  NewFalse = NewFalse / Scale;9263}9264 9265/// Some targets prefer to split a conditional branch like:9266/// \code9267///   %0 = icmp ne i32 %a, 09268///   %1 = icmp ne i32 %b, 09269///   %or.cond = or i1 %0, %19270///   br i1 %or.cond, label %TrueBB, label %FalseBB9271/// \endcode9272/// into multiple branch instructions like:9273/// \code9274///   bb1:9275///     %0 = icmp ne i32 %a, 09276///     br i1 %0, label %TrueBB, label %bb29277///   bb2:9278///     %1 = icmp ne i32 %b, 09279///     br i1 %1, label %TrueBB, label %FalseBB9280/// \endcode9281/// This usually allows instruction selection to do even further optimizations9282/// and combine the compare with the branch instruction. Currently this is9283/// applied for targets which have "cheap" jump instructions.9284///9285/// FIXME: Remove the (equivalent?) implementation in SelectionDAG.9286///9287bool CodeGenPrepare::splitBranchCondition(Function &F, ModifyDT &ModifiedDT) {9288  if (!TM->Options.EnableFastISel || TLI->isJumpExpensive())9289    return false;9290 9291  bool MadeChange = false;9292  for (auto &BB : F) {9293    // Does this BB end with the following?9294    //   %cond1 = icmp|fcmp|binary instruction ...9295    //   %cond2 = icmp|fcmp|binary instruction ...9296    //   %cond.or = or|and i1 %cond1, cond29297    //   br i1 %cond.or label %dest1, label %dest2"9298    Instruction *LogicOp;9299    BasicBlock *TBB, *FBB;9300    if (!match(BB.getTerminator(),9301               m_Br(m_OneUse(m_Instruction(LogicOp)), TBB, FBB)))9302      continue;9303 9304    auto *Br1 = cast<BranchInst>(BB.getTerminator());9305    if (Br1->getMetadata(LLVMContext::MD_unpredictable))9306      continue;9307 9308    // The merging of mostly empty BB can cause a degenerate branch.9309    if (TBB == FBB)9310      continue;9311 9312    unsigned Opc;9313    Value *Cond1, *Cond2;9314    if (match(LogicOp,9315              m_LogicalAnd(m_OneUse(m_Value(Cond1)), m_OneUse(m_Value(Cond2)))))9316      Opc = Instruction::And;9317    else if (match(LogicOp, m_LogicalOr(m_OneUse(m_Value(Cond1)),9318                                        m_OneUse(m_Value(Cond2)))))9319      Opc = Instruction::Or;9320    else9321      continue;9322 9323    auto IsGoodCond = [](Value *Cond) {9324      return match(9325          Cond,9326          m_CombineOr(m_Cmp(), m_CombineOr(m_LogicalAnd(m_Value(), m_Value()),9327                                           m_LogicalOr(m_Value(), m_Value()))));9328    };9329    if (!IsGoodCond(Cond1) || !IsGoodCond(Cond2))9330      continue;9331 9332    LLVM_DEBUG(dbgs() << "Before branch condition splitting\n"; BB.dump());9333 9334    // Create a new BB.9335    auto *TmpBB =9336        BasicBlock::Create(BB.getContext(), BB.getName() + ".cond.split",9337                           BB.getParent(), BB.getNextNode());9338    if (IsHugeFunc)9339      FreshBBs.insert(TmpBB);9340 9341    // Update original basic block by using the first condition directly by the9342    // branch instruction and removing the no longer needed and/or instruction.9343    Br1->setCondition(Cond1);9344    LogicOp->eraseFromParent();9345 9346    // Depending on the condition we have to either replace the true or the9347    // false successor of the original branch instruction.9348    if (Opc == Instruction::And)9349      Br1->setSuccessor(0, TmpBB);9350    else9351      Br1->setSuccessor(1, TmpBB);9352 9353    // Fill in the new basic block.9354    auto *Br2 = IRBuilder<>(TmpBB).CreateCondBr(Cond2, TBB, FBB);9355    if (auto *I = dyn_cast<Instruction>(Cond2)) {9356      I->removeFromParent();9357      I->insertBefore(Br2->getIterator());9358    }9359 9360    // Update PHI nodes in both successors. The original BB needs to be9361    // replaced in one successor's PHI nodes, because the branch comes now from9362    // the newly generated BB (NewBB). In the other successor we need to add one9363    // incoming edge to the PHI nodes, because both branch instructions target9364    // now the same successor. Depending on the original branch condition9365    // (and/or) we have to swap the successors (TrueDest, FalseDest), so that9366    // we perform the correct update for the PHI nodes.9367    // This doesn't change the successor order of the just created branch9368    // instruction (or any other instruction).9369    if (Opc == Instruction::Or)9370      std::swap(TBB, FBB);9371 9372    // Replace the old BB with the new BB.9373    TBB->replacePhiUsesWith(&BB, TmpBB);9374 9375    // Add another incoming edge from the new BB.9376    for (PHINode &PN : FBB->phis()) {9377      auto *Val = PN.getIncomingValueForBlock(&BB);9378      PN.addIncoming(Val, TmpBB);9379    }9380 9381    // Update the branch weights (from SelectionDAGBuilder::9382    // FindMergedConditions).9383    if (Opc == Instruction::Or) {9384      // Codegen X | Y as:9385      // BB1:9386      //   jmp_if_X TBB9387      //   jmp TmpBB9388      // TmpBB:9389      //   jmp_if_Y TBB9390      //   jmp FBB9391      //9392 9393      // We have flexibility in setting Prob for BB1 and Prob for NewBB.9394      // The requirement is that9395      //   TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)9396      //     = TrueProb for original BB.9397      // Assuming the original weights are A and B, one choice is to set BB1's9398      // weights to A and A+2B, and set TmpBB's weights to A and 2B. This choice9399      // assumes that9400      //   TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.9401      // Another choice is to assume TrueProb for BB1 equals to TrueProb for9402      // TmpBB, but the math is more complicated.9403      uint64_t TrueWeight, FalseWeight;9404      if (extractBranchWeights(*Br1, TrueWeight, FalseWeight)) {9405        uint64_t NewTrueWeight = TrueWeight;9406        uint64_t NewFalseWeight = TrueWeight + 2 * FalseWeight;9407        scaleWeights(NewTrueWeight, NewFalseWeight);9408        Br1->setMetadata(LLVMContext::MD_prof,9409                         MDBuilder(Br1->getContext())9410                             .createBranchWeights(TrueWeight, FalseWeight,9411                                                  hasBranchWeightOrigin(*Br1)));9412 9413        NewTrueWeight = TrueWeight;9414        NewFalseWeight = 2 * FalseWeight;9415        scaleWeights(NewTrueWeight, NewFalseWeight);9416        Br2->setMetadata(LLVMContext::MD_prof,9417                         MDBuilder(Br2->getContext())9418                             .createBranchWeights(TrueWeight, FalseWeight));9419      }9420    } else {9421      // Codegen X & Y as:9422      // BB1:9423      //   jmp_if_X TmpBB9424      //   jmp FBB9425      // TmpBB:9426      //   jmp_if_Y TBB9427      //   jmp FBB9428      //9429      //  This requires creation of TmpBB after CurBB.9430 9431      // We have flexibility in setting Prob for BB1 and Prob for TmpBB.9432      // The requirement is that9433      //   FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)9434      //     = FalseProb for original BB.9435      // Assuming the original weights are A and B, one choice is to set BB1's9436      // weights to 2A+B and B, and set TmpBB's weights to 2A and B. This choice9437      // assumes that9438      //   FalseProb for BB1 == TrueProb for BB1 * FalseProb for TmpBB.9439      uint64_t TrueWeight, FalseWeight;9440      if (extractBranchWeights(*Br1, TrueWeight, FalseWeight)) {9441        uint64_t NewTrueWeight = 2 * TrueWeight + FalseWeight;9442        uint64_t NewFalseWeight = FalseWeight;9443        scaleWeights(NewTrueWeight, NewFalseWeight);9444        Br1->setMetadata(LLVMContext::MD_prof,9445                         MDBuilder(Br1->getContext())9446                             .createBranchWeights(TrueWeight, FalseWeight));9447 9448        NewTrueWeight = 2 * TrueWeight;9449        NewFalseWeight = FalseWeight;9450        scaleWeights(NewTrueWeight, NewFalseWeight);9451        Br2->setMetadata(LLVMContext::MD_prof,9452                         MDBuilder(Br2->getContext())9453                             .createBranchWeights(TrueWeight, FalseWeight));9454      }9455    }9456 9457    ModifiedDT = ModifyDT::ModifyBBDT;9458    MadeChange = true;9459 9460    LLVM_DEBUG(dbgs() << "After branch condition splitting\n"; BB.dump();9461               TmpBB->dump());9462  }9463  return MadeChange;9464}9465