brintos / llvm-project-archived public Read only

1//===- InstCombineCalls.cpp -----------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the visitCall, visitInvoke, and visitCallBr functions.10//11//===----------------------------------------------------------------------===//12 13#include "InstCombineInternal.h"14#include "llvm/ADT/APFloat.h"15#include "llvm/ADT/APInt.h"16#include "llvm/ADT/APSInt.h"17#include "llvm/ADT/ArrayRef.h"18#include "llvm/ADT/STLFunctionalExtras.h"19#include "llvm/ADT/SmallBitVector.h"20#include "llvm/ADT/SmallVector.h"21#include "llvm/ADT/Statistic.h"22#include "llvm/ADT/StringExtras.h"23#include "llvm/Analysis/AliasAnalysis.h"24#include "llvm/Analysis/AssumeBundleQueries.h"25#include "llvm/Analysis/AssumptionCache.h"26#include "llvm/Analysis/InstructionSimplify.h"27#include "llvm/Analysis/Loads.h"28#include "llvm/Analysis/MemoryBuiltins.h"29#include "llvm/Analysis/ValueTracking.h"30#include "llvm/Analysis/VectorUtils.h"31#include "llvm/IR/AttributeMask.h"32#include "llvm/IR/Attributes.h"33#include "llvm/IR/BasicBlock.h"34#include "llvm/IR/Constant.h"35#include "llvm/IR/Constants.h"36#include "llvm/IR/DataLayout.h"37#include "llvm/IR/DebugInfo.h"38#include "llvm/IR/DerivedTypes.h"39#include "llvm/IR/Function.h"40#include "llvm/IR/GlobalVariable.h"41#include "llvm/IR/InlineAsm.h"42#include "llvm/IR/InstrTypes.h"43#include "llvm/IR/Instruction.h"44#include "llvm/IR/Instructions.h"45#include "llvm/IR/IntrinsicInst.h"46#include "llvm/IR/Intrinsics.h"47#include "llvm/IR/IntrinsicsAArch64.h"48#include "llvm/IR/IntrinsicsAMDGPU.h"49#include "llvm/IR/IntrinsicsARM.h"50#include "llvm/IR/IntrinsicsHexagon.h"51#include "llvm/IR/LLVMContext.h"52#include "llvm/IR/Metadata.h"53#include "llvm/IR/PatternMatch.h"54#include "llvm/IR/Statepoint.h"55#include "llvm/IR/Type.h"56#include "llvm/IR/User.h"57#include "llvm/IR/Value.h"58#include "llvm/IR/ValueHandle.h"59#include "llvm/Support/AtomicOrdering.h"60#include "llvm/Support/Casting.h"61#include "llvm/Support/CommandLine.h"62#include "llvm/Support/Compiler.h"63#include "llvm/Support/Debug.h"64#include "llvm/Support/ErrorHandling.h"65#include "llvm/Support/KnownBits.h"66#include "llvm/Support/KnownFPClass.h"67#include "llvm/Support/MathExtras.h"68#include "llvm/Support/TypeSize.h"69#include "llvm/Support/raw_ostream.h"70#include "llvm/Transforms/InstCombine/InstCombiner.h"71#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"72#include "llvm/Transforms/Utils/Local.h"73#include "llvm/Transforms/Utils/SimplifyLibCalls.h"74#include <algorithm>75#include <cassert>76#include <cstdint>77#include <optional>78#include <utility>79#include <vector>80 81#define DEBUG_TYPE "instcombine"82#include "llvm/Transforms/Utils/InstructionWorklist.h"83 84using namespace llvm;85using namespace PatternMatch;86 87STATISTIC(NumSimplified, "Number of library calls simplified");88 89static cl::opt<unsigned> GuardWideningWindow(90    "instcombine-guard-widening-window",91    cl::init(3),92    cl::desc("How wide an instruction window to bypass looking for "93             "another guard"));94 95/// Return the specified type promoted as it would be to pass though a va_arg96/// area.97static Type *getPromotedType(Type *Ty) {98  if (IntegerType* ITy = dyn_cast<IntegerType>(Ty)) {99    if (ITy->getBitWidth() < 32)100      return Type::getInt32Ty(Ty->getContext());101  }102  return Ty;103}104 105/// Recognize a memcpy/memmove from a trivially otherwise unused alloca.106/// TODO: This should probably be integrated with visitAllocSites, but that107/// requires a deeper change to allow either unread or unwritten objects.108static bool hasUndefSource(AnyMemTransferInst *MI) {109  auto *Src = MI->getRawSource();110  while (isa<GetElementPtrInst>(Src)) {111    if (!Src->hasOneUse())112      return false;113    Src = cast<Instruction>(Src)->getOperand(0);114  }115  return isa<AllocaInst>(Src) && Src->hasOneUse();116}117 118Instruction *InstCombinerImpl::SimplifyAnyMemTransfer(AnyMemTransferInst *MI) {119  Align DstAlign = getKnownAlignment(MI->getRawDest(), DL, MI, &AC, &DT);120  MaybeAlign CopyDstAlign = MI->getDestAlign();121  if (!CopyDstAlign || *CopyDstAlign < DstAlign) {122    MI->setDestAlignment(DstAlign);123    return MI;124  }125 126  Align SrcAlign = getKnownAlignment(MI->getRawSource(), DL, MI, &AC, &DT);127  MaybeAlign CopySrcAlign = MI->getSourceAlign();128  if (!CopySrcAlign || *CopySrcAlign < SrcAlign) {129    MI->setSourceAlignment(SrcAlign);130    return MI;131  }132 133  // If we have a store to a location which is known constant, we can conclude134  // that the store must be storing the constant value (else the memory135  // wouldn't be constant), and this must be a noop.136  if (!isModSet(AA->getModRefInfoMask(MI->getDest()))) {137    // Set the size of the copy to 0, it will be deleted on the next iteration.138    MI->setLength((uint64_t)0);139    return MI;140  }141 142  // If the source is provably undef, the memcpy/memmove doesn't do anything143  // (unless the transfer is volatile).144  if (hasUndefSource(MI) && !MI->isVolatile()) {145    // Set the size of the copy to 0, it will be deleted on the next iteration.146    MI->setLength((uint64_t)0);147    return MI;148  }149 150  // If MemCpyInst length is 1/2/4/8 bytes then replace memcpy with151  // load/store.152  ConstantInt *MemOpLength = dyn_cast<ConstantInt>(MI->getLength());153  if (!MemOpLength) return nullptr;154 155  // Source and destination pointer types are always "i8*" for intrinsic.  See156  // if the size is something we can handle with a single primitive load/store.157  // A single load+store correctly handles overlapping memory in the memmove158  // case.159  uint64_t Size = MemOpLength->getLimitedValue();160  assert(Size && "0-sized memory transferring should be removed already.");161 162  if (Size > 8 || (Size&(Size-1)))163    return nullptr;  // If not 1/2/4/8 bytes, exit.164 165  // If it is an atomic and alignment is less than the size then we will166  // introduce the unaligned memory access which will be later transformed167  // into libcall in CodeGen. This is not evident performance gain so disable168  // it now.169  if (MI->isAtomic())170    if (*CopyDstAlign < Size || *CopySrcAlign < Size)171      return nullptr;172 173  // Use an integer load+store unless we can find something better.174  IntegerType* IntType = IntegerType::get(MI->getContext(), Size<<3);175 176  // If the memcpy has metadata describing the members, see if we can get the177  // TBAA, scope and noalias tags describing our copy.178  AAMDNodes AACopyMD = MI->getAAMetadata().adjustForAccess(Size);179 180  Value *Src = MI->getArgOperand(1);181  Value *Dest = MI->getArgOperand(0);182  LoadInst *L = Builder.CreateLoad(IntType, Src);183  // Alignment from the mem intrinsic will be better, so use it.184  L->setAlignment(*CopySrcAlign);185  L->setAAMetadata(AACopyMD);186  MDNode *LoopMemParallelMD =187    MI->getMetadata(LLVMContext::MD_mem_parallel_loop_access);188  if (LoopMemParallelMD)189    L->setMetadata(LLVMContext::MD_mem_parallel_loop_access, LoopMemParallelMD);190  MDNode *AccessGroupMD = MI->getMetadata(LLVMContext::MD_access_group);191  if (AccessGroupMD)192    L->setMetadata(LLVMContext::MD_access_group, AccessGroupMD);193 194  StoreInst *S = Builder.CreateStore(L, Dest);195  // Alignment from the mem intrinsic will be better, so use it.196  S->setAlignment(*CopyDstAlign);197  S->setAAMetadata(AACopyMD);198  if (LoopMemParallelMD)199    S->setMetadata(LLVMContext::MD_mem_parallel_loop_access, LoopMemParallelMD);200  if (AccessGroupMD)201    S->setMetadata(LLVMContext::MD_access_group, AccessGroupMD);202  S->copyMetadata(*MI, LLVMContext::MD_DIAssignID);203 204  if (auto *MT = dyn_cast<MemTransferInst>(MI)) {205    // non-atomics can be volatile206    L->setVolatile(MT->isVolatile());207    S->setVolatile(MT->isVolatile());208  }209  if (MI->isAtomic()) {210    // atomics have to be unordered211    L->setOrdering(AtomicOrdering::Unordered);212    S->setOrdering(AtomicOrdering::Unordered);213  }214 215  // Set the size of the copy to 0, it will be deleted on the next iteration.216  MI->setLength((uint64_t)0);217  return MI;218}219 220Instruction *InstCombinerImpl::SimplifyAnyMemSet(AnyMemSetInst *MI) {221  const Align KnownAlignment =222      getKnownAlignment(MI->getDest(), DL, MI, &AC, &DT);223  MaybeAlign MemSetAlign = MI->getDestAlign();224  if (!MemSetAlign || *MemSetAlign < KnownAlignment) {225    MI->setDestAlignment(KnownAlignment);226    return MI;227  }228 229  // If we have a store to a location which is known constant, we can conclude230  // that the store must be storing the constant value (else the memory231  // wouldn't be constant), and this must be a noop.232  if (!isModSet(AA->getModRefInfoMask(MI->getDest()))) {233    // Set the size of the copy to 0, it will be deleted on the next iteration.234    MI->setLength((uint64_t)0);235    return MI;236  }237 238  // Remove memset with an undef value.239  // FIXME: This is technically incorrect because it might overwrite a poison240  // value. Change to PoisonValue once #52930 is resolved.241  if (isa<UndefValue>(MI->getValue())) {242    // Set the size of the copy to 0, it will be deleted on the next iteration.243    MI->setLength((uint64_t)0);244    return MI;245  }246 247  // Extract the length and alignment and fill if they are constant.248  ConstantInt *LenC = dyn_cast<ConstantInt>(MI->getLength());249  ConstantInt *FillC = dyn_cast<ConstantInt>(MI->getValue());250  if (!LenC || !FillC || !FillC->getType()->isIntegerTy(8))251    return nullptr;252  const uint64_t Len = LenC->getLimitedValue();253  assert(Len && "0-sized memory setting should be removed already.");254  const Align Alignment = MI->getDestAlign().valueOrOne();255 256  // If it is an atomic and alignment is less than the size then we will257  // introduce the unaligned memory access which will be later transformed258  // into libcall in CodeGen. This is not evident performance gain so disable259  // it now.260  if (MI->isAtomic() && Alignment < Len)261    return nullptr;262 263  // memset(s,c,n) -> store s, c (for n=1,2,4,8)264  if (Len <= 8 && isPowerOf2_32((uint32_t)Len)) {265    Value *Dest = MI->getDest();266 267    // Extract the fill value and store.268    Constant *FillVal = ConstantInt::get(269        MI->getContext(), APInt::getSplat(Len * 8, FillC->getValue()));270    StoreInst *S = Builder.CreateStore(FillVal, Dest, MI->isVolatile());271    S->copyMetadata(*MI, LLVMContext::MD_DIAssignID);272    for (DbgVariableRecord *DbgAssign : at::getDVRAssignmentMarkers(S)) {273      if (llvm::is_contained(DbgAssign->location_ops(), FillC))274        DbgAssign->replaceVariableLocationOp(FillC, FillVal);275    }276 277    S->setAlignment(Alignment);278    if (MI->isAtomic())279      S->setOrdering(AtomicOrdering::Unordered);280 281    // Set the size of the copy to 0, it will be deleted on the next iteration.282    MI->setLength((uint64_t)0);283    return MI;284  }285 286  return nullptr;287}288 289// TODO, Obvious Missing Transforms:290// * Narrow width by halfs excluding zero/undef lanes291Value *InstCombinerImpl::simplifyMaskedLoad(IntrinsicInst &II) {292  Value *LoadPtr = II.getArgOperand(0);293  const Align Alignment = II.getParamAlign(0).valueOrOne();294 295  // If the mask is all ones or undefs, this is a plain vector load of the 1st296  // argument.297  if (maskIsAllOneOrUndef(II.getArgOperand(1))) {298    LoadInst *L = Builder.CreateAlignedLoad(II.getType(), LoadPtr, Alignment,299                                            "unmaskedload");300    L->copyMetadata(II);301    return L;302  }303 304  // If we can unconditionally load from this address, replace with a305  // load/select idiom. TODO: use DT for context sensitive query306  if (isDereferenceablePointer(LoadPtr, II.getType(),307                               II.getDataLayout(), &II, &AC)) {308    LoadInst *LI = Builder.CreateAlignedLoad(II.getType(), LoadPtr, Alignment,309                                             "unmaskedload");310    LI->copyMetadata(II);311    return Builder.CreateSelect(II.getArgOperand(1), LI, II.getArgOperand(2));312  }313 314  return nullptr;315}316 317// TODO, Obvious Missing Transforms:318// * Single constant active lane -> store319// * Narrow width by halfs excluding zero/undef lanes320Instruction *InstCombinerImpl::simplifyMaskedStore(IntrinsicInst &II) {321  Value *StorePtr = II.getArgOperand(1);322  Align Alignment = II.getParamAlign(1).valueOrOne();323  auto *ConstMask = dyn_cast<Constant>(II.getArgOperand(2));324  if (!ConstMask)325    return nullptr;326 327  // If the mask is all zeros, this instruction does nothing.328  if (maskIsAllZeroOrUndef(ConstMask))329    return eraseInstFromFunction(II);330 331  // If the mask is all ones, this is a plain vector store of the 1st argument.332  if (maskIsAllOneOrUndef(ConstMask)) {333    StoreInst *S =334        new StoreInst(II.getArgOperand(0), StorePtr, false, Alignment);335    S->copyMetadata(II);336    return S;337  }338 339  if (isa<ScalableVectorType>(ConstMask->getType()))340    return nullptr;341 342  // Use masked off lanes to simplify operands via SimplifyDemandedVectorElts343  APInt DemandedElts = possiblyDemandedEltsInMask(ConstMask);344  APInt PoisonElts(DemandedElts.getBitWidth(), 0);345  if (Value *V = SimplifyDemandedVectorElts(II.getOperand(0), DemandedElts,346                                            PoisonElts))347    return replaceOperand(II, 0, V);348 349  return nullptr;350}351 352// TODO, Obvious Missing Transforms:353// * Single constant active lane load -> load354// * Dereferenceable address & few lanes -> scalarize speculative load/selects355// * Adjacent vector addresses -> masked.load356// * Narrow width by halfs excluding zero/undef lanes357// * Vector incrementing address -> vector masked load358Instruction *InstCombinerImpl::simplifyMaskedGather(IntrinsicInst &II) {359  auto *ConstMask = dyn_cast<Constant>(II.getArgOperand(1));360  if (!ConstMask)361    return nullptr;362 363  // Vector splat address w/known mask -> scalar load364  // Fold the gather to load the source vector first lane365  // because it is reloading the same value each time366  if (ConstMask->isAllOnesValue())367    if (auto *SplatPtr = getSplatValue(II.getArgOperand(0))) {368      auto *VecTy = cast<VectorType>(II.getType());369      const Align Alignment = II.getParamAlign(0).valueOrOne();370      LoadInst *L = Builder.CreateAlignedLoad(VecTy->getElementType(), SplatPtr,371                                              Alignment, "load.scalar");372      Value *Shuf =373          Builder.CreateVectorSplat(VecTy->getElementCount(), L, "broadcast");374      return replaceInstUsesWith(II, cast<Instruction>(Shuf));375    }376 377  return nullptr;378}379 380// TODO, Obvious Missing Transforms:381// * Single constant active lane -> store382// * Adjacent vector addresses -> masked.store383// * Narrow store width by halfs excluding zero/undef lanes384// * Vector incrementing address -> vector masked store385Instruction *InstCombinerImpl::simplifyMaskedScatter(IntrinsicInst &II) {386  auto *ConstMask = dyn_cast<Constant>(II.getArgOperand(2));387  if (!ConstMask)388    return nullptr;389 390  // If the mask is all zeros, a scatter does nothing.391  if (maskIsAllZeroOrUndef(ConstMask))392    return eraseInstFromFunction(II);393 394  // Vector splat address -> scalar store395  if (auto *SplatPtr = getSplatValue(II.getArgOperand(1))) {396    // scatter(splat(value), splat(ptr), non-zero-mask) -> store value, ptr397    if (auto *SplatValue = getSplatValue(II.getArgOperand(0))) {398      if (maskContainsAllOneOrUndef(ConstMask)) {399        Align Alignment = II.getParamAlign(1).valueOrOne();400        StoreInst *S = new StoreInst(SplatValue, SplatPtr, /*IsVolatile=*/false,401                                     Alignment);402        S->copyMetadata(II);403        return S;404      }405    }406    // scatter(vector, splat(ptr), splat(true)) -> store extract(vector,407    // lastlane), ptr408    if (ConstMask->isAllOnesValue()) {409      Align Alignment = II.getParamAlign(1).valueOrOne();410      VectorType *WideLoadTy = cast<VectorType>(II.getArgOperand(1)->getType());411      ElementCount VF = WideLoadTy->getElementCount();412      Value *RunTimeVF = Builder.CreateElementCount(Builder.getInt32Ty(), VF);413      Value *LastLane = Builder.CreateSub(RunTimeVF, Builder.getInt32(1));414      Value *Extract =415          Builder.CreateExtractElement(II.getArgOperand(0), LastLane);416      StoreInst *S =417          new StoreInst(Extract, SplatPtr, /*IsVolatile=*/false, Alignment);418      S->copyMetadata(II);419      return S;420    }421  }422  if (isa<ScalableVectorType>(ConstMask->getType()))423    return nullptr;424 425  // Use masked off lanes to simplify operands via SimplifyDemandedVectorElts426  APInt DemandedElts = possiblyDemandedEltsInMask(ConstMask);427  APInt PoisonElts(DemandedElts.getBitWidth(), 0);428  if (Value *V = SimplifyDemandedVectorElts(II.getOperand(0), DemandedElts,429                                            PoisonElts))430    return replaceOperand(II, 0, V);431  if (Value *V = SimplifyDemandedVectorElts(II.getOperand(1), DemandedElts,432                                            PoisonElts))433    return replaceOperand(II, 1, V);434 435  return nullptr;436}437 438/// This function transforms launder.invariant.group and strip.invariant.group439/// like:440/// launder(launder(%x)) -> launder(%x)       (the result is not the argument)441/// launder(strip(%x)) -> launder(%x)442/// strip(strip(%x)) -> strip(%x)             (the result is not the argument)443/// strip(launder(%x)) -> strip(%x)444/// This is legal because it preserves the most recent information about445/// the presence or absence of invariant.group.446static Instruction *simplifyInvariantGroupIntrinsic(IntrinsicInst &II,447                                                    InstCombinerImpl &IC) {448  auto *Arg = II.getArgOperand(0);449  auto *StrippedArg = Arg->stripPointerCasts();450  auto *StrippedInvariantGroupsArg = StrippedArg;451  while (auto *Intr = dyn_cast<IntrinsicInst>(StrippedInvariantGroupsArg)) {452    if (Intr->getIntrinsicID() != Intrinsic::launder_invariant_group &&453        Intr->getIntrinsicID() != Intrinsic::strip_invariant_group)454      break;455    StrippedInvariantGroupsArg = Intr->getArgOperand(0)->stripPointerCasts();456  }457  if (StrippedArg == StrippedInvariantGroupsArg)458    return nullptr; // No launders/strips to remove.459 460  Value *Result = nullptr;461 462  if (II.getIntrinsicID() == Intrinsic::launder_invariant_group)463    Result = IC.Builder.CreateLaunderInvariantGroup(StrippedInvariantGroupsArg);464  else if (II.getIntrinsicID() == Intrinsic::strip_invariant_group)465    Result = IC.Builder.CreateStripInvariantGroup(StrippedInvariantGroupsArg);466  else467    llvm_unreachable(468        "simplifyInvariantGroupIntrinsic only handles launder and strip");469  if (Result->getType()->getPointerAddressSpace() !=470      II.getType()->getPointerAddressSpace())471    Result = IC.Builder.CreateAddrSpaceCast(Result, II.getType());472 473  return cast<Instruction>(Result);474}475 476static Instruction *foldCttzCtlz(IntrinsicInst &II, InstCombinerImpl &IC) {477  assert((II.getIntrinsicID() == Intrinsic::cttz ||478          II.getIntrinsicID() == Intrinsic::ctlz) &&479         "Expected cttz or ctlz intrinsic");480  bool IsTZ = II.getIntrinsicID() == Intrinsic::cttz;481  Value *Op0 = II.getArgOperand(0);482  Value *Op1 = II.getArgOperand(1);483  Value *X;484  // ctlz(bitreverse(x)) -> cttz(x)485  // cttz(bitreverse(x)) -> ctlz(x)486  if (match(Op0, m_BitReverse(m_Value(X)))) {487    Intrinsic::ID ID = IsTZ ? Intrinsic::ctlz : Intrinsic::cttz;488    Function *F =489        Intrinsic::getOrInsertDeclaration(II.getModule(), ID, II.getType());490    return CallInst::Create(F, {X, II.getArgOperand(1)});491  }492 493  if (II.getType()->isIntOrIntVectorTy(1)) {494    // ctlz/cttz i1 Op0 --> not Op0495    if (match(Op1, m_Zero()))496      return BinaryOperator::CreateNot(Op0);497    // If zero is poison, then the input can be assumed to be "true", so the498    // instruction simplifies to "false".499    assert(match(Op1, m_One()) && "Expected ctlz/cttz operand to be 0 or 1");500    return IC.replaceInstUsesWith(II, ConstantInt::getNullValue(II.getType()));501  }502 503  // If ctlz/cttz is only used as a shift amount, set is_zero_poison to true.504  if (II.hasOneUse() && match(Op1, m_Zero()) &&505      match(II.user_back(), m_Shift(m_Value(), m_Specific(&II)))) {506    II.dropUBImplyingAttrsAndMetadata();507    return IC.replaceOperand(II, 1, IC.Builder.getTrue());508  }509 510  Constant *C;511 512  if (IsTZ) {513    // cttz(-x) -> cttz(x)514    if (match(Op0, m_Neg(m_Value(X))))515      return IC.replaceOperand(II, 0, X);516 517    // cttz(-x & x) -> cttz(x)518    if (match(Op0, m_c_And(m_Neg(m_Value(X)), m_Deferred(X))))519      return IC.replaceOperand(II, 0, X);520 521    // cttz(sext(x)) -> cttz(zext(x))522    if (match(Op0, m_OneUse(m_SExt(m_Value(X))))) {523      auto *Zext = IC.Builder.CreateZExt(X, II.getType());524      auto *CttzZext =525          IC.Builder.CreateBinaryIntrinsic(Intrinsic::cttz, Zext, Op1);526      return IC.replaceInstUsesWith(II, CttzZext);527    }528 529    // Zext doesn't change the number of trailing zeros, so narrow:530    // cttz(zext(x)) -> zext(cttz(x)) if the 'ZeroIsPoison' parameter is 'true'.531    if (match(Op0, m_OneUse(m_ZExt(m_Value(X)))) && match(Op1, m_One())) {532      auto *Cttz = IC.Builder.CreateBinaryIntrinsic(Intrinsic::cttz, X,533                                                    IC.Builder.getTrue());534      auto *ZextCttz = IC.Builder.CreateZExt(Cttz, II.getType());535      return IC.replaceInstUsesWith(II, ZextCttz);536    }537 538    // cttz(abs(x)) -> cttz(x)539    // cttz(nabs(x)) -> cttz(x)540    Value *Y;541    SelectPatternFlavor SPF = matchSelectPattern(Op0, X, Y).Flavor;542    if (SPF == SPF_ABS || SPF == SPF_NABS)543      return IC.replaceOperand(II, 0, X);544 545    if (match(Op0, m_Intrinsic<Intrinsic::abs>(m_Value(X))))546      return IC.replaceOperand(II, 0, X);547 548    // cttz(shl(%const, %val), 1) --> add(cttz(%const, 1), %val)549    if (match(Op0, m_Shl(m_ImmConstant(C), m_Value(X))) &&550        match(Op1, m_One())) {551      Value *ConstCttz =552          IC.Builder.CreateBinaryIntrinsic(Intrinsic::cttz, C, Op1);553      return BinaryOperator::CreateAdd(ConstCttz, X);554    }555 556    // cttz(lshr exact (%const, %val), 1) --> sub(cttz(%const, 1), %val)557    if (match(Op0, m_Exact(m_LShr(m_ImmConstant(C), m_Value(X)))) &&558        match(Op1, m_One())) {559      Value *ConstCttz =560          IC.Builder.CreateBinaryIntrinsic(Intrinsic::cttz, C, Op1);561      return BinaryOperator::CreateSub(ConstCttz, X);562    }563 564    // cttz(add(lshr(UINT_MAX, %val), 1)) --> sub(width, %val)565    if (match(Op0, m_Add(m_LShr(m_AllOnes(), m_Value(X)), m_One()))) {566      Value *Width =567          ConstantInt::get(II.getType(), II.getType()->getScalarSizeInBits());568      return BinaryOperator::CreateSub(Width, X);569    }570  } else {571    // ctlz(lshr(%const, %val), 1) --> add(ctlz(%const, 1), %val)572    if (match(Op0, m_LShr(m_ImmConstant(C), m_Value(X))) &&573        match(Op1, m_One())) {574      Value *ConstCtlz =575          IC.Builder.CreateBinaryIntrinsic(Intrinsic::ctlz, C, Op1);576      return BinaryOperator::CreateAdd(ConstCtlz, X);577    }578 579    // ctlz(shl nuw (%const, %val), 1) --> sub(ctlz(%const, 1), %val)580    if (match(Op0, m_NUWShl(m_ImmConstant(C), m_Value(X))) &&581        match(Op1, m_One())) {582      Value *ConstCtlz =583          IC.Builder.CreateBinaryIntrinsic(Intrinsic::ctlz, C, Op1);584      return BinaryOperator::CreateSub(ConstCtlz, X);585    }586 587    // ctlz(~x & (x - 1)) -> bitwidth - cttz(x, false)588    if (Op0->hasOneUse() &&589        match(Op0,590              m_c_And(m_Not(m_Value(X)), m_Add(m_Deferred(X), m_AllOnes())))) {591      Type *Ty = II.getType();592      unsigned BitWidth = Ty->getScalarSizeInBits();593      auto *Cttz = IC.Builder.CreateIntrinsic(Intrinsic::cttz, Ty,594                                              {X, IC.Builder.getFalse()});595      auto *Bw = ConstantInt::get(Ty, APInt(BitWidth, BitWidth));596      return IC.replaceInstUsesWith(II, IC.Builder.CreateSub(Bw, Cttz));597    }598  }599 600  // cttz(Pow2) -> Log2(Pow2)601  // ctlz(Pow2) -> BitWidth - 1 - Log2(Pow2)602  if (auto *R = IC.tryGetLog2(Op0, match(Op1, m_One()))) {603    if (IsTZ)604      return IC.replaceInstUsesWith(II, R);605    BinaryOperator *BO = BinaryOperator::CreateSub(606        ConstantInt::get(R->getType(), R->getType()->getScalarSizeInBits() - 1),607        R);608    BO->setHasNoSignedWrap();609    BO->setHasNoUnsignedWrap();610    return BO;611  }612 613  KnownBits Known = IC.computeKnownBits(Op0, &II);614 615  // Create a mask for bits above (ctlz) or below (cttz) the first known one.616  unsigned PossibleZeros = IsTZ ? Known.countMaxTrailingZeros()617                                : Known.countMaxLeadingZeros();618  unsigned DefiniteZeros = IsTZ ? Known.countMinTrailingZeros()619                                : Known.countMinLeadingZeros();620 621  // If all bits above (ctlz) or below (cttz) the first known one are known622  // zero, this value is constant.623  // FIXME: This should be in InstSimplify because we're replacing an624  // instruction with a constant.625  if (PossibleZeros == DefiniteZeros) {626    auto *C = ConstantInt::get(Op0->getType(), DefiniteZeros);627    return IC.replaceInstUsesWith(II, C);628  }629 630  // If the input to cttz/ctlz is known to be non-zero,631  // then change the 'ZeroIsPoison' parameter to 'true'632  // because we know the zero behavior can't affect the result.633  if (!Known.One.isZero() ||634      isKnownNonZero(Op0, IC.getSimplifyQuery().getWithInstruction(&II))) {635    if (!match(II.getArgOperand(1), m_One()))636      return IC.replaceOperand(II, 1, IC.Builder.getTrue());637  }638 639  // Add range attribute since known bits can't completely reflect what we know.640  unsigned BitWidth = Op0->getType()->getScalarSizeInBits();641  if (BitWidth != 1 && !II.hasRetAttr(Attribute::Range) &&642      !II.getMetadata(LLVMContext::MD_range)) {643    ConstantRange Range(APInt(BitWidth, DefiniteZeros),644                        APInt(BitWidth, PossibleZeros + 1));645    II.addRangeRetAttr(Range);646    return &II;647  }648 649  return nullptr;650}651 652static Instruction *foldCtpop(IntrinsicInst &II, InstCombinerImpl &IC) {653  assert(II.getIntrinsicID() == Intrinsic::ctpop &&654         "Expected ctpop intrinsic");655  Type *Ty = II.getType();656  unsigned BitWidth = Ty->getScalarSizeInBits();657  Value *Op0 = II.getArgOperand(0);658  Value *X, *Y;659 660  // ctpop(bitreverse(x)) -> ctpop(x)661  // ctpop(bswap(x)) -> ctpop(x)662  if (match(Op0, m_BitReverse(m_Value(X))) || match(Op0, m_BSwap(m_Value(X))))663    return IC.replaceOperand(II, 0, X);664 665  // ctpop(rot(x)) -> ctpop(x)666  if ((match(Op0, m_FShl(m_Value(X), m_Value(Y), m_Value())) ||667       match(Op0, m_FShr(m_Value(X), m_Value(Y), m_Value()))) &&668      X == Y)669    return IC.replaceOperand(II, 0, X);670 671  // ctpop(x | -x) -> bitwidth - cttz(x, false)672  if (Op0->hasOneUse() &&673      match(Op0, m_c_Or(m_Value(X), m_Neg(m_Deferred(X))))) {674    auto *Cttz = IC.Builder.CreateIntrinsic(Intrinsic::cttz, Ty,675                                            {X, IC.Builder.getFalse()});676    auto *Bw = ConstantInt::get(Ty, APInt(BitWidth, BitWidth));677    return IC.replaceInstUsesWith(II, IC.Builder.CreateSub(Bw, Cttz));678  }679 680  // ctpop(~x & (x - 1)) -> cttz(x, false)681  if (match(Op0,682            m_c_And(m_Not(m_Value(X)), m_Add(m_Deferred(X), m_AllOnes())))) {683    Function *F =684        Intrinsic::getOrInsertDeclaration(II.getModule(), Intrinsic::cttz, Ty);685    return CallInst::Create(F, {X, IC.Builder.getFalse()});686  }687 688  // Zext doesn't change the number of set bits, so narrow:689  // ctpop (zext X) --> zext (ctpop X)690  if (match(Op0, m_OneUse(m_ZExt(m_Value(X))))) {691    Value *NarrowPop = IC.Builder.CreateUnaryIntrinsic(Intrinsic::ctpop, X);692    return CastInst::Create(Instruction::ZExt, NarrowPop, Ty);693  }694 695  KnownBits Known(BitWidth);696  IC.computeKnownBits(Op0, Known, &II);697 698  // If all bits are zero except for exactly one fixed bit, then the result699  // must be 0 or 1, and we can get that answer by shifting to LSB:700  // ctpop (X & 32) --> (X & 32) >> 5701  // TODO: Investigate removing this as its likely unnecessary given the below702  // `isKnownToBeAPowerOfTwo` check.703  if ((~Known.Zero).isPowerOf2())704    return BinaryOperator::CreateLShr(705        Op0, ConstantInt::get(Ty, (~Known.Zero).exactLogBase2()));706 707  // More generally we can also handle non-constant power of 2 patterns such as708  // shl/shr(Pow2, X), (X & -X), etc... by transforming:709  // ctpop(Pow2OrZero) --> icmp ne X, 0710  if (IC.isKnownToBeAPowerOfTwo(Op0, /* OrZero */ true))711    return CastInst::Create(Instruction::ZExt,712                            IC.Builder.CreateICmp(ICmpInst::ICMP_NE, Op0,713                                                  Constant::getNullValue(Ty)),714                            Ty);715 716  // Add range attribute since known bits can't completely reflect what we know.717  if (BitWidth != 1) {718    ConstantRange OldRange =719        II.getRange().value_or(ConstantRange::getFull(BitWidth));720 721    unsigned Lower = Known.countMinPopulation();722    unsigned Upper = Known.countMaxPopulation() + 1;723 724    if (Lower == 0 && OldRange.contains(APInt::getZero(BitWidth)) &&725        isKnownNonZero(Op0, IC.getSimplifyQuery().getWithInstruction(&II)))726      Lower = 1;727 728    ConstantRange Range(APInt(BitWidth, Lower), APInt(BitWidth, Upper));729    Range = Range.intersectWith(OldRange, ConstantRange::Unsigned);730 731    if (Range != OldRange) {732      II.addRangeRetAttr(Range);733      return &II;734    }735  }736 737  return nullptr;738}739 740/// Convert a table lookup to shufflevector if the mask is constant.741/// This could benefit tbl1 if the mask is { 7,6,5,4,3,2,1,0 }, in742/// which case we could lower the shufflevector with rev64 instructions743/// as it's actually a byte reverse.744static Value *simplifyNeonTbl1(const IntrinsicInst &II,745                               InstCombiner::BuilderTy &Builder) {746  // Bail out if the mask is not a constant.747  auto *C = dyn_cast<Constant>(II.getArgOperand(1));748  if (!C)749    return nullptr;750 751  auto *VecTy = cast<FixedVectorType>(II.getType());752  unsigned NumElts = VecTy->getNumElements();753 754  // Only perform this transformation for <8 x i8> vector types.755  if (!VecTy->getElementType()->isIntegerTy(8) || NumElts != 8)756    return nullptr;757 758  int Indexes[8];759 760  for (unsigned I = 0; I < NumElts; ++I) {761    Constant *COp = C->getAggregateElement(I);762 763    if (!COp || !isa<ConstantInt>(COp))764      return nullptr;765 766    Indexes[I] = cast<ConstantInt>(COp)->getLimitedValue();767 768    // Make sure the mask indices are in range.769    if ((unsigned)Indexes[I] >= NumElts)770      return nullptr;771  }772 773  auto *V1 = II.getArgOperand(0);774  auto *V2 = Constant::getNullValue(V1->getType());775  return Builder.CreateShuffleVector(V1, V2, ArrayRef(Indexes));776}777 778// Returns true iff the 2 intrinsics have the same operands, limiting the779// comparison to the first NumOperands.780static bool haveSameOperands(const IntrinsicInst &I, const IntrinsicInst &E,781                             unsigned NumOperands) {782  assert(I.arg_size() >= NumOperands && "Not enough operands");783  assert(E.arg_size() >= NumOperands && "Not enough operands");784  for (unsigned i = 0; i < NumOperands; i++)785    if (I.getArgOperand(i) != E.getArgOperand(i))786      return false;787  return true;788}789 790// Remove trivially empty start/end intrinsic ranges, i.e. a start791// immediately followed by an end (ignoring debuginfo or other792// start/end intrinsics in between). As this handles only the most trivial793// cases, tracking the nesting level is not needed:794//795//   call @llvm.foo.start(i1 0)796//   call @llvm.foo.start(i1 0) ; This one won't be skipped: it will be removed797//   call @llvm.foo.end(i1 0)798//   call @llvm.foo.end(i1 0) ; &I799static bool800removeTriviallyEmptyRange(IntrinsicInst &EndI, InstCombinerImpl &IC,801                          std::function<bool(const IntrinsicInst &)> IsStart) {802  // We start from the end intrinsic and scan backwards, so that InstCombine803  // has already processed (and potentially removed) all the instructions804  // before the end intrinsic.805  BasicBlock::reverse_iterator BI(EndI), BE(EndI.getParent()->rend());806  for (; BI != BE; ++BI) {807    if (auto *I = dyn_cast<IntrinsicInst>(&*BI)) {808      if (I->isDebugOrPseudoInst() ||809          I->getIntrinsicID() == EndI.getIntrinsicID())810        continue;811      if (IsStart(*I)) {812        if (haveSameOperands(EndI, *I, EndI.arg_size())) {813          IC.eraseInstFromFunction(*I);814          IC.eraseInstFromFunction(EndI);815          return true;816        }817        // Skip start intrinsics that don't pair with this end intrinsic.818        continue;819      }820    }821    break;822  }823 824  return false;825}826 827Instruction *InstCombinerImpl::visitVAEndInst(VAEndInst &I) {828  removeTriviallyEmptyRange(I, *this, [&I](const IntrinsicInst &II) {829    // Bail out on the case where the source va_list of a va_copy is destroyed830    // immediately by a follow-up va_end.831    return II.getIntrinsicID() == Intrinsic::vastart ||832           (II.getIntrinsicID() == Intrinsic::vacopy &&833            I.getArgOperand(0) != II.getArgOperand(1));834  });835  return nullptr;836}837 838static CallInst *canonicalizeConstantArg0ToArg1(CallInst &Call) {839  assert(Call.arg_size() > 1 && "Need at least 2 args to swap");840  Value *Arg0 = Call.getArgOperand(0), *Arg1 = Call.getArgOperand(1);841  if (isa<Constant>(Arg0) && !isa<Constant>(Arg1)) {842    Call.setArgOperand(0, Arg1);843    Call.setArgOperand(1, Arg0);844    return &Call;845  }846  return nullptr;847}848 849/// Creates a result tuple for an overflow intrinsic \p II with a given850/// \p Result and a constant \p Overflow value.851static Instruction *createOverflowTuple(IntrinsicInst *II, Value *Result,852                                        Constant *Overflow) {853  Constant *V[] = {PoisonValue::get(Result->getType()), Overflow};854  StructType *ST = cast<StructType>(II->getType());855  Constant *Struct = ConstantStruct::get(ST, V);856  return InsertValueInst::Create(Struct, Result, 0);857}858 859Instruction *860InstCombinerImpl::foldIntrinsicWithOverflowCommon(IntrinsicInst *II) {861  WithOverflowInst *WO = cast<WithOverflowInst>(II);862  Value *OperationResult = nullptr;863  Constant *OverflowResult = nullptr;864  if (OptimizeOverflowCheck(WO->getBinaryOp(), WO->isSigned(), WO->getLHS(),865                            WO->getRHS(), *WO, OperationResult, OverflowResult))866    return createOverflowTuple(WO, OperationResult, OverflowResult);867 868  // See whether we can optimize the overflow check with assumption information.869  for (User *U : WO->users()) {870    if (!match(U, m_ExtractValue<1>(m_Value())))871      continue;872 873    for (auto &AssumeVH : AC.assumptionsFor(U)) {874      if (!AssumeVH)875        continue;876      CallInst *I = cast<CallInst>(AssumeVH);877      if (!match(I->getArgOperand(0), m_Not(m_Specific(U))))878        continue;879      if (!isValidAssumeForContext(I, II, /*DT=*/nullptr,880                                   /*AllowEphemerals=*/true))881        continue;882      Value *Result =883          Builder.CreateBinOp(WO->getBinaryOp(), WO->getLHS(), WO->getRHS());884      Result->takeName(WO);885      if (auto *Inst = dyn_cast<Instruction>(Result)) {886        if (WO->isSigned())887          Inst->setHasNoSignedWrap();888        else889          Inst->setHasNoUnsignedWrap();890      }891      return createOverflowTuple(WO, Result,892                                 ConstantInt::getFalse(U->getType()));893    }894  }895 896  return nullptr;897}898 899static bool inputDenormalIsIEEE(const Function &F, const Type *Ty) {900  Ty = Ty->getScalarType();901  return F.getDenormalMode(Ty->getFltSemantics()).Input == DenormalMode::IEEE;902}903 904static bool inputDenormalIsDAZ(const Function &F, const Type *Ty) {905  Ty = Ty->getScalarType();906  return F.getDenormalMode(Ty->getFltSemantics()).inputsAreZero();907}908 909/// \returns the compare predicate type if the test performed by910/// llvm.is.fpclass(x, \p Mask) is equivalent to fcmp o__ x, 0.0 with the911/// floating-point environment assumed for \p F for type \p Ty912static FCmpInst::Predicate fpclassTestIsFCmp0(FPClassTest Mask,913                                              const Function &F, Type *Ty) {914  switch (static_cast<unsigned>(Mask)) {915  case fcZero:916    if (inputDenormalIsIEEE(F, Ty))917      return FCmpInst::FCMP_OEQ;918    break;919  case fcZero | fcSubnormal:920    if (inputDenormalIsDAZ(F, Ty))921      return FCmpInst::FCMP_OEQ;922    break;923  case fcPositive | fcNegZero:924    if (inputDenormalIsIEEE(F, Ty))925      return FCmpInst::FCMP_OGE;926    break;927  case fcPositive | fcNegZero | fcNegSubnormal:928    if (inputDenormalIsDAZ(F, Ty))929      return FCmpInst::FCMP_OGE;930    break;931  case fcPosSubnormal | fcPosNormal | fcPosInf:932    if (inputDenormalIsIEEE(F, Ty))933      return FCmpInst::FCMP_OGT;934    break;935  case fcNegative | fcPosZero:936    if (inputDenormalIsIEEE(F, Ty))937      return FCmpInst::FCMP_OLE;938    break;939  case fcNegative | fcPosZero | fcPosSubnormal:940    if (inputDenormalIsDAZ(F, Ty))941      return FCmpInst::FCMP_OLE;942    break;943  case fcNegSubnormal | fcNegNormal | fcNegInf:944    if (inputDenormalIsIEEE(F, Ty))945      return FCmpInst::FCMP_OLT;946    break;947  case fcPosNormal | fcPosInf:948    if (inputDenormalIsDAZ(F, Ty))949      return FCmpInst::FCMP_OGT;950    break;951  case fcNegNormal | fcNegInf:952    if (inputDenormalIsDAZ(F, Ty))953      return FCmpInst::FCMP_OLT;954    break;955  case ~fcZero & ~fcNan:956    if (inputDenormalIsIEEE(F, Ty))957      return FCmpInst::FCMP_ONE;958    break;959  case ~(fcZero | fcSubnormal) & ~fcNan:960    if (inputDenormalIsDAZ(F, Ty))961      return FCmpInst::FCMP_ONE;962    break;963  default:964    break;965  }966 967  return FCmpInst::BAD_FCMP_PREDICATE;968}969 970Instruction *InstCombinerImpl::foldIntrinsicIsFPClass(IntrinsicInst &II) {971  Value *Src0 = II.getArgOperand(0);972  Value *Src1 = II.getArgOperand(1);973  const ConstantInt *CMask = cast<ConstantInt>(Src1);974  FPClassTest Mask = static_cast<FPClassTest>(CMask->getZExtValue());975  const bool IsUnordered = (Mask & fcNan) == fcNan;976  const bool IsOrdered = (Mask & fcNan) == fcNone;977  const FPClassTest OrderedMask = Mask & ~fcNan;978  const FPClassTest OrderedInvertedMask = ~OrderedMask & ~fcNan;979 980  const bool IsStrict =981      II.getFunction()->getAttributes().hasFnAttr(Attribute::StrictFP);982 983  Value *FNegSrc;984  if (match(Src0, m_FNeg(m_Value(FNegSrc)))) {985    // is.fpclass (fneg x), mask -> is.fpclass x, (fneg mask)986 987    II.setArgOperand(1, ConstantInt::get(Src1->getType(), fneg(Mask)));988    return replaceOperand(II, 0, FNegSrc);989  }990 991  Value *FAbsSrc;992  if (match(Src0, m_FAbs(m_Value(FAbsSrc)))) {993    II.setArgOperand(1, ConstantInt::get(Src1->getType(), inverse_fabs(Mask)));994    return replaceOperand(II, 0, FAbsSrc);995  }996 997  if ((OrderedMask == fcInf || OrderedInvertedMask == fcInf) &&998      (IsOrdered || IsUnordered) && !IsStrict) {999    // is.fpclass(x, fcInf) -> fcmp oeq fabs(x), +inf1000    // is.fpclass(x, ~fcInf) -> fcmp one fabs(x), +inf1001    // is.fpclass(x, fcInf|fcNan) -> fcmp ueq fabs(x), +inf1002    // is.fpclass(x, ~(fcInf|fcNan)) -> fcmp une fabs(x), +inf1003    Constant *Inf = ConstantFP::getInfinity(Src0->getType());1004    FCmpInst::Predicate Pred =1005        IsUnordered ? FCmpInst::FCMP_UEQ : FCmpInst::FCMP_OEQ;1006    if (OrderedInvertedMask == fcInf)1007      Pred = IsUnordered ? FCmpInst::FCMP_UNE : FCmpInst::FCMP_ONE;1008 1009    Value *Fabs = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, Src0);1010    Value *CmpInf = Builder.CreateFCmp(Pred, Fabs, Inf);1011    CmpInf->takeName(&II);1012    return replaceInstUsesWith(II, CmpInf);1013  }1014 1015  if ((OrderedMask == fcPosInf || OrderedMask == fcNegInf) &&1016      (IsOrdered || IsUnordered) && !IsStrict) {1017    // is.fpclass(x, fcPosInf) -> fcmp oeq x, +inf1018    // is.fpclass(x, fcNegInf) -> fcmp oeq x, -inf1019    // is.fpclass(x, fcPosInf|fcNan) -> fcmp ueq x, +inf1020    // is.fpclass(x, fcNegInf|fcNan) -> fcmp ueq x, -inf1021    Constant *Inf =1022        ConstantFP::getInfinity(Src0->getType(), OrderedMask == fcNegInf);1023    Value *EqInf = IsUnordered ? Builder.CreateFCmpUEQ(Src0, Inf)1024                               : Builder.CreateFCmpOEQ(Src0, Inf);1025 1026    EqInf->takeName(&II);1027    return replaceInstUsesWith(II, EqInf);1028  }1029 1030  if ((OrderedInvertedMask == fcPosInf || OrderedInvertedMask == fcNegInf) &&1031      (IsOrdered || IsUnordered) && !IsStrict) {1032    // is.fpclass(x, ~fcPosInf) -> fcmp one x, +inf1033    // is.fpclass(x, ~fcNegInf) -> fcmp one x, -inf1034    // is.fpclass(x, ~fcPosInf|fcNan) -> fcmp une x, +inf1035    // is.fpclass(x, ~fcNegInf|fcNan) -> fcmp une x, -inf1036    Constant *Inf = ConstantFP::getInfinity(Src0->getType(),1037                                            OrderedInvertedMask == fcNegInf);1038    Value *NeInf = IsUnordered ? Builder.CreateFCmpUNE(Src0, Inf)1039                               : Builder.CreateFCmpONE(Src0, Inf);1040    NeInf->takeName(&II);1041    return replaceInstUsesWith(II, NeInf);1042  }1043 1044  if (Mask == fcNan && !IsStrict) {1045    // Equivalent of isnan. Replace with standard fcmp if we don't care about FP1046    // exceptions.1047    Value *IsNan =1048        Builder.CreateFCmpUNO(Src0, ConstantFP::getZero(Src0->getType()));1049    IsNan->takeName(&II);1050    return replaceInstUsesWith(II, IsNan);1051  }1052 1053  if (Mask == (~fcNan & fcAllFlags) && !IsStrict) {1054    // Equivalent of !isnan. Replace with standard fcmp.1055    Value *FCmp =1056        Builder.CreateFCmpORD(Src0, ConstantFP::getZero(Src0->getType()));1057    FCmp->takeName(&II);1058    return replaceInstUsesWith(II, FCmp);1059  }1060 1061  FCmpInst::Predicate PredType = FCmpInst::BAD_FCMP_PREDICATE;1062 1063  // Try to replace with an fcmp with 01064  //1065  // is.fpclass(x, fcZero) -> fcmp oeq x, 0.01066  // is.fpclass(x, fcZero | fcNan) -> fcmp ueq x, 0.01067  // is.fpclass(x, ~fcZero & ~fcNan) -> fcmp one x, 0.01068  // is.fpclass(x, ~fcZero) -> fcmp une x, 0.01069  //1070  // is.fpclass(x, fcPosSubnormal | fcPosNormal | fcPosInf) -> fcmp ogt x, 0.01071  // is.fpclass(x, fcPositive | fcNegZero) -> fcmp oge x, 0.01072  //1073  // is.fpclass(x, fcNegSubnormal | fcNegNormal | fcNegInf) -> fcmp olt x, 0.01074  // is.fpclass(x, fcNegative | fcPosZero) -> fcmp ole x, 0.01075  //1076  if (!IsStrict && (IsOrdered || IsUnordered) &&1077      (PredType = fpclassTestIsFCmp0(OrderedMask, *II.getFunction(),1078                                     Src0->getType())) !=1079          FCmpInst::BAD_FCMP_PREDICATE) {1080    Constant *Zero = ConstantFP::getZero(Src0->getType());1081    // Equivalent of == 0.1082    Value *FCmp = Builder.CreateFCmp(1083        IsUnordered ? FCmpInst::getUnorderedPredicate(PredType) : PredType,1084        Src0, Zero);1085 1086    FCmp->takeName(&II);1087    return replaceInstUsesWith(II, FCmp);1088  }1089 1090  KnownFPClass Known = computeKnownFPClass(Src0, Mask, &II);1091 1092  // Clear test bits we know must be false from the source value.1093  // fp_class (nnan x), qnan|snan|other -> fp_class (nnan x), other1094  // fp_class (ninf x), ninf|pinf|other -> fp_class (ninf x), other1095  if ((Mask & Known.KnownFPClasses) != Mask) {1096    II.setArgOperand(1097        1, ConstantInt::get(Src1->getType(), Mask & Known.KnownFPClasses));1098    return &II;1099  }1100 1101  // If none of the tests which can return false are possible, fold to true.1102  // fp_class (nnan x), ~(qnan|snan) -> true1103  // fp_class (ninf x), ~(ninf|pinf) -> true1104  if (Mask == Known.KnownFPClasses)1105    return replaceInstUsesWith(II, ConstantInt::get(II.getType(), true));1106 1107  return nullptr;1108}1109 1110static std::optional<bool> getKnownSign(Value *Op, const SimplifyQuery &SQ) {1111  KnownBits Known = computeKnownBits(Op, SQ);1112  if (Known.isNonNegative())1113    return false;1114  if (Known.isNegative())1115    return true;1116 1117  Value *X, *Y;1118  if (match(Op, m_NSWSub(m_Value(X), m_Value(Y))))1119    return isImpliedByDomCondition(ICmpInst::ICMP_SLT, X, Y, SQ.CxtI, SQ.DL);1120 1121  return std::nullopt;1122}1123 1124static std::optional<bool> getKnownSignOrZero(Value *Op,1125                                              const SimplifyQuery &SQ) {1126  if (std::optional<bool> Sign = getKnownSign(Op, SQ))1127    return Sign;1128 1129  Value *X, *Y;1130  if (match(Op, m_NSWSub(m_Value(X), m_Value(Y))))1131    return isImpliedByDomCondition(ICmpInst::ICMP_SLE, X, Y, SQ.CxtI, SQ.DL);1132 1133  return std::nullopt;1134}1135 1136/// Return true if two values \p Op0 and \p Op1 are known to have the same sign.1137static bool signBitMustBeTheSame(Value *Op0, Value *Op1,1138                                 const SimplifyQuery &SQ) {1139  std::optional<bool> Known1 = getKnownSign(Op1, SQ);1140  if (!Known1)1141    return false;1142  std::optional<bool> Known0 = getKnownSign(Op0, SQ);1143  if (!Known0)1144    return false;1145  return *Known0 == *Known1;1146}1147 1148/// Try to canonicalize min/max(X + C0, C1) as min/max(X, C1 - C0) + C0. This1149/// can trigger other combines.1150static Instruction *moveAddAfterMinMax(IntrinsicInst *II,1151                                       InstCombiner::BuilderTy &Builder) {1152  Intrinsic::ID MinMaxID = II->getIntrinsicID();1153  assert((MinMaxID == Intrinsic::smax || MinMaxID == Intrinsic::smin ||1154          MinMaxID == Intrinsic::umax || MinMaxID == Intrinsic::umin) &&1155         "Expected a min or max intrinsic");1156 1157  // TODO: Match vectors with undef elements, but undef may not propagate.1158  Value *Op0 = II->getArgOperand(0), *Op1 = II->getArgOperand(1);1159  Value *X;1160  const APInt *C0, *C1;1161  if (!match(Op0, m_OneUse(m_Add(m_Value(X), m_APInt(C0)))) ||1162      !match(Op1, m_APInt(C1)))1163    return nullptr;1164 1165  // Check for necessary no-wrap and overflow constraints.1166  bool IsSigned = MinMaxID == Intrinsic::smax || MinMaxID == Intrinsic::smin;1167  auto *Add = cast<BinaryOperator>(Op0);1168  if ((IsSigned && !Add->hasNoSignedWrap()) ||1169      (!IsSigned && !Add->hasNoUnsignedWrap()))1170    return nullptr;1171 1172  // If the constant difference overflows, then instsimplify should reduce the1173  // min/max to the add or C1.1174  bool Overflow;1175  APInt CDiff =1176      IsSigned ? C1->ssub_ov(*C0, Overflow) : C1->usub_ov(*C0, Overflow);1177  assert(!Overflow && "Expected simplify of min/max");1178 1179  // min/max (add X, C0), C1 --> add (min/max X, C1 - C0), C01180  // Note: the "mismatched" no-overflow setting does not propagate.1181  Constant *NewMinMaxC = ConstantInt::get(II->getType(), CDiff);1182  Value *NewMinMax = Builder.CreateBinaryIntrinsic(MinMaxID, X, NewMinMaxC);1183  return IsSigned ? BinaryOperator::CreateNSWAdd(NewMinMax, Add->getOperand(1))1184                  : BinaryOperator::CreateNUWAdd(NewMinMax, Add->getOperand(1));1185}1186/// Match a sadd_sat or ssub_sat which is using min/max to clamp the value.1187Instruction *InstCombinerImpl::matchSAddSubSat(IntrinsicInst &MinMax1) {1188  Type *Ty = MinMax1.getType();1189 1190  // We are looking for a tree of:1191  // max(INT_MIN, min(INT_MAX, add(sext(A), sext(B))))1192  // Where the min and max could be reversed1193  Instruction *MinMax2;1194  BinaryOperator *AddSub;1195  const APInt *MinValue, *MaxValue;1196  if (match(&MinMax1, m_SMin(m_Instruction(MinMax2), m_APInt(MaxValue)))) {1197    if (!match(MinMax2, m_SMax(m_BinOp(AddSub), m_APInt(MinValue))))1198      return nullptr;1199  } else if (match(&MinMax1,1200                   m_SMax(m_Instruction(MinMax2), m_APInt(MinValue)))) {1201    if (!match(MinMax2, m_SMin(m_BinOp(AddSub), m_APInt(MaxValue))))1202      return nullptr;1203  } else1204    return nullptr;1205 1206  // Check that the constants clamp a saturate, and that the new type would be1207  // sensible to convert to.1208  if (!(*MaxValue + 1).isPowerOf2() || -*MinValue != *MaxValue + 1)1209    return nullptr;1210  // In what bitwidth can this be treated as saturating arithmetics?1211  unsigned NewBitWidth = (*MaxValue + 1).logBase2() + 1;1212  // FIXME: This isn't quite right for vectors, but using the scalar type is a1213  // good first approximation for what should be done there.1214  if (!shouldChangeType(Ty->getScalarType()->getIntegerBitWidth(), NewBitWidth))1215    return nullptr;1216 1217  // Also make sure that the inner min/max and the add/sub have one use.1218  if (!MinMax2->hasOneUse() || !AddSub->hasOneUse())1219    return nullptr;1220 1221  // Create the new type (which can be a vector type)1222  Type *NewTy = Ty->getWithNewBitWidth(NewBitWidth);1223 1224  Intrinsic::ID IntrinsicID;1225  if (AddSub->getOpcode() == Instruction::Add)1226    IntrinsicID = Intrinsic::sadd_sat;1227  else if (AddSub->getOpcode() == Instruction::Sub)1228    IntrinsicID = Intrinsic::ssub_sat;1229  else1230    return nullptr;1231 1232  // The two operands of the add/sub must be nsw-truncatable to the NewTy. This1233  // is usually achieved via a sext from a smaller type.1234  if (ComputeMaxSignificantBits(AddSub->getOperand(0), AddSub) > NewBitWidth ||1235      ComputeMaxSignificantBits(AddSub->getOperand(1), AddSub) > NewBitWidth)1236    return nullptr;1237 1238  // Finally create and return the sat intrinsic, truncated to the new type1239  Value *AT = Builder.CreateTrunc(AddSub->getOperand(0), NewTy);1240  Value *BT = Builder.CreateTrunc(AddSub->getOperand(1), NewTy);1241  Value *Sat = Builder.CreateIntrinsic(IntrinsicID, NewTy, {AT, BT});1242  return CastInst::Create(Instruction::SExt, Sat, Ty);1243}1244 1245 1246/// If we have a clamp pattern like max (min X, 42), 41 -- where the output1247/// can only be one of two possible constant values -- turn that into a select1248/// of constants.1249static Instruction *foldClampRangeOfTwo(IntrinsicInst *II,1250                                        InstCombiner::BuilderTy &Builder) {1251  Value *I0 = II->getArgOperand(0), *I1 = II->getArgOperand(1);1252  Value *X;1253  const APInt *C0, *C1;1254  if (!match(I1, m_APInt(C1)) || !I0->hasOneUse())1255    return nullptr;1256 1257  CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;1258  switch (II->getIntrinsicID()) {1259  case Intrinsic::smax:1260    if (match(I0, m_SMin(m_Value(X), m_APInt(C0))) && *C0 == *C1 + 1)1261      Pred = ICmpInst::ICMP_SGT;1262    break;1263  case Intrinsic::smin:1264    if (match(I0, m_SMax(m_Value(X), m_APInt(C0))) && *C1 == *C0 + 1)1265      Pred = ICmpInst::ICMP_SLT;1266    break;1267  case Intrinsic::umax:1268    if (match(I0, m_UMin(m_Value(X), m_APInt(C0))) && *C0 == *C1 + 1)1269      Pred = ICmpInst::ICMP_UGT;1270    break;1271  case Intrinsic::umin:1272    if (match(I0, m_UMax(m_Value(X), m_APInt(C0))) && *C1 == *C0 + 1)1273      Pred = ICmpInst::ICMP_ULT;1274    break;1275  default:1276    llvm_unreachable("Expected min/max intrinsic");1277  }1278  if (Pred == CmpInst::BAD_ICMP_PREDICATE)1279    return nullptr;1280 1281  // max (min X, 42), 41 --> X > 41 ? 42 : 411282  // min (max X, 42), 43 --> X < 43 ? 42 : 431283  Value *Cmp = Builder.CreateICmp(Pred, X, I1);1284  return SelectInst::Create(Cmp, ConstantInt::get(II->getType(), *C0), I1);1285}1286 1287/// If this min/max has a constant operand and an operand that is a matching1288/// min/max with a constant operand, constant-fold the 2 constant operands.1289static Value *reassociateMinMaxWithConstants(IntrinsicInst *II,1290                                             IRBuilderBase &Builder,1291                                             const SimplifyQuery &SQ) {1292  Intrinsic::ID MinMaxID = II->getIntrinsicID();1293  auto *LHS = dyn_cast<MinMaxIntrinsic>(II->getArgOperand(0));1294  if (!LHS)1295    return nullptr;1296 1297  Constant *C0, *C1;1298  if (!match(LHS->getArgOperand(1), m_ImmConstant(C0)) ||1299      !match(II->getArgOperand(1), m_ImmConstant(C1)))1300    return nullptr;1301 1302  // max (max X, C0), C1 --> max X, (max C0, C1)1303  // min (min X, C0), C1 --> min X, (min C0, C1)1304  // umax (smax X, nneg C0), nneg C1 --> smax X, (umax C0, C1)1305  // smin (umin X, nneg C0), nneg C1 --> umin X, (smin C0, C1)1306  Intrinsic::ID InnerMinMaxID = LHS->getIntrinsicID();1307  if (InnerMinMaxID != MinMaxID &&1308      !(((MinMaxID == Intrinsic::umax && InnerMinMaxID == Intrinsic::smax) ||1309         (MinMaxID == Intrinsic::smin && InnerMinMaxID == Intrinsic::umin)) &&1310        isKnownNonNegative(C0, SQ) && isKnownNonNegative(C1, SQ)))1311    return nullptr;1312 1313  ICmpInst::Predicate Pred = MinMaxIntrinsic::getPredicate(MinMaxID);1314  Value *CondC = Builder.CreateICmp(Pred, C0, C1);1315  Value *NewC = Builder.CreateSelect(CondC, C0, C1);1316  return Builder.CreateIntrinsic(InnerMinMaxID, II->getType(),1317                                 {LHS->getArgOperand(0), NewC});1318}1319 1320/// If this min/max has a matching min/max operand with a constant, try to push1321/// the constant operand into this instruction. This can enable more folds.1322static Instruction *1323reassociateMinMaxWithConstantInOperand(IntrinsicInst *II,1324                                       InstCombiner::BuilderTy &Builder) {1325  // Match and capture a min/max operand candidate.1326  Value *X, *Y;1327  Constant *C;1328  Instruction *Inner;1329  if (!match(II, m_c_MaxOrMin(m_OneUse(m_CombineAnd(1330                                  m_Instruction(Inner),1331                                  m_MaxOrMin(m_Value(X), m_ImmConstant(C)))),1332                              m_Value(Y))))1333    return nullptr;1334 1335  // The inner op must match. Check for constants to avoid infinite loops.1336  Intrinsic::ID MinMaxID = II->getIntrinsicID();1337  auto *InnerMM = dyn_cast<IntrinsicInst>(Inner);1338  if (!InnerMM || InnerMM->getIntrinsicID() != MinMaxID ||1339      match(X, m_ImmConstant()) || match(Y, m_ImmConstant()))1340    return nullptr;1341 1342  // max (max X, C), Y --> max (max X, Y), C1343  Function *MinMax = Intrinsic::getOrInsertDeclaration(II->getModule(),1344                                                       MinMaxID, II->getType());1345  Value *NewInner = Builder.CreateBinaryIntrinsic(MinMaxID, X, Y);1346  NewInner->takeName(Inner);1347  return CallInst::Create(MinMax, {NewInner, C});1348}1349 1350/// Reduce a sequence of min/max intrinsics with a common operand.1351static Instruction *factorizeMinMaxTree(IntrinsicInst *II) {1352  // Match 3 of the same min/max ops. Example: umin(umin(), umin()).1353  auto *LHS = dyn_cast<IntrinsicInst>(II->getArgOperand(0));1354  auto *RHS = dyn_cast<IntrinsicInst>(II->getArgOperand(1));1355  Intrinsic::ID MinMaxID = II->getIntrinsicID();1356  if (!LHS || !RHS || LHS->getIntrinsicID() != MinMaxID ||1357      RHS->getIntrinsicID() != MinMaxID ||1358      (!LHS->hasOneUse() && !RHS->hasOneUse()))1359    return nullptr;1360 1361  Value *A = LHS->getArgOperand(0);1362  Value *B = LHS->getArgOperand(1);1363  Value *C = RHS->getArgOperand(0);1364  Value *D = RHS->getArgOperand(1);1365 1366  // Look for a common operand.1367  Value *MinMaxOp = nullptr;1368  Value *ThirdOp = nullptr;1369  if (LHS->hasOneUse()) {1370    // If the LHS is only used in this chain and the RHS is used outside of it,1371    // reuse the RHS min/max because that will eliminate the LHS.1372    if (D == A || C == A) {1373      // min(min(a, b), min(c, a)) --> min(min(c, a), b)1374      // min(min(a, b), min(a, d)) --> min(min(a, d), b)1375      MinMaxOp = RHS;1376      ThirdOp = B;1377    } else if (D == B || C == B) {1378      // min(min(a, b), min(c, b)) --> min(min(c, b), a)1379      // min(min(a, b), min(b, d)) --> min(min(b, d), a)1380      MinMaxOp = RHS;1381      ThirdOp = A;1382    }1383  } else {1384    assert(RHS->hasOneUse() && "Expected one-use operand");1385    // Reuse the LHS. This will eliminate the RHS.1386    if (D == A || D == B) {1387      // min(min(a, b), min(c, a)) --> min(min(a, b), c)1388      // min(min(a, b), min(c, b)) --> min(min(a, b), c)1389      MinMaxOp = LHS;1390      ThirdOp = C;1391    } else if (C == A || C == B) {1392      // min(min(a, b), min(b, d)) --> min(min(a, b), d)1393      // min(min(a, b), min(c, b)) --> min(min(a, b), d)1394      MinMaxOp = LHS;1395      ThirdOp = D;1396    }1397  }1398 1399  if (!MinMaxOp || !ThirdOp)1400    return nullptr;1401 1402  Module *Mod = II->getModule();1403  Function *MinMax =1404      Intrinsic::getOrInsertDeclaration(Mod, MinMaxID, II->getType());1405  return CallInst::Create(MinMax, { MinMaxOp, ThirdOp });1406}1407 1408/// If all arguments of the intrinsic are unary shuffles with the same mask,1409/// try to shuffle after the intrinsic.1410Instruction *1411InstCombinerImpl::foldShuffledIntrinsicOperands(IntrinsicInst *II) {1412  if (!isTriviallyVectorizable(II->getIntrinsicID()) ||1413      !II->getCalledFunction()->isSpeculatable())1414    return nullptr;1415 1416  Value *X;1417  Constant *C;1418  ArrayRef<int> Mask;1419  auto *NonConstArg = find_if_not(II->args(), [&II](Use &Arg) {1420    return isa<Constant>(Arg.get()) ||1421           isVectorIntrinsicWithScalarOpAtArg(II->getIntrinsicID(),1422                                              Arg.getOperandNo(), nullptr);1423  });1424  if (!NonConstArg ||1425      !match(NonConstArg, m_Shuffle(m_Value(X), m_Poison(), m_Mask(Mask))))1426    return nullptr;1427 1428  // At least 1 operand must be a shuffle with 1 use because we are creating 21429  // instructions.1430  if (none_of(II->args(), match_fn(m_OneUse(m_Shuffle(m_Value(), m_Value())))))1431    return nullptr;1432 1433  // See if all arguments are shuffled with the same mask.1434  SmallVector<Value *, 4> NewArgs;1435  Type *SrcTy = X->getType();1436  for (Use &Arg : II->args()) {1437    if (isVectorIntrinsicWithScalarOpAtArg(II->getIntrinsicID(),1438                                           Arg.getOperandNo(), nullptr))1439      NewArgs.push_back(Arg);1440    else if (match(&Arg,1441                   m_Shuffle(m_Value(X), m_Poison(), m_SpecificMask(Mask))) &&1442             X->getType() == SrcTy)1443      NewArgs.push_back(X);1444    else if (match(&Arg, m_ImmConstant(C))) {1445      // If it's a constant, try find the constant that would be shuffled to C.1446      if (Constant *ShuffledC =1447              unshuffleConstant(Mask, C, cast<VectorType>(SrcTy)))1448        NewArgs.push_back(ShuffledC);1449      else1450        return nullptr;1451    } else1452      return nullptr;1453  }1454 1455  // intrinsic (shuf X, M), (shuf Y, M), ... --> shuf (intrinsic X, Y, ...), M1456  Instruction *FPI = isa<FPMathOperator>(II) ? II : nullptr;1457  // Result type might be a different vector width.1458  // TODO: Check that the result type isn't widened?1459  VectorType *ResTy =1460      VectorType::get(II->getType()->getScalarType(), cast<VectorType>(SrcTy));1461  Value *NewIntrinsic =1462      Builder.CreateIntrinsic(ResTy, II->getIntrinsicID(), NewArgs, FPI);1463  return new ShuffleVectorInst(NewIntrinsic, Mask);1464}1465 1466/// If all arguments of the intrinsic are reverses, try to pull the reverse1467/// after the intrinsic.1468Value *InstCombinerImpl::foldReversedIntrinsicOperands(IntrinsicInst *II) {1469  if (!isTriviallyVectorizable(II->getIntrinsicID()))1470    return nullptr;1471 1472  // At least 1 operand must be a reverse with 1 use because we are creating 21473  // instructions.1474  if (none_of(II->args(), [](Value *V) {1475        return match(V, m_OneUse(m_VecReverse(m_Value())));1476      }))1477    return nullptr;1478 1479  Value *X;1480  Constant *C;1481  SmallVector<Value *> NewArgs;1482  for (Use &Arg : II->args()) {1483    if (isVectorIntrinsicWithScalarOpAtArg(II->getIntrinsicID(),1484                                           Arg.getOperandNo(), nullptr))1485      NewArgs.push_back(Arg);1486    else if (match(&Arg, m_VecReverse(m_Value(X))))1487      NewArgs.push_back(X);1488    else if (isSplatValue(Arg))1489      NewArgs.push_back(Arg);1490    else if (match(&Arg, m_ImmConstant(C)))1491      NewArgs.push_back(Builder.CreateVectorReverse(C));1492    else1493      return nullptr;1494  }1495 1496  // intrinsic (reverse X), (reverse Y), ... --> reverse (intrinsic X, Y, ...)1497  Instruction *FPI = isa<FPMathOperator>(II) ? II : nullptr;1498  Instruction *NewIntrinsic = Builder.CreateIntrinsic(1499      II->getType(), II->getIntrinsicID(), NewArgs, FPI);1500  return Builder.CreateVectorReverse(NewIntrinsic);1501}1502 1503/// Fold the following cases and accepts bswap and bitreverse intrinsics:1504///   bswap(logic_op(bswap(x), y)) --> logic_op(x, bswap(y))1505///   bswap(logic_op(bswap(x), bswap(y))) --> logic_op(x, y) (ignores multiuse)1506template <Intrinsic::ID IntrID>1507static Instruction *foldBitOrderCrossLogicOp(Value *V,1508                                             InstCombiner::BuilderTy &Builder) {1509  static_assert(IntrID == Intrinsic::bswap || IntrID == Intrinsic::bitreverse,1510                "This helper only supports BSWAP and BITREVERSE intrinsics");1511 1512  Value *X, *Y;1513  // Find bitwise logic op. Check that it is a BinaryOperator explicitly so we1514  // don't match ConstantExpr that aren't meaningful for this transform.1515  if (match(V, m_OneUse(m_BitwiseLogic(m_Value(X), m_Value(Y)))) &&1516      isa<BinaryOperator>(V)) {1517    Value *OldReorderX, *OldReorderY;1518    BinaryOperator::BinaryOps Op = cast<BinaryOperator>(V)->getOpcode();1519 1520    // If both X and Y are bswap/bitreverse, the transform reduces the number1521    // of instructions even if there's multiuse.1522    // If only one operand is bswap/bitreverse, we need to ensure the operand1523    // have only one use.1524    if (match(X, m_Intrinsic<IntrID>(m_Value(OldReorderX))) &&1525        match(Y, m_Intrinsic<IntrID>(m_Value(OldReorderY)))) {1526      return BinaryOperator::Create(Op, OldReorderX, OldReorderY);1527    }1528 1529    if (match(X, m_OneUse(m_Intrinsic<IntrID>(m_Value(OldReorderX))))) {1530      Value *NewReorder = Builder.CreateUnaryIntrinsic(IntrID, Y);1531      return BinaryOperator::Create(Op, OldReorderX, NewReorder);1532    }1533 1534    if (match(Y, m_OneUse(m_Intrinsic<IntrID>(m_Value(OldReorderY))))) {1535      Value *NewReorder = Builder.CreateUnaryIntrinsic(IntrID, X);1536      return BinaryOperator::Create(Op, NewReorder, OldReorderY);1537    }1538  }1539  return nullptr;1540}1541 1542/// Helper to match idempotent binary intrinsics, namely, intrinsics where1543/// `f(f(x, y), y) == f(x, y)` holds.1544static bool isIdempotentBinaryIntrinsic(Intrinsic::ID IID) {1545  switch (IID) {1546  case Intrinsic::smax:1547  case Intrinsic::smin:1548  case Intrinsic::umax:1549  case Intrinsic::umin:1550  case Intrinsic::maximum:1551  case Intrinsic::minimum:1552  case Intrinsic::maximumnum:1553  case Intrinsic::minimumnum:1554  case Intrinsic::maxnum:1555  case Intrinsic::minnum:1556    return true;1557  default:1558    return false;1559  }1560}1561 1562/// Attempt to simplify value-accumulating recurrences of kind:1563///   %umax.acc = phi i8 [ %umax, %backedge ], [ %a, %entry ]1564///   %umax = call i8 @llvm.umax.i8(i8 %umax.acc, i8 %b)1565/// And let the idempotent binary intrinsic be hoisted, when the operands are1566/// known to be loop-invariant.1567static Value *foldIdempotentBinaryIntrinsicRecurrence(InstCombinerImpl &IC,1568                                                      IntrinsicInst *II) {1569  PHINode *PN;1570  Value *Init, *OtherOp;1571 1572  // A binary intrinsic recurrence with loop-invariant operands is equivalent to1573  // `call @llvm.binary.intrinsic(Init, OtherOp)`.1574  auto IID = II->getIntrinsicID();1575  if (!isIdempotentBinaryIntrinsic(IID) ||1576      !matchSimpleBinaryIntrinsicRecurrence(II, PN, Init, OtherOp) ||1577      !IC.getDominatorTree().dominates(OtherOp, PN))1578    return nullptr;1579 1580  auto *InvariantBinaryInst =1581      IC.Builder.CreateBinaryIntrinsic(IID, Init, OtherOp);1582  if (isa<FPMathOperator>(InvariantBinaryInst))1583    cast<Instruction>(InvariantBinaryInst)->copyFastMathFlags(II);1584  return InvariantBinaryInst;1585}1586 1587static Value *simplifyReductionOperand(Value *Arg, bool CanReorderLanes) {1588  if (!CanReorderLanes)1589    return nullptr;1590 1591  Value *V;1592  if (match(Arg, m_VecReverse(m_Value(V))))1593    return V;1594 1595  ArrayRef<int> Mask;1596  if (!isa<FixedVectorType>(Arg->getType()) ||1597      !match(Arg, m_Shuffle(m_Value(V), m_Undef(), m_Mask(Mask))) ||1598      !cast<ShuffleVectorInst>(Arg)->isSingleSource())1599    return nullptr;1600 1601  int Sz = Mask.size();1602  SmallBitVector UsedIndices(Sz);1603  for (int Idx : Mask) {1604    if (Idx == PoisonMaskElem || UsedIndices.test(Idx))1605      return nullptr;1606    UsedIndices.set(Idx);1607  }1608 1609  // Can remove shuffle iff just shuffled elements, no repeats, undefs, or1610  // other changes.1611  return UsedIndices.all() ? V : nullptr;1612}1613 1614/// Fold an unsigned minimum of trailing or leading zero bits counts:1615///   umin(cttz(CtOp, ZeroUndef), ConstOp) --> cttz(CtOp | (1 << ConstOp))1616///   umin(ctlz(CtOp, ZeroUndef), ConstOp) --> ctlz(CtOp | (SignedMin1617///                                              >> ConstOp))1618template <Intrinsic::ID IntrID>1619static Value *1620foldMinimumOverTrailingOrLeadingZeroCount(Value *I0, Value *I1,1621                                          const DataLayout &DL,1622                                          InstCombiner::BuilderTy &Builder) {1623  static_assert(IntrID == Intrinsic::cttz || IntrID == Intrinsic::ctlz,1624                "This helper only supports cttz and ctlz intrinsics");1625 1626  Value *CtOp;1627  Value *ZeroUndef;1628  if (!match(I0,1629             m_OneUse(m_Intrinsic<IntrID>(m_Value(CtOp), m_Value(ZeroUndef)))))1630    return nullptr;1631 1632  unsigned BitWidth = I1->getType()->getScalarSizeInBits();1633  auto LessBitWidth = [BitWidth](auto &C) { return C.ult(BitWidth); };1634  if (!match(I1, m_CheckedInt(LessBitWidth)))1635    // We have a constant >= BitWidth (which can be handled by CVP)1636    // or a non-splat vector with elements < and >= BitWidth1637    return nullptr;1638 1639  Type *Ty = I1->getType();1640  Constant *NewConst = ConstantFoldBinaryOpOperands(1641      IntrID == Intrinsic::cttz ? Instruction::Shl : Instruction::LShr,1642      IntrID == Intrinsic::cttz1643          ? ConstantInt::get(Ty, 1)1644          : ConstantInt::get(Ty, APInt::getSignedMinValue(BitWidth)),1645      cast<Constant>(I1), DL);1646  return Builder.CreateBinaryIntrinsic(1647      IntrID, Builder.CreateOr(CtOp, NewConst),1648      ConstantInt::getTrue(ZeroUndef->getType()));1649}1650 1651/// Return whether "X LOp (Y ROp Z)" is always equal to1652/// "(X LOp Y) ROp (X LOp Z)".1653static bool leftDistributesOverRight(Instruction::BinaryOps LOp, bool HasNUW,1654                                     bool HasNSW, Intrinsic::ID ROp) {1655  switch (ROp) {1656  case Intrinsic::umax:1657  case Intrinsic::umin:1658    if (HasNUW && LOp == Instruction::Add)1659      return true;1660    if (HasNUW && LOp == Instruction::Shl)1661      return true;1662    return false;1663  case Intrinsic::smax:1664  case Intrinsic::smin:1665    return HasNSW && LOp == Instruction::Add;1666  default:1667    return false;1668  }1669}1670 1671// Attempts to factorise a common term1672// in an instruction that has the form "(A op' B) op (C op' D)1673// where op is an intrinsic and op' is a binop1674static Value *1675foldIntrinsicUsingDistributiveLaws(IntrinsicInst *II,1676                                   InstCombiner::BuilderTy &Builder) {1677  Value *LHS = II->getOperand(0), *RHS = II->getOperand(1);1678  Intrinsic::ID TopLevelOpcode = II->getIntrinsicID();1679 1680  OverflowingBinaryOperator *Op0 = dyn_cast<OverflowingBinaryOperator>(LHS);1681  OverflowingBinaryOperator *Op1 = dyn_cast<OverflowingBinaryOperator>(RHS);1682 1683  if (!Op0 || !Op1)1684    return nullptr;1685 1686  if (Op0->getOpcode() != Op1->getOpcode())1687    return nullptr;1688 1689  if (!Op0->hasOneUse() || !Op1->hasOneUse())1690    return nullptr;1691 1692  Instruction::BinaryOps InnerOpcode =1693      static_cast<Instruction::BinaryOps>(Op0->getOpcode());1694  bool HasNUW = Op0->hasNoUnsignedWrap() && Op1->hasNoUnsignedWrap();1695  bool HasNSW = Op0->hasNoSignedWrap() && Op1->hasNoSignedWrap();1696 1697  if (!leftDistributesOverRight(InnerOpcode, HasNUW, HasNSW, TopLevelOpcode))1698    return nullptr;1699 1700  Value *A = Op0->getOperand(0);1701  Value *B = Op0->getOperand(1);1702  Value *C = Op1->getOperand(0);1703  Value *D = Op1->getOperand(1);1704 1705  // Attempts to swap variables such that A equals C or B equals D,1706  // if the inner operation is commutative.1707  if (Op0->isCommutative() && A != C && B != D) {1708    if (A == D || B == C)1709      std::swap(C, D);1710    else1711      return nullptr;1712  }1713 1714  BinaryOperator *NewBinop;1715  if (A == C) {1716    Value *NewIntrinsic = Builder.CreateBinaryIntrinsic(TopLevelOpcode, B, D);1717    NewBinop =1718        cast<BinaryOperator>(Builder.CreateBinOp(InnerOpcode, A, NewIntrinsic));1719  } else if (B == D) {1720    Value *NewIntrinsic = Builder.CreateBinaryIntrinsic(TopLevelOpcode, A, C);1721    NewBinop =1722        cast<BinaryOperator>(Builder.CreateBinOp(InnerOpcode, NewIntrinsic, B));1723  } else {1724    return nullptr;1725  }1726 1727  NewBinop->setHasNoUnsignedWrap(HasNUW);1728  NewBinop->setHasNoSignedWrap(HasNSW);1729 1730  return NewBinop;1731}1732 1733/// CallInst simplification. This mostly only handles folding of intrinsic1734/// instructions. For normal calls, it allows visitCallBase to do the heavy1735/// lifting.1736Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {1737  // Don't try to simplify calls without uses. It will not do anything useful,1738  // but will result in the following folds being skipped.1739  if (!CI.use_empty()) {1740    SmallVector<Value *, 8> Args(CI.args());1741    if (Value *V = simplifyCall(&CI, CI.getCalledOperand(), Args,1742                                SQ.getWithInstruction(&CI)))1743      return replaceInstUsesWith(CI, V);1744  }1745 1746  if (Value *FreedOp = getFreedOperand(&CI, &TLI))1747    return visitFree(CI, FreedOp);1748 1749  // If the caller function (i.e. us, the function that contains this CallInst)1750  // is nounwind, mark the call as nounwind, even if the callee isn't.1751  if (CI.getFunction()->doesNotThrow() && !CI.doesNotThrow()) {1752    CI.setDoesNotThrow();1753    return &CI;1754  }1755 1756  IntrinsicInst *II = dyn_cast<IntrinsicInst>(&CI);1757  if (!II)1758    return visitCallBase(CI);1759 1760  // Intrinsics cannot occur in an invoke or a callbr, so handle them here1761  // instead of in visitCallBase.1762  if (auto *MI = dyn_cast<AnyMemIntrinsic>(II)) {1763    if (auto NumBytes = MI->getLengthInBytes()) {1764      // memmove/cpy/set of zero bytes is a noop.1765      if (NumBytes->isZero())1766        return eraseInstFromFunction(CI);1767 1768      // For atomic unordered mem intrinsics if len is not a positive or1769      // not a multiple of element size then behavior is undefined.1770      if (MI->isAtomic() &&1771          (NumBytes->isNegative() ||1772           (NumBytes->getZExtValue() % MI->getElementSizeInBytes() != 0))) {1773        CreateNonTerminatorUnreachable(MI);1774        assert(MI->getType()->isVoidTy() &&1775               "non void atomic unordered mem intrinsic");1776        return eraseInstFromFunction(*MI);1777      }1778    }1779 1780    // No other transformations apply to volatile transfers.1781    if (MI->isVolatile())1782      return nullptr;1783 1784    if (AnyMemTransferInst *MTI = dyn_cast<AnyMemTransferInst>(MI)) {1785      // memmove(x,x,size) -> noop.1786      if (MTI->getSource() == MTI->getDest())1787        return eraseInstFromFunction(CI);1788    }1789 1790    auto IsPointerUndefined = [MI](Value *Ptr) {1791      return isa<ConstantPointerNull>(Ptr) &&1792             !NullPointerIsDefined(1793                 MI->getFunction(),1794                 cast<PointerType>(Ptr->getType())->getAddressSpace());1795    };1796    bool SrcIsUndefined = false;1797    // If we can determine a pointer alignment that is bigger than currently1798    // set, update the alignment.1799    if (auto *MTI = dyn_cast<AnyMemTransferInst>(MI)) {1800      if (Instruction *I = SimplifyAnyMemTransfer(MTI))1801        return I;1802      SrcIsUndefined = IsPointerUndefined(MTI->getRawSource());1803    } else if (auto *MSI = dyn_cast<AnyMemSetInst>(MI)) {1804      if (Instruction *I = SimplifyAnyMemSet(MSI))1805        return I;1806    }1807 1808    // If src/dest is null, this memory intrinsic must be a noop.1809    if (SrcIsUndefined || IsPointerUndefined(MI->getRawDest())) {1810      Builder.CreateAssumption(Builder.CreateIsNull(MI->getLength()));1811      return eraseInstFromFunction(CI);1812    }1813 1814    // If we have a memmove and the source operation is a constant global,1815    // then the source and dest pointers can't alias, so we can change this1816    // into a call to memcpy.1817    if (auto *MMI = dyn_cast<AnyMemMoveInst>(MI)) {1818      if (GlobalVariable *GVSrc = dyn_cast<GlobalVariable>(MMI->getSource()))1819        if (GVSrc->isConstant()) {1820          Module *M = CI.getModule();1821          Intrinsic::ID MemCpyID =1822              MMI->isAtomic()1823                  ? Intrinsic::memcpy_element_unordered_atomic1824                  : Intrinsic::memcpy;1825          Type *Tys[3] = { CI.getArgOperand(0)->getType(),1826                           CI.getArgOperand(1)->getType(),1827                           CI.getArgOperand(2)->getType() };1828          CI.setCalledFunction(1829              Intrinsic::getOrInsertDeclaration(M, MemCpyID, Tys));1830          return II;1831        }1832    }1833  }1834 1835  // For fixed width vector result intrinsics, use the generic demanded vector1836  // support.1837  if (auto *IIFVTy = dyn_cast<FixedVectorType>(II->getType())) {1838    auto VWidth = IIFVTy->getNumElements();1839    APInt PoisonElts(VWidth, 0);1840    APInt AllOnesEltMask(APInt::getAllOnes(VWidth));1841    if (Value *V = SimplifyDemandedVectorElts(II, AllOnesEltMask, PoisonElts)) {1842      if (V != II)1843        return replaceInstUsesWith(*II, V);1844      return II;1845    }1846  }1847 1848  if (II->isCommutative()) {1849    if (auto Pair = matchSymmetricPair(II->getOperand(0), II->getOperand(1))) {1850      replaceOperand(*II, 0, Pair->first);1851      replaceOperand(*II, 1, Pair->second);1852      return II;1853    }1854 1855    if (CallInst *NewCall = canonicalizeConstantArg0ToArg1(CI))1856      return NewCall;1857  }1858 1859  // Unused constrained FP intrinsic calls may have declared side effect, which1860  // prevents it from being removed. In some cases however the side effect is1861  // actually absent. To detect this case, call SimplifyConstrainedFPCall. If it1862  // returns a replacement, the call may be removed.1863  if (CI.use_empty() && isa<ConstrainedFPIntrinsic>(CI)) {1864    if (simplifyConstrainedFPCall(&CI, SQ.getWithInstruction(&CI)))1865      return eraseInstFromFunction(CI);1866  }1867 1868  Intrinsic::ID IID = II->getIntrinsicID();1869  switch (IID) {1870  case Intrinsic::objectsize: {1871    SmallVector<Instruction *> InsertedInstructions;1872    if (Value *V = lowerObjectSizeCall(II, DL, &TLI, AA, /*MustSucceed=*/false,1873                                       &InsertedInstructions)) {1874      for (Instruction *Inserted : InsertedInstructions)1875        Worklist.add(Inserted);1876      return replaceInstUsesWith(CI, V);1877    }1878    return nullptr;1879  }1880  case Intrinsic::abs: {1881    Value *IIOperand = II->getArgOperand(0);1882    bool IntMinIsPoison = cast<Constant>(II->getArgOperand(1))->isOneValue();1883 1884    // abs(-x) -> abs(x)1885    Value *X;1886    if (match(IIOperand, m_Neg(m_Value(X)))) {1887      if (cast<Instruction>(IIOperand)->hasNoSignedWrap() || IntMinIsPoison)1888        replaceOperand(*II, 1, Builder.getTrue());1889      return replaceOperand(*II, 0, X);1890    }1891    if (match(IIOperand, m_c_Select(m_Neg(m_Value(X)), m_Deferred(X))))1892      return replaceOperand(*II, 0, X);1893 1894    Value *Y;1895    // abs(a * abs(b)) -> abs(a * b)1896    if (match(IIOperand,1897              m_OneUse(m_c_Mul(m_Value(X),1898                               m_Intrinsic<Intrinsic::abs>(m_Value(Y)))))) {1899      bool NSW =1900          cast<Instruction>(IIOperand)->hasNoSignedWrap() && IntMinIsPoison;1901      auto *XY = NSW ? Builder.CreateNSWMul(X, Y) : Builder.CreateMul(X, Y);1902      return replaceOperand(*II, 0, XY);1903    }1904 1905    if (std::optional<bool> Known =1906            getKnownSignOrZero(IIOperand, SQ.getWithInstruction(II))) {1907      // abs(x) -> x if x >= 0 (include abs(x-y) --> x - y where x >= y)1908      // abs(x) -> x if x > 0 (include abs(x-y) --> x - y where x > y)1909      if (!*Known)1910        return replaceInstUsesWith(*II, IIOperand);1911 1912      // abs(x) -> -x if x < 01913      // abs(x) -> -x if x < = 0 (include abs(x-y) --> y - x where x <= y)1914      if (IntMinIsPoison)1915        return BinaryOperator::CreateNSWNeg(IIOperand);1916      return BinaryOperator::CreateNeg(IIOperand);1917    }1918 1919    // abs (sext X) --> zext (abs X*)1920    // Clear the IsIntMin (nsw) bit on the abs to allow narrowing.1921    if (match(IIOperand, m_OneUse(m_SExt(m_Value(X))))) {1922      Value *NarrowAbs =1923          Builder.CreateBinaryIntrinsic(Intrinsic::abs, X, Builder.getFalse());1924      return CastInst::Create(Instruction::ZExt, NarrowAbs, II->getType());1925    }1926 1927    // Match a complicated way to check if a number is odd/even:1928    // abs (srem X, 2) --> and X, 11929    const APInt *C;1930    if (match(IIOperand, m_SRem(m_Value(X), m_APInt(C))) && *C == 2)1931      return BinaryOperator::CreateAnd(X, ConstantInt::get(II->getType(), 1));1932 1933    break;1934  }1935  case Intrinsic::umin: {1936    Value *I0 = II->getArgOperand(0), *I1 = II->getArgOperand(1);1937    // umin(x, 1) == zext(x != 0)1938    if (match(I1, m_One())) {1939      assert(II->getType()->getScalarSizeInBits() != 1 &&1940             "Expected simplify of umin with max constant");1941      Value *Zero = Constant::getNullValue(I0->getType());1942      Value *Cmp = Builder.CreateICmpNE(I0, Zero);1943      return CastInst::Create(Instruction::ZExt, Cmp, II->getType());1944    }1945    // umin(cttz(x), const) --> cttz(x | (1 << const))1946    if (Value *FoldedCttz =1947            foldMinimumOverTrailingOrLeadingZeroCount<Intrinsic::cttz>(1948                I0, I1, DL, Builder))1949      return replaceInstUsesWith(*II, FoldedCttz);1950    // umin(ctlz(x), const) --> ctlz(x | (SignedMin >> const))1951    if (Value *FoldedCtlz =1952            foldMinimumOverTrailingOrLeadingZeroCount<Intrinsic::ctlz>(1953                I0, I1, DL, Builder))1954      return replaceInstUsesWith(*II, FoldedCtlz);1955    [[fallthrough]];1956  }1957  case Intrinsic::umax: {1958    Value *I0 = II->getArgOperand(0), *I1 = II->getArgOperand(1);1959    Value *X, *Y;1960    if (match(I0, m_ZExt(m_Value(X))) && match(I1, m_ZExt(m_Value(Y))) &&1961        (I0->hasOneUse() || I1->hasOneUse()) && X->getType() == Y->getType()) {1962      Value *NarrowMaxMin = Builder.CreateBinaryIntrinsic(IID, X, Y);1963      return CastInst::Create(Instruction::ZExt, NarrowMaxMin, II->getType());1964    }1965    Constant *C;1966    if (match(I0, m_ZExt(m_Value(X))) && match(I1, m_Constant(C)) &&1967        I0->hasOneUse()) {1968      if (Constant *NarrowC = getLosslessUnsignedTrunc(C, X->getType(), DL)) {1969        Value *NarrowMaxMin = Builder.CreateBinaryIntrinsic(IID, X, NarrowC);1970        return CastInst::Create(Instruction::ZExt, NarrowMaxMin, II->getType());1971      }1972    }1973    // If C is not 0:1974    //   umax(nuw_shl(x, C), x + 1) -> x == 0 ? 1 : nuw_shl(x, C)1975    // If C is not 0 or 1:1976    //   umax(nuw_mul(x, C), x + 1) -> x == 0 ? 1 : nuw_mul(x, C)1977    auto foldMaxMulShift = [&](Value *A, Value *B) -> Instruction * {1978      const APInt *C;1979      Value *X;1980      if (!match(A, m_NUWShl(m_Value(X), m_APInt(C))) &&1981          !(match(A, m_NUWMul(m_Value(X), m_APInt(C))) && !C->isOne()))1982        return nullptr;1983      if (C->isZero())1984        return nullptr;1985      if (!match(B, m_OneUse(m_Add(m_Specific(X), m_One()))))1986        return nullptr;1987 1988      Value *Cmp = Builder.CreateICmpEQ(X, ConstantInt::get(X->getType(), 0));1989      Value *NewSelect =1990          Builder.CreateSelect(Cmp, ConstantInt::get(X->getType(), 1), A);1991      return replaceInstUsesWith(*II, NewSelect);1992    };1993 1994    if (IID == Intrinsic::umax) {1995      if (Instruction *I = foldMaxMulShift(I0, I1))1996        return I;1997      if (Instruction *I = foldMaxMulShift(I1, I0))1998        return I;1999    }2000 2001    // If both operands of unsigned min/max are sign-extended, it is still ok2002    // to narrow the operation.2003    [[fallthrough]];2004  }2005  case Intrinsic::smax:2006  case Intrinsic::smin: {2007    Value *I0 = II->getArgOperand(0), *I1 = II->getArgOperand(1);2008    Value *X, *Y;2009    if (match(I0, m_SExt(m_Value(X))) && match(I1, m_SExt(m_Value(Y))) &&2010        (I0->hasOneUse() || I1->hasOneUse()) && X->getType() == Y->getType()) {2011      Value *NarrowMaxMin = Builder.CreateBinaryIntrinsic(IID, X, Y);2012      return CastInst::Create(Instruction::SExt, NarrowMaxMin, II->getType());2013    }2014 2015    Constant *C;2016    if (match(I0, m_SExt(m_Value(X))) && match(I1, m_Constant(C)) &&2017        I0->hasOneUse()) {2018      if (Constant *NarrowC = getLosslessSignedTrunc(C, X->getType(), DL)) {2019        Value *NarrowMaxMin = Builder.CreateBinaryIntrinsic(IID, X, NarrowC);2020        return CastInst::Create(Instruction::SExt, NarrowMaxMin, II->getType());2021      }2022    }2023 2024    // smax(smin(X, MinC), MaxC) -> smin(smax(X, MaxC), MinC) if MinC s>= MaxC2025    // umax(umin(X, MinC), MaxC) -> umin(umax(X, MaxC), MinC) if MinC u>= MaxC2026    const APInt *MinC, *MaxC;2027    auto CreateCanonicalClampForm = [&](bool IsSigned) {2028      auto MaxIID = IsSigned ? Intrinsic::smax : Intrinsic::umax;2029      auto MinIID = IsSigned ? Intrinsic::smin : Intrinsic::umin;2030      Value *NewMax = Builder.CreateBinaryIntrinsic(2031          MaxIID, X, ConstantInt::get(X->getType(), *MaxC));2032      return replaceInstUsesWith(2033          *II, Builder.CreateBinaryIntrinsic(2034                   MinIID, NewMax, ConstantInt::get(X->getType(), *MinC)));2035    };2036    if (IID == Intrinsic::smax &&2037        match(I0, m_OneUse(m_Intrinsic<Intrinsic::smin>(m_Value(X),2038                                                        m_APInt(MinC)))) &&2039        match(I1, m_APInt(MaxC)) && MinC->sgt(*MaxC))2040      return CreateCanonicalClampForm(true);2041    if (IID == Intrinsic::umax &&2042        match(I0, m_OneUse(m_Intrinsic<Intrinsic::umin>(m_Value(X),2043                                                        m_APInt(MinC)))) &&2044        match(I1, m_APInt(MaxC)) && MinC->ugt(*MaxC))2045      return CreateCanonicalClampForm(false);2046 2047    // umin(i1 X, i1 Y) -> and i1 X, Y2048    // smax(i1 X, i1 Y) -> and i1 X, Y2049    if ((IID == Intrinsic::umin || IID == Intrinsic::smax) &&2050        II->getType()->isIntOrIntVectorTy(1)) {2051      return BinaryOperator::CreateAnd(I0, I1);2052    }2053 2054    // umax(i1 X, i1 Y) -> or i1 X, Y2055    // smin(i1 X, i1 Y) -> or i1 X, Y2056    if ((IID == Intrinsic::umax || IID == Intrinsic::smin) &&2057        II->getType()->isIntOrIntVectorTy(1)) {2058      return BinaryOperator::CreateOr(I0, I1);2059    }2060 2061    // smin(smax(X, -1), 1) -> scmp(X, 0)2062    // smax(smin(X, 1), -1) -> scmp(X, 0)2063    // At this point, smax(smin(X, 1), -1) is changed to smin(smax(X, -1)2064    // And i1's have been changed to and/ors2065    // So we only need to check for smin2066    if (IID == Intrinsic::smin) {2067      if (match(I0, m_OneUse(m_SMax(m_Value(X), m_AllOnes()))) &&2068          match(I1, m_One())) {2069        Value *Zero = ConstantInt::get(X->getType(), 0);2070        return replaceInstUsesWith(2071            CI,2072            Builder.CreateIntrinsic(II->getType(), Intrinsic::scmp, {X, Zero}));2073      }2074    }2075 2076    if (IID == Intrinsic::smax || IID == Intrinsic::smin) {2077      // smax (neg nsw X), (neg nsw Y) --> neg nsw (smin X, Y)2078      // smin (neg nsw X), (neg nsw Y) --> neg nsw (smax X, Y)2079      // TODO: Canonicalize neg after min/max if I1 is constant.2080      if (match(I0, m_NSWNeg(m_Value(X))) && match(I1, m_NSWNeg(m_Value(Y))) &&2081          (I0->hasOneUse() || I1->hasOneUse())) {2082        Intrinsic::ID InvID = getInverseMinMaxIntrinsic(IID);2083        Value *InvMaxMin = Builder.CreateBinaryIntrinsic(InvID, X, Y);2084        return BinaryOperator::CreateNSWNeg(InvMaxMin);2085      }2086    }2087 2088    // (umax X, (xor X, Pow2))2089    //      -> (or X, Pow2)2090    // (umin X, (xor X, Pow2))2091    //      -> (and X, ~Pow2)2092    // (smax X, (xor X, Pos_Pow2))2093    //      -> (or X, Pos_Pow2)2094    // (smin X, (xor X, Pos_Pow2))2095    //      -> (and X, ~Pos_Pow2)2096    // (smax X, (xor X, Neg_Pow2))2097    //      -> (and X, ~Neg_Pow2)2098    // (smin X, (xor X, Neg_Pow2))2099    //      -> (or X, Neg_Pow2)2100    if ((match(I0, m_c_Xor(m_Specific(I1), m_Value(X))) ||2101         match(I1, m_c_Xor(m_Specific(I0), m_Value(X)))) &&2102        isKnownToBeAPowerOfTwo(X, /* OrZero */ true)) {2103      bool UseOr = IID == Intrinsic::smax || IID == Intrinsic::umax;2104      bool UseAndN = IID == Intrinsic::smin || IID == Intrinsic::umin;2105 2106      if (IID == Intrinsic::smax || IID == Intrinsic::smin) {2107        auto KnownSign = getKnownSign(X, SQ.getWithInstruction(II));2108        if (KnownSign == std::nullopt) {2109          UseOr = false;2110          UseAndN = false;2111        } else if (*KnownSign /* true is Signed. */) {2112          UseOr ^= true;2113          UseAndN ^= true;2114          Type *Ty = I0->getType();2115          // Negative power of 2 must be IntMin. It's possible to be able to2116          // prove negative / power of 2 without actually having known bits, so2117          // just get the value by hand.2118          X = Constant::getIntegerValue(2119              Ty, APInt::getSignedMinValue(Ty->getScalarSizeInBits()));2120        }2121      }2122      if (UseOr)2123        return BinaryOperator::CreateOr(I0, X);2124      else if (UseAndN)2125        return BinaryOperator::CreateAnd(I0, Builder.CreateNot(X));2126    }2127 2128    // If we can eliminate ~A and Y is free to invert:2129    // max ~A, Y --> ~(min A, ~Y)2130    //2131    // Examples:2132    // max ~A, ~Y --> ~(min A, Y)2133    // max ~A, C --> ~(min A, ~C)2134    // max ~A, (max ~Y, ~Z) --> ~min( A, (min Y, Z))2135    auto moveNotAfterMinMax = [&](Value *X, Value *Y) -> Instruction * {2136      Value *A;2137      if (match(X, m_OneUse(m_Not(m_Value(A)))) &&2138          !isFreeToInvert(A, A->hasOneUse())) {2139        if (Value *NotY = getFreelyInverted(Y, Y->hasOneUse(), &Builder)) {2140          Intrinsic::ID InvID = getInverseMinMaxIntrinsic(IID);2141          Value *InvMaxMin = Builder.CreateBinaryIntrinsic(InvID, A, NotY);2142          return BinaryOperator::CreateNot(InvMaxMin);2143        }2144      }2145      return nullptr;2146    };2147 2148    if (Instruction *I = moveNotAfterMinMax(I0, I1))2149      return I;2150    if (Instruction *I = moveNotAfterMinMax(I1, I0))2151      return I;2152 2153    if (Instruction *I = moveAddAfterMinMax(II, Builder))2154      return I;2155 2156    // minmax (X & NegPow2C, Y & NegPow2C) --> minmax(X, Y) & NegPow2C2157    const APInt *RHSC;2158    if (match(I0, m_OneUse(m_And(m_Value(X), m_NegatedPower2(RHSC)))) &&2159        match(I1, m_OneUse(m_And(m_Value(Y), m_SpecificInt(*RHSC)))))2160      return BinaryOperator::CreateAnd(Builder.CreateBinaryIntrinsic(IID, X, Y),2161                                       ConstantInt::get(II->getType(), *RHSC));2162 2163    // smax(X, -X) --> abs(X)2164    // smin(X, -X) --> -abs(X)2165    // umax(X, -X) --> -abs(X)2166    // umin(X, -X) --> abs(X)2167    if (isKnownNegation(I0, I1)) {2168      // We can choose either operand as the input to abs(), but if we can2169      // eliminate the only use of a value, that's better for subsequent2170      // transforms/analysis.2171      if (I0->hasOneUse() && !I1->hasOneUse())2172        std::swap(I0, I1);2173 2174      // This is some variant of abs(). See if we can propagate 'nsw' to the abs2175      // operation and potentially its negation.2176      bool IntMinIsPoison = isKnownNegation(I0, I1, /* NeedNSW */ true);2177      Value *Abs = Builder.CreateBinaryIntrinsic(2178          Intrinsic::abs, I0,2179          ConstantInt::getBool(II->getContext(), IntMinIsPoison));2180 2181      // We don't have a "nabs" intrinsic, so negate if needed based on the2182      // max/min operation.2183      if (IID == Intrinsic::smin || IID == Intrinsic::umax)2184        Abs = Builder.CreateNeg(Abs, "nabs", IntMinIsPoison);2185      return replaceInstUsesWith(CI, Abs);2186    }2187 2188    if (Instruction *Sel = foldClampRangeOfTwo(II, Builder))2189      return Sel;2190 2191    if (Instruction *SAdd = matchSAddSubSat(*II))2192      return SAdd;2193 2194    if (Value *NewMinMax = reassociateMinMaxWithConstants(II, Builder, SQ))2195      return replaceInstUsesWith(*II, NewMinMax);2196 2197    if (Instruction *R = reassociateMinMaxWithConstantInOperand(II, Builder))2198      return R;2199 2200    if (Instruction *NewMinMax = factorizeMinMaxTree(II))2201       return NewMinMax;2202 2203    // Try to fold minmax with constant RHS based on range information2204    if (match(I1, m_APIntAllowPoison(RHSC))) {2205      ICmpInst::Predicate Pred =2206          ICmpInst::getNonStrictPredicate(MinMaxIntrinsic::getPredicate(IID));2207      bool IsSigned = MinMaxIntrinsic::isSigned(IID);2208      ConstantRange LHS_CR = computeConstantRangeIncludingKnownBits(2209          I0, IsSigned, SQ.getWithInstruction(II));2210      if (!LHS_CR.isFullSet()) {2211        if (LHS_CR.icmp(Pred, *RHSC))2212          return replaceInstUsesWith(*II, I0);2213        if (LHS_CR.icmp(ICmpInst::getSwappedPredicate(Pred), *RHSC))2214          return replaceInstUsesWith(*II,2215                                     ConstantInt::get(II->getType(), *RHSC));2216      }2217    }2218 2219    if (Value *V = foldIntrinsicUsingDistributiveLaws(II, Builder))2220      return replaceInstUsesWith(*II, V);2221 2222    break;2223  }2224  case Intrinsic::scmp: {2225    Value *I0 = II->getArgOperand(0), *I1 = II->getArgOperand(1);2226    Value *LHS, *RHS;2227    if (match(I0, m_NSWSub(m_Value(LHS), m_Value(RHS))) && match(I1, m_Zero()))2228      return replaceInstUsesWith(2229          CI,2230          Builder.CreateIntrinsic(II->getType(), Intrinsic::scmp, {LHS, RHS}));2231    break;2232  }2233  case Intrinsic::bitreverse: {2234    Value *IIOperand = II->getArgOperand(0);2235    // bitrev (zext i1 X to ?) --> X ? SignBitC : 02236    Value *X;2237    if (match(IIOperand, m_ZExt(m_Value(X))) &&2238        X->getType()->isIntOrIntVectorTy(1)) {2239      Type *Ty = II->getType();2240      APInt SignBit = APInt::getSignMask(Ty->getScalarSizeInBits());2241      return SelectInst::Create(X, ConstantInt::get(Ty, SignBit),2242                                ConstantInt::getNullValue(Ty));2243    }2244 2245    if (Instruction *crossLogicOpFold =2246        foldBitOrderCrossLogicOp<Intrinsic::bitreverse>(IIOperand, Builder))2247      return crossLogicOpFold;2248 2249    break;2250  }2251  case Intrinsic::bswap: {2252    Value *IIOperand = II->getArgOperand(0);2253 2254    // Try to canonicalize bswap-of-logical-shift-by-8-bit-multiple as2255    // inverse-shift-of-bswap:2256    // bswap (shl X, Y) --> lshr (bswap X), Y2257    // bswap (lshr X, Y) --> shl (bswap X), Y2258    Value *X, *Y;2259    if (match(IIOperand, m_OneUse(m_LogicalShift(m_Value(X), m_Value(Y))))) {2260      unsigned BitWidth = IIOperand->getType()->getScalarSizeInBits();2261      if (MaskedValueIsZero(Y, APInt::getLowBitsSet(BitWidth, 3))) {2262        Value *NewSwap = Builder.CreateUnaryIntrinsic(Intrinsic::bswap, X);2263        BinaryOperator::BinaryOps InverseShift =2264            cast<BinaryOperator>(IIOperand)->getOpcode() == Instruction::Shl2265                ? Instruction::LShr2266                : Instruction::Shl;2267        return BinaryOperator::Create(InverseShift, NewSwap, Y);2268      }2269    }2270 2271    KnownBits Known = computeKnownBits(IIOperand, II);2272    uint64_t LZ = alignDown(Known.countMinLeadingZeros(), 8);2273    uint64_t TZ = alignDown(Known.countMinTrailingZeros(), 8);2274    unsigned BW = Known.getBitWidth();2275 2276    // bswap(x) -> shift(x) if x has exactly one "active byte"2277    if (BW - LZ - TZ == 8) {2278      assert(LZ != TZ && "active byte cannot be in the middle");2279      if (LZ > TZ)  // -> shl(x) if the "active byte" is in the low part of x2280        return BinaryOperator::CreateNUWShl(2281            IIOperand, ConstantInt::get(IIOperand->getType(), LZ - TZ));2282      // -> lshr(x) if the "active byte" is in the high part of x2283      return BinaryOperator::CreateExactLShr(2284            IIOperand, ConstantInt::get(IIOperand->getType(), TZ - LZ));2285    }2286 2287    // bswap(trunc(bswap(x))) -> trunc(lshr(x, c))2288    if (match(IIOperand, m_Trunc(m_BSwap(m_Value(X))))) {2289      unsigned C = X->getType()->getScalarSizeInBits() - BW;2290      Value *CV = ConstantInt::get(X->getType(), C);2291      Value *V = Builder.CreateLShr(X, CV);2292      return new TruncInst(V, IIOperand->getType());2293    }2294 2295    if (Instruction *crossLogicOpFold =2296            foldBitOrderCrossLogicOp<Intrinsic::bswap>(IIOperand, Builder)) {2297      return crossLogicOpFold;2298    }2299 2300    // Try to fold into bitreverse if bswap is the root of the expression tree.2301    if (Instruction *BitOp = matchBSwapOrBitReverse(*II, /*MatchBSwaps*/ false,2302                                                    /*MatchBitReversals*/ true))2303      return BitOp;2304    break;2305  }2306  case Intrinsic::masked_load:2307    if (Value *SimplifiedMaskedOp = simplifyMaskedLoad(*II))2308      return replaceInstUsesWith(CI, SimplifiedMaskedOp);2309    break;2310  case Intrinsic::masked_store:2311    return simplifyMaskedStore(*II);2312  case Intrinsic::masked_gather:2313    return simplifyMaskedGather(*II);2314  case Intrinsic::masked_scatter:2315    return simplifyMaskedScatter(*II);2316  case Intrinsic::launder_invariant_group:2317  case Intrinsic::strip_invariant_group:2318    if (auto *SkippedBarrier = simplifyInvariantGroupIntrinsic(*II, *this))2319      return replaceInstUsesWith(*II, SkippedBarrier);2320    break;2321  case Intrinsic::powi:2322    if (ConstantInt *Power = dyn_cast<ConstantInt>(II->getArgOperand(1))) {2323      // 0 and 1 are handled in instsimplify2324      // powi(x, -1) -> 1/x2325      if (Power->isMinusOne())2326        return BinaryOperator::CreateFDivFMF(ConstantFP::get(CI.getType(), 1.0),2327                                             II->getArgOperand(0), II);2328      // powi(x, 2) -> x*x2329      if (Power->equalsInt(2))2330        return BinaryOperator::CreateFMulFMF(II->getArgOperand(0),2331                                             II->getArgOperand(0), II);2332 2333      if (!Power->getValue()[0]) {2334        Value *X;2335        // If power is even:2336        // powi(-x, p) -> powi(x, p)2337        // powi(fabs(x), p) -> powi(x, p)2338        // powi(copysign(x, y), p) -> powi(x, p)2339        if (match(II->getArgOperand(0), m_FNeg(m_Value(X))) ||2340            match(II->getArgOperand(0), m_FAbs(m_Value(X))) ||2341            match(II->getArgOperand(0),2342                  m_Intrinsic<Intrinsic::copysign>(m_Value(X), m_Value())))2343          return replaceOperand(*II, 0, X);2344      }2345    }2346    break;2347 2348  case Intrinsic::cttz:2349  case Intrinsic::ctlz:2350    if (auto *I = foldCttzCtlz(*II, *this))2351      return I;2352    break;2353 2354  case Intrinsic::ctpop:2355    if (auto *I = foldCtpop(*II, *this))2356      return I;2357    break;2358 2359  case Intrinsic::fshl:2360  case Intrinsic::fshr: {2361    Value *Op0 = II->getArgOperand(0), *Op1 = II->getArgOperand(1);2362    Type *Ty = II->getType();2363    unsigned BitWidth = Ty->getScalarSizeInBits();2364    Constant *ShAmtC;2365    if (match(II->getArgOperand(2), m_ImmConstant(ShAmtC))) {2366      // Canonicalize a shift amount constant operand to modulo the bit-width.2367      Constant *WidthC = ConstantInt::get(Ty, BitWidth);2368      Constant *ModuloC =2369          ConstantFoldBinaryOpOperands(Instruction::URem, ShAmtC, WidthC, DL);2370      if (!ModuloC)2371        return nullptr;2372      if (ModuloC != ShAmtC)2373        return replaceOperand(*II, 2, ModuloC);2374 2375      assert(match(ConstantFoldCompareInstOperands(ICmpInst::ICMP_UGT, WidthC,2376                                                   ShAmtC, DL),2377                   m_One()) &&2378             "Shift amount expected to be modulo bitwidth");2379 2380      // Canonicalize funnel shift right by constant to funnel shift left. This2381      // is not entirely arbitrary. For historical reasons, the backend may2382      // recognize rotate left patterns but miss rotate right patterns.2383      if (IID == Intrinsic::fshr) {2384        // fshr X, Y, C --> fshl X, Y, (BitWidth - C) if C is not zero.2385        if (!isKnownNonZero(ShAmtC, SQ.getWithInstruction(II)))2386          return nullptr;2387 2388        Constant *LeftShiftC = ConstantExpr::getSub(WidthC, ShAmtC);2389        Module *Mod = II->getModule();2390        Function *Fshl =2391            Intrinsic::getOrInsertDeclaration(Mod, Intrinsic::fshl, Ty);2392        return CallInst::Create(Fshl, { Op0, Op1, LeftShiftC });2393      }2394      assert(IID == Intrinsic::fshl &&2395             "All funnel shifts by simple constants should go left");2396 2397      // fshl(X, 0, C) --> shl X, C2398      // fshl(X, undef, C) --> shl X, C2399      if (match(Op1, m_ZeroInt()) || match(Op1, m_Undef()))2400        return BinaryOperator::CreateShl(Op0, ShAmtC);2401 2402      // fshl(0, X, C) --> lshr X, (BW-C)2403      // fshl(undef, X, C) --> lshr X, (BW-C)2404      if (match(Op0, m_ZeroInt()) || match(Op0, m_Undef()))2405        return BinaryOperator::CreateLShr(Op1,2406                                          ConstantExpr::getSub(WidthC, ShAmtC));2407 2408      // fshl i16 X, X, 8 --> bswap i16 X (reduce to more-specific form)2409      if (Op0 == Op1 && BitWidth == 16 && match(ShAmtC, m_SpecificInt(8))) {2410        Module *Mod = II->getModule();2411        Function *Bswap =2412            Intrinsic::getOrInsertDeclaration(Mod, Intrinsic::bswap, Ty);2413        return CallInst::Create(Bswap, { Op0 });2414      }2415      if (Instruction *BitOp =2416              matchBSwapOrBitReverse(*II, /*MatchBSwaps*/ true,2417                                     /*MatchBitReversals*/ true))2418        return BitOp;2419 2420      // R = fshl(X, X, C2)2421      // fshl(R, R, C1) --> fshl(X, X, (C1 + C2) % bitsize)2422      Value *InnerOp;2423      const APInt *ShAmtInnerC, *ShAmtOuterC;2424      if (match(Op0, m_FShl(m_Value(InnerOp), m_Deferred(InnerOp),2425                            m_APInt(ShAmtInnerC))) &&2426          match(ShAmtC, m_APInt(ShAmtOuterC)) && Op0 == Op1) {2427        APInt Sum = *ShAmtOuterC + *ShAmtInnerC;2428        APInt Modulo = Sum.urem(APInt(Sum.getBitWidth(), BitWidth));2429        if (Modulo.isZero())2430          return replaceInstUsesWith(*II, InnerOp);2431        Constant *ModuloC = ConstantInt::get(Ty, Modulo);2432        return CallInst::Create(cast<IntrinsicInst>(Op0)->getCalledFunction(),2433                                {InnerOp, InnerOp, ModuloC});2434      }2435    }2436 2437    // fshl(X, X, Neg(Y)) --> fshr(X, X, Y)2438    // fshr(X, X, Neg(Y)) --> fshl(X, X, Y)2439    // if BitWidth is a power-of-22440    Value *Y;2441    if (Op0 == Op1 && isPowerOf2_32(BitWidth) &&2442        match(II->getArgOperand(2), m_Neg(m_Value(Y)))) {2443      Module *Mod = II->getModule();2444      Function *OppositeShift = Intrinsic::getOrInsertDeclaration(2445          Mod, IID == Intrinsic::fshl ? Intrinsic::fshr : Intrinsic::fshl, Ty);2446      return CallInst::Create(OppositeShift, {Op0, Op1, Y});2447    }2448 2449    // fshl(X, 0, Y) --> shl(X, and(Y, BitWidth - 1)) if bitwidth is a2450    // power-of-22451    if (IID == Intrinsic::fshl && isPowerOf2_32(BitWidth) &&2452        match(Op1, m_ZeroInt())) {2453      Value *Op2 = II->getArgOperand(2);2454      Value *And = Builder.CreateAnd(Op2, ConstantInt::get(Ty, BitWidth - 1));2455      return BinaryOperator::CreateShl(Op0, And);2456    }2457 2458    // Left or right might be masked.2459    if (SimplifyDemandedInstructionBits(*II))2460      return &CI;2461 2462    // The shift amount (operand 2) of a funnel shift is modulo the bitwidth,2463    // so only the low bits of the shift amount are demanded if the bitwidth is2464    // a power-of-2.2465    if (!isPowerOf2_32(BitWidth))2466      break;2467    APInt Op2Demanded = APInt::getLowBitsSet(BitWidth, Log2_32_Ceil(BitWidth));2468    KnownBits Op2Known(BitWidth);2469    if (SimplifyDemandedBits(II, 2, Op2Demanded, Op2Known))2470      return &CI;2471    break;2472  }2473  case Intrinsic::ptrmask: {2474    unsigned BitWidth = DL.getPointerTypeSizeInBits(II->getType());2475    KnownBits Known(BitWidth);2476    if (SimplifyDemandedInstructionBits(*II, Known))2477      return II;2478 2479    Value *InnerPtr, *InnerMask;2480    bool Changed = false;2481    // Combine:2482    // (ptrmask (ptrmask p, A), B)2483    //    -> (ptrmask p, (and A, B))2484    if (match(II->getArgOperand(0),2485              m_OneUse(m_Intrinsic<Intrinsic::ptrmask>(m_Value(InnerPtr),2486                                                       m_Value(InnerMask))))) {2487      assert(II->getArgOperand(1)->getType() == InnerMask->getType() &&2488             "Mask types must match");2489      // TODO: If InnerMask == Op1, we could copy attributes from inner2490      // callsite -> outer callsite.2491      Value *NewMask = Builder.CreateAnd(II->getArgOperand(1), InnerMask);2492      replaceOperand(CI, 0, InnerPtr);2493      replaceOperand(CI, 1, NewMask);2494      Changed = true;2495    }2496 2497    // See if we can deduce non-null.2498    if (!CI.hasRetAttr(Attribute::NonNull) &&2499        (Known.isNonZero() ||2500         isKnownNonZero(II, getSimplifyQuery().getWithInstruction(II)))) {2501      CI.addRetAttr(Attribute::NonNull);2502      Changed = true;2503    }2504 2505    unsigned NewAlignmentLog =2506        std::min(Value::MaxAlignmentExponent,2507                 std::min(BitWidth - 1, Known.countMinTrailingZeros()));2508    // Known bits will capture if we had alignment information associated with2509    // the pointer argument.2510    if (NewAlignmentLog > Log2(CI.getRetAlign().valueOrOne())) {2511      CI.addRetAttr(Attribute::getWithAlignment(2512          CI.getContext(), Align(uint64_t(1) << NewAlignmentLog)));2513      Changed = true;2514    }2515    if (Changed)2516      return &CI;2517    break;2518  }2519  case Intrinsic::uadd_with_overflow:2520  case Intrinsic::sadd_with_overflow: {2521    if (Instruction *I = foldIntrinsicWithOverflowCommon(II))2522      return I;2523 2524    // Given 2 constant operands whose sum does not overflow:2525    // uaddo (X +nuw C0), C1 -> uaddo X, C0 + C12526    // saddo (X +nsw C0), C1 -> saddo X, C0 + C12527    Value *X;2528    const APInt *C0, *C1;2529    Value *Arg0 = II->getArgOperand(0);2530    Value *Arg1 = II->getArgOperand(1);2531    bool IsSigned = IID == Intrinsic::sadd_with_overflow;2532    bool HasNWAdd = IsSigned2533                        ? match(Arg0, m_NSWAddLike(m_Value(X), m_APInt(C0)))2534                        : match(Arg0, m_NUWAddLike(m_Value(X), m_APInt(C0)));2535    if (HasNWAdd && match(Arg1, m_APInt(C1))) {2536      bool Overflow;2537      APInt NewC =2538          IsSigned ? C1->sadd_ov(*C0, Overflow) : C1->uadd_ov(*C0, Overflow);2539      if (!Overflow)2540        return replaceInstUsesWith(2541            *II, Builder.CreateBinaryIntrinsic(2542                     IID, X, ConstantInt::get(Arg1->getType(), NewC)));2543    }2544    break;2545  }2546 2547  case Intrinsic::umul_with_overflow:2548  case Intrinsic::smul_with_overflow:2549  case Intrinsic::usub_with_overflow:2550    if (Instruction *I = foldIntrinsicWithOverflowCommon(II))2551      return I;2552    break;2553 2554  case Intrinsic::ssub_with_overflow: {2555    if (Instruction *I = foldIntrinsicWithOverflowCommon(II))2556      return I;2557 2558    Constant *C;2559    Value *Arg0 = II->getArgOperand(0);2560    Value *Arg1 = II->getArgOperand(1);2561    // Given a constant C that is not the minimum signed value2562    // for an integer of a given bit width:2563    //2564    // ssubo X, C -> saddo X, -C2565    if (match(Arg1, m_Constant(C)) && C->isNotMinSignedValue()) {2566      Value *NegVal = ConstantExpr::getNeg(C);2567      // Build a saddo call that is equivalent to the discovered2568      // ssubo call.2569      return replaceInstUsesWith(2570          *II, Builder.CreateBinaryIntrinsic(Intrinsic::sadd_with_overflow,2571                                             Arg0, NegVal));2572    }2573 2574    break;2575  }2576 2577  case Intrinsic::uadd_sat:2578  case Intrinsic::sadd_sat:2579  case Intrinsic::usub_sat:2580  case Intrinsic::ssub_sat: {2581    SaturatingInst *SI = cast<SaturatingInst>(II);2582    Type *Ty = SI->getType();2583    Value *Arg0 = SI->getLHS();2584    Value *Arg1 = SI->getRHS();2585 2586    // Make use of known overflow information.2587    OverflowResult OR = computeOverflow(SI->getBinaryOp(), SI->isSigned(),2588                                        Arg0, Arg1, SI);2589    switch (OR) {2590      case OverflowResult::MayOverflow:2591        break;2592      case OverflowResult::NeverOverflows:2593        if (SI->isSigned())2594          return BinaryOperator::CreateNSW(SI->getBinaryOp(), Arg0, Arg1);2595        else2596          return BinaryOperator::CreateNUW(SI->getBinaryOp(), Arg0, Arg1);2597      case OverflowResult::AlwaysOverflowsLow: {2598        unsigned BitWidth = Ty->getScalarSizeInBits();2599        APInt Min = APSInt::getMinValue(BitWidth, !SI->isSigned());2600        return replaceInstUsesWith(*SI, ConstantInt::get(Ty, Min));2601      }2602      case OverflowResult::AlwaysOverflowsHigh: {2603        unsigned BitWidth = Ty->getScalarSizeInBits();2604        APInt Max = APSInt::getMaxValue(BitWidth, !SI->isSigned());2605        return replaceInstUsesWith(*SI, ConstantInt::get(Ty, Max));2606      }2607    }2608 2609    // usub_sat((sub nuw C, A), C1) -> usub_sat(usub_sat(C, C1), A)2610    // which after that:2611    // usub_sat((sub nuw C, A), C1) -> usub_sat(C - C1, A) if C1 u< C2612    // usub_sat((sub nuw C, A), C1) -> 0 otherwise2613    Constant *C, *C1;2614    Value *A;2615    if (IID == Intrinsic::usub_sat &&2616        match(Arg0, m_NUWSub(m_ImmConstant(C), m_Value(A))) &&2617        match(Arg1, m_ImmConstant(C1))) {2618      auto *NewC = Builder.CreateBinaryIntrinsic(Intrinsic::usub_sat, C, C1);2619      auto *NewSub =2620          Builder.CreateBinaryIntrinsic(Intrinsic::usub_sat, NewC, A);2621      return replaceInstUsesWith(*SI, NewSub);2622    }2623 2624    // ssub.sat(X, C) -> sadd.sat(X, -C) if C != MIN2625    if (IID == Intrinsic::ssub_sat && match(Arg1, m_Constant(C)) &&2626        C->isNotMinSignedValue()) {2627      Value *NegVal = ConstantExpr::getNeg(C);2628      return replaceInstUsesWith(2629          *II, Builder.CreateBinaryIntrinsic(2630              Intrinsic::sadd_sat, Arg0, NegVal));2631    }2632 2633    // sat(sat(X + Val2) + Val) -> sat(X + (Val+Val2))2634    // sat(sat(X - Val2) - Val) -> sat(X - (Val+Val2))2635    // if Val and Val2 have the same sign2636    if (auto *Other = dyn_cast<IntrinsicInst>(Arg0)) {2637      Value *X;2638      const APInt *Val, *Val2;2639      APInt NewVal;2640      bool IsUnsigned =2641          IID == Intrinsic::uadd_sat || IID == Intrinsic::usub_sat;2642      if (Other->getIntrinsicID() == IID &&2643          match(Arg1, m_APInt(Val)) &&2644          match(Other->getArgOperand(0), m_Value(X)) &&2645          match(Other->getArgOperand(1), m_APInt(Val2))) {2646        if (IsUnsigned)2647          NewVal = Val->uadd_sat(*Val2);2648        else if (Val->isNonNegative() == Val2->isNonNegative()) {2649          bool Overflow;2650          NewVal = Val->sadd_ov(*Val2, Overflow);2651          if (Overflow) {2652            // Both adds together may add more than SignedMaxValue2653            // without saturating the final result.2654            break;2655          }2656        } else {2657          // Cannot fold saturated addition with different signs.2658          break;2659        }2660 2661        return replaceInstUsesWith(2662            *II, Builder.CreateBinaryIntrinsic(2663                     IID, X, ConstantInt::get(II->getType(), NewVal)));2664      }2665    }2666    break;2667  }2668 2669  case Intrinsic::minnum:2670  case Intrinsic::maxnum:2671  case Intrinsic::minimum:2672  case Intrinsic::maximum: {2673    Value *Arg0 = II->getArgOperand(0);2674    Value *Arg1 = II->getArgOperand(1);2675    Value *X, *Y;2676    if (match(Arg0, m_FNeg(m_Value(X))) && match(Arg1, m_FNeg(m_Value(Y))) &&2677        (Arg0->hasOneUse() || Arg1->hasOneUse())) {2678      // If both operands are negated, invert the call and negate the result:2679      // min(-X, -Y) --> -(max(X, Y))2680      // max(-X, -Y) --> -(min(X, Y))2681      Intrinsic::ID NewIID;2682      switch (IID) {2683      case Intrinsic::maxnum:2684        NewIID = Intrinsic::minnum;2685        break;2686      case Intrinsic::minnum:2687        NewIID = Intrinsic::maxnum;2688        break;2689      case Intrinsic::maximum:2690        NewIID = Intrinsic::minimum;2691        break;2692      case Intrinsic::minimum:2693        NewIID = Intrinsic::maximum;2694        break;2695      default:2696        llvm_unreachable("unexpected intrinsic ID");2697      }2698      Value *NewCall = Builder.CreateBinaryIntrinsic(NewIID, X, Y, II);2699      Instruction *FNeg = UnaryOperator::CreateFNeg(NewCall);2700      FNeg->copyIRFlags(II);2701      return FNeg;2702    }2703 2704    // m(m(X, C2), C1) -> m(X, C)2705    const APFloat *C1, *C2;2706    if (auto *M = dyn_cast<IntrinsicInst>(Arg0)) {2707      if (M->getIntrinsicID() == IID && match(Arg1, m_APFloat(C1)) &&2708          ((match(M->getArgOperand(0), m_Value(X)) &&2709            match(M->getArgOperand(1), m_APFloat(C2))) ||2710           (match(M->getArgOperand(1), m_Value(X)) &&2711            match(M->getArgOperand(0), m_APFloat(C2))))) {2712        APFloat Res(0.0);2713        switch (IID) {2714        case Intrinsic::maxnum:2715          Res = maxnum(*C1, *C2);2716          break;2717        case Intrinsic::minnum:2718          Res = minnum(*C1, *C2);2719          break;2720        case Intrinsic::maximum:2721          Res = maximum(*C1, *C2);2722          break;2723        case Intrinsic::minimum:2724          Res = minimum(*C1, *C2);2725          break;2726        default:2727          llvm_unreachable("unexpected intrinsic ID");2728        }2729        // TODO: Conservatively intersecting FMF. If Res == C2, the transform2730        //       was a simplification (so Arg0 and its original flags could2731        //       propagate?)2732        Value *V = Builder.CreateBinaryIntrinsic(2733            IID, X, ConstantFP::get(Arg0->getType(), Res),2734            FMFSource::intersect(II, M));2735        return replaceInstUsesWith(*II, V);2736      }2737    }2738 2739    // m((fpext X), (fpext Y)) -> fpext (m(X, Y))2740    if (match(Arg0, m_OneUse(m_FPExt(m_Value(X)))) &&2741        match(Arg1, m_OneUse(m_FPExt(m_Value(Y)))) &&2742        X->getType() == Y->getType()) {2743      Value *NewCall =2744          Builder.CreateBinaryIntrinsic(IID, X, Y, II, II->getName());2745      return new FPExtInst(NewCall, II->getType());2746    }2747 2748    // max X, -X --> fabs X2749    // min X, -X --> -(fabs X)2750    // TODO: Remove one-use limitation? That is obviously better for max,2751    // hence why we don't check for one-use for that. However,2752    // it would be an extra instruction for min (fnabs), but2753    // that is still likely better for analysis and codegen.2754    auto IsMinMaxOrXNegX = [IID, &X](Value *Op0, Value *Op1) {2755      if (match(Op0, m_FNeg(m_Value(X))) && match(Op1, m_Specific(X)))2756        return Op0->hasOneUse() ||2757               (IID != Intrinsic::minimum && IID != Intrinsic::minnum);2758      return false;2759    };2760 2761    if (IsMinMaxOrXNegX(Arg0, Arg1) || IsMinMaxOrXNegX(Arg1, Arg0)) {2762      Value *R = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, X, II);2763      if (IID == Intrinsic::minimum || IID == Intrinsic::minnum)2764        R = Builder.CreateFNegFMF(R, II);2765      return replaceInstUsesWith(*II, R);2766    }2767 2768    break;2769  }2770  case Intrinsic::matrix_multiply: {2771    // Optimize negation in matrix multiplication.2772 2773    // -A * -B -> A * B2774    Value *A, *B;2775    if (match(II->getArgOperand(0), m_FNeg(m_Value(A))) &&2776        match(II->getArgOperand(1), m_FNeg(m_Value(B)))) {2777      replaceOperand(*II, 0, A);2778      replaceOperand(*II, 1, B);2779      return II;2780    }2781 2782    Value *Op0 = II->getOperand(0);2783    Value *Op1 = II->getOperand(1);2784    Value *OpNotNeg, *NegatedOp;2785    unsigned NegatedOpArg, OtherOpArg;2786    if (match(Op0, m_FNeg(m_Value(OpNotNeg)))) {2787      NegatedOp = Op0;2788      NegatedOpArg = 0;2789      OtherOpArg = 1;2790    } else if (match(Op1, m_FNeg(m_Value(OpNotNeg)))) {2791      NegatedOp = Op1;2792      NegatedOpArg = 1;2793      OtherOpArg = 0;2794    } else2795      // Multiplication doesn't have a negated operand.2796      break;2797 2798    // Only optimize if the negated operand has only one use.2799    if (!NegatedOp->hasOneUse())2800      break;2801 2802    Value *OtherOp = II->getOperand(OtherOpArg);2803    VectorType *RetTy = cast<VectorType>(II->getType());2804    VectorType *NegatedOpTy = cast<VectorType>(NegatedOp->getType());2805    VectorType *OtherOpTy = cast<VectorType>(OtherOp->getType());2806    ElementCount NegatedCount = NegatedOpTy->getElementCount();2807    ElementCount OtherCount = OtherOpTy->getElementCount();2808    ElementCount RetCount = RetTy->getElementCount();2809    // (-A) * B -> A * (-B), if it is cheaper to negate B and vice versa.2810    if (ElementCount::isKnownGT(NegatedCount, OtherCount) &&2811        ElementCount::isKnownLT(OtherCount, RetCount)) {2812      Value *InverseOtherOp = Builder.CreateFNeg(OtherOp);2813      replaceOperand(*II, NegatedOpArg, OpNotNeg);2814      replaceOperand(*II, OtherOpArg, InverseOtherOp);2815      return II;2816    }2817    // (-A) * B -> -(A * B), if it is cheaper to negate the result2818    if (ElementCount::isKnownGT(NegatedCount, RetCount)) {2819      SmallVector<Value *, 5> NewArgs(II->args());2820      NewArgs[NegatedOpArg] = OpNotNeg;2821      Instruction *NewMul =2822          Builder.CreateIntrinsic(II->getType(), IID, NewArgs, II);2823      return replaceInstUsesWith(*II, Builder.CreateFNegFMF(NewMul, II));2824    }2825    break;2826  }2827  case Intrinsic::fmuladd: {2828    // Try to simplify the underlying FMul.2829    if (Value *V =2830            simplifyFMulInst(II->getArgOperand(0), II->getArgOperand(1),2831                             II->getFastMathFlags(), SQ.getWithInstruction(II)))2832      return BinaryOperator::CreateFAddFMF(V, II->getArgOperand(2),2833                                           II->getFastMathFlags());2834 2835    [[fallthrough]];2836  }2837  case Intrinsic::fma: {2838    // fma fneg(x), fneg(y), z -> fma x, y, z2839    Value *Src0 = II->getArgOperand(0);2840    Value *Src1 = II->getArgOperand(1);2841    Value *Src2 = II->getArgOperand(2);2842    Value *X, *Y;2843    if (match(Src0, m_FNeg(m_Value(X))) && match(Src1, m_FNeg(m_Value(Y)))) {2844      replaceOperand(*II, 0, X);2845      replaceOperand(*II, 1, Y);2846      return II;2847    }2848 2849    // fma fabs(x), fabs(x), z -> fma x, x, z2850    if (match(Src0, m_FAbs(m_Value(X))) &&2851        match(Src1, m_FAbs(m_Specific(X)))) {2852      replaceOperand(*II, 0, X);2853      replaceOperand(*II, 1, X);2854      return II;2855    }2856 2857    // Try to simplify the underlying FMul. We can only apply simplifications2858    // that do not require rounding.2859    if (Value *V = simplifyFMAFMul(Src0, Src1, II->getFastMathFlags(),2860                                   SQ.getWithInstruction(II)))2861      return BinaryOperator::CreateFAddFMF(V, Src2, II->getFastMathFlags());2862 2863    // fma x, y, 0 -> fmul x, y2864    // This is always valid for -0.0, but requires nsz for +0.0 as2865    // -0.0 + 0.0 = 0.0, which would not be the same as the fmul on its own.2866    if (match(Src2, m_NegZeroFP()) ||2867        (match(Src2, m_PosZeroFP()) && II->getFastMathFlags().noSignedZeros()))2868      return BinaryOperator::CreateFMulFMF(Src0, Src1, II);2869 2870    // fma x, -1.0, y -> fsub y, x2871    if (match(Src1, m_SpecificFP(-1.0)))2872      return BinaryOperator::CreateFSubFMF(Src2, Src0, II);2873 2874    break;2875  }2876  case Intrinsic::copysign: {2877    Value *Mag = II->getArgOperand(0), *Sign = II->getArgOperand(1);2878    if (std::optional<bool> KnownSignBit = computeKnownFPSignBit(2879            Sign, getSimplifyQuery().getWithInstruction(II))) {2880      if (*KnownSignBit) {2881        // If we know that the sign argument is negative, reduce to FNABS:2882        // copysign Mag, -Sign --> fneg (fabs Mag)2883        Value *Fabs = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, Mag, II);2884        return replaceInstUsesWith(*II, Builder.CreateFNegFMF(Fabs, II));2885      }2886 2887      // If we know that the sign argument is positive, reduce to FABS:2888      // copysign Mag, +Sign --> fabs Mag2889      Value *Fabs = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, Mag, II);2890      return replaceInstUsesWith(*II, Fabs);2891    }2892 2893    // Propagate sign argument through nested calls:2894    // copysign Mag, (copysign ?, X) --> copysign Mag, X2895    Value *X;2896    if (match(Sign, m_Intrinsic<Intrinsic::copysign>(m_Value(), m_Value(X)))) {2897      Value *CopySign =2898          Builder.CreateCopySign(Mag, X, FMFSource::intersect(II, Sign));2899      return replaceInstUsesWith(*II, CopySign);2900    }2901 2902    // Clear sign-bit of constant magnitude:2903    // copysign -MagC, X --> copysign MagC, X2904    // TODO: Support constant folding for fabs2905    const APFloat *MagC;2906    if (match(Mag, m_APFloat(MagC)) && MagC->isNegative()) {2907      APFloat PosMagC = *MagC;2908      PosMagC.clearSign();2909      return replaceOperand(*II, 0, ConstantFP::get(Mag->getType(), PosMagC));2910    }2911 2912    // Peek through changes of magnitude's sign-bit. This call rewrites those:2913    // copysign (fabs X), Sign --> copysign X, Sign2914    // copysign (fneg X), Sign --> copysign X, Sign2915    if (match(Mag, m_FAbs(m_Value(X))) || match(Mag, m_FNeg(m_Value(X))))2916      return replaceOperand(*II, 0, X);2917 2918    break;2919  }2920  case Intrinsic::fabs: {2921    Value *Cond, *TVal, *FVal;2922    Value *Arg = II->getArgOperand(0);2923    Value *X;2924    // fabs (-X) --> fabs (X)2925    if (match(Arg, m_FNeg(m_Value(X)))) {2926        CallInst *Fabs = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, X, II);2927        return replaceInstUsesWith(CI, Fabs);2928    }2929 2930    if (match(Arg, m_Select(m_Value(Cond), m_Value(TVal), m_Value(FVal)))) {2931      // fabs (select Cond, TrueC, FalseC) --> select Cond, AbsT, AbsF2932      if (Arg->hasOneUse() ? (isa<Constant>(TVal) || isa<Constant>(FVal))2933                           : (isa<Constant>(TVal) && isa<Constant>(FVal))) {2934        CallInst *AbsT = Builder.CreateCall(II->getCalledFunction(), {TVal});2935        CallInst *AbsF = Builder.CreateCall(II->getCalledFunction(), {FVal});2936        SelectInst *SI = SelectInst::Create(Cond, AbsT, AbsF);2937        FastMathFlags FMF1 = II->getFastMathFlags();2938        FastMathFlags FMF2 = cast<SelectInst>(Arg)->getFastMathFlags();2939        FMF2.setNoSignedZeros(false);2940        SI->setFastMathFlags(FMF1 | FMF2);2941        return SI;2942      }2943      // fabs (select Cond, -FVal, FVal) --> fabs FVal2944      if (match(TVal, m_FNeg(m_Specific(FVal))))2945        return replaceOperand(*II, 0, FVal);2946      // fabs (select Cond, TVal, -TVal) --> fabs TVal2947      if (match(FVal, m_FNeg(m_Specific(TVal))))2948        return replaceOperand(*II, 0, TVal);2949    }2950 2951    Value *Magnitude, *Sign;2952    if (match(II->getArgOperand(0),2953              m_CopySign(m_Value(Magnitude), m_Value(Sign)))) {2954      // fabs (copysign x, y) -> (fabs x)2955      CallInst *AbsSign =2956          Builder.CreateUnaryIntrinsic(Intrinsic::fabs, Magnitude, II);2957      return replaceInstUsesWith(*II, AbsSign);2958    }2959 2960    [[fallthrough]];2961  }2962  case Intrinsic::ceil:2963  case Intrinsic::floor:2964  case Intrinsic::round:2965  case Intrinsic::roundeven:2966  case Intrinsic::nearbyint:2967  case Intrinsic::rint:2968  case Intrinsic::trunc: {2969    Value *ExtSrc;2970    if (match(II->getArgOperand(0), m_OneUse(m_FPExt(m_Value(ExtSrc))))) {2971      // Narrow the call: intrinsic (fpext x) -> fpext (intrinsic x)2972      Value *NarrowII = Builder.CreateUnaryIntrinsic(IID, ExtSrc, II);2973      return new FPExtInst(NarrowII, II->getType());2974    }2975    break;2976  }2977  case Intrinsic::cos:2978  case Intrinsic::amdgcn_cos: {2979    Value *X, *Sign;2980    Value *Src = II->getArgOperand(0);2981    if (match(Src, m_FNeg(m_Value(X))) || match(Src, m_FAbs(m_Value(X))) ||2982        match(Src, m_CopySign(m_Value(X), m_Value(Sign)))) {2983      // cos(-x) --> cos(x)2984      // cos(fabs(x)) --> cos(x)2985      // cos(copysign(x, y)) --> cos(x)2986      return replaceOperand(*II, 0, X);2987    }2988    break;2989  }2990  case Intrinsic::sin:2991  case Intrinsic::amdgcn_sin: {2992    Value *X;2993    if (match(II->getArgOperand(0), m_OneUse(m_FNeg(m_Value(X))))) {2994      // sin(-x) --> -sin(x)2995      Value *NewSin = Builder.CreateUnaryIntrinsic(IID, X, II);2996      return UnaryOperator::CreateFNegFMF(NewSin, II);2997    }2998    break;2999  }3000  case Intrinsic::ldexp: {3001    // ldexp(ldexp(x, a), b) -> ldexp(x, a + b)3002    //3003    // The danger is if the first ldexp would overflow to infinity or underflow3004    // to zero, but the combined exponent avoids it. We ignore this with3005    // reassoc.3006    //3007    // It's also safe to fold if we know both exponents are >= 0 or <= 0 since3008    // it would just double down on the overflow/underflow which would occur3009    // anyway.3010    //3011    // TODO: Could do better if we had range tracking for the input value3012    // exponent. Also could broaden sign check to cover == 0 case.3013    Value *Src = II->getArgOperand(0);3014    Value *Exp = II->getArgOperand(1);3015    Value *InnerSrc;3016    Value *InnerExp;3017    if (match(Src, m_OneUse(m_Intrinsic<Intrinsic::ldexp>(3018                       m_Value(InnerSrc), m_Value(InnerExp)))) &&3019        Exp->getType() == InnerExp->getType()) {3020      FastMathFlags FMF = II->getFastMathFlags();3021      FastMathFlags InnerFlags = cast<FPMathOperator>(Src)->getFastMathFlags();3022 3023      if ((FMF.allowReassoc() && InnerFlags.allowReassoc()) ||3024          signBitMustBeTheSame(Exp, InnerExp, SQ.getWithInstruction(II))) {3025        // TODO: Add nsw/nuw probably safe if integer type exceeds exponent3026        // width.3027        Value *NewExp = Builder.CreateAdd(InnerExp, Exp);3028        II->setArgOperand(1, NewExp);3029        II->setFastMathFlags(InnerFlags); // Or the inner flags.3030        return replaceOperand(*II, 0, InnerSrc);3031      }3032    }3033 3034    // ldexp(x, zext(i1 y)) -> fmul x, (select y, 2.0, 1.0)3035    // ldexp(x, sext(i1 y)) -> fmul x, (select y, 0.5, 1.0)3036    Value *ExtSrc;3037    if (match(Exp, m_ZExt(m_Value(ExtSrc))) &&3038        ExtSrc->getType()->getScalarSizeInBits() == 1) {3039      Value *Select =3040          Builder.CreateSelect(ExtSrc, ConstantFP::get(II->getType(), 2.0),3041                               ConstantFP::get(II->getType(), 1.0));3042      return BinaryOperator::CreateFMulFMF(Src, Select, II);3043    }3044    if (match(Exp, m_SExt(m_Value(ExtSrc))) &&3045        ExtSrc->getType()->getScalarSizeInBits() == 1) {3046      Value *Select =3047          Builder.CreateSelect(ExtSrc, ConstantFP::get(II->getType(), 0.5),3048                               ConstantFP::get(II->getType(), 1.0));3049      return BinaryOperator::CreateFMulFMF(Src, Select, II);3050    }3051 3052    // ldexp(x, c ? exp : 0) -> c ? ldexp(x, exp) : x3053    // ldexp(x, c ? 0 : exp) -> c ? x : ldexp(x, exp)3054    ///3055    // TODO: If we cared, should insert a canonicalize for x3056    Value *SelectCond, *SelectLHS, *SelectRHS;3057    if (match(II->getArgOperand(1),3058              m_OneUse(m_Select(m_Value(SelectCond), m_Value(SelectLHS),3059                                m_Value(SelectRHS))))) {3060      Value *NewLdexp = nullptr;3061      Value *Select = nullptr;3062      if (match(SelectRHS, m_ZeroInt())) {3063        NewLdexp = Builder.CreateLdexp(Src, SelectLHS, II);3064        Select = Builder.CreateSelect(SelectCond, NewLdexp, Src);3065      } else if (match(SelectLHS, m_ZeroInt())) {3066        NewLdexp = Builder.CreateLdexp(Src, SelectRHS, II);3067        Select = Builder.CreateSelect(SelectCond, Src, NewLdexp);3068      }3069 3070      if (NewLdexp) {3071        Select->takeName(II);3072        return replaceInstUsesWith(*II, Select);3073      }3074    }3075 3076    break;3077  }3078  case Intrinsic::ptrauth_auth:3079  case Intrinsic::ptrauth_resign: {3080    // We don't support this optimization on intrinsic calls with deactivation3081    // symbols, which are represented using operand bundles.3082    if (II->hasOperandBundles())3083      break;3084 3085    // (sign|resign) + (auth|resign) can be folded by omitting the middle3086    // sign+auth component if the key and discriminator match.3087    bool NeedSign = II->getIntrinsicID() == Intrinsic::ptrauth_resign;3088    Value *Ptr = II->getArgOperand(0);3089    Value *Key = II->getArgOperand(1);3090    Value *Disc = II->getArgOperand(2);3091 3092    // AuthKey will be the key we need to end up authenticating against in3093    // whatever we replace this sequence with.3094    Value *AuthKey = nullptr, *AuthDisc = nullptr, *BasePtr;3095    if (const auto *CI = dyn_cast<CallBase>(Ptr)) {3096      // We don't support this optimization on intrinsic calls with deactivation3097      // symbols, which are represented using operand bundles.3098      if (CI->hasOperandBundles())3099        break;3100 3101      BasePtr = CI->getArgOperand(0);3102      if (CI->getIntrinsicID() == Intrinsic::ptrauth_sign) {3103        if (CI->getArgOperand(1) != Key || CI->getArgOperand(2) != Disc)3104          break;3105      } else if (CI->getIntrinsicID() == Intrinsic::ptrauth_resign) {3106        if (CI->getArgOperand(3) != Key || CI->getArgOperand(4) != Disc)3107          break;3108        AuthKey = CI->getArgOperand(1);3109        AuthDisc = CI->getArgOperand(2);3110      } else3111        break;3112    } else if (const auto *PtrToInt = dyn_cast<PtrToIntOperator>(Ptr)) {3113      // ptrauth constants are equivalent to a call to @llvm.ptrauth.sign for3114      // our purposes, so check for that too.3115      const auto *CPA = dyn_cast<ConstantPtrAuth>(PtrToInt->getOperand(0));3116      if (!CPA || !CPA->isKnownCompatibleWith(Key, Disc, DL))3117        break;3118 3119      // resign(ptrauth(p,ks,ds),ks,ds,kr,dr) -> ptrauth(p,kr,dr)3120      if (NeedSign && isa<ConstantInt>(II->getArgOperand(4))) {3121        auto *SignKey = cast<ConstantInt>(II->getArgOperand(3));3122        auto *SignDisc = cast<ConstantInt>(II->getArgOperand(4));3123        auto *Null = ConstantPointerNull::get(Builder.getPtrTy());3124        auto *NewCPA = ConstantPtrAuth::get(CPA->getPointer(), SignKey,3125                                            SignDisc, /*AddrDisc=*/Null,3126                                            /*DeactivationSymbol=*/Null);3127        replaceInstUsesWith(3128            *II, ConstantExpr::getPointerCast(NewCPA, II->getType()));3129        return eraseInstFromFunction(*II);3130      }3131 3132      // auth(ptrauth(p,k,d),k,d) -> p3133      BasePtr = Builder.CreatePtrToInt(CPA->getPointer(), II->getType());3134    } else3135      break;3136 3137    unsigned NewIntrin;3138    if (AuthKey && NeedSign) {3139      // resign(0,1) + resign(1,2) = resign(0, 2)3140      NewIntrin = Intrinsic::ptrauth_resign;3141    } else if (AuthKey) {3142      // resign(0,1) + auth(1) = auth(0)3143      NewIntrin = Intrinsic::ptrauth_auth;3144    } else if (NeedSign) {3145      // sign(0) + resign(0, 1) = sign(1)3146      NewIntrin = Intrinsic::ptrauth_sign;3147    } else {3148      // sign(0) + auth(0) = nop3149      replaceInstUsesWith(*II, BasePtr);3150      return eraseInstFromFunction(*II);3151    }3152 3153    SmallVector<Value *, 4> CallArgs;3154    CallArgs.push_back(BasePtr);3155    if (AuthKey) {3156      CallArgs.push_back(AuthKey);3157      CallArgs.push_back(AuthDisc);3158    }3159 3160    if (NeedSign) {3161      CallArgs.push_back(II->getArgOperand(3));3162      CallArgs.push_back(II->getArgOperand(4));3163    }3164 3165    Function *NewFn =3166        Intrinsic::getOrInsertDeclaration(II->getModule(), NewIntrin);3167    return CallInst::Create(NewFn, CallArgs);3168  }3169  case Intrinsic::arm_neon_vtbl1:3170  case Intrinsic::aarch64_neon_tbl1:3171    if (Value *V = simplifyNeonTbl1(*II, Builder))3172      return replaceInstUsesWith(*II, V);3173    break;3174 3175  case Intrinsic::arm_neon_vmulls:3176  case Intrinsic::arm_neon_vmullu:3177  case Intrinsic::aarch64_neon_smull:3178  case Intrinsic::aarch64_neon_umull: {3179    Value *Arg0 = II->getArgOperand(0);3180    Value *Arg1 = II->getArgOperand(1);3181 3182    // Handle mul by zero first:3183    if (isa<ConstantAggregateZero>(Arg0) || isa<ConstantAggregateZero>(Arg1)) {3184      return replaceInstUsesWith(CI, ConstantAggregateZero::get(II->getType()));3185    }3186 3187    // Check for constant LHS & RHS - in this case we just simplify.3188    bool Zext = (IID == Intrinsic::arm_neon_vmullu ||3189                 IID == Intrinsic::aarch64_neon_umull);3190    VectorType *NewVT = cast<VectorType>(II->getType());3191    if (Constant *CV0 = dyn_cast<Constant>(Arg0)) {3192      if (Constant *CV1 = dyn_cast<Constant>(Arg1)) {3193        Value *V0 = Builder.CreateIntCast(CV0, NewVT, /*isSigned=*/!Zext);3194        Value *V1 = Builder.CreateIntCast(CV1, NewVT, /*isSigned=*/!Zext);3195        return replaceInstUsesWith(CI, Builder.CreateMul(V0, V1));3196      }3197 3198      // Couldn't simplify - canonicalize constant to the RHS.3199      std::swap(Arg0, Arg1);3200    }3201 3202    // Handle mul by one:3203    if (Constant *CV1 = dyn_cast<Constant>(Arg1))3204      if (ConstantInt *Splat =3205              dyn_cast_or_null<ConstantInt>(CV1->getSplatValue()))3206        if (Splat->isOne())3207          return CastInst::CreateIntegerCast(Arg0, II->getType(),3208                                             /*isSigned=*/!Zext);3209 3210    break;3211  }3212  case Intrinsic::arm_neon_aesd:3213  case Intrinsic::arm_neon_aese:3214  case Intrinsic::aarch64_crypto_aesd:3215  case Intrinsic::aarch64_crypto_aese:3216  case Intrinsic::aarch64_sve_aesd:3217  case Intrinsic::aarch64_sve_aese: {3218    Value *DataArg = II->getArgOperand(0);3219    Value *KeyArg  = II->getArgOperand(1);3220 3221    // Accept zero on either operand.3222    if (!match(KeyArg, m_ZeroInt()))3223      std::swap(KeyArg, DataArg);3224 3225    // Try to use the builtin XOR in AESE and AESD to eliminate a prior XOR3226    Value *Data, *Key;3227    if (match(KeyArg, m_ZeroInt()) &&3228        match(DataArg, m_Xor(m_Value(Data), m_Value(Key)))) {3229      replaceOperand(*II, 0, Data);3230      replaceOperand(*II, 1, Key);3231      return II;3232    }3233    break;3234  }3235  case Intrinsic::hexagon_V6_vandvrt:3236  case Intrinsic::hexagon_V6_vandvrt_128B: {3237    // Simplify Q -> V -> Q conversion.3238    if (auto Op0 = dyn_cast<IntrinsicInst>(II->getArgOperand(0))) {3239      Intrinsic::ID ID0 = Op0->getIntrinsicID();3240      if (ID0 != Intrinsic::hexagon_V6_vandqrt &&3241          ID0 != Intrinsic::hexagon_V6_vandqrt_128B)3242        break;3243      Value *Bytes = Op0->getArgOperand(1), *Mask = II->getArgOperand(1);3244      uint64_t Bytes1 = computeKnownBits(Bytes, Op0).One.getZExtValue();3245      uint64_t Mask1 = computeKnownBits(Mask, II).One.getZExtValue();3246      // Check if every byte has common bits in Bytes and Mask.3247      uint64_t C = Bytes1 & Mask1;3248      if ((C & 0xFF) && (C & 0xFF00) && (C & 0xFF0000) && (C & 0xFF000000))3249        return replaceInstUsesWith(*II, Op0->getArgOperand(0));3250    }3251    break;3252  }3253  case Intrinsic::stackrestore: {3254    enum class ClassifyResult {3255      None,3256      Alloca,3257      StackRestore,3258      CallWithSideEffects,3259    };3260    auto Classify = [](const Instruction *I) {3261      if (isa<AllocaInst>(I))3262        return ClassifyResult::Alloca;3263 3264      if (auto *CI = dyn_cast<CallInst>(I)) {3265        if (auto *II = dyn_cast<IntrinsicInst>(CI)) {3266          if (II->getIntrinsicID() == Intrinsic::stackrestore)3267            return ClassifyResult::StackRestore;3268 3269          if (II->mayHaveSideEffects())3270            return ClassifyResult::CallWithSideEffects;3271        } else {3272          // Consider all non-intrinsic calls to be side effects3273          return ClassifyResult::CallWithSideEffects;3274        }3275      }3276 3277      return ClassifyResult::None;3278    };3279 3280    // If the stacksave and the stackrestore are in the same BB, and there is3281    // no intervening call, alloca, or stackrestore of a different stacksave,3282    // remove the restore. This can happen when variable allocas are DCE'd.3283    if (IntrinsicInst *SS = dyn_cast<IntrinsicInst>(II->getArgOperand(0))) {3284      if (SS->getIntrinsicID() == Intrinsic::stacksave &&3285          SS->getParent() == II->getParent()) {3286        BasicBlock::iterator BI(SS);3287        bool CannotRemove = false;3288        for (++BI; &*BI != II; ++BI) {3289          switch (Classify(&*BI)) {3290          case ClassifyResult::None:3291            // So far so good, look at next instructions.3292            break;3293 3294          case ClassifyResult::StackRestore:3295            // If we found an intervening stackrestore for a different3296            // stacksave, we can't remove the stackrestore. Otherwise, continue.3297            if (cast<IntrinsicInst>(*BI).getArgOperand(0) != SS)3298              CannotRemove = true;3299            break;3300 3301          case ClassifyResult::Alloca:3302          case ClassifyResult::CallWithSideEffects:3303            // If we found an alloca, a non-intrinsic call, or an intrinsic3304            // call with side effects, we can't remove the stackrestore.3305            CannotRemove = true;3306            break;3307          }3308          if (CannotRemove)3309            break;3310        }3311 3312        if (!CannotRemove)3313          return eraseInstFromFunction(CI);3314      }3315    }3316 3317    // Scan down this block to see if there is another stack restore in the3318    // same block without an intervening call/alloca.3319    BasicBlock::iterator BI(II);3320    Instruction *TI = II->getParent()->getTerminator();3321    bool CannotRemove = false;3322    for (++BI; &*BI != TI; ++BI) {3323      switch (Classify(&*BI)) {3324      case ClassifyResult::None:3325        // So far so good, look at next instructions.3326        break;3327 3328      case ClassifyResult::StackRestore:3329        // If there is a stackrestore below this one, remove this one.3330        return eraseInstFromFunction(CI);3331 3332      case ClassifyResult::Alloca:3333      case ClassifyResult::CallWithSideEffects:3334        // If we found an alloca, a non-intrinsic call, or an intrinsic call3335        // with side effects (such as llvm.stacksave and llvm.read_register),3336        // we can't remove the stack restore.3337        CannotRemove = true;3338        break;3339      }3340      if (CannotRemove)3341        break;3342    }3343 3344    // If the stack restore is in a return, resume, or unwind block and if there3345    // are no allocas or calls between the restore and the return, nuke the3346    // restore.3347    if (!CannotRemove && (isa<ReturnInst>(TI) || isa<ResumeInst>(TI)))3348      return eraseInstFromFunction(CI);3349    break;3350  }3351  case Intrinsic::lifetime_end:3352    // Asan needs to poison memory to detect invalid access which is possible3353    // even for empty lifetime range.3354    if (II->getFunction()->hasFnAttribute(Attribute::SanitizeAddress) ||3355        II->getFunction()->hasFnAttribute(Attribute::SanitizeMemory) ||3356        II->getFunction()->hasFnAttribute(Attribute::SanitizeHWAddress))3357      break;3358 3359    if (removeTriviallyEmptyRange(*II, *this, [](const IntrinsicInst &I) {3360          return I.getIntrinsicID() == Intrinsic::lifetime_start;3361        }))3362      return nullptr;3363    break;3364  case Intrinsic::assume: {3365    Value *IIOperand = II->getArgOperand(0);3366    SmallVector<OperandBundleDef, 4> OpBundles;3367    II->getOperandBundlesAsDefs(OpBundles);3368 3369    /// This will remove the boolean Condition from the assume given as3370    /// argument and remove the assume if it becomes useless.3371    /// always returns nullptr for use as a return values.3372    auto RemoveConditionFromAssume = [&](Instruction *Assume) -> Instruction * {3373      assert(isa<AssumeInst>(Assume));3374      if (isAssumeWithEmptyBundle(*cast<AssumeInst>(II)))3375        return eraseInstFromFunction(CI);3376      replaceUse(II->getOperandUse(0), ConstantInt::getTrue(II->getContext()));3377      return nullptr;3378    };3379    // Remove an assume if it is followed by an identical assume.3380    // TODO: Do we need this? Unless there are conflicting assumptions, the3381    // computeKnownBits(IIOperand) below here eliminates redundant assumes.3382    Instruction *Next = II->getNextNode();3383    if (match(Next, m_Intrinsic<Intrinsic::assume>(m_Specific(IIOperand))))3384      return RemoveConditionFromAssume(Next);3385 3386    // Canonicalize assume(a && b) -> assume(a); assume(b);3387    // Note: New assumption intrinsics created here are registered by3388    // the InstCombineIRInserter object.3389    FunctionType *AssumeIntrinsicTy = II->getFunctionType();3390    Value *AssumeIntrinsic = II->getCalledOperand();3391    Value *A, *B;3392    if (match(IIOperand, m_LogicalAnd(m_Value(A), m_Value(B)))) {3393      Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic, A, OpBundles,3394                         II->getName());3395      Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic, B, II->getName());3396      return eraseInstFromFunction(*II);3397    }3398    // assume(!(a || b)) -> assume(!a); assume(!b);3399    if (match(IIOperand, m_Not(m_LogicalOr(m_Value(A), m_Value(B))))) {3400      Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic,3401                         Builder.CreateNot(A), OpBundles, II->getName());3402      Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic,3403                         Builder.CreateNot(B), II->getName());3404      return eraseInstFromFunction(*II);3405    }3406 3407    // assume( (load addr) != null ) -> add 'nonnull' metadata to load3408    // (if assume is valid at the load)3409    Instruction *LHS;3410    if (match(IIOperand, m_SpecificICmp(ICmpInst::ICMP_NE, m_Instruction(LHS),3411                                        m_Zero())) &&3412        LHS->getOpcode() == Instruction::Load &&3413        LHS->getType()->isPointerTy() &&3414        isValidAssumeForContext(II, LHS, &DT)) {3415      MDNode *MD = MDNode::get(II->getContext(), {});3416      LHS->setMetadata(LLVMContext::MD_nonnull, MD);3417      LHS->setMetadata(LLVMContext::MD_noundef, MD);3418      return RemoveConditionFromAssume(II);3419 3420      // TODO: apply nonnull return attributes to calls and invokes3421      // TODO: apply range metadata for range check patterns?3422    }3423 3424    for (unsigned Idx = 0; Idx < II->getNumOperandBundles(); Idx++) {3425      OperandBundleUse OBU = II->getOperandBundleAt(Idx);3426 3427      // Separate storage assumptions apply to the underlying allocations, not3428      // any particular pointer within them. When evaluating the hints for AA3429      // purposes we getUnderlyingObject them; by precomputing the answers here3430      // we can avoid having to do so repeatedly there.3431      if (OBU.getTagName() == "separate_storage") {3432        assert(OBU.Inputs.size() == 2);3433        auto MaybeSimplifyHint = [&](const Use &U) {3434          Value *Hint = U.get();3435          // Not having a limit is safe because InstCombine removes unreachable3436          // code.3437          Value *UnderlyingObject = getUnderlyingObject(Hint, /*MaxLookup*/ 0);3438          if (Hint != UnderlyingObject)3439            replaceUse(const_cast<Use &>(U), UnderlyingObject);3440        };3441        MaybeSimplifyHint(OBU.Inputs[0]);3442        MaybeSimplifyHint(OBU.Inputs[1]);3443      }3444 3445      // Try to remove redundant alignment assumptions.3446      if (OBU.getTagName() == "align" && OBU.Inputs.size() == 2) {3447        RetainedKnowledge RK = getKnowledgeFromOperandInAssume(3448            *cast<AssumeInst>(II), II->arg_size() + Idx);3449        if (!RK || RK.AttrKind != Attribute::Alignment ||3450            !isPowerOf2_64(RK.ArgValue) || !isa<ConstantInt>(RK.IRArgValue))3451          continue;3452 3453        // Remove align 1 bundles; they don't add any useful information.3454        if (RK.ArgValue == 1)3455          return CallBase::removeOperandBundle(II, OBU.getTagID());3456 3457        // Don't try to remove align assumptions for pointers derived from3458        // arguments. We might lose information if the function gets inline and3459        // the align argument attribute disappears.3460        Value *UO = getUnderlyingObject(RK.WasOn);3461        if (!UO || isa<Argument>(UO))3462          continue;3463 3464        // Compute known bits for the pointer, passing nullptr as context to3465        // avoid computeKnownBits using the assumption we are about to remove3466        // for reasoning.3467        KnownBits Known = computeKnownBits(RK.WasOn, /*CtxI=*/nullptr);3468        unsigned TZ = std::min(Known.countMinTrailingZeros(),3469                               Value::MaxAlignmentExponent);3470        if ((1ULL << TZ) < RK.ArgValue)3471          continue;3472        return CallBase::removeOperandBundle(II, OBU.getTagID());3473      }3474    }3475 3476    // Convert nonnull assume like:3477    // %A = icmp ne i32* %PTR, null3478    // call void @llvm.assume(i1 %A)3479    // into3480    // call void @llvm.assume(i1 true) [ "nonnull"(i32* %PTR) ]3481    if (EnableKnowledgeRetention &&3482        match(IIOperand,3483              m_SpecificICmp(ICmpInst::ICMP_NE, m_Value(A), m_Zero())) &&3484        A->getType()->isPointerTy()) {3485      if (auto *Replacement = buildAssumeFromKnowledge(3486              {RetainedKnowledge{Attribute::NonNull, 0, A}}, Next, &AC, &DT)) {3487 3488        Replacement->insertBefore(Next->getIterator());3489        AC.registerAssumption(Replacement);3490        return RemoveConditionFromAssume(II);3491      }3492    }3493 3494    // Convert alignment assume like:3495    // %B = ptrtoint i32* %A to i643496    // %C = and i64 %B, Constant3497    // %D = icmp eq i64 %C, 03498    // call void @llvm.assume(i1 %D)3499    // into3500    // call void @llvm.assume(i1 true) [ "align"(i32* [[A]], i64  Constant + 1)]3501    uint64_t AlignMask = 1;3502    if (EnableKnowledgeRetention &&3503        (match(IIOperand, m_Not(m_Trunc(m_Value(A)))) ||3504         match(IIOperand,3505               m_SpecificICmp(ICmpInst::ICMP_EQ,3506                              m_And(m_Value(A), m_ConstantInt(AlignMask)),3507                              m_Zero())))) {3508      if (isPowerOf2_64(AlignMask + 1)) {3509        uint64_t Offset = 0;3510        match(A, m_Add(m_Value(A), m_ConstantInt(Offset)));3511        if (match(A, m_PtrToIntOrAddr(m_Value(A)))) {3512          /// Note: this doesn't preserve the offset information but merges3513          /// offset and alignment.3514          /// TODO: we can generate a GEP instead of merging the alignment with3515          /// the offset.3516          RetainedKnowledge RK{Attribute::Alignment,3517                               (unsigned)MinAlign(Offset, AlignMask + 1), A};3518          if (auto *Replacement =3519                  buildAssumeFromKnowledge(RK, Next, &AC, &DT)) {3520 3521            Replacement->insertAfter(II->getIterator());3522            AC.registerAssumption(Replacement);3523          }3524          return RemoveConditionFromAssume(II);3525        }3526      }3527    }3528 3529    /// Canonicalize Knowledge in operand bundles.3530    if (EnableKnowledgeRetention && II->hasOperandBundles()) {3531      for (unsigned Idx = 0; Idx < II->getNumOperandBundles(); Idx++) {3532        auto &BOI = II->bundle_op_info_begin()[Idx];3533        RetainedKnowledge RK =3534          llvm::getKnowledgeFromBundle(cast<AssumeInst>(*II), BOI);3535        if (BOI.End - BOI.Begin > 2)3536          continue; // Prevent reducing knowledge in an align with offset since3537                    // extracting a RetainedKnowledge from them looses offset3538                    // information3539        RetainedKnowledge CanonRK =3540          llvm::simplifyRetainedKnowledge(cast<AssumeInst>(II), RK,3541                                          &getAssumptionCache(),3542                                          &getDominatorTree());3543        if (CanonRK == RK)3544          continue;3545        if (!CanonRK) {3546          if (BOI.End - BOI.Begin > 0) {3547            Worklist.pushValue(II->op_begin()[BOI.Begin]);3548            Value::dropDroppableUse(II->op_begin()[BOI.Begin]);3549          }3550          continue;3551        }3552        assert(RK.AttrKind == CanonRK.AttrKind);3553        if (BOI.End - BOI.Begin > 0)3554          II->op_begin()[BOI.Begin].set(CanonRK.WasOn);3555        if (BOI.End - BOI.Begin > 1)3556          II->op_begin()[BOI.Begin + 1].set(ConstantInt::get(3557              Type::getInt64Ty(II->getContext()), CanonRK.ArgValue));3558        if (RK.WasOn)3559          Worklist.pushValue(RK.WasOn);3560        return II;3561      }3562    }3563 3564    // If there is a dominating assume with the same condition as this one,3565    // then this one is redundant, and should be removed.3566    KnownBits Known(1);3567    computeKnownBits(IIOperand, Known, II);3568    if (Known.isAllOnes() && isAssumeWithEmptyBundle(cast<AssumeInst>(*II)))3569      return eraseInstFromFunction(*II);3570 3571    // assume(false) is unreachable.3572    if (match(IIOperand, m_CombineOr(m_Zero(), m_Undef()))) {3573      CreateNonTerminatorUnreachable(II);3574      return eraseInstFromFunction(*II);3575    }3576 3577    // Update the cache of affected values for this assumption (we might be3578    // here because we just simplified the condition).3579    AC.updateAffectedValues(cast<AssumeInst>(II));3580    break;3581  }3582  case Intrinsic::experimental_guard: {3583    // Is this guard followed by another guard?  We scan forward over a small3584    // fixed window of instructions to handle common cases with conditions3585    // computed between guards.3586    Instruction *NextInst = II->getNextNode();3587    for (unsigned i = 0; i < GuardWideningWindow; i++) {3588      // Note: Using context-free form to avoid compile time blow up3589      if (!isSafeToSpeculativelyExecute(NextInst))3590        break;3591      NextInst = NextInst->getNextNode();3592    }3593    Value *NextCond = nullptr;3594    if (match(NextInst,3595              m_Intrinsic<Intrinsic::experimental_guard>(m_Value(NextCond)))) {3596      Value *CurrCond = II->getArgOperand(0);3597 3598      // Remove a guard that it is immediately preceded by an identical guard.3599      // Otherwise canonicalize guard(a); guard(b) -> guard(a & b).3600      if (CurrCond != NextCond) {3601        Instruction *MoveI = II->getNextNode();3602        while (MoveI != NextInst) {3603          auto *Temp = MoveI;3604          MoveI = MoveI->getNextNode();3605          Temp->moveBefore(II->getIterator());3606        }3607        replaceOperand(*II, 0, Builder.CreateAnd(CurrCond, NextCond));3608      }3609      eraseInstFromFunction(*NextInst);3610      return II;3611    }3612    break;3613  }3614  case Intrinsic::vector_insert: {3615    Value *Vec = II->getArgOperand(0);3616    Value *SubVec = II->getArgOperand(1);3617    Value *Idx = II->getArgOperand(2);3618    auto *DstTy = dyn_cast<FixedVectorType>(II->getType());3619    auto *VecTy = dyn_cast<FixedVectorType>(Vec->getType());3620    auto *SubVecTy = dyn_cast<FixedVectorType>(SubVec->getType());3621 3622    // Only canonicalize if the destination vector, Vec, and SubVec are all3623    // fixed vectors.3624    if (DstTy && VecTy && SubVecTy) {3625      unsigned DstNumElts = DstTy->getNumElements();3626      unsigned VecNumElts = VecTy->getNumElements();3627      unsigned SubVecNumElts = SubVecTy->getNumElements();3628      unsigned IdxN = cast<ConstantInt>(Idx)->getZExtValue();3629 3630      // An insert that entirely overwrites Vec with SubVec is a nop.3631      if (VecNumElts == SubVecNumElts)3632        return replaceInstUsesWith(CI, SubVec);3633 3634      // Widen SubVec into a vector of the same width as Vec, since3635      // shufflevector requires the two input vectors to be the same width.3636      // Elements beyond the bounds of SubVec within the widened vector are3637      // undefined.3638      SmallVector<int, 8> WidenMask;3639      unsigned i;3640      for (i = 0; i != SubVecNumElts; ++i)3641        WidenMask.push_back(i);3642      for (; i != VecNumElts; ++i)3643        WidenMask.push_back(PoisonMaskElem);3644 3645      Value *WidenShuffle = Builder.CreateShuffleVector(SubVec, WidenMask);3646 3647      SmallVector<int, 8> Mask;3648      for (unsigned i = 0; i != IdxN; ++i)3649        Mask.push_back(i);3650      for (unsigned i = DstNumElts; i != DstNumElts + SubVecNumElts; ++i)3651        Mask.push_back(i);3652      for (unsigned i = IdxN + SubVecNumElts; i != DstNumElts; ++i)3653        Mask.push_back(i);3654 3655      Value *Shuffle = Builder.CreateShuffleVector(Vec, WidenShuffle, Mask);3656      return replaceInstUsesWith(CI, Shuffle);3657    }3658    break;3659  }3660  case Intrinsic::vector_extract: {3661    Value *Vec = II->getArgOperand(0);3662    Value *Idx = II->getArgOperand(1);3663 3664    Type *ReturnType = II->getType();3665    // (extract_vector (insert_vector InsertTuple, InsertValue, InsertIdx),3666    // ExtractIdx)3667    unsigned ExtractIdx = cast<ConstantInt>(Idx)->getZExtValue();3668    Value *InsertTuple, *InsertIdx, *InsertValue;3669    if (match(Vec, m_Intrinsic<Intrinsic::vector_insert>(m_Value(InsertTuple),3670                                                         m_Value(InsertValue),3671                                                         m_Value(InsertIdx))) &&3672        InsertValue->getType() == ReturnType) {3673      unsigned Index = cast<ConstantInt>(InsertIdx)->getZExtValue();3674      // Case where we get the same index right after setting it.3675      // extract.vector(insert.vector(InsertTuple, InsertValue, Idx), Idx) -->3676      // InsertValue3677      if (ExtractIdx == Index)3678        return replaceInstUsesWith(CI, InsertValue);3679      // If we are getting a different index than what was set in the3680      // insert.vector intrinsic. We can just set the input tuple to the one up3681      // in the chain. extract.vector(insert.vector(InsertTuple, InsertValue,3682      // InsertIndex), ExtractIndex)3683      // --> extract.vector(InsertTuple, ExtractIndex)3684      else3685        return replaceOperand(CI, 0, InsertTuple);3686    }3687 3688    auto *DstTy = dyn_cast<VectorType>(ReturnType);3689    auto *VecTy = dyn_cast<VectorType>(Vec->getType());3690 3691    if (DstTy && VecTy) {3692      auto DstEltCnt = DstTy->getElementCount();3693      auto VecEltCnt = VecTy->getElementCount();3694      unsigned IdxN = cast<ConstantInt>(Idx)->getZExtValue();3695 3696      // Extracting the entirety of Vec is a nop.3697      if (DstEltCnt == VecTy->getElementCount()) {3698        replaceInstUsesWith(CI, Vec);3699        return eraseInstFromFunction(CI);3700      }3701 3702      // Only canonicalize to shufflevector if the destination vector and3703      // Vec are fixed vectors.3704      if (VecEltCnt.isScalable() || DstEltCnt.isScalable())3705        break;3706 3707      SmallVector<int, 8> Mask;3708      for (unsigned i = 0; i != DstEltCnt.getKnownMinValue(); ++i)3709        Mask.push_back(IdxN + i);3710 3711      Value *Shuffle = Builder.CreateShuffleVector(Vec, Mask);3712      return replaceInstUsesWith(CI, Shuffle);3713    }3714    break;3715  }3716  case Intrinsic::experimental_vp_reverse: {3717    Value *X;3718    Value *Vec = II->getArgOperand(0);3719    Value *Mask = II->getArgOperand(1);3720    if (!match(Mask, m_AllOnes()))3721      break;3722    Value *EVL = II->getArgOperand(2);3723    // TODO: Canonicalize experimental.vp.reverse after unop/binops?3724    // rev(unop rev(X)) --> unop X3725    if (match(Vec,3726              m_OneUse(m_UnOp(m_Intrinsic<Intrinsic::experimental_vp_reverse>(3727                  m_Value(X), m_AllOnes(), m_Specific(EVL)))))) {3728      auto *OldUnOp = cast<UnaryOperator>(Vec);3729      auto *NewUnOp = UnaryOperator::CreateWithCopiedFlags(3730          OldUnOp->getOpcode(), X, OldUnOp, OldUnOp->getName(),3731          II->getIterator());3732      return replaceInstUsesWith(CI, NewUnOp);3733    }3734    break;3735  }3736  case Intrinsic::vector_reduce_or:3737  case Intrinsic::vector_reduce_and: {3738    // Canonicalize logical or/and reductions:3739    // Or reduction for i1 is represented as:3740    // %val = bitcast <ReduxWidth x i1> to iReduxWidth3741    // %res = cmp ne iReduxWidth %val, 03742    // And reduction for i1 is represented as:3743    // %val = bitcast <ReduxWidth x i1> to iReduxWidth3744    // %res = cmp eq iReduxWidth %val, 111113745    Value *Arg = II->getArgOperand(0);3746    Value *Vect;3747 3748    if (Value *NewOp =3749            simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {3750      replaceUse(II->getOperandUse(0), NewOp);3751      return II;3752    }3753 3754    if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {3755      if (auto *FTy = dyn_cast<FixedVectorType>(Vect->getType()))3756        if (FTy->getElementType() == Builder.getInt1Ty()) {3757          Value *Res = Builder.CreateBitCast(3758              Vect, Builder.getIntNTy(FTy->getNumElements()));3759          if (IID == Intrinsic::vector_reduce_and) {3760            Res = Builder.CreateICmpEQ(3761                Res, ConstantInt::getAllOnesValue(Res->getType()));3762          } else {3763            assert(IID == Intrinsic::vector_reduce_or &&3764                   "Expected or reduction.");3765            Res = Builder.CreateIsNotNull(Res);3766          }3767          if (Arg != Vect)3768            Res = Builder.CreateCast(cast<CastInst>(Arg)->getOpcode(), Res,3769                                     II->getType());3770          return replaceInstUsesWith(CI, Res);3771        }3772    }3773    [[fallthrough]];3774  }3775  case Intrinsic::vector_reduce_add: {3776    if (IID == Intrinsic::vector_reduce_add) {3777      // Convert vector_reduce_add(ZExt(<n x i1>)) to3778      // ZExtOrTrunc(ctpop(bitcast <n x i1> to in)).3779      // Convert vector_reduce_add(SExt(<n x i1>)) to3780      // -ZExtOrTrunc(ctpop(bitcast <n x i1> to in)).3781      // Convert vector_reduce_add(<n x i1>) to3782      // Trunc(ctpop(bitcast <n x i1> to in)).3783      Value *Arg = II->getArgOperand(0);3784      Value *Vect;3785 3786      if (Value *NewOp =3787              simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {3788        replaceUse(II->getOperandUse(0), NewOp);3789        return II;3790      }3791 3792      if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {3793        if (auto *FTy = dyn_cast<FixedVectorType>(Vect->getType()))3794          if (FTy->getElementType() == Builder.getInt1Ty()) {3795            Value *V = Builder.CreateBitCast(3796                Vect, Builder.getIntNTy(FTy->getNumElements()));3797            Value *Res = Builder.CreateUnaryIntrinsic(Intrinsic::ctpop, V);3798            if (Res->getType() != II->getType())3799              Res = Builder.CreateZExtOrTrunc(Res, II->getType());3800            if (Arg != Vect &&3801                cast<Instruction>(Arg)->getOpcode() == Instruction::SExt)3802              Res = Builder.CreateNeg(Res);3803            return replaceInstUsesWith(CI, Res);3804          }3805      }3806 3807      // vector.reduce.add.vNiM(splat(%x)) -> mul(%x, N)3808      if (Value *Splat = getSplatValue(Arg)) {3809        ElementCount VecToReduceCount =3810            cast<VectorType>(Arg->getType())->getElementCount();3811        if (VecToReduceCount.isFixed()) {3812          unsigned VectorSize = VecToReduceCount.getFixedValue();3813          return BinaryOperator::CreateMul(3814              Splat, ConstantInt::get(Splat->getType(), VectorSize));3815        }3816      }3817    }3818    [[fallthrough]];3819  }3820  case Intrinsic::vector_reduce_xor: {3821    if (IID == Intrinsic::vector_reduce_xor) {3822      // Exclusive disjunction reduction over the vector with3823      // (potentially-extended) i1 element type is actually a3824      // (potentially-extended) arithmetic `add` reduction over the original3825      // non-extended value:3826      //   vector_reduce_xor(?ext(<n x i1>))3827      //     -->3828      //   ?ext(vector_reduce_add(<n x i1>))3829      Value *Arg = II->getArgOperand(0);3830      Value *Vect;3831 3832      if (Value *NewOp =3833              simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {3834        replaceUse(II->getOperandUse(0), NewOp);3835        return II;3836      }3837 3838      if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {3839        if (auto *VTy = dyn_cast<VectorType>(Vect->getType()))3840          if (VTy->getElementType() == Builder.getInt1Ty()) {3841            Value *Res = Builder.CreateAddReduce(Vect);3842            if (Arg != Vect)3843              Res = Builder.CreateCast(cast<CastInst>(Arg)->getOpcode(), Res,3844                                       II->getType());3845            return replaceInstUsesWith(CI, Res);3846          }3847      }3848    }3849    [[fallthrough]];3850  }3851  case Intrinsic::vector_reduce_mul: {3852    if (IID == Intrinsic::vector_reduce_mul) {3853      // Multiplicative reduction over the vector with (potentially-extended)3854      // i1 element type is actually a (potentially zero-extended)3855      // logical `and` reduction over the original non-extended value:3856      //   vector_reduce_mul(?ext(<n x i1>))3857      //     -->3858      //   zext(vector_reduce_and(<n x i1>))3859      Value *Arg = II->getArgOperand(0);3860      Value *Vect;3861 3862      if (Value *NewOp =3863              simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {3864        replaceUse(II->getOperandUse(0), NewOp);3865        return II;3866      }3867 3868      if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {3869        if (auto *VTy = dyn_cast<VectorType>(Vect->getType()))3870          if (VTy->getElementType() == Builder.getInt1Ty()) {3871            Value *Res = Builder.CreateAndReduce(Vect);3872            if (Res->getType() != II->getType())3873              Res = Builder.CreateZExt(Res, II->getType());3874            return replaceInstUsesWith(CI, Res);3875          }3876      }3877    }3878    [[fallthrough]];3879  }3880  case Intrinsic::vector_reduce_umin:3881  case Intrinsic::vector_reduce_umax: {3882    if (IID == Intrinsic::vector_reduce_umin ||3883        IID == Intrinsic::vector_reduce_umax) {3884      // UMin/UMax reduction over the vector with (potentially-extended)3885      // i1 element type is actually a (potentially-extended)3886      // logical `and`/`or` reduction over the original non-extended value:3887      //   vector_reduce_u{min,max}(?ext(<n x i1>))3888      //     -->3889      //   ?ext(vector_reduce_{and,or}(<n x i1>))3890      Value *Arg = II->getArgOperand(0);3891      Value *Vect;3892 3893      if (Value *NewOp =3894              simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {3895        replaceUse(II->getOperandUse(0), NewOp);3896        return II;3897      }3898 3899      if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {3900        if (auto *VTy = dyn_cast<VectorType>(Vect->getType()))3901          if (VTy->getElementType() == Builder.getInt1Ty()) {3902            Value *Res = IID == Intrinsic::vector_reduce_umin3903                             ? Builder.CreateAndReduce(Vect)3904                             : Builder.CreateOrReduce(Vect);3905            if (Arg != Vect)3906              Res = Builder.CreateCast(cast<CastInst>(Arg)->getOpcode(), Res,3907                                       II->getType());3908            return replaceInstUsesWith(CI, Res);3909          }3910      }3911    }3912    [[fallthrough]];3913  }3914  case Intrinsic::vector_reduce_smin:3915  case Intrinsic::vector_reduce_smax: {3916    if (IID == Intrinsic::vector_reduce_smin ||3917        IID == Intrinsic::vector_reduce_smax) {3918      // SMin/SMax reduction over the vector with (potentially-extended)3919      // i1 element type is actually a (potentially-extended)3920      // logical `and`/`or` reduction over the original non-extended value:3921      //   vector_reduce_s{min,max}(<n x i1>)3922      //     -->3923      //   vector_reduce_{or,and}(<n x i1>)3924      // and3925      //   vector_reduce_s{min,max}(sext(<n x i1>))3926      //     -->3927      //   sext(vector_reduce_{or,and}(<n x i1>))3928      // and3929      //   vector_reduce_s{min,max}(zext(<n x i1>))3930      //     -->3931      //   zext(vector_reduce_{and,or}(<n x i1>))3932      Value *Arg = II->getArgOperand(0);3933      Value *Vect;3934 3935      if (Value *NewOp =3936              simplifyReductionOperand(Arg, /*CanReorderLanes=*/true)) {3937        replaceUse(II->getOperandUse(0), NewOp);3938        return II;3939      }3940 3941      if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {3942        if (auto *VTy = dyn_cast<VectorType>(Vect->getType()))3943          if (VTy->getElementType() == Builder.getInt1Ty()) {3944            Instruction::CastOps ExtOpc = Instruction::CastOps::CastOpsEnd;3945            if (Arg != Vect)3946              ExtOpc = cast<CastInst>(Arg)->getOpcode();3947            Value *Res = ((IID == Intrinsic::vector_reduce_smin) ==3948                          (ExtOpc == Instruction::CastOps::ZExt))3949                             ? Builder.CreateAndReduce(Vect)3950                             : Builder.CreateOrReduce(Vect);3951            if (Arg != Vect)3952              Res = Builder.CreateCast(ExtOpc, Res, II->getType());3953            return replaceInstUsesWith(CI, Res);3954          }3955      }3956    }3957    [[fallthrough]];3958  }3959  case Intrinsic::vector_reduce_fmax:3960  case Intrinsic::vector_reduce_fmin:3961  case Intrinsic::vector_reduce_fadd:3962  case Intrinsic::vector_reduce_fmul: {3963    bool CanReorderLanes = (IID != Intrinsic::vector_reduce_fadd &&3964                            IID != Intrinsic::vector_reduce_fmul) ||3965                           II->hasAllowReassoc();3966    const unsigned ArgIdx = (IID == Intrinsic::vector_reduce_fadd ||3967                             IID == Intrinsic::vector_reduce_fmul)3968                                ? 13969                                : 0;3970    Value *Arg = II->getArgOperand(ArgIdx);3971    if (Value *NewOp = simplifyReductionOperand(Arg, CanReorderLanes)) {3972      replaceUse(II->getOperandUse(ArgIdx), NewOp);3973      return nullptr;3974    }3975    break;3976  }3977  case Intrinsic::is_fpclass: {3978    if (Instruction *I = foldIntrinsicIsFPClass(*II))3979      return I;3980    break;3981  }3982  case Intrinsic::threadlocal_address: {3983    Align MinAlign = getKnownAlignment(II->getArgOperand(0), DL, II, &AC, &DT);3984    MaybeAlign Align = II->getRetAlign();3985    if (MinAlign > Align.valueOrOne()) {3986      II->addRetAttr(Attribute::getWithAlignment(II->getContext(), MinAlign));3987      return II;3988    }3989    break;3990  }3991  case Intrinsic::frexp: {3992    Value *X;3993    // The first result is idempotent with the added complication of the struct3994    // return, and the second result is zero because the value is already3995    // normalized.3996    if (match(II->getArgOperand(0), m_ExtractValue<0>(m_Value(X)))) {3997      if (match(X, m_Intrinsic<Intrinsic::frexp>(m_Value()))) {3998        X = Builder.CreateInsertValue(3999            X, Constant::getNullValue(II->getType()->getStructElementType(1)),4000            1);4001        return replaceInstUsesWith(*II, X);4002      }4003    }4004    break;4005  }4006  case Intrinsic::get_active_lane_mask: {4007    const APInt *Op0, *Op1;4008    if (match(II->getOperand(0), m_StrictlyPositive(Op0)) &&4009        match(II->getOperand(1), m_APInt(Op1))) {4010      Type *OpTy = II->getOperand(0)->getType();4011      return replaceInstUsesWith(4012          *II, Builder.CreateIntrinsic(4013                   II->getType(), Intrinsic::get_active_lane_mask,4014                   {Constant::getNullValue(OpTy),4015                    ConstantInt::get(OpTy, Op1->usub_sat(*Op0))}));4016    }4017    break;4018  }4019  case Intrinsic::experimental_get_vector_length: {4020    // get.vector.length(Cnt, MaxLanes) --> Cnt when Cnt <= MaxLanes4021    unsigned BitWidth =4022        std::max(II->getArgOperand(0)->getType()->getScalarSizeInBits(),4023                 II->getType()->getScalarSizeInBits());4024    ConstantRange Cnt =4025        computeConstantRangeIncludingKnownBits(II->getArgOperand(0), false,4026                                               SQ.getWithInstruction(II))4027            .zextOrTrunc(BitWidth);4028    ConstantRange MaxLanes = cast<ConstantInt>(II->getArgOperand(1))4029                                 ->getValue()4030                                 .zextOrTrunc(Cnt.getBitWidth());4031    if (cast<ConstantInt>(II->getArgOperand(2))->isOne())4032      MaxLanes = MaxLanes.multiply(4033          getVScaleRange(II->getFunction(), Cnt.getBitWidth()));4034 4035    if (Cnt.icmp(CmpInst::ICMP_ULE, MaxLanes))4036      return replaceInstUsesWith(4037          *II, Builder.CreateZExtOrTrunc(II->getArgOperand(0), II->getType()));4038    return nullptr;4039  }4040  default: {4041    // Handle target specific intrinsics4042    std::optional<Instruction *> V = targetInstCombineIntrinsic(*II);4043    if (V)4044      return *V;4045    break;4046  }4047  }4048 4049  // Try to fold intrinsic into select/phi operands. This is legal if:4050  //  * The intrinsic is speculatable.4051  //  * The operand is one of the following:4052  //    - a phi.4053  //    - a select with a scalar condition.4054  //    - a select with a vector condition and II is not a cross lane operation.4055  if (isSafeToSpeculativelyExecuteWithVariableReplaced(&CI)) {4056    for (Value *Op : II->args()) {4057      if (auto *Sel = dyn_cast<SelectInst>(Op)) {4058        bool IsVectorCond = Sel->getCondition()->getType()->isVectorTy();4059        if (IsVectorCond && !isNotCrossLaneOperation(II))4060          continue;4061        // Don't replace a scalar select with a more expensive vector select if4062        // we can't simplify both arms of the select.4063        bool SimplifyBothArms =4064            !Op->getType()->isVectorTy() && II->getType()->isVectorTy();4065        if (Instruction *R = FoldOpIntoSelect(4066                *II, Sel, /*FoldWithMultiUse=*/false, SimplifyBothArms))4067          return R;4068      }4069      if (auto *Phi = dyn_cast<PHINode>(Op))4070        if (Instruction *R = foldOpIntoPhi(*II, Phi))4071          return R;4072    }4073  }4074 4075  if (Instruction *Shuf = foldShuffledIntrinsicOperands(II))4076    return Shuf;4077 4078  if (Value *Reverse = foldReversedIntrinsicOperands(II))4079    return replaceInstUsesWith(*II, Reverse);4080 4081  if (Value *Res = foldIdempotentBinaryIntrinsicRecurrence(*this, II))4082    return replaceInstUsesWith(*II, Res);4083 4084  // Some intrinsics (like experimental_gc_statepoint) can be used in invoke4085  // context, so it is handled in visitCallBase and we should trigger it.4086  return visitCallBase(*II);4087}4088 4089// Fence instruction simplification4090Instruction *InstCombinerImpl::visitFenceInst(FenceInst &FI) {4091  auto *NFI = dyn_cast<FenceInst>(FI.getNextNode());4092  // This check is solely here to handle arbitrary target-dependent syncscopes.4093  // TODO: Can remove if does not matter in practice.4094  if (NFI && FI.isIdenticalTo(NFI))4095    return eraseInstFromFunction(FI);4096 4097  // Returns true if FI1 is identical or stronger fence than FI2.4098  auto isIdenticalOrStrongerFence = [](FenceInst *FI1, FenceInst *FI2) {4099    auto FI1SyncScope = FI1->getSyncScopeID();4100    // Consider same scope, where scope is global or single-thread.4101    if (FI1SyncScope != FI2->getSyncScopeID() ||4102        (FI1SyncScope != SyncScope::System &&4103         FI1SyncScope != SyncScope::SingleThread))4104      return false;4105 4106    return isAtLeastOrStrongerThan(FI1->getOrdering(), FI2->getOrdering());4107  };4108  if (NFI && isIdenticalOrStrongerFence(NFI, &FI))4109    return eraseInstFromFunction(FI);4110 4111  if (auto *PFI = dyn_cast_or_null<FenceInst>(FI.getPrevNode()))4112    if (isIdenticalOrStrongerFence(PFI, &FI))4113      return eraseInstFromFunction(FI);4114  return nullptr;4115}4116 4117// InvokeInst simplification4118Instruction *InstCombinerImpl::visitInvokeInst(InvokeInst &II) {4119  return visitCallBase(II);4120}4121 4122// CallBrInst simplification4123Instruction *InstCombinerImpl::visitCallBrInst(CallBrInst &CBI) {4124  return visitCallBase(CBI);4125}4126 4127static Value *optimizeModularFormat(CallInst *CI, IRBuilderBase &B) {4128  if (!CI->hasFnAttr("modular-format"))4129    return nullptr;4130 4131  SmallVector<StringRef> Args(4132      llvm::split(CI->getFnAttr("modular-format").getValueAsString(), ','));4133  // TODO: Make use of the first two arguments4134  unsigned FirstArgIdx;4135  [[maybe_unused]] bool Error;4136  Error = Args[2].getAsInteger(10, FirstArgIdx);4137  assert(!Error && "invalid first arg index");4138  --FirstArgIdx;4139  StringRef FnName = Args[3];4140  StringRef ImplName = Args[4];4141  ArrayRef<StringRef> AllAspects = ArrayRef<StringRef>(Args).drop_front(5);4142 4143  if (AllAspects.empty())4144    return nullptr;4145 4146  SmallVector<StringRef> NeededAspects;4147  for (StringRef Aspect : AllAspects) {4148    if (Aspect == "float") {4149      if (llvm::any_of(4150              llvm::make_range(std::next(CI->arg_begin(), FirstArgIdx),4151                               CI->arg_end()),4152              [](Value *V) { return V->getType()->isFloatingPointTy(); }))4153        NeededAspects.push_back("float");4154    } else {4155      // Unknown aspects are always considered to be needed.4156      NeededAspects.push_back(Aspect);4157    }4158  }4159 4160  if (NeededAspects.size() == AllAspects.size())4161    return nullptr;4162 4163  Module *M = CI->getModule();4164  LLVMContext &Ctx = M->getContext();4165  Function *Callee = CI->getCalledFunction();4166  FunctionCallee ModularFn = M->getOrInsertFunction(4167      FnName, Callee->getFunctionType(),4168      Callee->getAttributes().removeFnAttribute(Ctx, "modular-format"));4169  CallInst *New = cast<CallInst>(CI->clone());4170  New->setCalledFunction(ModularFn);4171  New->removeFnAttr("modular-format");4172  B.Insert(New);4173 4174  const auto ReferenceAspect = [&](StringRef Aspect) {4175    SmallString<20> Name = ImplName;4176    Name += '_';4177    Name += Aspect;4178    Function *RelocNoneFn =4179        Intrinsic::getOrInsertDeclaration(M, Intrinsic::reloc_none);4180    B.CreateCall(RelocNoneFn,4181                 {MetadataAsValue::get(Ctx, MDString::get(Ctx, Name))});4182  };4183 4184  llvm::sort(NeededAspects);4185  for (StringRef Request : NeededAspects)4186    ReferenceAspect(Request);4187 4188  return New;4189}4190 4191Instruction *InstCombinerImpl::tryOptimizeCall(CallInst *CI) {4192  if (!CI->getCalledFunction()) return nullptr;4193 4194  // Skip optimizing notail and musttail calls so4195  // LibCallSimplifier::optimizeCall doesn't have to preserve those invariants.4196  // LibCallSimplifier::optimizeCall should try to preserve tail calls though.4197  if (CI->isMustTailCall() || CI->isNoTailCall())4198    return nullptr;4199 4200  auto InstCombineRAUW = [this](Instruction *From, Value *With) {4201    replaceInstUsesWith(*From, With);4202  };4203  auto InstCombineErase = [this](Instruction *I) {4204    eraseInstFromFunction(*I);4205  };4206  LibCallSimplifier Simplifier(DL, &TLI, &DT, &DC, &AC, ORE, BFI, PSI,4207                               InstCombineRAUW, InstCombineErase);4208  if (Value *With = Simplifier.optimizeCall(CI, Builder)) {4209    ++NumSimplified;4210    return CI->use_empty() ? CI : replaceInstUsesWith(*CI, With);4211  }4212  if (Value *With = optimizeModularFormat(CI, Builder)) {4213    ++NumSimplified;4214    return CI->use_empty() ? CI : replaceInstUsesWith(*CI, With);4215  }4216 4217  return nullptr;4218}4219 4220static IntrinsicInst *findInitTrampolineFromAlloca(Value *TrampMem) {4221  // Strip off at most one level of pointer casts, looking for an alloca.  This4222  // is good enough in practice and simpler than handling any number of casts.4223  Value *Underlying = TrampMem->stripPointerCasts();4224  if (Underlying != TrampMem &&4225      (!Underlying->hasOneUse() || Underlying->user_back() != TrampMem))4226    return nullptr;4227  if (!isa<AllocaInst>(Underlying))4228    return nullptr;4229 4230  IntrinsicInst *InitTrampoline = nullptr;4231  for (User *U : TrampMem->users()) {4232    IntrinsicInst *II = dyn_cast<IntrinsicInst>(U);4233    if (!II)4234      return nullptr;4235    if (II->getIntrinsicID() == Intrinsic::init_trampoline) {4236      if (InitTrampoline)4237        // More than one init_trampoline writes to this value.  Give up.4238        return nullptr;4239      InitTrampoline = II;4240      continue;4241    }4242    if (II->getIntrinsicID() == Intrinsic::adjust_trampoline)4243      // Allow any number of calls to adjust.trampoline.4244      continue;4245    return nullptr;4246  }4247 4248  // No call to init.trampoline found.4249  if (!InitTrampoline)4250    return nullptr;4251 4252  // Check that the alloca is being used in the expected way.4253  if (InitTrampoline->getOperand(0) != TrampMem)4254    return nullptr;4255 4256  return InitTrampoline;4257}4258 4259static IntrinsicInst *findInitTrampolineFromBB(IntrinsicInst *AdjustTramp,4260                                               Value *TrampMem) {4261  // Visit all the previous instructions in the basic block, and try to find a4262  // init.trampoline which has a direct path to the adjust.trampoline.4263  for (BasicBlock::iterator I = AdjustTramp->getIterator(),4264                            E = AdjustTramp->getParent()->begin();4265       I != E;) {4266    Instruction *Inst = &*--I;4267    if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I))4268      if (II->getIntrinsicID() == Intrinsic::init_trampoline &&4269          II->getOperand(0) == TrampMem)4270        return II;4271    if (Inst->mayWriteToMemory())4272      return nullptr;4273  }4274  return nullptr;4275}4276 4277// Given a call to llvm.adjust.trampoline, find and return the corresponding4278// call to llvm.init.trampoline if the call to the trampoline can be optimized4279// to a direct call to a function.  Otherwise return NULL.4280static IntrinsicInst *findInitTrampoline(Value *Callee) {4281  Callee = Callee->stripPointerCasts();4282  IntrinsicInst *AdjustTramp = dyn_cast<IntrinsicInst>(Callee);4283  if (!AdjustTramp ||4284      AdjustTramp->getIntrinsicID() != Intrinsic::adjust_trampoline)4285    return nullptr;4286 4287  Value *TrampMem = AdjustTramp->getOperand(0);4288 4289  if (IntrinsicInst *IT = findInitTrampolineFromAlloca(TrampMem))4290    return IT;4291  if (IntrinsicInst *IT = findInitTrampolineFromBB(AdjustTramp, TrampMem))4292    return IT;4293  return nullptr;4294}4295 4296Instruction *InstCombinerImpl::foldPtrAuthIntrinsicCallee(CallBase &Call) {4297  const Value *Callee = Call.getCalledOperand();4298  const auto *IPC = dyn_cast<IntToPtrInst>(Callee);4299  if (!IPC || !IPC->isNoopCast(DL))4300    return nullptr;4301 4302  const auto *II = dyn_cast<IntrinsicInst>(IPC->getOperand(0));4303  if (!II)4304    return nullptr;4305 4306  Intrinsic::ID IIID = II->getIntrinsicID();4307  if (IIID != Intrinsic::ptrauth_resign && IIID != Intrinsic::ptrauth_sign)4308    return nullptr;4309 4310  // Isolate the ptrauth bundle from the others.4311  std::optional<OperandBundleUse> PtrAuthBundleOrNone;4312  SmallVector<OperandBundleDef, 2> NewBundles;4313  for (unsigned BI = 0, BE = Call.getNumOperandBundles(); BI != BE; ++BI) {4314    OperandBundleUse Bundle = Call.getOperandBundleAt(BI);4315    if (Bundle.getTagID() == LLVMContext::OB_ptrauth)4316      PtrAuthBundleOrNone = Bundle;4317    else4318      NewBundles.emplace_back(Bundle);4319  }4320 4321  if (!PtrAuthBundleOrNone)4322    return nullptr;4323 4324  Value *NewCallee = nullptr;4325  switch (IIID) {4326  // call(ptrauth.resign(p)), ["ptrauth"()] ->  call p, ["ptrauth"()]4327  // assuming the call bundle and the sign operands match.4328  case Intrinsic::ptrauth_resign: {4329    // Resign result key should match bundle.4330    if (II->getOperand(3) != PtrAuthBundleOrNone->Inputs[0])4331      return nullptr;4332    // Resign result discriminator should match bundle.4333    if (II->getOperand(4) != PtrAuthBundleOrNone->Inputs[1])4334      return nullptr;4335 4336    // Resign input (auth) key should also match: we can't change the key on4337    // the new call we're generating, because we don't know what keys are valid.4338    if (II->getOperand(1) != PtrAuthBundleOrNone->Inputs[0])4339      return nullptr;4340 4341    Value *NewBundleOps[] = {II->getOperand(1), II->getOperand(2)};4342    NewBundles.emplace_back("ptrauth", NewBundleOps);4343    NewCallee = II->getOperand(0);4344    break;4345  }4346 4347  // call(ptrauth.sign(p)), ["ptrauth"()] ->  call p4348  // assuming the call bundle and the sign operands match.4349  // Non-ptrauth indirect calls are undesirable, but so is ptrauth.sign.4350  case Intrinsic::ptrauth_sign: {4351    // Sign key should match bundle.4352    if (II->getOperand(1) != PtrAuthBundleOrNone->Inputs[0])4353      return nullptr;4354    // Sign discriminator should match bundle.4355    if (II->getOperand(2) != PtrAuthBundleOrNone->Inputs[1])4356      return nullptr;4357    NewCallee = II->getOperand(0);4358    break;4359  }4360  default:4361    llvm_unreachable("unexpected intrinsic ID");4362  }4363 4364  if (!NewCallee)4365    return nullptr;4366 4367  NewCallee = Builder.CreateBitOrPointerCast(NewCallee, Callee->getType());4368  CallBase *NewCall = CallBase::Create(&Call, NewBundles);4369  NewCall->setCalledOperand(NewCallee);4370  return NewCall;4371}4372 4373Instruction *InstCombinerImpl::foldPtrAuthConstantCallee(CallBase &Call) {4374  auto *CPA = dyn_cast<ConstantPtrAuth>(Call.getCalledOperand());4375  if (!CPA)4376    return nullptr;4377 4378  auto *CalleeF = dyn_cast<Function>(CPA->getPointer());4379  // If the ptrauth constant isn't based on a function pointer, bail out.4380  if (!CalleeF)4381    return nullptr;4382 4383  // Inspect the call ptrauth bundle to check it matches the ptrauth constant.4384  auto PAB = Call.getOperandBundle(LLVMContext::OB_ptrauth);4385  if (!PAB)4386    return nullptr;4387 4388  auto *Key = cast<ConstantInt>(PAB->Inputs[0]);4389  Value *Discriminator = PAB->Inputs[1];4390 4391  // If the bundle doesn't match, this is probably going to fail to auth.4392  if (!CPA->isKnownCompatibleWith(Key, Discriminator, DL))4393    return nullptr;4394 4395  // If the bundle matches the constant, proceed in making this a direct call.4396  auto *NewCall = CallBase::removeOperandBundle(&Call, LLVMContext::OB_ptrauth);4397  NewCall->setCalledOperand(CalleeF);4398  return NewCall;4399}4400 4401bool InstCombinerImpl::annotateAnyAllocSite(CallBase &Call,4402                                            const TargetLibraryInfo *TLI) {4403  // Note: We only handle cases which can't be driven from generic attributes4404  // here.  So, for example, nonnull and noalias (which are common properties4405  // of some allocation functions) are expected to be handled via annotation4406  // of the respective allocator declaration with generic attributes.4407  bool Changed = false;4408 4409  if (!Call.getType()->isPointerTy())4410    return Changed;4411 4412  std::optional<APInt> Size = getAllocSize(&Call, TLI);4413  if (Size && *Size != 0) {4414    // TODO: We really should just emit deref_or_null here and then4415    // let the generic inference code combine that with nonnull.4416    if (Call.hasRetAttr(Attribute::NonNull)) {4417      Changed = !Call.hasRetAttr(Attribute::Dereferenceable);4418      Call.addRetAttr(Attribute::getWithDereferenceableBytes(4419          Call.getContext(), Size->getLimitedValue()));4420    } else {4421      Changed = !Call.hasRetAttr(Attribute::DereferenceableOrNull);4422      Call.addRetAttr(Attribute::getWithDereferenceableOrNullBytes(4423          Call.getContext(), Size->getLimitedValue()));4424    }4425  }4426 4427  // Add alignment attribute if alignment is a power of two constant.4428  Value *Alignment = getAllocAlignment(&Call, TLI);4429  if (!Alignment)4430    return Changed;4431 4432  ConstantInt *AlignOpC = dyn_cast<ConstantInt>(Alignment);4433  if (AlignOpC && AlignOpC->getValue().ult(llvm::Value::MaximumAlignment)) {4434    uint64_t AlignmentVal = AlignOpC->getZExtValue();4435    if (llvm::isPowerOf2_64(AlignmentVal)) {4436      Align ExistingAlign = Call.getRetAlign().valueOrOne();4437      Align NewAlign = Align(AlignmentVal);4438      if (NewAlign > ExistingAlign) {4439        Call.addRetAttr(4440            Attribute::getWithAlignment(Call.getContext(), NewAlign));4441        Changed = true;4442      }4443    }4444  }4445  return Changed;4446}4447 4448/// Improvements for call, callbr and invoke instructions.4449Instruction *InstCombinerImpl::visitCallBase(CallBase &Call) {4450  bool Changed = annotateAnyAllocSite(Call, &TLI);4451 4452  // Mark any parameters that are known to be non-null with the nonnull4453  // attribute.  This is helpful for inlining calls to functions with null4454  // checks on their arguments.4455  SmallVector<unsigned, 4> ArgNos;4456  unsigned ArgNo = 0;4457 4458  for (Value *V : Call.args()) {4459    if (V->getType()->isPointerTy()) {4460      // Simplify the nonnull operand if the parameter is known to be nonnull.4461      // Otherwise, try to infer nonnull for it.4462      bool HasDereferenceable = Call.getParamDereferenceableBytes(ArgNo) > 0;4463      if (Call.paramHasAttr(ArgNo, Attribute::NonNull) ||4464          (HasDereferenceable &&4465           !NullPointerIsDefined(Call.getFunction(),4466                                 V->getType()->getPointerAddressSpace()))) {4467        if (Value *Res = simplifyNonNullOperand(V, HasDereferenceable)) {4468          replaceOperand(Call, ArgNo, Res);4469          Changed = true;4470        }4471      } else if (isKnownNonZero(V,4472                                getSimplifyQuery().getWithInstruction(&Call))) {4473        ArgNos.push_back(ArgNo);4474      }4475    }4476    ArgNo++;4477  }4478 4479  assert(ArgNo == Call.arg_size() && "Call arguments not processed correctly.");4480 4481  if (!ArgNos.empty()) {4482    AttributeList AS = Call.getAttributes();4483    LLVMContext &Ctx = Call.getContext();4484    AS = AS.addParamAttribute(Ctx, ArgNos,4485                              Attribute::get(Ctx, Attribute::NonNull));4486    Call.setAttributes(AS);4487    Changed = true;4488  }4489 4490  // If the callee is a pointer to a function, attempt to move any casts to the4491  // arguments of the call/callbr/invoke.4492  Value *Callee = Call.getCalledOperand();4493  Function *CalleeF = dyn_cast<Function>(Callee);4494  if ((!CalleeF || CalleeF->getFunctionType() != Call.getFunctionType()) &&4495      transformConstExprCastCall(Call))4496    return nullptr;4497 4498  if (CalleeF) {4499    // Remove the convergent attr on calls when the callee is not convergent.4500    if (Call.isConvergent() && !CalleeF->isConvergent() &&4501        !CalleeF->isIntrinsic()) {4502      LLVM_DEBUG(dbgs() << "Removing convergent attr from instr " << Call4503                        << "\n");4504      Call.setNotConvergent();4505      return &Call;4506    }4507 4508    // If the call and callee calling conventions don't match, and neither one4509    // of the calling conventions is compatible with C calling convention4510    // this call must be unreachable, as the call is undefined.4511    if ((CalleeF->getCallingConv() != Call.getCallingConv() &&4512         !(CalleeF->getCallingConv() == llvm::CallingConv::C &&4513           TargetLibraryInfoImpl::isCallingConvCCompatible(&Call)) &&4514         !(Call.getCallingConv() == llvm::CallingConv::C &&4515           TargetLibraryInfoImpl::isCallingConvCCompatible(CalleeF))) &&4516        // Only do this for calls to a function with a body.  A prototype may4517        // not actually end up matching the implementation's calling conv for a4518        // variety of reasons (e.g. it may be written in assembly).4519        !CalleeF->isDeclaration()) {4520      Instruction *OldCall = &Call;4521      CreateNonTerminatorUnreachable(OldCall);4522      // If OldCall does not return void then replaceInstUsesWith poison.4523      // This allows ValueHandlers and custom metadata to adjust itself.4524      if (!OldCall->getType()->isVoidTy())4525        replaceInstUsesWith(*OldCall, PoisonValue::get(OldCall->getType()));4526      if (isa<CallInst>(OldCall))4527        return eraseInstFromFunction(*OldCall);4528 4529      // We cannot remove an invoke or a callbr, because it would change thexi4530      // CFG, just change the callee to a null pointer.4531      cast<CallBase>(OldCall)->setCalledFunction(4532          CalleeF->getFunctionType(),4533          Constant::getNullValue(CalleeF->getType()));4534      return nullptr;4535    }4536  }4537 4538  // Calling a null function pointer is undefined if a null address isn't4539  // dereferenceable.4540  if ((isa<ConstantPointerNull>(Callee) &&4541       !NullPointerIsDefined(Call.getFunction())) ||4542      isa<UndefValue>(Callee)) {4543    // If Call does not return void then replaceInstUsesWith poison.4544    // This allows ValueHandlers and custom metadata to adjust itself.4545    if (!Call.getType()->isVoidTy())4546      replaceInstUsesWith(Call, PoisonValue::get(Call.getType()));4547 4548    if (Call.isTerminator()) {4549      // Can't remove an invoke or callbr because we cannot change the CFG.4550      return nullptr;4551    }4552 4553    // This instruction is not reachable, just remove it.4554    CreateNonTerminatorUnreachable(&Call);4555    return eraseInstFromFunction(Call);4556  }4557 4558  if (IntrinsicInst *II = findInitTrampoline(Callee))4559    return transformCallThroughTrampoline(Call, *II);4560 4561  // Combine calls involving pointer authentication intrinsics.4562  if (Instruction *NewCall = foldPtrAuthIntrinsicCallee(Call))4563    return NewCall;4564 4565  // Combine calls to ptrauth constants.4566  if (Instruction *NewCall = foldPtrAuthConstantCallee(Call))4567    return NewCall;4568 4569  if (isa<InlineAsm>(Callee) && !Call.doesNotThrow()) {4570    InlineAsm *IA = cast<InlineAsm>(Callee);4571    if (!IA->canThrow()) {4572      // Normal inline asm calls cannot throw - mark them4573      // 'nounwind'.4574      Call.setDoesNotThrow();4575      Changed = true;4576    }4577  }4578 4579  // Try to optimize the call if possible, we require DataLayout for most of4580  // this.  None of these calls are seen as possibly dead so go ahead and4581  // delete the instruction now.4582  if (CallInst *CI = dyn_cast<CallInst>(&Call)) {4583    Instruction *I = tryOptimizeCall(CI);4584    // If we changed something return the result, etc. Otherwise let4585    // the fallthrough check.4586    if (I) return eraseInstFromFunction(*I);4587  }4588 4589  if (!Call.use_empty() && !Call.isMustTailCall())4590    if (Value *ReturnedArg = Call.getReturnedArgOperand()) {4591      Type *CallTy = Call.getType();4592      Type *RetArgTy = ReturnedArg->getType();4593      if (RetArgTy->canLosslesslyBitCastTo(CallTy))4594        return replaceInstUsesWith(4595            Call, Builder.CreateBitOrPointerCast(ReturnedArg, CallTy));4596    }4597 4598  // Drop unnecessary callee_type metadata from calls that were converted4599  // into direct calls.4600  if (Call.getMetadata(LLVMContext::MD_callee_type) && !Call.isIndirectCall()) {4601    Call.setMetadata(LLVMContext::MD_callee_type, nullptr);4602    Changed = true;4603  }4604 4605  // Drop unnecessary kcfi operand bundles from calls that were converted4606  // into direct calls.4607  auto Bundle = Call.getOperandBundle(LLVMContext::OB_kcfi);4608  if (Bundle && !Call.isIndirectCall()) {4609    DEBUG_WITH_TYPE(DEBUG_TYPE "-kcfi", {4610      if (CalleeF) {4611        ConstantInt *FunctionType = nullptr;4612        ConstantInt *ExpectedType = cast<ConstantInt>(Bundle->Inputs[0]);4613 4614        if (MDNode *MD = CalleeF->getMetadata(LLVMContext::MD_kcfi_type))4615          FunctionType = mdconst::extract<ConstantInt>(MD->getOperand(0));4616 4617        if (FunctionType &&4618            FunctionType->getZExtValue() != ExpectedType->getZExtValue())4619          dbgs() << Call.getModule()->getName()4620                 << ": warning: kcfi: " << Call.getCaller()->getName()4621                 << ": call to " << CalleeF->getName()4622                 << " using a mismatching function pointer type\n";4623      }4624    });4625 4626    return CallBase::removeOperandBundle(&Call, LLVMContext::OB_kcfi);4627  }4628 4629  if (isRemovableAlloc(&Call, &TLI))4630    return visitAllocSite(Call);4631 4632  // Handle intrinsics which can be used in both call and invoke context.4633  switch (Call.getIntrinsicID()) {4634  case Intrinsic::experimental_gc_statepoint: {4635    GCStatepointInst &GCSP = *cast<GCStatepointInst>(&Call);4636    SmallPtrSet<Value *, 32> LiveGcValues;4637    for (const GCRelocateInst *Reloc : GCSP.getGCRelocates()) {4638      GCRelocateInst &GCR = *const_cast<GCRelocateInst *>(Reloc);4639 4640      // Remove the relocation if unused.4641      if (GCR.use_empty()) {4642        eraseInstFromFunction(GCR);4643        continue;4644      }4645 4646      Value *DerivedPtr = GCR.getDerivedPtr();4647      Value *BasePtr = GCR.getBasePtr();4648 4649      // Undef is undef, even after relocation.4650      if (isa<UndefValue>(DerivedPtr) || isa<UndefValue>(BasePtr)) {4651        replaceInstUsesWith(GCR, UndefValue::get(GCR.getType()));4652        eraseInstFromFunction(GCR);4653        continue;4654      }4655 4656      if (auto *PT = dyn_cast<PointerType>(GCR.getType())) {4657        // The relocation of null will be null for most any collector.4658        // TODO: provide a hook for this in GCStrategy.  There might be some4659        // weird collector this property does not hold for.4660        if (isa<ConstantPointerNull>(DerivedPtr)) {4661          // Use null-pointer of gc_relocate's type to replace it.4662          replaceInstUsesWith(GCR, ConstantPointerNull::get(PT));4663          eraseInstFromFunction(GCR);4664          continue;4665        }4666 4667        // isKnownNonNull -> nonnull attribute4668        if (!GCR.hasRetAttr(Attribute::NonNull) &&4669            isKnownNonZero(DerivedPtr,4670                           getSimplifyQuery().getWithInstruction(&Call))) {4671          GCR.addRetAttr(Attribute::NonNull);4672          // We discovered new fact, re-check users.4673          Worklist.pushUsersToWorkList(GCR);4674        }4675      }4676 4677      // If we have two copies of the same pointer in the statepoint argument4678      // list, canonicalize to one.  This may let us common gc.relocates.4679      if (GCR.getBasePtr() == GCR.getDerivedPtr() &&4680          GCR.getBasePtrIndex() != GCR.getDerivedPtrIndex()) {4681        auto *OpIntTy = GCR.getOperand(2)->getType();4682        GCR.setOperand(2, ConstantInt::get(OpIntTy, GCR.getBasePtrIndex()));4683      }4684 4685      // TODO: bitcast(relocate(p)) -> relocate(bitcast(p))4686      // Canonicalize on the type from the uses to the defs4687 4688      // TODO: relocate((gep p, C, C2, ...)) -> gep(relocate(p), C, C2, ...)4689      LiveGcValues.insert(BasePtr);4690      LiveGcValues.insert(DerivedPtr);4691    }4692    std::optional<OperandBundleUse> Bundle =4693        GCSP.getOperandBundle(LLVMContext::OB_gc_live);4694    unsigned NumOfGCLives = LiveGcValues.size();4695    if (!Bundle || NumOfGCLives == Bundle->Inputs.size())4696      break;4697    // We can reduce the size of gc live bundle.4698    DenseMap<Value *, unsigned> Val2Idx;4699    std::vector<Value *> NewLiveGc;4700    for (Value *V : Bundle->Inputs) {4701      auto [It, Inserted] = Val2Idx.try_emplace(V);4702      if (!Inserted)4703        continue;4704      if (LiveGcValues.count(V)) {4705        It->second = NewLiveGc.size();4706        NewLiveGc.push_back(V);4707      } else4708        It->second = NumOfGCLives;4709    }4710    // Update all gc.relocates4711    for (const GCRelocateInst *Reloc : GCSP.getGCRelocates()) {4712      GCRelocateInst &GCR = *const_cast<GCRelocateInst *>(Reloc);4713      Value *BasePtr = GCR.getBasePtr();4714      assert(Val2Idx.count(BasePtr) && Val2Idx[BasePtr] != NumOfGCLives &&4715             "Missed live gc for base pointer");4716      auto *OpIntTy1 = GCR.getOperand(1)->getType();4717      GCR.setOperand(1, ConstantInt::get(OpIntTy1, Val2Idx[BasePtr]));4718      Value *DerivedPtr = GCR.getDerivedPtr();4719      assert(Val2Idx.count(DerivedPtr) && Val2Idx[DerivedPtr] != NumOfGCLives &&4720             "Missed live gc for derived pointer");4721      auto *OpIntTy2 = GCR.getOperand(2)->getType();4722      GCR.setOperand(2, ConstantInt::get(OpIntTy2, Val2Idx[DerivedPtr]));4723    }4724    // Create new statepoint instruction.4725    OperandBundleDef NewBundle("gc-live", NewLiveGc);4726    return CallBase::Create(&Call, NewBundle);4727  }4728  default: { break; }4729  }4730 4731  return Changed ? &Call : nullptr;4732}4733 4734/// If the callee is a constexpr cast of a function, attempt to move the cast to4735/// the arguments of the call/invoke.4736/// CallBrInst is not supported.4737bool InstCombinerImpl::transformConstExprCastCall(CallBase &Call) {4738  auto *Callee =4739      dyn_cast<Function>(Call.getCalledOperand()->stripPointerCasts());4740  if (!Callee)4741    return false;4742 4743  assert(!isa<CallBrInst>(Call) &&4744         "CallBr's don't have a single point after a def to insert at");4745 4746  // Don't perform the transform for declarations, which may not be fully4747  // accurate. For example, void @foo() is commonly used as a placeholder for4748  // unknown prototypes.4749  if (Callee->isDeclaration())4750    return false;4751 4752  // If this is a call to a thunk function, don't remove the cast. Thunks are4753  // used to transparently forward all incoming parameters and outgoing return4754  // values, so it's important to leave the cast in place.4755  if (Callee->hasFnAttribute("thunk"))4756    return false;4757 4758  // If this is a call to a naked function, the assembly might be4759  // using an argument, or otherwise rely on the frame layout,4760  // the function prototype will mismatch.4761  if (Callee->hasFnAttribute(Attribute::Naked))4762    return false;4763 4764  // If this is a musttail call, the callee's prototype must match the caller's4765  // prototype with the exception of pointee types. The code below doesn't4766  // implement that, so we can't do this transform.4767  // TODO: Do the transform if it only requires adding pointer casts.4768  if (Call.isMustTailCall())4769    return false;4770 4771  Instruction *Caller = &Call;4772  const AttributeList &CallerPAL = Call.getAttributes();4773 4774  // Okay, this is a cast from a function to a different type.  Unless doing so4775  // would cause a type conversion of one of our arguments, change this call to4776  // be a direct call with arguments casted to the appropriate types.4777  FunctionType *FT = Callee->getFunctionType();4778  Type *OldRetTy = Caller->getType();4779  Type *NewRetTy = FT->getReturnType();4780 4781  // Check to see if we are changing the return type...4782  if (OldRetTy != NewRetTy) {4783 4784    if (NewRetTy->isStructTy())4785      return false; // TODO: Handle multiple return values.4786 4787    if (!CastInst::isBitOrNoopPointerCastable(NewRetTy, OldRetTy, DL)) {4788      if (!Caller->use_empty())4789        return false;   // Cannot transform this return value.4790    }4791 4792    if (!CallerPAL.isEmpty() && !Caller->use_empty()) {4793      AttrBuilder RAttrs(FT->getContext(), CallerPAL.getRetAttrs());4794      if (RAttrs.overlaps(AttributeFuncs::typeIncompatible(4795              NewRetTy, CallerPAL.getRetAttrs())))4796        return false;   // Attribute not compatible with transformed value.4797    }4798 4799    // If the callbase is an invoke instruction, and the return value is4800    // used by a PHI node in a successor, we cannot change the return type of4801    // the call because there is no place to put the cast instruction (without4802    // breaking the critical edge).  Bail out in this case.4803    if (!Caller->use_empty()) {4804      BasicBlock *PhisNotSupportedBlock = nullptr;4805      if (auto *II = dyn_cast<InvokeInst>(Caller))4806        PhisNotSupportedBlock = II->getNormalDest();4807      if (PhisNotSupportedBlock)4808        for (User *U : Caller->users())4809          if (PHINode *PN = dyn_cast<PHINode>(U))4810            if (PN->getParent() == PhisNotSupportedBlock)4811              return false;4812    }4813  }4814 4815  unsigned NumActualArgs = Call.arg_size();4816  unsigned NumCommonArgs = std::min(FT->getNumParams(), NumActualArgs);4817 4818  // Prevent us turning:4819  // declare void @takes_i32_inalloca(i32* inalloca)4820  //  call void bitcast (void (i32*)* @takes_i32_inalloca to void (i32)*)(i32 0)4821  //4822  // into:4823  //  call void @takes_i32_inalloca(i32* null)4824  //4825  //  Similarly, avoid folding away bitcasts of byval calls.4826  if (Callee->getAttributes().hasAttrSomewhere(Attribute::InAlloca) ||4827      Callee->getAttributes().hasAttrSomewhere(Attribute::Preallocated))4828    return false;4829 4830  auto AI = Call.arg_begin();4831  for (unsigned i = 0, e = NumCommonArgs; i != e; ++i, ++AI) {4832    Type *ParamTy = FT->getParamType(i);4833    Type *ActTy = (*AI)->getType();4834 4835    if (!CastInst::isBitOrNoopPointerCastable(ActTy, ParamTy, DL))4836      return false;   // Cannot transform this parameter value.4837 4838    // Check if there are any incompatible attributes we cannot drop safely.4839    if (AttrBuilder(FT->getContext(), CallerPAL.getParamAttrs(i))4840            .overlaps(AttributeFuncs::typeIncompatible(4841                ParamTy, CallerPAL.getParamAttrs(i),4842                AttributeFuncs::ASK_UNSAFE_TO_DROP)))4843      return false;   // Attribute not compatible with transformed value.4844 4845    if (Call.isInAllocaArgument(i) ||4846        CallerPAL.hasParamAttr(i, Attribute::Preallocated))4847      return false; // Cannot transform to and from inalloca/preallocated.4848 4849    if (CallerPAL.hasParamAttr(i, Attribute::SwiftError))4850      return false;4851 4852    if (CallerPAL.hasParamAttr(i, Attribute::ByVal) !=4853        Callee->getAttributes().hasParamAttr(i, Attribute::ByVal))4854      return false; // Cannot transform to or from byval.4855  }4856 4857  if (FT->getNumParams() < NumActualArgs && FT->isVarArg() &&4858      !CallerPAL.isEmpty()) {4859    // In this case we have more arguments than the new function type, but we4860    // won't be dropping them.  Check that these extra arguments have attributes4861    // that are compatible with being a vararg call argument.4862    unsigned SRetIdx;4863    if (CallerPAL.hasAttrSomewhere(Attribute::StructRet, &SRetIdx) &&4864        SRetIdx - AttributeList::FirstArgIndex >= FT->getNumParams())4865      return false;4866  }4867 4868  // Okay, we decided that this is a safe thing to do: go ahead and start4869  // inserting cast instructions as necessary.4870  SmallVector<Value *, 8> Args;4871  SmallVector<AttributeSet, 8> ArgAttrs;4872  Args.reserve(NumActualArgs);4873  ArgAttrs.reserve(NumActualArgs);4874 4875  // Get any return attributes.4876  AttrBuilder RAttrs(FT->getContext(), CallerPAL.getRetAttrs());4877 4878  // If the return value is not being used, the type may not be compatible4879  // with the existing attributes.  Wipe out any problematic attributes.4880  RAttrs.remove(4881      AttributeFuncs::typeIncompatible(NewRetTy, CallerPAL.getRetAttrs()));4882 4883  LLVMContext &Ctx = Call.getContext();4884  AI = Call.arg_begin();4885  for (unsigned i = 0; i != NumCommonArgs; ++i, ++AI) {4886    Type *ParamTy = FT->getParamType(i);4887 4888    Value *NewArg = *AI;4889    if ((*AI)->getType() != ParamTy)4890      NewArg = Builder.CreateBitOrPointerCast(*AI, ParamTy);4891    Args.push_back(NewArg);4892 4893    // Add any parameter attributes except the ones incompatible with the new4894    // type. Note that we made sure all incompatible ones are safe to drop.4895    AttributeMask IncompatibleAttrs = AttributeFuncs::typeIncompatible(4896        ParamTy, CallerPAL.getParamAttrs(i), AttributeFuncs::ASK_SAFE_TO_DROP);4897    ArgAttrs.push_back(4898        CallerPAL.getParamAttrs(i).removeAttributes(Ctx, IncompatibleAttrs));4899  }4900 4901  // If the function takes more arguments than the call was taking, add them4902  // now.4903  for (unsigned i = NumCommonArgs; i != FT->getNumParams(); ++i) {4904    Args.push_back(Constant::getNullValue(FT->getParamType(i)));4905    ArgAttrs.push_back(AttributeSet());4906  }4907 4908  // If we are removing arguments to the function, emit an obnoxious warning.4909  if (FT->getNumParams() < NumActualArgs) {4910    // TODO: if (!FT->isVarArg()) this call may be unreachable. PR147224911    if (FT->isVarArg()) {4912      // Add all of the arguments in their promoted form to the arg list.4913      for (unsigned i = FT->getNumParams(); i != NumActualArgs; ++i, ++AI) {4914        Type *PTy = getPromotedType((*AI)->getType());4915        Value *NewArg = *AI;4916        if (PTy != (*AI)->getType()) {4917          // Must promote to pass through va_arg area!4918          Instruction::CastOps opcode =4919            CastInst::getCastOpcode(*AI, false, PTy, false);4920          NewArg = Builder.CreateCast(opcode, *AI, PTy);4921        }4922        Args.push_back(NewArg);4923 4924        // Add any parameter attributes.4925        ArgAttrs.push_back(CallerPAL.getParamAttrs(i));4926      }4927    }4928  }4929 4930  AttributeSet FnAttrs = CallerPAL.getFnAttrs();4931 4932  if (NewRetTy->isVoidTy())4933    Caller->setName("");   // Void type should not have a name.4934 4935  assert((ArgAttrs.size() == FT->getNumParams() || FT->isVarArg()) &&4936         "missing argument attributes");4937  AttributeList NewCallerPAL = AttributeList::get(4938      Ctx, FnAttrs, AttributeSet::get(Ctx, RAttrs), ArgAttrs);4939 4940  SmallVector<OperandBundleDef, 1> OpBundles;4941  Call.getOperandBundlesAsDefs(OpBundles);4942 4943  CallBase *NewCall;4944  if (InvokeInst *II = dyn_cast<InvokeInst>(Caller)) {4945    NewCall = Builder.CreateInvoke(Callee, II->getNormalDest(),4946                                   II->getUnwindDest(), Args, OpBundles);4947  } else {4948    NewCall = Builder.CreateCall(Callee, Args, OpBundles);4949    cast<CallInst>(NewCall)->setTailCallKind(4950        cast<CallInst>(Caller)->getTailCallKind());4951  }4952  NewCall->takeName(Caller);4953  NewCall->setCallingConv(Call.getCallingConv());4954  NewCall->setAttributes(NewCallerPAL);4955 4956  // Preserve prof metadata if any.4957  NewCall->copyMetadata(*Caller, {LLVMContext::MD_prof});4958 4959  // Insert a cast of the return type as necessary.4960  Instruction *NC = NewCall;4961  Value *NV = NC;4962  if (OldRetTy != NV->getType() && !Caller->use_empty()) {4963    assert(!NV->getType()->isVoidTy());4964    NV = NC = CastInst::CreateBitOrPointerCast(NC, OldRetTy);4965    NC->setDebugLoc(Caller->getDebugLoc());4966 4967    auto OptInsertPt = NewCall->getInsertionPointAfterDef();4968    assert(OptInsertPt && "No place to insert cast");4969    InsertNewInstBefore(NC, *OptInsertPt);4970    Worklist.pushUsersToWorkList(*Caller);4971  }4972 4973  if (!Caller->use_empty())4974    replaceInstUsesWith(*Caller, NV);4975  else if (Caller->hasValueHandle()) {4976    if (OldRetTy == NV->getType())4977      ValueHandleBase::ValueIsRAUWd(Caller, NV);4978    else4979      // We cannot call ValueIsRAUWd with a different type, and the4980      // actual tracked value will disappear.4981      ValueHandleBase::ValueIsDeleted(Caller);4982  }4983 4984  eraseInstFromFunction(*Caller);4985  return true;4986}4987 4988/// Turn a call to a function created by init_trampoline / adjust_trampoline4989/// intrinsic pair into a direct call to the underlying function.4990Instruction *4991InstCombinerImpl::transformCallThroughTrampoline(CallBase &Call,4992                                                 IntrinsicInst &Tramp) {4993  FunctionType *FTy = Call.getFunctionType();4994  AttributeList Attrs = Call.getAttributes();4995 4996  // If the call already has the 'nest' attribute somewhere then give up -4997  // otherwise 'nest' would occur twice after splicing in the chain.4998  if (Attrs.hasAttrSomewhere(Attribute::Nest))4999    return nullptr;5000 5001  Function *NestF = cast<Function>(Tramp.getArgOperand(1)->stripPointerCasts());5002  FunctionType *NestFTy = NestF->getFunctionType();5003 5004  AttributeList NestAttrs = NestF->getAttributes();5005  if (!NestAttrs.isEmpty()) {5006    unsigned NestArgNo = 0;5007    Type *NestTy = nullptr;5008    AttributeSet NestAttr;5009 5010    // Look for a parameter marked with the 'nest' attribute.5011    for (FunctionType::param_iterator I = NestFTy->param_begin(),5012                                      E = NestFTy->param_end();5013         I != E; ++NestArgNo, ++I) {5014      AttributeSet AS = NestAttrs.getParamAttrs(NestArgNo);5015      if (AS.hasAttribute(Attribute::Nest)) {5016        // Record the parameter type and any other attributes.5017        NestTy = *I;5018        NestAttr = AS;5019        break;5020      }5021    }5022 5023    if (NestTy) {5024      std::vector<Value*> NewArgs;5025      std::vector<AttributeSet> NewArgAttrs;5026      NewArgs.reserve(Call.arg_size() + 1);5027      NewArgAttrs.reserve(Call.arg_size());5028 5029      // Insert the nest argument into the call argument list, which may5030      // mean appending it.  Likewise for attributes.5031 5032      {5033        unsigned ArgNo = 0;5034        auto I = Call.arg_begin(), E = Call.arg_end();5035        do {5036          if (ArgNo == NestArgNo) {5037            // Add the chain argument and attributes.5038            Value *NestVal = Tramp.getArgOperand(2);5039            if (NestVal->getType() != NestTy)5040              NestVal = Builder.CreateBitCast(NestVal, NestTy, "nest");5041            NewArgs.push_back(NestVal);5042            NewArgAttrs.push_back(NestAttr);5043          }5044 5045          if (I == E)5046            break;5047 5048          // Add the original argument and attributes.5049          NewArgs.push_back(*I);5050          NewArgAttrs.push_back(Attrs.getParamAttrs(ArgNo));5051 5052          ++ArgNo;5053          ++I;5054        } while (true);5055      }5056 5057      // The trampoline may have been bitcast to a bogus type (FTy).5058      // Handle this by synthesizing a new function type, equal to FTy5059      // with the chain parameter inserted.5060 5061      std::vector<Type*> NewTypes;5062      NewTypes.reserve(FTy->getNumParams()+1);5063 5064      // Insert the chain's type into the list of parameter types, which may5065      // mean appending it.5066      {5067        unsigned ArgNo = 0;5068        FunctionType::param_iterator I = FTy->param_begin(),5069          E = FTy->param_end();5070 5071        do {5072          if (ArgNo == NestArgNo)5073            // Add the chain's type.5074            NewTypes.push_back(NestTy);5075 5076          if (I == E)5077            break;5078 5079          // Add the original type.5080          NewTypes.push_back(*I);5081 5082          ++ArgNo;5083          ++I;5084        } while (true);5085      }5086 5087      // Replace the trampoline call with a direct call.  Let the generic5088      // code sort out any function type mismatches.5089      FunctionType *NewFTy =5090          FunctionType::get(FTy->getReturnType(), NewTypes, FTy->isVarArg());5091      AttributeList NewPAL =5092          AttributeList::get(FTy->getContext(), Attrs.getFnAttrs(),5093                             Attrs.getRetAttrs(), NewArgAttrs);5094 5095      SmallVector<OperandBundleDef, 1> OpBundles;5096      Call.getOperandBundlesAsDefs(OpBundles);5097 5098      Instruction *NewCaller;5099      if (InvokeInst *II = dyn_cast<InvokeInst>(&Call)) {5100        NewCaller = InvokeInst::Create(NewFTy, NestF, II->getNormalDest(),5101                                       II->getUnwindDest(), NewArgs, OpBundles);5102        cast<InvokeInst>(NewCaller)->setCallingConv(II->getCallingConv());5103        cast<InvokeInst>(NewCaller)->setAttributes(NewPAL);5104      } else if (CallBrInst *CBI = dyn_cast<CallBrInst>(&Call)) {5105        NewCaller =5106            CallBrInst::Create(NewFTy, NestF, CBI->getDefaultDest(),5107                               CBI->getIndirectDests(), NewArgs, OpBundles);5108        cast<CallBrInst>(NewCaller)->setCallingConv(CBI->getCallingConv());5109        cast<CallBrInst>(NewCaller)->setAttributes(NewPAL);5110      } else {5111        NewCaller = CallInst::Create(NewFTy, NestF, NewArgs, OpBundles);5112        cast<CallInst>(NewCaller)->setTailCallKind(5113            cast<CallInst>(Call).getTailCallKind());5114        cast<CallInst>(NewCaller)->setCallingConv(5115            cast<CallInst>(Call).getCallingConv());5116        cast<CallInst>(NewCaller)->setAttributes(NewPAL);5117      }5118      NewCaller->setDebugLoc(Call.getDebugLoc());5119 5120      return NewCaller;5121    }5122  }5123 5124  // Replace the trampoline call with a direct call.  Since there is no 'nest'5125  // parameter, there is no need to adjust the argument list.  Let the generic5126  // code sort out any function type mismatches.5127  Call.setCalledFunction(FTy, NestF);5128  return &Call;5129}5130