brintos / llvm-project-archived public Read only

1//===--- CGExprCXX.cpp - Emit LLVM Code for C++ expressions ---------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This contains code dealing with code generation of C++ expressions10//11//===----------------------------------------------------------------------===//12 13#include "CGCUDARuntime.h"14#include "CGCXXABI.h"15#include "CGDebugInfo.h"16#include "CGObjCRuntime.h"17#include "CodeGenFunction.h"18#include "ConstantEmitter.h"19#include "TargetInfo.h"20#include "clang/Basic/CodeGenOptions.h"21#include "clang/CodeGen/CGFunctionInfo.h"22#include "llvm/IR/Intrinsics.h"23 24using namespace clang;25using namespace CodeGen;26 27namespace {28struct MemberCallInfo {29  RequiredArgs ReqArgs;30  // Number of prefix arguments for the call. Ignores the `this` pointer.31  unsigned PrefixSize;32};33}34 35static MemberCallInfo36commonEmitCXXMemberOrOperatorCall(CodeGenFunction &CGF, GlobalDecl GD,37                                  llvm::Value *This, llvm::Value *ImplicitParam,38                                  QualType ImplicitParamTy, const CallExpr *CE,39                                  CallArgList &Args, CallArgList *RtlArgs) {40  auto *MD = cast<CXXMethodDecl>(GD.getDecl());41 42  assert(CE == nullptr || isa<CXXMemberCallExpr>(CE) ||43         isa<CXXOperatorCallExpr>(CE));44  assert(MD->isImplicitObjectMemberFunction() &&45         "Trying to emit a member or operator call expr on a static method!");46 47  // Push the this ptr.48  const CXXRecordDecl *RD =49      CGF.CGM.getCXXABI().getThisArgumentTypeForMethod(GD);50  Args.add(RValue::get(This), CGF.getTypes().DeriveThisType(RD, MD));51 52  // If there is an implicit parameter (e.g. VTT), emit it.53  if (ImplicitParam) {54    Args.add(RValue::get(ImplicitParam), ImplicitParamTy);55  }56 57  const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();58  RequiredArgs required = RequiredArgs::forPrototypePlus(FPT, Args.size());59  unsigned PrefixSize = Args.size() - 1;60 61  // And the rest of the call args.62  if (RtlArgs) {63    // Special case: if the caller emitted the arguments right-to-left already64    // (prior to emitting the *this argument), we're done. This happens for65    // assignment operators.66    Args.addFrom(*RtlArgs);67  } else if (CE) {68    // Special case: skip first argument of CXXOperatorCall (it is "this").69    unsigned ArgsToSkip = 0;70    if (const auto *Op = dyn_cast<CXXOperatorCallExpr>(CE)) {71      if (const auto *M = dyn_cast<CXXMethodDecl>(Op->getCalleeDecl()))72        ArgsToSkip =73            static_cast<unsigned>(!M->isExplicitObjectMemberFunction());74    }75    CGF.EmitCallArgs(Args, FPT, drop_begin(CE->arguments(), ArgsToSkip),76                     CE->getDirectCallee());77  } else {78    assert(79        FPT->getNumParams() == 0 &&80        "No CallExpr specified for function with non-zero number of arguments");81  }82  return {required, PrefixSize};83}84 85RValue CodeGenFunction::EmitCXXMemberOrOperatorCall(86    const CXXMethodDecl *MD, const CGCallee &Callee,87    ReturnValueSlot ReturnValue, llvm::Value *This, llvm::Value *ImplicitParam,88    QualType ImplicitParamTy, const CallExpr *CE, CallArgList *RtlArgs,89    llvm::CallBase **CallOrInvoke) {90  const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();91  CallArgList Args;92  MemberCallInfo CallInfo = commonEmitCXXMemberOrOperatorCall(93      *this, MD, This, ImplicitParam, ImplicitParamTy, CE, Args, RtlArgs);94  auto &FnInfo = CGM.getTypes().arrangeCXXMethodCall(95      Args, FPT, CallInfo.ReqArgs, CallInfo.PrefixSize);96  return EmitCall(FnInfo, Callee, ReturnValue, Args, CallOrInvoke,97                  CE && CE == MustTailCall,98                  CE ? CE->getExprLoc() : SourceLocation());99}100 101RValue CodeGenFunction::EmitCXXDestructorCall(102    GlobalDecl Dtor, const CGCallee &Callee, llvm::Value *This, QualType ThisTy,103    llvm::Value *ImplicitParam, QualType ImplicitParamTy, const CallExpr *CE,104    llvm::CallBase **CallOrInvoke) {105  const CXXMethodDecl *DtorDecl = cast<CXXMethodDecl>(Dtor.getDecl());106 107  assert(!ThisTy.isNull());108  assert(ThisTy->getAsCXXRecordDecl() == DtorDecl->getParent() &&109         "Pointer/Object mixup");110 111  LangAS SrcAS = ThisTy.getAddressSpace();112  LangAS DstAS = DtorDecl->getMethodQualifiers().getAddressSpace();113  if (SrcAS != DstAS) {114    QualType DstTy = DtorDecl->getThisType();115    llvm::Type *NewType = CGM.getTypes().ConvertType(DstTy);116    This = getTargetHooks().performAddrSpaceCast(*this, This, SrcAS, NewType);117  }118 119  CallArgList Args;120  commonEmitCXXMemberOrOperatorCall(*this, Dtor, This, ImplicitParam,121                                    ImplicitParamTy, CE, Args, nullptr);122  return EmitCall(CGM.getTypes().arrangeCXXStructorDeclaration(Dtor), Callee,123                  ReturnValueSlot(), Args, CallOrInvoke,124                  CE && CE == MustTailCall,125                  CE ? CE->getExprLoc() : SourceLocation{});126}127 128RValue CodeGenFunction::EmitCXXPseudoDestructorExpr(129                                            const CXXPseudoDestructorExpr *E) {130  QualType DestroyedType = E->getDestroyedType();131  if (DestroyedType.hasStrongOrWeakObjCLifetime()) {132    // Automatic Reference Counting:133    //   If the pseudo-expression names a retainable object with weak or134    //   strong lifetime, the object shall be released.135    Expr *BaseExpr = E->getBase();136    Address BaseValue = Address::invalid();137    Qualifiers BaseQuals;138 139    // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a scalar.140    if (E->isArrow()) {141      BaseValue = EmitPointerWithAlignment(BaseExpr);142      const auto *PTy = BaseExpr->getType()->castAs<PointerType>();143      BaseQuals = PTy->getPointeeType().getQualifiers();144    } else {145      LValue BaseLV = EmitLValue(BaseExpr);146      BaseValue = BaseLV.getAddress();147      QualType BaseTy = BaseExpr->getType();148      BaseQuals = BaseTy.getQualifiers();149    }150 151    switch (DestroyedType.getObjCLifetime()) {152    case Qualifiers::OCL_None:153    case Qualifiers::OCL_ExplicitNone:154    case Qualifiers::OCL_Autoreleasing:155      break;156 157    case Qualifiers::OCL_Strong:158      EmitARCRelease(Builder.CreateLoad(BaseValue,159                        DestroyedType.isVolatileQualified()),160                     ARCPreciseLifetime);161      break;162 163    case Qualifiers::OCL_Weak:164      EmitARCDestroyWeak(BaseValue);165      break;166    }167  } else {168    // C++ [expr.pseudo]p1:169    //   The result shall only be used as the operand for the function call170    //   operator (), and the result of such a call has type void. The only171    //   effect is the evaluation of the postfix-expression before the dot or172    //   arrow.173    EmitIgnoredExpr(E->getBase());174  }175 176  return RValue::get(nullptr);177}178 179static CXXRecordDecl *getCXXRecord(const Expr *E) {180  QualType T = E->getType();181  if (const PointerType *PTy = T->getAs<PointerType>())182    T = PTy->getPointeeType();183  return T->castAsCXXRecordDecl();184}185 186// Note: This function also emit constructor calls to support a MSVC187// extensions allowing explicit constructor function call.188RValue CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE,189                                              ReturnValueSlot ReturnValue,190                                              llvm::CallBase **CallOrInvoke) {191  const Expr *callee = CE->getCallee()->IgnoreParens();192 193  if (isa<BinaryOperator>(callee))194    return EmitCXXMemberPointerCallExpr(CE, ReturnValue, CallOrInvoke);195 196  const MemberExpr *ME = cast<MemberExpr>(callee);197  const CXXMethodDecl *MD = cast<CXXMethodDecl>(ME->getMemberDecl());198 199  if (MD->isStatic()) {200    // The method is static, emit it as we would a regular call.201    CGCallee callee =202        CGCallee::forDirect(CGM.GetAddrOfFunction(MD), GlobalDecl(MD));203    return EmitCall(getContext().getPointerType(MD->getType()), callee, CE,204                    ReturnValue, /*Chain=*/nullptr, CallOrInvoke);205  }206 207  bool HasQualifier = ME->hasQualifier();208  NestedNameSpecifier Qualifier = ME->getQualifier();209  bool IsArrow = ME->isArrow();210  const Expr *Base = ME->getBase();211 212  return EmitCXXMemberOrOperatorMemberCallExpr(CE, MD, ReturnValue,213                                               HasQualifier, Qualifier, IsArrow,214                                               Base, CallOrInvoke);215}216 217RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr(218    const CallExpr *CE, const CXXMethodDecl *MD, ReturnValueSlot ReturnValue,219    bool HasQualifier, NestedNameSpecifier Qualifier, bool IsArrow,220    const Expr *Base, llvm::CallBase **CallOrInvoke) {221  assert(isa<CXXMemberCallExpr>(CE) || isa<CXXOperatorCallExpr>(CE));222 223  // Compute the object pointer.224  bool CanUseVirtualCall = MD->isVirtual() && !HasQualifier;225 226  const CXXMethodDecl *DevirtualizedMethod = nullptr;227  if (CanUseVirtualCall &&228      MD->getDevirtualizedMethod(Base, getLangOpts().AppleKext)) {229    const CXXRecordDecl *BestDynamicDecl = Base->getBestDynamicClassType();230    DevirtualizedMethod = MD->getCorrespondingMethodInClass(BestDynamicDecl);231    assert(DevirtualizedMethod);232    const CXXRecordDecl *DevirtualizedClass = DevirtualizedMethod->getParent();233    const Expr *Inner = Base->IgnoreParenBaseCasts();234    if (DevirtualizedMethod->getReturnType().getCanonicalType() !=235        MD->getReturnType().getCanonicalType())236      // If the return types are not the same, this might be a case where more237      // code needs to run to compensate for it. For example, the derived238      // method might return a type that inherits form from the return239      // type of MD and has a prefix.240      // For now we just avoid devirtualizing these covariant cases.241      DevirtualizedMethod = nullptr;242    else if (getCXXRecord(Inner) == DevirtualizedClass)243      // If the class of the Inner expression is where the dynamic method244      // is defined, build the this pointer from it.245      Base = Inner;246    else if (getCXXRecord(Base) != DevirtualizedClass) {247      // If the method is defined in a class that is not the best dynamic248      // one or the one of the full expression, we would have to build249      // a derived-to-base cast to compute the correct this pointer, but250      // we don't have support for that yet, so do a virtual call.251      DevirtualizedMethod = nullptr;252    }253  }254 255  bool TrivialForCodegen =256      MD->isTrivial() || (MD->isDefaulted() && MD->getParent()->isUnion());257  bool TrivialAssignment =258      TrivialForCodegen &&259      (MD->isCopyAssignmentOperator() || MD->isMoveAssignmentOperator()) &&260      !MD->getParent()->mayInsertExtraPadding();261 262  // C++17 demands that we evaluate the RHS of a (possibly-compound) assignment263  // operator before the LHS.264  CallArgList RtlArgStorage;265  CallArgList *RtlArgs = nullptr;266  LValue TrivialAssignmentRHS;267  if (auto *OCE = dyn_cast<CXXOperatorCallExpr>(CE)) {268    if (OCE->isAssignmentOp()) {269      if (TrivialAssignment) {270        TrivialAssignmentRHS = EmitLValue(CE->getArg(1));271      } else {272        RtlArgs = &RtlArgStorage;273        EmitCallArgs(*RtlArgs, MD->getType()->castAs<FunctionProtoType>(),274                     drop_begin(CE->arguments(), 1), CE->getDirectCallee(),275                     /*ParamsToSkip*/0, EvaluationOrder::ForceRightToLeft);276      }277    }278  }279 280  LValue This;281  if (IsArrow) {282    LValueBaseInfo BaseInfo;283    TBAAAccessInfo TBAAInfo;284    Address ThisValue = EmitPointerWithAlignment(Base, &BaseInfo, &TBAAInfo);285    This = MakeAddrLValue(ThisValue, Base->getType()->getPointeeType(),286                          BaseInfo, TBAAInfo);287  } else {288    This = EmitLValue(Base);289  }290 291  if (const CXXConstructorDecl *Ctor = dyn_cast<CXXConstructorDecl>(MD)) {292    // This is the MSVC p->Ctor::Ctor(...) extension. We assume that's293    // constructing a new complete object of type Ctor.294    assert(!RtlArgs);295    assert(ReturnValue.isNull() && "Constructor shouldn't have return value");296    CallArgList Args;297    commonEmitCXXMemberOrOperatorCall(298        *this, {Ctor, Ctor_Complete}, This.getPointer(*this),299        /*ImplicitParam=*/nullptr,300        /*ImplicitParamTy=*/QualType(), CE, Args, nullptr);301 302    EmitCXXConstructorCall(Ctor, Ctor_Complete, /*ForVirtualBase=*/false,303                           /*Delegating=*/false, This.getAddress(), Args,304                           AggValueSlot::DoesNotOverlap, CE->getExprLoc(),305                           /*NewPointerIsChecked=*/false, CallOrInvoke);306    return RValue::get(nullptr);307  }308 309  if (TrivialForCodegen) {310    if (isa<CXXDestructorDecl>(MD))311      return RValue::get(nullptr);312 313    if (TrivialAssignment) {314      // We don't like to generate the trivial copy/move assignment operator315      // when it isn't necessary; just produce the proper effect here.316      // It's important that we use the result of EmitLValue here rather than317      // emitting call arguments, in order to preserve TBAA information from318      // the RHS.319      LValue RHS = isa<CXXOperatorCallExpr>(CE)320                       ? TrivialAssignmentRHS321                       : EmitLValue(*CE->arg_begin());322      EmitAggregateAssign(This, RHS, CE->getType());323      return RValue::get(This.getPointer(*this));324    }325 326    assert(MD->getParent()->mayInsertExtraPadding() &&327           "unknown trivial member function");328  }329 330  // Compute the function type we're calling.331  const CXXMethodDecl *CalleeDecl =332      DevirtualizedMethod ? DevirtualizedMethod : MD;333  const CGFunctionInfo *FInfo = nullptr;334  if (const auto *Dtor = dyn_cast<CXXDestructorDecl>(CalleeDecl))335    FInfo = &CGM.getTypes().arrangeCXXStructorDeclaration(336        GlobalDecl(Dtor, Dtor_Complete));337  else338    FInfo = &CGM.getTypes().arrangeCXXMethodDeclaration(CalleeDecl);339 340  llvm::FunctionType *Ty = CGM.getTypes().GetFunctionType(*FInfo);341 342  // C++11 [class.mfct.non-static]p2:343  //   If a non-static member function of a class X is called for an object that344  //   is not of type X, or of a type derived from X, the behavior is undefined.345  SourceLocation CallLoc;346  ASTContext &C = getContext();347  if (CE)348    CallLoc = CE->getExprLoc();349 350  SanitizerSet SkippedChecks;351  if (const auto *CMCE = dyn_cast<CXXMemberCallExpr>(CE)) {352    auto *IOA = CMCE->getImplicitObjectArgument();353    bool IsImplicitObjectCXXThis = IsWrappedCXXThis(IOA);354    if (IsImplicitObjectCXXThis)355      SkippedChecks.set(SanitizerKind::Alignment, true);356    if (IsImplicitObjectCXXThis || isa<DeclRefExpr>(IOA))357      SkippedChecks.set(SanitizerKind::Null, true);358  }359 360  if (sanitizePerformTypeCheck())361    EmitTypeCheck(CodeGenFunction::TCK_MemberCall, CallLoc,362                  This.emitRawPointer(*this),363                  C.getCanonicalTagType(CalleeDecl->getParent()),364                  /*Alignment=*/CharUnits::Zero(), SkippedChecks);365 366  // C++ [class.virtual]p12:367  //   Explicit qualification with the scope operator (5.1) suppresses the368  //   virtual call mechanism.369  //370  // We also don't emit a virtual call if the base expression has a record type371  // because then we know what the type is.372  bool UseVirtualCall = CanUseVirtualCall && !DevirtualizedMethod;373 374  if (const CXXDestructorDecl *Dtor = dyn_cast<CXXDestructorDecl>(CalleeDecl)) {375    assert(CE->arguments().empty() &&376           "Destructor shouldn't have explicit parameters");377    assert(ReturnValue.isNull() && "Destructor shouldn't have return value");378    if (UseVirtualCall) {379      CGM.getCXXABI().EmitVirtualDestructorCall(380          *this, Dtor, Dtor_Complete, This.getAddress(),381          cast<CXXMemberCallExpr>(CE), CallOrInvoke);382    } else {383      GlobalDecl GD(Dtor, Dtor_Complete);384      CGCallee Callee;385      if (getLangOpts().AppleKext && Dtor->isVirtual() && HasQualifier)386        Callee = BuildAppleKextVirtualCall(Dtor, Qualifier, Ty);387      else if (!DevirtualizedMethod)388        Callee =389            CGCallee::forDirect(CGM.getAddrOfCXXStructor(GD, FInfo, Ty), GD);390      else {391        Callee = CGCallee::forDirect(CGM.GetAddrOfFunction(GD, Ty), GD);392      }393 394      QualType ThisTy =395          IsArrow ? Base->getType()->getPointeeType() : Base->getType();396      EmitCXXDestructorCall(GD, Callee, This.getPointer(*this), ThisTy,397                            /*ImplicitParam=*/nullptr,398                            /*ImplicitParamTy=*/QualType(), CE, CallOrInvoke);399    }400    return RValue::get(nullptr);401  }402 403  // FIXME: Uses of 'MD' past this point need to be audited. We may need to use404  // 'CalleeDecl' instead.405 406  CGCallee Callee;407  if (UseVirtualCall) {408    Callee = CGCallee::forVirtual(CE, MD, This.getAddress(), Ty);409  } else {410    if (SanOpts.has(SanitizerKind::CFINVCall) &&411        MD->getParent()->isDynamicClass()) {412      llvm::Value *VTable;413      const CXXRecordDecl *RD;414      std::tie(VTable, RD) = CGM.getCXXABI().LoadVTablePtr(415          *this, This.getAddress(), CalleeDecl->getParent());416      EmitVTablePtrCheckForCall(RD, VTable, CFITCK_NVCall, CE->getBeginLoc());417    }418 419    if (getLangOpts().AppleKext && MD->isVirtual() && HasQualifier)420      Callee = BuildAppleKextVirtualCall(MD, Qualifier, Ty);421    else if (!DevirtualizedMethod)422      Callee =423          CGCallee::forDirect(CGM.GetAddrOfFunction(MD, Ty), GlobalDecl(MD));424    else {425      Callee =426          CGCallee::forDirect(CGM.GetAddrOfFunction(DevirtualizedMethod, Ty),427                              GlobalDecl(DevirtualizedMethod));428    }429  }430 431  if (MD->isVirtual()) {432    Address NewThisAddr =433        CGM.getCXXABI().adjustThisArgumentForVirtualFunctionCall(434            *this, CalleeDecl, This.getAddress(), UseVirtualCall);435    This.setAddress(NewThisAddr);436  }437 438  return EmitCXXMemberOrOperatorCall(439      CalleeDecl, Callee, ReturnValue, This.getPointer(*this),440      /*ImplicitParam=*/nullptr, QualType(), CE, RtlArgs, CallOrInvoke);441}442 443RValue444CodeGenFunction::EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,445                                              ReturnValueSlot ReturnValue,446                                              llvm::CallBase **CallOrInvoke) {447  const BinaryOperator *BO =448      cast<BinaryOperator>(E->getCallee()->IgnoreParens());449  const Expr *BaseExpr = BO->getLHS();450  const Expr *MemFnExpr = BO->getRHS();451 452  const auto *MPT = MemFnExpr->getType()->castAs<MemberPointerType>();453  const auto *FPT = MPT->getPointeeType()->castAs<FunctionProtoType>();454  const auto *RD = MPT->getMostRecentCXXRecordDecl();455 456  // Emit the 'this' pointer.457  Address This = Address::invalid();458  if (BO->getOpcode() == BO_PtrMemI)459    This = EmitPointerWithAlignment(BaseExpr, nullptr, nullptr, KnownNonNull);460  else461    This = EmitLValue(BaseExpr, KnownNonNull).getAddress();462 463  CanQualType ClassType = CGM.getContext().getCanonicalTagType(RD);464  EmitTypeCheck(TCK_MemberCall, E->getExprLoc(), This.emitRawPointer(*this),465                ClassType);466 467  // Get the member function pointer.468  llvm::Value *MemFnPtr = EmitScalarExpr(MemFnExpr);469 470  // Ask the ABI to load the callee.  Note that This is modified.471  llvm::Value *ThisPtrForCall = nullptr;472  CGCallee Callee =473    CGM.getCXXABI().EmitLoadOfMemberFunctionPointer(*this, BO, This,474                                             ThisPtrForCall, MemFnPtr, MPT);475 476  CallArgList Args;477 478  QualType ThisType = getContext().getPointerType(ClassType);479 480  // Push the this ptr.481  Args.add(RValue::get(ThisPtrForCall), ThisType);482 483  RequiredArgs required = RequiredArgs::forPrototypePlus(FPT, 1);484 485  // And the rest of the call args486  EmitCallArgs(Args, FPT, E->arguments());487  return EmitCall(CGM.getTypes().arrangeCXXMethodCall(Args, FPT, required,488                                                      /*PrefixSize=*/0),489                  Callee, ReturnValue, Args, CallOrInvoke, E == MustTailCall,490                  E->getExprLoc());491}492 493RValue CodeGenFunction::EmitCXXOperatorMemberCallExpr(494    const CXXOperatorCallExpr *E, const CXXMethodDecl *MD,495    ReturnValueSlot ReturnValue, llvm::CallBase **CallOrInvoke) {496  assert(MD->isImplicitObjectMemberFunction() &&497         "Trying to emit a member call expr on a static method!");498  return EmitCXXMemberOrOperatorMemberCallExpr(499      E, MD, ReturnValue, /*HasQualifier=*/false, /*Qualifier=*/std::nullopt,500      /*IsArrow=*/false, E->getArg(0), CallOrInvoke);501}502 503RValue CodeGenFunction::EmitCUDAKernelCallExpr(const CUDAKernelCallExpr *E,504                                               ReturnValueSlot ReturnValue,505                                               llvm::CallBase **CallOrInvoke) {506  // Emit as a device kernel call if CUDA device code is to be generated.507  if (getLangOpts().CUDAIsDevice)508    return CGM.getCUDARuntime().EmitCUDADeviceKernelCallExpr(509        *this, E, ReturnValue, CallOrInvoke);510  return CGM.getCUDARuntime().EmitCUDAKernelCallExpr(*this, E, ReturnValue,511                                                     CallOrInvoke);512}513 514static void EmitNullBaseClassInitialization(CodeGenFunction &CGF,515                                            Address DestPtr,516                                            const CXXRecordDecl *Base) {517  if (Base->isEmpty())518    return;519 520  DestPtr = DestPtr.withElementType(CGF.Int8Ty);521 522  const ASTRecordLayout &Layout = CGF.getContext().getASTRecordLayout(Base);523  CharUnits NVSize = Layout.getNonVirtualSize();524 525  // We cannot simply zero-initialize the entire base sub-object if vbptrs are526  // present, they are initialized by the most derived class before calling the527  // constructor.528  SmallVector<std::pair<CharUnits, CharUnits>, 1> Stores;529  Stores.emplace_back(CharUnits::Zero(), NVSize);530 531  // Each store is split by the existence of a vbptr.532  CharUnits VBPtrWidth = CGF.getPointerSize();533  std::vector<CharUnits> VBPtrOffsets =534      CGF.CGM.getCXXABI().getVBPtrOffsets(Base);535  for (CharUnits VBPtrOffset : VBPtrOffsets) {536    // Stop before we hit any virtual base pointers located in virtual bases.537    if (VBPtrOffset >= NVSize)538      break;539    std::pair<CharUnits, CharUnits> LastStore = Stores.pop_back_val();540    CharUnits LastStoreOffset = LastStore.first;541    CharUnits LastStoreSize = LastStore.second;542 543    CharUnits SplitBeforeOffset = LastStoreOffset;544    CharUnits SplitBeforeSize = VBPtrOffset - SplitBeforeOffset;545    assert(!SplitBeforeSize.isNegative() && "negative store size!");546    if (!SplitBeforeSize.isZero())547      Stores.emplace_back(SplitBeforeOffset, SplitBeforeSize);548 549    CharUnits SplitAfterOffset = VBPtrOffset + VBPtrWidth;550    CharUnits SplitAfterSize = LastStoreSize - SplitAfterOffset;551    assert(!SplitAfterSize.isNegative() && "negative store size!");552    if (!SplitAfterSize.isZero())553      Stores.emplace_back(SplitAfterOffset, SplitAfterSize);554  }555 556  // If the type contains a pointer to data member we can't memset it to zero.557  // Instead, create a null constant and copy it to the destination.558  // TODO: there are other patterns besides zero that we can usefully memset,559  // like -1, which happens to be the pattern used by member-pointers.560  // TODO: isZeroInitializable can be over-conservative in the case where a561  // virtual base contains a member pointer.562  llvm::Constant *NullConstantForBase = CGF.CGM.EmitNullConstantForBase(Base);563  if (!NullConstantForBase->isNullValue()) {564    llvm::GlobalVariable *NullVariable = new llvm::GlobalVariable(565        CGF.CGM.getModule(), NullConstantForBase->getType(),566        /*isConstant=*/true, llvm::GlobalVariable::PrivateLinkage,567        NullConstantForBase, Twine());568 569    CharUnits Align =570        std::max(Layout.getNonVirtualAlignment(), DestPtr.getAlignment());571    NullVariable->setAlignment(Align.getAsAlign());572 573    Address SrcPtr(NullVariable, CGF.Int8Ty, Align);574 575    // Get and call the appropriate llvm.memcpy overload.576    for (std::pair<CharUnits, CharUnits> Store : Stores) {577      CharUnits StoreOffset = Store.first;578      CharUnits StoreSize = Store.second;579      llvm::Value *StoreSizeVal = CGF.CGM.getSize(StoreSize);580      CGF.Builder.CreateMemCpy(581          CGF.Builder.CreateConstInBoundsByteGEP(DestPtr, StoreOffset),582          CGF.Builder.CreateConstInBoundsByteGEP(SrcPtr, StoreOffset),583          StoreSizeVal);584    }585 586  // Otherwise, just memset the whole thing to zero.  This is legal587  // because in LLVM, all default initializers (other than the ones we just588  // handled above) are guaranteed to have a bit pattern of all zeros.589  } else {590    for (std::pair<CharUnits, CharUnits> Store : Stores) {591      CharUnits StoreOffset = Store.first;592      CharUnits StoreSize = Store.second;593      llvm::Value *StoreSizeVal = CGF.CGM.getSize(StoreSize);594      CGF.Builder.CreateMemSet(595          CGF.Builder.CreateConstInBoundsByteGEP(DestPtr, StoreOffset),596          CGF.Builder.getInt8(0), StoreSizeVal);597    }598  }599}600 601void602CodeGenFunction::EmitCXXConstructExpr(const CXXConstructExpr *E,603                                      AggValueSlot Dest) {604  assert(!Dest.isIgnored() && "Must have a destination!");605  const CXXConstructorDecl *CD = E->getConstructor();606 607  // If we require zero initialization before (or instead of) calling the608  // constructor, as can be the case with a non-user-provided default609  // constructor, emit the zero initialization now, unless destination is610  // already zeroed.611  if (E->requiresZeroInitialization() && !Dest.isZeroed()) {612    switch (E->getConstructionKind()) {613    case CXXConstructionKind::Delegating:614    case CXXConstructionKind::Complete:615      EmitNullInitialization(Dest.getAddress(), E->getType());616      break;617    case CXXConstructionKind::VirtualBase:618    case CXXConstructionKind::NonVirtualBase:619      EmitNullBaseClassInitialization(*this, Dest.getAddress(),620                                      CD->getParent());621      break;622    }623  }624 625  // If this is a call to a trivial default constructor, do nothing.626  if (CD->isTrivial() && CD->isDefaultConstructor())627    return;628 629  // Elide the constructor if we're constructing from a temporary.630  if (getLangOpts().ElideConstructors && E->isElidable()) {631    // FIXME: This only handles the simplest case, where the source object632    //        is passed directly as the first argument to the constructor.633    //        This should also handle stepping though implicit casts and634    //        conversion sequences which involve two steps, with a635    //        conversion operator followed by a converting constructor.636    const Expr *SrcObj = E->getArg(0);637    assert(SrcObj->isTemporaryObject(getContext(), CD->getParent()));638    assert(639        getContext().hasSameUnqualifiedType(E->getType(), SrcObj->getType()));640    EmitAggExpr(SrcObj, Dest);641    return;642  }643 644  if (const ArrayType *arrayType645        = getContext().getAsArrayType(E->getType())) {646    EmitCXXAggrConstructorCall(CD, arrayType, Dest.getAddress(), E,647                               Dest.isSanitizerChecked());648  } else {649    CXXCtorType Type = Ctor_Complete;650    bool ForVirtualBase = false;651    bool Delegating = false;652 653    switch (E->getConstructionKind()) {654    case CXXConstructionKind::Delegating:655      // We should be emitting a constructor; GlobalDecl will assert this656      Type = CurGD.getCtorType();657      Delegating = true;658      break;659 660    case CXXConstructionKind::Complete:661      Type = Ctor_Complete;662      break;663 664    case CXXConstructionKind::VirtualBase:665      ForVirtualBase = true;666      [[fallthrough]];667 668    case CXXConstructionKind::NonVirtualBase:669      Type = Ctor_Base;670     }671 672     // Call the constructor.673     EmitCXXConstructorCall(CD, Type, ForVirtualBase, Delegating, Dest, E);674  }675}676 677void CodeGenFunction::EmitSynthesizedCXXCopyCtor(Address Dest, Address Src,678                                                 const Expr *Exp) {679  if (const ExprWithCleanups *E = dyn_cast<ExprWithCleanups>(Exp))680    Exp = E->getSubExpr();681  assert(isa<CXXConstructExpr>(Exp) &&682         "EmitSynthesizedCXXCopyCtor - unknown copy ctor expr");683  const CXXConstructExpr* E = cast<CXXConstructExpr>(Exp);684  const CXXConstructorDecl *CD = E->getConstructor();685  RunCleanupsScope Scope(*this);686 687  // If we require zero initialization before (or instead of) calling the688  // constructor, as can be the case with a non-user-provided default689  // constructor, emit the zero initialization now.690  // FIXME. Do I still need this for a copy ctor synthesis?691  if (E->requiresZeroInitialization())692    EmitNullInitialization(Dest, E->getType());693 694  assert(!getContext().getAsConstantArrayType(E->getType())695         && "EmitSynthesizedCXXCopyCtor - Copied-in Array");696  EmitSynthesizedCXXCopyCtorCall(CD, Dest, Src, E);697}698 699static CharUnits CalculateCookiePadding(CodeGenFunction &CGF,700                                        const CXXNewExpr *E) {701  if (!E->isArray())702    return CharUnits::Zero();703 704  // No cookie is required if the operator new[] being used is the705  // reserved placement operator new[].706  if (E->getOperatorNew()->isReservedGlobalPlacementOperator())707    return CharUnits::Zero();708 709  return CGF.CGM.getCXXABI().GetArrayCookieSize(E);710}711 712static llvm::Value *EmitCXXNewAllocSize(CodeGenFunction &CGF,713                                        const CXXNewExpr *e,714                                        unsigned minElements,715                                        llvm::Value *&numElements,716                                        llvm::Value *&sizeWithoutCookie) {717  QualType type = e->getAllocatedType();718 719  if (!e->isArray()) {720    CharUnits typeSize = CGF.getContext().getTypeSizeInChars(type);721    sizeWithoutCookie722      = llvm::ConstantInt::get(CGF.SizeTy, typeSize.getQuantity());723    return sizeWithoutCookie;724  }725 726  // The width of size_t.727  unsigned sizeWidth = CGF.SizeTy->getBitWidth();728 729  // Figure out the cookie size.730  llvm::APInt cookieSize(sizeWidth,731                         CalculateCookiePadding(CGF, e).getQuantity());732 733  // Emit the array size expression.734  // We multiply the size of all dimensions for NumElements.735  // e.g for 'int[2][3]', ElemType is 'int' and NumElements is 6.736  numElements = ConstantEmitter(CGF).tryEmitAbstract(737      *e->getArraySize(), (*e->getArraySize())->getType());738  if (!numElements)739    numElements = CGF.EmitScalarExpr(*e->getArraySize());740  assert(isa<llvm::IntegerType>(numElements->getType()));741 742  // The number of elements can be have an arbitrary integer type;743  // essentially, we need to multiply it by a constant factor, add a744  // cookie size, and verify that the result is representable as a745  // size_t.  That's just a gloss, though, and it's wrong in one746  // important way: if the count is negative, it's an error even if747  // the cookie size would bring the total size >= 0.748  bool isSigned749    = (*e->getArraySize())->getType()->isSignedIntegerOrEnumerationType();750  llvm::IntegerType *numElementsType751    = cast<llvm::IntegerType>(numElements->getType());752  unsigned numElementsWidth = numElementsType->getBitWidth();753 754  // Compute the constant factor.755  llvm::APInt arraySizeMultiplier(sizeWidth, 1);756  while (const ConstantArrayType *CAT757             = CGF.getContext().getAsConstantArrayType(type)) {758    type = CAT->getElementType();759    arraySizeMultiplier *= CAT->getSize();760  }761 762  CharUnits typeSize = CGF.getContext().getTypeSizeInChars(type);763  llvm::APInt typeSizeMultiplier(sizeWidth, typeSize.getQuantity());764  typeSizeMultiplier *= arraySizeMultiplier;765 766  // This will be a size_t.767  llvm::Value *size;768 769  // If someone is doing 'new int[42]' there is no need to do a dynamic check.770  // Don't bloat the -O0 code.771  if (llvm::ConstantInt *numElementsC =772        dyn_cast<llvm::ConstantInt>(numElements)) {773    const llvm::APInt &count = numElementsC->getValue();774 775    bool hasAnyOverflow = false;776 777    // If 'count' was a negative number, it's an overflow.778    if (isSigned && count.isNegative())779      hasAnyOverflow = true;780 781    // We want to do all this arithmetic in size_t.  If numElements is782    // wider than that, check whether it's already too big, and if so,783    // overflow.784    else if (numElementsWidth > sizeWidth &&785             numElementsWidth - sizeWidth > count.countl_zero())786      hasAnyOverflow = true;787 788    // Okay, compute a count at the right width.789    llvm::APInt adjustedCount = count.zextOrTrunc(sizeWidth);790 791    // If there is a brace-initializer, we cannot allocate fewer elements than792    // there are initializers. If we do, that's treated like an overflow.793    if (adjustedCount.ult(minElements))794      hasAnyOverflow = true;795 796    // Scale numElements by that.  This might overflow, but we don't797    // care because it only overflows if allocationSize does, too, and798    // if that overflows then we shouldn't use this.799    numElements = llvm::ConstantInt::get(CGF.SizeTy,800                                         adjustedCount * arraySizeMultiplier);801 802    // Compute the size before cookie, and track whether it overflowed.803    bool overflow;804    llvm::APInt allocationSize805      = adjustedCount.umul_ov(typeSizeMultiplier, overflow);806    hasAnyOverflow |= overflow;807 808    // Add in the cookie, and check whether it's overflowed.809    if (cookieSize != 0) {810      // Save the current size without a cookie.  This shouldn't be811      // used if there was overflow.812      sizeWithoutCookie = llvm::ConstantInt::get(CGF.SizeTy, allocationSize);813 814      allocationSize = allocationSize.uadd_ov(cookieSize, overflow);815      hasAnyOverflow |= overflow;816    }817 818    // On overflow, produce a -1 so operator new will fail.819    if (hasAnyOverflow) {820      size = llvm::Constant::getAllOnesValue(CGF.SizeTy);821    } else {822      size = llvm::ConstantInt::get(CGF.SizeTy, allocationSize);823    }824 825  // Otherwise, we might need to use the overflow intrinsics.826  } else {827    // There are up to five conditions we need to test for:828    // 1) if isSigned, we need to check whether numElements is negative;829    // 2) if numElementsWidth > sizeWidth, we need to check whether830    //   numElements is larger than something representable in size_t;831    // 3) if minElements > 0, we need to check whether numElements is smaller832    //    than that.833    // 4) we need to compute834    //      sizeWithoutCookie := numElements * typeSizeMultiplier835    //    and check whether it overflows; and836    // 5) if we need a cookie, we need to compute837    //      size := sizeWithoutCookie + cookieSize838    //    and check whether it overflows.839 840    llvm::Value *hasOverflow = nullptr;841 842    // If numElementsWidth > sizeWidth, then one way or another, we're843    // going to have to do a comparison for (2), and this happens to844    // take care of (1), too.845    if (numElementsWidth > sizeWidth) {846      llvm::APInt threshold =847          llvm::APInt::getOneBitSet(numElementsWidth, sizeWidth);848 849      llvm::Value *thresholdV850        = llvm::ConstantInt::get(numElementsType, threshold);851 852      hasOverflow = CGF.Builder.CreateICmpUGE(numElements, thresholdV);853      numElements = CGF.Builder.CreateTrunc(numElements, CGF.SizeTy);854 855    // Otherwise, if we're signed, we want to sext up to size_t.856    } else if (isSigned) {857      if (numElementsWidth < sizeWidth)858        numElements = CGF.Builder.CreateSExt(numElements, CGF.SizeTy);859 860      // If there's a non-1 type size multiplier, then we can do the861      // signedness check at the same time as we do the multiply862      // because a negative number times anything will cause an863      // unsigned overflow.  Otherwise, we have to do it here. But at least864      // in this case, we can subsume the >= minElements check.865      if (typeSizeMultiplier == 1)866        hasOverflow = CGF.Builder.CreateICmpSLT(numElements,867                              llvm::ConstantInt::get(CGF.SizeTy, minElements));868 869    // Otherwise, zext up to size_t if necessary.870    } else if (numElementsWidth < sizeWidth) {871      numElements = CGF.Builder.CreateZExt(numElements, CGF.SizeTy);872    }873 874    assert(numElements->getType() == CGF.SizeTy);875 876    if (minElements) {877      // Don't allow allocation of fewer elements than we have initializers.878      if (!hasOverflow) {879        hasOverflow = CGF.Builder.CreateICmpULT(numElements,880                              llvm::ConstantInt::get(CGF.SizeTy, minElements));881      } else if (numElementsWidth > sizeWidth) {882        // The other existing overflow subsumes this check.883        // We do an unsigned comparison, since any signed value < -1 is884        // taken care of either above or below.885        hasOverflow = CGF.Builder.CreateOr(hasOverflow,886                          CGF.Builder.CreateICmpULT(numElements,887                              llvm::ConstantInt::get(CGF.SizeTy, minElements)));888      }889    }890 891    size = numElements;892 893    // Multiply by the type size if necessary.  This multiplier894    // includes all the factors for nested arrays.895    //896    // This step also causes numElements to be scaled up by the897    // nested-array factor if necessary.  Overflow on this computation898    // can be ignored because the result shouldn't be used if899    // allocation fails.900    if (typeSizeMultiplier != 1) {901      llvm::Function *umul_with_overflow902        = CGF.CGM.getIntrinsic(llvm::Intrinsic::umul_with_overflow, CGF.SizeTy);903 904      llvm::Value *tsmV =905        llvm::ConstantInt::get(CGF.SizeTy, typeSizeMultiplier);906      llvm::Value *result =907          CGF.Builder.CreateCall(umul_with_overflow, {size, tsmV});908 909      llvm::Value *overflowed = CGF.Builder.CreateExtractValue(result, 1);910      if (hasOverflow)911        hasOverflow = CGF.Builder.CreateOr(hasOverflow, overflowed);912      else913        hasOverflow = overflowed;914 915      size = CGF.Builder.CreateExtractValue(result, 0);916 917      // Also scale up numElements by the array size multiplier.918      if (arraySizeMultiplier != 1) {919        // If the base element type size is 1, then we can re-use the920        // multiply we just did.921        if (typeSize.isOne()) {922          assert(arraySizeMultiplier == typeSizeMultiplier);923          numElements = size;924 925        // Otherwise we need a separate multiply.926        } else {927          llvm::Value *asmV =928            llvm::ConstantInt::get(CGF.SizeTy, arraySizeMultiplier);929          numElements = CGF.Builder.CreateMul(numElements, asmV);930        }931      }932    } else {933      // numElements doesn't need to be scaled.934      assert(arraySizeMultiplier == 1);935    }936 937    // Add in the cookie size if necessary.938    if (cookieSize != 0) {939      sizeWithoutCookie = size;940 941      llvm::Function *uadd_with_overflow942        = CGF.CGM.getIntrinsic(llvm::Intrinsic::uadd_with_overflow, CGF.SizeTy);943 944      llvm::Value *cookieSizeV = llvm::ConstantInt::get(CGF.SizeTy, cookieSize);945      llvm::Value *result =946          CGF.Builder.CreateCall(uadd_with_overflow, {size, cookieSizeV});947 948      llvm::Value *overflowed = CGF.Builder.CreateExtractValue(result, 1);949      if (hasOverflow)950        hasOverflow = CGF.Builder.CreateOr(hasOverflow, overflowed);951      else952        hasOverflow = overflowed;953 954      size = CGF.Builder.CreateExtractValue(result, 0);955    }956 957    // If we had any possibility of dynamic overflow, make a select to958    // overwrite 'size' with an all-ones value, which should cause959    // operator new to throw.960    if (hasOverflow)961      size = CGF.Builder.CreateSelect(hasOverflow,962                                 llvm::Constant::getAllOnesValue(CGF.SizeTy),963                                      size);964  }965 966  if (cookieSize == 0)967    sizeWithoutCookie = size;968  else969    assert(sizeWithoutCookie && "didn't set sizeWithoutCookie?");970 971  return size;972}973 974static void StoreAnyExprIntoOneUnit(CodeGenFunction &CGF, const Expr *Init,975                                    QualType AllocType, Address NewPtr,976                                    AggValueSlot::Overlap_t MayOverlap) {977  // FIXME: Refactor with EmitExprAsInit.978  switch (CGF.getEvaluationKind(AllocType)) {979  case TEK_Scalar:980    CGF.EmitScalarInit(Init, nullptr,981                       CGF.MakeAddrLValue(NewPtr, AllocType), false);982    return;983  case TEK_Complex:984    CGF.EmitComplexExprIntoLValue(Init, CGF.MakeAddrLValue(NewPtr, AllocType),985                                  /*isInit*/ true);986    return;987  case TEK_Aggregate: {988    AggValueSlot Slot989      = AggValueSlot::forAddr(NewPtr, AllocType.getQualifiers(),990                              AggValueSlot::IsDestructed,991                              AggValueSlot::DoesNotNeedGCBarriers,992                              AggValueSlot::IsNotAliased,993                              MayOverlap, AggValueSlot::IsNotZeroed,994                              AggValueSlot::IsSanitizerChecked);995    CGF.EmitAggExpr(Init, Slot);996    return;997  }998  }999  llvm_unreachable("bad evaluation kind");1000}1001 1002void CodeGenFunction::EmitNewArrayInitializer(1003    const CXXNewExpr *E, QualType ElementType, llvm::Type *ElementTy,1004    Address BeginPtr, llvm::Value *NumElements,1005    llvm::Value *AllocSizeWithoutCookie) {1006  // If we have a type with trivial initialization and no initializer,1007  // there's nothing to do.1008  if (!E->hasInitializer())1009    return;1010 1011  Address CurPtr = BeginPtr;1012 1013  unsigned InitListElements = 0;1014 1015  const Expr *Init = E->getInitializer();1016  Address EndOfInit = Address::invalid();1017  QualType::DestructionKind DtorKind = ElementType.isDestructedType();1018  CleanupDeactivationScope deactivation(*this);1019  bool pushedCleanup = false;1020 1021  CharUnits ElementSize = getContext().getTypeSizeInChars(ElementType);1022  CharUnits ElementAlign =1023    BeginPtr.getAlignment().alignmentOfArrayElement(ElementSize);1024 1025  // Attempt to perform zero-initialization using memset.1026  auto TryMemsetInitialization = [&]() -> bool {1027    // FIXME: If the type is a pointer-to-data-member under the Itanium ABI,1028    // we can initialize with a memset to -1.1029    if (!CGM.getTypes().isZeroInitializable(ElementType))1030      return false;1031 1032    // Optimization: since zero initialization will just set the memory1033    // to all zeroes, generate a single memset to do it in one shot.1034 1035    // Subtract out the size of any elements we've already initialized.1036    auto *RemainingSize = AllocSizeWithoutCookie;1037    if (InitListElements) {1038      // We know this can't overflow; we check this when doing the allocation.1039      auto *InitializedSize = llvm::ConstantInt::get(1040          RemainingSize->getType(),1041          getContext().getTypeSizeInChars(ElementType).getQuantity() *1042              InitListElements);1043      RemainingSize = Builder.CreateSub(RemainingSize, InitializedSize);1044    }1045 1046    // Create the memset.1047    Builder.CreateMemSet(CurPtr, Builder.getInt8(0), RemainingSize, false);1048    return true;1049  };1050 1051  const InitListExpr *ILE = dyn_cast<InitListExpr>(Init);1052  const CXXParenListInitExpr *CPLIE = nullptr;1053  const StringLiteral *SL = nullptr;1054  const ObjCEncodeExpr *OCEE = nullptr;1055  const Expr *IgnoreParen = nullptr;1056  if (!ILE) {1057    IgnoreParen = Init->IgnoreParenImpCasts();1058    CPLIE = dyn_cast<CXXParenListInitExpr>(IgnoreParen);1059    SL = dyn_cast<StringLiteral>(IgnoreParen);1060    OCEE = dyn_cast<ObjCEncodeExpr>(IgnoreParen);1061  }1062 1063  // If the initializer is an initializer list, first do the explicit elements.1064  if (ILE || CPLIE || SL || OCEE) {1065    // Initializing from a (braced) string literal is a special case; the init1066    // list element does not initialize a (single) array element.1067    if ((ILE && ILE->isStringLiteralInit()) || SL || OCEE) {1068      if (!ILE)1069        Init = IgnoreParen;1070      // Initialize the initial portion of length equal to that of the string1071      // literal. The allocation must be for at least this much; we emitted a1072      // check for that earlier.1073      AggValueSlot Slot =1074          AggValueSlot::forAddr(CurPtr, ElementType.getQualifiers(),1075                                AggValueSlot::IsDestructed,1076                                AggValueSlot::DoesNotNeedGCBarriers,1077                                AggValueSlot::IsNotAliased,1078                                AggValueSlot::DoesNotOverlap,1079                                AggValueSlot::IsNotZeroed,1080                                AggValueSlot::IsSanitizerChecked);1081      EmitAggExpr(ILE ? ILE->getInit(0) : Init, Slot);1082 1083      // Move past these elements.1084      InitListElements =1085          cast<ConstantArrayType>(Init->getType()->getAsArrayTypeUnsafe())1086              ->getZExtSize();1087      CurPtr = Builder.CreateConstInBoundsGEP(1088          CurPtr, InitListElements, "string.init.end");1089 1090      // Zero out the rest, if any remain.1091      llvm::ConstantInt *ConstNum = dyn_cast<llvm::ConstantInt>(NumElements);1092      if (!ConstNum || !ConstNum->equalsInt(InitListElements)) {1093        bool OK = TryMemsetInitialization();1094        (void)OK;1095        assert(OK && "couldn't memset character type?");1096      }1097      return;1098    }1099 1100    ArrayRef<const Expr *> InitExprs =1101        ILE ? ILE->inits() : CPLIE->getInitExprs();1102    InitListElements = InitExprs.size();1103 1104    // If this is a multi-dimensional array new, we will initialize multiple1105    // elements with each init list element.1106    QualType AllocType = E->getAllocatedType();1107    if (const ConstantArrayType *CAT = dyn_cast_or_null<ConstantArrayType>(1108            AllocType->getAsArrayTypeUnsafe())) {1109      ElementTy = ConvertTypeForMem(AllocType);1110      CurPtr = CurPtr.withElementType(ElementTy);1111      InitListElements *= getContext().getConstantArrayElementCount(CAT);1112    }1113 1114    // Enter a partial-destruction Cleanup if necessary.1115    if (DtorKind) {1116      AllocaTrackerRAII AllocaTracker(*this);1117      // In principle we could tell the Cleanup where we are more1118      // directly, but the control flow can get so varied here that it1119      // would actually be quite complex.  Therefore we go through an1120      // alloca.1121      llvm::Instruction *DominatingIP =1122          Builder.CreateFlagLoad(llvm::ConstantInt::getNullValue(Int8PtrTy));1123      EndOfInit = CreateTempAlloca(BeginPtr.getType(), getPointerAlign(),1124                                   "array.init.end");1125      pushIrregularPartialArrayCleanup(BeginPtr.emitRawPointer(*this),1126                                       EndOfInit, ElementType, ElementAlign,1127                                       getDestroyer(DtorKind));1128      cast<EHCleanupScope>(*EHStack.find(EHStack.stable_begin()))1129          .AddAuxAllocas(AllocaTracker.Take());1130      DeferredDeactivationCleanupStack.push_back(1131          {EHStack.stable_begin(), DominatingIP});1132      pushedCleanup = true;1133    }1134 1135    CharUnits StartAlign = CurPtr.getAlignment();1136    unsigned i = 0;1137    for (const Expr *IE : InitExprs) {1138      // Tell the cleanup that it needs to destroy up to this1139      // element.  TODO: some of these stores can be trivially1140      // observed to be unnecessary.1141      if (EndOfInit.isValid()) {1142        Builder.CreateStore(CurPtr.emitRawPointer(*this), EndOfInit);1143      }1144      // FIXME: If the last initializer is an incomplete initializer list for1145      // an array, and we have an array filler, we can fold together the two1146      // initialization loops.1147      StoreAnyExprIntoOneUnit(*this, IE, IE->getType(), CurPtr,1148                              AggValueSlot::DoesNotOverlap);1149      CurPtr = Address(Builder.CreateInBoundsGEP(CurPtr.getElementType(),1150                                                 CurPtr.emitRawPointer(*this),1151                                                 Builder.getSize(1),1152                                                 "array.exp.next"),1153                       CurPtr.getElementType(),1154                       StartAlign.alignmentAtOffset((++i) * ElementSize));1155    }1156 1157    // The remaining elements are filled with the array filler expression.1158    Init = ILE ? ILE->getArrayFiller() : CPLIE->getArrayFiller();1159 1160    // Extract the initializer for the individual array elements by pulling1161    // out the array filler from all the nested initializer lists. This avoids1162    // generating a nested loop for the initialization.1163    while (Init && Init->getType()->isConstantArrayType()) {1164      auto *SubILE = dyn_cast<InitListExpr>(Init);1165      if (!SubILE)1166        break;1167      assert(SubILE->getNumInits() == 0 && "explicit inits in array filler?");1168      Init = SubILE->getArrayFiller();1169    }1170 1171    // Switch back to initializing one base element at a time.1172    CurPtr = CurPtr.withElementType(BeginPtr.getElementType());1173  }1174 1175  // If all elements have already been initialized, skip any further1176  // initialization.1177  llvm::ConstantInt *ConstNum = dyn_cast<llvm::ConstantInt>(NumElements);1178  if (ConstNum && ConstNum->getZExtValue() <= InitListElements) {1179    return;1180  }1181 1182  assert(Init && "have trailing elements to initialize but no initializer");1183 1184  // If this is a constructor call, try to optimize it out, and failing that1185  // emit a single loop to initialize all remaining elements.1186  if (const CXXConstructExpr *CCE = dyn_cast<CXXConstructExpr>(Init)) {1187    CXXConstructorDecl *Ctor = CCE->getConstructor();1188    if (Ctor->isTrivial()) {1189      // If new expression did not specify value-initialization, then there1190      // is no initialization.1191      if (!CCE->requiresZeroInitialization() || Ctor->getParent()->isEmpty())1192        return;1193 1194      if (TryMemsetInitialization())1195        return;1196    }1197 1198    // Store the new Cleanup position for irregular Cleanups.1199    //1200    // FIXME: Share this cleanup with the constructor call emission rather than1201    // having it create a cleanup of its own.1202    if (EndOfInit.isValid())1203      Builder.CreateStore(CurPtr.emitRawPointer(*this), EndOfInit);1204 1205    // Emit a constructor call loop to initialize the remaining elements.1206    if (InitListElements)1207      NumElements = Builder.CreateSub(1208          NumElements,1209          llvm::ConstantInt::get(NumElements->getType(), InitListElements));1210    EmitCXXAggrConstructorCall(Ctor, NumElements, CurPtr, CCE,1211                               /*NewPointerIsChecked*/true,1212                               CCE->requiresZeroInitialization());1213    return;1214  }1215 1216  // If this is value-initialization, we can usually use memset.1217  ImplicitValueInitExpr IVIE(ElementType);1218  if (isa<ImplicitValueInitExpr>(Init)) {1219    if (TryMemsetInitialization())1220      return;1221 1222    // Switch to an ImplicitValueInitExpr for the element type. This handles1223    // only one case: multidimensional array new of pointers to members. In1224    // all other cases, we already have an initializer for the array element.1225    Init = &IVIE;1226  }1227 1228  // At this point we should have found an initializer for the individual1229  // elements of the array.1230  assert(getContext().hasSameUnqualifiedType(ElementType, Init->getType()) &&1231         "got wrong type of element to initialize");1232 1233  // If we have an empty initializer list, we can usually use memset.1234  if (auto *ILE = dyn_cast<InitListExpr>(Init))1235    if (ILE->getNumInits() == 0 && TryMemsetInitialization())1236      return;1237 1238  // If we have a struct whose every field is value-initialized, we can1239  // usually use memset.1240  if (auto *ILE = dyn_cast<InitListExpr>(Init)) {1241    if (const RecordType *RType =1242            ILE->getType()->getAsCanonical<RecordType>()) {1243      if (RType->getDecl()->isStruct()) {1244        const RecordDecl *RD = RType->getDecl()->getDefinitionOrSelf();1245        unsigned NumElements = 0;1246        if (auto *CXXRD = dyn_cast<CXXRecordDecl>(RD))1247          NumElements = CXXRD->getNumBases();1248        for (auto *Field : RD->fields())1249          if (!Field->isUnnamedBitField())1250            ++NumElements;1251        // FIXME: Recurse into nested InitListExprs.1252        if (ILE->getNumInits() == NumElements)1253          for (unsigned i = 0, e = ILE->getNumInits(); i != e; ++i)1254            if (!isa<ImplicitValueInitExpr>(ILE->getInit(i)))1255              --NumElements;1256        if (ILE->getNumInits() == NumElements && TryMemsetInitialization())1257          return;1258      }1259    }1260  }1261 1262  // Create the loop blocks.1263  llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();1264  llvm::BasicBlock *LoopBB = createBasicBlock("new.loop");1265  llvm::BasicBlock *ContBB = createBasicBlock("new.loop.end");1266 1267  // Find the end of the array, hoisted out of the loop.1268  llvm::Value *EndPtr = Builder.CreateInBoundsGEP(1269      BeginPtr.getElementType(), BeginPtr.emitRawPointer(*this), NumElements,1270      "array.end");1271 1272  // If the number of elements isn't constant, we have to now check if there is1273  // anything left to initialize.1274  if (!ConstNum) {1275    llvm::Value *IsEmpty = Builder.CreateICmpEQ(CurPtr.emitRawPointer(*this),1276                                                EndPtr, "array.isempty");1277    Builder.CreateCondBr(IsEmpty, ContBB, LoopBB);1278  }1279 1280  // Enter the loop.1281  EmitBlock(LoopBB);1282 1283  // Set up the current-element phi.1284  llvm::PHINode *CurPtrPhi =1285      Builder.CreatePHI(CurPtr.getType(), 2, "array.cur");1286  CurPtrPhi->addIncoming(CurPtr.emitRawPointer(*this), EntryBB);1287 1288  CurPtr = Address(CurPtrPhi, CurPtr.getElementType(), ElementAlign);1289 1290  // Store the new Cleanup position for irregular Cleanups.1291  if (EndOfInit.isValid())1292    Builder.CreateStore(CurPtr.emitRawPointer(*this), EndOfInit);1293 1294  // Enter a partial-destruction Cleanup if necessary.1295  if (!pushedCleanup && needsEHCleanup(DtorKind)) {1296    llvm::Instruction *DominatingIP =1297        Builder.CreateFlagLoad(llvm::ConstantInt::getNullValue(Int8PtrTy));1298    pushRegularPartialArrayCleanup(BeginPtr.emitRawPointer(*this),1299                                   CurPtr.emitRawPointer(*this), ElementType,1300                                   ElementAlign, getDestroyer(DtorKind));1301    DeferredDeactivationCleanupStack.push_back(1302        {EHStack.stable_begin(), DominatingIP});1303  }1304 1305  // Emit the initializer into this element.1306  StoreAnyExprIntoOneUnit(*this, Init, Init->getType(), CurPtr,1307                          AggValueSlot::DoesNotOverlap);1308 1309  // Leave the Cleanup if we entered one.1310  deactivation.ForceDeactivate();1311 1312  // Advance to the next element by adjusting the pointer type as necessary.1313  llvm::Value *NextPtr = Builder.CreateConstInBoundsGEP1_32(1314      ElementTy, CurPtr.emitRawPointer(*this), 1, "array.next");1315 1316  // Check whether we've gotten to the end of the array and, if so,1317  // exit the loop.1318  llvm::Value *IsEnd = Builder.CreateICmpEQ(NextPtr, EndPtr, "array.atend");1319  Builder.CreateCondBr(IsEnd, ContBB, LoopBB);1320  CurPtrPhi->addIncoming(NextPtr, Builder.GetInsertBlock());1321 1322  EmitBlock(ContBB);1323}1324 1325static void EmitNewInitializer(CodeGenFunction &CGF, const CXXNewExpr *E,1326                               QualType ElementType, llvm::Type *ElementTy,1327                               Address NewPtr, llvm::Value *NumElements,1328                               llvm::Value *AllocSizeWithoutCookie) {1329  ApplyDebugLocation DL(CGF, E);1330  if (E->isArray())1331    CGF.EmitNewArrayInitializer(E, ElementType, ElementTy, NewPtr, NumElements,1332                                AllocSizeWithoutCookie);1333  else if (const Expr *Init = E->getInitializer())1334    StoreAnyExprIntoOneUnit(CGF, Init, E->getAllocatedType(), NewPtr,1335                            AggValueSlot::DoesNotOverlap);1336}1337 1338/// Emit a call to an operator new or operator delete function, as implicitly1339/// created by new-expressions and delete-expressions.1340static RValue EmitNewDeleteCall(CodeGenFunction &CGF,1341                                const FunctionDecl *CalleeDecl,1342                                const FunctionProtoType *CalleeType,1343                                const CallArgList &Args) {1344  llvm::CallBase *CallOrInvoke;1345  llvm::Constant *CalleePtr = CGF.CGM.GetAddrOfFunction(CalleeDecl);1346  CGCallee Callee = CGCallee::forDirect(CalleePtr, GlobalDecl(CalleeDecl));1347  RValue RV =1348      CGF.EmitCall(CGF.CGM.getTypes().arrangeFreeFunctionCall(1349                       Args, CalleeType, /*ChainCall=*/false),1350                   Callee, ReturnValueSlot(), Args, &CallOrInvoke);1351 1352  /// C++1y [expr.new]p10:1353  ///   [In a new-expression,] an implementation is allowed to omit a call1354  ///   to a replaceable global allocation function.1355  ///1356  /// We model such elidable calls with the 'builtin' attribute.1357  llvm::Function *Fn = dyn_cast<llvm::Function>(CalleePtr);1358  if (CalleeDecl->isReplaceableGlobalAllocationFunction() &&1359      Fn && Fn->hasFnAttribute(llvm::Attribute::NoBuiltin)) {1360    CallOrInvoke->addFnAttr(llvm::Attribute::Builtin);1361  }1362 1363  return RV;1364}1365 1366RValue CodeGenFunction::EmitBuiltinNewDeleteCall(const FunctionProtoType *Type,1367                                                 const CallExpr *TheCall,1368                                                 bool IsDelete) {1369  CallArgList Args;1370  EmitCallArgs(Args, Type, TheCall->arguments());1371  // Find the allocation or deallocation function that we're calling.1372  ASTContext &Ctx = getContext();1373  DeclarationName Name = Ctx.DeclarationNames1374      .getCXXOperatorName(IsDelete ? OO_Delete : OO_New);1375 1376  for (auto *Decl : Ctx.getTranslationUnitDecl()->lookup(Name))1377    if (auto *FD = dyn_cast<FunctionDecl>(Decl))1378      if (Ctx.hasSameType(FD->getType(), QualType(Type, 0))) {1379        RValue RV = EmitNewDeleteCall(*this, FD, Type, Args);1380        if (auto *CB = dyn_cast_if_present<llvm::CallBase>(RV.getScalarVal())) {1381          if (SanOpts.has(SanitizerKind::AllocToken)) {1382            // Set !alloc_token metadata.1383            EmitAllocToken(CB, TheCall);1384          }1385        }1386        return RV;1387      }1388  llvm_unreachable("predeclared global operator new/delete is missing");1389}1390 1391namespace {1392  /// A cleanup to call the given 'operator delete' function upon abnormal1393  /// exit from a new expression. Templated on a traits type that deals with1394  /// ensuring that the arguments dominate the cleanup if necessary.1395  template<typename Traits>1396  class CallDeleteDuringNew final : public EHScopeStack::Cleanup {1397    /// Type used to hold llvm::Value*s.1398    typedef typename Traits::ValueTy ValueTy;1399    /// Type used to hold RValues.1400    typedef typename Traits::RValueTy RValueTy;1401    struct PlacementArg {1402      RValueTy ArgValue;1403      QualType ArgType;1404    };1405 1406    unsigned NumPlacementArgs : 30;1407    LLVM_PREFERRED_TYPE(AlignedAllocationMode)1408    unsigned PassAlignmentToPlacementDelete : 1;1409    const FunctionDecl *OperatorDelete;1410    RValueTy TypeIdentity;1411    ValueTy Ptr;1412    ValueTy AllocSize;1413    CharUnits AllocAlign;1414 1415    PlacementArg *getPlacementArgs() {1416      return reinterpret_cast<PlacementArg *>(this + 1);1417    }1418 1419  public:1420    static size_t getExtraSize(size_t NumPlacementArgs) {1421      return NumPlacementArgs * sizeof(PlacementArg);1422    }1423 1424    CallDeleteDuringNew(size_t NumPlacementArgs,1425                        const FunctionDecl *OperatorDelete,1426                        RValueTy TypeIdentity, ValueTy Ptr, ValueTy AllocSize,1427                        const ImplicitAllocationParameters &IAP,1428                        CharUnits AllocAlign)1429        : NumPlacementArgs(NumPlacementArgs),1430          PassAlignmentToPlacementDelete(1431              isAlignedAllocation(IAP.PassAlignment)),1432          OperatorDelete(OperatorDelete), TypeIdentity(TypeIdentity), Ptr(Ptr),1433          AllocSize(AllocSize), AllocAlign(AllocAlign) {}1434 1435    void setPlacementArg(unsigned I, RValueTy Arg, QualType Type) {1436      assert(I < NumPlacementArgs && "index out of range");1437      getPlacementArgs()[I] = {Arg, Type};1438    }1439 1440    void Emit(CodeGenFunction &CGF, Flags flags) override {1441      const auto *FPT = OperatorDelete->getType()->castAs<FunctionProtoType>();1442      CallArgList DeleteArgs;1443      unsigned FirstNonTypeArg = 0;1444      TypeAwareAllocationMode TypeAwareDeallocation =1445          TypeAwareAllocationMode::No;1446      if (OperatorDelete->isTypeAwareOperatorNewOrDelete()) {1447        TypeAwareDeallocation = TypeAwareAllocationMode::Yes;1448        QualType SpecializedTypeIdentity = FPT->getParamType(0);1449        ++FirstNonTypeArg;1450        DeleteArgs.add(Traits::get(CGF, TypeIdentity), SpecializedTypeIdentity);1451      }1452      // The first argument after type-identity parameter (if any) is always1453      // a void* (or C* for a destroying operator delete for class type C).1454      DeleteArgs.add(Traits::get(CGF, Ptr), FPT->getParamType(FirstNonTypeArg));1455 1456      // Figure out what other parameters we should be implicitly passing.1457      UsualDeleteParams Params;1458      if (NumPlacementArgs) {1459        // A placement deallocation function is implicitly passed an alignment1460        // if the placement allocation function was, but is never passed a size.1461        Params.Alignment =1462            alignedAllocationModeFromBool(PassAlignmentToPlacementDelete);1463        Params.TypeAwareDelete = TypeAwareDeallocation;1464        Params.Size = isTypeAwareAllocation(Params.TypeAwareDelete);1465      } else {1466        // For a non-placement new-expression, 'operator delete' can take a1467        // size and/or an alignment if it has the right parameters.1468        Params = OperatorDelete->getUsualDeleteParams();1469      }1470 1471      assert(!Params.DestroyingDelete &&1472             "should not call destroying delete in a new-expression");1473 1474      // The second argument can be a std::size_t (for non-placement delete).1475      if (Params.Size)1476        DeleteArgs.add(Traits::get(CGF, AllocSize),1477                       CGF.getContext().getSizeType());1478 1479      // The next (second or third) argument can be a std::align_val_t, which1480      // is an enum whose underlying type is std::size_t.1481      // FIXME: Use the right type as the parameter type. Note that in a call1482      // to operator delete(size_t, ...), we may not have it available.1483      if (isAlignedAllocation(Params.Alignment))1484        DeleteArgs.add(RValue::get(llvm::ConstantInt::get(1485                           CGF.SizeTy, AllocAlign.getQuantity())),1486                       CGF.getContext().getSizeType());1487 1488      // Pass the rest of the arguments, which must match exactly.1489      for (unsigned I = 0; I != NumPlacementArgs; ++I) {1490        auto Arg = getPlacementArgs()[I];1491        DeleteArgs.add(Traits::get(CGF, Arg.ArgValue), Arg.ArgType);1492      }1493 1494      // Call 'operator delete'.1495      EmitNewDeleteCall(CGF, OperatorDelete, FPT, DeleteArgs);1496    }1497  };1498}1499 1500/// Enter a cleanup to call 'operator delete' if the initializer in a1501/// new-expression throws.1502static void EnterNewDeleteCleanup(CodeGenFunction &CGF, const CXXNewExpr *E,1503                                  RValue TypeIdentity, Address NewPtr,1504                                  llvm::Value *AllocSize, CharUnits AllocAlign,1505                                  const CallArgList &NewArgs) {1506  unsigned NumNonPlacementArgs = E->getNumImplicitArgs();1507 1508  // If we're not inside a conditional branch, then the cleanup will1509  // dominate and we can do the easier (and more efficient) thing.1510  if (!CGF.isInConditionalBranch()) {1511    struct DirectCleanupTraits {1512      typedef llvm::Value *ValueTy;1513      typedef RValue RValueTy;1514      static RValue get(CodeGenFunction &, ValueTy V) { return RValue::get(V); }1515      static RValue get(CodeGenFunction &, RValueTy V) { return V; }1516    };1517 1518    typedef CallDeleteDuringNew<DirectCleanupTraits> DirectCleanup;1519 1520    DirectCleanup *Cleanup = CGF.EHStack.pushCleanupWithExtra<DirectCleanup>(1521        EHCleanup, E->getNumPlacementArgs(), E->getOperatorDelete(),1522        TypeIdentity, NewPtr.emitRawPointer(CGF), AllocSize,1523        E->implicitAllocationParameters(), AllocAlign);1524    for (unsigned I = 0, N = E->getNumPlacementArgs(); I != N; ++I) {1525      auto &Arg = NewArgs[I + NumNonPlacementArgs];1526      Cleanup->setPlacementArg(I, Arg.getRValue(CGF), Arg.Ty);1527    }1528 1529    return;1530  }1531 1532  // Otherwise, we need to save all this stuff.1533  DominatingValue<RValue>::saved_type SavedNewPtr =1534      DominatingValue<RValue>::save(CGF, RValue::get(NewPtr, CGF));1535  DominatingValue<RValue>::saved_type SavedAllocSize =1536    DominatingValue<RValue>::save(CGF, RValue::get(AllocSize));1537  DominatingValue<RValue>::saved_type SavedTypeIdentity =1538      DominatingValue<RValue>::save(CGF, TypeIdentity);1539  struct ConditionalCleanupTraits {1540    typedef DominatingValue<RValue>::saved_type ValueTy;1541    typedef DominatingValue<RValue>::saved_type RValueTy;1542    static RValue get(CodeGenFunction &CGF, ValueTy V) {1543      return V.restore(CGF);1544    }1545  };1546  typedef CallDeleteDuringNew<ConditionalCleanupTraits> ConditionalCleanup;1547 1548  ConditionalCleanup *Cleanup =1549      CGF.EHStack.pushCleanupWithExtra<ConditionalCleanup>(1550          EHCleanup, E->getNumPlacementArgs(), E->getOperatorDelete(),1551          SavedTypeIdentity, SavedNewPtr, SavedAllocSize,1552          E->implicitAllocationParameters(), AllocAlign);1553  for (unsigned I = 0, N = E->getNumPlacementArgs(); I != N; ++I) {1554    auto &Arg = NewArgs[I + NumNonPlacementArgs];1555    Cleanup->setPlacementArg(1556        I, DominatingValue<RValue>::save(CGF, Arg.getRValue(CGF)), Arg.Ty);1557  }1558 1559  CGF.initFullExprCleanup();1560}1561 1562llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {1563  // The element type being allocated.1564  QualType allocType = getContext().getBaseElementType(E->getAllocatedType());1565 1566  // 1. Build a call to the allocation function.1567  FunctionDecl *allocator = E->getOperatorNew();1568 1569  // If there is a brace-initializer or C++20 parenthesized initializer, cannot1570  // allocate fewer elements than inits.1571  unsigned minElements = 0;1572  unsigned IndexOfAlignArg = 1;1573  if (E->isArray() && E->hasInitializer()) {1574    const Expr *Init = E->getInitializer();1575    const InitListExpr *ILE = dyn_cast<InitListExpr>(Init);1576    const CXXParenListInitExpr *CPLIE = dyn_cast<CXXParenListInitExpr>(Init);1577    const Expr *IgnoreParen = Init->IgnoreParenImpCasts();1578    if ((ILE && ILE->isStringLiteralInit()) ||1579        isa<StringLiteral>(IgnoreParen) || isa<ObjCEncodeExpr>(IgnoreParen)) {1580      minElements =1581          cast<ConstantArrayType>(Init->getType()->getAsArrayTypeUnsafe())1582              ->getZExtSize();1583    } else if (ILE || CPLIE) {1584      minElements = ILE ? ILE->getNumInits() : CPLIE->getInitExprs().size();1585    }1586  }1587 1588  llvm::Value *numElements = nullptr;1589  llvm::Value *allocSizeWithoutCookie = nullptr;1590  llvm::Value *allocSize =1591    EmitCXXNewAllocSize(*this, E, minElements, numElements,1592                        allocSizeWithoutCookie);1593  CharUnits allocAlign = getContext().getTypeAlignInChars(allocType);1594 1595  // Emit the allocation call.  If the allocator is a global placement1596  // operator, just "inline" it directly.1597  Address allocation = Address::invalid();1598  CallArgList allocatorArgs;1599  RValue TypeIdentityArg;1600  if (allocator->isReservedGlobalPlacementOperator()) {1601    assert(E->getNumPlacementArgs() == 1);1602    const Expr *arg = *E->placement_arguments().begin();1603 1604    LValueBaseInfo BaseInfo;1605    allocation = EmitPointerWithAlignment(arg, &BaseInfo);1606 1607    // The pointer expression will, in many cases, be an opaque void*.1608    // In these cases, discard the computed alignment and use the1609    // formal alignment of the allocated type.1610    if (BaseInfo.getAlignmentSource() != AlignmentSource::Decl)1611      allocation.setAlignment(allocAlign);1612 1613    // Set up allocatorArgs for the call to operator delete if it's not1614    // the reserved global operator.1615    if (E->getOperatorDelete() &&1616        !E->getOperatorDelete()->isReservedGlobalPlacementOperator()) {1617      allocatorArgs.add(RValue::get(allocSize), getContext().getSizeType());1618      allocatorArgs.add(RValue::get(allocation, *this), arg->getType());1619    }1620 1621  } else {1622    const FunctionProtoType *allocatorType =1623      allocator->getType()->castAs<FunctionProtoType>();1624    ImplicitAllocationParameters IAP = E->implicitAllocationParameters();1625    unsigned ParamsToSkip = 0;1626    if (isTypeAwareAllocation(IAP.PassTypeIdentity)) {1627      QualType SpecializedTypeIdentity = allocatorType->getParamType(0);1628      CXXScalarValueInitExpr TypeIdentityParam(SpecializedTypeIdentity, nullptr,1629                                               SourceLocation());1630      TypeIdentityArg = EmitAnyExprToTemp(&TypeIdentityParam);1631      allocatorArgs.add(TypeIdentityArg, SpecializedTypeIdentity);1632      ++ParamsToSkip;1633      ++IndexOfAlignArg;1634    }1635    // The allocation size is the first argument.1636    QualType sizeType = getContext().getSizeType();1637    allocatorArgs.add(RValue::get(allocSize), sizeType);1638    ++ParamsToSkip;1639 1640    if (allocSize != allocSizeWithoutCookie) {1641      CharUnits cookieAlign = getSizeAlign(); // FIXME: Ask the ABI.1642      allocAlign = std::max(allocAlign, cookieAlign);1643    }1644 1645    // The allocation alignment may be passed as the second argument.1646    if (isAlignedAllocation(IAP.PassAlignment)) {1647      QualType AlignValT = sizeType;1648      if (allocatorType->getNumParams() > IndexOfAlignArg) {1649        AlignValT = allocatorType->getParamType(IndexOfAlignArg);1650        assert(getContext().hasSameUnqualifiedType(1651                   AlignValT->castAsEnumDecl()->getIntegerType(), sizeType) &&1652               "wrong type for alignment parameter");1653        ++ParamsToSkip;1654      } else {1655        // Corner case, passing alignment to 'operator new(size_t, ...)'.1656        assert(allocator->isVariadic() && "can't pass alignment to allocator");1657      }1658      allocatorArgs.add(1659          RValue::get(llvm::ConstantInt::get(SizeTy, allocAlign.getQuantity())),1660          AlignValT);1661    }1662 1663    // FIXME: Why do we not pass a CalleeDecl here?1664    EmitCallArgs(allocatorArgs, allocatorType, E->placement_arguments(),1665                 /*AC*/AbstractCallee(), /*ParamsToSkip*/ParamsToSkip);1666 1667    RValue RV =1668      EmitNewDeleteCall(*this, allocator, allocatorType, allocatorArgs);1669 1670    if (auto *newCall = dyn_cast<llvm::CallBase>(RV.getScalarVal())) {1671      if (auto *CGDI = getDebugInfo()) {1672        // Set !heapallocsite metadata on the call to operator new.1673        CGDI->addHeapAllocSiteMetadata(newCall, allocType, E->getExprLoc());1674      }1675      if (SanOpts.has(SanitizerKind::AllocToken)) {1676        // Set !alloc_token metadata.1677        EmitAllocToken(newCall, allocType);1678      }1679    }1680 1681    // If this was a call to a global replaceable allocation function that does1682    // not take an alignment argument, the allocator is known to produce1683    // storage that's suitably aligned for any object that fits, up to a known1684    // threshold. Otherwise assume it's suitably aligned for the allocated type.1685    CharUnits allocationAlign = allocAlign;1686    if (!E->passAlignment() &&1687        allocator->isReplaceableGlobalAllocationFunction()) {1688      unsigned AllocatorAlign = llvm::bit_floor(std::min<uint64_t>(1689          Target.getNewAlign(), getContext().getTypeSize(allocType)));1690      allocationAlign = std::max(1691          allocationAlign, getContext().toCharUnitsFromBits(AllocatorAlign));1692    }1693 1694    allocation = Address(RV.getScalarVal(), Int8Ty, allocationAlign);1695  }1696 1697  // Emit a null check on the allocation result if the allocation1698  // function is allowed to return null (because it has a non-throwing1699  // exception spec or is the reserved placement new) and we have an1700  // interesting initializer will be running sanitizers on the initialization.1701  bool nullCheck = E->shouldNullCheckAllocation() &&1702                   (!allocType.isPODType(getContext()) || E->hasInitializer() ||1703                    sanitizePerformTypeCheck());1704 1705  llvm::BasicBlock *nullCheckBB = nullptr;1706  llvm::BasicBlock *contBB = nullptr;1707 1708  // The null-check means that the initializer is conditionally1709  // evaluated.1710  ConditionalEvaluation conditional(*this);1711 1712  if (nullCheck) {1713    conditional.begin(*this);1714 1715    nullCheckBB = Builder.GetInsertBlock();1716    llvm::BasicBlock *notNullBB = createBasicBlock("new.notnull");1717    contBB = createBasicBlock("new.cont");1718 1719    llvm::Value *isNull = Builder.CreateIsNull(allocation, "new.isnull");1720    Builder.CreateCondBr(isNull, contBB, notNullBB);1721    EmitBlock(notNullBB);1722  }1723 1724  // If there's an operator delete, enter a cleanup to call it if an1725  // exception is thrown.1726  EHScopeStack::stable_iterator operatorDeleteCleanup;1727  llvm::Instruction *cleanupDominator = nullptr;1728  if (E->getOperatorDelete() &&1729      !E->getOperatorDelete()->isReservedGlobalPlacementOperator()) {1730    EnterNewDeleteCleanup(*this, E, TypeIdentityArg, allocation, allocSize,1731                          allocAlign, allocatorArgs);1732    operatorDeleteCleanup = EHStack.stable_begin();1733    cleanupDominator = Builder.CreateUnreachable();1734  }1735 1736  assert((allocSize == allocSizeWithoutCookie) ==1737         CalculateCookiePadding(*this, E).isZero());1738  if (allocSize != allocSizeWithoutCookie) {1739    assert(E->isArray());1740    allocation = CGM.getCXXABI().InitializeArrayCookie(*this, allocation,1741                                                       numElements,1742                                                       E, allocType);1743  }1744 1745  llvm::Type *elementTy = ConvertTypeForMem(allocType);1746  Address result = allocation.withElementType(elementTy);1747 1748  // Passing pointer through launder.invariant.group to avoid propagation of1749  // vptrs information which may be included in previous type.1750  // To not break LTO with different optimizations levels, we do it regardless1751  // of optimization level.1752  if (CGM.getCodeGenOpts().StrictVTablePointers &&1753      allocator->isReservedGlobalPlacementOperator())1754    result = Builder.CreateLaunderInvariantGroup(result);1755 1756  // Emit sanitizer checks for pointer value now, so that in the case of an1757  // array it was checked only once and not at each constructor call. We may1758  // have already checked that the pointer is non-null.1759  // FIXME: If we have an array cookie and a potentially-throwing allocator,1760  // we'll null check the wrong pointer here.1761  SanitizerSet SkippedChecks;1762  SkippedChecks.set(SanitizerKind::Null, nullCheck);1763  EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall,1764                E->getAllocatedTypeSourceInfo()->getTypeLoc().getBeginLoc(),1765                result, allocType, result.getAlignment(), SkippedChecks,1766                numElements);1767 1768  EmitNewInitializer(*this, E, allocType, elementTy, result, numElements,1769                     allocSizeWithoutCookie);1770  llvm::Value *resultPtr = result.emitRawPointer(*this);1771 1772  // Deactivate the 'operator delete' cleanup if we finished1773  // initialization.1774  if (operatorDeleteCleanup.isValid()) {1775    DeactivateCleanupBlock(operatorDeleteCleanup, cleanupDominator);1776    cleanupDominator->eraseFromParent();1777  }1778 1779  if (nullCheck) {1780    conditional.end(*this);1781 1782    llvm::BasicBlock *notNullBB = Builder.GetInsertBlock();1783    EmitBlock(contBB);1784 1785    llvm::PHINode *PHI = Builder.CreatePHI(resultPtr->getType(), 2);1786    PHI->addIncoming(resultPtr, notNullBB);1787    PHI->addIncoming(llvm::Constant::getNullValue(resultPtr->getType()),1788                     nullCheckBB);1789 1790    resultPtr = PHI;1791  }1792 1793  return resultPtr;1794}1795 1796void CodeGenFunction::EmitDeleteCall(const FunctionDecl *DeleteFD,1797                                     llvm::Value *DeletePtr, QualType DeleteTy,1798                                     llvm::Value *NumElements,1799                                     CharUnits CookieSize) {1800  assert((!NumElements && CookieSize.isZero()) ||1801         DeleteFD->getOverloadedOperator() == OO_Array_Delete);1802 1803  const auto *DeleteFTy = DeleteFD->getType()->castAs<FunctionProtoType>();1804  CallArgList DeleteArgs;1805 1806  auto Params = DeleteFD->getUsualDeleteParams();1807  auto ParamTypeIt = DeleteFTy->param_type_begin();1808 1809  std::optional<llvm::AllocaInst *> TagAlloca;1810  auto EmitTag = [&](QualType TagType, const char *TagName) {1811    assert(!TagAlloca);1812    llvm::Type *Ty = getTypes().ConvertType(TagType);1813    CharUnits Align = CGM.getNaturalTypeAlignment(TagType);1814    llvm::AllocaInst *TagAllocation = CreateTempAlloca(Ty, TagName);1815    TagAllocation->setAlignment(Align.getAsAlign());1816    DeleteArgs.add(RValue::getAggregate(Address(TagAllocation, Ty, Align)),1817                   TagType);1818    TagAlloca = TagAllocation;1819  };1820 1821  // Pass std::type_identity tag if present1822  if (isTypeAwareAllocation(Params.TypeAwareDelete))1823    EmitTag(*ParamTypeIt++, "typeaware.delete.tag");1824 1825  // Pass the pointer itself.1826  QualType ArgTy = *ParamTypeIt++;1827  DeleteArgs.add(RValue::get(DeletePtr), ArgTy);1828 1829  // Pass the std::destroying_delete tag if present.1830  if (Params.DestroyingDelete)1831    EmitTag(*ParamTypeIt++, "destroying.delete.tag");1832 1833  // Pass the size if the delete function has a size_t parameter.1834  if (Params.Size) {1835    QualType SizeType = *ParamTypeIt++;1836    CharUnits DeleteTypeSize = getContext().getTypeSizeInChars(DeleteTy);1837    llvm::Value *Size = llvm::ConstantInt::get(ConvertType(SizeType),1838                                               DeleteTypeSize.getQuantity());1839 1840    // For array new, multiply by the number of elements.1841    if (NumElements)1842      Size = Builder.CreateMul(Size, NumElements);1843 1844    // If there is a cookie, add the cookie size.1845    if (!CookieSize.isZero())1846      Size = Builder.CreateAdd(1847          Size, llvm::ConstantInt::get(SizeTy, CookieSize.getQuantity()));1848 1849    DeleteArgs.add(RValue::get(Size), SizeType);1850  }1851 1852  // Pass the alignment if the delete function has an align_val_t parameter.1853  if (isAlignedAllocation(Params.Alignment)) {1854    QualType AlignValType = *ParamTypeIt++;1855    CharUnits DeleteTypeAlign =1856        getContext().toCharUnitsFromBits(getContext().getTypeAlignIfKnown(1857            DeleteTy, true /* NeedsPreferredAlignment */));1858    llvm::Value *Align = llvm::ConstantInt::get(ConvertType(AlignValType),1859                                                DeleteTypeAlign.getQuantity());1860    DeleteArgs.add(RValue::get(Align), AlignValType);1861  }1862 1863  assert(ParamTypeIt == DeleteFTy->param_type_end() &&1864         "unknown parameter to usual delete function");1865 1866  // Emit the call to delete.1867  EmitNewDeleteCall(*this, DeleteFD, DeleteFTy, DeleteArgs);1868 1869  // If call argument lowering didn't use a generated tag argument alloca we1870  // remove them1871  if (TagAlloca && (*TagAlloca)->use_empty())1872    (*TagAlloca)->eraseFromParent();1873}1874namespace {1875  /// Calls the given 'operator delete' on a single object.1876  struct CallObjectDelete final : EHScopeStack::Cleanup {1877    llvm::Value *Ptr;1878    const FunctionDecl *OperatorDelete;1879    QualType ElementType;1880 1881    CallObjectDelete(llvm::Value *Ptr,1882                     const FunctionDecl *OperatorDelete,1883                     QualType ElementType)1884      : Ptr(Ptr), OperatorDelete(OperatorDelete), ElementType(ElementType) {}1885 1886    void Emit(CodeGenFunction &CGF, Flags flags) override {1887      CGF.EmitDeleteCall(OperatorDelete, Ptr, ElementType);1888    }1889  };1890}1891 1892void1893CodeGenFunction::pushCallObjectDeleteCleanup(const FunctionDecl *OperatorDelete,1894                                             llvm::Value *CompletePtr,1895                                             QualType ElementType) {1896  EHStack.pushCleanup<CallObjectDelete>(NormalAndEHCleanup, CompletePtr,1897                                        OperatorDelete, ElementType);1898}1899 1900/// Emit the code for deleting a single object with a destroying operator1901/// delete. If the element type has a non-virtual destructor, Ptr has already1902/// been converted to the type of the parameter of 'operator delete'. Otherwise1903/// Ptr points to an object of the static type.1904static void EmitDestroyingObjectDelete(CodeGenFunction &CGF,1905                                       const CXXDeleteExpr *DE, Address Ptr,1906                                       QualType ElementType) {1907  auto *Dtor = ElementType->getAsCXXRecordDecl()->getDestructor();1908  if (Dtor && Dtor->isVirtual())1909    CGF.CGM.getCXXABI().emitVirtualObjectDelete(CGF, DE, Ptr, ElementType,1910                                                Dtor);1911  else1912    CGF.EmitDeleteCall(DE->getOperatorDelete(), Ptr.emitRawPointer(CGF),1913                       ElementType);1914}1915 1916/// Emit the code for deleting a single object.1917/// \return \c true if we started emitting UnconditionalDeleteBlock, \c false1918/// if not.1919static bool EmitObjectDelete(CodeGenFunction &CGF,1920                             const CXXDeleteExpr *DE,1921                             Address Ptr,1922                             QualType ElementType,1923                             llvm::BasicBlock *UnconditionalDeleteBlock) {1924  // C++11 [expr.delete]p3:1925  //   If the static type of the object to be deleted is different from its1926  //   dynamic type, the static type shall be a base class of the dynamic type1927  //   of the object to be deleted and the static type shall have a virtual1928  //   destructor or the behavior is undefined.1929  CGF.EmitTypeCheck(CodeGenFunction::TCK_MemberCall, DE->getExprLoc(), Ptr,1930                    ElementType);1931 1932  const FunctionDecl *OperatorDelete = DE->getOperatorDelete();1933  assert(!OperatorDelete->isDestroyingOperatorDelete());1934 1935  // Find the destructor for the type, if applicable.  If the1936  // destructor is virtual, we'll just emit the vcall and return.1937  const CXXDestructorDecl *Dtor = nullptr;1938  if (const auto *RD = ElementType->getAsCXXRecordDecl()) {1939    if (RD->hasDefinition() && !RD->hasTrivialDestructor()) {1940      Dtor = RD->getDestructor();1941 1942      if (Dtor->isVirtual()) {1943        bool UseVirtualCall = true;1944        const Expr *Base = DE->getArgument();1945        if (auto *DevirtualizedDtor =1946                dyn_cast_or_null<const CXXDestructorDecl>(1947                    Dtor->getDevirtualizedMethod(1948                        Base, CGF.CGM.getLangOpts().AppleKext))) {1949          UseVirtualCall = false;1950          const CXXRecordDecl *DevirtualizedClass =1951              DevirtualizedDtor->getParent();1952          if (declaresSameEntity(getCXXRecord(Base), DevirtualizedClass)) {1953            // Devirtualized to the class of the base type (the type of the1954            // whole expression).1955            Dtor = DevirtualizedDtor;1956          } else {1957            // Devirtualized to some other type. Would need to cast the this1958            // pointer to that type but we don't have support for that yet, so1959            // do a virtual call. FIXME: handle the case where it is1960            // devirtualized to the derived type (the type of the inner1961            // expression) as in EmitCXXMemberOrOperatorMemberCallExpr.1962            UseVirtualCall = true;1963          }1964        }1965        if (UseVirtualCall) {1966          CGF.CGM.getCXXABI().emitVirtualObjectDelete(CGF, DE, Ptr, ElementType,1967                                                      Dtor);1968          return false;1969        }1970      }1971    }1972  }1973 1974  // Make sure that we call delete even if the dtor throws.1975  // This doesn't have to a conditional cleanup because we're going1976  // to pop it off in a second.1977  CGF.EHStack.pushCleanup<CallObjectDelete>(1978      NormalAndEHCleanup, Ptr.emitRawPointer(CGF), OperatorDelete, ElementType);1979 1980  if (Dtor)1981    CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete,1982                              /*ForVirtualBase=*/false,1983                              /*Delegating=*/false,1984                              Ptr, ElementType);1985  else if (auto Lifetime = ElementType.getObjCLifetime()) {1986    switch (Lifetime) {1987    case Qualifiers::OCL_None:1988    case Qualifiers::OCL_ExplicitNone:1989    case Qualifiers::OCL_Autoreleasing:1990      break;1991 1992    case Qualifiers::OCL_Strong:1993      CGF.EmitARCDestroyStrong(Ptr, ARCPreciseLifetime);1994      break;1995 1996    case Qualifiers::OCL_Weak:1997      CGF.EmitARCDestroyWeak(Ptr);1998      break;1999    }2000  }2001 2002  // When optimizing for size, call 'operator delete' unconditionally.2003  if (CGF.CGM.getCodeGenOpts().OptimizeSize > 1) {2004    CGF.EmitBlock(UnconditionalDeleteBlock);2005    CGF.PopCleanupBlock();2006    return true;2007  }2008 2009  CGF.PopCleanupBlock();2010  return false;2011}2012 2013namespace {2014  /// Calls the given 'operator delete' on an array of objects.2015  struct CallArrayDelete final : EHScopeStack::Cleanup {2016    llvm::Value *Ptr;2017    const FunctionDecl *OperatorDelete;2018    llvm::Value *NumElements;2019    QualType ElementType;2020    CharUnits CookieSize;2021 2022    CallArrayDelete(llvm::Value *Ptr,2023                    const FunctionDecl *OperatorDelete,2024                    llvm::Value *NumElements,2025                    QualType ElementType,2026                    CharUnits CookieSize)2027      : Ptr(Ptr), OperatorDelete(OperatorDelete), NumElements(NumElements),2028        ElementType(ElementType), CookieSize(CookieSize) {}2029 2030    void Emit(CodeGenFunction &CGF, Flags flags) override {2031      CGF.EmitDeleteCall(OperatorDelete, Ptr, ElementType, NumElements,2032                         CookieSize);2033    }2034  };2035}2036 2037/// Emit the code for deleting an array of objects.2038static void EmitArrayDelete(CodeGenFunction &CGF,2039                            const CXXDeleteExpr *E,2040                            Address deletedPtr,2041                            QualType elementType) {2042  llvm::Value *numElements = nullptr;2043  llvm::Value *allocatedPtr = nullptr;2044  CharUnits cookieSize;2045  CGF.CGM.getCXXABI().ReadArrayCookie(CGF, deletedPtr, E, elementType,2046                                      numElements, allocatedPtr, cookieSize);2047 2048  assert(allocatedPtr && "ReadArrayCookie didn't set allocated pointer");2049 2050  // Make sure that we call delete even if one of the dtors throws.2051  const FunctionDecl *operatorDelete = E->getOperatorDelete();2052  CGF.EHStack.pushCleanup<CallArrayDelete>(NormalAndEHCleanup,2053                                           allocatedPtr, operatorDelete,2054                                           numElements, elementType,2055                                           cookieSize);2056 2057  // Destroy the elements.2058  if (QualType::DestructionKind dtorKind = elementType.isDestructedType()) {2059    assert(numElements && "no element count for a type with a destructor!");2060 2061    CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType);2062    CharUnits elementAlign =2063      deletedPtr.getAlignment().alignmentOfArrayElement(elementSize);2064 2065    llvm::Value *arrayBegin = deletedPtr.emitRawPointer(CGF);2066    llvm::Value *arrayEnd = CGF.Builder.CreateInBoundsGEP(2067      deletedPtr.getElementType(), arrayBegin, numElements, "delete.end");2068 2069    // Note that it is legal to allocate a zero-length array, and we2070    // can never fold the check away because the length should always2071    // come from a cookie.2072    CGF.emitArrayDestroy(arrayBegin, arrayEnd, elementType, elementAlign,2073                         CGF.getDestroyer(dtorKind),2074                         /*checkZeroLength*/ true,2075                         CGF.needsEHCleanup(dtorKind));2076  }2077 2078  // Pop the cleanup block.2079  CGF.PopCleanupBlock();2080}2081 2082void CodeGenFunction::EmitCXXDeleteExpr(const CXXDeleteExpr *E) {2083  const Expr *Arg = E->getArgument();2084  Address Ptr = EmitPointerWithAlignment(Arg);2085 2086  // Null check the pointer.2087  //2088  // We could avoid this null check if we can determine that the object2089  // destruction is trivial and doesn't require an array cookie; we can2090  // unconditionally perform the operator delete call in that case. For now, we2091  // assume that deleted pointers are null rarely enough that it's better to2092  // keep the branch. This might be worth revisiting for a -O0 code size win.2093  llvm::BasicBlock *DeleteNotNull = createBasicBlock("delete.notnull");2094  llvm::BasicBlock *DeleteEnd = createBasicBlock("delete.end");2095 2096  llvm::Value *IsNull = Builder.CreateIsNull(Ptr, "isnull");2097 2098  Builder.CreateCondBr(IsNull, DeleteEnd, DeleteNotNull);2099  EmitBlock(DeleteNotNull);2100  Ptr.setKnownNonNull();2101 2102  QualType DeleteTy = E->getDestroyedType();2103 2104  // A destroying operator delete overrides the entire operation of the2105  // delete expression.2106  if (E->getOperatorDelete()->isDestroyingOperatorDelete()) {2107    EmitDestroyingObjectDelete(*this, E, Ptr, DeleteTy);2108    EmitBlock(DeleteEnd);2109    return;2110  }2111 2112  // We might be deleting a pointer to array.2113  DeleteTy = getContext().getBaseElementType(DeleteTy);2114  Ptr = Ptr.withElementType(ConvertTypeForMem(DeleteTy));2115 2116  if (E->isArrayForm()) {2117    EmitArrayDelete(*this, E, Ptr, DeleteTy);2118    EmitBlock(DeleteEnd);2119  } else {2120    if (!EmitObjectDelete(*this, E, Ptr, DeleteTy, DeleteEnd))2121      EmitBlock(DeleteEnd);2122  }2123}2124 2125static llvm::Value *EmitTypeidFromVTable(CodeGenFunction &CGF, const Expr *E,2126                                         llvm::Type *StdTypeInfoPtrTy,2127                                         bool HasNullCheck) {2128  // Get the vtable pointer.2129  Address ThisPtr = CGF.EmitLValue(E).getAddress();2130 2131  QualType SrcRecordTy = E->getType();2132 2133  // C++ [class.cdtor]p4:2134  //   If the operand of typeid refers to the object under construction or2135  //   destruction and the static type of the operand is neither the constructor2136  //   or destructor’s class nor one of its bases, the behavior is undefined.2137  CGF.EmitTypeCheck(CodeGenFunction::TCK_DynamicOperation, E->getExprLoc(),2138                    ThisPtr, SrcRecordTy);2139 2140  // Whether we need an explicit null pointer check. For example, with the2141  // Microsoft ABI, if this is a call to __RTtypeid, the null pointer check and2142  // exception throw is inside the __RTtypeid(nullptr) call2143  if (HasNullCheck &&2144      CGF.CGM.getCXXABI().shouldTypeidBeNullChecked(SrcRecordTy)) {2145    llvm::BasicBlock *BadTypeidBlock =2146        CGF.createBasicBlock("typeid.bad_typeid");2147    llvm::BasicBlock *EndBlock = CGF.createBasicBlock("typeid.end");2148 2149    llvm::Value *IsNull = CGF.Builder.CreateIsNull(ThisPtr);2150    CGF.Builder.CreateCondBr(IsNull, BadTypeidBlock, EndBlock);2151 2152    CGF.EmitBlock(BadTypeidBlock);2153    CGF.CGM.getCXXABI().EmitBadTypeidCall(CGF);2154    CGF.EmitBlock(EndBlock);2155  }2156 2157  return CGF.CGM.getCXXABI().EmitTypeid(CGF, SrcRecordTy, ThisPtr,2158                                        StdTypeInfoPtrTy);2159}2160 2161llvm::Value *CodeGenFunction::EmitCXXTypeidExpr(const CXXTypeidExpr *E) {2162  // Ideally, we would like to use GlobalsInt8PtrTy here, however, we cannot,2163  // primarily because the result of applying typeid is a value of type2164  // type_info, which is declared & defined by the standard library2165  // implementation and expects to operate on the generic (default) AS.2166  // https://reviews.llvm.org/D157452 has more context, and a possible solution.2167  llvm::Type *PtrTy = Int8PtrTy;2168  LangAS GlobAS = CGM.GetGlobalVarAddressSpace(nullptr);2169 2170  auto MaybeASCast = [=](auto &&TypeInfo) {2171    if (GlobAS == LangAS::Default)2172      return TypeInfo;2173    return getTargetHooks().performAddrSpaceCast(CGM, TypeInfo, GlobAS, PtrTy);2174  };2175 2176  if (E->isTypeOperand()) {2177    llvm::Constant *TypeInfo =2178        CGM.GetAddrOfRTTIDescriptor(E->getTypeOperand(getContext()));2179    return MaybeASCast(TypeInfo);2180  }2181 2182  // C++ [expr.typeid]p2:2183  //   When typeid is applied to a glvalue expression whose type is a2184  //   polymorphic class type, the result refers to a std::type_info object2185  //   representing the type of the most derived object (that is, the dynamic2186  //   type) to which the glvalue refers.2187  // If the operand is already most derived object, no need to look up vtable.2188  if (E->isPotentiallyEvaluated() && !E->isMostDerived(getContext()))2189    return EmitTypeidFromVTable(*this, E->getExprOperand(), PtrTy,2190                                E->hasNullCheck());2191 2192  QualType OperandTy = E->getExprOperand()->getType();2193  return MaybeASCast(CGM.GetAddrOfRTTIDescriptor(OperandTy));2194}2195 2196static llvm::Value *EmitDynamicCastToNull(CodeGenFunction &CGF,2197                                          QualType DestTy) {2198  llvm::Type *DestLTy = CGF.ConvertType(DestTy);2199  if (DestTy->isPointerType())2200    return llvm::Constant::getNullValue(DestLTy);2201 2202  /// C++ [expr.dynamic.cast]p9:2203  ///   A failed cast to reference type throws std::bad_cast2204  if (!CGF.CGM.getCXXABI().EmitBadCastCall(CGF))2205    return nullptr;2206 2207  CGF.Builder.ClearInsertionPoint();2208  return llvm::PoisonValue::get(DestLTy);2209}2210 2211llvm::Value *CodeGenFunction::EmitDynamicCast(Address ThisAddr,2212                                              const CXXDynamicCastExpr *DCE) {2213  CGM.EmitExplicitCastExprType(DCE, this);2214  QualType DestTy = DCE->getTypeAsWritten();2215 2216  QualType SrcTy = DCE->getSubExpr()->getType();2217 2218  // C++ [expr.dynamic.cast]p7:2219  //   If T is "pointer to cv void," then the result is a pointer to the most2220  //   derived object pointed to by v.2221  bool IsDynamicCastToVoid = DestTy->isVoidPointerType();2222  QualType SrcRecordTy;2223  QualType DestRecordTy;2224  if (IsDynamicCastToVoid) {2225    SrcRecordTy = SrcTy->getPointeeType();2226    // No DestRecordTy.2227  } else if (const PointerType *DestPTy = DestTy->getAs<PointerType>()) {2228    SrcRecordTy = SrcTy->castAs<PointerType>()->getPointeeType();2229    DestRecordTy = DestPTy->getPointeeType();2230  } else {2231    SrcRecordTy = SrcTy;2232    DestRecordTy = DestTy->castAs<ReferenceType>()->getPointeeType();2233  }2234 2235  // C++ [class.cdtor]p5:2236  //   If the operand of the dynamic_cast refers to the object under2237  //   construction or destruction and the static type of the operand is not a2238  //   pointer to or object of the constructor or destructor’s own class or one2239  //   of its bases, the dynamic_cast results in undefined behavior.2240  EmitTypeCheck(TCK_DynamicOperation, DCE->getExprLoc(), ThisAddr, SrcRecordTy);2241 2242  if (DCE->isAlwaysNull()) {2243    if (llvm::Value *T = EmitDynamicCastToNull(*this, DestTy)) {2244      // Expression emission is expected to retain a valid insertion point.2245      if (!Builder.GetInsertBlock())2246        EmitBlock(createBasicBlock("dynamic_cast.unreachable"));2247      return T;2248    }2249  }2250 2251  assert(SrcRecordTy->isRecordType() && "source type must be a record type!");2252 2253  // If the destination is effectively final, the cast succeeds if and only2254  // if the dynamic type of the pointer is exactly the destination type.2255  bool IsExact = !IsDynamicCastToVoid &&2256                 CGM.getCodeGenOpts().OptimizationLevel > 0 &&2257                 DestRecordTy->getAsCXXRecordDecl()->isEffectivelyFinal() &&2258                 CGM.getCXXABI().shouldEmitExactDynamicCast(DestRecordTy);2259 2260  std::optional<CGCXXABI::ExactDynamicCastInfo> ExactCastInfo;2261  if (IsExact) {2262    ExactCastInfo = CGM.getCXXABI().getExactDynamicCastInfo(SrcRecordTy, DestTy,2263                                                            DestRecordTy);2264    if (!ExactCastInfo) {2265      llvm::Value *NullValue = EmitDynamicCastToNull(*this, DestTy);2266      if (!Builder.GetInsertBlock())2267        EmitBlock(createBasicBlock("dynamic_cast.unreachable"));2268      return NullValue;2269    }2270  }2271 2272  // C++ [expr.dynamic.cast]p4:2273  //   If the value of v is a null pointer value in the pointer case, the result2274  //   is the null pointer value of type T.2275  bool ShouldNullCheckSrcValue =2276      IsExact || CGM.getCXXABI().shouldDynamicCastCallBeNullChecked(2277                     SrcTy->isPointerType(), SrcRecordTy);2278 2279  llvm::BasicBlock *CastNull = nullptr;2280  llvm::BasicBlock *CastNotNull = nullptr;2281  llvm::BasicBlock *CastEnd = createBasicBlock("dynamic_cast.end");2282 2283  if (ShouldNullCheckSrcValue) {2284    CastNull = createBasicBlock("dynamic_cast.null");2285    CastNotNull = createBasicBlock("dynamic_cast.notnull");2286 2287    llvm::Value *IsNull = Builder.CreateIsNull(ThisAddr);2288    Builder.CreateCondBr(IsNull, CastNull, CastNotNull);2289    EmitBlock(CastNotNull);2290  }2291 2292  llvm::Value *Value;2293  if (IsDynamicCastToVoid) {2294    Value = CGM.getCXXABI().emitDynamicCastToVoid(*this, ThisAddr, SrcRecordTy);2295  } else if (IsExact) {2296    // If the destination type is effectively final, this pointer points to the2297    // right type if and only if its vptr has the right value.2298    Value = CGM.getCXXABI().emitExactDynamicCast(2299        *this, ThisAddr, SrcRecordTy, DestTy, DestRecordTy, *ExactCastInfo,2300        CastEnd, CastNull);2301  } else {2302    assert(DestRecordTy->isRecordType() &&2303           "destination type must be a record type!");2304    Value = CGM.getCXXABI().emitDynamicCastCall(*this, ThisAddr, SrcRecordTy,2305                                                DestTy, DestRecordTy, CastEnd);2306  }2307  CastNotNull = Builder.GetInsertBlock();2308 2309  llvm::Value *NullValue = nullptr;2310  if (ShouldNullCheckSrcValue) {2311    EmitBranch(CastEnd);2312 2313    EmitBlock(CastNull);2314    NullValue = EmitDynamicCastToNull(*this, DestTy);2315    CastNull = Builder.GetInsertBlock();2316 2317    EmitBranch(CastEnd);2318  }2319 2320  EmitBlock(CastEnd);2321 2322  if (CastNull) {2323    llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2);2324    PHI->addIncoming(Value, CastNotNull);2325    PHI->addIncoming(NullValue, CastNull);2326 2327    Value = PHI;2328  }2329 2330  return Value;2331}2332