brintos / llvm-project-archived public Read only

1//===--- CGClass.cpp - Emit LLVM Code for C++ classes -----------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This contains code dealing with C++ code generation of classes10//11//===----------------------------------------------------------------------===//12 13#include "ABIInfoImpl.h"14#include "CGBlocks.h"15#include "CGCXXABI.h"16#include "CGDebugInfo.h"17#include "CGRecordLayout.h"18#include "CodeGenFunction.h"19#include "TargetInfo.h"20#include "clang/AST/Attr.h"21#include "clang/AST/CXXInheritance.h"22#include "clang/AST/CharUnits.h"23#include "clang/AST/DeclTemplate.h"24#include "clang/AST/EvaluatedExprVisitor.h"25#include "clang/AST/RecordLayout.h"26#include "clang/AST/StmtCXX.h"27#include "clang/Basic/CodeGenOptions.h"28#include "clang/CodeGen/CGFunctionInfo.h"29#include "llvm/IR/Intrinsics.h"30#include "llvm/IR/Metadata.h"31#include "llvm/Support/SaveAndRestore.h"32#include "llvm/Transforms/Utils/SanitizerStats.h"33#include <optional>34 35using namespace clang;36using namespace CodeGen;37 38/// Return the best known alignment for an unknown pointer to a39/// particular class.40CharUnits CodeGenModule::getClassPointerAlignment(const CXXRecordDecl *RD) {41  if (!RD->hasDefinition())42    return CharUnits::One(); // Hopefully won't be used anywhere.43 44  auto &layout = getContext().getASTRecordLayout(RD);45 46  // If the class is final, then we know that the pointer points to an47  // object of that type and can use the full alignment.48  if (RD->isEffectivelyFinal())49    return layout.getAlignment();50 51  // Otherwise, we have to assume it could be a subclass.52  return layout.getNonVirtualAlignment();53}54 55/// Return the smallest possible amount of storage that might be allocated56/// starting from the beginning of an object of a particular class.57///58/// This may be smaller than sizeof(RD) if RD has virtual base classes.59CharUnits CodeGenModule::getMinimumClassObjectSize(const CXXRecordDecl *RD) {60  if (!RD->hasDefinition())61    return CharUnits::One();62 63  auto &layout = getContext().getASTRecordLayout(RD);64 65  // If the class is final, then we know that the pointer points to an66  // object of that type and can use the full alignment.67  if (RD->isEffectivelyFinal())68    return layout.getSize();69 70  // Otherwise, we have to assume it could be a subclass.71  return std::max(layout.getNonVirtualSize(), CharUnits::One());72}73 74/// Return the best known alignment for a pointer to a virtual base,75/// given the alignment of a pointer to the derived class.76CharUnits CodeGenModule::getVBaseAlignment(CharUnits actualDerivedAlign,77                                           const CXXRecordDecl *derivedClass,78                                           const CXXRecordDecl *vbaseClass) {79  // The basic idea here is that an underaligned derived pointer might80  // indicate an underaligned base pointer.81 82  assert(vbaseClass->isCompleteDefinition());83  auto &baseLayout = getContext().getASTRecordLayout(vbaseClass);84  CharUnits expectedVBaseAlign = baseLayout.getNonVirtualAlignment();85 86  return getDynamicOffsetAlignment(actualDerivedAlign, derivedClass,87                                   expectedVBaseAlign);88}89 90CharUnits91CodeGenModule::getDynamicOffsetAlignment(CharUnits actualBaseAlign,92                                         const CXXRecordDecl *baseDecl,93                                         CharUnits expectedTargetAlign) {94  // If the base is an incomplete type (which is, alas, possible with95  // member pointers), be pessimistic.96  if (!baseDecl->isCompleteDefinition())97    return std::min(actualBaseAlign, expectedTargetAlign);98 99  auto &baseLayout = getContext().getASTRecordLayout(baseDecl);100  CharUnits expectedBaseAlign = baseLayout.getNonVirtualAlignment();101 102  // If the class is properly aligned, assume the target offset is, too.103  //104  // This actually isn't necessarily the right thing to do --- if the105  // class is a complete object, but it's only properly aligned for a106  // base subobject, then the alignments of things relative to it are107  // probably off as well.  (Note that this requires the alignment of108  // the target to be greater than the NV alignment of the derived109  // class.)110  //111  // However, our approach to this kind of under-alignment can only112  // ever be best effort; after all, we're never going to propagate113  // alignments through variables or parameters.  Note, in particular,114  // that constructing a polymorphic type in an address that's less115  // than pointer-aligned will generally trap in the constructor,116  // unless we someday add some sort of attribute to change the117  // assumed alignment of 'this'.  So our goal here is pretty much118  // just to allow the user to explicitly say that a pointer is119  // under-aligned and then safely access its fields and vtables.120  if (actualBaseAlign >= expectedBaseAlign) {121    return expectedTargetAlign;122  }123 124  // Otherwise, we might be offset by an arbitrary multiple of the125  // actual alignment.  The correct adjustment is to take the min of126  // the two alignments.127  return std::min(actualBaseAlign, expectedTargetAlign);128}129 130Address CodeGenFunction::LoadCXXThisAddress() {131  assert(CurFuncDecl && "loading 'this' without a func declaration?");132  auto *MD = cast<CXXMethodDecl>(CurFuncDecl);133 134  // Lazily compute CXXThisAlignment.135  if (CXXThisAlignment.isZero()) {136    // Just use the best known alignment for the parent.137    // TODO: if we're currently emitting a complete-object ctor/dtor,138    // we can always use the complete-object alignment.139    CXXThisAlignment = CGM.getClassPointerAlignment(MD->getParent());140  }141 142  return makeNaturalAddressForPointer(143      LoadCXXThis(), MD->getFunctionObjectParameterType(), CXXThisAlignment,144      false, nullptr, nullptr, KnownNonNull);145}146 147/// Emit the address of a field using a member data pointer.148///149/// \param E Only used for emergency diagnostics150Address CodeGenFunction::EmitCXXMemberDataPointerAddress(151    const Expr *E, Address base, llvm::Value *memberPtr,152    const MemberPointerType *memberPtrType, bool IsInBounds,153    LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo) {154  // Ask the ABI to compute the actual address.155  llvm::Value *ptr = CGM.getCXXABI().EmitMemberDataPointerAddress(156      *this, E, base, memberPtr, memberPtrType, IsInBounds);157 158  QualType memberType = memberPtrType->getPointeeType();159  CharUnits memberAlign =160      CGM.getNaturalTypeAlignment(memberType, BaseInfo, TBAAInfo);161  memberAlign = CGM.getDynamicOffsetAlignment(162      base.getAlignment(), memberPtrType->getMostRecentCXXRecordDecl(),163      memberAlign);164  return Address(ptr, ConvertTypeForMem(memberPtrType->getPointeeType()),165                 memberAlign);166}167 168CharUnits CodeGenModule::computeNonVirtualBaseClassOffset(169    const CXXRecordDecl *DerivedClass, CastExpr::path_const_iterator Start,170    CastExpr::path_const_iterator End) {171  CharUnits Offset = CharUnits::Zero();172 173  const ASTContext &Context = getContext();174  const CXXRecordDecl *RD = DerivedClass;175 176  for (CastExpr::path_const_iterator I = Start; I != End; ++I) {177    const CXXBaseSpecifier *Base = *I;178    assert(!Base->isVirtual() && "Should not see virtual bases here!");179 180    // Get the layout.181    const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);182 183    const auto *BaseDecl = Base->getType()->castAsCXXRecordDecl();184    // Add the offset.185    Offset += Layout.getBaseClassOffset(BaseDecl);186 187    RD = BaseDecl;188  }189 190  return Offset;191}192 193llvm::Constant *194CodeGenModule::GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl,195                                   CastExpr::path_const_iterator PathBegin,196                                   CastExpr::path_const_iterator PathEnd) {197  assert(PathBegin != PathEnd && "Base path should not be empty!");198 199  CharUnits Offset =200      computeNonVirtualBaseClassOffset(ClassDecl, PathBegin, PathEnd);201  if (Offset.isZero())202    return nullptr;203 204  llvm::Type *PtrDiffTy =205      getTypes().ConvertType(getContext().getPointerDiffType());206 207  return llvm::ConstantInt::get(PtrDiffTy, Offset.getQuantity());208}209 210/// Gets the address of a direct base class within a complete object.211/// This should only be used for (1) non-virtual bases or (2) virtual bases212/// when the type is known to be complete (e.g. in complete destructors).213///214/// The object pointed to by 'This' is assumed to be non-null.215Address216CodeGenFunction::GetAddressOfDirectBaseInCompleteClass(Address This,217                                                   const CXXRecordDecl *Derived,218                                                   const CXXRecordDecl *Base,219                                                   bool BaseIsVirtual) {220  // 'this' must be a pointer (in some address space) to Derived.221  assert(This.getElementType() == ConvertType(Derived));222 223  // Compute the offset of the virtual base.224  CharUnits Offset;225  const ASTRecordLayout &Layout = getContext().getASTRecordLayout(Derived);226  if (BaseIsVirtual)227    Offset = Layout.getVBaseClassOffset(Base);228  else229    Offset = Layout.getBaseClassOffset(Base);230 231  // Shift and cast down to the base type.232  // TODO: for complete types, this should be possible with a GEP.233  Address V = This;234  if (!Offset.isZero()) {235    V = V.withElementType(Int8Ty);236    V = Builder.CreateConstInBoundsByteGEP(V, Offset);237  }238  return V.withElementType(ConvertType(Base));239}240 241static Address242ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, Address addr,243                                CharUnits nonVirtualOffset,244                                llvm::Value *virtualOffset,245                                const CXXRecordDecl *derivedClass,246                                const CXXRecordDecl *nearestVBase) {247  // Assert that we have something to do.248  assert(!nonVirtualOffset.isZero() || virtualOffset != nullptr);249 250  // Compute the offset from the static and dynamic components.251  llvm::Value *baseOffset;252  if (!nonVirtualOffset.isZero()) {253    llvm::Type *OffsetType =254        (CGF.CGM.getTarget().getCXXABI().isItaniumFamily() &&255         CGF.CGM.getItaniumVTableContext().isRelativeLayout())256            ? CGF.Int32Ty257            : CGF.PtrDiffTy;258    baseOffset =259        llvm::ConstantInt::get(OffsetType, nonVirtualOffset.getQuantity());260    if (virtualOffset) {261      baseOffset = CGF.Builder.CreateAdd(virtualOffset, baseOffset);262    }263  } else {264    baseOffset = virtualOffset;265  }266 267  // Apply the base offset.268  llvm::Value *ptr = addr.emitRawPointer(CGF);269  ptr = CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ptr, baseOffset, "add.ptr");270 271  // If we have a virtual component, the alignment of the result will272  // be relative only to the known alignment of that vbase.273  CharUnits alignment;274  if (virtualOffset) {275    assert(nearestVBase && "virtual offset without vbase?");276    alignment = CGF.CGM.getVBaseAlignment(addr.getAlignment(),277                                          derivedClass, nearestVBase);278  } else {279    alignment = addr.getAlignment();280  }281  alignment = alignment.alignmentAtOffset(nonVirtualOffset);282 283  return Address(ptr, CGF.Int8Ty, alignment);284}285 286Address CodeGenFunction::GetAddressOfBaseClass(287    Address Value, const CXXRecordDecl *Derived,288    CastExpr::path_const_iterator PathBegin,289    CastExpr::path_const_iterator PathEnd, bool NullCheckValue,290    SourceLocation Loc) {291  assert(PathBegin != PathEnd && "Base path should not be empty!");292 293  CastExpr::path_const_iterator Start = PathBegin;294  const CXXRecordDecl *VBase = nullptr;295 296  // Sema has done some convenient canonicalization here: if the297  // access path involved any virtual steps, the conversion path will298  // *start* with a step down to the correct virtual base subobject,299  // and hence will not require any further steps.300  if ((*Start)->isVirtual()) {301    VBase = (*Start)->getType()->castAsCXXRecordDecl();302    ++Start;303  }304 305  // Compute the static offset of the ultimate destination within its306  // allocating subobject (the virtual base, if there is one, or else307  // the "complete" object that we see).308  CharUnits NonVirtualOffset = CGM.computeNonVirtualBaseClassOffset(309      VBase ? VBase : Derived, Start, PathEnd);310 311  // If there's a virtual step, we can sometimes "devirtualize" it.312  // For now, that's limited to when the derived type is final.313  // TODO: "devirtualize" this for accesses to known-complete objects.314  if (VBase && Derived->hasAttr<FinalAttr>()) {315    const ASTRecordLayout &layout = getContext().getASTRecordLayout(Derived);316    CharUnits vBaseOffset = layout.getVBaseClassOffset(VBase);317    NonVirtualOffset += vBaseOffset;318    VBase = nullptr; // we no longer have a virtual step319  }320 321  // Get the base pointer type.322  llvm::Type *BaseValueTy = ConvertType((PathEnd[-1])->getType());323  llvm::Type *PtrTy = llvm::PointerType::get(324      CGM.getLLVMContext(), Value.getType()->getPointerAddressSpace());325 326  CanQualType DerivedTy = getContext().getCanonicalTagType(Derived);327  CharUnits DerivedAlign = CGM.getClassPointerAlignment(Derived);328 329  // If the static offset is zero and we don't have a virtual step,330  // just do a bitcast; null checks are unnecessary.331  if (NonVirtualOffset.isZero() && !VBase) {332    if (sanitizePerformTypeCheck()) {333      SanitizerSet SkippedChecks;334      SkippedChecks.set(SanitizerKind::Null, !NullCheckValue);335      EmitTypeCheck(TCK_Upcast, Loc, Value.emitRawPointer(*this), DerivedTy,336                    DerivedAlign, SkippedChecks);337    }338    return Value.withElementType(BaseValueTy);339  }340 341  llvm::BasicBlock *origBB = nullptr;342  llvm::BasicBlock *endBB = nullptr;343 344  // Skip over the offset (and the vtable load) if we're supposed to345  // null-check the pointer.346  if (NullCheckValue) {347    origBB = Builder.GetInsertBlock();348    llvm::BasicBlock *notNullBB = createBasicBlock("cast.notnull");349    endBB = createBasicBlock("cast.end");350 351    llvm::Value *isNull = Builder.CreateIsNull(Value);352    Builder.CreateCondBr(isNull, endBB, notNullBB);353    EmitBlock(notNullBB);354  }355 356  if (sanitizePerformTypeCheck()) {357    SanitizerSet SkippedChecks;358    SkippedChecks.set(SanitizerKind::Null, true);359    EmitTypeCheck(VBase ? TCK_UpcastToVirtualBase : TCK_Upcast, Loc,360                  Value.emitRawPointer(*this), DerivedTy, DerivedAlign,361                  SkippedChecks);362  }363 364  // Compute the virtual offset.365  llvm::Value *VirtualOffset = nullptr;366  if (VBase) {367    VirtualOffset =368        CGM.getCXXABI().GetVirtualBaseClassOffset(*this, Value, Derived, VBase);369  }370 371  // Apply both offsets.372  Value = ApplyNonVirtualAndVirtualOffset(*this, Value, NonVirtualOffset,373                                          VirtualOffset, Derived, VBase);374 375  // Cast to the destination type.376  Value = Value.withElementType(BaseValueTy);377 378  // Build a phi if we needed a null check.379  if (NullCheckValue) {380    llvm::BasicBlock *notNullBB = Builder.GetInsertBlock();381    Builder.CreateBr(endBB);382    EmitBlock(endBB);383 384    llvm::PHINode *PHI = Builder.CreatePHI(PtrTy, 2, "cast.result");385    PHI->addIncoming(Value.emitRawPointer(*this), notNullBB);386    PHI->addIncoming(llvm::Constant::getNullValue(PtrTy), origBB);387    Value = Value.withPointer(PHI, NotKnownNonNull);388  }389 390  return Value;391}392 393Address394CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr,395                                          const CXXRecordDecl *Derived,396                                        CastExpr::path_const_iterator PathBegin,397                                          CastExpr::path_const_iterator PathEnd,398                                          bool NullCheckValue) {399  assert(PathBegin != PathEnd && "Base path should not be empty!");400 401  CanQualType DerivedTy = getContext().getCanonicalTagType(Derived);402  llvm::Type *DerivedValueTy = ConvertType(DerivedTy);403 404  llvm::Value *NonVirtualOffset =405    CGM.GetNonVirtualBaseClassOffset(Derived, PathBegin, PathEnd);406 407  if (!NonVirtualOffset) {408    // No offset, we can just cast back.409    return BaseAddr.withElementType(DerivedValueTy);410  }411 412  llvm::BasicBlock *CastNull = nullptr;413  llvm::BasicBlock *CastNotNull = nullptr;414  llvm::BasicBlock *CastEnd = nullptr;415 416  if (NullCheckValue) {417    CastNull = createBasicBlock("cast.null");418    CastNotNull = createBasicBlock("cast.notnull");419    CastEnd = createBasicBlock("cast.end");420 421    llvm::Value *IsNull = Builder.CreateIsNull(BaseAddr);422    Builder.CreateCondBr(IsNull, CastNull, CastNotNull);423    EmitBlock(CastNotNull);424  }425 426  // Apply the offset.427  Address Addr = BaseAddr.withElementType(Int8Ty);428  Addr = Builder.CreateInBoundsGEP(429      Addr, Builder.CreateNeg(NonVirtualOffset), Int8Ty,430      CGM.getClassPointerAlignment(Derived), "sub.ptr");431 432  // Just cast.433  Addr = Addr.withElementType(DerivedValueTy);434 435  // Produce a PHI if we had a null-check.436  if (NullCheckValue) {437    Builder.CreateBr(CastEnd);438    EmitBlock(CastNull);439    Builder.CreateBr(CastEnd);440    EmitBlock(CastEnd);441 442    llvm::Value *Value = Addr.emitRawPointer(*this);443    llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2);444    PHI->addIncoming(Value, CastNotNull);445    PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), CastNull);446    return Address(PHI, Addr.getElementType(),447                   CGM.getClassPointerAlignment(Derived));448  }449 450  return Addr;451}452 453llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD,454                                              bool ForVirtualBase,455                                              bool Delegating) {456  if (!CGM.getCXXABI().NeedsVTTParameter(GD)) {457    // This constructor/destructor does not need a VTT parameter.458    return nullptr;459  }460 461  const CXXRecordDecl *RD = cast<CXXMethodDecl>(CurCodeDecl)->getParent();462  const CXXRecordDecl *Base = cast<CXXMethodDecl>(GD.getDecl())->getParent();463 464  uint64_t SubVTTIndex;465 466  if (Delegating) {467    // If this is a delegating constructor call, just load the VTT.468    return LoadCXXVTT();469  } else if (RD == Base) {470    // If the record matches the base, this is the complete ctor/dtor471    // variant calling the base variant in a class with virtual bases.472    assert(!CGM.getCXXABI().NeedsVTTParameter(CurGD) &&473           "doing no-op VTT offset in base dtor/ctor?");474    assert(!ForVirtualBase && "Can't have same class as virtual base!");475    SubVTTIndex = 0;476  } else {477    const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);478    CharUnits BaseOffset = ForVirtualBase ?479      Layout.getVBaseClassOffset(Base) :480      Layout.getBaseClassOffset(Base);481 482    SubVTTIndex =483      CGM.getVTables().getSubVTTIndex(RD, BaseSubobject(Base, BaseOffset));484    assert(SubVTTIndex != 0 && "Sub-VTT index must be greater than zero!");485  }486 487  if (CGM.getCXXABI().NeedsVTTParameter(CurGD)) {488    // A VTT parameter was passed to the constructor, use it.489    llvm::Value *VTT = LoadCXXVTT();490    return Builder.CreateConstInBoundsGEP1_64(VoidPtrTy, VTT, SubVTTIndex);491  } else {492    // We're the complete constructor, so get the VTT by name.493    llvm::GlobalValue *VTT = CGM.getVTables().GetAddrOfVTT(RD);494    return Builder.CreateConstInBoundsGEP2_64(495        VTT->getValueType(), VTT, 0, SubVTTIndex);496  }497}498 499namespace {500  /// Call the destructor for a direct base class.501  struct CallBaseDtor final : EHScopeStack::Cleanup {502    const CXXRecordDecl *BaseClass;503    bool BaseIsVirtual;504    CallBaseDtor(const CXXRecordDecl *Base, bool BaseIsVirtual)505      : BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {}506 507    void Emit(CodeGenFunction &CGF, Flags flags) override {508      const CXXRecordDecl *DerivedClass =509        cast<CXXMethodDecl>(CGF.CurCodeDecl)->getParent();510 511      const CXXDestructorDecl *D = BaseClass->getDestructor();512      // We are already inside a destructor, so presumably the object being513      // destroyed should have the expected type.514      QualType ThisTy = D->getFunctionObjectParameterType();515      Address Addr =516        CGF.GetAddressOfDirectBaseInCompleteClass(CGF.LoadCXXThisAddress(),517                                                  DerivedClass, BaseClass,518                                                  BaseIsVirtual);519      CGF.EmitCXXDestructorCall(D, Dtor_Base, BaseIsVirtual,520                                /*Delegating=*/false, Addr, ThisTy);521    }522  };523 524  /// A visitor which checks whether an initializer uses 'this' in a525  /// way which requires the vtable to be properly set.526  struct DynamicThisUseChecker : ConstEvaluatedExprVisitor<DynamicThisUseChecker> {527    typedef ConstEvaluatedExprVisitor<DynamicThisUseChecker> super;528 529    bool UsesThis;530 531    DynamicThisUseChecker(const ASTContext &C) : super(C), UsesThis(false) {}532 533    // Black-list all explicit and implicit references to 'this'.534    //535    // Do we need to worry about external references to 'this' derived536    // from arbitrary code?  If so, then anything which runs arbitrary537    // external code might potentially access the vtable.538    void VisitCXXThisExpr(const CXXThisExpr *E) { UsesThis = true; }539  };540} // end anonymous namespace541 542static bool BaseInitializerUsesThis(ASTContext &C, const Expr *Init) {543  DynamicThisUseChecker Checker(C);544  Checker.Visit(Init);545  return Checker.UsesThis;546}547 548static void EmitBaseInitializer(CodeGenFunction &CGF,549                                const CXXRecordDecl *ClassDecl,550                                CXXCtorInitializer *BaseInit) {551  assert(BaseInit->isBaseInitializer() &&552         "Must have base initializer!");553 554  Address ThisPtr = CGF.LoadCXXThisAddress();555 556  const auto *BaseClassDecl = BaseInit->getBaseClass()->castAsCXXRecordDecl();557 558  bool isBaseVirtual = BaseInit->isBaseVirtual();559 560  // If the initializer for the base (other than the constructor561  // itself) accesses 'this' in any way, we need to initialize the562  // vtables.563  if (BaseInitializerUsesThis(CGF.getContext(), BaseInit->getInit()))564    CGF.InitializeVTablePointers(ClassDecl);565 566  // We can pretend to be a complete class because it only matters for567  // virtual bases, and we only do virtual bases for complete ctors.568  Address V =569    CGF.GetAddressOfDirectBaseInCompleteClass(ThisPtr, ClassDecl,570                                              BaseClassDecl,571                                              isBaseVirtual);572  AggValueSlot AggSlot =573      AggValueSlot::forAddr(574          V, Qualifiers(),575          AggValueSlot::IsDestructed,576          AggValueSlot::DoesNotNeedGCBarriers,577          AggValueSlot::IsNotAliased,578          CGF.getOverlapForBaseInit(ClassDecl, BaseClassDecl, isBaseVirtual));579 580  CGF.EmitAggExpr(BaseInit->getInit(), AggSlot);581 582  if (CGF.CGM.getLangOpts().Exceptions &&583      !BaseClassDecl->hasTrivialDestructor())584    CGF.EHStack.pushCleanup<CallBaseDtor>(EHCleanup, BaseClassDecl,585                                          isBaseVirtual);586}587 588static bool isMemcpyEquivalentSpecialMember(const CXXMethodDecl *D) {589  auto *CD = dyn_cast<CXXConstructorDecl>(D);590  if (!(CD && CD->isCopyOrMoveConstructor()) &&591      !D->isCopyAssignmentOperator() && !D->isMoveAssignmentOperator())592    return false;593 594  // We can emit a memcpy for a trivial copy or move constructor/assignment.595  if (D->isTrivial() && !D->getParent()->mayInsertExtraPadding())596    return true;597 598  // We *must* emit a memcpy for a defaulted union copy or move op.599  if (D->getParent()->isUnion() && D->isDefaulted())600    return true;601 602  return false;603}604 605static void EmitLValueForAnyFieldInitialization(CodeGenFunction &CGF,606                                                CXXCtorInitializer *MemberInit,607                                                LValue &LHS) {608  FieldDecl *Field = MemberInit->getAnyMember();609  if (MemberInit->isIndirectMemberInitializer()) {610    // If we are initializing an anonymous union field, drill down to the field.611    IndirectFieldDecl *IndirectField = MemberInit->getIndirectMember();612    for (const auto *I : IndirectField->chain())613      LHS = CGF.EmitLValueForFieldInitialization(LHS, cast<FieldDecl>(I));614  } else {615    LHS = CGF.EmitLValueForFieldInitialization(LHS, Field);616  }617}618 619static void EmitMemberInitializer(CodeGenFunction &CGF,620                                  const CXXRecordDecl *ClassDecl,621                                  CXXCtorInitializer *MemberInit,622                                  const CXXConstructorDecl *Constructor,623                                  FunctionArgList &Args) {624  ApplyAtomGroup Grp(CGF.getDebugInfo());625  ApplyDebugLocation Loc(CGF, MemberInit->getSourceLocation());626  assert(MemberInit->isAnyMemberInitializer() &&627         "Must have member initializer!");628  assert(MemberInit->getInit() && "Must have initializer!");629 630  // non-static data member initializers.631  FieldDecl *Field = MemberInit->getAnyMember();632  QualType FieldType = Field->getType();633 634  llvm::Value *ThisPtr = CGF.LoadCXXThis();635  CanQualType RecordTy = CGF.getContext().getCanonicalTagType(ClassDecl);636  LValue LHS;637 638  // If a base constructor is being emitted, create an LValue that has the639  // non-virtual alignment.640  if (CGF.CurGD.getCtorType() == Ctor_Base)641    LHS = CGF.MakeNaturalAlignPointeeAddrLValue(ThisPtr, RecordTy);642  else643    LHS = CGF.MakeNaturalAlignAddrLValue(ThisPtr, RecordTy);644 645  EmitLValueForAnyFieldInitialization(CGF, MemberInit, LHS);646 647  // Special case: if we are in a copy or move constructor, and we are copying648  // an array of PODs or classes with trivial copy constructors, ignore the649  // AST and perform the copy we know is equivalent.650  // FIXME: This is hacky at best... if we had a bit more explicit information651  // in the AST, we could generalize it more easily.652  const ConstantArrayType *Array653    = CGF.getContext().getAsConstantArrayType(FieldType);654  if (Array && Constructor->isDefaulted() &&655      Constructor->isCopyOrMoveConstructor()) {656    QualType BaseElementTy = CGF.getContext().getBaseElementType(Array);657    CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit());658    if (BaseElementTy.isPODType(CGF.getContext()) ||659        (CE && isMemcpyEquivalentSpecialMember(CE->getConstructor()))) {660      unsigned SrcArgIndex =661          CGF.CGM.getCXXABI().getSrcArgforCopyCtor(Constructor, Args);662      llvm::Value *SrcPtr663        = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(Args[SrcArgIndex]));664      LValue ThisRHSLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy);665      LValue Src = CGF.EmitLValueForFieldInitialization(ThisRHSLV, Field);666 667      // Copy the aggregate.668      CGF.EmitAggregateCopy(LHS, Src, FieldType, CGF.getOverlapForFieldInit(Field),669                            LHS.isVolatileQualified());670      // Ensure that we destroy the objects if an exception is thrown later in671      // the constructor.672      QualType::DestructionKind dtorKind = FieldType.isDestructedType();673      if (CGF.needsEHCleanup(dtorKind))674        CGF.pushEHDestroy(dtorKind, LHS.getAddress(), FieldType);675      return;676    }677  }678 679  CGF.EmitInitializerForField(Field, LHS, MemberInit->getInit());680}681 682void CodeGenFunction::EmitInitializerForField(FieldDecl *Field, LValue LHS,683                                              Expr *Init) {684  QualType FieldType = Field->getType();685  switch (getEvaluationKind(FieldType)) {686  case TEK_Scalar:687    if (LHS.isSimple()) {688      EmitExprAsInit(Init, Field, LHS, false);689    } else {690      RValue RHS = RValue::get(EmitScalarExpr(Init));691      EmitStoreThroughLValue(RHS, LHS);692    }693    break;694  case TEK_Complex:695    EmitComplexExprIntoLValue(Init, LHS, /*isInit*/ true);696    break;697  case TEK_Aggregate: {698    AggValueSlot Slot = AggValueSlot::forLValue(699        LHS, AggValueSlot::IsDestructed, AggValueSlot::DoesNotNeedGCBarriers,700        AggValueSlot::IsNotAliased, getOverlapForFieldInit(Field),701        AggValueSlot::IsNotZeroed,702        // Checks are made by the code that calls constructor.703        AggValueSlot::IsSanitizerChecked);704    EmitAggExpr(Init, Slot);705    break;706  }707  }708 709  // Ensure that we destroy this object if an exception is thrown710  // later in the constructor.711  QualType::DestructionKind dtorKind = FieldType.isDestructedType();712  if (needsEHCleanup(dtorKind))713    pushEHDestroy(dtorKind, LHS.getAddress(), FieldType);714}715 716/// Checks whether the given constructor is a valid subject for the717/// complete-to-base constructor delegation optimization, i.e.718/// emitting the complete constructor as a simple call to the base719/// constructor.720bool CodeGenFunction::IsConstructorDelegationValid(721    const CXXConstructorDecl *Ctor) {722 723  // Currently we disable the optimization for classes with virtual724  // bases because (1) the addresses of parameter variables need to be725  // consistent across all initializers but (2) the delegate function726  // call necessarily creates a second copy of the parameter variable.727  //728  // The limiting example (purely theoretical AFAIK):729  //   struct A { A(int &c) { c++; } };730  //   struct B : virtual A {731  //     B(int count) : A(count) { printf("%d\n", count); }732  //   };733  // ...although even this example could in principle be emitted as a734  // delegation since the address of the parameter doesn't escape.735  if (Ctor->getParent()->getNumVBases()) {736    // TODO: white-list trivial vbase initializers.  This case wouldn't737    // be subject to the restrictions below.738 739    // TODO: white-list cases where:740    //  - there are no non-reference parameters to the constructor741    //  - the initializers don't access any non-reference parameters742    //  - the initializers don't take the address of non-reference743    //    parameters744    //  - etc.745    // If we ever add any of the above cases, remember that:746    //  - function-try-blocks will always exclude this optimization747    //  - we need to perform the constructor prologue and cleanup in748    //    EmitConstructorBody.749 750    return false;751  }752 753  // We also disable the optimization for variadic functions because754  // it's impossible to "re-pass" varargs.755  if (Ctor->getType()->castAs<FunctionProtoType>()->isVariadic())756    return false;757 758  // FIXME: Decide if we can do a delegation of a delegating constructor.759  if (Ctor->isDelegatingConstructor())760    return false;761 762  return true;763}764 765// Emit code in ctor (Prologue==true) or dtor (Prologue==false)766// to poison the extra field paddings inserted under767// -fsanitize-address-field-padding=1|2.768void CodeGenFunction::EmitAsanPrologueOrEpilogue(bool Prologue) {769  ASTContext &Context = getContext();770  const CXXRecordDecl *ClassDecl =771      Prologue ? cast<CXXConstructorDecl>(CurGD.getDecl())->getParent()772               : cast<CXXDestructorDecl>(CurGD.getDecl())->getParent();773  if (!ClassDecl->mayInsertExtraPadding()) return;774 775  struct SizeAndOffset {776    uint64_t Size;777    uint64_t Offset;778  };779 780  unsigned PtrSize = CGM.getDataLayout().getPointerSizeInBits();781  const ASTRecordLayout &Info = Context.getASTRecordLayout(ClassDecl);782 783  // Populate sizes and offsets of fields.784  SmallVector<SizeAndOffset, 16> SSV(Info.getFieldCount());785  for (unsigned i = 0, e = Info.getFieldCount(); i != e; ++i)786    SSV[i].Offset =787        Context.toCharUnitsFromBits(Info.getFieldOffset(i)).getQuantity();788 789  size_t NumFields = 0;790  for (const auto *Field : ClassDecl->fields()) {791    const FieldDecl *D = Field;792    auto FieldInfo = Context.getTypeInfoInChars(D->getType());793    CharUnits FieldSize = FieldInfo.Width;794    assert(NumFields < SSV.size());795    SSV[NumFields].Size = D->isBitField() ? 0 : FieldSize.getQuantity();796    NumFields++;797  }798  assert(NumFields == SSV.size());799  if (SSV.size() <= 1) return;800 801  // We will insert calls to __asan_* run-time functions.802  // LLVM AddressSanitizer pass may decide to inline them later.803  llvm::Type *Args[2] = {IntPtrTy, IntPtrTy};804  llvm::FunctionType *FTy =805      llvm::FunctionType::get(CGM.VoidTy, Args, false);806  llvm::FunctionCallee F = CGM.CreateRuntimeFunction(807      FTy, Prologue ? "__asan_poison_intra_object_redzone"808                    : "__asan_unpoison_intra_object_redzone");809 810  llvm::Value *ThisPtr = LoadCXXThis();811  ThisPtr = Builder.CreatePtrToInt(ThisPtr, IntPtrTy);812  uint64_t TypeSize = Info.getNonVirtualSize().getQuantity();813  // For each field check if it has sufficient padding,814  // if so (un)poison it with a call.815  for (size_t i = 0; i < SSV.size(); i++) {816    uint64_t AsanAlignment = 8;817    uint64_t NextField = i == SSV.size() - 1 ? TypeSize : SSV[i + 1].Offset;818    uint64_t PoisonSize = NextField - SSV[i].Offset - SSV[i].Size;819    uint64_t EndOffset = SSV[i].Offset + SSV[i].Size;820    if (PoisonSize < AsanAlignment || !SSV[i].Size ||821        (NextField % AsanAlignment) != 0)822      continue;823    Builder.CreateCall(824        F, {Builder.CreateAdd(ThisPtr, Builder.getIntN(PtrSize, EndOffset)),825            Builder.getIntN(PtrSize, PoisonSize)});826  }827}828 829/// EmitConstructorBody - Emits the body of the current constructor.830void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) {831  EmitAsanPrologueOrEpilogue(true);832  const CXXConstructorDecl *Ctor = cast<CXXConstructorDecl>(CurGD.getDecl());833  CXXCtorType CtorType = CurGD.getCtorType();834 835  assert((CGM.getTarget().getCXXABI().hasConstructorVariants() ||836          CtorType == Ctor_Complete) &&837         "can only generate complete ctor for this ABI");838 839  // Before we go any further, try the complete->base constructor840  // delegation optimization.841  if (CtorType == Ctor_Complete && IsConstructorDelegationValid(Ctor) &&842      CGM.getTarget().getCXXABI().hasConstructorVariants()) {843    EmitDelegateCXXConstructorCall(Ctor, Ctor_Base, Args, Ctor->getEndLoc());844    return;845  }846 847  const FunctionDecl *Definition = nullptr;848  Stmt *Body = Ctor->getBody(Definition);849  assert(Definition == Ctor && "emitting wrong constructor body");850 851  // Enter the function-try-block before the constructor prologue if852  // applicable.853  bool IsTryBody = isa_and_nonnull<CXXTryStmt>(Body);854  if (IsTryBody)855    EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true);856 857  incrementProfileCounter(Body);858  maybeCreateMCDCCondBitmap();859 860  RunCleanupsScope RunCleanups(*this);861 862  // TODO: in restricted cases, we can emit the vbase initializers of863  // a complete ctor and then delegate to the base ctor.864 865  // Emit the constructor prologue, i.e. the base and member866  // initializers.867  EmitCtorPrologue(Ctor, CtorType, Args);868 869  // Emit the body of the statement.870  if (IsTryBody)871    EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock());872  else if (Body)873    EmitStmt(Body);874 875  // Emit any cleanup blocks associated with the member or base876  // initializers, which includes (along the exceptional path) the877  // destructors for those members and bases that were fully878  // constructed.879  RunCleanups.ForceCleanup();880 881  if (IsTryBody)882    ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true);883}884 885namespace {886  /// RAII object to indicate that codegen is copying the value representation887  /// instead of the object representation. Useful when copying a struct or888  /// class which has uninitialized members and we're only performing889  /// lvalue-to-rvalue conversion on the object but not its members.890  class CopyingValueRepresentation {891  public:892    explicit CopyingValueRepresentation(CodeGenFunction &CGF)893        : CGF(CGF), OldSanOpts(CGF.SanOpts) {894      CGF.SanOpts.set(SanitizerKind::Bool, false);895      CGF.SanOpts.set(SanitizerKind::Enum, false);896    }897    ~CopyingValueRepresentation() {898      CGF.SanOpts = OldSanOpts;899    }900  private:901    CodeGenFunction &CGF;902    SanitizerSet OldSanOpts;903  };904} // end anonymous namespace905 906namespace {907  class FieldMemcpyizer {908  public:909    FieldMemcpyizer(CodeGenFunction &CGF, const CXXRecordDecl *ClassDecl,910                    const VarDecl *SrcRec)911      : CGF(CGF), ClassDecl(ClassDecl), SrcRec(SrcRec),912        RecLayout(CGF.getContext().getASTRecordLayout(ClassDecl)),913        FirstField(nullptr), LastField(nullptr), FirstFieldOffset(0),914        LastFieldOffset(0), LastAddedFieldIndex(0) {}915 916    bool isMemcpyableField(FieldDecl *F) const {917      // Never memcpy fields when we are adding poisoned paddings.918      if (CGF.getContext().getLangOpts().SanitizeAddressFieldPadding)919        return false;920      Qualifiers Qual = F->getType().getQualifiers();921      if (Qual.hasVolatile() || Qual.hasObjCLifetime())922        return false;923      if (PointerAuthQualifier Q = F->getType().getPointerAuth();924          Q && Q.isAddressDiscriminated())925        return false;926      return true;927    }928 929    void addMemcpyableField(FieldDecl *F) {930      if (isEmptyFieldForLayout(CGF.getContext(), F))931        return;932      if (!FirstField)933        addInitialField(F);934      else935        addNextField(F);936    }937 938    CharUnits getMemcpySize(uint64_t FirstByteOffset) const {939      ASTContext &Ctx = CGF.getContext();940      unsigned LastFieldSize =941          LastField->isBitField()942              ? LastField->getBitWidthValue()943              : Ctx.toBits(944                    Ctx.getTypeInfoDataSizeInChars(LastField->getType()).Width);945      uint64_t MemcpySizeBits = LastFieldOffset + LastFieldSize -946                                FirstByteOffset + Ctx.getCharWidth() - 1;947      CharUnits MemcpySize = Ctx.toCharUnitsFromBits(MemcpySizeBits);948      return MemcpySize;949    }950 951    void emitMemcpy() {952      // Give the subclass a chance to bail out if it feels the memcpy isn't953      // worth it (e.g. Hasn't aggregated enough data).954      if (!FirstField) {955        return;956      }957 958      uint64_t FirstByteOffset;959      if (FirstField->isBitField()) {960        const CGRecordLayout &RL =961          CGF.getTypes().getCGRecordLayout(FirstField->getParent());962        const CGBitFieldInfo &BFInfo = RL.getBitFieldInfo(FirstField);963        // FirstFieldOffset is not appropriate for bitfields,964        // we need to use the storage offset instead.965        FirstByteOffset = CGF.getContext().toBits(BFInfo.StorageOffset);966      } else {967        FirstByteOffset = FirstFieldOffset;968      }969 970      CharUnits MemcpySize = getMemcpySize(FirstByteOffset);971      CanQualType RecordTy = CGF.getContext().getCanonicalTagType(ClassDecl);972      Address ThisPtr = CGF.LoadCXXThisAddress();973      LValue DestLV = CGF.MakeAddrLValue(ThisPtr, RecordTy);974      LValue Dest = CGF.EmitLValueForFieldInitialization(DestLV, FirstField);975      llvm::Value *SrcPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(SrcRec));976      LValue SrcLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy);977      LValue Src = CGF.EmitLValueForFieldInitialization(SrcLV, FirstField);978 979      emitMemcpyIR(980          Dest.isBitField() ? Dest.getBitFieldAddress() : Dest.getAddress(),981          Src.isBitField() ? Src.getBitFieldAddress() : Src.getAddress(),982          MemcpySize);983      reset();984    }985 986    void reset() {987      FirstField = nullptr;988    }989 990  protected:991    CodeGenFunction &CGF;992    const CXXRecordDecl *ClassDecl;993 994  private:995    void emitMemcpyIR(Address DestPtr, Address SrcPtr, CharUnits Size) {996      DestPtr = DestPtr.withElementType(CGF.Int8Ty);997      SrcPtr = SrcPtr.withElementType(CGF.Int8Ty);998      auto *I = CGF.Builder.CreateMemCpy(DestPtr, SrcPtr, Size.getQuantity());999      CGF.addInstToCurrentSourceAtom(I, nullptr);1000    }1001 1002    void addInitialField(FieldDecl *F) {1003      FirstField = F;1004      LastField = F;1005      FirstFieldOffset = RecLayout.getFieldOffset(F->getFieldIndex());1006      LastFieldOffset = FirstFieldOffset;1007      LastAddedFieldIndex = F->getFieldIndex();1008    }1009 1010    void addNextField(FieldDecl *F) {1011      // For the most part, the following invariant will hold:1012      //   F->getFieldIndex() == LastAddedFieldIndex + 11013      // The one exception is that Sema won't add a copy-initializer for an1014      // unnamed bitfield, which will show up here as a gap in the sequence.1015      assert(F->getFieldIndex() >= LastAddedFieldIndex + 1 &&1016             "Cannot aggregate fields out of order.");1017      LastAddedFieldIndex = F->getFieldIndex();1018 1019      // The 'first' and 'last' fields are chosen by offset, rather than field1020      // index. This allows the code to support bitfields, as well as regular1021      // fields.1022      uint64_t FOffset = RecLayout.getFieldOffset(F->getFieldIndex());1023      if (FOffset < FirstFieldOffset) {1024        FirstField = F;1025        FirstFieldOffset = FOffset;1026      } else if (FOffset >= LastFieldOffset) {1027        LastField = F;1028        LastFieldOffset = FOffset;1029      }1030    }1031 1032    const VarDecl *SrcRec;1033    const ASTRecordLayout &RecLayout;1034    FieldDecl *FirstField;1035    FieldDecl *LastField;1036    uint64_t FirstFieldOffset, LastFieldOffset;1037    unsigned LastAddedFieldIndex;1038  };1039 1040  class ConstructorMemcpyizer : public FieldMemcpyizer {1041  private:1042    /// Get source argument for copy constructor. Returns null if not a copy1043    /// constructor.1044    static const VarDecl *getTrivialCopySource(CodeGenFunction &CGF,1045                                               const CXXConstructorDecl *CD,1046                                               FunctionArgList &Args) {1047      if (CD->isCopyOrMoveConstructor() && CD->isDefaulted())1048        return Args[CGF.CGM.getCXXABI().getSrcArgforCopyCtor(CD, Args)];1049      return nullptr;1050    }1051 1052    // Returns true if a CXXCtorInitializer represents a member initialization1053    // that can be rolled into a memcpy.1054    bool isMemberInitMemcpyable(CXXCtorInitializer *MemberInit) const {1055      if (!MemcpyableCtor)1056        return false;1057      FieldDecl *Field = MemberInit->getMember();1058      assert(Field && "No field for member init.");1059      QualType FieldType = Field->getType();1060      CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit());1061 1062      // Bail out on non-memcpyable, not-trivially-copyable members.1063      if (!(CE && isMemcpyEquivalentSpecialMember(CE->getConstructor())) &&1064          !(FieldType.isTriviallyCopyableType(CGF.getContext()) ||1065            FieldType->isReferenceType()))1066        return false;1067 1068      // Bail out on volatile fields.1069      if (!isMemcpyableField(Field))1070        return false;1071 1072      // Otherwise we're good.1073      return true;1074    }1075 1076  public:1077    ConstructorMemcpyizer(CodeGenFunction &CGF, const CXXConstructorDecl *CD,1078                          FunctionArgList &Args)1079      : FieldMemcpyizer(CGF, CD->getParent(), getTrivialCopySource(CGF, CD, Args)),1080        ConstructorDecl(CD),1081        MemcpyableCtor(CD->isDefaulted() &&1082                       CD->isCopyOrMoveConstructor() &&1083                       CGF.getLangOpts().getGC() == LangOptions::NonGC),1084        Args(Args) { }1085 1086    void addMemberInitializer(CXXCtorInitializer *MemberInit) {1087      if (isMemberInitMemcpyable(MemberInit)) {1088        AggregatedInits.push_back(MemberInit);1089        addMemcpyableField(MemberInit->getMember());1090      } else {1091        emitAggregatedInits();1092        EmitMemberInitializer(CGF, ConstructorDecl->getParent(), MemberInit,1093                              ConstructorDecl, Args);1094      }1095    }1096 1097    void emitAggregatedInits() {1098      if (AggregatedInits.size() <= 1) {1099        // This memcpy is too small to be worthwhile. Fall back on default1100        // codegen.1101        if (!AggregatedInits.empty()) {1102          CopyingValueRepresentation CVR(CGF);1103          EmitMemberInitializer(CGF, ConstructorDecl->getParent(),1104                                AggregatedInits[0], ConstructorDecl, Args);1105          AggregatedInits.clear();1106        }1107        reset();1108        return;1109      }1110 1111      pushEHDestructors();1112      ApplyAtomGroup Grp(CGF.getDebugInfo());1113      emitMemcpy();1114      AggregatedInits.clear();1115    }1116 1117    void pushEHDestructors() {1118      Address ThisPtr = CGF.LoadCXXThisAddress();1119      CanQualType RecordTy = CGF.getContext().getCanonicalTagType(ClassDecl);1120      LValue LHS = CGF.MakeAddrLValue(ThisPtr, RecordTy);1121 1122      for (unsigned i = 0; i < AggregatedInits.size(); ++i) {1123        CXXCtorInitializer *MemberInit = AggregatedInits[i];1124        QualType FieldType = MemberInit->getAnyMember()->getType();1125        QualType::DestructionKind dtorKind = FieldType.isDestructedType();1126        if (!CGF.needsEHCleanup(dtorKind))1127          continue;1128        LValue FieldLHS = LHS;1129        EmitLValueForAnyFieldInitialization(CGF, MemberInit, FieldLHS);1130        CGF.pushEHDestroy(dtorKind, FieldLHS.getAddress(), FieldType);1131      }1132    }1133 1134    void finish() {1135      emitAggregatedInits();1136    }1137 1138  private:1139    const CXXConstructorDecl *ConstructorDecl;1140    bool MemcpyableCtor;1141    FunctionArgList &Args;1142    SmallVector<CXXCtorInitializer*, 16> AggregatedInits;1143  };1144 1145  class AssignmentMemcpyizer : public FieldMemcpyizer {1146  private:1147    // Returns the memcpyable field copied by the given statement, if one1148    // exists. Otherwise returns null.1149    FieldDecl *getMemcpyableField(Stmt *S) {1150      if (!AssignmentsMemcpyable)1151        return nullptr;1152      if (BinaryOperator *BO = dyn_cast<BinaryOperator>(S)) {1153        // Recognise trivial assignments.1154        if (BO->getOpcode() != BO_Assign)1155          return nullptr;1156        MemberExpr *ME = dyn_cast<MemberExpr>(BO->getLHS());1157        if (!ME)1158          return nullptr;1159        FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl());1160        if (!Field || !isMemcpyableField(Field))1161          return nullptr;1162        Stmt *RHS = BO->getRHS();1163        if (ImplicitCastExpr *EC = dyn_cast<ImplicitCastExpr>(RHS))1164          RHS = EC->getSubExpr();1165        if (!RHS)1166          return nullptr;1167        if (MemberExpr *ME2 = dyn_cast<MemberExpr>(RHS)) {1168          if (ME2->getMemberDecl() == Field)1169            return Field;1170        }1171        return nullptr;1172      } else if (CXXMemberCallExpr *MCE = dyn_cast<CXXMemberCallExpr>(S)) {1173        CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MCE->getCalleeDecl());1174        if (!(MD && isMemcpyEquivalentSpecialMember(MD)))1175          return nullptr;1176        MemberExpr *IOA = dyn_cast<MemberExpr>(MCE->getImplicitObjectArgument());1177        if (!IOA)1178          return nullptr;1179        FieldDecl *Field = dyn_cast<FieldDecl>(IOA->getMemberDecl());1180        if (!Field || !isMemcpyableField(Field))1181          return nullptr;1182        MemberExpr *Arg0 = dyn_cast<MemberExpr>(MCE->getArg(0));1183        if (!Arg0 || Field != dyn_cast<FieldDecl>(Arg0->getMemberDecl()))1184          return nullptr;1185        return Field;1186      } else if (CallExpr *CE = dyn_cast<CallExpr>(S)) {1187        FunctionDecl *FD = dyn_cast<FunctionDecl>(CE->getCalleeDecl());1188        if (!FD || FD->getBuiltinID() != Builtin::BI__builtin_memcpy)1189          return nullptr;1190        Expr *DstPtr = CE->getArg(0);1191        if (ImplicitCastExpr *DC = dyn_cast<ImplicitCastExpr>(DstPtr))1192          DstPtr = DC->getSubExpr();1193        UnaryOperator *DUO = dyn_cast<UnaryOperator>(DstPtr);1194        if (!DUO || DUO->getOpcode() != UO_AddrOf)1195          return nullptr;1196        MemberExpr *ME = dyn_cast<MemberExpr>(DUO->getSubExpr());1197        if (!ME)1198          return nullptr;1199        FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl());1200        if (!Field || !isMemcpyableField(Field))1201          return nullptr;1202        Expr *SrcPtr = CE->getArg(1);1203        if (ImplicitCastExpr *SC = dyn_cast<ImplicitCastExpr>(SrcPtr))1204          SrcPtr = SC->getSubExpr();1205        UnaryOperator *SUO = dyn_cast<UnaryOperator>(SrcPtr);1206        if (!SUO || SUO->getOpcode() != UO_AddrOf)1207          return nullptr;1208        MemberExpr *ME2 = dyn_cast<MemberExpr>(SUO->getSubExpr());1209        if (!ME2 || Field != dyn_cast<FieldDecl>(ME2->getMemberDecl()))1210          return nullptr;1211        return Field;1212      }1213 1214      return nullptr;1215    }1216 1217    bool AssignmentsMemcpyable;1218    SmallVector<Stmt*, 16> AggregatedStmts;1219 1220  public:1221    AssignmentMemcpyizer(CodeGenFunction &CGF, const CXXMethodDecl *AD,1222                         FunctionArgList &Args)1223      : FieldMemcpyizer(CGF, AD->getParent(), Args[Args.size() - 1]),1224        AssignmentsMemcpyable(CGF.getLangOpts().getGC() == LangOptions::NonGC) {1225      assert(Args.size() == 2);1226    }1227 1228    void emitAssignment(Stmt *S) {1229      FieldDecl *F = getMemcpyableField(S);1230      if (F) {1231        addMemcpyableField(F);1232        AggregatedStmts.push_back(S);1233      } else {1234        emitAggregatedStmts();1235        CGF.EmitStmt(S);1236      }1237    }1238 1239    void emitAggregatedStmts() {1240      if (AggregatedStmts.size() <= 1) {1241        if (!AggregatedStmts.empty()) {1242          CopyingValueRepresentation CVR(CGF);1243          CGF.EmitStmt(AggregatedStmts[0]);1244        }1245        reset();1246      }1247 1248      ApplyAtomGroup Grp(CGF.getDebugInfo());1249      emitMemcpy();1250      AggregatedStmts.clear();1251    }1252 1253    void finish() {1254      emitAggregatedStmts();1255    }1256  };1257} // end anonymous namespace1258 1259static bool isInitializerOfDynamicClass(const CXXCtorInitializer *BaseInit) {1260  const Type *BaseType = BaseInit->getBaseClass();1261  return BaseType->castAsCXXRecordDecl()->isDynamicClass();1262}1263 1264/// EmitCtorPrologue - This routine generates necessary code to initialize1265/// base classes and non-static data members belonging to this constructor.1266void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,1267                                       CXXCtorType CtorType,1268                                       FunctionArgList &Args) {1269  if (CD->isDelegatingConstructor())1270    return EmitDelegatingCXXConstructorCall(CD, Args);1271 1272  const CXXRecordDecl *ClassDecl = CD->getParent();1273 1274  // Virtual base initializers aren't needed if:1275  // - This is a base ctor variant1276  // - There are no vbases1277  // - The class is abstract, so a complete object of it cannot be constructed1278  //1279  // The check for an abstract class is necessary because sema may not have1280  // marked virtual base destructors referenced.1281  bool ConstructVBases = CtorType != Ctor_Base &&1282                         ClassDecl->getNumVBases() != 0 &&1283                         !ClassDecl->isAbstract();1284 1285  // In the Microsoft C++ ABI, there are no constructor variants. Instead, the1286  // constructor of a class with virtual bases takes an additional parameter to1287  // conditionally construct the virtual bases. Emit that check here.1288  llvm::BasicBlock *BaseCtorContinueBB = nullptr;1289  if (ConstructVBases &&1290      !CGM.getTarget().getCXXABI().hasConstructorVariants()) {1291    BaseCtorContinueBB =1292        CGM.getCXXABI().EmitCtorCompleteObjectHandler(*this, ClassDecl);1293    assert(BaseCtorContinueBB);1294  }1295 1296  // Create three separate ranges for the different types of initializers.1297  auto AllInits = CD->inits();1298 1299  // Find the boundaries between the three groups.1300  auto VirtualBaseEnd = std::find_if(1301      AllInits.begin(), AllInits.end(), [](const CXXCtorInitializer *Init) {1302        return !(Init->isBaseInitializer() && Init->isBaseVirtual());1303      });1304 1305  auto NonVirtualBaseEnd = std::find_if(VirtualBaseEnd, AllInits.end(),1306                                        [](const CXXCtorInitializer *Init) {1307                                          return !Init->isBaseInitializer();1308                                        });1309 1310  // Create the three ranges.1311  auto VirtualBaseInits = llvm::make_range(AllInits.begin(), VirtualBaseEnd);1312  auto NonVirtualBaseInits =1313      llvm::make_range(VirtualBaseEnd, NonVirtualBaseEnd);1314  auto MemberInits = llvm::make_range(NonVirtualBaseEnd, AllInits.end());1315 1316  // Process virtual base initializers, if necessary.1317  if (ConstructVBases) {1318    for (CXXCtorInitializer *Initializer : VirtualBaseInits) {1319      SaveAndRestore ThisRAII(CXXThisValue);1320      if (CGM.getCodeGenOpts().StrictVTablePointers &&1321          CGM.getCodeGenOpts().OptimizationLevel > 0 &&1322          isInitializerOfDynamicClass(Initializer))1323        CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());1324      EmitBaseInitializer(*this, ClassDecl, Initializer);1325    }1326  }1327 1328  if (BaseCtorContinueBB) {1329    // Complete object handler should continue to the remaining initializers.1330    Builder.CreateBr(BaseCtorContinueBB);1331    EmitBlock(BaseCtorContinueBB);1332  }1333 1334  // Then, non-virtual base initializers.1335  for (CXXCtorInitializer *Initializer : NonVirtualBaseInits) {1336    assert(!Initializer->isBaseVirtual());1337    SaveAndRestore ThisRAII(CXXThisValue);1338    if (CGM.getCodeGenOpts().StrictVTablePointers &&1339        CGM.getCodeGenOpts().OptimizationLevel > 0 &&1340        isInitializerOfDynamicClass(Initializer))1341      CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());1342    EmitBaseInitializer(*this, ClassDecl, Initializer);1343  }1344 1345  InitializeVTablePointers(ClassDecl);1346 1347  // And finally, initialize class members.1348  FieldConstructionScope FCS(*this, LoadCXXThisAddress());1349  ConstructorMemcpyizer CM(*this, CD, Args);1350  for (CXXCtorInitializer *Member : MemberInits) {1351    assert(!Member->isBaseInitializer());1352    assert(Member->isAnyMemberInitializer() &&1353           "Delegating initializer on non-delegating constructor");1354    CM.addMemberInitializer(Member);1355  }1356 1357  CM.finish();1358}1359 1360static bool1361FieldHasTrivialDestructorBody(ASTContext &Context, const FieldDecl *Field);1362 1363static bool1364HasTrivialDestructorBody(ASTContext &Context,1365                         const CXXRecordDecl *BaseClassDecl,1366                         const CXXRecordDecl *MostDerivedClassDecl)1367{1368  // If the destructor is trivial we don't have to check anything else.1369  if (BaseClassDecl->hasTrivialDestructor())1370    return true;1371 1372  if (!BaseClassDecl->getDestructor()->hasTrivialBody())1373    return false;1374 1375  // Check fields.1376  for (const auto *Field : BaseClassDecl->fields())1377    if (!FieldHasTrivialDestructorBody(Context, Field))1378      return false;1379 1380  // Check non-virtual bases.1381  for (const auto &I : BaseClassDecl->bases()) {1382    if (I.isVirtual())1383      continue;1384 1385    const auto *NonVirtualBase = I.getType()->castAsCXXRecordDecl();1386    if (!HasTrivialDestructorBody(Context, NonVirtualBase,1387                                  MostDerivedClassDecl))1388      return false;1389  }1390 1391  if (BaseClassDecl == MostDerivedClassDecl) {1392    // Check virtual bases.1393    for (const auto &I : BaseClassDecl->vbases()) {1394      const auto *VirtualBase = I.getType()->castAsCXXRecordDecl();1395      if (!HasTrivialDestructorBody(Context, VirtualBase,1396                                    MostDerivedClassDecl))1397        return false;1398    }1399  }1400 1401  return true;1402}1403 1404static bool1405FieldHasTrivialDestructorBody(ASTContext &Context,1406                                          const FieldDecl *Field)1407{1408  QualType FieldBaseElementType = Context.getBaseElementType(Field->getType());1409 1410  auto *FieldClassDecl = FieldBaseElementType->getAsCXXRecordDecl();1411  if (!FieldClassDecl)1412    return true;1413 1414  // The destructor for an implicit anonymous union member is never invoked.1415  if (FieldClassDecl->isUnion() && FieldClassDecl->isAnonymousStructOrUnion())1416    return true;1417 1418  return HasTrivialDestructorBody(Context, FieldClassDecl, FieldClassDecl);1419}1420 1421/// CanSkipVTablePointerInitialization - Check whether we need to initialize1422/// any vtable pointers before calling this destructor.1423static bool CanSkipVTablePointerInitialization(CodeGenFunction &CGF,1424                                               const CXXDestructorDecl *Dtor) {1425  const CXXRecordDecl *ClassDecl = Dtor->getParent();1426  if (!ClassDecl->isDynamicClass())1427    return true;1428 1429  // For a final class, the vtable pointer is known to already point to the1430  // class's vtable.1431  if (ClassDecl->isEffectivelyFinal())1432    return true;1433 1434  if (!Dtor->hasTrivialBody())1435    return false;1436 1437  // Check the fields.1438  for (const auto *Field : ClassDecl->fields())1439    if (!FieldHasTrivialDestructorBody(CGF.getContext(), Field))1440      return false;1441 1442  return true;1443}1444 1445/// EmitDestructorBody - Emits the body of the current destructor.1446void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) {1447  const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CurGD.getDecl());1448  CXXDtorType DtorType = CurGD.getDtorType();1449 1450  // For an abstract class, non-base destructors are never used (and can't1451  // be emitted in general, because vbase dtors may not have been validated1452  // by Sema), but the Itanium ABI doesn't make them optional and Clang may1453  // in fact emit references to them from other compilations, so emit them1454  // as functions containing a trap instruction.1455  if (DtorType != Dtor_Base && Dtor->getParent()->isAbstract()) {1456    llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);1457    TrapCall->setDoesNotReturn();1458    TrapCall->setDoesNotThrow();1459    Builder.CreateUnreachable();1460    Builder.ClearInsertionPoint();1461    return;1462  }1463 1464  Stmt *Body = Dtor->getBody();1465  if (Body) {1466    incrementProfileCounter(Body);1467    maybeCreateMCDCCondBitmap();1468  }1469 1470  // The call to operator delete in a deleting destructor happens1471  // outside of the function-try-block, which means it's always1472  // possible to delegate the destructor body to the complete1473  // destructor.  Do so.1474  if (DtorType == Dtor_Deleting) {1475    RunCleanupsScope DtorEpilogue(*this);1476    EnterDtorCleanups(Dtor, Dtor_Deleting);1477    if (HaveInsertPoint()) {1478      QualType ThisTy = Dtor->getFunctionObjectParameterType();1479      EmitCXXDestructorCall(Dtor, Dtor_Complete, /*ForVirtualBase=*/false,1480                            /*Delegating=*/false, LoadCXXThisAddress(), ThisTy);1481    }1482    return;1483  }1484 1485  // If the body is a function-try-block, enter the try before1486  // anything else.1487  bool isTryBody = isa_and_nonnull<CXXTryStmt>(Body);1488  if (isTryBody)1489    EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true);1490  EmitAsanPrologueOrEpilogue(false);1491 1492  // Enter the epilogue cleanups.1493  RunCleanupsScope DtorEpilogue(*this);1494 1495  // If this is the complete variant, just invoke the base variant;1496  // the epilogue will destruct the virtual bases.  But we can't do1497  // this optimization if the body is a function-try-block, because1498  // we'd introduce *two* handler blocks.  In the Microsoft ABI, we1499  // always delegate because we might not have a definition in this TU.1500  switch (DtorType) {1501  case Dtor_Unified:1502    llvm_unreachable("not expecting a unified dtor");1503  case Dtor_Comdat: llvm_unreachable("not expecting a COMDAT");1504  case Dtor_Deleting: llvm_unreachable("already handled deleting case");1505 1506  case Dtor_Complete:1507    assert((Body || getTarget().getCXXABI().isMicrosoft()) &&1508           "can't emit a dtor without a body for non-Microsoft ABIs");1509 1510    // Enter the cleanup scopes for virtual bases.1511    EnterDtorCleanups(Dtor, Dtor_Complete);1512 1513    if (!isTryBody) {1514      QualType ThisTy = Dtor->getFunctionObjectParameterType();1515      EmitCXXDestructorCall(Dtor, Dtor_Base, /*ForVirtualBase=*/false,1516                            /*Delegating=*/false, LoadCXXThisAddress(), ThisTy);1517      break;1518    }1519 1520    // Fallthrough: act like we're in the base variant.1521    [[fallthrough]];1522 1523  case Dtor_Base:1524    assert(Body);1525 1526    // Enter the cleanup scopes for fields and non-virtual bases.1527    EnterDtorCleanups(Dtor, Dtor_Base);1528 1529    // Initialize the vtable pointers before entering the body.1530    if (!CanSkipVTablePointerInitialization(*this, Dtor)) {1531      // Insert the llvm.launder.invariant.group intrinsic before initializing1532      // the vptrs to cancel any previous assumptions we might have made.1533      if (CGM.getCodeGenOpts().StrictVTablePointers &&1534          CGM.getCodeGenOpts().OptimizationLevel > 0)1535        CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());1536      InitializeVTablePointers(Dtor->getParent());1537    }1538 1539    if (isTryBody)1540      EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock());1541    else if (Body)1542      EmitStmt(Body);1543    else {1544      assert(Dtor->isImplicit() && "bodyless dtor not implicit");1545      // nothing to do besides what's in the epilogue1546    }1547    // -fapple-kext must inline any call to this dtor into1548    // the caller's body.1549    if (getLangOpts().AppleKext)1550      CurFn->addFnAttr(llvm::Attribute::AlwaysInline);1551 1552    break;1553  }1554 1555  // Jump out through the epilogue cleanups.1556  DtorEpilogue.ForceCleanup();1557 1558  // Exit the try if applicable.1559  if (isTryBody)1560    ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true);1561}1562 1563void CodeGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &Args) {1564  const CXXMethodDecl *AssignOp = cast<CXXMethodDecl>(CurGD.getDecl());1565  const Stmt *RootS = AssignOp->getBody();1566  assert(isa<CompoundStmt>(RootS) &&1567         "Body of an implicit assignment operator should be compound stmt.");1568  const CompoundStmt *RootCS = cast<CompoundStmt>(RootS);1569 1570  LexicalScope Scope(*this, RootCS->getSourceRange());1571 1572  incrementProfileCounter(RootCS);1573  maybeCreateMCDCCondBitmap();1574  AssignmentMemcpyizer AM(*this, AssignOp, Args);1575  for (auto *I : RootCS->body())1576    AM.emitAssignment(I);1577 1578  AM.finish();1579}1580 1581namespace {1582  llvm::Value *LoadThisForDtorDelete(CodeGenFunction &CGF,1583                                     const CXXDestructorDecl *DD) {1584    if (Expr *ThisArg = DD->getOperatorDeleteThisArg())1585      return CGF.EmitScalarExpr(ThisArg);1586    return CGF.LoadCXXThis();1587  }1588 1589  /// Call the operator delete associated with the current destructor.1590  struct CallDtorDelete final : EHScopeStack::Cleanup {1591    CallDtorDelete() {}1592 1593    void Emit(CodeGenFunction &CGF, Flags flags) override {1594      const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CGF.CurCodeDecl);1595      const CXXRecordDecl *ClassDecl = Dtor->getParent();1596      CGF.EmitDeleteCall(Dtor->getOperatorDelete(),1597                         LoadThisForDtorDelete(CGF, Dtor),1598                         CGF.getContext().getCanonicalTagType(ClassDecl));1599    }1600  };1601 1602  // This function implements generation of scalar deleting destructor body for1603  // the case when the destructor also accepts an implicit flag. Right now only1604  // Microsoft ABI requires deleting destructors to accept implicit flags.1605  // The flag indicates whether an operator delete should be called and whether1606  // it should be a class-specific operator delete or a global one.1607  void EmitConditionalDtorDeleteCall(CodeGenFunction &CGF,1608                                     llvm::Value *ShouldDeleteCondition,1609                                     bool ReturnAfterDelete) {1610    const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CGF.CurCodeDecl);1611    const CXXRecordDecl *ClassDecl = Dtor->getParent();1612    const FunctionDecl *OD = Dtor->getOperatorDelete();1613    assert(OD->isDestroyingOperatorDelete() == ReturnAfterDelete &&1614           "unexpected value for ReturnAfterDelete");1615    auto *CondTy = cast<llvm::IntegerType>(ShouldDeleteCondition->getType());1616    // MSVC calls global operator delete inside of the dtor body, but clang1617    // aligned with this behavior only after a particular version. This is not1618    // ABI-compatible with previous versions.1619    ASTContext &Context = CGF.getContext();1620    bool CallGlobDelete =1621        Context.getTargetInfo().callGlobalDeleteInDeletingDtor(1622            Context.getLangOpts());1623    if (CallGlobDelete && OD->isDestroyingOperatorDelete()) {1624      llvm::BasicBlock *CallDtor = CGF.createBasicBlock("dtor.call_dtor");1625      llvm::BasicBlock *DontCallDtor = CGF.createBasicBlock("dtor.entry_cont");1626      // Third bit set signals that global operator delete is called. That means1627      // despite class having destroying operator delete which is responsible1628      // for calling dtor, we need to call dtor because global operator delete1629      // won't do that.1630      llvm::Value *Check3rdBit = CGF.Builder.CreateAnd(1631          ShouldDeleteCondition, llvm::ConstantInt::get(CondTy, 4));1632      llvm::Value *ShouldCallDtor = CGF.Builder.CreateIsNull(Check3rdBit);1633      CGF.Builder.CreateCondBr(ShouldCallDtor, DontCallDtor, CallDtor);1634      CGF.EmitBlock(CallDtor);1635      QualType ThisTy = Dtor->getFunctionObjectParameterType();1636      CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, /*ForVirtualBase=*/false,1637                                /*Delegating=*/false, CGF.LoadCXXThisAddress(),1638                                ThisTy);1639      CGF.Builder.CreateBr(DontCallDtor);1640      CGF.EmitBlock(DontCallDtor);1641    }1642    llvm::BasicBlock *callDeleteBB = CGF.createBasicBlock("dtor.call_delete");1643    llvm::BasicBlock *continueBB = CGF.createBasicBlock("dtor.continue");1644    // First bit set signals that operator delete must be called.1645    llvm::Value *Check1stBit = CGF.Builder.CreateAnd(1646        ShouldDeleteCondition, llvm::ConstantInt::get(CondTy, 1));1647    llvm::Value *ShouldCallDelete = CGF.Builder.CreateIsNull(Check1stBit);1648    CGF.Builder.CreateCondBr(ShouldCallDelete, continueBB, callDeleteBB);1649 1650    CGF.EmitBlock(callDeleteBB);1651    auto EmitDeleteAndGoToEnd = [&](const FunctionDecl *DeleteOp) {1652      CGF.EmitDeleteCall(DeleteOp, LoadThisForDtorDelete(CGF, Dtor),1653                         Context.getCanonicalTagType(ClassDecl));1654      if (ReturnAfterDelete)1655        CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);1656      else1657        CGF.Builder.CreateBr(continueBB);1658    };1659    // If Sema only found a global operator delete previously, the dtor can1660    // always call it. Otherwise we need to check the third bit and call the1661    // appropriate operator delete, i.e. global or class-specific.1662    if (const FunctionDecl *GlobOD = Dtor->getOperatorGlobalDelete();1663        isa<CXXMethodDecl>(OD) && GlobOD && CallGlobDelete) {1664      // Third bit set signals that global operator delete is called, i.e.1665      // ::delete appears on the callsite.1666      llvm::Value *CheckTheBitForGlobDeleteCall = CGF.Builder.CreateAnd(1667          ShouldDeleteCondition, llvm::ConstantInt::get(CondTy, 4));1668      llvm::Value *ShouldCallGlobDelete =1669          CGF.Builder.CreateIsNull(CheckTheBitForGlobDeleteCall);1670      llvm::BasicBlock *GlobDelete =1671          CGF.createBasicBlock("dtor.call_glob_delete");1672      llvm::BasicBlock *ClassDelete =1673          CGF.createBasicBlock("dtor.call_class_delete");1674      CGF.Builder.CreateCondBr(ShouldCallGlobDelete, ClassDelete, GlobDelete);1675      CGF.EmitBlock(GlobDelete);1676 1677      EmitDeleteAndGoToEnd(GlobOD);1678      CGF.EmitBlock(ClassDelete);1679    }1680    EmitDeleteAndGoToEnd(OD);1681    CGF.EmitBlock(continueBB);1682  }1683 1684  struct CallDtorDeleteConditional final : EHScopeStack::Cleanup {1685    llvm::Value *ShouldDeleteCondition;1686 1687  public:1688    CallDtorDeleteConditional(llvm::Value *ShouldDeleteCondition)1689        : ShouldDeleteCondition(ShouldDeleteCondition) {1690      assert(ShouldDeleteCondition != nullptr);1691    }1692 1693    void Emit(CodeGenFunction &CGF, Flags flags) override {1694      EmitConditionalDtorDeleteCall(CGF, ShouldDeleteCondition,1695                                    /*ReturnAfterDelete*/false);1696    }1697  };1698 1699  class DestroyField  final : public EHScopeStack::Cleanup {1700    const FieldDecl *field;1701    CodeGenFunction::Destroyer *destroyer;1702    bool useEHCleanupForArray;1703 1704  public:1705    DestroyField(const FieldDecl *field, CodeGenFunction::Destroyer *destroyer,1706                 bool useEHCleanupForArray)1707        : field(field), destroyer(destroyer),1708          useEHCleanupForArray(useEHCleanupForArray) {}1709 1710    void Emit(CodeGenFunction &CGF, Flags flags) override {1711      // Find the address of the field.1712      Address thisValue = CGF.LoadCXXThisAddress();1713      CanQualType RecordTy =1714          CGF.getContext().getCanonicalTagType(field->getParent());1715      LValue ThisLV = CGF.MakeAddrLValue(thisValue, RecordTy);1716      LValue LV = CGF.EmitLValueForField(ThisLV, field);1717      assert(LV.isSimple());1718 1719      CGF.emitDestroy(LV.getAddress(), field->getType(), destroyer,1720                      flags.isForNormalCleanup() && useEHCleanupForArray);1721    }1722  };1723 1724  class DeclAsInlineDebugLocation {1725    CGDebugInfo *DI;1726    llvm::MDNode *InlinedAt;1727    std::optional<ApplyDebugLocation> Location;1728 1729  public:1730    DeclAsInlineDebugLocation(CodeGenFunction &CGF, const NamedDecl &Decl)1731        : DI(CGF.getDebugInfo()) {1732      if (!DI)1733        return;1734      InlinedAt = DI->getInlinedAt();1735      DI->setInlinedAt(CGF.Builder.getCurrentDebugLocation());1736      Location.emplace(CGF, Decl.getLocation());1737    }1738 1739    ~DeclAsInlineDebugLocation() {1740      if (!DI)1741        return;1742      Location.reset();1743      DI->setInlinedAt(InlinedAt);1744    }1745  };1746 1747  static void EmitSanitizerDtorCallback(1748      CodeGenFunction &CGF, StringRef Name, llvm::Value *Ptr,1749      std::optional<CharUnits::QuantityType> PoisonSize = {}) {1750    CodeGenFunction::SanitizerScope SanScope(&CGF);1751    // Pass in void pointer and size of region as arguments to runtime1752    // function1753    SmallVector<llvm::Value *, 2> Args = {Ptr};1754    SmallVector<llvm::Type *, 2> ArgTypes = {CGF.VoidPtrTy};1755 1756    if (PoisonSize.has_value()) {1757      Args.emplace_back(llvm::ConstantInt::get(CGF.SizeTy, *PoisonSize));1758      ArgTypes.emplace_back(CGF.SizeTy);1759    }1760 1761    llvm::FunctionType *FnType =1762        llvm::FunctionType::get(CGF.VoidTy, ArgTypes, false);1763    llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(FnType, Name);1764 1765    CGF.EmitNounwindRuntimeCall(Fn, Args);1766  }1767 1768  static void1769  EmitSanitizerDtorFieldsCallback(CodeGenFunction &CGF, llvm::Value *Ptr,1770                                  CharUnits::QuantityType PoisonSize) {1771    EmitSanitizerDtorCallback(CGF, "__sanitizer_dtor_callback_fields", Ptr,1772                              PoisonSize);1773  }1774 1775  /// Poison base class with a trivial destructor.1776  struct SanitizeDtorTrivialBase final : EHScopeStack::Cleanup {1777    const CXXRecordDecl *BaseClass;1778    bool BaseIsVirtual;1779    SanitizeDtorTrivialBase(const CXXRecordDecl *Base, bool BaseIsVirtual)1780        : BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {}1781 1782    void Emit(CodeGenFunction &CGF, Flags flags) override {1783      const CXXRecordDecl *DerivedClass =1784          cast<CXXMethodDecl>(CGF.CurCodeDecl)->getParent();1785 1786      Address Addr = CGF.GetAddressOfDirectBaseInCompleteClass(1787          CGF.LoadCXXThisAddress(), DerivedClass, BaseClass, BaseIsVirtual);1788 1789      const ASTRecordLayout &BaseLayout =1790          CGF.getContext().getASTRecordLayout(BaseClass);1791      CharUnits BaseSize = BaseLayout.getSize();1792 1793      if (!BaseSize.isPositive())1794        return;1795 1796      // Use the base class declaration location as inline DebugLocation. All1797      // fields of the class are destroyed.1798      DeclAsInlineDebugLocation InlineHere(CGF, *BaseClass);1799      EmitSanitizerDtorFieldsCallback(CGF, Addr.emitRawPointer(CGF),1800                                      BaseSize.getQuantity());1801 1802      // Prevent the current stack frame from disappearing from the stack trace.1803      CGF.CurFn->addFnAttr("disable-tail-calls", "true");1804    }1805  };1806 1807  class SanitizeDtorFieldRange final : public EHScopeStack::Cleanup {1808    const CXXDestructorDecl *Dtor;1809    unsigned StartIndex;1810    unsigned EndIndex;1811 1812  public:1813    SanitizeDtorFieldRange(const CXXDestructorDecl *Dtor, unsigned StartIndex,1814                           unsigned EndIndex)1815        : Dtor(Dtor), StartIndex(StartIndex), EndIndex(EndIndex) {}1816 1817    // Generate function call for handling object poisoning.1818    // Disables tail call elimination, to prevent the current stack frame1819    // from disappearing from the stack trace.1820    void Emit(CodeGenFunction &CGF, Flags flags) override {1821      const ASTContext &Context = CGF.getContext();1822      const ASTRecordLayout &Layout =1823          Context.getASTRecordLayout(Dtor->getParent());1824 1825      // It's a first trivial field so it should be at the begining of a char,1826      // still round up start offset just in case.1827      CharUnits PoisonStart = Context.toCharUnitsFromBits(1828          Layout.getFieldOffset(StartIndex) + Context.getCharWidth() - 1);1829      llvm::ConstantInt *OffsetSizePtr =1830          llvm::ConstantInt::get(CGF.SizeTy, PoisonStart.getQuantity());1831 1832      llvm::Value *OffsetPtr =1833          CGF.Builder.CreateGEP(CGF.Int8Ty, CGF.LoadCXXThis(), OffsetSizePtr);1834 1835      CharUnits PoisonEnd;1836      if (EndIndex >= Layout.getFieldCount()) {1837        PoisonEnd = Layout.getNonVirtualSize();1838      } else {1839        PoisonEnd =1840            Context.toCharUnitsFromBits(Layout.getFieldOffset(EndIndex));1841      }1842      CharUnits PoisonSize = PoisonEnd - PoisonStart;1843      if (!PoisonSize.isPositive())1844        return;1845 1846      // Use the top field declaration location as inline DebugLocation.1847      DeclAsInlineDebugLocation InlineHere(1848          CGF, **std::next(Dtor->getParent()->field_begin(), StartIndex));1849      EmitSanitizerDtorFieldsCallback(CGF, OffsetPtr, PoisonSize.getQuantity());1850 1851      // Prevent the current stack frame from disappearing from the stack trace.1852      CGF.CurFn->addFnAttr("disable-tail-calls", "true");1853    }1854  };1855 1856 class SanitizeDtorVTable final : public EHScopeStack::Cleanup {1857    const CXXDestructorDecl *Dtor;1858 1859  public:1860    SanitizeDtorVTable(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {}1861 1862    // Generate function call for handling vtable pointer poisoning.1863    void Emit(CodeGenFunction &CGF, Flags flags) override {1864      assert(Dtor->getParent()->isDynamicClass());1865      (void)Dtor;1866      // Poison vtable and vtable ptr if they exist for this class.1867      llvm::Value *VTablePtr = CGF.LoadCXXThis();1868 1869      // Pass in void pointer and size of region as arguments to runtime1870      // function1871      EmitSanitizerDtorCallback(CGF, "__sanitizer_dtor_callback_vptr",1872                                VTablePtr);1873    }1874 };1875 1876 class SanitizeDtorCleanupBuilder {1877   ASTContext &Context;1878   EHScopeStack &EHStack;1879   const CXXDestructorDecl *DD;1880   std::optional<unsigned> StartIndex;1881 1882 public:1883   SanitizeDtorCleanupBuilder(ASTContext &Context, EHScopeStack &EHStack,1884                              const CXXDestructorDecl *DD)1885       : Context(Context), EHStack(EHStack), DD(DD), StartIndex(std::nullopt) {}1886   void PushCleanupForField(const FieldDecl *Field) {1887     if (isEmptyFieldForLayout(Context, Field))1888       return;1889     unsigned FieldIndex = Field->getFieldIndex();1890     if (FieldHasTrivialDestructorBody(Context, Field)) {1891       if (!StartIndex)1892         StartIndex = FieldIndex;1893     } else if (StartIndex) {1894       EHStack.pushCleanup<SanitizeDtorFieldRange>(NormalAndEHCleanup, DD,1895                                                   *StartIndex, FieldIndex);1896       StartIndex = std::nullopt;1897     }1898   }1899   void End() {1900     if (StartIndex)1901       EHStack.pushCleanup<SanitizeDtorFieldRange>(NormalAndEHCleanup, DD,1902                                                   *StartIndex, -1);1903   }1904 };1905} // end anonymous namespace1906 1907/// Emit all code that comes at the end of class's1908/// destructor. This is to call destructors on members and base classes1909/// in reverse order of their construction.1910///1911/// For a deleting destructor, this also handles the case where a destroying1912/// operator delete completely overrides the definition.1913void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,1914                                        CXXDtorType DtorType) {1915  assert((!DD->isTrivial() || DD->hasAttr<DLLExportAttr>()) &&1916         "Should not emit dtor epilogue for non-exported trivial dtor!");1917 1918  // The deleting-destructor phase just needs to call the appropriate1919  // operator delete that Sema picked up.1920  if (DtorType == Dtor_Deleting) {1921    assert(DD->getOperatorDelete() &&1922           "operator delete missing - EnterDtorCleanups");1923    if (CXXStructorImplicitParamValue) {1924      // If there is an implicit param to the deleting dtor, it's a boolean1925      // telling whether this is a deleting destructor.1926      if (DD->getOperatorDelete()->isDestroyingOperatorDelete())1927        EmitConditionalDtorDeleteCall(*this, CXXStructorImplicitParamValue,1928                                      /*ReturnAfterDelete*/true);1929      else1930        EHStack.pushCleanup<CallDtorDeleteConditional>(1931            NormalAndEHCleanup, CXXStructorImplicitParamValue);1932    } else {1933      if (DD->getOperatorDelete()->isDestroyingOperatorDelete()) {1934        const CXXRecordDecl *ClassDecl = DD->getParent();1935        EmitDeleteCall(DD->getOperatorDelete(),1936                       LoadThisForDtorDelete(*this, DD),1937                       getContext().getCanonicalTagType(ClassDecl));1938        EmitBranchThroughCleanup(ReturnBlock);1939      } else {1940        EHStack.pushCleanup<CallDtorDelete>(NormalAndEHCleanup);1941      }1942    }1943    return;1944  }1945 1946  const CXXRecordDecl *ClassDecl = DD->getParent();1947 1948  // Unions have no bases and do not call field destructors.1949  if (ClassDecl->isUnion())1950    return;1951 1952  // The complete-destructor phase just destructs all the virtual bases.1953  if (DtorType == Dtor_Complete) {1954    // Poison the vtable pointer such that access after the base1955    // and member destructors are invoked is invalid.1956    if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&1957        SanOpts.has(SanitizerKind::Memory) && ClassDecl->getNumVBases() &&1958        ClassDecl->isPolymorphic())1959      EHStack.pushCleanup<SanitizeDtorVTable>(NormalAndEHCleanup, DD);1960 1961    // We push them in the forward order so that they'll be popped in1962    // the reverse order.1963    for (const auto &Base : ClassDecl->vbases()) {1964      auto *BaseClassDecl = Base.getType()->castAsCXXRecordDecl();1965      if (BaseClassDecl->hasTrivialDestructor()) {1966        // Under SanitizeMemoryUseAfterDtor, poison the trivial base class1967        // memory. For non-trival base classes the same is done in the class1968        // destructor.1969        if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&1970            SanOpts.has(SanitizerKind::Memory) && !BaseClassDecl->isEmpty())1971          EHStack.pushCleanup<SanitizeDtorTrivialBase>(NormalAndEHCleanup,1972                                                       BaseClassDecl,1973                                                       /*BaseIsVirtual*/ true);1974      } else {1975        EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup, BaseClassDecl,1976                                          /*BaseIsVirtual*/ true);1977      }1978    }1979 1980    return;1981  }1982 1983  assert(DtorType == Dtor_Base);1984  // Poison the vtable pointer if it has no virtual bases, but inherits1985  // virtual functions.1986  if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&1987      SanOpts.has(SanitizerKind::Memory) && !ClassDecl->getNumVBases() &&1988      ClassDecl->isPolymorphic())1989    EHStack.pushCleanup<SanitizeDtorVTable>(NormalAndEHCleanup, DD);1990 1991  // Destroy non-virtual bases.1992  for (const auto &Base : ClassDecl->bases()) {1993    // Ignore virtual bases.1994    if (Base.isVirtual())1995      continue;1996 1997    CXXRecordDecl *BaseClassDecl = Base.getType()->getAsCXXRecordDecl();1998 1999    if (BaseClassDecl->hasTrivialDestructor()) {2000      if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&2001          SanOpts.has(SanitizerKind::Memory) && !BaseClassDecl->isEmpty())2002        EHStack.pushCleanup<SanitizeDtorTrivialBase>(NormalAndEHCleanup,2003                                                     BaseClassDecl,2004                                                     /*BaseIsVirtual*/ false);2005    } else {2006      EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup, BaseClassDecl,2007                                        /*BaseIsVirtual*/ false);2008    }2009  }2010 2011  // Poison fields such that access after their destructors are2012  // invoked, and before the base class destructor runs, is invalid.2013  bool SanitizeFields = CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&2014                        SanOpts.has(SanitizerKind::Memory);2015  SanitizeDtorCleanupBuilder SanitizeBuilder(getContext(), EHStack, DD);2016 2017  // Destroy direct fields.2018  for (const auto *Field : ClassDecl->fields()) {2019    if (SanitizeFields)2020      SanitizeBuilder.PushCleanupForField(Field);2021 2022    QualType type = Field->getType();2023    QualType::DestructionKind dtorKind = type.isDestructedType();2024    if (!dtorKind)2025      continue;2026 2027    // Anonymous union members do not have their destructors called.2028    const RecordType *RT = type->getAsUnionType();2029    if (RT && RT->getDecl()->isAnonymousStructOrUnion())2030      continue;2031 2032    CleanupKind cleanupKind = getCleanupKind(dtorKind);2033    EHStack.pushCleanup<DestroyField>(2034        cleanupKind, Field, getDestroyer(dtorKind), cleanupKind & EHCleanup);2035  }2036 2037  if (SanitizeFields)2038    SanitizeBuilder.End();2039}2040 2041/// EmitCXXAggrConstructorCall - Emit a loop to call a particular2042/// constructor for each of several members of an array.2043///2044/// \param ctor the constructor to call for each element2045/// \param arrayType the type of the array to initialize2046/// \param arrayBegin an arrayType*2047/// \param zeroInitialize true if each element should be2048///   zero-initialized before it is constructed2049void CodeGenFunction::EmitCXXAggrConstructorCall(2050    const CXXConstructorDecl *ctor, const ArrayType *arrayType,2051    Address arrayBegin, const CXXConstructExpr *E, bool NewPointerIsChecked,2052    bool zeroInitialize) {2053  QualType elementType;2054  llvm::Value *numElements =2055    emitArrayLength(arrayType, elementType, arrayBegin);2056 2057  EmitCXXAggrConstructorCall(ctor, numElements, arrayBegin, E,2058                             NewPointerIsChecked, zeroInitialize);2059}2060 2061/// EmitCXXAggrConstructorCall - Emit a loop to call a particular2062/// constructor for each of several members of an array.2063///2064/// \param ctor the constructor to call for each element2065/// \param numElements the number of elements in the array;2066///   may be zero2067/// \param arrayBase a T*, where T is the type constructed by ctor2068/// \param zeroInitialize true if each element should be2069///   zero-initialized before it is constructed2070void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor,2071                                                 llvm::Value *numElements,2072                                                 Address arrayBase,2073                                                 const CXXConstructExpr *E,2074                                                 bool NewPointerIsChecked,2075                                                 bool zeroInitialize) {2076  // It's legal for numElements to be zero.  This can happen both2077  // dynamically, because x can be zero in 'new A[x]', and statically,2078  // because of GCC extensions that permit zero-length arrays.  There2079  // are probably legitimate places where we could assume that this2080  // doesn't happen, but it's not clear that it's worth it.2081  llvm::BranchInst *zeroCheckBranch = nullptr;2082 2083  // Optimize for a constant count.2084  llvm::ConstantInt *constantCount2085    = dyn_cast<llvm::ConstantInt>(numElements);2086  if (constantCount) {2087    // Just skip out if the constant count is zero.2088    if (constantCount->isZero()) return;2089 2090  // Otherwise, emit the check.2091  } else {2092    llvm::BasicBlock *loopBB = createBasicBlock("new.ctorloop");2093    llvm::Value *iszero = Builder.CreateIsNull(numElements, "isempty");2094    zeroCheckBranch = Builder.CreateCondBr(iszero, loopBB, loopBB);2095    EmitBlock(loopBB);2096  }2097 2098  // Find the end of the array.2099  llvm::Type *elementType = arrayBase.getElementType();2100  llvm::Value *arrayBegin = arrayBase.emitRawPointer(*this);2101  llvm::Value *arrayEnd = Builder.CreateInBoundsGEP(2102      elementType, arrayBegin, numElements, "arrayctor.end");2103 2104  // Enter the loop, setting up a phi for the current location to initialize.2105  llvm::BasicBlock *entryBB = Builder.GetInsertBlock();2106  llvm::BasicBlock *loopBB = createBasicBlock("arrayctor.loop");2107  EmitBlock(loopBB);2108  llvm::PHINode *cur = Builder.CreatePHI(arrayBegin->getType(), 2,2109                                         "arrayctor.cur");2110  cur->addIncoming(arrayBegin, entryBB);2111 2112  // Inside the loop body, emit the constructor call on the array element.2113  if (CGM.shouldEmitConvergenceTokens())2114    ConvergenceTokenStack.push_back(emitConvergenceLoopToken(loopBB));2115 2116  // The alignment of the base, adjusted by the size of a single element,2117  // provides a conservative estimate of the alignment of every element.2118  // (This assumes we never start tracking offsetted alignments.)2119  //2120  // Note that these are complete objects and so we don't need to2121  // use the non-virtual size or alignment.2122  CanQualType type = getContext().getCanonicalTagType(ctor->getParent());2123  CharUnits eltAlignment =2124    arrayBase.getAlignment()2125             .alignmentOfArrayElement(getContext().getTypeSizeInChars(type));2126  Address curAddr = Address(cur, elementType, eltAlignment);2127 2128  // Zero initialize the storage, if requested.2129  if (zeroInitialize)2130    EmitNullInitialization(curAddr, type);2131 2132  // C++ [class.temporary]p4:2133  // There are two contexts in which temporaries are destroyed at a different2134  // point than the end of the full-expression. The first context is when a2135  // default constructor is called to initialize an element of an array.2136  // If the constructor has one or more default arguments, the destruction of2137  // every temporary created in a default argument expression is sequenced2138  // before the construction of the next array element, if any.2139 2140  {2141    RunCleanupsScope Scope(*this);2142 2143    // Evaluate the constructor and its arguments in a regular2144    // partial-destroy cleanup.2145    if (getLangOpts().Exceptions &&2146        !ctor->getParent()->hasTrivialDestructor()) {2147      Destroyer *destroyer = destroyCXXObject;2148      pushRegularPartialArrayCleanup(arrayBegin, cur, type, eltAlignment,2149                                     *destroyer);2150    }2151    auto currAVS = AggValueSlot::forAddr(2152        curAddr, type.getQualifiers(), AggValueSlot::IsDestructed,2153        AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,2154        AggValueSlot::DoesNotOverlap, AggValueSlot::IsNotZeroed,2155        NewPointerIsChecked ? AggValueSlot::IsSanitizerChecked2156                            : AggValueSlot::IsNotSanitizerChecked);2157    EmitCXXConstructorCall(ctor, Ctor_Complete, /*ForVirtualBase=*/false,2158                           /*Delegating=*/false, currAVS, E);2159  }2160 2161  // Go to the next element.2162  llvm::Value *next = Builder.CreateInBoundsGEP(2163      elementType, cur, llvm::ConstantInt::get(SizeTy, 1), "arrayctor.next");2164  cur->addIncoming(next, Builder.GetInsertBlock());2165 2166  // Check whether that's the end of the loop.2167  llvm::Value *done = Builder.CreateICmpEQ(next, arrayEnd, "arrayctor.done");2168  llvm::BasicBlock *contBB = createBasicBlock("arrayctor.cont");2169  Builder.CreateCondBr(done, contBB, loopBB);2170 2171  // Patch the earlier check to skip over the loop.2172  if (zeroCheckBranch) zeroCheckBranch->setSuccessor(0, contBB);2173 2174  if (CGM.shouldEmitConvergenceTokens())2175    ConvergenceTokenStack.pop_back();2176 2177  EmitBlock(contBB);2178}2179 2180void CodeGenFunction::destroyCXXObject(CodeGenFunction &CGF,2181                                       Address addr,2182                                       QualType type) {2183  const CXXDestructorDecl *dtor = type->castAsCXXRecordDecl()->getDestructor();2184  assert(!dtor->isTrivial());2185  CGF.EmitCXXDestructorCall(dtor, Dtor_Complete, /*for vbase*/ false,2186                            /*Delegating=*/false, addr, type);2187}2188 2189void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,2190                                             CXXCtorType Type,2191                                             bool ForVirtualBase,2192                                             bool Delegating,2193                                             AggValueSlot ThisAVS,2194                                             const CXXConstructExpr *E) {2195  CallArgList Args;2196  Address This = ThisAVS.getAddress();2197  LangAS SlotAS = ThisAVS.getQualifiers().getAddressSpace();2198  LangAS ThisAS = D->getFunctionObjectParameterType().getAddressSpace();2199  llvm::Value *ThisPtr =2200      getAsNaturalPointerTo(This, D->getThisType()->getPointeeType());2201 2202  if (SlotAS != ThisAS) {2203    unsigned TargetThisAS = getContext().getTargetAddressSpace(ThisAS);2204    llvm::Type *NewType =2205        llvm::PointerType::get(getLLVMContext(), TargetThisAS);2206    ThisPtr =2207        getTargetHooks().performAddrSpaceCast(*this, ThisPtr, ThisAS, NewType);2208  }2209 2210  // Push the this ptr.2211  Args.add(RValue::get(ThisPtr), D->getThisType());2212 2213  // If this is a trivial constructor, emit a memcpy now before we lose2214  // the alignment information on the argument.2215  // FIXME: It would be better to preserve alignment information into CallArg.2216  if (isMemcpyEquivalentSpecialMember(D)) {2217    assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor");2218 2219    const Expr *Arg = E->getArg(0);2220    LValue Src = EmitLValue(Arg);2221    CanQualType DestTy = getContext().getCanonicalTagType(D->getParent());2222    LValue Dest = MakeAddrLValue(This, DestTy);2223    EmitAggregateCopyCtor(Dest, Src, ThisAVS.mayOverlap());2224    return;2225  }2226 2227  // Add the rest of the user-supplied arguments.2228  const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();2229  EvaluationOrder Order = E->isListInitialization()2230                              ? EvaluationOrder::ForceLeftToRight2231                              : EvaluationOrder::Default;2232  EmitCallArgs(Args, FPT, E->arguments(), E->getConstructor(),2233               /*ParamsToSkip*/ 0, Order);2234 2235  EmitCXXConstructorCall(D, Type, ForVirtualBase, Delegating, This, Args,2236                         ThisAVS.mayOverlap(), E->getExprLoc(),2237                         ThisAVS.isSanitizerChecked());2238}2239 2240static bool canEmitDelegateCallArgs(CodeGenFunction &CGF,2241                                    const CXXConstructorDecl *Ctor,2242                                    CXXCtorType Type, CallArgList &Args) {2243  // We can't forward a variadic call.2244  if (Ctor->isVariadic())2245    return false;2246 2247  if (CGF.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {2248    // If the parameters are callee-cleanup, it's not safe to forward.2249    for (auto *P : Ctor->parameters())2250      if (P->needsDestruction(CGF.getContext()))2251        return false;2252 2253    // Likewise if they're inalloca.2254    const CGFunctionInfo &Info =2255        CGF.CGM.getTypes().arrangeCXXConstructorCall(Args, Ctor, Type, 0, 0);2256    if (Info.usesInAlloca())2257      return false;2258  }2259 2260  // Anything else should be OK.2261  return true;2262}2263 2264void CodeGenFunction::EmitCXXConstructorCall(2265    const CXXConstructorDecl *D, CXXCtorType Type, bool ForVirtualBase,2266    bool Delegating, Address This, CallArgList &Args,2267    AggValueSlot::Overlap_t Overlap, SourceLocation Loc,2268    bool NewPointerIsChecked, llvm::CallBase **CallOrInvoke) {2269  const CXXRecordDecl *ClassDecl = D->getParent();2270 2271  if (!NewPointerIsChecked)2272    EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This,2273                  getContext().getCanonicalTagType(ClassDecl),2274                  CharUnits::Zero());2275 2276  if (D->isTrivial() && D->isDefaultConstructor()) {2277    assert(Args.size() == 1 && "trivial default ctor with args");2278    return;2279  }2280 2281  // If this is a trivial constructor, just emit what's needed. If this is a2282  // union copy constructor, we must emit a memcpy, because the AST does not2283  // model that copy.2284  if (isMemcpyEquivalentSpecialMember(D)) {2285    assert(Args.size() == 2 && "unexpected argcount for trivial ctor");2286    QualType SrcTy = D->getParamDecl(0)->getType().getNonReferenceType();2287    Address Src = makeNaturalAddressForPointer(2288        Args[1].getRValue(*this).getScalarVal(), SrcTy);2289    LValue SrcLVal = MakeAddrLValue(Src, SrcTy);2290    CanQualType DestTy = getContext().getCanonicalTagType(ClassDecl);2291    LValue DestLVal = MakeAddrLValue(This, DestTy);2292    EmitAggregateCopyCtor(DestLVal, SrcLVal, Overlap);2293    return;2294  }2295 2296  bool PassPrototypeArgs = true;2297  // Check whether we can actually emit the constructor before trying to do so.2298  if (auto Inherited = D->getInheritedConstructor()) {2299    PassPrototypeArgs = getTypes().inheritingCtorHasParams(Inherited, Type);2300    if (PassPrototypeArgs && !canEmitDelegateCallArgs(*this, D, Type, Args)) {2301      EmitInlinedInheritingCXXConstructorCall(D, Type, ForVirtualBase,2302                                              Delegating, Args);2303      return;2304    }2305  }2306 2307  // Insert any ABI-specific implicit constructor arguments.2308  CGCXXABI::AddedStructorArgCounts ExtraArgs =2309      CGM.getCXXABI().addImplicitConstructorArgs(*this, D, Type, ForVirtualBase,2310                                                 Delegating, Args);2311 2312  // Emit the call.2313  llvm::Constant *CalleePtr = CGM.getAddrOfCXXStructor(GlobalDecl(D, Type));2314  const CGFunctionInfo &Info = CGM.getTypes().arrangeCXXConstructorCall(2315      Args, D, Type, ExtraArgs.Prefix, ExtraArgs.Suffix, PassPrototypeArgs);2316  CGCallee Callee = CGCallee::forDirect(CalleePtr, GlobalDecl(D, Type));2317  EmitCall(Info, Callee, ReturnValueSlot(), Args, CallOrInvoke, false, Loc);2318 2319  // Generate vtable assumptions if we're constructing a complete object2320  // with a vtable.  We don't do this for base subobjects for two reasons:2321  // first, it's incorrect for classes with virtual bases, and second, we're2322  // about to overwrite the vptrs anyway.2323  // We also have to make sure if we can refer to vtable:2324  // - Otherwise we can refer to vtable if it's safe to speculatively emit.2325  // FIXME: If vtable is used by ctor/dtor, or if vtable is external and we are2326  // sure that definition of vtable is not hidden,2327  // then we are always safe to refer to it.2328  // FIXME: It looks like InstCombine is very inefficient on dealing with2329  // assumes. Make assumption loads require -fstrict-vtable-pointers temporarily.2330  if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&2331      ClassDecl->isDynamicClass() && Type != Ctor_Base &&2332      CGM.getCXXABI().canSpeculativelyEmitVTable(ClassDecl) &&2333      CGM.getCodeGenOpts().StrictVTablePointers)2334    EmitVTableAssumptionLoads(ClassDecl, This);2335}2336 2337void CodeGenFunction::EmitInheritedCXXConstructorCall(2338    const CXXConstructorDecl *D, bool ForVirtualBase, Address This,2339    bool InheritedFromVBase, const CXXInheritedCtorInitExpr *E) {2340  CallArgList Args;2341  CallArg ThisArg(RValue::get(getAsNaturalPointerTo(2342                      This, D->getThisType()->getPointeeType())),2343                  D->getThisType());2344 2345  // Forward the parameters.2346  if (InheritedFromVBase &&2347      CGM.getTarget().getCXXABI().hasConstructorVariants()) {2348    // Nothing to do; this construction is not responsible for constructing2349    // the base class containing the inherited constructor.2350    // FIXME: Can we just pass undef's for the remaining arguments if we don't2351    // have constructor variants?2352    Args.push_back(ThisArg);2353  } else if (!CXXInheritedCtorInitExprArgs.empty()) {2354    // The inheriting constructor was inlined; just inject its arguments.2355    assert(CXXInheritedCtorInitExprArgs.size() >= D->getNumParams() &&2356           "wrong number of parameters for inherited constructor call");2357    Args = CXXInheritedCtorInitExprArgs;2358    Args[0] = ThisArg;2359  } else {2360    // The inheriting constructor was not inlined. Emit delegating arguments.2361    Args.push_back(ThisArg);2362    const auto *OuterCtor = cast<CXXConstructorDecl>(CurCodeDecl);2363    assert(OuterCtor->getNumParams() == D->getNumParams());2364    assert(!OuterCtor->isVariadic() && "should have been inlined");2365 2366    for (const auto *Param : OuterCtor->parameters()) {2367      assert(getContext().hasSameUnqualifiedType(2368          OuterCtor->getParamDecl(Param->getFunctionScopeIndex())->getType(),2369          Param->getType()));2370      EmitDelegateCallArg(Args, Param, E->getLocation());2371 2372      // Forward __attribute__(pass_object_size).2373      if (Param->hasAttr<PassObjectSizeAttr>()) {2374        auto *POSParam = SizeArguments[Param];2375        assert(POSParam && "missing pass_object_size value for forwarding");2376        EmitDelegateCallArg(Args, POSParam, E->getLocation());2377      }2378    }2379  }2380 2381  EmitCXXConstructorCall(D, Ctor_Base, ForVirtualBase, /*Delegating*/false,2382                         This, Args, AggValueSlot::MayOverlap,2383                         E->getLocation(), /*NewPointerIsChecked*/true);2384}2385 2386void CodeGenFunction::EmitInlinedInheritingCXXConstructorCall(2387    const CXXConstructorDecl *Ctor, CXXCtorType CtorType, bool ForVirtualBase,2388    bool Delegating, CallArgList &Args) {2389  GlobalDecl GD(Ctor, CtorType);2390  InlinedInheritingConstructorScope Scope(*this, GD);2391  ApplyInlineDebugLocation DebugScope(*this, GD);2392  RunCleanupsScope RunCleanups(*this);2393 2394  // Save the arguments to be passed to the inherited constructor.2395  CXXInheritedCtorInitExprArgs = Args;2396 2397  FunctionArgList Params;2398  QualType RetType = BuildFunctionArgList(CurGD, Params);2399  FnRetTy = RetType;2400 2401  // Insert any ABI-specific implicit constructor arguments.2402  CGM.getCXXABI().addImplicitConstructorArgs(*this, Ctor, CtorType,2403                                             ForVirtualBase, Delegating, Args);2404 2405  // Emit a simplified prolog. We only need to emit the implicit params.2406  assert(Args.size() >= Params.size() && "too few arguments for call");2407  for (unsigned I = 0, N = Args.size(); I != N; ++I) {2408    if (I < Params.size() && isa<ImplicitParamDecl>(Params[I])) {2409      const RValue &RV = Args[I].getRValue(*this);2410      assert(!RV.isComplex() && "complex indirect params not supported");2411      ParamValue Val = RV.isScalar()2412                           ? ParamValue::forDirect(RV.getScalarVal())2413                           : ParamValue::forIndirect(RV.getAggregateAddress());2414      EmitParmDecl(*Params[I], Val, I + 1);2415    }2416  }2417 2418  // Create a return value slot if the ABI implementation wants one.2419  // FIXME: This is dumb, we should ask the ABI not to try to set the return2420  // value instead.2421  if (!RetType->isVoidType())2422    ReturnValue = CreateIRTemp(RetType, "retval.inhctor");2423 2424  CGM.getCXXABI().EmitInstanceFunctionProlog(*this);2425  CXXThisValue = CXXABIThisValue;2426 2427  // Directly emit the constructor initializers.2428  EmitCtorPrologue(Ctor, CtorType, Params);2429}2430 2431void CodeGenFunction::EmitVTableAssumptionLoad(const VPtr &Vptr, Address This) {2432  llvm::Value *VTableGlobal =2433      CGM.getCXXABI().getVTableAddressPoint(Vptr.Base, Vptr.VTableClass);2434  if (!VTableGlobal)2435    return;2436 2437  // We can just use the base offset in the complete class.2438  CharUnits NonVirtualOffset = Vptr.Base.getBaseOffset();2439 2440  if (!NonVirtualOffset.isZero())2441    This =2442        ApplyNonVirtualAndVirtualOffset(*this, This, NonVirtualOffset, nullptr,2443                                        Vptr.VTableClass, Vptr.NearestVBase);2444 2445  llvm::Value *VPtrValue =2446      GetVTablePtr(This, VTableGlobal->getType(), Vptr.VTableClass);2447  llvm::Value *Cmp =2448      Builder.CreateICmpEQ(VPtrValue, VTableGlobal, "cmp.vtables");2449  Builder.CreateAssumption(Cmp);2450}2451 2452void CodeGenFunction::EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl,2453                                                Address This) {2454  if (CGM.getCXXABI().doStructorsInitializeVPtrs(ClassDecl))2455    for (const VPtr &Vptr : getVTablePointers(ClassDecl))2456      EmitVTableAssumptionLoad(Vptr, This);2457}2458 2459void2460CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,2461                                                Address This, Address Src,2462                                                const CXXConstructExpr *E) {2463  const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();2464 2465  CallArgList Args;2466 2467  // Push the this ptr.2468  Args.add(RValue::get(getAsNaturalPointerTo(This, D->getThisType())),2469           D->getThisType());2470 2471  // Push the src ptr.2472  QualType QT = *(FPT->param_type_begin());2473  llvm::Type *t = CGM.getTypes().ConvertType(QT);2474  llvm::Value *Val = getAsNaturalPointerTo(Src, D->getThisType());2475  llvm::Value *SrcVal = Builder.CreateBitCast(Val, t);2476  Args.add(RValue::get(SrcVal), QT);2477 2478  // Skip over first argument (Src).2479  EmitCallArgs(Args, FPT, drop_begin(E->arguments(), 1), E->getConstructor(),2480               /*ParamsToSkip*/ 1);2481 2482  EmitCXXConstructorCall(D, Ctor_Complete, /*ForVirtualBase*/false,2483                         /*Delegating*/false, This, Args,2484                         AggValueSlot::MayOverlap, E->getExprLoc(),2485                         /*NewPointerIsChecked*/false);2486}2487 2488void2489CodeGenFunction::EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,2490                                                CXXCtorType CtorType,2491                                                const FunctionArgList &Args,2492                                                SourceLocation Loc) {2493  CallArgList DelegateArgs;2494 2495  FunctionArgList::const_iterator I = Args.begin(), E = Args.end();2496  assert(I != E && "no parameters to constructor");2497 2498  // this2499  Address This = LoadCXXThisAddress();2500  DelegateArgs.add(RValue::get(getAsNaturalPointerTo(2501                       This, (*I)->getType()->getPointeeType())),2502                   (*I)->getType());2503  ++I;2504 2505  // FIXME: The location of the VTT parameter in the parameter list is2506  // specific to the Itanium ABI and shouldn't be hardcoded here.2507  if (CGM.getCXXABI().NeedsVTTParameter(CurGD)) {2508    assert(I != E && "cannot skip vtt parameter, already done with args");2509    assert((*I)->getType()->isPointerType() &&2510           "skipping parameter not of vtt type");2511    ++I;2512  }2513 2514  // Explicit arguments.2515  for (; I != E; ++I) {2516    const VarDecl *param = *I;2517    // FIXME: per-argument source location2518    EmitDelegateCallArg(DelegateArgs, param, Loc);2519  }2520 2521  EmitCXXConstructorCall(Ctor, CtorType, /*ForVirtualBase=*/false,2522                         /*Delegating=*/true, This, DelegateArgs,2523                         AggValueSlot::MayOverlap, Loc,2524                         /*NewPointerIsChecked=*/true);2525}2526 2527namespace {2528  struct CallDelegatingCtorDtor final : EHScopeStack::Cleanup {2529    const CXXDestructorDecl *Dtor;2530    Address Addr;2531    CXXDtorType Type;2532 2533    CallDelegatingCtorDtor(const CXXDestructorDecl *D, Address Addr,2534                           CXXDtorType Type)2535      : Dtor(D), Addr(Addr), Type(Type) {}2536 2537    void Emit(CodeGenFunction &CGF, Flags flags) override {2538      // We are calling the destructor from within the constructor.2539      // Therefore, "this" should have the expected type.2540      QualType ThisTy = Dtor->getFunctionObjectParameterType();2541      CGF.EmitCXXDestructorCall(Dtor, Type, /*ForVirtualBase=*/false,2542                                /*Delegating=*/true, Addr, ThisTy);2543    }2544  };2545} // end anonymous namespace2546 2547void2548CodeGenFunction::EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,2549                                                  const FunctionArgList &Args) {2550  assert(Ctor->isDelegatingConstructor());2551 2552  Address ThisPtr = LoadCXXThisAddress();2553 2554  AggValueSlot AggSlot =2555    AggValueSlot::forAddr(ThisPtr, Qualifiers(),2556                          AggValueSlot::IsDestructed,2557                          AggValueSlot::DoesNotNeedGCBarriers,2558                          AggValueSlot::IsNotAliased,2559                          AggValueSlot::MayOverlap,2560                          AggValueSlot::IsNotZeroed,2561                          // Checks are made by the code that calls constructor.2562                          AggValueSlot::IsSanitizerChecked);2563 2564  EmitAggExpr(Ctor->init_begin()[0]->getInit(), AggSlot);2565 2566  const CXXRecordDecl *ClassDecl = Ctor->getParent();2567  if (CGM.getLangOpts().Exceptions && !ClassDecl->hasTrivialDestructor()) {2568    CXXDtorType Type =2569      CurGD.getCtorType() == Ctor_Complete ? Dtor_Complete : Dtor_Base;2570 2571    EHStack.pushCleanup<CallDelegatingCtorDtor>(EHCleanup,2572                                                ClassDecl->getDestructor(),2573                                                ThisPtr, Type);2574  }2575}2576 2577void CodeGenFunction::EmitCXXDestructorCall(const CXXDestructorDecl *DD,2578                                            CXXDtorType Type,2579                                            bool ForVirtualBase,2580                                            bool Delegating, Address This,2581                                            QualType ThisTy) {2582  CGM.getCXXABI().EmitDestructorCall(*this, DD, Type, ForVirtualBase,2583                                     Delegating, This, ThisTy);2584}2585 2586namespace {2587  struct CallLocalDtor final : EHScopeStack::Cleanup {2588    const CXXDestructorDecl *Dtor;2589    Address Addr;2590    QualType Ty;2591 2592    CallLocalDtor(const CXXDestructorDecl *D, Address Addr, QualType Ty)2593        : Dtor(D), Addr(Addr), Ty(Ty) {}2594 2595    void Emit(CodeGenFunction &CGF, Flags flags) override {2596      CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete,2597                                /*ForVirtualBase=*/false,2598                                /*Delegating=*/false, Addr, Ty);2599    }2600  };2601} // end anonymous namespace2602 2603void CodeGenFunction::PushDestructorCleanup(const CXXDestructorDecl *D,2604                                            QualType T, Address Addr) {2605  EHStack.pushCleanup<CallLocalDtor>(NormalAndEHCleanup, D, Addr, T);2606}2607 2608void CodeGenFunction::PushDestructorCleanup(QualType T, Address Addr) {2609  CXXRecordDecl *ClassDecl = T->getAsCXXRecordDecl();2610  if (!ClassDecl) return;2611  if (ClassDecl->hasTrivialDestructor()) return;2612 2613  const CXXDestructorDecl *D = ClassDecl->getDestructor();2614  assert(D && D->isUsed() && "destructor not marked as used!");2615  PushDestructorCleanup(D, T, Addr);2616}2617 2618void CodeGenFunction::InitializeVTablePointer(const VPtr &Vptr) {2619  // Compute the address point.2620  llvm::Value *VTableAddressPoint =2621      CGM.getCXXABI().getVTableAddressPointInStructor(2622          *this, Vptr.VTableClass, Vptr.Base, Vptr.NearestVBase);2623 2624  if (!VTableAddressPoint)2625    return;2626 2627  // Compute where to store the address point.2628  llvm::Value *VirtualOffset = nullptr;2629  CharUnits NonVirtualOffset = CharUnits::Zero();2630 2631  if (CGM.getCXXABI().isVirtualOffsetNeededForVTableField(*this, Vptr)) {2632    // We need to use the virtual base offset offset because the virtual base2633    // might have a different offset in the most derived class.2634 2635    VirtualOffset = CGM.getCXXABI().GetVirtualBaseClassOffset(2636        *this, LoadCXXThisAddress(), Vptr.VTableClass, Vptr.NearestVBase);2637    NonVirtualOffset = Vptr.OffsetFromNearestVBase;2638  } else {2639    // We can just use the base offset in the complete class.2640    NonVirtualOffset = Vptr.Base.getBaseOffset();2641  }2642 2643  // Apply the offsets.2644  Address VTableField = LoadCXXThisAddress();2645  if (!NonVirtualOffset.isZero() || VirtualOffset)2646    VTableField = ApplyNonVirtualAndVirtualOffset(2647        *this, VTableField, NonVirtualOffset, VirtualOffset, Vptr.VTableClass,2648        Vptr.NearestVBase);2649 2650  // Finally, store the address point. Use the same LLVM types as the field to2651  // support optimization.2652  unsigned GlobalsAS = CGM.getDataLayout().getDefaultGlobalsAddressSpace();2653  llvm::Type *PtrTy = llvm::PointerType::get(CGM.getLLVMContext(), GlobalsAS);2654  // vtable field is derived from `this` pointer, therefore they should be in2655  // the same addr space. Note that this might not be LLVM address space 0.2656  VTableField = VTableField.withElementType(PtrTy);2657 2658  if (auto AuthenticationInfo = CGM.getVTablePointerAuthInfo(2659          this, Vptr.Base.getBase(), VTableField.emitRawPointer(*this)))2660    VTableAddressPoint =2661        EmitPointerAuthSign(*AuthenticationInfo, VTableAddressPoint);2662 2663  llvm::StoreInst *Store = Builder.CreateStore(VTableAddressPoint, VTableField);2664  TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(PtrTy);2665  CGM.DecorateInstructionWithTBAA(Store, TBAAInfo);2666  if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&2667      CGM.getCodeGenOpts().StrictVTablePointers)2668    CGM.DecorateInstructionWithInvariantGroup(Store, Vptr.VTableClass);2669}2670 2671CodeGenFunction::VPtrsVector2672CodeGenFunction::getVTablePointers(const CXXRecordDecl *VTableClass) {2673  CodeGenFunction::VPtrsVector VPtrsResult;2674  VisitedVirtualBasesSetTy VBases;2675  getVTablePointers(BaseSubobject(VTableClass, CharUnits::Zero()),2676                    /*NearestVBase=*/nullptr,2677                    /*OffsetFromNearestVBase=*/CharUnits::Zero(),2678                    /*BaseIsNonVirtualPrimaryBase=*/false, VTableClass, VBases,2679                    VPtrsResult);2680  return VPtrsResult;2681}2682 2683void CodeGenFunction::getVTablePointers(BaseSubobject Base,2684                                        const CXXRecordDecl *NearestVBase,2685                                        CharUnits OffsetFromNearestVBase,2686                                        bool BaseIsNonVirtualPrimaryBase,2687                                        const CXXRecordDecl *VTableClass,2688                                        VisitedVirtualBasesSetTy &VBases,2689                                        VPtrsVector &Vptrs) {2690  // If this base is a non-virtual primary base the address point has already2691  // been set.2692  if (!BaseIsNonVirtualPrimaryBase) {2693    // Initialize the vtable pointer for this base.2694    VPtr Vptr = {Base, NearestVBase, OffsetFromNearestVBase, VTableClass};2695    Vptrs.push_back(Vptr);2696  }2697 2698  const CXXRecordDecl *RD = Base.getBase();2699 2700  // Traverse bases.2701  for (const auto &I : RD->bases()) {2702    auto *BaseDecl = I.getType()->castAsCXXRecordDecl();2703    // Ignore classes without a vtable.2704    if (!BaseDecl->isDynamicClass())2705      continue;2706 2707    CharUnits BaseOffset;2708    CharUnits BaseOffsetFromNearestVBase;2709    bool BaseDeclIsNonVirtualPrimaryBase;2710 2711    if (I.isVirtual()) {2712      // Check if we've visited this virtual base before.2713      if (!VBases.insert(BaseDecl).second)2714        continue;2715 2716      const ASTRecordLayout &Layout =2717        getContext().getASTRecordLayout(VTableClass);2718 2719      BaseOffset = Layout.getVBaseClassOffset(BaseDecl);2720      BaseOffsetFromNearestVBase = CharUnits::Zero();2721      BaseDeclIsNonVirtualPrimaryBase = false;2722    } else {2723      const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);2724 2725      BaseOffset = Base.getBaseOffset() + Layout.getBaseClassOffset(BaseDecl);2726      BaseOffsetFromNearestVBase =2727        OffsetFromNearestVBase + Layout.getBaseClassOffset(BaseDecl);2728      BaseDeclIsNonVirtualPrimaryBase = Layout.getPrimaryBase() == BaseDecl;2729    }2730 2731    getVTablePointers(2732        BaseSubobject(BaseDecl, BaseOffset),2733        I.isVirtual() ? BaseDecl : NearestVBase, BaseOffsetFromNearestVBase,2734        BaseDeclIsNonVirtualPrimaryBase, VTableClass, VBases, Vptrs);2735  }2736}2737 2738void CodeGenFunction::InitializeVTablePointers(const CXXRecordDecl *RD) {2739  // Ignore classes without a vtable.2740  if (!RD->isDynamicClass())2741    return;2742 2743  // Initialize the vtable pointers for this class and all of its bases.2744  if (CGM.getCXXABI().doStructorsInitializeVPtrs(RD))2745    for (const VPtr &Vptr : getVTablePointers(RD))2746      InitializeVTablePointer(Vptr);2747 2748  if (RD->getNumVBases())2749    CGM.getCXXABI().initializeHiddenVirtualInheritanceMembers(*this, RD);2750}2751 2752llvm::Value *CodeGenFunction::GetVTablePtr(Address This,2753                                           llvm::Type *VTableTy,2754                                           const CXXRecordDecl *RD,2755                                           VTableAuthMode AuthMode) {2756  Address VTablePtrSrc = This.withElementType(VTableTy);2757  llvm::Instruction *VTable = Builder.CreateLoad(VTablePtrSrc, "vtable");2758  TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTableTy);2759  CGM.DecorateInstructionWithTBAA(VTable, TBAAInfo);2760 2761  if (auto AuthenticationInfo =2762          CGM.getVTablePointerAuthInfo(this, RD, This.emitRawPointer(*this))) {2763    if (AuthMode != VTableAuthMode::UnsafeUbsanStrip) {2764      VTable = cast<llvm::Instruction>(2765          EmitPointerAuthAuth(*AuthenticationInfo, VTable));2766      if (AuthMode == VTableAuthMode::MustTrap) {2767        // This is clearly suboptimal but until we have an ability2768        // to rely on the authentication intrinsic trapping and force2769        // an authentication to occur we don't really have a choice.2770        VTable =2771            cast<llvm::Instruction>(Builder.CreateBitCast(VTable, Int8PtrTy));2772        Builder.CreateLoad(RawAddress(VTable, Int8Ty, CGM.getPointerAlign()),2773                           /* IsVolatile */ true);2774      }2775    } else {2776      VTable = cast<llvm::Instruction>(EmitPointerAuthAuth(2777          CGPointerAuthInfo(0, PointerAuthenticationMode::Strip, false, false,2778                            nullptr),2779          VTable));2780    }2781  }2782 2783  if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&2784      CGM.getCodeGenOpts().StrictVTablePointers)2785    CGM.DecorateInstructionWithInvariantGroup(VTable, RD);2786 2787  return VTable;2788}2789 2790// If a class has a single non-virtual base and does not introduce or override2791// virtual member functions or fields, it will have the same layout as its base.2792// This function returns the least derived such class.2793//2794// Casting an instance of a base class to such a derived class is technically2795// undefined behavior, but it is a relatively common hack for introducing member2796// functions on class instances with specific properties (e.g. llvm::Operator)2797// that works under most compilers and should not have security implications, so2798// we allow it by default. It can be disabled with -fsanitize=cfi-cast-strict.2799static const CXXRecordDecl *2800LeastDerivedClassWithSameLayout(const CXXRecordDecl *RD) {2801  if (!RD->field_empty())2802    return RD;2803 2804  if (RD->getNumVBases() != 0)2805    return RD;2806 2807  if (RD->getNumBases() != 1)2808    return RD;2809 2810  for (const CXXMethodDecl *MD : RD->methods()) {2811    if (MD->isVirtual()) {2812      // Virtual member functions are only ok if they are implicit destructors2813      // because the implicit destructor will have the same semantics as the2814      // base class's destructor if no fields are added.2815      if (isa<CXXDestructorDecl>(MD) && MD->isImplicit())2816        continue;2817      return RD;2818    }2819  }2820 2821  return LeastDerivedClassWithSameLayout(2822      RD->bases_begin()->getType()->getAsCXXRecordDecl());2823}2824 2825void CodeGenFunction::EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,2826                                                   llvm::Value *VTable,2827                                                   SourceLocation Loc) {2828  if (SanOpts.has(SanitizerKind::CFIVCall))2829    EmitVTablePtrCheckForCall(RD, VTable, CodeGenFunction::CFITCK_VCall, Loc);2830  else if (CGM.getCodeGenOpts().WholeProgramVTables &&2831           // Don't insert type test assumes if we are forcing public2832           // visibility.2833           !CGM.AlwaysHasLTOVisibilityPublic(RD)) {2834    CanQualType Ty = CGM.getContext().getCanonicalTagType(RD);2835    llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(Ty);2836    llvm::Value *TypeId =2837        llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD);2838 2839    // If we already know that the call has hidden LTO visibility, emit2840    // @llvm.type.test(). Otherwise emit @llvm.public.type.test(), which WPD2841    // will convert to @llvm.type.test() if we assert at link time that we have2842    // whole program visibility.2843    llvm::Intrinsic::ID IID = CGM.HasHiddenLTOVisibility(RD)2844                                  ? llvm::Intrinsic::type_test2845                                  : llvm::Intrinsic::public_type_test;2846    llvm::Value *TypeTest =2847        Builder.CreateCall(CGM.getIntrinsic(IID), {VTable, TypeId});2848    Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::assume), TypeTest);2849  }2850}2851 2852/// Converts the CFITypeCheckKind into SanitizerKind::SanitizerOrdinal and2853/// llvm::SanitizerStatKind.2854static std::pair<SanitizerKind::SanitizerOrdinal, llvm::SanitizerStatKind>2855SanitizerInfoFromCFICheckKind(CodeGenFunction::CFITypeCheckKind TCK) {2856  switch (TCK) {2857  case CodeGenFunction::CFITCK_VCall:2858    return std::make_pair(SanitizerKind::SO_CFIVCall, llvm::SanStat_CFI_VCall);2859  case CodeGenFunction::CFITCK_NVCall:2860    return std::make_pair(SanitizerKind::SO_CFINVCall,2861                          llvm::SanStat_CFI_NVCall);2862  case CodeGenFunction::CFITCK_DerivedCast:2863    return std::make_pair(SanitizerKind::SO_CFIDerivedCast,2864                          llvm::SanStat_CFI_DerivedCast);2865  case CodeGenFunction::CFITCK_UnrelatedCast:2866    return std::make_pair(SanitizerKind::SO_CFIUnrelatedCast,2867                          llvm::SanStat_CFI_UnrelatedCast);2868  case CodeGenFunction::CFITCK_ICall:2869  case CodeGenFunction::CFITCK_NVMFCall:2870  case CodeGenFunction::CFITCK_VMFCall:2871    llvm_unreachable("unexpected sanitizer kind");2872  }2873  llvm_unreachable("Unknown CFITypeCheckKind enum");2874}2875 2876void CodeGenFunction::EmitVTablePtrCheckForCall(const CXXRecordDecl *RD,2877                                                llvm::Value *VTable,2878                                                CFITypeCheckKind TCK,2879                                                SourceLocation Loc) {2880  if (!SanOpts.has(SanitizerKind::CFICastStrict))2881    RD = LeastDerivedClassWithSameLayout(RD);2882 2883  auto [Ordinal, _] = SanitizerInfoFromCFICheckKind(TCK);2884  SanitizerDebugLocation SanScope(this, {Ordinal},2885                                  SanitizerHandler::CFICheckFail);2886 2887  EmitVTablePtrCheck(RD, VTable, TCK, Loc);2888}2889 2890void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, Address Derived,2891                                                bool MayBeNull,2892                                                CFITypeCheckKind TCK,2893                                                SourceLocation Loc) {2894  if (!getLangOpts().CPlusPlus)2895    return;2896 2897  const auto *ClassDecl = T->getAsCXXRecordDecl();2898  if (!ClassDecl)2899    return;2900 2901  if (!ClassDecl->isCompleteDefinition() || !ClassDecl->isDynamicClass())2902    return;2903 2904  if (!SanOpts.has(SanitizerKind::CFICastStrict))2905    ClassDecl = LeastDerivedClassWithSameLayout(ClassDecl);2906 2907  auto [Ordinal, _] = SanitizerInfoFromCFICheckKind(TCK);2908  SanitizerDebugLocation SanScope(this, {Ordinal},2909                                  SanitizerHandler::CFICheckFail);2910 2911  llvm::BasicBlock *ContBlock = nullptr;2912 2913  if (MayBeNull) {2914    llvm::Value *DerivedNotNull =2915        Builder.CreateIsNotNull(Derived.emitRawPointer(*this), "cast.nonnull");2916 2917    llvm::BasicBlock *CheckBlock = createBasicBlock("cast.check");2918    ContBlock = createBasicBlock("cast.cont");2919 2920    Builder.CreateCondBr(DerivedNotNull, CheckBlock, ContBlock);2921 2922    EmitBlock(CheckBlock);2923  }2924 2925  llvm::Value *VTable;2926  std::tie(VTable, ClassDecl) =2927      CGM.getCXXABI().LoadVTablePtr(*this, Derived, ClassDecl);2928 2929  EmitVTablePtrCheck(ClassDecl, VTable, TCK, Loc);2930 2931  if (MayBeNull) {2932    Builder.CreateBr(ContBlock);2933    EmitBlock(ContBlock);2934  }2935}2936 2937void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD,2938                                         llvm::Value *VTable,2939                                         CFITypeCheckKind TCK,2940                                         SourceLocation Loc) {2941  assert(IsSanitizerScope);2942 2943  if (!CGM.getCodeGenOpts().SanitizeCfiCrossDso &&2944      !CGM.HasHiddenLTOVisibility(RD))2945    return;2946 2947  auto [M, SSK] = SanitizerInfoFromCFICheckKind(TCK);2948 2949  std::string TypeName = RD->getQualifiedNameAsString();2950  if (getContext().getNoSanitizeList().containsType(2951          SanitizerMask::bitPosToMask(M), TypeName))2952    return;2953 2954  EmitSanitizerStatReport(SSK);2955 2956  CanQualType T = CGM.getContext().getCanonicalTagType(RD);2957  llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(T);2958  llvm::Value *TypeId = llvm::MetadataAsValue::get(getLLVMContext(), MD);2959 2960  llvm::Value *TypeTest = Builder.CreateCall(2961      CGM.getIntrinsic(llvm::Intrinsic::type_test), {VTable, TypeId});2962 2963  llvm::Constant *StaticData[] = {2964      llvm::ConstantInt::get(Int8Ty, TCK),2965      EmitCheckSourceLocation(Loc),2966      EmitCheckTypeDescriptor(T),2967  };2968 2969  auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);2970  if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && CrossDsoTypeId) {2971    EmitCfiSlowPathCheck(M, TypeTest, CrossDsoTypeId, VTable, StaticData);2972    return;2973  }2974 2975  if (CGM.getCodeGenOpts().SanitizeTrap.has(M)) {2976    bool NoMerge = !CGM.getCodeGenOpts().SanitizeMergeHandlers.has(M);2977    EmitTrapCheck(TypeTest, SanitizerHandler::CFICheckFail, NoMerge);2978    return;2979  }2980 2981  llvm::Value *AllVtables = llvm::MetadataAsValue::get(2982      CGM.getLLVMContext(),2983      llvm::MDString::get(CGM.getLLVMContext(), "all-vtables"));2984  llvm::Value *ValidVtable = Builder.CreateCall(2985      CGM.getIntrinsic(llvm::Intrinsic::type_test), {VTable, AllVtables});2986  EmitCheck(std::make_pair(TypeTest, M), SanitizerHandler::CFICheckFail,2987            StaticData, {VTable, ValidVtable});2988}2989 2990bool CodeGenFunction::ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD) {2991  if (!CGM.getCodeGenOpts().WholeProgramVTables ||2992      !CGM.HasHiddenLTOVisibility(RD))2993    return false;2994 2995  if (CGM.getCodeGenOpts().VirtualFunctionElimination)2996    return true;2997 2998  if (!SanOpts.has(SanitizerKind::CFIVCall) ||2999      !CGM.getCodeGenOpts().SanitizeTrap.has(SanitizerKind::CFIVCall))3000    return false;3001 3002  std::string TypeName = RD->getQualifiedNameAsString();3003  return !getContext().getNoSanitizeList().containsType(SanitizerKind::CFIVCall,3004                                                        TypeName);3005}3006 3007llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad(3008    const CXXRecordDecl *RD, llvm::Value *VTable, llvm::Type *VTableTy,3009    uint64_t VTableByteOffset) {3010  auto CheckOrdinal = SanitizerKind::SO_CFIVCall;3011  auto CheckHandler = SanitizerHandler::CFICheckFail;3012  SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);3013 3014  EmitSanitizerStatReport(llvm::SanStat_CFI_VCall);3015 3016  CanQualType T = CGM.getContext().getCanonicalTagType(RD);3017  llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(T);3018  llvm::Value *TypeId = llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD);3019 3020  auto CheckedLoadIntrinsic = CGM.getVTables().useRelativeLayout()3021                                  ? llvm::Intrinsic::type_checked_load_relative3022                                  : llvm::Intrinsic::type_checked_load;3023  llvm::Value *CheckedLoad = Builder.CreateCall(3024      CGM.getIntrinsic(CheckedLoadIntrinsic),3025      {VTable, llvm::ConstantInt::get(Int32Ty, VTableByteOffset), TypeId});3026 3027  llvm::Value *CheckResult = Builder.CreateExtractValue(CheckedLoad, 1);3028 3029  std::string TypeName = RD->getQualifiedNameAsString();3030  if (SanOpts.has(SanitizerKind::CFIVCall) &&3031      !getContext().getNoSanitizeList().containsType(SanitizerKind::CFIVCall,3032                                                     TypeName)) {3033    EmitCheck(std::make_pair(CheckResult, CheckOrdinal), CheckHandler, {}, {});3034  }3035 3036  return Builder.CreateBitCast(Builder.CreateExtractValue(CheckedLoad, 0),3037                               VTableTy);3038}3039 3040void CodeGenFunction::EmitForwardingCallToLambda(3041    const CXXMethodDecl *callOperator, CallArgList &callArgs,3042    const CGFunctionInfo *calleeFnInfo, llvm::Constant *calleePtr) {3043  // Get the address of the call operator.3044  if (!calleeFnInfo)3045    calleeFnInfo = &CGM.getTypes().arrangeCXXMethodDeclaration(callOperator);3046 3047  if (!calleePtr)3048    calleePtr =3049        CGM.GetAddrOfFunction(GlobalDecl(callOperator),3050                              CGM.getTypes().GetFunctionType(*calleeFnInfo));3051 3052  // Prepare the return slot.3053  const FunctionProtoType *FPT =3054    callOperator->getType()->castAs<FunctionProtoType>();3055  QualType resultType = FPT->getReturnType();3056  ReturnValueSlot returnSlot;3057  if (!resultType->isVoidType() &&3058      calleeFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect &&3059      !hasScalarEvaluationKind(calleeFnInfo->getReturnType()))3060    returnSlot =3061        ReturnValueSlot(ReturnValue, resultType.isVolatileQualified(),3062                        /*IsUnused=*/false, /*IsExternallyDestructed=*/true);3063 3064  // We don't need to separately arrange the call arguments because3065  // the call can't be variadic anyway --- it's impossible to forward3066  // variadic arguments.3067 3068  // Now emit our call.3069  auto callee = CGCallee::forDirect(calleePtr, GlobalDecl(callOperator));3070  RValue RV = EmitCall(*calleeFnInfo, callee, returnSlot, callArgs);3071 3072  // If necessary, copy the returned value into the slot.3073  if (!resultType->isVoidType() && returnSlot.isNull()) {3074    if (getLangOpts().ObjCAutoRefCount && resultType->isObjCRetainableType()) {3075      RV = RValue::get(EmitARCRetainAutoreleasedReturnValue(RV.getScalarVal()));3076    }3077    EmitReturnOfRValue(RV, resultType);3078  } else3079    EmitBranchThroughCleanup(ReturnBlock);3080}3081 3082void CodeGenFunction::EmitLambdaBlockInvokeBody() {3083  const BlockDecl *BD = BlockInfo->getBlockDecl();3084  const VarDecl *variable = BD->capture_begin()->getVariable();3085  const CXXRecordDecl *Lambda = variable->getType()->getAsCXXRecordDecl();3086  const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();3087 3088  if (CallOp->isVariadic()) {3089    // FIXME: Making this work correctly is nasty because it requires either3090    // cloning the body of the call operator or making the call operator3091    // forward.3092    CGM.ErrorUnsupported(CurCodeDecl, "lambda conversion to variadic function");3093    return;3094  }3095 3096  // Start building arguments for forwarding call3097  CallArgList CallArgs;3098 3099  CanQualType ThisType =3100      getContext().getPointerType(getContext().getCanonicalTagType(Lambda));3101  Address ThisPtr = GetAddrOfBlockDecl(variable);3102  CallArgs.add(RValue::get(getAsNaturalPointerTo(ThisPtr, ThisType)), ThisType);3103 3104  // Add the rest of the parameters.3105  for (auto *param : BD->parameters())3106    EmitDelegateCallArg(CallArgs, param, param->getBeginLoc());3107 3108  assert(!Lambda->isGenericLambda() &&3109            "generic lambda interconversion to block not implemented");3110  EmitForwardingCallToLambda(CallOp, CallArgs);3111}3112 3113void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) {3114  if (MD->isVariadic()) {3115    // FIXME: Making this work correctly is nasty because it requires either3116    // cloning the body of the call operator or making the call operator3117    // forward.3118    CGM.ErrorUnsupported(MD, "lambda conversion to variadic function");3119    return;3120  }3121 3122  const CXXRecordDecl *Lambda = MD->getParent();3123 3124  // Start building arguments for forwarding call3125  CallArgList CallArgs;3126 3127  CanQualType LambdaType = getContext().getCanonicalTagType(Lambda);3128  CanQualType ThisType = getContext().getPointerType(LambdaType);3129  Address ThisPtr = CreateMemTemp(LambdaType, "unused.capture");3130  CallArgs.add(RValue::get(ThisPtr.emitRawPointer(*this)), ThisType);3131 3132  EmitLambdaDelegatingInvokeBody(MD, CallArgs);3133}3134 3135void CodeGenFunction::EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD,3136                                                     CallArgList &CallArgs) {3137  // Add the rest of the forwarded parameters.3138  for (auto *Param : MD->parameters())3139    EmitDelegateCallArg(CallArgs, Param, Param->getBeginLoc());3140 3141  const CXXRecordDecl *Lambda = MD->getParent();3142  const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();3143  // For a generic lambda, find the corresponding call operator specialization3144  // to which the call to the static-invoker shall be forwarded.3145  if (Lambda->isGenericLambda()) {3146    assert(MD->isFunctionTemplateSpecialization());3147    const TemplateArgumentList *TAL = MD->getTemplateSpecializationArgs();3148    FunctionTemplateDecl *CallOpTemplate = CallOp->getDescribedFunctionTemplate();3149    void *InsertPos = nullptr;3150    FunctionDecl *CorrespondingCallOpSpecialization =3151        CallOpTemplate->findSpecialization(TAL->asArray(), InsertPos);3152    assert(CorrespondingCallOpSpecialization);3153    CallOp = cast<CXXMethodDecl>(CorrespondingCallOpSpecialization);3154  }3155 3156  // Special lambda forwarding when there are inalloca parameters.3157  if (hasInAllocaArg(MD)) {3158    const CGFunctionInfo *ImplFnInfo = nullptr;3159    llvm::Function *ImplFn = nullptr;3160    EmitLambdaInAllocaImplFn(CallOp, &ImplFnInfo, &ImplFn);3161 3162    EmitForwardingCallToLambda(CallOp, CallArgs, ImplFnInfo, ImplFn);3163    return;3164  }3165 3166  EmitForwardingCallToLambda(CallOp, CallArgs);3167}3168 3169void CodeGenFunction::EmitLambdaInAllocaCallOpBody(const CXXMethodDecl *MD) {3170  if (MD->isVariadic()) {3171    // FIXME: Making this work correctly is nasty because it requires either3172    // cloning the body of the call operator or making the call operator forward.3173    CGM.ErrorUnsupported(MD, "lambda conversion to variadic function");3174    return;3175  }3176 3177  // Forward %this argument.3178  CallArgList CallArgs;3179  CanQualType LambdaType = getContext().getCanonicalTagType(MD->getParent());3180  CanQualType ThisType = getContext().getPointerType(LambdaType);3181  llvm::Value *ThisArg = CurFn->getArg(0);3182  CallArgs.add(RValue::get(ThisArg), ThisType);3183 3184  EmitLambdaDelegatingInvokeBody(MD, CallArgs);3185}3186 3187void CodeGenFunction::EmitLambdaInAllocaImplFn(3188    const CXXMethodDecl *CallOp, const CGFunctionInfo **ImplFnInfo,3189    llvm::Function **ImplFn) {3190  const CGFunctionInfo &FnInfo =3191      CGM.getTypes().arrangeCXXMethodDeclaration(CallOp);3192  llvm::Function *CallOpFn =3193      cast<llvm::Function>(CGM.GetAddrOfFunction(GlobalDecl(CallOp)));3194 3195  // Emit function containing the original call op body. __invoke will delegate3196  // to this function.3197  SmallVector<CanQualType, 4> ArgTypes;3198  for (auto I = FnInfo.arg_begin(); I != FnInfo.arg_end(); ++I)3199    ArgTypes.push_back(I->type);3200  *ImplFnInfo = &CGM.getTypes().arrangeLLVMFunctionInfo(3201      FnInfo.getReturnType(), FnInfoOpts::IsDelegateCall, ArgTypes,3202      FnInfo.getExtInfo(), {}, FnInfo.getRequiredArgs());3203 3204  // Create mangled name as if this was a method named __impl. If for some3205  // reason the name doesn't look as expected then just tack __impl to the3206  // front.3207  // TODO: Use the name mangler to produce the right name instead of using3208  // string replacement.3209  StringRef CallOpName = CallOpFn->getName();3210  std::string ImplName;3211  if (size_t Pos = CallOpName.find_first_of("<lambda"))3212    ImplName = ("?__impl@" + CallOpName.drop_front(Pos)).str();3213  else3214    ImplName = ("__impl" + CallOpName).str();3215 3216  llvm::Function *Fn = CallOpFn->getParent()->getFunction(ImplName);3217  if (!Fn) {3218    Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(**ImplFnInfo),3219                                llvm::GlobalValue::InternalLinkage, ImplName,3220                                CGM.getModule());3221    CGM.SetInternalFunctionAttributes(CallOp, Fn, **ImplFnInfo);3222 3223    const GlobalDecl &GD = GlobalDecl(CallOp);3224    const auto *D = cast<FunctionDecl>(GD.getDecl());3225    CodeGenFunction(CGM).GenerateCode(GD, Fn, **ImplFnInfo);3226    CGM.SetLLVMFunctionAttributesForDefinition(D, Fn);3227  }3228  *ImplFn = Fn;3229}3230