brintos / llvm-project-archived public Read only

1//===--- CGExprAgg.cpp - Emit LLVM Code from Aggregate Expressions --------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This contains code to emit Aggregate Expr nodes as LLVM code.10//11//===----------------------------------------------------------------------===//12 13#include "CGCXXABI.h"14#include "CGDebugInfo.h"15#include "CGHLSLRuntime.h"16#include "CGObjCRuntime.h"17#include "CGRecordLayout.h"18#include "CodeGenFunction.h"19#include "CodeGenModule.h"20#include "ConstantEmitter.h"21#include "EHScopeStack.h"22#include "TargetInfo.h"23#include "clang/AST/ASTContext.h"24#include "clang/AST/Attr.h"25#include "clang/AST/DeclCXX.h"26#include "clang/AST/DeclTemplate.h"27#include "clang/AST/StmtVisitor.h"28#include "llvm/IR/Constants.h"29#include "llvm/IR/Function.h"30#include "llvm/IR/GlobalVariable.h"31#include "llvm/IR/Instruction.h"32#include "llvm/IR/IntrinsicInst.h"33#include "llvm/IR/Intrinsics.h"34using namespace clang;35using namespace CodeGen;36 37//===----------------------------------------------------------------------===//38//                        Aggregate Expression Emitter39//===----------------------------------------------------------------------===//40 41namespace llvm {42extern cl::opt<bool> EnableSingleByteCoverage;43} // namespace llvm44 45namespace {46class AggExprEmitter : public StmtVisitor<AggExprEmitter> {47  CodeGenFunction &CGF;48  CGBuilderTy &Builder;49  AggValueSlot Dest;50  bool IsResultUnused;51 52  AggValueSlot EnsureSlot(QualType T) {53    if (!Dest.isIgnored()) return Dest;54    return CGF.CreateAggTemp(T, "agg.tmp.ensured");55  }56  void EnsureDest(QualType T) {57    if (!Dest.isIgnored()) return;58    Dest = CGF.CreateAggTemp(T, "agg.tmp.ensured");59  }60 61  // Calls `Fn` with a valid return value slot, potentially creating a temporary62  // to do so. If a temporary is created, an appropriate copy into `Dest` will63  // be emitted, as will lifetime markers.64  //65  // The given function should take a ReturnValueSlot, and return an RValue that66  // points to said slot.67  void withReturnValueSlot(const Expr *E,68                           llvm::function_ref<RValue(ReturnValueSlot)> Fn);69 70  void DoZeroInitPadding(uint64_t &PaddingStart, uint64_t PaddingEnd,71                         const FieldDecl *NextField);72 73public:74  AggExprEmitter(CodeGenFunction &cgf, AggValueSlot Dest, bool IsResultUnused)75    : CGF(cgf), Builder(CGF.Builder), Dest(Dest),76    IsResultUnused(IsResultUnused) { }77 78  //===--------------------------------------------------------------------===//79  //                               Utilities80  //===--------------------------------------------------------------------===//81 82  /// EmitAggLoadOfLValue - Given an expression with aggregate type that83  /// represents a value lvalue, this method emits the address of the lvalue,84  /// then loads the result into DestPtr.85  void EmitAggLoadOfLValue(const Expr *E);86 87  /// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.88  /// SrcIsRValue is true if source comes from an RValue.89  void EmitFinalDestCopy(QualType type, const LValue &src,90                         CodeGenFunction::ExprValueKind SrcValueKind =91                             CodeGenFunction::EVK_NonRValue);92  void EmitFinalDestCopy(QualType type, RValue src);93  void EmitCopy(QualType type, const AggValueSlot &dest,94                const AggValueSlot &src);95 96  void EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, QualType ArrayQTy,97                     Expr *ExprToVisit, ArrayRef<Expr *> Args,98                     Expr *ArrayFiller);99 100  AggValueSlot::NeedsGCBarriers_t needsGC(QualType T) {101    if (CGF.getLangOpts().getGC() && TypeRequiresGCollection(T))102      return AggValueSlot::NeedsGCBarriers;103    return AggValueSlot::DoesNotNeedGCBarriers;104  }105 106  bool TypeRequiresGCollection(QualType T);107 108  //===--------------------------------------------------------------------===//109  //                            Visitor Methods110  //===--------------------------------------------------------------------===//111 112  void Visit(Expr *E) {113    ApplyDebugLocation DL(CGF, E);114    StmtVisitor<AggExprEmitter>::Visit(E);115  }116 117  void VisitStmt(Stmt *S) {118    CGF.ErrorUnsupported(S, "aggregate expression");119  }120  void VisitParenExpr(ParenExpr *PE) { Visit(PE->getSubExpr()); }121  void VisitGenericSelectionExpr(GenericSelectionExpr *GE) {122    Visit(GE->getResultExpr());123  }124  void VisitCoawaitExpr(CoawaitExpr *E) {125    CGF.EmitCoawaitExpr(*E, Dest, IsResultUnused);126  }127  void VisitCoyieldExpr(CoyieldExpr *E) {128    CGF.EmitCoyieldExpr(*E, Dest, IsResultUnused);129  }130  void VisitUnaryCoawait(UnaryOperator *E) { Visit(E->getSubExpr()); }131  void VisitUnaryExtension(UnaryOperator *E) { Visit(E->getSubExpr()); }132  void VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *E) {133    return Visit(E->getReplacement());134  }135 136  void VisitConstantExpr(ConstantExpr *E) {137    EnsureDest(E->getType());138 139    if (llvm::Value *Result = ConstantEmitter(CGF).tryEmitConstantExpr(E)) {140      CGF.CreateCoercedStore(141          Result, Dest.getAddress(),142          llvm::TypeSize::getFixed(143              Dest.getPreferredSize(CGF.getContext(), E->getType())144                  .getQuantity()),145          E->getType().isVolatileQualified());146      return;147    }148    return Visit(E->getSubExpr());149  }150 151  // l-values.152  void VisitDeclRefExpr(DeclRefExpr *E) { EmitAggLoadOfLValue(E); }153  void VisitMemberExpr(MemberExpr *ME) { EmitAggLoadOfLValue(ME); }154  void VisitUnaryDeref(UnaryOperator *E) { EmitAggLoadOfLValue(E); }155  void VisitStringLiteral(StringLiteral *E) { EmitAggLoadOfLValue(E); }156  void VisitCompoundLiteralExpr(CompoundLiteralExpr *E);157  void VisitArraySubscriptExpr(ArraySubscriptExpr *E) {158    EmitAggLoadOfLValue(E);159  }160  void VisitPredefinedExpr(const PredefinedExpr *E) {161    EmitAggLoadOfLValue(E);162  }163 164  // Operators.165  void VisitCastExpr(CastExpr *E);166  void VisitCallExpr(const CallExpr *E);167  void VisitStmtExpr(const StmtExpr *E);168  void VisitBinaryOperator(const BinaryOperator *BO);169  void VisitPointerToDataMemberBinaryOperator(const BinaryOperator *BO);170  void VisitBinAssign(const BinaryOperator *E);171  void VisitBinComma(const BinaryOperator *E);172  void VisitBinCmp(const BinaryOperator *E);173  void VisitCXXRewrittenBinaryOperator(CXXRewrittenBinaryOperator *E) {174    Visit(E->getSemanticForm());175  }176 177  void VisitObjCMessageExpr(ObjCMessageExpr *E);178  void VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) {179    EmitAggLoadOfLValue(E);180  }181 182  void VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E);183  void VisitAbstractConditionalOperator(const AbstractConditionalOperator *CO);184  void VisitChooseExpr(const ChooseExpr *CE);185  void VisitInitListExpr(InitListExpr *E);186  void VisitCXXParenListOrInitListExpr(Expr *ExprToVisit, ArrayRef<Expr *> Args,187                                       FieldDecl *InitializedFieldInUnion,188                                       Expr *ArrayFiller);189  void VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,190                              llvm::Value *outerBegin = nullptr);191  void VisitImplicitValueInitExpr(ImplicitValueInitExpr *E);192  void VisitNoInitExpr(NoInitExpr *E) { } // Do nothing.193  void VisitCXXDefaultArgExpr(CXXDefaultArgExpr *DAE) {194    CodeGenFunction::CXXDefaultArgExprScope Scope(CGF, DAE);195    Visit(DAE->getExpr());196  }197  void VisitCXXDefaultInitExpr(CXXDefaultInitExpr *DIE) {198    CodeGenFunction::CXXDefaultInitExprScope Scope(CGF, DIE);199    Visit(DIE->getExpr());200  }201  void VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E);202  void VisitCXXConstructExpr(const CXXConstructExpr *E);203  void VisitCXXInheritedCtorInitExpr(const CXXInheritedCtorInitExpr *E);204  void VisitLambdaExpr(LambdaExpr *E);205  void VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E);206  void VisitExprWithCleanups(ExprWithCleanups *E);207  void VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E);208  void VisitCXXTypeidExpr(CXXTypeidExpr *E) { EmitAggLoadOfLValue(E); }209  void VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E);210  void VisitOpaqueValueExpr(OpaqueValueExpr *E);211 212  void VisitPseudoObjectExpr(PseudoObjectExpr *E) {213    if (E->isGLValue()) {214      LValue LV = CGF.EmitPseudoObjectLValue(E);215      return EmitFinalDestCopy(E->getType(), LV);216    }217 218    AggValueSlot Slot = EnsureSlot(E->getType());219    bool NeedsDestruction =220        !Slot.isExternallyDestructed() &&221        E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;222    if (NeedsDestruction)223      Slot.setExternallyDestructed();224    CGF.EmitPseudoObjectRValue(E, Slot);225    if (NeedsDestruction)226      CGF.pushDestroy(QualType::DK_nontrivial_c_struct, Slot.getAddress(),227                      E->getType());228  }229 230  void VisitVAArgExpr(VAArgExpr *E);231  void VisitCXXParenListInitExpr(CXXParenListInitExpr *E);232  void VisitCXXParenListOrInitListExpr(Expr *ExprToVisit, ArrayRef<Expr *> Args,233                                       Expr *ArrayFiller);234 235  void EmitInitializationToLValue(Expr *E, LValue Address);236  void EmitNullInitializationToLValue(LValue Address);237  //  case Expr::ChooseExprClass:238  void VisitCXXThrowExpr(const CXXThrowExpr *E) { CGF.EmitCXXThrowExpr(E); }239  void VisitAtomicExpr(AtomicExpr *E) {240    RValue Res = CGF.EmitAtomicExpr(E);241    EmitFinalDestCopy(E->getType(), Res);242  }243  void VisitPackIndexingExpr(PackIndexingExpr *E) {244    Visit(E->getSelectedExpr());245  }246};247}  // end anonymous namespace.248 249//===----------------------------------------------------------------------===//250//                                Utilities251//===----------------------------------------------------------------------===//252 253/// EmitAggLoadOfLValue - Given an expression with aggregate type that254/// represents a value lvalue, this method emits the address of the lvalue,255/// then loads the result into DestPtr.256void AggExprEmitter::EmitAggLoadOfLValue(const Expr *E) {257  LValue LV = CGF.EmitLValue(E);258 259  // If the type of the l-value is atomic, then do an atomic load.260  if (LV.getType()->isAtomicType() || CGF.LValueIsSuitableForInlineAtomic(LV)) {261    CGF.EmitAtomicLoad(LV, E->getExprLoc(), Dest);262    return;263  }264 265  EmitFinalDestCopy(E->getType(), LV);266}267 268/// True if the given aggregate type requires special GC API calls.269bool AggExprEmitter::TypeRequiresGCollection(QualType T) {270  // Only record types have members that might require garbage collection.271  const auto *Record = T->getAsRecordDecl();272  if (!Record)273    return false;274 275  // Don't mess with non-trivial C++ types.276  if (isa<CXXRecordDecl>(Record) &&277      (cast<CXXRecordDecl>(Record)->hasNonTrivialCopyConstructor() ||278       !cast<CXXRecordDecl>(Record)->hasTrivialDestructor()))279    return false;280 281  // Check whether the type has an object member.282  return Record->hasObjectMember();283}284 285void AggExprEmitter::withReturnValueSlot(286    const Expr *E, llvm::function_ref<RValue(ReturnValueSlot)> EmitCall) {287  QualType RetTy = E->getType();288  bool RequiresDestruction =289      !Dest.isExternallyDestructed() &&290      RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct;291 292  // If it makes no observable difference, save a memcpy + temporary.293  //294  // We need to always provide our own temporary if destruction is required.295  // Otherwise, EmitCall will emit its own, notice that it's "unused", and end296  // its lifetime before we have the chance to emit a proper destructor call.297  bool UseTemp = Dest.isPotentiallyAliased() || Dest.requiresGCollection() ||298                 (RequiresDestruction && Dest.isIgnored());299 300  Address RetAddr = Address::invalid();301 302  EHScopeStack::stable_iterator LifetimeEndBlock;303  llvm::IntrinsicInst *LifetimeStartInst = nullptr;304  if (!UseTemp) {305    RetAddr = Dest.getAddress();306  } else {307    RetAddr = CGF.CreateMemTempWithoutCast(RetTy, "tmp");308    if (CGF.EmitLifetimeStart(RetAddr.getBasePointer())) {309      LifetimeStartInst =310          cast<llvm::IntrinsicInst>(std::prev(Builder.GetInsertPoint()));311      assert(LifetimeStartInst->getIntrinsicID() ==312                 llvm::Intrinsic::lifetime_start &&313             "Last insertion wasn't a lifetime.start?");314 315      CGF.pushFullExprCleanup<CodeGenFunction::CallLifetimeEnd>(316          NormalEHLifetimeMarker, RetAddr);317      LifetimeEndBlock = CGF.EHStack.stable_begin();318    }319  }320 321  RValue Src =322      EmitCall(ReturnValueSlot(RetAddr, Dest.isVolatile(), IsResultUnused,323                               Dest.isExternallyDestructed()));324 325  if (!UseTemp)326    return;327 328  assert(Dest.isIgnored() || Dest.emitRawPointer(CGF) !=329                                 Src.getAggregatePointer(E->getType(), CGF));330  EmitFinalDestCopy(E->getType(), Src);331 332  if (!RequiresDestruction && LifetimeStartInst) {333    // If there's no dtor to run, the copy was the last use of our temporary.334    // Since we're not guaranteed to be in an ExprWithCleanups, clean up335    // eagerly.336    CGF.DeactivateCleanupBlock(LifetimeEndBlock, LifetimeStartInst);337    CGF.EmitLifetimeEnd(RetAddr.getBasePointer());338  }339}340 341/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.342void AggExprEmitter::EmitFinalDestCopy(QualType type, RValue src) {343  assert(src.isAggregate() && "value must be aggregate value!");344  LValue srcLV = CGF.MakeAddrLValue(src.getAggregateAddress(), type);345  EmitFinalDestCopy(type, srcLV, CodeGenFunction::EVK_RValue);346}347 348/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.349void AggExprEmitter::EmitFinalDestCopy(350    QualType type, const LValue &src,351    CodeGenFunction::ExprValueKind SrcValueKind) {352  // If Dest is ignored, then we're evaluating an aggregate expression353  // in a context that doesn't care about the result.  Note that loads354  // from volatile l-values force the existence of a non-ignored355  // destination.356  if (Dest.isIgnored())357    return;358 359  // Copy non-trivial C structs here.360  LValue DstLV = CGF.MakeAddrLValue(361      Dest.getAddress(), Dest.isVolatile() ? type.withVolatile() : type);362 363  if (SrcValueKind == CodeGenFunction::EVK_RValue) {364    if (type.isNonTrivialToPrimitiveDestructiveMove() == QualType::PCK_Struct) {365      if (Dest.isPotentiallyAliased())366        CGF.callCStructMoveAssignmentOperator(DstLV, src);367      else368        CGF.callCStructMoveConstructor(DstLV, src);369      return;370    }371  } else {372    if (type.isNonTrivialToPrimitiveCopy() == QualType::PCK_Struct) {373      if (Dest.isPotentiallyAliased())374        CGF.callCStructCopyAssignmentOperator(DstLV, src);375      else376        CGF.callCStructCopyConstructor(DstLV, src);377      return;378    }379  }380 381  AggValueSlot srcAgg = AggValueSlot::forLValue(382      src, AggValueSlot::IsDestructed, needsGC(type), AggValueSlot::IsAliased,383      AggValueSlot::MayOverlap);384  EmitCopy(type, Dest, srcAgg);385}386 387/// Perform a copy from the source into the destination.388///389/// \param type - the type of the aggregate being copied; qualifiers are390///   ignored391void AggExprEmitter::EmitCopy(QualType type, const AggValueSlot &dest,392                              const AggValueSlot &src) {393  if (dest.requiresGCollection()) {394    CharUnits sz = dest.getPreferredSize(CGF.getContext(), type);395    llvm::Value *size = llvm::ConstantInt::get(CGF.SizeTy, sz.getQuantity());396    CGF.CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF,397                                                      dest.getAddress(),398                                                      src.getAddress(),399                                                      size);400    return;401  }402 403  // If the result of the assignment is used, copy the LHS there also.404  // It's volatile if either side is.  Use the minimum alignment of405  // the two sides.406  LValue DestLV = CGF.MakeAddrLValue(dest.getAddress(), type);407  LValue SrcLV = CGF.MakeAddrLValue(src.getAddress(), type);408  CGF.EmitAggregateCopy(DestLV, SrcLV, type, dest.mayOverlap(),409                        dest.isVolatile() || src.isVolatile());410}411 412/// Emit the initializer for a std::initializer_list initialized with a413/// real initializer list.414void415AggExprEmitter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) {416  // Emit an array containing the elements.  The array is externally destructed417  // if the std::initializer_list object is.418  ASTContext &Ctx = CGF.getContext();419  LValue Array = CGF.EmitLValue(E->getSubExpr());420  assert(Array.isSimple() && "initializer_list array not a simple lvalue");421  Address ArrayPtr = Array.getAddress();422 423  const ConstantArrayType *ArrayType =424      Ctx.getAsConstantArrayType(E->getSubExpr()->getType());425  assert(ArrayType && "std::initializer_list constructed from non-array");426 427  auto *Record = E->getType()->castAsRecordDecl();428  RecordDecl::field_iterator Field = Record->field_begin();429  assert(Field != Record->field_end() &&430         Ctx.hasSameType(Field->getType()->getPointeeType(),431                         ArrayType->getElementType()) &&432         "Expected std::initializer_list first field to be const E *");433 434  // Start pointer.435  AggValueSlot Dest = EnsureSlot(E->getType());436  LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());437  LValue Start = CGF.EmitLValueForFieldInitialization(DestLV, *Field);438  llvm::Value *ArrayStart = ArrayPtr.emitRawPointer(CGF);439  CGF.EmitStoreThroughLValue(RValue::get(ArrayStart), Start);440  ++Field;441  assert(Field != Record->field_end() &&442         "Expected std::initializer_list to have two fields");443 444  llvm::Value *Size = Builder.getInt(ArrayType->getSize());445  LValue EndOrLength = CGF.EmitLValueForFieldInitialization(DestLV, *Field);446  if (Ctx.hasSameType(Field->getType(), Ctx.getSizeType())) {447    // Length.448    CGF.EmitStoreThroughLValue(RValue::get(Size), EndOrLength);449 450  } else {451    // End pointer.452    assert(Field->getType()->isPointerType() &&453           Ctx.hasSameType(Field->getType()->getPointeeType(),454                           ArrayType->getElementType()) &&455           "Expected std::initializer_list second field to be const E *");456    llvm::Value *Zero = llvm::ConstantInt::get(CGF.PtrDiffTy, 0);457    llvm::Value *IdxEnd[] = { Zero, Size };458    llvm::Value *ArrayEnd = Builder.CreateInBoundsGEP(459        ArrayPtr.getElementType(), ArrayPtr.emitRawPointer(CGF), IdxEnd,460        "arrayend");461    CGF.EmitStoreThroughLValue(RValue::get(ArrayEnd), EndOrLength);462  }463 464  assert(++Field == Record->field_end() &&465         "Expected std::initializer_list to only have two fields");466}467 468/// Determine if E is a trivial array filler, that is, one that is469/// equivalent to zero-initialization.470static bool isTrivialFiller(Expr *E) {471  if (!E)472    return true;473 474  if (isa<ImplicitValueInitExpr>(E))475    return true;476 477  if (auto *ILE = dyn_cast<InitListExpr>(E)) {478    if (ILE->getNumInits())479      return false;480    return isTrivialFiller(ILE->getArrayFiller());481  }482 483  if (auto *Cons = dyn_cast_or_null<CXXConstructExpr>(E))484    return Cons->getConstructor()->isDefaultConstructor() &&485           Cons->getConstructor()->isTrivial();486 487  // FIXME: Are there other cases where we can avoid emitting an initializer?488  return false;489}490 491// emit an elementwise cast where the RHS is a scalar or vector492// or emit an aggregate splat cast493static void EmitHLSLScalarElementwiseAndSplatCasts(CodeGenFunction &CGF,494                                                   LValue DestVal,495                                                   llvm::Value *SrcVal,496                                                   QualType SrcTy,497                                                   SourceLocation Loc) {498  // Flatten our destination499  SmallVector<LValue, 16> StoreList;500  CGF.FlattenAccessAndTypeLValue(DestVal, StoreList);501 502  bool isVector = false;503  if (auto *VT = SrcTy->getAs<VectorType>()) {504    isVector = true;505    SrcTy = VT->getElementType();506    assert(StoreList.size() <= VT->getNumElements() &&507           "Cannot perform HLSL flat cast when vector source \508           object has less elements than flattened destination \509           object.");510  }511 512  for (unsigned I = 0, Size = StoreList.size(); I < Size; I++) {513    LValue DestLVal = StoreList[I];514    llvm::Value *Load =515        isVector ? CGF.Builder.CreateExtractElement(SrcVal, I, "vec.load")516                 : SrcVal;517    llvm::Value *Cast =518        CGF.EmitScalarConversion(Load, SrcTy, DestLVal.getType(), Loc);519    CGF.EmitStoreThroughLValue(RValue::get(Cast), DestLVal);520  }521}522 523// emit a flat cast where the RHS is an aggregate524static void EmitHLSLElementwiseCast(CodeGenFunction &CGF, LValue DestVal,525                                    LValue SrcVal, SourceLocation Loc) {526  // Flatten our destination527  SmallVector<LValue, 16> StoreList;528  CGF.FlattenAccessAndTypeLValue(DestVal, StoreList);529  // Flatten our src530  SmallVector<LValue, 16> LoadList;531  CGF.FlattenAccessAndTypeLValue(SrcVal, LoadList);532 533  assert(StoreList.size() <= LoadList.size() &&534         "Cannot perform HLSL elementwise cast when flattened source object \535          has less elements than flattened destination object.");536  // apply casts to what we load from LoadList537  // and store result in Dest538  for (unsigned I = 0, E = StoreList.size(); I < E; I++) {539    LValue DestLVal = StoreList[I];540    LValue SrcLVal = LoadList[I];541    RValue RVal = CGF.EmitLoadOfLValue(SrcLVal, Loc);542    assert(RVal.isScalar() && "All flattened source values should be scalars");543    llvm::Value *Val = RVal.getScalarVal();544    llvm::Value *Cast = CGF.EmitScalarConversion(Val, SrcLVal.getType(),545                                                 DestLVal.getType(), Loc);546    CGF.EmitStoreThroughLValue(RValue::get(Cast), DestLVal);547  }548}549 550/// Emit initialization of an array from an initializer list. ExprToVisit must551/// be either an InitListEpxr a CXXParenInitListExpr.552void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,553                                   QualType ArrayQTy, Expr *ExprToVisit,554                                   ArrayRef<Expr *> Args, Expr *ArrayFiller) {555  uint64_t NumInitElements = Args.size();556 557  uint64_t NumArrayElements = AType->getNumElements();558  for (const auto *Init : Args) {559    if (const auto *Embed = dyn_cast<EmbedExpr>(Init->IgnoreParenImpCasts())) {560      NumInitElements += Embed->getDataElementCount() - 1;561      if (NumInitElements > NumArrayElements) {562        NumInitElements = NumArrayElements;563        break;564      }565    }566  }567 568  assert(NumInitElements <= NumArrayElements);569 570  QualType elementType =571      CGF.getContext().getAsArrayType(ArrayQTy)->getElementType();572  CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType);573  CharUnits elementAlign =574    DestPtr.getAlignment().alignmentOfArrayElement(elementSize);575  llvm::Type *llvmElementType = CGF.ConvertTypeForMem(elementType);576 577  // Consider initializing the array by copying from a global. For this to be578  // more efficient than per-element initialization, the size of the elements579  // with explicit initializers should be large enough.580  if (NumInitElements * elementSize.getQuantity() > 16 &&581      elementType.isTriviallyCopyableType(CGF.getContext())) {582    CodeGen::CodeGenModule &CGM = CGF.CGM;583    ConstantEmitter Emitter(CGF);584    QualType GVArrayQTy = CGM.getContext().getAddrSpaceQualType(585        CGM.getContext().removeAddrSpaceQualType(ArrayQTy),586        CGM.GetGlobalConstantAddressSpace());587    LangAS AS = GVArrayQTy.getAddressSpace();588    if (llvm::Constant *C =589            Emitter.tryEmitForInitializer(ExprToVisit, AS, GVArrayQTy)) {590      auto GV = new llvm::GlobalVariable(591          CGM.getModule(), C->getType(),592          /* isConstant= */ true, llvm::GlobalValue::PrivateLinkage, C,593          "constinit",594          /* InsertBefore= */ nullptr, llvm::GlobalVariable::NotThreadLocal,595          CGM.getContext().getTargetAddressSpace(AS));596      Emitter.finalize(GV);597      CharUnits Align = CGM.getContext().getTypeAlignInChars(GVArrayQTy);598      GV->setAlignment(Align.getAsAlign());599      Address GVAddr(GV, GV->getValueType(), Align);600      EmitFinalDestCopy(ArrayQTy, CGF.MakeAddrLValue(GVAddr, GVArrayQTy));601      return;602    }603  }604 605  // Exception safety requires us to destroy all the606  // already-constructed members if an initializer throws.607  // For that, we'll need an EH cleanup.608  QualType::DestructionKind dtorKind = elementType.isDestructedType();609  Address endOfInit = Address::invalid();610  CodeGenFunction::CleanupDeactivationScope deactivation(CGF);611 612  llvm::Value *begin = DestPtr.emitRawPointer(CGF);613  if (dtorKind) {614    CodeGenFunction::AllocaTrackerRAII allocaTracker(CGF);615    // In principle we could tell the cleanup where we are more616    // directly, but the control flow can get so varied here that it617    // would actually be quite complex.  Therefore we go through an618    // alloca.619    llvm::Instruction *dominatingIP =620        Builder.CreateFlagLoad(llvm::ConstantInt::getNullValue(CGF.Int8PtrTy));621    endOfInit = CGF.CreateTempAlloca(begin->getType(), CGF.getPointerAlign(),622                                     "arrayinit.endOfInit");623    Builder.CreateStore(begin, endOfInit);624    CGF.pushIrregularPartialArrayCleanup(begin, endOfInit, elementType,625                                         elementAlign,626                                         CGF.getDestroyer(dtorKind));627    cast<EHCleanupScope>(*CGF.EHStack.find(CGF.EHStack.stable_begin()))628        .AddAuxAllocas(allocaTracker.Take());629 630    CGF.DeferredDeactivationCleanupStack.push_back(631        {CGF.EHStack.stable_begin(), dominatingIP});632  }633 634  llvm::Value *one = llvm::ConstantInt::get(CGF.SizeTy, 1);635 636  auto Emit = [&](Expr *Init, uint64_t ArrayIndex) {637    llvm::Value *element = begin;638    if (ArrayIndex > 0) {639      element = Builder.CreateInBoundsGEP(640          llvmElementType, begin,641          llvm::ConstantInt::get(CGF.SizeTy, ArrayIndex), "arrayinit.element");642 643      // Tell the cleanup that it needs to destroy up to this644      // element.  TODO: some of these stores can be trivially645      // observed to be unnecessary.646      if (endOfInit.isValid())647        Builder.CreateStore(element, endOfInit);648    }649 650    LValue elementLV = CGF.MakeAddrLValue(651        Address(element, llvmElementType, elementAlign), elementType);652    EmitInitializationToLValue(Init, elementLV);653    return true;654  };655 656  unsigned ArrayIndex = 0;657  // Emit the explicit initializers.658  for (uint64_t i = 0; i != NumInitElements; ++i) {659    if (ArrayIndex >= NumInitElements)660      break;661    if (auto *EmbedS = dyn_cast<EmbedExpr>(Args[i]->IgnoreParenImpCasts())) {662      EmbedS->doForEachDataElement(Emit, ArrayIndex);663    } else {664      Emit(Args[i], ArrayIndex);665      ArrayIndex++;666    }667  }668 669  // Check whether there's a non-trivial array-fill expression.670  bool hasTrivialFiller = isTrivialFiller(ArrayFiller);671 672  // Any remaining elements need to be zero-initialized, possibly673  // using the filler expression.  We can skip this if the we're674  // emitting to zeroed memory.675  if (NumInitElements != NumArrayElements &&676      !(Dest.isZeroed() && hasTrivialFiller &&677        CGF.getTypes().isZeroInitializable(elementType))) {678 679    // Use an actual loop.  This is basically680    //   do { *array++ = filler; } while (array != end);681 682    // Advance to the start of the rest of the array.683    llvm::Value *element = begin;684    if (NumInitElements) {685      element = Builder.CreateInBoundsGEP(686          llvmElementType, element,687          llvm::ConstantInt::get(CGF.SizeTy, NumInitElements),688          "arrayinit.start");689      if (endOfInit.isValid()) Builder.CreateStore(element, endOfInit);690    }691 692    // Compute the end of the array.693    llvm::Value *end = Builder.CreateInBoundsGEP(694        llvmElementType, begin,695        llvm::ConstantInt::get(CGF.SizeTy, NumArrayElements), "arrayinit.end");696 697    llvm::BasicBlock *entryBB = Builder.GetInsertBlock();698    llvm::BasicBlock *bodyBB = CGF.createBasicBlock("arrayinit.body");699 700    // Jump into the body.701    CGF.EmitBlock(bodyBB);702    llvm::PHINode *currentElement =703      Builder.CreatePHI(element->getType(), 2, "arrayinit.cur");704    currentElement->addIncoming(element, entryBB);705 706    // Emit the actual filler expression.707    {708      // C++1z [class.temporary]p5:709      //   when a default constructor is called to initialize an element of710      //   an array with no corresponding initializer [...] the destruction of711      //   every temporary created in a default argument is sequenced before712      //   the construction of the next array element, if any713      CodeGenFunction::RunCleanupsScope CleanupsScope(CGF);714      LValue elementLV = CGF.MakeAddrLValue(715          Address(currentElement, llvmElementType, elementAlign), elementType);716      if (ArrayFiller)717        EmitInitializationToLValue(ArrayFiller, elementLV);718      else719        EmitNullInitializationToLValue(elementLV);720    }721 722    // Move on to the next element.723    llvm::Value *nextElement = Builder.CreateInBoundsGEP(724        llvmElementType, currentElement, one, "arrayinit.next");725 726    // Tell the EH cleanup that we finished with the last element.727    if (endOfInit.isValid()) Builder.CreateStore(nextElement, endOfInit);728 729    // Leave the loop if we're done.730    llvm::Value *done = Builder.CreateICmpEQ(nextElement, end,731                                             "arrayinit.done");732    llvm::BasicBlock *endBB = CGF.createBasicBlock("arrayinit.end");733    Builder.CreateCondBr(done, endBB, bodyBB);734    currentElement->addIncoming(nextElement, Builder.GetInsertBlock());735 736    CGF.EmitBlock(endBB);737  }738}739 740//===----------------------------------------------------------------------===//741//                            Visitor Methods742//===----------------------------------------------------------------------===//743 744void AggExprEmitter::VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E){745  Visit(E->getSubExpr());746}747 748void AggExprEmitter::VisitOpaqueValueExpr(OpaqueValueExpr *e) {749  // If this is a unique OVE, just visit its source expression.750  if (e->isUnique())751    Visit(e->getSourceExpr());752  else753    EmitFinalDestCopy(e->getType(), CGF.getOrCreateOpaqueLValueMapping(e));754}755 756void757AggExprEmitter::VisitCompoundLiteralExpr(CompoundLiteralExpr *E) {758  if (Dest.isPotentiallyAliased()) {759    // Just emit a load of the lvalue + a copy, because our compound literal760    // might alias the destination.761    EmitAggLoadOfLValue(E);762    return;763  }764 765  AggValueSlot Slot = EnsureSlot(E->getType());766 767  // Block-scope compound literals are destroyed at the end of the enclosing768  // scope in C.769  bool Destruct =770      !CGF.getLangOpts().CPlusPlus && !Slot.isExternallyDestructed();771  if (Destruct)772    Slot.setExternallyDestructed();773 774  CGF.EmitAggExpr(E->getInitializer(), Slot);775 776  if (Destruct)777    if (QualType::DestructionKind DtorKind = E->getType().isDestructedType())778      CGF.pushLifetimeExtendedDestroy(779          CGF.getCleanupKind(DtorKind), Slot.getAddress(), E->getType(),780          CGF.getDestroyer(DtorKind), DtorKind & EHCleanup);781}782 783/// Attempt to look through various unimportant expressions to find a784/// cast of the given kind.785static Expr *findPeephole(Expr *op, CastKind kind, const ASTContext &ctx) {786  op = op->IgnoreParenNoopCasts(ctx);787  if (auto castE = dyn_cast<CastExpr>(op)) {788    if (castE->getCastKind() == kind)789      return castE->getSubExpr();790  }791  return nullptr;792}793 794void AggExprEmitter::VisitCastExpr(CastExpr *E) {795  if (const auto *ECE = dyn_cast<ExplicitCastExpr>(E))796    CGF.CGM.EmitExplicitCastExprType(ECE, &CGF);797  switch (E->getCastKind()) {798  case CK_Dynamic: {799    // FIXME: Can this actually happen? We have no test coverage for it.800    assert(isa<CXXDynamicCastExpr>(E) && "CK_Dynamic without a dynamic_cast?");801    LValue LV = CGF.EmitCheckedLValue(E->getSubExpr(),802                                      CodeGenFunction::TCK_Load);803    // FIXME: Do we also need to handle property references here?804    if (LV.isSimple())805      CGF.EmitDynamicCast(LV.getAddress(), cast<CXXDynamicCastExpr>(E));806    else807      CGF.CGM.ErrorUnsupported(E, "non-simple lvalue dynamic_cast");808 809    if (!Dest.isIgnored())810      CGF.CGM.ErrorUnsupported(E, "lvalue dynamic_cast with a destination");811    break;812  }813 814  case CK_ToUnion: {815    // Evaluate even if the destination is ignored.816    if (Dest.isIgnored()) {817      CGF.EmitAnyExpr(E->getSubExpr(), AggValueSlot::ignored(),818                      /*ignoreResult=*/true);819      break;820    }821 822    // GCC union extension823    QualType Ty = E->getSubExpr()->getType();824    Address CastPtr = Dest.getAddress().withElementType(CGF.ConvertType(Ty));825    EmitInitializationToLValue(E->getSubExpr(),826                               CGF.MakeAddrLValue(CastPtr, Ty));827    break;828  }829 830  case CK_LValueToRValueBitCast: {831    if (Dest.isIgnored()) {832      CGF.EmitAnyExpr(E->getSubExpr(), AggValueSlot::ignored(),833                      /*ignoreResult=*/true);834      break;835    }836 837    LValue SourceLV = CGF.EmitLValue(E->getSubExpr());838    Address SourceAddress = SourceLV.getAddress().withElementType(CGF.Int8Ty);839    Address DestAddress = Dest.getAddress().withElementType(CGF.Int8Ty);840    llvm::Value *SizeVal = llvm::ConstantInt::get(841        CGF.SizeTy,842        CGF.getContext().getTypeSizeInChars(E->getType()).getQuantity());843    Builder.CreateMemCpy(DestAddress, SourceAddress, SizeVal);844    break;845  }846 847  case CK_DerivedToBase:848  case CK_BaseToDerived:849  case CK_UncheckedDerivedToBase: {850    llvm_unreachable("cannot perform hierarchy conversion in EmitAggExpr: "851                "should have been unpacked before we got here");852  }853 854  case CK_NonAtomicToAtomic:855  case CK_AtomicToNonAtomic: {856    bool isToAtomic = (E->getCastKind() == CK_NonAtomicToAtomic);857 858    // Determine the atomic and value types.859    QualType atomicType = E->getSubExpr()->getType();860    QualType valueType = E->getType();861    if (isToAtomic) std::swap(atomicType, valueType);862 863    assert(atomicType->isAtomicType());864    assert(CGF.getContext().hasSameUnqualifiedType(valueType,865                          atomicType->castAs<AtomicType>()->getValueType()));866 867    // Just recurse normally if we're ignoring the result or the868    // atomic type doesn't change representation.869    if (Dest.isIgnored() || !CGF.CGM.isPaddedAtomicType(atomicType)) {870      return Visit(E->getSubExpr());871    }872 873    CastKind peepholeTarget =874      (isToAtomic ? CK_AtomicToNonAtomic : CK_NonAtomicToAtomic);875 876    // These two cases are reverses of each other; try to peephole them.877    if (Expr *op =878            findPeephole(E->getSubExpr(), peepholeTarget, CGF.getContext())) {879      assert(CGF.getContext().hasSameUnqualifiedType(op->getType(),880                                                     E->getType()) &&881           "peephole significantly changed types?");882      return Visit(op);883    }884 885    // If we're converting an r-value of non-atomic type to an r-value886    // of atomic type, just emit directly into the relevant sub-object.887    if (isToAtomic) {888      AggValueSlot valueDest = Dest;889      if (!valueDest.isIgnored() && CGF.CGM.isPaddedAtomicType(atomicType)) {890        // Zero-initialize.  (Strictly speaking, we only need to initialize891        // the padding at the end, but this is simpler.)892        if (!Dest.isZeroed())893          CGF.EmitNullInitialization(Dest.getAddress(), atomicType);894 895        // Build a GEP to refer to the subobject.896        Address valueAddr =897            CGF.Builder.CreateStructGEP(valueDest.getAddress(), 0);898        valueDest = AggValueSlot::forAddr(valueAddr,899                                          valueDest.getQualifiers(),900                                          valueDest.isExternallyDestructed(),901                                          valueDest.requiresGCollection(),902                                          valueDest.isPotentiallyAliased(),903                                          AggValueSlot::DoesNotOverlap,904                                          AggValueSlot::IsZeroed);905      }906 907      CGF.EmitAggExpr(E->getSubExpr(), valueDest);908      return;909    }910 911    // Otherwise, we're converting an atomic type to a non-atomic type.912    // Make an atomic temporary, emit into that, and then copy the value out.913    AggValueSlot atomicSlot =914      CGF.CreateAggTemp(atomicType, "atomic-to-nonatomic.temp");915    CGF.EmitAggExpr(E->getSubExpr(), atomicSlot);916 917    Address valueAddr = Builder.CreateStructGEP(atomicSlot.getAddress(), 0);918    RValue rvalue = RValue::getAggregate(valueAddr, atomicSlot.isVolatile());919    return EmitFinalDestCopy(valueType, rvalue);920  }921  case CK_AddressSpaceConversion:922     return Visit(E->getSubExpr());923 924  case CK_LValueToRValue:925    // If we're loading from a volatile type, force the destination926    // into existence.927    if (E->getSubExpr()->getType().isVolatileQualified()) {928      bool Destruct =929          !Dest.isExternallyDestructed() &&930          E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;931      if (Destruct)932        Dest.setExternallyDestructed();933      EnsureDest(E->getType());934      Visit(E->getSubExpr());935 936      if (Destruct)937        CGF.pushDestroy(QualType::DK_nontrivial_c_struct, Dest.getAddress(),938                        E->getType());939 940      return;941    }942 943    [[fallthrough]];944 945  case CK_HLSLArrayRValue:946    Visit(E->getSubExpr());947    break;948  case CK_HLSLAggregateSplatCast: {949    Expr *Src = E->getSubExpr();950    QualType SrcTy = Src->getType();951    RValue RV = CGF.EmitAnyExpr(Src);952    LValue DestLVal = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());953    SourceLocation Loc = E->getExprLoc();954 955    assert(RV.isScalar() && SrcTy->isScalarType() &&956           "RHS of HLSL splat cast must be a scalar.");957    llvm::Value *SrcVal = RV.getScalarVal();958    EmitHLSLScalarElementwiseAndSplatCasts(CGF, DestLVal, SrcVal, SrcTy, Loc);959    break;960  }961  case CK_HLSLElementwiseCast: {962    Expr *Src = E->getSubExpr();963    QualType SrcTy = Src->getType();964    RValue RV = CGF.EmitAnyExpr(Src);965    LValue DestLVal = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());966    SourceLocation Loc = E->getExprLoc();967 968    if (RV.isScalar()) {969      llvm::Value *SrcVal = RV.getScalarVal();970      assert(SrcTy->isVectorType() &&971             "HLSL Elementwise cast doesn't handle splatting.");972      EmitHLSLScalarElementwiseAndSplatCasts(CGF, DestLVal, SrcVal, SrcTy, Loc);973    } else {974      assert(RV.isAggregate() &&975             "Can't perform HLSL Aggregate cast on a complex type.");976      Address SrcVal = RV.getAggregateAddress();977      EmitHLSLElementwiseCast(CGF, DestLVal, CGF.MakeAddrLValue(SrcVal, SrcTy),978                              Loc);979    }980    break;981  }982  case CK_NoOp:983  case CK_UserDefinedConversion:984  case CK_ConstructorConversion:985    assert(CGF.getContext().hasSameUnqualifiedType(E->getSubExpr()->getType(),986                                                   E->getType()) &&987           "Implicit cast types must be compatible");988    Visit(E->getSubExpr());989    break;990 991  case CK_LValueBitCast:992    llvm_unreachable("should not be emitting lvalue bitcast as rvalue");993 994  case CK_Dependent:995  case CK_BitCast:996  case CK_ArrayToPointerDecay:997  case CK_FunctionToPointerDecay:998  case CK_NullToPointer:999  case CK_NullToMemberPointer:1000  case CK_BaseToDerivedMemberPointer:1001  case CK_DerivedToBaseMemberPointer:1002  case CK_MemberPointerToBoolean:1003  case CK_ReinterpretMemberPointer:1004  case CK_IntegralToPointer:1005  case CK_PointerToIntegral:1006  case CK_PointerToBoolean:1007  case CK_ToVoid:1008  case CK_VectorSplat:1009  case CK_IntegralCast:1010  case CK_BooleanToSignedIntegral:1011  case CK_IntegralToBoolean:1012  case CK_IntegralToFloating:1013  case CK_FloatingToIntegral:1014  case CK_FloatingToBoolean:1015  case CK_FloatingCast:1016  case CK_CPointerToObjCPointerCast:1017  case CK_BlockPointerToObjCPointerCast:1018  case CK_AnyPointerToBlockPointerCast:1019  case CK_ObjCObjectLValueCast:1020  case CK_FloatingRealToComplex:1021  case CK_FloatingComplexToReal:1022  case CK_FloatingComplexToBoolean:1023  case CK_FloatingComplexCast:1024  case CK_FloatingComplexToIntegralComplex:1025  case CK_IntegralRealToComplex:1026  case CK_IntegralComplexToReal:1027  case CK_IntegralComplexToBoolean:1028  case CK_IntegralComplexCast:1029  case CK_IntegralComplexToFloatingComplex:1030  case CK_ARCProduceObject:1031  case CK_ARCConsumeObject:1032  case CK_ARCReclaimReturnedObject:1033  case CK_ARCExtendBlockObject:1034  case CK_CopyAndAutoreleaseBlockObject:1035  case CK_BuiltinFnToFnPtr:1036  case CK_ZeroToOCLOpaqueType:1037  case CK_MatrixCast:1038  case CK_HLSLVectorTruncation:1039 1040  case CK_IntToOCLSampler:1041  case CK_FloatingToFixedPoint:1042  case CK_FixedPointToFloating:1043  case CK_FixedPointCast:1044  case CK_FixedPointToBoolean:1045  case CK_FixedPointToIntegral:1046  case CK_IntegralToFixedPoint:1047    llvm_unreachable("cast kind invalid for aggregate types");1048  }1049}1050 1051void AggExprEmitter::VisitCallExpr(const CallExpr *E) {1052  if (E->getCallReturnType(CGF.getContext())->isReferenceType()) {1053    EmitAggLoadOfLValue(E);1054    return;1055  }1056 1057  withReturnValueSlot(E, [&](ReturnValueSlot Slot) {1058    return CGF.EmitCallExpr(E, Slot);1059  });1060}1061 1062void AggExprEmitter::VisitObjCMessageExpr(ObjCMessageExpr *E) {1063  withReturnValueSlot(E, [&](ReturnValueSlot Slot) {1064    return CGF.EmitObjCMessageExpr(E, Slot);1065  });1066}1067 1068void AggExprEmitter::VisitBinComma(const BinaryOperator *E) {1069  CGF.EmitIgnoredExpr(E->getLHS());1070  Visit(E->getRHS());1071}1072 1073void AggExprEmitter::VisitStmtExpr(const StmtExpr *E) {1074  CodeGenFunction::StmtExprEvaluation eval(CGF);1075  CGF.EmitCompoundStmt(*E->getSubStmt(), true, Dest);1076}1077 1078enum CompareKind {1079  CK_Less,1080  CK_Greater,1081  CK_Equal,1082};1083 1084static llvm::Value *EmitCompare(CGBuilderTy &Builder, CodeGenFunction &CGF,1085                                const BinaryOperator *E, llvm::Value *LHS,1086                                llvm::Value *RHS, CompareKind Kind,1087                                const char *NameSuffix = "") {1088  QualType ArgTy = E->getLHS()->getType();1089  if (const ComplexType *CT = ArgTy->getAs<ComplexType>())1090    ArgTy = CT->getElementType();1091 1092  if (const auto *MPT = ArgTy->getAs<MemberPointerType>()) {1093    assert(Kind == CK_Equal &&1094           "member pointers may only be compared for equality");1095    return CGF.CGM.getCXXABI().EmitMemberPointerComparison(1096        CGF, LHS, RHS, MPT, /*IsInequality*/ false);1097  }1098 1099  // Compute the comparison instructions for the specified comparison kind.1100  struct CmpInstInfo {1101    const char *Name;1102    llvm::CmpInst::Predicate FCmp;1103    llvm::CmpInst::Predicate SCmp;1104    llvm::CmpInst::Predicate UCmp;1105  };1106  CmpInstInfo InstInfo = [&]() -> CmpInstInfo {1107    using FI = llvm::FCmpInst;1108    using II = llvm::ICmpInst;1109    switch (Kind) {1110    case CK_Less:1111      return {"cmp.lt", FI::FCMP_OLT, II::ICMP_SLT, II::ICMP_ULT};1112    case CK_Greater:1113      return {"cmp.gt", FI::FCMP_OGT, II::ICMP_SGT, II::ICMP_UGT};1114    case CK_Equal:1115      return {"cmp.eq", FI::FCMP_OEQ, II::ICMP_EQ, II::ICMP_EQ};1116    }1117    llvm_unreachable("Unrecognised CompareKind enum");1118  }();1119 1120  if (ArgTy->hasFloatingRepresentation())1121    return Builder.CreateFCmp(InstInfo.FCmp, LHS, RHS,1122                              llvm::Twine(InstInfo.Name) + NameSuffix);1123  if (ArgTy->isIntegralOrEnumerationType() || ArgTy->isPointerType()) {1124    auto Inst =1125        ArgTy->hasSignedIntegerRepresentation() ? InstInfo.SCmp : InstInfo.UCmp;1126    return Builder.CreateICmp(Inst, LHS, RHS,1127                              llvm::Twine(InstInfo.Name) + NameSuffix);1128  }1129 1130  llvm_unreachable("unsupported aggregate binary expression should have "1131                   "already been handled");1132}1133 1134void AggExprEmitter::VisitBinCmp(const BinaryOperator *E) {1135  using llvm::BasicBlock;1136  using llvm::PHINode;1137  using llvm::Value;1138  assert(CGF.getContext().hasSameType(E->getLHS()->getType(),1139                                      E->getRHS()->getType()));1140  const ComparisonCategoryInfo &CmpInfo =1141      CGF.getContext().CompCategories.getInfoForType(E->getType());1142  assert(CmpInfo.Record->isTriviallyCopyable() &&1143         "cannot copy non-trivially copyable aggregate");1144 1145  QualType ArgTy = E->getLHS()->getType();1146 1147  if (!ArgTy->isIntegralOrEnumerationType() && !ArgTy->isRealFloatingType() &&1148      !ArgTy->isNullPtrType() && !ArgTy->isPointerType() &&1149      !ArgTy->isMemberPointerType() && !ArgTy->isAnyComplexType()) {1150    return CGF.ErrorUnsupported(E, "aggregate three-way comparison");1151  }1152  bool IsComplex = ArgTy->isAnyComplexType();1153 1154  // Evaluate the operands to the expression and extract their values.1155  auto EmitOperand = [&](Expr *E) -> std::pair<Value *, Value *> {1156    RValue RV = CGF.EmitAnyExpr(E);1157    if (RV.isScalar())1158      return {RV.getScalarVal(), nullptr};1159    if (RV.isAggregate())1160      return {RV.getAggregatePointer(E->getType(), CGF), nullptr};1161    assert(RV.isComplex());1162    return RV.getComplexVal();1163  };1164  auto LHSValues = EmitOperand(E->getLHS()),1165       RHSValues = EmitOperand(E->getRHS());1166 1167  auto EmitCmp = [&](CompareKind K) {1168    Value *Cmp = EmitCompare(Builder, CGF, E, LHSValues.first, RHSValues.first,1169                             K, IsComplex ? ".r" : "");1170    if (!IsComplex)1171      return Cmp;1172    assert(K == CompareKind::CK_Equal);1173    Value *CmpImag = EmitCompare(Builder, CGF, E, LHSValues.second,1174                                 RHSValues.second, K, ".i");1175    return Builder.CreateAnd(Cmp, CmpImag, "and.eq");1176  };1177  auto EmitCmpRes = [&](const ComparisonCategoryInfo::ValueInfo *VInfo) {1178    return Builder.getInt(VInfo->getIntValue());1179  };1180 1181  Value *Select;1182  if (ArgTy->isNullPtrType()) {1183    Select = EmitCmpRes(CmpInfo.getEqualOrEquiv());1184  } else if (!CmpInfo.isPartial()) {1185    Value *SelectOne =1186        Builder.CreateSelect(EmitCmp(CK_Less), EmitCmpRes(CmpInfo.getLess()),1187                             EmitCmpRes(CmpInfo.getGreater()), "sel.lt");1188    Select = Builder.CreateSelect(EmitCmp(CK_Equal),1189                                  EmitCmpRes(CmpInfo.getEqualOrEquiv()),1190                                  SelectOne, "sel.eq");1191  } else {1192    Value *SelectEq = Builder.CreateSelect(1193        EmitCmp(CK_Equal), EmitCmpRes(CmpInfo.getEqualOrEquiv()),1194        EmitCmpRes(CmpInfo.getUnordered()), "sel.eq");1195    Value *SelectGT = Builder.CreateSelect(EmitCmp(CK_Greater),1196                                           EmitCmpRes(CmpInfo.getGreater()),1197                                           SelectEq, "sel.gt");1198    Select = Builder.CreateSelect(1199        EmitCmp(CK_Less), EmitCmpRes(CmpInfo.getLess()), SelectGT, "sel.lt");1200  }1201  // Create the return value in the destination slot.1202  EnsureDest(E->getType());1203  LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());1204 1205  // Emit the address of the first (and only) field in the comparison category1206  // type, and initialize it from the constant integer value selected above.1207  LValue FieldLV = CGF.EmitLValueForFieldInitialization(1208      DestLV, *CmpInfo.Record->field_begin());1209  CGF.EmitStoreThroughLValue(RValue::get(Select), FieldLV, /*IsInit*/ true);1210 1211  // All done! The result is in the Dest slot.1212}1213 1214void AggExprEmitter::VisitBinaryOperator(const BinaryOperator *E) {1215  if (E->getOpcode() == BO_PtrMemD || E->getOpcode() == BO_PtrMemI)1216    VisitPointerToDataMemberBinaryOperator(E);1217  else1218    CGF.ErrorUnsupported(E, "aggregate binary expression");1219}1220 1221void AggExprEmitter::VisitPointerToDataMemberBinaryOperator(1222                                                    const BinaryOperator *E) {1223  LValue LV = CGF.EmitPointerToDataMemberBinaryExpr(E);1224  EmitFinalDestCopy(E->getType(), LV);1225}1226 1227/// Is the value of the given expression possibly a reference to or1228/// into a __block variable?1229static bool isBlockVarRef(const Expr *E) {1230  // Make sure we look through parens.1231  E = E->IgnoreParens();1232 1233  // Check for a direct reference to a __block variable.1234  if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E)) {1235    const VarDecl *var = dyn_cast<VarDecl>(DRE->getDecl());1236    return (var && var->hasAttr<BlocksAttr>());1237  }1238 1239  // More complicated stuff.1240 1241  // Binary operators.1242  if (const BinaryOperator *op = dyn_cast<BinaryOperator>(E)) {1243    // For an assignment or pointer-to-member operation, just care1244    // about the LHS.1245    if (op->isAssignmentOp() || op->isPtrMemOp())1246      return isBlockVarRef(op->getLHS());1247 1248    // For a comma, just care about the RHS.1249    if (op->getOpcode() == BO_Comma)1250      return isBlockVarRef(op->getRHS());1251 1252    // FIXME: pointer arithmetic?1253    return false;1254 1255  // Check both sides of a conditional operator.1256  } else if (const AbstractConditionalOperator *op1257               = dyn_cast<AbstractConditionalOperator>(E)) {1258    return isBlockVarRef(op->getTrueExpr())1259        || isBlockVarRef(op->getFalseExpr());1260 1261  // OVEs are required to support BinaryConditionalOperators.1262  } else if (const OpaqueValueExpr *op1263               = dyn_cast<OpaqueValueExpr>(E)) {1264    if (const Expr *src = op->getSourceExpr())1265      return isBlockVarRef(src);1266 1267  // Casts are necessary to get things like (*(int*)&var) = foo().1268  // We don't really care about the kind of cast here, except1269  // we don't want to look through l2r casts, because it's okay1270  // to get the *value* in a __block variable.1271  } else if (const CastExpr *cast = dyn_cast<CastExpr>(E)) {1272    if (cast->getCastKind() == CK_LValueToRValue)1273      return false;1274    return isBlockVarRef(cast->getSubExpr());1275 1276  // Handle unary operators.  Again, just aggressively look through1277  // it, ignoring the operation.1278  } else if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(E)) {1279    return isBlockVarRef(uop->getSubExpr());1280 1281  // Look into the base of a field access.1282  } else if (const MemberExpr *mem = dyn_cast<MemberExpr>(E)) {1283    return isBlockVarRef(mem->getBase());1284 1285  // Look into the base of a subscript.1286  } else if (const ArraySubscriptExpr *sub = dyn_cast<ArraySubscriptExpr>(E)) {1287    return isBlockVarRef(sub->getBase());1288  }1289 1290  return false;1291}1292 1293void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) {1294  ApplyAtomGroup Grp(CGF.getDebugInfo());1295  // For an assignment to work, the value on the right has1296  // to be compatible with the value on the left.1297  assert(CGF.getContext().hasSameUnqualifiedType(E->getLHS()->getType(),1298                                                 E->getRHS()->getType())1299         && "Invalid assignment");1300 1301  // If the LHS might be a __block variable, and the RHS can1302  // potentially cause a block copy, we need to evaluate the RHS first1303  // so that the assignment goes the right place.1304  // This is pretty semantically fragile.1305  if (isBlockVarRef(E->getLHS()) &&1306      E->getRHS()->HasSideEffects(CGF.getContext())) {1307    // Ensure that we have a destination, and evaluate the RHS into that.1308    EnsureDest(E->getRHS()->getType());1309    Visit(E->getRHS());1310 1311    // Now emit the LHS and copy into it.1312    LValue LHS = CGF.EmitCheckedLValue(E->getLHS(), CodeGenFunction::TCK_Store);1313 1314    // That copy is an atomic copy if the LHS is atomic.1315    if (LHS.getType()->isAtomicType() ||1316        CGF.LValueIsSuitableForInlineAtomic(LHS)) {1317      CGF.EmitAtomicStore(Dest.asRValue(), LHS, /*isInit*/ false);1318      return;1319    }1320 1321    EmitCopy(E->getLHS()->getType(),1322             AggValueSlot::forLValue(LHS, AggValueSlot::IsDestructed,1323                                     needsGC(E->getLHS()->getType()),1324                                     AggValueSlot::IsAliased,1325                                     AggValueSlot::MayOverlap),1326             Dest);1327    return;1328  }1329 1330  LValue LHS = CGF.EmitLValue(E->getLHS());1331 1332  // If we have an atomic type, evaluate into the destination and then1333  // do an atomic copy.1334  if (LHS.getType()->isAtomicType() ||1335      CGF.LValueIsSuitableForInlineAtomic(LHS)) {1336    EnsureDest(E->getRHS()->getType());1337    Visit(E->getRHS());1338    CGF.EmitAtomicStore(Dest.asRValue(), LHS, /*isInit*/ false);1339    return;1340  }1341 1342  // Codegen the RHS so that it stores directly into the LHS.1343  AggValueSlot LHSSlot = AggValueSlot::forLValue(1344      LHS, AggValueSlot::IsDestructed, needsGC(E->getLHS()->getType()),1345      AggValueSlot::IsAliased, AggValueSlot::MayOverlap);1346  // A non-volatile aggregate destination might have volatile member.1347  if (!LHSSlot.isVolatile() &&1348      CGF.hasVolatileMember(E->getLHS()->getType()))1349    LHSSlot.setVolatile(true);1350 1351  CGF.EmitAggExpr(E->getRHS(), LHSSlot);1352 1353  // Copy into the destination if the assignment isn't ignored.1354  EmitFinalDestCopy(E->getType(), LHS);1355 1356  if (!Dest.isIgnored() && !Dest.isExternallyDestructed() &&1357      E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct)1358    CGF.pushDestroy(QualType::DK_nontrivial_c_struct, Dest.getAddress(),1359                    E->getType());1360}1361 1362void AggExprEmitter::1363VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {1364  llvm::BasicBlock *LHSBlock = CGF.createBasicBlock("cond.true");1365  llvm::BasicBlock *RHSBlock = CGF.createBasicBlock("cond.false");1366  llvm::BasicBlock *ContBlock = CGF.createBasicBlock("cond.end");1367 1368  // Bind the common expression if necessary.1369  CodeGenFunction::OpaqueValueMapping binding(CGF, E);1370 1371  CodeGenFunction::ConditionalEvaluation eval(CGF);1372  CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock,1373                           CGF.getProfileCount(E));1374 1375  // Save whether the destination's lifetime is externally managed.1376  bool isExternallyDestructed = Dest.isExternallyDestructed();1377  bool destructNonTrivialCStruct =1378      !isExternallyDestructed &&1379      E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;1380  isExternallyDestructed |= destructNonTrivialCStruct;1381  Dest.setExternallyDestructed(isExternallyDestructed);1382 1383  eval.begin(CGF);1384  CGF.EmitBlock(LHSBlock);1385  if (llvm::EnableSingleByteCoverage)1386    CGF.incrementProfileCounter(E->getTrueExpr());1387  else1388    CGF.incrementProfileCounter(E);1389  Visit(E->getTrueExpr());1390  eval.end(CGF);1391 1392  assert(CGF.HaveInsertPoint() && "expression evaluation ended with no IP!");1393  CGF.Builder.CreateBr(ContBlock);1394 1395  // If the result of an agg expression is unused, then the emission1396  // of the LHS might need to create a destination slot.  That's fine1397  // with us, and we can safely emit the RHS into the same slot, but1398  // we shouldn't claim that it's already being destructed.1399  Dest.setExternallyDestructed(isExternallyDestructed);1400 1401  eval.begin(CGF);1402  CGF.EmitBlock(RHSBlock);1403  if (llvm::EnableSingleByteCoverage)1404    CGF.incrementProfileCounter(E->getFalseExpr());1405  Visit(E->getFalseExpr());1406  eval.end(CGF);1407 1408  if (destructNonTrivialCStruct)1409    CGF.pushDestroy(QualType::DK_nontrivial_c_struct, Dest.getAddress(),1410                    E->getType());1411 1412  CGF.EmitBlock(ContBlock);1413  if (llvm::EnableSingleByteCoverage)1414    CGF.incrementProfileCounter(E);1415}1416 1417void AggExprEmitter::VisitChooseExpr(const ChooseExpr *CE) {1418  Visit(CE->getChosenSubExpr());1419}1420 1421void AggExprEmitter::VisitVAArgExpr(VAArgExpr *VE) {1422  Address ArgValue = Address::invalid();1423  CGF.EmitVAArg(VE, ArgValue, Dest);1424 1425  // If EmitVAArg fails, emit an error.1426  if (!ArgValue.isValid()) {1427    CGF.ErrorUnsupported(VE, "aggregate va_arg expression");1428    return;1429  }1430}1431 1432void AggExprEmitter::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {1433  // Ensure that we have a slot, but if we already do, remember1434  // whether it was externally destructed.1435  bool wasExternallyDestructed = Dest.isExternallyDestructed();1436  EnsureDest(E->getType());1437 1438  // We're going to push a destructor if there isn't already one.1439  Dest.setExternallyDestructed();1440 1441  Visit(E->getSubExpr());1442 1443  // Push that destructor we promised.1444  if (!wasExternallyDestructed)1445    CGF.EmitCXXTemporary(E->getTemporary(), E->getType(), Dest.getAddress());1446}1447 1448void1449AggExprEmitter::VisitCXXConstructExpr(const CXXConstructExpr *E) {1450  AggValueSlot Slot = EnsureSlot(E->getType());1451  CGF.EmitCXXConstructExpr(E, Slot);1452}1453 1454void AggExprEmitter::VisitCXXInheritedCtorInitExpr(1455    const CXXInheritedCtorInitExpr *E) {1456  AggValueSlot Slot = EnsureSlot(E->getType());1457  CGF.EmitInheritedCXXConstructorCall(1458      E->getConstructor(), E->constructsVBase(), Slot.getAddress(),1459      E->inheritedFromVBase(), E);1460}1461 1462void1463AggExprEmitter::VisitLambdaExpr(LambdaExpr *E) {1464  AggValueSlot Slot = EnsureSlot(E->getType());1465  LValue SlotLV = CGF.MakeAddrLValue(Slot.getAddress(), E->getType());1466 1467  // We'll need to enter cleanup scopes in case any of the element1468  // initializers throws an exception or contains branch out of the expressions.1469  CodeGenFunction::CleanupDeactivationScope scope(CGF);1470 1471  CXXRecordDecl::field_iterator CurField = E->getLambdaClass()->field_begin();1472  for (LambdaExpr::const_capture_init_iterator i = E->capture_init_begin(),1473                                               e = E->capture_init_end();1474       i != e; ++i, ++CurField) {1475    // Emit initialization1476    LValue LV = CGF.EmitLValueForFieldInitialization(SlotLV, *CurField);1477    if (CurField->hasCapturedVLAType()) {1478      CGF.EmitLambdaVLACapture(CurField->getCapturedVLAType(), LV);1479      continue;1480    }1481 1482    EmitInitializationToLValue(*i, LV);1483 1484    // Push a destructor if necessary.1485    if (QualType::DestructionKind DtorKind =1486            CurField->getType().isDestructedType()) {1487      assert(LV.isSimple());1488      if (DtorKind)1489        CGF.pushDestroyAndDeferDeactivation(NormalAndEHCleanup, LV.getAddress(),1490                                            CurField->getType(),1491                                            CGF.getDestroyer(DtorKind), false);1492    }1493  }1494}1495 1496void AggExprEmitter::VisitExprWithCleanups(ExprWithCleanups *E) {1497  CodeGenFunction::RunCleanupsScope cleanups(CGF);1498  Visit(E->getSubExpr());1499}1500 1501void AggExprEmitter::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) {1502  QualType T = E->getType();1503  AggValueSlot Slot = EnsureSlot(T);1504  EmitNullInitializationToLValue(CGF.MakeAddrLValue(Slot.getAddress(), T));1505}1506 1507void AggExprEmitter::VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) {1508  QualType T = E->getType();1509  AggValueSlot Slot = EnsureSlot(T);1510  EmitNullInitializationToLValue(CGF.MakeAddrLValue(Slot.getAddress(), T));1511}1512 1513/// Determine whether the given cast kind is known to always convert values1514/// with all zero bits in their value representation to values with all zero1515/// bits in their value representation.1516static bool castPreservesZero(const CastExpr *CE) {1517  switch (CE->getCastKind()) {1518    // No-ops.1519  case CK_NoOp:1520  case CK_UserDefinedConversion:1521  case CK_ConstructorConversion:1522  case CK_BitCast:1523  case CK_ToUnion:1524  case CK_ToVoid:1525    // Conversions between (possibly-complex) integral, (possibly-complex)1526    // floating-point, and bool.1527  case CK_BooleanToSignedIntegral:1528  case CK_FloatingCast:1529  case CK_FloatingComplexCast:1530  case CK_FloatingComplexToBoolean:1531  case CK_FloatingComplexToIntegralComplex:1532  case CK_FloatingComplexToReal:1533  case CK_FloatingRealToComplex:1534  case CK_FloatingToBoolean:1535  case CK_FloatingToIntegral:1536  case CK_IntegralCast:1537  case CK_IntegralComplexCast:1538  case CK_IntegralComplexToBoolean:1539  case CK_IntegralComplexToFloatingComplex:1540  case CK_IntegralComplexToReal:1541  case CK_IntegralRealToComplex:1542  case CK_IntegralToBoolean:1543  case CK_IntegralToFloating:1544    // Reinterpreting integers as pointers and vice versa.1545  case CK_IntegralToPointer:1546  case CK_PointerToIntegral:1547    // Language extensions.1548  case CK_VectorSplat:1549  case CK_MatrixCast:1550  case CK_NonAtomicToAtomic:1551  case CK_AtomicToNonAtomic:1552  case CK_HLSLVectorTruncation:1553  case CK_HLSLElementwiseCast:1554  case CK_HLSLAggregateSplatCast:1555    return true;1556 1557  case CK_BaseToDerivedMemberPointer:1558  case CK_DerivedToBaseMemberPointer:1559  case CK_MemberPointerToBoolean:1560  case CK_NullToMemberPointer:1561  case CK_ReinterpretMemberPointer:1562    // FIXME: ABI-dependent.1563    return false;1564 1565  case CK_AnyPointerToBlockPointerCast:1566  case CK_BlockPointerToObjCPointerCast:1567  case CK_CPointerToObjCPointerCast:1568  case CK_ObjCObjectLValueCast:1569  case CK_IntToOCLSampler:1570  case CK_ZeroToOCLOpaqueType:1571    // FIXME: Check these.1572    return false;1573 1574  case CK_FixedPointCast:1575  case CK_FixedPointToBoolean:1576  case CK_FixedPointToFloating:1577  case CK_FixedPointToIntegral:1578  case CK_FloatingToFixedPoint:1579  case CK_IntegralToFixedPoint:1580    // FIXME: Do all fixed-point types represent zero as all 0 bits?1581    return false;1582 1583  case CK_AddressSpaceConversion:1584  case CK_BaseToDerived:1585  case CK_DerivedToBase:1586  case CK_Dynamic:1587  case CK_NullToPointer:1588  case CK_PointerToBoolean:1589    // FIXME: Preserves zeroes only if zero pointers and null pointers have the1590    // same representation in all involved address spaces.1591    return false;1592 1593  case CK_ARCConsumeObject:1594  case CK_ARCExtendBlockObject:1595  case CK_ARCProduceObject:1596  case CK_ARCReclaimReturnedObject:1597  case CK_CopyAndAutoreleaseBlockObject:1598  case CK_ArrayToPointerDecay:1599  case CK_FunctionToPointerDecay:1600  case CK_BuiltinFnToFnPtr:1601  case CK_Dependent:1602  case CK_LValueBitCast:1603  case CK_LValueToRValue:1604  case CK_LValueToRValueBitCast:1605  case CK_UncheckedDerivedToBase:1606  case CK_HLSLArrayRValue:1607    return false;1608  }1609  llvm_unreachable("Unhandled clang::CastKind enum");1610}1611 1612/// isSimpleZero - If emitting this value will obviously just cause a store of1613/// zero to memory, return true.  This can return false if uncertain, so it just1614/// handles simple cases.1615static bool isSimpleZero(const Expr *E, CodeGenFunction &CGF) {1616  E = E->IgnoreParens();1617  while (auto *CE = dyn_cast<CastExpr>(E)) {1618    if (!castPreservesZero(CE))1619      break;1620    E = CE->getSubExpr()->IgnoreParens();1621  }1622 1623  // 01624  if (const IntegerLiteral *IL = dyn_cast<IntegerLiteral>(E))1625    return IL->getValue() == 0;1626  // +0.01627  if (const FloatingLiteral *FL = dyn_cast<FloatingLiteral>(E))1628    return FL->getValue().isPosZero();1629  // int()1630  if ((isa<ImplicitValueInitExpr>(E) || isa<CXXScalarValueInitExpr>(E)) &&1631      CGF.getTypes().isZeroInitializable(E->getType()))1632    return true;1633  // (int*)0 - Null pointer expressions.1634  if (const CastExpr *ICE = dyn_cast<CastExpr>(E))1635    return ICE->getCastKind() == CK_NullToPointer &&1636           CGF.getTypes().isPointerZeroInitializable(E->getType()) &&1637           !E->HasSideEffects(CGF.getContext());1638  // '\0'1639  if (const CharacterLiteral *CL = dyn_cast<CharacterLiteral>(E))1640    return CL->getValue() == 0;1641 1642  // Otherwise, hard case: conservatively return false.1643  return false;1644}1645 1646 1647void1648AggExprEmitter::EmitInitializationToLValue(Expr *E, LValue LV) {1649  QualType type = LV.getType();1650  // FIXME: Ignore result?1651  // FIXME: Are initializers affected by volatile?1652  if (Dest.isZeroed() && isSimpleZero(E, CGF)) {1653    // Storing "i32 0" to a zero'd memory location is a noop.1654    return;1655  } else if (isa<ImplicitValueInitExpr>(E) || isa<CXXScalarValueInitExpr>(E)) {1656    return EmitNullInitializationToLValue(LV);1657  } else if (isa<NoInitExpr>(E)) {1658    // Do nothing.1659    return;1660  } else if (type->isReferenceType()) {1661    RValue RV = CGF.EmitReferenceBindingToExpr(E);1662    return CGF.EmitStoreThroughLValue(RV, LV);1663  }1664 1665  CGF.EmitInitializationToLValue(E, LV, Dest.isZeroed());1666}1667 1668void AggExprEmitter::EmitNullInitializationToLValue(LValue lv) {1669  QualType type = lv.getType();1670 1671  // If the destination slot is already zeroed out before the aggregate is1672  // copied into it, we don't have to emit any zeros here.1673  if (Dest.isZeroed() && CGF.getTypes().isZeroInitializable(type))1674    return;1675 1676  if (CGF.hasScalarEvaluationKind(type)) {1677    // For non-aggregates, we can store the appropriate null constant.1678    llvm::Value *null = CGF.CGM.EmitNullConstant(type);1679    // Note that the following is not equivalent to1680    // EmitStoreThroughBitfieldLValue for ARC types.1681    if (lv.isBitField()) {1682      CGF.EmitStoreThroughBitfieldLValue(RValue::get(null), lv);1683    } else {1684      assert(lv.isSimple());1685      CGF.EmitStoreOfScalar(null, lv, /* isInitialization */ true);1686    }1687  } else {1688    // There's a potential optimization opportunity in combining1689    // memsets; that would be easy for arrays, but relatively1690    // difficult for structures with the current code.1691    CGF.EmitNullInitialization(lv.getAddress(), lv.getType());1692  }1693}1694 1695void AggExprEmitter::VisitCXXParenListInitExpr(CXXParenListInitExpr *E) {1696  VisitCXXParenListOrInitListExpr(E, E->getInitExprs(),1697                                  E->getInitializedFieldInUnion(),1698                                  E->getArrayFiller());1699}1700 1701void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {1702  if (E->hadArrayRangeDesignator())1703    CGF.ErrorUnsupported(E, "GNU array range designator extension");1704 1705  if (E->isTransparent())1706    return Visit(E->getInit(0));1707 1708  VisitCXXParenListOrInitListExpr(1709      E, E->inits(), E->getInitializedFieldInUnion(), E->getArrayFiller());1710}1711 1712void AggExprEmitter::VisitCXXParenListOrInitListExpr(1713    Expr *ExprToVisit, ArrayRef<Expr *> InitExprs,1714    FieldDecl *InitializedFieldInUnion, Expr *ArrayFiller) {1715#if 01716  // FIXME: Assess perf here?  Figure out what cases are worth optimizing here1717  // (Length of globals? Chunks of zeroed-out space?).1718  //1719  // If we can, prefer a copy from a global; this is a lot less code for long1720  // globals, and it's easier for the current optimizers to analyze.1721  if (llvm::Constant *C =1722          CGF.CGM.EmitConstantExpr(ExprToVisit, ExprToVisit->getType(), &CGF)) {1723    llvm::GlobalVariable* GV =1724    new llvm::GlobalVariable(CGF.CGM.getModule(), C->getType(), true,1725                             llvm::GlobalValue::InternalLinkage, C, "");1726    EmitFinalDestCopy(ExprToVisit->getType(),1727                      CGF.MakeAddrLValue(GV, ExprToVisit->getType()));1728    return;1729  }1730#endif1731 1732  // HLSL initialization lists in the AST are an expansion which can contain1733  // side-effecting expressions wrapped in opaque value expressions. To properly1734  // emit these we need to emit the opaque values before we emit the argument1735  // expressions themselves. This is a little hacky, but it prevents us needing1736  // to do a bigger AST-level change for a language feature that we need1737  // deprecate in the near future. See related HLSL language proposals:1738  // * 0005-strict-initializer-lists.md1739  // * https://github.com/microsoft/hlsl-specs/pull/3251740  if (CGF.getLangOpts().HLSL && isa<InitListExpr>(ExprToVisit))1741    CGF.CGM.getHLSLRuntime().emitInitListOpaqueValues(1742        CGF, cast<InitListExpr>(ExprToVisit));1743 1744  AggValueSlot Dest = EnsureSlot(ExprToVisit->getType());1745 1746  LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), ExprToVisit->getType());1747 1748  // Handle initialization of an array.1749  if (ExprToVisit->getType()->isConstantArrayType()) {1750    auto AType = cast<llvm::ArrayType>(Dest.getAddress().getElementType());1751    EmitArrayInit(Dest.getAddress(), AType, ExprToVisit->getType(), ExprToVisit,1752                  InitExprs, ArrayFiller);1753    return;1754  } else if (ExprToVisit->getType()->isVariableArrayType()) {1755    // A variable array type that has an initializer can only do empty1756    // initialization. And because this feature is not exposed as an extension1757    // in C++, we can safely memset the array memory to zero.1758    assert(InitExprs.size() == 0 &&1759           "you can only use an empty initializer with VLAs");1760    CGF.EmitNullInitialization(Dest.getAddress(), ExprToVisit->getType());1761    return;1762  }1763 1764  assert(ExprToVisit->getType()->isRecordType() &&1765         "Only support structs/unions here!");1766 1767  // Do struct initialization; this code just sets each individual member1768  // to the approprate value.  This makes bitfield support automatic;1769  // the disadvantage is that the generated code is more difficult for1770  // the optimizer, especially with bitfields.1771  unsigned NumInitElements = InitExprs.size();1772  RecordDecl *record = ExprToVisit->getType()->castAsRecordDecl();1773 1774  // We'll need to enter cleanup scopes in case any of the element1775  // initializers throws an exception.1776  CodeGenFunction::CleanupDeactivationScope DeactivateCleanups(CGF);1777 1778  unsigned curInitIndex = 0;1779 1780  // Emit initialization of base classes.1781  if (auto *CXXRD = dyn_cast<CXXRecordDecl>(record)) {1782    assert(NumInitElements >= CXXRD->getNumBases() &&1783           "missing initializer for base class");1784    for (auto &Base : CXXRD->bases()) {1785      assert(!Base.isVirtual() && "should not see vbases here");1786      auto *BaseRD = Base.getType()->getAsCXXRecordDecl();1787      Address V = CGF.GetAddressOfDirectBaseInCompleteClass(1788          Dest.getAddress(), CXXRD, BaseRD,1789          /*isBaseVirtual*/ false);1790      AggValueSlot AggSlot = AggValueSlot::forAddr(1791          V, Qualifiers(),1792          AggValueSlot::IsDestructed,1793          AggValueSlot::DoesNotNeedGCBarriers,1794          AggValueSlot::IsNotAliased,1795          CGF.getOverlapForBaseInit(CXXRD, BaseRD, Base.isVirtual()));1796      CGF.EmitAggExpr(InitExprs[curInitIndex++], AggSlot);1797 1798      if (QualType::DestructionKind dtorKind =1799              Base.getType().isDestructedType())1800        CGF.pushDestroyAndDeferDeactivation(dtorKind, V, Base.getType());1801    }1802  }1803 1804  // Prepare a 'this' for CXXDefaultInitExprs.1805  CodeGenFunction::FieldConstructionScope FCS(CGF, Dest.getAddress());1806 1807  const bool ZeroInitPadding =1808      CGF.CGM.shouldZeroInitPadding() && !Dest.isZeroed();1809 1810  if (record->isUnion()) {1811    // Only initialize one field of a union. The field itself is1812    // specified by the initializer list.1813    if (!InitializedFieldInUnion) {1814      // Empty union; we have nothing to do.1815 1816#ifndef NDEBUG1817      // Make sure that it's really an empty and not a failure of1818      // semantic analysis.1819      for (const auto *Field : record->fields())1820        assert(1821            (Field->isUnnamedBitField() || Field->isAnonymousStructOrUnion()) &&1822            "Only unnamed bitfields or anonymous class allowed");1823#endif1824      return;1825    }1826 1827    // FIXME: volatility1828    FieldDecl *Field = InitializedFieldInUnion;1829 1830    LValue FieldLoc = CGF.EmitLValueForFieldInitialization(DestLV, Field);1831    if (NumInitElements) {1832      // Store the initializer into the field1833      EmitInitializationToLValue(InitExprs[0], FieldLoc);1834      if (ZeroInitPadding) {1835        uint64_t TotalSize = CGF.getContext().toBits(1836            Dest.getPreferredSize(CGF.getContext(), DestLV.getType()));1837        uint64_t FieldSize = CGF.getContext().getTypeSize(FieldLoc.getType());1838        DoZeroInitPadding(FieldSize, TotalSize, nullptr);1839      }1840    } else {1841      // Default-initialize to null.1842      if (ZeroInitPadding)1843        EmitNullInitializationToLValue(DestLV);1844      else1845        EmitNullInitializationToLValue(FieldLoc);1846    }1847    return;1848  }1849 1850  // Here we iterate over the fields; this makes it simpler to both1851  // default-initialize fields and skip over unnamed fields.1852  const ASTRecordLayout &Layout = CGF.getContext().getASTRecordLayout(record);1853  uint64_t PaddingStart = 0;1854 1855  for (const auto *field : record->fields()) {1856    // We're done once we hit the flexible array member.1857    if (field->getType()->isIncompleteArrayType())1858      break;1859 1860    // Always skip anonymous bitfields.1861    if (field->isUnnamedBitField())1862      continue;1863 1864    // We're done if we reach the end of the explicit initializers, we1865    // have a zeroed object, and the rest of the fields are1866    // zero-initializable.1867    if (curInitIndex == NumInitElements && Dest.isZeroed() &&1868        CGF.getTypes().isZeroInitializable(ExprToVisit->getType()))1869      break;1870 1871    if (ZeroInitPadding)1872      DoZeroInitPadding(PaddingStart,1873                        Layout.getFieldOffset(field->getFieldIndex()), field);1874 1875    LValue LV = CGF.EmitLValueForFieldInitialization(DestLV, field);1876    // We never generate write-barries for initialized fields.1877    LV.setNonGC(true);1878 1879    if (curInitIndex < NumInitElements) {1880      // Store the initializer into the field.1881      EmitInitializationToLValue(InitExprs[curInitIndex++], LV);1882    } else {1883      // We're out of initializers; default-initialize to null1884      EmitNullInitializationToLValue(LV);1885    }1886 1887    // Push a destructor if necessary.1888    // FIXME: if we have an array of structures, all explicitly1889    // initialized, we can end up pushing a linear number of cleanups.1890    if (QualType::DestructionKind dtorKind1891          = field->getType().isDestructedType()) {1892      assert(LV.isSimple());1893      if (dtorKind) {1894        CGF.pushDestroyAndDeferDeactivation(NormalAndEHCleanup, LV.getAddress(),1895                                            field->getType(),1896                                            CGF.getDestroyer(dtorKind), false);1897      }1898    }1899  }1900  if (ZeroInitPadding) {1901    uint64_t TotalSize = CGF.getContext().toBits(1902        Dest.getPreferredSize(CGF.getContext(), DestLV.getType()));1903    DoZeroInitPadding(PaddingStart, TotalSize, nullptr);1904  }1905}1906 1907void AggExprEmitter::DoZeroInitPadding(uint64_t &PaddingStart,1908                                       uint64_t PaddingEnd,1909                                       const FieldDecl *NextField) {1910 1911  auto InitBytes = [&](uint64_t StartBit, uint64_t EndBit) {1912    CharUnits Start = CGF.getContext().toCharUnitsFromBits(StartBit);1913    CharUnits End = CGF.getContext().toCharUnitsFromBits(EndBit);1914    Address Addr = Dest.getAddress().withElementType(CGF.CharTy);1915    if (!Start.isZero())1916      Addr = Builder.CreateConstGEP(Addr, Start.getQuantity());1917    llvm::Constant *SizeVal = Builder.getInt64((End - Start).getQuantity());1918    CGF.Builder.CreateMemSet(Addr, Builder.getInt8(0), SizeVal, false);1919  };1920 1921  if (NextField != nullptr && NextField->isBitField()) {1922    // For bitfield, zero init StorageSize before storing the bits. So we don't1923    // need to handle big/little endian.1924    const CGRecordLayout &RL =1925        CGF.getTypes().getCGRecordLayout(NextField->getParent());1926    const CGBitFieldInfo &Info = RL.getBitFieldInfo(NextField);1927    uint64_t StorageStart = CGF.getContext().toBits(Info.StorageOffset);1928    if (StorageStart + Info.StorageSize > PaddingStart) {1929      if (StorageStart > PaddingStart)1930        InitBytes(PaddingStart, StorageStart);1931      Address Addr = Dest.getAddress();1932      if (!Info.StorageOffset.isZero())1933        Addr = Builder.CreateConstGEP(Addr.withElementType(CGF.CharTy),1934                                      Info.StorageOffset.getQuantity());1935      Addr = Addr.withElementType(1936          llvm::Type::getIntNTy(CGF.getLLVMContext(), Info.StorageSize));1937      Builder.CreateStore(Builder.getIntN(Info.StorageSize, 0), Addr);1938      PaddingStart = StorageStart + Info.StorageSize;1939    }1940    return;1941  }1942 1943  if (PaddingStart < PaddingEnd)1944    InitBytes(PaddingStart, PaddingEnd);1945  if (NextField != nullptr)1946    PaddingStart =1947        PaddingEnd + CGF.getContext().getTypeSize(NextField->getType());1948}1949 1950void AggExprEmitter::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,1951                                            llvm::Value *outerBegin) {1952  // Emit the common subexpression.1953  CodeGenFunction::OpaqueValueMapping binding(CGF, E->getCommonExpr());1954 1955  Address destPtr = EnsureSlot(E->getType()).getAddress();1956  uint64_t numElements = E->getArraySize().getZExtValue();1957 1958  if (!numElements)1959    return;1960 1961  // destPtr is an array*. Construct an elementType* by drilling down a level.1962  llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0);1963  llvm::Value *indices[] = {zero, zero};1964  llvm::Value *begin = Builder.CreateInBoundsGEP(destPtr.getElementType(),1965                                                 destPtr.emitRawPointer(CGF),1966                                                 indices, "arrayinit.begin");1967 1968  // Prepare to special-case multidimensional array initialization: we avoid1969  // emitting multiple destructor loops in that case.1970  if (!outerBegin)1971    outerBegin = begin;1972  ArrayInitLoopExpr *InnerLoop = dyn_cast<ArrayInitLoopExpr>(E->getSubExpr());1973 1974  QualType elementType =1975      CGF.getContext().getAsArrayType(E->getType())->getElementType();1976  CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType);1977  CharUnits elementAlign =1978      destPtr.getAlignment().alignmentOfArrayElement(elementSize);1979  llvm::Type *llvmElementType = CGF.ConvertTypeForMem(elementType);1980 1981  llvm::BasicBlock *entryBB = Builder.GetInsertBlock();1982  llvm::BasicBlock *bodyBB = CGF.createBasicBlock("arrayinit.body");1983 1984  // Jump into the body.1985  CGF.EmitBlock(bodyBB);1986  llvm::PHINode *index =1987      Builder.CreatePHI(zero->getType(), 2, "arrayinit.index");1988  index->addIncoming(zero, entryBB);1989  llvm::Value *element =1990      Builder.CreateInBoundsGEP(llvmElementType, begin, index);1991 1992  // Prepare for a cleanup.1993  QualType::DestructionKind dtorKind = elementType.isDestructedType();1994  EHScopeStack::stable_iterator cleanup;1995  if (CGF.needsEHCleanup(dtorKind) && !InnerLoop) {1996    if (outerBegin->getType() != element->getType())1997      outerBegin = Builder.CreateBitCast(outerBegin, element->getType());1998    CGF.pushRegularPartialArrayCleanup(outerBegin, element, elementType,1999                                       elementAlign,2000                                       CGF.getDestroyer(dtorKind));2001    cleanup = CGF.EHStack.stable_begin();2002  } else {2003    dtorKind = QualType::DK_none;2004  }2005 2006  // Emit the actual filler expression.2007  {2008    // Temporaries created in an array initialization loop are destroyed2009    // at the end of each iteration.2010    CodeGenFunction::RunCleanupsScope CleanupsScope(CGF);2011    CodeGenFunction::ArrayInitLoopExprScope Scope(CGF, index);2012    LValue elementLV = CGF.MakeAddrLValue(2013        Address(element, llvmElementType, elementAlign), elementType);2014 2015    if (InnerLoop) {2016      // If the subexpression is an ArrayInitLoopExpr, share its cleanup.2017      auto elementSlot = AggValueSlot::forLValue(2018          elementLV, AggValueSlot::IsDestructed,2019          AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,2020          AggValueSlot::DoesNotOverlap);2021      AggExprEmitter(CGF, elementSlot, false)2022          .VisitArrayInitLoopExpr(InnerLoop, outerBegin);2023    } else2024      EmitInitializationToLValue(E->getSubExpr(), elementLV);2025  }2026 2027  // Move on to the next element.2028  llvm::Value *nextIndex = Builder.CreateNUWAdd(2029      index, llvm::ConstantInt::get(CGF.SizeTy, 1), "arrayinit.next");2030  index->addIncoming(nextIndex, Builder.GetInsertBlock());2031 2032  // Leave the loop if we're done.2033  llvm::Value *done = Builder.CreateICmpEQ(2034      nextIndex, llvm::ConstantInt::get(CGF.SizeTy, numElements),2035      "arrayinit.done");2036  llvm::BasicBlock *endBB = CGF.createBasicBlock("arrayinit.end");2037  Builder.CreateCondBr(done, endBB, bodyBB);2038 2039  CGF.EmitBlock(endBB);2040 2041  // Leave the partial-array cleanup if we entered one.2042  if (dtorKind)2043    CGF.DeactivateCleanupBlock(cleanup, index);2044}2045 2046void AggExprEmitter::VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E) {2047  AggValueSlot Dest = EnsureSlot(E->getType());2048 2049  LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());2050  EmitInitializationToLValue(E->getBase(), DestLV);2051  VisitInitListExpr(E->getUpdater());2052}2053 2054//===----------------------------------------------------------------------===//2055//                        Entry Points into this File2056//===----------------------------------------------------------------------===//2057 2058/// GetNumNonZeroBytesInInit - Get an approximate count of the number of2059/// non-zero bytes that will be stored when outputting the initializer for the2060/// specified initializer expression.2061static CharUnits GetNumNonZeroBytesInInit(const Expr *E, CodeGenFunction &CGF) {2062  if (auto *MTE = dyn_cast<MaterializeTemporaryExpr>(E))2063    E = MTE->getSubExpr();2064  E = E->IgnoreParenNoopCasts(CGF.getContext());2065 2066  // 0 and 0.0 won't require any non-zero stores!2067  if (isSimpleZero(E, CGF)) return CharUnits::Zero();2068 2069  // If this is an initlist expr, sum up the size of sizes of the (present)2070  // elements.  If this is something weird, assume the whole thing is non-zero.2071  const InitListExpr *ILE = dyn_cast<InitListExpr>(E);2072  while (ILE && ILE->isTransparent())2073    ILE = dyn_cast<InitListExpr>(ILE->getInit(0));2074  if (!ILE || !CGF.getTypes().isZeroInitializable(ILE->getType()))2075    return CGF.getContext().getTypeSizeInChars(E->getType());2076 2077  // InitListExprs for structs have to be handled carefully.  If there are2078  // reference members, we need to consider the size of the reference, not the2079  // referencee.  InitListExprs for unions and arrays can't have references.2080  if (const RecordType *RT = E->getType()->getAsCanonical<RecordType>()) {2081    if (!RT->isUnionType()) {2082      RecordDecl *SD = RT->getDecl()->getDefinitionOrSelf();2083      CharUnits NumNonZeroBytes = CharUnits::Zero();2084 2085      unsigned ILEElement = 0;2086      if (auto *CXXRD = dyn_cast<CXXRecordDecl>(SD))2087        while (ILEElement != CXXRD->getNumBases())2088          NumNonZeroBytes +=2089              GetNumNonZeroBytesInInit(ILE->getInit(ILEElement++), CGF);2090      for (const auto *Field : SD->fields()) {2091        // We're done once we hit the flexible array member or run out of2092        // InitListExpr elements.2093        if (Field->getType()->isIncompleteArrayType() ||2094            ILEElement == ILE->getNumInits())2095          break;2096        if (Field->isUnnamedBitField())2097          continue;2098 2099        const Expr *E = ILE->getInit(ILEElement++);2100 2101        // Reference values are always non-null and have the width of a pointer.2102        if (Field->getType()->isReferenceType())2103          NumNonZeroBytes += CGF.getContext().toCharUnitsFromBits(2104              CGF.getTarget().getPointerWidth(LangAS::Default));2105        else2106          NumNonZeroBytes += GetNumNonZeroBytesInInit(E, CGF);2107      }2108 2109      return NumNonZeroBytes;2110    }2111  }2112 2113  // FIXME: This overestimates the number of non-zero bytes for bit-fields.2114  CharUnits NumNonZeroBytes = CharUnits::Zero();2115  for (unsigned i = 0, e = ILE->getNumInits(); i != e; ++i)2116    NumNonZeroBytes += GetNumNonZeroBytesInInit(ILE->getInit(i), CGF);2117  return NumNonZeroBytes;2118}2119 2120/// CheckAggExprForMemSetUse - If the initializer is large and has a lot of2121/// zeros in it, emit a memset and avoid storing the individual zeros.2122///2123static void CheckAggExprForMemSetUse(AggValueSlot &Slot, const Expr *E,2124                                     CodeGenFunction &CGF) {2125  // If the slot is already known to be zeroed, nothing to do.  Don't mess with2126  // volatile stores.2127  if (Slot.isZeroed() || Slot.isVolatile() || !Slot.getAddress().isValid())2128    return;2129 2130  // C++ objects with a user-declared constructor don't need zero'ing.2131  if (CGF.getLangOpts().CPlusPlus)2132    if (const RecordType *RT = CGF.getContext()2133                                   .getBaseElementType(E->getType())2134                                   ->getAsCanonical<RecordType>()) {2135      const auto *RD = cast<CXXRecordDecl>(RT->getDecl());2136      if (RD->hasUserDeclaredConstructor())2137        return;2138    }2139 2140  // If the type is 16-bytes or smaller, prefer individual stores over memset.2141  CharUnits Size = Slot.getPreferredSize(CGF.getContext(), E->getType());2142  if (Size <= CharUnits::fromQuantity(16))2143    return;2144 2145  // Check to see if over 3/4 of the initializer are known to be zero.  If so,2146  // we prefer to emit memset + individual stores for the rest.2147  CharUnits NumNonZeroBytes = GetNumNonZeroBytesInInit(E, CGF);2148  if (NumNonZeroBytes*4 > Size)2149    return;2150 2151  // Okay, it seems like a good idea to use an initial memset, emit the call.2152  llvm::Constant *SizeVal = CGF.Builder.getInt64(Size.getQuantity());2153 2154  Address Loc = Slot.getAddress().withElementType(CGF.Int8Ty);2155  CGF.Builder.CreateMemSet(Loc, CGF.Builder.getInt8(0), SizeVal, false);2156 2157  // Tell the AggExprEmitter that the slot is known zero.2158  Slot.setZeroed();2159}2160 2161 2162 2163 2164/// EmitAggExpr - Emit the computation of the specified expression of aggregate2165/// type.  The result is computed into DestPtr.  Note that if DestPtr is null,2166/// the value of the aggregate expression is not needed.  If VolatileDest is2167/// true, DestPtr cannot be 0.2168void CodeGenFunction::EmitAggExpr(const Expr *E, AggValueSlot Slot) {2169  assert(E && hasAggregateEvaluationKind(E->getType()) &&2170         "Invalid aggregate expression to emit");2171  assert((Slot.getAddress().isValid() || Slot.isIgnored()) &&2172         "slot has bits but no address");2173 2174  // Optimize the slot if possible.2175  CheckAggExprForMemSetUse(Slot, E, *this);2176 2177  AggExprEmitter(*this, Slot, Slot.isIgnored()).Visit(const_cast<Expr*>(E));2178}2179 2180LValue CodeGenFunction::EmitAggExprToLValue(const Expr *E) {2181  assert(hasAggregateEvaluationKind(E->getType()) && "Invalid argument!");2182  Address Temp = CreateMemTemp(E->getType());2183  LValue LV = MakeAddrLValue(Temp, E->getType());2184  EmitAggExpr(E, AggValueSlot::forLValue(LV, AggValueSlot::IsNotDestructed,2185                                         AggValueSlot::DoesNotNeedGCBarriers,2186                                         AggValueSlot::IsNotAliased,2187                                         AggValueSlot::DoesNotOverlap));2188  return LV;2189}2190 2191void CodeGenFunction::EmitAggFinalDestCopy(QualType Type, AggValueSlot Dest,2192                                           const LValue &Src,2193                                           ExprValueKind SrcKind) {2194  return AggExprEmitter(*this, Dest, Dest.isIgnored())2195      .EmitFinalDestCopy(Type, Src, SrcKind);2196}2197 2198AggValueSlot::Overlap_t2199CodeGenFunction::getOverlapForFieldInit(const FieldDecl *FD) {2200  if (!FD->hasAttr<NoUniqueAddressAttr>() || !FD->getType()->isRecordType())2201    return AggValueSlot::DoesNotOverlap;2202 2203  // Empty fields can overlap earlier fields.2204  if (FD->getType()->getAsCXXRecordDecl()->isEmpty())2205    return AggValueSlot::MayOverlap;2206 2207  // If the field lies entirely within the enclosing class's nvsize, its tail2208  // padding cannot overlap any already-initialized object. (The only subobjects2209  // with greater addresses that might already be initialized are vbases.)2210  const RecordDecl *ClassRD = FD->getParent();2211  const ASTRecordLayout &Layout = getContext().getASTRecordLayout(ClassRD);2212  if (Layout.getFieldOffset(FD->getFieldIndex()) +2213          getContext().getTypeSize(FD->getType()) <=2214      (uint64_t)getContext().toBits(Layout.getNonVirtualSize()))2215    return AggValueSlot::DoesNotOverlap;2216 2217  // The tail padding may contain values we need to preserve.2218  return AggValueSlot::MayOverlap;2219}2220 2221AggValueSlot::Overlap_t CodeGenFunction::getOverlapForBaseInit(2222    const CXXRecordDecl *RD, const CXXRecordDecl *BaseRD, bool IsVirtual) {2223  // If the most-derived object is a field declared with [[no_unique_address]],2224  // the tail padding of any virtual base could be reused for other subobjects2225  // of that field's class.2226  if (IsVirtual)2227    return AggValueSlot::MayOverlap;2228 2229  // Empty bases can overlap earlier bases.2230  if (BaseRD->isEmpty())2231    return AggValueSlot::MayOverlap;2232 2233  // If the base class is laid out entirely within the nvsize of the derived2234  // class, its tail padding cannot yet be initialized, so we can issue2235  // stores at the full width of the base class.2236  const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);2237  if (Layout.getBaseClassOffset(BaseRD) +2238          getContext().getASTRecordLayout(BaseRD).getSize() <=2239      Layout.getNonVirtualSize())2240    return AggValueSlot::DoesNotOverlap;2241 2242  // The tail padding may contain values we need to preserve.2243  return AggValueSlot::MayOverlap;2244}2245 2246void CodeGenFunction::EmitAggregateCopy(LValue Dest, LValue Src, QualType Ty,2247                                        AggValueSlot::Overlap_t MayOverlap,2248                                        bool isVolatile) {2249  assert(!Ty->isAnyComplexType() && "Shouldn't happen for complex");2250 2251  Address DestPtr = Dest.getAddress();2252  Address SrcPtr = Src.getAddress();2253 2254  if (getLangOpts().CPlusPlus) {2255    if (const auto *Record = Ty->getAsCXXRecordDecl()) {2256      assert((Record->hasTrivialCopyConstructor() ||2257              Record->hasTrivialCopyAssignment() ||2258              Record->hasTrivialMoveConstructor() ||2259              Record->hasTrivialMoveAssignment() ||2260              Record->hasAttr<TrivialABIAttr>() || Record->isUnion()) &&2261             "Trying to aggregate-copy a type without a trivial copy/move "2262             "constructor or assignment operator");2263      // Ignore empty classes in C++.2264      if (Record->isEmpty())2265        return;2266    }2267  }2268 2269  if (getLangOpts().CUDAIsDevice) {2270    if (Ty->isCUDADeviceBuiltinSurfaceType()) {2271      if (getTargetHooks().emitCUDADeviceBuiltinSurfaceDeviceCopy(*this, Dest,2272                                                                  Src))2273        return;2274    } else if (Ty->isCUDADeviceBuiltinTextureType()) {2275      if (getTargetHooks().emitCUDADeviceBuiltinTextureDeviceCopy(*this, Dest,2276                                                                  Src))2277        return;2278    }2279  }2280 2281  if (getLangOpts().HLSL && Ty.getAddressSpace() == LangAS::hlsl_constant)2282    if (CGM.getHLSLRuntime().emitBufferCopy(*this, DestPtr, SrcPtr, Ty))2283      return;2284 2285  // Aggregate assignment turns into llvm.memcpy.  This is almost valid per2286  // C99 6.5.16.1p3, which states "If the value being stored in an object is2287  // read from another object that overlaps in anyway the storage of the first2288  // object, then the overlap shall be exact and the two objects shall have2289  // qualified or unqualified versions of a compatible type."2290  //2291  // memcpy is not defined if the source and destination pointers are exactly2292  // equal, but other compilers do this optimization, and almost every memcpy2293  // implementation handles this case safely.  If there is a libc that does not2294  // safely handle this, we can add a target hook.2295 2296  // Get data size info for this aggregate. Don't copy the tail padding if this2297  // might be a potentially-overlapping subobject, since the tail padding might2298  // be occupied by a different object. Otherwise, copying it is fine.2299  TypeInfoChars TypeInfo;2300  if (MayOverlap)2301    TypeInfo = getContext().getTypeInfoDataSizeInChars(Ty);2302  else2303    TypeInfo = getContext().getTypeInfoInChars(Ty);2304 2305  llvm::Value *SizeVal = nullptr;2306  if (TypeInfo.Width.isZero()) {2307    // But note that getTypeInfo returns 0 for a VLA.2308    if (auto *VAT = dyn_cast_or_null<VariableArrayType>(2309            getContext().getAsArrayType(Ty))) {2310      QualType BaseEltTy;2311      SizeVal = emitArrayLength(VAT, BaseEltTy, DestPtr);2312      TypeInfo = getContext().getTypeInfoInChars(BaseEltTy);2313      assert(!TypeInfo.Width.isZero());2314      SizeVal = Builder.CreateNUWMul(2315          SizeVal,2316          llvm::ConstantInt::get(SizeTy, TypeInfo.Width.getQuantity()));2317    }2318  }2319  if (!SizeVal) {2320    SizeVal = llvm::ConstantInt::get(SizeTy, TypeInfo.Width.getQuantity());2321  }2322 2323  // FIXME: If we have a volatile struct, the optimizer can remove what might2324  // appear to be `extra' memory ops:2325  //2326  // volatile struct { int i; } a, b;2327  //2328  // int main() {2329  //   a = b;2330  //   a = b;2331  // }2332  //2333  // we need to use a different call here.  We use isVolatile to indicate when2334  // either the source or the destination is volatile.2335 2336  DestPtr = DestPtr.withElementType(Int8Ty);2337  SrcPtr = SrcPtr.withElementType(Int8Ty);2338 2339  // Don't do any of the memmove_collectable tests if GC isn't set.2340  if (CGM.getLangOpts().getGC() == LangOptions::NonGC) {2341    // fall through2342  } else if (const auto *Record = Ty->getAsRecordDecl()) {2343    if (Record->hasObjectMember()) {2344      CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestPtr, SrcPtr,2345                                                    SizeVal);2346      return;2347    }2348  } else if (Ty->isArrayType()) {2349    QualType BaseType = getContext().getBaseElementType(Ty);2350    if (const auto *Record = BaseType->getAsRecordDecl()) {2351      if (Record->hasObjectMember()) {2352        CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestPtr, SrcPtr,2353                                                      SizeVal);2354        return;2355      }2356    }2357  }2358 2359  auto *Inst = Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, isVolatile);2360  addInstToCurrentSourceAtom(Inst, nullptr);2361 2362  // Determine the metadata to describe the position of any padding in this2363  // memcpy, as well as the TBAA tags for the members of the struct, in case2364  // the optimizer wishes to expand it in to scalar memory operations.2365  if (llvm::MDNode *TBAAStructTag = CGM.getTBAAStructInfo(Ty))2366    Inst->setMetadata(llvm::LLVMContext::MD_tbaa_struct, TBAAStructTag);2367 2368  if (CGM.getCodeGenOpts().NewStructPathTBAA) {2369    TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForMemoryTransfer(2370        Dest.getTBAAInfo(), Src.getTBAAInfo());2371    CGM.DecorateInstructionWithTBAA(Inst, TBAAInfo);2372  }2373}2374