brintos / llvm-project-archived public Read only

1//===--- CGBlocks.cpp - Emit LLVM Code for declarations ---------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This contains code to emit blocks.10//11//===----------------------------------------------------------------------===//12 13#include "CGBlocks.h"14#include "CGCXXABI.h"15#include "CGDebugInfo.h"16#include "CGObjCRuntime.h"17#include "CGOpenCLRuntime.h"18#include "CodeGenFunction.h"19#include "CodeGenModule.h"20#include "CodeGenPGO.h"21#include "ConstantEmitter.h"22#include "TargetInfo.h"23#include "clang/AST/Attr.h"24#include "clang/AST/DeclObjC.h"25#include "clang/CodeGen/ConstantInitBuilder.h"26#include "llvm/IR/DataLayout.h"27#include "llvm/IR/Module.h"28#include "llvm/Support/ScopedPrinter.h"29#include <algorithm>30#include <cstdio>31 32using namespace clang;33using namespace CodeGen;34 35CGBlockInfo::CGBlockInfo(const BlockDecl *block, StringRef name)36    : Name(name), CXXThisIndex(0), CanBeGlobal(false), NeedsCopyDispose(false),37      NoEscape(false), HasCXXObject(false), UsesStret(false),38      HasCapturedVariableLayout(false), CapturesNonExternalType(false),39      LocalAddress(RawAddress::invalid()), StructureType(nullptr),40      Block(block) {41 42  // Skip asm prefix, if any.  'name' is usually taken directly from43  // the mangled name of the enclosing function.44  name.consume_front("\01");45}46 47// Anchor the vtable to this translation unit.48BlockByrefHelpers::~BlockByrefHelpers() {}49 50/// Build the given block as a global block.51static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM,52                                        const CGBlockInfo &blockInfo,53                                        llvm::Constant *blockFn);54 55/// Build the helper function to copy a block.56static llvm::Constant *buildCopyHelper(CodeGenModule &CGM,57                                       const CGBlockInfo &blockInfo) {58  return CodeGenFunction(CGM).GenerateCopyHelperFunction(blockInfo);59}60 61/// Build the helper function to dispose of a block.62static llvm::Constant *buildDisposeHelper(CodeGenModule &CGM,63                                          const CGBlockInfo &blockInfo) {64  return CodeGenFunction(CGM).GenerateDestroyHelperFunction(blockInfo);65}66 67namespace {68 69enum class CaptureStrKind {70  // String for the copy helper.71  CopyHelper,72  // String for the dispose helper.73  DisposeHelper,74  // Merge the strings for the copy helper and dispose helper.75  Merged76};77 78} // end anonymous namespace79 80static std::string getBlockCaptureStr(const CGBlockInfo::Capture &Cap,81                                      CaptureStrKind StrKind,82                                      CharUnits BlockAlignment,83                                      CodeGenModule &CGM);84 85static std::string getBlockDescriptorName(const CGBlockInfo &BlockInfo,86                                          CodeGenModule &CGM) {87  std::string Name = "__block_descriptor_";88  Name += llvm::to_string(BlockInfo.BlockSize.getQuantity()) + "_";89 90  if (BlockInfo.NeedsCopyDispose) {91    if (CGM.getLangOpts().Exceptions)92      Name += "e";93    if (CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)94      Name += "a";95    Name += llvm::to_string(BlockInfo.BlockAlign.getQuantity()) + "_";96 97    for (auto &Cap : BlockInfo.SortedCaptures) {98      if (Cap.isConstantOrTrivial())99        continue;100 101      Name += llvm::to_string(Cap.getOffset().getQuantity());102 103      if (Cap.CopyKind == Cap.DisposeKind) {104        // If CopyKind and DisposeKind are the same, merge the capture105        // information.106        assert(Cap.CopyKind != BlockCaptureEntityKind::None &&107               "shouldn't see BlockCaptureManagedEntity that is None");108        Name += getBlockCaptureStr(Cap, CaptureStrKind::Merged,109                                   BlockInfo.BlockAlign, CGM);110      } else {111        // If CopyKind and DisposeKind are not the same, which can happen when112        // either Kind is None or the captured object is a __strong block,113        // concatenate the copy and dispose strings.114        Name += getBlockCaptureStr(Cap, CaptureStrKind::CopyHelper,115                                   BlockInfo.BlockAlign, CGM);116        Name += getBlockCaptureStr(Cap, CaptureStrKind::DisposeHelper,117                                   BlockInfo.BlockAlign, CGM);118      }119    }120    Name += "_";121  }122 123  std::string TypeAtEncoding;124 125  if (!CGM.getCodeGenOpts().DisableBlockSignatureString) {126    TypeAtEncoding =127        CGM.getContext().getObjCEncodingForBlock(BlockInfo.getBlockExpr());128    /// Replace occurrences of '@' with '\1'. '@' is reserved on ELF platforms129    /// as a separator between symbol name and symbol version.130    llvm::replace(TypeAtEncoding, '@', '\1');131  }132  Name += "e" + llvm::to_string(TypeAtEncoding.size()) + "_" + TypeAtEncoding;133  Name += "l" + CGM.getObjCRuntime().getRCBlockLayoutStr(CGM, BlockInfo);134  return Name;135}136 137/// buildBlockDescriptor - Build the block descriptor meta-data for a block.138/// buildBlockDescriptor is accessed from 5th field of the Block_literal139/// meta-data and contains stationary information about the block literal.140/// Its definition will have 4 (or optionally 6) words.141/// \code142/// struct Block_descriptor {143///   unsigned long reserved;144///   unsigned long size;  // size of Block_literal metadata in bytes.145///   void *copy_func_helper_decl;  // optional copy helper.146///   void *destroy_func_decl; // optional destructor helper.147///   void *block_method_encoding_address; // @encode for block literal signature.148///   void *block_layout_info; // encoding of captured block variables.149/// };150/// \endcode151static llvm::Constant *buildBlockDescriptor(CodeGenModule &CGM,152                                            const CGBlockInfo &blockInfo) {153  ASTContext &C = CGM.getContext();154 155  llvm::IntegerType *ulong =156    cast<llvm::IntegerType>(CGM.getTypes().ConvertType(C.UnsignedLongTy));157  llvm::PointerType *i8p = nullptr;158  if (CGM.getLangOpts().OpenCL)159    i8p = llvm::PointerType::get(160        CGM.getLLVMContext(), C.getTargetAddressSpace(LangAS::opencl_constant));161  else162    i8p = CGM.VoidPtrTy;163 164  std::string descName;165 166  // If an equivalent block descriptor global variable exists, return it.167  if (C.getLangOpts().ObjC &&168      CGM.getLangOpts().getGC() == LangOptions::NonGC) {169    descName = getBlockDescriptorName(blockInfo, CGM);170    if (llvm::GlobalValue *desc = CGM.getModule().getNamedValue(descName))171      return desc;172  }173 174  // If there isn't an equivalent block descriptor global variable, create a new175  // one.176  ConstantInitBuilder builder(CGM);177  auto elements = builder.beginStruct();178 179  // reserved180  elements.addInt(ulong, 0);181 182  // Size183  // FIXME: What is the right way to say this doesn't fit?  We should give184  // a user diagnostic in that case.  Better fix would be to change the185  // API to size_t.186  elements.addInt(ulong, blockInfo.BlockSize.getQuantity());187 188  // Optional copy/dispose helpers.189  bool hasInternalHelper = false;190  if (blockInfo.NeedsCopyDispose) {191    auto &Schema = CGM.getCodeGenOpts().PointerAuth.BlockHelperFunctionPointers;192    // copy_func_helper_decl193    llvm::Constant *copyHelper = buildCopyHelper(CGM, blockInfo);194    elements.addSignedPointer(copyHelper, Schema, GlobalDecl(), QualType());195 196    // destroy_func_decl197    llvm::Constant *disposeHelper = buildDisposeHelper(CGM, blockInfo);198    elements.addSignedPointer(disposeHelper, Schema, GlobalDecl(), QualType());199 200    if (cast<llvm::Function>(copyHelper->stripPointerCasts())201            ->hasInternalLinkage() ||202        cast<llvm::Function>(disposeHelper->stripPointerCasts())203            ->hasInternalLinkage())204      hasInternalHelper = true;205  }206 207  // Signature.  Mandatory ObjC-style method descriptor @encode sequence.208  if (CGM.getCodeGenOpts().DisableBlockSignatureString) {209    elements.addNullPointer(i8p);210  } else {211    std::string typeAtEncoding =212        CGM.getContext().getObjCEncodingForBlock(blockInfo.getBlockExpr());213    elements.add(CGM.GetAddrOfConstantCString(typeAtEncoding).getPointer());214  }215 216  // GC layout.217  if (C.getLangOpts().ObjC) {218    if (CGM.getLangOpts().getGC() != LangOptions::NonGC)219      elements.add(CGM.getObjCRuntime().BuildGCBlockLayout(CGM, blockInfo));220    else221      elements.add(CGM.getObjCRuntime().BuildRCBlockLayout(CGM, blockInfo));222  }223  else224    elements.addNullPointer(i8p);225 226  unsigned AddrSpace = 0;227  if (C.getLangOpts().OpenCL)228    AddrSpace = C.getTargetAddressSpace(LangAS::opencl_constant);229 230  llvm::GlobalValue::LinkageTypes linkage;231  if (descName.empty()) {232    linkage = llvm::GlobalValue::InternalLinkage;233    descName = "__block_descriptor_tmp";234  } else if (hasInternalHelper) {235    // If either the copy helper or the dispose helper has internal linkage,236    // the block descriptor must have internal linkage too.237    linkage = llvm::GlobalValue::InternalLinkage;238  } else {239    linkage = llvm::GlobalValue::LinkOnceODRLinkage;240  }241 242  llvm::GlobalVariable *global =243      elements.finishAndCreateGlobal(descName, CGM.getPointerAlign(),244                                     /*constant*/ true, linkage, AddrSpace);245 246  if (linkage == llvm::GlobalValue::LinkOnceODRLinkage) {247    if (CGM.supportsCOMDAT())248      global->setComdat(CGM.getModule().getOrInsertComdat(descName));249    global->setVisibility(llvm::GlobalValue::HiddenVisibility);250    global->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);251  }252 253  return global;254}255 256/*257  Purely notional variadic template describing the layout of a block.258 259  template <class _ResultType, class... _ParamTypes, class... _CaptureTypes>260  struct Block_literal {261    /// Initialized to one of:262    ///   extern void *_NSConcreteStackBlock[];263    ///   extern void *_NSConcreteGlobalBlock[];264    ///265    /// In theory, we could start one off malloc'ed by setting266    /// BLOCK_NEEDS_FREE, giving it a refcount of 1, and using267    /// this isa:268    ///   extern void *_NSConcreteMallocBlock[];269    struct objc_class *isa;270 271    /// These are the flags (with corresponding bit number) that the272    /// compiler is actually supposed to know about.273    ///  23. BLOCK_IS_NOESCAPE - indicates that the block is non-escaping274    ///  25. BLOCK_HAS_COPY_DISPOSE - indicates that the block275    ///   descriptor provides copy and dispose helper functions276    ///  26. BLOCK_HAS_CXX_OBJ - indicates that there's a captured277    ///   object with a nontrivial destructor or copy constructor278    ///  28. BLOCK_IS_GLOBAL - indicates that the block is allocated279    ///   as global memory280    ///  29. BLOCK_USE_STRET - indicates that the block function281    ///   uses stret, which objc_msgSend needs to know about282    ///  30. BLOCK_HAS_SIGNATURE - indicates that the block has an283    ///   @encoded signature string284    /// And we're not supposed to manipulate these:285    ///  24. BLOCK_NEEDS_FREE - indicates that the block has been moved286    ///   to malloc'ed memory287    ///  27. BLOCK_IS_GC - indicates that the block has been moved to288    ///   to GC-allocated memory289    /// Additionally, the bottom 16 bits are a reference count which290    /// should be zero on the stack.291    int flags;292 293    /// Reserved;  should be zero-initialized.294    int reserved;295 296    /// Function pointer generated from block literal.297    _ResultType (*invoke)(Block_literal *, _ParamTypes...);298 299    /// Block description metadata generated from block literal.300    struct Block_descriptor *block_descriptor;301 302    /// Captured values follow.303    _CapturesTypes captures...;304  };305 */306 307namespace {308  /// A chunk of data that we actually have to capture in the block.309  struct BlockLayoutChunk {310    CharUnits Alignment;311    CharUnits Size;312    const BlockDecl::Capture *Capture; // null for 'this'313    llvm::Type *Type;314    QualType FieldType;315    BlockCaptureEntityKind CopyKind, DisposeKind;316    BlockFieldFlags CopyFlags, DisposeFlags;317 318    BlockLayoutChunk(CharUnits align, CharUnits size,319                     const BlockDecl::Capture *capture, llvm::Type *type,320                     QualType fieldType, BlockCaptureEntityKind CopyKind,321                     BlockFieldFlags CopyFlags,322                     BlockCaptureEntityKind DisposeKind,323                     BlockFieldFlags DisposeFlags)324        : Alignment(align), Size(size), Capture(capture), Type(type),325          FieldType(fieldType), CopyKind(CopyKind), DisposeKind(DisposeKind),326          CopyFlags(CopyFlags), DisposeFlags(DisposeFlags) {}327 328    /// Tell the block info that this chunk has the given field index.329    void setIndex(CGBlockInfo &info, unsigned index, CharUnits offset) {330      if (!Capture) {331        info.CXXThisIndex = index;332        info.CXXThisOffset = offset;333      } else {334        info.SortedCaptures.push_back(CGBlockInfo::Capture::makeIndex(335            index, offset, FieldType, CopyKind, CopyFlags, DisposeKind,336            DisposeFlags, Capture));337      }338    }339 340    bool isTrivial() const {341      return CopyKind == BlockCaptureEntityKind::None &&342             DisposeKind == BlockCaptureEntityKind::None;343    }344  };345 346  /// Order by 1) all __strong together 2) next, all block together 3) next,347  /// all byref together 4) next, all __weak together. Preserve descending348  /// alignment in all situations.349  bool operator<(const BlockLayoutChunk &left, const BlockLayoutChunk &right) {350    if (left.Alignment != right.Alignment)351      return left.Alignment > right.Alignment;352 353    auto getPrefOrder = [](const BlockLayoutChunk &chunk) {354      switch (chunk.CopyKind) {355      case BlockCaptureEntityKind::ARCStrong:356        return 0;357      case BlockCaptureEntityKind::BlockObject:358        switch (chunk.CopyFlags.getBitMask()) {359        case BLOCK_FIELD_IS_OBJECT:360          return 0;361        case BLOCK_FIELD_IS_BLOCK:362          return 1;363        case BLOCK_FIELD_IS_BYREF:364          return 2;365        default:366          break;367        }368        break;369      case BlockCaptureEntityKind::ARCWeak:370        return 3;371      default:372        break;373      }374      return 4;375    };376 377    return getPrefOrder(left) < getPrefOrder(right);378  }379} // end anonymous namespace380 381static std::pair<BlockCaptureEntityKind, BlockFieldFlags>382computeCopyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,383                               const LangOptions &LangOpts);384 385static std::pair<BlockCaptureEntityKind, BlockFieldFlags>386computeDestroyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,387                                  const LangOptions &LangOpts);388 389static void addBlockLayout(CharUnits align, CharUnits size,390                           const BlockDecl::Capture *capture, llvm::Type *type,391                           QualType fieldType,392                           SmallVectorImpl<BlockLayoutChunk> &Layout,393                           CGBlockInfo &Info, CodeGenModule &CGM) {394  if (!capture) {395    // 'this' capture.396    Layout.push_back(BlockLayoutChunk(397        align, size, capture, type, fieldType, BlockCaptureEntityKind::None,398        BlockFieldFlags(), BlockCaptureEntityKind::None, BlockFieldFlags()));399    return;400  }401 402  const LangOptions &LangOpts = CGM.getLangOpts();403  BlockCaptureEntityKind CopyKind, DisposeKind;404  BlockFieldFlags CopyFlags, DisposeFlags;405 406  std::tie(CopyKind, CopyFlags) =407      computeCopyInfoForBlockCapture(*capture, fieldType, LangOpts);408  std::tie(DisposeKind, DisposeFlags) =409      computeDestroyInfoForBlockCapture(*capture, fieldType, LangOpts);410  Layout.push_back(BlockLayoutChunk(align, size, capture, type, fieldType,411                                    CopyKind, CopyFlags, DisposeKind,412                                    DisposeFlags));413 414  if (Info.NoEscape)415    return;416 417  if (!Layout.back().isTrivial())418    Info.NeedsCopyDispose = true;419}420 421/// Determines if the given type is safe for constant capture in C++.422static bool isSafeForCXXConstantCapture(QualType type) {423  const auto *record = type->getBaseElementTypeUnsafe()->getAsCXXRecordDecl();424 425  // Only records can be unsafe.426  if (!record)427    return true;428 429  // Maintain semantics for classes with non-trivial dtors or copy ctors.430  if (!record->hasTrivialDestructor()) return false;431  if (record->hasNonTrivialCopyConstructor()) return false;432 433  // Otherwise, we just have to make sure there aren't any mutable434  // fields that might have changed since initialization.435  return !record->hasMutableFields();436}437 438/// It is illegal to modify a const object after initialization.439/// Therefore, if a const object has a constant initializer, we don't440/// actually need to keep storage for it in the block; we'll just441/// rematerialize it at the start of the block function.  This is442/// acceptable because we make no promises about address stability of443/// captured variables.444static llvm::Constant *tryCaptureAsConstant(CodeGenModule &CGM,445                                            CodeGenFunction *CGF,446                                            const VarDecl *var) {447  // Return if this is a function parameter. We shouldn't try to448  // rematerialize default arguments of function parameters.449  if (isa<ParmVarDecl>(var))450    return nullptr;451 452  QualType type = var->getType();453 454  // We can only do this if the variable is const.455  if (!type.isConstQualified()) return nullptr;456 457  // Furthermore, in C++ we have to worry about mutable fields:458  // C++ [dcl.type.cv]p4:459  //   Except that any class member declared mutable can be460  //   modified, any attempt to modify a const object during its461  //   lifetime results in undefined behavior.462  if (CGM.getLangOpts().CPlusPlus && !isSafeForCXXConstantCapture(type))463    return nullptr;464 465  // If the variable doesn't have any initializer (shouldn't this be466  // invalid?), it's not clear what we should do.  Maybe capture as467  // zero?468  const Expr *init = var->getInit();469  if (!init) return nullptr;470 471  return ConstantEmitter(CGM, CGF).tryEmitAbstractForInitializer(*var);472}473 474/// Get the low bit of a nonzero character count.  This is the475/// alignment of the nth byte if the 0th byte is universally aligned.476static CharUnits getLowBit(CharUnits v) {477  return CharUnits::fromQuantity(v.getQuantity() & (~v.getQuantity() + 1));478}479 480static void initializeForBlockHeader(CodeGenModule &CGM, CGBlockInfo &info,481                             SmallVectorImpl<llvm::Type*> &elementTypes) {482 483  assert(elementTypes.empty());484  if (CGM.getLangOpts().OpenCL) {485    // The header is basically 'struct { int; int; generic void *;486    // custom_fields; }'. Assert that struct is packed.487    auto GenPtrAlign = CharUnits::fromQuantity(488        CGM.getTarget().getPointerAlign(LangAS::opencl_generic) / 8);489    auto GenPtrSize = CharUnits::fromQuantity(490        CGM.getTarget().getPointerWidth(LangAS::opencl_generic) / 8);491    assert(CGM.getIntSize() <= GenPtrSize);492    assert(CGM.getIntAlign() <= GenPtrAlign);493    assert((2 * CGM.getIntSize()).isMultipleOf(GenPtrAlign));494    elementTypes.push_back(CGM.IntTy); /* total size */495    elementTypes.push_back(CGM.IntTy); /* align */496    elementTypes.push_back(497        CGM.getOpenCLRuntime()498            .getGenericVoidPointerType()); /* invoke function */499    unsigned Offset =500        2 * CGM.getIntSize().getQuantity() + GenPtrSize.getQuantity();501    unsigned BlockAlign = GenPtrAlign.getQuantity();502    if (auto *Helper =503            CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {504      for (auto *I : Helper->getCustomFieldTypes()) /* custom fields */ {505        // TargetOpenCLBlockHelp needs to make sure the struct is packed.506        // If necessary, add padding fields to the custom fields.507        unsigned Align = CGM.getDataLayout().getABITypeAlign(I).value();508        if (BlockAlign < Align)509          BlockAlign = Align;510        assert(Offset % Align == 0);511        Offset += CGM.getDataLayout().getTypeAllocSize(I);512        elementTypes.push_back(I);513      }514    }515    info.BlockAlign = CharUnits::fromQuantity(BlockAlign);516    info.BlockSize = CharUnits::fromQuantity(Offset);517  } else {518    // The header is basically 'struct { void *; int; int; void *; void *; }'.519    // Assert that the struct is packed.520    assert(CGM.getIntSize() <= CGM.getPointerSize());521    assert(CGM.getIntAlign() <= CGM.getPointerAlign());522    assert((2 * CGM.getIntSize()).isMultipleOf(CGM.getPointerAlign()));523    info.BlockAlign = CGM.getPointerAlign();524    info.BlockSize = 3 * CGM.getPointerSize() + 2 * CGM.getIntSize();525    elementTypes.push_back(CGM.VoidPtrTy);526    elementTypes.push_back(CGM.IntTy);527    elementTypes.push_back(CGM.IntTy);528    elementTypes.push_back(CGM.VoidPtrTy);529    elementTypes.push_back(CGM.getBlockDescriptorType());530  }531}532 533static QualType getCaptureFieldType(const CodeGenFunction &CGF,534                                    const BlockDecl::Capture &CI) {535  const VarDecl *VD = CI.getVariable();536 537  // If the variable is captured by an enclosing block or lambda expression,538  // use the type of the capture field.539  if (CGF.BlockInfo && CI.isNested())540    return CGF.BlockInfo->getCapture(VD).fieldType();541  if (auto *FD = CGF.LambdaCaptureFields.lookup(VD))542    return FD->getType();543  // If the captured variable is a non-escaping __block variable, the field544  // type is the reference type. If the variable is a __block variable that545  // already has a reference type, the field type is the variable's type.546  return VD->isNonEscapingByref() ?547         CGF.getContext().getLValueReferenceType(VD->getType()) : VD->getType();548}549 550/// Compute the layout of the given block.  Attempts to lay the block551/// out with minimal space requirements.552static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF,553                             CGBlockInfo &info) {554  ASTContext &C = CGM.getContext();555  const BlockDecl *block = info.getBlockDecl();556 557  SmallVector<llvm::Type*, 8> elementTypes;558  initializeForBlockHeader(CGM, info, elementTypes);559  bool hasNonConstantCustomFields = false;560  if (auto *OpenCLHelper =561          CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper())562    hasNonConstantCustomFields =563        !OpenCLHelper->areAllCustomFieldValuesConstant(info);564  if (!block->hasCaptures() && !hasNonConstantCustomFields) {565    info.StructureType =566      llvm::StructType::get(CGM.getLLVMContext(), elementTypes, true);567    info.CanBeGlobal = true;568    return;569  } else if (C.getLangOpts().ObjC &&570             CGM.getLangOpts().getGC() == LangOptions::NonGC)571    info.HasCapturedVariableLayout = true;572 573  if (block->doesNotEscape())574    info.NoEscape = true;575 576  // Collect the layout chunks.577  SmallVector<BlockLayoutChunk, 16> layout;578  layout.reserve(block->capturesCXXThis() +579                 (block->capture_end() - block->capture_begin()));580 581  CharUnits maxFieldAlign;582 583  // First, 'this'.584  if (block->capturesCXXThis()) {585    assert(CGF && isa_and_nonnull<CXXMethodDecl>(CGF->CurFuncDecl) &&586           "Can't capture 'this' outside a method");587    QualType thisType = cast<CXXMethodDecl>(CGF->CurFuncDecl)->getThisType();588 589    // Theoretically, this could be in a different address space, so590    // don't assume standard pointer size/align.591    llvm::Type *llvmType = CGM.getTypes().ConvertType(thisType);592    auto TInfo = CGM.getContext().getTypeInfoInChars(thisType);593    maxFieldAlign = std::max(maxFieldAlign, TInfo.Align);594 595    addBlockLayout(TInfo.Align, TInfo.Width, nullptr, llvmType, thisType,596                   layout, info, CGM);597  }598 599  // Next, all the block captures.600  for (const auto &CI : block->captures()) {601    const VarDecl *variable = CI.getVariable();602 603    if (CI.isEscapingByref()) {604      // Just use void* instead of a pointer to the byref type.605      CharUnits align = CGM.getPointerAlign();606      maxFieldAlign = std::max(maxFieldAlign, align);607 608      // Since a __block variable cannot be captured by lambdas, its type and609      // the capture field type should always match.610      assert(CGF && getCaptureFieldType(*CGF, CI) == variable->getType() &&611             "capture type differs from the variable type");612      addBlockLayout(align, CGM.getPointerSize(), &CI, CGM.VoidPtrTy,613                     variable->getType(), layout, info, CGM);614      continue;615    }616 617    // Otherwise, build a layout chunk with the size and alignment of618    // the declaration.619    if (llvm::Constant *constant = tryCaptureAsConstant(CGM, CGF, variable)) {620      info.SortedCaptures.push_back(621          CGBlockInfo::Capture::makeConstant(constant, &CI));622      continue;623    }624 625    QualType VT = getCaptureFieldType(*CGF, CI);626 627    if (CGM.getLangOpts().CPlusPlus)628      if (const CXXRecordDecl *record = VT->getAsCXXRecordDecl())629        if (CI.hasCopyExpr() || !record->hasTrivialDestructor()) {630          info.HasCXXObject = true;631          if (!record->isExternallyVisible())632            info.CapturesNonExternalType = true;633        }634 635    CharUnits size = C.getTypeSizeInChars(VT);636    CharUnits align = C.getDeclAlign(variable);637 638    maxFieldAlign = std::max(maxFieldAlign, align);639 640    llvm::Type *llvmType =641      CGM.getTypes().ConvertTypeForMem(VT);642 643    addBlockLayout(align, size, &CI, llvmType, VT, layout, info, CGM);644  }645 646  // If that was everything, we're done here.647  if (layout.empty()) {648    info.StructureType =649      llvm::StructType::get(CGM.getLLVMContext(), elementTypes, true);650    info.CanBeGlobal = true;651    info.buildCaptureMap();652    return;653  }654 655  // Sort the layout by alignment.  We have to use a stable sort here656  // to get reproducible results.  There should probably be an657  // llvm::array_pod_stable_sort.658  llvm::stable_sort(layout);659 660  // Needed for blocks layout info.661  info.BlockHeaderForcedGapOffset = info.BlockSize;662  info.BlockHeaderForcedGapSize = CharUnits::Zero();663 664  CharUnits &blockSize = info.BlockSize;665  info.BlockAlign = std::max(maxFieldAlign, info.BlockAlign);666 667  // Assuming that the first byte in the header is maximally aligned,668  // get the alignment of the first byte following the header.669  CharUnits endAlign = getLowBit(blockSize);670 671  // If the end of the header isn't satisfactorily aligned for the672  // maximum thing, look for things that are okay with the header-end673  // alignment, and keep appending them until we get something that's674  // aligned right.  This algorithm is only guaranteed optimal if675  // that condition is satisfied at some point; otherwise we can get676  // things like:677  //   header                 // next byte has alignment 4678  //   something_with_size_5; // next byte has alignment 1679  //   something_with_alignment_8;680  // which has 7 bytes of padding, as opposed to the naive solution681  // which might have less (?).682  if (endAlign < maxFieldAlign) {683    SmallVectorImpl<BlockLayoutChunk>::iterator684      li = layout.begin() + 1, le = layout.end();685 686    // Look for something that the header end is already687    // satisfactorily aligned for.688    for (; li != le && endAlign < li->Alignment; ++li)689      ;690 691    // If we found something that's naturally aligned for the end of692    // the header, keep adding things...693    if (li != le) {694      SmallVectorImpl<BlockLayoutChunk>::iterator first = li;695      for (; li != le; ++li) {696        assert(endAlign >= li->Alignment);697 698        li->setIndex(info, elementTypes.size(), blockSize);699        elementTypes.push_back(li->Type);700        blockSize += li->Size;701        endAlign = getLowBit(blockSize);702 703        // ...until we get to the alignment of the maximum field.704        if (endAlign >= maxFieldAlign) {705          ++li;706          break;707        }708      }709      // Don't re-append everything we just appended.710      layout.erase(first, li);711    }712  }713 714  assert(endAlign == getLowBit(blockSize));715 716  // At this point, we just have to add padding if the end align still717  // isn't aligned right.718  if (endAlign < maxFieldAlign) {719    CharUnits newBlockSize = blockSize.alignTo(maxFieldAlign);720    CharUnits padding = newBlockSize - blockSize;721 722    // If we haven't yet added any fields, remember that there was an723    // initial gap; this need to go into the block layout bit map.724    if (blockSize == info.BlockHeaderForcedGapOffset) {725      info.BlockHeaderForcedGapSize = padding;726    }727 728    elementTypes.push_back(llvm::ArrayType::get(CGM.Int8Ty,729                                                padding.getQuantity()));730    blockSize = newBlockSize;731    endAlign = getLowBit(blockSize); // might be > maxFieldAlign732  }733 734  assert(endAlign >= maxFieldAlign);735  assert(endAlign == getLowBit(blockSize));736  // Slam everything else on now.  This works because they have737  // strictly decreasing alignment and we expect that size is always a738  // multiple of alignment.739  for (SmallVectorImpl<BlockLayoutChunk>::iterator740         li = layout.begin(), le = layout.end(); li != le; ++li) {741    if (endAlign < li->Alignment) {742      // size may not be multiple of alignment. This can only happen with743      // an over-aligned variable. We will be adding a padding field to744      // make the size be multiple of alignment.745      CharUnits padding = li->Alignment - endAlign;746      elementTypes.push_back(llvm::ArrayType::get(CGM.Int8Ty,747                                                  padding.getQuantity()));748      blockSize += padding;749      endAlign = getLowBit(blockSize);750    }751    assert(endAlign >= li->Alignment);752    li->setIndex(info, elementTypes.size(), blockSize);753    elementTypes.push_back(li->Type);754    blockSize += li->Size;755    endAlign = getLowBit(blockSize);756  }757 758  info.buildCaptureMap();759  info.StructureType =760    llvm::StructType::get(CGM.getLLVMContext(), elementTypes, true);761}762 763/// Emit a block literal expression in the current function.764llvm::Value *CodeGenFunction::EmitBlockLiteral(const BlockExpr *blockExpr) {765  // If the block has no captures, we won't have a pre-computed766  // layout for it.767  if (!blockExpr->getBlockDecl()->hasCaptures())768    // The block literal is emitted as a global variable, and the block invoke769    // function has to be extracted from its initializer.770    if (llvm::Constant *Block = CGM.getAddrOfGlobalBlockIfEmitted(blockExpr))771      return Block;772 773  CGBlockInfo blockInfo(blockExpr->getBlockDecl(), CurFn->getName());774  computeBlockInfo(CGM, this, blockInfo);775  blockInfo.BlockExpression = blockExpr;776  if (!blockInfo.CanBeGlobal)777    blockInfo.LocalAddress = CreateTempAlloca(blockInfo.StructureType,778                                              blockInfo.BlockAlign, "block");779  return EmitBlockLiteral(blockInfo);780}781 782llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {783  bool IsOpenCL = CGM.getContext().getLangOpts().OpenCL;784  llvm::PointerType *GenVoidPtrTy =785      IsOpenCL ? CGM.getOpenCLRuntime().getGenericVoidPointerType() : VoidPtrTy;786  LangAS GenVoidPtrAddr = IsOpenCL ? LangAS::opencl_generic : LangAS::Default;787  auto GenVoidPtrSize = CharUnits::fromQuantity(788      CGM.getTarget().getPointerWidth(GenVoidPtrAddr) / 8);789  // Using the computed layout, generate the actual block function.790  bool isLambdaConv = blockInfo.getBlockDecl()->isConversionFromLambda();791  CodeGenFunction BlockCGF{CGM, true};792  BlockCGF.SanOpts = SanOpts;793  auto *InvokeFn = BlockCGF.GenerateBlockFunction(794      CurGD, blockInfo, LocalDeclMap, isLambdaConv, blockInfo.CanBeGlobal);795  auto *blockFn = llvm::ConstantExpr::getPointerCast(InvokeFn, GenVoidPtrTy);796 797  // If there is nothing to capture, we can emit this as a global block.798  if (blockInfo.CanBeGlobal)799    return CGM.getAddrOfGlobalBlockIfEmitted(blockInfo.BlockExpression);800 801  // Otherwise, we have to emit this as a local block.802 803  RawAddress blockAddr = blockInfo.LocalAddress;804  assert(blockAddr.isValid() && "block has no address!");805 806  llvm::Constant *isa;807  llvm::Constant *descriptor;808  BlockFlags flags;809  if (!IsOpenCL) {810    // If the block is non-escaping, set field 'isa 'to NSConcreteGlobalBlock811    // and set the BLOCK_IS_GLOBAL bit of field 'flags'. Copying a non-escaping812    // block just returns the original block and releasing it is a no-op.813    llvm::Constant *blockISA = blockInfo.NoEscape814                                   ? CGM.getNSConcreteGlobalBlock()815                                   : CGM.getNSConcreteStackBlock();816    isa = blockISA;817 818    // Compute the initial on-stack block flags.819    if (!CGM.getCodeGenOpts().DisableBlockSignatureString)820      flags = BLOCK_HAS_SIGNATURE;821    if (blockInfo.HasCapturedVariableLayout)822      flags |= BLOCK_HAS_EXTENDED_LAYOUT;823    if (blockInfo.NeedsCopyDispose)824      flags |= BLOCK_HAS_COPY_DISPOSE;825    if (blockInfo.HasCXXObject)826      flags |= BLOCK_HAS_CXX_OBJ;827    if (blockInfo.UsesStret)828      flags |= BLOCK_USE_STRET;829    if (blockInfo.NoEscape)830      flags |= BLOCK_IS_NOESCAPE | BLOCK_IS_GLOBAL;831 832    // Build the block descriptor.833    descriptor = buildBlockDescriptor(CGM, blockInfo);834  }835 836  auto projectField = [&](unsigned index, const Twine &name) -> Address {837    return Builder.CreateStructGEP(blockAddr, index, name);838  };839  auto storeField = [&](llvm::Value *value, unsigned index, const Twine &name) {840    Builder.CreateStore(value, projectField(index, name));841  };842 843  // Initialize the block header.844  {845    // We assume all the header fields are densely packed.846    unsigned index = 0;847    CharUnits offset;848    auto addHeaderField = [&](llvm::Value *value, CharUnits size,849                              const Twine &name) {850      storeField(value, index, name);851      offset += size;852      index++;853    };854    auto addSignedHeaderField =855        [&](llvm::Value *Value, const PointerAuthSchema &Schema,856            GlobalDecl Decl, QualType Type, CharUnits Size, const Twine &Name) {857          auto StorageAddress = projectField(index, Name);858          if (Schema) {859            auto AuthInfo = EmitPointerAuthInfo(860                Schema, StorageAddress.emitRawPointer(*this), Decl, Type);861            Value = EmitPointerAuthSign(AuthInfo, Value);862          }863          Builder.CreateStore(Value, StorageAddress);864          offset += Size;865          index++;866        };867 868    if (!IsOpenCL) {869      addSignedHeaderField(870          isa, CGM.getCodeGenOpts().PointerAuth.ObjCIsaPointers, GlobalDecl(),871          QualType(), getPointerSize(), "block.isa");872      addHeaderField(llvm::ConstantInt::get(IntTy, flags.getBitMask()),873                     getIntSize(), "block.flags");874      addHeaderField(llvm::ConstantInt::get(IntTy, 0), getIntSize(),875                     "block.reserved");876    } else {877      addHeaderField(878          llvm::ConstantInt::get(IntTy, blockInfo.BlockSize.getQuantity()),879          getIntSize(), "block.size");880      addHeaderField(881          llvm::ConstantInt::get(IntTy, blockInfo.BlockAlign.getQuantity()),882          getIntSize(), "block.align");883    }884 885    if (!IsOpenCL) {886      llvm::Value *blockFnPtr =887          llvm::ConstantExpr::getBitCast(InvokeFn, VoidPtrTy);888      QualType type = blockInfo.getBlockExpr()889                          ->getType()890                          ->castAs<BlockPointerType>()891                          ->getPointeeType();892      addSignedHeaderField(893          blockFnPtr,894          CGM.getCodeGenOpts().PointerAuth.BlockInvocationFunctionPointers,895          GlobalDecl(), type, getPointerSize(), "block.invoke");896 897      addSignedHeaderField(898          descriptor, CGM.getCodeGenOpts().PointerAuth.BlockDescriptorPointers,899          GlobalDecl(), type, getPointerSize(), "block.descriptor");900    } else if (auto *Helper =901                   CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {902      addHeaderField(blockFn, GenVoidPtrSize, "block.invoke");903      for (auto I : Helper->getCustomFieldValues(*this, blockInfo)) {904        addHeaderField(905            I.first,906            CharUnits::fromQuantity(907                CGM.getDataLayout().getTypeAllocSize(I.first->getType())),908            I.second);909      }910    } else911      addHeaderField(blockFn, GenVoidPtrSize, "block.invoke");912  }913 914  // Finally, capture all the values into the block.915  const BlockDecl *blockDecl = blockInfo.getBlockDecl();916 917  // First, 'this'.918  if (blockDecl->capturesCXXThis()) {919    Address addr =920        projectField(blockInfo.CXXThisIndex, "block.captured-this.addr");921    Builder.CreateStore(LoadCXXThis(), addr);922  }923 924  // Next, captured variables.925  for (const auto &CI : blockDecl->captures()) {926    const VarDecl *variable = CI.getVariable();927    const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable);928 929    // Ignore constant captures.930    if (capture.isConstant()) continue;931 932    QualType type = capture.fieldType();933 934    // This will be a [[type]]*, except that a byref entry will just be935    // an i8**.936    Address blockField = projectField(capture.getIndex(), "block.captured");937 938    // Compute the address of the thing we're going to move into the939    // block literal.940    Address src = Address::invalid();941 942    if (blockDecl->isConversionFromLambda()) {943      // The lambda capture in a lambda's conversion-to-block-pointer is944      // special; we'll simply emit it directly.945      src = Address::invalid();946    } else if (CI.isEscapingByref()) {947      if (BlockInfo && CI.isNested()) {948        // We need to use the capture from the enclosing block.949        const CGBlockInfo::Capture &enclosingCapture =950            BlockInfo->getCapture(variable);951 952        // This is a [[type]]*, except that a byref entry will just be an i8**.953        src = Builder.CreateStructGEP(LoadBlockStruct(),954                                      enclosingCapture.getIndex(),955                                      "block.capture.addr");956      } else {957        auto I = LocalDeclMap.find(variable);958        assert(I != LocalDeclMap.end());959        src = I->second;960      }961    } else {962      DeclRefExpr declRef(getContext(), const_cast<VarDecl *>(variable),963                          /*RefersToEnclosingVariableOrCapture*/ CI.isNested(),964                          type.getNonReferenceType(), VK_LValue,965                          SourceLocation());966      src = EmitDeclRefLValue(&declRef).getAddress();967    };968 969    // For byrefs, we just write the pointer to the byref struct into970    // the block field.  There's no need to chase the forwarding971    // pointer at this point, since we're building something that will972    // live a shorter life than the stack byref anyway.973    if (CI.isEscapingByref()) {974      // Get a void* that points to the byref struct.975      llvm::Value *byrefPointer;976      if (CI.isNested())977        byrefPointer = Builder.CreateLoad(src, "byref.capture");978      else979        byrefPointer = src.emitRawPointer(*this);980 981      // Write that void* into the capture field.982      Builder.CreateStore(byrefPointer, blockField);983 984    // If we have a copy constructor, evaluate that into the block field.985    } else if (const Expr *copyExpr = CI.getCopyExpr()) {986      if (blockDecl->isConversionFromLambda()) {987        // If we have a lambda conversion, emit the expression988        // directly into the block instead.989        AggValueSlot Slot =990            AggValueSlot::forAddr(blockField, Qualifiers(),991                                  AggValueSlot::IsDestructed,992                                  AggValueSlot::DoesNotNeedGCBarriers,993                                  AggValueSlot::IsNotAliased,994                                  AggValueSlot::DoesNotOverlap);995        EmitAggExpr(copyExpr, Slot);996      } else {997        EmitSynthesizedCXXCopyCtor(blockField, src, copyExpr);998      }999 1000    // If it's a reference variable, copy the reference into the block field.1001    } else if (type->getAs<ReferenceType>()) {1002      Builder.CreateStore(src.emitRawPointer(*this), blockField);1003 1004      // If type is const-qualified, copy the value into the block field.1005    } else if (type.isConstQualified() &&1006               type.getObjCLifetime() == Qualifiers::OCL_Strong &&1007               CGM.getCodeGenOpts().OptimizationLevel != 0) {1008      llvm::Value *value = Builder.CreateLoad(src, "captured");1009      Builder.CreateStore(value, blockField);1010 1011    // If this is an ARC __strong block-pointer variable, don't do a1012    // block copy.1013    //1014    // TODO: this can be generalized into the normal initialization logic:1015    // we should never need to do a block-copy when initializing a local1016    // variable, because the local variable's lifetime should be strictly1017    // contained within the stack block's.1018    } else if (type.getObjCLifetime() == Qualifiers::OCL_Strong &&1019               type->isBlockPointerType()) {1020      // Load the block and do a simple retain.1021      llvm::Value *value = Builder.CreateLoad(src, "block.captured_block");1022      value = EmitARCRetainNonBlock(value);1023 1024      // Do a primitive store to the block field.1025      Builder.CreateStore(value, blockField);1026 1027    // Otherwise, fake up a POD copy into the block field.1028    } else {1029      // Fake up a new variable so that EmitScalarInit doesn't think1030      // we're referring to the variable in its own initializer.1031      ImplicitParamDecl BlockFieldPseudoVar(getContext(), type,1032                                            ImplicitParamKind::Other);1033 1034      // We use one of these or the other depending on whether the1035      // reference is nested.1036      DeclRefExpr declRef(getContext(), const_cast<VarDecl *>(variable),1037                          /*RefersToEnclosingVariableOrCapture*/ CI.isNested(),1038                          type, VK_LValue, SourceLocation());1039 1040      ImplicitCastExpr l2r(ImplicitCastExpr::OnStack, type, CK_LValueToRValue,1041                           &declRef, VK_PRValue, FPOptionsOverride());1042      // FIXME: Pass a specific location for the expr init so that the store is1043      // attributed to a reasonable location - otherwise it may be attributed to1044      // locations of subexpressions in the initialization.1045      EmitExprAsInit(&l2r, &BlockFieldPseudoVar,1046                     MakeAddrLValue(blockField, type, AlignmentSource::Decl),1047                     /*captured by init*/ false);1048    }1049 1050    // Push a cleanup for the capture if necessary.1051    if (!blockInfo.NoEscape && !blockInfo.NeedsCopyDispose)1052      continue;1053 1054    // Ignore __block captures; there's nothing special in the on-stack block1055    // that we need to do for them.1056    if (CI.isByRef())1057      continue;1058 1059    // Ignore objects that aren't destructed.1060    QualType::DestructionKind dtorKind = type.isDestructedType();1061    if (dtorKind == QualType::DK_none)1062      continue;1063 1064    CodeGenFunction::Destroyer *destroyer;1065 1066    // Block captures count as local values and have imprecise semantics.1067    // They also can't be arrays, so need to worry about that.1068    //1069    // For const-qualified captures, emit clang.arc.use to ensure the captured1070    // object doesn't get released while we are still depending on its validity1071    // within the block.1072    if (type.isConstQualified() &&1073        type.getObjCLifetime() == Qualifiers::OCL_Strong &&1074        CGM.getCodeGenOpts().OptimizationLevel != 0) {1075      assert(CGM.getLangOpts().ObjCAutoRefCount &&1076             "expected ObjC ARC to be enabled");1077      destroyer = emitARCIntrinsicUse;1078    } else if (dtorKind == QualType::DK_objc_strong_lifetime) {1079      destroyer = destroyARCStrongImprecise;1080    } else {1081      destroyer = getDestroyer(dtorKind);1082    }1083 1084    CleanupKind cleanupKind = NormalCleanup;1085    bool useArrayEHCleanup = needsEHCleanup(dtorKind);1086    if (useArrayEHCleanup)1087      cleanupKind = NormalAndEHCleanup;1088 1089    // Extend the lifetime of the capture to the end of the scope enclosing the1090    // block expression except when the block decl is in the list of RetExpr's1091    // cleanup objects, in which case its lifetime ends after the full1092    // expression.1093    auto IsBlockDeclInRetExpr = [&]() {1094      auto *EWC = llvm::dyn_cast_or_null<ExprWithCleanups>(RetExpr);1095      if (EWC)1096        for (auto &C : EWC->getObjects())1097          if (auto *BD = C.dyn_cast<BlockDecl *>())1098            if (BD == blockDecl)1099              return true;1100      return false;1101    };1102 1103    if (IsBlockDeclInRetExpr())1104      pushDestroy(cleanupKind, blockField, type, destroyer, useArrayEHCleanup);1105    else1106      pushLifetimeExtendedDestroy(cleanupKind, blockField, type, destroyer,1107                                  useArrayEHCleanup);1108  }1109 1110  // Cast to the converted block-pointer type, which happens (somewhat1111  // unfortunately) to be a pointer to function type.1112  llvm::Value *result = Builder.CreatePointerCast(1113      blockAddr.getPointer(), ConvertType(blockInfo.getBlockExpr()->getType()));1114 1115  if (IsOpenCL) {1116    CGM.getOpenCLRuntime().recordBlockInfo(blockInfo.BlockExpression, InvokeFn,1117                                           result, blockInfo.StructureType);1118  }1119 1120  return result;1121}1122 1123 1124llvm::Type *CodeGenModule::getBlockDescriptorType() {1125  if (BlockDescriptorType)1126    return BlockDescriptorType;1127 1128  unsigned AddrSpace = 0;1129  if (getLangOpts().OpenCL)1130    AddrSpace = getContext().getTargetAddressSpace(LangAS::opencl_constant);1131  BlockDescriptorType = llvm::PointerType::get(getLLVMContext(), AddrSpace);1132  return BlockDescriptorType;1133}1134 1135llvm::Type *CodeGenModule::getGenericBlockLiteralType() {1136  if (GenericBlockLiteralType)1137    return GenericBlockLiteralType;1138 1139  llvm::Type *BlockDescPtrTy = getBlockDescriptorType();1140 1141  if (getLangOpts().OpenCL) {1142    // struct __opencl_block_literal_generic {1143    //   int __size;1144    //   int __align;1145    //   __generic void *__invoke;1146    //   /* custom fields */1147    // };1148    SmallVector<llvm::Type *, 8> StructFields(1149        {IntTy, IntTy, getOpenCLRuntime().getGenericVoidPointerType()});1150    if (auto *Helper = getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {1151      llvm::append_range(StructFields, Helper->getCustomFieldTypes());1152    }1153    GenericBlockLiteralType = llvm::StructType::create(1154        StructFields, "struct.__opencl_block_literal_generic");1155  } else {1156    // struct __block_literal_generic {1157    //   void *__isa;1158    //   int __flags;1159    //   int __reserved;1160    //   void (*__invoke)(void *);1161    //   struct __block_descriptor *__descriptor;1162    // };1163    GenericBlockLiteralType =1164        llvm::StructType::create("struct.__block_literal_generic", VoidPtrTy,1165                                 IntTy, IntTy, VoidPtrTy, BlockDescPtrTy);1166  }1167 1168  return GenericBlockLiteralType;1169}1170 1171RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E,1172                                          ReturnValueSlot ReturnValue,1173                                          llvm::CallBase **CallOrInvoke) {1174  const auto *BPT = E->getCallee()->getType()->castAs<BlockPointerType>();1175  llvm::Value *BlockPtr = EmitScalarExpr(E->getCallee());1176  llvm::Type *GenBlockTy = CGM.getGenericBlockLiteralType();1177  llvm::Value *Func = nullptr;1178  QualType FnType = BPT->getPointeeType();1179  ASTContext &Ctx = getContext();1180  CallArgList Args;1181 1182  llvm::Value *FuncPtr = nullptr;1183 1184  if (getLangOpts().OpenCL) {1185    // For OpenCL, BlockPtr is already casted to generic block literal.1186 1187    // First argument of a block call is a generic block literal casted to1188    // generic void pointer, i.e. i8 addrspace(4)*1189    llvm::Type *GenericVoidPtrTy =1190        CGM.getOpenCLRuntime().getGenericVoidPointerType();1191    llvm::Value *BlockDescriptor = Builder.CreatePointerCast(1192        BlockPtr, GenericVoidPtrTy);1193    QualType VoidPtrQualTy = Ctx.getPointerType(1194        Ctx.getAddrSpaceQualType(Ctx.VoidTy, LangAS::opencl_generic));1195    Args.add(RValue::get(BlockDescriptor), VoidPtrQualTy);1196    // And the rest of the arguments.1197    EmitCallArgs(Args, FnType->getAs<FunctionProtoType>(), E->arguments());1198 1199    // We *can* call the block directly unless it is a function argument.1200    if (!isa<ParmVarDecl>(E->getCalleeDecl()))1201      Func = CGM.getOpenCLRuntime().getInvokeFunction(E->getCallee());1202    else {1203      FuncPtr = Builder.CreateStructGEP(GenBlockTy, BlockPtr, 2);1204      Func = Builder.CreateAlignedLoad(GenericVoidPtrTy, FuncPtr,1205                                       getPointerAlign());1206    }1207  } else {1208    // Bitcast the block literal to a generic block literal.1209    BlockPtr =1210        Builder.CreatePointerCast(BlockPtr, DefaultPtrTy, "block.literal");1211    // Get pointer to the block invoke function1212    FuncPtr = Builder.CreateStructGEP(GenBlockTy, BlockPtr, 3);1213 1214    // First argument is a block literal casted to a void pointer1215    BlockPtr = Builder.CreatePointerCast(BlockPtr, VoidPtrTy);1216    Args.add(RValue::get(BlockPtr), Ctx.VoidPtrTy);1217    // And the rest of the arguments.1218    EmitCallArgs(Args, FnType->getAs<FunctionProtoType>(), E->arguments());1219 1220    // Load the function.1221    Func = Builder.CreateAlignedLoad(VoidPtrTy, FuncPtr, getPointerAlign());1222  }1223 1224  const FunctionType *FuncTy = FnType->castAs<FunctionType>();1225  const CGFunctionInfo &FnInfo =1226    CGM.getTypes().arrangeBlockFunctionCall(Args, FuncTy);1227 1228  // Prepare the callee.1229  CGPointerAuthInfo PointerAuth;1230  if (auto &AuthSchema =1231          CGM.getCodeGenOpts().PointerAuth.BlockInvocationFunctionPointers) {1232    assert(FuncPtr != nullptr && "Missing function pointer for AuthInfo");1233    PointerAuth =1234        EmitPointerAuthInfo(AuthSchema, FuncPtr, GlobalDecl(), FnType);1235  }1236 1237  CGCallee Callee(CGCalleeInfo(), Func, PointerAuth);1238 1239  // And call the block.1240  return EmitCall(FnInfo, Callee, ReturnValue, Args, CallOrInvoke);1241}1242 1243Address CodeGenFunction::GetAddrOfBlockDecl(const VarDecl *variable) {1244  assert(BlockInfo && "evaluating block ref without block information?");1245  const CGBlockInfo::Capture &capture = BlockInfo->getCapture(variable);1246 1247  // Handle constant captures.1248  if (capture.isConstant()) return LocalDeclMap.find(variable)->second;1249 1250  Address addr = Builder.CreateStructGEP(LoadBlockStruct(), capture.getIndex(),1251                                         "block.capture.addr");1252 1253  if (variable->isEscapingByref()) {1254    // addr should be a void** right now.  Load, then cast the result1255    // to byref*.1256 1257    auto &byrefInfo = getBlockByrefInfo(variable);1258    addr = Address(Builder.CreateLoad(addr), byrefInfo.Type,1259                   byrefInfo.ByrefAlignment);1260 1261    addr = emitBlockByrefAddress(addr, byrefInfo, /*follow*/ true,1262                                 variable->getName());1263  }1264 1265  assert((!variable->isNonEscapingByref() ||1266          capture.fieldType()->isReferenceType()) &&1267         "the capture field of a non-escaping variable should have a "1268         "reference type");1269  if (capture.fieldType()->isReferenceType())1270    addr = EmitLoadOfReference(MakeAddrLValue(addr, capture.fieldType()));1271 1272  return addr;1273}1274 1275void CodeGenModule::setAddrOfGlobalBlock(const BlockExpr *BE,1276                                         llvm::Constant *Addr) {1277  bool Ok = EmittedGlobalBlocks.insert(std::make_pair(BE, Addr)).second;1278  (void)Ok;1279  assert(Ok && "Trying to replace an already-existing global block!");1280}1281 1282llvm::Constant *1283CodeGenModule::GetAddrOfGlobalBlock(const BlockExpr *BE,1284                                    StringRef Name) {1285  if (llvm::Constant *Block = getAddrOfGlobalBlockIfEmitted(BE))1286    return Block;1287 1288  CGBlockInfo blockInfo(BE->getBlockDecl(), Name);1289  blockInfo.BlockExpression = BE;1290 1291  // Compute information about the layout, etc., of this block.1292  computeBlockInfo(*this, nullptr, blockInfo);1293 1294  // Using that metadata, generate the actual block function.1295  {1296    CodeGenFunction::DeclMapTy LocalDeclMap;1297    CodeGenFunction(*this).GenerateBlockFunction(1298        GlobalDecl(), blockInfo, LocalDeclMap,1299        /*IsLambdaConversionToBlock*/ false, /*BuildGlobalBlock*/ true);1300  }1301 1302  return getAddrOfGlobalBlockIfEmitted(BE);1303}1304 1305static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM,1306                                        const CGBlockInfo &blockInfo,1307                                        llvm::Constant *blockFn) {1308  assert(blockInfo.CanBeGlobal);1309  // Callers should detect this case on their own: calling this function1310  // generally requires computing layout information, which is a waste of time1311  // if we've already emitted this block.1312  assert(!CGM.getAddrOfGlobalBlockIfEmitted(blockInfo.BlockExpression) &&1313         "Refusing to re-emit a global block.");1314 1315  // Generate the constants for the block literal initializer.1316  ConstantInitBuilder builder(CGM);1317  auto fields = builder.beginStruct();1318 1319  bool IsOpenCL = CGM.getLangOpts().OpenCL;1320  bool IsWindows = CGM.getTarget().getTriple().isOSWindows();1321  auto &CGOPointerAuth = CGM.getCodeGenOpts().PointerAuth;1322  if (!IsOpenCL) {1323    // isa1324    if (IsWindows)1325      fields.addNullPointer(CGM.Int8PtrPtrTy);1326    else1327      fields.addSignedPointer(CGM.getNSConcreteGlobalBlock(),1328                              CGOPointerAuth.ObjCIsaPointers, GlobalDecl(),1329                              QualType());1330 1331    // __flags1332    BlockFlags flags = BLOCK_IS_GLOBAL;1333    if (!CGM.getCodeGenOpts().DisableBlockSignatureString)1334      flags |= BLOCK_HAS_SIGNATURE;1335    if (blockInfo.UsesStret)1336      flags |= BLOCK_USE_STRET;1337 1338    fields.addInt(CGM.IntTy, flags.getBitMask());1339 1340    // Reserved1341    fields.addInt(CGM.IntTy, 0);1342  } else {1343    fields.addInt(CGM.IntTy, blockInfo.BlockSize.getQuantity());1344    fields.addInt(CGM.IntTy, blockInfo.BlockAlign.getQuantity());1345  }1346 1347  // Function1348  if (auto &Schema = CGOPointerAuth.BlockInvocationFunctionPointers) {1349    QualType FnType = blockInfo.getBlockExpr()1350                          ->getType()1351                          ->castAs<BlockPointerType>()1352                          ->getPointeeType();1353    fields.addSignedPointer(blockFn, Schema, GlobalDecl(), FnType);1354  } else1355    fields.add(blockFn);1356 1357  if (!IsOpenCL) {1358    // Descriptor1359    llvm::Constant *Descriptor = buildBlockDescriptor(CGM, blockInfo);1360    fields.addSignedPointer(Descriptor, CGOPointerAuth.BlockDescriptorPointers,1361                            GlobalDecl(), QualType());1362  } else if (auto *Helper =1363                 CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {1364    for (auto *I : Helper->getCustomFieldValues(CGM, blockInfo)) {1365      fields.add(I);1366    }1367  }1368 1369  unsigned AddrSpace = 0;1370  if (CGM.getContext().getLangOpts().OpenCL)1371    AddrSpace = CGM.getContext().getTargetAddressSpace(LangAS::opencl_global);1372 1373  llvm::GlobalVariable *literal = fields.finishAndCreateGlobal(1374      "__block_literal_global", blockInfo.BlockAlign,1375      /*constant*/ !IsWindows, llvm::GlobalVariable::InternalLinkage, AddrSpace);1376 1377  literal->addAttribute("objc_arc_inert");1378 1379  // Windows does not allow globals to be initialised to point to globals in1380  // different DLLs.  Any such variables must run code to initialise them.1381  if (IsWindows) {1382    auto *Init = llvm::Function::Create(llvm::FunctionType::get(CGM.VoidTy,1383          {}), llvm::GlobalValue::InternalLinkage, ".block_isa_init",1384        &CGM.getModule());1385    llvm::IRBuilder<> b(llvm::BasicBlock::Create(CGM.getLLVMContext(), "entry",1386          Init));1387    b.CreateAlignedStore(CGM.getNSConcreteGlobalBlock(),1388                         b.CreateStructGEP(literal->getValueType(), literal, 0),1389                         CGM.getPointerAlign().getAsAlign());1390    b.CreateRetVoid();1391    // We can't use the normal LLVM global initialisation array, because we1392    // need to specify that this runs early in library initialisation.1393    auto *InitVar = new llvm::GlobalVariable(CGM.getModule(), Init->getType(),1394        /*isConstant*/true, llvm::GlobalValue::InternalLinkage,1395        Init, ".block_isa_init_ptr");1396    InitVar->setSection(".CRT$XCLa");1397    CGM.addUsedGlobal(InitVar);1398  }1399 1400  // Return a constant of the appropriately-casted type.1401  llvm::Type *RequiredType =1402    CGM.getTypes().ConvertType(blockInfo.getBlockExpr()->getType());1403  llvm::Constant *Result =1404      llvm::ConstantExpr::getPointerCast(literal, RequiredType);1405  CGM.setAddrOfGlobalBlock(blockInfo.BlockExpression, Result);1406  if (CGM.getContext().getLangOpts().OpenCL)1407    CGM.getOpenCLRuntime().recordBlockInfo(1408        blockInfo.BlockExpression,1409        cast<llvm::Function>(blockFn->stripPointerCasts()), Result,1410        literal->getValueType());1411  return Result;1412}1413 1414void CodeGenFunction::setBlockContextParameter(const ImplicitParamDecl *D,1415                                               unsigned argNum,1416                                               llvm::Value *arg) {1417  assert(BlockInfo && "not emitting prologue of block invocation function?!");1418 1419  // Allocate a stack slot like for any local variable to guarantee optimal1420  // debug info at -O0. The mem2reg pass will eliminate it when optimizing.1421  RawAddress alloc = CreateMemTemp(D->getType(), D->getName() + ".addr");1422  Builder.CreateStore(arg, alloc);1423  if (CGDebugInfo *DI = getDebugInfo()) {1424    if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {1425      DI->setLocation(D->getLocation());1426      DI->EmitDeclareOfBlockLiteralArgVariable(1427          *BlockInfo, D->getName(), argNum,1428          cast<llvm::AllocaInst>(alloc.getPointer()->stripPointerCasts()),1429          Builder);1430    }1431  }1432 1433  SourceLocation StartLoc = BlockInfo->getBlockExpr()->getBody()->getBeginLoc();1434  ApplyDebugLocation Scope(*this, StartLoc);1435 1436  // Instead of messing around with LocalDeclMap, just set the value1437  // directly as BlockPointer.1438  BlockPointer = Builder.CreatePointerCast(1439      arg,1440      llvm::PointerType::get(1441          getLLVMContext(),1442          getContext().getLangOpts().OpenCL1443              ? getContext().getTargetAddressSpace(LangAS::opencl_generic)1444              : 0),1445      "block");1446}1447 1448Address CodeGenFunction::LoadBlockStruct() {1449  assert(BlockInfo && "not in a block invocation function!");1450  assert(BlockPointer && "no block pointer set!");1451  return Address(BlockPointer, BlockInfo->StructureType, BlockInfo->BlockAlign);1452}1453 1454llvm::Function *CodeGenFunction::GenerateBlockFunction(1455    GlobalDecl GD, const CGBlockInfo &blockInfo, const DeclMapTy &ldm,1456    bool IsLambdaConversionToBlock, bool BuildGlobalBlock) {1457  const BlockDecl *blockDecl = blockInfo.getBlockDecl();1458 1459  CurGD = GD;1460 1461  CurEHLocation = blockInfo.getBlockExpr()->getEndLoc();1462 1463  BlockInfo = &blockInfo;1464 1465  // Arrange for local static and local extern declarations to appear1466  // to be local to this function as well, in case they're directly1467  // referenced in a block.1468  for (const auto &KV : ldm) {1469    const auto *var = dyn_cast<VarDecl>(KV.first);1470    if (var && !var->hasLocalStorage())1471      setAddrOfLocalVar(var, KV.second);1472  }1473 1474  // Begin building the function declaration.1475 1476  // Build the argument list.1477  FunctionArgList args;1478 1479  // The first argument is the block pointer.  Just take it as a void*1480  // and cast it later.1481  QualType selfTy = getContext().VoidPtrTy;1482 1483  // For OpenCL passed block pointer can be private AS local variable or1484  // global AS program scope variable (for the case with and without captures).1485  // Generic AS is used therefore to be able to accommodate both private and1486  // generic AS in one implementation.1487  if (getLangOpts().OpenCL)1488    selfTy = getContext().getPointerType(getContext().getAddrSpaceQualType(1489        getContext().VoidTy, LangAS::opencl_generic));1490 1491  const IdentifierInfo *II = &CGM.getContext().Idents.get(".block_descriptor");1492 1493  ImplicitParamDecl SelfDecl(getContext(), const_cast<BlockDecl *>(blockDecl),1494                             SourceLocation(), II, selfTy,1495                             ImplicitParamKind::ObjCSelf);1496  args.push_back(&SelfDecl);1497 1498  // Now add the rest of the parameters.1499  args.append(blockDecl->param_begin(), blockDecl->param_end());1500 1501  // Create the function declaration.1502  const FunctionProtoType *fnType = blockInfo.getBlockExpr()->getFunctionType();1503  const CGFunctionInfo &fnInfo =1504    CGM.getTypes().arrangeBlockFunctionDeclaration(fnType, args);1505  if (CGM.ReturnSlotInterferesWithArgs(fnInfo))1506    blockInfo.UsesStret = true;1507 1508  llvm::FunctionType *fnLLVMType = CGM.getTypes().GetFunctionType(fnInfo);1509 1510  StringRef name = CGM.getBlockMangledName(GD, blockDecl);1511  llvm::Function *fn = llvm::Function::Create(1512      fnLLVMType, llvm::GlobalValue::InternalLinkage, name, &CGM.getModule());1513  CGM.SetInternalFunctionAttributes(blockDecl, fn, fnInfo);1514 1515  if (BuildGlobalBlock) {1516    auto GenVoidPtrTy = getContext().getLangOpts().OpenCL1517                            ? CGM.getOpenCLRuntime().getGenericVoidPointerType()1518                            : VoidPtrTy;1519    buildGlobalBlock(CGM, blockInfo,1520                     llvm::ConstantExpr::getPointerCast(fn, GenVoidPtrTy));1521  }1522 1523  // Begin generating the function.1524  StartFunction(blockDecl, fnType->getReturnType(), fn, fnInfo, args,1525                blockDecl->getLocation(),1526                blockInfo.getBlockExpr()->getBody()->getBeginLoc());1527 1528  // Okay.  Undo some of what StartFunction did.1529 1530  // At -O0 we generate an explicit alloca for the BlockPointer, so the RA1531  // won't delete the dbg.declare intrinsics for captured variables.1532  llvm::Value *BlockPointerDbgLoc = BlockPointer;1533  if (CGM.getCodeGenOpts().OptimizationLevel == 0) {1534    // Allocate a stack slot for it, so we can point the debugger to it1535    Address Alloca = CreateTempAlloca(BlockPointer->getType(),1536                                      getPointerAlign(),1537                                      "block.addr");1538    // Set the DebugLocation to empty, so the store is recognized as a1539    // frame setup instruction by llvm::DwarfDebug::beginFunction().1540    auto NL = ApplyDebugLocation::CreateEmpty(*this);1541    Builder.CreateStore(BlockPointer, Alloca);1542    BlockPointerDbgLoc = Alloca.emitRawPointer(*this);1543  }1544 1545  // If we have a C++ 'this' reference, go ahead and force it into1546  // existence now.1547  if (blockDecl->capturesCXXThis()) {1548    Address addr = Builder.CreateStructGEP(1549        LoadBlockStruct(), blockInfo.CXXThisIndex, "block.captured-this");1550    CXXThisValue = Builder.CreateLoad(addr, "this");1551  }1552 1553  // Also force all the constant captures.1554  for (const auto &CI : blockDecl->captures()) {1555    const VarDecl *variable = CI.getVariable();1556    const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable);1557    if (!capture.isConstant()) continue;1558 1559    CharUnits align = getContext().getDeclAlign(variable);1560    Address alloca =1561      CreateMemTemp(variable->getType(), align, "block.captured-const");1562 1563    Builder.CreateStore(capture.getConstant(), alloca);1564 1565    setAddrOfLocalVar(variable, alloca);1566  }1567 1568  // Save a spot to insert the debug information for all the DeclRefExprs.1569  llvm::BasicBlock *entry = Builder.GetInsertBlock();1570  llvm::BasicBlock::iterator entry_ptr = Builder.GetInsertPoint();1571  --entry_ptr;1572 1573  if (IsLambdaConversionToBlock)1574    EmitLambdaBlockInvokeBody();1575  else {1576    PGO->assignRegionCounters(GlobalDecl(blockDecl), fn);1577    incrementProfileCounter(blockDecl->getBody());1578    EmitStmt(blockDecl->getBody());1579  }1580 1581  // Remember where we were...1582  llvm::BasicBlock *resume = Builder.GetInsertBlock();1583 1584  // Go back to the entry.1585  if (entry_ptr->getNextNode())1586    entry_ptr = entry_ptr->getNextNode()->getIterator();1587  else1588    entry_ptr = entry->end();1589  Builder.SetInsertPoint(entry, entry_ptr);1590 1591  // Emit debug information for all the DeclRefExprs.1592  // FIXME: also for 'this'1593  if (CGDebugInfo *DI = getDebugInfo()) {1594    for (const auto &CI : blockDecl->captures()) {1595      const VarDecl *variable = CI.getVariable();1596      DI->EmitLocation(Builder, variable->getLocation());1597 1598      if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {1599        const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable);1600        if (capture.isConstant()) {1601          auto addr = LocalDeclMap.find(variable)->second;1602          (void)DI->EmitDeclareOfAutoVariable(1603              variable, addr.emitRawPointer(*this), Builder);1604          continue;1605        }1606 1607        DI->EmitDeclareOfBlockDeclRefVariable(1608            variable, BlockPointerDbgLoc, Builder, blockInfo,1609            entry_ptr == entry->end() ? nullptr : &*entry_ptr);1610      }1611    }1612    // Recover location if it was changed in the above loop.1613    DI->EmitLocation(Builder,1614                     cast<CompoundStmt>(blockDecl->getBody())->getRBracLoc());1615  }1616 1617  // And resume where we left off.1618  if (resume == nullptr)1619    Builder.ClearInsertionPoint();1620  else1621    Builder.SetInsertPoint(resume);1622 1623  FinishFunction(cast<CompoundStmt>(blockDecl->getBody())->getRBracLoc());1624 1625  return fn;1626}1627 1628static std::pair<BlockCaptureEntityKind, BlockFieldFlags>1629computeCopyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,1630                               const LangOptions &LangOpts) {1631  if (CI.getCopyExpr()) {1632    assert(!CI.isByRef());1633    // don't bother computing flags1634    return std::make_pair(BlockCaptureEntityKind::CXXRecord, BlockFieldFlags());1635  }1636  BlockFieldFlags Flags;1637  if (CI.isEscapingByref()) {1638    Flags = BLOCK_FIELD_IS_BYREF;1639    if (T.isObjCGCWeak())1640      Flags |= BLOCK_FIELD_IS_WEAK;1641    return std::make_pair(BlockCaptureEntityKind::BlockObject, Flags);1642  }1643 1644  if (T.hasAddressDiscriminatedPointerAuth())1645    return std::make_pair(1646        BlockCaptureEntityKind::AddressDiscriminatedPointerAuth, Flags);1647 1648  Flags = BLOCK_FIELD_IS_OBJECT;1649  bool isBlockPointer = T->isBlockPointerType();1650  if (isBlockPointer)1651    Flags = BLOCK_FIELD_IS_BLOCK;1652 1653  switch (T.isNonTrivialToPrimitiveCopy()) {1654  case QualType::PCK_Struct:1655    return std::make_pair(BlockCaptureEntityKind::NonTrivialCStruct,1656                          BlockFieldFlags());1657  case QualType::PCK_ARCWeak:1658    // We need to register __weak direct captures with the runtime.1659    return std::make_pair(BlockCaptureEntityKind::ARCWeak, Flags);1660  case QualType::PCK_ARCStrong:1661    // We need to retain the copied value for __strong direct captures.1662    // If it's a block pointer, we have to copy the block and assign that to1663    // the destination pointer, so we might as well use _Block_object_assign.1664    // Otherwise we can avoid that.1665    return std::make_pair(!isBlockPointer ? BlockCaptureEntityKind::ARCStrong1666                                          : BlockCaptureEntityKind::BlockObject,1667                          Flags);1668  case QualType::PCK_PtrAuth:1669    return std::make_pair(1670        BlockCaptureEntityKind::AddressDiscriminatedPointerAuth,1671        BlockFieldFlags());1672  case QualType::PCK_Trivial:1673  case QualType::PCK_VolatileTrivial: {1674    if (!T->isObjCRetainableType())1675      // For all other types, the memcpy is fine.1676      return std::make_pair(BlockCaptureEntityKind::None, BlockFieldFlags());1677 1678    // Honor the inert __unsafe_unretained qualifier, which doesn't actually1679    // make it into the type system.1680    if (T->isObjCInertUnsafeUnretainedType())1681      return std::make_pair(BlockCaptureEntityKind::None, BlockFieldFlags());1682 1683    // Special rules for ARC captures:1684    Qualifiers QS = T.getQualifiers();1685 1686    // Non-ARC captures of retainable pointers are strong and1687    // therefore require a call to _Block_object_assign.1688    if (!QS.getObjCLifetime() && !LangOpts.ObjCAutoRefCount)1689      return std::make_pair(BlockCaptureEntityKind::BlockObject, Flags);1690 1691    // Otherwise the memcpy is fine.1692    return std::make_pair(BlockCaptureEntityKind::None, BlockFieldFlags());1693  }1694  }1695  llvm_unreachable("after exhaustive PrimitiveCopyKind switch");1696}1697 1698namespace {1699/// Release a __block variable.1700struct CallBlockRelease final : EHScopeStack::Cleanup {1701  Address Addr;1702  BlockFieldFlags FieldFlags;1703  bool LoadBlockVarAddr, CanThrow;1704 1705  CallBlockRelease(Address Addr, BlockFieldFlags Flags, bool LoadValue,1706                   bool CT)1707      : Addr(Addr), FieldFlags(Flags), LoadBlockVarAddr(LoadValue),1708        CanThrow(CT) {}1709 1710  void Emit(CodeGenFunction &CGF, Flags flags) override {1711    llvm::Value *BlockVarAddr;1712    if (LoadBlockVarAddr) {1713      BlockVarAddr = CGF.Builder.CreateLoad(Addr);1714    } else {1715      BlockVarAddr = Addr.emitRawPointer(CGF);1716    }1717 1718    CGF.BuildBlockRelease(BlockVarAddr, FieldFlags, CanThrow);1719  }1720};1721} // end anonymous namespace1722 1723/// Check if \p T is a C++ class that has a destructor that can throw.1724bool CodeGenFunction::cxxDestructorCanThrow(QualType T) {1725  if (const auto *RD = T->getAsCXXRecordDecl())1726    if (const CXXDestructorDecl *DD = RD->getDestructor())1727      return DD->getType()->castAs<FunctionProtoType>()->canThrow();1728  return false;1729}1730 1731// Return a string that has the information about a capture.1732static std::string getBlockCaptureStr(const CGBlockInfo::Capture &Cap,1733                                      CaptureStrKind StrKind,1734                                      CharUnits BlockAlignment,1735                                      CodeGenModule &CGM) {1736  std::string Str;1737  ASTContext &Ctx = CGM.getContext();1738  const BlockDecl::Capture &CI = *Cap.Cap;1739  QualType CaptureTy = CI.getVariable()->getType();1740 1741  BlockCaptureEntityKind Kind;1742  BlockFieldFlags Flags;1743 1744  // CaptureStrKind::Merged should be passed only when the operations and the1745  // flags are the same for copy and dispose.1746  assert((StrKind != CaptureStrKind::Merged ||1747          (Cap.CopyKind == Cap.DisposeKind &&1748           Cap.CopyFlags == Cap.DisposeFlags)) &&1749         "different operations and flags");1750 1751  if (StrKind == CaptureStrKind::DisposeHelper) {1752    Kind = Cap.DisposeKind;1753    Flags = Cap.DisposeFlags;1754  } else {1755    Kind = Cap.CopyKind;1756    Flags = Cap.CopyFlags;1757  }1758 1759  switch (Kind) {1760  case BlockCaptureEntityKind::CXXRecord: {1761    Str += "c";1762    SmallString<256> TyStr;1763    llvm::raw_svector_ostream Out(TyStr);1764    CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(CaptureTy, Out);1765    Str += llvm::to_string(TyStr.size()) + TyStr.c_str();1766    break;1767  }1768  case BlockCaptureEntityKind::ARCWeak:1769    Str += "w";1770    break;1771  case BlockCaptureEntityKind::ARCStrong:1772    Str += "s";1773    break;1774  case BlockCaptureEntityKind::AddressDiscriminatedPointerAuth: {1775    auto PtrAuth = CaptureTy.getPointerAuth();1776    assert(PtrAuth && PtrAuth.isAddressDiscriminated());1777    Str += "p" + llvm::to_string(PtrAuth.getKey()) + "d" +1778           llvm::to_string(PtrAuth.getExtraDiscriminator());1779    break;1780  }1781  case BlockCaptureEntityKind::BlockObject: {1782    const VarDecl *Var = CI.getVariable();1783    unsigned F = Flags.getBitMask();1784    if (F & BLOCK_FIELD_IS_BYREF) {1785      Str += "r";1786      if (F & BLOCK_FIELD_IS_WEAK)1787        Str += "w";1788      else {1789        // If CaptureStrKind::Merged is passed, check both the copy expression1790        // and the destructor.1791        if (StrKind != CaptureStrKind::DisposeHelper) {1792          if (Ctx.getBlockVarCopyInit(Var).canThrow())1793            Str += "c";1794        }1795        if (StrKind != CaptureStrKind::CopyHelper) {1796          if (CodeGenFunction::cxxDestructorCanThrow(CaptureTy))1797            Str += "d";1798        }1799      }1800    } else {1801      assert((F & BLOCK_FIELD_IS_OBJECT) && "unexpected flag value");1802      if (F == BLOCK_FIELD_IS_BLOCK)1803        Str += "b";1804      else1805        Str += "o";1806    }1807    break;1808  }1809  case BlockCaptureEntityKind::NonTrivialCStruct: {1810    bool IsVolatile = CaptureTy.isVolatileQualified();1811    CharUnits Alignment = BlockAlignment.alignmentAtOffset(Cap.getOffset());1812 1813    Str += "n";1814    std::string FuncStr;1815    if (StrKind == CaptureStrKind::DisposeHelper)1816      FuncStr = CodeGenFunction::getNonTrivialDestructorStr(1817          CaptureTy, Alignment, IsVolatile, Ctx);1818    else1819      // If CaptureStrKind::Merged is passed, use the copy constructor string.1820      // It has all the information that the destructor string has.1821      FuncStr = CodeGenFunction::getNonTrivialCopyConstructorStr(1822          CaptureTy, Alignment, IsVolatile, Ctx);1823    // The underscore is necessary here because non-trivial copy constructor1824    // and destructor strings can start with a number.1825    Str += llvm::to_string(FuncStr.size()) + "_" + FuncStr;1826    break;1827  }1828  case BlockCaptureEntityKind::None:1829    break;1830  }1831 1832  return Str;1833}1834 1835static std::string getCopyDestroyHelperFuncName(1836    const SmallVectorImpl<CGBlockInfo::Capture> &Captures,1837    CharUnits BlockAlignment, CaptureStrKind StrKind, CodeGenModule &CGM) {1838  assert((StrKind == CaptureStrKind::CopyHelper ||1839          StrKind == CaptureStrKind::DisposeHelper) &&1840         "unexpected CaptureStrKind");1841  std::string Name = StrKind == CaptureStrKind::CopyHelper1842                         ? "__copy_helper_block_"1843                         : "__destroy_helper_block_";1844  if (CGM.getLangOpts().Exceptions)1845    Name += "e";1846  if (CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)1847    Name += "a";1848  Name += llvm::to_string(BlockAlignment.getQuantity()) + "_";1849 1850  for (auto &Cap : Captures) {1851    if (Cap.isConstantOrTrivial())1852      continue;1853    Name += llvm::to_string(Cap.getOffset().getQuantity());1854    Name += getBlockCaptureStr(Cap, StrKind, BlockAlignment, CGM);1855  }1856 1857  return Name;1858}1859 1860static void pushCaptureCleanup(BlockCaptureEntityKind CaptureKind,1861                               Address Field, QualType CaptureType,1862                               BlockFieldFlags Flags, bool ForCopyHelper,1863                               VarDecl *Var, CodeGenFunction &CGF) {1864  bool EHOnly = ForCopyHelper;1865 1866  switch (CaptureKind) {1867  case BlockCaptureEntityKind::CXXRecord:1868  case BlockCaptureEntityKind::ARCWeak:1869  case BlockCaptureEntityKind::NonTrivialCStruct:1870  case BlockCaptureEntityKind::ARCStrong: {1871    if (CaptureType.isDestructedType() &&1872        (!EHOnly || CGF.needsEHCleanup(CaptureType.isDestructedType()))) {1873      CodeGenFunction::Destroyer *Destroyer =1874          CaptureKind == BlockCaptureEntityKind::ARCStrong1875              ? CodeGenFunction::destroyARCStrongImprecise1876              : CGF.getDestroyer(CaptureType.isDestructedType());1877      CleanupKind Kind =1878          EHOnly ? EHCleanup1879                 : CGF.getCleanupKind(CaptureType.isDestructedType());1880      CGF.pushDestroy(Kind, Field, CaptureType, Destroyer, Kind & EHCleanup);1881    }1882    break;1883  }1884  case BlockCaptureEntityKind::BlockObject: {1885    if (!EHOnly || CGF.getLangOpts().Exceptions) {1886      CleanupKind Kind = EHOnly ? EHCleanup : NormalAndEHCleanup;1887      // Calls to _Block_object_dispose along the EH path in the copy helper1888      // function don't throw as newly-copied __block variables always have a1889      // reference count of 2.1890      bool CanThrow =1891          !ForCopyHelper && CGF.cxxDestructorCanThrow(CaptureType);1892      CGF.enterByrefCleanup(Kind, Field, Flags, /*LoadBlockVarAddr*/ true,1893                            CanThrow);1894    }1895    break;1896  }1897  case BlockCaptureEntityKind::AddressDiscriminatedPointerAuth:1898  case BlockCaptureEntityKind::None:1899    break;1900  }1901}1902 1903static void setBlockHelperAttributesVisibility(bool CapturesNonExternalType,1904                                               llvm::Function *Fn,1905                                               const CGFunctionInfo &FI,1906                                               CodeGenModule &CGM) {1907  if (CapturesNonExternalType) {1908    CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FI);1909  } else {1910    Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);1911    Fn->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);1912    CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, Fn, /*IsThunk=*/false);1913    CGM.SetLLVMFunctionAttributesForDefinition(nullptr, Fn);1914  }1915}1916/// Generate the copy-helper function for a block closure object:1917///   static void block_copy_helper(block_t *dst, block_t *src);1918/// The runtime will have previously initialized 'dst' by doing a1919/// bit-copy of 'src'.1920///1921/// Note that this copies an entire block closure object to the heap;1922/// it should not be confused with a 'byref copy helper', which moves1923/// the contents of an individual __block variable to the heap.1924llvm::Constant *1925CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {1926  std::string FuncName = getCopyDestroyHelperFuncName(1927      blockInfo.SortedCaptures, blockInfo.BlockAlign,1928      CaptureStrKind::CopyHelper, CGM);1929 1930  if (llvm::GlobalValue *Func = CGM.getModule().getNamedValue(FuncName))1931    return Func;1932 1933  ASTContext &C = getContext();1934 1935  QualType ReturnTy = C.VoidTy;1936 1937  FunctionArgList args;1938  ImplicitParamDecl DstDecl(C, C.VoidPtrTy, ImplicitParamKind::Other);1939  args.push_back(&DstDecl);1940  ImplicitParamDecl SrcDecl(C, C.VoidPtrTy, ImplicitParamKind::Other);1941  args.push_back(&SrcDecl);1942 1943  const CGFunctionInfo &FI =1944      CGM.getTypes().arrangeBuiltinFunctionDeclaration(ReturnTy, args);1945 1946  // FIXME: it would be nice if these were mergeable with things with1947  // identical semantics.1948  llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(FI);1949 1950  llvm::Function *Fn =1951    llvm::Function::Create(LTy, llvm::GlobalValue::LinkOnceODRLinkage,1952                           FuncName, &CGM.getModule());1953  if (CGM.supportsCOMDAT())1954    Fn->setComdat(CGM.getModule().getOrInsertComdat(FuncName));1955 1956  SmallVector<QualType, 2> ArgTys;1957  ArgTys.push_back(C.VoidPtrTy);1958  ArgTys.push_back(C.VoidPtrTy);1959 1960  setBlockHelperAttributesVisibility(blockInfo.CapturesNonExternalType, Fn, FI,1961                                     CGM);1962  StartFunction(GlobalDecl(), ReturnTy, Fn, FI, args);1963  auto AL = ApplyDebugLocation::CreateArtificial(*this);1964 1965  Address src = GetAddrOfLocalVar(&SrcDecl);1966  src = Address(Builder.CreateLoad(src), blockInfo.StructureType,1967                blockInfo.BlockAlign);1968 1969  Address dst = GetAddrOfLocalVar(&DstDecl);1970  dst = Address(Builder.CreateLoad(dst), blockInfo.StructureType,1971                blockInfo.BlockAlign);1972 1973  for (auto &capture : blockInfo.SortedCaptures) {1974    if (capture.isConstantOrTrivial())1975      continue;1976 1977    const BlockDecl::Capture &CI = *capture.Cap;1978    QualType captureType = CI.getVariable()->getType();1979    BlockFieldFlags flags = capture.CopyFlags;1980 1981    unsigned index = capture.getIndex();1982    Address srcField = Builder.CreateStructGEP(src, index);1983    Address dstField = Builder.CreateStructGEP(dst, index);1984 1985    switch (capture.CopyKind) {1986    case BlockCaptureEntityKind::CXXRecord:1987      // If there's an explicit copy expression, we do that.1988      assert(CI.getCopyExpr() && "copy expression for variable is missing");1989      EmitSynthesizedCXXCopyCtor(dstField, srcField, CI.getCopyExpr());1990      break;1991    case BlockCaptureEntityKind::ARCWeak:1992      EmitARCCopyWeak(dstField, srcField);1993      break;1994    case BlockCaptureEntityKind::AddressDiscriminatedPointerAuth: {1995      QualType Type = CI.getVariable()->getType();1996      PointerAuthQualifier PointerAuth = Type.getPointerAuth();1997      assert(PointerAuth && PointerAuth.isAddressDiscriminated());1998      EmitPointerAuthCopy(PointerAuth, Type, dstField, srcField);1999      // We don't need to push cleanups for ptrauth types.2000      continue;2001    }2002    case BlockCaptureEntityKind::NonTrivialCStruct: {2003      // If this is a C struct that requires non-trivial copy construction,2004      // emit a call to its copy constructor.2005      QualType varType = CI.getVariable()->getType();2006      callCStructCopyConstructor(MakeAddrLValue(dstField, varType),2007                                 MakeAddrLValue(srcField, varType));2008      break;2009    }2010    case BlockCaptureEntityKind::ARCStrong: {2011      llvm::Value *srcValue = Builder.CreateLoad(srcField, "blockcopy.src");2012      // At -O0, store null into the destination field (so that the2013      // storeStrong doesn't over-release) and then call storeStrong.2014      // This is a workaround to not having an initStrong call.2015      if (CGM.getCodeGenOpts().OptimizationLevel == 0) {2016        auto *ty = cast<llvm::PointerType>(srcValue->getType());2017        llvm::Value *null = llvm::ConstantPointerNull::get(ty);2018        Builder.CreateStore(null, dstField);2019        EmitARCStoreStrongCall(dstField, srcValue, true);2020 2021      // With optimization enabled, take advantage of the fact that2022      // the blocks runtime guarantees a memcpy of the block data, and2023      // just emit a retain of the src field.2024      } else {2025        EmitARCRetainNonBlock(srcValue);2026 2027        // Unless EH cleanup is required, we don't need this anymore, so kill2028        // it. It's not quite worth the annoyance to avoid creating it in the2029        // first place.2030        if (!needsEHCleanup(captureType.isDestructedType()))2031          if (auto *I = cast_or_null<llvm::Instruction>(2032                  dstField.getPointerIfNotSigned()))2033            I->eraseFromParent();2034      }2035      break;2036    }2037    case BlockCaptureEntityKind::BlockObject: {2038      llvm::Value *srcValue = Builder.CreateLoad(srcField, "blockcopy.src");2039      llvm::Value *dstAddr = dstField.emitRawPointer(*this);2040      llvm::Value *args[] = {2041        dstAddr, srcValue, llvm::ConstantInt::get(Int32Ty, flags.getBitMask())2042      };2043 2044      if (CI.isByRef() && C.getBlockVarCopyInit(CI.getVariable()).canThrow())2045        EmitRuntimeCallOrInvoke(CGM.getBlockObjectAssign(), args);2046      else2047        EmitNounwindRuntimeCall(CGM.getBlockObjectAssign(), args);2048      break;2049    }2050    case BlockCaptureEntityKind::None:2051      continue;2052    }2053 2054    // Ensure that we destroy the copied object if an exception is thrown later2055    // in the helper function.2056    pushCaptureCleanup(capture.CopyKind, dstField, captureType, flags,2057                       /*ForCopyHelper*/ true, CI.getVariable(), *this);2058  }2059 2060  FinishFunction();2061 2062  return Fn;2063}2064 2065static BlockFieldFlags2066getBlockFieldFlagsForObjCObjectPointer(const BlockDecl::Capture &CI,2067                                       QualType T) {2068  BlockFieldFlags Flags = BLOCK_FIELD_IS_OBJECT;2069  if (T->isBlockPointerType())2070    Flags = BLOCK_FIELD_IS_BLOCK;2071  return Flags;2072}2073 2074static std::pair<BlockCaptureEntityKind, BlockFieldFlags>2075computeDestroyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,2076                                  const LangOptions &LangOpts) {2077  if (CI.isEscapingByref()) {2078    BlockFieldFlags Flags = BLOCK_FIELD_IS_BYREF;2079    if (T.isObjCGCWeak())2080      Flags |= BLOCK_FIELD_IS_WEAK;2081    return std::make_pair(BlockCaptureEntityKind::BlockObject, Flags);2082  }2083 2084  switch (T.isDestructedType()) {2085  case QualType::DK_cxx_destructor:2086    return std::make_pair(BlockCaptureEntityKind::CXXRecord, BlockFieldFlags());2087  case QualType::DK_objc_strong_lifetime:2088    // Use objc_storeStrong for __strong direct captures; the2089    // dynamic tools really like it when we do this.2090    return std::make_pair(BlockCaptureEntityKind::ARCStrong,2091                          getBlockFieldFlagsForObjCObjectPointer(CI, T));2092  case QualType::DK_objc_weak_lifetime:2093    // Support __weak direct captures.2094    return std::make_pair(BlockCaptureEntityKind::ARCWeak,2095                          getBlockFieldFlagsForObjCObjectPointer(CI, T));2096  case QualType::DK_nontrivial_c_struct:2097    return std::make_pair(BlockCaptureEntityKind::NonTrivialCStruct,2098                          BlockFieldFlags());2099  case QualType::DK_none: {2100    // Non-ARC captures are strong, and we need to use _Block_object_dispose.2101    // But honor the inert __unsafe_unretained qualifier, which doesn't actually2102    // make it into the type system.2103    if (T->isObjCRetainableType() && !T.getQualifiers().hasObjCLifetime() &&2104        !LangOpts.ObjCAutoRefCount && !T->isObjCInertUnsafeUnretainedType())2105      return std::make_pair(BlockCaptureEntityKind::BlockObject,2106                            getBlockFieldFlagsForObjCObjectPointer(CI, T));2107    // Otherwise, we have nothing to do.2108    return std::make_pair(BlockCaptureEntityKind::None, BlockFieldFlags());2109  }2110  }2111  llvm_unreachable("after exhaustive DestructionKind switch");2112}2113 2114/// Generate the destroy-helper function for a block closure object:2115///   static void block_destroy_helper(block_t *theBlock);2116///2117/// Note that this destroys a heap-allocated block closure object;2118/// it should not be confused with a 'byref destroy helper', which2119/// destroys the heap-allocated contents of an individual __block2120/// variable.2121llvm::Constant *2122CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) {2123  std::string FuncName = getCopyDestroyHelperFuncName(2124      blockInfo.SortedCaptures, blockInfo.BlockAlign,2125      CaptureStrKind::DisposeHelper, CGM);2126 2127  if (llvm::GlobalValue *Func = CGM.getModule().getNamedValue(FuncName))2128    return Func;2129 2130  ASTContext &C = getContext();2131 2132  QualType ReturnTy = C.VoidTy;2133 2134  FunctionArgList args;2135  ImplicitParamDecl SrcDecl(C, C.VoidPtrTy, ImplicitParamKind::Other);2136  args.push_back(&SrcDecl);2137 2138  const CGFunctionInfo &FI =2139      CGM.getTypes().arrangeBuiltinFunctionDeclaration(ReturnTy, args);2140 2141  // FIXME: We'd like to put these into a mergable by content, with2142  // internal linkage.2143  llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(FI);2144 2145  llvm::Function *Fn =2146    llvm::Function::Create(LTy, llvm::GlobalValue::LinkOnceODRLinkage,2147                           FuncName, &CGM.getModule());2148  if (CGM.supportsCOMDAT())2149    Fn->setComdat(CGM.getModule().getOrInsertComdat(FuncName));2150 2151  SmallVector<QualType, 1> ArgTys;2152  ArgTys.push_back(C.VoidPtrTy);2153 2154  setBlockHelperAttributesVisibility(blockInfo.CapturesNonExternalType, Fn, FI,2155                                     CGM);2156  StartFunction(GlobalDecl(), ReturnTy, Fn, FI, args);2157  markAsIgnoreThreadCheckingAtRuntime(Fn);2158 2159  auto AL = ApplyDebugLocation::CreateArtificial(*this);2160 2161  Address src = GetAddrOfLocalVar(&SrcDecl);2162  src = Address(Builder.CreateLoad(src), blockInfo.StructureType,2163                blockInfo.BlockAlign);2164 2165  CodeGenFunction::RunCleanupsScope cleanups(*this);2166 2167  for (auto &capture : blockInfo.SortedCaptures) {2168    if (capture.isConstantOrTrivial())2169      continue;2170 2171    const BlockDecl::Capture &CI = *capture.Cap;2172    BlockFieldFlags flags = capture.DisposeFlags;2173 2174    Address srcField = Builder.CreateStructGEP(src, capture.getIndex());2175 2176    pushCaptureCleanup(capture.DisposeKind, srcField,2177                       CI.getVariable()->getType(), flags,2178                       /*ForCopyHelper*/ false, CI.getVariable(), *this);2179  }2180 2181  cleanups.ForceCleanup();2182 2183  FinishFunction();2184 2185  return Fn;2186}2187 2188namespace {2189 2190/// Emits the copy/dispose helper functions for a __block object of id type.2191class ObjectByrefHelpers final : public BlockByrefHelpers {2192  BlockFieldFlags Flags;2193 2194public:2195  ObjectByrefHelpers(CharUnits alignment, BlockFieldFlags flags)2196    : BlockByrefHelpers(alignment), Flags(flags) {}2197 2198  void emitCopy(CodeGenFunction &CGF, Address destField,2199                Address srcField) override {2200    destField = destField.withElementType(CGF.Int8Ty);2201 2202    srcField = srcField.withElementType(CGF.Int8PtrTy);2203    llvm::Value *srcValue = CGF.Builder.CreateLoad(srcField);2204 2205    unsigned flags = (Flags | BLOCK_BYREF_CALLER).getBitMask();2206 2207    llvm::Value *flagsVal = llvm::ConstantInt::get(CGF.Int32Ty, flags);2208    llvm::FunctionCallee fn = CGF.CGM.getBlockObjectAssign();2209 2210    llvm::Value *args[] = {destField.emitRawPointer(CGF), srcValue, flagsVal};2211    CGF.EmitNounwindRuntimeCall(fn, args);2212  }2213 2214  void emitDispose(CodeGenFunction &CGF, Address field) override {2215    field = field.withElementType(CGF.Int8PtrTy);2216    llvm::Value *value = CGF.Builder.CreateLoad(field);2217 2218    CGF.BuildBlockRelease(value, Flags | BLOCK_BYREF_CALLER, false);2219  }2220 2221  void profileImpl(llvm::FoldingSetNodeID &id) const override {2222    id.AddInteger(Flags.getBitMask());2223  }2224};2225 2226/// Emits the copy/dispose helpers for an ARC __block __weak variable.2227class ARCWeakByrefHelpers final : public BlockByrefHelpers {2228public:2229  ARCWeakByrefHelpers(CharUnits alignment) : BlockByrefHelpers(alignment) {}2230 2231  void emitCopy(CodeGenFunction &CGF, Address destField,2232                Address srcField) override {2233    CGF.EmitARCMoveWeak(destField, srcField);2234  }2235 2236  void emitDispose(CodeGenFunction &CGF, Address field) override {2237    CGF.EmitARCDestroyWeak(field);2238  }2239 2240  void profileImpl(llvm::FoldingSetNodeID &id) const override {2241    // 0 is distinguishable from all pointers and byref flags2242    id.AddInteger(0);2243  }2244};2245 2246/// Emits the copy/dispose helpers for an ARC __block __strong variable2247/// that's not of block-pointer type.2248class ARCStrongByrefHelpers final : public BlockByrefHelpers {2249public:2250  ARCStrongByrefHelpers(CharUnits alignment) : BlockByrefHelpers(alignment) {}2251 2252  void emitCopy(CodeGenFunction &CGF, Address destField,2253                Address srcField) override {2254    // Do a "move" by copying the value and then zeroing out the old2255    // variable.2256 2257    llvm::Value *value = CGF.Builder.CreateLoad(srcField);2258 2259    llvm::Value *null =2260      llvm::ConstantPointerNull::get(cast<llvm::PointerType>(value->getType()));2261 2262    if (CGF.CGM.getCodeGenOpts().OptimizationLevel == 0) {2263      CGF.Builder.CreateStore(null, destField);2264      CGF.EmitARCStoreStrongCall(destField, value, /*ignored*/ true);2265      CGF.EmitARCStoreStrongCall(srcField, null, /*ignored*/ true);2266      return;2267    }2268    CGF.Builder.CreateStore(value, destField);2269    CGF.Builder.CreateStore(null, srcField);2270  }2271 2272  void emitDispose(CodeGenFunction &CGF, Address field) override {2273    CGF.EmitARCDestroyStrong(field, ARCImpreciseLifetime);2274  }2275 2276  void profileImpl(llvm::FoldingSetNodeID &id) const override {2277    // 1 is distinguishable from all pointers and byref flags2278    id.AddInteger(1);2279  }2280};2281 2282/// Emits the copy/dispose helpers for an ARC __block __strong2283/// variable that's of block-pointer type.2284class ARCStrongBlockByrefHelpers final : public BlockByrefHelpers {2285public:2286  ARCStrongBlockByrefHelpers(CharUnits alignment)2287    : BlockByrefHelpers(alignment) {}2288 2289  void emitCopy(CodeGenFunction &CGF, Address destField,2290                Address srcField) override {2291    // Do the copy with objc_retainBlock; that's all that2292    // _Block_object_assign would do anyway, and we'd have to pass the2293    // right arguments to make sure it doesn't get no-op'ed.2294    llvm::Value *oldValue = CGF.Builder.CreateLoad(srcField);2295    llvm::Value *copy = CGF.EmitARCRetainBlock(oldValue, /*mandatory*/ true);2296    CGF.Builder.CreateStore(copy, destField);2297  }2298 2299  void emitDispose(CodeGenFunction &CGF, Address field) override {2300    CGF.EmitARCDestroyStrong(field, ARCImpreciseLifetime);2301  }2302 2303  void profileImpl(llvm::FoldingSetNodeID &id) const override {2304    // 2 is distinguishable from all pointers and byref flags2305    id.AddInteger(2);2306  }2307};2308 2309/// Emits the copy/dispose helpers for a __block variable with a2310/// nontrivial copy constructor or destructor.2311class CXXByrefHelpers final : public BlockByrefHelpers {2312  QualType VarType;2313  const Expr *CopyExpr;2314 2315public:2316  CXXByrefHelpers(CharUnits alignment, QualType type,2317                  const Expr *copyExpr)2318    : BlockByrefHelpers(alignment), VarType(type), CopyExpr(copyExpr) {}2319 2320  bool needsCopy() const override { return CopyExpr != nullptr; }2321  void emitCopy(CodeGenFunction &CGF, Address destField,2322                Address srcField) override {2323    if (!CopyExpr) return;2324    CGF.EmitSynthesizedCXXCopyCtor(destField, srcField, CopyExpr);2325  }2326 2327  void emitDispose(CodeGenFunction &CGF, Address field) override {2328    EHScopeStack::stable_iterator cleanupDepth = CGF.EHStack.stable_begin();2329    CGF.PushDestructorCleanup(VarType, field);2330    CGF.PopCleanupBlocks(cleanupDepth);2331  }2332 2333  void profileImpl(llvm::FoldingSetNodeID &id) const override {2334    id.AddPointer(VarType.getCanonicalType().getAsOpaquePtr());2335  }2336};2337 2338/// Emits the copy/dispose helpers for a __block variable with2339/// address-discriminated pointer authentication.2340class AddressDiscriminatedByrefHelpers final : public BlockByrefHelpers {2341  QualType VarType;2342 2343public:2344  AddressDiscriminatedByrefHelpers(CharUnits Alignment, QualType Type)2345      : BlockByrefHelpers(Alignment), VarType(Type) {2346    assert(Type.hasAddressDiscriminatedPointerAuth());2347  }2348 2349  void emitCopy(CodeGenFunction &CGF, Address DestField,2350                Address SrcField) override {2351    CGF.EmitPointerAuthCopy(VarType.getPointerAuth(), VarType, DestField,2352                            SrcField);2353  }2354 2355  bool needsDispose() const override { return false; }2356  void emitDispose(CodeGenFunction &CGF, Address Field) override {2357    llvm_unreachable("should never be called");2358  }2359 2360  void profileImpl(llvm::FoldingSetNodeID &ID) const override {2361    ID.AddPointer(VarType.getCanonicalType().getAsOpaquePtr());2362  }2363};2364 2365/// Emits the copy/dispose helpers for a __block variable that is a non-trivial2366/// C struct.2367class NonTrivialCStructByrefHelpers final : public BlockByrefHelpers {2368  QualType VarType;2369 2370public:2371  NonTrivialCStructByrefHelpers(CharUnits alignment, QualType type)2372    : BlockByrefHelpers(alignment), VarType(type) {}2373 2374  void emitCopy(CodeGenFunction &CGF, Address destField,2375                Address srcField) override {2376    CGF.callCStructMoveConstructor(CGF.MakeAddrLValue(destField, VarType),2377                                   CGF.MakeAddrLValue(srcField, VarType));2378  }2379 2380  bool needsDispose() const override {2381    return VarType.isDestructedType();2382  }2383 2384  void emitDispose(CodeGenFunction &CGF, Address field) override {2385    EHScopeStack::stable_iterator cleanupDepth = CGF.EHStack.stable_begin();2386    CGF.pushDestroy(VarType.isDestructedType(), field, VarType);2387    CGF.PopCleanupBlocks(cleanupDepth);2388  }2389 2390  void profileImpl(llvm::FoldingSetNodeID &id) const override {2391    id.AddPointer(VarType.getCanonicalType().getAsOpaquePtr());2392  }2393};2394} // end anonymous namespace2395 2396static llvm::Constant *2397generateByrefCopyHelper(CodeGenFunction &CGF, const BlockByrefInfo &byrefInfo,2398                        BlockByrefHelpers &generator) {2399  ASTContext &Context = CGF.getContext();2400 2401  QualType ReturnTy = Context.VoidTy;2402 2403  FunctionArgList args;2404  ImplicitParamDecl Dst(Context, Context.VoidPtrTy, ImplicitParamKind::Other);2405  args.push_back(&Dst);2406 2407  ImplicitParamDecl Src(Context, Context.VoidPtrTy, ImplicitParamKind::Other);2408  args.push_back(&Src);2409 2410  const CGFunctionInfo &FI =2411      CGF.CGM.getTypes().arrangeBuiltinFunctionDeclaration(ReturnTy, args);2412 2413  llvm::FunctionType *LTy = CGF.CGM.getTypes().GetFunctionType(FI);2414 2415  // FIXME: We'd like to put these into a mergable by content, with2416  // internal linkage.2417  llvm::Function *Fn =2418    llvm::Function::Create(LTy, llvm::GlobalValue::InternalLinkage,2419                           "__Block_byref_object_copy_", &CGF.CGM.getModule());2420 2421  SmallVector<QualType, 2> ArgTys;2422  ArgTys.push_back(Context.VoidPtrTy);2423  ArgTys.push_back(Context.VoidPtrTy);2424 2425  CGF.CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FI);2426 2427  CGF.StartFunction(GlobalDecl(), ReturnTy, Fn, FI, args);2428    // Create a scope with an artificial location for the body of this function.2429  auto AL = ApplyDebugLocation::CreateArtificial(CGF);2430 2431  if (generator.needsCopy()) {2432    // dst->x2433    Address destField = CGF.GetAddrOfLocalVar(&Dst);2434    destField = Address(CGF.Builder.CreateLoad(destField), byrefInfo.Type,2435                        byrefInfo.ByrefAlignment);2436    destField =2437        CGF.emitBlockByrefAddress(destField, byrefInfo, false, "dest-object");2438 2439    // src->x2440    Address srcField = CGF.GetAddrOfLocalVar(&Src);2441    srcField = Address(CGF.Builder.CreateLoad(srcField), byrefInfo.Type,2442                       byrefInfo.ByrefAlignment);2443    srcField =2444        CGF.emitBlockByrefAddress(srcField, byrefInfo, false, "src-object");2445 2446    generator.emitCopy(CGF, destField, srcField);2447  }2448 2449  CGF.FinishFunction();2450 2451  return Fn;2452}2453 2454/// Build the copy helper for a __block variable.2455static llvm::Constant *buildByrefCopyHelper(CodeGenModule &CGM,2456                                            const BlockByrefInfo &byrefInfo,2457                                            BlockByrefHelpers &generator) {2458  CodeGenFunction CGF(CGM);2459  return generateByrefCopyHelper(CGF, byrefInfo, generator);2460}2461 2462/// Generate code for a __block variable's dispose helper.2463static llvm::Constant *2464generateByrefDisposeHelper(CodeGenFunction &CGF,2465                           const BlockByrefInfo &byrefInfo,2466                           BlockByrefHelpers &generator) {2467  ASTContext &Context = CGF.getContext();2468  QualType R = Context.VoidTy;2469 2470  FunctionArgList args;2471  ImplicitParamDecl Src(CGF.getContext(), Context.VoidPtrTy,2472                        ImplicitParamKind::Other);2473  args.push_back(&Src);2474 2475  const CGFunctionInfo &FI =2476    CGF.CGM.getTypes().arrangeBuiltinFunctionDeclaration(R, args);2477 2478  llvm::FunctionType *LTy = CGF.CGM.getTypes().GetFunctionType(FI);2479 2480  // FIXME: We'd like to put these into a mergable by content, with2481  // internal linkage.2482  llvm::Function *Fn =2483    llvm::Function::Create(LTy, llvm::GlobalValue::InternalLinkage,2484                           "__Block_byref_object_dispose_",2485                           &CGF.CGM.getModule());2486 2487  SmallVector<QualType, 1> ArgTys;2488  ArgTys.push_back(Context.VoidPtrTy);2489 2490  CGF.CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FI);2491 2492  CGF.StartFunction(GlobalDecl(), R, Fn, FI, args);2493    // Create a scope with an artificial location for the body of this function.2494  auto AL = ApplyDebugLocation::CreateArtificial(CGF);2495 2496  if (generator.needsDispose()) {2497    Address addr = CGF.GetAddrOfLocalVar(&Src);2498    addr = Address(CGF.Builder.CreateLoad(addr), byrefInfo.Type,2499                   byrefInfo.ByrefAlignment);2500    addr = CGF.emitBlockByrefAddress(addr, byrefInfo, false, "object");2501 2502    generator.emitDispose(CGF, addr);2503  }2504 2505  CGF.FinishFunction();2506 2507  return Fn;2508}2509 2510/// Build the dispose helper for a __block variable.2511static llvm::Constant *buildByrefDisposeHelper(CodeGenModule &CGM,2512                                               const BlockByrefInfo &byrefInfo,2513                                               BlockByrefHelpers &generator) {2514  CodeGenFunction CGF(CGM);2515  return generateByrefDisposeHelper(CGF, byrefInfo, generator);2516}2517 2518/// Lazily build the copy and dispose helpers for a __block variable2519/// with the given information.2520template <class T>2521static T *buildByrefHelpers(CodeGenModule &CGM, const BlockByrefInfo &byrefInfo,2522                            T &&generator) {2523  llvm::FoldingSetNodeID id;2524  generator.Profile(id);2525 2526  void *insertPos;2527  BlockByrefHelpers *node2528    = CGM.ByrefHelpersCache.FindNodeOrInsertPos(id, insertPos);2529  if (node) return static_cast<T*>(node);2530 2531  generator.CopyHelper = buildByrefCopyHelper(CGM, byrefInfo, generator);2532  generator.DisposeHelper = buildByrefDisposeHelper(CGM, byrefInfo, generator);2533 2534  T *copy = new (CGM.getContext()) T(std::forward<T>(generator));2535  CGM.ByrefHelpersCache.InsertNode(copy, insertPos);2536  return copy;2537}2538 2539/// Build the copy and dispose helpers for the given __block variable2540/// emission.  Places the helpers in the global cache.  Returns null2541/// if no helpers are required.2542BlockByrefHelpers *2543CodeGenFunction::buildByrefHelpers(llvm::StructType &byrefType,2544                                   const AutoVarEmission &emission) {2545  const VarDecl &var = *emission.Variable;2546  assert(var.isEscapingByref() &&2547         "only escaping __block variables need byref helpers");2548 2549  QualType type = var.getType();2550 2551  auto &byrefInfo = getBlockByrefInfo(&var);2552 2553  // The alignment we care about for the purposes of uniquing byref2554  // helpers is the alignment of the actual byref value field.2555  CharUnits valueAlignment =2556    byrefInfo.ByrefAlignment.alignmentAtOffset(byrefInfo.FieldOffset);2557 2558  if (const CXXRecordDecl *record = type->getAsCXXRecordDecl()) {2559    const Expr *copyExpr =2560        CGM.getContext().getBlockVarCopyInit(&var).getCopyExpr();2561    if (!copyExpr && record->hasTrivialDestructor()) return nullptr;2562 2563    return ::buildByrefHelpers(2564        CGM, byrefInfo, CXXByrefHelpers(valueAlignment, type, copyExpr));2565  }2566  if (type.hasAddressDiscriminatedPointerAuth()) {2567    return ::buildByrefHelpers(2568        CGM, byrefInfo, AddressDiscriminatedByrefHelpers(valueAlignment, type));2569  }2570  // If type is a non-trivial C struct type that is non-trivial to2571  // destructly move or destroy, build the copy and dispose helpers.2572  if (type.isNonTrivialToPrimitiveDestructiveMove() == QualType::PCK_Struct ||2573      type.isDestructedType() == QualType::DK_nontrivial_c_struct)2574    return ::buildByrefHelpers(2575        CGM, byrefInfo, NonTrivialCStructByrefHelpers(valueAlignment, type));2576 2577  // Otherwise, if we don't have a retainable type, there's nothing to do.2578  // that the runtime does extra copies.2579  if (!type->isObjCRetainableType()) return nullptr;2580 2581  Qualifiers qs = type.getQualifiers();2582 2583  // If we have lifetime, that dominates.2584  if (Qualifiers::ObjCLifetime lifetime = qs.getObjCLifetime()) {2585    switch (lifetime) {2586    case Qualifiers::OCL_None: llvm_unreachable("impossible");2587 2588    // These are just bits as far as the runtime is concerned.2589    case Qualifiers::OCL_ExplicitNone:2590    case Qualifiers::OCL_Autoreleasing:2591      return nullptr;2592 2593    // Tell the runtime that this is ARC __weak, called by the2594    // byref routines.2595    case Qualifiers::OCL_Weak:2596      return ::buildByrefHelpers(CGM, byrefInfo,2597                                 ARCWeakByrefHelpers(valueAlignment));2598 2599    // ARC __strong __block variables need to be retained.2600    case Qualifiers::OCL_Strong:2601      // Block pointers need to be copied, and there's no direct2602      // transfer possible.2603      if (type->isBlockPointerType()) {2604        return ::buildByrefHelpers(CGM, byrefInfo,2605                                   ARCStrongBlockByrefHelpers(valueAlignment));2606 2607      // Otherwise, we transfer ownership of the retain from the stack2608      // to the heap.2609      } else {2610        return ::buildByrefHelpers(CGM, byrefInfo,2611                                   ARCStrongByrefHelpers(valueAlignment));2612      }2613    }2614    llvm_unreachable("fell out of lifetime switch!");2615  }2616 2617  BlockFieldFlags flags;2618  if (type->isBlockPointerType()) {2619    flags |= BLOCK_FIELD_IS_BLOCK;2620  } else if (CGM.getContext().isObjCNSObjectType(type) ||2621             type->isObjCObjectPointerType()) {2622    flags |= BLOCK_FIELD_IS_OBJECT;2623  } else {2624    return nullptr;2625  }2626 2627  if (type.isObjCGCWeak())2628    flags |= BLOCK_FIELD_IS_WEAK;2629 2630  return ::buildByrefHelpers(CGM, byrefInfo,2631                             ObjectByrefHelpers(valueAlignment, flags));2632}2633 2634Address CodeGenFunction::emitBlockByrefAddress(Address baseAddr,2635                                               const VarDecl *var,2636                                               bool followForward) {2637  auto &info = getBlockByrefInfo(var);2638  return emitBlockByrefAddress(baseAddr, info, followForward, var->getName());2639}2640 2641Address CodeGenFunction::emitBlockByrefAddress(Address baseAddr,2642                                               const BlockByrefInfo &info,2643                                               bool followForward,2644                                               const llvm::Twine &name) {2645  // Chase the forwarding address if requested.2646  if (followForward) {2647    Address forwardingAddr = Builder.CreateStructGEP(baseAddr, 1, "forwarding");2648    baseAddr = Address(Builder.CreateLoad(forwardingAddr), info.Type,2649                       info.ByrefAlignment);2650  }2651 2652  return Builder.CreateStructGEP(baseAddr, info.FieldIndex, name);2653}2654 2655/// BuildByrefInfo - This routine changes a __block variable declared as T x2656///   into:2657///2658///      struct {2659///        void *__isa;2660///        void *__forwarding;2661///        int32_t __flags;2662///        int32_t __size;2663///        void *__copy_helper;       // only if needed2664///        void *__destroy_helper;    // only if needed2665///        void *__byref_variable_layout;// only if needed2666///        char padding[X];           // only if needed2667///        T x;2668///      } x2669///2670const BlockByrefInfo &CodeGenFunction::getBlockByrefInfo(const VarDecl *D) {2671  auto it = BlockByrefInfos.find(D);2672  if (it != BlockByrefInfos.end())2673    return it->second;2674 2675  QualType Ty = D->getType();2676 2677  CharUnits size;2678  SmallVector<llvm::Type *, 8> types;2679 2680  // void *__isa;2681  types.push_back(VoidPtrTy);2682  size += getPointerSize();2683 2684  // void *__forwarding;2685  types.push_back(VoidPtrTy);2686  size += getPointerSize();2687 2688  // int32_t __flags;2689  types.push_back(Int32Ty);2690  size += CharUnits::fromQuantity(4);2691 2692  // int32_t __size;2693  types.push_back(Int32Ty);2694  size += CharUnits::fromQuantity(4);2695 2696  // Note that this must match *exactly* the logic in buildByrefHelpers.2697  bool hasCopyAndDispose = getContext().BlockRequiresCopying(Ty, D);2698  if (hasCopyAndDispose) {2699    /// void *__copy_helper;2700    types.push_back(VoidPtrTy);2701    size += getPointerSize();2702 2703    /// void *__destroy_helper;2704    types.push_back(VoidPtrTy);2705    size += getPointerSize();2706  }2707 2708  bool HasByrefExtendedLayout = false;2709  Qualifiers::ObjCLifetime Lifetime = Qualifiers::OCL_None;2710  if (getContext().getByrefLifetime(Ty, Lifetime, HasByrefExtendedLayout) &&2711      HasByrefExtendedLayout) {2712    /// void *__byref_variable_layout;2713    types.push_back(VoidPtrTy);2714    size += CharUnits::fromQuantity(PointerSizeInBytes);2715  }2716 2717  // T x;2718  llvm::Type *varTy = ConvertTypeForMem(Ty);2719 2720  bool packed = false;2721  CharUnits varAlign = getContext().getDeclAlign(D);2722  CharUnits varOffset = size.alignTo(varAlign);2723 2724  // We may have to insert padding.2725  if (varOffset != size) {2726    llvm::Type *paddingTy =2727      llvm::ArrayType::get(Int8Ty, (varOffset - size).getQuantity());2728 2729    types.push_back(paddingTy);2730    size = varOffset;2731 2732  // Conversely, we might have to prevent LLVM from inserting padding.2733  } else if (CGM.getDataLayout().getABITypeAlign(varTy) >2734             uint64_t(varAlign.getQuantity())) {2735    packed = true;2736  }2737  types.push_back(varTy);2738 2739  llvm::StructType *byrefType = llvm::StructType::create(2740      getLLVMContext(), types, "struct.__block_byref_" + D->getNameAsString(),2741      packed);2742 2743  BlockByrefInfo info;2744  info.Type = byrefType;2745  info.FieldIndex = types.size() - 1;2746  info.FieldOffset = varOffset;2747  info.ByrefAlignment = std::max(varAlign, getPointerAlign());2748 2749  auto pair = BlockByrefInfos.insert({D, info});2750  assert(pair.second && "info was inserted recursively?");2751  return pair.first->second;2752}2753 2754/// Initialize the structural components of a __block variable, i.e.2755/// everything but the actual object.2756void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) {2757  // Find the address of the local.2758  Address addr = emission.Addr;2759 2760  // That's an alloca of the byref structure type.2761  llvm::StructType *byrefType = cast<llvm::StructType>(addr.getElementType());2762 2763  unsigned nextHeaderIndex = 0;2764  CharUnits nextHeaderOffset;2765  auto storeHeaderField = [&](llvm::Value *value, CharUnits fieldSize,2766                              const Twine &name, bool isFunction = false) {2767    auto fieldAddr = Builder.CreateStructGEP(addr, nextHeaderIndex, name);2768    if (isFunction) {2769      if (auto &Schema = CGM.getCodeGenOpts()2770                             .PointerAuth.BlockByrefHelperFunctionPointers) {2771        auto PointerAuth = EmitPointerAuthInfo(2772            Schema, fieldAddr.emitRawPointer(*this), GlobalDecl(), QualType());2773        value = EmitPointerAuthSign(PointerAuth, value);2774      }2775    }2776    Builder.CreateStore(value, fieldAddr);2777 2778    nextHeaderIndex++;2779    nextHeaderOffset += fieldSize;2780  };2781 2782  // Build the byref helpers if necessary.  This is null if we don't need any.2783  BlockByrefHelpers *helpers = buildByrefHelpers(*byrefType, emission);2784 2785  const VarDecl &D = *emission.Variable;2786  QualType type = D.getType();2787 2788  bool HasByrefExtendedLayout = false;2789  Qualifiers::ObjCLifetime ByrefLifetime = Qualifiers::OCL_None;2790  bool ByRefHasLifetime =2791    getContext().getByrefLifetime(type, ByrefLifetime, HasByrefExtendedLayout);2792 2793  llvm::Value *V;2794 2795  // Initialize the 'isa', which is just 0 or 1.2796  int isa = 0;2797  if (type.isObjCGCWeak())2798    isa = 1;2799  V = Builder.CreateIntToPtr(Builder.getInt32(isa), Int8PtrTy, "isa");2800  storeHeaderField(V, getPointerSize(), "byref.isa");2801 2802  // Store the address of the variable into its own forwarding pointer.2803  storeHeaderField(addr.emitRawPointer(*this), getPointerSize(),2804                   "byref.forwarding");2805 2806  // Blocks ABI:2807  //   c) the flags field is set to either 0 if no helper functions are2808  //      needed or BLOCK_BYREF_HAS_COPY_DISPOSE if they are,2809  BlockFlags flags;2810  if (helpers) flags |= BLOCK_BYREF_HAS_COPY_DISPOSE;2811  if (ByRefHasLifetime) {2812    if (HasByrefExtendedLayout) flags |= BLOCK_BYREF_LAYOUT_EXTENDED;2813      else switch (ByrefLifetime) {2814        case Qualifiers::OCL_Strong:2815          flags |= BLOCK_BYREF_LAYOUT_STRONG;2816          break;2817        case Qualifiers::OCL_Weak:2818          flags |= BLOCK_BYREF_LAYOUT_WEAK;2819          break;2820        case Qualifiers::OCL_ExplicitNone:2821          flags |= BLOCK_BYREF_LAYOUT_UNRETAINED;2822          break;2823        case Qualifiers::OCL_None:2824          if (!type->isObjCObjectPointerType() && !type->isBlockPointerType())2825            flags |= BLOCK_BYREF_LAYOUT_NON_OBJECT;2826          break;2827        default:2828          break;2829      }2830    if (CGM.getLangOpts().ObjCGCBitmapPrint) {2831      printf("\n Inline flag for BYREF variable layout (%d):", flags.getBitMask());2832      if (flags & BLOCK_BYREF_HAS_COPY_DISPOSE)2833        printf(" BLOCK_BYREF_HAS_COPY_DISPOSE");2834      if (flags & BLOCK_BYREF_LAYOUT_MASK) {2835        BlockFlags ThisFlag(flags.getBitMask() & BLOCK_BYREF_LAYOUT_MASK);2836        if (ThisFlag ==  BLOCK_BYREF_LAYOUT_EXTENDED)2837          printf(" BLOCK_BYREF_LAYOUT_EXTENDED");2838        if (ThisFlag ==  BLOCK_BYREF_LAYOUT_STRONG)2839          printf(" BLOCK_BYREF_LAYOUT_STRONG");2840        if (ThisFlag == BLOCK_BYREF_LAYOUT_WEAK)2841          printf(" BLOCK_BYREF_LAYOUT_WEAK");2842        if (ThisFlag == BLOCK_BYREF_LAYOUT_UNRETAINED)2843          printf(" BLOCK_BYREF_LAYOUT_UNRETAINED");2844        if (ThisFlag == BLOCK_BYREF_LAYOUT_NON_OBJECT)2845          printf(" BLOCK_BYREF_LAYOUT_NON_OBJECT");2846      }2847      printf("\n");2848    }2849  }2850  storeHeaderField(llvm::ConstantInt::get(IntTy, flags.getBitMask()),2851                   getIntSize(), "byref.flags");2852 2853  CharUnits byrefSize = CGM.GetTargetTypeStoreSize(byrefType);2854  V = llvm::ConstantInt::get(IntTy, byrefSize.getQuantity());2855  storeHeaderField(V, getIntSize(), "byref.size");2856 2857  if (helpers) {2858    storeHeaderField(helpers->CopyHelper, getPointerSize(), "byref.copyHelper",2859                     /*isFunction=*/true);2860    storeHeaderField(helpers->DisposeHelper, getPointerSize(),2861                     "byref.disposeHelper", /*isFunction=*/true);2862  }2863 2864  if (ByRefHasLifetime && HasByrefExtendedLayout) {2865    auto layoutInfo = CGM.getObjCRuntime().BuildByrefLayout(CGM, type);2866    storeHeaderField(layoutInfo, getPointerSize(), "byref.layout");2867  }2868}2869 2870void CodeGenFunction::BuildBlockRelease(llvm::Value *V, BlockFieldFlags flags,2871                                        bool CanThrow) {2872  llvm::FunctionCallee F = CGM.getBlockObjectDispose();2873  llvm::Value *args[] = {V,2874                         llvm::ConstantInt::get(Int32Ty, flags.getBitMask())};2875 2876  if (CanThrow)2877    EmitRuntimeCallOrInvoke(F, args);2878  else2879    EmitNounwindRuntimeCall(F, args);2880}2881 2882void CodeGenFunction::enterByrefCleanup(CleanupKind Kind, Address Addr,2883                                        BlockFieldFlags Flags,2884                                        bool LoadBlockVarAddr, bool CanThrow) {2885  EHStack.pushCleanup<CallBlockRelease>(Kind, Addr, Flags, LoadBlockVarAddr,2886                                        CanThrow);2887}2888 2889/// Adjust the declaration of something from the blocks API.2890static void configureBlocksRuntimeObject(CodeGenModule &CGM,2891                                         llvm::Constant *C) {2892  auto *GV = cast<llvm::GlobalValue>(C->stripPointerCasts());2893 2894  if (!CGM.getCodeGenOpts().StaticClosure &&2895      CGM.getTarget().getTriple().isOSBinFormatCOFF()) {2896    const IdentifierInfo &II = CGM.getContext().Idents.get(C->getName());2897    TranslationUnitDecl *TUDecl = CGM.getContext().getTranslationUnitDecl();2898    DeclContext *DC = TranslationUnitDecl::castToDeclContext(TUDecl);2899 2900    assert((isa<llvm::Function>(C->stripPointerCasts()) ||2901            isa<llvm::GlobalVariable>(C->stripPointerCasts())) &&2902           "expected Function or GlobalVariable");2903 2904    const NamedDecl *ND = nullptr;2905    for (const auto *Result : DC->lookup(&II))2906      if ((ND = dyn_cast<FunctionDecl>(Result)) ||2907          (ND = dyn_cast<VarDecl>(Result)))2908        break;2909 2910    if (GV->isDeclaration() && (!ND || !ND->hasAttr<DLLExportAttr>())) {2911      GV->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass);2912      GV->setLinkage(llvm::GlobalValue::ExternalLinkage);2913    } else {2914      GV->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass);2915      GV->setLinkage(llvm::GlobalValue::ExternalLinkage);2916    }2917  }2918 2919  if (CGM.getLangOpts().BlocksRuntimeOptional && GV->isDeclaration() &&2920      GV->hasExternalLinkage())2921    GV->setLinkage(llvm::GlobalValue::ExternalWeakLinkage);2922 2923  CGM.setDSOLocal(GV);2924}2925 2926llvm::FunctionCallee CodeGenModule::getBlockObjectDispose() {2927  if (BlockObjectDispose)2928    return BlockObjectDispose;2929 2930  QualType args[] = {Context.VoidPtrTy, Context.IntTy};2931  BlockObjectDispose =2932      CreateRuntimeFunction(Context.VoidTy, args, "_Block_object_dispose");2933  configureBlocksRuntimeObject(2934      *this, cast<llvm::Constant>(BlockObjectDispose.getCallee()));2935  return BlockObjectDispose;2936}2937 2938llvm::FunctionCallee CodeGenModule::getBlockObjectAssign() {2939  if (BlockObjectAssign)2940    return BlockObjectAssign;2941 2942  QualType args[] = {Context.VoidPtrTy, Context.VoidPtrTy, Context.IntTy};2943  BlockObjectAssign =2944      CreateRuntimeFunction(Context.VoidTy, args, "_Block_object_assign");2945  configureBlocksRuntimeObject(2946      *this, cast<llvm::Constant>(BlockObjectAssign.getCallee()));2947  return BlockObjectAssign;2948}2949 2950llvm::Constant *CodeGenModule::getNSConcreteGlobalBlock() {2951  if (NSConcreteGlobalBlock)2952    return NSConcreteGlobalBlock;2953 2954  NSConcreteGlobalBlock = GetOrCreateLLVMGlobal(2955      "_NSConcreteGlobalBlock", Int8PtrTy, LangAS::Default, nullptr);2956  configureBlocksRuntimeObject(*this, NSConcreteGlobalBlock);2957  return NSConcreteGlobalBlock;2958}2959 2960llvm::Constant *CodeGenModule::getNSConcreteStackBlock() {2961  if (NSConcreteStackBlock)2962    return NSConcreteStackBlock;2963 2964  NSConcreteStackBlock = GetOrCreateLLVMGlobal(2965      "_NSConcreteStackBlock", Int8PtrTy, LangAS::Default, nullptr);2966  configureBlocksRuntimeObject(*this, NSConcreteStackBlock);2967  return NSConcreteStackBlock;2968}2969