5596 lines · c
1//===-- CodeGenFunction.h - Per-Function state for LLVM CodeGen -*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This is the internal per-function state used for llvm translation.10//11//===----------------------------------------------------------------------===//12 13#ifndef LLVM_CLANG_LIB_CODEGEN_CODEGENFUNCTION_H14#define LLVM_CLANG_LIB_CODEGEN_CODEGENFUNCTION_H15 16#include "CGBuilder.h"17#include "CGLoopInfo.h"18#include "CGValue.h"19#include "CodeGenModule.h"20#include "EHScopeStack.h"21#include "SanitizerHandler.h"22#include "VarBypassDetector.h"23#include "clang/AST/CharUnits.h"24#include "clang/AST/CurrentSourceLocExprScope.h"25#include "clang/AST/ExprCXX.h"26#include "clang/AST/ExprObjC.h"27#include "clang/AST/ExprOpenMP.h"28#include "clang/AST/StmtOpenACC.h"29#include "clang/AST/StmtOpenMP.h"30#include "clang/AST/StmtSYCL.h"31#include "clang/AST/Type.h"32#include "clang/Basic/ABI.h"33#include "clang/Basic/CapturedStmt.h"34#include "clang/Basic/CodeGenOptions.h"35#include "clang/Basic/OpenMPKinds.h"36#include "clang/Basic/TargetInfo.h"37#include "llvm/ADT/ArrayRef.h"38#include "llvm/ADT/DenseMap.h"39#include "llvm/ADT/MapVector.h"40#include "llvm/ADT/SmallVector.h"41#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"42#include "llvm/IR/Instructions.h"43#include "llvm/IR/ValueHandle.h"44#include "llvm/Support/Debug.h"45#include "llvm/Transforms/Utils/SanitizerStats.h"46#include <optional>47 48namespace llvm {49class BasicBlock;50class ConvergenceControlInst;51class LLVMContext;52class MDNode;53class SwitchInst;54class Twine;55class Value;56class CanonicalLoopInfo;57} // namespace llvm58 59namespace clang {60class ASTContext;61class CXXDestructorDecl;62class CXXForRangeStmt;63class CXXTryStmt;64class Decl;65class LabelDecl;66class FunctionDecl;67class FunctionProtoType;68class LabelStmt;69class ObjCContainerDecl;70class ObjCInterfaceDecl;71class ObjCIvarDecl;72class ObjCMethodDecl;73class ObjCImplementationDecl;74class ObjCPropertyImplDecl;75class TargetInfo;76class VarDecl;77class ObjCForCollectionStmt;78class ObjCAtTryStmt;79class ObjCAtThrowStmt;80class ObjCAtSynchronizedStmt;81class ObjCAutoreleasePoolStmt;82class OMPUseDevicePtrClause;83class OMPUseDeviceAddrClause;84class SVETypeFlags;85class OMPExecutableDirective;86 87namespace analyze_os_log {88class OSLogBufferLayout;89}90 91namespace CodeGen {92class CodeGenTypes;93class CodeGenPGO;94class CGCallee;95class CGFunctionInfo;96class CGBlockInfo;97class CGCXXABI;98class BlockByrefHelpers;99class BlockByrefInfo;100class BlockFieldFlags;101class RegionCodeGenTy;102class TargetCodeGenInfo;103struct OMPTaskDataTy;104struct CGCoroData;105 106// clang-format off107/// The kind of evaluation to perform on values of a particular108/// type. Basically, is the code in CGExprScalar, CGExprComplex, or109/// CGExprAgg?110///111/// TODO: should vectors maybe be split out into their own thing?112enum TypeEvaluationKind {113 TEK_Scalar,114 TEK_Complex,115 TEK_Aggregate116};117// clang-format on118 119/// Helper class with most of the code for saving a value for a120/// conditional expression cleanup.121struct DominatingLLVMValue {122 typedef llvm::PointerIntPair<llvm::Value *, 1, bool> saved_type;123 124 /// Answer whether the given value needs extra work to be saved.125 static bool needsSaving(llvm::Value *value) {126 if (!value)127 return false;128 129 // If it's not an instruction, we don't need to save.130 if (!isa<llvm::Instruction>(value))131 return false;132 133 // If it's an instruction in the entry block, we don't need to save.134 llvm::BasicBlock *block = cast<llvm::Instruction>(value)->getParent();135 return (block != &block->getParent()->getEntryBlock());136 }137 138 static saved_type save(CodeGenFunction &CGF, llvm::Value *value);139 static llvm::Value *restore(CodeGenFunction &CGF, saved_type value);140};141 142/// A partial specialization of DominatingValue for llvm::Values that143/// might be llvm::Instructions.144template <class T> struct DominatingPointer<T, true> : DominatingLLVMValue {145 typedef T *type;146 static type restore(CodeGenFunction &CGF, saved_type value) {147 return static_cast<T *>(DominatingLLVMValue::restore(CGF, value));148 }149};150 151/// A specialization of DominatingValue for Address.152template <> struct DominatingValue<Address> {153 typedef Address type;154 155 struct saved_type {156 DominatingLLVMValue::saved_type BasePtr;157 llvm::Type *ElementType;158 CharUnits Alignment;159 DominatingLLVMValue::saved_type Offset;160 llvm::PointerType *EffectiveType;161 };162 163 static bool needsSaving(type value) {164 if (DominatingLLVMValue::needsSaving(value.getBasePointer()) ||165 DominatingLLVMValue::needsSaving(value.getOffset()))166 return true;167 return false;168 }169 static saved_type save(CodeGenFunction &CGF, type value) {170 return {DominatingLLVMValue::save(CGF, value.getBasePointer()),171 value.getElementType(), value.getAlignment(),172 DominatingLLVMValue::save(CGF, value.getOffset()), value.getType()};173 }174 static type restore(CodeGenFunction &CGF, saved_type value) {175 return Address(DominatingLLVMValue::restore(CGF, value.BasePtr),176 value.ElementType, value.Alignment, CGPointerAuthInfo(),177 DominatingLLVMValue::restore(CGF, value.Offset));178 }179};180 181/// A specialization of DominatingValue for RValue.182template <> struct DominatingValue<RValue> {183 typedef RValue type;184 class saved_type {185 enum Kind {186 ScalarLiteral,187 ScalarAddress,188 AggregateLiteral,189 AggregateAddress,190 ComplexAddress191 };192 union {193 struct {194 DominatingLLVMValue::saved_type first, second;195 } Vals;196 DominatingValue<Address>::saved_type AggregateAddr;197 };198 LLVM_PREFERRED_TYPE(Kind)199 unsigned K : 3;200 201 saved_type(DominatingLLVMValue::saved_type Val1, unsigned K)202 : Vals{Val1, DominatingLLVMValue::saved_type()}, K(K) {}203 204 saved_type(DominatingLLVMValue::saved_type Val1,205 DominatingLLVMValue::saved_type Val2)206 : Vals{Val1, Val2}, K(ComplexAddress) {}207 208 saved_type(DominatingValue<Address>::saved_type AggregateAddr, unsigned K)209 : AggregateAddr(AggregateAddr), K(K) {}210 211 public:212 static bool needsSaving(RValue value);213 static saved_type save(CodeGenFunction &CGF, RValue value);214 RValue restore(CodeGenFunction &CGF);215 216 // implementations in CGCleanup.cpp217 };218 219 static bool needsSaving(type value) { return saved_type::needsSaving(value); }220 static saved_type save(CodeGenFunction &CGF, type value) {221 return saved_type::save(CGF, value);222 }223 static type restore(CodeGenFunction &CGF, saved_type value) {224 return value.restore(CGF);225 }226};227 228/// A scoped helper to set the current source atom group for229/// CGDebugInfo::addInstToCurrentSourceAtom. A source atom is a source construct230/// that is "interesting" for debug stepping purposes. We use an atom group231/// number to track the instruction(s) that implement the functionality for the232/// atom, plus backup instructions/source locations.233class ApplyAtomGroup {234 uint64_t OriginalAtom = 0;235 CGDebugInfo *DI = nullptr;236 237 ApplyAtomGroup(const ApplyAtomGroup &) = delete;238 void operator=(const ApplyAtomGroup &) = delete;239 240public:241 ApplyAtomGroup(CGDebugInfo *DI);242 ~ApplyAtomGroup();243};244 245/// CodeGenFunction - This class organizes the per-function state that is used246/// while generating LLVM code.247class CodeGenFunction : public CodeGenTypeCache {248 CodeGenFunction(const CodeGenFunction &) = delete;249 void operator=(const CodeGenFunction &) = delete;250 251 friend class CGCXXABI;252 253public:254 /// A jump destination is an abstract label, branching to which may255 /// require a jump out through normal cleanups.256 struct JumpDest {257 JumpDest() : Block(nullptr), Index(0) {}258 JumpDest(llvm::BasicBlock *Block, EHScopeStack::stable_iterator Depth,259 unsigned Index)260 : Block(Block), ScopeDepth(Depth), Index(Index) {}261 262 bool isValid() const { return Block != nullptr; }263 llvm::BasicBlock *getBlock() const { return Block; }264 EHScopeStack::stable_iterator getScopeDepth() const { return ScopeDepth; }265 unsigned getDestIndex() const { return Index; }266 267 // This should be used cautiously.268 void setScopeDepth(EHScopeStack::stable_iterator depth) {269 ScopeDepth = depth;270 }271 272 private:273 llvm::BasicBlock *Block;274 EHScopeStack::stable_iterator ScopeDepth;275 unsigned Index;276 };277 278 CodeGenModule &CGM; // Per-module state.279 const TargetInfo &Target;280 281 // For EH/SEH outlined funclets, this field points to parent's CGF282 CodeGenFunction *ParentCGF = nullptr;283 284 typedef std::pair<llvm::Value *, llvm::Value *> ComplexPairTy;285 LoopInfoStack LoopStack;286 CGBuilderTy Builder;287 288 // Stores variables for which we can't generate correct lifetime markers289 // because of jumps.290 VarBypassDetector Bypasses;291 292 /// List of recently emitted OMPCanonicalLoops.293 ///294 /// Since OMPCanonicalLoops are nested inside other statements (in particular295 /// CapturedStmt generated by OMPExecutableDirective and non-perfectly nested296 /// loops), we cannot directly call OMPEmitOMPCanonicalLoop and receive its297 /// llvm::CanonicalLoopInfo. Instead, we call EmitStmt and any298 /// OMPEmitOMPCanonicalLoop called by it will add its CanonicalLoopInfo to299 /// this stack when done. Entering a new loop requires clearing this list; it300 /// either means we start parsing a new loop nest (in which case the previous301 /// loop nest goes out of scope) or a second loop in the same level in which302 /// case it would be ambiguous into which of the two (or more) loops the loop303 /// nest would extend.304 SmallVector<llvm::CanonicalLoopInfo *, 4> OMPLoopNestStack;305 306 /// Stack to track the Logical Operator recursion nest for MC/DC.307 SmallVector<const BinaryOperator *, 16> MCDCLogOpStack;308 309 /// Stack to track the controlled convergence tokens.310 SmallVector<llvm::ConvergenceControlInst *, 4> ConvergenceTokenStack;311 312 /// Number of nested loop to be consumed by the last surrounding313 /// loop-associated directive.314 int ExpectedOMPLoopDepth = 0;315 316 // CodeGen lambda for loops and support for ordered clause317 typedef llvm::function_ref<void(CodeGenFunction &, const OMPLoopDirective &,318 JumpDest)>319 CodeGenLoopTy;320 typedef llvm::function_ref<void(CodeGenFunction &, SourceLocation,321 const unsigned, const bool)>322 CodeGenOrderedTy;323 324 // Codegen lambda for loop bounds in worksharing loop constructs325 typedef llvm::function_ref<std::pair<LValue, LValue>(326 CodeGenFunction &, const OMPExecutableDirective &S)>327 CodeGenLoopBoundsTy;328 329 // Codegen lambda for loop bounds in dispatch-based loop implementation330 typedef llvm::function_ref<std::pair<llvm::Value *, llvm::Value *>(331 CodeGenFunction &, const OMPExecutableDirective &S, Address LB,332 Address UB)>333 CodeGenDispatchBoundsTy;334 335 /// CGBuilder insert helper. This function is called after an336 /// instruction is created using Builder.337 void InsertHelper(llvm::Instruction *I, const llvm::Twine &Name,338 llvm::BasicBlock::iterator InsertPt) const;339 340 /// CurFuncDecl - Holds the Decl for the current outermost341 /// non-closure context.342 const Decl *CurFuncDecl = nullptr;343 /// CurCodeDecl - This is the inner-most code context, which includes blocks.344 const Decl *CurCodeDecl = nullptr;345 const CGFunctionInfo *CurFnInfo = nullptr;346 QualType FnRetTy;347 llvm::Function *CurFn = nullptr;348 349 /// If a cast expression is being visited, this holds the current cast's350 /// expression.351 const CastExpr *CurCast = nullptr;352 353 /// Save Parameter Decl for coroutine.354 llvm::SmallVector<const ParmVarDecl *, 4> FnArgs;355 356 // Holds coroutine data if the current function is a coroutine. We use a357 // wrapper to manage its lifetime, so that we don't have to define CGCoroData358 // in this header.359 struct CGCoroInfo {360 std::unique_ptr<CGCoroData> Data;361 bool InSuspendBlock = false;362 CGCoroInfo();363 ~CGCoroInfo();364 };365 CGCoroInfo CurCoro;366 367 bool isCoroutine() const { return CurCoro.Data != nullptr; }368 369 bool inSuspendBlock() const {370 return isCoroutine() && CurCoro.InSuspendBlock;371 }372 373 // Holds FramePtr for await_suspend wrapper generation,374 // so that __builtin_coro_frame call can be lowered375 // directly to value of its second argument376 struct AwaitSuspendWrapperInfo {377 llvm::Value *FramePtr = nullptr;378 };379 AwaitSuspendWrapperInfo CurAwaitSuspendWrapper;380 381 // Generates wrapper function for `llvm.coro.await.suspend.*` intrinisics.382 // It encapsulates SuspendExpr in a function, to separate it's body383 // from the main coroutine to avoid miscompilations. Intrinisic384 // is lowered to this function call in CoroSplit pass385 // Function signature is:386 // <type> __await_suspend_wrapper_<name>(ptr %awaiter, ptr %hdl)387 // where type is one of (void, i1, ptr)388 llvm::Function *generateAwaitSuspendWrapper(Twine const &CoroName,389 Twine const &SuspendPointName,390 CoroutineSuspendExpr const &S);391 392 /// CurGD - The GlobalDecl for the current function being compiled.393 GlobalDecl CurGD;394 395 /// PrologueCleanupDepth - The cleanup depth enclosing all the396 /// cleanups associated with the parameters.397 EHScopeStack::stable_iterator PrologueCleanupDepth;398 399 /// ReturnBlock - Unified return block.400 JumpDest ReturnBlock;401 402 /// ReturnValue - The temporary alloca to hold the return403 /// value. This is invalid iff the function has no return value.404 Address ReturnValue = Address::invalid();405 406 /// ReturnValuePointer - The temporary alloca to hold a pointer to sret.407 /// This is invalid if sret is not in use.408 Address ReturnValuePointer = Address::invalid();409 410 /// If a return statement is being visited, this holds the return statment's411 /// result expression.412 const Expr *RetExpr = nullptr;413 414 /// Return true if a label was seen in the current scope.415 bool hasLabelBeenSeenInCurrentScope() const {416 if (CurLexicalScope)417 return CurLexicalScope->hasLabels();418 return !LabelMap.empty();419 }420 421 /// AllocaInsertPoint - This is an instruction in the entry block before which422 /// we prefer to insert allocas.423 llvm::AssertingVH<llvm::Instruction> AllocaInsertPt;424 425private:426 /// PostAllocaInsertPt - This is a place in the prologue where code can be427 /// inserted that will be dominated by all the static allocas. This helps428 /// achieve two things:429 /// 1. Contiguity of all static allocas (within the prologue) is maintained.430 /// 2. All other prologue code (which are dominated by static allocas) do431 /// appear in the source order immediately after all static allocas.432 ///433 /// PostAllocaInsertPt will be lazily created when it is *really* required.434 llvm::AssertingVH<llvm::Instruction> PostAllocaInsertPt = nullptr;435 436public:437 /// Return PostAllocaInsertPt. If it is not yet created, then insert it438 /// immediately after AllocaInsertPt.439 llvm::Instruction *getPostAllocaInsertPoint() {440 if (!PostAllocaInsertPt) {441 assert(AllocaInsertPt &&442 "Expected static alloca insertion point at function prologue");443 assert(AllocaInsertPt->getParent()->isEntryBlock() &&444 "EBB should be entry block of the current code gen function");445 PostAllocaInsertPt = AllocaInsertPt->clone();446 PostAllocaInsertPt->setName("postallocapt");447 PostAllocaInsertPt->insertAfter(AllocaInsertPt->getIterator());448 }449 450 return PostAllocaInsertPt;451 }452 453 /// API for captured statement code generation.454 class CGCapturedStmtInfo {455 public:456 explicit CGCapturedStmtInfo(CapturedRegionKind K = CR_Default)457 : Kind(K), ThisValue(nullptr), CXXThisFieldDecl(nullptr) {}458 explicit CGCapturedStmtInfo(const CapturedStmt &S,459 CapturedRegionKind K = CR_Default)460 : Kind(K), ThisValue(nullptr), CXXThisFieldDecl(nullptr) {461 462 RecordDecl::field_iterator Field =463 S.getCapturedRecordDecl()->field_begin();464 for (CapturedStmt::const_capture_iterator I = S.capture_begin(),465 E = S.capture_end();466 I != E; ++I, ++Field) {467 if (I->capturesThis())468 CXXThisFieldDecl = *Field;469 else if (I->capturesVariable())470 CaptureFields[I->getCapturedVar()->getCanonicalDecl()] = *Field;471 else if (I->capturesVariableByCopy())472 CaptureFields[I->getCapturedVar()->getCanonicalDecl()] = *Field;473 }474 }475 476 virtual ~CGCapturedStmtInfo();477 478 CapturedRegionKind getKind() const { return Kind; }479 480 virtual void setContextValue(llvm::Value *V) { ThisValue = V; }481 // Retrieve the value of the context parameter.482 virtual llvm::Value *getContextValue() const { return ThisValue; }483 484 /// Lookup the captured field decl for a variable.485 virtual const FieldDecl *lookup(const VarDecl *VD) const {486 return CaptureFields.lookup(VD->getCanonicalDecl());487 }488 489 bool isCXXThisExprCaptured() const { return getThisFieldDecl() != nullptr; }490 virtual FieldDecl *getThisFieldDecl() const { return CXXThisFieldDecl; }491 492 static bool classof(const CGCapturedStmtInfo *) { return true; }493 494 /// Emit the captured statement body.495 virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S) {496 CGF.incrementProfileCounter(S);497 CGF.EmitStmt(S);498 }499 500 /// Get the name of the capture helper.501 virtual StringRef getHelperName() const { return "__captured_stmt"; }502 503 /// Get the CaptureFields504 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> getCaptureFields() {505 return CaptureFields;506 }507 508 private:509 /// The kind of captured statement being generated.510 CapturedRegionKind Kind;511 512 /// Keep the map between VarDecl and FieldDecl.513 llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields;514 515 /// The base address of the captured record, passed in as the first516 /// argument of the parallel region function.517 llvm::Value *ThisValue;518 519 /// Captured 'this' type.520 FieldDecl *CXXThisFieldDecl;521 };522 CGCapturedStmtInfo *CapturedStmtInfo = nullptr;523 524 /// RAII for correct setting/restoring of CapturedStmtInfo.525 class CGCapturedStmtRAII {526 private:527 CodeGenFunction &CGF;528 CGCapturedStmtInfo *PrevCapturedStmtInfo;529 530 public:531 CGCapturedStmtRAII(CodeGenFunction &CGF,532 CGCapturedStmtInfo *NewCapturedStmtInfo)533 : CGF(CGF), PrevCapturedStmtInfo(CGF.CapturedStmtInfo) {534 CGF.CapturedStmtInfo = NewCapturedStmtInfo;535 }536 ~CGCapturedStmtRAII() { CGF.CapturedStmtInfo = PrevCapturedStmtInfo; }537 };538 539 /// An abstract representation of regular/ObjC call/message targets.540 class AbstractCallee {541 /// The function declaration of the callee.542 const Decl *CalleeDecl;543 544 public:545 AbstractCallee() : CalleeDecl(nullptr) {}546 AbstractCallee(const FunctionDecl *FD) : CalleeDecl(FD) {}547 AbstractCallee(const ObjCMethodDecl *OMD) : CalleeDecl(OMD) {}548 bool hasFunctionDecl() const {549 return isa_and_nonnull<FunctionDecl>(CalleeDecl);550 }551 const Decl *getDecl() const { return CalleeDecl; }552 unsigned getNumParams() const {553 if (const auto *FD = dyn_cast<FunctionDecl>(CalleeDecl))554 return FD->getNumParams();555 return cast<ObjCMethodDecl>(CalleeDecl)->param_size();556 }557 const ParmVarDecl *getParamDecl(unsigned I) const {558 if (const auto *FD = dyn_cast<FunctionDecl>(CalleeDecl))559 return FD->getParamDecl(I);560 return *(cast<ObjCMethodDecl>(CalleeDecl)->param_begin() + I);561 }562 };563 564 /// Sanitizers enabled for this function.565 SanitizerSet SanOpts;566 567 /// True if CodeGen currently emits code implementing sanitizer checks.568 bool IsSanitizerScope = false;569 570 /// RAII object to set/unset CodeGenFunction::IsSanitizerScope.571 class SanitizerScope {572 CodeGenFunction *CGF;573 574 public:575 SanitizerScope(CodeGenFunction *CGF);576 ~SanitizerScope();577 };578 579 /// In C++, whether we are code generating a thunk. This controls whether we580 /// should emit cleanups.581 bool CurFuncIsThunk = false;582 583 /// In ARC, whether we should autorelease the return value.584 bool AutoreleaseResult = false;585 586 /// Whether we processed a Microsoft-style asm block during CodeGen. These can587 /// potentially set the return value.588 bool SawAsmBlock = false;589 590 GlobalDecl CurSEHParent;591 592 /// True if the current function is an outlined SEH helper. This can be a593 /// finally block or filter expression.594 bool IsOutlinedSEHHelper = false;595 596 /// True if CodeGen currently emits code inside presereved access index597 /// region.598 bool IsInPreservedAIRegion = false;599 600 /// True if the current statement has nomerge attribute.601 bool InNoMergeAttributedStmt = false;602 603 /// True if the current statement has noinline attribute.604 bool InNoInlineAttributedStmt = false;605 606 /// True if the current statement has always_inline attribute.607 bool InAlwaysInlineAttributedStmt = false;608 609 /// True if the current statement has noconvergent attribute.610 bool InNoConvergentAttributedStmt = false;611 612 /// HLSL Branch attribute.613 HLSLControlFlowHintAttr::Spelling HLSLControlFlowAttr =614 HLSLControlFlowHintAttr::SpellingNotCalculated;615 616 // The CallExpr within the current statement that the musttail attribute617 // applies to. nullptr if there is no 'musttail' on the current statement.618 const CallExpr *MustTailCall = nullptr;619 620 /// Returns true if a function must make progress, which means the621 /// mustprogress attribute can be added.622 bool checkIfFunctionMustProgress() {623 if (CGM.getCodeGenOpts().getFiniteLoops() ==624 CodeGenOptions::FiniteLoopsKind::Never)625 return false;626 627 // C++11 and later guarantees that a thread eventually will do one of the628 // following (C++11 [intro.multithread]p24 and C++17 [intro.progress]p1):629 // - terminate,630 // - make a call to a library I/O function,631 // - perform an access through a volatile glvalue, or632 // - perform a synchronization operation or an atomic operation.633 //634 // Hence each function is 'mustprogress' in C++11 or later.635 return getLangOpts().CPlusPlus11;636 }637 638 /// Returns true if a loop must make progress, which means the mustprogress639 /// attribute can be added. \p HasConstantCond indicates whether the branch640 /// condition is a known constant.641 bool checkIfLoopMustProgress(const Expr *, bool HasEmptyBody);642 643 const CodeGen::CGBlockInfo *BlockInfo = nullptr;644 llvm::Value *BlockPointer = nullptr;645 646 llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;647 FieldDecl *LambdaThisCaptureField = nullptr;648 649 /// A mapping from NRVO variables to the flags used to indicate650 /// when the NRVO has been applied to this variable.651 llvm::DenseMap<const VarDecl *, llvm::Value *> NRVOFlags;652 653 EHScopeStack EHStack;654 llvm::SmallVector<char, 256> LifetimeExtendedCleanupStack;655 656 // A stack of cleanups which were added to EHStack but have to be deactivated657 // later before being popped or emitted. These are usually deactivated on658 // exiting a `CleanupDeactivationScope` scope. For instance, after a659 // full-expr.660 //661 // These are specially useful for correctly emitting cleanups while662 // encountering branches out of expression (through stmt-expr or coroutine663 // suspensions).664 struct DeferredDeactivateCleanup {665 EHScopeStack::stable_iterator Cleanup;666 llvm::Instruction *DominatingIP;667 };668 llvm::SmallVector<DeferredDeactivateCleanup> DeferredDeactivationCleanupStack;669 670 // Enters a new scope for capturing cleanups which are deferred to be671 // deactivated, all of which will be deactivated once the scope is exited.672 struct CleanupDeactivationScope {673 CodeGenFunction &CGF;674 size_t OldDeactivateCleanupStackSize;675 bool Deactivated;676 CleanupDeactivationScope(CodeGenFunction &CGF)677 : CGF(CGF), OldDeactivateCleanupStackSize(678 CGF.DeferredDeactivationCleanupStack.size()),679 Deactivated(false) {}680 681 void ForceDeactivate() {682 assert(!Deactivated && "Deactivating already deactivated scope");683 auto &Stack = CGF.DeferredDeactivationCleanupStack;684 for (size_t I = Stack.size(); I > OldDeactivateCleanupStackSize; I--) {685 CGF.DeactivateCleanupBlock(Stack[I - 1].Cleanup,686 Stack[I - 1].DominatingIP);687 Stack[I - 1].DominatingIP->eraseFromParent();688 }689 Stack.resize(OldDeactivateCleanupStackSize);690 Deactivated = true;691 }692 693 ~CleanupDeactivationScope() {694 if (Deactivated)695 return;696 ForceDeactivate();697 }698 };699 700 llvm::SmallVector<const JumpDest *, 2> SEHTryEpilogueStack;701 702 llvm::Instruction *CurrentFuncletPad = nullptr;703 704 class CallLifetimeEnd final : public EHScopeStack::Cleanup {705 bool isRedundantBeforeReturn() override { return true; }706 707 llvm::Value *Addr;708 709 public:710 CallLifetimeEnd(RawAddress addr) : Addr(addr.getPointer()) {}711 712 void Emit(CodeGenFunction &CGF, Flags flags) override {713 CGF.EmitLifetimeEnd(Addr);714 }715 };716 717 // We are using objects of this 'cleanup' class to emit fake.use calls718 // for -fextend-variable-liveness. They are placed at the end of a variable's719 // scope analogous to lifetime markers.720 class FakeUse final : public EHScopeStack::Cleanup {721 Address Addr;722 723 public:724 FakeUse(Address addr) : Addr(addr) {}725 726 void Emit(CodeGenFunction &CGF, Flags flags) override {727 CGF.EmitFakeUse(Addr);728 }729 };730 731 /// Header for data within LifetimeExtendedCleanupStack.732 struct alignas(uint64_t) LifetimeExtendedCleanupHeader {733 /// The size of the following cleanup object.734 unsigned Size;735 /// The kind of cleanup to push.736 LLVM_PREFERRED_TYPE(CleanupKind)737 unsigned Kind : 31;738 /// Whether this is a conditional cleanup.739 LLVM_PREFERRED_TYPE(bool)740 unsigned IsConditional : 1;741 742 size_t getSize() const { return Size; }743 CleanupKind getKind() const { return (CleanupKind)Kind; }744 bool isConditional() const { return IsConditional; }745 };746 747 /// i32s containing the indexes of the cleanup destinations.748 RawAddress NormalCleanupDest = RawAddress::invalid();749 750 unsigned NextCleanupDestIndex = 1;751 752 /// EHResumeBlock - Unified block containing a call to llvm.eh.resume.753 llvm::BasicBlock *EHResumeBlock = nullptr;754 755 /// The exception slot. All landing pads write the current exception pointer756 /// into this alloca.757 llvm::Value *ExceptionSlot = nullptr;758 759 /// The selector slot. Under the MandatoryCleanup model, all landing pads760 /// write the current selector value into this alloca.761 llvm::AllocaInst *EHSelectorSlot = nullptr;762 763 /// A stack of exception code slots. Entering an __except block pushes a slot764 /// on the stack and leaving pops one. The __exception_code() intrinsic loads765 /// a value from the top of the stack.766 SmallVector<Address, 1> SEHCodeSlotStack;767 768 /// Value returned by __exception_info intrinsic.769 llvm::Value *SEHInfo = nullptr;770 771 /// Emits a landing pad for the current EH stack.772 llvm::BasicBlock *EmitLandingPad();773 774 llvm::BasicBlock *getInvokeDestImpl();775 776 /// Parent loop-based directive for scan directive.777 const OMPExecutableDirective *OMPParentLoopDirectiveForScan = nullptr;778 llvm::BasicBlock *OMPBeforeScanBlock = nullptr;779 llvm::BasicBlock *OMPAfterScanBlock = nullptr;780 llvm::BasicBlock *OMPScanExitBlock = nullptr;781 llvm::BasicBlock *OMPScanDispatch = nullptr;782 bool OMPFirstScanLoop = false;783 784 /// Manages parent directive for scan directives.785 class ParentLoopDirectiveForScanRegion {786 CodeGenFunction &CGF;787 const OMPExecutableDirective *ParentLoopDirectiveForScan;788 789 public:790 ParentLoopDirectiveForScanRegion(791 CodeGenFunction &CGF,792 const OMPExecutableDirective &ParentLoopDirectiveForScan)793 : CGF(CGF),794 ParentLoopDirectiveForScan(CGF.OMPParentLoopDirectiveForScan) {795 CGF.OMPParentLoopDirectiveForScan = &ParentLoopDirectiveForScan;796 }797 ~ParentLoopDirectiveForScanRegion() {798 CGF.OMPParentLoopDirectiveForScan = ParentLoopDirectiveForScan;799 }800 };801 802 template <class T>803 typename DominatingValue<T>::saved_type saveValueInCond(T value) {804 return DominatingValue<T>::save(*this, value);805 }806 807 class CGFPOptionsRAII {808 public:809 CGFPOptionsRAII(CodeGenFunction &CGF, FPOptions FPFeatures);810 CGFPOptionsRAII(CodeGenFunction &CGF, const Expr *E);811 ~CGFPOptionsRAII();812 813 private:814 void ConstructorHelper(FPOptions FPFeatures);815 CodeGenFunction &CGF;816 FPOptions OldFPFeatures;817 llvm::fp::ExceptionBehavior OldExcept;818 llvm::RoundingMode OldRounding;819 std::optional<CGBuilderTy::FastMathFlagGuard> FMFGuard;820 };821 FPOptions CurFPFeatures;822 823 class CGAtomicOptionsRAII {824 public:825 CGAtomicOptionsRAII(CodeGenModule &CGM_, AtomicOptions AO)826 : CGM(CGM_), SavedAtomicOpts(CGM.getAtomicOpts()) {827 CGM.setAtomicOpts(AO);828 }829 CGAtomicOptionsRAII(CodeGenModule &CGM_, const AtomicAttr *AA)830 : CGM(CGM_), SavedAtomicOpts(CGM.getAtomicOpts()) {831 if (!AA)832 return;833 AtomicOptions AO = SavedAtomicOpts;834 for (auto Option : AA->atomicOptions()) {835 switch (Option) {836 case AtomicAttr::remote_memory:837 AO.remote_memory = true;838 break;839 case AtomicAttr::no_remote_memory:840 AO.remote_memory = false;841 break;842 case AtomicAttr::fine_grained_memory:843 AO.fine_grained_memory = true;844 break;845 case AtomicAttr::no_fine_grained_memory:846 AO.fine_grained_memory = false;847 break;848 case AtomicAttr::ignore_denormal_mode:849 AO.ignore_denormal_mode = true;850 break;851 case AtomicAttr::no_ignore_denormal_mode:852 AO.ignore_denormal_mode = false;853 break;854 }855 }856 CGM.setAtomicOpts(AO);857 }858 859 CGAtomicOptionsRAII(const CGAtomicOptionsRAII &) = delete;860 CGAtomicOptionsRAII &operator=(const CGAtomicOptionsRAII &) = delete;861 ~CGAtomicOptionsRAII() { CGM.setAtomicOpts(SavedAtomicOpts); }862 863 private:864 CodeGenModule &CGM;865 AtomicOptions SavedAtomicOpts;866 };867 868public:869 /// ObjCEHValueStack - Stack of Objective-C exception values, used for870 /// rethrows.871 SmallVector<llvm::Value *, 8> ObjCEHValueStack;872 873 /// A class controlling the emission of a finally block.874 class FinallyInfo {875 /// Where the catchall's edge through the cleanup should go.876 JumpDest RethrowDest;877 878 /// A function to call to enter the catch.879 llvm::FunctionCallee BeginCatchFn;880 881 /// An i1 variable indicating whether or not the @finally is882 /// running for an exception.883 llvm::AllocaInst *ForEHVar = nullptr;884 885 /// An i8* variable into which the exception pointer to rethrow886 /// has been saved.887 llvm::AllocaInst *SavedExnVar = nullptr;888 889 public:890 void enter(CodeGenFunction &CGF, const Stmt *Finally,891 llvm::FunctionCallee beginCatchFn,892 llvm::FunctionCallee endCatchFn, llvm::FunctionCallee rethrowFn);893 void exit(CodeGenFunction &CGF);894 };895 896 /// Returns true inside SEH __try blocks.897 bool isSEHTryScope() const { return !SEHTryEpilogueStack.empty(); }898 899 /// Returns true while emitting a cleanuppad.900 bool isCleanupPadScope() const {901 return CurrentFuncletPad && isa<llvm::CleanupPadInst>(CurrentFuncletPad);902 }903 904 /// pushFullExprCleanup - Push a cleanup to be run at the end of the905 /// current full-expression. Safe against the possibility that906 /// we're currently inside a conditionally-evaluated expression.907 template <class T, class... As>908 void pushFullExprCleanup(CleanupKind kind, As... A) {909 // If we're not in a conditional branch, or if none of the910 // arguments requires saving, then use the unconditional cleanup.911 if (!isInConditionalBranch())912 return EHStack.pushCleanup<T>(kind, A...);913 914 // Stash values in a tuple so we can guarantee the order of saves.915 typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple;916 SavedTuple Saved{saveValueInCond(A)...};917 918 typedef EHScopeStack::ConditionalCleanup<T, As...> CleanupType;919 EHStack.pushCleanupTuple<CleanupType>(kind, Saved);920 initFullExprCleanup();921 }922 923 /// Queue a cleanup to be pushed after finishing the current full-expression,924 /// potentially with an active flag.925 template <class T, class... As>926 void pushCleanupAfterFullExpr(CleanupKind Kind, As... A) {927 if (!isInConditionalBranch())928 return pushCleanupAfterFullExprWithActiveFlag<T>(929 Kind, RawAddress::invalid(), A...);930 931 RawAddress ActiveFlag = createCleanupActiveFlag();932 assert(!DominatingValue<Address>::needsSaving(ActiveFlag) &&933 "cleanup active flag should never need saving");934 935 typedef std::tuple<typename DominatingValue<As>::saved_type...> SavedTuple;936 SavedTuple Saved{saveValueInCond(A)...};937 938 typedef EHScopeStack::ConditionalCleanup<T, As...> CleanupType;939 pushCleanupAfterFullExprWithActiveFlag<CleanupType>(Kind, ActiveFlag,940 Saved);941 }942 943 template <class T, class... As>944 void pushCleanupAfterFullExprWithActiveFlag(CleanupKind Kind,945 RawAddress ActiveFlag, As... A) {946 LifetimeExtendedCleanupHeader Header = {sizeof(T), Kind,947 ActiveFlag.isValid()};948 949 size_t OldSize = LifetimeExtendedCleanupStack.size();950 LifetimeExtendedCleanupStack.resize(951 LifetimeExtendedCleanupStack.size() + sizeof(Header) + Header.Size +952 (Header.IsConditional ? sizeof(ActiveFlag) : 0));953 954 static_assert((alignof(LifetimeExtendedCleanupHeader) == alignof(T)) &&955 (alignof(T) == alignof(RawAddress)),956 "Cleanup will be allocated on misaligned address");957 char *Buffer = &LifetimeExtendedCleanupStack[OldSize];958 new (Buffer) LifetimeExtendedCleanupHeader(Header);959 new (Buffer + sizeof(Header)) T(A...);960 if (Header.IsConditional)961 new (Buffer + sizeof(Header) + sizeof(T)) RawAddress(ActiveFlag);962 }963 964 // Push a cleanup onto EHStack and deactivate it later. It is usually965 // deactivated when exiting a `CleanupDeactivationScope` (for example: after a966 // full expression).967 template <class T, class... As>968 void pushCleanupAndDeferDeactivation(CleanupKind Kind, As... A) {969 // Placeholder dominating IP for this cleanup.970 llvm::Instruction *DominatingIP =971 Builder.CreateFlagLoad(llvm::Constant::getNullValue(Int8PtrTy));972 EHStack.pushCleanup<T>(Kind, A...);973 DeferredDeactivationCleanupStack.push_back(974 {EHStack.stable_begin(), DominatingIP});975 }976 977 /// Set up the last cleanup that was pushed as a conditional978 /// full-expression cleanup.979 void initFullExprCleanup() {980 initFullExprCleanupWithFlag(createCleanupActiveFlag());981 }982 983 void initFullExprCleanupWithFlag(RawAddress ActiveFlag);984 RawAddress createCleanupActiveFlag();985 986 /// PushDestructorCleanup - Push a cleanup to call the987 /// complete-object destructor of an object of the given type at the988 /// given address. Does nothing if T is not a C++ class type with a989 /// non-trivial destructor.990 void PushDestructorCleanup(QualType T, Address Addr);991 992 /// PushDestructorCleanup - Push a cleanup to call the993 /// complete-object variant of the given destructor on the object at994 /// the given address.995 void PushDestructorCleanup(const CXXDestructorDecl *Dtor, QualType T,996 Address Addr);997 998 /// PopCleanupBlock - Will pop the cleanup entry on the stack and999 /// process all branch fixups.1000 void PopCleanupBlock(bool FallThroughIsBranchThrough = false,1001 bool ForDeactivation = false);1002 1003 /// DeactivateCleanupBlock - Deactivates the given cleanup block.1004 /// The block cannot be reactivated. Pops it if it's the top of the1005 /// stack.1006 ///1007 /// \param DominatingIP - An instruction which is known to1008 /// dominate the current IP (if set) and which lies along1009 /// all paths of execution between the current IP and the1010 /// the point at which the cleanup comes into scope.1011 void DeactivateCleanupBlock(EHScopeStack::stable_iterator Cleanup,1012 llvm::Instruction *DominatingIP);1013 1014 /// ActivateCleanupBlock - Activates an initially-inactive cleanup.1015 /// Cannot be used to resurrect a deactivated cleanup.1016 ///1017 /// \param DominatingIP - An instruction which is known to1018 /// dominate the current IP (if set) and which lies along1019 /// all paths of execution between the current IP and the1020 /// the point at which the cleanup comes into scope.1021 void ActivateCleanupBlock(EHScopeStack::stable_iterator Cleanup,1022 llvm::Instruction *DominatingIP);1023 1024 /// Enters a new scope for capturing cleanups, all of which1025 /// will be executed once the scope is exited.1026 class RunCleanupsScope {1027 EHScopeStack::stable_iterator CleanupStackDepth, OldCleanupScopeDepth;1028 size_t LifetimeExtendedCleanupStackSize;1029 CleanupDeactivationScope DeactivateCleanups;1030 bool OldDidCallStackSave;1031 1032 protected:1033 bool PerformCleanup;1034 1035 private:1036 RunCleanupsScope(const RunCleanupsScope &) = delete;1037 void operator=(const RunCleanupsScope &) = delete;1038 1039 protected:1040 CodeGenFunction &CGF;1041 1042 public:1043 /// Enter a new cleanup scope.1044 explicit RunCleanupsScope(CodeGenFunction &CGF)1045 : DeactivateCleanups(CGF), PerformCleanup(true), CGF(CGF) {1046 CleanupStackDepth = CGF.EHStack.stable_begin();1047 LifetimeExtendedCleanupStackSize =1048 CGF.LifetimeExtendedCleanupStack.size();1049 OldDidCallStackSave = CGF.DidCallStackSave;1050 CGF.DidCallStackSave = false;1051 OldCleanupScopeDepth = CGF.CurrentCleanupScopeDepth;1052 CGF.CurrentCleanupScopeDepth = CleanupStackDepth;1053 }1054 1055 /// Exit this cleanup scope, emitting any accumulated cleanups.1056 ~RunCleanupsScope() {1057 if (PerformCleanup)1058 ForceCleanup();1059 }1060 1061 /// Determine whether this scope requires any cleanups.1062 bool requiresCleanups() const {1063 return CGF.EHStack.stable_begin() != CleanupStackDepth;1064 }1065 1066 /// Force the emission of cleanups now, instead of waiting1067 /// until this object is destroyed.1068 /// \param ValuesToReload - A list of values that need to be available at1069 /// the insertion point after cleanup emission. If cleanup emission created1070 /// a shared cleanup block, these value pointers will be rewritten.1071 /// Otherwise, they not will be modified.1072 void1073 ForceCleanup(std::initializer_list<llvm::Value **> ValuesToReload = {}) {1074 assert(PerformCleanup && "Already forced cleanup");1075 CGF.DidCallStackSave = OldDidCallStackSave;1076 DeactivateCleanups.ForceDeactivate();1077 CGF.PopCleanupBlocks(CleanupStackDepth, LifetimeExtendedCleanupStackSize,1078 ValuesToReload);1079 PerformCleanup = false;1080 CGF.CurrentCleanupScopeDepth = OldCleanupScopeDepth;1081 }1082 };1083 1084 // Cleanup stack depth of the RunCleanupsScope that was pushed most recently.1085 EHScopeStack::stable_iterator CurrentCleanupScopeDepth =1086 EHScopeStack::stable_end();1087 1088 class LexicalScope : public RunCleanupsScope {1089 SourceRange Range;1090 SmallVector<const LabelDecl *, 4> Labels;1091 LexicalScope *ParentScope;1092 1093 LexicalScope(const LexicalScope &) = delete;1094 void operator=(const LexicalScope &) = delete;1095 1096 public:1097 /// Enter a new cleanup scope.1098 explicit LexicalScope(CodeGenFunction &CGF, SourceRange Range);1099 1100 void addLabel(const LabelDecl *label) {1101 assert(PerformCleanup && "adding label to dead scope?");1102 Labels.push_back(label);1103 }1104 1105 /// Exit this cleanup scope, emitting any accumulated1106 /// cleanups.1107 ~LexicalScope();1108 1109 /// Force the emission of cleanups now, instead of waiting1110 /// until this object is destroyed.1111 void ForceCleanup() {1112 CGF.CurLexicalScope = ParentScope;1113 RunCleanupsScope::ForceCleanup();1114 1115 if (!Labels.empty())1116 rescopeLabels();1117 }1118 1119 bool hasLabels() const { return !Labels.empty(); }1120 1121 void rescopeLabels();1122 };1123 1124 typedef llvm::DenseMap<const Decl *, Address> DeclMapTy;1125 1126 /// The class used to assign some variables some temporarily addresses.1127 class OMPMapVars {1128 DeclMapTy SavedLocals;1129 DeclMapTy SavedTempAddresses;1130 OMPMapVars(const OMPMapVars &) = delete;1131 void operator=(const OMPMapVars &) = delete;1132 1133 public:1134 explicit OMPMapVars() = default;1135 ~OMPMapVars() {1136 assert(SavedLocals.empty() && "Did not restored original addresses.");1137 };1138 1139 /// Sets the address of the variable \p LocalVD to be \p TempAddr in1140 /// function \p CGF.1141 /// \return true if at least one variable was set already, false otherwise.1142 bool setVarAddr(CodeGenFunction &CGF, const VarDecl *LocalVD,1143 Address TempAddr) {1144 LocalVD = LocalVD->getCanonicalDecl();1145 // Only save it once.1146 if (SavedLocals.count(LocalVD))1147 return false;1148 1149 // Copy the existing local entry to SavedLocals.1150 auto it = CGF.LocalDeclMap.find(LocalVD);1151 if (it != CGF.LocalDeclMap.end())1152 SavedLocals.try_emplace(LocalVD, it->second);1153 else1154 SavedLocals.try_emplace(LocalVD, Address::invalid());1155 1156 // Generate the private entry.1157 QualType VarTy = LocalVD->getType();1158 if (VarTy->isReferenceType()) {1159 Address Temp = CGF.CreateMemTemp(VarTy);1160 CGF.Builder.CreateStore(TempAddr.emitRawPointer(CGF), Temp);1161 TempAddr = Temp;1162 }1163 SavedTempAddresses.try_emplace(LocalVD, TempAddr);1164 1165 return true;1166 }1167 1168 /// Applies new addresses to the list of the variables.1169 /// \return true if at least one variable is using new address, false1170 /// otherwise.1171 bool apply(CodeGenFunction &CGF) {1172 copyInto(SavedTempAddresses, CGF.LocalDeclMap);1173 SavedTempAddresses.clear();1174 return !SavedLocals.empty();1175 }1176 1177 /// Restores original addresses of the variables.1178 void restore(CodeGenFunction &CGF) {1179 if (!SavedLocals.empty()) {1180 copyInto(SavedLocals, CGF.LocalDeclMap);1181 SavedLocals.clear();1182 }1183 }1184 1185 private:1186 /// Copy all the entries in the source map over the corresponding1187 /// entries in the destination, which must exist.1188 static void copyInto(const DeclMapTy &Src, DeclMapTy &Dest) {1189 for (auto &[Decl, Addr] : Src) {1190 if (!Addr.isValid())1191 Dest.erase(Decl);1192 else1193 Dest.insert_or_assign(Decl, Addr);1194 }1195 }1196 };1197 1198 /// The scope used to remap some variables as private in the OpenMP loop body1199 /// (or other captured region emitted without outlining), and to restore old1200 /// vars back on exit.1201 class OMPPrivateScope : public RunCleanupsScope {1202 OMPMapVars MappedVars;1203 OMPPrivateScope(const OMPPrivateScope &) = delete;1204 void operator=(const OMPPrivateScope &) = delete;1205 1206 public:1207 /// Enter a new OpenMP private scope.1208 explicit OMPPrivateScope(CodeGenFunction &CGF) : RunCleanupsScope(CGF) {}1209 1210 /// Registers \p LocalVD variable as a private with \p Addr as the address1211 /// of the corresponding private variable. \p1212 /// PrivateGen is the address of the generated private variable.1213 /// \return true if the variable is registered as private, false if it has1214 /// been privatized already.1215 bool addPrivate(const VarDecl *LocalVD, Address Addr) {1216 assert(PerformCleanup && "adding private to dead scope");1217 return MappedVars.setVarAddr(CGF, LocalVD, Addr);1218 }1219 1220 /// Privatizes local variables previously registered as private.1221 /// Registration is separate from the actual privatization to allow1222 /// initializers use values of the original variables, not the private one.1223 /// This is important, for example, if the private variable is a class1224 /// variable initialized by a constructor that references other private1225 /// variables. But at initialization original variables must be used, not1226 /// private copies.1227 /// \return true if at least one variable was privatized, false otherwise.1228 bool Privatize() { return MappedVars.apply(CGF); }1229 1230 void ForceCleanup() {1231 RunCleanupsScope::ForceCleanup();1232 restoreMap();1233 }1234 1235 /// Exit scope - all the mapped variables are restored.1236 ~OMPPrivateScope() {1237 if (PerformCleanup)1238 ForceCleanup();1239 }1240 1241 /// Checks if the global variable is captured in current function.1242 bool isGlobalVarCaptured(const VarDecl *VD) const {1243 VD = VD->getCanonicalDecl();1244 return !VD->isLocalVarDeclOrParm() && CGF.LocalDeclMap.count(VD) > 0;1245 }1246 1247 /// Restore all mapped variables w/o clean up. This is usefully when we want1248 /// to reference the original variables but don't want the clean up because1249 /// that could emit lifetime end too early, causing backend issue #56913.1250 void restoreMap() { MappedVars.restore(CGF); }1251 };1252 1253 /// Save/restore original map of previously emitted local vars in case when we1254 /// need to duplicate emission of the same code several times in the same1255 /// function for OpenMP code.1256 class OMPLocalDeclMapRAII {1257 CodeGenFunction &CGF;1258 DeclMapTy SavedMap;1259 1260 public:1261 OMPLocalDeclMapRAII(CodeGenFunction &CGF)1262 : CGF(CGF), SavedMap(CGF.LocalDeclMap) {}1263 ~OMPLocalDeclMapRAII() { SavedMap.swap(CGF.LocalDeclMap); }1264 };1265 1266 /// Takes the old cleanup stack size and emits the cleanup blocks1267 /// that have been added.1268 void1269 PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize,1270 std::initializer_list<llvm::Value **> ValuesToReload = {});1271 1272 /// Takes the old cleanup stack size and emits the cleanup blocks1273 /// that have been added, then adds all lifetime-extended cleanups from1274 /// the given position to the stack.1275 void1276 PopCleanupBlocks(EHScopeStack::stable_iterator OldCleanupStackSize,1277 size_t OldLifetimeExtendedStackSize,1278 std::initializer_list<llvm::Value **> ValuesToReload = {});1279 1280 void ResolveBranchFixups(llvm::BasicBlock *Target);1281 1282 /// The given basic block lies in the current EH scope, but may be a1283 /// target of a potentially scope-crossing jump; get a stable handle1284 /// to which we can perform this jump later.1285 JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target) {1286 return JumpDest(Target, EHStack.getInnermostNormalCleanup(),1287 NextCleanupDestIndex++);1288 }1289 1290 /// The given basic block lies in the current EH scope, but may be a1291 /// target of a potentially scope-crossing jump; get a stable handle1292 /// to which we can perform this jump later.1293 JumpDest getJumpDestInCurrentScope(StringRef Name = StringRef()) {1294 return getJumpDestInCurrentScope(createBasicBlock(Name));1295 }1296 1297 /// EmitBranchThroughCleanup - Emit a branch from the current insert1298 /// block through the normal cleanup handling code (if any) and then1299 /// on to \arg Dest.1300 void EmitBranchThroughCleanup(JumpDest Dest);1301 1302 /// isObviouslyBranchWithoutCleanups - Return true if a branch to the1303 /// specified destination obviously has no cleanups to run. 'false' is always1304 /// a conservatively correct answer for this method.1305 bool isObviouslyBranchWithoutCleanups(JumpDest Dest) const;1306 1307 /// popCatchScope - Pops the catch scope at the top of the EHScope1308 /// stack, emitting any required code (other than the catch handlers1309 /// themselves).1310 void popCatchScope();1311 1312 llvm::BasicBlock *getEHResumeBlock(bool isCleanup);1313 llvm::BasicBlock *getEHDispatchBlock(EHScopeStack::stable_iterator scope);1314 llvm::BasicBlock *1315 getFuncletEHDispatchBlock(EHScopeStack::stable_iterator scope);1316 1317 /// An object to manage conditionally-evaluated expressions.1318 class ConditionalEvaluation {1319 llvm::BasicBlock *StartBB;1320 1321 public:1322 ConditionalEvaluation(CodeGenFunction &CGF)1323 : StartBB(CGF.Builder.GetInsertBlock()) {}1324 1325 void begin(CodeGenFunction &CGF) {1326 assert(CGF.OutermostConditional != this);1327 if (!CGF.OutermostConditional)1328 CGF.OutermostConditional = this;1329 }1330 1331 void end(CodeGenFunction &CGF) {1332 assert(CGF.OutermostConditional != nullptr);1333 if (CGF.OutermostConditional == this)1334 CGF.OutermostConditional = nullptr;1335 }1336 1337 /// Returns a block which will be executed prior to each1338 /// evaluation of the conditional code.1339 llvm::BasicBlock *getStartingBlock() const { return StartBB; }1340 };1341 1342 /// isInConditionalBranch - Return true if we're currently emitting1343 /// one branch or the other of a conditional expression.1344 bool isInConditionalBranch() const { return OutermostConditional != nullptr; }1345 1346 void setBeforeOutermostConditional(llvm::Value *value, Address addr,1347 CodeGenFunction &CGF) {1348 assert(isInConditionalBranch());1349 llvm::BasicBlock *block = OutermostConditional->getStartingBlock();1350 auto store = new llvm::StoreInst(value, addr.emitRawPointer(CGF),1351 block->back().getIterator());1352 store->setAlignment(addr.getAlignment().getAsAlign());1353 }1354 1355 /// An RAII object to record that we're evaluating a statement1356 /// expression.1357 class StmtExprEvaluation {1358 CodeGenFunction &CGF;1359 1360 /// We have to save the outermost conditional: cleanups in a1361 /// statement expression aren't conditional just because the1362 /// StmtExpr is.1363 ConditionalEvaluation *SavedOutermostConditional;1364 1365 public:1366 StmtExprEvaluation(CodeGenFunction &CGF)1367 : CGF(CGF), SavedOutermostConditional(CGF.OutermostConditional) {1368 CGF.OutermostConditional = nullptr;1369 }1370 1371 ~StmtExprEvaluation() {1372 CGF.OutermostConditional = SavedOutermostConditional;1373 CGF.EnsureInsertPoint();1374 }1375 };1376 1377 /// An object which temporarily prevents a value from being1378 /// destroyed by aggressive peephole optimizations that assume that1379 /// all uses of a value have been realized in the IR.1380 class PeepholeProtection {1381 llvm::Instruction *Inst = nullptr;1382 friend class CodeGenFunction;1383 1384 public:1385 PeepholeProtection() = default;1386 };1387 1388 /// A non-RAII class containing all the information about a bound1389 /// opaque value. OpaqueValueMapping, below, is a RAII wrapper for1390 /// this which makes individual mappings very simple; using this1391 /// class directly is useful when you have a variable number of1392 /// opaque values or don't want the RAII functionality for some1393 /// reason.1394 class OpaqueValueMappingData {1395 const OpaqueValueExpr *OpaqueValue;1396 bool BoundLValue;1397 CodeGenFunction::PeepholeProtection Protection;1398 1399 OpaqueValueMappingData(const OpaqueValueExpr *ov, bool boundLValue)1400 : OpaqueValue(ov), BoundLValue(boundLValue) {}1401 1402 public:1403 OpaqueValueMappingData() : OpaqueValue(nullptr) {}1404 1405 static bool shouldBindAsLValue(const Expr *expr) {1406 // gl-values should be bound as l-values for obvious reasons.1407 // Records should be bound as l-values because IR generation1408 // always keeps them in memory. Expressions of function type1409 // act exactly like l-values but are formally required to be1410 // r-values in C.1411 return expr->isGLValue() || expr->getType()->isFunctionType() ||1412 hasAggregateEvaluationKind(expr->getType());1413 }1414 1415 static OpaqueValueMappingData1416 bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const Expr *e) {1417 if (shouldBindAsLValue(ov))1418 return bind(CGF, ov, CGF.EmitLValue(e));1419 return bind(CGF, ov, CGF.EmitAnyExpr(e));1420 }1421 1422 static OpaqueValueMappingData1423 bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const LValue &lv) {1424 assert(shouldBindAsLValue(ov));1425 CGF.OpaqueLValues.insert(std::make_pair(ov, lv));1426 return OpaqueValueMappingData(ov, true);1427 }1428 1429 static OpaqueValueMappingData1430 bind(CodeGenFunction &CGF, const OpaqueValueExpr *ov, const RValue &rv) {1431 assert(!shouldBindAsLValue(ov));1432 CGF.OpaqueRValues.insert(std::make_pair(ov, rv));1433 1434 OpaqueValueMappingData data(ov, false);1435 1436 // Work around an extremely aggressive peephole optimization in1437 // EmitScalarConversion which assumes that all other uses of a1438 // value are extant.1439 data.Protection = CGF.protectFromPeepholes(rv);1440 1441 return data;1442 }1443 1444 bool isValid() const { return OpaqueValue != nullptr; }1445 void clear() { OpaqueValue = nullptr; }1446 1447 void unbind(CodeGenFunction &CGF) {1448 assert(OpaqueValue && "no data to unbind!");1449 1450 if (BoundLValue) {1451 CGF.OpaqueLValues.erase(OpaqueValue);1452 } else {1453 CGF.OpaqueRValues.erase(OpaqueValue);1454 CGF.unprotectFromPeepholes(Protection);1455 }1456 }1457 };1458 1459 /// An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.1460 class OpaqueValueMapping {1461 CodeGenFunction &CGF;1462 OpaqueValueMappingData Data;1463 1464 public:1465 static bool shouldBindAsLValue(const Expr *expr) {1466 return OpaqueValueMappingData::shouldBindAsLValue(expr);1467 }1468 1469 /// Build the opaque value mapping for the given conditional1470 /// operator if it's the GNU ?: extension. This is a common1471 /// enough pattern that the convenience operator is really1472 /// helpful.1473 ///1474 OpaqueValueMapping(CodeGenFunction &CGF,1475 const AbstractConditionalOperator *op)1476 : CGF(CGF) {1477 if (isa<ConditionalOperator>(op))1478 // Leave Data empty.1479 return;1480 1481 const BinaryConditionalOperator *e = cast<BinaryConditionalOperator>(op);1482 Data = OpaqueValueMappingData::bind(CGF, e->getOpaqueValue(),1483 e->getCommon());1484 }1485 1486 /// Build the opaque value mapping for an OpaqueValueExpr whose source1487 /// expression is set to the expression the OVE represents.1488 OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *OV)1489 : CGF(CGF) {1490 if (OV) {1491 assert(OV->getSourceExpr() && "wrong form of OpaqueValueMapping used "1492 "for OVE with no source expression");1493 Data = OpaqueValueMappingData::bind(CGF, OV, OV->getSourceExpr());1494 }1495 }1496 1497 OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *opaqueValue,1498 LValue lvalue)1499 : CGF(CGF),1500 Data(OpaqueValueMappingData::bind(CGF, opaqueValue, lvalue)) {}1501 1502 OpaqueValueMapping(CodeGenFunction &CGF, const OpaqueValueExpr *opaqueValue,1503 RValue rvalue)1504 : CGF(CGF),1505 Data(OpaqueValueMappingData::bind(CGF, opaqueValue, rvalue)) {}1506 1507 void pop() {1508 Data.unbind(CGF);1509 Data.clear();1510 }1511 1512 ~OpaqueValueMapping() {1513 if (Data.isValid())1514 Data.unbind(CGF);1515 }1516 };1517 1518private:1519 CGDebugInfo *DebugInfo;1520 /// Used to create unique names for artificial VLA size debug info variables.1521 unsigned VLAExprCounter = 0;1522 bool DisableDebugInfo = false;1523 1524 /// DidCallStackSave - Whether llvm.stacksave has been called. Used to avoid1525 /// calling llvm.stacksave for multiple VLAs in the same scope.1526 bool DidCallStackSave = false;1527 1528 /// IndirectBranch - The first time an indirect goto is seen we create a block1529 /// with an indirect branch. Every time we see the address of a label taken,1530 /// we add the label to the indirect goto. Every subsequent indirect goto is1531 /// codegen'd as a jump to the IndirectBranch's basic block.1532 llvm::IndirectBrInst *IndirectBranch = nullptr;1533 1534 /// LocalDeclMap - This keeps track of the LLVM allocas or globals for local C1535 /// decls.1536 DeclMapTy LocalDeclMap;1537 1538 // Keep track of the cleanups for callee-destructed parameters pushed to the1539 // cleanup stack so that they can be deactivated later.1540 llvm::DenseMap<const ParmVarDecl *, EHScopeStack::stable_iterator>1541 CalleeDestructedParamCleanups;1542 1543 /// SizeArguments - If a ParmVarDecl had the pass_object_size attribute, this1544 /// will contain a mapping from said ParmVarDecl to its implicit "object_size"1545 /// parameter.1546 llvm::SmallDenseMap<const ParmVarDecl *, const ImplicitParamDecl *, 2>1547 SizeArguments;1548 1549 /// Track escaped local variables with auto storage. Used during SEH1550 /// outlining to produce a call to llvm.localescape.1551 llvm::DenseMap<llvm::AllocaInst *, int> EscapedLocals;1552 1553 /// LabelMap - This keeps track of the LLVM basic block for each C label.1554 llvm::DenseMap<const LabelDecl *, JumpDest> LabelMap;1555 1556 // BreakContinueStack - This keeps track of where break and continue1557 // statements should jump to.1558 struct BreakContinue {1559 BreakContinue(const Stmt &LoopOrSwitch, JumpDest Break, JumpDest Continue)1560 : LoopOrSwitch(&LoopOrSwitch), BreakBlock(Break),1561 ContinueBlock(Continue) {}1562 1563 const Stmt *LoopOrSwitch;1564 JumpDest BreakBlock;1565 JumpDest ContinueBlock;1566 };1567 SmallVector<BreakContinue, 8> BreakContinueStack;1568 1569 /// Handles cancellation exit points in OpenMP-related constructs.1570 class OpenMPCancelExitStack {1571 /// Tracks cancellation exit point and join point for cancel-related exit1572 /// and normal exit.1573 struct CancelExit {1574 CancelExit() = default;1575 CancelExit(OpenMPDirectiveKind Kind, JumpDest ExitBlock,1576 JumpDest ContBlock)1577 : Kind(Kind), ExitBlock(ExitBlock), ContBlock(ContBlock) {}1578 OpenMPDirectiveKind Kind = llvm::omp::OMPD_unknown;1579 /// true if the exit block has been emitted already by the special1580 /// emitExit() call, false if the default codegen is used.1581 bool HasBeenEmitted = false;1582 JumpDest ExitBlock;1583 JumpDest ContBlock;1584 };1585 1586 SmallVector<CancelExit, 8> Stack;1587 1588 public:1589 OpenMPCancelExitStack() : Stack(1) {}1590 ~OpenMPCancelExitStack() = default;1591 /// Fetches the exit block for the current OpenMP construct.1592 JumpDest getExitBlock() const { return Stack.back().ExitBlock; }1593 /// Emits exit block with special codegen procedure specific for the related1594 /// OpenMP construct + emits code for normal construct cleanup.1595 void emitExit(CodeGenFunction &CGF, OpenMPDirectiveKind Kind,1596 const llvm::function_ref<void(CodeGenFunction &)> CodeGen) {1597 if (Stack.back().Kind == Kind && getExitBlock().isValid()) {1598 assert(CGF.getOMPCancelDestination(Kind).isValid());1599 assert(CGF.HaveInsertPoint());1600 assert(!Stack.back().HasBeenEmitted);1601 auto IP = CGF.Builder.saveAndClearIP();1602 CGF.EmitBlock(Stack.back().ExitBlock.getBlock());1603 CodeGen(CGF);1604 CGF.EmitBranch(Stack.back().ContBlock.getBlock());1605 CGF.Builder.restoreIP(IP);1606 Stack.back().HasBeenEmitted = true;1607 }1608 CodeGen(CGF);1609 }1610 /// Enter the cancel supporting \a Kind construct.1611 /// \param Kind OpenMP directive that supports cancel constructs.1612 /// \param HasCancel true, if the construct has inner cancel directive,1613 /// false otherwise.1614 void enter(CodeGenFunction &CGF, OpenMPDirectiveKind Kind, bool HasCancel) {1615 Stack.push_back({Kind,1616 HasCancel ? CGF.getJumpDestInCurrentScope("cancel.exit")1617 : JumpDest(),1618 HasCancel ? CGF.getJumpDestInCurrentScope("cancel.cont")1619 : JumpDest()});1620 }1621 /// Emits default exit point for the cancel construct (if the special one1622 /// has not be used) + join point for cancel/normal exits.1623 void exit(CodeGenFunction &CGF) {1624 if (getExitBlock().isValid()) {1625 assert(CGF.getOMPCancelDestination(Stack.back().Kind).isValid());1626 bool HaveIP = CGF.HaveInsertPoint();1627 if (!Stack.back().HasBeenEmitted) {1628 if (HaveIP)1629 CGF.EmitBranchThroughCleanup(Stack.back().ContBlock);1630 CGF.EmitBlock(Stack.back().ExitBlock.getBlock());1631 CGF.EmitBranchThroughCleanup(Stack.back().ContBlock);1632 }1633 CGF.EmitBlock(Stack.back().ContBlock.getBlock());1634 if (!HaveIP) {1635 CGF.Builder.CreateUnreachable();1636 CGF.Builder.ClearInsertionPoint();1637 }1638 }1639 Stack.pop_back();1640 }1641 };1642 OpenMPCancelExitStack OMPCancelStack;1643 1644 /// Lower the Likelihood knowledge about the \p Cond via llvm.expect intrin.1645 llvm::Value *emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,1646 Stmt::Likelihood LH);1647 1648 std::unique_ptr<CodeGenPGO> PGO;1649 1650 /// Bitmap used by MC/DC to track condition outcomes of a boolean expression.1651 Address MCDCCondBitmapAddr = Address::invalid();1652 1653 /// Calculate branch weights appropriate for PGO data1654 llvm::MDNode *createProfileWeights(uint64_t TrueCount,1655 uint64_t FalseCount) const;1656 llvm::MDNode *createProfileWeights(ArrayRef<uint64_t> Weights) const;1657 llvm::MDNode *createProfileWeightsForLoop(const Stmt *Cond,1658 uint64_t LoopCount) const;1659 1660public:1661 std::pair<bool, bool> getIsCounterPair(const Stmt *S) const;1662 void markStmtAsUsed(bool Skipped, const Stmt *S);1663 void markStmtMaybeUsed(const Stmt *S);1664 1665 /// Increment the profiler's counter for the given statement by \p StepV.1666 /// If \p StepV is null, the default increment is 1.1667 void incrementProfileCounter(const Stmt *S, llvm::Value *StepV = nullptr);1668 1669 bool isMCDCCoverageEnabled() const {1670 return (CGM.getCodeGenOpts().hasProfileClangInstr() &&1671 CGM.getCodeGenOpts().MCDCCoverage &&1672 !CurFn->hasFnAttribute(llvm::Attribute::NoProfile));1673 }1674 1675 /// Allocate a temp value on the stack that MCDC can use to track condition1676 /// results.1677 void maybeCreateMCDCCondBitmap();1678 1679 bool isBinaryLogicalOp(const Expr *E) const {1680 const BinaryOperator *BOp = dyn_cast<BinaryOperator>(E->IgnoreParens());1681 return (BOp && BOp->isLogicalOp());1682 }1683 1684 /// Zero-init the MCDC temp value.1685 void maybeResetMCDCCondBitmap(const Expr *E);1686 1687 /// Increment the profiler's counter for the given expression by \p StepV.1688 /// If \p StepV is null, the default increment is 1.1689 void maybeUpdateMCDCTestVectorBitmap(const Expr *E);1690 1691 /// Update the MCDC temp value with the condition's evaluated result.1692 void maybeUpdateMCDCCondBitmap(const Expr *E, llvm::Value *Val);1693 1694 /// Get the profiler's count for the given statement.1695 uint64_t getProfileCount(const Stmt *S);1696 1697 /// Set the profiler's current count.1698 void setCurrentProfileCount(uint64_t Count);1699 1700 /// Get the profiler's current count. This is generally the count for the most1701 /// recently incremented counter.1702 uint64_t getCurrentProfileCount();1703 1704 /// See CGDebugInfo::addInstToCurrentSourceAtom.1705 void addInstToCurrentSourceAtom(llvm::Instruction *KeyInstruction,1706 llvm::Value *Backup);1707 1708 /// See CGDebugInfo::addInstToSpecificSourceAtom.1709 void addInstToSpecificSourceAtom(llvm::Instruction *KeyInstruction,1710 llvm::Value *Backup, uint64_t Atom);1711 1712 /// Add \p KeyInstruction and an optional \p Backup instruction to a new atom1713 /// group (See ApplyAtomGroup for more info).1714 void addInstToNewSourceAtom(llvm::Instruction *KeyInstruction,1715 llvm::Value *Backup);1716 1717private:1718 /// SwitchInsn - This is nearest current switch instruction. It is null if1719 /// current context is not in a switch.1720 llvm::SwitchInst *SwitchInsn = nullptr;1721 /// The branch weights of SwitchInsn when doing instrumentation based PGO.1722 SmallVector<uint64_t, 16> *SwitchWeights = nullptr;1723 1724 /// The likelihood attributes of the SwitchCase.1725 SmallVector<Stmt::Likelihood, 16> *SwitchLikelihood = nullptr;1726 1727 /// CaseRangeBlock - This block holds if condition check for last case1728 /// statement range in current switch instruction.1729 llvm::BasicBlock *CaseRangeBlock = nullptr;1730 1731 /// OpaqueLValues - Keeps track of the current set of opaque value1732 /// expressions.1733 llvm::DenseMap<const OpaqueValueExpr *, LValue> OpaqueLValues;1734 llvm::DenseMap<const OpaqueValueExpr *, RValue> OpaqueRValues;1735 1736 // VLASizeMap - This keeps track of the associated size for each VLA type.1737 // We track this by the size expression rather than the type itself because1738 // in certain situations, like a const qualifier applied to an VLA typedef,1739 // multiple VLA types can share the same size expression.1740 // FIXME: Maybe this could be a stack of maps that is pushed/popped as we1741 // enter/leave scopes.1742 llvm::DenseMap<const Expr *, llvm::Value *> VLASizeMap;1743 1744 /// A block containing a single 'unreachable' instruction. Created1745 /// lazily by getUnreachableBlock().1746 llvm::BasicBlock *UnreachableBlock = nullptr;1747 1748 /// Counts of the number return expressions in the function.1749 unsigned NumReturnExprs = 0;1750 1751 /// Count the number of simple (constant) return expressions in the function.1752 unsigned NumSimpleReturnExprs = 0;1753 1754 /// The last regular (non-return) debug location (breakpoint) in the function.1755 SourceLocation LastStopPoint;1756 1757public:1758 /// Source location information about the default argument or member1759 /// initializer expression we're evaluating, if any.1760 CurrentSourceLocExprScope CurSourceLocExprScope;1761 using SourceLocExprScopeGuard =1762 CurrentSourceLocExprScope::SourceLocExprScopeGuard;1763 1764 /// A scope within which we are constructing the fields of an object which1765 /// might use a CXXDefaultInitExpr. This stashes away a 'this' value to use1766 /// if we need to evaluate a CXXDefaultInitExpr within the evaluation.1767 class FieldConstructionScope {1768 public:1769 FieldConstructionScope(CodeGenFunction &CGF, Address This)1770 : CGF(CGF), OldCXXDefaultInitExprThis(CGF.CXXDefaultInitExprThis) {1771 CGF.CXXDefaultInitExprThis = This;1772 }1773 ~FieldConstructionScope() {1774 CGF.CXXDefaultInitExprThis = OldCXXDefaultInitExprThis;1775 }1776 1777 private:1778 CodeGenFunction &CGF;1779 Address OldCXXDefaultInitExprThis;1780 };1781 1782 /// The scope of a CXXDefaultInitExpr. Within this scope, the value of 'this'1783 /// is overridden to be the object under construction.1784 class CXXDefaultInitExprScope {1785 public:1786 CXXDefaultInitExprScope(CodeGenFunction &CGF, const CXXDefaultInitExpr *E)1787 : CGF(CGF), OldCXXThisValue(CGF.CXXThisValue),1788 OldCXXThisAlignment(CGF.CXXThisAlignment),1789 SourceLocScope(E, CGF.CurSourceLocExprScope) {1790 CGF.CXXThisValue = CGF.CXXDefaultInitExprThis.getBasePointer();1791 CGF.CXXThisAlignment = CGF.CXXDefaultInitExprThis.getAlignment();1792 }1793 ~CXXDefaultInitExprScope() {1794 CGF.CXXThisValue = OldCXXThisValue;1795 CGF.CXXThisAlignment = OldCXXThisAlignment;1796 }1797 1798 public:1799 CodeGenFunction &CGF;1800 llvm::Value *OldCXXThisValue;1801 CharUnits OldCXXThisAlignment;1802 SourceLocExprScopeGuard SourceLocScope;1803 };1804 1805 struct CXXDefaultArgExprScope : SourceLocExprScopeGuard {1806 CXXDefaultArgExprScope(CodeGenFunction &CGF, const CXXDefaultArgExpr *E)1807 : SourceLocExprScopeGuard(E, CGF.CurSourceLocExprScope) {}1808 };1809 1810 /// The scope of an ArrayInitLoopExpr. Within this scope, the value of the1811 /// current loop index is overridden.1812 class ArrayInitLoopExprScope {1813 public:1814 ArrayInitLoopExprScope(CodeGenFunction &CGF, llvm::Value *Index)1815 : CGF(CGF), OldArrayInitIndex(CGF.ArrayInitIndex) {1816 CGF.ArrayInitIndex = Index;1817 }1818 ~ArrayInitLoopExprScope() { CGF.ArrayInitIndex = OldArrayInitIndex; }1819 1820 private:1821 CodeGenFunction &CGF;1822 llvm::Value *OldArrayInitIndex;1823 };1824 1825 class InlinedInheritingConstructorScope {1826 public:1827 InlinedInheritingConstructorScope(CodeGenFunction &CGF, GlobalDecl GD)1828 : CGF(CGF), OldCurGD(CGF.CurGD), OldCurFuncDecl(CGF.CurFuncDecl),1829 OldCurCodeDecl(CGF.CurCodeDecl),1830 OldCXXABIThisDecl(CGF.CXXABIThisDecl),1831 OldCXXABIThisValue(CGF.CXXABIThisValue),1832 OldCXXThisValue(CGF.CXXThisValue),1833 OldCXXABIThisAlignment(CGF.CXXABIThisAlignment),1834 OldCXXThisAlignment(CGF.CXXThisAlignment),1835 OldReturnValue(CGF.ReturnValue), OldFnRetTy(CGF.FnRetTy),1836 OldCXXInheritedCtorInitExprArgs(1837 std::move(CGF.CXXInheritedCtorInitExprArgs)) {1838 CGF.CurGD = GD;1839 CGF.CurFuncDecl = CGF.CurCodeDecl =1840 cast<CXXConstructorDecl>(GD.getDecl());1841 CGF.CXXABIThisDecl = nullptr;1842 CGF.CXXABIThisValue = nullptr;1843 CGF.CXXThisValue = nullptr;1844 CGF.CXXABIThisAlignment = CharUnits();1845 CGF.CXXThisAlignment = CharUnits();1846 CGF.ReturnValue = Address::invalid();1847 CGF.FnRetTy = QualType();1848 CGF.CXXInheritedCtorInitExprArgs.clear();1849 }1850 ~InlinedInheritingConstructorScope() {1851 CGF.CurGD = OldCurGD;1852 CGF.CurFuncDecl = OldCurFuncDecl;1853 CGF.CurCodeDecl = OldCurCodeDecl;1854 CGF.CXXABIThisDecl = OldCXXABIThisDecl;1855 CGF.CXXABIThisValue = OldCXXABIThisValue;1856 CGF.CXXThisValue = OldCXXThisValue;1857 CGF.CXXABIThisAlignment = OldCXXABIThisAlignment;1858 CGF.CXXThisAlignment = OldCXXThisAlignment;1859 CGF.ReturnValue = OldReturnValue;1860 CGF.FnRetTy = OldFnRetTy;1861 CGF.CXXInheritedCtorInitExprArgs =1862 std::move(OldCXXInheritedCtorInitExprArgs);1863 }1864 1865 private:1866 CodeGenFunction &CGF;1867 GlobalDecl OldCurGD;1868 const Decl *OldCurFuncDecl;1869 const Decl *OldCurCodeDecl;1870 ImplicitParamDecl *OldCXXABIThisDecl;1871 llvm::Value *OldCXXABIThisValue;1872 llvm::Value *OldCXXThisValue;1873 CharUnits OldCXXABIThisAlignment;1874 CharUnits OldCXXThisAlignment;1875 Address OldReturnValue;1876 QualType OldFnRetTy;1877 CallArgList OldCXXInheritedCtorInitExprArgs;1878 };1879 1880 // Helper class for the OpenMP IR Builder. Allows reusability of code used for1881 // region body, and finalization codegen callbacks. This will class will also1882 // contain privatization functions used by the privatization call backs1883 //1884 // TODO: this is temporary class for things that are being moved out of1885 // CGOpenMPRuntime, new versions of current CodeGenFunction methods, or1886 // utility function for use with the OMPBuilder. Once that move to use the1887 // OMPBuilder is done, everything here will either become part of CodeGenFunc.1888 // directly, or a new helper class that will contain functions used by both1889 // this and the OMPBuilder1890 1891 struct OMPBuilderCBHelpers {1892 1893 OMPBuilderCBHelpers() = delete;1894 OMPBuilderCBHelpers(const OMPBuilderCBHelpers &) = delete;1895 OMPBuilderCBHelpers &operator=(const OMPBuilderCBHelpers &) = delete;1896 1897 using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;1898 1899 /// Cleanup action for allocate support.1900 class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup {1901 1902 private:1903 llvm::CallInst *RTLFnCI;1904 1905 public:1906 OMPAllocateCleanupTy(llvm::CallInst *RLFnCI) : RTLFnCI(RLFnCI) {1907 RLFnCI->removeFromParent();1908 }1909 1910 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override {1911 if (!CGF.HaveInsertPoint())1912 return;1913 CGF.Builder.Insert(RTLFnCI);1914 }1915 };1916 1917 /// Returns address of the threadprivate variable for the current1918 /// thread. This Also create any necessary OMP runtime calls.1919 ///1920 /// \param VD VarDecl for Threadprivate variable.1921 /// \param VDAddr Address of the Vardecl1922 /// \param Loc The location where the barrier directive was encountered1923 static Address getAddrOfThreadPrivate(CodeGenFunction &CGF,1924 const VarDecl *VD, Address VDAddr,1925 SourceLocation Loc);1926 1927 /// Gets the OpenMP-specific address of the local variable /p VD.1928 static Address getAddressOfLocalVariable(CodeGenFunction &CGF,1929 const VarDecl *VD);1930 /// Get the platform-specific name separator.1931 /// \param Parts different parts of the final name that needs separation1932 /// \param FirstSeparator First separator used between the initial two1933 /// parts of the name.1934 /// \param Separator separator used between all of the rest consecutinve1935 /// parts of the name1936 static std::string getNameWithSeparators(ArrayRef<StringRef> Parts,1937 StringRef FirstSeparator = ".",1938 StringRef Separator = ".");1939 /// Emit the Finalization for an OMP region1940 /// \param CGF The Codegen function this belongs to1941 /// \param IP Insertion point for generating the finalization code.1942 static void FinalizeOMPRegion(CodeGenFunction &CGF, InsertPointTy IP) {1943 CGBuilderTy::InsertPointGuard IPG(CGF.Builder);1944 assert(IP.getBlock()->end() != IP.getPoint() &&1945 "OpenMP IR Builder should cause terminated block!");1946 1947 llvm::BasicBlock *IPBB = IP.getBlock();1948 llvm::BasicBlock *DestBB = IPBB->getUniqueSuccessor();1949 assert(DestBB && "Finalization block should have one successor!");1950 1951 // erase and replace with cleanup branch.1952 IPBB->getTerminator()->eraseFromParent();1953 CGF.Builder.SetInsertPoint(IPBB);1954 CodeGenFunction::JumpDest Dest = CGF.getJumpDestInCurrentScope(DestBB);1955 CGF.EmitBranchThroughCleanup(Dest);1956 }1957 1958 /// Emit the body of an OMP region1959 /// \param CGF The Codegen function this belongs to1960 /// \param RegionBodyStmt The body statement for the OpenMP region being1961 /// generated1962 /// \param AllocaIP Where to insert alloca instructions1963 /// \param CodeGenIP Where to insert the region code1964 /// \param RegionName Name to be used for new blocks1965 static void EmitOMPInlinedRegionBody(CodeGenFunction &CGF,1966 const Stmt *RegionBodyStmt,1967 InsertPointTy AllocaIP,1968 InsertPointTy CodeGenIP,1969 Twine RegionName);1970 1971 static void EmitCaptureStmt(CodeGenFunction &CGF, InsertPointTy CodeGenIP,1972 llvm::BasicBlock &FiniBB, llvm::Function *Fn,1973 ArrayRef<llvm::Value *> Args) {1974 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();1975 if (llvm::Instruction *CodeGenIPBBTI = CodeGenIPBB->getTerminator())1976 CodeGenIPBBTI->eraseFromParent();1977 1978 CGF.Builder.SetInsertPoint(CodeGenIPBB);1979 1980 if (Fn->doesNotThrow())1981 CGF.EmitNounwindRuntimeCall(Fn, Args);1982 else1983 CGF.EmitRuntimeCall(Fn, Args);1984 1985 if (CGF.Builder.saveIP().isSet())1986 CGF.Builder.CreateBr(&FiniBB);1987 }1988 1989 /// Emit the body of an OMP region that will be outlined in1990 /// OpenMPIRBuilder::finalize().1991 /// \param CGF The Codegen function this belongs to1992 /// \param RegionBodyStmt The body statement for the OpenMP region being1993 /// generated1994 /// \param AllocaIP Where to insert alloca instructions1995 /// \param CodeGenIP Where to insert the region code1996 /// \param RegionName Name to be used for new blocks1997 static void EmitOMPOutlinedRegionBody(CodeGenFunction &CGF,1998 const Stmt *RegionBodyStmt,1999 InsertPointTy AllocaIP,2000 InsertPointTy CodeGenIP,2001 Twine RegionName);2002 2003 /// RAII for preserving necessary info during Outlined region body codegen.2004 class OutlinedRegionBodyRAII {2005 2006 llvm::AssertingVH<llvm::Instruction> OldAllocaIP;2007 CodeGenFunction::JumpDest OldReturnBlock;2008 CodeGenFunction &CGF;2009 2010 public:2011 OutlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,2012 llvm::BasicBlock &RetBB)2013 : CGF(cgf) {2014 assert(AllocaIP.isSet() &&2015 "Must specify Insertion point for allocas of outlined function");2016 OldAllocaIP = CGF.AllocaInsertPt;2017 CGF.AllocaInsertPt = &*AllocaIP.getPoint();2018 2019 OldReturnBlock = CGF.ReturnBlock;2020 CGF.ReturnBlock = CGF.getJumpDestInCurrentScope(&RetBB);2021 }2022 2023 ~OutlinedRegionBodyRAII() {2024 CGF.AllocaInsertPt = OldAllocaIP;2025 CGF.ReturnBlock = OldReturnBlock;2026 }2027 };2028 2029 /// RAII for preserving necessary info during inlined region body codegen.2030 class InlinedRegionBodyRAII {2031 2032 llvm::AssertingVH<llvm::Instruction> OldAllocaIP;2033 CodeGenFunction &CGF;2034 2035 public:2036 InlinedRegionBodyRAII(CodeGenFunction &cgf, InsertPointTy &AllocaIP,2037 llvm::BasicBlock &FiniBB)2038 : CGF(cgf) {2039 // Alloca insertion block should be in the entry block of the containing2040 // function so it expects an empty AllocaIP in which case will reuse the2041 // old alloca insertion point, or a new AllocaIP in the same block as2042 // the old one2043 assert((!AllocaIP.isSet() ||2044 CGF.AllocaInsertPt->getParent() == AllocaIP.getBlock()) &&2045 "Insertion point should be in the entry block of containing "2046 "function!");2047 OldAllocaIP = CGF.AllocaInsertPt;2048 if (AllocaIP.isSet())2049 CGF.AllocaInsertPt = &*AllocaIP.getPoint();2050 2051 // TODO: Remove the call, after making sure the counter is not used by2052 // the EHStack.2053 // Since this is an inlined region, it should not modify the2054 // ReturnBlock, and should reuse the one for the enclosing outlined2055 // region. So, the JumpDest being return by the function is discarded2056 (void)CGF.getJumpDestInCurrentScope(&FiniBB);2057 }2058 2059 ~InlinedRegionBodyRAII() { CGF.AllocaInsertPt = OldAllocaIP; }2060 };2061 };2062 2063private:2064 /// CXXThisDecl - When generating code for a C++ member function,2065 /// this will hold the implicit 'this' declaration.2066 ImplicitParamDecl *CXXABIThisDecl = nullptr;2067 llvm::Value *CXXABIThisValue = nullptr;2068 llvm::Value *CXXThisValue = nullptr;2069 CharUnits CXXABIThisAlignment;2070 CharUnits CXXThisAlignment;2071 2072 /// The value of 'this' to use when evaluating CXXDefaultInitExprs within2073 /// this expression.2074 Address CXXDefaultInitExprThis = Address::invalid();2075 2076 /// The current array initialization index when evaluating an2077 /// ArrayInitIndexExpr within an ArrayInitLoopExpr.2078 llvm::Value *ArrayInitIndex = nullptr;2079 2080 /// The values of function arguments to use when evaluating2081 /// CXXInheritedCtorInitExprs within this context.2082 CallArgList CXXInheritedCtorInitExprArgs;2083 2084 /// CXXStructorImplicitParamDecl - When generating code for a constructor or2085 /// destructor, this will hold the implicit argument (e.g. VTT).2086 ImplicitParamDecl *CXXStructorImplicitParamDecl = nullptr;2087 llvm::Value *CXXStructorImplicitParamValue = nullptr;2088 2089 /// OutermostConditional - Points to the outermost active2090 /// conditional control. This is used so that we know if a2091 /// temporary should be destroyed conditionally.2092 ConditionalEvaluation *OutermostConditional = nullptr;2093 2094 /// The current lexical scope.2095 LexicalScope *CurLexicalScope = nullptr;2096 2097 /// The current source location that should be used for exception2098 /// handling code.2099 SourceLocation CurEHLocation;2100 2101 /// BlockByrefInfos - For each __block variable, contains2102 /// information about the layout of the variable.2103 llvm::DenseMap<const ValueDecl *, BlockByrefInfo> BlockByrefInfos;2104 2105 /// Used by -fsanitize=nullability-return to determine whether the return2106 /// value can be checked.2107 llvm::Value *RetValNullabilityPrecondition = nullptr;2108 2109 /// Check if -fsanitize=nullability-return instrumentation is required for2110 /// this function.2111 bool requiresReturnValueNullabilityCheck() const {2112 return RetValNullabilityPrecondition;2113 }2114 2115 /// Used to store precise source locations for return statements by the2116 /// runtime return value checks.2117 Address ReturnLocation = Address::invalid();2118 2119 /// Check if the return value of this function requires sanitization.2120 bool requiresReturnValueCheck() const;2121 2122 bool isInAllocaArgument(CGCXXABI &ABI, QualType Ty);2123 bool hasInAllocaArg(const CXXMethodDecl *MD);2124 2125 llvm::BasicBlock *TerminateLandingPad = nullptr;2126 llvm::BasicBlock *TerminateHandler = nullptr;2127 llvm::SmallVector<llvm::BasicBlock *, 2> TrapBBs;2128 2129 /// Terminate funclets keyed by parent funclet pad.2130 llvm::MapVector<llvm::Value *, llvm::BasicBlock *> TerminateFunclets;2131 2132 /// Largest vector width used in ths function. Will be used to create a2133 /// function attribute.2134 unsigned LargestVectorWidth = 0;2135 2136 /// True if we need emit the life-time markers. This is initially set in2137 /// the constructor, but could be overwritten to true if this is a coroutine.2138 bool ShouldEmitLifetimeMarkers;2139 2140 /// Add OpenCL kernel arg metadata and the kernel attribute metadata to2141 /// the function metadata.2142 void EmitKernelMetadata(const FunctionDecl *FD, llvm::Function *Fn);2143 2144public:2145 CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext = false);2146 ~CodeGenFunction();2147 2148 CodeGenTypes &getTypes() const { return CGM.getTypes(); }2149 ASTContext &getContext() const { return CGM.getContext(); }2150 CGDebugInfo *getDebugInfo() {2151 if (DisableDebugInfo)2152 return nullptr;2153 return DebugInfo;2154 }2155 void disableDebugInfo() { DisableDebugInfo = true; }2156 void enableDebugInfo() { DisableDebugInfo = false; }2157 2158 bool shouldUseFusedARCCalls() {2159 return CGM.getCodeGenOpts().OptimizationLevel == 0;2160 }2161 2162 const LangOptions &getLangOpts() const { return CGM.getLangOpts(); }2163 2164 /// Returns a pointer to the function's exception object and selector slot,2165 /// which is assigned in every landing pad.2166 Address getExceptionSlot();2167 Address getEHSelectorSlot();2168 2169 /// Returns the contents of the function's exception object and selector2170 /// slots.2171 llvm::Value *getExceptionFromSlot();2172 llvm::Value *getSelectorFromSlot();2173 2174 RawAddress getNormalCleanupDestSlot();2175 2176 llvm::BasicBlock *getUnreachableBlock() {2177 if (!UnreachableBlock) {2178 UnreachableBlock = createBasicBlock("unreachable");2179 new llvm::UnreachableInst(getLLVMContext(), UnreachableBlock);2180 }2181 return UnreachableBlock;2182 }2183 2184 llvm::BasicBlock *getInvokeDest() {2185 if (!EHStack.requiresLandingPad())2186 return nullptr;2187 return getInvokeDestImpl();2188 }2189 2190 bool currentFunctionUsesSEHTry() const { return !!CurSEHParent; }2191 2192 const TargetInfo &getTarget() const { return Target; }2193 llvm::LLVMContext &getLLVMContext() { return CGM.getLLVMContext(); }2194 const TargetCodeGenInfo &getTargetHooks() const {2195 return CGM.getTargetCodeGenInfo();2196 }2197 2198 //===--------------------------------------------------------------------===//2199 // Cleanups2200 //===--------------------------------------------------------------------===//2201 2202 typedef void Destroyer(CodeGenFunction &CGF, Address addr, QualType ty);2203 2204 void pushIrregularPartialArrayCleanup(llvm::Value *arrayBegin,2205 Address arrayEndPointer,2206 QualType elementType,2207 CharUnits elementAlignment,2208 Destroyer *destroyer);2209 void pushRegularPartialArrayCleanup(llvm::Value *arrayBegin,2210 llvm::Value *arrayEnd,2211 QualType elementType,2212 CharUnits elementAlignment,2213 Destroyer *destroyer);2214 2215 void pushDestroy(QualType::DestructionKind dtorKind, Address addr,2216 QualType type);2217 void pushEHDestroy(QualType::DestructionKind dtorKind, Address addr,2218 QualType type);2219 void pushDestroy(CleanupKind kind, Address addr, QualType type,2220 Destroyer *destroyer, bool useEHCleanupForArray);2221 void pushDestroyAndDeferDeactivation(QualType::DestructionKind dtorKind,2222 Address addr, QualType type);2223 void pushDestroyAndDeferDeactivation(CleanupKind cleanupKind, Address addr,2224 QualType type, Destroyer *destroyer,2225 bool useEHCleanupForArray);2226 void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr,2227 QualType type, Destroyer *destroyer,2228 bool useEHCleanupForArray);2229 void pushLifetimeExtendedDestroy(QualType::DestructionKind dtorKind,2230 Address addr, QualType type);2231 void pushCallObjectDeleteCleanup(const FunctionDecl *OperatorDelete,2232 llvm::Value *CompletePtr,2233 QualType ElementType);2234 void pushStackRestore(CleanupKind kind, Address SPMem);2235 void pushKmpcAllocFree(CleanupKind Kind,2236 std::pair<llvm::Value *, llvm::Value *> AddrSizePair);2237 void emitDestroy(Address addr, QualType type, Destroyer *destroyer,2238 bool useEHCleanupForArray);2239 llvm::Function *generateDestroyHelper(Address addr, QualType type,2240 Destroyer *destroyer,2241 bool useEHCleanupForArray,2242 const VarDecl *VD);2243 void emitArrayDestroy(llvm::Value *begin, llvm::Value *end,2244 QualType elementType, CharUnits elementAlign,2245 Destroyer *destroyer, bool checkZeroLength,2246 bool useEHCleanup);2247 2248 Destroyer *getDestroyer(QualType::DestructionKind destructionKind);2249 2250 /// Determines whether an EH cleanup is required to destroy a type2251 /// with the given destruction kind.2252 bool needsEHCleanup(QualType::DestructionKind kind) {2253 switch (kind) {2254 case QualType::DK_none:2255 return false;2256 case QualType::DK_cxx_destructor:2257 case QualType::DK_objc_weak_lifetime:2258 case QualType::DK_nontrivial_c_struct:2259 return getLangOpts().Exceptions;2260 case QualType::DK_objc_strong_lifetime:2261 return getLangOpts().Exceptions &&2262 CGM.getCodeGenOpts().ObjCAutoRefCountExceptions;2263 }2264 llvm_unreachable("bad destruction kind");2265 }2266 2267 CleanupKind getCleanupKind(QualType::DestructionKind kind) {2268 return (needsEHCleanup(kind) ? NormalAndEHCleanup : NormalCleanup);2269 }2270 2271 //===--------------------------------------------------------------------===//2272 // Objective-C2273 //===--------------------------------------------------------------------===//2274 2275 void GenerateObjCMethod(const ObjCMethodDecl *OMD);2276 2277 void StartObjCMethod(const ObjCMethodDecl *MD, const ObjCContainerDecl *CD);2278 2279 /// GenerateObjCGetter - Synthesize an Objective-C property getter function.2280 void GenerateObjCGetter(ObjCImplementationDecl *IMP,2281 const ObjCPropertyImplDecl *PID);2282 void generateObjCGetterBody(const ObjCImplementationDecl *classImpl,2283 const ObjCPropertyImplDecl *propImpl,2284 const ObjCMethodDecl *GetterMothodDecl,2285 llvm::Constant *AtomicHelperFn);2286 2287 void GenerateObjCCtorDtorMethod(ObjCImplementationDecl *IMP,2288 ObjCMethodDecl *MD, bool ctor);2289 2290 /// GenerateObjCSetter - Synthesize an Objective-C property setter function2291 /// for the given property.2292 void GenerateObjCSetter(ObjCImplementationDecl *IMP,2293 const ObjCPropertyImplDecl *PID);2294 void generateObjCSetterBody(const ObjCImplementationDecl *classImpl,2295 const ObjCPropertyImplDecl *propImpl,2296 llvm::Constant *AtomicHelperFn);2297 2298 //===--------------------------------------------------------------------===//2299 // Block Bits2300 //===--------------------------------------------------------------------===//2301 2302 /// Emit block literal.2303 /// \return an LLVM value which is a pointer to a struct which contains2304 /// information about the block, including the block invoke function, the2305 /// captured variables, etc.2306 llvm::Value *EmitBlockLiteral(const BlockExpr *);2307 2308 llvm::Function *GenerateBlockFunction(GlobalDecl GD, const CGBlockInfo &Info,2309 const DeclMapTy &ldm,2310 bool IsLambdaConversionToBlock,2311 bool BuildGlobalBlock);2312 2313 /// Check if \p T is a C++ class that has a destructor that can throw.2314 static bool cxxDestructorCanThrow(QualType T);2315 2316 llvm::Constant *GenerateCopyHelperFunction(const CGBlockInfo &blockInfo);2317 llvm::Constant *GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo);2318 llvm::Constant *2319 GenerateObjCAtomicSetterCopyHelperFunction(const ObjCPropertyImplDecl *PID);2320 llvm::Constant *2321 GenerateObjCAtomicGetterCopyHelperFunction(const ObjCPropertyImplDecl *PID);2322 llvm::Value *EmitBlockCopyAndAutorelease(llvm::Value *Block, QualType Ty);2323 2324 void BuildBlockRelease(llvm::Value *DeclPtr, BlockFieldFlags flags,2325 bool CanThrow);2326 2327 class AutoVarEmission;2328 2329 void emitByrefStructureInit(const AutoVarEmission &emission);2330 2331 /// Enter a cleanup to destroy a __block variable. Note that this2332 /// cleanup should be a no-op if the variable hasn't left the stack2333 /// yet; if a cleanup is required for the variable itself, that needs2334 /// to be done externally.2335 ///2336 /// \param Kind Cleanup kind.2337 ///2338 /// \param Addr When \p LoadBlockVarAddr is false, the address of the __block2339 /// structure that will be passed to _Block_object_dispose. When2340 /// \p LoadBlockVarAddr is true, the address of the field of the block2341 /// structure that holds the address of the __block structure.2342 ///2343 /// \param Flags The flag that will be passed to _Block_object_dispose.2344 ///2345 /// \param LoadBlockVarAddr Indicates whether we need to emit a load from2346 /// \p Addr to get the address of the __block structure.2347 void enterByrefCleanup(CleanupKind Kind, Address Addr, BlockFieldFlags Flags,2348 bool LoadBlockVarAddr, bool CanThrow);2349 2350 void setBlockContextParameter(const ImplicitParamDecl *D, unsigned argNum,2351 llvm::Value *ptr);2352 2353 Address LoadBlockStruct();2354 Address GetAddrOfBlockDecl(const VarDecl *var);2355 2356 /// BuildBlockByrefAddress - Computes the location of the2357 /// data in a variable which is declared as __block.2358 Address emitBlockByrefAddress(Address baseAddr, const VarDecl *V,2359 bool followForward = true);2360 Address emitBlockByrefAddress(Address baseAddr, const BlockByrefInfo &info,2361 bool followForward, const llvm::Twine &name);2362 2363 const BlockByrefInfo &getBlockByrefInfo(const VarDecl *var);2364 2365 QualType BuildFunctionArgList(GlobalDecl GD, FunctionArgList &Args);2366 2367 void GenerateCode(GlobalDecl GD, llvm::Function *Fn,2368 const CGFunctionInfo &FnInfo);2369 2370 /// Annotate the function with an attribute that disables TSan checking at2371 /// runtime.2372 void markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn);2373 2374 /// Emit code for the start of a function.2375 /// \param Loc The location to be associated with the function.2376 /// \param StartLoc The location of the function body.2377 void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn,2378 const CGFunctionInfo &FnInfo, const FunctionArgList &Args,2379 SourceLocation Loc = SourceLocation(),2380 SourceLocation StartLoc = SourceLocation());2381 2382 static bool IsConstructorDelegationValid(const CXXConstructorDecl *Ctor);2383 2384 void EmitConstructorBody(FunctionArgList &Args);2385 void EmitDestructorBody(FunctionArgList &Args);2386 void emitImplicitAssignmentOperatorBody(FunctionArgList &Args);2387 void EmitFunctionBody(const Stmt *Body);2388 void EmitBlockWithFallThrough(llvm::BasicBlock *BB, const Stmt *S);2389 2390 void EmitForwardingCallToLambda(const CXXMethodDecl *LambdaCallOperator,2391 CallArgList &CallArgs,2392 const CGFunctionInfo *CallOpFnInfo = nullptr,2393 llvm::Constant *CallOpFn = nullptr);2394 void EmitLambdaBlockInvokeBody();2395 void EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD);2396 void EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD,2397 CallArgList &CallArgs);2398 void EmitLambdaInAllocaImplFn(const CXXMethodDecl *CallOp,2399 const CGFunctionInfo **ImplFnInfo,2400 llvm::Function **ImplFn);2401 void EmitLambdaInAllocaCallOpBody(const CXXMethodDecl *MD);2402 void EmitLambdaVLACapture(const VariableArrayType *VAT, LValue LV) {2403 EmitStoreThroughLValue(RValue::get(VLASizeMap[VAT->getSizeExpr()]), LV);2404 }2405 void EmitAsanPrologueOrEpilogue(bool Prologue);2406 2407 /// Emit the unified return block, trying to avoid its emission when2408 /// possible.2409 /// \return The debug location of the user written return statement if the2410 /// return block is avoided.2411 llvm::DebugLoc EmitReturnBlock();2412 2413 /// FinishFunction - Complete IR generation of the current function. It is2414 /// legal to call this function even if there is no current insertion point.2415 void FinishFunction(SourceLocation EndLoc = SourceLocation());2416 2417 void StartThunk(llvm::Function *Fn, GlobalDecl GD,2418 const CGFunctionInfo &FnInfo, bool IsUnprototyped);2419 2420 void EmitCallAndReturnForThunk(llvm::FunctionCallee Callee,2421 const ThunkInfo *Thunk, bool IsUnprototyped);2422 2423 void FinishThunk();2424 2425 /// Emit a musttail call for a thunk with a potentially adjusted this pointer.2426 void EmitMustTailThunk(GlobalDecl GD, llvm::Value *AdjustedThisPtr,2427 llvm::FunctionCallee Callee);2428 2429 /// Generate a thunk for the given method.2430 void generateThunk(llvm::Function *Fn, const CGFunctionInfo &FnInfo,2431 GlobalDecl GD, const ThunkInfo &Thunk,2432 bool IsUnprototyped);2433 2434 llvm::Function *GenerateVarArgsThunk(llvm::Function *Fn,2435 const CGFunctionInfo &FnInfo,2436 GlobalDecl GD, const ThunkInfo &Thunk);2437 2438 void EmitCtorPrologue(const CXXConstructorDecl *CD, CXXCtorType Type,2439 FunctionArgList &Args);2440 2441 void EmitInitializerForField(FieldDecl *Field, LValue LHS, Expr *Init);2442 2443 /// Struct with all information about dynamic [sub]class needed to set vptr.2444 struct VPtr {2445 BaseSubobject Base;2446 const CXXRecordDecl *NearestVBase;2447 CharUnits OffsetFromNearestVBase;2448 const CXXRecordDecl *VTableClass;2449 };2450 2451 /// Initialize the vtable pointer of the given subobject.2452 void InitializeVTablePointer(const VPtr &vptr);2453 2454 typedef llvm::SmallVector<VPtr, 4> VPtrsVector;2455 2456 typedef llvm::SmallPtrSet<const CXXRecordDecl *, 4> VisitedVirtualBasesSetTy;2457 VPtrsVector getVTablePointers(const CXXRecordDecl *VTableClass);2458 2459 void getVTablePointers(BaseSubobject Base, const CXXRecordDecl *NearestVBase,2460 CharUnits OffsetFromNearestVBase,2461 bool BaseIsNonVirtualPrimaryBase,2462 const CXXRecordDecl *VTableClass,2463 VisitedVirtualBasesSetTy &VBases, VPtrsVector &vptrs);2464 2465 void InitializeVTablePointers(const CXXRecordDecl *ClassDecl);2466 2467 // VTableTrapMode - whether we guarantee that loading the2468 // vtable is guaranteed to trap on authentication failure,2469 // even if the resulting vtable pointer is unused.2470 enum class VTableAuthMode {2471 Authenticate,2472 MustTrap,2473 UnsafeUbsanStrip // Should only be used for Vptr UBSan check2474 };2475 /// GetVTablePtr - Return the Value of the vtable pointer member pointed2476 /// to by This.2477 llvm::Value *2478 GetVTablePtr(Address This, llvm::Type *VTableTy,2479 const CXXRecordDecl *VTableClass,2480 VTableAuthMode AuthMode = VTableAuthMode::Authenticate);2481 2482 enum CFITypeCheckKind {2483 CFITCK_VCall,2484 CFITCK_NVCall,2485 CFITCK_DerivedCast,2486 CFITCK_UnrelatedCast,2487 CFITCK_ICall,2488 CFITCK_NVMFCall,2489 CFITCK_VMFCall,2490 };2491 2492 /// Derived is the presumed address of an object of type T after a2493 /// cast. If T is a polymorphic class type, emit a check that the virtual2494 /// table for Derived belongs to a class derived from T.2495 void EmitVTablePtrCheckForCast(QualType T, Address Derived, bool MayBeNull,2496 CFITypeCheckKind TCK, SourceLocation Loc);2497 2498 /// EmitVTablePtrCheckForCall - Virtual method MD is being called via VTable.2499 /// If vptr CFI is enabled, emit a check that VTable is valid.2500 void EmitVTablePtrCheckForCall(const CXXRecordDecl *RD, llvm::Value *VTable,2501 CFITypeCheckKind TCK, SourceLocation Loc);2502 2503 /// EmitVTablePtrCheck - Emit a check that VTable is a valid virtual table for2504 /// RD using llvm.type.test.2505 void EmitVTablePtrCheck(const CXXRecordDecl *RD, llvm::Value *VTable,2506 CFITypeCheckKind TCK, SourceLocation Loc);2507 2508 /// If whole-program virtual table optimization is enabled, emit an assumption2509 /// that VTable is a member of RD's type identifier. Or, if vptr CFI is2510 /// enabled, emit a check that VTable is a member of RD's type identifier.2511 void EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,2512 llvm::Value *VTable, SourceLocation Loc);2513 2514 /// Returns whether we should perform a type checked load when loading a2515 /// virtual function for virtual calls to members of RD. This is generally2516 /// true when both vcall CFI and whole-program-vtables are enabled.2517 bool ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD);2518 2519 /// Emit a type checked load from the given vtable.2520 llvm::Value *EmitVTableTypeCheckedLoad(const CXXRecordDecl *RD,2521 llvm::Value *VTable,2522 llvm::Type *VTableTy,2523 uint64_t VTableByteOffset);2524 2525 /// EnterDtorCleanups - Enter the cleanups necessary to complete the2526 /// given phase of destruction for a destructor. The end result2527 /// should call destructors on members and base classes in reverse2528 /// order of their construction.2529 void EnterDtorCleanups(const CXXDestructorDecl *Dtor, CXXDtorType Type);2530 2531 /// ShouldInstrumentFunction - Return true if the current function should be2532 /// instrumented with __cyg_profile_func_* calls2533 bool ShouldInstrumentFunction();2534 2535 /// ShouldSkipSanitizerInstrumentation - Return true if the current function2536 /// should not be instrumented with sanitizers.2537 bool ShouldSkipSanitizerInstrumentation();2538 2539 /// ShouldXRayInstrument - Return true if the current function should be2540 /// instrumented with XRay nop sleds.2541 bool ShouldXRayInstrumentFunction() const;2542 2543 /// AlwaysEmitXRayCustomEvents - Return true if we must unconditionally emit2544 /// XRay custom event handling calls.2545 bool AlwaysEmitXRayCustomEvents() const;2546 2547 /// AlwaysEmitXRayTypedEvents - Return true if clang must unconditionally emit2548 /// XRay typed event handling calls.2549 bool AlwaysEmitXRayTypedEvents() const;2550 2551 /// Return a type hash constant for a function instrumented by2552 /// -fsanitize=function.2553 llvm::ConstantInt *getUBSanFunctionTypeHash(QualType T) const;2554 2555 /// EmitFunctionProlog - Emit the target specific LLVM code to load the2556 /// arguments for the given function. This is also responsible for naming the2557 /// LLVM function arguments.2558 void EmitFunctionProlog(const CGFunctionInfo &FI, llvm::Function *Fn,2559 const FunctionArgList &Args);2560 2561 /// EmitFunctionEpilog - Emit the target specific LLVM code to return the2562 /// given temporary. Specify the source location atom group (Key Instructions2563 /// debug info feature) for the `ret` using \p RetKeyInstructionsSourceAtom.2564 /// If it's 0, the `ret` will get added to a new source atom group.2565 void EmitFunctionEpilog(const CGFunctionInfo &FI, bool EmitRetDbgLoc,2566 SourceLocation EndLoc,2567 uint64_t RetKeyInstructionsSourceAtom);2568 2569 /// Emit a test that checks if the return value \p RV is nonnull.2570 void EmitReturnValueCheck(llvm::Value *RV);2571 2572 /// EmitStartEHSpec - Emit the start of the exception spec.2573 void EmitStartEHSpec(const Decl *D);2574 2575 /// EmitEndEHSpec - Emit the end of the exception spec.2576 void EmitEndEHSpec(const Decl *D);2577 2578 /// getTerminateLandingPad - Return a landing pad that just calls terminate.2579 llvm::BasicBlock *getTerminateLandingPad();2580 2581 /// getTerminateLandingPad - Return a cleanup funclet that just calls2582 /// terminate.2583 llvm::BasicBlock *getTerminateFunclet();2584 2585 /// getTerminateHandler - Return a handler (not a landing pad, just2586 /// a catch handler) that just calls terminate. This is used when2587 /// a terminate scope encloses a try.2588 llvm::BasicBlock *getTerminateHandler();2589 2590 llvm::Type *ConvertTypeForMem(QualType T);2591 llvm::Type *ConvertType(QualType T);2592 llvm::Type *convertTypeForLoadStore(QualType ASTTy,2593 llvm::Type *LLVMTy = nullptr);2594 llvm::Type *ConvertType(const TypeDecl *T) {2595 return ConvertType(getContext().getTypeDeclType(T));2596 }2597 2598 /// LoadObjCSelf - Load the value of self. This function is only valid while2599 /// generating code for an Objective-C method.2600 llvm::Value *LoadObjCSelf();2601 2602 /// TypeOfSelfObject - Return type of object that this self represents.2603 QualType TypeOfSelfObject();2604 2605 /// getEvaluationKind - Return the TypeEvaluationKind of QualType \c T.2606 static TypeEvaluationKind getEvaluationKind(QualType T);2607 2608 static bool hasScalarEvaluationKind(QualType T) {2609 return getEvaluationKind(T) == TEK_Scalar;2610 }2611 2612 static bool hasAggregateEvaluationKind(QualType T) {2613 return getEvaluationKind(T) == TEK_Aggregate;2614 }2615 2616 /// createBasicBlock - Create an LLVM basic block.2617 llvm::BasicBlock *createBasicBlock(const Twine &name = "",2618 llvm::Function *parent = nullptr,2619 llvm::BasicBlock *before = nullptr) {2620 return llvm::BasicBlock::Create(getLLVMContext(), name, parent, before);2621 }2622 2623 /// getBasicBlockForLabel - Return the LLVM basicblock that the specified2624 /// label maps to.2625 JumpDest getJumpDestForLabel(const LabelDecl *S);2626 2627 /// SimplifyForwardingBlocks - If the given basic block is only a branch to2628 /// another basic block, simplify it. This assumes that no other code could2629 /// potentially reference the basic block.2630 void SimplifyForwardingBlocks(llvm::BasicBlock *BB);2631 2632 /// EmitBlock - Emit the given block \arg BB and set it as the insert point,2633 /// adding a fall-through branch from the current insert block if2634 /// necessary. It is legal to call this function even if there is no current2635 /// insertion point.2636 ///2637 /// IsFinished - If true, indicates that the caller has finished emitting2638 /// branches to the given block and does not expect to emit code into it. This2639 /// means the block can be ignored if it is unreachable.2640 void EmitBlock(llvm::BasicBlock *BB, bool IsFinished = false);2641 2642 /// EmitBlockAfterUses - Emit the given block somewhere hopefully2643 /// near its uses, and leave the insertion point in it.2644 void EmitBlockAfterUses(llvm::BasicBlock *BB);2645 2646 /// EmitBranch - Emit a branch to the specified basic block from the current2647 /// insert block, taking care to avoid creation of branches from dummy2648 /// blocks. It is legal to call this function even if there is no current2649 /// insertion point.2650 ///2651 /// This function clears the current insertion point. The caller should follow2652 /// calls to this function with calls to Emit*Block prior to generation new2653 /// code.2654 void EmitBranch(llvm::BasicBlock *Block);2655 2656 /// HaveInsertPoint - True if an insertion point is defined. If not, this2657 /// indicates that the current code being emitted is unreachable.2658 bool HaveInsertPoint() const { return Builder.GetInsertBlock() != nullptr; }2659 2660 /// EnsureInsertPoint - Ensure that an insertion point is defined so that2661 /// emitted IR has a place to go. Note that by definition, if this function2662 /// creates a block then that block is unreachable; callers may do better to2663 /// detect when no insertion point is defined and simply skip IR generation.2664 void EnsureInsertPoint() {2665 if (!HaveInsertPoint())2666 EmitBlock(createBasicBlock());2667 }2668 2669 /// ErrorUnsupported - Print out an error that codegen doesn't support the2670 /// specified stmt yet.2671 void ErrorUnsupported(const Stmt *S, const char *Type);2672 2673 //===--------------------------------------------------------------------===//2674 // Helpers2675 //===--------------------------------------------------------------------===//2676 2677 Address mergeAddressesInConditionalExpr(Address LHS, Address RHS,2678 llvm::BasicBlock *LHSBlock,2679 llvm::BasicBlock *RHSBlock,2680 llvm::BasicBlock *MergeBlock,2681 QualType MergedType) {2682 Builder.SetInsertPoint(MergeBlock);2683 llvm::PHINode *PtrPhi = Builder.CreatePHI(LHS.getType(), 2, "cond");2684 PtrPhi->addIncoming(LHS.getBasePointer(), LHSBlock);2685 PtrPhi->addIncoming(RHS.getBasePointer(), RHSBlock);2686 LHS.replaceBasePointer(PtrPhi);2687 LHS.setAlignment(std::min(LHS.getAlignment(), RHS.getAlignment()));2688 return LHS;2689 }2690 2691 /// Construct an address with the natural alignment of T. If a pointer to T2692 /// is expected to be signed, the pointer passed to this function must have2693 /// been signed, and the returned Address will have the pointer authentication2694 /// information needed to authenticate the signed pointer.2695 Address makeNaturalAddressForPointer(2696 llvm::Value *Ptr, QualType T, CharUnits Alignment = CharUnits::Zero(),2697 bool ForPointeeType = false, LValueBaseInfo *BaseInfo = nullptr,2698 TBAAAccessInfo *TBAAInfo = nullptr,2699 KnownNonNull_t IsKnownNonNull = NotKnownNonNull) {2700 if (Alignment.isZero())2701 Alignment =2702 CGM.getNaturalTypeAlignment(T, BaseInfo, TBAAInfo, ForPointeeType);2703 return Address(Ptr, ConvertTypeForMem(T), Alignment,2704 CGM.getPointerAuthInfoForPointeeType(T), /*Offset=*/nullptr,2705 IsKnownNonNull);2706 }2707 2708 LValue MakeAddrLValue(Address Addr, QualType T,2709 AlignmentSource Source = AlignmentSource::Type) {2710 return MakeAddrLValue(Addr, T, LValueBaseInfo(Source),2711 CGM.getTBAAAccessInfo(T));2712 }2713 2714 LValue MakeAddrLValue(Address Addr, QualType T, LValueBaseInfo BaseInfo,2715 TBAAAccessInfo TBAAInfo) {2716 return LValue::MakeAddr(Addr, T, getContext(), BaseInfo, TBAAInfo);2717 }2718 2719 LValue MakeAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,2720 AlignmentSource Source = AlignmentSource::Type) {2721 return MakeAddrLValue(makeNaturalAddressForPointer(V, T, Alignment), T,2722 LValueBaseInfo(Source), CGM.getTBAAAccessInfo(T));2723 }2724 2725 /// Same as MakeAddrLValue above except that the pointer is known to be2726 /// unsigned.2727 LValue MakeRawAddrLValue(llvm::Value *V, QualType T, CharUnits Alignment,2728 AlignmentSource Source = AlignmentSource::Type) {2729 Address Addr(V, ConvertTypeForMem(T), Alignment);2730 return LValue::MakeAddr(Addr, T, getContext(), LValueBaseInfo(Source),2731 CGM.getTBAAAccessInfo(T));2732 }2733 2734 LValue2735 MakeAddrLValueWithoutTBAA(Address Addr, QualType T,2736 AlignmentSource Source = AlignmentSource::Type) {2737 return LValue::MakeAddr(Addr, T, getContext(), LValueBaseInfo(Source),2738 TBAAAccessInfo());2739 }2740 2741 /// Given a value of type T* that may not be to a complete object, construct2742 /// an l-value with the natural pointee alignment of T.2743 LValue MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T);2744 2745 LValue2746 MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T,2747 KnownNonNull_t IsKnownNonNull = NotKnownNonNull);2748 2749 /// Same as MakeNaturalAlignPointeeAddrLValue except that the pointer is known2750 /// to be unsigned.2751 LValue MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V, QualType T);2752 2753 LValue MakeNaturalAlignRawAddrLValue(llvm::Value *V, QualType T);2754 2755 Address EmitLoadOfReference(LValue RefLVal,2756 LValueBaseInfo *PointeeBaseInfo = nullptr,2757 TBAAAccessInfo *PointeeTBAAInfo = nullptr);2758 LValue EmitLoadOfReferenceLValue(LValue RefLVal);2759 LValue2760 EmitLoadOfReferenceLValue(Address RefAddr, QualType RefTy,2761 AlignmentSource Source = AlignmentSource::Type) {2762 LValue RefLVal = MakeAddrLValue(RefAddr, RefTy, LValueBaseInfo(Source),2763 CGM.getTBAAAccessInfo(RefTy));2764 return EmitLoadOfReferenceLValue(RefLVal);2765 }2766 2767 /// Load a pointer with type \p PtrTy stored at address \p Ptr.2768 /// Note that \p PtrTy is the type of the loaded pointer, not the addresses2769 /// it is loaded from.2770 Address EmitLoadOfPointer(Address Ptr, const PointerType *PtrTy,2771 LValueBaseInfo *BaseInfo = nullptr,2772 TBAAAccessInfo *TBAAInfo = nullptr);2773 LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy);2774 2775private:2776 struct AllocaTracker {2777 void Add(llvm::AllocaInst *I) { Allocas.push_back(I); }2778 llvm::SmallVector<llvm::AllocaInst *> Take() { return std::move(Allocas); }2779 2780 private:2781 llvm::SmallVector<llvm::AllocaInst *> Allocas;2782 };2783 AllocaTracker *Allocas = nullptr;2784 2785 /// CGDecl helper.2786 void emitStoresForConstant(const VarDecl &D, Address Loc, bool isVolatile,2787 llvm::Constant *constant, bool IsAutoInit);2788 /// CGDecl helper.2789 void emitStoresForZeroInit(const VarDecl &D, Address Loc, bool isVolatile);2790 /// CGDecl helper.2791 void emitStoresForPatternInit(const VarDecl &D, Address Loc, bool isVolatile);2792 /// CGDecl helper.2793 void emitStoresForInitAfterBZero(llvm::Constant *Init, Address Loc,2794 bool isVolatile, bool IsAutoInit);2795 2796public:2797 // Captures all the allocas created during the scope of its RAII object.2798 struct AllocaTrackerRAII {2799 AllocaTrackerRAII(CodeGenFunction &CGF)2800 : CGF(CGF), OldTracker(CGF.Allocas) {2801 CGF.Allocas = &Tracker;2802 }2803 ~AllocaTrackerRAII() { CGF.Allocas = OldTracker; }2804 2805 llvm::SmallVector<llvm::AllocaInst *> Take() { return Tracker.Take(); }2806 2807 private:2808 CodeGenFunction &CGF;2809 AllocaTracker *OldTracker;2810 AllocaTracker Tracker;2811 };2812 2813private:2814 /// If \p Alloca is not in the same address space as \p DestLangAS, insert an2815 /// address space cast and return a new RawAddress based on this value.2816 RawAddress MaybeCastStackAddressSpace(RawAddress Alloca, LangAS DestLangAS,2817 llvm::Value *ArraySize = nullptr);2818 2819public:2820 /// CreateTempAlloca - This creates an alloca and inserts it into the entry2821 /// block if \p ArraySize is nullptr, otherwise inserts it at the current2822 /// insertion point of the builder. The caller is responsible for setting an2823 /// appropriate alignment on2824 /// the alloca.2825 ///2826 /// \p ArraySize is the number of array elements to be allocated if it2827 /// is not nullptr.2828 ///2829 /// LangAS::Default is the address space of pointers to local variables and2830 /// temporaries, as exposed in the source language. In certain2831 /// configurations, this is not the same as the alloca address space, and a2832 /// cast is needed to lift the pointer from the alloca AS into2833 /// LangAS::Default. This can happen when the target uses a restricted2834 /// address space for the stack but the source language requires2835 /// LangAS::Default to be a generic address space. The latter condition is2836 /// common for most programming languages; OpenCL is an exception in that2837 /// LangAS::Default is the private address space, which naturally maps2838 /// to the stack.2839 ///2840 /// Because the address of a temporary is often exposed to the program in2841 /// various ways, this function will perform the cast. The original alloca2842 /// instruction is returned through \p Alloca if it is not nullptr.2843 ///2844 /// The cast is not performaed in CreateTempAllocaWithoutCast. This is2845 /// more efficient if the caller knows that the address will not be exposed.2846 llvm::AllocaInst *CreateTempAlloca(llvm::Type *Ty, const Twine &Name = "tmp",2847 llvm::Value *ArraySize = nullptr);2848 2849 /// CreateTempAlloca - This creates a alloca and inserts it into the entry2850 /// block. The alloca is casted to the address space of \p UseAddrSpace if2851 /// necessary.2852 RawAddress CreateTempAlloca(llvm::Type *Ty, LangAS UseAddrSpace,2853 CharUnits align, const Twine &Name = "tmp",2854 llvm::Value *ArraySize = nullptr,2855 RawAddress *Alloca = nullptr);2856 2857 /// CreateTempAlloca - This creates a alloca and inserts it into the entry2858 /// block. The alloca is casted to default address space if necessary.2859 ///2860 /// FIXME: This version should be removed, and context should provide the2861 /// context use address space used instead of default.2862 RawAddress CreateTempAlloca(llvm::Type *Ty, CharUnits align,2863 const Twine &Name = "tmp",2864 llvm::Value *ArraySize = nullptr,2865 RawAddress *Alloca = nullptr) {2866 return CreateTempAlloca(Ty, LangAS::Default, align, Name, ArraySize,2867 Alloca);2868 }2869 2870 RawAddress CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits align,2871 const Twine &Name = "tmp",2872 llvm::Value *ArraySize = nullptr);2873 2874 /// CreateDefaultAlignedTempAlloca - This creates an alloca with the2875 /// default ABI alignment of the given LLVM type.2876 ///2877 /// IMPORTANT NOTE: This is *not* generally the right alignment for2878 /// any given AST type that happens to have been lowered to the2879 /// given IR type. This should only ever be used for function-local,2880 /// IR-driven manipulations like saving and restoring a value. Do2881 /// not hand this address off to arbitrary IRGen routines, and especially2882 /// do not pass it as an argument to a function that might expect a2883 /// properly ABI-aligned value.2884 RawAddress CreateDefaultAlignTempAlloca(llvm::Type *Ty,2885 const Twine &Name = "tmp");2886 2887 /// CreateIRTemp - Create a temporary IR object of the given type, with2888 /// appropriate alignment. This routine should only be used when an temporary2889 /// value needs to be stored into an alloca (for example, to avoid explicit2890 /// PHI construction), but the type is the IR type, not the type appropriate2891 /// for storing in memory.2892 ///2893 /// That is, this is exactly equivalent to CreateMemTemp, but calling2894 /// ConvertType instead of ConvertTypeForMem.2895 RawAddress CreateIRTemp(QualType T, const Twine &Name = "tmp");2896 2897 /// CreateMemTemp - Create a temporary memory object of the given type, with2898 /// appropriate alignmen and cast it to the default address space. Returns2899 /// the original alloca instruction by \p Alloca if it is not nullptr.2900 RawAddress CreateMemTemp(QualType T, const Twine &Name = "tmp",2901 RawAddress *Alloca = nullptr);2902 RawAddress CreateMemTemp(QualType T, CharUnits Align,2903 const Twine &Name = "tmp",2904 RawAddress *Alloca = nullptr);2905 2906 /// CreateMemTemp - Create a temporary memory object of the given type, with2907 /// appropriate alignmen without casting it to the default address space.2908 RawAddress CreateMemTempWithoutCast(QualType T, const Twine &Name = "tmp");2909 RawAddress CreateMemTempWithoutCast(QualType T, CharUnits Align,2910 const Twine &Name = "tmp");2911 2912 /// CreateAggTemp - Create a temporary memory object for the given2913 /// aggregate type.2914 AggValueSlot CreateAggTemp(QualType T, const Twine &Name = "tmp",2915 RawAddress *Alloca = nullptr) {2916 return AggValueSlot::forAddr(2917 CreateMemTemp(T, Name, Alloca), T.getQualifiers(),2918 AggValueSlot::IsNotDestructed, AggValueSlot::DoesNotNeedGCBarriers,2919 AggValueSlot::IsNotAliased, AggValueSlot::DoesNotOverlap);2920 }2921 2922 /// EvaluateExprAsBool - Perform the usual unary conversions on the specified2923 /// expression and compare the result against zero, returning an Int1Ty value.2924 llvm::Value *EvaluateExprAsBool(const Expr *E);2925 2926 /// Retrieve the implicit cast expression of the rhs in a binary operator2927 /// expression by passing pointers to Value and QualType2928 /// This is used for implicit bitfield conversion checks, which2929 /// must compare with the value before potential truncation.2930 llvm::Value *EmitWithOriginalRHSBitfieldAssignment(const BinaryOperator *E,2931 llvm::Value **Previous,2932 QualType *SrcType);2933 2934 /// Emit a check that an [implicit] conversion of a bitfield. It is not UB,2935 /// so we use the value after conversion.2936 void EmitBitfieldConversionCheck(llvm::Value *Src, QualType SrcType,2937 llvm::Value *Dst, QualType DstType,2938 const CGBitFieldInfo &Info,2939 SourceLocation Loc);2940 2941 /// EmitIgnoredExpr - Emit an expression in a context which ignores the2942 /// result.2943 void EmitIgnoredExpr(const Expr *E);2944 2945 /// EmitAnyExpr - Emit code to compute the specified expression which can have2946 /// any type. The result is returned as an RValue struct. If this is an2947 /// aggregate expression, the aggloc/agglocvolatile arguments indicate where2948 /// the result should be returned.2949 ///2950 /// \param ignoreResult True if the resulting value isn't used.2951 RValue EmitAnyExpr(const Expr *E,2952 AggValueSlot aggSlot = AggValueSlot::ignored(),2953 bool ignoreResult = false);2954 2955 // EmitVAListRef - Emit a "reference" to a va_list; this is either the address2956 // or the value of the expression, depending on how va_list is defined.2957 Address EmitVAListRef(const Expr *E);2958 2959 /// Emit a "reference" to a __builtin_ms_va_list; this is2960 /// always the value of the expression, because a __builtin_ms_va_list is a2961 /// pointer to a char.2962 Address EmitMSVAListRef(const Expr *E);2963 2964 /// EmitAnyExprToTemp - Similarly to EmitAnyExpr(), however, the result will2965 /// always be accessible even if no aggregate location is provided.2966 RValue EmitAnyExprToTemp(const Expr *E);2967 2968 /// EmitAnyExprToMem - Emits the code necessary to evaluate an2969 /// arbitrary expression into the given memory location.2970 void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals,2971 bool IsInitializer);2972 2973 void EmitAnyExprToExn(const Expr *E, Address Addr);2974 2975 /// EmitInitializationToLValue - Emit an initializer to an LValue.2976 void EmitInitializationToLValue(2977 const Expr *E, LValue LV,2978 AggValueSlot::IsZeroed_t IsZeroed = AggValueSlot::IsNotZeroed);2979 2980 /// EmitExprAsInit - Emits the code necessary to initialize a2981 /// location in memory with the given initializer.2982 void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue,2983 bool capturedByInit);2984 2985 /// hasVolatileMember - returns true if aggregate type has a volatile2986 /// member.2987 bool hasVolatileMember(QualType T) {2988 if (const auto *RD = T->getAsRecordDecl())2989 return RD->hasVolatileMember();2990 return false;2991 }2992 2993 /// Determine whether a return value slot may overlap some other object.2994 AggValueSlot::Overlap_t getOverlapForReturnValue() {2995 // FIXME: Assuming no overlap here breaks guaranteed copy elision for base2996 // class subobjects. These cases may need to be revisited depending on the2997 // resolution of the relevant core issue.2998 return AggValueSlot::DoesNotOverlap;2999 }3000 3001 /// Determine whether a field initialization may overlap some other object.3002 AggValueSlot::Overlap_t getOverlapForFieldInit(const FieldDecl *FD);3003 3004 /// Determine whether a base class initialization may overlap some other3005 /// object.3006 AggValueSlot::Overlap_t getOverlapForBaseInit(const CXXRecordDecl *RD,3007 const CXXRecordDecl *BaseRD,3008 bool IsVirtual);3009 3010 /// Emit an aggregate assignment.3011 void EmitAggregateAssign(LValue Dest, LValue Src, QualType EltTy) {3012 ApplyAtomGroup Grp(getDebugInfo());3013 bool IsVolatile = hasVolatileMember(EltTy);3014 EmitAggregateCopy(Dest, Src, EltTy, AggValueSlot::MayOverlap, IsVolatile);3015 }3016 3017 void EmitAggregateCopyCtor(LValue Dest, LValue Src,3018 AggValueSlot::Overlap_t MayOverlap) {3019 EmitAggregateCopy(Dest, Src, Src.getType(), MayOverlap);3020 }3021 3022 /// EmitAggregateCopy - Emit an aggregate copy.3023 ///3024 /// \param isVolatile \c true iff either the source or the destination is3025 /// volatile.3026 /// \param MayOverlap Whether the tail padding of the destination might be3027 /// occupied by some other object. More efficient code can often be3028 /// generated if not.3029 void EmitAggregateCopy(LValue Dest, LValue Src, QualType EltTy,3030 AggValueSlot::Overlap_t MayOverlap,3031 bool isVolatile = false);3032 3033 /// GetAddrOfLocalVar - Return the address of a local variable.3034 Address GetAddrOfLocalVar(const VarDecl *VD) {3035 auto it = LocalDeclMap.find(VD);3036 assert(it != LocalDeclMap.end() &&3037 "Invalid argument to GetAddrOfLocalVar(), no decl!");3038 return it->second;3039 }3040 3041 /// Given an opaque value expression, return its LValue mapping if it exists,3042 /// otherwise create one.3043 LValue getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e);3044 3045 /// Given an opaque value expression, return its RValue mapping if it exists,3046 /// otherwise create one.3047 RValue getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e);3048 3049 /// isOpaqueValueEmitted - Return true if the opaque value expression has3050 /// already been emitted.3051 bool isOpaqueValueEmitted(const OpaqueValueExpr *E);3052 3053 /// Get the index of the current ArrayInitLoopExpr, if any.3054 llvm::Value *getArrayInitIndex() { return ArrayInitIndex; }3055 3056 /// getAccessedFieldNo - Given an encoded value and a result number, return3057 /// the input field number being accessed.3058 static unsigned getAccessedFieldNo(unsigned Idx, const llvm::Constant *Elts);3059 3060 llvm::BlockAddress *GetAddrOfLabel(const LabelDecl *L);3061 llvm::BasicBlock *GetIndirectGotoBlock();3062 3063 /// Check if \p E is a C++ "this" pointer wrapped in value-preserving casts.3064 static bool IsWrappedCXXThis(const Expr *E);3065 3066 /// EmitNullInitialization - Generate code to set a value of the given type to3067 /// null, If the type contains data member pointers, they will be initialized3068 /// to -1 in accordance with the Itanium C++ ABI.3069 void EmitNullInitialization(Address DestPtr, QualType Ty);3070 3071 /// Emits a call to an LLVM variable-argument intrinsic, either3072 /// \c llvm.va_start or \c llvm.va_end.3073 /// \param ArgValue A reference to the \c va_list as emitted by either3074 /// \c EmitVAListRef or \c EmitMSVAListRef.3075 /// \param IsStart If \c true, emits a call to \c llvm.va_start; otherwise,3076 /// calls \c llvm.va_end.3077 llvm::Value *EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart);3078 3079 /// Generate code to get an argument from the passed in pointer3080 /// and update it accordingly.3081 /// \param VE The \c VAArgExpr for which to generate code.3082 /// \param VAListAddr Receives a reference to the \c va_list as emitted by3083 /// either \c EmitVAListRef or \c EmitMSVAListRef.3084 /// \returns A pointer to the argument.3085 // FIXME: We should be able to get rid of this method and use the va_arg3086 // instruction in LLVM instead once it works well enough.3087 RValue EmitVAArg(VAArgExpr *VE, Address &VAListAddr,3088 AggValueSlot Slot = AggValueSlot::ignored());3089 3090 /// emitArrayLength - Compute the length of an array, even if it's a3091 /// VLA, and drill down to the base element type.3092 llvm::Value *emitArrayLength(const ArrayType *arrayType, QualType &baseType,3093 Address &addr);3094 3095 /// EmitVLASize - Capture all the sizes for the VLA expressions in3096 /// the given variably-modified type and store them in the VLASizeMap.3097 ///3098 /// This function can be called with a null (unreachable) insert point.3099 void EmitVariablyModifiedType(QualType Ty);3100 3101 struct VlaSizePair {3102 llvm::Value *NumElts;3103 QualType Type;3104 3105 VlaSizePair(llvm::Value *NE, QualType T) : NumElts(NE), Type(T) {}3106 };3107 3108 /// Return the number of elements for a single dimension3109 /// for the given array type.3110 VlaSizePair getVLAElements1D(const VariableArrayType *vla);3111 VlaSizePair getVLAElements1D(QualType vla);3112 3113 /// Returns an LLVM value that corresponds to the size,3114 /// in non-variably-sized elements, of a variable length array type,3115 /// plus that largest non-variably-sized element type. Assumes that3116 /// the type has already been emitted with EmitVariablyModifiedType.3117 VlaSizePair getVLASize(const VariableArrayType *vla);3118 VlaSizePair getVLASize(QualType vla);3119 3120 /// LoadCXXThis - Load the value of 'this'. This function is only valid while3121 /// generating code for an C++ member function.3122 llvm::Value *LoadCXXThis() {3123 assert(CXXThisValue && "no 'this' value for this function");3124 return CXXThisValue;3125 }3126 Address LoadCXXThisAddress();3127 3128 /// LoadCXXVTT - Load the VTT parameter to base constructors/destructors have3129 /// virtual bases.3130 // FIXME: Every place that calls LoadCXXVTT is something3131 // that needs to be abstracted properly.3132 llvm::Value *LoadCXXVTT() {3133 assert(CXXStructorImplicitParamValue && "no VTT value for this function");3134 return CXXStructorImplicitParamValue;3135 }3136 3137 /// GetAddressOfBaseOfCompleteClass - Convert the given pointer to a3138 /// complete class to the given direct base.3139 Address GetAddressOfDirectBaseInCompleteClass(Address Value,3140 const CXXRecordDecl *Derived,3141 const CXXRecordDecl *Base,3142 bool BaseIsVirtual);3143 3144 static bool ShouldNullCheckClassCastValue(const CastExpr *Cast);3145 3146 /// GetAddressOfBaseClass - This function will add the necessary delta to the3147 /// load of 'this' and returns address of the base class.3148 Address GetAddressOfBaseClass(Address Value, const CXXRecordDecl *Derived,3149 CastExpr::path_const_iterator PathBegin,3150 CastExpr::path_const_iterator PathEnd,3151 bool NullCheckValue, SourceLocation Loc);3152 3153 Address GetAddressOfDerivedClass(Address Value, const CXXRecordDecl *Derived,3154 CastExpr::path_const_iterator PathBegin,3155 CastExpr::path_const_iterator PathEnd,3156 bool NullCheckValue);3157 3158 /// GetVTTParameter - Return the VTT parameter that should be passed to a3159 /// base constructor/destructor with virtual bases.3160 /// FIXME: VTTs are Itanium ABI-specific, so the definition should move3161 /// to ItaniumCXXABI.cpp together with all the references to VTT.3162 llvm::Value *GetVTTParameter(GlobalDecl GD, bool ForVirtualBase,3163 bool Delegating);3164 3165 void EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,3166 CXXCtorType CtorType,3167 const FunctionArgList &Args,3168 SourceLocation Loc);3169 // It's important not to confuse this and the previous function. Delegating3170 // constructors are the C++0x feature. The constructor delegate optimization3171 // is used to reduce duplication in the base and complete consturctors where3172 // they are substantially the same.3173 void EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,3174 const FunctionArgList &Args);3175 3176 /// Emit a call to an inheriting constructor (that is, one that invokes a3177 /// constructor inherited from a base class) by inlining its definition. This3178 /// is necessary if the ABI does not support forwarding the arguments to the3179 /// base class constructor (because they're variadic or similar).3180 void EmitInlinedInheritingCXXConstructorCall(const CXXConstructorDecl *Ctor,3181 CXXCtorType CtorType,3182 bool ForVirtualBase,3183 bool Delegating,3184 CallArgList &Args);3185 3186 /// Emit a call to a constructor inherited from a base class, passing the3187 /// current constructor's arguments along unmodified (without even making3188 /// a copy).3189 void EmitInheritedCXXConstructorCall(const CXXConstructorDecl *D,3190 bool ForVirtualBase, Address This,3191 bool InheritedFromVBase,3192 const CXXInheritedCtorInitExpr *E);3193 3194 void EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type,3195 bool ForVirtualBase, bool Delegating,3196 AggValueSlot ThisAVS, const CXXConstructExpr *E);3197 3198 void EmitCXXConstructorCall(const CXXConstructorDecl *D, CXXCtorType Type,3199 bool ForVirtualBase, bool Delegating,3200 Address This, CallArgList &Args,3201 AggValueSlot::Overlap_t Overlap,3202 SourceLocation Loc, bool NewPointerIsChecked,3203 llvm::CallBase **CallOrInvoke = nullptr);3204 3205 /// Emit assumption load for all bases. Requires to be called only on3206 /// most-derived class and not under construction of the object.3207 void EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl, Address This);3208 3209 /// Emit assumption that vptr load == global vtable.3210 void EmitVTableAssumptionLoad(const VPtr &vptr, Address This);3211 3212 void EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D, Address This,3213 Address Src, const CXXConstructExpr *E);3214 3215 void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,3216 const ArrayType *ArrayTy, Address ArrayPtr,3217 const CXXConstructExpr *E,3218 bool NewPointerIsChecked,3219 bool ZeroInitialization = false);3220 3221 void EmitCXXAggrConstructorCall(const CXXConstructorDecl *D,3222 llvm::Value *NumElements, Address ArrayPtr,3223 const CXXConstructExpr *E,3224 bool NewPointerIsChecked,3225 bool ZeroInitialization = false);3226 3227 static Destroyer destroyCXXObject;3228 3229 void EmitCXXDestructorCall(const CXXDestructorDecl *D, CXXDtorType Type,3230 bool ForVirtualBase, bool Delegating, Address This,3231 QualType ThisTy);3232 3233 void EmitNewArrayInitializer(const CXXNewExpr *E, QualType elementType,3234 llvm::Type *ElementTy, Address NewPtr,3235 llvm::Value *NumElements,3236 llvm::Value *AllocSizeWithoutCookie);3237 3238 void EmitCXXTemporary(const CXXTemporary *Temporary, QualType TempType,3239 Address Ptr);3240 3241 void EmitSehCppScopeBegin();3242 void EmitSehCppScopeEnd();3243 void EmitSehTryScopeBegin();3244 void EmitSehTryScopeEnd();3245 3246 bool EmitLifetimeStart(llvm::Value *Addr);3247 void EmitLifetimeEnd(llvm::Value *Addr);3248 3249 llvm::Value *EmitCXXNewExpr(const CXXNewExpr *E);3250 void EmitCXXDeleteExpr(const CXXDeleteExpr *E);3251 3252 void EmitDeleteCall(const FunctionDecl *DeleteFD, llvm::Value *Ptr,3253 QualType DeleteTy, llvm::Value *NumElements = nullptr,3254 CharUnits CookieSize = CharUnits());3255 3256 RValue EmitBuiltinNewDeleteCall(const FunctionProtoType *Type,3257 const CallExpr *TheCallExpr, bool IsDelete);3258 3259 llvm::Value *EmitCXXTypeidExpr(const CXXTypeidExpr *E);3260 llvm::Value *EmitDynamicCast(Address V, const CXXDynamicCastExpr *DCE);3261 Address EmitCXXUuidofExpr(const CXXUuidofExpr *E);3262 3263 /// Situations in which we might emit a check for the suitability of a3264 /// pointer or glvalue. Needs to be kept in sync with ubsan_handlers.cpp in3265 /// compiler-rt.3266 enum TypeCheckKind {3267 /// Checking the operand of a load. Must be suitably sized and aligned.3268 TCK_Load,3269 /// Checking the destination of a store. Must be suitably sized and aligned.3270 TCK_Store,3271 /// Checking the bound value in a reference binding. Must be suitably sized3272 /// and aligned, but is not required to refer to an object (until the3273 /// reference is used), per core issue 453.3274 TCK_ReferenceBinding,3275 /// Checking the object expression in a non-static data member access. Must3276 /// be an object within its lifetime.3277 TCK_MemberAccess,3278 /// Checking the 'this' pointer for a call to a non-static member function.3279 /// Must be an object within its lifetime.3280 TCK_MemberCall,3281 /// Checking the 'this' pointer for a constructor call.3282 TCK_ConstructorCall,3283 /// Checking the operand of a static_cast to a derived pointer type. Must be3284 /// null or an object within its lifetime.3285 TCK_DowncastPointer,3286 /// Checking the operand of a static_cast to a derived reference type. Must3287 /// be an object within its lifetime.3288 TCK_DowncastReference,3289 /// Checking the operand of a cast to a base object. Must be suitably sized3290 /// and aligned.3291 TCK_Upcast,3292 /// Checking the operand of a cast to a virtual base object. Must be an3293 /// object within its lifetime.3294 TCK_UpcastToVirtualBase,3295 /// Checking the value assigned to a _Nonnull pointer. Must not be null.3296 TCK_NonnullAssign,3297 /// Checking the operand of a dynamic_cast or a typeid expression. Must be3298 /// null or an object within its lifetime.3299 TCK_DynamicOperation3300 };3301 3302 /// Determine whether the pointer type check \p TCK permits null pointers.3303 static bool isNullPointerAllowed(TypeCheckKind TCK);3304 3305 /// Determine whether the pointer type check \p TCK requires a vptr check.3306 static bool isVptrCheckRequired(TypeCheckKind TCK, QualType Ty);3307 3308 /// Whether any type-checking sanitizers are enabled. If \c false,3309 /// calls to EmitTypeCheck can be skipped.3310 bool sanitizePerformTypeCheck() const;3311 3312 void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, LValue LV,3313 QualType Type, SanitizerSet SkippedChecks = SanitizerSet(),3314 llvm::Value *ArraySize = nullptr) {3315 if (!sanitizePerformTypeCheck())3316 return;3317 EmitTypeCheck(TCK, Loc, LV.emitRawPointer(*this), Type, LV.getAlignment(),3318 SkippedChecks, ArraySize);3319 }3320 3321 void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, Address Addr,3322 QualType Type, CharUnits Alignment = CharUnits::Zero(),3323 SanitizerSet SkippedChecks = SanitizerSet(),3324 llvm::Value *ArraySize = nullptr) {3325 if (!sanitizePerformTypeCheck())3326 return;3327 EmitTypeCheck(TCK, Loc, Addr.emitRawPointer(*this), Type, Alignment,3328 SkippedChecks, ArraySize);3329 }3330 3331 /// Emit a check that \p V is the address of storage of the3332 /// appropriate size and alignment for an object of type \p Type3333 /// (or if ArraySize is provided, for an array of that bound).3334 void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, llvm::Value *V,3335 QualType Type, CharUnits Alignment = CharUnits::Zero(),3336 SanitizerSet SkippedChecks = SanitizerSet(),3337 llvm::Value *ArraySize = nullptr);3338 3339 /// Emit a check that \p Base points into an array object, which3340 /// we can access at index \p Index. \p Accessed should be \c false if we3341 /// this expression is used as an lvalue, for instance in "&Arr[Idx]".3342 void EmitBoundsCheck(const Expr *E, const Expr *Base, llvm::Value *Index,3343 QualType IndexType, bool Accessed);3344 void EmitBoundsCheckImpl(const Expr *E, llvm::Value *Bound,3345 llvm::Value *Index, QualType IndexType,3346 QualType IndexedType, bool Accessed);3347 3348 /// Returns debug info, with additional annotation if3349 /// CGM.getCodeGenOpts().SanitizeAnnotateDebugInfo[Ordinal] is enabled for3350 /// any of the ordinals.3351 llvm::DILocation *3352 SanitizerAnnotateDebugInfo(ArrayRef<SanitizerKind::SanitizerOrdinal> Ordinals,3353 SanitizerHandler Handler);3354 3355 /// Build metadata used by the AllocToken instrumentation.3356 llvm::MDNode *buildAllocToken(QualType AllocType);3357 /// Emit and set additional metadata used by the AllocToken instrumentation.3358 void EmitAllocToken(llvm::CallBase *CB, QualType AllocType);3359 /// Build additional metadata used by the AllocToken instrumentation,3360 /// inferring the type from an allocation call expression.3361 llvm::MDNode *buildAllocToken(const CallExpr *E);3362 /// Emit and set additional metadata used by the AllocToken instrumentation,3363 /// inferring the type from an allocation call expression.3364 void EmitAllocToken(llvm::CallBase *CB, const CallExpr *E);3365 3366 llvm::Value *GetCountedByFieldExprGEP(const Expr *Base, const FieldDecl *FD,3367 const FieldDecl *CountDecl);3368 3369 /// Build an expression accessing the "counted_by" field.3370 llvm::Value *EmitLoadOfCountedByField(const Expr *Base, const FieldDecl *FD,3371 const FieldDecl *CountDecl);3372 3373 // Emit bounds checking for flexible array and pointer members with the3374 // counted_by attribute.3375 void EmitCountedByBoundsChecking(const Expr *E, llvm::Value *Idx,3376 Address Addr, QualType IdxTy,3377 QualType ArrayTy, bool Accessed,3378 bool FlexibleArray);3379 3380 llvm::Value *EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,3381 bool isInc, bool isPre);3382 ComplexPairTy EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,3383 bool isInc, bool isPre);3384 3385 /// Converts Location to a DebugLoc, if debug information is enabled.3386 llvm::DebugLoc SourceLocToDebugLoc(SourceLocation Location);3387 3388 /// Get the record field index as represented in debug info.3389 unsigned getDebugInfoFIndex(const RecordDecl *Rec, unsigned FieldIndex);3390 3391 //===--------------------------------------------------------------------===//3392 // Declaration Emission3393 //===--------------------------------------------------------------------===//3394 3395 /// EmitDecl - Emit a declaration.3396 ///3397 /// This function can be called with a null (unreachable) insert point.3398 void EmitDecl(const Decl &D, bool EvaluateConditionDecl = false);3399 3400 /// EmitVarDecl - Emit a local variable declaration.3401 ///3402 /// This function can be called with a null (unreachable) insert point.3403 void EmitVarDecl(const VarDecl &D);3404 3405 void EmitScalarInit(const Expr *init, const ValueDecl *D, LValue lvalue,3406 bool capturedByInit);3407 3408 typedef void SpecialInitFn(CodeGenFunction &Init, const VarDecl &D,3409 llvm::Value *Address);3410 3411 /// Determine whether the given initializer is trivial in the sense3412 /// that it requires no code to be generated.3413 bool isTrivialInitializer(const Expr *Init);3414 3415 /// EmitAutoVarDecl - Emit an auto variable declaration.3416 ///3417 /// This function can be called with a null (unreachable) insert point.3418 void EmitAutoVarDecl(const VarDecl &D);3419 3420 class AutoVarEmission {3421 friend class CodeGenFunction;3422 3423 const VarDecl *Variable;3424 3425 /// The address of the alloca for languages with explicit address space3426 /// (e.g. OpenCL) or alloca casted to generic pointer for address space3427 /// agnostic languages (e.g. C++). Invalid if the variable was emitted3428 /// as a global constant.3429 Address Addr;3430 3431 llvm::Value *NRVOFlag;3432 3433 /// True if the variable is a __block variable that is captured by an3434 /// escaping block.3435 bool IsEscapingByRef;3436 3437 /// True if the variable is of aggregate type and has a constant3438 /// initializer.3439 bool IsConstantAggregate;3440 3441 /// True if lifetime markers should be used.3442 bool UseLifetimeMarkers;3443 3444 /// Address with original alloca instruction. Invalid if the variable was3445 /// emitted as a global constant.3446 RawAddress AllocaAddr;3447 3448 struct Invalid {};3449 AutoVarEmission(Invalid)3450 : Variable(nullptr), Addr(Address::invalid()),3451 AllocaAddr(RawAddress::invalid()) {}3452 3453 AutoVarEmission(const VarDecl &variable)3454 : Variable(&variable), Addr(Address::invalid()), NRVOFlag(nullptr),3455 IsEscapingByRef(false), IsConstantAggregate(false),3456 UseLifetimeMarkers(false), AllocaAddr(RawAddress::invalid()) {}3457 3458 bool wasEmittedAsGlobal() const { return !Addr.isValid(); }3459 3460 public:3461 static AutoVarEmission invalid() { return AutoVarEmission(Invalid()); }3462 3463 bool useLifetimeMarkers() const { return UseLifetimeMarkers; }3464 3465 /// Returns the raw, allocated address, which is not necessarily3466 /// the address of the object itself. It is casted to default3467 /// address space for address space agnostic languages.3468 Address getAllocatedAddress() const { return Addr; }3469 3470 /// Returns the address for the original alloca instruction.3471 RawAddress getOriginalAllocatedAddress() const { return AllocaAddr; }3472 3473 /// Returns the address of the object within this declaration.3474 /// Note that this does not chase the forwarding pointer for3475 /// __block decls.3476 Address getObjectAddress(CodeGenFunction &CGF) const {3477 if (!IsEscapingByRef)3478 return Addr;3479 3480 return CGF.emitBlockByrefAddress(Addr, Variable, /*forward*/ false);3481 }3482 };3483 AutoVarEmission EmitAutoVarAlloca(const VarDecl &var);3484 void EmitAutoVarInit(const AutoVarEmission &emission);3485 void EmitAutoVarCleanups(const AutoVarEmission &emission);3486 void emitAutoVarTypeCleanup(const AutoVarEmission &emission,3487 QualType::DestructionKind dtorKind);3488 3489 void MaybeEmitDeferredVarDeclInit(const VarDecl *var);3490 3491 /// Emits the alloca and debug information for the size expressions for each3492 /// dimension of an array. It registers the association of its (1-dimensional)3493 /// QualTypes and size expression's debug node, so that CGDebugInfo can3494 /// reference this node when creating the DISubrange object to describe the3495 /// array types.3496 void EmitAndRegisterVariableArrayDimensions(CGDebugInfo *DI, const VarDecl &D,3497 bool EmitDebugInfo);3498 3499 void EmitStaticVarDecl(const VarDecl &D,3500 llvm::GlobalValue::LinkageTypes Linkage);3501 3502 class ParamValue {3503 union {3504 Address Addr;3505 llvm::Value *Value;3506 };3507 3508 bool IsIndirect;3509 3510 ParamValue(llvm::Value *V) : Value(V), IsIndirect(false) {}3511 ParamValue(Address A) : Addr(A), IsIndirect(true) {}3512 3513 public:3514 static ParamValue forDirect(llvm::Value *value) {3515 return ParamValue(value);3516 }3517 static ParamValue forIndirect(Address addr) {3518 assert(!addr.getAlignment().isZero());3519 return ParamValue(addr);3520 }3521 3522 bool isIndirect() const { return IsIndirect; }3523 llvm::Value *getAnyValue() const {3524 if (!isIndirect())3525 return Value;3526 assert(!Addr.hasOffset() && "unexpected offset");3527 return Addr.getBasePointer();3528 }3529 3530 llvm::Value *getDirectValue() const {3531 assert(!isIndirect());3532 return Value;3533 }3534 3535 Address getIndirectAddress() const {3536 assert(isIndirect());3537 return Addr;3538 }3539 };3540 3541 /// EmitParmDecl - Emit a ParmVarDecl or an ImplicitParamDecl.3542 void EmitParmDecl(const VarDecl &D, ParamValue Arg, unsigned ArgNo);3543 3544 /// protectFromPeepholes - Protect a value that we're intending to3545 /// store to the side, but which will probably be used later, from3546 /// aggressive peepholing optimizations that might delete it.3547 ///3548 /// Pass the result to unprotectFromPeepholes to declare that3549 /// protection is no longer required.3550 ///3551 /// There's no particular reason why this shouldn't apply to3552 /// l-values, it's just that no existing peepholes work on pointers.3553 PeepholeProtection protectFromPeepholes(RValue rvalue);3554 void unprotectFromPeepholes(PeepholeProtection protection);3555 3556 void emitAlignmentAssumptionCheck(llvm::Value *Ptr, QualType Ty,3557 SourceLocation Loc,3558 SourceLocation AssumptionLoc,3559 llvm::Value *Alignment,3560 llvm::Value *OffsetValue,3561 llvm::Value *TheCheck,3562 llvm::Instruction *Assumption);3563 3564 void emitAlignmentAssumption(llvm::Value *PtrValue, QualType Ty,3565 SourceLocation Loc, SourceLocation AssumptionLoc,3566 llvm::Value *Alignment,3567 llvm::Value *OffsetValue = nullptr);3568 3569 void emitAlignmentAssumption(llvm::Value *PtrValue, const Expr *E,3570 SourceLocation AssumptionLoc,3571 llvm::Value *Alignment,3572 llvm::Value *OffsetValue = nullptr);3573 3574 //===--------------------------------------------------------------------===//3575 // Statement Emission3576 //===--------------------------------------------------------------------===//3577 3578 /// EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.3579 void EmitStopPoint(const Stmt *S);3580 3581 /// EmitStmt - Emit the code for the statement \arg S. It is legal to call3582 /// this function even if there is no current insertion point.3583 ///3584 /// This function may clear the current insertion point; callers should use3585 /// EnsureInsertPoint if they wish to subsequently generate code without first3586 /// calling EmitBlock, EmitBranch, or EmitStmt.3587 void EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs = {});3588 3589 /// EmitSimpleStmt - Try to emit a "simple" statement which does not3590 /// necessarily require an insertion point or debug information; typically3591 /// because the statement amounts to a jump or a container of other3592 /// statements.3593 ///3594 /// \return True if the statement was handled.3595 bool EmitSimpleStmt(const Stmt *S, ArrayRef<const Attr *> Attrs);3596 3597 Address EmitCompoundStmt(const CompoundStmt &S, bool GetLast = false,3598 AggValueSlot AVS = AggValueSlot::ignored());3599 Address3600 EmitCompoundStmtWithoutScope(const CompoundStmt &S, bool GetLast = false,3601 AggValueSlot AVS = AggValueSlot::ignored());3602 3603 /// EmitLabel - Emit the block for the given label. It is legal to call this3604 /// function even if there is no current insertion point.3605 void EmitLabel(const LabelDecl *D); // helper for EmitLabelStmt.3606 3607 void EmitLabelStmt(const LabelStmt &S);3608 void EmitAttributedStmt(const AttributedStmt &S);3609 void EmitGotoStmt(const GotoStmt &S);3610 void EmitIndirectGotoStmt(const IndirectGotoStmt &S);3611 void EmitIfStmt(const IfStmt &S);3612 3613 void EmitWhileStmt(const WhileStmt &S, ArrayRef<const Attr *> Attrs = {});3614 void EmitDoStmt(const DoStmt &S, ArrayRef<const Attr *> Attrs = {});3615 void EmitForStmt(const ForStmt &S, ArrayRef<const Attr *> Attrs = {});3616 void EmitReturnStmt(const ReturnStmt &S);3617 void EmitDeclStmt(const DeclStmt &S);3618 void EmitBreakStmt(const BreakStmt &S);3619 void EmitContinueStmt(const ContinueStmt &S);3620 void EmitSwitchStmt(const SwitchStmt &S);3621 void EmitDefaultStmt(const DefaultStmt &S, ArrayRef<const Attr *> Attrs);3622 void EmitCaseStmt(const CaseStmt &S, ArrayRef<const Attr *> Attrs);3623 void EmitCaseStmtRange(const CaseStmt &S, ArrayRef<const Attr *> Attrs);3624 void EmitAsmStmt(const AsmStmt &S);3625 3626 const BreakContinue *GetDestForLoopControlStmt(const LoopControlStmt &S);3627 3628 void EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S);3629 void EmitObjCAtTryStmt(const ObjCAtTryStmt &S);3630 void EmitObjCAtThrowStmt(const ObjCAtThrowStmt &S);3631 void EmitObjCAtSynchronizedStmt(const ObjCAtSynchronizedStmt &S);3632 void EmitObjCAutoreleasePoolStmt(const ObjCAutoreleasePoolStmt &S);3633 3634 void EmitCoroutineBody(const CoroutineBodyStmt &S);3635 void EmitCoreturnStmt(const CoreturnStmt &S);3636 RValue EmitCoawaitExpr(const CoawaitExpr &E,3637 AggValueSlot aggSlot = AggValueSlot::ignored(),3638 bool ignoreResult = false);3639 LValue EmitCoawaitLValue(const CoawaitExpr *E);3640 RValue EmitCoyieldExpr(const CoyieldExpr &E,3641 AggValueSlot aggSlot = AggValueSlot::ignored(),3642 bool ignoreResult = false);3643 LValue EmitCoyieldLValue(const CoyieldExpr *E);3644 RValue EmitCoroutineIntrinsic(const CallExpr *E, unsigned int IID);3645 3646 void EnterCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false);3647 void ExitCXXTryStmt(const CXXTryStmt &S, bool IsFnTryBlock = false);3648 3649 void EmitCXXTryStmt(const CXXTryStmt &S);3650 void EmitSEHTryStmt(const SEHTryStmt &S);3651 void EmitSEHLeaveStmt(const SEHLeaveStmt &S);3652 void EnterSEHTryStmt(const SEHTryStmt &S);3653 void ExitSEHTryStmt(const SEHTryStmt &S);3654 void VolatilizeTryBlocks(llvm::BasicBlock *BB,3655 llvm::SmallPtrSet<llvm::BasicBlock *, 10> &V);3656 3657 void pushSEHCleanup(CleanupKind kind, llvm::Function *FinallyFunc);3658 void startOutlinedSEHHelper(CodeGenFunction &ParentCGF, bool IsFilter,3659 const Stmt *OutlinedStmt);3660 3661 llvm::Function *GenerateSEHFilterFunction(CodeGenFunction &ParentCGF,3662 const SEHExceptStmt &Except);3663 3664 llvm::Function *GenerateSEHFinallyFunction(CodeGenFunction &ParentCGF,3665 const SEHFinallyStmt &Finally);3666 3667 void EmitSEHExceptionCodeSave(CodeGenFunction &ParentCGF,3668 llvm::Value *ParentFP, llvm::Value *EntryEBP);3669 llvm::Value *EmitSEHExceptionCode();3670 llvm::Value *EmitSEHExceptionInfo();3671 llvm::Value *EmitSEHAbnormalTermination();3672 3673 /// Emit simple code for OpenMP directives in Simd-only mode.3674 void EmitSimpleOMPExecutableDirective(const OMPExecutableDirective &D);3675 3676 /// Scan the outlined statement for captures from the parent function. For3677 /// each capture, mark the capture as escaped and emit a call to3678 /// llvm.localrecover. Insert the localrecover result into the LocalDeclMap.3679 void EmitCapturedLocals(CodeGenFunction &ParentCGF, const Stmt *OutlinedStmt,3680 bool IsFilter);3681 3682 /// Recovers the address of a local in a parent function. ParentVar is the3683 /// address of the variable used in the immediate parent function. It can3684 /// either be an alloca or a call to llvm.localrecover if there are nested3685 /// outlined functions. ParentFP is the frame pointer of the outermost parent3686 /// frame.3687 Address recoverAddrOfEscapedLocal(CodeGenFunction &ParentCGF,3688 Address ParentVar, llvm::Value *ParentFP);3689 3690 void EmitCXXForRangeStmt(const CXXForRangeStmt &S,3691 ArrayRef<const Attr *> Attrs = {});3692 3693 /// Controls insertion of cancellation exit blocks in worksharing constructs.3694 class OMPCancelStackRAII {3695 CodeGenFunction &CGF;3696 3697 public:3698 OMPCancelStackRAII(CodeGenFunction &CGF, OpenMPDirectiveKind Kind,3699 bool HasCancel)3700 : CGF(CGF) {3701 CGF.OMPCancelStack.enter(CGF, Kind, HasCancel);3702 }3703 ~OMPCancelStackRAII() { CGF.OMPCancelStack.exit(CGF); }3704 };3705 3706 /// Returns calculated size of the specified type.3707 llvm::Value *getTypeSize(QualType Ty);3708 LValue InitCapturedStruct(const CapturedStmt &S);3709 llvm::Function *EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K);3710 llvm::Function *GenerateCapturedStmtFunction(const CapturedStmt &S);3711 Address GenerateCapturedStmtArgument(const CapturedStmt &S);3712 llvm::Function *3713 GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S,3714 const OMPExecutableDirective &D);3715 void GenerateOpenMPCapturedVars(const CapturedStmt &S,3716 SmallVectorImpl<llvm::Value *> &CapturedVars);3717 void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy,3718 SourceLocation Loc);3719 /// Perform element by element copying of arrays with type \a3720 /// OriginalType from \a SrcAddr to \a DestAddr using copying procedure3721 /// generated by \a CopyGen.3722 ///3723 /// \param DestAddr Address of the destination array.3724 /// \param SrcAddr Address of the source array.3725 /// \param OriginalType Type of destination and source arrays.3726 /// \param CopyGen Copying procedure that copies value of single array element3727 /// to another single array element.3728 void EmitOMPAggregateAssign(3729 Address DestAddr, Address SrcAddr, QualType OriginalType,3730 const llvm::function_ref<void(Address, Address)> CopyGen);3731 /// Emit proper copying of data from one variable to another.3732 ///3733 /// \param OriginalType Original type of the copied variables.3734 /// \param DestAddr Destination address.3735 /// \param SrcAddr Source address.3736 /// \param DestVD Destination variable used in \a CopyExpr (for arrays, has3737 /// type of the base array element).3738 /// \param SrcVD Source variable used in \a CopyExpr (for arrays, has type of3739 /// the base array element).3740 /// \param Copy Actual copygin expression for copying data from \a SrcVD to \a3741 /// DestVD.3742 void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr,3743 const VarDecl *DestVD, const VarDecl *SrcVD,3744 const Expr *Copy);3745 /// Emit atomic update code for constructs: \a X = \a X \a BO \a E or3746 /// \a X = \a E \a BO \a E.3747 ///3748 /// \param X Value to be updated.3749 /// \param E Update value.3750 /// \param BO Binary operation for update operation.3751 /// \param IsXLHSInRHSPart true if \a X is LHS in RHS part of the update3752 /// expression, false otherwise.3753 /// \param AO Atomic ordering of the generated atomic instructions.3754 /// \param CommonGen Code generator for complex expressions that cannot be3755 /// expressed through atomicrmw instruction.3756 /// \returns <true, OldAtomicValue> if simple 'atomicrmw' instruction was3757 /// generated, <false, RValue::get(nullptr)> otherwise.3758 std::pair<bool, RValue> EmitOMPAtomicSimpleUpdateExpr(3759 LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,3760 llvm::AtomicOrdering AO, SourceLocation Loc,3761 const llvm::function_ref<RValue(RValue)> CommonGen);3762 bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D,3763 OMPPrivateScope &PrivateScope);3764 void EmitOMPPrivateClause(const OMPExecutableDirective &D,3765 OMPPrivateScope &PrivateScope);3766 void EmitOMPUseDevicePtrClause(3767 const OMPUseDevicePtrClause &C, OMPPrivateScope &PrivateScope,3768 const llvm::DenseMap<const ValueDecl *, llvm::Value *>3769 CaptureDeviceAddrMap);3770 void EmitOMPUseDeviceAddrClause(3771 const OMPUseDeviceAddrClause &C, OMPPrivateScope &PrivateScope,3772 const llvm::DenseMap<const ValueDecl *, llvm::Value *>3773 CaptureDeviceAddrMap);3774 /// Emit code for copyin clause in \a D directive. The next code is3775 /// generated at the start of outlined functions for directives:3776 /// \code3777 /// threadprivate_var1 = master_threadprivate_var1;3778 /// operator=(threadprivate_var2, master_threadprivate_var2);3779 /// ...3780 /// __kmpc_barrier(&loc, global_tid);3781 /// \endcode3782 ///3783 /// \param D OpenMP directive possibly with 'copyin' clause(s).3784 /// \returns true if at least one copyin variable is found, false otherwise.3785 bool EmitOMPCopyinClause(const OMPExecutableDirective &D);3786 /// Emit initial code for lastprivate variables. If some variable is3787 /// not also firstprivate, then the default initialization is used. Otherwise3788 /// initialization of this variable is performed by EmitOMPFirstprivateClause3789 /// method.3790 ///3791 /// \param D Directive that may have 'lastprivate' directives.3792 /// \param PrivateScope Private scope for capturing lastprivate variables for3793 /// proper codegen in internal captured statement.3794 ///3795 /// \returns true if there is at least one lastprivate variable, false3796 /// otherwise.3797 bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D,3798 OMPPrivateScope &PrivateScope);3799 /// Emit final copying of lastprivate values to original variables at3800 /// the end of the worksharing or simd directive.3801 ///3802 /// \param D Directive that has at least one 'lastprivate' directives.3803 /// \param IsLastIterCond Boolean condition that must be set to 'i1 true' if3804 /// it is the last iteration of the loop code in associated directive, or to3805 /// 'i1 false' otherwise. If this item is nullptr, no final check is required.3806 void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D,3807 bool NoFinals,3808 llvm::Value *IsLastIterCond = nullptr);3809 /// Emit initial code for linear clauses.3810 void EmitOMPLinearClause(const OMPLoopDirective &D,3811 CodeGenFunction::OMPPrivateScope &PrivateScope);3812 /// Emit final code for linear clauses.3813 /// \param CondGen Optional conditional code for final part of codegen for3814 /// linear clause.3815 void EmitOMPLinearClauseFinal(3816 const OMPLoopDirective &D,3817 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen);3818 /// Emit initial code for reduction variables. Creates reduction copies3819 /// and initializes them with the values according to OpenMP standard.3820 ///3821 /// \param D Directive (possibly) with the 'reduction' clause.3822 /// \param PrivateScope Private scope for capturing reduction variables for3823 /// proper codegen in internal captured statement.3824 ///3825 void EmitOMPReductionClauseInit(const OMPExecutableDirective &D,3826 OMPPrivateScope &PrivateScope,3827 bool ForInscan = false);3828 /// Emit final update of reduction values to original variables at3829 /// the end of the directive.3830 ///3831 /// \param D Directive that has at least one 'reduction' directives.3832 /// \param ReductionKind The kind of reduction to perform.3833 void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D,3834 const OpenMPDirectiveKind ReductionKind);3835 /// Emit initial code for linear variables. Creates private copies3836 /// and initializes them with the values according to OpenMP standard.3837 ///3838 /// \param D Directive (possibly) with the 'linear' clause.3839 /// \return true if at least one linear variable is found that should be3840 /// initialized with the value of the original variable, false otherwise.3841 bool EmitOMPLinearClauseInit(const OMPLoopDirective &D);3842 3843 typedef const llvm::function_ref<void(CodeGenFunction & /*CGF*/,3844 llvm::Function * /*OutlinedFn*/,3845 const OMPTaskDataTy & /*Data*/)>3846 TaskGenTy;3847 void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S,3848 const OpenMPDirectiveKind CapturedRegion,3849 const RegionCodeGenTy &BodyGen,3850 const TaskGenTy &TaskGen, OMPTaskDataTy &Data);3851 struct OMPTargetDataInfo {3852 Address BasePointersArray = Address::invalid();3853 Address PointersArray = Address::invalid();3854 Address SizesArray = Address::invalid();3855 Address MappersArray = Address::invalid();3856 unsigned NumberOfTargetItems = 0;3857 explicit OMPTargetDataInfo() = default;3858 OMPTargetDataInfo(Address BasePointersArray, Address PointersArray,3859 Address SizesArray, Address MappersArray,3860 unsigned NumberOfTargetItems)3861 : BasePointersArray(BasePointersArray), PointersArray(PointersArray),3862 SizesArray(SizesArray), MappersArray(MappersArray),3863 NumberOfTargetItems(NumberOfTargetItems) {}3864 };3865 void EmitOMPTargetTaskBasedDirective(const OMPExecutableDirective &S,3866 const RegionCodeGenTy &BodyGen,3867 OMPTargetDataInfo &InputInfo);3868 void processInReduction(const OMPExecutableDirective &S, OMPTaskDataTy &Data,3869 CodeGenFunction &CGF, const CapturedStmt *CS,3870 OMPPrivateScope &Scope);3871 void EmitOMPMetaDirective(const OMPMetaDirective &S);3872 void EmitOMPParallelDirective(const OMPParallelDirective &S);3873 void EmitOMPSimdDirective(const OMPSimdDirective &S);3874 void EmitOMPTileDirective(const OMPTileDirective &S);3875 void EmitOMPStripeDirective(const OMPStripeDirective &S);3876 void EmitOMPUnrollDirective(const OMPUnrollDirective &S);3877 void EmitOMPReverseDirective(const OMPReverseDirective &S);3878 void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S);3879 void EmitOMPFuseDirective(const OMPFuseDirective &S);3880 void EmitOMPForDirective(const OMPForDirective &S);3881 void EmitOMPForSimdDirective(const OMPForSimdDirective &S);3882 void EmitOMPScopeDirective(const OMPScopeDirective &S);3883 void EmitOMPSectionsDirective(const OMPSectionsDirective &S);3884 void EmitOMPSectionDirective(const OMPSectionDirective &S);3885 void EmitOMPSingleDirective(const OMPSingleDirective &S);3886 void EmitOMPMasterDirective(const OMPMasterDirective &S);3887 void EmitOMPMaskedDirective(const OMPMaskedDirective &S);3888 void EmitOMPCriticalDirective(const OMPCriticalDirective &S);3889 void EmitOMPParallelForDirective(const OMPParallelForDirective &S);3890 void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S);3891 void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S);3892 void EmitOMPParallelMasterDirective(const OMPParallelMasterDirective &S);3893 void EmitOMPTaskDirective(const OMPTaskDirective &S);3894 void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S);3895 void EmitOMPErrorDirective(const OMPErrorDirective &S);3896 void EmitOMPBarrierDirective(const OMPBarrierDirective &S);3897 void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S);3898 void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S);3899 void EmitOMPFlushDirective(const OMPFlushDirective &S);3900 void EmitOMPDepobjDirective(const OMPDepobjDirective &S);3901 void EmitOMPScanDirective(const OMPScanDirective &S);3902 void EmitOMPOrderedDirective(const OMPOrderedDirective &S);3903 void EmitOMPAtomicDirective(const OMPAtomicDirective &S);3904 void EmitOMPTargetDirective(const OMPTargetDirective &S);3905 void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S);3906 void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S);3907 void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S);3908 void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S);3909 void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S);3910 void3911 EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S);3912 void EmitOMPTeamsDirective(const OMPTeamsDirective &S);3913 void3914 EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S);3915 void EmitOMPCancelDirective(const OMPCancelDirective &S);3916 void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S);3917 void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S);3918 void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S);3919 void EmitOMPMasterTaskLoopDirective(const OMPMasterTaskLoopDirective &S);3920 void EmitOMPMaskedTaskLoopDirective(const OMPMaskedTaskLoopDirective &S);3921 void3922 EmitOMPMasterTaskLoopSimdDirective(const OMPMasterTaskLoopSimdDirective &S);3923 void3924 EmitOMPMaskedTaskLoopSimdDirective(const OMPMaskedTaskLoopSimdDirective &S);3925 void EmitOMPParallelMasterTaskLoopDirective(3926 const OMPParallelMasterTaskLoopDirective &S);3927 void EmitOMPParallelMaskedTaskLoopDirective(3928 const OMPParallelMaskedTaskLoopDirective &S);3929 void EmitOMPParallelMasterTaskLoopSimdDirective(3930 const OMPParallelMasterTaskLoopSimdDirective &S);3931 void EmitOMPParallelMaskedTaskLoopSimdDirective(3932 const OMPParallelMaskedTaskLoopSimdDirective &S);3933 void EmitOMPDistributeDirective(const OMPDistributeDirective &S);3934 void EmitOMPDistributeParallelForDirective(3935 const OMPDistributeParallelForDirective &S);3936 void EmitOMPDistributeParallelForSimdDirective(3937 const OMPDistributeParallelForSimdDirective &S);3938 void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S);3939 void EmitOMPTargetParallelForSimdDirective(3940 const OMPTargetParallelForSimdDirective &S);3941 void EmitOMPTargetSimdDirective(const OMPTargetSimdDirective &S);3942 void EmitOMPTeamsDistributeDirective(const OMPTeamsDistributeDirective &S);3943 void3944 EmitOMPTeamsDistributeSimdDirective(const OMPTeamsDistributeSimdDirective &S);3945 void EmitOMPTeamsDistributeParallelForSimdDirective(3946 const OMPTeamsDistributeParallelForSimdDirective &S);3947 void EmitOMPTeamsDistributeParallelForDirective(3948 const OMPTeamsDistributeParallelForDirective &S);3949 void EmitOMPTargetTeamsDirective(const OMPTargetTeamsDirective &S);3950 void EmitOMPTargetTeamsDistributeDirective(3951 const OMPTargetTeamsDistributeDirective &S);3952 void EmitOMPTargetTeamsDistributeParallelForDirective(3953 const OMPTargetTeamsDistributeParallelForDirective &S);3954 void EmitOMPTargetTeamsDistributeParallelForSimdDirective(3955 const OMPTargetTeamsDistributeParallelForSimdDirective &S);3956 void EmitOMPTargetTeamsDistributeSimdDirective(3957 const OMPTargetTeamsDistributeSimdDirective &S);3958 void EmitOMPGenericLoopDirective(const OMPGenericLoopDirective &S);3959 void EmitOMPParallelGenericLoopDirective(const OMPLoopDirective &S);3960 void EmitOMPTargetParallelGenericLoopDirective(3961 const OMPTargetParallelGenericLoopDirective &S);3962 void EmitOMPTargetTeamsGenericLoopDirective(3963 const OMPTargetTeamsGenericLoopDirective &S);3964 void EmitOMPTeamsGenericLoopDirective(const OMPTeamsGenericLoopDirective &S);3965 void EmitOMPInteropDirective(const OMPInteropDirective &S);3966 void EmitOMPParallelMaskedDirective(const OMPParallelMaskedDirective &S);3967 void EmitOMPAssumeDirective(const OMPAssumeDirective &S);3968 3969 /// Emit device code for the target directive.3970 static void EmitOMPTargetDeviceFunction(CodeGenModule &CGM,3971 StringRef ParentName,3972 const OMPTargetDirective &S);3973 static void3974 EmitOMPTargetParallelDeviceFunction(CodeGenModule &CGM, StringRef ParentName,3975 const OMPTargetParallelDirective &S);3976 /// Emit device code for the target parallel for directive.3977 static void EmitOMPTargetParallelForDeviceFunction(3978 CodeGenModule &CGM, StringRef ParentName,3979 const OMPTargetParallelForDirective &S);3980 /// Emit device code for the target parallel for simd directive.3981 static void EmitOMPTargetParallelForSimdDeviceFunction(3982 CodeGenModule &CGM, StringRef ParentName,3983 const OMPTargetParallelForSimdDirective &S);3984 /// Emit device code for the target teams directive.3985 static void3986 EmitOMPTargetTeamsDeviceFunction(CodeGenModule &CGM, StringRef ParentName,3987 const OMPTargetTeamsDirective &S);3988 /// Emit device code for the target teams distribute directive.3989 static void EmitOMPTargetTeamsDistributeDeviceFunction(3990 CodeGenModule &CGM, StringRef ParentName,3991 const OMPTargetTeamsDistributeDirective &S);3992 /// Emit device code for the target teams distribute simd directive.3993 static void EmitOMPTargetTeamsDistributeSimdDeviceFunction(3994 CodeGenModule &CGM, StringRef ParentName,3995 const OMPTargetTeamsDistributeSimdDirective &S);3996 /// Emit device code for the target simd directive.3997 static void EmitOMPTargetSimdDeviceFunction(CodeGenModule &CGM,3998 StringRef ParentName,3999 const OMPTargetSimdDirective &S);4000 /// Emit device code for the target teams distribute parallel for simd4001 /// directive.4002 static void EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(4003 CodeGenModule &CGM, StringRef ParentName,4004 const OMPTargetTeamsDistributeParallelForSimdDirective &S);4005 4006 /// Emit device code for the target teams loop directive.4007 static void EmitOMPTargetTeamsGenericLoopDeviceFunction(4008 CodeGenModule &CGM, StringRef ParentName,4009 const OMPTargetTeamsGenericLoopDirective &S);4010 4011 /// Emit device code for the target parallel loop directive.4012 static void EmitOMPTargetParallelGenericLoopDeviceFunction(4013 CodeGenModule &CGM, StringRef ParentName,4014 const OMPTargetParallelGenericLoopDirective &S);4015 4016 static void EmitOMPTargetTeamsDistributeParallelForDeviceFunction(4017 CodeGenModule &CGM, StringRef ParentName,4018 const OMPTargetTeamsDistributeParallelForDirective &S);4019 4020 /// Emit the Stmt \p S and return its topmost canonical loop, if any.4021 /// TODO: The \p Depth paramter is not yet implemented and must be 1. In the4022 /// future it is meant to be the number of loops expected in the loop nests4023 /// (usually specified by the "collapse" clause) that are collapsed to a4024 /// single loop by this function.4025 llvm::CanonicalLoopInfo *EmitOMPCollapsedCanonicalLoopNest(const Stmt *S,4026 int Depth);4027 4028 /// Emit an OMPCanonicalLoop using the OpenMPIRBuilder.4029 void EmitOMPCanonicalLoop(const OMPCanonicalLoop *S);4030 4031 /// Emit inner loop of the worksharing/simd construct.4032 ///4033 /// \param S Directive, for which the inner loop must be emitted.4034 /// \param RequiresCleanup true, if directive has some associated private4035 /// variables.4036 /// \param LoopCond Bollean condition for loop continuation.4037 /// \param IncExpr Increment expression for loop control variable.4038 /// \param BodyGen Generator for the inner body of the inner loop.4039 /// \param PostIncGen Genrator for post-increment code (required for ordered4040 /// loop directvies).4041 void EmitOMPInnerLoop(4042 const OMPExecutableDirective &S, bool RequiresCleanup,4043 const Expr *LoopCond, const Expr *IncExpr,4044 const llvm::function_ref<void(CodeGenFunction &)> BodyGen,4045 const llvm::function_ref<void(CodeGenFunction &)> PostIncGen);4046 4047 JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind);4048 /// Emit initial code for loop counters of loop-based directives.4049 void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S,4050 OMPPrivateScope &LoopScope);4051 4052 /// Helper for the OpenMP loop directives.4053 void EmitOMPLoopBody(const OMPLoopDirective &D, JumpDest LoopExit);4054 4055 /// Emit code for the worksharing loop-based directive.4056 /// \return true, if this construct has any lastprivate clause, false -4057 /// otherwise.4058 bool EmitOMPWorksharingLoop(const OMPLoopDirective &S, Expr *EUB,4059 const CodeGenLoopBoundsTy &CodeGenLoopBounds,4060 const CodeGenDispatchBoundsTy &CGDispatchBounds);4061 4062 /// Emit code for the distribute loop-based directive.4063 void EmitOMPDistributeLoop(const OMPLoopDirective &S,4064 const CodeGenLoopTy &CodeGenLoop, Expr *IncExpr);4065 4066 /// Helpers for the OpenMP loop directives.4067 void EmitOMPSimdInit(const OMPLoopDirective &D);4068 void EmitOMPSimdFinal(4069 const OMPLoopDirective &D,4070 const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen);4071 4072 /// Emits the lvalue for the expression with possibly captured variable.4073 LValue EmitOMPSharedLValue(const Expr *E);4074 4075private:4076 /// Helpers for blocks.4077 llvm::Value *EmitBlockLiteral(const CGBlockInfo &Info);4078 4079 /// struct with the values to be passed to the OpenMP loop-related functions4080 struct OMPLoopArguments {4081 /// loop lower bound4082 Address LB = Address::invalid();4083 /// loop upper bound4084 Address UB = Address::invalid();4085 /// loop stride4086 Address ST = Address::invalid();4087 /// isLastIteration argument for runtime functions4088 Address IL = Address::invalid();4089 /// Chunk value generated by sema4090 llvm::Value *Chunk = nullptr;4091 /// EnsureUpperBound4092 Expr *EUB = nullptr;4093 /// IncrementExpression4094 Expr *IncExpr = nullptr;4095 /// Loop initialization4096 Expr *Init = nullptr;4097 /// Loop exit condition4098 Expr *Cond = nullptr;4099 /// Update of LB after a whole chunk has been executed4100 Expr *NextLB = nullptr;4101 /// Update of UB after a whole chunk has been executed4102 Expr *NextUB = nullptr;4103 /// Distinguish between the for distribute and sections4104 OpenMPDirectiveKind DKind = llvm::omp::OMPD_unknown;4105 OMPLoopArguments() = default;4106 OMPLoopArguments(Address LB, Address UB, Address ST, Address IL,4107 llvm::Value *Chunk = nullptr, Expr *EUB = nullptr,4108 Expr *IncExpr = nullptr, Expr *Init = nullptr,4109 Expr *Cond = nullptr, Expr *NextLB = nullptr,4110 Expr *NextUB = nullptr)4111 : LB(LB), UB(UB), ST(ST), IL(IL), Chunk(Chunk), EUB(EUB),4112 IncExpr(IncExpr), Init(Init), Cond(Cond), NextLB(NextLB),4113 NextUB(NextUB) {}4114 };4115 void EmitOMPOuterLoop(bool DynamicOrOrdered, bool IsMonotonic,4116 const OMPLoopDirective &S, OMPPrivateScope &LoopScope,4117 const OMPLoopArguments &LoopArgs,4118 const CodeGenLoopTy &CodeGenLoop,4119 const CodeGenOrderedTy &CodeGenOrdered);4120 void EmitOMPForOuterLoop(const OpenMPScheduleTy &ScheduleKind,4121 bool IsMonotonic, const OMPLoopDirective &S,4122 OMPPrivateScope &LoopScope, bool Ordered,4123 const OMPLoopArguments &LoopArgs,4124 const CodeGenDispatchBoundsTy &CGDispatchBounds);4125 void EmitOMPDistributeOuterLoop(OpenMPDistScheduleClauseKind ScheduleKind,4126 const OMPLoopDirective &S,4127 OMPPrivateScope &LoopScope,4128 const OMPLoopArguments &LoopArgs,4129 const CodeGenLoopTy &CodeGenLoopContent);4130 /// Emit code for sections directive.4131 void EmitSections(const OMPExecutableDirective &S);4132 4133public:4134 //===--------------------------------------------------------------------===//4135 // OpenACC Emission4136 //===--------------------------------------------------------------------===//4137 void EmitOpenACCComputeConstruct(const OpenACCComputeConstruct &S) {4138 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',4139 // simply emitting its structured block, but in the future we will implement4140 // some sort of IR.4141 EmitStmt(S.getStructuredBlock());4142 }4143 4144 void EmitOpenACCLoopConstruct(const OpenACCLoopConstruct &S) {4145 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',4146 // simply emitting its loop, but in the future we will implement4147 // some sort of IR.4148 EmitStmt(S.getLoop());4149 }4150 4151 void EmitOpenACCCombinedConstruct(const OpenACCCombinedConstruct &S) {4152 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',4153 // simply emitting its loop, but in the future we will implement4154 // some sort of IR.4155 EmitStmt(S.getLoop());4156 }4157 4158 void EmitOpenACCDataConstruct(const OpenACCDataConstruct &S) {4159 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',4160 // simply emitting its structured block, but in the future we will implement4161 // some sort of IR.4162 EmitStmt(S.getStructuredBlock());4163 }4164 4165 void EmitOpenACCEnterDataConstruct(const OpenACCEnterDataConstruct &S) {4166 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',4167 // but in the future we will implement some sort of IR.4168 }4169 4170 void EmitOpenACCExitDataConstruct(const OpenACCExitDataConstruct &S) {4171 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',4172 // but in the future we will implement some sort of IR.4173 }4174 4175 void EmitOpenACCHostDataConstruct(const OpenACCHostDataConstruct &S) {4176 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',4177 // simply emitting its structured block, but in the future we will implement4178 // some sort of IR.4179 EmitStmt(S.getStructuredBlock());4180 }4181 4182 void EmitOpenACCWaitConstruct(const OpenACCWaitConstruct &S) {4183 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',4184 // but in the future we will implement some sort of IR.4185 }4186 4187 void EmitOpenACCInitConstruct(const OpenACCInitConstruct &S) {4188 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',4189 // but in the future we will implement some sort of IR.4190 }4191 4192 void EmitOpenACCShutdownConstruct(const OpenACCShutdownConstruct &S) {4193 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',4194 // but in the future we will implement some sort of IR.4195 }4196 4197 void EmitOpenACCSetConstruct(const OpenACCSetConstruct &S) {4198 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',4199 // but in the future we will implement some sort of IR.4200 }4201 4202 void EmitOpenACCUpdateConstruct(const OpenACCUpdateConstruct &S) {4203 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',4204 // but in the future we will implement some sort of IR.4205 }4206 4207 void EmitOpenACCAtomicConstruct(const OpenACCAtomicConstruct &S) {4208 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',4209 // simply emitting its associated stmt, but in the future we will implement4210 // some sort of IR.4211 EmitStmt(S.getAssociatedStmt());4212 }4213 void EmitOpenACCCacheConstruct(const OpenACCCacheConstruct &S) {4214 // TODO OpenACC: Implement this. It is currently implemented as a 'no-op',4215 // but in the future we will implement some sort of IR.4216 }4217 4218 //===--------------------------------------------------------------------===//4219 // LValue Expression Emission4220 //===--------------------------------------------------------------------===//4221 4222 /// Create a check that a scalar RValue is non-null.4223 llvm::Value *EmitNonNullRValueCheck(RValue RV, QualType T);4224 4225 /// GetUndefRValue - Get an appropriate 'undef' rvalue for the given type.4226 RValue GetUndefRValue(QualType Ty);4227 4228 /// EmitUnsupportedRValue - Emit a dummy r-value using the type of E4229 /// and issue an ErrorUnsupported style diagnostic (using the4230 /// provided Name).4231 RValue EmitUnsupportedRValue(const Expr *E, const char *Name);4232 4233 /// EmitUnsupportedLValue - Emit a dummy l-value using the type of E and issue4234 /// an ErrorUnsupported style diagnostic (using the provided Name).4235 LValue EmitUnsupportedLValue(const Expr *E, const char *Name);4236 4237 /// EmitLValue - Emit code to compute a designator that specifies the location4238 /// of the expression.4239 ///4240 /// This can return one of two things: a simple address or a bitfield4241 /// reference. In either case, the LLVM Value* in the LValue structure is4242 /// guaranteed to be an LLVM pointer type.4243 ///4244 /// If this returns a bitfield reference, nothing about the pointee type of4245 /// the LLVM value is known: For example, it may not be a pointer to an4246 /// integer.4247 ///4248 /// If this returns a normal address, and if the lvalue's C type is fixed4249 /// size, this method guarantees that the returned pointer type will point to4250 /// an LLVM type of the same size of the lvalue's type. If the lvalue has a4251 /// variable length type, this is not possible.4252 ///4253 LValue EmitLValue(const Expr *E,4254 KnownNonNull_t IsKnownNonNull = NotKnownNonNull);4255 4256private:4257 LValue EmitLValueHelper(const Expr *E, KnownNonNull_t IsKnownNonNull);4258 4259public:4260 /// Same as EmitLValue but additionally we generate checking code to4261 /// guard against undefined behavior. This is only suitable when we know4262 /// that the address will be used to access the object.4263 LValue EmitCheckedLValue(const Expr *E, TypeCheckKind TCK);4264 4265 RValue convertTempToRValue(Address addr, QualType type, SourceLocation Loc);4266 4267 void EmitAtomicInit(Expr *E, LValue lvalue);4268 4269 bool LValueIsSuitableForInlineAtomic(LValue Src);4270 4271 RValue EmitAtomicLoad(LValue LV, SourceLocation SL,4272 AggValueSlot Slot = AggValueSlot::ignored());4273 4274 RValue EmitAtomicLoad(LValue lvalue, SourceLocation loc,4275 llvm::AtomicOrdering AO, bool IsVolatile = false,4276 AggValueSlot slot = AggValueSlot::ignored());4277 4278 void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit);4279 4280 void EmitAtomicStore(RValue rvalue, LValue lvalue, llvm::AtomicOrdering AO,4281 bool IsVolatile, bool isInit);4282 4283 std::pair<RValue, llvm::Value *> EmitAtomicCompareExchange(4284 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,4285 llvm::AtomicOrdering Success =4286 llvm::AtomicOrdering::SequentiallyConsistent,4287 llvm::AtomicOrdering Failure =4288 llvm::AtomicOrdering::SequentiallyConsistent,4289 bool IsWeak = false, AggValueSlot Slot = AggValueSlot::ignored());4290 4291 /// Emit an atomicrmw instruction, and applying relevant metadata when4292 /// applicable.4293 llvm::AtomicRMWInst *emitAtomicRMWInst(4294 llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val,4295 llvm::AtomicOrdering Order = llvm::AtomicOrdering::SequentiallyConsistent,4296 llvm::SyncScope::ID SSID = llvm::SyncScope::System,4297 const AtomicExpr *AE = nullptr);4298 4299 void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO,4300 const llvm::function_ref<RValue(RValue)> &UpdateOp,4301 bool IsVolatile);4302 4303 /// EmitToMemory - Change a scalar value from its value4304 /// representation to its in-memory representation.4305 llvm::Value *EmitToMemory(llvm::Value *Value, QualType Ty);4306 4307 /// EmitFromMemory - Change a scalar value from its memory4308 /// representation to its value representation.4309 llvm::Value *EmitFromMemory(llvm::Value *Value, QualType Ty);4310 4311 /// Check if the scalar \p Value is within the valid range for the given4312 /// type \p Ty.4313 ///4314 /// Returns true if a check is needed (even if the range is unknown).4315 bool EmitScalarRangeCheck(llvm::Value *Value, QualType Ty,4316 SourceLocation Loc);4317 4318 /// EmitLoadOfScalar - Load a scalar value from an address, taking4319 /// care to appropriately convert from the memory representation to4320 /// the LLVM value representation.4321 llvm::Value *EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty,4322 SourceLocation Loc,4323 AlignmentSource Source = AlignmentSource::Type,4324 bool isNontemporal = false) {4325 return EmitLoadOfScalar(Addr, Volatile, Ty, Loc, LValueBaseInfo(Source),4326 CGM.getTBAAAccessInfo(Ty), isNontemporal);4327 }4328 4329 llvm::Value *EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty,4330 SourceLocation Loc, LValueBaseInfo BaseInfo,4331 TBAAAccessInfo TBAAInfo,4332 bool isNontemporal = false);4333 4334 /// EmitLoadOfScalar - Load a scalar value from an address, taking4335 /// care to appropriately convert from the memory representation to4336 /// the LLVM value representation. The l-value must be a simple4337 /// l-value.4338 llvm::Value *EmitLoadOfScalar(LValue lvalue, SourceLocation Loc);4339 4340 /// EmitStoreOfScalar - Store a scalar value to an address, taking4341 /// care to appropriately convert from the memory representation to4342 /// the LLVM value representation.4343 void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile,4344 QualType Ty,4345 AlignmentSource Source = AlignmentSource::Type,4346 bool isInit = false, bool isNontemporal = false) {4347 EmitStoreOfScalar(Value, Addr, Volatile, Ty, LValueBaseInfo(Source),4348 CGM.getTBAAAccessInfo(Ty), isInit, isNontemporal);4349 }4350 4351 void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile,4352 QualType Ty, LValueBaseInfo BaseInfo,4353 TBAAAccessInfo TBAAInfo, bool isInit = false,4354 bool isNontemporal = false);4355 4356 /// EmitStoreOfScalar - Store a scalar value to an address, taking4357 /// care to appropriately convert from the memory representation to4358 /// the LLVM value representation. The l-value must be a simple4359 /// l-value. The isInit flag indicates whether this is an initialization.4360 /// If so, atomic qualifiers are ignored and the store is always non-atomic.4361 void EmitStoreOfScalar(llvm::Value *value, LValue lvalue,4362 bool isInit = false);4363 4364 /// EmitLoadOfLValue - Given an expression that represents a value lvalue,4365 /// this method emits the address of the lvalue, then loads the result as an4366 /// rvalue, returning the rvalue.4367 RValue EmitLoadOfLValue(LValue V, SourceLocation Loc);4368 RValue EmitLoadOfExtVectorElementLValue(LValue V);4369 RValue EmitLoadOfBitfieldLValue(LValue LV, SourceLocation Loc);4370 RValue EmitLoadOfGlobalRegLValue(LValue LV);4371 4372 /// Like EmitLoadOfLValue but also handles complex and aggregate types.4373 RValue EmitLoadOfAnyValue(LValue V,4374 AggValueSlot Slot = AggValueSlot::ignored(),4375 SourceLocation Loc = {});4376 4377 /// EmitStoreThroughLValue - Store the specified rvalue into the specified4378 /// lvalue, where both are guaranteed to the have the same type, and that type4379 /// is 'Ty'.4380 void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit = false);4381 void EmitStoreThroughExtVectorComponentLValue(RValue Src, LValue Dst);4382 void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst);4383 4384 /// EmitStoreThroughBitfieldLValue - Store Src into Dst with same constraints4385 /// as EmitStoreThroughLValue.4386 ///4387 /// \param Result [out] - If non-null, this will be set to a Value* for the4388 /// bit-field contents after the store, appropriate for use as the result of4389 /// an assignment to the bit-field.4390 void EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,4391 llvm::Value **Result = nullptr);4392 4393 /// Emit an l-value for an assignment (simple or compound) of complex type.4394 LValue EmitComplexAssignmentLValue(const BinaryOperator *E);4395 LValue EmitComplexCompoundAssignmentLValue(const CompoundAssignOperator *E);4396 LValue EmitScalarCompoundAssignWithComplex(const CompoundAssignOperator *E,4397 llvm::Value *&Result);4398 4399 // Note: only available for agg return types4400 LValue EmitBinaryOperatorLValue(const BinaryOperator *E);4401 LValue EmitCompoundAssignmentLValue(const CompoundAssignOperator *E);4402 // Note: only available for agg return types4403 LValue EmitCallExprLValue(const CallExpr *E,4404 llvm::CallBase **CallOrInvoke = nullptr);4405 // Note: only available for agg return types4406 LValue EmitVAArgExprLValue(const VAArgExpr *E);4407 LValue EmitDeclRefLValue(const DeclRefExpr *E);4408 LValue EmitStringLiteralLValue(const StringLiteral *E);4409 LValue EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E);4410 LValue EmitPredefinedLValue(const PredefinedExpr *E);4411 LValue EmitUnaryOpLValue(const UnaryOperator *E);4412 LValue EmitArraySubscriptExpr(const ArraySubscriptExpr *E,4413 bool Accessed = false);4414 llvm::Value *EmitMatrixIndexExpr(const Expr *E);4415 LValue EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E);4416 LValue EmitArraySectionExpr(const ArraySectionExpr *E,4417 bool IsLowerBound = true);4418 LValue EmitExtVectorElementExpr(const ExtVectorElementExpr *E);4419 LValue EmitMemberExpr(const MemberExpr *E);4420 LValue EmitObjCIsaExpr(const ObjCIsaExpr *E);4421 LValue EmitCompoundLiteralLValue(const CompoundLiteralExpr *E);4422 LValue EmitInitListLValue(const InitListExpr *E);4423 void EmitIgnoredConditionalOperator(const AbstractConditionalOperator *E);4424 LValue EmitConditionalOperatorLValue(const AbstractConditionalOperator *E);4425 LValue EmitCastLValue(const CastExpr *E);4426 LValue EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *E);4427 LValue EmitOpaqueValueLValue(const OpaqueValueExpr *e);4428 LValue EmitHLSLArrayAssignLValue(const BinaryOperator *E);4429 4430 std::pair<LValue, LValue> EmitHLSLOutArgLValues(const HLSLOutArgExpr *E,4431 QualType Ty);4432 LValue EmitHLSLOutArgExpr(const HLSLOutArgExpr *E, CallArgList &Args,4433 QualType Ty);4434 4435 Address EmitExtVectorElementLValue(LValue V);4436 4437 RValue EmitRValueForField(LValue LV, const FieldDecl *FD, SourceLocation Loc);4438 4439 Address EmitArrayToPointerDecay(const Expr *Array,4440 LValueBaseInfo *BaseInfo = nullptr,4441 TBAAAccessInfo *TBAAInfo = nullptr);4442 4443 class ConstantEmission {4444 llvm::PointerIntPair<llvm::Constant *, 1, bool> ValueAndIsReference;4445 ConstantEmission(llvm::Constant *C, bool isReference)4446 : ValueAndIsReference(C, isReference) {}4447 4448 public:4449 ConstantEmission() {}4450 static ConstantEmission forReference(llvm::Constant *C) {4451 return ConstantEmission(C, true);4452 }4453 static ConstantEmission forValue(llvm::Constant *C) {4454 return ConstantEmission(C, false);4455 }4456 4457 explicit operator bool() const {4458 return ValueAndIsReference.getOpaqueValue() != nullptr;4459 }4460 4461 bool isReference() const { return ValueAndIsReference.getInt(); }4462 LValue getReferenceLValue(CodeGenFunction &CGF, const Expr *RefExpr) const {4463 assert(isReference());4464 return CGF.MakeNaturalAlignAddrLValue(ValueAndIsReference.getPointer(),4465 RefExpr->getType());4466 }4467 4468 llvm::Constant *getValue() const {4469 assert(!isReference());4470 return ValueAndIsReference.getPointer();4471 }4472 };4473 4474 ConstantEmission tryEmitAsConstant(const DeclRefExpr *RefExpr);4475 ConstantEmission tryEmitAsConstant(const MemberExpr *ME);4476 llvm::Value *emitScalarConstant(const ConstantEmission &Constant, Expr *E);4477 4478 RValue EmitPseudoObjectRValue(const PseudoObjectExpr *e,4479 AggValueSlot slot = AggValueSlot::ignored());4480 LValue EmitPseudoObjectLValue(const PseudoObjectExpr *e);4481 4482 void FlattenAccessAndTypeLValue(LValue LVal,4483 SmallVectorImpl<LValue> &AccessList);4484 4485 llvm::Value *EmitIvarOffset(const ObjCInterfaceDecl *Interface,4486 const ObjCIvarDecl *Ivar);4487 llvm::Value *EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl *Interface,4488 const ObjCIvarDecl *Ivar);4489 LValue EmitLValueForField(LValue Base, const FieldDecl *Field,4490 bool IsInBounds = true);4491 LValue EmitLValueForLambdaField(const FieldDecl *Field);4492 LValue EmitLValueForLambdaField(const FieldDecl *Field,4493 llvm::Value *ThisValue);4494 4495 /// EmitLValueForFieldInitialization - Like EmitLValueForField, except that4496 /// if the Field is a reference, this will return the address of the reference4497 /// and not the address of the value stored in the reference.4498 LValue EmitLValueForFieldInitialization(LValue Base, const FieldDecl *Field);4499 4500 LValue EmitLValueForIvar(QualType ObjectTy, llvm::Value *Base,4501 const ObjCIvarDecl *Ivar, unsigned CVRQualifiers);4502 4503 LValue EmitCXXConstructLValue(const CXXConstructExpr *E);4504 LValue EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E);4505 LValue EmitCXXTypeidLValue(const CXXTypeidExpr *E);4506 LValue EmitCXXUuidofLValue(const CXXUuidofExpr *E);4507 4508 LValue EmitObjCMessageExprLValue(const ObjCMessageExpr *E);4509 LValue EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E);4510 LValue EmitStmtExprLValue(const StmtExpr *E);4511 LValue EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E);4512 LValue EmitObjCSelectorLValue(const ObjCSelectorExpr *E);4513 void EmitDeclRefExprDbgValue(const DeclRefExpr *E, const APValue &Init);4514 4515 //===--------------------------------------------------------------------===//4516 // Scalar Expression Emission4517 //===--------------------------------------------------------------------===//4518 4519 /// EmitCall - Generate a call of the given function, expecting the given4520 /// result type, and using the given argument list which specifies both the4521 /// LLVM arguments and the types they were derived from.4522 RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee,4523 ReturnValueSlot ReturnValue, const CallArgList &Args,4524 llvm::CallBase **CallOrInvoke, bool IsMustTail,4525 SourceLocation Loc,4526 bool IsVirtualFunctionPointerThunk = false);4527 RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee,4528 ReturnValueSlot ReturnValue, const CallArgList &Args,4529 llvm::CallBase **CallOrInvoke = nullptr,4530 bool IsMustTail = false) {4531 return EmitCall(CallInfo, Callee, ReturnValue, Args, CallOrInvoke,4532 IsMustTail, SourceLocation());4533 }4534 RValue EmitCall(QualType FnType, const CGCallee &Callee, const CallExpr *E,4535 ReturnValueSlot ReturnValue, llvm::Value *Chain = nullptr,4536 llvm::CallBase **CallOrInvoke = nullptr,4537 CGFunctionInfo const **ResolvedFnInfo = nullptr);4538 4539 // If a Call or Invoke instruction was emitted for this CallExpr, this method4540 // writes the pointer to `CallOrInvoke` if it's not null.4541 RValue EmitCallExpr(const CallExpr *E,4542 ReturnValueSlot ReturnValue = ReturnValueSlot(),4543 llvm::CallBase **CallOrInvoke = nullptr);4544 RValue EmitSimpleCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue,4545 llvm::CallBase **CallOrInvoke = nullptr);4546 CGCallee EmitCallee(const Expr *E);4547 4548 void checkTargetFeatures(const CallExpr *E, const FunctionDecl *TargetDecl);4549 void checkTargetFeatures(SourceLocation Loc, const FunctionDecl *TargetDecl);4550 4551 llvm::CallInst *EmitRuntimeCall(llvm::FunctionCallee callee,4552 const Twine &name = "");4553 llvm::CallInst *EmitRuntimeCall(llvm::FunctionCallee callee,4554 ArrayRef<llvm::Value *> args,4555 const Twine &name = "");4556 llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,4557 const Twine &name = "");4558 llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,4559 ArrayRef<Address> args,4560 const Twine &name = "");4561 llvm::CallInst *EmitNounwindRuntimeCall(llvm::FunctionCallee callee,4562 ArrayRef<llvm::Value *> args,4563 const Twine &name = "");4564 4565 SmallVector<llvm::OperandBundleDef, 1>4566 getBundlesForFunclet(llvm::Value *Callee);4567 4568 llvm::CallBase *EmitCallOrInvoke(llvm::FunctionCallee Callee,4569 ArrayRef<llvm::Value *> Args,4570 const Twine &Name = "");4571 llvm::CallBase *EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,4572 ArrayRef<llvm::Value *> args,4573 const Twine &name = "");4574 llvm::CallBase *EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,4575 const Twine &name = "");4576 void EmitNoreturnRuntimeCallOrInvoke(llvm::FunctionCallee callee,4577 ArrayRef<llvm::Value *> args);4578 4579 CGCallee BuildAppleKextVirtualCall(const CXXMethodDecl *MD,4580 NestedNameSpecifier Qual, llvm::Type *Ty);4581 4582 CGCallee BuildAppleKextVirtualDestructorCall(const CXXDestructorDecl *DD,4583 CXXDtorType Type,4584 const CXXRecordDecl *RD);4585 4586 bool isPointerKnownNonNull(const Expr *E);4587 /// Check whether the underlying base pointer is a constant null.4588 bool isUnderlyingBasePointerConstantNull(const Expr *E);4589 4590 /// Create the discriminator from the storage address and the entity hash.4591 llvm::Value *EmitPointerAuthBlendDiscriminator(llvm::Value *StorageAddress,4592 llvm::Value *Discriminator);4593 CGPointerAuthInfo EmitPointerAuthInfo(const PointerAuthSchema &Schema,4594 llvm::Value *StorageAddress,4595 GlobalDecl SchemaDecl,4596 QualType SchemaType);4597 4598 llvm::Value *EmitPointerAuthSign(const CGPointerAuthInfo &Info,4599 llvm::Value *Pointer);4600 4601 llvm::Value *EmitPointerAuthAuth(const CGPointerAuthInfo &Info,4602 llvm::Value *Pointer);4603 4604 llvm::Value *emitPointerAuthResign(llvm::Value *Pointer, QualType PointerType,4605 const CGPointerAuthInfo &CurAuthInfo,4606 const CGPointerAuthInfo &NewAuthInfo,4607 bool IsKnownNonNull);4608 llvm::Value *emitPointerAuthResignCall(llvm::Value *Pointer,4609 const CGPointerAuthInfo &CurInfo,4610 const CGPointerAuthInfo &NewInfo);4611 4612 void EmitPointerAuthOperandBundle(4613 const CGPointerAuthInfo &Info,4614 SmallVectorImpl<llvm::OperandBundleDef> &Bundles);4615 4616 CGPointerAuthInfo EmitPointerAuthInfo(PointerAuthQualifier Qualifier,4617 Address StorageAddress);4618 llvm::Value *EmitPointerAuthQualify(PointerAuthQualifier Qualifier,4619 llvm::Value *Pointer, QualType ValueType,4620 Address StorageAddress,4621 bool IsKnownNonNull);4622 llvm::Value *EmitPointerAuthQualify(PointerAuthQualifier Qualifier,4623 const Expr *PointerExpr,4624 Address StorageAddress);4625 llvm::Value *EmitPointerAuthUnqualify(PointerAuthQualifier Qualifier,4626 llvm::Value *Pointer,4627 QualType PointerType,4628 Address StorageAddress,4629 bool IsKnownNonNull);4630 void EmitPointerAuthCopy(PointerAuthQualifier Qualifier, QualType Type,4631 Address DestField, Address SrcField);4632 4633 std::pair<llvm::Value *, CGPointerAuthInfo>4634 EmitOrigPointerRValue(const Expr *E);4635 4636 llvm::Value *authPointerToPointerCast(llvm::Value *ResultPtr,4637 QualType SourceType, QualType DestType);4638 Address authPointerToPointerCast(Address Ptr, QualType SourceType,4639 QualType DestType);4640 4641 Address getAsNaturalAddressOf(Address Addr, QualType PointeeTy);4642 4643 llvm::Value *getAsNaturalPointerTo(Address Addr, QualType PointeeType) {4644 return getAsNaturalAddressOf(Addr, PointeeType).getBasePointer();4645 }4646 4647 // Return the copy constructor name with the prefix "__copy_constructor_"4648 // removed.4649 static std::string getNonTrivialCopyConstructorStr(QualType QT,4650 CharUnits Alignment,4651 bool IsVolatile,4652 ASTContext &Ctx);4653 4654 // Return the destructor name with the prefix "__destructor_" removed.4655 static std::string getNonTrivialDestructorStr(QualType QT,4656 CharUnits Alignment,4657 bool IsVolatile,4658 ASTContext &Ctx);4659 4660 // These functions emit calls to the special functions of non-trivial C4661 // structs.4662 void defaultInitNonTrivialCStructVar(LValue Dst);4663 void callCStructDefaultConstructor(LValue Dst);4664 void callCStructDestructor(LValue Dst);4665 void callCStructCopyConstructor(LValue Dst, LValue Src);4666 void callCStructMoveConstructor(LValue Dst, LValue Src);4667 void callCStructCopyAssignmentOperator(LValue Dst, LValue Src);4668 void callCStructMoveAssignmentOperator(LValue Dst, LValue Src);4669 4670 RValue EmitCXXMemberOrOperatorCall(4671 const CXXMethodDecl *Method, const CGCallee &Callee,4672 ReturnValueSlot ReturnValue, llvm::Value *This,4673 llvm::Value *ImplicitParam, QualType ImplicitParamTy, const CallExpr *E,4674 CallArgList *RtlArgs, llvm::CallBase **CallOrInvoke);4675 RValue EmitCXXDestructorCall(GlobalDecl Dtor, const CGCallee &Callee,4676 llvm::Value *This, QualType ThisTy,4677 llvm::Value *ImplicitParam,4678 QualType ImplicitParamTy, const CallExpr *E,4679 llvm::CallBase **CallOrInvoke = nullptr);4680 RValue EmitCXXMemberCallExpr(const CXXMemberCallExpr *E,4681 ReturnValueSlot ReturnValue,4682 llvm::CallBase **CallOrInvoke = nullptr);4683 RValue EmitCXXMemberOrOperatorMemberCallExpr(4684 const CallExpr *CE, const CXXMethodDecl *MD, ReturnValueSlot ReturnValue,4685 bool HasQualifier, NestedNameSpecifier Qualifier, bool IsArrow,4686 const Expr *Base, llvm::CallBase **CallOrInvoke);4687 // Compute the object pointer.4688 Address EmitCXXMemberDataPointerAddress(4689 const Expr *E, Address base, llvm::Value *memberPtr,4690 const MemberPointerType *memberPtrType, bool IsInBounds,4691 LValueBaseInfo *BaseInfo = nullptr, TBAAAccessInfo *TBAAInfo = nullptr);4692 RValue EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,4693 ReturnValueSlot ReturnValue,4694 llvm::CallBase **CallOrInvoke);4695 4696 RValue EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E,4697 const CXXMethodDecl *MD,4698 ReturnValueSlot ReturnValue,4699 llvm::CallBase **CallOrInvoke);4700 RValue EmitCXXPseudoDestructorExpr(const CXXPseudoDestructorExpr *E);4701 4702 RValue EmitCUDAKernelCallExpr(const CUDAKernelCallExpr *E,4703 ReturnValueSlot ReturnValue,4704 llvm::CallBase **CallOrInvoke);4705 4706 RValue EmitNVPTXDevicePrintfCallExpr(const CallExpr *E);4707 RValue EmitAMDGPUDevicePrintfCallExpr(const CallExpr *E);4708 4709 RValue EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,4710 const CallExpr *E, ReturnValueSlot ReturnValue);4711 4712 RValue emitRotate(const CallExpr *E, bool IsRotateRight);4713 4714 /// Emit IR for __builtin_os_log_format.4715 RValue emitBuiltinOSLogFormat(const CallExpr &E);4716 4717 /// Emit IR for __builtin_is_aligned.4718 RValue EmitBuiltinIsAligned(const CallExpr *E);4719 /// Emit IR for __builtin_align_up/__builtin_align_down.4720 RValue EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp);4721 4722 llvm::Function *generateBuiltinOSLogHelperFunction(4723 const analyze_os_log::OSLogBufferLayout &Layout,4724 CharUnits BufferAlignment);4725 4726 RValue EmitBlockCallExpr(const CallExpr *E, ReturnValueSlot ReturnValue,4727 llvm::CallBase **CallOrInvoke);4728 4729 /// EmitTargetBuiltinExpr - Emit the given builtin call. Returns 0 if the call4730 /// is unhandled by the current target.4731 llvm::Value *EmitTargetBuiltinExpr(unsigned BuiltinID, const CallExpr *E,4732 ReturnValueSlot ReturnValue);4733 4734 llvm::Value *4735 EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty,4736 const llvm::CmpInst::Predicate Pred,4737 const llvm::Twine &Name = "");4738 llvm::Value *EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E,4739 ReturnValueSlot ReturnValue,4740 llvm::Triple::ArchType Arch);4741 llvm::Value *EmitARMMVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E,4742 ReturnValueSlot ReturnValue,4743 llvm::Triple::ArchType Arch);4744 llvm::Value *EmitARMCDEBuiltinExpr(unsigned BuiltinID, const CallExpr *E,4745 ReturnValueSlot ReturnValue,4746 llvm::Triple::ArchType Arch);4747 llvm::Value *EmitCMSEClearRecord(llvm::Value *V, llvm::IntegerType *ITy,4748 QualType RTy);4749 llvm::Value *EmitCMSEClearRecord(llvm::Value *V, llvm::ArrayType *ATy,4750 QualType RTy);4751 4752 llvm::Value *4753 EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic,4754 unsigned AltLLVMIntrinsic, const char *NameHint,4755 unsigned Modifier, const CallExpr *E,4756 SmallVectorImpl<llvm::Value *> &Ops, Address PtrOp0,4757 Address PtrOp1, llvm::Triple::ArchType Arch);4758 4759 llvm::Function *LookupNeonLLVMIntrinsic(unsigned IntrinsicID,4760 unsigned Modifier, llvm::Type *ArgTy,4761 const CallExpr *E);4762 llvm::Value *EmitNeonCall(llvm::Function *F,4763 SmallVectorImpl<llvm::Value *> &O, const char *name,4764 unsigned shift = 0, bool rightshift = false);4765 llvm::Value *EmitFP8NeonCall(unsigned IID, ArrayRef<llvm::Type *> Tys,4766 SmallVectorImpl<llvm::Value *> &O,4767 const CallExpr *E, const char *name);4768 llvm::Value *EmitFP8NeonCvtCall(unsigned IID, llvm::Type *Ty0,4769 llvm::Type *Ty1, bool Extract,4770 SmallVectorImpl<llvm::Value *> &Ops,4771 const CallExpr *E, const char *name);4772 llvm::Value *EmitFP8NeonFDOTCall(unsigned IID, bool ExtendLaneArg,4773 llvm::Type *RetTy,4774 SmallVectorImpl<llvm::Value *> &Ops,4775 const CallExpr *E, const char *name);4776 llvm::Value *EmitFP8NeonFMLACall(unsigned IID, bool ExtendLaneArg,4777 llvm::Type *RetTy,4778 SmallVectorImpl<llvm::Value *> &Ops,4779 const CallExpr *E, const char *name);4780 llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx,4781 const llvm::ElementCount &Count);4782 llvm::Value *EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx);4783 llvm::Value *EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty,4784 bool negateForRightShift);4785 llvm::Value *EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt,4786 llvm::Type *Ty, bool usgn, const char *name);4787 llvm::Value *vectorWrapScalar16(llvm::Value *Op);4788 /// SVEBuiltinMemEltTy - Returns the memory element type for this memory4789 /// access builtin. Only required if it can't be inferred from the base4790 /// pointer operand.4791 llvm::Type *SVEBuiltinMemEltTy(const SVETypeFlags &TypeFlags);4792 4793 SmallVector<llvm::Type *, 2>4794 getSVEOverloadTypes(const SVETypeFlags &TypeFlags, llvm::Type *ReturnType,4795 ArrayRef<llvm::Value *> Ops);4796 llvm::Type *getEltType(const SVETypeFlags &TypeFlags);4797 llvm::ScalableVectorType *getSVEType(const SVETypeFlags &TypeFlags);4798 llvm::ScalableVectorType *getSVEPredType(const SVETypeFlags &TypeFlags);4799 llvm::Value *EmitSVETupleSetOrGet(const SVETypeFlags &TypeFlags,4800 ArrayRef<llvm::Value *> Ops);4801 llvm::Value *EmitSVETupleCreate(const SVETypeFlags &TypeFlags,4802 llvm::Type *ReturnType,4803 ArrayRef<llvm::Value *> Ops);4804 llvm::Value *EmitSVEAllTruePred(const SVETypeFlags &TypeFlags);4805 llvm::Value *EmitSVEDupX(llvm::Value *Scalar);4806 llvm::Value *EmitSVEDupX(llvm::Value *Scalar, llvm::Type *Ty);4807 llvm::Value *EmitSVEReinterpret(llvm::Value *Val, llvm::Type *Ty);4808 llvm::Value *EmitSVEPMull(const SVETypeFlags &TypeFlags,4809 llvm::SmallVectorImpl<llvm::Value *> &Ops,4810 unsigned BuiltinID);4811 llvm::Value *EmitSVEMovl(const SVETypeFlags &TypeFlags,4812 llvm::ArrayRef<llvm::Value *> Ops,4813 unsigned BuiltinID);4814 llvm::Value *EmitSVEPredicateCast(llvm::Value *Pred,4815 llvm::ScalableVectorType *VTy);4816 llvm::Value *EmitSVEPredicateTupleCast(llvm::Value *PredTuple,4817 llvm::StructType *Ty);4818 llvm::Value *EmitSVEGatherLoad(const SVETypeFlags &TypeFlags,4819 llvm::SmallVectorImpl<llvm::Value *> &Ops,4820 unsigned IntID);4821 llvm::Value *EmitSVEScatterStore(const SVETypeFlags &TypeFlags,4822 llvm::SmallVectorImpl<llvm::Value *> &Ops,4823 unsigned IntID);4824 llvm::Value *EmitSVEMaskedLoad(const CallExpr *, llvm::Type *ReturnTy,4825 SmallVectorImpl<llvm::Value *> &Ops,4826 unsigned BuiltinID, bool IsZExtReturn);4827 llvm::Value *EmitSVEMaskedStore(const CallExpr *,4828 SmallVectorImpl<llvm::Value *> &Ops,4829 unsigned BuiltinID);4830 llvm::Value *EmitSVEPrefetchLoad(const SVETypeFlags &TypeFlags,4831 SmallVectorImpl<llvm::Value *> &Ops,4832 unsigned BuiltinID);4833 llvm::Value *EmitSVEGatherPrefetch(const SVETypeFlags &TypeFlags,4834 SmallVectorImpl<llvm::Value *> &Ops,4835 unsigned IntID);4836 llvm::Value *EmitSVEStructLoad(const SVETypeFlags &TypeFlags,4837 SmallVectorImpl<llvm::Value *> &Ops,4838 unsigned IntID);4839 llvm::Value *EmitSVEStructStore(const SVETypeFlags &TypeFlags,4840 SmallVectorImpl<llvm::Value *> &Ops,4841 unsigned IntID);4842 llvm::Value *EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);4843 4844 llvm::Value *EmitSMELd1St1(const SVETypeFlags &TypeFlags,4845 llvm::SmallVectorImpl<llvm::Value *> &Ops,4846 unsigned IntID);4847 llvm::Value *EmitSMEReadWrite(const SVETypeFlags &TypeFlags,4848 llvm::SmallVectorImpl<llvm::Value *> &Ops,4849 unsigned IntID);4850 llvm::Value *EmitSMEZero(const SVETypeFlags &TypeFlags,4851 llvm::SmallVectorImpl<llvm::Value *> &Ops,4852 unsigned IntID);4853 llvm::Value *EmitSMELdrStr(const SVETypeFlags &TypeFlags,4854 llvm::SmallVectorImpl<llvm::Value *> &Ops,4855 unsigned IntID);4856 4857 void GetAArch64SVEProcessedOperands(unsigned BuiltinID, const CallExpr *E,4858 SmallVectorImpl<llvm::Value *> &Ops,4859 SVETypeFlags TypeFlags);4860 4861 llvm::Value *EmitAArch64SMEBuiltinExpr(unsigned BuiltinID, const CallExpr *E);4862 4863 llvm::Value *EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E,4864 llvm::Triple::ArchType Arch);4865 llvm::Value *EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E);4866 4867 llvm::Value *BuildVector(ArrayRef<llvm::Value *> Ops);4868 llvm::Value *EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E);4869 llvm::Value *EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E);4870 llvm::Value *EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E);4871 llvm::Value *EmitHLSLBuiltinExpr(unsigned BuiltinID, const CallExpr *E,4872 ReturnValueSlot ReturnValue);4873 4874 // Returns a builtin function that the SPIR-V backend will expand into a spec4875 // constant.4876 llvm::Function *4877 getSpecConstantFunction(const clang::QualType &SpecConstantType);4878 4879 llvm::Value *EmitDirectXBuiltinExpr(unsigned BuiltinID, const CallExpr *E);4880 llvm::Value *EmitSPIRVBuiltinExpr(unsigned BuiltinID, const CallExpr *E);4881 llvm::Value *EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx,4882 const CallExpr *E);4883 llvm::Value *EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E);4884 llvm::Value *EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E);4885 llvm::Value *EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,4886 const CallExpr *E);4887 llvm::Value *EmitHexagonBuiltinExpr(unsigned BuiltinID, const CallExpr *E);4888 llvm::Value *EmitRISCVBuiltinExpr(unsigned BuiltinID, const CallExpr *E,4889 ReturnValueSlot ReturnValue);4890 4891 llvm::Value *EmitRISCVCpuSupports(const CallExpr *E);4892 llvm::Value *EmitRISCVCpuSupports(ArrayRef<StringRef> FeaturesStrs);4893 llvm::Value *EmitRISCVCpuInit();4894 llvm::Value *EmitRISCVCpuIs(const CallExpr *E);4895 llvm::Value *EmitRISCVCpuIs(StringRef CPUStr);4896 4897 void AddAMDGPUFenceAddressSpaceMMRA(llvm::Instruction *Inst,4898 const CallExpr *E);4899 void ProcessOrderScopeAMDGCN(llvm::Value *Order, llvm::Value *Scope,4900 llvm::AtomicOrdering &AO,4901 llvm::SyncScope::ID &SSID);4902 4903 enum class MSVCIntrin;4904 llvm::Value *EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E);4905 4906 llvm::Value *EmitBuiltinAvailable(const VersionTuple &Version);4907 4908 llvm::Value *EmitObjCProtocolExpr(const ObjCProtocolExpr *E);4909 llvm::Value *EmitObjCStringLiteral(const ObjCStringLiteral *E);4910 llvm::Value *EmitObjCBoxedExpr(const ObjCBoxedExpr *E);4911 llvm::Value *EmitObjCArrayLiteral(const ObjCArrayLiteral *E);4912 llvm::Value *EmitObjCDictionaryLiteral(const ObjCDictionaryLiteral *E);4913 llvm::Value *4914 EmitObjCCollectionLiteral(const Expr *E,4915 const ObjCMethodDecl *MethodWithObjects);4916 llvm::Value *EmitObjCSelectorExpr(const ObjCSelectorExpr *E);4917 RValue EmitObjCMessageExpr(const ObjCMessageExpr *E,4918 ReturnValueSlot Return = ReturnValueSlot());4919 4920 /// Retrieves the default cleanup kind for an ARC cleanup.4921 /// Except under -fobjc-arc-eh, ARC cleanups are normal-only.4922 CleanupKind getARCCleanupKind() {4923 return CGM.getCodeGenOpts().ObjCAutoRefCountExceptions ? NormalAndEHCleanup4924 : NormalCleanup;4925 }4926 4927 // ARC primitives.4928 void EmitARCInitWeak(Address addr, llvm::Value *value);4929 void EmitARCDestroyWeak(Address addr);4930 llvm::Value *EmitARCLoadWeak(Address addr);4931 llvm::Value *EmitARCLoadWeakRetained(Address addr);4932 llvm::Value *EmitARCStoreWeak(Address addr, llvm::Value *value, bool ignored);4933 void emitARCCopyAssignWeak(QualType Ty, Address DstAddr, Address SrcAddr);4934 void emitARCMoveAssignWeak(QualType Ty, Address DstAddr, Address SrcAddr);4935 void EmitARCCopyWeak(Address dst, Address src);4936 void EmitARCMoveWeak(Address dst, Address src);4937 llvm::Value *EmitARCRetainAutorelease(QualType type, llvm::Value *value);4938 llvm::Value *EmitARCRetainAutoreleaseNonBlock(llvm::Value *value);4939 llvm::Value *EmitARCStoreStrong(LValue lvalue, llvm::Value *value,4940 bool resultIgnored);4941 llvm::Value *EmitARCStoreStrongCall(Address addr, llvm::Value *value,4942 bool resultIgnored);4943 llvm::Value *EmitARCRetain(QualType type, llvm::Value *value);4944 llvm::Value *EmitARCRetainNonBlock(llvm::Value *value);4945 llvm::Value *EmitARCRetainBlock(llvm::Value *value, bool mandatory);4946 void EmitARCDestroyStrong(Address addr, ARCPreciseLifetime_t precise);4947 void EmitARCRelease(llvm::Value *value, ARCPreciseLifetime_t precise);4948 llvm::Value *EmitARCAutorelease(llvm::Value *value);4949 llvm::Value *EmitARCAutoreleaseReturnValue(llvm::Value *value);4950 llvm::Value *EmitARCRetainAutoreleaseReturnValue(llvm::Value *value);4951 llvm::Value *EmitARCRetainAutoreleasedReturnValue(llvm::Value *value);4952 llvm::Value *EmitARCUnsafeClaimAutoreleasedReturnValue(llvm::Value *value);4953 4954 llvm::Value *EmitObjCAutorelease(llvm::Value *value, llvm::Type *returnType);4955 llvm::Value *EmitObjCRetainNonBlock(llvm::Value *value,4956 llvm::Type *returnType);4957 void EmitObjCRelease(llvm::Value *value, ARCPreciseLifetime_t precise);4958 4959 std::pair<LValue, llvm::Value *>4960 EmitARCStoreAutoreleasing(const BinaryOperator *e);4961 std::pair<LValue, llvm::Value *> EmitARCStoreStrong(const BinaryOperator *e,4962 bool ignored);4963 std::pair<LValue, llvm::Value *>4964 EmitARCStoreUnsafeUnretained(const BinaryOperator *e, bool ignored);4965 4966 llvm::Value *EmitObjCAlloc(llvm::Value *value, llvm::Type *returnType);4967 llvm::Value *EmitObjCAllocWithZone(llvm::Value *value,4968 llvm::Type *returnType);4969 llvm::Value *EmitObjCAllocInit(llvm::Value *value, llvm::Type *resultType);4970 4971 llvm::Value *EmitObjCThrowOperand(const Expr *expr);4972 llvm::Value *EmitObjCConsumeObject(QualType T, llvm::Value *Ptr);4973 llvm::Value *EmitObjCExtendObjectLifetime(QualType T, llvm::Value *Ptr);4974 4975 llvm::Value *EmitARCExtendBlockObject(const Expr *expr);4976 llvm::Value *EmitARCReclaimReturnedObject(const Expr *e,4977 bool allowUnsafeClaim);4978 llvm::Value *EmitARCRetainScalarExpr(const Expr *expr);4979 llvm::Value *EmitARCRetainAutoreleaseScalarExpr(const Expr *expr);4980 llvm::Value *EmitARCUnsafeUnretainedScalarExpr(const Expr *expr);4981 4982 void EmitARCIntrinsicUse(ArrayRef<llvm::Value *> values);4983 4984 void EmitARCNoopIntrinsicUse(ArrayRef<llvm::Value *> values);4985 4986 static Destroyer destroyARCStrongImprecise;4987 static Destroyer destroyARCStrongPrecise;4988 static Destroyer destroyARCWeak;4989 static Destroyer emitARCIntrinsicUse;4990 static Destroyer destroyNonTrivialCStruct;4991 4992 void EmitObjCAutoreleasePoolPop(llvm::Value *Ptr);4993 llvm::Value *EmitObjCAutoreleasePoolPush();4994 llvm::Value *EmitObjCMRRAutoreleasePoolPush();4995 void EmitObjCAutoreleasePoolCleanup(llvm::Value *Ptr);4996 void EmitObjCMRRAutoreleasePoolPop(llvm::Value *Ptr);4997 4998 /// Emits a reference binding to the passed in expression.4999 RValue EmitReferenceBindingToExpr(const Expr *E);5000 5001 //===--------------------------------------------------------------------===//5002 // Expression Emission5003 //===--------------------------------------------------------------------===//5004 5005 // Expressions are broken into three classes: scalar, complex, aggregate.5006 5007 /// EmitScalarExpr - Emit the computation of the specified expression of LLVM5008 /// scalar type, returning the result.5009 llvm::Value *EmitScalarExpr(const Expr *E, bool IgnoreResultAssign = false);5010 5011 /// Emit a conversion from the specified type to the specified destination5012 /// type, both of which are LLVM scalar types.5013 llvm::Value *EmitScalarConversion(llvm::Value *Src, QualType SrcTy,5014 QualType DstTy, SourceLocation Loc);5015 5016 /// Emit a conversion from the specified complex type to the specified5017 /// destination type, where the destination type is an LLVM scalar type.5018 llvm::Value *EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy,5019 QualType DstTy,5020 SourceLocation Loc);5021 5022 /// EmitAggExpr - Emit the computation of the specified expression5023 /// of aggregate type. The result is computed into the given slot,5024 /// which may be null to indicate that the value is not needed.5025 void EmitAggExpr(const Expr *E, AggValueSlot AS);5026 5027 /// EmitAggExprToLValue - Emit the computation of the specified expression of5028 /// aggregate type into a temporary LValue.5029 LValue EmitAggExprToLValue(const Expr *E);5030 5031 enum ExprValueKind { EVK_RValue, EVK_NonRValue };5032 5033 /// EmitAggFinalDestCopy - Emit copy of the specified aggregate into5034 /// destination address.5035 void EmitAggFinalDestCopy(QualType Type, AggValueSlot Dest, const LValue &Src,5036 ExprValueKind SrcKind);5037 5038 /// Create a store to \arg DstPtr from \arg Src, truncating the stored value5039 /// to at most \arg DstSize bytes.5040 void CreateCoercedStore(llvm::Value *Src, Address Dst, llvm::TypeSize DstSize,5041 bool DstIsVolatile);5042 5043 /// EmitExtendGCLifetime - Given a pointer to an Objective-C object,5044 /// make sure it survives garbage collection until this point.5045 void EmitExtendGCLifetime(llvm::Value *object);5046 5047 /// EmitComplexExpr - Emit the computation of the specified expression of5048 /// complex type, returning the result.5049 ComplexPairTy EmitComplexExpr(const Expr *E, bool IgnoreReal = false,5050 bool IgnoreImag = false);5051 5052 /// EmitComplexExprIntoLValue - Emit the given expression of complex5053 /// type and place its result into the specified l-value.5054 void EmitComplexExprIntoLValue(const Expr *E, LValue dest, bool isInit);5055 5056 /// EmitStoreOfComplex - Store a complex number into the specified l-value.5057 void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit);5058 5059 /// EmitLoadOfComplex - Load a complex number from the specified l-value.5060 ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc);5061 5062 ComplexPairTy EmitPromotedComplexExpr(const Expr *E, QualType PromotionType);5063 llvm::Value *EmitPromotedScalarExpr(const Expr *E, QualType PromotionType);5064 ComplexPairTy EmitPromotedValue(ComplexPairTy result, QualType PromotionType);5065 ComplexPairTy EmitUnPromotedValue(ComplexPairTy result,5066 QualType PromotionType);5067 5068 Address emitAddrOfRealComponent(Address complex, QualType complexType);5069 Address emitAddrOfImagComponent(Address complex, QualType complexType);5070 5071 /// AddInitializerToStaticVarDecl - Add the initializer for 'D' to the5072 /// global variable that has already been created for it. If the initializer5073 /// has a different type than GV does, this may free GV and return a different5074 /// one. Otherwise it just returns GV.5075 llvm::GlobalVariable *AddInitializerToStaticVarDecl(const VarDecl &D,5076 llvm::GlobalVariable *GV);5077 5078 // Emit an @llvm.invariant.start call for the given memory region.5079 void EmitInvariantStart(llvm::Constant *Addr, CharUnits Size);5080 5081 /// EmitCXXGlobalVarDeclInit - Create the initializer for a C++5082 /// variable with global storage.5083 void EmitCXXGlobalVarDeclInit(const VarDecl &D, llvm::GlobalVariable *GV,5084 bool PerformInit);5085 5086 llvm::Constant *createAtExitStub(const VarDecl &VD, llvm::FunctionCallee Dtor,5087 llvm::Constant *Addr);5088 5089 llvm::Function *createTLSAtExitStub(const VarDecl &VD,5090 llvm::FunctionCallee Dtor,5091 llvm::Constant *Addr,5092 llvm::FunctionCallee &AtExit);5093 5094 /// Call atexit() with a function that passes the given argument to5095 /// the given function.5096 void registerGlobalDtorWithAtExit(const VarDecl &D, llvm::FunctionCallee fn,5097 llvm::Constant *addr);5098 5099 /// Registers the dtor using 'llvm.global_dtors' for platforms that do not5100 /// support an 'atexit()' function.5101 void registerGlobalDtorWithLLVM(const VarDecl &D, llvm::FunctionCallee fn,5102 llvm::Constant *addr);5103 5104 /// Call atexit() with function dtorStub.5105 void registerGlobalDtorWithAtExit(llvm::Constant *dtorStub);5106 5107 /// Call unatexit() with function dtorStub.5108 llvm::Value *unregisterGlobalDtorWithUnAtExit(llvm::Constant *dtorStub);5109 5110 /// Emit code in this function to perform a guarded variable5111 /// initialization. Guarded initializations are used when it's not5112 /// possible to prove that an initialization will be done exactly5113 /// once, e.g. with a static local variable or a static data member5114 /// of a class template.5115 void EmitCXXGuardedInit(const VarDecl &D, llvm::GlobalVariable *DeclPtr,5116 bool PerformInit);5117 5118 enum class GuardKind { VariableGuard, TlsGuard };5119 5120 /// Emit a branch to select whether or not to perform guarded initialization.5121 void EmitCXXGuardedInitBranch(llvm::Value *NeedsInit,5122 llvm::BasicBlock *InitBlock,5123 llvm::BasicBlock *NoInitBlock, GuardKind Kind,5124 const VarDecl *D);5125 5126 /// GenerateCXXGlobalInitFunc - Generates code for initializing global5127 /// variables.5128 void5129 GenerateCXXGlobalInitFunc(llvm::Function *Fn,5130 ArrayRef<llvm::Function *> CXXThreadLocals,5131 ConstantAddress Guard = ConstantAddress::invalid());5132 5133 /// GenerateCXXGlobalCleanUpFunc - Generates code for cleaning up global5134 /// variables.5135 void GenerateCXXGlobalCleanUpFunc(5136 llvm::Function *Fn,5137 ArrayRef<std::tuple<llvm::FunctionType *, llvm::WeakTrackingVH,5138 llvm::Constant *>>5139 DtorsOrStermFinalizers);5140 5141 void GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn, const VarDecl *D,5142 llvm::GlobalVariable *Addr,5143 bool PerformInit);5144 5145 void EmitCXXConstructExpr(const CXXConstructExpr *E, AggValueSlot Dest);5146 5147 void EmitSynthesizedCXXCopyCtor(Address Dest, Address Src, const Expr *Exp);5148 5149 void EmitCXXThrowExpr(const CXXThrowExpr *E, bool KeepInsertionPoint = true);5150 5151 RValue EmitAtomicExpr(AtomicExpr *E);5152 5153 void EmitFakeUse(Address Addr);5154 5155 //===--------------------------------------------------------------------===//5156 // Annotations Emission5157 //===--------------------------------------------------------------------===//5158 5159 /// Emit an annotation call (intrinsic).5160 llvm::Value *EmitAnnotationCall(llvm::Function *AnnotationFn,5161 llvm::Value *AnnotatedVal,5162 StringRef AnnotationStr,5163 SourceLocation Location,5164 const AnnotateAttr *Attr);5165 5166 /// Emit local annotations for the local variable V, declared by D.5167 void EmitVarAnnotations(const VarDecl *D, llvm::Value *V);5168 5169 /// Emit field annotations for the given field & value. Returns the5170 /// annotation result.5171 Address EmitFieldAnnotations(const FieldDecl *D, Address V);5172 5173 //===--------------------------------------------------------------------===//5174 // Internal Helpers5175 //===--------------------------------------------------------------------===//5176 5177 /// ContainsLabel - Return true if the statement contains a label in it. If5178 /// this statement is not executed normally, it not containing a label means5179 /// that we can just remove the code.5180 static bool ContainsLabel(const Stmt *S, bool IgnoreCaseStmts = false);5181 5182 /// containsBreak - Return true if the statement contains a break out of it.5183 /// If the statement (recursively) contains a switch or loop with a break5184 /// inside of it, this is fine.5185 static bool containsBreak(const Stmt *S);5186 5187 /// Determine if the given statement might introduce a declaration into the5188 /// current scope, by being a (possibly-labelled) DeclStmt.5189 static bool mightAddDeclToScope(const Stmt *S);5190 5191 /// ConstantFoldsToSimpleInteger - If the specified expression does not fold5192 /// to a constant, or if it does but contains a label, return false. If it5193 /// constant folds return true and set the boolean result in Result.5194 bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result,5195 bool AllowLabels = false);5196 5197 /// ConstantFoldsToSimpleInteger - If the specified expression does not fold5198 /// to a constant, or if it does but contains a label, return false. If it5199 /// constant folds return true and set the folded value.5200 bool ConstantFoldsToSimpleInteger(const Expr *Cond, llvm::APSInt &Result,5201 bool AllowLabels = false);5202 5203 /// Ignore parentheses and logical-NOT to track conditions consistently.5204 static const Expr *stripCond(const Expr *C);5205 5206 /// isInstrumentedCondition - Determine whether the given condition is an5207 /// instrumentable condition (i.e. no "&&" or "||").5208 static bool isInstrumentedCondition(const Expr *C);5209 5210 /// EmitBranchToCounterBlock - Emit a conditional branch to a new block that5211 /// increments a profile counter based on the semantics of the given logical5212 /// operator opcode. This is used to instrument branch condition coverage5213 /// for logical operators.5214 void EmitBranchToCounterBlock(const Expr *Cond, BinaryOperator::Opcode LOp,5215 llvm::BasicBlock *TrueBlock,5216 llvm::BasicBlock *FalseBlock,5217 uint64_t TrueCount = 0,5218 Stmt::Likelihood LH = Stmt::LH_None,5219 const Expr *CntrIdx = nullptr);5220 5221 /// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an5222 /// if statement) to the specified blocks. Based on the condition, this might5223 /// try to simplify the codegen of the conditional based on the branch.5224 /// TrueCount should be the number of times we expect the condition to5225 /// evaluate to true based on PGO data.5226 void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock,5227 llvm::BasicBlock *FalseBlock, uint64_t TrueCount,5228 Stmt::Likelihood LH = Stmt::LH_None,5229 const Expr *ConditionalOp = nullptr,5230 const VarDecl *ConditionalDecl = nullptr);5231 5232 /// Given an assignment `*LHS = RHS`, emit a test that checks if \p RHS is5233 /// nonnull, if \p LHS is marked _Nonnull.5234 void EmitNullabilityCheck(LValue LHS, llvm::Value *RHS, SourceLocation Loc);5235 5236 /// An enumeration which makes it easier to specify whether or not an5237 /// operation is a subtraction.5238 enum { NotSubtraction = false, IsSubtraction = true };5239 5240 /// Emit pointer + index arithmetic.5241 llvm::Value *EmitPointerArithmetic(const BinaryOperator *BO,5242 Expr *pointerOperand, llvm::Value *pointer,5243 Expr *indexOperand, llvm::Value *index,5244 bool isSubtraction);5245 5246 /// Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to5247 /// detect undefined behavior when the pointer overflow sanitizer is enabled.5248 /// \p SignedIndices indicates whether any of the GEP indices are signed.5249 /// \p IsSubtraction indicates whether the expression used to form the GEP5250 /// is a subtraction.5251 llvm::Value *EmitCheckedInBoundsGEP(llvm::Type *ElemTy, llvm::Value *Ptr,5252 ArrayRef<llvm::Value *> IdxList,5253 bool SignedIndices, bool IsSubtraction,5254 SourceLocation Loc,5255 const Twine &Name = "");5256 5257 Address EmitCheckedInBoundsGEP(Address Addr, ArrayRef<llvm::Value *> IdxList,5258 llvm::Type *elementType, bool SignedIndices,5259 bool IsSubtraction, SourceLocation Loc,5260 CharUnits Align, const Twine &Name = "");5261 5262 /// Specifies which type of sanitizer check to apply when handling a5263 /// particular builtin.5264 enum BuiltinCheckKind {5265 BCK_CTZPassedZero,5266 BCK_CLZPassedZero,5267 BCK_AssumePassedFalse,5268 };5269 5270 /// Emits an argument for a call to a builtin. If the builtin sanitizer is5271 /// enabled, a runtime check specified by \p Kind is also emitted.5272 llvm::Value *EmitCheckedArgForBuiltin(const Expr *E, BuiltinCheckKind Kind);5273 5274 /// Emits an argument for a call to a `__builtin_assume`. If the builtin5275 /// sanitizer is enabled, a runtime check is also emitted.5276 llvm::Value *EmitCheckedArgForAssume(const Expr *E);5277 5278 /// Emit a description of a type in a format suitable for passing to5279 /// a runtime sanitizer handler.5280 llvm::Constant *EmitCheckTypeDescriptor(QualType T);5281 5282 /// Convert a value into a format suitable for passing to a runtime5283 /// sanitizer handler.5284 llvm::Value *EmitCheckValue(llvm::Value *V);5285 5286 /// Emit a description of a source location in a format suitable for5287 /// passing to a runtime sanitizer handler.5288 llvm::Constant *EmitCheckSourceLocation(SourceLocation Loc);5289 5290 void EmitKCFIOperandBundle(const CGCallee &Callee,5291 SmallVectorImpl<llvm::OperandBundleDef> &Bundles);5292 5293 /// Create a basic block that will either trap or call a handler function in5294 /// the UBSan runtime with the provided arguments, and create a conditional5295 /// branch to it.5296 void5297 EmitCheck(ArrayRef<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>>5298 Checked,5299 SanitizerHandler Check, ArrayRef<llvm::Constant *> StaticArgs,5300 ArrayRef<llvm::Value *> DynamicArgs,5301 const TrapReason *TR = nullptr);5302 5303 /// Emit a slow path cross-DSO CFI check which calls __cfi_slowpath5304 /// if Cond if false.5305 void EmitCfiSlowPathCheck(SanitizerKind::SanitizerOrdinal Ordinal,5306 llvm::Value *Cond, llvm::ConstantInt *TypeId,5307 llvm::Value *Ptr,5308 ArrayRef<llvm::Constant *> StaticArgs);5309 5310 /// Emit a reached-unreachable diagnostic if \p Loc is valid and runtime5311 /// checking is enabled. Otherwise, just emit an unreachable instruction.5312 void EmitUnreachable(SourceLocation Loc);5313 5314 /// Create a basic block that will call the trap intrinsic, and emit a5315 /// conditional branch to it, for the -ftrapv checks.5316 void EmitTrapCheck(llvm::Value *Checked, SanitizerHandler CheckHandlerID,5317 bool NoMerge = false, const TrapReason *TR = nullptr);5318 5319 /// Emit a call to trap or debugtrap and attach function attribute5320 /// "trap-func-name" if specified.5321 llvm::CallInst *EmitTrapCall(llvm::Intrinsic::ID IntrID);5322 5323 /// Emit a stub for the cross-DSO CFI check function.5324 void EmitCfiCheckStub();5325 5326 /// Emit a cross-DSO CFI failure handling function.5327 void EmitCfiCheckFail();5328 5329 /// Create a check for a function parameter that may potentially be5330 /// declared as non-null.5331 void EmitNonNullArgCheck(RValue RV, QualType ArgType, SourceLocation ArgLoc,5332 AbstractCallee AC, unsigned ParmNum);5333 5334 void EmitNonNullArgCheck(Address Addr, QualType ArgType,5335 SourceLocation ArgLoc, AbstractCallee AC,5336 unsigned ParmNum);5337 5338 /// EmitWriteback - Emit callbacks for function.5339 void EmitWritebacks(const CallArgList &Args);5340 5341 /// EmitCallArg - Emit a single call argument.5342 void EmitCallArg(CallArgList &args, const Expr *E, QualType ArgType);5343 5344 /// EmitDelegateCallArg - We are performing a delegate call; that5345 /// is, the current function is delegating to another one. Produce5346 /// a r-value suitable for passing the given parameter.5347 void EmitDelegateCallArg(CallArgList &args, const VarDecl *param,5348 SourceLocation loc);5349 5350 /// SetFPAccuracy - Set the minimum required accuracy of the given floating5351 /// point operation, expressed as the maximum relative error in ulp.5352 void SetFPAccuracy(llvm::Value *Val, float Accuracy);5353 5354 /// Set the minimum required accuracy of the given sqrt operation5355 /// based on CodeGenOpts.5356 void SetSqrtFPAccuracy(llvm::Value *Val);5357 5358 /// Set the minimum required accuracy of the given sqrt operation based on5359 /// CodeGenOpts.5360 void SetDivFPAccuracy(llvm::Value *Val);5361 5362 /// Set the codegen fast-math flags.5363 void SetFastMathFlags(FPOptions FPFeatures);5364 5365 // Truncate or extend a boolean vector to the requested number of elements.5366 llvm::Value *emitBoolVecConversion(llvm::Value *SrcVec,5367 unsigned NumElementsDst,5368 const llvm::Twine &Name = "");5369 5370 void maybeAttachRangeForLoad(llvm::LoadInst *Load, QualType Ty,5371 SourceLocation Loc);5372 5373private:5374 // Emits a convergence_loop instruction for the given |BB|, with |ParentToken|5375 // as it's parent convergence instr.5376 llvm::ConvergenceControlInst *emitConvergenceLoopToken(llvm::BasicBlock *BB);5377 5378 // Adds a convergence_ctrl token with |ParentToken| as parent convergence5379 // instr to the call |Input|.5380 llvm::CallBase *addConvergenceControlToken(llvm::CallBase *Input);5381 5382 // Find the convergence_entry instruction |F|, or emits ones if none exists.5383 // Returns the convergence instruction.5384 llvm::ConvergenceControlInst *5385 getOrEmitConvergenceEntryToken(llvm::Function *F);5386 5387private:5388 llvm::MDNode *getRangeForLoadFromType(QualType Ty);5389 void EmitReturnOfRValue(RValue RV, QualType Ty);5390 5391 void deferPlaceholderReplacement(llvm::Instruction *Old, llvm::Value *New);5392 5393 llvm::SmallVector<std::pair<llvm::WeakTrackingVH, llvm::Value *>, 4>5394 DeferredReplacements;5395 5396 /// Set the address of a local variable.5397 void setAddrOfLocalVar(const VarDecl *VD, Address Addr) {5398 assert(!LocalDeclMap.count(VD) && "Decl already exists in LocalDeclMap!");5399 LocalDeclMap.insert({VD, Addr});5400 }5401 5402 /// ExpandTypeFromArgs - Reconstruct a structure of type \arg Ty5403 /// from function arguments into \arg Dst. See ABIArgInfo::Expand.5404 ///5405 /// \param AI - The first function argument of the expansion.5406 void ExpandTypeFromArgs(QualType Ty, LValue Dst,5407 llvm::Function::arg_iterator &AI);5408 5409 /// ExpandTypeToArgs - Expand an CallArg \arg Arg, with the LLVM type for \arg5410 /// Ty, into individual arguments on the provided vector \arg IRCallArgs,5411 /// starting at index \arg IRCallArgPos. See ABIArgInfo::Expand.5412 void ExpandTypeToArgs(QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,5413 SmallVectorImpl<llvm::Value *> &IRCallArgs,5414 unsigned &IRCallArgPos);5415 5416 std::pair<llvm::Value *, llvm::Type *>5417 EmitAsmInput(const TargetInfo::ConstraintInfo &Info, const Expr *InputExpr,5418 std::string &ConstraintStr);5419 5420 std::pair<llvm::Value *, llvm::Type *>5421 EmitAsmInputLValue(const TargetInfo::ConstraintInfo &Info, LValue InputValue,5422 QualType InputType, std::string &ConstraintStr,5423 SourceLocation Loc);5424 5425 /// Attempts to statically evaluate the object size of E. If that5426 /// fails, emits code to figure the size of E out for us. This is5427 /// pass_object_size aware.5428 ///5429 /// If EmittedExpr is non-null, this will use that instead of re-emitting E.5430 llvm::Value *evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,5431 llvm::IntegerType *ResType,5432 llvm::Value *EmittedE,5433 bool IsDynamic);5434 5435 /// Emits the size of E, as required by __builtin_object_size. This5436 /// function is aware of pass_object_size parameters, and will act accordingly5437 /// if E is a parameter with the pass_object_size attribute.5438 llvm::Value *emitBuiltinObjectSize(const Expr *E, unsigned Type,5439 llvm::IntegerType *ResType,5440 llvm::Value *EmittedE, bool IsDynamic);5441 5442 llvm::Value *emitCountedBySize(const Expr *E, llvm::Value *EmittedE,5443 unsigned Type, llvm::IntegerType *ResType);5444 5445 llvm::Value *emitCountedByMemberSize(const MemberExpr *E, const Expr *Idx,5446 llvm::Value *EmittedE,5447 QualType CastedArrayElementTy,5448 unsigned Type,5449 llvm::IntegerType *ResType);5450 5451 llvm::Value *emitCountedByPointerSize(const ImplicitCastExpr *E,5452 const Expr *Idx, llvm::Value *EmittedE,5453 QualType CastedArrayElementTy,5454 unsigned Type,5455 llvm::IntegerType *ResType);5456 5457 void emitZeroOrPatternForAutoVarInit(QualType type, const VarDecl &D,5458 Address Loc);5459 5460public:5461 enum class EvaluationOrder {5462 ///! No language constraints on evaluation order.5463 Default,5464 ///! Language semantics require left-to-right evaluation.5465 ForceLeftToRight,5466 ///! Language semantics require right-to-left evaluation.5467 ForceRightToLeft5468 };5469 5470 // Wrapper for function prototype sources. Wraps either a FunctionProtoType or5471 // an ObjCMethodDecl.5472 struct PrototypeWrapper {5473 llvm::PointerUnion<const FunctionProtoType *, const ObjCMethodDecl *> P;5474 5475 PrototypeWrapper(const FunctionProtoType *FT) : P(FT) {}5476 PrototypeWrapper(const ObjCMethodDecl *MD) : P(MD) {}5477 };5478 5479 void EmitCallArgs(CallArgList &Args, PrototypeWrapper Prototype,5480 llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,5481 AbstractCallee AC = AbstractCallee(),5482 unsigned ParamsToSkip = 0,5483 EvaluationOrder Order = EvaluationOrder::Default);5484 5485 /// EmitPointerWithAlignment - Given an expression with a pointer type,5486 /// emit the value and compute our best estimate of the alignment of the5487 /// pointee.5488 ///5489 /// \param BaseInfo - If non-null, this will be initialized with5490 /// information about the source of the alignment and the may-alias5491 /// attribute. Note that this function will conservatively fall back on5492 /// the type when it doesn't recognize the expression and may-alias will5493 /// be set to false.5494 ///5495 /// One reasonable way to use this information is when there's a language5496 /// guarantee that the pointer must be aligned to some stricter value, and5497 /// we're simply trying to ensure that sufficiently obvious uses of under-5498 /// aligned objects don't get miscompiled; for example, a placement new5499 /// into the address of a local variable. In such a case, it's quite5500 /// reasonable to just ignore the returned alignment when it isn't from an5501 /// explicit source.5502 Address5503 EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo = nullptr,5504 TBAAAccessInfo *TBAAInfo = nullptr,5505 KnownNonNull_t IsKnownNonNull = NotKnownNonNull);5506 5507 /// If \p E references a parameter with pass_object_size info or a constant5508 /// array size modifier, emit the object size divided by the size of \p EltTy.5509 /// Otherwise return null.5510 llvm::Value *LoadPassedObjectSize(const Expr *E, QualType EltTy);5511 5512 void EmitSanitizerStatReport(llvm::SanitizerStatKind SSK);5513 5514 struct FMVResolverOption {5515 llvm::Function *Function;5516 llvm::SmallVector<StringRef, 8> Features;5517 std::optional<StringRef> Architecture;5518 5519 FMVResolverOption(llvm::Function *F, ArrayRef<StringRef> Feats,5520 std::optional<StringRef> Arch = std::nullopt)5521 : Function(F), Features(Feats), Architecture(Arch) {}5522 };5523 5524 // Emits the body of a multiversion function's resolver. Assumes that the5525 // options are already sorted in the proper order, with the 'default' option5526 // last (if it exists).5527 void EmitMultiVersionResolver(llvm::Function *Resolver,5528 ArrayRef<FMVResolverOption> Options);5529 void EmitX86MultiVersionResolver(llvm::Function *Resolver,5530 ArrayRef<FMVResolverOption> Options);5531 void EmitAArch64MultiVersionResolver(llvm::Function *Resolver,5532 ArrayRef<FMVResolverOption> Options);5533 void EmitRISCVMultiVersionResolver(llvm::Function *Resolver,5534 ArrayRef<FMVResolverOption> Options);5535 5536private:5537 QualType getVarArgType(const Expr *Arg);5538 5539 void EmitDeclMetadata();5540 5541 BlockByrefHelpers *buildByrefHelpers(llvm::StructType &byrefType,5542 const AutoVarEmission &emission);5543 5544 void AddObjCARCExceptionMetadata(llvm::Instruction *Inst);5545 5546 llvm::Value *GetValueForARMHint(unsigned BuiltinID);5547 llvm::Value *EmitX86CpuIs(const CallExpr *E);5548 llvm::Value *EmitX86CpuIs(StringRef CPUStr);5549 llvm::Value *EmitX86CpuSupports(const CallExpr *E);5550 llvm::Value *EmitX86CpuSupports(ArrayRef<StringRef> FeatureStrs);5551 llvm::Value *EmitX86CpuSupports(std::array<uint32_t, 4> FeatureMask);5552 llvm::Value *EmitX86CpuInit();5553 llvm::Value *FormX86ResolverCondition(const FMVResolverOption &RO);5554 llvm::Value *EmitAArch64CpuInit();5555 llvm::Value *FormAArch64ResolverCondition(const FMVResolverOption &RO);5556 llvm::Value *EmitAArch64CpuSupports(const CallExpr *E);5557 llvm::Value *EmitAArch64CpuSupports(ArrayRef<StringRef> FeatureStrs);5558};5559 5560inline DominatingLLVMValue::saved_type5561DominatingLLVMValue::save(CodeGenFunction &CGF, llvm::Value *value) {5562 if (!needsSaving(value))5563 return saved_type(value, false);5564 5565 // Otherwise, we need an alloca.5566 auto align = CharUnits::fromQuantity(5567 CGF.CGM.getDataLayout().getPrefTypeAlign(value->getType()));5568 Address alloca =5569 CGF.CreateTempAlloca(value->getType(), align, "cond-cleanup.save");5570 CGF.Builder.CreateStore(value, alloca);5571 5572 return saved_type(alloca.emitRawPointer(CGF), true);5573}5574 5575inline llvm::Value *DominatingLLVMValue::restore(CodeGenFunction &CGF,5576 saved_type value) {5577 // If the value says it wasn't saved, trust that it's still dominating.5578 if (!value.getInt())5579 return value.getPointer();5580 5581 // Otherwise, it should be an alloca instruction, as set up in save().5582 auto alloca = cast<llvm::AllocaInst>(value.getPointer());5583 return CGF.Builder.CreateAlignedLoad(alloca->getAllocatedType(), alloca,5584 alloca->getAlign());5585}5586 5587} // end namespace CodeGen5588 5589// Map the LangOption for floating point exception behavior into5590// the corresponding enum in the IR.5591llvm::fp::ExceptionBehavior5592ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind);5593} // end namespace clang5594 5595#endif5596