3386 lines · cpp
1//===--- CodeGenFunction.cpp - Emit LLVM Code from ASTs for a Function ----===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This coordinates the per-function state used while generating code.10//11//===----------------------------------------------------------------------===//12 13#include "CodeGenFunction.h"14#include "CGBlocks.h"15#include "CGCUDARuntime.h"16#include "CGCXXABI.h"17#include "CGCleanup.h"18#include "CGDebugInfo.h"19#include "CGHLSLRuntime.h"20#include "CGOpenMPRuntime.h"21#include "CodeGenModule.h"22#include "CodeGenPGO.h"23#include "TargetInfo.h"24#include "clang/AST/ASTContext.h"25#include "clang/AST/ASTLambda.h"26#include "clang/AST/Attr.h"27#include "clang/AST/Decl.h"28#include "clang/AST/DeclCXX.h"29#include "clang/AST/Expr.h"30#include "clang/AST/StmtCXX.h"31#include "clang/AST/StmtObjC.h"32#include "clang/Basic/Builtins.h"33#include "clang/Basic/CodeGenOptions.h"34#include "clang/Basic/DiagnosticFrontend.h"35#include "clang/Basic/TargetBuiltins.h"36#include "clang/Basic/TargetInfo.h"37#include "clang/CodeGen/CGFunctionInfo.h"38#include "llvm/ADT/ArrayRef.h"39#include "llvm/ADT/ScopeExit.h"40#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"41#include "llvm/IR/DataLayout.h"42#include "llvm/IR/Dominators.h"43#include "llvm/IR/FPEnv.h"44#include "llvm/IR/Instruction.h"45#include "llvm/IR/IntrinsicInst.h"46#include "llvm/IR/Intrinsics.h"47#include "llvm/IR/MDBuilder.h"48#include "llvm/Support/CRC.h"49#include "llvm/Support/xxhash.h"50#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"51#include "llvm/Transforms/Utils/PromoteMemToReg.h"52#include <optional>53 54using namespace clang;55using namespace CodeGen;56 57namespace llvm {58extern cl::opt<bool> EnableSingleByteCoverage;59} // namespace llvm60 61/// shouldEmitLifetimeMarkers - Decide whether we need emit the life-time62/// markers.63static bool shouldEmitLifetimeMarkers(const CodeGenOptions &CGOpts,64 const LangOptions &LangOpts) {65 if (CGOpts.DisableLifetimeMarkers)66 return false;67 68 // Sanitizers may use markers.69 if (CGOpts.SanitizeAddressUseAfterScope ||70 LangOpts.Sanitize.has(SanitizerKind::HWAddress) ||71 LangOpts.Sanitize.has(SanitizerKind::Memory))72 return true;73 74 // For now, only in optimized builds.75 return CGOpts.OptimizationLevel != 0;76}77 78CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext)79 : CodeGenTypeCache(cgm), CGM(cgm), Target(cgm.getTarget()),80 Builder(cgm, cgm.getModule().getContext(), llvm::ConstantFolder(),81 CGBuilderInserterTy(this)),82 SanOpts(CGM.getLangOpts().Sanitize), CurFPFeatures(CGM.getLangOpts()),83 DebugInfo(CGM.getModuleDebugInfo()),84 PGO(std::make_unique<CodeGenPGO>(cgm)),85 ShouldEmitLifetimeMarkers(86 shouldEmitLifetimeMarkers(CGM.getCodeGenOpts(), CGM.getLangOpts())) {87 if (!suppressNewContext)88 CGM.getCXXABI().getMangleContext().startNewFunction();89 EHStack.setCGF(this);90 91 SetFastMathFlags(CurFPFeatures);92}93 94CodeGenFunction::~CodeGenFunction() {95 assert(LifetimeExtendedCleanupStack.empty() && "failed to emit a cleanup");96 assert(DeferredDeactivationCleanupStack.empty() &&97 "missed to deactivate a cleanup");98 99 if (getLangOpts().OpenMP && CurFn)100 CGM.getOpenMPRuntime().functionFinished(*this);101 102 // If we have an OpenMPIRBuilder we want to finalize functions (incl.103 // outlining etc) at some point. Doing it once the function codegen is done104 // seems to be a reasonable spot. We do it here, as opposed to the deletion105 // time of the CodeGenModule, because we have to ensure the IR has not yet106 // been "emitted" to the outside, thus, modifications are still sensible.107 if (CGM.getLangOpts().OpenMPIRBuilder && CurFn)108 CGM.getOpenMPRuntime().getOMPBuilder().finalize(CurFn);109}110 111// Map the LangOption for exception behavior into112// the corresponding enum in the IR.113llvm::fp::ExceptionBehavior114clang::ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind) {115 116 switch (Kind) {117 case LangOptions::FPE_Ignore: return llvm::fp::ebIgnore;118 case LangOptions::FPE_MayTrap: return llvm::fp::ebMayTrap;119 case LangOptions::FPE_Strict: return llvm::fp::ebStrict;120 default:121 llvm_unreachable("Unsupported FP Exception Behavior");122 }123}124 125void CodeGenFunction::SetFastMathFlags(FPOptions FPFeatures) {126 llvm::FastMathFlags FMF;127 FMF.setAllowReassoc(FPFeatures.getAllowFPReassociate());128 FMF.setNoNaNs(FPFeatures.getNoHonorNaNs());129 FMF.setNoInfs(FPFeatures.getNoHonorInfs());130 FMF.setNoSignedZeros(FPFeatures.getNoSignedZero());131 FMF.setAllowReciprocal(FPFeatures.getAllowReciprocal());132 FMF.setApproxFunc(FPFeatures.getAllowApproxFunc());133 FMF.setAllowContract(FPFeatures.allowFPContractAcrossStatement());134 Builder.setFastMathFlags(FMF);135}136 137CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,138 const Expr *E)139 : CGF(CGF) {140 ConstructorHelper(E->getFPFeaturesInEffect(CGF.getLangOpts()));141}142 143CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,144 FPOptions FPFeatures)145 : CGF(CGF) {146 ConstructorHelper(FPFeatures);147}148 149void CodeGenFunction::CGFPOptionsRAII::ConstructorHelper(FPOptions FPFeatures) {150 OldFPFeatures = CGF.CurFPFeatures;151 CGF.CurFPFeatures = FPFeatures;152 153 OldExcept = CGF.Builder.getDefaultConstrainedExcept();154 OldRounding = CGF.Builder.getDefaultConstrainedRounding();155 156 if (OldFPFeatures == FPFeatures)157 return;158 159 FMFGuard.emplace(CGF.Builder);160 161 llvm::RoundingMode NewRoundingBehavior = FPFeatures.getRoundingMode();162 CGF.Builder.setDefaultConstrainedRounding(NewRoundingBehavior);163 auto NewExceptionBehavior =164 ToConstrainedExceptMD(FPFeatures.getExceptionMode());165 CGF.Builder.setDefaultConstrainedExcept(NewExceptionBehavior);166 167 CGF.SetFastMathFlags(FPFeatures);168 169 assert((CGF.CurFuncDecl == nullptr || CGF.Builder.getIsFPConstrained() ||170 isa<CXXConstructorDecl>(CGF.CurFuncDecl) ||171 isa<CXXDestructorDecl>(CGF.CurFuncDecl) ||172 (NewExceptionBehavior == llvm::fp::ebIgnore &&173 NewRoundingBehavior == llvm::RoundingMode::NearestTiesToEven)) &&174 "FPConstrained should be enabled on entire function");175 176 auto mergeFnAttrValue = [&](StringRef Name, bool Value) {177 auto OldValue =178 CGF.CurFn->getFnAttribute(Name).getValueAsBool();179 auto NewValue = OldValue & Value;180 if (OldValue != NewValue)181 CGF.CurFn->addFnAttr(Name, llvm::toStringRef(NewValue));182 };183 mergeFnAttrValue("no-infs-fp-math", FPFeatures.getNoHonorInfs());184 mergeFnAttrValue("no-nans-fp-math", FPFeatures.getNoHonorNaNs());185 mergeFnAttrValue("no-signed-zeros-fp-math", FPFeatures.getNoSignedZero());186}187 188CodeGenFunction::CGFPOptionsRAII::~CGFPOptionsRAII() {189 CGF.CurFPFeatures = OldFPFeatures;190 CGF.Builder.setDefaultConstrainedExcept(OldExcept);191 CGF.Builder.setDefaultConstrainedRounding(OldRounding);192}193 194static LValue195makeNaturalAlignAddrLValue(llvm::Value *V, QualType T, bool ForPointeeType,196 bool MightBeSigned, CodeGenFunction &CGF,197 KnownNonNull_t IsKnownNonNull = NotKnownNonNull) {198 LValueBaseInfo BaseInfo;199 TBAAAccessInfo TBAAInfo;200 CharUnits Alignment =201 CGF.CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo, ForPointeeType);202 Address Addr =203 MightBeSigned204 ? CGF.makeNaturalAddressForPointer(V, T, Alignment, false, nullptr,205 nullptr, IsKnownNonNull)206 : Address(V, CGF.ConvertTypeForMem(T), Alignment, IsKnownNonNull);207 return CGF.MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);208}209 210LValue211CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T,212 KnownNonNull_t IsKnownNonNull) {213 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false,214 /*MightBeSigned*/ true, *this,215 IsKnownNonNull);216}217 218LValue219CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) {220 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true,221 /*MightBeSigned*/ true, *this);222}223 224LValue CodeGenFunction::MakeNaturalAlignRawAddrLValue(llvm::Value *V,225 QualType T) {226 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ false,227 /*MightBeSigned*/ false, *this);228}229 230LValue CodeGenFunction::MakeNaturalAlignPointeeRawAddrLValue(llvm::Value *V,231 QualType T) {232 return ::makeNaturalAlignAddrLValue(V, T, /*ForPointeeType*/ true,233 /*MightBeSigned*/ false, *this);234}235 236llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) {237 return CGM.getTypes().ConvertTypeForMem(T);238}239 240llvm::Type *CodeGenFunction::ConvertType(QualType T) {241 return CGM.getTypes().ConvertType(T);242}243 244llvm::Type *CodeGenFunction::convertTypeForLoadStore(QualType ASTTy,245 llvm::Type *LLVMTy) {246 return CGM.getTypes().convertTypeForLoadStore(ASTTy, LLVMTy);247}248 249TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) {250 type = type.getCanonicalType();251 while (true) {252 switch (type->getTypeClass()) {253#define TYPE(name, parent)254#define ABSTRACT_TYPE(name, parent)255#define NON_CANONICAL_TYPE(name, parent) case Type::name:256#define DEPENDENT_TYPE(name, parent) case Type::name:257#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(name, parent) case Type::name:258#include "clang/AST/TypeNodes.inc"259 llvm_unreachable("non-canonical or dependent type in IR-generation");260 261 case Type::Auto:262 case Type::DeducedTemplateSpecialization:263 llvm_unreachable("undeduced type in IR-generation");264 265 // Various scalar types.266 case Type::Builtin:267 case Type::Pointer:268 case Type::BlockPointer:269 case Type::LValueReference:270 case Type::RValueReference:271 case Type::MemberPointer:272 case Type::Vector:273 case Type::ExtVector:274 case Type::ConstantMatrix:275 case Type::FunctionProto:276 case Type::FunctionNoProto:277 case Type::Enum:278 case Type::ObjCObjectPointer:279 case Type::Pipe:280 case Type::BitInt:281 case Type::HLSLAttributedResource:282 case Type::HLSLInlineSpirv:283 return TEK_Scalar;284 285 // Complexes.286 case Type::Complex:287 return TEK_Complex;288 289 // Arrays, records, and Objective-C objects.290 case Type::ConstantArray:291 case Type::IncompleteArray:292 case Type::VariableArray:293 case Type::Record:294 case Type::ObjCObject:295 case Type::ObjCInterface:296 case Type::ArrayParameter:297 return TEK_Aggregate;298 299 // We operate on atomic values according to their underlying type.300 case Type::Atomic:301 type = cast<AtomicType>(type)->getValueType();302 continue;303 }304 llvm_unreachable("unknown type kind!");305 }306}307 308llvm::DebugLoc CodeGenFunction::EmitReturnBlock() {309 // For cleanliness, we try to avoid emitting the return block for310 // simple cases.311 llvm::BasicBlock *CurBB = Builder.GetInsertBlock();312 313 if (CurBB) {314 assert(!CurBB->getTerminator() && "Unexpected terminated block.");315 316 // We have a valid insert point, reuse it if it is empty or there are no317 // explicit jumps to the return block.318 if (CurBB->empty() || ReturnBlock.getBlock()->use_empty()) {319 ReturnBlock.getBlock()->replaceAllUsesWith(CurBB);320 delete ReturnBlock.getBlock();321 ReturnBlock = JumpDest();322 } else323 EmitBlock(ReturnBlock.getBlock());324 return llvm::DebugLoc();325 }326 327 // Otherwise, if the return block is the target of a single direct328 // branch then we can just put the code in that block instead. This329 // cleans up functions which started with a unified return block.330 if (ReturnBlock.getBlock()->hasOneUse()) {331 llvm::BranchInst *BI =332 dyn_cast<llvm::BranchInst>(*ReturnBlock.getBlock()->user_begin());333 if (BI && BI->isUnconditional() &&334 BI->getSuccessor(0) == ReturnBlock.getBlock()) {335 // Record/return the DebugLoc of the simple 'return' expression to be used336 // later by the actual 'ret' instruction.337 llvm::DebugLoc Loc = BI->getDebugLoc();338 Builder.SetInsertPoint(BI->getParent());339 BI->eraseFromParent();340 delete ReturnBlock.getBlock();341 ReturnBlock = JumpDest();342 return Loc;343 }344 }345 346 // FIXME: We are at an unreachable point, there is no reason to emit the block347 // unless it has uses. However, we still need a place to put the debug348 // region.end for now.349 350 EmitBlock(ReturnBlock.getBlock());351 return llvm::DebugLoc();352}353 354static void EmitIfUsed(CodeGenFunction &CGF, llvm::BasicBlock *BB) {355 if (!BB) return;356 if (!BB->use_empty()) {357 CGF.CurFn->insert(CGF.CurFn->end(), BB);358 return;359 }360 delete BB;361}362 363void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {364 assert(BreakContinueStack.empty() &&365 "mismatched push/pop in break/continue stack!");366 assert(LifetimeExtendedCleanupStack.empty() &&367 "mismatched push/pop of cleanups in EHStack!");368 assert(DeferredDeactivationCleanupStack.empty() &&369 "mismatched activate/deactivate of cleanups!");370 371 if (CGM.shouldEmitConvergenceTokens()) {372 ConvergenceTokenStack.pop_back();373 assert(ConvergenceTokenStack.empty() &&374 "mismatched push/pop in convergence stack!");375 }376 377 bool OnlySimpleReturnStmts = NumSimpleReturnExprs > 0378 && NumSimpleReturnExprs == NumReturnExprs379 && ReturnBlock.getBlock()->use_empty();380 // Usually the return expression is evaluated before the cleanup381 // code. If the function contains only a simple return statement,382 // such as a constant, the location before the cleanup code becomes383 // the last useful breakpoint in the function, because the simple384 // return expression will be evaluated after the cleanup code. To be385 // safe, set the debug location for cleanup code to the location of386 // the return statement. Otherwise the cleanup code should be at the387 // end of the function's lexical scope.388 //389 // If there are multiple branches to the return block, the branch390 // instructions will get the location of the return statements and391 // all will be fine.392 if (CGDebugInfo *DI = getDebugInfo()) {393 if (OnlySimpleReturnStmts)394 DI->EmitLocation(Builder, LastStopPoint);395 else396 DI->EmitLocation(Builder, EndLoc);397 }398 399 // Pop any cleanups that might have been associated with the400 // parameters. Do this in whatever block we're currently in; it's401 // important to do this before we enter the return block or return402 // edges will be *really* confused.403 bool HasCleanups = EHStack.stable_begin() != PrologueCleanupDepth;404 bool HasOnlyNoopCleanups =405 HasCleanups && EHStack.containsOnlyNoopCleanups(PrologueCleanupDepth);406 bool EmitRetDbgLoc = !HasCleanups || HasOnlyNoopCleanups;407 408 std::optional<ApplyDebugLocation> OAL;409 if (HasCleanups) {410 // Make sure the line table doesn't jump back into the body for411 // the ret after it's been at EndLoc.412 if (CGDebugInfo *DI = getDebugInfo()) {413 if (OnlySimpleReturnStmts)414 DI->EmitLocation(Builder, EndLoc);415 else416 // We may not have a valid end location. Try to apply it anyway, and417 // fall back to an artificial location if needed.418 OAL = ApplyDebugLocation::CreateDefaultArtificial(*this, EndLoc);419 }420 421 PopCleanupBlocks(PrologueCleanupDepth);422 }423 424 // Emit function epilog (to return).425 llvm::DebugLoc Loc = EmitReturnBlock();426 427 if (ShouldInstrumentFunction()) {428 if (CGM.getCodeGenOpts().InstrumentFunctions)429 CurFn->addFnAttr("instrument-function-exit", "__cyg_profile_func_exit");430 if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)431 CurFn->addFnAttr("instrument-function-exit-inlined",432 "__cyg_profile_func_exit");433 }434 435 // Emit debug descriptor for function end.436 if (CGDebugInfo *DI = getDebugInfo())437 DI->EmitFunctionEnd(Builder, CurFn);438 439 // Reset the debug location to that of the simple 'return' expression, if any440 // rather than that of the end of the function's scope '}'.441 uint64_t RetKeyInstructionsAtomGroup = Loc ? Loc->getAtomGroup() : 0;442 ApplyDebugLocation AL(*this, Loc);443 EmitFunctionEpilog(*CurFnInfo, EmitRetDbgLoc, EndLoc,444 RetKeyInstructionsAtomGroup);445 EmitEndEHSpec(CurCodeDecl);446 447 assert(EHStack.empty() &&448 "did not remove all scopes from cleanup stack!");449 450 // If someone did an indirect goto, emit the indirect goto block at the end of451 // the function.452 if (IndirectBranch) {453 EmitBlock(IndirectBranch->getParent());454 Builder.ClearInsertionPoint();455 }456 457 // If some of our locals escaped, insert a call to llvm.localescape in the458 // entry block.459 if (!EscapedLocals.empty()) {460 // Invert the map from local to index into a simple vector. There should be461 // no holes.462 SmallVector<llvm::Value *, 4> EscapeArgs;463 EscapeArgs.resize(EscapedLocals.size());464 for (auto &Pair : EscapedLocals)465 EscapeArgs[Pair.second] = Pair.first;466 llvm::Function *FrameEscapeFn = llvm::Intrinsic::getOrInsertDeclaration(467 &CGM.getModule(), llvm::Intrinsic::localescape);468 CGBuilderTy(*this, AllocaInsertPt).CreateCall(FrameEscapeFn, EscapeArgs);469 }470 471 // Remove the AllocaInsertPt instruction, which is just a convenience for us.472 llvm::Instruction *Ptr = AllocaInsertPt;473 AllocaInsertPt = nullptr;474 Ptr->eraseFromParent();475 476 // PostAllocaInsertPt, if created, was lazily created when it was required,477 // remove it now since it was just created for our own convenience.478 if (PostAllocaInsertPt) {479 llvm::Instruction *PostPtr = PostAllocaInsertPt;480 PostAllocaInsertPt = nullptr;481 PostPtr->eraseFromParent();482 }483 484 // If someone took the address of a label but never did an indirect goto, we485 // made a zero entry PHI node, which is illegal, zap it now.486 if (IndirectBranch) {487 llvm::PHINode *PN = cast<llvm::PHINode>(IndirectBranch->getAddress());488 if (PN->getNumIncomingValues() == 0) {489 PN->replaceAllUsesWith(llvm::PoisonValue::get(PN->getType()));490 PN->eraseFromParent();491 }492 }493 494 EmitIfUsed(*this, EHResumeBlock);495 EmitIfUsed(*this, TerminateLandingPad);496 EmitIfUsed(*this, TerminateHandler);497 EmitIfUsed(*this, UnreachableBlock);498 499 for (const auto &FuncletAndParent : TerminateFunclets)500 EmitIfUsed(*this, FuncletAndParent.second);501 502 if (CGM.getCodeGenOpts().EmitDeclMetadata)503 EmitDeclMetadata();504 505 for (const auto &R : DeferredReplacements) {506 if (llvm::Value *Old = R.first) {507 Old->replaceAllUsesWith(R.second);508 cast<llvm::Instruction>(Old)->eraseFromParent();509 }510 }511 DeferredReplacements.clear();512 513 // Eliminate CleanupDestSlot alloca by replacing it with SSA values and514 // PHIs if the current function is a coroutine. We don't do it for all515 // functions as it may result in slight increase in numbers of instructions516 // if compiled with no optimizations. We do it for coroutine as the lifetime517 // of CleanupDestSlot alloca make correct coroutine frame building very518 // difficult.519 if (NormalCleanupDest.isValid() && isCoroutine()) {520 llvm::DominatorTree DT(*CurFn);521 llvm::PromoteMemToReg(522 cast<llvm::AllocaInst>(NormalCleanupDest.getPointer()), DT);523 NormalCleanupDest = Address::invalid();524 }525 526 // Scan function arguments for vector width.527 for (llvm::Argument &A : CurFn->args())528 if (auto *VT = dyn_cast<llvm::VectorType>(A.getType()))529 LargestVectorWidth =530 std::max((uint64_t)LargestVectorWidth,531 VT->getPrimitiveSizeInBits().getKnownMinValue());532 533 // Update vector width based on return type.534 if (auto *VT = dyn_cast<llvm::VectorType>(CurFn->getReturnType()))535 LargestVectorWidth =536 std::max((uint64_t)LargestVectorWidth,537 VT->getPrimitiveSizeInBits().getKnownMinValue());538 539 if (CurFnInfo->getMaxVectorWidth() > LargestVectorWidth)540 LargestVectorWidth = CurFnInfo->getMaxVectorWidth();541 542 // Add the min-legal-vector-width attribute. This contains the max width from:543 // 1. min-vector-width attribute used in the source program.544 // 2. Any builtins used that have a vector width specified.545 // 3. Values passed in and out of inline assembly.546 // 4. Width of vector arguments and return types for this function.547 // 5. Width of vector arguments and return types for functions called by this548 // function.549 if (getContext().getTargetInfo().getTriple().isX86())550 CurFn->addFnAttr("min-legal-vector-width",551 llvm::utostr(LargestVectorWidth));552 553 // If we generated an unreachable return block, delete it now.554 if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty()) {555 Builder.ClearInsertionPoint();556 ReturnBlock.getBlock()->eraseFromParent();557 }558 if (ReturnValue.isValid()) {559 auto *RetAlloca =560 dyn_cast<llvm::AllocaInst>(ReturnValue.emitRawPointer(*this));561 if (RetAlloca && RetAlloca->use_empty()) {562 RetAlloca->eraseFromParent();563 ReturnValue = Address::invalid();564 }565 }566}567 568/// ShouldInstrumentFunction - Return true if the current function should be569/// instrumented with __cyg_profile_func_* calls570bool CodeGenFunction::ShouldInstrumentFunction() {571 if (!CGM.getCodeGenOpts().InstrumentFunctions &&572 !CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining &&573 !CGM.getCodeGenOpts().InstrumentFunctionEntryBare)574 return false;575 if (!CurFuncDecl || CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>())576 return false;577 return true;578}579 580bool CodeGenFunction::ShouldSkipSanitizerInstrumentation() {581 if (!CurFuncDecl)582 return false;583 return CurFuncDecl->hasAttr<DisableSanitizerInstrumentationAttr>();584}585 586/// ShouldXRayInstrument - Return true if the current function should be587/// instrumented with XRay nop sleds.588bool CodeGenFunction::ShouldXRayInstrumentFunction() const {589 return CGM.getCodeGenOpts().XRayInstrumentFunctions;590}591 592/// AlwaysEmitXRayCustomEvents - Return true if we should emit IR for calls to593/// the __xray_customevent(...) builtin calls, when doing XRay instrumentation.594bool CodeGenFunction::AlwaysEmitXRayCustomEvents() const {595 return CGM.getCodeGenOpts().XRayInstrumentFunctions &&596 (CGM.getCodeGenOpts().XRayAlwaysEmitCustomEvents ||597 CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==598 XRayInstrKind::Custom);599}600 601bool CodeGenFunction::AlwaysEmitXRayTypedEvents() const {602 return CGM.getCodeGenOpts().XRayInstrumentFunctions &&603 (CGM.getCodeGenOpts().XRayAlwaysEmitTypedEvents ||604 CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==605 XRayInstrKind::Typed);606}607 608llvm::ConstantInt *609CodeGenFunction::getUBSanFunctionTypeHash(QualType Ty) const {610 // Remove any (C++17) exception specifications, to allow calling e.g. a611 // noexcept function through a non-noexcept pointer.612 if (!Ty->isFunctionNoProtoType())613 Ty = getContext().getFunctionTypeWithExceptionSpec(Ty, EST_None);614 std::string Mangled;615 llvm::raw_string_ostream Out(Mangled);616 CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(Ty, Out, false);617 return llvm::ConstantInt::get(618 CGM.Int32Ty, static_cast<uint32_t>(llvm::xxh3_64bits(Mangled)));619}620 621void CodeGenFunction::EmitKernelMetadata(const FunctionDecl *FD,622 llvm::Function *Fn) {623 if (!FD->hasAttr<DeviceKernelAttr>() && !FD->hasAttr<CUDAGlobalAttr>())624 return;625 626 llvm::LLVMContext &Context = getLLVMContext();627 628 CGM.GenKernelArgMetadata(Fn, FD, this);629 630 if (!(getLangOpts().OpenCL ||631 (getLangOpts().CUDA &&632 getContext().getTargetInfo().getTriple().isSPIRV())))633 return;634 635 if (const VecTypeHintAttr *A = FD->getAttr<VecTypeHintAttr>()) {636 QualType HintQTy = A->getTypeHint();637 const ExtVectorType *HintEltQTy = HintQTy->getAs<ExtVectorType>();638 bool IsSignedInteger =639 HintQTy->isSignedIntegerType() ||640 (HintEltQTy && HintEltQTy->getElementType()->isSignedIntegerType());641 llvm::Metadata *AttrMDArgs[] = {642 llvm::ConstantAsMetadata::get(llvm::PoisonValue::get(643 CGM.getTypes().ConvertType(A->getTypeHint()))),644 llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(645 llvm::IntegerType::get(Context, 32),646 llvm::APInt(32, (uint64_t)(IsSignedInteger ? 1 : 0))))};647 Fn->setMetadata("vec_type_hint", llvm::MDNode::get(Context, AttrMDArgs));648 }649 650 if (const WorkGroupSizeHintAttr *A = FD->getAttr<WorkGroupSizeHintAttr>()) {651 auto Eval = [&](Expr *E) {652 return E->EvaluateKnownConstInt(FD->getASTContext()).getExtValue();653 };654 llvm::Metadata *AttrMDArgs[] = {655 llvm::ConstantAsMetadata::get(Builder.getInt32(Eval(A->getXDim()))),656 llvm::ConstantAsMetadata::get(Builder.getInt32(Eval(A->getYDim()))),657 llvm::ConstantAsMetadata::get(Builder.getInt32(Eval(A->getZDim())))};658 Fn->setMetadata("work_group_size_hint", llvm::MDNode::get(Context, AttrMDArgs));659 }660 661 if (const ReqdWorkGroupSizeAttr *A = FD->getAttr<ReqdWorkGroupSizeAttr>()) {662 auto Eval = [&](Expr *E) {663 return E->EvaluateKnownConstInt(FD->getASTContext()).getExtValue();664 };665 llvm::Metadata *AttrMDArgs[] = {666 llvm::ConstantAsMetadata::get(Builder.getInt32(Eval(A->getXDim()))),667 llvm::ConstantAsMetadata::get(Builder.getInt32(Eval(A->getYDim()))),668 llvm::ConstantAsMetadata::get(Builder.getInt32(Eval(A->getZDim())))};669 Fn->setMetadata("reqd_work_group_size", llvm::MDNode::get(Context, AttrMDArgs));670 }671 672 if (const OpenCLIntelReqdSubGroupSizeAttr *A =673 FD->getAttr<OpenCLIntelReqdSubGroupSizeAttr>()) {674 llvm::Metadata *AttrMDArgs[] = {675 llvm::ConstantAsMetadata::get(Builder.getInt32(A->getSubGroupSize()))};676 Fn->setMetadata("intel_reqd_sub_group_size",677 llvm::MDNode::get(Context, AttrMDArgs));678 }679}680 681/// Determine whether the function F ends with a return stmt.682static bool endsWithReturn(const Decl* F) {683 const Stmt *Body = nullptr;684 if (auto *FD = dyn_cast_or_null<FunctionDecl>(F))685 Body = FD->getBody();686 else if (auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(F))687 Body = OMD->getBody();688 689 if (auto *CS = dyn_cast_or_null<CompoundStmt>(Body)) {690 auto LastStmt = CS->body_rbegin();691 if (LastStmt != CS->body_rend())692 return isa<ReturnStmt>(*LastStmt);693 }694 return false;695}696 697void CodeGenFunction::markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn) {698 if (SanOpts.has(SanitizerKind::Thread)) {699 Fn->addFnAttr("sanitize_thread_no_checking_at_run_time");700 Fn->removeFnAttr(llvm::Attribute::SanitizeThread);701 }702}703 704/// Check if the return value of this function requires sanitization.705bool CodeGenFunction::requiresReturnValueCheck() const {706 return requiresReturnValueNullabilityCheck() ||707 (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute) && CurCodeDecl &&708 CurCodeDecl->getAttr<ReturnsNonNullAttr>());709}710 711static bool matchesStlAllocatorFn(const Decl *D, const ASTContext &Ctx) {712 auto *MD = dyn_cast_or_null<CXXMethodDecl>(D);713 if (!MD || !MD->getDeclName().getAsIdentifierInfo() ||714 !MD->getDeclName().getAsIdentifierInfo()->isStr("allocate") ||715 (MD->getNumParams() != 1 && MD->getNumParams() != 2))716 return false;717 718 if (!Ctx.hasSameType(MD->parameters()[0]->getType(), Ctx.getSizeType()))719 return false;720 721 if (MD->getNumParams() == 2) {722 auto *PT = MD->parameters()[1]->getType()->getAs<PointerType>();723 if (!PT || !PT->isVoidPointerType() ||724 !PT->getPointeeType().isConstQualified())725 return false;726 }727 728 return true;729}730 731bool CodeGenFunction::isInAllocaArgument(CGCXXABI &ABI, QualType Ty) {732 const CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();733 return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;734}735 736bool CodeGenFunction::hasInAllocaArg(const CXXMethodDecl *MD) {737 return getTarget().getTriple().getArch() == llvm::Triple::x86 &&738 getTarget().getCXXABI().isMicrosoft() &&739 llvm::any_of(MD->parameters(), [&](ParmVarDecl *P) {740 return isInAllocaArgument(CGM.getCXXABI(), P->getType());741 });742}743 744/// Return the UBSan prologue signature for \p FD if one is available.745static llvm::Constant *getPrologueSignature(CodeGenModule &CGM,746 const FunctionDecl *FD) {747 if (const auto *MD = dyn_cast<CXXMethodDecl>(FD))748 if (!MD->isStatic())749 return nullptr;750 return CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM);751}752 753void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,754 llvm::Function *Fn,755 const CGFunctionInfo &FnInfo,756 const FunctionArgList &Args,757 SourceLocation Loc,758 SourceLocation StartLoc) {759 assert(!CurFn &&760 "Do not use a CodeGenFunction object for more than one function");761 762 const Decl *D = GD.getDecl();763 764 DidCallStackSave = false;765 CurCodeDecl = D;766 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D);767 if (FD && FD->usesSEHTry())768 CurSEHParent = GD;769 CurFuncDecl = (D ? D->getNonClosureContext() : nullptr);770 FnRetTy = RetTy;771 CurFn = Fn;772 CurFnInfo = &FnInfo;773 assert(CurFn->isDeclaration() && "Function already has body?");774 775 // If this function is ignored for any of the enabled sanitizers,776 // disable the sanitizer for the function.777 do {778#define SANITIZER(NAME, ID) \779 if (SanOpts.empty()) \780 break; \781 if (SanOpts.has(SanitizerKind::ID)) \782 if (CGM.isInNoSanitizeList(SanitizerKind::ID, Fn, Loc)) \783 SanOpts.set(SanitizerKind::ID, false);784 785#include "clang/Basic/Sanitizers.def"786#undef SANITIZER787 } while (false);788 789 if (D) {790 const bool SanitizeBounds = SanOpts.hasOneOf(SanitizerKind::Bounds);791 SanitizerMask no_sanitize_mask;792 bool NoSanitizeCoverage = false;793 794 for (auto *Attr : D->specific_attrs<NoSanitizeAttr>()) {795 no_sanitize_mask |= Attr->getMask();796 // SanitizeCoverage is not handled by SanOpts.797 if (Attr->hasCoverage())798 NoSanitizeCoverage = true;799 }800 801 // Apply the no_sanitize* attributes to SanOpts.802 SanOpts.Mask &= ~no_sanitize_mask;803 if (no_sanitize_mask & SanitizerKind::Address)804 SanOpts.set(SanitizerKind::KernelAddress, false);805 if (no_sanitize_mask & SanitizerKind::KernelAddress)806 SanOpts.set(SanitizerKind::Address, false);807 if (no_sanitize_mask & SanitizerKind::HWAddress)808 SanOpts.set(SanitizerKind::KernelHWAddress, false);809 if (no_sanitize_mask & SanitizerKind::KernelHWAddress)810 SanOpts.set(SanitizerKind::HWAddress, false);811 812 if (SanitizeBounds && !SanOpts.hasOneOf(SanitizerKind::Bounds))813 Fn->addFnAttr(llvm::Attribute::NoSanitizeBounds);814 815 if (NoSanitizeCoverage && CGM.getCodeGenOpts().hasSanitizeCoverage())816 Fn->addFnAttr(llvm::Attribute::NoSanitizeCoverage);817 818 // Some passes need the non-negated no_sanitize attribute. Pass them on.819 if (CGM.getCodeGenOpts().hasSanitizeBinaryMetadata()) {820 if (no_sanitize_mask & SanitizerKind::Thread)821 Fn->addFnAttr("no_sanitize_thread");822 }823 }824 825 if (ShouldSkipSanitizerInstrumentation()) {826 CurFn->addFnAttr(llvm::Attribute::DisableSanitizerInstrumentation);827 } else {828 // Apply sanitizer attributes to the function.829 if (SanOpts.hasOneOf(SanitizerKind::Address | SanitizerKind::KernelAddress))830 Fn->addFnAttr(llvm::Attribute::SanitizeAddress);831 if (SanOpts.hasOneOf(SanitizerKind::HWAddress |832 SanitizerKind::KernelHWAddress))833 Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress);834 if (SanOpts.has(SanitizerKind::MemtagStack))835 Fn->addFnAttr(llvm::Attribute::SanitizeMemTag);836 if (SanOpts.has(SanitizerKind::Thread))837 Fn->addFnAttr(llvm::Attribute::SanitizeThread);838 if (SanOpts.has(SanitizerKind::Type))839 Fn->addFnAttr(llvm::Attribute::SanitizeType);840 if (SanOpts.has(SanitizerKind::NumericalStability))841 Fn->addFnAttr(llvm::Attribute::SanitizeNumericalStability);842 if (SanOpts.hasOneOf(SanitizerKind::Memory | SanitizerKind::KernelMemory))843 Fn->addFnAttr(llvm::Attribute::SanitizeMemory);844 if (SanOpts.has(SanitizerKind::AllocToken))845 Fn->addFnAttr(llvm::Attribute::SanitizeAllocToken);846 }847 if (SanOpts.has(SanitizerKind::SafeStack))848 Fn->addFnAttr(llvm::Attribute::SafeStack);849 if (SanOpts.has(SanitizerKind::ShadowCallStack))850 Fn->addFnAttr(llvm::Attribute::ShadowCallStack);851 852 if (SanOpts.has(SanitizerKind::Realtime))853 if (FD && FD->getASTContext().hasAnyFunctionEffects())854 for (const FunctionEffectWithCondition &Fe : FD->getFunctionEffects()) {855 if (Fe.Effect.kind() == FunctionEffect::Kind::NonBlocking)856 Fn->addFnAttr(llvm::Attribute::SanitizeRealtime);857 else if (Fe.Effect.kind() == FunctionEffect::Kind::Blocking)858 Fn->addFnAttr(llvm::Attribute::SanitizeRealtimeBlocking);859 }860 861 // Apply fuzzing attribute to the function.862 if (SanOpts.hasOneOf(SanitizerKind::Fuzzer | SanitizerKind::FuzzerNoLink))863 Fn->addFnAttr(llvm::Attribute::OptForFuzzing);864 865 // Ignore TSan memory acesses from within ObjC/ObjC++ dealloc, initialize,866 // .cxx_destruct, __destroy_helper_block_ and all of their calees at run time.867 if (SanOpts.has(SanitizerKind::Thread)) {868 if (const auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(D)) {869 const IdentifierInfo *II = OMD->getSelector().getIdentifierInfoForSlot(0);870 if (OMD->getMethodFamily() == OMF_dealloc ||871 OMD->getMethodFamily() == OMF_initialize ||872 (OMD->getSelector().isUnarySelector() && II->isStr(".cxx_destruct"))) {873 markAsIgnoreThreadCheckingAtRuntime(Fn);874 }875 }876 }877 878 // Ignore unrelated casts in STL allocate() since the allocator must cast879 // from void* to T* before object initialization completes. Don't match on the880 // namespace because not all allocators are in std::881 if (D && SanOpts.has(SanitizerKind::CFIUnrelatedCast)) {882 if (matchesStlAllocatorFn(D, getContext()))883 SanOpts.Mask &= ~SanitizerKind::CFIUnrelatedCast;884 }885 886 // Ignore null checks in coroutine functions since the coroutines passes887 // are not aware of how to move the extra UBSan instructions across the split888 // coroutine boundaries.889 if (D && SanOpts.has(SanitizerKind::Null))890 if (FD && FD->getBody() &&891 FD->getBody()->getStmtClass() == Stmt::CoroutineBodyStmtClass)892 SanOpts.Mask &= ~SanitizerKind::Null;893 894 // Apply xray attributes to the function (as a string, for now)895 bool AlwaysXRayAttr = false;896 if (const auto *XRayAttr = D ? D->getAttr<XRayInstrumentAttr>() : nullptr) {897 if (CGM.getCodeGenOpts().XRayInstrumentationBundle.has(898 XRayInstrKind::FunctionEntry) ||899 CGM.getCodeGenOpts().XRayInstrumentationBundle.has(900 XRayInstrKind::FunctionExit)) {901 if (XRayAttr->alwaysXRayInstrument() && ShouldXRayInstrumentFunction()) {902 Fn->addFnAttr("function-instrument", "xray-always");903 AlwaysXRayAttr = true;904 }905 if (XRayAttr->neverXRayInstrument())906 Fn->addFnAttr("function-instrument", "xray-never");907 if (const auto *LogArgs = D->getAttr<XRayLogArgsAttr>())908 if (ShouldXRayInstrumentFunction())909 Fn->addFnAttr("xray-log-args",910 llvm::utostr(LogArgs->getArgumentCount()));911 }912 } else {913 if (ShouldXRayInstrumentFunction() && !CGM.imbueXRayAttrs(Fn, Loc))914 Fn->addFnAttr(915 "xray-instruction-threshold",916 llvm::itostr(CGM.getCodeGenOpts().XRayInstructionThreshold));917 }918 919 if (ShouldXRayInstrumentFunction()) {920 if (CGM.getCodeGenOpts().XRayIgnoreLoops)921 Fn->addFnAttr("xray-ignore-loops");922 923 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(924 XRayInstrKind::FunctionExit))925 Fn->addFnAttr("xray-skip-exit");926 927 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(928 XRayInstrKind::FunctionEntry))929 Fn->addFnAttr("xray-skip-entry");930 931 auto FuncGroups = CGM.getCodeGenOpts().XRayTotalFunctionGroups;932 if (FuncGroups > 1) {933 auto FuncName = llvm::ArrayRef<uint8_t>(CurFn->getName().bytes_begin(),934 CurFn->getName().bytes_end());935 auto Group = crc32(FuncName) % FuncGroups;936 if (Group != CGM.getCodeGenOpts().XRaySelectedFunctionGroup &&937 !AlwaysXRayAttr)938 Fn->addFnAttr("function-instrument", "xray-never");939 }940 }941 942 if (CGM.getCodeGenOpts().getProfileInstr() !=943 llvm::driver::ProfileInstrKind::ProfileNone) {944 switch (CGM.isFunctionBlockedFromProfileInstr(Fn, Loc)) {945 case ProfileList::Skip:946 Fn->addFnAttr(llvm::Attribute::SkipProfile);947 break;948 case ProfileList::Forbid:949 Fn->addFnAttr(llvm::Attribute::NoProfile);950 break;951 case ProfileList::Allow:952 break;953 }954 }955 956 unsigned Count, Offset;957 StringRef Section;958 if (const auto *Attr =959 D ? D->getAttr<PatchableFunctionEntryAttr>() : nullptr) {960 Count = Attr->getCount();961 Offset = Attr->getOffset();962 Section = Attr->getSection();963 } else {964 Count = CGM.getCodeGenOpts().PatchableFunctionEntryCount;965 Offset = CGM.getCodeGenOpts().PatchableFunctionEntryOffset;966 }967 if (Section.empty())968 Section = CGM.getCodeGenOpts().PatchableFunctionEntrySection;969 if (Count && Offset <= Count) {970 Fn->addFnAttr("patchable-function-entry", std::to_string(Count - Offset));971 if (Offset)972 Fn->addFnAttr("patchable-function-prefix", std::to_string(Offset));973 if (!Section.empty())974 Fn->addFnAttr("patchable-function-entry-section", Section);975 }976 // Instruct that functions for COFF/CodeView targets should start with a977 // patchable instruction, but only on x86/x64. Don't forward this to ARM/ARM64978 // backends as they don't need it -- instructions on these architectures are979 // always atomically patchable at runtime.980 if (CGM.getCodeGenOpts().HotPatch &&981 getContext().getTargetInfo().getTriple().isX86() &&982 getContext().getTargetInfo().getTriple().getEnvironment() !=983 llvm::Triple::CODE16)984 Fn->addFnAttr("patchable-function", "prologue-short-redirect");985 986 // Add no-jump-tables value.987 if (CGM.getCodeGenOpts().NoUseJumpTables)988 Fn->addFnAttr("no-jump-tables", "true");989 990 // Add no-inline-line-tables value.991 if (CGM.getCodeGenOpts().NoInlineLineTables)992 Fn->addFnAttr("no-inline-line-tables");993 994 // Add profile-sample-accurate value.995 if (CGM.getCodeGenOpts().ProfileSampleAccurate)996 Fn->addFnAttr("profile-sample-accurate");997 998 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())999 Fn->addFnAttr("use-sample-profile");1000 1001 if (D && D->hasAttr<CFICanonicalJumpTableAttr>())1002 Fn->addFnAttr("cfi-canonical-jump-table");1003 1004 if (D && D->hasAttr<NoProfileFunctionAttr>())1005 Fn->addFnAttr(llvm::Attribute::NoProfile);1006 1007 if (D && D->hasAttr<HybridPatchableAttr>())1008 Fn->addFnAttr(llvm::Attribute::HybridPatchable);1009 1010 if (D) {1011 // Function attributes take precedence over command line flags.1012 if (auto *A = D->getAttr<FunctionReturnThunksAttr>()) {1013 switch (A->getThunkType()) {1014 case FunctionReturnThunksAttr::Kind::Keep:1015 break;1016 case FunctionReturnThunksAttr::Kind::Extern:1017 Fn->addFnAttr(llvm::Attribute::FnRetThunkExtern);1018 break;1019 }1020 } else if (CGM.getCodeGenOpts().FunctionReturnThunks)1021 Fn->addFnAttr(llvm::Attribute::FnRetThunkExtern);1022 }1023 1024 if (FD && (getLangOpts().OpenCL ||1025 (getLangOpts().CUDA &&1026 getContext().getTargetInfo().getTriple().isSPIRV()) ||1027 ((getLangOpts().HIP || getLangOpts().OffloadViaLLVM) &&1028 getLangOpts().CUDAIsDevice))) {1029 // Add metadata for a kernel function.1030 EmitKernelMetadata(FD, Fn);1031 }1032 1033 if (FD && FD->hasAttr<ClspvLibclcBuiltinAttr>()) {1034 Fn->setMetadata("clspv_libclc_builtin",1035 llvm::MDNode::get(getLLVMContext(), {}));1036 }1037 1038 // If we are checking function types, emit a function type signature as1039 // prologue data.1040 if (FD && SanOpts.has(SanitizerKind::Function)) {1041 if (llvm::Constant *PrologueSig = getPrologueSignature(CGM, FD)) {1042 llvm::LLVMContext &Ctx = Fn->getContext();1043 llvm::MDBuilder MDB(Ctx);1044 Fn->setMetadata(1045 llvm::LLVMContext::MD_func_sanitize,1046 MDB.createRTTIPointerPrologue(1047 PrologueSig, getUBSanFunctionTypeHash(FD->getType())));1048 }1049 }1050 1051 // If we're checking nullability, we need to know whether we can check the1052 // return value. Initialize the flag to 'true' and refine it in EmitParmDecl.1053 if (SanOpts.has(SanitizerKind::NullabilityReturn)) {1054 auto Nullability = FnRetTy->getNullability();1055 if (Nullability && *Nullability == NullabilityKind::NonNull &&1056 !FnRetTy->isRecordType()) {1057 if (!(SanOpts.has(SanitizerKind::ReturnsNonnullAttribute) &&1058 CurCodeDecl && CurCodeDecl->getAttr<ReturnsNonNullAttr>()))1059 RetValNullabilityPrecondition =1060 llvm::ConstantInt::getTrue(getLLVMContext());1061 }1062 }1063 1064 // If we're in C++ mode and the function name is "main", it is guaranteed1065 // to be norecurse by the standard (3.6.1.3 "The function main shall not be1066 // used within a program").1067 //1068 // OpenCL C 2.0 v2.2-11 s6.9.i:1069 // Recursion is not supported.1070 //1071 // HLSL1072 // Recursion is not supported.1073 //1074 // SYCL v1.2.1 s3.10:1075 // kernels cannot include RTTI information, exception classes,1076 // recursive code, virtual functions or make use of C++ libraries that1077 // are not compiled for the device.1078 if (FD &&1079 ((getLangOpts().CPlusPlus && FD->isMain()) || getLangOpts().OpenCL ||1080 getLangOpts().HLSL || getLangOpts().SYCLIsDevice ||1081 (getLangOpts().CUDA && FD->hasAttr<CUDAGlobalAttr>())))1082 Fn->addFnAttr(llvm::Attribute::NoRecurse);1083 1084 llvm::RoundingMode RM = getLangOpts().getDefaultRoundingMode();1085 llvm::fp::ExceptionBehavior FPExceptionBehavior =1086 ToConstrainedExceptMD(getLangOpts().getDefaultExceptionMode());1087 Builder.setDefaultConstrainedRounding(RM);1088 Builder.setDefaultConstrainedExcept(FPExceptionBehavior);1089 if ((FD && (FD->UsesFPIntrin() || FD->hasAttr<StrictFPAttr>())) ||1090 (!FD && (FPExceptionBehavior != llvm::fp::ebIgnore ||1091 RM != llvm::RoundingMode::NearestTiesToEven))) {1092 Builder.setIsFPConstrained(true);1093 Fn->addFnAttr(llvm::Attribute::StrictFP);1094 }1095 1096 // If a custom alignment is used, force realigning to this alignment on1097 // any main function which certainly will need it.1098 if (FD && ((FD->isMain() || FD->isMSVCRTEntryPoint()) &&1099 CGM.getCodeGenOpts().StackAlignment))1100 Fn->addFnAttr("stackrealign");1101 1102 // "main" doesn't need to zero out call-used registers.1103 if (FD && FD->isMain())1104 Fn->removeFnAttr("zero-call-used-regs");1105 1106 // Add vscale_range attribute if appropriate.1107 llvm::StringMap<bool> FeatureMap;1108 auto IsArmStreaming = TargetInfo::ArmStreamingKind::NotStreaming;1109 if (FD) {1110 getContext().getFunctionFeatureMap(FeatureMap, FD);1111 if (const auto *T = FD->getType()->getAs<FunctionProtoType>())1112 if (T->getAArch64SMEAttributes() &1113 FunctionType::SME_PStateSMCompatibleMask)1114 IsArmStreaming = TargetInfo::ArmStreamingKind::StreamingCompatible;1115 1116 if (IsArmStreamingFunction(FD, true))1117 IsArmStreaming = TargetInfo::ArmStreamingKind::Streaming;1118 }1119 std::optional<std::pair<unsigned, unsigned>> VScaleRange =1120 getContext().getTargetInfo().getVScaleRange(getLangOpts(), IsArmStreaming,1121 &FeatureMap);1122 if (VScaleRange) {1123 CurFn->addFnAttr(llvm::Attribute::getWithVScaleRangeArgs(1124 getLLVMContext(), VScaleRange->first, VScaleRange->second));1125 }1126 1127 llvm::BasicBlock *EntryBB = createBasicBlock("entry", CurFn);1128 1129 // Create a marker to make it easy to insert allocas into the entryblock1130 // later. Don't create this with the builder, because we don't want it1131 // folded.1132 llvm::Value *Poison = llvm::PoisonValue::get(Int32Ty);1133 AllocaInsertPt = new llvm::BitCastInst(Poison, Int32Ty, "allocapt", EntryBB);1134 1135 ReturnBlock = getJumpDestInCurrentScope("return");1136 1137 Builder.SetInsertPoint(EntryBB);1138 1139 // If we're checking the return value, allocate space for a pointer to a1140 // precise source location of the checked return statement.1141 if (requiresReturnValueCheck()) {1142 ReturnLocation = CreateDefaultAlignTempAlloca(Int8PtrTy, "return.sloc.ptr");1143 Builder.CreateStore(llvm::ConstantPointerNull::get(Int8PtrTy),1144 ReturnLocation);1145 }1146 1147 // Emit subprogram debug descriptor.1148 if (CGDebugInfo *DI = getDebugInfo()) {1149 // Reconstruct the type from the argument list so that implicit parameters,1150 // such as 'this' and 'vtt', show up in the debug info. Preserve the calling1151 // convention.1152 DI->emitFunctionStart(GD, Loc, StartLoc,1153 DI->getFunctionType(FD, RetTy, Args), CurFn,1154 CurFuncIsThunk);1155 }1156 1157 if (ShouldInstrumentFunction()) {1158 if (CGM.getCodeGenOpts().InstrumentFunctions)1159 CurFn->addFnAttr("instrument-function-entry", "__cyg_profile_func_enter");1160 if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)1161 CurFn->addFnAttr("instrument-function-entry-inlined",1162 "__cyg_profile_func_enter");1163 if (CGM.getCodeGenOpts().InstrumentFunctionEntryBare)1164 CurFn->addFnAttr("instrument-function-entry-inlined",1165 "__cyg_profile_func_enter_bare");1166 }1167 1168 // Since emitting the mcount call here impacts optimizations such as function1169 // inlining, we just add an attribute to insert a mcount call in backend.1170 // The attribute "counting-function" is set to mcount function name which is1171 // architecture dependent.1172 if (CGM.getCodeGenOpts().InstrumentForProfiling) {1173 // Calls to fentry/mcount should not be generated if function has1174 // the no_instrument_function attribute.1175 if (!CurFuncDecl || !CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>()) {1176 if (CGM.getCodeGenOpts().CallFEntry)1177 Fn->addFnAttr("fentry-call", "true");1178 else {1179 Fn->addFnAttr("instrument-function-entry-inlined",1180 getTarget().getMCountName());1181 }1182 if (CGM.getCodeGenOpts().MNopMCount) {1183 if (!CGM.getCodeGenOpts().CallFEntry)1184 CGM.getDiags().Report(diag::err_opt_not_valid_without_opt)1185 << "-mnop-mcount" << "-mfentry";1186 Fn->addFnAttr("mnop-mcount");1187 }1188 1189 if (CGM.getCodeGenOpts().RecordMCount) {1190 if (!CGM.getCodeGenOpts().CallFEntry)1191 CGM.getDiags().Report(diag::err_opt_not_valid_without_opt)1192 << "-mrecord-mcount" << "-mfentry";1193 Fn->addFnAttr("mrecord-mcount");1194 }1195 }1196 }1197 1198 if (CGM.getCodeGenOpts().PackedStack) {1199 if (getContext().getTargetInfo().getTriple().getArch() !=1200 llvm::Triple::systemz)1201 CGM.getDiags().Report(diag::err_opt_not_valid_on_target)1202 << "-mpacked-stack";1203 Fn->addFnAttr("packed-stack");1204 }1205 1206 if (CGM.getCodeGenOpts().WarnStackSize != UINT_MAX &&1207 !CGM.getDiags().isIgnored(diag::warn_fe_backend_frame_larger_than, Loc))1208 Fn->addFnAttr("warn-stack-size",1209 std::to_string(CGM.getCodeGenOpts().WarnStackSize));1210 1211 if (RetTy->isVoidType()) {1212 // Void type; nothing to return.1213 ReturnValue = Address::invalid();1214 1215 // Count the implicit return.1216 if (!endsWithReturn(D))1217 ++NumReturnExprs;1218 } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect) {1219 // Indirect return; emit returned value directly into sret slot.1220 // This reduces code size, and affects correctness in C++.1221 auto AI = CurFn->arg_begin();1222 if (CurFnInfo->getReturnInfo().isSRetAfterThis())1223 ++AI;1224 ReturnValue = makeNaturalAddressForPointer(1225 &*AI, RetTy, CurFnInfo->getReturnInfo().getIndirectAlign(), false,1226 nullptr, nullptr, KnownNonNull);1227 if (!CurFnInfo->getReturnInfo().getIndirectByVal()) {1228 ReturnValuePointer =1229 CreateDefaultAlignTempAlloca(ReturnValue.getType(), "result.ptr");1230 Builder.CreateStore(ReturnValue.emitRawPointer(*this),1231 ReturnValuePointer);1232 }1233 } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca &&1234 !hasScalarEvaluationKind(CurFnInfo->getReturnType())) {1235 // Load the sret pointer from the argument struct and return into that.1236 unsigned Idx = CurFnInfo->getReturnInfo().getInAllocaFieldIndex();1237 llvm::Function::arg_iterator EI = CurFn->arg_end();1238 --EI;1239 llvm::Value *Addr = Builder.CreateStructGEP(1240 CurFnInfo->getArgStruct(), &*EI, Idx);1241 llvm::Type *Ty =1242 cast<llvm::GetElementPtrInst>(Addr)->getResultElementType();1243 ReturnValuePointer = Address(Addr, Ty, getPointerAlign());1244 Addr = Builder.CreateAlignedLoad(Ty, Addr, getPointerAlign(), "agg.result");1245 ReturnValue = Address(Addr, ConvertType(RetTy),1246 CGM.getNaturalTypeAlignment(RetTy), KnownNonNull);1247 } else {1248 ReturnValue = CreateIRTemp(RetTy, "retval");1249 1250 // Tell the epilog emitter to autorelease the result. We do this1251 // now so that various specialized functions can suppress it1252 // during their IR-generation.1253 if (getLangOpts().ObjCAutoRefCount &&1254 !CurFnInfo->isReturnsRetained() &&1255 RetTy->isObjCRetainableType())1256 AutoreleaseResult = true;1257 }1258 1259 EmitStartEHSpec(CurCodeDecl);1260 1261 PrologueCleanupDepth = EHStack.stable_begin();1262 1263 // Emit OpenMP specific initialization of the device functions.1264 if (getLangOpts().OpenMP && CurCodeDecl)1265 CGM.getOpenMPRuntime().emitFunctionProlog(*this, CurCodeDecl);1266 1267 if (FD && getLangOpts().HLSL) {1268 // Handle emitting HLSL entry functions.1269 if (FD->hasAttr<HLSLShaderAttr>()) {1270 CGM.getHLSLRuntime().emitEntryFunction(FD, Fn);1271 }1272 }1273 1274 EmitFunctionProlog(*CurFnInfo, CurFn, Args);1275 1276 if (const CXXMethodDecl *MD = dyn_cast_if_present<CXXMethodDecl>(D);1277 MD && !MD->isStatic()) {1278 bool IsInLambda =1279 MD->getParent()->isLambda() && MD->getOverloadedOperator() == OO_Call;1280 if (MD->isImplicitObjectMemberFunction())1281 CGM.getCXXABI().EmitInstanceFunctionProlog(*this);1282 if (IsInLambda) {1283 // We're in a lambda; figure out the captures.1284 MD->getParent()->getCaptureFields(LambdaCaptureFields,1285 LambdaThisCaptureField);1286 if (LambdaThisCaptureField) {1287 // If the lambda captures the object referred to by '*this' - either by1288 // value or by reference, make sure CXXThisValue points to the correct1289 // object.1290 1291 // Get the lvalue for the field (which is a copy of the enclosing object1292 // or contains the address of the enclosing object).1293 LValue ThisFieldLValue = EmitLValueForLambdaField(LambdaThisCaptureField);1294 if (!LambdaThisCaptureField->getType()->isPointerType()) {1295 // If the enclosing object was captured by value, just use its1296 // address. Sign this pointer.1297 CXXThisValue = ThisFieldLValue.getPointer(*this);1298 } else {1299 // Load the lvalue pointed to by the field, since '*this' was captured1300 // by reference.1301 CXXThisValue =1302 EmitLoadOfLValue(ThisFieldLValue, SourceLocation()).getScalarVal();1303 }1304 }1305 for (auto *FD : MD->getParent()->fields()) {1306 if (FD->hasCapturedVLAType()) {1307 auto *ExprArg = EmitLoadOfLValue(EmitLValueForLambdaField(FD),1308 SourceLocation()).getScalarVal();1309 auto VAT = FD->getCapturedVLAType();1310 VLASizeMap[VAT->getSizeExpr()] = ExprArg;1311 }1312 }1313 } else if (MD->isImplicitObjectMemberFunction()) {1314 // Not in a lambda; just use 'this' from the method.1315 // FIXME: Should we generate a new load for each use of 'this'? The1316 // fast register allocator would be happier...1317 CXXThisValue = CXXABIThisValue;1318 }1319 1320 // Check the 'this' pointer once per function, if it's available.1321 if (CXXABIThisValue) {1322 SanitizerSet SkippedChecks;1323 SkippedChecks.set(SanitizerKind::ObjectSize, true);1324 QualType ThisTy = MD->getThisType();1325 1326 // If this is the call operator of a lambda with no captures, it1327 // may have a static invoker function, which may call this operator with1328 // a null 'this' pointer.1329 if (isLambdaCallOperator(MD) && MD->getParent()->isCapturelessLambda())1330 SkippedChecks.set(SanitizerKind::Null, true);1331 1332 EmitTypeCheck(1333 isa<CXXConstructorDecl>(MD) ? TCK_ConstructorCall : TCK_MemberCall,1334 Loc, CXXABIThisValue, ThisTy, CXXABIThisAlignment, SkippedChecks);1335 }1336 }1337 1338 // If any of the arguments have a variably modified type, make sure to1339 // emit the type size, but only if the function is not naked. Naked functions1340 // have no prolog to run this evaluation.1341 if (!FD || !FD->hasAttr<NakedAttr>()) {1342 for (const VarDecl *VD : Args) {1343 // Dig out the type as written from ParmVarDecls; it's unclear whether1344 // the standard (C99 6.9.1p10) requires this, but we're following the1345 // precedent set by gcc.1346 QualType Ty;1347 if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(VD))1348 Ty = PVD->getOriginalType();1349 else1350 Ty = VD->getType();1351 1352 if (Ty->isVariablyModifiedType())1353 EmitVariablyModifiedType(Ty);1354 }1355 }1356 // Emit a location at the end of the prologue.1357 if (CGDebugInfo *DI = getDebugInfo())1358 DI->EmitLocation(Builder, StartLoc);1359 // TODO: Do we need to handle this in two places like we do with1360 // target-features/target-cpu?1361 if (CurFuncDecl)1362 if (const auto *VecWidth = CurFuncDecl->getAttr<MinVectorWidthAttr>())1363 LargestVectorWidth = VecWidth->getVectorWidth();1364 1365 if (CGM.shouldEmitConvergenceTokens())1366 ConvergenceTokenStack.push_back(getOrEmitConvergenceEntryToken(CurFn));1367}1368 1369void CodeGenFunction::EmitFunctionBody(const Stmt *Body) {1370 incrementProfileCounter(Body);1371 maybeCreateMCDCCondBitmap();1372 if (const CompoundStmt *S = dyn_cast<CompoundStmt>(Body))1373 EmitCompoundStmtWithoutScope(*S);1374 else1375 EmitStmt(Body);1376}1377 1378/// When instrumenting to collect profile data, the counts for some blocks1379/// such as switch cases need to not include the fall-through counts, so1380/// emit a branch around the instrumentation code. When not instrumenting,1381/// this just calls EmitBlock().1382void CodeGenFunction::EmitBlockWithFallThrough(llvm::BasicBlock *BB,1383 const Stmt *S) {1384 llvm::BasicBlock *SkipCountBB = nullptr;1385 // Do not skip over the instrumentation when single byte coverage mode is1386 // enabled.1387 if (HaveInsertPoint() && CGM.getCodeGenOpts().hasProfileClangInstr() &&1388 !llvm::EnableSingleByteCoverage) {1389 // When instrumenting for profiling, the fallthrough to certain1390 // statements needs to skip over the instrumentation code so that we1391 // get an accurate count.1392 SkipCountBB = createBasicBlock("skipcount");1393 EmitBranch(SkipCountBB);1394 }1395 EmitBlock(BB);1396 uint64_t CurrentCount = getCurrentProfileCount();1397 incrementProfileCounter(S);1398 setCurrentProfileCount(getCurrentProfileCount() + CurrentCount);1399 if (SkipCountBB)1400 EmitBlock(SkipCountBB);1401}1402 1403/// Tries to mark the given function nounwind based on the1404/// non-existence of any throwing calls within it. We believe this is1405/// lightweight enough to do at -O0.1406static void TryMarkNoThrow(llvm::Function *F) {1407 // LLVM treats 'nounwind' on a function as part of the type, so we1408 // can't do this on functions that can be overwritten.1409 if (F->isInterposable()) return;1410 1411 for (llvm::BasicBlock &BB : *F)1412 for (llvm::Instruction &I : BB)1413 if (I.mayThrow())1414 return;1415 1416 F->setDoesNotThrow();1417}1418 1419QualType CodeGenFunction::BuildFunctionArgList(GlobalDecl GD,1420 FunctionArgList &Args) {1421 const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());1422 QualType ResTy = FD->getReturnType();1423 1424 const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD);1425 if (MD && MD->isImplicitObjectMemberFunction()) {1426 if (CGM.getCXXABI().HasThisReturn(GD))1427 ResTy = MD->getThisType();1428 else if (CGM.getCXXABI().hasMostDerivedReturn(GD))1429 ResTy = CGM.getContext().VoidPtrTy;1430 CGM.getCXXABI().buildThisParam(*this, Args);1431 }1432 1433 // The base version of an inheriting constructor whose constructed base is a1434 // virtual base is not passed any arguments (because it doesn't actually call1435 // the inherited constructor).1436 bool PassedParams = true;1437 if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(FD))1438 if (auto Inherited = CD->getInheritedConstructor())1439 PassedParams =1440 getTypes().inheritingCtorHasParams(Inherited, GD.getCtorType());1441 1442 if (PassedParams) {1443 for (auto *Param : FD->parameters()) {1444 Args.push_back(Param);1445 if (!Param->hasAttr<PassObjectSizeAttr>())1446 continue;1447 1448 auto *Implicit = ImplicitParamDecl::Create(1449 getContext(), Param->getDeclContext(), Param->getLocation(),1450 /*Id=*/nullptr, getContext().getSizeType(), ImplicitParamKind::Other);1451 SizeArguments[Param] = Implicit;1452 Args.push_back(Implicit);1453 }1454 }1455 1456 if (MD && (isa<CXXConstructorDecl>(MD) || isa<CXXDestructorDecl>(MD)))1457 CGM.getCXXABI().addImplicitStructorParams(*this, ResTy, Args);1458 1459 return ResTy;1460}1461 1462void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,1463 const CGFunctionInfo &FnInfo) {1464 assert(Fn && "generating code for null Function");1465 const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());1466 CurGD = GD;1467 1468 FunctionArgList Args;1469 QualType ResTy = BuildFunctionArgList(GD, Args);1470 1471 CGM.getTargetCodeGenInfo().checkFunctionABI(CGM, FD);1472 1473 if (FD->isInlineBuiltinDeclaration()) {1474 // When generating code for a builtin with an inline declaration, use a1475 // mangled name to hold the actual body, while keeping an external1476 // definition in case the function pointer is referenced somewhere.1477 std::string FDInlineName = (Fn->getName() + ".inline").str();1478 llvm::Module *M = Fn->getParent();1479 llvm::Function *Clone = M->getFunction(FDInlineName);1480 if (!Clone) {1481 Clone = llvm::Function::Create(Fn->getFunctionType(),1482 llvm::GlobalValue::InternalLinkage,1483 Fn->getAddressSpace(), FDInlineName, M);1484 Clone->addFnAttr(llvm::Attribute::AlwaysInline);1485 }1486 Fn->setLinkage(llvm::GlobalValue::ExternalLinkage);1487 Fn = Clone;1488 } else {1489 // Detect the unusual situation where an inline version is shadowed by a1490 // non-inline version. In that case we should pick the external one1491 // everywhere. That's GCC behavior too. Unfortunately, I cannot find a way1492 // to detect that situation before we reach codegen, so do some late1493 // replacement.1494 for (const FunctionDecl *PD = FD->getPreviousDecl(); PD;1495 PD = PD->getPreviousDecl()) {1496 if (LLVM_UNLIKELY(PD->isInlineBuiltinDeclaration())) {1497 std::string FDInlineName = (Fn->getName() + ".inline").str();1498 llvm::Module *M = Fn->getParent();1499 if (llvm::Function *Clone = M->getFunction(FDInlineName)) {1500 Clone->replaceAllUsesWith(Fn);1501 Clone->eraseFromParent();1502 }1503 break;1504 }1505 }1506 }1507 1508 // Check if we should generate debug info for this function.1509 if (FD->hasAttr<NoDebugAttr>()) {1510 // Clear non-distinct debug info that was possibly attached to the function1511 // due to an earlier declaration without the nodebug attribute1512 Fn->setSubprogram(nullptr);1513 // Disable debug info indefinitely for this function1514 DebugInfo = nullptr;1515 }1516 // Finalize function debug info on exit.1517 auto Cleanup = llvm::make_scope_exit([this] {1518 if (CGDebugInfo *DI = getDebugInfo())1519 DI->completeFunction();1520 });1521 1522 // The function might not have a body if we're generating thunks for a1523 // function declaration.1524 SourceRange BodyRange;1525 if (Stmt *Body = FD->getBody())1526 BodyRange = Body->getSourceRange();1527 else1528 BodyRange = FD->getLocation();1529 CurEHLocation = BodyRange.getEnd();1530 1531 // Use the location of the start of the function to determine where1532 // the function definition is located. By default use the location1533 // of the declaration as the location for the subprogram. A function1534 // may lack a declaration in the source code if it is created by code1535 // gen. (examples: _GLOBAL__I_a, __cxx_global_array_dtor, thunk).1536 SourceLocation Loc = FD->getLocation();1537 1538 // If this is a function specialization then use the pattern body1539 // as the location for the function.1540 if (const FunctionDecl *SpecDecl = FD->getTemplateInstantiationPattern())1541 if (SpecDecl->hasBody(SpecDecl))1542 Loc = SpecDecl->getLocation();1543 1544 Stmt *Body = FD->getBody();1545 1546 if (Body) {1547 // Coroutines always emit lifetime markers.1548 if (isa<CoroutineBodyStmt>(Body))1549 ShouldEmitLifetimeMarkers = true;1550 1551 // Initialize helper which will detect jumps which can cause invalid1552 // lifetime markers.1553 if (ShouldEmitLifetimeMarkers)1554 Bypasses.Init(CGM, Body);1555 }1556 1557 // Emit the standard function prologue.1558 StartFunction(GD, ResTy, Fn, FnInfo, Args, Loc, BodyRange.getBegin());1559 1560 // Save parameters for coroutine function.1561 if (Body && isa_and_nonnull<CoroutineBodyStmt>(Body))1562 llvm::append_range(FnArgs, FD->parameters());1563 1564 // Ensure that the function adheres to the forward progress guarantee, which1565 // is required by certain optimizations.1566 // In C++11 and up, the attribute will be removed if the body contains a1567 // trivial empty loop.1568 if (checkIfFunctionMustProgress())1569 CurFn->addFnAttr(llvm::Attribute::MustProgress);1570 1571 // Generate the body of the function.1572 PGO->assignRegionCounters(GD, CurFn);1573 if (isa<CXXDestructorDecl>(FD))1574 EmitDestructorBody(Args);1575 else if (isa<CXXConstructorDecl>(FD))1576 EmitConstructorBody(Args);1577 else if (getLangOpts().CUDA &&1578 !getLangOpts().CUDAIsDevice &&1579 FD->hasAttr<CUDAGlobalAttr>())1580 CGM.getCUDARuntime().emitDeviceStub(*this, Args);1581 else if (isa<CXXMethodDecl>(FD) &&1582 cast<CXXMethodDecl>(FD)->isLambdaStaticInvoker()) {1583 // The lambda static invoker function is special, because it forwards or1584 // clones the body of the function call operator (but is actually static).1585 EmitLambdaStaticInvokeBody(cast<CXXMethodDecl>(FD));1586 } else if (isa<CXXMethodDecl>(FD) &&1587 isLambdaCallOperator(cast<CXXMethodDecl>(FD)) &&1588 !FnInfo.isDelegateCall() &&1589 cast<CXXMethodDecl>(FD)->getParent()->getLambdaStaticInvoker() &&1590 hasInAllocaArg(cast<CXXMethodDecl>(FD))) {1591 // If emitting a lambda with static invoker on X86 Windows, change1592 // the call operator body.1593 // Make sure that this is a call operator with an inalloca arg and check1594 // for delegate call to make sure this is the original call op and not the1595 // new forwarding function for the static invoker.1596 EmitLambdaInAllocaCallOpBody(cast<CXXMethodDecl>(FD));1597 } else if (FD->isDefaulted() && isa<CXXMethodDecl>(FD) &&1598 (cast<CXXMethodDecl>(FD)->isCopyAssignmentOperator() ||1599 cast<CXXMethodDecl>(FD)->isMoveAssignmentOperator())) {1600 // Implicit copy-assignment gets the same special treatment as implicit1601 // copy-constructors.1602 emitImplicitAssignmentOperatorBody(Args);1603 } else if (DeviceKernelAttr::isOpenCLSpelling(1604 FD->getAttr<DeviceKernelAttr>()) &&1605 GD.getKernelReferenceKind() == KernelReferenceKind::Kernel) {1606 CallArgList CallArgs;1607 for (unsigned i = 0; i < Args.size(); ++i) {1608 Address ArgAddr = GetAddrOfLocalVar(Args[i]);1609 QualType ArgQualType = Args[i]->getType();1610 RValue ArgRValue = convertTempToRValue(ArgAddr, ArgQualType, Loc);1611 CallArgs.add(ArgRValue, ArgQualType);1612 }1613 GlobalDecl GDStub = GlobalDecl(FD, KernelReferenceKind::Stub);1614 const FunctionType *FT = cast<FunctionType>(FD->getType());1615 CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FT);1616 const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall(1617 CallArgs, FT, /*ChainCall=*/false);1618 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FnInfo);1619 llvm::Constant *GDStubFunctionPointer =1620 CGM.getRawFunctionPointer(GDStub, FTy);1621 CGCallee GDStubCallee = CGCallee::forDirect(GDStubFunctionPointer, GDStub);1622 EmitCall(FnInfo, GDStubCallee, ReturnValueSlot(), CallArgs, nullptr, false,1623 Loc);1624 } else if (Body) {1625 EmitFunctionBody(Body);1626 } else1627 llvm_unreachable("no definition for emitted function");1628 1629 // C++11 [stmt.return]p2:1630 // Flowing off the end of a function [...] results in undefined behavior in1631 // a value-returning function.1632 // C11 6.9.1p12:1633 // If the '}' that terminates a function is reached, and the value of the1634 // function call is used by the caller, the behavior is undefined.1635 if (getLangOpts().CPlusPlus && !FD->hasImplicitReturnZero() && !SawAsmBlock &&1636 !FD->getReturnType()->isVoidType() && Builder.GetInsertBlock()) {1637 bool ShouldEmitUnreachable =1638 CGM.getCodeGenOpts().StrictReturn ||1639 !CGM.MayDropFunctionReturn(FD->getASTContext(), FD->getReturnType());1640 if (SanOpts.has(SanitizerKind::Return)) {1641 auto CheckOrdinal = SanitizerKind::SO_Return;1642 auto CheckHandler = SanitizerHandler::MissingReturn;1643 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);1644 llvm::Value *IsFalse = Builder.getFalse();1645 EmitCheck(std::make_pair(IsFalse, CheckOrdinal), CheckHandler,1646 EmitCheckSourceLocation(FD->getLocation()), {});1647 } else if (ShouldEmitUnreachable) {1648 if (CGM.getCodeGenOpts().OptimizationLevel == 0)1649 EmitTrapCall(llvm::Intrinsic::trap);1650 }1651 if (SanOpts.has(SanitizerKind::Return) || ShouldEmitUnreachable) {1652 Builder.CreateUnreachable();1653 Builder.ClearInsertionPoint();1654 }1655 }1656 1657 // Emit the standard function epilogue.1658 FinishFunction(BodyRange.getEnd());1659 1660 PGO->verifyCounterMap();1661 1662 // If we haven't marked the function nothrow through other means, do1663 // a quick pass now to see if we can.1664 if (!CurFn->doesNotThrow())1665 TryMarkNoThrow(CurFn);1666}1667 1668/// ContainsLabel - Return true if the statement contains a label in it. If1669/// this statement is not executed normally, it not containing a label means1670/// that we can just remove the code.1671bool CodeGenFunction::ContainsLabel(const Stmt *S, bool IgnoreCaseStmts) {1672 // Null statement, not a label!1673 if (!S) return false;1674 1675 // If this is a label, we have to emit the code, consider something like:1676 // if (0) { ... foo: bar(); } goto foo;1677 //1678 // TODO: If anyone cared, we could track __label__'s, since we know that you1679 // can't jump to one from outside their declared region.1680 if (isa<LabelStmt>(S))1681 return true;1682 1683 // If this is a case/default statement, and we haven't seen a switch, we have1684 // to emit the code.1685 if (isa<SwitchCase>(S) && !IgnoreCaseStmts)1686 return true;1687 1688 // If this is a switch statement, we want to ignore cases below it.1689 if (isa<SwitchStmt>(S))1690 IgnoreCaseStmts = true;1691 1692 // Scan subexpressions for verboten labels.1693 for (const Stmt *SubStmt : S->children())1694 if (ContainsLabel(SubStmt, IgnoreCaseStmts))1695 return true;1696 1697 return false;1698}1699 1700/// containsBreak - Return true if the statement contains a break out of it.1701/// If the statement (recursively) contains a switch or loop with a break1702/// inside of it, this is fine.1703bool CodeGenFunction::containsBreak(const Stmt *S) {1704 // Null statement, not a label!1705 if (!S) return false;1706 1707 // If this is a switch or loop that defines its own break scope, then we can1708 // include it and anything inside of it.1709 if (isa<SwitchStmt>(S) || isa<WhileStmt>(S) || isa<DoStmt>(S) ||1710 isa<ForStmt>(S))1711 return false;1712 1713 if (isa<BreakStmt>(S))1714 return true;1715 1716 // Scan subexpressions for verboten breaks.1717 for (const Stmt *SubStmt : S->children())1718 if (containsBreak(SubStmt))1719 return true;1720 1721 return false;1722}1723 1724bool CodeGenFunction::mightAddDeclToScope(const Stmt *S) {1725 if (!S) return false;1726 1727 // Some statement kinds add a scope and thus never add a decl to the current1728 // scope. Note, this list is longer than the list of statements that might1729 // have an unscoped decl nested within them, but this way is conservatively1730 // correct even if more statement kinds are added.1731 if (isa<IfStmt>(S) || isa<SwitchStmt>(S) || isa<WhileStmt>(S) ||1732 isa<DoStmt>(S) || isa<ForStmt>(S) || isa<CompoundStmt>(S) ||1733 isa<CXXForRangeStmt>(S) || isa<CXXTryStmt>(S) ||1734 isa<ObjCForCollectionStmt>(S) || isa<ObjCAtTryStmt>(S))1735 return false;1736 1737 if (isa<DeclStmt>(S))1738 return true;1739 1740 for (const Stmt *SubStmt : S->children())1741 if (mightAddDeclToScope(SubStmt))1742 return true;1743 1744 return false;1745}1746 1747/// ConstantFoldsToSimpleInteger - If the specified expression does not fold1748/// to a constant, or if it does but contains a label, return false. If it1749/// constant folds return true and set the boolean result in Result.1750bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,1751 bool &ResultBool,1752 bool AllowLabels) {1753 // If MC/DC is enabled, disable folding so that we can instrument all1754 // conditions to yield complete test vectors. We still keep track of1755 // folded conditions during region mapping and visualization.1756 if (!AllowLabels && CGM.getCodeGenOpts().hasProfileClangInstr() &&1757 CGM.getCodeGenOpts().MCDCCoverage)1758 return false;1759 1760 llvm::APSInt ResultInt;1761 if (!ConstantFoldsToSimpleInteger(Cond, ResultInt, AllowLabels))1762 return false;1763 1764 ResultBool = ResultInt.getBoolValue();1765 return true;1766}1767 1768/// ConstantFoldsToSimpleInteger - If the specified expression does not fold1769/// to a constant, or if it does but contains a label, return false. If it1770/// constant folds return true and set the folded value.1771bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,1772 llvm::APSInt &ResultInt,1773 bool AllowLabels) {1774 // FIXME: Rename and handle conversion of other evaluatable things1775 // to bool.1776 Expr::EvalResult Result;1777 if (!Cond->EvaluateAsInt(Result, getContext()))1778 return false; // Not foldable, not integer or not fully evaluatable.1779 1780 llvm::APSInt Int = Result.Val.getInt();1781 if (!AllowLabels && CodeGenFunction::ContainsLabel(Cond))1782 return false; // Contains a label.1783 1784 PGO->markStmtMaybeUsed(Cond);1785 ResultInt = Int;1786 return true;1787}1788 1789/// Strip parentheses and simplistic logical-NOT operators.1790const Expr *CodeGenFunction::stripCond(const Expr *C) {1791 while (const UnaryOperator *Op = dyn_cast<UnaryOperator>(C->IgnoreParens())) {1792 if (Op->getOpcode() != UO_LNot)1793 break;1794 C = Op->getSubExpr();1795 }1796 return C->IgnoreParens();1797}1798 1799/// Determine whether the given condition is an instrumentable condition1800/// (i.e. no "&&" or "||").1801bool CodeGenFunction::isInstrumentedCondition(const Expr *C) {1802 const BinaryOperator *BOp = dyn_cast<BinaryOperator>(stripCond(C));1803 return (!BOp || !BOp->isLogicalOp());1804}1805 1806/// EmitBranchToCounterBlock - Emit a conditional branch to a new block that1807/// increments a profile counter based on the semantics of the given logical1808/// operator opcode. This is used to instrument branch condition coverage for1809/// logical operators.1810void CodeGenFunction::EmitBranchToCounterBlock(1811 const Expr *Cond, BinaryOperator::Opcode LOp, llvm::BasicBlock *TrueBlock,1812 llvm::BasicBlock *FalseBlock, uint64_t TrueCount /* = 0 */,1813 Stmt::Likelihood LH /* =None */, const Expr *CntrIdx /* = nullptr */) {1814 // If not instrumenting, just emit a branch.1815 bool InstrumentRegions = CGM.getCodeGenOpts().hasProfileClangInstr();1816 if (!InstrumentRegions || !isInstrumentedCondition(Cond))1817 return EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount, LH);1818 1819 const Stmt *CntrStmt = (CntrIdx ? CntrIdx : Cond);1820 1821 llvm::BasicBlock *ThenBlock = nullptr;1822 llvm::BasicBlock *ElseBlock = nullptr;1823 llvm::BasicBlock *NextBlock = nullptr;1824 1825 // Create the block we'll use to increment the appropriate counter.1826 llvm::BasicBlock *CounterIncrBlock = createBasicBlock("lop.rhscnt");1827 1828 // Set block pointers according to Logical-AND (BO_LAnd) semantics. This1829 // means we need to evaluate the condition and increment the counter on TRUE:1830 //1831 // if (Cond)1832 // goto CounterIncrBlock;1833 // else1834 // goto FalseBlock;1835 //1836 // CounterIncrBlock:1837 // Counter++;1838 // goto TrueBlock;1839 1840 if (LOp == BO_LAnd) {1841 ThenBlock = CounterIncrBlock;1842 ElseBlock = FalseBlock;1843 NextBlock = TrueBlock;1844 }1845 1846 // Set block pointers according to Logical-OR (BO_LOr) semantics. This means1847 // we need to evaluate the condition and increment the counter on FALSE:1848 //1849 // if (Cond)1850 // goto TrueBlock;1851 // else1852 // goto CounterIncrBlock;1853 //1854 // CounterIncrBlock:1855 // Counter++;1856 // goto FalseBlock;1857 1858 else if (LOp == BO_LOr) {1859 ThenBlock = TrueBlock;1860 ElseBlock = CounterIncrBlock;1861 NextBlock = FalseBlock;1862 } else {1863 llvm_unreachable("Expected Opcode must be that of a Logical Operator");1864 }1865 1866 // Emit Branch based on condition.1867 EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, TrueCount, LH);1868 1869 // Emit the block containing the counter increment(s).1870 EmitBlock(CounterIncrBlock);1871 1872 // Increment corresponding counter; if index not provided, use Cond as index.1873 incrementProfileCounter(CntrStmt);1874 1875 // Go to the next block.1876 EmitBranch(NextBlock);1877}1878 1879/// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an if1880/// statement) to the specified blocks. Based on the condition, this might try1881/// to simplify the codegen of the conditional based on the branch.1882/// \param LH The value of the likelihood attribute on the True branch.1883/// \param ConditionalOp Used by MC/DC code coverage to track the result of the1884/// ConditionalOperator (ternary) through a recursive call for the operator's1885/// LHS and RHS nodes.1886void CodeGenFunction::EmitBranchOnBoolExpr(1887 const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock,1888 uint64_t TrueCount, Stmt::Likelihood LH, const Expr *ConditionalOp,1889 const VarDecl *ConditionalDecl) {1890 Cond = Cond->IgnoreParens();1891 1892 if (const BinaryOperator *CondBOp = dyn_cast<BinaryOperator>(Cond)) {1893 // Handle X && Y in a condition.1894 if (CondBOp->getOpcode() == BO_LAnd) {1895 MCDCLogOpStack.push_back(CondBOp);1896 1897 // If we have "1 && X", simplify the code. "0 && X" would have constant1898 // folded if the case was simple enough.1899 bool ConstantBool = false;1900 if (ConstantFoldsToSimpleInteger(CondBOp->getLHS(), ConstantBool) &&1901 ConstantBool) {1902 // br(1 && X) -> br(X).1903 incrementProfileCounter(CondBOp);1904 EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LAnd, TrueBlock,1905 FalseBlock, TrueCount, LH);1906 MCDCLogOpStack.pop_back();1907 return;1908 }1909 1910 // If we have "X && 1", simplify the code to use an uncond branch.1911 // "X && 0" would have been constant folded to 0.1912 if (ConstantFoldsToSimpleInteger(CondBOp->getRHS(), ConstantBool) &&1913 ConstantBool) {1914 // br(X && 1) -> br(X).1915 EmitBranchToCounterBlock(CondBOp->getLHS(), BO_LAnd, TrueBlock,1916 FalseBlock, TrueCount, LH, CondBOp);1917 MCDCLogOpStack.pop_back();1918 return;1919 }1920 1921 // Emit the LHS as a conditional. If the LHS conditional is false, we1922 // want to jump to the FalseBlock.1923 llvm::BasicBlock *LHSTrue = createBasicBlock("land.lhs.true");1924 // The counter tells us how often we evaluate RHS, and all of TrueCount1925 // can be propagated to that branch.1926 uint64_t RHSCount = getProfileCount(CondBOp->getRHS());1927 1928 ConditionalEvaluation eval(*this);1929 {1930 ApplyDebugLocation DL(*this, Cond);1931 // Propagate the likelihood attribute like __builtin_expect1932 // __builtin_expect(X && Y, 1) -> X and Y are likely1933 // __builtin_expect(X && Y, 0) -> only Y is unlikely1934 EmitBranchOnBoolExpr(CondBOp->getLHS(), LHSTrue, FalseBlock, RHSCount,1935 LH == Stmt::LH_Unlikely ? Stmt::LH_None : LH);1936 EmitBlock(LHSTrue);1937 }1938 1939 incrementProfileCounter(CondBOp);1940 setCurrentProfileCount(getProfileCount(CondBOp->getRHS()));1941 1942 // Any temporaries created here are conditional.1943 eval.begin(*this);1944 EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LAnd, TrueBlock,1945 FalseBlock, TrueCount, LH);1946 eval.end(*this);1947 MCDCLogOpStack.pop_back();1948 return;1949 }1950 1951 if (CondBOp->getOpcode() == BO_LOr) {1952 MCDCLogOpStack.push_back(CondBOp);1953 1954 // If we have "0 || X", simplify the code. "1 || X" would have constant1955 // folded if the case was simple enough.1956 bool ConstantBool = false;1957 if (ConstantFoldsToSimpleInteger(CondBOp->getLHS(), ConstantBool) &&1958 !ConstantBool) {1959 // br(0 || X) -> br(X).1960 incrementProfileCounter(CondBOp);1961 EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LOr, TrueBlock,1962 FalseBlock, TrueCount, LH);1963 MCDCLogOpStack.pop_back();1964 return;1965 }1966 1967 // If we have "X || 0", simplify the code to use an uncond branch.1968 // "X || 1" would have been constant folded to 1.1969 if (ConstantFoldsToSimpleInteger(CondBOp->getRHS(), ConstantBool) &&1970 !ConstantBool) {1971 // br(X || 0) -> br(X).1972 EmitBranchToCounterBlock(CondBOp->getLHS(), BO_LOr, TrueBlock,1973 FalseBlock, TrueCount, LH, CondBOp);1974 MCDCLogOpStack.pop_back();1975 return;1976 }1977 // Emit the LHS as a conditional. If the LHS conditional is true, we1978 // want to jump to the TrueBlock.1979 llvm::BasicBlock *LHSFalse = createBasicBlock("lor.lhs.false");1980 // We have the count for entry to the RHS and for the whole expression1981 // being true, so we can divy up True count between the short circuit and1982 // the RHS.1983 uint64_t LHSCount =1984 getCurrentProfileCount() - getProfileCount(CondBOp->getRHS());1985 uint64_t RHSCount = TrueCount - LHSCount;1986 1987 ConditionalEvaluation eval(*this);1988 {1989 // Propagate the likelihood attribute like __builtin_expect1990 // __builtin_expect(X || Y, 1) -> only Y is likely1991 // __builtin_expect(X || Y, 0) -> both X and Y are unlikely1992 ApplyDebugLocation DL(*this, Cond);1993 EmitBranchOnBoolExpr(CondBOp->getLHS(), TrueBlock, LHSFalse, LHSCount,1994 LH == Stmt::LH_Likely ? Stmt::LH_None : LH);1995 EmitBlock(LHSFalse);1996 }1997 1998 incrementProfileCounter(CondBOp);1999 setCurrentProfileCount(getProfileCount(CondBOp->getRHS()));2000 2001 // Any temporaries created here are conditional.2002 eval.begin(*this);2003 EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LOr, TrueBlock, FalseBlock,2004 RHSCount, LH);2005 2006 eval.end(*this);2007 MCDCLogOpStack.pop_back();2008 return;2009 }2010 }2011 2012 if (const UnaryOperator *CondUOp = dyn_cast<UnaryOperator>(Cond)) {2013 // br(!x, t, f) -> br(x, f, t)2014 // Avoid doing this optimization when instrumenting a condition for MC/DC.2015 // LNot is taken as part of the condition for simplicity, and changing its2016 // sense negatively impacts test vector tracking.2017 bool MCDCCondition = CGM.getCodeGenOpts().hasProfileClangInstr() &&2018 CGM.getCodeGenOpts().MCDCCoverage &&2019 isInstrumentedCondition(Cond);2020 if (CondUOp->getOpcode() == UO_LNot && !MCDCCondition) {2021 // Negate the count.2022 uint64_t FalseCount = getCurrentProfileCount() - TrueCount;2023 // The values of the enum are chosen to make this negation possible.2024 LH = static_cast<Stmt::Likelihood>(-LH);2025 // Negate the condition and swap the destination blocks.2026 return EmitBranchOnBoolExpr(CondUOp->getSubExpr(), FalseBlock, TrueBlock,2027 FalseCount, LH);2028 }2029 }2030 2031 if (const ConditionalOperator *CondOp = dyn_cast<ConditionalOperator>(Cond)) {2032 // br(c ? x : y, t, f) -> br(c, br(x, t, f), br(y, t, f))2033 llvm::BasicBlock *LHSBlock = createBasicBlock("cond.true");2034 llvm::BasicBlock *RHSBlock = createBasicBlock("cond.false");2035 2036 // The ConditionalOperator itself has no likelihood information for its2037 // true and false branches. This matches the behavior of __builtin_expect.2038 ConditionalEvaluation cond(*this);2039 EmitBranchOnBoolExpr(CondOp->getCond(), LHSBlock, RHSBlock,2040 getProfileCount(CondOp), Stmt::LH_None);2041 2042 // When computing PGO branch weights, we only know the overall count for2043 // the true block. This code is essentially doing tail duplication of the2044 // naive code-gen, introducing new edges for which counts are not2045 // available. Divide the counts proportionally between the LHS and RHS of2046 // the conditional operator.2047 uint64_t LHSScaledTrueCount = 0;2048 if (TrueCount) {2049 double LHSRatio =2050 getProfileCount(CondOp) / (double)getCurrentProfileCount();2051 LHSScaledTrueCount = TrueCount * LHSRatio;2052 }2053 2054 cond.begin(*this);2055 EmitBlock(LHSBlock);2056 incrementProfileCounter(CondOp);2057 {2058 ApplyDebugLocation DL(*this, Cond);2059 EmitBranchOnBoolExpr(CondOp->getLHS(), TrueBlock, FalseBlock,2060 LHSScaledTrueCount, LH, CondOp);2061 }2062 cond.end(*this);2063 2064 cond.begin(*this);2065 EmitBlock(RHSBlock);2066 EmitBranchOnBoolExpr(CondOp->getRHS(), TrueBlock, FalseBlock,2067 TrueCount - LHSScaledTrueCount, LH, CondOp);2068 cond.end(*this);2069 2070 return;2071 }2072 2073 if (const CXXThrowExpr *Throw = dyn_cast<CXXThrowExpr>(Cond)) {2074 // Conditional operator handling can give us a throw expression as a2075 // condition for a case like:2076 // br(c ? throw x : y, t, f) -> br(c, br(throw x, t, f), br(y, t, f)2077 // Fold this to:2078 // br(c, throw x, br(y, t, f))2079 EmitCXXThrowExpr(Throw, /*KeepInsertionPoint*/false);2080 return;2081 }2082 2083 // Emit the code with the fully general case.2084 llvm::Value *CondV;2085 {2086 ApplyDebugLocation DL(*this, Cond);2087 CondV = EvaluateExprAsBool(Cond);2088 }2089 2090 MaybeEmitDeferredVarDeclInit(ConditionalDecl);2091 2092 // If not at the top of the logical operator nest, update MCDC temp with the2093 // boolean result of the evaluated condition.2094 if (!MCDCLogOpStack.empty()) {2095 const Expr *MCDCBaseExpr = Cond;2096 // When a nested ConditionalOperator (ternary) is encountered in a boolean2097 // expression, MC/DC tracks the result of the ternary, and this is tied to2098 // the ConditionalOperator expression and not the ternary's LHS or RHS. If2099 // this is the case, the ConditionalOperator expression is passed through2100 // the ConditionalOp parameter and then used as the MCDC base expression.2101 if (ConditionalOp)2102 MCDCBaseExpr = ConditionalOp;2103 2104 maybeUpdateMCDCCondBitmap(MCDCBaseExpr, CondV);2105 }2106 2107 llvm::MDNode *Weights = nullptr;2108 llvm::MDNode *Unpredictable = nullptr;2109 2110 // If the branch has a condition wrapped by __builtin_unpredictable,2111 // create metadata that specifies that the branch is unpredictable.2112 // Don't bother if not optimizing because that metadata would not be used.2113 auto *Call = dyn_cast<CallExpr>(Cond->IgnoreImpCasts());2114 if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {2115 auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl());2116 if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {2117 llvm::MDBuilder MDHelper(getLLVMContext());2118 Unpredictable = MDHelper.createUnpredictable();2119 }2120 }2121 2122 // If there is a Likelihood knowledge for the cond, lower it.2123 // Note that if not optimizing this won't emit anything.2124 llvm::Value *NewCondV = emitCondLikelihoodViaExpectIntrinsic(CondV, LH);2125 if (CondV != NewCondV)2126 CondV = NewCondV;2127 else {2128 // Otherwise, lower profile counts. Note that we do this even at -O0.2129 uint64_t CurrentCount = std::max(getCurrentProfileCount(), TrueCount);2130 Weights = createProfileWeights(TrueCount, CurrentCount - TrueCount);2131 }2132 2133 llvm::Instruction *BrInst = Builder.CreateCondBr(CondV, TrueBlock, FalseBlock,2134 Weights, Unpredictable);2135 addInstToNewSourceAtom(BrInst, CondV);2136 2137 switch (HLSLControlFlowAttr) {2138 case HLSLControlFlowHintAttr::Microsoft_branch:2139 case HLSLControlFlowHintAttr::Microsoft_flatten: {2140 llvm::MDBuilder MDHelper(CGM.getLLVMContext());2141 2142 llvm::ConstantInt *BranchHintConstant =2143 HLSLControlFlowAttr ==2144 HLSLControlFlowHintAttr::Spelling::Microsoft_branch2145 ? llvm::ConstantInt::get(CGM.Int32Ty, 1)2146 : llvm::ConstantInt::get(CGM.Int32Ty, 2);2147 2148 SmallVector<llvm::Metadata *, 2> Vals(2149 {MDHelper.createString("hlsl.controlflow.hint"),2150 MDHelper.createConstant(BranchHintConstant)});2151 BrInst->setMetadata("hlsl.controlflow.hint",2152 llvm::MDNode::get(CGM.getLLVMContext(), Vals));2153 break;2154 }2155 // This is required to avoid warnings during compilation2156 case HLSLControlFlowHintAttr::SpellingNotCalculated:2157 break;2158 }2159}2160 2161/// ErrorUnsupported - Print out an error that codegen doesn't support the2162/// specified stmt yet.2163void CodeGenFunction::ErrorUnsupported(const Stmt *S, const char *Type) {2164 CGM.ErrorUnsupported(S, Type);2165}2166 2167/// emitNonZeroVLAInit - Emit the "zero" initialization of a2168/// variable-length array whose elements have a non-zero bit-pattern.2169///2170/// \param baseType the inner-most element type of the array2171/// \param src - a char* pointing to the bit-pattern for a single2172/// base element of the array2173/// \param sizeInChars - the total size of the VLA, in chars2174static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,2175 Address dest, Address src,2176 llvm::Value *sizeInChars) {2177 CGBuilderTy &Builder = CGF.Builder;2178 2179 CharUnits baseSize = CGF.getContext().getTypeSizeInChars(baseType);2180 llvm::Value *baseSizeInChars2181 = llvm::ConstantInt::get(CGF.IntPtrTy, baseSize.getQuantity());2182 2183 Address begin = dest.withElementType(CGF.Int8Ty);2184 llvm::Value *end = Builder.CreateInBoundsGEP(begin.getElementType(),2185 begin.emitRawPointer(CGF),2186 sizeInChars, "vla.end");2187 2188 llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock();2189 llvm::BasicBlock *loopBB = CGF.createBasicBlock("vla-init.loop");2190 llvm::BasicBlock *contBB = CGF.createBasicBlock("vla-init.cont");2191 2192 // Make a loop over the VLA. C99 guarantees that the VLA element2193 // count must be nonzero.2194 CGF.EmitBlock(loopBB);2195 2196 llvm::PHINode *cur = Builder.CreatePHI(begin.getType(), 2, "vla.cur");2197 cur->addIncoming(begin.emitRawPointer(CGF), originBB);2198 2199 CharUnits curAlign =2200 dest.getAlignment().alignmentOfArrayElement(baseSize);2201 2202 // memcpy the individual element bit-pattern.2203 Builder.CreateMemCpy(Address(cur, CGF.Int8Ty, curAlign), src, baseSizeInChars,2204 /*volatile*/ false);2205 2206 // Go to the next element.2207 llvm::Value *next =2208 Builder.CreateInBoundsGEP(CGF.Int8Ty, cur, baseSizeInChars, "vla.next");2209 2210 // Leave if that's the end of the VLA.2211 llvm::Value *done = Builder.CreateICmpEQ(next, end, "vla-init.isdone");2212 Builder.CreateCondBr(done, contBB, loopBB);2213 cur->addIncoming(next, loopBB);2214 2215 CGF.EmitBlock(contBB);2216}2217 2218void2219CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) {2220 // Ignore empty classes in C++.2221 if (getLangOpts().CPlusPlus)2222 if (const auto *RD = Ty->getAsCXXRecordDecl(); RD && RD->isEmpty())2223 return;2224 2225 if (DestPtr.getElementType() != Int8Ty)2226 DestPtr = DestPtr.withElementType(Int8Ty);2227 2228 // Get size and alignment info for this aggregate.2229 CharUnits size = getContext().getTypeSizeInChars(Ty);2230 2231 llvm::Value *SizeVal;2232 const VariableArrayType *vla;2233 2234 // Don't bother emitting a zero-byte memset.2235 if (size.isZero()) {2236 // But note that getTypeInfo returns 0 for a VLA.2237 if (const VariableArrayType *vlaType =2238 dyn_cast_or_null<VariableArrayType>(2239 getContext().getAsArrayType(Ty))) {2240 auto VlaSize = getVLASize(vlaType);2241 SizeVal = VlaSize.NumElts;2242 CharUnits eltSize = getContext().getTypeSizeInChars(VlaSize.Type);2243 if (!eltSize.isOne())2244 SizeVal = Builder.CreateNUWMul(SizeVal, CGM.getSize(eltSize));2245 vla = vlaType;2246 } else {2247 return;2248 }2249 } else {2250 SizeVal = CGM.getSize(size);2251 vla = nullptr;2252 }2253 2254 // If the type contains a pointer to data member we can't memset it to zero.2255 // Instead, create a null constant and copy it to the destination.2256 // TODO: there are other patterns besides zero that we can usefully memset,2257 // like -1, which happens to be the pattern used by member-pointers.2258 if (!CGM.getTypes().isZeroInitializable(Ty)) {2259 // For a VLA, emit a single element, then splat that over the VLA.2260 if (vla) Ty = getContext().getBaseElementType(vla);2261 2262 llvm::Constant *NullConstant = CGM.EmitNullConstant(Ty);2263 2264 llvm::GlobalVariable *NullVariable =2265 new llvm::GlobalVariable(CGM.getModule(), NullConstant->getType(),2266 /*isConstant=*/true,2267 llvm::GlobalVariable::PrivateLinkage,2268 NullConstant, Twine());2269 CharUnits NullAlign = DestPtr.getAlignment();2270 NullVariable->setAlignment(NullAlign.getAsAlign());2271 Address SrcPtr(NullVariable, Builder.getInt8Ty(), NullAlign);2272 2273 if (vla) return emitNonZeroVLAInit(*this, Ty, DestPtr, SrcPtr, SizeVal);2274 2275 // Get and call the appropriate llvm.memcpy overload.2276 Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, false);2277 return;2278 }2279 2280 // Otherwise, just memset the whole thing to zero. This is legal2281 // because in LLVM, all default initializers (other than the ones we just2282 // handled above) are guaranteed to have a bit pattern of all zeros.2283 Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal, false);2284}2285 2286llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelDecl *L) {2287 // Make sure that there is a block for the indirect goto.2288 if (!IndirectBranch)2289 GetIndirectGotoBlock();2290 2291 llvm::BasicBlock *BB = getJumpDestForLabel(L).getBlock();2292 2293 // Make sure the indirect branch includes all of the address-taken blocks.2294 IndirectBranch->addDestination(BB);2295 return llvm::BlockAddress::get(CurFn->getType(), BB);2296}2297 2298llvm::BasicBlock *CodeGenFunction::GetIndirectGotoBlock() {2299 // If we already made the indirect branch for indirect goto, return its block.2300 if (IndirectBranch) return IndirectBranch->getParent();2301 2302 CGBuilderTy TmpBuilder(*this, createBasicBlock("indirectgoto"));2303 2304 // Create the PHI node that indirect gotos will add entries to.2305 llvm::Value *DestVal = TmpBuilder.CreatePHI(Int8PtrTy, 0,2306 "indirect.goto.dest");2307 2308 // Create the indirect branch instruction.2309 IndirectBranch = TmpBuilder.CreateIndirectBr(DestVal);2310 return IndirectBranch->getParent();2311}2312 2313/// Computes the length of an array in elements, as well as the base2314/// element type and a properly-typed first element pointer.2315llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,2316 QualType &baseType,2317 Address &addr) {2318 const ArrayType *arrayType = origArrayType;2319 2320 // If it's a VLA, we have to load the stored size. Note that2321 // this is the size of the VLA in bytes, not its size in elements.2322 llvm::Value *numVLAElements = nullptr;2323 if (isa<VariableArrayType>(arrayType)) {2324 numVLAElements = getVLASize(cast<VariableArrayType>(arrayType)).NumElts;2325 2326 // Walk into all VLAs. This doesn't require changes to addr,2327 // which has type T* where T is the first non-VLA element type.2328 do {2329 QualType elementType = arrayType->getElementType();2330 arrayType = getContext().getAsArrayType(elementType);2331 2332 // If we only have VLA components, 'addr' requires no adjustment.2333 if (!arrayType) {2334 baseType = elementType;2335 return numVLAElements;2336 }2337 } while (isa<VariableArrayType>(arrayType));2338 2339 // We get out here only if we find a constant array type2340 // inside the VLA.2341 }2342 2343 // We have some number of constant-length arrays, so addr should2344 // have LLVM type [M x [N x [...]]]*. Build a GEP that walks2345 // down to the first element of addr.2346 SmallVector<llvm::Value*, 8> gepIndices;2347 2348 // GEP down to the array type.2349 llvm::ConstantInt *zero = Builder.getInt32(0);2350 gepIndices.push_back(zero);2351 2352 uint64_t countFromCLAs = 1;2353 QualType eltType;2354 2355 llvm::ArrayType *llvmArrayType =2356 dyn_cast<llvm::ArrayType>(addr.getElementType());2357 while (llvmArrayType) {2358 assert(isa<ConstantArrayType>(arrayType));2359 assert(cast<ConstantArrayType>(arrayType)->getZExtSize() ==2360 llvmArrayType->getNumElements());2361 2362 gepIndices.push_back(zero);2363 countFromCLAs *= llvmArrayType->getNumElements();2364 eltType = arrayType->getElementType();2365 2366 llvmArrayType =2367 dyn_cast<llvm::ArrayType>(llvmArrayType->getElementType());2368 arrayType = getContext().getAsArrayType(arrayType->getElementType());2369 assert((!llvmArrayType || arrayType) &&2370 "LLVM and Clang types are out-of-synch");2371 }2372 2373 if (arrayType) {2374 // From this point onwards, the Clang array type has been emitted2375 // as some other type (probably a packed struct). Compute the array2376 // size, and just emit the 'begin' expression as a bitcast.2377 while (arrayType) {2378 countFromCLAs *= cast<ConstantArrayType>(arrayType)->getZExtSize();2379 eltType = arrayType->getElementType();2380 arrayType = getContext().getAsArrayType(eltType);2381 }2382 2383 llvm::Type *baseType = ConvertType(eltType);2384 addr = addr.withElementType(baseType);2385 } else {2386 // Create the actual GEP.2387 addr = Address(Builder.CreateInBoundsGEP(addr.getElementType(),2388 addr.emitRawPointer(*this),2389 gepIndices, "array.begin"),2390 ConvertTypeForMem(eltType), addr.getAlignment());2391 }2392 2393 baseType = eltType;2394 2395 llvm::Value *numElements2396 = llvm::ConstantInt::get(SizeTy, countFromCLAs);2397 2398 // If we had any VLA dimensions, factor them in.2399 if (numVLAElements)2400 numElements = Builder.CreateNUWMul(numVLAElements, numElements);2401 2402 return numElements;2403}2404 2405CodeGenFunction::VlaSizePair CodeGenFunction::getVLASize(QualType type) {2406 const VariableArrayType *vla = getContext().getAsVariableArrayType(type);2407 assert(vla && "type was not a variable array type!");2408 return getVLASize(vla);2409}2410 2411CodeGenFunction::VlaSizePair2412CodeGenFunction::getVLASize(const VariableArrayType *type) {2413 // The number of elements so far; always size_t.2414 llvm::Value *numElements = nullptr;2415 2416 QualType elementType;2417 do {2418 elementType = type->getElementType();2419 llvm::Value *vlaSize = VLASizeMap[type->getSizeExpr()];2420 assert(vlaSize && "no size for VLA!");2421 assert(vlaSize->getType() == SizeTy);2422 2423 if (!numElements) {2424 numElements = vlaSize;2425 } else {2426 // It's undefined behavior if this wraps around, so mark it that way.2427 // FIXME: Teach -fsanitize=undefined to trap this.2428 numElements = Builder.CreateNUWMul(numElements, vlaSize);2429 }2430 } while ((type = getContext().getAsVariableArrayType(elementType)));2431 2432 return { numElements, elementType };2433}2434 2435CodeGenFunction::VlaSizePair2436CodeGenFunction::getVLAElements1D(QualType type) {2437 const VariableArrayType *vla = getContext().getAsVariableArrayType(type);2438 assert(vla && "type was not a variable array type!");2439 return getVLAElements1D(vla);2440}2441 2442CodeGenFunction::VlaSizePair2443CodeGenFunction::getVLAElements1D(const VariableArrayType *Vla) {2444 llvm::Value *VlaSize = VLASizeMap[Vla->getSizeExpr()];2445 assert(VlaSize && "no size for VLA!");2446 assert(VlaSize->getType() == SizeTy);2447 return { VlaSize, Vla->getElementType() };2448}2449 2450void CodeGenFunction::EmitVariablyModifiedType(QualType type) {2451 assert(type->isVariablyModifiedType() &&2452 "Must pass variably modified type to EmitVLASizes!");2453 2454 EnsureInsertPoint();2455 2456 // We're going to walk down into the type and look for VLA2457 // expressions.2458 do {2459 assert(type->isVariablyModifiedType());2460 2461 const Type *ty = type.getTypePtr();2462 switch (ty->getTypeClass()) {2463 2464#define TYPE(Class, Base)2465#define ABSTRACT_TYPE(Class, Base)2466#define NON_CANONICAL_TYPE(Class, Base)2467#define DEPENDENT_TYPE(Class, Base) case Type::Class:2468#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base)2469#include "clang/AST/TypeNodes.inc"2470 llvm_unreachable("unexpected dependent type!");2471 2472 // These types are never variably-modified.2473 case Type::Builtin:2474 case Type::Complex:2475 case Type::Vector:2476 case Type::ExtVector:2477 case Type::ConstantMatrix:2478 case Type::Record:2479 case Type::Enum:2480 case Type::Using:2481 case Type::TemplateSpecialization:2482 case Type::ObjCTypeParam:2483 case Type::ObjCObject:2484 case Type::ObjCInterface:2485 case Type::ObjCObjectPointer:2486 case Type::BitInt:2487 case Type::HLSLInlineSpirv:2488 case Type::PredefinedSugar:2489 llvm_unreachable("type class is never variably-modified!");2490 2491 case Type::Adjusted:2492 type = cast<AdjustedType>(ty)->getAdjustedType();2493 break;2494 2495 case Type::Decayed:2496 type = cast<DecayedType>(ty)->getPointeeType();2497 break;2498 2499 case Type::Pointer:2500 type = cast<PointerType>(ty)->getPointeeType();2501 break;2502 2503 case Type::BlockPointer:2504 type = cast<BlockPointerType>(ty)->getPointeeType();2505 break;2506 2507 case Type::LValueReference:2508 case Type::RValueReference:2509 type = cast<ReferenceType>(ty)->getPointeeType();2510 break;2511 2512 case Type::MemberPointer:2513 type = cast<MemberPointerType>(ty)->getPointeeType();2514 break;2515 2516 case Type::ArrayParameter:2517 case Type::ConstantArray:2518 case Type::IncompleteArray:2519 // Losing element qualification here is fine.2520 type = cast<ArrayType>(ty)->getElementType();2521 break;2522 2523 case Type::VariableArray: {2524 // Losing element qualification here is fine.2525 const VariableArrayType *vat = cast<VariableArrayType>(ty);2526 2527 // Unknown size indication requires no size computation.2528 // Otherwise, evaluate and record it.2529 if (const Expr *sizeExpr = vat->getSizeExpr()) {2530 // It's possible that we might have emitted this already,2531 // e.g. with a typedef and a pointer to it.2532 llvm::Value *&entry = VLASizeMap[sizeExpr];2533 if (!entry) {2534 llvm::Value *size = EmitScalarExpr(sizeExpr);2535 2536 // C11 6.7.6.2p5:2537 // If the size is an expression that is not an integer constant2538 // expression [...] each time it is evaluated it shall have a value2539 // greater than zero.2540 if (SanOpts.has(SanitizerKind::VLABound)) {2541 auto CheckOrdinal = SanitizerKind::SO_VLABound;2542 auto CheckHandler = SanitizerHandler::VLABoundNotPositive;2543 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);2544 llvm::Value *Zero = llvm::Constant::getNullValue(size->getType());2545 clang::QualType SEType = sizeExpr->getType();2546 llvm::Value *CheckCondition =2547 SEType->isSignedIntegerType()2548 ? Builder.CreateICmpSGT(size, Zero)2549 : Builder.CreateICmpUGT(size, Zero);2550 llvm::Constant *StaticArgs[] = {2551 EmitCheckSourceLocation(sizeExpr->getBeginLoc()),2552 EmitCheckTypeDescriptor(SEType)};2553 EmitCheck(std::make_pair(CheckCondition, CheckOrdinal),2554 CheckHandler, StaticArgs, size);2555 }2556 2557 // Always zexting here would be wrong if it weren't2558 // undefined behavior to have a negative bound.2559 // FIXME: What about when size's type is larger than size_t?2560 entry = Builder.CreateIntCast(size, SizeTy, /*signed*/ false);2561 }2562 }2563 type = vat->getElementType();2564 break;2565 }2566 2567 case Type::FunctionProto:2568 case Type::FunctionNoProto:2569 type = cast<FunctionType>(ty)->getReturnType();2570 break;2571 2572 case Type::Paren:2573 case Type::TypeOf:2574 case Type::UnaryTransform:2575 case Type::Attributed:2576 case Type::BTFTagAttributed:2577 case Type::HLSLAttributedResource:2578 case Type::SubstTemplateTypeParm:2579 case Type::MacroQualified:2580 case Type::CountAttributed:2581 // Keep walking after single level desugaring.2582 type = type.getSingleStepDesugaredType(getContext());2583 break;2584 2585 case Type::Typedef:2586 case Type::Decltype:2587 case Type::Auto:2588 case Type::DeducedTemplateSpecialization:2589 case Type::PackIndexing:2590 // Stop walking: nothing to do.2591 return;2592 2593 case Type::TypeOfExpr:2594 // Stop walking: emit typeof expression.2595 EmitIgnoredExpr(cast<TypeOfExprType>(ty)->getUnderlyingExpr());2596 return;2597 2598 case Type::Atomic:2599 type = cast<AtomicType>(ty)->getValueType();2600 break;2601 2602 case Type::Pipe:2603 type = cast<PipeType>(ty)->getElementType();2604 break;2605 }2606 } while (type->isVariablyModifiedType());2607}2608 2609Address CodeGenFunction::EmitVAListRef(const Expr* E) {2610 if (getContext().getBuiltinVaListType()->isArrayType())2611 return EmitPointerWithAlignment(E);2612 return EmitLValue(E).getAddress();2613}2614 2615Address CodeGenFunction::EmitMSVAListRef(const Expr *E) {2616 return EmitLValue(E).getAddress();2617}2618 2619void CodeGenFunction::EmitDeclRefExprDbgValue(const DeclRefExpr *E,2620 const APValue &Init) {2621 assert(Init.hasValue() && "Invalid DeclRefExpr initializer!");2622 if (CGDebugInfo *Dbg = getDebugInfo())2623 if (CGM.getCodeGenOpts().hasReducedDebugInfo())2624 Dbg->EmitGlobalVariable(E->getDecl(), Init);2625}2626 2627CodeGenFunction::PeepholeProtection2628CodeGenFunction::protectFromPeepholes(RValue rvalue) {2629 // At the moment, the only aggressive peephole we do in IR gen2630 // is trunc(zext) folding, but if we add more, we can easily2631 // extend this protection.2632 2633 if (!rvalue.isScalar()) return PeepholeProtection();2634 llvm::Value *value = rvalue.getScalarVal();2635 if (!isa<llvm::ZExtInst>(value)) return PeepholeProtection();2636 2637 // Just make an extra bitcast.2638 assert(HaveInsertPoint());2639 llvm::Instruction *inst = new llvm::BitCastInst(value, value->getType(), "",2640 Builder.GetInsertBlock());2641 2642 PeepholeProtection protection;2643 protection.Inst = inst;2644 return protection;2645}2646 2647void CodeGenFunction::unprotectFromPeepholes(PeepholeProtection protection) {2648 if (!protection.Inst) return;2649 2650 // In theory, we could try to duplicate the peepholes now, but whatever.2651 protection.Inst->eraseFromParent();2652}2653 2654void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,2655 QualType Ty, SourceLocation Loc,2656 SourceLocation AssumptionLoc,2657 llvm::Value *Alignment,2658 llvm::Value *OffsetValue) {2659 if (Alignment->getType() != IntPtrTy)2660 Alignment =2661 Builder.CreateIntCast(Alignment, IntPtrTy, false, "casted.align");2662 if (OffsetValue && OffsetValue->getType() != IntPtrTy)2663 OffsetValue =2664 Builder.CreateIntCast(OffsetValue, IntPtrTy, true, "casted.offset");2665 llvm::Value *TheCheck = nullptr;2666 if (SanOpts.has(SanitizerKind::Alignment)) {2667 llvm::Value *PtrIntValue =2668 Builder.CreatePtrToInt(PtrValue, IntPtrTy, "ptrint");2669 2670 if (OffsetValue) {2671 bool IsOffsetZero = false;2672 if (const auto *CI = dyn_cast<llvm::ConstantInt>(OffsetValue))2673 IsOffsetZero = CI->isZero();2674 2675 if (!IsOffsetZero)2676 PtrIntValue = Builder.CreateSub(PtrIntValue, OffsetValue, "offsetptr");2677 }2678 2679 llvm::Value *Zero = llvm::ConstantInt::get(IntPtrTy, 0);2680 llvm::Value *Mask =2681 Builder.CreateSub(Alignment, llvm::ConstantInt::get(IntPtrTy, 1));2682 llvm::Value *MaskedPtr = Builder.CreateAnd(PtrIntValue, Mask, "maskedptr");2683 TheCheck = Builder.CreateICmpEQ(MaskedPtr, Zero, "maskcond");2684 }2685 llvm::Instruction *Assumption = Builder.CreateAlignmentAssumption(2686 CGM.getDataLayout(), PtrValue, Alignment, OffsetValue);2687 2688 if (!SanOpts.has(SanitizerKind::Alignment))2689 return;2690 emitAlignmentAssumptionCheck(PtrValue, Ty, Loc, AssumptionLoc, Alignment,2691 OffsetValue, TheCheck, Assumption);2692}2693 2694void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,2695 const Expr *E,2696 SourceLocation AssumptionLoc,2697 llvm::Value *Alignment,2698 llvm::Value *OffsetValue) {2699 QualType Ty = E->getType();2700 SourceLocation Loc = E->getExprLoc();2701 2702 emitAlignmentAssumption(PtrValue, Ty, Loc, AssumptionLoc, Alignment,2703 OffsetValue);2704}2705 2706llvm::Value *CodeGenFunction::EmitAnnotationCall(llvm::Function *AnnotationFn,2707 llvm::Value *AnnotatedVal,2708 StringRef AnnotationStr,2709 SourceLocation Location,2710 const AnnotateAttr *Attr) {2711 SmallVector<llvm::Value *, 5> Args = {2712 AnnotatedVal,2713 CGM.EmitAnnotationString(AnnotationStr),2714 CGM.EmitAnnotationUnit(Location),2715 CGM.EmitAnnotationLineNo(Location),2716 };2717 if (Attr)2718 Args.push_back(CGM.EmitAnnotationArgs(Attr));2719 return Builder.CreateCall(AnnotationFn, Args);2720}2721 2722void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) {2723 assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");2724 for (const auto *I : D->specific_attrs<AnnotateAttr>())2725 EmitAnnotationCall(CGM.getIntrinsic(llvm::Intrinsic::var_annotation,2726 {V->getType(), CGM.ConstGlobalsPtrTy}),2727 V, I->getAnnotation(), D->getLocation(), I);2728}2729 2730Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D,2731 Address Addr) {2732 assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");2733 llvm::Value *V = Addr.emitRawPointer(*this);2734 llvm::Type *VTy = V->getType();2735 auto *PTy = dyn_cast<llvm::PointerType>(VTy);2736 unsigned AS = PTy ? PTy->getAddressSpace() : 0;2737 llvm::PointerType *IntrinTy =2738 llvm::PointerType::get(CGM.getLLVMContext(), AS);2739 llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::ptr_annotation,2740 {IntrinTy, CGM.ConstGlobalsPtrTy});2741 2742 for (const auto *I : D->specific_attrs<AnnotateAttr>()) {2743 // FIXME Always emit the cast inst so we can differentiate between2744 // annotation on the first field of a struct and annotation on the struct2745 // itself.2746 if (VTy != IntrinTy)2747 V = Builder.CreateBitCast(V, IntrinTy);2748 V = EmitAnnotationCall(F, V, I->getAnnotation(), D->getLocation(), I);2749 V = Builder.CreateBitCast(V, VTy);2750 }2751 2752 return Address(V, Addr.getElementType(), Addr.getAlignment());2753}2754 2755CodeGenFunction::CGCapturedStmtInfo::~CGCapturedStmtInfo() { }2756 2757CodeGenFunction::SanitizerScope::SanitizerScope(CodeGenFunction *CGF)2758 : CGF(CGF) {2759 assert(!CGF->IsSanitizerScope);2760 CGF->IsSanitizerScope = true;2761}2762 2763CodeGenFunction::SanitizerScope::~SanitizerScope() {2764 CGF->IsSanitizerScope = false;2765}2766 2767void CodeGenFunction::InsertHelper(llvm::Instruction *I,2768 const llvm::Twine &Name,2769 llvm::BasicBlock::iterator InsertPt) const {2770 LoopStack.InsertHelper(I);2771 if (IsSanitizerScope)2772 I->setNoSanitizeMetadata();2773}2774 2775void CGBuilderInserter::InsertHelper(2776 llvm::Instruction *I, const llvm::Twine &Name,2777 llvm::BasicBlock::iterator InsertPt) const {2778 llvm::IRBuilderDefaultInserter::InsertHelper(I, Name, InsertPt);2779 if (CGF)2780 CGF->InsertHelper(I, Name, InsertPt);2781}2782 2783// Emits an error if we don't have a valid set of target features for the2784// called function.2785void CodeGenFunction::checkTargetFeatures(const CallExpr *E,2786 const FunctionDecl *TargetDecl) {2787 // SemaChecking cannot handle below x86 builtins because they have different2788 // parameter ranges with different TargetAttribute of caller.2789 if (CGM.getContext().getTargetInfo().getTriple().isX86()) {2790 unsigned BuiltinID = TargetDecl->getBuiltinID();2791 if (BuiltinID == X86::BI__builtin_ia32_cmpps ||2792 BuiltinID == X86::BI__builtin_ia32_cmpss ||2793 BuiltinID == X86::BI__builtin_ia32_cmppd ||2794 BuiltinID == X86::BI__builtin_ia32_cmpsd) {2795 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl);2796 llvm::StringMap<bool> TargetFetureMap;2797 CGM.getContext().getFunctionFeatureMap(TargetFetureMap, FD);2798 llvm::APSInt Result =2799 *(E->getArg(2)->getIntegerConstantExpr(CGM.getContext()));2800 if (Result.getSExtValue() > 7 && !TargetFetureMap.lookup("avx"))2801 CGM.getDiags().Report(E->getBeginLoc(), diag::err_builtin_needs_feature)2802 << TargetDecl->getDeclName() << "avx";2803 }2804 }2805 return checkTargetFeatures(E->getBeginLoc(), TargetDecl);2806}2807 2808// Emits an error if we don't have a valid set of target features for the2809// called function.2810void CodeGenFunction::checkTargetFeatures(SourceLocation Loc,2811 const FunctionDecl *TargetDecl) {2812 // Early exit if this is an indirect call.2813 if (!TargetDecl)2814 return;2815 2816 // Get the current enclosing function if it exists. If it doesn't2817 // we can't check the target features anyhow.2818 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl);2819 if (!FD)2820 return;2821 2822 bool IsAlwaysInline = TargetDecl->hasAttr<AlwaysInlineAttr>();2823 bool IsFlatten = FD && FD->hasAttr<FlattenAttr>();2824 2825 // Grab the required features for the call. For a builtin this is listed in2826 // the td file with the default cpu, for an always_inline function this is any2827 // listed cpu and any listed features.2828 unsigned BuiltinID = TargetDecl->getBuiltinID();2829 std::string MissingFeature;2830 llvm::StringMap<bool> CallerFeatureMap;2831 CGM.getContext().getFunctionFeatureMap(CallerFeatureMap, FD);2832 // When compiling in HipStdPar mode we have to be conservative in rejecting2833 // target specific features in the FE, and defer the possible error to the2834 // AcceleratorCodeSelection pass, wherein iff an unsupported target builtin is2835 // referenced by an accelerator executable function, we emit an error.2836 bool IsHipStdPar = getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice;2837 if (BuiltinID) {2838 StringRef FeatureList(CGM.getContext().BuiltinInfo.getRequiredFeatures(BuiltinID));2839 if (!Builtin::evaluateRequiredTargetFeatures(2840 FeatureList, CallerFeatureMap) && !IsHipStdPar) {2841 CGM.getDiags().Report(Loc, diag::err_builtin_needs_feature)2842 << TargetDecl->getDeclName()2843 << FeatureList;2844 }2845 } else if (!TargetDecl->isMultiVersion() &&2846 TargetDecl->hasAttr<TargetAttr>()) {2847 // Get the required features for the callee.2848 2849 const TargetAttr *TD = TargetDecl->getAttr<TargetAttr>();2850 ParsedTargetAttr ParsedAttr =2851 CGM.getContext().filterFunctionTargetAttrs(TD);2852 2853 SmallVector<StringRef, 1> ReqFeatures;2854 llvm::StringMap<bool> CalleeFeatureMap;2855 CGM.getContext().getFunctionFeatureMap(CalleeFeatureMap, TargetDecl);2856 2857 for (const auto &F : ParsedAttr.Features) {2858 if (F[0] == '+' && CalleeFeatureMap.lookup(F.substr(1)))2859 ReqFeatures.push_back(StringRef(F).substr(1));2860 }2861 2862 for (const auto &F : CalleeFeatureMap) {2863 // Only positive features are "required".2864 if (F.getValue())2865 ReqFeatures.push_back(F.getKey());2866 }2867 if (!llvm::all_of(ReqFeatures,2868 [&](StringRef Feature) {2869 if (!CallerFeatureMap.lookup(Feature)) {2870 MissingFeature = Feature.str();2871 return false;2872 }2873 return true;2874 }) &&2875 !IsHipStdPar) {2876 if (IsAlwaysInline)2877 CGM.getDiags().Report(Loc, diag::err_function_needs_feature)2878 << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature;2879 else if (IsFlatten)2880 CGM.getDiags().Report(Loc, diag::err_flatten_function_needs_feature)2881 << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature;2882 }2883 2884 } else if (!FD->isMultiVersion() && FD->hasAttr<TargetAttr>()) {2885 llvm::StringMap<bool> CalleeFeatureMap;2886 CGM.getContext().getFunctionFeatureMap(CalleeFeatureMap, TargetDecl);2887 2888 for (const auto &F : CalleeFeatureMap) {2889 if (F.getValue() &&2890 (!CallerFeatureMap.lookup(F.getKey()) ||2891 !CallerFeatureMap.find(F.getKey())->getValue()) &&2892 !IsHipStdPar) {2893 if (IsAlwaysInline)2894 CGM.getDiags().Report(Loc, diag::err_function_needs_feature)2895 << FD->getDeclName() << TargetDecl->getDeclName() << F.getKey();2896 else if (IsFlatten)2897 CGM.getDiags().Report(Loc, diag::err_flatten_function_needs_feature)2898 << FD->getDeclName() << TargetDecl->getDeclName() << F.getKey();2899 }2900 }2901 }2902}2903 2904void CodeGenFunction::EmitSanitizerStatReport(llvm::SanitizerStatKind SSK) {2905 if (!CGM.getCodeGenOpts().SanitizeStats)2906 return;2907 2908 llvm::IRBuilder<> IRB(Builder.GetInsertBlock(), Builder.GetInsertPoint());2909 IRB.SetCurrentDebugLocation(Builder.getCurrentDebugLocation());2910 CGM.getSanStats().create(IRB, SSK);2911}2912 2913void CodeGenFunction::EmitKCFIOperandBundle(2914 const CGCallee &Callee, SmallVectorImpl<llvm::OperandBundleDef> &Bundles) {2915 const CGCalleeInfo &CI = Callee.getAbstractInfo();2916 const FunctionProtoType *FP = CI.getCalleeFunctionProtoType();2917 if (!FP)2918 return;2919 2920 StringRef Salt;2921 if (const auto &Info = FP->getExtraAttributeInfo())2922 Salt = Info.CFISalt;2923 2924 Bundles.emplace_back("kcfi", CGM.CreateKCFITypeId(FP->desugar(), Salt));2925}2926 2927llvm::Value *2928CodeGenFunction::FormAArch64ResolverCondition(const FMVResolverOption &RO) {2929 return RO.Features.empty() ? nullptr : EmitAArch64CpuSupports(RO.Features);2930}2931 2932llvm::Value *2933CodeGenFunction::FormX86ResolverCondition(const FMVResolverOption &RO) {2934 llvm::Value *Condition = nullptr;2935 2936 if (RO.Architecture) {2937 StringRef Arch = *RO.Architecture;2938 // If arch= specifies an x86-64 micro-architecture level, test the feature2939 // with __builtin_cpu_supports, otherwise use __builtin_cpu_is.2940 if (Arch.starts_with("x86-64"))2941 Condition = EmitX86CpuSupports({Arch});2942 else2943 Condition = EmitX86CpuIs(Arch);2944 }2945 2946 if (!RO.Features.empty()) {2947 llvm::Value *FeatureCond = EmitX86CpuSupports(RO.Features);2948 Condition =2949 Condition ? Builder.CreateAnd(Condition, FeatureCond) : FeatureCond;2950 }2951 return Condition;2952}2953 2954static void CreateMultiVersionResolverReturn(CodeGenModule &CGM,2955 llvm::Function *Resolver,2956 CGBuilderTy &Builder,2957 llvm::Function *FuncToReturn,2958 bool SupportsIFunc) {2959 if (SupportsIFunc) {2960 Builder.CreateRet(FuncToReturn);2961 return;2962 }2963 2964 llvm::SmallVector<llvm::Value *, 10> Args(2965 llvm::make_pointer_range(Resolver->args()));2966 2967 llvm::CallInst *Result = Builder.CreateCall(FuncToReturn, Args);2968 Result->setTailCallKind(llvm::CallInst::TCK_MustTail);2969 2970 if (Resolver->getReturnType()->isVoidTy())2971 Builder.CreateRetVoid();2972 else2973 Builder.CreateRet(Result);2974}2975 2976void CodeGenFunction::EmitMultiVersionResolver(2977 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {2978 2979 llvm::Triple::ArchType ArchType =2980 getContext().getTargetInfo().getTriple().getArch();2981 2982 switch (ArchType) {2983 case llvm::Triple::x86:2984 case llvm::Triple::x86_64:2985 EmitX86MultiVersionResolver(Resolver, Options);2986 return;2987 case llvm::Triple::aarch64:2988 EmitAArch64MultiVersionResolver(Resolver, Options);2989 return;2990 case llvm::Triple::riscv32:2991 case llvm::Triple::riscv64:2992 EmitRISCVMultiVersionResolver(Resolver, Options);2993 return;2994 2995 default:2996 assert(false && "Only implemented for x86, AArch64 and RISC-V targets");2997 }2998}2999 3000void CodeGenFunction::EmitRISCVMultiVersionResolver(3001 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {3002 3003 if (getContext().getTargetInfo().getTriple().getOS() !=3004 llvm::Triple::OSType::Linux) {3005 CGM.getDiags().Report(diag::err_os_unsupport_riscv_fmv);3006 return;3007 }3008 3009 llvm::BasicBlock *CurBlock = createBasicBlock("resolver_entry", Resolver);3010 Builder.SetInsertPoint(CurBlock);3011 EmitRISCVCpuInit();3012 3013 bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();3014 bool HasDefault = false;3015 unsigned DefaultIndex = 0;3016 3017 // Check the each candidate function.3018 for (unsigned Index = 0; Index < Options.size(); Index++) {3019 3020 if (Options[Index].Features.empty()) {3021 HasDefault = true;3022 DefaultIndex = Index;3023 continue;3024 }3025 3026 Builder.SetInsertPoint(CurBlock);3027 3028 // FeaturesCondition: The bitmask of the required extension has been3029 // enabled by the runtime object.3030 // (__riscv_feature_bits.features[i] & REQUIRED_BITMASK) ==3031 // REQUIRED_BITMASK3032 //3033 // When condition is met, return this version of the function.3034 // Otherwise, try the next version.3035 //3036 // if (FeaturesConditionVersion1)3037 // return Version1;3038 // else if (FeaturesConditionVersion2)3039 // return Version2;3040 // else if (FeaturesConditionVersion3)3041 // return Version3;3042 // ...3043 // else3044 // return DefaultVersion;3045 3046 // TODO: Add a condition to check the length before accessing elements.3047 // Without checking the length first, we may access an incorrect memory3048 // address when using different versions.3049 llvm::SmallVector<StringRef, 8> CurrTargetAttrFeats;3050 llvm::SmallVector<std::string, 8> TargetAttrFeats;3051 3052 for (StringRef Feat : Options[Index].Features) {3053 std::vector<std::string> FeatStr =3054 getContext().getTargetInfo().parseTargetAttr(Feat).Features;3055 3056 assert(FeatStr.size() == 1 && "Feature string not delimited");3057 3058 std::string &CurrFeat = FeatStr.front();3059 if (CurrFeat[0] == '+')3060 TargetAttrFeats.push_back(CurrFeat.substr(1));3061 }3062 3063 if (TargetAttrFeats.empty())3064 continue;3065 3066 for (std::string &Feat : TargetAttrFeats)3067 CurrTargetAttrFeats.push_back(Feat);3068 3069 Builder.SetInsertPoint(CurBlock);3070 llvm::Value *FeatsCondition = EmitRISCVCpuSupports(CurrTargetAttrFeats);3071 3072 llvm::BasicBlock *RetBlock = createBasicBlock("resolver_return", Resolver);3073 CGBuilderTy RetBuilder(*this, RetBlock);3074 CreateMultiVersionResolverReturn(CGM, Resolver, RetBuilder,3075 Options[Index].Function, SupportsIFunc);3076 llvm::BasicBlock *ElseBlock = createBasicBlock("resolver_else", Resolver);3077 3078 Builder.SetInsertPoint(CurBlock);3079 Builder.CreateCondBr(FeatsCondition, RetBlock, ElseBlock);3080 3081 CurBlock = ElseBlock;3082 }3083 3084 // Finally, emit the default one.3085 if (HasDefault) {3086 Builder.SetInsertPoint(CurBlock);3087 CreateMultiVersionResolverReturn(3088 CGM, Resolver, Builder, Options[DefaultIndex].Function, SupportsIFunc);3089 return;3090 }3091 3092 // If no generic/default, emit an unreachable.3093 Builder.SetInsertPoint(CurBlock);3094 llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);3095 TrapCall->setDoesNotReturn();3096 TrapCall->setDoesNotThrow();3097 Builder.CreateUnreachable();3098 Builder.ClearInsertionPoint();3099}3100 3101void CodeGenFunction::EmitAArch64MultiVersionResolver(3102 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {3103 assert(!Options.empty() && "No multiversion resolver options found");3104 assert(Options.back().Features.size() == 0 && "Default case must be last");3105 bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();3106 assert(SupportsIFunc &&3107 "Multiversion resolver requires target IFUNC support");3108 bool AArch64CpuInitialized = false;3109 llvm::BasicBlock *CurBlock = createBasicBlock("resolver_entry", Resolver);3110 3111 for (const FMVResolverOption &RO : Options) {3112 Builder.SetInsertPoint(CurBlock);3113 llvm::Value *Condition = FormAArch64ResolverCondition(RO);3114 3115 // The 'default' or 'all features enabled' case.3116 if (!Condition) {3117 CreateMultiVersionResolverReturn(CGM, Resolver, Builder, RO.Function,3118 SupportsIFunc);3119 return;3120 }3121 3122 if (!AArch64CpuInitialized) {3123 Builder.SetInsertPoint(CurBlock, CurBlock->begin());3124 EmitAArch64CpuInit();3125 AArch64CpuInitialized = true;3126 Builder.SetInsertPoint(CurBlock);3127 }3128 3129 llvm::BasicBlock *RetBlock = createBasicBlock("resolver_return", Resolver);3130 CGBuilderTy RetBuilder(*this, RetBlock);3131 CreateMultiVersionResolverReturn(CGM, Resolver, RetBuilder, RO.Function,3132 SupportsIFunc);3133 CurBlock = createBasicBlock("resolver_else", Resolver);3134 Builder.CreateCondBr(Condition, RetBlock, CurBlock);3135 }3136 3137 // If no default, emit an unreachable.3138 Builder.SetInsertPoint(CurBlock);3139 llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);3140 TrapCall->setDoesNotReturn();3141 TrapCall->setDoesNotThrow();3142 Builder.CreateUnreachable();3143 Builder.ClearInsertionPoint();3144}3145 3146void CodeGenFunction::EmitX86MultiVersionResolver(3147 llvm::Function *Resolver, ArrayRef<FMVResolverOption> Options) {3148 3149 bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();3150 3151 // Main function's basic block.3152 llvm::BasicBlock *CurBlock = createBasicBlock("resolver_entry", Resolver);3153 Builder.SetInsertPoint(CurBlock);3154 EmitX86CpuInit();3155 3156 for (const FMVResolverOption &RO : Options) {3157 Builder.SetInsertPoint(CurBlock);3158 llvm::Value *Condition = FormX86ResolverCondition(RO);3159 3160 // The 'default' or 'generic' case.3161 if (!Condition) {3162 assert(&RO == Options.end() - 1 &&3163 "Default or Generic case must be last");3164 CreateMultiVersionResolverReturn(CGM, Resolver, Builder, RO.Function,3165 SupportsIFunc);3166 return;3167 }3168 3169 llvm::BasicBlock *RetBlock = createBasicBlock("resolver_return", Resolver);3170 CGBuilderTy RetBuilder(*this, RetBlock);3171 CreateMultiVersionResolverReturn(CGM, Resolver, RetBuilder, RO.Function,3172 SupportsIFunc);3173 CurBlock = createBasicBlock("resolver_else", Resolver);3174 Builder.CreateCondBr(Condition, RetBlock, CurBlock);3175 }3176 3177 // If no generic/default, emit an unreachable.3178 Builder.SetInsertPoint(CurBlock);3179 llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);3180 TrapCall->setDoesNotReturn();3181 TrapCall->setDoesNotThrow();3182 Builder.CreateUnreachable();3183 Builder.ClearInsertionPoint();3184}3185 3186// Loc - where the diagnostic will point, where in the source code this3187// alignment has failed.3188// SecondaryLoc - if present (will be present if sufficiently different from3189// Loc), the diagnostic will additionally point a "Note:" to this location.3190// It should be the location where the __attribute__((assume_aligned))3191// was written e.g.3192void CodeGenFunction::emitAlignmentAssumptionCheck(3193 llvm::Value *Ptr, QualType Ty, SourceLocation Loc,3194 SourceLocation SecondaryLoc, llvm::Value *Alignment,3195 llvm::Value *OffsetValue, llvm::Value *TheCheck,3196 llvm::Instruction *Assumption) {3197 assert(isa_and_nonnull<llvm::CallInst>(Assumption) &&3198 cast<llvm::CallInst>(Assumption)->getCalledOperand() ==3199 llvm::Intrinsic::getOrInsertDeclaration(3200 Builder.GetInsertBlock()->getParent()->getParent(),3201 llvm::Intrinsic::assume) &&3202 "Assumption should be a call to llvm.assume().");3203 assert(&(Builder.GetInsertBlock()->back()) == Assumption &&3204 "Assumption should be the last instruction of the basic block, "3205 "since the basic block is still being generated.");3206 3207 if (!SanOpts.has(SanitizerKind::Alignment))3208 return;3209 3210 // Don't check pointers to volatile data. The behavior here is implementation-3211 // defined.3212 if (Ty->getPointeeType().isVolatileQualified())3213 return;3214 3215 // We need to temorairly remove the assumption so we can insert the3216 // sanitizer check before it, else the check will be dropped by optimizations.3217 Assumption->removeFromParent();3218 3219 {3220 auto CheckOrdinal = SanitizerKind::SO_Alignment;3221 auto CheckHandler = SanitizerHandler::AlignmentAssumption;3222 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);3223 3224 if (!OffsetValue)3225 OffsetValue = Builder.getInt1(false); // no offset.3226 3227 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc),3228 EmitCheckSourceLocation(SecondaryLoc),3229 EmitCheckTypeDescriptor(Ty)};3230 llvm::Value *DynamicData[] = {Ptr, Alignment, OffsetValue};3231 EmitCheck({std::make_pair(TheCheck, CheckOrdinal)}, CheckHandler,3232 StaticData, DynamicData);3233 }3234 3235 // We are now in the (new, empty) "cont" basic block.3236 // Reintroduce the assumption.3237 Builder.Insert(Assumption);3238 // FIXME: Assumption still has it's original basic block as it's Parent.3239}3240 3241llvm::DebugLoc CodeGenFunction::SourceLocToDebugLoc(SourceLocation Location) {3242 if (CGDebugInfo *DI = getDebugInfo())3243 return DI->SourceLocToDebugLoc(Location);3244 3245 return llvm::DebugLoc();3246}3247 3248llvm::Value *3249CodeGenFunction::emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,3250 Stmt::Likelihood LH) {3251 switch (LH) {3252 case Stmt::LH_None:3253 return Cond;3254 case Stmt::LH_Likely:3255 case Stmt::LH_Unlikely:3256 // Don't generate llvm.expect on -O0 as the backend won't use it for3257 // anything.3258 if (CGM.getCodeGenOpts().OptimizationLevel == 0)3259 return Cond;3260 llvm::Type *CondTy = Cond->getType();3261 assert(CondTy->isIntegerTy(1) && "expecting condition to be a boolean");3262 llvm::Function *FnExpect =3263 CGM.getIntrinsic(llvm::Intrinsic::expect, CondTy);3264 llvm::Value *ExpectedValueOfCond =3265 llvm::ConstantInt::getBool(CondTy, LH == Stmt::LH_Likely);3266 return Builder.CreateCall(FnExpect, {Cond, ExpectedValueOfCond},3267 Cond->getName() + ".expval");3268 }3269 llvm_unreachable("Unknown Likelihood");3270}3271 3272llvm::Value *CodeGenFunction::emitBoolVecConversion(llvm::Value *SrcVec,3273 unsigned NumElementsDst,3274 const llvm::Twine &Name) {3275 auto *SrcTy = cast<llvm::FixedVectorType>(SrcVec->getType());3276 unsigned NumElementsSrc = SrcTy->getNumElements();3277 if (NumElementsSrc == NumElementsDst)3278 return SrcVec;3279 3280 std::vector<int> ShuffleMask(NumElementsDst, -1);3281 for (unsigned MaskIdx = 0;3282 MaskIdx < std::min<>(NumElementsDst, NumElementsSrc); ++MaskIdx)3283 ShuffleMask[MaskIdx] = MaskIdx;3284 3285 return Builder.CreateShuffleVector(SrcVec, ShuffleMask, Name);3286}3287 3288void CodeGenFunction::EmitPointerAuthOperandBundle(3289 const CGPointerAuthInfo &PointerAuth,3290 SmallVectorImpl<llvm::OperandBundleDef> &Bundles) {3291 if (!PointerAuth.isSigned())3292 return;3293 3294 auto *Key = Builder.getInt32(PointerAuth.getKey());3295 3296 llvm::Value *Discriminator = PointerAuth.getDiscriminator();3297 if (!Discriminator)3298 Discriminator = Builder.getSize(0);3299 3300 llvm::Value *Args[] = {Key, Discriminator};3301 Bundles.emplace_back("ptrauth", Args);3302}3303 3304static llvm::Value *EmitPointerAuthCommon(CodeGenFunction &CGF,3305 const CGPointerAuthInfo &PointerAuth,3306 llvm::Value *Pointer,3307 unsigned IntrinsicID) {3308 if (!PointerAuth)3309 return Pointer;3310 3311 auto Key = CGF.Builder.getInt32(PointerAuth.getKey());3312 3313 llvm::Value *Discriminator = PointerAuth.getDiscriminator();3314 if (!Discriminator) {3315 Discriminator = CGF.Builder.getSize(0);3316 }3317 3318 // Convert the pointer to intptr_t before signing it.3319 auto OrigType = Pointer->getType();3320 Pointer = CGF.Builder.CreatePtrToInt(Pointer, CGF.IntPtrTy);3321 3322 // call i64 @llvm.ptrauth.sign.i64(i64 %pointer, i32 %key, i64 %discriminator)3323 auto Intrinsic = CGF.CGM.getIntrinsic(IntrinsicID);3324 Pointer = CGF.EmitRuntimeCall(Intrinsic, {Pointer, Key, Discriminator});3325 3326 // Convert back to the original type.3327 Pointer = CGF.Builder.CreateIntToPtr(Pointer, OrigType);3328 return Pointer;3329}3330 3331llvm::Value *3332CodeGenFunction::EmitPointerAuthSign(const CGPointerAuthInfo &PointerAuth,3333 llvm::Value *Pointer) {3334 if (!PointerAuth.shouldSign())3335 return Pointer;3336 return EmitPointerAuthCommon(*this, PointerAuth, Pointer,3337 llvm::Intrinsic::ptrauth_sign);3338}3339 3340static llvm::Value *EmitStrip(CodeGenFunction &CGF,3341 const CGPointerAuthInfo &PointerAuth,3342 llvm::Value *Pointer) {3343 auto StripIntrinsic = CGF.CGM.getIntrinsic(llvm::Intrinsic::ptrauth_strip);3344 3345 auto Key = CGF.Builder.getInt32(PointerAuth.getKey());3346 // Convert the pointer to intptr_t before signing it.3347 auto OrigType = Pointer->getType();3348 Pointer = CGF.EmitRuntimeCall(3349 StripIntrinsic, {CGF.Builder.CreatePtrToInt(Pointer, CGF.IntPtrTy), Key});3350 return CGF.Builder.CreateIntToPtr(Pointer, OrigType);3351}3352 3353llvm::Value *3354CodeGenFunction::EmitPointerAuthAuth(const CGPointerAuthInfo &PointerAuth,3355 llvm::Value *Pointer) {3356 if (PointerAuth.shouldStrip()) {3357 return EmitStrip(*this, PointerAuth, Pointer);3358 }3359 if (!PointerAuth.shouldAuth()) {3360 return Pointer;3361 }3362 3363 return EmitPointerAuthCommon(*this, PointerAuth, Pointer,3364 llvm::Intrinsic::ptrauth_auth);3365}3366 3367void CodeGenFunction::addInstToCurrentSourceAtom(3368 llvm::Instruction *KeyInstruction, llvm::Value *Backup) {3369 if (CGDebugInfo *DI = getDebugInfo())3370 DI->addInstToCurrentSourceAtom(KeyInstruction, Backup);3371}3372 3373void CodeGenFunction::addInstToSpecificSourceAtom(3374 llvm::Instruction *KeyInstruction, llvm::Value *Backup, uint64_t Atom) {3375 if (CGDebugInfo *DI = getDebugInfo())3376 DI->addInstToSpecificSourceAtom(KeyInstruction, Backup, Atom);3377}3378 3379void CodeGenFunction::addInstToNewSourceAtom(llvm::Instruction *KeyInstruction,3380 llvm::Value *Backup) {3381 if (CGDebugInfo *DI = getDebugInfo()) {3382 ApplyAtomGroup Grp(getDebugInfo());3383 DI->addInstToCurrentSourceAtom(KeyInstruction, Backup);3384 }3385}3386