6995 lines · cpp
1//===--- CGExpr.cpp - Emit LLVM Code from Expressions ---------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This contains code to emit Expr nodes as LLVM code.10//11//===----------------------------------------------------------------------===//12 13#include "ABIInfoImpl.h"14#include "CGCUDARuntime.h"15#include "CGCXXABI.h"16#include "CGCall.h"17#include "CGCleanup.h"18#include "CGDebugInfo.h"19#include "CGHLSLRuntime.h"20#include "CGObjCRuntime.h"21#include "CGOpenMPRuntime.h"22#include "CGRecordLayout.h"23#include "CodeGenFunction.h"24#include "CodeGenModule.h"25#include "CodeGenPGO.h"26#include "ConstantEmitter.h"27#include "TargetInfo.h"28#include "clang/AST/ASTContext.h"29#include "clang/AST/ASTLambda.h"30#include "clang/AST/Attr.h"31#include "clang/AST/DeclObjC.h"32#include "clang/AST/InferAlloc.h"33#include "clang/AST/NSAPI.h"34#include "clang/AST/ParentMapContext.h"35#include "clang/AST/StmtVisitor.h"36#include "clang/Basic/Builtins.h"37#include "clang/Basic/CodeGenOptions.h"38#include "clang/Basic/Module.h"39#include "clang/Basic/SourceManager.h"40#include "llvm/ADT/STLExtras.h"41#include "llvm/ADT/ScopeExit.h"42#include "llvm/ADT/StringExtras.h"43#include "llvm/IR/DataLayout.h"44#include "llvm/IR/Intrinsics.h"45#include "llvm/IR/LLVMContext.h"46#include "llvm/IR/MDBuilder.h"47#include "llvm/IR/MatrixBuilder.h"48#include "llvm/Support/ConvertUTF.h"49#include "llvm/Support/Endian.h"50#include "llvm/Support/MathExtras.h"51#include "llvm/Support/Path.h"52#include "llvm/Support/xxhash.h"53#include "llvm/Transforms/Utils/SanitizerStats.h"54 55#include <numeric>56#include <optional>57#include <string>58 59using namespace clang;60using namespace CodeGen;61 62namespace clang {63// TODO: consider deprecating ClSanitizeGuardChecks; functionality is subsumed64// by -fsanitize-skip-hot-cutoff65llvm::cl::opt<bool> ClSanitizeGuardChecks(66 "ubsan-guard-checks", llvm::cl::Optional,67 llvm::cl::desc("Guard UBSAN checks with `llvm.allow.ubsan.check()`."));68 69} // namespace clang70 71//===--------------------------------------------------------------------===//72// Defines for metadata73//===--------------------------------------------------------------------===//74 75// Those values are crucial to be the SAME as in ubsan runtime library.76enum VariableTypeDescriptorKind : uint16_t {77 /// An integer type.78 TK_Integer = 0x0000,79 /// A floating-point type.80 TK_Float = 0x0001,81 /// An _BitInt(N) type.82 TK_BitInt = 0x0002,83 /// Any other type. The value representation is unspecified.84 TK_Unknown = 0xffff85};86 87//===--------------------------------------------------------------------===//88// Miscellaneous Helper Methods89//===--------------------------------------------------------------------===//90 91static llvm::StringRef GetUBSanTrapForHandler(SanitizerHandler ID) {92 switch (ID) {93#define SANITIZER_CHECK(Enum, Name, Version, Msg) \94 case SanitizerHandler::Enum: \95 return Msg;96 LIST_SANITIZER_CHECKS97#undef SANITIZER_CHECK98 }99 llvm_unreachable("unhandled switch case");100}101 102/// CreateTempAlloca - This creates a alloca and inserts it into the entry103/// block.104RawAddress105CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits Align,106 const Twine &Name,107 llvm::Value *ArraySize) {108 auto Alloca = CreateTempAlloca(Ty, Name, ArraySize);109 Alloca->setAlignment(Align.getAsAlign());110 return RawAddress(Alloca, Ty, Align, KnownNonNull);111}112 113RawAddress CodeGenFunction::MaybeCastStackAddressSpace(RawAddress Alloca,114 LangAS DestLangAS,115 llvm::Value *ArraySize) {116 117 llvm::Value *V = Alloca.getPointer();118 // Alloca always returns a pointer in alloca address space, which may119 // be different from the type defined by the language. For example,120 // in C++ the auto variables are in the default address space. Therefore121 // cast alloca to the default address space when necessary.122 123 unsigned DestAddrSpace = getContext().getTargetAddressSpace(DestLangAS);124 if (DestAddrSpace != Alloca.getAddressSpace()) {125 llvm::IRBuilderBase::InsertPointGuard IPG(Builder);126 // When ArraySize is nullptr, alloca is inserted at AllocaInsertPt,127 // otherwise alloca is inserted at the current insertion point of the128 // builder.129 if (!ArraySize)130 Builder.SetInsertPoint(getPostAllocaInsertPoint());131 V = getTargetHooks().performAddrSpaceCast(132 *this, V, getASTAllocaAddressSpace(), Builder.getPtrTy(DestAddrSpace),133 /*IsNonNull=*/true);134 }135 136 return RawAddress(V, Alloca.getElementType(), Alloca.getAlignment(),137 KnownNonNull);138}139 140RawAddress CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, LangAS DestLangAS,141 CharUnits Align, const Twine &Name,142 llvm::Value *ArraySize,143 RawAddress *AllocaAddr) {144 RawAddress Alloca = CreateTempAllocaWithoutCast(Ty, Align, Name, ArraySize);145 if (AllocaAddr)146 *AllocaAddr = Alloca;147 return MaybeCastStackAddressSpace(Alloca, DestLangAS, ArraySize);148}149 150/// CreateTempAlloca - This creates an alloca and inserts it into the entry151/// block if \p ArraySize is nullptr, otherwise inserts it at the current152/// insertion point of the builder.153llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty,154 const Twine &Name,155 llvm::Value *ArraySize) {156 llvm::AllocaInst *Alloca;157 if (ArraySize)158 Alloca = Builder.CreateAlloca(Ty, ArraySize, Name);159 else160 Alloca =161 new llvm::AllocaInst(Ty, CGM.getDataLayout().getAllocaAddrSpace(),162 ArraySize, Name, AllocaInsertPt->getIterator());163 if (SanOpts.Mask & SanitizerKind::Address) {164 Alloca->addAnnotationMetadata({"alloca_name_altered", Name.str()});165 }166 if (Allocas) {167 Allocas->Add(Alloca);168 }169 return Alloca;170}171 172/// CreateDefaultAlignTempAlloca - This creates an alloca with the173/// default alignment of the corresponding LLVM type, which is *not*174/// guaranteed to be related in any way to the expected alignment of175/// an AST type that might have been lowered to Ty.176RawAddress CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty,177 const Twine &Name) {178 CharUnits Align =179 CharUnits::fromQuantity(CGM.getDataLayout().getPrefTypeAlign(Ty));180 return CreateTempAlloca(Ty, Align, Name);181}182 183RawAddress CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) {184 CharUnits Align = getContext().getTypeAlignInChars(Ty);185 return CreateTempAlloca(ConvertType(Ty), Align, Name);186}187 188RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name,189 RawAddress *Alloca) {190 // FIXME: Should we prefer the preferred type alignment here?191 return CreateMemTemp(Ty, getContext().getTypeAlignInChars(Ty), Name, Alloca);192}193 194RawAddress CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align,195 const Twine &Name,196 RawAddress *Alloca) {197 RawAddress Result = CreateTempAlloca(ConvertTypeForMem(Ty), Align, Name,198 /*ArraySize=*/nullptr, Alloca);199 200 if (Ty->isConstantMatrixType()) {201 auto *ArrayTy = cast<llvm::ArrayType>(Result.getElementType());202 auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(),203 ArrayTy->getNumElements());204 205 Result = Address(Result.getPointer(), VectorTy, Result.getAlignment(),206 KnownNonNull);207 }208 return Result;209}210 211RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty,212 CharUnits Align,213 const Twine &Name) {214 return CreateTempAllocaWithoutCast(ConvertTypeForMem(Ty), Align, Name);215}216 217RawAddress CodeGenFunction::CreateMemTempWithoutCast(QualType Ty,218 const Twine &Name) {219 return CreateMemTempWithoutCast(Ty, getContext().getTypeAlignInChars(Ty),220 Name);221}222 223/// EvaluateExprAsBool - Perform the usual unary conversions on the specified224/// expression and compare the result against zero, returning an Int1Ty value.225llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) {226 PGO->setCurrentStmt(E);227 if (const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>()) {228 llvm::Value *MemPtr = EmitScalarExpr(E);229 return CGM.getCXXABI().EmitMemberPointerIsNotNull(*this, MemPtr, MPT);230 }231 232 QualType BoolTy = getContext().BoolTy;233 SourceLocation Loc = E->getExprLoc();234 CGFPOptionsRAII FPOptsRAII(*this, E);235 if (!E->getType()->isAnyComplexType())236 return EmitScalarConversion(EmitScalarExpr(E), E->getType(), BoolTy, Loc);237 238 return EmitComplexToScalarConversion(EmitComplexExpr(E), E->getType(), BoolTy,239 Loc);240}241 242/// EmitIgnoredExpr - Emit code to compute the specified expression,243/// ignoring the result.244void CodeGenFunction::EmitIgnoredExpr(const Expr *E) {245 if (E->isPRValue())246 return (void)EmitAnyExpr(E, AggValueSlot::ignored(), true);247 248 // if this is a bitfield-resulting conditional operator, we can special case249 // emit this. The normal 'EmitLValue' version of this is particularly250 // difficult to codegen for, since creating a single "LValue" for two251 // different sized arguments here is not particularly doable.252 if (const auto *CondOp = dyn_cast<AbstractConditionalOperator>(253 E->IgnoreParenNoopCasts(getContext()))) {254 if (CondOp->getObjectKind() == OK_BitField)255 return EmitIgnoredConditionalOperator(CondOp);256 }257 258 // Just emit it as an l-value and drop the result.259 EmitLValue(E);260}261 262/// EmitAnyExpr - Emit code to compute the specified expression which263/// can have any type. The result is returned as an RValue struct.264/// If this is an aggregate expression, AggSlot indicates where the265/// result should be returned.266RValue CodeGenFunction::EmitAnyExpr(const Expr *E,267 AggValueSlot aggSlot,268 bool ignoreResult) {269 switch (getEvaluationKind(E->getType())) {270 case TEK_Scalar:271 return RValue::get(EmitScalarExpr(E, ignoreResult));272 case TEK_Complex:273 return RValue::getComplex(EmitComplexExpr(E, ignoreResult, ignoreResult));274 case TEK_Aggregate:275 if (!ignoreResult && aggSlot.isIgnored())276 aggSlot = CreateAggTemp(E->getType(), "agg-temp");277 EmitAggExpr(E, aggSlot);278 return aggSlot.asRValue();279 }280 llvm_unreachable("bad evaluation kind");281}282 283/// EmitAnyExprToTemp - Similar to EmitAnyExpr(), however, the result will284/// always be accessible even if no aggregate location is provided.285RValue CodeGenFunction::EmitAnyExprToTemp(const Expr *E) {286 AggValueSlot AggSlot = AggValueSlot::ignored();287 288 if (hasAggregateEvaluationKind(E->getType()))289 AggSlot = CreateAggTemp(E->getType(), "agg.tmp");290 return EmitAnyExpr(E, AggSlot);291}292 293/// EmitAnyExprToMem - Evaluate an expression into a given memory294/// location.295void CodeGenFunction::EmitAnyExprToMem(const Expr *E,296 Address Location,297 Qualifiers Quals,298 bool IsInit) {299 // FIXME: This function should take an LValue as an argument.300 switch (getEvaluationKind(E->getType())) {301 case TEK_Complex:302 EmitComplexExprIntoLValue(E, MakeAddrLValue(Location, E->getType()),303 /*isInit*/ false);304 return;305 306 case TEK_Aggregate: {307 EmitAggExpr(E, AggValueSlot::forAddr(Location, Quals,308 AggValueSlot::IsDestructed_t(IsInit),309 AggValueSlot::DoesNotNeedGCBarriers,310 AggValueSlot::IsAliased_t(!IsInit),311 AggValueSlot::MayOverlap));312 return;313 }314 315 case TEK_Scalar: {316 RValue RV = RValue::get(EmitScalarExpr(E, /*Ignore*/ false));317 LValue LV = MakeAddrLValue(Location, E->getType());318 EmitStoreThroughLValue(RV, LV);319 return;320 }321 }322 llvm_unreachable("bad evaluation kind");323}324 325void CodeGenFunction::EmitInitializationToLValue(326 const Expr *E, LValue LV, AggValueSlot::IsZeroed_t IsZeroed) {327 QualType Type = LV.getType();328 switch (getEvaluationKind(Type)) {329 case TEK_Complex:330 EmitComplexExprIntoLValue(E, LV, /*isInit*/ true);331 return;332 case TEK_Aggregate:333 EmitAggExpr(E, AggValueSlot::forLValue(LV, AggValueSlot::IsDestructed,334 AggValueSlot::DoesNotNeedGCBarriers,335 AggValueSlot::IsNotAliased,336 AggValueSlot::MayOverlap, IsZeroed));337 return;338 case TEK_Scalar:339 if (LV.isSimple())340 EmitScalarInit(E, /*D=*/nullptr, LV, /*Captured=*/false);341 else342 EmitStoreThroughLValue(RValue::get(EmitScalarExpr(E)), LV);343 return;344 }345 llvm_unreachable("bad evaluation kind");346}347 348static void349pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M,350 const Expr *E, Address ReferenceTemporary) {351 // Objective-C++ ARC:352 // If we are binding a reference to a temporary that has ownership, we353 // need to perform retain/release operations on the temporary.354 //355 // FIXME: This should be looking at E, not M.356 if (auto Lifetime = M->getType().getObjCLifetime()) {357 switch (Lifetime) {358 case Qualifiers::OCL_None:359 case Qualifiers::OCL_ExplicitNone:360 // Carry on to normal cleanup handling.361 break;362 363 case Qualifiers::OCL_Autoreleasing:364 // Nothing to do; cleaned up by an autorelease pool.365 return;366 367 case Qualifiers::OCL_Strong:368 case Qualifiers::OCL_Weak:369 switch (StorageDuration Duration = M->getStorageDuration()) {370 case SD_Static:371 // Note: we intentionally do not register a cleanup to release372 // the object on program termination.373 return;374 375 case SD_Thread:376 // FIXME: We should probably register a cleanup in this case.377 return;378 379 case SD_Automatic:380 case SD_FullExpression:381 CodeGenFunction::Destroyer *Destroy;382 CleanupKind CleanupKind;383 if (Lifetime == Qualifiers::OCL_Strong) {384 const ValueDecl *VD = M->getExtendingDecl();385 bool Precise = isa_and_nonnull<VarDecl>(VD) &&386 VD->hasAttr<ObjCPreciseLifetimeAttr>();387 CleanupKind = CGF.getARCCleanupKind();388 Destroy = Precise ? &CodeGenFunction::destroyARCStrongPrecise389 : &CodeGenFunction::destroyARCStrongImprecise;390 } else {391 // __weak objects always get EH cleanups; otherwise, exceptions392 // could cause really nasty crashes instead of mere leaks.393 CleanupKind = NormalAndEHCleanup;394 Destroy = &CodeGenFunction::destroyARCWeak;395 }396 if (Duration == SD_FullExpression)397 CGF.pushDestroy(CleanupKind, ReferenceTemporary,398 M->getType(), *Destroy,399 CleanupKind & EHCleanup);400 else401 CGF.pushLifetimeExtendedDestroy(CleanupKind, ReferenceTemporary,402 M->getType(),403 *Destroy, CleanupKind & EHCleanup);404 return;405 406 case SD_Dynamic:407 llvm_unreachable("temporary cannot have dynamic storage duration");408 }409 llvm_unreachable("unknown storage duration");410 }411 }412 413 QualType::DestructionKind DK = E->getType().isDestructedType();414 if (DK != QualType::DK_none) {415 switch (M->getStorageDuration()) {416 case SD_Static:417 case SD_Thread: {418 CXXDestructorDecl *ReferenceTemporaryDtor = nullptr;419 if (const auto *ClassDecl =420 E->getType()->getBaseElementTypeUnsafe()->getAsCXXRecordDecl();421 ClassDecl && !ClassDecl->hasTrivialDestructor())422 // Get the destructor for the reference temporary.423 ReferenceTemporaryDtor = ClassDecl->getDestructor();424 425 if (!ReferenceTemporaryDtor)426 return;427 428 llvm::FunctionCallee CleanupFn;429 llvm::Constant *CleanupArg;430 if (E->getType()->isArrayType()) {431 CleanupFn = CodeGenFunction(CGF.CGM).generateDestroyHelper(432 ReferenceTemporary, E->getType(), CodeGenFunction::destroyCXXObject,433 CGF.getLangOpts().Exceptions,434 dyn_cast_or_null<VarDecl>(M->getExtendingDecl()));435 CleanupArg = llvm::Constant::getNullValue(CGF.Int8PtrTy);436 } else {437 CleanupFn = CGF.CGM.getAddrAndTypeOfCXXStructor(438 GlobalDecl(ReferenceTemporaryDtor, Dtor_Complete));439 CleanupArg =440 cast<llvm::Constant>(ReferenceTemporary.emitRawPointer(CGF));441 }442 CGF.CGM.getCXXABI().registerGlobalDtor(443 CGF, *cast<VarDecl>(M->getExtendingDecl()), CleanupFn, CleanupArg);444 } break;445 case SD_FullExpression:446 CGF.pushDestroy(DK, ReferenceTemporary, E->getType());447 break;448 case SD_Automatic:449 CGF.pushLifetimeExtendedDestroy(DK, ReferenceTemporary, E->getType());450 break;451 case SD_Dynamic:452 llvm_unreachable("temporary cannot have dynamic storage duration");453 }454 }455}456 457static RawAddress createReferenceTemporary(CodeGenFunction &CGF,458 const MaterializeTemporaryExpr *M,459 const Expr *Inner,460 RawAddress *Alloca = nullptr) {461 auto &TCG = CGF.getTargetHooks();462 switch (M->getStorageDuration()) {463 case SD_FullExpression:464 case SD_Automatic: {465 // If we have a constant temporary array or record try to promote it into a466 // constant global under the same rules a normal constant would've been467 // promoted. This is easier on the optimizer and generally emits fewer468 // instructions.469 QualType Ty = Inner->getType();470 if (CGF.CGM.getCodeGenOpts().MergeAllConstants &&471 (Ty->isArrayType() || Ty->isRecordType()) &&472 Ty.isConstantStorage(CGF.getContext(), true, false))473 if (auto Init = ConstantEmitter(CGF).tryEmitAbstract(Inner, Ty)) {474 auto AS = CGF.CGM.GetGlobalConstantAddressSpace();475 auto *GV = new llvm::GlobalVariable(476 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,477 llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp", nullptr,478 llvm::GlobalValue::NotThreadLocal,479 CGF.getContext().getTargetAddressSpace(AS));480 CharUnits alignment = CGF.getContext().getTypeAlignInChars(Ty);481 GV->setAlignment(alignment.getAsAlign());482 llvm::Constant *C = GV;483 if (AS != LangAS::Default)484 C = TCG.performAddrSpaceCast(485 CGF.CGM, GV, AS,486 llvm::PointerType::get(487 CGF.getLLVMContext(),488 CGF.getContext().getTargetAddressSpace(LangAS::Default)));489 // FIXME: Should we put the new global into a COMDAT?490 return RawAddress(C, GV->getValueType(), alignment);491 }492 return CGF.CreateMemTemp(Ty, "ref.tmp", Alloca);493 }494 case SD_Thread:495 case SD_Static:496 return CGF.CGM.GetAddrOfGlobalTemporary(M, Inner);497 498 case SD_Dynamic:499 llvm_unreachable("temporary can't have dynamic storage duration");500 }501 llvm_unreachable("unknown storage duration");502}503 504/// Helper method to check if the underlying ABI is AAPCS505static bool isAAPCS(const TargetInfo &TargetInfo) {506 return TargetInfo.getABI().starts_with("aapcs");507}508 509LValue CodeGenFunction::510EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) {511 const Expr *E = M->getSubExpr();512 513 assert((!M->getExtendingDecl() || !isa<VarDecl>(M->getExtendingDecl()) ||514 !cast<VarDecl>(M->getExtendingDecl())->isARCPseudoStrong()) &&515 "Reference should never be pseudo-strong!");516 517 // FIXME: ideally this would use EmitAnyExprToMem, however, we cannot do so518 // as that will cause the lifetime adjustment to be lost for ARC519 auto ownership = M->getType().getObjCLifetime();520 if (ownership != Qualifiers::OCL_None &&521 ownership != Qualifiers::OCL_ExplicitNone) {522 RawAddress Object = createReferenceTemporary(*this, M, E);523 if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object.getPointer())) {524 llvm::Type *Ty = ConvertTypeForMem(E->getType());525 Object = Object.withElementType(Ty);526 527 // createReferenceTemporary will promote the temporary to a global with a528 // constant initializer if it can. It can only do this to a value of529 // ARC-manageable type if the value is global and therefore "immune" to530 // ref-counting operations. Therefore we have no need to emit either a531 // dynamic initialization or a cleanup and we can just return the address532 // of the temporary.533 if (Var->hasInitializer())534 return MakeAddrLValue(Object, M->getType(), AlignmentSource::Decl);535 536 Var->setInitializer(CGM.EmitNullConstant(E->getType()));537 }538 LValue RefTempDst = MakeAddrLValue(Object, M->getType(),539 AlignmentSource::Decl);540 541 switch (getEvaluationKind(E->getType())) {542 default: llvm_unreachable("expected scalar or aggregate expression");543 case TEK_Scalar:544 EmitScalarInit(E, M->getExtendingDecl(), RefTempDst, false);545 break;546 case TEK_Aggregate: {547 EmitAggExpr(E, AggValueSlot::forAddr(Object,548 E->getType().getQualifiers(),549 AggValueSlot::IsDestructed,550 AggValueSlot::DoesNotNeedGCBarriers,551 AggValueSlot::IsNotAliased,552 AggValueSlot::DoesNotOverlap));553 break;554 }555 }556 557 pushTemporaryCleanup(*this, M, E, Object);558 return RefTempDst;559 }560 561 SmallVector<const Expr *, 2> CommaLHSs;562 SmallVector<SubobjectAdjustment, 2> Adjustments;563 E = E->skipRValueSubobjectAdjustments(CommaLHSs, Adjustments);564 565 for (const auto &Ignored : CommaLHSs)566 EmitIgnoredExpr(Ignored);567 568 if (const auto *opaque = dyn_cast<OpaqueValueExpr>(E)) {569 if (opaque->getType()->isRecordType()) {570 assert(Adjustments.empty());571 return EmitOpaqueValueLValue(opaque);572 }573 }574 575 // Create and initialize the reference temporary.576 RawAddress Alloca = Address::invalid();577 RawAddress Object = createReferenceTemporary(*this, M, E, &Alloca);578 if (auto *Var = dyn_cast<llvm::GlobalVariable>(579 Object.getPointer()->stripPointerCasts())) {580 llvm::Type *TemporaryType = ConvertTypeForMem(E->getType());581 Object = Object.withElementType(TemporaryType);582 // If the temporary is a global and has a constant initializer or is a583 // constant temporary that we promoted to a global, we may have already584 // initialized it.585 if (!Var->hasInitializer()) {586 Var->setInitializer(CGM.EmitNullConstant(E->getType()));587 QualType RefType = M->getType().withoutLocalFastQualifiers();588 if (RefType.getPointerAuth()) {589 // Use the qualifier of the reference temporary to sign the pointer.590 LValue LV = MakeRawAddrLValue(Object.getPointer(), RefType,591 Object.getAlignment());592 EmitScalarInit(E, M->getExtendingDecl(), LV, false);593 } else {594 EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/ true);595 }596 }597 } else {598 switch (M->getStorageDuration()) {599 case SD_Automatic:600 if (EmitLifetimeStart(Alloca.getPointer())) {601 pushCleanupAfterFullExpr<CallLifetimeEnd>(NormalEHLifetimeMarker,602 Alloca);603 }604 break;605 606 case SD_FullExpression: {607 if (!ShouldEmitLifetimeMarkers)608 break;609 610 // Avoid creating a conditional cleanup just to hold an llvm.lifetime.end611 // marker. Instead, start the lifetime of a conditional temporary earlier612 // so that it's unconditional. Don't do this with sanitizers which need613 // more precise lifetime marks. However when inside an "await.suspend"614 // block, we should always avoid conditional cleanup because it creates615 // boolean marker that lives across await_suspend, which can destroy coro616 // frame.617 ConditionalEvaluation *OldConditional = nullptr;618 CGBuilderTy::InsertPoint OldIP;619 if (isInConditionalBranch() && !E->getType().isDestructedType() &&620 ((!SanOpts.has(SanitizerKind::HWAddress) &&621 !SanOpts.has(SanitizerKind::Memory) &&622 !CGM.getCodeGenOpts().SanitizeAddressUseAfterScope) ||623 inSuspendBlock())) {624 OldConditional = OutermostConditional;625 OutermostConditional = nullptr;626 627 OldIP = Builder.saveIP();628 llvm::BasicBlock *Block = OldConditional->getStartingBlock();629 Builder.restoreIP(CGBuilderTy::InsertPoint(630 Block, llvm::BasicBlock::iterator(Block->back())));631 }632 633 if (EmitLifetimeStart(Alloca.getPointer())) {634 pushFullExprCleanup<CallLifetimeEnd>(NormalEHLifetimeMarker, Alloca);635 }636 637 if (OldConditional) {638 OutermostConditional = OldConditional;639 Builder.restoreIP(OldIP);640 }641 break;642 }643 644 default:645 break;646 }647 EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/true);648 }649 pushTemporaryCleanup(*this, M, E, Object);650 651 // Perform derived-to-base casts and/or field accesses, to get from the652 // temporary object we created (and, potentially, for which we extended653 // the lifetime) to the subobject we're binding the reference to.654 for (SubobjectAdjustment &Adjustment : llvm::reverse(Adjustments)) {655 switch (Adjustment.Kind) {656 case SubobjectAdjustment::DerivedToBaseAdjustment:657 Object =658 GetAddressOfBaseClass(Object, Adjustment.DerivedToBase.DerivedClass,659 Adjustment.DerivedToBase.BasePath->path_begin(),660 Adjustment.DerivedToBase.BasePath->path_end(),661 /*NullCheckValue=*/ false, E->getExprLoc());662 break;663 664 case SubobjectAdjustment::FieldAdjustment: {665 LValue LV = MakeAddrLValue(Object, E->getType(), AlignmentSource::Decl);666 LV = EmitLValueForField(LV, Adjustment.Field);667 assert(LV.isSimple() &&668 "materialized temporary field is not a simple lvalue");669 Object = LV.getAddress();670 break;671 }672 673 case SubobjectAdjustment::MemberPointerAdjustment: {674 llvm::Value *Ptr = EmitScalarExpr(Adjustment.Ptr.RHS);675 Object = EmitCXXMemberDataPointerAddress(676 E, Object, Ptr, Adjustment.Ptr.MPT, /*IsInBounds=*/true);677 break;678 }679 }680 }681 682 return MakeAddrLValue(Object, M->getType(), AlignmentSource::Decl);683}684 685RValue686CodeGenFunction::EmitReferenceBindingToExpr(const Expr *E) {687 // Emit the expression as an lvalue.688 LValue LV = EmitLValue(E);689 assert(LV.isSimple());690 llvm::Value *Value = LV.getPointer(*this);691 692 if (sanitizePerformTypeCheck() && !E->getType()->isFunctionType()) {693 // C++11 [dcl.ref]p5 (as amended by core issue 453):694 // If a glvalue to which a reference is directly bound designates neither695 // an existing object or function of an appropriate type nor a region of696 // storage of suitable size and alignment to contain an object of the697 // reference's type, the behavior is undefined.698 QualType Ty = E->getType();699 EmitTypeCheck(TCK_ReferenceBinding, E->getExprLoc(), Value, Ty);700 }701 702 return RValue::get(Value);703}704 705 706/// getAccessedFieldNo - Given an encoded value and a result number, return the707/// input field number being accessed.708unsigned CodeGenFunction::getAccessedFieldNo(unsigned Idx,709 const llvm::Constant *Elts) {710 return cast<llvm::ConstantInt>(Elts->getAggregateElement(Idx))711 ->getZExtValue();712}713 714static llvm::Value *emitHashMix(CGBuilderTy &Builder, llvm::Value *Acc,715 llvm::Value *Ptr) {716 llvm::Value *A0 =717 Builder.CreateMul(Ptr, Builder.getInt64(0xbf58476d1ce4e5b9u));718 llvm::Value *A1 =719 Builder.CreateXor(A0, Builder.CreateLShr(A0, Builder.getInt64(31)));720 return Builder.CreateXor(Acc, A1);721}722 723bool CodeGenFunction::isNullPointerAllowed(TypeCheckKind TCK) {724 return TCK == TCK_DowncastPointer || TCK == TCK_Upcast ||725 TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation;726}727 728bool CodeGenFunction::isVptrCheckRequired(TypeCheckKind TCK, QualType Ty) {729 CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();730 return (RD && RD->hasDefinition() && RD->isDynamicClass()) &&731 (TCK == TCK_MemberAccess || TCK == TCK_MemberCall ||732 TCK == TCK_DowncastPointer || TCK == TCK_DowncastReference ||733 TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation);734}735 736bool CodeGenFunction::sanitizePerformTypeCheck() const {737 return SanOpts.has(SanitizerKind::Null) ||738 SanOpts.has(SanitizerKind::Alignment) ||739 SanOpts.has(SanitizerKind::ObjectSize) ||740 SanOpts.has(SanitizerKind::Vptr);741}742 743void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,744 llvm::Value *Ptr, QualType Ty,745 CharUnits Alignment,746 SanitizerSet SkippedChecks,747 llvm::Value *ArraySize) {748 if (!sanitizePerformTypeCheck())749 return;750 751 // Don't check pointers outside the default address space. The null check752 // isn't correct, the object-size check isn't supported by LLVM, and we can't753 // communicate the addresses to the runtime handler for the vptr check.754 if (Ptr->getType()->getPointerAddressSpace())755 return;756 757 // Don't check pointers to volatile data. The behavior here is implementation-758 // defined.759 if (Ty.isVolatileQualified())760 return;761 762 // Quickly determine whether we have a pointer to an alloca. It's possible763 // to skip null checks, and some alignment checks, for these pointers. This764 // can reduce compile-time significantly.765 auto PtrToAlloca = dyn_cast<llvm::AllocaInst>(Ptr->stripPointerCasts());766 767 llvm::Value *IsNonNull = nullptr;768 bool IsGuaranteedNonNull =769 SkippedChecks.has(SanitizerKind::Null) || PtrToAlloca;770 771 llvm::BasicBlock *Done = nullptr;772 bool DoneViaNullSanitize = false;773 774 {775 auto CheckHandler = SanitizerHandler::TypeMismatch;776 SanitizerDebugLocation SanScope(this,777 {SanitizerKind::SO_Null,778 SanitizerKind::SO_ObjectSize,779 SanitizerKind::SO_Alignment},780 CheckHandler);781 782 SmallVector<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>, 3>783 Checks;784 785 llvm::Value *True = llvm::ConstantInt::getTrue(getLLVMContext());786 bool AllowNullPointers = isNullPointerAllowed(TCK);787 if ((SanOpts.has(SanitizerKind::Null) || AllowNullPointers) &&788 !IsGuaranteedNonNull) {789 // The glvalue must not be an empty glvalue.790 IsNonNull = Builder.CreateIsNotNull(Ptr);791 792 // The IR builder can constant-fold the null check if the pointer points793 // to a constant.794 IsGuaranteedNonNull = IsNonNull == True;795 796 // Skip the null check if the pointer is known to be non-null.797 if (!IsGuaranteedNonNull) {798 if (AllowNullPointers) {799 // When performing pointer casts, it's OK if the value is null.800 // Skip the remaining checks in that case.801 Done = createBasicBlock("null");802 DoneViaNullSanitize = true;803 llvm::BasicBlock *Rest = createBasicBlock("not.null");804 Builder.CreateCondBr(IsNonNull, Rest, Done);805 EmitBlock(Rest);806 } else {807 Checks.push_back(std::make_pair(IsNonNull, SanitizerKind::SO_Null));808 }809 }810 }811 812 if (SanOpts.has(SanitizerKind::ObjectSize) &&813 !SkippedChecks.has(SanitizerKind::ObjectSize) &&814 !Ty->isIncompleteType()) {815 uint64_t TySize = CGM.getMinimumObjectSize(Ty).getQuantity();816 llvm::Value *Size = llvm::ConstantInt::get(IntPtrTy, TySize);817 if (ArraySize)818 Size = Builder.CreateMul(Size, ArraySize);819 820 // Degenerate case: new X[0] does not need an objectsize check.821 llvm::Constant *ConstantSize = dyn_cast<llvm::Constant>(Size);822 if (!ConstantSize || !ConstantSize->isNullValue()) {823 // The glvalue must refer to a large enough storage region.824 // FIXME: If Address Sanitizer is enabled, insert dynamic825 // instrumentation826 // to check this.827 // FIXME: Get object address space828 llvm::Type *Tys[2] = {IntPtrTy, Int8PtrTy};829 llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::objectsize, Tys);830 llvm::Value *Min = Builder.getFalse();831 llvm::Value *NullIsUnknown = Builder.getFalse();832 llvm::Value *Dynamic = Builder.getFalse();833 llvm::Value *LargeEnough = Builder.CreateICmpUGE(834 Builder.CreateCall(F, {Ptr, Min, NullIsUnknown, Dynamic}), Size);835 Checks.push_back(836 std::make_pair(LargeEnough, SanitizerKind::SO_ObjectSize));837 }838 }839 840 llvm::MaybeAlign AlignVal;841 llvm::Value *PtrAsInt = nullptr;842 843 if (SanOpts.has(SanitizerKind::Alignment) &&844 !SkippedChecks.has(SanitizerKind::Alignment)) {845 AlignVal = Alignment.getAsMaybeAlign();846 if (!Ty->isIncompleteType() && !AlignVal)847 AlignVal = CGM.getNaturalTypeAlignment(Ty, nullptr, nullptr,848 /*ForPointeeType=*/true)849 .getAsMaybeAlign();850 851 // The glvalue must be suitably aligned.852 if (AlignVal && *AlignVal > llvm::Align(1) &&853 (!PtrToAlloca || PtrToAlloca->getAlign() < *AlignVal)) {854 PtrAsInt = Builder.CreatePtrToInt(Ptr, IntPtrTy);855 llvm::Value *Align = Builder.CreateAnd(856 PtrAsInt, llvm::ConstantInt::get(IntPtrTy, AlignVal->value() - 1));857 llvm::Value *Aligned =858 Builder.CreateICmpEQ(Align, llvm::ConstantInt::get(IntPtrTy, 0));859 if (Aligned != True)860 Checks.push_back(861 std::make_pair(Aligned, SanitizerKind::SO_Alignment));862 }863 }864 865 if (Checks.size() > 0) {866 llvm::Constant *StaticData[] = {867 EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(Ty),868 llvm::ConstantInt::get(Int8Ty, AlignVal ? llvm::Log2(*AlignVal) : 1),869 llvm::ConstantInt::get(Int8Ty, TCK)};870 EmitCheck(Checks, CheckHandler, StaticData, PtrAsInt ? PtrAsInt : Ptr);871 }872 }873 874 // If possible, check that the vptr indicates that there is a subobject of875 // type Ty at offset zero within this object.876 //877 // C++11 [basic.life]p5,6:878 // [For storage which does not refer to an object within its lifetime]879 // The program has undefined behavior if:880 // -- the [pointer or glvalue] is used to access a non-static data member881 // or call a non-static member function882 if (SanOpts.has(SanitizerKind::Vptr) &&883 !SkippedChecks.has(SanitizerKind::Vptr) && isVptrCheckRequired(TCK, Ty)) {884 SanitizerDebugLocation SanScope(this, {SanitizerKind::SO_Vptr},885 SanitizerHandler::DynamicTypeCacheMiss);886 887 // Ensure that the pointer is non-null before loading it. If there is no888 // compile-time guarantee, reuse the run-time null check or emit a new one.889 if (!IsGuaranteedNonNull) {890 if (!IsNonNull)891 IsNonNull = Builder.CreateIsNotNull(Ptr);892 if (!Done)893 Done = createBasicBlock("vptr.null");894 llvm::BasicBlock *VptrNotNull = createBasicBlock("vptr.not.null");895 Builder.CreateCondBr(IsNonNull, VptrNotNull, Done);896 EmitBlock(VptrNotNull);897 }898 899 // Compute a deterministic hash of the mangled name of the type.900 SmallString<64> MangledName;901 llvm::raw_svector_ostream Out(MangledName);902 CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty.getUnqualifiedType(),903 Out);904 905 // Contained in NoSanitizeList based on the mangled type.906 if (!CGM.getContext().getNoSanitizeList().containsType(SanitizerKind::Vptr,907 Out.str())) {908 // Load the vptr, and mix it with TypeHash.909 llvm::Value *TypeHash =910 llvm::ConstantInt::get(Int64Ty, xxh3_64bits(Out.str()));911 912 llvm::Type *VPtrTy = llvm::PointerType::get(getLLVMContext(), 0);913 Address VPtrAddr(Ptr, IntPtrTy, getPointerAlign());914 llvm::Value *VPtrVal = GetVTablePtr(VPtrAddr, VPtrTy,915 Ty->getAsCXXRecordDecl(),916 VTableAuthMode::UnsafeUbsanStrip);917 VPtrVal = Builder.CreateBitOrPointerCast(VPtrVal, IntPtrTy);918 919 llvm::Value *Hash =920 emitHashMix(Builder, TypeHash, Builder.CreateZExt(VPtrVal, Int64Ty));921 Hash = Builder.CreateTrunc(Hash, IntPtrTy);922 923 // Look the hash up in our cache.924 const int CacheSize = 128;925 llvm::Type *HashTable = llvm::ArrayType::get(IntPtrTy, CacheSize);926 llvm::Value *Cache = CGM.CreateRuntimeVariable(HashTable,927 "__ubsan_vptr_type_cache");928 llvm::Value *Slot = Builder.CreateAnd(Hash,929 llvm::ConstantInt::get(IntPtrTy,930 CacheSize-1));931 llvm::Value *Indices[] = { Builder.getInt32(0), Slot };932 llvm::Value *CacheVal = Builder.CreateAlignedLoad(933 IntPtrTy, Builder.CreateInBoundsGEP(HashTable, Cache, Indices),934 getPointerAlign());935 936 // If the hash isn't in the cache, call a runtime handler to perform the937 // hard work of checking whether the vptr is for an object of the right938 // type. This will either fill in the cache and return, or produce a939 // diagnostic.940 llvm::Value *EqualHash = Builder.CreateICmpEQ(CacheVal, Hash);941 llvm::Constant *StaticData[] = {942 EmitCheckSourceLocation(Loc),943 EmitCheckTypeDescriptor(Ty),944 CGM.GetAddrOfRTTIDescriptor(Ty.getUnqualifiedType()),945 llvm::ConstantInt::get(Int8Ty, TCK)946 };947 llvm::Value *DynamicData[] = { Ptr, Hash };948 EmitCheck(std::make_pair(EqualHash, SanitizerKind::SO_Vptr),949 SanitizerHandler::DynamicTypeCacheMiss, StaticData,950 DynamicData);951 }952 }953 954 if (Done) {955 SanitizerDebugLocation SanScope(956 this,957 {DoneViaNullSanitize ? SanitizerKind::SO_Null : SanitizerKind::SO_Vptr},958 DoneViaNullSanitize ? SanitizerHandler::TypeMismatch959 : SanitizerHandler::DynamicTypeCacheMiss);960 Builder.CreateBr(Done);961 EmitBlock(Done);962 }963}964 965llvm::Value *CodeGenFunction::LoadPassedObjectSize(const Expr *E,966 QualType EltTy) {967 ASTContext &C = getContext();968 uint64_t EltSize = C.getTypeSizeInChars(EltTy).getQuantity();969 if (!EltSize)970 return nullptr;971 972 auto *ArrayDeclRef = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts());973 if (!ArrayDeclRef)974 return nullptr;975 976 auto *ParamDecl = dyn_cast<ParmVarDecl>(ArrayDeclRef->getDecl());977 if (!ParamDecl)978 return nullptr;979 980 auto *POSAttr = ParamDecl->getAttr<PassObjectSizeAttr>();981 if (!POSAttr)982 return nullptr;983 984 // Don't load the size if it's a lower bound.985 int POSType = POSAttr->getType();986 if (POSType != 0 && POSType != 1)987 return nullptr;988 989 // Find the implicit size parameter.990 auto PassedSizeIt = SizeArguments.find(ParamDecl);991 if (PassedSizeIt == SizeArguments.end())992 return nullptr;993 994 const ImplicitParamDecl *PassedSizeDecl = PassedSizeIt->second;995 assert(LocalDeclMap.count(PassedSizeDecl) && "Passed size not loadable");996 Address AddrOfSize = LocalDeclMap.find(PassedSizeDecl)->second;997 llvm::Value *SizeInBytes = EmitLoadOfScalar(AddrOfSize, /*Volatile=*/false,998 C.getSizeType(), E->getExprLoc());999 llvm::Value *SizeOfElement =1000 llvm::ConstantInt::get(SizeInBytes->getType(), EltSize);1001 return Builder.CreateUDiv(SizeInBytes, SizeOfElement);1002}1003 1004/// If Base is known to point to the start of an array, return the length of1005/// that array. Return 0 if the length cannot be determined.1006static llvm::Value *getArrayIndexingBound(CodeGenFunction &CGF,1007 const Expr *Base,1008 QualType &IndexedType,1009 LangOptions::StrictFlexArraysLevelKind1010 StrictFlexArraysLevel) {1011 // For the vector indexing extension, the bound is the number of elements.1012 if (const VectorType *VT = Base->getType()->getAs<VectorType>()) {1013 IndexedType = Base->getType();1014 return CGF.Builder.getInt32(VT->getNumElements());1015 }1016 1017 Base = Base->IgnoreParens();1018 1019 if (const auto *CE = dyn_cast<CastExpr>(Base)) {1020 if (CE->getCastKind() == CK_ArrayToPointerDecay &&1021 !CE->getSubExpr()->isFlexibleArrayMemberLike(CGF.getContext(),1022 StrictFlexArraysLevel)) {1023 CodeGenFunction::SanitizerScope SanScope(&CGF);1024 1025 IndexedType = CE->getSubExpr()->getType();1026 const ArrayType *AT = IndexedType->castAsArrayTypeUnsafe();1027 if (const auto *CAT = dyn_cast<ConstantArrayType>(AT))1028 return CGF.Builder.getInt(CAT->getSize());1029 1030 if (const auto *VAT = dyn_cast<VariableArrayType>(AT))1031 return CGF.getVLASize(VAT).NumElts;1032 // Ignore pass_object_size here. It's not applicable on decayed pointers.1033 }1034 }1035 1036 CodeGenFunction::SanitizerScope SanScope(&CGF);1037 1038 QualType EltTy{Base->getType()->getPointeeOrArrayElementType(), 0};1039 if (llvm::Value *POS = CGF.LoadPassedObjectSize(Base, EltTy)) {1040 IndexedType = Base->getType();1041 return POS;1042 }1043 1044 return nullptr;1045}1046 1047namespace {1048 1049/// \p StructAccessBase returns the base \p Expr of a field access. It returns1050/// either a \p DeclRefExpr, representing the base pointer to the struct, i.e.:1051///1052/// p in p-> a.b.c1053///1054/// or a \p MemberExpr, if the \p MemberExpr has the \p RecordDecl we're1055/// looking for:1056///1057/// struct s {1058/// struct s *ptr;1059/// int count;1060/// char array[] __attribute__((counted_by(count)));1061/// };1062///1063/// If we have an expression like \p p->ptr->array[index], we want the1064/// \p MemberExpr for \p p->ptr instead of \p p.1065class StructAccessBase1066 : public ConstStmtVisitor<StructAccessBase, const Expr *> {1067 const RecordDecl *ExpectedRD;1068 1069 bool IsExpectedRecordDecl(const Expr *E) const {1070 QualType Ty = E->getType();1071 if (Ty->isPointerType())1072 Ty = Ty->getPointeeType();1073 return ExpectedRD == Ty->getAsRecordDecl();1074 }1075 1076public:1077 StructAccessBase(const RecordDecl *ExpectedRD) : ExpectedRD(ExpectedRD) {}1078 1079 //===--------------------------------------------------------------------===//1080 // Visitor Methods1081 //===--------------------------------------------------------------------===//1082 1083 // NOTE: If we build C++ support for counted_by, then we'll have to handle1084 // horrors like this:1085 //1086 // struct S {1087 // int x, y;1088 // int blah[] __attribute__((counted_by(x)));1089 // } s;1090 //1091 // int foo(int index, int val) {1092 // int (S::*IHatePMDs)[] = &S::blah;1093 // (s.*IHatePMDs)[index] = val;1094 // }1095 1096 const Expr *Visit(const Expr *E) {1097 return ConstStmtVisitor<StructAccessBase, const Expr *>::Visit(E);1098 }1099 1100 const Expr *VisitStmt(const Stmt *S) { return nullptr; }1101 1102 // These are the types we expect to return (in order of most to least1103 // likely):1104 //1105 // 1. DeclRefExpr - This is the expression for the base of the structure.1106 // It's exactly what we want to build an access to the \p counted_by1107 // field.1108 // 2. MemberExpr - This is the expression that has the same \p RecordDecl1109 // as the flexble array member's lexical enclosing \p RecordDecl. This1110 // allows us to catch things like: "p->p->array"1111 // 3. CompoundLiteralExpr - This is for people who create something1112 // heretical like (struct foo has a flexible array member):1113 //1114 // (struct foo){ 1, 2 }.blah[idx];1115 const Expr *VisitDeclRefExpr(const DeclRefExpr *E) {1116 return IsExpectedRecordDecl(E) ? E : nullptr;1117 }1118 const Expr *VisitMemberExpr(const MemberExpr *E) {1119 if (IsExpectedRecordDecl(E) && E->isArrow())1120 return E;1121 const Expr *Res = Visit(E->getBase());1122 return !Res && IsExpectedRecordDecl(E) ? E : Res;1123 }1124 const Expr *VisitCompoundLiteralExpr(const CompoundLiteralExpr *E) {1125 return IsExpectedRecordDecl(E) ? E : nullptr;1126 }1127 const Expr *VisitCallExpr(const CallExpr *E) {1128 return IsExpectedRecordDecl(E) ? E : nullptr;1129 }1130 1131 const Expr *VisitArraySubscriptExpr(const ArraySubscriptExpr *E) {1132 if (IsExpectedRecordDecl(E))1133 return E;1134 return Visit(E->getBase());1135 }1136 const Expr *VisitCastExpr(const CastExpr *E) {1137 if (E->getCastKind() == CK_LValueToRValue)1138 return IsExpectedRecordDecl(E) ? E : nullptr;1139 return Visit(E->getSubExpr());1140 }1141 const Expr *VisitParenExpr(const ParenExpr *E) {1142 return Visit(E->getSubExpr());1143 }1144 const Expr *VisitUnaryAddrOf(const UnaryOperator *E) {1145 return Visit(E->getSubExpr());1146 }1147 const Expr *VisitUnaryDeref(const UnaryOperator *E) {1148 return Visit(E->getSubExpr());1149 }1150};1151 1152} // end anonymous namespace1153 1154using RecIndicesTy = SmallVector<llvm::Value *, 8>;1155 1156static bool getGEPIndicesToField(CodeGenFunction &CGF, const RecordDecl *RD,1157 const FieldDecl *Field,1158 RecIndicesTy &Indices) {1159 const CGRecordLayout &Layout = CGF.CGM.getTypes().getCGRecordLayout(RD);1160 int64_t FieldNo = -1;1161 for (const FieldDecl *FD : RD->fields()) {1162 if (!Layout.containsFieldDecl(FD))1163 // This could happen if the field has a struct type that's empty. I don't1164 // know why either.1165 continue;1166 1167 FieldNo = Layout.getLLVMFieldNo(FD);1168 if (FD == Field) {1169 Indices.emplace_back(CGF.Builder.getInt32(FieldNo));1170 return true;1171 }1172 1173 QualType Ty = FD->getType();1174 if (Ty->isRecordType()) {1175 if (getGEPIndicesToField(CGF, Ty->getAsRecordDecl(), Field, Indices)) {1176 if (RD->isUnion())1177 FieldNo = 0;1178 Indices.emplace_back(CGF.Builder.getInt32(FieldNo));1179 return true;1180 }1181 }1182 }1183 1184 return false;1185}1186 1187llvm::Value *CodeGenFunction::GetCountedByFieldExprGEP(1188 const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl) {1189 const RecordDecl *RD = CountDecl->getParent()->getOuterLexicalRecordContext();1190 1191 // Find the base struct expr (i.e. p in p->a.b.c.d).1192 const Expr *StructBase = StructAccessBase(RD).Visit(Base);1193 if (!StructBase || StructBase->HasSideEffects(getContext()))1194 return nullptr;1195 1196 llvm::Value *Res = nullptr;1197 if (StructBase->getType()->isPointerType()) {1198 LValueBaseInfo BaseInfo;1199 TBAAAccessInfo TBAAInfo;1200 Address Addr = EmitPointerWithAlignment(StructBase, &BaseInfo, &TBAAInfo);1201 Res = Addr.emitRawPointer(*this);1202 } else if (StructBase->isLValue()) {1203 LValue LV = EmitLValue(StructBase);1204 Address Addr = LV.getAddress();1205 Res = Addr.emitRawPointer(*this);1206 } else {1207 return nullptr;1208 }1209 1210 RecIndicesTy Indices;1211 getGEPIndicesToField(*this, RD, CountDecl, Indices);1212 if (Indices.empty())1213 return nullptr;1214 1215 Indices.push_back(Builder.getInt32(0));1216 CanQualType T = CGM.getContext().getCanonicalTagType(RD);1217 return Builder.CreateInBoundsGEP(ConvertType(T), Res,1218 RecIndicesTy(llvm::reverse(Indices)),1219 "counted_by.gep");1220}1221 1222/// This method is typically called in contexts where we can't generate1223/// side-effects, like in __builtin_dynamic_object_size. When finding1224/// expressions, only choose those that have either already been emitted or can1225/// be loaded without side-effects.1226///1227/// - \p FAMDecl: the \p Decl for the flexible array member. It may not be1228/// within the top-level struct.1229/// - \p CountDecl: must be within the same non-anonymous struct as \p FAMDecl.1230llvm::Value *CodeGenFunction::EmitLoadOfCountedByField(1231 const Expr *Base, const FieldDecl *FAMDecl, const FieldDecl *CountDecl) {1232 if (llvm::Value *GEP = GetCountedByFieldExprGEP(Base, FAMDecl, CountDecl))1233 return Builder.CreateAlignedLoad(ConvertType(CountDecl->getType()), GEP,1234 getIntAlign(), "counted_by.load");1235 return nullptr;1236}1237 1238void CodeGenFunction::EmitBoundsCheck(const Expr *E, const Expr *Base,1239 llvm::Value *Index, QualType IndexType,1240 bool Accessed) {1241 assert(SanOpts.has(SanitizerKind::ArrayBounds) &&1242 "should not be called unless adding bounds checks");1243 const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =1244 getLangOpts().getStrictFlexArraysLevel();1245 QualType IndexedType;1246 llvm::Value *Bound =1247 getArrayIndexingBound(*this, Base, IndexedType, StrictFlexArraysLevel);1248 1249 EmitBoundsCheckImpl(E, Bound, Index, IndexType, IndexedType, Accessed);1250}1251 1252void CodeGenFunction::EmitBoundsCheckImpl(const Expr *E, llvm::Value *Bound,1253 llvm::Value *Index,1254 QualType IndexType,1255 QualType IndexedType, bool Accessed) {1256 if (!Bound)1257 return;1258 1259 auto CheckKind = SanitizerKind::SO_ArrayBounds;1260 auto CheckHandler = SanitizerHandler::OutOfBounds;1261 SanitizerDebugLocation SanScope(this, {CheckKind}, CheckHandler);1262 1263 bool IndexSigned = IndexType->isSignedIntegerOrEnumerationType();1264 llvm::Value *IndexVal = Builder.CreateIntCast(Index, SizeTy, IndexSigned);1265 llvm::Value *BoundVal = Builder.CreateIntCast(Bound, SizeTy, false);1266 1267 llvm::Constant *StaticData[] = {1268 EmitCheckSourceLocation(E->getExprLoc()),1269 EmitCheckTypeDescriptor(IndexedType),1270 EmitCheckTypeDescriptor(IndexType)1271 };1272 llvm::Value *Check = Accessed ? Builder.CreateICmpULT(IndexVal, BoundVal)1273 : Builder.CreateICmpULE(IndexVal, BoundVal);1274 EmitCheck(std::make_pair(Check, CheckKind), CheckHandler, StaticData, Index);1275}1276 1277llvm::MDNode *CodeGenFunction::buildAllocToken(QualType AllocType) {1278 auto ATMD = infer_alloc::getAllocTokenMetadata(AllocType, getContext());1279 if (!ATMD)1280 return nullptr;1281 1282 llvm::MDBuilder MDB(getLLVMContext());1283 auto *TypeNameMD = MDB.createString(ATMD->TypeName);1284 auto *ContainsPtrC = Builder.getInt1(ATMD->ContainsPointer);1285 auto *ContainsPtrMD = MDB.createConstant(ContainsPtrC);1286 1287 // Format: !{<type-name>, <contains-pointer>}1288 return llvm::MDNode::get(CGM.getLLVMContext(), {TypeNameMD, ContainsPtrMD});1289}1290 1291void CodeGenFunction::EmitAllocToken(llvm::CallBase *CB, QualType AllocType) {1292 assert(SanOpts.has(SanitizerKind::AllocToken) &&1293 "Only needed with -fsanitize=alloc-token");1294 CB->setMetadata(llvm::LLVMContext::MD_alloc_token,1295 buildAllocToken(AllocType));1296}1297 1298llvm::MDNode *CodeGenFunction::buildAllocToken(const CallExpr *E) {1299 QualType AllocType = infer_alloc::inferPossibleType(E, getContext(), CurCast);1300 if (!AllocType.isNull())1301 return buildAllocToken(AllocType);1302 return nullptr;1303}1304 1305void CodeGenFunction::EmitAllocToken(llvm::CallBase *CB, const CallExpr *E) {1306 assert(SanOpts.has(SanitizerKind::AllocToken) &&1307 "Only needed with -fsanitize=alloc-token");1308 if (llvm::MDNode *MDN = buildAllocToken(E))1309 CB->setMetadata(llvm::LLVMContext::MD_alloc_token, MDN);1310}1311 1312CodeGenFunction::ComplexPairTy CodeGenFunction::1313EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,1314 bool isInc, bool isPre) {1315 ComplexPairTy InVal = EmitLoadOfComplex(LV, E->getExprLoc());1316 1317 llvm::Value *NextVal;1318 if (isa<llvm::IntegerType>(InVal.first->getType())) {1319 uint64_t AmountVal = isInc ? 1 : -1;1320 NextVal = llvm::ConstantInt::get(InVal.first->getType(), AmountVal, true);1321 1322 // Add the inc/dec to the real part.1323 NextVal = Builder.CreateAdd(InVal.first, NextVal, isInc ? "inc" : "dec");1324 } else {1325 QualType ElemTy = E->getType()->castAs<ComplexType>()->getElementType();1326 llvm::APFloat FVal(getContext().getFloatTypeSemantics(ElemTy), 1);1327 if (!isInc)1328 FVal.changeSign();1329 NextVal = llvm::ConstantFP::get(getLLVMContext(), FVal);1330 1331 // Add the inc/dec to the real part.1332 NextVal = Builder.CreateFAdd(InVal.first, NextVal, isInc ? "inc" : "dec");1333 }1334 1335 ComplexPairTy IncVal(NextVal, InVal.second);1336 1337 // Store the updated result through the lvalue.1338 EmitStoreOfComplex(IncVal, LV, /*init*/ false);1339 if (getLangOpts().OpenMP)1340 CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(*this,1341 E->getSubExpr());1342 1343 // If this is a postinc, return the value read from memory, otherwise use the1344 // updated value.1345 return isPre ? IncVal : InVal;1346}1347 1348void CodeGenModule::EmitExplicitCastExprType(const ExplicitCastExpr *E,1349 CodeGenFunction *CGF) {1350 // Bind VLAs in the cast type.1351 if (CGF && E->getType()->isVariablyModifiedType())1352 CGF->EmitVariablyModifiedType(E->getType());1353 1354 if (CGDebugInfo *DI = getModuleDebugInfo())1355 DI->EmitExplicitCastType(E->getType());1356}1357 1358//===----------------------------------------------------------------------===//1359// LValue Expression Emission1360//===----------------------------------------------------------------------===//1361 1362static CharUnits getArrayElementAlign(CharUnits arrayAlign, llvm::Value *idx,1363 CharUnits eltSize) {1364 // If we have a constant index, we can use the exact offset of the1365 // element we're accessing.1366 if (auto *constantIdx = dyn_cast<llvm::ConstantInt>(idx)) {1367 CharUnits offset = constantIdx->getZExtValue() * eltSize;1368 return arrayAlign.alignmentAtOffset(offset);1369 }1370 1371 // Otherwise, use the worst-case alignment for any element.1372 return arrayAlign.alignmentOfArrayElement(eltSize);1373}1374 1375/// Emit pointer + index arithmetic.1376static Address emitPointerArithmetic(CodeGenFunction &CGF,1377 const BinaryOperator *BO,1378 LValueBaseInfo *BaseInfo,1379 TBAAAccessInfo *TBAAInfo,1380 KnownNonNull_t IsKnownNonNull) {1381 assert(BO->isAdditiveOp() && "Expect an addition or subtraction.");1382 Expr *pointerOperand = BO->getLHS();1383 Expr *indexOperand = BO->getRHS();1384 bool isSubtraction = BO->getOpcode() == BO_Sub;1385 1386 Address BaseAddr = Address::invalid();1387 llvm::Value *index = nullptr;1388 // In a subtraction, the LHS is always the pointer.1389 // Note: do not change the evaluation order.1390 if (!isSubtraction && !pointerOperand->getType()->isAnyPointerType()) {1391 std::swap(pointerOperand, indexOperand);1392 index = CGF.EmitScalarExpr(indexOperand);1393 BaseAddr = CGF.EmitPointerWithAlignment(pointerOperand, BaseInfo, TBAAInfo,1394 NotKnownNonNull);1395 } else {1396 BaseAddr = CGF.EmitPointerWithAlignment(pointerOperand, BaseInfo, TBAAInfo,1397 NotKnownNonNull);1398 index = CGF.EmitScalarExpr(indexOperand);1399 }1400 1401 llvm::Value *pointer = BaseAddr.getBasePointer();1402 llvm::Value *Res = CGF.EmitPointerArithmetic(1403 BO, pointerOperand, pointer, indexOperand, index, isSubtraction);1404 QualType PointeeTy = BO->getType()->getPointeeType();1405 CharUnits Align =1406 getArrayElementAlign(BaseAddr.getAlignment(), index,1407 CGF.getContext().getTypeSizeInChars(PointeeTy));1408 return Address(Res, CGF.ConvertTypeForMem(PointeeTy), Align,1409 CGF.CGM.getPointerAuthInfoForPointeeType(PointeeTy),1410 /*Offset=*/nullptr, IsKnownNonNull);1411}1412 1413static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo,1414 TBAAAccessInfo *TBAAInfo,1415 KnownNonNull_t IsKnownNonNull,1416 CodeGenFunction &CGF) {1417 // We allow this with ObjC object pointers because of fragile ABIs.1418 assert(E->getType()->isPointerType() ||1419 E->getType()->isObjCObjectPointerType());1420 E = E->IgnoreParens();1421 1422 // Casts:1423 if (const CastExpr *CE = dyn_cast<CastExpr>(E)) {1424 if (const auto *ECE = dyn_cast<ExplicitCastExpr>(CE))1425 CGF.CGM.EmitExplicitCastExprType(ECE, &CGF);1426 1427 switch (CE->getCastKind()) {1428 // Non-converting casts (but not C's implicit conversion from void*).1429 case CK_BitCast:1430 case CK_NoOp:1431 case CK_AddressSpaceConversion:1432 if (auto PtrTy = CE->getSubExpr()->getType()->getAs<PointerType>()) {1433 if (PtrTy->getPointeeType()->isVoidType())1434 break;1435 1436 LValueBaseInfo InnerBaseInfo;1437 TBAAAccessInfo InnerTBAAInfo;1438 Address Addr = CGF.EmitPointerWithAlignment(1439 CE->getSubExpr(), &InnerBaseInfo, &InnerTBAAInfo, IsKnownNonNull);1440 if (BaseInfo) *BaseInfo = InnerBaseInfo;1441 if (TBAAInfo) *TBAAInfo = InnerTBAAInfo;1442 1443 if (isa<ExplicitCastExpr>(CE)) {1444 LValueBaseInfo TargetTypeBaseInfo;1445 TBAAAccessInfo TargetTypeTBAAInfo;1446 CharUnits Align = CGF.CGM.getNaturalPointeeTypeAlignment(1447 E->getType(), &TargetTypeBaseInfo, &TargetTypeTBAAInfo);1448 if (TBAAInfo)1449 *TBAAInfo =1450 CGF.CGM.mergeTBAAInfoForCast(*TBAAInfo, TargetTypeTBAAInfo);1451 // If the source l-value is opaque, honor the alignment of the1452 // casted-to type.1453 if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) {1454 if (BaseInfo)1455 BaseInfo->mergeForCast(TargetTypeBaseInfo);1456 Addr.setAlignment(Align);1457 }1458 }1459 1460 if (CGF.SanOpts.has(SanitizerKind::CFIUnrelatedCast) &&1461 CE->getCastKind() == CK_BitCast) {1462 if (auto PT = E->getType()->getAs<PointerType>())1463 CGF.EmitVTablePtrCheckForCast(PT->getPointeeType(), Addr,1464 /*MayBeNull=*/true,1465 CodeGenFunction::CFITCK_UnrelatedCast,1466 CE->getBeginLoc());1467 }1468 1469 llvm::Type *ElemTy =1470 CGF.ConvertTypeForMem(E->getType()->getPointeeType());1471 Addr = Addr.withElementType(ElemTy);1472 if (CE->getCastKind() == CK_AddressSpaceConversion)1473 Addr = CGF.Builder.CreateAddrSpaceCast(1474 Addr, CGF.ConvertType(E->getType()), ElemTy);1475 1476 return CGF.authPointerToPointerCast(Addr, CE->getSubExpr()->getType(),1477 CE->getType());1478 }1479 break;1480 1481 // Array-to-pointer decay.1482 case CK_ArrayToPointerDecay:1483 return CGF.EmitArrayToPointerDecay(CE->getSubExpr(), BaseInfo, TBAAInfo);1484 1485 // Derived-to-base conversions.1486 case CK_UncheckedDerivedToBase:1487 case CK_DerivedToBase: {1488 // TODO: Support accesses to members of base classes in TBAA. For now, we1489 // conservatively pretend that the complete object is of the base class1490 // type.1491 if (TBAAInfo)1492 *TBAAInfo = CGF.CGM.getTBAAAccessInfo(E->getType());1493 Address Addr = CGF.EmitPointerWithAlignment(1494 CE->getSubExpr(), BaseInfo, nullptr,1495 (KnownNonNull_t)(IsKnownNonNull ||1496 CE->getCastKind() == CK_UncheckedDerivedToBase));1497 auto Derived = CE->getSubExpr()->getType()->getPointeeCXXRecordDecl();1498 return CGF.GetAddressOfBaseClass(1499 Addr, Derived, CE->path_begin(), CE->path_end(),1500 CGF.ShouldNullCheckClassCastValue(CE), CE->getExprLoc());1501 }1502 1503 // TODO: Is there any reason to treat base-to-derived conversions1504 // specially?1505 default:1506 break;1507 }1508 }1509 1510 // Unary &.1511 if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(E)) {1512 if (UO->getOpcode() == UO_AddrOf) {1513 LValue LV = CGF.EmitLValue(UO->getSubExpr(), IsKnownNonNull);1514 if (BaseInfo) *BaseInfo = LV.getBaseInfo();1515 if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();1516 return LV.getAddress();1517 }1518 }1519 1520 // std::addressof and variants.1521 if (auto *Call = dyn_cast<CallExpr>(E)) {1522 switch (Call->getBuiltinCallee()) {1523 default:1524 break;1525 case Builtin::BIaddressof:1526 case Builtin::BI__addressof:1527 case Builtin::BI__builtin_addressof: {1528 LValue LV = CGF.EmitLValue(Call->getArg(0), IsKnownNonNull);1529 if (BaseInfo) *BaseInfo = LV.getBaseInfo();1530 if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();1531 return LV.getAddress();1532 }1533 }1534 }1535 1536 // Pointer arithmetic: pointer +/- index.1537 if (auto *BO = dyn_cast<BinaryOperator>(E)) {1538 if (BO->isAdditiveOp())1539 return emitPointerArithmetic(CGF, BO, BaseInfo, TBAAInfo, IsKnownNonNull);1540 }1541 1542 // TODO: conditional operators, comma.1543 1544 // Otherwise, use the alignment of the type.1545 return CGF.makeNaturalAddressForPointer(1546 CGF.EmitScalarExpr(E), E->getType()->getPointeeType(), CharUnits(),1547 /*ForPointeeType=*/true, BaseInfo, TBAAInfo, IsKnownNonNull);1548}1549 1550/// EmitPointerWithAlignment - Given an expression of pointer type, try to1551/// derive a more accurate bound on the alignment of the pointer.1552Address CodeGenFunction::EmitPointerWithAlignment(1553 const Expr *E, LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo,1554 KnownNonNull_t IsKnownNonNull) {1555 Address Addr =1556 ::EmitPointerWithAlignment(E, BaseInfo, TBAAInfo, IsKnownNonNull, *this);1557 if (IsKnownNonNull && !Addr.isKnownNonNull())1558 Addr.setKnownNonNull();1559 return Addr;1560}1561 1562llvm::Value *CodeGenFunction::EmitNonNullRValueCheck(RValue RV, QualType T) {1563 llvm::Value *V = RV.getScalarVal();1564 if (auto MPT = T->getAs<MemberPointerType>())1565 return CGM.getCXXABI().EmitMemberPointerIsNotNull(*this, V, MPT);1566 return Builder.CreateICmpNE(V, llvm::Constant::getNullValue(V->getType()));1567}1568 1569RValue CodeGenFunction::GetUndefRValue(QualType Ty) {1570 if (Ty->isVoidType())1571 return RValue::get(nullptr);1572 1573 switch (getEvaluationKind(Ty)) {1574 case TEK_Complex: {1575 llvm::Type *EltTy =1576 ConvertType(Ty->castAs<ComplexType>()->getElementType());1577 llvm::Value *U = llvm::UndefValue::get(EltTy);1578 return RValue::getComplex(std::make_pair(U, U));1579 }1580 1581 // If this is a use of an undefined aggregate type, the aggregate must have an1582 // identifiable address. Just because the contents of the value are undefined1583 // doesn't mean that the address can't be taken and compared.1584 case TEK_Aggregate: {1585 Address DestPtr = CreateMemTemp(Ty, "undef.agg.tmp");1586 return RValue::getAggregate(DestPtr);1587 }1588 1589 case TEK_Scalar:1590 return RValue::get(llvm::UndefValue::get(ConvertType(Ty)));1591 }1592 llvm_unreachable("bad evaluation kind");1593}1594 1595RValue CodeGenFunction::EmitUnsupportedRValue(const Expr *E,1596 const char *Name) {1597 ErrorUnsupported(E, Name);1598 return GetUndefRValue(E->getType());1599}1600 1601LValue CodeGenFunction::EmitUnsupportedLValue(const Expr *E,1602 const char *Name) {1603 ErrorUnsupported(E, Name);1604 llvm::Type *ElTy = ConvertType(E->getType());1605 llvm::Type *Ty = DefaultPtrTy;1606 return MakeAddrLValue(1607 Address(llvm::UndefValue::get(Ty), ElTy, CharUnits::One()), E->getType());1608}1609 1610bool CodeGenFunction::IsWrappedCXXThis(const Expr *Obj) {1611 const Expr *Base = Obj;1612 while (!isa<CXXThisExpr>(Base)) {1613 // The result of a dynamic_cast can be null.1614 if (isa<CXXDynamicCastExpr>(Base))1615 return false;1616 1617 if (const auto *CE = dyn_cast<CastExpr>(Base)) {1618 Base = CE->getSubExpr();1619 } else if (const auto *PE = dyn_cast<ParenExpr>(Base)) {1620 Base = PE->getSubExpr();1621 } else if (const auto *UO = dyn_cast<UnaryOperator>(Base)) {1622 if (UO->getOpcode() == UO_Extension)1623 Base = UO->getSubExpr();1624 else1625 return false;1626 } else {1627 return false;1628 }1629 }1630 return true;1631}1632 1633LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) {1634 LValue LV;1635 if (SanOpts.has(SanitizerKind::ArrayBounds) && isa<ArraySubscriptExpr>(E))1636 LV = EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E), /*Accessed*/true);1637 else1638 LV = EmitLValue(E);1639 if (!isa<DeclRefExpr>(E) && !LV.isBitField() && LV.isSimple()) {1640 SanitizerSet SkippedChecks;1641 if (const auto *ME = dyn_cast<MemberExpr>(E)) {1642 bool IsBaseCXXThis = IsWrappedCXXThis(ME->getBase());1643 if (IsBaseCXXThis)1644 SkippedChecks.set(SanitizerKind::Alignment, true);1645 if (IsBaseCXXThis || isa<DeclRefExpr>(ME->getBase()))1646 SkippedChecks.set(SanitizerKind::Null, true);1647 }1648 EmitTypeCheck(TCK, E->getExprLoc(), LV, E->getType(), SkippedChecks);1649 }1650 return LV;1651}1652 1653/// EmitLValue - Emit code to compute a designator that specifies the location1654/// of the expression.1655///1656/// This can return one of two things: a simple address or a bitfield reference.1657/// In either case, the LLVM Value* in the LValue structure is guaranteed to be1658/// an LLVM pointer type.1659///1660/// If this returns a bitfield reference, nothing about the pointee type of the1661/// LLVM value is known: For example, it may not be a pointer to an integer.1662///1663/// If this returns a normal address, and if the lvalue's C type is fixed size,1664/// this method guarantees that the returned pointer type will point to an LLVM1665/// type of the same size of the lvalue's type. If the lvalue has a variable1666/// length type, this is not possible.1667///1668LValue CodeGenFunction::EmitLValue(const Expr *E,1669 KnownNonNull_t IsKnownNonNull) {1670 // Running with sufficient stack space to avoid deeply nested expressions1671 // cause a stack overflow.1672 LValue LV;1673 CGM.runWithSufficientStackSpace(1674 E->getExprLoc(), [&] { LV = EmitLValueHelper(E, IsKnownNonNull); });1675 1676 if (IsKnownNonNull && !LV.isKnownNonNull())1677 LV.setKnownNonNull();1678 return LV;1679}1680 1681static QualType getConstantExprReferredType(const FullExpr *E,1682 const ASTContext &Ctx) {1683 const Expr *SE = E->getSubExpr()->IgnoreImplicit();1684 if (isa<OpaqueValueExpr>(SE))1685 return SE->getType();1686 return cast<CallExpr>(SE)->getCallReturnType(Ctx)->getPointeeType();1687}1688 1689LValue CodeGenFunction::EmitLValueHelper(const Expr *E,1690 KnownNonNull_t IsKnownNonNull) {1691 ApplyDebugLocation DL(*this, E);1692 switch (E->getStmtClass()) {1693 default: return EmitUnsupportedLValue(E, "l-value expression");1694 1695 case Expr::ObjCPropertyRefExprClass:1696 llvm_unreachable("cannot emit a property reference directly");1697 1698 case Expr::ObjCSelectorExprClass:1699 return EmitObjCSelectorLValue(cast<ObjCSelectorExpr>(E));1700 case Expr::ObjCIsaExprClass:1701 return EmitObjCIsaExpr(cast<ObjCIsaExpr>(E));1702 case Expr::BinaryOperatorClass:1703 return EmitBinaryOperatorLValue(cast<BinaryOperator>(E));1704 case Expr::CompoundAssignOperatorClass: {1705 QualType Ty = E->getType();1706 if (const AtomicType *AT = Ty->getAs<AtomicType>())1707 Ty = AT->getValueType();1708 if (!Ty->isAnyComplexType())1709 return EmitCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));1710 return EmitComplexCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));1711 }1712 case Expr::CallExprClass:1713 case Expr::CXXMemberCallExprClass:1714 case Expr::CXXOperatorCallExprClass:1715 case Expr::UserDefinedLiteralClass:1716 return EmitCallExprLValue(cast<CallExpr>(E));1717 case Expr::CXXRewrittenBinaryOperatorClass:1718 return EmitLValue(cast<CXXRewrittenBinaryOperator>(E)->getSemanticForm(),1719 IsKnownNonNull);1720 case Expr::VAArgExprClass:1721 return EmitVAArgExprLValue(cast<VAArgExpr>(E));1722 case Expr::DeclRefExprClass:1723 return EmitDeclRefLValue(cast<DeclRefExpr>(E));1724 case Expr::ConstantExprClass: {1725 const ConstantExpr *CE = cast<ConstantExpr>(E);1726 if (llvm::Value *Result = ConstantEmitter(*this).tryEmitConstantExpr(CE)) {1727 QualType RetType = getConstantExprReferredType(CE, getContext());1728 return MakeNaturalAlignAddrLValue(Result, RetType);1729 }1730 return EmitLValue(cast<ConstantExpr>(E)->getSubExpr(), IsKnownNonNull);1731 }1732 case Expr::ParenExprClass:1733 return EmitLValue(cast<ParenExpr>(E)->getSubExpr(), IsKnownNonNull);1734 case Expr::GenericSelectionExprClass:1735 return EmitLValue(cast<GenericSelectionExpr>(E)->getResultExpr(),1736 IsKnownNonNull);1737 case Expr::PredefinedExprClass:1738 return EmitPredefinedLValue(cast<PredefinedExpr>(E));1739 case Expr::StringLiteralClass:1740 return EmitStringLiteralLValue(cast<StringLiteral>(E));1741 case Expr::ObjCEncodeExprClass:1742 return EmitObjCEncodeExprLValue(cast<ObjCEncodeExpr>(E));1743 case Expr::PseudoObjectExprClass:1744 return EmitPseudoObjectLValue(cast<PseudoObjectExpr>(E));1745 case Expr::InitListExprClass:1746 return EmitInitListLValue(cast<InitListExpr>(E));1747 case Expr::CXXTemporaryObjectExprClass:1748 case Expr::CXXConstructExprClass:1749 return EmitCXXConstructLValue(cast<CXXConstructExpr>(E));1750 case Expr::CXXBindTemporaryExprClass:1751 return EmitCXXBindTemporaryLValue(cast<CXXBindTemporaryExpr>(E));1752 case Expr::CXXUuidofExprClass:1753 return EmitCXXUuidofLValue(cast<CXXUuidofExpr>(E));1754 case Expr::LambdaExprClass:1755 return EmitAggExprToLValue(E);1756 1757 case Expr::ExprWithCleanupsClass: {1758 const auto *cleanups = cast<ExprWithCleanups>(E);1759 RunCleanupsScope Scope(*this);1760 LValue LV = EmitLValue(cleanups->getSubExpr(), IsKnownNonNull);1761 if (LV.isSimple()) {1762 // Defend against branches out of gnu statement expressions surrounded by1763 // cleanups.1764 Address Addr = LV.getAddress();1765 llvm::Value *V = Addr.getBasePointer();1766 Scope.ForceCleanup({&V});1767 Addr.replaceBasePointer(V);1768 return LValue::MakeAddr(Addr, LV.getType(), getContext(),1769 LV.getBaseInfo(), LV.getTBAAInfo());1770 }1771 // FIXME: Is it possible to create an ExprWithCleanups that produces a1772 // bitfield lvalue or some other non-simple lvalue?1773 return LV;1774 }1775 1776 case Expr::CXXDefaultArgExprClass: {1777 auto *DAE = cast<CXXDefaultArgExpr>(E);1778 CXXDefaultArgExprScope Scope(*this, DAE);1779 return EmitLValue(DAE->getExpr(), IsKnownNonNull);1780 }1781 case Expr::CXXDefaultInitExprClass: {1782 auto *DIE = cast<CXXDefaultInitExpr>(E);1783 CXXDefaultInitExprScope Scope(*this, DIE);1784 return EmitLValue(DIE->getExpr(), IsKnownNonNull);1785 }1786 case Expr::CXXTypeidExprClass:1787 return EmitCXXTypeidLValue(cast<CXXTypeidExpr>(E));1788 1789 case Expr::ObjCMessageExprClass:1790 return EmitObjCMessageExprLValue(cast<ObjCMessageExpr>(E));1791 case Expr::ObjCIvarRefExprClass:1792 return EmitObjCIvarRefLValue(cast<ObjCIvarRefExpr>(E));1793 case Expr::StmtExprClass:1794 return EmitStmtExprLValue(cast<StmtExpr>(E));1795 case Expr::UnaryOperatorClass:1796 return EmitUnaryOpLValue(cast<UnaryOperator>(E));1797 case Expr::ArraySubscriptExprClass:1798 return EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E));1799 case Expr::MatrixSubscriptExprClass:1800 return EmitMatrixSubscriptExpr(cast<MatrixSubscriptExpr>(E));1801 case Expr::ArraySectionExprClass:1802 return EmitArraySectionExpr(cast<ArraySectionExpr>(E));1803 case Expr::ExtVectorElementExprClass:1804 return EmitExtVectorElementExpr(cast<ExtVectorElementExpr>(E));1805 case Expr::CXXThisExprClass:1806 return MakeAddrLValue(LoadCXXThisAddress(), E->getType());1807 case Expr::MemberExprClass:1808 return EmitMemberExpr(cast<MemberExpr>(E));1809 case Expr::CompoundLiteralExprClass:1810 return EmitCompoundLiteralLValue(cast<CompoundLiteralExpr>(E));1811 case Expr::ConditionalOperatorClass:1812 return EmitConditionalOperatorLValue(cast<ConditionalOperator>(E));1813 case Expr::BinaryConditionalOperatorClass:1814 return EmitConditionalOperatorLValue(cast<BinaryConditionalOperator>(E));1815 case Expr::ChooseExprClass:1816 return EmitLValue(cast<ChooseExpr>(E)->getChosenSubExpr(), IsKnownNonNull);1817 case Expr::OpaqueValueExprClass:1818 return EmitOpaqueValueLValue(cast<OpaqueValueExpr>(E));1819 case Expr::SubstNonTypeTemplateParmExprClass:1820 return EmitLValue(cast<SubstNonTypeTemplateParmExpr>(E)->getReplacement(),1821 IsKnownNonNull);1822 case Expr::ImplicitCastExprClass:1823 case Expr::CStyleCastExprClass:1824 case Expr::CXXFunctionalCastExprClass:1825 case Expr::CXXStaticCastExprClass:1826 case Expr::CXXDynamicCastExprClass:1827 case Expr::CXXReinterpretCastExprClass:1828 case Expr::CXXConstCastExprClass:1829 case Expr::CXXAddrspaceCastExprClass:1830 case Expr::ObjCBridgedCastExprClass:1831 return EmitCastLValue(cast<CastExpr>(E));1832 1833 case Expr::MaterializeTemporaryExprClass:1834 return EmitMaterializeTemporaryExpr(cast<MaterializeTemporaryExpr>(E));1835 1836 case Expr::CoawaitExprClass:1837 return EmitCoawaitLValue(cast<CoawaitExpr>(E));1838 case Expr::CoyieldExprClass:1839 return EmitCoyieldLValue(cast<CoyieldExpr>(E));1840 case Expr::PackIndexingExprClass:1841 return EmitLValue(cast<PackIndexingExpr>(E)->getSelectedExpr());1842 case Expr::HLSLOutArgExprClass:1843 llvm_unreachable("cannot emit a HLSL out argument directly");1844 }1845}1846 1847/// Given an object of the given canonical type, can we safely copy a1848/// value out of it based on its initializer?1849static bool isConstantEmittableObjectType(QualType type) {1850 assert(type.isCanonical());1851 assert(!type->isReferenceType());1852 1853 // Must be const-qualified but non-volatile.1854 Qualifiers qs = type.getLocalQualifiers();1855 if (!qs.hasConst() || qs.hasVolatile()) return false;1856 1857 // Otherwise, all object types satisfy this except C++ classes with1858 // mutable subobjects or non-trivial copy/destroy behavior.1859 if (const auto *RT = dyn_cast<RecordType>(type))1860 if (const auto *RD = dyn_cast<CXXRecordDecl>(RT->getDecl())) {1861 RD = RD->getDefinitionOrSelf();1862 if (RD->hasMutableFields() || !RD->isTrivial())1863 return false;1864 }1865 1866 return true;1867}1868 1869/// Can we constant-emit a load of a reference to a variable of the1870/// given type? This is different from predicates like1871/// Decl::mightBeUsableInConstantExpressions because we do want it to apply1872/// in situations that don't necessarily satisfy the language's rules1873/// for this (e.g. C++'s ODR-use rules). For example, we want to able1874/// to do this with const float variables even if those variables1875/// aren't marked 'constexpr'.1876enum ConstantEmissionKind {1877 CEK_None,1878 CEK_AsReferenceOnly,1879 CEK_AsValueOrReference,1880 CEK_AsValueOnly1881};1882static ConstantEmissionKind checkVarTypeForConstantEmission(QualType type) {1883 type = type.getCanonicalType();1884 if (const auto *ref = dyn_cast<ReferenceType>(type)) {1885 if (isConstantEmittableObjectType(ref->getPointeeType()))1886 return CEK_AsValueOrReference;1887 return CEK_AsReferenceOnly;1888 }1889 if (isConstantEmittableObjectType(type))1890 return CEK_AsValueOnly;1891 return CEK_None;1892}1893 1894/// Try to emit a reference to the given value without producing it as1895/// an l-value. This is just an optimization, but it avoids us needing1896/// to emit global copies of variables if they're named without triggering1897/// a formal use in a context where we can't emit a direct reference to them,1898/// for instance if a block or lambda or a member of a local class uses a1899/// const int variable or constexpr variable from an enclosing function.1900CodeGenFunction::ConstantEmission1901CodeGenFunction::tryEmitAsConstant(const DeclRefExpr *RefExpr) {1902 const ValueDecl *Value = RefExpr->getDecl();1903 1904 // The value needs to be an enum constant or a constant variable.1905 ConstantEmissionKind CEK;1906 if (isa<ParmVarDecl>(Value)) {1907 CEK = CEK_None;1908 } else if (const auto *var = dyn_cast<VarDecl>(Value)) {1909 CEK = checkVarTypeForConstantEmission(var->getType());1910 } else if (isa<EnumConstantDecl>(Value)) {1911 CEK = CEK_AsValueOnly;1912 } else {1913 CEK = CEK_None;1914 }1915 if (CEK == CEK_None) return ConstantEmission();1916 1917 Expr::EvalResult result;1918 bool resultIsReference;1919 QualType resultType;1920 1921 // It's best to evaluate all the way as an r-value if that's permitted.1922 if (CEK != CEK_AsReferenceOnly &&1923 RefExpr->EvaluateAsRValue(result, getContext())) {1924 resultIsReference = false;1925 resultType = RefExpr->getType().getUnqualifiedType();1926 1927 // Otherwise, try to evaluate as an l-value.1928 } else if (CEK != CEK_AsValueOnly &&1929 RefExpr->EvaluateAsLValue(result, getContext())) {1930 resultIsReference = true;1931 resultType = Value->getType();1932 1933 // Failure.1934 } else {1935 return ConstantEmission();1936 }1937 1938 // In any case, if the initializer has side-effects, abandon ship.1939 if (result.HasSideEffects)1940 return ConstantEmission();1941 1942 // In CUDA/HIP device compilation, a lambda may capture a reference variable1943 // referencing a global host variable by copy. In this case the lambda should1944 // make a copy of the value of the global host variable. The DRE of the1945 // captured reference variable cannot be emitted as load from the host1946 // global variable as compile time constant, since the host variable is not1947 // accessible on device. The DRE of the captured reference variable has to be1948 // loaded from captures.1949 if (CGM.getLangOpts().CUDAIsDevice && result.Val.isLValue() &&1950 RefExpr->refersToEnclosingVariableOrCapture()) {1951 auto *MD = dyn_cast_or_null<CXXMethodDecl>(CurCodeDecl);1952 if (isLambdaMethod(MD) && MD->getOverloadedOperator() == OO_Call) {1953 const APValue::LValueBase &base = result.Val.getLValueBase();1954 if (const ValueDecl *D = base.dyn_cast<const ValueDecl *>()) {1955 if (const VarDecl *VD = dyn_cast<const VarDecl>(D)) {1956 if (!VD->hasAttr<CUDADeviceAttr>()) {1957 return ConstantEmission();1958 }1959 }1960 }1961 }1962 }1963 1964 // Emit as a constant.1965 llvm::Constant *C = ConstantEmitter(*this).emitAbstract(1966 RefExpr->getLocation(), result.Val, resultType);1967 1968 // Make sure we emit a debug reference to the global variable.1969 // This should probably fire even for1970 if (isa<VarDecl>(Value)) {1971 if (!getContext().DeclMustBeEmitted(cast<VarDecl>(Value)))1972 EmitDeclRefExprDbgValue(RefExpr, result.Val);1973 } else {1974 assert(isa<EnumConstantDecl>(Value));1975 EmitDeclRefExprDbgValue(RefExpr, result.Val);1976 }1977 1978 // If we emitted a reference constant, we need to dereference that.1979 if (resultIsReference)1980 return ConstantEmission::forReference(C);1981 1982 return ConstantEmission::forValue(C);1983}1984 1985static DeclRefExpr *tryToConvertMemberExprToDeclRefExpr(CodeGenFunction &CGF,1986 const MemberExpr *ME) {1987 if (auto *VD = dyn_cast<VarDecl>(ME->getMemberDecl())) {1988 // Try to emit static variable member expressions as DREs.1989 return DeclRefExpr::Create(1990 CGF.getContext(), NestedNameSpecifierLoc(), SourceLocation(), VD,1991 /*RefersToEnclosingVariableOrCapture=*/false, ME->getExprLoc(),1992 ME->getType(), ME->getValueKind(), nullptr, nullptr, ME->isNonOdrUse());1993 }1994 return nullptr;1995}1996 1997CodeGenFunction::ConstantEmission1998CodeGenFunction::tryEmitAsConstant(const MemberExpr *ME) {1999 if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(*this, ME))2000 return tryEmitAsConstant(DRE);2001 return ConstantEmission();2002}2003 2004llvm::Value *CodeGenFunction::emitScalarConstant(2005 const CodeGenFunction::ConstantEmission &Constant, Expr *E) {2006 assert(Constant && "not a constant");2007 if (Constant.isReference())2008 return EmitLoadOfLValue(Constant.getReferenceLValue(*this, E),2009 E->getExprLoc())2010 .getScalarVal();2011 return Constant.getValue();2012}2013 2014llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue,2015 SourceLocation Loc) {2016 return EmitLoadOfScalar(lvalue.getAddress(), lvalue.isVolatile(),2017 lvalue.getType(), Loc, lvalue.getBaseInfo(),2018 lvalue.getTBAAInfo(), lvalue.isNontemporal());2019}2020 2021static bool getRangeForType(CodeGenFunction &CGF, QualType Ty,2022 llvm::APInt &Min, llvm::APInt &End,2023 bool StrictEnums, bool IsBool) {2024 const auto *ED = Ty->getAsEnumDecl();2025 bool IsRegularCPlusPlusEnum =2026 CGF.getLangOpts().CPlusPlus && StrictEnums && ED && !ED->isFixed();2027 if (!IsBool && !IsRegularCPlusPlusEnum)2028 return false;2029 2030 if (IsBool) {2031 Min = llvm::APInt(CGF.getContext().getTypeSize(Ty), 0);2032 End = llvm::APInt(CGF.getContext().getTypeSize(Ty), 2);2033 } else {2034 ED->getValueRange(End, Min);2035 }2036 return true;2037}2038 2039llvm::MDNode *CodeGenFunction::getRangeForLoadFromType(QualType Ty) {2040 llvm::APInt Min, End;2041 if (!getRangeForType(*this, Ty, Min, End, CGM.getCodeGenOpts().StrictEnums,2042 Ty->hasBooleanRepresentation() && !Ty->isVectorType()))2043 return nullptr;2044 2045 llvm::MDBuilder MDHelper(getLLVMContext());2046 return MDHelper.createRange(Min, End);2047}2048 2049void CodeGenFunction::maybeAttachRangeForLoad(llvm::LoadInst *Load, QualType Ty,2050 SourceLocation Loc) {2051 if (EmitScalarRangeCheck(Load, Ty, Loc)) {2052 // In order to prevent the optimizer from throwing away the check, don't2053 // attach range metadata to the load.2054 } else if (CGM.getCodeGenOpts().OptimizationLevel > 0) {2055 if (llvm::MDNode *RangeInfo = getRangeForLoadFromType(Ty)) {2056 Load->setMetadata(llvm::LLVMContext::MD_range, RangeInfo);2057 Load->setMetadata(llvm::LLVMContext::MD_noundef,2058 llvm::MDNode::get(CGM.getLLVMContext(), {}));2059 }2060 }2061}2062 2063bool CodeGenFunction::EmitScalarRangeCheck(llvm::Value *Value, QualType Ty,2064 SourceLocation Loc) {2065 bool HasBoolCheck = SanOpts.has(SanitizerKind::Bool);2066 bool HasEnumCheck = SanOpts.has(SanitizerKind::Enum);2067 if (!HasBoolCheck && !HasEnumCheck)2068 return false;2069 2070 bool IsBool = (Ty->hasBooleanRepresentation() && !Ty->isVectorType()) ||2071 NSAPI(CGM.getContext()).isObjCBOOLType(Ty);2072 bool NeedsBoolCheck = HasBoolCheck && IsBool;2073 bool NeedsEnumCheck = HasEnumCheck && Ty->isEnumeralType();2074 if (!NeedsBoolCheck && !NeedsEnumCheck)2075 return false;2076 2077 // Single-bit booleans don't need to be checked. Special-case this to avoid2078 // a bit width mismatch when handling bitfield values. This is handled by2079 // EmitFromMemory for the non-bitfield case.2080 if (IsBool &&2081 cast<llvm::IntegerType>(Value->getType())->getBitWidth() == 1)2082 return false;2083 2084 if (NeedsEnumCheck &&2085 getContext().isTypeIgnoredBySanitizer(SanitizerKind::Enum, Ty))2086 return false;2087 2088 llvm::APInt Min, End;2089 if (!getRangeForType(*this, Ty, Min, End, /*StrictEnums=*/true, IsBool))2090 return true;2091 2092 SanitizerKind::SanitizerOrdinal Kind =2093 NeedsEnumCheck ? SanitizerKind::SO_Enum : SanitizerKind::SO_Bool;2094 2095 auto &Ctx = getLLVMContext();2096 auto CheckHandler = SanitizerHandler::LoadInvalidValue;2097 SanitizerDebugLocation SanScope(this, {Kind}, CheckHandler);2098 llvm::Value *Check;2099 --End;2100 if (!Min) {2101 Check = Builder.CreateICmpULE(Value, llvm::ConstantInt::get(Ctx, End));2102 } else {2103 llvm::Value *Upper =2104 Builder.CreateICmpSLE(Value, llvm::ConstantInt::get(Ctx, End));2105 llvm::Value *Lower =2106 Builder.CreateICmpSGE(Value, llvm::ConstantInt::get(Ctx, Min));2107 Check = Builder.CreateAnd(Upper, Lower);2108 }2109 llvm::Constant *StaticArgs[] = {EmitCheckSourceLocation(Loc),2110 EmitCheckTypeDescriptor(Ty)};2111 EmitCheck(std::make_pair(Check, Kind), CheckHandler, StaticArgs, Value);2112 return true;2113}2114 2115llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,2116 QualType Ty,2117 SourceLocation Loc,2118 LValueBaseInfo BaseInfo,2119 TBAAAccessInfo TBAAInfo,2120 bool isNontemporal) {2121 if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getBasePointer()))2122 if (GV->isThreadLocal())2123 Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV),2124 NotKnownNonNull);2125 2126 if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {2127 // Boolean vectors use `iN` as storage type.2128 if (ClangVecTy->isPackedVectorBoolType(getContext())) {2129 llvm::Type *ValTy = ConvertType(Ty);2130 unsigned ValNumElems =2131 cast<llvm::FixedVectorType>(ValTy)->getNumElements();2132 // Load the `iP` storage object (P is the padded vector size).2133 auto *RawIntV = Builder.CreateLoad(Addr, Volatile, "load_bits");2134 const auto *RawIntTy = RawIntV->getType();2135 assert(RawIntTy->isIntegerTy() && "compressed iN storage for bitvectors");2136 // Bitcast iP --> <P x i1>.2137 auto *PaddedVecTy = llvm::FixedVectorType::get(2138 Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits());2139 llvm::Value *V = Builder.CreateBitCast(RawIntV, PaddedVecTy);2140 // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).2141 V = emitBoolVecConversion(V, ValNumElems, "extractvec");2142 2143 return EmitFromMemory(V, Ty);2144 }2145 2146 // Handles vectors of sizes that are likely to be expanded to a larger size2147 // to optimize performance.2148 auto *VTy = cast<llvm::FixedVectorType>(Addr.getElementType());2149 auto *NewVecTy =2150 CGM.getABIInfo().getOptimalVectorMemoryType(VTy, getLangOpts());2151 2152 if (VTy != NewVecTy) {2153 Address Cast = Addr.withElementType(NewVecTy);2154 llvm::Value *V = Builder.CreateLoad(Cast, Volatile, "loadVecN");2155 unsigned OldNumElements = VTy->getNumElements();2156 SmallVector<int, 16> Mask(OldNumElements);2157 std::iota(Mask.begin(), Mask.end(), 0);2158 V = Builder.CreateShuffleVector(V, Mask, "extractVec");2159 return EmitFromMemory(V, Ty);2160 }2161 }2162 2163 // Atomic operations have to be done on integral types.2164 LValue AtomicLValue =2165 LValue::MakeAddr(Addr, Ty, getContext(), BaseInfo, TBAAInfo);2166 if (Ty->isAtomicType() || LValueIsSuitableForInlineAtomic(AtomicLValue)) {2167 return EmitAtomicLoad(AtomicLValue, Loc).getScalarVal();2168 }2169 2170 Addr =2171 Addr.withElementType(convertTypeForLoadStore(Ty, Addr.getElementType()));2172 2173 llvm::LoadInst *Load = Builder.CreateLoad(Addr, Volatile);2174 if (isNontemporal) {2175 llvm::MDNode *Node = llvm::MDNode::get(2176 Load->getContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));2177 Load->setMetadata(llvm::LLVMContext::MD_nontemporal, Node);2178 }2179 2180 CGM.DecorateInstructionWithTBAA(Load, TBAAInfo);2181 2182 maybeAttachRangeForLoad(Load, Ty, Loc);2183 2184 return EmitFromMemory(Load, Ty);2185}2186 2187/// Converts a scalar value from its primary IR type (as returned2188/// by ConvertType) to its load/store type (as returned by2189/// convertTypeForLoadStore).2190llvm::Value *CodeGenFunction::EmitToMemory(llvm::Value *Value, QualType Ty) {2191 if (auto *AtomicTy = Ty->getAs<AtomicType>())2192 Ty = AtomicTy->getValueType();2193 2194 if (Ty->isExtVectorBoolType()) {2195 llvm::Type *StoreTy = convertTypeForLoadStore(Ty, Value->getType());2196 if (StoreTy->isVectorTy() && StoreTy->getScalarSizeInBits() >2197 Value->getType()->getScalarSizeInBits())2198 return Builder.CreateZExt(Value, StoreTy);2199 2200 // Expand to the memory bit width.2201 unsigned MemNumElems = StoreTy->getPrimitiveSizeInBits();2202 // <N x i1> --> <P x i1>.2203 Value = emitBoolVecConversion(Value, MemNumElems, "insertvec");2204 // <P x i1> --> iP.2205 Value = Builder.CreateBitCast(Value, StoreTy);2206 }2207 2208 if (Ty->hasBooleanRepresentation() || Ty->isBitIntType()) {2209 llvm::Type *StoreTy = convertTypeForLoadStore(Ty, Value->getType());2210 bool Signed = Ty->isSignedIntegerOrEnumerationType();2211 return Builder.CreateIntCast(Value, StoreTy, Signed, "storedv");2212 }2213 2214 return Value;2215}2216 2217/// Converts a scalar value from its load/store type (as returned2218/// by convertTypeForLoadStore) to its primary IR type (as returned2219/// by ConvertType).2220llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) {2221 if (auto *AtomicTy = Ty->getAs<AtomicType>())2222 Ty = AtomicTy->getValueType();2223 2224 if (Ty->isPackedVectorBoolType(getContext())) {2225 const auto *RawIntTy = Value->getType();2226 2227 // Bitcast iP --> <P x i1>.2228 auto *PaddedVecTy = llvm::FixedVectorType::get(2229 Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits());2230 auto *V = Builder.CreateBitCast(Value, PaddedVecTy);2231 // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).2232 llvm::Type *ValTy = ConvertType(Ty);2233 unsigned ValNumElems = cast<llvm::FixedVectorType>(ValTy)->getNumElements();2234 return emitBoolVecConversion(V, ValNumElems, "extractvec");2235 }2236 2237 llvm::Type *ResTy = ConvertType(Ty);2238 if (Ty->hasBooleanRepresentation() || Ty->isBitIntType() ||2239 Ty->isExtVectorBoolType())2240 return Builder.CreateTrunc(Value, ResTy, "loadedv");2241 2242 return Value;2243}2244 2245// Convert the pointer of \p Addr to a pointer to a vector (the value type of2246// MatrixType), if it points to a array (the memory type of MatrixType).2247static RawAddress MaybeConvertMatrixAddress(RawAddress Addr,2248 CodeGenFunction &CGF,2249 bool IsVector = true) {2250 auto *ArrayTy = dyn_cast<llvm::ArrayType>(Addr.getElementType());2251 if (ArrayTy && IsVector) {2252 auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(),2253 ArrayTy->getNumElements());2254 2255 return Addr.withElementType(VectorTy);2256 }2257 auto *VectorTy = dyn_cast<llvm::VectorType>(Addr.getElementType());2258 if (VectorTy && !IsVector) {2259 auto *ArrayTy = llvm::ArrayType::get(2260 VectorTy->getElementType(),2261 cast<llvm::FixedVectorType>(VectorTy)->getNumElements());2262 2263 return Addr.withElementType(ArrayTy);2264 }2265 2266 return Addr;2267}2268 2269// Emit a store of a matrix LValue. This may require casting the original2270// pointer to memory address (ArrayType) to a pointer to the value type2271// (VectorType).2272static void EmitStoreOfMatrixScalar(llvm::Value *value, LValue lvalue,2273 bool isInit, CodeGenFunction &CGF) {2274 Address Addr = MaybeConvertMatrixAddress(lvalue.getAddress(), CGF,2275 value->getType()->isVectorTy());2276 CGF.EmitStoreOfScalar(value, Addr, lvalue.isVolatile(), lvalue.getType(),2277 lvalue.getBaseInfo(), lvalue.getTBAAInfo(), isInit,2278 lvalue.isNontemporal());2279}2280 2281void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr,2282 bool Volatile, QualType Ty,2283 LValueBaseInfo BaseInfo,2284 TBAAAccessInfo TBAAInfo,2285 bool isInit, bool isNontemporal) {2286 if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getBasePointer()))2287 if (GV->isThreadLocal())2288 Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV),2289 NotKnownNonNull);2290 2291 // Handles vectors of sizes that are likely to be expanded to a larger size2292 // to optimize performance.2293 llvm::Type *SrcTy = Value->getType();2294 if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {2295 if (auto *VecTy = dyn_cast<llvm::FixedVectorType>(SrcTy)) {2296 auto *NewVecTy =2297 CGM.getABIInfo().getOptimalVectorMemoryType(VecTy, getLangOpts());2298 if (!ClangVecTy->isPackedVectorBoolType(getContext()) &&2299 VecTy != NewVecTy) {2300 SmallVector<int, 16> Mask(NewVecTy->getNumElements(),2301 VecTy->getNumElements());2302 std::iota(Mask.begin(), Mask.begin() + VecTy->getNumElements(), 0);2303 // Use undef instead of poison for the padding lanes, to make sure no2304 // padding bits are poisoned, which may break coercion.2305 Value = Builder.CreateShuffleVector(Value, llvm::UndefValue::get(VecTy),2306 Mask, "extractVec");2307 SrcTy = NewVecTy;2308 }2309 if (Addr.getElementType() != SrcTy)2310 Addr = Addr.withElementType(SrcTy);2311 }2312 }2313 2314 Value = EmitToMemory(Value, Ty);2315 2316 LValue AtomicLValue =2317 LValue::MakeAddr(Addr, Ty, getContext(), BaseInfo, TBAAInfo);2318 if (Ty->isAtomicType() ||2319 (!isInit && LValueIsSuitableForInlineAtomic(AtomicLValue))) {2320 EmitAtomicStore(RValue::get(Value), AtomicLValue, isInit);2321 return;2322 }2323 2324 llvm::StoreInst *Store = Builder.CreateStore(Value, Addr, Volatile);2325 addInstToCurrentSourceAtom(Store, Value);2326 2327 if (isNontemporal) {2328 llvm::MDNode *Node =2329 llvm::MDNode::get(Store->getContext(),2330 llvm::ConstantAsMetadata::get(Builder.getInt32(1)));2331 Store->setMetadata(llvm::LLVMContext::MD_nontemporal, Node);2332 }2333 2334 CGM.DecorateInstructionWithTBAA(Store, TBAAInfo);2335}2336 2337void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue,2338 bool isInit) {2339 if (lvalue.getType()->isConstantMatrixType()) {2340 EmitStoreOfMatrixScalar(value, lvalue, isInit, *this);2341 return;2342 }2343 2344 EmitStoreOfScalar(value, lvalue.getAddress(), lvalue.isVolatile(),2345 lvalue.getType(), lvalue.getBaseInfo(),2346 lvalue.getTBAAInfo(), isInit, lvalue.isNontemporal());2347}2348 2349// Emit a load of a LValue of matrix type. This may require casting the pointer2350// to memory address (ArrayType) to a pointer to the value type (VectorType).2351static RValue EmitLoadOfMatrixLValue(LValue LV, SourceLocation Loc,2352 CodeGenFunction &CGF) {2353 assert(LV.getType()->isConstantMatrixType());2354 Address Addr = MaybeConvertMatrixAddress(LV.getAddress(), CGF);2355 LV.setAddress(Addr);2356 return RValue::get(CGF.EmitLoadOfScalar(LV, Loc));2357}2358 2359RValue CodeGenFunction::EmitLoadOfAnyValue(LValue LV, AggValueSlot Slot,2360 SourceLocation Loc) {2361 QualType Ty = LV.getType();2362 switch (getEvaluationKind(Ty)) {2363 case TEK_Scalar:2364 return EmitLoadOfLValue(LV, Loc);2365 case TEK_Complex:2366 return RValue::getComplex(EmitLoadOfComplex(LV, Loc));2367 case TEK_Aggregate:2368 EmitAggFinalDestCopy(Ty, Slot, LV, EVK_NonRValue);2369 return Slot.asRValue();2370 }2371 llvm_unreachable("bad evaluation kind");2372}2373 2374/// EmitLoadOfLValue - Given an expression that represents a value lvalue, this2375/// method emits the address of the lvalue, then loads the result as an rvalue,2376/// returning the rvalue.2377RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, SourceLocation Loc) {2378 // Load from __ptrauth.2379 if (PointerAuthQualifier PtrAuth = LV.getQuals().getPointerAuth()) {2380 LV.getQuals().removePointerAuth();2381 llvm::Value *Value = EmitLoadOfLValue(LV, Loc).getScalarVal();2382 return RValue::get(EmitPointerAuthUnqualify(PtrAuth, Value, LV.getType(),2383 LV.getAddress(),2384 /*known nonnull*/ false));2385 }2386 2387 if (LV.isObjCWeak()) {2388 // load of a __weak object.2389 Address AddrWeakObj = LV.getAddress();2390 return RValue::get(CGM.getObjCRuntime().EmitObjCWeakRead(*this,2391 AddrWeakObj));2392 }2393 if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) {2394 // In MRC mode, we do a load+autorelease.2395 if (!getLangOpts().ObjCAutoRefCount) {2396 return RValue::get(EmitARCLoadWeak(LV.getAddress()));2397 }2398 2399 // In ARC mode, we load retained and then consume the value.2400 llvm::Value *Object = EmitARCLoadWeakRetained(LV.getAddress());2401 Object = EmitObjCConsumeObject(LV.getType(), Object);2402 return RValue::get(Object);2403 }2404 2405 if (LV.isSimple()) {2406 assert(!LV.getType()->isFunctionType());2407 2408 if (LV.getType()->isConstantMatrixType())2409 return EmitLoadOfMatrixLValue(LV, Loc, *this);2410 2411 // Everything needs a load.2412 return RValue::get(EmitLoadOfScalar(LV, Loc));2413 }2414 2415 if (LV.isVectorElt()) {2416 llvm::LoadInst *Load = Builder.CreateLoad(LV.getVectorAddress(),2417 LV.isVolatileQualified());2418 return RValue::get(Builder.CreateExtractElement(Load, LV.getVectorIdx(),2419 "vecext"));2420 }2421 2422 // If this is a reference to a subset of the elements of a vector, either2423 // shuffle the input or extract/insert them as appropriate.2424 if (LV.isExtVectorElt()) {2425 return EmitLoadOfExtVectorElementLValue(LV);2426 }2427 2428 // Global Register variables always invoke intrinsics2429 if (LV.isGlobalReg())2430 return EmitLoadOfGlobalRegLValue(LV);2431 2432 if (LV.isMatrixElt()) {2433 llvm::Value *Idx = LV.getMatrixIdx();2434 if (CGM.getCodeGenOpts().OptimizationLevel > 0) {2435 const auto *const MatTy = LV.getType()->castAs<ConstantMatrixType>();2436 llvm::MatrixBuilder MB(Builder);2437 MB.CreateIndexAssumption(Idx, MatTy->getNumElementsFlattened());2438 }2439 llvm::LoadInst *Load =2440 Builder.CreateLoad(LV.getMatrixAddress(), LV.isVolatileQualified());2441 return RValue::get(Builder.CreateExtractElement(Load, Idx, "matrixext"));2442 }2443 2444 assert(LV.isBitField() && "Unknown LValue type!");2445 return EmitLoadOfBitfieldLValue(LV, Loc);2446}2447 2448RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV,2449 SourceLocation Loc) {2450 const CGBitFieldInfo &Info = LV.getBitFieldInfo();2451 2452 // Get the output type.2453 llvm::Type *ResLTy = ConvertType(LV.getType());2454 2455 Address Ptr = LV.getBitFieldAddress();2456 llvm::Value *Val =2457 Builder.CreateLoad(Ptr, LV.isVolatileQualified(), "bf.load");2458 2459 bool UseVolatile = LV.isVolatileQualified() &&2460 Info.VolatileStorageSize != 0 && isAAPCS(CGM.getTarget());2461 const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;2462 const unsigned StorageSize =2463 UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;2464 if (Info.IsSigned) {2465 assert(static_cast<unsigned>(Offset + Info.Size) <= StorageSize);2466 unsigned HighBits = StorageSize - Offset - Info.Size;2467 if (HighBits)2468 Val = Builder.CreateShl(Val, HighBits, "bf.shl");2469 if (Offset + HighBits)2470 Val = Builder.CreateAShr(Val, Offset + HighBits, "bf.ashr");2471 } else {2472 if (Offset)2473 Val = Builder.CreateLShr(Val, Offset, "bf.lshr");2474 if (static_cast<unsigned>(Offset) + Info.Size < StorageSize)2475 Val = Builder.CreateAnd(2476 Val, llvm::APInt::getLowBitsSet(StorageSize, Info.Size), "bf.clear");2477 }2478 Val = Builder.CreateIntCast(Val, ResLTy, Info.IsSigned, "bf.cast");2479 EmitScalarRangeCheck(Val, LV.getType(), Loc);2480 return RValue::get(Val);2481}2482 2483// If this is a reference to a subset of the elements of a vector, create an2484// appropriate shufflevector.2485RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) {2486 llvm::Value *Vec = Builder.CreateLoad(LV.getExtVectorAddress(),2487 LV.isVolatileQualified());2488 2489 // HLSL allows treating scalars as one-element vectors. Converting the scalar2490 // IR value to a vector here allows the rest of codegen to behave as normal.2491 if (getLangOpts().HLSL && !Vec->getType()->isVectorTy()) {2492 llvm::Type *DstTy = llvm::FixedVectorType::get(Vec->getType(), 1);2493 llvm::Value *Zero = llvm::Constant::getNullValue(CGM.Int64Ty);2494 Vec = Builder.CreateInsertElement(DstTy, Vec, Zero, "cast.splat");2495 }2496 2497 const llvm::Constant *Elts = LV.getExtVectorElts();2498 2499 // If the result of the expression is a non-vector type, we must be extracting2500 // a single element. Just codegen as an extractelement.2501 const VectorType *ExprVT = LV.getType()->getAs<VectorType>();2502 if (!ExprVT) {2503 unsigned InIdx = getAccessedFieldNo(0, Elts);2504 llvm::Value *Elt = llvm::ConstantInt::get(SizeTy, InIdx);2505 2506 llvm::Value *Element = Builder.CreateExtractElement(Vec, Elt);2507 2508 llvm::Type *LVTy = ConvertType(LV.getType());2509 if (Element->getType()->getPrimitiveSizeInBits() >2510 LVTy->getPrimitiveSizeInBits())2511 Element = Builder.CreateTrunc(Element, LVTy);2512 2513 return RValue::get(Element);2514 }2515 2516 // Always use shuffle vector to try to retain the original program structure2517 unsigned NumResultElts = ExprVT->getNumElements();2518 2519 SmallVector<int, 4> Mask;2520 for (unsigned i = 0; i != NumResultElts; ++i)2521 Mask.push_back(getAccessedFieldNo(i, Elts));2522 2523 Vec = Builder.CreateShuffleVector(Vec, Mask);2524 2525 if (LV.getType()->isExtVectorBoolType())2526 Vec = Builder.CreateTrunc(Vec, ConvertType(LV.getType()), "truncv");2527 2528 return RValue::get(Vec);2529}2530 2531/// Generates lvalue for partial ext_vector access.2532Address CodeGenFunction::EmitExtVectorElementLValue(LValue LV) {2533 Address VectorAddress = LV.getExtVectorAddress();2534 QualType EQT = LV.getType()->castAs<VectorType>()->getElementType();2535 llvm::Type *VectorElementTy = CGM.getTypes().ConvertType(EQT);2536 2537 Address CastToPointerElement = VectorAddress.withElementType(VectorElementTy);2538 2539 const llvm::Constant *Elts = LV.getExtVectorElts();2540 unsigned ix = getAccessedFieldNo(0, Elts);2541 2542 Address VectorBasePtrPlusIx =2543 Builder.CreateConstInBoundsGEP(CastToPointerElement, ix,2544 "vector.elt");2545 2546 return VectorBasePtrPlusIx;2547}2548 2549/// Load of global named registers are always calls to intrinsics.2550RValue CodeGenFunction::EmitLoadOfGlobalRegLValue(LValue LV) {2551 assert((LV.getType()->isIntegerType() || LV.getType()->isPointerType()) &&2552 "Bad type for register variable");2553 llvm::MDNode *RegName = cast<llvm::MDNode>(2554 cast<llvm::MetadataAsValue>(LV.getGlobalReg())->getMetadata());2555 2556 // We accept integer and pointer types only2557 llvm::Type *OrigTy = CGM.getTypes().ConvertType(LV.getType());2558 llvm::Type *Ty = OrigTy;2559 if (OrigTy->isPointerTy())2560 Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);2561 llvm::Type *Types[] = { Ty };2562 2563 llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types);2564 llvm::Value *Call = Builder.CreateCall(2565 F, llvm::MetadataAsValue::get(Ty->getContext(), RegName));2566 if (OrigTy->isPointerTy())2567 Call = Builder.CreateIntToPtr(Call, OrigTy);2568 return RValue::get(Call);2569}2570 2571/// EmitStoreThroughLValue - Store the specified rvalue into the specified2572/// lvalue, where both are guaranteed to the have the same type, and that type2573/// is 'Ty'.2574void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst,2575 bool isInit) {2576 if (!Dst.isSimple()) {2577 if (Dst.isVectorElt()) {2578 if (getLangOpts().HLSL) {2579 // HLSL allows direct access to vector elements, so storing to2580 // individual elements of a vector through VectorElt is handled as2581 // separate store instructions.2582 Address DstAddr = Dst.getVectorAddress();2583 llvm::Type *DestAddrTy = DstAddr.getElementType();2584 llvm::Type *ElemTy = DestAddrTy->getScalarType();2585 CharUnits ElemAlign = CharUnits::fromQuantity(2586 CGM.getDataLayout().getPrefTypeAlign(ElemTy));2587 2588 assert(ElemTy->getScalarSizeInBits() >= 8 &&2589 "vector element type must be at least byte-sized");2590 2591 llvm::Value *Val = Src.getScalarVal();2592 if (Val->getType()->getPrimitiveSizeInBits() <2593 ElemTy->getScalarSizeInBits())2594 Val = Builder.CreateZExt(Val, ElemTy->getScalarType());2595 2596 llvm::Value *Idx = Dst.getVectorIdx();2597 llvm::Value *Zero = llvm::ConstantInt::get(Int32Ty, 0);2598 Address DstElemAddr =2599 Builder.CreateGEP(DstAddr, {Zero, Idx}, DestAddrTy, ElemAlign);2600 Builder.CreateStore(Val, DstElemAddr, Dst.isVolatileQualified());2601 return;2602 }2603 2604 // Read/modify/write the vector, inserting the new element.2605 llvm::Value *Vec = Builder.CreateLoad(Dst.getVectorAddress(),2606 Dst.isVolatileQualified());2607 llvm::Type *VecTy = Vec->getType();2608 llvm::Value *SrcVal = Src.getScalarVal();2609 2610 if (SrcVal->getType()->getPrimitiveSizeInBits() <2611 VecTy->getScalarSizeInBits())2612 SrcVal = Builder.CreateZExt(SrcVal, VecTy->getScalarType());2613 2614 auto *IRStoreTy = dyn_cast<llvm::IntegerType>(Vec->getType());2615 if (IRStoreTy) {2616 auto *IRVecTy = llvm::FixedVectorType::get(2617 Builder.getInt1Ty(), IRStoreTy->getPrimitiveSizeInBits());2618 Vec = Builder.CreateBitCast(Vec, IRVecTy);2619 // iN --> <N x i1>.2620 }2621 2622 // Allow inserting `<1 x T>` into an `<N x T>`. It can happen with scalar2623 // types which are mapped to vector LLVM IR types (e.g. for implementing2624 // an ABI).2625 if (auto *EltTy = dyn_cast<llvm::FixedVectorType>(SrcVal->getType());2626 EltTy && EltTy->getNumElements() == 1)2627 SrcVal = Builder.CreateBitCast(SrcVal, EltTy->getElementType());2628 2629 Vec = Builder.CreateInsertElement(Vec, SrcVal, Dst.getVectorIdx(),2630 "vecins");2631 if (IRStoreTy) {2632 // <N x i1> --> <iN>.2633 Vec = Builder.CreateBitCast(Vec, IRStoreTy);2634 }2635 2636 auto *I = Builder.CreateStore(Vec, Dst.getVectorAddress(),2637 Dst.isVolatileQualified());2638 addInstToCurrentSourceAtom(I, Vec);2639 return;2640 }2641 2642 // If this is an update of extended vector elements, insert them as2643 // appropriate.2644 if (Dst.isExtVectorElt())2645 return EmitStoreThroughExtVectorComponentLValue(Src, Dst);2646 2647 if (Dst.isGlobalReg())2648 return EmitStoreThroughGlobalRegLValue(Src, Dst);2649 2650 if (Dst.isMatrixElt()) {2651 llvm::Value *Idx = Dst.getMatrixIdx();2652 if (CGM.getCodeGenOpts().OptimizationLevel > 0) {2653 const auto *const MatTy = Dst.getType()->castAs<ConstantMatrixType>();2654 llvm::MatrixBuilder MB(Builder);2655 MB.CreateIndexAssumption(Idx, MatTy->getNumElementsFlattened());2656 }2657 llvm::Instruction *Load = Builder.CreateLoad(Dst.getMatrixAddress());2658 llvm::Value *Vec =2659 Builder.CreateInsertElement(Load, Src.getScalarVal(), Idx, "matins");2660 auto *I = Builder.CreateStore(Vec, Dst.getMatrixAddress(),2661 Dst.isVolatileQualified());2662 addInstToCurrentSourceAtom(I, Vec);2663 return;2664 }2665 2666 assert(Dst.isBitField() && "Unknown LValue type");2667 return EmitStoreThroughBitfieldLValue(Src, Dst);2668 }2669 2670 // Handle __ptrauth qualification by re-signing the value.2671 if (PointerAuthQualifier PointerAuth = Dst.getQuals().getPointerAuth()) {2672 Src = RValue::get(EmitPointerAuthQualify(PointerAuth, Src.getScalarVal(),2673 Dst.getType(), Dst.getAddress(),2674 /*known nonnull*/ false));2675 }2676 2677 // There's special magic for assigning into an ARC-qualified l-value.2678 if (Qualifiers::ObjCLifetime Lifetime = Dst.getQuals().getObjCLifetime()) {2679 switch (Lifetime) {2680 case Qualifiers::OCL_None:2681 llvm_unreachable("present but none");2682 2683 case Qualifiers::OCL_ExplicitNone:2684 // nothing special2685 break;2686 2687 case Qualifiers::OCL_Strong:2688 if (isInit) {2689 Src = RValue::get(EmitARCRetain(Dst.getType(), Src.getScalarVal()));2690 break;2691 }2692 EmitARCStoreStrong(Dst, Src.getScalarVal(), /*ignore*/ true);2693 return;2694 2695 case Qualifiers::OCL_Weak:2696 if (isInit)2697 // Initialize and then skip the primitive store.2698 EmitARCInitWeak(Dst.getAddress(), Src.getScalarVal());2699 else2700 EmitARCStoreWeak(Dst.getAddress(), Src.getScalarVal(),2701 /*ignore*/ true);2702 return;2703 2704 case Qualifiers::OCL_Autoreleasing:2705 Src = RValue::get(EmitObjCExtendObjectLifetime(Dst.getType(),2706 Src.getScalarVal()));2707 // fall into the normal path2708 break;2709 }2710 }2711 2712 if (Dst.isObjCWeak() && !Dst.isNonGC()) {2713 // load of a __weak object.2714 Address LvalueDst = Dst.getAddress();2715 llvm::Value *src = Src.getScalarVal();2716 CGM.getObjCRuntime().EmitObjCWeakAssign(*this, src, LvalueDst);2717 return;2718 }2719 2720 if (Dst.isObjCStrong() && !Dst.isNonGC()) {2721 // load of a __strong object.2722 Address LvalueDst = Dst.getAddress();2723 llvm::Value *src = Src.getScalarVal();2724 if (Dst.isObjCIvar()) {2725 assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL");2726 llvm::Type *ResultType = IntPtrTy;2727 Address dst = EmitPointerWithAlignment(Dst.getBaseIvarExp());2728 llvm::Value *RHS = dst.emitRawPointer(*this);2729 RHS = Builder.CreatePtrToInt(RHS, ResultType, "sub.ptr.rhs.cast");2730 llvm::Value *LHS = Builder.CreatePtrToInt(LvalueDst.emitRawPointer(*this),2731 ResultType, "sub.ptr.lhs.cast");2732 llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, "ivar.offset");2733 CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst, BytesBetween);2734 } else if (Dst.isGlobalObjCRef()) {2735 CGM.getObjCRuntime().EmitObjCGlobalAssign(*this, src, LvalueDst,2736 Dst.isThreadLocalRef());2737 }2738 else2739 CGM.getObjCRuntime().EmitObjCStrongCastAssign(*this, src, LvalueDst);2740 return;2741 }2742 2743 assert(Src.isScalar() && "Can't emit an agg store with this method");2744 EmitStoreOfScalar(Src.getScalarVal(), Dst, isInit);2745}2746 2747void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,2748 llvm::Value **Result) {2749 const CGBitFieldInfo &Info = Dst.getBitFieldInfo();2750 llvm::Type *ResLTy = convertTypeForLoadStore(Dst.getType());2751 Address Ptr = Dst.getBitFieldAddress();2752 2753 // Get the source value, truncated to the width of the bit-field.2754 llvm::Value *SrcVal = Src.getScalarVal();2755 2756 // Cast the source to the storage type and shift it into place.2757 SrcVal = Builder.CreateIntCast(SrcVal, Ptr.getElementType(),2758 /*isSigned=*/false);2759 llvm::Value *MaskedVal = SrcVal;2760 2761 const bool UseVolatile =2762 CGM.getCodeGenOpts().AAPCSBitfieldWidth && Dst.isVolatileQualified() &&2763 Info.VolatileStorageSize != 0 && isAAPCS(CGM.getTarget());2764 const unsigned StorageSize =2765 UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;2766 const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;2767 // See if there are other bits in the bitfield's storage we'll need to load2768 // and mask together with source before storing.2769 if (StorageSize != Info.Size) {2770 assert(StorageSize > Info.Size && "Invalid bitfield size.");2771 llvm::Value *Val =2772 Builder.CreateLoad(Ptr, Dst.isVolatileQualified(), "bf.load");2773 2774 // Mask the source value as needed.2775 if (!Dst.getType()->hasBooleanRepresentation())2776 SrcVal = Builder.CreateAnd(2777 SrcVal, llvm::APInt::getLowBitsSet(StorageSize, Info.Size),2778 "bf.value");2779 MaskedVal = SrcVal;2780 if (Offset)2781 SrcVal = Builder.CreateShl(SrcVal, Offset, "bf.shl");2782 2783 // Mask out the original value.2784 Val = Builder.CreateAnd(2785 Val, ~llvm::APInt::getBitsSet(StorageSize, Offset, Offset + Info.Size),2786 "bf.clear");2787 2788 // Or together the unchanged values and the source value.2789 SrcVal = Builder.CreateOr(Val, SrcVal, "bf.set");2790 } else {2791 assert(Offset == 0);2792 // According to the AACPS:2793 // When a volatile bit-field is written, and its container does not overlap2794 // with any non-bit-field member, its container must be read exactly once2795 // and written exactly once using the access width appropriate to the type2796 // of the container. The two accesses are not atomic.2797 if (Dst.isVolatileQualified() && isAAPCS(CGM.getTarget()) &&2798 CGM.getCodeGenOpts().ForceAAPCSBitfieldLoad)2799 Builder.CreateLoad(Ptr, true, "bf.load");2800 }2801 2802 // Write the new value back out.2803 auto *I = Builder.CreateStore(SrcVal, Ptr, Dst.isVolatileQualified());2804 addInstToCurrentSourceAtom(I, SrcVal);2805 2806 // Return the new value of the bit-field, if requested.2807 if (Result) {2808 llvm::Value *ResultVal = MaskedVal;2809 2810 // Sign extend the value if needed.2811 if (Info.IsSigned) {2812 assert(Info.Size <= StorageSize);2813 unsigned HighBits = StorageSize - Info.Size;2814 if (HighBits) {2815 ResultVal = Builder.CreateShl(ResultVal, HighBits, "bf.result.shl");2816 ResultVal = Builder.CreateAShr(ResultVal, HighBits, "bf.result.ashr");2817 }2818 }2819 2820 ResultVal = Builder.CreateIntCast(ResultVal, ResLTy, Info.IsSigned,2821 "bf.result.cast");2822 *Result = EmitFromMemory(ResultVal, Dst.getType());2823 }2824}2825 2826void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,2827 LValue Dst) {2828 llvm::Value *SrcVal = Src.getScalarVal();2829 Address DstAddr = Dst.getExtVectorAddress();2830 const llvm::Constant *Elts = Dst.getExtVectorElts();2831 if (DstAddr.getElementType()->getScalarSizeInBits() >2832 SrcVal->getType()->getScalarSizeInBits())2833 SrcVal = Builder.CreateZExt(2834 SrcVal, convertTypeForLoadStore(Dst.getType(), SrcVal->getType()));2835 2836 if (getLangOpts().HLSL) {2837 llvm::Type *DestAddrTy = DstAddr.getElementType();2838 // HLSL allows storing to scalar values through ExtVector component LValues.2839 // To support this we need to handle the case where the destination address2840 // is a scalar.2841 if (!DestAddrTy->isVectorTy()) {2842 assert(!Dst.getType()->isVectorType() &&2843 "this should only occur for non-vector l-values");2844 Builder.CreateStore(SrcVal, DstAddr, Dst.isVolatileQualified());2845 return;2846 }2847 2848 // HLSL allows direct access to vector elements, so storing to individual2849 // elements of a vector through ExtVector is handled as separate store2850 // instructions.2851 // If we are updating multiple elements, Dst and Src are vectors; for2852 // a single element update they are scalars.2853 const VectorType *VTy = Dst.getType()->getAs<VectorType>();2854 unsigned NumSrcElts = VTy ? VTy->getNumElements() : 1;2855 CharUnits ElemAlign = CharUnits::fromQuantity(2856 CGM.getDataLayout().getPrefTypeAlign(DestAddrTy->getScalarType()));2857 llvm::Value *Zero = llvm::ConstantInt::get(Int32Ty, 0);2858 2859 for (unsigned I = 0; I != NumSrcElts; ++I) {2860 llvm::Value *Val = VTy ? Builder.CreateExtractElement(2861 SrcVal, llvm::ConstantInt::get(Int32Ty, I))2862 : SrcVal;2863 unsigned FieldNo = getAccessedFieldNo(I, Elts);2864 Address DstElemAddr = Address::invalid();2865 if (FieldNo == 0)2866 DstElemAddr = DstAddr.withAlignment(ElemAlign);2867 else2868 DstElemAddr = Builder.CreateGEP(2869 DstAddr, {Zero, llvm::ConstantInt::get(Int32Ty, FieldNo)},2870 DestAddrTy, ElemAlign);2871 Builder.CreateStore(Val, DstElemAddr, Dst.isVolatileQualified());2872 }2873 return;2874 }2875 2876 // This access turns into a read/modify/write of the vector. Load the input2877 // value now.2878 llvm::Value *Vec = Builder.CreateLoad(DstAddr, Dst.isVolatileQualified());2879 llvm::Type *VecTy = Vec->getType();2880 2881 if (const VectorType *VTy = Dst.getType()->getAs<VectorType>()) {2882 unsigned NumSrcElts = VTy->getNumElements();2883 unsigned NumDstElts = cast<llvm::FixedVectorType>(VecTy)->getNumElements();2884 if (NumDstElts == NumSrcElts) {2885 // Use shuffle vector is the src and destination are the same number of2886 // elements and restore the vector mask since it is on the side it will be2887 // stored.2888 SmallVector<int, 4> Mask(NumDstElts);2889 for (unsigned i = 0; i != NumSrcElts; ++i)2890 Mask[getAccessedFieldNo(i, Elts)] = i;2891 2892 Vec = Builder.CreateShuffleVector(SrcVal, Mask);2893 } else if (NumDstElts > NumSrcElts) {2894 // Extended the source vector to the same length and then shuffle it2895 // into the destination.2896 // FIXME: since we're shuffling with undef, can we just use the indices2897 // into that? This could be simpler.2898 SmallVector<int, 4> ExtMask;2899 for (unsigned i = 0; i != NumSrcElts; ++i)2900 ExtMask.push_back(i);2901 ExtMask.resize(NumDstElts, -1);2902 llvm::Value *ExtSrcVal = Builder.CreateShuffleVector(SrcVal, ExtMask);2903 // build identity2904 SmallVector<int, 4> Mask;2905 for (unsigned i = 0; i != NumDstElts; ++i)2906 Mask.push_back(i);2907 2908 // When the vector size is odd and .odd or .hi is used, the last element2909 // of the Elts constant array will be one past the size of the vector.2910 // Ignore the last element here, if it is greater than the mask size.2911 if (getAccessedFieldNo(NumSrcElts - 1, Elts) == Mask.size())2912 NumSrcElts--;2913 2914 // modify when what gets shuffled in2915 for (unsigned i = 0; i != NumSrcElts; ++i)2916 Mask[getAccessedFieldNo(i, Elts)] = i + NumDstElts;2917 Vec = Builder.CreateShuffleVector(Vec, ExtSrcVal, Mask);2918 } else {2919 // We should never shorten the vector2920 llvm_unreachable("unexpected shorten vector length");2921 }2922 } else {2923 // If the Src is a scalar (not a vector), and the target is a vector it must2924 // be updating one element.2925 unsigned InIdx = getAccessedFieldNo(0, Elts);2926 llvm::Value *Elt = llvm::ConstantInt::get(SizeTy, InIdx);2927 2928 Vec = Builder.CreateInsertElement(Vec, SrcVal, Elt);2929 }2930 2931 Builder.CreateStore(Vec, Dst.getExtVectorAddress(),2932 Dst.isVolatileQualified());2933}2934 2935/// Store of global named registers are always calls to intrinsics.2936void CodeGenFunction::EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst) {2937 assert((Dst.getType()->isIntegerType() || Dst.getType()->isPointerType()) &&2938 "Bad type for register variable");2939 llvm::MDNode *RegName = cast<llvm::MDNode>(2940 cast<llvm::MetadataAsValue>(Dst.getGlobalReg())->getMetadata());2941 assert(RegName && "Register LValue is not metadata");2942 2943 // We accept integer and pointer types only2944 llvm::Type *OrigTy = CGM.getTypes().ConvertType(Dst.getType());2945 llvm::Type *Ty = OrigTy;2946 if (OrigTy->isPointerTy())2947 Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);2948 llvm::Type *Types[] = { Ty };2949 2950 llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types);2951 llvm::Value *Value = Src.getScalarVal();2952 if (OrigTy->isPointerTy())2953 Value = Builder.CreatePtrToInt(Value, Ty);2954 Builder.CreateCall(2955 F, {llvm::MetadataAsValue::get(Ty->getContext(), RegName), Value});2956}2957 2958// setObjCGCLValueClass - sets class of the lvalue for the purpose of2959// generating write-barries API. It is currently a global, ivar,2960// or neither.2961static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,2962 LValue &LV,2963 bool IsMemberAccess=false) {2964 if (Ctx.getLangOpts().getGC() == LangOptions::NonGC)2965 return;2966 2967 if (isa<ObjCIvarRefExpr>(E)) {2968 QualType ExpTy = E->getType();2969 if (IsMemberAccess && ExpTy->isPointerType()) {2970 // If ivar is a structure pointer, assigning to field of2971 // this struct follows gcc's behavior and makes it a non-ivar2972 // writer-barrier conservatively.2973 ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();2974 if (ExpTy->isRecordType()) {2975 LV.setObjCIvar(false);2976 return;2977 }2978 }2979 LV.setObjCIvar(true);2980 auto *Exp = cast<ObjCIvarRefExpr>(const_cast<Expr *>(E));2981 LV.setBaseIvarExp(Exp->getBase());2982 LV.setObjCArray(E->getType()->isArrayType());2983 return;2984 }2985 2986 if (const auto *Exp = dyn_cast<DeclRefExpr>(E)) {2987 if (const auto *VD = dyn_cast<VarDecl>(Exp->getDecl())) {2988 if (VD->hasGlobalStorage()) {2989 LV.setGlobalObjCRef(true);2990 LV.setThreadLocalRef(VD->getTLSKind() != VarDecl::TLS_None);2991 }2992 }2993 LV.setObjCArray(E->getType()->isArrayType());2994 return;2995 }2996 2997 if (const auto *Exp = dyn_cast<UnaryOperator>(E)) {2998 setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);2999 return;3000 }3001 3002 if (const auto *Exp = dyn_cast<ParenExpr>(E)) {3003 setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);3004 if (LV.isObjCIvar()) {3005 // If cast is to a structure pointer, follow gcc's behavior and make it3006 // a non-ivar write-barrier.3007 QualType ExpTy = E->getType();3008 if (ExpTy->isPointerType())3009 ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();3010 if (ExpTy->isRecordType())3011 LV.setObjCIvar(false);3012 }3013 return;3014 }3015 3016 if (const auto *Exp = dyn_cast<GenericSelectionExpr>(E)) {3017 setObjCGCLValueClass(Ctx, Exp->getResultExpr(), LV);3018 return;3019 }3020 3021 if (const auto *Exp = dyn_cast<ImplicitCastExpr>(E)) {3022 setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);3023 return;3024 }3025 3026 if (const auto *Exp = dyn_cast<CStyleCastExpr>(E)) {3027 setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);3028 return;3029 }3030 3031 if (const auto *Exp = dyn_cast<ObjCBridgedCastExpr>(E)) {3032 setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);3033 return;3034 }3035 3036 if (const auto *Exp = dyn_cast<ArraySubscriptExpr>(E)) {3037 setObjCGCLValueClass(Ctx, Exp->getBase(), LV);3038 if (LV.isObjCIvar() && !LV.isObjCArray())3039 // Using array syntax to assigning to what an ivar points to is not3040 // same as assigning to the ivar itself. {id *Names;} Names[i] = 0;3041 LV.setObjCIvar(false);3042 else if (LV.isGlobalObjCRef() && !LV.isObjCArray())3043 // Using array syntax to assigning to what global points to is not3044 // same as assigning to the global itself. {id *G;} G[i] = 0;3045 LV.setGlobalObjCRef(false);3046 return;3047 }3048 3049 if (const auto *Exp = dyn_cast<MemberExpr>(E)) {3050 setObjCGCLValueClass(Ctx, Exp->getBase(), LV, true);3051 // We don't know if member is an 'ivar', but this flag is looked at3052 // only in the context of LV.isObjCIvar().3053 LV.setObjCArray(E->getType()->isArrayType());3054 return;3055 }3056}3057 3058static LValue EmitThreadPrivateVarDeclLValue(3059 CodeGenFunction &CGF, const VarDecl *VD, QualType T, Address Addr,3060 llvm::Type *RealVarTy, SourceLocation Loc) {3061 if (CGF.CGM.getLangOpts().OpenMPIRBuilder)3062 Addr = CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate(3063 CGF, VD, Addr, Loc);3064 else3065 Addr =3066 CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, Addr, Loc);3067 3068 Addr = Addr.withElementType(RealVarTy);3069 return CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);3070}3071 3072static Address emitDeclTargetVarDeclLValue(CodeGenFunction &CGF,3073 const VarDecl *VD, QualType T) {3074 std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =3075 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);3076 // Return an invalid address if variable is MT_To (or MT_Enter starting with3077 // OpenMP 5.2) and unified memory is not enabled. For all other cases: MT_Link3078 // and MT_To (or MT_Enter) with unified memory, return a valid address.3079 if (!Res || ((*Res == OMPDeclareTargetDeclAttr::MT_To ||3080 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&3081 !CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory()))3082 return Address::invalid();3083 assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||3084 ((*Res == OMPDeclareTargetDeclAttr::MT_To ||3085 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&3086 CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) &&3087 "Expected link clause OR to clause with unified memory enabled.");3088 QualType PtrTy = CGF.getContext().getPointerType(VD->getType());3089 Address Addr = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);3090 return CGF.EmitLoadOfPointer(Addr, PtrTy->castAs<PointerType>());3091}3092 3093Address3094CodeGenFunction::EmitLoadOfReference(LValue RefLVal,3095 LValueBaseInfo *PointeeBaseInfo,3096 TBAAAccessInfo *PointeeTBAAInfo) {3097 llvm::LoadInst *Load =3098 Builder.CreateLoad(RefLVal.getAddress(), RefLVal.isVolatile());3099 CGM.DecorateInstructionWithTBAA(Load, RefLVal.getTBAAInfo());3100 QualType PTy = RefLVal.getType()->getPointeeType();3101 CharUnits Align = CGM.getNaturalTypeAlignment(3102 PTy, PointeeBaseInfo, PointeeTBAAInfo, /*ForPointeeType=*/true);3103 if (!PTy->isIncompleteType()) {3104 llvm::LLVMContext &Ctx = getLLVMContext();3105 llvm::MDBuilder MDB(Ctx);3106 // Emit !nonnull metadata3107 if (CGM.getTypes().getTargetAddressSpace(PTy) == 0 &&3108 !CGM.getCodeGenOpts().NullPointerIsValid)3109 Load->setMetadata(llvm::LLVMContext::MD_nonnull,3110 llvm::MDNode::get(Ctx, {}));3111 // Emit !align metadata3112 if (PTy->isObjectType()) {3113 auto AlignVal = Align.getQuantity();3114 if (AlignVal > 1) {3115 Load->setMetadata(3116 llvm::LLVMContext::MD_align,3117 llvm::MDNode::get(Ctx, MDB.createConstant(llvm::ConstantInt::get(3118 Builder.getInt64Ty(), AlignVal))));3119 }3120 }3121 }3122 return makeNaturalAddressForPointer(Load, PTy, Align,3123 /*ForPointeeType=*/true, PointeeBaseInfo,3124 PointeeTBAAInfo);3125}3126 3127LValue CodeGenFunction::EmitLoadOfReferenceLValue(LValue RefLVal) {3128 LValueBaseInfo PointeeBaseInfo;3129 TBAAAccessInfo PointeeTBAAInfo;3130 Address PointeeAddr = EmitLoadOfReference(RefLVal, &PointeeBaseInfo,3131 &PointeeTBAAInfo);3132 return MakeAddrLValue(PointeeAddr, RefLVal.getType()->getPointeeType(),3133 PointeeBaseInfo, PointeeTBAAInfo);3134}3135 3136Address CodeGenFunction::EmitLoadOfPointer(Address Ptr,3137 const PointerType *PtrTy,3138 LValueBaseInfo *BaseInfo,3139 TBAAAccessInfo *TBAAInfo) {3140 llvm::Value *Addr = Builder.CreateLoad(Ptr);3141 return makeNaturalAddressForPointer(Addr, PtrTy->getPointeeType(),3142 CharUnits(), /*ForPointeeType=*/true,3143 BaseInfo, TBAAInfo);3144}3145 3146LValue CodeGenFunction::EmitLoadOfPointerLValue(Address PtrAddr,3147 const PointerType *PtrTy) {3148 LValueBaseInfo BaseInfo;3149 TBAAAccessInfo TBAAInfo;3150 Address Addr = EmitLoadOfPointer(PtrAddr, PtrTy, &BaseInfo, &TBAAInfo);3151 return MakeAddrLValue(Addr, PtrTy->getPointeeType(), BaseInfo, TBAAInfo);3152}3153 3154static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,3155 const Expr *E, const VarDecl *VD) {3156 QualType T = E->getType();3157 3158 // If it's thread_local, emit a call to its wrapper function instead.3159 if (VD->getTLSKind() == VarDecl::TLS_Dynamic &&3160 CGF.CGM.getCXXABI().usesThreadWrapperFunction(VD))3161 return CGF.CGM.getCXXABI().EmitThreadLocalVarDeclLValue(CGF, VD, T);3162 // Check if the variable is marked as declare target with link clause in3163 // device codegen.3164 if (CGF.getLangOpts().OpenMPIsTargetDevice) {3165 Address Addr = emitDeclTargetVarDeclLValue(CGF, VD, T);3166 if (Addr.isValid())3167 return CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);3168 }3169 3170 llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(VD);3171 3172 if (VD->getTLSKind() != VarDecl::TLS_None)3173 V = CGF.Builder.CreateThreadLocalAddress(V);3174 3175 llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(VD->getType());3176 CharUnits Alignment = CGF.getContext().getDeclAlign(VD);3177 Address Addr(V, RealVarTy, Alignment);3178 // Emit reference to the private copy of the variable if it is an OpenMP3179 // threadprivate variable.3180 if (CGF.getLangOpts().OpenMP && !CGF.getLangOpts().OpenMPSimd &&3181 VD->hasAttr<OMPThreadPrivateDeclAttr>()) {3182 return EmitThreadPrivateVarDeclLValue(CGF, VD, T, Addr, RealVarTy,3183 E->getExprLoc());3184 }3185 LValue LV = VD->getType()->isReferenceType() ?3186 CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(),3187 AlignmentSource::Decl) :3188 CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);3189 setObjCGCLValueClass(CGF.getContext(), E, LV);3190 return LV;3191}3192 3193llvm::Constant *CodeGenModule::getRawFunctionPointer(GlobalDecl GD,3194 llvm::Type *Ty) {3195 const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());3196 if (FD->hasAttr<WeakRefAttr>()) {3197 ConstantAddress aliasee = GetWeakRefReference(FD);3198 return aliasee.getPointer();3199 }3200 3201 llvm::Constant *V = GetAddrOfFunction(GD, Ty);3202 return V;3203}3204 3205static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF, const Expr *E,3206 GlobalDecl GD) {3207 const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());3208 llvm::Constant *V = CGF.CGM.getFunctionPointer(GD);3209 QualType ETy = E->getType();3210 if (ETy->isCFIUncheckedCalleeFunctionType()) {3211 if (auto *GV = dyn_cast<llvm::GlobalValue>(V))3212 V = llvm::NoCFIValue::get(GV);3213 }3214 CharUnits Alignment = CGF.getContext().getDeclAlign(FD);3215 return CGF.MakeAddrLValue(V, ETy, Alignment, AlignmentSource::Decl);3216}3217 3218static LValue EmitCapturedFieldLValue(CodeGenFunction &CGF, const FieldDecl *FD,3219 llvm::Value *ThisValue) {3220 3221 return CGF.EmitLValueForLambdaField(FD, ThisValue);3222}3223 3224/// Named Registers are named metadata pointing to the register name3225/// which will be read from/written to as an argument to the intrinsic3226/// @llvm.read/write_register.3227/// So far, only the name is being passed down, but other options such as3228/// register type, allocation type or even optimization options could be3229/// passed down via the metadata node.3230static LValue EmitGlobalNamedRegister(const VarDecl *VD, CodeGenModule &CGM) {3231 SmallString<64> Name("llvm.named.register.");3232 AsmLabelAttr *Asm = VD->getAttr<AsmLabelAttr>();3233 assert(Asm->getLabel().size() < 64-Name.size() &&3234 "Register name too big");3235 Name.append(Asm->getLabel());3236 llvm::NamedMDNode *M =3237 CGM.getModule().getOrInsertNamedMetadata(Name);3238 if (M->getNumOperands() == 0) {3239 llvm::MDString *Str = llvm::MDString::get(CGM.getLLVMContext(),3240 Asm->getLabel());3241 llvm::Metadata *Ops[] = {Str};3242 M->addOperand(llvm::MDNode::get(CGM.getLLVMContext(), Ops));3243 }3244 3245 CharUnits Alignment = CGM.getContext().getDeclAlign(VD);3246 3247 llvm::Value *Ptr =3248 llvm::MetadataAsValue::get(CGM.getLLVMContext(), M->getOperand(0));3249 return LValue::MakeGlobalReg(Ptr, Alignment, VD->getType());3250}3251 3252/// Determine whether we can emit a reference to \p VD from the current3253/// context, despite not necessarily having seen an odr-use of the variable in3254/// this context.3255static bool canEmitSpuriousReferenceToVariable(CodeGenFunction &CGF,3256 const DeclRefExpr *E,3257 const VarDecl *VD) {3258 // For a variable declared in an enclosing scope, do not emit a spurious3259 // reference even if we have a capture, as that will emit an unwarranted3260 // reference to our capture state, and will likely generate worse code than3261 // emitting a local copy.3262 if (E->refersToEnclosingVariableOrCapture())3263 return false;3264 3265 // For a local declaration declared in this function, we can always reference3266 // it even if we don't have an odr-use.3267 if (VD->hasLocalStorage()) {3268 return VD->getDeclContext() ==3269 dyn_cast_or_null<DeclContext>(CGF.CurCodeDecl);3270 }3271 3272 // For a global declaration, we can emit a reference to it if we know3273 // for sure that we are able to emit a definition of it.3274 VD = VD->getDefinition(CGF.getContext());3275 if (!VD)3276 return false;3277 3278 // Don't emit a spurious reference if it might be to a variable that only3279 // exists on a different device / target.3280 // FIXME: This is unnecessarily broad. Check whether this would actually be a3281 // cross-target reference.3282 if (CGF.getLangOpts().OpenMP || CGF.getLangOpts().CUDA ||3283 CGF.getLangOpts().OpenCL) {3284 return false;3285 }3286 3287 // We can emit a spurious reference only if the linkage implies that we'll3288 // be emitting a non-interposable symbol that will be retained until link3289 // time.3290 switch (CGF.CGM.getLLVMLinkageVarDefinition(VD)) {3291 case llvm::GlobalValue::ExternalLinkage:3292 case llvm::GlobalValue::LinkOnceODRLinkage:3293 case llvm::GlobalValue::WeakODRLinkage:3294 case llvm::GlobalValue::InternalLinkage:3295 case llvm::GlobalValue::PrivateLinkage:3296 return true;3297 default:3298 return false;3299 }3300}3301 3302LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {3303 const NamedDecl *ND = E->getDecl();3304 QualType T = E->getType();3305 3306 assert(E->isNonOdrUse() != NOUR_Unevaluated &&3307 "should not emit an unevaluated operand");3308 3309 if (const auto *VD = dyn_cast<VarDecl>(ND)) {3310 // Global Named registers access via intrinsics only3311 if (VD->getStorageClass() == SC_Register &&3312 VD->hasAttr<AsmLabelAttr>() && !VD->isLocalVarDecl())3313 return EmitGlobalNamedRegister(VD, CGM);3314 3315 // If this DeclRefExpr does not constitute an odr-use of the variable,3316 // we're not permitted to emit a reference to it in general, and it might3317 // not be captured if capture would be necessary for a use. Emit the3318 // constant value directly instead.3319 if (E->isNonOdrUse() == NOUR_Constant &&3320 (VD->getType()->isReferenceType() ||3321 !canEmitSpuriousReferenceToVariable(*this, E, VD))) {3322 VD->getAnyInitializer(VD);3323 llvm::Constant *Val = ConstantEmitter(*this).emitAbstract(3324 E->getLocation(), *VD->evaluateValue(), VD->getType());3325 assert(Val && "failed to emit constant expression");3326 3327 Address Addr = Address::invalid();3328 if (!VD->getType()->isReferenceType()) {3329 // Spill the constant value to a global.3330 Addr = CGM.createUnnamedGlobalFrom(*VD, Val,3331 getContext().getDeclAlign(VD));3332 llvm::Type *VarTy = getTypes().ConvertTypeForMem(VD->getType());3333 auto *PTy = llvm::PointerType::get(3334 getLLVMContext(), getTypes().getTargetAddressSpace(VD->getType()));3335 Addr = Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, PTy, VarTy);3336 } else {3337 // Should we be using the alignment of the constant pointer we emitted?3338 CharUnits Alignment =3339 CGM.getNaturalTypeAlignment(E->getType(),3340 /* BaseInfo= */ nullptr,3341 /* TBAAInfo= */ nullptr,3342 /* forPointeeType= */ true);3343 Addr = makeNaturalAddressForPointer(Val, T, Alignment);3344 }3345 return MakeAddrLValue(Addr, T, AlignmentSource::Decl);3346 }3347 3348 // FIXME: Handle other kinds of non-odr-use DeclRefExprs.3349 3350 // Check for captured variables.3351 if (E->refersToEnclosingVariableOrCapture()) {3352 VD = VD->getCanonicalDecl();3353 if (auto *FD = LambdaCaptureFields.lookup(VD))3354 return EmitCapturedFieldLValue(*this, FD, CXXABIThisValue);3355 if (CapturedStmtInfo) {3356 auto I = LocalDeclMap.find(VD);3357 if (I != LocalDeclMap.end()) {3358 LValue CapLVal;3359 if (VD->getType()->isReferenceType())3360 CapLVal = EmitLoadOfReferenceLValue(I->second, VD->getType(),3361 AlignmentSource::Decl);3362 else3363 CapLVal = MakeAddrLValue(I->second, T);3364 // Mark lvalue as nontemporal if the variable is marked as nontemporal3365 // in simd context.3366 if (getLangOpts().OpenMP &&3367 CGM.getOpenMPRuntime().isNontemporalDecl(VD))3368 CapLVal.setNontemporal(/*Value=*/true);3369 return CapLVal;3370 }3371 LValue CapLVal =3372 EmitCapturedFieldLValue(*this, CapturedStmtInfo->lookup(VD),3373 CapturedStmtInfo->getContextValue());3374 Address LValueAddress = CapLVal.getAddress();3375 CapLVal = MakeAddrLValue(Address(LValueAddress.emitRawPointer(*this),3376 LValueAddress.getElementType(),3377 getContext().getDeclAlign(VD)),3378 CapLVal.getType(),3379 LValueBaseInfo(AlignmentSource::Decl),3380 CapLVal.getTBAAInfo());3381 // Mark lvalue as nontemporal if the variable is marked as nontemporal3382 // in simd context.3383 if (getLangOpts().OpenMP &&3384 CGM.getOpenMPRuntime().isNontemporalDecl(VD))3385 CapLVal.setNontemporal(/*Value=*/true);3386 return CapLVal;3387 }3388 3389 assert(isa<BlockDecl>(CurCodeDecl));3390 Address addr = GetAddrOfBlockDecl(VD);3391 return MakeAddrLValue(addr, T, AlignmentSource::Decl);3392 }3393 }3394 3395 // FIXME: We should be able to assert this for FunctionDecls as well!3396 // FIXME: We should be able to assert this for all DeclRefExprs, not just3397 // those with a valid source location.3398 assert((ND->isUsed(false) || !isa<VarDecl>(ND) || E->isNonOdrUse() ||3399 !E->getLocation().isValid()) &&3400 "Should not use decl without marking it used!");3401 3402 if (ND->hasAttr<WeakRefAttr>()) {3403 const auto *VD = cast<ValueDecl>(ND);3404 ConstantAddress Aliasee = CGM.GetWeakRefReference(VD);3405 return MakeAddrLValue(Aliasee, T, AlignmentSource::Decl);3406 }3407 3408 if (const auto *VD = dyn_cast<VarDecl>(ND)) {3409 // Check if this is a global variable.3410 if (VD->hasLinkage() || VD->isStaticDataMember())3411 return EmitGlobalVarDeclLValue(*this, E, VD);3412 3413 Address addr = Address::invalid();3414 3415 // The variable should generally be present in the local decl map.3416 auto iter = LocalDeclMap.find(VD);3417 if (iter != LocalDeclMap.end()) {3418 addr = iter->second;3419 3420 // Otherwise, it might be static local we haven't emitted yet for3421 // some reason; most likely, because it's in an outer function.3422 } else if (VD->isStaticLocal()) {3423 llvm::Constant *var = CGM.getOrCreateStaticVarDecl(3424 *VD, CGM.getLLVMLinkageVarDefinition(VD));3425 addr = Address(3426 var, ConvertTypeForMem(VD->getType()), getContext().getDeclAlign(VD));3427 3428 // No other cases for now.3429 } else {3430 llvm_unreachable("DeclRefExpr for Decl not entered in LocalDeclMap?");3431 }3432 3433 // Handle threadlocal function locals.3434 if (VD->getTLSKind() != VarDecl::TLS_None)3435 addr = addr.withPointer(3436 Builder.CreateThreadLocalAddress(addr.getBasePointer()),3437 NotKnownNonNull);3438 3439 // Check for OpenMP threadprivate variables.3440 if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd &&3441 VD->hasAttr<OMPThreadPrivateDeclAttr>()) {3442 return EmitThreadPrivateVarDeclLValue(3443 *this, VD, T, addr, getTypes().ConvertTypeForMem(VD->getType()),3444 E->getExprLoc());3445 }3446 3447 // Drill into block byref variables.3448 bool isBlockByref = VD->isEscapingByref();3449 if (isBlockByref) {3450 addr = emitBlockByrefAddress(addr, VD);3451 }3452 3453 // Drill into reference types.3454 LValue LV = VD->getType()->isReferenceType() ?3455 EmitLoadOfReferenceLValue(addr, VD->getType(), AlignmentSource::Decl) :3456 MakeAddrLValue(addr, T, AlignmentSource::Decl);3457 3458 bool isLocalStorage = VD->hasLocalStorage();3459 3460 bool NonGCable = isLocalStorage &&3461 !VD->getType()->isReferenceType() &&3462 !isBlockByref;3463 if (NonGCable) {3464 LV.getQuals().removeObjCGCAttr();3465 LV.setNonGC(true);3466 }3467 3468 bool isImpreciseLifetime =3469 (isLocalStorage && !VD->hasAttr<ObjCPreciseLifetimeAttr>());3470 if (isImpreciseLifetime)3471 LV.setARCPreciseLifetime(ARCImpreciseLifetime);3472 setObjCGCLValueClass(getContext(), E, LV);3473 return LV;3474 }3475 3476 if (const auto *FD = dyn_cast<FunctionDecl>(ND))3477 return EmitFunctionDeclLValue(*this, E, FD);3478 3479 // FIXME: While we're emitting a binding from an enclosing scope, all other3480 // DeclRefExprs we see should be implicitly treated as if they also refer to3481 // an enclosing scope.3482 if (const auto *BD = dyn_cast<BindingDecl>(ND)) {3483 if (E->refersToEnclosingVariableOrCapture()) {3484 auto *FD = LambdaCaptureFields.lookup(BD);3485 return EmitCapturedFieldLValue(*this, FD, CXXABIThisValue);3486 }3487 // Suppress debug location updates when visiting the binding, since the3488 // binding may emit instructions that would otherwise be associated with the3489 // binding itself, rather than the expression referencing the binding. (this3490 // leads to jumpy debug stepping behavior where the location/debugger jump3491 // back to the binding declaration, then back to the expression referencing3492 // the binding)3493 DisableDebugLocationUpdates D(*this);3494 return EmitLValue(BD->getBinding(), NotKnownNonNull);3495 }3496 3497 // We can form DeclRefExprs naming GUID declarations when reconstituting3498 // non-type template parameters into expressions.3499 if (const auto *GD = dyn_cast<MSGuidDecl>(ND))3500 return MakeAddrLValue(CGM.GetAddrOfMSGuidDecl(GD), T,3501 AlignmentSource::Decl);3502 3503 if (const auto *TPO = dyn_cast<TemplateParamObjectDecl>(ND)) {3504 auto ATPO = CGM.GetAddrOfTemplateParamObject(TPO);3505 auto AS = getLangASFromTargetAS(ATPO.getAddressSpace());3506 3507 if (AS != T.getAddressSpace()) {3508 auto TargetAS = getContext().getTargetAddressSpace(T.getAddressSpace());3509 auto PtrTy = llvm::PointerType::get(CGM.getLLVMContext(), TargetAS);3510 auto ASC = getTargetHooks().performAddrSpaceCast(CGM, ATPO.getPointer(),3511 AS, PtrTy);3512 ATPO = ConstantAddress(ASC, ATPO.getElementType(), ATPO.getAlignment());3513 }3514 3515 return MakeAddrLValue(ATPO, T, AlignmentSource::Decl);3516 }3517 3518 llvm_unreachable("Unhandled DeclRefExpr");3519}3520 3521LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) {3522 // __extension__ doesn't affect lvalue-ness.3523 if (E->getOpcode() == UO_Extension)3524 return EmitLValue(E->getSubExpr());3525 3526 QualType ExprTy = getContext().getCanonicalType(E->getSubExpr()->getType());3527 switch (E->getOpcode()) {3528 default: llvm_unreachable("Unknown unary operator lvalue!");3529 case UO_Deref: {3530 QualType T = E->getSubExpr()->getType()->getPointeeType();3531 assert(!T.isNull() && "CodeGenFunction::EmitUnaryOpLValue: Illegal type");3532 3533 LValueBaseInfo BaseInfo;3534 TBAAAccessInfo TBAAInfo;3535 Address Addr = EmitPointerWithAlignment(E->getSubExpr(), &BaseInfo,3536 &TBAAInfo);3537 LValue LV = MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);3538 LV.getQuals().setAddressSpace(ExprTy.getAddressSpace());3539 3540 // We should not generate __weak write barrier on indirect reference3541 // of a pointer to object; as in void foo (__weak id *param); *param = 0;3542 // But, we continue to generate __strong write barrier on indirect write3543 // into a pointer to object.3544 if (getLangOpts().ObjC &&3545 getLangOpts().getGC() != LangOptions::NonGC &&3546 LV.isObjCWeak())3547 LV.setNonGC(!E->isOBJCGCCandidate(getContext()));3548 return LV;3549 }3550 case UO_Real:3551 case UO_Imag: {3552 LValue LV = EmitLValue(E->getSubExpr());3553 assert(LV.isSimple() && "real/imag on non-ordinary l-value");3554 3555 // __real is valid on scalars. This is a faster way of testing that.3556 // __imag can only produce an rvalue on scalars.3557 if (E->getOpcode() == UO_Real &&3558 !LV.getAddress().getElementType()->isStructTy()) {3559 assert(E->getSubExpr()->getType()->isArithmeticType());3560 return LV;3561 }3562 3563 QualType T = ExprTy->castAs<ComplexType>()->getElementType();3564 3565 Address Component =3566 (E->getOpcode() == UO_Real3567 ? emitAddrOfRealComponent(LV.getAddress(), LV.getType())3568 : emitAddrOfImagComponent(LV.getAddress(), LV.getType()));3569 LValue ElemLV = MakeAddrLValue(Component, T, LV.getBaseInfo(),3570 CGM.getTBAAInfoForSubobject(LV, T));3571 ElemLV.getQuals().addQualifiers(LV.getQuals());3572 return ElemLV;3573 }3574 case UO_PreInc:3575 case UO_PreDec: {3576 LValue LV = EmitLValue(E->getSubExpr());3577 bool isInc = E->getOpcode() == UO_PreInc;3578 3579 if (E->getType()->isAnyComplexType())3580 EmitComplexPrePostIncDec(E, LV, isInc, true/*isPre*/);3581 else3582 EmitScalarPrePostIncDec(E, LV, isInc, true/*isPre*/);3583 return LV;3584 }3585 }3586}3587 3588LValue CodeGenFunction::EmitStringLiteralLValue(const StringLiteral *E) {3589 return MakeAddrLValue(CGM.GetAddrOfConstantStringFromLiteral(E),3590 E->getType(), AlignmentSource::Decl);3591}3592 3593LValue CodeGenFunction::EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E) {3594 return MakeAddrLValue(CGM.GetAddrOfConstantStringFromObjCEncode(E),3595 E->getType(), AlignmentSource::Decl);3596}3597 3598LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {3599 auto SL = E->getFunctionName();3600 assert(SL != nullptr && "No StringLiteral name in PredefinedExpr");3601 StringRef FnName = CurFn->getName();3602 FnName.consume_front("\01");3603 StringRef NameItems[] = {3604 PredefinedExpr::getIdentKindName(E->getIdentKind()), FnName};3605 std::string GVName = llvm::join(NameItems, NameItems + 2, ".");3606 if (auto *BD = dyn_cast_or_null<BlockDecl>(CurCodeDecl)) {3607 std::string Name = std::string(SL->getString());3608 if (!Name.empty()) {3609 unsigned Discriminator =3610 CGM.getCXXABI().getMangleContext().getBlockId(BD, true);3611 if (Discriminator)3612 Name += "_" + Twine(Discriminator + 1).str();3613 auto C = CGM.GetAddrOfConstantCString(Name, GVName);3614 return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl);3615 } else {3616 auto C = CGM.GetAddrOfConstantCString(std::string(FnName), GVName);3617 return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl);3618 }3619 }3620 auto C = CGM.GetAddrOfConstantStringFromLiteral(SL, GVName);3621 return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl);3622}3623 3624/// Emit a type description suitable for use by a runtime sanitizer library. The3625/// format of a type descriptor is3626///3627/// \code3628/// { i16 TypeKind, i16 TypeInfo }3629/// \endcode3630///3631/// followed by an array of i8 containing the type name with extra information3632/// for BitInt. TypeKind is TK_Integer(0) for an integer, TK_Float(1) for a3633/// floating point value, TK_BitInt(2) for BitInt and TK_Unknown(0xFFFF) for3634/// anything else.3635llvm::Constant *CodeGenFunction::EmitCheckTypeDescriptor(QualType T) {3636 // Only emit each type's descriptor once.3637 if (llvm::Constant *C = CGM.getTypeDescriptorFromMap(T))3638 return C;3639 3640 uint16_t TypeKind = TK_Unknown;3641 uint16_t TypeInfo = 0;3642 bool IsBitInt = false;3643 3644 if (T->isIntegerType()) {3645 TypeKind = TK_Integer;3646 TypeInfo = (llvm::Log2_32(getContext().getTypeSize(T)) << 1) |3647 (T->isSignedIntegerType() ? 1 : 0);3648 // Follow suggestion from discussion of issue 64100.3649 // So we can write the exact amount of bits in TypeName after '\0'3650 // making it <diagnostic-like type name>.'\0'.<32-bit width>.3651 if (T->isSignedIntegerType() && T->getAs<BitIntType>()) {3652 // Do a sanity checks as we are using 32-bit type to store bit length.3653 assert(getContext().getTypeSize(T) > 0 &&3654 " non positive amount of bits in __BitInt type");3655 assert(getContext().getTypeSize(T) <= 0xFFFFFFFF &&3656 " too many bits in __BitInt type");3657 3658 // Redefine TypeKind with the actual __BitInt type if we have signed3659 // BitInt.3660 TypeKind = TK_BitInt;3661 IsBitInt = true;3662 }3663 } else if (T->isFloatingType()) {3664 TypeKind = TK_Float;3665 TypeInfo = getContext().getTypeSize(T);3666 }3667 3668 // Format the type name as if for a diagnostic, including quotes and3669 // optionally an 'aka'.3670 SmallString<32> Buffer;3671 CGM.getDiags().ConvertArgToString(DiagnosticsEngine::ak_qualtype,3672 (intptr_t)T.getAsOpaquePtr(), StringRef(),3673 StringRef(), {}, Buffer, {});3674 3675 if (IsBitInt) {3676 // The Structure is: 0 to end the string, 32 bit unsigned integer in target3677 // endianness, zero.3678 char S[6] = {'\0', '\0', '\0', '\0', '\0', '\0'};3679 const auto *EIT = T->castAs<BitIntType>();3680 uint32_t Bits = EIT->getNumBits();3681 llvm::support::endian::write32(S + 1, Bits,3682 getTarget().isBigEndian()3683 ? llvm::endianness::big3684 : llvm::endianness::little);3685 StringRef Str = StringRef(S, sizeof(S) / sizeof(decltype(S[0])));3686 Buffer.append(Str);3687 }3688 3689 llvm::Constant *Components[] = {3690 Builder.getInt16(TypeKind), Builder.getInt16(TypeInfo),3691 llvm::ConstantDataArray::getString(getLLVMContext(), Buffer)3692 };3693 llvm::Constant *Descriptor = llvm::ConstantStruct::getAnon(Components);3694 3695 auto *GV = new llvm::GlobalVariable(3696 CGM.getModule(), Descriptor->getType(),3697 /*isConstant=*/true, llvm::GlobalVariable::PrivateLinkage, Descriptor);3698 GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);3699 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV);3700 3701 // Remember the descriptor for this type.3702 CGM.setTypeDescriptorInMap(T, GV);3703 3704 return GV;3705}3706 3707llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) {3708 llvm::Type *TargetTy = IntPtrTy;3709 3710 if (V->getType() == TargetTy)3711 return V;3712 3713 // Floating-point types which fit into intptr_t are bitcast to integers3714 // and then passed directly (after zero-extension, if necessary).3715 if (V->getType()->isFloatingPointTy()) {3716 unsigned Bits = V->getType()->getPrimitiveSizeInBits().getFixedValue();3717 if (Bits <= TargetTy->getIntegerBitWidth())3718 V = Builder.CreateBitCast(V, llvm::Type::getIntNTy(getLLVMContext(),3719 Bits));3720 }3721 3722 // Integers which fit in intptr_t are zero-extended and passed directly.3723 if (V->getType()->isIntegerTy() &&3724 V->getType()->getIntegerBitWidth() <= TargetTy->getIntegerBitWidth())3725 return Builder.CreateZExt(V, TargetTy);3726 3727 // Pointers are passed directly, everything else is passed by address.3728 if (!V->getType()->isPointerTy()) {3729 RawAddress Ptr = CreateDefaultAlignTempAlloca(V->getType());3730 Builder.CreateStore(V, Ptr);3731 V = Ptr.getPointer();3732 }3733 return Builder.CreatePtrToInt(V, TargetTy);3734}3735 3736/// Emit a representation of a SourceLocation for passing to a handler3737/// in a sanitizer runtime library. The format for this data is:3738/// \code3739/// struct SourceLocation {3740/// const char *Filename;3741/// int32_t Line, Column;3742/// };3743/// \endcode3744/// For an invalid SourceLocation, the Filename pointer is null.3745llvm::Constant *CodeGenFunction::EmitCheckSourceLocation(SourceLocation Loc) {3746 llvm::Constant *Filename;3747 int Line, Column;3748 3749 PresumedLoc PLoc = getContext().getSourceManager().getPresumedLoc(Loc);3750 if (PLoc.isValid()) {3751 StringRef FilenameString = PLoc.getFilename();3752 3753 int PathComponentsToStrip =3754 CGM.getCodeGenOpts().EmitCheckPathComponentsToStrip;3755 if (PathComponentsToStrip < 0) {3756 assert(PathComponentsToStrip != INT_MIN);3757 int PathComponentsToKeep = -PathComponentsToStrip;3758 auto I = llvm::sys::path::rbegin(FilenameString);3759 auto E = llvm::sys::path::rend(FilenameString);3760 while (I != E && --PathComponentsToKeep)3761 ++I;3762 3763 FilenameString = FilenameString.substr(I - E);3764 } else if (PathComponentsToStrip > 0) {3765 auto I = llvm::sys::path::begin(FilenameString);3766 auto E = llvm::sys::path::end(FilenameString);3767 while (I != E && PathComponentsToStrip--)3768 ++I;3769 3770 if (I != E)3771 FilenameString =3772 FilenameString.substr(I - llvm::sys::path::begin(FilenameString));3773 else3774 FilenameString = llvm::sys::path::filename(FilenameString);3775 }3776 3777 auto FilenameGV =3778 CGM.GetAddrOfConstantCString(std::string(FilenameString), ".src");3779 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(3780 cast<llvm::GlobalVariable>(3781 FilenameGV.getPointer()->stripPointerCasts()));3782 Filename = FilenameGV.getPointer();3783 Line = PLoc.getLine();3784 Column = PLoc.getColumn();3785 } else {3786 Filename = llvm::Constant::getNullValue(Int8PtrTy);3787 Line = Column = 0;3788 }3789 3790 llvm::Constant *Data[] = {Filename, Builder.getInt32(Line),3791 Builder.getInt32(Column)};3792 3793 return llvm::ConstantStruct::getAnon(Data);3794}3795 3796namespace {3797/// Specify under what conditions this check can be recovered3798enum class CheckRecoverableKind {3799 /// Always terminate program execution if this check fails.3800 Unrecoverable,3801 /// Check supports recovering, runtime has both fatal (noreturn) and3802 /// non-fatal handlers for this check.3803 Recoverable,3804 /// Runtime conditionally aborts, always need to support recovery.3805 AlwaysRecoverable3806};3807}3808 3809static CheckRecoverableKind3810getRecoverableKind(SanitizerKind::SanitizerOrdinal Ordinal) {3811 if (Ordinal == SanitizerKind::SO_Vptr)3812 return CheckRecoverableKind::AlwaysRecoverable;3813 else if (Ordinal == SanitizerKind::SO_Return ||3814 Ordinal == SanitizerKind::SO_Unreachable)3815 return CheckRecoverableKind::Unrecoverable;3816 else3817 return CheckRecoverableKind::Recoverable;3818}3819 3820namespace {3821struct SanitizerHandlerInfo {3822 char const *const Name;3823 unsigned Version;3824};3825}3826 3827const SanitizerHandlerInfo SanitizerHandlers[] = {3828#define SANITIZER_CHECK(Enum, Name, Version, Msg) {#Name, Version},3829 LIST_SANITIZER_CHECKS3830#undef SANITIZER_CHECK3831};3832 3833static void emitCheckHandlerCall(CodeGenFunction &CGF,3834 llvm::FunctionType *FnType,3835 ArrayRef<llvm::Value *> FnArgs,3836 SanitizerHandler CheckHandler,3837 CheckRecoverableKind RecoverKind, bool IsFatal,3838 llvm::BasicBlock *ContBB, bool NoMerge) {3839 assert(IsFatal || RecoverKind != CheckRecoverableKind::Unrecoverable);3840 std::optional<ApplyDebugLocation> DL;3841 if (!CGF.Builder.getCurrentDebugLocation()) {3842 // Ensure that the call has at least an artificial debug location.3843 DL.emplace(CGF, SourceLocation());3844 }3845 bool NeedsAbortSuffix =3846 IsFatal && RecoverKind != CheckRecoverableKind::Unrecoverable;3847 bool MinimalRuntime = CGF.CGM.getCodeGenOpts().SanitizeMinimalRuntime;3848 bool HandlerPreserveAllRegs =3849 CGF.CGM.getCodeGenOpts().SanitizeHandlerPreserveAllRegs;3850 const SanitizerHandlerInfo &CheckInfo = SanitizerHandlers[CheckHandler];3851 const StringRef CheckName = CheckInfo.Name;3852 std::string FnName = "__ubsan_handle_" + CheckName.str();3853 if (CheckInfo.Version && !MinimalRuntime)3854 FnName += "_v" + llvm::utostr(CheckInfo.Version);3855 if (MinimalRuntime)3856 FnName += "_minimal";3857 if (NeedsAbortSuffix)3858 FnName += "_abort";3859 if (HandlerPreserveAllRegs && !NeedsAbortSuffix)3860 FnName += "_preserve";3861 bool MayReturn =3862 !IsFatal || RecoverKind == CheckRecoverableKind::AlwaysRecoverable;3863 3864 llvm::AttrBuilder B(CGF.getLLVMContext());3865 if (!MayReturn) {3866 B.addAttribute(llvm::Attribute::NoReturn)3867 .addAttribute(llvm::Attribute::NoUnwind);3868 }3869 B.addUWTableAttr(llvm::UWTableKind::Default);3870 3871 llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(3872 FnType, FnName,3873 llvm::AttributeList::get(CGF.getLLVMContext(),3874 llvm::AttributeList::FunctionIndex, B),3875 /*Local=*/true);3876 llvm::CallInst *HandlerCall = CGF.EmitNounwindRuntimeCall(Fn, FnArgs);3877 NoMerge = NoMerge || !CGF.CGM.getCodeGenOpts().OptimizationLevel ||3878 (CGF.CurCodeDecl && CGF.CurCodeDecl->hasAttr<OptimizeNoneAttr>());3879 if (NoMerge)3880 HandlerCall->addFnAttr(llvm::Attribute::NoMerge);3881 if (HandlerPreserveAllRegs && !NeedsAbortSuffix) {3882 // N.B. there is also a clang::CallingConv which is not what we want here.3883 HandlerCall->setCallingConv(llvm::CallingConv::PreserveAll);3884 }3885 if (!MayReturn) {3886 HandlerCall->setDoesNotReturn();3887 CGF.Builder.CreateUnreachable();3888 } else {3889 CGF.Builder.CreateBr(ContBB);3890 }3891}3892 3893void CodeGenFunction::EmitCheck(3894 ArrayRef<std::pair<llvm::Value *, SanitizerKind::SanitizerOrdinal>> Checked,3895 SanitizerHandler CheckHandler, ArrayRef<llvm::Constant *> StaticArgs,3896 ArrayRef<llvm::Value *> DynamicArgs, const TrapReason *TR) {3897 assert(IsSanitizerScope);3898 assert(Checked.size() > 0);3899 assert(CheckHandler >= 0 &&3900 size_t(CheckHandler) < std::size(SanitizerHandlers));3901 const StringRef CheckName = SanitizerHandlers[CheckHandler].Name;3902 3903 llvm::Value *FatalCond = nullptr;3904 llvm::Value *RecoverableCond = nullptr;3905 llvm::Value *TrapCond = nullptr;3906 bool NoMerge = false;3907 // Expand checks into:3908 // (Check1 || !allow_ubsan_check) && (Check2 || !allow_ubsan_check) ...3909 // We need separate allow_ubsan_check intrinsics because they have separately3910 // specified cutoffs.3911 // This expression looks expensive but will be simplified after3912 // LowerAllowCheckPass.3913 for (auto &[Check, Ord] : Checked) {3914 llvm::Value *GuardedCheck = Check;3915 if (ClSanitizeGuardChecks ||3916 (CGM.getCodeGenOpts().SanitizeSkipHotCutoffs[Ord] > 0)) {3917 llvm::Value *Allow = Builder.CreateCall(3918 CGM.getIntrinsic(llvm::Intrinsic::allow_ubsan_check),3919 llvm::ConstantInt::get(CGM.Int8Ty, Ord));3920 GuardedCheck = Builder.CreateOr(Check, Builder.CreateNot(Allow));3921 }3922 3923 // -fsanitize-trap= overrides -fsanitize-recover=.3924 llvm::Value *&Cond = CGM.getCodeGenOpts().SanitizeTrap.has(Ord) ? TrapCond3925 : CGM.getCodeGenOpts().SanitizeRecover.has(Ord)3926 ? RecoverableCond3927 : FatalCond;3928 Cond = Cond ? Builder.CreateAnd(Cond, GuardedCheck) : GuardedCheck;3929 3930 if (!CGM.getCodeGenOpts().SanitizeMergeHandlers.has(Ord))3931 NoMerge = true;3932 }3933 3934 if (TrapCond)3935 EmitTrapCheck(TrapCond, CheckHandler, NoMerge, TR);3936 if (!FatalCond && !RecoverableCond)3937 return;3938 3939 llvm::Value *JointCond;3940 if (FatalCond && RecoverableCond)3941 JointCond = Builder.CreateAnd(FatalCond, RecoverableCond);3942 else3943 JointCond = FatalCond ? FatalCond : RecoverableCond;3944 assert(JointCond);3945 3946 CheckRecoverableKind RecoverKind = getRecoverableKind(Checked[0].second);3947 assert(SanOpts.has(Checked[0].second));3948#ifndef NDEBUG3949 for (int i = 1, n = Checked.size(); i < n; ++i) {3950 assert(RecoverKind == getRecoverableKind(Checked[i].second) &&3951 "All recoverable kinds in a single check must be same!");3952 assert(SanOpts.has(Checked[i].second));3953 }3954#endif3955 3956 llvm::BasicBlock *Cont = createBasicBlock("cont");3957 llvm::BasicBlock *Handlers = createBasicBlock("handler." + CheckName);3958 llvm::Instruction *Branch = Builder.CreateCondBr(JointCond, Cont, Handlers);3959 // Give hint that we very much don't expect to execute the handler3960 llvm::MDBuilder MDHelper(getLLVMContext());3961 llvm::MDNode *Node = MDHelper.createLikelyBranchWeights();3962 Branch->setMetadata(llvm::LLVMContext::MD_prof, Node);3963 EmitBlock(Handlers);3964 3965 // Clear arguments for the MinimalRuntime handler.3966 if (CGM.getCodeGenOpts().SanitizeMinimalRuntime) {3967 StaticArgs = {};3968 DynamicArgs = {};3969 }3970 3971 // Handler functions take an i8* pointing to the (handler-specific) static3972 // information block, followed by a sequence of intptr_t arguments3973 // representing operand values.3974 SmallVector<llvm::Value *, 4> Args;3975 SmallVector<llvm::Type *, 4> ArgTypes;3976 3977 Args.reserve(DynamicArgs.size() + 1);3978 ArgTypes.reserve(DynamicArgs.size() + 1);3979 3980 // Emit handler arguments and create handler function type.3981 if (!StaticArgs.empty()) {3982 llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs);3983 auto *InfoPtr = new llvm::GlobalVariable(3984 CGM.getModule(), Info->getType(),3985 // Non-constant global is used in a handler to deduplicate reports.3986 // TODO: change deduplication logic and make it constant.3987 /*isConstant=*/false, llvm::GlobalVariable::PrivateLinkage, Info, "",3988 nullptr, llvm::GlobalVariable::NotThreadLocal,3989 CGM.getDataLayout().getDefaultGlobalsAddressSpace());3990 InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);3991 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(InfoPtr);3992 Args.push_back(InfoPtr);3993 ArgTypes.push_back(Args.back()->getType());3994 }3995 3996 for (llvm::Value *DynamicArg : DynamicArgs) {3997 Args.push_back(EmitCheckValue(DynamicArg));3998 ArgTypes.push_back(IntPtrTy);3999 }4000 4001 llvm::FunctionType *FnType =4002 llvm::FunctionType::get(CGM.VoidTy, ArgTypes, false);4003 4004 if (!FatalCond || !RecoverableCond) {4005 // Simple case: we need to generate a single handler call, either4006 // fatal, or non-fatal.4007 emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind,4008 (FatalCond != nullptr), Cont, NoMerge);4009 } else {4010 // Emit two handler calls: first one for set of unrecoverable checks,4011 // another one for recoverable.4012 llvm::BasicBlock *NonFatalHandlerBB =4013 createBasicBlock("non_fatal." + CheckName);4014 llvm::BasicBlock *FatalHandlerBB = createBasicBlock("fatal." + CheckName);4015 Builder.CreateCondBr(FatalCond, NonFatalHandlerBB, FatalHandlerBB);4016 EmitBlock(FatalHandlerBB);4017 emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind, true,4018 NonFatalHandlerBB, NoMerge);4019 EmitBlock(NonFatalHandlerBB);4020 emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind, false,4021 Cont, NoMerge);4022 }4023 4024 EmitBlock(Cont);4025}4026 4027void CodeGenFunction::EmitCfiSlowPathCheck(4028 SanitizerKind::SanitizerOrdinal Ordinal, llvm::Value *Cond,4029 llvm::ConstantInt *TypeId, llvm::Value *Ptr,4030 ArrayRef<llvm::Constant *> StaticArgs) {4031 llvm::BasicBlock *Cont = createBasicBlock("cfi.cont");4032 4033 llvm::BasicBlock *CheckBB = createBasicBlock("cfi.slowpath");4034 llvm::BranchInst *BI = Builder.CreateCondBr(Cond, Cont, CheckBB);4035 4036 llvm::MDBuilder MDHelper(getLLVMContext());4037 llvm::MDNode *Node = MDHelper.createLikelyBranchWeights();4038 BI->setMetadata(llvm::LLVMContext::MD_prof, Node);4039 4040 EmitBlock(CheckBB);4041 4042 bool WithDiag = !CGM.getCodeGenOpts().SanitizeTrap.has(Ordinal);4043 4044 llvm::CallInst *CheckCall;4045 llvm::FunctionCallee SlowPathFn;4046 if (WithDiag) {4047 llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs);4048 auto *InfoPtr =4049 new llvm::GlobalVariable(CGM.getModule(), Info->getType(), false,4050 llvm::GlobalVariable::PrivateLinkage, Info);4051 InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);4052 CGM.getSanitizerMetadata()->disableSanitizerForGlobal(InfoPtr);4053 4054 SlowPathFn = CGM.getModule().getOrInsertFunction(4055 "__cfi_slowpath_diag",4056 llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy, Int8PtrTy},4057 false));4058 CheckCall = Builder.CreateCall(SlowPathFn, {TypeId, Ptr, InfoPtr});4059 } else {4060 SlowPathFn = CGM.getModule().getOrInsertFunction(4061 "__cfi_slowpath",4062 llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy}, false));4063 CheckCall = Builder.CreateCall(SlowPathFn, {TypeId, Ptr});4064 }4065 4066 CGM.setDSOLocal(4067 cast<llvm::GlobalValue>(SlowPathFn.getCallee()->stripPointerCasts()));4068 CheckCall->setDoesNotThrow();4069 4070 EmitBlock(Cont);4071}4072 4073// Emit a stub for __cfi_check function so that the linker knows about this4074// symbol in LTO mode.4075void CodeGenFunction::EmitCfiCheckStub() {4076 llvm::Module *M = &CGM.getModule();4077 ASTContext &C = getContext();4078 QualType QInt64Ty = C.getIntTypeForBitwidth(64, false);4079 4080 FunctionArgList FnArgs;4081 ImplicitParamDecl ArgCallsiteTypeId(C, QInt64Ty, ImplicitParamKind::Other);4082 ImplicitParamDecl ArgAddr(C, C.VoidPtrTy, ImplicitParamKind::Other);4083 ImplicitParamDecl ArgCFICheckFailData(C, C.VoidPtrTy,4084 ImplicitParamKind::Other);4085 FnArgs.push_back(&ArgCallsiteTypeId);4086 FnArgs.push_back(&ArgAddr);4087 FnArgs.push_back(&ArgCFICheckFailData);4088 const CGFunctionInfo &FI =4089 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, FnArgs);4090 4091 llvm::Function *F = llvm::Function::Create(4092 llvm::FunctionType::get(VoidTy, {Int64Ty, VoidPtrTy, VoidPtrTy}, false),4093 llvm::GlobalValue::WeakAnyLinkage, "__cfi_check", M);4094 CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, F, /*IsThunk=*/false);4095 CGM.SetLLVMFunctionAttributesForDefinition(nullptr, F);4096 F->setAlignment(llvm::Align(4096));4097 CGM.setDSOLocal(F);4098 4099 llvm::LLVMContext &Ctx = M->getContext();4100 llvm::BasicBlock *BB = llvm::BasicBlock::Create(Ctx, "entry", F);4101 // CrossDSOCFI pass is not executed if there is no executable code.4102 SmallVector<llvm::Value*> Args{F->getArg(2), F->getArg(1)};4103 llvm::CallInst::Create(M->getFunction("__cfi_check_fail"), Args, "", BB);4104 llvm::ReturnInst::Create(Ctx, nullptr, BB);4105}4106 4107// This function is basically a switch over the CFI failure kind, which is4108// extracted from CFICheckFailData (1st function argument). Each case is either4109// llvm.trap or a call to one of the two runtime handlers, based on4110// -fsanitize-trap and -fsanitize-recover settings. Default case (invalid4111// failure kind) traps, but this should really never happen. CFICheckFailData4112// can be nullptr if the calling module has -fsanitize-trap behavior for this4113// check kind; in this case __cfi_check_fail traps as well.4114void CodeGenFunction::EmitCfiCheckFail() {4115 auto CheckHandler = SanitizerHandler::CFICheckFail;4116 // TODO: the SanitizerKind is not yet determined for this check (and might4117 // not even be available, if Data == nullptr). However, we still want to4118 // annotate the instrumentation. We approximate this by using all the CFI4119 // kinds.4120 SanitizerDebugLocation SanScope(4121 this,4122 {SanitizerKind::SO_CFIVCall, SanitizerKind::SO_CFINVCall,4123 SanitizerKind::SO_CFIDerivedCast, SanitizerKind::SO_CFIUnrelatedCast,4124 SanitizerKind::SO_CFIICall},4125 CheckHandler);4126 FunctionArgList Args;4127 ImplicitParamDecl ArgData(getContext(), getContext().VoidPtrTy,4128 ImplicitParamKind::Other);4129 ImplicitParamDecl ArgAddr(getContext(), getContext().VoidPtrTy,4130 ImplicitParamKind::Other);4131 Args.push_back(&ArgData);4132 Args.push_back(&ArgAddr);4133 4134 const CGFunctionInfo &FI =4135 CGM.getTypes().arrangeBuiltinFunctionDeclaration(getContext().VoidTy, Args);4136 4137 llvm::Function *F = llvm::Function::Create(4138 llvm::FunctionType::get(VoidTy, {VoidPtrTy, VoidPtrTy}, false),4139 llvm::GlobalValue::WeakODRLinkage, "__cfi_check_fail", &CGM.getModule());4140 4141 CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, F, /*IsThunk=*/false);4142 CGM.SetLLVMFunctionAttributesForDefinition(nullptr, F);4143 F->setVisibility(llvm::GlobalValue::HiddenVisibility);4144 4145 StartFunction(GlobalDecl(), CGM.getContext().VoidTy, F, FI, Args,4146 SourceLocation());4147 4148 ApplyDebugLocation ADL = ApplyDebugLocation::CreateArtificial(*this);4149 4150 // This function is not affected by NoSanitizeList. This function does4151 // not have a source location, but "src:*" would still apply. Revert any4152 // changes to SanOpts made in StartFunction.4153 SanOpts = CGM.getLangOpts().Sanitize;4154 4155 llvm::Value *Data =4156 EmitLoadOfScalar(GetAddrOfLocalVar(&ArgData), /*Volatile=*/false,4157 CGM.getContext().VoidPtrTy, ArgData.getLocation());4158 llvm::Value *Addr =4159 EmitLoadOfScalar(GetAddrOfLocalVar(&ArgAddr), /*Volatile=*/false,4160 CGM.getContext().VoidPtrTy, ArgAddr.getLocation());4161 4162 // Data == nullptr means the calling module has trap behaviour for this check.4163 llvm::Value *DataIsNotNullPtr =4164 Builder.CreateICmpNE(Data, llvm::ConstantPointerNull::get(Int8PtrTy));4165 // TODO: since there is no data, we don't know the CheckKind, and therefore4166 // cannot inspect CGM.getCodeGenOpts().SanitizeMergeHandlers. We default to4167 // NoMerge = false. Users can disable merging by disabling optimization.4168 EmitTrapCheck(DataIsNotNullPtr, SanitizerHandler::CFICheckFail,4169 /*NoMerge=*/false);4170 4171 llvm::StructType *SourceLocationTy =4172 llvm::StructType::get(VoidPtrTy, Int32Ty, Int32Ty);4173 llvm::StructType *CfiCheckFailDataTy =4174 llvm::StructType::get(Int8Ty, SourceLocationTy, VoidPtrTy);4175 4176 llvm::Value *V = Builder.CreateConstGEP2_32(4177 CfiCheckFailDataTy, Builder.CreatePointerCast(Data, DefaultPtrTy), 0, 0);4178 4179 Address CheckKindAddr(V, Int8Ty, getIntAlign());4180 llvm::Value *CheckKind = Builder.CreateLoad(CheckKindAddr);4181 4182 llvm::Value *AllVtables = llvm::MetadataAsValue::get(4183 CGM.getLLVMContext(),4184 llvm::MDString::get(CGM.getLLVMContext(), "all-vtables"));4185 llvm::Value *ValidVtable = Builder.CreateZExt(4186 Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::type_test),4187 {Addr, AllVtables}),4188 IntPtrTy);4189 4190 const std::pair<int, SanitizerKind::SanitizerOrdinal> CheckKinds[] = {4191 {CFITCK_VCall, SanitizerKind::SO_CFIVCall},4192 {CFITCK_NVCall, SanitizerKind::SO_CFINVCall},4193 {CFITCK_DerivedCast, SanitizerKind::SO_CFIDerivedCast},4194 {CFITCK_UnrelatedCast, SanitizerKind::SO_CFIUnrelatedCast},4195 {CFITCK_ICall, SanitizerKind::SO_CFIICall}};4196 4197 for (auto CheckKindOrdinalPair : CheckKinds) {4198 int Kind = CheckKindOrdinalPair.first;4199 SanitizerKind::SanitizerOrdinal Ordinal = CheckKindOrdinalPair.second;4200 4201 // TODO: we could apply SanitizerAnnotateDebugInfo(Ordinal) instead of4202 // relying on the SanitizerScope with all CFI ordinals4203 4204 llvm::Value *Cond =4205 Builder.CreateICmpNE(CheckKind, llvm::ConstantInt::get(Int8Ty, Kind));4206 if (CGM.getLangOpts().Sanitize.has(Ordinal))4207 EmitCheck(std::make_pair(Cond, Ordinal), SanitizerHandler::CFICheckFail,4208 {}, {Data, Addr, ValidVtable});4209 else4210 // TODO: we can't rely on CGM.getCodeGenOpts().SanitizeMergeHandlers.4211 // Although the compiler allows SanitizeMergeHandlers to be set4212 // independently of CGM.getLangOpts().Sanitize, Driver/SanitizerArgs.cpp4213 // requires that SanitizeMergeHandlers is a subset of Sanitize.4214 EmitTrapCheck(Cond, CheckHandler, /*NoMerge=*/false);4215 }4216 4217 FinishFunction();4218 // The only reference to this function will be created during LTO link.4219 // Make sure it survives until then.4220 CGM.addUsedGlobal(F);4221}4222 4223void CodeGenFunction::EmitUnreachable(SourceLocation Loc) {4224 if (SanOpts.has(SanitizerKind::Unreachable)) {4225 auto CheckOrdinal = SanitizerKind::SO_Unreachable;4226 auto CheckHandler = SanitizerHandler::BuiltinUnreachable;4227 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);4228 EmitCheck(std::make_pair(static_cast<llvm::Value *>(Builder.getFalse()),4229 CheckOrdinal),4230 CheckHandler, EmitCheckSourceLocation(Loc), {});4231 }4232 Builder.CreateUnreachable();4233}4234 4235void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked,4236 SanitizerHandler CheckHandlerID,4237 bool NoMerge, const TrapReason *TR) {4238 llvm::BasicBlock *Cont = createBasicBlock("cont");4239 4240 // If we're optimizing, collapse all calls to trap down to just one per4241 // check-type per function to save on code size.4242 if ((int)TrapBBs.size() <= CheckHandlerID)4243 TrapBBs.resize(CheckHandlerID + 1);4244 4245 llvm::BasicBlock *&TrapBB = TrapBBs[CheckHandlerID];4246 4247 llvm::DILocation *TrapLocation = Builder.getCurrentDebugLocation();4248 llvm::StringRef TrapMessage;4249 llvm::StringRef TrapCategory;4250 auto DebugTrapReasonKind = CGM.getCodeGenOpts().getSanitizeDebugTrapReasons();4251 if (TR && !TR->isEmpty() &&4252 DebugTrapReasonKind ==4253 CodeGenOptions::SanitizeDebugTrapReasonKind::Detailed) {4254 TrapMessage = TR->getMessage();4255 TrapCategory = TR->getCategory();4256 } else {4257 TrapMessage = GetUBSanTrapForHandler(CheckHandlerID);4258 TrapCategory = "Undefined Behavior Sanitizer";4259 }4260 4261 if (getDebugInfo() && !TrapMessage.empty() &&4262 DebugTrapReasonKind !=4263 CodeGenOptions::SanitizeDebugTrapReasonKind::None &&4264 TrapLocation) {4265 TrapLocation = getDebugInfo()->CreateTrapFailureMessageFor(4266 TrapLocation, TrapCategory, TrapMessage);4267 }4268 4269 NoMerge = NoMerge || !CGM.getCodeGenOpts().OptimizationLevel ||4270 (CurCodeDecl && CurCodeDecl->hasAttr<OptimizeNoneAttr>());4271 4272 llvm::MDBuilder MDHelper(getLLVMContext());4273 if (TrapBB && !NoMerge) {4274 auto Call = TrapBB->begin();4275 assert(isa<llvm::CallInst>(Call) && "Expected call in trap BB");4276 4277 Call->applyMergedLocation(Call->getDebugLoc(), TrapLocation);4278 4279 Builder.CreateCondBr(Checked, Cont, TrapBB,4280 MDHelper.createLikelyBranchWeights());4281 } else {4282 TrapBB = createBasicBlock("trap");4283 Builder.CreateCondBr(Checked, Cont, TrapBB,4284 MDHelper.createLikelyBranchWeights());4285 EmitBlock(TrapBB);4286 4287 ApplyDebugLocation applyTrapDI(*this, TrapLocation);4288 4289 llvm::CallInst *TrapCall =4290 Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::ubsantrap),4291 llvm::ConstantInt::get(CGM.Int8Ty, CheckHandlerID));4292 4293 if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {4294 auto A = llvm::Attribute::get(getLLVMContext(), "trap-func-name",4295 CGM.getCodeGenOpts().TrapFuncName);4296 TrapCall->addFnAttr(A);4297 }4298 if (NoMerge)4299 TrapCall->addFnAttr(llvm::Attribute::NoMerge);4300 TrapCall->setDoesNotReturn();4301 TrapCall->setDoesNotThrow();4302 Builder.CreateUnreachable();4303 }4304 4305 EmitBlock(Cont);4306}4307 4308llvm::CallInst *CodeGenFunction::EmitTrapCall(llvm::Intrinsic::ID IntrID) {4309 llvm::CallInst *TrapCall =4310 Builder.CreateCall(CGM.getIntrinsic(IntrID));4311 4312 if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {4313 auto A = llvm::Attribute::get(getLLVMContext(), "trap-func-name",4314 CGM.getCodeGenOpts().TrapFuncName);4315 TrapCall->addFnAttr(A);4316 }4317 4318 if (InNoMergeAttributedStmt)4319 TrapCall->addFnAttr(llvm::Attribute::NoMerge);4320 return TrapCall;4321}4322 4323Address CodeGenFunction::EmitArrayToPointerDecay(const Expr *E,4324 LValueBaseInfo *BaseInfo,4325 TBAAAccessInfo *TBAAInfo) {4326 assert(E->getType()->isArrayType() &&4327 "Array to pointer decay must have array source type!");4328 4329 // Expressions of array type can't be bitfields or vector elements.4330 LValue LV = EmitLValue(E);4331 Address Addr = LV.getAddress();4332 4333 // If the array type was an incomplete type, we need to make sure4334 // the decay ends up being the right type.4335 llvm::Type *NewTy = ConvertType(E->getType());4336 Addr = Addr.withElementType(NewTy);4337 4338 // Note that VLA pointers are always decayed, so we don't need to do4339 // anything here.4340 if (!E->getType()->isVariableArrayType()) {4341 assert(isa<llvm::ArrayType>(Addr.getElementType()) &&4342 "Expected pointer to array");4343 Addr = Builder.CreateConstArrayGEP(Addr, 0, "arraydecay");4344 }4345 4346 // The result of this decay conversion points to an array element within the4347 // base lvalue. However, since TBAA currently does not support representing4348 // accesses to elements of member arrays, we conservatively represent accesses4349 // to the pointee object as if it had no any base lvalue specified.4350 // TODO: Support TBAA for member arrays.4351 QualType EltType = E->getType()->castAsArrayTypeUnsafe()->getElementType();4352 if (BaseInfo) *BaseInfo = LV.getBaseInfo();4353 if (TBAAInfo) *TBAAInfo = CGM.getTBAAAccessInfo(EltType);4354 4355 return Addr.withElementType(ConvertTypeForMem(EltType));4356}4357 4358/// isSimpleArrayDecayOperand - If the specified expr is a simple decay from an4359/// array to pointer, return the array subexpression.4360static const Expr *isSimpleArrayDecayOperand(const Expr *E) {4361 // If this isn't just an array->pointer decay, bail out.4362 const auto *CE = dyn_cast<CastExpr>(E);4363 if (!CE || CE->getCastKind() != CK_ArrayToPointerDecay)4364 return nullptr;4365 4366 // If this is a decay from variable width array, bail out.4367 const Expr *SubExpr = CE->getSubExpr();4368 if (SubExpr->getType()->isVariableArrayType())4369 return nullptr;4370 4371 return SubExpr;4372}4373 4374static llvm::Value *emitArraySubscriptGEP(CodeGenFunction &CGF,4375 llvm::Type *elemType,4376 llvm::Value *ptr,4377 ArrayRef<llvm::Value*> indices,4378 bool inbounds,4379 bool signedIndices,4380 SourceLocation loc,4381 const llvm::Twine &name = "arrayidx") {4382 if (inbounds) {4383 return CGF.EmitCheckedInBoundsGEP(elemType, ptr, indices, signedIndices,4384 CodeGenFunction::NotSubtraction, loc,4385 name);4386 } else {4387 return CGF.Builder.CreateGEP(elemType, ptr, indices, name);4388 }4389}4390 4391static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,4392 ArrayRef<llvm::Value *> indices,4393 llvm::Type *elementType, bool inbounds,4394 bool signedIndices, SourceLocation loc,4395 CharUnits align,4396 const llvm::Twine &name = "arrayidx") {4397 if (inbounds) {4398 return CGF.EmitCheckedInBoundsGEP(addr, indices, elementType, signedIndices,4399 CodeGenFunction::NotSubtraction, loc,4400 align, name);4401 } else {4402 return CGF.Builder.CreateGEP(addr, indices, elementType, align, name);4403 }4404}4405 4406static QualType getFixedSizeElementType(const ASTContext &ctx,4407 const VariableArrayType *vla) {4408 QualType eltType;4409 do {4410 eltType = vla->getElementType();4411 } while ((vla = ctx.getAsVariableArrayType(eltType)));4412 return eltType;4413}4414 4415static bool hasBPFPreserveStaticOffset(const RecordDecl *D) {4416 return D && D->hasAttr<BPFPreserveStaticOffsetAttr>();4417}4418 4419static bool hasBPFPreserveStaticOffset(const Expr *E) {4420 if (!E)4421 return false;4422 QualType PointeeType = E->getType()->getPointeeType();4423 if (PointeeType.isNull())4424 return false;4425 if (const auto *BaseDecl = PointeeType->getAsRecordDecl())4426 return hasBPFPreserveStaticOffset(BaseDecl);4427 return false;4428}4429 4430// Wraps Addr with a call to llvm.preserve.static.offset intrinsic.4431static Address wrapWithBPFPreserveStaticOffset(CodeGenFunction &CGF,4432 Address &Addr) {4433 if (!CGF.getTarget().getTriple().isBPF())4434 return Addr;4435 4436 llvm::Function *Fn =4437 CGF.CGM.getIntrinsic(llvm::Intrinsic::preserve_static_offset);4438 llvm::CallInst *Call = CGF.Builder.CreateCall(Fn, {Addr.emitRawPointer(CGF)});4439 return Address(Call, Addr.getElementType(), Addr.getAlignment());4440}4441 4442/// Given an array base, check whether its member access belongs to a record4443/// with preserve_access_index attribute or not.4444static bool IsPreserveAIArrayBase(CodeGenFunction &CGF, const Expr *ArrayBase) {4445 if (!ArrayBase || !CGF.getDebugInfo())4446 return false;4447 4448 // Only support base as either a MemberExpr or DeclRefExpr.4449 // DeclRefExpr to cover cases like:4450 // struct s { int a; int b[10]; };4451 // struct s *p;4452 // p[1].a4453 // p[1] will generate a DeclRefExpr and p[1].a is a MemberExpr.4454 // p->b[5] is a MemberExpr example.4455 const Expr *E = ArrayBase->IgnoreImpCasts();4456 if (const auto *ME = dyn_cast<MemberExpr>(E))4457 return ME->getMemberDecl()->hasAttr<BPFPreserveAccessIndexAttr>();4458 4459 if (const auto *DRE = dyn_cast<DeclRefExpr>(E)) {4460 const auto *VarDef = dyn_cast<VarDecl>(DRE->getDecl());4461 if (!VarDef)4462 return false;4463 4464 const auto *PtrT = VarDef->getType()->getAs<PointerType>();4465 if (!PtrT)4466 return false;4467 4468 const auto *PointeeT = PtrT->getPointeeType()4469 ->getUnqualifiedDesugaredType();4470 if (const auto *RecT = dyn_cast<RecordType>(PointeeT))4471 return RecT->getDecl()4472 ->getMostRecentDecl()4473 ->hasAttr<BPFPreserveAccessIndexAttr>();4474 return false;4475 }4476 4477 return false;4478}4479 4480static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,4481 ArrayRef<llvm::Value *> indices,4482 QualType eltType, bool inbounds,4483 bool signedIndices, SourceLocation loc,4484 QualType *arrayType = nullptr,4485 const Expr *Base = nullptr,4486 const llvm::Twine &name = "arrayidx") {4487 // All the indices except that last must be zero.4488#ifndef NDEBUG4489 for (auto *idx : indices.drop_back())4490 assert(isa<llvm::ConstantInt>(idx) &&4491 cast<llvm::ConstantInt>(idx)->isZero());4492#endif4493 4494 // Determine the element size of the statically-sized base. This is4495 // the thing that the indices are expressed in terms of.4496 if (auto vla = CGF.getContext().getAsVariableArrayType(eltType)) {4497 eltType = getFixedSizeElementType(CGF.getContext(), vla);4498 }4499 4500 // We can use that to compute the best alignment of the element.4501 CharUnits eltSize = CGF.getContext().getTypeSizeInChars(eltType);4502 CharUnits eltAlign =4503 getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize);4504 4505 if (hasBPFPreserveStaticOffset(Base))4506 addr = wrapWithBPFPreserveStaticOffset(CGF, addr);4507 4508 llvm::Value *eltPtr;4509 auto LastIndex = dyn_cast<llvm::ConstantInt>(indices.back());4510 if (!LastIndex ||4511 (!CGF.IsInPreservedAIRegion && !IsPreserveAIArrayBase(CGF, Base))) {4512 addr = emitArraySubscriptGEP(CGF, addr, indices,4513 CGF.ConvertTypeForMem(eltType), inbounds,4514 signedIndices, loc, eltAlign, name);4515 return addr;4516 } else {4517 // Remember the original array subscript for bpf target4518 unsigned idx = LastIndex->getZExtValue();4519 llvm::DIType *DbgInfo = nullptr;4520 if (arrayType)4521 DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(*arrayType, loc);4522 eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(4523 addr.getElementType(), addr.emitRawPointer(CGF), indices.size() - 1,4524 idx, DbgInfo);4525 }4526 4527 return Address(eltPtr, CGF.ConvertTypeForMem(eltType), eltAlign);4528}4529 4530namespace {4531 4532/// StructFieldAccess is a simple visitor class to grab the first l-value to4533/// r-value cast Expr.4534struct StructFieldAccess4535 : public ConstStmtVisitor<StructFieldAccess, const Expr *> {4536 const Expr *VisitCastExpr(const CastExpr *E) {4537 if (E->getCastKind() == CK_LValueToRValue)4538 return E;4539 return Visit(E->getSubExpr());4540 }4541 const Expr *VisitParenExpr(const ParenExpr *E) {4542 return Visit(E->getSubExpr());4543 }4544};4545 4546} // end anonymous namespace4547 4548/// The offset of a field from the beginning of the record.4549static bool getFieldOffsetInBits(CodeGenFunction &CGF, const RecordDecl *RD,4550 const FieldDecl *Field, int64_t &Offset) {4551 ASTContext &Ctx = CGF.getContext();4552 const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(RD);4553 unsigned FieldNo = 0;4554 4555 for (const FieldDecl *FD : RD->fields()) {4556 if (FD == Field) {4557 Offset += Layout.getFieldOffset(FieldNo);4558 return true;4559 }4560 4561 QualType Ty = FD->getType();4562 if (Ty->isRecordType())4563 if (getFieldOffsetInBits(CGF, Ty->getAsRecordDecl(), Field, Offset)) {4564 Offset += Layout.getFieldOffset(FieldNo);4565 return true;4566 }4567 4568 if (!RD->isUnion())4569 ++FieldNo;4570 }4571 4572 return false;4573}4574 4575/// Returns the relative offset difference between \p FD1 and \p FD2.4576/// \code4577/// offsetof(struct foo, FD1) - offsetof(struct foo, FD2)4578/// \endcode4579/// Both fields must be within the same struct.4580static std::optional<int64_t> getOffsetDifferenceInBits(CodeGenFunction &CGF,4581 const FieldDecl *FD1,4582 const FieldDecl *FD2) {4583 const RecordDecl *FD1OuterRec =4584 FD1->getParent()->getOuterLexicalRecordContext();4585 const RecordDecl *FD2OuterRec =4586 FD2->getParent()->getOuterLexicalRecordContext();4587 4588 if (FD1OuterRec != FD2OuterRec)4589 // Fields must be within the same RecordDecl.4590 return std::optional<int64_t>();4591 4592 int64_t FD1Offset = 0;4593 if (!getFieldOffsetInBits(CGF, FD1OuterRec, FD1, FD1Offset))4594 return std::optional<int64_t>();4595 4596 int64_t FD2Offset = 0;4597 if (!getFieldOffsetInBits(CGF, FD2OuterRec, FD2, FD2Offset))4598 return std::optional<int64_t>();4599 4600 return std::make_optional<int64_t>(FD1Offset - FD2Offset);4601}4602 4603/// EmitCountedByBoundsChecking - If the array being accessed has a "counted_by"4604/// attribute, generate bounds checking code. The "count" field is at the top4605/// level of the struct or in an anonymous struct, that's also at the top level.4606/// Future expansions may allow the "count" to reside at any place in the4607/// struct, but the value of "counted_by" will be a "simple" path to the count,4608/// i.e. "a.b.count", so we shouldn't need the full force of EmitLValue or4609/// similar to emit the correct GEP.4610void CodeGenFunction::EmitCountedByBoundsChecking(4611 const Expr *E, llvm::Value *Idx, Address Addr, QualType IdxTy,4612 QualType ArrayTy, bool Accessed, bool FlexibleArray) {4613 const auto *ME = dyn_cast<MemberExpr>(E->IgnoreImpCasts());4614 if (!ME || !ME->getMemberDecl()->getType()->isCountAttributedType())4615 return;4616 4617 const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =4618 getLangOpts().getStrictFlexArraysLevel();4619 if (FlexibleArray &&4620 !ME->isFlexibleArrayMemberLike(getContext(), StrictFlexArraysLevel))4621 return;4622 4623 const FieldDecl *FD = cast<FieldDecl>(ME->getMemberDecl());4624 const FieldDecl *CountFD = FD->findCountedByField();4625 if (!CountFD)4626 return;4627 4628 if (std::optional<int64_t> Diff =4629 getOffsetDifferenceInBits(*this, CountFD, FD)) {4630 if (!Addr.isValid()) {4631 // An invalid Address indicates we're checking a pointer array access.4632 // Emit the checked L-Value here.4633 LValue LV = EmitCheckedLValue(E, TCK_MemberAccess);4634 Addr = LV.getAddress();4635 }4636 4637 // FIXME: The 'static_cast' is necessary, otherwise the result turns into a4638 // uint64_t, which messes things up if we have a negative offset difference.4639 Diff = *Diff / static_cast<int64_t>(CGM.getContext().getCharWidth());4640 4641 // Create a GEP with the byte offset between the counted object and the4642 // count and use that to load the count value.4643 Addr = Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Int8PtrTy, Int8Ty);4644 4645 llvm::Type *CountTy = ConvertType(CountFD->getType());4646 llvm::Value *Res =4647 Builder.CreateInBoundsGEP(Int8Ty, Addr.emitRawPointer(*this),4648 Builder.getInt32(*Diff), ".counted_by.gep");4649 Res = Builder.CreateAlignedLoad(CountTy, Res, getIntAlign(),4650 ".counted_by.load");4651 4652 // Now emit the bounds checking.4653 EmitBoundsCheckImpl(E, Res, Idx, IdxTy, ArrayTy, Accessed);4654 }4655}4656 4657LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,4658 bool Accessed) {4659 // The index must always be an integer, which is not an aggregate. Emit it4660 // in lexical order (this complexity is, sadly, required by C++17).4661 llvm::Value *IdxPre =4662 (E->getLHS() == E->getIdx()) ? EmitScalarExpr(E->getIdx()) : nullptr;4663 bool SignedIndices = false;4664 auto EmitIdxAfterBase = [&, IdxPre](bool Promote) -> llvm::Value * {4665 auto *Idx = IdxPre;4666 if (E->getLHS() != E->getIdx()) {4667 assert(E->getRHS() == E->getIdx() && "index was neither LHS nor RHS");4668 Idx = EmitScalarExpr(E->getIdx());4669 }4670 4671 QualType IdxTy = E->getIdx()->getType();4672 bool IdxSigned = IdxTy->isSignedIntegerOrEnumerationType();4673 SignedIndices |= IdxSigned;4674 4675 if (SanOpts.has(SanitizerKind::ArrayBounds))4676 EmitBoundsCheck(E, E->getBase(), Idx, IdxTy, Accessed);4677 4678 // Extend or truncate the index type to 32 or 64-bits.4679 if (Promote && Idx->getType() != IntPtrTy)4680 Idx = Builder.CreateIntCast(Idx, IntPtrTy, IdxSigned, "idxprom");4681 4682 return Idx;4683 };4684 IdxPre = nullptr;4685 4686 // If the base is a vector type, then we are forming a vector element lvalue4687 // with this subscript.4688 if (E->getBase()->getType()->isSubscriptableVectorType() &&4689 !isa<ExtVectorElementExpr>(E->getBase())) {4690 // Emit the vector as an lvalue to get its address.4691 LValue LHS = EmitLValue(E->getBase());4692 auto *Idx = EmitIdxAfterBase(/*Promote*/false);4693 assert(LHS.isSimple() && "Can only subscript lvalue vectors here!");4694 return LValue::MakeVectorElt(LHS.getAddress(), Idx, E->getBase()->getType(),4695 LHS.getBaseInfo(), TBAAAccessInfo());4696 }4697 4698 // The HLSL runtime handles subscript expressions on global resource arrays4699 // and objects with HLSL buffer layouts.4700 if (getLangOpts().HLSL) {4701 std::optional<LValue> LV;4702 if (E->getType()->isHLSLResourceRecord() ||4703 E->getType()->isHLSLResourceRecordArray()) {4704 LV = CGM.getHLSLRuntime().emitResourceArraySubscriptExpr(E, *this);4705 } else if (E->getType().getAddressSpace() == LangAS::hlsl_constant) {4706 LV = CGM.getHLSLRuntime().emitBufferArraySubscriptExpr(E, *this,4707 EmitIdxAfterBase);4708 }4709 if (LV.has_value())4710 return *LV;4711 }4712 4713 // All the other cases basically behave like simple offsetting.4714 4715 // Handle the extvector case we ignored above.4716 if (isa<ExtVectorElementExpr>(E->getBase())) {4717 LValue LV = EmitLValue(E->getBase());4718 auto *Idx = EmitIdxAfterBase(/*Promote*/true);4719 Address Addr = EmitExtVectorElementLValue(LV);4720 4721 QualType EltType = LV.getType()->castAs<VectorType>()->getElementType();4722 Addr = emitArraySubscriptGEP(*this, Addr, Idx, EltType, /*inbounds*/ true,4723 SignedIndices, E->getExprLoc());4724 return MakeAddrLValue(Addr, EltType, LV.getBaseInfo(),4725 CGM.getTBAAInfoForSubobject(LV, EltType));4726 }4727 4728 LValueBaseInfo EltBaseInfo;4729 TBAAAccessInfo EltTBAAInfo;4730 Address Addr = Address::invalid();4731 if (const VariableArrayType *vla =4732 getContext().getAsVariableArrayType(E->getType())) {4733 // The base must be a pointer, which is not an aggregate. Emit4734 // it. It needs to be emitted first in case it's what captures4735 // the VLA bounds.4736 Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);4737 auto *Idx = EmitIdxAfterBase(/*Promote*/true);4738 4739 // The element count here is the total number of non-VLA elements.4740 llvm::Value *numElements = getVLASize(vla).NumElts;4741 4742 // Effectively, the multiply by the VLA size is part of the GEP.4743 // GEP indexes are signed, and scaling an index isn't permitted to4744 // signed-overflow, so we use the same semantics for our explicit4745 // multiply. We suppress this if overflow is not undefined behavior.4746 if (getLangOpts().PointerOverflowDefined) {4747 Idx = Builder.CreateMul(Idx, numElements);4748 } else {4749 Idx = Builder.CreateNSWMul(Idx, numElements);4750 }4751 4752 Addr = emitArraySubscriptGEP(*this, Addr, Idx, vla->getElementType(),4753 !getLangOpts().PointerOverflowDefined,4754 SignedIndices, E->getExprLoc());4755 4756 } else if (const ObjCObjectType *OIT = E->getType()->getAs<ObjCObjectType>()){4757 // Indexing over an interface, as in "NSString *P; P[4];"4758 4759 // Emit the base pointer.4760 Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);4761 auto *Idx = EmitIdxAfterBase(/*Promote*/true);4762 4763 CharUnits InterfaceSize = getContext().getTypeSizeInChars(OIT);4764 llvm::Value *InterfaceSizeVal =4765 llvm::ConstantInt::get(Idx->getType(), InterfaceSize.getQuantity());4766 4767 llvm::Value *ScaledIdx = Builder.CreateMul(Idx, InterfaceSizeVal);4768 4769 // We don't necessarily build correct LLVM struct types for ObjC4770 // interfaces, so we can't rely on GEP to do this scaling4771 // correctly, so we need to cast to i8*. FIXME: is this actually4772 // true? A lot of other things in the fragile ABI would break...4773 llvm::Type *OrigBaseElemTy = Addr.getElementType();4774 4775 // Do the GEP.4776 CharUnits EltAlign =4777 getArrayElementAlign(Addr.getAlignment(), Idx, InterfaceSize);4778 llvm::Value *EltPtr =4779 emitArraySubscriptGEP(*this, Int8Ty, Addr.emitRawPointer(*this),4780 ScaledIdx, false, SignedIndices, E->getExprLoc());4781 Addr = Address(EltPtr, OrigBaseElemTy, EltAlign);4782 } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) {4783 // If this is A[i] where A is an array, the frontend will have decayed the4784 // base to be a ArrayToPointerDecay implicit cast. While correct, it is4785 // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a4786 // "gep x, i" here. Emit one "gep A, 0, i".4787 assert(Array->getType()->isArrayType() &&4788 "Array to pointer decay must have array source type!");4789 LValue ArrayLV;4790 // For simple multidimensional array indexing, set the 'accessed' flag for4791 // better bounds-checking of the base expression.4792 if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Array))4793 ArrayLV = EmitArraySubscriptExpr(ASE, /*Accessed*/ true);4794 else4795 ArrayLV = EmitLValue(Array);4796 auto *Idx = EmitIdxAfterBase(/*Promote*/true);4797 4798 if (SanOpts.has(SanitizerKind::ArrayBounds))4799 EmitCountedByBoundsChecking(Array, Idx, ArrayLV.getAddress(),4800 E->getIdx()->getType(), Array->getType(),4801 Accessed, /*FlexibleArray=*/true);4802 4803 // Propagate the alignment from the array itself to the result.4804 QualType arrayType = Array->getType();4805 Addr = emitArraySubscriptGEP(4806 *this, ArrayLV.getAddress(), {CGM.getSize(CharUnits::Zero()), Idx},4807 E->getType(), !getLangOpts().PointerOverflowDefined, SignedIndices,4808 E->getExprLoc(), &arrayType, E->getBase());4809 EltBaseInfo = ArrayLV.getBaseInfo();4810 if (!CGM.getCodeGenOpts().NewStructPathTBAA) {4811 // Since CodeGenTBAA::getTypeInfoHelper only handles array types for4812 // new struct path TBAA, we must a use a plain access.4813 EltTBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, E->getType());4814 } else if (ArrayLV.getTBAAInfo().isMayAlias()) {4815 EltTBAAInfo = TBAAAccessInfo::getMayAliasInfo();4816 } else if (ArrayLV.getTBAAInfo().isIncomplete()) {4817 // The array element is complete, even if the array is not.4818 EltTBAAInfo = CGM.getTBAAAccessInfo(E->getType());4819 } else {4820 // The TBAA access info from the array (base) lvalue is ordinary. We will4821 // adapt it to create access info for the element.4822 EltTBAAInfo = ArrayLV.getTBAAInfo();4823 4824 // We retain the TBAA struct path (BaseType and Offset members) from the4825 // array. In the TBAA representation, we map any array access to the4826 // element at index 0, as the index is generally a runtime value. This4827 // element has the same offset in the base type as the array itself.4828 // If the array lvalue had no base type, there is no point trying to4829 // generate one, since an array itself is not a valid base type.4830 4831 // We also retain the access type from the base lvalue, but the access4832 // size must be updated to the size of an individual element.4833 EltTBAAInfo.Size =4834 getContext().getTypeSizeInChars(E->getType()).getQuantity();4835 }4836 } else {4837 // The base must be a pointer; emit it with an estimate of its alignment.4838 Address BaseAddr =4839 EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);4840 auto *Idx = EmitIdxAfterBase(/*Promote*/true);4841 QualType ptrType = E->getBase()->getType();4842 Addr = emitArraySubscriptGEP(*this, BaseAddr, Idx, E->getType(),4843 !getLangOpts().PointerOverflowDefined,4844 SignedIndices, E->getExprLoc(), &ptrType,4845 E->getBase());4846 4847 if (SanOpts.has(SanitizerKind::ArrayBounds)) {4848 StructFieldAccess Visitor;4849 const Expr *Base = Visitor.Visit(E->getBase());4850 4851 if (const auto *CE = dyn_cast_if_present<CastExpr>(Base);4852 CE && CE->getCastKind() == CK_LValueToRValue)4853 EmitCountedByBoundsChecking(CE, Idx, Address::invalid(),4854 E->getIdx()->getType(), ptrType, Accessed,4855 /*FlexibleArray=*/false);4856 }4857 }4858 4859 LValue LV = MakeAddrLValue(Addr, E->getType(), EltBaseInfo, EltTBAAInfo);4860 4861 if (getLangOpts().ObjC &&4862 getLangOpts().getGC() != LangOptions::NonGC) {4863 LV.setNonGC(!E->isOBJCGCCandidate(getContext()));4864 setObjCGCLValueClass(getContext(), E, LV);4865 }4866 return LV;4867}4868 4869llvm::Value *CodeGenFunction::EmitMatrixIndexExpr(const Expr *E) {4870 llvm::Value *Idx = EmitScalarExpr(E);4871 if (Idx->getType() == IntPtrTy)4872 return Idx;4873 bool IsSigned = E->getType()->isSignedIntegerOrEnumerationType();4874 return Builder.CreateIntCast(Idx, IntPtrTy, IsSigned);4875}4876 4877LValue CodeGenFunction::EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E) {4878 assert(4879 !E->isIncomplete() &&4880 "incomplete matrix subscript expressions should be rejected during Sema");4881 LValue Base = EmitLValue(E->getBase());4882 4883 // Extend or truncate the index type to 32 or 64-bits if needed.4884 llvm::Value *RowIdx = EmitMatrixIndexExpr(E->getRowIdx());4885 llvm::Value *ColIdx = EmitMatrixIndexExpr(E->getColumnIdx());4886 4887 llvm::Value *NumRows = Builder.getIntN(4888 RowIdx->getType()->getScalarSizeInBits(),4889 E->getBase()->getType()->castAs<ConstantMatrixType>()->getNumRows());4890 llvm::Value *FinalIdx =4891 Builder.CreateAdd(Builder.CreateMul(ColIdx, NumRows), RowIdx);4892 return LValue::MakeMatrixElt(4893 MaybeConvertMatrixAddress(Base.getAddress(), *this), FinalIdx,4894 E->getBase()->getType(), Base.getBaseInfo(), TBAAAccessInfo());4895}4896 4897static Address emitOMPArraySectionBase(CodeGenFunction &CGF, const Expr *Base,4898 LValueBaseInfo &BaseInfo,4899 TBAAAccessInfo &TBAAInfo,4900 QualType BaseTy, QualType ElTy,4901 bool IsLowerBound) {4902 LValue BaseLVal;4903 if (auto *ASE = dyn_cast<ArraySectionExpr>(Base->IgnoreParenImpCasts())) {4904 BaseLVal = CGF.EmitArraySectionExpr(ASE, IsLowerBound);4905 if (BaseTy->isArrayType()) {4906 Address Addr = BaseLVal.getAddress();4907 BaseInfo = BaseLVal.getBaseInfo();4908 4909 // If the array type was an incomplete type, we need to make sure4910 // the decay ends up being the right type.4911 llvm::Type *NewTy = CGF.ConvertType(BaseTy);4912 Addr = Addr.withElementType(NewTy);4913 4914 // Note that VLA pointers are always decayed, so we don't need to do4915 // anything here.4916 if (!BaseTy->isVariableArrayType()) {4917 assert(isa<llvm::ArrayType>(Addr.getElementType()) &&4918 "Expected pointer to array");4919 Addr = CGF.Builder.CreateConstArrayGEP(Addr, 0, "arraydecay");4920 }4921 4922 return Addr.withElementType(CGF.ConvertTypeForMem(ElTy));4923 }4924 LValueBaseInfo TypeBaseInfo;4925 TBAAAccessInfo TypeTBAAInfo;4926 CharUnits Align =4927 CGF.CGM.getNaturalTypeAlignment(ElTy, &TypeBaseInfo, &TypeTBAAInfo);4928 BaseInfo.mergeForCast(TypeBaseInfo);4929 TBAAInfo = CGF.CGM.mergeTBAAInfoForCast(TBAAInfo, TypeTBAAInfo);4930 return Address(CGF.Builder.CreateLoad(BaseLVal.getAddress()),4931 CGF.ConvertTypeForMem(ElTy), Align);4932 }4933 return CGF.EmitPointerWithAlignment(Base, &BaseInfo, &TBAAInfo);4934}4935 4936LValue CodeGenFunction::EmitArraySectionExpr(const ArraySectionExpr *E,4937 bool IsLowerBound) {4938 4939 assert(!E->isOpenACCArraySection() &&4940 "OpenACC Array section codegen not implemented");4941 4942 QualType BaseTy = ArraySectionExpr::getBaseOriginalType(E->getBase());4943 QualType ResultExprTy;4944 if (auto *AT = getContext().getAsArrayType(BaseTy))4945 ResultExprTy = AT->getElementType();4946 else4947 ResultExprTy = BaseTy->getPointeeType();4948 llvm::Value *Idx = nullptr;4949 if (IsLowerBound || E->getColonLocFirst().isInvalid()) {4950 // Requesting lower bound or upper bound, but without provided length and4951 // without ':' symbol for the default length -> length = 1.4952 // Idx = LowerBound ?: 0;4953 if (auto *LowerBound = E->getLowerBound()) {4954 Idx = Builder.CreateIntCast(4955 EmitScalarExpr(LowerBound), IntPtrTy,4956 LowerBound->getType()->hasSignedIntegerRepresentation());4957 } else4958 Idx = llvm::ConstantInt::getNullValue(IntPtrTy);4959 } else {4960 // Try to emit length or lower bound as constant. If this is possible, 14961 // is subtracted from constant length or lower bound. Otherwise, emit LLVM4962 // IR (LB + Len) - 1.4963 auto &C = CGM.getContext();4964 auto *Length = E->getLength();4965 llvm::APSInt ConstLength;4966 if (Length) {4967 // Idx = LowerBound + Length - 1;4968 if (std::optional<llvm::APSInt> CL = Length->getIntegerConstantExpr(C)) {4969 ConstLength = CL->zextOrTrunc(PointerWidthInBits);4970 Length = nullptr;4971 }4972 auto *LowerBound = E->getLowerBound();4973 llvm::APSInt ConstLowerBound(PointerWidthInBits, /*isUnsigned=*/false);4974 if (LowerBound) {4975 if (std::optional<llvm::APSInt> LB =4976 LowerBound->getIntegerConstantExpr(C)) {4977 ConstLowerBound = LB->zextOrTrunc(PointerWidthInBits);4978 LowerBound = nullptr;4979 }4980 }4981 if (!Length)4982 --ConstLength;4983 else if (!LowerBound)4984 --ConstLowerBound;4985 4986 if (Length || LowerBound) {4987 auto *LowerBoundVal =4988 LowerBound4989 ? Builder.CreateIntCast(4990 EmitScalarExpr(LowerBound), IntPtrTy,4991 LowerBound->getType()->hasSignedIntegerRepresentation())4992 : llvm::ConstantInt::get(IntPtrTy, ConstLowerBound);4993 auto *LengthVal =4994 Length4995 ? Builder.CreateIntCast(4996 EmitScalarExpr(Length), IntPtrTy,4997 Length->getType()->hasSignedIntegerRepresentation())4998 : llvm::ConstantInt::get(IntPtrTy, ConstLength);4999 Idx = Builder.CreateAdd(LowerBoundVal, LengthVal, "lb_add_len",5000 /*HasNUW=*/false,5001 !getLangOpts().PointerOverflowDefined);5002 if (Length && LowerBound) {5003 Idx = Builder.CreateSub(5004 Idx, llvm::ConstantInt::get(IntPtrTy, /*V=*/1), "idx_sub_1",5005 /*HasNUW=*/false, !getLangOpts().PointerOverflowDefined);5006 }5007 } else5008 Idx = llvm::ConstantInt::get(IntPtrTy, ConstLength + ConstLowerBound);5009 } else {5010 // Idx = ArraySize - 1;5011 QualType ArrayTy = BaseTy->isPointerType()5012 ? E->getBase()->IgnoreParenImpCasts()->getType()5013 : BaseTy;5014 if (auto *VAT = C.getAsVariableArrayType(ArrayTy)) {5015 Length = VAT->getSizeExpr();5016 if (std::optional<llvm::APSInt> L = Length->getIntegerConstantExpr(C)) {5017 ConstLength = *L;5018 Length = nullptr;5019 }5020 } else {5021 auto *CAT = C.getAsConstantArrayType(ArrayTy);5022 assert(CAT && "unexpected type for array initializer");5023 ConstLength = CAT->getSize();5024 }5025 if (Length) {5026 auto *LengthVal = Builder.CreateIntCast(5027 EmitScalarExpr(Length), IntPtrTy,5028 Length->getType()->hasSignedIntegerRepresentation());5029 Idx = Builder.CreateSub(5030 LengthVal, llvm::ConstantInt::get(IntPtrTy, /*V=*/1), "len_sub_1",5031 /*HasNUW=*/false, !getLangOpts().PointerOverflowDefined);5032 } else {5033 ConstLength = ConstLength.zextOrTrunc(PointerWidthInBits);5034 --ConstLength;5035 Idx = llvm::ConstantInt::get(IntPtrTy, ConstLength);5036 }5037 }5038 }5039 assert(Idx);5040 5041 Address EltPtr = Address::invalid();5042 LValueBaseInfo BaseInfo;5043 TBAAAccessInfo TBAAInfo;5044 if (auto *VLA = getContext().getAsVariableArrayType(ResultExprTy)) {5045 // The base must be a pointer, which is not an aggregate. Emit5046 // it. It needs to be emitted first in case it's what captures5047 // the VLA bounds.5048 Address Base =5049 emitOMPArraySectionBase(*this, E->getBase(), BaseInfo, TBAAInfo,5050 BaseTy, VLA->getElementType(), IsLowerBound);5051 // The element count here is the total number of non-VLA elements.5052 llvm::Value *NumElements = getVLASize(VLA).NumElts;5053 5054 // Effectively, the multiply by the VLA size is part of the GEP.5055 // GEP indexes are signed, and scaling an index isn't permitted to5056 // signed-overflow, so we use the same semantics for our explicit5057 // multiply. We suppress this if overflow is not undefined behavior.5058 if (getLangOpts().PointerOverflowDefined)5059 Idx = Builder.CreateMul(Idx, NumElements);5060 else5061 Idx = Builder.CreateNSWMul(Idx, NumElements);5062 EltPtr = emitArraySubscriptGEP(*this, Base, Idx, VLA->getElementType(),5063 !getLangOpts().PointerOverflowDefined,5064 /*signedIndices=*/false, E->getExprLoc());5065 } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) {5066 // If this is A[i] where A is an array, the frontend will have decayed the5067 // base to be a ArrayToPointerDecay implicit cast. While correct, it is5068 // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a5069 // "gep x, i" here. Emit one "gep A, 0, i".5070 assert(Array->getType()->isArrayType() &&5071 "Array to pointer decay must have array source type!");5072 LValue ArrayLV;5073 // For simple multidimensional array indexing, set the 'accessed' flag for5074 // better bounds-checking of the base expression.5075 if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Array))5076 ArrayLV = EmitArraySubscriptExpr(ASE, /*Accessed*/ true);5077 else5078 ArrayLV = EmitLValue(Array);5079 5080 // Propagate the alignment from the array itself to the result.5081 EltPtr = emitArraySubscriptGEP(5082 *this, ArrayLV.getAddress(), {CGM.getSize(CharUnits::Zero()), Idx},5083 ResultExprTy, !getLangOpts().PointerOverflowDefined,5084 /*signedIndices=*/false, E->getExprLoc());5085 BaseInfo = ArrayLV.getBaseInfo();5086 TBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, ResultExprTy);5087 } else {5088 Address Base =5089 emitOMPArraySectionBase(*this, E->getBase(), BaseInfo, TBAAInfo, BaseTy,5090 ResultExprTy, IsLowerBound);5091 EltPtr = emitArraySubscriptGEP(*this, Base, Idx, ResultExprTy,5092 !getLangOpts().PointerOverflowDefined,5093 /*signedIndices=*/false, E->getExprLoc());5094 }5095 5096 return MakeAddrLValue(EltPtr, ResultExprTy, BaseInfo, TBAAInfo);5097}5098 5099LValue CodeGenFunction::5100EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {5101 // Emit the base vector as an l-value.5102 LValue Base;5103 5104 // ExtVectorElementExpr's base can either be a vector or pointer to vector.5105 if (E->isArrow()) {5106 // If it is a pointer to a vector, emit the address and form an lvalue with5107 // it.5108 LValueBaseInfo BaseInfo;5109 TBAAAccessInfo TBAAInfo;5110 Address Ptr = EmitPointerWithAlignment(E->getBase(), &BaseInfo, &TBAAInfo);5111 const auto *PT = E->getBase()->getType()->castAs<PointerType>();5112 Base = MakeAddrLValue(Ptr, PT->getPointeeType(), BaseInfo, TBAAInfo);5113 Base.getQuals().removeObjCGCAttr();5114 } else if (E->getBase()->isGLValue()) {5115 // Otherwise, if the base is an lvalue ( as in the case of foo.x.x),5116 // emit the base as an lvalue.5117 assert(E->getBase()->getType()->isVectorType());5118 Base = EmitLValue(E->getBase());5119 } else {5120 // Otherwise, the base is a normal rvalue (as in (V+V).x), emit it as such.5121 assert(E->getBase()->getType()->isVectorType() &&5122 "Result must be a vector");5123 llvm::Value *Vec = EmitScalarExpr(E->getBase());5124 5125 // Store the vector to memory (because LValue wants an address).5126 Address VecMem = CreateMemTemp(E->getBase()->getType());5127 // need to zero extend an hlsl boolean vector to store it back to memory5128 QualType Ty = E->getBase()->getType();5129 llvm::Type *LTy = convertTypeForLoadStore(Ty, Vec->getType());5130 if (LTy->getScalarSizeInBits() > Vec->getType()->getScalarSizeInBits())5131 Vec = Builder.CreateZExt(Vec, LTy);5132 Builder.CreateStore(Vec, VecMem);5133 Base = MakeAddrLValue(VecMem, Ty, AlignmentSource::Decl);5134 }5135 5136 QualType type =5137 E->getType().withCVRQualifiers(Base.getQuals().getCVRQualifiers());5138 5139 // Encode the element access list into a vector of unsigned indices.5140 SmallVector<uint32_t, 4> Indices;5141 E->getEncodedElementAccess(Indices);5142 5143 if (Base.isSimple()) {5144 llvm::Constant *CV =5145 llvm::ConstantDataVector::get(getLLVMContext(), Indices);5146 return LValue::MakeExtVectorElt(Base.getAddress(), CV, type,5147 Base.getBaseInfo(), TBAAAccessInfo());5148 }5149 assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");5150 5151 llvm::Constant *BaseElts = Base.getExtVectorElts();5152 SmallVector<llvm::Constant *, 4> CElts;5153 5154 for (unsigned Index : Indices)5155 CElts.push_back(BaseElts->getAggregateElement(Index));5156 llvm::Constant *CV = llvm::ConstantVector::get(CElts);5157 return LValue::MakeExtVectorElt(Base.getExtVectorAddress(), CV, type,5158 Base.getBaseInfo(), TBAAAccessInfo());5159}5160 5161bool CodeGenFunction::isUnderlyingBasePointerConstantNull(const Expr *E) {5162 const Expr *UnderlyingBaseExpr = E->IgnoreParens();5163 while (auto *BaseMemberExpr = dyn_cast<MemberExpr>(UnderlyingBaseExpr))5164 UnderlyingBaseExpr = BaseMemberExpr->getBase()->IgnoreParens();5165 return getContext().isSentinelNullExpr(UnderlyingBaseExpr);5166}5167 5168LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {5169 if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(*this, E)) {5170 EmitIgnoredExpr(E->getBase());5171 return EmitDeclRefLValue(DRE);5172 }5173 if (getLangOpts().HLSL &&5174 E->getType().getAddressSpace() == LangAS::hlsl_constant) {5175 // We have an HLSL buffer - emit using HLSL's layout rules.5176 return CGM.getHLSLRuntime().emitBufferMemberExpr(*this, E);5177 }5178 5179 Expr *BaseExpr = E->getBase();5180 // Check whether the underlying base pointer is a constant null.5181 // If so, we do not set inbounds flag for GEP to avoid breaking some5182 // old-style offsetof idioms.5183 bool IsInBounds = !getLangOpts().PointerOverflowDefined &&5184 !isUnderlyingBasePointerConstantNull(BaseExpr);5185 // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a scalar.5186 LValue BaseLV;5187 if (E->isArrow()) {5188 LValueBaseInfo BaseInfo;5189 TBAAAccessInfo TBAAInfo;5190 Address Addr = EmitPointerWithAlignment(BaseExpr, &BaseInfo, &TBAAInfo);5191 QualType PtrTy = BaseExpr->getType()->getPointeeType();5192 SanitizerSet SkippedChecks;5193 bool IsBaseCXXThis = IsWrappedCXXThis(BaseExpr);5194 if (IsBaseCXXThis)5195 SkippedChecks.set(SanitizerKind::Alignment, true);5196 if (IsBaseCXXThis || isa<DeclRefExpr>(BaseExpr))5197 SkippedChecks.set(SanitizerKind::Null, true);5198 EmitTypeCheck(TCK_MemberAccess, E->getExprLoc(), Addr, PtrTy,5199 /*Alignment=*/CharUnits::Zero(), SkippedChecks);5200 BaseLV = MakeAddrLValue(Addr, PtrTy, BaseInfo, TBAAInfo);5201 } else5202 BaseLV = EmitCheckedLValue(BaseExpr, TCK_MemberAccess);5203 5204 NamedDecl *ND = E->getMemberDecl();5205 if (auto *Field = dyn_cast<FieldDecl>(ND)) {5206 LValue LV = EmitLValueForField(BaseLV, Field, IsInBounds);5207 setObjCGCLValueClass(getContext(), E, LV);5208 if (getLangOpts().OpenMP) {5209 // If the member was explicitly marked as nontemporal, mark it as5210 // nontemporal. If the base lvalue is marked as nontemporal, mark access5211 // to children as nontemporal too.5212 if ((IsWrappedCXXThis(BaseExpr) &&5213 CGM.getOpenMPRuntime().isNontemporalDecl(Field)) ||5214 BaseLV.isNontemporal())5215 LV.setNontemporal(/*Value=*/true);5216 }5217 return LV;5218 }5219 5220 if (const auto *FD = dyn_cast<FunctionDecl>(ND))5221 return EmitFunctionDeclLValue(*this, E, FD);5222 5223 llvm_unreachable("Unhandled member declaration!");5224}5225 5226/// Given that we are currently emitting a lambda, emit an l-value for5227/// one of its members.5228///5229LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field,5230 llvm::Value *ThisValue) {5231 bool HasExplicitObjectParameter = false;5232 const auto *MD = dyn_cast_if_present<CXXMethodDecl>(CurCodeDecl);5233 if (MD) {5234 HasExplicitObjectParameter = MD->isExplicitObjectMemberFunction();5235 assert(MD->getParent()->isLambda());5236 assert(MD->getParent() == Field->getParent());5237 }5238 LValue LambdaLV;5239 if (HasExplicitObjectParameter) {5240 const VarDecl *D = cast<CXXMethodDecl>(CurCodeDecl)->getParamDecl(0);5241 auto It = LocalDeclMap.find(D);5242 assert(It != LocalDeclMap.end() && "explicit parameter not loaded?");5243 Address AddrOfExplicitObject = It->getSecond();5244 if (D->getType()->isReferenceType())5245 LambdaLV = EmitLoadOfReferenceLValue(AddrOfExplicitObject, D->getType(),5246 AlignmentSource::Decl);5247 else5248 LambdaLV = MakeAddrLValue(AddrOfExplicitObject,5249 D->getType().getNonReferenceType());5250 5251 // Make sure we have an lvalue to the lambda itself and not a derived class.5252 auto *ThisTy = D->getType().getNonReferenceType()->getAsCXXRecordDecl();5253 auto *LambdaTy = cast<CXXRecordDecl>(Field->getParent());5254 if (ThisTy != LambdaTy) {5255 const CXXCastPath &BasePathArray = getContext().LambdaCastPaths.at(MD);5256 Address Base = GetAddressOfBaseClass(5257 LambdaLV.getAddress(), ThisTy, BasePathArray.begin(),5258 BasePathArray.end(), /*NullCheckValue=*/false, SourceLocation());5259 CanQualType T = getContext().getCanonicalTagType(LambdaTy);5260 LambdaLV = MakeAddrLValue(Base, T);5261 }5262 } else {5263 CanQualType LambdaTagType =5264 getContext().getCanonicalTagType(Field->getParent());5265 LambdaLV = MakeNaturalAlignAddrLValue(ThisValue, LambdaTagType);5266 }5267 return EmitLValueForField(LambdaLV, Field);5268}5269 5270LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field) {5271 return EmitLValueForLambdaField(Field, CXXABIThisValue);5272}5273 5274/// Get the field index in the debug info. The debug info structure/union5275/// will ignore the unnamed bitfields.5276unsigned CodeGenFunction::getDebugInfoFIndex(const RecordDecl *Rec,5277 unsigned FieldIndex) {5278 unsigned I = 0, Skipped = 0;5279 5280 for (auto *F : Rec->getDefinition()->fields()) {5281 if (I == FieldIndex)5282 break;5283 if (F->isUnnamedBitField())5284 Skipped++;5285 I++;5286 }5287 5288 return FieldIndex - Skipped;5289}5290 5291/// Get the address of a zero-sized field within a record. The resulting5292/// address doesn't necessarily have the right type.5293static Address emitAddrOfZeroSizeField(CodeGenFunction &CGF, Address Base,5294 const FieldDecl *Field,5295 bool IsInBounds) {5296 CharUnits Offset = CGF.getContext().toCharUnitsFromBits(5297 CGF.getContext().getFieldOffset(Field));5298 if (Offset.isZero())5299 return Base;5300 Base = Base.withElementType(CGF.Int8Ty);5301 if (!IsInBounds)5302 return CGF.Builder.CreateConstByteGEP(Base, Offset);5303 return CGF.Builder.CreateConstInBoundsByteGEP(Base, Offset);5304}5305 5306/// Drill down to the storage of a field without walking into5307/// reference types.5308///5309/// The resulting address doesn't necessarily have the right type.5310static Address emitAddrOfFieldStorage(CodeGenFunction &CGF, Address base,5311 const FieldDecl *field, bool IsInBounds) {5312 if (isEmptyFieldForLayout(CGF.getContext(), field))5313 return emitAddrOfZeroSizeField(CGF, base, field, IsInBounds);5314 5315 const RecordDecl *rec = field->getParent();5316 5317 unsigned idx =5318 CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field);5319 5320 if (!IsInBounds)5321 return CGF.Builder.CreateConstGEP2_32(base, 0, idx, field->getName());5322 5323 return CGF.Builder.CreateStructGEP(base, idx, field->getName());5324}5325 5326static Address emitPreserveStructAccess(CodeGenFunction &CGF, LValue base,5327 Address addr, const FieldDecl *field) {5328 const RecordDecl *rec = field->getParent();5329 llvm::DIType *DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(5330 base.getType(), rec->getLocation());5331 5332 unsigned idx =5333 CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field);5334 5335 return CGF.Builder.CreatePreserveStructAccessIndex(5336 addr, idx, CGF.getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo);5337}5338 5339static bool hasAnyVptr(const QualType Type, const ASTContext &Context) {5340 const auto *RD = Type.getTypePtr()->getAsCXXRecordDecl();5341 if (!RD)5342 return false;5343 5344 if (RD->isDynamicClass())5345 return true;5346 5347 for (const auto &Base : RD->bases())5348 if (hasAnyVptr(Base.getType(), Context))5349 return true;5350 5351 for (const FieldDecl *Field : RD->fields())5352 if (hasAnyVptr(Field->getType(), Context))5353 return true;5354 5355 return false;5356}5357 5358LValue CodeGenFunction::EmitLValueForField(LValue base, const FieldDecl *field,5359 bool IsInBounds) {5360 LValueBaseInfo BaseInfo = base.getBaseInfo();5361 5362 if (field->isBitField()) {5363 const CGRecordLayout &RL =5364 CGM.getTypes().getCGRecordLayout(field->getParent());5365 const CGBitFieldInfo &Info = RL.getBitFieldInfo(field);5366 const bool UseVolatile = isAAPCS(CGM.getTarget()) &&5367 CGM.getCodeGenOpts().AAPCSBitfieldWidth &&5368 Info.VolatileStorageSize != 0 &&5369 field->getType()5370 .withCVRQualifiers(base.getVRQualifiers())5371 .isVolatileQualified();5372 Address Addr = base.getAddress();5373 unsigned Idx = RL.getLLVMFieldNo(field);5374 const RecordDecl *rec = field->getParent();5375 if (hasBPFPreserveStaticOffset(rec))5376 Addr = wrapWithBPFPreserveStaticOffset(*this, Addr);5377 if (!UseVolatile) {5378 if (!IsInPreservedAIRegion &&5379 (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>())) {5380 if (Idx != 0) {5381 // For structs, we GEP to the field that the record layout suggests.5382 if (!IsInBounds)5383 Addr = Builder.CreateConstGEP2_32(Addr, 0, Idx, field->getName());5384 else5385 Addr = Builder.CreateStructGEP(Addr, Idx, field->getName());5386 }5387 } else {5388 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateRecordType(5389 getContext().getCanonicalTagType(rec), rec->getLocation());5390 Addr = Builder.CreatePreserveStructAccessIndex(5391 Addr, Idx, getDebugInfoFIndex(rec, field->getFieldIndex()),5392 DbgInfo);5393 }5394 }5395 const unsigned SS =5396 UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;5397 // Get the access type.5398 llvm::Type *FieldIntTy = llvm::Type::getIntNTy(getLLVMContext(), SS);5399 Addr = Addr.withElementType(FieldIntTy);5400 if (UseVolatile) {5401 const unsigned VolatileOffset = Info.VolatileStorageOffset.getQuantity();5402 if (VolatileOffset)5403 Addr = Builder.CreateConstInBoundsGEP(Addr, VolatileOffset);5404 }5405 5406 QualType fieldType =5407 field->getType().withCVRQualifiers(base.getVRQualifiers());5408 // TODO: Support TBAA for bit fields.5409 LValueBaseInfo FieldBaseInfo(BaseInfo.getAlignmentSource());5410 return LValue::MakeBitfield(Addr, Info, fieldType, FieldBaseInfo,5411 TBAAAccessInfo());5412 }5413 5414 // Fields of may-alias structures are may-alias themselves.5415 // FIXME: this should get propagated down through anonymous structs5416 // and unions.5417 QualType FieldType = field->getType();5418 const RecordDecl *rec = field->getParent();5419 AlignmentSource BaseAlignSource = BaseInfo.getAlignmentSource();5420 LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(BaseAlignSource));5421 TBAAAccessInfo FieldTBAAInfo;5422 if (base.getTBAAInfo().isMayAlias() ||5423 rec->hasAttr<MayAliasAttr>() || FieldType->isVectorType()) {5424 FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();5425 } else if (rec->isUnion()) {5426 // TODO: Support TBAA for unions.5427 FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();5428 } else {5429 // If no base type been assigned for the base access, then try to generate5430 // one for this base lvalue.5431 FieldTBAAInfo = base.getTBAAInfo();5432 if (!FieldTBAAInfo.BaseType) {5433 FieldTBAAInfo.BaseType = CGM.getTBAABaseTypeInfo(base.getType());5434 assert(!FieldTBAAInfo.Offset &&5435 "Nonzero offset for an access with no base type!");5436 }5437 5438 // Adjust offset to be relative to the base type.5439 const ASTRecordLayout &Layout =5440 getContext().getASTRecordLayout(field->getParent());5441 unsigned CharWidth = getContext().getCharWidth();5442 if (FieldTBAAInfo.BaseType)5443 FieldTBAAInfo.Offset +=5444 Layout.getFieldOffset(field->getFieldIndex()) / CharWidth;5445 5446 // Update the final access type and size.5447 FieldTBAAInfo.AccessType = CGM.getTBAATypeInfo(FieldType);5448 FieldTBAAInfo.Size =5449 getContext().getTypeSizeInChars(FieldType).getQuantity();5450 }5451 5452 Address addr = base.getAddress();5453 if (hasBPFPreserveStaticOffset(rec))5454 addr = wrapWithBPFPreserveStaticOffset(*this, addr);5455 if (auto *ClassDef = dyn_cast<CXXRecordDecl>(rec)) {5456 if (CGM.getCodeGenOpts().StrictVTablePointers &&5457 ClassDef->isDynamicClass()) {5458 // Getting to any field of dynamic object requires stripping dynamic5459 // information provided by invariant.group. This is because accessing5460 // fields may leak the real address of dynamic object, which could result5461 // in miscompilation when leaked pointer would be compared.5462 auto *stripped =5463 Builder.CreateStripInvariantGroup(addr.emitRawPointer(*this));5464 addr = Address(stripped, addr.getElementType(), addr.getAlignment());5465 }5466 }5467 5468 unsigned RecordCVR = base.getVRQualifiers();5469 if (rec->isUnion()) {5470 // For unions, there is no pointer adjustment.5471 if (CGM.getCodeGenOpts().StrictVTablePointers &&5472 hasAnyVptr(FieldType, getContext()))5473 // Because unions can easily skip invariant.barriers, we need to add5474 // a barrier every time CXXRecord field with vptr is referenced.5475 addr = Builder.CreateLaunderInvariantGroup(addr);5476 5477 if (IsInPreservedAIRegion ||5478 (getDebugInfo() && rec->hasAttr<BPFPreserveAccessIndexAttr>())) {5479 // Remember the original union field index5480 llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(base.getType(),5481 rec->getLocation());5482 addr =5483 Address(Builder.CreatePreserveUnionAccessIndex(5484 addr.emitRawPointer(*this),5485 getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo),5486 addr.getElementType(), addr.getAlignment());5487 }5488 5489 if (FieldType->isReferenceType())5490 addr = addr.withElementType(CGM.getTypes().ConvertTypeForMem(FieldType));5491 } else {5492 if (!IsInPreservedAIRegion &&5493 (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>()))5494 // For structs, we GEP to the field that the record layout suggests.5495 addr = emitAddrOfFieldStorage(*this, addr, field, IsInBounds);5496 else5497 // Remember the original struct field index5498 addr = emitPreserveStructAccess(*this, base, addr, field);5499 }5500 5501 // If this is a reference field, load the reference right now.5502 if (FieldType->isReferenceType()) {5503 LValue RefLVal =5504 MakeAddrLValue(addr, FieldType, FieldBaseInfo, FieldTBAAInfo);5505 if (RecordCVR & Qualifiers::Volatile)5506 RefLVal.getQuals().addVolatile();5507 addr = EmitLoadOfReference(RefLVal, &FieldBaseInfo, &FieldTBAAInfo);5508 5509 // Qualifiers on the struct don't apply to the referencee.5510 RecordCVR = 0;5511 FieldType = FieldType->getPointeeType();5512 }5513 5514 // Make sure that the address is pointing to the right type. This is critical5515 // for both unions and structs.5516 addr = addr.withElementType(CGM.getTypes().ConvertTypeForMem(FieldType));5517 5518 if (field->hasAttr<AnnotateAttr>())5519 addr = EmitFieldAnnotations(field, addr);5520 5521 LValue LV = MakeAddrLValue(addr, FieldType, FieldBaseInfo, FieldTBAAInfo);5522 LV.getQuals().addCVRQualifiers(RecordCVR);5523 5524 // __weak attribute on a field is ignored.5525 if (LV.getQuals().getObjCGCAttr() == Qualifiers::Weak)5526 LV.getQuals().removeObjCGCAttr();5527 5528 return LV;5529}5530 5531LValue5532CodeGenFunction::EmitLValueForFieldInitialization(LValue Base,5533 const FieldDecl *Field) {5534 QualType FieldType = Field->getType();5535 5536 if (!FieldType->isReferenceType())5537 return EmitLValueForField(Base, Field);5538 5539 Address V = emitAddrOfFieldStorage(5540 *this, Base.getAddress(), Field,5541 /*IsInBounds=*/!getLangOpts().PointerOverflowDefined);5542 5543 // Make sure that the address is pointing to the right type.5544 llvm::Type *llvmType = ConvertTypeForMem(FieldType);5545 V = V.withElementType(llvmType);5546 5547 // TODO: Generate TBAA information that describes this access as a structure5548 // member access and not just an access to an object of the field's type. This5549 // should be similar to what we do in EmitLValueForField().5550 LValueBaseInfo BaseInfo = Base.getBaseInfo();5551 AlignmentSource FieldAlignSource = BaseInfo.getAlignmentSource();5552 LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(FieldAlignSource));5553 return MakeAddrLValue(V, FieldType, FieldBaseInfo,5554 CGM.getTBAAInfoForSubobject(Base, FieldType));5555}5556 5557LValue CodeGenFunction::EmitCompoundLiteralLValue(const CompoundLiteralExpr *E){5558 if (E->isFileScope()) {5559 ConstantAddress GlobalPtr = CGM.GetAddrOfConstantCompoundLiteral(E);5560 return MakeAddrLValue(GlobalPtr, E->getType(), AlignmentSource::Decl);5561 }5562 if (E->getType()->isVariablyModifiedType())5563 // make sure to emit the VLA size.5564 EmitVariablyModifiedType(E->getType());5565 5566 Address DeclPtr = CreateMemTemp(E->getType(), ".compoundliteral");5567 const Expr *InitExpr = E->getInitializer();5568 LValue Result = MakeAddrLValue(DeclPtr, E->getType(), AlignmentSource::Decl);5569 5570 EmitAnyExprToMem(InitExpr, DeclPtr, E->getType().getQualifiers(),5571 /*Init*/ true);5572 5573 // Block-scope compound literals are destroyed at the end of the enclosing5574 // scope in C.5575 if (!getLangOpts().CPlusPlus)5576 if (QualType::DestructionKind DtorKind = E->getType().isDestructedType())5577 pushLifetimeExtendedDestroy(getCleanupKind(DtorKind), DeclPtr,5578 E->getType(), getDestroyer(DtorKind),5579 DtorKind & EHCleanup);5580 5581 return Result;5582}5583 5584LValue CodeGenFunction::EmitInitListLValue(const InitListExpr *E) {5585 if (!E->isGLValue())5586 // Initializing an aggregate temporary in C++11: T{...}.5587 return EmitAggExprToLValue(E);5588 5589 // An lvalue initializer list must be initializing a reference.5590 assert(E->isTransparent() && "non-transparent glvalue init list");5591 return EmitLValue(E->getInit(0));5592}5593 5594/// Emit the operand of a glvalue conditional operator. This is either a glvalue5595/// or a (possibly-parenthesized) throw-expression. If this is a throw, no5596/// LValue is returned and the current block has been terminated.5597static std::optional<LValue> EmitLValueOrThrowExpression(CodeGenFunction &CGF,5598 const Expr *Operand) {5599 if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Operand->IgnoreParens())) {5600 CGF.EmitCXXThrowExpr(ThrowExpr, /*KeepInsertionPoint*/false);5601 return std::nullopt;5602 }5603 5604 return CGF.EmitLValue(Operand);5605}5606 5607namespace {5608// Handle the case where the condition is a constant evaluatable simple integer,5609// which means we don't have to separately handle the true/false blocks.5610std::optional<LValue> HandleConditionalOperatorLValueSimpleCase(5611 CodeGenFunction &CGF, const AbstractConditionalOperator *E) {5612 const Expr *condExpr = E->getCond();5613 bool CondExprBool;5614 if (CGF.ConstantFoldsToSimpleInteger(condExpr, CondExprBool)) {5615 const Expr *Live = E->getTrueExpr(), *Dead = E->getFalseExpr();5616 if (!CondExprBool)5617 std::swap(Live, Dead);5618 5619 if (!CGF.ContainsLabel(Dead)) {5620 // If the true case is live, we need to track its region.5621 if (CondExprBool)5622 CGF.incrementProfileCounter(E);5623 CGF.markStmtMaybeUsed(Dead);5624 // If a throw expression we emit it and return an undefined lvalue5625 // because it can't be used.5626 if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Live->IgnoreParens())) {5627 CGF.EmitCXXThrowExpr(ThrowExpr);5628 llvm::Type *ElemTy = CGF.ConvertType(Dead->getType());5629 llvm::Type *Ty = CGF.DefaultPtrTy;5630 return CGF.MakeAddrLValue(5631 Address(llvm::UndefValue::get(Ty), ElemTy, CharUnits::One()),5632 Dead->getType());5633 }5634 return CGF.EmitLValue(Live);5635 }5636 }5637 return std::nullopt;5638}5639struct ConditionalInfo {5640 llvm::BasicBlock *lhsBlock, *rhsBlock;5641 std::optional<LValue> LHS, RHS;5642};5643 5644// Create and generate the 3 blocks for a conditional operator.5645// Leaves the 'current block' in the continuation basic block.5646template<typename FuncTy>5647ConditionalInfo EmitConditionalBlocks(CodeGenFunction &CGF,5648 const AbstractConditionalOperator *E,5649 const FuncTy &BranchGenFunc) {5650 ConditionalInfo Info{CGF.createBasicBlock("cond.true"),5651 CGF.createBasicBlock("cond.false"), std::nullopt,5652 std::nullopt};5653 llvm::BasicBlock *endBlock = CGF.createBasicBlock("cond.end");5654 5655 CodeGenFunction::ConditionalEvaluation eval(CGF);5656 CGF.EmitBranchOnBoolExpr(E->getCond(), Info.lhsBlock, Info.rhsBlock,5657 CGF.getProfileCount(E));5658 5659 // Any temporaries created here are conditional.5660 CGF.EmitBlock(Info.lhsBlock);5661 CGF.incrementProfileCounter(E);5662 eval.begin(CGF);5663 Info.LHS = BranchGenFunc(CGF, E->getTrueExpr());5664 eval.end(CGF);5665 Info.lhsBlock = CGF.Builder.GetInsertBlock();5666 5667 if (Info.LHS)5668 CGF.Builder.CreateBr(endBlock);5669 5670 // Any temporaries created here are conditional.5671 CGF.EmitBlock(Info.rhsBlock);5672 eval.begin(CGF);5673 Info.RHS = BranchGenFunc(CGF, E->getFalseExpr());5674 eval.end(CGF);5675 Info.rhsBlock = CGF.Builder.GetInsertBlock();5676 CGF.EmitBlock(endBlock);5677 5678 return Info;5679}5680} // namespace5681 5682void CodeGenFunction::EmitIgnoredConditionalOperator(5683 const AbstractConditionalOperator *E) {5684 if (!E->isGLValue()) {5685 // ?: here should be an aggregate.5686 assert(hasAggregateEvaluationKind(E->getType()) &&5687 "Unexpected conditional operator!");5688 return (void)EmitAggExprToLValue(E);5689 }5690 5691 OpaqueValueMapping binding(*this, E);5692 if (HandleConditionalOperatorLValueSimpleCase(*this, E))5693 return;5694 5695 EmitConditionalBlocks(*this, E, [](CodeGenFunction &CGF, const Expr *E) {5696 CGF.EmitIgnoredExpr(E);5697 return LValue{};5698 });5699}5700LValue CodeGenFunction::EmitConditionalOperatorLValue(5701 const AbstractConditionalOperator *expr) {5702 if (!expr->isGLValue()) {5703 // ?: here should be an aggregate.5704 assert(hasAggregateEvaluationKind(expr->getType()) &&5705 "Unexpected conditional operator!");5706 return EmitAggExprToLValue(expr);5707 }5708 5709 OpaqueValueMapping binding(*this, expr);5710 if (std::optional<LValue> Res =5711 HandleConditionalOperatorLValueSimpleCase(*this, expr))5712 return *Res;5713 5714 ConditionalInfo Info = EmitConditionalBlocks(5715 *this, expr, [](CodeGenFunction &CGF, const Expr *E) {5716 return EmitLValueOrThrowExpression(CGF, E);5717 });5718 5719 if ((Info.LHS && !Info.LHS->isSimple()) ||5720 (Info.RHS && !Info.RHS->isSimple()))5721 return EmitUnsupportedLValue(expr, "conditional operator");5722 5723 if (Info.LHS && Info.RHS) {5724 Address lhsAddr = Info.LHS->getAddress();5725 Address rhsAddr = Info.RHS->getAddress();5726 Address result = mergeAddressesInConditionalExpr(5727 lhsAddr, rhsAddr, Info.lhsBlock, Info.rhsBlock,5728 Builder.GetInsertBlock(), expr->getType());5729 AlignmentSource alignSource =5730 std::max(Info.LHS->getBaseInfo().getAlignmentSource(),5731 Info.RHS->getBaseInfo().getAlignmentSource());5732 TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForConditionalOperator(5733 Info.LHS->getTBAAInfo(), Info.RHS->getTBAAInfo());5734 return MakeAddrLValue(result, expr->getType(), LValueBaseInfo(alignSource),5735 TBAAInfo);5736 } else {5737 assert((Info.LHS || Info.RHS) &&5738 "both operands of glvalue conditional are throw-expressions?");5739 return Info.LHS ? *Info.LHS : *Info.RHS;5740 }5741}5742 5743/// EmitCastLValue - Casts are never lvalues unless that cast is to a reference5744/// type. If the cast is to a reference, we can have the usual lvalue result,5745/// otherwise if a cast is needed by the code generator in an lvalue context,5746/// then it must mean that we need the address of an aggregate in order to5747/// access one of its members. This can happen for all the reasons that casts5748/// are permitted with aggregate result, including noop aggregate casts, and5749/// cast from scalar to union.5750LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {5751 auto RestoreCurCast =5752 llvm::make_scope_exit([this, Prev = CurCast] { CurCast = Prev; });5753 CurCast = E;5754 switch (E->getCastKind()) {5755 case CK_ToVoid:5756 case CK_BitCast:5757 case CK_LValueToRValueBitCast:5758 case CK_ArrayToPointerDecay:5759 case CK_FunctionToPointerDecay:5760 case CK_NullToMemberPointer:5761 case CK_NullToPointer:5762 case CK_IntegralToPointer:5763 case CK_PointerToIntegral:5764 case CK_PointerToBoolean:5765 case CK_IntegralCast:5766 case CK_BooleanToSignedIntegral:5767 case CK_IntegralToBoolean:5768 case CK_IntegralToFloating:5769 case CK_FloatingToIntegral:5770 case CK_FloatingToBoolean:5771 case CK_FloatingCast:5772 case CK_FloatingRealToComplex:5773 case CK_FloatingComplexToReal:5774 case CK_FloatingComplexToBoolean:5775 case CK_FloatingComplexCast:5776 case CK_FloatingComplexToIntegralComplex:5777 case CK_IntegralRealToComplex:5778 case CK_IntegralComplexToReal:5779 case CK_IntegralComplexToBoolean:5780 case CK_IntegralComplexCast:5781 case CK_IntegralComplexToFloatingComplex:5782 case CK_DerivedToBaseMemberPointer:5783 case CK_BaseToDerivedMemberPointer:5784 case CK_MemberPointerToBoolean:5785 case CK_ReinterpretMemberPointer:5786 case CK_AnyPointerToBlockPointerCast:5787 case CK_ARCProduceObject:5788 case CK_ARCConsumeObject:5789 case CK_ARCReclaimReturnedObject:5790 case CK_ARCExtendBlockObject:5791 case CK_CopyAndAutoreleaseBlockObject:5792 case CK_IntToOCLSampler:5793 case CK_FloatingToFixedPoint:5794 case CK_FixedPointToFloating:5795 case CK_FixedPointCast:5796 case CK_FixedPointToBoolean:5797 case CK_FixedPointToIntegral:5798 case CK_IntegralToFixedPoint:5799 case CK_MatrixCast:5800 case CK_HLSLVectorTruncation:5801 case CK_HLSLArrayRValue:5802 case CK_HLSLElementwiseCast:5803 case CK_HLSLAggregateSplatCast:5804 return EmitUnsupportedLValue(E, "unexpected cast lvalue");5805 5806 case CK_Dependent:5807 llvm_unreachable("dependent cast kind in IR gen!");5808 5809 case CK_BuiltinFnToFnPtr:5810 llvm_unreachable("builtin functions are handled elsewhere");5811 5812 // These are never l-values; just use the aggregate emission code.5813 case CK_NonAtomicToAtomic:5814 case CK_AtomicToNonAtomic:5815 return EmitAggExprToLValue(E);5816 5817 case CK_Dynamic: {5818 LValue LV = EmitLValue(E->getSubExpr());5819 Address V = LV.getAddress();5820 const auto *DCE = cast<CXXDynamicCastExpr>(E);5821 return MakeNaturalAlignRawAddrLValue(EmitDynamicCast(V, DCE), E->getType());5822 }5823 5824 case CK_ConstructorConversion:5825 case CK_UserDefinedConversion:5826 case CK_CPointerToObjCPointerCast:5827 case CK_BlockPointerToObjCPointerCast:5828 case CK_LValueToRValue:5829 return EmitLValue(E->getSubExpr());5830 5831 case CK_NoOp: {5832 // CK_NoOp can model a qualification conversion, which can remove an array5833 // bound and change the IR type.5834 // FIXME: Once pointee types are removed from IR, remove this.5835 LValue LV = EmitLValue(E->getSubExpr());5836 // Propagate the volatile qualifer to LValue, if exist in E.5837 if (E->changesVolatileQualification())5838 LV.getQuals() = E->getType().getQualifiers();5839 if (LV.isSimple()) {5840 Address V = LV.getAddress();5841 if (V.isValid()) {5842 llvm::Type *T = ConvertTypeForMem(E->getType());5843 if (V.getElementType() != T)5844 LV.setAddress(V.withElementType(T));5845 }5846 }5847 return LV;5848 }5849 5850 case CK_UncheckedDerivedToBase:5851 case CK_DerivedToBase: {5852 auto *DerivedClassDecl = E->getSubExpr()->getType()->castAsCXXRecordDecl();5853 LValue LV = EmitLValue(E->getSubExpr());5854 Address This = LV.getAddress();5855 5856 // Perform the derived-to-base conversion5857 Address Base = GetAddressOfBaseClass(5858 This, DerivedClassDecl, E->path_begin(), E->path_end(),5859 /*NullCheckValue=*/false, E->getExprLoc());5860 5861 // TODO: Support accesses to members of base classes in TBAA. For now, we5862 // conservatively pretend that the complete object is of the base class5863 // type.5864 return MakeAddrLValue(Base, E->getType(), LV.getBaseInfo(),5865 CGM.getTBAAInfoForSubobject(LV, E->getType()));5866 }5867 case CK_ToUnion:5868 return EmitAggExprToLValue(E);5869 case CK_BaseToDerived: {5870 auto *DerivedClassDecl = E->getType()->castAsCXXRecordDecl();5871 LValue LV = EmitLValue(E->getSubExpr());5872 5873 // Perform the base-to-derived conversion5874 Address Derived = GetAddressOfDerivedClass(5875 LV.getAddress(), DerivedClassDecl, E->path_begin(), E->path_end(),5876 /*NullCheckValue=*/false);5877 5878 // C++11 [expr.static.cast]p2: Behavior is undefined if a downcast is5879 // performed and the object is not of the derived type.5880 if (sanitizePerformTypeCheck())5881 EmitTypeCheck(TCK_DowncastReference, E->getExprLoc(), Derived,5882 E->getType());5883 5884 if (SanOpts.has(SanitizerKind::CFIDerivedCast))5885 EmitVTablePtrCheckForCast(E->getType(), Derived,5886 /*MayBeNull=*/false, CFITCK_DerivedCast,5887 E->getBeginLoc());5888 5889 return MakeAddrLValue(Derived, E->getType(), LV.getBaseInfo(),5890 CGM.getTBAAInfoForSubobject(LV, E->getType()));5891 }5892 case CK_LValueBitCast: {5893 // This must be a reinterpret_cast (or c-style equivalent).5894 const auto *CE = cast<ExplicitCastExpr>(E);5895 5896 CGM.EmitExplicitCastExprType(CE, this);5897 LValue LV = EmitLValue(E->getSubExpr());5898 Address V = LV.getAddress().withElementType(5899 ConvertTypeForMem(CE->getTypeAsWritten()->getPointeeType()));5900 5901 if (SanOpts.has(SanitizerKind::CFIUnrelatedCast))5902 EmitVTablePtrCheckForCast(E->getType(), V,5903 /*MayBeNull=*/false, CFITCK_UnrelatedCast,5904 E->getBeginLoc());5905 5906 return MakeAddrLValue(V, E->getType(), LV.getBaseInfo(),5907 CGM.getTBAAInfoForSubobject(LV, E->getType()));5908 }5909 case CK_AddressSpaceConversion: {5910 LValue LV = EmitLValue(E->getSubExpr());5911 QualType DestTy = getContext().getPointerType(E->getType());5912 llvm::Value *V = getTargetHooks().performAddrSpaceCast(5913 *this, LV.getPointer(*this),5914 E->getSubExpr()->getType().getAddressSpace(), ConvertType(DestTy));5915 return MakeAddrLValue(Address(V, ConvertTypeForMem(E->getType()),5916 LV.getAddress().getAlignment()),5917 E->getType(), LV.getBaseInfo(), LV.getTBAAInfo());5918 }5919 case CK_ObjCObjectLValueCast: {5920 LValue LV = EmitLValue(E->getSubExpr());5921 Address V = LV.getAddress().withElementType(ConvertType(E->getType()));5922 return MakeAddrLValue(V, E->getType(), LV.getBaseInfo(),5923 CGM.getTBAAInfoForSubobject(LV, E->getType()));5924 }5925 case CK_ZeroToOCLOpaqueType:5926 llvm_unreachable("NULL to OpenCL opaque type lvalue cast is not valid");5927 5928 case CK_VectorSplat: {5929 // LValue results of vector splats are only supported in HLSL.5930 if (!getLangOpts().HLSL)5931 return EmitUnsupportedLValue(E, "unexpected cast lvalue");5932 return EmitLValue(E->getSubExpr());5933 }5934 }5935 5936 llvm_unreachable("Unhandled lvalue cast kind?");5937}5938 5939LValue CodeGenFunction::EmitOpaqueValueLValue(const OpaqueValueExpr *e) {5940 assert(OpaqueValueMappingData::shouldBindAsLValue(e));5941 return getOrCreateOpaqueLValueMapping(e);5942}5943 5944std::pair<LValue, LValue>5945CodeGenFunction::EmitHLSLOutArgLValues(const HLSLOutArgExpr *E, QualType Ty) {5946 // Emitting the casted temporary through an opaque value.5947 LValue BaseLV = EmitLValue(E->getArgLValue());5948 OpaqueValueMappingData::bind(*this, E->getOpaqueArgLValue(), BaseLV);5949 5950 QualType ExprTy = E->getType();5951 Address OutTemp = CreateIRTemp(ExprTy);5952 LValue TempLV = MakeAddrLValue(OutTemp, ExprTy);5953 5954 if (E->isInOut())5955 EmitInitializationToLValue(E->getCastedTemporary()->getSourceExpr(),5956 TempLV);5957 5958 OpaqueValueMappingData::bind(*this, E->getCastedTemporary(), TempLV);5959 return std::make_pair(BaseLV, TempLV);5960}5961 5962LValue CodeGenFunction::EmitHLSLOutArgExpr(const HLSLOutArgExpr *E,5963 CallArgList &Args, QualType Ty) {5964 5965 auto [BaseLV, TempLV] = EmitHLSLOutArgLValues(E, Ty);5966 5967 llvm::Value *Addr = TempLV.getAddress().getBasePointer();5968 llvm::Type *ElTy = ConvertTypeForMem(TempLV.getType());5969 5970 EmitLifetimeStart(Addr);5971 5972 Address TmpAddr(Addr, ElTy, TempLV.getAlignment());5973 Args.addWriteback(BaseLV, TmpAddr, nullptr, E->getWritebackCast());5974 Args.add(RValue::get(TmpAddr, *this), Ty);5975 return TempLV;5976}5977 5978LValue5979CodeGenFunction::getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e) {5980 assert(OpaqueValueMapping::shouldBindAsLValue(e));5981 5982 llvm::DenseMap<const OpaqueValueExpr*,LValue>::iterator5983 it = OpaqueLValues.find(e);5984 5985 if (it != OpaqueLValues.end())5986 return it->second;5987 5988 assert(e->isUnique() && "LValue for a nonunique OVE hasn't been emitted");5989 return EmitLValue(e->getSourceExpr());5990}5991 5992RValue5993CodeGenFunction::getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e) {5994 assert(!OpaqueValueMapping::shouldBindAsLValue(e));5995 5996 llvm::DenseMap<const OpaqueValueExpr*,RValue>::iterator5997 it = OpaqueRValues.find(e);5998 5999 if (it != OpaqueRValues.end())6000 return it->second;6001 6002 assert(e->isUnique() && "RValue for a nonunique OVE hasn't been emitted");6003 return EmitAnyExpr(e->getSourceExpr());6004}6005 6006bool CodeGenFunction::isOpaqueValueEmitted(const OpaqueValueExpr *E) {6007 if (OpaqueValueMapping::shouldBindAsLValue(E))6008 return OpaqueLValues.contains(E);6009 return OpaqueRValues.contains(E);6010}6011 6012RValue CodeGenFunction::EmitRValueForField(LValue LV,6013 const FieldDecl *FD,6014 SourceLocation Loc) {6015 QualType FT = FD->getType();6016 LValue FieldLV = EmitLValueForField(LV, FD);6017 switch (getEvaluationKind(FT)) {6018 case TEK_Complex:6019 return RValue::getComplex(EmitLoadOfComplex(FieldLV, Loc));6020 case TEK_Aggregate:6021 return FieldLV.asAggregateRValue();6022 case TEK_Scalar:6023 // This routine is used to load fields one-by-one to perform a copy, so6024 // don't load reference fields.6025 if (FD->getType()->isReferenceType())6026 return RValue::get(FieldLV.getPointer(*this));6027 // Call EmitLoadOfScalar except when the lvalue is a bitfield to emit a6028 // primitive load.6029 if (FieldLV.isBitField())6030 return EmitLoadOfLValue(FieldLV, Loc);6031 return RValue::get(EmitLoadOfScalar(FieldLV, Loc));6032 }6033 llvm_unreachable("bad evaluation kind");6034}6035 6036//===--------------------------------------------------------------------===//6037// Expression Emission6038//===--------------------------------------------------------------------===//6039 6040RValue CodeGenFunction::EmitCallExpr(const CallExpr *E,6041 ReturnValueSlot ReturnValue,6042 llvm::CallBase **CallOrInvoke) {6043 llvm::CallBase *CallOrInvokeStorage;6044 if (!CallOrInvoke) {6045 CallOrInvoke = &CallOrInvokeStorage;6046 }6047 6048 auto AddCoroElideSafeOnExit = llvm::make_scope_exit([&] {6049 if (E->isCoroElideSafe()) {6050 auto *I = *CallOrInvoke;6051 if (I)6052 I->addFnAttr(llvm::Attribute::CoroElideSafe);6053 }6054 });6055 6056 // Builtins never have block type.6057 if (E->getCallee()->getType()->isBlockPointerType())6058 return EmitBlockCallExpr(E, ReturnValue, CallOrInvoke);6059 6060 if (const auto *CE = dyn_cast<CXXMemberCallExpr>(E))6061 return EmitCXXMemberCallExpr(CE, ReturnValue, CallOrInvoke);6062 6063 if (const auto *CE = dyn_cast<CUDAKernelCallExpr>(E))6064 return EmitCUDAKernelCallExpr(CE, ReturnValue, CallOrInvoke);6065 6066 // A CXXOperatorCallExpr is created even for explicit object methods, but6067 // these should be treated like static function call.6068 if (const auto *CE = dyn_cast<CXXOperatorCallExpr>(E))6069 if (const auto *MD =6070 dyn_cast_if_present<CXXMethodDecl>(CE->getCalleeDecl());6071 MD && MD->isImplicitObjectMemberFunction())6072 return EmitCXXOperatorMemberCallExpr(CE, MD, ReturnValue, CallOrInvoke);6073 6074 CGCallee callee = EmitCallee(E->getCallee());6075 6076 if (callee.isBuiltin()) {6077 return EmitBuiltinExpr(callee.getBuiltinDecl(), callee.getBuiltinID(),6078 E, ReturnValue);6079 }6080 6081 if (callee.isPseudoDestructor()) {6082 return EmitCXXPseudoDestructorExpr(callee.getPseudoDestructorExpr());6083 }6084 6085 return EmitCall(E->getCallee()->getType(), callee, E, ReturnValue,6086 /*Chain=*/nullptr, CallOrInvoke);6087}6088 6089/// Emit a CallExpr without considering whether it might be a subclass.6090RValue CodeGenFunction::EmitSimpleCallExpr(const CallExpr *E,6091 ReturnValueSlot ReturnValue,6092 llvm::CallBase **CallOrInvoke) {6093 CGCallee Callee = EmitCallee(E->getCallee());6094 return EmitCall(E->getCallee()->getType(), Callee, E, ReturnValue,6095 /*Chain=*/nullptr, CallOrInvoke);6096}6097 6098// Detect the unusual situation where an inline version is shadowed by a6099// non-inline version. In that case we should pick the external one6100// everywhere. That's GCC behavior too.6101static bool OnlyHasInlineBuiltinDeclaration(const FunctionDecl *FD) {6102 for (const FunctionDecl *PD = FD; PD; PD = PD->getPreviousDecl())6103 if (!PD->isInlineBuiltinDeclaration())6104 return false;6105 return true;6106}6107 6108static CGCallee EmitDirectCallee(CodeGenFunction &CGF, GlobalDecl GD) {6109 const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());6110 6111 if (auto builtinID = FD->getBuiltinID()) {6112 std::string NoBuiltinFD = ("no-builtin-" + FD->getName()).str();6113 std::string NoBuiltins = "no-builtins";6114 6115 StringRef Ident = CGF.CGM.getMangledName(GD);6116 std::string FDInlineName = (Ident + ".inline").str();6117 6118 bool IsPredefinedLibFunction =6119 CGF.getContext().BuiltinInfo.isPredefinedLibFunction(builtinID);6120 bool HasAttributeNoBuiltin =6121 CGF.CurFn->getAttributes().hasFnAttr(NoBuiltinFD) ||6122 CGF.CurFn->getAttributes().hasFnAttr(NoBuiltins);6123 6124 // When directing calling an inline builtin, call it through it's mangled6125 // name to make it clear it's not the actual builtin.6126 if (CGF.CurFn->getName() != FDInlineName &&6127 OnlyHasInlineBuiltinDeclaration(FD)) {6128 llvm::Constant *CalleePtr = CGF.CGM.getRawFunctionPointer(GD);6129 llvm::Function *Fn = llvm::cast<llvm::Function>(CalleePtr);6130 llvm::Module *M = Fn->getParent();6131 llvm::Function *Clone = M->getFunction(FDInlineName);6132 if (!Clone) {6133 Clone = llvm::Function::Create(Fn->getFunctionType(),6134 llvm::GlobalValue::InternalLinkage,6135 Fn->getAddressSpace(), FDInlineName, M);6136 Clone->addFnAttr(llvm::Attribute::AlwaysInline);6137 }6138 return CGCallee::forDirect(Clone, GD);6139 }6140 6141 // Replaceable builtins provide their own implementation of a builtin. If we6142 // are in an inline builtin implementation, avoid trivial infinite6143 // recursion. Honor __attribute__((no_builtin("foo"))) or6144 // __attribute__((no_builtin)) on the current function unless foo is6145 // not a predefined library function which means we must generate the6146 // builtin no matter what.6147 else if (!IsPredefinedLibFunction || !HasAttributeNoBuiltin)6148 return CGCallee::forBuiltin(builtinID, FD);6149 }6150 6151 llvm::Constant *CalleePtr = CGF.CGM.getRawFunctionPointer(GD);6152 if (CGF.CGM.getLangOpts().CUDA && !CGF.CGM.getLangOpts().CUDAIsDevice &&6153 FD->hasAttr<CUDAGlobalAttr>())6154 CalleePtr = CGF.CGM.getCUDARuntime().getKernelStub(6155 cast<llvm::GlobalValue>(CalleePtr->stripPointerCasts()));6156 6157 return CGCallee::forDirect(CalleePtr, GD);6158}6159 6160static GlobalDecl getGlobalDeclForDirectCall(const FunctionDecl *FD) {6161 if (DeviceKernelAttr::isOpenCLSpelling(FD->getAttr<DeviceKernelAttr>()))6162 return GlobalDecl(FD, KernelReferenceKind::Stub);6163 return GlobalDecl(FD);6164}6165 6166CGCallee CodeGenFunction::EmitCallee(const Expr *E) {6167 E = E->IgnoreParens();6168 6169 // Look through function-to-pointer decay.6170 if (auto ICE = dyn_cast<ImplicitCastExpr>(E)) {6171 if (ICE->getCastKind() == CK_FunctionToPointerDecay ||6172 ICE->getCastKind() == CK_BuiltinFnToFnPtr) {6173 return EmitCallee(ICE->getSubExpr());6174 }6175 6176 // Try to remember the original __ptrauth qualifier for loads of6177 // function pointers.6178 if (ICE->getCastKind() == CK_LValueToRValue) {6179 const Expr *SubExpr = ICE->getSubExpr();6180 if (const auto *PtrType = SubExpr->getType()->getAs<PointerType>()) {6181 std::pair<llvm::Value *, CGPointerAuthInfo> Result =6182 EmitOrigPointerRValue(E);6183 6184 QualType FunctionType = PtrType->getPointeeType();6185 assert(FunctionType->isFunctionType());6186 6187 GlobalDecl GD;6188 if (const auto *VD =6189 dyn_cast_or_null<VarDecl>(E->getReferencedDeclOfCallee())) {6190 GD = GlobalDecl(VD);6191 }6192 CGCalleeInfo CalleeInfo(FunctionType->getAs<FunctionProtoType>(), GD);6193 CGCallee Callee(CalleeInfo, Result.first, Result.second);6194 return Callee;6195 }6196 }6197 6198 // Resolve direct calls.6199 } else if (auto DRE = dyn_cast<DeclRefExpr>(E)) {6200 if (auto FD = dyn_cast<FunctionDecl>(DRE->getDecl())) {6201 return EmitDirectCallee(*this, getGlobalDeclForDirectCall(FD));6202 }6203 } else if (auto ME = dyn_cast<MemberExpr>(E)) {6204 if (auto FD = dyn_cast<FunctionDecl>(ME->getMemberDecl())) {6205 EmitIgnoredExpr(ME->getBase());6206 return EmitDirectCallee(*this, FD);6207 }6208 6209 // Look through template substitutions.6210 } else if (auto NTTP = dyn_cast<SubstNonTypeTemplateParmExpr>(E)) {6211 return EmitCallee(NTTP->getReplacement());6212 6213 // Treat pseudo-destructor calls differently.6214 } else if (auto PDE = dyn_cast<CXXPseudoDestructorExpr>(E)) {6215 return CGCallee::forPseudoDestructor(PDE);6216 }6217 6218 // Otherwise, we have an indirect reference.6219 llvm::Value *calleePtr;6220 QualType functionType;6221 if (auto ptrType = E->getType()->getAs<PointerType>()) {6222 calleePtr = EmitScalarExpr(E);6223 functionType = ptrType->getPointeeType();6224 } else {6225 functionType = E->getType();6226 calleePtr = EmitLValue(E, KnownNonNull).getPointer(*this);6227 }6228 assert(functionType->isFunctionType());6229 6230 GlobalDecl GD;6231 if (const auto *VD =6232 dyn_cast_or_null<VarDecl>(E->getReferencedDeclOfCallee()))6233 GD = GlobalDecl(VD);6234 6235 CGCalleeInfo calleeInfo(functionType->getAs<FunctionProtoType>(), GD);6236 CGPointerAuthInfo pointerAuth = CGM.getFunctionPointerAuthInfo(functionType);6237 CGCallee callee(calleeInfo, calleePtr, pointerAuth);6238 return callee;6239}6240 6241LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {6242 // Comma expressions just emit their LHS then their RHS as an l-value.6243 if (E->getOpcode() == BO_Comma) {6244 EmitIgnoredExpr(E->getLHS());6245 EnsureInsertPoint();6246 return EmitLValue(E->getRHS());6247 }6248 6249 if (E->getOpcode() == BO_PtrMemD ||6250 E->getOpcode() == BO_PtrMemI)6251 return EmitPointerToDataMemberBinaryExpr(E);6252 6253 assert(E->getOpcode() == BO_Assign && "unexpected binary l-value");6254 6255 // Create a Key Instructions source location atom group that covers both6256 // LHS and RHS expressions. Nested RHS expressions may get subsequently6257 // separately grouped (1 below):6258 //6259 // 1. `a = b = c` -> Two atoms.6260 // 2. `x = new(1)` -> One atom (for both addr store and value store).6261 // 3. Complex and agg assignment -> One atom.6262 ApplyAtomGroup Grp(getDebugInfo());6263 6264 // Note that in all of these cases, __block variables need the RHS6265 // evaluated first just in case the variable gets moved by the RHS.6266 6267 switch (getEvaluationKind(E->getType())) {6268 case TEK_Scalar: {6269 if (PointerAuthQualifier PtrAuth =6270 E->getLHS()->getType().getPointerAuth()) {6271 LValue LV = EmitCheckedLValue(E->getLHS(), TCK_Store);6272 LValue CopiedLV = LV;6273 CopiedLV.getQuals().removePointerAuth();6274 llvm::Value *RV =6275 EmitPointerAuthQualify(PtrAuth, E->getRHS(), CopiedLV.getAddress());6276 EmitNullabilityCheck(CopiedLV, RV, E->getExprLoc());6277 EmitStoreThroughLValue(RValue::get(RV), CopiedLV);6278 return LV;6279 }6280 6281 switch (E->getLHS()->getType().getObjCLifetime()) {6282 case Qualifiers::OCL_Strong:6283 return EmitARCStoreStrong(E, /*ignored*/ false).first;6284 6285 case Qualifiers::OCL_Autoreleasing:6286 return EmitARCStoreAutoreleasing(E).first;6287 6288 // No reason to do any of these differently.6289 case Qualifiers::OCL_None:6290 case Qualifiers::OCL_ExplicitNone:6291 case Qualifiers::OCL_Weak:6292 break;6293 }6294 6295 // TODO: Can we de-duplicate this code with the corresponding code in6296 // CGExprScalar, similar to the way EmitCompoundAssignmentLValue works?6297 RValue RV;6298 llvm::Value *Previous = nullptr;6299 QualType SrcType = E->getRHS()->getType();6300 // Check if LHS is a bitfield, if RHS contains an implicit cast expression6301 // we want to extract that value and potentially (if the bitfield sanitizer6302 // is enabled) use it to check for an implicit conversion.6303 if (E->getLHS()->refersToBitField()) {6304 llvm::Value *RHS =6305 EmitWithOriginalRHSBitfieldAssignment(E, &Previous, &SrcType);6306 RV = RValue::get(RHS);6307 } else6308 RV = EmitAnyExpr(E->getRHS());6309 6310 LValue LV = EmitCheckedLValue(E->getLHS(), TCK_Store);6311 6312 if (RV.isScalar())6313 EmitNullabilityCheck(LV, RV.getScalarVal(), E->getExprLoc());6314 6315 if (LV.isBitField()) {6316 llvm::Value *Result = nullptr;6317 // If bitfield sanitizers are enabled we want to use the result6318 // to check whether a truncation or sign change has occurred.6319 if (SanOpts.has(SanitizerKind::ImplicitBitfieldConversion))6320 EmitStoreThroughBitfieldLValue(RV, LV, &Result);6321 else6322 EmitStoreThroughBitfieldLValue(RV, LV);6323 6324 // If the expression contained an implicit conversion, make sure6325 // to use the value before the scalar conversion.6326 llvm::Value *Src = Previous ? Previous : RV.getScalarVal();6327 QualType DstType = E->getLHS()->getType();6328 EmitBitfieldConversionCheck(Src, SrcType, Result, DstType,6329 LV.getBitFieldInfo(), E->getExprLoc());6330 } else6331 EmitStoreThroughLValue(RV, LV);6332 6333 if (getLangOpts().OpenMP)6334 CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(*this,6335 E->getLHS());6336 return LV;6337 }6338 6339 case TEK_Complex:6340 return EmitComplexAssignmentLValue(E);6341 6342 case TEK_Aggregate:6343 // If the lang opt is HLSL and the LHS is a constant array6344 // then we are performing a copy assignment and call a special6345 // function because EmitAggExprToLValue emits to a temporary LValue6346 if (getLangOpts().HLSL && E->getLHS()->getType()->isConstantArrayType())6347 return EmitHLSLArrayAssignLValue(E);6348 6349 return EmitAggExprToLValue(E);6350 }6351 llvm_unreachable("bad evaluation kind");6352}6353 6354// This function implements trivial copy assignment for HLSL's6355// assignable constant arrays.6356LValue CodeGenFunction::EmitHLSLArrayAssignLValue(const BinaryOperator *E) {6357 // Don't emit an LValue for the RHS because it might not be an LValue6358 LValue LHS = EmitLValue(E->getLHS());6359 // In C the RHS of an assignment operator is an RValue.6360 // EmitAggregateAssign takes anan LValue for the RHS. Instead we can call6361 // EmitInitializationToLValue to emit an RValue into an LValue.6362 EmitInitializationToLValue(E->getRHS(), LHS);6363 return LHS;6364}6365 6366LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E,6367 llvm::CallBase **CallOrInvoke) {6368 RValue RV = EmitCallExpr(E, ReturnValueSlot(), CallOrInvoke);6369 6370 if (!RV.isScalar())6371 return MakeAddrLValue(RV.getAggregateAddress(), E->getType(),6372 AlignmentSource::Decl);6373 6374 assert(E->getCallReturnType(getContext())->isReferenceType() &&6375 "Can't have a scalar return unless the return type is a "6376 "reference type!");6377 6378 return MakeNaturalAlignPointeeAddrLValue(RV.getScalarVal(), E->getType());6379}6380 6381LValue CodeGenFunction::EmitVAArgExprLValue(const VAArgExpr *E) {6382 // FIXME: This shouldn't require another copy.6383 return EmitAggExprToLValue(E);6384}6385 6386LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) {6387 assert(E->getType()->getAsCXXRecordDecl()->hasTrivialDestructor()6388 && "binding l-value to type which needs a temporary");6389 AggValueSlot Slot = CreateAggTemp(E->getType());6390 EmitCXXConstructExpr(E, Slot);6391 return MakeAddrLValue(Slot.getAddress(), E->getType(), AlignmentSource::Decl);6392}6393 6394LValue6395CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) {6396 return MakeNaturalAlignRawAddrLValue(EmitCXXTypeidExpr(E), E->getType());6397}6398 6399Address CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) {6400 return CGM.GetAddrOfMSGuidDecl(E->getGuidDecl())6401 .withElementType(ConvertType(E->getType()));6402}6403 6404LValue CodeGenFunction::EmitCXXUuidofLValue(const CXXUuidofExpr *E) {6405 return MakeAddrLValue(EmitCXXUuidofExpr(E), E->getType(),6406 AlignmentSource::Decl);6407}6408 6409LValue6410CodeGenFunction::EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E) {6411 AggValueSlot Slot = CreateAggTemp(E->getType(), "temp.lvalue");6412 Slot.setExternallyDestructed();6413 EmitAggExpr(E->getSubExpr(), Slot);6414 EmitCXXTemporary(E->getTemporary(), E->getType(), Slot.getAddress());6415 return MakeAddrLValue(Slot.getAddress(), E->getType(), AlignmentSource::Decl);6416}6417 6418LValue CodeGenFunction::EmitObjCMessageExprLValue(const ObjCMessageExpr *E) {6419 RValue RV = EmitObjCMessageExpr(E);6420 6421 if (!RV.isScalar())6422 return MakeAddrLValue(RV.getAggregateAddress(), E->getType(),6423 AlignmentSource::Decl);6424 6425 assert(E->getMethodDecl()->getReturnType()->isReferenceType() &&6426 "Can't have a scalar return unless the return type is a "6427 "reference type!");6428 6429 return MakeNaturalAlignPointeeAddrLValue(RV.getScalarVal(), E->getType());6430}6431 6432LValue CodeGenFunction::EmitObjCSelectorLValue(const ObjCSelectorExpr *E) {6433 Address V =6434 CGM.getObjCRuntime().GetAddrOfSelector(*this, E->getSelector());6435 return MakeAddrLValue(V, E->getType(), AlignmentSource::Decl);6436}6437 6438llvm::Value *CodeGenFunction::EmitIvarOffset(const ObjCInterfaceDecl *Interface,6439 const ObjCIvarDecl *Ivar) {6440 return CGM.getObjCRuntime().EmitIvarOffset(*this, Interface, Ivar);6441}6442 6443llvm::Value *6444CodeGenFunction::EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl *Interface,6445 const ObjCIvarDecl *Ivar) {6446 llvm::Value *OffsetValue = EmitIvarOffset(Interface, Ivar);6447 QualType PointerDiffType = getContext().getPointerDiffType();6448 return Builder.CreateZExtOrTrunc(OffsetValue,6449 getTypes().ConvertType(PointerDiffType));6450}6451 6452LValue CodeGenFunction::EmitLValueForIvar(QualType ObjectTy,6453 llvm::Value *BaseValue,6454 const ObjCIvarDecl *Ivar,6455 unsigned CVRQualifiers) {6456 return CGM.getObjCRuntime().EmitObjCValueForIvar(*this, ObjectTy, BaseValue,6457 Ivar, CVRQualifiers);6458}6459 6460LValue CodeGenFunction::EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E) {6461 // FIXME: A lot of the code below could be shared with EmitMemberExpr.6462 llvm::Value *BaseValue = nullptr;6463 const Expr *BaseExpr = E->getBase();6464 Qualifiers BaseQuals;6465 QualType ObjectTy;6466 if (E->isArrow()) {6467 BaseValue = EmitScalarExpr(BaseExpr);6468 ObjectTy = BaseExpr->getType()->getPointeeType();6469 BaseQuals = ObjectTy.getQualifiers();6470 } else {6471 LValue BaseLV = EmitLValue(BaseExpr);6472 BaseValue = BaseLV.getPointer(*this);6473 ObjectTy = BaseExpr->getType();6474 BaseQuals = ObjectTy.getQualifiers();6475 }6476 6477 LValue LV =6478 EmitLValueForIvar(ObjectTy, BaseValue, E->getDecl(),6479 BaseQuals.getCVRQualifiers());6480 setObjCGCLValueClass(getContext(), E, LV);6481 return LV;6482}6483 6484LValue CodeGenFunction::EmitStmtExprLValue(const StmtExpr *E) {6485 // Can only get l-value for message expression returning aggregate type6486 RValue RV = EmitAnyExprToTemp(E);6487 return MakeAddrLValue(RV.getAggregateAddress(), E->getType(),6488 AlignmentSource::Decl);6489}6490 6491RValue CodeGenFunction::EmitCall(QualType CalleeType,6492 const CGCallee &OrigCallee, const CallExpr *E,6493 ReturnValueSlot ReturnValue,6494 llvm::Value *Chain,6495 llvm::CallBase **CallOrInvoke,6496 CGFunctionInfo const **ResolvedFnInfo) {6497 // Get the actual function type. The callee type will always be a pointer to6498 // function type or a block pointer type.6499 assert(CalleeType->isFunctionPointerType() &&6500 "Call must have function pointer type!");6501 6502 const Decl *TargetDecl =6503 OrigCallee.getAbstractInfo().getCalleeDecl().getDecl();6504 6505 assert((!isa_and_present<FunctionDecl>(TargetDecl) ||6506 !cast<FunctionDecl>(TargetDecl)->isImmediateFunction()) &&6507 "trying to emit a call to an immediate function");6508 6509 CalleeType = getContext().getCanonicalType(CalleeType);6510 6511 auto PointeeType = cast<PointerType>(CalleeType)->getPointeeType();6512 6513 CGCallee Callee = OrigCallee;6514 6515 if (SanOpts.has(SanitizerKind::Function) &&6516 (!TargetDecl || !isa<FunctionDecl>(TargetDecl)) &&6517 !isa<FunctionNoProtoType>(PointeeType)) {6518 if (llvm::Constant *PrefixSig =6519 CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM)) {6520 auto CheckOrdinal = SanitizerKind::SO_Function;6521 auto CheckHandler = SanitizerHandler::FunctionTypeMismatch;6522 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);6523 auto *TypeHash = getUBSanFunctionTypeHash(PointeeType);6524 6525 llvm::Type *PrefixSigType = PrefixSig->getType();6526 llvm::StructType *PrefixStructTy = llvm::StructType::get(6527 CGM.getLLVMContext(), {PrefixSigType, Int32Ty}, /*isPacked=*/true);6528 6529 llvm::Value *CalleePtr = Callee.getFunctionPointer();6530 if (CGM.getCodeGenOpts().PointerAuth.FunctionPointers) {6531 // Use raw pointer since we are using the callee pointer as data here.6532 Address Addr =6533 Address(CalleePtr, CalleePtr->getType(),6534 CharUnits::fromQuantity(6535 CalleePtr->getPointerAlignment(CGM.getDataLayout())),6536 Callee.getPointerAuthInfo(), nullptr);6537 CalleePtr = Addr.emitRawPointer(*this);6538 }6539 6540 // On 32-bit Arm, the low bit of a function pointer indicates whether6541 // it's using the Arm or Thumb instruction set. The actual first6542 // instruction lives at the same address either way, so we must clear6543 // that low bit before using the function address to find the prefix6544 // structure.6545 //6546 // This applies to both Arm and Thumb target triples, because6547 // either one could be used in an interworking context where it6548 // might be passed function pointers of both types.6549 llvm::Value *AlignedCalleePtr;6550 if (CGM.getTriple().isARM() || CGM.getTriple().isThumb()) {6551 llvm::Value *CalleeAddress =6552 Builder.CreatePtrToInt(CalleePtr, IntPtrTy);6553 llvm::Value *Mask = llvm::ConstantInt::get(IntPtrTy, ~1);6554 llvm::Value *AlignedCalleeAddress =6555 Builder.CreateAnd(CalleeAddress, Mask);6556 AlignedCalleePtr =6557 Builder.CreateIntToPtr(AlignedCalleeAddress, CalleePtr->getType());6558 } else {6559 AlignedCalleePtr = CalleePtr;6560 }6561 6562 llvm::Value *CalleePrefixStruct = AlignedCalleePtr;6563 llvm::Value *CalleeSigPtr =6564 Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, -1, 0);6565 llvm::Value *CalleeSig =6566 Builder.CreateAlignedLoad(PrefixSigType, CalleeSigPtr, getIntAlign());6567 llvm::Value *CalleeSigMatch = Builder.CreateICmpEQ(CalleeSig, PrefixSig);6568 6569 llvm::BasicBlock *Cont = createBasicBlock("cont");6570 llvm::BasicBlock *TypeCheck = createBasicBlock("typecheck");6571 Builder.CreateCondBr(CalleeSigMatch, TypeCheck, Cont);6572 6573 EmitBlock(TypeCheck);6574 llvm::Value *CalleeTypeHash = Builder.CreateAlignedLoad(6575 Int32Ty,6576 Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, -1, 1),6577 getPointerAlign());6578 llvm::Value *CalleeTypeHashMatch =6579 Builder.CreateICmpEQ(CalleeTypeHash, TypeHash);6580 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(E->getBeginLoc()),6581 EmitCheckTypeDescriptor(CalleeType)};6582 EmitCheck(std::make_pair(CalleeTypeHashMatch, CheckOrdinal), CheckHandler,6583 StaticData, {CalleePtr});6584 6585 Builder.CreateBr(Cont);6586 EmitBlock(Cont);6587 }6588 }6589 6590 const auto *FnType = cast<FunctionType>(PointeeType);6591 6592 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl);6593 FD && DeviceKernelAttr::isOpenCLSpelling(FD->getAttr<DeviceKernelAttr>()))6594 CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FnType);6595 6596 bool CFIUnchecked =6597 CalleeType->hasPointeeToToCFIUncheckedCalleeFunctionType();6598 6599 // If we are checking indirect calls and this call is indirect, check that the6600 // function pointer is a member of the bit set for the function type.6601 if (SanOpts.has(SanitizerKind::CFIICall) &&6602 (!TargetDecl || !isa<FunctionDecl>(TargetDecl)) && !CFIUnchecked) {6603 auto CheckOrdinal = SanitizerKind::SO_CFIICall;6604 auto CheckHandler = SanitizerHandler::CFICheckFail;6605 SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);6606 EmitSanitizerStatReport(llvm::SanStat_CFI_ICall);6607 6608 llvm::Metadata *MD =6609 CGM.CreateMetadataIdentifierForFnType(QualType(FnType, 0));6610 6611 llvm::Value *TypeId = llvm::MetadataAsValue::get(getLLVMContext(), MD);6612 6613 llvm::Value *CalleePtr = Callee.getFunctionPointer();6614 llvm::Value *TypeTest = Builder.CreateCall(6615 CGM.getIntrinsic(llvm::Intrinsic::type_test), {CalleePtr, TypeId});6616 6617 auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);6618 llvm::Constant *StaticData[] = {6619 llvm::ConstantInt::get(Int8Ty, CFITCK_ICall),6620 EmitCheckSourceLocation(E->getBeginLoc()),6621 EmitCheckTypeDescriptor(QualType(FnType, 0)),6622 };6623 if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && CrossDsoTypeId) {6624 EmitCfiSlowPathCheck(CheckOrdinal, TypeTest, CrossDsoTypeId, CalleePtr,6625 StaticData);6626 } else {6627 EmitCheck(std::make_pair(TypeTest, CheckOrdinal), CheckHandler,6628 StaticData, {CalleePtr, llvm::UndefValue::get(IntPtrTy)});6629 }6630 }6631 6632 CallArgList Args;6633 if (Chain)6634 Args.add(RValue::get(Chain), CGM.getContext().VoidPtrTy);6635 6636 // C++17 requires that we evaluate arguments to a call using assignment syntax6637 // right-to-left, and that we evaluate arguments to certain other operators6638 // left-to-right. Note that we allow this to override the order dictated by6639 // the calling convention on the MS ABI, which means that parameter6640 // destruction order is not necessarily reverse construction order.6641 // FIXME: Revisit this based on C++ committee response to unimplementability.6642 EvaluationOrder Order = EvaluationOrder::Default;6643 bool StaticOperator = false;6644 if (auto *OCE = dyn_cast<CXXOperatorCallExpr>(E)) {6645 if (OCE->isAssignmentOp())6646 Order = EvaluationOrder::ForceRightToLeft;6647 else {6648 switch (OCE->getOperator()) {6649 case OO_LessLess:6650 case OO_GreaterGreater:6651 case OO_AmpAmp:6652 case OO_PipePipe:6653 case OO_Comma:6654 case OO_ArrowStar:6655 Order = EvaluationOrder::ForceLeftToRight;6656 break;6657 default:6658 break;6659 }6660 }6661 6662 if (const auto *MD =6663 dyn_cast_if_present<CXXMethodDecl>(OCE->getCalleeDecl());6664 MD && MD->isStatic())6665 StaticOperator = true;6666 }6667 6668 auto Arguments = E->arguments();6669 if (StaticOperator) {6670 // If we're calling a static operator, we need to emit the object argument6671 // and ignore it.6672 EmitIgnoredExpr(E->getArg(0));6673 Arguments = drop_begin(Arguments, 1);6674 }6675 EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), Arguments,6676 E->getDirectCallee(), /*ParamsToSkip=*/0, Order);6677 6678 const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall(6679 Args, FnType, /*ChainCall=*/Chain);6680 6681 if (ResolvedFnInfo)6682 *ResolvedFnInfo = &FnInfo;6683 6684 // HIP function pointer contains kernel handle when it is used in triple6685 // chevron. The kernel stub needs to be loaded from kernel handle and used6686 // as callee.6687 if (CGM.getLangOpts().HIP && !CGM.getLangOpts().CUDAIsDevice &&6688 isa<CUDAKernelCallExpr>(E) &&6689 (!TargetDecl || !isa<FunctionDecl>(TargetDecl))) {6690 llvm::Value *Handle = Callee.getFunctionPointer();6691 auto *Stub = Builder.CreateLoad(6692 Address(Handle, Handle->getType(), CGM.getPointerAlign()));6693 Callee.setFunctionPointer(Stub);6694 }6695 llvm::CallBase *LocalCallOrInvoke = nullptr;6696 RValue Call = EmitCall(FnInfo, Callee, ReturnValue, Args, &LocalCallOrInvoke,6697 E == MustTailCall, E->getExprLoc());6698 6699 if (auto *CalleeDecl = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {6700 if (CalleeDecl->hasAttr<RestrictAttr>() ||6701 CalleeDecl->hasAttr<MallocSpanAttr>() ||6702 CalleeDecl->hasAttr<AllocSizeAttr>()) {6703 // Function has 'malloc' (aka. 'restrict') or 'alloc_size' attribute.6704 if (SanOpts.has(SanitizerKind::AllocToken)) {6705 // Set !alloc_token metadata.6706 EmitAllocToken(LocalCallOrInvoke, E);6707 }6708 }6709 }6710 if (CallOrInvoke)6711 *CallOrInvoke = LocalCallOrInvoke;6712 6713 return Call;6714}6715 6716LValue CodeGenFunction::6717EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E) {6718 Address BaseAddr = Address::invalid();6719 if (E->getOpcode() == BO_PtrMemI) {6720 BaseAddr = EmitPointerWithAlignment(E->getLHS());6721 } else {6722 BaseAddr = EmitLValue(E->getLHS()).getAddress();6723 }6724 6725 llvm::Value *OffsetV = EmitScalarExpr(E->getRHS());6726 const auto *MPT = E->getRHS()->getType()->castAs<MemberPointerType>();6727 6728 LValueBaseInfo BaseInfo;6729 TBAAAccessInfo TBAAInfo;6730 bool IsInBounds = !getLangOpts().PointerOverflowDefined &&6731 !isUnderlyingBasePointerConstantNull(E->getLHS());6732 Address MemberAddr = EmitCXXMemberDataPointerAddress(6733 E, BaseAddr, OffsetV, MPT, IsInBounds, &BaseInfo, &TBAAInfo);6734 6735 return MakeAddrLValue(MemberAddr, MPT->getPointeeType(), BaseInfo, TBAAInfo);6736}6737 6738/// Given the address of a temporary variable, produce an r-value of6739/// its type.6740RValue CodeGenFunction::convertTempToRValue(Address addr,6741 QualType type,6742 SourceLocation loc) {6743 LValue lvalue = MakeAddrLValue(addr, type, AlignmentSource::Decl);6744 switch (getEvaluationKind(type)) {6745 case TEK_Complex:6746 return RValue::getComplex(EmitLoadOfComplex(lvalue, loc));6747 case TEK_Aggregate:6748 return lvalue.asAggregateRValue();6749 case TEK_Scalar:6750 return RValue::get(EmitLoadOfScalar(lvalue, loc));6751 }6752 llvm_unreachable("bad evaluation kind");6753}6754 6755void CodeGenFunction::SetFPAccuracy(llvm::Value *Val, float Accuracy) {6756 assert(Val->getType()->isFPOrFPVectorTy());6757 if (Accuracy == 0.0 || !isa<llvm::Instruction>(Val))6758 return;6759 6760 llvm::MDBuilder MDHelper(getLLVMContext());6761 llvm::MDNode *Node = MDHelper.createFPMath(Accuracy);6762 6763 cast<llvm::Instruction>(Val)->setMetadata(llvm::LLVMContext::MD_fpmath, Node);6764}6765 6766void CodeGenFunction::SetSqrtFPAccuracy(llvm::Value *Val) {6767 llvm::Type *EltTy = Val->getType()->getScalarType();6768 if (!EltTy->isFloatTy())6769 return;6770 6771 if ((getLangOpts().OpenCL &&6772 !CGM.getCodeGenOpts().OpenCLCorrectlyRoundedDivSqrt) ||6773 (getLangOpts().HIP && getLangOpts().CUDAIsDevice &&6774 !CGM.getCodeGenOpts().HIPCorrectlyRoundedDivSqrt)) {6775 // OpenCL v1.1 s7.4: minimum accuracy of single precision / is 3ulp6776 //6777 // OpenCL v1.2 s5.6.4.2: The -cl-fp32-correctly-rounded-divide-sqrt6778 // build option allows an application to specify that single precision6779 // floating-point divide (x/y and 1/x) and sqrt used in the program6780 // source are correctly rounded.6781 //6782 // TODO: CUDA has a prec-sqrt flag6783 SetFPAccuracy(Val, 3.0f);6784 }6785}6786 6787void CodeGenFunction::SetDivFPAccuracy(llvm::Value *Val) {6788 llvm::Type *EltTy = Val->getType()->getScalarType();6789 if (!EltTy->isFloatTy())6790 return;6791 6792 if ((getLangOpts().OpenCL &&6793 !CGM.getCodeGenOpts().OpenCLCorrectlyRoundedDivSqrt) ||6794 (getLangOpts().HIP && getLangOpts().CUDAIsDevice &&6795 !CGM.getCodeGenOpts().HIPCorrectlyRoundedDivSqrt)) {6796 // OpenCL v1.1 s7.4: minimum accuracy of single precision / is 2.5ulp6797 //6798 // OpenCL v1.2 s5.6.4.2: The -cl-fp32-correctly-rounded-divide-sqrt6799 // build option allows an application to specify that single precision6800 // floating-point divide (x/y and 1/x) and sqrt used in the program6801 // source are correctly rounded.6802 //6803 // TODO: CUDA has a prec-div flag6804 SetFPAccuracy(Val, 2.5f);6805 }6806}6807 6808namespace {6809 struct LValueOrRValue {6810 LValue LV;6811 RValue RV;6812 };6813}6814 6815static LValueOrRValue emitPseudoObjectExpr(CodeGenFunction &CGF,6816 const PseudoObjectExpr *E,6817 bool forLValue,6818 AggValueSlot slot) {6819 SmallVector<CodeGenFunction::OpaqueValueMappingData, 4> opaques;6820 6821 // Find the result expression, if any.6822 const Expr *resultExpr = E->getResultExpr();6823 LValueOrRValue result;6824 6825 for (PseudoObjectExpr::const_semantics_iterator6826 i = E->semantics_begin(), e = E->semantics_end(); i != e; ++i) {6827 const Expr *semantic = *i;6828 6829 // If this semantic expression is an opaque value, bind it6830 // to the result of its source expression.6831 if (const auto *ov = dyn_cast<OpaqueValueExpr>(semantic)) {6832 // Skip unique OVEs.6833 if (ov->isUnique()) {6834 assert(ov != resultExpr &&6835 "A unique OVE cannot be used as the result expression");6836 continue;6837 }6838 6839 // If this is the result expression, we may need to evaluate6840 // directly into the slot.6841 typedef CodeGenFunction::OpaqueValueMappingData OVMA;6842 OVMA opaqueData;6843 if (ov == resultExpr && ov->isPRValue() && !forLValue &&6844 CodeGenFunction::hasAggregateEvaluationKind(ov->getType())) {6845 CGF.EmitAggExpr(ov->getSourceExpr(), slot);6846 LValue LV = CGF.MakeAddrLValue(slot.getAddress(), ov->getType(),6847 AlignmentSource::Decl);6848 opaqueData = OVMA::bind(CGF, ov, LV);6849 result.RV = slot.asRValue();6850 6851 // Otherwise, emit as normal.6852 } else {6853 opaqueData = OVMA::bind(CGF, ov, ov->getSourceExpr());6854 6855 // If this is the result, also evaluate the result now.6856 if (ov == resultExpr) {6857 if (forLValue)6858 result.LV = CGF.EmitLValue(ov);6859 else6860 result.RV = CGF.EmitAnyExpr(ov, slot);6861 }6862 }6863 6864 opaques.push_back(opaqueData);6865 6866 // Otherwise, if the expression is the result, evaluate it6867 // and remember the result.6868 } else if (semantic == resultExpr) {6869 if (forLValue)6870 result.LV = CGF.EmitLValue(semantic);6871 else6872 result.RV = CGF.EmitAnyExpr(semantic, slot);6873 6874 // Otherwise, evaluate the expression in an ignored context.6875 } else {6876 CGF.EmitIgnoredExpr(semantic);6877 }6878 }6879 6880 // Unbind all the opaques now.6881 for (CodeGenFunction::OpaqueValueMappingData &opaque : opaques)6882 opaque.unbind(CGF);6883 6884 return result;6885}6886 6887RValue CodeGenFunction::EmitPseudoObjectRValue(const PseudoObjectExpr *E,6888 AggValueSlot slot) {6889 return emitPseudoObjectExpr(*this, E, false, slot).RV;6890}6891 6892LValue CodeGenFunction::EmitPseudoObjectLValue(const PseudoObjectExpr *E) {6893 return emitPseudoObjectExpr(*this, E, true, AggValueSlot::ignored()).LV;6894}6895 6896void CodeGenFunction::FlattenAccessAndTypeLValue(6897 LValue Val, SmallVectorImpl<LValue> &AccessList) {6898 6899 llvm::SmallVector<6900 std::tuple<LValue, QualType, llvm::SmallVector<llvm::Value *, 4>>, 16>6901 WorkList;6902 llvm::IntegerType *IdxTy = llvm::IntegerType::get(getLLVMContext(), 32);6903 WorkList.push_back({Val, Val.getType(), {llvm::ConstantInt::get(IdxTy, 0)}});6904 6905 while (!WorkList.empty()) {6906 auto [LVal, T, IdxList] = WorkList.pop_back_val();6907 T = T.getCanonicalType().getUnqualifiedType();6908 assert(!isa<MatrixType>(T) && "Matrix types not yet supported in HLSL");6909 6910 if (const auto *CAT = dyn_cast<ConstantArrayType>(T)) {6911 uint64_t Size = CAT->getZExtSize();6912 for (int64_t I = Size - 1; I > -1; I--) {6913 llvm::SmallVector<llvm::Value *, 4> IdxListCopy = IdxList;6914 IdxListCopy.push_back(llvm::ConstantInt::get(IdxTy, I));6915 WorkList.emplace_back(LVal, CAT->getElementType(), IdxListCopy);6916 }6917 } else if (const auto *RT = dyn_cast<RecordType>(T)) {6918 const RecordDecl *Record = RT->getDecl()->getDefinitionOrSelf();6919 assert(!Record->isUnion() && "Union types not supported in flat cast.");6920 6921 const CXXRecordDecl *CXXD = dyn_cast<CXXRecordDecl>(Record);6922 6923 llvm::SmallVector<6924 std::tuple<LValue, QualType, llvm::SmallVector<llvm::Value *, 4>>, 16>6925 ReverseList;6926 if (CXXD && CXXD->isStandardLayout())6927 Record = CXXD->getStandardLayoutBaseWithFields();6928 6929 // deal with potential base classes6930 if (CXXD && !CXXD->isStandardLayout()) {6931 if (CXXD->getNumBases() > 0) {6932 assert(CXXD->getNumBases() == 1 &&6933 "HLSL doesn't support multiple inheritance.");6934 auto Base = CXXD->bases_begin();6935 llvm::SmallVector<llvm::Value *, 4> IdxListCopy = IdxList;6936 IdxListCopy.push_back(llvm::ConstantInt::get(6937 IdxTy, 0)); // base struct should be at index zero6938 ReverseList.emplace_back(LVal, Base->getType(), IdxListCopy);6939 }6940 }6941 6942 const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(Record);6943 6944 llvm::Type *LLVMT = ConvertTypeForMem(T);6945 CharUnits Align = getContext().getTypeAlignInChars(T);6946 LValue RLValue;6947 bool createdGEP = false;6948 for (auto *FD : Record->fields()) {6949 if (FD->isBitField()) {6950 if (FD->isUnnamedBitField())6951 continue;6952 if (!createdGEP) {6953 createdGEP = true;6954 Address GEP = Builder.CreateInBoundsGEP(LVal.getAddress(), IdxList,6955 LLVMT, Align, "gep");6956 RLValue = MakeAddrLValue(GEP, T);6957 }6958 LValue FieldLVal = EmitLValueForField(RLValue, FD, true);6959 ReverseList.push_back({FieldLVal, FD->getType(), {}});6960 } else {6961 llvm::SmallVector<llvm::Value *, 4> IdxListCopy = IdxList;6962 IdxListCopy.push_back(6963 llvm::ConstantInt::get(IdxTy, Layout.getLLVMFieldNo(FD)));6964 ReverseList.emplace_back(LVal, FD->getType(), IdxListCopy);6965 }6966 }6967 6968 std::reverse(ReverseList.begin(), ReverseList.end());6969 llvm::append_range(WorkList, ReverseList);6970 } else if (const auto *VT = dyn_cast<VectorType>(T)) {6971 llvm::Type *LLVMT = ConvertTypeForMem(T);6972 CharUnits Align = getContext().getTypeAlignInChars(T);6973 Address GEP = Builder.CreateInBoundsGEP(LVal.getAddress(), IdxList, LLVMT,6974 Align, "vector.gep");6975 LValue Base = MakeAddrLValue(GEP, T);6976 for (unsigned I = 0, E = VT->getNumElements(); I < E; I++) {6977 llvm::Constant *Idx = llvm::ConstantInt::get(IdxTy, I);6978 LValue LV =6979 LValue::MakeVectorElt(Base.getAddress(), Idx, VT->getElementType(),6980 Base.getBaseInfo(), TBAAAccessInfo());6981 AccessList.emplace_back(LV);6982 }6983 } else { // a scalar/builtin type6984 if (!IdxList.empty()) {6985 llvm::Type *LLVMT = ConvertTypeForMem(T);6986 CharUnits Align = getContext().getTypeAlignInChars(T);6987 Address GEP = Builder.CreateInBoundsGEP(LVal.getAddress(), IdxList,6988 LLVMT, Align, "gep");6989 AccessList.emplace_back(MakeAddrLValue(GEP, T));6990 } else // must be a bitfield we already created an lvalue for6991 AccessList.emplace_back(LVal);6992 }6993 }6994}6995