6336 lines · cpp
1//===--- CGCall.cpp - Encapsulate calling convention details --------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// These classes wrap the information about a call or function10// definition used to handle ABI compliancy.11//12//===----------------------------------------------------------------------===//13 14#include "CGCall.h"15#include "ABIInfo.h"16#include "ABIInfoImpl.h"17#include "CGBlocks.h"18#include "CGCXXABI.h"19#include "CGCleanup.h"20#include "CGDebugInfo.h"21#include "CGRecordLayout.h"22#include "CodeGenFunction.h"23#include "CodeGenModule.h"24#include "CodeGenPGO.h"25#include "TargetInfo.h"26#include "clang/AST/Attr.h"27#include "clang/AST/Decl.h"28#include "clang/AST/DeclCXX.h"29#include "clang/AST/DeclObjC.h"30#include "clang/Basic/CodeGenOptions.h"31#include "clang/Basic/TargetInfo.h"32#include "clang/CodeGen/CGFunctionInfo.h"33#include "clang/CodeGen/SwiftCallingConv.h"34#include "llvm/ADT/StringExtras.h"35#include "llvm/Analysis/ValueTracking.h"36#include "llvm/IR/Assumptions.h"37#include "llvm/IR/AttributeMask.h"38#include "llvm/IR/Attributes.h"39#include "llvm/IR/CallingConv.h"40#include "llvm/IR/DataLayout.h"41#include "llvm/IR/DebugInfoMetadata.h"42#include "llvm/IR/InlineAsm.h"43#include "llvm/IR/IntrinsicInst.h"44#include "llvm/IR/Intrinsics.h"45#include "llvm/IR/Type.h"46#include "llvm/Transforms/Utils/Local.h"47#include <optional>48using namespace clang;49using namespace CodeGen;50 51/***/52 53unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {54 switch (CC) {55 default:56 return llvm::CallingConv::C;57 case CC_X86StdCall:58 return llvm::CallingConv::X86_StdCall;59 case CC_X86FastCall:60 return llvm::CallingConv::X86_FastCall;61 case CC_X86RegCall:62 return llvm::CallingConv::X86_RegCall;63 case CC_X86ThisCall:64 return llvm::CallingConv::X86_ThisCall;65 case CC_Win64:66 return llvm::CallingConv::Win64;67 case CC_X86_64SysV:68 return llvm::CallingConv::X86_64_SysV;69 case CC_AAPCS:70 return llvm::CallingConv::ARM_AAPCS;71 case CC_AAPCS_VFP:72 return llvm::CallingConv::ARM_AAPCS_VFP;73 case CC_IntelOclBicc:74 return llvm::CallingConv::Intel_OCL_BI;75 // TODO: Add support for __pascal to LLVM.76 case CC_X86Pascal:77 return llvm::CallingConv::C;78 // TODO: Add support for __vectorcall to LLVM.79 case CC_X86VectorCall:80 return llvm::CallingConv::X86_VectorCall;81 case CC_AArch64VectorCall:82 return llvm::CallingConv::AArch64_VectorCall;83 case CC_AArch64SVEPCS:84 return llvm::CallingConv::AArch64_SVE_VectorCall;85 case CC_SpirFunction:86 return llvm::CallingConv::SPIR_FUNC;87 case CC_DeviceKernel:88 return CGM.getTargetCodeGenInfo().getDeviceKernelCallingConv();89 case CC_PreserveMost:90 return llvm::CallingConv::PreserveMost;91 case CC_PreserveAll:92 return llvm::CallingConv::PreserveAll;93 case CC_Swift:94 return llvm::CallingConv::Swift;95 case CC_SwiftAsync:96 return llvm::CallingConv::SwiftTail;97 case CC_M68kRTD:98 return llvm::CallingConv::M68k_RTD;99 case CC_PreserveNone:100 return llvm::CallingConv::PreserveNone;101 // clang-format off102 case CC_RISCVVectorCall: return llvm::CallingConv::RISCV_VectorCall;103 // clang-format on104#define CC_VLS_CASE(ABI_VLEN) \105 case CC_RISCVVLSCall_##ABI_VLEN: \106 return llvm::CallingConv::RISCV_VLSCall_##ABI_VLEN;107 CC_VLS_CASE(32)108 CC_VLS_CASE(64)109 CC_VLS_CASE(128)110 CC_VLS_CASE(256)111 CC_VLS_CASE(512)112 CC_VLS_CASE(1024)113 CC_VLS_CASE(2048)114 CC_VLS_CASE(4096)115 CC_VLS_CASE(8192)116 CC_VLS_CASE(16384)117 CC_VLS_CASE(32768)118 CC_VLS_CASE(65536)119#undef CC_VLS_CASE120 }121}122 123/// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR124/// qualification. Either or both of RD and MD may be null. A null RD indicates125/// that there is no meaningful 'this' type, and a null MD can occur when126/// calling a method pointer.127CanQualType CodeGenTypes::DeriveThisType(const CXXRecordDecl *RD,128 const CXXMethodDecl *MD) {129 CanQualType RecTy;130 if (RD)131 RecTy = Context.getCanonicalTagType(RD);132 else133 RecTy = Context.VoidTy;134 135 if (MD)136 RecTy = CanQualType::CreateUnsafe(Context.getAddrSpaceQualType(137 RecTy, MD->getMethodQualifiers().getAddressSpace()));138 return Context.getPointerType(RecTy);139}140 141/// Returns the canonical formal type of the given C++ method.142static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {143 return MD->getType()144 ->getCanonicalTypeUnqualified()145 .getAs<FunctionProtoType>();146}147 148/// Returns the "extra-canonicalized" return type, which discards149/// qualifiers on the return type. Codegen doesn't care about them,150/// and it makes ABI code a little easier to be able to assume that151/// all parameter and return types are top-level unqualified.152static CanQualType GetReturnType(QualType RetTy) {153 return RetTy->getCanonicalTypeUnqualified().getUnqualifiedType();154}155 156/// Arrange the argument and result information for a value of the given157/// unprototyped freestanding function type.158const CGFunctionInfo &159CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) {160 // When translating an unprototyped function type, always use a161 // variadic type.162 return arrangeLLVMFunctionInfo(FTNP->getReturnType().getUnqualifiedType(),163 FnInfoOpts::None, {}, FTNP->getExtInfo(), {},164 RequiredArgs(0));165}166 167static void addExtParameterInfosForCall(168 llvm::SmallVectorImpl<FunctionProtoType::ExtParameterInfo> ¶mInfos,169 const FunctionProtoType *proto, unsigned prefixArgs, unsigned totalArgs) {170 assert(proto->hasExtParameterInfos());171 assert(paramInfos.size() <= prefixArgs);172 assert(proto->getNumParams() + prefixArgs <= totalArgs);173 174 paramInfos.reserve(totalArgs);175 176 // Add default infos for any prefix args that don't already have infos.177 paramInfos.resize(prefixArgs);178 179 // Add infos for the prototype.180 for (const auto &ParamInfo : proto->getExtParameterInfos()) {181 paramInfos.push_back(ParamInfo);182 // pass_object_size params have no parameter info.183 if (ParamInfo.hasPassObjectSize())184 paramInfos.emplace_back();185 }186 187 assert(paramInfos.size() <= totalArgs &&188 "Did we forget to insert pass_object_size args?");189 // Add default infos for the variadic and/or suffix arguments.190 paramInfos.resize(totalArgs);191}192 193/// Adds the formal parameters in FPT to the given prefix. If any parameter in194/// FPT has pass_object_size attrs, then we'll add parameters for those, too.195static void appendParameterTypes(196 const CodeGenTypes &CGT, SmallVectorImpl<CanQualType> &prefix,197 SmallVectorImpl<FunctionProtoType::ExtParameterInfo> ¶mInfos,198 CanQual<FunctionProtoType> FPT) {199 // Fast path: don't touch param info if we don't need to.200 if (!FPT->hasExtParameterInfos()) {201 assert(paramInfos.empty() &&202 "We have paramInfos, but the prototype doesn't?");203 prefix.append(FPT->param_type_begin(), FPT->param_type_end());204 return;205 }206 207 unsigned PrefixSize = prefix.size();208 // In the vast majority of cases, we'll have precisely FPT->getNumParams()209 // parameters; the only thing that can change this is the presence of210 // pass_object_size. So, we preallocate for the common case.211 prefix.reserve(prefix.size() + FPT->getNumParams());212 213 auto ExtInfos = FPT->getExtParameterInfos();214 assert(ExtInfos.size() == FPT->getNumParams());215 for (unsigned I = 0, E = FPT->getNumParams(); I != E; ++I) {216 prefix.push_back(FPT->getParamType(I));217 if (ExtInfos[I].hasPassObjectSize())218 prefix.push_back(CGT.getContext().getCanonicalSizeType());219 }220 221 addExtParameterInfosForCall(paramInfos, FPT.getTypePtr(), PrefixSize,222 prefix.size());223}224 225using ExtParameterInfoList =226 SmallVector<FunctionProtoType::ExtParameterInfo, 16>;227 228/// Arrange the LLVM function layout for a value of the given function229/// type, on top of any implicit parameters already stored.230static const CGFunctionInfo &231arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,232 SmallVectorImpl<CanQualType> &prefix,233 CanQual<FunctionProtoType> FTP) {234 ExtParameterInfoList paramInfos;235 RequiredArgs Required = RequiredArgs::forPrototypePlus(FTP, prefix.size());236 appendParameterTypes(CGT, prefix, paramInfos, FTP);237 CanQualType resultType = FTP->getReturnType().getUnqualifiedType();238 239 FnInfoOpts opts =240 instanceMethod ? FnInfoOpts::IsInstanceMethod : FnInfoOpts::None;241 return CGT.arrangeLLVMFunctionInfo(resultType, opts, prefix,242 FTP->getExtInfo(), paramInfos, Required);243}244 245using CanQualTypeList = SmallVector<CanQualType, 16>;246 247/// Arrange the argument and result information for a value of the248/// given freestanding function type.249const CGFunctionInfo &250CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {251 CanQualTypeList argTypes;252 return ::arrangeLLVMFunctionInfo(*this, /*instanceMethod=*/false, argTypes,253 FTP);254}255 256static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D,257 bool IsTargetDefaultMSABI) {258 // Set the appropriate calling convention for the Function.259 if (D->hasAttr<StdCallAttr>())260 return CC_X86StdCall;261 262 if (D->hasAttr<FastCallAttr>())263 return CC_X86FastCall;264 265 if (D->hasAttr<RegCallAttr>())266 return CC_X86RegCall;267 268 if (D->hasAttr<ThisCallAttr>())269 return CC_X86ThisCall;270 271 if (D->hasAttr<VectorCallAttr>())272 return CC_X86VectorCall;273 274 if (D->hasAttr<PascalAttr>())275 return CC_X86Pascal;276 277 if (PcsAttr *PCS = D->getAttr<PcsAttr>())278 return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);279 280 if (D->hasAttr<AArch64VectorPcsAttr>())281 return CC_AArch64VectorCall;282 283 if (D->hasAttr<AArch64SVEPcsAttr>())284 return CC_AArch64SVEPCS;285 286 if (D->hasAttr<DeviceKernelAttr>())287 return CC_DeviceKernel;288 289 if (D->hasAttr<IntelOclBiccAttr>())290 return CC_IntelOclBicc;291 292 if (D->hasAttr<MSABIAttr>())293 return IsTargetDefaultMSABI ? CC_C : CC_Win64;294 295 if (D->hasAttr<SysVABIAttr>())296 return IsTargetDefaultMSABI ? CC_X86_64SysV : CC_C;297 298 if (D->hasAttr<PreserveMostAttr>())299 return CC_PreserveMost;300 301 if (D->hasAttr<PreserveAllAttr>())302 return CC_PreserveAll;303 304 if (D->hasAttr<M68kRTDAttr>())305 return CC_M68kRTD;306 307 if (D->hasAttr<PreserveNoneAttr>())308 return CC_PreserveNone;309 310 if (D->hasAttr<RISCVVectorCCAttr>())311 return CC_RISCVVectorCall;312 313 if (RISCVVLSCCAttr *PCS = D->getAttr<RISCVVLSCCAttr>()) {314 switch (PCS->getVectorWidth()) {315 default:316 llvm_unreachable("Invalid RISC-V VLS ABI VLEN");317#define CC_VLS_CASE(ABI_VLEN) \318 case ABI_VLEN: \319 return CC_RISCVVLSCall_##ABI_VLEN;320 CC_VLS_CASE(32)321 CC_VLS_CASE(64)322 CC_VLS_CASE(128)323 CC_VLS_CASE(256)324 CC_VLS_CASE(512)325 CC_VLS_CASE(1024)326 CC_VLS_CASE(2048)327 CC_VLS_CASE(4096)328 CC_VLS_CASE(8192)329 CC_VLS_CASE(16384)330 CC_VLS_CASE(32768)331 CC_VLS_CASE(65536)332#undef CC_VLS_CASE333 }334 }335 336 return CC_C;337}338 339/// Arrange the argument and result information for a call to an340/// unknown C++ non-static member function of the given abstract type.341/// (A null RD means we don't have any meaningful "this" argument type,342/// so fall back to a generic pointer type).343/// The member function must be an ordinary function, i.e. not a344/// constructor or destructor.345const CGFunctionInfo &346CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,347 const FunctionProtoType *FTP,348 const CXXMethodDecl *MD) {349 CanQualTypeList argTypes;350 351 // Add the 'this' pointer.352 argTypes.push_back(DeriveThisType(RD, MD));353 354 return ::arrangeLLVMFunctionInfo(355 *this, /*instanceMethod=*/true, argTypes,356 FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());357}358 359/// Set calling convention for CUDA/HIP kernel.360static void setCUDAKernelCallingConvention(CanQualType &FTy, CodeGenModule &CGM,361 const FunctionDecl *FD) {362 if (FD->hasAttr<CUDAGlobalAttr>()) {363 const FunctionType *FT = FTy->getAs<FunctionType>();364 CGM.getTargetCodeGenInfo().setCUDAKernelCallingConvention(FT);365 FTy = FT->getCanonicalTypeUnqualified();366 }367}368 369/// Arrange the argument and result information for a declaration or370/// definition of the given C++ non-static member function. The371/// member function must be an ordinary function, i.e. not a372/// constructor or destructor.373const CGFunctionInfo &374CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) {375 assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");376 assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");377 378 CanQualType FT = GetFormalType(MD).getAs<Type>();379 setCUDAKernelCallingConvention(FT, CGM, MD);380 auto prototype = FT.getAs<FunctionProtoType>();381 382 if (MD->isImplicitObjectMemberFunction()) {383 // The abstract case is perfectly fine.384 const CXXRecordDecl *ThisType =385 getCXXABI().getThisArgumentTypeForMethod(MD);386 return arrangeCXXMethodType(ThisType, prototype.getTypePtr(), MD);387 }388 389 return arrangeFreeFunctionType(prototype);390}391 392bool CodeGenTypes::inheritingCtorHasParams(393 const InheritedConstructor &Inherited, CXXCtorType Type) {394 // Parameters are unnecessary if we're constructing a base class subobject395 // and the inherited constructor lives in a virtual base.396 return Type == Ctor_Complete ||397 !Inherited.getShadowDecl()->constructsVirtualBase() ||398 !Target.getCXXABI().hasConstructorVariants();399}400 401const CGFunctionInfo &402CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) {403 auto *MD = cast<CXXMethodDecl>(GD.getDecl());404 405 CanQualTypeList argTypes;406 ExtParameterInfoList paramInfos;407 408 const CXXRecordDecl *ThisType = getCXXABI().getThisArgumentTypeForMethod(GD);409 argTypes.push_back(DeriveThisType(ThisType, MD));410 411 bool PassParams = true;412 413 if (auto *CD = dyn_cast<CXXConstructorDecl>(MD)) {414 // A base class inheriting constructor doesn't get forwarded arguments415 // needed to construct a virtual base (or base class thereof).416 if (auto Inherited = CD->getInheritedConstructor())417 PassParams = inheritingCtorHasParams(Inherited, GD.getCtorType());418 }419 420 CanQual<FunctionProtoType> FTP = GetFormalType(MD);421 422 // Add the formal parameters.423 if (PassParams)424 appendParameterTypes(*this, argTypes, paramInfos, FTP);425 426 CGCXXABI::AddedStructorArgCounts AddedArgs =427 getCXXABI().buildStructorSignature(GD, argTypes);428 if (!paramInfos.empty()) {429 // Note: prefix implies after the first param.430 if (AddedArgs.Prefix)431 paramInfos.insert(paramInfos.begin() + 1, AddedArgs.Prefix,432 FunctionProtoType::ExtParameterInfo{});433 if (AddedArgs.Suffix)434 paramInfos.append(AddedArgs.Suffix,435 FunctionProtoType::ExtParameterInfo{});436 }437 438 RequiredArgs required =439 (PassParams && MD->isVariadic() ? RequiredArgs(argTypes.size())440 : RequiredArgs::All);441 442 FunctionType::ExtInfo extInfo = FTP->getExtInfo();443 CanQualType resultType = getCXXABI().HasThisReturn(GD) ? argTypes.front()444 : getCXXABI().hasMostDerivedReturn(GD)445 ? CGM.getContext().VoidPtrTy446 : Context.VoidTy;447 return arrangeLLVMFunctionInfo(resultType, FnInfoOpts::IsInstanceMethod,448 argTypes, extInfo, paramInfos, required);449}450 451static CanQualTypeList getArgTypesForCall(ASTContext &ctx,452 const CallArgList &args) {453 CanQualTypeList argTypes;454 for (auto &arg : args)455 argTypes.push_back(ctx.getCanonicalParamType(arg.Ty));456 return argTypes;457}458 459static CanQualTypeList getArgTypesForDeclaration(ASTContext &ctx,460 const FunctionArgList &args) {461 CanQualTypeList argTypes;462 for (auto &arg : args)463 argTypes.push_back(ctx.getCanonicalParamType(arg->getType()));464 return argTypes;465}466 467static ExtParameterInfoList468getExtParameterInfosForCall(const FunctionProtoType *proto, unsigned prefixArgs,469 unsigned totalArgs) {470 ExtParameterInfoList result;471 if (proto->hasExtParameterInfos()) {472 addExtParameterInfosForCall(result, proto, prefixArgs, totalArgs);473 }474 return result;475}476 477/// Arrange a call to a C++ method, passing the given arguments.478///479/// ExtraPrefixArgs is the number of ABI-specific args passed after the `this`480/// parameter.481/// ExtraSuffixArgs is the number of ABI-specific args passed at the end of482/// args.483/// PassProtoArgs indicates whether `args` has args for the parameters in the484/// given CXXConstructorDecl.485const CGFunctionInfo &CodeGenTypes::arrangeCXXConstructorCall(486 const CallArgList &args, const CXXConstructorDecl *D, CXXCtorType CtorKind,487 unsigned ExtraPrefixArgs, unsigned ExtraSuffixArgs, bool PassProtoArgs) {488 CanQualTypeList ArgTypes;489 for (const auto &Arg : args)490 ArgTypes.push_back(Context.getCanonicalParamType(Arg.Ty));491 492 // +1 for implicit this, which should always be args[0].493 unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs;494 495 CanQual<FunctionProtoType> FPT = GetFormalType(D);496 RequiredArgs Required = PassProtoArgs497 ? RequiredArgs::forPrototypePlus(498 FPT, TotalPrefixArgs + ExtraSuffixArgs)499 : RequiredArgs::All;500 501 GlobalDecl GD(D, CtorKind);502 CanQualType ResultType = getCXXABI().HasThisReturn(GD) ? ArgTypes.front()503 : getCXXABI().hasMostDerivedReturn(GD)504 ? CGM.getContext().VoidPtrTy505 : Context.VoidTy;506 507 FunctionType::ExtInfo Info = FPT->getExtInfo();508 ExtParameterInfoList ParamInfos;509 // If the prototype args are elided, we should only have ABI-specific args,510 // which never have param info.511 if (PassProtoArgs && FPT->hasExtParameterInfos()) {512 // ABI-specific suffix arguments are treated the same as variadic arguments.513 addExtParameterInfosForCall(ParamInfos, FPT.getTypePtr(), TotalPrefixArgs,514 ArgTypes.size());515 }516 517 return arrangeLLVMFunctionInfo(ResultType, FnInfoOpts::IsInstanceMethod,518 ArgTypes, Info, ParamInfos, Required);519}520 521/// Arrange the argument and result information for the declaration or522/// definition of the given function.523const CGFunctionInfo &524CodeGenTypes::arrangeFunctionDeclaration(const GlobalDecl GD) {525 const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());526 if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD))527 if (MD->isImplicitObjectMemberFunction())528 return arrangeCXXMethodDeclaration(MD);529 530 CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();531 532 assert(isa<FunctionType>(FTy));533 setCUDAKernelCallingConvention(FTy, CGM, FD);534 535 if (DeviceKernelAttr::isOpenCLSpelling(FD->getAttr<DeviceKernelAttr>()) &&536 GD.getKernelReferenceKind() == KernelReferenceKind::Stub) {537 const FunctionType *FT = FTy->getAs<FunctionType>();538 CGM.getTargetCodeGenInfo().setOCLKernelStubCallingConvention(FT);539 FTy = FT->getCanonicalTypeUnqualified();540 }541 542 // When declaring a function without a prototype, always use a543 // non-variadic type.544 if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) {545 return arrangeLLVMFunctionInfo(noProto->getReturnType(), FnInfoOpts::None,546 {}, noProto->getExtInfo(), {},547 RequiredArgs::All);548 }549 550 return arrangeFreeFunctionType(FTy.castAs<FunctionProtoType>());551}552 553/// Arrange the argument and result information for the declaration or554/// definition of an Objective-C method.555const CGFunctionInfo &556CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) {557 // It happens that this is the same as a call with no optional558 // arguments, except also using the formal 'self' type.559 return arrangeObjCMessageSendSignature(MD, MD->getSelfDecl()->getType());560}561 562/// Arrange the argument and result information for the function type563/// through which to perform a send to the given Objective-C method,564/// using the given receiver type. The receiver type is not always565/// the 'self' type of the method or even an Objective-C pointer type.566/// This is *not* the right method for actually performing such a567/// message send, due to the possibility of optional arguments.568const CGFunctionInfo &569CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,570 QualType receiverType) {571 CanQualTypeList argTys;572 ExtParameterInfoList extParamInfos(MD->isDirectMethod() ? 1 : 2);573 argTys.push_back(Context.getCanonicalParamType(receiverType));574 if (!MD->isDirectMethod())575 argTys.push_back(Context.getCanonicalParamType(Context.getObjCSelType()));576 for (const auto *I : MD->parameters()) {577 argTys.push_back(Context.getCanonicalParamType(I->getType()));578 auto extParamInfo = FunctionProtoType::ExtParameterInfo().withIsNoEscape(579 I->hasAttr<NoEscapeAttr>());580 extParamInfos.push_back(extParamInfo);581 }582 583 FunctionType::ExtInfo einfo;584 bool IsTargetDefaultMSABI =585 getContext().getTargetInfo().getTriple().isOSWindows() ||586 getContext().getTargetInfo().getTriple().isUEFI();587 einfo = einfo.withCallingConv(588 getCallingConventionForDecl(MD, IsTargetDefaultMSABI));589 590 if (getContext().getLangOpts().ObjCAutoRefCount &&591 MD->hasAttr<NSReturnsRetainedAttr>())592 einfo = einfo.withProducesResult(true);593 594 RequiredArgs required =595 (MD->isVariadic() ? RequiredArgs(argTys.size()) : RequiredArgs::All);596 597 return arrangeLLVMFunctionInfo(GetReturnType(MD->getReturnType()),598 FnInfoOpts::None, argTys, einfo, extParamInfos,599 required);600}601 602const CGFunctionInfo &603CodeGenTypes::arrangeUnprototypedObjCMessageSend(QualType returnType,604 const CallArgList &args) {605 CanQualTypeList argTypes = getArgTypesForCall(Context, args);606 FunctionType::ExtInfo einfo;607 608 return arrangeLLVMFunctionInfo(GetReturnType(returnType), FnInfoOpts::None,609 argTypes, einfo, {}, RequiredArgs::All);610}611 612const CGFunctionInfo &CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) {613 // FIXME: Do we need to handle ObjCMethodDecl?614 if (isa<CXXConstructorDecl>(GD.getDecl()) ||615 isa<CXXDestructorDecl>(GD.getDecl()))616 return arrangeCXXStructorDeclaration(GD);617 618 return arrangeFunctionDeclaration(GD);619}620 621/// Arrange a thunk that takes 'this' as the first parameter followed by622/// varargs. Return a void pointer, regardless of the actual return type.623/// The body of the thunk will end in a musttail call to a function of the624/// correct type, and the caller will bitcast the function to the correct625/// prototype.626const CGFunctionInfo &627CodeGenTypes::arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD) {628 assert(MD->isVirtual() && "only methods have thunks");629 CanQual<FunctionProtoType> FTP = GetFormalType(MD);630 CanQualType ArgTys[] = {DeriveThisType(MD->getParent(), MD)};631 return arrangeLLVMFunctionInfo(Context.VoidTy, FnInfoOpts::None, ArgTys,632 FTP->getExtInfo(), {}, RequiredArgs(1));633}634 635const CGFunctionInfo &636CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD,637 CXXCtorType CT) {638 assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);639 640 CanQual<FunctionProtoType> FTP = GetFormalType(CD);641 SmallVector<CanQualType, 2> ArgTys;642 const CXXRecordDecl *RD = CD->getParent();643 ArgTys.push_back(DeriveThisType(RD, CD));644 if (CT == Ctor_CopyingClosure)645 ArgTys.push_back(*FTP->param_type_begin());646 if (RD->getNumVBases() > 0)647 ArgTys.push_back(Context.IntTy);648 CallingConv CC = Context.getDefaultCallingConvention(649 /*IsVariadic=*/false, /*IsCXXMethod=*/true);650 return arrangeLLVMFunctionInfo(Context.VoidTy, FnInfoOpts::IsInstanceMethod,651 ArgTys, FunctionType::ExtInfo(CC), {},652 RequiredArgs::All);653}654 655/// Arrange a call as unto a free function, except possibly with an656/// additional number of formal parameters considered required.657static const CGFunctionInfo &658arrangeFreeFunctionLikeCall(CodeGenTypes &CGT, CodeGenModule &CGM,659 const CallArgList &args, const FunctionType *fnType,660 unsigned numExtraRequiredArgs, bool chainCall) {661 assert(args.size() >= numExtraRequiredArgs);662 663 ExtParameterInfoList paramInfos;664 665 // In most cases, there are no optional arguments.666 RequiredArgs required = RequiredArgs::All;667 668 // If we have a variadic prototype, the required arguments are the669 // extra prefix plus the arguments in the prototype.670 if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(fnType)) {671 if (proto->isVariadic())672 required = RequiredArgs::forPrototypePlus(proto, numExtraRequiredArgs);673 674 if (proto->hasExtParameterInfos())675 addExtParameterInfosForCall(paramInfos, proto, numExtraRequiredArgs,676 args.size());677 678 // If we don't have a prototype at all, but we're supposed to679 // explicitly use the variadic convention for unprototyped calls,680 // treat all of the arguments as required but preserve the nominal681 // possibility of variadics.682 } else if (CGM.getTargetCodeGenInfo().isNoProtoCallVariadic(683 args, cast<FunctionNoProtoType>(fnType))) {684 required = RequiredArgs(args.size());685 }686 687 CanQualTypeList argTypes;688 for (const auto &arg : args)689 argTypes.push_back(CGT.getContext().getCanonicalParamType(arg.Ty));690 FnInfoOpts opts = chainCall ? FnInfoOpts::IsChainCall : FnInfoOpts::None;691 return CGT.arrangeLLVMFunctionInfo(GetReturnType(fnType->getReturnType()),692 opts, argTypes, fnType->getExtInfo(),693 paramInfos, required);694}695 696/// Figure out the rules for calling a function with the given formal697/// type using the given arguments. The arguments are necessary698/// because the function might be unprototyped, in which case it's699/// target-dependent in crazy ways.700const CGFunctionInfo &CodeGenTypes::arrangeFreeFunctionCall(701 const CallArgList &args, const FunctionType *fnType, bool chainCall) {702 return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType,703 chainCall ? 1 : 0, chainCall);704}705 706/// A block function is essentially a free function with an707/// extra implicit argument.708const CGFunctionInfo &709CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args,710 const FunctionType *fnType) {711 return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType, 1,712 /*chainCall=*/false);713}714 715const CGFunctionInfo &716CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto,717 const FunctionArgList ¶ms) {718 ExtParameterInfoList paramInfos =719 getExtParameterInfosForCall(proto, 1, params.size());720 CanQualTypeList argTypes = getArgTypesForDeclaration(Context, params);721 722 return arrangeLLVMFunctionInfo(GetReturnType(proto->getReturnType()),723 FnInfoOpts::None, argTypes,724 proto->getExtInfo(), paramInfos,725 RequiredArgs::forPrototypePlus(proto, 1));726}727 728const CGFunctionInfo &729CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType,730 const CallArgList &args) {731 CanQualTypeList argTypes;732 for (const auto &Arg : args)733 argTypes.push_back(Context.getCanonicalParamType(Arg.Ty));734 return arrangeLLVMFunctionInfo(GetReturnType(resultType), FnInfoOpts::None,735 argTypes, FunctionType::ExtInfo(),736 /*paramInfos=*/{}, RequiredArgs::All);737}738 739const CGFunctionInfo &740CodeGenTypes::arrangeBuiltinFunctionDeclaration(QualType resultType,741 const FunctionArgList &args) {742 CanQualTypeList argTypes = getArgTypesForDeclaration(Context, args);743 744 return arrangeLLVMFunctionInfo(GetReturnType(resultType), FnInfoOpts::None,745 argTypes, FunctionType::ExtInfo(), {},746 RequiredArgs::All);747}748 749const CGFunctionInfo &CodeGenTypes::arrangeBuiltinFunctionDeclaration(750 CanQualType resultType, ArrayRef<CanQualType> argTypes) {751 return arrangeLLVMFunctionInfo(resultType, FnInfoOpts::None, argTypes,752 FunctionType::ExtInfo(), {},753 RequiredArgs::All);754}755 756const CGFunctionInfo &CodeGenTypes::arrangeDeviceKernelCallerDeclaration(757 QualType resultType, const FunctionArgList &args) {758 CanQualTypeList argTypes = getArgTypesForDeclaration(Context, args);759 760 return arrangeLLVMFunctionInfo(GetReturnType(resultType), FnInfoOpts::None,761 argTypes,762 FunctionType::ExtInfo(CC_DeviceKernel),763 /*paramInfos=*/{}, RequiredArgs::All);764}765 766/// Arrange a call to a C++ method, passing the given arguments.767///768/// numPrefixArgs is the number of ABI-specific prefix arguments we have. It769/// does not count `this`.770const CGFunctionInfo &CodeGenTypes::arrangeCXXMethodCall(771 const CallArgList &args, const FunctionProtoType *proto,772 RequiredArgs required, unsigned numPrefixArgs) {773 assert(numPrefixArgs + 1 <= args.size() &&774 "Emitting a call with less args than the required prefix?");775 // Add one to account for `this`. It's a bit awkward here, but we don't count776 // `this` in similar places elsewhere.777 ExtParameterInfoList paramInfos =778 getExtParameterInfosForCall(proto, numPrefixArgs + 1, args.size());779 780 CanQualTypeList argTypes = getArgTypesForCall(Context, args);781 782 FunctionType::ExtInfo info = proto->getExtInfo();783 return arrangeLLVMFunctionInfo(GetReturnType(proto->getReturnType()),784 FnInfoOpts::IsInstanceMethod, argTypes, info,785 paramInfos, required);786}787 788const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {789 return arrangeLLVMFunctionInfo(getContext().VoidTy, FnInfoOpts::None, {},790 FunctionType::ExtInfo(), {},791 RequiredArgs::All);792}793 794const CGFunctionInfo &CodeGenTypes::arrangeCall(const CGFunctionInfo &signature,795 const CallArgList &args) {796 assert(signature.arg_size() <= args.size());797 if (signature.arg_size() == args.size())798 return signature;799 800 ExtParameterInfoList paramInfos;801 auto sigParamInfos = signature.getExtParameterInfos();802 if (!sigParamInfos.empty()) {803 paramInfos.append(sigParamInfos.begin(), sigParamInfos.end());804 paramInfos.resize(args.size());805 }806 807 CanQualTypeList argTypes = getArgTypesForCall(Context, args);808 809 assert(signature.getRequiredArgs().allowsOptionalArgs());810 FnInfoOpts opts = FnInfoOpts::None;811 if (signature.isInstanceMethod())812 opts |= FnInfoOpts::IsInstanceMethod;813 if (signature.isChainCall())814 opts |= FnInfoOpts::IsChainCall;815 if (signature.isDelegateCall())816 opts |= FnInfoOpts::IsDelegateCall;817 return arrangeLLVMFunctionInfo(signature.getReturnType(), opts, argTypes,818 signature.getExtInfo(), paramInfos,819 signature.getRequiredArgs());820}821 822namespace clang {823namespace CodeGen {824void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI);825}826} // namespace clang827 828/// Arrange the argument and result information for an abstract value829/// of a given function type. This is the method which all of the830/// above functions ultimately defer to.831const CGFunctionInfo &CodeGenTypes::arrangeLLVMFunctionInfo(832 CanQualType resultType, FnInfoOpts opts, ArrayRef<CanQualType> argTypes,833 FunctionType::ExtInfo info,834 ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,835 RequiredArgs required) {836 assert(llvm::all_of(argTypes,837 [](CanQualType T) { return T.isCanonicalAsParam(); }));838 839 // Lookup or create unique function info.840 llvm::FoldingSetNodeID ID;841 bool isInstanceMethod =842 (opts & FnInfoOpts::IsInstanceMethod) == FnInfoOpts::IsInstanceMethod;843 bool isChainCall =844 (opts & FnInfoOpts::IsChainCall) == FnInfoOpts::IsChainCall;845 bool isDelegateCall =846 (opts & FnInfoOpts::IsDelegateCall) == FnInfoOpts::IsDelegateCall;847 CGFunctionInfo::Profile(ID, isInstanceMethod, isChainCall, isDelegateCall,848 info, paramInfos, required, resultType, argTypes);849 850 void *insertPos = nullptr;851 CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, insertPos);852 if (FI)853 return *FI;854 855 unsigned CC = ClangCallConvToLLVMCallConv(info.getCC());856 857 // Construct the function info. We co-allocate the ArgInfos.858 FI = CGFunctionInfo::create(CC, isInstanceMethod, isChainCall, isDelegateCall,859 info, paramInfos, resultType, argTypes, required);860 FunctionInfos.InsertNode(FI, insertPos);861 862 bool inserted = FunctionsBeingProcessed.insert(FI).second;863 (void)inserted;864 assert(inserted && "Recursively being processed?");865 866 // Compute ABI information.867 if (CC == llvm::CallingConv::SPIR_KERNEL) {868 // Force target independent argument handling for the host visible869 // kernel functions.870 computeSPIRKernelABIInfo(CGM, *FI);871 } else if (info.getCC() == CC_Swift || info.getCC() == CC_SwiftAsync) {872 swiftcall::computeABIInfo(CGM, *FI);873 } else {874 CGM.getABIInfo().computeInfo(*FI);875 }876 877 // Loop over all of the computed argument and return value info. If any of878 // them are direct or extend without a specified coerce type, specify the879 // default now.880 ABIArgInfo &retInfo = FI->getReturnInfo();881 if (retInfo.canHaveCoerceToType() && retInfo.getCoerceToType() == nullptr)882 retInfo.setCoerceToType(ConvertType(FI->getReturnType()));883 884 for (auto &I : FI->arguments())885 if (I.info.canHaveCoerceToType() && I.info.getCoerceToType() == nullptr)886 I.info.setCoerceToType(ConvertType(I.type));887 888 bool erased = FunctionsBeingProcessed.erase(FI);889 (void)erased;890 assert(erased && "Not in set?");891 892 return *FI;893}894 895CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC, bool instanceMethod,896 bool chainCall, bool delegateCall,897 const FunctionType::ExtInfo &info,898 ArrayRef<ExtParameterInfo> paramInfos,899 CanQualType resultType,900 ArrayRef<CanQualType> argTypes,901 RequiredArgs required) {902 assert(paramInfos.empty() || paramInfos.size() == argTypes.size());903 assert(!required.allowsOptionalArgs() ||904 required.getNumRequiredArgs() <= argTypes.size());905 906 void *buffer = operator new(totalSizeToAlloc<ArgInfo, ExtParameterInfo>(907 argTypes.size() + 1, paramInfos.size()));908 909 CGFunctionInfo *FI = new (buffer) CGFunctionInfo();910 FI->CallingConvention = llvmCC;911 FI->EffectiveCallingConvention = llvmCC;912 FI->ASTCallingConvention = info.getCC();913 FI->InstanceMethod = instanceMethod;914 FI->ChainCall = chainCall;915 FI->DelegateCall = delegateCall;916 FI->CmseNSCall = info.getCmseNSCall();917 FI->NoReturn = info.getNoReturn();918 FI->ReturnsRetained = info.getProducesResult();919 FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();920 FI->NoCfCheck = info.getNoCfCheck();921 FI->Required = required;922 FI->HasRegParm = info.getHasRegParm();923 FI->RegParm = info.getRegParm();924 FI->ArgStruct = nullptr;925 FI->ArgStructAlign = 0;926 FI->NumArgs = argTypes.size();927 FI->HasExtParameterInfos = !paramInfos.empty();928 FI->getArgsBuffer()[0].type = resultType;929 FI->MaxVectorWidth = 0;930 for (unsigned i = 0, e = argTypes.size(); i != e; ++i)931 FI->getArgsBuffer()[i + 1].type = argTypes[i];932 for (unsigned i = 0, e = paramInfos.size(); i != e; ++i)933 FI->getExtParameterInfosBuffer()[i] = paramInfos[i];934 return FI;935}936 937/***/938 939namespace {940// ABIArgInfo::Expand implementation.941 942// Specifies the way QualType passed as ABIArgInfo::Expand is expanded.943struct TypeExpansion {944 enum TypeExpansionKind {945 // Elements of constant arrays are expanded recursively.946 TEK_ConstantArray,947 // Record fields are expanded recursively (but if record is a union, only948 // the field with the largest size is expanded).949 TEK_Record,950 // For complex types, real and imaginary parts are expanded recursively.951 TEK_Complex,952 // All other types are not expandable.953 TEK_None954 };955 956 const TypeExpansionKind Kind;957 958 TypeExpansion(TypeExpansionKind K) : Kind(K) {}959 virtual ~TypeExpansion() {}960};961 962struct ConstantArrayExpansion : TypeExpansion {963 QualType EltTy;964 uint64_t NumElts;965 966 ConstantArrayExpansion(QualType EltTy, uint64_t NumElts)967 : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}968 static bool classof(const TypeExpansion *TE) {969 return TE->Kind == TEK_ConstantArray;970 }971};972 973struct RecordExpansion : TypeExpansion {974 SmallVector<const CXXBaseSpecifier *, 1> Bases;975 976 SmallVector<const FieldDecl *, 1> Fields;977 978 RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases,979 SmallVector<const FieldDecl *, 1> &&Fields)980 : TypeExpansion(TEK_Record), Bases(std::move(Bases)),981 Fields(std::move(Fields)) {}982 static bool classof(const TypeExpansion *TE) {983 return TE->Kind == TEK_Record;984 }985};986 987struct ComplexExpansion : TypeExpansion {988 QualType EltTy;989 990 ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {}991 static bool classof(const TypeExpansion *TE) {992 return TE->Kind == TEK_Complex;993 }994};995 996struct NoExpansion : TypeExpansion {997 NoExpansion() : TypeExpansion(TEK_None) {}998 static bool classof(const TypeExpansion *TE) { return TE->Kind == TEK_None; }999};1000} // namespace1001 1002static std::unique_ptr<TypeExpansion>1003getTypeExpansion(QualType Ty, const ASTContext &Context) {1004 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {1005 return std::make_unique<ConstantArrayExpansion>(AT->getElementType(),1006 AT->getZExtSize());1007 }1008 if (const auto *RD = Ty->getAsRecordDecl()) {1009 SmallVector<const CXXBaseSpecifier *, 1> Bases;1010 SmallVector<const FieldDecl *, 1> Fields;1011 assert(!RD->hasFlexibleArrayMember() &&1012 "Cannot expand structure with flexible array.");1013 if (RD->isUnion()) {1014 // Unions can be here only in degenerative cases - all the fields are same1015 // after flattening. Thus we have to use the "largest" field.1016 const FieldDecl *LargestFD = nullptr;1017 CharUnits UnionSize = CharUnits::Zero();1018 1019 for (const auto *FD : RD->fields()) {1020 if (FD->isZeroLengthBitField())1021 continue;1022 assert(!FD->isBitField() &&1023 "Cannot expand structure with bit-field members.");1024 CharUnits FieldSize = Context.getTypeSizeInChars(FD->getType());1025 if (UnionSize < FieldSize) {1026 UnionSize = FieldSize;1027 LargestFD = FD;1028 }1029 }1030 if (LargestFD)1031 Fields.push_back(LargestFD);1032 } else {1033 if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {1034 assert(!CXXRD->isDynamicClass() &&1035 "cannot expand vtable pointers in dynamic classes");1036 llvm::append_range(Bases, llvm::make_pointer_range(CXXRD->bases()));1037 }1038 1039 for (const auto *FD : RD->fields()) {1040 if (FD->isZeroLengthBitField())1041 continue;1042 assert(!FD->isBitField() &&1043 "Cannot expand structure with bit-field members.");1044 Fields.push_back(FD);1045 }1046 }1047 return std::make_unique<RecordExpansion>(std::move(Bases),1048 std::move(Fields));1049 }1050 if (const ComplexType *CT = Ty->getAs<ComplexType>()) {1051 return std::make_unique<ComplexExpansion>(CT->getElementType());1052 }1053 return std::make_unique<NoExpansion>();1054}1055 1056static int getExpansionSize(QualType Ty, const ASTContext &Context) {1057 auto Exp = getTypeExpansion(Ty, Context);1058 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {1059 return CAExp->NumElts * getExpansionSize(CAExp->EltTy, Context);1060 }1061 if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {1062 int Res = 0;1063 for (auto BS : RExp->Bases)1064 Res += getExpansionSize(BS->getType(), Context);1065 for (auto FD : RExp->Fields)1066 Res += getExpansionSize(FD->getType(), Context);1067 return Res;1068 }1069 if (isa<ComplexExpansion>(Exp.get()))1070 return 2;1071 assert(isa<NoExpansion>(Exp.get()));1072 return 1;1073}1074 1075void CodeGenTypes::getExpandedTypes(1076 QualType Ty, SmallVectorImpl<llvm::Type *>::iterator &TI) {1077 auto Exp = getTypeExpansion(Ty, Context);1078 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {1079 for (int i = 0, n = CAExp->NumElts; i < n; i++) {1080 getExpandedTypes(CAExp->EltTy, TI);1081 }1082 } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {1083 for (auto BS : RExp->Bases)1084 getExpandedTypes(BS->getType(), TI);1085 for (auto FD : RExp->Fields)1086 getExpandedTypes(FD->getType(), TI);1087 } else if (auto CExp = dyn_cast<ComplexExpansion>(Exp.get())) {1088 llvm::Type *EltTy = ConvertType(CExp->EltTy);1089 *TI++ = EltTy;1090 *TI++ = EltTy;1091 } else {1092 assert(isa<NoExpansion>(Exp.get()));1093 *TI++ = ConvertType(Ty);1094 }1095}1096 1097static void forConstantArrayExpansion(CodeGenFunction &CGF,1098 ConstantArrayExpansion *CAE,1099 Address BaseAddr,1100 llvm::function_ref<void(Address)> Fn) {1101 for (int i = 0, n = CAE->NumElts; i < n; i++) {1102 Address EltAddr = CGF.Builder.CreateConstGEP2_32(BaseAddr, 0, i);1103 Fn(EltAddr);1104 }1105}1106 1107void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,1108 llvm::Function::arg_iterator &AI) {1109 assert(LV.isSimple() &&1110 "Unexpected non-simple lvalue during struct expansion.");1111 1112 auto Exp = getTypeExpansion(Ty, getContext());1113 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {1114 forConstantArrayExpansion(1115 *this, CAExp, LV.getAddress(), [&](Address EltAddr) {1116 LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy);1117 ExpandTypeFromArgs(CAExp->EltTy, LV, AI);1118 });1119 } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {1120 Address This = LV.getAddress();1121 for (const CXXBaseSpecifier *BS : RExp->Bases) {1122 // Perform a single step derived-to-base conversion.1123 Address Base =1124 GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,1125 /*NullCheckValue=*/false, SourceLocation());1126 LValue SubLV = MakeAddrLValue(Base, BS->getType());1127 1128 // Recurse onto bases.1129 ExpandTypeFromArgs(BS->getType(), SubLV, AI);1130 }1131 for (auto FD : RExp->Fields) {1132 // FIXME: What are the right qualifiers here?1133 LValue SubLV = EmitLValueForFieldInitialization(LV, FD);1134 ExpandTypeFromArgs(FD->getType(), SubLV, AI);1135 }1136 } else if (isa<ComplexExpansion>(Exp.get())) {1137 auto realValue = &*AI++;1138 auto imagValue = &*AI++;1139 EmitStoreOfComplex(ComplexPairTy(realValue, imagValue), LV, /*init*/ true);1140 } else {1141 // Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a1142 // primitive store.1143 assert(isa<NoExpansion>(Exp.get()));1144 llvm::Value *Arg = &*AI++;1145 if (LV.isBitField()) {1146 EmitStoreThroughLValue(RValue::get(Arg), LV);1147 } else {1148 // TODO: currently there are some places are inconsistent in what LLVM1149 // pointer type they use (see D118744). Once clang uses opaque pointers1150 // all LLVM pointer types will be the same and we can remove this check.1151 if (Arg->getType()->isPointerTy()) {1152 Address Addr = LV.getAddress();1153 Arg = Builder.CreateBitCast(Arg, Addr.getElementType());1154 }1155 EmitStoreOfScalar(Arg, LV);1156 }1157 }1158}1159 1160void CodeGenFunction::ExpandTypeToArgs(1161 QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,1162 SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {1163 auto Exp = getTypeExpansion(Ty, getContext());1164 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {1165 Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress()1166 : Arg.getKnownRValue().getAggregateAddress();1167 forConstantArrayExpansion(*this, CAExp, Addr, [&](Address EltAddr) {1168 CallArg EltArg =1169 CallArg(convertTempToRValue(EltAddr, CAExp->EltTy, SourceLocation()),1170 CAExp->EltTy);1171 ExpandTypeToArgs(CAExp->EltTy, EltArg, IRFuncTy, IRCallArgs,1172 IRCallArgPos);1173 });1174 } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {1175 Address This = Arg.hasLValue() ? Arg.getKnownLValue().getAddress()1176 : Arg.getKnownRValue().getAggregateAddress();1177 for (const CXXBaseSpecifier *BS : RExp->Bases) {1178 // Perform a single step derived-to-base conversion.1179 Address Base =1180 GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,1181 /*NullCheckValue=*/false, SourceLocation());1182 CallArg BaseArg = CallArg(RValue::getAggregate(Base), BS->getType());1183 1184 // Recurse onto bases.1185 ExpandTypeToArgs(BS->getType(), BaseArg, IRFuncTy, IRCallArgs,1186 IRCallArgPos);1187 }1188 1189 LValue LV = MakeAddrLValue(This, Ty);1190 for (auto FD : RExp->Fields) {1191 CallArg FldArg =1192 CallArg(EmitRValueForField(LV, FD, SourceLocation()), FD->getType());1193 ExpandTypeToArgs(FD->getType(), FldArg, IRFuncTy, IRCallArgs,1194 IRCallArgPos);1195 }1196 } else if (isa<ComplexExpansion>(Exp.get())) {1197 ComplexPairTy CV = Arg.getKnownRValue().getComplexVal();1198 IRCallArgs[IRCallArgPos++] = CV.first;1199 IRCallArgs[IRCallArgPos++] = CV.second;1200 } else {1201 assert(isa<NoExpansion>(Exp.get()));1202 auto RV = Arg.getKnownRValue();1203 assert(RV.isScalar() &&1204 "Unexpected non-scalar rvalue during struct expansion.");1205 1206 // Insert a bitcast as needed.1207 llvm::Value *V = RV.getScalarVal();1208 if (IRCallArgPos < IRFuncTy->getNumParams() &&1209 V->getType() != IRFuncTy->getParamType(IRCallArgPos))1210 V = Builder.CreateBitCast(V, IRFuncTy->getParamType(IRCallArgPos));1211 1212 IRCallArgs[IRCallArgPos++] = V;1213 }1214}1215 1216/// Create a temporary allocation for the purposes of coercion.1217static RawAddress CreateTempAllocaForCoercion(CodeGenFunction &CGF,1218 llvm::Type *Ty,1219 CharUnits MinAlign,1220 const Twine &Name = "tmp") {1221 // Don't use an alignment that's worse than what LLVM would prefer.1222 auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlign(Ty);1223 CharUnits Align = std::max(MinAlign, CharUnits::fromQuantity(PrefAlign));1224 1225 return CGF.CreateTempAlloca(Ty, Align, Name + ".coerce");1226}1227 1228/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are1229/// accessing some number of bytes out of it, try to gep into the struct to get1230/// at its inner goodness. Dive as deep as possible without entering an element1231/// with an in-memory size smaller than DstSize.1232static Address EnterStructPointerForCoercedAccess(Address SrcPtr,1233 llvm::StructType *SrcSTy,1234 uint64_t DstSize,1235 CodeGenFunction &CGF) {1236 // We can't dive into a zero-element struct.1237 if (SrcSTy->getNumElements() == 0)1238 return SrcPtr;1239 1240 llvm::Type *FirstElt = SrcSTy->getElementType(0);1241 1242 // If the first elt is at least as large as what we're looking for, or if the1243 // first element is the same size as the whole struct, we can enter it. The1244 // comparison must be made on the store size and not the alloca size. Using1245 // the alloca size may overstate the size of the load.1246 uint64_t FirstEltSize = CGF.CGM.getDataLayout().getTypeStoreSize(FirstElt);1247 if (FirstEltSize < DstSize &&1248 FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(SrcSTy))1249 return SrcPtr;1250 1251 // GEP into the first element.1252 SrcPtr = CGF.Builder.CreateStructGEP(SrcPtr, 0, "coerce.dive");1253 1254 // If the first element is a struct, recurse.1255 llvm::Type *SrcTy = SrcPtr.getElementType();1256 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))1257 return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);1258 1259 return SrcPtr;1260}1261 1262/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both1263/// are either integers or pointers. This does a truncation of the value if it1264/// is too large or a zero extension if it is too small.1265///1266/// This behaves as if the value were coerced through memory, so on big-endian1267/// targets the high bits are preserved in a truncation, while little-endian1268/// targets preserve the low bits.1269static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val, llvm::Type *Ty,1270 CodeGenFunction &CGF) {1271 if (Val->getType() == Ty)1272 return Val;1273 1274 if (isa<llvm::PointerType>(Val->getType())) {1275 // If this is Pointer->Pointer avoid conversion to and from int.1276 if (isa<llvm::PointerType>(Ty))1277 return CGF.Builder.CreateBitCast(Val, Ty, "coerce.val");1278 1279 // Convert the pointer to an integer so we can play with its width.1280 Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi");1281 }1282 1283 llvm::Type *DestIntTy = Ty;1284 if (isa<llvm::PointerType>(DestIntTy))1285 DestIntTy = CGF.IntPtrTy;1286 1287 if (Val->getType() != DestIntTy) {1288 const llvm::DataLayout &DL = CGF.CGM.getDataLayout();1289 if (DL.isBigEndian()) {1290 // Preserve the high bits on big-endian targets.1291 // That is what memory coercion does.1292 uint64_t SrcSize = DL.getTypeSizeInBits(Val->getType());1293 uint64_t DstSize = DL.getTypeSizeInBits(DestIntTy);1294 1295 if (SrcSize > DstSize) {1296 Val = CGF.Builder.CreateLShr(Val, SrcSize - DstSize, "coerce.highbits");1297 Val = CGF.Builder.CreateTrunc(Val, DestIntTy, "coerce.val.ii");1298 } else {1299 Val = CGF.Builder.CreateZExt(Val, DestIntTy, "coerce.val.ii");1300 Val = CGF.Builder.CreateShl(Val, DstSize - SrcSize, "coerce.highbits");1301 }1302 } else {1303 // Little-endian targets preserve the low bits. No shifts required.1304 Val = CGF.Builder.CreateIntCast(Val, DestIntTy, false, "coerce.val.ii");1305 }1306 }1307 1308 if (isa<llvm::PointerType>(Ty))1309 Val = CGF.Builder.CreateIntToPtr(Val, Ty, "coerce.val.ip");1310 return Val;1311}1312 1313/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as1314/// a pointer to an object of type \arg Ty, known to be aligned to1315/// \arg SrcAlign bytes.1316///1317/// This safely handles the case when the src type is smaller than the1318/// destination type; in this situation the values of bits which not1319/// present in the src are undefined.1320static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,1321 CodeGenFunction &CGF) {1322 llvm::Type *SrcTy = Src.getElementType();1323 1324 // If SrcTy and Ty are the same, just do a load.1325 if (SrcTy == Ty)1326 return CGF.Builder.CreateLoad(Src);1327 1328 llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);1329 1330 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {1331 Src = EnterStructPointerForCoercedAccess(Src, SrcSTy,1332 DstSize.getFixedValue(), CGF);1333 SrcTy = Src.getElementType();1334 }1335 1336 llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);1337 1338 // If the source and destination are integer or pointer types, just do an1339 // extension or truncation to the desired type.1340 if ((isa<llvm::IntegerType>(Ty) || isa<llvm::PointerType>(Ty)) &&1341 (isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy))) {1342 llvm::Value *Load = CGF.Builder.CreateLoad(Src);1343 return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF);1344 }1345 1346 // If load is legal, just bitcast the src pointer.1347 if (!SrcSize.isScalable() && !DstSize.isScalable() &&1348 SrcSize.getFixedValue() >= DstSize.getFixedValue()) {1349 // Generally SrcSize is never greater than DstSize, since this means we are1350 // losing bits. However, this can happen in cases where the structure has1351 // additional padding, for example due to a user specified alignment.1352 //1353 // FIXME: Assert that we aren't truncating non-padding bits when have access1354 // to that information.1355 Src = Src.withElementType(Ty);1356 return CGF.Builder.CreateLoad(Src);1357 }1358 1359 // If coercing a fixed vector to a scalable vector for ABI compatibility, and1360 // the types match, use the llvm.vector.insert intrinsic to perform the1361 // conversion.1362 if (auto *ScalableDstTy = dyn_cast<llvm::ScalableVectorType>(Ty)) {1363 if (auto *FixedSrcTy = dyn_cast<llvm::FixedVectorType>(SrcTy)) {1364 // If we are casting a fixed i8 vector to a scalable i1 predicate1365 // vector, use a vector insert and bitcast the result.1366 if (ScalableDstTy->getElementType()->isIntegerTy(1) &&1367 FixedSrcTy->getElementType()->isIntegerTy(8)) {1368 ScalableDstTy = llvm::ScalableVectorType::get(1369 FixedSrcTy->getElementType(),1370 llvm::divideCeil(1371 ScalableDstTy->getElementCount().getKnownMinValue(), 8));1372 }1373 if (ScalableDstTy->getElementType() == FixedSrcTy->getElementType()) {1374 auto *Load = CGF.Builder.CreateLoad(Src);1375 auto *PoisonVec = llvm::PoisonValue::get(ScalableDstTy);1376 llvm::Value *Result = CGF.Builder.CreateInsertVector(1377 ScalableDstTy, PoisonVec, Load, uint64_t(0), "cast.scalable");1378 ScalableDstTy = cast<llvm::ScalableVectorType>(1379 llvm::VectorType::getWithSizeAndScalar(ScalableDstTy, Ty));1380 if (Result->getType() != ScalableDstTy)1381 Result = CGF.Builder.CreateBitCast(Result, ScalableDstTy);1382 if (Result->getType() != Ty)1383 Result = CGF.Builder.CreateExtractVector(Ty, Result, uint64_t(0));1384 return Result;1385 }1386 }1387 }1388 1389 // Otherwise do coercion through memory. This is stupid, but simple.1390 RawAddress Tmp =1391 CreateTempAllocaForCoercion(CGF, Ty, Src.getAlignment(), Src.getName());1392 CGF.Builder.CreateMemCpy(1393 Tmp.getPointer(), Tmp.getAlignment().getAsAlign(),1394 Src.emitRawPointer(CGF), Src.getAlignment().getAsAlign(),1395 llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize.getKnownMinValue()));1396 return CGF.Builder.CreateLoad(Tmp);1397}1398 1399void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, Address Dst,1400 llvm::TypeSize DstSize,1401 bool DstIsVolatile) {1402 if (!DstSize)1403 return;1404 1405 llvm::Type *SrcTy = Src->getType();1406 llvm::TypeSize SrcSize = CGM.getDataLayout().getTypeAllocSize(SrcTy);1407 1408 // GEP into structs to try to make types match.1409 // FIXME: This isn't really that useful with opaque types, but it impacts a1410 // lot of regression tests.1411 if (SrcTy != Dst.getElementType()) {1412 if (llvm::StructType *DstSTy =1413 dyn_cast<llvm::StructType>(Dst.getElementType())) {1414 assert(!SrcSize.isScalable());1415 Dst = EnterStructPointerForCoercedAccess(Dst, DstSTy,1416 SrcSize.getFixedValue(), *this);1417 }1418 }1419 1420 if (SrcSize.isScalable() || SrcSize <= DstSize) {1421 if (SrcTy->isIntegerTy() && Dst.getElementType()->isPointerTy() &&1422 SrcSize == CGM.getDataLayout().getTypeAllocSize(Dst.getElementType())) {1423 // If the value is supposed to be a pointer, convert it before storing it.1424 Src = CoerceIntOrPtrToIntOrPtr(Src, Dst.getElementType(), *this);1425 auto *I = Builder.CreateStore(Src, Dst, DstIsVolatile);1426 addInstToCurrentSourceAtom(I, Src);1427 } else if (llvm::StructType *STy =1428 dyn_cast<llvm::StructType>(Src->getType())) {1429 // Prefer scalar stores to first-class aggregate stores.1430 Dst = Dst.withElementType(SrcTy);1431 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {1432 Address EltPtr = Builder.CreateStructGEP(Dst, i);1433 llvm::Value *Elt = Builder.CreateExtractValue(Src, i);1434 auto *I = Builder.CreateStore(Elt, EltPtr, DstIsVolatile);1435 addInstToCurrentSourceAtom(I, Elt);1436 }1437 } else {1438 auto *I =1439 Builder.CreateStore(Src, Dst.withElementType(SrcTy), DstIsVolatile);1440 addInstToCurrentSourceAtom(I, Src);1441 }1442 } else if (SrcTy->isIntegerTy()) {1443 // If the source is a simple integer, coerce it directly.1444 llvm::Type *DstIntTy = Builder.getIntNTy(DstSize.getFixedValue() * 8);1445 Src = CoerceIntOrPtrToIntOrPtr(Src, DstIntTy, *this);1446 auto *I =1447 Builder.CreateStore(Src, Dst.withElementType(DstIntTy), DstIsVolatile);1448 addInstToCurrentSourceAtom(I, Src);1449 } else {1450 // Otherwise do coercion through memory. This is stupid, but1451 // simple.1452 1453 // Generally SrcSize is never greater than DstSize, since this means we are1454 // losing bits. However, this can happen in cases where the structure has1455 // additional padding, for example due to a user specified alignment.1456 //1457 // FIXME: Assert that we aren't truncating non-padding bits when have access1458 // to that information.1459 RawAddress Tmp =1460 CreateTempAllocaForCoercion(*this, SrcTy, Dst.getAlignment());1461 Builder.CreateStore(Src, Tmp);1462 auto *I = Builder.CreateMemCpy(1463 Dst.emitRawPointer(*this), Dst.getAlignment().getAsAlign(),1464 Tmp.getPointer(), Tmp.getAlignment().getAsAlign(),1465 Builder.CreateTypeSize(IntPtrTy, DstSize));1466 addInstToCurrentSourceAtom(I, Src);1467 }1468}1469 1470static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr,1471 const ABIArgInfo &info) {1472 if (unsigned offset = info.getDirectOffset()) {1473 addr = addr.withElementType(CGF.Int8Ty);1474 addr = CGF.Builder.CreateConstInBoundsByteGEP(1475 addr, CharUnits::fromQuantity(offset));1476 addr = addr.withElementType(info.getCoerceToType());1477 }1478 return addr;1479}1480 1481static std::pair<llvm::Value *, bool>1482CoerceScalableToFixed(CodeGenFunction &CGF, llvm::FixedVectorType *ToTy,1483 llvm::ScalableVectorType *FromTy, llvm::Value *V,1484 StringRef Name = "") {1485 // If we are casting a scalable i1 predicate vector to a fixed i81486 // vector, first bitcast the source.1487 if (FromTy->getElementType()->isIntegerTy(1) &&1488 ToTy->getElementType() == CGF.Builder.getInt8Ty()) {1489 if (!FromTy->getElementCount().isKnownMultipleOf(8)) {1490 FromTy = llvm::ScalableVectorType::get(1491 FromTy->getElementType(),1492 llvm::alignTo<8>(FromTy->getElementCount().getKnownMinValue()));1493 llvm::Value *ZeroVec = llvm::Constant::getNullValue(FromTy);1494 V = CGF.Builder.CreateInsertVector(FromTy, ZeroVec, V, uint64_t(0));1495 }1496 FromTy = llvm::ScalableVectorType::get(1497 ToTy->getElementType(),1498 FromTy->getElementCount().getKnownMinValue() / 8);1499 V = CGF.Builder.CreateBitCast(V, FromTy);1500 }1501 if (FromTy->getElementType() == ToTy->getElementType()) {1502 V->setName(Name + ".coerce");1503 V = CGF.Builder.CreateExtractVector(ToTy, V, uint64_t(0), "cast.fixed");1504 return {V, true};1505 }1506 return {V, false};1507}1508 1509namespace {1510 1511/// Encapsulates information about the way function arguments from1512/// CGFunctionInfo should be passed to actual LLVM IR function.1513class ClangToLLVMArgMapping {1514 static const unsigned InvalidIndex = ~0U;1515 unsigned InallocaArgNo;1516 unsigned SRetArgNo;1517 unsigned TotalIRArgs;1518 1519 /// Arguments of LLVM IR function corresponding to single Clang argument.1520 struct IRArgs {1521 unsigned PaddingArgIndex;1522 // Argument is expanded to IR arguments at positions1523 // [FirstArgIndex, FirstArgIndex + NumberOfArgs).1524 unsigned FirstArgIndex;1525 unsigned NumberOfArgs;1526 1527 IRArgs()1528 : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),1529 NumberOfArgs(0) {}1530 };1531 1532 SmallVector<IRArgs, 8> ArgInfo;1533 1534public:1535 ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI,1536 bool OnlyRequiredArgs = false)1537 : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),1538 ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {1539 construct(Context, FI, OnlyRequiredArgs);1540 }1541 1542 bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }1543 unsigned getInallocaArgNo() const {1544 assert(hasInallocaArg());1545 return InallocaArgNo;1546 }1547 1548 bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }1549 unsigned getSRetArgNo() const {1550 assert(hasSRetArg());1551 return SRetArgNo;1552 }1553 1554 unsigned totalIRArgs() const { return TotalIRArgs; }1555 1556 bool hasPaddingArg(unsigned ArgNo) const {1557 assert(ArgNo < ArgInfo.size());1558 return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;1559 }1560 unsigned getPaddingArgNo(unsigned ArgNo) const {1561 assert(hasPaddingArg(ArgNo));1562 return ArgInfo[ArgNo].PaddingArgIndex;1563 }1564 1565 /// Returns index of first IR argument corresponding to ArgNo, and their1566 /// quantity.1567 std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {1568 assert(ArgNo < ArgInfo.size());1569 return std::make_pair(ArgInfo[ArgNo].FirstArgIndex,1570 ArgInfo[ArgNo].NumberOfArgs);1571 }1572 1573private:1574 void construct(const ASTContext &Context, const CGFunctionInfo &FI,1575 bool OnlyRequiredArgs);1576};1577 1578void ClangToLLVMArgMapping::construct(const ASTContext &Context,1579 const CGFunctionInfo &FI,1580 bool OnlyRequiredArgs) {1581 unsigned IRArgNo = 0;1582 bool SwapThisWithSRet = false;1583 const ABIArgInfo &RetAI = FI.getReturnInfo();1584 1585 if (RetAI.getKind() == ABIArgInfo::Indirect) {1586 SwapThisWithSRet = RetAI.isSRetAfterThis();1587 SRetArgNo = SwapThisWithSRet ? 1 : IRArgNo++;1588 }1589 1590 unsigned ArgNo = 0;1591 unsigned NumArgs = OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size();1592 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs;1593 ++I, ++ArgNo) {1594 assert(I != FI.arg_end());1595 QualType ArgType = I->type;1596 const ABIArgInfo &AI = I->info;1597 // Collect data about IR arguments corresponding to Clang argument ArgNo.1598 auto &IRArgs = ArgInfo[ArgNo];1599 1600 if (AI.getPaddingType())1601 IRArgs.PaddingArgIndex = IRArgNo++;1602 1603 switch (AI.getKind()) {1604 case ABIArgInfo::TargetSpecific:1605 case ABIArgInfo::Extend:1606 case ABIArgInfo::Direct: {1607 // FIXME: handle sseregparm someday...1608 llvm::StructType *STy = dyn_cast<llvm::StructType>(AI.getCoerceToType());1609 if (AI.isDirect() && AI.getCanBeFlattened() && STy) {1610 IRArgs.NumberOfArgs = STy->getNumElements();1611 } else {1612 IRArgs.NumberOfArgs = 1;1613 }1614 break;1615 }1616 case ABIArgInfo::Indirect:1617 case ABIArgInfo::IndirectAliased:1618 IRArgs.NumberOfArgs = 1;1619 break;1620 case ABIArgInfo::Ignore:1621 case ABIArgInfo::InAlloca:1622 // ignore and inalloca doesn't have matching LLVM parameters.1623 IRArgs.NumberOfArgs = 0;1624 break;1625 case ABIArgInfo::CoerceAndExpand:1626 IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size();1627 break;1628 case ABIArgInfo::Expand:1629 IRArgs.NumberOfArgs = getExpansionSize(ArgType, Context);1630 break;1631 }1632 1633 if (IRArgs.NumberOfArgs > 0) {1634 IRArgs.FirstArgIndex = IRArgNo;1635 IRArgNo += IRArgs.NumberOfArgs;1636 }1637 1638 // Skip over the sret parameter when it comes second. We already handled it1639 // above.1640 if (IRArgNo == 1 && SwapThisWithSRet)1641 IRArgNo++;1642 }1643 assert(ArgNo == ArgInfo.size());1644 1645 if (FI.usesInAlloca())1646 InallocaArgNo = IRArgNo++;1647 1648 TotalIRArgs = IRArgNo;1649}1650} // namespace1651 1652/***/1653 1654bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {1655 const auto &RI = FI.getReturnInfo();1656 return RI.isIndirect() || (RI.isInAlloca() && RI.getInAllocaSRet());1657}1658 1659bool CodeGenModule::ReturnTypeHasInReg(const CGFunctionInfo &FI) {1660 const auto &RI = FI.getReturnInfo();1661 return RI.getInReg();1662}1663 1664bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) {1665 return ReturnTypeUsesSRet(FI) &&1666 getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs();1667}1668 1669bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {1670 if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {1671 switch (BT->getKind()) {1672 default:1673 return false;1674 case BuiltinType::Float:1675 return getTarget().useObjCFPRetForRealType(FloatModeKind::Float);1676 case BuiltinType::Double:1677 return getTarget().useObjCFPRetForRealType(FloatModeKind::Double);1678 case BuiltinType::LongDouble:1679 return getTarget().useObjCFPRetForRealType(FloatModeKind::LongDouble);1680 }1681 }1682 1683 return false;1684}1685 1686bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {1687 if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {1688 if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {1689 if (BT->getKind() == BuiltinType::LongDouble)1690 return getTarget().useObjCFP2RetForComplexLongDouble();1691 }1692 }1693 1694 return false;1695}1696 1697llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {1698 const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD);1699 return GetFunctionType(FI);1700}1701 1702llvm::FunctionType *CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {1703 1704 bool Inserted = FunctionsBeingProcessed.insert(&FI).second;1705 (void)Inserted;1706 assert(Inserted && "Recursively being processed?");1707 1708 llvm::Type *resultType = nullptr;1709 const ABIArgInfo &retAI = FI.getReturnInfo();1710 switch (retAI.getKind()) {1711 case ABIArgInfo::Expand:1712 case ABIArgInfo::IndirectAliased:1713 llvm_unreachable("Invalid ABI kind for return argument");1714 1715 case ABIArgInfo::TargetSpecific:1716 case ABIArgInfo::Extend:1717 case ABIArgInfo::Direct:1718 resultType = retAI.getCoerceToType();1719 break;1720 1721 case ABIArgInfo::InAlloca:1722 if (retAI.getInAllocaSRet()) {1723 // sret things on win32 aren't void, they return the sret pointer.1724 QualType ret = FI.getReturnType();1725 unsigned addressSpace = CGM.getTypes().getTargetAddressSpace(ret);1726 resultType = llvm::PointerType::get(getLLVMContext(), addressSpace);1727 } else {1728 resultType = llvm::Type::getVoidTy(getLLVMContext());1729 }1730 break;1731 1732 case ABIArgInfo::Indirect:1733 case ABIArgInfo::Ignore:1734 resultType = llvm::Type::getVoidTy(getLLVMContext());1735 break;1736 1737 case ABIArgInfo::CoerceAndExpand:1738 resultType = retAI.getUnpaddedCoerceAndExpandType();1739 break;1740 }1741 1742 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true);1743 SmallVector<llvm::Type *, 8> ArgTypes(IRFunctionArgs.totalIRArgs());1744 1745 // Add type for sret argument.1746 if (IRFunctionArgs.hasSRetArg()) {1747 ArgTypes[IRFunctionArgs.getSRetArgNo()] = llvm::PointerType::get(1748 getLLVMContext(), FI.getReturnInfo().getIndirectAddrSpace());1749 }1750 1751 // Add type for inalloca argument.1752 if (IRFunctionArgs.hasInallocaArg())1753 ArgTypes[IRFunctionArgs.getInallocaArgNo()] =1754 llvm::PointerType::getUnqual(getLLVMContext());1755 1756 // Add in all of the required arguments.1757 unsigned ArgNo = 0;1758 CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),1759 ie = it + FI.getNumRequiredArgs();1760 for (; it != ie; ++it, ++ArgNo) {1761 const ABIArgInfo &ArgInfo = it->info;1762 1763 // Insert a padding type to ensure proper alignment.1764 if (IRFunctionArgs.hasPaddingArg(ArgNo))1765 ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =1766 ArgInfo.getPaddingType();1767 1768 unsigned FirstIRArg, NumIRArgs;1769 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);1770 1771 switch (ArgInfo.getKind()) {1772 case ABIArgInfo::Ignore:1773 case ABIArgInfo::InAlloca:1774 assert(NumIRArgs == 0);1775 break;1776 1777 case ABIArgInfo::Indirect:1778 assert(NumIRArgs == 1);1779 // indirect arguments are always on the stack, which is alloca addr space.1780 ArgTypes[FirstIRArg] = llvm::PointerType::get(1781 getLLVMContext(), CGM.getDataLayout().getAllocaAddrSpace());1782 break;1783 case ABIArgInfo::IndirectAliased:1784 assert(NumIRArgs == 1);1785 ArgTypes[FirstIRArg] = llvm::PointerType::get(1786 getLLVMContext(), ArgInfo.getIndirectAddrSpace());1787 break;1788 case ABIArgInfo::TargetSpecific:1789 case ABIArgInfo::Extend:1790 case ABIArgInfo::Direct: {1791 // Fast-isel and the optimizer generally like scalar values better than1792 // FCAs, so we flatten them if this is safe to do for this argument.1793 llvm::Type *argType = ArgInfo.getCoerceToType();1794 llvm::StructType *st = dyn_cast<llvm::StructType>(argType);1795 if (st && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {1796 assert(NumIRArgs == st->getNumElements());1797 for (unsigned i = 0, e = st->getNumElements(); i != e; ++i)1798 ArgTypes[FirstIRArg + i] = st->getElementType(i);1799 } else {1800 assert(NumIRArgs == 1);1801 ArgTypes[FirstIRArg] = argType;1802 }1803 break;1804 }1805 1806 case ABIArgInfo::CoerceAndExpand: {1807 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;1808 for (auto *EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) {1809 *ArgTypesIter++ = EltTy;1810 }1811 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);1812 break;1813 }1814 1815 case ABIArgInfo::Expand:1816 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;1817 getExpandedTypes(it->type, ArgTypesIter);1818 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);1819 break;1820 }1821 }1822 1823 bool Erased = FunctionsBeingProcessed.erase(&FI);1824 (void)Erased;1825 assert(Erased && "Not in set?");1826 1827 return llvm::FunctionType::get(resultType, ArgTypes, FI.isVariadic());1828}1829 1830llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {1831 const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());1832 const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();1833 1834 if (!isFuncTypeConvertible(FPT))1835 return llvm::StructType::get(getLLVMContext());1836 1837 return GetFunctionType(GD);1838}1839 1840static void AddAttributesFromFunctionProtoType(ASTContext &Ctx,1841 llvm::AttrBuilder &FuncAttrs,1842 const FunctionProtoType *FPT) {1843 if (!FPT)1844 return;1845 1846 if (!isUnresolvedExceptionSpec(FPT->getExceptionSpecType()) &&1847 FPT->isNothrow())1848 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);1849 1850 unsigned SMEBits = FPT->getAArch64SMEAttributes();1851 if (SMEBits & FunctionType::SME_PStateSMEnabledMask)1852 FuncAttrs.addAttribute("aarch64_pstate_sm_enabled");1853 if (SMEBits & FunctionType::SME_PStateSMCompatibleMask)1854 FuncAttrs.addAttribute("aarch64_pstate_sm_compatible");1855 if (SMEBits & FunctionType::SME_AgnosticZAStateMask)1856 FuncAttrs.addAttribute("aarch64_za_state_agnostic");1857 1858 // ZA1859 if (FunctionType::getArmZAState(SMEBits) == FunctionType::ARM_Preserves)1860 FuncAttrs.addAttribute("aarch64_preserves_za");1861 if (FunctionType::getArmZAState(SMEBits) == FunctionType::ARM_In)1862 FuncAttrs.addAttribute("aarch64_in_za");1863 if (FunctionType::getArmZAState(SMEBits) == FunctionType::ARM_Out)1864 FuncAttrs.addAttribute("aarch64_out_za");1865 if (FunctionType::getArmZAState(SMEBits) == FunctionType::ARM_InOut)1866 FuncAttrs.addAttribute("aarch64_inout_za");1867 1868 // ZT01869 if (FunctionType::getArmZT0State(SMEBits) == FunctionType::ARM_Preserves)1870 FuncAttrs.addAttribute("aarch64_preserves_zt0");1871 if (FunctionType::getArmZT0State(SMEBits) == FunctionType::ARM_In)1872 FuncAttrs.addAttribute("aarch64_in_zt0");1873 if (FunctionType::getArmZT0State(SMEBits) == FunctionType::ARM_Out)1874 FuncAttrs.addAttribute("aarch64_out_zt0");1875 if (FunctionType::getArmZT0State(SMEBits) == FunctionType::ARM_InOut)1876 FuncAttrs.addAttribute("aarch64_inout_zt0");1877}1878 1879static void AddAttributesFromOMPAssumes(llvm::AttrBuilder &FuncAttrs,1880 const Decl *Callee) {1881 if (!Callee)1882 return;1883 1884 SmallVector<StringRef, 4> Attrs;1885 1886 for (const OMPAssumeAttr *AA : Callee->specific_attrs<OMPAssumeAttr>())1887 AA->getAssumption().split(Attrs, ",");1888 1889 if (!Attrs.empty())1890 FuncAttrs.addAttribute(llvm::AssumptionAttrKey,1891 llvm::join(Attrs.begin(), Attrs.end(), ","));1892}1893 1894bool CodeGenModule::MayDropFunctionReturn(const ASTContext &Context,1895 QualType ReturnType) const {1896 // We can't just discard the return value for a record type with a1897 // complex destructor or a non-trivially copyable type.1898 if (const RecordType *RT =1899 ReturnType.getCanonicalType()->getAsCanonical<RecordType>()) {1900 if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl()))1901 return ClassDecl->hasTrivialDestructor();1902 }1903 return ReturnType.isTriviallyCopyableType(Context);1904}1905 1906static bool HasStrictReturn(const CodeGenModule &Module, QualType RetTy,1907 const Decl *TargetDecl) {1908 // As-is msan can not tolerate noundef mismatch between caller and1909 // implementation. Mismatch is possible for e.g. indirect calls from C-caller1910 // into C++. Such mismatches lead to confusing false reports. To avoid1911 // expensive workaround on msan we enforce initialization event in uncommon1912 // cases where it's allowed.1913 if (Module.getLangOpts().Sanitize.has(SanitizerKind::Memory))1914 return true;1915 // C++ explicitly makes returning undefined values UB. C's rule only applies1916 // to used values, so we never mark them noundef for now.1917 if (!Module.getLangOpts().CPlusPlus)1918 return false;1919 if (TargetDecl) {1920 if (const FunctionDecl *FDecl = dyn_cast<FunctionDecl>(TargetDecl)) {1921 if (FDecl->isExternC())1922 return false;1923 } else if (const VarDecl *VDecl = dyn_cast<VarDecl>(TargetDecl)) {1924 // Function pointer.1925 if (VDecl->isExternC())1926 return false;1927 }1928 }1929 1930 // We don't want to be too aggressive with the return checking, unless1931 // it's explicit in the code opts or we're using an appropriate sanitizer.1932 // Try to respect what the programmer intended.1933 return Module.getCodeGenOpts().StrictReturn ||1934 !Module.MayDropFunctionReturn(Module.getContext(), RetTy) ||1935 Module.getLangOpts().Sanitize.has(SanitizerKind::Return);1936}1937 1938/// Add denormal-fp-math and denormal-fp-math-f32 as appropriate for the1939/// requested denormal behavior, accounting for the overriding behavior of the1940/// -f32 case.1941static void addDenormalModeAttrs(llvm::DenormalMode FPDenormalMode,1942 llvm::DenormalMode FP32DenormalMode,1943 llvm::AttrBuilder &FuncAttrs) {1944 if (FPDenormalMode != llvm::DenormalMode::getDefault())1945 FuncAttrs.addAttribute("denormal-fp-math", FPDenormalMode.str());1946 1947 if (FP32DenormalMode != FPDenormalMode && FP32DenormalMode.isValid())1948 FuncAttrs.addAttribute("denormal-fp-math-f32", FP32DenormalMode.str());1949}1950 1951/// Add default attributes to a function, which have merge semantics under1952/// -mlink-builtin-bitcode and should not simply overwrite any existing1953/// attributes in the linked library.1954static void1955addMergableDefaultFunctionAttributes(const CodeGenOptions &CodeGenOpts,1956 llvm::AttrBuilder &FuncAttrs) {1957 addDenormalModeAttrs(CodeGenOpts.FPDenormalMode, CodeGenOpts.FP32DenormalMode,1958 FuncAttrs);1959}1960 1961static void getTrivialDefaultFunctionAttributes(1962 StringRef Name, bool HasOptnone, const CodeGenOptions &CodeGenOpts,1963 const LangOptions &LangOpts, bool AttrOnCallSite,1964 llvm::AttrBuilder &FuncAttrs) {1965 // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.1966 if (!HasOptnone) {1967 if (CodeGenOpts.OptimizeSize)1968 FuncAttrs.addAttribute(llvm::Attribute::OptimizeForSize);1969 if (CodeGenOpts.OptimizeSize == 2)1970 FuncAttrs.addAttribute(llvm::Attribute::MinSize);1971 }1972 1973 if (CodeGenOpts.DisableRedZone)1974 FuncAttrs.addAttribute(llvm::Attribute::NoRedZone);1975 if (CodeGenOpts.IndirectTlsSegRefs)1976 FuncAttrs.addAttribute("indirect-tls-seg-refs");1977 if (CodeGenOpts.NoImplicitFloat)1978 FuncAttrs.addAttribute(llvm::Attribute::NoImplicitFloat);1979 1980 if (AttrOnCallSite) {1981 // Attributes that should go on the call site only.1982 // FIXME: Look for 'BuiltinAttr' on the function rather than re-checking1983 // the -fno-builtin-foo list.1984 if (!CodeGenOpts.SimplifyLibCalls || LangOpts.isNoBuiltinFunc(Name))1985 FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin);1986 if (!CodeGenOpts.TrapFuncName.empty())1987 FuncAttrs.addAttribute("trap-func-name", CodeGenOpts.TrapFuncName);1988 } else {1989 switch (CodeGenOpts.getFramePointer()) {1990 case CodeGenOptions::FramePointerKind::None:1991 // This is the default behavior.1992 break;1993 case CodeGenOptions::FramePointerKind::Reserved:1994 case CodeGenOptions::FramePointerKind::NonLeafNoReserve:1995 case CodeGenOptions::FramePointerKind::NonLeaf:1996 case CodeGenOptions::FramePointerKind::All:1997 FuncAttrs.addAttribute("frame-pointer",1998 CodeGenOptions::getFramePointerKindName(1999 CodeGenOpts.getFramePointer()));2000 }2001 2002 if (CodeGenOpts.LessPreciseFPMAD)2003 FuncAttrs.addAttribute("less-precise-fpmad", "true");2004 2005 if (CodeGenOpts.NullPointerIsValid)2006 FuncAttrs.addAttribute(llvm::Attribute::NullPointerIsValid);2007 2008 if (LangOpts.getDefaultExceptionMode() == LangOptions::FPE_Ignore)2009 FuncAttrs.addAttribute("no-trapping-math", "true");2010 2011 // TODO: Are these all needed?2012 // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.2013 if (LangOpts.NoHonorInfs)2014 FuncAttrs.addAttribute("no-infs-fp-math", "true");2015 if (LangOpts.NoHonorNaNs)2016 FuncAttrs.addAttribute("no-nans-fp-math", "true");2017 if (CodeGenOpts.SoftFloat)2018 FuncAttrs.addAttribute("use-soft-float", "true");2019 FuncAttrs.addAttribute("stack-protector-buffer-size",2020 llvm::utostr(CodeGenOpts.SSPBufferSize));2021 if (LangOpts.NoSignedZero)2022 FuncAttrs.addAttribute("no-signed-zeros-fp-math", "true");2023 2024 // TODO: Reciprocal estimate codegen options should apply to instructions?2025 const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals;2026 if (!Recips.empty())2027 FuncAttrs.addAttribute("reciprocal-estimates", llvm::join(Recips, ","));2028 2029 if (!CodeGenOpts.PreferVectorWidth.empty() &&2030 CodeGenOpts.PreferVectorWidth != "none")2031 FuncAttrs.addAttribute("prefer-vector-width",2032 CodeGenOpts.PreferVectorWidth);2033 2034 if (CodeGenOpts.StackRealignment)2035 FuncAttrs.addAttribute("stackrealign");2036 if (CodeGenOpts.Backchain)2037 FuncAttrs.addAttribute("backchain");2038 if (CodeGenOpts.EnableSegmentedStacks)2039 FuncAttrs.addAttribute("split-stack");2040 2041 if (CodeGenOpts.SpeculativeLoadHardening)2042 FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);2043 2044 // Add zero-call-used-regs attribute.2045 switch (CodeGenOpts.getZeroCallUsedRegs()) {2046 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Skip:2047 FuncAttrs.removeAttribute("zero-call-used-regs");2048 break;2049 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPRArg:2050 FuncAttrs.addAttribute("zero-call-used-regs", "used-gpr-arg");2051 break;2052 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPR:2053 FuncAttrs.addAttribute("zero-call-used-regs", "used-gpr");2054 break;2055 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedArg:2056 FuncAttrs.addAttribute("zero-call-used-regs", "used-arg");2057 break;2058 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Used:2059 FuncAttrs.addAttribute("zero-call-used-regs", "used");2060 break;2061 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPRArg:2062 FuncAttrs.addAttribute("zero-call-used-regs", "all-gpr-arg");2063 break;2064 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPR:2065 FuncAttrs.addAttribute("zero-call-used-regs", "all-gpr");2066 break;2067 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllArg:2068 FuncAttrs.addAttribute("zero-call-used-regs", "all-arg");2069 break;2070 case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::All:2071 FuncAttrs.addAttribute("zero-call-used-regs", "all");2072 break;2073 }2074 }2075 2076 if (LangOpts.assumeFunctionsAreConvergent()) {2077 // Conservatively, mark all functions and calls in CUDA and OpenCL as2078 // convergent (meaning, they may call an intrinsically convergent op, such2079 // as __syncthreads() / barrier(), and so can't have certain optimizations2080 // applied around them). LLVM will remove this attribute where it safely2081 // can.2082 FuncAttrs.addAttribute(llvm::Attribute::Convergent);2083 }2084 2085 // TODO: NoUnwind attribute should be added for other GPU modes HIP,2086 // OpenMP offload. AFAIK, neither of them support exceptions in device code.2087 if ((LangOpts.CUDA && LangOpts.CUDAIsDevice) || LangOpts.OpenCL ||2088 LangOpts.SYCLIsDevice) {2089 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);2090 }2091 2092 if (CodeGenOpts.SaveRegParams && !AttrOnCallSite)2093 FuncAttrs.addAttribute("save-reg-params");2094 2095 for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) {2096 StringRef Var, Value;2097 std::tie(Var, Value) = Attr.split('=');2098 FuncAttrs.addAttribute(Var, Value);2099 }2100 2101 TargetInfo::BranchProtectionInfo BPI(LangOpts);2102 TargetCodeGenInfo::initBranchProtectionFnAttributes(BPI, FuncAttrs);2103}2104 2105/// Merges `target-features` from \TargetOpts and \F, and sets the result in2106/// \FuncAttr2107/// * features from \F are always kept2108/// * a feature from \TargetOpts is kept if itself and its opposite are absent2109/// from \F2110static void2111overrideFunctionFeaturesWithTargetFeatures(llvm::AttrBuilder &FuncAttr,2112 const llvm::Function &F,2113 const TargetOptions &TargetOpts) {2114 auto FFeatures = F.getFnAttribute("target-features");2115 2116 llvm::StringSet<> MergedNames;2117 SmallVector<StringRef> MergedFeatures;2118 MergedFeatures.reserve(TargetOpts.Features.size());2119 2120 auto AddUnmergedFeatures = [&](auto &&FeatureRange) {2121 for (StringRef Feature : FeatureRange) {2122 if (Feature.empty())2123 continue;2124 assert(Feature[0] == '+' || Feature[0] == '-');2125 StringRef Name = Feature.drop_front(1);2126 bool Merged = !MergedNames.insert(Name).second;2127 if (!Merged)2128 MergedFeatures.push_back(Feature);2129 }2130 };2131 2132 if (FFeatures.isValid())2133 AddUnmergedFeatures(llvm::split(FFeatures.getValueAsString(), ','));2134 AddUnmergedFeatures(TargetOpts.Features);2135 2136 if (!MergedFeatures.empty()) {2137 llvm::sort(MergedFeatures);2138 FuncAttr.addAttribute("target-features", llvm::join(MergedFeatures, ","));2139 }2140}2141 2142void CodeGen::mergeDefaultFunctionDefinitionAttributes(2143 llvm::Function &F, const CodeGenOptions &CodeGenOpts,2144 const LangOptions &LangOpts, const TargetOptions &TargetOpts,2145 bool WillInternalize) {2146 2147 llvm::AttrBuilder FuncAttrs(F.getContext());2148 // Here we only extract the options that are relevant compared to the version2149 // from GetCPUAndFeaturesAttributes.2150 if (!TargetOpts.CPU.empty())2151 FuncAttrs.addAttribute("target-cpu", TargetOpts.CPU);2152 if (!TargetOpts.TuneCPU.empty())2153 FuncAttrs.addAttribute("tune-cpu", TargetOpts.TuneCPU);2154 2155 ::getTrivialDefaultFunctionAttributes(F.getName(), F.hasOptNone(),2156 CodeGenOpts, LangOpts,2157 /*AttrOnCallSite=*/false, FuncAttrs);2158 2159 if (!WillInternalize && F.isInterposable()) {2160 // Do not promote "dynamic" denormal-fp-math to this translation unit's2161 // setting for weak functions that won't be internalized. The user has no2162 // real control for how builtin bitcode is linked, so we shouldn't assume2163 // later copies will use a consistent mode.2164 F.addFnAttrs(FuncAttrs);2165 return;2166 }2167 2168 llvm::AttributeMask AttrsToRemove;2169 2170 llvm::DenormalMode DenormModeToMerge = F.getDenormalModeRaw();2171 llvm::DenormalMode DenormModeToMergeF32 = F.getDenormalModeF32Raw();2172 llvm::DenormalMode Merged =2173 CodeGenOpts.FPDenormalMode.mergeCalleeMode(DenormModeToMerge);2174 llvm::DenormalMode MergedF32 = CodeGenOpts.FP32DenormalMode;2175 2176 if (DenormModeToMergeF32.isValid()) {2177 MergedF32 =2178 CodeGenOpts.FP32DenormalMode.mergeCalleeMode(DenormModeToMergeF32);2179 }2180 2181 if (Merged == llvm::DenormalMode::getDefault()) {2182 AttrsToRemove.addAttribute("denormal-fp-math");2183 } else if (Merged != DenormModeToMerge) {2184 // Overwrite existing attribute2185 FuncAttrs.addAttribute("denormal-fp-math",2186 CodeGenOpts.FPDenormalMode.str());2187 }2188 2189 if (MergedF32 == llvm::DenormalMode::getDefault()) {2190 AttrsToRemove.addAttribute("denormal-fp-math-f32");2191 } else if (MergedF32 != DenormModeToMergeF32) {2192 // Overwrite existing attribute2193 FuncAttrs.addAttribute("denormal-fp-math-f32",2194 CodeGenOpts.FP32DenormalMode.str());2195 }2196 2197 F.removeFnAttrs(AttrsToRemove);2198 addDenormalModeAttrs(Merged, MergedF32, FuncAttrs);2199 2200 overrideFunctionFeaturesWithTargetFeatures(FuncAttrs, F, TargetOpts);2201 2202 F.addFnAttrs(FuncAttrs);2203}2204 2205void CodeGenModule::getTrivialDefaultFunctionAttributes(2206 StringRef Name, bool HasOptnone, bool AttrOnCallSite,2207 llvm::AttrBuilder &FuncAttrs) {2208 ::getTrivialDefaultFunctionAttributes(Name, HasOptnone, getCodeGenOpts(),2209 getLangOpts(), AttrOnCallSite,2210 FuncAttrs);2211}2212 2213void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,2214 bool HasOptnone,2215 bool AttrOnCallSite,2216 llvm::AttrBuilder &FuncAttrs) {2217 getTrivialDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite,2218 FuncAttrs);2219 2220 if (!AttrOnCallSite)2221 TargetCodeGenInfo::initPointerAuthFnAttributes(CodeGenOpts.PointerAuth,2222 FuncAttrs);2223 2224 // If we're just getting the default, get the default values for mergeable2225 // attributes.2226 if (!AttrOnCallSite)2227 addMergableDefaultFunctionAttributes(CodeGenOpts, FuncAttrs);2228}2229 2230void CodeGenModule::addDefaultFunctionDefinitionAttributes(2231 llvm::AttrBuilder &attrs) {2232 getDefaultFunctionAttributes(/*function name*/ "", /*optnone*/ false,2233 /*for call*/ false, attrs);2234 GetCPUAndFeaturesAttributes(GlobalDecl(), attrs);2235}2236 2237static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,2238 const LangOptions &LangOpts,2239 const NoBuiltinAttr *NBA = nullptr) {2240 auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {2241 SmallString<32> AttributeName;2242 AttributeName += "no-builtin-";2243 AttributeName += BuiltinName;2244 FuncAttrs.addAttribute(AttributeName);2245 };2246 2247 // First, handle the language options passed through -fno-builtin.2248 if (LangOpts.NoBuiltin) {2249 // -fno-builtin disables them all.2250 FuncAttrs.addAttribute("no-builtins");2251 return;2252 }2253 2254 // Then, add attributes for builtins specified through -fno-builtin-<name>.2255 llvm::for_each(LangOpts.NoBuiltinFuncs, AddNoBuiltinAttr);2256 2257 // Now, let's check the __attribute__((no_builtin("...")) attribute added to2258 // the source.2259 if (!NBA)2260 return;2261 2262 // If there is a wildcard in the builtin names specified through the2263 // attribute, disable them all.2264 if (llvm::is_contained(NBA->builtinNames(), "*")) {2265 FuncAttrs.addAttribute("no-builtins");2266 return;2267 }2268 2269 // And last, add the rest of the builtin names.2270 llvm::for_each(NBA->builtinNames(), AddNoBuiltinAttr);2271}2272 2273static bool DetermineNoUndef(QualType QTy, CodeGenTypes &Types,2274 const llvm::DataLayout &DL, const ABIArgInfo &AI,2275 bool CheckCoerce = true) {2276 llvm::Type *Ty = Types.ConvertTypeForMem(QTy);2277 if (AI.getKind() == ABIArgInfo::Indirect ||2278 AI.getKind() == ABIArgInfo::IndirectAliased)2279 return true;2280 if (AI.getKind() == ABIArgInfo::Extend && !AI.isNoExt())2281 return true;2282 if (!DL.typeSizeEqualsStoreSize(Ty))2283 // TODO: This will result in a modest amount of values not marked noundef2284 // when they could be. We care about values that *invisibly* contain undef2285 // bits from the perspective of LLVM IR.2286 return false;2287 if (CheckCoerce && AI.canHaveCoerceToType()) {2288 llvm::Type *CoerceTy = AI.getCoerceToType();2289 if (llvm::TypeSize::isKnownGT(DL.getTypeSizeInBits(CoerceTy),2290 DL.getTypeSizeInBits(Ty)))2291 // If we're coercing to a type with a greater size than the canonical one,2292 // we're introducing new undef bits.2293 // Coercing to a type of smaller or equal size is ok, as we know that2294 // there's no internal padding (typeSizeEqualsStoreSize).2295 return false;2296 }2297 if (QTy->isBitIntType())2298 return true;2299 if (QTy->isReferenceType())2300 return true;2301 if (QTy->isNullPtrType())2302 return false;2303 if (QTy->isMemberPointerType())2304 // TODO: Some member pointers are `noundef`, but it depends on the ABI. For2305 // now, never mark them.2306 return false;2307 if (QTy->isScalarType()) {2308 if (const ComplexType *Complex = dyn_cast<ComplexType>(QTy))2309 return DetermineNoUndef(Complex->getElementType(), Types, DL, AI, false);2310 return true;2311 }2312 if (const VectorType *Vector = dyn_cast<VectorType>(QTy))2313 return DetermineNoUndef(Vector->getElementType(), Types, DL, AI, false);2314 if (const MatrixType *Matrix = dyn_cast<MatrixType>(QTy))2315 return DetermineNoUndef(Matrix->getElementType(), Types, DL, AI, false);2316 if (const ArrayType *Array = dyn_cast<ArrayType>(QTy))2317 return DetermineNoUndef(Array->getElementType(), Types, DL, AI, false);2318 2319 // TODO: Some structs may be `noundef`, in specific situations.2320 return false;2321}2322 2323/// Check if the argument of a function has maybe_undef attribute.2324static bool IsArgumentMaybeUndef(const Decl *TargetDecl,2325 unsigned NumRequiredArgs, unsigned ArgNo) {2326 const auto *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl);2327 if (!FD)2328 return false;2329 2330 // Assume variadic arguments do not have maybe_undef attribute.2331 if (ArgNo >= NumRequiredArgs)2332 return false;2333 2334 // Check if argument has maybe_undef attribute.2335 if (ArgNo < FD->getNumParams()) {2336 const ParmVarDecl *Param = FD->getParamDecl(ArgNo);2337 if (Param && Param->hasAttr<MaybeUndefAttr>())2338 return true;2339 }2340 2341 return false;2342}2343 2344/// Test if it's legal to apply nofpclass for the given parameter type and it's2345/// lowered IR type.2346static bool canApplyNoFPClass(const ABIArgInfo &AI, QualType ParamType,2347 bool IsReturn) {2348 // Should only apply to FP types in the source, not ABI promoted.2349 if (!ParamType->hasFloatingRepresentation())2350 return false;2351 2352 // The promoted-to IR type also needs to support nofpclass.2353 llvm::Type *IRTy = AI.getCoerceToType();2354 if (llvm::AttributeFuncs::isNoFPClassCompatibleType(IRTy))2355 return true;2356 2357 if (llvm::StructType *ST = dyn_cast<llvm::StructType>(IRTy)) {2358 return !IsReturn && AI.getCanBeFlattened() &&2359 llvm::all_of(ST->elements(),2360 llvm::AttributeFuncs::isNoFPClassCompatibleType);2361 }2362 2363 return false;2364}2365 2366/// Return the nofpclass mask that can be applied to floating-point parameters.2367static llvm::FPClassTest getNoFPClassTestMask(const LangOptions &LangOpts) {2368 llvm::FPClassTest Mask = llvm::fcNone;2369 if (LangOpts.NoHonorInfs)2370 Mask |= llvm::fcInf;2371 if (LangOpts.NoHonorNaNs)2372 Mask |= llvm::fcNan;2373 return Mask;2374}2375 2376void CodeGenModule::AdjustMemoryAttribute(StringRef Name,2377 CGCalleeInfo CalleeInfo,2378 llvm::AttributeList &Attrs) {2379 if (Attrs.getMemoryEffects().getModRef() == llvm::ModRefInfo::NoModRef) {2380 Attrs = Attrs.removeFnAttribute(getLLVMContext(), llvm::Attribute::Memory);2381 llvm::Attribute MemoryAttr = llvm::Attribute::getWithMemoryEffects(2382 getLLVMContext(), llvm::MemoryEffects::writeOnly());2383 Attrs = Attrs.addFnAttribute(getLLVMContext(), MemoryAttr);2384 }2385}2386 2387/// Construct the IR attribute list of a function or call.2388///2389/// When adding an attribute, please consider where it should be handled:2390///2391/// - getDefaultFunctionAttributes is for attributes that are essentially2392/// part of the global target configuration (but perhaps can be2393/// overridden on a per-function basis). Adding attributes there2394/// will cause them to also be set in frontends that build on Clang's2395/// target-configuration logic, as well as for code defined in library2396/// modules such as CUDA's libdevice.2397///2398/// - ConstructAttributeList builds on top of getDefaultFunctionAttributes2399/// and adds declaration-specific, convention-specific, and2400/// frontend-specific logic. The last is of particular importance:2401/// attributes that restrict how the frontend generates code must be2402/// added here rather than getDefaultFunctionAttributes.2403///2404void CodeGenModule::ConstructAttributeList(StringRef Name,2405 const CGFunctionInfo &FI,2406 CGCalleeInfo CalleeInfo,2407 llvm::AttributeList &AttrList,2408 unsigned &CallingConv,2409 bool AttrOnCallSite, bool IsThunk) {2410 llvm::AttrBuilder FuncAttrs(getLLVMContext());2411 llvm::AttrBuilder RetAttrs(getLLVMContext());2412 2413 // Collect function IR attributes from the CC lowering.2414 // We'll collect the paramete and result attributes later.2415 CallingConv = FI.getEffectiveCallingConvention();2416 if (FI.isNoReturn())2417 FuncAttrs.addAttribute(llvm::Attribute::NoReturn);2418 if (FI.isCmseNSCall())2419 FuncAttrs.addAttribute("cmse_nonsecure_call");2420 2421 // Collect function IR attributes from the callee prototype if we have one.2422 AddAttributesFromFunctionProtoType(getContext(), FuncAttrs,2423 CalleeInfo.getCalleeFunctionProtoType());2424 const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl();2425 2426 // Attach assumption attributes to the declaration. If this is a call2427 // site, attach assumptions from the caller to the call as well.2428 AddAttributesFromOMPAssumes(FuncAttrs, TargetDecl);2429 2430 bool HasOptnone = false;2431 // The NoBuiltinAttr attached to the target FunctionDecl.2432 const NoBuiltinAttr *NBA = nullptr;2433 2434 // Some ABIs may result in additional accesses to arguments that may2435 // otherwise not be present.2436 std::optional<llvm::Attribute::AttrKind> MemAttrForPtrArgs;2437 bool AddedPotentialArgAccess = false;2438 auto AddPotentialArgAccess = [&]() {2439 AddedPotentialArgAccess = true;2440 llvm::Attribute A = FuncAttrs.getAttribute(llvm::Attribute::Memory);2441 if (A.isValid())2442 FuncAttrs.addMemoryAttr(A.getMemoryEffects() |2443 llvm::MemoryEffects::argMemOnly());2444 };2445 2446 // Collect function IR attributes based on declaration-specific2447 // information.2448 // FIXME: handle sseregparm someday...2449 if (TargetDecl) {2450 if (TargetDecl->hasAttr<ReturnsTwiceAttr>())2451 FuncAttrs.addAttribute(llvm::Attribute::ReturnsTwice);2452 if (TargetDecl->hasAttr<NoThrowAttr>())2453 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);2454 if (TargetDecl->hasAttr<NoReturnAttr>())2455 FuncAttrs.addAttribute(llvm::Attribute::NoReturn);2456 if (TargetDecl->hasAttr<ColdAttr>())2457 FuncAttrs.addAttribute(llvm::Attribute::Cold);2458 if (TargetDecl->hasAttr<HotAttr>())2459 FuncAttrs.addAttribute(llvm::Attribute::Hot);2460 if (TargetDecl->hasAttr<NoDuplicateAttr>())2461 FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);2462 if (TargetDecl->hasAttr<ConvergentAttr>())2463 FuncAttrs.addAttribute(llvm::Attribute::Convergent);2464 2465 if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {2466 AddAttributesFromFunctionProtoType(2467 getContext(), FuncAttrs, Fn->getType()->getAs<FunctionProtoType>());2468 if (AttrOnCallSite && Fn->isReplaceableGlobalAllocationFunction()) {2469 // A sane operator new returns a non-aliasing pointer.2470 auto Kind = Fn->getDeclName().getCXXOverloadedOperator();2471 if (getCodeGenOpts().AssumeSaneOperatorNew &&2472 (Kind == OO_New || Kind == OO_Array_New))2473 RetAttrs.addAttribute(llvm::Attribute::NoAlias);2474 }2475 const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Fn);2476 const bool IsVirtualCall = MD && MD->isVirtual();2477 // Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a2478 // virtual function. These attributes are not inherited by overloads.2479 if (!(AttrOnCallSite && IsVirtualCall)) {2480 if (Fn->isNoReturn())2481 FuncAttrs.addAttribute(llvm::Attribute::NoReturn);2482 NBA = Fn->getAttr<NoBuiltinAttr>();2483 }2484 }2485 2486 if (isa<FunctionDecl>(TargetDecl) || isa<VarDecl>(TargetDecl)) {2487 // Only place nomerge attribute on call sites, never functions. This2488 // allows it to work on indirect virtual function calls.2489 if (AttrOnCallSite && TargetDecl->hasAttr<NoMergeAttr>())2490 FuncAttrs.addAttribute(llvm::Attribute::NoMerge);2491 }2492 2493 // 'const', 'pure' and 'noalias' attributed functions are also nounwind.2494 if (TargetDecl->hasAttr<ConstAttr>()) {2495 FuncAttrs.addMemoryAttr(llvm::MemoryEffects::none());2496 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);2497 // gcc specifies that 'const' functions have greater restrictions than2498 // 'pure' functions, so they also cannot have infinite loops.2499 FuncAttrs.addAttribute(llvm::Attribute::WillReturn);2500 MemAttrForPtrArgs = llvm::Attribute::ReadNone;2501 } else if (TargetDecl->hasAttr<PureAttr>()) {2502 FuncAttrs.addMemoryAttr(llvm::MemoryEffects::readOnly());2503 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);2504 // gcc specifies that 'pure' functions cannot have infinite loops.2505 FuncAttrs.addAttribute(llvm::Attribute::WillReturn);2506 MemAttrForPtrArgs = llvm::Attribute::ReadOnly;2507 } else if (TargetDecl->hasAttr<NoAliasAttr>()) {2508 FuncAttrs.addMemoryAttr(llvm::MemoryEffects::inaccessibleOrArgMemOnly());2509 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);2510 }2511 if (const auto *RA = TargetDecl->getAttr<RestrictAttr>();2512 RA && RA->getDeallocator() == nullptr)2513 RetAttrs.addAttribute(llvm::Attribute::NoAlias);2514 if (TargetDecl->hasAttr<ReturnsNonNullAttr>() &&2515 !CodeGenOpts.NullPointerIsValid)2516 RetAttrs.addAttribute(llvm::Attribute::NonNull);2517 if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>())2518 FuncAttrs.addAttribute("no_caller_saved_registers");2519 if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>())2520 FuncAttrs.addAttribute(llvm::Attribute::NoCfCheck);2521 if (TargetDecl->hasAttr<LeafAttr>())2522 FuncAttrs.addAttribute(llvm::Attribute::NoCallback);2523 if (TargetDecl->hasAttr<BPFFastCallAttr>())2524 FuncAttrs.addAttribute("bpf_fastcall");2525 2526 HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();2527 if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {2528 std::optional<unsigned> NumElemsParam;2529 if (AllocSize->getNumElemsParam().isValid())2530 NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();2531 FuncAttrs.addAllocSizeAttr(AllocSize->getElemSizeParam().getLLVMIndex(),2532 NumElemsParam);2533 }2534 2535 if (DeviceKernelAttr::isOpenCLSpelling(2536 TargetDecl->getAttr<DeviceKernelAttr>()) &&2537 CallingConv != CallingConv::CC_C &&2538 CallingConv != CallingConv::CC_SpirFunction) {2539 // Check CallingConv to avoid adding uniform-work-group-size attribute to2540 // OpenCL Kernel Stub2541 if (getLangOpts().OpenCLVersion <= 120) {2542 // OpenCL v1.2 Work groups are always uniform2543 FuncAttrs.addAttribute("uniform-work-group-size", "true");2544 } else {2545 // OpenCL v2.0 Work groups may be whether uniform or not.2546 // '-cl-uniform-work-group-size' compile option gets a hint2547 // to the compiler that the global work-size be a multiple of2548 // the work-group size specified to clEnqueueNDRangeKernel2549 // (i.e. work groups are uniform).2550 FuncAttrs.addAttribute(2551 "uniform-work-group-size",2552 llvm::toStringRef(getLangOpts().OffloadUniformBlock));2553 }2554 }2555 2556 if (TargetDecl->hasAttr<CUDAGlobalAttr>() &&2557 getLangOpts().OffloadUniformBlock)2558 FuncAttrs.addAttribute("uniform-work-group-size", "true");2559 2560 if (TargetDecl->hasAttr<ArmLocallyStreamingAttr>())2561 FuncAttrs.addAttribute("aarch64_pstate_sm_body");2562 }2563 2564 // Attach "no-builtins" attributes to:2565 // * call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>".2566 // * definitions: "no-builtins" or "no-builtin-<name>" only.2567 // The attributes can come from:2568 // * LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name>2569 // * FunctionDecl attributes: __attribute__((no_builtin(...)))2570 addNoBuiltinAttributes(FuncAttrs, getLangOpts(), NBA);2571 2572 // Collect function IR attributes based on global settiings.2573 getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs);2574 2575 // Override some default IR attributes based on declaration-specific2576 // information.2577 if (TargetDecl) {2578 if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>())2579 FuncAttrs.removeAttribute(llvm::Attribute::SpeculativeLoadHardening);2580 if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>())2581 FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);2582 if (TargetDecl->hasAttr<NoSplitStackAttr>())2583 FuncAttrs.removeAttribute("split-stack");2584 if (TargetDecl->hasAttr<ZeroCallUsedRegsAttr>()) {2585 // A function "__attribute__((...))" overrides the command-line flag.2586 auto Kind =2587 TargetDecl->getAttr<ZeroCallUsedRegsAttr>()->getZeroCallUsedRegs();2588 FuncAttrs.removeAttribute("zero-call-used-regs");2589 FuncAttrs.addAttribute(2590 "zero-call-used-regs",2591 ZeroCallUsedRegsAttr::ConvertZeroCallUsedRegsKindToStr(Kind));2592 }2593 2594 // Add NonLazyBind attribute to function declarations when -fno-plt2595 // is used.2596 // FIXME: what if we just haven't processed the function definition2597 // yet, or if it's an external definition like C99 inline?2598 if (CodeGenOpts.NoPLT) {2599 if (auto *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {2600 if (!Fn->isDefined() && !AttrOnCallSite) {2601 FuncAttrs.addAttribute(llvm::Attribute::NonLazyBind);2602 }2603 }2604 }2605 // Remove 'convergent' if requested.2606 if (TargetDecl->hasAttr<NoConvergentAttr>())2607 FuncAttrs.removeAttribute(llvm::Attribute::Convergent);2608 }2609 2610 // Add "sample-profile-suffix-elision-policy" attribute for internal linkage2611 // functions with -funique-internal-linkage-names.2612 if (TargetDecl && CodeGenOpts.UniqueInternalLinkageNames) {2613 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {2614 if (!FD->isExternallyVisible())2615 FuncAttrs.addAttribute("sample-profile-suffix-elision-policy",2616 "selected");2617 }2618 }2619 2620 // Collect non-call-site function IR attributes from declaration-specific2621 // information.2622 if (!AttrOnCallSite) {2623 if (TargetDecl && TargetDecl->hasAttr<CmseNSEntryAttr>())2624 FuncAttrs.addAttribute("cmse_nonsecure_entry");2625 2626 // Whether tail calls are enabled.2627 auto shouldDisableTailCalls = [&] {2628 // Should this be honored in getDefaultFunctionAttributes?2629 if (CodeGenOpts.DisableTailCalls)2630 return true;2631 2632 if (!TargetDecl)2633 return false;2634 2635 if (TargetDecl->hasAttr<DisableTailCallsAttr>() ||2636 TargetDecl->hasAttr<AnyX86InterruptAttr>())2637 return true;2638 2639 if (CodeGenOpts.NoEscapingBlockTailCalls) {2640 if (const auto *BD = dyn_cast<BlockDecl>(TargetDecl))2641 if (!BD->doesNotEscape())2642 return true;2643 }2644 2645 return false;2646 };2647 if (shouldDisableTailCalls())2648 FuncAttrs.addAttribute("disable-tail-calls", "true");2649 2650 // These functions require the returns_twice attribute for correct codegen,2651 // but the attribute may not be added if -fno-builtin is specified. We2652 // explicitly add that attribute here.2653 static const llvm::StringSet<> ReturnsTwiceFn{2654 "_setjmpex", "setjmp", "_setjmp", "vfork",2655 "sigsetjmp", "__sigsetjmp", "savectx", "getcontext"};2656 if (ReturnsTwiceFn.contains(Name))2657 FuncAttrs.addAttribute(llvm::Attribute::ReturnsTwice);2658 2659 // CPU/feature overrides. addDefaultFunctionDefinitionAttributes2660 // handles these separately to set them based on the global defaults.2661 GetCPUAndFeaturesAttributes(CalleeInfo.getCalleeDecl(), FuncAttrs);2662 2663 // Windows hotpatching support2664 if (!MSHotPatchFunctions.empty()) {2665 bool IsHotPatched = llvm::binary_search(MSHotPatchFunctions, Name);2666 if (IsHotPatched)2667 FuncAttrs.addAttribute("marked_for_windows_hot_patching");2668 }2669 }2670 2671 // Mark functions that are replaceable by the loader.2672 if (CodeGenOpts.isLoaderReplaceableFunctionName(Name))2673 FuncAttrs.addAttribute("loader-replaceable");2674 2675 // Collect attributes from arguments and return values.2676 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);2677 2678 QualType RetTy = FI.getReturnType();2679 const ABIArgInfo &RetAI = FI.getReturnInfo();2680 const llvm::DataLayout &DL = getDataLayout();2681 2682 // Determine if the return type could be partially undef2683 if (CodeGenOpts.EnableNoundefAttrs &&2684 HasStrictReturn(*this, RetTy, TargetDecl)) {2685 if (!RetTy->isVoidType() && RetAI.getKind() != ABIArgInfo::Indirect &&2686 DetermineNoUndef(RetTy, getTypes(), DL, RetAI))2687 RetAttrs.addAttribute(llvm::Attribute::NoUndef);2688 }2689 2690 switch (RetAI.getKind()) {2691 case ABIArgInfo::Extend:2692 if (RetAI.isSignExt())2693 RetAttrs.addAttribute(llvm::Attribute::SExt);2694 else if (RetAI.isZeroExt())2695 RetAttrs.addAttribute(llvm::Attribute::ZExt);2696 else2697 RetAttrs.addAttribute(llvm::Attribute::NoExt);2698 [[fallthrough]];2699 case ABIArgInfo::TargetSpecific:2700 case ABIArgInfo::Direct:2701 if (RetAI.getInReg())2702 RetAttrs.addAttribute(llvm::Attribute::InReg);2703 2704 if (canApplyNoFPClass(RetAI, RetTy, true))2705 RetAttrs.addNoFPClassAttr(getNoFPClassTestMask(getLangOpts()));2706 2707 break;2708 case ABIArgInfo::Ignore:2709 break;2710 2711 case ABIArgInfo::InAlloca:2712 case ABIArgInfo::Indirect: {2713 // inalloca and sret disable readnone and readonly2714 AddPotentialArgAccess();2715 break;2716 }2717 2718 case ABIArgInfo::CoerceAndExpand:2719 break;2720 2721 case ABIArgInfo::Expand:2722 case ABIArgInfo::IndirectAliased:2723 llvm_unreachable("Invalid ABI kind for return argument");2724 }2725 2726 if (!IsThunk) {2727 // FIXME: fix this properly, https://reviews.llvm.org/D1003882728 if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {2729 QualType PTy = RefTy->getPointeeType();2730 if (!PTy->isIncompleteType() && PTy->isConstantSizeType())2731 RetAttrs.addDereferenceableAttr(2732 getMinimumObjectSize(PTy).getQuantity());2733 if (getTypes().getTargetAddressSpace(PTy) == 0 &&2734 !CodeGenOpts.NullPointerIsValid)2735 RetAttrs.addAttribute(llvm::Attribute::NonNull);2736 if (PTy->isObjectType()) {2737 llvm::Align Alignment =2738 getNaturalPointeeTypeAlignment(RetTy).getAsAlign();2739 RetAttrs.addAlignmentAttr(Alignment);2740 }2741 }2742 }2743 2744 bool hasUsedSRet = false;2745 SmallVector<llvm::AttributeSet, 4> ArgAttrs(IRFunctionArgs.totalIRArgs());2746 2747 // Attach attributes to sret.2748 if (IRFunctionArgs.hasSRetArg()) {2749 llvm::AttrBuilder SRETAttrs(getLLVMContext());2750 SRETAttrs.addStructRetAttr(getTypes().ConvertTypeForMem(RetTy));2751 SRETAttrs.addAttribute(llvm::Attribute::Writable);2752 SRETAttrs.addAttribute(llvm::Attribute::DeadOnUnwind);2753 hasUsedSRet = true;2754 if (RetAI.getInReg())2755 SRETAttrs.addAttribute(llvm::Attribute::InReg);2756 SRETAttrs.addAlignmentAttr(RetAI.getIndirectAlign().getQuantity());2757 ArgAttrs[IRFunctionArgs.getSRetArgNo()] =2758 llvm::AttributeSet::get(getLLVMContext(), SRETAttrs);2759 }2760 2761 // Attach attributes to inalloca argument.2762 if (IRFunctionArgs.hasInallocaArg()) {2763 llvm::AttrBuilder Attrs(getLLVMContext());2764 Attrs.addInAllocaAttr(FI.getArgStruct());2765 ArgAttrs[IRFunctionArgs.getInallocaArgNo()] =2766 llvm::AttributeSet::get(getLLVMContext(), Attrs);2767 }2768 2769 // Apply `nonnull`, `dereferenceable(N)` and `align N` to the `this` argument,2770 // unless this is a thunk function.2771 // FIXME: fix this properly, https://reviews.llvm.org/D1003882772 if (FI.isInstanceMethod() && !IRFunctionArgs.hasInallocaArg() &&2773 !FI.arg_begin()->type->isVoidPointerType() && !IsThunk) {2774 auto IRArgs = IRFunctionArgs.getIRArgs(0);2775 2776 assert(IRArgs.second == 1 && "Expected only a single `this` pointer.");2777 2778 llvm::AttrBuilder Attrs(getLLVMContext());2779 2780 QualType ThisTy = FI.arg_begin()->type.getTypePtr()->getPointeeType();2781 2782 if (!CodeGenOpts.NullPointerIsValid &&2783 getTypes().getTargetAddressSpace(FI.arg_begin()->type) == 0) {2784 Attrs.addAttribute(llvm::Attribute::NonNull);2785 Attrs.addDereferenceableAttr(getMinimumObjectSize(ThisTy).getQuantity());2786 } else {2787 // FIXME dereferenceable should be correct here, regardless of2788 // NullPointerIsValid. However, dereferenceable currently does not always2789 // respect NullPointerIsValid and may imply nonnull and break the program.2790 // See https://reviews.llvm.org/D66618 for discussions.2791 Attrs.addDereferenceableOrNullAttr(2792 getMinimumObjectSize(2793 FI.arg_begin()->type.castAs<PointerType>()->getPointeeType())2794 .getQuantity());2795 }2796 2797 llvm::Align Alignment =2798 getNaturalTypeAlignment(ThisTy, /*BaseInfo=*/nullptr,2799 /*TBAAInfo=*/nullptr, /*forPointeeType=*/true)2800 .getAsAlign();2801 Attrs.addAlignmentAttr(Alignment);2802 2803 ArgAttrs[IRArgs.first] = llvm::AttributeSet::get(getLLVMContext(), Attrs);2804 }2805 2806 unsigned ArgNo = 0;2807 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(), E = FI.arg_end();2808 I != E; ++I, ++ArgNo) {2809 QualType ParamType = I->type;2810 const ABIArgInfo &AI = I->info;2811 llvm::AttrBuilder Attrs(getLLVMContext());2812 2813 // Add attribute for padding argument, if necessary.2814 if (IRFunctionArgs.hasPaddingArg(ArgNo)) {2815 if (AI.getPaddingInReg()) {2816 ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =2817 llvm::AttributeSet::get(getLLVMContext(),2818 llvm::AttrBuilder(getLLVMContext())2819 .addAttribute(llvm::Attribute::InReg));2820 }2821 }2822 2823 // Decide whether the argument we're handling could be partially undef2824 if (CodeGenOpts.EnableNoundefAttrs &&2825 DetermineNoUndef(ParamType, getTypes(), DL, AI)) {2826 Attrs.addAttribute(llvm::Attribute::NoUndef);2827 }2828 2829 // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we2830 // have the corresponding parameter variable. It doesn't make2831 // sense to do it here because parameters are so messed up.2832 switch (AI.getKind()) {2833 case ABIArgInfo::Extend:2834 if (AI.isSignExt())2835 Attrs.addAttribute(llvm::Attribute::SExt);2836 else if (AI.isZeroExt())2837 Attrs.addAttribute(llvm::Attribute::ZExt);2838 else2839 Attrs.addAttribute(llvm::Attribute::NoExt);2840 [[fallthrough]];2841 case ABIArgInfo::TargetSpecific:2842 case ABIArgInfo::Direct:2843 if (ArgNo == 0 && FI.isChainCall())2844 Attrs.addAttribute(llvm::Attribute::Nest);2845 else if (AI.getInReg())2846 Attrs.addAttribute(llvm::Attribute::InReg);2847 Attrs.addStackAlignmentAttr(llvm::MaybeAlign(AI.getDirectAlign()));2848 2849 if (canApplyNoFPClass(AI, ParamType, false))2850 Attrs.addNoFPClassAttr(getNoFPClassTestMask(getLangOpts()));2851 break;2852 case ABIArgInfo::Indirect: {2853 if (AI.getInReg())2854 Attrs.addAttribute(llvm::Attribute::InReg);2855 2856 // HLSL out and inout parameters must not be marked with ByVal or2857 // DeadOnReturn attributes because stores to these parameters by the2858 // callee are visible to the caller.2859 if (auto ParamABI = FI.getExtParameterInfo(ArgNo).getABI();2860 ParamABI != ParameterABI::HLSLOut &&2861 ParamABI != ParameterABI::HLSLInOut) {2862 2863 // Depending on the ABI, this may be either a byval or a dead_on_return2864 // argument.2865 if (AI.getIndirectByVal()) {2866 Attrs.addByValAttr(getTypes().ConvertTypeForMem(ParamType));2867 } else {2868 // Add dead_on_return when the object's lifetime ends in the callee.2869 // This includes trivially-destructible objects, as well as objects2870 // whose destruction / clean-up is carried out within the callee2871 // (e.g., Obj-C ARC-managed structs, MSVC callee-destroyed objects).2872 if (!ParamType.isDestructedType() || !ParamType->isRecordType() ||2873 ParamType->castAsRecordDecl()->isParamDestroyedInCallee())2874 Attrs.addAttribute(llvm::Attribute::DeadOnReturn);2875 }2876 }2877 2878 auto *Decl = ParamType->getAsRecordDecl();2879 if (CodeGenOpts.PassByValueIsNoAlias && Decl &&2880 Decl->getArgPassingRestrictions() ==2881 RecordArgPassingKind::CanPassInRegs)2882 // When calling the function, the pointer passed in will be the only2883 // reference to the underlying object. Mark it accordingly.2884 Attrs.addAttribute(llvm::Attribute::NoAlias);2885 2886 // TODO: We could add the byref attribute if not byval, but it would2887 // require updating many testcases.2888 2889 CharUnits Align = AI.getIndirectAlign();2890 2891 // In a byval argument, it is important that the required2892 // alignment of the type is honored, as LLVM might be creating a2893 // *new* stack object, and needs to know what alignment to give2894 // it. (Sometimes it can deduce a sensible alignment on its own,2895 // but not if clang decides it must emit a packed struct, or the2896 // user specifies increased alignment requirements.)2897 //2898 // This is different from indirect *not* byval, where the object2899 // exists already, and the align attribute is purely2900 // informative.2901 assert(!Align.isZero());2902 2903 // For now, only add this when we have a byval argument.2904 // TODO: be less lazy about updating test cases.2905 if (AI.getIndirectByVal())2906 Attrs.addAlignmentAttr(Align.getQuantity());2907 2908 // byval disables readnone and readonly.2909 AddPotentialArgAccess();2910 break;2911 }2912 case ABIArgInfo::IndirectAliased: {2913 CharUnits Align = AI.getIndirectAlign();2914 Attrs.addByRefAttr(getTypes().ConvertTypeForMem(ParamType));2915 Attrs.addAlignmentAttr(Align.getQuantity());2916 break;2917 }2918 case ABIArgInfo::Ignore:2919 case ABIArgInfo::Expand:2920 case ABIArgInfo::CoerceAndExpand:2921 break;2922 2923 case ABIArgInfo::InAlloca:2924 // inalloca disables readnone and readonly.2925 AddPotentialArgAccess();2926 continue;2927 }2928 2929 if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {2930 QualType PTy = RefTy->getPointeeType();2931 if (!PTy->isIncompleteType() && PTy->isConstantSizeType())2932 Attrs.addDereferenceableAttr(getMinimumObjectSize(PTy).getQuantity());2933 if (getTypes().getTargetAddressSpace(PTy) == 0 &&2934 !CodeGenOpts.NullPointerIsValid)2935 Attrs.addAttribute(llvm::Attribute::NonNull);2936 if (PTy->isObjectType()) {2937 llvm::Align Alignment =2938 getNaturalPointeeTypeAlignment(ParamType).getAsAlign();2939 Attrs.addAlignmentAttr(Alignment);2940 }2941 }2942 2943 // From OpenCL spec v3.0.10 section 6.3.5 Alignment of Types:2944 // > For arguments to a __kernel function declared to be a pointer to a2945 // > data type, the OpenCL compiler can assume that the pointee is always2946 // > appropriately aligned as required by the data type.2947 if (TargetDecl &&2948 DeviceKernelAttr::isOpenCLSpelling(2949 TargetDecl->getAttr<DeviceKernelAttr>()) &&2950 ParamType->isPointerType()) {2951 QualType PTy = ParamType->getPointeeType();2952 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {2953 llvm::Align Alignment =2954 getNaturalPointeeTypeAlignment(ParamType).getAsAlign();2955 Attrs.addAlignmentAttr(Alignment);2956 }2957 }2958 2959 switch (FI.getExtParameterInfo(ArgNo).getABI()) {2960 case ParameterABI::HLSLOut:2961 case ParameterABI::HLSLInOut:2962 Attrs.addAttribute(llvm::Attribute::NoAlias);2963 break;2964 case ParameterABI::Ordinary:2965 break;2966 2967 case ParameterABI::SwiftIndirectResult: {2968 // Add 'sret' if we haven't already used it for something, but2969 // only if the result is void.2970 if (!hasUsedSRet && RetTy->isVoidType()) {2971 Attrs.addStructRetAttr(getTypes().ConvertTypeForMem(ParamType));2972 hasUsedSRet = true;2973 }2974 2975 // Add 'noalias' in either case.2976 Attrs.addAttribute(llvm::Attribute::NoAlias);2977 2978 // Add 'dereferenceable' and 'alignment'.2979 auto PTy = ParamType->getPointeeType();2980 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {2981 auto info = getContext().getTypeInfoInChars(PTy);2982 Attrs.addDereferenceableAttr(info.Width.getQuantity());2983 Attrs.addAlignmentAttr(info.Align.getAsAlign());2984 }2985 break;2986 }2987 2988 case ParameterABI::SwiftErrorResult:2989 Attrs.addAttribute(llvm::Attribute::SwiftError);2990 break;2991 2992 case ParameterABI::SwiftContext:2993 Attrs.addAttribute(llvm::Attribute::SwiftSelf);2994 break;2995 2996 case ParameterABI::SwiftAsyncContext:2997 Attrs.addAttribute(llvm::Attribute::SwiftAsync);2998 break;2999 }3000 3001 if (FI.getExtParameterInfo(ArgNo).isNoEscape())3002 Attrs.addCapturesAttr(llvm::CaptureInfo::none());3003 3004 if (Attrs.hasAttributes()) {3005 unsigned FirstIRArg, NumIRArgs;3006 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);3007 for (unsigned i = 0; i < NumIRArgs; i++)3008 ArgAttrs[FirstIRArg + i] = ArgAttrs[FirstIRArg + i].addAttributes(3009 getLLVMContext(), llvm::AttributeSet::get(getLLVMContext(), Attrs));3010 }3011 }3012 assert(ArgNo == FI.arg_size());3013 3014 ArgNo = 0;3015 if (AddedPotentialArgAccess && MemAttrForPtrArgs) {3016 llvm::FunctionType *FunctionType = getTypes().GetFunctionType(FI);3017 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(),3018 E = FI.arg_end();3019 I != E; ++I, ++ArgNo) {3020 if (I->info.isDirect() || I->info.isExpand() ||3021 I->info.isCoerceAndExpand()) {3022 unsigned FirstIRArg, NumIRArgs;3023 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);3024 for (unsigned i = FirstIRArg; i < FirstIRArg + NumIRArgs; ++i) {3025 if (FunctionType->getParamType(i)->isPointerTy()) {3026 ArgAttrs[i] =3027 ArgAttrs[i].addAttribute(getLLVMContext(), *MemAttrForPtrArgs);3028 }3029 }3030 }3031 }3032 }3033 3034 AttrList = llvm::AttributeList::get(3035 getLLVMContext(), llvm::AttributeSet::get(getLLVMContext(), FuncAttrs),3036 llvm::AttributeSet::get(getLLVMContext(), RetAttrs), ArgAttrs);3037}3038 3039/// An argument came in as a promoted argument; demote it back to its3040/// declared type.3041static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,3042 const VarDecl *var,3043 llvm::Value *value) {3044 llvm::Type *varType = CGF.ConvertType(var->getType());3045 3046 // This can happen with promotions that actually don't change the3047 // underlying type, like the enum promotions.3048 if (value->getType() == varType)3049 return value;3050 3051 assert((varType->isIntegerTy() || varType->isFloatingPointTy()) &&3052 "unexpected promotion type");3053 3054 if (isa<llvm::IntegerType>(varType))3055 return CGF.Builder.CreateTrunc(value, varType, "arg.unpromote");3056 3057 return CGF.Builder.CreateFPCast(value, varType, "arg.unpromote");3058}3059 3060/// Returns the attribute (either parameter attribute, or function3061/// attribute), which declares argument ArgNo to be non-null.3062static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD,3063 QualType ArgType, unsigned ArgNo) {3064 // FIXME: __attribute__((nonnull)) can also be applied to:3065 // - references to pointers, where the pointee is known to be3066 // nonnull (apparently a Clang extension)3067 // - transparent unions containing pointers3068 // In the former case, LLVM IR cannot represent the constraint. In3069 // the latter case, we have no guarantee that the transparent union3070 // is in fact passed as a pointer.3071 if (!ArgType->isAnyPointerType() && !ArgType->isBlockPointerType())3072 return nullptr;3073 // First, check attribute on parameter itself.3074 if (PVD) {3075 if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>())3076 return ParmNNAttr;3077 }3078 // Check function attributes.3079 if (!FD)3080 return nullptr;3081 for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {3082 if (NNAttr->isNonNull(ArgNo))3083 return NNAttr;3084 }3085 return nullptr;3086}3087 3088namespace {3089struct CopyBackSwiftError final : EHScopeStack::Cleanup {3090 Address Temp;3091 Address Arg;3092 CopyBackSwiftError(Address temp, Address arg) : Temp(temp), Arg(arg) {}3093 void Emit(CodeGenFunction &CGF, Flags flags) override {3094 llvm::Value *errorValue = CGF.Builder.CreateLoad(Temp);3095 CGF.Builder.CreateStore(errorValue, Arg);3096 }3097};3098} // namespace3099 3100void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,3101 llvm::Function *Fn,3102 const FunctionArgList &Args) {3103 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>())3104 // Naked functions don't have prologues.3105 return;3106 3107 // If this is an implicit-return-zero function, go ahead and3108 // initialize the return value. TODO: it might be nice to have3109 // a more general mechanism for this that didn't require synthesized3110 // return statements.3111 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl)) {3112 if (FD->hasImplicitReturnZero()) {3113 QualType RetTy = FD->getReturnType().getUnqualifiedType();3114 llvm::Type *LLVMTy = CGM.getTypes().ConvertType(RetTy);3115 llvm::Constant *Zero = llvm::Constant::getNullValue(LLVMTy);3116 Builder.CreateStore(Zero, ReturnValue);3117 }3118 }3119 3120 // FIXME: We no longer need the types from FunctionArgList; lift up and3121 // simplify.3122 3123 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);3124 assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs());3125 3126 // If we're using inalloca, all the memory arguments are GEPs off of the last3127 // parameter, which is a pointer to the complete memory area.3128 Address ArgStruct = Address::invalid();3129 if (IRFunctionArgs.hasInallocaArg())3130 ArgStruct = Address(Fn->getArg(IRFunctionArgs.getInallocaArgNo()),3131 FI.getArgStruct(), FI.getArgStructAlignment());3132 3133 // Name the struct return parameter.3134 if (IRFunctionArgs.hasSRetArg()) {3135 auto AI = Fn->getArg(IRFunctionArgs.getSRetArgNo());3136 AI->setName("agg.result");3137 AI->addAttr(llvm::Attribute::NoAlias);3138 }3139 3140 // Track if we received the parameter as a pointer (indirect, byval, or3141 // inalloca). If already have a pointer, EmitParmDecl doesn't need to copy it3142 // into a local alloca for us.3143 SmallVector<ParamValue, 16> ArgVals;3144 ArgVals.reserve(Args.size());3145 3146 // Create a pointer value for every parameter declaration. This usually3147 // entails copying one or more LLVM IR arguments into an alloca. Don't push3148 // any cleanups or do anything that might unwind. We do that separately, so3149 // we can push the cleanups in the correct order for the ABI.3150 assert(FI.arg_size() == Args.size() &&3151 "Mismatch between function signature & arguments.");3152 unsigned ArgNo = 0;3153 CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();3154 for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end(); i != e;3155 ++i, ++info_it, ++ArgNo) {3156 const VarDecl *Arg = *i;3157 const ABIArgInfo &ArgI = info_it->info;3158 3159 bool isPromoted =3160 isa<ParmVarDecl>(Arg) && cast<ParmVarDecl>(Arg)->isKNRPromoted();3161 // We are converting from ABIArgInfo type to VarDecl type directly, unless3162 // the parameter is promoted. In this case we convert to3163 // CGFunctionInfo::ArgInfo type with subsequent argument demotion.3164 QualType Ty = isPromoted ? info_it->type : Arg->getType();3165 assert(hasScalarEvaluationKind(Ty) ==3166 hasScalarEvaluationKind(Arg->getType()));3167 3168 unsigned FirstIRArg, NumIRArgs;3169 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);3170 3171 switch (ArgI.getKind()) {3172 case ABIArgInfo::InAlloca: {3173 assert(NumIRArgs == 0);3174 auto FieldIndex = ArgI.getInAllocaFieldIndex();3175 Address V =3176 Builder.CreateStructGEP(ArgStruct, FieldIndex, Arg->getName());3177 if (ArgI.getInAllocaIndirect())3178 V = Address(Builder.CreateLoad(V), ConvertTypeForMem(Ty),3179 getContext().getTypeAlignInChars(Ty));3180 ArgVals.push_back(ParamValue::forIndirect(V));3181 break;3182 }3183 3184 case ABIArgInfo::Indirect:3185 case ABIArgInfo::IndirectAliased: {3186 assert(NumIRArgs == 1);3187 Address ParamAddr = makeNaturalAddressForPointer(3188 Fn->getArg(FirstIRArg), Ty, ArgI.getIndirectAlign(), false, nullptr,3189 nullptr, KnownNonNull);3190 3191 if (!hasScalarEvaluationKind(Ty)) {3192 // Aggregates and complex variables are accessed by reference. All we3193 // need to do is realign the value, if requested. Also, if the address3194 // may be aliased, copy it to ensure that the parameter variable is3195 // mutable and has a unique adress, as C requires.3196 if (ArgI.getIndirectRealign() || ArgI.isIndirectAliased()) {3197 RawAddress AlignedTemp = CreateMemTemp(Ty, "coerce");3198 3199 // Copy from the incoming argument pointer to the temporary with the3200 // appropriate alignment.3201 //3202 // FIXME: We should have a common utility for generating an aggregate3203 // copy.3204 CharUnits Size = getContext().getTypeSizeInChars(Ty);3205 Builder.CreateMemCpy(3206 AlignedTemp.getPointer(), AlignedTemp.getAlignment().getAsAlign(),3207 ParamAddr.emitRawPointer(*this),3208 ParamAddr.getAlignment().getAsAlign(),3209 llvm::ConstantInt::get(IntPtrTy, Size.getQuantity()));3210 ParamAddr = AlignedTemp;3211 }3212 ArgVals.push_back(ParamValue::forIndirect(ParamAddr));3213 } else {3214 // Load scalar value from indirect argument.3215 llvm::Value *V =3216 EmitLoadOfScalar(ParamAddr, false, Ty, Arg->getBeginLoc());3217 3218 if (isPromoted)3219 V = emitArgumentDemotion(*this, Arg, V);3220 ArgVals.push_back(ParamValue::forDirect(V));3221 }3222 break;3223 }3224 3225 case ABIArgInfo::Extend:3226 case ABIArgInfo::Direct: {3227 auto AI = Fn->getArg(FirstIRArg);3228 llvm::Type *LTy = ConvertType(Arg->getType());3229 3230 // Prepare parameter attributes. So far, only attributes for pointer3231 // parameters are prepared. See3232 // http://llvm.org/docs/LangRef.html#paramattrs.3233 if (ArgI.getDirectOffset() == 0 && LTy->isPointerTy() &&3234 ArgI.getCoerceToType()->isPointerTy()) {3235 assert(NumIRArgs == 1);3236 3237 if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Arg)) {3238 // Set `nonnull` attribute if any.3239 if (getNonNullAttr(CurCodeDecl, PVD, PVD->getType(),3240 PVD->getFunctionScopeIndex()) &&3241 !CGM.getCodeGenOpts().NullPointerIsValid)3242 AI->addAttr(llvm::Attribute::NonNull);3243 3244 QualType OTy = PVD->getOriginalType();3245 if (const auto *ArrTy = getContext().getAsConstantArrayType(OTy)) {3246 // A C99 array parameter declaration with the static keyword also3247 // indicates dereferenceability, and if the size is constant we can3248 // use the dereferenceable attribute (which requires the size in3249 // bytes).3250 if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {3251 QualType ETy = ArrTy->getElementType();3252 llvm::Align Alignment =3253 CGM.getNaturalTypeAlignment(ETy).getAsAlign();3254 AI->addAttrs(llvm::AttrBuilder(getLLVMContext())3255 .addAlignmentAttr(Alignment));3256 uint64_t ArrSize = ArrTy->getZExtSize();3257 if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&3258 ArrSize) {3259 llvm::AttrBuilder Attrs(getLLVMContext());3260 Attrs.addDereferenceableAttr(3261 getContext().getTypeSizeInChars(ETy).getQuantity() *3262 ArrSize);3263 AI->addAttrs(Attrs);3264 } else if (getContext().getTargetInfo().getNullPointerValue(3265 ETy.getAddressSpace()) == 0 &&3266 !CGM.getCodeGenOpts().NullPointerIsValid) {3267 AI->addAttr(llvm::Attribute::NonNull);3268 }3269 }3270 } else if (const auto *ArrTy =3271 getContext().getAsVariableArrayType(OTy)) {3272 // For C99 VLAs with the static keyword, we don't know the size so3273 // we can't use the dereferenceable attribute, but in addrspace(0)3274 // we know that it must be nonnull.3275 if (ArrTy->getSizeModifier() == ArraySizeModifier::Static) {3276 QualType ETy = ArrTy->getElementType();3277 llvm::Align Alignment =3278 CGM.getNaturalTypeAlignment(ETy).getAsAlign();3279 AI->addAttrs(llvm::AttrBuilder(getLLVMContext())3280 .addAlignmentAttr(Alignment));3281 if (!getTypes().getTargetAddressSpace(ETy) &&3282 !CGM.getCodeGenOpts().NullPointerIsValid)3283 AI->addAttr(llvm::Attribute::NonNull);3284 }3285 }3286 3287 // Set `align` attribute if any.3288 const auto *AVAttr = PVD->getAttr<AlignValueAttr>();3289 if (!AVAttr)3290 if (const auto *TOTy = OTy->getAs<TypedefType>())3291 AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();3292 if (AVAttr && !SanOpts.has(SanitizerKind::Alignment)) {3293 // If alignment-assumption sanitizer is enabled, we do *not* add3294 // alignment attribute here, but emit normal alignment assumption,3295 // so the UBSAN check could function.3296 llvm::ConstantInt *AlignmentCI =3297 cast<llvm::ConstantInt>(EmitScalarExpr(AVAttr->getAlignment()));3298 uint64_t AlignmentInt =3299 AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment);3300 if (AI->getParamAlign().valueOrOne() < AlignmentInt) {3301 AI->removeAttr(llvm::Attribute::AttrKind::Alignment);3302 AI->addAttrs(llvm::AttrBuilder(getLLVMContext())3303 .addAlignmentAttr(llvm::Align(AlignmentInt)));3304 }3305 }3306 }3307 3308 // Set 'noalias' if an argument type has the `restrict` qualifier.3309 if (Arg->getType().isRestrictQualified())3310 AI->addAttr(llvm::Attribute::NoAlias);3311 }3312 3313 // Prepare the argument value. If we have the trivial case, handle it3314 // with no muss and fuss.3315 if (!isa<llvm::StructType>(ArgI.getCoerceToType()) &&3316 ArgI.getCoerceToType() == ConvertType(Ty) &&3317 ArgI.getDirectOffset() == 0) {3318 assert(NumIRArgs == 1);3319 3320 // LLVM expects swifterror parameters to be used in very restricted3321 // ways. Copy the value into a less-restricted temporary.3322 llvm::Value *V = AI;3323 if (FI.getExtParameterInfo(ArgNo).getABI() ==3324 ParameterABI::SwiftErrorResult) {3325 QualType pointeeTy = Ty->getPointeeType();3326 assert(pointeeTy->isPointerType());3327 RawAddress temp =3328 CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");3329 Address arg = makeNaturalAddressForPointer(3330 V, pointeeTy, getContext().getTypeAlignInChars(pointeeTy));3331 llvm::Value *incomingErrorValue = Builder.CreateLoad(arg);3332 Builder.CreateStore(incomingErrorValue, temp);3333 V = temp.getPointer();3334 3335 // Push a cleanup to copy the value back at the end of the function.3336 // The convention does not guarantee that the value will be written3337 // back if the function exits with an unwind exception.3338 EHStack.pushCleanup<CopyBackSwiftError>(NormalCleanup, temp, arg);3339 }3340 3341 // Ensure the argument is the correct type.3342 if (V->getType() != ArgI.getCoerceToType())3343 V = Builder.CreateBitCast(V, ArgI.getCoerceToType());3344 3345 if (isPromoted)3346 V = emitArgumentDemotion(*this, Arg, V);3347 3348 // Because of merging of function types from multiple decls it is3349 // possible for the type of an argument to not match the corresponding3350 // type in the function type. Since we are codegening the callee3351 // in here, add a cast to the argument type.3352 llvm::Type *LTy = ConvertType(Arg->getType());3353 if (V->getType() != LTy)3354 V = Builder.CreateBitCast(V, LTy);3355 3356 ArgVals.push_back(ParamValue::forDirect(V));3357 break;3358 }3359 3360 // VLST arguments are coerced to VLATs at the function boundary for3361 // ABI consistency. If this is a VLST that was coerced to3362 // a VLAT at the function boundary and the types match up, use3363 // llvm.vector.extract to convert back to the original VLST.3364 if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(ConvertType(Ty))) {3365 llvm::Value *ArgVal = Fn->getArg(FirstIRArg);3366 if (auto *VecTyFrom =3367 dyn_cast<llvm::ScalableVectorType>(ArgVal->getType())) {3368 auto [Coerced, Extracted] = CoerceScalableToFixed(3369 *this, VecTyTo, VecTyFrom, ArgVal, Arg->getName());3370 if (Extracted) {3371 assert(NumIRArgs == 1);3372 ArgVals.push_back(ParamValue::forDirect(Coerced));3373 break;3374 }3375 }3376 }3377 3378 llvm::StructType *STy =3379 dyn_cast<llvm::StructType>(ArgI.getCoerceToType());3380 Address Alloca =3381 CreateMemTemp(Ty, getContext().getDeclAlign(Arg), Arg->getName());3382 3383 // Pointer to store into.3384 Address Ptr = emitAddressAtOffset(*this, Alloca, ArgI);3385 3386 // Fast-isel and the optimizer generally like scalar values better than3387 // FCAs, so we flatten them if this is safe to do for this argument.3388 if (ArgI.isDirect() && ArgI.getCanBeFlattened() && STy &&3389 STy->getNumElements() > 1) {3390 llvm::TypeSize StructSize = CGM.getDataLayout().getTypeAllocSize(STy);3391 llvm::TypeSize PtrElementSize =3392 CGM.getDataLayout().getTypeAllocSize(Ptr.getElementType());3393 if (StructSize.isScalable()) {3394 assert(STy->containsHomogeneousScalableVectorTypes() &&3395 "ABI only supports structure with homogeneous scalable vector "3396 "type");3397 assert(StructSize == PtrElementSize &&3398 "Only allow non-fractional movement of structure with"3399 "homogeneous scalable vector type");3400 assert(STy->getNumElements() == NumIRArgs);3401 3402 llvm::Value *LoadedStructValue = llvm::PoisonValue::get(STy);3403 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {3404 auto *AI = Fn->getArg(FirstIRArg + i);3405 AI->setName(Arg->getName() + ".coerce" + Twine(i));3406 LoadedStructValue =3407 Builder.CreateInsertValue(LoadedStructValue, AI, i);3408 }3409 3410 Builder.CreateStore(LoadedStructValue, Ptr);3411 } else {3412 uint64_t SrcSize = StructSize.getFixedValue();3413 uint64_t DstSize = PtrElementSize.getFixedValue();3414 3415 Address AddrToStoreInto = Address::invalid();3416 if (SrcSize <= DstSize) {3417 AddrToStoreInto = Ptr.withElementType(STy);3418 } else {3419 AddrToStoreInto =3420 CreateTempAlloca(STy, Alloca.getAlignment(), "coerce");3421 }3422 3423 assert(STy->getNumElements() == NumIRArgs);3424 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {3425 auto AI = Fn->getArg(FirstIRArg + i);3426 AI->setName(Arg->getName() + ".coerce" + Twine(i));3427 Address EltPtr = Builder.CreateStructGEP(AddrToStoreInto, i);3428 Builder.CreateStore(AI, EltPtr);3429 }3430 3431 if (SrcSize > DstSize) {3432 Builder.CreateMemCpy(Ptr, AddrToStoreInto, DstSize);3433 }3434 }3435 } else {3436 // Simple case, just do a coerced store of the argument into the alloca.3437 assert(NumIRArgs == 1);3438 auto AI = Fn->getArg(FirstIRArg);3439 AI->setName(Arg->getName() + ".coerce");3440 CreateCoercedStore(3441 AI, Ptr,3442 llvm::TypeSize::getFixed(3443 getContext().getTypeSizeInChars(Ty).getQuantity() -3444 ArgI.getDirectOffset()),3445 /*DstIsVolatile=*/false);3446 }3447 3448 // Match to what EmitParmDecl is expecting for this type.3449 if (CodeGenFunction::hasScalarEvaluationKind(Ty)) {3450 llvm::Value *V =3451 EmitLoadOfScalar(Alloca, false, Ty, Arg->getBeginLoc());3452 if (isPromoted)3453 V = emitArgumentDemotion(*this, Arg, V);3454 ArgVals.push_back(ParamValue::forDirect(V));3455 } else {3456 ArgVals.push_back(ParamValue::forIndirect(Alloca));3457 }3458 break;3459 }3460 3461 case ABIArgInfo::CoerceAndExpand: {3462 // Reconstruct into a temporary.3463 Address alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));3464 ArgVals.push_back(ParamValue::forIndirect(alloca));3465 3466 auto coercionType = ArgI.getCoerceAndExpandType();3467 auto unpaddedCoercionType = ArgI.getUnpaddedCoerceAndExpandType();3468 auto *unpaddedStruct = dyn_cast<llvm::StructType>(unpaddedCoercionType);3469 3470 alloca = alloca.withElementType(coercionType);3471 3472 unsigned argIndex = FirstIRArg;3473 unsigned unpaddedIndex = 0;3474 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {3475 llvm::Type *eltType = coercionType->getElementType(i);3476 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))3477 continue;3478 3479 auto eltAddr = Builder.CreateStructGEP(alloca, i);3480 llvm::Value *elt = Fn->getArg(argIndex++);3481 3482 auto paramType = unpaddedStruct3483 ? unpaddedStruct->getElementType(unpaddedIndex++)3484 : unpaddedCoercionType;3485 3486 if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(eltType)) {3487 if (auto *VecTyFrom = dyn_cast<llvm::ScalableVectorType>(paramType)) {3488 bool Extracted;3489 std::tie(elt, Extracted) = CoerceScalableToFixed(3490 *this, VecTyTo, VecTyFrom, elt, elt->getName());3491 assert(Extracted && "Unexpected scalable to fixed vector coercion");3492 }3493 }3494 Builder.CreateStore(elt, eltAddr);3495 }3496 assert(argIndex == FirstIRArg + NumIRArgs);3497 break;3498 }3499 3500 case ABIArgInfo::Expand: {3501 // If this structure was expanded into multiple arguments then3502 // we need to create a temporary and reconstruct it from the3503 // arguments.3504 Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));3505 LValue LV = MakeAddrLValue(Alloca, Ty);3506 ArgVals.push_back(ParamValue::forIndirect(Alloca));3507 3508 auto FnArgIter = Fn->arg_begin() + FirstIRArg;3509 ExpandTypeFromArgs(Ty, LV, FnArgIter);3510 assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs);3511 for (unsigned i = 0, e = NumIRArgs; i != e; ++i) {3512 auto AI = Fn->getArg(FirstIRArg + i);3513 AI->setName(Arg->getName() + "." + Twine(i));3514 }3515 break;3516 }3517 3518 case ABIArgInfo::TargetSpecific: {3519 auto *AI = Fn->getArg(FirstIRArg);3520 AI->setName(Arg->getName() + ".target_coerce");3521 Address Alloca =3522 CreateMemTemp(Ty, getContext().getDeclAlign(Arg), Arg->getName());3523 Address Ptr = emitAddressAtOffset(*this, Alloca, ArgI);3524 CGM.getABIInfo().createCoercedStore(AI, Ptr, ArgI, false, *this);3525 if (CodeGenFunction::hasScalarEvaluationKind(Ty)) {3526 llvm::Value *V =3527 EmitLoadOfScalar(Alloca, false, Ty, Arg->getBeginLoc());3528 if (isPromoted) {3529 V = emitArgumentDemotion(*this, Arg, V);3530 }3531 ArgVals.push_back(ParamValue::forDirect(V));3532 } else {3533 ArgVals.push_back(ParamValue::forIndirect(Alloca));3534 }3535 break;3536 }3537 case ABIArgInfo::Ignore:3538 assert(NumIRArgs == 0);3539 // Initialize the local variable appropriately.3540 if (!hasScalarEvaluationKind(Ty)) {3541 ArgVals.push_back(ParamValue::forIndirect(CreateMemTemp(Ty)));3542 } else {3543 llvm::Value *U = llvm::UndefValue::get(ConvertType(Arg->getType()));3544 ArgVals.push_back(ParamValue::forDirect(U));3545 }3546 break;3547 }3548 }3549 3550 if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {3551 for (int I = Args.size() - 1; I >= 0; --I)3552 EmitParmDecl(*Args[I], ArgVals[I], I + 1);3553 } else {3554 for (unsigned I = 0, E = Args.size(); I != E; ++I)3555 EmitParmDecl(*Args[I], ArgVals[I], I + 1);3556 }3557}3558 3559static void eraseUnusedBitCasts(llvm::Instruction *insn) {3560 while (insn->use_empty()) {3561 llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(insn);3562 if (!bitcast)3563 return;3564 3565 // This is "safe" because we would have used a ConstantExpr otherwise.3566 insn = cast<llvm::Instruction>(bitcast->getOperand(0));3567 bitcast->eraseFromParent();3568 }3569}3570 3571/// Try to emit a fused autorelease of a return result.3572static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,3573 llvm::Value *result) {3574 // We must be immediately followed the cast.3575 llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();3576 if (BB->empty())3577 return nullptr;3578 if (&BB->back() != result)3579 return nullptr;3580 3581 llvm::Type *resultType = result->getType();3582 3583 // result is in a BasicBlock and is therefore an Instruction.3584 llvm::Instruction *generator = cast<llvm::Instruction>(result);3585 3586 SmallVector<llvm::Instruction *, 4> InstsToKill;3587 3588 // Look for:3589 // %generator = bitcast %type1* %generator2 to %type2*3590 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(generator)) {3591 // We would have emitted this as a constant if the operand weren't3592 // an Instruction.3593 generator = cast<llvm::Instruction>(bitcast->getOperand(0));3594 3595 // Require the generator to be immediately followed by the cast.3596 if (generator->getNextNode() != bitcast)3597 return nullptr;3598 3599 InstsToKill.push_back(bitcast);3600 }3601 3602 // Look for:3603 // %generator = call i8* @objc_retain(i8* %originalResult)3604 // or3605 // %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)3606 llvm::CallInst *call = dyn_cast<llvm::CallInst>(generator);3607 if (!call)3608 return nullptr;3609 3610 bool doRetainAutorelease;3611 3612 if (call->getCalledOperand() == CGF.CGM.getObjCEntrypoints().objc_retain) {3613 doRetainAutorelease = true;3614 } else if (call->getCalledOperand() ==3615 CGF.CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue) {3616 doRetainAutorelease = false;3617 3618 // If we emitted an assembly marker for this call (and the3619 // ARCEntrypoints field should have been set if so), go looking3620 // for that call. If we can't find it, we can't do this3621 // optimization. But it should always be the immediately previous3622 // instruction, unless we needed bitcasts around the call.3623 if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) {3624 llvm::Instruction *prev = call->getPrevNode();3625 assert(prev);3626 if (isa<llvm::BitCastInst>(prev)) {3627 prev = prev->getPrevNode();3628 assert(prev);3629 }3630 assert(isa<llvm::CallInst>(prev));3631 assert(cast<llvm::CallInst>(prev)->getCalledOperand() ==3632 CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker);3633 InstsToKill.push_back(prev);3634 }3635 } else {3636 return nullptr;3637 }3638 3639 result = call->getArgOperand(0);3640 InstsToKill.push_back(call);3641 3642 // Keep killing bitcasts, for sanity. Note that we no longer care3643 // about precise ordering as long as there's exactly one use.3644 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(result)) {3645 if (!bitcast->hasOneUse())3646 break;3647 InstsToKill.push_back(bitcast);3648 result = bitcast->getOperand(0);3649 }3650 3651 // Delete all the unnecessary instructions, from latest to earliest.3652 for (auto *I : InstsToKill)3653 I->eraseFromParent();3654 3655 // Do the fused retain/autorelease if we were asked to.3656 if (doRetainAutorelease)3657 result = CGF.EmitARCRetainAutoreleaseReturnValue(result);3658 3659 // Cast back to the result type.3660 return CGF.Builder.CreateBitCast(result, resultType);3661}3662 3663/// If this is a +1 of the value of an immutable 'self', remove it.3664static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,3665 llvm::Value *result) {3666 // This is only applicable to a method with an immutable 'self'.3667 const ObjCMethodDecl *method =3668 dyn_cast_or_null<ObjCMethodDecl>(CGF.CurCodeDecl);3669 if (!method)3670 return nullptr;3671 const VarDecl *self = method->getSelfDecl();3672 if (!self->getType().isConstQualified())3673 return nullptr;3674 3675 // Look for a retain call. Note: stripPointerCasts looks through returned arg3676 // functions, which would cause us to miss the retain.3677 llvm::CallInst *retainCall = dyn_cast<llvm::CallInst>(result);3678 if (!retainCall || retainCall->getCalledOperand() !=3679 CGF.CGM.getObjCEntrypoints().objc_retain)3680 return nullptr;3681 3682 // Look for an ordinary load of 'self'.3683 llvm::Value *retainedValue = retainCall->getArgOperand(0);3684 llvm::LoadInst *load =3685 dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts());3686 if (!load || load->isAtomic() || load->isVolatile() ||3687 load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getBasePointer())3688 return nullptr;3689 3690 // Okay! Burn it all down. This relies for correctness on the3691 // assumption that the retain is emitted as part of the return and3692 // that thereafter everything is used "linearly".3693 llvm::Type *resultType = result->getType();3694 eraseUnusedBitCasts(cast<llvm::Instruction>(result));3695 assert(retainCall->use_empty());3696 retainCall->eraseFromParent();3697 eraseUnusedBitCasts(cast<llvm::Instruction>(retainedValue));3698 3699 return CGF.Builder.CreateBitCast(load, resultType);3700}3701 3702/// Emit an ARC autorelease of the result of a function.3703///3704/// \return the value to actually return from the function3705static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,3706 llvm::Value *result) {3707 // If we're returning 'self', kill the initial retain. This is a3708 // heuristic attempt to "encourage correctness" in the really unfortunate3709 // case where we have a return of self during a dealloc and we desperately3710 // need to avoid the possible autorelease.3711 if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))3712 return self;3713 3714 // At -O0, try to emit a fused retain/autorelease.3715 if (CGF.shouldUseFusedARCCalls())3716 if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))3717 return fused;3718 3719 return CGF.EmitARCAutoreleaseReturnValue(result);3720}3721 3722/// Heuristically search for a dominating store to the return-value slot.3723static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {3724 llvm::Value *ReturnValuePtr = CGF.ReturnValue.getBasePointer();3725 3726 // Check if a User is a store which pointerOperand is the ReturnValue.3727 // We are looking for stores to the ReturnValue, not for stores of the3728 // ReturnValue to some other location.3729 auto GetStoreIfValid = [&CGF,3730 ReturnValuePtr](llvm::User *U) -> llvm::StoreInst * {3731 auto *SI = dyn_cast<llvm::StoreInst>(U);3732 if (!SI || SI->getPointerOperand() != ReturnValuePtr ||3733 SI->getValueOperand()->getType() != CGF.ReturnValue.getElementType())3734 return nullptr;3735 // These aren't actually possible for non-coerced returns, and we3736 // only care about non-coerced returns on this code path.3737 // All memory instructions inside __try block are volatile.3738 assert(!SI->isAtomic() &&3739 (!SI->isVolatile() || CGF.currentFunctionUsesSEHTry()));3740 return SI;3741 };3742 // If there are multiple uses of the return-value slot, just check3743 // for something immediately preceding the IP. Sometimes this can3744 // happen with how we generate implicit-returns; it can also happen3745 // with noreturn cleanups.3746 if (!ReturnValuePtr->hasOneUse()) {3747 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();3748 if (IP->empty())3749 return nullptr;3750 3751 // Look at directly preceding instruction, skipping bitcasts, lifetime3752 // markers, and fake uses and their operands.3753 const llvm::Instruction *LoadIntoFakeUse = nullptr;3754 for (llvm::Instruction &I : llvm::reverse(*IP)) {3755 // Ignore instructions that are just loads for fake uses; the load should3756 // immediately precede the fake use, so we only need to remember the3757 // operand for the last fake use seen.3758 if (LoadIntoFakeUse == &I)3759 continue;3760 if (isa<llvm::BitCastInst>(&I))3761 continue;3762 if (auto *II = dyn_cast<llvm::IntrinsicInst>(&I)) {3763 if (II->getIntrinsicID() == llvm::Intrinsic::lifetime_end)3764 continue;3765 3766 if (II->getIntrinsicID() == llvm::Intrinsic::fake_use) {3767 LoadIntoFakeUse = dyn_cast<llvm::Instruction>(II->getArgOperand(0));3768 continue;3769 }3770 }3771 return GetStoreIfValid(&I);3772 }3773 return nullptr;3774 }3775 3776 llvm::StoreInst *store = GetStoreIfValid(ReturnValuePtr->user_back());3777 if (!store)3778 return nullptr;3779 3780 // Now do a first-and-dirty dominance check: just walk up the3781 // single-predecessors chain from the current insertion point.3782 llvm::BasicBlock *StoreBB = store->getParent();3783 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();3784 llvm::SmallPtrSet<llvm::BasicBlock *, 4> SeenBBs;3785 while (IP != StoreBB) {3786 if (!SeenBBs.insert(IP).second || !(IP = IP->getSinglePredecessor()))3787 return nullptr;3788 }3789 3790 // Okay, the store's basic block dominates the insertion point; we3791 // can do our thing.3792 return store;3793}3794 3795// Helper functions for EmitCMSEClearRecord3796 3797// Set the bits corresponding to a field having width `BitWidth` and located at3798// offset `BitOffset` (from the least significant bit) within a storage unit of3799// `Bits.size()` bytes. Each element of `Bits` corresponds to one target byte.3800// Use little-endian layout, i.e.`Bits[0]` is the LSB.3801static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int BitOffset,3802 int BitWidth, int CharWidth) {3803 assert(CharWidth <= 64);3804 assert(static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth);3805 3806 int Pos = 0;3807 if (BitOffset >= CharWidth) {3808 Pos += BitOffset / CharWidth;3809 BitOffset = BitOffset % CharWidth;3810 }3811 3812 const uint64_t Used = (uint64_t(1) << CharWidth) - 1;3813 if (BitOffset + BitWidth >= CharWidth) {3814 Bits[Pos++] |= (Used << BitOffset) & Used;3815 BitWidth -= CharWidth - BitOffset;3816 BitOffset = 0;3817 }3818 3819 while (BitWidth >= CharWidth) {3820 Bits[Pos++] = Used;3821 BitWidth -= CharWidth;3822 }3823 3824 if (BitWidth > 0)3825 Bits[Pos++] |= (Used >> (CharWidth - BitWidth)) << BitOffset;3826}3827 3828// Set the bits corresponding to a field having width `BitWidth` and located at3829// offset `BitOffset` (from the least significant bit) within a storage unit of3830// `StorageSize` bytes, located at `StorageOffset` in `Bits`. Each element of3831// `Bits` corresponds to one target byte. Use target endian layout.3832static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int StorageOffset,3833 int StorageSize, int BitOffset, int BitWidth,3834 int CharWidth, bool BigEndian) {3835 3836 SmallVector<uint64_t, 8> TmpBits(StorageSize);3837 setBitRange(TmpBits, BitOffset, BitWidth, CharWidth);3838 3839 if (BigEndian)3840 std::reverse(TmpBits.begin(), TmpBits.end());3841 3842 for (uint64_t V : TmpBits)3843 Bits[StorageOffset++] |= V;3844}3845 3846static void setUsedBits(CodeGenModule &, QualType, int,3847 SmallVectorImpl<uint64_t> &);3848 3849// Set the bits in `Bits`, which correspond to the value representations of3850// the actual members of the record type `RTy`. Note that this function does3851// not handle base classes, virtual tables, etc, since they cannot happen in3852// CMSE function arguments or return. The bit mask corresponds to the target3853// memory layout, i.e. it's endian dependent.3854static void setUsedBits(CodeGenModule &CGM, const RecordType *RTy, int Offset,3855 SmallVectorImpl<uint64_t> &Bits) {3856 ASTContext &Context = CGM.getContext();3857 int CharWidth = Context.getCharWidth();3858 const RecordDecl *RD = RTy->getDecl()->getDefinition();3859 const ASTRecordLayout &ASTLayout = Context.getASTRecordLayout(RD);3860 const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(RD);3861 3862 int Idx = 0;3863 for (auto I = RD->field_begin(), E = RD->field_end(); I != E; ++I, ++Idx) {3864 const FieldDecl *F = *I;3865 3866 if (F->isUnnamedBitField() || F->isZeroLengthBitField() ||3867 F->getType()->isIncompleteArrayType())3868 continue;3869 3870 if (F->isBitField()) {3871 const CGBitFieldInfo &BFI = Layout.getBitFieldInfo(F);3872 setBitRange(Bits, Offset + BFI.StorageOffset.getQuantity(),3873 BFI.StorageSize / CharWidth, BFI.Offset, BFI.Size, CharWidth,3874 CGM.getDataLayout().isBigEndian());3875 continue;3876 }3877 3878 setUsedBits(CGM, F->getType(),3879 Offset + ASTLayout.getFieldOffset(Idx) / CharWidth, Bits);3880 }3881}3882 3883// Set the bits in `Bits`, which correspond to the value representations of3884// the elements of an array type `ATy`.3885static void setUsedBits(CodeGenModule &CGM, const ConstantArrayType *ATy,3886 int Offset, SmallVectorImpl<uint64_t> &Bits) {3887 const ASTContext &Context = CGM.getContext();3888 3889 QualType ETy = Context.getBaseElementType(ATy);3890 int Size = Context.getTypeSizeInChars(ETy).getQuantity();3891 SmallVector<uint64_t, 4> TmpBits(Size);3892 setUsedBits(CGM, ETy, 0, TmpBits);3893 3894 for (int I = 0, N = Context.getConstantArrayElementCount(ATy); I < N; ++I) {3895 auto Src = TmpBits.begin();3896 auto Dst = Bits.begin() + Offset + I * Size;3897 for (int J = 0; J < Size; ++J)3898 *Dst++ |= *Src++;3899 }3900}3901 3902// Set the bits in `Bits`, which correspond to the value representations of3903// the type `QTy`.3904static void setUsedBits(CodeGenModule &CGM, QualType QTy, int Offset,3905 SmallVectorImpl<uint64_t> &Bits) {3906 if (const auto *RTy = QTy->getAsCanonical<RecordType>())3907 return setUsedBits(CGM, RTy, Offset, Bits);3908 3909 ASTContext &Context = CGM.getContext();3910 if (const auto *ATy = Context.getAsConstantArrayType(QTy))3911 return setUsedBits(CGM, ATy, Offset, Bits);3912 3913 int Size = Context.getTypeSizeInChars(QTy).getQuantity();3914 if (Size <= 0)3915 return;3916 3917 std::fill_n(Bits.begin() + Offset, Size,3918 (uint64_t(1) << Context.getCharWidth()) - 1);3919}3920 3921static uint64_t buildMultiCharMask(const SmallVectorImpl<uint64_t> &Bits,3922 int Pos, int Size, int CharWidth,3923 bool BigEndian) {3924 assert(Size > 0);3925 uint64_t Mask = 0;3926 if (BigEndian) {3927 for (auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size; P != E;3928 ++P)3929 Mask = (Mask << CharWidth) | *P;3930 } else {3931 auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos;3932 do3933 Mask = (Mask << CharWidth) | *--P;3934 while (P != End);3935 }3936 return Mask;3937}3938 3939// Emit code to clear the bits in a record, which aren't a part of any user3940// declared member, when the record is a function return.3941llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,3942 llvm::IntegerType *ITy,3943 QualType QTy) {3944 assert(Src->getType() == ITy);3945 assert(ITy->getScalarSizeInBits() <= 64);3946 3947 const llvm::DataLayout &DataLayout = CGM.getDataLayout();3948 int Size = DataLayout.getTypeStoreSize(ITy);3949 SmallVector<uint64_t, 4> Bits(Size);3950 setUsedBits(CGM, QTy->castAsCanonical<RecordType>(), 0, Bits);3951 3952 int CharWidth = CGM.getContext().getCharWidth();3953 uint64_t Mask =3954 buildMultiCharMask(Bits, 0, Size, CharWidth, DataLayout.isBigEndian());3955 3956 return Builder.CreateAnd(Src, Mask, "cmse.clear");3957}3958 3959// Emit code to clear the bits in a record, which aren't a part of any user3960// declared member, when the record is a function argument.3961llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,3962 llvm::ArrayType *ATy,3963 QualType QTy) {3964 const llvm::DataLayout &DataLayout = CGM.getDataLayout();3965 int Size = DataLayout.getTypeStoreSize(ATy);3966 SmallVector<uint64_t, 16> Bits(Size);3967 setUsedBits(CGM, QTy->castAsCanonical<RecordType>(), 0, Bits);3968 3969 // Clear each element of the LLVM array.3970 int CharWidth = CGM.getContext().getCharWidth();3971 int CharsPerElt =3972 ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth;3973 int MaskIndex = 0;3974 llvm::Value *R = llvm::PoisonValue::get(ATy);3975 for (int I = 0, N = ATy->getArrayNumElements(); I != N; ++I) {3976 uint64_t Mask = buildMultiCharMask(Bits, MaskIndex, CharsPerElt, CharWidth,3977 DataLayout.isBigEndian());3978 MaskIndex += CharsPerElt;3979 llvm::Value *T0 = Builder.CreateExtractValue(Src, I);3980 llvm::Value *T1 = Builder.CreateAnd(T0, Mask, "cmse.clear");3981 R = Builder.CreateInsertValue(R, T1, I);3982 }3983 3984 return R;3985}3986 3987void CodeGenFunction::EmitFunctionEpilog(3988 const CGFunctionInfo &FI, bool EmitRetDbgLoc, SourceLocation EndLoc,3989 uint64_t RetKeyInstructionsSourceAtom) {3990 if (FI.isNoReturn()) {3991 // Noreturn functions don't return.3992 EmitUnreachable(EndLoc);3993 return;3994 }3995 3996 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>()) {3997 // Naked functions don't have epilogues.3998 Builder.CreateUnreachable();3999 return;4000 }4001 4002 // Functions with no result always return void.4003 if (!ReturnValue.isValid()) {4004 auto *I = Builder.CreateRetVoid();4005 if (RetKeyInstructionsSourceAtom)4006 addInstToSpecificSourceAtom(I, nullptr, RetKeyInstructionsSourceAtom);4007 else4008 addInstToNewSourceAtom(I, nullptr);4009 return;4010 }4011 4012 llvm::DebugLoc RetDbgLoc;4013 llvm::Value *RV = nullptr;4014 QualType RetTy = FI.getReturnType();4015 const ABIArgInfo &RetAI = FI.getReturnInfo();4016 4017 switch (RetAI.getKind()) {4018 case ABIArgInfo::InAlloca:4019 // Aggregates get evaluated directly into the destination. Sometimes we4020 // need to return the sret value in a register, though.4021 assert(hasAggregateEvaluationKind(RetTy));4022 if (RetAI.getInAllocaSRet()) {4023 llvm::Function::arg_iterator EI = CurFn->arg_end();4024 --EI;4025 llvm::Value *ArgStruct = &*EI;4026 llvm::Value *SRet = Builder.CreateStructGEP(4027 FI.getArgStruct(), ArgStruct, RetAI.getInAllocaFieldIndex());4028 llvm::Type *Ty =4029 cast<llvm::GetElementPtrInst>(SRet)->getResultElementType();4030 RV = Builder.CreateAlignedLoad(Ty, SRet, getPointerAlign(), "sret");4031 }4032 break;4033 4034 case ABIArgInfo::Indirect: {4035 auto AI = CurFn->arg_begin();4036 if (RetAI.isSRetAfterThis())4037 ++AI;4038 switch (getEvaluationKind(RetTy)) {4039 case TEK_Complex: {4040 ComplexPairTy RT =4041 EmitLoadOfComplex(MakeAddrLValue(ReturnValue, RetTy), EndLoc);4042 EmitStoreOfComplex(RT, MakeNaturalAlignAddrLValue(&*AI, RetTy),4043 /*isInit*/ true);4044 break;4045 }4046 case TEK_Aggregate:4047 // Do nothing; aggregates get evaluated directly into the destination.4048 break;4049 case TEK_Scalar: {4050 LValueBaseInfo BaseInfo;4051 TBAAAccessInfo TBAAInfo;4052 CharUnits Alignment =4053 CGM.getNaturalTypeAlignment(RetTy, &BaseInfo, &TBAAInfo);4054 Address ArgAddr(&*AI, ConvertType(RetTy), Alignment);4055 LValue ArgVal =4056 LValue::MakeAddr(ArgAddr, RetTy, getContext(), BaseInfo, TBAAInfo);4057 EmitStoreOfScalar(4058 EmitLoadOfScalar(MakeAddrLValue(ReturnValue, RetTy), EndLoc), ArgVal,4059 /*isInit*/ true);4060 break;4061 }4062 }4063 break;4064 }4065 4066 case ABIArgInfo::Extend:4067 case ABIArgInfo::Direct:4068 if (RetAI.getCoerceToType() == ConvertType(RetTy) &&4069 RetAI.getDirectOffset() == 0) {4070 // The internal return value temp always will have pointer-to-return-type4071 // type, just do a load.4072 4073 // If there is a dominating store to ReturnValue, we can elide4074 // the load, zap the store, and usually zap the alloca.4075 if (llvm::StoreInst *SI = findDominatingStoreToReturnValue(*this)) {4076 // Reuse the debug location from the store unless there is4077 // cleanup code to be emitted between the store and return4078 // instruction.4079 if (EmitRetDbgLoc && !AutoreleaseResult)4080 RetDbgLoc = SI->getDebugLoc();4081 // Get the stored value and nuke the now-dead store.4082 RV = SI->getValueOperand();4083 SI->eraseFromParent();4084 4085 // Otherwise, we have to do a simple load.4086 } else {4087 RV = Builder.CreateLoad(ReturnValue);4088 }4089 } else {4090 // If the value is offset in memory, apply the offset now.4091 Address V = emitAddressAtOffset(*this, ReturnValue, RetAI);4092 4093 RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this);4094 }4095 4096 // In ARC, end functions that return a retainable type with a call4097 // to objc_autoreleaseReturnValue.4098 if (AutoreleaseResult) {4099#ifndef NDEBUG4100 // Type::isObjCRetainabletype has to be called on a QualType that hasn't4101 // been stripped of the typedefs, so we cannot use RetTy here. Get the4102 // original return type of FunctionDecl, CurCodeDecl, and BlockDecl from4103 // CurCodeDecl or BlockInfo.4104 QualType RT;4105 4106 if (auto *FD = dyn_cast<FunctionDecl>(CurCodeDecl))4107 RT = FD->getReturnType();4108 else if (auto *MD = dyn_cast<ObjCMethodDecl>(CurCodeDecl))4109 RT = MD->getReturnType();4110 else if (isa<BlockDecl>(CurCodeDecl))4111 RT = BlockInfo->BlockExpression->getFunctionType()->getReturnType();4112 else4113 llvm_unreachable("Unexpected function/method type");4114 4115 assert(getLangOpts().ObjCAutoRefCount && !FI.isReturnsRetained() &&4116 RT->isObjCRetainableType());4117#endif4118 RV = emitAutoreleaseOfResult(*this, RV);4119 }4120 4121 break;4122 4123 case ABIArgInfo::Ignore:4124 break;4125 4126 case ABIArgInfo::CoerceAndExpand: {4127 auto coercionType = RetAI.getCoerceAndExpandType();4128 auto unpaddedCoercionType = RetAI.getUnpaddedCoerceAndExpandType();4129 auto *unpaddedStruct = dyn_cast<llvm::StructType>(unpaddedCoercionType);4130 4131 // Load all of the coerced elements out into results.4132 llvm::SmallVector<llvm::Value *, 4> results;4133 Address addr = ReturnValue.withElementType(coercionType);4134 unsigned unpaddedIndex = 0;4135 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {4136 auto coercedEltType = coercionType->getElementType(i);4137 if (ABIArgInfo::isPaddingForCoerceAndExpand(coercedEltType))4138 continue;4139 4140 auto eltAddr = Builder.CreateStructGEP(addr, i);4141 llvm::Value *elt = CreateCoercedLoad(4142 eltAddr,4143 unpaddedStruct ? unpaddedStruct->getElementType(unpaddedIndex++)4144 : unpaddedCoercionType,4145 *this);4146 results.push_back(elt);4147 }4148 4149 // If we have one result, it's the single direct result type.4150 if (results.size() == 1) {4151 RV = results[0];4152 4153 // Otherwise, we need to make a first-class aggregate.4154 } else {4155 // Construct a return type that lacks padding elements.4156 llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType();4157 4158 RV = llvm::PoisonValue::get(returnType);4159 for (unsigned i = 0, e = results.size(); i != e; ++i) {4160 RV = Builder.CreateInsertValue(RV, results[i], i);4161 }4162 }4163 break;4164 }4165 case ABIArgInfo::TargetSpecific: {4166 Address V = emitAddressAtOffset(*this, ReturnValue, RetAI);4167 RV = CGM.getABIInfo().createCoercedLoad(V, RetAI, *this);4168 break;4169 }4170 case ABIArgInfo::Expand:4171 case ABIArgInfo::IndirectAliased:4172 llvm_unreachable("Invalid ABI kind for return argument");4173 }4174 4175 llvm::Instruction *Ret;4176 if (RV) {4177 if (CurFuncDecl && CurFuncDecl->hasAttr<CmseNSEntryAttr>()) {4178 // For certain return types, clear padding bits, as they may reveal4179 // sensitive information.4180 // Small struct/union types are passed as integers.4181 auto *ITy = dyn_cast<llvm::IntegerType>(RV->getType());4182 if (ITy != nullptr && isa<RecordType>(RetTy.getCanonicalType()))4183 RV = EmitCMSEClearRecord(RV, ITy, RetTy);4184 }4185 EmitReturnValueCheck(RV);4186 Ret = Builder.CreateRet(RV);4187 } else {4188 Ret = Builder.CreateRetVoid();4189 }4190 4191 if (RetDbgLoc)4192 Ret->setDebugLoc(std::move(RetDbgLoc));4193 4194 llvm::Value *Backup = RV ? Ret->getOperand(0) : nullptr;4195 if (RetKeyInstructionsSourceAtom)4196 addInstToSpecificSourceAtom(Ret, Backup, RetKeyInstructionsSourceAtom);4197 else4198 addInstToNewSourceAtom(Ret, Backup);4199}4200 4201void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {4202 // A current decl may not be available when emitting vtable thunks.4203 if (!CurCodeDecl)4204 return;4205 4206 // If the return block isn't reachable, neither is this check, so don't emit4207 // it.4208 if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty())4209 return;4210 4211 ReturnsNonNullAttr *RetNNAttr = nullptr;4212 if (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute))4213 RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>();4214 4215 if (!RetNNAttr && !requiresReturnValueNullabilityCheck())4216 return;4217 4218 // Prefer the returns_nonnull attribute if it's present.4219 SourceLocation AttrLoc;4220 SanitizerKind::SanitizerOrdinal CheckKind;4221 SanitizerHandler Handler;4222 if (RetNNAttr) {4223 assert(!requiresReturnValueNullabilityCheck() &&4224 "Cannot check nullability and the nonnull attribute");4225 AttrLoc = RetNNAttr->getLocation();4226 CheckKind = SanitizerKind::SO_ReturnsNonnullAttribute;4227 Handler = SanitizerHandler::NonnullReturn;4228 } else {4229 if (auto *DD = dyn_cast<DeclaratorDecl>(CurCodeDecl))4230 if (auto *TSI = DD->getTypeSourceInfo())4231 if (auto FTL = TSI->getTypeLoc().getAsAdjusted<FunctionTypeLoc>())4232 AttrLoc = FTL.getReturnLoc().findNullabilityLoc();4233 CheckKind = SanitizerKind::SO_NullabilityReturn;4234 Handler = SanitizerHandler::NullabilityReturn;4235 }4236 4237 SanitizerDebugLocation SanScope(this, {CheckKind}, Handler);4238 4239 // Make sure the "return" source location is valid. If we're checking a4240 // nullability annotation, make sure the preconditions for the check are met.4241 llvm::BasicBlock *Check = createBasicBlock("nullcheck");4242 llvm::BasicBlock *NoCheck = createBasicBlock("no.nullcheck");4243 llvm::Value *SLocPtr = Builder.CreateLoad(ReturnLocation, "return.sloc.load");4244 llvm::Value *CanNullCheck = Builder.CreateIsNotNull(SLocPtr);4245 if (requiresReturnValueNullabilityCheck())4246 CanNullCheck =4247 Builder.CreateAnd(CanNullCheck, RetValNullabilityPrecondition);4248 Builder.CreateCondBr(CanNullCheck, Check, NoCheck);4249 EmitBlock(Check);4250 4251 // Now do the null check.4252 llvm::Value *Cond = Builder.CreateIsNotNull(RV);4253 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(AttrLoc)};4254 llvm::Value *DynamicData[] = {SLocPtr};4255 EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, DynamicData);4256 4257 EmitBlock(NoCheck);4258 4259#ifndef NDEBUG4260 // The return location should not be used after the check has been emitted.4261 ReturnLocation = Address::invalid();4262#endif4263}4264 4265static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) {4266 const CXXRecordDecl *RD = type->getAsCXXRecordDecl();4267 return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;4268}4269 4270static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF, QualType Ty) {4271 // FIXME: Generate IR in one pass, rather than going back and fixing up these4272 // placeholders.4273 llvm::Type *IRTy = CGF.ConvertTypeForMem(Ty);4274 llvm::Type *IRPtrTy = llvm::PointerType::getUnqual(CGF.getLLVMContext());4275 llvm::Value *Placeholder = llvm::PoisonValue::get(IRPtrTy);4276 4277 // FIXME: When we generate this IR in one pass, we shouldn't need4278 // this win32-specific alignment hack.4279 CharUnits Align = CharUnits::fromQuantity(4);4280 Placeholder = CGF.Builder.CreateAlignedLoad(IRPtrTy, Placeholder, Align);4281 4282 return AggValueSlot::forAddr(4283 Address(Placeholder, IRTy, Align), Ty.getQualifiers(),4284 AggValueSlot::IsNotDestructed, AggValueSlot::DoesNotNeedGCBarriers,4285 AggValueSlot::IsNotAliased, AggValueSlot::DoesNotOverlap);4286}4287 4288void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,4289 const VarDecl *param,4290 SourceLocation loc) {4291 // StartFunction converted the ABI-lowered parameter(s) into a4292 // local alloca. We need to turn that into an r-value suitable4293 // for EmitCall.4294 Address local = GetAddrOfLocalVar(param);4295 4296 QualType type = param->getType();4297 4298 // GetAddrOfLocalVar returns a pointer-to-pointer for references,4299 // but the argument needs to be the original pointer.4300 if (type->isReferenceType()) {4301 args.add(RValue::get(Builder.CreateLoad(local)), type);4302 4303 // In ARC, move out of consumed arguments so that the release cleanup4304 // entered by StartFunction doesn't cause an over-release. This isn't4305 // optimal -O0 code generation, but it should get cleaned up when4306 // optimization is enabled. This also assumes that delegate calls are4307 // performed exactly once for a set of arguments, but that should be safe.4308 } else if (getLangOpts().ObjCAutoRefCount &&4309 param->hasAttr<NSConsumedAttr>() && type->isObjCRetainableType()) {4310 llvm::Value *ptr = Builder.CreateLoad(local);4311 auto null =4312 llvm::ConstantPointerNull::get(cast<llvm::PointerType>(ptr->getType()));4313 Builder.CreateStore(null, local);4314 args.add(RValue::get(ptr), type);4315 4316 // For the most part, we just need to load the alloca, except that4317 // aggregate r-values are actually pointers to temporaries.4318 } else {4319 args.add(convertTempToRValue(local, type, loc), type);4320 }4321 4322 // Deactivate the cleanup for the callee-destructed param that was pushed.4323 if (type->isRecordType() && !CurFuncIsThunk &&4324 type->castAsRecordDecl()->isParamDestroyedInCallee() &&4325 param->needsDestruction(getContext())) {4326 EHScopeStack::stable_iterator cleanup =4327 CalleeDestructedParamCleanups.lookup(cast<ParmVarDecl>(param));4328 assert(cleanup.isValid() &&4329 "cleanup for callee-destructed param not recorded");4330 // This unreachable is a temporary marker which will be removed later.4331 llvm::Instruction *isActive = Builder.CreateUnreachable();4332 args.addArgCleanupDeactivation(cleanup, isActive);4333 }4334}4335 4336static bool isProvablyNull(llvm::Value *addr) {4337 return llvm::isa_and_nonnull<llvm::ConstantPointerNull>(addr);4338}4339 4340static bool isProvablyNonNull(Address Addr, CodeGenFunction &CGF) {4341 return llvm::isKnownNonZero(Addr.getBasePointer(), CGF.CGM.getDataLayout());4342}4343 4344/// Emit the actual writing-back of a writeback.4345static void emitWriteback(CodeGenFunction &CGF,4346 const CallArgList::Writeback &writeback) {4347 const LValue &srcLV = writeback.Source;4348 Address srcAddr = srcLV.getAddress();4349 assert(!isProvablyNull(srcAddr.getBasePointer()) &&4350 "shouldn't have writeback for provably null argument");4351 4352 if (writeback.WritebackExpr) {4353 CGF.EmitIgnoredExpr(writeback.WritebackExpr);4354 CGF.EmitLifetimeEnd(writeback.Temporary.getBasePointer());4355 return;4356 }4357 4358 llvm::BasicBlock *contBB = nullptr;4359 4360 // If the argument wasn't provably non-null, we need to null check4361 // before doing the store.4362 bool provablyNonNull = isProvablyNonNull(srcAddr, CGF);4363 4364 if (!provablyNonNull) {4365 llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback");4366 contBB = CGF.createBasicBlock("icr.done");4367 4368 llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull");4369 CGF.Builder.CreateCondBr(isNull, contBB, writebackBB);4370 CGF.EmitBlock(writebackBB);4371 }4372 4373 // Load the value to writeback.4374 llvm::Value *value = CGF.Builder.CreateLoad(writeback.Temporary);4375 4376 // Cast it back, in case we're writing an id to a Foo* or something.4377 value = CGF.Builder.CreateBitCast(value, srcAddr.getElementType(),4378 "icr.writeback-cast");4379 4380 // Perform the writeback.4381 4382 // If we have a "to use" value, it's something we need to emit a use4383 // of. This has to be carefully threaded in: if it's done after the4384 // release it's potentially undefined behavior (and the optimizer4385 // will ignore it), and if it happens before the retain then the4386 // optimizer could move the release there.4387 if (writeback.ToUse) {4388 assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);4389 4390 // Retain the new value. No need to block-copy here: the block's4391 // being passed up the stack.4392 value = CGF.EmitARCRetainNonBlock(value);4393 4394 // Emit the intrinsic use here.4395 CGF.EmitARCIntrinsicUse(writeback.ToUse);4396 4397 // Load the old value (primitively).4398 llvm::Value *oldValue = CGF.EmitLoadOfScalar(srcLV, SourceLocation());4399 4400 // Put the new value in place (primitively).4401 CGF.EmitStoreOfScalar(value, srcLV, /*init*/ false);4402 4403 // Release the old value.4404 CGF.EmitARCRelease(oldValue, srcLV.isARCPreciseLifetime());4405 4406 // Otherwise, we can just do a normal lvalue store.4407 } else {4408 CGF.EmitStoreThroughLValue(RValue::get(value), srcLV);4409 }4410 4411 // Jump to the continuation block.4412 if (!provablyNonNull)4413 CGF.EmitBlock(contBB);4414}4415 4416static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF,4417 const CallArgList &CallArgs) {4418 ArrayRef<CallArgList::CallArgCleanup> Cleanups =4419 CallArgs.getCleanupsToDeactivate();4420 // Iterate in reverse to increase the likelihood of popping the cleanup.4421 for (const auto &I : llvm::reverse(Cleanups)) {4422 CGF.DeactivateCleanupBlock(I.Cleanup, I.IsActiveIP);4423 I.IsActiveIP->eraseFromParent();4424 }4425}4426 4427static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) {4428 if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(E->IgnoreParens()))4429 if (uop->getOpcode() == UO_AddrOf)4430 return uop->getSubExpr();4431 return nullptr;4432}4433 4434/// Emit an argument that's being passed call-by-writeback. That is,4435/// we are passing the address of an __autoreleased temporary; it4436/// might be copy-initialized with the current value of the given4437/// address, but it will definitely be copied out of after the call.4438static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,4439 const ObjCIndirectCopyRestoreExpr *CRE) {4440 LValue srcLV;4441 4442 // Make an optimistic effort to emit the address as an l-value.4443 // This can fail if the argument expression is more complicated.4444 if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(CRE->getSubExpr())) {4445 srcLV = CGF.EmitLValue(lvExpr);4446 4447 // Otherwise, just emit it as a scalar.4448 } else {4449 Address srcAddr = CGF.EmitPointerWithAlignment(CRE->getSubExpr());4450 4451 QualType srcAddrType =4452 CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();4453 srcLV = CGF.MakeAddrLValue(srcAddr, srcAddrType);4454 }4455 Address srcAddr = srcLV.getAddress();4456 4457 // The dest and src types don't necessarily match in LLVM terms4458 // because of the crazy ObjC compatibility rules.4459 4460 llvm::PointerType *destType =4461 cast<llvm::PointerType>(CGF.ConvertType(CRE->getType()));4462 llvm::Type *destElemType =4463 CGF.ConvertTypeForMem(CRE->getType()->getPointeeType());4464 4465 // If the address is a constant null, just pass the appropriate null.4466 if (isProvablyNull(srcAddr.getBasePointer())) {4467 args.add(RValue::get(llvm::ConstantPointerNull::get(destType)),4468 CRE->getType());4469 return;4470 }4471 4472 // Create the temporary.4473 Address temp =4474 CGF.CreateTempAlloca(destElemType, CGF.getPointerAlign(), "icr.temp");4475 // Loading an l-value can introduce a cleanup if the l-value is __weak,4476 // and that cleanup will be conditional if we can't prove that the l-value4477 // isn't null, so we need to register a dominating point so that the cleanups4478 // system will make valid IR.4479 CodeGenFunction::ConditionalEvaluation condEval(CGF);4480 4481 // Zero-initialize it if we're not doing a copy-initialization.4482 bool shouldCopy = CRE->shouldCopy();4483 if (!shouldCopy) {4484 llvm::Value *null =4485 llvm::ConstantPointerNull::get(cast<llvm::PointerType>(destElemType));4486 CGF.Builder.CreateStore(null, temp);4487 }4488 4489 llvm::BasicBlock *contBB = nullptr;4490 llvm::BasicBlock *originBB = nullptr;4491 4492 // If the address is *not* known to be non-null, we need to switch.4493 llvm::Value *finalArgument;4494 4495 bool provablyNonNull = isProvablyNonNull(srcAddr, CGF);4496 4497 if (provablyNonNull) {4498 finalArgument = temp.emitRawPointer(CGF);4499 } else {4500 llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull");4501 4502 finalArgument = CGF.Builder.CreateSelect(4503 isNull, llvm::ConstantPointerNull::get(destType),4504 temp.emitRawPointer(CGF), "icr.argument");4505 4506 // If we need to copy, then the load has to be conditional, which4507 // means we need control flow.4508 if (shouldCopy) {4509 originBB = CGF.Builder.GetInsertBlock();4510 contBB = CGF.createBasicBlock("icr.cont");4511 llvm::BasicBlock *copyBB = CGF.createBasicBlock("icr.copy");4512 CGF.Builder.CreateCondBr(isNull, contBB, copyBB);4513 CGF.EmitBlock(copyBB);4514 condEval.begin(CGF);4515 }4516 }4517 4518 llvm::Value *valueToUse = nullptr;4519 4520 // Perform a copy if necessary.4521 if (shouldCopy) {4522 RValue srcRV = CGF.EmitLoadOfLValue(srcLV, SourceLocation());4523 assert(srcRV.isScalar());4524 4525 llvm::Value *src = srcRV.getScalarVal();4526 src = CGF.Builder.CreateBitCast(src, destElemType, "icr.cast");4527 4528 // Use an ordinary store, not a store-to-lvalue.4529 CGF.Builder.CreateStore(src, temp);4530 4531 // If optimization is enabled, and the value was held in a4532 // __strong variable, we need to tell the optimizer that this4533 // value has to stay alive until we're doing the store back.4534 // This is because the temporary is effectively unretained,4535 // and so otherwise we can violate the high-level semantics.4536 if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 &&4537 srcLV.getObjCLifetime() == Qualifiers::OCL_Strong) {4538 valueToUse = src;4539 }4540 }4541 4542 // Finish the control flow if we needed it.4543 if (shouldCopy && !provablyNonNull) {4544 llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock();4545 CGF.EmitBlock(contBB);4546 4547 // Make a phi for the value to intrinsically use.4548 if (valueToUse) {4549 llvm::PHINode *phiToUse =4550 CGF.Builder.CreatePHI(valueToUse->getType(), 2, "icr.to-use");4551 phiToUse->addIncoming(valueToUse, copyBB);4552 phiToUse->addIncoming(llvm::PoisonValue::get(valueToUse->getType()),4553 originBB);4554 valueToUse = phiToUse;4555 }4556 4557 condEval.end(CGF);4558 }4559 4560 args.addWriteback(srcLV, temp, valueToUse);4561 args.add(RValue::get(finalArgument), CRE->getType());4562}4563 4564void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) {4565 assert(!StackBase);4566 4567 // Save the stack.4568 StackBase = CGF.Builder.CreateStackSave("inalloca.save");4569}4570 4571void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const {4572 if (StackBase) {4573 // Restore the stack after the call.4574 CGF.Builder.CreateStackRestore(StackBase);4575 }4576}4577 4578void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,4579 SourceLocation ArgLoc,4580 AbstractCallee AC, unsigned ParmNum) {4581 if (!AC.getDecl() || !(SanOpts.has(SanitizerKind::NonnullAttribute) ||4582 SanOpts.has(SanitizerKind::NullabilityArg)))4583 return;4584 4585 // The param decl may be missing in a variadic function.4586 auto PVD = ParmNum < AC.getNumParams() ? AC.getParamDecl(ParmNum) : nullptr;4587 unsigned ArgNo = PVD ? PVD->getFunctionScopeIndex() : ParmNum;4588 4589 // Prefer the nonnull attribute if it's present.4590 const NonNullAttr *NNAttr = nullptr;4591 if (SanOpts.has(SanitizerKind::NonnullAttribute))4592 NNAttr = getNonNullAttr(AC.getDecl(), PVD, ArgType, ArgNo);4593 4594 bool CanCheckNullability = false;4595 if (SanOpts.has(SanitizerKind::NullabilityArg) && !NNAttr && PVD &&4596 !PVD->getType()->isRecordType()) {4597 auto Nullability = PVD->getType()->getNullability();4598 CanCheckNullability = Nullability &&4599 *Nullability == NullabilityKind::NonNull &&4600 PVD->getTypeSourceInfo();4601 }4602 4603 if (!NNAttr && !CanCheckNullability)4604 return;4605 4606 SourceLocation AttrLoc;4607 SanitizerKind::SanitizerOrdinal CheckKind;4608 SanitizerHandler Handler;4609 if (NNAttr) {4610 AttrLoc = NNAttr->getLocation();4611 CheckKind = SanitizerKind::SO_NonnullAttribute;4612 Handler = SanitizerHandler::NonnullArg;4613 } else {4614 AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc();4615 CheckKind = SanitizerKind::SO_NullabilityArg;4616 Handler = SanitizerHandler::NullabilityArg;4617 }4618 4619 SanitizerDebugLocation SanScope(this, {CheckKind}, Handler);4620 llvm::Value *Cond = EmitNonNullRValueCheck(RV, ArgType);4621 llvm::Constant *StaticData[] = {4622 EmitCheckSourceLocation(ArgLoc),4623 EmitCheckSourceLocation(AttrLoc),4624 llvm::ConstantInt::get(Int32Ty, ArgNo + 1),4625 };4626 EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, {});4627}4628 4629void CodeGenFunction::EmitNonNullArgCheck(Address Addr, QualType ArgType,4630 SourceLocation ArgLoc,4631 AbstractCallee AC, unsigned ParmNum) {4632 if (!AC.getDecl() || !(SanOpts.has(SanitizerKind::NonnullAttribute) ||4633 SanOpts.has(SanitizerKind::NullabilityArg)))4634 return;4635 4636 EmitNonNullArgCheck(RValue::get(Addr, *this), ArgType, ArgLoc, AC, ParmNum);4637}4638 4639// Check if the call is going to use the inalloca convention. This needs to4640// agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged4641// later, so we can't check it directly.4642static bool hasInAllocaArgs(CodeGenModule &CGM, CallingConv ExplicitCC,4643 ArrayRef<QualType> ArgTypes) {4644 // The Swift calling conventions don't go through the target-specific4645 // argument classification, they never use inalloca.4646 // TODO: Consider limiting inalloca use to only calling conventions supported4647 // by MSVC.4648 if (ExplicitCC == CC_Swift || ExplicitCC == CC_SwiftAsync)4649 return false;4650 if (!CGM.getTarget().getCXXABI().isMicrosoft())4651 return false;4652 return llvm::any_of(ArgTypes, [&](QualType Ty) {4653 return isInAllocaArgument(CGM.getCXXABI(), Ty);4654 });4655}4656 4657#ifndef NDEBUG4658// Determine whether the given argument is an Objective-C method4659// that may have type parameters in its signature.4660static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) {4661 const DeclContext *dc = method->getDeclContext();4662 if (const ObjCInterfaceDecl *classDecl = dyn_cast<ObjCInterfaceDecl>(dc)) {4663 return classDecl->getTypeParamListAsWritten();4664 }4665 4666 if (const ObjCCategoryDecl *catDecl = dyn_cast<ObjCCategoryDecl>(dc)) {4667 return catDecl->getTypeParamList();4668 }4669 4670 return false;4671}4672#endif4673 4674/// EmitCallArgs - Emit call arguments for a function.4675void CodeGenFunction::EmitCallArgs(4676 CallArgList &Args, PrototypeWrapper Prototype,4677 llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,4678 AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) {4679 SmallVector<QualType, 16> ArgTypes;4680 4681 assert((ParamsToSkip == 0 || Prototype.P) &&4682 "Can't skip parameters if type info is not provided");4683 4684 // This variable only captures *explicitly* written conventions, not those4685 // applied by default via command line flags or target defaults, such as4686 // thiscall, aapcs, stdcall via -mrtd, etc. Computing that correctly would4687 // require knowing if this is a C++ instance method or being able to see4688 // unprototyped FunctionTypes.4689 CallingConv ExplicitCC = CC_C;4690 4691 // First, if a prototype was provided, use those argument types.4692 bool IsVariadic = false;4693 if (Prototype.P) {4694 const auto *MD = dyn_cast<const ObjCMethodDecl *>(Prototype.P);4695 if (MD) {4696 IsVariadic = MD->isVariadic();4697 ExplicitCC = getCallingConventionForDecl(4698 MD, CGM.getTarget().getTriple().isOSWindows());4699 ArgTypes.assign(MD->param_type_begin() + ParamsToSkip,4700 MD->param_type_end());4701 } else {4702 const auto *FPT = cast<const FunctionProtoType *>(Prototype.P);4703 IsVariadic = FPT->isVariadic();4704 ExplicitCC = FPT->getExtInfo().getCC();4705 ArgTypes.assign(FPT->param_type_begin() + ParamsToSkip,4706 FPT->param_type_end());4707 }4708 4709#ifndef NDEBUG4710 // Check that the prototyped types match the argument expression types.4711 bool isGenericMethod = MD && isObjCMethodWithTypeParams(MD);4712 CallExpr::const_arg_iterator Arg = ArgRange.begin();4713 for (QualType Ty : ArgTypes) {4714 assert(Arg != ArgRange.end() && "Running over edge of argument list!");4715 assert(4716 (isGenericMethod || Ty->isVariablyModifiedType() ||4717 Ty.getNonReferenceType()->isObjCRetainableType() ||4718 getContext()4719 .getCanonicalType(Ty.getNonReferenceType())4720 .getTypePtr() ==4721 getContext().getCanonicalType((*Arg)->getType()).getTypePtr()) &&4722 "type mismatch in call argument!");4723 ++Arg;4724 }4725 4726 // Either we've emitted all the call args, or we have a call to variadic4727 // function.4728 assert((Arg == ArgRange.end() || IsVariadic) &&4729 "Extra arguments in non-variadic function!");4730#endif4731 }4732 4733 // If we still have any arguments, emit them using the type of the argument.4734 for (auto *A : llvm::drop_begin(ArgRange, ArgTypes.size()))4735 ArgTypes.push_back(IsVariadic ? getVarArgType(A) : A->getType());4736 assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));4737 4738 // We must evaluate arguments from right to left in the MS C++ ABI,4739 // because arguments are destroyed left to right in the callee. As a special4740 // case, there are certain language constructs that require left-to-right4741 // evaluation, and in those cases we consider the evaluation order requirement4742 // to trump the "destruction order is reverse construction order" guarantee.4743 bool LeftToRight =4744 CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()4745 ? Order == EvaluationOrder::ForceLeftToRight4746 : Order != EvaluationOrder::ForceRightToLeft;4747 4748 auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg,4749 RValue EmittedArg) {4750 if (!AC.hasFunctionDecl() || I >= AC.getNumParams())4751 return;4752 auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>();4753 if (PS == nullptr)4754 return;4755 4756 const auto &Context = getContext();4757 auto SizeTy = Context.getSizeType();4758 auto T = Builder.getIntNTy(Context.getTypeSize(SizeTy));4759 assert(EmittedArg.getScalarVal() && "We emitted nothing for the arg?");4760 llvm::Value *V = evaluateOrEmitBuiltinObjectSize(4761 Arg, PS->getType(), T, EmittedArg.getScalarVal(), PS->isDynamic());4762 Args.add(RValue::get(V), SizeTy);4763 // If we're emitting args in reverse, be sure to do so with4764 // pass_object_size, as well.4765 if (!LeftToRight)4766 std::swap(Args.back(), *(&Args.back() - 1));4767 };4768 4769 // Insert a stack save if we're going to need any inalloca args.4770 if (hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) {4771 assert(getTarget().getTriple().getArch() == llvm::Triple::x86 &&4772 "inalloca only supported on x86");4773 Args.allocateArgumentMemory(*this);4774 }4775 4776 // Evaluate each argument in the appropriate order.4777 size_t CallArgsStart = Args.size();4778 for (unsigned I = 0, E = ArgTypes.size(); I != E; ++I) {4779 unsigned Idx = LeftToRight ? I : E - I - 1;4780 CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx;4781 unsigned InitialArgSize = Args.size();4782 // If *Arg is an ObjCIndirectCopyRestoreExpr, check that either the types of4783 // the argument and parameter match or the objc method is parameterized.4784 assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) ||4785 getContext().hasSameUnqualifiedType((*Arg)->getType(),4786 ArgTypes[Idx]) ||4787 (isa<ObjCMethodDecl>(AC.getDecl()) &&4788 isObjCMethodWithTypeParams(cast<ObjCMethodDecl>(AC.getDecl())))) &&4789 "Argument and parameter types don't match");4790 EmitCallArg(Args, *Arg, ArgTypes[Idx]);4791 // In particular, we depend on it being the last arg in Args, and the4792 // objectsize bits depend on there only being one arg if !LeftToRight.4793 assert(InitialArgSize + 1 == Args.size() &&4794 "The code below depends on only adding one arg per EmitCallArg");4795 (void)InitialArgSize;4796 // Since pointer argument are never emitted as LValue, it is safe to emit4797 // non-null argument check for r-value only.4798 if (!Args.back().hasLValue()) {4799 RValue RVArg = Args.back().getKnownRValue();4800 EmitNonNullArgCheck(RVArg, ArgTypes[Idx], (*Arg)->getExprLoc(), AC,4801 ParamsToSkip + Idx);4802 // @llvm.objectsize should never have side-effects and shouldn't need4803 // destruction/cleanups, so we can safely "emit" it after its arg,4804 // regardless of right-to-leftness4805 MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg);4806 }4807 }4808 4809 if (!LeftToRight) {4810 // Un-reverse the arguments we just evaluated so they match up with the LLVM4811 // IR function.4812 std::reverse(Args.begin() + CallArgsStart, Args.end());4813 4814 // Reverse the writebacks to match the MSVC ABI.4815 Args.reverseWritebacks();4816 }4817}4818 4819namespace {4820 4821struct DestroyUnpassedArg final : EHScopeStack::Cleanup {4822 DestroyUnpassedArg(Address Addr, QualType Ty) : Addr(Addr), Ty(Ty) {}4823 4824 Address Addr;4825 QualType Ty;4826 4827 void Emit(CodeGenFunction &CGF, Flags flags) override {4828 QualType::DestructionKind DtorKind = Ty.isDestructedType();4829 if (DtorKind == QualType::DK_cxx_destructor) {4830 const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();4831 assert(!Dtor->isTrivial());4832 CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, /*for vbase*/ false,4833 /*Delegating=*/false, Addr, Ty);4834 } else {4835 CGF.callCStructDestructor(CGF.MakeAddrLValue(Addr, Ty));4836 }4837 }4838};4839 4840} // end anonymous namespace4841 4842RValue CallArg::getRValue(CodeGenFunction &CGF) const {4843 if (!HasLV)4844 return RV;4845 LValue Copy = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty), Ty);4846 CGF.EmitAggregateCopy(Copy, LV, Ty, AggValueSlot::DoesNotOverlap,4847 LV.isVolatile());4848 IsUsed = true;4849 return RValue::getAggregate(Copy.getAddress());4850}4851 4852void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const {4853 LValue Dst = CGF.MakeAddrLValue(Addr, Ty);4854 if (!HasLV && RV.isScalar())4855 CGF.EmitStoreOfScalar(RV.getScalarVal(), Dst, /*isInit=*/true);4856 else if (!HasLV && RV.isComplex())4857 CGF.EmitStoreOfComplex(RV.getComplexVal(), Dst, /*init=*/true);4858 else {4859 auto Addr = HasLV ? LV.getAddress() : RV.getAggregateAddress();4860 LValue SrcLV = CGF.MakeAddrLValue(Addr, Ty);4861 // We assume that call args are never copied into subobjects.4862 CGF.EmitAggregateCopy(Dst, SrcLV, Ty, AggValueSlot::DoesNotOverlap,4863 HasLV ? LV.isVolatileQualified()4864 : RV.isVolatileQualified());4865 }4866 IsUsed = true;4867}4868 4869void CodeGenFunction::EmitWritebacks(const CallArgList &args) {4870 for (const auto &I : args.writebacks())4871 emitWriteback(*this, I);4872}4873 4874void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,4875 QualType type) {4876 std::optional<DisableDebugLocationUpdates> Dis;4877 if (isa<CXXDefaultArgExpr>(E))4878 Dis.emplace(*this);4879 if (const ObjCIndirectCopyRestoreExpr *CRE =4880 dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) {4881 assert(getLangOpts().ObjCAutoRefCount);4882 return emitWritebackArg(*this, args, CRE);4883 }4884 4885 // Add writeback for HLSLOutParamExpr.4886 // Needs to be before the assert below because HLSLOutArgExpr is an LValue4887 // and is not a reference.4888 if (const HLSLOutArgExpr *OE = dyn_cast<HLSLOutArgExpr>(E)) {4889 EmitHLSLOutArgExpr(OE, args, type);4890 return;4891 }4892 4893 assert(type->isReferenceType() == E->isGLValue() &&4894 "reference binding to unmaterialized r-value!");4895 4896 if (E->isGLValue()) {4897 assert(E->getObjectKind() == OK_Ordinary);4898 return args.add(EmitReferenceBindingToExpr(E), type);4899 }4900 4901 bool HasAggregateEvalKind = hasAggregateEvaluationKind(type);4902 4903 // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.4904 // However, we still have to push an EH-only cleanup in case we unwind before4905 // we make it to the call.4906 if (type->isRecordType() &&4907 type->castAsRecordDecl()->isParamDestroyedInCallee()) {4908 // If we're using inalloca, use the argument memory. Otherwise, use a4909 // temporary.4910 AggValueSlot Slot = args.isUsingInAlloca()4911 ? createPlaceholderSlot(*this, type)4912 : CreateAggTemp(type, "agg.tmp");4913 4914 bool DestroyedInCallee = true, NeedsCleanup = true;4915 if (const auto *RD = type->getAsCXXRecordDecl())4916 DestroyedInCallee = RD->hasNonTrivialDestructor();4917 else4918 NeedsCleanup = type.isDestructedType();4919 4920 if (DestroyedInCallee)4921 Slot.setExternallyDestructed();4922 4923 EmitAggExpr(E, Slot);4924 RValue RV = Slot.asRValue();4925 args.add(RV, type);4926 4927 if (DestroyedInCallee && NeedsCleanup) {4928 // Create a no-op GEP between the placeholder and the cleanup so we can4929 // RAUW it successfully. It also serves as a marker of the first4930 // instruction where the cleanup is active.4931 pushFullExprCleanup<DestroyUnpassedArg>(NormalAndEHCleanup,4932 Slot.getAddress(), type);4933 // This unreachable is a temporary marker which will be removed later.4934 llvm::Instruction *IsActive =4935 Builder.CreateFlagLoad(llvm::Constant::getNullValue(Int8PtrTy));4936 args.addArgCleanupDeactivation(EHStack.stable_begin(), IsActive);4937 }4938 return;4939 }4940 4941 if (HasAggregateEvalKind && isa<ImplicitCastExpr>(E) &&4942 cast<CastExpr>(E)->getCastKind() == CK_LValueToRValue &&4943 !type->isArrayParameterType() && !type.isNonTrivialToPrimitiveCopy()) {4944 LValue L = EmitLValue(cast<CastExpr>(E)->getSubExpr());4945 assert(L.isSimple());4946 args.addUncopiedAggregate(L, type);4947 return;4948 }4949 4950 args.add(EmitAnyExprToTemp(E), type);4951}4952 4953QualType CodeGenFunction::getVarArgType(const Expr *Arg) {4954 // System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC4955 // implicitly widens null pointer constants that are arguments to varargs4956 // functions to pointer-sized ints.4957 if (!getTarget().getTriple().isOSWindows())4958 return Arg->getType();4959 4960 if (Arg->getType()->isIntegerType() &&4961 getContext().getTypeSize(Arg->getType()) <4962 getContext().getTargetInfo().getPointerWidth(LangAS::Default) &&4963 Arg->isNullPointerConstant(getContext(),4964 Expr::NPC_ValueDependentIsNotNull)) {4965 return getContext().getIntPtrType();4966 }4967 4968 return Arg->getType();4969}4970 4971// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC4972// optimizer it can aggressively ignore unwind edges.4973void CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {4974 if (CGM.getCodeGenOpts().OptimizationLevel != 0 &&4975 !CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)4976 Inst->setMetadata("clang.arc.no_objc_arc_exceptions",4977 CGM.getNoObjCARCExceptionsMetadata());4978}4979 4980/// Emits a call to the given no-arguments nounwind runtime function.4981llvm::CallInst *4982CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,4983 const llvm::Twine &name) {4984 return EmitNounwindRuntimeCall(callee, ArrayRef<llvm::Value *>(), name);4985}4986 4987/// Emits a call to the given nounwind runtime function.4988llvm::CallInst *4989CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,4990 ArrayRef<Address> args,4991 const llvm::Twine &name) {4992 SmallVector<llvm::Value *, 3> values;4993 for (auto arg : args)4994 values.push_back(arg.emitRawPointer(*this));4995 return EmitNounwindRuntimeCall(callee, values, name);4996}4997 4998llvm::CallInst *4999CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,5000 ArrayRef<llvm::Value *> args,5001 const llvm::Twine &name) {5002 llvm::CallInst *call = EmitRuntimeCall(callee, args, name);5003 call->setDoesNotThrow();5004 return call;5005}5006 5007/// Emits a simple call (never an invoke) to the given no-arguments5008/// runtime function.5009llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,5010 const llvm::Twine &name) {5011 return EmitRuntimeCall(callee, {}, name);5012}5013 5014// Calls which may throw must have operand bundles indicating which funclet5015// they are nested within.5016SmallVector<llvm::OperandBundleDef, 1>5017CodeGenFunction::getBundlesForFunclet(llvm::Value *Callee) {5018 // There is no need for a funclet operand bundle if we aren't inside a5019 // funclet.5020 if (!CurrentFuncletPad)5021 return (SmallVector<llvm::OperandBundleDef, 1>());5022 5023 // Skip intrinsics which cannot throw (as long as they don't lower into5024 // regular function calls in the course of IR transformations).5025 if (auto *CalleeFn = dyn_cast<llvm::Function>(Callee->stripPointerCasts())) {5026 if (CalleeFn->isIntrinsic() && CalleeFn->doesNotThrow()) {5027 auto IID = CalleeFn->getIntrinsicID();5028 if (!llvm::IntrinsicInst::mayLowerToFunctionCall(IID))5029 return (SmallVector<llvm::OperandBundleDef, 1>());5030 }5031 }5032 5033 SmallVector<llvm::OperandBundleDef, 1> BundleList;5034 BundleList.emplace_back("funclet", CurrentFuncletPad);5035 return BundleList;5036}5037 5038/// Emits a simple call (never an invoke) to the given runtime function.5039llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,5040 ArrayRef<llvm::Value *> args,5041 const llvm::Twine &name) {5042 llvm::CallInst *call = Builder.CreateCall(5043 callee, args, getBundlesForFunclet(callee.getCallee()), name);5044 call->setCallingConv(getRuntimeCC());5045 5046 if (CGM.shouldEmitConvergenceTokens() && call->isConvergent())5047 return cast<llvm::CallInst>(addConvergenceControlToken(call));5048 return call;5049}5050 5051/// Emits a call or invoke to the given noreturn runtime function.5052void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(5053 llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args) {5054 SmallVector<llvm::OperandBundleDef, 1> BundleList =5055 getBundlesForFunclet(callee.getCallee());5056 5057 if (getInvokeDest()) {5058 llvm::InvokeInst *invoke = Builder.CreateInvoke(5059 callee, getUnreachableBlock(), getInvokeDest(), args, BundleList);5060 invoke->setDoesNotReturn();5061 invoke->setCallingConv(getRuntimeCC());5062 } else {5063 llvm::CallInst *call = Builder.CreateCall(callee, args, BundleList);5064 call->setDoesNotReturn();5065 call->setCallingConv(getRuntimeCC());5066 Builder.CreateUnreachable();5067 }5068}5069 5070/// Emits a call or invoke instruction to the given nullary runtime function.5071llvm::CallBase *5072CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,5073 const Twine &name) {5074 return EmitRuntimeCallOrInvoke(callee, {}, name);5075}5076 5077/// Emits a call or invoke instruction to the given runtime function.5078llvm::CallBase *5079CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,5080 ArrayRef<llvm::Value *> args,5081 const Twine &name) {5082 llvm::CallBase *call = EmitCallOrInvoke(callee, args, name);5083 call->setCallingConv(getRuntimeCC());5084 return call;5085}5086 5087/// Emits a call or invoke instruction to the given function, depending5088/// on the current state of the EH stack.5089llvm::CallBase *CodeGenFunction::EmitCallOrInvoke(llvm::FunctionCallee Callee,5090 ArrayRef<llvm::Value *> Args,5091 const Twine &Name) {5092 llvm::BasicBlock *InvokeDest = getInvokeDest();5093 SmallVector<llvm::OperandBundleDef, 1> BundleList =5094 getBundlesForFunclet(Callee.getCallee());5095 5096 llvm::CallBase *Inst;5097 if (!InvokeDest)5098 Inst = Builder.CreateCall(Callee, Args, BundleList, Name);5099 else {5100 llvm::BasicBlock *ContBB = createBasicBlock("invoke.cont");5101 Inst = Builder.CreateInvoke(Callee, ContBB, InvokeDest, Args, BundleList,5102 Name);5103 EmitBlock(ContBB);5104 }5105 5106 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC5107 // optimizer it can aggressively ignore unwind edges.5108 if (CGM.getLangOpts().ObjCAutoRefCount)5109 AddObjCARCExceptionMetadata(Inst);5110 5111 return Inst;5112}5113 5114void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,5115 llvm::Value *New) {5116 DeferredReplacements.push_back(5117 std::make_pair(llvm::WeakTrackingVH(Old), New));5118}5119 5120namespace {5121 5122/// Specify given \p NewAlign as the alignment of return value attribute. If5123/// such attribute already exists, re-set it to the maximal one of two options.5124[[nodiscard]] llvm::AttributeList5125maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx,5126 const llvm::AttributeList &Attrs,5127 llvm::Align NewAlign) {5128 llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne();5129 if (CurAlign >= NewAlign)5130 return Attrs;5131 llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Ctx, NewAlign);5132 return Attrs.removeRetAttribute(Ctx, llvm::Attribute::AttrKind::Alignment)5133 .addRetAttribute(Ctx, AlignAttr);5134}5135 5136template <typename AlignedAttrTy> class AbstractAssumeAlignedAttrEmitter {5137protected:5138 CodeGenFunction &CGF;5139 5140 /// We do nothing if this is, or becomes, nullptr.5141 const AlignedAttrTy *AA = nullptr;5142 5143 llvm::Value *Alignment = nullptr; // May or may not be a constant.5144 llvm::ConstantInt *OffsetCI = nullptr; // Constant, hopefully zero.5145 5146 AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)5147 : CGF(CGF_) {5148 if (!FuncDecl)5149 return;5150 AA = FuncDecl->getAttr<AlignedAttrTy>();5151 }5152 5153public:5154 /// If we can, materialize the alignment as an attribute on return value.5155 [[nodiscard]] llvm::AttributeList5156 TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {5157 if (!AA || OffsetCI || CGF.SanOpts.has(SanitizerKind::Alignment))5158 return Attrs;5159 const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);5160 if (!AlignmentCI)5161 return Attrs;5162 // We may legitimately have non-power-of-2 alignment here.5163 // If so, this is UB land, emit it via `@llvm.assume` instead.5164 if (!AlignmentCI->getValue().isPowerOf2())5165 return Attrs;5166 llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(5167 CGF.getLLVMContext(), Attrs,5168 llvm::Align(5169 AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));5170 AA = nullptr; // We're done. Disallow doing anything else.5171 return NewAttrs;5172 }5173 5174 /// Emit alignment assumption.5175 /// This is a general fallback that we take if either there is an offset,5176 /// or the alignment is variable or we are sanitizing for alignment.5177 void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {5178 if (!AA)5179 return;5180 CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,5181 AA->getLocation(), Alignment, OffsetCI);5182 AA = nullptr; // We're done. Disallow doing anything else.5183 }5184};5185 5186/// Helper data structure to emit `AssumeAlignedAttr`.5187class AssumeAlignedAttrEmitter final5188 : public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> {5189public:5190 AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)5191 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {5192 if (!AA)5193 return;5194 // It is guaranteed that the alignment/offset are constants.5195 Alignment = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AA->getAlignment()));5196 if (Expr *Offset = AA->getOffset()) {5197 OffsetCI = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(Offset));5198 if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset.5199 OffsetCI = nullptr;5200 }5201 }5202};5203 5204/// Helper data structure to emit `AllocAlignAttr`.5205class AllocAlignAttrEmitter final5206 : public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> {5207public:5208 AllocAlignAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl,5209 const CallArgList &CallArgs)5210 : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {5211 if (!AA)5212 return;5213 // Alignment may or may not be a constant, and that is okay.5214 Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()]5215 .getRValue(CGF)5216 .getScalarVal();5217 }5218};5219 5220} // namespace5221 5222static unsigned getMaxVectorWidth(const llvm::Type *Ty) {5223 if (auto *VT = dyn_cast<llvm::VectorType>(Ty))5224 return VT->getPrimitiveSizeInBits().getKnownMinValue();5225 if (auto *AT = dyn_cast<llvm::ArrayType>(Ty))5226 return getMaxVectorWidth(AT->getElementType());5227 5228 unsigned MaxVectorWidth = 0;5229 if (auto *ST = dyn_cast<llvm::StructType>(Ty))5230 for (auto *I : ST->elements())5231 MaxVectorWidth = std::max(MaxVectorWidth, getMaxVectorWidth(I));5232 return MaxVectorWidth;5233}5234 5235RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,5236 const CGCallee &Callee,5237 ReturnValueSlot ReturnValue,5238 const CallArgList &CallArgs,5239 llvm::CallBase **callOrInvoke, bool IsMustTail,5240 SourceLocation Loc,5241 bool IsVirtualFunctionPointerThunk) {5242 // FIXME: We no longer need the types from CallArgs; lift up and simplify.5243 5244 assert(Callee.isOrdinary() || Callee.isVirtual());5245 5246 // Handle struct-return functions by passing a pointer to the5247 // location that we would like to return into.5248 QualType RetTy = CallInfo.getReturnType();5249 const ABIArgInfo &RetAI = CallInfo.getReturnInfo();5250 5251 llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(CallInfo);5252 5253 const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl();5254 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {5255 // We can only guarantee that a function is called from the correct5256 // context/function based on the appropriate target attributes,5257 // so only check in the case where we have both always_inline and target5258 // since otherwise we could be making a conditional call after a check for5259 // the proper cpu features (and it won't cause code generation issues due to5260 // function based code generation).5261 if ((TargetDecl->hasAttr<AlwaysInlineAttr>() &&5262 (TargetDecl->hasAttr<TargetAttr>() ||5263 (CurFuncDecl && CurFuncDecl->hasAttr<TargetAttr>()))) ||5264 (CurFuncDecl && CurFuncDecl->hasAttr<FlattenAttr>() &&5265 (CurFuncDecl->hasAttr<TargetAttr>() ||5266 TargetDecl->hasAttr<TargetAttr>())))5267 checkTargetFeatures(Loc, FD);5268 }5269 5270 // Some architectures (such as x86-64) have the ABI changed based on5271 // attribute-target/features. Give them a chance to diagnose.5272 const FunctionDecl *CallerDecl = dyn_cast_or_null<FunctionDecl>(CurCodeDecl);5273 const FunctionDecl *CalleeDecl = dyn_cast_or_null<FunctionDecl>(TargetDecl);5274 CGM.getTargetCodeGenInfo().checkFunctionCallABI(CGM, Loc, CallerDecl,5275 CalleeDecl, CallArgs, RetTy);5276 5277 // 1. Set up the arguments.5278 5279 // If we're using inalloca, insert the allocation after the stack save.5280 // FIXME: Do this earlier rather than hacking it in here!5281 RawAddress ArgMemory = RawAddress::invalid();5282 if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {5283 const llvm::DataLayout &DL = CGM.getDataLayout();5284 llvm::Instruction *IP = CallArgs.getStackBase();5285 llvm::AllocaInst *AI;5286 if (IP) {5287 IP = IP->getNextNode();5288 AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(), "argmem",5289 IP->getIterator());5290 } else {5291 AI = CreateTempAlloca(ArgStruct, "argmem");5292 }5293 auto Align = CallInfo.getArgStructAlignment();5294 AI->setAlignment(Align.getAsAlign());5295 AI->setUsedWithInAlloca(true);5296 assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());5297 ArgMemory = RawAddress(AI, ArgStruct, Align);5298 }5299 5300 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);5301 SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs());5302 5303 // If the call returns a temporary with struct return, create a temporary5304 // alloca to hold the result, unless one is given to us.5305 Address SRetPtr = Address::invalid();5306 bool NeedSRetLifetimeEnd = false;5307 if (RetAI.isIndirect() || RetAI.isInAlloca() || RetAI.isCoerceAndExpand()) {5308 // For virtual function pointer thunks and musttail calls, we must always5309 // forward an incoming SRet pointer to the callee, because a local alloca5310 // would be de-allocated before the call. These cases both guarantee that5311 // there will be an incoming SRet argument of the correct type.5312 if ((IsVirtualFunctionPointerThunk || IsMustTail) && RetAI.isIndirect()) {5313 SRetPtr = makeNaturalAddressForPointer(CurFn->arg_begin() +5314 IRFunctionArgs.getSRetArgNo(),5315 RetTy, CharUnits::fromQuantity(1));5316 } else if (!ReturnValue.isNull()) {5317 SRetPtr = ReturnValue.getAddress();5318 } else {5319 SRetPtr = CreateMemTempWithoutCast(RetTy, "tmp");5320 if (HaveInsertPoint() && ReturnValue.isUnused())5321 NeedSRetLifetimeEnd = EmitLifetimeStart(SRetPtr.getBasePointer());5322 }5323 if (IRFunctionArgs.hasSRetArg()) {5324 // A mismatch between the allocated return value's AS and the target's5325 // chosen IndirectAS can happen e.g. when passing the this pointer through5326 // a chain involving stores to / loads from the DefaultAS; we address this5327 // here, symmetrically with the handling we have for normal pointer args.5328 if (SRetPtr.getAddressSpace() != RetAI.getIndirectAddrSpace()) {5329 llvm::Value *V = SRetPtr.getBasePointer();5330 LangAS SAS = getLangASFromTargetAS(SRetPtr.getAddressSpace());5331 llvm::Type *Ty = llvm::PointerType::get(getLLVMContext(),5332 RetAI.getIndirectAddrSpace());5333 5334 SRetPtr = SRetPtr.withPointer(5335 getTargetHooks().performAddrSpaceCast(*this, V, SAS, Ty, true),5336 SRetPtr.isKnownNonNull());5337 }5338 IRCallArgs[IRFunctionArgs.getSRetArgNo()] =5339 getAsNaturalPointerTo(SRetPtr, RetTy);5340 } else if (RetAI.isInAlloca()) {5341 Address Addr =5342 Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex());5343 Builder.CreateStore(getAsNaturalPointerTo(SRetPtr, RetTy), Addr);5344 }5345 }5346 5347 RawAddress swiftErrorTemp = RawAddress::invalid();5348 Address swiftErrorArg = Address::invalid();5349 5350 // When passing arguments using temporary allocas, we need to add the5351 // appropriate lifetime markers. This vector keeps track of all the lifetime5352 // markers that need to be ended right after the call.5353 SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall;5354 5355 // Translate all of the arguments as necessary to match the IR lowering.5356 assert(CallInfo.arg_size() == CallArgs.size() &&5357 "Mismatch between function signature & arguments.");5358 unsigned ArgNo = 0;5359 CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();5360 for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();5361 I != E; ++I, ++info_it, ++ArgNo) {5362 const ABIArgInfo &ArgInfo = info_it->info;5363 5364 // Insert a padding argument to ensure proper alignment.5365 if (IRFunctionArgs.hasPaddingArg(ArgNo))5366 IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =5367 llvm::UndefValue::get(ArgInfo.getPaddingType());5368 5369 unsigned FirstIRArg, NumIRArgs;5370 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);5371 5372 bool ArgHasMaybeUndefAttr =5373 IsArgumentMaybeUndef(TargetDecl, CallInfo.getNumRequiredArgs(), ArgNo);5374 5375 switch (ArgInfo.getKind()) {5376 case ABIArgInfo::InAlloca: {5377 assert(NumIRArgs == 0);5378 assert(getTarget().getTriple().getArch() == llvm::Triple::x86);5379 if (I->isAggregate()) {5380 RawAddress Addr = I->hasLValue()5381 ? I->getKnownLValue().getAddress()5382 : I->getKnownRValue().getAggregateAddress();5383 llvm::Instruction *Placeholder =5384 cast<llvm::Instruction>(Addr.getPointer());5385 5386 if (!ArgInfo.getInAllocaIndirect()) {5387 // Replace the placeholder with the appropriate argument slot GEP.5388 CGBuilderTy::InsertPoint IP = Builder.saveIP();5389 Builder.SetInsertPoint(Placeholder);5390 Addr = Builder.CreateStructGEP(ArgMemory,5391 ArgInfo.getInAllocaFieldIndex());5392 Builder.restoreIP(IP);5393 } else {5394 // For indirect things such as overaligned structs, replace the5395 // placeholder with a regular aggregate temporary alloca. Store the5396 // address of this alloca into the struct.5397 Addr = CreateMemTemp(info_it->type, "inalloca.indirect.tmp");5398 Address ArgSlot = Builder.CreateStructGEP(5399 ArgMemory, ArgInfo.getInAllocaFieldIndex());5400 Builder.CreateStore(Addr.getPointer(), ArgSlot);5401 }5402 deferPlaceholderReplacement(Placeholder, Addr.getPointer());5403 } else if (ArgInfo.getInAllocaIndirect()) {5404 // Make a temporary alloca and store the address of it into the argument5405 // struct.5406 RawAddress Addr = CreateMemTempWithoutCast(5407 I->Ty, getContext().getTypeAlignInChars(I->Ty),5408 "indirect-arg-temp");5409 I->copyInto(*this, Addr);5410 Address ArgSlot =5411 Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());5412 Builder.CreateStore(Addr.getPointer(), ArgSlot);5413 } else {5414 // Store the RValue into the argument struct.5415 Address Addr =5416 Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());5417 Addr = Addr.withElementType(ConvertTypeForMem(I->Ty));5418 I->copyInto(*this, Addr);5419 }5420 break;5421 }5422 5423 case ABIArgInfo::Indirect:5424 case ABIArgInfo::IndirectAliased: {5425 assert(NumIRArgs == 1);5426 if (I->isAggregate()) {5427 // We want to avoid creating an unnecessary temporary+copy here;5428 // however, we need one in three cases:5429 // 1. If the argument is not byval, and we are required to copy the5430 // source. (This case doesn't occur on any common architecture.)5431 // 2. If the argument is byval, RV is not sufficiently aligned, and5432 // we cannot force it to be sufficiently aligned.5433 // 3. If the argument is byval, but RV is not located in default5434 // or alloca address space.5435 Address Addr = I->hasLValue()5436 ? I->getKnownLValue().getAddress()5437 : I->getKnownRValue().getAggregateAddress();5438 CharUnits Align = ArgInfo.getIndirectAlign();5439 const llvm::DataLayout *TD = &CGM.getDataLayout();5440 5441 assert((FirstIRArg >= IRFuncTy->getNumParams() ||5442 IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() ==5443 TD->getAllocaAddrSpace()) &&5444 "indirect argument must be in alloca address space");5445 5446 bool NeedCopy = false;5447 if (Addr.getAlignment() < Align &&5448 llvm::getOrEnforceKnownAlignment(Addr.emitRawPointer(*this),5449 Align.getAsAlign(),5450 *TD) < Align.getAsAlign()) {5451 NeedCopy = true;5452 } else if (I->hasLValue()) {5453 auto LV = I->getKnownLValue();5454 5455 bool isByValOrRef =5456 ArgInfo.isIndirectAliased() || ArgInfo.getIndirectByVal();5457 5458 if (!isByValOrRef ||5459 (LV.getAlignment() < getContext().getTypeAlignInChars(I->Ty))) {5460 NeedCopy = true;5461 }5462 5463 if (isByValOrRef && Addr.getType()->getAddressSpace() !=5464 ArgInfo.getIndirectAddrSpace()) {5465 NeedCopy = true;5466 }5467 }5468 5469 if (!NeedCopy) {5470 // Skip the extra memcpy call.5471 llvm::Value *V = getAsNaturalPointerTo(Addr, I->Ty);5472 auto *T = llvm::PointerType::get(CGM.getLLVMContext(),5473 ArgInfo.getIndirectAddrSpace());5474 5475 // FIXME: This should not depend on the language address spaces, and5476 // only the contextual values. If the address space mismatches, see if5477 // we can look through a cast to a compatible address space value,5478 // otherwise emit a copy.5479 llvm::Value *Val = getTargetHooks().performAddrSpaceCast(5480 *this, V, I->Ty.getAddressSpace(), T, true);5481 if (ArgHasMaybeUndefAttr)5482 Val = Builder.CreateFreeze(Val);5483 IRCallArgs[FirstIRArg] = Val;5484 break;5485 }5486 } else if (I->getType()->isArrayParameterType()) {5487 // Don't produce a temporary for ArrayParameterType arguments.5488 // ArrayParameterType arguments are only created from5489 // HLSL_ArrayRValue casts and HLSLOutArgExpr expressions, both5490 // of which create temporaries already. This allows us to just use the5491 // scalar for the decayed array pointer as the argument directly.5492 IRCallArgs[FirstIRArg] = I->getKnownRValue().getScalarVal();5493 break;5494 }5495 5496 // For non-aggregate args and aggregate args meeting conditions above5497 // we need to create an aligned temporary, and copy to it.5498 RawAddress AI = CreateMemTempWithoutCast(5499 I->Ty, ArgInfo.getIndirectAlign(), "byval-temp");5500 llvm::Value *Val = getAsNaturalPointerTo(AI, I->Ty);5501 if (ArgHasMaybeUndefAttr)5502 Val = Builder.CreateFreeze(Val);5503 IRCallArgs[FirstIRArg] = Val;5504 5505 // Emit lifetime markers for the temporary alloca and add cleanup code to5506 // emit the end lifetime marker after the call.5507 if (EmitLifetimeStart(AI.getPointer()))5508 CallLifetimeEndAfterCall.emplace_back(AI);5509 5510 // Generate the copy.5511 I->copyInto(*this, AI);5512 break;5513 }5514 5515 case ABIArgInfo::Ignore:5516 assert(NumIRArgs == 0);5517 break;5518 5519 case ABIArgInfo::Extend:5520 case ABIArgInfo::Direct: {5521 if (!isa<llvm::StructType>(ArgInfo.getCoerceToType()) &&5522 ArgInfo.getCoerceToType() == ConvertType(info_it->type) &&5523 ArgInfo.getDirectOffset() == 0) {5524 assert(NumIRArgs == 1);5525 llvm::Value *V;5526 if (!I->isAggregate())5527 V = I->getKnownRValue().getScalarVal();5528 else5529 V = Builder.CreateLoad(5530 I->hasLValue() ? I->getKnownLValue().getAddress()5531 : I->getKnownRValue().getAggregateAddress());5532 5533 // Implement swifterror by copying into a new swifterror argument.5534 // We'll write back in the normal path out of the call.5535 if (CallInfo.getExtParameterInfo(ArgNo).getABI() ==5536 ParameterABI::SwiftErrorResult) {5537 assert(!swiftErrorTemp.isValid() && "multiple swifterror args");5538 5539 QualType pointeeTy = I->Ty->getPointeeType();5540 swiftErrorArg = makeNaturalAddressForPointer(5541 V, pointeeTy, getContext().getTypeAlignInChars(pointeeTy));5542 5543 swiftErrorTemp =5544 CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");5545 V = swiftErrorTemp.getPointer();5546 cast<llvm::AllocaInst>(V)->setSwiftError(true);5547 5548 llvm::Value *errorValue = Builder.CreateLoad(swiftErrorArg);5549 Builder.CreateStore(errorValue, swiftErrorTemp);5550 }5551 5552 // We might have to widen integers, but we should never truncate.5553 if (ArgInfo.getCoerceToType() != V->getType() &&5554 V->getType()->isIntegerTy())5555 V = Builder.CreateZExt(V, ArgInfo.getCoerceToType());5556 5557 // The only plausible mismatch here would be for pointer address spaces.5558 // We assume that the target has a reasonable mapping for the DefaultAS5559 // (it can be casted to from incoming specific ASes), and insert an AS5560 // cast to address the mismatch.5561 if (FirstIRArg < IRFuncTy->getNumParams() &&5562 V->getType() != IRFuncTy->getParamType(FirstIRArg)) {5563 assert(V->getType()->isPointerTy() && "Only pointers can mismatch!");5564 auto ActualAS = I->Ty.getAddressSpace();5565 V = getTargetHooks().performAddrSpaceCast(5566 *this, V, ActualAS, IRFuncTy->getParamType(FirstIRArg));5567 }5568 5569 if (ArgHasMaybeUndefAttr)5570 V = Builder.CreateFreeze(V);5571 IRCallArgs[FirstIRArg] = V;5572 break;5573 }5574 5575 llvm::StructType *STy =5576 dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType());5577 5578 // FIXME: Avoid the conversion through memory if possible.5579 Address Src = Address::invalid();5580 if (!I->isAggregate()) {5581 Src = CreateMemTemp(I->Ty, "coerce");5582 I->copyInto(*this, Src);5583 } else {5584 Src = I->hasLValue() ? I->getKnownLValue().getAddress()5585 : I->getKnownRValue().getAggregateAddress();5586 }5587 5588 // If the value is offset in memory, apply the offset now.5589 Src = emitAddressAtOffset(*this, Src, ArgInfo);5590 5591 // Fast-isel and the optimizer generally like scalar values better than5592 // FCAs, so we flatten them if this is safe to do for this argument.5593 if (STy && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {5594 llvm::Type *SrcTy = Src.getElementType();5595 llvm::TypeSize SrcTypeSize =5596 CGM.getDataLayout().getTypeAllocSize(SrcTy);5597 llvm::TypeSize DstTypeSize = CGM.getDataLayout().getTypeAllocSize(STy);5598 if (SrcTypeSize.isScalable()) {5599 assert(STy->containsHomogeneousScalableVectorTypes() &&5600 "ABI only supports structure with homogeneous scalable vector "5601 "type");5602 assert(SrcTypeSize == DstTypeSize &&5603 "Only allow non-fractional movement of structure with "5604 "homogeneous scalable vector type");5605 assert(NumIRArgs == STy->getNumElements());5606 5607 llvm::Value *StoredStructValue =5608 Builder.CreateLoad(Src, Src.getName() + ".tuple");5609 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {5610 llvm::Value *Extract = Builder.CreateExtractValue(5611 StoredStructValue, i, Src.getName() + ".extract" + Twine(i));5612 IRCallArgs[FirstIRArg + i] = Extract;5613 }5614 } else {5615 uint64_t SrcSize = SrcTypeSize.getFixedValue();5616 uint64_t DstSize = DstTypeSize.getFixedValue();5617 5618 // If the source type is smaller than the destination type of the5619 // coerce-to logic, copy the source value into a temp alloca the size5620 // of the destination type to allow loading all of it. The bits past5621 // the source value are left undef.5622 if (SrcSize < DstSize) {5623 Address TempAlloca = CreateTempAlloca(STy, Src.getAlignment(),5624 Src.getName() + ".coerce");5625 Builder.CreateMemCpy(TempAlloca, Src, SrcSize);5626 Src = TempAlloca;5627 } else {5628 Src = Src.withElementType(STy);5629 }5630 5631 assert(NumIRArgs == STy->getNumElements());5632 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {5633 Address EltPtr = Builder.CreateStructGEP(Src, i);5634 llvm::Value *LI = Builder.CreateLoad(EltPtr);5635 if (ArgHasMaybeUndefAttr)5636 LI = Builder.CreateFreeze(LI);5637 IRCallArgs[FirstIRArg + i] = LI;5638 }5639 }5640 } else {5641 // In the simple case, just pass the coerced loaded value.5642 assert(NumIRArgs == 1);5643 llvm::Value *Load =5644 CreateCoercedLoad(Src, ArgInfo.getCoerceToType(), *this);5645 5646 if (CallInfo.isCmseNSCall()) {5647 // For certain parameter types, clear padding bits, as they may reveal5648 // sensitive information.5649 // Small struct/union types are passed as integer arrays.5650 auto *ATy = dyn_cast<llvm::ArrayType>(Load->getType());5651 if (ATy != nullptr && isa<RecordType>(I->Ty.getCanonicalType()))5652 Load = EmitCMSEClearRecord(Load, ATy, I->Ty);5653 }5654 5655 if (ArgHasMaybeUndefAttr)5656 Load = Builder.CreateFreeze(Load);5657 IRCallArgs[FirstIRArg] = Load;5658 }5659 5660 break;5661 }5662 5663 case ABIArgInfo::CoerceAndExpand: {5664 auto coercionType = ArgInfo.getCoerceAndExpandType();5665 auto layout = CGM.getDataLayout().getStructLayout(coercionType);5666 auto unpaddedCoercionType = ArgInfo.getUnpaddedCoerceAndExpandType();5667 auto *unpaddedStruct = dyn_cast<llvm::StructType>(unpaddedCoercionType);5668 5669 Address addr = Address::invalid();5670 RawAddress AllocaAddr = RawAddress::invalid();5671 bool NeedLifetimeEnd = false;5672 if (I->isAggregate()) {5673 addr = I->hasLValue() ? I->getKnownLValue().getAddress()5674 : I->getKnownRValue().getAggregateAddress();5675 5676 } else {5677 RValue RV = I->getKnownRValue();5678 assert(RV.isScalar()); // complex should always just be direct5679 5680 llvm::Type *scalarType = RV.getScalarVal()->getType();5681 auto scalarAlign = CGM.getDataLayout().getPrefTypeAlign(scalarType);5682 5683 // Materialize to a temporary.5684 addr = CreateTempAlloca(RV.getScalarVal()->getType(),5685 CharUnits::fromQuantity(std::max(5686 layout->getAlignment(), scalarAlign)),5687 "tmp",5688 /*ArraySize=*/nullptr, &AllocaAddr);5689 NeedLifetimeEnd = EmitLifetimeStart(AllocaAddr.getPointer());5690 5691 Builder.CreateStore(RV.getScalarVal(), addr);5692 }5693 5694 addr = addr.withElementType(coercionType);5695 5696 unsigned IRArgPos = FirstIRArg;5697 unsigned unpaddedIndex = 0;5698 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {5699 llvm::Type *eltType = coercionType->getElementType(i);5700 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))5701 continue;5702 Address eltAddr = Builder.CreateStructGEP(addr, i);5703 llvm::Value *elt = CreateCoercedLoad(5704 eltAddr,5705 unpaddedStruct ? unpaddedStruct->getElementType(unpaddedIndex++)5706 : unpaddedCoercionType,5707 *this);5708 if (ArgHasMaybeUndefAttr)5709 elt = Builder.CreateFreeze(elt);5710 IRCallArgs[IRArgPos++] = elt;5711 }5712 assert(IRArgPos == FirstIRArg + NumIRArgs);5713 5714 if (NeedLifetimeEnd)5715 EmitLifetimeEnd(AllocaAddr.getPointer());5716 break;5717 }5718 5719 case ABIArgInfo::Expand: {5720 unsigned IRArgPos = FirstIRArg;5721 ExpandTypeToArgs(I->Ty, *I, IRFuncTy, IRCallArgs, IRArgPos);5722 assert(IRArgPos == FirstIRArg + NumIRArgs);5723 break;5724 }5725 5726 case ABIArgInfo::TargetSpecific: {5727 Address Src = Address::invalid();5728 if (!I->isAggregate()) {5729 Src = CreateMemTemp(I->Ty, "target_coerce");5730 I->copyInto(*this, Src);5731 } else {5732 Src = I->hasLValue() ? I->getKnownLValue().getAddress()5733 : I->getKnownRValue().getAggregateAddress();5734 }5735 5736 // If the value is offset in memory, apply the offset now.5737 Src = emitAddressAtOffset(*this, Src, ArgInfo);5738 llvm::Value *Load =5739 CGM.getABIInfo().createCoercedLoad(Src, ArgInfo, *this);5740 IRCallArgs[FirstIRArg] = Load;5741 break;5742 }5743 }5744 }5745 5746 const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(*this);5747 llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer();5748 5749 // If we're using inalloca, set up that argument.5750 if (ArgMemory.isValid()) {5751 llvm::Value *Arg = ArgMemory.getPointer();5752 assert(IRFunctionArgs.hasInallocaArg());5753 IRCallArgs[IRFunctionArgs.getInallocaArgNo()] = Arg;5754 }5755 5756 // 2. Prepare the function pointer.5757 5758 // If the callee is a bitcast of a non-variadic function to have a5759 // variadic function pointer type, check to see if we can remove the5760 // bitcast. This comes up with unprototyped functions.5761 //5762 // This makes the IR nicer, but more importantly it ensures that we5763 // can inline the function at -O0 if it is marked always_inline.5764 auto simplifyVariadicCallee = [](llvm::FunctionType *CalleeFT,5765 llvm::Value *Ptr) -> llvm::Function * {5766 if (!CalleeFT->isVarArg())5767 return nullptr;5768 5769 // Get underlying value if it's a bitcast5770 if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Ptr)) {5771 if (CE->getOpcode() == llvm::Instruction::BitCast)5772 Ptr = CE->getOperand(0);5773 }5774 5775 llvm::Function *OrigFn = dyn_cast<llvm::Function>(Ptr);5776 if (!OrigFn)5777 return nullptr;5778 5779 llvm::FunctionType *OrigFT = OrigFn->getFunctionType();5780 5781 // If the original type is variadic, or if any of the component types5782 // disagree, we cannot remove the cast.5783 if (OrigFT->isVarArg() ||5784 OrigFT->getNumParams() != CalleeFT->getNumParams() ||5785 OrigFT->getReturnType() != CalleeFT->getReturnType())5786 return nullptr;5787 5788 for (unsigned i = 0, e = OrigFT->getNumParams(); i != e; ++i)5789 if (OrigFT->getParamType(i) != CalleeFT->getParamType(i))5790 return nullptr;5791 5792 return OrigFn;5793 };5794 5795 if (llvm::Function *OrigFn = simplifyVariadicCallee(IRFuncTy, CalleePtr)) {5796 CalleePtr = OrigFn;5797 IRFuncTy = OrigFn->getFunctionType();5798 }5799 5800 // 3. Perform the actual call.5801 5802 // Deactivate any cleanups that we're supposed to do immediately before5803 // the call.5804 if (!CallArgs.getCleanupsToDeactivate().empty())5805 deactivateArgCleanupsBeforeCall(*this, CallArgs);5806 5807 // Update the largest vector width if any arguments have vector types.5808 for (unsigned i = 0; i < IRCallArgs.size(); ++i)5809 LargestVectorWidth = std::max(LargestVectorWidth,5810 getMaxVectorWidth(IRCallArgs[i]->getType()));5811 5812 // Compute the calling convention and attributes.5813 unsigned CallingConv;5814 llvm::AttributeList Attrs;5815 CGM.ConstructAttributeList(CalleePtr->getName(), CallInfo,5816 Callee.getAbstractInfo(), Attrs, CallingConv,5817 /*AttrOnCallSite=*/true,5818 /*IsThunk=*/false);5819 5820 if (CallingConv == llvm::CallingConv::X86_VectorCall &&5821 getTarget().getTriple().isWindowsArm64EC()) {5822 CGM.Error(Loc, "__vectorcall calling convention is not currently "5823 "supported");5824 }5825 5826 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl)) {5827 if (FD->hasAttr<StrictFPAttr>())5828 // All calls within a strictfp function are marked strictfp5829 Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::StrictFP);5830 5831 // If -ffast-math is enabled and the function is guarded by an5832 // '__attribute__((optnone)) adjust the memory attribute so the BE emits the5833 // library call instead of the intrinsic.5834 if (FD->hasAttr<OptimizeNoneAttr>() && getLangOpts().FastMath)5835 CGM.AdjustMemoryAttribute(CalleePtr->getName(), Callee.getAbstractInfo(),5836 Attrs);5837 }5838 // Add call-site nomerge attribute if exists.5839 if (InNoMergeAttributedStmt)5840 Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::NoMerge);5841 5842 // Add call-site noinline attribute if exists.5843 if (InNoInlineAttributedStmt)5844 Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::NoInline);5845 5846 // Add call-site always_inline attribute if exists.5847 // Note: This corresponds to the [[clang::always_inline]] statement attribute.5848 if (InAlwaysInlineAttributedStmt &&5849 !CGM.getTargetCodeGenInfo().wouldInliningViolateFunctionCallABI(5850 CallerDecl, CalleeDecl))5851 Attrs =5852 Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::AlwaysInline);5853 5854 // Remove call-site convergent attribute if requested.5855 if (InNoConvergentAttributedStmt)5856 Attrs =5857 Attrs.removeFnAttribute(getLLVMContext(), llvm::Attribute::Convergent);5858 5859 // Apply some call-site-specific attributes.5860 // TODO: work this into building the attribute set.5861 5862 // Apply always_inline to all calls within flatten functions.5863 // FIXME: should this really take priority over __try, below?5864 if (CurCodeDecl && CurCodeDecl->hasAttr<FlattenAttr>() &&5865 !InNoInlineAttributedStmt &&5866 !(TargetDecl && TargetDecl->hasAttr<NoInlineAttr>()) &&5867 !CGM.getTargetCodeGenInfo().wouldInliningViolateFunctionCallABI(5868 CallerDecl, CalleeDecl)) {5869 Attrs =5870 Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::AlwaysInline);5871 }5872 5873 // Disable inlining inside SEH __try blocks.5874 if (isSEHTryScope()) {5875 Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::NoInline);5876 }5877 5878 // Decide whether to use a call or an invoke.5879 bool CannotThrow;5880 if (currentFunctionUsesSEHTry()) {5881 // SEH cares about asynchronous exceptions, so everything can "throw."5882 CannotThrow = false;5883 } else if (isCleanupPadScope() &&5884 EHPersonality::get(*this).isMSVCXXPersonality()) {5885 // The MSVC++ personality will implicitly terminate the program if an5886 // exception is thrown during a cleanup outside of a try/catch.5887 // We don't need to model anything in IR to get this behavior.5888 CannotThrow = true;5889 } else {5890 // Otherwise, nounwind call sites will never throw.5891 CannotThrow = Attrs.hasFnAttr(llvm::Attribute::NoUnwind);5892 5893 if (auto *FPtr = dyn_cast<llvm::Function>(CalleePtr))5894 if (FPtr->hasFnAttribute(llvm::Attribute::NoUnwind))5895 CannotThrow = true;5896 }5897 5898 // If we made a temporary, be sure to clean up after ourselves. Note that we5899 // can't depend on being inside of an ExprWithCleanups, so we need to manually5900 // pop this cleanup later on. Being eager about this is OK, since this5901 // temporary is 'invisible' outside of the callee.5902 if (NeedSRetLifetimeEnd)5903 pushFullExprCleanup<CallLifetimeEnd>(NormalEHLifetimeMarker, SRetPtr);5904 5905 llvm::BasicBlock *InvokeDest = CannotThrow ? nullptr : getInvokeDest();5906 5907 SmallVector<llvm::OperandBundleDef, 1> BundleList =5908 getBundlesForFunclet(CalleePtr);5909 5910 if (SanOpts.has(SanitizerKind::KCFI) &&5911 !isa_and_nonnull<FunctionDecl>(TargetDecl))5912 EmitKCFIOperandBundle(ConcreteCallee, BundleList);5913 5914 // Add the pointer-authentication bundle.5915 EmitPointerAuthOperandBundle(ConcreteCallee.getPointerAuthInfo(), BundleList);5916 5917 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl))5918 if (FD->hasAttr<StrictFPAttr>())5919 // All calls within a strictfp function are marked strictfp5920 Attrs = Attrs.addFnAttribute(getLLVMContext(), llvm::Attribute::StrictFP);5921 5922 AssumeAlignedAttrEmitter AssumeAlignedAttrEmitter(*this, TargetDecl);5923 Attrs = AssumeAlignedAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);5924 5925 AllocAlignAttrEmitter AllocAlignAttrEmitter(*this, TargetDecl, CallArgs);5926 Attrs = AllocAlignAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);5927 5928 // Emit the actual call/invoke instruction.5929 llvm::CallBase *CI;5930 if (!InvokeDest) {5931 CI = Builder.CreateCall(IRFuncTy, CalleePtr, IRCallArgs, BundleList);5932 } else {5933 llvm::BasicBlock *Cont = createBasicBlock("invoke.cont");5934 CI = Builder.CreateInvoke(IRFuncTy, CalleePtr, Cont, InvokeDest, IRCallArgs,5935 BundleList);5936 EmitBlock(Cont);5937 }5938 if (CI->getCalledFunction() && CI->getCalledFunction()->hasName() &&5939 CI->getCalledFunction()->getName().starts_with("_Z4sqrt")) {5940 SetSqrtFPAccuracy(CI);5941 }5942 if (callOrInvoke) {5943 *callOrInvoke = CI;5944 if (CGM.getCodeGenOpts().CallGraphSection) {5945 QualType CST;5946 if (TargetDecl && TargetDecl->getFunctionType())5947 CST = QualType(TargetDecl->getFunctionType(), 0);5948 else if (const auto *FPT =5949 Callee.getAbstractInfo().getCalleeFunctionProtoType())5950 CST = QualType(FPT, 0);5951 else5952 llvm_unreachable(5953 "Cannot find the callee type to generate callee_type metadata.");5954 5955 // Set type identifier metadata of indirect calls for call graph section.5956 if (!CST.isNull())5957 CGM.createCalleeTypeMetadataForIcall(CST, *callOrInvoke);5958 }5959 }5960 5961 // If this is within a function that has the guard(nocf) attribute and is an5962 // indirect call, add the "guard_nocf" attribute to this call to indicate that5963 // Control Flow Guard checks should not be added, even if the call is inlined.5964 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl)) {5965 if (const auto *A = FD->getAttr<CFGuardAttr>()) {5966 if (A->getGuard() == CFGuardAttr::GuardArg::nocf &&5967 !CI->getCalledFunction())5968 Attrs = Attrs.addFnAttribute(getLLVMContext(), "guard_nocf");5969 }5970 }5971 5972 // Apply the attributes and calling convention.5973 CI->setAttributes(Attrs);5974 CI->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));5975 5976 // Apply various metadata.5977 5978 if (!CI->getType()->isVoidTy())5979 CI->setName("call");5980 5981 if (CGM.shouldEmitConvergenceTokens() && CI->isConvergent())5982 CI = addConvergenceControlToken(CI);5983 5984 // Update largest vector width from the return type.5985 LargestVectorWidth =5986 std::max(LargestVectorWidth, getMaxVectorWidth(CI->getType()));5987 5988 // Insert instrumentation or attach profile metadata at indirect call sites.5989 // For more details, see the comment before the definition of5990 // IPVK_IndirectCallTarget in InstrProfData.inc.5991 if (!CI->getCalledFunction())5992 PGO->valueProfile(Builder, llvm::IPVK_IndirectCallTarget, CI, CalleePtr);5993 5994 // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC5995 // optimizer it can aggressively ignore unwind edges.5996 if (CGM.getLangOpts().ObjCAutoRefCount)5997 AddObjCARCExceptionMetadata(CI);5998 5999 // Set tail call kind if necessary.6000 if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(CI)) {6001 if (TargetDecl && TargetDecl->hasAttr<NotTailCalledAttr>())6002 Call->setTailCallKind(llvm::CallInst::TCK_NoTail);6003 else if (IsMustTail) {6004 if (getTarget().getTriple().isPPC()) {6005 if (getTarget().getTriple().isOSAIX())6006 CGM.getDiags().Report(Loc, diag::err_aix_musttail_unsupported);6007 else if (!getTarget().hasFeature("pcrelative-memops")) {6008 if (getTarget().hasFeature("longcall"))6009 CGM.getDiags().Report(Loc, diag::err_ppc_impossible_musttail) << 0;6010 else if (Call->isIndirectCall())6011 CGM.getDiags().Report(Loc, diag::err_ppc_impossible_musttail) << 1;6012 else if (isa_and_nonnull<FunctionDecl>(TargetDecl)) {6013 if (!cast<FunctionDecl>(TargetDecl)->isDefined())6014 // The undefined callee may be a forward declaration. Without6015 // knowning all symbols in the module, we won't know the symbol is6016 // defined or not. Collect all these symbols for later diagnosing.6017 CGM.addUndefinedGlobalForTailCall(6018 {cast<FunctionDecl>(TargetDecl), Loc});6019 else {6020 llvm::GlobalValue::LinkageTypes Linkage = CGM.getFunctionLinkage(6021 GlobalDecl(cast<FunctionDecl>(TargetDecl)));6022 if (llvm::GlobalValue::isWeakForLinker(Linkage) ||6023 llvm::GlobalValue::isDiscardableIfUnused(Linkage))6024 CGM.getDiags().Report(Loc, diag::err_ppc_impossible_musttail)6025 << 2;6026 }6027 }6028 }6029 }6030 Call->setTailCallKind(llvm::CallInst::TCK_MustTail);6031 }6032 }6033 6034 // Add metadata for calls to MSAllocator functions6035 if (getDebugInfo() && TargetDecl && TargetDecl->hasAttr<MSAllocatorAttr>())6036 getDebugInfo()->addHeapAllocSiteMetadata(CI, RetTy->getPointeeType(), Loc);6037 6038 // Add metadata if calling an __attribute__((error(""))) or warning fn.6039 if (TargetDecl && TargetDecl->hasAttr<ErrorAttr>()) {6040 llvm::ConstantInt *Line =6041 llvm::ConstantInt::get(Int64Ty, Loc.getRawEncoding());6042 llvm::ConstantAsMetadata *MD = llvm::ConstantAsMetadata::get(Line);6043 llvm::MDTuple *MDT = llvm::MDNode::get(getLLVMContext(), {MD});6044 CI->setMetadata("srcloc", MDT);6045 }6046 6047 // 4. Finish the call.6048 6049 // If the call doesn't return, finish the basic block and clear the6050 // insertion point; this allows the rest of IRGen to discard6051 // unreachable code.6052 if (CI->doesNotReturn()) {6053 if (NeedSRetLifetimeEnd)6054 PopCleanupBlock();6055 6056 // Strip away the noreturn attribute to better diagnose unreachable UB.6057 if (SanOpts.has(SanitizerKind::Unreachable)) {6058 // Also remove from function since CallBase::hasFnAttr additionally checks6059 // attributes of the called function.6060 if (auto *F = CI->getCalledFunction())6061 F->removeFnAttr(llvm::Attribute::NoReturn);6062 CI->removeFnAttr(llvm::Attribute::NoReturn);6063 6064 // Avoid incompatibility with ASan which relies on the `noreturn`6065 // attribute to insert handler calls.6066 if (SanOpts.hasOneOf(SanitizerKind::Address |6067 SanitizerKind::KernelAddress)) {6068 SanitizerScope SanScope(this);6069 llvm::IRBuilder<>::InsertPointGuard IPGuard(Builder);6070 Builder.SetInsertPoint(CI);6071 auto *FnType = llvm::FunctionType::get(CGM.VoidTy, /*isVarArg=*/false);6072 llvm::FunctionCallee Fn =6073 CGM.CreateRuntimeFunction(FnType, "__asan_handle_no_return");6074 EmitNounwindRuntimeCall(Fn);6075 }6076 }6077 6078 EmitUnreachable(Loc);6079 Builder.ClearInsertionPoint();6080 6081 // FIXME: For now, emit a dummy basic block because expr emitters in6082 // generally are not ready to handle emitting expressions at unreachable6083 // points.6084 EnsureInsertPoint();6085 6086 // Return a reasonable RValue.6087 return GetUndefRValue(RetTy);6088 }6089 6090 // If this is a musttail call, return immediately. We do not branch to the6091 // epilogue in this case.6092 if (IsMustTail) {6093 for (auto it = EHStack.find(CurrentCleanupScopeDepth); it != EHStack.end();6094 ++it) {6095 EHCleanupScope *Cleanup = dyn_cast<EHCleanupScope>(&*it);6096 // Fake uses can be safely emitted immediately prior to the tail call, so6097 // we choose to emit them just before the call here.6098 if (Cleanup && Cleanup->isFakeUse()) {6099 CGBuilderTy::InsertPointGuard IPG(Builder);6100 Builder.SetInsertPoint(CI);6101 Cleanup->getCleanup()->Emit(*this, EHScopeStack::Cleanup::Flags());6102 } else if (!(Cleanup &&6103 Cleanup->getCleanup()->isRedundantBeforeReturn())) {6104 CGM.ErrorUnsupported(MustTailCall, "tail call skipping over cleanups");6105 }6106 }6107 if (CI->getType()->isVoidTy())6108 Builder.CreateRetVoid();6109 else6110 Builder.CreateRet(CI);6111 Builder.ClearInsertionPoint();6112 EnsureInsertPoint();6113 return GetUndefRValue(RetTy);6114 }6115 6116 // Perform the swifterror writeback.6117 if (swiftErrorTemp.isValid()) {6118 llvm::Value *errorResult = Builder.CreateLoad(swiftErrorTemp);6119 Builder.CreateStore(errorResult, swiftErrorArg);6120 }6121 6122 // Emit any call-associated writebacks immediately. Arguably this6123 // should happen after any return-value munging.6124 if (CallArgs.hasWritebacks())6125 EmitWritebacks(CallArgs);6126 6127 // The stack cleanup for inalloca arguments has to run out of the normal6128 // lexical order, so deactivate it and run it manually here.6129 CallArgs.freeArgumentMemory(*this);6130 6131 // Extract the return value.6132 RValue Ret;6133 6134 // If the current function is a virtual function pointer thunk, avoid copying6135 // the return value of the musttail call to a temporary.6136 if (IsVirtualFunctionPointerThunk) {6137 Ret = RValue::get(CI);6138 } else {6139 Ret = [&] {6140 switch (RetAI.getKind()) {6141 case ABIArgInfo::CoerceAndExpand: {6142 auto coercionType = RetAI.getCoerceAndExpandType();6143 6144 Address addr = SRetPtr.withElementType(coercionType);6145 6146 assert(CI->getType() == RetAI.getUnpaddedCoerceAndExpandType());6147 bool requiresExtract = isa<llvm::StructType>(CI->getType());6148 6149 unsigned unpaddedIndex = 0;6150 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {6151 llvm::Type *eltType = coercionType->getElementType(i);6152 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))6153 continue;6154 Address eltAddr = Builder.CreateStructGEP(addr, i);6155 llvm::Value *elt = CI;6156 if (requiresExtract)6157 elt = Builder.CreateExtractValue(elt, unpaddedIndex++);6158 else6159 assert(unpaddedIndex == 0);6160 Builder.CreateStore(elt, eltAddr);6161 }6162 [[fallthrough]];6163 }6164 6165 case ABIArgInfo::InAlloca:6166 case ABIArgInfo::Indirect: {6167 RValue ret = convertTempToRValue(SRetPtr, RetTy, SourceLocation());6168 if (NeedSRetLifetimeEnd)6169 PopCleanupBlock();6170 return ret;6171 }6172 6173 case ABIArgInfo::Ignore:6174 // If we are ignoring an argument that had a result, make sure to6175 // construct the appropriate return value for our caller.6176 return GetUndefRValue(RetTy);6177 6178 case ABIArgInfo::Extend:6179 case ABIArgInfo::Direct: {6180 llvm::Type *RetIRTy = ConvertType(RetTy);6181 if (RetAI.getCoerceToType() == RetIRTy &&6182 RetAI.getDirectOffset() == 0) {6183 switch (getEvaluationKind(RetTy)) {6184 case TEK_Complex: {6185 llvm::Value *Real = Builder.CreateExtractValue(CI, 0);6186 llvm::Value *Imag = Builder.CreateExtractValue(CI, 1);6187 return RValue::getComplex(std::make_pair(Real, Imag));6188 }6189 case TEK_Aggregate:6190 break;6191 case TEK_Scalar: {6192 // If the argument doesn't match, perform a bitcast to coerce it.6193 // This can happen due to trivial type mismatches.6194 llvm::Value *V = CI;6195 if (V->getType() != RetIRTy)6196 V = Builder.CreateBitCast(V, RetIRTy);6197 return RValue::get(V);6198 }6199 }6200 }6201 6202 // If coercing a fixed vector from a scalable vector for ABI6203 // compatibility, and the types match, use the llvm.vector.extract6204 // intrinsic to perform the conversion.6205 if (auto *FixedDstTy = dyn_cast<llvm::FixedVectorType>(RetIRTy)) {6206 llvm::Value *V = CI;6207 if (auto *ScalableSrcTy =6208 dyn_cast<llvm::ScalableVectorType>(V->getType())) {6209 if (FixedDstTy->getElementType() ==6210 ScalableSrcTy->getElementType()) {6211 V = Builder.CreateExtractVector(FixedDstTy, V, uint64_t(0),6212 "cast.fixed");6213 return RValue::get(V);6214 }6215 }6216 }6217 6218 Address DestPtr = ReturnValue.getValue();6219 bool DestIsVolatile = ReturnValue.isVolatile();6220 uint64_t DestSize =6221 getContext().getTypeInfoDataSizeInChars(RetTy).Width.getQuantity();6222 6223 if (!DestPtr.isValid()) {6224 DestPtr = CreateMemTemp(RetTy, "coerce");6225 DestIsVolatile = false;6226 DestSize = getContext().getTypeSizeInChars(RetTy).getQuantity();6227 }6228 6229 // An empty record can overlap other data (if declared with6230 // no_unique_address); omit the store for such types - as there is no6231 // actual data to store.6232 if (!isEmptyRecord(getContext(), RetTy, true)) {6233 // If the value is offset in memory, apply the offset now.6234 Address StorePtr = emitAddressAtOffset(*this, DestPtr, RetAI);6235 CreateCoercedStore(6236 CI, StorePtr,6237 llvm::TypeSize::getFixed(DestSize - RetAI.getDirectOffset()),6238 DestIsVolatile);6239 }6240 6241 return convertTempToRValue(DestPtr, RetTy, SourceLocation());6242 }6243 6244 case ABIArgInfo::TargetSpecific: {6245 Address DestPtr = ReturnValue.getValue();6246 Address StorePtr = emitAddressAtOffset(*this, DestPtr, RetAI);6247 bool DestIsVolatile = ReturnValue.isVolatile();6248 if (!DestPtr.isValid()) {6249 DestPtr = CreateMemTemp(RetTy, "target_coerce");6250 DestIsVolatile = false;6251 }6252 CGM.getABIInfo().createCoercedStore(CI, StorePtr, RetAI, DestIsVolatile,6253 *this);6254 return convertTempToRValue(DestPtr, RetTy, SourceLocation());6255 }6256 6257 case ABIArgInfo::Expand:6258 case ABIArgInfo::IndirectAliased:6259 llvm_unreachable("Invalid ABI kind for return argument");6260 }6261 6262 llvm_unreachable("Unhandled ABIArgInfo::Kind");6263 }();6264 }6265 6266 // Emit the assume_aligned check on the return value.6267 if (Ret.isScalar() && TargetDecl) {6268 AssumeAlignedAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);6269 AllocAlignAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);6270 }6271 6272 // Explicitly call CallLifetimeEnd::Emit just to re-use the code even though6273 // we can't use the full cleanup mechanism.6274 for (CallLifetimeEnd &LifetimeEnd : CallLifetimeEndAfterCall)6275 LifetimeEnd.Emit(*this, /*Flags=*/{});6276 6277 if (!ReturnValue.isExternallyDestructed() &&6278 RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct)6279 pushDestroy(QualType::DK_nontrivial_c_struct, Ret.getAggregateAddress(),6280 RetTy);6281 6282 // Generate function declaration DISuprogram in order to be used6283 // in debug info about call sites.6284 if (CGDebugInfo *DI = getDebugInfo()) {6285 // Ensure call site info would actually be emitted before collecting6286 // further callee info.6287 if (CalleeDecl && !CalleeDecl->hasAttr<NoDebugAttr>() &&6288 DI->getCallSiteRelatedAttrs() != llvm::DINode::FlagZero) {6289 CodeGenFunction CalleeCGF(CGM);6290 const GlobalDecl &CalleeGlobalDecl =6291 Callee.getAbstractInfo().getCalleeDecl();6292 CalleeCGF.CurGD = CalleeGlobalDecl;6293 FunctionArgList Args;6294 QualType ResTy = CalleeCGF.BuildFunctionArgList(CalleeGlobalDecl, Args);6295 DI->EmitFuncDeclForCallSite(6296 CI, DI->getFunctionType(CalleeDecl, ResTy, Args), CalleeGlobalDecl);6297 }6298 }6299 6300 return Ret;6301}6302 6303CGCallee CGCallee::prepareConcreteCallee(CodeGenFunction &CGF) const {6304 if (isVirtual()) {6305 const CallExpr *CE = getVirtualCallExpr();6306 return CGF.CGM.getCXXABI().getVirtualFunctionPointer(6307 CGF, getVirtualMethodDecl(), getThisAddress(), getVirtualFunctionType(),6308 CE ? CE->getBeginLoc() : SourceLocation());6309 }6310 6311 return *this;6312}6313 6314/* VarArg handling */6315 6316RValue CodeGenFunction::EmitVAArg(VAArgExpr *VE, Address &VAListAddr,6317 AggValueSlot Slot) {6318 VAListAddr = VE->isMicrosoftABI() ? EmitMSVAListRef(VE->getSubExpr())6319 : EmitVAListRef(VE->getSubExpr());6320 QualType Ty = VE->getType();6321 if (Ty->isVariablyModifiedType())6322 EmitVariablyModifiedType(Ty);6323 if (VE->isMicrosoftABI())6324 return CGM.getABIInfo().EmitMSVAArg(*this, VAListAddr, Ty, Slot);6325 return CGM.getABIInfo().EmitVAArg(*this, VAListAddr, Ty, Slot);6326}6327 6328DisableDebugLocationUpdates::DisableDebugLocationUpdates(CodeGenFunction &CGF)6329 : CGF(CGF) {6330 CGF.disableDebugInfo();6331}6332 6333DisableDebugLocationUpdates::~DisableDebugLocationUpdates() {6334 CGF.enableDebugInfo();6335}6336