1378 lines · cpp
1//===- AArch64.cpp --------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "ABIInfoImpl.h"10#include "TargetInfo.h"11#include "clang/AST/Decl.h"12#include "clang/Basic/DiagnosticFrontend.h"13#include "llvm/TargetParser/AArch64TargetParser.h"14 15using namespace clang;16using namespace clang::CodeGen;17 18//===----------------------------------------------------------------------===//19// AArch64 ABI Implementation20//===----------------------------------------------------------------------===//21 22namespace {23 24class AArch64ABIInfo : public ABIInfo {25 AArch64ABIKind Kind;26 27 std::unique_ptr<TargetCodeGenInfo> WinX86_64CodegenInfo;28 29public:30 AArch64ABIInfo(CodeGenModule &CGM, AArch64ABIKind Kind)31 : ABIInfo(CGM.getTypes()), Kind(Kind) {32 if (getTarget().getTriple().isWindowsArm64EC()) {33 WinX86_64CodegenInfo =34 createWinX86_64TargetCodeGenInfo(CGM, X86AVXABILevel::None);35 }36 }37 38 bool isSoftFloat() const { return Kind == AArch64ABIKind::AAPCSSoft; }39 40private:41 AArch64ABIKind getABIKind() const { return Kind; }42 bool isDarwinPCS() const { return Kind == AArch64ABIKind::DarwinPCS; }43 44 ABIArgInfo classifyReturnType(QualType RetTy, bool IsVariadicFn) const;45 ABIArgInfo classifyArgumentType(QualType RetTy, bool IsVariadicFn,46 bool IsNamedArg, unsigned CallingConvention,47 unsigned &NSRN, unsigned &NPRN) const;48 llvm::Type *convertFixedToScalableVectorType(const VectorType *VT) const;49 ABIArgInfo coerceIllegalVector(QualType Ty, unsigned &NSRN,50 unsigned &NPRN) const;51 ABIArgInfo coerceAndExpandPureScalableAggregate(52 QualType Ty, bool IsNamedArg, unsigned NVec, unsigned NPred,53 const SmallVectorImpl<llvm::Type *> &UnpaddedCoerceToSeq, unsigned &NSRN,54 unsigned &NPRN) const;55 bool isHomogeneousAggregateBaseType(QualType Ty) const override;56 bool isHomogeneousAggregateSmallEnough(const Type *Ty,57 uint64_t Members) const override;58 bool isZeroLengthBitfieldPermittedInHomogeneousAggregate() const override;59 60 bool isIllegalVectorType(QualType Ty) const;61 62 bool passAsAggregateType(QualType Ty) const;63 bool passAsPureScalableType(QualType Ty, unsigned &NV, unsigned &NP,64 SmallVectorImpl<llvm::Type *> &CoerceToSeq) const;65 66 void flattenType(llvm::Type *Ty,67 SmallVectorImpl<llvm::Type *> &Flattened) const;68 69 void computeInfo(CGFunctionInfo &FI) const override {70 if (!::classifyReturnType(getCXXABI(), FI, *this))71 FI.getReturnInfo() =72 classifyReturnType(FI.getReturnType(), FI.isVariadic());73 74 unsigned ArgNo = 0;75 unsigned NSRN = 0, NPRN = 0;76 for (auto &it : FI.arguments()) {77 const bool IsNamedArg =78 !FI.isVariadic() || ArgNo < FI.getRequiredArgs().getNumRequiredArgs();79 ++ArgNo;80 it.info = classifyArgumentType(it.type, FI.isVariadic(), IsNamedArg,81 FI.getCallingConvention(), NSRN, NPRN);82 }83 }84 85 RValue EmitDarwinVAArg(Address VAListAddr, QualType Ty, CodeGenFunction &CGF,86 AggValueSlot Slot) const;87 88 RValue EmitAAPCSVAArg(Address VAListAddr, QualType Ty, CodeGenFunction &CGF,89 AArch64ABIKind Kind, AggValueSlot Slot) const;90 91 RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,92 AggValueSlot Slot) const override {93 llvm::Type *BaseTy = CGF.ConvertType(Ty);94 if (isa<llvm::ScalableVectorType>(BaseTy))95 llvm::report_fatal_error("Passing SVE types to variadic functions is "96 "currently not supported");97 98 return Kind == AArch64ABIKind::Win6499 ? EmitMSVAArg(CGF, VAListAddr, Ty, Slot)100 : isDarwinPCS() ? EmitDarwinVAArg(VAListAddr, Ty, CGF, Slot)101 : EmitAAPCSVAArg(VAListAddr, Ty, CGF, Kind, Slot);102 }103 104 RValue EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,105 AggValueSlot Slot) const override;106 107 bool allowBFloatArgsAndRet() const override {108 return getTarget().hasBFloat16Type();109 }110 111 using ABIInfo::appendAttributeMangling;112 void appendAttributeMangling(TargetClonesAttr *Attr, unsigned Index,113 raw_ostream &Out) const override;114 void appendAttributeMangling(StringRef AttrStr,115 raw_ostream &Out) const override;116};117 118class AArch64SwiftABIInfo : public SwiftABIInfo {119public:120 explicit AArch64SwiftABIInfo(CodeGenTypes &CGT)121 : SwiftABIInfo(CGT, /*SwiftErrorInRegister=*/true) {}122 123 bool isLegalVectorType(CharUnits VectorSize, llvm::Type *EltTy,124 unsigned NumElts) const override;125};126 127class AArch64TargetCodeGenInfo : public TargetCodeGenInfo {128public:129 AArch64TargetCodeGenInfo(CodeGenModule &CGM, AArch64ABIKind Kind)130 : TargetCodeGenInfo(std::make_unique<AArch64ABIInfo>(CGM, Kind)) {131 SwiftInfo = std::make_unique<AArch64SwiftABIInfo>(CGM.getTypes());132 }133 134 StringRef getARCRetainAutoreleasedReturnValueMarker() const override {135 return "mov\tfp, fp\t\t// marker for objc_retainAutoreleaseReturnValue";136 }137 138 int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {139 return 31;140 }141 142 bool doesReturnSlotInterfereWithArgs() const override { return false; }143 144 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,145 CodeGen::CodeGenModule &CGM) const override {146 auto *Fn = dyn_cast<llvm::Function>(GV);147 if (!Fn)148 return;149 150 const auto *FD = dyn_cast_or_null<FunctionDecl>(D);151 TargetInfo::BranchProtectionInfo BPI(CGM.getLangOpts());152 153 if (FD && FD->hasAttr<TargetAttr>()) {154 const auto *TA = FD->getAttr<TargetAttr>();155 ParsedTargetAttr Attr =156 CGM.getTarget().parseTargetAttr(TA->getFeaturesStr());157 if (!Attr.BranchProtection.empty()) {158 StringRef Error;159 (void)CGM.getTarget().validateBranchProtection(160 Attr.BranchProtection, Attr.CPU, BPI, CGM.getLangOpts(), Error);161 assert(Error.empty());162 }163 }164 setBranchProtectionFnAttributes(BPI, *Fn);165 setPointerAuthFnAttributes(CGM.getCodeGenOpts().PointerAuth, *Fn);166 }167 168 bool isScalarizableAsmOperand(CodeGen::CodeGenFunction &CGF,169 llvm::Type *Ty) const override {170 if (CGF.getTarget().hasFeature("ls64")) {171 auto *ST = dyn_cast<llvm::StructType>(Ty);172 if (ST && ST->getNumElements() == 1) {173 auto *AT = dyn_cast<llvm::ArrayType>(ST->getElementType(0));174 if (AT && AT->getNumElements() == 8 &&175 AT->getElementType()->isIntegerTy(64))176 return true;177 }178 }179 return TargetCodeGenInfo::isScalarizableAsmOperand(CGF, Ty);180 }181 182 void checkFunctionABI(CodeGenModule &CGM,183 const FunctionDecl *Decl) const override;184 185 void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,186 const FunctionDecl *Caller,187 const FunctionDecl *Callee, const CallArgList &Args,188 QualType ReturnType) const override;189 190 bool wouldInliningViolateFunctionCallABI(191 const FunctionDecl *Caller, const FunctionDecl *Callee) const override;192 193private:194 // Diagnose calls between functions with incompatible Streaming SVE195 // attributes.196 void checkFunctionCallABIStreaming(CodeGenModule &CGM, SourceLocation CallLoc,197 const FunctionDecl *Caller,198 const FunctionDecl *Callee) const;199 // Diagnose calls which must pass arguments in floating-point registers when200 // the selected target does not have floating-point registers.201 void checkFunctionCallABISoftFloat(CodeGenModule &CGM, SourceLocation CallLoc,202 const FunctionDecl *Caller,203 const FunctionDecl *Callee,204 const CallArgList &Args,205 QualType ReturnType) const;206};207 208class WindowsAArch64TargetCodeGenInfo : public AArch64TargetCodeGenInfo {209public:210 WindowsAArch64TargetCodeGenInfo(CodeGenModule &CGM, AArch64ABIKind K)211 : AArch64TargetCodeGenInfo(CGM, K) {}212 213 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,214 CodeGen::CodeGenModule &CGM) const override;215 216 void getDependentLibraryOption(llvm::StringRef Lib,217 llvm::SmallString<24> &Opt) const override {218 Opt = "/DEFAULTLIB:" + qualifyWindowsLibrary(Lib);219 }220 221 void getDetectMismatchOption(llvm::StringRef Name, llvm::StringRef Value,222 llvm::SmallString<32> &Opt) const override {223 Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";224 }225};226 227void WindowsAArch64TargetCodeGenInfo::setTargetAttributes(228 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {229 AArch64TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);230 if (GV->isDeclaration())231 return;232 addStackProbeTargetAttributes(D, GV, CGM);233}234}235 236llvm::Type *237AArch64ABIInfo::convertFixedToScalableVectorType(const VectorType *VT) const {238 assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");239 240 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {241 assert(VT->getElementType()->castAs<BuiltinType>()->getKind() ==242 BuiltinType::UChar &&243 "unexpected builtin type for SVE predicate!");244 return llvm::ScalableVectorType::get(llvm::Type::getInt1Ty(getVMContext()),245 16);246 }247 248 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {249 const auto *BT = VT->getElementType()->castAs<BuiltinType>();250 switch (BT->getKind()) {251 default:252 llvm_unreachable("unexpected builtin type for SVE vector!");253 254 case BuiltinType::SChar:255 case BuiltinType::UChar:256 case BuiltinType::MFloat8:257 return llvm::ScalableVectorType::get(258 llvm::Type::getInt8Ty(getVMContext()), 16);259 260 case BuiltinType::Short:261 case BuiltinType::UShort:262 return llvm::ScalableVectorType::get(263 llvm::Type::getInt16Ty(getVMContext()), 8);264 265 case BuiltinType::Int:266 case BuiltinType::UInt:267 return llvm::ScalableVectorType::get(268 llvm::Type::getInt32Ty(getVMContext()), 4);269 270 case BuiltinType::Long:271 case BuiltinType::ULong:272 return llvm::ScalableVectorType::get(273 llvm::Type::getInt64Ty(getVMContext()), 2);274 275 case BuiltinType::Half:276 return llvm::ScalableVectorType::get(277 llvm::Type::getHalfTy(getVMContext()), 8);278 279 case BuiltinType::Float:280 return llvm::ScalableVectorType::get(281 llvm::Type::getFloatTy(getVMContext()), 4);282 283 case BuiltinType::Double:284 return llvm::ScalableVectorType::get(285 llvm::Type::getDoubleTy(getVMContext()), 2);286 287 case BuiltinType::BFloat16:288 return llvm::ScalableVectorType::get(289 llvm::Type::getBFloatTy(getVMContext()), 8);290 }291 }292 293 llvm_unreachable("expected fixed-length SVE vector");294}295 296ABIArgInfo AArch64ABIInfo::coerceIllegalVector(QualType Ty, unsigned &NSRN,297 unsigned &NPRN) const {298 assert(Ty->isVectorType() && "expected vector type!");299 300 const auto *VT = Ty->castAs<VectorType>();301 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {302 assert(VT->getElementType()->isBuiltinType() && "expected builtin type!");303 assert(VT->getElementType()->castAs<BuiltinType>()->getKind() ==304 BuiltinType::UChar &&305 "unexpected builtin type for SVE predicate!");306 NPRN = std::min(NPRN + 1, 4u);307 return ABIArgInfo::getDirect(llvm::ScalableVectorType::get(308 llvm::Type::getInt1Ty(getVMContext()), 16));309 }310 311 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {312 NSRN = std::min(NSRN + 1, 8u);313 return ABIArgInfo::getDirect(convertFixedToScalableVectorType(VT));314 }315 316 uint64_t Size = getContext().getTypeSize(Ty);317 // Android promotes <2 x i8> to i16, not i32318 if ((isAndroid() || isOHOSFamily()) && (Size <= 16)) {319 llvm::Type *ResType = llvm::Type::getInt16Ty(getVMContext());320 return ABIArgInfo::getDirect(ResType);321 }322 if (Size <= 32) {323 llvm::Type *ResType = llvm::Type::getInt32Ty(getVMContext());324 return ABIArgInfo::getDirect(ResType);325 }326 if (Size == 64) {327 NSRN = std::min(NSRN + 1, 8u);328 auto *ResType =329 llvm::FixedVectorType::get(llvm::Type::getInt32Ty(getVMContext()), 2);330 return ABIArgInfo::getDirect(ResType);331 }332 if (Size == 128) {333 NSRN = std::min(NSRN + 1, 8u);334 auto *ResType =335 llvm::FixedVectorType::get(llvm::Type::getInt32Ty(getVMContext()), 4);336 return ABIArgInfo::getDirect(ResType);337 }338 339 return getNaturalAlignIndirect(Ty, getDataLayout().getAllocaAddrSpace(),340 /*ByVal=*/false);341}342 343ABIArgInfo AArch64ABIInfo::coerceAndExpandPureScalableAggregate(344 QualType Ty, bool IsNamedArg, unsigned NVec, unsigned NPred,345 const SmallVectorImpl<llvm::Type *> &UnpaddedCoerceToSeq, unsigned &NSRN,346 unsigned &NPRN) const {347 if (!IsNamedArg || NSRN + NVec > 8 || NPRN + NPred > 4)348 return getNaturalAlignIndirect(Ty, getDataLayout().getAllocaAddrSpace(),349 /*ByVal=*/false);350 NSRN += NVec;351 NPRN += NPred;352 353 // Handle SVE vector tuples.354 if (Ty->isSVESizelessBuiltinType())355 return ABIArgInfo::getDirect();356 357 llvm::Type *UnpaddedCoerceToType =358 UnpaddedCoerceToSeq.size() == 1359 ? UnpaddedCoerceToSeq[0]360 : llvm::StructType::get(CGT.getLLVMContext(), UnpaddedCoerceToSeq,361 true);362 363 SmallVector<llvm::Type *> CoerceToSeq;364 flattenType(CGT.ConvertType(Ty), CoerceToSeq);365 auto *CoerceToType =366 llvm::StructType::get(CGT.getLLVMContext(), CoerceToSeq, false);367 368 return ABIArgInfo::getCoerceAndExpand(CoerceToType, UnpaddedCoerceToType);369}370 371ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadicFn,372 bool IsNamedArg,373 unsigned CallingConvention,374 unsigned &NSRN,375 unsigned &NPRN) const {376 Ty = useFirstFieldIfTransparentUnion(Ty);377 378 if (IsVariadicFn && getTarget().getTriple().isWindowsArm64EC()) {379 // Arm64EC varargs functions use the x86_64 classification rules,380 // not the AArch64 ABI rules.381 return WinX86_64CodegenInfo->getABIInfo().classifyArgForArm64ECVarArg(Ty);382 }383 384 // Handle illegal vector types here.385 if (isIllegalVectorType(Ty))386 return coerceIllegalVector(Ty, NSRN, NPRN);387 388 if (!passAsAggregateType(Ty)) {389 // Treat an enum type as its underlying type.390 if (const auto *ED = Ty->getAsEnumDecl())391 Ty = ED->getIntegerType();392 393 if (const auto *EIT = Ty->getAs<BitIntType>())394 if (EIT->getNumBits() > 128)395 return getNaturalAlignIndirect(Ty, getDataLayout().getAllocaAddrSpace(),396 false);397 398 if (Ty->isVectorType())399 NSRN = std::min(NSRN + 1, 8u);400 else if (const auto *BT = Ty->getAs<BuiltinType>()) {401 if (BT->isFloatingPoint())402 NSRN = std::min(NSRN + 1, 8u);403 else {404 switch (BT->getKind()) {405 case BuiltinType::SveBool:406 case BuiltinType::SveCount:407 NPRN = std::min(NPRN + 1, 4u);408 break;409 case BuiltinType::SveBoolx2:410 NPRN = std::min(NPRN + 2, 4u);411 break;412 case BuiltinType::SveBoolx4:413 NPRN = std::min(NPRN + 4, 4u);414 break;415 default:416 if (BT->isSVESizelessBuiltinType())417 NSRN = std::min(418 NSRN + getContext().getBuiltinVectorTypeInfo(BT).NumVectors,419 8u);420 }421 }422 }423 424 return (isPromotableIntegerTypeForABI(Ty) && isDarwinPCS()425 ? ABIArgInfo::getExtend(Ty, CGT.ConvertType(Ty))426 : ABIArgInfo::getDirect());427 }428 429 // Structures with either a non-trivial destructor or a non-trivial430 // copy constructor are always indirect.431 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) {432 return getNaturalAlignIndirect(433 Ty, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),434 /*ByVal=*/RAA == CGCXXABI::RAA_DirectInMemory);435 }436 437 // Empty records:438 // AAPCS64 does not say that empty records are ignored as arguments,439 // but other compilers do so in certain situations, and we copy that behavior.440 // Those situations are in fact language-mode-specific, which seems really441 // unfortunate, but it's something we just have to accept. If this doesn't442 // apply, just fall through to the standard argument-handling path.443 // Darwin overrides the psABI here to ignore all empty records in all modes.444 uint64_t Size = getContext().getTypeSize(Ty);445 bool IsEmpty = isEmptyRecord(getContext(), Ty, true);446 if (!Ty->isSVESizelessBuiltinType() && (IsEmpty || Size == 0)) {447 // Empty records are ignored in C mode, and in C++ on Darwin.448 if (!getContext().getLangOpts().CPlusPlus || isDarwinPCS())449 return ABIArgInfo::getIgnore();450 451 // In C++ mode, arguments which have sizeof() == 0 (which are non-standard452 // C++) are ignored. This isn't defined by any standard, so we copy GCC's453 // behaviour here.454 if (Size == 0)455 return ABIArgInfo::getIgnore();456 }457 458 // Homogeneous Floating-point Aggregates (HFAs) need to be expanded.459 const Type *Base = nullptr;460 uint64_t Members = 0;461 bool IsWin64 = Kind == AArch64ABIKind::Win64 ||462 CallingConvention == llvm::CallingConv::Win64;463 bool IsWinVariadic = IsWin64 && IsVariadicFn;464 // In variadic functions on Windows, all composite types are treated alike,465 // no special handling of HFAs/HVAs.466 if (!IsWinVariadic && isHomogeneousAggregate(Ty, Base, Members)) {467 NSRN = std::min(NSRN + Members, uint64_t(8));468 if (Kind != AArch64ABIKind::AAPCS)469 return ABIArgInfo::getDirect(470 llvm::ArrayType::get(CGT.ConvertType(QualType(Base, 0)), Members));471 472 // For HFAs/HVAs, cap the argument alignment to 16, otherwise473 // set it to 8 according to the AAPCS64 document.474 unsigned Align =475 getContext().getTypeUnadjustedAlignInChars(Ty).getQuantity();476 Align = (Align >= 16) ? 16 : 8;477 return ABIArgInfo::getDirect(478 llvm::ArrayType::get(CGT.ConvertType(QualType(Base, 0)), Members), 0,479 nullptr, true, Align);480 }481 482 // In AAPCS named arguments of a Pure Scalable Type are passed expanded in483 // registers, or indirectly if there are not enough registers.484 if (Kind == AArch64ABIKind::AAPCS) {485 unsigned NVec = 0, NPred = 0;486 SmallVector<llvm::Type *> UnpaddedCoerceToSeq;487 if (passAsPureScalableType(Ty, NVec, NPred, UnpaddedCoerceToSeq) &&488 (NVec + NPred) > 0)489 return coerceAndExpandPureScalableAggregate(490 Ty, IsNamedArg, NVec, NPred, UnpaddedCoerceToSeq, NSRN, NPRN);491 }492 493 // Aggregates <= 16 bytes are passed directly in registers or on the stack.494 if (Size <= 128) {495 unsigned Alignment;496 if (Kind == AArch64ABIKind::AAPCS) {497 Alignment = getContext().getTypeUnadjustedAlign(Ty);498 Alignment = Alignment < 128 ? 64 : 128;499 } else {500 Alignment =501 std::max(getContext().getTypeAlign(Ty),502 (unsigned)getTarget().getPointerWidth(LangAS::Default));503 }504 Size = llvm::alignTo(Size, Alignment);505 506 // If the Aggregate is made up of pointers, use an array of pointers for the507 // coerced type. This prevents having to convert ptr2int->int2ptr through508 // the call, allowing alias analysis to produce better code.509 auto ContainsOnlyPointers = [&](const auto &Self, QualType Ty) {510 if (isEmptyRecord(getContext(), Ty, true))511 return false;512 const auto *RD = Ty->getAsRecordDecl();513 if (!RD)514 return false;515 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {516 for (const auto &I : CXXRD->bases())517 if (!Self(Self, I.getType()))518 return false;519 }520 return all_of(RD->fields(), [&](FieldDecl *FD) {521 QualType FDTy = FD->getType();522 if (FDTy->isArrayType())523 FDTy = getContext().getBaseElementType(FDTy);524 return (FDTy->isPointerOrReferenceType() &&525 getContext().getTypeSize(FDTy) == 64 &&526 !FDTy->getPointeeType().hasAddressSpace()) ||527 Self(Self, FDTy);528 });529 };530 531 // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.532 // For aggregates with 16-byte alignment, we use i128.533 llvm::Type *BaseTy = llvm::Type::getIntNTy(getVMContext(), Alignment);534 if ((Size == 64 || Size == 128) && Alignment == 64 &&535 ContainsOnlyPointers(ContainsOnlyPointers, Ty))536 BaseTy = llvm::PointerType::getUnqual(getVMContext());537 return ABIArgInfo::getDirect(538 Size == Alignment ? BaseTy539 : llvm::ArrayType::get(BaseTy, Size / Alignment));540 }541 542 return getNaturalAlignIndirect(Ty, getDataLayout().getAllocaAddrSpace(),543 /*ByVal=*/false);544}545 546ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy,547 bool IsVariadicFn) const {548 if (RetTy->isVoidType())549 return ABIArgInfo::getIgnore();550 551 if (const auto *VT = RetTy->getAs<VectorType>()) {552 if (VT->getVectorKind() == VectorKind::SveFixedLengthData ||553 VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {554 unsigned NSRN = 0, NPRN = 0;555 return coerceIllegalVector(RetTy, NSRN, NPRN);556 }557 }558 559 // Large vector types should be returned via memory.560 if (RetTy->isVectorType() && getContext().getTypeSize(RetTy) > 128)561 return getNaturalAlignIndirect(RetTy, getDataLayout().getAllocaAddrSpace());562 563 if (!passAsAggregateType(RetTy)) {564 // Treat an enum type as its underlying type.565 if (const auto *ED = RetTy->getAsEnumDecl())566 RetTy = ED->getIntegerType();567 568 if (const auto *EIT = RetTy->getAs<BitIntType>())569 if (EIT->getNumBits() > 128)570 return getNaturalAlignIndirect(RetTy,571 getDataLayout().getAllocaAddrSpace());572 573 return (isPromotableIntegerTypeForABI(RetTy) && isDarwinPCS()574 ? ABIArgInfo::getExtend(RetTy)575 : ABIArgInfo::getDirect());576 }577 578 uint64_t Size = getContext().getTypeSize(RetTy);579 if (!RetTy->isSVESizelessBuiltinType() &&580 (isEmptyRecord(getContext(), RetTy, true) || Size == 0))581 return ABIArgInfo::getIgnore();582 583 const Type *Base = nullptr;584 uint64_t Members = 0;585 if (isHomogeneousAggregate(RetTy, Base, Members) &&586 !(getTarget().getTriple().getArch() == llvm::Triple::aarch64_32 &&587 IsVariadicFn))588 // Homogeneous Floating-point Aggregates (HFAs) are returned directly.589 return ABIArgInfo::getDirect();590 591 // In AAPCS return values of a Pure Scalable type are treated as a single592 // named argument and passed expanded in registers, or indirectly if there are593 // not enough registers.594 if (Kind == AArch64ABIKind::AAPCS) {595 unsigned NSRN = 0, NPRN = 0;596 unsigned NVec = 0, NPred = 0;597 SmallVector<llvm::Type *> UnpaddedCoerceToSeq;598 if (passAsPureScalableType(RetTy, NVec, NPred, UnpaddedCoerceToSeq) &&599 (NVec + NPred) > 0)600 return coerceAndExpandPureScalableAggregate(601 RetTy, /* IsNamedArg */ true, NVec, NPred, UnpaddedCoerceToSeq, NSRN,602 NPRN);603 }604 605 // Aggregates <= 16 bytes are returned directly in registers or on the stack.606 if (Size <= 128) {607 if (Size <= 64 && getDataLayout().isLittleEndian()) {608 // Composite types are returned in lower bits of a 64-bit register for LE,609 // and in higher bits for BE. However, integer types are always returned610 // in lower bits for both LE and BE, and they are not rounded up to611 // 64-bits. We can skip rounding up of composite types for LE, but not for612 // BE, otherwise composite types will be indistinguishable from integer613 // types.614 return ABIArgInfo::getDirect(615 llvm::IntegerType::get(getVMContext(), Size));616 }617 618 unsigned Alignment = getContext().getTypeAlign(RetTy);619 Size = llvm::alignTo(Size, 64); // round up to multiple of 8 bytes620 621 // We use a pair of i64 for 16-byte aggregate with 8-byte alignment.622 // For aggregates with 16-byte alignment, we use i128.623 if (Alignment < 128 && Size == 128) {624 llvm::Type *BaseTy = llvm::Type::getInt64Ty(getVMContext());625 return ABIArgInfo::getDirect(llvm::ArrayType::get(BaseTy, Size / 64));626 }627 return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Size));628 }629 630 return getNaturalAlignIndirect(RetTy, getDataLayout().getAllocaAddrSpace());631}632 633/// isIllegalVectorType - check whether the vector type is legal for AArch64.634bool AArch64ABIInfo::isIllegalVectorType(QualType Ty) const {635 if (const VectorType *VT = Ty->getAs<VectorType>()) {636 // Check whether VT is a fixed-length SVE vector. These types are637 // represented as scalable vectors in function args/return and must be638 // coerced from fixed vectors.639 if (VT->getVectorKind() == VectorKind::SveFixedLengthData ||640 VT->getVectorKind() == VectorKind::SveFixedLengthPredicate)641 return true;642 643 // Check whether VT is legal.644 unsigned NumElements = VT->getNumElements();645 uint64_t Size = getContext().getTypeSize(VT);646 // NumElements should be power of 2.647 if (!llvm::isPowerOf2_32(NumElements))648 return true;649 650 // arm64_32 has to be compatible with the ARM logic here, which allows huge651 // vectors for some reason.652 llvm::Triple Triple = getTarget().getTriple();653 if (Triple.getArch() == llvm::Triple::aarch64_32 &&654 Triple.isOSBinFormatMachO())655 return Size <= 32;656 657 return Size != 64 && (Size != 128 || NumElements == 1);658 }659 return false;660}661 662bool AArch64SwiftABIInfo::isLegalVectorType(CharUnits VectorSize,663 llvm::Type *EltTy,664 unsigned NumElts) const {665 if (!llvm::isPowerOf2_32(NumElts))666 return false;667 if (VectorSize.getQuantity() != 8 &&668 (VectorSize.getQuantity() != 16 || NumElts == 1))669 return false;670 return true;671}672 673bool AArch64ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {674 // For the soft-float ABI variant, no types are considered to be homogeneous675 // aggregates.676 if (isSoftFloat())677 return false;678 679 // Homogeneous aggregates for AAPCS64 must have base types of a floating680 // point type or a short-vector type. This is the same as the 32-bit ABI,681 // but with the difference that any floating-point type is allowed,682 // including __fp16.683 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {684 if (BT->isFloatingPoint())685 return true;686 } else if (const VectorType *VT = Ty->getAs<VectorType>()) {687 if (auto Kind = VT->getVectorKind();688 Kind == VectorKind::SveFixedLengthData ||689 Kind == VectorKind::SveFixedLengthPredicate)690 return false;691 692 unsigned VecSize = getContext().getTypeSize(VT);693 if (VecSize == 64 || VecSize == 128)694 return true;695 }696 return false;697}698 699bool AArch64ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base,700 uint64_t Members) const {701 return Members <= 4;702}703 704bool AArch64ABIInfo::isZeroLengthBitfieldPermittedInHomogeneousAggregate()705 const {706 // AAPCS64 says that the rule for whether something is a homogeneous707 // aggregate is applied to the output of the data layout decision. So708 // anything that doesn't affect the data layout also does not affect709 // homogeneity. In particular, zero-length bitfields don't stop a struct710 // being homogeneous.711 return true;712}713 714bool AArch64ABIInfo::passAsAggregateType(QualType Ty) const {715 if (Kind == AArch64ABIKind::AAPCS && Ty->isSVESizelessBuiltinType()) {716 const auto *BT = Ty->castAs<BuiltinType>();717 return !BT->isSVECount() &&718 getContext().getBuiltinVectorTypeInfo(BT).NumVectors > 1;719 }720 return isAggregateTypeForABI(Ty);721}722 723// Check if a type needs to be passed in registers as a Pure Scalable Type (as724// defined by AAPCS64). Return the number of data vectors and the number of725// predicate vectors in the type, into `NVec` and `NPred`, respectively. Upon726// return `CoerceToSeq` contains an expanded sequence of LLVM IR types, one727// element for each non-composite member. For practical purposes, limit the728// length of `CoerceToSeq` to about 12 (the maximum that could possibly fit729// in registers) and return false, the effect of which will be to pass the730// argument under the rules for a large (> 128 bytes) composite.731bool AArch64ABIInfo::passAsPureScalableType(732 QualType Ty, unsigned &NVec, unsigned &NPred,733 SmallVectorImpl<llvm::Type *> &CoerceToSeq) const {734 if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {735 uint64_t NElt = AT->getZExtSize();736 if (NElt == 0)737 return false;738 739 unsigned NV = 0, NP = 0;740 SmallVector<llvm::Type *> EltCoerceToSeq;741 if (!passAsPureScalableType(AT->getElementType(), NV, NP, EltCoerceToSeq))742 return false;743 744 if (CoerceToSeq.size() + NElt * EltCoerceToSeq.size() > 12)745 return false;746 747 for (uint64_t I = 0; I < NElt; ++I)748 llvm::append_range(CoerceToSeq, EltCoerceToSeq);749 750 NVec += NElt * NV;751 NPred += NElt * NP;752 return true;753 }754 755 if (const RecordType *RT = Ty->getAsCanonical<RecordType>()) {756 // If the record cannot be passed in registers, then it's not a PST.757 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI());758 RAA != CGCXXABI::RAA_Default)759 return false;760 761 // Pure scalable types are never unions and never contain unions.762 const RecordDecl *RD = RT->getDecl()->getDefinitionOrSelf();763 if (RD->isUnion())764 return false;765 766 // If this is a C++ record, check the bases.767 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {768 for (const auto &I : CXXRD->bases()) {769 if (isEmptyRecord(getContext(), I.getType(), true))770 continue;771 if (!passAsPureScalableType(I.getType(), NVec, NPred, CoerceToSeq))772 return false;773 }774 }775 776 // Check members.777 for (const auto *FD : RD->fields()) {778 QualType FT = FD->getType();779 if (isEmptyField(getContext(), FD, /* AllowArrays */ true))780 continue;781 if (!passAsPureScalableType(FT, NVec, NPred, CoerceToSeq))782 return false;783 }784 785 return true;786 }787 788 if (const auto *VT = Ty->getAs<VectorType>()) {789 if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) {790 ++NPred;791 if (CoerceToSeq.size() + 1 > 12)792 return false;793 CoerceToSeq.push_back(convertFixedToScalableVectorType(VT));794 return true;795 }796 797 if (VT->getVectorKind() == VectorKind::SveFixedLengthData) {798 ++NVec;799 if (CoerceToSeq.size() + 1 > 12)800 return false;801 CoerceToSeq.push_back(convertFixedToScalableVectorType(VT));802 return true;803 }804 805 return false;806 }807 808 if (!Ty->isBuiltinType())809 return false;810 811 bool isPredicate;812 switch (Ty->castAs<BuiltinType>()->getKind()) {813#define SVE_VECTOR_TYPE(Name, MangledName, Id, SingletonId) \814 case BuiltinType::Id: \815 isPredicate = false; \816 break;817#define SVE_PREDICATE_TYPE(Name, MangledName, Id, SingletonId) \818 case BuiltinType::Id: \819 isPredicate = true; \820 break;821#include "clang/Basic/AArch64ACLETypes.def"822 default:823 return false;824 }825 826 ASTContext::BuiltinVectorTypeInfo Info =827 getContext().getBuiltinVectorTypeInfo(cast<BuiltinType>(Ty));828 assert(Info.NumVectors > 0 && Info.NumVectors <= 4 &&829 "Expected 1, 2, 3 or 4 vectors!");830 if (isPredicate)831 NPred += Info.NumVectors;832 else833 NVec += Info.NumVectors;834 llvm::Type *EltTy = Info.ElementType->isMFloat8Type()835 ? llvm::Type::getInt8Ty(getVMContext())836 : CGT.ConvertType(Info.ElementType);837 auto *VTy = llvm::ScalableVectorType::get(EltTy, Info.EC.getKnownMinValue());838 839 if (CoerceToSeq.size() + Info.NumVectors > 12)840 return false;841 std::fill_n(std::back_inserter(CoerceToSeq), Info.NumVectors, VTy);842 843 return true;844}845 846// Expand an LLVM IR type into a sequence with a element for each non-struct,847// non-array member of the type, with the exception of the padding types, which848// are retained.849void AArch64ABIInfo::flattenType(850 llvm::Type *Ty, SmallVectorImpl<llvm::Type *> &Flattened) const {851 852 if (ABIArgInfo::isPaddingForCoerceAndExpand(Ty)) {853 Flattened.push_back(Ty);854 return;855 }856 857 if (const auto *AT = dyn_cast<llvm::ArrayType>(Ty)) {858 uint64_t NElt = AT->getNumElements();859 if (NElt == 0)860 return;861 862 SmallVector<llvm::Type *> EltFlattened;863 flattenType(AT->getElementType(), EltFlattened);864 865 for (uint64_t I = 0; I < NElt; ++I)866 llvm::append_range(Flattened, EltFlattened);867 return;868 }869 870 if (const auto *ST = dyn_cast<llvm::StructType>(Ty)) {871 for (auto *ET : ST->elements())872 flattenType(ET, Flattened);873 return;874 }875 876 Flattened.push_back(Ty);877}878 879RValue AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty,880 CodeGenFunction &CGF, AArch64ABIKind Kind,881 AggValueSlot Slot) const {882 // These numbers are not used for variadic arguments, hence it doesn't matter883 // they don't retain their values across multiple calls to884 // `classifyArgumentType` here.885 unsigned NSRN = 0, NPRN = 0;886 ABIArgInfo AI =887 classifyArgumentType(Ty, /*IsVariadicFn=*/true, /* IsNamedArg */ false,888 CGF.CurFnInfo->getCallingConvention(), NSRN, NPRN);889 // Empty records are ignored for parameter passing purposes.890 if (AI.isIgnore())891 return Slot.asRValue();892 893 bool IsIndirect = AI.isIndirect();894 895 llvm::Type *BaseTy = CGF.ConvertType(Ty);896 if (IsIndirect)897 BaseTy = llvm::PointerType::getUnqual(BaseTy->getContext());898 else if (AI.getCoerceToType())899 BaseTy = AI.getCoerceToType();900 901 unsigned NumRegs = 1;902 if (llvm::ArrayType *ArrTy = dyn_cast<llvm::ArrayType>(BaseTy)) {903 BaseTy = ArrTy->getElementType();904 NumRegs = ArrTy->getNumElements();905 }906 bool IsFPR =907 !isSoftFloat() && (BaseTy->isFloatingPointTy() || BaseTy->isVectorTy());908 909 // The AArch64 va_list type and handling is specified in the Procedure Call910 // Standard, section B.4:911 //912 // struct {913 // void *__stack;914 // void *__gr_top;915 // void *__vr_top;916 // int __gr_offs;917 // int __vr_offs;918 // };919 920 llvm::BasicBlock *MaybeRegBlock = CGF.createBasicBlock("vaarg.maybe_reg");921 llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg");922 llvm::BasicBlock *OnStackBlock = CGF.createBasicBlock("vaarg.on_stack");923 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end");924 925 CharUnits TySize = getContext().getTypeSizeInChars(Ty);926 CharUnits TyAlign = getContext().getTypeUnadjustedAlignInChars(Ty);927 928 Address reg_offs_p = Address::invalid();929 llvm::Value *reg_offs = nullptr;930 int reg_top_index;931 int RegSize = IsIndirect ? 8 : TySize.getQuantity();932 if (!IsFPR) {933 // 3 is the field number of __gr_offs934 reg_offs_p = CGF.Builder.CreateStructGEP(VAListAddr, 3, "gr_offs_p");935 reg_offs = CGF.Builder.CreateLoad(reg_offs_p, "gr_offs");936 reg_top_index = 1; // field number for __gr_top937 RegSize = llvm::alignTo(RegSize, 8);938 } else {939 // 4 is the field number of __vr_offs.940 reg_offs_p = CGF.Builder.CreateStructGEP(VAListAddr, 4, "vr_offs_p");941 reg_offs = CGF.Builder.CreateLoad(reg_offs_p, "vr_offs");942 reg_top_index = 2; // field number for __vr_top943 RegSize = 16 * NumRegs;944 }945 946 //=======================================947 // Find out where argument was passed948 //=======================================949 950 // If reg_offs >= 0 we're already using the stack for this type of951 // argument. We don't want to keep updating reg_offs (in case it overflows,952 // though anyone passing 2GB of arguments, each at most 16 bytes, deserves953 // whatever they get).954 llvm::Value *UsingStack = nullptr;955 UsingStack = CGF.Builder.CreateICmpSGE(956 reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, 0));957 958 CGF.Builder.CreateCondBr(UsingStack, OnStackBlock, MaybeRegBlock);959 960 // Otherwise, at least some kind of argument could go in these registers, the961 // question is whether this particular type is too big.962 CGF.EmitBlock(MaybeRegBlock);963 964 // Integer arguments may need to correct register alignment (for example a965 // "struct { __int128 a; };" gets passed in x_2N, x_{2N+1}). In this case we966 // align __gr_offs to calculate the potential address.967 if (!IsFPR && !IsIndirect && TyAlign.getQuantity() > 8) {968 int Align = TyAlign.getQuantity();969 970 reg_offs = CGF.Builder.CreateAdd(971 reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, Align - 1),972 "align_regoffs");973 reg_offs = CGF.Builder.CreateAnd(974 reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, -Align),975 "aligned_regoffs");976 }977 978 // Update the gr_offs/vr_offs pointer for next call to va_arg on this va_list.979 // The fact that this is done unconditionally reflects the fact that980 // allocating an argument to the stack also uses up all the remaining981 // registers of the appropriate kind.982 llvm::Value *NewOffset = nullptr;983 NewOffset = CGF.Builder.CreateAdd(984 reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, RegSize), "new_reg_offs");985 CGF.Builder.CreateStore(NewOffset, reg_offs_p);986 987 // Now we're in a position to decide whether this argument really was in988 // registers or not.989 llvm::Value *InRegs = nullptr;990 InRegs = CGF.Builder.CreateICmpSLE(991 NewOffset, llvm::ConstantInt::get(CGF.Int32Ty, 0), "inreg");992 993 CGF.Builder.CreateCondBr(InRegs, InRegBlock, OnStackBlock);994 995 //=======================================996 // Argument was in registers997 //=======================================998 999 // Now we emit the code for if the argument was originally passed in1000 // registers. First start the appropriate block:1001 CGF.EmitBlock(InRegBlock);1002 1003 llvm::Value *reg_top = nullptr;1004 Address reg_top_p =1005 CGF.Builder.CreateStructGEP(VAListAddr, reg_top_index, "reg_top_p");1006 reg_top = CGF.Builder.CreateLoad(reg_top_p, "reg_top");1007 Address BaseAddr(CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, reg_top, reg_offs),1008 CGF.Int8Ty, CharUnits::fromQuantity(IsFPR ? 16 : 8));1009 Address RegAddr = Address::invalid();1010 llvm::Type *MemTy = CGF.ConvertTypeForMem(Ty), *ElementTy = MemTy;1011 1012 if (IsIndirect) {1013 // If it's been passed indirectly (actually a struct), whatever we find from1014 // stored registers or on the stack will actually be a struct **.1015 MemTy = llvm::PointerType::getUnqual(MemTy->getContext());1016 }1017 1018 const Type *Base = nullptr;1019 uint64_t NumMembers = 0;1020 bool IsHFA = isHomogeneousAggregate(Ty, Base, NumMembers);1021 if (IsHFA && NumMembers > 1) {1022 // Homogeneous aggregates passed in registers will have their elements split1023 // and stored 16-bytes apart regardless of size (they're notionally in qN,1024 // qN+1, ...). We reload and store into a temporary local variable1025 // contiguously.1026 assert(!IsIndirect && "Homogeneous aggregates should be passed directly");1027 auto BaseTyInfo = getContext().getTypeInfoInChars(QualType(Base, 0));1028 llvm::Type *BaseTy = CGF.ConvertType(QualType(Base, 0));1029 llvm::Type *HFATy = llvm::ArrayType::get(BaseTy, NumMembers);1030 Address Tmp = CGF.CreateTempAlloca(HFATy,1031 std::max(TyAlign, BaseTyInfo.Align));1032 1033 // On big-endian platforms, the value will be right-aligned in its slot.1034 int Offset = 0;1035 if (CGF.CGM.getDataLayout().isBigEndian() &&1036 BaseTyInfo.Width.getQuantity() < 16)1037 Offset = 16 - BaseTyInfo.Width.getQuantity();1038 1039 for (unsigned i = 0; i < NumMembers; ++i) {1040 CharUnits BaseOffset = CharUnits::fromQuantity(16 * i + Offset);1041 Address LoadAddr =1042 CGF.Builder.CreateConstInBoundsByteGEP(BaseAddr, BaseOffset);1043 LoadAddr = LoadAddr.withElementType(BaseTy);1044 1045 Address StoreAddr = CGF.Builder.CreateConstArrayGEP(Tmp, i);1046 1047 llvm::Value *Elem = CGF.Builder.CreateLoad(LoadAddr);1048 CGF.Builder.CreateStore(Elem, StoreAddr);1049 }1050 1051 RegAddr = Tmp.withElementType(MemTy);1052 } else {1053 // Otherwise the object is contiguous in memory.1054 1055 // It might be right-aligned in its slot.1056 CharUnits SlotSize = BaseAddr.getAlignment();1057 if (CGF.CGM.getDataLayout().isBigEndian() && !IsIndirect &&1058 (IsHFA || !isAggregateTypeForABI(Ty)) &&1059 TySize < SlotSize) {1060 CharUnits Offset = SlotSize - TySize;1061 BaseAddr = CGF.Builder.CreateConstInBoundsByteGEP(BaseAddr, Offset);1062 }1063 1064 RegAddr = BaseAddr.withElementType(MemTy);1065 }1066 1067 CGF.EmitBranch(ContBlock);1068 1069 //=======================================1070 // Argument was on the stack1071 //=======================================1072 CGF.EmitBlock(OnStackBlock);1073 1074 Address stack_p = CGF.Builder.CreateStructGEP(VAListAddr, 0, "stack_p");1075 llvm::Value *OnStackPtr = CGF.Builder.CreateLoad(stack_p, "stack");1076 1077 // Again, stack arguments may need realignment. In this case both integer and1078 // floating-point ones might be affected.1079 if (!IsIndirect && TyAlign.getQuantity() > 8) {1080 OnStackPtr = emitRoundPointerUpToAlignment(CGF, OnStackPtr, TyAlign);1081 }1082 Address OnStackAddr = Address(OnStackPtr, CGF.Int8Ty,1083 std::max(CharUnits::fromQuantity(8), TyAlign));1084 1085 // All stack slots are multiples of 8 bytes.1086 CharUnits StackSlotSize = CharUnits::fromQuantity(8);1087 CharUnits StackSize;1088 if (IsIndirect)1089 StackSize = StackSlotSize;1090 else1091 StackSize = TySize.alignTo(StackSlotSize);1092 1093 llvm::Value *StackSizeC = CGF.Builder.getSize(StackSize);1094 llvm::Value *NewStack = CGF.Builder.CreateInBoundsGEP(1095 CGF.Int8Ty, OnStackPtr, StackSizeC, "new_stack");1096 1097 // Write the new value of __stack for the next call to va_arg1098 CGF.Builder.CreateStore(NewStack, stack_p);1099 1100 if (CGF.CGM.getDataLayout().isBigEndian() && !isAggregateTypeForABI(Ty) &&1101 TySize < StackSlotSize) {1102 CharUnits Offset = StackSlotSize - TySize;1103 OnStackAddr = CGF.Builder.CreateConstInBoundsByteGEP(OnStackAddr, Offset);1104 }1105 1106 OnStackAddr = OnStackAddr.withElementType(MemTy);1107 1108 CGF.EmitBranch(ContBlock);1109 1110 //=======================================1111 // Tidy up1112 //=======================================1113 CGF.EmitBlock(ContBlock);1114 1115 Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock, OnStackAddr,1116 OnStackBlock, "vaargs.addr");1117 1118 if (IsIndirect)1119 return CGF.EmitLoadOfAnyValue(1120 CGF.MakeAddrLValue(1121 Address(CGF.Builder.CreateLoad(ResAddr, "vaarg.addr"), ElementTy,1122 TyAlign),1123 Ty),1124 Slot);1125 1126 return CGF.EmitLoadOfAnyValue(CGF.MakeAddrLValue(ResAddr, Ty), Slot);1127}1128 1129RValue AArch64ABIInfo::EmitDarwinVAArg(Address VAListAddr, QualType Ty,1130 CodeGenFunction &CGF,1131 AggValueSlot Slot) const {1132 // The backend's lowering doesn't support va_arg for aggregates or1133 // illegal vector types. Lower VAArg here for these cases and use1134 // the LLVM va_arg instruction for everything else.1135 if (!isAggregateTypeForABI(Ty) && !isIllegalVectorType(Ty))1136 return CGF.EmitLoadOfAnyValue(1137 CGF.MakeAddrLValue(1138 EmitVAArgInstr(CGF, VAListAddr, Ty, ABIArgInfo::getDirect()), Ty),1139 Slot);1140 1141 uint64_t PointerSize = getTarget().getPointerWidth(LangAS::Default) / 8;1142 CharUnits SlotSize = CharUnits::fromQuantity(PointerSize);1143 1144 // Empty records are ignored for parameter passing purposes.1145 if (isEmptyRecord(getContext(), Ty, true))1146 return Slot.asRValue();1147 1148 // The size of the actual thing passed, which might end up just1149 // being a pointer for indirect types.1150 auto TyInfo = getContext().getTypeInfoInChars(Ty);1151 1152 // Arguments bigger than 16 bytes which aren't homogeneous1153 // aggregates should be passed indirectly.1154 bool IsIndirect = false;1155 if (TyInfo.Width.getQuantity() > 16) {1156 const Type *Base = nullptr;1157 uint64_t Members = 0;1158 IsIndirect = !isHomogeneousAggregate(Ty, Base, Members);1159 }1160 1161 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, TyInfo, SlotSize,1162 /*AllowHigherAlign*/ true, Slot);1163}1164 1165RValue AArch64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,1166 QualType Ty, AggValueSlot Slot) const {1167 bool IsIndirect = false;1168 1169 if (getTarget().getTriple().isWindowsArm64EC()) {1170 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is1171 // not 1, 2, 4, or 8 bytes, must be passed by reference."1172 uint64_t Width = getContext().getTypeSize(Ty);1173 IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Width);1174 } else {1175 // Composites larger than 16 bytes are passed by reference.1176 if (isAggregateTypeForABI(Ty) && getContext().getTypeSize(Ty) > 128)1177 IsIndirect = true;1178 }1179 1180 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,1181 CGF.getContext().getTypeInfoInChars(Ty),1182 CharUnits::fromQuantity(8),1183 /*allowHigherAlign*/ false, Slot);1184}1185 1186static bool isStreamingCompatible(const FunctionDecl *F) {1187 if (const auto *T = F->getType()->getAs<FunctionProtoType>())1188 return T->getAArch64SMEAttributes() &1189 FunctionType::SME_PStateSMCompatibleMask;1190 return false;1191}1192 1193// Report an error if an argument or return value of type Ty would need to be1194// passed in a floating-point register.1195static void diagnoseIfNeedsFPReg(DiagnosticsEngine &Diags,1196 const StringRef ABIName,1197 const AArch64ABIInfo &ABIInfo,1198 const QualType &Ty, const NamedDecl *D,1199 SourceLocation loc) {1200 const Type *HABase = nullptr;1201 uint64_t HAMembers = 0;1202 if (Ty->isFloatingType() || Ty->isVectorType() ||1203 ABIInfo.isHomogeneousAggregate(Ty, HABase, HAMembers)) {1204 Diags.Report(loc, diag::err_target_unsupported_type_for_abi)1205 << D->getDeclName() << Ty << ABIName;1206 }1207}1208 1209// If we are using a hard-float ABI, but do not have floating point registers,1210// then report an error for any function arguments or returns which would be1211// passed in floating-pint registers.1212void AArch64TargetCodeGenInfo::checkFunctionABI(1213 CodeGenModule &CGM, const FunctionDecl *FuncDecl) const {1214 const AArch64ABIInfo &ABIInfo = getABIInfo<AArch64ABIInfo>();1215 const TargetInfo &TI = ABIInfo.getContext().getTargetInfo();1216 1217 if (!TI.hasFeature("fp") && !ABIInfo.isSoftFloat()) {1218 diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo,1219 FuncDecl->getReturnType(), FuncDecl,1220 FuncDecl->getLocation());1221 for (ParmVarDecl *PVD : FuncDecl->parameters()) {1222 diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo, PVD->getType(),1223 PVD, FuncDecl->getLocation());1224 }1225 }1226}1227 1228enum class ArmSMEInlinability : uint8_t {1229 Ok = 0,1230 ErrorCalleeRequiresNewZA = 1 << 0,1231 ErrorCalleeRequiresNewZT0 = 1 << 1,1232 WarnIncompatibleStreamingModes = 1 << 2,1233 ErrorIncompatibleStreamingModes = 1 << 3,1234 1235 IncompatibleStreamingModes =1236 WarnIncompatibleStreamingModes | ErrorIncompatibleStreamingModes,1237 1238 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/ErrorIncompatibleStreamingModes),1239};1240 1241/// Determines if there are any Arm SME ABI issues with inlining \p Callee into1242/// \p Caller. Returns the issue (if any) in the ArmSMEInlinability bit enum.1243static ArmSMEInlinability GetArmSMEInlinability(const FunctionDecl *Caller,1244 const FunctionDecl *Callee) {1245 bool CallerIsStreaming =1246 IsArmStreamingFunction(Caller, /*IncludeLocallyStreaming=*/true);1247 bool CalleeIsStreaming =1248 IsArmStreamingFunction(Callee, /*IncludeLocallyStreaming=*/true);1249 bool CallerIsStreamingCompatible = isStreamingCompatible(Caller);1250 bool CalleeIsStreamingCompatible = isStreamingCompatible(Callee);1251 1252 ArmSMEInlinability Inlinability = ArmSMEInlinability::Ok;1253 1254 if (!CalleeIsStreamingCompatible &&1255 (CallerIsStreaming != CalleeIsStreaming || CallerIsStreamingCompatible)) {1256 if (CalleeIsStreaming)1257 Inlinability |= ArmSMEInlinability::ErrorIncompatibleStreamingModes;1258 else1259 Inlinability |= ArmSMEInlinability::WarnIncompatibleStreamingModes;1260 }1261 if (auto *NewAttr = Callee->getAttr<ArmNewAttr>()) {1262 if (NewAttr->isNewZA())1263 Inlinability |= ArmSMEInlinability::ErrorCalleeRequiresNewZA;1264 if (NewAttr->isNewZT0())1265 Inlinability |= ArmSMEInlinability::ErrorCalleeRequiresNewZT0;1266 }1267 1268 return Inlinability;1269}1270 1271void AArch64TargetCodeGenInfo::checkFunctionCallABIStreaming(1272 CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,1273 const FunctionDecl *Callee) const {1274 if (!Caller || !Callee || !Callee->hasAttr<AlwaysInlineAttr>())1275 return;1276 1277 ArmSMEInlinability Inlinability = GetArmSMEInlinability(Caller, Callee);1278 1279 if ((Inlinability & ArmSMEInlinability::IncompatibleStreamingModes) !=1280 ArmSMEInlinability::Ok)1281 CGM.getDiags().Report(1282 CallLoc,1283 (Inlinability & ArmSMEInlinability::ErrorIncompatibleStreamingModes) ==1284 ArmSMEInlinability::ErrorIncompatibleStreamingModes1285 ? diag::err_function_always_inline_attribute_mismatch1286 : diag::warn_function_always_inline_attribute_mismatch)1287 << Caller->getDeclName() << Callee->getDeclName() << "streaming";1288 1289 if ((Inlinability & ArmSMEInlinability::ErrorCalleeRequiresNewZA) ==1290 ArmSMEInlinability::ErrorCalleeRequiresNewZA)1291 CGM.getDiags().Report(CallLoc, diag::err_function_always_inline_new_za)1292 << Callee->getDeclName();1293 1294 if ((Inlinability & ArmSMEInlinability::ErrorCalleeRequiresNewZT0) ==1295 ArmSMEInlinability::ErrorCalleeRequiresNewZT0)1296 CGM.getDiags().Report(CallLoc, diag::err_function_always_inline_new_zt0)1297 << Callee->getDeclName();1298}1299 1300// If the target does not have floating-point registers, but we are using a1301// hard-float ABI, there is no way to pass floating-point, vector or HFA values1302// to functions, so we report an error.1303void AArch64TargetCodeGenInfo::checkFunctionCallABISoftFloat(1304 CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,1305 const FunctionDecl *Callee, const CallArgList &Args,1306 QualType ReturnType) const {1307 const AArch64ABIInfo &ABIInfo = getABIInfo<AArch64ABIInfo>();1308 const TargetInfo &TI = ABIInfo.getContext().getTargetInfo();1309 1310 if (!Caller || TI.hasFeature("fp") || ABIInfo.isSoftFloat())1311 return;1312 1313 diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo, ReturnType,1314 Callee ? Callee : Caller, CallLoc);1315 1316 for (const CallArg &Arg : Args)1317 diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo, Arg.getType(),1318 Callee ? Callee : Caller, CallLoc);1319}1320 1321void AArch64TargetCodeGenInfo::checkFunctionCallABI(CodeGenModule &CGM,1322 SourceLocation CallLoc,1323 const FunctionDecl *Caller,1324 const FunctionDecl *Callee,1325 const CallArgList &Args,1326 QualType ReturnType) const {1327 checkFunctionCallABIStreaming(CGM, CallLoc, Caller, Callee);1328 checkFunctionCallABISoftFloat(CGM, CallLoc, Caller, Callee, Args, ReturnType);1329}1330 1331bool AArch64TargetCodeGenInfo::wouldInliningViolateFunctionCallABI(1332 const FunctionDecl *Caller, const FunctionDecl *Callee) const {1333 return Caller && Callee &&1334 GetArmSMEInlinability(Caller, Callee) != ArmSMEInlinability::Ok;1335}1336 1337void AArch64ABIInfo::appendAttributeMangling(TargetClonesAttr *Attr,1338 unsigned Index,1339 raw_ostream &Out) const {1340 appendAttributeMangling(Attr->getFeatureStr(Index), Out);1341}1342 1343void AArch64ABIInfo::appendAttributeMangling(StringRef AttrStr,1344 raw_ostream &Out) const {1345 if (AttrStr == "default") {1346 Out << ".default";1347 return;1348 }1349 1350 Out << "._";1351 SmallVector<StringRef, 8> Features;1352 AttrStr.split(Features, "+");1353 for (auto &Feat : Features)1354 Feat = Feat.trim();1355 1356 llvm::sort(Features, [](const StringRef LHS, const StringRef RHS) {1357 return LHS.compare(RHS) < 0;1358 });1359 1360 llvm::SmallDenseSet<StringRef, 8> UniqueFeats;1361 for (auto &Feat : Features)1362 if (auto Ext = llvm::AArch64::parseFMVExtension(Feat))1363 if (UniqueFeats.insert(Ext->Name).second)1364 Out << 'M' << Ext->Name;1365}1366 1367std::unique_ptr<TargetCodeGenInfo>1368CodeGen::createAArch64TargetCodeGenInfo(CodeGenModule &CGM,1369 AArch64ABIKind Kind) {1370 return std::make_unique<AArch64TargetCodeGenInfo>(CGM, Kind);1371}1372 1373std::unique_ptr<TargetCodeGenInfo>1374CodeGen::createWindowsAArch64TargetCodeGenInfo(CodeGenModule &CGM,1375 AArch64ABIKind K) {1376 return std::make_unique<WindowsAArch64TargetCodeGenInfo>(CGM, K);1377}1378