3543 lines · cpp
1//===- X86.cpp ------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "ABIInfoImpl.h"10#include "TargetInfo.h"11#include "clang/Basic/DiagnosticFrontend.h"12#include "llvm/ADT/SmallBitVector.h"13 14using namespace clang;15using namespace clang::CodeGen;16 17namespace {18 19/// IsX86_MMXType - Return true if this is an MMX type.20bool IsX86_MMXType(llvm::Type *IRType) {21 // Return true if the type is an MMX type <2 x i32>, <4 x i16>, or <8 x i8>.22 return IRType->isVectorTy() && IRType->getPrimitiveSizeInBits() == 64 &&23 cast<llvm::VectorType>(IRType)->getElementType()->isIntegerTy() &&24 IRType->getScalarSizeInBits() != 64;25}26 27static llvm::Type *X86AdjustInlineAsmType(CodeGen::CodeGenFunction &CGF,28 StringRef Constraint,29 llvm::Type *Ty) {30 bool IsMMXCons = llvm::StringSwitch<bool>(Constraint)31 .Cases({"y", "&y", "^Ym"}, true)32 .Default(false);33 if (IsMMXCons && Ty->isVectorTy() &&34 cast<llvm::VectorType>(Ty)->getPrimitiveSizeInBits().getFixedValue() !=35 64)36 return nullptr; // Invalid MMX constraint37 38 if (Constraint == "k") {39 llvm::Type *Int1Ty = llvm::Type::getInt1Ty(CGF.getLLVMContext());40 return llvm::FixedVectorType::get(Int1Ty, Ty->getScalarSizeInBits());41 }42 43 // No operation needed44 return Ty;45}46 47/// Returns true if this type can be passed in SSE registers with the48/// X86_VectorCall calling convention. Shared between x86_32 and x86_64.49static bool isX86VectorTypeForVectorCall(ASTContext &Context, QualType Ty) {50 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {51 if (BT->isFloatingPoint() && BT->getKind() != BuiltinType::Half) {52 if (BT->getKind() == BuiltinType::LongDouble) {53 if (&Context.getTargetInfo().getLongDoubleFormat() ==54 &llvm::APFloat::x87DoubleExtended())55 return false;56 }57 return true;58 }59 } else if (const VectorType *VT = Ty->getAs<VectorType>()) {60 // vectorcall can pass XMM, YMM, and ZMM vectors. We don't pass SSE1 MMX61 // registers specially.62 unsigned VecSize = Context.getTypeSize(VT);63 if (VecSize == 128 || VecSize == 256 || VecSize == 512)64 return true;65 }66 return false;67}68 69/// Returns true if this aggregate is small enough to be passed in SSE registers70/// in the X86_VectorCall calling convention. Shared between x86_32 and x86_64.71static bool isX86VectorCallAggregateSmallEnough(uint64_t NumMembers) {72 return NumMembers <= 4;73}74 75/// Returns a Homogeneous Vector Aggregate ABIArgInfo, used in X86.76static ABIArgInfo getDirectX86Hva(llvm::Type* T = nullptr) {77 auto AI = ABIArgInfo::getDirect(T);78 AI.setInReg(true);79 AI.setCanBeFlattened(false);80 return AI;81}82 83//===----------------------------------------------------------------------===//84// X86-32 ABI Implementation85//===----------------------------------------------------------------------===//86 87/// Similar to llvm::CCState, but for Clang.88struct CCState {89 CCState(CGFunctionInfo &FI)90 : IsPreassigned(FI.arg_size()), CC(FI.getCallingConvention()),91 Required(FI.getRequiredArgs()), IsDelegateCall(FI.isDelegateCall()) {}92 93 llvm::SmallBitVector IsPreassigned;94 unsigned CC = CallingConv::CC_C;95 unsigned FreeRegs = 0;96 unsigned FreeSSERegs = 0;97 RequiredArgs Required;98 bool IsDelegateCall = false;99};100 101/// X86_32ABIInfo - The X86-32 ABI information.102class X86_32ABIInfo : public ABIInfo {103 enum Class {104 Integer,105 Float106 };107 108 static const unsigned MinABIStackAlignInBytes = 4;109 110 bool IsDarwinVectorABI;111 bool IsRetSmallStructInRegABI;112 bool IsWin32StructABI;113 bool IsSoftFloatABI;114 bool IsMCUABI;115 bool IsLinuxABI;116 unsigned DefaultNumRegisterParameters;117 118 static bool isRegisterSize(unsigned Size) {119 return (Size == 8 || Size == 16 || Size == 32 || Size == 64);120 }121 122 bool isHomogeneousAggregateBaseType(QualType Ty) const override {123 // FIXME: Assumes vectorcall is in use.124 return isX86VectorTypeForVectorCall(getContext(), Ty);125 }126 127 bool isHomogeneousAggregateSmallEnough(const Type *Ty,128 uint64_t NumMembers) const override {129 // FIXME: Assumes vectorcall is in use.130 return isX86VectorCallAggregateSmallEnough(NumMembers);131 }132 133 bool shouldReturnTypeInRegister(QualType Ty, ASTContext &Context) const;134 135 /// getIndirectResult - Give a source type \arg Ty, return a suitable result136 /// such that the argument will be passed in memory.137 ABIArgInfo getIndirectResult(QualType Ty, bool ByVal, CCState &State) const;138 139 ABIArgInfo getIndirectReturnResult(QualType Ty, CCState &State) const;140 141 /// Return the alignment to use for the given type on the stack.142 unsigned getTypeStackAlignInBytes(QualType Ty, unsigned Align) const;143 144 Class classify(QualType Ty) const;145 ABIArgInfo classifyReturnType(QualType RetTy, CCState &State) const;146 ABIArgInfo classifyArgumentType(QualType RetTy, CCState &State,147 unsigned ArgIndex) const;148 149 /// Updates the number of available free registers, returns150 /// true if any registers were allocated.151 bool updateFreeRegs(QualType Ty, CCState &State) const;152 153 bool shouldAggregateUseDirect(QualType Ty, CCState &State, bool &InReg,154 bool &NeedsPadding) const;155 bool shouldPrimitiveUseInReg(QualType Ty, CCState &State) const;156 157 bool canExpandIndirectArgument(QualType Ty) const;158 159 /// Rewrite the function info so that all memory arguments use160 /// inalloca.161 void rewriteWithInAlloca(CGFunctionInfo &FI) const;162 163 void addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,164 CharUnits &StackOffset, ABIArgInfo &Info,165 QualType Type) const;166 void runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const;167 168public:169 170 void computeInfo(CGFunctionInfo &FI) const override;171 RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,172 AggValueSlot Slot) const override;173 174 X86_32ABIInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,175 bool RetSmallStructInRegABI, bool Win32StructABI,176 unsigned NumRegisterParameters, bool SoftFloatABI)177 : ABIInfo(CGT), IsDarwinVectorABI(DarwinVectorABI),178 IsRetSmallStructInRegABI(RetSmallStructInRegABI),179 IsWin32StructABI(Win32StructABI), IsSoftFloatABI(SoftFloatABI),180 IsMCUABI(CGT.getTarget().getTriple().isOSIAMCU()),181 IsLinuxABI(CGT.getTarget().getTriple().isOSLinux() ||182 CGT.getTarget().getTriple().isOSCygMing()),183 DefaultNumRegisterParameters(NumRegisterParameters) {}184};185 186class X86_32SwiftABIInfo : public SwiftABIInfo {187public:188 explicit X86_32SwiftABIInfo(CodeGenTypes &CGT)189 : SwiftABIInfo(CGT, /*SwiftErrorInRegister=*/false) {}190 191 bool shouldPassIndirectly(ArrayRef<llvm::Type *> ComponentTys,192 bool AsReturnValue) const override {193 // LLVM's x86-32 lowering currently only assigns up to three194 // integer registers and three fp registers. Oddly, it'll use up to195 // four vector registers for vectors, but those can overlap with the196 // scalar registers.197 return occupiesMoreThan(ComponentTys, /*total=*/3);198 }199};200 201class X86_32TargetCodeGenInfo : public TargetCodeGenInfo {202public:203 X86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,204 bool RetSmallStructInRegABI, bool Win32StructABI,205 unsigned NumRegisterParameters, bool SoftFloatABI)206 : TargetCodeGenInfo(std::make_unique<X86_32ABIInfo>(207 CGT, DarwinVectorABI, RetSmallStructInRegABI, Win32StructABI,208 NumRegisterParameters, SoftFloatABI)) {209 SwiftInfo = std::make_unique<X86_32SwiftABIInfo>(CGT);210 }211 212 static bool isStructReturnInRegABI(213 const llvm::Triple &Triple, const CodeGenOptions &Opts);214 215 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,216 CodeGen::CodeGenModule &CGM) const override;217 218 int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {219 // Darwin uses different dwarf register numbers for EH.220 if (CGM.getTarget().getTriple().isOSDarwin()) return 5;221 return 4;222 }223 224 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,225 llvm::Value *Address) const override;226 227 llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,228 StringRef Constraint,229 llvm::Type* Ty) const override {230 return X86AdjustInlineAsmType(CGF, Constraint, Ty);231 }232 233 void addReturnRegisterOutputs(CodeGenFunction &CGF, LValue ReturnValue,234 std::string &Constraints,235 std::vector<llvm::Type *> &ResultRegTypes,236 std::vector<llvm::Type *> &ResultTruncRegTypes,237 std::vector<LValue> &ResultRegDests,238 std::string &AsmString,239 unsigned NumOutputs) const override;240 241 StringRef getARCRetainAutoreleasedReturnValueMarker() const override {242 return "movl\t%ebp, %ebp"243 "\t\t// marker for objc_retainAutoreleaseReturnValue";244 }245};246 247}248 249/// Rewrite input constraint references after adding some output constraints.250/// In the case where there is one output and one input and we add one output,251/// we need to replace all operand references greater than or equal to 1:252/// mov $0, $1253/// mov eax, $1254/// The result will be:255/// mov $0, $2256/// mov eax, $2257static void rewriteInputConstraintReferences(unsigned FirstIn,258 unsigned NumNewOuts,259 std::string &AsmString) {260 std::string Buf;261 llvm::raw_string_ostream OS(Buf);262 size_t Pos = 0;263 while (Pos < AsmString.size()) {264 size_t DollarStart = AsmString.find('$', Pos);265 if (DollarStart == std::string::npos)266 DollarStart = AsmString.size();267 size_t DollarEnd = AsmString.find_first_not_of('$', DollarStart);268 if (DollarEnd == std::string::npos)269 DollarEnd = AsmString.size();270 OS << StringRef(&AsmString[Pos], DollarEnd - Pos);271 Pos = DollarEnd;272 size_t NumDollars = DollarEnd - DollarStart;273 if (NumDollars % 2 != 0 && Pos < AsmString.size()) {274 // We have an operand reference.275 size_t DigitStart = Pos;276 if (AsmString[DigitStart] == '{') {277 OS << '{';278 ++DigitStart;279 }280 size_t DigitEnd = AsmString.find_first_not_of("0123456789", DigitStart);281 if (DigitEnd == std::string::npos)282 DigitEnd = AsmString.size();283 StringRef OperandStr(&AsmString[DigitStart], DigitEnd - DigitStart);284 unsigned OperandIndex;285 if (!OperandStr.getAsInteger(10, OperandIndex)) {286 if (OperandIndex >= FirstIn)287 OperandIndex += NumNewOuts;288 OS << OperandIndex;289 } else {290 OS << OperandStr;291 }292 Pos = DigitEnd;293 }294 }295 AsmString = std::move(Buf);296}297 298/// Add output constraints for EAX:EDX because they are return registers.299void X86_32TargetCodeGenInfo::addReturnRegisterOutputs(300 CodeGenFunction &CGF, LValue ReturnSlot, std::string &Constraints,301 std::vector<llvm::Type *> &ResultRegTypes,302 std::vector<llvm::Type *> &ResultTruncRegTypes,303 std::vector<LValue> &ResultRegDests, std::string &AsmString,304 unsigned NumOutputs) const {305 uint64_t RetWidth = CGF.getContext().getTypeSize(ReturnSlot.getType());306 307 // Use the EAX constraint if the width is 32 or smaller and EAX:EDX if it is308 // larger.309 if (!Constraints.empty())310 Constraints += ',';311 if (RetWidth <= 32) {312 Constraints += "={eax}";313 ResultRegTypes.push_back(CGF.Int32Ty);314 } else {315 // Use the 'A' constraint for EAX:EDX.316 Constraints += "=A";317 ResultRegTypes.push_back(CGF.Int64Ty);318 }319 320 // Truncate EAX or EAX:EDX to an integer of the appropriate size.321 llvm::Type *CoerceTy = llvm::IntegerType::get(CGF.getLLVMContext(), RetWidth);322 ResultTruncRegTypes.push_back(CoerceTy);323 324 // Coerce the integer by bitcasting the return slot pointer.325 ReturnSlot.setAddress(ReturnSlot.getAddress().withElementType(CoerceTy));326 ResultRegDests.push_back(ReturnSlot);327 328 rewriteInputConstraintReferences(NumOutputs, 1, AsmString);329}330 331/// shouldReturnTypeInRegister - Determine if the given type should be332/// returned in a register (for the Darwin and MCU ABI).333bool X86_32ABIInfo::shouldReturnTypeInRegister(QualType Ty,334 ASTContext &Context) const {335 uint64_t Size = Context.getTypeSize(Ty);336 337 // For i386, type must be register sized.338 // For the MCU ABI, it only needs to be <= 8-byte339 if ((IsMCUABI && Size > 64) || (!IsMCUABI && !isRegisterSize(Size)))340 return false;341 342 if (Ty->isVectorType()) {343 // 64- and 128- bit vectors inside structures are not returned in344 // registers.345 if (Size == 64 || Size == 128)346 return false;347 348 return true;349 }350 351 // If this is a builtin, pointer, enum, complex type, member pointer, or352 // member function pointer it is ok.353 if (Ty->getAs<BuiltinType>() || Ty->hasPointerRepresentation() ||354 Ty->isAnyComplexType() || Ty->isEnumeralType() ||355 Ty->isBlockPointerType() || Ty->isMemberPointerType())356 return true;357 358 // Arrays are treated like records.359 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty))360 return shouldReturnTypeInRegister(AT->getElementType(), Context);361 362 // Otherwise, it must be a record type.363 const auto *RD = Ty->getAsRecordDecl();364 if (!RD)365 return false;366 367 // FIXME: Traverse bases here too.368 369 // Structure types are passed in register if all fields would be370 // passed in a register.371 for (const auto *FD : RD->fields()) {372 // Empty fields are ignored.373 if (isEmptyField(Context, FD, true))374 continue;375 376 // Check fields recursively.377 if (!shouldReturnTypeInRegister(FD->getType(), Context))378 return false;379 }380 return true;381}382 383static bool is32Or64BitBasicType(QualType Ty, ASTContext &Context) {384 // Treat complex types as the element type.385 if (const ComplexType *CTy = Ty->getAs<ComplexType>())386 Ty = CTy->getElementType();387 388 // Check for a type which we know has a simple scalar argument-passing389 // convention without any padding. (We're specifically looking for 32390 // and 64-bit integer and integer-equivalents, float, and double.)391 if (!Ty->getAs<BuiltinType>() && !Ty->hasPointerRepresentation() &&392 !Ty->isEnumeralType() && !Ty->isBlockPointerType())393 return false;394 395 uint64_t Size = Context.getTypeSize(Ty);396 return Size == 32 || Size == 64;397}398 399static bool addFieldSizes(ASTContext &Context, const RecordDecl *RD,400 uint64_t &Size) {401 for (const auto *FD : RD->fields()) {402 // Scalar arguments on the stack get 4 byte alignment on x86. If the403 // argument is smaller than 32-bits, expanding the struct will create404 // alignment padding.405 if (!is32Or64BitBasicType(FD->getType(), Context))406 return false;407 408 // FIXME: Reject bit-fields wholesale; there are two problems, we don't know409 // how to expand them yet, and the predicate for telling if a bitfield still410 // counts as "basic" is more complicated than what we were doing previously.411 if (FD->isBitField())412 return false;413 414 Size += Context.getTypeSize(FD->getType());415 }416 return true;417}418 419static bool addBaseAndFieldSizes(ASTContext &Context, const CXXRecordDecl *RD,420 uint64_t &Size) {421 // Don't do this if there are any non-empty bases.422 for (const CXXBaseSpecifier &Base : RD->bases()) {423 if (!addBaseAndFieldSizes(Context, Base.getType()->getAsCXXRecordDecl(),424 Size))425 return false;426 }427 if (!addFieldSizes(Context, RD, Size))428 return false;429 return true;430}431 432/// Test whether an argument type which is to be passed indirectly (on the433/// stack) would have the equivalent layout if it was expanded into separate434/// arguments. If so, we prefer to do the latter to avoid inhibiting435/// optimizations.436bool X86_32ABIInfo::canExpandIndirectArgument(QualType Ty) const {437 // We can only expand structure types.438 const RecordDecl *RD = Ty->getAsRecordDecl();439 if (!RD)440 return false;441 uint64_t Size = 0;442 if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {443 if (!IsWin32StructABI) {444 // On non-Windows, we have to conservatively match our old bitcode445 // prototypes in order to be ABI-compatible at the bitcode level.446 if (!CXXRD->isCLike())447 return false;448 } else {449 // Don't do this for dynamic classes.450 if (CXXRD->isDynamicClass())451 return false;452 }453 if (!addBaseAndFieldSizes(getContext(), CXXRD, Size))454 return false;455 } else {456 if (!addFieldSizes(getContext(), RD, Size))457 return false;458 }459 460 // We can do this if there was no alignment padding.461 return Size == getContext().getTypeSize(Ty);462}463 464ABIArgInfo X86_32ABIInfo::getIndirectReturnResult(QualType RetTy, CCState &State) const {465 // If the return value is indirect, then the hidden argument is consuming one466 // integer register.467 if (State.CC != llvm::CallingConv::X86_FastCall &&468 State.CC != llvm::CallingConv::X86_VectorCall && State.FreeRegs) {469 --State.FreeRegs;470 if (!IsMCUABI)471 return getNaturalAlignIndirectInReg(RetTy);472 }473 return getNaturalAlignIndirect(474 RetTy, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),475 /*ByVal=*/false);476}477 478ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy,479 CCState &State) const {480 if (RetTy->isVoidType())481 return ABIArgInfo::getIgnore();482 483 const Type *Base = nullptr;484 uint64_t NumElts = 0;485 if ((State.CC == llvm::CallingConv::X86_VectorCall ||486 State.CC == llvm::CallingConv::X86_RegCall) &&487 isHomogeneousAggregate(RetTy, Base, NumElts)) {488 // The LLVM struct type for such an aggregate should lower properly.489 return ABIArgInfo::getDirect();490 }491 492 if (const VectorType *VT = RetTy->getAs<VectorType>()) {493 // On Darwin, some vectors are returned in registers.494 if (IsDarwinVectorABI) {495 uint64_t Size = getContext().getTypeSize(RetTy);496 497 // 128-bit vectors are a special case; they are returned in498 // registers and we need to make sure to pick a type the LLVM499 // backend will like.500 if (Size == 128)501 return ABIArgInfo::getDirect(llvm::FixedVectorType::get(502 llvm::Type::getInt64Ty(getVMContext()), 2));503 504 // Always return in register if it fits in a general purpose505 // register, or if it is 64 bits and has a single element.506 if ((Size == 8 || Size == 16 || Size == 32) ||507 (Size == 64 && VT->getNumElements() == 1))508 return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),509 Size));510 511 return getIndirectReturnResult(RetTy, State);512 }513 514 return ABIArgInfo::getDirect();515 }516 517 if (isAggregateTypeForABI(RetTy)) {518 if (const auto *RD = RetTy->getAsRecordDecl();519 RD && RD->hasFlexibleArrayMember())520 // Structures with flexible arrays are always indirect.521 return getIndirectReturnResult(RetTy, State);522 523 // If specified, structs and unions are always indirect.524 if (!IsRetSmallStructInRegABI && !RetTy->isAnyComplexType())525 return getIndirectReturnResult(RetTy, State);526 527 // Ignore empty structs/unions.528 if (isEmptyRecord(getContext(), RetTy, true))529 return ABIArgInfo::getIgnore();530 531 // Return complex of _Float16 as <2 x half> so the backend will use xmm0.532 if (const ComplexType *CT = RetTy->getAs<ComplexType>()) {533 QualType ET = getContext().getCanonicalType(CT->getElementType());534 if (ET->isFloat16Type())535 return ABIArgInfo::getDirect(llvm::FixedVectorType::get(536 llvm::Type::getHalfTy(getVMContext()), 2));537 }538 539 // Small structures which are register sized are generally returned540 // in a register.541 if (shouldReturnTypeInRegister(RetTy, getContext())) {542 uint64_t Size = getContext().getTypeSize(RetTy);543 544 // As a special-case, if the struct is a "single-element" struct, and545 // the field is of type "float" or "double", return it in a546 // floating-point register. (MSVC does not apply this special case.)547 // We apply a similar transformation for pointer types to improve the548 // quality of the generated IR.549 if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext()))550 if ((!IsWin32StructABI && SeltTy->isRealFloatingType())551 || SeltTy->hasPointerRepresentation())552 return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));553 554 // FIXME: We should be able to narrow this integer in cases with dead555 // padding.556 return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),Size));557 }558 559 return getIndirectReturnResult(RetTy, State);560 }561 562 // Treat an enum type as its underlying type.563 if (const auto *ED = RetTy->getAsEnumDecl())564 RetTy = ED->getIntegerType();565 566 if (const auto *EIT = RetTy->getAs<BitIntType>())567 if (EIT->getNumBits() > 64)568 return getIndirectReturnResult(RetTy, State);569 570 return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)571 : ABIArgInfo::getDirect());572}573 574unsigned X86_32ABIInfo::getTypeStackAlignInBytes(QualType Ty,575 unsigned Align) const {576 // Otherwise, if the alignment is less than or equal to the minimum ABI577 // alignment, just use the default; the backend will handle this.578 if (Align <= MinABIStackAlignInBytes)579 return 0; // Use default alignment.580 581 if (IsLinuxABI) {582 // Exclude other System V OS (e.g Darwin, PS4 and FreeBSD) since we don't583 // want to spend any effort dealing with the ramifications of ABI breaks.584 //585 // If the vector type is __m128/__m256/__m512, return the default alignment.586 if (Ty->isVectorType() && (Align == 16 || Align == 32 || Align == 64))587 return Align;588 }589 // On non-Darwin, the stack type alignment is always 4.590 if (!IsDarwinVectorABI) {591 // Set explicit alignment, since we may need to realign the top.592 return MinABIStackAlignInBytes;593 }594 595 // Otherwise, if the type contains an SSE vector type, the alignment is 16.596 if (Align >= 16 && (isSIMDVectorType(getContext(), Ty) ||597 isRecordWithSIMDVectorType(getContext(), Ty)))598 return 16;599 600 return MinABIStackAlignInBytes;601}602 603ABIArgInfo X86_32ABIInfo::getIndirectResult(QualType Ty, bool ByVal,604 CCState &State) const {605 if (!ByVal) {606 if (State.FreeRegs) {607 --State.FreeRegs; // Non-byval indirects just use one pointer.608 if (!IsMCUABI)609 return getNaturalAlignIndirectInReg(Ty);610 }611 return getNaturalAlignIndirect(Ty, getDataLayout().getAllocaAddrSpace(),612 false);613 }614 615 // Compute the byval alignment.616 unsigned TypeAlign = getContext().getTypeAlign(Ty) / 8;617 unsigned StackAlign = getTypeStackAlignInBytes(Ty, TypeAlign);618 if (StackAlign == 0)619 return ABIArgInfo::getIndirect(620 CharUnits::fromQuantity(4),621 /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),622 /*ByVal=*/true);623 624 // If the stack alignment is less than the type alignment, realign the625 // argument.626 bool Realign = TypeAlign > StackAlign;627 return ABIArgInfo::getIndirect(628 CharUnits::fromQuantity(StackAlign),629 /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(), /*ByVal=*/true,630 Realign);631}632 633X86_32ABIInfo::Class X86_32ABIInfo::classify(QualType Ty) const {634 const Type *T = isSingleElementStruct(Ty, getContext());635 if (!T)636 T = Ty.getTypePtr();637 638 if (const BuiltinType *BT = T->getAs<BuiltinType>()) {639 BuiltinType::Kind K = BT->getKind();640 if (K == BuiltinType::Float || K == BuiltinType::Double)641 return Float;642 }643 return Integer;644}645 646bool X86_32ABIInfo::updateFreeRegs(QualType Ty, CCState &State) const {647 if (!IsSoftFloatABI) {648 Class C = classify(Ty);649 if (C == Float)650 return false;651 }652 653 unsigned Size = getContext().getTypeSize(Ty);654 unsigned SizeInRegs = (Size + 31) / 32;655 656 if (SizeInRegs == 0)657 return false;658 659 if (!IsMCUABI) {660 if (SizeInRegs > State.FreeRegs) {661 State.FreeRegs = 0;662 return false;663 }664 } else {665 // The MCU psABI allows passing parameters in-reg even if there are666 // earlier parameters that are passed on the stack. Also,667 // it does not allow passing >8-byte structs in-register,668 // even if there are 3 free registers available.669 if (SizeInRegs > State.FreeRegs || SizeInRegs > 2)670 return false;671 }672 673 State.FreeRegs -= SizeInRegs;674 return true;675}676 677bool X86_32ABIInfo::shouldAggregateUseDirect(QualType Ty, CCState &State,678 bool &InReg,679 bool &NeedsPadding) const {680 // On Windows, aggregates other than HFAs are never passed in registers, and681 // they do not consume register slots. Homogenous floating-point aggregates682 // (HFAs) have already been dealt with at this point.683 if (IsWin32StructABI && isAggregateTypeForABI(Ty))684 return false;685 686 NeedsPadding = false;687 InReg = !IsMCUABI;688 689 if (!updateFreeRegs(Ty, State))690 return false;691 692 if (IsMCUABI)693 return true;694 695 if (State.CC == llvm::CallingConv::X86_FastCall ||696 State.CC == llvm::CallingConv::X86_VectorCall ||697 State.CC == llvm::CallingConv::X86_RegCall) {698 if (getContext().getTypeSize(Ty) <= 32 && State.FreeRegs)699 NeedsPadding = true;700 701 return false;702 }703 704 return true;705}706 707bool X86_32ABIInfo::shouldPrimitiveUseInReg(QualType Ty, CCState &State) const {708 bool IsPtrOrInt = (getContext().getTypeSize(Ty) <= 32) &&709 (Ty->isIntegralOrEnumerationType() || Ty->isPointerType() ||710 Ty->isReferenceType());711 712 if (!IsPtrOrInt && (State.CC == llvm::CallingConv::X86_FastCall ||713 State.CC == llvm::CallingConv::X86_VectorCall))714 return false;715 716 if (!updateFreeRegs(Ty, State))717 return false;718 719 if (!IsPtrOrInt && State.CC == llvm::CallingConv::X86_RegCall)720 return false;721 722 // Return true to apply inreg to all legal parameters except for MCU targets.723 return !IsMCUABI;724}725 726void X86_32ABIInfo::runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const {727 // Vectorcall x86 works subtly different than in x64, so the format is728 // a bit different than the x64 version. First, all vector types (not HVAs)729 // are assigned, with the first 6 ending up in the [XYZ]MM0-5 registers.730 // This differs from the x64 implementation, where the first 6 by INDEX get731 // registers.732 // In the second pass over the arguments, HVAs are passed in the remaining733 // vector registers if possible, or indirectly by address. The address will be734 // passed in ECX/EDX if available. Any other arguments are passed according to735 // the usual fastcall rules.736 MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();737 for (int I = 0, E = Args.size(); I < E; ++I) {738 const Type *Base = nullptr;739 uint64_t NumElts = 0;740 const QualType &Ty = Args[I].type;741 if ((Ty->isVectorType() || Ty->isBuiltinType()) &&742 isHomogeneousAggregate(Ty, Base, NumElts)) {743 if (State.FreeSSERegs >= NumElts) {744 State.FreeSSERegs -= NumElts;745 Args[I].info = ABIArgInfo::getDirectInReg();746 State.IsPreassigned.set(I);747 }748 }749 }750}751 752ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, CCState &State,753 unsigned ArgIndex) const {754 // FIXME: Set alignment on indirect arguments.755 bool IsFastCall = State.CC == llvm::CallingConv::X86_FastCall;756 bool IsRegCall = State.CC == llvm::CallingConv::X86_RegCall;757 bool IsVectorCall = State.CC == llvm::CallingConv::X86_VectorCall;758 759 Ty = useFirstFieldIfTransparentUnion(Ty);760 TypeInfo TI = getContext().getTypeInfo(Ty);761 762 // Check with the C++ ABI first.763 const RecordType *RT = Ty->getAsCanonical<RecordType>();764 if (RT) {765 CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI());766 if (RAA == CGCXXABI::RAA_Indirect) {767 return getIndirectResult(Ty, false, State);768 } else if (State.IsDelegateCall) {769 // Avoid having different alignments on delegate call args by always770 // setting the alignment to 4, which is what we do for inallocas.771 ABIArgInfo Res = getIndirectResult(Ty, false, State);772 Res.setIndirectAlign(CharUnits::fromQuantity(4));773 return Res;774 } else if (RAA == CGCXXABI::RAA_DirectInMemory) {775 // The field index doesn't matter, we'll fix it up later.776 return ABIArgInfo::getInAlloca(/*FieldIndex=*/0);777 }778 }779 780 // Regcall uses the concept of a homogenous vector aggregate, similar781 // to other targets.782 const Type *Base = nullptr;783 uint64_t NumElts = 0;784 if ((IsRegCall || IsVectorCall) &&785 isHomogeneousAggregate(Ty, Base, NumElts)) {786 if (State.FreeSSERegs >= NumElts) {787 State.FreeSSERegs -= NumElts;788 789 // Vectorcall passes HVAs directly and does not flatten them, but regcall790 // does.791 if (IsVectorCall)792 return getDirectX86Hva();793 794 if (Ty->isBuiltinType() || Ty->isVectorType())795 return ABIArgInfo::getDirect();796 return ABIArgInfo::getExpand();797 }798 if (IsVectorCall && Ty->isBuiltinType())799 return ABIArgInfo::getDirect();800 return getIndirectResult(Ty, /*ByVal=*/false, State);801 }802 803 if (isAggregateTypeForABI(Ty)) {804 // Structures with flexible arrays are always indirect.805 // FIXME: This should not be byval!806 if (RT && RT->getDecl()->getDefinitionOrSelf()->hasFlexibleArrayMember())807 return getIndirectResult(Ty, true, State);808 809 // Ignore empty structs/unions on non-Windows.810 if (!IsWin32StructABI && isEmptyRecord(getContext(), Ty, true))811 return ABIArgInfo::getIgnore();812 813 // Ignore 0 sized structs.814 if (TI.Width == 0)815 return ABIArgInfo::getIgnore();816 817 llvm::LLVMContext &LLVMContext = getVMContext();818 llvm::IntegerType *Int32 = llvm::Type::getInt32Ty(LLVMContext);819 bool NeedsPadding = false;820 bool InReg;821 if (shouldAggregateUseDirect(Ty, State, InReg, NeedsPadding)) {822 unsigned SizeInRegs = (TI.Width + 31) / 32;823 SmallVector<llvm::Type*, 3> Elements(SizeInRegs, Int32);824 llvm::Type *Result = llvm::StructType::get(LLVMContext, Elements);825 if (InReg)826 return ABIArgInfo::getDirectInReg(Result);827 else828 return ABIArgInfo::getDirect(Result);829 }830 llvm::IntegerType *PaddingType = NeedsPadding ? Int32 : nullptr;831 832 // Pass over-aligned aggregates to non-variadic functions on Windows833 // indirectly. This behavior was added in MSVC 2015. Use the required834 // alignment from the record layout, since that may be less than the835 // regular type alignment, and types with required alignment of less than 4836 // bytes are not passed indirectly.837 if (IsWin32StructABI && State.Required.isRequiredArg(ArgIndex)) {838 unsigned AlignInBits = 0;839 if (RT) {840 const ASTRecordLayout &Layout =841 getContext().getASTRecordLayout(RT->getDecl());842 AlignInBits = getContext().toBits(Layout.getRequiredAlignment());843 } else if (TI.isAlignRequired()) {844 AlignInBits = TI.Align;845 }846 if (AlignInBits > 32)847 return getIndirectResult(Ty, /*ByVal=*/false, State);848 }849 850 // Expand small (<= 128-bit) record types when we know that the stack layout851 // of those arguments will match the struct. This is important because the852 // LLVM backend isn't smart enough to remove byval, which inhibits many853 // optimizations.854 // Don't do this for the MCU if there are still free integer registers855 // (see X86_64 ABI for full explanation).856 if (TI.Width <= 4 * 32 && (!IsMCUABI || State.FreeRegs == 0) &&857 canExpandIndirectArgument(Ty))858 return ABIArgInfo::getExpandWithPadding(859 IsFastCall || IsVectorCall || IsRegCall, PaddingType);860 861 return getIndirectResult(Ty, true, State);862 }863 864 if (const VectorType *VT = Ty->getAs<VectorType>()) {865 // On Windows, vectors are passed directly if registers are available, or866 // indirectly if not. This avoids the need to align argument memory. Pass867 // user-defined vector types larger than 512 bits indirectly for simplicity.868 if (IsWin32StructABI) {869 if (TI.Width <= 512 && State.FreeSSERegs > 0) {870 --State.FreeSSERegs;871 return ABIArgInfo::getDirectInReg();872 }873 return getIndirectResult(Ty, /*ByVal=*/false, State);874 }875 876 // On Darwin, some vectors are passed in memory, we handle this by passing877 // it as an i8/i16/i32/i64.878 if (IsDarwinVectorABI) {879 if ((TI.Width == 8 || TI.Width == 16 || TI.Width == 32) ||880 (TI.Width == 64 && VT->getNumElements() == 1))881 return ABIArgInfo::getDirect(882 llvm::IntegerType::get(getVMContext(), TI.Width));883 }884 885 if (IsX86_MMXType(CGT.ConvertType(Ty)))886 return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), 64));887 888 return ABIArgInfo::getDirect();889 }890 891 if (const auto *ED = Ty->getAsEnumDecl())892 Ty = ED->getIntegerType();893 894 bool InReg = shouldPrimitiveUseInReg(Ty, State);895 896 if (isPromotableIntegerTypeForABI(Ty)) {897 if (InReg)898 return ABIArgInfo::getExtendInReg(Ty, CGT.ConvertType(Ty));899 return ABIArgInfo::getExtend(Ty, CGT.ConvertType(Ty));900 }901 902 if (const auto *EIT = Ty->getAs<BitIntType>()) {903 if (EIT->getNumBits() <= 64) {904 if (InReg)905 return ABIArgInfo::getDirectInReg();906 return ABIArgInfo::getDirect();907 }908 return getIndirectResult(Ty, /*ByVal=*/false, State);909 }910 911 if (InReg)912 return ABIArgInfo::getDirectInReg();913 return ABIArgInfo::getDirect();914}915 916void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const {917 CCState State(FI);918 if (IsMCUABI)919 State.FreeRegs = 3;920 else if (State.CC == llvm::CallingConv::X86_FastCall) {921 State.FreeRegs = 2;922 State.FreeSSERegs = 3;923 } else if (State.CC == llvm::CallingConv::X86_VectorCall) {924 State.FreeRegs = 2;925 State.FreeSSERegs = 6;926 } else if (FI.getHasRegParm())927 State.FreeRegs = FI.getRegParm();928 else if (State.CC == llvm::CallingConv::X86_RegCall) {929 State.FreeRegs = 5;930 State.FreeSSERegs = 8;931 } else if (IsWin32StructABI) {932 // Since MSVC 2015, the first three SSE vectors have been passed in933 // registers. The rest are passed indirectly.934 State.FreeRegs = DefaultNumRegisterParameters;935 State.FreeSSERegs = 3;936 } else937 State.FreeRegs = DefaultNumRegisterParameters;938 939 if (!::classifyReturnType(getCXXABI(), FI, *this)) {940 FI.getReturnInfo() = classifyReturnType(FI.getReturnType(), State);941 } else if (FI.getReturnInfo().isIndirect()) {942 // The C++ ABI is not aware of register usage, so we have to check if the943 // return value was sret and put it in a register ourselves if appropriate.944 if (State.FreeRegs) {945 --State.FreeRegs; // The sret parameter consumes a register.946 if (!IsMCUABI)947 FI.getReturnInfo().setInReg(true);948 }949 }950 951 // The chain argument effectively gives us another free register.952 if (FI.isChainCall())953 ++State.FreeRegs;954 955 // For vectorcall, do a first pass over the arguments, assigning FP and vector956 // arguments to XMM registers as available.957 if (State.CC == llvm::CallingConv::X86_VectorCall)958 runVectorCallFirstPass(FI, State);959 960 bool UsedInAlloca = false;961 MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();962 for (unsigned I = 0, E = Args.size(); I < E; ++I) {963 // Skip arguments that have already been assigned.964 if (State.IsPreassigned.test(I))965 continue;966 967 Args[I].info =968 classifyArgumentType(Args[I].type, State, I);969 UsedInAlloca |= (Args[I].info.getKind() == ABIArgInfo::InAlloca);970 }971 972 // If we needed to use inalloca for any argument, do a second pass and rewrite973 // all the memory arguments to use inalloca.974 if (UsedInAlloca)975 rewriteWithInAlloca(FI);976}977 978void979X86_32ABIInfo::addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,980 CharUnits &StackOffset, ABIArgInfo &Info,981 QualType Type) const {982 // Arguments are always 4-byte-aligned.983 CharUnits WordSize = CharUnits::fromQuantity(4);984 assert(StackOffset.isMultipleOf(WordSize) && "unaligned inalloca struct");985 986 // sret pointers and indirect things will require an extra pointer987 // indirection, unless they are byval. Most things are byval, and will not988 // require this indirection.989 bool IsIndirect = false;990 if (Info.isIndirect() && !Info.getIndirectByVal())991 IsIndirect = true;992 Info = ABIArgInfo::getInAlloca(FrameFields.size(), IsIndirect);993 llvm::Type *LLTy = CGT.ConvertTypeForMem(Type);994 if (IsIndirect)995 LLTy = llvm::PointerType::getUnqual(getVMContext());996 FrameFields.push_back(LLTy);997 StackOffset += IsIndirect ? WordSize : getContext().getTypeSizeInChars(Type);998 999 // Insert padding bytes to respect alignment.1000 CharUnits FieldEnd = StackOffset;1001 StackOffset = FieldEnd.alignTo(WordSize);1002 if (StackOffset != FieldEnd) {1003 CharUnits NumBytes = StackOffset - FieldEnd;1004 llvm::Type *Ty = llvm::Type::getInt8Ty(getVMContext());1005 Ty = llvm::ArrayType::get(Ty, NumBytes.getQuantity());1006 FrameFields.push_back(Ty);1007 }1008}1009 1010static bool isArgInAlloca(const ABIArgInfo &Info) {1011 // Leave ignored and inreg arguments alone.1012 switch (Info.getKind()) {1013 case ABIArgInfo::InAlloca:1014 return true;1015 case ABIArgInfo::Ignore:1016 case ABIArgInfo::IndirectAliased:1017 case ABIArgInfo::TargetSpecific:1018 return false;1019 case ABIArgInfo::Indirect:1020 case ABIArgInfo::Direct:1021 case ABIArgInfo::Extend:1022 return !Info.getInReg();1023 case ABIArgInfo::Expand:1024 case ABIArgInfo::CoerceAndExpand:1025 // These are aggregate types which are never passed in registers when1026 // inalloca is involved.1027 return true;1028 }1029 llvm_unreachable("invalid enum");1030}1031 1032void X86_32ABIInfo::rewriteWithInAlloca(CGFunctionInfo &FI) const {1033 assert(IsWin32StructABI && "inalloca only supported on win32");1034 1035 // Build a packed struct type for all of the arguments in memory.1036 SmallVector<llvm::Type *, 6> FrameFields;1037 1038 // The stack alignment is always 4.1039 CharUnits StackAlign = CharUnits::fromQuantity(4);1040 1041 CharUnits StackOffset;1042 CGFunctionInfo::arg_iterator I = FI.arg_begin(), E = FI.arg_end();1043 1044 // Put 'this' into the struct before 'sret', if necessary.1045 bool IsThisCall =1046 FI.getCallingConvention() == llvm::CallingConv::X86_ThisCall;1047 ABIArgInfo &Ret = FI.getReturnInfo();1048 if (Ret.isIndirect() && Ret.isSRetAfterThis() && !IsThisCall &&1049 isArgInAlloca(I->info)) {1050 addFieldToArgStruct(FrameFields, StackOffset, I->info, I->type);1051 ++I;1052 }1053 1054 // Put the sret parameter into the inalloca struct if it's in memory.1055 if (Ret.isIndirect() && !Ret.getInReg()) {1056 addFieldToArgStruct(FrameFields, StackOffset, Ret, FI.getReturnType());1057 // On Windows, the hidden sret parameter is always returned in eax.1058 Ret.setInAllocaSRet(IsWin32StructABI);1059 }1060 1061 // Skip the 'this' parameter in ecx.1062 if (IsThisCall)1063 ++I;1064 1065 // Put arguments passed in memory into the struct.1066 for (; I != E; ++I) {1067 if (isArgInAlloca(I->info))1068 addFieldToArgStruct(FrameFields, StackOffset, I->info, I->type);1069 }1070 1071 FI.setArgStruct(llvm::StructType::get(getVMContext(), FrameFields,1072 /*isPacked=*/true),1073 StackAlign);1074}1075 1076RValue X86_32ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,1077 QualType Ty, AggValueSlot Slot) const {1078 1079 auto TypeInfo = getContext().getTypeInfoInChars(Ty);1080 1081 CCState State(*const_cast<CGFunctionInfo *>(CGF.CurFnInfo));1082 ABIArgInfo AI = classifyArgumentType(Ty, State, /*ArgIndex*/ 0);1083 // Empty records are ignored for parameter passing purposes.1084 if (AI.isIgnore())1085 return Slot.asRValue();1086 1087 // x86-32 changes the alignment of certain arguments on the stack.1088 //1089 // Just messing with TypeInfo like this works because we never pass1090 // anything indirectly.1091 TypeInfo.Align = CharUnits::fromQuantity(1092 getTypeStackAlignInBytes(Ty, TypeInfo.Align.getQuantity()));1093 1094 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false, TypeInfo,1095 CharUnits::fromQuantity(4),1096 /*AllowHigherAlign*/ true, Slot);1097}1098 1099bool X86_32TargetCodeGenInfo::isStructReturnInRegABI(1100 const llvm::Triple &Triple, const CodeGenOptions &Opts) {1101 assert(Triple.getArch() == llvm::Triple::x86);1102 1103 switch (Opts.getStructReturnConvention()) {1104 case CodeGenOptions::SRCK_Default:1105 break;1106 case CodeGenOptions::SRCK_OnStack: // -fpcc-struct-return1107 return false;1108 case CodeGenOptions::SRCK_InRegs: // -freg-struct-return1109 return true;1110 }1111 1112 if (Triple.isOSDarwin() || Triple.isOSIAMCU())1113 return true;1114 1115 switch (Triple.getOS()) {1116 case llvm::Triple::DragonFly:1117 case llvm::Triple::FreeBSD:1118 case llvm::Triple::OpenBSD:1119 case llvm::Triple::Win32:1120 return true;1121 default:1122 return false;1123 }1124}1125 1126static void addX86InterruptAttrs(const FunctionDecl *FD, llvm::GlobalValue *GV,1127 CodeGen::CodeGenModule &CGM) {1128 if (!FD->hasAttr<AnyX86InterruptAttr>())1129 return;1130 1131 llvm::Function *Fn = cast<llvm::Function>(GV);1132 Fn->setCallingConv(llvm::CallingConv::X86_INTR);1133 if (FD->getNumParams() == 0)1134 return;1135 1136 auto PtrTy = cast<PointerType>(FD->getParamDecl(0)->getType());1137 llvm::Type *ByValTy = CGM.getTypes().ConvertType(PtrTy->getPointeeType());1138 llvm::Attribute NewAttr = llvm::Attribute::getWithByValType(1139 Fn->getContext(), ByValTy);1140 Fn->addParamAttr(0, NewAttr);1141}1142 1143void X86_32TargetCodeGenInfo::setTargetAttributes(1144 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {1145 if (GV->isDeclaration())1146 return;1147 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {1148 if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {1149 llvm::Function *Fn = cast<llvm::Function>(GV);1150 Fn->addFnAttr("stackrealign");1151 }1152 1153 addX86InterruptAttrs(FD, GV, CGM);1154 }1155}1156 1157bool X86_32TargetCodeGenInfo::initDwarfEHRegSizeTable(1158 CodeGen::CodeGenFunction &CGF,1159 llvm::Value *Address) const {1160 CodeGen::CGBuilderTy &Builder = CGF.Builder;1161 1162 llvm::Value *Four8 = llvm::ConstantInt::get(CGF.Int8Ty, 4);1163 1164 // 0-7 are the eight integer registers; the order is different1165 // on Darwin (for EH), but the range is the same.1166 // 8 is %eip.1167 AssignToArrayRange(Builder, Address, Four8, 0, 8);1168 1169 if (CGF.CGM.getTarget().getTriple().isOSDarwin()) {1170 // 12-16 are st(0..4). Not sure why we stop at 4.1171 // These have size 16, which is sizeof(long double) on1172 // platforms with 8-byte alignment for that type.1173 llvm::Value *Sixteen8 = llvm::ConstantInt::get(CGF.Int8Ty, 16);1174 AssignToArrayRange(Builder, Address, Sixteen8, 12, 16);1175 1176 } else {1177 // 9 is %eflags, which doesn't get a size on Darwin for some1178 // reason.1179 Builder.CreateAlignedStore(1180 Four8, Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, Address, 9),1181 CharUnits::One());1182 1183 // 11-16 are st(0..5). Not sure why we stop at 5.1184 // These have size 12, which is sizeof(long double) on1185 // platforms with 4-byte alignment for that type.1186 llvm::Value *Twelve8 = llvm::ConstantInt::get(CGF.Int8Ty, 12);1187 AssignToArrayRange(Builder, Address, Twelve8, 11, 16);1188 }1189 1190 return false;1191}1192 1193//===----------------------------------------------------------------------===//1194// X86-64 ABI Implementation1195//===----------------------------------------------------------------------===//1196 1197 1198namespace {1199 1200/// \p returns the size in bits of the largest (native) vector for \p AVXLevel.1201static unsigned getNativeVectorSizeForAVXABI(X86AVXABILevel AVXLevel) {1202 switch (AVXLevel) {1203 case X86AVXABILevel::AVX512:1204 return 512;1205 case X86AVXABILevel::AVX:1206 return 256;1207 case X86AVXABILevel::None:1208 return 128;1209 }1210 llvm_unreachable("Unknown AVXLevel");1211}1212 1213/// X86_64ABIInfo - The X86_64 ABI information.1214class X86_64ABIInfo : public ABIInfo {1215 enum Class {1216 Integer = 0,1217 SSE,1218 SSEUp,1219 X87,1220 X87Up,1221 ComplexX87,1222 NoClass,1223 Memory1224 };1225 1226 /// merge - Implement the X86_64 ABI merging algorithm.1227 ///1228 /// Merge an accumulating classification \arg Accum with a field1229 /// classification \arg Field.1230 ///1231 /// \param Accum - The accumulating classification. This should1232 /// always be either NoClass or the result of a previous merge1233 /// call. In addition, this should never be Memory (the caller1234 /// should just return Memory for the aggregate).1235 static Class merge(Class Accum, Class Field);1236 1237 /// postMerge - Implement the X86_64 ABI post merging algorithm.1238 ///1239 /// Post merger cleanup, reduces a malformed Hi and Lo pair to1240 /// final MEMORY or SSE classes when necessary.1241 ///1242 /// \param AggregateSize - The size of the current aggregate in1243 /// the classification process.1244 ///1245 /// \param Lo - The classification for the parts of the type1246 /// residing in the low word of the containing object.1247 ///1248 /// \param Hi - The classification for the parts of the type1249 /// residing in the higher words of the containing object.1250 ///1251 void postMerge(unsigned AggregateSize, Class &Lo, Class &Hi) const;1252 1253 /// classify - Determine the x86_64 register classes in which the1254 /// given type T should be passed.1255 ///1256 /// \param Lo - The classification for the parts of the type1257 /// residing in the low word of the containing object.1258 ///1259 /// \param Hi - The classification for the parts of the type1260 /// residing in the high word of the containing object.1261 ///1262 /// \param OffsetBase - The bit offset of this type in the1263 /// containing object. Some parameters are classified different1264 /// depending on whether they straddle an eightbyte boundary.1265 ///1266 /// \param isNamedArg - Whether the argument in question is a "named"1267 /// argument, as used in AMD64-ABI 3.5.7.1268 ///1269 /// \param IsRegCall - Whether the calling conversion is regcall.1270 ///1271 /// If a word is unused its result will be NoClass; if a type should1272 /// be passed in Memory then at least the classification of \arg Lo1273 /// will be Memory.1274 ///1275 /// The \arg Lo class will be NoClass iff the argument is ignored.1276 ///1277 /// If the \arg Lo class is ComplexX87, then the \arg Hi class will1278 /// also be ComplexX87.1279 void classify(QualType T, uint64_t OffsetBase, Class &Lo, Class &Hi,1280 bool isNamedArg, bool IsRegCall = false) const;1281 1282 llvm::Type *GetByteVectorType(QualType Ty) const;1283 llvm::Type *GetSSETypeAtOffset(llvm::Type *IRType,1284 unsigned IROffset, QualType SourceTy,1285 unsigned SourceOffset) const;1286 llvm::Type *GetINTEGERTypeAtOffset(llvm::Type *IRType,1287 unsigned IROffset, QualType SourceTy,1288 unsigned SourceOffset) const;1289 1290 /// getIndirectResult - Give a source type \arg Ty, return a suitable result1291 /// such that the argument will be returned in memory.1292 ABIArgInfo getIndirectReturnResult(QualType Ty) const;1293 1294 /// getIndirectResult - Give a source type \arg Ty, return a suitable result1295 /// such that the argument will be passed in memory.1296 ///1297 /// \param freeIntRegs - The number of free integer registers remaining1298 /// available.1299 ABIArgInfo getIndirectResult(QualType Ty, unsigned freeIntRegs) const;1300 1301 ABIArgInfo classifyReturnType(QualType RetTy) const;1302 1303 ABIArgInfo classifyArgumentType(QualType Ty, unsigned freeIntRegs,1304 unsigned &neededInt, unsigned &neededSSE,1305 bool isNamedArg,1306 bool IsRegCall = false) const;1307 1308 ABIArgInfo classifyRegCallStructType(QualType Ty, unsigned &NeededInt,1309 unsigned &NeededSSE,1310 unsigned &MaxVectorWidth) const;1311 1312 ABIArgInfo classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,1313 unsigned &NeededSSE,1314 unsigned &MaxVectorWidth) const;1315 1316 bool IsIllegalVectorType(QualType Ty) const;1317 1318 /// The 0.98 ABI revision clarified a lot of ambiguities,1319 /// unfortunately in ways that were not always consistent with1320 /// certain previous compilers. In particular, platforms which1321 /// required strict binary compatibility with older versions of GCC1322 /// may need to exempt themselves.1323 bool honorsRevision0_98() const {1324 return !getTarget().getTriple().isOSDarwin();1325 }1326 1327 /// GCC classifies <1 x long long> as SSE but some platform ABIs choose to1328 /// classify it as INTEGER (for compatibility with older clang compilers).1329 bool classifyIntegerMMXAsSSE() const {1330 // Clang <= 3.8 did not do this.1331 if (getContext().getLangOpts().getClangABICompat() <=1332 LangOptions::ClangABI::Ver3_8)1333 return false;1334 1335 const llvm::Triple &Triple = getTarget().getTriple();1336 if (Triple.isOSDarwin() || Triple.isPS() || Triple.isOSFreeBSD())1337 return false;1338 return true;1339 }1340 1341 // GCC classifies vectors of __int128 as memory.1342 bool passInt128VectorsInMem() const {1343 // Clang <= 9.0 did not do this.1344 if (getContext().getLangOpts().getClangABICompat() <=1345 LangOptions::ClangABI::Ver9)1346 return false;1347 1348 const llvm::Triple &T = getTarget().getTriple();1349 return T.isOSLinux() || T.isOSNetBSD();1350 }1351 1352 bool returnCXXRecordGreaterThan128InMem() const {1353 // Clang <= 20.0 did not do this, and PlayStation does not do this.1354 if (getContext().getLangOpts().getClangABICompat() <=1355 LangOptions::ClangABI::Ver20 ||1356 getTarget().getTriple().isPS())1357 return false;1358 1359 return true;1360 }1361 1362 X86AVXABILevel AVXLevel;1363 // Some ABIs (e.g. X32 ABI and Native Client OS) use 32 bit pointers on1364 // 64-bit hardware.1365 bool Has64BitPointers;1366 1367public:1368 X86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)1369 : ABIInfo(CGT), AVXLevel(AVXLevel),1370 Has64BitPointers(CGT.getDataLayout().getPointerSize(0) == 8) {}1371 1372 bool isPassedUsingAVXType(QualType type) const {1373 unsigned neededInt, neededSSE;1374 // The freeIntRegs argument doesn't matter here.1375 ABIArgInfo info = classifyArgumentType(type, 0, neededInt, neededSSE,1376 /*isNamedArg*/true);1377 if (info.isDirect()) {1378 llvm::Type *ty = info.getCoerceToType();1379 if (llvm::VectorType *vectorTy = dyn_cast_or_null<llvm::VectorType>(ty))1380 return vectorTy->getPrimitiveSizeInBits().getFixedValue() > 128;1381 }1382 return false;1383 }1384 1385 void computeInfo(CGFunctionInfo &FI) const override;1386 1387 RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,1388 AggValueSlot Slot) const override;1389 RValue EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,1390 AggValueSlot Slot) const override;1391 1392 bool has64BitPointers() const {1393 return Has64BitPointers;1394 }1395};1396 1397/// WinX86_64ABIInfo - The Windows X86_64 ABI information.1398class WinX86_64ABIInfo : public ABIInfo {1399public:1400 WinX86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)1401 : ABIInfo(CGT), AVXLevel(AVXLevel),1402 IsMingw64(getTarget().getTriple().isWindowsGNUEnvironment()) {}1403 1404 void computeInfo(CGFunctionInfo &FI) const override;1405 1406 RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,1407 AggValueSlot Slot) const override;1408 1409 bool isHomogeneousAggregateBaseType(QualType Ty) const override {1410 // FIXME: Assumes vectorcall is in use.1411 return isX86VectorTypeForVectorCall(getContext(), Ty);1412 }1413 1414 bool isHomogeneousAggregateSmallEnough(const Type *Ty,1415 uint64_t NumMembers) const override {1416 // FIXME: Assumes vectorcall is in use.1417 return isX86VectorCallAggregateSmallEnough(NumMembers);1418 }1419 1420 ABIArgInfo classifyArgForArm64ECVarArg(QualType Ty) const override {1421 unsigned FreeSSERegs = 0;1422 return classify(Ty, FreeSSERegs, /*IsReturnType=*/false,1423 /*IsVectorCall=*/false, /*IsRegCall=*/false);1424 }1425 1426private:1427 ABIArgInfo classify(QualType Ty, unsigned &FreeSSERegs, bool IsReturnType,1428 bool IsVectorCall, bool IsRegCall) const;1429 ABIArgInfo reclassifyHvaArgForVectorCall(QualType Ty, unsigned &FreeSSERegs,1430 const ABIArgInfo ¤t) const;1431 1432 X86AVXABILevel AVXLevel;1433 1434 bool IsMingw64;1435};1436 1437class X86_64TargetCodeGenInfo : public TargetCodeGenInfo {1438public:1439 X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)1440 : TargetCodeGenInfo(std::make_unique<X86_64ABIInfo>(CGT, AVXLevel)) {1441 SwiftInfo =1442 std::make_unique<SwiftABIInfo>(CGT, /*SwiftErrorInRegister=*/true);1443 }1444 1445 /// Disable tail call on x86-64. The epilogue code before the tail jump blocks1446 /// autoreleaseRV/retainRV and autoreleaseRV/unsafeClaimRV optimizations.1447 bool markARCOptimizedReturnCallsAsNoTail() const override { return true; }1448 1449 int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {1450 return 7;1451 }1452 1453 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,1454 llvm::Value *Address) const override {1455 llvm::Value *Eight8 = llvm::ConstantInt::get(CGF.Int8Ty, 8);1456 1457 // 0-15 are the 16 integer registers.1458 // 16 is %rip.1459 AssignToArrayRange(CGF.Builder, Address, Eight8, 0, 16);1460 return false;1461 }1462 1463 llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,1464 StringRef Constraint,1465 llvm::Type* Ty) const override {1466 return X86AdjustInlineAsmType(CGF, Constraint, Ty);1467 }1468 1469 bool isNoProtoCallVariadic(const CallArgList &args,1470 const FunctionNoProtoType *fnType) const override {1471 // The default CC on x86-64 sets %al to the number of SSA1472 // registers used, and GCC sets this when calling an unprototyped1473 // function, so we override the default behavior. However, don't do1474 // that when AVX types are involved: the ABI explicitly states it is1475 // undefined, and it doesn't work in practice because of how the ABI1476 // defines varargs anyway.1477 if (fnType->getCallConv() == CC_C) {1478 bool HasAVXType = false;1479 for (const CallArg &arg : args) {1480 if (getABIInfo<X86_64ABIInfo>().isPassedUsingAVXType(arg.Ty)) {1481 HasAVXType = true;1482 break;1483 }1484 }1485 1486 if (!HasAVXType)1487 return true;1488 }1489 1490 return TargetCodeGenInfo::isNoProtoCallVariadic(args, fnType);1491 }1492 1493 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,1494 CodeGen::CodeGenModule &CGM) const override {1495 if (GV->isDeclaration())1496 return;1497 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {1498 if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {1499 llvm::Function *Fn = cast<llvm::Function>(GV);1500 Fn->addFnAttr("stackrealign");1501 }1502 1503 addX86InterruptAttrs(FD, GV, CGM);1504 }1505 }1506 1507 void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,1508 const FunctionDecl *Caller,1509 const FunctionDecl *Callee, const CallArgList &Args,1510 QualType ReturnType) const override;1511};1512} // namespace1513 1514static void initFeatureMaps(const ASTContext &Ctx,1515 llvm::StringMap<bool> &CallerMap,1516 const FunctionDecl *Caller,1517 llvm::StringMap<bool> &CalleeMap,1518 const FunctionDecl *Callee) {1519 if (CalleeMap.empty() && CallerMap.empty()) {1520 // The caller is potentially nullptr in the case where the call isn't in a1521 // function. In this case, the getFunctionFeatureMap ensures we just get1522 // the TU level setting (since it cannot be modified by 'target'..1523 Ctx.getFunctionFeatureMap(CallerMap, Caller);1524 Ctx.getFunctionFeatureMap(CalleeMap, Callee);1525 }1526}1527 1528static bool checkAVXParamFeature(DiagnosticsEngine &Diag,1529 SourceLocation CallLoc,1530 const FunctionDecl &Callee,1531 const llvm::StringMap<bool> &CallerMap,1532 const llvm::StringMap<bool> &CalleeMap,1533 QualType Ty, StringRef Feature,1534 bool IsArgument) {1535 bool CallerHasFeat = CallerMap.lookup(Feature);1536 bool CalleeHasFeat = CalleeMap.lookup(Feature);1537 // No explicit features and the function is internal, be permissive.1538 if (!CallerHasFeat && !CalleeHasFeat &&1539 (!Callee.isExternallyVisible() || Callee.hasAttr<AlwaysInlineAttr>()))1540 return false;1541 1542 if (!CallerHasFeat && !CalleeHasFeat)1543 return Diag.Report(CallLoc, diag::warn_avx_calling_convention)1544 << IsArgument << Ty << Feature;1545 1546 // Mixing calling conventions here is very clearly an error.1547 if (!CallerHasFeat || !CalleeHasFeat)1548 return Diag.Report(CallLoc, diag::err_avx_calling_convention)1549 << IsArgument << Ty << Feature;1550 1551 // Else, both caller and callee have the required feature, so there is no need1552 // to diagnose.1553 return false;1554}1555 1556static bool checkAVXParam(DiagnosticsEngine &Diag, ASTContext &Ctx,1557 SourceLocation CallLoc, const FunctionDecl &Callee,1558 const llvm::StringMap<bool> &CallerMap,1559 const llvm::StringMap<bool> &CalleeMap, QualType Ty,1560 bool IsArgument) {1561 uint64_t Size = Ctx.getTypeSize(Ty);1562 if (Size > 256)1563 return checkAVXParamFeature(Diag, CallLoc, Callee, CallerMap, CalleeMap, Ty,1564 "avx512f", IsArgument);1565 1566 if (Size > 128)1567 return checkAVXParamFeature(Diag, CallLoc, Callee, CallerMap, CalleeMap, Ty,1568 "avx", IsArgument);1569 1570 return false;1571}1572 1573void X86_64TargetCodeGenInfo::checkFunctionCallABI(CodeGenModule &CGM,1574 SourceLocation CallLoc,1575 const FunctionDecl *Caller,1576 const FunctionDecl *Callee,1577 const CallArgList &Args,1578 QualType ReturnType) const {1579 if (!Callee)1580 return;1581 1582 llvm::StringMap<bool> CallerMap;1583 llvm::StringMap<bool> CalleeMap;1584 unsigned ArgIndex = 0;1585 1586 // We need to loop through the actual call arguments rather than the1587 // function's parameters, in case this variadic.1588 for (const CallArg &Arg : Args) {1589 // The "avx" feature changes how vectors >128 in size are passed. "avx512f"1590 // additionally changes how vectors >256 in size are passed. Like GCC, we1591 // warn when a function is called with an argument where this will change.1592 // Unlike GCC, we also error when it is an obvious ABI mismatch, that is,1593 // the caller and callee features are mismatched.1594 // Unfortunately, we cannot do this diagnostic in SEMA, since the callee can1595 // change its ABI with attribute-target after this call.1596 if (Arg.getType()->isVectorType() &&1597 CGM.getContext().getTypeSize(Arg.getType()) > 128) {1598 initFeatureMaps(CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);1599 QualType Ty = Arg.getType();1600 // The CallArg seems to have desugared the type already, so for clearer1601 // diagnostics, replace it with the type in the FunctionDecl if possible.1602 if (ArgIndex < Callee->getNumParams())1603 Ty = Callee->getParamDecl(ArgIndex)->getType();1604 1605 if (checkAVXParam(CGM.getDiags(), CGM.getContext(), CallLoc, *Callee,1606 CallerMap, CalleeMap, Ty, /*IsArgument*/ true))1607 return;1608 }1609 ++ArgIndex;1610 }1611 1612 // Check return always, as we don't have a good way of knowing in codegen1613 // whether this value is used, tail-called, etc.1614 if (Callee->getReturnType()->isVectorType() &&1615 CGM.getContext().getTypeSize(Callee->getReturnType()) > 128) {1616 initFeatureMaps(CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);1617 checkAVXParam(CGM.getDiags(), CGM.getContext(), CallLoc, *Callee, CallerMap,1618 CalleeMap, Callee->getReturnType(),1619 /*IsArgument*/ false);1620 }1621}1622 1623std::string TargetCodeGenInfo::qualifyWindowsLibrary(StringRef Lib) {1624 // If the argument does not end in .lib, automatically add the suffix.1625 // If the argument contains a space, enclose it in quotes.1626 // This matches the behavior of MSVC.1627 bool Quote = Lib.contains(' ');1628 std::string ArgStr = Quote ? "\"" : "";1629 ArgStr += Lib;1630 if (!Lib.ends_with_insensitive(".lib") && !Lib.ends_with_insensitive(".a"))1631 ArgStr += ".lib";1632 ArgStr += Quote ? "\"" : "";1633 return ArgStr;1634}1635 1636namespace {1637class WinX86_32TargetCodeGenInfo : public X86_32TargetCodeGenInfo {1638public:1639 WinX86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,1640 bool DarwinVectorABI, bool RetSmallStructInRegABI, bool Win32StructABI,1641 unsigned NumRegisterParameters)1642 : X86_32TargetCodeGenInfo(CGT, DarwinVectorABI, RetSmallStructInRegABI,1643 Win32StructABI, NumRegisterParameters, false) {}1644 1645 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,1646 CodeGen::CodeGenModule &CGM) const override;1647 1648 void getDependentLibraryOption(llvm::StringRef Lib,1649 llvm::SmallString<24> &Opt) const override {1650 Opt = "/DEFAULTLIB:";1651 Opt += qualifyWindowsLibrary(Lib);1652 }1653 1654 void getDetectMismatchOption(llvm::StringRef Name,1655 llvm::StringRef Value,1656 llvm::SmallString<32> &Opt) const override {1657 Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";1658 }1659};1660} // namespace1661 1662void WinX86_32TargetCodeGenInfo::setTargetAttributes(1663 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {1664 X86_32TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);1665 if (GV->isDeclaration())1666 return;1667 addStackProbeTargetAttributes(D, GV, CGM);1668}1669 1670namespace {1671class WinX86_64TargetCodeGenInfo : public TargetCodeGenInfo {1672public:1673 WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,1674 X86AVXABILevel AVXLevel)1675 : TargetCodeGenInfo(std::make_unique<WinX86_64ABIInfo>(CGT, AVXLevel)) {1676 SwiftInfo =1677 std::make_unique<SwiftABIInfo>(CGT, /*SwiftErrorInRegister=*/true);1678 }1679 1680 void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,1681 CodeGen::CodeGenModule &CGM) const override;1682 1683 int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {1684 return 7;1685 }1686 1687 bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,1688 llvm::Value *Address) const override {1689 llvm::Value *Eight8 = llvm::ConstantInt::get(CGF.Int8Ty, 8);1690 1691 // 0-15 are the 16 integer registers.1692 // 16 is %rip.1693 AssignToArrayRange(CGF.Builder, Address, Eight8, 0, 16);1694 return false;1695 }1696 1697 void getDependentLibraryOption(llvm::StringRef Lib,1698 llvm::SmallString<24> &Opt) const override {1699 Opt = "/DEFAULTLIB:";1700 Opt += qualifyWindowsLibrary(Lib);1701 }1702 1703 void getDetectMismatchOption(llvm::StringRef Name,1704 llvm::StringRef Value,1705 llvm::SmallString<32> &Opt) const override {1706 Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";1707 }1708};1709} // namespace1710 1711void WinX86_64TargetCodeGenInfo::setTargetAttributes(1712 const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {1713 TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);1714 if (GV->isDeclaration())1715 return;1716 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {1717 if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {1718 llvm::Function *Fn = cast<llvm::Function>(GV);1719 Fn->addFnAttr("stackrealign");1720 }1721 1722 addX86InterruptAttrs(FD, GV, CGM);1723 }1724 1725 addStackProbeTargetAttributes(D, GV, CGM);1726}1727 1728void X86_64ABIInfo::postMerge(unsigned AggregateSize, Class &Lo,1729 Class &Hi) const {1730 // AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done:1731 //1732 // (a) If one of the classes is Memory, the whole argument is passed in1733 // memory.1734 //1735 // (b) If X87UP is not preceded by X87, the whole argument is passed in1736 // memory.1737 //1738 // (c) If the size of the aggregate exceeds two eightbytes and the first1739 // eightbyte isn't SSE or any other eightbyte isn't SSEUP, the whole1740 // argument is passed in memory. NOTE: This is necessary to keep the1741 // ABI working for processors that don't support the __m256 type.1742 //1743 // (d) If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE.1744 //1745 // Some of these are enforced by the merging logic. Others can arise1746 // only with unions; for example:1747 // union { _Complex double; unsigned; }1748 //1749 // Note that clauses (b) and (c) were added in 0.98.1750 //1751 if (Hi == Memory)1752 Lo = Memory;1753 if (Hi == X87Up && Lo != X87 && honorsRevision0_98())1754 Lo = Memory;1755 if (AggregateSize > 128 && (Lo != SSE || Hi != SSEUp))1756 Lo = Memory;1757 if (Hi == SSEUp && Lo != SSE)1758 Hi = SSE;1759}1760 1761X86_64ABIInfo::Class X86_64ABIInfo::merge(Class Accum, Class Field) {1762 // AMD64-ABI 3.2.3p2: Rule 4. Each field of an object is1763 // classified recursively so that always two fields are1764 // considered. The resulting class is calculated according to1765 // the classes of the fields in the eightbyte:1766 //1767 // (a) If both classes are equal, this is the resulting class.1768 //1769 // (b) If one of the classes is NO_CLASS, the resulting class is1770 // the other class.1771 //1772 // (c) If one of the classes is MEMORY, the result is the MEMORY1773 // class.1774 //1775 // (d) If one of the classes is INTEGER, the result is the1776 // INTEGER.1777 //1778 // (e) If one of the classes is X87, X87UP, COMPLEX_X87 class,1779 // MEMORY is used as class.1780 //1781 // (f) Otherwise class SSE is used.1782 1783 // Accum should never be memory (we should have returned) or1784 // ComplexX87 (because this cannot be passed in a structure).1785 assert((Accum != Memory && Accum != ComplexX87) &&1786 "Invalid accumulated classification during merge.");1787 if (Accum == Field || Field == NoClass)1788 return Accum;1789 if (Field == Memory)1790 return Memory;1791 if (Accum == NoClass)1792 return Field;1793 if (Accum == Integer || Field == Integer)1794 return Integer;1795 if (Field == X87 || Field == X87Up || Field == ComplexX87 ||1796 Accum == X87 || Accum == X87Up)1797 return Memory;1798 return SSE;1799}1800 1801void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,1802 Class &Hi, bool isNamedArg, bool IsRegCall) const {1803 // FIXME: This code can be simplified by introducing a simple value class for1804 // Class pairs with appropriate constructor methods for the various1805 // situations.1806 1807 // FIXME: Some of the split computations are wrong; unaligned vectors1808 // shouldn't be passed in registers for example, so there is no chance they1809 // can straddle an eightbyte. Verify & simplify.1810 1811 Lo = Hi = NoClass;1812 1813 Class &Current = OffsetBase < 64 ? Lo : Hi;1814 Current = Memory;1815 1816 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {1817 BuiltinType::Kind k = BT->getKind();1818 1819 if (k == BuiltinType::Void) {1820 Current = NoClass;1821 } else if (k == BuiltinType::Int128 || k == BuiltinType::UInt128) {1822 Lo = Integer;1823 Hi = Integer;1824 } else if (k >= BuiltinType::Bool && k <= BuiltinType::LongLong) {1825 Current = Integer;1826 } else if (k == BuiltinType::Float || k == BuiltinType::Double ||1827 k == BuiltinType::Float16 || k == BuiltinType::BFloat16) {1828 Current = SSE;1829 } else if (k == BuiltinType::Float128) {1830 Lo = SSE;1831 Hi = SSEUp;1832 } else if (k == BuiltinType::LongDouble) {1833 const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();1834 if (LDF == &llvm::APFloat::IEEEquad()) {1835 Lo = SSE;1836 Hi = SSEUp;1837 } else if (LDF == &llvm::APFloat::x87DoubleExtended()) {1838 Lo = X87;1839 Hi = X87Up;1840 } else if (LDF == &llvm::APFloat::IEEEdouble()) {1841 Current = SSE;1842 } else1843 llvm_unreachable("unexpected long double representation!");1844 }1845 // FIXME: _Decimal32 and _Decimal64 are SSE.1846 // FIXME: _float128 and _Decimal128 are (SSE, SSEUp).1847 return;1848 }1849 1850 if (const auto *ED = Ty->getAsEnumDecl()) {1851 // Classify the underlying integer type.1852 classify(ED->getIntegerType(), OffsetBase, Lo, Hi, isNamedArg);1853 return;1854 }1855 1856 if (Ty->hasPointerRepresentation()) {1857 Current = Integer;1858 return;1859 }1860 1861 if (Ty->isMemberPointerType()) {1862 if (Ty->isMemberFunctionPointerType()) {1863 if (Has64BitPointers) {1864 // If Has64BitPointers, this is an {i64, i64}, so classify both1865 // Lo and Hi now.1866 Lo = Hi = Integer;1867 } else {1868 // Otherwise, with 32-bit pointers, this is an {i32, i32}. If that1869 // straddles an eightbyte boundary, Hi should be classified as well.1870 uint64_t EB_FuncPtr = (OffsetBase) / 64;1871 uint64_t EB_ThisAdj = (OffsetBase + 64 - 1) / 64;1872 if (EB_FuncPtr != EB_ThisAdj) {1873 Lo = Hi = Integer;1874 } else {1875 Current = Integer;1876 }1877 }1878 } else {1879 Current = Integer;1880 }1881 return;1882 }1883 1884 if (const VectorType *VT = Ty->getAs<VectorType>()) {1885 uint64_t Size = getContext().getTypeSize(VT);1886 if (Size == 1 || Size == 8 || Size == 16 || Size == 32) {1887 // gcc passes the following as integer:1888 // 4 bytes - <4 x char>, <2 x short>, <1 x int>, <1 x float>1889 // 2 bytes - <2 x char>, <1 x short>1890 // 1 byte - <1 x char>1891 Current = Integer;1892 1893 // If this type crosses an eightbyte boundary, it should be1894 // split.1895 uint64_t EB_Lo = (OffsetBase) / 64;1896 uint64_t EB_Hi = (OffsetBase + Size - 1) / 64;1897 if (EB_Lo != EB_Hi)1898 Hi = Lo;1899 } else if (Size == 64) {1900 QualType ElementType = VT->getElementType();1901 1902 // gcc passes <1 x double> in memory. :(1903 if (ElementType->isSpecificBuiltinType(BuiltinType::Double))1904 return;1905 1906 // gcc passes <1 x long long> as SSE but clang used to unconditionally1907 // pass them as integer. For platforms where clang is the de facto1908 // platform compiler, we must continue to use integer.1909 if (!classifyIntegerMMXAsSSE() &&1910 (ElementType->isSpecificBuiltinType(BuiltinType::LongLong) ||1911 ElementType->isSpecificBuiltinType(BuiltinType::ULongLong) ||1912 ElementType->isSpecificBuiltinType(BuiltinType::Long) ||1913 ElementType->isSpecificBuiltinType(BuiltinType::ULong)))1914 Current = Integer;1915 else1916 Current = SSE;1917 1918 // If this type crosses an eightbyte boundary, it should be1919 // split.1920 if (OffsetBase && OffsetBase != 64)1921 Hi = Lo;1922 } else if (Size == 128 ||1923 (isNamedArg && Size <= getNativeVectorSizeForAVXABI(AVXLevel))) {1924 QualType ElementType = VT->getElementType();1925 1926 // gcc passes 256 and 512 bit <X x __int128> vectors in memory. :(1927 if (passInt128VectorsInMem() && Size != 128 &&1928 (ElementType->isSpecificBuiltinType(BuiltinType::Int128) ||1929 ElementType->isSpecificBuiltinType(BuiltinType::UInt128)))1930 return;1931 1932 // Arguments of 256-bits are split into four eightbyte chunks. The1933 // least significant one belongs to class SSE and all the others to class1934 // SSEUP. The original Lo and Hi design considers that types can't be1935 // greater than 128-bits, so a 64-bit split in Hi and Lo makes sense.1936 // This design isn't correct for 256-bits, but since there're no cases1937 // where the upper parts would need to be inspected, avoid adding1938 // complexity and just consider Hi to match the 64-256 part.1939 //1940 // Note that per 3.5.7 of AMD64-ABI, 256-bit args are only passed in1941 // registers if they are "named", i.e. not part of the "..." of a1942 // variadic function.1943 //1944 // Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args are1945 // split into eight eightbyte chunks, one SSE and seven SSEUP.1946 Lo = SSE;1947 Hi = SSEUp;1948 }1949 return;1950 }1951 1952 if (const ComplexType *CT = Ty->getAs<ComplexType>()) {1953 QualType ET = getContext().getCanonicalType(CT->getElementType());1954 1955 uint64_t Size = getContext().getTypeSize(Ty);1956 if (ET->isIntegralOrEnumerationType()) {1957 if (Size <= 64)1958 Current = Integer;1959 else if (Size <= 128)1960 Lo = Hi = Integer;1961 } else if (ET->isFloat16Type() || ET == getContext().FloatTy ||1962 ET->isBFloat16Type()) {1963 Current = SSE;1964 } else if (ET == getContext().DoubleTy) {1965 Lo = Hi = SSE;1966 } else if (ET == getContext().LongDoubleTy) {1967 const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();1968 if (LDF == &llvm::APFloat::IEEEquad())1969 Current = Memory;1970 else if (LDF == &llvm::APFloat::x87DoubleExtended())1971 Current = ComplexX87;1972 else if (LDF == &llvm::APFloat::IEEEdouble())1973 Lo = Hi = SSE;1974 else1975 llvm_unreachable("unexpected long double representation!");1976 }1977 1978 // If this complex type crosses an eightbyte boundary then it1979 // should be split.1980 uint64_t EB_Real = (OffsetBase) / 64;1981 uint64_t EB_Imag = (OffsetBase + getContext().getTypeSize(ET)) / 64;1982 if (Hi == NoClass && EB_Real != EB_Imag)1983 Hi = Lo;1984 1985 return;1986 }1987 1988 if (const auto *EITy = Ty->getAs<BitIntType>()) {1989 if (EITy->getNumBits() <= 64)1990 Current = Integer;1991 else if (EITy->getNumBits() <= 128)1992 Lo = Hi = Integer;1993 // Larger values need to get passed in memory.1994 return;1995 }1996 1997 if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {1998 // Arrays are treated like structures.1999 2000 uint64_t Size = getContext().getTypeSize(Ty);2001 2002 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger2003 // than eight eightbytes, ..., it has class MEMORY.2004 // regcall ABI doesn't have limitation to an object. The only limitation2005 // is the free registers, which will be checked in computeInfo.2006 if (!IsRegCall && Size > 512)2007 return;2008 2009 // AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned2010 // fields, it has class MEMORY.2011 //2012 // Only need to check alignment of array base.2013 if (OffsetBase % getContext().getTypeAlign(AT->getElementType()))2014 return;2015 2016 // Otherwise implement simplified merge. We could be smarter about2017 // this, but it isn't worth it and would be harder to verify.2018 Current = NoClass;2019 uint64_t EltSize = getContext().getTypeSize(AT->getElementType());2020 uint64_t ArraySize = AT->getZExtSize();2021 2022 // The only case a 256-bit wide vector could be used is when the array2023 // contains a single 256-bit element. Since Lo and Hi logic isn't extended2024 // to work for sizes wider than 128, early check and fallback to memory.2025 //2026 if (Size > 128 &&2027 (Size != EltSize || Size > getNativeVectorSizeForAVXABI(AVXLevel)))2028 return;2029 2030 for (uint64_t i=0, Offset=OffsetBase; i<ArraySize; ++i, Offset += EltSize) {2031 Class FieldLo, FieldHi;2032 classify(AT->getElementType(), Offset, FieldLo, FieldHi, isNamedArg);2033 Lo = merge(Lo, FieldLo);2034 Hi = merge(Hi, FieldHi);2035 if (Lo == Memory || Hi == Memory)2036 break;2037 }2038 2039 postMerge(Size, Lo, Hi);2040 assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp array classification.");2041 return;2042 }2043 2044 if (const RecordType *RT = Ty->getAsCanonical<RecordType>()) {2045 uint64_t Size = getContext().getTypeSize(Ty);2046 2047 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger2048 // than eight eightbytes, ..., it has class MEMORY.2049 if (Size > 512)2050 return;2051 2052 // AMD64-ABI 3.2.3p2: Rule 2. If a C++ object has either a non-trivial2053 // copy constructor or a non-trivial destructor, it is passed by invisible2054 // reference.2055 if (getRecordArgABI(RT, getCXXABI()))2056 return;2057 2058 const RecordDecl *RD = RT->getDecl()->getDefinitionOrSelf();2059 2060 // Assume variable sized types are passed in memory.2061 if (RD->hasFlexibleArrayMember())2062 return;2063 2064 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);2065 2066 // Reset Lo class, this will be recomputed.2067 Current = NoClass;2068 2069 // If this is a C++ record, classify the bases first.2070 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {2071 for (const auto &I : CXXRD->bases()) {2072 assert(!I.isVirtual() && !I.getType()->isDependentType() &&2073 "Unexpected base class!");2074 const auto *Base = I.getType()->castAsCXXRecordDecl();2075 // Classify this field.2076 //2077 // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate exceeds a2078 // single eightbyte, each is classified separately. Each eightbyte gets2079 // initialized to class NO_CLASS.2080 Class FieldLo, FieldHi;2081 uint64_t Offset =2082 OffsetBase + getContext().toBits(Layout.getBaseClassOffset(Base));2083 classify(I.getType(), Offset, FieldLo, FieldHi, isNamedArg);2084 Lo = merge(Lo, FieldLo);2085 Hi = merge(Hi, FieldHi);2086 if (returnCXXRecordGreaterThan128InMem() &&2087 (Size > 128 && (Size != getContext().getTypeSize(I.getType()) ||2088 Size > getNativeVectorSizeForAVXABI(AVXLevel)))) {2089 // The only case a 256(or 512)-bit wide vector could be used to return2090 // is when CXX record contains a single 256(or 512)-bit element.2091 Lo = Memory;2092 }2093 if (Lo == Memory || Hi == Memory) {2094 postMerge(Size, Lo, Hi);2095 return;2096 }2097 }2098 }2099 2100 // Classify the fields one at a time, merging the results.2101 unsigned idx = 0;2102 bool UseClang11Compat = getContext().getLangOpts().getClangABICompat() <=2103 LangOptions::ClangABI::Ver11 ||2104 getContext().getTargetInfo().getTriple().isPS();2105 bool IsUnion = RT->isUnionType() && !UseClang11Compat;2106 2107 for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();2108 i != e; ++i, ++idx) {2109 uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);2110 bool BitField = i->isBitField();2111 2112 // Ignore padding bit-fields.2113 if (BitField && i->isUnnamedBitField())2114 continue;2115 2116 // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger than2117 // eight eightbytes, or it contains unaligned fields, it has class MEMORY.2118 //2119 // The only case a 256-bit or a 512-bit wide vector could be used is when2120 // the struct contains a single 256-bit or 512-bit element. Early check2121 // and fallback to memory.2122 //2123 // FIXME: Extended the Lo and Hi logic properly to work for size wider2124 // than 128.2125 if (Size > 128 &&2126 ((!IsUnion && Size != getContext().getTypeSize(i->getType())) ||2127 Size > getNativeVectorSizeForAVXABI(AVXLevel))) {2128 Lo = Memory;2129 postMerge(Size, Lo, Hi);2130 return;2131 }2132 2133 bool IsInMemory =2134 Offset % getContext().getTypeAlign(i->getType().getCanonicalType());2135 // Note, skip this test for bit-fields, see below.2136 if (!BitField && IsInMemory) {2137 Lo = Memory;2138 postMerge(Size, Lo, Hi);2139 return;2140 }2141 2142 // Classify this field.2143 //2144 // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate2145 // exceeds a single eightbyte, each is classified2146 // separately. Each eightbyte gets initialized to class2147 // NO_CLASS.2148 Class FieldLo, FieldHi;2149 2150 // Bit-fields require special handling, they do not force the2151 // structure to be passed in memory even if unaligned, and2152 // therefore they can straddle an eightbyte.2153 if (BitField) {2154 assert(!i->isUnnamedBitField());2155 uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);2156 uint64_t Size = i->getBitWidthValue();2157 2158 uint64_t EB_Lo = Offset / 64;2159 uint64_t EB_Hi = (Offset + Size - 1) / 64;2160 2161 if (EB_Lo) {2162 assert(EB_Hi == EB_Lo && "Invalid classification, type > 16 bytes.");2163 FieldLo = NoClass;2164 FieldHi = Integer;2165 } else {2166 FieldLo = Integer;2167 FieldHi = EB_Hi ? Integer : NoClass;2168 }2169 } else2170 classify(i->getType(), Offset, FieldLo, FieldHi, isNamedArg);2171 Lo = merge(Lo, FieldLo);2172 Hi = merge(Hi, FieldHi);2173 if (Lo == Memory || Hi == Memory)2174 break;2175 }2176 2177 postMerge(Size, Lo, Hi);2178 }2179}2180 2181ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const {2182 // If this is a scalar LLVM value then assume LLVM will pass it in the right2183 // place naturally.2184 if (!isAggregateTypeForABI(Ty)) {2185 // Treat an enum type as its underlying type.2186 if (const auto *ED = Ty->getAsEnumDecl())2187 Ty = ED->getIntegerType();2188 2189 if (Ty->isBitIntType())2190 return getNaturalAlignIndirect(Ty, getDataLayout().getAllocaAddrSpace());2191 2192 return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)2193 : ABIArgInfo::getDirect());2194 }2195 2196 return getNaturalAlignIndirect(Ty, getDataLayout().getAllocaAddrSpace());2197}2198 2199bool X86_64ABIInfo::IsIllegalVectorType(QualType Ty) const {2200 if (const VectorType *VecTy = Ty->getAs<VectorType>()) {2201 uint64_t Size = getContext().getTypeSize(VecTy);2202 unsigned LargestVector = getNativeVectorSizeForAVXABI(AVXLevel);2203 if (Size <= 64 || Size > LargestVector)2204 return true;2205 QualType EltTy = VecTy->getElementType();2206 if (passInt128VectorsInMem() &&2207 (EltTy->isSpecificBuiltinType(BuiltinType::Int128) ||2208 EltTy->isSpecificBuiltinType(BuiltinType::UInt128)))2209 return true;2210 }2211 2212 return false;2213}2214 2215ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty,2216 unsigned freeIntRegs) const {2217 // If this is a scalar LLVM value then assume LLVM will pass it in the right2218 // place naturally.2219 //2220 // This assumption is optimistic, as there could be free registers available2221 // when we need to pass this argument in memory, and LLVM could try to pass2222 // the argument in the free register. This does not seem to happen currently,2223 // but this code would be much safer if we could mark the argument with2224 // 'onstack'. See PR12193.2225 if (!isAggregateTypeForABI(Ty) && !IsIllegalVectorType(Ty) &&2226 !Ty->isBitIntType()) {2227 // Treat an enum type as its underlying type.2228 if (const auto *ED = Ty->getAsEnumDecl())2229 Ty = ED->getIntegerType();2230 2231 return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)2232 : ABIArgInfo::getDirect());2233 }2234 2235 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))2236 return getNaturalAlignIndirect(Ty, getDataLayout().getAllocaAddrSpace(),2237 RAA == CGCXXABI::RAA_DirectInMemory);2238 2239 // Compute the byval alignment. We specify the alignment of the byval in all2240 // cases so that the mid-level optimizer knows the alignment of the byval.2241 unsigned Align = std::max(getContext().getTypeAlign(Ty) / 8, 8U);2242 2243 // Attempt to avoid passing indirect results using byval when possible. This2244 // is important for good codegen.2245 //2246 // We do this by coercing the value into a scalar type which the backend can2247 // handle naturally (i.e., without using byval).2248 //2249 // For simplicity, we currently only do this when we have exhausted all of the2250 // free integer registers. Doing this when there are free integer registers2251 // would require more care, as we would have to ensure that the coerced value2252 // did not claim the unused register. That would require either reording the2253 // arguments to the function (so that any subsequent inreg values came first),2254 // or only doing this optimization when there were no following arguments that2255 // might be inreg.2256 //2257 // We currently expect it to be rare (particularly in well written code) for2258 // arguments to be passed on the stack when there are still free integer2259 // registers available (this would typically imply large structs being passed2260 // by value), so this seems like a fair tradeoff for now.2261 //2262 // We can revisit this if the backend grows support for 'onstack' parameter2263 // attributes. See PR12193.2264 if (freeIntRegs == 0) {2265 uint64_t Size = getContext().getTypeSize(Ty);2266 2267 // If this type fits in an eightbyte, coerce it into the matching integral2268 // type, which will end up on the stack (with alignment 8).2269 if (Align == 8 && Size <= 64)2270 return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),2271 Size));2272 }2273 2274 return ABIArgInfo::getIndirect(CharUnits::fromQuantity(Align),2275 getDataLayout().getAllocaAddrSpace());2276}2277 2278/// The ABI specifies that a value should be passed in a full vector XMM/YMM2279/// register. Pick an LLVM IR type that will be passed as a vector register.2280llvm::Type *X86_64ABIInfo::GetByteVectorType(QualType Ty) const {2281 // Wrapper structs/arrays that only contain vectors are passed just like2282 // vectors; strip them off if present.2283 if (const Type *InnerTy = isSingleElementStruct(Ty, getContext()))2284 Ty = QualType(InnerTy, 0);2285 2286 llvm::Type *IRType = CGT.ConvertType(Ty);2287 if (isa<llvm::VectorType>(IRType)) {2288 // Don't pass vXi128 vectors in their native type, the backend can't2289 // legalize them.2290 if (passInt128VectorsInMem() &&2291 cast<llvm::VectorType>(IRType)->getElementType()->isIntegerTy(128)) {2292 // Use a vXi64 vector.2293 uint64_t Size = getContext().getTypeSize(Ty);2294 return llvm::FixedVectorType::get(llvm::Type::getInt64Ty(getVMContext()),2295 Size / 64);2296 }2297 2298 return IRType;2299 }2300 2301 if (IRType->getTypeID() == llvm::Type::FP128TyID)2302 return IRType;2303 2304 // We couldn't find the preferred IR vector type for 'Ty'.2305 uint64_t Size = getContext().getTypeSize(Ty);2306 assert((Size == 128 || Size == 256 || Size == 512) && "Invalid type found!");2307 2308 2309 // Return a LLVM IR vector type based on the size of 'Ty'.2310 return llvm::FixedVectorType::get(llvm::Type::getDoubleTy(getVMContext()),2311 Size / 64);2312}2313 2314/// BitsContainNoUserData - Return true if the specified [start,end) bit range2315/// is known to either be off the end of the specified type or being in2316/// alignment padding. The user type specified is known to be at most 128 bits2317/// in size, and have passed through X86_64ABIInfo::classify with a successful2318/// classification that put one of the two halves in the INTEGER class.2319///2320/// It is conservatively correct to return false.2321static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,2322 unsigned EndBit, ASTContext &Context) {2323 // If the bytes being queried are off the end of the type, there is no user2324 // data hiding here. This handles analysis of builtins, vectors and other2325 // types that don't contain interesting padding.2326 unsigned TySize = (unsigned)Context.getTypeSize(Ty);2327 if (TySize <= StartBit)2328 return true;2329 2330 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {2331 unsigned EltSize = (unsigned)Context.getTypeSize(AT->getElementType());2332 unsigned NumElts = (unsigned)AT->getZExtSize();2333 2334 // Check each element to see if the element overlaps with the queried range.2335 for (unsigned i = 0; i != NumElts; ++i) {2336 // If the element is after the span we care about, then we're done..2337 unsigned EltOffset = i*EltSize;2338 if (EltOffset >= EndBit) break;2339 2340 unsigned EltStart = EltOffset < StartBit ? StartBit-EltOffset :0;2341 if (!BitsContainNoUserData(AT->getElementType(), EltStart,2342 EndBit-EltOffset, Context))2343 return false;2344 }2345 // If it overlaps no elements, then it is safe to process as padding.2346 return true;2347 }2348 2349 if (const auto *RD = Ty->getAsRecordDecl()) {2350 const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);2351 2352 // If this is a C++ record, check the bases first.2353 if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {2354 for (const auto &I : CXXRD->bases()) {2355 assert(!I.isVirtual() && !I.getType()->isDependentType() &&2356 "Unexpected base class!");2357 const auto *Base = I.getType()->castAsCXXRecordDecl();2358 2359 // If the base is after the span we care about, ignore it.2360 unsigned BaseOffset = Context.toBits(Layout.getBaseClassOffset(Base));2361 if (BaseOffset >= EndBit) continue;2362 2363 unsigned BaseStart = BaseOffset < StartBit ? StartBit-BaseOffset :0;2364 if (!BitsContainNoUserData(I.getType(), BaseStart,2365 EndBit-BaseOffset, Context))2366 return false;2367 }2368 }2369 2370 // Verify that no field has data that overlaps the region of interest. Yes2371 // this could be sped up a lot by being smarter about queried fields,2372 // however we're only looking at structs up to 16 bytes, so we don't care2373 // much.2374 unsigned idx = 0;2375 for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();2376 i != e; ++i, ++idx) {2377 unsigned FieldOffset = (unsigned)Layout.getFieldOffset(idx);2378 2379 // If we found a field after the region we care about, then we're done.2380 if (FieldOffset >= EndBit) break;2381 2382 unsigned FieldStart = FieldOffset < StartBit ? StartBit-FieldOffset :0;2383 if (!BitsContainNoUserData(i->getType(), FieldStart, EndBit-FieldOffset,2384 Context))2385 return false;2386 }2387 2388 // If nothing in this record overlapped the area of interest, then we're2389 // clean.2390 return true;2391 }2392 2393 return false;2394}2395 2396/// getFPTypeAtOffset - Return a floating point type at the specified offset.2397static llvm::Type *getFPTypeAtOffset(llvm::Type *IRType, unsigned IROffset,2398 const llvm::DataLayout &TD) {2399 if (IROffset == 0 && IRType->isFloatingPointTy())2400 return IRType;2401 2402 // If this is a struct, recurse into the field at the specified offset.2403 if (llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {2404 if (!STy->getNumContainedTypes())2405 return nullptr;2406 2407 const llvm::StructLayout *SL = TD.getStructLayout(STy);2408 unsigned Elt = SL->getElementContainingOffset(IROffset);2409 IROffset -= SL->getElementOffset(Elt);2410 return getFPTypeAtOffset(STy->getElementType(Elt), IROffset, TD);2411 }2412 2413 // If this is an array, recurse into the field at the specified offset.2414 if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {2415 llvm::Type *EltTy = ATy->getElementType();2416 unsigned EltSize = TD.getTypeAllocSize(EltTy);2417 IROffset -= IROffset / EltSize * EltSize;2418 return getFPTypeAtOffset(EltTy, IROffset, TD);2419 }2420 2421 return nullptr;2422}2423 2424/// GetSSETypeAtOffset - Return a type that will be passed by the backend in the2425/// low 8 bytes of an XMM register, corresponding to the SSE class.2426llvm::Type *X86_64ABIInfo::2427GetSSETypeAtOffset(llvm::Type *IRType, unsigned IROffset,2428 QualType SourceTy, unsigned SourceOffset) const {2429 const llvm::DataLayout &TD = getDataLayout();2430 unsigned SourceSize =2431 (unsigned)getContext().getTypeSize(SourceTy) / 8 - SourceOffset;2432 llvm::Type *T0 = getFPTypeAtOffset(IRType, IROffset, TD);2433 if (!T0 || T0->isDoubleTy())2434 return llvm::Type::getDoubleTy(getVMContext());2435 2436 // Get the adjacent FP type.2437 llvm::Type *T1 = nullptr;2438 unsigned T0Size = TD.getTypeAllocSize(T0);2439 if (SourceSize > T0Size)2440 T1 = getFPTypeAtOffset(IRType, IROffset + T0Size, TD);2441 if (T1 == nullptr) {2442 // Check if IRType is a half/bfloat + float. float type will be in IROffset+4 due2443 // to its alignment.2444 if (T0->is16bitFPTy() && SourceSize > 4)2445 T1 = getFPTypeAtOffset(IRType, IROffset + 4, TD);2446 // If we can't get a second FP type, return a simple half or float.2447 // avx512fp16-abi.c:pr51813_2 shows it works to return float for2448 // {float, i8} too.2449 if (T1 == nullptr)2450 return T0;2451 }2452 2453 if (T0->isFloatTy() && T1->isFloatTy())2454 return llvm::FixedVectorType::get(T0, 2);2455 2456 if (T0->is16bitFPTy() && T1->is16bitFPTy()) {2457 llvm::Type *T2 = nullptr;2458 if (SourceSize > 4)2459 T2 = getFPTypeAtOffset(IRType, IROffset + 4, TD);2460 if (T2 == nullptr)2461 return llvm::FixedVectorType::get(T0, 2);2462 return llvm::FixedVectorType::get(T0, 4);2463 }2464 2465 if (T0->is16bitFPTy() || T1->is16bitFPTy())2466 return llvm::FixedVectorType::get(llvm::Type::getHalfTy(getVMContext()), 4);2467 2468 return llvm::Type::getDoubleTy(getVMContext());2469}2470 2471/// GetINTEGERTypeAtOffset - The ABI specifies that a value should be passed in2472/// one or more 8-byte GPRs. This means that we either have a scalar or we are2473/// talking about the high and/or low part of an up-to-16-byte struct. This2474/// routine picks the best LLVM IR type to represent this, which may be i64 or2475/// may be anything else that the backend will pass in GPRs that works better2476/// (e.g. i8, %foo*, etc).2477///2478/// PrefType is an LLVM IR type that corresponds to (part of) the IR type for2479/// the source type. IROffset is an offset in bytes into the LLVM IR type that2480/// the 8-byte value references. PrefType may be null.2481///2482/// SourceTy is the source-level type for the entire argument. SourceOffset is2483/// an offset into this that we're processing (which is always either 0 or 8).2484///2485llvm::Type *X86_64ABIInfo::2486GetINTEGERTypeAtOffset(llvm::Type *IRType, unsigned IROffset,2487 QualType SourceTy, unsigned SourceOffset) const {2488 // If we're dealing with an un-offset LLVM IR type, then it means that we're2489 // returning an 8-byte unit starting with it. See if we can safely use it.2490 if (IROffset == 0) {2491 // Pointers and int64's always fill the 8-byte unit.2492 if ((isa<llvm::PointerType>(IRType) && Has64BitPointers) ||2493 IRType->isIntegerTy(64))2494 return IRType;2495 2496 // If we have a 1/2/4-byte integer, we can use it only if the rest of the2497 // goodness in the source type is just tail padding. This is allowed to2498 // kick in for struct {double,int} on the int, but not on2499 // struct{double,int,int} because we wouldn't return the second int. We2500 // have to do this analysis on the source type because we can't depend on2501 // unions being lowered a specific way etc.2502 if (IRType->isIntegerTy(8) || IRType->isIntegerTy(16) ||2503 IRType->isIntegerTy(32) ||2504 (isa<llvm::PointerType>(IRType) && !Has64BitPointers)) {2505 unsigned BitWidth = isa<llvm::PointerType>(IRType) ? 32 :2506 cast<llvm::IntegerType>(IRType)->getBitWidth();2507 2508 if (BitsContainNoUserData(SourceTy, SourceOffset*8+BitWidth,2509 SourceOffset*8+64, getContext()))2510 return IRType;2511 }2512 }2513 2514 if (llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {2515 // If this is a struct, recurse into the field at the specified offset.2516 const llvm::StructLayout *SL = getDataLayout().getStructLayout(STy);2517 if (IROffset < SL->getSizeInBytes()) {2518 unsigned FieldIdx = SL->getElementContainingOffset(IROffset);2519 IROffset -= SL->getElementOffset(FieldIdx);2520 2521 return GetINTEGERTypeAtOffset(STy->getElementType(FieldIdx), IROffset,2522 SourceTy, SourceOffset);2523 }2524 }2525 2526 if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {2527 llvm::Type *EltTy = ATy->getElementType();2528 unsigned EltSize = getDataLayout().getTypeAllocSize(EltTy);2529 unsigned EltOffset = IROffset/EltSize*EltSize;2530 return GetINTEGERTypeAtOffset(EltTy, IROffset-EltOffset, SourceTy,2531 SourceOffset);2532 }2533 2534 // if we have a 128-bit integer, we can pass it safely using an i1282535 // so we return that2536 if (IRType->isIntegerTy(128)) {2537 assert(IROffset == 0);2538 return IRType;2539 }2540 2541 // Okay, we don't have any better idea of what to pass, so we pass this in an2542 // integer register that isn't too big to fit the rest of the struct.2543 unsigned TySizeInBytes =2544 (unsigned)getContext().getTypeSizeInChars(SourceTy).getQuantity();2545 2546 assert(TySizeInBytes != SourceOffset && "Empty field?");2547 2548 // It is always safe to classify this as an integer type up to i64 that2549 // isn't larger than the structure.2550 return llvm::IntegerType::get(getVMContext(),2551 std::min(TySizeInBytes-SourceOffset, 8U)*8);2552}2553 2554 2555/// GetX86_64ByValArgumentPair - Given a high and low type that can ideally2556/// be used as elements of a two register pair to pass or return, return a2557/// first class aggregate to represent them. For example, if the low part of2558/// a by-value argument should be passed as i32* and the high part as float,2559/// return {i32*, float}.2560static llvm::Type *2561GetX86_64ByValArgumentPair(llvm::Type *Lo, llvm::Type *Hi,2562 const llvm::DataLayout &TD) {2563 // In order to correctly satisfy the ABI, we need to the high part to start2564 // at offset 8. If the high and low parts we inferred are both 4-byte types2565 // (e.g. i32 and i32) then the resultant struct type ({i32,i32}) won't have2566 // the second element at offset 8. Check for this:2567 unsigned LoSize = (unsigned)TD.getTypeAllocSize(Lo);2568 llvm::Align HiAlign = TD.getABITypeAlign(Hi);2569 unsigned HiStart = llvm::alignTo(LoSize, HiAlign);2570 assert(HiStart != 0 && HiStart <= 8 && "Invalid x86-64 argument pair!");2571 2572 // To handle this, we have to increase the size of the low part so that the2573 // second element will start at an 8 byte offset. We can't increase the size2574 // of the second element because it might make us access off the end of the2575 // struct.2576 if (HiStart != 8) {2577 // There are usually two sorts of types the ABI generation code can produce2578 // for the low part of a pair that aren't 8 bytes in size: half, float or2579 // i8/i16/i32. This can also include pointers when they are 32-bit (X32).2580 // Promote these to a larger type.2581 if (Lo->isHalfTy() || Lo->isFloatTy())2582 Lo = llvm::Type::getDoubleTy(Lo->getContext());2583 else {2584 assert((Lo->isIntegerTy() || Lo->isPointerTy())2585 && "Invalid/unknown lo type");2586 Lo = llvm::Type::getInt64Ty(Lo->getContext());2587 }2588 }2589 2590 llvm::StructType *Result = llvm::StructType::get(Lo, Hi);2591 2592 // Verify that the second element is at an 8-byte offset.2593 assert(TD.getStructLayout(Result)->getElementOffset(1) == 8 &&2594 "Invalid x86-64 argument pair!");2595 return Result;2596}2597 2598ABIArgInfo X86_64ABIInfo::classifyReturnType(QualType RetTy) const {2599 // AMD64-ABI 3.2.3p4: Rule 1. Classify the return type with the2600 // classification algorithm.2601 X86_64ABIInfo::Class Lo, Hi;2602 classify(RetTy, 0, Lo, Hi, /*isNamedArg*/ true);2603 2604 // Check some invariants.2605 assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");2606 assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");2607 2608 llvm::Type *ResType = nullptr;2609 switch (Lo) {2610 case NoClass:2611 if (Hi == NoClass)2612 return ABIArgInfo::getIgnore();2613 // If the low part is just padding, it takes no register, leave ResType2614 // null.2615 assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&2616 "Unknown missing lo part");2617 break;2618 2619 case SSEUp:2620 case X87Up:2621 llvm_unreachable("Invalid classification for lo word.");2622 2623 // AMD64-ABI 3.2.3p4: Rule 2. Types of class memory are returned via2624 // hidden argument.2625 case Memory:2626 return getIndirectReturnResult(RetTy);2627 2628 // AMD64-ABI 3.2.3p4: Rule 3. If the class is INTEGER, the next2629 // available register of the sequence %rax, %rdx is used.2630 case Integer:2631 ResType = GetINTEGERTypeAtOffset(CGT.ConvertType(RetTy), 0, RetTy, 0);2632 2633 // If we have a sign or zero extended integer, make sure to return Extend2634 // so that the parameter gets the right LLVM IR attributes.2635 if (Hi == NoClass && isa<llvm::IntegerType>(ResType)) {2636 // Treat an enum type as its underlying type.2637 if (const auto *ED = RetTy->getAsEnumDecl())2638 RetTy = ED->getIntegerType();2639 2640 if (RetTy->isIntegralOrEnumerationType() &&2641 isPromotableIntegerTypeForABI(RetTy))2642 return ABIArgInfo::getExtend(RetTy);2643 }2644 2645 if (ResType->isIntegerTy(128)) {2646 // i128 are passed directly2647 assert(Hi == Integer);2648 return ABIArgInfo::getDirect(ResType);2649 }2650 break;2651 2652 // AMD64-ABI 3.2.3p4: Rule 4. If the class is SSE, the next2653 // available SSE register of the sequence %xmm0, %xmm1 is used.2654 case SSE:2655 ResType = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 0, RetTy, 0);2656 break;2657 2658 // AMD64-ABI 3.2.3p4: Rule 6. If the class is X87, the value is2659 // returned on the X87 stack in %st0 as 80-bit x87 number.2660 case X87:2661 ResType = llvm::Type::getX86_FP80Ty(getVMContext());2662 break;2663 2664 // AMD64-ABI 3.2.3p4: Rule 8. If the class is COMPLEX_X87, the real2665 // part of the value is returned in %st0 and the imaginary part in2666 // %st1.2667 case ComplexX87:2668 assert(Hi == ComplexX87 && "Unexpected ComplexX87 classification.");2669 ResType = llvm::StructType::get(llvm::Type::getX86_FP80Ty(getVMContext()),2670 llvm::Type::getX86_FP80Ty(getVMContext()));2671 break;2672 }2673 2674 llvm::Type *HighPart = nullptr;2675 switch (Hi) {2676 // Memory was handled previously and X87 should2677 // never occur as a hi class.2678 case Memory:2679 case X87:2680 llvm_unreachable("Invalid classification for hi word.");2681 2682 case ComplexX87: // Previously handled.2683 case NoClass:2684 break;2685 2686 case Integer:2687 HighPart = GetINTEGERTypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);2688 if (Lo == NoClass) // Return HighPart at offset 8 in memory.2689 return ABIArgInfo::getDirect(HighPart, 8);2690 break;2691 case SSE:2692 HighPart = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);2693 if (Lo == NoClass) // Return HighPart at offset 8 in memory.2694 return ABIArgInfo::getDirect(HighPart, 8);2695 break;2696 2697 // AMD64-ABI 3.2.3p4: Rule 5. If the class is SSEUP, the eightbyte2698 // is passed in the next available eightbyte chunk if the last used2699 // vector register.2700 //2701 // SSEUP should always be preceded by SSE, just widen.2702 case SSEUp:2703 assert(Lo == SSE && "Unexpected SSEUp classification.");2704 ResType = GetByteVectorType(RetTy);2705 break;2706 2707 // AMD64-ABI 3.2.3p4: Rule 7. If the class is X87UP, the value is2708 // returned together with the previous X87 value in %st0.2709 case X87Up:2710 // If X87Up is preceded by X87, we don't need to do2711 // anything. However, in some cases with unions it may not be2712 // preceded by X87. In such situations we follow gcc and pass the2713 // extra bits in an SSE reg.2714 if (Lo != X87) {2715 HighPart = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);2716 if (Lo == NoClass) // Return HighPart at offset 8 in memory.2717 return ABIArgInfo::getDirect(HighPart, 8);2718 }2719 break;2720 }2721 2722 // If a high part was specified, merge it together with the low part. It is2723 // known to pass in the high eightbyte of the result. We do this by forming a2724 // first class struct aggregate with the high and low part: {low, high}2725 if (HighPart)2726 ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getDataLayout());2727 2728 return ABIArgInfo::getDirect(ResType);2729}2730 2731ABIArgInfo2732X86_64ABIInfo::classifyArgumentType(QualType Ty, unsigned freeIntRegs,2733 unsigned &neededInt, unsigned &neededSSE,2734 bool isNamedArg, bool IsRegCall) const {2735 Ty = useFirstFieldIfTransparentUnion(Ty);2736 2737 X86_64ABIInfo::Class Lo, Hi;2738 classify(Ty, 0, Lo, Hi, isNamedArg, IsRegCall);2739 2740 // Check some invariants.2741 // FIXME: Enforce these by construction.2742 assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");2743 assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");2744 2745 neededInt = 0;2746 neededSSE = 0;2747 llvm::Type *ResType = nullptr;2748 switch (Lo) {2749 case NoClass:2750 if (Hi == NoClass)2751 return ABIArgInfo::getIgnore();2752 // If the low part is just padding, it takes no register, leave ResType2753 // null.2754 assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&2755 "Unknown missing lo part");2756 break;2757 2758 // AMD64-ABI 3.2.3p3: Rule 1. If the class is MEMORY, pass the argument2759 // on the stack.2760 case Memory:2761 2762 // AMD64-ABI 3.2.3p3: Rule 5. If the class is X87, X87UP or2763 // COMPLEX_X87, it is passed in memory.2764 case X87:2765 case ComplexX87:2766 if (getRecordArgABI(Ty, getCXXABI()) == CGCXXABI::RAA_Indirect)2767 ++neededInt;2768 return getIndirectResult(Ty, freeIntRegs);2769 2770 case SSEUp:2771 case X87Up:2772 llvm_unreachable("Invalid classification for lo word.");2773 2774 // AMD64-ABI 3.2.3p3: Rule 2. If the class is INTEGER, the next2775 // available register of the sequence %rdi, %rsi, %rdx, %rcx, %r82776 // and %r9 is used.2777 case Integer:2778 ++neededInt;2779 2780 // Pick an 8-byte type based on the preferred type.2781 ResType = GetINTEGERTypeAtOffset(CGT.ConvertType(Ty), 0, Ty, 0);2782 2783 // If we have a sign or zero extended integer, make sure to return Extend2784 // so that the parameter gets the right LLVM IR attributes.2785 if (Hi == NoClass && isa<llvm::IntegerType>(ResType)) {2786 // Treat an enum type as its underlying type.2787 if (const auto *ED = Ty->getAsEnumDecl())2788 Ty = ED->getIntegerType();2789 2790 if (Ty->isIntegralOrEnumerationType() &&2791 isPromotableIntegerTypeForABI(Ty))2792 return ABIArgInfo::getExtend(Ty, CGT.ConvertType(Ty));2793 }2794 2795 if (ResType->isIntegerTy(128)) {2796 assert(Hi == Integer);2797 ++neededInt;2798 return ABIArgInfo::getDirect(ResType);2799 }2800 break;2801 2802 // AMD64-ABI 3.2.3p3: Rule 3. If the class is SSE, the next2803 // available SSE register is used, the registers are taken in the2804 // order from %xmm0 to %xmm7.2805 case SSE: {2806 llvm::Type *IRType = CGT.ConvertType(Ty);2807 ResType = GetSSETypeAtOffset(IRType, 0, Ty, 0);2808 ++neededSSE;2809 break;2810 }2811 }2812 2813 llvm::Type *HighPart = nullptr;2814 switch (Hi) {2815 // Memory was handled previously, ComplexX87 and X87 should2816 // never occur as hi classes, and X87Up must be preceded by X87,2817 // which is passed in memory.2818 case Memory:2819 case X87:2820 case ComplexX87:2821 llvm_unreachable("Invalid classification for hi word.");2822 2823 case NoClass: break;2824 2825 case Integer:2826 ++neededInt;2827 // Pick an 8-byte type based on the preferred type.2828 HighPart = GetINTEGERTypeAtOffset(CGT.ConvertType(Ty), 8, Ty, 8);2829 2830 if (Lo == NoClass) // Pass HighPart at offset 8 in memory.2831 return ABIArgInfo::getDirect(HighPart, 8);2832 break;2833 2834 // X87Up generally doesn't occur here (long double is passed in2835 // memory), except in situations involving unions.2836 case X87Up:2837 case SSE:2838 ++neededSSE;2839 HighPart = GetSSETypeAtOffset(CGT.ConvertType(Ty), 8, Ty, 8);2840 2841 if (Lo == NoClass) // Pass HighPart at offset 8 in memory.2842 return ABIArgInfo::getDirect(HighPart, 8);2843 break;2844 2845 // AMD64-ABI 3.2.3p3: Rule 4. If the class is SSEUP, the2846 // eightbyte is passed in the upper half of the last used SSE2847 // register. This only happens when 128-bit vectors are passed.2848 case SSEUp:2849 assert(Lo == SSE && "Unexpected SSEUp classification");2850 ResType = GetByteVectorType(Ty);2851 break;2852 }2853 2854 // If a high part was specified, merge it together with the low part. It is2855 // known to pass in the high eightbyte of the result. We do this by forming a2856 // first class struct aggregate with the high and low part: {low, high}2857 if (HighPart)2858 ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getDataLayout());2859 2860 return ABIArgInfo::getDirect(ResType);2861}2862 2863ABIArgInfo2864X86_64ABIInfo::classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,2865 unsigned &NeededSSE,2866 unsigned &MaxVectorWidth) const {2867 auto *RD =2868 cast<RecordType>(Ty.getCanonicalType())->getDecl()->getDefinitionOrSelf();2869 2870 if (RD->hasFlexibleArrayMember())2871 return getIndirectReturnResult(Ty);2872 2873 // Sum up bases2874 if (auto CXXRD = dyn_cast<CXXRecordDecl>(RD)) {2875 if (CXXRD->isDynamicClass()) {2876 NeededInt = NeededSSE = 0;2877 return getIndirectReturnResult(Ty);2878 }2879 2880 for (const auto &I : CXXRD->bases())2881 if (classifyRegCallStructTypeImpl(I.getType(), NeededInt, NeededSSE,2882 MaxVectorWidth)2883 .isIndirect()) {2884 NeededInt = NeededSSE = 0;2885 return getIndirectReturnResult(Ty);2886 }2887 }2888 2889 // Sum up members2890 for (const auto *FD : RD->fields()) {2891 QualType MTy = FD->getType();2892 if (MTy->isRecordType() && !MTy->isUnionType()) {2893 if (classifyRegCallStructTypeImpl(MTy, NeededInt, NeededSSE,2894 MaxVectorWidth)2895 .isIndirect()) {2896 NeededInt = NeededSSE = 0;2897 return getIndirectReturnResult(Ty);2898 }2899 } else {2900 unsigned LocalNeededInt, LocalNeededSSE;2901 if (classifyArgumentType(MTy, UINT_MAX, LocalNeededInt, LocalNeededSSE,2902 true, true)2903 .isIndirect()) {2904 NeededInt = NeededSSE = 0;2905 return getIndirectReturnResult(Ty);2906 }2907 if (const auto *AT = getContext().getAsConstantArrayType(MTy))2908 MTy = AT->getElementType();2909 if (const auto *VT = MTy->getAs<VectorType>())2910 if (getContext().getTypeSize(VT) > MaxVectorWidth)2911 MaxVectorWidth = getContext().getTypeSize(VT);2912 NeededInt += LocalNeededInt;2913 NeededSSE += LocalNeededSSE;2914 }2915 }2916 2917 return ABIArgInfo::getDirect();2918}2919 2920ABIArgInfo2921X86_64ABIInfo::classifyRegCallStructType(QualType Ty, unsigned &NeededInt,2922 unsigned &NeededSSE,2923 unsigned &MaxVectorWidth) const {2924 2925 NeededInt = 0;2926 NeededSSE = 0;2927 MaxVectorWidth = 0;2928 2929 return classifyRegCallStructTypeImpl(Ty, NeededInt, NeededSSE,2930 MaxVectorWidth);2931}2932 2933void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {2934 2935 const unsigned CallingConv = FI.getCallingConvention();2936 // It is possible to force Win64 calling convention on any x86_64 target by2937 // using __attribute__((ms_abi)). In such case to correctly emit Win642938 // compatible code delegate this call to WinX86_64ABIInfo::computeInfo.2939 if (CallingConv == llvm::CallingConv::Win64) {2940 WinX86_64ABIInfo Win64ABIInfo(CGT, AVXLevel);2941 Win64ABIInfo.computeInfo(FI);2942 return;2943 }2944 2945 bool IsRegCall = CallingConv == llvm::CallingConv::X86_RegCall;2946 2947 // Keep track of the number of assigned registers.2948 unsigned FreeIntRegs = IsRegCall ? 11 : 6;2949 unsigned FreeSSERegs = IsRegCall ? 16 : 8;2950 unsigned NeededInt = 0, NeededSSE = 0, MaxVectorWidth = 0;2951 2952 if (!::classifyReturnType(getCXXABI(), FI, *this)) {2953 if (IsRegCall && FI.getReturnType()->getTypePtr()->isRecordType() &&2954 !FI.getReturnType()->getTypePtr()->isUnionType()) {2955 FI.getReturnInfo() = classifyRegCallStructType(2956 FI.getReturnType(), NeededInt, NeededSSE, MaxVectorWidth);2957 if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {2958 FreeIntRegs -= NeededInt;2959 FreeSSERegs -= NeededSSE;2960 } else {2961 FI.getReturnInfo() = getIndirectReturnResult(FI.getReturnType());2962 }2963 } else if (IsRegCall && FI.getReturnType()->getAs<ComplexType>() &&2964 getContext().getCanonicalType(FI.getReturnType()2965 ->getAs<ComplexType>()2966 ->getElementType()) ==2967 getContext().LongDoubleTy)2968 // Complex Long Double Type is passed in Memory when Regcall2969 // calling convention is used.2970 FI.getReturnInfo() = getIndirectReturnResult(FI.getReturnType());2971 else2972 FI.getReturnInfo() = classifyReturnType(FI.getReturnType());2973 }2974 2975 // If the return value is indirect, then the hidden argument is consuming one2976 // integer register.2977 if (FI.getReturnInfo().isIndirect())2978 --FreeIntRegs;2979 else if (NeededSSE && MaxVectorWidth > 0)2980 FI.setMaxVectorWidth(MaxVectorWidth);2981 2982 // The chain argument effectively gives us another free register.2983 if (FI.isChainCall())2984 ++FreeIntRegs;2985 2986 unsigned NumRequiredArgs = FI.getNumRequiredArgs();2987 // AMD64-ABI 3.2.3p3: Once arguments are classified, the registers2988 // get assigned (in left-to-right order) for passing as follows...2989 unsigned ArgNo = 0;2990 for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();2991 it != ie; ++it, ++ArgNo) {2992 bool IsNamedArg = ArgNo < NumRequiredArgs;2993 2994 if (IsRegCall && it->type->isStructureOrClassType())2995 it->info = classifyRegCallStructType(it->type, NeededInt, NeededSSE,2996 MaxVectorWidth);2997 else2998 it->info = classifyArgumentType(it->type, FreeIntRegs, NeededInt,2999 NeededSSE, IsNamedArg);3000 3001 // AMD64-ABI 3.2.3p3: If there are no registers available for any3002 // eightbyte of an argument, the whole argument is passed on the3003 // stack. If registers have already been assigned for some3004 // eightbytes of such an argument, the assignments get reverted.3005 if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {3006 FreeIntRegs -= NeededInt;3007 FreeSSERegs -= NeededSSE;3008 if (MaxVectorWidth > FI.getMaxVectorWidth())3009 FI.setMaxVectorWidth(MaxVectorWidth);3010 } else {3011 it->info = getIndirectResult(it->type, FreeIntRegs);3012 }3013 }3014}3015 3016static Address EmitX86_64VAArgFromMemory(CodeGenFunction &CGF,3017 Address VAListAddr, QualType Ty) {3018 Address overflow_arg_area_p =3019 CGF.Builder.CreateStructGEP(VAListAddr, 2, "overflow_arg_area_p");3020 llvm::Value *overflow_arg_area =3021 CGF.Builder.CreateLoad(overflow_arg_area_p, "overflow_arg_area");3022 3023 // AMD64-ABI 3.5.7p5: Step 7. Align l->overflow_arg_area upwards to a 163024 // byte boundary if alignment needed by type exceeds 8 byte boundary.3025 // It isn't stated explicitly in the standard, but in practice we use3026 // alignment greater than 16 where necessary.3027 CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);3028 if (Align > CharUnits::fromQuantity(8)) {3029 overflow_arg_area = emitRoundPointerUpToAlignment(CGF, overflow_arg_area,3030 Align);3031 }3032 3033 // AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.3034 llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);3035 llvm::Value *Res = overflow_arg_area;3036 3037 // AMD64-ABI 3.5.7p5: Step 9. Set l->overflow_arg_area to:3038 // l->overflow_arg_area + sizeof(type).3039 // AMD64-ABI 3.5.7p5: Step 10. Align l->overflow_arg_area upwards to3040 // an 8 byte boundary.3041 3042 uint64_t SizeInBytes = (CGF.getContext().getTypeSize(Ty) + 7) / 8;3043 llvm::Value *Offset =3044 llvm::ConstantInt::get(CGF.Int32Ty, (SizeInBytes + 7) & ~7);3045 overflow_arg_area = CGF.Builder.CreateGEP(CGF.Int8Ty, overflow_arg_area,3046 Offset, "overflow_arg_area.next");3047 CGF.Builder.CreateStore(overflow_arg_area, overflow_arg_area_p);3048 3049 // AMD64-ABI 3.5.7p5: Step 11. Return the fetched type.3050 return Address(Res, LTy, Align);3051}3052 3053RValue X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,3054 QualType Ty, AggValueSlot Slot) const {3055 // Assume that va_list type is correct; should be pointer to LLVM type:3056 // struct {3057 // i32 gp_offset;3058 // i32 fp_offset;3059 // i8* overflow_arg_area;3060 // i8* reg_save_area;3061 // };3062 unsigned neededInt, neededSSE;3063 3064 Ty = getContext().getCanonicalType(Ty);3065 ABIArgInfo AI = classifyArgumentType(Ty, 0, neededInt, neededSSE,3066 /*isNamedArg*/false);3067 3068 // Empty records are ignored for parameter passing purposes.3069 if (AI.isIgnore())3070 return Slot.asRValue();3071 3072 // AMD64-ABI 3.5.7p5: Step 1. Determine whether type may be passed3073 // in the registers. If not go to step 7.3074 if (!neededInt && !neededSSE)3075 return CGF.EmitLoadOfAnyValue(3076 CGF.MakeAddrLValue(EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty), Ty),3077 Slot);3078 3079 // AMD64-ABI 3.5.7p5: Step 2. Compute num_gp to hold the number of3080 // general purpose registers needed to pass type and num_fp to hold3081 // the number of floating point registers needed.3082 3083 // AMD64-ABI 3.5.7p5: Step 3. Verify whether arguments fit into3084 // registers. In the case: l->gp_offset > 48 - num_gp * 8 or3085 // l->fp_offset > 304 - num_fp * 16 go to step 7.3086 //3087 // NOTE: 304 is a typo, there are (6 * 8 + 8 * 16) = 176 bytes of3088 // register save space).3089 3090 llvm::Value *InRegs = nullptr;3091 Address gp_offset_p = Address::invalid(), fp_offset_p = Address::invalid();3092 llvm::Value *gp_offset = nullptr, *fp_offset = nullptr;3093 if (neededInt) {3094 gp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 0, "gp_offset_p");3095 gp_offset = CGF.Builder.CreateLoad(gp_offset_p, "gp_offset");3096 InRegs = llvm::ConstantInt::get(CGF.Int32Ty, 48 - neededInt * 8);3097 InRegs = CGF.Builder.CreateICmpULE(gp_offset, InRegs, "fits_in_gp");3098 }3099 3100 if (neededSSE) {3101 fp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 1, "fp_offset_p");3102 fp_offset = CGF.Builder.CreateLoad(fp_offset_p, "fp_offset");3103 llvm::Value *FitsInFP =3104 llvm::ConstantInt::get(CGF.Int32Ty, 176 - neededSSE * 16);3105 FitsInFP = CGF.Builder.CreateICmpULE(fp_offset, FitsInFP, "fits_in_fp");3106 InRegs = InRegs ? CGF.Builder.CreateAnd(InRegs, FitsInFP) : FitsInFP;3107 }3108 3109 llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg");3110 llvm::BasicBlock *InMemBlock = CGF.createBasicBlock("vaarg.in_mem");3111 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end");3112 CGF.Builder.CreateCondBr(InRegs, InRegBlock, InMemBlock);3113 3114 // Emit code to load the value if it was passed in registers.3115 3116 CGF.EmitBlock(InRegBlock);3117 3118 // AMD64-ABI 3.5.7p5: Step 4. Fetch type from l->reg_save_area with3119 // an offset of l->gp_offset and/or l->fp_offset. This may require3120 // copying to a temporary location in case the parameter is passed3121 // in different register classes or requires an alignment greater3122 // than 8 for general purpose registers and 16 for XMM registers.3123 //3124 // FIXME: This really results in shameful code when we end up needing to3125 // collect arguments from different places; often what should result in a3126 // simple assembling of a structure from scattered addresses has many more3127 // loads than necessary. Can we clean this up?3128 llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);3129 llvm::Value *RegSaveArea = CGF.Builder.CreateLoad(3130 CGF.Builder.CreateStructGEP(VAListAddr, 3), "reg_save_area");3131 3132 Address RegAddr = Address::invalid();3133 if (neededInt && neededSSE) {3134 // FIXME: Cleanup.3135 assert(AI.isDirect() && "Unexpected ABI info for mixed regs");3136 llvm::StructType *ST = cast<llvm::StructType>(AI.getCoerceToType());3137 Address Tmp = CGF.CreateMemTemp(Ty);3138 Tmp = Tmp.withElementType(ST);3139 assert(ST->getNumElements() == 2 && "Unexpected ABI info for mixed regs");3140 llvm::Type *TyLo = ST->getElementType(0);3141 llvm::Type *TyHi = ST->getElementType(1);3142 assert((TyLo->isFPOrFPVectorTy() ^ TyHi->isFPOrFPVectorTy()) &&3143 "Unexpected ABI info for mixed regs");3144 llvm::Value *GPAddr =3145 CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, gp_offset);3146 llvm::Value *FPAddr =3147 CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, fp_offset);3148 llvm::Value *RegLoAddr = TyLo->isFPOrFPVectorTy() ? FPAddr : GPAddr;3149 llvm::Value *RegHiAddr = TyLo->isFPOrFPVectorTy() ? GPAddr : FPAddr;3150 3151 // Copy the first element.3152 // FIXME: Our choice of alignment here and below is probably pessimistic.3153 llvm::Value *V = CGF.Builder.CreateAlignedLoad(3154 TyLo, RegLoAddr,3155 CharUnits::fromQuantity(getDataLayout().getABITypeAlign(TyLo)));3156 CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0));3157 3158 // Copy the second element.3159 V = CGF.Builder.CreateAlignedLoad(3160 TyHi, RegHiAddr,3161 CharUnits::fromQuantity(getDataLayout().getABITypeAlign(TyHi)));3162 CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1));3163 3164 RegAddr = Tmp.withElementType(LTy);3165 } else if (neededInt || neededSSE == 1) {3166 // Copy to a temporary if necessary to ensure the appropriate alignment.3167 auto TInfo = getContext().getTypeInfoInChars(Ty);3168 uint64_t TySize = TInfo.Width.getQuantity();3169 CharUnits TyAlign = TInfo.Align;3170 llvm::Type *CoTy = nullptr;3171 if (AI.isDirect())3172 CoTy = AI.getCoerceToType();3173 3174 llvm::Value *GpOrFpOffset = neededInt ? gp_offset : fp_offset;3175 uint64_t Alignment = neededInt ? 8 : 16;3176 uint64_t RegSize = neededInt ? neededInt * 8 : 16;3177 // There are two cases require special handling:3178 // 1)3179 // ```3180 // struct {3181 // struct {} a[8];3182 // int b;3183 // };3184 // ```3185 // The lower 8 bytes of the structure are not stored,3186 // so an 8-byte offset is needed when accessing the structure.3187 // 2)3188 // ```3189 // struct {3190 // long long a;3191 // struct {} b;3192 // };3193 // ```3194 // The stored size of this structure is smaller than its actual size,3195 // which may lead to reading past the end of the register save area.3196 if (CoTy && (AI.getDirectOffset() == 8 || RegSize < TySize)) {3197 Address Tmp = CGF.CreateMemTemp(Ty);3198 llvm::Value *Addr =3199 CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, GpOrFpOffset);3200 llvm::Value *Src = CGF.Builder.CreateAlignedLoad(CoTy, Addr, TyAlign);3201 llvm::Value *PtrOffset =3202 llvm::ConstantInt::get(CGF.Int32Ty, AI.getDirectOffset());3203 Address Dst = Address(3204 CGF.Builder.CreateGEP(CGF.Int8Ty, Tmp.getBasePointer(), PtrOffset),3205 LTy, TyAlign);3206 CGF.Builder.CreateStore(Src, Dst);3207 RegAddr = Tmp.withElementType(LTy);3208 } else {3209 RegAddr =3210 Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, GpOrFpOffset),3211 LTy, CharUnits::fromQuantity(Alignment));3212 3213 // Copy into a temporary if the type is more aligned than the3214 // register save area.3215 if (neededInt && TyAlign.getQuantity() > 8) {3216 Address Tmp = CGF.CreateMemTemp(Ty);3217 CGF.Builder.CreateMemCpy(Tmp, RegAddr, TySize, false);3218 RegAddr = Tmp;3219 }3220 }3221 3222 } else {3223 assert(neededSSE == 2 && "Invalid number of needed registers!");3224 // SSE registers are spaced 16 bytes apart in the register save3225 // area, we need to collect the two eightbytes together.3226 // The ABI isn't explicit about this, but it seems reasonable3227 // to assume that the slots are 16-byte aligned, since the stack is3228 // naturally 16-byte aligned and the prologue is expected to store3229 // all the SSE registers to the RSA.3230 Address RegAddrLo = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea,3231 fp_offset),3232 CGF.Int8Ty, CharUnits::fromQuantity(16));3233 Address RegAddrHi =3234 CGF.Builder.CreateConstInBoundsByteGEP(RegAddrLo,3235 CharUnits::fromQuantity(16));3236 llvm::Type *ST = AI.canHaveCoerceToType()3237 ? AI.getCoerceToType()3238 : llvm::StructType::get(CGF.DoubleTy, CGF.DoubleTy);3239 llvm::Value *V;3240 Address Tmp = CGF.CreateMemTemp(Ty);3241 Tmp = Tmp.withElementType(ST);3242 V = CGF.Builder.CreateLoad(3243 RegAddrLo.withElementType(ST->getStructElementType(0)));3244 CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0));3245 V = CGF.Builder.CreateLoad(3246 RegAddrHi.withElementType(ST->getStructElementType(1)));3247 CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1));3248 3249 RegAddr = Tmp.withElementType(LTy);3250 }3251 3252 // AMD64-ABI 3.5.7p5: Step 5. Set:3253 // l->gp_offset = l->gp_offset + num_gp * 83254 // l->fp_offset = l->fp_offset + num_fp * 16.3255 if (neededInt) {3256 llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int32Ty, neededInt * 8);3257 CGF.Builder.CreateStore(CGF.Builder.CreateAdd(gp_offset, Offset),3258 gp_offset_p);3259 }3260 if (neededSSE) {3261 llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int32Ty, neededSSE * 16);3262 CGF.Builder.CreateStore(CGF.Builder.CreateAdd(fp_offset, Offset),3263 fp_offset_p);3264 }3265 CGF.EmitBranch(ContBlock);3266 3267 // Emit code to load the value if it was passed in memory.3268 3269 CGF.EmitBlock(InMemBlock);3270 Address MemAddr = EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);3271 3272 // Return the appropriate result.3273 3274 CGF.EmitBlock(ContBlock);3275 Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock, MemAddr, InMemBlock,3276 "vaarg.addr");3277 return CGF.EmitLoadOfAnyValue(CGF.MakeAddrLValue(ResAddr, Ty), Slot);3278}3279 3280RValue X86_64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,3281 QualType Ty, AggValueSlot Slot) const {3282 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is3283 // not 1, 2, 4, or 8 bytes, must be passed by reference."3284 uint64_t Width = getContext().getTypeSize(Ty);3285 bool IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Width);3286 3287 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,3288 CGF.getContext().getTypeInfoInChars(Ty),3289 CharUnits::fromQuantity(8),3290 /*allowHigherAlign*/ false, Slot);3291}3292 3293ABIArgInfo WinX86_64ABIInfo::reclassifyHvaArgForVectorCall(3294 QualType Ty, unsigned &FreeSSERegs, const ABIArgInfo ¤t) const {3295 const Type *Base = nullptr;3296 uint64_t NumElts = 0;3297 3298 if (!Ty->isBuiltinType() && !Ty->isVectorType() &&3299 isHomogeneousAggregate(Ty, Base, NumElts) && FreeSSERegs >= NumElts) {3300 FreeSSERegs -= NumElts;3301 return getDirectX86Hva();3302 }3303 return current;3304}3305 3306ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,3307 bool IsReturnType, bool IsVectorCall,3308 bool IsRegCall) const {3309 3310 if (Ty->isVoidType())3311 return ABIArgInfo::getIgnore();3312 3313 if (const auto *ED = Ty->getAsEnumDecl())3314 Ty = ED->getIntegerType();3315 3316 TypeInfo Info = getContext().getTypeInfo(Ty);3317 uint64_t Width = Info.Width;3318 CharUnits Align = getContext().toCharUnitsFromBits(Info.Align);3319 3320 const RecordType *RT = Ty->getAsCanonical<RecordType>();3321 if (RT) {3322 if (!IsReturnType) {3323 if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI()))3324 return getNaturalAlignIndirect(Ty, getDataLayout().getAllocaAddrSpace(),3325 RAA == CGCXXABI::RAA_DirectInMemory);3326 }3327 3328 if (RT->getDecl()->getDefinitionOrSelf()->hasFlexibleArrayMember())3329 return getNaturalAlignIndirect(Ty, getDataLayout().getAllocaAddrSpace(),3330 /*ByVal=*/false);3331 }3332 3333 const Type *Base = nullptr;3334 uint64_t NumElts = 0;3335 // vectorcall adds the concept of a homogenous vector aggregate, similar to3336 // other targets.3337 if ((IsVectorCall || IsRegCall) &&3338 isHomogeneousAggregate(Ty, Base, NumElts)) {3339 if (IsRegCall) {3340 if (FreeSSERegs >= NumElts) {3341 FreeSSERegs -= NumElts;3342 if (IsReturnType || Ty->isBuiltinType() || Ty->isVectorType())3343 return ABIArgInfo::getDirect();3344 return ABIArgInfo::getExpand();3345 }3346 return ABIArgInfo::getIndirect(3347 Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),3348 /*ByVal=*/false);3349 } else if (IsVectorCall) {3350 if (FreeSSERegs >= NumElts &&3351 (IsReturnType || Ty->isBuiltinType() || Ty->isVectorType())) {3352 FreeSSERegs -= NumElts;3353 return ABIArgInfo::getDirect();3354 } else if (IsReturnType) {3355 return ABIArgInfo::getExpand();3356 } else if (!Ty->isBuiltinType() && !Ty->isVectorType()) {3357 // HVAs are delayed and reclassified in the 2nd step.3358 return ABIArgInfo::getIndirect(3359 Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),3360 /*ByVal=*/false);3361 }3362 }3363 }3364 3365 if (Ty->isMemberPointerType()) {3366 // If the member pointer is represented by an LLVM int or ptr, pass it3367 // directly.3368 llvm::Type *LLTy = CGT.ConvertType(Ty);3369 if (LLTy->isPointerTy() || LLTy->isIntegerTy())3370 return ABIArgInfo::getDirect();3371 }3372 3373 if (RT || Ty->isAnyComplexType() || Ty->isMemberPointerType()) {3374 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is3375 // not 1, 2, 4, or 8 bytes, must be passed by reference."3376 if (Width > 64 || !llvm::isPowerOf2_64(Width))3377 return getNaturalAlignIndirect(Ty, getDataLayout().getAllocaAddrSpace(),3378 /*ByVal=*/false);3379 3380 // Otherwise, coerce it to a small integer.3381 return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Width));3382 }3383 3384 if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {3385 switch (BT->getKind()) {3386 case BuiltinType::Bool:3387 // Bool type is always extended to the ABI, other builtin types are not3388 // extended.3389 return ABIArgInfo::getExtend(Ty);3390 3391 case BuiltinType::LongDouble:3392 // Mingw64 GCC uses the old 80 bit extended precision floating point3393 // unit. It passes them indirectly through memory.3394 if (IsMingw64) {3395 const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();3396 if (LDF == &llvm::APFloat::x87DoubleExtended())3397 return ABIArgInfo::getIndirect(3398 Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),3399 /*ByVal=*/false);3400 }3401 break;3402 3403 case BuiltinType::Int128:3404 case BuiltinType::UInt128:3405 case BuiltinType::Float128:3406 // 128-bit float and integer types share the same ABI.3407 3408 // If it's a parameter type, the normal ABI rule is that arguments larger3409 // than 8 bytes are passed indirectly. GCC follows it. We follow it too,3410 // even though it isn't particularly efficient.3411 if (!IsReturnType)3412 return ABIArgInfo::getIndirect(3413 Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),3414 /*ByVal=*/false);3415 3416 // Mingw64 GCC returns i128 in XMM0. Coerce to v2i64 to handle that.3417 // Clang matches them for compatibility.3418 // NOTE: GCC actually returns f128 indirectly but will hopefully change.3419 // See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=115054#c8.3420 return ABIArgInfo::getDirect(llvm::FixedVectorType::get(3421 llvm::Type::getInt64Ty(getVMContext()), 2));3422 3423 default:3424 break;3425 }3426 }3427 3428 if (Ty->isBitIntType()) {3429 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is3430 // not 1, 2, 4, or 8 bytes, must be passed by reference."3431 // However, non-power-of-two bit-precise integers will be passed as 1, 2, 4,3432 // or 8 bytes anyway as long is it fits in them, so we don't have to check3433 // the power of 2.3434 if (Width <= 64)3435 return ABIArgInfo::getDirect();3436 return ABIArgInfo::getIndirect(3437 Align, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(),3438 /*ByVal=*/false);3439 }3440 3441 return ABIArgInfo::getDirect();3442}3443 3444void WinX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {3445 const unsigned CC = FI.getCallingConvention();3446 bool IsVectorCall = CC == llvm::CallingConv::X86_VectorCall;3447 bool IsRegCall = CC == llvm::CallingConv::X86_RegCall;3448 3449 // If __attribute__((sysv_abi)) is in use, use the SysV argument3450 // classification rules.3451 if (CC == llvm::CallingConv::X86_64_SysV) {3452 X86_64ABIInfo SysVABIInfo(CGT, AVXLevel);3453 SysVABIInfo.computeInfo(FI);3454 return;3455 }3456 3457 unsigned FreeSSERegs = 0;3458 if (IsVectorCall) {3459 // We can use up to 4 SSE return registers with vectorcall.3460 FreeSSERegs = 4;3461 } else if (IsRegCall) {3462 // RegCall gives us 16 SSE registers.3463 FreeSSERegs = 16;3464 }3465 3466 if (!getCXXABI().classifyReturnType(FI))3467 FI.getReturnInfo() = classify(FI.getReturnType(), FreeSSERegs, true,3468 IsVectorCall, IsRegCall);3469 3470 if (IsVectorCall) {3471 // We can use up to 6 SSE register parameters with vectorcall.3472 FreeSSERegs = 6;3473 } else if (IsRegCall) {3474 // RegCall gives us 16 SSE registers, we can reuse the return registers.3475 FreeSSERegs = 16;3476 }3477 3478 unsigned ArgNum = 0;3479 unsigned ZeroSSERegs = 0;3480 for (auto &I : FI.arguments()) {3481 // Vectorcall in x64 only permits the first 6 arguments to be passed as3482 // XMM/YMM registers. After the sixth argument, pretend no vector3483 // registers are left.3484 unsigned *MaybeFreeSSERegs =3485 (IsVectorCall && ArgNum >= 6) ? &ZeroSSERegs : &FreeSSERegs;3486 I.info =3487 classify(I.type, *MaybeFreeSSERegs, false, IsVectorCall, IsRegCall);3488 ++ArgNum;3489 }3490 3491 if (IsVectorCall) {3492 // For vectorcall, assign aggregate HVAs to any free vector registers in a3493 // second pass.3494 for (auto &I : FI.arguments())3495 I.info = reclassifyHvaArgForVectorCall(I.type, FreeSSERegs, I.info);3496 }3497}3498 3499RValue WinX86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,3500 QualType Ty, AggValueSlot Slot) const {3501 // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is3502 // not 1, 2, 4, or 8 bytes, must be passed by reference."3503 uint64_t Width = getContext().getTypeSize(Ty);3504 bool IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Width);3505 3506 return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,3507 CGF.getContext().getTypeInfoInChars(Ty),3508 CharUnits::fromQuantity(8),3509 /*allowHigherAlign*/ false, Slot);3510}3511 3512std::unique_ptr<TargetCodeGenInfo> CodeGen::createX86_32TargetCodeGenInfo(3513 CodeGenModule &CGM, bool DarwinVectorABI, bool Win32StructABI,3514 unsigned NumRegisterParameters, bool SoftFloatABI) {3515 bool RetSmallStructInRegABI = X86_32TargetCodeGenInfo::isStructReturnInRegABI(3516 CGM.getTriple(), CGM.getCodeGenOpts());3517 return std::make_unique<X86_32TargetCodeGenInfo>(3518 CGM.getTypes(), DarwinVectorABI, RetSmallStructInRegABI, Win32StructABI,3519 NumRegisterParameters, SoftFloatABI);3520}3521 3522std::unique_ptr<TargetCodeGenInfo> CodeGen::createWinX86_32TargetCodeGenInfo(3523 CodeGenModule &CGM, bool DarwinVectorABI, bool Win32StructABI,3524 unsigned NumRegisterParameters) {3525 bool RetSmallStructInRegABI = X86_32TargetCodeGenInfo::isStructReturnInRegABI(3526 CGM.getTriple(), CGM.getCodeGenOpts());3527 return std::make_unique<WinX86_32TargetCodeGenInfo>(3528 CGM.getTypes(), DarwinVectorABI, RetSmallStructInRegABI, Win32StructABI,3529 NumRegisterParameters);3530}3531 3532std::unique_ptr<TargetCodeGenInfo>3533CodeGen::createX86_64TargetCodeGenInfo(CodeGenModule &CGM,3534 X86AVXABILevel AVXLevel) {3535 return std::make_unique<X86_64TargetCodeGenInfo>(CGM.getTypes(), AVXLevel);3536}3537 3538std::unique_ptr<TargetCodeGenInfo>3539CodeGen::createWinX86_64TargetCodeGenInfo(CodeGenModule &CGM,3540 X86AVXABILevel AVXLevel) {3541 return std::make_unique<WinX86_64TargetCodeGenInfo>(CGM.getTypes(), AVXLevel);3542}3543