Sign in Sign up
brintos

brintos / llvm-project-archived public Read only

0
0
Fork 0
Text · 119.9 KiB · a61bbe5 Raw
3026 lines · cpp
1//===- llvm/lib/Target/X86/X86ISelCallLowering.cpp - Call lowering --------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9/// \file10/// This file implements the lowering of LLVM calls to DAG nodes.11//12//===----------------------------------------------------------------------===//13 14#include "MCTargetDesc/X86MCAsmInfo.h"15#include "X86.h"16#include "X86CallingConv.h"17#include "X86FrameLowering.h"18#include "X86ISelLowering.h"19#include "X86InstrBuilder.h"20#include "X86MachineFunctionInfo.h"21#include "X86TargetMachine.h"22#include "llvm/ADT/Statistic.h"23#include "llvm/Analysis/ObjCARCUtil.h"24#include "llvm/CodeGen/MachineJumpTableInfo.h"25#include "llvm/CodeGen/MachineModuleInfo.h"26#include "llvm/CodeGen/WinEHFuncInfo.h"27#include "llvm/IR/DiagnosticInfo.h"28#include "llvm/IR/IRBuilder.h"29#include "llvm/IR/Module.h"30 31#define DEBUG_TYPE "x86-isel"32 33using namespace llvm;34 35STATISTIC(NumTailCalls, "Number of tail calls");36 37/// Call this when the user attempts to do something unsupported, like38/// returning a double without SSE2 enabled on x86_64. This is not fatal, unlike39/// report_fatal_error, so calling code should attempt to recover without40/// crashing.41static void errorUnsupported(SelectionDAG &DAG, const SDLoc &dl,42                             const char *Msg) {43  MachineFunction &MF = DAG.getMachineFunction();44  DAG.getContext()->diagnose(45      DiagnosticInfoUnsupported(MF.getFunction(), Msg, dl.getDebugLoc()));46}47 48/// Returns true if a CC can dynamically exclude a register from the list of49/// callee-saved-registers (TargetRegistryInfo::getCalleeSavedRegs()) based on50/// the return registers.51static bool shouldDisableRetRegFromCSR(CallingConv::ID CC) {52  switch (CC) {53  default:54    return false;55  case CallingConv::X86_RegCall:56  case CallingConv::PreserveMost:57  case CallingConv::PreserveAll:58    return true;59  }60}61 62/// Returns true if a CC can dynamically exclude a register from the list of63/// callee-saved-registers (TargetRegistryInfo::getCalleeSavedRegs()) based on64/// the parameters.65static bool shouldDisableArgRegFromCSR(CallingConv::ID CC) {66  return CC == CallingConv::X86_RegCall;67}68 69static std::pair<MVT, unsigned>70handleMaskRegisterForCallingConv(unsigned NumElts, CallingConv::ID CC,71                                 const X86Subtarget &Subtarget) {72  // v2i1/v4i1/v8i1/v16i1 all pass in xmm registers unless the calling73  // convention is one that uses k registers.74  if (NumElts == 2)75    return {MVT::v2i64, 1};76  if (NumElts == 4)77    return {MVT::v4i32, 1};78  if (NumElts == 8 && CC != CallingConv::X86_RegCall &&79      CC != CallingConv::Intel_OCL_BI)80    return {MVT::v8i16, 1};81  if (NumElts == 16 && CC != CallingConv::X86_RegCall &&82      CC != CallingConv::Intel_OCL_BI)83    return {MVT::v16i8, 1};84  // v32i1 passes in ymm unless we have BWI and the calling convention is85  // regcall.86  if (NumElts == 32 && (!Subtarget.hasBWI() || CC != CallingConv::X86_RegCall))87    return {MVT::v32i8, 1};88  // Split v64i1 vectors if we don't have v64i8 available.89  if (NumElts == 64 && Subtarget.hasBWI() && CC != CallingConv::X86_RegCall) {90    if (Subtarget.useAVX512Regs())91      return {MVT::v64i8, 1};92    return {MVT::v32i8, 2};93  }94 95  // Break wide or odd vXi1 vectors into scalars to match avx2 behavior.96  if (!isPowerOf2_32(NumElts) || (NumElts == 64 && !Subtarget.hasBWI()) ||97      NumElts > 64)98    return {MVT::i8, NumElts};99 100  return {MVT::INVALID_SIMPLE_VALUE_TYPE, 0};101}102 103MVT X86TargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,104                                                     CallingConv::ID CC,105                                                     EVT VT) const {106  if (VT.isVector()) {107    if (VT.getVectorElementType() == MVT::i1 && Subtarget.hasAVX512()) {108      unsigned NumElts = VT.getVectorNumElements();109 110      MVT RegisterVT;111      unsigned NumRegisters;112      std::tie(RegisterVT, NumRegisters) =113          handleMaskRegisterForCallingConv(NumElts, CC, Subtarget);114      if (RegisterVT != MVT::INVALID_SIMPLE_VALUE_TYPE)115        return RegisterVT;116    }117 118    if (VT.getVectorElementType() == MVT::f16 && VT.getVectorNumElements() < 8)119      return MVT::v8f16;120  }121 122  // We will use more GPRs for f64 and f80 on 32 bits when x87 is disabled.123  if ((VT == MVT::f64 || VT == MVT::f80) && !Subtarget.is64Bit() &&124      !Subtarget.hasX87())125    return MVT::i32;126 127  if (isTypeLegal(MVT::f16)) {128    if (VT.isVector() && VT.getVectorElementType() == MVT::bf16)129      return getRegisterTypeForCallingConv(130          Context, CC, VT.changeVectorElementType(MVT::f16));131 132    if (VT == MVT::bf16)133      return MVT::f16;134  }135 136  return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);137}138 139unsigned X86TargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,140                                                          CallingConv::ID CC,141                                                          EVT VT) const {142  if (VT.isVector()) {143    if (VT.getVectorElementType() == MVT::i1 && Subtarget.hasAVX512()) {144      unsigned NumElts = VT.getVectorNumElements();145 146      MVT RegisterVT;147      unsigned NumRegisters;148      std::tie(RegisterVT, NumRegisters) =149          handleMaskRegisterForCallingConv(NumElts, CC, Subtarget);150      if (RegisterVT != MVT::INVALID_SIMPLE_VALUE_TYPE)151        return NumRegisters;152    }153 154    if (VT.getVectorElementType() == MVT::f16 && VT.getVectorNumElements() < 8)155      return 1;156  }157 158  // We have to split f64 to 2 registers and f80 to 3 registers on 32 bits if159  // x87 is disabled.160  if (!Subtarget.is64Bit() && !Subtarget.hasX87()) {161    if (VT == MVT::f64)162      return 2;163    if (VT == MVT::f80)164      return 3;165  }166 167  if (VT.isVector() && VT.getVectorElementType() == MVT::bf16 &&168      isTypeLegal(MVT::f16))169    return getNumRegistersForCallingConv(Context, CC,170                                         VT.changeVectorElementType(MVT::f16));171 172  return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);173}174 175unsigned X86TargetLowering::getVectorTypeBreakdownForCallingConv(176    LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,177    unsigned &NumIntermediates, MVT &RegisterVT) const {178  // Break wide or odd vXi1 vectors into scalars to match avx2 behavior.179  if (VT.isVector() && VT.getVectorElementType() == MVT::i1 &&180      Subtarget.hasAVX512() &&181      (!isPowerOf2_32(VT.getVectorNumElements()) ||182       (VT.getVectorNumElements() == 64 && !Subtarget.hasBWI()) ||183       VT.getVectorNumElements() > 64)) {184    RegisterVT = MVT::i8;185    IntermediateVT = MVT::i1;186    NumIntermediates = VT.getVectorNumElements();187    return NumIntermediates;188  }189 190  // Split v64i1 vectors if we don't have v64i8 available.191  if (VT == MVT::v64i1 && Subtarget.hasBWI() && !Subtarget.useAVX512Regs() &&192      CC != CallingConv::X86_RegCall) {193    RegisterVT = MVT::v32i8;194    IntermediateVT = MVT::v32i1;195    NumIntermediates = 2;196    return 2;197  }198 199  // Split vNbf16 vectors according to vNf16.200  if (VT.isVector() && VT.getVectorElementType() == MVT::bf16 &&201      isTypeLegal(MVT::f16))202    VT = VT.changeVectorElementType(MVT::f16);203 204  return TargetLowering::getVectorTypeBreakdownForCallingConv(Context, CC, VT, IntermediateVT,205                                              NumIntermediates, RegisterVT);206}207 208EVT X86TargetLowering::getSetCCResultType(const DataLayout &DL,209                                          LLVMContext& Context,210                                          EVT VT) const {211  if (!VT.isVector())212    return MVT::i8;213 214  if (Subtarget.hasAVX512()) {215    // Figure out what this type will be legalized to.216    EVT LegalVT = VT;217    while (getTypeAction(Context, LegalVT) != TypeLegal)218      LegalVT = getTypeToTransformTo(Context, LegalVT);219 220    // If we got a 512-bit vector then we'll definitely have a vXi1 compare.221    if (LegalVT.getSimpleVT().is512BitVector())222      return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount());223 224    if (LegalVT.getSimpleVT().isVector() && Subtarget.hasVLX()) {225      // If we legalized to less than a 512-bit vector, then we will use a vXi1226      // compare for vXi32/vXi64 for sure. If we have BWI we will also support227      // vXi16/vXi8.228      MVT EltVT = LegalVT.getSimpleVT().getVectorElementType();229      if (Subtarget.hasBWI() || EltVT.getSizeInBits() >= 32)230        return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount());231    }232  }233 234  return VT.changeVectorElementTypeToInteger();235}236 237bool X86TargetLowering::functionArgumentNeedsConsecutiveRegisters(238    Type *Ty, CallingConv::ID CallConv, bool isVarArg,239    const DataLayout &DL) const {240  // On x86-64 i128 is split into two i64s and needs to be allocated to two241  // consecutive registers, or spilled to the stack as a whole. On x86-32 i128242  // is split to four i32s and never actually passed in registers, but we use243  // the consecutive register mark to match it in TableGen.244  if (Ty->isIntegerTy(128))245    return true;246 247  // On x86-32, fp128 acts the same as i128.248  if (Subtarget.is32Bit() && Ty->isFP128Ty())249    return true;250 251  return false;252}253 254/// Helper for getByValTypeAlignment to determine255/// the desired ByVal argument alignment.256static void getMaxByValAlign(Type *Ty, Align &MaxAlign) {257  if (MaxAlign == 16)258    return;259  if (VectorType *VTy = dyn_cast<VectorType>(Ty)) {260    if (VTy->getPrimitiveSizeInBits().getFixedValue() == 128)261      MaxAlign = Align(16);262  } else if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {263    Align EltAlign;264    getMaxByValAlign(ATy->getElementType(), EltAlign);265    if (EltAlign > MaxAlign)266      MaxAlign = EltAlign;267  } else if (StructType *STy = dyn_cast<StructType>(Ty)) {268    for (auto *EltTy : STy->elements()) {269      Align EltAlign;270      getMaxByValAlign(EltTy, EltAlign);271      if (EltAlign > MaxAlign)272        MaxAlign = EltAlign;273      if (MaxAlign == 16)274        break;275    }276  }277}278 279/// Return the desired alignment for ByVal aggregate280/// function arguments in the caller parameter area. For X86, aggregates281/// that contain SSE vectors are placed at 16-byte boundaries while the rest282/// are at 4-byte boundaries.283Align X86TargetLowering::getByValTypeAlignment(Type *Ty,284                                               const DataLayout &DL) const {285  if (Subtarget.is64Bit())286    return std::max(DL.getABITypeAlign(Ty), Align::Constant<8>());287 288  Align Alignment(4);289  if (Subtarget.hasSSE1())290    getMaxByValAlign(Ty, Alignment);291  return Alignment;292}293 294/// It returns EVT::Other if the type should be determined using generic295/// target-independent logic.296/// For vector ops we check that the overall size isn't larger than our297/// preferred vector width.298EVT X86TargetLowering::getOptimalMemOpType(299    LLVMContext &Context, const MemOp &Op,300    const AttributeList &FuncAttributes) const {301  if (!FuncAttributes.hasFnAttr(Attribute::NoImplicitFloat)) {302    if (Op.size() >= 16 &&303        (!Subtarget.isUnalignedMem16Slow() || Op.isAligned(Align(16)))) {304      // FIXME: Check if unaligned 64-byte accesses are slow.305      if (Op.size() >= 64 && Subtarget.hasAVX512() &&306          (Subtarget.getPreferVectorWidth() >= 512)) {307        return Subtarget.hasBWI() ? MVT::v64i8 : MVT::v16i32;308      }309      // FIXME: Check if unaligned 32-byte accesses are slow.310      if (Op.size() >= 32 && Subtarget.hasAVX() &&311          Subtarget.useLight256BitInstructions()) {312        // Although this isn't a well-supported type for AVX1, we'll let313        // legalization and shuffle lowering produce the optimal codegen. If we314        // choose an optimal type with a vector element larger than a byte,315        // getMemsetStores() may create an intermediate splat (using an integer316        // multiply) before we splat as a vector.317        return MVT::v32i8;318      }319      if (Subtarget.hasSSE2() && (Subtarget.getPreferVectorWidth() >= 128))320        return MVT::v16i8;321      // TODO: Can SSE1 handle a byte vector?322      // If we have SSE1 registers we should be able to use them.323      if (Subtarget.hasSSE1() && (Subtarget.is64Bit() || Subtarget.hasX87()) &&324          (Subtarget.getPreferVectorWidth() >= 128))325        return MVT::v4f32;326    } else if (((Op.isMemcpy() && !Op.isMemcpyStrSrc()) || Op.isZeroMemset()) &&327               Op.size() >= 8 && !Subtarget.is64Bit() && Subtarget.hasSSE2()) {328      // Do not use f64 to lower memcpy if source is string constant. It's329      // better to use i32 to avoid the loads.330      // Also, do not use f64 to lower memset unless this is a memset of zeros.331      // The gymnastics of splatting a byte value into an XMM register and then332      // only using 8-byte stores (because this is a CPU with slow unaligned333      // 16-byte accesses) makes that a loser.334      return MVT::f64;335    }336  }337  // This is a compromise. If we reach here, unaligned accesses may be slow on338  // this target. However, creating smaller, aligned accesses could be even339  // slower and would certainly be a lot more code.340  if (Subtarget.is64Bit() && Op.size() >= 8)341    return MVT::i64;342  return MVT::i32;343}344 345bool X86TargetLowering::isSafeMemOpType(MVT VT) const {346  if (VT == MVT::f32)347    return Subtarget.hasSSE1();348  if (VT == MVT::f64)349    return Subtarget.hasSSE2();350  return true;351}352 353static bool isBitAligned(Align Alignment, uint64_t SizeInBits) {354  return (8 * Alignment.value()) % SizeInBits == 0;355}356 357bool X86TargetLowering::isMemoryAccessFast(EVT VT, Align Alignment) const {358  if (isBitAligned(Alignment, VT.getSizeInBits()))359    return true;360  switch (VT.getSizeInBits()) {361  default:362    // 8-byte and under are always assumed to be fast.363    return true;364  case 128:365    return !Subtarget.isUnalignedMem16Slow();366  case 256:367    return !Subtarget.isUnalignedMem32Slow();368    // TODO: What about AVX-512 (512-bit) accesses?369  }370}371 372bool X86TargetLowering::allowsMisalignedMemoryAccesses(373    EVT VT, unsigned, Align Alignment, MachineMemOperand::Flags Flags,374    unsigned *Fast) const {375  if (Fast)376    *Fast = isMemoryAccessFast(VT, Alignment);377  // NonTemporal vector memory ops must be aligned.378  if (!!(Flags & MachineMemOperand::MONonTemporal) && VT.isVector()) {379    // NT loads can only be vector aligned, so if its less aligned than the380    // minimum vector size (which we can split the vector down to), we might as381    // well use a regular unaligned vector load.382    // We don't have any NT loads pre-SSE41.383    if (!!(Flags & MachineMemOperand::MOLoad))384      return (Alignment < 16 || !Subtarget.hasSSE41());385    return false;386  }387  // Misaligned accesses of any size are always allowed.388  return true;389}390 391bool X86TargetLowering::allowsMemoryAccess(LLVMContext &Context,392                                           const DataLayout &DL, EVT VT,393                                           unsigned AddrSpace, Align Alignment,394                                           MachineMemOperand::Flags Flags,395                                           unsigned *Fast) const {396  if (Fast)397    *Fast = isMemoryAccessFast(VT, Alignment);398  if (!!(Flags & MachineMemOperand::MONonTemporal) && VT.isVector()) {399    if (allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment, Flags,400                                       /*Fast=*/nullptr))401      return true;402    // NonTemporal vector memory ops are special, and must be aligned.403    if (!isBitAligned(Alignment, VT.getSizeInBits()))404      return false;405    switch (VT.getSizeInBits()) {406    case 128:407      if (!!(Flags & MachineMemOperand::MOLoad) && Subtarget.hasSSE41())408        return true;409      if (!!(Flags & MachineMemOperand::MOStore) && Subtarget.hasSSE2())410        return true;411      return false;412    case 256:413      if (!!(Flags & MachineMemOperand::MOLoad) && Subtarget.hasAVX2())414        return true;415      if (!!(Flags & MachineMemOperand::MOStore) && Subtarget.hasAVX())416        return true;417      return false;418    case 512:419      if (Subtarget.hasAVX512())420        return true;421      return false;422    default:423      return false; // Don't have NonTemporal vector memory ops of this size.424    }425  }426  return true;427}428 429/// Return the entry encoding for a jump table in the430/// current function.  The returned value is a member of the431/// MachineJumpTableInfo::JTEntryKind enum.432unsigned X86TargetLowering::getJumpTableEncoding() const {433  // In GOT pic mode, each entry in the jump table is emitted as a @GOTOFF434  // symbol.435  if (isPositionIndependent() && Subtarget.isPICStyleGOT())436    return MachineJumpTableInfo::EK_Custom32;437  if (isPositionIndependent() &&438      getTargetMachine().getCodeModel() == CodeModel::Large &&439      !Subtarget.isTargetCOFF())440    return MachineJumpTableInfo::EK_LabelDifference64;441 442  // Otherwise, use the normal jump table encoding heuristics.443  return TargetLowering::getJumpTableEncoding();444}445 446bool X86TargetLowering::useSoftFloat() const {447  return Subtarget.useSoftFloat();448}449 450void X86TargetLowering::markLibCallAttributes(MachineFunction *MF, unsigned CC,451                                              ArgListTy &Args) const {452 453  // Only relabel X86-32 for C / Stdcall CCs.454  if (Subtarget.is64Bit())455    return;456  if (CC != CallingConv::C && CC != CallingConv::X86_StdCall)457    return;458  unsigned ParamRegs = 0;459  if (auto *M = MF->getFunction().getParent())460    ParamRegs = M->getNumberRegisterParameters();461 462  // Mark the first N int arguments as having reg463  for (auto &Arg : Args) {464    Type *T = Arg.Ty;465    if (T->isIntOrPtrTy())466      if (MF->getDataLayout().getTypeAllocSize(T) <= 8) {467        unsigned numRegs = 1;468        if (MF->getDataLayout().getTypeAllocSize(T) > 4)469          numRegs = 2;470        if (ParamRegs < numRegs)471          return;472        ParamRegs -= numRegs;473        Arg.IsInReg = true;474      }475  }476}477 478const MCExpr *479X86TargetLowering::LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI,480                                             const MachineBasicBlock *MBB,481                                             unsigned uid,MCContext &Ctx) const{482  assert(isPositionIndependent() && Subtarget.isPICStyleGOT());483  // In 32-bit ELF systems, our jump table entries are formed with @GOTOFF484  // entries.485  return MCSymbolRefExpr::create(MBB->getSymbol(), X86::S_GOTOFF, Ctx);486}487 488/// Returns relocation base for the given PIC jumptable.489SDValue X86TargetLowering::getPICJumpTableRelocBase(SDValue Table,490                                                    SelectionDAG &DAG) const {491  if (!Subtarget.is64Bit())492    // This doesn't have SDLoc associated with it, but is not really the493    // same as a Register.494    return DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(),495                       getPointerTy(DAG.getDataLayout()));496  return Table;497}498 499/// This returns the relocation base for the given PIC jumptable,500/// the same as getPICJumpTableRelocBase, but as an MCExpr.501const MCExpr *X86TargetLowering::502getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI,503                             MCContext &Ctx) const {504  // X86-64 uses RIP relative addressing based on the jump table label.505  if (Subtarget.isPICStyleRIPRel() ||506      (Subtarget.is64Bit() &&507       getTargetMachine().getCodeModel() == CodeModel::Large))508    return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx);509 510  // Otherwise, the reference is relative to the PIC base.511  return MCSymbolRefExpr::create(MF->getPICBaseSymbol(), Ctx);512}513 514std::pair<const TargetRegisterClass *, uint8_t>515X86TargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,516                                           MVT VT) const {517  const TargetRegisterClass *RRC = nullptr;518  uint8_t Cost = 1;519  switch (VT.SimpleTy) {520  default:521    return TargetLowering::findRepresentativeClass(TRI, VT);522  case MVT::i8: case MVT::i16: case MVT::i32: case MVT::i64:523    RRC = Subtarget.is64Bit() ? &X86::GR64RegClass : &X86::GR32RegClass;524    break;525  case MVT::x86mmx:526    RRC = &X86::VR64RegClass;527    break;528  case MVT::f32: case MVT::f64:529  case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64:530  case MVT::v4f32: case MVT::v2f64:531  case MVT::v32i8: case MVT::v16i16: case MVT::v8i32: case MVT::v4i64:532  case MVT::v8f32: case MVT::v4f64:533  case MVT::v64i8: case MVT::v32i16: case MVT::v16i32: case MVT::v8i64:534  case MVT::v16f32: case MVT::v8f64:535    RRC = &X86::VR128XRegClass;536    break;537  }538  return std::make_pair(RRC, Cost);539}540 541unsigned X86TargetLowering::getAddressSpace() const {542  if (Subtarget.is64Bit())543    return (getTargetMachine().getCodeModel() == CodeModel::Kernel) ? X86AS::GS544                                                                    : X86AS::FS;545  return X86AS::GS;546}547 548static bool hasStackGuardSlotTLS(const Triple &TargetTriple) {549  return TargetTriple.isOSGlibc() || TargetTriple.isOSFuchsia() ||550         TargetTriple.isAndroid();551}552 553static Constant* SegmentOffset(IRBuilderBase &IRB,554                               int Offset, unsigned AddressSpace) {555  return ConstantExpr::getIntToPtr(556      ConstantInt::get(Type::getInt32Ty(IRB.getContext()), Offset),557      IRB.getPtrTy(AddressSpace));558}559 560Value *X86TargetLowering::getIRStackGuard(IRBuilderBase &IRB) const {561  // glibc, bionic, and Fuchsia have a special slot for the stack guard in562  // tcbhead_t; use it instead of the usual global variable (see563  // sysdeps/{i386,x86_64}/nptl/tls.h)564  if (hasStackGuardSlotTLS(Subtarget.getTargetTriple())) {565    unsigned AddressSpace = getAddressSpace();566 567    // <zircon/tls.h> defines ZX_TLS_STACK_GUARD_OFFSET with this value.568    if (Subtarget.isTargetFuchsia())569      return SegmentOffset(IRB, 0x10, AddressSpace);570 571    Module *M = IRB.GetInsertBlock()->getParent()->getParent();572    // Specially, some users may customize the base reg and offset.573    int Offset = M->getStackProtectorGuardOffset();574    // If we don't set -stack-protector-guard-offset value:575    // %fs:0x28, unless we're using a Kernel code model, in which case576    // it's %gs:0x28.  gs:0x14 on i386.577    if (Offset == INT_MAX)578      Offset = (Subtarget.is64Bit()) ? 0x28 : 0x14;579 580    StringRef GuardReg = M->getStackProtectorGuardReg();581    if (GuardReg == "fs")582      AddressSpace = X86AS::FS;583    else if (GuardReg == "gs")584      AddressSpace = X86AS::GS;585 586    // Use symbol guard if user specify.587    StringRef GuardSymb = M->getStackProtectorGuardSymbol();588    if (!GuardSymb.empty()) {589      GlobalVariable *GV = M->getGlobalVariable(GuardSymb);590      if (!GV) {591        Type *Ty = Subtarget.is64Bit() ? Type::getInt64Ty(M->getContext())592                                       : Type::getInt32Ty(M->getContext());593        GV = new GlobalVariable(*M, Ty, false, GlobalValue::ExternalLinkage,594                                nullptr, GuardSymb, nullptr,595                                GlobalValue::NotThreadLocal, AddressSpace);596        if (!Subtarget.isTargetDarwin())597          GV->setDSOLocal(M->getDirectAccessExternalData());598      }599      return GV;600    }601 602    return SegmentOffset(IRB, Offset, AddressSpace);603  }604  return TargetLowering::getIRStackGuard(IRB);605}606 607void X86TargetLowering::insertSSPDeclarations(Module &M) const {608  // MSVC CRT provides functionalities for stack protection.609  RTLIB::LibcallImpl SecurityCheckCookieLibcall =610      getLibcallImpl(RTLIB::SECURITY_CHECK_COOKIE);611 612  RTLIB::LibcallImpl SecurityCookieVar =613      getLibcallImpl(RTLIB::STACK_CHECK_GUARD);614  if (SecurityCheckCookieLibcall != RTLIB::Unsupported &&615      SecurityCookieVar != RTLIB::Unsupported) {616    // MSVC CRT provides functionalities for stack protection.617    // MSVC CRT has a global variable holding security cookie.618    M.getOrInsertGlobal(getLibcallImplName(SecurityCookieVar),619                        PointerType::getUnqual(M.getContext()));620 621    // MSVC CRT has a function to validate security cookie.622    FunctionCallee SecurityCheckCookie =623        M.getOrInsertFunction(getLibcallImplName(SecurityCheckCookieLibcall),624                              Type::getVoidTy(M.getContext()),625                              PointerType::getUnqual(M.getContext()));626 627    if (Function *F = dyn_cast<Function>(SecurityCheckCookie.getCallee())) {628      F->setCallingConv(CallingConv::X86_FastCall);629      F->addParamAttr(0, Attribute::AttrKind::InReg);630    }631    return;632  }633 634  StringRef GuardMode = M.getStackProtectorGuard();635 636  // glibc, bionic, and Fuchsia have a special slot for the stack guard.637  if ((GuardMode == "tls" || GuardMode.empty()) &&638      hasStackGuardSlotTLS(Subtarget.getTargetTriple()))639    return;640  TargetLowering::insertSSPDeclarations(M);641}642 643Value *644X86TargetLowering::getSafeStackPointerLocation(IRBuilderBase &IRB) const {645  // Android provides a fixed TLS slot for the SafeStack pointer. See the646  // definition of TLS_SLOT_SAFESTACK in647  // https://android.googlesource.com/platform/bionic/+/master/libc/private/bionic_tls.h648  if (Subtarget.isTargetAndroid()) {649    // %fs:0x48, unless we're using a Kernel code model, in which case it's %gs:650    // %gs:0x24 on i386651    int Offset = (Subtarget.is64Bit()) ? 0x48 : 0x24;652    return SegmentOffset(IRB, Offset, getAddressSpace());653  }654 655  // Fuchsia is similar.656  if (Subtarget.isTargetFuchsia()) {657    // <zircon/tls.h> defines ZX_TLS_UNSAFE_SP_OFFSET with this value.658    return SegmentOffset(IRB, 0x18, getAddressSpace());659  }660 661  return TargetLowering::getSafeStackPointerLocation(IRB);662}663 664//===----------------------------------------------------------------------===//665//               Return Value Calling Convention Implementation666//===----------------------------------------------------------------------===//667 668bool X86TargetLowering::CanLowerReturn(669    CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg,670    const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,671    const Type *RetTy) const {672  SmallVector<CCValAssign, 16> RVLocs;673  CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);674  return CCInfo.CheckReturn(Outs, RetCC_X86);675}676 677const MCPhysReg *X86TargetLowering::getScratchRegisters(CallingConv::ID) const {678  static const MCPhysReg ScratchRegs[] = { X86::R11, 0 };679  return ScratchRegs;680}681 682ArrayRef<MCPhysReg> X86TargetLowering::getRoundingControlRegisters() const {683  static const MCPhysReg RCRegs[] = {X86::FPCW, X86::MXCSR};684  return RCRegs;685}686 687/// Lowers masks values (v*i1) to the local register values688/// \returns DAG node after lowering to register type689static SDValue lowerMasksToReg(const SDValue &ValArg, const EVT &ValLoc,690                               const SDLoc &DL, SelectionDAG &DAG) {691  EVT ValVT = ValArg.getValueType();692 693  if (ValVT == MVT::v1i1)694    return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ValLoc, ValArg,695                       DAG.getIntPtrConstant(0, DL));696 697  if ((ValVT == MVT::v8i1 && (ValLoc == MVT::i8 || ValLoc == MVT::i32)) ||698      (ValVT == MVT::v16i1 && (ValLoc == MVT::i16 || ValLoc == MVT::i32))) {699    // Two stage lowering might be required700    // bitcast:   v8i1 -> i8 / v16i1 -> i16701    // anyextend: i8   -> i32 / i16   -> i32702    EVT TempValLoc = ValVT == MVT::v8i1 ? MVT::i8 : MVT::i16;703    SDValue ValToCopy = DAG.getBitcast(TempValLoc, ValArg);704    if (ValLoc == MVT::i32)705      ValToCopy = DAG.getNode(ISD::ANY_EXTEND, DL, ValLoc, ValToCopy);706    return ValToCopy;707  }708 709  if ((ValVT == MVT::v32i1 && ValLoc == MVT::i32) ||710      (ValVT == MVT::v64i1 && ValLoc == MVT::i64)) {711    // One stage lowering is required712    // bitcast:   v32i1 -> i32 / v64i1 -> i64713    return DAG.getBitcast(ValLoc, ValArg);714  }715 716  return DAG.getNode(ISD::ANY_EXTEND, DL, ValLoc, ValArg);717}718 719/// Breaks v64i1 value into two registers and adds the new node to the DAG720static void Passv64i1ArgInRegs(721    const SDLoc &DL, SelectionDAG &DAG, SDValue &Arg,722    SmallVectorImpl<std::pair<Register, SDValue>> &RegsToPass, CCValAssign &VA,723    CCValAssign &NextVA, const X86Subtarget &Subtarget) {724  assert(Subtarget.hasBWI() && "Expected AVX512BW target!");725  assert(Subtarget.is32Bit() && "Expecting 32 bit target");726  assert(Arg.getValueType() == MVT::i64 && "Expecting 64 bit value");727  assert(VA.isRegLoc() && NextVA.isRegLoc() &&728         "The value should reside in two registers");729 730  // Before splitting the value we cast it to i64731  Arg = DAG.getBitcast(MVT::i64, Arg);732 733  // Splitting the value into two i32 types734  SDValue Lo, Hi;735  std::tie(Lo, Hi) = DAG.SplitScalar(Arg, DL, MVT::i32, MVT::i32);736 737  // Attach the two i32 types into corresponding registers738  RegsToPass.push_back(std::make_pair(VA.getLocReg(), Lo));739  RegsToPass.push_back(std::make_pair(NextVA.getLocReg(), Hi));740}741 742SDValue743X86TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,744                               bool isVarArg,745                               const SmallVectorImpl<ISD::OutputArg> &Outs,746                               const SmallVectorImpl<SDValue> &OutVals,747                               const SDLoc &dl, SelectionDAG &DAG) const {748  MachineFunction &MF = DAG.getMachineFunction();749  X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();750 751  // In some cases we need to disable registers from the default CSR list.752  // For example, when they are used as return registers (preserve_* and X86's753  // regcall) or for argument passing (X86's regcall).754  bool ShouldDisableCalleeSavedRegister =755      shouldDisableRetRegFromCSR(CallConv) ||756      MF.getFunction().hasFnAttribute("no_caller_saved_registers");757 758  if (CallConv == CallingConv::X86_INTR && !Outs.empty())759    report_fatal_error("X86 interrupts may not return any value");760 761  SmallVector<CCValAssign, 16> RVLocs;762  CCState CCInfo(CallConv, isVarArg, MF, RVLocs, *DAG.getContext());763  CCInfo.AnalyzeReturn(Outs, RetCC_X86);764 765  SmallVector<std::pair<Register, SDValue>, 4> RetVals;766  for (unsigned I = 0, OutsIndex = 0, E = RVLocs.size(); I != E;767       ++I, ++OutsIndex) {768    CCValAssign &VA = RVLocs[I];769    assert(VA.isRegLoc() && "Can only return in registers!");770 771    // Add the register to the CalleeSaveDisableRegs list.772    if (ShouldDisableCalleeSavedRegister)773      MF.getRegInfo().disableCalleeSavedRegister(VA.getLocReg());774 775    SDValue ValToCopy = OutVals[OutsIndex];776    EVT ValVT = ValToCopy.getValueType();777 778    // Promote values to the appropriate types.779    if (VA.getLocInfo() == CCValAssign::SExt)780      ValToCopy = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), ValToCopy);781    else if (VA.getLocInfo() == CCValAssign::ZExt)782      ValToCopy = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), ValToCopy);783    else if (VA.getLocInfo() == CCValAssign::AExt) {784      if (ValVT.isVector() && ValVT.getVectorElementType() == MVT::i1)785        ValToCopy = lowerMasksToReg(ValToCopy, VA.getLocVT(), dl, DAG);786      else787        ValToCopy = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), ValToCopy);788    }789    else if (VA.getLocInfo() == CCValAssign::BCvt)790      ValToCopy = DAG.getBitcast(VA.getLocVT(), ValToCopy);791 792    assert(VA.getLocInfo() != CCValAssign::FPExt &&793           "Unexpected FP-extend for return value.");794 795    // Report an error if we have attempted to return a value via an XMM796    // register and SSE was disabled.797    if (!Subtarget.hasSSE1() && X86::FR32XRegClass.contains(VA.getLocReg())) {798      errorUnsupported(DAG, dl, "SSE register return with SSE disabled");799      VA.convertToReg(X86::FP0); // Set reg to FP0, avoid hitting asserts.800    } else if (!Subtarget.hasSSE2() &&801               X86::FR64XRegClass.contains(VA.getLocReg()) &&802               ValVT == MVT::f64) {803      // When returning a double via an XMM register, report an error if SSE2 is804      // not enabled.805      errorUnsupported(DAG, dl, "SSE2 register return with SSE2 disabled");806      VA.convertToReg(X86::FP0); // Set reg to FP0, avoid hitting asserts.807    }808 809    // Returns in ST0/ST1 are handled specially: these are pushed as operands to810    // the RET instruction and handled by the FP Stackifier.811    if (VA.getLocReg() == X86::FP0 ||812        VA.getLocReg() == X86::FP1) {813      // If this is a copy from an xmm register to ST(0), use an FPExtend to814      // change the value to the FP stack register class.815      if (isScalarFPTypeInSSEReg(VA.getValVT()))816        ValToCopy = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f80, ValToCopy);817      RetVals.push_back(std::make_pair(VA.getLocReg(), ValToCopy));818      // Don't emit a copytoreg.819      continue;820    }821 822    // 64-bit vector (MMX) values are returned in XMM0 / XMM1 except for v1i64823    // which is returned in RAX / RDX.824    if (Subtarget.is64Bit()) {825      if (ValVT == MVT::x86mmx) {826        if (VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1) {827          ValToCopy = DAG.getBitcast(MVT::i64, ValToCopy);828          ValToCopy = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64,829                                  ValToCopy);830          // If we don't have SSE2 available, convert to v4f32 so the generated831          // register is legal.832          if (!Subtarget.hasSSE2())833            ValToCopy = DAG.getBitcast(MVT::v4f32, ValToCopy);834        }835      }836    }837 838    if (VA.needsCustom()) {839      assert(VA.getValVT() == MVT::v64i1 &&840             "Currently the only custom case is when we split v64i1 to 2 regs");841 842      Passv64i1ArgInRegs(dl, DAG, ValToCopy, RetVals, VA, RVLocs[++I],843                         Subtarget);844 845      // Add the second register to the CalleeSaveDisableRegs list.846      if (ShouldDisableCalleeSavedRegister)847        MF.getRegInfo().disableCalleeSavedRegister(RVLocs[I].getLocReg());848    } else {849      RetVals.push_back(std::make_pair(VA.getLocReg(), ValToCopy));850    }851  }852 853  SDValue Glue;854  SmallVector<SDValue, 6> RetOps;855  RetOps.push_back(Chain); // Operand #0 = Chain (updated below)856  // Operand #1 = Bytes To Pop857  RetOps.push_back(DAG.getTargetConstant(FuncInfo->getBytesToPopOnReturn(), dl,858                   MVT::i32));859 860  // Copy the result values into the output registers.861  for (auto &RetVal : RetVals) {862    if (RetVal.first == X86::FP0 || RetVal.first == X86::FP1) {863      RetOps.push_back(RetVal.second);864      continue; // Don't emit a copytoreg.865    }866 867    Chain = DAG.getCopyToReg(Chain, dl, RetVal.first, RetVal.second, Glue);868    Glue = Chain.getValue(1);869    RetOps.push_back(870        DAG.getRegister(RetVal.first, RetVal.second.getValueType()));871  }872 873  // Swift calling convention does not require we copy the sret argument874  // into %rax/%eax for the return, and SRetReturnReg is not set for Swift.875 876  // All x86 ABIs require that for returning structs by value we copy877  // the sret argument into %rax/%eax (depending on ABI) for the return.878  // We saved the argument into a virtual register in the entry block,879  // so now we copy the value out and into %rax/%eax.880  //881  // Checking Function.hasStructRetAttr() here is insufficient because the IR882  // may not have an explicit sret argument. If FuncInfo.CanLowerReturn is883  // false, then an sret argument may be implicitly inserted in the SelDAG. In884  // either case FuncInfo->setSRetReturnReg() will have been called.885  if (Register SRetReg = FuncInfo->getSRetReturnReg()) {886    // When we have both sret and another return value, we should use the887    // original Chain stored in RetOps[0], instead of the current Chain updated888    // in the above loop. If we only have sret, RetOps[0] equals to Chain.889 890    // For the case of sret and another return value, we have891    //   Chain_0 at the function entry892    //   Chain_1 = getCopyToReg(Chain_0) in the above loop893    // If we use Chain_1 in getCopyFromReg, we will have894    //   Val = getCopyFromReg(Chain_1)895    //   Chain_2 = getCopyToReg(Chain_1, Val) from below896 897    // getCopyToReg(Chain_0) will be glued together with898    // getCopyToReg(Chain_1, Val) into Unit A, getCopyFromReg(Chain_1) will be899    // in Unit B, and we will have cyclic dependency between Unit A and Unit B:900    //   Data dependency from Unit B to Unit A due to usage of Val in901    //     getCopyToReg(Chain_1, Val)902    //   Chain dependency from Unit A to Unit B903 904    // So here, we use RetOps[0] (i.e Chain_0) for getCopyFromReg.905    SDValue Val = DAG.getCopyFromReg(RetOps[0], dl, SRetReg,906                                     getPointerTy(MF.getDataLayout()));907 908    Register RetValReg909        = (Subtarget.is64Bit() && !Subtarget.isTarget64BitILP32()) ?910          X86::RAX : X86::EAX;911    Chain = DAG.getCopyToReg(Chain, dl, RetValReg, Val, Glue);912    Glue = Chain.getValue(1);913 914    // RAX/EAX now acts like a return value.915    RetOps.push_back(916        DAG.getRegister(RetValReg, getPointerTy(DAG.getDataLayout())));917 918    // Add the returned register to the CalleeSaveDisableRegs list. Don't do919    // this however for preserve_most/preserve_all to minimize the number of920    // callee-saved registers for these CCs.921    if (ShouldDisableCalleeSavedRegister &&922        CallConv != CallingConv::PreserveAll &&923        CallConv != CallingConv::PreserveMost)924      MF.getRegInfo().disableCalleeSavedRegister(RetValReg);925  }926 927  const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();928  const MCPhysReg *I =929      TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction());930  if (I) {931    for (; *I; ++I) {932      if (X86::GR64RegClass.contains(*I))933        RetOps.push_back(DAG.getRegister(*I, MVT::i64));934      else935        llvm_unreachable("Unexpected register class in CSRsViaCopy!");936    }937  }938 939  RetOps[0] = Chain;  // Update chain.940 941  // Add the glue if we have it.942  if (Glue.getNode())943    RetOps.push_back(Glue);944 945  X86ISD::NodeType opcode = X86ISD::RET_GLUE;946  if (CallConv == CallingConv::X86_INTR)947    opcode = X86ISD::IRET;948  return DAG.getNode(opcode, dl, MVT::Other, RetOps);949}950 951bool X86TargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const {952  if (N->getNumValues() != 1 || !N->hasNUsesOfValue(1, 0))953    return false;954 955  SDValue TCChain = Chain;956  SDNode *Copy = *N->user_begin();957  if (Copy->getOpcode() == ISD::CopyToReg) {958    // If the copy has a glue operand, we conservatively assume it isn't safe to959    // perform a tail call.960    if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue)961      return false;962    TCChain = Copy->getOperand(0);963  } else if (Copy->getOpcode() != ISD::FP_EXTEND)964    return false;965 966  bool HasRet = false;967  for (const SDNode *U : Copy->users()) {968    if (U->getOpcode() != X86ISD::RET_GLUE)969      return false;970    // If we are returning more than one value, we can definitely971    // not make a tail call see PR19530972    if (U->getNumOperands() > 4)973      return false;974    if (U->getNumOperands() == 4 &&975        U->getOperand(U->getNumOperands() - 1).getValueType() != MVT::Glue)976      return false;977    HasRet = true;978  }979 980  if (!HasRet)981    return false;982 983  Chain = TCChain;984  return true;985}986 987EVT X86TargetLowering::getTypeForExtReturn(LLVMContext &Context, EVT VT,988                                           ISD::NodeType ExtendKind) const {989  MVT ReturnMVT = MVT::i32;990 991  bool Darwin = Subtarget.getTargetTriple().isOSDarwin();992  if (VT == MVT::i1 || (!Darwin && (VT == MVT::i8 || VT == MVT::i16))) {993    // The ABI does not require i1, i8 or i16 to be extended.994    //995    // On Darwin, there is code in the wild relying on Clang's old behaviour of996    // always extending i8/i16 return values, so keep doing that for now.997    // (PR26665).998    ReturnMVT = MVT::i8;999  }1000 1001  EVT MinVT = getRegisterType(Context, ReturnMVT);1002  return VT.bitsLT(MinVT) ? MinVT : VT;1003}1004 1005/// Reads two 32 bit registers and creates a 64 bit mask value.1006/// \param VA The current 32 bit value that need to be assigned.1007/// \param NextVA The next 32 bit value that need to be assigned.1008/// \param Root The parent DAG node.1009/// \param [in,out] InGlue Represents SDvalue in the parent DAG node for1010///                        glue purposes. In the case the DAG is already using1011///                        physical register instead of virtual, we should glue1012///                        our new SDValue to InGlue SDvalue.1013/// \return a new SDvalue of size 64bit.1014static SDValue getv64i1Argument(CCValAssign &VA, CCValAssign &NextVA,1015                                SDValue &Root, SelectionDAG &DAG,1016                                const SDLoc &DL, const X86Subtarget &Subtarget,1017                                SDValue *InGlue = nullptr) {1018  assert((Subtarget.hasBWI()) && "Expected AVX512BW target!");1019  assert(Subtarget.is32Bit() && "Expecting 32 bit target");1020  assert(VA.getValVT() == MVT::v64i1 &&1021         "Expecting first location of 64 bit width type");1022  assert(NextVA.getValVT() == VA.getValVT() &&1023         "The locations should have the same type");1024  assert(VA.isRegLoc() && NextVA.isRegLoc() &&1025         "The values should reside in two registers");1026 1027  SDValue Lo, Hi;1028  SDValue ArgValueLo, ArgValueHi;1029 1030  MachineFunction &MF = DAG.getMachineFunction();1031  const TargetRegisterClass *RC = &X86::GR32RegClass;1032 1033  // Read a 32 bit value from the registers.1034  if (nullptr == InGlue) {1035    // When no physical register is present,1036    // create an intermediate virtual register.1037    Register Reg = MF.addLiveIn(VA.getLocReg(), RC);1038    ArgValueLo = DAG.getCopyFromReg(Root, DL, Reg, MVT::i32);1039    Reg = MF.addLiveIn(NextVA.getLocReg(), RC);1040    ArgValueHi = DAG.getCopyFromReg(Root, DL, Reg, MVT::i32);1041  } else {1042    // When a physical register is available read the value from it and glue1043    // the reads together.1044    ArgValueLo =1045      DAG.getCopyFromReg(Root, DL, VA.getLocReg(), MVT::i32, *InGlue);1046    *InGlue = ArgValueLo.getValue(2);1047    ArgValueHi =1048      DAG.getCopyFromReg(Root, DL, NextVA.getLocReg(), MVT::i32, *InGlue);1049    *InGlue = ArgValueHi.getValue(2);1050  }1051 1052  // Convert the i32 type into v32i1 type.1053  Lo = DAG.getBitcast(MVT::v32i1, ArgValueLo);1054 1055  // Convert the i32 type into v32i1 type.1056  Hi = DAG.getBitcast(MVT::v32i1, ArgValueHi);1057 1058  // Concatenate the two values together.1059  return DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v64i1, Lo, Hi);1060}1061 1062/// The function will lower a register of various sizes (8/16/32/64)1063/// to a mask value of the expected size (v8i1/v16i1/v32i1/v64i1)1064/// \returns a DAG node contains the operand after lowering to mask type.1065static SDValue lowerRegToMasks(const SDValue &ValArg, const EVT &ValVT,1066                               const EVT &ValLoc, const SDLoc &DL,1067                               SelectionDAG &DAG) {1068  SDValue ValReturned = ValArg;1069 1070  if (ValVT == MVT::v1i1)1071    return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1, ValReturned);1072 1073  if (ValVT == MVT::v64i1) {1074    // In 32 bit machine, this case is handled by getv64i1Argument1075    assert(ValLoc == MVT::i64 && "Expecting only i64 locations");1076    // In 64 bit machine, There is no need to truncate the value only bitcast1077  } else {1078    MVT MaskLenVT;1079    switch (ValVT.getSimpleVT().SimpleTy) {1080    case MVT::v8i1:1081      MaskLenVT = MVT::i8;1082      break;1083    case MVT::v16i1:1084      MaskLenVT = MVT::i16;1085      break;1086    case MVT::v32i1:1087      MaskLenVT = MVT::i32;1088      break;1089    default:1090      llvm_unreachable("Expecting a vector of i1 types");1091    }1092 1093    ValReturned = DAG.getNode(ISD::TRUNCATE, DL, MaskLenVT, ValReturned);1094  }1095  return DAG.getBitcast(ValVT, ValReturned);1096}1097 1098static SDValue getPopFromX87Reg(SelectionDAG &DAG, SDValue Chain,1099                                const SDLoc &dl, Register Reg, EVT VT,1100                                SDValue Glue) {1101  SDVTList VTs = DAG.getVTList(VT, MVT::Other, MVT::Glue);1102  SDValue Ops[] = {Chain, DAG.getRegister(Reg, VT), Glue};1103  return DAG.getNode(X86ISD::POP_FROM_X87_REG, dl, VTs,1104                     ArrayRef(Ops, Glue.getNode() ? 3 : 2));1105}1106 1107/// Lower the result values of a call into the1108/// appropriate copies out of appropriate physical registers.1109///1110SDValue X86TargetLowering::LowerCallResult(1111    SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool isVarArg,1112    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,1113    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,1114    uint32_t *RegMask) const {1115 1116  const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();1117  // Assign locations to each value returned by this call.1118  SmallVector<CCValAssign, 16> RVLocs;1119  CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,1120                 *DAG.getContext());1121  CCInfo.AnalyzeCallResult(Ins, RetCC_X86);1122 1123  // Copy all of the result registers out of their specified physreg.1124  for (unsigned I = 0, InsIndex = 0, E = RVLocs.size(); I != E;1125       ++I, ++InsIndex) {1126    CCValAssign &VA = RVLocs[I];1127    EVT CopyVT = VA.getLocVT();1128 1129    // In some calling conventions we need to remove the used registers1130    // from the register mask.1131    if (RegMask) {1132      for (MCPhysReg SubReg : TRI->subregs_inclusive(VA.getLocReg()))1133        RegMask[SubReg / 32] &= ~(1u << (SubReg % 32));1134    }1135 1136    // Report an error if there was an attempt to return FP values via XMM1137    // registers.1138    if (!Subtarget.hasSSE1() && X86::FR32XRegClass.contains(VA.getLocReg())) {1139      errorUnsupported(DAG, dl, "SSE register return with SSE disabled");1140      if (VA.getLocReg() == X86::XMM1)1141        VA.convertToReg(X86::FP1); // Set reg to FP1, avoid hitting asserts.1142      else1143        VA.convertToReg(X86::FP0); // Set reg to FP0, avoid hitting asserts.1144    } else if (!Subtarget.hasSSE2() &&1145               X86::FR64XRegClass.contains(VA.getLocReg()) &&1146               CopyVT == MVT::f64) {1147      errorUnsupported(DAG, dl, "SSE2 register return with SSE2 disabled");1148      if (VA.getLocReg() == X86::XMM1)1149        VA.convertToReg(X86::FP1); // Set reg to FP1, avoid hitting asserts.1150      else1151        VA.convertToReg(X86::FP0); // Set reg to FP0, avoid hitting asserts.1152    }1153 1154    // If we prefer to use the value in xmm registers, copy it out as f80 and1155    // use a truncate to move it from fp stack reg to xmm reg.1156    bool RoundAfterCopy = false;1157    bool X87Result = VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1;1158    if (X87Result && isScalarFPTypeInSSEReg(VA.getValVT())) {1159      if (!Subtarget.hasX87())1160        report_fatal_error("X87 register return with X87 disabled");1161      CopyVT = MVT::f80;1162      RoundAfterCopy = (CopyVT != VA.getLocVT());1163    }1164 1165    SDValue Val;1166    if (VA.needsCustom()) {1167      assert(VA.getValVT() == MVT::v64i1 &&1168             "Currently the only custom case is when we split v64i1 to 2 regs");1169      Val =1170          getv64i1Argument(VA, RVLocs[++I], Chain, DAG, dl, Subtarget, &InGlue);1171    } else {1172      Chain =1173          X87Result1174              ? getPopFromX87Reg(DAG, Chain, dl, VA.getLocReg(), CopyVT, InGlue)1175                    .getValue(1)1176              : DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), CopyVT, InGlue)1177                    .getValue(1);1178      Val = Chain.getValue(0);1179      InGlue = Chain.getValue(2);1180    }1181 1182    if (RoundAfterCopy)1183      Val = DAG.getNode(ISD::FP_ROUND, dl, VA.getValVT(), Val,1184                        // This truncation won't change the value.1185                        DAG.getIntPtrConstant(1, dl, /*isTarget=*/true));1186 1187    if (VA.isExtInLoc()) {1188      if (VA.getValVT().isVector() &&1189          VA.getValVT().getScalarType() == MVT::i1 &&1190          ((VA.getLocVT() == MVT::i64) || (VA.getLocVT() == MVT::i32) ||1191           (VA.getLocVT() == MVT::i16) || (VA.getLocVT() == MVT::i8))) {1192        // promoting a mask type (v*i1) into a register of type i64/i32/i16/i81193        Val = lowerRegToMasks(Val, VA.getValVT(), VA.getLocVT(), dl, DAG);1194      } else1195        Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val);1196    }1197 1198    if (VA.getLocInfo() == CCValAssign::BCvt)1199      Val = DAG.getBitcast(VA.getValVT(), Val);1200 1201    InVals.push_back(Val);1202  }1203 1204  return Chain;1205}1206 1207//===----------------------------------------------------------------------===//1208//                C & StdCall & Fast Calling Convention implementation1209//===----------------------------------------------------------------------===//1210//  StdCall calling convention seems to be standard for many Windows' API1211//  routines and around. It differs from C calling convention just a little:1212//  callee should clean up the stack, not caller. Symbols should be also1213//  decorated in some fancy way :) It doesn't support any vector arguments.1214//  For info on fast calling convention see Fast Calling Convention (tail call)1215//  implementation LowerX86_32FastCCCallTo.1216 1217/// Determines whether Args, either a set of outgoing arguments to a call, or a1218/// set of incoming args of a call, contains an sret pointer that the callee1219/// pops1220template <typename T>1221static bool hasCalleePopSRet(const SmallVectorImpl<T> &Args,1222                             const X86Subtarget &Subtarget) {1223  // Not C++20 (yet), so no concepts available.1224  static_assert(std::is_same_v<T, ISD::OutputArg> ||1225                    std::is_same_v<T, ISD::InputArg>,1226                "requires ISD::OutputArg or ISD::InputArg");1227 1228  // Only 32-bit pops the sret.  It's a 64-bit world these days, so early-out1229  // for most compilations.1230  if (!Subtarget.is32Bit())1231    return false;1232 1233  if (Args.empty())1234    return false;1235 1236  // Most calls do not have an sret argument, check the arg next.1237  const ISD::ArgFlagsTy &Flags = Args[0].Flags;1238  if (!Flags.isSRet() || Flags.isInReg())1239    return false;1240 1241  // The MSVCabi does not pop the sret.1242  if (Subtarget.getTargetTriple().isOSMSVCRT())1243    return false;1244 1245  // MCUs don't pop the sret1246  if (Subtarget.isTargetMCU())1247    return false;1248 1249  // Callee pops argument1250  return true;1251}1252 1253/// Make a copy of an aggregate at address specified by "Src" to address1254/// "Dst" with size and alignment information specified by the specific1255/// parameter attribute. The copy will be passed as a byval function parameter.1256static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,1257                                         SDValue Chain, ISD::ArgFlagsTy Flags,1258                                         SelectionDAG &DAG, const SDLoc &dl) {1259  SDValue SizeNode = DAG.getIntPtrConstant(Flags.getByValSize(), dl);1260 1261  return DAG.getMemcpy(1262      Chain, dl, Dst, Src, SizeNode, Flags.getNonZeroByValAlign(),1263      /*isVolatile*/ false, /*AlwaysInline=*/true,1264      /*CI=*/nullptr, std::nullopt, MachinePointerInfo(), MachinePointerInfo());1265}1266 1267/// Return true if the calling convention is one that we can guarantee TCO for.1268static bool canGuaranteeTCO(CallingConv::ID CC) {1269  return (CC == CallingConv::Fast || CC == CallingConv::GHC ||1270          CC == CallingConv::X86_RegCall || CC == CallingConv::HiPE ||1271          CC == CallingConv::Tail || CC == CallingConv::SwiftTail);1272}1273 1274/// Return true if we might ever do TCO for calls with this calling convention.1275static bool mayTailCallThisCC(CallingConv::ID CC) {1276  switch (CC) {1277  // C calling conventions:1278  case CallingConv::C:1279  case CallingConv::Win64:1280  case CallingConv::X86_64_SysV:1281  case CallingConv::PreserveNone:1282  // Callee pop conventions:1283  case CallingConv::X86_ThisCall:1284  case CallingConv::X86_StdCall:1285  case CallingConv::X86_VectorCall:1286  case CallingConv::X86_FastCall:1287  // Swift:1288  case CallingConv::Swift:1289    return true;1290  default:1291    return canGuaranteeTCO(CC);1292  }1293}1294 1295/// Return true if the function is being made into a tailcall target by1296/// changing its ABI.1297static bool shouldGuaranteeTCO(CallingConv::ID CC, bool GuaranteedTailCallOpt) {1298  return (GuaranteedTailCallOpt && canGuaranteeTCO(CC)) ||1299         CC == CallingConv::Tail || CC == CallingConv::SwiftTail;1300}1301 1302bool X86TargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {1303  if (!CI->isTailCall())1304    return false;1305 1306  CallingConv::ID CalleeCC = CI->getCallingConv();1307  if (!mayTailCallThisCC(CalleeCC))1308    return false;1309 1310  return true;1311}1312 1313SDValue1314X86TargetLowering::LowerMemArgument(SDValue Chain, CallingConv::ID CallConv,1315                                    const SmallVectorImpl<ISD::InputArg> &Ins,1316                                    const SDLoc &dl, SelectionDAG &DAG,1317                                    const CCValAssign &VA,1318                                    MachineFrameInfo &MFI, unsigned i) const {1319  // Create the nodes corresponding to a load from this parameter slot.1320  ISD::ArgFlagsTy Flags = Ins[i].Flags;1321  bool AlwaysUseMutable = shouldGuaranteeTCO(1322      CallConv, DAG.getTarget().Options.GuaranteedTailCallOpt);1323  bool isImmutable = !AlwaysUseMutable && !Flags.isByVal();1324  EVT ValVT;1325  MVT PtrVT = getPointerTy(DAG.getDataLayout());1326 1327  // If value is passed by pointer we have address passed instead of the value1328  // itself. No need to extend if the mask value and location share the same1329  // absolute size.1330  bool ExtendedInMem =1331      VA.isExtInLoc() && VA.getValVT().getScalarType() == MVT::i1 &&1332      VA.getValVT().getSizeInBits() != VA.getLocVT().getSizeInBits();1333 1334  if (VA.getLocInfo() == CCValAssign::Indirect || ExtendedInMem)1335    ValVT = VA.getLocVT();1336  else1337    ValVT = VA.getValVT();1338 1339  // FIXME: For now, all byval parameter objects are marked mutable. This can be1340  // changed with more analysis.1341  // In case of tail call optimization mark all arguments mutable. Since they1342  // could be overwritten by lowering of arguments in case of a tail call.1343  if (Flags.isByVal()) {1344    unsigned Bytes = Flags.getByValSize();1345    if (Bytes == 0) Bytes = 1; // Don't create zero-sized stack objects.1346 1347    // FIXME: For now, all byval parameter objects are marked as aliasing. This1348    // can be improved with deeper analysis.1349    int FI = MFI.CreateFixedObject(Bytes, VA.getLocMemOffset(), isImmutable,1350                                   /*isAliased=*/true);1351    return DAG.getFrameIndex(FI, PtrVT);1352  }1353 1354  EVT ArgVT = Ins[i].ArgVT;1355 1356  // If this is a vector that has been split into multiple parts, don't elide1357  // the copy. The layout on the stack may not match the packed in-memory1358  // layout.1359  bool ScalarizedVector = ArgVT.isVector() && !VA.getLocVT().isVector();1360 1361  // This is an argument in memory. We might be able to perform copy elision.1362  // If the argument is passed directly in memory without any extension, then we1363  // can perform copy elision. Large vector types, for example, may be passed1364  // indirectly by pointer.1365  if (Flags.isCopyElisionCandidate() &&1366      VA.getLocInfo() != CCValAssign::Indirect && !ExtendedInMem &&1367      !ScalarizedVector) {1368    SDValue PartAddr;1369    if (Ins[i].PartOffset == 0) {1370      // If this is a one-part value or the first part of a multi-part value,1371      // create a stack object for the entire argument value type and return a1372      // load from our portion of it. This assumes that if the first part of an1373      // argument is in memory, the rest will also be in memory.1374      int FI = MFI.CreateFixedObject(ArgVT.getStoreSize(), VA.getLocMemOffset(),1375                                     /*IsImmutable=*/false);1376      PartAddr = DAG.getFrameIndex(FI, PtrVT);1377      return DAG.getLoad(1378          ValVT, dl, Chain, PartAddr,1379          MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI));1380    }1381 1382    // This is not the first piece of an argument in memory. See if there is1383    // already a fixed stack object including this offset. If so, assume it1384    // was created by the PartOffset == 0 branch above and create a load from1385    // the appropriate offset into it.1386    int64_t PartBegin = VA.getLocMemOffset();1387    int64_t PartEnd = PartBegin + ValVT.getSizeInBits() / 8;1388    int FI = MFI.getObjectIndexBegin();1389    for (; MFI.isFixedObjectIndex(FI); ++FI) {1390      int64_t ObjBegin = MFI.getObjectOffset(FI);1391      int64_t ObjEnd = ObjBegin + MFI.getObjectSize(FI);1392      if (ObjBegin <= PartBegin && PartEnd <= ObjEnd)1393        break;1394    }1395    if (MFI.isFixedObjectIndex(FI)) {1396      SDValue Addr =1397          DAG.getNode(ISD::ADD, dl, PtrVT, DAG.getFrameIndex(FI, PtrVT),1398                      DAG.getIntPtrConstant(Ins[i].PartOffset, dl));1399      return DAG.getLoad(ValVT, dl, Chain, Addr,1400                         MachinePointerInfo::getFixedStack(1401                             DAG.getMachineFunction(), FI, Ins[i].PartOffset));1402    }1403  }1404 1405  int FI = MFI.CreateFixedObject(ValVT.getSizeInBits() / 8,1406                                 VA.getLocMemOffset(), isImmutable);1407 1408  // Set SExt or ZExt flag.1409  if (VA.getLocInfo() == CCValAssign::ZExt) {1410    MFI.setObjectZExt(FI, true);1411  } else if (VA.getLocInfo() == CCValAssign::SExt) {1412    MFI.setObjectSExt(FI, true);1413  }1414 1415  MaybeAlign Alignment;1416  if (Subtarget.isTargetWindowsMSVC() && !Subtarget.is64Bit() &&1417      ValVT != MVT::f80)1418    Alignment = MaybeAlign(4);1419  SDValue FIN = DAG.getFrameIndex(FI, PtrVT);1420  SDValue Val = DAG.getLoad(1421      ValVT, dl, Chain, FIN,1422      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI),1423      Alignment);1424  return ExtendedInMem1425             ? (VA.getValVT().isVector()1426                    ? DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VA.getValVT(), Val)1427                    : DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val))1428             : Val;1429}1430 1431// FIXME: Get this from tablegen.1432static ArrayRef<MCPhysReg> get64BitArgumentGPRs(CallingConv::ID CallConv,1433                                                const X86Subtarget &Subtarget) {1434  assert(Subtarget.is64Bit());1435 1436  if (Subtarget.isCallingConvWin64(CallConv)) {1437    static const MCPhysReg GPR64ArgRegsWin64[] = {1438      X86::RCX, X86::RDX, X86::R8,  X86::R91439    };1440    return GPR64ArgRegsWin64;1441  }1442 1443  static const MCPhysReg GPR64ArgRegs64Bit[] = {1444    X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R91445  };1446  return GPR64ArgRegs64Bit;1447}1448 1449// FIXME: Get this from tablegen.1450static ArrayRef<MCPhysReg> get64BitArgumentXMMs(MachineFunction &MF,1451                                                CallingConv::ID CallConv,1452                                                const X86Subtarget &Subtarget) {1453  assert(Subtarget.is64Bit());1454  if (Subtarget.isCallingConvWin64(CallConv)) {1455    // The XMM registers which might contain var arg parameters are shadowed1456    // in their paired GPR.  So we only need to save the GPR to their home1457    // slots.1458    // TODO: __vectorcall will change this.1459    return {};1460  }1461 1462  bool isSoftFloat = Subtarget.useSoftFloat();1463  if (isSoftFloat || !Subtarget.hasSSE1())1464    // Kernel mode asks for SSE to be disabled, so there are no XMM argument1465    // registers.1466    return {};1467 1468  static const MCPhysReg XMMArgRegs64Bit[] = {1469    X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,1470    X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM71471  };1472  return XMMArgRegs64Bit;1473}1474 1475#ifndef NDEBUG1476static bool isSortedByValueNo(ArrayRef<CCValAssign> ArgLocs) {1477  return llvm::is_sorted(1478      ArgLocs, [](const CCValAssign &A, const CCValAssign &B) -> bool {1479        return A.getValNo() < B.getValNo();1480      });1481}1482#endif1483 1484namespace {1485/// This is a helper class for lowering variable arguments parameters.1486class VarArgsLoweringHelper {1487public:1488  VarArgsLoweringHelper(X86MachineFunctionInfo *FuncInfo, const SDLoc &Loc,1489                        SelectionDAG &DAG, const X86Subtarget &Subtarget,1490                        CallingConv::ID CallConv, CCState &CCInfo)1491      : FuncInfo(FuncInfo), DL(Loc), DAG(DAG), Subtarget(Subtarget),1492        TheMachineFunction(DAG.getMachineFunction()),1493        TheFunction(TheMachineFunction.getFunction()),1494        FrameInfo(TheMachineFunction.getFrameInfo()),1495        FrameLowering(*Subtarget.getFrameLowering()),1496        TargLowering(DAG.getTargetLoweringInfo()), CallConv(CallConv),1497        CCInfo(CCInfo) {}1498 1499  // Lower variable arguments parameters.1500  void lowerVarArgsParameters(SDValue &Chain, unsigned StackSize);1501 1502private:1503  void createVarArgAreaAndStoreRegisters(SDValue &Chain, unsigned StackSize);1504 1505  void forwardMustTailParameters(SDValue &Chain);1506 1507  bool is64Bit() const { return Subtarget.is64Bit(); }1508  bool isWin64() const { return Subtarget.isCallingConvWin64(CallConv); }1509 1510  X86MachineFunctionInfo *FuncInfo;1511  const SDLoc &DL;1512  SelectionDAG &DAG;1513  const X86Subtarget &Subtarget;1514  MachineFunction &TheMachineFunction;1515  const Function &TheFunction;1516  MachineFrameInfo &FrameInfo;1517  const TargetFrameLowering &FrameLowering;1518  const TargetLowering &TargLowering;1519  CallingConv::ID CallConv;1520  CCState &CCInfo;1521};1522} // namespace1523 1524void VarArgsLoweringHelper::createVarArgAreaAndStoreRegisters(1525    SDValue &Chain, unsigned StackSize) {1526  // If the function takes variable number of arguments, make a frame index for1527  // the start of the first vararg value... for expansion of llvm.va_start. We1528  // can skip this if there are no va_start calls.1529  if (is64Bit() || (CallConv != CallingConv::X86_FastCall &&1530                    CallConv != CallingConv::X86_ThisCall)) {1531    FuncInfo->setVarArgsFrameIndex(1532        FrameInfo.CreateFixedObject(1, StackSize, true));1533  }1534 1535  // 64-bit calling conventions support varargs and register parameters, so we1536  // have to do extra work to spill them in the prologue.1537  if (is64Bit()) {1538    // Find the first unallocated argument registers.1539    ArrayRef<MCPhysReg> ArgGPRs = get64BitArgumentGPRs(CallConv, Subtarget);1540    ArrayRef<MCPhysReg> ArgXMMs =1541        get64BitArgumentXMMs(TheMachineFunction, CallConv, Subtarget);1542    unsigned NumIntRegs = CCInfo.getFirstUnallocated(ArgGPRs);1543    unsigned NumXMMRegs = CCInfo.getFirstUnallocated(ArgXMMs);1544 1545    assert(!(NumXMMRegs && !Subtarget.hasSSE1()) &&1546           "SSE register cannot be used when SSE is disabled!");1547 1548    if (isWin64()) {1549      // Get to the caller-allocated home save location.  Add 8 to account1550      // for the return address.1551      int HomeOffset = FrameLowering.getOffsetOfLocalArea() + 8;1552      FuncInfo->setRegSaveFrameIndex(1553          FrameInfo.CreateFixedObject(1, NumIntRegs * 8 + HomeOffset, false));1554      // Fixup to set vararg frame on shadow area (4 x i64).1555      if (NumIntRegs < 4)1556        FuncInfo->setVarArgsFrameIndex(FuncInfo->getRegSaveFrameIndex());1557    } else {1558      // For X86-64, if there are vararg parameters that are passed via1559      // registers, then we must store them to their spots on the stack so1560      // they may be loaded by dereferencing the result of va_next.1561      FuncInfo->setVarArgsGPOffset(NumIntRegs * 8);1562      FuncInfo->setVarArgsFPOffset(ArgGPRs.size() * 8 + NumXMMRegs * 16);1563      FuncInfo->setRegSaveFrameIndex(FrameInfo.CreateStackObject(1564          ArgGPRs.size() * 8 + ArgXMMs.size() * 16, Align(16), false));1565    }1566 1567    SmallVector<SDValue, 6>1568        LiveGPRs; // list of SDValue for GPR registers keeping live input value1569    SmallVector<SDValue, 8> LiveXMMRegs; // list of SDValue for XMM registers1570                                         // keeping live input value1571    SDValue ALVal; // if applicable keeps SDValue for %al register1572 1573    // Gather all the live in physical registers.1574    for (MCPhysReg Reg : ArgGPRs.slice(NumIntRegs)) {1575      Register GPR = TheMachineFunction.addLiveIn(Reg, &X86::GR64RegClass);1576      LiveGPRs.push_back(DAG.getCopyFromReg(Chain, DL, GPR, MVT::i64));1577    }1578    const auto &AvailableXmms = ArgXMMs.slice(NumXMMRegs);1579    if (!AvailableXmms.empty()) {1580      Register AL = TheMachineFunction.addLiveIn(X86::AL, &X86::GR8RegClass);1581      ALVal = DAG.getCopyFromReg(Chain, DL, AL, MVT::i8);1582      for (MCPhysReg Reg : AvailableXmms) {1583        // FastRegisterAllocator spills virtual registers at basic1584        // block boundary. That leads to usages of xmm registers1585        // outside of check for %al. Pass physical registers to1586        // VASTART_SAVE_XMM_REGS to avoid unneccessary spilling.1587        TheMachineFunction.getRegInfo().addLiveIn(Reg);1588        LiveXMMRegs.push_back(DAG.getRegister(Reg, MVT::v4f32));1589      }1590    }1591 1592    // Store the integer parameter registers.1593    SmallVector<SDValue, 8> MemOps;1594    SDValue RSFIN =1595        DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(),1596                          TargLowering.getPointerTy(DAG.getDataLayout()));1597    unsigned Offset = FuncInfo->getVarArgsGPOffset();1598    for (SDValue Val : LiveGPRs) {1599      SDValue FIN = DAG.getNode(ISD::ADD, DL,1600                                TargLowering.getPointerTy(DAG.getDataLayout()),1601                                RSFIN, DAG.getIntPtrConstant(Offset, DL));1602      SDValue Store =1603          DAG.getStore(Val.getValue(1), DL, Val, FIN,1604                       MachinePointerInfo::getFixedStack(1605                           DAG.getMachineFunction(),1606                           FuncInfo->getRegSaveFrameIndex(), Offset));1607      MemOps.push_back(Store);1608      Offset += 8;1609    }1610 1611    // Now store the XMM (fp + vector) parameter registers.1612    if (!LiveXMMRegs.empty()) {1613      SmallVector<SDValue, 12> SaveXMMOps;1614      SaveXMMOps.push_back(Chain);1615      SaveXMMOps.push_back(ALVal);1616      SaveXMMOps.push_back(RSFIN);1617      SaveXMMOps.push_back(1618          DAG.getTargetConstant(FuncInfo->getVarArgsFPOffset(), DL, MVT::i32));1619      llvm::append_range(SaveXMMOps, LiveXMMRegs);1620      MachineMemOperand *StoreMMO =1621          DAG.getMachineFunction().getMachineMemOperand(1622              MachinePointerInfo::getFixedStack(1623                  DAG.getMachineFunction(), FuncInfo->getRegSaveFrameIndex(),1624                  Offset),1625              MachineMemOperand::MOStore, 128, Align(16));1626      MemOps.push_back(DAG.getMemIntrinsicNode(X86ISD::VASTART_SAVE_XMM_REGS,1627                                               DL, DAG.getVTList(MVT::Other),1628                                               SaveXMMOps, MVT::i8, StoreMMO));1629    }1630 1631    if (!MemOps.empty())1632      Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);1633  }1634}1635 1636void VarArgsLoweringHelper::forwardMustTailParameters(SDValue &Chain) {1637  // Find the largest legal vector type.1638  MVT VecVT = MVT::Other;1639  // FIXME: Only some x86_32 calling conventions support AVX512.1640  if (Subtarget.useAVX512Regs() &&1641      (is64Bit() || (CallConv == CallingConv::X86_VectorCall ||1642                     CallConv == CallingConv::Intel_OCL_BI)))1643    VecVT = MVT::v16f32;1644  else if (Subtarget.hasAVX())1645    VecVT = MVT::v8f32;1646  else if (Subtarget.hasSSE2())1647    VecVT = MVT::v4f32;1648 1649  // We forward some GPRs and some vector types.1650  SmallVector<MVT, 2> RegParmTypes;1651  MVT IntVT = is64Bit() ? MVT::i64 : MVT::i32;1652  RegParmTypes.push_back(IntVT);1653  if (VecVT != MVT::Other)1654    RegParmTypes.push_back(VecVT);1655 1656  // Compute the set of forwarded registers. The rest are scratch.1657  SmallVectorImpl<ForwardedRegister> &Forwards =1658      FuncInfo->getForwardedMustTailRegParms();1659  CCInfo.analyzeMustTailForwardedRegisters(Forwards, RegParmTypes, CC_X86);1660 1661  // Forward AL for SysV x86_64 targets, since it is used for varargs.1662  if (is64Bit() && !isWin64() && !CCInfo.isAllocated(X86::AL)) {1663    Register ALVReg = TheMachineFunction.addLiveIn(X86::AL, &X86::GR8RegClass);1664    Forwards.push_back(ForwardedRegister(ALVReg, X86::AL, MVT::i8));1665  }1666 1667  // Copy all forwards from physical to virtual registers.1668  for (ForwardedRegister &FR : Forwards) {1669    // FIXME: Can we use a less constrained schedule?1670    SDValue RegVal = DAG.getCopyFromReg(Chain, DL, FR.VReg, FR.VT);1671    FR.VReg = TheMachineFunction.getRegInfo().createVirtualRegister(1672        TargLowering.getRegClassFor(FR.VT));1673    Chain = DAG.getCopyToReg(Chain, DL, FR.VReg, RegVal);1674  }1675}1676 1677void VarArgsLoweringHelper::lowerVarArgsParameters(SDValue &Chain,1678                                                   unsigned StackSize) {1679  // Set FrameIndex to the 0xAAAAAAA value to mark unset state.1680  // If necessary, it would be set into the correct value later.1681  FuncInfo->setVarArgsFrameIndex(0xAAAAAAA);1682  FuncInfo->setRegSaveFrameIndex(0xAAAAAAA);1683 1684  if (FrameInfo.hasVAStart())1685    createVarArgAreaAndStoreRegisters(Chain, StackSize);1686 1687  if (FrameInfo.hasMustTailInVarArgFunc())1688    forwardMustTailParameters(Chain);1689}1690 1691SDValue X86TargetLowering::LowerFormalArguments(1692    SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,1693    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,1694    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {1695  MachineFunction &MF = DAG.getMachineFunction();1696  X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();1697 1698  const Function &F = MF.getFunction();1699  if (F.hasExternalLinkage() && Subtarget.isTargetCygMing() &&1700      F.getName() == "main")1701    FuncInfo->setForceFramePointer(true);1702 1703  MachineFrameInfo &MFI = MF.getFrameInfo();1704  bool Is64Bit = Subtarget.is64Bit();1705  bool IsWin64 = Subtarget.isCallingConvWin64(CallConv);1706 1707  assert(1708      !(IsVarArg && canGuaranteeTCO(CallConv)) &&1709      "Var args not supported with calling conv' regcall, fastcc, ghc or hipe");1710 1711  // Assign locations to all of the incoming arguments.1712  SmallVector<CCValAssign, 16> ArgLocs;1713  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());1714 1715  // Allocate shadow area for Win64.1716  if (IsWin64)1717    CCInfo.AllocateStack(32, Align(8));1718 1719  CCInfo.AnalyzeArguments(Ins, CC_X86);1720 1721  // In vectorcall calling convention a second pass is required for the HVA1722  // types.1723  if (CallingConv::X86_VectorCall == CallConv) {1724    CCInfo.AnalyzeArgumentsSecondPass(Ins, CC_X86);1725  }1726 1727  // The next loop assumes that the locations are in the same order of the1728  // input arguments.1729  assert(isSortedByValueNo(ArgLocs) &&1730         "Argument Location list must be sorted before lowering");1731 1732  SDValue ArgValue;1733  for (unsigned I = 0, InsIndex = 0, E = ArgLocs.size(); I != E;1734       ++I, ++InsIndex) {1735    assert(InsIndex < Ins.size() && "Invalid Ins index");1736    CCValAssign &VA = ArgLocs[I];1737 1738    if (VA.isRegLoc()) {1739      EVT RegVT = VA.getLocVT();1740      if (VA.needsCustom()) {1741        assert(1742            VA.getValVT() == MVT::v64i1 &&1743            "Currently the only custom case is when we split v64i1 to 2 regs");1744 1745        // v64i1 values, in regcall calling convention, that are1746        // compiled to 32 bit arch, are split up into two registers.1747        ArgValue =1748            getv64i1Argument(VA, ArgLocs[++I], Chain, DAG, dl, Subtarget);1749      } else {1750        const TargetRegisterClass *RC;1751        if (RegVT == MVT::i8)1752          RC = &X86::GR8RegClass;1753        else if (RegVT == MVT::i16)1754          RC = &X86::GR16RegClass;1755        else if (RegVT == MVT::i32)1756          RC = &X86::GR32RegClass;1757        else if (Is64Bit && RegVT == MVT::i64)1758          RC = &X86::GR64RegClass;1759        else if (RegVT == MVT::f16)1760          RC = Subtarget.hasAVX512() ? &X86::FR16XRegClass : &X86::FR16RegClass;1761        else if (RegVT == MVT::f32)1762          RC = Subtarget.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass;1763        else if (RegVT == MVT::f64)1764          RC = Subtarget.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass;1765        else if (RegVT == MVT::f80)1766          RC = &X86::RFP80RegClass;1767        else if (RegVT == MVT::f128)1768          RC = &X86::VR128RegClass;1769        else if (RegVT.is512BitVector())1770          RC = &X86::VR512RegClass;1771        else if (RegVT.is256BitVector())1772          RC = Subtarget.hasVLX() ? &X86::VR256XRegClass : &X86::VR256RegClass;1773        else if (RegVT.is128BitVector())1774          RC = Subtarget.hasVLX() ? &X86::VR128XRegClass : &X86::VR128RegClass;1775        else if (RegVT == MVT::x86mmx)1776          RC = &X86::VR64RegClass;1777        else if (RegVT == MVT::v1i1)1778          RC = &X86::VK1RegClass;1779        else if (RegVT == MVT::v8i1)1780          RC = &X86::VK8RegClass;1781        else if (RegVT == MVT::v16i1)1782          RC = &X86::VK16RegClass;1783        else if (RegVT == MVT::v32i1)1784          RC = &X86::VK32RegClass;1785        else if (RegVT == MVT::v64i1)1786          RC = &X86::VK64RegClass;1787        else1788          llvm_unreachable("Unknown argument type!");1789 1790        Register Reg = MF.addLiveIn(VA.getLocReg(), RC);1791        ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT);1792      }1793 1794      // If this is an 8 or 16-bit value, it is really passed promoted to 321795      // bits.  Insert an assert[sz]ext to capture this, then truncate to the1796      // right size.1797      if (VA.getLocInfo() == CCValAssign::SExt)1798        ArgValue = DAG.getNode(ISD::AssertSext, dl, RegVT, ArgValue,1799                               DAG.getValueType(VA.getValVT()));1800      else if (VA.getLocInfo() == CCValAssign::ZExt)1801        ArgValue = DAG.getNode(ISD::AssertZext, dl, RegVT, ArgValue,1802                               DAG.getValueType(VA.getValVT()));1803      else if (VA.getLocInfo() == CCValAssign::BCvt)1804        ArgValue = DAG.getBitcast(VA.getValVT(), ArgValue);1805 1806      if (VA.isExtInLoc()) {1807        // Handle MMX values passed in XMM regs.1808        if (RegVT.isVector() && VA.getValVT().getScalarType() != MVT::i1)1809          ArgValue = DAG.getNode(X86ISD::MOVDQ2Q, dl, VA.getValVT(), ArgValue);1810        else if (VA.getValVT().isVector() &&1811                 VA.getValVT().getScalarType() == MVT::i1 &&1812                 ((VA.getLocVT() == MVT::i64) || (VA.getLocVT() == MVT::i32) ||1813                  (VA.getLocVT() == MVT::i16) || (VA.getLocVT() == MVT::i8))) {1814          // Promoting a mask type (v*i1) into a register of type i64/i32/i16/i81815          ArgValue = lowerRegToMasks(ArgValue, VA.getValVT(), RegVT, dl, DAG);1816        } else1817          ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue);1818      }1819    } else {1820      assert(VA.isMemLoc());1821      ArgValue =1822          LowerMemArgument(Chain, CallConv, Ins, dl, DAG, VA, MFI, InsIndex);1823    }1824 1825    // If value is passed via pointer - do a load.1826    if (VA.getLocInfo() == CCValAssign::Indirect &&1827        !(Ins[I].Flags.isByVal() && VA.isRegLoc())) {1828      ArgValue =1829          DAG.getLoad(VA.getValVT(), dl, Chain, ArgValue, MachinePointerInfo());1830    }1831 1832    InVals.push_back(ArgValue);1833  }1834 1835  for (unsigned I = 0, E = Ins.size(); I != E; ++I) {1836    if (Ins[I].Flags.isSwiftAsync()) {1837      auto X86FI = MF.getInfo<X86MachineFunctionInfo>();1838      if (X86::isExtendedSwiftAsyncFrameSupported(Subtarget, MF))1839        X86FI->setHasSwiftAsyncContext(true);1840      else {1841        int PtrSize = Subtarget.is64Bit() ? 8 : 4;1842        int FI =1843            MF.getFrameInfo().CreateStackObject(PtrSize, Align(PtrSize), false);1844        X86FI->setSwiftAsyncContextFrameIdx(FI);1845        SDValue St = DAG.getStore(1846            DAG.getEntryNode(), dl, InVals[I],1847            DAG.getFrameIndex(FI, PtrSize == 8 ? MVT::i64 : MVT::i32),1848            MachinePointerInfo::getFixedStack(MF, FI));1849        Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, St, Chain);1850      }1851    }1852 1853    // Swift calling convention does not require we copy the sret argument1854    // into %rax/%eax for the return. We don't set SRetReturnReg for Swift.1855    if (CallConv == CallingConv::Swift || CallConv == CallingConv::SwiftTail)1856      continue;1857 1858    // All x86 ABIs require that for returning structs by value we copy the1859    // sret argument into %rax/%eax (depending on ABI) for the return. Save1860    // the argument into a virtual register so that we can access it from the1861    // return points.1862    if (Ins[I].Flags.isSRet()) {1863      assert(!FuncInfo->getSRetReturnReg() &&1864             "SRet return has already been set");1865      MVT PtrTy = getPointerTy(DAG.getDataLayout());1866      Register Reg =1867          MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrTy));1868      FuncInfo->setSRetReturnReg(Reg);1869      SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), dl, Reg, InVals[I]);1870      Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Copy, Chain);1871      break;1872    }1873  }1874 1875  unsigned StackSize = CCInfo.getStackSize();1876  // Align stack specially for tail calls.1877  if (shouldGuaranteeTCO(CallConv,1878                         MF.getTarget().Options.GuaranteedTailCallOpt))1879    StackSize = GetAlignedArgumentStackSize(StackSize, DAG);1880 1881  if (IsVarArg)1882    VarArgsLoweringHelper(FuncInfo, dl, DAG, Subtarget, CallConv, CCInfo)1883        .lowerVarArgsParameters(Chain, StackSize);1884 1885  // Some CCs need callee pop.1886  if (X86::isCalleePop(CallConv, Is64Bit, IsVarArg,1887                       MF.getTarget().Options.GuaranteedTailCallOpt)) {1888    FuncInfo->setBytesToPopOnReturn(StackSize); // Callee pops everything.1889  } else if (CallConv == CallingConv::X86_INTR && Ins.size() == 2) {1890    // X86 interrupts must pop the error code (and the alignment padding) if1891    // present.1892    FuncInfo->setBytesToPopOnReturn(Is64Bit ? 16 : 4);1893  } else {1894    FuncInfo->setBytesToPopOnReturn(0); // Callee pops nothing.1895    // If this is an sret function, the return should pop the hidden pointer.1896    if (!canGuaranteeTCO(CallConv) && hasCalleePopSRet(Ins, Subtarget))1897      FuncInfo->setBytesToPopOnReturn(4);1898  }1899 1900  if (!Is64Bit) {1901    // RegSaveFrameIndex is X86-64 only.1902    FuncInfo->setRegSaveFrameIndex(0xAAAAAAA);1903  }1904 1905  FuncInfo->setArgumentStackSize(StackSize);1906 1907  if (WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo()) {1908    EHPersonality Personality = classifyEHPersonality(F.getPersonalityFn());1909    if (Personality == EHPersonality::CoreCLR) {1910      assert(Is64Bit);1911      // TODO: Add a mechanism to frame lowering that will allow us to indicate1912      // that we'd prefer this slot be allocated towards the bottom of the frame1913      // (i.e. near the stack pointer after allocating the frame).  Every1914      // funclet needs a copy of this slot in its (mostly empty) frame, and the1915      // offset from the bottom of this and each funclet's frame must be the1916      // same, so the size of funclets' (mostly empty) frames is dictated by1917      // how far this slot is from the bottom (since they allocate just enough1918      // space to accommodate holding this slot at the correct offset).1919      int PSPSymFI = MFI.CreateStackObject(8, Align(8), /*isSpillSlot=*/false);1920      EHInfo->PSPSymFrameIdx = PSPSymFI;1921    }1922  }1923 1924  if (shouldDisableArgRegFromCSR(CallConv) ||1925      F.hasFnAttribute("no_caller_saved_registers")) {1926    MachineRegisterInfo &MRI = MF.getRegInfo();1927    for (std::pair<MCRegister, Register> Pair : MRI.liveins())1928      MRI.disableCalleeSavedRegister(Pair.first);1929  }1930 1931  if (CallingConv::PreserveNone == CallConv)1932    for (const ISD::InputArg &In : Ins) {1933      if (In.Flags.isSwiftSelf() || In.Flags.isSwiftAsync() ||1934          In.Flags.isSwiftError()) {1935        errorUnsupported(DAG, dl,1936                         "Swift attributes can't be used with preserve_none");1937        break;1938      }1939    }1940 1941  return Chain;1942}1943 1944SDValue X86TargetLowering::LowerMemOpCallTo(SDValue Chain, SDValue StackPtr,1945                                            SDValue Arg, const SDLoc &dl,1946                                            SelectionDAG &DAG,1947                                            const CCValAssign &VA,1948                                            ISD::ArgFlagsTy Flags,1949                                            bool isByVal) const {1950  unsigned LocMemOffset = VA.getLocMemOffset();1951  SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);1952  PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),1953                       StackPtr, PtrOff);1954  if (isByVal)1955    return CreateCopyOfByValArgument(Arg, PtrOff, Chain, Flags, DAG, dl);1956 1957  MaybeAlign Alignment;1958  if (Subtarget.isTargetWindowsMSVC() && !Subtarget.is64Bit() &&1959      Arg.getSimpleValueType() != MVT::f80)1960    Alignment = MaybeAlign(4);1961  return DAG.getStore(1962      Chain, dl, Arg, PtrOff,1963      MachinePointerInfo::getStack(DAG.getMachineFunction(), LocMemOffset),1964      Alignment);1965}1966 1967/// Emit a load of return address if tail call1968/// optimization is performed and it is required.1969SDValue X86TargetLowering::EmitTailCallLoadRetAddr(1970    SelectionDAG &DAG, SDValue &OutRetAddr, SDValue Chain, bool IsTailCall,1971    bool Is64Bit, int FPDiff, const SDLoc &dl) const {1972  // Adjust the Return address stack slot.1973  EVT VT = getPointerTy(DAG.getDataLayout());1974  OutRetAddr = getReturnAddressFrameIndex(DAG);1975 1976  // Load the "old" Return address.1977  OutRetAddr = DAG.getLoad(VT, dl, Chain, OutRetAddr, MachinePointerInfo());1978  return SDValue(OutRetAddr.getNode(), 1);1979}1980 1981/// Emit a store of the return address if tail call1982/// optimization is performed and it is required (FPDiff!=0).1983static SDValue EmitTailCallStoreRetAddr(SelectionDAG &DAG, MachineFunction &MF,1984                                        SDValue Chain, SDValue RetAddrFrIdx,1985                                        EVT PtrVT, unsigned SlotSize,1986                                        int FPDiff, const SDLoc &dl) {1987  // Store the return address to the appropriate stack slot.1988  if (!FPDiff) return Chain;1989  // Calculate the new stack slot for the return address.1990  int NewReturnAddrFI =1991    MF.getFrameInfo().CreateFixedObject(SlotSize, (int64_t)FPDiff - SlotSize,1992                                         false);1993  SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, PtrVT);1994  Chain = DAG.getStore(Chain, dl, RetAddrFrIdx, NewRetAddrFrIdx,1995                       MachinePointerInfo::getFixedStack(1996                           DAG.getMachineFunction(), NewReturnAddrFI));1997  return Chain;1998}1999 2000/// Returns a vector_shuffle mask for an movs{s|d}, movd2001/// operation of specified width.2002SDValue X86TargetLowering::getMOVL(SelectionDAG &DAG, const SDLoc &dl, MVT VT,2003                                   SDValue V1, SDValue V2) const {2004  unsigned NumElems = VT.getVectorNumElements();2005  SmallVector<int, 8> Mask;2006  Mask.push_back(NumElems);2007  for (unsigned i = 1; i != NumElems; ++i)2008    Mask.push_back(i);2009  return DAG.getVectorShuffle(VT, dl, V1, V2, Mask);2010}2011 2012SDValue2013X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,2014                             SmallVectorImpl<SDValue> &InVals) const {2015  SelectionDAG &DAG                     = CLI.DAG;2016  SDLoc &dl                             = CLI.DL;2017  SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;2018  SmallVectorImpl<SDValue> &OutVals     = CLI.OutVals;2019  SmallVectorImpl<ISD::InputArg> &Ins   = CLI.Ins;2020  SDValue Chain                         = CLI.Chain;2021  SDValue Callee                        = CLI.Callee;2022  CallingConv::ID CallConv              = CLI.CallConv;2023  bool &isTailCall                      = CLI.IsTailCall;2024  bool isVarArg                         = CLI.IsVarArg;2025  const auto *CB                        = CLI.CB;2026 2027  MachineFunction &MF = DAG.getMachineFunction();2028  bool Is64Bit        = Subtarget.is64Bit();2029  bool IsWin64        = Subtarget.isCallingConvWin64(CallConv);2030  bool IsSibcall      = false;2031  bool IsGuaranteeTCO = MF.getTarget().Options.GuaranteedTailCallOpt ||2032      CallConv == CallingConv::Tail || CallConv == CallingConv::SwiftTail;2033  bool IsCalleePopSRet = !IsGuaranteeTCO && hasCalleePopSRet(Outs, Subtarget);2034  X86MachineFunctionInfo *X86Info = MF.getInfo<X86MachineFunctionInfo>();2035  bool HasNCSR = (CB && isa<CallInst>(CB) &&2036                  CB->hasFnAttr("no_caller_saved_registers"));2037  bool IsIndirectCall = (CB && isa<CallInst>(CB) && CB->isIndirectCall());2038  bool IsCFICall = IsIndirectCall && CLI.CFIType;2039  const Module *M = MF.getFunction().getParent();2040 2041  // If the indirect call target has the nocf_check attribute, the call needs2042  // the NOTRACK prefix. For simplicity just disable tail calls as there are2043  // so many variants.2044  bool IsNoTrackIndirectCall = IsIndirectCall && CB->doesNoCfCheck() &&2045                               M->getModuleFlag("cf-protection-branch");2046  if (IsNoTrackIndirectCall)2047    isTailCall = false;2048 2049  MachineFunction::CallSiteInfo CSInfo;2050  if (CallConv == CallingConv::X86_INTR)2051    report_fatal_error("X86 interrupts may not be called directly");2052 2053  // Set type id for call site info.2054  if (MF.getTarget().Options.EmitCallGraphSection && CB && CB->isIndirectCall())2055    CSInfo = MachineFunction::CallSiteInfo(*CB);2056 2057  if (IsIndirectCall && !IsWin64 &&2058      M->getModuleFlag("import-call-optimization"))2059    errorUnsupported(DAG, dl,2060                     "Indirect calls must have a normal calling convention if "2061                     "Import Call Optimization is enabled");2062 2063  // Analyze operands of the call, assigning locations to each operand.2064  SmallVector<CCValAssign, 16> ArgLocs;2065  CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext());2066 2067  // Allocate shadow area for Win64.2068  if (IsWin64)2069    CCInfo.AllocateStack(32, Align(8));2070 2071  CCInfo.AnalyzeArguments(Outs, CC_X86);2072 2073  // In vectorcall calling convention a second pass is required for the HVA2074  // types.2075  if (CallingConv::X86_VectorCall == CallConv) {2076    CCInfo.AnalyzeArgumentsSecondPass(Outs, CC_X86);2077  }2078 2079  bool IsMustTail = CLI.CB && CLI.CB->isMustTailCall();2080  if (Subtarget.isPICStyleGOT() && !IsGuaranteeTCO && !IsMustTail) {2081    // If we are using a GOT, disable tail calls to external symbols with2082    // default visibility. Tail calling such a symbol requires using a GOT2083    // relocation, which forces early binding of the symbol. This breaks code2084    // that require lazy function symbol resolution. Using musttail or2085    // GuaranteedTailCallOpt will override this.2086    GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee);2087    if (!G || (!G->getGlobal()->hasLocalLinkage() &&2088               G->getGlobal()->hasDefaultVisibility()))2089      isTailCall = false;2090  }2091 2092  if (isTailCall && !IsMustTail) {2093    // Check if it's really possible to do a tail call.2094    isTailCall = IsEligibleForTailCallOptimization(CLI, CCInfo, ArgLocs,2095                                                   IsCalleePopSRet);2096 2097    // Sibcalls are automatically detected tailcalls which do not require2098    // ABI changes.2099    if (!IsGuaranteeTCO && isTailCall)2100      IsSibcall = true;2101 2102    if (isTailCall)2103      ++NumTailCalls;2104  }2105 2106  if (IsMustTail && !isTailCall)2107    report_fatal_error("failed to perform tail call elimination on a call "2108                       "site marked musttail");2109 2110  assert(!(isVarArg && canGuaranteeTCO(CallConv)) &&2111         "Var args not supported with calling convention fastcc, ghc or hipe");2112 2113  // Get a count of how many bytes are to be pushed on the stack.2114  unsigned NumBytes = CCInfo.getAlignedCallFrameSize();2115  if (IsSibcall)2116    // This is a sibcall. The memory operands are available in caller's2117    // own caller's stack.2118    NumBytes = 0;2119  else if (IsGuaranteeTCO && canGuaranteeTCO(CallConv))2120    NumBytes = GetAlignedArgumentStackSize(NumBytes, DAG);2121 2122  int FPDiff = 0;2123  if (isTailCall &&2124      shouldGuaranteeTCO(CallConv,2125                         MF.getTarget().Options.GuaranteedTailCallOpt)) {2126    // Lower arguments at fp - stackoffset + fpdiff.2127    unsigned NumBytesCallerPushed = X86Info->getBytesToPopOnReturn();2128 2129    FPDiff = NumBytesCallerPushed - NumBytes;2130 2131    // Set the delta of movement of the returnaddr stackslot.2132    // But only set if delta is greater than previous delta.2133    if (FPDiff < X86Info->getTCReturnAddrDelta())2134      X86Info->setTCReturnAddrDelta(FPDiff);2135  }2136 2137  unsigned NumBytesToPush = NumBytes;2138  unsigned NumBytesToPop = NumBytes;2139 2140  // If we have an inalloca argument, all stack space has already been allocated2141  // for us and be right at the top of the stack.  We don't support multiple2142  // arguments passed in memory when using inalloca.2143  if (!Outs.empty() && Outs.back().Flags.isInAlloca()) {2144    NumBytesToPush = 0;2145    if (!ArgLocs.back().isMemLoc())2146      report_fatal_error("cannot use inalloca attribute on a register "2147                         "parameter");2148    if (ArgLocs.back().getLocMemOffset() != 0)2149      report_fatal_error("any parameter with the inalloca attribute must be "2150                         "the only memory argument");2151  } else if (CLI.IsPreallocated) {2152    assert(ArgLocs.back().isMemLoc() &&2153           "cannot use preallocated attribute on a register "2154           "parameter");2155    SmallVector<size_t, 4> PreallocatedOffsets;2156    for (size_t i = 0; i < CLI.OutVals.size(); ++i) {2157      if (CLI.CB->paramHasAttr(i, Attribute::Preallocated)) {2158        PreallocatedOffsets.push_back(ArgLocs[i].getLocMemOffset());2159      }2160    }2161    auto *MFI = DAG.getMachineFunction().getInfo<X86MachineFunctionInfo>();2162    size_t PreallocatedId = MFI->getPreallocatedIdForCallSite(CLI.CB);2163    MFI->setPreallocatedStackSize(PreallocatedId, NumBytes);2164    MFI->setPreallocatedArgOffsets(PreallocatedId, PreallocatedOffsets);2165    NumBytesToPush = 0;2166  }2167 2168  if (!IsSibcall && !IsMustTail)2169    Chain = DAG.getCALLSEQ_START(Chain, NumBytesToPush,2170                                 NumBytes - NumBytesToPush, dl);2171 2172  SDValue RetAddrFrIdx;2173  // Load return address for tail calls.2174  if (isTailCall && FPDiff)2175    Chain = EmitTailCallLoadRetAddr(DAG, RetAddrFrIdx, Chain, isTailCall,2176                                    Is64Bit, FPDiff, dl);2177 2178  SmallVector<std::pair<Register, SDValue>, 8> RegsToPass;2179  SmallVector<SDValue, 8> MemOpChains;2180  SDValue StackPtr;2181 2182  // The next loop assumes that the locations are in the same order of the2183  // input arguments.2184  assert(isSortedByValueNo(ArgLocs) &&2185         "Argument Location list must be sorted before lowering");2186 2187  // Walk the register/memloc assignments, inserting copies/loads.  In the case2188  // of tail call optimization arguments are handle later.2189  const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();2190  for (unsigned I = 0, OutIndex = 0, E = ArgLocs.size(); I != E;2191       ++I, ++OutIndex) {2192    assert(OutIndex < Outs.size() && "Invalid Out index");2193    // Skip inalloca/preallocated arguments, they have already been written.2194    ISD::ArgFlagsTy Flags = Outs[OutIndex].Flags;2195    if (Flags.isInAlloca() || Flags.isPreallocated())2196      continue;2197 2198    CCValAssign &VA = ArgLocs[I];2199    EVT RegVT = VA.getLocVT();2200    SDValue Arg = OutVals[OutIndex];2201    bool isByVal = Flags.isByVal();2202 2203    // Promote the value if needed.2204    switch (VA.getLocInfo()) {2205    default: llvm_unreachable("Unknown loc info!");2206    case CCValAssign::Full: break;2207    case CCValAssign::SExt:2208      Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, RegVT, Arg);2209      break;2210    case CCValAssign::ZExt:2211      Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, RegVT, Arg);2212      break;2213    case CCValAssign::AExt:2214      if (Arg.getValueType().isVector() &&2215          Arg.getValueType().getVectorElementType() == MVT::i1)2216        Arg = lowerMasksToReg(Arg, RegVT, dl, DAG);2217      else if (RegVT.is128BitVector()) {2218        // Special case: passing MMX values in XMM registers.2219        Arg = DAG.getBitcast(MVT::i64, Arg);2220        Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Arg);2221        Arg = getMOVL(DAG, dl, MVT::v2i64, DAG.getUNDEF(MVT::v2i64), Arg);2222      } else2223        Arg = DAG.getNode(ISD::ANY_EXTEND, dl, RegVT, Arg);2224      break;2225    case CCValAssign::BCvt:2226      Arg = DAG.getBitcast(RegVT, Arg);2227      break;2228    case CCValAssign::Indirect: {2229      if (isByVal) {2230        // Memcpy the argument to a temporary stack slot to prevent2231        // the caller from seeing any modifications the callee may make2232        // as guaranteed by the `byval` attribute.2233        int FrameIdx = MF.getFrameInfo().CreateStackObject(2234            Flags.getByValSize(),2235            std::max(Align(16), Flags.getNonZeroByValAlign()), false);2236        SDValue StackSlot =2237            DAG.getFrameIndex(FrameIdx, getPointerTy(DAG.getDataLayout()));2238        Chain =2239            CreateCopyOfByValArgument(Arg, StackSlot, Chain, Flags, DAG, dl);2240        // From now on treat this as a regular pointer2241        Arg = StackSlot;2242        isByVal = false;2243      } else {2244        // Store the argument.2245        SDValue SpillSlot = DAG.CreateStackTemporary(VA.getValVT());2246        int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();2247        Chain = DAG.getStore(2248            Chain, dl, Arg, SpillSlot,2249            MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI));2250        Arg = SpillSlot;2251      }2252      break;2253    }2254    }2255 2256    if (VA.needsCustom()) {2257      assert(VA.getValVT() == MVT::v64i1 &&2258             "Currently the only custom case is when we split v64i1 to 2 regs");2259      // Split v64i1 value into two registers2260      Passv64i1ArgInRegs(dl, DAG, Arg, RegsToPass, VA, ArgLocs[++I], Subtarget);2261    } else if (VA.isRegLoc()) {2262      RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));2263      const TargetOptions &Options = DAG.getTarget().Options;2264      if (Options.EmitCallSiteInfo)2265        CSInfo.ArgRegPairs.emplace_back(VA.getLocReg(), I);2266      if (isVarArg && IsWin64) {2267        // Win64 ABI requires argument XMM reg to be copied to the corresponding2268        // shadow reg if callee is a varargs function.2269        Register ShadowReg;2270        switch (VA.getLocReg()) {2271        case X86::XMM0: ShadowReg = X86::RCX; break;2272        case X86::XMM1: ShadowReg = X86::RDX; break;2273        case X86::XMM2: ShadowReg = X86::R8; break;2274        case X86::XMM3: ShadowReg = X86::R9; break;2275        }2276        if (ShadowReg)2277          RegsToPass.push_back(std::make_pair(ShadowReg, Arg));2278      }2279    } else if (!IsSibcall && (!isTailCall || isByVal)) {2280      assert(VA.isMemLoc());2281      if (!StackPtr.getNode())2282        StackPtr = DAG.getCopyFromReg(Chain, dl, RegInfo->getStackRegister(),2283                                      getPointerTy(DAG.getDataLayout()));2284      MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Arg,2285                                             dl, DAG, VA, Flags, isByVal));2286    }2287  }2288 2289  if (!MemOpChains.empty())2290    Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);2291 2292  if (Subtarget.isPICStyleGOT()) {2293    // ELF / PIC requires GOT in the EBX register before function calls via PLT2294    // GOT pointer (except regcall).2295    if (!isTailCall) {2296      // Indirect call with RegCall calling convertion may use up all the2297      // general registers, so it is not suitable to bind EBX reister for2298      // GOT address, just let register allocator handle it.2299      if (CallConv != CallingConv::X86_RegCall)2300        RegsToPass.push_back(std::make_pair(2301          Register(X86::EBX), DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(),2302                                          getPointerTy(DAG.getDataLayout()))));2303    } else {2304      // If we are tail calling and generating PIC/GOT style code load the2305      // address of the callee into ECX. The value in ecx is used as target of2306      // the tail jump. This is done to circumvent the ebx/callee-saved problem2307      // for tail calls on PIC/GOT architectures. Normally we would just put the2308      // address of GOT into ebx and then call target@PLT. But for tail calls2309      // ebx would be restored (since ebx is callee saved) before jumping to the2310      // target@PLT.2311 2312      // Note: The actual moving to ECX is done further down.2313      GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee);2314      if (G && !G->getGlobal()->hasLocalLinkage() &&2315          G->getGlobal()->hasDefaultVisibility())2316        Callee = LowerGlobalAddress(Callee, DAG);2317      else if (isa<ExternalSymbolSDNode>(Callee))2318        Callee = LowerExternalSymbol(Callee, DAG);2319    }2320  }2321 2322  if (Is64Bit && isVarArg && !IsWin64 && !IsMustTail &&2323      (Subtarget.hasSSE1() || !M->getModuleFlag("SkipRaxSetup"))) {2324    // From AMD64 ABI document:2325    // For calls that may call functions that use varargs or stdargs2326    // (prototype-less calls or calls to functions containing ellipsis (...) in2327    // the declaration) %al is used as hidden argument to specify the number2328    // of SSE registers used. The contents of %al do not need to match exactly2329    // the number of registers, but must be an ubound on the number of SSE2330    // registers used and is in the range 0 - 8 inclusive.2331 2332    // Count the number of XMM registers allocated.2333    static const MCPhysReg XMMArgRegs[] = {2334      X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,2335      X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM72336    };2337    unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs);2338    assert((Subtarget.hasSSE1() || !NumXMMRegs)2339           && "SSE registers cannot be used when SSE is disabled");2340    RegsToPass.push_back(std::make_pair(Register(X86::AL),2341                                        DAG.getConstant(NumXMMRegs, dl,2342                                                        MVT::i8)));2343  }2344 2345  if (isVarArg && IsMustTail) {2346    const auto &Forwards = X86Info->getForwardedMustTailRegParms();2347    for (const auto &F : Forwards) {2348      SDValue Val = DAG.getCopyFromReg(Chain, dl, F.VReg, F.VT);2349      RegsToPass.push_back(std::make_pair(F.PReg, Val));2350    }2351  }2352 2353  // For tail calls lower the arguments to the 'real' stack slots.  Sibcalls2354  // don't need this because the eligibility check rejects calls that require2355  // shuffling arguments passed in memory.2356  if (!IsSibcall && isTailCall) {2357    // Force all the incoming stack arguments to be loaded from the stack2358    // before any new outgoing arguments or the return address are stored to the2359    // stack, because the outgoing stack slots may alias the incoming argument2360    // stack slots, and the alias isn't otherwise explicit. This is slightly2361    // more conservative than necessary, because it means that each store2362    // effectively depends on every argument instead of just those arguments it2363    // would clobber.2364    Chain = DAG.getStackArgumentTokenFactor(Chain);2365 2366    SmallVector<SDValue, 8> MemOpChains2;2367    SDValue FIN;2368    int FI = 0;2369    for (unsigned I = 0, OutsIndex = 0, E = ArgLocs.size(); I != E;2370         ++I, ++OutsIndex) {2371      CCValAssign &VA = ArgLocs[I];2372 2373      if (VA.isRegLoc()) {2374        if (VA.needsCustom()) {2375          assert((CallConv == CallingConv::X86_RegCall) &&2376                 "Expecting custom case only in regcall calling convention");2377          // This means that we are in special case where one argument was2378          // passed through two register locations - Skip the next location2379          ++I;2380        }2381 2382        continue;2383      }2384 2385      assert(VA.isMemLoc());2386      SDValue Arg = OutVals[OutsIndex];2387      ISD::ArgFlagsTy Flags = Outs[OutsIndex].Flags;2388      // Skip inalloca/preallocated arguments.  They don't require any work.2389      if (Flags.isInAlloca() || Flags.isPreallocated())2390        continue;2391      // Create frame index.2392      int32_t Offset = VA.getLocMemOffset()+FPDiff;2393      uint32_t OpSize = (VA.getLocVT().getSizeInBits()+7)/8;2394      FI = MF.getFrameInfo().CreateFixedObject(OpSize, Offset, true);2395      FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));2396 2397      if (Flags.isByVal()) {2398        // Copy relative to framepointer.2399        SDValue Source = DAG.getIntPtrConstant(VA.getLocMemOffset(), dl);2400        if (!StackPtr.getNode())2401          StackPtr = DAG.getCopyFromReg(Chain, dl, RegInfo->getStackRegister(),2402                                        getPointerTy(DAG.getDataLayout()));2403        Source = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),2404                             StackPtr, Source);2405 2406        MemOpChains2.push_back(2407            CreateCopyOfByValArgument(Source, FIN, Chain, Flags, DAG, dl));2408      } else {2409        // Store relative to framepointer.2410        MemOpChains2.push_back(DAG.getStore(2411            Chain, dl, Arg, FIN,2412            MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI)));2413      }2414    }2415 2416    if (!MemOpChains2.empty())2417      Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains2);2418 2419    // Store the return address to the appropriate stack slot.2420    Chain = EmitTailCallStoreRetAddr(DAG, MF, Chain, RetAddrFrIdx,2421                                     getPointerTy(DAG.getDataLayout()),2422                                     RegInfo->getSlotSize(), FPDiff, dl);2423  }2424 2425  // Build a sequence of copy-to-reg nodes chained together with token chain2426  // and glue operands which copy the outgoing args into registers.2427  SDValue InGlue;2428  for (const auto &[Reg, N] : RegsToPass) {2429    Chain = DAG.getCopyToReg(Chain, dl, Reg, N, InGlue);2430    InGlue = Chain.getValue(1);2431  }2432 2433  bool IsImpCall = false;2434  if (DAG.getTarget().getCodeModel() == CodeModel::Large) {2435    assert(Is64Bit && "Large code model is only legal in 64-bit mode.");2436    // In the 64-bit large code model, we have to make all calls2437    // through a register, since the call instruction's 32-bit2438    // pc-relative offset may not be large enough to hold the whole2439    // address.2440  } else if (Callee->getOpcode() == ISD::GlobalAddress ||2441             Callee->getOpcode() == ISD::ExternalSymbol) {2442    // Lower direct calls to global addresses and external symbols. Setting2443    // ForCall to true here has the effect of removing WrapperRIP when possible2444    // to allow direct calls to be selected without first materializing the2445    // address into a register.2446    Callee = LowerGlobalOrExternal(Callee, DAG, /*ForCall=*/true, &IsImpCall);2447  } else if (Subtarget.isTarget64BitILP32() &&2448             Callee.getValueType() == MVT::i32) {2449    // Zero-extend the 32-bit Callee address into a 64-bit according to x32 ABI2450    Callee = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Callee);2451  }2452 2453  SmallVector<SDValue, 8> Ops;2454 2455  if (!IsSibcall && isTailCall && !IsMustTail) {2456    Chain = DAG.getCALLSEQ_END(Chain, NumBytesToPop, 0, InGlue, dl);2457    InGlue = Chain.getValue(1);2458  }2459 2460  Ops.push_back(Chain);2461  Ops.push_back(Callee);2462 2463  if (isTailCall)2464    Ops.push_back(DAG.getSignedTargetConstant(FPDiff, dl, MVT::i32));2465 2466  // Add argument registers to the end of the list so that they are known live2467  // into the call.2468  for (const auto &[Reg, N] : RegsToPass)2469    Ops.push_back(DAG.getRegister(Reg, N.getValueType()));2470 2471  // Add a register mask operand representing the call-preserved registers.2472  const uint32_t *Mask = [&]() {2473    auto AdaptedCC = CallConv;2474    // If HasNCSR is asserted (attribute NoCallerSavedRegisters exists),2475    // use X86_INTR calling convention because it has the same CSR mask2476    // (same preserved registers).2477    if (HasNCSR)2478      AdaptedCC = (CallingConv::ID)CallingConv::X86_INTR;2479    // If NoCalleeSavedRegisters is requested, than use GHC since it happens2480    // to use the CSR_NoRegs_RegMask.2481    if (CB && CB->hasFnAttr("no_callee_saved_registers"))2482      AdaptedCC = (CallingConv::ID)CallingConv::GHC;2483    return RegInfo->getCallPreservedMask(MF, AdaptedCC);2484  }();2485  assert(Mask && "Missing call preserved mask for calling convention");2486 2487  if (MachineOperand::clobbersPhysReg(Mask, RegInfo->getFramePtr())) {2488    X86Info->setFPClobberedByCall(true);2489    if (CLI.CB && isa<InvokeInst>(CLI.CB))2490      X86Info->setFPClobberedByInvoke(true);2491  }2492  if (MachineOperand::clobbersPhysReg(Mask, RegInfo->getBaseRegister())) {2493    X86Info->setBPClobberedByCall(true);2494    if (CLI.CB && isa<InvokeInst>(CLI.CB))2495      X86Info->setBPClobberedByInvoke(true);2496  }2497 2498  // If this is an invoke in a 32-bit function using a funclet-based2499  // personality, assume the function clobbers all registers. If an exception2500  // is thrown, the runtime will not restore CSRs.2501  // FIXME: Model this more precisely so that we can register allocate across2502  // the normal edge and spill and fill across the exceptional edge.2503  if (!Is64Bit && CLI.CB && isa<InvokeInst>(CLI.CB)) {2504    const Function &CallerFn = MF.getFunction();2505    EHPersonality Pers =2506        CallerFn.hasPersonalityFn()2507            ? classifyEHPersonality(CallerFn.getPersonalityFn())2508            : EHPersonality::Unknown;2509    if (isFuncletEHPersonality(Pers))2510      Mask = RegInfo->getNoPreservedMask();2511  }2512 2513  // Define a new register mask from the existing mask.2514  uint32_t *RegMask = nullptr;2515 2516  // In some calling conventions we need to remove the used physical registers2517  // from the reg mask. Create a new RegMask for such calling conventions.2518  // RegMask for calling conventions that disable only return registers (e.g.2519  // preserve_most) will be modified later in LowerCallResult.2520  bool ShouldDisableArgRegs = shouldDisableArgRegFromCSR(CallConv) || HasNCSR;2521  if (ShouldDisableArgRegs || shouldDisableRetRegFromCSR(CallConv)) {2522    const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();2523 2524    // Allocate a new Reg Mask and copy Mask.2525    RegMask = MF.allocateRegMask();2526    unsigned RegMaskSize = MachineOperand::getRegMaskSize(TRI->getNumRegs());2527    memcpy(RegMask, Mask, sizeof(RegMask[0]) * RegMaskSize);2528 2529    // Make sure all sub registers of the argument registers are reset2530    // in the RegMask.2531    if (ShouldDisableArgRegs) {2532      for (auto const &RegPair : RegsToPass)2533        for (MCPhysReg SubReg : TRI->subregs_inclusive(RegPair.first))2534          RegMask[SubReg / 32] &= ~(1u << (SubReg % 32));2535    }2536 2537    // Create the RegMask Operand according to our updated mask.2538    Ops.push_back(DAG.getRegisterMask(RegMask));2539  } else {2540    // Create the RegMask Operand according to the static mask.2541    Ops.push_back(DAG.getRegisterMask(Mask));2542  }2543 2544  if (InGlue.getNode())2545    Ops.push_back(InGlue);2546 2547  if (isTailCall) {2548    // We used to do:2549    //// If this is the first return lowered for this function, add the regs2550    //// to the liveout set for the function.2551    // This isn't right, although it's probably harmless on x86; liveouts2552    // should be computed from returns not tail calls.  Consider a void2553    // function making a tail call to a function returning int.2554    MF.getFrameInfo().setHasTailCall();2555    SDValue Ret = DAG.getNode(X86ISD::TC_RETURN, dl, MVT::Other, Ops);2556 2557    if (IsCFICall)2558      Ret.getNode()->setCFIType(CLI.CFIType->getZExtValue());2559 2560    DAG.addNoMergeSiteInfo(Ret.getNode(), CLI.NoMerge);2561    DAG.addCallSiteInfo(Ret.getNode(), std::move(CSInfo));2562    return Ret;2563  }2564 2565  // Returns a chain & a glue for retval copy to use.2566  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);2567  if (IsImpCall) {2568    Chain = DAG.getNode(X86ISD::IMP_CALL, dl, NodeTys, Ops);2569  } else if (IsNoTrackIndirectCall) {2570    Chain = DAG.getNode(X86ISD::NT_CALL, dl, NodeTys, Ops);2571  } else if (CLI.CB && objcarc::hasAttachedCallOpBundle(CLI.CB)) {2572    // Calls with a "clang.arc.attachedcall" bundle are special. They should be2573    // expanded to the call, directly followed by a special marker sequence and2574    // a call to a ObjC library function. Use the CALL_RVMARKER to do that.2575    assert(!isTailCall &&2576           "tail calls cannot be marked with clang.arc.attachedcall");2577    assert(Is64Bit && "clang.arc.attachedcall is only supported in 64bit mode");2578 2579    // Add a target global address for the retainRV/claimRV runtime function2580    // just before the call target.2581    Function *ARCFn = *objcarc::getAttachedARCFunction(CLI.CB);2582    auto PtrVT = getPointerTy(DAG.getDataLayout());2583    auto GA = DAG.getTargetGlobalAddress(ARCFn, dl, PtrVT);2584    Ops.insert(Ops.begin() + 1, GA);2585    Chain = DAG.getNode(X86ISD::CALL_RVMARKER, dl, NodeTys, Ops);2586  } else {2587    Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, Ops);2588  }2589 2590  if (IsCFICall)2591    Chain.getNode()->setCFIType(CLI.CFIType->getZExtValue());2592 2593  InGlue = Chain.getValue(1);2594  DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);2595  DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo));2596 2597  // Save heapallocsite metadata.2598  if (CLI.CB)2599    if (MDNode *HeapAlloc = CLI.CB->getMetadata("heapallocsite"))2600      DAG.addHeapAllocSite(Chain.getNode(), HeapAlloc);2601 2602  // Create the CALLSEQ_END node.2603  unsigned NumBytesForCalleeToPop = 0; // Callee pops nothing.2604  if (X86::isCalleePop(CallConv, Is64Bit, isVarArg,2605                       DAG.getTarget().Options.GuaranteedTailCallOpt))2606    NumBytesForCalleeToPop = NumBytes;    // Callee pops everything2607  else if (!canGuaranteeTCO(CallConv) && IsCalleePopSRet)2608    // If this call passes a struct-return pointer, the callee2609    // pops that struct pointer.2610    NumBytesForCalleeToPop = 4;2611 2612  // Returns a glue for retval copy to use.2613  if (!IsSibcall) {2614    Chain = DAG.getCALLSEQ_END(Chain, NumBytesToPop, NumBytesForCalleeToPop,2615                               InGlue, dl);2616    InGlue = Chain.getValue(1);2617  }2618 2619  if (CallingConv::PreserveNone == CallConv)2620    for (const ISD::OutputArg &Out : Outs) {2621      if (Out.Flags.isSwiftSelf() || Out.Flags.isSwiftAsync() ||2622          Out.Flags.isSwiftError()) {2623        errorUnsupported(DAG, dl,2624                         "Swift attributes can't be used with preserve_none");2625        break;2626      }2627    }2628 2629  // Handle result values, copying them out of physregs into vregs that we2630  // return.2631  return LowerCallResult(Chain, InGlue, CallConv, isVarArg, Ins, dl, DAG,2632                         InVals, RegMask);2633}2634 2635//===----------------------------------------------------------------------===//2636//                Fast Calling Convention (tail call) implementation2637//===----------------------------------------------------------------------===//2638 2639//  Like std call, callee cleans arguments, convention except that ECX is2640//  reserved for storing the tail called function address. Only 2 registers are2641//  free for argument passing (inreg). Tail call optimization is performed2642//  provided:2643//                * tailcallopt is enabled2644//                * caller/callee are fastcc2645//  On X86_64 architecture with GOT-style position independent code only local2646//  (within module) calls are supported at the moment.2647//  To keep the stack aligned according to platform abi the function2648//  GetAlignedArgumentStackSize ensures that argument delta is always multiples2649//  of stack alignment. (Dynamic linkers need this - Darwin's dyld for example)2650//  If a tail called function callee has more arguments than the caller the2651//  caller needs to make sure that there is room to move the RETADDR to. This is2652//  achieved by reserving an area the size of the argument delta right after the2653//  original RETADDR, but before the saved framepointer or the spilled registers2654//  e.g. caller(arg1, arg2) calls callee(arg1, arg2,arg3,arg4)2655//  stack layout:2656//    arg12657//    arg22658//    RETADDR2659//    [ new RETADDR2660//      move area ]2661//    (possible EBP)2662//    ESI2663//    EDI2664//    local1 ..2665 2666/// Make the stack size align e.g 16n + 12 aligned for a 16-byte align2667/// requirement.2668unsigned2669X86TargetLowering::GetAlignedArgumentStackSize(const unsigned StackSize,2670                                               SelectionDAG &DAG) const {2671  const Align StackAlignment = Subtarget.getFrameLowering()->getStackAlign();2672  const uint64_t SlotSize = Subtarget.getRegisterInfo()->getSlotSize();2673  assert(StackSize % SlotSize == 0 &&2674         "StackSize must be a multiple of SlotSize");2675  return alignTo(StackSize + SlotSize, StackAlignment) - SlotSize;2676}2677 2678/// Return true if the given stack call argument is already available in the2679/// same position (relatively) of the caller's incoming argument stack.2680static2681bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags,2682                         MachineFrameInfo &MFI, const MachineRegisterInfo *MRI,2683                         const X86InstrInfo *TII, const CCValAssign &VA) {2684  unsigned Bytes = Arg.getValueSizeInBits() / 8;2685 2686  for (;;) {2687    // Look through nodes that don't alter the bits of the incoming value.2688    unsigned Op = Arg.getOpcode();2689    if (Op == ISD::ZERO_EXTEND || Op == ISD::ANY_EXTEND || Op == ISD::BITCAST ||2690        Op == ISD::AssertZext) {2691      Arg = Arg.getOperand(0);2692      continue;2693    }2694    if (Op == ISD::TRUNCATE) {2695      const SDValue &TruncInput = Arg.getOperand(0);2696      if (TruncInput.getOpcode() == ISD::AssertZext &&2697          cast<VTSDNode>(TruncInput.getOperand(1))->getVT() ==2698              Arg.getValueType()) {2699        Arg = TruncInput.getOperand(0);2700        continue;2701      }2702    }2703    break;2704  }2705 2706  int FI = INT_MAX;2707  if (Arg.getOpcode() == ISD::CopyFromReg) {2708    Register VR = cast<RegisterSDNode>(Arg.getOperand(1))->getReg();2709    if (!VR.isVirtual())2710      return false;2711    MachineInstr *Def = MRI->getVRegDef(VR);2712    if (!Def)2713      return false;2714    if (!Flags.isByVal()) {2715      if (!TII->isLoadFromStackSlot(*Def, FI))2716        return false;2717    } else {2718      unsigned Opcode = Def->getOpcode();2719      if ((Opcode == X86::LEA32r || Opcode == X86::LEA64r ||2720           Opcode == X86::LEA64_32r) &&2721          Def->getOperand(1).isFI()) {2722        FI = Def->getOperand(1).getIndex();2723        Bytes = Flags.getByValSize();2724      } else2725        return false;2726    }2727  } else if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Arg)) {2728    if (Flags.isByVal())2729      // ByVal argument is passed in as a pointer but it's now being2730      // dereferenced. e.g.2731      // define @foo(%struct.X* %A) {2732      //   tail call @bar(%struct.X* byval %A)2733      // }2734      return false;2735    SDValue Ptr = Ld->getBasePtr();2736    FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr);2737    if (!FINode)2738      return false;2739    FI = FINode->getIndex();2740  } else if (Arg.getOpcode() == ISD::FrameIndex && Flags.isByVal()) {2741    FrameIndexSDNode *FINode = cast<FrameIndexSDNode>(Arg);2742    FI = FINode->getIndex();2743    Bytes = Flags.getByValSize();2744  } else2745    return false;2746 2747  assert(FI != INT_MAX);2748  if (!MFI.isFixedObjectIndex(FI))2749    return false;2750 2751  if (Offset != MFI.getObjectOffset(FI))2752    return false;2753 2754  // If this is not byval, check that the argument stack object is immutable.2755  // inalloca and argument copy elision can create mutable argument stack2756  // objects. Byval objects can be mutated, but a byval call intends to pass the2757  // mutated memory.2758  if (!Flags.isByVal() && !MFI.isImmutableObjectIndex(FI))2759    return false;2760 2761  if (VA.getLocVT().getFixedSizeInBits() >2762      Arg.getValueSizeInBits().getFixedValue()) {2763    // If the argument location is wider than the argument type, check that any2764    // extension flags match.2765    if (Flags.isZExt() != MFI.isObjectZExt(FI) ||2766        Flags.isSExt() != MFI.isObjectSExt(FI)) {2767      return false;2768    }2769  }2770 2771  return Bytes == MFI.getObjectSize(FI);2772}2773 2774static bool2775mayBeSRetTailCallCompatible(const TargetLowering::CallLoweringInfo &CLI,2776                            Register CallerSRetReg) {2777  const auto &Outs = CLI.Outs;2778  const auto &OutVals = CLI.OutVals;2779 2780  // We know the caller has a sret pointer argument (CallerSRetReg). Locate the2781  // operand index within the callee that may have a sret pointer too.2782  unsigned Pos = 0;2783  for (unsigned E = Outs.size(); Pos != E; ++Pos)2784    if (Outs[Pos].Flags.isSRet())2785      break;2786  // Bail out if the callee has not any sret argument.2787  if (Pos == Outs.size())2788    return false;2789 2790  // At this point, either the caller is forwarding its sret argument to the2791  // callee, or the callee is being passed a different sret pointer. We now look2792  // for a CopyToReg, where the callee sret argument is written into a new vreg2793  // (which should later be %rax/%eax, if this is returned).2794  SDValue SRetArgVal = OutVals[Pos];2795  for (SDNode *User : SRetArgVal->users()) {2796    if (User->getOpcode() != ISD::CopyToReg)2797      continue;2798    Register Reg = cast<RegisterSDNode>(User->getOperand(1))->getReg();2799    if (Reg == CallerSRetReg && User->getOperand(2) == SRetArgVal)2800      return true;2801  }2802 2803  return false;2804}2805 2806/// Check whether the call is eligible for tail call optimization. Targets2807/// that want to do tail call optimization should implement this function.2808/// Note that the x86 backend does not check musttail calls for eligibility! The2809/// rest of x86 tail call lowering must be prepared to forward arguments of any2810/// type.2811bool X86TargetLowering::IsEligibleForTailCallOptimization(2812    TargetLowering::CallLoweringInfo &CLI, CCState &CCInfo,2813    SmallVectorImpl<CCValAssign> &ArgLocs, bool IsCalleePopSRet) const {2814  SelectionDAG &DAG = CLI.DAG;2815  const SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;2816  const SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;2817  const SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;2818  SDValue Callee = CLI.Callee;2819  CallingConv::ID CalleeCC = CLI.CallConv;2820  bool isVarArg = CLI.IsVarArg;2821 2822  if (!mayTailCallThisCC(CalleeCC))2823    return false;2824 2825  // If -tailcallopt is specified, make fastcc functions tail-callable.2826  MachineFunction &MF = DAG.getMachineFunction();2827  X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();2828  const Function &CallerF = MF.getFunction();2829 2830  // If the function return type is x86_fp80 and the callee return type is not,2831  // then the FP_EXTEND of the call result is not a nop. It's not safe to2832  // perform a tailcall optimization here.2833  if (CallerF.getReturnType()->isX86_FP80Ty() && !CLI.RetTy->isX86_FP80Ty())2834    return false;2835 2836  CallingConv::ID CallerCC = CallerF.getCallingConv();2837  bool CCMatch = CallerCC == CalleeCC;2838  bool IsCalleeWin64 = Subtarget.isCallingConvWin64(CalleeCC);2839  bool IsCallerWin64 = Subtarget.isCallingConvWin64(CallerCC);2840  bool IsGuaranteeTCO = DAG.getTarget().Options.GuaranteedTailCallOpt ||2841      CalleeCC == CallingConv::Tail || CalleeCC == CallingConv::SwiftTail;2842 2843  // Win64 functions have extra shadow space for argument homing. Don't do the2844  // sibcall if the caller and callee have mismatched expectations for this2845  // space.2846  if (IsCalleeWin64 != IsCallerWin64)2847    return false;2848 2849  if (IsGuaranteeTCO) {2850    if (canGuaranteeTCO(CalleeCC) && CCMatch)2851      return true;2852    return false;2853  }2854 2855  // Look for obvious safe cases to perform tail call optimization that do not2856  // require ABI changes. This is what gcc calls sibcall.2857 2858  // Can't do sibcall if stack needs to be dynamically re-aligned. PEI needs to2859  // emit a special epilogue.2860  const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();2861  if (RegInfo->hasStackRealignment(MF))2862    return false;2863 2864  // Avoid sibcall optimization if we are an sret return function and the callee2865  // is incompatible, unless such premises are proven wrong. See comment in2866  // LowerReturn about why hasStructRetAttr is insufficient.2867  if (Register SRetReg = FuncInfo->getSRetReturnReg()) {2868    // For a compatible tail call the callee must return our sret pointer. So it2869    // needs to be (a) an sret function itself and (b) we pass our sret as its2870    // sret. Condition #b is harder to determine.2871    if (!mayBeSRetTailCallCompatible(CLI, SRetReg))2872      return false;2873  } else if (IsCalleePopSRet)2874    // The callee pops an sret, so we cannot tail-call, as our caller doesn't2875    // expect that.2876    return false;2877 2878  // Do not sibcall optimize vararg calls unless all arguments are passed via2879  // registers.2880  LLVMContext &C = *DAG.getContext();2881  if (isVarArg && !Outs.empty()) {2882    // Optimizing for varargs on Win64 is unlikely to be safe without2883    // additional testing.2884    if (IsCalleeWin64 || IsCallerWin64)2885      return false;2886 2887    for (const auto &VA : ArgLocs)2888      if (!VA.isRegLoc())2889        return false;2890  }2891 2892  // If the call result is in ST0 / ST1, it needs to be popped off the x872893  // stack.  Therefore, if it's not used by the call it is not safe to optimize2894  // this into a sibcall.2895  bool Unused = false;2896  for (const auto &In : Ins) {2897    if (!In.Used) {2898      Unused = true;2899      break;2900    }2901  }2902  if (Unused) {2903    SmallVector<CCValAssign, 16> RVLocs;2904    CCState RVCCInfo(CalleeCC, false, MF, RVLocs, C);2905    RVCCInfo.AnalyzeCallResult(Ins, RetCC_X86);2906    for (const auto &VA : RVLocs) {2907      if (VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1)2908        return false;2909    }2910  }2911 2912  // Check that the call results are passed in the same way.2913  if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, C, Ins,2914                                  RetCC_X86, RetCC_X86))2915    return false;2916  // The callee has to preserve all registers the caller needs to preserve.2917  const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();2918  const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);2919  if (!CCMatch) {2920    const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);2921    if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))2922      return false;2923  }2924 2925  // The stack frame of the caller cannot be replaced by the tail-callee one's2926  // if the function is required to preserve all the registers. Conservatively2927  // prevent tail optimization even if hypothetically all the registers are used2928  // for passing formal parameters or returning values.2929  if (CallerF.hasFnAttribute("no_caller_saved_registers"))2930    return false;2931 2932  unsigned StackArgsSize = CCInfo.getStackSize();2933 2934  // If the callee takes no arguments then go on to check the results of the2935  // call.2936  if (!Outs.empty()) {2937    if (StackArgsSize > 0) {2938      // Check if the arguments are already laid out in the right way as2939      // the caller's fixed stack objects.2940      MachineFrameInfo &MFI = MF.getFrameInfo();2941      const MachineRegisterInfo *MRI = &MF.getRegInfo();2942      const X86InstrInfo *TII = Subtarget.getInstrInfo();2943      for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {2944        const CCValAssign &VA = ArgLocs[I];2945        SDValue Arg = OutVals[I];2946        ISD::ArgFlagsTy Flags = Outs[I].Flags;2947        if (VA.getLocInfo() == CCValAssign::Indirect)2948          return false;2949        if (!VA.isRegLoc()) {2950          if (!MatchingStackOffset(Arg, VA.getLocMemOffset(), Flags, MFI, MRI,2951                                   TII, VA))2952            return false;2953        }2954      }2955    }2956 2957    bool PositionIndependent = isPositionIndependent();2958    // If the tailcall address may be in a register, then make sure it's2959    // possible to register allocate for it. In 32-bit, the call address can2960    // only target EAX, EDX, or ECX since the tail call must be scheduled after2961    // callee-saved registers are restored. These happen to be the same2962    // registers used to pass 'inreg' arguments so watch out for those.2963    if (!Subtarget.is64Bit() && ((!isa<GlobalAddressSDNode>(Callee) &&2964                                  !isa<ExternalSymbolSDNode>(Callee)) ||2965                                 PositionIndependent)) {2966      unsigned NumInRegs = 0;2967      // In PIC we need an extra register to formulate the address computation2968      // for the callee.2969      unsigned MaxInRegs = PositionIndependent ? 2 : 3;2970 2971      for (const auto &VA : ArgLocs) {2972        if (!VA.isRegLoc())2973          continue;2974        Register Reg = VA.getLocReg();2975        switch (Reg) {2976        default: break;2977        case X86::EAX: case X86::EDX: case X86::ECX:2978          if (++NumInRegs == MaxInRegs)2979            return false;2980          break;2981        }2982      }2983    }2984 2985    const MachineRegisterInfo &MRI = MF.getRegInfo();2986    if (!parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals))2987      return false;2988  }2989 2990  bool CalleeWillPop =2991      X86::isCalleePop(CalleeCC, Subtarget.is64Bit(), isVarArg,2992                       MF.getTarget().Options.GuaranteedTailCallOpt);2993 2994  if (unsigned BytesToPop = FuncInfo->getBytesToPopOnReturn()) {2995    // If we have bytes to pop, the callee must pop them.2996    bool CalleePopMatches = CalleeWillPop && BytesToPop == StackArgsSize;2997    if (!CalleePopMatches)2998      return false;2999  } else if (CalleeWillPop && StackArgsSize > 0) {3000    // If we don't have bytes to pop, make sure the callee doesn't pop any.3001    return false;3002  }3003 3004  return true;3005}3006 3007/// Determines whether the callee is required to pop its own arguments.3008/// Callee pop is necessary to support tail calls.3009bool X86::isCalleePop(CallingConv::ID CallingConv,3010                      bool is64Bit, bool IsVarArg, bool GuaranteeTCO) {3011  // If GuaranteeTCO is true, we force some calls to be callee pop so that we3012  // can guarantee TCO.3013  if (!IsVarArg && shouldGuaranteeTCO(CallingConv, GuaranteeTCO))3014    return true;3015 3016  switch (CallingConv) {3017  default:3018    return false;3019  case CallingConv::X86_StdCall:3020  case CallingConv::X86_FastCall:3021  case CallingConv::X86_ThisCall:3022  case CallingConv::X86_VectorCall:3023    return !is64Bit;3024  }3025}3026