1406 lines · cpp
1//===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8///9/// \file10/// This file implements some simple delegations needed for call lowering.11///12//===----------------------------------------------------------------------===//13 14#include "llvm/CodeGen/GlobalISel/CallLowering.h"15#include "llvm/CodeGen/Analysis.h"16#include "llvm/CodeGen/CallingConvLower.h"17#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"18#include "llvm/CodeGen/GlobalISel/Utils.h"19#include "llvm/CodeGen/MachineFrameInfo.h"20#include "llvm/CodeGen/MachineOperand.h"21#include "llvm/CodeGen/MachineRegisterInfo.h"22#include "llvm/CodeGen/TargetLowering.h"23#include "llvm/IR/DataLayout.h"24#include "llvm/IR/LLVMContext.h"25#include "llvm/IR/Module.h"26#include "llvm/Target/TargetMachine.h"27 28#define DEBUG_TYPE "call-lowering"29 30using namespace llvm;31 32void CallLowering::anchor() {}33 34/// Helper function which updates \p Flags when \p AttrFn returns true.35static void36addFlagsUsingAttrFn(ISD::ArgFlagsTy &Flags,37 const std::function<bool(Attribute::AttrKind)> &AttrFn) {38 // TODO: There are missing flags. Add them here.39 if (AttrFn(Attribute::SExt))40 Flags.setSExt();41 if (AttrFn(Attribute::ZExt))42 Flags.setZExt();43 if (AttrFn(Attribute::InReg))44 Flags.setInReg();45 if (AttrFn(Attribute::StructRet))46 Flags.setSRet();47 if (AttrFn(Attribute::Nest))48 Flags.setNest();49 if (AttrFn(Attribute::ByVal))50 Flags.setByVal();51 if (AttrFn(Attribute::ByRef))52 Flags.setByRef();53 if (AttrFn(Attribute::Preallocated))54 Flags.setPreallocated();55 if (AttrFn(Attribute::InAlloca))56 Flags.setInAlloca();57 if (AttrFn(Attribute::Returned))58 Flags.setReturned();59 if (AttrFn(Attribute::SwiftSelf))60 Flags.setSwiftSelf();61 if (AttrFn(Attribute::SwiftAsync))62 Flags.setSwiftAsync();63 if (AttrFn(Attribute::SwiftError))64 Flags.setSwiftError();65}66 67ISD::ArgFlagsTy CallLowering::getAttributesForArgIdx(const CallBase &Call,68 unsigned ArgIdx) const {69 ISD::ArgFlagsTy Flags;70 addFlagsUsingAttrFn(Flags, [&Call, &ArgIdx](Attribute::AttrKind Attr) {71 return Call.paramHasAttr(ArgIdx, Attr);72 });73 return Flags;74}75 76ISD::ArgFlagsTy77CallLowering::getAttributesForReturn(const CallBase &Call) const {78 ISD::ArgFlagsTy Flags;79 addFlagsUsingAttrFn(Flags, [&Call](Attribute::AttrKind Attr) {80 return Call.hasRetAttr(Attr);81 });82 return Flags;83}84 85void CallLowering::addArgFlagsFromAttributes(ISD::ArgFlagsTy &Flags,86 const AttributeList &Attrs,87 unsigned OpIdx) const {88 addFlagsUsingAttrFn(Flags, [&Attrs, &OpIdx](Attribute::AttrKind Attr) {89 return Attrs.hasAttributeAtIndex(OpIdx, Attr);90 });91}92 93bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,94 ArrayRef<Register> ResRegs,95 ArrayRef<ArrayRef<Register>> ArgRegs,96 Register SwiftErrorVReg,97 std::optional<PtrAuthInfo> PAI,98 Register ConvergenceCtrlToken,99 std::function<Register()> GetCalleeReg) const {100 CallLoweringInfo Info;101 const DataLayout &DL = MIRBuilder.getDataLayout();102 MachineFunction &MF = MIRBuilder.getMF();103 MachineRegisterInfo &MRI = MF.getRegInfo();104 bool CanBeTailCalled = CB.isTailCall() &&105 isInTailCallPosition(CB, MF.getTarget()) &&106 (MF.getFunction()107 .getFnAttribute("disable-tail-calls")108 .getValueAsString() != "true");109 110 CallingConv::ID CallConv = CB.getCallingConv();111 Type *RetTy = CB.getType();112 bool IsVarArg = CB.getFunctionType()->isVarArg();113 114 SmallVector<BaseArgInfo, 4> SplitArgs;115 getReturnInfo(CallConv, RetTy, CB.getAttributes(), SplitArgs, DL);116 Info.CanLowerReturn = canLowerReturn(MF, CallConv, SplitArgs, IsVarArg);117 118 Info.IsConvergent = CB.isConvergent();119 120 if (!Info.CanLowerReturn) {121 // Callee requires sret demotion.122 insertSRetOutgoingArgument(MIRBuilder, CB, Info);123 124 // The sret demotion isn't compatible with tail-calls, since the sret125 // argument points into the caller's stack frame.126 CanBeTailCalled = false;127 }128 129 // First step is to marshall all the function's parameters into the correct130 // physregs and memory locations. Gather the sequence of argument types that131 // we'll pass to the assigner function.132 unsigned i = 0;133 unsigned NumFixedArgs = CB.getFunctionType()->getNumParams();134 for (const auto &Arg : CB.args()) {135 ArgInfo OrigArg{ArgRegs[i], *Arg.get(), i, getAttributesForArgIdx(CB, i)};136 setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, CB);137 if (i >= NumFixedArgs)138 OrigArg.Flags[0].setVarArg();139 140 // If we have an explicit sret argument that is an Instruction, (i.e., it141 // might point to function-local memory), we can't meaningfully tail-call.142 if (OrigArg.Flags[0].isSRet() && isa<Instruction>(&Arg))143 CanBeTailCalled = false;144 145 Info.OrigArgs.push_back(OrigArg);146 ++i;147 }148 149 // Try looking through a bitcast from one function type to another.150 // Commonly happens with calls to objc_msgSend().151 const Value *CalleeV = CB.getCalledOperand()->stripPointerCasts();152 153 // If IRTranslator chose to drop the ptrauth info, we can turn this into154 // a direct call.155 if (!PAI && CB.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {156 CalleeV = cast<ConstantPtrAuth>(CalleeV)->getPointer();157 assert(isa<Function>(CalleeV));158 }159 160 if (const Function *F = dyn_cast<Function>(CalleeV)) {161 if (F->hasFnAttribute(Attribute::NonLazyBind)) {162 LLT Ty = getLLTForType(*F->getType(), DL);163 Register Reg = MIRBuilder.buildGlobalValue(Ty, F).getReg(0);164 Info.Callee = MachineOperand::CreateReg(Reg, false);165 } else {166 Info.Callee = MachineOperand::CreateGA(F, 0);167 }168 } else if (isa<GlobalIFunc>(CalleeV) || isa<GlobalAlias>(CalleeV)) {169 // IR IFuncs and Aliases can't be forward declared (only defined), so the170 // callee must be in the same TU and therefore we can direct-call it without171 // worrying about it being out of range.172 Info.Callee = MachineOperand::CreateGA(cast<GlobalValue>(CalleeV), 0);173 } else174 Info.Callee = MachineOperand::CreateReg(GetCalleeReg(), false);175 176 Register ReturnHintAlignReg;177 Align ReturnHintAlign;178 179 Info.OrigRet = ArgInfo{ResRegs, RetTy, 0, getAttributesForReturn(CB)};180 181 if (!Info.OrigRet.Ty->isVoidTy()) {182 setArgFlags(Info.OrigRet, AttributeList::ReturnIndex, DL, CB);183 184 if (MaybeAlign Alignment = CB.getRetAlign()) {185 if (*Alignment > Align(1)) {186 ReturnHintAlignReg = MRI.cloneVirtualRegister(ResRegs[0]);187 Info.OrigRet.Regs[0] = ReturnHintAlignReg;188 ReturnHintAlign = *Alignment;189 }190 }191 }192 193 auto Bundle = CB.getOperandBundle(LLVMContext::OB_kcfi);194 if (Bundle && CB.isIndirectCall()) {195 Info.CFIType = cast<ConstantInt>(Bundle->Inputs[0]);196 assert(Info.CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");197 }198 199 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_deactivation_symbol)) {200 Info.DeactivationSymbol = cast<GlobalValue>(Bundle->Inputs[0]);201 }202 203 Info.CB = &CB;204 Info.KnownCallees = CB.getMetadata(LLVMContext::MD_callees);205 Info.CallConv = CallConv;206 Info.SwiftErrorVReg = SwiftErrorVReg;207 Info.PAI = PAI;208 Info.ConvergenceCtrlToken = ConvergenceCtrlToken;209 Info.IsMustTailCall = CB.isMustTailCall();210 Info.IsTailCall = CanBeTailCalled;211 Info.IsVarArg = IsVarArg;212 if (!lowerCall(MIRBuilder, Info))213 return false;214 215 if (ReturnHintAlignReg && !Info.LoweredTailCall) {216 MIRBuilder.buildAssertAlign(ResRegs[0], ReturnHintAlignReg,217 ReturnHintAlign);218 }219 220 return true;221}222 223template <typename FuncInfoTy>224void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx,225 const DataLayout &DL,226 const FuncInfoTy &FuncInfo) const {227 auto &Flags = Arg.Flags[0];228 const AttributeList &Attrs = FuncInfo.getAttributes();229 addArgFlagsFromAttributes(Flags, Attrs, OpIdx);230 231 PointerType *PtrTy = dyn_cast<PointerType>(Arg.Ty->getScalarType());232 if (PtrTy) {233 Flags.setPointer();234 Flags.setPointerAddrSpace(PtrTy->getPointerAddressSpace());235 }236 237 Align MemAlign = DL.getABITypeAlign(Arg.Ty);238 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||239 Flags.isByRef()) {240 assert(OpIdx >= AttributeList::FirstArgIndex);241 unsigned ParamIdx = OpIdx - AttributeList::FirstArgIndex;242 243 Type *ElementTy = FuncInfo.getParamByValType(ParamIdx);244 if (!ElementTy)245 ElementTy = FuncInfo.getParamByRefType(ParamIdx);246 if (!ElementTy)247 ElementTy = FuncInfo.getParamInAllocaType(ParamIdx);248 if (!ElementTy)249 ElementTy = FuncInfo.getParamPreallocatedType(ParamIdx);250 251 assert(ElementTy && "Must have byval, inalloca or preallocated type");252 253 uint64_t MemSize = DL.getTypeAllocSize(ElementTy);254 if (Flags.isByRef())255 Flags.setByRefSize(MemSize);256 else257 Flags.setByValSize(MemSize);258 259 // For ByVal, alignment should be passed from FE. BE will guess if260 // this info is not there but there are cases it cannot get right.261 if (auto ParamAlign = FuncInfo.getParamStackAlign(ParamIdx))262 MemAlign = *ParamAlign;263 else if ((ParamAlign = FuncInfo.getParamAlign(ParamIdx)))264 MemAlign = *ParamAlign;265 else266 MemAlign = getTLI()->getByValTypeAlignment(ElementTy, DL);267 } else if (OpIdx >= AttributeList::FirstArgIndex) {268 if (auto ParamAlign =269 FuncInfo.getParamStackAlign(OpIdx - AttributeList::FirstArgIndex))270 MemAlign = *ParamAlign;271 }272 Flags.setMemAlign(MemAlign);273 Flags.setOrigAlign(DL.getABITypeAlign(Arg.Ty));274 275 // Don't try to use the returned attribute if the argument is marked as276 // swiftself, since it won't be passed in x0.277 if (Flags.isSwiftSelf())278 Flags.setReturned(false);279}280 281template void282CallLowering::setArgFlags<Function>(CallLowering::ArgInfo &Arg, unsigned OpIdx,283 const DataLayout &DL,284 const Function &FuncInfo) const;285 286template void287CallLowering::setArgFlags<CallBase>(CallLowering::ArgInfo &Arg, unsigned OpIdx,288 const DataLayout &DL,289 const CallBase &FuncInfo) const;290 291void CallLowering::splitToValueTypes(const ArgInfo &OrigArg,292 SmallVectorImpl<ArgInfo> &SplitArgs,293 const DataLayout &DL,294 CallingConv::ID CallConv,295 SmallVectorImpl<uint64_t> *Offsets) const {296 LLVMContext &Ctx = OrigArg.Ty->getContext();297 298 SmallVector<EVT, 4> SplitVTs;299 ComputeValueVTs(*TLI, DL, OrigArg.Ty, SplitVTs, /*MemVTs=*/nullptr, Offsets,300 0);301 302 if (SplitVTs.size() == 0)303 return;304 305 if (SplitVTs.size() == 1) {306 // No splitting to do, but we want to replace the original type (e.g. [1 x307 // double] -> double).308 SplitArgs.emplace_back(OrigArg.Regs[0], SplitVTs[0].getTypeForEVT(Ctx),309 OrigArg.OrigArgIndex, OrigArg.Flags[0],310 OrigArg.OrigValue);311 return;312 }313 314 // Create one ArgInfo for each virtual register in the original ArgInfo.315 assert(OrigArg.Regs.size() == SplitVTs.size() && "Regs / types mismatch");316 317 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(318 OrigArg.Ty, CallConv, false, DL);319 for (unsigned i = 0, e = SplitVTs.size(); i < e; ++i) {320 Type *SplitTy = SplitVTs[i].getTypeForEVT(Ctx);321 SplitArgs.emplace_back(OrigArg.Regs[i], SplitTy, OrigArg.OrigArgIndex,322 OrigArg.Flags[0]);323 if (NeedsRegBlock)324 SplitArgs.back().Flags[0].setInConsecutiveRegs();325 }326 327 SplitArgs.back().Flags[0].setInConsecutiveRegsLast();328}329 330/// Pack values \p SrcRegs to cover the vector type result \p DstRegs.331static MachineInstrBuilder332mergeVectorRegsToResultRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,333 ArrayRef<Register> SrcRegs) {334 MachineRegisterInfo &MRI = *B.getMRI();335 LLT LLTy = MRI.getType(DstRegs[0]);336 LLT PartLLT = MRI.getType(SrcRegs[0]);337 338 // Deal with v3s16 split into v2s16339 LLT LCMTy = getCoverTy(LLTy, PartLLT);340 if (LCMTy == LLTy) {341 // Common case where no padding is needed.342 assert(DstRegs.size() == 1);343 return B.buildConcatVectors(DstRegs[0], SrcRegs);344 }345 346 // We need to create an unmerge to the result registers, which may require347 // widening the original value.348 Register UnmergeSrcReg;349 if (LCMTy != PartLLT) {350 assert(DstRegs.size() == 1);351 return B.buildDeleteTrailingVectorElements(352 DstRegs[0], B.buildMergeLikeInstr(LCMTy, SrcRegs));353 } else {354 // We don't need to widen anything if we're extracting a scalar which was355 // promoted to a vector e.g. s8 -> v4s8 -> s8356 assert(SrcRegs.size() == 1);357 UnmergeSrcReg = SrcRegs[0];358 }359 360 int NumDst = LCMTy.getSizeInBits() / LLTy.getSizeInBits();361 362 SmallVector<Register, 8> PadDstRegs(NumDst);363 llvm::copy(DstRegs, PadDstRegs.begin());364 365 // Create the excess dead defs for the unmerge.366 for (int I = DstRegs.size(); I != NumDst; ++I)367 PadDstRegs[I] = MRI.createGenericVirtualRegister(LLTy);368 369 if (PadDstRegs.size() == 1)370 return B.buildDeleteTrailingVectorElements(DstRegs[0], UnmergeSrcReg);371 return B.buildUnmerge(PadDstRegs, UnmergeSrcReg);372}373 374/// Create a sequence of instructions to combine pieces split into register375/// typed values to the original IR value. \p OrigRegs contains the destination376/// value registers of type \p LLTy, and \p Regs contains the legalized pieces377/// with type \p PartLLT. This is used for incoming values (physregs to vregs).378static void buildCopyFromRegs(MachineIRBuilder &B, ArrayRef<Register> OrigRegs,379 ArrayRef<Register> Regs, LLT LLTy, LLT PartLLT,380 const ISD::ArgFlagsTy Flags) {381 MachineRegisterInfo &MRI = *B.getMRI();382 383 if (PartLLT == LLTy) {384 // We should have avoided introducing a new virtual register, and just385 // directly assigned here.386 assert(OrigRegs[0] == Regs[0]);387 return;388 }389 390 if (PartLLT.getSizeInBits() == LLTy.getSizeInBits() && OrigRegs.size() == 1 &&391 Regs.size() == 1) {392 B.buildBitcast(OrigRegs[0], Regs[0]);393 return;394 }395 396 // A vector PartLLT needs extending to LLTy's element size.397 // E.g. <2 x s64> = G_SEXT <2 x s32>.398 if (PartLLT.isVector() == LLTy.isVector() &&399 PartLLT.getScalarSizeInBits() > LLTy.getScalarSizeInBits() &&400 (!PartLLT.isVector() ||401 PartLLT.getElementCount() == LLTy.getElementCount()) &&402 OrigRegs.size() == 1 && Regs.size() == 1) {403 Register SrcReg = Regs[0];404 405 LLT LocTy = MRI.getType(SrcReg);406 407 if (Flags.isSExt()) {408 SrcReg = B.buildAssertSExt(LocTy, SrcReg, LLTy.getScalarSizeInBits())409 .getReg(0);410 } else if (Flags.isZExt()) {411 SrcReg = B.buildAssertZExt(LocTy, SrcReg, LLTy.getScalarSizeInBits())412 .getReg(0);413 }414 415 // Sometimes pointers are passed zero extended.416 LLT OrigTy = MRI.getType(OrigRegs[0]);417 if (OrigTy.isPointer()) {418 LLT IntPtrTy = LLT::scalar(OrigTy.getSizeInBits());419 B.buildIntToPtr(OrigRegs[0], B.buildTrunc(IntPtrTy, SrcReg));420 return;421 }422 423 B.buildTrunc(OrigRegs[0], SrcReg);424 return;425 }426 427 if (!LLTy.isVector() && !PartLLT.isVector()) {428 assert(OrigRegs.size() == 1);429 LLT OrigTy = MRI.getType(OrigRegs[0]);430 431 unsigned SrcSize = PartLLT.getSizeInBits().getFixedValue() * Regs.size();432 if (SrcSize == OrigTy.getSizeInBits())433 B.buildMergeValues(OrigRegs[0], Regs);434 else {435 auto Widened = B.buildMergeLikeInstr(LLT::scalar(SrcSize), Regs);436 B.buildTrunc(OrigRegs[0], Widened);437 }438 439 return;440 }441 442 if (PartLLT.isVector()) {443 assert(OrigRegs.size() == 1);444 SmallVector<Register> CastRegs(Regs);445 446 // If PartLLT is a mismatched vector in both number of elements and element447 // size, e.g. PartLLT == v2s64 and LLTy is v3s32, then first coerce it to448 // have the same elt type, i.e. v4s32.449 // TODO: Extend this coersion to element multiples other than just 2.450 if (TypeSize::isKnownGT(PartLLT.getSizeInBits(), LLTy.getSizeInBits()) &&451 PartLLT.getScalarSizeInBits() == LLTy.getScalarSizeInBits() * 2 &&452 Regs.size() == 1) {453 LLT NewTy = PartLLT.changeElementType(LLTy.getElementType())454 .changeElementCount(PartLLT.getElementCount() * 2);455 CastRegs[0] = B.buildBitcast(NewTy, Regs[0]).getReg(0);456 PartLLT = NewTy;457 }458 459 if (LLTy.getScalarType() == PartLLT.getElementType()) {460 mergeVectorRegsToResultRegs(B, OrigRegs, CastRegs);461 } else {462 unsigned I = 0;463 LLT GCDTy = getGCDType(LLTy, PartLLT);464 465 // We are both splitting a vector, and bitcasting its element types. Cast466 // the source pieces into the appropriate number of pieces with the result467 // element type.468 for (Register SrcReg : CastRegs)469 CastRegs[I++] = B.buildBitcast(GCDTy, SrcReg).getReg(0);470 mergeVectorRegsToResultRegs(B, OrigRegs, CastRegs);471 }472 473 return;474 }475 476 assert(LLTy.isVector() && !PartLLT.isVector());477 478 LLT DstEltTy = LLTy.getElementType();479 480 // Pointer information was discarded. We'll need to coerce some register types481 // to avoid violating type constraints.482 LLT RealDstEltTy = MRI.getType(OrigRegs[0]).getElementType();483 484 assert(DstEltTy.getSizeInBits() == RealDstEltTy.getSizeInBits());485 486 if (DstEltTy == PartLLT) {487 // Vector was trivially scalarized.488 489 if (RealDstEltTy.isPointer()) {490 for (Register Reg : Regs)491 MRI.setType(Reg, RealDstEltTy);492 }493 494 B.buildBuildVector(OrigRegs[0], Regs);495 } else if (DstEltTy.getSizeInBits() > PartLLT.getSizeInBits()) {496 // Deal with vector with 64-bit elements decomposed to 32-bit497 // registers. Need to create intermediate 64-bit elements.498 SmallVector<Register, 8> EltMerges;499 int PartsPerElt =500 divideCeil(DstEltTy.getSizeInBits(), PartLLT.getSizeInBits());501 LLT ExtendedPartTy = LLT::scalar(PartLLT.getSizeInBits() * PartsPerElt);502 503 for (int I = 0, NumElts = LLTy.getNumElements(); I != NumElts; ++I) {504 auto Merge =505 B.buildMergeLikeInstr(ExtendedPartTy, Regs.take_front(PartsPerElt));506 if (ExtendedPartTy.getSizeInBits() > RealDstEltTy.getSizeInBits())507 Merge = B.buildTrunc(RealDstEltTy, Merge);508 // Fix the type in case this is really a vector of pointers.509 MRI.setType(Merge.getReg(0), RealDstEltTy);510 EltMerges.push_back(Merge.getReg(0));511 Regs = Regs.drop_front(PartsPerElt);512 }513 514 B.buildBuildVector(OrigRegs[0], EltMerges);515 } else {516 // Vector was split, and elements promoted to a wider type.517 // FIXME: Should handle floating point promotions.518 unsigned NumElts = LLTy.getNumElements();519 LLT BVType = LLT::fixed_vector(NumElts, PartLLT);520 521 Register BuildVec;522 if (NumElts == Regs.size())523 BuildVec = B.buildBuildVector(BVType, Regs).getReg(0);524 else {525 // Vector elements are packed in the inputs.526 // e.g. we have a <4 x s16> but 2 x s32 in regs.527 assert(NumElts > Regs.size());528 LLT SrcEltTy = MRI.getType(Regs[0]);529 530 LLT OriginalEltTy = MRI.getType(OrigRegs[0]).getElementType();531 532 // Input registers contain packed elements.533 // Determine how many elements per reg.534 assert((SrcEltTy.getSizeInBits() % OriginalEltTy.getSizeInBits()) == 0);535 unsigned EltPerReg =536 (SrcEltTy.getSizeInBits() / OriginalEltTy.getSizeInBits());537 538 SmallVector<Register, 0> BVRegs;539 BVRegs.reserve(Regs.size() * EltPerReg);540 for (Register R : Regs) {541 auto Unmerge = B.buildUnmerge(OriginalEltTy, R);542 for (unsigned K = 0; K < EltPerReg; ++K)543 BVRegs.push_back(B.buildAnyExt(PartLLT, Unmerge.getReg(K)).getReg(0));544 }545 546 // We may have some more elements in BVRegs, e.g. if we have 2 s32 pieces547 // for a <3 x s16> vector. We should have less than EltPerReg extra items.548 if (BVRegs.size() > NumElts) {549 assert((BVRegs.size() - NumElts) < EltPerReg);550 BVRegs.truncate(NumElts);551 }552 BuildVec = B.buildBuildVector(BVType, BVRegs).getReg(0);553 }554 B.buildTrunc(OrigRegs[0], BuildVec);555 }556}557 558/// Create a sequence of instructions to expand the value in \p SrcReg (of type559/// \p SrcTy) to the types in \p DstRegs (of type \p PartTy). \p ExtendOp should560/// contain the type of scalar value extension if necessary.561///562/// This is used for outgoing values (vregs to physregs)563static void buildCopyToRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,564 Register SrcReg, LLT SrcTy, LLT PartTy,565 unsigned ExtendOp = TargetOpcode::G_ANYEXT) {566 // We could just insert a regular copy, but this is unreachable at the moment.567 assert(SrcTy != PartTy && "identical part types shouldn't reach here");568 569 const TypeSize PartSize = PartTy.getSizeInBits();570 571 if (PartTy.isVector() == SrcTy.isVector() &&572 PartTy.getScalarSizeInBits() > SrcTy.getScalarSizeInBits()) {573 assert(DstRegs.size() == 1);574 B.buildInstr(ExtendOp, {DstRegs[0]}, {SrcReg});575 return;576 }577 578 if (SrcTy.isVector() && !PartTy.isVector() &&579 TypeSize::isKnownGT(PartSize, SrcTy.getElementType().getSizeInBits())) {580 // Vector was scalarized, and the elements extended.581 auto UnmergeToEltTy = B.buildUnmerge(SrcTy.getElementType(), SrcReg);582 for (int i = 0, e = DstRegs.size(); i != e; ++i)583 B.buildAnyExt(DstRegs[i], UnmergeToEltTy.getReg(i));584 return;585 }586 587 if (SrcTy.isVector() && PartTy.isVector() &&588 PartTy.getSizeInBits() == SrcTy.getSizeInBits() &&589 ElementCount::isKnownLT(SrcTy.getElementCount(),590 PartTy.getElementCount())) {591 // A coercion like: v2f32 -> v4f32 or nxv2f32 -> nxv4f32592 Register DstReg = DstRegs.front();593 B.buildPadVectorWithUndefElements(DstReg, SrcReg);594 return;595 }596 597 LLT GCDTy = getGCDType(SrcTy, PartTy);598 if (GCDTy == PartTy) {599 // If this already evenly divisible, we can create a simple unmerge.600 B.buildUnmerge(DstRegs, SrcReg);601 return;602 }603 604 if (SrcTy.isVector() && !PartTy.isVector() &&605 SrcTy.getScalarSizeInBits() > PartTy.getSizeInBits()) {606 LLT ExtTy =607 LLT::vector(SrcTy.getElementCount(),608 LLT::scalar(PartTy.getScalarSizeInBits() * DstRegs.size() /609 SrcTy.getNumElements()));610 auto Ext = B.buildAnyExt(ExtTy, SrcReg);611 B.buildUnmerge(DstRegs, Ext);612 return;613 }614 615 MachineRegisterInfo &MRI = *B.getMRI();616 LLT DstTy = MRI.getType(DstRegs[0]);617 LLT LCMTy = getCoverTy(SrcTy, PartTy);618 619 if (PartTy.isVector() && LCMTy == PartTy) {620 assert(DstRegs.size() == 1);621 B.buildPadVectorWithUndefElements(DstRegs[0], SrcReg);622 return;623 }624 625 const unsigned DstSize = DstTy.getSizeInBits();626 const unsigned SrcSize = SrcTy.getSizeInBits();627 unsigned CoveringSize = LCMTy.getSizeInBits();628 629 Register UnmergeSrc = SrcReg;630 631 if (!LCMTy.isVector() && CoveringSize != SrcSize) {632 // For scalars, it's common to be able to use a simple extension.633 if (SrcTy.isScalar() && DstTy.isScalar()) {634 CoveringSize = alignTo(SrcSize, DstSize);635 LLT CoverTy = LLT::scalar(CoveringSize);636 UnmergeSrc = B.buildInstr(ExtendOp, {CoverTy}, {SrcReg}).getReg(0);637 } else {638 // Widen to the common type.639 // FIXME: This should respect the extend type640 Register Undef = B.buildUndef(SrcTy).getReg(0);641 SmallVector<Register, 8> MergeParts(1, SrcReg);642 for (unsigned Size = SrcSize; Size != CoveringSize; Size += SrcSize)643 MergeParts.push_back(Undef);644 UnmergeSrc = B.buildMergeLikeInstr(LCMTy, MergeParts).getReg(0);645 }646 }647 648 if (LCMTy.isVector() && CoveringSize != SrcSize)649 UnmergeSrc = B.buildPadVectorWithUndefElements(LCMTy, SrcReg).getReg(0);650 651 B.buildUnmerge(DstRegs, UnmergeSrc);652}653 654bool CallLowering::determineAndHandleAssignments(655 ValueHandler &Handler, ValueAssigner &Assigner,656 SmallVectorImpl<ArgInfo> &Args, MachineIRBuilder &MIRBuilder,657 CallingConv::ID CallConv, bool IsVarArg,658 ArrayRef<Register> ThisReturnRegs) const {659 MachineFunction &MF = MIRBuilder.getMF();660 const Function &F = MF.getFunction();661 SmallVector<CCValAssign, 16> ArgLocs;662 663 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, F.getContext());664 if (!determineAssignments(Assigner, Args, CCInfo))665 return false;666 667 return handleAssignments(Handler, Args, CCInfo, ArgLocs, MIRBuilder,668 ThisReturnRegs);669}670 671static unsigned extendOpFromFlags(llvm::ISD::ArgFlagsTy Flags) {672 if (Flags.isSExt())673 return TargetOpcode::G_SEXT;674 if (Flags.isZExt())675 return TargetOpcode::G_ZEXT;676 return TargetOpcode::G_ANYEXT;677}678 679bool CallLowering::determineAssignments(ValueAssigner &Assigner,680 SmallVectorImpl<ArgInfo> &Args,681 CCState &CCInfo) const {682 LLVMContext &Ctx = CCInfo.getContext();683 const CallingConv::ID CallConv = CCInfo.getCallingConv();684 685 unsigned NumArgs = Args.size();686 for (unsigned i = 0; i != NumArgs; ++i) {687 EVT CurVT = EVT::getEVT(Args[i].Ty);688 689 MVT NewVT = TLI->getRegisterTypeForCallingConv(Ctx, CallConv, CurVT);690 691 // If we need to split the type over multiple regs, check it's a scenario692 // we currently support.693 unsigned NumParts =694 TLI->getNumRegistersForCallingConv(Ctx, CallConv, CurVT);695 696 if (NumParts == 1) {697 // Try to use the register type if we couldn't assign the VT.698 if (Assigner.assignArg(i, CurVT, NewVT, NewVT, CCValAssign::Full, Args[i],699 Args[i].Flags[0], CCInfo))700 return false;701 continue;702 }703 704 // For incoming arguments (physregs to vregs), we could have values in705 // physregs (or memlocs) which we want to extract and copy to vregs.706 // During this, we might have to deal with the LLT being split across707 // multiple regs, so we have to record this information for later.708 //709 // If we have outgoing args, then we have the opposite case. We have a710 // vreg with an LLT which we want to assign to a physical location, and711 // we might have to record that the value has to be split later.712 713 // We're handling an incoming arg which is split over multiple regs.714 // E.g. passing an s128 on AArch64.715 ISD::ArgFlagsTy OrigFlags = Args[i].Flags[0];716 Args[i].Flags.clear();717 718 for (unsigned Part = 0; Part < NumParts; ++Part) {719 ISD::ArgFlagsTy Flags = OrigFlags;720 if (Part == 0) {721 Flags.setSplit();722 } else {723 Flags.setOrigAlign(Align(1));724 if (Part == NumParts - 1)725 Flags.setSplitEnd();726 }727 728 Args[i].Flags.push_back(Flags);729 if (Assigner.assignArg(i, CurVT, NewVT, NewVT, CCValAssign::Full, Args[i],730 Args[i].Flags[Part], CCInfo)) {731 // Still couldn't assign this smaller part type for some reason.732 return false;733 }734 }735 }736 737 return true;738}739 740bool CallLowering::handleAssignments(ValueHandler &Handler,741 SmallVectorImpl<ArgInfo> &Args,742 CCState &CCInfo,743 SmallVectorImpl<CCValAssign> &ArgLocs,744 MachineIRBuilder &MIRBuilder,745 ArrayRef<Register> ThisReturnRegs) const {746 MachineFunction &MF = MIRBuilder.getMF();747 MachineRegisterInfo &MRI = MF.getRegInfo();748 const Function &F = MF.getFunction();749 const DataLayout &DL = F.getDataLayout();750 751 const unsigned NumArgs = Args.size();752 753 // Stores thunks for outgoing register assignments. This is used so we delay754 // generating register copies until mem loc assignments are done. We do this755 // so that if the target is using the delayed stack protector feature, we can756 // find the split point of the block accurately. E.g. if we have:757 // G_STORE %val, %memloc758 // $x0 = COPY %foo759 // $x1 = COPY %bar760 // CALL func761 // ... then the split point for the block will correctly be at, and including,762 // the copy to $x0. If instead the G_STORE instruction immediately precedes763 // the CALL, then we'd prematurely choose the CALL as the split point, thus764 // generating a split block with a CALL that uses undefined physregs.765 SmallVector<std::function<void()>> DelayedOutgoingRegAssignments;766 767 for (unsigned i = 0, j = 0; i != NumArgs; ++i, ++j) {768 assert(j < ArgLocs.size() && "Skipped too many arg locs");769 CCValAssign &VA = ArgLocs[j];770 assert(VA.getValNo() == i && "Location doesn't correspond to current arg");771 772 if (VA.needsCustom()) {773 std::function<void()> Thunk;774 unsigned NumArgRegs = Handler.assignCustomValue(775 Args[i], ArrayRef(ArgLocs).slice(j), &Thunk);776 if (Thunk)777 DelayedOutgoingRegAssignments.emplace_back(Thunk);778 if (!NumArgRegs)779 return false;780 j += (NumArgRegs - 1);781 continue;782 }783 784 auto AllocaAddressSpace = MF.getDataLayout().getAllocaAddrSpace();785 786 const MVT ValVT = VA.getValVT();787 const MVT LocVT = VA.getLocVT();788 789 const LLT LocTy(LocVT);790 const LLT ValTy(ValVT);791 const LLT NewLLT = Handler.isIncomingArgumentHandler() ? LocTy : ValTy;792 const EVT OrigVT = EVT::getEVT(Args[i].Ty);793 const LLT OrigTy = getLLTForType(*Args[i].Ty, DL);794 const LLT PointerTy = LLT::pointer(795 AllocaAddressSpace, DL.getPointerSizeInBits(AllocaAddressSpace));796 797 // Expected to be multiple regs for a single incoming arg.798 // There should be Regs.size() ArgLocs per argument.799 // This should be the same as getNumRegistersForCallingConv800 const unsigned NumParts = Args[i].Flags.size();801 802 // Now split the registers into the assigned types.803 Args[i].OrigRegs.assign(Args[i].Regs.begin(), Args[i].Regs.end());804 805 if (NumParts != 1 || NewLLT != OrigTy) {806 // If we can't directly assign the register, we need one or more807 // intermediate values.808 Args[i].Regs.resize(NumParts);809 810 // When we have indirect parameter passing we are receiving a pointer,811 // that points to the actual value, so we need one "temporary" pointer.812 if (VA.getLocInfo() == CCValAssign::Indirect) {813 if (Handler.isIncomingArgumentHandler())814 Args[i].Regs[0] = MRI.createGenericVirtualRegister(PointerTy);815 } else {816 // For each split register, create and assign a vreg that will store817 // the incoming component of the larger value. These will later be818 // merged to form the final vreg.819 for (unsigned Part = 0; Part < NumParts; ++Part)820 Args[i].Regs[Part] = MRI.createGenericVirtualRegister(NewLLT);821 }822 }823 824 assert((j + (NumParts - 1)) < ArgLocs.size() &&825 "Too many regs for number of args");826 827 // Coerce into outgoing value types before register assignment.828 if (!Handler.isIncomingArgumentHandler() && OrigTy != ValTy &&829 VA.getLocInfo() != CCValAssign::Indirect) {830 assert(Args[i].OrigRegs.size() == 1);831 buildCopyToRegs(MIRBuilder, Args[i].Regs, Args[i].OrigRegs[0], OrigTy,832 ValTy, extendOpFromFlags(Args[i].Flags[0]));833 }834 835 bool IndirectParameterPassingHandled = false;836 bool BigEndianPartOrdering = TLI->hasBigEndianPartOrdering(OrigVT, DL);837 for (unsigned Part = 0; Part < NumParts; ++Part) {838 assert((VA.getLocInfo() != CCValAssign::Indirect || Part == 0) &&839 "Only the first parameter should be processed when "840 "handling indirect passing!");841 Register ArgReg = Args[i].Regs[Part];842 // There should be Regs.size() ArgLocs per argument.843 unsigned Idx = BigEndianPartOrdering ? NumParts - 1 - Part : Part;844 CCValAssign &VA = ArgLocs[j + Idx];845 const ISD::ArgFlagsTy Flags = Args[i].Flags[Part];846 847 // We found an indirect parameter passing, and we have an848 // OutgoingValueHandler as our handler (so we are at the call site or the849 // return value). In this case, start the construction of the following850 // GMIR, that is responsible for the preparation of indirect parameter851 // passing:852 //853 // %1(indirectly passed type) = The value to pass854 // %3(pointer) = G_FRAME_INDEX %stack.0855 // G_STORE %1, %3 :: (store (s128), align 8)856 //857 // After this GMIR, the remaining part of the loop body will decide how858 // to get the value to the caller and we break out of the loop.859 if (VA.getLocInfo() == CCValAssign::Indirect &&860 !Handler.isIncomingArgumentHandler()) {861 Align AlignmentForStored = DL.getPrefTypeAlign(Args[i].Ty);862 MachineFrameInfo &MFI = MF.getFrameInfo();863 // Get some space on the stack for the value, so later we can pass it864 // as a reference.865 int FrameIdx = MFI.CreateStackObject(OrigTy.getScalarSizeInBits(),866 AlignmentForStored, false);867 Register PointerToStackReg =868 MIRBuilder.buildFrameIndex(PointerTy, FrameIdx).getReg(0);869 MachinePointerInfo StackPointerMPO =870 MachinePointerInfo::getFixedStack(MF, FrameIdx);871 // Store the value in the previously created stack space.872 MIRBuilder.buildStore(Args[i].OrigRegs[Part], PointerToStackReg,873 StackPointerMPO,874 inferAlignFromPtrInfo(MF, StackPointerMPO));875 876 ArgReg = PointerToStackReg;877 IndirectParameterPassingHandled = true;878 }879 880 if (VA.isMemLoc() && !Flags.isByVal()) {881 // Individual pieces may have been spilled to the stack and others882 // passed in registers.883 884 // TODO: The memory size may be larger than the value we need to885 // store. We may need to adjust the offset for big endian targets.886 LLT MemTy = Handler.getStackValueStoreType(DL, VA, Flags);887 888 MachinePointerInfo MPO;889 Register StackAddr =890 Handler.getStackAddress(VA.getLocInfo() == CCValAssign::Indirect891 ? PointerTy.getSizeInBytes()892 : MemTy.getSizeInBytes(),893 VA.getLocMemOffset(), MPO, Flags);894 895 // Finish the handling of indirect passing from the passers896 // (OutgoingParameterHandler) side.897 // This branch is needed, so the pointer to the value is loaded onto the898 // stack.899 if (VA.getLocInfo() == CCValAssign::Indirect)900 Handler.assignValueToAddress(ArgReg, StackAddr, PointerTy, MPO, VA);901 else902 Handler.assignValueToAddress(Args[i], Part, StackAddr, MemTy, MPO,903 VA);904 } else if (VA.isMemLoc() && Flags.isByVal()) {905 assert(Args[i].Regs.size() == 1 && "didn't expect split byval pointer");906 907 if (Handler.isIncomingArgumentHandler()) {908 // We just need to copy the frame index value to the pointer.909 MachinePointerInfo MPO;910 Register StackAddr = Handler.getStackAddress(911 Flags.getByValSize(), VA.getLocMemOffset(), MPO, Flags);912 MIRBuilder.buildCopy(Args[i].Regs[0], StackAddr);913 } else {914 // For outgoing byval arguments, insert the implicit copy byval915 // implies, such that writes in the callee do not modify the caller's916 // value.917 uint64_t MemSize = Flags.getByValSize();918 int64_t Offset = VA.getLocMemOffset();919 920 MachinePointerInfo DstMPO;921 Register StackAddr =922 Handler.getStackAddress(MemSize, Offset, DstMPO, Flags);923 924 MachinePointerInfo SrcMPO(Args[i].OrigValue);925 if (!Args[i].OrigValue) {926 // We still need to accurately track the stack address space if we927 // don't know the underlying value.928 const LLT PtrTy = MRI.getType(StackAddr);929 SrcMPO = MachinePointerInfo(PtrTy.getAddressSpace());930 }931 932 Align DstAlign = std::max(Flags.getNonZeroByValAlign(),933 inferAlignFromPtrInfo(MF, DstMPO));934 935 Align SrcAlign = std::max(Flags.getNonZeroByValAlign(),936 inferAlignFromPtrInfo(MF, SrcMPO));937 938 Handler.copyArgumentMemory(Args[i], StackAddr, Args[i].Regs[0],939 DstMPO, DstAlign, SrcMPO, SrcAlign,940 MemSize, VA);941 }942 } else if (i == 0 && !ThisReturnRegs.empty() &&943 Handler.isIncomingArgumentHandler() &&944 isTypeIsValidForThisReturn(ValVT)) {945 Handler.assignValueToReg(ArgReg, ThisReturnRegs[Part], VA);946 } else if (Handler.isIncomingArgumentHandler()) {947 Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA);948 } else {949 DelayedOutgoingRegAssignments.emplace_back([=, &Handler]() {950 Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA);951 });952 }953 954 // Finish the handling of indirect parameter passing when receiving955 // the value (we are in the called function or the caller when receiving956 // the return value).957 if (VA.getLocInfo() == CCValAssign::Indirect &&958 Handler.isIncomingArgumentHandler()) {959 Align Alignment = DL.getABITypeAlign(Args[i].Ty);960 MachinePointerInfo MPO = MachinePointerInfo::getUnknownStack(MF);961 962 // Since we are doing indirect parameter passing, we know that the value963 // in the temporary register is not the value passed to the function,964 // but rather a pointer to that value. Let's load that value into the965 // virtual register where the parameter should go.966 MIRBuilder.buildLoad(Args[i].OrigRegs[0], Args[i].Regs[0], MPO,967 Alignment);968 969 IndirectParameterPassingHandled = true;970 }971 972 if (IndirectParameterPassingHandled)973 break;974 }975 976 // Now that all pieces have been assigned, re-pack the register typed values977 // into the original value typed registers. This is only necessary, when978 // the value was passed in multiple registers, not indirectly.979 if (Handler.isIncomingArgumentHandler() && OrigVT != LocVT &&980 !IndirectParameterPassingHandled) {981 // Merge the split registers into the expected larger result vregs of982 // the original call.983 buildCopyFromRegs(MIRBuilder, Args[i].OrigRegs, Args[i].Regs, OrigTy,984 LocTy, Args[i].Flags[0]);985 }986 987 j += NumParts - 1;988 }989 for (auto &Fn : DelayedOutgoingRegAssignments)990 Fn();991 992 return true;993}994 995void CallLowering::insertSRetLoads(MachineIRBuilder &MIRBuilder, Type *RetTy,996 ArrayRef<Register> VRegs, Register DemoteReg,997 int FI) const {998 MachineFunction &MF = MIRBuilder.getMF();999 MachineRegisterInfo &MRI = MF.getRegInfo();1000 const DataLayout &DL = MF.getDataLayout();1001 1002 SmallVector<EVT, 4> SplitVTs;1003 SmallVector<uint64_t, 4> Offsets;1004 ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, /*MemVTs=*/nullptr, &Offsets, 0);1005 1006 assert(VRegs.size() == SplitVTs.size());1007 1008 unsigned NumValues = SplitVTs.size();1009 Align BaseAlign = DL.getPrefTypeAlign(RetTy);1010 Type *RetPtrTy =1011 PointerType::get(RetTy->getContext(), DL.getAllocaAddrSpace());1012 LLT OffsetLLTy = getLLTForType(*DL.getIndexType(RetPtrTy), DL);1013 1014 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);1015 1016 for (unsigned I = 0; I < NumValues; ++I) {1017 Register Addr;1018 MIRBuilder.materializeObjectPtrOffset(Addr, DemoteReg, OffsetLLTy,1019 Offsets[I]);1020 auto *MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad,1021 MRI.getType(VRegs[I]),1022 commonAlignment(BaseAlign, Offsets[I]));1023 MIRBuilder.buildLoad(VRegs[I], Addr, *MMO);1024 }1025}1026 1027void CallLowering::insertSRetStores(MachineIRBuilder &MIRBuilder, Type *RetTy,1028 ArrayRef<Register> VRegs,1029 Register DemoteReg) const {1030 MachineFunction &MF = MIRBuilder.getMF();1031 MachineRegisterInfo &MRI = MF.getRegInfo();1032 const DataLayout &DL = MF.getDataLayout();1033 1034 SmallVector<EVT, 4> SplitVTs;1035 SmallVector<uint64_t, 4> Offsets;1036 ComputeValueVTs(*TLI, DL, RetTy, SplitVTs, /*MemVTs=*/nullptr, &Offsets, 0);1037 1038 assert(VRegs.size() == SplitVTs.size());1039 1040 unsigned NumValues = SplitVTs.size();1041 Align BaseAlign = DL.getPrefTypeAlign(RetTy);1042 unsigned AS = DL.getAllocaAddrSpace();1043 LLT OffsetLLTy = getLLTForType(*DL.getIndexType(RetTy->getContext(), AS), DL);1044 1045 MachinePointerInfo PtrInfo(AS);1046 1047 for (unsigned I = 0; I < NumValues; ++I) {1048 Register Addr;1049 MIRBuilder.materializeObjectPtrOffset(Addr, DemoteReg, OffsetLLTy,1050 Offsets[I]);1051 auto *MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,1052 MRI.getType(VRegs[I]),1053 commonAlignment(BaseAlign, Offsets[I]));1054 MIRBuilder.buildStore(VRegs[I], Addr, *MMO);1055 }1056}1057 1058void CallLowering::insertSRetIncomingArgument(1059 const Function &F, SmallVectorImpl<ArgInfo> &SplitArgs, Register &DemoteReg,1060 MachineRegisterInfo &MRI, const DataLayout &DL) const {1061 unsigned AS = DL.getAllocaAddrSpace();1062 DemoteReg = MRI.createGenericVirtualRegister(1063 LLT::pointer(AS, DL.getPointerSizeInBits(AS)));1064 1065 Type *PtrTy = PointerType::get(F.getContext(), AS);1066 1067 SmallVector<EVT, 1> ValueVTs;1068 ComputeValueVTs(*TLI, DL, PtrTy, ValueVTs);1069 1070 // NOTE: Assume that a pointer won't get split into more than one VT.1071 assert(ValueVTs.size() == 1);1072 1073 ArgInfo DemoteArg(DemoteReg, ValueVTs[0].getTypeForEVT(PtrTy->getContext()),1074 ArgInfo::NoArgIndex);1075 setArgFlags(DemoteArg, AttributeList::ReturnIndex, DL, F);1076 DemoteArg.Flags[0].setSRet();1077 SplitArgs.insert(SplitArgs.begin(), DemoteArg);1078}1079 1080void CallLowering::insertSRetOutgoingArgument(MachineIRBuilder &MIRBuilder,1081 const CallBase &CB,1082 CallLoweringInfo &Info) const {1083 const DataLayout &DL = MIRBuilder.getDataLayout();1084 Type *RetTy = CB.getType();1085 unsigned AS = DL.getAllocaAddrSpace();1086 LLT FramePtrTy = LLT::pointer(AS, DL.getPointerSizeInBits(AS));1087 1088 int FI = MIRBuilder.getMF().getFrameInfo().CreateStackObject(1089 DL.getTypeAllocSize(RetTy), DL.getPrefTypeAlign(RetTy), false);1090 1091 Register DemoteReg = MIRBuilder.buildFrameIndex(FramePtrTy, FI).getReg(0);1092 ArgInfo DemoteArg(DemoteReg, PointerType::get(RetTy->getContext(), AS),1093 ArgInfo::NoArgIndex);1094 setArgFlags(DemoteArg, AttributeList::ReturnIndex, DL, CB);1095 DemoteArg.Flags[0].setSRet();1096 1097 Info.OrigArgs.insert(Info.OrigArgs.begin(), DemoteArg);1098 Info.DemoteStackIndex = FI;1099 Info.DemoteRegister = DemoteReg;1100}1101 1102bool CallLowering::checkReturn(CCState &CCInfo,1103 SmallVectorImpl<BaseArgInfo> &Outs,1104 CCAssignFn *Fn) const {1105 for (unsigned I = 0, E = Outs.size(); I < E; ++I) {1106 MVT VT = MVT::getVT(Outs[I].Ty);1107 if (Fn(I, VT, VT, CCValAssign::Full, Outs[I].Flags[0], Outs[I].Ty, CCInfo))1108 return false;1109 }1110 return true;1111}1112 1113void CallLowering::getReturnInfo(CallingConv::ID CallConv, Type *RetTy,1114 AttributeList Attrs,1115 SmallVectorImpl<BaseArgInfo> &Outs,1116 const DataLayout &DL) const {1117 LLVMContext &Context = RetTy->getContext();1118 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();1119 1120 SmallVector<EVT, 4> SplitVTs;1121 ComputeValueVTs(*TLI, DL, RetTy, SplitVTs);1122 addArgFlagsFromAttributes(Flags, Attrs, AttributeList::ReturnIndex);1123 1124 for (EVT VT : SplitVTs) {1125 unsigned NumParts =1126 TLI->getNumRegistersForCallingConv(Context, CallConv, VT);1127 MVT RegVT = TLI->getRegisterTypeForCallingConv(Context, CallConv, VT);1128 Type *PartTy = EVT(RegVT).getTypeForEVT(Context);1129 1130 for (unsigned I = 0; I < NumParts; ++I) {1131 Outs.emplace_back(PartTy, Flags);1132 }1133 }1134}1135 1136bool CallLowering::checkReturnTypeForCallConv(MachineFunction &MF) const {1137 const auto &F = MF.getFunction();1138 Type *ReturnType = F.getReturnType();1139 CallingConv::ID CallConv = F.getCallingConv();1140 1141 SmallVector<BaseArgInfo, 4> SplitArgs;1142 getReturnInfo(CallConv, ReturnType, F.getAttributes(), SplitArgs,1143 MF.getDataLayout());1144 return canLowerReturn(MF, CallConv, SplitArgs, F.isVarArg());1145}1146 1147bool CallLowering::parametersInCSRMatch(1148 const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask,1149 const SmallVectorImpl<CCValAssign> &OutLocs,1150 const SmallVectorImpl<ArgInfo> &OutArgs) const {1151 for (unsigned i = 0; i < OutLocs.size(); ++i) {1152 const auto &ArgLoc = OutLocs[i];1153 // If it's not a register, it's fine.1154 if (!ArgLoc.isRegLoc())1155 continue;1156 1157 MCRegister PhysReg = ArgLoc.getLocReg();1158 1159 // Only look at callee-saved registers.1160 if (MachineOperand::clobbersPhysReg(CallerPreservedMask, PhysReg))1161 continue;1162 1163 LLVM_DEBUG(1164 dbgs()1165 << "... Call has an argument passed in a callee-saved register.\n");1166 1167 // Check if it was copied from.1168 const ArgInfo &OutInfo = OutArgs[i];1169 1170 if (OutInfo.Regs.size() > 1) {1171 LLVM_DEBUG(1172 dbgs() << "... Cannot handle arguments in multiple registers.\n");1173 return false;1174 }1175 1176 // Check if we copy the register, walking through copies from virtual1177 // registers. Note that getDefIgnoringCopies does not ignore copies from1178 // physical registers.1179 MachineInstr *RegDef = getDefIgnoringCopies(OutInfo.Regs[0], MRI);1180 if (!RegDef || RegDef->getOpcode() != TargetOpcode::COPY) {1181 LLVM_DEBUG(1182 dbgs()1183 << "... Parameter was not copied into a VReg, cannot tail call.\n");1184 return false;1185 }1186 1187 // Got a copy. Verify that it's the same as the register we want.1188 Register CopyRHS = RegDef->getOperand(1).getReg();1189 if (CopyRHS != PhysReg) {1190 LLVM_DEBUG(dbgs() << "... Callee-saved register was not copied into "1191 "VReg, cannot tail call.\n");1192 return false;1193 }1194 }1195 1196 return true;1197}1198 1199bool CallLowering::resultsCompatible(CallLoweringInfo &Info,1200 MachineFunction &MF,1201 SmallVectorImpl<ArgInfo> &InArgs,1202 ValueAssigner &CalleeAssigner,1203 ValueAssigner &CallerAssigner) const {1204 const Function &F = MF.getFunction();1205 CallingConv::ID CalleeCC = Info.CallConv;1206 CallingConv::ID CallerCC = F.getCallingConv();1207 1208 if (CallerCC == CalleeCC)1209 return true;1210 1211 SmallVector<CCValAssign, 16> ArgLocs1;1212 CCState CCInfo1(CalleeCC, Info.IsVarArg, MF, ArgLocs1, F.getContext());1213 if (!determineAssignments(CalleeAssigner, InArgs, CCInfo1))1214 return false;1215 1216 SmallVector<CCValAssign, 16> ArgLocs2;1217 CCState CCInfo2(CallerCC, F.isVarArg(), MF, ArgLocs2, F.getContext());1218 if (!determineAssignments(CallerAssigner, InArgs, CCInfo2))1219 return false;1220 1221 // We need the argument locations to match up exactly. If there's more in1222 // one than the other, then we are done.1223 if (ArgLocs1.size() != ArgLocs2.size())1224 return false;1225 1226 // Make sure that each location is passed in exactly the same way.1227 for (unsigned i = 0, e = ArgLocs1.size(); i < e; ++i) {1228 const CCValAssign &Loc1 = ArgLocs1[i];1229 const CCValAssign &Loc2 = ArgLocs2[i];1230 1231 // We need both of them to be the same. So if one is a register and one1232 // isn't, we're done.1233 if (Loc1.isRegLoc() != Loc2.isRegLoc())1234 return false;1235 1236 if (Loc1.isRegLoc()) {1237 // If they don't have the same register location, we're done.1238 if (Loc1.getLocReg() != Loc2.getLocReg())1239 return false;1240 1241 // They matched, so we can move to the next ArgLoc.1242 continue;1243 }1244 1245 // Loc1 wasn't a RegLoc, so they both must be MemLocs. Check if they match.1246 if (Loc1.getLocMemOffset() != Loc2.getLocMemOffset())1247 return false;1248 }1249 1250 return true;1251}1252 1253LLT CallLowering::ValueHandler::getStackValueStoreType(1254 const DataLayout &DL, const CCValAssign &VA, ISD::ArgFlagsTy Flags) const {1255 const MVT ValVT = VA.getValVT();1256 if (ValVT != MVT::iPTR) {1257 LLT ValTy(ValVT);1258 1259 // We lost the pointeriness going through CCValAssign, so try to restore it1260 // based on the flags.1261 if (Flags.isPointer()) {1262 LLT PtrTy = LLT::pointer(Flags.getPointerAddrSpace(),1263 ValTy.getScalarSizeInBits());1264 if (ValVT.isVector() && ValVT.getVectorNumElements() != 1)1265 return LLT::vector(ValTy.getElementCount(), PtrTy);1266 return PtrTy;1267 }1268 1269 return ValTy;1270 }1271 1272 unsigned AddrSpace = Flags.getPointerAddrSpace();1273 return LLT::pointer(AddrSpace, DL.getPointerSize(AddrSpace));1274}1275 1276void CallLowering::ValueHandler::copyArgumentMemory(1277 const ArgInfo &Arg, Register DstPtr, Register SrcPtr,1278 const MachinePointerInfo &DstPtrInfo, Align DstAlign,1279 const MachinePointerInfo &SrcPtrInfo, Align SrcAlign, uint64_t MemSize,1280 CCValAssign &VA) const {1281 MachineFunction &MF = MIRBuilder.getMF();1282 MachineMemOperand *SrcMMO = MF.getMachineMemOperand(1283 SrcPtrInfo,1284 MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable, MemSize,1285 SrcAlign);1286 1287 MachineMemOperand *DstMMO = MF.getMachineMemOperand(1288 DstPtrInfo,1289 MachineMemOperand::MOStore | MachineMemOperand::MODereferenceable,1290 MemSize, DstAlign);1291 1292 const LLT PtrTy = MRI.getType(DstPtr);1293 const LLT SizeTy = LLT::scalar(PtrTy.getSizeInBits());1294 1295 auto SizeConst = MIRBuilder.buildConstant(SizeTy, MemSize);1296 MIRBuilder.buildMemCpy(DstPtr, SrcPtr, SizeConst, *DstMMO, *SrcMMO);1297}1298 1299Register CallLowering::ValueHandler::extendRegister(Register ValReg,1300 const CCValAssign &VA,1301 unsigned MaxSizeBits) {1302 LLT LocTy{VA.getLocVT()};1303 LLT ValTy{VA.getValVT()};1304 1305 if (LocTy.getSizeInBits() == ValTy.getSizeInBits())1306 return ValReg;1307 1308 if (LocTy.isScalar() && MaxSizeBits && MaxSizeBits < LocTy.getSizeInBits()) {1309 if (MaxSizeBits <= ValTy.getSizeInBits())1310 return ValReg;1311 LocTy = LLT::scalar(MaxSizeBits);1312 }1313 1314 const LLT ValRegTy = MRI.getType(ValReg);1315 if (ValRegTy.isPointer()) {1316 // The x32 ABI wants to zero extend 32-bit pointers to 64-bit registers, so1317 // we have to cast to do the extension.1318 LLT IntPtrTy = LLT::scalar(ValRegTy.getSizeInBits());1319 ValReg = MIRBuilder.buildPtrToInt(IntPtrTy, ValReg).getReg(0);1320 }1321 1322 switch (VA.getLocInfo()) {1323 default:1324 break;1325 case CCValAssign::Full:1326 case CCValAssign::BCvt:1327 // FIXME: bitconverting between vector types may or may not be a1328 // nop in big-endian situations.1329 return ValReg;1330 case CCValAssign::AExt: {1331 auto MIB = MIRBuilder.buildAnyExt(LocTy, ValReg);1332 return MIB.getReg(0);1333 }1334 case CCValAssign::SExt: {1335 Register NewReg = MRI.createGenericVirtualRegister(LocTy);1336 MIRBuilder.buildSExt(NewReg, ValReg);1337 return NewReg;1338 }1339 case CCValAssign::ZExt: {1340 Register NewReg = MRI.createGenericVirtualRegister(LocTy);1341 MIRBuilder.buildZExt(NewReg, ValReg);1342 return NewReg;1343 }1344 }1345 llvm_unreachable("unable to extend register");1346}1347 1348void CallLowering::ValueAssigner::anchor() {}1349 1350Register CallLowering::IncomingValueHandler::buildExtensionHint(1351 const CCValAssign &VA, Register SrcReg, LLT NarrowTy) {1352 switch (VA.getLocInfo()) {1353 case CCValAssign::LocInfo::ZExt: {1354 return MIRBuilder1355 .buildAssertZExt(MRI.cloneVirtualRegister(SrcReg), SrcReg,1356 NarrowTy.getScalarSizeInBits())1357 .getReg(0);1358 }1359 case CCValAssign::LocInfo::SExt: {1360 return MIRBuilder1361 .buildAssertSExt(MRI.cloneVirtualRegister(SrcReg), SrcReg,1362 NarrowTy.getScalarSizeInBits())1363 .getReg(0);1364 break;1365 }1366 default:1367 return SrcReg;1368 }1369}1370 1371/// Check if we can use a basic COPY instruction between the two types.1372///1373/// We're currently building on top of the infrastructure using MVT, which loses1374/// pointer information in the CCValAssign. We accept copies from physical1375/// registers that have been reported as integers if it's to an equivalent sized1376/// pointer LLT.1377static bool isCopyCompatibleType(LLT SrcTy, LLT DstTy) {1378 if (SrcTy == DstTy)1379 return true;1380 1381 if (SrcTy.getSizeInBits() != DstTy.getSizeInBits())1382 return false;1383 1384 SrcTy = SrcTy.getScalarType();1385 DstTy = DstTy.getScalarType();1386 1387 return (SrcTy.isPointer() && DstTy.isScalar()) ||1388 (DstTy.isPointer() && SrcTy.isScalar());1389}1390 1391void CallLowering::IncomingValueHandler::assignValueToReg(1392 Register ValVReg, Register PhysReg, const CCValAssign &VA) {1393 const MVT LocVT = VA.getLocVT();1394 const LLT LocTy(LocVT);1395 const LLT RegTy = MRI.getType(ValVReg);1396 1397 if (isCopyCompatibleType(RegTy, LocTy)) {1398 MIRBuilder.buildCopy(ValVReg, PhysReg);1399 return;1400 }1401 1402 auto Copy = MIRBuilder.buildCopy(LocTy, PhysReg);1403 auto Hint = buildExtensionHint(VA, Copy.getReg(0), RegTy);1404 MIRBuilder.buildTrunc(ValVReg, Hint);1405}1406