1669 lines · cpp
1//===-- llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp - Call lowering -----===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8///9/// \file10/// This file implements the lowering of LLVM calls to machine code calls for11/// GlobalISel.12///13//===----------------------------------------------------------------------===//14 15#include "AMDGPUCallLowering.h"16#include "AMDGPU.h"17#include "AMDGPULegalizerInfo.h"18#include "SIMachineFunctionInfo.h"19#include "SIRegisterInfo.h"20#include "llvm/CodeGen/Analysis.h"21#include "llvm/CodeGen/FunctionLoweringInfo.h"22#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"23#include "llvm/CodeGen/MachineFrameInfo.h"24#include "llvm/IR/IntrinsicsAMDGPU.h"25 26#define DEBUG_TYPE "amdgpu-call-lowering"27 28using namespace llvm;29 30namespace {31 32/// Wrapper around extendRegister to ensure we extend to a full 32-bit register.33static Register extendRegisterMin32(CallLowering::ValueHandler &Handler,34 Register ValVReg, const CCValAssign &VA) {35 if (VA.getLocVT().getSizeInBits() < 32) {36 // 16-bit types are reported as legal for 32-bit registers. We need to37 // extend and do a 32-bit copy to avoid the verifier complaining about it.38 return Handler.MIRBuilder.buildAnyExt(LLT::scalar(32), ValVReg).getReg(0);39 }40 41 return Handler.extendRegister(ValVReg, VA);42}43 44struct AMDGPUOutgoingValueHandler : public CallLowering::OutgoingValueHandler {45 AMDGPUOutgoingValueHandler(MachineIRBuilder &B, MachineRegisterInfo &MRI,46 MachineInstrBuilder MIB)47 : OutgoingValueHandler(B, MRI), MIB(MIB) {}48 49 MachineInstrBuilder MIB;50 51 Register getStackAddress(uint64_t Size, int64_t Offset,52 MachinePointerInfo &MPO,53 ISD::ArgFlagsTy Flags) override {54 llvm_unreachable("not implemented");55 }56 57 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,58 const MachinePointerInfo &MPO,59 const CCValAssign &VA) override {60 llvm_unreachable("not implemented");61 }62 63 void assignValueToReg(Register ValVReg, Register PhysReg,64 const CCValAssign &VA) override {65 Register ExtReg = extendRegisterMin32(*this, ValVReg, VA);66 67 // If this is a scalar return, insert a readfirstlane just in case the value68 // ends up in a VGPR.69 // FIXME: Assert this is a shader return.70 const SIRegisterInfo *TRI71 = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());72 if (TRI->isSGPRReg(MRI, PhysReg)) {73 LLT Ty = MRI.getType(ExtReg);74 LLT S32 = LLT::scalar(32);75 if (Ty != S32) {76 // FIXME: We should probably support readfirstlane intrinsics with all77 // legal 32-bit types.78 assert(Ty.getSizeInBits() == 32);79 if (Ty.isPointer())80 ExtReg = MIRBuilder.buildPtrToInt(S32, ExtReg).getReg(0);81 else82 ExtReg = MIRBuilder.buildBitcast(S32, ExtReg).getReg(0);83 }84 85 auto ToSGPR = MIRBuilder86 .buildIntrinsic(Intrinsic::amdgcn_readfirstlane,87 {MRI.getType(ExtReg)})88 .addReg(ExtReg);89 ExtReg = ToSGPR.getReg(0);90 }91 92 MIRBuilder.buildCopy(PhysReg, ExtReg);93 MIB.addUse(PhysReg, RegState::Implicit);94 }95};96 97struct AMDGPUIncomingArgHandler : public CallLowering::IncomingValueHandler {98 uint64_t StackUsed = 0;99 100 AMDGPUIncomingArgHandler(MachineIRBuilder &B, MachineRegisterInfo &MRI)101 : IncomingValueHandler(B, MRI) {}102 103 Register getStackAddress(uint64_t Size, int64_t Offset,104 MachinePointerInfo &MPO,105 ISD::ArgFlagsTy Flags) override {106 auto &MFI = MIRBuilder.getMF().getFrameInfo();107 108 // Byval is assumed to be writable memory, but other stack passed arguments109 // are not.110 const bool IsImmutable = !Flags.isByVal();111 int FI = MFI.CreateFixedObject(Size, Offset, IsImmutable);112 MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI);113 auto AddrReg = MIRBuilder.buildFrameIndex(114 LLT::pointer(AMDGPUAS::PRIVATE_ADDRESS, 32), FI);115 StackUsed = std::max(StackUsed, Size + Offset);116 return AddrReg.getReg(0);117 }118 119 void assignValueToReg(Register ValVReg, Register PhysReg,120 const CCValAssign &VA) override {121 markPhysRegUsed(PhysReg);122 123 if (VA.getLocVT().getSizeInBits() < 32) {124 // 16-bit types are reported as legal for 32-bit registers. We need to do125 // a 32-bit copy, and truncate to avoid the verifier complaining about it.126 auto Copy = MIRBuilder.buildCopy(LLT::scalar(32), PhysReg);127 128 // If we have signext/zeroext, it applies to the whole 32-bit register129 // before truncation.130 auto Extended =131 buildExtensionHint(VA, Copy.getReg(0), LLT(VA.getLocVT()));132 MIRBuilder.buildTrunc(ValVReg, Extended);133 return;134 }135 136 IncomingValueHandler::assignValueToReg(ValVReg, PhysReg, VA);137 }138 139 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,140 const MachinePointerInfo &MPO,141 const CCValAssign &VA) override {142 MachineFunction &MF = MIRBuilder.getMF();143 144 auto *MMO = MF.getMachineMemOperand(145 MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, MemTy,146 inferAlignFromPtrInfo(MF, MPO));147 MIRBuilder.buildLoad(ValVReg, Addr, *MMO);148 }149 150 /// How the physical register gets marked varies between formal151 /// parameters (it's a basic-block live-in), and a call instruction152 /// (it's an implicit-def of the BL).153 virtual void markPhysRegUsed(unsigned PhysReg) = 0;154};155 156struct FormalArgHandler : public AMDGPUIncomingArgHandler {157 FormalArgHandler(MachineIRBuilder &B, MachineRegisterInfo &MRI)158 : AMDGPUIncomingArgHandler(B, MRI) {}159 160 void markPhysRegUsed(unsigned PhysReg) override {161 MIRBuilder.getMBB().addLiveIn(PhysReg);162 }163};164 165struct CallReturnHandler : public AMDGPUIncomingArgHandler {166 CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,167 MachineInstrBuilder MIB)168 : AMDGPUIncomingArgHandler(MIRBuilder, MRI), MIB(MIB) {}169 170 void markPhysRegUsed(unsigned PhysReg) override {171 MIB.addDef(PhysReg, RegState::Implicit);172 }173 174 MachineInstrBuilder MIB;175};176 177struct AMDGPUOutgoingArgHandler : public AMDGPUOutgoingValueHandler {178 /// For tail calls, the byte offset of the call's argument area from the179 /// callee's. Unused elsewhere.180 int FPDiff;181 182 // Cache the SP register vreg if we need it more than once in this call site.183 Register SPReg;184 185 bool IsTailCall;186 187 AMDGPUOutgoingArgHandler(MachineIRBuilder &MIRBuilder,188 MachineRegisterInfo &MRI, MachineInstrBuilder MIB,189 bool IsTailCall = false, int FPDiff = 0)190 : AMDGPUOutgoingValueHandler(MIRBuilder, MRI, MIB), FPDiff(FPDiff),191 IsTailCall(IsTailCall) {}192 193 Register getStackAddress(uint64_t Size, int64_t Offset,194 MachinePointerInfo &MPO,195 ISD::ArgFlagsTy Flags) override {196 MachineFunction &MF = MIRBuilder.getMF();197 const LLT PtrTy = LLT::pointer(AMDGPUAS::PRIVATE_ADDRESS, 32);198 const LLT S32 = LLT::scalar(32);199 200 if (IsTailCall) {201 Offset += FPDiff;202 int FI = MF.getFrameInfo().CreateFixedObject(Size, Offset, true);203 auto FIReg = MIRBuilder.buildFrameIndex(PtrTy, FI);204 MPO = MachinePointerInfo::getFixedStack(MF, FI);205 return FIReg.getReg(0);206 }207 208 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();209 210 if (!SPReg) {211 const GCNSubtarget &ST = MIRBuilder.getMF().getSubtarget<GCNSubtarget>();212 if (ST.enableFlatScratch()) {213 // The stack is accessed unswizzled, so we can use a regular copy.214 SPReg = MIRBuilder.buildCopy(PtrTy,215 MFI->getStackPtrOffsetReg()).getReg(0);216 } else {217 // The address we produce here, without knowing the use context, is going218 // to be interpreted as a vector address, so we need to convert to a219 // swizzled address.220 SPReg = MIRBuilder.buildInstr(AMDGPU::G_AMDGPU_WAVE_ADDRESS, {PtrTy},221 {MFI->getStackPtrOffsetReg()}).getReg(0);222 }223 }224 225 auto OffsetReg = MIRBuilder.buildConstant(S32, Offset);226 227 auto AddrReg = MIRBuilder.buildPtrAdd(PtrTy, SPReg, OffsetReg);228 MPO = MachinePointerInfo::getStack(MF, Offset);229 return AddrReg.getReg(0);230 }231 232 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,233 const MachinePointerInfo &MPO,234 const CCValAssign &VA) override {235 MachineFunction &MF = MIRBuilder.getMF();236 uint64_t LocMemOffset = VA.getLocMemOffset();237 const auto &ST = MF.getSubtarget<GCNSubtarget>();238 239 auto *MMO = MF.getMachineMemOperand(240 MPO, MachineMemOperand::MOStore, MemTy,241 commonAlignment(ST.getStackAlignment(), LocMemOffset));242 MIRBuilder.buildStore(ValVReg, Addr, *MMO);243 }244 245 void assignValueToAddress(const CallLowering::ArgInfo &Arg,246 unsigned ValRegIndex, Register Addr, LLT MemTy,247 const MachinePointerInfo &MPO,248 const CCValAssign &VA) override {249 Register ValVReg = VA.getLocInfo() != CCValAssign::LocInfo::FPExt250 ? extendRegister(Arg.Regs[ValRegIndex], VA)251 : Arg.Regs[ValRegIndex];252 assignValueToAddress(ValVReg, Addr, MemTy, MPO, VA);253 }254};255} // anonymous namespace256 257AMDGPUCallLowering::AMDGPUCallLowering(const AMDGPUTargetLowering &TLI)258 : CallLowering(&TLI) {259}260 261// FIXME: Compatibility shim262static ISD::NodeType extOpcodeToISDExtOpcode(unsigned MIOpc) {263 switch (MIOpc) {264 case TargetOpcode::G_SEXT:265 return ISD::SIGN_EXTEND;266 case TargetOpcode::G_ZEXT:267 return ISD::ZERO_EXTEND;268 case TargetOpcode::G_ANYEXT:269 return ISD::ANY_EXTEND;270 default:271 llvm_unreachable("not an extend opcode");272 }273}274 275bool AMDGPUCallLowering::canLowerReturn(MachineFunction &MF,276 CallingConv::ID CallConv,277 SmallVectorImpl<BaseArgInfo> &Outs,278 bool IsVarArg) const {279 // For shaders. Vector types should be explicitly handled by CC.280 if (AMDGPU::isEntryFunctionCC(CallConv))281 return true;282 283 SmallVector<CCValAssign, 16> ArgLocs;284 const SITargetLowering &TLI = *getTLI<SITargetLowering>();285 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs,286 MF.getFunction().getContext());287 288 return checkReturn(CCInfo, Outs, TLI.CCAssignFnForReturn(CallConv, IsVarArg));289}290 291/// Lower the return value for the already existing \p Ret. This assumes that292/// \p B's insertion point is correct.293bool AMDGPUCallLowering::lowerReturnVal(MachineIRBuilder &B,294 const Value *Val, ArrayRef<Register> VRegs,295 MachineInstrBuilder &Ret) const {296 if (!Val)297 return true;298 299 auto &MF = B.getMF();300 const auto &F = MF.getFunction();301 const DataLayout &DL = MF.getDataLayout();302 MachineRegisterInfo *MRI = B.getMRI();303 LLVMContext &Ctx = F.getContext();304 305 CallingConv::ID CC = F.getCallingConv();306 const SITargetLowering &TLI = *getTLI<SITargetLowering>();307 308 SmallVector<EVT, 8> SplitEVTs;309 ComputeValueVTs(TLI, DL, Val->getType(), SplitEVTs);310 assert(VRegs.size() == SplitEVTs.size() &&311 "For each split Type there should be exactly one VReg.");312 313 SmallVector<ArgInfo, 8> SplitRetInfos;314 315 for (unsigned i = 0; i < SplitEVTs.size(); ++i) {316 EVT VT = SplitEVTs[i];317 Register Reg = VRegs[i];318 ArgInfo RetInfo(Reg, VT.getTypeForEVT(Ctx), 0);319 setArgFlags(RetInfo, AttributeList::ReturnIndex, DL, F);320 321 if (VT.isScalarInteger()) {322 unsigned ExtendOp = TargetOpcode::G_ANYEXT;323 if (RetInfo.Flags[0].isSExt()) {324 assert(RetInfo.Regs.size() == 1 && "expect only simple return values");325 ExtendOp = TargetOpcode::G_SEXT;326 } else if (RetInfo.Flags[0].isZExt()) {327 assert(RetInfo.Regs.size() == 1 && "expect only simple return values");328 ExtendOp = TargetOpcode::G_ZEXT;329 }330 331 EVT ExtVT = TLI.getTypeForExtReturn(Ctx, VT,332 extOpcodeToISDExtOpcode(ExtendOp));333 if (ExtVT != VT) {334 RetInfo.Ty = ExtVT.getTypeForEVT(Ctx);335 LLT ExtTy = getLLTForType(*RetInfo.Ty, DL);336 Reg = B.buildInstr(ExtendOp, {ExtTy}, {Reg}).getReg(0);337 }338 }339 340 if (Reg != RetInfo.Regs[0]) {341 RetInfo.Regs[0] = Reg;342 // Reset the arg flags after modifying Reg.343 setArgFlags(RetInfo, AttributeList::ReturnIndex, DL, F);344 }345 346 splitToValueTypes(RetInfo, SplitRetInfos, DL, CC);347 }348 349 CCAssignFn *AssignFn = TLI.CCAssignFnForReturn(CC, F.isVarArg());350 351 OutgoingValueAssigner Assigner(AssignFn);352 AMDGPUOutgoingValueHandler RetHandler(B, *MRI, Ret);353 return determineAndHandleAssignments(RetHandler, Assigner, SplitRetInfos, B,354 CC, F.isVarArg());355}356 357bool AMDGPUCallLowering::lowerReturn(MachineIRBuilder &B, const Value *Val,358 ArrayRef<Register> VRegs,359 FunctionLoweringInfo &FLI) const {360 361 MachineFunction &MF = B.getMF();362 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();363 MFI->setIfReturnsVoid(!Val);364 365 assert(!Val == VRegs.empty() && "Return value without a vreg");366 367 CallingConv::ID CC = B.getMF().getFunction().getCallingConv();368 const bool IsShader = AMDGPU::isShader(CC);369 const bool IsWaveEnd =370 (IsShader && MFI->returnsVoid()) || AMDGPU::isKernel(CC);371 if (IsWaveEnd) {372 B.buildInstr(AMDGPU::S_ENDPGM)373 .addImm(0);374 return true;375 }376 377 const bool IsWholeWave = MFI->isWholeWaveFunction();378 unsigned ReturnOpc = IsWholeWave ? AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_RETURN379 : IsShader ? AMDGPU::SI_RETURN_TO_EPILOG380 : AMDGPU::SI_RETURN;381 auto Ret = B.buildInstrNoInsert(ReturnOpc);382 383 if (!FLI.CanLowerReturn)384 insertSRetStores(B, Val->getType(), VRegs, FLI.DemoteRegister);385 else if (!lowerReturnVal(B, Val, VRegs, Ret))386 return false;387 388 if (IsWholeWave)389 addOriginalExecToReturn(B.getMF(), Ret);390 391 // TODO: Handle CalleeSavedRegsViaCopy.392 393 B.insertInstr(Ret);394 return true;395}396 397void AMDGPUCallLowering::lowerParameterPtr(Register DstReg, MachineIRBuilder &B,398 uint64_t Offset) const {399 MachineFunction &MF = B.getMF();400 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();401 MachineRegisterInfo &MRI = MF.getRegInfo();402 Register KernArgSegmentPtr =403 MFI->getPreloadedReg(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);404 Register KernArgSegmentVReg = MRI.getLiveInVirtReg(KernArgSegmentPtr);405 406 auto OffsetReg = B.buildConstant(LLT::scalar(64), Offset);407 408 B.buildPtrAdd(DstReg, KernArgSegmentVReg, OffsetReg);409}410 411void AMDGPUCallLowering::lowerParameter(MachineIRBuilder &B, ArgInfo &OrigArg,412 uint64_t Offset,413 Align Alignment) const {414 MachineFunction &MF = B.getMF();415 const Function &F = MF.getFunction();416 const DataLayout &DL = F.getDataLayout();417 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);418 419 LLT PtrTy = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);420 421 SmallVector<ArgInfo, 32> SplitArgs;422 SmallVector<uint64_t> FieldOffsets;423 splitToValueTypes(OrigArg, SplitArgs, DL, F.getCallingConv(), &FieldOffsets);424 425 unsigned Idx = 0;426 for (ArgInfo &SplitArg : SplitArgs) {427 Register PtrReg = B.getMRI()->createGenericVirtualRegister(PtrTy);428 lowerParameterPtr(PtrReg, B, Offset + FieldOffsets[Idx]);429 430 LLT ArgTy = getLLTForType(*SplitArg.Ty, DL);431 if (SplitArg.Flags[0].isPointer()) {432 // Compensate for losing pointeriness in splitValueTypes.433 LLT PtrTy = LLT::pointer(SplitArg.Flags[0].getPointerAddrSpace(),434 ArgTy.getScalarSizeInBits());435 ArgTy = ArgTy.isVector() ? LLT::vector(ArgTy.getElementCount(), PtrTy)436 : PtrTy;437 }438 439 MachineMemOperand *MMO = MF.getMachineMemOperand(440 PtrInfo,441 MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |442 MachineMemOperand::MOInvariant,443 ArgTy, commonAlignment(Alignment, FieldOffsets[Idx]));444 445 assert(SplitArg.Regs.size() == 1);446 447 B.buildLoad(SplitArg.Regs[0], PtrReg, *MMO);448 ++Idx;449 }450}451 452// Allocate special inputs passed in user SGPRs.453static void allocateHSAUserSGPRs(CCState &CCInfo,454 MachineIRBuilder &B,455 MachineFunction &MF,456 const SIRegisterInfo &TRI,457 SIMachineFunctionInfo &Info) {458 // FIXME: How should these inputs interact with inreg / custom SGPR inputs?459 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info.getUserSGPRInfo();460 if (UserSGPRInfo.hasPrivateSegmentBuffer()) {461 Register PrivateSegmentBufferReg = Info.addPrivateSegmentBuffer(TRI);462 MF.addLiveIn(PrivateSegmentBufferReg, &AMDGPU::SGPR_128RegClass);463 CCInfo.AllocateReg(PrivateSegmentBufferReg);464 }465 466 if (UserSGPRInfo.hasDispatchPtr()) {467 Register DispatchPtrReg = Info.addDispatchPtr(TRI);468 MF.addLiveIn(DispatchPtrReg, &AMDGPU::SGPR_64RegClass);469 CCInfo.AllocateReg(DispatchPtrReg);470 }471 472 if (UserSGPRInfo.hasQueuePtr()) {473 Register QueuePtrReg = Info.addQueuePtr(TRI);474 MF.addLiveIn(QueuePtrReg, &AMDGPU::SGPR_64RegClass);475 CCInfo.AllocateReg(QueuePtrReg);476 }477 478 if (UserSGPRInfo.hasKernargSegmentPtr()) {479 MachineRegisterInfo &MRI = MF.getRegInfo();480 Register InputPtrReg = Info.addKernargSegmentPtr(TRI);481 const LLT P4 = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);482 Register VReg = MRI.createGenericVirtualRegister(P4);483 MRI.addLiveIn(InputPtrReg, VReg);484 B.getMBB().addLiveIn(InputPtrReg);485 B.buildCopy(VReg, InputPtrReg);486 CCInfo.AllocateReg(InputPtrReg);487 }488 489 if (UserSGPRInfo.hasDispatchID()) {490 Register DispatchIDReg = Info.addDispatchID(TRI);491 MF.addLiveIn(DispatchIDReg, &AMDGPU::SGPR_64RegClass);492 CCInfo.AllocateReg(DispatchIDReg);493 }494 495 if (UserSGPRInfo.hasFlatScratchInit()) {496 Register FlatScratchInitReg = Info.addFlatScratchInit(TRI);497 MF.addLiveIn(FlatScratchInitReg, &AMDGPU::SGPR_64RegClass);498 CCInfo.AllocateReg(FlatScratchInitReg);499 }500 501 if (UserSGPRInfo.hasPrivateSegmentSize()) {502 Register PrivateSegmentSizeReg = Info.addPrivateSegmentSize(TRI);503 MF.addLiveIn(PrivateSegmentSizeReg, &AMDGPU::SGPR_32RegClass);504 CCInfo.AllocateReg(PrivateSegmentSizeReg);505 }506 507 // TODO: Add GridWorkGroupCount user SGPRs when used. For now with HSA we read508 // these from the dispatch pointer.509}510 511bool AMDGPUCallLowering::lowerFormalArgumentsKernel(512 MachineIRBuilder &B, const Function &F,513 ArrayRef<ArrayRef<Register>> VRegs) const {514 MachineFunction &MF = B.getMF();515 const GCNSubtarget *Subtarget = &MF.getSubtarget<GCNSubtarget>();516 MachineRegisterInfo &MRI = MF.getRegInfo();517 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();518 const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();519 const SITargetLowering &TLI = *getTLI<SITargetLowering>();520 const DataLayout &DL = F.getDataLayout();521 522 SmallVector<CCValAssign, 16> ArgLocs;523 CCState CCInfo(F.getCallingConv(), F.isVarArg(), MF, ArgLocs, F.getContext());524 525 allocateHSAUserSGPRs(CCInfo, B, MF, *TRI, *Info);526 527 unsigned i = 0;528 const Align KernArgBaseAlign(16);529 const unsigned BaseOffset = Subtarget->getExplicitKernelArgOffset();530 uint64_t ExplicitArgOffset = 0;531 532 // TODO: Align down to dword alignment and extract bits for extending loads.533 for (auto &Arg : F.args()) {534 // TODO: Add support for kernarg preload.535 if (Arg.hasAttribute("amdgpu-hidden-argument")) {536 LLVM_DEBUG(dbgs() << "Preloading hidden arguments is not supported\n");537 return false;538 }539 540 const bool IsByRef = Arg.hasByRefAttr();541 Type *ArgTy = IsByRef ? Arg.getParamByRefType() : Arg.getType();542 unsigned AllocSize = DL.getTypeAllocSize(ArgTy);543 if (AllocSize == 0)544 continue;545 546 MaybeAlign ParamAlign = IsByRef ? Arg.getParamAlign() : std::nullopt;547 Align ABIAlign = DL.getValueOrABITypeAlignment(ParamAlign, ArgTy);548 549 uint64_t ArgOffset = alignTo(ExplicitArgOffset, ABIAlign) + BaseOffset;550 ExplicitArgOffset = alignTo(ExplicitArgOffset, ABIAlign) + AllocSize;551 552 if (Arg.use_empty()) {553 ++i;554 continue;555 }556 557 Align Alignment = commonAlignment(KernArgBaseAlign, ArgOffset);558 559 if (IsByRef) {560 unsigned ByRefAS = cast<PointerType>(Arg.getType())->getAddressSpace();561 562 assert(VRegs[i].size() == 1 &&563 "expected only one register for byval pointers");564 if (ByRefAS == AMDGPUAS::CONSTANT_ADDRESS) {565 lowerParameterPtr(VRegs[i][0], B, ArgOffset);566 } else {567 const LLT ConstPtrTy = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);568 Register PtrReg = MRI.createGenericVirtualRegister(ConstPtrTy);569 lowerParameterPtr(PtrReg, B, ArgOffset);570 571 B.buildAddrSpaceCast(VRegs[i][0], PtrReg);572 }573 } else {574 ArgInfo OrigArg(VRegs[i], Arg, i);575 const unsigned OrigArgIdx = i + AttributeList::FirstArgIndex;576 setArgFlags(OrigArg, OrigArgIdx, DL, F);577 lowerParameter(B, OrigArg, ArgOffset, Alignment);578 }579 580 ++i;581 }582 583 if (Info->getNumKernargPreloadedSGPRs())584 Info->setNumWaveDispatchSGPRs(Info->getNumUserSGPRs());585 586 TLI.allocateSpecialEntryInputVGPRs(CCInfo, MF, *TRI, *Info);587 TLI.allocateSystemSGPRs(CCInfo, MF, *Info, F.getCallingConv(), false);588 return true;589}590 591bool AMDGPUCallLowering::lowerFormalArguments(592 MachineIRBuilder &B, const Function &F, ArrayRef<ArrayRef<Register>> VRegs,593 FunctionLoweringInfo &FLI) const {594 CallingConv::ID CC = F.getCallingConv();595 596 // The infrastructure for normal calling convention lowering is essentially597 // useless for kernels. We want to avoid any kind of legalization or argument598 // splitting.599 if (CC == CallingConv::AMDGPU_KERNEL)600 return lowerFormalArgumentsKernel(B, F, VRegs);601 602 const bool IsGraphics = AMDGPU::isGraphics(CC);603 const bool IsEntryFunc = AMDGPU::isEntryFunctionCC(CC);604 605 MachineFunction &MF = B.getMF();606 MachineBasicBlock &MBB = B.getMBB();607 MachineRegisterInfo &MRI = MF.getRegInfo();608 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();609 const GCNSubtarget &Subtarget = MF.getSubtarget<GCNSubtarget>();610 const SIRegisterInfo *TRI = Subtarget.getRegisterInfo();611 const DataLayout &DL = F.getDataLayout();612 613 SmallVector<CCValAssign, 16> ArgLocs;614 CCState CCInfo(CC, F.isVarArg(), MF, ArgLocs, F.getContext());615 const GCNUserSGPRUsageInfo &UserSGPRInfo = Info->getUserSGPRInfo();616 617 if (UserSGPRInfo.hasImplicitBufferPtr()) {618 Register ImplicitBufferPtrReg = Info->addImplicitBufferPtr(*TRI);619 MF.addLiveIn(ImplicitBufferPtrReg, &AMDGPU::SGPR_64RegClass);620 CCInfo.AllocateReg(ImplicitBufferPtrReg);621 }622 623 // FIXME: This probably isn't defined for mesa624 if (UserSGPRInfo.hasFlatScratchInit() && !Subtarget.isAmdPalOS()) {625 Register FlatScratchInitReg = Info->addFlatScratchInit(*TRI);626 MF.addLiveIn(FlatScratchInitReg, &AMDGPU::SGPR_64RegClass);627 CCInfo.AllocateReg(FlatScratchInitReg);628 }629 630 SmallVector<ArgInfo, 32> SplitArgs;631 unsigned Idx = 0;632 unsigned PSInputNum = 0;633 634 // Insert the hidden sret parameter if the return value won't fit in the635 // return registers.636 if (!FLI.CanLowerReturn)637 insertSRetIncomingArgument(F, SplitArgs, FLI.DemoteRegister, MRI, DL);638 639 for (auto &Arg : F.args()) {640 if (DL.getTypeStoreSize(Arg.getType()) == 0)641 continue;642 643 if (Info->isWholeWaveFunction() && Idx == 0) {644 assert(VRegs[Idx].size() == 1 && "Expected only one register");645 646 // The first argument for whole wave functions is the original EXEC value.647 B.buildInstr(AMDGPU::G_AMDGPU_WHOLE_WAVE_FUNC_SETUP)648 .addDef(VRegs[Idx][0]);649 650 ++Idx;651 continue;652 }653 654 const bool InReg = Arg.hasAttribute(Attribute::InReg);655 656 if (Arg.hasAttribute(Attribute::SwiftSelf) ||657 Arg.hasAttribute(Attribute::SwiftError) ||658 Arg.hasAttribute(Attribute::Nest))659 return false;660 661 if (CC == CallingConv::AMDGPU_PS && !InReg && PSInputNum <= 15) {662 const bool ArgUsed = !Arg.use_empty();663 bool SkipArg = !ArgUsed && !Info->isPSInputAllocated(PSInputNum);664 665 if (!SkipArg) {666 Info->markPSInputAllocated(PSInputNum);667 if (ArgUsed)668 Info->markPSInputEnabled(PSInputNum);669 }670 671 ++PSInputNum;672 673 if (SkipArg) {674 for (Register R : VRegs[Idx])675 B.buildUndef(R);676 677 ++Idx;678 continue;679 }680 }681 682 ArgInfo OrigArg(VRegs[Idx], Arg, Idx);683 const unsigned OrigArgIdx = Idx + AttributeList::FirstArgIndex;684 setArgFlags(OrigArg, OrigArgIdx, DL, F);685 686 splitToValueTypes(OrigArg, SplitArgs, DL, CC);687 ++Idx;688 }689 690 // At least one interpolation mode must be enabled or else the GPU will691 // hang.692 //693 // Check PSInputAddr instead of PSInputEnable. The idea is that if the user694 // set PSInputAddr, the user wants to enable some bits after the compilation695 // based on run-time states. Since we can't know what the final PSInputEna696 // will look like, so we shouldn't do anything here and the user should take697 // responsibility for the correct programming.698 //699 // Otherwise, the following restrictions apply:700 // - At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled.701 // - If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be702 // enabled too.703 if (CC == CallingConv::AMDGPU_PS) {704 if ((Info->getPSInputAddr() & 0x7F) == 0 ||705 ((Info->getPSInputAddr() & 0xF) == 0 &&706 Info->isPSInputAllocated(11))) {707 CCInfo.AllocateReg(AMDGPU::VGPR0);708 CCInfo.AllocateReg(AMDGPU::VGPR1);709 Info->markPSInputAllocated(0);710 Info->markPSInputEnabled(0);711 }712 713 if (Subtarget.isAmdPalOS()) {714 // For isAmdPalOS, the user does not enable some bits after compilation715 // based on run-time states; the register values being generated here are716 // the final ones set in hardware. Therefore we need to apply the717 // workaround to PSInputAddr and PSInputEnable together. (The case where718 // a bit is set in PSInputAddr but not PSInputEnable is where the frontend719 // set up an input arg for a particular interpolation mode, but nothing720 // uses that input arg. Really we should have an earlier pass that removes721 // such an arg.)722 unsigned PsInputBits = Info->getPSInputAddr() & Info->getPSInputEnable();723 if ((PsInputBits & 0x7F) == 0 ||724 ((PsInputBits & 0xF) == 0 &&725 (PsInputBits >> 11 & 1)))726 Info->markPSInputEnabled(llvm::countr_zero(Info->getPSInputAddr()));727 }728 }729 730 const SITargetLowering &TLI = *getTLI<SITargetLowering>();731 CCAssignFn *AssignFn = TLI.CCAssignFnForCall(CC, F.isVarArg());732 733 if (!MBB.empty())734 B.setInstr(*MBB.begin());735 736 if (!IsEntryFunc && !IsGraphics) {737 // For the fixed ABI, pass workitem IDs in the last argument register.738 TLI.allocateSpecialInputVGPRsFixed(CCInfo, MF, *TRI, *Info);739 740 if (!Subtarget.enableFlatScratch())741 CCInfo.AllocateReg(Info->getScratchRSrcReg());742 TLI.allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info);743 }744 745 IncomingValueAssigner Assigner(AssignFn);746 if (!determineAssignments(Assigner, SplitArgs, CCInfo))747 return false;748 749 if (IsEntryFunc) {750 // This assumes the registers are allocated by CCInfo in ascending order751 // with no gaps.752 Info->setNumWaveDispatchSGPRs(753 CCInfo.getFirstUnallocated(AMDGPU::SGPR_32RegClass.getRegisters()));754 Info->setNumWaveDispatchVGPRs(755 CCInfo.getFirstUnallocated(AMDGPU::VGPR_32RegClass.getRegisters()));756 }757 758 FormalArgHandler Handler(B, MRI);759 if (!handleAssignments(Handler, SplitArgs, CCInfo, ArgLocs, B))760 return false;761 762 uint64_t StackSize = Assigner.StackSize;763 764 // Start adding system SGPRs.765 if (IsEntryFunc)766 TLI.allocateSystemSGPRs(CCInfo, MF, *Info, CC, IsGraphics);767 768 // When we tail call, we need to check if the callee's arguments will fit on769 // the caller's stack. So, whenever we lower formal arguments, we should keep770 // track of this information, since we might lower a tail call in this771 // function later.772 Info->setBytesInStackArgArea(StackSize);773 774 // Move back to the end of the basic block.775 B.setMBB(MBB);776 777 return true;778}779 780bool AMDGPUCallLowering::passSpecialInputs(MachineIRBuilder &MIRBuilder,781 CCState &CCInfo,782 SmallVectorImpl<std::pair<MCRegister, Register>> &ArgRegs,783 CallLoweringInfo &Info) const {784 MachineFunction &MF = MIRBuilder.getMF();785 786 // If there's no call site, this doesn't correspond to a call from the IR and787 // doesn't need implicit inputs.788 if (!Info.CB)789 return true;790 791 const AMDGPUFunctionArgInfo *CalleeArgInfo792 = &AMDGPUArgumentUsageInfo::FixedABIFunctionInfo;793 794 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();795 const AMDGPUFunctionArgInfo &CallerArgInfo = MFI->getArgInfo();796 797 798 // TODO: Unify with private memory register handling. This is complicated by799 // the fact that at least in kernels, the input argument is not necessarily800 // in the same location as the input.801 AMDGPUFunctionArgInfo::PreloadedValue InputRegs[] = {802 AMDGPUFunctionArgInfo::DISPATCH_PTR,803 AMDGPUFunctionArgInfo::QUEUE_PTR,804 AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR,805 AMDGPUFunctionArgInfo::DISPATCH_ID,806 AMDGPUFunctionArgInfo::WORKGROUP_ID_X,807 AMDGPUFunctionArgInfo::WORKGROUP_ID_Y,808 AMDGPUFunctionArgInfo::WORKGROUP_ID_Z,809 AMDGPUFunctionArgInfo::LDS_KERNEL_ID,810 };811 812 static constexpr StringLiteral ImplicitAttrNames[][2] = {813 {"amdgpu-no-dispatch-ptr", ""},814 {"amdgpu-no-queue-ptr", ""},815 {"amdgpu-no-implicitarg-ptr", ""},816 {"amdgpu-no-dispatch-id", ""},817 {"amdgpu-no-workgroup-id-x", "amdgpu-no-cluster-id-x"},818 {"amdgpu-no-workgroup-id-y", "amdgpu-no-cluster-id-y"},819 {"amdgpu-no-workgroup-id-z", "amdgpu-no-cluster-id-z"},820 {"amdgpu-no-lds-kernel-id", ""},821 };822 823 MachineRegisterInfo &MRI = MF.getRegInfo();824 825 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();826 const AMDGPULegalizerInfo *LI827 = static_cast<const AMDGPULegalizerInfo*>(ST.getLegalizerInfo());828 829 unsigned I = 0;830 for (auto InputID : InputRegs) {831 const ArgDescriptor *OutgoingArg;832 const TargetRegisterClass *ArgRC;833 LLT ArgTy;834 835 // If the callee does not use the attribute value, skip copying the value.836 if (all_of(ImplicitAttrNames[I++], [&](StringRef AttrName) {837 return AttrName.empty() || Info.CB->hasFnAttr(AttrName);838 }))839 continue;840 841 std::tie(OutgoingArg, ArgRC, ArgTy) =842 CalleeArgInfo->getPreloadedValue(InputID);843 if (!OutgoingArg)844 continue;845 846 const ArgDescriptor *IncomingArg;847 const TargetRegisterClass *IncomingArgRC;848 std::tie(IncomingArg, IncomingArgRC, ArgTy) =849 CallerArgInfo.getPreloadedValue(InputID);850 assert(IncomingArgRC == ArgRC);851 852 Register InputReg = MRI.createGenericVirtualRegister(ArgTy);853 854 if (IncomingArg) {855 LI->buildLoadInputValue(InputReg, MIRBuilder, IncomingArg, ArgRC, ArgTy);856 } else if (InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR) {857 LI->getImplicitArgPtr(InputReg, MRI, MIRBuilder);858 } else if (InputID == AMDGPUFunctionArgInfo::LDS_KERNEL_ID) {859 std::optional<uint32_t> Id =860 AMDGPUMachineFunction::getLDSKernelIdMetadata(MF.getFunction());861 if (Id) {862 MIRBuilder.buildConstant(InputReg, *Id);863 } else {864 MIRBuilder.buildUndef(InputReg);865 }866 } else {867 // We may have proven the input wasn't needed, although the ABI is868 // requiring it. We just need to allocate the register appropriately.869 MIRBuilder.buildUndef(InputReg);870 }871 872 if (OutgoingArg->isRegister()) {873 ArgRegs.emplace_back(OutgoingArg->getRegister(), InputReg);874 if (!CCInfo.AllocateReg(OutgoingArg->getRegister()))875 report_fatal_error("failed to allocate implicit input argument");876 } else {877 LLVM_DEBUG(dbgs() << "Unhandled stack passed implicit input argument\n");878 return false;879 }880 }881 882 // Pack workitem IDs into a single register or pass it as is if already883 // packed.884 const ArgDescriptor *OutgoingArg;885 const TargetRegisterClass *ArgRC;886 LLT ArgTy;887 888 std::tie(OutgoingArg, ArgRC, ArgTy) =889 CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X);890 if (!OutgoingArg)891 std::tie(OutgoingArg, ArgRC, ArgTy) =892 CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y);893 if (!OutgoingArg)894 std::tie(OutgoingArg, ArgRC, ArgTy) =895 CalleeArgInfo->getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z);896 if (!OutgoingArg)897 return false;898 899 auto WorkitemIDX =900 CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X);901 auto WorkitemIDY =902 CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y);903 auto WorkitemIDZ =904 CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z);905 906 const ArgDescriptor *IncomingArgX = std::get<0>(WorkitemIDX);907 const ArgDescriptor *IncomingArgY = std::get<0>(WorkitemIDY);908 const ArgDescriptor *IncomingArgZ = std::get<0>(WorkitemIDZ);909 const LLT S32 = LLT::scalar(32);910 911 const bool NeedWorkItemIDX = !Info.CB->hasFnAttr("amdgpu-no-workitem-id-x");912 const bool NeedWorkItemIDY = !Info.CB->hasFnAttr("amdgpu-no-workitem-id-y");913 const bool NeedWorkItemIDZ = !Info.CB->hasFnAttr("amdgpu-no-workitem-id-z");914 915 // If incoming ids are not packed we need to pack them.916 // FIXME: Should consider known workgroup size to eliminate known 0 cases.917 Register InputReg;918 if (IncomingArgX && !IncomingArgX->isMasked() && CalleeArgInfo->WorkItemIDX &&919 NeedWorkItemIDX) {920 if (ST.getMaxWorkitemID(MF.getFunction(), 0) != 0) {921 InputReg = MRI.createGenericVirtualRegister(S32);922 LI->buildLoadInputValue(InputReg, MIRBuilder, IncomingArgX,923 std::get<1>(WorkitemIDX),924 std::get<2>(WorkitemIDX));925 } else {926 InputReg = MIRBuilder.buildConstant(S32, 0).getReg(0);927 }928 }929 930 if (IncomingArgY && !IncomingArgY->isMasked() && CalleeArgInfo->WorkItemIDY &&931 NeedWorkItemIDY && ST.getMaxWorkitemID(MF.getFunction(), 1) != 0) {932 Register Y = MRI.createGenericVirtualRegister(S32);933 LI->buildLoadInputValue(Y, MIRBuilder, IncomingArgY,934 std::get<1>(WorkitemIDY), std::get<2>(WorkitemIDY));935 936 Y = MIRBuilder.buildShl(S32, Y, MIRBuilder.buildConstant(S32, 10)).getReg(0);937 InputReg = InputReg ? MIRBuilder.buildOr(S32, InputReg, Y).getReg(0) : Y;938 }939 940 if (IncomingArgZ && !IncomingArgZ->isMasked() && CalleeArgInfo->WorkItemIDZ &&941 NeedWorkItemIDZ && ST.getMaxWorkitemID(MF.getFunction(), 2) != 0) {942 Register Z = MRI.createGenericVirtualRegister(S32);943 LI->buildLoadInputValue(Z, MIRBuilder, IncomingArgZ,944 std::get<1>(WorkitemIDZ), std::get<2>(WorkitemIDZ));945 946 Z = MIRBuilder.buildShl(S32, Z, MIRBuilder.buildConstant(S32, 20)).getReg(0);947 InputReg = InputReg ? MIRBuilder.buildOr(S32, InputReg, Z).getReg(0) : Z;948 }949 950 if (!InputReg &&951 (NeedWorkItemIDX || NeedWorkItemIDY || NeedWorkItemIDZ)) {952 InputReg = MRI.createGenericVirtualRegister(S32);953 if (!IncomingArgX && !IncomingArgY && !IncomingArgZ) {954 // We're in a situation where the outgoing function requires the workitem955 // ID, but the calling function does not have it (e.g a graphics function956 // calling a C calling convention function). This is illegal, but we need957 // to produce something.958 MIRBuilder.buildUndef(InputReg);959 } else {960 // Workitem ids are already packed, any of present incoming arguments will961 // carry all required fields.962 ArgDescriptor IncomingArg = ArgDescriptor::createArg(963 IncomingArgX ? *IncomingArgX :964 IncomingArgY ? *IncomingArgY : *IncomingArgZ, ~0u);965 LI->buildLoadInputValue(InputReg, MIRBuilder, &IncomingArg,966 &AMDGPU::VGPR_32RegClass, S32);967 }968 }969 970 if (OutgoingArg->isRegister()) {971 if (InputReg)972 ArgRegs.emplace_back(OutgoingArg->getRegister(), InputReg);973 974 if (!CCInfo.AllocateReg(OutgoingArg->getRegister()))975 report_fatal_error("failed to allocate implicit input argument");976 } else {977 LLVM_DEBUG(dbgs() << "Unhandled stack passed implicit input argument\n");978 return false;979 }980 981 return true;982}983 984/// Returns a pair containing the fixed CCAssignFn and the vararg CCAssignFn for985/// CC.986static std::pair<CCAssignFn *, CCAssignFn *>987getAssignFnsForCC(CallingConv::ID CC, const SITargetLowering &TLI) {988 return {TLI.CCAssignFnForCall(CC, false), TLI.CCAssignFnForCall(CC, true)};989}990 991static unsigned getCallOpcode(const MachineFunction &CallerF, bool IsIndirect,992 bool IsTailCall, bool IsWave32,993 CallingConv::ID CC,994 bool IsDynamicVGPRChainCall = false) {995 // For calls to amdgpu_cs_chain functions, the address is known to be uniform.996 assert((AMDGPU::isChainCC(CC) || !IsIndirect || !IsTailCall) &&997 "Indirect calls can't be tail calls, "998 "because the address can be divergent");999 if (!IsTailCall)1000 return AMDGPU::G_SI_CALL;1001 1002 if (AMDGPU::isChainCC(CC)) {1003 if (IsDynamicVGPRChainCall)1004 return IsWave32 ? AMDGPU::SI_CS_CHAIN_TC_W32_DVGPR1005 : AMDGPU::SI_CS_CHAIN_TC_W64_DVGPR;1006 return IsWave32 ? AMDGPU::SI_CS_CHAIN_TC_W32 : AMDGPU::SI_CS_CHAIN_TC_W64;1007 }1008 1009 if (CallerF.getFunction().getCallingConv() ==1010 CallingConv::AMDGPU_Gfx_WholeWave)1011 return AMDGPU::SI_TCRETURN_GFX_WholeWave;1012 1013 if (CC == CallingConv::AMDGPU_Gfx || CC == CallingConv::AMDGPU_Gfx_WholeWave)1014 return AMDGPU::SI_TCRETURN_GFX;1015 1016 return AMDGPU::SI_TCRETURN;1017}1018 1019// Add operands to call instruction to track the callee.1020static bool addCallTargetOperands(MachineInstrBuilder &CallInst,1021 MachineIRBuilder &MIRBuilder,1022 AMDGPUCallLowering::CallLoweringInfo &Info,1023 bool IsDynamicVGPRChainCall = false) {1024 if (Info.Callee.isReg()) {1025 CallInst.addReg(Info.Callee.getReg());1026 CallInst.addImm(0);1027 } else if (Info.Callee.isGlobal() && Info.Callee.getOffset() == 0) {1028 // The call lowering lightly assumed we can directly encode a call target in1029 // the instruction, which is not the case. Materialize the address here.1030 const GlobalValue *GV = Info.Callee.getGlobal();1031 auto Ptr = MIRBuilder.buildGlobalValue(1032 LLT::pointer(GV->getAddressSpace(), 64), GV);1033 CallInst.addReg(Ptr.getReg(0));1034 1035 if (IsDynamicVGPRChainCall) {1036 // DynamicVGPR chain calls are always indirect.1037 CallInst.addImm(0);1038 } else1039 CallInst.add(Info.Callee);1040 } else1041 return false;1042 1043 return true;1044}1045 1046bool AMDGPUCallLowering::doCallerAndCalleePassArgsTheSameWay(1047 CallLoweringInfo &Info, MachineFunction &MF,1048 SmallVectorImpl<ArgInfo> &InArgs) const {1049 const Function &CallerF = MF.getFunction();1050 CallingConv::ID CalleeCC = Info.CallConv;1051 CallingConv::ID CallerCC = CallerF.getCallingConv();1052 1053 // If the calling conventions match, then everything must be the same.1054 if (CalleeCC == CallerCC)1055 return true;1056 1057 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();1058 1059 // Make sure that the caller and callee preserve all of the same registers.1060 const auto *TRI = ST.getRegisterInfo();1061 1062 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);1063 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);1064 if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))1065 return false;1066 1067 // Check if the caller and callee will handle arguments in the same way.1068 const SITargetLowering &TLI = *getTLI<SITargetLowering>();1069 CCAssignFn *CalleeAssignFnFixed;1070 CCAssignFn *CalleeAssignFnVarArg;1071 std::tie(CalleeAssignFnFixed, CalleeAssignFnVarArg) =1072 getAssignFnsForCC(CalleeCC, TLI);1073 1074 CCAssignFn *CallerAssignFnFixed;1075 CCAssignFn *CallerAssignFnVarArg;1076 std::tie(CallerAssignFnFixed, CallerAssignFnVarArg) =1077 getAssignFnsForCC(CallerCC, TLI);1078 1079 // FIXME: We are not accounting for potential differences in implicitly passed1080 // inputs, but only the fixed ABI is supported now anyway.1081 IncomingValueAssigner CalleeAssigner(CalleeAssignFnFixed,1082 CalleeAssignFnVarArg);1083 IncomingValueAssigner CallerAssigner(CallerAssignFnFixed,1084 CallerAssignFnVarArg);1085 return resultsCompatible(Info, MF, InArgs, CalleeAssigner, CallerAssigner);1086}1087 1088bool AMDGPUCallLowering::areCalleeOutgoingArgsTailCallable(1089 CallLoweringInfo &Info, MachineFunction &MF,1090 SmallVectorImpl<ArgInfo> &OutArgs) const {1091 // If there are no outgoing arguments, then we are done.1092 if (OutArgs.empty())1093 return true;1094 1095 const Function &CallerF = MF.getFunction();1096 CallingConv::ID CalleeCC = Info.CallConv;1097 CallingConv::ID CallerCC = CallerF.getCallingConv();1098 const SITargetLowering &TLI = *getTLI<SITargetLowering>();1099 1100 CCAssignFn *AssignFnFixed;1101 CCAssignFn *AssignFnVarArg;1102 std::tie(AssignFnFixed, AssignFnVarArg) = getAssignFnsForCC(CalleeCC, TLI);1103 1104 // We have outgoing arguments. Make sure that we can tail call with them.1105 SmallVector<CCValAssign, 16> OutLocs;1106 CCState OutInfo(CalleeCC, false, MF, OutLocs, CallerF.getContext());1107 OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg);1108 1109 if (!determineAssignments(Assigner, OutArgs, OutInfo)) {1110 LLVM_DEBUG(dbgs() << "... Could not analyze call operands.\n");1111 return false;1112 }1113 1114 // Make sure that they can fit on the caller's stack.1115 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();1116 if (OutInfo.getStackSize() > FuncInfo->getBytesInStackArgArea()) {1117 LLVM_DEBUG(dbgs() << "... Cannot fit call operands on caller's stack.\n");1118 return false;1119 }1120 1121 // Verify that the parameters in callee-saved registers match.1122 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();1123 const SIRegisterInfo *TRI = ST.getRegisterInfo();1124 const uint32_t *CallerPreservedMask = TRI->getCallPreservedMask(MF, CallerCC);1125 MachineRegisterInfo &MRI = MF.getRegInfo();1126 return parametersInCSRMatch(MRI, CallerPreservedMask, OutLocs, OutArgs);1127}1128 1129bool AMDGPUCallLowering::isEligibleForTailCallOptimization(1130 MachineIRBuilder &B, CallLoweringInfo &Info,1131 SmallVectorImpl<ArgInfo> &InArgs, SmallVectorImpl<ArgInfo> &OutArgs) const {1132 // Must pass all target-independent checks in order to tail call optimize.1133 if (!Info.IsTailCall)1134 return false;1135 1136 // Indirect calls can't be tail calls, because the address can be divergent.1137 // TODO Check divergence info if the call really is divergent.1138 if (Info.Callee.isReg())1139 return false;1140 1141 MachineFunction &MF = B.getMF();1142 const Function &CallerF = MF.getFunction();1143 CallingConv::ID CalleeCC = Info.CallConv;1144 CallingConv::ID CallerCC = CallerF.getCallingConv();1145 1146 const SIRegisterInfo *TRI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();1147 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);1148 // Kernels aren't callable, and don't have a live in return address so it1149 // doesn't make sense to do a tail call with entry functions.1150 if (!CallerPreserved)1151 return false;1152 1153 if (!AMDGPU::mayTailCallThisCC(CalleeCC)) {1154 LLVM_DEBUG(dbgs() << "... Calling convention cannot be tail called.\n");1155 return false;1156 }1157 1158 if (any_of(CallerF.args(), [](const Argument &A) {1159 return A.hasByValAttr() || A.hasSwiftErrorAttr();1160 })) {1161 LLVM_DEBUG(dbgs() << "... Cannot tail call from callers with byval "1162 "or swifterror arguments\n");1163 return false;1164 }1165 1166 // If we have -tailcallopt, then we're done.1167 if (MF.getTarget().Options.GuaranteedTailCallOpt) {1168 return AMDGPU::canGuaranteeTCO(CalleeCC) &&1169 CalleeCC == CallerF.getCallingConv();1170 }1171 1172 // Verify that the incoming and outgoing arguments from the callee are1173 // safe to tail call.1174 if (!doCallerAndCalleePassArgsTheSameWay(Info, MF, InArgs)) {1175 LLVM_DEBUG(1176 dbgs()1177 << "... Caller and callee have incompatible calling conventions.\n");1178 return false;1179 }1180 1181 // FIXME: We need to check if any arguments passed in SGPR are uniform. If1182 // they are not, this cannot be a tail call. If they are uniform, but may be1183 // VGPR, we need to insert readfirstlanes.1184 if (!areCalleeOutgoingArgsTailCallable(Info, MF, OutArgs))1185 return false;1186 1187 LLVM_DEBUG(dbgs() << "... Call is eligible for tail call optimization.\n");1188 return true;1189}1190 1191// Insert outgoing implicit arguments for a call, by inserting copies to the1192// implicit argument registers and adding the necessary implicit uses to the1193// call instruction.1194void AMDGPUCallLowering::handleImplicitCallArguments(1195 MachineIRBuilder &MIRBuilder, MachineInstrBuilder &CallInst,1196 const GCNSubtarget &ST, const SIMachineFunctionInfo &FuncInfo,1197 CallingConv::ID CalleeCC,1198 ArrayRef<std::pair<MCRegister, Register>> ImplicitArgRegs) const {1199 if (!ST.enableFlatScratch()) {1200 // Insert copies for the SRD. In the HSA case, this should be an identity1201 // copy.1202 auto ScratchRSrcReg = MIRBuilder.buildCopy(LLT::fixed_vector(4, 32),1203 FuncInfo.getScratchRSrcReg());1204 1205 auto CalleeRSrcReg = AMDGPU::isChainCC(CalleeCC)1206 ? AMDGPU::SGPR48_SGPR49_SGPR50_SGPR511207 : AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3;1208 1209 MIRBuilder.buildCopy(CalleeRSrcReg, ScratchRSrcReg);1210 CallInst.addReg(CalleeRSrcReg, RegState::Implicit);1211 }1212 1213 for (std::pair<MCRegister, Register> ArgReg : ImplicitArgRegs) {1214 MIRBuilder.buildCopy((Register)ArgReg.first, ArgReg.second);1215 CallInst.addReg(ArgReg.first, RegState::Implicit);1216 }1217}1218 1219namespace {1220// Chain calls have special arguments that we need to handle. These have the1221// same index as they do in the llvm.amdgcn.cs.chain intrinsic.1222enum ChainCallArgIdx {1223 Exec = 1,1224 Flags = 4,1225 NumVGPRs = 5,1226 FallbackExec = 6,1227 FallbackCallee = 7,1228};1229} // anonymous namespace1230 1231bool AMDGPUCallLowering::lowerTailCall(1232 MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info,1233 SmallVectorImpl<ArgInfo> &OutArgs) const {1234 MachineFunction &MF = MIRBuilder.getMF();1235 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();1236 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();1237 const Function &F = MF.getFunction();1238 MachineRegisterInfo &MRI = MF.getRegInfo();1239 const SIInstrInfo *TII = ST.getInstrInfo();1240 const SIRegisterInfo *TRI = ST.getRegisterInfo();1241 const SITargetLowering &TLI = *getTLI<SITargetLowering>();1242 1243 // True when we're tail calling, but without -tailcallopt.1244 bool IsSibCall = !MF.getTarget().Options.GuaranteedTailCallOpt;1245 1246 // Find out which ABI gets to decide where things go.1247 CallingConv::ID CalleeCC = Info.CallConv;1248 CCAssignFn *AssignFnFixed;1249 CCAssignFn *AssignFnVarArg;1250 std::tie(AssignFnFixed, AssignFnVarArg) = getAssignFnsForCC(CalleeCC, TLI);1251 1252 MachineInstrBuilder CallSeqStart;1253 if (!IsSibCall)1254 CallSeqStart = MIRBuilder.buildInstr(AMDGPU::ADJCALLSTACKUP);1255 1256 bool IsChainCall = AMDGPU::isChainCC(Info.CallConv);1257 bool IsDynamicVGPRChainCall = false;1258 1259 if (IsChainCall) {1260 ArgInfo FlagsArg = Info.OrigArgs[ChainCallArgIdx::Flags];1261 const APInt &FlagsValue = cast<ConstantInt>(FlagsArg.OrigValue)->getValue();1262 if (FlagsValue.isZero()) {1263 if (Info.OrigArgs.size() != 5) {1264 LLVM_DEBUG(dbgs() << "No additional args allowed if flags == 0\n");1265 return false;1266 }1267 } else if (FlagsValue.isOneBitSet(0)) {1268 IsDynamicVGPRChainCall = true;1269 1270 if (Info.OrigArgs.size() != 8) {1271 LLVM_DEBUG(dbgs() << "Expected 3 additional args\n");1272 return false;1273 }1274 1275 // On GFX12, we can only change the VGPR allocation for wave32.1276 if (!ST.isWave32()) {1277 F.getContext().diagnose(DiagnosticInfoUnsupported(1278 F, "dynamic VGPR mode is only supported for wave32"));1279 return false;1280 }1281 1282 ArgInfo FallbackExecArg = Info.OrigArgs[ChainCallArgIdx::FallbackExec];1283 assert(FallbackExecArg.Regs.size() == 1 &&1284 "Expected single register for fallback EXEC");1285 if (!FallbackExecArg.Ty->isIntegerTy(ST.getWavefrontSize())) {1286 LLVM_DEBUG(dbgs() << "Bad type for fallback EXEC\n");1287 return false;1288 }1289 }1290 }1291 1292 unsigned Opc = getCallOpcode(MF, Info.Callee.isReg(), /*IsTailCall*/ true,1293 ST.isWave32(), CalleeCC, IsDynamicVGPRChainCall);1294 auto MIB = MIRBuilder.buildInstrNoInsert(Opc);1295 1296 if (FuncInfo->isWholeWaveFunction())1297 addOriginalExecToReturn(MF, MIB);1298 1299 // Keep track of the index of the next operand to be added to the call1300 unsigned CalleeIdx = MIB->getNumOperands();1301 1302 if (!addCallTargetOperands(MIB, MIRBuilder, Info, IsDynamicVGPRChainCall))1303 return false;1304 1305 // Byte offset for the tail call. When we are sibcalling, this will always1306 // be 0.1307 MIB.addImm(0);1308 1309 // If this is a chain call, we need to pass in the EXEC mask as well as any1310 // other special args.1311 if (IsChainCall) {1312 auto AddRegOrImm = [&](const ArgInfo &Arg) {1313 if (auto CI = dyn_cast<ConstantInt>(Arg.OrigValue)) {1314 MIB.addImm(CI->getSExtValue());1315 } else {1316 MIB.addReg(Arg.Regs[0]);1317 unsigned Idx = MIB->getNumOperands() - 1;1318 MIB->getOperand(Idx).setReg(constrainOperandRegClass(1319 MF, *TRI, MRI, *TII, *ST.getRegBankInfo(), *MIB, MIB->getDesc(),1320 MIB->getOperand(Idx), Idx));1321 }1322 };1323 1324 ArgInfo ExecArg = Info.OrigArgs[ChainCallArgIdx::Exec];1325 assert(ExecArg.Regs.size() == 1 && "Too many regs for EXEC");1326 1327 if (!ExecArg.Ty->isIntegerTy(ST.getWavefrontSize())) {1328 LLVM_DEBUG(dbgs() << "Bad type for EXEC");1329 return false;1330 }1331 1332 AddRegOrImm(ExecArg);1333 if (IsDynamicVGPRChainCall)1334 std::for_each(Info.OrigArgs.begin() + ChainCallArgIdx::NumVGPRs,1335 Info.OrigArgs.end(), AddRegOrImm);1336 }1337 1338 // Tell the call which registers are clobbered.1339 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CalleeCC);1340 MIB.addRegMask(Mask);1341 1342 // FPDiff is the byte offset of the call's argument area from the callee's.1343 // Stores to callee stack arguments will be placed in FixedStackSlots offset1344 // by this amount for a tail call. In a sibling call it must be 0 because the1345 // caller will deallocate the entire stack and the callee still expects its1346 // arguments to begin at SP+0.1347 int FPDiff = 0;1348 1349 // This will be 0 for sibcalls, potentially nonzero for tail calls produced1350 // by -tailcallopt. For sibcalls, the memory operands for the call are1351 // already available in the caller's incoming argument space.1352 unsigned NumBytes = 0;1353 if (!IsSibCall) {1354 // We aren't sibcalling, so we need to compute FPDiff. We need to do this1355 // before handling assignments, because FPDiff must be known for memory1356 // arguments.1357 unsigned NumReusableBytes = FuncInfo->getBytesInStackArgArea();1358 SmallVector<CCValAssign, 16> OutLocs;1359 CCState OutInfo(CalleeCC, false, MF, OutLocs, F.getContext());1360 1361 // FIXME: Not accounting for callee implicit inputs1362 OutgoingValueAssigner CalleeAssigner(AssignFnFixed, AssignFnVarArg);1363 if (!determineAssignments(CalleeAssigner, OutArgs, OutInfo))1364 return false;1365 1366 // The callee will pop the argument stack as a tail call. Thus, we must1367 // keep it 16-byte aligned.1368 NumBytes = alignTo(OutInfo.getStackSize(), ST.getStackAlignment());1369 1370 // FPDiff will be negative if this tail call requires more space than we1371 // would automatically have in our incoming argument space. Positive if we1372 // actually shrink the stack.1373 FPDiff = NumReusableBytes - NumBytes;1374 1375 // The stack pointer must be 16-byte aligned at all times it's used for a1376 // memory operation, which in practice means at *all* times and in1377 // particular across call boundaries. Therefore our own arguments started at1378 // a 16-byte aligned SP and the delta applied for the tail call should1379 // satisfy the same constraint.1380 assert(isAligned(ST.getStackAlignment(), FPDiff) &&1381 "unaligned stack on tail call");1382 }1383 1384 SmallVector<CCValAssign, 16> ArgLocs;1385 CCState CCInfo(Info.CallConv, Info.IsVarArg, MF, ArgLocs, F.getContext());1386 1387 // We could pass MIB and directly add the implicit uses to the call1388 // now. However, as an aesthetic choice, place implicit argument operands1389 // after the ordinary user argument registers.1390 SmallVector<std::pair<MCRegister, Register>, 12> ImplicitArgRegs;1391 1392 if (Info.CallConv != CallingConv::AMDGPU_Gfx &&1393 Info.CallConv != CallingConv::AMDGPU_Gfx_WholeWave &&1394 !AMDGPU::isChainCC(Info.CallConv)) {1395 // With a fixed ABI, allocate fixed registers before user arguments.1396 if (!passSpecialInputs(MIRBuilder, CCInfo, ImplicitArgRegs, Info))1397 return false;1398 }1399 1400 OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg);1401 1402 if (!determineAssignments(Assigner, OutArgs, CCInfo))1403 return false;1404 1405 // Do the actual argument marshalling.1406 AMDGPUOutgoingArgHandler Handler(MIRBuilder, MRI, MIB, true, FPDiff);1407 if (!handleAssignments(Handler, OutArgs, CCInfo, ArgLocs, MIRBuilder))1408 return false;1409 1410 if (Info.ConvergenceCtrlToken) {1411 MIB.addUse(Info.ConvergenceCtrlToken, RegState::Implicit);1412 }1413 handleImplicitCallArguments(MIRBuilder, MIB, ST, *FuncInfo, CalleeCC,1414 ImplicitArgRegs);1415 1416 // If we have -tailcallopt, we need to adjust the stack. We'll do the call1417 // sequence start and end here.1418 if (!IsSibCall) {1419 MIB->getOperand(CalleeIdx + 1).setImm(FPDiff);1420 CallSeqStart.addImm(NumBytes).addImm(0);1421 // End the call sequence *before* emitting the call. Normally, we would1422 // tidy the frame up after the call. However, here, we've laid out the1423 // parameters so that when SP is reset, they will be in the correct1424 // location.1425 MIRBuilder.buildInstr(AMDGPU::ADJCALLSTACKDOWN).addImm(NumBytes).addImm(0);1426 }1427 1428 // Now we can add the actual call instruction to the correct basic block.1429 MIRBuilder.insertInstr(MIB);1430 1431 // If this is a whole wave tail call, we need to constrain the register for1432 // the original EXEC.1433 if (MIB->getOpcode() == AMDGPU::SI_TCRETURN_GFX_WholeWave) {1434 MIB->getOperand(0).setReg(1435 constrainOperandRegClass(MF, *TRI, MRI, *TII, *ST.getRegBankInfo(),1436 *MIB, MIB->getDesc(), MIB->getOperand(0), 0));1437 }1438 1439 // If Callee is a reg, since it is used by a target specific1440 // instruction, it must have a register class matching the1441 // constraint of that instruction.1442 1443 // FIXME: We should define regbankselectable call instructions to handle1444 // divergent call targets.1445 if (MIB->getOperand(CalleeIdx).isReg()) {1446 MIB->getOperand(CalleeIdx).setReg(constrainOperandRegClass(1447 MF, *TRI, MRI, *TII, *ST.getRegBankInfo(), *MIB, MIB->getDesc(),1448 MIB->getOperand(CalleeIdx), CalleeIdx));1449 }1450 1451 MF.getFrameInfo().setHasTailCall();1452 Info.LoweredTailCall = true;1453 return true;1454}1455 1456/// Lower a call to the @llvm.amdgcn.cs.chain intrinsic.1457bool AMDGPUCallLowering::lowerChainCall(MachineIRBuilder &MIRBuilder,1458 CallLoweringInfo &Info) const {1459 ArgInfo Callee = Info.OrigArgs[0];1460 ArgInfo SGPRArgs = Info.OrigArgs[2];1461 ArgInfo VGPRArgs = Info.OrigArgs[3];1462 1463 MachineFunction &MF = MIRBuilder.getMF();1464 const Function &F = MF.getFunction();1465 const DataLayout &DL = F.getDataLayout();1466 1467 // The function to jump to is actually the first argument, so we'll change the1468 // Callee and other info to match that before using our existing helper.1469 const Value *CalleeV = Callee.OrigValue->stripPointerCasts();1470 if (const Function *F = dyn_cast<Function>(CalleeV)) {1471 Info.Callee = MachineOperand::CreateGA(F, 0);1472 Info.CallConv = F->getCallingConv();1473 } else {1474 assert(Callee.Regs.size() == 1 && "Too many regs for the callee");1475 Info.Callee = MachineOperand::CreateReg(Callee.Regs[0], false);1476 Info.CallConv = CallingConv::AMDGPU_CS_Chain; // amdgpu_cs_chain_preserve1477 // behaves the same here.1478 }1479 1480 // The function that we're calling cannot be vararg (only the intrinsic is).1481 Info.IsVarArg = false;1482 1483 assert(1484 all_of(SGPRArgs.Flags, [](ISD::ArgFlagsTy F) { return F.isInReg(); }) &&1485 "SGPR arguments should be marked inreg");1486 assert(1487 none_of(VGPRArgs.Flags, [](ISD::ArgFlagsTy F) { return F.isInReg(); }) &&1488 "VGPR arguments should not be marked inreg");1489 1490 SmallVector<ArgInfo, 8> OutArgs;1491 splitToValueTypes(SGPRArgs, OutArgs, DL, Info.CallConv);1492 splitToValueTypes(VGPRArgs, OutArgs, DL, Info.CallConv);1493 1494 Info.IsMustTailCall = true;1495 return lowerTailCall(MIRBuilder, Info, OutArgs);1496}1497 1498bool AMDGPUCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,1499 CallLoweringInfo &Info) const {1500 if (Function *F = Info.CB->getCalledFunction())1501 if (F->isIntrinsic()) {1502 switch (F->getIntrinsicID()) {1503 case Intrinsic::amdgcn_cs_chain:1504 return lowerChainCall(MIRBuilder, Info);1505 case Intrinsic::amdgcn_call_whole_wave:1506 Info.CallConv = CallingConv::AMDGPU_Gfx_WholeWave;1507 1508 // Get the callee from the original instruction, so it doesn't look like1509 // this is an indirect call.1510 Info.Callee = MachineOperand::CreateGA(1511 cast<GlobalValue>(Info.CB->getOperand(0)), /*Offset=*/0);1512 Info.OrigArgs.erase(Info.OrigArgs.begin());1513 Info.IsVarArg = false;1514 break;1515 default:1516 llvm_unreachable("Unexpected intrinsic call");1517 }1518 }1519 1520 if (Info.IsVarArg) {1521 LLVM_DEBUG(dbgs() << "Variadic functions not implemented\n");1522 return false;1523 }1524 1525 MachineFunction &MF = MIRBuilder.getMF();1526 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();1527 const SIRegisterInfo *TRI = ST.getRegisterInfo();1528 1529 const Function &F = MF.getFunction();1530 MachineRegisterInfo &MRI = MF.getRegInfo();1531 const SITargetLowering &TLI = *getTLI<SITargetLowering>();1532 const DataLayout &DL = F.getDataLayout();1533 1534 SmallVector<ArgInfo, 8> OutArgs;1535 for (auto &OrigArg : Info.OrigArgs)1536 splitToValueTypes(OrigArg, OutArgs, DL, Info.CallConv);1537 1538 SmallVector<ArgInfo, 8> InArgs;1539 if (Info.CanLowerReturn && !Info.OrigRet.Ty->isVoidTy())1540 splitToValueTypes(Info.OrigRet, InArgs, DL, Info.CallConv);1541 1542 // If we can lower as a tail call, do that instead.1543 bool CanTailCallOpt =1544 isEligibleForTailCallOptimization(MIRBuilder, Info, InArgs, OutArgs);1545 1546 // We must emit a tail call if we have musttail.1547 if (Info.IsMustTailCall && !CanTailCallOpt) {1548 LLVM_DEBUG(dbgs() << "Failed to lower musttail call as tail call\n");1549 return false;1550 }1551 1552 Info.IsTailCall = CanTailCallOpt;1553 if (CanTailCallOpt)1554 return lowerTailCall(MIRBuilder, Info, OutArgs);1555 1556 // Find out which ABI gets to decide where things go.1557 CCAssignFn *AssignFnFixed;1558 CCAssignFn *AssignFnVarArg;1559 std::tie(AssignFnFixed, AssignFnVarArg) =1560 getAssignFnsForCC(Info.CallConv, TLI);1561 1562 MIRBuilder.buildInstr(AMDGPU::ADJCALLSTACKUP)1563 .addImm(0)1564 .addImm(0);1565 1566 // Create a temporarily-floating call instruction so we can add the implicit1567 // uses of arg registers.1568 unsigned Opc = getCallOpcode(MF, Info.Callee.isReg(), false, ST.isWave32(),1569 Info.CallConv);1570 1571 auto MIB = MIRBuilder.buildInstrNoInsert(Opc);1572 MIB.addDef(TRI->getReturnAddressReg(MF));1573 1574 if (!Info.IsConvergent)1575 MIB.setMIFlag(MachineInstr::NoConvergent);1576 1577 if (!addCallTargetOperands(MIB, MIRBuilder, Info))1578 return false;1579 1580 // Tell the call which registers are clobbered.1581 const uint32_t *Mask = TRI->getCallPreservedMask(MF, Info.CallConv);1582 MIB.addRegMask(Mask);1583 1584 SmallVector<CCValAssign, 16> ArgLocs;1585 CCState CCInfo(Info.CallConv, Info.IsVarArg, MF, ArgLocs, F.getContext());1586 1587 // We could pass MIB and directly add the implicit uses to the call1588 // now. However, as an aesthetic choice, place implicit argument operands1589 // after the ordinary user argument registers.1590 SmallVector<std::pair<MCRegister, Register>, 12> ImplicitArgRegs;1591 1592 if (Info.CallConv != CallingConv::AMDGPU_Gfx &&1593 Info.CallConv != CallingConv::AMDGPU_Gfx_WholeWave) {1594 // With a fixed ABI, allocate fixed registers before user arguments.1595 if (!passSpecialInputs(MIRBuilder, CCInfo, ImplicitArgRegs, Info))1596 return false;1597 }1598 1599 // Do the actual argument marshalling.1600 OutgoingValueAssigner Assigner(AssignFnFixed, AssignFnVarArg);1601 if (!determineAssignments(Assigner, OutArgs, CCInfo))1602 return false;1603 1604 AMDGPUOutgoingArgHandler Handler(MIRBuilder, MRI, MIB, false);1605 if (!handleAssignments(Handler, OutArgs, CCInfo, ArgLocs, MIRBuilder))1606 return false;1607 1608 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();1609 1610 if (Info.ConvergenceCtrlToken) {1611 MIB.addUse(Info.ConvergenceCtrlToken, RegState::Implicit);1612 }1613 handleImplicitCallArguments(MIRBuilder, MIB, ST, *MFI, Info.CallConv,1614 ImplicitArgRegs);1615 1616 // Get a count of how many bytes are to be pushed on the stack.1617 unsigned NumBytes = CCInfo.getStackSize();1618 1619 // If Callee is a reg, since it is used by a target specific1620 // instruction, it must have a register class matching the1621 // constraint of that instruction.1622 1623 // FIXME: We should define regbankselectable call instructions to handle1624 // divergent call targets.1625 if (MIB->getOperand(1).isReg()) {1626 MIB->getOperand(1).setReg(constrainOperandRegClass(1627 MF, *TRI, MRI, *ST.getInstrInfo(),1628 *ST.getRegBankInfo(), *MIB, MIB->getDesc(), MIB->getOperand(1),1629 1));1630 }1631 1632 // Now we can add the actual call instruction to the correct position.1633 MIRBuilder.insertInstr(MIB);1634 1635 // Finally we can copy the returned value back into its virtual-register. In1636 // symmetry with the arguments, the physical register must be an1637 // implicit-define of the call instruction.1638 if (Info.CanLowerReturn && !Info.OrigRet.Ty->isVoidTy()) {1639 CCAssignFn *RetAssignFn = TLI.CCAssignFnForReturn(Info.CallConv,1640 Info.IsVarArg);1641 IncomingValueAssigner Assigner(RetAssignFn);1642 CallReturnHandler Handler(MIRBuilder, MRI, MIB);1643 if (!determineAndHandleAssignments(Handler, Assigner, InArgs, MIRBuilder,1644 Info.CallConv, Info.IsVarArg))1645 return false;1646 }1647 1648 uint64_t CalleePopBytes = NumBytes;1649 1650 MIRBuilder.buildInstr(AMDGPU::ADJCALLSTACKDOWN)1651 .addImm(0)1652 .addImm(CalleePopBytes);1653 1654 if (!Info.CanLowerReturn) {1655 insertSRetLoads(MIRBuilder, Info.OrigRet.Ty, Info.OrigRet.Regs,1656 Info.DemoteRegister, Info.DemoteStackIndex);1657 }1658 1659 return true;1660}1661 1662void AMDGPUCallLowering::addOriginalExecToReturn(1663 MachineFunction &MF, MachineInstrBuilder &Ret) const {1664 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();1665 const SIInstrInfo *TII = ST.getInstrInfo();1666 const MachineInstr *Setup = TII->getWholeWaveFunctionSetup(MF);1667 Ret.addReg(Setup->getOperand(0).getReg());1668}1669