432 lines · cpp
1//===- llvm/lib/Target/X86/X86CallLowering.cpp - Call lowering ------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9/// \file10/// This file implements the lowering of LLVM calls to machine code calls for11/// GlobalISel.12//13//===----------------------------------------------------------------------===//14 15#include "X86CallLowering.h"16#include "X86CallingConv.h"17#include "X86ISelLowering.h"18#include "X86InstrInfo.h"19#include "X86MachineFunctionInfo.h"20#include "X86RegisterInfo.h"21#include "X86Subtarget.h"22#include "llvm/ADT/ArrayRef.h"23#include "llvm/ADT/SmallVector.h"24#include "llvm/CodeGen/Analysis.h"25#include "llvm/CodeGen/CallingConvLower.h"26#include "llvm/CodeGen/FunctionLoweringInfo.h"27#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"28#include "llvm/CodeGen/GlobalISel/Utils.h"29#include "llvm/CodeGen/LowLevelTypeUtils.h"30#include "llvm/CodeGen/MachineBasicBlock.h"31#include "llvm/CodeGen/MachineFrameInfo.h"32#include "llvm/CodeGen/MachineFunction.h"33#include "llvm/CodeGen/MachineInstrBuilder.h"34#include "llvm/CodeGen/MachineMemOperand.h"35#include "llvm/CodeGen/MachineOperand.h"36#include "llvm/CodeGen/MachineRegisterInfo.h"37#include "llvm/CodeGen/TargetInstrInfo.h"38#include "llvm/CodeGen/TargetSubtargetInfo.h"39#include "llvm/CodeGen/ValueTypes.h"40#include "llvm/CodeGenTypes/LowLevelType.h"41#include "llvm/CodeGenTypes/MachineValueType.h"42#include "llvm/IR/Attributes.h"43#include "llvm/IR/DataLayout.h"44#include "llvm/IR/Function.h"45#include "llvm/IR/Value.h"46#include <cassert>47#include <cstdint>48 49using namespace llvm;50 51X86CallLowering::X86CallLowering(const X86TargetLowering &TLI)52 : CallLowering(&TLI) {}53 54namespace {55 56struct X86OutgoingValueAssigner : public CallLowering::OutgoingValueAssigner {57private:58 uint64_t StackSize = 0;59 unsigned NumXMMRegs = 0;60 61public:62 uint64_t getStackSize() { return StackSize; }63 unsigned getNumXmmRegs() { return NumXMMRegs; }64 65 X86OutgoingValueAssigner(CCAssignFn *AssignFn_)66 : CallLowering::OutgoingValueAssigner(AssignFn_) {}67 68 bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT,69 CCValAssign::LocInfo LocInfo,70 const CallLowering::ArgInfo &Info, ISD::ArgFlagsTy Flags,71 CCState &State) override {72 bool Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, Info.Ty, State);73 StackSize = State.getStackSize();74 75 static const MCPhysReg XMMArgRegs[] = {X86::XMM0, X86::XMM1, X86::XMM2,76 X86::XMM3, X86::XMM4, X86::XMM5,77 X86::XMM6, X86::XMM7};78 if (Flags.isVarArg())79 NumXMMRegs = State.getFirstUnallocated(XMMArgRegs);80 81 return Res;82 }83};84 85struct X86OutgoingValueHandler : public CallLowering::OutgoingValueHandler {86 X86OutgoingValueHandler(MachineIRBuilder &MIRBuilder,87 MachineRegisterInfo &MRI, MachineInstrBuilder &MIB)88 : OutgoingValueHandler(MIRBuilder, MRI), MIB(MIB),89 DL(MIRBuilder.getMF().getDataLayout()),90 STI(MIRBuilder.getMF().getSubtarget<X86Subtarget>()) {}91 92 Register getStackAddress(uint64_t Size, int64_t Offset,93 MachinePointerInfo &MPO,94 ISD::ArgFlagsTy Flags) override {95 LLT p0 = LLT::pointer(0, DL.getPointerSizeInBits(0));96 LLT SType = LLT::scalar(DL.getPointerSizeInBits(0));97 auto SPReg =98 MIRBuilder.buildCopy(p0, STI.getRegisterInfo()->getStackRegister());99 100 auto OffsetReg = MIRBuilder.buildConstant(SType, Offset);101 102 auto AddrReg = MIRBuilder.buildPtrAdd(p0, SPReg, OffsetReg);103 104 MPO = MachinePointerInfo::getStack(MIRBuilder.getMF(), Offset);105 return AddrReg.getReg(0);106 }107 108 void assignValueToReg(Register ValVReg, Register PhysReg,109 const CCValAssign &VA) override {110 MIB.addUse(PhysReg, RegState::Implicit);111 Register ExtReg = extendRegister(ValVReg, VA);112 MIRBuilder.buildCopy(PhysReg, ExtReg);113 }114 115 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,116 const MachinePointerInfo &MPO,117 const CCValAssign &VA) override {118 MachineFunction &MF = MIRBuilder.getMF();119 Register ExtReg = extendRegister(ValVReg, VA);120 121 auto *MMO = MF.getMachineMemOperand(MPO, MachineMemOperand::MOStore, MemTy,122 inferAlignFromPtrInfo(MF, MPO));123 MIRBuilder.buildStore(ExtReg, Addr, *MMO);124 }125 126protected:127 MachineInstrBuilder &MIB;128 const DataLayout &DL;129 const X86Subtarget &STI;130};131 132} // end anonymous namespace133 134bool X86CallLowering::canLowerReturn(135 MachineFunction &MF, CallingConv::ID CallConv,136 SmallVectorImpl<CallLowering::BaseArgInfo> &Outs, bool IsVarArg) const {137 LLVMContext &Context = MF.getFunction().getContext();138 SmallVector<CCValAssign, 16> RVLocs;139 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);140 return checkReturn(CCInfo, Outs, RetCC_X86);141}142 143bool X86CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,144 const Value *Val, ArrayRef<Register> VRegs,145 FunctionLoweringInfo &FLI) const {146 assert(((Val && !VRegs.empty()) || (!Val && VRegs.empty())) &&147 "Return value without a vreg");148 MachineFunction &MF = MIRBuilder.getMF();149 auto MIB = MIRBuilder.buildInstrNoInsert(X86::RET).addImm(0);150 auto FuncInfo = MF.getInfo<X86MachineFunctionInfo>();151 const auto &STI = MF.getSubtarget<X86Subtarget>();152 Register RetReg = STI.is64Bit() ? X86::RAX : X86::EAX;153 154 if (!FLI.CanLowerReturn) {155 insertSRetStores(MIRBuilder, Val->getType(), VRegs, FLI.DemoteRegister);156 MIRBuilder.buildCopy(RetReg, FLI.DemoteRegister);157 MIB.addReg(RetReg);158 } else if (Register Reg = FuncInfo->getSRetReturnReg()) {159 MIRBuilder.buildCopy(RetReg, Reg);160 MIB.addReg(RetReg);161 } else if (!VRegs.empty()) {162 const Function &F = MF.getFunction();163 MachineRegisterInfo &MRI = MF.getRegInfo();164 const DataLayout &DL = MF.getDataLayout();165 166 ArgInfo OrigRetInfo(VRegs, Val->getType(), 0);167 setArgFlags(OrigRetInfo, AttributeList::ReturnIndex, DL, F);168 169 SmallVector<ArgInfo, 4> SplitRetInfos;170 splitToValueTypes(OrigRetInfo, SplitRetInfos, DL, F.getCallingConv());171 172 X86OutgoingValueAssigner Assigner(RetCC_X86);173 X86OutgoingValueHandler Handler(MIRBuilder, MRI, MIB);174 if (!determineAndHandleAssignments(Handler, Assigner, SplitRetInfos,175 MIRBuilder, F.getCallingConv(),176 F.isVarArg()))177 return false;178 }179 180 MIRBuilder.insertInstr(MIB);181 return true;182}183 184namespace {185 186struct X86IncomingValueHandler : public CallLowering::IncomingValueHandler {187 X86IncomingValueHandler(MachineIRBuilder &MIRBuilder,188 MachineRegisterInfo &MRI)189 : IncomingValueHandler(MIRBuilder, MRI),190 DL(MIRBuilder.getMF().getDataLayout()) {}191 192 Register getStackAddress(uint64_t Size, int64_t Offset,193 MachinePointerInfo &MPO,194 ISD::ArgFlagsTy Flags) override {195 auto &MFI = MIRBuilder.getMF().getFrameInfo();196 197 // Byval is assumed to be writable memory, but other stack passed arguments198 // are not.199 const bool IsImmutable = !Flags.isByVal();200 201 int FI = MFI.CreateFixedObject(Size, Offset, IsImmutable);202 MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI);203 204 return MIRBuilder205 .buildFrameIndex(LLT::pointer(0, DL.getPointerSizeInBits(0)), FI)206 .getReg(0);207 }208 209 void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,210 const MachinePointerInfo &MPO,211 const CCValAssign &VA) override {212 MachineFunction &MF = MIRBuilder.getMF();213 auto *MMO = MF.getMachineMemOperand(214 MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, MemTy,215 inferAlignFromPtrInfo(MF, MPO));216 MIRBuilder.buildLoad(ValVReg, Addr, *MMO);217 }218 219 void assignValueToReg(Register ValVReg, Register PhysReg,220 const CCValAssign &VA) override {221 markPhysRegUsed(PhysReg.asMCReg());222 IncomingValueHandler::assignValueToReg(ValVReg, PhysReg, VA);223 }224 225 /// How the physical register gets marked varies between formal226 /// parameters (it's a basic-block live-in), and a call instruction227 /// (it's an implicit-def of the BL).228 virtual void markPhysRegUsed(MCRegister PhysReg) = 0;229 230protected:231 const DataLayout &DL;232};233 234struct FormalArgHandler : public X86IncomingValueHandler {235 FormalArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI)236 : X86IncomingValueHandler(MIRBuilder, MRI) {}237 238 void markPhysRegUsed(MCRegister PhysReg) override {239 MIRBuilder.getMRI()->addLiveIn(PhysReg);240 MIRBuilder.getMBB().addLiveIn(PhysReg);241 }242};243 244struct CallReturnHandler : public X86IncomingValueHandler {245 CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,246 MachineInstrBuilder &MIB)247 : X86IncomingValueHandler(MIRBuilder, MRI), MIB(MIB) {}248 249 void markPhysRegUsed(MCRegister PhysReg) override {250 MIB.addDef(PhysReg, RegState::Implicit);251 }252 253protected:254 MachineInstrBuilder &MIB;255};256 257} // end anonymous namespace258 259bool X86CallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,260 const Function &F,261 ArrayRef<ArrayRef<Register>> VRegs,262 FunctionLoweringInfo &FLI) const {263 MachineFunction &MF = MIRBuilder.getMF();264 MachineRegisterInfo &MRI = MF.getRegInfo();265 auto DL = MF.getDataLayout();266 auto FuncInfo = MF.getInfo<X86MachineFunctionInfo>();267 268 SmallVector<ArgInfo, 8> SplitArgs;269 270 if (!FLI.CanLowerReturn)271 insertSRetIncomingArgument(F, SplitArgs, FLI.DemoteRegister, MRI, DL);272 273 // TODO: handle variadic function274 if (F.isVarArg())275 return false;276 277 unsigned Idx = 0;278 for (const auto &Arg : F.args()) {279 // TODO: handle not simple cases.280 if (Arg.hasAttribute(Attribute::ByVal) ||281 Arg.hasAttribute(Attribute::InReg) ||282 Arg.hasAttribute(Attribute::SwiftSelf) ||283 Arg.hasAttribute(Attribute::SwiftError) || VRegs[Idx].size() > 1)284 return false;285 286 if (Arg.hasAttribute(Attribute::StructRet)) {287 assert(VRegs[Idx].size() == 1 &&288 "Unexpected amount of registers for sret argument.");289 FuncInfo->setSRetReturnReg(VRegs[Idx][0]);290 }291 292 ArgInfo OrigArg(VRegs[Idx], Arg.getType(), Idx);293 setArgFlags(OrigArg, Idx + AttributeList::FirstArgIndex, DL, F);294 splitToValueTypes(OrigArg, SplitArgs, DL, F.getCallingConv());295 Idx++;296 }297 298 if (SplitArgs.empty())299 return true;300 301 MachineBasicBlock &MBB = MIRBuilder.getMBB();302 if (!MBB.empty())303 MIRBuilder.setInstr(*MBB.begin());304 305 X86OutgoingValueAssigner Assigner(CC_X86);306 FormalArgHandler Handler(MIRBuilder, MRI);307 if (!determineAndHandleAssignments(Handler, Assigner, SplitArgs, MIRBuilder,308 F.getCallingConv(), F.isVarArg()))309 return false;310 311 // Move back to the end of the basic block.312 MIRBuilder.setMBB(MBB);313 314 return true;315}316 317bool X86CallLowering::lowerCall(MachineIRBuilder &MIRBuilder,318 CallLoweringInfo &Info) const {319 MachineFunction &MF = MIRBuilder.getMF();320 const Function &F = MF.getFunction();321 MachineRegisterInfo &MRI = MF.getRegInfo();322 const DataLayout &DL = F.getDataLayout();323 const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();324 const TargetInstrInfo &TII = *STI.getInstrInfo();325 const X86RegisterInfo *TRI = STI.getRegisterInfo();326 327 // Handle only Linux C, X86_64_SysV calling conventions for now.328 if (!STI.isTargetLinux() || !(Info.CallConv == CallingConv::C ||329 Info.CallConv == CallingConv::X86_64_SysV))330 return false;331 332 unsigned AdjStackDown = TII.getCallFrameSetupOpcode();333 auto CallSeqStart = MIRBuilder.buildInstr(AdjStackDown);334 335 // Create a temporarily-floating call instruction so we can add the implicit336 // uses of arg registers.337 bool Is64Bit = STI.is64Bit();338 unsigned CallOpc = Info.Callee.isReg()339 ? (Is64Bit ? X86::CALL64r : X86::CALL32r)340 : (Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32);341 342 auto MIB = MIRBuilder.buildInstrNoInsert(CallOpc)343 .add(Info.Callee)344 .addRegMask(TRI->getCallPreservedMask(MF, Info.CallConv));345 346 SmallVector<ArgInfo, 8> SplitArgs;347 for (const auto &OrigArg : Info.OrigArgs) {348 349 // TODO: handle not simple cases.350 if (OrigArg.Flags[0].isByVal())351 return false;352 353 if (OrigArg.Regs.size() > 1)354 return false;355 356 splitToValueTypes(OrigArg, SplitArgs, DL, Info.CallConv);357 }358 // Do the actual argument marshalling.359 X86OutgoingValueAssigner Assigner(CC_X86);360 X86OutgoingValueHandler Handler(MIRBuilder, MRI, MIB);361 if (!determineAndHandleAssignments(Handler, Assigner, SplitArgs, MIRBuilder,362 Info.CallConv, Info.IsVarArg))363 return false;364 365 bool IsFixed =366 Info.OrigArgs.empty() ? true : !Info.OrigArgs.back().Flags[0].isVarArg();367 if (STI.is64Bit() && !IsFixed && !STI.isCallingConvWin64(Info.CallConv)) {368 // From AMD64 ABI document:369 // For calls that may call functions that use varargs or stdargs370 // (prototype-less calls or calls to functions containing ellipsis (...) in371 // the declaration) %al is used as hidden argument to specify the number372 // of SSE registers used. The contents of %al do not need to match exactly373 // the number of registers, but must be an ubound on the number of SSE374 // registers used and is in the range 0 - 8 inclusive.375 376 MIRBuilder.buildInstr(X86::MOV8ri)377 .addDef(X86::AL)378 .addImm(Assigner.getNumXmmRegs());379 MIB.addUse(X86::AL, RegState::Implicit);380 }381 382 // Now we can add the actual call instruction to the correct basic block.383 MIRBuilder.insertInstr(MIB);384 385 // If Callee is a reg, since it is used by a target specific386 // instruction, it must have a register class matching the387 // constraint of that instruction.388 if (Info.Callee.isReg())389 MIB->getOperand(0).setReg(constrainOperandRegClass(390 MF, *TRI, MRI, *MF.getSubtarget().getInstrInfo(),391 *MF.getSubtarget().getRegBankInfo(), *MIB, MIB->getDesc(), Info.Callee,392 0));393 394 // Finally we can copy the returned value back into its virtual-register. In395 // symmetry with the arguments, the physical register must be an396 // implicit-define of the call instruction.397 398 if (Info.CanLowerReturn && !Info.OrigRet.Ty->isVoidTy()) {399 if (Info.OrigRet.Regs.size() > 1)400 return false;401 402 SplitArgs.clear();403 SmallVector<Register, 8> NewRegs;404 405 splitToValueTypes(Info.OrigRet, SplitArgs, DL, Info.CallConv);406 407 X86OutgoingValueAssigner Assigner(RetCC_X86);408 CallReturnHandler Handler(MIRBuilder, MRI, MIB);409 if (!determineAndHandleAssignments(Handler, Assigner, SplitArgs, MIRBuilder,410 Info.CallConv, Info.IsVarArg))411 return false;412 413 if (!NewRegs.empty())414 MIRBuilder.buildMergeLikeInstr(Info.OrigRet.Regs[0], NewRegs);415 }416 417 CallSeqStart.addImm(Assigner.getStackSize())418 .addImm(0 /* see getFrameTotalSize */)419 .addImm(0 /* see getFrameAdjustment */);420 421 unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();422 MIRBuilder.buildInstr(AdjStackUp)423 .addImm(Assigner.getStackSize())424 .addImm(0 /* NumBytesForCalleeToPop */);425 426 if (!Info.CanLowerReturn)427 insertSRetLoads(MIRBuilder, Info.OrigRet.Ty, Info.OrigRet.Regs,428 Info.DemoteRegister, Info.DemoteStackIndex);429 430 return true;431}432