333 lines · cpp
1//=== ARMCallingConv.cpp - ARM Custom CC Routines ---------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file contains the custom routines for the ARM Calling Convention that10// aren't done by tablegen, and includes the table generated implementations.11//12//===----------------------------------------------------------------------===//13 14#include "ARMCallingConv.h"15#include "ARM.h"16#include "ARMSubtarget.h"17using namespace llvm;18 19// APCS f64 is in register pairs, possibly split to stack20static bool f64AssignAPCS(unsigned ValNo, MVT ValVT, MVT LocVT,21 CCValAssign::LocInfo LocInfo,22 CCState &State, bool CanFail) {23 static const MCPhysReg RegList[] = { ARM::R0, ARM::R1, ARM::R2, ARM::R3 };24 25 // Try to get the first register.26 if (MCRegister Reg = State.AllocateReg(RegList))27 State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));28 else {29 // For the 2nd half of a v2f64, do not fail.30 if (CanFail)31 return false;32 33 // Put the whole thing on the stack.34 State.addLoc(CCValAssign::getCustomMem(35 ValNo, ValVT, State.AllocateStack(8, Align(4)), LocVT, LocInfo));36 return true;37 }38 39 // Try to get the second register.40 if (MCRegister Reg = State.AllocateReg(RegList))41 State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));42 else43 State.addLoc(CCValAssign::getCustomMem(44 ValNo, ValVT, State.AllocateStack(4, Align(4)), LocVT, LocInfo));45 return true;46}47 48static bool CC_ARM_APCS_Custom_f64(unsigned ValNo, MVT ValVT, MVT LocVT,49 CCValAssign::LocInfo LocInfo,50 ISD::ArgFlagsTy ArgFlags,51 CCState &State) {52 if (!f64AssignAPCS(ValNo, ValVT, LocVT, LocInfo, State, true))53 return false;54 if (LocVT == MVT::v2f64 &&55 !f64AssignAPCS(ValNo, ValVT, LocVT, LocInfo, State, false))56 return false;57 return true; // we handled it58}59 60// AAPCS f64 is in aligned register pairs61static bool f64AssignAAPCS(unsigned ValNo, MVT ValVT, MVT LocVT,62 CCValAssign::LocInfo LocInfo,63 CCState &State, bool CanFail) {64 static const MCPhysReg HiRegList[] = { ARM::R0, ARM::R2 };65 static const MCPhysReg LoRegList[] = { ARM::R1, ARM::R3 };66 static const MCPhysReg ShadowRegList[] = { ARM::R0, ARM::R1 };67 static const MCPhysReg GPRArgRegs[] = { ARM::R0, ARM::R1, ARM::R2, ARM::R3 };68 69 MCRegister Reg = State.AllocateReg(HiRegList, ShadowRegList);70 if (!Reg) {71 72 // If we had R3 unallocated only, now we still must to waste it.73 Reg = State.AllocateReg(GPRArgRegs);74 assert((!Reg || Reg == ARM::R3) && "Wrong GPRs usage for f64");75 76 // For the 2nd half of a v2f64, do not just fail.77 if (CanFail)78 return false;79 80 // Put the whole thing on the stack.81 State.addLoc(CCValAssign::getCustomMem(82 ValNo, ValVT, State.AllocateStack(8, Align(8)), LocVT, LocInfo));83 return true;84 }85 86 unsigned i;87 for (i = 0; i < 2; ++i)88 if (HiRegList[i] == Reg)89 break;90 91 MCRegister T = State.AllocateReg(LoRegList[i]);92 (void)T;93 assert(T == LoRegList[i] && "Could not allocate register");94 95 State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));96 State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, LoRegList[i],97 LocVT, LocInfo));98 return true;99}100 101static bool CC_ARM_AAPCS_Custom_f64(unsigned ValNo, MVT ValVT, MVT LocVT,102 CCValAssign::LocInfo LocInfo,103 ISD::ArgFlagsTy ArgFlags,104 CCState &State) {105 if (!f64AssignAAPCS(ValNo, ValVT, LocVT, LocInfo, State, true))106 return false;107 if (LocVT == MVT::v2f64 &&108 !f64AssignAAPCS(ValNo, ValVT, LocVT, LocInfo, State, false))109 return false;110 return true; // we handled it111}112 113static bool f64RetAssign(unsigned ValNo, MVT ValVT, MVT LocVT,114 CCValAssign::LocInfo LocInfo, CCState &State) {115 static const MCPhysReg HiRegList[] = { ARM::R0, ARM::R2 };116 static const MCPhysReg LoRegList[] = { ARM::R1, ARM::R3 };117 118 MCRegister Reg = State.AllocateReg(HiRegList, LoRegList);119 if (!Reg)120 return false; // we didn't handle it121 122 unsigned i;123 for (i = 0; i < 2; ++i)124 if (HiRegList[i] == Reg)125 break;126 127 State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));128 State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, LoRegList[i],129 LocVT, LocInfo));130 return true;131}132 133static bool RetCC_ARM_APCS_Custom_f64(unsigned ValNo, MVT ValVT, MVT LocVT,134 CCValAssign::LocInfo LocInfo,135 ISD::ArgFlagsTy ArgFlags,136 CCState &State) {137 if (!f64RetAssign(ValNo, ValVT, LocVT, LocInfo, State))138 return false;139 if (LocVT == MVT::v2f64 && !f64RetAssign(ValNo, ValVT, LocVT, LocInfo, State))140 return false;141 return true; // we handled it142}143 144static bool RetCC_ARM_AAPCS_Custom_f64(unsigned ValNo, MVT ValVT, MVT LocVT,145 CCValAssign::LocInfo LocInfo,146 ISD::ArgFlagsTy ArgFlags,147 CCState &State) {148 return RetCC_ARM_APCS_Custom_f64(ValNo, ValVT, LocVT, LocInfo, ArgFlags,149 State);150}151 152static const MCPhysReg RRegList[] = { ARM::R0, ARM::R1, ARM::R2, ARM::R3 };153 154static const MCPhysReg SRegList[] = { ARM::S0, ARM::S1, ARM::S2, ARM::S3,155 ARM::S4, ARM::S5, ARM::S6, ARM::S7,156 ARM::S8, ARM::S9, ARM::S10, ARM::S11,157 ARM::S12, ARM::S13, ARM::S14, ARM::S15 };158static const MCPhysReg DRegList[] = { ARM::D0, ARM::D1, ARM::D2, ARM::D3,159 ARM::D4, ARM::D5, ARM::D6, ARM::D7 };160static const MCPhysReg QRegList[] = { ARM::Q0, ARM::Q1, ARM::Q2, ARM::Q3 };161 162 163// Allocate part of an AAPCS HFA or HVA. We assume that each member of the HA164// has InConsecutiveRegs set, and that the last member also has165// InConsecutiveRegsLast set. We must process all members of the HA before166// we can allocate it, as we need to know the total number of registers that167// will be needed in order to (attempt to) allocate a contiguous block.168static bool CC_ARM_AAPCS_Custom_Aggregate(unsigned ValNo, MVT ValVT,169 MVT LocVT,170 CCValAssign::LocInfo LocInfo,171 ISD::ArgFlagsTy ArgFlags,172 CCState &State) {173 SmallVectorImpl<CCValAssign> &PendingMembers = State.getPendingLocs();174 175 // AAPCS HFAs must have 1-4 elements, all of the same type176 if (PendingMembers.size() > 0)177 assert(PendingMembers[0].getLocVT() == LocVT);178 179 // Add the argument to the list to be allocated once we know the size of the180 // aggregate. Store the type's required alignment as extra info for later: in181 // the [N x i64] case all trace has been removed by the time we actually get182 // to do allocation.183 PendingMembers.push_back(CCValAssign::getPending(184 ValNo, ValVT, LocVT, LocInfo, ArgFlags.getNonZeroOrigAlign().value()));185 186 if (!ArgFlags.isInConsecutiveRegsLast())187 return true;188 189 // Try to allocate a contiguous block of registers, each of the correct190 // size to hold one member.191 auto &DL = State.getMachineFunction().getDataLayout();192 const MaybeAlign StackAlign = DL.getStackAlignment();193 assert(StackAlign && "data layout string is missing stack alignment");194 const Align FirstMemberAlign(PendingMembers[0].getExtraInfo());195 Align Alignment = std::min(FirstMemberAlign, *StackAlign);196 197 ArrayRef<MCPhysReg> RegList;198 switch (LocVT.SimpleTy) {199 case MVT::i32: {200 RegList = RRegList;201 unsigned RegIdx = State.getFirstUnallocated(RegList);202 203 // First consume all registers that would give an unaligned object. Whether204 // we go on stack or in regs, no-one will be using them in future.205 unsigned RegAlign = alignTo(Alignment.value(), 4) / 4;206 while (RegIdx % RegAlign != 0 && RegIdx < RegList.size())207 State.AllocateReg(RegList[RegIdx++]);208 209 break;210 }211 case MVT::f16:212 case MVT::bf16:213 case MVT::f32:214 RegList = SRegList;215 break;216 case MVT::v4f16:217 case MVT::v4bf16:218 case MVT::f64:219 RegList = DRegList;220 break;221 case MVT::v8f16:222 case MVT::v8bf16:223 case MVT::v2f64:224 RegList = QRegList;225 break;226 default:227 llvm_unreachable("Unexpected member type for block aggregate");228 break;229 }230 231 ArrayRef<MCPhysReg> RegResult =232 State.AllocateRegBlock(RegList, PendingMembers.size());233 if (!RegResult.empty()) {234 for (const auto &[PendingMember, Reg] : zip(PendingMembers, RegResult)) {235 PendingMember.convertToReg(Reg);236 State.addLoc(PendingMember);237 }238 PendingMembers.clear();239 return true;240 }241 242 // Register allocation failed, we'll be needing the stack243 unsigned Size = LocVT.getSizeInBits() / 8;244 if (LocVT == MVT::i32 && State.getStackSize() == 0) {245 // If nothing else has used the stack until this point, a non-HFA aggregate246 // can be split between regs and stack.247 unsigned RegIdx = State.getFirstUnallocated(RegList);248 for (auto &It : PendingMembers) {249 if (RegIdx >= RegList.size())250 It.convertToMem(State.AllocateStack(Size, Align(Size)));251 else252 It.convertToReg(State.AllocateReg(RegList[RegIdx++]));253 254 State.addLoc(It);255 }256 PendingMembers.clear();257 return true;258 }259 260 if (LocVT != MVT::i32)261 RegList = SRegList;262 263 // Mark all regs as unavailable (AAPCS rule C.2.vfp for VFP, C.6 for core)264 for (auto Reg : RegList)265 State.AllocateReg(Reg);266 267 // Clamp the alignment between 4 and 8.268 if (State.getMachineFunction().getSubtarget<ARMSubtarget>().isTargetAEABI())269 Alignment = ArgFlags.getNonZeroMemAlign() <= 4 ? Align(4) : Align(8);270 271 // After the first item has been allocated, the rest are packed as tightly as272 // possible. (E.g. an incoming i64 would have starting Align of 8, but we'll273 // be allocating a bunch of i32 slots).274 for (auto &It : PendingMembers) {275 It.convertToMem(State.AllocateStack(Size, Alignment));276 State.addLoc(It);277 Alignment = Align(1);278 }279 280 // All pending members have now been allocated281 PendingMembers.clear();282 283 // This will be allocated by the last member of the aggregate284 return true;285}286 287static bool CustomAssignInRegList(unsigned ValNo, MVT ValVT, MVT LocVT,288 CCValAssign::LocInfo LocInfo, CCState &State,289 ArrayRef<MCPhysReg> RegList) {290 MCRegister Reg = State.AllocateReg(RegList);291 if (Reg) {292 State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));293 return true;294 }295 return false;296}297 298static bool CC_ARM_AAPCS_Custom_f16(unsigned ValNo, MVT ValVT, MVT LocVT,299 CCValAssign::LocInfo LocInfo,300 ISD::ArgFlagsTy ArgFlags, CCState &State) {301 // f16 and bf16 arguments are extended to i32 and assigned to a register in302 // [r0, r3].303 return CustomAssignInRegList(ValNo, ValVT, MVT::i32, LocInfo, State,304 RRegList);305}306 307static bool CC_ARM_AAPCS_VFP_Custom_f16(unsigned ValNo, MVT ValVT, MVT LocVT,308 CCValAssign::LocInfo LocInfo,309 ISD::ArgFlagsTy ArgFlags,310 CCState &State) {311 // f16 and bf16 arguments are extended to f32 and assigned to a register in312 // [s0, s15].313 return CustomAssignInRegList(ValNo, ValVT, MVT::f32, LocInfo, State,314 SRegList);315}316 317static bool CC_ARM_AAPCS_Common_Custom_f16_Stack(unsigned ValNo, MVT ValVT,318 MVT LocVT,319 CCValAssign::LocInfo LocInfo,320 ISD::ArgFlagsTy ArgFlags,321 CCState &State) {322 // f16 and bf16 (if not passed in a register) are assigned to a 32-bit stack323 // slot, with the most-significant 16 bits unspecified. The 32-bit slot is324 // important to make sure that the byte ordering is correct for big endian325 // targets.326 State.addLoc(CCValAssign::getCustomMem(327 ValNo, ValVT, State.AllocateStack(4, Align(4)), MVT::i32, LocInfo));328 return true;329}330 331// Include the table generated calling convention implementations.332#include "ARMGenCallingConv.inc"333