1424 lines · cpp
1//===- AArch64RegisterInfo.cpp - AArch64 Register Information -------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file contains the AArch64 implementation of the TargetRegisterInfo10// class.11//12//===----------------------------------------------------------------------===//13 14#include "AArch64RegisterInfo.h"15#include "AArch64FrameLowering.h"16#include "AArch64InstrInfo.h"17#include "AArch64MachineFunctionInfo.h"18#include "AArch64SMEAttributes.h"19#include "AArch64Subtarget.h"20#include "MCTargetDesc/AArch64AddressingModes.h"21#include "MCTargetDesc/AArch64InstPrinter.h"22#include "llvm/ADT/BitVector.h"23#include "llvm/BinaryFormat/Dwarf.h"24#include "llvm/CodeGen/LiveRegMatrix.h"25#include "llvm/CodeGen/MachineFrameInfo.h"26#include "llvm/CodeGen/MachineInstrBuilder.h"27#include "llvm/CodeGen/MachineRegisterInfo.h"28#include "llvm/CodeGen/RegisterScavenging.h"29#include "llvm/CodeGen/TargetFrameLowering.h"30#include "llvm/IR/DebugInfoMetadata.h"31#include "llvm/IR/DiagnosticInfo.h"32#include "llvm/IR/Function.h"33#include "llvm/Target/TargetOptions.h"34#include "llvm/TargetParser/Triple.h"35 36using namespace llvm;37 38#define GET_CC_REGISTER_LISTS39#include "AArch64GenCallingConv.inc"40#define GET_REGINFO_TARGET_DESC41#include "AArch64GenRegisterInfo.inc"42 43AArch64RegisterInfo::AArch64RegisterInfo(const Triple &TT, unsigned HwMode)44 : AArch64GenRegisterInfo(AArch64::LR, 0, 0, 0, HwMode), TT(TT) {45 AArch64_MC::initLLVMToCVRegMapping(this);46}47 48/// Return whether the register needs a CFI entry. Not all unwinders may know49/// about SVE registers, so we assume the lowest common denominator, i.e. the50/// callee-saves required by the base ABI. For the SVE registers z8-z15 only the51/// lower 64-bits (d8-d15) need to be saved. The lower 64-bits subreg is52/// returned in \p RegToUseForCFI.53bool AArch64RegisterInfo::regNeedsCFI(MCRegister Reg,54 MCRegister &RegToUseForCFI) const {55 if (AArch64::PPRRegClass.contains(Reg))56 return false;57 58 if (AArch64::ZPRRegClass.contains(Reg)) {59 RegToUseForCFI = getSubReg(Reg, AArch64::dsub);60 for (int I = 0; CSR_AArch64_AAPCS_SaveList[I]; ++I) {61 if (CSR_AArch64_AAPCS_SaveList[I] == RegToUseForCFI)62 return true;63 }64 return false;65 }66 67 RegToUseForCFI = Reg;68 return true;69}70 71const MCPhysReg *72AArch64RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {73 assert(MF && "Invalid MachineFunction pointer.");74 75 auto &AFI = *MF->getInfo<AArch64FunctionInfo>();76 const auto &F = MF->getFunction();77 const auto *TLI = MF->getSubtarget<AArch64Subtarget>().getTargetLowering();78 const bool Darwin = MF->getSubtarget<AArch64Subtarget>().isTargetDarwin();79 const bool Windows = MF->getSubtarget<AArch64Subtarget>().isTargetWindows();80 81 if (TLI->supportSwiftError() &&82 F.getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {83 if (Darwin)84 return CSR_Darwin_AArch64_AAPCS_SwiftError_SaveList;85 if (Windows)86 return CSR_Win_AArch64_AAPCS_SwiftError_SaveList;87 return CSR_AArch64_AAPCS_SwiftError_SaveList;88 }89 90 switch (F.getCallingConv()) {91 case CallingConv::GHC:92 // GHC set of callee saved regs is empty as all those regs are93 // used for passing STG regs around94 return CSR_AArch64_NoRegs_SaveList;95 96 case CallingConv::PreserveNone:97 // FIXME: Windows likely need this to be altered for properly unwinding.98 return CSR_AArch64_NoneRegs_SaveList;99 100 case CallingConv::AnyReg:101 return CSR_AArch64_AllRegs_SaveList;102 103 case CallingConv::ARM64EC_Thunk_X64:104 return CSR_Win_AArch64_Arm64EC_Thunk_SaveList;105 106 case CallingConv::PreserveMost:107 if (Darwin)108 return CSR_Darwin_AArch64_RT_MostRegs_SaveList;109 if (Windows)110 return CSR_Win_AArch64_RT_MostRegs_SaveList;111 return CSR_AArch64_RT_MostRegs_SaveList;112 113 case CallingConv::PreserveAll:114 if (Darwin)115 return CSR_Darwin_AArch64_RT_AllRegs_SaveList;116 if (Windows)117 return CSR_Win_AArch64_RT_AllRegs_SaveList;118 return CSR_AArch64_RT_AllRegs_SaveList;119 120 case CallingConv::CFGuard_Check:121 if (Darwin)122 report_fatal_error(123 "Calling convention CFGuard_Check is unsupported on Darwin.");124 return CSR_Win_AArch64_CFGuard_Check_SaveList;125 126 case CallingConv::SwiftTail:127 if (Darwin)128 return CSR_Darwin_AArch64_AAPCS_SwiftTail_SaveList;129 if (Windows)130 return CSR_Win_AArch64_AAPCS_SwiftTail_SaveList;131 return CSR_AArch64_AAPCS_SwiftTail_SaveList;132 133 case CallingConv::AArch64_VectorCall:134 if (Darwin)135 return CSR_Darwin_AArch64_AAVPCS_SaveList;136 if (Windows)137 return CSR_Win_AArch64_AAVPCS_SaveList;138 return CSR_AArch64_AAVPCS_SaveList;139 140 case CallingConv::AArch64_SVE_VectorCall:141 if (Darwin)142 report_fatal_error(143 "Calling convention SVE_VectorCall is unsupported on Darwin.");144 if (Windows)145 return CSR_Win_AArch64_SVE_AAPCS_SaveList;146 return CSR_AArch64_SVE_AAPCS_SaveList;147 148 case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0:149 report_fatal_error(150 "Calling convention "151 "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is only "152 "supported to improve calls to SME ACLE save/restore/disable-za "153 "functions, and is not intended to be used beyond that scope.");154 155 case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1:156 report_fatal_error(157 "Calling convention "158 "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1 is "159 "only supported to improve calls to SME ACLE __arm_get_current_vg "160 "function, and is not intended to be used beyond that scope.");161 162 case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2:163 report_fatal_error(164 "Calling convention "165 "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is "166 "only supported to improve calls to SME ACLE __arm_sme_state "167 "and is not intended to be used beyond that scope.");168 169 case CallingConv::Win64:170 if (Darwin)171 return CSR_Darwin_AArch64_AAPCS_Win64_SaveList;172 if (Windows)173 return CSR_Win_AArch64_AAPCS_SaveList;174 return CSR_AArch64_AAPCS_X18_SaveList;175 176 case CallingConv::CXX_FAST_TLS:177 if (Darwin)178 return AFI.isSplitCSR() ? CSR_Darwin_AArch64_CXX_TLS_PE_SaveList179 : CSR_Darwin_AArch64_CXX_TLS_SaveList;180 // FIXME: this likely should be a `report_fatal_error` condition, however,181 // that would be a departure from the previously implemented behaviour.182 LLVM_FALLTHROUGH;183 184 default:185 if (Darwin)186 return AFI.hasSVE_AAPCS(*MF) ? CSR_Darwin_AArch64_SVE_AAPCS_SaveList187 : CSR_Darwin_AArch64_AAPCS_SaveList;188 if (Windows)189 return AFI.hasSVE_AAPCS(*MF) ? CSR_Win_AArch64_SVE_AAPCS_SaveList190 : CSR_Win_AArch64_AAPCS_SaveList;191 return AFI.hasSVE_AAPCS(*MF) ? CSR_AArch64_SVE_AAPCS_SaveList192 : CSR_AArch64_AAPCS_SaveList;193 }194}195 196const MCPhysReg *AArch64RegisterInfo::getCalleeSavedRegsViaCopy(197 const MachineFunction *MF) const {198 assert(MF && "Invalid MachineFunction pointer.");199 if (MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS &&200 MF->getInfo<AArch64FunctionInfo>()->isSplitCSR())201 return CSR_Darwin_AArch64_CXX_TLS_ViaCopy_SaveList;202 return nullptr;203}204 205void AArch64RegisterInfo::UpdateCustomCalleeSavedRegs(206 MachineFunction &MF) const {207 const MCPhysReg *CSRs = getCalleeSavedRegs(&MF);208 SmallVector<MCPhysReg, 32> UpdatedCSRs;209 for (const MCPhysReg *I = CSRs; *I; ++I)210 UpdatedCSRs.push_back(*I);211 212 for (size_t i = 0; i < AArch64::GPR64commonRegClass.getNumRegs(); ++i) {213 if (MF.getSubtarget<AArch64Subtarget>().isXRegCustomCalleeSaved(i)) {214 UpdatedCSRs.push_back(AArch64::GPR64commonRegClass.getRegister(i));215 }216 }217 // Register lists are zero-terminated.218 UpdatedCSRs.push_back(0);219 MF.getRegInfo().setCalleeSavedRegs(UpdatedCSRs);220}221 222const TargetRegisterClass *223AArch64RegisterInfo::getSubClassWithSubReg(const TargetRegisterClass *RC,224 unsigned Idx) const {225 // edge case for GPR/FPR register classes226 if (RC == &AArch64::GPR32allRegClass && Idx == AArch64::hsub)227 return &AArch64::FPR32RegClass;228 else if (RC == &AArch64::GPR64allRegClass && Idx == AArch64::hsub)229 return &AArch64::FPR64RegClass;230 231 // Forward to TableGen's default version.232 return AArch64GenRegisterInfo::getSubClassWithSubReg(RC, Idx);233}234 235const uint32_t *236AArch64RegisterInfo::getDarwinCallPreservedMask(const MachineFunction &MF,237 CallingConv::ID CC) const {238 assert(MF.getSubtarget<AArch64Subtarget>().isTargetDarwin() &&239 "Invalid subtarget for getDarwinCallPreservedMask");240 241 if (CC == CallingConv::CXX_FAST_TLS)242 return CSR_Darwin_AArch64_CXX_TLS_RegMask;243 if (CC == CallingConv::AArch64_VectorCall)244 return CSR_Darwin_AArch64_AAVPCS_RegMask;245 if (CC == CallingConv::AArch64_SVE_VectorCall)246 return CSR_Darwin_AArch64_SVE_AAPCS_RegMask;247 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0)248 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask;249 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1)250 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1_RegMask;251 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2)252 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2_RegMask;253 if (CC == CallingConv::CFGuard_Check)254 report_fatal_error(255 "Calling convention CFGuard_Check is unsupported on Darwin.");256 if (MF.getSubtarget<AArch64Subtarget>()257 .getTargetLowering()258 ->supportSwiftError() &&259 MF.getFunction().getAttributes().hasAttrSomewhere(Attribute::SwiftError))260 return CSR_Darwin_AArch64_AAPCS_SwiftError_RegMask;261 if (CC == CallingConv::SwiftTail)262 return CSR_Darwin_AArch64_AAPCS_SwiftTail_RegMask;263 if (CC == CallingConv::PreserveMost)264 return CSR_Darwin_AArch64_RT_MostRegs_RegMask;265 if (CC == CallingConv::PreserveAll)266 return CSR_Darwin_AArch64_RT_AllRegs_RegMask;267 return CSR_Darwin_AArch64_AAPCS_RegMask;268}269 270const uint32_t *271AArch64RegisterInfo::getCallPreservedMask(const MachineFunction &MF,272 CallingConv::ID CC) const {273 bool SCS = MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack);274 if (CC == CallingConv::GHC)275 // This is academic because all GHC calls are (supposed to be) tail calls276 return SCS ? CSR_AArch64_NoRegs_SCS_RegMask : CSR_AArch64_NoRegs_RegMask;277 if (CC == CallingConv::PreserveNone)278 return SCS ? CSR_AArch64_NoneRegs_SCS_RegMask279 : CSR_AArch64_NoneRegs_RegMask;280 if (CC == CallingConv::AnyReg)281 return SCS ? CSR_AArch64_AllRegs_SCS_RegMask : CSR_AArch64_AllRegs_RegMask;282 283 // All the following calling conventions are handled differently on Darwin.284 if (MF.getSubtarget<AArch64Subtarget>().isTargetDarwin()) {285 if (SCS)286 report_fatal_error("ShadowCallStack attribute not supported on Darwin.");287 return getDarwinCallPreservedMask(MF, CC);288 }289 290 if (CC == CallingConv::AArch64_VectorCall)291 return SCS ? CSR_AArch64_AAVPCS_SCS_RegMask : CSR_AArch64_AAVPCS_RegMask;292 if (CC == CallingConv::AArch64_SVE_VectorCall)293 return SCS ? CSR_AArch64_SVE_AAPCS_SCS_RegMask294 : CSR_AArch64_SVE_AAPCS_RegMask;295 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0)296 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask;297 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1)298 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1_RegMask;299 if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2)300 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2_RegMask;301 if (CC == CallingConv::CFGuard_Check)302 return CSR_Win_AArch64_CFGuard_Check_RegMask;303 if (MF.getSubtarget<AArch64Subtarget>().getTargetLowering()304 ->supportSwiftError() &&305 MF.getFunction().getAttributes().hasAttrSomewhere(Attribute::SwiftError))306 return SCS ? CSR_AArch64_AAPCS_SwiftError_SCS_RegMask307 : CSR_AArch64_AAPCS_SwiftError_RegMask;308 if (CC == CallingConv::SwiftTail) {309 if (SCS)310 report_fatal_error("ShadowCallStack attribute not supported with swifttail");311 return CSR_AArch64_AAPCS_SwiftTail_RegMask;312 }313 if (CC == CallingConv::PreserveMost)314 return SCS ? CSR_AArch64_RT_MostRegs_SCS_RegMask315 : CSR_AArch64_RT_MostRegs_RegMask;316 if (CC == CallingConv::PreserveAll)317 return SCS ? CSR_AArch64_RT_AllRegs_SCS_RegMask318 : CSR_AArch64_RT_AllRegs_RegMask;319 320 return SCS ? CSR_AArch64_AAPCS_SCS_RegMask : CSR_AArch64_AAPCS_RegMask;321}322 323const uint32_t *AArch64RegisterInfo::getCustomEHPadPreservedMask(324 const MachineFunction &MF) const {325 if (MF.getSubtarget<AArch64Subtarget>().isTargetLinux())326 return CSR_AArch64_AAPCS_RegMask;327 328 return nullptr;329}330 331const uint32_t *AArch64RegisterInfo::getTLSCallPreservedMask() const {332 if (TT.isOSDarwin())333 return CSR_Darwin_AArch64_TLS_RegMask;334 335 assert(TT.isOSBinFormatELF() && "Invalid target");336 return CSR_AArch64_TLS_ELF_RegMask;337}338 339void AArch64RegisterInfo::UpdateCustomCallPreservedMask(MachineFunction &MF,340 const uint32_t **Mask) const {341 uint32_t *UpdatedMask = MF.allocateRegMask();342 unsigned RegMaskSize = MachineOperand::getRegMaskSize(getNumRegs());343 memcpy(UpdatedMask, *Mask, sizeof(UpdatedMask[0]) * RegMaskSize);344 345 for (size_t i = 0; i < AArch64::GPR64commonRegClass.getNumRegs(); ++i) {346 if (MF.getSubtarget<AArch64Subtarget>().isXRegCustomCalleeSaved(i)) {347 for (MCPhysReg SubReg :348 subregs_inclusive(AArch64::GPR64commonRegClass.getRegister(i))) {349 // See TargetRegisterInfo::getCallPreservedMask for how to interpret the350 // register mask.351 UpdatedMask[SubReg / 32] |= 1u << (SubReg % 32);352 }353 }354 }355 *Mask = UpdatedMask;356}357 358const uint32_t *AArch64RegisterInfo::getSMStartStopCallPreservedMask() const {359 return CSR_AArch64_SMStartStop_RegMask;360}361 362const uint32_t *363AArch64RegisterInfo::SMEABISupportRoutinesCallPreservedMaskFromX0() const {364 return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask;365}366 367const uint32_t *AArch64RegisterInfo::getNoPreservedMask() const {368 return CSR_AArch64_NoRegs_RegMask;369}370 371const uint32_t *372AArch64RegisterInfo::getThisReturnPreservedMask(const MachineFunction &MF,373 CallingConv::ID CC) const {374 // This should return a register mask that is the same as that returned by375 // getCallPreservedMask but that additionally preserves the register used for376 // the first i64 argument (which must also be the register used to return a377 // single i64 return value)378 //379 // In case that the calling convention does not use the same register for380 // both, the function should return NULL (does not currently apply)381 assert(CC != CallingConv::GHC && "should not be GHC calling convention.");382 if (MF.getSubtarget<AArch64Subtarget>().isTargetDarwin())383 return CSR_Darwin_AArch64_AAPCS_ThisReturn_RegMask;384 return CSR_AArch64_AAPCS_ThisReturn_RegMask;385}386 387const uint32_t *AArch64RegisterInfo::getWindowsStackProbePreservedMask() const {388 return CSR_AArch64_StackProbe_Windows_RegMask;389}390 391std::optional<std::string>392AArch64RegisterInfo::explainReservedReg(const MachineFunction &MF,393 MCRegister PhysReg) const {394 if (hasBasePointer(MF) && MCRegisterInfo::regsOverlap(PhysReg, AArch64::X19))395 return std::string("X19 is used as the frame base pointer register.");396 397 if (MF.getSubtarget<AArch64Subtarget>().isWindowsArm64EC()) {398 bool warn = false;399 if (MCRegisterInfo::regsOverlap(PhysReg, AArch64::X13) ||400 MCRegisterInfo::regsOverlap(PhysReg, AArch64::X14) ||401 MCRegisterInfo::regsOverlap(PhysReg, AArch64::X23) ||402 MCRegisterInfo::regsOverlap(PhysReg, AArch64::X24) ||403 MCRegisterInfo::regsOverlap(PhysReg, AArch64::X28))404 warn = true;405 406 for (unsigned i = AArch64::B16; i <= AArch64::B31; ++i)407 if (MCRegisterInfo::regsOverlap(PhysReg, i))408 warn = true;409 410 if (warn)411 return std::string(AArch64InstPrinter::getRegisterName(PhysReg)) +412 " is clobbered by asynchronous signals when using Arm64EC.";413 }414 415 return {};416}417 418BitVector419AArch64RegisterInfo::getStrictlyReservedRegs(const MachineFunction &MF) const {420 const AArch64FrameLowering *TFI = getFrameLowering(MF);421 422 // FIXME: avoid re-calculating this every time.423 BitVector Reserved(getNumRegs());424 markSuperRegs(Reserved, AArch64::WSP);425 markSuperRegs(Reserved, AArch64::WZR);426 427 if (TFI->isFPReserved(MF))428 markSuperRegs(Reserved, AArch64::W29);429 430 if (MF.getSubtarget<AArch64Subtarget>().isWindowsArm64EC()) {431 // x13, x14, x23, x24, x28, and v16-v31 are clobbered by asynchronous432 // signals, so we can't ever use them.433 markSuperRegs(Reserved, AArch64::W13);434 markSuperRegs(Reserved, AArch64::W14);435 markSuperRegs(Reserved, AArch64::W23);436 markSuperRegs(Reserved, AArch64::W24);437 markSuperRegs(Reserved, AArch64::W28);438 for (unsigned i = AArch64::B16; i <= AArch64::B31; ++i)439 markSuperRegs(Reserved, i);440 }441 442 for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) {443 if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(i))444 markSuperRegs(Reserved, AArch64::GPR32commonRegClass.getRegister(i));445 }446 447 if (hasBasePointer(MF))448 markSuperRegs(Reserved, AArch64::W19);449 450 // SLH uses register W16/X16 as the taint register.451 if (MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening))452 markSuperRegs(Reserved, AArch64::W16);453 454 // FFR is modelled as global state that cannot be allocated.455 if (MF.getSubtarget<AArch64Subtarget>().hasSVE())456 Reserved.set(AArch64::FFR);457 458 // SME tiles are not allocatable.459 if (MF.getSubtarget<AArch64Subtarget>().hasSME()) {460 for (MCPhysReg SubReg : subregs_inclusive(AArch64::ZA))461 Reserved.set(SubReg);462 }463 464 // VG cannot be allocated465 Reserved.set(AArch64::VG);466 467 if (MF.getSubtarget<AArch64Subtarget>().hasSME2()) {468 for (MCSubRegIterator SubReg(AArch64::ZT0, this, /*self=*/true);469 SubReg.isValid(); ++SubReg)470 Reserved.set(*SubReg);471 }472 473 markSuperRegs(Reserved, AArch64::FPCR);474 markSuperRegs(Reserved, AArch64::FPMR);475 markSuperRegs(Reserved, AArch64::FPSR);476 477 if (MF.getFunction().getCallingConv() == CallingConv::GRAAL) {478 markSuperRegs(Reserved, AArch64::X27);479 markSuperRegs(Reserved, AArch64::X28);480 markSuperRegs(Reserved, AArch64::W27);481 markSuperRegs(Reserved, AArch64::W28);482 }483 484 assert(checkAllSuperRegsMarked(Reserved));485 486 // Add _HI registers after checkAllSuperRegsMarked as this check otherwise487 // becomes considerably more expensive.488 Reserved.set(AArch64::WSP_HI);489 Reserved.set(AArch64::WZR_HI);490 static_assert(AArch64::W30_HI - AArch64::W0_HI == 30,491 "Unexpected order of registers");492 Reserved.set(AArch64::W0_HI, AArch64::W30_HI);493 static_assert(AArch64::B31_HI - AArch64::B0_HI == 31,494 "Unexpected order of registers");495 Reserved.set(AArch64::B0_HI, AArch64::B31_HI);496 static_assert(AArch64::H31_HI - AArch64::H0_HI == 31,497 "Unexpected order of registers");498 Reserved.set(AArch64::H0_HI, AArch64::H31_HI);499 static_assert(AArch64::S31_HI - AArch64::S0_HI == 31,500 "Unexpected order of registers");501 Reserved.set(AArch64::S0_HI, AArch64::S31_HI);502 static_assert(AArch64::D31_HI - AArch64::D0_HI == 31,503 "Unexpected order of registers");504 Reserved.set(AArch64::D0_HI, AArch64::D31_HI);505 static_assert(AArch64::Q31_HI - AArch64::Q0_HI == 31,506 "Unexpected order of registers");507 Reserved.set(AArch64::Q0_HI, AArch64::Q31_HI);508 509 return Reserved;510}511 512BitVector513AArch64RegisterInfo::getUserReservedRegs(const MachineFunction &MF) const {514 BitVector Reserved(getNumRegs());515 for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) {516 // ReserveXRegister is set for registers manually reserved517 // through +reserve-x#i.518 if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(i))519 markSuperRegs(Reserved, AArch64::GPR32commonRegClass.getRegister(i));520 }521 return Reserved;522}523 524BitVector525AArch64RegisterInfo::getReservedRegs(const MachineFunction &MF) const {526 BitVector Reserved(getNumRegs());527 for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) {528 if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReservedForRA(i))529 markSuperRegs(Reserved, AArch64::GPR32commonRegClass.getRegister(i));530 }531 532 if (MF.getSubtarget<AArch64Subtarget>().isLRReservedForRA()) {533 // In order to prevent the register allocator from using LR, we need to534 // mark it as reserved. However we don't want to keep it reserved throughout535 // the pipeline since it prevents other infrastructure from reasoning about536 // it's liveness. We use the NoVRegs property instead of IsSSA because537 // IsSSA is removed before VirtRegRewriter runs.538 if (!MF.getProperties().hasNoVRegs())539 markSuperRegs(Reserved, AArch64::LR);540 }541 542 assert(checkAllSuperRegsMarked(Reserved));543 544 // Handle strictlyReservedRegs separately to avoid re-evaluating the assert,545 // which becomes considerably expensive when considering the _HI registers.546 Reserved |= getStrictlyReservedRegs(MF);547 548 return Reserved;549}550 551bool AArch64RegisterInfo::isReservedReg(const MachineFunction &MF,552 MCRegister Reg) const {553 return getReservedRegs(MF)[Reg];554}555 556bool AArch64RegisterInfo::isUserReservedReg(const MachineFunction &MF,557 MCRegister Reg) const {558 return getUserReservedRegs(MF)[Reg];559}560 561bool AArch64RegisterInfo::isStrictlyReservedReg(const MachineFunction &MF,562 MCRegister Reg) const {563 return getStrictlyReservedRegs(MF)[Reg];564}565 566bool AArch64RegisterInfo::isAnyArgRegReserved(const MachineFunction &MF) const {567 return llvm::any_of(*AArch64::GPR64argRegClass.MC, [this, &MF](MCPhysReg r) {568 return isStrictlyReservedReg(MF, r);569 });570}571 572void AArch64RegisterInfo::emitReservedArgRegCallError(573 const MachineFunction &MF) const {574 const Function &F = MF.getFunction();575 F.getContext().diagnose(DiagnosticInfoUnsupported{F, ("AArch64 doesn't support"576 " function calls if any of the argument registers is reserved.")});577}578 579bool AArch64RegisterInfo::isAsmClobberable(const MachineFunction &MF,580 MCRegister PhysReg) const {581 // SLH uses register X16 as the taint register but it will fallback to a different582 // method if the user clobbers it. So X16 is not reserved for inline asm but is583 // for normal codegen.584 if (MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening) &&585 MCRegisterInfo::regsOverlap(PhysReg, AArch64::X16))586 return true;587 588 // ZA/ZT0 registers are reserved but may be permitted in the clobber list.589 if (PhysReg == AArch64::ZA || PhysReg == AArch64::ZT0)590 return true;591 592 return !isReservedReg(MF, PhysReg);593}594 595const TargetRegisterClass *596AArch64RegisterInfo::getPointerRegClass(unsigned Kind) const {597 return &AArch64::GPR64spRegClass;598}599 600const TargetRegisterClass *601AArch64RegisterInfo::getCrossCopyRegClass(const TargetRegisterClass *RC) const {602 if (RC == &AArch64::CCRRegClass)603 return &AArch64::GPR64RegClass; // Only MSR & MRS copy NZCV.604 return RC;605}606 607MCRegister AArch64RegisterInfo::getBaseRegister() const { return AArch64::X19; }608 609bool AArch64RegisterInfo::hasBasePointer(const MachineFunction &MF) const {610 const MachineFrameInfo &MFI = MF.getFrameInfo();611 612 // In the presence of variable sized objects or funclets, if the fixed stack613 // size is large enough that referencing from the FP won't result in things614 // being in range relatively often, we can use a base pointer to allow access615 // from the other direction like the SP normally works.616 //617 // Furthermore, if both variable sized objects are present, and the618 // stack needs to be dynamically re-aligned, the base pointer is the only619 // reliable way to reference the locals.620 if (MFI.hasVarSizedObjects() || MF.hasEHFunclets()) {621 if (hasStackRealignment(MF))622 return true;623 624 auto &ST = MF.getSubtarget<AArch64Subtarget>();625 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();626 if (ST.hasSVE() || ST.isStreaming()) {627 // Frames that have variable sized objects and scalable SVE objects,628 // should always use a basepointer.629 if (!AFI->hasCalculatedStackSizeSVE() || AFI->hasSVEStackSize())630 return true;631 }632 633 // Frames with hazard padding can have a large offset between the frame634 // pointer and GPR locals, which includes the emergency spill slot. If the635 // emergency spill slot is not within range of the load/store instructions636 // (which have a signed 9-bit range), we will fail to compile if it is used.637 // Since hasBasePointer() is called before we know if we have hazard padding638 // or an emergency spill slot we need to enable the basepointer639 // conservatively.640 if (ST.getStreamingHazardSize() &&641 !AFI->getSMEFnAttrs().hasNonStreamingInterfaceAndBody()) {642 return true;643 }644 645 // Conservatively estimate whether the negative offset from the frame646 // pointer will be sufficient to reach. If a function has a smallish647 // frame, it's less likely to have lots of spills and callee saved648 // space, so it's all more likely to be within range of the frame pointer.649 // If it's wrong, we'll materialize the constant and still get to the650 // object; it's just suboptimal. Negative offsets use the unscaled651 // load/store instructions, which have a 9-bit signed immediate.652 return MFI.getLocalFrameSize() >= 256;653 }654 655 return false;656}657 658bool AArch64RegisterInfo::isArgumentRegister(const MachineFunction &MF,659 MCRegister Reg) const {660 CallingConv::ID CC = MF.getFunction().getCallingConv();661 const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();662 bool IsVarArg = STI.isCallingConvWin64(MF.getFunction().getCallingConv(),663 MF.getFunction().isVarArg());664 665 auto HasReg = [](ArrayRef<MCRegister> RegList, MCRegister Reg) {666 return llvm::is_contained(RegList, Reg);667 };668 669 switch (CC) {670 default:671 report_fatal_error("Unsupported calling convention.");672 case CallingConv::GHC:673 return HasReg(CC_AArch64_GHC_ArgRegs, Reg);674 case CallingConv::PreserveNone:675 if (!MF.getFunction().isVarArg())676 return HasReg(CC_AArch64_Preserve_None_ArgRegs, Reg);677 [[fallthrough]];678 case CallingConv::C:679 case CallingConv::Fast:680 case CallingConv::PreserveMost:681 case CallingConv::PreserveAll:682 case CallingConv::CXX_FAST_TLS:683 case CallingConv::Swift:684 case CallingConv::SwiftTail:685 case CallingConv::Tail:686 if (STI.isTargetWindows()) {687 if (IsVarArg)688 return HasReg(CC_AArch64_Win64_VarArg_ArgRegs, Reg);689 switch (CC) {690 default:691 return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg);692 case CallingConv::Swift:693 case CallingConv::SwiftTail:694 return HasReg(CC_AArch64_Win64PCS_Swift_ArgRegs, Reg) ||695 HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg);696 }697 }698 if (!STI.isTargetDarwin()) {699 switch (CC) {700 default:701 return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg);702 case CallingConv::Swift:703 case CallingConv::SwiftTail:704 return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg) ||705 HasReg(CC_AArch64_AAPCS_Swift_ArgRegs, Reg);706 }707 }708 if (!IsVarArg) {709 switch (CC) {710 default:711 return HasReg(CC_AArch64_DarwinPCS_ArgRegs, Reg);712 case CallingConv::Swift:713 case CallingConv::SwiftTail:714 return HasReg(CC_AArch64_DarwinPCS_ArgRegs, Reg) ||715 HasReg(CC_AArch64_DarwinPCS_Swift_ArgRegs, Reg);716 }717 }718 if (STI.isTargetILP32())719 return HasReg(CC_AArch64_DarwinPCS_ILP32_VarArg_ArgRegs, Reg);720 return HasReg(CC_AArch64_DarwinPCS_VarArg_ArgRegs, Reg);721 case CallingConv::Win64:722 if (IsVarArg)723 HasReg(CC_AArch64_Win64_VarArg_ArgRegs, Reg);724 return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg);725 case CallingConv::CFGuard_Check:726 return HasReg(CC_AArch64_Win64_CFGuard_Check_ArgRegs, Reg);727 case CallingConv::AArch64_VectorCall:728 case CallingConv::AArch64_SVE_VectorCall:729 case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0:730 case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1:731 case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2:732 if (STI.isTargetWindows())733 return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg);734 return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg);735 }736}737 738Register739AArch64RegisterInfo::getFrameRegister(const MachineFunction &MF) const {740 const AArch64FrameLowering *TFI = getFrameLowering(MF);741 return TFI->hasFP(MF) ? AArch64::FP : AArch64::SP;742}743 744bool AArch64RegisterInfo::requiresRegisterScavenging(745 const MachineFunction &MF) const {746 return true;747}748 749bool AArch64RegisterInfo::requiresVirtualBaseRegisters(750 const MachineFunction &MF) const {751 return true;752}753 754bool755AArch64RegisterInfo::useFPForScavengingIndex(const MachineFunction &MF) const {756 // This function indicates whether the emergency spillslot should be placed757 // close to the beginning of the stackframe (closer to FP) or the end758 // (closer to SP).759 //760 // The beginning works most reliably if we have a frame pointer.761 // In the presence of any non-constant space between FP and locals,762 // (e.g. in case of stack realignment or a scalable SVE area), it is763 // better to use SP or BP.764 const AArch64FrameLowering &TFI = *getFrameLowering(MF);765 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();766 assert((!MF.getSubtarget<AArch64Subtarget>().hasSVE() ||767 AFI->hasCalculatedStackSizeSVE()) &&768 "Expected SVE area to be calculated by this point");769 return TFI.hasFP(MF) && !hasStackRealignment(MF) && !AFI->hasSVEStackSize() &&770 !AFI->hasStackHazardSlotIndex();771}772 773bool AArch64RegisterInfo::requiresFrameIndexScavenging(774 const MachineFunction &MF) const {775 return true;776}777 778bool779AArch64RegisterInfo::cannotEliminateFrame(const MachineFunction &MF) const {780 const MachineFrameInfo &MFI = MF.getFrameInfo();781 if (MF.getTarget().Options.DisableFramePointerElim(MF) && MFI.adjustsStack())782 return true;783 return MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken();784}785 786/// needsFrameBaseReg - Returns true if the instruction's frame index787/// reference would be better served by a base register other than FP788/// or SP. Used by LocalStackFrameAllocation to determine which frame index789/// references it should create new base registers for.790bool AArch64RegisterInfo::needsFrameBaseReg(MachineInstr *MI,791 int64_t Offset) const {792 for (unsigned i = 0; !MI->getOperand(i).isFI(); ++i)793 assert(i < MI->getNumOperands() &&794 "Instr doesn't have FrameIndex operand!");795 796 // It's the load/store FI references that cause issues, as it can be difficult797 // to materialize the offset if it won't fit in the literal field. Estimate798 // based on the size of the local frame and some conservative assumptions799 // about the rest of the stack frame (note, this is pre-regalloc, so800 // we don't know everything for certain yet) whether this offset is likely801 // to be out of range of the immediate. Return true if so.802 803 // We only generate virtual base registers for loads and stores, so804 // return false for everything else.805 if (!MI->mayLoad() && !MI->mayStore())806 return false;807 808 // Without a virtual base register, if the function has variable sized809 // objects, all fixed-size local references will be via the frame pointer,810 // Approximate the offset and see if it's legal for the instruction.811 // Note that the incoming offset is based on the SP value at function entry,812 // so it'll be negative.813 MachineFunction &MF = *MI->getParent()->getParent();814 const AArch64FrameLowering *TFI = getFrameLowering(MF);815 MachineFrameInfo &MFI = MF.getFrameInfo();816 817 // Estimate an offset from the frame pointer.818 // Conservatively assume all GPR callee-saved registers get pushed.819 // FP, LR, X19-X28, D8-D15. 64-bits each.820 int64_t FPOffset = Offset - 16 * 20;821 // Estimate an offset from the stack pointer.822 // The incoming offset is relating to the SP at the start of the function,823 // but when we access the local it'll be relative to the SP after local824 // allocation, so adjust our SP-relative offset by that allocation size.825 Offset += MFI.getLocalFrameSize();826 // Assume that we'll have at least some spill slots allocated.827 // FIXME: This is a total SWAG number. We should run some statistics828 // and pick a real one.829 Offset += 128; // 128 bytes of spill slots830 831 // If there is a frame pointer, try using it.832 // The FP is only available if there is no dynamic realignment. We833 // don't know for sure yet whether we'll need that, so we guess based834 // on whether there are any local variables that would trigger it.835 if (TFI->hasFP(MF) && isFrameOffsetLegal(MI, AArch64::FP, FPOffset))836 return false;837 838 // If we can reference via the stack pointer or base pointer, try that.839 // FIXME: This (and the code that resolves the references) can be improved840 // to only disallow SP relative references in the live range of841 // the VLA(s). In practice, it's unclear how much difference that842 // would make, but it may be worth doing.843 if (isFrameOffsetLegal(MI, AArch64::SP, Offset))844 return false;845 846 // If even offset 0 is illegal, we don't want a virtual base register.847 if (!isFrameOffsetLegal(MI, AArch64::SP, 0))848 return false;849 850 // The offset likely isn't legal; we want to allocate a virtual base register.851 return true;852}853 854bool AArch64RegisterInfo::isFrameOffsetLegal(const MachineInstr *MI,855 Register BaseReg,856 int64_t Offset) const {857 assert(MI && "Unable to get the legal offset for nil instruction.");858 StackOffset SaveOffset = StackOffset::getFixed(Offset);859 return isAArch64FrameOffsetLegal(*MI, SaveOffset) & AArch64FrameOffsetIsLegal;860}861 862/// Insert defining instruction(s) for BaseReg to be a pointer to FrameIdx863/// at the beginning of the basic block.864Register865AArch64RegisterInfo::materializeFrameBaseRegister(MachineBasicBlock *MBB,866 int FrameIdx,867 int64_t Offset) const {868 MachineBasicBlock::iterator Ins = MBB->begin();869 DebugLoc DL; // Defaults to "unknown"870 if (Ins != MBB->end())871 DL = Ins->getDebugLoc();872 const MachineFunction &MF = *MBB->getParent();873 const AArch64InstrInfo *TII =874 MF.getSubtarget<AArch64Subtarget>().getInstrInfo();875 const MCInstrDesc &MCID = TII->get(AArch64::ADDXri);876 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();877 Register BaseReg = MRI.createVirtualRegister(&AArch64::GPR64spRegClass);878 MRI.constrainRegClass(BaseReg, TII->getRegClass(MCID, 0));879 unsigned Shifter = AArch64_AM::getShifterImm(AArch64_AM::LSL, 0);880 881 BuildMI(*MBB, Ins, DL, MCID, BaseReg)882 .addFrameIndex(FrameIdx)883 .addImm(Offset)884 .addImm(Shifter);885 886 return BaseReg;887}888 889void AArch64RegisterInfo::resolveFrameIndex(MachineInstr &MI, Register BaseReg,890 int64_t Offset) const {891 // ARM doesn't need the general 64-bit offsets892 StackOffset Off = StackOffset::getFixed(Offset);893 894 unsigned i = 0;895 while (!MI.getOperand(i).isFI()) {896 ++i;897 assert(i < MI.getNumOperands() && "Instr doesn't have FrameIndex operand!");898 }899 900 const MachineFunction *MF = MI.getParent()->getParent();901 const AArch64InstrInfo *TII =902 MF->getSubtarget<AArch64Subtarget>().getInstrInfo();903 bool Done = rewriteAArch64FrameIndex(MI, i, BaseReg, Off, TII);904 assert(Done && "Unable to resolve frame index!");905 (void)Done;906}907 908// Create a scratch register for the frame index elimination in an instruction.909// This function has special handling of stack tagging loop pseudos, in which910// case it can also change the instruction opcode.911static Register912createScratchRegisterForInstruction(MachineInstr &MI, unsigned FIOperandNum,913 const AArch64InstrInfo *TII) {914 // ST*Gloop have a reserved scratch register in operand 1. Use it, and also915 // replace the instruction with the writeback variant because it will now916 // satisfy the operand constraints for it.917 Register ScratchReg;918 if (MI.getOpcode() == AArch64::STGloop ||919 MI.getOpcode() == AArch64::STZGloop) {920 assert(FIOperandNum == 3 &&921 "Wrong frame index operand for STGloop/STZGloop");922 unsigned Op = MI.getOpcode() == AArch64::STGloop ? AArch64::STGloop_wback923 : AArch64::STZGloop_wback;924 ScratchReg = MI.getOperand(1).getReg();925 MI.getOperand(3).ChangeToRegister(ScratchReg, false, false, true);926 MI.setDesc(TII->get(Op));927 MI.tieOperands(1, 3);928 } else {929 ScratchReg =930 MI.getMF()->getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass);931 MI.getOperand(FIOperandNum)932 .ChangeToRegister(ScratchReg, false, false, true);933 }934 return ScratchReg;935}936 937void AArch64RegisterInfo::getOffsetOpcodes(938 const StackOffset &Offset, SmallVectorImpl<uint64_t> &Ops) const {939 // The smallest scalable element supported by scaled SVE addressing940 // modes are predicates, which are 2 scalable bytes in size. So the scalable941 // byte offset must always be a multiple of 2.942 assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset");943 944 // Add fixed-sized offset using existing DIExpression interface.945 DIExpression::appendOffset(Ops, Offset.getFixed());946 947 unsigned VG = getDwarfRegNum(AArch64::VG, true);948 int64_t VGSized = Offset.getScalable() / 2;949 if (VGSized > 0) {950 Ops.push_back(dwarf::DW_OP_constu);951 Ops.push_back(VGSized);952 Ops.append({dwarf::DW_OP_bregx, VG, 0ULL});953 Ops.push_back(dwarf::DW_OP_mul);954 Ops.push_back(dwarf::DW_OP_plus);955 } else if (VGSized < 0) {956 Ops.push_back(dwarf::DW_OP_constu);957 Ops.push_back(-VGSized);958 Ops.append({dwarf::DW_OP_bregx, VG, 0ULL});959 Ops.push_back(dwarf::DW_OP_mul);960 Ops.push_back(dwarf::DW_OP_minus);961 }962}963 964bool AArch64RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,965 int SPAdj, unsigned FIOperandNum,966 RegScavenger *RS) const {967 assert(SPAdj == 0 && "Unexpected");968 969 MachineInstr &MI = *II;970 MachineBasicBlock &MBB = *MI.getParent();971 MachineFunction &MF = *MBB.getParent();972 const MachineFrameInfo &MFI = MF.getFrameInfo();973 const AArch64InstrInfo *TII =974 MF.getSubtarget<AArch64Subtarget>().getInstrInfo();975 const AArch64FrameLowering *TFI = getFrameLowering(MF);976 int FrameIndex = MI.getOperand(FIOperandNum).getIndex();977 bool Tagged =978 MI.getOperand(FIOperandNum).getTargetFlags() & AArch64II::MO_TAGGED;979 Register FrameReg;980 981 // Special handling of dbg_value, stackmap patchpoint statepoint instructions.982 if (MI.getOpcode() == TargetOpcode::STACKMAP ||983 MI.getOpcode() == TargetOpcode::PATCHPOINT ||984 MI.getOpcode() == TargetOpcode::STATEPOINT) {985 StackOffset Offset =986 TFI->resolveFrameIndexReference(MF, FrameIndex, FrameReg,987 /*PreferFP=*/true,988 /*ForSimm=*/false);989 Offset += StackOffset::getFixed(MI.getOperand(FIOperandNum + 1).getImm());990 MI.getOperand(FIOperandNum).ChangeToRegister(FrameReg, false /*isDef*/);991 MI.getOperand(FIOperandNum + 1).ChangeToImmediate(Offset.getFixed());992 return false;993 }994 995 if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE) {996 MachineOperand &FI = MI.getOperand(FIOperandNum);997 StackOffset Offset = TFI->getNonLocalFrameIndexReference(MF, FrameIndex);998 assert(!Offset.getScalable() &&999 "Frame offsets with a scalable component are not supported");1000 FI.ChangeToImmediate(Offset.getFixed());1001 return false;1002 }1003 1004 StackOffset Offset;1005 if (MI.getOpcode() == AArch64::TAGPstack) {1006 // TAGPstack must use the virtual frame register in its 3rd operand.1007 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();1008 FrameReg = MI.getOperand(3).getReg();1009 Offset = StackOffset::getFixed(MFI.getObjectOffset(FrameIndex) +1010 AFI->getTaggedBasePointerOffset());1011 } else if (Tagged) {1012 StackOffset SPOffset = StackOffset::getFixed(1013 MFI.getObjectOffset(FrameIndex) + (int64_t)MFI.getStackSize());1014 if (MFI.hasVarSizedObjects() ||1015 isAArch64FrameOffsetLegal(MI, SPOffset, nullptr, nullptr, nullptr) !=1016 (AArch64FrameOffsetCanUpdate | AArch64FrameOffsetIsLegal)) {1017 // Can't update to SP + offset in place. Precalculate the tagged pointer1018 // in a scratch register.1019 Offset = TFI->resolveFrameIndexReference(1020 MF, FrameIndex, FrameReg, /*PreferFP=*/false, /*ForSimm=*/true);1021 Register ScratchReg =1022 MF.getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass);1023 emitFrameOffset(MBB, II, MI.getDebugLoc(), ScratchReg, FrameReg, Offset,1024 TII);1025 BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(AArch64::LDG), ScratchReg)1026 .addReg(ScratchReg)1027 .addReg(ScratchReg)1028 .addImm(0);1029 MI.getOperand(FIOperandNum)1030 .ChangeToRegister(ScratchReg, false, false, true);1031 return false;1032 }1033 FrameReg = AArch64::SP;1034 Offset = StackOffset::getFixed(MFI.getObjectOffset(FrameIndex) +1035 (int64_t)MFI.getStackSize());1036 } else {1037 Offset = TFI->resolveFrameIndexReference(1038 MF, FrameIndex, FrameReg, /*PreferFP=*/false, /*ForSimm=*/true);1039 }1040 1041 // Modify MI as necessary to handle as much of 'Offset' as possible1042 if (rewriteAArch64FrameIndex(MI, FIOperandNum, FrameReg, Offset, TII))1043 return true;1044 1045 assert((!RS || !RS->isScavengingFrameIndex(FrameIndex)) &&1046 "Emergency spill slot is out of reach");1047 1048 // If we get here, the immediate doesn't fit into the instruction. We folded1049 // as much as possible above. Handle the rest, providing a register that is1050 // SP+LargeImm.1051 Register ScratchReg =1052 createScratchRegisterForInstruction(MI, FIOperandNum, TII);1053 emitFrameOffset(MBB, II, MI.getDebugLoc(), ScratchReg, FrameReg, Offset, TII);1054 return false;1055}1056 1057unsigned AArch64RegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,1058 MachineFunction &MF) const {1059 const AArch64FrameLowering *TFI = getFrameLowering(MF);1060 1061 switch (RC->getID()) {1062 default:1063 return 0;1064 case AArch64::GPR32RegClassID:1065 case AArch64::GPR32spRegClassID:1066 case AArch64::GPR32allRegClassID:1067 case AArch64::GPR64spRegClassID:1068 case AArch64::GPR64allRegClassID:1069 case AArch64::GPR64RegClassID:1070 case AArch64::GPR32commonRegClassID:1071 case AArch64::GPR64commonRegClassID:1072 return 32 - 1 // XZR/SP1073 - (TFI->hasFP(MF) || TT.isOSDarwin()) // FP1074 - MF.getSubtarget<AArch64Subtarget>().getNumXRegisterReserved()1075 - hasBasePointer(MF); // X191076 case AArch64::FPR8RegClassID:1077 case AArch64::FPR16RegClassID:1078 case AArch64::FPR32RegClassID:1079 case AArch64::FPR64RegClassID:1080 case AArch64::FPR128RegClassID:1081 return 32;1082 1083 case AArch64::MatrixIndexGPR32_8_11RegClassID:1084 case AArch64::MatrixIndexGPR32_12_15RegClassID:1085 return 4;1086 1087 case AArch64::DDRegClassID:1088 case AArch64::DDDRegClassID:1089 case AArch64::DDDDRegClassID:1090 case AArch64::QQRegClassID:1091 case AArch64::QQQRegClassID:1092 case AArch64::QQQQRegClassID:1093 return 32;1094 1095 case AArch64::FPR128_loRegClassID:1096 case AArch64::FPR64_loRegClassID:1097 case AArch64::FPR16_loRegClassID:1098 return 16;1099 case AArch64::FPR128_0to7RegClassID:1100 return 8;1101 }1102}1103 1104// We add regalloc hints for different cases:1105// * Choosing a better destination operand for predicated SVE instructions1106// where the inactive lanes are undef, by choosing a register that is not1107// unique to the other operands of the instruction.1108//1109// * Improve register allocation for SME multi-vector instructions where we can1110// benefit from the strided- and contiguous register multi-vector tuples.1111//1112// Here FORM_TRANSPOSED_REG_TUPLE nodes are created to improve register1113// allocation where a consecutive multi-vector tuple is constructed from the1114// same indices of multiple strided loads. This may still result in1115// unnecessary copies between the loads and the tuple. Here we try to return a1116// hint to assign the contiguous ZPRMulReg starting at the same register as1117// the first operand of the pseudo, which should be a subregister of the first1118// strided load.1119//1120// For example, if the first strided load has been assigned $z16_z20_z24_z281121// and the operands of the pseudo are each accessing subregister zsub2, we1122// should look through through Order to find a contiguous register which1123// begins with $z24 (i.e. $z24_z25_z26_z27).1124bool AArch64RegisterInfo::getRegAllocationHints(1125 Register VirtReg, ArrayRef<MCPhysReg> Order,1126 SmallVectorImpl<MCPhysReg> &Hints, const MachineFunction &MF,1127 const VirtRegMap *VRM, const LiveRegMatrix *Matrix) const {1128 auto &ST = MF.getSubtarget<AArch64Subtarget>();1129 const AArch64InstrInfo *TII =1130 MF.getSubtarget<AArch64Subtarget>().getInstrInfo();1131 const MachineRegisterInfo &MRI = MF.getRegInfo();1132 1133 // For predicated SVE instructions where the inactive lanes are undef,1134 // pick a destination register that is not unique to avoid introducing1135 // a movprfx.1136 const TargetRegisterClass *RegRC = MRI.getRegClass(VirtReg);1137 if (AArch64::ZPRRegClass.hasSubClassEq(RegRC)) {1138 bool ConsiderOnlyHints = TargetRegisterInfo::getRegAllocationHints(1139 VirtReg, Order, Hints, MF, VRM);1140 1141 for (const MachineOperand &DefOp : MRI.def_operands(VirtReg)) {1142 const MachineInstr &Def = *DefOp.getParent();1143 if (DefOp.isImplicit() ||1144 (TII->get(Def.getOpcode()).TSFlags & AArch64::FalseLanesMask) !=1145 AArch64::FalseLanesUndef)1146 continue;1147 1148 unsigned InstFlags =1149 TII->get(AArch64::getSVEPseudoMap(Def.getOpcode())).TSFlags;1150 1151 for (MCPhysReg R : Order) {1152 auto AddHintIfSuitable = [&](MCPhysReg R,1153 const MachineOperand &MO) -> bool {1154 // R is a suitable register hint if R can reuse one of the other1155 // source operands.1156 if (VRM->getPhys(MO.getReg()) != R)1157 return false;1158 Hints.push_back(R);1159 return true;1160 };1161 1162 switch (InstFlags & AArch64::DestructiveInstTypeMask) {1163 default:1164 break;1165 case AArch64::DestructiveTernaryCommWithRev:1166 AddHintIfSuitable(R, Def.getOperand(2)) ||1167 AddHintIfSuitable(R, Def.getOperand(3)) ||1168 AddHintIfSuitable(R, Def.getOperand(4));1169 break;1170 case AArch64::DestructiveBinaryComm:1171 case AArch64::DestructiveBinaryCommWithRev:1172 AddHintIfSuitable(R, Def.getOperand(2)) ||1173 AddHintIfSuitable(R, Def.getOperand(3));1174 break;1175 case AArch64::DestructiveBinary:1176 case AArch64::DestructiveBinaryImm:1177 AddHintIfSuitable(R, Def.getOperand(2));1178 break;1179 }1180 }1181 }1182 1183 if (Hints.size())1184 return ConsiderOnlyHints;1185 }1186 1187 if (!ST.hasSME() || !ST.isStreaming())1188 return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, MF,1189 VRM);1190 1191 // The SVE calling convention preserves registers Z8-Z23. As a result, there1192 // are no ZPR2Strided or ZPR4Strided registers that do not overlap with the1193 // callee-saved registers and so by default these will be pushed to the back1194 // of the allocation order for the ZPRStridedOrContiguous classes.1195 // If any of the instructions which define VirtReg are used by the1196 // FORM_TRANSPOSED_REG_TUPLE pseudo, we want to favour reducing copy1197 // instructions over reducing the number of clobbered callee-save registers,1198 // so we add the strided registers as a hint.1199 unsigned RegID = RegRC->getID();1200 if (RegID == AArch64::ZPR2StridedOrContiguousRegClassID ||1201 RegID == AArch64::ZPR4StridedOrContiguousRegClassID) {1202 1203 // Look through uses of the register for FORM_TRANSPOSED_REG_TUPLE.1204 for (const MachineInstr &Use : MRI.use_nodbg_instructions(VirtReg)) {1205 if (Use.getOpcode() != AArch64::FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO &&1206 Use.getOpcode() != AArch64::FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO)1207 continue;1208 1209 unsigned UseOps = Use.getNumOperands() - 1;1210 const TargetRegisterClass *StridedRC;1211 switch (RegID) {1212 case AArch64::ZPR2StridedOrContiguousRegClassID:1213 StridedRC = &AArch64::ZPR2StridedRegClass;1214 break;1215 case AArch64::ZPR4StridedOrContiguousRegClassID:1216 StridedRC = &AArch64::ZPR4StridedRegClass;1217 break;1218 default:1219 llvm_unreachable("Unexpected RegID");1220 }1221 1222 SmallVector<MCPhysReg, 4> StridedOrder;1223 for (MCPhysReg Reg : Order)1224 if (StridedRC->contains(Reg))1225 StridedOrder.push_back(Reg);1226 1227 int OpIdx = Use.findRegisterUseOperandIdx(VirtReg, this);1228 assert(OpIdx != -1 && "Expected operand index from register use.");1229 1230 unsigned TupleID = MRI.getRegClass(Use.getOperand(0).getReg())->getID();1231 bool IsMulZPR = TupleID == AArch64::ZPR2Mul2RegClassID ||1232 TupleID == AArch64::ZPR4Mul4RegClassID;1233 1234 const MachineOperand *AssignedRegOp = llvm::find_if(1235 make_range(Use.operands_begin() + 1, Use.operands_end()),1236 [&VRM](const MachineOperand &Op) {1237 return VRM->hasPhys(Op.getReg());1238 });1239 1240 // Example:1241 //1242 // When trying to find a suitable register allocation for VirtReg %v2 in:1243 //1244 // %v0:zpr2stridedorcontiguous = ld1 p0/z, [...]1245 // %v1:zpr2stridedorcontiguous = ld1 p0/z, [...]1246 // %v2:zpr2stridedorcontiguous = ld1 p0/z, [...]1247 // %v3:zpr2stridedorcontiguous = ld1 p0/z, [...]1248 // %v4:zpr4mul4 = FORM_TRANSPOSED_X4 %v0:0, %v1:0, %v2:0, %v3:01249 //1250 // One such suitable allocation would be:1251 //1252 // { z0, z8 } = ld1 p0/z, [...]1253 // { z1, z9 } = ld1 p0/z, [...]1254 // { z2, z10 } = ld1 p0/z, [...]1255 // { z3, z11 } = ld1 p0/z, [...]1256 // { z0, z1, z2, z3 } =1257 // FORM_TRANSPOSED_X4 {z0, z8}:0, {z1, z9}:0, {z2, z10}:0, {z3, z11}:01258 //1259 // Below we distinguish two cases when trying to find a register:1260 // * None of the registers used by FORM_TRANSPOSED_X4 have been assigned1261 // yet. In this case the code muse ensure that there are at least UseOps1262 // free consecutive registers. If IsMulZPR is true, then the first of1263 // registers must also be a multiple of UseOps, e.g. { z0, z1, z2, z3 }1264 // is valid but { z1, z2, z3, z5 } is not.1265 // * One or more of the registers used by FORM_TRANSPOSED_X4 is already1266 // assigned a physical register, which means only checking that a1267 // consecutive range of free tuple registers exists which includes1268 // the assigned register.1269 // e.g. in the example above, if { z0, z8 } is already allocated for1270 // %v0, we just need to ensure that { z1, z9 }, { z2, z10 } and1271 // { z3, z11 } are also free. If so, we add { z2, z10 }.1272 1273 if (AssignedRegOp == Use.operands_end()) {1274 // There are no registers already assigned to any of the pseudo1275 // operands. Look for a valid starting register for the group.1276 for (unsigned I = 0; I < StridedOrder.size(); ++I) {1277 MCPhysReg Reg = StridedOrder[I];1278 1279 // If the FORM_TRANSPOSE nodes use the ZPRMul classes, the starting1280 // register of the first load should be a multiple of 2 or 4.1281 unsigned SubRegIdx = Use.getOperand(OpIdx).getSubReg();1282 if (IsMulZPR && (getSubReg(Reg, SubRegIdx) - AArch64::Z0) % UseOps !=1283 ((unsigned)OpIdx - 1))1284 continue;1285 1286 // In the example above, if VirtReg is the third operand of the1287 // tuple (%v2) and Reg == Z2_Z10, then we need to make sure that1288 // Z0_Z8, Z1_Z9 and Z3_Z11 are also available.1289 auto IsFreeConsecutiveReg = [&](unsigned UseOp) {1290 unsigned R = Reg - (OpIdx - 1) + UseOp;1291 return StridedRC->contains(R) &&1292 (UseOp == 0 ||1293 ((getSubReg(R, AArch64::zsub0) - AArch64::Z0) ==1294 (getSubReg(R - 1, AArch64::zsub0) - AArch64::Z0) + 1)) &&1295 !Matrix->isPhysRegUsed(R);1296 };1297 if (all_of(iota_range<unsigned>(0U, UseOps, /*Inclusive=*/false),1298 IsFreeConsecutiveReg))1299 Hints.push_back(Reg);1300 }1301 } else {1302 // At least one operand already has a physical register assigned.1303 // Find the starting sub-register of this and use it to work out the1304 // correct strided register to suggest based on the current op index.1305 MCPhysReg TargetStartReg =1306 getSubReg(VRM->getPhys(AssignedRegOp->getReg()), AArch64::zsub0) +1307 (OpIdx - AssignedRegOp->getOperandNo());1308 1309 for (unsigned I = 0; I < StridedOrder.size(); ++I)1310 if (getSubReg(StridedOrder[I], AArch64::zsub0) == TargetStartReg)1311 Hints.push_back(StridedOrder[I]);1312 }1313 1314 if (!Hints.empty())1315 return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints,1316 MF, VRM);1317 }1318 }1319 1320 for (MachineInstr &MI : MRI.def_instructions(VirtReg)) {1321 if (MI.getOpcode() != AArch64::FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO &&1322 MI.getOpcode() != AArch64::FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO)1323 return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints,1324 MF, VRM);1325 1326 unsigned FirstOpSubReg = MI.getOperand(1).getSubReg();1327 switch (FirstOpSubReg) {1328 case AArch64::zsub0:1329 case AArch64::zsub1:1330 case AArch64::zsub2:1331 case AArch64::zsub3:1332 break;1333 default:1334 continue;1335 }1336 1337 // Look up the physical register mapped to the first operand of the pseudo.1338 Register FirstOpVirtReg = MI.getOperand(1).getReg();1339 if (!VRM->hasPhys(FirstOpVirtReg))1340 continue;1341 1342 MCRegister TupleStartReg =1343 getSubReg(VRM->getPhys(FirstOpVirtReg), FirstOpSubReg);1344 for (unsigned I = 0; I < Order.size(); ++I)1345 if (MCRegister R = getSubReg(Order[I], AArch64::zsub0))1346 if (R == TupleStartReg)1347 Hints.push_back(Order[I]);1348 }1349 1350 return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, MF,1351 VRM);1352}1353 1354unsigned AArch64RegisterInfo::getLocalAddressRegister(1355 const MachineFunction &MF) const {1356 const auto &MFI = MF.getFrameInfo();1357 if (!MF.hasEHFunclets() && !MFI.hasVarSizedObjects())1358 return AArch64::SP;1359 else if (hasStackRealignment(MF))1360 return getBaseRegister();1361 return getFrameRegister(MF);1362}1363 1364/// SrcRC and DstRC will be morphed into NewRC if this returns true1365bool AArch64RegisterInfo::shouldCoalesce(1366 MachineInstr *MI, const TargetRegisterClass *SrcRC, unsigned SubReg,1367 const TargetRegisterClass *DstRC, unsigned DstSubReg,1368 const TargetRegisterClass *NewRC, LiveIntervals &LIS) const {1369 MachineRegisterInfo &MRI = MI->getMF()->getRegInfo();1370 1371 if (MI->isCopy() &&1372 ((DstRC->getID() == AArch64::GPR64RegClassID) ||1373 (DstRC->getID() == AArch64::GPR64commonRegClassID)) &&1374 MI->getOperand(0).getSubReg() && MI->getOperand(1).getSubReg())1375 // Do not coalesce in the case of a 32-bit subregister copy1376 // which implements a 32 to 64 bit zero extension1377 // which relies on the upper 32 bits being zeroed.1378 return false;1379 1380 auto IsCoalescerBarrier = [](const MachineInstr &MI) {1381 switch (MI.getOpcode()) {1382 case AArch64::COALESCER_BARRIER_FPR16:1383 case AArch64::COALESCER_BARRIER_FPR32:1384 case AArch64::COALESCER_BARRIER_FPR64:1385 case AArch64::COALESCER_BARRIER_FPR128:1386 return true;1387 default:1388 return false;1389 }1390 };1391 1392 // For calls that temporarily have to toggle streaming mode as part of the1393 // call-sequence, we need to be more careful when coalescing copy instructions1394 // so that we don't end up coalescing the NEON/FP result or argument register1395 // with a whole Z-register, such that after coalescing the register allocator1396 // will try to spill/reload the entire Z register.1397 //1398 // We do this by checking if the node has any defs/uses that are1399 // COALESCER_BARRIER pseudos. These are 'nops' in practice, but they exist to1400 // instruct the coalescer to avoid coalescing the copy.1401 if (MI->isCopy() && SubReg != DstSubReg &&1402 (AArch64::ZPRRegClass.hasSubClassEq(DstRC) ||1403 AArch64::ZPRRegClass.hasSubClassEq(SrcRC))) {1404 unsigned SrcReg = MI->getOperand(1).getReg();1405 if (any_of(MRI.def_instructions(SrcReg), IsCoalescerBarrier))1406 return false;1407 unsigned DstReg = MI->getOperand(0).getReg();1408 if (any_of(MRI.use_nodbg_instructions(DstReg), IsCoalescerBarrier))1409 return false;1410 }1411 1412 return true;1413}1414 1415bool AArch64RegisterInfo::shouldAnalyzePhysregInMachineLoopInfo(1416 MCRegister R) const {1417 return R == AArch64::VG;1418}1419 1420bool AArch64RegisterInfo::isIgnoredCVReg(MCRegister LLVMReg) const {1421 return (LLVMReg >= AArch64::Z0 && LLVMReg <= AArch64::Z31) ||1422 (LLVMReg >= AArch64::P0 && LLVMReg <= AArch64::P15);1423}1424