1255 lines · cpp
1//===- AArch64RegisterBankInfo.cpp ----------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8/// \file9/// This file implements the targeting of the RegisterBankInfo class for10/// AArch64.11/// \todo This should be generated by TableGen.12//===----------------------------------------------------------------------===//13 14#include "AArch64RegisterBankInfo.h"15#include "AArch64RegisterInfo.h"16#include "AArch64Subtarget.h"17#include "MCTargetDesc/AArch64MCTargetDesc.h"18#include "llvm/ADT/STLExtras.h"19#include "llvm/ADT/SmallVector.h"20#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"21#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"22#include "llvm/CodeGen/GlobalISel/Utils.h"23#include "llvm/CodeGen/LowLevelTypeUtils.h"24#include "llvm/CodeGen/MachineFunction.h"25#include "llvm/CodeGen/MachineInstr.h"26#include "llvm/CodeGen/MachineOperand.h"27#include "llvm/CodeGen/MachineRegisterInfo.h"28#include "llvm/CodeGen/RegisterBank.h"29#include "llvm/CodeGen/RegisterBankInfo.h"30#include "llvm/CodeGen/TargetOpcodes.h"31#include "llvm/CodeGen/TargetRegisterInfo.h"32#include "llvm/CodeGen/TargetSubtargetInfo.h"33#include "llvm/IR/IntrinsicsAArch64.h"34#include "llvm/Support/ErrorHandling.h"35#include "llvm/Support/Threading.h"36#include <cassert>37 38#define GET_TARGET_REGBANK_IMPL39#include "AArch64GenRegisterBank.inc"40 41// This file will be TableGen'ed at some point.42#include "AArch64GenRegisterBankInfo.def"43 44using namespace llvm;45static const unsigned CustomMappingID = 1;46 47AArch64RegisterBankInfo::AArch64RegisterBankInfo(48 const TargetRegisterInfo &TRI) {49 static llvm::once_flag InitializeRegisterBankFlag;50 51 static auto InitializeRegisterBankOnce = [&]() {52 // We have only one set of register banks, whatever the subtarget53 // is. Therefore, the initialization of the RegBanks table should be54 // done only once. Indeed the table of all register banks55 // (AArch64::RegBanks) is unique in the compiler. At some point, it56 // will get tablegen'ed and the whole constructor becomes empty.57 58 const RegisterBank &RBGPR = getRegBank(AArch64::GPRRegBankID);59 (void)RBGPR;60 assert(&AArch64::GPRRegBank == &RBGPR &&61 "The order in RegBanks is messed up");62 63 const RegisterBank &RBFPR = getRegBank(AArch64::FPRRegBankID);64 (void)RBFPR;65 assert(&AArch64::FPRRegBank == &RBFPR &&66 "The order in RegBanks is messed up");67 68 const RegisterBank &RBCCR = getRegBank(AArch64::CCRegBankID);69 (void)RBCCR;70 assert(&AArch64::CCRegBank == &RBCCR &&71 "The order in RegBanks is messed up");72 73 // The GPR register bank is fully defined by all the registers in74 // GR64all + its subclasses.75 assert(RBGPR.covers(*TRI.getRegClass(AArch64::GPR32RegClassID)) &&76 "Subclass not added?");77 assert(getMaximumSize(RBGPR.getID()) == 128 &&78 "GPRs should hold up to 128-bit");79 80 // The FPR register bank is fully defined by all the registers in81 // GR64all + its subclasses.82 assert(RBFPR.covers(*TRI.getRegClass(AArch64::QQRegClassID)) &&83 "Subclass not added?");84 assert(RBFPR.covers(*TRI.getRegClass(AArch64::FPR64RegClassID)) &&85 "Subclass not added?");86 assert(getMaximumSize(RBFPR.getID()) == 512 &&87 "FPRs should hold up to 512-bit via QQQQ sequence");88 89 assert(RBCCR.covers(*TRI.getRegClass(AArch64::CCRRegClassID)) &&90 "Class not added?");91 assert(getMaximumSize(RBCCR.getID()) == 32 &&92 "CCR should hold up to 32-bit");93 94 // Check that the TableGen'ed like file is in sync we our expectations.95 // First, the Idx.96 assert(checkPartialMappingIdx(PMI_FirstGPR, PMI_LastGPR,97 {PMI_GPR32, PMI_GPR64, PMI_GPR128}) &&98 "PartialMappingIdx's are incorrectly ordered");99 assert(checkPartialMappingIdx(PMI_FirstFPR, PMI_LastFPR,100 {PMI_FPR16, PMI_FPR32, PMI_FPR64, PMI_FPR128,101 PMI_FPR256, PMI_FPR512}) &&102 "PartialMappingIdx's are incorrectly ordered");103// Now, the content.104// Check partial mapping.105#define CHECK_PARTIALMAP(Idx, ValStartIdx, ValLength, RB) \106 do { \107 assert( \108 checkPartialMap(PartialMappingIdx::Idx, ValStartIdx, ValLength, RB) && \109 #Idx " is incorrectly initialized"); \110 } while (false)111 112 CHECK_PARTIALMAP(PMI_GPR32, 0, 32, RBGPR);113 CHECK_PARTIALMAP(PMI_GPR64, 0, 64, RBGPR);114 CHECK_PARTIALMAP(PMI_GPR128, 0, 128, RBGPR);115 CHECK_PARTIALMAP(PMI_FPR16, 0, 16, RBFPR);116 CHECK_PARTIALMAP(PMI_FPR32, 0, 32, RBFPR);117 CHECK_PARTIALMAP(PMI_FPR64, 0, 64, RBFPR);118 CHECK_PARTIALMAP(PMI_FPR128, 0, 128, RBFPR);119 CHECK_PARTIALMAP(PMI_FPR256, 0, 256, RBFPR);120 CHECK_PARTIALMAP(PMI_FPR512, 0, 512, RBFPR);121 122// Check value mapping.123#define CHECK_VALUEMAP_IMPL(RBName, Size, Offset) \124 do { \125 assert(checkValueMapImpl(PartialMappingIdx::PMI_##RBName##Size, \126 PartialMappingIdx::PMI_First##RBName, Size, \127 Offset) && \128 #RBName #Size " " #Offset " is incorrectly initialized"); \129 } while (false)130 131#define CHECK_VALUEMAP(RBName, Size) CHECK_VALUEMAP_IMPL(RBName, Size, 0)132 133 CHECK_VALUEMAP(GPR, 32);134 CHECK_VALUEMAP(GPR, 64);135 CHECK_VALUEMAP(GPR, 128);136 CHECK_VALUEMAP(FPR, 16);137 CHECK_VALUEMAP(FPR, 32);138 CHECK_VALUEMAP(FPR, 64);139 CHECK_VALUEMAP(FPR, 128);140 CHECK_VALUEMAP(FPR, 256);141 CHECK_VALUEMAP(FPR, 512);142 143// Check the value mapping for 3-operands instructions where all the operands144// map to the same value mapping.145#define CHECK_VALUEMAP_3OPS(RBName, Size) \146 do { \147 CHECK_VALUEMAP_IMPL(RBName, Size, 0); \148 CHECK_VALUEMAP_IMPL(RBName, Size, 1); \149 CHECK_VALUEMAP_IMPL(RBName, Size, 2); \150 } while (false)151 152 CHECK_VALUEMAP_3OPS(GPR, 32);153 CHECK_VALUEMAP_3OPS(GPR, 64);154 CHECK_VALUEMAP_3OPS(GPR, 128);155 CHECK_VALUEMAP_3OPS(FPR, 32);156 CHECK_VALUEMAP_3OPS(FPR, 64);157 CHECK_VALUEMAP_3OPS(FPR, 128);158 CHECK_VALUEMAP_3OPS(FPR, 256);159 CHECK_VALUEMAP_3OPS(FPR, 512);160 161#define CHECK_VALUEMAP_CROSSREGCPY(RBNameDst, RBNameSrc, Size) \162 do { \163 unsigned PartialMapDstIdx = PMI_##RBNameDst##Size - PMI_Min; \164 unsigned PartialMapSrcIdx = PMI_##RBNameSrc##Size - PMI_Min; \165 (void)PartialMapDstIdx; \166 (void)PartialMapSrcIdx; \167 const ValueMapping *Map = getCopyMapping(AArch64::RBNameDst##RegBankID, \168 AArch64::RBNameSrc##RegBankID, \169 TypeSize::getFixed(Size)); \170 (void)Map; \171 assert(Map[0].BreakDown == \172 &AArch64GenRegisterBankInfo::PartMappings[PartialMapDstIdx] && \173 Map[0].NumBreakDowns == 1 && \174 #RBNameDst #Size " Dst is incorrectly initialized"); \175 assert(Map[1].BreakDown == \176 &AArch64GenRegisterBankInfo::PartMappings[PartialMapSrcIdx] && \177 Map[1].NumBreakDowns == 1 && \178 #RBNameSrc #Size " Src is incorrectly initialized"); \179 \180 } while (false)181 182 CHECK_VALUEMAP_CROSSREGCPY(GPR, GPR, 32);183 CHECK_VALUEMAP_CROSSREGCPY(GPR, FPR, 32);184 CHECK_VALUEMAP_CROSSREGCPY(GPR, GPR, 64);185 CHECK_VALUEMAP_CROSSREGCPY(GPR, FPR, 64);186 CHECK_VALUEMAP_CROSSREGCPY(FPR, FPR, 32);187 CHECK_VALUEMAP_CROSSREGCPY(FPR, GPR, 32);188 CHECK_VALUEMAP_CROSSREGCPY(FPR, FPR, 64);189 CHECK_VALUEMAP_CROSSREGCPY(FPR, GPR, 64);190 191#define CHECK_VALUEMAP_FPEXT(DstSize, SrcSize) \192 do { \193 unsigned PartialMapDstIdx = PMI_FPR##DstSize - PMI_Min; \194 unsigned PartialMapSrcIdx = PMI_FPR##SrcSize - PMI_Min; \195 (void)PartialMapDstIdx; \196 (void)PartialMapSrcIdx; \197 const ValueMapping *Map = getFPExtMapping(DstSize, SrcSize); \198 (void)Map; \199 assert(Map[0].BreakDown == \200 &AArch64GenRegisterBankInfo::PartMappings[PartialMapDstIdx] && \201 Map[0].NumBreakDowns == 1 && "FPR" #DstSize \202 " Dst is incorrectly initialized"); \203 assert(Map[1].BreakDown == \204 &AArch64GenRegisterBankInfo::PartMappings[PartialMapSrcIdx] && \205 Map[1].NumBreakDowns == 1 && "FPR" #SrcSize \206 " Src is incorrectly initialized"); \207 \208 } while (false)209 210 CHECK_VALUEMAP_FPEXT(32, 16);211 CHECK_VALUEMAP_FPEXT(64, 16);212 CHECK_VALUEMAP_FPEXT(64, 32);213 CHECK_VALUEMAP_FPEXT(128, 64);214 215 assert(verify(TRI) && "Invalid register bank information");216 };217 218 llvm::call_once(InitializeRegisterBankFlag, InitializeRegisterBankOnce);219}220 221unsigned AArch64RegisterBankInfo::copyCost(const RegisterBank &A,222 const RegisterBank &B,223 const TypeSize Size) const {224 // What do we do with different size?225 // copy are same size.226 // Will introduce other hooks for different size:227 // * extract cost.228 // * build_sequence cost.229 230 // Copy from (resp. to) GPR to (resp. from) FPR involves FMOV.231 // FIXME: This should be deduced from the scheduling model.232 if (&A == &AArch64::GPRRegBank && &B == &AArch64::FPRRegBank)233 // FMOVXDr or FMOVWSr.234 return 5;235 if (&A == &AArch64::FPRRegBank && &B == &AArch64::GPRRegBank)236 // FMOVDXr or FMOVSWr.237 return 4;238 239 return RegisterBankInfo::copyCost(A, B, Size);240}241 242const RegisterBank &243AArch64RegisterBankInfo::getRegBankFromRegClass(const TargetRegisterClass &RC,244 LLT Ty) const {245 switch (RC.getID()) {246 case AArch64::GPR64sponlyRegClassID:247 return getRegBank(AArch64::GPRRegBankID);248 default:249 return AArch64GenRegisterBankInfo::getRegBankFromRegClass(RC, Ty);250 }251}252 253RegisterBankInfo::InstructionMappings254AArch64RegisterBankInfo::getInstrAlternativeMappings(255 const MachineInstr &MI) const {256 const MachineFunction &MF = *MI.getParent()->getParent();257 const TargetSubtargetInfo &STI = MF.getSubtarget();258 const TargetRegisterInfo &TRI = *STI.getRegisterInfo();259 const MachineRegisterInfo &MRI = MF.getRegInfo();260 261 switch (MI.getOpcode()) {262 case TargetOpcode::G_OR: {263 // 32 and 64-bit or can be mapped on either FPR or264 // GPR for the same cost.265 TypeSize Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI);266 if (Size != 32 && Size != 64)267 break;268 269 // If the instruction has any implicit-defs or uses,270 // do not mess with it.271 if (MI.getNumOperands() != 3)272 break;273 InstructionMappings AltMappings;274 const InstructionMapping &GPRMapping = getInstructionMapping(275 /*ID*/ 1, /*Cost*/ 1, getValueMapping(PMI_FirstGPR, Size),276 /*NumOperands*/ 3);277 const InstructionMapping &FPRMapping = getInstructionMapping(278 /*ID*/ 2, /*Cost*/ 1, getValueMapping(PMI_FirstFPR, Size),279 /*NumOperands*/ 3);280 281 AltMappings.push_back(&GPRMapping);282 AltMappings.push_back(&FPRMapping);283 return AltMappings;284 }285 case TargetOpcode::G_BITCAST: {286 TypeSize Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI);287 if (Size != 32 && Size != 64)288 break;289 290 // If the instruction has any implicit-defs or uses,291 // do not mess with it.292 if (MI.getNumOperands() != 2)293 break;294 295 InstructionMappings AltMappings;296 const InstructionMapping &GPRMapping = getInstructionMapping(297 /*ID*/ 1, /*Cost*/ 1,298 getCopyMapping(AArch64::GPRRegBankID, AArch64::GPRRegBankID, Size),299 /*NumOperands*/ 2);300 const InstructionMapping &FPRMapping = getInstructionMapping(301 /*ID*/ 2, /*Cost*/ 1,302 getCopyMapping(AArch64::FPRRegBankID, AArch64::FPRRegBankID, Size),303 /*NumOperands*/ 2);304 const InstructionMapping &GPRToFPRMapping = getInstructionMapping(305 /*ID*/ 3,306 /*Cost*/307 copyCost(AArch64::GPRRegBank, AArch64::FPRRegBank,308 TypeSize::getFixed(Size)),309 getCopyMapping(AArch64::FPRRegBankID, AArch64::GPRRegBankID, Size),310 /*NumOperands*/ 2);311 const InstructionMapping &FPRToGPRMapping = getInstructionMapping(312 /*ID*/ 3,313 /*Cost*/314 copyCost(AArch64::GPRRegBank, AArch64::FPRRegBank,315 TypeSize::getFixed(Size)),316 getCopyMapping(AArch64::GPRRegBankID, AArch64::FPRRegBankID, Size),317 /*NumOperands*/ 2);318 319 AltMappings.push_back(&GPRMapping);320 AltMappings.push_back(&FPRMapping);321 AltMappings.push_back(&GPRToFPRMapping);322 AltMappings.push_back(&FPRToGPRMapping);323 return AltMappings;324 }325 case TargetOpcode::G_LOAD: {326 TypeSize Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI);327 if (Size != 64)328 break;329 330 // If the instruction has any implicit-defs or uses,331 // do not mess with it.332 if (MI.getNumOperands() != 2)333 break;334 335 InstructionMappings AltMappings;336 const InstructionMapping &GPRMapping = getInstructionMapping(337 /*ID*/ 1, /*Cost*/ 1,338 getOperandsMapping(339 {getValueMapping(PMI_FirstGPR, Size),340 // Addresses are GPR 64-bit.341 getValueMapping(PMI_FirstGPR, TypeSize::getFixed(64))}),342 /*NumOperands*/ 2);343 const InstructionMapping &FPRMapping = getInstructionMapping(344 /*ID*/ 2, /*Cost*/ 1,345 getOperandsMapping(346 {getValueMapping(PMI_FirstFPR, Size),347 // Addresses are GPR 64-bit.348 getValueMapping(PMI_FirstGPR, TypeSize::getFixed(64))}),349 /*NumOperands*/ 2);350 351 AltMappings.push_back(&GPRMapping);352 AltMappings.push_back(&FPRMapping);353 return AltMappings;354 }355 default:356 break;357 }358 return RegisterBankInfo::getInstrAlternativeMappings(MI);359}360 361void AArch64RegisterBankInfo::applyMappingImpl(362 MachineIRBuilder &Builder, const OperandsMapper &OpdMapper) const {363 MachineInstr &MI = OpdMapper.getMI();364 MachineRegisterInfo &MRI = OpdMapper.getMRI();365 366 switch (MI.getOpcode()) {367 case TargetOpcode::G_OR:368 case TargetOpcode::G_BITCAST:369 case TargetOpcode::G_LOAD:370 // Those ID must match getInstrAlternativeMappings.371 assert((OpdMapper.getInstrMapping().getID() >= 1 &&372 OpdMapper.getInstrMapping().getID() <= 4) &&373 "Don't know how to handle that ID");374 return applyDefaultMapping(OpdMapper);375 case TargetOpcode::G_INSERT_VECTOR_ELT: {376 // Extend smaller gpr operands to 32 bit.377 Builder.setInsertPt(*MI.getParent(), MI.getIterator());378 auto Ext = Builder.buildAnyExt(LLT::scalar(32), MI.getOperand(2).getReg());379 MRI.setRegBank(Ext.getReg(0), getRegBank(AArch64::GPRRegBankID));380 MI.getOperand(2).setReg(Ext.getReg(0));381 return applyDefaultMapping(OpdMapper);382 }383 case AArch64::G_DUP: {384 // Extend smaller gpr to 32-bits385 assert(MRI.getType(MI.getOperand(1).getReg()).getSizeInBits() < 32 &&386 "Expected sources smaller than 32-bits");387 Builder.setInsertPt(*MI.getParent(), MI.getIterator());388 389 Register ConstReg;390 auto ConstMI = MRI.getVRegDef(MI.getOperand(1).getReg());391 if (ConstMI->getOpcode() == TargetOpcode::G_CONSTANT) {392 auto CstVal = ConstMI->getOperand(1).getCImm()->getValue();393 ConstReg =394 Builder.buildConstant(LLT::scalar(32), CstVal.sext(32)).getReg(0);395 } else {396 ConstReg = Builder.buildAnyExt(LLT::scalar(32), MI.getOperand(1).getReg())397 .getReg(0);398 }399 MRI.setRegBank(ConstReg, getRegBank(AArch64::GPRRegBankID));400 MI.getOperand(1).setReg(ConstReg);401 return applyDefaultMapping(OpdMapper);402 }403 default:404 llvm_unreachable("Don't know how to handle that operation");405 }406}407 408const RegisterBankInfo::InstructionMapping &409AArch64RegisterBankInfo::getSameKindOfOperandsMapping(410 const MachineInstr &MI) const {411 const unsigned Opc = MI.getOpcode();412 const MachineFunction &MF = *MI.getParent()->getParent();413 const MachineRegisterInfo &MRI = MF.getRegInfo();414 415 unsigned NumOperands = MI.getNumOperands();416 assert(NumOperands <= 3 &&417 "This code is for instructions with 3 or less operands");418 419 LLT Ty = MRI.getType(MI.getOperand(0).getReg());420 TypeSize Size = Ty.getSizeInBits();421 bool IsFPR = Ty.isVector() || isPreISelGenericFloatingPointOpcode(Opc);422 423 PartialMappingIdx RBIdx = IsFPR ? PMI_FirstFPR : PMI_FirstGPR;424 425#ifndef NDEBUG426 // Make sure all the operands are using similar size and type.427 // Should probably be checked by the machine verifier.428 // This code won't catch cases where the number of lanes is429 // different between the operands.430 // If we want to go to that level of details, it is probably431 // best to check that the types are the same, period.432 // Currently, we just check that the register banks are the same433 // for each types.434 for (unsigned Idx = 1; Idx != NumOperands; ++Idx) {435 LLT OpTy = MRI.getType(MI.getOperand(Idx).getReg());436 assert(437 AArch64GenRegisterBankInfo::getRegBankBaseIdxOffset(438 RBIdx, OpTy.getSizeInBits()) ==439 AArch64GenRegisterBankInfo::getRegBankBaseIdxOffset(RBIdx, Size) &&440 "Operand has incompatible size");441 bool OpIsFPR = OpTy.isVector() || isPreISelGenericFloatingPointOpcode(Opc);442 (void)OpIsFPR;443 assert(IsFPR == OpIsFPR && "Operand has incompatible type");444 }445#endif // End NDEBUG.446 447 return getInstructionMapping(DefaultMappingID, 1,448 getValueMapping(RBIdx, Size), NumOperands);449}450 451/// \returns true if a given intrinsic only uses and defines FPRs.452static bool isFPIntrinsic(const MachineRegisterInfo &MRI,453 const MachineInstr &MI) {454 // TODO: Add more intrinsics.455 switch (cast<GIntrinsic>(MI).getIntrinsicID()) {456 default:457 return false;458 case Intrinsic::aarch64_neon_uaddlv:459 case Intrinsic::aarch64_neon_uaddv:460 case Intrinsic::aarch64_neon_saddv:461 case Intrinsic::aarch64_neon_umaxv:462 case Intrinsic::aarch64_neon_smaxv:463 case Intrinsic::aarch64_neon_uminv:464 case Intrinsic::aarch64_neon_sminv:465 case Intrinsic::aarch64_neon_faddv:466 case Intrinsic::aarch64_neon_fmaxv:467 case Intrinsic::aarch64_neon_fminv:468 case Intrinsic::aarch64_neon_fmaxnmv:469 case Intrinsic::aarch64_neon_fminnmv:470 case Intrinsic::aarch64_neon_fmulx:471 case Intrinsic::aarch64_neon_frecpe:472 case Intrinsic::aarch64_neon_frecps:473 case Intrinsic::aarch64_neon_frecpx:474 case Intrinsic::aarch64_neon_frsqrte:475 case Intrinsic::aarch64_neon_frsqrts:476 case Intrinsic::aarch64_neon_facge:477 case Intrinsic::aarch64_neon_facgt:478 case Intrinsic::aarch64_neon_fabd:479 case Intrinsic::aarch64_sisd_fabd:480 case Intrinsic::aarch64_neon_sqrdmlah:481 case Intrinsic::aarch64_neon_sqrdmlsh:482 case Intrinsic::aarch64_neon_sqrdmulh:483 case Intrinsic::aarch64_neon_sqadd:484 case Intrinsic::aarch64_neon_sqsub:485 case Intrinsic::aarch64_crypto_sha1h:486 case Intrinsic::aarch64_crypto_sha1c:487 case Intrinsic::aarch64_crypto_sha1p:488 case Intrinsic::aarch64_crypto_sha1m:489 case Intrinsic::aarch64_sisd_fcvtxn:490 return true;491 case Intrinsic::aarch64_neon_saddlv: {492 const LLT SrcTy = MRI.getType(MI.getOperand(2).getReg());493 return SrcTy.getElementType().getSizeInBits() >= 16 &&494 SrcTy.getElementCount().getFixedValue() >= 4;495 }496 }497}498 499bool AArch64RegisterBankInfo::isPHIWithFPConstraints(500 const MachineInstr &MI, const MachineRegisterInfo &MRI,501 const AArch64RegisterInfo &TRI, const unsigned Depth) const {502 if (!MI.isPHI() || Depth > MaxFPRSearchDepth)503 return false;504 505 return any_of(MRI.use_nodbg_instructions(MI.getOperand(0).getReg()),506 [&](const MachineInstr &UseMI) {507 if (onlyUsesFP(UseMI, MRI, TRI, Depth + 1))508 return true;509 return isPHIWithFPConstraints(UseMI, MRI, TRI, Depth + 1);510 });511}512 513bool AArch64RegisterBankInfo::hasFPConstraints(const MachineInstr &MI,514 const MachineRegisterInfo &MRI,515 const AArch64RegisterInfo &TRI,516 unsigned Depth) const {517 unsigned Op = MI.getOpcode();518 if (Op == TargetOpcode::G_INTRINSIC && isFPIntrinsic(MRI, MI))519 return true;520 521 // Do we have an explicit floating point instruction?522 if (isPreISelGenericFloatingPointOpcode(Op))523 return true;524 525 // No. Check if we have a copy-like instruction. If we do, then we could526 // still be fed by floating point instructions.527 if (Op != TargetOpcode::COPY && !MI.isPHI() &&528 !isPreISelGenericOptimizationHint(Op))529 return false;530 531 // Check if we already know the register bank.532 auto *RB = getRegBank(MI.getOperand(0).getReg(), MRI, TRI);533 if (RB == &AArch64::FPRRegBank)534 return true;535 if (RB == &AArch64::GPRRegBank)536 return false;537 538 // We don't know anything.539 //540 // If we have a phi, we may be able to infer that it will be assigned a FPR541 // based off of its inputs.542 if (!MI.isPHI() || Depth > MaxFPRSearchDepth)543 return false;544 545 return any_of(MI.explicit_uses(), [&](const MachineOperand &Op) {546 return Op.isReg() &&547 onlyDefinesFP(*MRI.getVRegDef(Op.getReg()), MRI, TRI, Depth + 1);548 });549}550 551bool AArch64RegisterBankInfo::onlyUsesFP(const MachineInstr &MI,552 const MachineRegisterInfo &MRI,553 const AArch64RegisterInfo &TRI,554 unsigned Depth) const {555 switch (MI.getOpcode()) {556 case TargetOpcode::G_FPTOSI:557 case TargetOpcode::G_FPTOUI:558 case TargetOpcode::G_FPTOSI_SAT:559 case TargetOpcode::G_FPTOUI_SAT:560 case TargetOpcode::G_FCMP:561 case TargetOpcode::G_LROUND:562 case TargetOpcode::G_LLROUND:563 case AArch64::G_PMULL:564 return true;565 case TargetOpcode::G_INTRINSIC:566 switch (cast<GIntrinsic>(MI).getIntrinsicID()) {567 case Intrinsic::aarch64_neon_fcvtas:568 case Intrinsic::aarch64_neon_fcvtau:569 case Intrinsic::aarch64_neon_fcvtzs:570 case Intrinsic::aarch64_neon_fcvtzu:571 case Intrinsic::aarch64_neon_fcvtms:572 case Intrinsic::aarch64_neon_fcvtmu:573 case Intrinsic::aarch64_neon_fcvtns:574 case Intrinsic::aarch64_neon_fcvtnu:575 case Intrinsic::aarch64_neon_fcvtps:576 case Intrinsic::aarch64_neon_fcvtpu:577 return true;578 default:579 break;580 }581 break;582 default:583 break;584 }585 return hasFPConstraints(MI, MRI, TRI, Depth);586}587 588bool AArch64RegisterBankInfo::onlyDefinesFP(const MachineInstr &MI,589 const MachineRegisterInfo &MRI,590 const AArch64RegisterInfo &TRI,591 unsigned Depth) const {592 switch (MI.getOpcode()) {593 case AArch64::G_DUP:594 case AArch64::G_SADDLP:595 case AArch64::G_UADDLP:596 case TargetOpcode::G_SITOFP:597 case TargetOpcode::G_UITOFP:598 case TargetOpcode::G_EXTRACT_VECTOR_ELT:599 case TargetOpcode::G_INSERT_VECTOR_ELT:600 case TargetOpcode::G_BUILD_VECTOR:601 case TargetOpcode::G_BUILD_VECTOR_TRUNC:602 return true;603 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:604 switch (cast<GIntrinsic>(MI).getIntrinsicID()) {605 case Intrinsic::aarch64_neon_ld1x2:606 case Intrinsic::aarch64_neon_ld1x3:607 case Intrinsic::aarch64_neon_ld1x4:608 case Intrinsic::aarch64_neon_ld2:609 case Intrinsic::aarch64_neon_ld2lane:610 case Intrinsic::aarch64_neon_ld2r:611 case Intrinsic::aarch64_neon_ld3:612 case Intrinsic::aarch64_neon_ld3lane:613 case Intrinsic::aarch64_neon_ld3r:614 case Intrinsic::aarch64_neon_ld4:615 case Intrinsic::aarch64_neon_ld4lane:616 case Intrinsic::aarch64_neon_ld4r:617 return true;618 default:619 break;620 }621 break;622 default:623 break;624 }625 return hasFPConstraints(MI, MRI, TRI, Depth);626}627 628bool AArch64RegisterBankInfo::prefersFPUse(const MachineInstr &MI,629 const MachineRegisterInfo &MRI,630 const AArch64RegisterInfo &TRI,631 unsigned Depth) const {632 switch (MI.getOpcode()) {633 case TargetOpcode::G_SITOFP:634 case TargetOpcode::G_UITOFP:635 return MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() ==636 MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();637 }638 return onlyDefinesFP(MI, MRI, TRI, Depth);639}640 641bool AArch64RegisterBankInfo::isLoadFromFPType(const MachineInstr &MI) const {642 // GMemOperation because we also want to match indexed loads.643 auto *MemOp = cast<GMemOperation>(&MI);644 const Value *LdVal = MemOp->getMMO().getValue();645 if (!LdVal)646 return false;647 648 Type *EltTy = nullptr;649 if (const GlobalValue *GV = dyn_cast<GlobalValue>(LdVal)) {650 EltTy = GV->getValueType();651 // Look at the first element of the struct to determine the type we are652 // loading653 while (StructType *StructEltTy = dyn_cast<StructType>(EltTy)) {654 if (StructEltTy->getNumElements() == 0)655 break;656 EltTy = StructEltTy->getTypeAtIndex(0U);657 }658 // Look at the first element of the array to determine its type659 if (isa<ArrayType>(EltTy))660 EltTy = EltTy->getArrayElementType();661 } else if (!isa<Constant>(LdVal)) {662 // FIXME: grubbing around uses is pretty ugly, but with no more663 // `getPointerElementType` there's not much else we can do.664 for (const auto *LdUser : LdVal->users()) {665 if (isa<LoadInst>(LdUser)) {666 EltTy = LdUser->getType();667 break;668 }669 if (isa<StoreInst>(LdUser) && LdUser->getOperand(1) == LdVal) {670 EltTy = LdUser->getOperand(0)->getType();671 break;672 }673 }674 }675 return EltTy && EltTy->isFPOrFPVectorTy();676}677 678const RegisterBankInfo::InstructionMapping &679AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {680 const unsigned Opc = MI.getOpcode();681 682 // Try the default logic for non-generic instructions that are either copies683 // or already have some operands assigned to banks.684 if ((Opc != TargetOpcode::COPY && !isPreISelGenericOpcode(Opc)) ||685 Opc == TargetOpcode::G_PHI) {686 const RegisterBankInfo::InstructionMapping &Mapping =687 getInstrMappingImpl(MI);688 if (Mapping.isValid())689 return Mapping;690 }691 692 const MachineFunction &MF = *MI.getParent()->getParent();693 const MachineRegisterInfo &MRI = MF.getRegInfo();694 const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();695 const AArch64RegisterInfo &TRI = *STI.getRegisterInfo();696 697 switch (Opc) {698 // G_{F|S|U}REM are not listed because they are not legal.699 // Arithmetic ops.700 case TargetOpcode::G_ADD:701 case TargetOpcode::G_SUB:702 case TargetOpcode::G_PTR_ADD:703 case TargetOpcode::G_MUL:704 case TargetOpcode::G_SDIV:705 case TargetOpcode::G_UDIV:706 // Bitwise ops.707 case TargetOpcode::G_AND:708 case TargetOpcode::G_OR:709 case TargetOpcode::G_XOR:710 // Floating point ops.711 case TargetOpcode::G_FADD:712 case TargetOpcode::G_FSUB:713 case TargetOpcode::G_FMUL:714 case TargetOpcode::G_FDIV:715 case TargetOpcode::G_FMAXIMUM:716 case TargetOpcode::G_FMINIMUM:717 return getSameKindOfOperandsMapping(MI);718 case TargetOpcode::G_FPEXT: {719 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());720 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());721 return getInstructionMapping(722 DefaultMappingID, /*Cost*/ 1,723 getFPExtMapping(DstTy.getSizeInBits(), SrcTy.getSizeInBits()),724 /*NumOperands*/ 2);725 }726 // Shifts.727 case TargetOpcode::G_SHL:728 case TargetOpcode::G_LSHR:729 case TargetOpcode::G_ASHR: {730 LLT ShiftAmtTy = MRI.getType(MI.getOperand(2).getReg());731 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());732 if (ShiftAmtTy.getSizeInBits() == 64 && SrcTy.getSizeInBits() == 32)733 return getInstructionMapping(DefaultMappingID, 1,734 &ValMappings[Shift64Imm], 3);735 return getSameKindOfOperandsMapping(MI);736 }737 case TargetOpcode::COPY: {738 Register DstReg = MI.getOperand(0).getReg();739 Register SrcReg = MI.getOperand(1).getReg();740 // Check if one of the register is not a generic register.741 if ((DstReg.isPhysical() || !MRI.getType(DstReg).isValid()) ||742 (SrcReg.isPhysical() || !MRI.getType(SrcReg).isValid())) {743 const RegisterBank *DstRB = getRegBank(DstReg, MRI, TRI);744 const RegisterBank *SrcRB = getRegBank(SrcReg, MRI, TRI);745 if (!DstRB)746 DstRB = SrcRB;747 else if (!SrcRB)748 SrcRB = DstRB;749 // If both RB are null that means both registers are generic.750 // We shouldn't be here.751 assert(DstRB && SrcRB && "Both RegBank were nullptr");752 TypeSize Size = getSizeInBits(DstReg, MRI, TRI);753 return getInstructionMapping(754 DefaultMappingID, copyCost(*DstRB, *SrcRB, Size),755 getCopyMapping(DstRB->getID(), SrcRB->getID(), Size),756 // We only care about the mapping of the destination.757 /*NumOperands*/ 1);758 }759 // Both registers are generic, use G_BITCAST.760 [[fallthrough]];761 }762 case TargetOpcode::G_BITCAST: {763 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());764 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());765 TypeSize Size = DstTy.getSizeInBits();766 bool DstIsGPR = !DstTy.isVector() && DstTy.getSizeInBits() <= 64;767 bool SrcIsGPR = !SrcTy.isVector() && SrcTy.getSizeInBits() <= 64;768 const RegisterBank &DstRB =769 DstIsGPR ? AArch64::GPRRegBank : AArch64::FPRRegBank;770 const RegisterBank &SrcRB =771 SrcIsGPR ? AArch64::GPRRegBank : AArch64::FPRRegBank;772 return getInstructionMapping(773 DefaultMappingID, copyCost(DstRB, SrcRB, Size),774 getCopyMapping(DstRB.getID(), SrcRB.getID(), Size),775 // We only care about the mapping of the destination for COPY.776 /*NumOperands*/ Opc == TargetOpcode::G_BITCAST ? 2 : 1);777 }778 default:779 break;780 }781 782 unsigned NumOperands = MI.getNumOperands();783 unsigned MappingID = DefaultMappingID;784 785 // Track the size and bank of each register. We don't do partial mappings.786 SmallVector<unsigned, 4> OpSize(NumOperands);787 SmallVector<PartialMappingIdx, 4> OpRegBankIdx(NumOperands);788 for (unsigned Idx = 0; Idx < NumOperands; ++Idx) {789 auto &MO = MI.getOperand(Idx);790 if (!MO.isReg() || !MO.getReg())791 continue;792 793 LLT Ty = MRI.getType(MO.getReg());794 if (!Ty.isValid())795 continue;796 OpSize[Idx] = Ty.getSizeInBits().getKnownMinValue();797 798 // As a top-level guess, vectors including both scalable and non-scalable799 // ones go in FPRs, scalars and pointers in GPRs.800 // For floating-point instructions, scalars go in FPRs.801 if (Ty.isVector())802 OpRegBankIdx[Idx] = PMI_FirstFPR;803 else if (isPreISelGenericFloatingPointOpcode(Opc) ||804 (MO.isDef() && onlyDefinesFP(MI, MRI, TRI)) ||805 (MO.isUse() && onlyUsesFP(MI, MRI, TRI)) ||806 Ty.getSizeInBits() > 64)807 OpRegBankIdx[Idx] = PMI_FirstFPR;808 else809 OpRegBankIdx[Idx] = PMI_FirstGPR;810 }811 812 unsigned Cost = 1;813 // Some of the floating-point instructions have mixed GPR and FPR operands:814 // fine-tune the computed mapping.815 switch (Opc) {816 case AArch64::G_DUP: {817 Register ScalarReg = MI.getOperand(1).getReg();818 LLT ScalarTy = MRI.getType(ScalarReg);819 auto ScalarDef = MRI.getVRegDef(ScalarReg);820 // We want to select dup(load) into LD1R.821 if (ScalarDef->getOpcode() == TargetOpcode::G_LOAD)822 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};823 // s8 is an exception for G_DUP, which we always want on gpr.824 else if (ScalarTy.getSizeInBits() != 8 &&825 (getRegBank(ScalarReg, MRI, TRI) == &AArch64::FPRRegBank ||826 onlyDefinesFP(*ScalarDef, MRI, TRI)))827 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};828 else {829 if (ScalarTy.getSizeInBits() < 32 &&830 getRegBank(ScalarReg, MRI, TRI) == &AArch64::GPRRegBank) {831 // Calls applyMappingImpl()832 MappingID = CustomMappingID;833 }834 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR};835 }836 break;837 }838 case TargetOpcode::G_TRUNC: {839 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());840 if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 128)841 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};842 break;843 }844 case TargetOpcode::G_SITOFP:845 case TargetOpcode::G_UITOFP: {846 if (MRI.getType(MI.getOperand(0).getReg()).isVector())847 break;848 // Integer to FP conversions don't necessarily happen between GPR -> FPR849 // regbanks. They can also be done within an FPR register.850 Register SrcReg = MI.getOperand(1).getReg();851 if (getRegBank(SrcReg, MRI, TRI) == &AArch64::FPRRegBank &&852 MRI.getType(SrcReg).getSizeInBits() ==853 MRI.getType(MI.getOperand(0).getReg()).getSizeInBits())854 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};855 else856 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR};857 break;858 }859 case TargetOpcode::G_FPTOSI_SAT:860 case TargetOpcode::G_FPTOUI_SAT:861 case TargetOpcode::G_FPTOSI:862 case TargetOpcode::G_FPTOUI: {863 LLT DstType = MRI.getType(MI.getOperand(0).getReg());864 if (DstType.isVector())865 break;866 if (DstType == LLT::scalar(16)) {867 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};868 break;869 }870 TypeSize DstSize = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI);871 TypeSize SrcSize = getSizeInBits(MI.getOperand(1).getReg(), MRI, TRI);872 if (((DstSize == SrcSize) || STI.hasFeature(AArch64::FeatureFPRCVT)) &&873 all_of(MRI.use_nodbg_instructions(MI.getOperand(0).getReg()),874 [&](const MachineInstr &UseMI) {875 return onlyUsesFP(UseMI, MRI, TRI) ||876 prefersFPUse(UseMI, MRI, TRI);877 }))878 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};879 else880 OpRegBankIdx = {PMI_FirstGPR, PMI_FirstFPR};881 break;882 }883 case TargetOpcode::G_INTRINSIC_LRINT:884 case TargetOpcode::G_INTRINSIC_LLRINT:885 if (MRI.getType(MI.getOperand(0).getReg()).isVector())886 break;887 OpRegBankIdx = {PMI_FirstGPR, PMI_FirstFPR};888 break;889 case TargetOpcode::G_FCMP: {890 // If the result is a vector, it must use a FPR.891 AArch64GenRegisterBankInfo::PartialMappingIdx Idx0 =892 MRI.getType(MI.getOperand(0).getReg()).isVector() ? PMI_FirstFPR893 : PMI_FirstGPR;894 OpRegBankIdx = {Idx0,895 /* Predicate */ PMI_None, PMI_FirstFPR, PMI_FirstFPR};896 break;897 }898 case TargetOpcode::G_BITCAST:899 // This is going to be a cross register bank copy and this is expensive.900 if (OpRegBankIdx[0] != OpRegBankIdx[1])901 Cost = copyCost(902 *AArch64GenRegisterBankInfo::PartMappings[OpRegBankIdx[0]].RegBank,903 *AArch64GenRegisterBankInfo::PartMappings[OpRegBankIdx[1]].RegBank,904 TypeSize::getFixed(OpSize[0]));905 break;906 case TargetOpcode::G_LOAD: {907 // Loading in vector unit is slightly more expensive.908 // This is actually only true for the LD1R and co instructions,909 // but anyway for the fast mode this number does not matter and910 // for the greedy mode the cost of the cross bank copy will911 // offset this number.912 // FIXME: Should be derived from the scheduling model.913 if (OpRegBankIdx[0] != PMI_FirstGPR) {914 Cost = 2;915 break;916 }917 918 if (cast<GLoad>(MI).isAtomic()) {919 // Atomics always use GPR destinations. Don't refine any further.920 OpRegBankIdx[0] = PMI_FirstGPR;921 break;922 }923 924 // Try to guess the type of the load from the MMO.925 if (isLoadFromFPType(MI)) {926 OpRegBankIdx[0] = PMI_FirstFPR;927 break;928 }929 930 // Check if that load feeds fp instructions.931 // In that case, we want the default mapping to be on FPR932 // instead of blind map every scalar to GPR.933 if (any_of(MRI.use_nodbg_instructions(MI.getOperand(0).getReg()),934 [&](const MachineInstr &UseMI) {935 // If we have at least one direct or indirect use936 // in a FP instruction,937 // assume this was a floating point load in the IR. If it was938 // not, we would have had a bitcast before reaching that939 // instruction.940 //941 // Int->FP conversion operations are also captured in942 // prefersFPUse().943 944 if (isPHIWithFPConstraints(UseMI, MRI, TRI))945 return true;946 947 return onlyUsesFP(UseMI, MRI, TRI) ||948 prefersFPUse(UseMI, MRI, TRI);949 }))950 OpRegBankIdx[0] = PMI_FirstFPR;951 break;952 }953 case TargetOpcode::G_STORE:954 // Check if that store is fed by fp instructions.955 if (OpRegBankIdx[0] == PMI_FirstGPR) {956 Register VReg = MI.getOperand(0).getReg();957 if (!VReg)958 break;959 MachineInstr *DefMI = MRI.getVRegDef(VReg);960 if (onlyDefinesFP(*DefMI, MRI, TRI))961 OpRegBankIdx[0] = PMI_FirstFPR;962 break;963 }964 break;965 case TargetOpcode::G_INDEXED_STORE:966 if (OpRegBankIdx[1] == PMI_FirstGPR) {967 Register VReg = MI.getOperand(1).getReg();968 if (!VReg)969 break;970 MachineInstr *DefMI = MRI.getVRegDef(VReg);971 if (onlyDefinesFP(*DefMI, MRI, TRI))972 OpRegBankIdx[1] = PMI_FirstFPR;973 break;974 }975 break;976 case TargetOpcode::G_INDEXED_SEXTLOAD:977 case TargetOpcode::G_INDEXED_ZEXTLOAD:978 // These should always be GPR.979 OpRegBankIdx[0] = PMI_FirstGPR;980 break;981 case TargetOpcode::G_INDEXED_LOAD: {982 if (isLoadFromFPType(MI))983 OpRegBankIdx[0] = PMI_FirstFPR;984 break;985 }986 case TargetOpcode::G_SELECT: {987 // If the destination is FPR, preserve that.988 if (OpRegBankIdx[0] != PMI_FirstGPR)989 break;990 991 // If we're taking in vectors, we have no choice but to put everything on992 // FPRs, except for the condition. The condition must always be on a GPR.993 LLT SrcTy = MRI.getType(MI.getOperand(2).getReg());994 if (SrcTy.isVector()) {995 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR, PMI_FirstFPR, PMI_FirstFPR};996 break;997 }998 999 // Try to minimize the number of copies. If we have more floating point1000 // constrained values than not, then we'll put everything on FPR. Otherwise,1001 // everything has to be on GPR.1002 unsigned NumFP = 0;1003 1004 // Check if the uses of the result always produce floating point values.1005 //1006 // For example:1007 //1008 // %z = G_SELECT %cond %x %y1009 // fpr = G_FOO %z ...1010 if (any_of(MRI.use_nodbg_instructions(MI.getOperand(0).getReg()),1011 [&](MachineInstr &MI) { return onlyUsesFP(MI, MRI, TRI); }))1012 ++NumFP;1013 1014 // Check if the defs of the source values always produce floating point1015 // values.1016 //1017 // For example:1018 //1019 // %x = G_SOMETHING_ALWAYS_FLOAT %a ...1020 // %z = G_SELECT %cond %x %y1021 //1022 // Also check whether or not the sources have already been decided to be1023 // FPR. Keep track of this.1024 //1025 // This doesn't check the condition, since it's just whatever is in NZCV.1026 // This isn't passed explicitly in a register to fcsel/csel.1027 for (unsigned Idx = 2; Idx < 4; ++Idx) {1028 Register VReg = MI.getOperand(Idx).getReg();1029 MachineInstr *DefMI = MRI.getVRegDef(VReg);1030 if (getRegBank(VReg, MRI, TRI) == &AArch64::FPRRegBank ||1031 onlyDefinesFP(*DefMI, MRI, TRI))1032 ++NumFP;1033 }1034 1035 // If we have more FP constraints than not, then move everything over to1036 // FPR.1037 if (NumFP >= 2)1038 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR, PMI_FirstFPR, PMI_FirstFPR};1039 1040 break;1041 }1042 case TargetOpcode::G_UNMERGE_VALUES: {1043 // If the first operand belongs to a FPR register bank, then make sure that1044 // we preserve that.1045 if (OpRegBankIdx[0] != PMI_FirstGPR)1046 break;1047 1048 LLT SrcTy = MRI.getType(MI.getOperand(MI.getNumOperands()-1).getReg());1049 // UNMERGE into scalars from a vector should always use FPR.1050 // Likewise if any of the uses are FP instructions.1051 if (SrcTy.isVector() || SrcTy == LLT::scalar(128) ||1052 any_of(MRI.use_nodbg_instructions(MI.getOperand(0).getReg()),1053 [&](MachineInstr &MI) { return onlyUsesFP(MI, MRI, TRI); })) {1054 // Set the register bank of every operand to FPR.1055 for (unsigned Idx = 0, NumOperands = MI.getNumOperands();1056 Idx < NumOperands; ++Idx)1057 OpRegBankIdx[Idx] = PMI_FirstFPR;1058 }1059 break;1060 }1061 case TargetOpcode::G_EXTRACT_VECTOR_ELT:1062 // Destination and source need to be FPRs.1063 OpRegBankIdx[0] = PMI_FirstFPR;1064 OpRegBankIdx[1] = PMI_FirstFPR;1065 1066 // Index needs to be a GPR.1067 OpRegBankIdx[2] = PMI_FirstGPR;1068 break;1069 case TargetOpcode::G_INSERT_VECTOR_ELT:1070 OpRegBankIdx[0] = PMI_FirstFPR;1071 OpRegBankIdx[1] = PMI_FirstFPR;1072 1073 // The element may be either a GPR or FPR. Preserve that behaviour.1074 if (getRegBank(MI.getOperand(2).getReg(), MRI, TRI) == &AArch64::FPRRegBank)1075 OpRegBankIdx[2] = PMI_FirstFPR;1076 else {1077 // If the type is i8/i16, and the regank will be GPR, then we change the1078 // type to i32 in applyMappingImpl.1079 LLT Ty = MRI.getType(MI.getOperand(2).getReg());1080 if (Ty.getSizeInBits() == 8 || Ty.getSizeInBits() == 16) {1081 // Calls applyMappingImpl()1082 MappingID = CustomMappingID;1083 }1084 OpRegBankIdx[2] = PMI_FirstGPR;1085 }1086 1087 // Index needs to be a GPR.1088 OpRegBankIdx[3] = PMI_FirstGPR;1089 break;1090 case TargetOpcode::G_EXTRACT: {1091 // For s128 sources we have to use fpr unless we know otherwise.1092 auto Src = MI.getOperand(1).getReg();1093 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());1094 if (SrcTy.getSizeInBits() != 128)1095 break;1096 auto Idx = MRI.getRegClassOrNull(Src) == &AArch64::XSeqPairsClassRegClass1097 ? PMI_FirstGPR1098 : PMI_FirstFPR;1099 OpRegBankIdx[0] = Idx;1100 OpRegBankIdx[1] = Idx;1101 break;1102 }1103 case TargetOpcode::G_BUILD_VECTOR: {1104 // If the first source operand belongs to a FPR register bank, then make1105 // sure that we preserve that.1106 if (OpRegBankIdx[1] != PMI_FirstGPR)1107 break;1108 Register VReg = MI.getOperand(1).getReg();1109 if (!VReg)1110 break;1111 1112 // Get the instruction that defined the source operand reg, and check if1113 // it's a floating point operation. Or, if it's a type like s16 which1114 // doesn't have a exact size gpr register class. The exception is if the1115 // build_vector has all constant operands, which may be better to leave as1116 // gpr without copies, so it can be matched in imported patterns.1117 MachineInstr *DefMI = MRI.getVRegDef(VReg);1118 unsigned DefOpc = DefMI->getOpcode();1119 const LLT SrcTy = MRI.getType(VReg);1120 if (all_of(MI.operands(), [&](const MachineOperand &Op) {1121 return Op.isDef() || MRI.getVRegDef(Op.getReg())->getOpcode() ==1122 TargetOpcode::G_CONSTANT;1123 }))1124 break;1125 if (isPreISelGenericFloatingPointOpcode(DefOpc) ||1126 SrcTy.getSizeInBits() < 32 ||1127 getRegBank(VReg, MRI, TRI) == &AArch64::FPRRegBank) {1128 // Have a floating point op.1129 // Make sure every operand gets mapped to a FPR register class.1130 unsigned NumOperands = MI.getNumOperands();1131 for (unsigned Idx = 0; Idx < NumOperands; ++Idx)1132 OpRegBankIdx[Idx] = PMI_FirstFPR;1133 }1134 break;1135 }1136 case TargetOpcode::G_VECREDUCE_FADD:1137 case TargetOpcode::G_VECREDUCE_FMUL:1138 case TargetOpcode::G_VECREDUCE_FMAX:1139 case TargetOpcode::G_VECREDUCE_FMIN:1140 case TargetOpcode::G_VECREDUCE_FMAXIMUM:1141 case TargetOpcode::G_VECREDUCE_FMINIMUM:1142 case TargetOpcode::G_VECREDUCE_ADD:1143 case TargetOpcode::G_VECREDUCE_MUL:1144 case TargetOpcode::G_VECREDUCE_AND:1145 case TargetOpcode::G_VECREDUCE_OR:1146 case TargetOpcode::G_VECREDUCE_XOR:1147 case TargetOpcode::G_VECREDUCE_SMAX:1148 case TargetOpcode::G_VECREDUCE_SMIN:1149 case TargetOpcode::G_VECREDUCE_UMAX:1150 case TargetOpcode::G_VECREDUCE_UMIN:1151 // Reductions produce a scalar value from a vector, the scalar should be on1152 // FPR bank.1153 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};1154 break;1155 case TargetOpcode::G_VECREDUCE_SEQ_FADD:1156 case TargetOpcode::G_VECREDUCE_SEQ_FMUL:1157 // These reductions also take a scalar accumulator input.1158 // Assign them FPR for now.1159 OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR, PMI_FirstFPR};1160 break;1161 case TargetOpcode::G_INTRINSIC:1162 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: {1163 switch (cast<GIntrinsic>(MI).getIntrinsicID()) {1164 case Intrinsic::aarch64_neon_fcvtas:1165 case Intrinsic::aarch64_neon_fcvtau:1166 case Intrinsic::aarch64_neon_fcvtzs:1167 case Intrinsic::aarch64_neon_fcvtzu:1168 case Intrinsic::aarch64_neon_fcvtms:1169 case Intrinsic::aarch64_neon_fcvtmu:1170 case Intrinsic::aarch64_neon_fcvtns:1171 case Intrinsic::aarch64_neon_fcvtnu:1172 case Intrinsic::aarch64_neon_fcvtps:1173 case Intrinsic::aarch64_neon_fcvtpu: {1174 OpRegBankIdx[2] = PMI_FirstFPR;1175 if (MRI.getType(MI.getOperand(0).getReg()).isVector()) {1176 OpRegBankIdx[0] = PMI_FirstFPR;1177 break;1178 }1179 TypeSize DstSize = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI);1180 TypeSize SrcSize = getSizeInBits(MI.getOperand(2).getReg(), MRI, TRI);1181 if (((DstSize == SrcSize) || STI.hasFeature(AArch64::FeatureFPRCVT)) &&1182 all_of(MRI.use_nodbg_instructions(MI.getOperand(0).getReg()),1183 [&](const MachineInstr &UseMI) {1184 return onlyUsesFP(UseMI, MRI, TRI) ||1185 prefersFPUse(UseMI, MRI, TRI);1186 }))1187 OpRegBankIdx[0] = PMI_FirstFPR;1188 else1189 OpRegBankIdx[0] = PMI_FirstGPR;1190 break;1191 }1192 case Intrinsic::aarch64_neon_vcvtfxs2fp:1193 case Intrinsic::aarch64_neon_vcvtfxu2fp:1194 case Intrinsic::aarch64_neon_vcvtfp2fxs:1195 case Intrinsic::aarch64_neon_vcvtfp2fxu:1196 // Override these intrinsics, because they would have a partial1197 // mapping. This is needed for 'half' types, which otherwise don't1198 // get legalised correctly.1199 OpRegBankIdx[0] = PMI_FirstFPR;1200 OpRegBankIdx[2] = PMI_FirstFPR;1201 // OpRegBankIdx[1] is the intrinsic ID.1202 // OpRegBankIdx[3] is an integer immediate.1203 break;1204 default: {1205 // Check if we know that the intrinsic has any constraints on its register1206 // banks. If it does, then update the mapping accordingly.1207 unsigned Idx = 0;1208 if (onlyDefinesFP(MI, MRI, TRI))1209 for (const auto &Op : MI.defs()) {1210 if (Op.isReg())1211 OpRegBankIdx[Idx] = PMI_FirstFPR;1212 ++Idx;1213 }1214 else1215 Idx += MI.getNumExplicitDefs();1216 1217 if (onlyUsesFP(MI, MRI, TRI))1218 for (const auto &Op : MI.explicit_uses()) {1219 if (Op.isReg())1220 OpRegBankIdx[Idx] = PMI_FirstFPR;1221 ++Idx;1222 }1223 break;1224 }1225 }1226 break;1227 }1228 case TargetOpcode::G_LROUND:1229 case TargetOpcode::G_LLROUND: {1230 // Source is always floating point and destination is always integer.1231 OpRegBankIdx = {PMI_FirstGPR, PMI_FirstFPR};1232 break;1233 }1234 }1235 1236 // Finally construct the computed mapping.1237 SmallVector<const ValueMapping *, 8> OpdsMapping(NumOperands);1238 for (unsigned Idx = 0; Idx < NumOperands; ++Idx) {1239 if (MI.getOperand(Idx).isReg() && MI.getOperand(Idx).getReg()) {1240 LLT Ty = MRI.getType(MI.getOperand(Idx).getReg());1241 if (!Ty.isValid())1242 continue;1243 auto Mapping =1244 getValueMapping(OpRegBankIdx[Idx], TypeSize::getFixed(OpSize[Idx]));1245 if (!Mapping->isValid())1246 return getInvalidInstructionMapping();1247 1248 OpdsMapping[Idx] = Mapping;1249 }1250 }1251 1252 return getInstructionMapping(MappingID, Cost, getOperandsMapping(OpdsMapping),1253 NumOperands);1254}1255