1014 lines · cpp
1//===-- AMDGPURegBankLegalizeRules.cpp ------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9/// Definitions of RegBankLegalize Rules for all opcodes.10/// Implementation of container for all the Rules and search.11/// Fast search for most common case when Rule.Predicate checks LLT and12/// uniformity of register in operand 0.13//14//===----------------------------------------------------------------------===//15 16#include "AMDGPURegBankLegalizeRules.h"17#include "AMDGPUInstrInfo.h"18#include "GCNSubtarget.h"19#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"20#include "llvm/CodeGen/MachineUniformityAnalysis.h"21#include "llvm/IR/IntrinsicsAMDGPU.h"22#include "llvm/Support/AMDGPUAddrSpace.h"23 24#define DEBUG_TYPE "amdgpu-regbanklegalize"25 26using namespace llvm;27using namespace AMDGPU;28 29bool AMDGPU::isAnyPtr(LLT Ty, unsigned Width) {30 return Ty.isPointer() && Ty.getSizeInBits() == Width;31}32 33RegBankLLTMapping::RegBankLLTMapping(34 std::initializer_list<RegBankLLTMappingApplyID> DstOpMappingList,35 std::initializer_list<RegBankLLTMappingApplyID> SrcOpMappingList,36 LoweringMethodID LoweringMethod)37 : DstOpMapping(DstOpMappingList), SrcOpMapping(SrcOpMappingList),38 LoweringMethod(LoweringMethod) {}39 40PredicateMapping::PredicateMapping(41 std::initializer_list<UniformityLLTOpPredicateID> OpList,42 std::function<bool(const MachineInstr &)> TestFunc)43 : OpUniformityAndTypes(OpList), TestFunc(TestFunc) {}44 45bool matchUniformityAndLLT(Register Reg, UniformityLLTOpPredicateID UniID,46 const MachineUniformityInfo &MUI,47 const MachineRegisterInfo &MRI) {48 switch (UniID) {49 case S1:50 return MRI.getType(Reg) == LLT::scalar(1);51 case S16:52 return MRI.getType(Reg) == LLT::scalar(16);53 case S32:54 return MRI.getType(Reg) == LLT::scalar(32);55 case S64:56 return MRI.getType(Reg) == LLT::scalar(64);57 case S128:58 return MRI.getType(Reg) == LLT::scalar(128);59 case P0:60 return MRI.getType(Reg) == LLT::pointer(0, 64);61 case P1:62 return MRI.getType(Reg) == LLT::pointer(1, 64);63 case P3:64 return MRI.getType(Reg) == LLT::pointer(3, 32);65 case P4:66 return MRI.getType(Reg) == LLT::pointer(4, 64);67 case P5:68 return MRI.getType(Reg) == LLT::pointer(5, 32);69 case P8:70 return MRI.getType(Reg) == LLT::pointer(8, 128);71 case Ptr32:72 return isAnyPtr(MRI.getType(Reg), 32);73 case Ptr64:74 return isAnyPtr(MRI.getType(Reg), 64);75 case Ptr128:76 return isAnyPtr(MRI.getType(Reg), 128);77 case V2S32:78 return MRI.getType(Reg) == LLT::fixed_vector(2, 32);79 case V4S32:80 return MRI.getType(Reg) == LLT::fixed_vector(4, 32);81 case B32:82 return MRI.getType(Reg).getSizeInBits() == 32;83 case B64:84 return MRI.getType(Reg).getSizeInBits() == 64;85 case B96:86 return MRI.getType(Reg).getSizeInBits() == 96;87 case B128:88 return MRI.getType(Reg).getSizeInBits() == 128;89 case B256:90 return MRI.getType(Reg).getSizeInBits() == 256;91 case B512:92 return MRI.getType(Reg).getSizeInBits() == 512;93 case UniS1:94 return MRI.getType(Reg) == LLT::scalar(1) && MUI.isUniform(Reg);95 case UniS16:96 return MRI.getType(Reg) == LLT::scalar(16) && MUI.isUniform(Reg);97 case UniS32:98 return MRI.getType(Reg) == LLT::scalar(32) && MUI.isUniform(Reg);99 case UniS64:100 return MRI.getType(Reg) == LLT::scalar(64) && MUI.isUniform(Reg);101 case UniS128:102 return MRI.getType(Reg) == LLT::scalar(128) && MUI.isUniform(Reg);103 case UniP0:104 return MRI.getType(Reg) == LLT::pointer(0, 64) && MUI.isUniform(Reg);105 case UniP1:106 return MRI.getType(Reg) == LLT::pointer(1, 64) && MUI.isUniform(Reg);107 case UniP3:108 return MRI.getType(Reg) == LLT::pointer(3, 32) && MUI.isUniform(Reg);109 case UniP4:110 return MRI.getType(Reg) == LLT::pointer(4, 64) && MUI.isUniform(Reg);111 case UniP5:112 return MRI.getType(Reg) == LLT::pointer(5, 32) && MUI.isUniform(Reg);113 case UniP8:114 return MRI.getType(Reg) == LLT::pointer(8, 128) && MUI.isUniform(Reg);115 case UniPtr32:116 return isAnyPtr(MRI.getType(Reg), 32) && MUI.isUniform(Reg);117 case UniPtr64:118 return isAnyPtr(MRI.getType(Reg), 64) && MUI.isUniform(Reg);119 case UniPtr128:120 return isAnyPtr(MRI.getType(Reg), 128) && MUI.isUniform(Reg);121 case UniV2S16:122 return MRI.getType(Reg) == LLT::fixed_vector(2, 16) && MUI.isUniform(Reg);123 case UniB32:124 return MRI.getType(Reg).getSizeInBits() == 32 && MUI.isUniform(Reg);125 case UniB64:126 return MRI.getType(Reg).getSizeInBits() == 64 && MUI.isUniform(Reg);127 case UniB96:128 return MRI.getType(Reg).getSizeInBits() == 96 && MUI.isUniform(Reg);129 case UniB128:130 return MRI.getType(Reg).getSizeInBits() == 128 && MUI.isUniform(Reg);131 case UniB256:132 return MRI.getType(Reg).getSizeInBits() == 256 && MUI.isUniform(Reg);133 case UniB512:134 return MRI.getType(Reg).getSizeInBits() == 512 && MUI.isUniform(Reg);135 case DivS1:136 return MRI.getType(Reg) == LLT::scalar(1) && MUI.isDivergent(Reg);137 case DivS16:138 return MRI.getType(Reg) == LLT::scalar(16) && MUI.isDivergent(Reg);139 case DivS32:140 return MRI.getType(Reg) == LLT::scalar(32) && MUI.isDivergent(Reg);141 case DivS64:142 return MRI.getType(Reg) == LLT::scalar(64) && MUI.isDivergent(Reg);143 case DivS128:144 return MRI.getType(Reg) == LLT::scalar(128) && MUI.isDivergent(Reg);145 case DivP0:146 return MRI.getType(Reg) == LLT::pointer(0, 64) && MUI.isDivergent(Reg);147 case DivP1:148 return MRI.getType(Reg) == LLT::pointer(1, 64) && MUI.isDivergent(Reg);149 case DivP3:150 return MRI.getType(Reg) == LLT::pointer(3, 32) && MUI.isDivergent(Reg);151 case DivP4:152 return MRI.getType(Reg) == LLT::pointer(4, 64) && MUI.isDivergent(Reg);153 case DivP5:154 return MRI.getType(Reg) == LLT::pointer(5, 32) && MUI.isDivergent(Reg);155 case DivPtr32:156 return isAnyPtr(MRI.getType(Reg), 32) && MUI.isDivergent(Reg);157 case DivPtr64:158 return isAnyPtr(MRI.getType(Reg), 64) && MUI.isDivergent(Reg);159 case DivPtr128:160 return isAnyPtr(MRI.getType(Reg), 128) && MUI.isDivergent(Reg);161 case DivV2S16:162 return MRI.getType(Reg) == LLT::fixed_vector(2, 16) && MUI.isDivergent(Reg);163 case DivB32:164 return MRI.getType(Reg).getSizeInBits() == 32 && MUI.isDivergent(Reg);165 case DivB64:166 return MRI.getType(Reg).getSizeInBits() == 64 && MUI.isDivergent(Reg);167 case DivB96:168 return MRI.getType(Reg).getSizeInBits() == 96 && MUI.isDivergent(Reg);169 case DivB128:170 return MRI.getType(Reg).getSizeInBits() == 128 && MUI.isDivergent(Reg);171 case DivB256:172 return MRI.getType(Reg).getSizeInBits() == 256 && MUI.isDivergent(Reg);173 case DivB512:174 return MRI.getType(Reg).getSizeInBits() == 512 && MUI.isDivergent(Reg);175 case _:176 return true;177 default:178 llvm_unreachable("missing matchUniformityAndLLT");179 }180}181 182bool PredicateMapping::match(const MachineInstr &MI,183 const MachineUniformityInfo &MUI,184 const MachineRegisterInfo &MRI) const {185 // Check LLT signature.186 for (unsigned i = 0; i < OpUniformityAndTypes.size(); ++i) {187 if (OpUniformityAndTypes[i] == _) {188 if (MI.getOperand(i).isReg())189 return false;190 continue;191 }192 193 // Remaining IDs check registers.194 if (!MI.getOperand(i).isReg())195 return false;196 197 if (!matchUniformityAndLLT(MI.getOperand(i).getReg(),198 OpUniformityAndTypes[i], MUI, MRI))199 return false;200 }201 202 // More complex check.203 if (TestFunc)204 return TestFunc(MI);205 206 return true;207}208 209SetOfRulesForOpcode::SetOfRulesForOpcode() = default;210 211SetOfRulesForOpcode::SetOfRulesForOpcode(FastRulesTypes FastTypes)212 : FastTypes(FastTypes) {}213 214UniformityLLTOpPredicateID LLTToId(LLT Ty) {215 if (Ty == LLT::scalar(16))216 return S16;217 if (Ty == LLT::scalar(32))218 return S32;219 if (Ty == LLT::scalar(64))220 return S64;221 if (Ty == LLT::fixed_vector(2, 16))222 return V2S16;223 if (Ty == LLT::fixed_vector(2, 32))224 return V2S32;225 if (Ty == LLT::fixed_vector(3, 32))226 return V3S32;227 if (Ty == LLT::fixed_vector(4, 32))228 return V4S32;229 return _;230}231 232UniformityLLTOpPredicateID LLTToBId(LLT Ty) {233 if (Ty == LLT::scalar(32) || Ty == LLT::fixed_vector(2, 16) ||234 isAnyPtr(Ty, 32))235 return B32;236 if (Ty == LLT::scalar(64) || Ty == LLT::fixed_vector(2, 32) ||237 Ty == LLT::fixed_vector(4, 16) || isAnyPtr(Ty, 64))238 return B64;239 if (Ty == LLT::fixed_vector(3, 32))240 return B96;241 if (Ty == LLT::fixed_vector(4, 32) || isAnyPtr(Ty, 128))242 return B128;243 return _;244}245 246const RegBankLLTMapping &247SetOfRulesForOpcode::findMappingForMI(const MachineInstr &MI,248 const MachineRegisterInfo &MRI,249 const MachineUniformityInfo &MUI) const {250 // Search in "Fast Rules".251 // Note: if fast rules are enabled, RegBankLLTMapping must be added in each252 // slot that could "match fast Predicate". If not, InvalidMapping is253 // returned which results in failure, does not search "Slow Rules".254 if (FastTypes != NoFastRules) {255 Register Reg = MI.getOperand(0).getReg();256 int Slot;257 if (FastTypes == StandardB)258 Slot = getFastPredicateSlot(LLTToBId(MRI.getType(Reg)));259 else260 Slot = getFastPredicateSlot(LLTToId(MRI.getType(Reg)));261 262 if (Slot != -1)263 return MUI.isUniform(Reg) ? Uni[Slot] : Div[Slot];264 }265 266 // Slow search for more complex rules.267 for (const RegBankLegalizeRule &Rule : Rules) {268 if (Rule.Predicate.match(MI, MUI, MRI))269 return Rule.OperandMapping;270 }271 272 LLVM_DEBUG(dbgs() << "MI: "; MI.dump(););273 llvm_unreachable("None of the rules defined for MI's opcode matched MI");274}275 276void SetOfRulesForOpcode::addRule(RegBankLegalizeRule Rule) {277 Rules.push_back(Rule);278}279 280void SetOfRulesForOpcode::addFastRuleDivergent(UniformityLLTOpPredicateID Ty,281 RegBankLLTMapping RuleApplyIDs) {282 int Slot = getFastPredicateSlot(Ty);283 assert(Slot != -1 && "Ty unsupported in this FastRulesTypes");284 Div[Slot] = RuleApplyIDs;285}286 287void SetOfRulesForOpcode::addFastRuleUniform(UniformityLLTOpPredicateID Ty,288 RegBankLLTMapping RuleApplyIDs) {289 int Slot = getFastPredicateSlot(Ty);290 assert(Slot != -1 && "Ty unsupported in this FastRulesTypes");291 Uni[Slot] = RuleApplyIDs;292}293 294int SetOfRulesForOpcode::getFastPredicateSlot(295 UniformityLLTOpPredicateID Ty) const {296 switch (FastTypes) {297 case Standard: {298 switch (Ty) {299 case S32:300 return 0;301 case S16:302 return 1;303 case S64:304 return 2;305 case V2S16:306 return 3;307 default:308 return -1;309 }310 }311 case StandardB: {312 switch (Ty) {313 case B32:314 return 0;315 case B64:316 return 1;317 case B96:318 return 2;319 case B128:320 return 3;321 default:322 return -1;323 }324 }325 case Vector: {326 switch (Ty) {327 case S32:328 return 0;329 case V2S32:330 return 1;331 case V3S32:332 return 2;333 case V4S32:334 return 3;335 default:336 return -1;337 }338 }339 default:340 return -1;341 }342}343 344RegBankLegalizeRules::RuleSetInitializer345RegBankLegalizeRules::addRulesForGOpcs(std::initializer_list<unsigned> OpcList,346 FastRulesTypes FastTypes) {347 return RuleSetInitializer(OpcList, GRulesAlias, GRules, FastTypes);348}349 350RegBankLegalizeRules::RuleSetInitializer351RegBankLegalizeRules::addRulesForIOpcs(std::initializer_list<unsigned> OpcList,352 FastRulesTypes FastTypes) {353 return RuleSetInitializer(OpcList, IRulesAlias, IRules, FastTypes);354}355 356const SetOfRulesForOpcode &357RegBankLegalizeRules::getRulesForOpc(MachineInstr &MI) const {358 unsigned Opc = MI.getOpcode();359 if (Opc == AMDGPU::G_INTRINSIC || Opc == AMDGPU::G_INTRINSIC_CONVERGENT ||360 Opc == AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS ||361 Opc == AMDGPU::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS) {362 unsigned IntrID = cast<GIntrinsic>(MI).getIntrinsicID();363 auto IRAIt = IRulesAlias.find(IntrID);364 if (IRAIt == IRulesAlias.end()) {365 LLVM_DEBUG(dbgs() << "MI: "; MI.dump(););366 llvm_unreachable("No rules defined for intrinsic opcode");367 }368 return IRules.at(IRAIt->second);369 }370 371 auto GRAIt = GRulesAlias.find(Opc);372 if (GRAIt == GRulesAlias.end()) {373 LLVM_DEBUG(dbgs() << "MI: "; MI.dump(););374 llvm_unreachable("No rules defined for generic opcode");375 }376 return GRules.at(GRAIt->second);377}378 379// Syntactic sugar wrapper for predicate lambda that enables '&&', '||' and '!'.380class Predicate {381private:382 struct Elt {383 // Save formula composed of Pred, '&&', '||' and '!' as a jump table.384 // Sink ! to Pred. For example !((A && !B) || C) -> (!A || B) && !C385 // Sequences of && and || will be represented by jumps, for example:386 // (A && B && ... X) or (A && B && ... X) || Y387 // A == true jump to B388 // A == false jump to end or Y, result is A(false) or Y389 // (A || B || ... X) or (A || B || ... X) && Y390 // A == true jump to end or Y, result is A(true) or Y391 // A == false jump to B392 // Notice that when negating expression, we simply flip Neg on each Pred393 // and swap TJumpOffset and FJumpOffset (&& becomes ||, || becomes &&).394 std::function<bool(const MachineInstr &)> Pred;395 bool Neg; // Neg of Pred is calculated before jump396 unsigned TJumpOffset;397 unsigned FJumpOffset;398 };399 400 SmallVector<Elt, 8> Expression;401 402 Predicate(SmallVectorImpl<Elt> &&Expr) { Expression.swap(Expr); };403 404public:405 Predicate(std::function<bool(const MachineInstr &)> Pred) {406 Expression.push_back({Pred, false, 1, 1});407 };408 409 bool operator()(const MachineInstr &MI) const {410 unsigned Idx = 0;411 unsigned ResultIdx = Expression.size();412 bool Result;413 do {414 Result = Expression[Idx].Pred(MI);415 Result = Expression[Idx].Neg ? !Result : Result;416 if (Result) {417 Idx += Expression[Idx].TJumpOffset;418 } else {419 Idx += Expression[Idx].FJumpOffset;420 }421 } while ((Idx != ResultIdx));422 423 return Result;424 };425 426 Predicate operator!() const {427 SmallVector<Elt, 8> NegExpression;428 for (const Elt &ExprElt : Expression) {429 NegExpression.push_back({ExprElt.Pred, !ExprElt.Neg, ExprElt.FJumpOffset,430 ExprElt.TJumpOffset});431 }432 return Predicate(std::move(NegExpression));433 };434 435 Predicate operator&&(const Predicate &RHS) const {436 SmallVector<Elt, 8> AndExpression = Expression;437 438 unsigned RHSSize = RHS.Expression.size();439 unsigned ResultIdx = Expression.size();440 for (unsigned i = 0; i < ResultIdx; ++i) {441 // LHS results in false, whole expression results in false.442 if (i + AndExpression[i].FJumpOffset == ResultIdx)443 AndExpression[i].FJumpOffset += RHSSize;444 }445 446 AndExpression.append(RHS.Expression);447 448 return Predicate(std::move(AndExpression));449 }450 451 Predicate operator||(const Predicate &RHS) const {452 SmallVector<Elt, 8> OrExpression = Expression;453 454 unsigned RHSSize = RHS.Expression.size();455 unsigned ResultIdx = Expression.size();456 for (unsigned i = 0; i < ResultIdx; ++i) {457 // LHS results in true, whole expression results in true.458 if (i + OrExpression[i].TJumpOffset == ResultIdx)459 OrExpression[i].TJumpOffset += RHSSize;460 }461 462 OrExpression.append(RHS.Expression);463 464 return Predicate(std::move(OrExpression));465 }466};467 468// Initialize rules469RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,470 MachineRegisterInfo &_MRI)471 : ST(&_ST), MRI(&_MRI) {472 473 addRulesForGOpcs({G_ADD, G_SUB}, Standard)474 .Uni(S16, {{Sgpr32Trunc}, {Sgpr32AExt, Sgpr32AExt}})475 .Div(S16, {{Vgpr16}, {Vgpr16, Vgpr16}})476 .Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32}})477 .Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}})478 .Uni(V2S16, {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackAExt})479 .Div(V2S16, {{VgprV2S16}, {VgprV2S16, VgprV2S16}})480 .Uni(S64, {{Sgpr64}, {Sgpr64, Sgpr64}})481 .Div(S64, {{Vgpr64}, {Vgpr64, Vgpr64}});482 483 addRulesForGOpcs({G_UADDO, G_USUBO}, Standard)484 .Uni(S32, {{Sgpr32, Sgpr32Trunc}, {Sgpr32, Sgpr32}})485 .Div(S32, {{Vgpr32, Vcc}, {Vgpr32, Vgpr32}});486 487 addRulesForGOpcs({G_UADDE, G_USUBE}, Standard)488 .Uni(S32, {{Sgpr32, Sgpr32Trunc}, {Sgpr32, Sgpr32, Sgpr32AExtBoolInReg}})489 .Div(S32, {{Vgpr32, Vcc}, {Vgpr32, Vgpr32, Vcc}});490 491 addRulesForGOpcs({G_MUL}, Standard).Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}});492 493 addRulesForGOpcs({G_XOR, G_OR, G_AND}, StandardB)494 .Any({{UniS1}, {{Sgpr32Trunc}, {Sgpr32AExt, Sgpr32AExt}}})495 .Any({{DivS1}, {{Vcc}, {Vcc, Vcc}}})496 .Any({{UniS16}, {{Sgpr16}, {Sgpr16, Sgpr16}}})497 .Any({{DivS16}, {{Vgpr16}, {Vgpr16, Vgpr16}}})498 .Uni(B32, {{SgprB32}, {SgprB32, SgprB32}})499 .Div(B32, {{VgprB32}, {VgprB32, VgprB32}})500 .Uni(B64, {{SgprB64}, {SgprB64, SgprB64}})501 .Div(B64, {{VgprB64}, {VgprB64, VgprB64}, SplitTo32});502 503 addRulesForGOpcs({G_SHL}, Standard)504 .Uni(S16, {{Sgpr32Trunc}, {Sgpr32AExt, Sgpr32ZExt}})505 .Div(S16, {{Vgpr16}, {Vgpr16, Vgpr16}})506 .Uni(V2S16, {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackBitShift})507 .Div(V2S16, {{VgprV2S16}, {VgprV2S16, VgprV2S16}})508 .Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32}})509 .Uni(S64, {{Sgpr64}, {Sgpr64, Sgpr32}})510 .Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}})511 .Div(S64, {{Vgpr64}, {Vgpr64, Vgpr32}});512 513 addRulesForGOpcs({G_LSHR}, Standard)514 .Uni(S16, {{Sgpr32Trunc}, {Sgpr32ZExt, Sgpr32ZExt}})515 .Div(S16, {{Vgpr16}, {Vgpr16, Vgpr16}})516 .Uni(V2S16, {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackBitShift})517 .Div(V2S16, {{VgprV2S16}, {VgprV2S16, VgprV2S16}})518 .Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32}})519 .Uni(S64, {{Sgpr64}, {Sgpr64, Sgpr32}})520 .Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}})521 .Div(S64, {{Vgpr64}, {Vgpr64, Vgpr32}});522 523 addRulesForGOpcs({G_ASHR}, Standard)524 .Uni(S16, {{Sgpr32Trunc}, {Sgpr32SExt, Sgpr32ZExt}})525 .Div(S16, {{Vgpr16}, {Vgpr16, Vgpr16}})526 .Uni(V2S16, {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackBitShift})527 .Div(V2S16, {{VgprV2S16}, {VgprV2S16, VgprV2S16}})528 .Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32}})529 .Uni(S64, {{Sgpr64}, {Sgpr64, Sgpr32}})530 .Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}})531 .Div(S64, {{Vgpr64}, {Vgpr64, Vgpr32}});532 533 addRulesForGOpcs({G_FSHR}, Standard)534 .Uni(S32, {{UniInVgprS32}, {Vgpr32, Vgpr32, Vgpr32}})535 .Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}});536 537 addRulesForGOpcs({G_FRAME_INDEX}).Any({{UniP5, _}, {{SgprP5}, {None}}});538 539 addRulesForGOpcs({G_UBFX, G_SBFX}, Standard)540 .Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32, Sgpr32}, S_BFE})541 .Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}})542 .Uni(S64, {{Sgpr64}, {Sgpr64, Sgpr32, Sgpr32}, S_BFE})543 .Div(S64, {{Vgpr64}, {Vgpr64, Vgpr32, Vgpr32}, V_BFE});544 545 addRulesForGOpcs({G_SMIN, G_SMAX}, Standard)546 .Uni(S16, {{Sgpr32Trunc}, {Sgpr32SExt, Sgpr32SExt}})547 .Div(S16, {{Vgpr16}, {Vgpr16, Vgpr16}})548 .Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32}})549 .Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}})550 .Uni(V2S16, {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackMinMax})551 .Div(V2S16, {{VgprV2S16}, {VgprV2S16, VgprV2S16}});552 553 addRulesForGOpcs({G_UMIN, G_UMAX}, Standard)554 .Uni(S16, {{Sgpr32Trunc}, {Sgpr32ZExt, Sgpr32ZExt}})555 .Div(S16, {{Vgpr16}, {Vgpr16, Vgpr16}})556 .Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32}})557 .Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}})558 .Uni(V2S16, {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackMinMax})559 .Div(V2S16, {{VgprV2S16}, {VgprV2S16, VgprV2S16}});560 561 // Note: we only write S1 rules for G_IMPLICIT_DEF, G_CONSTANT, G_FCONSTANT562 // and G_FREEZE here, rest is trivially regbankselected earlier563 addRulesForGOpcs({G_IMPLICIT_DEF}).Any({{UniS1}, {{Sgpr32Trunc}, {}}});564 addRulesForGOpcs({G_CONSTANT})565 .Any({{UniS1, _}, {{Sgpr32Trunc}, {None}, UniCstExt}});566 addRulesForGOpcs({G_FREEZE}).Any({{DivS1}, {{Vcc}, {Vcc}}});567 568 addRulesForGOpcs({G_ICMP})569 .Any({{UniS1, _, S32}, {{Sgpr32Trunc}, {None, Sgpr32, Sgpr32}}})570 .Any({{DivS1, _, S32}, {{Vcc}, {None, Vgpr32, Vgpr32}}})571 .Any({{DivS1, _, S64}, {{Vcc}, {None, Vgpr64, Vgpr64}}});572 573 addRulesForGOpcs({G_FCMP})574 .Any({{UniS1, _, S32}, {{UniInVcc}, {None, Vgpr32, Vgpr32}}})575 .Any({{DivS1, _, S32}, {{Vcc}, {None, Vgpr32, Vgpr32}}});576 577 addRulesForGOpcs({G_BRCOND})578 .Any({{UniS1}, {{}, {Sgpr32AExtBoolInReg}}})579 .Any({{DivS1}, {{}, {Vcc}}});580 581 addRulesForGOpcs({G_BR}).Any({{_}, {{}, {None}}});582 583 addRulesForGOpcs({G_SELECT}, StandardB)584 .Any({{DivS16}, {{Vgpr16}, {Vcc, Vgpr16, Vgpr16}}})585 .Any({{UniS16}, {{Sgpr16}, {Sgpr32AExtBoolInReg, Sgpr16, Sgpr16}}})586 .Div(B32, {{VgprB32}, {Vcc, VgprB32, VgprB32}})587 .Uni(B32, {{SgprB32}, {Sgpr32AExtBoolInReg, SgprB32, SgprB32}})588 .Div(B64, {{VgprB64}, {Vcc, VgprB64, VgprB64}, SplitTo32Select})589 .Uni(B64, {{SgprB64}, {Sgpr32AExtBoolInReg, SgprB64, SgprB64}});590 591 addRulesForGOpcs({G_ANYEXT})592 .Any({{UniS16, S1}, {{None}, {None}}}) // should be combined away593 .Any({{UniS32, S1}, {{None}, {None}}}) // should be combined away594 .Any({{UniS64, S1}, {{None}, {None}}}) // should be combined away595 .Any({{DivS16, S1}, {{Vgpr16}, {Vcc}, VccExtToSel}})596 .Any({{DivS32, S1}, {{Vgpr32}, {Vcc}, VccExtToSel}})597 .Any({{DivS64, S1}, {{Vgpr64}, {Vcc}, VccExtToSel}})598 .Any({{UniS64, S32}, {{Sgpr64}, {Sgpr32}, Ext32To64}})599 .Any({{DivS64, S32}, {{Vgpr64}, {Vgpr32}, Ext32To64}})600 .Any({{UniS32, S16}, {{Sgpr32}, {Sgpr16}}})601 .Any({{DivS32, S16}, {{Vgpr32}, {Vgpr16}}});602 603 // In global-isel G_TRUNC in-reg is treated as no-op, inst selected into COPY.604 // It is up to user to deal with truncated bits.605 addRulesForGOpcs({G_TRUNC})606 .Any({{UniS1, UniS16}, {{None}, {None}}}) // should be combined away607 .Any({{UniS1, UniS32}, {{None}, {None}}}) // should be combined away608 .Any({{UniS1, UniS64}, {{None}, {None}}}) // should be combined away609 .Any({{UniS16, S32}, {{Sgpr16}, {Sgpr32}}})610 .Any({{DivS16, S32}, {{Vgpr16}, {Vgpr32}}})611 .Any({{UniS32, S64}, {{Sgpr32}, {Sgpr64}}})612 .Any({{DivS32, S64}, {{Vgpr32}, {Vgpr64}}})613 .Any({{UniV2S16, V2S32}, {{SgprV2S16}, {SgprV2S32}}})614 .Any({{DivV2S16, V2S32}, {{VgprV2S16}, {VgprV2S32}}})615 // This is non-trivial. VgprToVccCopy is done using compare instruction.616 .Any({{DivS1, DivS16}, {{Vcc}, {Vgpr16}, VgprToVccCopy}})617 .Any({{DivS1, DivS32}, {{Vcc}, {Vgpr32}, VgprToVccCopy}})618 .Any({{DivS1, DivS64}, {{Vcc}, {Vgpr64}, VgprToVccCopy}});619 620 addRulesForGOpcs({G_ZEXT})621 .Any({{UniS16, S1}, {{Sgpr32Trunc}, {Sgpr32AExtBoolInReg}, UniExtToSel}})622 .Any({{UniS32, S1}, {{Sgpr32}, {Sgpr32AExtBoolInReg}, UniExtToSel}})623 .Any({{UniS64, S1}, {{Sgpr64}, {Sgpr32AExtBoolInReg}, UniExtToSel}})624 .Any({{DivS16, S1}, {{Vgpr16}, {Vcc}, VccExtToSel}})625 .Any({{DivS32, S1}, {{Vgpr32}, {Vcc}, VccExtToSel}})626 .Any({{DivS64, S1}, {{Vgpr64}, {Vcc}, VccExtToSel}})627 .Any({{UniS64, S32}, {{Sgpr64}, {Sgpr32}, Ext32To64}})628 .Any({{DivS64, S32}, {{Vgpr64}, {Vgpr32}, Ext32To64}})629 // not extending S16 to S32 is questionable.630 .Any({{UniS64, S16}, {{Sgpr64}, {Sgpr32ZExt}, Ext32To64}})631 .Any({{DivS64, S16}, {{Vgpr64}, {Vgpr32ZExt}, Ext32To64}})632 .Any({{UniS32, S16}, {{Sgpr32}, {Sgpr16}}})633 .Any({{DivS32, S16}, {{Vgpr32}, {Vgpr16}}});634 635 addRulesForGOpcs({G_SEXT})636 .Any({{UniS16, S1}, {{Sgpr32Trunc}, {Sgpr32AExtBoolInReg}, UniExtToSel}})637 .Any({{UniS32, S1}, {{Sgpr32}, {Sgpr32AExtBoolInReg}, UniExtToSel}})638 .Any({{UniS64, S1}, {{Sgpr64}, {Sgpr32AExtBoolInReg}, UniExtToSel}})639 .Any({{DivS16, S1}, {{Vgpr16}, {Vcc}, VccExtToSel}})640 .Any({{DivS32, S1}, {{Vgpr32}, {Vcc}, VccExtToSel}})641 .Any({{DivS64, S1}, {{Vgpr64}, {Vcc}, VccExtToSel}})642 .Any({{UniS64, S32}, {{Sgpr64}, {Sgpr32}, Ext32To64}})643 .Any({{DivS64, S32}, {{Vgpr64}, {Vgpr32}, Ext32To64}})644 // not extending S16 to S32 is questionable.645 .Any({{UniS64, S16}, {{Sgpr64}, {Sgpr32SExt}, Ext32To64}})646 .Any({{DivS64, S16}, {{Vgpr64}, {Vgpr32SExt}, Ext32To64}})647 .Any({{UniS32, S16}, {{Sgpr32}, {Sgpr16}}})648 .Any({{DivS32, S16}, {{Vgpr32}, {Vgpr16}}});649 650 addRulesForGOpcs({G_SEXT_INREG})651 .Any({{UniS32, S32}, {{Sgpr32}, {Sgpr32}}})652 .Any({{DivS32, S32}, {{Vgpr32}, {Vgpr32}}})653 .Any({{UniS64, S64}, {{Sgpr64}, {Sgpr64}}})654 .Any({{DivS64, S64}, {{Vgpr64}, {Vgpr64}, SplitTo32SExtInReg}});655 656 addRulesForGOpcs({G_ASSERT_ZEXT, G_ASSERT_SEXT}, Standard)657 .Uni(S32, {{Sgpr32}, {Sgpr32, Imm}})658 .Div(S32, {{Vgpr32}, {Vgpr32, Imm}})659 .Uni(S64, {{Sgpr64}, {Sgpr64, Imm}})660 .Div(S64, {{Vgpr64}, {Vgpr64, Imm}});661 662 bool hasSMRDx3 = ST->hasScalarDwordx3Loads();663 bool hasSMRDSmall = ST->hasScalarSubwordLoads();664 bool usesTrue16 = ST->useRealTrue16Insts();665 666 Predicate isAlign16([](const MachineInstr &MI) -> bool {667 return (*MI.memoperands_begin())->getAlign() >= Align(16);668 });669 670 Predicate isAlign4([](const MachineInstr &MI) -> bool {671 return (*MI.memoperands_begin())->getAlign() >= Align(4);672 });673 674 Predicate isAtomicMMO([](const MachineInstr &MI) -> bool {675 return (*MI.memoperands_begin())->isAtomic();676 });677 678 Predicate isUniMMO([](const MachineInstr &MI) -> bool {679 return AMDGPU::isUniformMMO(*MI.memoperands_begin());680 });681 682 Predicate isConst([](const MachineInstr &MI) -> bool {683 // Address space in MMO be different then address space on pointer.684 const MachineMemOperand *MMO = *MI.memoperands_begin();685 const unsigned AS = MMO->getAddrSpace();686 return AS == AMDGPUAS::CONSTANT_ADDRESS ||687 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT;688 });689 690 Predicate isVolatileMMO([](const MachineInstr &MI) -> bool {691 return (*MI.memoperands_begin())->isVolatile();692 });693 694 Predicate isInvMMO([](const MachineInstr &MI) -> bool {695 return (*MI.memoperands_begin())->isInvariant();696 });697 698 Predicate isNoClobberMMO([](const MachineInstr &MI) -> bool {699 return (*MI.memoperands_begin())->getFlags() & MONoClobber;700 });701 702 Predicate isNaturalAligned([](const MachineInstr &MI) -> bool {703 const MachineMemOperand *MMO = *MI.memoperands_begin();704 return MMO->getAlign() >= Align(MMO->getSize().getValue());705 });706 707 Predicate is8Or16BitMMO([](const MachineInstr &MI) -> bool {708 const MachineMemOperand *MMO = *MI.memoperands_begin();709 const unsigned MemSize = 8 * MMO->getSize().getValue();710 return MemSize == 16 || MemSize == 8;711 });712 713 Predicate is32BitMMO([](const MachineInstr &MI) -> bool {714 const MachineMemOperand *MMO = *MI.memoperands_begin();715 return 8 * MMO->getSize().getValue() == 32;716 });717 718 auto isUL = !isAtomicMMO && isUniMMO && (isConst || !isVolatileMMO) &&719 (isConst || isInvMMO || isNoClobberMMO);720 721 // clang-format off722 // TODO: S32Dst, 16-bit any-extending load should not appear on True16 targets723 addRulesForGOpcs({G_LOAD})724 // flat, addrspace(0), never uniform - flat_load725 .Any({{DivS16, P0}, {{Vgpr16}, {VgprP0}}}, usesTrue16)726 .Any({{DivB32, P0}, {{VgprB32}, {VgprP0}}}) // 32-bit load, 8-bit and 16-bit any-extending load727 .Any({{DivB64, P0}, {{VgprB64}, {VgprP0}}})728 .Any({{DivB96, P0}, {{VgprB96}, {VgprP0}}})729 .Any({{DivB128, P0}, {{VgprB128}, {VgprP0}}})730 731 // global, addrspace(1)732 // divergent - global_load733 .Any({{DivS16, P1}, {{Vgpr16}, {VgprP1}}}, usesTrue16)734 .Any({{DivB32, P1}, {{VgprB32}, {VgprP1}}}) //32-bit load, 8-bit and 16-bit any-extending load735 .Any({{DivB64, P1}, {{VgprB64}, {VgprP1}}})736 .Any({{DivB96, P1}, {{VgprB96}, {VgprP1}}})737 .Any({{DivB128, P1}, {{VgprB128}, {VgprP1}}})738 .Any({{DivB256, P1}, {{VgprB256}, {VgprP1}, SplitLoad}})739 .Any({{DivB512, P1}, {{VgprB512}, {VgprP1}, SplitLoad}})740 741 // uniform - s_load742 .Any({{{UniS16, P1}, isNaturalAligned && isUL}, {{Sgpr32Trunc}, {SgprP1}}}, usesTrue16 && hasSMRDSmall) // s16 load743 .Any({{{UniS16, P1}, isAlign4 && isUL}, {{Sgpr32Trunc}, {SgprP1}, WidenMMOToS32}}, usesTrue16 && !hasSMRDSmall) // s16 load to 32-bit load744 .Any({{{UniB32, P1}, isNaturalAligned && isUL}, {{SgprB32}, {SgprP1}}}, hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load745 // TODO: SplitLoad when !isNaturalAligned && isUL and target hasSMRDSmall746 .Any({{{UniB32, P1}, is8Or16BitMMO && isAlign4 && isUL}, {{SgprB32}, {SgprP1}, WidenMMOToS32}}, !hasSMRDSmall) //8-bit and 16-bit any-extending load to 32-bit load747 .Any({{{UniB32, P1}, is32BitMMO && isAlign4 && isUL}, {{SgprB32}, {SgprP1}}}) //32-bit load748 .Any({{{UniB64, P1}, isAlign4 && isUL}, {{SgprB64}, {SgprP1}}})749 .Any({{{UniB96, P1}, isAlign16 && isUL}, {{SgprB96}, {SgprP1}, WidenLoad}}, !hasSMRDx3)750 .Any({{{UniB96, P1}, isAlign4 && !isAlign16 && isUL}, {{SgprB96}, {SgprP1}, SplitLoad}}, !hasSMRDx3)751 .Any({{{UniB96, P1}, isAlign4 && isUL}, {{SgprB96}, {SgprP1}}}, hasSMRDx3)752 .Any({{{UniB128, P1}, isAlign4 && isUL}, {{SgprB128}, {SgprP1}}})753 .Any({{{UniB256, P1}, isAlign4 && isUL}, {{SgprB256}, {SgprP1}}})754 .Any({{{UniB512, P1}, isAlign4 && isUL}, {{SgprB512}, {SgprP1}}})755 756 // Uniform via global or buffer load, for example volatile or non-aligned757 // uniform load. Not using standard {{UniInVgprTy}, {VgprP1}} since it is758 // selected as global_load, use SgprP1 for pointer instead to match759 // patterns without flat-for-global, default for GFX7 and older.760 // -> +flat-for-global + {{UniInVgprTy}, {SgprP1}} - global_load761 // -> -flat-for-global + {{UniInVgprTy}, {SgprP1}} - buffer_load762 .Any({{{UniS16, P1}, !isNaturalAligned || !isUL}, {{UniInVgprS16}, {SgprP1}}}, usesTrue16 && hasSMRDSmall) // s16 load763 .Any({{{UniS16, P1}, !isAlign4 || !isUL}, {{UniInVgprS16}, {SgprP1}}}, usesTrue16 && !hasSMRDSmall) // s16 load764 .Any({{{UniB32, P1}, !isNaturalAligned || !isUL}, {{UniInVgprB32}, {SgprP1}}}, hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load765 .Any({{{UniB32, P1}, !isAlign4 || !isUL}, {{UniInVgprB32}, {SgprP1}}}, !hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load766 .Any({{{UniB64, P1}, !isAlign4 || !isUL}, {{UniInVgprB64}, {SgprP1}}})767 .Any({{{UniB96, P1}, !isAlign4 || !isUL}, {{UniInVgprB96}, {SgprP1}}})768 .Any({{{UniB128, P1}, !isAlign4 || !isUL}, {{UniInVgprB128}, {SgprP1}}})769 .Any({{{UniB256, P1}, !isAlign4 || !isUL}, {{UniInVgprB256}, {SgprP1}, SplitLoad}})770 .Any({{{UniB512, P1}, !isAlign4 || !isUL}, {{UniInVgprB512}, {SgprP1}, SplitLoad}})771 772 // local, addrspace(3) - ds_load773 .Any({{DivS16, P3}, {{Vgpr16}, {VgprP3}}}, usesTrue16)774 .Any({{DivB32, P3}, {{VgprB32}, {VgprP3}}}) // 32-bit load, 8-bit and 16-bit any-extending load775 .Any({{DivB64, P3}, {{VgprB64}, {VgprP3}}})776 .Any({{DivB96, P3}, {{VgprB96}, {VgprP3}}})777 .Any({{DivB128, P3}, {{VgprB128}, {VgprP3}}})778 779 .Any({{UniS16, P3}, {{UniInVgprS16}, {SgprP3}}}, usesTrue16) // 16-bit load780 .Any({{UniB32, P3}, {{UniInVgprB32}, {VgprP3}}}) // 32-bit load, 8-bit and 16-bit any-extending load781 .Any({{UniB64, P3}, {{UniInVgprB64}, {VgprP3}}})782 .Any({{UniB96, P3}, {{UniInVgprB96}, {VgprP3}}})783 .Any({{UniB128, P3}, {{UniInVgprB128}, {VgprP3}}})784 785 // constant, addrspace(4)786 // divergent - global_load787 .Any({{DivS16, P4}, {{Vgpr16}, {VgprP4}}}, usesTrue16)788 .Any({{DivB32, P4}, {{VgprB32}, {VgprP4}}}) //32-bit load, 8-bit and 16-bit any-extending load789 .Any({{DivB64, P4}, {{VgprB64}, {VgprP4}}})790 .Any({{DivB96, P4}, {{VgprB96}, {VgprP4}}})791 .Any({{DivB128, P4}, {{VgprB128}, {VgprP4}}})792 .Any({{DivB256, P4}, {{VgprB256}, {VgprP4}, SplitLoad}})793 .Any({{DivB512, P4}, {{VgprB512}, {VgprP4}, SplitLoad}})794 795 // uniform - s_load796 .Any({{{UniS16, P4}, isNaturalAligned && isUL}, {{Sgpr32Trunc}, {SgprP4}}}, usesTrue16 && hasSMRDSmall) // s16 load797 .Any({{{UniS16, P4}, isAlign4 && isUL}, {{Sgpr32Trunc}, {SgprP4}, WidenMMOToS32}}, usesTrue16 && !hasSMRDSmall) // s16 load to 32-bit load798 .Any({{{UniB32, P4}, isNaturalAligned && isUL}, {{SgprB32}, {SgprP4}}}, hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load799 .Any({{{UniB32, P4}, is8Or16BitMMO && isAlign4 && isUL}, {{SgprB32}, {SgprP4}, WidenMMOToS32}}, !hasSMRDSmall) //8-bit and 16-bit any-extending load to 32-bit load800 .Any({{{UniB32, P4}, is32BitMMO && isAlign4 && isUL}, {{SgprB32}, {SgprP4}}}) //32-bit load801 .Any({{{UniB64, P4}, isAlign4 && isUL}, {{SgprB64}, {SgprP4}}})802 .Any({{{UniB96, P4}, isAlign16 && isUL}, {{SgprB96}, {SgprP4}, WidenLoad}}, !hasSMRDx3)803 .Any({{{UniB96, P4}, isAlign4 && !isAlign16 && isUL}, {{SgprB96}, {SgprP4}, SplitLoad}}, !hasSMRDx3)804 .Any({{{UniB96, P4}, isAlign4 && isUL}, {{SgprB96}, {SgprP4}}}, hasSMRDx3)805 .Any({{{UniB128, P4}, isAlign4 && isUL}, {{SgprB128}, {SgprP4}}})806 .Any({{{UniB256, P4}, isAlign4 && isUL}, {{SgprB256}, {SgprP4}}})807 .Any({{{UniB512, P4}, isAlign4 && isUL}, {{SgprB512}, {SgprP4}}})808 809 // uniform in vgpr - global_load or buffer_load810 .Any({{{UniS16, P4}, !isNaturalAligned || !isUL}, {{UniInVgprS16}, {SgprP4}}}, usesTrue16 && hasSMRDSmall) // s16 load811 .Any({{{UniS16, P4}, !isAlign4 || !isUL}, {{UniInVgprS16}, {SgprP4}}}, usesTrue16 && !hasSMRDSmall) // s16 load812 .Any({{{UniB32, P4}, !isNaturalAligned || !isUL}, {{UniInVgprB32}, {SgprP4}}}, hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load813 .Any({{{UniB32, P4}, !isAlign4 || !isUL}, {{UniInVgprB32}, {SgprP4}}}, !hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load814 .Any({{{UniB64, P4}, !isAlign4 || !isUL}, {{UniInVgprB64}, {SgprP4}}})815 .Any({{{UniB96, P4}, !isAlign4 || !isUL}, {{UniInVgprB96}, {SgprP4}}})816 .Any({{{UniB128, P4}, !isAlign4 || !isUL}, {{UniInVgprB128}, {SgprP4}}})817 .Any({{{UniB256, P4}, !isAlign4 || !isUL}, {{UniInVgprB256}, {SgprP4}, SplitLoad}})818 .Any({{{UniB512, P4}, !isAlign4 || !isUL}, {{UniInVgprB512}, {SgprP4}, SplitLoad}})819 820 // private, addrspace(5), never uniform - scratch_load821 .Any({{DivS16, P5}, {{Vgpr16}, {VgprP5}}}, usesTrue16)822 .Any({{DivB32, P5}, {{VgprB32}, {VgprP5}}}) // 32-bit load, 8-bit and 16-bit any-extending load823 .Any({{DivB64, P5}, {{VgprB64}, {VgprP5}}})824 .Any({{DivB96, P5}, {{VgprB96}, {VgprP5}}})825 .Any({{DivB128, P5}, {{VgprB128}, {VgprP5}}})826 827 .Any({{DivS32, Ptr128}, {{Vgpr32}, {VgprPtr128}}});828 829 830 addRulesForGOpcs({G_ZEXTLOAD, G_SEXTLOAD}) // i8 and i16 zeroextending loads831 .Any({{DivS32, P0}, {{Vgpr32}, {VgprP0}}})832 833 .Any({{DivS32, P1}, {{Vgpr32}, {VgprP1}}})834 .Any({{{UniS32, P1}, isAlign4 && isUL}, {{Sgpr32}, {SgprP1}, WidenMMOToS32}}, !hasSMRDSmall)835 .Any({{{UniS32, P1}, isNaturalAligned && isUL}, {{Sgpr32}, {SgprP1}}}, hasSMRDSmall)836 .Any({{{UniS32, P1}, !isAlign4 || !isUL}, {{UniInVgprS32}, {SgprP1}}}, !hasSMRDSmall)837 .Any({{{UniS32, P1}, !isNaturalAligned || !isUL}, {{UniInVgprS32}, {SgprP1}}}, hasSMRDSmall)838 839 .Any({{DivS32, P3}, {{Vgpr32}, {VgprP3}}})840 .Any({{UniS32, P3}, {{UniInVgprS32}, {VgprP3}}})841 842 .Any({{DivS32, P4}, {{Vgpr32}, {VgprP4}}})843 .Any({{{UniS32, P4}, isAlign4 && isUL}, {{Sgpr32}, {SgprP4}, WidenMMOToS32}}, !hasSMRDSmall)844 .Any({{{UniS32, P4}, isNaturalAligned && isUL}, {{Sgpr32}, {SgprP4}}}, hasSMRDSmall)845 .Any({{{UniS32, P4}, !isAlign4 || !isUL}, {{UniInVgprS32}, {SgprP4}}}, !hasSMRDSmall)846 .Any({{{UniS32, P4}, !isNaturalAligned || !isUL}, {{UniInVgprS32}, {SgprP4}}}, hasSMRDSmall)847 848 .Any({{DivS32, P5}, {{Vgpr32}, {VgprP5}}});849 850 addRulesForGOpcs({G_STORE})851 // addrspace(0)852 .Any({{S16, P0}, {{}, {Vgpr16, VgprP0}}}, usesTrue16) // 16-bit store853 .Any({{B32, P0}, {{}, {VgprB32, VgprP0}}}) // 32-bit store, 8-bit and 16-bit truncating store854 .Any({{B64, P0}, {{}, {VgprB64, VgprP0}}})855 .Any({{B96, P0}, {{}, {VgprB96, VgprP0}}})856 .Any({{B128, P0}, {{}, {VgprB128, VgprP0}}})857 858 // addrspace(1), there are no stores to addrspace(4)859 // For targets:860 // - with "+flat-for-global" - global_store861 // - without(-flat-for-global) - buffer_store addr64862 .Any({{S16, DivP1}, {{}, {Vgpr16, VgprP1}}}, usesTrue16) // 16-bit store863 .Any({{B32, DivP1}, {{}, {VgprB32, VgprP1}}}) // 32-bit store, 8-bit and 16-bit truncating store864 .Any({{B64, DivP1}, {{}, {VgprB64, VgprP1}}})865 .Any({{B96, DivP1}, {{}, {VgprB96, VgprP1}}})866 .Any({{B128, DivP1}, {{}, {VgprB128, VgprP1}}})867 868 // For UniP1, use sgpr ptr to match flat-for-global patterns. Targets:869 // - with "+flat-for-global" - global_store for both sgpr and vgpr ptr870 // - without(-flat-for-global) - need sgpr ptr to select buffer_store871 .Any({{S16, UniP1}, {{}, {Vgpr16, SgprP1}}}, usesTrue16) // 16-bit store872 .Any({{B32, UniP1}, {{}, {VgprB32, SgprP1}}}) // 32-bit store, 8-bit and 16-bit truncating store873 .Any({{B64, UniP1}, {{}, {VgprB64, SgprP1}}})874 .Any({{B96, UniP1}, {{}, {VgprB96, SgprP1}}})875 .Any({{B128, UniP1}, {{}, {VgprB128, SgprP1}}})876 877 // addrspace(3) and addrspace(5)878 .Any({{S16, Ptr32}, {{}, {Vgpr16, VgprPtr32}}}, usesTrue16) // 16-bit store879 .Any({{B32, Ptr32}, {{}, {VgprB32, VgprPtr32}}}) // 32-bit store, 8-bit and 16-bit truncating store880 .Any({{B64, Ptr32}, {{}, {VgprB64, VgprPtr32}}})881 .Any({{B96, Ptr32}, {{}, {VgprB96, VgprPtr32}}})882 .Any({{B128, Ptr32}, {{}, {VgprB128, VgprPtr32}}});883 // clang-format on884 885 addRulesForGOpcs({G_AMDGPU_BUFFER_LOAD, G_AMDGPU_BUFFER_LOAD_FORMAT,886 G_AMDGPU_TBUFFER_LOAD_FORMAT},887 StandardB)888 .Div(B32, {{VgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})889 .Uni(B32, {{UniInVgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})890 .Div(B64, {{VgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})891 .Uni(B64, {{UniInVgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})892 .Div(B96, {{VgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})893 .Uni(B96, {{UniInVgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})894 .Div(B128, {{VgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})895 .Uni(B128, {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});896 897 addRulesForGOpcs({G_AMDGPU_BUFFER_STORE})898 .Any({{S32}, {{}, {Vgpr32, SgprV4S32, Vgpr32, Vgpr32, Sgpr32}}});899 900 addRulesForGOpcs({G_PTR_ADD})901 .Any({{UniPtr32}, {{SgprPtr32}, {SgprPtr32, Sgpr32}}})902 .Any({{DivPtr32}, {{VgprPtr32}, {VgprPtr32, Vgpr32}}})903 .Any({{UniPtr64}, {{SgprPtr64}, {SgprPtr64, Sgpr64}}})904 .Any({{DivPtr64}, {{VgprPtr64}, {VgprPtr64, Vgpr64}}});905 906 addRulesForGOpcs({G_INTTOPTR})907 .Any({{UniPtr32}, {{SgprPtr32}, {Sgpr32}}})908 .Any({{DivPtr32}, {{VgprPtr32}, {Vgpr32}}})909 .Any({{UniPtr64}, {{SgprPtr64}, {Sgpr64}}})910 .Any({{DivPtr64}, {{VgprPtr64}, {Vgpr64}}})911 .Any({{UniPtr128}, {{SgprPtr128}, {Sgpr128}}})912 .Any({{DivPtr128}, {{VgprPtr128}, {Vgpr128}}});913 914 addRulesForGOpcs({G_PTRTOINT})915 .Any({{UniS32}, {{Sgpr32}, {SgprPtr32}}})916 .Any({{DivS32}, {{Vgpr32}, {VgprPtr32}}})917 .Any({{UniS64}, {{Sgpr64}, {SgprPtr64}}})918 .Any({{DivS64}, {{Vgpr64}, {VgprPtr64}}})919 .Any({{UniS128}, {{Sgpr128}, {SgprPtr128}}})920 .Any({{DivS128}, {{Vgpr128}, {VgprPtr128}}});921 922 addRulesForGOpcs({G_ABS}, Standard).Uni(S16, {{Sgpr32Trunc}, {Sgpr32SExt}});923 924 addRulesForGOpcs({G_FENCE}).Any({{{}}, {{}, {}}});925 926 addRulesForGOpcs({G_READSTEADYCOUNTER, G_READCYCLECOUNTER}, Standard)927 .Uni(S64, {{Sgpr64}, {}});928 929 addRulesForGOpcs({G_BLOCK_ADDR}).Any({{UniP0}, {{SgprP0}, {}}});930 931 addRulesForGOpcs({G_GLOBAL_VALUE})932 .Any({{UniP0}, {{SgprP0}, {}}})933 .Any({{UniP1}, {{SgprP1}, {}}})934 .Any({{UniP3}, {{SgprP3}, {}}})935 .Any({{UniP4}, {{SgprP4}, {}}})936 .Any({{UniP8}, {{SgprP8}, {}}});937 938 addRulesForGOpcs({G_AMDGPU_WAVE_ADDRESS}).Any({{UniP5}, {{SgprP5}, {}}});939 940 bool hasSALUFloat = ST->hasSALUFloatInsts();941 942 addRulesForGOpcs({G_FADD, G_FMUL}, Standard)943 .Uni(S16, {{UniInVgprS16}, {Vgpr16, Vgpr16}}, !hasSALUFloat)944 .Uni(S16, {{Sgpr16}, {Sgpr16, Sgpr16}}, hasSALUFloat)945 .Div(S16, {{Vgpr16}, {Vgpr16, Vgpr16}})946 .Uni(S32, {{Sgpr32}, {Sgpr32, Sgpr32}}, hasSALUFloat)947 .Uni(S32, {{UniInVgprS32}, {Vgpr32, Vgpr32}}, !hasSALUFloat)948 .Div(S32, {{Vgpr32}, {Vgpr32, Vgpr32}})949 .Uni(S64, {{UniInVgprS64}, {Vgpr64, Vgpr64}})950 .Div(S64, {{Vgpr64}, {Vgpr64, Vgpr64}})951 .Uni(V2S16, {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}}, !hasSALUFloat)952 .Uni(V2S16, {{SgprV2S16}, {SgprV2S16, SgprV2S16}, ScalarizeToS16},953 hasSALUFloat)954 .Div(V2S16, {{VgprV2S16}, {VgprV2S16, VgprV2S16}})955 .Any({{UniV2S32}, {{UniInVgprV2S32}, {VgprV2S32, VgprV2S32}}})956 .Any({{DivV2S32}, {{VgprV2S32}, {VgprV2S32, VgprV2S32}}});957 958 // FNEG and FABS are either folded as source modifiers or can be selected as959 // bitwise XOR and AND with Mask. XOR and AND are available on SALU but for960 // targets without SALU float we still select them as VGPR since there would961 // be no real sgpr use.962 addRulesForGOpcs({G_FNEG, G_FABS}, Standard)963 .Uni(S16, {{UniInVgprS16}, {Vgpr16}}, !hasSALUFloat)964 .Uni(S16, {{Sgpr16}, {Sgpr16}}, hasSALUFloat)965 .Div(S16, {{Vgpr16}, {Vgpr16}})966 .Uni(S32, {{UniInVgprS32}, {Vgpr32}}, !hasSALUFloat)967 .Uni(S32, {{Sgpr32}, {Sgpr32}}, hasSALUFloat)968 .Div(S32, {{Vgpr32}, {Vgpr32}})969 .Uni(S64, {{UniInVgprS64}, {Vgpr64}})970 .Div(S64, {{Vgpr64}, {Vgpr64}})971 .Uni(V2S16, {{UniInVgprV2S16}, {VgprV2S16}}, !hasSALUFloat)972 .Uni(V2S16, {{SgprV2S16}, {SgprV2S16}, ScalarizeToS16}, hasSALUFloat)973 .Div(V2S16, {{VgprV2S16}, {VgprV2S16}})974 .Any({{UniV2S32}, {{UniInVgprV2S32}, {VgprV2S32}}})975 .Any({{DivV2S32}, {{VgprV2S32}, {VgprV2S32}}});976 977 addRulesForGOpcs({G_FPTOUI})978 .Any({{UniS32, S32}, {{Sgpr32}, {Sgpr32}}}, hasSALUFloat)979 .Any({{UniS32, S32}, {{UniInVgprS32}, {Vgpr32}}}, !hasSALUFloat);980 981 addRulesForGOpcs({G_UITOFP})982 .Any({{DivS32, S32}, {{Vgpr32}, {Vgpr32}}})983 .Any({{UniS32, S32}, {{Sgpr32}, {Sgpr32}}}, hasSALUFloat)984 .Any({{UniS32, S32}, {{UniInVgprS32}, {Vgpr32}}}, !hasSALUFloat);985 986 addRulesForGOpcs({G_IS_FPCLASS})987 .Any({{DivS1, S16}, {{Vcc}, {Vgpr16}}})988 .Any({{UniS1, S16}, {{UniInVcc}, {Vgpr16}}})989 .Any({{DivS1, S32}, {{Vcc}, {Vgpr32}}})990 .Any({{UniS1, S32}, {{UniInVcc}, {Vgpr32}}})991 .Any({{DivS1, S64}, {{Vcc}, {Vgpr64}}})992 .Any({{UniS1, S64}, {{UniInVcc}, {Vgpr64}}});993 994 using namespace Intrinsic;995 996 addRulesForIOpcs({amdgcn_s_getpc}).Any({{UniS64, _}, {{Sgpr64}, {None}}});997 998 // This is "intrinsic lane mask" it was set to i32/i64 in llvm-ir.999 addRulesForIOpcs({amdgcn_end_cf}).Any({{_, S32}, {{}, {None, Sgpr32}}});1000 1001 addRulesForIOpcs({amdgcn_if_break}, Standard)1002 .Uni(S32, {{Sgpr32}, {IntrId, Vcc, Sgpr32}});1003 1004 addRulesForIOpcs({amdgcn_mbcnt_lo, amdgcn_mbcnt_hi}, Standard)1005 .Div(S32, {{}, {Vgpr32, None, Vgpr32, Vgpr32}});1006 1007 addRulesForIOpcs({amdgcn_readfirstlane})1008 .Any({{UniS32, _, DivS32}, {{}, {Sgpr32, None, Vgpr32}}})1009 // this should not exist in the first place, it is from call lowering1010 // readfirstlaning just in case register is not in sgpr.1011 .Any({{UniS32, _, UniS32}, {{}, {Sgpr32, None, Vgpr32}}});1012 1013} // end initialize rules1014