395 lines · c
1//===- AMDGPURegBankLegalizeRules --------------------------------*- C++ -*-==//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUREGBANKLEGALIZERULES_H10#define LLVM_LIB_TARGET_AMDGPU_AMDGPUREGBANKLEGALIZERULES_H11 12#include "llvm/ADT/DenseMap.h"13#include "llvm/ADT/SmallVector.h"14#include <functional>15 16namespace llvm {17 18class LLT;19class MachineRegisterInfo;20class MachineInstr;21class GCNSubtarget;22class MachineFunction;23template <typename T> class GenericUniformityInfo;24template <typename T> class GenericSSAContext;25using MachineSSAContext = GenericSSAContext<MachineFunction>;26using MachineUniformityInfo = GenericUniformityInfo<MachineSSAContext>;27 28namespace AMDGPU {29 30/// \returns true if \p Ty is a pointer type with size \p Width.31bool isAnyPtr(LLT Ty, unsigned Width);32 33// IDs used to build predicate for RegBankLegalizeRule. Predicate can have one34// or more IDs and each represents a check for 'uniform or divergent' + LLT or35// just LLT on register operand.36// Most often checking one operand is enough to decide which RegBankLLTMapping37// to apply (see Fast Rules), IDs are useful when two or more operands need to38// be checked.39enum UniformityLLTOpPredicateID {40 _,41 // scalars42 S1,43 S16,44 S32,45 S64,46 S128,47 48 UniS1,49 UniS16,50 UniS32,51 UniS64,52 UniS128,53 54 DivS1,55 DivS16,56 DivS32,57 DivS64,58 DivS128,59 60 // pointers61 P0,62 P1,63 P3,64 P4,65 P5,66 P8,67 Ptr32,68 Ptr64,69 Ptr128,70 71 UniP0,72 UniP1,73 UniP3,74 UniP4,75 UniP5,76 UniP8,77 UniPtr32,78 UniPtr64,79 UniPtr128,80 81 DivP0,82 DivP1,83 DivP3,84 DivP4,85 DivP5,86 DivPtr32,87 DivPtr64,88 DivPtr128,89 90 // vectors91 V2S16,92 V2S32,93 V3S32,94 V4S32,95 96 UniV2S16,97 UniV2S32,98 99 DivV2S16,100 DivV2S32,101 102 // B types103 B32,104 B64,105 B96,106 B128,107 B256,108 B512,109 110 UniB32,111 UniB64,112 UniB96,113 UniB128,114 UniB256,115 UniB512,116 117 DivB32,118 DivB64,119 DivB96,120 DivB128,121 DivB256,122 DivB512,123};124 125// How to apply register bank on register operand.126// In most cases, this serves as a LLT and register bank assert.127// Can change operands and insert copies, extends, truncs, and read-any-lanes.128// Anything more complicated requires LoweringMethod.129enum RegBankLLTMappingApplyID {130 InvalidMapping,131 None,132 IntrId,133 Imm,134 Vcc,135 136 // sgpr scalars, pointers, vectors and B-types137 Sgpr16,138 Sgpr32,139 Sgpr64,140 Sgpr128,141 SgprP0,142 SgprP1,143 SgprP3,144 SgprP4,145 SgprP5,146 SgprP8,147 SgprPtr32,148 SgprPtr64,149 SgprPtr128,150 SgprV2S16,151 SgprV4S32,152 SgprV2S32,153 SgprB32,154 SgprB64,155 SgprB96,156 SgprB128,157 SgprB256,158 SgprB512,159 160 // vgpr scalars, pointers, vectors and B-types161 Vgpr16,162 Vgpr32,163 Vgpr64,164 Vgpr128,165 VgprP0,166 VgprP1,167 VgprP3,168 VgprP4,169 VgprP5,170 VgprPtr32,171 VgprPtr64,172 VgprPtr128,173 VgprV2S16,174 VgprV2S32,175 VgprB32,176 VgprB64,177 VgprB96,178 VgprB128,179 VgprB256,180 VgprB512,181 VgprV4S32,182 183 // Dst only modifiers: read-any-lane and truncs184 UniInVcc,185 UniInVgprS16,186 UniInVgprS32,187 UniInVgprS64,188 UniInVgprV2S16,189 UniInVgprV2S32,190 UniInVgprV4S32,191 UniInVgprB32,192 UniInVgprB64,193 UniInVgprB96,194 UniInVgprB128,195 UniInVgprB256,196 UniInVgprB512,197 198 Sgpr32Trunc,199 200 // Src only modifiers: execute in waterfall loop if divergent201 Sgpr32_WF,202 SgprV4S32_WF,203 204 // Src only modifiers: extends205 Sgpr32AExt,206 Sgpr32AExtBoolInReg,207 Sgpr32SExt,208 Sgpr32ZExt,209 Vgpr32SExt,210 Vgpr32ZExt,211};212 213// Instruction needs to be replaced with sequence of instructions. Lowering was214// not done by legalizer since instructions is available in either sgpr or vgpr.215// For example S64 AND is available on sgpr, for that reason S64 AND is legal in216// context of Legalizer that only checks LLT. But S64 AND is not available on217// vgpr. Lower it to two S32 vgpr ANDs.218enum LoweringMethodID {219 DoNotLower,220 VccExtToSel,221 UniExtToSel,222 UnpackBitShift,223 UnpackMinMax,224 S_BFE,225 V_BFE,226 VgprToVccCopy,227 SplitTo32,228 ScalarizeToS16,229 SplitTo32Select,230 SplitTo32SExtInReg,231 Ext32To64,232 UniCstExt,233 SplitLoad,234 WidenLoad,235 WidenMMOToS32,236 UnpackAExt237};238 239enum FastRulesTypes {240 NoFastRules,241 Standard, // S16, S32, S64, V2S16242 StandardB, // B32, B64, B96, B128243 Vector, // S32, V2S32, V3S32, V4S32244};245 246struct RegBankLLTMapping {247 SmallVector<RegBankLLTMappingApplyID, 2> DstOpMapping;248 SmallVector<RegBankLLTMappingApplyID, 4> SrcOpMapping;249 LoweringMethodID LoweringMethod;250 RegBankLLTMapping(251 std::initializer_list<RegBankLLTMappingApplyID> DstOpMappingList,252 std::initializer_list<RegBankLLTMappingApplyID> SrcOpMappingList,253 LoweringMethodID LoweringMethod = DoNotLower);254};255 256struct PredicateMapping {257 SmallVector<UniformityLLTOpPredicateID, 4> OpUniformityAndTypes;258 std::function<bool(const MachineInstr &)> TestFunc;259 PredicateMapping(260 std::initializer_list<UniformityLLTOpPredicateID> OpList,261 std::function<bool(const MachineInstr &)> TestFunc = nullptr);262 263 bool match(const MachineInstr &MI, const MachineUniformityInfo &MUI,264 const MachineRegisterInfo &MRI) const;265};266 267struct RegBankLegalizeRule {268 PredicateMapping Predicate;269 RegBankLLTMapping OperandMapping;270};271 272class SetOfRulesForOpcode {273 // "Slow Rules". More complex 'Rules[i].Predicate', check them one by one.274 SmallVector<RegBankLegalizeRule, 4> Rules;275 276 // "Fast Rules"277 // Instead of testing each 'Rules[i].Predicate' we do direct access to278 // RegBankLLTMapping using getFastPredicateSlot. For example if:279 // - FastTypes == Standard Uni[0] holds Mapping in case Op 0 is uniform S32280 // - FastTypes == Vector Div[3] holds Mapping in case Op 0 is divergent V4S32281 FastRulesTypes FastTypes = NoFastRules;282#define InvMapping RegBankLLTMapping({InvalidMapping}, {InvalidMapping})283 RegBankLLTMapping Uni[4] = {InvMapping, InvMapping, InvMapping, InvMapping};284 RegBankLLTMapping Div[4] = {InvMapping, InvMapping, InvMapping, InvMapping};285 286public:287 SetOfRulesForOpcode();288 SetOfRulesForOpcode(FastRulesTypes FastTypes);289 290 const RegBankLLTMapping &291 findMappingForMI(const MachineInstr &MI, const MachineRegisterInfo &MRI,292 const MachineUniformityInfo &MUI) const;293 294 void addRule(RegBankLegalizeRule Rule);295 296 void addFastRuleDivergent(UniformityLLTOpPredicateID Ty,297 RegBankLLTMapping RuleApplyIDs);298 void addFastRuleUniform(UniformityLLTOpPredicateID Ty,299 RegBankLLTMapping RuleApplyIDs);300 301private:302 int getFastPredicateSlot(UniformityLLTOpPredicateID Ty) const;303};304 305// Essentially 'map<Opcode(or intrinsic_opcode), SetOfRulesForOpcode>' but a306// little more efficient.307class RegBankLegalizeRules {308 const GCNSubtarget *ST;309 MachineRegisterInfo *MRI;310 // Separate maps for G-opcodes and instrinsics since they are in different311 // enums. Multiple opcodes can share same set of rules.312 // RulesAlias = map<Opcode, KeyOpcode>313 // Rules = map<KeyOpcode, SetOfRulesForOpcode>314 SmallDenseMap<unsigned, unsigned, 256> GRulesAlias;315 SmallDenseMap<unsigned, SetOfRulesForOpcode, 128> GRules;316 SmallDenseMap<unsigned, unsigned, 128> IRulesAlias;317 SmallDenseMap<unsigned, SetOfRulesForOpcode, 64> IRules;318 class RuleSetInitializer {319 SetOfRulesForOpcode *RuleSet;320 321 public:322 // Used for clang-format line breaks and to force writing all rules for323 // opcode in same place.324 template <class AliasMap, class RulesMap>325 RuleSetInitializer(std::initializer_list<unsigned> OpcList,326 AliasMap &RulesAlias, RulesMap &Rules,327 FastRulesTypes FastTypes = NoFastRules) {328 unsigned KeyOpcode = *OpcList.begin();329 for (unsigned Opc : OpcList) {330 [[maybe_unused]] auto [_, NewInput] =331 RulesAlias.try_emplace(Opc, KeyOpcode);332 assert(NewInput && "Can't redefine existing Rules");333 }334 335 auto [DenseMapIter, NewInput] = Rules.try_emplace(KeyOpcode, FastTypes);336 assert(NewInput && "Can't redefine existing Rules");337 338 RuleSet = &DenseMapIter->second;339 }340 341 RuleSetInitializer(const RuleSetInitializer &) = delete;342 RuleSetInitializer &operator=(const RuleSetInitializer &) = delete;343 RuleSetInitializer(RuleSetInitializer &&) = delete;344 RuleSetInitializer &operator=(RuleSetInitializer &&) = delete;345 ~RuleSetInitializer() = default;346 347 RuleSetInitializer &Div(UniformityLLTOpPredicateID Ty,348 RegBankLLTMapping RuleApplyIDs,349 bool STPred = true) {350 if (STPred)351 RuleSet->addFastRuleDivergent(Ty, RuleApplyIDs);352 return *this;353 }354 355 RuleSetInitializer &Uni(UniformityLLTOpPredicateID Ty,356 RegBankLLTMapping RuleApplyIDs,357 bool STPred = true) {358 if (STPred)359 RuleSet->addFastRuleUniform(Ty, RuleApplyIDs);360 return *this;361 }362 363 RuleSetInitializer &Any(RegBankLegalizeRule Init, bool STPred = true) {364 if (STPred)365 RuleSet->addRule(Init);366 return *this;367 }368 };369 370 RuleSetInitializer addRulesForGOpcs(std::initializer_list<unsigned> OpcList,371 FastRulesTypes FastTypes = NoFastRules);372 373 RuleSetInitializer addRulesForIOpcs(std::initializer_list<unsigned> OpcList,374 FastRulesTypes FastTypes = NoFastRules);375 376public:377 // Initialize rules for all opcodes.378 RegBankLegalizeRules(const GCNSubtarget &ST, MachineRegisterInfo &MRI);379 380 // In case we don't want to regenerate same rules, we can use already381 // generated rules but need to refresh references to objects that are382 // created for this run.383 void refreshRefs(const GCNSubtarget &_ST, MachineRegisterInfo &_MRI) {384 ST = &_ST;385 MRI = &_MRI;386 };387 388 const SetOfRulesForOpcode &getRulesForOpc(MachineInstr &MI) const;389};390 391} // end namespace AMDGPU392} // end namespace llvm393 394#endif395