516 lines · cpp
1//===-- AMDGPURegBankLegalize.cpp -----------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9/// Lower G_ instructions that can't be inst-selected with register bank10/// assignment from AMDGPURegBankSelect based on machine uniformity info.11/// Given types on all operands, some register bank assignments require lowering12/// while others do not.13/// Note: cases where all register bank assignments would require lowering are14/// lowered in legalizer.15/// For example vgpr S64 G_AND requires lowering to S32 while sgpr S64 does not.16/// Eliminate sgpr S1 by lowering to sgpr S32.17//18//===----------------------------------------------------------------------===//19 20#include "AMDGPU.h"21#include "AMDGPUGlobalISelUtils.h"22#include "AMDGPURegBankLegalizeHelper.h"23#include "GCNSubtarget.h"24#include "llvm/CodeGen/GlobalISel/CSEInfo.h"25#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"26#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"27#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"28#include "llvm/CodeGen/GlobalISel/Utils.h"29#include "llvm/CodeGen/MachineFunctionPass.h"30#include "llvm/CodeGen/MachineUniformityAnalysis.h"31#include "llvm/CodeGen/TargetPassConfig.h"32#include "llvm/InitializePasses.h"33 34#define DEBUG_TYPE "amdgpu-regbanklegalize"35 36using namespace llvm;37using namespace AMDGPU;38using namespace llvm::MIPatternMatch;39 40namespace {41 42// AMDGPU-specific pattern matchers43template <typename SrcTy>44inline UnaryOp_match<SrcTy, AMDGPU::G_AMDGPU_READANYLANE>45m_GAMDGPUReadAnyLane(const SrcTy &Src) {46 return UnaryOp_match<SrcTy, AMDGPU::G_AMDGPU_READANYLANE>(Src);47}48 49class AMDGPURegBankLegalize : public MachineFunctionPass {50public:51 static char ID;52 53public:54 AMDGPURegBankLegalize() : MachineFunctionPass(ID) {}55 56 bool runOnMachineFunction(MachineFunction &MF) override;57 58 StringRef getPassName() const override {59 return "AMDGPU Register Bank Legalize";60 }61 62 void getAnalysisUsage(AnalysisUsage &AU) const override {63 AU.addRequired<TargetPassConfig>();64 AU.addRequired<GISelCSEAnalysisWrapperPass>();65 AU.addRequired<MachineUniformityAnalysisPass>();66 MachineFunctionPass::getAnalysisUsage(AU);67 }68 69 // If there were no phis and we do waterfall expansion machine verifier would70 // fail.71 MachineFunctionProperties getClearedProperties() const override {72 return MachineFunctionProperties().setNoPHIs();73 }74};75 76} // End anonymous namespace.77 78INITIALIZE_PASS_BEGIN(AMDGPURegBankLegalize, DEBUG_TYPE,79 "AMDGPU Register Bank Legalize", false, false)80INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)81INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass)82INITIALIZE_PASS_DEPENDENCY(MachineUniformityAnalysisPass)83INITIALIZE_PASS_END(AMDGPURegBankLegalize, DEBUG_TYPE,84 "AMDGPU Register Bank Legalize", false, false)85 86char AMDGPURegBankLegalize::ID = 0;87 88char &llvm::AMDGPURegBankLegalizeID = AMDGPURegBankLegalize::ID;89 90FunctionPass *llvm::createAMDGPURegBankLegalizePass() {91 return new AMDGPURegBankLegalize();92}93 94const RegBankLegalizeRules &getRules(const GCNSubtarget &ST,95 MachineRegisterInfo &MRI) {96 static std::mutex GlobalMutex;97 static SmallDenseMap<unsigned, std::unique_ptr<RegBankLegalizeRules>>98 CacheForRuleSet;99 std::lock_guard<std::mutex> Lock(GlobalMutex);100 auto [It, Inserted] = CacheForRuleSet.try_emplace(ST.getGeneration());101 if (Inserted)102 It->second = std::make_unique<RegBankLegalizeRules>(ST, MRI);103 else104 It->second->refreshRefs(ST, MRI);105 return *It->second;106}107 108class AMDGPURegBankLegalizeCombiner {109 MachineIRBuilder &B;110 MachineRegisterInfo &MRI;111 const SIRegisterInfo &TRI;112 const RegisterBank *SgprRB;113 const RegisterBank *VgprRB;114 const RegisterBank *VccRB;115 116 static constexpr LLT S1 = LLT::scalar(1);117 static constexpr LLT S16 = LLT::scalar(16);118 static constexpr LLT S32 = LLT::scalar(32);119 static constexpr LLT S64 = LLT::scalar(64);120 121public:122 AMDGPURegBankLegalizeCombiner(MachineIRBuilder &B, const SIRegisterInfo &TRI,123 const RegisterBankInfo &RBI)124 : B(B), MRI(*B.getMRI()), TRI(TRI),125 SgprRB(&RBI.getRegBank(AMDGPU::SGPRRegBankID)),126 VgprRB(&RBI.getRegBank(AMDGPU::VGPRRegBankID)),127 VccRB(&RBI.getRegBank(AMDGPU::VCCRegBankID)) {};128 129 bool isLaneMask(Register Reg);130 std::pair<MachineInstr *, Register> tryMatch(Register Src, unsigned Opcode);131 std::pair<GUnmerge *, int> tryMatchRALFromUnmerge(Register Src);132 Register getReadAnyLaneSrc(Register Src);133 void replaceRegWithOrBuildCopy(Register Dst, Register Src);134 135 bool tryEliminateReadAnyLane(MachineInstr &Copy);136 void tryCombineCopy(MachineInstr &MI);137 void tryCombineS1AnyExt(MachineInstr &MI);138};139 140bool AMDGPURegBankLegalizeCombiner::isLaneMask(Register Reg) {141 const RegisterBank *RB = MRI.getRegBankOrNull(Reg);142 if (RB && RB->getID() == AMDGPU::VCCRegBankID)143 return true;144 145 const TargetRegisterClass *RC = MRI.getRegClassOrNull(Reg);146 return RC && TRI.isSGPRClass(RC) && MRI.getType(Reg) == LLT::scalar(1);147}148 149std::pair<MachineInstr *, Register>150AMDGPURegBankLegalizeCombiner::tryMatch(Register Src, unsigned Opcode) {151 MachineInstr *MatchMI = MRI.getVRegDef(Src);152 if (MatchMI->getOpcode() != Opcode)153 return {nullptr, Register()};154 return {MatchMI, MatchMI->getOperand(1).getReg()};155}156 157std::pair<GUnmerge *, int>158AMDGPURegBankLegalizeCombiner::tryMatchRALFromUnmerge(Register Src) {159 MachineInstr *ReadAnyLane = MRI.getVRegDef(Src);160 if (ReadAnyLane->getOpcode() != AMDGPU::G_AMDGPU_READANYLANE)161 return {nullptr, -1};162 163 Register RALSrc = ReadAnyLane->getOperand(1).getReg();164 if (auto *UnMerge = getOpcodeDef<GUnmerge>(RALSrc, MRI))165 return {UnMerge, UnMerge->findRegisterDefOperandIdx(RALSrc, nullptr)};166 167 return {nullptr, -1};168}169 170Register AMDGPURegBankLegalizeCombiner::getReadAnyLaneSrc(Register Src) {171 // Src = G_AMDGPU_READANYLANE RALSrc172 Register RALSrc;173 if (mi_match(Src, MRI, m_GAMDGPUReadAnyLane(m_Reg(RALSrc))))174 return RALSrc;175 176 // RALSrc = G_ANYEXT S16Src177 // TruncSrc = G_AMDGPU_READANYLANE RALSrc178 // Src = G_TRUNC TruncSrc179 if (mi_match(Src, MRI,180 m_GTrunc(m_GAMDGPUReadAnyLane(m_GAnyExt(m_Reg(RALSrc)))))) {181 return RALSrc;182 }183 184 // TruncSrc = G_AMDGPU_READANYLANE RALSrc185 // AextSrc = G_TRUNC TruncSrc186 // Src = G_ANYEXT AextSrc187 if (mi_match(Src, MRI,188 m_GAnyExt(m_GTrunc(m_GAMDGPUReadAnyLane(m_Reg(RALSrc)))))) {189 return RALSrc;190 }191 192 // LoVgpr, HiVgpr = G_UNMERGE_VALUES UnmergeSrc193 // LoSgpr = G_AMDGPU_READANYLANE LoVgpr194 // HiSgpr = G_AMDGPU_READANYLANE HiVgpr195 // Src G_MERGE_VALUES LoSgpr, HiSgpr196 auto *Merge = getOpcodeDef<GMergeLikeInstr>(Src, MRI);197 if (Merge) {198 unsigned NumElts = Merge->getNumSources();199 auto [Unmerge, Idx] = tryMatchRALFromUnmerge(Merge->getSourceReg(0));200 if (!Unmerge || Unmerge->getNumDefs() != NumElts || Idx != 0)201 return {};202 203 // Check if all elements are from same unmerge and there is no shuffling.204 for (unsigned i = 1; i < NumElts; ++i) {205 auto [UnmergeI, IdxI] = tryMatchRALFromUnmerge(Merge->getSourceReg(i));206 if (UnmergeI != Unmerge || (unsigned)IdxI != i)207 return {};208 }209 return Unmerge->getSourceReg();210 }211 212 // SrcRegIdx = G_AMDGPU_READANYLANE RALElSrc213 // SourceReg G_MERGE_VALUES ..., SrcRegIdx, ...214 // ..., Src, ... = G_UNMERGE_VALUES SourceReg215 auto *UnMerge = getOpcodeDef<GUnmerge>(Src, MRI);216 if (!UnMerge)217 return {};218 219 int Idx = UnMerge->findRegisterDefOperandIdx(Src, nullptr);220 Merge = getOpcodeDef<GMergeLikeInstr>(UnMerge->getSourceReg(), MRI);221 if (!Merge || UnMerge->getNumDefs() != Merge->getNumSources())222 return {};223 224 Register SrcRegIdx = Merge->getSourceReg(Idx);225 if (MRI.getType(Src) != MRI.getType(SrcRegIdx))226 return {};227 228 auto [RALEl, RALElSrc] = tryMatch(SrcRegIdx, AMDGPU::G_AMDGPU_READANYLANE);229 if (RALEl)230 return RALElSrc;231 232 return {};233}234 235void AMDGPURegBankLegalizeCombiner::replaceRegWithOrBuildCopy(Register Dst,236 Register Src) {237 if (Dst.isVirtual())238 MRI.replaceRegWith(Dst, Src);239 else240 B.buildCopy(Dst, Src);241}242 243bool AMDGPURegBankLegalizeCombiner::tryEliminateReadAnyLane(244 MachineInstr &Copy) {245 Register Dst = Copy.getOperand(0).getReg();246 Register Src = Copy.getOperand(1).getReg();247 248 // Skip non-vgpr Dst249 if (Dst.isVirtual() ? (MRI.getRegBankOrNull(Dst) != VgprRB)250 : !TRI.isVGPR(MRI, Dst))251 return false;252 253 // Skip physical source registers and source registers with register class254 if (!Src.isVirtual() || MRI.getRegClassOrNull(Src))255 return false;256 257 Register RALDst = Src;258 MachineInstr &SrcMI = *MRI.getVRegDef(Src);259 if (SrcMI.getOpcode() == AMDGPU::G_BITCAST)260 RALDst = SrcMI.getOperand(1).getReg();261 262 Register RALSrc = getReadAnyLaneSrc(RALDst);263 if (!RALSrc)264 return false;265 266 B.setInstr(Copy);267 if (SrcMI.getOpcode() != AMDGPU::G_BITCAST) {268 // Src = READANYLANE RALSrc Src = READANYLANE RALSrc269 // Dst = Copy Src $Dst = Copy Src270 // -> ->271 // Dst = RALSrc $Dst = Copy RALSrc272 replaceRegWithOrBuildCopy(Dst, RALSrc);273 } else {274 // RALDst = READANYLANE RALSrc RALDst = READANYLANE RALSrc275 // Src = G_BITCAST RALDst Src = G_BITCAST RALDst276 // Dst = Copy Src Dst = Copy Src277 // -> ->278 // NewVgpr = G_BITCAST RALDst NewVgpr = G_BITCAST RALDst279 // Dst = NewVgpr $Dst = Copy NewVgpr280 auto Bitcast = B.buildBitcast({VgprRB, MRI.getType(Src)}, RALSrc);281 replaceRegWithOrBuildCopy(Dst, Bitcast.getReg(0));282 }283 284 eraseInstr(Copy, MRI);285 return true;286}287 288void AMDGPURegBankLegalizeCombiner::tryCombineCopy(MachineInstr &MI) {289 if (tryEliminateReadAnyLane(MI))290 return;291 292 Register Dst = MI.getOperand(0).getReg();293 Register Src = MI.getOperand(1).getReg();294 // Skip copies of physical registers.295 if (!Dst.isVirtual() || !Src.isVirtual())296 return;297 298 // This is a cross bank copy, sgpr S1 to lane mask.299 //300 // %Src:sgpr(s1) = G_TRUNC %TruncS32Src:sgpr(s32)301 // %Dst:lane-mask(s1) = COPY %Src:sgpr(s1)302 // ->303 // %BoolSrc:sgpr(s32) = G_AND %TruncS32Src:sgpr(s32), 1304 // %Dst:lane-mask(s1) = G_AMDGPU_COPY_VCC_SCC %BoolSrc:sgpr(s32)305 if (isLaneMask(Dst) && MRI.getRegBankOrNull(Src) == SgprRB) {306 auto [Trunc, TruncS32Src] = tryMatch(Src, AMDGPU::G_TRUNC);307 assert(Trunc && MRI.getType(TruncS32Src) == S32 &&308 "sgpr S1 must be result of G_TRUNC of sgpr S32");309 310 B.setInstr(MI);311 // Ensure that truncated bits in BoolSrc are 0.312 auto One = B.buildConstant({SgprRB, S32}, 1);313 auto BoolSrc = B.buildAnd({SgprRB, S32}, TruncS32Src, One);314 B.buildInstr(AMDGPU::G_AMDGPU_COPY_VCC_SCC, {Dst}, {BoolSrc});315 eraseInstr(MI, MRI);316 }317}318 319void AMDGPURegBankLegalizeCombiner::tryCombineS1AnyExt(MachineInstr &MI) {320 // %Src:sgpr(S1) = G_TRUNC %TruncSrc321 // %Dst = G_ANYEXT %Src:sgpr(S1)322 // ->323 // %Dst = G_... %TruncSrc324 Register Dst = MI.getOperand(0).getReg();325 Register Src = MI.getOperand(1).getReg();326 if (MRI.getType(Src) != S1)327 return;328 329 auto [Trunc, TruncSrc] = tryMatch(Src, AMDGPU::G_TRUNC);330 if (!Trunc)331 return;332 333 LLT DstTy = MRI.getType(Dst);334 LLT TruncSrcTy = MRI.getType(TruncSrc);335 336 if (DstTy == TruncSrcTy) {337 MRI.replaceRegWith(Dst, TruncSrc);338 eraseInstr(MI, MRI);339 return;340 }341 342 B.setInstr(MI);343 344 if (DstTy == S32 && TruncSrcTy == S64) {345 auto Unmerge = B.buildUnmerge({SgprRB, S32}, TruncSrc);346 MRI.replaceRegWith(Dst, Unmerge.getReg(0));347 eraseInstr(MI, MRI);348 return;349 }350 351 if (DstTy == S64 && TruncSrcTy == S32) {352 B.buildMergeLikeInstr(MI.getOperand(0).getReg(),353 {TruncSrc, B.buildUndef({SgprRB, S32})});354 eraseInstr(MI, MRI);355 return;356 }357 358 if (DstTy == S32 && TruncSrcTy == S16) {359 B.buildAnyExt(Dst, TruncSrc);360 eraseInstr(MI, MRI);361 return;362 }363 364 if (DstTy == S16 && TruncSrcTy == S32) {365 B.buildTrunc(Dst, TruncSrc);366 eraseInstr(MI, MRI);367 return;368 }369 370 llvm_unreachable("missing anyext + trunc combine");371}372 373// Search through MRI for virtual registers with sgpr register bank and S1 LLT.374[[maybe_unused]] static Register getAnySgprS1(const MachineRegisterInfo &MRI) {375 const LLT S1 = LLT::scalar(1);376 for (unsigned i = 0; i < MRI.getNumVirtRegs(); ++i) {377 Register Reg = Register::index2VirtReg(i);378 if (MRI.def_empty(Reg) || MRI.getType(Reg) != S1)379 continue;380 381 const RegisterBank *RB = MRI.getRegBankOrNull(Reg);382 if (RB && RB->getID() == AMDGPU::SGPRRegBankID) {383 LLVM_DEBUG(dbgs() << "Warning: detected sgpr S1 register in: ";384 MRI.getVRegDef(Reg)->dump(););385 return Reg;386 }387 }388 389 return {};390}391 392bool AMDGPURegBankLegalize::runOnMachineFunction(MachineFunction &MF) {393 if (MF.getProperties().hasFailedISel())394 return false;395 396 // Setup the instruction builder with CSE.397 const TargetPassConfig &TPC = getAnalysis<TargetPassConfig>();398 GISelCSEAnalysisWrapper &Wrapper =399 getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();400 GISelCSEInfo &CSEInfo = Wrapper.get(TPC.getCSEConfig());401 GISelObserverWrapper Observer;402 Observer.addObserver(&CSEInfo);403 404 CSEMIRBuilder B(MF);405 B.setCSEInfo(&CSEInfo);406 B.setChangeObserver(Observer);407 408 RAIIDelegateInstaller DelegateInstaller(MF, &Observer);409 RAIIMFObserverInstaller MFObserverInstaller(MF, Observer);410 411 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();412 MachineRegisterInfo &MRI = MF.getRegInfo();413 const RegisterBankInfo &RBI = *ST.getRegBankInfo();414 const MachineUniformityInfo &MUI =415 getAnalysis<MachineUniformityAnalysisPass>().getUniformityInfo();416 417 // RegBankLegalizeRules is initialized with assigning sets of IDs to opcodes.418 const RegBankLegalizeRules &RBLRules = getRules(ST, MRI);419 420 // Logic that does legalization based on IDs assigned to Opcode.421 RegBankLegalizeHelper RBLHelper(B, MUI, RBI, RBLRules);422 423 SmallVector<MachineInstr *> AllInst;424 425 for (MachineBasicBlock &MBB : MF) {426 for (MachineInstr &MI : MBB) {427 AllInst.push_back(&MI);428 }429 }430 431 for (MachineInstr *MI : AllInst) {432 if (!MI->isPreISelOpcode())433 continue;434 435 unsigned Opc = MI->getOpcode();436 // Insert point for use operands needs some calculation.437 if (Opc == AMDGPU::G_PHI) {438 RBLHelper.applyMappingPHI(*MI);439 continue;440 }441 442 // Opcodes that support pretty much all combinations of reg banks and LLTs443 // (except S1). There is no point in writing rules for them.444 if (Opc == AMDGPU::G_BUILD_VECTOR || Opc == AMDGPU::G_UNMERGE_VALUES ||445 Opc == AMDGPU::G_MERGE_VALUES || Opc == AMDGPU::G_BITCAST) {446 RBLHelper.applyMappingTrivial(*MI);447 continue;448 }449 450 // Opcodes that also support S1.451 if (Opc == G_FREEZE &&452 MRI.getType(MI->getOperand(0).getReg()) != LLT::scalar(1)) {453 RBLHelper.applyMappingTrivial(*MI);454 continue;455 }456 457 if ((Opc == AMDGPU::G_CONSTANT || Opc == AMDGPU::G_FCONSTANT ||458 Opc == AMDGPU::G_IMPLICIT_DEF)) {459 Register Dst = MI->getOperand(0).getReg();460 // Non S1 types are trivially accepted.461 if (MRI.getType(Dst) != LLT::scalar(1)) {462 assert(MRI.getRegBank(Dst)->getID() == AMDGPU::SGPRRegBankID);463 continue;464 }465 466 // S1 rules are in RegBankLegalizeRules.467 }468 469 RBLHelper.findRuleAndApplyMapping(*MI);470 }471 472 // Sgpr S1 clean up combines:473 // - Sgpr S1(S32) to sgpr S1(S32) Copy: anyext + trunc combine.474 // In RegBankLegalize 'S1 Dst' are legalized into S32 as475 // 'S1Dst = Trunc S32Dst' and 'S1 Src' into 'S32Src = Anyext S1Src'.476 // S1 Truncs and Anyexts that come from legalizer, that can have non-S32477 // types e.g. S16 = Anyext S1 or S1 = Trunc S64, will also be cleaned up.478 // - Sgpr S1(S32) to vcc Copy: G_AMDGPU_COPY_VCC_SCC combine.479 // Divergent instruction uses sgpr S1 as input that should be lane mask(vcc)480 // Legalizing this use creates sgpr S1(S32) to vcc Copy.481 482 // Note: Remaining S1 copies, S1s are either sgpr S1(S32) or vcc S1:483 // - Vcc to vcc Copy: nothing to do here, just a regular copy.484 // - Vcc to sgpr S1 Copy: Should not exist in a form of COPY instruction(*).485 // Note: For 'uniform-in-vcc to sgpr-S1 copy' G_AMDGPU_COPY_SCC_VCC is used486 // instead. When only available instruction creates vcc result, use of487 // UniformInVcc results in creating G_AMDGPU_COPY_SCC_VCC.488 489 // (*)Explanation for 'sgpr S1(uniform) = COPY vcc(divergent)':490 // Copy from divergent to uniform register indicates an error in either:491 // - Uniformity analysis: Uniform instruction has divergent input. If one of492 // the inputs is divergent, instruction should be divergent!493 // - RegBankLegalizer not executing in waterfall loop (missing implementation)494 495 AMDGPURegBankLegalizeCombiner Combiner(B, *ST.getRegisterInfo(), RBI);496 497 for (MachineBasicBlock &MBB : MF) {498 for (MachineInstr &MI : make_early_inc_range(MBB)) {499 if (MI.getOpcode() == AMDGPU::COPY) {500 Combiner.tryCombineCopy(MI);501 continue;502 }503 if (MI.getOpcode() == AMDGPU::G_ANYEXT) {504 Combiner.tryCombineS1AnyExt(MI);505 continue;506 }507 }508 }509 510 assert(!getAnySgprS1(MRI).isValid() &&511 "Registers with sgpr reg bank and S1 LLT are not legal after "512 "AMDGPURegBankLegalize. Should lower to sgpr S32");513 514 return true;515}516