473 lines · c
1//===- AMDGPUInstructionSelector --------------------------------*- C++ -*-==//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8/// \file9/// This file declares the targeting of the InstructionSelector class for10/// AMDGPU.11//===----------------------------------------------------------------------===//12 13#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUINSTRUCTIONSELECTOR_H14#define LLVM_LIB_TARGET_AMDGPU_AMDGPUINSTRUCTIONSELECTOR_H15 16#include "SIDefines.h"17#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"18#include "llvm/IR/InstrTypes.h"19 20namespace {21#define GET_GLOBALISEL_PREDICATE_BITSET22#define AMDGPUSubtarget GCNSubtarget23#include "AMDGPUGenGlobalISel.inc"24#undef GET_GLOBALISEL_PREDICATE_BITSET25#undef AMDGPUSubtarget26}27 28namespace llvm {29 30namespace AMDGPU {31struct ImageDimIntrinsicInfo;32}33 34class AMDGPURegisterBankInfo;35class AMDGPUTargetMachine;36class BlockFrequencyInfo;37class ProfileSummaryInfo;38class GCNSubtarget;39class MachineInstr;40class MachineIRBuilder;41class MachineOperand;42class MachineRegisterInfo;43class RegisterBank;44class SIInstrInfo;45class SIRegisterInfo;46class TargetRegisterClass;47 48class AMDGPUInstructionSelector final : public InstructionSelector {49private:50 MachineRegisterInfo *MRI;51 const GCNSubtarget *Subtarget;52 53public:54 AMDGPUInstructionSelector(const GCNSubtarget &STI,55 const AMDGPURegisterBankInfo &RBI,56 const AMDGPUTargetMachine &TM);57 58 bool select(MachineInstr &I) override;59 static const char *getName();60 61 void setupMF(MachineFunction &MF, GISelValueTracking *VT,62 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,63 BlockFrequencyInfo *BFI) override;64 65private:66 struct GEPInfo {67 SmallVector<unsigned, 2> SgprParts;68 SmallVector<unsigned, 2> VgprParts;69 int64_t Imm = 0;70 };71 72 bool isSGPR(Register Reg) const;73 74 bool isInstrUniform(const MachineInstr &MI) const;75 bool isVCC(Register Reg, const MachineRegisterInfo &MRI) const;76 77 const RegisterBank *getArtifactRegBank(78 Register Reg, const MachineRegisterInfo &MRI,79 const TargetRegisterInfo &TRI) const;80 81 /// tblgen-erated 'select' implementation.82 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;83 84 MachineOperand getSubOperand64(MachineOperand &MO,85 const TargetRegisterClass &SubRC,86 unsigned SubIdx) const;87 88 bool constrainCopyLikeIntrin(MachineInstr &MI, unsigned NewOpc) const;89 bool selectCOPY(MachineInstr &I) const;90 bool selectCOPY_SCC_VCC(MachineInstr &I) const;91 bool selectCOPY_VCC_SCC(MachineInstr &I) const;92 bool selectReadAnyLane(MachineInstr &I) const;93 bool selectPHI(MachineInstr &I) const;94 bool selectG_TRUNC(MachineInstr &I) const;95 bool selectG_SZA_EXT(MachineInstr &I) const;96 bool selectG_FPEXT(MachineInstr &I) const;97 bool selectG_FNEG(MachineInstr &I) const;98 bool selectG_FABS(MachineInstr &I) const;99 bool selectG_AND_OR_XOR(MachineInstr &I) const;100 bool selectG_ADD_SUB(MachineInstr &I) const;101 bool selectG_UADDO_USUBO_UADDE_USUBE(MachineInstr &I) const;102 bool selectG_AMDGPU_MAD_64_32(MachineInstr &I) const;103 bool selectG_EXTRACT(MachineInstr &I) const;104 bool selectG_FMA_FMAD(MachineInstr &I) const;105 bool selectG_MERGE_VALUES(MachineInstr &I) const;106 bool selectG_UNMERGE_VALUES(MachineInstr &I) const;107 bool selectG_BUILD_VECTOR(MachineInstr &I) const;108 bool selectG_IMPLICIT_DEF(MachineInstr &I) const;109 bool selectG_INSERT(MachineInstr &I) const;110 bool selectG_SBFX_UBFX(MachineInstr &I) const;111 112 bool selectInterpP1F16(MachineInstr &MI) const;113 bool selectWritelane(MachineInstr &MI) const;114 bool selectDivScale(MachineInstr &MI) const;115 bool selectIntrinsicCmp(MachineInstr &MI) const;116 bool selectBallot(MachineInstr &I) const;117 bool selectRelocConstant(MachineInstr &I) const;118 bool selectGroupStaticSize(MachineInstr &I) const;119 bool selectReturnAddress(MachineInstr &I) const;120 bool selectG_INTRINSIC(MachineInstr &I) const;121 122 bool selectEndCfIntrinsic(MachineInstr &MI) const;123 bool selectDSOrderedIntrinsic(MachineInstr &MI, Intrinsic::ID IID) const;124 bool selectDSGWSIntrinsic(MachineInstr &MI, Intrinsic::ID IID) const;125 bool selectDSAppendConsume(MachineInstr &MI, bool IsAppend) const;126 bool selectInitWholeWave(MachineInstr &MI) const;127 bool selectDSBvhStackIntrinsic(MachineInstr &MI) const;128 129 bool selectImageIntrinsic(MachineInstr &MI,130 const AMDGPU::ImageDimIntrinsicInfo *Intr) const;131 bool selectG_INTRINSIC_W_SIDE_EFFECTS(MachineInstr &I) const;132 int getS_CMPOpcode(CmpInst::Predicate P, unsigned Size) const;133 bool selectG_ICMP_or_FCMP(MachineInstr &I) const;134 bool hasVgprParts(ArrayRef<GEPInfo> AddrInfo) const;135 void getAddrModeInfo(const MachineInstr &Load, const MachineRegisterInfo &MRI,136 SmallVectorImpl<GEPInfo> &AddrInfo) const;137 138 void initM0(MachineInstr &I) const;139 bool selectG_LOAD_STORE_ATOMICRMW(MachineInstr &I) const;140 bool selectG_SELECT(MachineInstr &I) const;141 bool selectG_BRCOND(MachineInstr &I) const;142 bool selectG_GLOBAL_VALUE(MachineInstr &I) const;143 bool selectG_PTRMASK(MachineInstr &I) const;144 bool selectG_EXTRACT_VECTOR_ELT(MachineInstr &I) const;145 bool selectG_INSERT_VECTOR_ELT(MachineInstr &I) const;146 bool selectBufferLoadLds(MachineInstr &MI) const;147 bool selectGlobalLoadLds(MachineInstr &MI) const;148 bool selectBVHIntersectRayIntrinsic(MachineInstr &I) const;149 bool selectSMFMACIntrin(MachineInstr &I) const;150 bool selectPermlaneSwapIntrin(MachineInstr &I, Intrinsic::ID IntrID) const;151 bool selectWaveAddress(MachineInstr &I) const;152 bool selectBITOP3(MachineInstr &I) const;153 bool selectStackRestore(MachineInstr &MI) const;154 bool selectNamedBarrierInit(MachineInstr &I, Intrinsic::ID IID) const;155 bool selectNamedBarrierInst(MachineInstr &I, Intrinsic::ID IID) const;156 bool selectSBarrierSignalIsfirst(MachineInstr &I, Intrinsic::ID IID) const;157 bool selectSGetBarrierState(MachineInstr &I, Intrinsic::ID IID) const;158 bool selectSBarrierLeave(MachineInstr &I) const;159 160 std::pair<Register, unsigned> selectVOP3ModsImpl(Register Src,161 bool IsCanonicalizing = true,162 bool AllowAbs = true,163 bool OpSel = false) const;164 165 Register copyToVGPRIfSrcFolded(Register Src, unsigned Mods,166 MachineOperand Root, MachineInstr *InsertPt,167 bool ForceVGPR = false) const;168 169 InstructionSelector::ComplexRendererFns170 selectVCSRC(MachineOperand &Root) const;171 172 InstructionSelector::ComplexRendererFns173 selectVSRC0(MachineOperand &Root) const;174 175 InstructionSelector::ComplexRendererFns176 selectVOP3Mods0(MachineOperand &Root) const;177 InstructionSelector::ComplexRendererFns178 selectVOP3BMods0(MachineOperand &Root) const;179 InstructionSelector::ComplexRendererFns180 selectVOP3OMods(MachineOperand &Root) const;181 InstructionSelector::ComplexRendererFns182 selectVOP3Mods(MachineOperand &Root) const;183 InstructionSelector::ComplexRendererFns184 selectVOP3ModsNonCanonicalizing(MachineOperand &Root) const;185 InstructionSelector::ComplexRendererFns186 selectVOP3BMods(MachineOperand &Root) const;187 188 ComplexRendererFns selectVOP3NoMods(MachineOperand &Root) const;189 190 std::pair<Register, unsigned>191 selectVOP3PModsImpl(Register RootReg, const MachineRegisterInfo &MRI,192 bool IsDOT = false) const;193 InstructionSelector::ComplexRendererFns194 selectVOP3PRetHelper(MachineOperand &Root, bool IsDOT = false) const;195 196 InstructionSelector::ComplexRendererFns197 selectVOP3PMods(MachineOperand &Root) const;198 199 InstructionSelector::ComplexRendererFns200 selectVOP3PModsDOT(MachineOperand &Root) const;201 202 InstructionSelector::ComplexRendererFns203 selectWMMAOpSelVOP3PMods(MachineOperand &Root) const;204 205 InstructionSelector::ComplexRendererFns206 selectWMMAModsF32NegAbs(MachineOperand &Root) const;207 InstructionSelector::ComplexRendererFns208 selectWMMAModsF16Neg(MachineOperand &Root) const;209 InstructionSelector::ComplexRendererFns210 selectWMMAModsF16NegAbs(MachineOperand &Root) const;211 InstructionSelector::ComplexRendererFns212 selectWMMAVISrc(MachineOperand &Root) const;213 InstructionSelector::ComplexRendererFns214 selectSWMMACIndex8(MachineOperand &Root) const;215 InstructionSelector::ComplexRendererFns216 selectSWMMACIndex16(MachineOperand &Root) const;217 InstructionSelector::ComplexRendererFns218 selectSWMMACIndex32(MachineOperand &Root) const;219 220 InstructionSelector::ComplexRendererFns221 selectVOP3OpSelMods(MachineOperand &Root) const;222 223 InstructionSelector::ComplexRendererFns224 selectVINTERPMods(MachineOperand &Root) const;225 InstructionSelector::ComplexRendererFns226 selectVINTERPModsHi(MachineOperand &Root) const;227 228 bool selectScaleOffset(MachineOperand &Root, Register &Offset,229 bool IsSigned) const;230 bool selectSmrdOffset(MachineOperand &Root, Register &Base, Register *SOffset,231 int64_t *Offset, bool *ScaleOffset) const;232 InstructionSelector::ComplexRendererFns233 selectSmrdImm(MachineOperand &Root) const;234 InstructionSelector::ComplexRendererFns235 selectSmrdImm32(MachineOperand &Root) const;236 InstructionSelector::ComplexRendererFns237 selectSmrdSgpr(MachineOperand &Root) const;238 InstructionSelector::ComplexRendererFns239 selectSmrdSgprImm(MachineOperand &Root) const;240 241 std::pair<Register, int> selectFlatOffsetImpl(MachineOperand &Root,242 uint64_t FlatVariant) const;243 244 InstructionSelector::ComplexRendererFns245 selectFlatOffset(MachineOperand &Root) const;246 InstructionSelector::ComplexRendererFns247 selectGlobalOffset(MachineOperand &Root) const;248 InstructionSelector::ComplexRendererFns249 selectScratchOffset(MachineOperand &Root) const;250 251 InstructionSelector::ComplexRendererFns252 selectGlobalSAddr(MachineOperand &Root, unsigned CPolBits,253 bool NeedIOffset = true) const;254 InstructionSelector::ComplexRendererFns255 selectGlobalSAddr(MachineOperand &Root) const;256 InstructionSelector::ComplexRendererFns257 selectGlobalSAddrCPol(MachineOperand &Root) const;258 InstructionSelector::ComplexRendererFns259 selectGlobalSAddrCPolM0(MachineOperand &Root) const;260 InstructionSelector::ComplexRendererFns261 selectGlobalSAddrGLC(MachineOperand &Root) const;262 InstructionSelector::ComplexRendererFns263 selectGlobalSAddrNoIOffset(MachineOperand &Root) const;264 InstructionSelector::ComplexRendererFns265 selectGlobalSAddrNoIOffsetM0(MachineOperand &Root) const;266 267 InstructionSelector::ComplexRendererFns268 selectScratchSAddr(MachineOperand &Root) const;269 bool checkFlatScratchSVSSwizzleBug(Register VAddr, Register SAddr,270 uint64_t ImmOffset) const;271 InstructionSelector::ComplexRendererFns272 selectScratchSVAddr(MachineOperand &Root) const;273 274 InstructionSelector::ComplexRendererFns275 selectMUBUFScratchOffen(MachineOperand &Root) const;276 InstructionSelector::ComplexRendererFns277 selectMUBUFScratchOffset(MachineOperand &Root) const;278 279 bool isDSOffsetLegal(Register Base, int64_t Offset) const;280 bool isDSOffset2Legal(Register Base, int64_t Offset0, int64_t Offset1,281 unsigned Size) const;282 bool isFlatScratchBaseLegal(Register Addr) const;283 bool isFlatScratchBaseLegalSV(Register Addr) const;284 bool isFlatScratchBaseLegalSVImm(Register Addr) const;285 286 std::pair<Register, unsigned>287 selectDS1Addr1OffsetImpl(MachineOperand &Root) const;288 InstructionSelector::ComplexRendererFns289 selectDS1Addr1Offset(MachineOperand &Root) const;290 291 InstructionSelector::ComplexRendererFns292 selectDS64Bit4ByteAligned(MachineOperand &Root) const;293 294 InstructionSelector::ComplexRendererFns295 selectDS128Bit8ByteAligned(MachineOperand &Root) const;296 297 std::pair<Register, unsigned> selectDSReadWrite2Impl(MachineOperand &Root,298 unsigned size) const;299 InstructionSelector::ComplexRendererFns300 selectDSReadWrite2(MachineOperand &Root, unsigned size) const;301 302 std::tuple<Register, int64_t, bool>303 getPtrBaseWithConstantOffset(Register Root,304 const MachineRegisterInfo &MRI) const;305 306 // Parse out a chain of up to two g_ptr_add instructions.307 // g_ptr_add (n0, _)308 // g_ptr_add (n0, (n1 = g_ptr_add n2, n3))309 struct MUBUFAddressData {310 Register N0, N2, N3;311 int64_t Offset = 0;312 };313 314 bool shouldUseAddr64(MUBUFAddressData AddrData) const;315 316 void splitIllegalMUBUFOffset(MachineIRBuilder &B,317 Register &SOffset, int64_t &ImmOffset) const;318 319 MUBUFAddressData parseMUBUFAddress(Register Src) const;320 321 bool selectMUBUFAddr64Impl(MachineOperand &Root, Register &VAddr,322 Register &RSrcReg, Register &SOffset,323 int64_t &Offset) const;324 325 bool selectMUBUFOffsetImpl(MachineOperand &Root, Register &RSrcReg,326 Register &SOffset, int64_t &Offset) const;327 328 InstructionSelector::ComplexRendererFns329 selectBUFSOffset(MachineOperand &Root) const;330 331 InstructionSelector::ComplexRendererFns332 selectMUBUFAddr64(MachineOperand &Root) const;333 334 InstructionSelector::ComplexRendererFns335 selectMUBUFOffset(MachineOperand &Root) const;336 337 ComplexRendererFns selectSMRDBufferImm(MachineOperand &Root) const;338 ComplexRendererFns selectSMRDBufferImm32(MachineOperand &Root) const;339 ComplexRendererFns selectSMRDBufferSgprImm(MachineOperand &Root) const;340 341 std::pair<Register, unsigned> selectVOP3PMadMixModsImpl(MachineOperand &Root,342 bool &Matched) const;343 ComplexRendererFns selectVOP3PMadMixModsExt(MachineOperand &Root) const;344 ComplexRendererFns selectVOP3PMadMixMods(MachineOperand &Root) const;345 346 void renderTruncImm32(MachineInstrBuilder &MIB, const MachineInstr &MI,347 int OpIdx = -1) const;348 349 void renderTruncTImm(MachineInstrBuilder &MIB, const MachineInstr &MI,350 int OpIdx) const;351 void renderZextBoolTImm(MachineInstrBuilder &MIB, const MachineInstr &MI,352 int OpIdx) const;353 354 void renderOpSelTImm(MachineInstrBuilder &MIB, const MachineInstr &MI,355 int OpIdx) const;356 357 void renderSrcAndDstSelToOpSelXForm_0_0(MachineInstrBuilder &MIB,358 const MachineInstr &MI,359 int OpIdx) const;360 361 void renderSrcAndDstSelToOpSelXForm_0_1(MachineInstrBuilder &MIB,362 const MachineInstr &MI,363 int OpIdx) const;364 365 void renderSrcAndDstSelToOpSelXForm_1_0(MachineInstrBuilder &MIB,366 const MachineInstr &MI,367 int OpIdx) const;368 369 void renderSrcAndDstSelToOpSelXForm_1_1(MachineInstrBuilder &MIB,370 const MachineInstr &MI,371 int OpIdx) const;372 373 void renderDstSelToOpSelXForm(MachineInstrBuilder &MIB,374 const MachineInstr &MI, int OpIdx) const;375 376 void renderSrcSelToOpSelXForm(MachineInstrBuilder &MIB,377 const MachineInstr &MI, int OpIdx) const;378 379 void renderSrcAndDstSelToOpSelXForm_2_0(MachineInstrBuilder &MIB,380 const MachineInstr &MI,381 int OpIdx) const;382 383 void renderDstSelToOpSel3XFormXForm(MachineInstrBuilder &MIB,384 const MachineInstr &MI, int OpIdx) const;385 386 void renderNegateImm(MachineInstrBuilder &MIB, const MachineInstr &MI,387 int OpIdx) const;388 389 void renderBitcastFPImm(MachineInstrBuilder &MIB, const MachineInstr &MI,390 int OpIdx) const;391 392 void renderBitcastFPImm32(MachineInstrBuilder &MIB, const MachineInstr &MI,393 int OpIdx) const {394 renderBitcastFPImm(MIB, MI, OpIdx);395 }396 void renderBitcastFPImm64(MachineInstrBuilder &MIB, const MachineInstr &MI,397 int OpIdx) const {398 renderBitcastFPImm(MIB, MI, OpIdx);399 }400 401 void renderPopcntImm(MachineInstrBuilder &MIB, const MachineInstr &MI,402 int OpIdx) const;403 void renderExtractCPol(MachineInstrBuilder &MIB, const MachineInstr &MI,404 int OpIdx) const;405 void renderExtractSWZ(MachineInstrBuilder &MIB, const MachineInstr &MI,406 int OpIdx) const;407 void renderExtractCpolSetGLC(MachineInstrBuilder &MIB, const MachineInstr &MI,408 int OpIdx) const;409 410 void renderFrameIndex(MachineInstrBuilder &MIB, const MachineInstr &MI,411 int OpIdx) const;412 413 void renderFPPow2ToExponent(MachineInstrBuilder &MIB, const MachineInstr &MI,414 int OpIdx) const;415 416 void renderRoundMode(MachineInstrBuilder &MIB, const MachineInstr &MI,417 int OpIdx) const;418 419 void renderVOP3PModsNeg(MachineInstrBuilder &MIB, const MachineInstr &MI,420 int OpIdx) const;421 void renderVOP3PModsNegs(MachineInstrBuilder &MIB, const MachineInstr &MI,422 int OpIdx) const;423 void renderVOP3PModsNegAbs(MachineInstrBuilder &MIB, const MachineInstr &MI,424 int OpIdx) const;425 426 void renderPrefetchLoc(MachineInstrBuilder &MIB, const MachineInstr &MI,427 int OpIdx) const;428 429 void renderScaledMAIIntrinsicOperand(MachineInstrBuilder &MIB,430 const MachineInstr &MI, int OpIdx) const;431 432 bool isInlineImmediate(const APInt &Imm) const;433 bool isInlineImmediate(const APFloat &Imm) const;434 435 // Returns true if TargetOpcode::G_AND MachineInstr `MI`'s masking of the436 // shift amount operand's `ShAmtBits` bits is unneeded.437 bool isUnneededShiftMask(const MachineInstr &MI, unsigned ShAmtBits) const;438 439 /// Match a zero extend from a 32-bit value to 64-bits.440 Register matchZeroExtendFromS32(Register Reg) const;441 /// Match a sign extend from a 32-bit value to 64-bits.442 Register matchSignExtendFromS32(Register Reg) const;443 /// Match a zero extend from a 32-bit value to 64-bits, or \p Reg itself if it444 /// is 32-bit.445 Register matchZeroExtendFromS32OrS32(Register Reg) const;446 /// Match a sign extend from a 32-bit value to 64-bits, or \p Reg itself if it447 /// is 32-bit.448 Register matchSignExtendFromS32OrS32(Register Reg) const;449 /// Match either sign or zero extend depending on the \p IsSigned from a450 /// 32-bit value to 64-bits, or \p Reg itself if it is 32-bit.451 Register matchExtendFromS32OrS32(Register Reg, bool IsSigned) const;452 /// Match an any extend from a 32-bit value to 64-bit.453 Register matchAnyExtendFromS32(Register Reg) const;454 455 const SIInstrInfo &TII;456 const SIRegisterInfo &TRI;457 const AMDGPURegisterBankInfo &RBI;458 const AMDGPUTargetMachine &TM;459 const GCNSubtarget &STI;460#define GET_GLOBALISEL_PREDICATES_DECL461#define AMDGPUSubtarget GCNSubtarget462#include "AMDGPUGenGlobalISel.inc"463#undef GET_GLOBALISEL_PREDICATES_DECL464#undef AMDGPUSubtarget465 466#define GET_GLOBALISEL_TEMPORARIES_DECL467#include "AMDGPUGenGlobalISel.inc"468#undef GET_GLOBALISEL_TEMPORARIES_DECL469};470 471} // End llvm namespace.472#endif473