1994 lines · c
1//=====-- GCNSubtarget.h - Define GCN Subtarget for AMDGPU ------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//==-----------------------------------------------------------------------===//8//9/// \file10/// AMD GCN specific subclass of TargetSubtarget.11//12//===----------------------------------------------------------------------===//13 14#ifndef LLVM_LIB_TARGET_AMDGPU_GCNSUBTARGET_H15#define LLVM_LIB_TARGET_AMDGPU_GCNSUBTARGET_H16 17#include "AMDGPUCallLowering.h"18#include "AMDGPURegisterBankInfo.h"19#include "AMDGPUSubtarget.h"20#include "SIFrameLowering.h"21#include "SIISelLowering.h"22#include "SIInstrInfo.h"23#include "Utils/AMDGPUBaseInfo.h"24#include "llvm/Support/ErrorHandling.h"25 26#define GET_SUBTARGETINFO_HEADER27#include "AMDGPUGenSubtargetInfo.inc"28 29namespace llvm {30 31class GCNTargetMachine;32 33class GCNSubtarget final : public AMDGPUGenSubtargetInfo,34 public AMDGPUSubtarget {35public:36 using AMDGPUSubtarget::getMaxWavesPerEU;37 38 // Following 2 enums are documented at:39 // - https://llvm.org/docs/AMDGPUUsage.html#trap-handler-abi40 enum class TrapHandlerAbi {41 NONE = 0x00,42 AMDHSA = 0x01,43 };44 45 enum class TrapID {46 LLVMAMDHSATrap = 0x02,47 LLVMAMDHSADebugTrap = 0x03,48 };49 50private:51 /// SelectionDAGISel related APIs.52 std::unique_ptr<const SelectionDAGTargetInfo> TSInfo;53 54 /// GlobalISel related APIs.55 std::unique_ptr<AMDGPUCallLowering> CallLoweringInfo;56 std::unique_ptr<InlineAsmLowering> InlineAsmLoweringInfo;57 std::unique_ptr<InstructionSelector> InstSelector;58 std::unique_ptr<LegalizerInfo> Legalizer;59 std::unique_ptr<AMDGPURegisterBankInfo> RegBankInfo;60 61protected:62 // Basic subtarget description.63 AMDGPU::IsaInfo::AMDGPUTargetID TargetID;64 unsigned Gen = INVALID;65 InstrItineraryData InstrItins;66 int LDSBankCount = 0;67 unsigned MaxPrivateElementSize = 0;68 69 // Possibly statically set by tablegen, but may want to be overridden.70 bool FastDenormalF32 = false;71 bool HalfRate64Ops = false;72 bool FullRate64Ops = false;73 74 // Dynamically set bits that enable features.75 bool FlatForGlobal = false;76 bool AutoWaitcntBeforeBarrier = false;77 bool BackOffBarrier = false;78 bool UnalignedScratchAccess = false;79 bool UnalignedAccessMode = false;80 bool RelaxedBufferOOBMode = false;81 bool HasApertureRegs = false;82 bool SupportsXNACK = false;83 bool KernargPreload = false;84 85 // This should not be used directly. 'TargetID' tracks the dynamic settings86 // for XNACK.87 bool EnableXNACK = false;88 89 bool EnableTgSplit = false;90 bool EnableCuMode = false;91 bool TrapHandler = false;92 bool EnablePreciseMemory = false;93 94 // Used as options.95 bool EnableLoadStoreOpt = false;96 bool EnableUnsafeDSOffsetFolding = false;97 bool EnableSIScheduler = false;98 bool EnableDS128 = false;99 bool EnablePRTStrictNull = false;100 bool DumpCode = false;101 bool AssemblerPermissiveWavesize = false;102 103 // Subtarget statically properties set by tablegen104 bool FP64 = false;105 bool FMA = false;106 bool MIMG_R128 = false;107 bool CIInsts = false;108 bool GFX8Insts = false;109 bool GFX9Insts = false;110 bool GFX90AInsts = false;111 bool GFX940Insts = false;112 bool GFX950Insts = false;113 bool GFX10Insts = false;114 bool GFX11Insts = false;115 bool GFX12Insts = false;116 bool GFX1250Insts = false;117 bool GFX10_3Insts = false;118 bool GFX7GFX8GFX9Insts = false;119 bool SGPRInitBug = false;120 bool UserSGPRInit16Bug = false;121 bool NegativeScratchOffsetBug = false;122 bool NegativeUnalignedScratchOffsetBug = false;123 bool HasSMemRealTime = false;124 bool HasIntClamp = false;125 bool HasFmaMixInsts = false;126 bool HasFmaMixBF16Insts = false;127 bool HasMovrel = false;128 bool HasVGPRIndexMode = false;129 bool HasScalarDwordx3Loads = false;130 bool HasScalarStores = false;131 bool HasScalarAtomics = false;132 bool HasSDWAOmod = false;133 bool HasSDWAScalar = false;134 bool HasSDWASdst = false;135 bool HasSDWAMac = false;136 bool HasSDWAOutModsVOPC = false;137 bool HasDPP = false;138 bool HasDPP8 = false;139 bool HasDPALU_DPP = false;140 bool HasDPPSrc1SGPR = false;141 bool HasPackedFP32Ops = false;142 bool HasImageInsts = false;143 bool HasExtendedImageInsts = false;144 bool HasR128A16 = false;145 bool HasA16 = false;146 bool HasG16 = false;147 bool HasNSAEncoding = false;148 bool HasPartialNSAEncoding = false;149 bool GFX10_AEncoding = false;150 bool GFX10_BEncoding = false;151 bool HasDLInsts = false;152 bool HasFmacF64Inst = false;153 bool HasDot1Insts = false;154 bool HasDot2Insts = false;155 bool HasDot3Insts = false;156 bool HasDot4Insts = false;157 bool HasDot5Insts = false;158 bool HasDot6Insts = false;159 bool HasDot7Insts = false;160 bool HasDot8Insts = false;161 bool HasDot9Insts = false;162 bool HasDot10Insts = false;163 bool HasDot11Insts = false;164 bool HasDot12Insts = false;165 bool HasDot13Insts = false;166 bool HasMAIInsts = false;167 bool HasFP8Insts = false;168 bool HasFP8ConversionInsts = false;169 bool HasCubeInsts = false;170 bool HasLerpInst = false;171 bool HasSadInsts = false;172 bool HasQsadInsts = false;173 bool HasCvtNormInsts = false;174 bool HasCvtPkNormVOP2Insts = false;175 bool HasCvtPkNormVOP3Insts = false;176 bool HasFP8E5M3Insts = false;177 bool HasCvtFP8Vop1Bug = false;178 bool HasPkFmacF16Inst = false;179 bool HasAtomicFMinFMaxF32GlobalInsts = false;180 bool HasAtomicFMinFMaxF64GlobalInsts = false;181 bool HasAtomicFMinFMaxF32FlatInsts = false;182 bool HasAtomicFMinFMaxF64FlatInsts = false;183 bool HasAtomicDsPkAdd16Insts = false;184 bool HasAtomicFlatPkAdd16Insts = false;185 bool HasAtomicFaddRtnInsts = false;186 bool HasAtomicFaddNoRtnInsts = false;187 bool HasMemoryAtomicFaddF32DenormalSupport = false;188 bool HasAtomicBufferGlobalPkAddF16NoRtnInsts = false;189 bool HasAtomicBufferGlobalPkAddF16Insts = false;190 bool HasAtomicCSubNoRtnInsts = false;191 bool HasAtomicGlobalPkAddBF16Inst = false;192 bool HasAtomicBufferPkAddBF16Inst = false;193 bool HasFlatAtomicFaddF32Inst = false;194 bool HasFlatBufferGlobalAtomicFaddF64Inst = false;195 bool HasDefaultComponentZero = false;196 bool HasAgentScopeFineGrainedRemoteMemoryAtomics = false;197 bool HasEmulatedSystemScopeAtomics = false;198 bool HasDefaultComponentBroadcast = false;199 bool HasXF32Insts = false;200 /// The maximum number of instructions that may be placed within an S_CLAUSE,201 /// which is one greater than the maximum argument to S_CLAUSE. A value of 0202 /// indicates a lack of S_CLAUSE support.203 unsigned MaxHardClauseLength = 0;204 bool SupportsSRAMECC = false;205 bool DynamicVGPR = false;206 bool DynamicVGPRBlockSize32 = false;207 bool HasVMemToLDSLoad = false;208 bool RequiresAlignVGPR = false;209 210 // This should not be used directly. 'TargetID' tracks the dynamic settings211 // for SRAMECC.212 bool EnableSRAMECC = false;213 214 bool HasNoSdstCMPX = false;215 bool HasVscnt = false;216 bool HasWaitXcnt = false;217 bool HasGetWaveIdInst = false;218 bool HasSMemTimeInst = false;219 bool HasShaderCyclesRegister = false;220 bool HasShaderCyclesHiLoRegisters = false;221 bool HasVOP3Literal = false;222 bool HasNoDataDepHazard = false;223 bool FlatAddressSpace = false;224 bool FlatInstOffsets = false;225 bool FlatGlobalInsts = false;226 bool FlatScratchInsts = false;227 bool FlatGVSMode = false;228 bool ScalarFlatScratchInsts = false;229 bool HasArchitectedFlatScratch = false;230 bool EnableFlatScratch = false;231 bool HasArchitectedSGPRs = false;232 bool HasGDS = false;233 bool HasGWS = false;234 bool AddNoCarryInsts = false;235 bool HasUnpackedD16VMem = false;236 bool LDSMisalignedBug = false;237 bool HasMFMAInlineLiteralBug = false;238 bool UnalignedBufferAccess = false;239 bool UnalignedDSAccess = false;240 bool HasPackedTID = false;241 bool ScalarizeGlobal = false;242 bool HasSALUFloatInsts = false;243 bool HasPseudoScalarTrans = false;244 bool HasRestrictedSOffset = false;245 bool Has64BitLiterals = false;246 bool Has1024AddressableVGPRs = false;247 bool HasBitOp3Insts = false;248 bool HasTanhInsts = false;249 bool HasTensorCvtLutInsts = false;250 bool HasTransposeLoadF4F6Insts = false;251 bool HasPrngInst = false;252 bool HasBVHDualAndBVH8Insts = false;253 bool HasPermlane16Swap = false;254 bool HasPermlane32Swap = false;255 bool HasVcmpxPermlaneHazard = false;256 bool HasVMEMtoScalarWriteHazard = false;257 bool HasSMEMtoVectorWriteHazard = false;258 bool HasInstFwdPrefetchBug = false;259 bool HasVmemPrefInsts = false;260 bool HasSafeSmemPrefetch = false;261 bool HasSafeCUPrefetch = false;262 bool HasVcmpxExecWARHazard = false;263 bool HasLdsBranchVmemWARHazard = false;264 bool HasNSAtoVMEMBug = false;265 bool HasNSAClauseBug = false;266 bool HasOffset3fBug = false;267 bool HasFlatSegmentOffsetBug = false;268 bool HasImageStoreD16Bug = false;269 bool HasImageGather4D16Bug = false;270 bool HasMSAALoadDstSelBug = false;271 bool HasPrivEnabledTrap2NopBug = false;272 bool Has1_5xVGPRs = false;273 bool HasMADIntraFwdBug = false;274 bool HasVOPDInsts = false;275 bool HasVALUTransUseHazard = false;276 bool HasRequiredExportPriority = false;277 bool HasVmemWriteVgprInOrder = false;278 bool HasAshrPkInsts = false;279 bool HasIEEEMinimumMaximumInsts = false;280 bool HasMinimum3Maximum3F32 = false;281 bool HasMinimum3Maximum3F16 = false;282 bool HasMin3Max3PKF16 = false;283 bool HasMinimum3Maximum3PKF16 = false;284 bool HasLshlAddU64Inst = false;285 bool HasAddSubU64Insts = false;286 bool HasMadU32Inst = false;287 bool HasAddMinMaxInsts = false;288 bool HasPkAddMinMaxInsts = false;289 bool HasPointSampleAccel = false;290 bool HasLdsBarrierArriveAtomic = false;291 bool HasSetPrioIncWgInst = false;292 293 bool RequiresCOV6 = false;294 bool UseBlockVGPROpsForCSR = false;295 bool HasGloballyAddressableScratch = false;296 297 bool Has45BitNumRecordsBufferResource = false;298 299 bool HasClusters = false;300 bool RequiresWaitsBeforeSystemScopeStores = false;301 302 // Dummy feature to use for assembler in tablegen.303 bool FeatureDisable = false;304 305private:306 SIInstrInfo InstrInfo;307 SITargetLowering TLInfo;308 SIFrameLowering FrameLowering;309 310public:311 GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,312 const GCNTargetMachine &TM);313 ~GCNSubtarget() override;314 315 GCNSubtarget &initializeSubtargetDependencies(const Triple &TT,316 StringRef GPU, StringRef FS);317 318 /// Diagnose inconsistent subtarget features before attempting to codegen319 /// function \p F.320 void checkSubtargetFeatures(const Function &F) const;321 322 const SIInstrInfo *getInstrInfo() const override {323 return &InstrInfo;324 }325 326 const SIFrameLowering *getFrameLowering() const override {327 return &FrameLowering;328 }329 330 const SITargetLowering *getTargetLowering() const override {331 return &TLInfo;332 }333 334 const SIRegisterInfo *getRegisterInfo() const override {335 return &InstrInfo.getRegisterInfo();336 }337 338 const SelectionDAGTargetInfo *getSelectionDAGInfo() const override;339 340 const CallLowering *getCallLowering() const override {341 return CallLoweringInfo.get();342 }343 344 const InlineAsmLowering *getInlineAsmLowering() const override {345 return InlineAsmLoweringInfo.get();346 }347 348 InstructionSelector *getInstructionSelector() const override {349 return InstSelector.get();350 }351 352 const LegalizerInfo *getLegalizerInfo() const override {353 return Legalizer.get();354 }355 356 const AMDGPURegisterBankInfo *getRegBankInfo() const override {357 return RegBankInfo.get();358 }359 360 const AMDGPU::IsaInfo::AMDGPUTargetID &getTargetID() const {361 return TargetID;362 }363 364 const InstrItineraryData *getInstrItineraryData() const override {365 return &InstrItins;366 }367 368 void ParseSubtargetFeatures(StringRef CPU, StringRef TuneCPU, StringRef FS);369 370 Generation getGeneration() const {371 return (Generation)Gen;372 }373 374 unsigned getMaxWaveScratchSize() const {375 // See COMPUTE_TMPRING_SIZE.WAVESIZE.376 if (getGeneration() >= GFX12) {377 // 18-bit field in units of 64-dword.378 return (64 * 4) * ((1 << 18) - 1);379 }380 if (getGeneration() == GFX11) {381 // 15-bit field in units of 64-dword.382 return (64 * 4) * ((1 << 15) - 1);383 }384 // 13-bit field in units of 256-dword.385 return (256 * 4) * ((1 << 13) - 1);386 }387 388 /// Return the number of high bits known to be zero for a frame index.389 unsigned getKnownHighZeroBitsForFrameIndex() const {390 return llvm::countl_zero(getMaxWaveScratchSize()) + getWavefrontSizeLog2();391 }392 393 int getLDSBankCount() const {394 return LDSBankCount;395 }396 397 unsigned getMaxPrivateElementSize(bool ForBufferRSrc = false) const {398 return (ForBufferRSrc || !enableFlatScratch()) ? MaxPrivateElementSize : 16;399 }400 401 unsigned getConstantBusLimit(unsigned Opcode) const;402 403 /// Returns if the result of this instruction with a 16-bit result returned in404 /// a 32-bit register implicitly zeroes the high 16-bits, rather than preserve405 /// the original value.406 bool zeroesHigh16BitsOfDest(unsigned Opcode) const;407 408 bool supportsWGP() const {409 if (GFX1250Insts)410 return false;411 return getGeneration() >= GFX10;412 }413 414 bool hasIntClamp() const {415 return HasIntClamp;416 }417 418 bool hasFP64() const {419 return FP64;420 }421 422 bool hasMIMG_R128() const {423 return MIMG_R128;424 }425 426 bool hasHWFP64() const {427 return FP64;428 }429 430 bool hasHalfRate64Ops() const {431 return HalfRate64Ops;432 }433 434 bool hasFullRate64Ops() const {435 return FullRate64Ops;436 }437 438 bool hasAddr64() const {439 return (getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS);440 }441 442 bool hasFlat() const {443 return (getGeneration() > AMDGPUSubtarget::SOUTHERN_ISLANDS);444 }445 446 // Return true if the target only has the reverse operand versions of VALU447 // shift instructions (e.g. v_lshrrev_b32, and no v_lshr_b32).448 bool hasOnlyRevVALUShifts() const {449 return getGeneration() >= VOLCANIC_ISLANDS;450 }451 452 bool hasFractBug() const {453 return getGeneration() == SOUTHERN_ISLANDS;454 }455 456 bool hasBFE() const {457 return true;458 }459 460 bool hasBFI() const {461 return true;462 }463 464 bool hasBFM() const {465 return hasBFE();466 }467 468 bool hasBCNT(unsigned Size) const {469 return true;470 }471 472 bool hasFFBL() const {473 return true;474 }475 476 bool hasFFBH() const {477 return true;478 }479 480 bool hasMed3_16() const {481 return getGeneration() >= AMDGPUSubtarget::GFX9;482 }483 484 bool hasMin3Max3_16() const {485 return getGeneration() >= AMDGPUSubtarget::GFX9;486 }487 488 bool hasFmaMixInsts() const {489 return HasFmaMixInsts;490 }491 492 bool hasFmaMixBF16Insts() const { return HasFmaMixBF16Insts; }493 494 bool hasCARRY() const {495 return true;496 }497 498 bool hasFMA() const {499 return FMA;500 }501 502 bool hasSwap() const {503 return GFX9Insts;504 }505 506 bool hasScalarPackInsts() const {507 return GFX9Insts;508 }509 510 bool hasScalarMulHiInsts() const {511 return GFX9Insts;512 }513 514 bool hasScalarSubwordLoads() const { return getGeneration() >= GFX12; }515 516 TrapHandlerAbi getTrapHandlerAbi() const {517 return isAmdHsaOS() ? TrapHandlerAbi::AMDHSA : TrapHandlerAbi::NONE;518 }519 520 bool supportsGetDoorbellID() const {521 // The S_GETREG DOORBELL_ID is supported by all GFX9 onward targets.522 return getGeneration() >= GFX9;523 }524 525 /// True if the offset field of DS instructions works as expected. On SI, the526 /// offset uses a 16-bit adder and does not always wrap properly.527 bool hasUsableDSOffset() const {528 return getGeneration() >= SEA_ISLANDS;529 }530 531 bool unsafeDSOffsetFoldingEnabled() const {532 return EnableUnsafeDSOffsetFolding;533 }534 535 /// Condition output from div_scale is usable.536 bool hasUsableDivScaleConditionOutput() const {537 return getGeneration() != SOUTHERN_ISLANDS;538 }539 540 /// Extra wait hazard is needed in some cases before541 /// s_cbranch_vccnz/s_cbranch_vccz.542 bool hasReadVCCZBug() const {543 return getGeneration() <= SEA_ISLANDS;544 }545 546 /// Writes to VCC_LO/VCC_HI update the VCCZ flag.547 bool partialVCCWritesUpdateVCCZ() const {548 return getGeneration() >= GFX10;549 }550 551 /// A read of an SGPR by SMRD instruction requires 4 wait states when the SGPR552 /// was written by a VALU instruction.553 bool hasSMRDReadVALUDefHazard() const {554 return getGeneration() == SOUTHERN_ISLANDS;555 }556 557 /// A read of an SGPR by a VMEM instruction requires 5 wait states when the558 /// SGPR was written by a VALU Instruction.559 bool hasVMEMReadSGPRVALUDefHazard() const {560 return getGeneration() >= VOLCANIC_ISLANDS;561 }562 563 bool hasRFEHazards() const {564 return getGeneration() >= VOLCANIC_ISLANDS;565 }566 567 /// Number of hazard wait states for s_setreg_b32/s_setreg_imm32_b32.568 unsigned getSetRegWaitStates() const {569 return getGeneration() <= SEA_ISLANDS ? 1 : 2;570 }571 572 bool dumpCode() const {573 return DumpCode;574 }575 576 /// Return the amount of LDS that can be used that will not restrict the577 /// occupancy lower than WaveCount.578 unsigned getMaxLocalMemSizeWithWaveCount(unsigned WaveCount,579 const Function &) const;580 581 bool supportsMinMaxDenormModes() const {582 return getGeneration() >= AMDGPUSubtarget::GFX9;583 }584 585 /// \returns If target supports S_DENORM_MODE.586 bool hasDenormModeInst() const {587 return getGeneration() >= AMDGPUSubtarget::GFX10;588 }589 590 bool useFlatForGlobal() const {591 return FlatForGlobal;592 }593 594 /// \returns If target supports ds_read/write_b128 and user enables generation595 /// of ds_read/write_b128.596 bool useDS128() const {597 return CIInsts && EnableDS128;598 }599 600 /// \return If target supports ds_read/write_b96/128.601 bool hasDS96AndDS128() const {602 return CIInsts;603 }604 605 /// Have v_trunc_f64, v_ceil_f64, v_rndne_f64606 bool haveRoundOpsF64() const {607 return CIInsts;608 }609 610 /// \returns If MUBUF instructions always perform range checking, even for611 /// buffer resources used for private memory access.612 bool privateMemoryResourceIsRangeChecked() const {613 return getGeneration() < AMDGPUSubtarget::GFX9;614 }615 616 /// \returns If target requires PRT Struct NULL support (zero result registers617 /// for sparse texture support).618 bool usePRTStrictNull() const {619 return EnablePRTStrictNull;620 }621 622 bool hasAutoWaitcntBeforeBarrier() const {623 return AutoWaitcntBeforeBarrier;624 }625 626 /// \returns true if the target supports backing off of s_barrier instructions627 /// when an exception is raised.628 bool supportsBackOffBarrier() const {629 return BackOffBarrier;630 }631 632 bool hasUnalignedBufferAccess() const {633 return UnalignedBufferAccess;634 }635 636 bool hasUnalignedBufferAccessEnabled() const {637 return UnalignedBufferAccess && UnalignedAccessMode;638 }639 640 bool hasUnalignedDSAccess() const {641 return UnalignedDSAccess;642 }643 644 bool hasUnalignedDSAccessEnabled() const {645 return UnalignedDSAccess && UnalignedAccessMode;646 }647 648 bool hasUnalignedScratchAccess() const {649 return UnalignedScratchAccess;650 }651 652 bool hasUnalignedScratchAccessEnabled() const {653 return UnalignedScratchAccess && UnalignedAccessMode;654 }655 656 bool hasUnalignedAccessMode() const {657 return UnalignedAccessMode;658 }659 660 bool hasRelaxedBufferOOBMode() const { return RelaxedBufferOOBMode; }661 662 bool hasApertureRegs() const {663 return HasApertureRegs;664 }665 666 bool isTrapHandlerEnabled() const {667 return TrapHandler;668 }669 670 bool isXNACKEnabled() const {671 return TargetID.isXnackOnOrAny();672 }673 674 bool isTgSplitEnabled() const {675 return EnableTgSplit;676 }677 678 bool isCuModeEnabled() const {679 return EnableCuMode;680 }681 682 bool isPreciseMemoryEnabled() const { return EnablePreciseMemory; }683 684 bool hasFlatAddressSpace() const {685 return FlatAddressSpace;686 }687 688 bool hasFlatScrRegister() const {689 return hasFlatAddressSpace();690 }691 692 bool hasFlatInstOffsets() const {693 return FlatInstOffsets;694 }695 696 bool hasFlatGlobalInsts() const {697 return FlatGlobalInsts;698 }699 700 bool hasFlatScratchInsts() const {701 return FlatScratchInsts;702 }703 704 // Check if target supports ST addressing mode with FLAT scratch instructions.705 // The ST addressing mode means no registers are used, either VGPR or SGPR,706 // but only immediate offset is swizzled and added to the FLAT scratch base.707 bool hasFlatScratchSTMode() const {708 return hasFlatScratchInsts() && (hasGFX10_3Insts() || hasGFX940Insts());709 }710 711 bool hasFlatScratchSVSMode() const { return GFX940Insts || GFX11Insts; }712 713 bool hasScalarFlatScratchInsts() const {714 return ScalarFlatScratchInsts;715 }716 717 bool enableFlatScratch() const {718 return flatScratchIsArchitected() ||719 (EnableFlatScratch && hasFlatScratchInsts());720 }721 722 bool hasGlobalAddTidInsts() const {723 return GFX10_BEncoding;724 }725 726 bool hasAtomicCSub() const {727 return GFX10_BEncoding;728 }729 730 bool hasMTBUFInsts() const { return !hasGFX1250Insts(); }731 732 bool hasFormattedMUBUFInsts() const { return !hasGFX1250Insts(); }733 734 bool hasExportInsts() const {735 return !hasGFX940Insts() && !hasGFX1250Insts();736 }737 738 bool hasVINTERPEncoding() const { return GFX11Insts && !hasGFX1250Insts(); }739 740 // DS_ADD_F64/DS_ADD_RTN_F64741 bool hasLdsAtomicAddF64() const {742 return hasGFX90AInsts() || hasGFX1250Insts();743 }744 745 bool hasMultiDwordFlatScratchAddressing() const {746 return getGeneration() >= GFX9;747 }748 749 bool hasFlatSegmentOffsetBug() const {750 return HasFlatSegmentOffsetBug;751 }752 753 bool hasFlatLgkmVMemCountInOrder() const {754 return getGeneration() > GFX9;755 }756 757 bool hasD16LoadStore() const {758 return getGeneration() >= GFX9;759 }760 761 bool d16PreservesUnusedBits() const {762 return hasD16LoadStore() && !TargetID.isSramEccOnOrAny();763 }764 765 bool hasD16Images() const {766 return getGeneration() >= VOLCANIC_ISLANDS;767 }768 769 /// Return if most LDS instructions have an m0 use that require m0 to be770 /// initialized.771 bool ldsRequiresM0Init() const {772 return getGeneration() < GFX9;773 }774 775 // True if the hardware rewinds and replays GWS operations if a wave is776 // preempted.777 //778 // If this is false, a GWS operation requires testing if a nack set the779 // MEM_VIOL bit, and repeating if so.780 bool hasGWSAutoReplay() const {781 return getGeneration() >= GFX9;782 }783 784 /// \returns if target has ds_gws_sema_release_all instruction.785 bool hasGWSSemaReleaseAll() const {786 return CIInsts;787 }788 789 /// \returns true if the target has integer add/sub instructions that do not790 /// produce a carry-out. This includes v_add_[iu]32, v_sub_[iu]32,791 /// v_add_[iu]16, and v_sub_[iu]16, all of which support the clamp modifier792 /// for saturation.793 bool hasAddNoCarry() const {794 return AddNoCarryInsts;795 }796 797 bool hasScalarAddSub64() const { return getGeneration() >= GFX12; }798 799 bool hasScalarSMulU64() const { return getGeneration() >= GFX12; }800 801 bool hasUnpackedD16VMem() const {802 return HasUnpackedD16VMem;803 }804 805 // Covers VS/PS/CS graphics shaders806 bool isMesaGfxShader(const Function &F) const {807 return isMesa3DOS() && AMDGPU::isShader(F.getCallingConv());808 }809 810 bool hasMad64_32() const {811 return getGeneration() >= SEA_ISLANDS;812 }813 814 bool hasSDWAOmod() const {815 return HasSDWAOmod;816 }817 818 bool hasSDWAScalar() const {819 return HasSDWAScalar;820 }821 822 bool hasSDWASdst() const {823 return HasSDWASdst;824 }825 826 bool hasSDWAMac() const {827 return HasSDWAMac;828 }829 830 bool hasSDWAOutModsVOPC() const {831 return HasSDWAOutModsVOPC;832 }833 834 bool hasDLInsts() const {835 return HasDLInsts;836 }837 838 bool hasFmacF64Inst() const { return HasFmacF64Inst; }839 840 bool hasDot1Insts() const {841 return HasDot1Insts;842 }843 844 bool hasDot2Insts() const {845 return HasDot2Insts;846 }847 848 bool hasDot3Insts() const {849 return HasDot3Insts;850 }851 852 bool hasDot4Insts() const {853 return HasDot4Insts;854 }855 856 bool hasDot5Insts() const {857 return HasDot5Insts;858 }859 860 bool hasDot6Insts() const {861 return HasDot6Insts;862 }863 864 bool hasDot7Insts() const {865 return HasDot7Insts;866 }867 868 bool hasDot8Insts() const {869 return HasDot8Insts;870 }871 872 bool hasDot9Insts() const {873 return HasDot9Insts;874 }875 876 bool hasDot10Insts() const {877 return HasDot10Insts;878 }879 880 bool hasDot11Insts() const {881 return HasDot11Insts;882 }883 884 bool hasDot12Insts() const {885 return HasDot12Insts;886 }887 888 bool hasDot13Insts() const {889 return HasDot13Insts;890 }891 892 bool hasMAIInsts() const {893 return HasMAIInsts;894 }895 896 bool hasFP8Insts() const {897 return HasFP8Insts;898 }899 900 bool hasFP8ConversionInsts() const { return HasFP8ConversionInsts; }901 902 bool hasCubeInsts() const { return HasCubeInsts; }903 904 bool hasLerpInst() const { return HasLerpInst; }905 906 bool hasSadInsts() const { return HasSadInsts; }907 908 bool hasQsadInsts() const { return HasQsadInsts; }909 910 bool hasCvtNormInsts() const { return HasCvtNormInsts; }911 912 bool hasCvtPkNormVOP2Insts() const { return HasCvtPkNormVOP2Insts; }913 914 bool hasCvtPkNormVOP3Insts() const { return HasCvtPkNormVOP3Insts; }915 916 bool hasFP8E5M3Insts() const { return HasFP8E5M3Insts; }917 918 bool hasPkFmacF16Inst() const {919 return HasPkFmacF16Inst;920 }921 922 bool hasAtomicFMinFMaxF32GlobalInsts() const {923 return HasAtomicFMinFMaxF32GlobalInsts;924 }925 926 bool hasAtomicFMinFMaxF64GlobalInsts() const {927 return HasAtomicFMinFMaxF64GlobalInsts;928 }929 930 bool hasAtomicFMinFMaxF32FlatInsts() const {931 return HasAtomicFMinFMaxF32FlatInsts;932 }933 934 bool hasAtomicFMinFMaxF64FlatInsts() const {935 return HasAtomicFMinFMaxF64FlatInsts;936 }937 938 bool hasAtomicDsPkAdd16Insts() const { return HasAtomicDsPkAdd16Insts; }939 940 bool hasAtomicFlatPkAdd16Insts() const { return HasAtomicFlatPkAdd16Insts; }941 942 bool hasAtomicFaddInsts() const {943 return HasAtomicFaddRtnInsts || HasAtomicFaddNoRtnInsts;944 }945 946 bool hasAtomicFaddRtnInsts() const { return HasAtomicFaddRtnInsts; }947 948 bool hasAtomicFaddNoRtnInsts() const { return HasAtomicFaddNoRtnInsts; }949 950 bool hasAtomicBufferGlobalPkAddF16NoRtnInsts() const {951 return HasAtomicBufferGlobalPkAddF16NoRtnInsts;952 }953 954 bool hasAtomicBufferGlobalPkAddF16Insts() const {955 return HasAtomicBufferGlobalPkAddF16Insts;956 }957 958 bool hasAtomicGlobalPkAddBF16Inst() const {959 return HasAtomicGlobalPkAddBF16Inst;960 }961 962 bool hasAtomicBufferPkAddBF16Inst() const {963 return HasAtomicBufferPkAddBF16Inst;964 }965 966 bool hasFlatAtomicFaddF32Inst() const { return HasFlatAtomicFaddF32Inst; }967 968 /// \return true if the target has flat, global, and buffer atomic fadd for969 /// double.970 bool hasFlatBufferGlobalAtomicFaddF64Inst() const {971 return HasFlatBufferGlobalAtomicFaddF64Inst;972 }973 974 /// \return true if the target's flat, global, and buffer atomic fadd for975 /// float supports denormal handling.976 bool hasMemoryAtomicFaddF32DenormalSupport() const {977 return HasMemoryAtomicFaddF32DenormalSupport;978 }979 980 /// \return true if atomic operations targeting fine-grained memory work981 /// correctly at device scope, in allocations in host or peer PCIe device982 /// memory.983 bool supportsAgentScopeFineGrainedRemoteMemoryAtomics() const {984 return HasAgentScopeFineGrainedRemoteMemoryAtomics;985 }986 987 /// \return true is HW emulates system scope atomics unsupported by the PCI-e988 /// via CAS loop.989 bool hasEmulatedSystemScopeAtomics() const {990 return HasEmulatedSystemScopeAtomics;991 }992 993 bool hasDefaultComponentZero() const { return HasDefaultComponentZero; }994 995 bool hasDefaultComponentBroadcast() const {996 return HasDefaultComponentBroadcast;997 }998 999 bool hasNoSdstCMPX() const {1000 return HasNoSdstCMPX;1001 }1002 1003 bool hasVscnt() const {1004 return HasVscnt;1005 }1006 1007 bool hasGetWaveIdInst() const {1008 return HasGetWaveIdInst;1009 }1010 1011 bool hasSMemTimeInst() const {1012 return HasSMemTimeInst;1013 }1014 1015 bool hasShaderCyclesRegister() const {1016 return HasShaderCyclesRegister;1017 }1018 1019 bool hasShaderCyclesHiLoRegisters() const {1020 return HasShaderCyclesHiLoRegisters;1021 }1022 1023 bool hasVOP3Literal() const {1024 return HasVOP3Literal;1025 }1026 1027 bool hasNoDataDepHazard() const {1028 return HasNoDataDepHazard;1029 }1030 1031 bool vmemWriteNeedsExpWaitcnt() const {1032 return getGeneration() < SEA_ISLANDS;1033 }1034 1035 bool hasInstPrefetch() const {1036 return getGeneration() == GFX10 || getGeneration() == GFX11;1037 }1038 1039 bool hasPrefetch() const { return GFX12Insts; }1040 1041 bool hasVmemPrefInsts() const { return HasVmemPrefInsts; }1042 1043 bool hasSafeSmemPrefetch() const { return HasSafeSmemPrefetch; }1044 1045 bool hasSafeCUPrefetch() const { return HasSafeCUPrefetch; }1046 1047 // Has s_cmpk_* instructions.1048 bool hasSCmpK() const { return getGeneration() < GFX12; }1049 1050 // Scratch is allocated in 256 dword per wave blocks for the entire1051 // wavefront. When viewed from the perspective of an arbitrary workitem, this1052 // is 4-byte aligned.1053 //1054 // Only 4-byte alignment is really needed to access anything. Transformations1055 // on the pointer value itself may rely on the alignment / known low bits of1056 // the pointer. Set this to something above the minimum to avoid needing1057 // dynamic realignment in common cases.1058 Align getStackAlignment() const { return Align(16); }1059 1060 bool enableMachineScheduler() const override {1061 return true;1062 }1063 1064 bool useAA() const override;1065 1066 bool enableSubRegLiveness() const override {1067 return true;1068 }1069 1070 void setScalarizeGlobalBehavior(bool b) { ScalarizeGlobal = b; }1071 bool getScalarizeGlobalBehavior() const { return ScalarizeGlobal; }1072 1073 // static wrappers1074 static bool hasHalfRate64Ops(const TargetSubtargetInfo &STI);1075 1076 // XXX - Why is this here if it isn't in the default pass set?1077 bool enableEarlyIfConversion() const override {1078 return true;1079 }1080 1081 void overrideSchedPolicy(MachineSchedPolicy &Policy,1082 const SchedRegion &Region) const override;1083 1084 void overridePostRASchedPolicy(MachineSchedPolicy &Policy,1085 const SchedRegion &Region) const override;1086 1087 void mirFileLoaded(MachineFunction &MF) const override;1088 1089 unsigned getMaxNumUserSGPRs() const {1090 return AMDGPU::getMaxNumUserSGPRs(*this);1091 }1092 1093 bool hasSMemRealTime() const {1094 return HasSMemRealTime;1095 }1096 1097 bool hasMovrel() const {1098 return HasMovrel;1099 }1100 1101 bool hasVGPRIndexMode() const {1102 return HasVGPRIndexMode;1103 }1104 1105 bool useVGPRIndexMode() const;1106 1107 bool hasScalarCompareEq64() const {1108 return getGeneration() >= VOLCANIC_ISLANDS;1109 }1110 1111 bool hasScalarDwordx3Loads() const { return HasScalarDwordx3Loads; }1112 1113 bool hasScalarStores() const {1114 return HasScalarStores;1115 }1116 1117 bool hasScalarAtomics() const {1118 return HasScalarAtomics;1119 }1120 1121 bool hasLDSFPAtomicAddF32() const { return GFX8Insts; }1122 bool hasLDSFPAtomicAddF64() const { return GFX90AInsts || GFX1250Insts; }1123 1124 /// \returns true if the subtarget has the v_permlanex16_b32 instruction.1125 bool hasPermLaneX16() const { return getGeneration() >= GFX10; }1126 1127 /// \returns true if the subtarget has the v_permlane64_b32 instruction.1128 bool hasPermLane64() const { return getGeneration() >= GFX11; }1129 1130 bool hasDPP() const {1131 return HasDPP;1132 }1133 1134 bool hasDPPBroadcasts() const {1135 return HasDPP && getGeneration() < GFX10;1136 }1137 1138 bool hasDPPWavefrontShifts() const {1139 return HasDPP && getGeneration() < GFX10;1140 }1141 1142 bool hasDPP8() const {1143 return HasDPP8;1144 }1145 1146 bool hasDPALU_DPP() const {1147 return HasDPALU_DPP;1148 }1149 1150 bool hasDPPSrc1SGPR() const { return HasDPPSrc1SGPR; }1151 1152 bool hasPackedFP32Ops() const {1153 return HasPackedFP32Ops;1154 }1155 1156 // Has V_PK_MOV_B32 opcode1157 bool hasPkMovB32() const {1158 return GFX90AInsts;1159 }1160 1161 bool hasFmaakFmamkF32Insts() const {1162 return getGeneration() >= GFX10 || hasGFX940Insts();1163 }1164 1165 bool hasFmaakFmamkF64Insts() const { return hasGFX1250Insts(); }1166 1167 bool hasImageInsts() const {1168 return HasImageInsts;1169 }1170 1171 bool hasExtendedImageInsts() const {1172 return HasExtendedImageInsts;1173 }1174 1175 bool hasR128A16() const {1176 return HasR128A16;1177 }1178 1179 bool hasA16() const { return HasA16; }1180 1181 bool hasG16() const { return HasG16; }1182 1183 bool hasOffset3fBug() const {1184 return HasOffset3fBug;1185 }1186 1187 bool hasImageStoreD16Bug() const { return HasImageStoreD16Bug; }1188 1189 bool hasImageGather4D16Bug() const { return HasImageGather4D16Bug; }1190 1191 bool hasMADIntraFwdBug() const { return HasMADIntraFwdBug; }1192 1193 bool hasMSAALoadDstSelBug() const { return HasMSAALoadDstSelBug; }1194 1195 bool hasPrivEnabledTrap2NopBug() const { return HasPrivEnabledTrap2NopBug; }1196 1197 bool hasNSAEncoding() const { return HasNSAEncoding; }1198 1199 bool hasNonNSAEncoding() const { return getGeneration() < GFX12; }1200 1201 bool hasPartialNSAEncoding() const { return HasPartialNSAEncoding; }1202 1203 unsigned getNSAMaxSize(bool HasSampler = false) const {1204 return AMDGPU::getNSAMaxSize(*this, HasSampler);1205 }1206 1207 bool hasGFX10_AEncoding() const {1208 return GFX10_AEncoding;1209 }1210 1211 bool hasGFX10_BEncoding() const {1212 return GFX10_BEncoding;1213 }1214 1215 bool hasGFX10_3Insts() const {1216 return GFX10_3Insts;1217 }1218 1219 bool hasMadF16() const;1220 1221 bool hasMovB64() const { return GFX940Insts || GFX1250Insts; }1222 1223 bool hasLshlAddU64Inst() const { return HasLshlAddU64Inst; }1224 1225 // Scalar and global loads support scale_offset bit.1226 bool hasScaleOffset() const { return GFX1250Insts; }1227 1228 bool hasFlatGVSMode() const { return FlatGVSMode; }1229 1230 // FLAT GLOBAL VOffset is signed1231 bool hasSignedGVSOffset() const { return GFX1250Insts; }1232 1233 bool enableSIScheduler() const {1234 return EnableSIScheduler;1235 }1236 1237 bool loadStoreOptEnabled() const {1238 return EnableLoadStoreOpt;1239 }1240 1241 bool hasSGPRInitBug() const {1242 return SGPRInitBug;1243 }1244 1245 bool hasUserSGPRInit16Bug() const {1246 return UserSGPRInit16Bug && isWave32();1247 }1248 1249 bool hasNegativeScratchOffsetBug() const { return NegativeScratchOffsetBug; }1250 1251 bool hasNegativeUnalignedScratchOffsetBug() const {1252 return NegativeUnalignedScratchOffsetBug;1253 }1254 1255 bool hasMFMAInlineLiteralBug() const {1256 return HasMFMAInlineLiteralBug;1257 }1258 1259 bool has12DWordStoreHazard() const {1260 return getGeneration() != AMDGPUSubtarget::SOUTHERN_ISLANDS;1261 }1262 1263 // \returns true if the subtarget supports DWORDX3 load/store instructions.1264 bool hasDwordx3LoadStores() const {1265 return CIInsts;1266 }1267 1268 bool hasReadM0MovRelInterpHazard() const {1269 return getGeneration() == AMDGPUSubtarget::GFX9;1270 }1271 1272 bool hasReadM0SendMsgHazard() const {1273 return getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&1274 getGeneration() <= AMDGPUSubtarget::GFX9;1275 }1276 1277 bool hasReadM0LdsDmaHazard() const {1278 return getGeneration() == AMDGPUSubtarget::GFX9;1279 }1280 1281 bool hasReadM0LdsDirectHazard() const {1282 return getGeneration() == AMDGPUSubtarget::GFX9;1283 }1284 1285 bool hasVcmpxPermlaneHazard() const {1286 return HasVcmpxPermlaneHazard;1287 }1288 1289 bool hasVMEMtoScalarWriteHazard() const {1290 return HasVMEMtoScalarWriteHazard;1291 }1292 1293 bool hasSMEMtoVectorWriteHazard() const {1294 return HasSMEMtoVectorWriteHazard;1295 }1296 1297 bool hasLDSMisalignedBug() const {1298 return LDSMisalignedBug && !EnableCuMode;1299 }1300 1301 bool hasInstFwdPrefetchBug() const {1302 return HasInstFwdPrefetchBug;1303 }1304 1305 bool hasVcmpxExecWARHazard() const {1306 return HasVcmpxExecWARHazard;1307 }1308 1309 bool hasLdsBranchVmemWARHazard() const {1310 return HasLdsBranchVmemWARHazard;1311 }1312 1313 // Shift amount of a 64 bit shift cannot be a highest allocated register1314 // if also at the end of the allocation block.1315 bool hasShift64HighRegBug() const {1316 return GFX90AInsts && !GFX940Insts;1317 }1318 1319 // Has one cycle hazard on transcendental instruction feeding a1320 // non transcendental VALU.1321 bool hasTransForwardingHazard() const { return GFX940Insts; }1322 1323 // Has one cycle hazard on a VALU instruction partially writing dst with1324 // a shift of result bits feeding another VALU instruction.1325 bool hasDstSelForwardingHazard() const { return GFX940Insts; }1326 1327 // Cannot use op_sel with v_dot instructions.1328 bool hasDOTOpSelHazard() const { return GFX940Insts || GFX11Insts; }1329 1330 // Does not have HW interlocs for VALU writing and then reading SGPRs.1331 bool hasVDecCoExecHazard() const {1332 return GFX940Insts;1333 }1334 1335 bool hasNSAtoVMEMBug() const {1336 return HasNSAtoVMEMBug;1337 }1338 1339 bool hasNSAClauseBug() const { return HasNSAClauseBug; }1340 1341 bool hasHardClauses() const { return MaxHardClauseLength > 0; }1342 1343 bool hasGFX90AInsts() const { return GFX90AInsts; }1344 1345 bool hasFPAtomicToDenormModeHazard() const {1346 return getGeneration() == GFX10;1347 }1348 1349 bool hasVOP3DPP() const { return getGeneration() >= GFX11; }1350 1351 bool hasLdsDirect() const { return getGeneration() >= GFX11; }1352 1353 bool hasLdsWaitVMSRC() const { return getGeneration() >= GFX12; }1354 1355 bool hasVALUPartialForwardingHazard() const {1356 return getGeneration() == GFX11;1357 }1358 1359 bool hasVALUTransUseHazard() const { return HasVALUTransUseHazard; }1360 1361 bool hasCvtScaleForwardingHazard() const { return GFX950Insts; }1362 1363 bool requiresCodeObjectV6() const { return RequiresCOV6; }1364 1365 bool useVGPRBlockOpsForCSR() const { return UseBlockVGPROpsForCSR; }1366 1367 bool hasGloballyAddressableScratch() const {1368 return HasGloballyAddressableScratch;1369 }1370 1371 bool hasVALUMaskWriteHazard() const { return getGeneration() == GFX11; }1372 1373 bool hasVALUReadSGPRHazard() const { return GFX12Insts && !GFX1250Insts; }1374 1375 bool setRegModeNeedsVNOPs() const {1376 return GFX1250Insts && getGeneration() == GFX12;1377 }1378 1379 /// Return if operations acting on VGPR tuples require even alignment.1380 bool needsAlignedVGPRs() const { return RequiresAlignVGPR; }1381 1382 /// Return true if the target has the S_PACK_HL_B32_B16 instruction.1383 bool hasSPackHL() const { return GFX11Insts; }1384 1385 /// Return true if the target's EXP instruction has the COMPR flag, which1386 /// affects the meaning of the EN (enable) bits.1387 bool hasCompressedExport() const { return !GFX11Insts; }1388 1389 /// Return true if the target's EXP instruction supports the NULL export1390 /// target.1391 bool hasNullExportTarget() const { return !GFX11Insts; }1392 1393 bool has1_5xVGPRs() const { return Has1_5xVGPRs; }1394 1395 bool hasVOPDInsts() const { return HasVOPDInsts; }1396 1397 bool hasFlatScratchSVSSwizzleBug() const { return getGeneration() == GFX11; }1398 1399 /// Return true if the target has the S_DELAY_ALU instruction.1400 bool hasDelayAlu() const { return GFX11Insts; }1401 1402 bool hasPackedTID() const { return HasPackedTID; }1403 1404 // GFX94* is a derivation to GFX90A. hasGFX940Insts() being true implies that1405 // hasGFX90AInsts is also true.1406 bool hasGFX940Insts() const { return GFX940Insts; }1407 1408 // GFX950 is a derivation to GFX94*. hasGFX950Insts() implies that1409 // hasGFX940Insts and hasGFX90AInsts are also true.1410 bool hasGFX950Insts() const { return GFX950Insts; }1411 1412 /// Returns true if the target supports1413 /// global_load_lds_dwordx3/global_load_lds_dwordx4 or1414 /// buffer_load_dwordx3/buffer_load_dwordx4 with the lds bit.1415 bool hasLDSLoadB96_B128() const {1416 return hasGFX950Insts();1417 }1418 1419 bool hasVMemToLDSLoad() const { return HasVMemToLDSLoad; }1420 1421 bool hasSALUFloatInsts() const { return HasSALUFloatInsts; }1422 1423 bool hasPseudoScalarTrans() const { return HasPseudoScalarTrans; }1424 1425 bool hasRestrictedSOffset() const { return HasRestrictedSOffset; }1426 1427 bool hasRequiredExportPriority() const { return HasRequiredExportPriority; }1428 1429 bool hasVmemWriteVgprInOrder() const { return HasVmemWriteVgprInOrder; }1430 1431 /// \returns true if the target uses LOADcnt/SAMPLEcnt/BVHcnt, DScnt/KMcnt1432 /// and STOREcnt rather than VMcnt, LGKMcnt and VScnt respectively.1433 bool hasExtendedWaitCounts() const { return getGeneration() >= GFX12; }1434 1435 /// \returns true if inline constants are not supported for F16 pseudo1436 /// scalar transcendentals.1437 bool hasNoF16PseudoScalarTransInlineConstants() const {1438 return getGeneration() == GFX12;1439 }1440 1441 /// \returns true if the target has instructions with xf32 format support.1442 bool hasXF32Insts() const { return HasXF32Insts; }1443 1444 /// \returns true if the target has packed f32 instructions that only read 321445 /// bits from a scalar operand (SGPR or literal) and replicates the bits to1446 /// both channels.1447 bool hasPKF32InstsReplicatingLower32BitsOfScalarInput() const {1448 return getGeneration() == GFX12 && GFX1250Insts;1449 }1450 1451 bool hasBitOp3Insts() const { return HasBitOp3Insts; }1452 1453 bool hasPermlane16Swap() const { return HasPermlane16Swap; }1454 bool hasPermlane32Swap() const { return HasPermlane32Swap; }1455 bool hasAshrPkInsts() const { return HasAshrPkInsts; }1456 1457 bool hasMinimum3Maximum3F32() const {1458 return HasMinimum3Maximum3F32;1459 }1460 1461 bool hasMinimum3Maximum3F16() const {1462 return HasMinimum3Maximum3F16;1463 }1464 1465 bool hasMin3Max3PKF16() const { return HasMin3Max3PKF16; }1466 1467 bool hasTanhInsts() const { return HasTanhInsts; }1468 1469 bool hasTensorCvtLutInsts() const { return HasTensorCvtLutInsts; }1470 1471 bool hasAddPC64Inst() const { return GFX1250Insts; }1472 1473 bool has1024AddressableVGPRs() const { return Has1024AddressableVGPRs; }1474 1475 bool hasMinimum3Maximum3PKF16() const {1476 return HasMinimum3Maximum3PKF16;1477 }1478 1479 bool hasTransposeLoadF4F6Insts() const { return HasTransposeLoadF4F6Insts; }1480 1481 /// \returns true if the target has s_wait_xcnt insertion. Supported for1482 /// GFX1250.1483 bool hasWaitXCnt() const { return HasWaitXcnt; }1484 1485 // A single DWORD instructions can use a 64-bit literal.1486 bool has64BitLiterals() const { return Has64BitLiterals; }1487 1488 bool hasPointSampleAccel() const { return HasPointSampleAccel; }1489 1490 bool hasLdsBarrierArriveAtomic() const { return HasLdsBarrierArriveAtomic; }1491 1492 /// \returns The maximum number of instructions that can be enclosed in an1493 /// S_CLAUSE on the given subtarget, or 0 for targets that do not support that1494 /// instruction.1495 unsigned maxHardClauseLength() const { return MaxHardClauseLength; }1496 1497 bool hasPrngInst() const { return HasPrngInst; }1498 1499 bool hasBVHDualAndBVH8Insts() const { return HasBVHDualAndBVH8Insts; }1500 1501 /// Return the maximum number of waves per SIMD for kernels using \p SGPRs1502 /// SGPRs1503 unsigned getOccupancyWithNumSGPRs(unsigned SGPRs) const;1504 1505 /// Return the maximum number of waves per SIMD for kernels using \p VGPRs1506 /// VGPRs1507 unsigned getOccupancyWithNumVGPRs(unsigned VGPRs,1508 unsigned DynamicVGPRBlockSize) const;1509 1510 /// Subtarget's minimum/maximum occupancy, in number of waves per EU, that can1511 /// be achieved when the only function running on a CU is \p F, each workgroup1512 /// uses \p LDSSize bytes of LDS, and each wave uses \p NumSGPRs SGPRs and \p1513 /// NumVGPRs VGPRs. The flat workgroup sizes associated to the function are a1514 /// range, so this returns a range as well.1515 ///1516 /// Note that occupancy can be affected by the scratch allocation as well, but1517 /// we do not have enough information to compute it.1518 std::pair<unsigned, unsigned> computeOccupancy(const Function &F,1519 unsigned LDSSize = 0,1520 unsigned NumSGPRs = 0,1521 unsigned NumVGPRs = 0) const;1522 1523 /// \returns true if the flat_scratch register should be initialized with the1524 /// pointer to the wave's scratch memory rather than a size and offset.1525 bool flatScratchIsPointer() const {1526 return getGeneration() >= AMDGPUSubtarget::GFX9;1527 }1528 1529 /// \returns true if the flat_scratch register is initialized by the HW.1530 /// In this case it is readonly.1531 bool flatScratchIsArchitected() const { return HasArchitectedFlatScratch; }1532 1533 /// \returns true if the architected SGPRs are enabled.1534 bool hasArchitectedSGPRs() const { return HasArchitectedSGPRs; }1535 1536 /// \returns true if Global Data Share is supported.1537 bool hasGDS() const { return HasGDS; }1538 1539 /// \returns true if Global Wave Sync is supported.1540 bool hasGWS() const { return HasGWS; }1541 1542 /// \returns true if the machine has merged shaders in which s0-s7 are1543 /// reserved by the hardware and user SGPRs start at s81544 bool hasMergedShaders() const {1545 return getGeneration() >= GFX9;1546 }1547 1548 // \returns true if the target supports the pre-NGG legacy geometry path.1549 bool hasLegacyGeometry() const { return getGeneration() < GFX11; }1550 1551 // \returns true if preloading kernel arguments is supported.1552 bool hasKernargPreload() const { return KernargPreload; }1553 1554 // \returns true if the target has split barriers feature1555 bool hasSplitBarriers() const { return getGeneration() >= GFX12; }1556 1557 // \returns true if FP8/BF8 VOP1 form of conversion to F32 is unreliable.1558 bool hasCvtFP8VOP1Bug() const { return HasCvtFP8Vop1Bug; }1559 1560 // \returns true if CSUB (a.k.a. SUB_CLAMP on GFX12) atomics support a1561 // no-return form.1562 bool hasAtomicCSubNoRtnInsts() const { return HasAtomicCSubNoRtnInsts; }1563 1564 // \returns true if the target has DX10_CLAMP kernel descriptor mode bit1565 bool hasDX10ClampMode() const { return getGeneration() < GFX12; }1566 1567 // \returns true if the target has IEEE kernel descriptor mode bit1568 bool hasIEEEMode() const { return getGeneration() < GFX12; }1569 1570 // \returns true if the target has IEEE fminimum/fmaximum instructions1571 bool hasIEEEMinimumMaximumInsts() const { return HasIEEEMinimumMaximumInsts; }1572 1573 // \returns true if the target has WG_RR_MODE kernel descriptor mode bit1574 bool hasRrWGMode() const { return getGeneration() >= GFX12; }1575 1576 /// \returns true if VADDR and SADDR fields in VSCRATCH can use negative1577 /// values.1578 bool hasSignedScratchOffsets() const { return getGeneration() >= GFX12; }1579 1580 bool hasGFX1250Insts() const { return GFX1250Insts; }1581 1582 bool hasVOPD3() const { return GFX1250Insts; }1583 1584 // \returns true if the target has V_ADD_U64/V_SUB_U64 instructions.1585 bool hasAddSubU64Insts() const { return HasAddSubU64Insts; }1586 1587 // \returns true if the target has V_MAD_U32 instruction.1588 bool hasMadU32Inst() const { return HasMadU32Inst; }1589 1590 // \returns true if the target has V_MUL_U64/V_MUL_I64 instructions.1591 bool hasVectorMulU64() const { return GFX1250Insts; }1592 1593 // \returns true if the target has V_MAD_NC_U64_U32/V_MAD_NC_I64_I321594 // instructions.1595 bool hasMadU64U32NoCarry() const { return GFX1250Insts; }1596 1597 // \returns true if the target has V_{MIN|MAX}_{I|U}64 instructions.1598 bool hasIntMinMax64() const { return GFX1250Insts; }1599 1600 // \returns true if the target has V_ADD_{MIN|MAX}_{I|U}32 instructions.1601 bool hasAddMinMaxInsts() const { return HasAddMinMaxInsts; }1602 1603 // \returns true if the target has V_PK_ADD_{MIN|MAX}_{I|U}16 instructions.1604 bool hasPkAddMinMaxInsts() const { return HasPkAddMinMaxInsts; }1605 1606 // \returns true if the target has V_PK_{MIN|MAX}3_{I|U}16 instructions.1607 bool hasPkMinMax3Insts() const { return GFX1250Insts; }1608 1609 // \returns ture if target has S_GET_SHADER_CYCLES_U64 instruction.1610 bool hasSGetShaderCyclesInst() const { return GFX1250Insts; }1611 1612 // \returns true if target has S_SETPRIO_INC_WG instruction.1613 bool hasSetPrioIncWgInst() const { return HasSetPrioIncWgInst; }1614 1615 // \returns true if S_GETPC_B64 zero-extends the result from 48 bits instead1616 // of sign-extending. Note that GFX1250 has not only fixed the bug but also1617 // extended VA to 57 bits.1618 bool hasGetPCZeroExtension() const { return GFX12Insts && !GFX1250Insts; }1619 1620 // \returns true if the target needs to create a prolog for backward1621 // compatibility when preloading kernel arguments.1622 bool needsKernArgPreloadProlog() const {1623 return hasKernargPreload() && !GFX1250Insts;1624 }1625 1626 /// \returns SGPR allocation granularity supported by the subtarget.1627 unsigned getSGPRAllocGranule() const {1628 return AMDGPU::IsaInfo::getSGPRAllocGranule(this);1629 }1630 1631 /// \returns SGPR encoding granularity supported by the subtarget.1632 unsigned getSGPREncodingGranule() const {1633 return AMDGPU::IsaInfo::getSGPREncodingGranule(this);1634 }1635 1636 /// \returns Total number of SGPRs supported by the subtarget.1637 unsigned getTotalNumSGPRs() const {1638 return AMDGPU::IsaInfo::getTotalNumSGPRs(this);1639 }1640 1641 /// \returns Addressable number of SGPRs supported by the subtarget.1642 unsigned getAddressableNumSGPRs() const {1643 return AMDGPU::IsaInfo::getAddressableNumSGPRs(this);1644 }1645 1646 /// \returns Minimum number of SGPRs that meets the given number of waves per1647 /// execution unit requirement supported by the subtarget.1648 unsigned getMinNumSGPRs(unsigned WavesPerEU) const {1649 return AMDGPU::IsaInfo::getMinNumSGPRs(this, WavesPerEU);1650 }1651 1652 /// \returns Maximum number of SGPRs that meets the given number of waves per1653 /// execution unit requirement supported by the subtarget.1654 unsigned getMaxNumSGPRs(unsigned WavesPerEU, bool Addressable) const {1655 return AMDGPU::IsaInfo::getMaxNumSGPRs(this, WavesPerEU, Addressable);1656 }1657 1658 /// \returns Reserved number of SGPRs. This is common1659 /// utility function called by MachineFunction and1660 /// Function variants of getReservedNumSGPRs.1661 unsigned getBaseReservedNumSGPRs(const bool HasFlatScratch) const;1662 /// \returns Reserved number of SGPRs for given machine function \p MF.1663 unsigned getReservedNumSGPRs(const MachineFunction &MF) const;1664 1665 /// \returns Reserved number of SGPRs for given function \p F.1666 unsigned getReservedNumSGPRs(const Function &F) const;1667 1668 /// \returns Maximum number of preloaded SGPRs for the subtarget.1669 unsigned getMaxNumPreloadedSGPRs() const;1670 1671 /// \returns max num SGPRs. This is the common utility1672 /// function called by MachineFunction and Function1673 /// variants of getMaxNumSGPRs.1674 unsigned getBaseMaxNumSGPRs(const Function &F,1675 std::pair<unsigned, unsigned> WavesPerEU,1676 unsigned PreloadedSGPRs,1677 unsigned ReservedNumSGPRs) const;1678 1679 /// \returns Maximum number of SGPRs that meets number of waves per execution1680 /// unit requirement for function \p MF, or number of SGPRs explicitly1681 /// requested using "amdgpu-num-sgpr" attribute attached to function \p MF.1682 ///1683 /// \returns Value that meets number of waves per execution unit requirement1684 /// if explicitly requested value cannot be converted to integer, violates1685 /// subtarget's specifications, or does not meet number of waves per execution1686 /// unit requirement.1687 unsigned getMaxNumSGPRs(const MachineFunction &MF) const;1688 1689 /// \returns Maximum number of SGPRs that meets number of waves per execution1690 /// unit requirement for function \p F, or number of SGPRs explicitly1691 /// requested using "amdgpu-num-sgpr" attribute attached to function \p F.1692 ///1693 /// \returns Value that meets number of waves per execution unit requirement1694 /// if explicitly requested value cannot be converted to integer, violates1695 /// subtarget's specifications, or does not meet number of waves per execution1696 /// unit requirement.1697 unsigned getMaxNumSGPRs(const Function &F) const;1698 1699 /// \returns VGPR allocation granularity supported by the subtarget.1700 unsigned getVGPRAllocGranule(unsigned DynamicVGPRBlockSize) const {1701 return AMDGPU::IsaInfo::getVGPRAllocGranule(this, DynamicVGPRBlockSize);1702 }1703 1704 /// \returns VGPR encoding granularity supported by the subtarget.1705 unsigned getVGPREncodingGranule() const {1706 return AMDGPU::IsaInfo::getVGPREncodingGranule(this);1707 }1708 1709 /// \returns Total number of VGPRs supported by the subtarget.1710 unsigned getTotalNumVGPRs() const {1711 return AMDGPU::IsaInfo::getTotalNumVGPRs(this);1712 }1713 1714 /// \returns Addressable number of architectural VGPRs supported by the1715 /// subtarget.1716 unsigned getAddressableNumArchVGPRs() const {1717 return AMDGPU::IsaInfo::getAddressableNumArchVGPRs(this);1718 }1719 1720 /// \returns Addressable number of VGPRs supported by the subtarget.1721 unsigned getAddressableNumVGPRs(unsigned DynamicVGPRBlockSize) const {1722 return AMDGPU::IsaInfo::getAddressableNumVGPRs(this, DynamicVGPRBlockSize);1723 }1724 1725 /// \returns the minimum number of VGPRs that will prevent achieving more than1726 /// the specified number of waves \p WavesPerEU.1727 unsigned getMinNumVGPRs(unsigned WavesPerEU,1728 unsigned DynamicVGPRBlockSize) const {1729 return AMDGPU::IsaInfo::getMinNumVGPRs(this, WavesPerEU,1730 DynamicVGPRBlockSize);1731 }1732 1733 /// \returns the maximum number of VGPRs that can be used and still achieved1734 /// at least the specified number of waves \p WavesPerEU.1735 unsigned getMaxNumVGPRs(unsigned WavesPerEU,1736 unsigned DynamicVGPRBlockSize) const {1737 return AMDGPU::IsaInfo::getMaxNumVGPRs(this, WavesPerEU,1738 DynamicVGPRBlockSize);1739 }1740 1741 /// \returns max num VGPRs. This is the common utility function1742 /// called by MachineFunction and Function variants of getMaxNumVGPRs.1743 unsigned1744 getBaseMaxNumVGPRs(const Function &F,1745 std::pair<unsigned, unsigned> NumVGPRBounds) const;1746 1747 /// \returns Maximum number of VGPRs that meets number of waves per execution1748 /// unit requirement for function \p F, or number of VGPRs explicitly1749 /// requested using "amdgpu-num-vgpr" attribute attached to function \p F.1750 ///1751 /// \returns Value that meets number of waves per execution unit requirement1752 /// if explicitly requested value cannot be converted to integer, violates1753 /// subtarget's specifications, or does not meet number of waves per execution1754 /// unit requirement.1755 unsigned getMaxNumVGPRs(const Function &F) const;1756 1757 unsigned getMaxNumAGPRs(const Function &F) const {1758 return getMaxNumVGPRs(F);1759 }1760 1761 /// Return a pair of maximum numbers of VGPRs and AGPRs that meet the number1762 /// of waves per execution unit required for the function \p MF.1763 std::pair<unsigned, unsigned> getMaxNumVectorRegs(const Function &F) const;1764 1765 /// \returns Maximum number of VGPRs that meets number of waves per execution1766 /// unit requirement for function \p MF, or number of VGPRs explicitly1767 /// requested using "amdgpu-num-vgpr" attribute attached to function \p MF.1768 ///1769 /// \returns Value that meets number of waves per execution unit requirement1770 /// if explicitly requested value cannot be converted to integer, violates1771 /// subtarget's specifications, or does not meet number of waves per execution1772 /// unit requirement.1773 unsigned getMaxNumVGPRs(const MachineFunction &MF) const;1774 1775 bool supportsWave32() const { return getGeneration() >= GFX10; }1776 1777 bool supportsWave64() const { return !hasGFX1250Insts(); }1778 1779 bool isWave32() const {1780 return getWavefrontSize() == 32;1781 }1782 1783 bool isWave64() const {1784 return getWavefrontSize() == 64;1785 }1786 1787 /// Returns if the wavesize of this subtarget is known reliable. This is false1788 /// only for the a default target-cpu that does not have an explicit1789 /// +wavefrontsize target feature.1790 bool isWaveSizeKnown() const {1791 return hasFeature(AMDGPU::FeatureWavefrontSize32) ||1792 hasFeature(AMDGPU::FeatureWavefrontSize64);1793 }1794 1795 const TargetRegisterClass *getBoolRC() const {1796 return getRegisterInfo()->getBoolRC();1797 }1798 1799 /// \returns Maximum number of work groups per compute unit supported by the1800 /// subtarget and limited by given \p FlatWorkGroupSize.1801 unsigned getMaxWorkGroupsPerCU(unsigned FlatWorkGroupSize) const override {1802 return AMDGPU::IsaInfo::getMaxWorkGroupsPerCU(this, FlatWorkGroupSize);1803 }1804 1805 /// \returns Minimum flat work group size supported by the subtarget.1806 unsigned getMinFlatWorkGroupSize() const override {1807 return AMDGPU::IsaInfo::getMinFlatWorkGroupSize(this);1808 }1809 1810 /// \returns Maximum flat work group size supported by the subtarget.1811 unsigned getMaxFlatWorkGroupSize() const override {1812 return AMDGPU::IsaInfo::getMaxFlatWorkGroupSize(this);1813 }1814 1815 /// \returns Number of waves per execution unit required to support the given1816 /// \p FlatWorkGroupSize.1817 unsigned1818 getWavesPerEUForWorkGroup(unsigned FlatWorkGroupSize) const override {1819 return AMDGPU::IsaInfo::getWavesPerEUForWorkGroup(this, FlatWorkGroupSize);1820 }1821 1822 /// \returns Minimum number of waves per execution unit supported by the1823 /// subtarget.1824 unsigned getMinWavesPerEU() const override {1825 return AMDGPU::IsaInfo::getMinWavesPerEU(this);1826 }1827 1828 void adjustSchedDependency(SUnit *Def, int DefOpIdx, SUnit *Use, int UseOpIdx,1829 SDep &Dep,1830 const TargetSchedModel *SchedModel) const override;1831 1832 // \returns true if it's beneficial on this subtarget for the scheduler to1833 // cluster stores as well as loads.1834 bool shouldClusterStores() const { return getGeneration() >= GFX11; }1835 1836 // \returns the number of address arguments from which to enable MIMG NSA1837 // on supported architectures.1838 unsigned getNSAThreshold(const MachineFunction &MF) const;1839 1840 // \returns true if the subtarget has a hazard requiring an "s_nop 0"1841 // instruction before "s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)".1842 bool requiresNopBeforeDeallocVGPRs() const { return !GFX1250Insts; }1843 1844 // \returns true if the subtarget needs S_WAIT_ALU 0 before S_GETREG_B32 on1845 // STATUS, STATE_PRIV, EXCP_FLAG_PRIV, or EXCP_FLAG_USER.1846 bool requiresWaitIdleBeforeGetReg() const { return GFX1250Insts; }1847 1848 bool isDynamicVGPREnabled() const { return DynamicVGPR; }1849 unsigned getDynamicVGPRBlockSize() const {1850 return DynamicVGPRBlockSize32 ? 32 : 16;1851 }1852 1853 bool requiresDisjointEarlyClobberAndUndef() const override {1854 // AMDGPU doesn't care if early-clobber and undef operands are allocated1855 // to the same register.1856 return false;1857 }1858 1859 // DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 shall not be claused with anything1860 // and surronded by S_WAIT_ALU(0xFFE3).1861 bool hasDsAtomicAsyncBarrierArriveB64PipeBug() const {1862 return getGeneration() == GFX12;1863 }1864 1865 // Requires s_wait_alu(0) after s102/s103 write and src_flat_scratch_base1866 // read.1867 bool hasScratchBaseForwardingHazard() const {1868 return GFX1250Insts && getGeneration() == GFX12;1869 }1870 1871 /// \returns true if the subtarget supports clusters of workgroups.1872 bool hasClusters() const { return HasClusters; }1873 1874 /// \returns true if the subtarget requires a wait for xcnt before VMEM1875 /// accesses that must never be repeated in the event of a page fault/re-try.1876 /// Atomic stores/rmw and all volatile accesses fall under this criteria.1877 bool requiresWaitXCntForSingleAccessInstructions() const {1878 return GFX1250Insts;1879 }1880 1881 /// \returns the number of significant bits in the immediate field of the1882 /// S_NOP instruction.1883 unsigned getSNopBits() const {1884 if (getGeneration() >= AMDGPUSubtarget::GFX12)1885 return 7;1886 if (getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)1887 return 4;1888 return 3;1889 }1890 1891 /// \returns true if the sub-target supports buffer resource (V#) with 45-bit1892 /// num_records.1893 bool has45BitNumRecordsBufferResource() const {1894 return Has45BitNumRecordsBufferResource;1895 }1896 1897 bool requiresWaitsBeforeSystemScopeStores() const {1898 return RequiresWaitsBeforeSystemScopeStores;1899 }1900};1901 1902class GCNUserSGPRUsageInfo {1903public:1904 bool hasImplicitBufferPtr() const { return ImplicitBufferPtr; }1905 1906 bool hasPrivateSegmentBuffer() const { return PrivateSegmentBuffer; }1907 1908 bool hasDispatchPtr() const { return DispatchPtr; }1909 1910 bool hasQueuePtr() const { return QueuePtr; }1911 1912 bool hasKernargSegmentPtr() const { return KernargSegmentPtr; }1913 1914 bool hasDispatchID() const { return DispatchID; }1915 1916 bool hasFlatScratchInit() const { return FlatScratchInit; }1917 1918 bool hasPrivateSegmentSize() const { return PrivateSegmentSize; }1919 1920 unsigned getNumKernargPreloadSGPRs() const { return NumKernargPreloadSGPRs; }1921 1922 unsigned getNumUsedUserSGPRs() const { return NumUsedUserSGPRs; }1923 1924 unsigned getNumFreeUserSGPRs();1925 1926 void allocKernargPreloadSGPRs(unsigned NumSGPRs);1927 1928 enum UserSGPRID : unsigned {1929 ImplicitBufferPtrID = 0,1930 PrivateSegmentBufferID = 1,1931 DispatchPtrID = 2,1932 QueuePtrID = 3,1933 KernargSegmentPtrID = 4,1934 DispatchIdID = 5,1935 FlatScratchInitID = 6,1936 PrivateSegmentSizeID = 71937 };1938 1939 // Returns the size in number of SGPRs for preload user SGPR field.1940 static unsigned getNumUserSGPRForField(UserSGPRID ID) {1941 switch (ID) {1942 case ImplicitBufferPtrID:1943 return 2;1944 case PrivateSegmentBufferID:1945 return 4;1946 case DispatchPtrID:1947 return 2;1948 case QueuePtrID:1949 return 2;1950 case KernargSegmentPtrID:1951 return 2;1952 case DispatchIdID:1953 return 2;1954 case FlatScratchInitID:1955 return 2;1956 case PrivateSegmentSizeID:1957 return 1;1958 }1959 llvm_unreachable("Unknown UserSGPRID.");1960 }1961 1962 GCNUserSGPRUsageInfo(const Function &F, const GCNSubtarget &ST);1963 1964private:1965 const GCNSubtarget &ST;1966 1967 // Private memory buffer1968 // Compute directly in sgpr[0:1]1969 // Other shaders indirect 64-bits at sgpr[0:1]1970 bool ImplicitBufferPtr = false;1971 1972 bool PrivateSegmentBuffer = false;1973 1974 bool DispatchPtr = false;1975 1976 bool QueuePtr = false;1977 1978 bool KernargSegmentPtr = false;1979 1980 bool DispatchID = false;1981 1982 bool FlatScratchInit = false;1983 1984 bool PrivateSegmentSize = false;1985 1986 unsigned NumKernargPreloadSGPRs = 0;1987 1988 unsigned NumUsedUserSGPRs = 0;1989};1990 1991} // end namespace llvm1992 1993#endif // LLVM_LIB_TARGET_AMDGPU_GCNSUBTARGET_H1994